From 804fca606649b2b3d6b8f9636b6beea01c8cabb3 Mon Sep 17 00:00:00 2001
From: Lucas Nussbaum <lucas.nussbaum@loria.fr>
Date: Sat, 17 Nov 2018 00:13:49 +0100
Subject: [PATCH] [dev] port puppet generators
---
Rakefile | 17 +-
lib/refrepo/gen/puppet/bindg5k.rb | 267 ++++++++----------
lib/refrepo/gen/puppet/conmang5k.rb | 83 ++----
lib/refrepo/gen/puppet/dhcpg5k.rb | 188 +++++-------
lib/refrepo/gen/puppet/kadeployg5k.rb | 239 +++++++---------
lib/refrepo/gen/puppet/kavlang5k.rb | 103 +++----
lib/refrepo/gen/puppet/templates/conman.erb | 2 +-
lib/refrepo/gen/puppet/templates/dhcp.erb | 2 +-
.../gen/puppet/templates/kadeployg5k.conf.erb | 2 +-
9 files changed, 365 insertions(+), 538 deletions(-)
diff --git a/Rakefile b/Rakefile
index 410b242053..1985ef4954 100644
--- a/Rakefile
+++ b/Rakefile
@@ -15,6 +15,12 @@ REFAPI_DIR = "./generators/reference-api"
PUPPET_DIR = "./generators/puppet"
VALIDATORS_DIR = "./generators/input-validators"
+if Dir::exists?(ENV['HOME'] + '/.gpuppet/repo')
+ PUPPET_ODIR = ENV['HOME'] + '/.gpuppet/repo'
+else
+ PUPPET_ODIR = '/tmp/puppet'
+end
+
G5K_SITES = RefRepo::Utils::get_sites
namespace :valid do
@@ -156,9 +162,16 @@ namespace :puppet do
all_puppet_tasks = [:bindg5k, :conmang5k, :dhcpg5k, :kadeployg5k, :lanpowerg5k, :kavlang5k]
all_puppet_tasks.each { |t|
- desc "Generate #{t} configuration"
+ desc "Generate #{t} configuration -- parameters: SITE={grenoble,...} OUTPUTDIR=[default: #{PUPPET_ODIR}] [CONFDIR=...] VERBOSE=1"
task t do
- invoke_script "#{PUPPET_DIR}/#{t}.rb"
+ require "refrepo/gen/puppet/#{t}"
+ options = {}
+ options[:sites] = ( ENV['SITE'] ? ENV['SITE'].split(',') : G5K_SITES )
+ options[:output_dir] = ENV['OUTPUTDIR'] || PUPPET_ODIR
+ options[:verbose] = (ENV['VERBOSE'] != nil)
+ options[:conf_dir] = ENV['CONFDIR'] if ENV['CONFDIR']
+ send("generate_puppet_#{t}", options)
+
end
}
diff --git a/lib/refrepo/gen/puppet/bindg5k.rb b/lib/refrepo/gen/puppet/bindg5k.rb
index fe9dde7c25..1b0c50b269 100644
--- a/lib/refrepo/gen/puppet/bindg5k.rb
+++ b/lib/refrepo/gen/puppet/bindg5k.rb
@@ -1,11 +1,3 @@
-#!/usr/bin/ruby
-# coding: utf-8
-
-if RUBY_VERSION < "2.1"
- puts "This script requires ruby >= 2.1"
- exit
-end
-
# See also: https://www.grid5000.fr/mediawiki/index.php/DNS_server
require 'pp'
@@ -14,44 +6,7 @@ require 'pathname'
require 'fileutils'
require 'optparse'
require 'dns/zone'
-require_relative '../lib/input_loader'
-
-input_data_dir = "../../input/grid5000/"
-
-refapi = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
-
-$options = {}
-$options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia toulouse}
-$options[:output_dir] = "/tmp/puppet"
-$options[:verbose] = false
-
-OptionParser.new do |opts|
- opts.banner = "Usage: bindg5k.rb [options]"
-
- opts.separator ""
- opts.separator "Example: ruby bindg5k.rb -s nancy -o /tmp/puppet"
-
- opts.on('-o', '--output-dir dir', String, 'Select the puppet repo path', "Default: " + $options[:output_dir]) do |d|
- $options[:output_dir] = d
- end
-
- opts.separator ""
- opts.separator "Filters:"
-
- opts.on('-s', '--sites a,b,c', Array, 'Select site(s)', "Default: " + $options[:sites].join(", ")) do |s|
- raise "Wrong argument for -s option." unless (s - $options[:sites]).empty?
- $options[:sites] = s
- end
-
- opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
- $options[:verbose] = true
- end
-
- opts.on_tail("-h", "--help", "Show this message") do
- puts opts
- exit
- end
-end.parse!
+require 'refrepo/input_loader'
#Prettier aligned dump of records
class DNS::Zone::RR::A
@@ -467,140 +422,146 @@ def write_zone(zone)
File.write(zone.file_path, zone.dump)
end
-puts "Writing DNS configuration files to: #{$options[:output_dir]}"
-puts "For site(s): #{$options[:sites].join(', ')}"
+# main method
+def generate_puppet_bindg5k(options)
+ $options = options
+ puts "Writing DNS configuration files to: #{$options[:output_dir]}"
+ puts "For site(s): #{$options[:sites].join(', ')}"
-puts "Note: if you modify *-manual.db files you will have to manually update the serial in managed db file for changes to be applied"
+ puts "Note: if you modify *-manual.db files you will have to manually update the serial in managed db file for changes to be applied"
-$written_files = []
+ $written_files = []
-# Loop over Grid'5000 sites
-refapi["sites"].each { |site_uid, site|
-
- next unless $options[:sites].include?(site_uid)
+ refapi = load_yaml_file_hierarchy
- dest_dir = "#{$options[:output_dir]}/platforms/production/modules/generated/files/bind/"
- zones_dir = File.join(dest_dir, "zones/#{site_uid}")
+ # Loop over Grid'5000 sites
+ refapi["sites"].each { |site_uid, site|
- site_records = {}
+ next unless $options[:sites].include?(site_uid)
- # Servers
- site_records['servers'] = get_servers_records(site) unless site['servers'].nil?
+ dest_dir = "#{$options[:output_dir]}/platforms/production/modules/generated/files/bind/"
+ zones_dir = File.join(dest_dir, "zones/#{site_uid}")
- # PDUs
- site_records['pdus'] = get_pdus_records(site) unless site['pdus'].nil?
+ site_records = {}
- # Networks and laptops (same input format)
- site_records['networks'] = get_networks_records(site, 'networks') unless site['networks'].nil?
- site_records['laptops'] = get_networks_records(site, 'laptops') unless site['laptops'].nil?
+ # Servers
+ site_records['servers'] = get_servers_records(site) unless site['servers'].nil?
- site.fetch("clusters", []).sort.each { |cluster_uid, cluster|
+ # PDUs
+ site_records['pdus'] = get_pdus_records(site) unless site['pdus'].nil?
- cluster.fetch('nodes').select { |node_uid, node|
- node != nil && node["status"] != "retired" && node.has_key?('network_adapters')
- }.each_sort_by_node_uid { |node_uid, node|
+ # Networks and laptops (same input format)
+ site_records['networks'] = get_networks_records(site, 'networks') unless site['networks'].nil?
+ site_records['laptops'] = get_networks_records(site, 'laptops') unless site['laptops'].nil?
- network_adapters = {}
+ site.fetch("clusters", []).sort.each { |cluster_uid, cluster|
- # Nodes
- node.fetch('network_adapters').each { |net_uid, net_hash|
- network_adapters[net_uid] = {"ip" => net_hash["ip"], "mounted" => net_hash["mounted"], "alias" => net_hash["alias"]}
- }
-
- # Mic
- if node['mic'] && node['mic']['ip']
- network_adapters['mic0'] = {"ip" => node['mic']['ip']}
- end
+ cluster.fetch('nodes').select { |node_uid, node|
+ node != nil && node["status"] != "retired" && node.has_key?('network_adapters')
+ }.each_sort_by_node_uid { |node_uid, node|
- site_records[cluster_uid] ||= []
- site_records[cluster_uid] += get_node_records(cluster_uid, node_uid, network_adapters)
+ network_adapters = {}
- # Kavlan
- if node['kavlan']
- kavlan_adapters = {}
- node.fetch('kavlan').each { |net_uid, net_hash|
- net_hash.each { |kavlan_net_uid, ip|
- kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"] = {"ip" => ip, "mounted" => node['network_adapters'][net_uid]['mounted']}
- }
+ # Nodes
+ node.fetch('network_adapters').each { |net_uid, net_hash|
+ network_adapters[net_uid] = {"ip" => net_hash["ip"], "mounted" => net_hash["mounted"], "alias" => net_hash["alias"]}
}
- site_records["#{cluster_uid}-kavlan"] ||= []
- site_records["#{cluster_uid}-kavlan"] += get_node_kavlan_records(cluster_uid, node_uid, network_adapters, kavlan_adapters)
- end
- } # each nodes
- } # each cluster
- reverse_records = {} # one hash entry per reverse dns file
+ # Mic
+ if node['mic'] && node['mic']['ip']
+ network_adapters['mic0'] = {"ip" => node['mic']['ip']}
+ end
+
+ site_records[cluster_uid] ||= []
+ site_records[cluster_uid] += get_node_records(cluster_uid, node_uid, network_adapters)
+
+ # Kavlan
+ if node['kavlan']
+ kavlan_adapters = {}
+ node.fetch('kavlan').each { |net_uid, net_hash|
+ net_hash.each { |kavlan_net_uid, ip|
+ kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"] = {"ip" => ip, "mounted" => node['network_adapters'][net_uid]['mounted']}
+ }
+ }
+ site_records["#{cluster_uid}-kavlan"] ||= []
+ site_records["#{cluster_uid}-kavlan"] += get_node_kavlan_records(cluster_uid, node_uid, network_adapters, kavlan_adapters)
+ end
+ } # each nodes
+ } # each cluster
+
+ reverse_records = {} # one hash entry per reverse dns file
+
+ site_records.each { |zone, records|
+
+ #Sort records
+ site_records[zone] = sort_records(records)
+
+ records.each{ |record|
+ #get Reverse records
+ reverse_file_name, reverse_record = get_reverse_record(record, site_uid)
+ if reverse_file_name != nil
+ reverse_records[reverse_file_name] ||= []
+ reverse_records[reverse_file_name].each {|r|
+ if r.label == reverse_record.label
+ puts "Warning: reverse entry with IP #{reverse_record.label} already exists in #{reverse_file_name}, #{reverse_record.name} is duplicate"
+ end
+ }
+ reverse_records[reverse_file_name] << reverse_record
+ end
+ }
+ }
- site_records.each { |zone, records|
+ zones = []
- #Sort records
- site_records[zone] = sort_records(records)
+ #Sort reverse records and create reverse zone from files
+ reverse_records.each{ |file_name, records|
+ records.sort!{ |a, b|
+ a.label.to_i <=> b.label.to_i
+ }
- records.each{ |record|
- #get Reverse records
- reverse_file_name, reverse_record = get_reverse_record(record, site_uid)
- if reverse_file_name != nil
- reverse_records[reverse_file_name] ||= []
- reverse_records[reverse_file_name].each {|r|
- if r.label == reverse_record.label
- puts "Warning: reverse entry with IP #{reverse_record.label} already exists in #{reverse_file_name}, #{reverse_record.name} is duplicate"
- end
- }
- reverse_records[reverse_file_name] << reverse_record
+ reverse_file_path = File.join(zones_dir, file_name)
+ zone = load_zone(reverse_file_path, site_uid, site, true)
+ if diff_zone_file(zone, records)
+ zone.soa.serial = update_serial(zone.soa.serial)
end
+ zone.records = records;
+ zones << zone
}
- }
-
- zones = []
- #Sort reverse records and create reverse zone from files
- reverse_records.each{ |file_name, records|
- records.sort!{ |a, b|
- a.label.to_i <=> b.label.to_i
+ #Manage site zone (SITE.db file)
+ #It only contains header and inclusion of other db files
+ #Check modification in included files and update serial accordingly
+ site_zone_path = File.join(zones_dir, site_uid + ".db")
+ site_zone = load_zone(site_zone_path, site_uid, site, true)
+ site_zone_changed = false
+
+ site_records.each{ |type, records|
+ next if records.empty?
+ zone_file_path = File.join(zones_dir, site_uid + "-" + type + ".db")
+ zone = load_zone(zone_file_path, site_uid, site, false)
+ if diff_zone_file(zone, records)
+ puts "Zone file changed: #{zone.file_path}" if $options[:verbose]
+ site_zone_changed = true
+ end
+ zone.records = records
+ site_zone.include += "$INCLUDE /etc/bind/zones/#{site_uid}/#{File.basename(zone_file_path)}\n"
+ zones << zone
}
- reverse_file_path = File.join(zones_dir, file_name)
- zone = load_zone(reverse_file_path, site_uid, site, true)
- if diff_zone_file(zone, records)
- zone.soa.serial = update_serial(zone.soa.serial)
+ if (site_zone_changed)
+ site_zone.soa.serial = update_serial(site_zone.soa.serial)
end
- zone.records = records;
- zones << zone
- }
-
- #Manage site zone (SITE.db file)
- #It only contains header and inclusion of other db files
- #Check modification in included files and update serial accordingly
- site_zone_path = File.join(zones_dir, site_uid + ".db")
- site_zone = load_zone(site_zone_path, site_uid, site, true)
- site_zone_changed = false
-
- site_records.each{ |type, records|
- next if records.empty?
- zone_file_path = File.join(zones_dir, site_uid + "-" + type + ".db")
- zone = load_zone(zone_file_path, site_uid, site, false)
- if diff_zone_file(zone, records)
- puts "Zone file changed: #{zone.file_path}" if $options[:verbose]
- site_zone_changed = true
- end
- zone.records = records
- site_zone.include += "$INCLUDE /etc/bind/zones/#{site_uid}/#{File.basename(zone_file_path)}\n"
- zones << zone
- }
- if (site_zone_changed)
- site_zone.soa.serial = update_serial(site_zone.soa.serial)
- end
-
- zones << site_zone
+ zones << site_zone
- zones += get_orphan_reverse_manual_zones(zones_dir, site_uid, site)
+ zones += get_orphan_reverse_manual_zones(zones_dir, site_uid, site)
- zones.each{ |zone|
- write_zone(zone)
- }
+ zones.each{ |zone|
+ write_zone(zone)
+ }
- write_site_conf(site_uid, dest_dir, zones_dir)
- write_site_local_conf(site_uid, dest_dir, zones_dir)
+ write_site_conf(site_uid, dest_dir, zones_dir)
+ write_site_local_conf(site_uid, dest_dir, zones_dir)
-} # each sites
+ } # each sites
+end
diff --git a/lib/refrepo/gen/puppet/conmang5k.rb b/lib/refrepo/gen/puppet/conmang5k.rb
index 4742cddb58..b3f85e54c2 100644
--- a/lib/refrepo/gen/puppet/conmang5k.rb
+++ b/lib/refrepo/gen/puppet/conmang5k.rb
@@ -1,81 +1,44 @@
-#!/usr/bin/ruby
-
-if RUBY_VERSION < "2.1"
- puts "This script requires ruby >= 2.1"
- exit
-end
-
# This script generates conmang5k/files/<site_uid>/conman.conf from input/, conf/console.yaml and conf/console-password.yaml
require 'pp'
require 'erb'
require 'pathname'
require 'optparse'
-require_relative '../lib/input_loader'
-require_relative '../lib/hash/hash.rb'
+require 'refrepo/input_loader'
+require 'refrepo/hash/hash'
-options = {}
-options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
-options[:output_dir] = "/tmp/puppet-repo"
-options[:conf_dir] = File.expand_path("conf-examples/", File.dirname(__FILE__))
+# Apply ERB template and save result to file
+def write_conman_file(site_uid, site_refapi, site_config, site_credentials, options)
+ output = ERB.new(File.read(File.expand_path('templates/conman.erb', File.dirname(__FILE__)))).result(binding)
-OptionParser.new do |opts|
- opts.banner = "Usage: conmang5k.rb [options]"
+ output_file = Pathname("#{options[:output_dir]}/platforms/production/modules/generated/files/conman/server/#{site_uid}.conf")
- opts.separator ""
- opts.separator "Example: ruby conmang5k.rb -s nancy -o /tmp/puppet-repo"
- opts.separator "Example: ruby conmang5k.rb -s nancy -o $GPUPPET_PREFIX/repo"
+ output_file.dirname.mkpath()
+ File.write(output_file, output)
+end
- opts.on('-o', '--output-dir dir', String, 'Select the puppet repo path', "Default: " + options[:output_dir]) do |d|
- options[:output_dir] = d
+def generate_puppet_conmang5k(options)
+ if not options[:confdir]
options[:conf_dir] = "#{options[:output_dir]}/platforms/production/generators/ipmitools/"
end
- opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: #{options[:conf_dir]}") do |d|
- options[:conf_dir] = d
- end
+ raise("Error: #{options[:conf_dir]} does not exist. The given configuration path is incorrect") unless Pathname(options[:conf_dir].to_s).exist?
- opts.separator ""
- opts.separator "Filters:"
+ puts "Writing Conman configuration files to: #{options[:output_dir]}"
+ puts "Using configuration directory: #{options[:conf_dir]}"
+ puts "For site(s): #{options[:sites].join(', ')}"
- opts.on('-s', '--sites a,b,c', Array, 'Select site(s)', "Default: " + options[:sites].join(", ")) do |s|
- raise "Wrong argument for -s option." unless (s - options[:sites]).empty?
- options[:sites] = s
- end
+ refapi = load_yaml_file_hierarchy
- opts.on_tail("-h", "--help", "Show this message") do
- puts opts
- exit
- end
-end.parse!
+ config = YAML::load_file(options[:conf_dir] + 'console.yaml')
+ credentials = YAML::load_file(options[:conf_dir] + 'console-password.yaml')
-raise("Error: #{options[:conf_dir]} does not exist. The given configuration path is incorrect") unless Pathname(options[:conf_dir].to_s).exist?
-puts "Writing Conman configuration files to: #{options[:output_dir]}"
-puts "Using configuration directory: #{options[:conf_dir]}"
-puts "For site(s): #{options[:sites].join(', ')}"
+ # Loop over each site
+ refapi["sites"].each { |site_uid, site_refapi|
-#Input
-input_data_dir = "../../input/grid5000/"
-refapi = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
+ next unless options[:sites].include?(site_uid)
-config = YAML::load_file(options[:conf_dir] + 'console.yaml')
-credentials = YAML::load_file(options[:conf_dir] + 'console-password.yaml')
-
-# Apply ERB template and save result to file
-def write_conman_file(site_uid, site_refapi, site_config, site_credentials, options)
- output = ERB.new(File.read(File.expand_path('templates/conman.erb', File.dirname(__FILE__)))).result(binding)
-
- output_file = Pathname("#{options[:output_dir]}/platforms/production/modules/generated/files/conman/server/#{site_uid}.conf")
-
- output_file.dirname.mkpath()
- File.write(output_file, output)
+ write_conman_file(site_uid, site_refapi, config[site_uid], credentials[site_uid], options)
+ }
end
-
-# Loop over each site
-refapi["sites"].each { |site_uid, site_refapi|
-
- next unless options[:sites].include?(site_uid)
-
- write_conman_file(site_uid, site_refapi, config[site_uid], credentials[site_uid], options)
-}
diff --git a/lib/refrepo/gen/puppet/dhcpg5k.rb b/lib/refrepo/gen/puppet/dhcpg5k.rb
index 5fe8a5d8cf..f86fedd95b 100644
--- a/lib/refrepo/gen/puppet/dhcpg5k.rb
+++ b/lib/refrepo/gen/puppet/dhcpg5k.rb
@@ -1,47 +1,8 @@
-#!/usr/bin/ruby
-
-if RUBY_VERSION < "2.1"
- puts "This script requires ruby >= 2.1"
- exit
-end
-
require 'pp'
require 'erb'
require 'pathname'
require 'optparse'
-require_relative '../lib/input_loader'
-
-input_data_dir = "../../input/grid5000/"
-
-global_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
-
-options = {}
-options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
-options[:output_dir] = "/tmp/puppet-repo"
-
-OptionParser.new do |opts|
- opts.banner = "Usage: dhcpg5k.rb [options]"
-
- opts.separator ""
- opts.separator "Example: ruby dhcpg5k.rb -s nancy -o /tmp/puppet-repo"
-
- opts.on('-o', '--output-dir dir', String, 'Select the puppet repo path', "Default: " + options[:output_dir]) do |d|
- options[:output_dir] = d
- end
-
- opts.separator ""
- opts.separator "Filters:"
-
- opts.on('-s', '--sites a,b,c', Array, 'Select site(s)', "Default: " + options[:sites].join(", ")) do |s|
- raise "Wrong argument for -s option." unless (s - options[:sites]).empty?
- options[:sites] = s
- end
-
- opts.on_tail("-h", "--help", "Show this message") do
- puts opts
- exit
- end
-end.parse!
+require 'refrepo/input_loader'
# Get the mac and ip of a node. Throw exception if error.
def get_network_info(node_hash, network_interface)
@@ -80,86 +41,91 @@ def write_dhcp_file(data, options)
File.write(output_file, output)
end
-puts "Writing DHCP configuration files to: #{options[:output_dir]}"
-puts "For site(s): #{options[:sites].join(', ')}"
+def generate_puppet_dhcpg5k(options)
+ global_hash = load_yaml_file_hierarchy
-# Loop over Grid'5000 sites
-global_hash["sites"].each { |site_uid, site_hash|
- next unless options[:sites].include?(site_uid)
+ puts "Writing DHCP configuration files to: #{options[:output_dir]}"
+ puts "For site(s): #{options[:sites].join(', ')}"
- puts site_uid
+ # Loop over Grid'5000 sites
+ global_hash["sites"].each { |site_uid, site_hash|
- #
- # eth, bmc and mic0
- #
+ next unless options[:sites].include?(site_uid)
- # Relocate ip/mac info of MIC
- site_hash.fetch("clusters").each { |cluster_uid, cluster_hash|
- cluster_hash.fetch('nodes').each { |node_uid, node_hash|
- next if node_hash == nil || node_hash['status'] == 'retired'
+ puts site_uid
- if node_hash['mic'] && node_hash['mic']['ip'] && node_hash['mic']['mac']
- node_hash['network_adapters'] ||= {}
- node_hash['network_adapters']['mic0'] ||= {}
- node_hash['network_adapters']['mic0']['ip'] = node_hash['mic'].delete('ip')
- node_hash['network_adapters']['mic0']['mac'] = node_hash['mic'].delete('mac')
- end
- }
- }
+ #
+ # eth, bmc and mic0
+ #
- # One file for each clusters
- site_hash.fetch("clusters").each { |cluster_uid, cluster_hash|
- # networks = ["eth", "bmc"]
- # networks << 'mic0' if cluster_hash['nodes'].values.any? {|x| x['network_adapters']['mic0'] }
-
- write_dhcp_file({
- "filename" => "cluster-" + cluster_uid + ".conf",
- "site_uid" => site_uid,
- "nodes" => cluster_hash.fetch('nodes'),
- "network_adapters" => ["eth", "bmc", "mic0"],
- "optional_network_adapters" => ["mic0"]
- }, options)
- }
+ # Relocate ip/mac info of MIC
+ site_hash.fetch("clusters").each { |cluster_uid, cluster_hash|
+ cluster_hash.fetch('nodes').each { |node_uid, node_hash|
+ next if node_hash == nil || node_hash['status'] == 'retired'
- #
- #
- #
-
- # Other dhcp files
- ["networks", "laptops", "servers"].each { |key|
- write_dhcp_file({
- "filename" => key + ".conf",
- "site_uid" => site_uid,
- "nodes" => site_hash[key],
- "network_adapters" => ["default", "eth", "bmc", "adm"],
- "optional_network_adapters" => ["eth", "bmc", "adm"]
- }, options) unless site_hash[key].nil?
- }
+ if node_hash['mic'] && node_hash['mic']['ip'] && node_hash['mic']['mac']
+ node_hash['network_adapters'] ||= {}
+ node_hash['network_adapters']['mic0'] ||= {}
+ node_hash['network_adapters']['mic0']['ip'] = node_hash['mic'].delete('ip')
+ node_hash['network_adapters']['mic0']['mac'] = node_hash['mic'].delete('mac')
+ end
+ }
+ }
- #
- # PDUs
- #
-
- if ! site_hash['pdus'].nil?
- # Relocate ip/mac info of PDUS
- site_hash['pdus'].each { |pdu_uid, pdu_hash|
- if pdu_hash['ip'] && pdu_hash['mac']
- pdu_hash['network_adapters'] ||= {}
- pdu_hash['network_adapters']['pdu'] ||= {}
- pdu_hash['network_adapters']['pdu']['ip'] = pdu_hash.delete('ip')
- pdu_hash['network_adapters']['pdu']['mac'] = pdu_hash.delete('mac')
- end
+ # One file for each clusters
+ site_hash.fetch("clusters").each { |cluster_uid, cluster_hash|
+ # networks = ["eth", "bmc"]
+ # networks << 'mic0' if cluster_hash['nodes'].values.any? {|x| x['network_adapters']['mic0'] }
+
+ write_dhcp_file({
+ "filename" => "cluster-" + cluster_uid + ".conf",
+ "site_uid" => site_uid,
+ "nodes" => cluster_hash.fetch('nodes'),
+ "network_adapters" => ["eth", "bmc", "mic0"],
+ "optional_network_adapters" => ["mic0"]
+ }, options)
}
- key = 'pdus'
- write_dhcp_file({
- "filename" => key + ".conf",
- "site_uid" => site_uid,
- "nodes" => site_hash['pdus'],
- "network_adapters" => ['pdu'],
- "optional_network_adapters" => []
- }, options)
- end
+ #
+ #
+ #
+
+ # Other dhcp files
+ ["networks", "laptops", "servers"].each { |key|
+ write_dhcp_file({
+ "filename" => key + ".conf",
+ "site_uid" => site_uid,
+ "nodes" => site_hash[key],
+ "network_adapters" => ["default", "eth", "bmc", "adm"],
+ "optional_network_adapters" => ["eth", "bmc", "adm"]
+ }, options) unless site_hash[key].nil?
+ }
-}
+ #
+ # PDUs
+ #
+
+ if ! site_hash['pdus'].nil?
+ # Relocate ip/mac info of PDUS
+ site_hash['pdus'].each { |pdu_uid, pdu_hash|
+ if pdu_hash['ip'] && pdu_hash['mac']
+ pdu_hash['network_adapters'] ||= {}
+ pdu_hash['network_adapters']['pdu'] ||= {}
+ pdu_hash['network_adapters']['pdu']['ip'] = pdu_hash.delete('ip')
+ pdu_hash['network_adapters']['pdu']['mac'] = pdu_hash.delete('mac')
+ end
+ }
+
+ key = 'pdus'
+ write_dhcp_file({
+ "filename" => key + ".conf",
+ "site_uid" => site_uid,
+ "nodes" => site_hash['pdus'],
+ "network_adapters" => ['pdu'],
+ "optional_network_adapters" => []
+ }, options)
+ end
+
+ }
+end
diff --git a/lib/refrepo/gen/puppet/kadeployg5k.rb b/lib/refrepo/gen/puppet/kadeployg5k.rb
index 226d98f39f..8b6e73eeba 100644
--- a/lib/refrepo/gen/puppet/kadeployg5k.rb
+++ b/lib/refrepo/gen/puppet/kadeployg5k.rb
@@ -1,63 +1,9 @@
-#!/usr/bin/ruby
-
-# This script generates:
-# - kadeployg5k/files/<site_uid>/server_conf[_dev]/clusters.conf from input/
-# - kadeployg5k/files/<site_uid>/server_conf[_dev]/<cluster_uid>-cluster.conf from kadeployg5k[-dev].yaml and template/kadeployg5k.yaml.erb
-
-if RUBY_VERSION < "2.1"
- puts "This script requires ruby >= 2.1"
- exit
-end
-
require 'pp'
require 'erb'
require 'pathname'
require 'optparse'
-require_relative '../lib/input_loader'
-require_relative '../lib/hash/hash.rb'
-
-input_data_dir = "../../input/grid5000/"
-global_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
-
-options = {}
-options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
-options[:output_dir] = "/tmp/puppet-repo"
-options[:conf_dir] = File.expand_path("conf-examples/", File.dirname(__FILE__))
-
-OptionParser.new do |opts|
- opts.banner = "Usage: kadeployg5k.rb [options]"
-
- opts.separator ""
- opts.separator "Example: ruby kadeployg5k.rb -s nancy -o /tmp/puppet-repo -c $(dirname path_to_kadeployg5k.yaml)"
-
- opts.on('-o', '--output-dir dir', String, 'Select the puppet repo path', "Default: " + options[:output_dir]) do |d|
- options[:output_dir] = d
- options[:conf_dir] = "#{options[:output_dir]}/platforms/production/generators/kadeploy"
- end
-
- opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: #{options[:conf_dir]}") do |d|
- options[:conf_dir] = d
- end
-
- opts.separator ""
- opts.separator "Filters:"
-
- opts.on('-s', '--sites a,b,c', Array, 'Select site(s)', "Default: " + options[:sites].join(", ")) do |s|
- raise "Wrong argument for -s option." unless (s - options[:sites]).empty?
- options[:sites] = s
- end
-
- opts.on_tail("-h", "--help", "Show this message") do
- puts opts
- exit
- end
-end.parse!
-
-raise("Error: #{options[:conf_dir]} does not exist. The given configuration path is incorrect") unless Pathname(options[:conf_dir].to_s).exist?
-
-puts "Writing Kadeploy configuration files to: #{options[:output_dir]}"
-puts "Using configuration directory: #{options[:conf_dir]}"
-puts "For site(s): #{options[:sites].join(', ')}"
+require 'refrepo/input_loader'
+require 'refrepo/hash/hash'
# Compute cluster prefix
# input: cluster_list = ['graoully', 'graphene', 'griffon', ...]
@@ -92,114 +38,129 @@ def get_ip(node)
}
end
-# There is two kadeploy servers : kadeploy and kadeploy-dev
-['', '-dev'].each {|suffix|
+def generate_puppet_kadeployg5k(options)
- global_hash['sites'].each { |site_uid, site|
+ global_hash = load_yaml_file_hierarchy
- next unless options[:sites].include?(site_uid)
+ if not options[:confdir]
+ options[:conf_dir] = "#{options[:output_dir]}/platforms/production/generators/kadeploy"
+ end
- #
- # Generate site/<site_uid>/servers_conf[_dev]/clusters.conf
- #
+ raise("Error: #{options[:conf_dir]} does not exist. The given configuration path is incorrect") unless Pathname(options[:conf_dir].to_s).exist?
- clusters_conf = { 'clusters'=> [] } # output clusters.conf
- prefix = cluster_prefix(site['clusters'].keys)
+ puts "Writing Kadeploy configuration files to: #{options[:output_dir]}"
+ puts "Using configuration directory: #{options[:conf_dir]}"
+ puts "For site(s): #{options[:sites].join(', ')}"
- site['clusters'].sort.each { |cluster_uid, cluster|
+ # There is two kadeploy servers : kadeploy and kadeploy-dev
+ ['', '-dev'].each {|suffix|
- # clusters:
- # - name: griffon
- # prefix: gri
- # conf_file: /etc/kadeploy3/griffon-cluster.conf
- # nodes:
- # - name: griffon-[1-92].nancy.grid5000.fr
- # address: 172.16.65.[1-92]
+ global_hash['sites'].each { |site_uid, site|
- cluster_conf = {}
- cluster_conf['name'] = cluster_uid
- cluster_conf['prefix'] = prefix[cluster_uid]
- cluster_conf['conf_file'] = "/etc/kadeploy3#{suffix}/#{cluster_uid}-cluster.conf"
- cluster_conf['nodes'] = []
+ next unless options[:sites].include?(site_uid)
- # init
- first = last = c_uid = -1
- first_ip = ['0', '0', '0', 0]
+ #
+ # Generate site/<site_uid>/servers_conf[_dev]/clusters.conf
+ #
- # group nodes by range (griffon-[1-92] -> 172.16.65.[1-92])
- cluster['nodes'].each_sort_by_node_uid { |node_uid, node|
+ clusters_conf = { 'clusters'=> [] } # output clusters.conf
+ prefix = cluster_prefix(site['clusters'].keys)
- next if node == nil || (node['status'] && node['status'] == 'retired')
+ site['clusters'].sort.each { |cluster_uid, cluster|
- c, id = node_uid.scan(/^([^\d]*)(\d*)$/).first
- id = id.to_i
- ip = get_ip(node).split('.')
- ip[3] = ip[3].to_i
+ # clusters:
+ # - name: griffon
+ # prefix: gri
+ # conf_file: /etc/kadeploy3/griffon-cluster.conf
+ # nodes:
+ # - name: griffon-[1-92].nancy.grid5000.fr
+ # address: 172.16.65.[1-92]
- if c == c_uid && id == last + 1 && ip[0..2] == first_ip[0..2] && ip[3] == first_ip[3] + id - first
- # extend range
- last = id
- else
- if c_uid != -1
- node = {}
- node['name'] = "#{c_uid}[#{first}-#{last}].#{site_uid}.grid5000.fr"
- node['address'] = "#{first_ip[0..2].join('.')}.[#{first_ip[3]}-#{first_ip[3]+last-first}]"
- cluster_conf['nodes'] << node
- end
+ cluster_conf = {}
+ cluster_conf['name'] = cluster_uid
+ cluster_conf['prefix'] = prefix[cluster_uid]
+ cluster_conf['conf_file'] = "/etc/kadeploy3#{suffix}/#{cluster_uid}-cluster.conf"
+ cluster_conf['nodes'] = []
+
+ # init
+ first = last = c_uid = -1
+ first_ip = ['0', '0', '0', 0]
- # new range
- first = last = id
- first_ip = ip
- c_uid = c
+ # group nodes by range (griffon-[1-92] -> 172.16.65.[1-92])
+ cluster['nodes'].each_sort_by_node_uid { |node_uid, node|
+
+ next if node == nil || (node['status'] && node['status'] == 'retired')
+
+ c, id = node_uid.scan(/^([^\d]*)(\d*)$/).first
+ id = id.to_i
+ ip = get_ip(node).split('.')
+ ip[3] = ip[3].to_i
+
+ if c == c_uid && id == last + 1 && ip[0..2] == first_ip[0..2] && ip[3] == first_ip[3] + id - first
+ # extend range
+ last = id
+ else
+ if c_uid != -1
+ node = {}
+ node['name'] = "#{c_uid}[#{first}-#{last}].#{site_uid}.grid5000.fr"
+ node['address'] = "#{first_ip[0..2].join('.')}.[#{first_ip[3]}-#{first_ip[3]+last-first}]"
+ cluster_conf['nodes'] << node
+ end
+
+ # new range
+ first = last = id
+ first_ip = ip
+ c_uid = c
+ end
+ }
+ # last range
+ if c_uid != -1
+ node = {}
+ node['name'] = "#{c_uid}[#{first}-#{last}].#{site_uid}.grid5000.fr"
+ node['address'] = "#{first_ip[0..2].join('.')}.[#{first_ip[3]}-#{first_ip[3]+last-first}]"
+ cluster_conf['nodes'] << node
end
- }
- # last range
- if c_uid != -1
- node = {}
- node['name'] = "#{c_uid}[#{first}-#{last}].#{site_uid}.grid5000.fr"
- node['address'] = "#{first_ip[0..2].join('.')}.[#{first_ip[3]}-#{first_ip[3]+last-first}]"
- cluster_conf['nodes'] << node
- end
- clusters_conf['clusters'] << cluster_conf
+ clusters_conf['clusters'] << cluster_conf
- } # site['clusters'].each
+ } # site['clusters'].each
- output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kadeploy/server#{suffix.tr('-', '_')}/#{site_uid}/clusters.conf")
+ output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kadeploy/server#{suffix.tr('-', '_')}/#{site_uid}/clusters.conf")
- output_file.dirname.mkpath()
- write_yaml(output_file, clusters_conf)
- add_header(output_file)
+ output_file.dirname.mkpath()
+ write_yaml(output_file, clusters_conf)
+ add_header(output_file)
- #
- # Generate <cluster_uid>-cluster.conf files
- #
+ #
+ # Generate <cluster_uid>-cluster.conf files
+ #
- # Load 'conf/kadeployg5k.yaml' data and fill up the kadeployg5k.conf.erb template for each cluster
+ # Load 'conf/kadeployg5k.yaml' data and fill up the kadeployg5k.conf.erb template for each cluster
- conf = YAML::load(ERB.new(File.read("#{options[:conf_dir]}/kadeployg5k#{suffix}.yaml")).result(binding))
+ conf = YAML::load(ERB.new(File.read("#{options[:conf_dir]}/kadeployg5k#{suffix}.yaml")).result(binding))
- site['clusters'].each { |cluster_uid, cluster|
- defaults = conf['defaults']
- overrides = conf[site_uid][cluster_uid]
- dupes = (defaults.to_a & overrides.to_a)
- if not dupes.empty?
- puts "Warning: cluster-specific configuration for #{cluster_uid} overrides default values: #{dupes.to_s}"
- end
- data = data = defaults.merge(overrides)
- if data.nil?
- puts "Warning: configuration not found in #{options[:conf_dir]}/kadeployg5k#{suffix}.yaml for #{cluster_uid}. Skipped"
- next
- end
+ site['clusters'].each { |cluster_uid, cluster|
+ defaults = conf['defaults']
+ overrides = conf[site_uid][cluster_uid]
+ dupes = (defaults.to_a & overrides.to_a)
+ if not dupes.empty?
+ puts "Warning: cluster-specific configuration for #{cluster_uid} overrides default values: #{dupes.to_s}"
+ end
+ data = data = defaults.merge(overrides)
+ if data.nil?
+ puts "Warning: configuration not found in #{options[:conf_dir]}/kadeployg5k#{suffix}.yaml for #{cluster_uid}. Skipped"
+ next
+ end
- output = ERB.new(File.read(File.expand_path('templates/kadeployg5k.conf.erb', File.dirname(__FILE__)))).result(binding)
+ output = ERB.new(File.read(File.expand_path('templates/kadeployg5k.conf.erb', File.dirname(__FILE__)))).result(binding)
- output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kadeploy/server#{suffix.tr('-', '_')}/#{site_uid}/#{cluster_uid}-cluster.conf")
+ output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kadeploy/server#{suffix.tr('-', '_')}/#{site_uid}/#{cluster_uid}-cluster.conf")
- output_file.dirname.mkpath()
- File.write(output_file, output)
+ output_file.dirname.mkpath()
+ File.write(output_file, output)
- }
+ }
+ }
}
-}
+end
diff --git a/lib/refrepo/gen/puppet/kavlang5k.rb b/lib/refrepo/gen/puppet/kavlang5k.rb
index 48704ee582..f79bfc9f7d 100755
--- a/lib/refrepo/gen/puppet/kavlang5k.rb
+++ b/lib/refrepo/gen/puppet/kavlang5k.rb
@@ -1,97 +1,60 @@
-#!/usr/bin/env ruby
-
# !!! Require to "gem install ruby-ip", do not install "ip" gem !!!
-
require 'json'
require 'fileutils'
-if RUBY_VERSION < "2.1"
- puts "This script requires ruby >= 2.1"
- exit
-end
-
require 'pp'
require 'yaml'
require 'pathname'
require 'optparse'
-require_relative '../lib/input_loader'
-require_relative '../lib/hash/hash.rb'
-
-options = {}
-options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
-options[:output_dir] = "/tmp/puppet-repo"
-options[:conf_dir] = File.expand_path("./conf-examples/", File.dirname(__FILE__))
+require 'refrepo/input_loader'
+require 'refrepo/hash/hash'
-OptionParser.new do |opts|
- opts.banner = "Usage: kavlang5k.rb [options]"
+def generate_puppet_kavlang5k(options)
- opts.separator ""
- opts.separator "Example: ruby kavlang5k.rb -s nancy -o /tmp/puppet-repo"
-
- opts.on('-o', '--output-dir dir', String, 'Select the puppet repo path', "Default: " + options[:output_dir]) do |d|
- options[:output_dir] = d
+ if not options[:confdir]
options[:conf_dir] = "#{options[:output_dir]}/platforms/production/generators/kavlan"
end
- opts.on('-c', '--conf-dir dir', String, 'Select the kavlan module configuration path', "Default: ./conf-examples") do |d|
- options[:conf_dir] = d
- end
-
- opts.separator ""
- opts.separator "Filters:"
-
- opts.on('-s', '--sites a,b,c', Array, 'Select site(s)', "Default: " + options[:sites].join(", ")) do |s|
- raise "Wrong argument for -s option." unless (s - options[:sites]).empty?
- options[:sites] = s
- end
-
- opts.on_tail("-h", "--help", "Show this message") do
- puts opts
- exit
- end
-end.parse!
+ raise("Error: #{options[:conf_dir]} does not exist. The given configuration path is incorrect") unless Pathname(options[:conf_dir].to_s).exist?
-raise("Error: #{options[:conf_dir]} does not exist. The given configuration path is incorrect") unless Pathname(options[:conf_dir].to_s).exist?
+ puts "Writing kavlan configuration files to: #{options[:output_dir]}"
+ puts "Using configuration directory: #{options[:conf_dir]}"
+ puts "For site(s): #{options[:sites].join(', ')}"
-puts "Writing kavlan configuration files to: #{options[:output_dir]}"
-puts "Using configuration directory: #{options[:conf_dir]}"
-puts "For site(s): #{options[:sites].join(', ')}"
+ refapi = load_yaml_file_hierarchy
-input_data_dir = "../../input/grid5000/"
+ refapi['sites'].each { |site_uid, site_refapi|
-refapi = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
+ next unless options[:sites].include?(site_uid)
-refapi['sites'].each { |site_uid, site_refapi|
+ conf = YAML::load(ERB.new(File.read("#{options[:conf_dir]}/kavlang5k.yaml"), nil, '-').result(binding))[site_uid]
+ if not conf
+ warn "No generator configuration for site #{site_uid} found in #{options[:conf_dir]}/kavlang5k.yaml, skipping kavlan.conf"
+ else
+ output = ERB.new(File.read(File.expand_path('templates/kavlan.conf.erb', File.dirname(__FILE__))), nil, '-').result(binding)
+ output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kavlan/#{site_uid}/kavlan.conf")
+ output_file.dirname.mkpath()
+ File.write(output_file, output)
+ end
- next unless options[:sites].include?(site_uid)
-
- conf = YAML::load(ERB.new(File.read("#{options[:conf_dir]}/kavlang5k.yaml"), nil, '-').result(binding))[site_uid]
- if not conf
- warn "No generator configuration for site #{site_uid} found in #{options[:conf_dir]}/kavlang5k.yaml, skipping kavlan.conf"
- else
- output = ERB.new(File.read(File.expand_path('templates/kavlan.conf.erb', File.dirname(__FILE__))), nil, '-').result(binding)
- output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kavlan/#{site_uid}/kavlan.conf")
+ output = ERB.new(File.read(File.expand_path('templates/kavlan-cluster.conf.erb', File.dirname(__FILE__))), nil, '-').result(binding)
+ output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kavlan/#{site_uid}/#{site_uid}.conf")
output_file.dirname.mkpath()
File.write(output_file, output)
- end
- output = ERB.new(File.read(File.expand_path('templates/kavlan-cluster.conf.erb', File.dirname(__FILE__))), nil, '-').result(binding)
- output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kavlan/#{site_uid}/#{site_uid}.conf")
- output_file.dirname.mkpath()
- File.write(output_file, output)
+ (1..9).each do |kavlan_id|
+ output = ERB.new(File.read(File.expand_path('templates/kavlan-dhcp.conf.erb', File.dirname(__FILE__))), nil, '-').result(binding)
+ output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kavlan/#{site_uid}/dhcp/dhcpd-#{kavlan_id}.conf")
+ output_file.dirname.mkpath()
+ File.write(output_file, output)
+ end
- (1..9).each do |kavlan_id|
+ # Look for site's global kavlan
+ kavlan_id = refapi['sites'][site_uid]['kavlans'].each_key.select {|k| k.is_a?(Numeric) and k>9}.pop()
output = ERB.new(File.read(File.expand_path('templates/kavlan-dhcp.conf.erb', File.dirname(__FILE__))), nil, '-').result(binding)
- output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kavlan/#{site_uid}/dhcp/dhcpd-#{kavlan_id}.conf")
- output_file.dirname.mkpath()
+ output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kavlan/#{site_uid}/dhcp/dhcpd-0.conf")
File.write(output_file, output)
- end
- # Look for site's global kavlan
- kavlan_id = refapi['sites'][site_uid]['kavlans'].each_key.select {|k| k.is_a?(Numeric) and k>9}.pop()
- output = ERB.new(File.read(File.expand_path('templates/kavlan-dhcp.conf.erb', File.dirname(__FILE__))), nil, '-').result(binding)
- output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kavlan/#{site_uid}/dhcp/dhcpd-0.conf")
- File.write(output_file, output)
-
-}
+ }
+end
diff --git a/lib/refrepo/gen/puppet/templates/conman.erb b/lib/refrepo/gen/puppet/templates/conman.erb
index 15121b4b4a..9c5f8cd66a 100644
--- a/lib/refrepo/gen/puppet/templates/conman.erb
+++ b/lib/refrepo/gen/puppet/templates/conman.erb
@@ -1,5 +1,5 @@
#
-# This file was generated by reference-repository.git/generators/puppet/<%= File.basename($PROGRAM_NAME) %>
+# This file was generated from reference-repository.git
# Do not edit this file by hand. Your changes will be overwritten.
#
diff --git a/lib/refrepo/gen/puppet/templates/dhcp.erb b/lib/refrepo/gen/puppet/templates/dhcp.erb
index eb8d93fe7c..ed8e14d02c 100644
--- a/lib/refrepo/gen/puppet/templates/dhcp.erb
+++ b/lib/refrepo/gen/puppet/templates/dhcp.erb
@@ -1,5 +1,5 @@
#
-# This file was generated by reference-repository.git/generators/puppet/<%= File.basename($PROGRAM_NAME) %>
+# This file was generated from reference-repository.git
# Do not edit this file by hand. Your changes will be overwritten.
#
<%
diff --git a/lib/refrepo/gen/puppet/templates/kadeployg5k.conf.erb b/lib/refrepo/gen/puppet/templates/kadeployg5k.conf.erb
index 82034e2a7a..175d4f2f8c 100644
--- a/lib/refrepo/gen/puppet/templates/kadeployg5k.conf.erb
+++ b/lib/refrepo/gen/puppet/templates/kadeployg5k.conf.erb
@@ -1,5 +1,5 @@
#
-# This file was generated by reference-repository.git/generators/puppet/<%= File.basename($PROGRAM_NAME) %>
+# This file was generated from reference-repository.git
# Do not edit this file by hand. Your changes will be overwritten.
#
---
--
GitLab