Attention une mise à jour du serveur va être effectuée le lundi 17 mai entre 13h et 13h30. Cette mise à jour va générer une interruption du service de quelques minutes.

Commit 804fca60 authored by Lucas Nussbaum's avatar Lucas Nussbaum

[dev] port puppet generators

parent 78141f1b
......@@ -15,6 +15,12 @@ REFAPI_DIR = "./generators/reference-api"
PUPPET_DIR = "./generators/puppet"
VALIDATORS_DIR = "./generators/input-validators"
if Dir::exists?(ENV['HOME'] + '/.gpuppet/repo')
PUPPET_ODIR = ENV['HOME'] + '/.gpuppet/repo'
else
PUPPET_ODIR = '/tmp/puppet'
end
G5K_SITES = RefRepo::Utils::get_sites
namespace :valid do
......@@ -156,9 +162,16 @@ namespace :puppet do
all_puppet_tasks = [:bindg5k, :conmang5k, :dhcpg5k, :kadeployg5k, :lanpowerg5k, :kavlang5k]
all_puppet_tasks.each { |t|
desc "Generate #{t} configuration"
desc "Generate #{t} configuration -- parameters: SITE={grenoble,...} OUTPUTDIR=[default: #{PUPPET_ODIR}] [CONFDIR=...] VERBOSE=1"
task t do
invoke_script "#{PUPPET_DIR}/#{t}.rb"
require "refrepo/gen/puppet/#{t}"
options = {}
options[:sites] = ( ENV['SITE'] ? ENV['SITE'].split(',') : G5K_SITES )
options[:output_dir] = ENV['OUTPUTDIR'] || PUPPET_ODIR
options[:verbose] = (ENV['VERBOSE'] != nil)
options[:conf_dir] = ENV['CONFDIR'] if ENV['CONFDIR']
send("generate_puppet_#{t}", options)
end
}
......
#!/usr/bin/ruby
# coding: utf-8
if RUBY_VERSION < "2.1"
puts "This script requires ruby >= 2.1"
exit
end
# See also: https://www.grid5000.fr/mediawiki/index.php/DNS_server
require 'pp'
......@@ -14,44 +6,7 @@ require 'pathname'
require 'fileutils'
require 'optparse'
require 'dns/zone'
require_relative '../lib/input_loader'
input_data_dir = "../../input/grid5000/"
refapi = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
$options = {}
$options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia toulouse}
$options[:output_dir] = "/tmp/puppet"
$options[:verbose] = false
OptionParser.new do |opts|
opts.banner = "Usage: bindg5k.rb [options]"
opts.separator ""
opts.separator "Example: ruby bindg5k.rb -s nancy -o /tmp/puppet"
opts.on('-o', '--output-dir dir', String, 'Select the puppet repo path', "Default: " + $options[:output_dir]) do |d|
$options[:output_dir] = d
end
opts.separator ""
opts.separator "Filters:"
opts.on('-s', '--sites a,b,c', Array, 'Select site(s)', "Default: " + $options[:sites].join(", ")) do |s|
raise "Wrong argument for -s option." unless (s - $options[:sites]).empty?
$options[:sites] = s
end
opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
$options[:verbose] = true
end
opts.on_tail("-h", "--help", "Show this message") do
puts opts
exit
end
end.parse!
require 'refrepo/input_loader'
#Prettier aligned dump of records
class DNS::Zone::RR::A
......@@ -467,140 +422,146 @@ def write_zone(zone)
File.write(zone.file_path, zone.dump)
end
puts "Writing DNS configuration files to: #{$options[:output_dir]}"
puts "For site(s): #{$options[:sites].join(', ')}"
# main method
def generate_puppet_bindg5k(options)
$options = options
puts "Writing DNS configuration files to: #{$options[:output_dir]}"
puts "For site(s): #{$options[:sites].join(', ')}"
puts "Note: if you modify *-manual.db files you will have to manually update the serial in managed db file for changes to be applied"
puts "Note: if you modify *-manual.db files you will have to manually update the serial in managed db file for changes to be applied"
$written_files = []
$written_files = []
# Loop over Grid'5000 sites
refapi["sites"].each { |site_uid, site|
next unless $options[:sites].include?(site_uid)
refapi = load_yaml_file_hierarchy
dest_dir = "#{$options[:output_dir]}/platforms/production/modules/generated/files/bind/"
zones_dir = File.join(dest_dir, "zones/#{site_uid}")
# Loop over Grid'5000 sites
refapi["sites"].each { |site_uid, site|
site_records = {}
next unless $options[:sites].include?(site_uid)
# Servers
site_records['servers'] = get_servers_records(site) unless site['servers'].nil?
dest_dir = "#{$options[:output_dir]}/platforms/production/modules/generated/files/bind/"
zones_dir = File.join(dest_dir, "zones/#{site_uid}")
# PDUs
site_records['pdus'] = get_pdus_records(site) unless site['pdus'].nil?
site_records = {}
# Networks and laptops (same input format)
site_records['networks'] = get_networks_records(site, 'networks') unless site['networks'].nil?
site_records['laptops'] = get_networks_records(site, 'laptops') unless site['laptops'].nil?
# Servers
site_records['servers'] = get_servers_records(site) unless site['servers'].nil?
site.fetch("clusters", []).sort.each { |cluster_uid, cluster|
# PDUs
site_records['pdus'] = get_pdus_records(site) unless site['pdus'].nil?
cluster.fetch('nodes').select { |node_uid, node|
node != nil && node["status"] != "retired" && node.has_key?('network_adapters')
}.each_sort_by_node_uid { |node_uid, node|
# Networks and laptops (same input format)
site_records['networks'] = get_networks_records(site, 'networks') unless site['networks'].nil?
site_records['laptops'] = get_networks_records(site, 'laptops') unless site['laptops'].nil?
network_adapters = {}
site.fetch("clusters", []).sort.each { |cluster_uid, cluster|
# Nodes
node.fetch('network_adapters').each { |net_uid, net_hash|
network_adapters[net_uid] = {"ip" => net_hash["ip"], "mounted" => net_hash["mounted"], "alias" => net_hash["alias"]}
}
# Mic
if node['mic'] && node['mic']['ip']
network_adapters['mic0'] = {"ip" => node['mic']['ip']}
end
cluster.fetch('nodes').select { |node_uid, node|
node != nil && node["status"] != "retired" && node.has_key?('network_adapters')
}.each_sort_by_node_uid { |node_uid, node|
site_records[cluster_uid] ||= []
site_records[cluster_uid] += get_node_records(cluster_uid, node_uid, network_adapters)
network_adapters = {}
# Kavlan
if node['kavlan']
kavlan_adapters = {}
node.fetch('kavlan').each { |net_uid, net_hash|
net_hash.each { |kavlan_net_uid, ip|
kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"] = {"ip" => ip, "mounted" => node['network_adapters'][net_uid]['mounted']}
}
# Nodes
node.fetch('network_adapters').each { |net_uid, net_hash|
network_adapters[net_uid] = {"ip" => net_hash["ip"], "mounted" => net_hash["mounted"], "alias" => net_hash["alias"]}
}
site_records["#{cluster_uid}-kavlan"] ||= []
site_records["#{cluster_uid}-kavlan"] += get_node_kavlan_records(cluster_uid, node_uid, network_adapters, kavlan_adapters)
end
} # each nodes
} # each cluster
reverse_records = {} # one hash entry per reverse dns file
# Mic
if node['mic'] && node['mic']['ip']
network_adapters['mic0'] = {"ip" => node['mic']['ip']}
end
site_records[cluster_uid] ||= []
site_records[cluster_uid] += get_node_records(cluster_uid, node_uid, network_adapters)
# Kavlan
if node['kavlan']
kavlan_adapters = {}
node.fetch('kavlan').each { |net_uid, net_hash|
net_hash.each { |kavlan_net_uid, ip|
kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"] = {"ip" => ip, "mounted" => node['network_adapters'][net_uid]['mounted']}
}
}
site_records["#{cluster_uid}-kavlan"] ||= []
site_records["#{cluster_uid}-kavlan"] += get_node_kavlan_records(cluster_uid, node_uid, network_adapters, kavlan_adapters)
end
} # each nodes
} # each cluster
reverse_records = {} # one hash entry per reverse dns file
site_records.each { |zone, records|
#Sort records
site_records[zone] = sort_records(records)
records.each{ |record|
#get Reverse records
reverse_file_name, reverse_record = get_reverse_record(record, site_uid)
if reverse_file_name != nil
reverse_records[reverse_file_name] ||= []
reverse_records[reverse_file_name].each {|r|
if r.label == reverse_record.label
puts "Warning: reverse entry with IP #{reverse_record.label} already exists in #{reverse_file_name}, #{reverse_record.name} is duplicate"
end
}
reverse_records[reverse_file_name] << reverse_record
end
}
}
site_records.each { |zone, records|
zones = []
#Sort records
site_records[zone] = sort_records(records)
#Sort reverse records and create reverse zone from files
reverse_records.each{ |file_name, records|
records.sort!{ |a, b|
a.label.to_i <=> b.label.to_i
}
records.each{ |record|
#get Reverse records
reverse_file_name, reverse_record = get_reverse_record(record, site_uid)
if reverse_file_name != nil
reverse_records[reverse_file_name] ||= []
reverse_records[reverse_file_name].each {|r|
if r.label == reverse_record.label
puts "Warning: reverse entry with IP #{reverse_record.label} already exists in #{reverse_file_name}, #{reverse_record.name} is duplicate"
end
}
reverse_records[reverse_file_name] << reverse_record
reverse_file_path = File.join(zones_dir, file_name)
zone = load_zone(reverse_file_path, site_uid, site, true)
if diff_zone_file(zone, records)
zone.soa.serial = update_serial(zone.soa.serial)
end
zone.records = records;
zones << zone
}
}
zones = []
#Sort reverse records and create reverse zone from files
reverse_records.each{ |file_name, records|
records.sort!{ |a, b|
a.label.to_i <=> b.label.to_i
#Manage site zone (SITE.db file)
#It only contains header and inclusion of other db files
#Check modification in included files and update serial accordingly
site_zone_path = File.join(zones_dir, site_uid + ".db")
site_zone = load_zone(site_zone_path, site_uid, site, true)
site_zone_changed = false
site_records.each{ |type, records|
next if records.empty?
zone_file_path = File.join(zones_dir, site_uid + "-" + type + ".db")
zone = load_zone(zone_file_path, site_uid, site, false)
if diff_zone_file(zone, records)
puts "Zone file changed: #{zone.file_path}" if $options[:verbose]
site_zone_changed = true
end
zone.records = records
site_zone.include += "$INCLUDE /etc/bind/zones/#{site_uid}/#{File.basename(zone_file_path)}\n"
zones << zone
}
reverse_file_path = File.join(zones_dir, file_name)
zone = load_zone(reverse_file_path, site_uid, site, true)
if diff_zone_file(zone, records)
zone.soa.serial = update_serial(zone.soa.serial)
if (site_zone_changed)
site_zone.soa.serial = update_serial(site_zone.soa.serial)
end
zone.records = records;
zones << zone
}
#Manage site zone (SITE.db file)
#It only contains header and inclusion of other db files
#Check modification in included files and update serial accordingly
site_zone_path = File.join(zones_dir, site_uid + ".db")
site_zone = load_zone(site_zone_path, site_uid, site, true)
site_zone_changed = false
site_records.each{ |type, records|
next if records.empty?
zone_file_path = File.join(zones_dir, site_uid + "-" + type + ".db")
zone = load_zone(zone_file_path, site_uid, site, false)
if diff_zone_file(zone, records)
puts "Zone file changed: #{zone.file_path}" if $options[:verbose]
site_zone_changed = true
end
zone.records = records
site_zone.include += "$INCLUDE /etc/bind/zones/#{site_uid}/#{File.basename(zone_file_path)}\n"
zones << zone
}
if (site_zone_changed)
site_zone.soa.serial = update_serial(site_zone.soa.serial)
end
zones << site_zone
zones << site_zone
zones += get_orphan_reverse_manual_zones(zones_dir, site_uid, site)
zones += get_orphan_reverse_manual_zones(zones_dir, site_uid, site)
zones.each{ |zone|
write_zone(zone)
}
zones.each{ |zone|
write_zone(zone)
}
write_site_conf(site_uid, dest_dir, zones_dir)
write_site_local_conf(site_uid, dest_dir, zones_dir)
write_site_conf(site_uid, dest_dir, zones_dir)
write_site_local_conf(site_uid, dest_dir, zones_dir)
} # each sites
} # each sites
end
#!/usr/bin/ruby
if RUBY_VERSION < "2.1"
puts "This script requires ruby >= 2.1"
exit
end
# This script generates conmang5k/files/<site_uid>/conman.conf from input/, conf/console.yaml and conf/console-password.yaml
require 'pp'
require 'erb'
require 'pathname'
require 'optparse'
require_relative '../lib/input_loader'
require_relative '../lib/hash/hash.rb'
require 'refrepo/input_loader'
require 'refrepo/hash/hash'
options = {}
options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
options[:output_dir] = "/tmp/puppet-repo"
options[:conf_dir] = File.expand_path("conf-examples/", File.dirname(__FILE__))
# Apply ERB template and save result to file
def write_conman_file(site_uid, site_refapi, site_config, site_credentials, options)
output = ERB.new(File.read(File.expand_path('templates/conman.erb', File.dirname(__FILE__)))).result(binding)
OptionParser.new do |opts|
opts.banner = "Usage: conmang5k.rb [options]"
output_file = Pathname("#{options[:output_dir]}/platforms/production/modules/generated/files/conman/server/#{site_uid}.conf")
opts.separator ""
opts.separator "Example: ruby conmang5k.rb -s nancy -o /tmp/puppet-repo"
opts.separator "Example: ruby conmang5k.rb -s nancy -o $GPUPPET_PREFIX/repo"
output_file.dirname.mkpath()
File.write(output_file, output)
end
opts.on('-o', '--output-dir dir', String, 'Select the puppet repo path', "Default: " + options[:output_dir]) do |d|
options[:output_dir] = d
def generate_puppet_conmang5k(options)
if not options[:confdir]
options[:conf_dir] = "#{options[:output_dir]}/platforms/production/generators/ipmitools/"
end
opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: #{options[:conf_dir]}") do |d|
options[:conf_dir] = d
end
raise("Error: #{options[:conf_dir]} does not exist. The given configuration path is incorrect") unless Pathname(options[:conf_dir].to_s).exist?
opts.separator ""
opts.separator "Filters:"
puts "Writing Conman configuration files to: #{options[:output_dir]}"
puts "Using configuration directory: #{options[:conf_dir]}"
puts "For site(s): #{options[:sites].join(', ')}"
opts.on('-s', '--sites a,b,c', Array, 'Select site(s)', "Default: " + options[:sites].join(", ")) do |s|
raise "Wrong argument for -s option." unless (s - options[:sites]).empty?
options[:sites] = s
end
refapi = load_yaml_file_hierarchy
opts.on_tail("-h", "--help", "Show this message") do
puts opts
exit
end
end.parse!
config = YAML::load_file(options[:conf_dir] + 'console.yaml')
credentials = YAML::load_file(options[:conf_dir] + 'console-password.yaml')
raise("Error: #{options[:conf_dir]} does not exist. The given configuration path is incorrect") unless Pathname(options[:conf_dir].to_s).exist?
puts "Writing Conman configuration files to: #{options[:output_dir]}"
puts "Using configuration directory: #{options[:conf_dir]}"
puts "For site(s): #{options[:sites].join(', ')}"
# Loop over each site
refapi["sites"].each { |site_uid, site_refapi|
#Input
input_data_dir = "../../input/grid5000/"
refapi = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
next unless options[:sites].include?(site_uid)
config = YAML::load_file(options[:conf_dir] + 'console.yaml')
credentials = YAML::load_file(options[:conf_dir] + 'console-password.yaml')
# Apply ERB template and save result to file
def write_conman_file(site_uid, site_refapi, site_config, site_credentials, options)
output = ERB.new(File.read(File.expand_path('templates/conman.erb', File.dirname(__FILE__)))).result(binding)
output_file = Pathname("#{options[:output_dir]}/platforms/production/modules/generated/files/conman/server/#{site_uid}.conf")
output_file.dirname.mkpath()
File.write(output_file, output)
write_conman_file(site_uid, site_refapi, config[site_uid], credentials[site_uid], options)
}
end
# Loop over each site
refapi["sites"].each { |site_uid, site_refapi|
next unless options[:sites].include?(site_uid)
write_conman_file(site_uid, site_refapi, config[site_uid], credentials[site_uid], options)
}
#!/usr/bin/ruby
if RUBY_VERSION < "2.1"
puts "This script requires ruby >= 2.1"
exit
end
require 'pp'
require 'erb'
require 'pathname'
require 'optparse'
require_relative '../lib/input_loader'
input_data_dir = "../../input/grid5000/"
global_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
options = {}
options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
options[:output_dir] = "/tmp/puppet-repo"
OptionParser.new do |opts|
opts.banner = "Usage: dhcpg5k.rb [options]"
opts.separator ""
opts.separator "Example: ruby dhcpg5k.rb -s nancy -o /tmp/puppet-repo"
opts.on('-o', '--output-dir dir', String, 'Select the puppet repo path', "Default: " + options[:output_dir]) do |d|
options[:output_dir] = d
end
opts.separator ""
opts.separator "Filters:"
opts.on('-s', '--sites a,b,c', Array, 'Select site(s)', "Default: " + options[:sites].join(", ")) do |s|
raise "Wrong argument for -s option." unless (s - options[:sites]).empty?
options[:sites] = s
end
opts.on_tail("-h", "--help", "Show this message") do
puts opts
exit
end
end.parse!
require 'refrepo/input_loader'
# Get the mac and ip of a node. Throw exception if error.
def get_network_info(node_hash, network_interface)
......@@ -80,86 +41,91 @@ def write_dhcp_file(data, options)
File.write(output_file, output)
end
puts "Writing DHCP configuration files to: #{options[:output_dir]}"
puts "For site(s): #{options[:sites].join(', ')}"
def generate_puppet_dhcpg5k(options)
global_hash = load_yaml_file_hierarchy
# Loop over Grid'5000 sites
global_hash["sites"].each { |site_uid, site_hash|
next unless options[:sites].include?(site_uid)
puts "Writing DHCP configuration files to: #{options[:output_dir]}"
puts "For site(s): #{options[:sites].join(', ')}"
puts site_uid
# Loop over Grid'5000 sites
global_hash["sites"].each { |site_uid, site_hash|
#
# eth, bmc and mic0
#
next unless options[:sites].include?(site_uid)
# Relocate ip/mac info of MIC
site_hash.fetch("clusters").each { |cluster_uid, cluster_hash|
cluster_hash.fetch('nodes').each { |node_uid, node_hash|
next if node_hash == nil || node_hash['status'] == 'retired'
puts site_uid
if node_hash['mic'] && node_hash['mic']['ip'] && node_hash['mic']['mac']
node_hash['network_adapters'] ||= {}
node_hash['network_adapters']['mic0'] ||= {}
node_hash['network_adapters']['mic0']['ip'] = node_hash['mic'].delete('ip')
node_hash['network_adapters']['mic0']['mac'] = node_hash['mic'].delete('mac')
end
}
}
#
# eth, bmc and mic0
#
# One file for each clusters
site_hash.fetch("clusters").each { |cluster_uid, cluster_hash|
# networks = ["eth", "bmc"]
# networks << 'mic0' if cluster_hash['nodes'].values.any? {|x| x['network_adapters']['mic0'] }
write_dhcp_file({
"filename" => "cluster-" + cluster_uid + ".conf",
"site_uid" => site_uid,
"nodes" => cluster_hash.fetch('nodes'),
"network_adapters" => ["eth", "bmc", "mic0"],
"optional_network_adapters" => ["mic0"]
}, options)
}
# Relocate ip/mac info of MIC
site_hash.fetch("clusters").each { |cluster_uid, cluster_hash|
cluster_hash.fetch('nodes').each { |node_uid, node_hash|
next if node_hash == nil || node_hash['status'] == 'retired'
#
#
#
# Other dhcp files
["networks", "laptops", "servers"].each { |key|
write_dhcp_file({
"filename" => key + ".conf",
"site_uid" => site_uid,
"nodes" => site_hash[key],
"network_adapters" => ["default", "eth", "bmc", "adm"],
"optional_network_adapters" => ["eth", "bmc", "adm"]
}, options) unless site_hash[key].nil?
}
if node_hash['mic'] && node_hash['mic']['ip'] && node_hash['mic']['mac']
node_hash['network_adapters'] ||= {}
node_hash['network_adapters']['mic0'] ||= {}
node_hash['network_adapters']['mic0']['ip'] = node_hash['mic'].delete('ip')
node_hash['network_adapters']['mic0']['mac'] = node_hash['mic'].delete('mac')
end
}
}
#
# PDUs
#
if ! site_hash['pdus'].nil?
# Relocate ip/mac info of PDUS
site_hash['pdus'].each { |pdu_uid, pdu_hash|
if pdu_hash['ip'] && pdu_hash['mac']
pdu_hash['network_adapters'] ||= {}