Skip to content

GitLab

Attention une mise à jour du serveur va être effectuée le lundi 17 mai entre 13h et 13h30. Cette mise à jour va générer une interruption du service de quelques minutes.

Commit c57a4e7f authored by Gaetan SIMO's avatar Gaetan SIMO
Browse files
Options

[Rakefile] Organized files structure to ease generation by sites/cluster +... 

[Rakefile] Organized files structure to ease generation by sites/cluster + environments + network api. All features on 'rake -T'
parent 89b7c0e7
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1025 additions and 4955 deletions
Rakefile
View file @ c57a4e7f
require 'rubygems'
require 'fileutils'
require 'json'
require 'logger'
require 'yaml'
require 'pp'
require 'restfully'
ROOT_DIR = File.expand_path File.dirname(__FILE__)
LIB_DIR = File.join(ROOT_DIR, "generators", "lib")
$LOAD_PATH.unshift(LIB_DIR) unless $LOAD_PATH.include?(LIB_DIR)
EXTRA_DIR = File.join(ROOT_DIR, "extra", "lib")
$LOAD_PATH.unshift(EXTRA_DIR) unless $LOAD_PATH.include?(EXTRA_DIR)
Rake.application.options.trace = true
require 'grid5000'
#require 'naming-pattern'
task :environment do
Dir.chdir(ROOT_DIR)
@logger = Logger.new(STDERR)
@logger.level = Logger.const_get((ENV['DEBUG'] || 'INFO').upcase)
end
task :api_sites do
api_logger = Logger.new("/dev/null")
api_logger.level = Logger::FATAL
@api = Restfully::Session.new(:configuration_file => File.expand_path("~/.restfully/api.grid5000.fr.yml"),:logger => api_logger)
@api_sites = if ENV['SITE']
[@api.root.sites[ENV['SITE'].to_sym]]
else
@api.root.sites
end
end
task :hosts do
# HOSTS=gw.lille
# HOSTS=*.lille
......@@ -40,305 +44,64 @@ task :hosts do
@host = host
end
namespace :netlinks do
desc "Probe a remote network equipement and retrieve its neighbors in a yaml file. HOST is its grid5000 FQDN."
task :probe => [:environment,:hosts] do
# Scan the site net-links yaml file to find properties about this host (snmp community,...)
# These properties are going to be used to probe the given host
namespace :g5k do
desc "Generates the JSON files based on the generators, for all sites.\nUse SITE=<SITE-NAME> if you wish to restrict the generation to a specific site.\nUse DRY=1 to simulate the execution."
task :generate => [:environment,:hosts] do
host,site = @host.scan(/(\S+)\.(\S+)/).flatten
dirs = Dir.glob("generators/input/#{site}/")
if dirs.empty?
@logger.error "Failed to find a directory containing the net-links yaml file for your site '#{site}'"
next
end
message = []
dirs.each do |dir|
site = File.basename(dir)
net_links_file = File.join(dir,"net-links.yaml")
net_links_dir = File.join(dir,"net-links")
probes = []
root_dir_input = "#{ROOT_DIR}/generators/input/sites"
command = File.join(ROOT_DIR, "generators", "grid5000")
command += " " + File.join(root_dir_input, site,"#{site}.rb")
command += " " + File.join(root_dir_input, site,"clusters","#{host}.rb")
command += " " + File.join(root_dir_input, site,"clusters","#{host}.yaml")
command << " -s" if ENV['DRY'] == "yes"
# puts command
sh command
end
end
net_links = YAML::load_file(net_links_file)
net_links.each do |host_uid,properties|
next if host_uid.match(Regexp.new(host.gsub(/\*/,'\S+'))).nil?
# gather information about this host
probes.push({:uid=>host_uid, :snmp_community => properties["snmp_community"]})
end
if probes.empty?
@logger.warn "Failed to find any host described within the file #{net_links_file}."
else
# copy the 'extra' dir on the site,
# execute all the probing on all hosts
# retrieve the result from all hosts
prober = "weathermap.#{site}.grid5000.fr"
sh "rsync -av extra #{prober}:"
sh "ssh #{prober} 'cd extra && bundle install'"
# rake deadnodes:reasons
# rake deadnodes:tofix
namespace :deadnodes do
probes.each do |info|
sh "ssh #{prober} 'cd extra && ./bin/net-links.rb --host #{info[:uid]} --community #{info[:snmp_community]} --logger stdout:warn'"
end
probes.each do |info|
sh "rsync -av #{prober}:/tmp/#{info[:uid]}.yaml #{net_links_dir}/"
message.push "* #{net_links_dir}/#{info[:uid]}.yaml"
end
end
end
# Print the result on a pretty message
if message.empty?
@logger.warn "No host net-link has been download locally. Please correct warnings first."
else
message.unshift "Network links were saved into the following files : "
message_size = message.max{|a,b|a.size <=> b.size}.size
puts "+-#{"-" * message_size}-+"
puts message.map{|m| "| #{m} "}.join("\n")
puts "+-#{"-" * message_size}-+"
end
desc "List all dead nodes and the reason why they are dead. (SITE=)"
task :reasons => [:environment,:api_sites] do
@logger.level = Logger::INFO
@reasons = true
Rake::Task["deadnodes:browse"].execute
end
def format_vlan(coord,raw_port,linecards)
# puts "Vlan : #{coord.inspect} #{raw_port.inspect}"
desc "List all nodes which have they state not in synch with they comment. (SITE=)"
task :tofix => [:environment,:api_sites] do
@logger.level = Logger::ERROR
@tofix = true
Rake::Task["deadnodes:browse"].execute
end
def format_channel(coord,raw_port,linecards)
# puts "Channel : #{coord.inspect} #{raw_port.inspect}"
end
def format_port(coord,raw_port,linecards)
# puts "#{coord.inspect} #{raw_port.inspect}"
neighbor,site = nil
if raw_port.has_key? :fqdn
neighbor,site = raw_port[:fqdn].scan(/([^.]+)\.([^.]+)\.grid5000\.fr/).flatten
elsif raw_port.has_key? :uid
neighbor,site = raw_port[:uid].scan(/([^.]+)\.([^.]+)\.grid5000\.fr/).flatten
neighbor = raw_port[:uid] if neighbor.nil?
end
return if neighbor.nil?
l = (coord["linecard"]||0)
p = coord["port"]
linecards[l] = {"ports"=>{}} unless linecards.has_key? l
ports = linecards[l]["ports"]
formated_port = ports[p]
if formated_port.nil?
ports[p] = neighbor
elsif formated_port.is_a? Hash
ports[p]["uid"] = neighbor
else
ports[p] = neighbor
end
end
def browse_naming_patterns(dico,patterns,&block)
dico.each do |key,value|
if value.is_a? Hash
browse_naming_patterns(value,patterns,&block)
else
if key == "naming_pattern"
pattern = value
if patterns.has_key? pattern
@logger.warn "Naming Pattern already defined '#{pattern}'. Skipping this redefinition."
task :browse do
def comment_ok?(comment)
# comment.nil? or comment == "OK"
comment == "OK"
end
@api_sites.each do |site|
site.status["nodes"].each do |uid,status|
comment = status["comment"]
state = status["hard"].downcase
if comment_ok?(comment)
if state == "dead"
@logger.error "Node '#{uid}' of state '#{state}' should not have comment '#{comment}'" if @tofix
else
patterns[pattern] = block
# nothing, good state
end
end
end
end
end
desc "Updated formated net-links with the raw information gathered from network equipments."
task :format => [:environment,:hosts] do
# Read the network links fresh out of network equipment
# Read the configuration formated
# update the formated configuration with raw information from the net-links yaml file
host,site = @host.scan(/(\S+)\.(\S+)/).flatten
dirs = Dir.glob("generators/input/#{site}/")
if dirs.empty?
@logger.error "Failed to find a directory containing the net-links yaml file for your site '#{site}'"
next
end
message = []
dirs.each do |dir|
site = File.basename(dir)
net_links_file = File.join(dir,"net-links.yaml")
net_links_dir = File.join(dir,"net-links")
net_links = YAML::load_file(net_links_file)
hosts = {}
net_links.each do |uid,properties|
hosts[uid] = properties unless uid.match(Regexp.new(host.gsub(/\*/,'\S+'))).nil?
end
if hosts.empty?
@logger.warn "Failed to find any host described within the file #{net_links_file}."
else
# Read the network links fresh out of network equipment
# Read the configuration formated
# update the formated configuration with raw information from the net-links yaml file
hosts.each do |uid,properties|
raw_file = File.join(net_links_dir,"#{uid}.yaml")
raw = YAML::load_file(raw_file)
# Go through all formated config, and register a call back that will encode the naming_pattern
patterns = {}
browse_naming_patterns(properties["vlans"],patterns) do |dict,raw_port|
format_vlan(dict,raw_port,properties["vlans"])
end
browse_naming_patterns(properties["channels"],patterns) do |dict,raw_port|
format_channel(dict,raw_port,properties["channels"])
end
browse_naming_patterns(properties["linecards"],patterns) do |dict,raw_port|
format_port(dict,raw_port,properties["linecards"])
end
# Go through all ports to find they linecard and port index,
raw.each do |port|
ifname = port[:ifname]
next if ifname.nil?
# find the naming_pattern that correspond to this ifname
patterns.each do |np,cb|
# scan the ifname with each naming_pattern
dict = NamingPattern.encode(np,ifname)
unless dict.empty?
# Found the naming_pattern for this ifname
# So Place it where it belongs
cb.call(dict,port)
break
end
end
#
end
end
# When all net links in a site are formated, we write them in they file.
File.open(net_links_file,'w'){|f| YAML::dump(net_links,f)}
message.push "#{net_links_file} hosts=#{hosts.keys.inspect}"
end
end
# Print the result on a pretty message
if message.empty?
@logger.warn "No host net-link has been formated. Please correct warnings and your cli parameters."
else
message.map!{|m| "* #{m}"}
message.unshift "Formated Network links were saved into the following files : "
message_size = message.max{|a,b|a.size <=> b.size}.size
puts "+-#{"-" * message_size}-+"
puts message.map{|m| "| #{m} "}.join("\n")
puts "+-#{"-" * message_size}-+"
end
end
desc "Update net-links.yaml with the kavlan like config file."
task :kavlan => [:environment,:hosts] do
config_file = ENV['CONF']
abort "You must provide a CONF=, which is the path to the kavlan like config file." if config_file.nil?
host,site = @host.scan(/(\S+)\.(\S+)/).flatten
dirs = Dir.glob("generators/input/#{site}/")
if dirs.empty?
@logger.error "Failed to find a directory containing the net-links yaml file for your site '#{site}'"
next
end
message = []
dirs.each do |dir|
site = File.basename(dir)
net_links_file = File.join(dir,"net-links.yaml")
net_links_dir = File.join(dir,"net-links")
net_links = YAML::load_file(net_links_file)
net_links.keep_if do |uid,properties|
uid.match(Regexp.new(host.gsub(/\*/,'\S+'))) != nil
end
if net_links.empty?
@logger.warn "Failed to find any host described within the file #{net_links_file}."
else
# parse the kavlan like config file
config = {}
File.read(config_file).lines.each do |line|
if ((scan = line.strip.scan(/^([^.]+)\.([^.]+)\.grid5000\.fr\s+(\S+)\s+(\S+)$/)).size > 0)
uid,site,ifname,router = scan.flatten
router = router.downcase
config[router] = [] unless config.has_key? router
# if ((scan = uid.scan(/^(\S+-\d+)-(\S+)$/)).size > 0)
# uid,port = scan.flatten
# config[router].push({:uid=>uid,:ifname=>ifname,:port=>port})
# else
config[router].push({:uid=>uid,:ifname=>ifname})
# end
end
end
updated = []
# Update the net-links with kavlan config file
net_links.each do |uid,properties|
router = config[uid]
next if router.nil?
updated.push uid
# register interfaces naming patterns
patterns = {}
browse_naming_patterns(properties["linecards"],patterns) do |dict,raw_port|
format_port(dict,raw_port,properties["linecards"])
end
# parse the ifname
router.each do |neighbor|
ifname = neighbor[:ifname]
# find the naming_pattern that correspond to this ifname
patterns.each do |np,cb|
# scan the ifname with each naming_pattern
dict = NamingPattern.encode(np,ifname)
unless dict.empty?
# Found the naming_pattern for this ifname
# So Place it where it belongs
cb.call(dict,neighbor)
break
end
end
else
if state == "dead"
@logger.info "Node '#{uid}' is dead because '#{comment}'" if @reasons
else
@logger.error "Node '#{uid}' should have the not-dead-comment 'OK', since its state is '#{state}'. Instead, it has comment '#{comment}'." if @tofix
end
end
# When all net links in a site are formated, we write them in they file.
File.open(net_links_file,'w'){|f| YAML::dump(net_links,f)}
message.push "#{net_links_file} hosts=#{updated.inspect}"
end
end
# Print the result on a pretty message
if message.empty?
@logger.warn "No host net-link has been formated. Please correct warnings and/or your cli parameters."
else
message.map!{|m| "* #{m}"}
message.unshift "Formated Network links were saved into the following files : "
message_size = message.max{|a,b|a.size <=> b.size}.size
puts "+-#{"-" * message_size}-+"
puts message.map{|m| "| #{m} "}.join("\n")
puts "+-#{"-" * message_size}-+"
end
end
desc "Generates The JSON files that will be used for network API.\nUse DRY=1 to simulate the execution. "
task :generate => [:environment,:hosts] do
host,site = @host.scan(/(\S+)\.(\S+)/).flatten
dirs = Dir.glob("generators/input/#{site}/")
if dirs.empty?
@logger.error "Failed to find a directory containing the net-links yaml file for your site '#{site}'"
next
end
generator = "#{File.join(ROOT_DIR, "generators", "grid5000")}"
net_link_generator = "#{File.join(ROOT_DIR, "generators", "input", "net-links-generator.rb")}"
dirs.each do |dir|
site = File.basename(dir)
net_links_file = File.expand_path(File.join(dir,"net-links.yaml"))
site_file = File.expand_path(File.join(dir,"..","#{site}.rb"))
command = "#{generator} #{site_file} #{net_link_generator} #{net_links_file}"
command << " -s" if ENV['DRY'] && ENV['DRY'] != "0"
sh command
end
end
end
namespace :g5k do
desc "Generates the JSON files based on the generators, for all sites.\nUse SITE=<SITE-NAME> if you wish to restrict the generation to a specific site.\nUse DRY=1 to simulate the execution."
task :generate => :environment do
site = if ENV['SITE']
ENV['SITE']
else
"*"
end
command = "#{File.join(ROOT_DIR, "generators", "grid5000")} #{File.join(ROOT_DIR, "generators", "input", "#{site}.rb")} #{File.join(ROOT_DIR, "generators", "input", "#{site}.yaml")}"
command << " -s" if ENV['DRY'] && ENV['DRY'] != "0"
@logger.info "Executing #{command.inspect}..."
system command
end
end
......@@ -432,3 +195,56 @@ namespace :oar do
end
end
end
namespace :netlinks do
desc "Generates network API JSON files based on net-links yaml files.\nUse DRY=yes to simulate the execution. "
task :generate => [:environment,:hosts] do
host,site = @host.scan(/(\S+)\.(\S+)/).flatten
root_dir_input = "#{ROOT_DIR}/generators/input/sites"
command = File.join(ROOT_DIR, "generators", "grid5000")
command += " " + File.join(root_dir_input, "net-links.rb")
command += " " + File.join(root_dir_input, site,"#{site}.rb")
command += " " + File.join(root_dir_input, site,"net-links","#{host}.yaml")
command << " -s" if ENV['DRY'] == "yes"
# puts command
sh command
end
end
namespace :env do
desc "Generates environment JSON files .\nUse DRY=yes to simulate the execution. "
task :generate => [:environment] do
env_name = ENV["ENV_NAME"]
abort "You must provide ENV_NAME=" if env_name.nil?
root_dir_input = "#{ROOT_DIR}/generators/input"
command = File.join(ROOT_DIR, "generators", "grid5000")
command += " " + File.join(root_dir_input, "environments","#{env_name}")
command << " -s" if ENV['DRY'] == "yes"
sh command
end
end
=begin
task :mm => [:environment,:hosts] do
host,site = @host.scan(/(\S+)\.(\S+)/).flatten
root_dir_input = "#{ROOT_DIR}/generators/input"
Dir.glob("#{root_dir_input}/#{site}*").each do |file|
filename = File.basename(file)
if ((scan = filename.scan(/(\S+)-(\S+)\.(rb|yaml)/)).size > 0)
site,cluster = scan.flatten
cmd = "mkdir -p #{root_dir_input}/#{site}/clusters"
cmd += " && mv #{file} #{root_dir_input}/#{site}/clusters/#{cluster}#{File.extname(filename)}"
sh cmd
# puts cmd
elsif ((scan = filename.scan(/(\S+)\.rb/)).size > 0)
site = scan.first.first
cmd = "mkdir -p #{root_dir_input}/#{site}/clusters"
cmd += " && mv #{file} #{root_dir_input}/#{site}/#{site}.rb"
sh cmd
end
end
end
=end
generators/input/globals.yaml deleted 100644 → 0
View file @ 89b7c0e7
sites:
- bordeaux
- grenoble
- lille
- lyon
- nancy
- orsay
- rennes
- sophia
- toulouse
\ No newline at end of file
generators/input/input.description.ignore deleted 100644 → 0
View file @ 89b7c0e7
site :rennes do |site_uid| # General description of the site.
name "Rennes"
location "Rennes, France"
web "http://www.irisa.fr"
description ""
latitude 48.114722
longitude -1.679444
email_contact
sys_admin_contact
security_contact
user_support_contact
# Describes the list of environments available on the site.
# Notice the :refer_to option that creates a hard link to an existing file, starting from the ./data directory.
%w{sid-x64-base-1.0}.each do |env_id|
environment env_id, :refer_to => "grid5000/environments/#{env_id}"
end
cluster :paraquad do |cluster_uid|
model "Dell PowerEdge 1950"
created_at Time.parse("2006-12-01").httpdate # Date of arrival of the cluster into Grid5000.
# Do not forget the .httpdate call.
# Describes the nodes that compose the cluster. To avoid repetition, you may enclose the description in a loop:
64.times do |i| # index is starting at 0
node "#{cluster_uid}-#{i+1}" do |node_uid| # node UID.
# lists the supported job types (true, false or string value)
supported_job_types({:deploy => true, :besteffort => true, :virtual => "ivt"})
# describes the architecture of the node
architecture({
:smp_size => 2, # number of physical CPUs
:smt_size => 4, # total number of logical CPUs (cores)
:platform_type => "x86_64"
})
# describes the processors
processor({
:vendor => "Intel",
:model => "Intel Xeon",
:version => "5148 LV",
:clock_speed => 2.33.G, # G = 10^9, M = 10^6, etc.
:instruction_set => "", # x86-64, IA-64, ...
:other_description => "",
:cache_l1 => nil, # first-level unified cache size of the processor
:cache_l1i => nil, # first-level instruction cache size of the processor
:cache_l1d => nil, # first-level data cache size of the processor
:cache_l2 => nil # second-level unified cache size of the processor
})
main_memory({
:ram_size => 4.GiB, # the amount of RAM. MAKE SURE YOU USE GiB or MiB or KiB, and not GB, MB, KB...
:virtual_size => nil # the amount of virtual memory (RAM+Swap)
})
# Information about the default operating system of the node
# $ lsb_release -a.
# =>
# Distributor ID: Ubuntu = :name
# Description: Ubuntu 6.10
# Release: 6.10 = :release
# Codename: edgy = :version
operating_system({
:name => nil, # name of the operating system
:release => nil, # release of the operating system
:version => nil # version of the operating system
})
# Describes the list of storage devices available on the node
# Notice the use of ***GB*** (and not GiB)
# Required properties include: :interface, :size
# You may add the :driver property, or a :raid property (e.g. :raid => "0") if needed.
storage_devices [
{:interface => 'SATA', :size => 160.GB, :driver => "mptsas"}
]
# Describes the list of network adapters available on the node
# Required properties include: :interface, :rate, :switch, :network_address, :ip, :enabled
# You may add properties such as: :driver, :vendor, :version
# You may include adapters reserved for management (in this case, set enabled to false)
network_adapters [
{:interface => 'Myri-10G', :rate => 10.G,
:switch => "c6509-grid", :network_address => "#{node_uid}.#{site_uid}.grid5000.fr", :ip => dns_lookup("#{node_uid}.#{site_uid}.grid5000.fr"),
:vendor => 'Myrinet', :version => "10G-PCIE-8A-C", :enabled => true},
{:interface => 'Ethernet', :rate => 1.G, :enabled => true},
{:interface => 'Ethernet', :rate => 1.G, :enabled => false}
]
end
end
end
end
\ No newline at end of file
generators/input/lille.yaml deleted 100644 → 0
View file @ 89b7c0e7
This diff is collapsed.
generators/input/lille/lille.rb deleted 100644 → 0
View file @ 89b7c0e7
site :lille do
lookup("net-links").each do |equipment_uid, properties|
network_equipment equipment_uid do |uid|
model properties["model"]
kind properties["kind"]
snmp_community properties["snmp_community"]
vlans properties["vlans"]
routes properties["routes"]
channels properties["channels"]
linecards_array = []
properties["linecards"].each do |linecard_index,linecard|
ports = []
linecard.delete("ports").each do |port_index,port|
port = {"uid"=>port} if port.is_a? String
ports[port_index] = port
end
linecard["ports"] = ports.map{|p| p || {}}
linecards_array[linecard_index] = linecard