Attention une mise à jour du serveur va être effectuée le lundi 17 mai entre 13h et 13h30. Cette mise à jour va générer une interruption du service de quelques minutes.

Commit a995db96 authored by Jérémie Gaidamour's avatar Jérémie Gaidamour

[dev] Removed old generators

parent 99eef878

Too many changes to show.

To preserve performance only 1000 of 1000+ files are displayed.
source :rubygems
gem 'json_pure', '~> 1.5'
gem 'restfully','1.0.5'
gem "rake", ">= 0.9.2.2"
gem "net-admin-common",
:git => "ssh://g5kadmin@git.grid5000.fr/srv/git/repos/net-admin-common",
:tag => "2.5.3"
gem "cacti-play",
:git => "ssh://g5kadmin@git.grid5000.fr/srv/git/repos/cacti-play",
:tag => "2.8"
gem "weathermap",
:git => "ssh://g5kadmin@git.grid5000.fr/srv/git/repos/weathermap",
:tag => "5.8.4"
GIT
remote: ssh://g5kadmin@git.grid5000.fr/srv/git/repos/cacti-play
revision: 4a2b6cb6ae2d9f791a4529737491ea02eeeb58c8
tag: 2.8
specs:
cacti-play (2.7.4)
net-admin-common (>= 2.5.3)
GIT
remote: ssh://g5kadmin@git.grid5000.fr/srv/git/repos/net-admin-common
revision: 3985600064a0a583ee9a47b4d51c97bc94f291a2
tag: 2.5.3
specs:
net-admin-common (2.5.3)
net-ssh (>= 2.3.0)
net-ssh-multi (>= 1.1)
rest-client (>= 1.6.7)
GIT
remote: ssh://g5kadmin@git.grid5000.fr/srv/git/repos/weathermap
revision: a258691fb1bdfa8db04897fd593bc2f9d3264b53
tag: 5.8.4
specs:
weathermap (5.8.2)
cacti-play (>= 2.7.4)
interpolator (>= 0.15)
ipaddress (>= 0.8.0)
json (>= 1.7.0)
net-admin-common (>= 2.5.3)
GEM
remote: http://rubygems.org/
specs:
addressable (2.3.2)
backports (2.6.3)
bond (0.4.2)
interpolator (0.15)
ipaddress (0.8.0)
json (1.7.4)
json_pure (1.7.4)
mime-types (1.19)
net-ssh (2.6.2)
net-ssh-gateway (1.1.0)
net-ssh (>= 1.99.1)
net-ssh-multi (1.1)
net-ssh (>= 2.1.4)
net-ssh-gateway (>= 0.99.0)
rack (1.4.1)
rack-cache (1.2)
rack (>= 0.4)
rake (0.9.2.2)
rb-readline (0.4.2)
rest-client (1.6.7)
mime-types (>= 1.16)
rest-client-components (1.2.0)
rack (>= 1.0.1)
rest-client (>= 1.6.0, < 1.7.0)
restfully (1.0.5)
addressable
backports
json (~> 1.5)
rack-cache
rb-readline
rest-client (~> 1.6)
rest-client-components
ripl (= 0.6.1)
ripl-color_streams
ripl-multi_line
ripl-play (~> 0.2.1)
ripl-short_errors
ripl (0.6.1)
bond (~> 0.4.0)
ripl-color_streams (0.1.2)
ripl (>= 0.2.8)
ripl-multi_line (0.3.0)
ripl (>= 0.3.6)
ripl-play (0.2.1)
ripl (>= 0.3.0)
ripl-short_errors (0.1.0)
ripl (>= 0.2.8)
PLATFORMS
ruby
DEPENDENCIES
cacti-play!
json_pure (~> 1.5)
net-admin-common!
rake (>= 0.9.2.2)
restfully (= 1.0.5)
weathermap!
#
require 'fileutils'
require 'json'
require 'logger'
require 'restfully'
ROOT_DIR = File.expand_path File.dirname(__FILE__)
LIB_DIR = File.join(ROOT_DIR, "generators", "lib")
$LOAD_PATH.unshift(LIB_DIR) unless $LOAD_PATH.include?(LIB_DIR)
require 'grid5000'
task :environment do
Dir.chdir(ROOT_DIR)
@logger = Logger.new(STDERR)
@logger.level = Logger.const_get((ENV['DEBUG'] || 'INFO').upcase)
end
task :api_sites do
api_logger = Logger.new("/dev/null")
api_logger.level = Logger::FATAL
@api = Restfully::Session.new(:configuration_file => File.expand_path("~/.restfully/api.grid5000.fr.yml"),:logger => api_logger,:base_uri => 'https://api.grid5000.fr/sid')
@api_sites = if ENV['SITE']
[@api.root.sites[ENV['SITE'].to_sym]]
else
@api.root.sites
end
end
task :hosts do
# HOSTS=gw.lille
# HOSTS=*.lille
# SITES=lille => HOSTS=*.lille
# SITES=* => HOSTS=*.*
site = ENV['SITE']
host = ENV['HOST']
if site != nil
host = "*.#{site}"
elsif host != nil
abort "HOST must be on the form <hostname>.<site>. You provided '#{host}'." if host.scan(/^(\S+)\.(\S+)$/).empty?
else
abort "You must provide SITE= , (SITE=lille, or SITE=*), or a HOST, (HOST=gw.lille, or HOST=*.lille, or HOST=sw-*.lille)"
end
abort "You must provide HOST= , (HOST=gw.lille, or HOST=*.lille, or HOST=sw-*.lille)" if host.nil?
@host = host
end
namespace :g5k do
desc "Generates the JSON files based on the generators, for all sites.\nUse SITE=<SITE-NAME> if you wish to restrict the generation to a specific site.\nUse DRY=yes to simulate the execution."
task :generate => [:environment,:hosts] do
host,site = @host.scan(/(\S+)\.(\S+)/).flatten
root_dir_input = "#{ROOT_DIR}/generators/input/sites"
command = File.join(ROOT_DIR, "generators", "grid5000")
command += " " + File.join(root_dir_input, site,"#{site}.rb")
command += " " + File.join(root_dir_input, site,"clusters", "*", "#{host}.rb")
command += " " + File.join(root_dir_input, site,"clusters", "*", "nodes", "#{host}*.yaml")
command += " " + File.join(root_dir_input, site,"clusters", "*", "#{host}_manual.yaml")
command += " " + File.join(root_dir_input, site,"pdus.rb")
command << " -s" if ENV['DRY'] == "yes"
# puts command
sh command
end
end
# rake deadnodes:reasons
# rake deadnodes:tofix
namespace :deadnodes do
desc "List all dead nodes and the reason why they are dead. (SITE=)"
task :reasons => [:environment,:api_sites] do
@logger.level = Logger::INFO
@reasons = true
Rake::Task["deadnodes:browse"].execute
end
desc "List all nodes which have they state not in synch with they comment. (SITE=)"
task :tofix => [:environment,:api_sites] do
@logger.level = Logger::ERROR
@tofix = true
Rake::Task["deadnodes:browse"].execute
end
task :browse do
def comment_ok?(comment)
comment == "OK"
end
phoenix = []
@api_sites.each do |site|
reg = /^([^-]+)-(\d+)/
site.status["nodes"].sort{|a,b|
a_cluster,a_id = a[0].scan(reg).flatten
b_cluster,b_id = b[0].scan(reg).flatten
[a_cluster,a_id.to_i] <=> [b_cluster,b_id.to_i]
}.each do |uid,status|
comment = status["comment"]
state = status["hard"].downcase
if comment_ok?(comment)
if state == "dead"
@logger.error "Node '#{uid}' of state '#{state}' should not have comment '#{comment}'" if @tofix
else
# nothing, good state
end
else
if state == "dead"
@logger.info "Node '#{uid}' is dead because '#{comment}'" if @reasons
else
@logger.error "Node '#{uid}' should have the not-dead-comment 'OK', since its state is '#{state}'. Instead, it has comment '#{comment}'." if @tofix
phoenix.push uid if comment.match(/^\[phoenix\]/) != nil
end
end
end
end
puts phoenix if ENV["PHOENIX"] == "yes"
end
end
# TESTS
# Deletion:
# rake -s oar:generate FROM=4cfebf92e9cce05315782b51e05eded4ab4f0e7e TO=7d2648eaad7dbbc6f1fdb9c0279f73d374ccd47a
#
# Update:
# rake -s oar:generate FROM=7d2648eaad7dbbc6f1fdb9c0279f73d374ccd47a TO=bb528643003757942521942eaeab74b15aaa976d
#
# Add:
# rake -s oar:generate FROM=be9f7338b9750ce675447c13d172157992041ec1 TO=7dc3a4101a657230b7ad0534025a7ca93c905411
#
# All:
# rake -s oar:generate FROM=be9f7338b9750ce675447c13d172157992041ec1 TO=7d2648eaad7dbbc6f1fdb9c0279f73d374ccd47a
#
namespace :oar do
desc "Generates the oaradmin lines to update the OAR database after a change in the reference repository.\nUse FROM=<SHA-ID> and TO=<SHA-ID> to specify the starting and ending commits.\n Use -s to suppress the 'in directory' announcement.\n Use SUDO=sudo to prefix the commandes.\n use PROPS=<comma-separated-list> to limit the update to some properties"
task :generate => :environment do
if ENV['FROM'].nil? || ENV['FROM'].empty?
@logger.fatal "You MUST specify a commit id from where to start using the FROM=<SHA-ID> argument. Ex: rake -s oar:generate FROM=be9f7338b9750ce675447c13d172157992041ec1 TO=7dc3a4101a657230b7ad0534025a7ca93c905411 2> /dev/null"
exit(1)
end
ENV['TO'] ||= 'HEAD'
ENV['SUDO'] ||= ''
ENV['PROPS'] ||= ''
properties = ENV['PROPS'].split(',')
@logger.info "Analysing changes between #{ENV['FROM']}..#{ENV['TO']}..."
commands = []
diff = `git diff --name-status #{ENV['FROM']}..#{ENV['TO']}`
diff.split("\n").each do |line|
action, filename = line.split("\t")
next unless filename =~ %r{.+/nodes/.+json}
node_uid, site_uid, grid_uid = filename.gsub(/\.json/,'').split("/").values_at(-1, -5, -7)
cluster_uid = node_uid.split("-")[0]
host = [node_uid, site_uid, grid_uid, "fr"].flatten.join(".")
if ENV['SITE'] && !ENV['SITE'].split(",").include?(site_uid)
@logger.info "Skipping #{host} since you only want changes that occured on #{ENV['SITE'].inspect}"
next
end
command = ""
case action
when "A", "C", "M"
node_properties = JSON.parse(File.read(filename))
cluster_properties = JSON.parse(File.read(filename.gsub(%r{/nodes.*}, "/#{cluster_uid}.json")))
cluster = Grid5000::Cluster.new(cluster_properties)
node = Grid5000::Node.new(cluster, node_properties)
begin
export = node.export("oar-2.4")
rescue Grid5000::MissingProperty => e
@logger.warn "Error when exporting #{host}: #{e.message}. Skipped."
next
end
if action == "M" # modification of a file
command.concat("#{ENV["SUDO"]} oarnodesetting -h #{host} ")
command.concat(" -p ").concat( export.to_a.map{|(k,v)|
if v.nil?
nil
elsif properties.size > 0
if properties.include?(k)
"#{k}=#{v.inspect.gsub("'", "\\'").gsub("\"", "'")}"
else
nil
end
else
"#{k}=#{v.inspect.gsub("'", "\\'").gsub("\"", "'")}"
end
}.compact.join(" -p ") )
else # new file
node_number = node_uid.split("-")[1]
command.concat("oar_resources_add -H 1 --host0 #{node_number} --host-prefix #{cluster_uid}- --host-suffix .#{site_uid}.#{grid_uid}.fr -C #{node.properties['architecture']['smp_size']} -c #{export['cpucore']}")
command.concat(" -a")
# Add other properties
command.concat(' -A "')
if ENV['MAINTENANCE'] && ENV['MAINTENANCE']=='NO'
command.concat(' -p maintenance=\'NO\'')
else
# by default, maintenance is YES when creating new resources
command.concat(' -p maintenance=\'YES\'')
end
# by default, an Alive node has comment "OK"
command.concat(' -p comment=\'OK\'')
command.concat(" -p ").concat( export.to_a.map{|(k,v)|
if v.nil?
nil
else
"#{k}=#{v.inspect.gsub("'", "\\'").gsub("\"", "'")}"
end
}.compact.join(" -p ") )
command.concat('"')
end
when "D" # deletion of a file
command.concat("oarnodesetting -s Dead -h #{host}")
else
@logger.warn "Don't know what to do with #{line.inspect}. Ignoring."
next
end
if ENV['COMMIT'] && ENV['COMMIT']=='YES'
command.concat(' -c')
end
commands << command
end
commands.each do |command|
puts command
end
end
end
namespace :netlinks do
desc "Generates network API JSON files based on net-links yaml files.\nUse DRY=yes to simulate the execution. "
task :generate => [:environment,:hosts] do
host,site = @host.scan(/(\S+)\.(\S+)/).flatten
root_dir_input = File.join(ROOT_DIR, "generators","input")
command = File.join(ROOT_DIR, "generators", "grid5000")
command += " " + File.join(root_dir_input, "net-links.rb")
command += " " + File.join(root_dir_input,"sites", site,"#{site}.rb")
command += " " + File.join(root_dir_input,"sites", site,"net-links","#{host}.yaml")
command << " -s" if ENV['DRY'] == "yes"
# puts command
sh command
end
end
namespace :env do
desc "Generates environment JSON files .\nUse DRY=yes to simulate the execution. "
task :generate => [:environment] do
env_name = ENV["ENV_NAME"]
abort "You must provide ENV_NAME=" if env_name.nil?
root_dir_input = "#{ROOT_DIR}/generators/input"
command = File.join(ROOT_DIR, "generators", "grid5000")
command += " " + File.join(root_dir_input, "environments","#{env_name}")
command << " -s" if ENV['DRY'] == "yes"
sh command
end
end
namespace :weathermap do
@weathermap_options = ""
task :hosts do
@weathermap_site = ENV['SITE']
@weathermap_host = ENV['HOST']
abort "You must provide the SITE= " if @weathermap_site.nil?
abort "You must provide the HOST= name (uid) in its site " if @weathermap_host.nil? or @weathermap_host.match(/\.grid5000\.fr/) != nil
end
task :execute => ["weathermap:hosts"] do
cmd = "bundle exec weathermap"
cmd += " --site '#{@weathermap_site}' --host '#{@weathermap_host}' --api-path #{ROOT_DIR} #{@weathermap_options}"
sh cmd
end
desc "Create weathermaps for host HOST without data."
task :testing => ["weathermap:hosts"] do
@weathermap_options.replace("--action write --use-cacti no")
Rake::Task['weathermap:execute'].invoke
end
desc "Create weathermaps for host HOST with RRD from cacti."
task :production => ["weathermap:hosts"] do
@weathermap_options.replace("--action write --use-cacti yes")
Rake::Task['weathermap:execute'].invoke
end
desc "Display network links description amongst network equipments."
task :display do
@weathermap_options.replace("--action display")
Rake::Task['weathermap:execute'].invoke
end
end
#!/usr/bin/env ruby
require 'rubygems'
# Set up gems listed in the Gemfile.
gemfile = File.expand_path('../../Gemfile', __FILE__)
begin
ENV['BUNDLE_GEMFILE'] = gemfile
require 'bundler'
Bundler.setup
rescue LoadError => e
STDERR.puts e.message
STDERR.puts "Try installing bundler: `sudo gem install bundler`"
exit!
rescue Bundler::GemNotFound => e
STDERR.puts e.message
STDERR.puts "Try running `sudo bundle install`."
exit!
end
require 'pp'
require 'fileutils'
require 'json'
require 'yaml'
require 'time'
require 'optparse'
require File.dirname(__FILE__)+'/lib/core_extensions'
require File.dirname(__FILE__)+'/lib/g5k_generator'
options = {:simulate => false}
option_parser = OptionParser.new do |opts|
opts.banner = %{
A tool to generate the Grid5000 reference data.
Usage:
./grid5000 input_files [config_files] [options]
Notes:
* input_files MUST be .rb files.
* config_files MUST be .yml or .yaml files.
Examples:
./grid5000 input/*.rb input/*.yaml -s
./grid5000 input/nancy.rb input/nancy.yaml
./grid5000 input/rennes.rb input/lille.rb -s
Options:}
opts.on("-s", "--simulate", "Run in simulation mode (do not write changes). Default to false.") do |v|
options[:simulate] = v
end
opts.on_tail("-h", "--help", "Show this message") do
puts opts
exit
end
end
option_parser.parse!
files = ARGV
if files.empty?
$stderr.puts option_parser.help
exit 1
elsif (files.map{|file| File.exists?(file)}.include? false)
$stderr.puts "Error: one of your input or config file does not exist."
exit 2
else
input = {}
config = {}
files.each do |filename|
case (ext = File.extname(filename))
when ".rb"
# all sites have a pdus.rb file. Breaks generation if all sites are generated at the same time
input[File.basename(filename, ext)]="" unless input.has_key?(File.basename(filename, ext))
input[File.basename(filename, ext)] += File.read(filename)
when ".yml", ".yaml"
config[File.basename(filename, ext)] = YAML.load_file(filename)
else
raise ArgumentError, "The file #{filename} is neither an input file nor a config file."
end
end
puts "[Input files:\t\t #{input.keys.join(", ")}]"
puts "[Config files:\t\t #{config.keys.join(", ")}]"
puts "[Simulation mode:\t #{options[:simulate]}]"
generator = G5K::ReferenceGenerator.new({:uid => "grid5000", :type => "grid"}, :input => input, :config => config)
data = generator.generate
puts "[Add to the raw data for derived properties:\ #{true}]"
data = generator.annotate
directory_to_write = File.expand_path File.join(File.dirname(__FILE__), "../data")
generator.write(directory_to_write, options)
exit 0
end
# Parse net-links given through @config parameter
# @config parameter contains nothin but net-links yaml configurations
@config.each do |equipment_uid, properties|
properties = properties[equipment_uid]
site_name = properties["site"]
site site_name.to_sym do |site|
network_equipment equipment_uid do |uid|
# Most properties are directly written as they are from the net-links YAML file to the the JSON file
model properties["model"]
kind properties["kind"]
@context.recursive_merge!(:site => site)
snmp_community properties["snmp_community"]
weathermap properties["weathermap"] || {}
vlans properties["vlans"]
routes properties["routes"]
channels properties["channels"]
backplane_bps properties["backplane_bps"]
mtu properties["mtu"]
# Change the format of linecard from Hash to Array
linecards_array = []
properties["linecards"].each do |linecard_index,linecard|
ports = []
linecard.delete("ports").each do |port_index,port|
port = {"uid"=>port} if port.is_a? String
ports[port_index] = port
end
linecard["ports"] = ports.map{|p| p || {}}
linecards_array[linecard_index] = linecard
end
linecards linecards_array.map{|l| l || {}}
end
end
end
site :grenoble do |site_uid|
cluster :adonis do |cluster_uid|
model "Bull R422-E2 dual mobo + Tesla S1070"
created_at Time.parse("2010-09-02").httpdate
kavlan true
queues ['default', 'admin']
10.times do |i|
node "#{cluster_uid}-#{i+1}" do |node_uid|
performance({
:core_flops => 7139000000,
:node_flops => 55510000000
})
supported_job_types({
:deploy => true,
:besteffort => true,
:virtual => lookup(node_uid, node_uid, 'supported_job_types', 'virtual'),
:max_walltime => 0,
:queues => ['default', 'admin']
})
architecture({
:smp_size => lookup(node_uid, node_uid, 'architecture', 'smp_size'),
:smt_size => lookup(node_uid, node_uid, 'architecture', 'smt_size'),
:platform_type => lookup(node_uid, node_uid, 'architecture', 'platform_type')
})
processor({
:vendor => lookup(node_uid, node_uid, 'processor', 'vendor'),
:model => lookup(node_uid, node_uid, 'processor', 'model'),
:version => lookup(node_uid, node_uid, 'processor', 'version'),
:clock_speed => lookup(node_uid, node_uid, 'processor', 'clock_speed'),
:instruction_set => lookup(node_uid, node_uid, 'processor', 'instruction_set'),
:other_description => lookup(node_uid, node_uid, 'processor', 'other_description'),
:cache_l1 => lookup(node_uid, node_uid, 'processor', 'cache_l1'),
:cache_l1i => lookup(node_uid, node_uid, 'processor', 'cache_l1i'),
:cache_l1d => lookup(node_uid, node_uid, 'processor', 'cache_l1d'),
:cache_l2 => lookup(node_uid, node_uid, 'processor', 'cache_l2'),
:cache_l3 => lookup(node_uid, node_uid, 'processor', 'cache_l3')
})
main_memory({
:ram_size => lookup(node_uid, node_uid, 'main_memory', 'ram_size'),
:virtual_size => nil
})
operating_system({
:name => "debian",
:release => "Jessie",
:version => "8.2",
:kernel => "3.16.0-4-amd64"
})