Mentions légales du service

Skip to content
Snippets Groups Projects
Commit 7989e807 authored by Lucas Nussbaum's avatar Lucas Nussbaum
Browse files

[dev] move OAR properties checker to lib/

parent 9644a8a3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Rakefile
+ 23
− 3
View file @ 7989e807
......@@ -40,7 +40,7 @@ namespace :oar do
end
namespace :validators do
namespace :valid do
desc "Check homogeneity of clusters"
task "homogeneity" do
......@@ -51,10 +51,29 @@ namespace :validators do
task "schema" do
invoke_script "#{VALIDATORS_DIR}/yaml-input-schema-validator.rb"
end
desc "Check OAR properties. Parameters: [SITE={grenoble,...}] [CLUSTER={yeti,...}] [VERBOSE=1]"
task "oar-properties" do
options = {}
options[:verbose] = true if ENV['VERBOSE']
if ENV['SITE']
options[:sites] = ENV['SITE'].split(',')
else
options[:sites] = G5K_SITES
end
if ENV['CLUSTER']
options[:clusters] = ENV['CLUSTER'].split(',')
else
options[:clusters] = []
end
ret = RefRepo::Valid::OarProperties::check(options)
exit(ret)
end
end
namespace :gen do
desc "Run wiki generator. Parameter: NAME={hardware,site_hardware,...} SITE={global,grenoble,...} DO={diff,print,update}"
desc "Run wiki generator. Parameters: NAME={hardware,site_hardware,...} SITE={global,grenoble,...} DO={diff,print,update}"
task "wiki" do
options = {}
if ENV['SITE']
......@@ -81,7 +100,8 @@ namespace :gen do
puts "You must specify something to do using DO="
exit(1)
end
RefRepo::Gen::Wiki::wikigen(options)
ret = RefRepo::Gen::Wiki::wikigen(options)
exit(ret)
end
end
......
lib/refrepo.rb
+ 4
− 1
View file @ 7989e807
# pre-declare those modules here
module RefRepo
end
module RefRepo::Gen
end
module RefRepo::Valid
end
require 'refrepo/valid/oar-properties'
require 'refrepo/gen/wiki'
lib/refrepo/utils.rb 0 → 100644
+ 28
− 0
View file @ 7989e807
module RefRepo::Utils
def self.get_api_config
conf = ENV['HOME']+'/.grid5000_api.yml'
yconf = YAML::load(IO::read(conf)) rescue {}
yconf['uri'] ||= 'https://api.grid5000.fr/'
yconf['version'] ||= 'stable'
return yconf
end
def self.get_api(path)
conf = get_api_config
if conf['username'] and conf['password']
o = { :http_basic_authentication => [conf['username'], conf['password']] }
else
o = {}
end
d = open("#{conf['uri']}/#{conf['version']}/#{path}", o).read
return JSON::parse(d)
end
end
# Various monkey patches
class Hash
def slice(*extract)
h2 = self.select{|key, value| extract.include?(key) }
h2
end
end
lib/refrepo/valid/oar-properties.rb
+ 89
− 126
View file @ 7989e807
......@@ -13,6 +13,7 @@ require 'hashdiff'
require 'optparse'
require 'net/ssh'
require 'open-uri'
require 'refrepo/utils'
# propriétés ignorées
IGNORED_PROPERTIES=%w{chassis chunks thread}
......@@ -28,132 +29,94 @@ class Hash
end
end
def parse_command_line_parameters
options = {}
options[:sites] = %w(grenoble lille luxembourg lyon nancy nantes rennes sophia)
options[:clusters] = []
OptionParser.new do |opts|
opts.banner = 'Usage: oar-properties-check.rb [options]'
opts.separator ''
opts.separator 'Example: ruby oar-properties-check.rb -v -s nancy'
opts.separator ''
opts.separator 'Filters:'
opts.on('-s', '--sites a,b,c', Array, 'Select site(s)',
'Default: ' + options[:sites].join(', ')) do |s|
raise 'Wrong argument for -s option.' unless (s - options[:sites]).empty?
options[:sites] = s
end
opts.on('-c', '--clusters a,b,c', Array, 'Select clusters(s). Default: all') do |s|
options[:clusters] = s
end
opts.separator ''
opts.separator 'Common options:'
opts.on('-v', '--[no-]verbose', 'Run verbosely', 'Multiple -v options increase the verbosity. The maximum is 3.') do | |
options[:verbose] ||= 0
options[:verbose] = options[:verbose] + 1
end
# Print an options summary.
opts.on_tail('-h', '--help', 'Show this message') do
puts opts
exit
end
end.parse!
puts "Options: #{options}" if options[:verbose]
return options
end
ret = true
options = parse_command_line_parameters
options[:sites].each do |site|
puts "Checking site #{site}..."
resources = JSON::parse(open("https://api-proxy.nancy.grid5000.fr/3.0/sites/#{site}/internal/oarapi/resources/details.json?limit=1000000", {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read)['items']
default_resources = resources.select { |e| e['type'] == 'default' }.sort_by { |e| e['id'] }
# Checking scheduler_priority
default_resources.each do |r|
if r['scheduler_priority'] < 0
puts "Invalid scheduler_priority value on #{r['id']}/#{r['network_address']}: #{r['scheduler_priority']}"
ret = false
end
end
# Checking list of properties
names = default_resources.map { |e| e.keys.sort }.uniq.first - IGNORED_PROPERTIES
if names != G5K_PROPERTIES
puts "ERROR: wrong list of properties:"
ret = false
puts "- " + (G5K_PROPERTIES - names).join(' ')
puts "+ " + (names - G5K_PROPERTIES).join(' ')
end
# 'core' must be globally unique
dupe_cores = default_resources.map { |e| e.slice('id', 'core', 'host', 'cpu', 'cpuset') }.group_by { |e| e['core'] }.to_a.select { |e| e[1].length > 1 }
unless dupe_cores.empty?
puts "ERROR: some resources have the same 'core' value. it should be globally unique over the site."
ret = false
pp dupe_cores if options[:verbose]
end
# 'cpu' must be unique to a 'host'
dupe_cpus = default_resources.map { |e| [e['cpu'], e['host'] ]}.uniq.group_by { |e| e[0] }.to_a.select { |e| e[1].length > 1 }
unless dupe_cpus.empty?
puts "ERROR: some hosts have the same 'cpu' value. it should be globally unique over the site."
ret = false
pp dupe_cores if options[:verbose]
end
# for each host ...
default_resources.map { |e| e['host'] }.uniq.each do |host|
host_resources = default_resources.select { |e| e['host'] == host }
cluster = host_resources.first['cluster']
next if not options[:clusters].empty? and not options[:clusters].include?(cluster)
# compute nbcores.
# cpucore is cores per cpu. to know the number of cpus, we devide memnode per memcpu.
nbcores = host_resources.map { |e| e['cpucore'] * (e['memnode'] / e['memcpu']) }.uniq
if nbcores.length > 1
raise "Invalid: varying nbcores inside cluster!"
end
nbcores = nbcores.first
if host_resources.length != nbcores
puts "ERROR: invalid number of resources for #{host}. should be nbcores."
ret = false
end
# ids and cores should be in the same order
host_cores = host_resources.map { |e| e['core'] }
host_cores_min = host_cores.first
host_cores_max = host_cores.last
if host_cores_max - host_cores_min + 1 != nbcores
puts "ERROR: core values for #{host} are not sequential"
ret = false
end
# the first cpuset should be 0
host_cpusets = host_resources.map { |e| e['cpuset'] }.sort
host_cpusets_min = host_cpusets.first
host_cpusets_max = host_cpusets.last
if host_cpusets_min != 0
puts "ERROR: first cpuset value for #{host} should be 0"
ret = false
end
# the last cpuset should be nbcores-1
if host_cpusets_max - host_cpusets_min + 1 != nbcores
puts "ERROR: cpuset values for #{host} are not sequential"
ret = false
end
if options[:verbose] and (host_cpusets_max - host_cpusets_min + 1 != nbcores or host_cores_max - host_cores_min + 1 != nbcores)
puts "id cpu core cpuset"
pp host_resources.map { |e| [e['id'], e['cpu'], e['core'], e['cpuset'] ] }
module RefRepo::Valid::OarProperties
def self.check(options)
ret = true
options[:sites].each do |site|
puts "Checking site #{site}..."
resources = RefRepo::Utils::get_api("sites/#{site}/internal/oarapi/resources/details.json?limit=1000000")['items']
default_resources = resources.select { |e| e['type'] == 'default' }.sort_by { |e| e['id'] }
# Checking scheduler_priority
default_resources.each do |r|
if r['scheduler_priority'] < 0
puts "Invalid scheduler_priority value on #{r['id']}/#{r['network_address']}: #{r['scheduler_priority']}"
ret = false
end
end
# Checking list of properties
names = default_resources.map { |e| e.keys.sort }.uniq.first - IGNORED_PROPERTIES
if names != G5K_PROPERTIES
puts "ERROR: wrong list of properties:"
ret = false
puts "- " + (G5K_PROPERTIES - names).join(' ')
puts "+ " + (names - G5K_PROPERTIES).join(' ')
end
# 'core' must be globally unique
dupe_cores = default_resources.map { |e| e.slice('id', 'core', 'host', 'cpu', 'cpuset') }.group_by { |e| e['core'] }.to_a.select { |e| e[1].length > 1 }
unless dupe_cores.empty?
puts "ERROR: some resources have the same 'core' value. it should be globally unique over the site."
ret = false
pp dupe_cores if options[:verbose]
end
# 'cpu' must be unique to a 'host'
dupe_cpus = default_resources.map { |e| [e['cpu'], e['host'] ]}.uniq.group_by { |e| e[0] }.to_a.select { |e| e[1].length > 1 }
unless dupe_cpus.empty?
puts "ERROR: some hosts have the same 'cpu' value. it should be globally unique over the site."
ret = false
pp dupe_cores if options[:verbose]
end
# for each host ...
default_resources.map { |e| e['host'] }.uniq.each do |host|
host_resources = default_resources.select { |e| e['host'] == host }
cluster = host_resources.first['cluster']
next if not options[:clusters].empty? and not options[:clusters].include?(cluster)
# compute nbcores.
# cpucore is cores per cpu. to know the number of cpus, we devide memnode per memcpu.
nbcores = host_resources.map { |e| e['cpucore'] * (e['memnode'] / e['memcpu']) }.uniq
if nbcores.length > 1
raise "Invalid: varying nbcores inside cluster!"
end
nbcores = nbcores.first
if host_resources.length != nbcores
puts "ERROR: invalid number of resources for #{host}. should be nbcores."
ret = false
end
# ids and cores should be in the same order
host_cores = host_resources.map { |e| e['core'] }
host_cores_min = host_cores.first
host_cores_max = host_cores.last
if host_cores_max - host_cores_min + 1 != nbcores
puts "ERROR: core values for #{host} are not sequential"
ret = false
end
# the first cpuset should be 0
host_cpusets = host_resources.map { |e| e['cpuset'] }.sort
host_cpusets_min = host_cpusets.first
host_cpusets_max = host_cpusets.last
if host_cpusets_min != 0
puts "ERROR: first cpuset value for #{host} should be 0"
ret = false
end
# the last cpuset should be nbcores-1
if host_cpusets_max - host_cpusets_min + 1 != nbcores
puts "ERROR: cpuset values for #{host} are not sequential"
ret = false
end
if options[:verbose] and (host_cpusets_max - host_cpusets_min + 1 != nbcores or host_cores_max - host_cores_min + 1 != nbcores)
puts "id cpu core cpuset"
pp host_resources.map { |e| [e['id'], e['cpu'], e['core'], e['cpuset'] ] }
end
end
end
return ret
end
end
exit ret
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment