Mentions légales du service

Skip to content
Snippets Groups Projects
Commit 7989e807 authored by Lucas Nussbaum's avatar Lucas Nussbaum
Browse files

[dev] move OAR properties checker to lib/

parent 9644a8a3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Rakefile
+ 23
− 3
View file @ 7989e807
...@@ -40,7 +40,7 @@ namespace :oar do ...@@ -40,7 +40,7 @@ namespace :oar do
end end
namespace :validators do namespace :valid do
desc "Check homogeneity of clusters" desc "Check homogeneity of clusters"
task "homogeneity" do task "homogeneity" do
...@@ -51,10 +51,29 @@ namespace :validators do ...@@ -51,10 +51,29 @@ namespace :validators do
task "schema" do task "schema" do
invoke_script "#{VALIDATORS_DIR}/yaml-input-schema-validator.rb" invoke_script "#{VALIDATORS_DIR}/yaml-input-schema-validator.rb"
end end
desc "Check OAR properties. Parameters: [SITE={grenoble,...}] [CLUSTER={yeti,...}] [VERBOSE=1]"
task "oar-properties" do
options = {}
options[:verbose] = true if ENV['VERBOSE']
if ENV['SITE']
options[:sites] = ENV['SITE'].split(',')
else
options[:sites] = G5K_SITES
end
if ENV['CLUSTER']
options[:clusters] = ENV['CLUSTER'].split(',')
else
options[:clusters] = []
end
ret = RefRepo::Valid::OarProperties::check(options)
exit(ret)
end
end end
namespace :gen do namespace :gen do
desc "Run wiki generator. Parameter: NAME={hardware,site_hardware,...} SITE={global,grenoble,...} DO={diff,print,update}" desc "Run wiki generator. Parameters: NAME={hardware,site_hardware,...} SITE={global,grenoble,...} DO={diff,print,update}"
task "wiki" do task "wiki" do
options = {} options = {}
if ENV['SITE'] if ENV['SITE']
...@@ -81,7 +100,8 @@ namespace :gen do ...@@ -81,7 +100,8 @@ namespace :gen do
puts "You must specify something to do using DO=" puts "You must specify something to do using DO="
exit(1) exit(1)
end end
RefRepo::Gen::Wiki::wikigen(options) ret = RefRepo::Gen::Wiki::wikigen(options)
exit(ret)
end end
end end
......
lib/refrepo.rb
+ 4
− 1
View file @ 7989e807
# pre-declare those modules here
module RefRepo module RefRepo
end end
module RefRepo::Gen module RefRepo::Gen
end end
module RefRepo::Valid
end
require 'refrepo/valid/oar-properties'
require 'refrepo/gen/wiki' require 'refrepo/gen/wiki'
lib/refrepo/utils.rb 0 → 100644
+ 28
− 0
View file @ 7989e807
module RefRepo::Utils
def self.get_api_config
conf = ENV['HOME']+'/.grid5000_api.yml'
yconf = YAML::load(IO::read(conf)) rescue {}
yconf['uri'] ||= 'https://api.grid5000.fr/'
yconf['version'] ||= 'stable'
return yconf
end
def self.get_api(path)
conf = get_api_config
if conf['username'] and conf['password']
o = { :http_basic_authentication => [conf['username'], conf['password']] }
else
o = {}
end
d = open("#{conf['uri']}/#{conf['version']}/#{path}", o).read
return JSON::parse(d)
end
end
# Various monkey patches
class Hash
def slice(*extract)
h2 = self.select{|key, value| extract.include?(key) }
h2
end
end
lib/refrepo/valid/oar-properties.rb
+ 89
− 126
View file @ 7989e807
...@@ -13,6 +13,7 @@ require 'hashdiff' ...@@ -13,6 +13,7 @@ require 'hashdiff'
require 'optparse' require 'optparse'
require 'net/ssh' require 'net/ssh'
require 'open-uri' require 'open-uri'
require 'refrepo/utils'
# propriétés ignorées # propriétés ignorées
IGNORED_PROPERTIES=%w{chassis chunks thread} IGNORED_PROPERTIES=%w{chassis chunks thread}
...@@ -28,132 +29,94 @@ class Hash ...@@ -28,132 +29,94 @@ class Hash
end end
end end
def parse_command_line_parameters module RefRepo::Valid::OarProperties
options = {} def self.check(options)
options[:sites] = %w(grenoble lille luxembourg lyon nancy nantes rennes sophia) ret = true
options[:clusters] = [] options[:sites].each do |site|
puts "Checking site #{site}..."
OptionParser.new do |opts| resources = RefRepo::Utils::get_api("sites/#{site}/internal/oarapi/resources/details.json?limit=1000000")['items']
opts.banner = 'Usage: oar-properties-check.rb [options]'
opts.separator '' default_resources = resources.select { |e| e['type'] == 'default' }.sort_by { |e| e['id'] }
opts.separator 'Example: ruby oar-properties-check.rb -v -s nancy'
opts.separator '' # Checking scheduler_priority
default_resources.each do |r|
opts.separator 'Filters:' if r['scheduler_priority'] < 0
opts.on('-s', '--sites a,b,c', Array, 'Select site(s)', puts "Invalid scheduler_priority value on #{r['id']}/#{r['network_address']}: #{r['scheduler_priority']}"
'Default: ' + options[:sites].join(', ')) do |s| ret = false
raise 'Wrong argument for -s option.' unless (s - options[:sites]).empty? end
options[:sites] = s end
end
opts.on('-c', '--clusters a,b,c', Array, 'Select clusters(s). Default: all') do |s| # Checking list of properties
options[:clusters] = s names = default_resources.map { |e| e.keys.sort }.uniq.first - IGNORED_PROPERTIES
end if names != G5K_PROPERTIES
opts.separator '' puts "ERROR: wrong list of properties:"
opts.separator 'Common options:' ret = false
opts.on('-v', '--[no-]verbose', 'Run verbosely', 'Multiple -v options increase the verbosity. The maximum is 3.') do | | puts "- " + (G5K_PROPERTIES - names).join(' ')
options[:verbose] ||= 0 puts "+ " + (names - G5K_PROPERTIES).join(' ')
options[:verbose] = options[:verbose] + 1 end
end
# 'core' must be globally unique
# Print an options summary. dupe_cores = default_resources.map { |e| e.slice('id', 'core', 'host', 'cpu', 'cpuset') }.group_by { |e| e['core'] }.to_a.select { |e| e[1].length > 1 }
opts.on_tail('-h', '--help', 'Show this message') do unless dupe_cores.empty?
puts opts puts "ERROR: some resources have the same 'core' value. it should be globally unique over the site."
exit ret = false
end pp dupe_cores if options[:verbose]
end.parse! end
puts "Options: #{options}" if options[:verbose] # 'cpu' must be unique to a 'host'
dupe_cpus = default_resources.map { |e| [e['cpu'], e['host'] ]}.uniq.group_by { |e| e[0] }.to_a.select { |e| e[1].length > 1 }
return options unless dupe_cpus.empty?
end puts "ERROR: some hosts have the same 'cpu' value. it should be globally unique over the site."
ret = false
ret = true pp dupe_cores if options[:verbose]
options = parse_command_line_parameters end
options[:sites].each do |site| # for each host ...
puts "Checking site #{site}..." default_resources.map { |e| e['host'] }.uniq.each do |host|
resources = JSON::parse(open("https://api-proxy.nancy.grid5000.fr/3.0/sites/#{site}/internal/oarapi/resources/details.json?limit=1000000", {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read)['items'] host_resources = default_resources.select { |e| e['host'] == host }
cluster = host_resources.first['cluster']
default_resources = resources.select { |e| e['type'] == 'default' }.sort_by { |e| e['id'] } next if not options[:clusters].empty? and not options[:clusters].include?(cluster)
# Checking scheduler_priority # compute nbcores.
default_resources.each do |r| # cpucore is cores per cpu. to know the number of cpus, we devide memnode per memcpu.
if r['scheduler_priority'] < 0 nbcores = host_resources.map { |e| e['cpucore'] * (e['memnode'] / e['memcpu']) }.uniq
puts "Invalid scheduler_priority value on #{r['id']}/#{r['network_address']}: #{r['scheduler_priority']}" if nbcores.length > 1
ret = false raise "Invalid: varying nbcores inside cluster!"
end end
end nbcores = nbcores.first
# Checking list of properties if host_resources.length != nbcores
names = default_resources.map { |e| e.keys.sort }.uniq.first - IGNORED_PROPERTIES puts "ERROR: invalid number of resources for #{host}. should be nbcores."
if names != G5K_PROPERTIES ret = false
puts "ERROR: wrong list of properties:" end
ret = false
puts "- " + (G5K_PROPERTIES - names).join(' ') # ids and cores should be in the same order
puts "+ " + (names - G5K_PROPERTIES).join(' ') host_cores = host_resources.map { |e| e['core'] }
end host_cores_min = host_cores.first
host_cores_max = host_cores.last
# 'core' must be globally unique if host_cores_max - host_cores_min + 1 != nbcores
dupe_cores = default_resources.map { |e| e.slice('id', 'core', 'host', 'cpu', 'cpuset') }.group_by { |e| e['core'] }.to_a.select { |e| e[1].length > 1 } puts "ERROR: core values for #{host} are not sequential"
unless dupe_cores.empty? ret = false
puts "ERROR: some resources have the same 'core' value. it should be globally unique over the site." end
ret = false # the first cpuset should be 0
pp dupe_cores if options[:verbose] host_cpusets = host_resources.map { |e| e['cpuset'] }.sort
end host_cpusets_min = host_cpusets.first
host_cpusets_max = host_cpusets.last
# 'cpu' must be unique to a 'host' if host_cpusets_min != 0
dupe_cpus = default_resources.map { |e| [e['cpu'], e['host'] ]}.uniq.group_by { |e| e[0] }.to_a.select { |e| e[1].length > 1 } puts "ERROR: first cpuset value for #{host} should be 0"
unless dupe_cpus.empty? ret = false
puts "ERROR: some hosts have the same 'cpu' value. it should be globally unique over the site." end
ret = false # the last cpuset should be nbcores-1
pp dupe_cores if options[:verbose] if host_cpusets_max - host_cpusets_min + 1 != nbcores
end puts "ERROR: cpuset values for #{host} are not sequential"
ret = false
# for each host ... end
default_resources.map { |e| e['host'] }.uniq.each do |host| if options[:verbose] and (host_cpusets_max - host_cpusets_min + 1 != nbcores or host_cores_max - host_cores_min + 1 != nbcores)
host_resources = default_resources.select { |e| e['host'] == host } puts "id cpu core cpuset"
cluster = host_resources.first['cluster'] pp host_resources.map { |e| [e['id'], e['cpu'], e['core'], e['cpuset'] ] }
next if not options[:clusters].empty? and not options[:clusters].include?(cluster) end
end
# compute nbcores.
# cpucore is cores per cpu. to know the number of cpus, we devide memnode per memcpu.
nbcores = host_resources.map { |e| e['cpucore'] * (e['memnode'] / e['memcpu']) }.uniq
if nbcores.length > 1
raise "Invalid: varying nbcores inside cluster!"
end
nbcores = nbcores.first
if host_resources.length != nbcores
puts "ERROR: invalid number of resources for #{host}. should be nbcores."
ret = false
end
# ids and cores should be in the same order
host_cores = host_resources.map { |e| e['core'] }
host_cores_min = host_cores.first
host_cores_max = host_cores.last
if host_cores_max - host_cores_min + 1 != nbcores
puts "ERROR: core values for #{host} are not sequential"
ret = false
end
# the first cpuset should be 0
host_cpusets = host_resources.map { |e| e['cpuset'] }.sort
host_cpusets_min = host_cpusets.first
host_cpusets_max = host_cpusets.last
if host_cpusets_min != 0
puts "ERROR: first cpuset value for #{host} should be 0"
ret = false
end
# the last cpuset should be nbcores-1
if host_cpusets_max - host_cpusets_min + 1 != nbcores
puts "ERROR: cpuset values for #{host} are not sequential"
ret = false
end
if options[:verbose] and (host_cpusets_max - host_cpusets_min + 1 != nbcores or host_cores_max - host_cores_min + 1 != nbcores)
puts "id cpu core cpuset"
pp host_resources.map { |e| [e['id'], e['cpu'], e['core'], e['cpuset'] ] }
end end
return ret
end end
end end
exit ret
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment