diff --git a/Rakefile b/Rakefile index 6b5d609f568bc2ca8a6557ec348f427cf351a64b..58565c8172a45e4039fc0db93f4b64c012b3f737 100644 --- a/Rakefile +++ b/Rakefile @@ -100,10 +100,12 @@ namespace :valid do end namespace :gen do - desc "Run wiki generator -- parameters: NAME={hardware,site_hardware,oar_properties,...} SITE={global,grenoble,...} DO={diff,print,update}" + desc "Run wiki generator -- parameters: NAME={hardware,site_hardware,oar_properties,...} SITE={global,grenoble,...} DO={diff,print,update} [CONFDIR=...]" task "wiki" do require 'refrepo/gen/wiki' options = {} + options[:conf_dir] = ENV['CONFDIR'] if ENV['CONFDIR'] + options[:output_dir] = PUPPET_ODIR options[:sites] = ( ENV['SITE'] ? ENV['SITE'].split(',') : ['global'] + G5K_SITES ) if ENV['NAME'] options[:generators] = ENV['NAME'].split(',') @@ -201,7 +203,7 @@ namespace :gen do namespace :puppet do - all_puppet_tasks = [:bindg5k, :conmang5k, :dhcpg5k, :kadeployg5k, :lanpowerg5k, :kavlang5k, :kwollectg5k, :network_monitoring, :'refapi-subset', :oxidizedg5k, :'sub-simplifier-aliases'] + all_puppet_tasks = [:bindg5k, :conmang5k, :dhcpg5k, :kadeployg5k, :lanpowerg5k, :kavlang5k, :kwollectg5k, :network_monitoring, :'refapi-subset', :oxidizedg5k, :'oarsub-simplifier-aliases'] all_puppet_tasks.each { |t| generated_desc = (t == :'refapi-subset') ? 'description' : 'configuration' diff --git a/lib/refrepo/gen/puppet/oarsub-simplifier-aliases.rb b/lib/refrepo/gen/puppet/oarsub-simplifier-aliases.rb index 54759e7f2cde67aeca15c5d7c734228b7eba1275..d63eb8b5c1bec07d46cc76f7b35616d32455b059 100644 --- a/lib/refrepo/gen/puppet/oarsub-simplifier-aliases.rb +++ b/lib/refrepo/gen/puppet/oarsub-simplifier-aliases.rb @@ -11,37 +11,60 @@ def get_sub_simplifier_default_aliases(options) raise("Error: file #{options[:conf_dir]}/aliases.yaml does not exist. The given configuration path is incorrect") unless Pathname("#{options[:conf_dir]}/aliases.yaml").exist? default_aliases = YAML.load(File.read("#{options[:conf_dir]}/aliases.yaml")) - default_aliases.merge!(GPURef.get_all_aliases.transform_values { |value| "gpu_model='#{value}'" }) + + gpu_aliases = GPURef.get_all_aliases.map do |al, model| + [al, {'value' => "gpu_model='#{model}'", + 'desc' => "Select node(s) with #{model} GPU", + 'category' => 'GPUs'}] + end.to_h + + default_aliases.merge!(gpu_aliases) + mem_aliases = {} mem_multipliers = [24, 32] mem_multipliers.each do |i| while i <= 1024 do - default_aliases["#{i}GB"] = "memnode>=#{i*1024}" + mem_aliases["#{i}GB"] = {} + mem_aliases["#{i}GB"]['value'] = "memnode>=#{i*1024}" + mem_aliases["#{i}GB"]['category'] = 'Memory' + mem_aliases["#{i}GB"]['desc'] = "Select node(s) with #{i}GB RAM or more" i *= 2 end end + mem_aliases = mem_aliases.sort_by { |key, _| key.scan(/\d+/).first.to_i }.to_h + default_aliases.merge!(mem_aliases) default_aliases end def generate_puppet_oarsub_simplifier_aliases(options) default_aliases = get_sub_simplifier_default_aliases(options) + sites_aliases = generate_all_sites_aliases options[:sites].each do |site| output_file = "#{options[:output_dir]}/platforms/production/modules/generated/files/grid5000/oar-sub-simplifier/#{site}-aliases.yaml" - gen_aliases_yaml(site, output_file, default_aliases) + generate_site_aliases_yaml(output_file, default_aliases, sites_aliases[site]) end end -def gen_aliases_yaml(site, output_path, default_aliases) +def generate_all_sites_aliases aliases = {} - aliases.merge!(default_aliases) - site_data_hierarchy = load_data_hierarchy - site_data_hierarchy['sites'][site]['clusters'].each_key do |cluster| - aliases[cluster] = "cluster='#{cluster}'" - aliases["#{cluster}-%d"] = "host='#{cluster}-%d.#{site}.grid5000.fr'" + site_data_hierarchy['sites'].each_key do |site| + aliases[site] = {} + site_data_hierarchy['sites'][site]['clusters'].each_key do |cluster| + aliases[site][cluster] = "cluster='#{cluster}'" + aliases[site]["#{cluster}-%d"] = "host='#{cluster}-%d.#{site}.grid5000.fr'" + end end + aliases +end + +def generate_site_aliases_yaml(output_path, default_aliases, site_aliases) + aliases = {} + aliases.merge!(default_aliases.map { |al, tr| [al, tr['value']] }.to_h) + aliases.merge!(site_aliases) + output_file = File.new(output_path, 'w') output_file.write(YAML.dump(aliases)) end diff --git a/lib/refrepo/gen/wiki.rb b/lib/refrepo/gen/wiki.rb index 192f836ff25bf82e18df331795c5a4615abecdf9..03582c6f639cfbe1d1294395cb2b854c8f7d5444 100644 --- a/lib/refrepo/gen/wiki.rb +++ b/lib/refrepo/gen/wiki.rb @@ -9,6 +9,7 @@ require 'refrepo/gen/wiki/generators/site_network' require 'refrepo/gen/wiki/generators/status' require 'refrepo/gen/wiki/generators/group_storage' require 'refrepo/gen/wiki/generators/kwollect_metrics' +require 'refrepo/gen/wiki/generators/oarsub_simplifier_aliases' module RefRepo::Gen::Wiki @@ -40,6 +41,10 @@ module RefRepo::Gen::Wiki 'kwollect_metrics' => { :gen => KwollectMetricsGenerator, :page => 'Generated/KwollectMetrics' + }, + 'oarsub_simplifier_aliases' => { + :gen => OarsubSimplifierAliasesGenerator, + :page => 'Generated/OarsubSimplifierAliases' } } SITE_GENERATORS = { diff --git a/lib/refrepo/gen/wiki/generators/cpu_parameters.rb b/lib/refrepo/gen/wiki/generators/cpu_parameters.rb index 550765aba45fb335babaf94632958ac28252fa6d..ea365a7c4f252c56fe00d1d2fd312baa4299df40 100644 --- a/lib/refrepo/gen/wiki/generators/cpu_parameters.rb +++ b/lib/refrepo/gen/wiki/generators/cpu_parameters.rb @@ -5,7 +5,7 @@ class CPUParametersGenerator < WikiGenerator super(page_name) end - def generate_content + def generate_content(_options) table_columns = ["Installation date", "Site", "Cluster", "CPU Family", "CPU Version", "Microarchitecture", "Frequency", "Server type", "HT enabled", "Turboboost enabled", "P-State driver", "C-State driver"] table_data = [] diff --git a/lib/refrepo/gen/wiki/generators/disk_reservation.rb b/lib/refrepo/gen/wiki/generators/disk_reservation.rb index 8489b70f41af7288403aa72a992033736ec7f61d..a3bc7a918a3fb2e5510b1f482fe780c032ec5953 100644 --- a/lib/refrepo/gen/wiki/generators/disk_reservation.rb +++ b/lib/refrepo/gen/wiki/generators/disk_reservation.rb @@ -6,7 +6,7 @@ class DiskReservationGenerator < WikiGenerator super(page_name) end - def generate_content + def generate_content(_options) table_columns = ["Site", "Cluster", "Number of nodes", "Number of reservable disks per node"] table_data = [] global_hash = get_global_hash diff --git a/lib/refrepo/gen/wiki/generators/group_storage.rb b/lib/refrepo/gen/wiki/generators/group_storage.rb index 9e6f9d6422cac7bae88a0f8db58ca8b9423c3e0d..5350889e8132d2547f3e7eadb458997e105450f3 100644 --- a/lib/refrepo/gen/wiki/generators/group_storage.rb +++ b/lib/refrepo/gen/wiki/generators/group_storage.rb @@ -6,7 +6,7 @@ class GroupStorageGenerator < WikiGenerator super(page_name) end - def generate_content + def generate_content(_options) table_columns = ["Site", "Server Name", "Size", "Link Speed", "Notes"] table_data = [] global_hash = get_global_hash diff --git a/lib/refrepo/gen/wiki/generators/hardware.rb b/lib/refrepo/gen/wiki/generators/hardware.rb index c662eaad11886b33f543b4359df70769036ca7cb..28f25ea6f88315aee793f410d9349b4af0825b67 100644 --- a/lib/refrepo/gen/wiki/generators/hardware.rb +++ b/lib/refrepo/gen/wiki/generators/hardware.rb @@ -7,7 +7,7 @@ class G5KHardwareGenerator < WikiGenerator super(page_name) end - def generate_content + def generate_content(_options) @global_hash = get_global_hash @site_uids = G5K::SITES diff --git a/lib/refrepo/gen/wiki/generators/kwollect_metrics.rb b/lib/refrepo/gen/wiki/generators/kwollect_metrics.rb index d0061e4da7296cfcb7fa44efb6e9e4d0f8544738..3ba3532cd8cc0f844380544d68f785d9ea4afe8f 100644 --- a/lib/refrepo/gen/wiki/generators/kwollect_metrics.rb +++ b/lib/refrepo/gen/wiki/generators/kwollect_metrics.rb @@ -6,7 +6,7 @@ class KwollectMetricsGenerator < WikiGenerator super(page_name) end - def generate_content + def generate_content(_options) @generated_content = "__NOEDITSECTION__\n" @generated_content = "\nMetrics marked with * must be activated on demand, and metrics marked with ** are activated on non-deploy jobs by default.\n\n" @generated_content += "{|class=\"wikitable\"\n" diff --git a/lib/refrepo/gen/wiki/generators/oar_properties.rb b/lib/refrepo/gen/wiki/generators/oar_properties.rb index c4976e740c2cdc06004567083d8d9af50e9dc184..3f77b51d35aff8ca912bbe08d8e58844045703ae 100644 --- a/lib/refrepo/gen/wiki/generators/oar_properties.rb +++ b/lib/refrepo/gen/wiki/generators/oar_properties.rb @@ -264,7 +264,7 @@ class OarPropertiesGenerator < WikiGenerator "String" end - def generate_content + def generate_content(_options) refapi = load_data_hierarchy #Properties generated from oar-properties generator props = {} diff --git a/lib/refrepo/gen/wiki/generators/oarsub_simplifier_aliases.rb b/lib/refrepo/gen/wiki/generators/oarsub_simplifier_aliases.rb new file mode 100644 index 0000000000000000000000000000000000000000..ebcd89264a601b315e017231fead71f152eace04 --- /dev/null +++ b/lib/refrepo/gen/wiki/generators/oarsub_simplifier_aliases.rb @@ -0,0 +1,37 @@ +# coding: utf-8 + +require 'refrepo/gen/puppet/oarsub-simplifier-aliases' + +class OarsubSimplifierAliasesGenerator < WikiGenerator + def initialize(page_name) + super(page_name) + end + + def generate_content(options) + default_aliases = get_sub_simplifier_default_aliases(options) + default_aliases_by_category = {} + default_aliases.each do |al, data| + default_aliases_by_category[data['category']] ||= {} + default_aliases_by_category[data['category']][al] = data + end + + @generated_content = "__NOEDITSECTION__\n" + default_aliases_by_category.sort.each do |cat, data| + @generated_content += "=== #{cat} ===\n" + @generated_content += "{|class=\"wikitable\"\n" + @generated_content += "! style=\"width: 15%\" | Alias\n" + @generated_content += "! style=\"width: 30%\" | Description\n" + @generated_content += "! style=\"width: 30%\" | Translate to\n" + + data.each do |al, c| + @generated_content += "|-\n" + @generated_content += "|#{al}\n" + @generated_content += "|#{c['desc']}\n" + @generated_content += "|<code>#{c['value']}</code>\n" + end + + @generated_content += "|}\n" + @generated_content += "\n" + end + end +end diff --git a/lib/refrepo/gen/wiki/generators/site_hardware.rb b/lib/refrepo/gen/wiki/generators/site_hardware.rb index 3445690bd46010e58323d4f24a80c63e05fdfaa8..992b8517783750f15a27b91616362dd1cda816e0 100644 --- a/lib/refrepo/gen/wiki/generators/site_hardware.rb +++ b/lib/refrepo/gen/wiki/generators/site_hardware.rb @@ -9,7 +9,7 @@ class SiteHardwareGenerator < WikiGenerator @site = site end - def generate_content + def generate_content(_options) has_reservable_disks = false G5K::get_global_hash['sites'][@site]['clusters'].each do |_,c| c['nodes'].each do |_,n| diff --git a/lib/refrepo/gen/wiki/generators/site_network.rb b/lib/refrepo/gen/wiki/generators/site_network.rb index 2c8c50eb9c3f70ea53a4f3d09be1db7c100aaa92..44b601bcda7db4e8e3136631f793aa7f1df9c0e5 100644 --- a/lib/refrepo/gen/wiki/generators/site_network.rb +++ b/lib/refrepo/gen/wiki/generators/site_network.rb @@ -26,7 +26,7 @@ class SiteNetworkGenerator < WikiGenerator return h['network_equipments'].to_a.map { |e| "* #{e[0]}: #{e[1]['model']}" }.sort.join("\n") end - def generate_content + def generate_content(_options) @generated_content = "__NOTOC__\n__NOEDITSECTION__\n" @generated_content += "= Network devices models =\n" @generated_content += generate_equipments + "\n" diff --git a/lib/refrepo/gen/wiki/generators/status.rb b/lib/refrepo/gen/wiki/generators/status.rb index 8e44d75e89043033e853b079cb61fda1bfb8aa02..5ae0d4c3c0252ebb77befa542a1cf56580c0aae5 100644 --- a/lib/refrepo/gen/wiki/generators/status.rb +++ b/lib/refrepo/gen/wiki/generators/status.rb @@ -6,7 +6,7 @@ class StatusGenerator < WikiGenerator super(page_name) end - def generate_content + def generate_content(_options) @global_hash = get_global_hash @site_uids = G5K::SITES diff --git a/lib/refrepo/gen/wiki/wiki_generator.rb b/lib/refrepo/gen/wiki/wiki_generator.rb index 96731a8fc1940e3885a99a1536820fb71fa272e1..fa6c54f0c298c11284424e98feff61f9a40a71ee 100644 --- a/lib/refrepo/gen/wiki/wiki_generator.rb +++ b/lib/refrepo/gen/wiki/wiki_generator.rb @@ -29,7 +29,7 @@ class WikiGenerator end end - def generate_content + def generate_content(_options) raise "To be implemented in actual generators" end @@ -114,7 +114,7 @@ class WikiGenerator #Execute actions on generator based on given options def exec(options) - generate_content() + generate_content(options) ret = true #Login only if we need to