From 743d94f3dd0498d429f80573521f30feab341261 Mon Sep 17 00:00:00 2001
From: Samir Noir <samir.noir@inria.fr>
Date: Wed, 12 Jan 2022 17:31:40 +0100
Subject: [PATCH] [gen/wiki] add oarsub_simplifier_aliases generator
---
Rakefile | 6 +-
.../gen/puppet/oarsub-simplifier-aliases.rb | 41 +++++++++---
lib/refrepo/gen/wiki.rb | 5 ++
.../gen/wiki/generators/cpu_parameters.rb | 2 +-
.../gen/wiki/generators/disk_reservation.rb | 2 +-
.../gen/wiki/generators/group_storage.rb | 2 +-
lib/refrepo/gen/wiki/generators/hardware.rb | 2 +-
.../gen/wiki/generators/kwollect_metrics.rb | 2 +-
.../gen/wiki/generators/oar_properties.rb | 2 +-
.../generators/oarsub_simplifier_aliases.rb | 65 +++++++++++++++++++
.../gen/wiki/generators/site_hardware.rb | 2 +-
.../gen/wiki/generators/site_network.rb | 2 +-
lib/refrepo/gen/wiki/generators/status.rb | 2 +-
lib/refrepo/gen/wiki/wiki_generator.rb | 4 +-
14 files changed, 117 insertions(+), 22 deletions(-)
create mode 100644 lib/refrepo/gen/wiki/generators/oarsub_simplifier_aliases.rb
diff --git a/Rakefile b/Rakefile
index 6b5d609f568..58565c8172a 100644
--- a/Rakefile
+++ b/Rakefile
@@ -100,10 +100,12 @@ namespace :valid do
end
namespace :gen do
- desc "Run wiki generator -- parameters: NAME={hardware,site_hardware,oar_properties,...} SITE={global,grenoble,...} DO={diff,print,update}"
+ desc "Run wiki generator -- parameters: NAME={hardware,site_hardware,oar_properties,...} SITE={global,grenoble,...} DO={diff,print,update} [CONFDIR=...]"
task "wiki" do
require 'refrepo/gen/wiki'
options = {}
+ options[:conf_dir] = ENV['CONFDIR'] if ENV['CONFDIR']
+ options[:output_dir] = PUPPET_ODIR
options[:sites] = ( ENV['SITE'] ? ENV['SITE'].split(',') : ['global'] + G5K_SITES )
if ENV['NAME']
options[:generators] = ENV['NAME'].split(',')
@@ -201,7 +203,7 @@ namespace :gen do
namespace :puppet do
- all_puppet_tasks = [:bindg5k, :conmang5k, :dhcpg5k, :kadeployg5k, :lanpowerg5k, :kavlang5k, :kwollectg5k, :network_monitoring, :'refapi-subset', :oxidizedg5k, :'sub-simplifier-aliases']
+ all_puppet_tasks = [:bindg5k, :conmang5k, :dhcpg5k, :kadeployg5k, :lanpowerg5k, :kavlang5k, :kwollectg5k, :network_monitoring, :'refapi-subset', :oxidizedg5k, :'oarsub-simplifier-aliases']
all_puppet_tasks.each { |t|
generated_desc = (t == :'refapi-subset') ? 'description' : 'configuration'
diff --git a/lib/refrepo/gen/puppet/oarsub-simplifier-aliases.rb b/lib/refrepo/gen/puppet/oarsub-simplifier-aliases.rb
index 54759e7f2cd..d63eb8b5c1b 100644
--- a/lib/refrepo/gen/puppet/oarsub-simplifier-aliases.rb
+++ b/lib/refrepo/gen/puppet/oarsub-simplifier-aliases.rb
@@ -11,37 +11,60 @@ def get_sub_simplifier_default_aliases(options)
raise("Error: file #{options[:conf_dir]}/aliases.yaml does not exist. The given configuration path is incorrect") unless Pathname("#{options[:conf_dir]}/aliases.yaml").exist?
default_aliases = YAML.load(File.read("#{options[:conf_dir]}/aliases.yaml"))
- default_aliases.merge!(GPURef.get_all_aliases.transform_values { |value| "gpu_model='#{value}'" })
+
+ gpu_aliases = GPURef.get_all_aliases.map do |al, model|
+ [al, {'value' => "gpu_model='#{model}'",
+ 'desc' => "Select node(s) with #{model} GPU",
+ 'category' => 'GPUs'}]
+ end.to_h
+
+ default_aliases.merge!(gpu_aliases)
+ mem_aliases = {}
mem_multipliers = [24, 32]
mem_multipliers.each do |i|
while i <= 1024 do
- default_aliases["#{i}GB"] = "memnode>=#{i*1024}"
+ mem_aliases["#{i}GB"] = {}
+ mem_aliases["#{i}GB"]['value'] = "memnode>=#{i*1024}"
+ mem_aliases["#{i}GB"]['category'] = 'Memory'
+ mem_aliases["#{i}GB"]['desc'] = "Select node(s) with #{i}GB RAM or more"
i *= 2
end
end
+ mem_aliases = mem_aliases.sort_by { |key, _| key.scan(/\d+/).first.to_i }.to_h
+ default_aliases.merge!(mem_aliases)
default_aliases
end
def generate_puppet_oarsub_simplifier_aliases(options)
default_aliases = get_sub_simplifier_default_aliases(options)
+ sites_aliases = generate_all_sites_aliases
options[:sites].each do |site|
output_file = "#{options[:output_dir]}/platforms/production/modules/generated/files/grid5000/oar-sub-simplifier/#{site}-aliases.yaml"
- gen_aliases_yaml(site, output_file, default_aliases)
+ generate_site_aliases_yaml(output_file, default_aliases, sites_aliases[site])
end
end
-def gen_aliases_yaml(site, output_path, default_aliases)
+def generate_all_sites_aliases
aliases = {}
- aliases.merge!(default_aliases)
-
site_data_hierarchy = load_data_hierarchy
- site_data_hierarchy['sites'][site]['clusters'].each_key do |cluster|
- aliases[cluster] = "cluster='#{cluster}'"
- aliases["#{cluster}-%d"] = "host='#{cluster}-%d.#{site}.grid5000.fr'"
+ site_data_hierarchy['sites'].each_key do |site|
+ aliases[site] = {}
+ site_data_hierarchy['sites'][site]['clusters'].each_key do |cluster|
+ aliases[site][cluster] = "cluster='#{cluster}'"
+ aliases[site]["#{cluster}-%d"] = "host='#{cluster}-%d.#{site}.grid5000.fr'"
+ end
end
+ aliases
+end
+
+def generate_site_aliases_yaml(output_path, default_aliases, site_aliases)
+ aliases = {}
+ aliases.merge!(default_aliases.map { |al, tr| [al, tr['value']] }.to_h)
+ aliases.merge!(site_aliases)
+
output_file = File.new(output_path, 'w')
output_file.write(YAML.dump(aliases))
end
diff --git a/lib/refrepo/gen/wiki.rb b/lib/refrepo/gen/wiki.rb
index 192f836ff25..03582c6f639 100644
--- a/lib/refrepo/gen/wiki.rb
+++ b/lib/refrepo/gen/wiki.rb
@@ -9,6 +9,7 @@ require 'refrepo/gen/wiki/generators/site_network'
require 'refrepo/gen/wiki/generators/status'
require 'refrepo/gen/wiki/generators/group_storage'
require 'refrepo/gen/wiki/generators/kwollect_metrics'
+require 'refrepo/gen/wiki/generators/oarsub_simplifier_aliases'
module RefRepo::Gen::Wiki
@@ -40,6 +41,10 @@ module RefRepo::Gen::Wiki
'kwollect_metrics' => {
:gen => KwollectMetricsGenerator,
:page => 'Generated/KwollectMetrics'
+ },
+ 'oarsub_simplifier_aliases' => {
+ :gen => OarsubSimplifierAliasesGenerator,
+ :page => 'Generated/OarsubSimplifierAliases'
}
}
SITE_GENERATORS = {
diff --git a/lib/refrepo/gen/wiki/generators/cpu_parameters.rb b/lib/refrepo/gen/wiki/generators/cpu_parameters.rb
index 550765aba45..ea365a7c4f2 100644
--- a/lib/refrepo/gen/wiki/generators/cpu_parameters.rb
+++ b/lib/refrepo/gen/wiki/generators/cpu_parameters.rb
@@ -5,7 +5,7 @@ class CPUParametersGenerator < WikiGenerator
super(page_name)
end
- def generate_content
+ def generate_content(_options)
table_columns = ["Installation date", "Site", "Cluster", "CPU Family", "CPU Version", "Microarchitecture", "Frequency", "Server type", "HT enabled", "Turboboost enabled", "P-State driver", "C-State driver"]
table_data = []
diff --git a/lib/refrepo/gen/wiki/generators/disk_reservation.rb b/lib/refrepo/gen/wiki/generators/disk_reservation.rb
index 8489b70f41a..a3bc7a918a3 100644
--- a/lib/refrepo/gen/wiki/generators/disk_reservation.rb
+++ b/lib/refrepo/gen/wiki/generators/disk_reservation.rb
@@ -6,7 +6,7 @@ class DiskReservationGenerator < WikiGenerator
super(page_name)
end
- def generate_content
+ def generate_content(_options)
table_columns = ["Site", "Cluster", "Number of nodes", "Number of reservable disks per node"]
table_data = []
global_hash = get_global_hash
diff --git a/lib/refrepo/gen/wiki/generators/group_storage.rb b/lib/refrepo/gen/wiki/generators/group_storage.rb
index 9e6f9d6422c..5350889e813 100644
--- a/lib/refrepo/gen/wiki/generators/group_storage.rb
+++ b/lib/refrepo/gen/wiki/generators/group_storage.rb
@@ -6,7 +6,7 @@ class GroupStorageGenerator < WikiGenerator
super(page_name)
end
- def generate_content
+ def generate_content(_options)
table_columns = ["Site", "Server Name", "Size", "Link Speed", "Notes"]
table_data = []
global_hash = get_global_hash
diff --git a/lib/refrepo/gen/wiki/generators/hardware.rb b/lib/refrepo/gen/wiki/generators/hardware.rb
index c662eaad118..28f25ea6f88 100644
--- a/lib/refrepo/gen/wiki/generators/hardware.rb
+++ b/lib/refrepo/gen/wiki/generators/hardware.rb
@@ -7,7 +7,7 @@ class G5KHardwareGenerator < WikiGenerator
super(page_name)
end
- def generate_content
+ def generate_content(_options)
@global_hash = get_global_hash
@site_uids = G5K::SITES
diff --git a/lib/refrepo/gen/wiki/generators/kwollect_metrics.rb b/lib/refrepo/gen/wiki/generators/kwollect_metrics.rb
index d0061e4da72..3ba3532cd8c 100644
--- a/lib/refrepo/gen/wiki/generators/kwollect_metrics.rb
+++ b/lib/refrepo/gen/wiki/generators/kwollect_metrics.rb
@@ -6,7 +6,7 @@ class KwollectMetricsGenerator < WikiGenerator
super(page_name)
end
- def generate_content
+ def generate_content(_options)
@generated_content = "__NOEDITSECTION__\n"
@generated_content = "\nMetrics marked with * must be activated on demand, and metrics marked with ** are activated on non-deploy jobs by default.\n\n"
@generated_content += "{|class=\"wikitable\"\n"
diff --git a/lib/refrepo/gen/wiki/generators/oar_properties.rb b/lib/refrepo/gen/wiki/generators/oar_properties.rb
index c4976e740c2..3f77b51d35a 100644
--- a/lib/refrepo/gen/wiki/generators/oar_properties.rb
+++ b/lib/refrepo/gen/wiki/generators/oar_properties.rb
@@ -264,7 +264,7 @@ class OarPropertiesGenerator < WikiGenerator
"String"
end
- def generate_content
+ def generate_content(_options)
refapi = load_data_hierarchy
#Properties generated from oar-properties generator
props = {}
diff --git a/lib/refrepo/gen/wiki/generators/oarsub_simplifier_aliases.rb b/lib/refrepo/gen/wiki/generators/oarsub_simplifier_aliases.rb
new file mode 100644
index 00000000000..3921295d22c
--- /dev/null
+++ b/lib/refrepo/gen/wiki/generators/oarsub_simplifier_aliases.rb
@@ -0,0 +1,65 @@
+# coding: utf-8
+
+require 'refrepo/gen/puppet/oarsub-simplifier-aliases'
+
+class OarsubSimplifierAliasesGenerator < WikiGenerator
+ def initialize(page_name)
+ super(page_name)
+ end
+
+ def generate_content(options)
+ default_aliases = get_sub_simplifier_default_aliases(options)
+ sites_aliases = generate_all_sites_aliases
+
+ default_aliases_by_category = {}
+ default_aliases.each do |al, data|
+ default_aliases_by_category[data['category']] ||= {}
+ default_aliases_by_category[data['category']][al] = data
+ end
+
+ @generated_content = "__NOEDITSECTION__\n"
+
+ @generated_content += "=== Clusters and nodes ===\n"
+ @generated_content += "<code>%d</code> is a digit representing a node's number.<br/><br/>\n"
+
+ sites_aliases.each do |site, aliases|
+ @generated_content += "'''#{site.capitalize}'''\n"
+ @generated_content += "{|class=\"wikitable mw-collapsible mw-collapsed\"\n"
+ @generated_content += "! style=\"width: 15%\" | Alias\n"
+ @generated_content += "! style=\"width: 30%\" | Description\n"
+ @generated_content += "! style=\"width: 30%\" | Translate to\n"
+ aliases.each do |al, value|
+ desc = if al.match(/([a-zA-Z]+)-\%d/)
+ "Select a node from cluster #{$1.capitalize}"
+ else
+ "Select node(s) from cluster #{al.capitalize}"
+ end
+
+ @generated_content += "|-\n"
+ @generated_content += "|''#{al}''\n"
+ @generated_content += "|#{desc}\n"
+ @generated_content += "|<code>#{value}</code>\n"
+ end
+ @generated_content += "|}\n"
+ @generated_content += "\n"
+ end
+
+ default_aliases_by_category.sort.each do |cat, data|
+ @generated_content += "=== #{cat} ===\n"
+ @generated_content += "{|class=\"wikitable\"\n"
+ @generated_content += "! style=\"width: 15%\" | Alias\n"
+ @generated_content += "! style=\"width: 30%\" | Description\n"
+ @generated_content += "! style=\"width: 30%\" | Translate to\n"
+
+ data.each do |al, c|
+ @generated_content += "|-\n"
+ @generated_content += "|''#{al}''\n"
+ @generated_content += "|#{c['desc']}\n"
+ @generated_content += "|<code>#{c['value']}</code>\n"
+ end
+
+ @generated_content += "|}\n"
+ @generated_content += "\n"
+ end
+ end
+end
diff --git a/lib/refrepo/gen/wiki/generators/site_hardware.rb b/lib/refrepo/gen/wiki/generators/site_hardware.rb
index 3445690bd46..992b8517783 100644
--- a/lib/refrepo/gen/wiki/generators/site_hardware.rb
+++ b/lib/refrepo/gen/wiki/generators/site_hardware.rb
@@ -9,7 +9,7 @@ class SiteHardwareGenerator < WikiGenerator
@site = site
end
- def generate_content
+ def generate_content(_options)
has_reservable_disks = false
G5K::get_global_hash['sites'][@site]['clusters'].each do |_,c|
c['nodes'].each do |_,n|
diff --git a/lib/refrepo/gen/wiki/generators/site_network.rb b/lib/refrepo/gen/wiki/generators/site_network.rb
index 2c8c50eb9c3..44b601bcda7 100644
--- a/lib/refrepo/gen/wiki/generators/site_network.rb
+++ b/lib/refrepo/gen/wiki/generators/site_network.rb
@@ -26,7 +26,7 @@ class SiteNetworkGenerator < WikiGenerator
return h['network_equipments'].to_a.map { |e| "* #{e[0]}: #{e[1]['model']}" }.sort.join("\n")
end
- def generate_content
+ def generate_content(_options)
@generated_content = "__NOTOC__\n__NOEDITSECTION__\n"
@generated_content += "= Network devices models =\n"
@generated_content += generate_equipments + "\n"
diff --git a/lib/refrepo/gen/wiki/generators/status.rb b/lib/refrepo/gen/wiki/generators/status.rb
index 8e44d75e890..5ae0d4c3c02 100644
--- a/lib/refrepo/gen/wiki/generators/status.rb
+++ b/lib/refrepo/gen/wiki/generators/status.rb
@@ -6,7 +6,7 @@ class StatusGenerator < WikiGenerator
super(page_name)
end
- def generate_content
+ def generate_content(_options)
@global_hash = get_global_hash
@site_uids = G5K::SITES
diff --git a/lib/refrepo/gen/wiki/wiki_generator.rb b/lib/refrepo/gen/wiki/wiki_generator.rb
index 96731a8fc19..fa6c54f0c29 100644
--- a/lib/refrepo/gen/wiki/wiki_generator.rb
+++ b/lib/refrepo/gen/wiki/wiki_generator.rb
@@ -29,7 +29,7 @@ class WikiGenerator
end
end
- def generate_content
+ def generate_content(_options)
raise "To be implemented in actual generators"
end
@@ -114,7 +114,7 @@ class WikiGenerator
#Execute actions on generator based on given options
def exec(options)
- generate_content()
+ generate_content(options)
ret = true
#Login only if we need to
--
GitLab