From a565845374e3478c78a586ed34a61b918035e6a8 Mon Sep 17 00:00:00 2001
From: Dimitri DELABROYE <dimitri.delabroye@inria.fr>
Date: Tue, 2 Jul 2019 08:18:23 +0200
Subject: [PATCH] [lib/refrepos/gen/wiki] use data instead of input
---
.../gen/wiki/generators/cpu_parameters.rb | 2 +-
.../gen/wiki/generators/disk_reservation.rb | 2 +-
lib/refrepo/gen/wiki/generators/hardware.rb | 49 ++++++++-------
.../gen/wiki/generators/site_hardware.rb | 59 +++++++++++++++----
.../gen/wiki/generators/site_network.rb | 2 +-
lib/refrepo/gen/wiki/mw_utils.rb | 14 ++---
6 files changed, 83 insertions(+), 45 deletions(-)
diff --git a/lib/refrepo/gen/wiki/generators/cpu_parameters.rb b/lib/refrepo/gen/wiki/generators/cpu_parameters.rb
index d0d7d0e15a9..013c1f7b996 100644
--- a/lib/refrepo/gen/wiki/generators/cpu_parameters.rb
+++ b/lib/refrepo/gen/wiki/generators/cpu_parameters.rb
@@ -28,7 +28,7 @@ class CPUParametersGenerator < WikiGenerator
#One line per cluster
table_data << [
- DateTime.new(*cluster_hash["created_at"].to_s.scan(/\d+/).map {|i| i.to_i}).strftime("%Y-%m-%d"),
+ DateTime.parse(*cluster_hash["created_at"]).strftime("%Y-%m-%d"),
site_uid,
cluster_uid,
cpu_family,
diff --git a/lib/refrepo/gen/wiki/generators/disk_reservation.rb b/lib/refrepo/gen/wiki/generators/disk_reservation.rb
index 5a5ebf775a5..b88db2dc21a 100644
--- a/lib/refrepo/gen/wiki/generators/disk_reservation.rb
+++ b/lib/refrepo/gen/wiki/generators/disk_reservation.rb
@@ -17,7 +17,7 @@ class DiskReservationGenerator < WikiGenerator
disk_info = {}
cluster_hash.fetch('nodes').sort.to_h.each { |node_uid, node_hash|
next if node_hash['status'] == 'retired'
- reservable_disks = node_hash['storage_devices'].select{ |k, v| v['reservation'] == true }.count
+ reservable_disks = node_hash['storage_devices'].select{ |v| v['reservation'] == true }.count
add(disk_info, node_uid, reservable_disks)
}
diff --git a/lib/refrepo/gen/wiki/generators/hardware.rb b/lib/refrepo/gen/wiki/generators/hardware.rb
index 0098d770acc..23f7cc07cd3 100644
--- a/lib/refrepo/gen/wiki/generators/hardware.rb
+++ b/lib/refrepo/gen/wiki/generators/hardware.rb
@@ -42,10 +42,10 @@ class G5KHardwareGenerator < WikiGenerator
if node_hash['gpu'] and node_hash['gpu']['gpu_count']
gpus += node_hash['gpu']['gpu_count']
end
- ssds += node_hash['storage_devices'].values.select { |d| d['storage'] == 'SSD' }.length
- hdds += node_hash['storage_devices'].values.select { |d| d['storage'] == 'HDD' }.length
- node_hash['storage_devices'].each_pair do |k, e|
- storage_space += e['size']
+ ssds += node_hash['storage_devices'].select { |d| d['storage'] == 'SSD' }.length
+ hdds += node_hash['storage_devices'].select { |d| d['storage'] == 'HDD' }.length
+ node_hash['storage_devices'].each do |i|
+ storage_space += i['size']
end
end
end
@@ -108,12 +108,12 @@ class G5KHardwareGenerator < WikiGenerator
data['ram_size'][key][site_uid] += 1
# HPC Networks
- interfaces = node_hash['network_adapters'].select{ |k, v|
+ interfaces = node_hash['network_adapters'].select{ |v|
v['enabled'] and
(v['mounted'] or v['mountable']) and
not v['management'] and
- (k =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
- }.map{ |k, v|
+ (v['device'] =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
+ }.map{ |v|
[
{
text: v['interface'] + ' ' + G5K.get_rate(v['rate']),
@@ -129,12 +129,12 @@ class G5KHardwareGenerator < WikiGenerator
}
# NIC models
- interfaces = node_hash['network_adapters'].select{ |k, v|
+ interfaces = node_hash['network_adapters'].select{ |v|
v['enabled'] and
(v['mounted'] or v['mountable']) and
not v['management'] and
- (k =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
- }.map{ |k, v|
+ (v['device'] =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
+ }.map{ |v|
t = (v['vendor'] || 'N/A') + ' ' + (v['model'] || 'N/A');
[
{
@@ -314,21 +314,21 @@ class G5KHardwareGenerator < WikiGenerator
cluster_hash.fetch('nodes').sort.to_h.each do |node_uid, node_hash|
next if node_hash['status'] == 'retired'
sd = node_hash['storage_devices']
- reservable_disks = sd.to_a.select{ |v| v[1]['reservation'] == true }.count > 0
- maindisk = sd.to_a.select { |v| v[0] == 'sda' }.first[1]
+ reservable_disks = sd.select{ |v| v['reservation'] == true }.count > 0
+ maindisk = sd.select { |v| v['device'] == 'sda' }[0]
maindisk_t = maindisk['storage'] + ' ' + G5K.get_size(maindisk['size'],'metric')
- other = sd.to_a.select { |d| d[0] != 'sda' }
- hdds = other.select { |d| d[1]['storage'] == 'HDD' }
+ other = sd.select { |d| d['device'] != 'sda' }
+ hdds = other.select { |d| d['storage'] == 'HDD' }
if hdds.count == 0
hdd_t = "0"
else
- hdd_t = hdds.count.to_s + " (" + hdds.map { |d| G5K.get_size(d[1]['size'],'metric') }.join(', ') + ")"
+ hdd_t = hdds.count.to_s + " (" + hdds.map { |d| G5K.get_size(d['size'],'metric') }.join(', ') + ")"
end
- ssds = other.select { |d| d[1]['storage'] == 'SSD' }
+ ssds = other.select { |d| d['storage'] == 'SSD' }
if ssds.count == 0
ssd_t = "0"
else
- ssd_t = ssds.count.to_s + " (" + ssds.map { |d| G5K.get_size(d[1]['size'],'metric') }.join(', ') + ")"
+ ssd_t = ssds.count.to_s + " (" + ssds.map { |d| G5K.get_size(d['size'],'metric') }.join(', ') + ")"
end
queues = cluster_hash['queues'] - ['admin', 'default']
queue_t = (queues.nil? || (queues.empty? ? '' : "_.28" + queues[0].gsub(' ', '_') + ' queue.29'))
@@ -373,13 +373,18 @@ class G5KHardwareGenerator < WikiGenerator
cluster_hash.fetch('nodes').sort.to_h.each { |node_uid, node_hash|
next if node_hash['status'] == 'retired'
if node_hash['network_adapters']
- node_interfaces = node_hash['network_adapters'].select{ |k, v| v['interface'] == 'Ethernet' and v['enabled'] == true and (v['mounted'] == true or v['mountable'] == true) and v['management'] == false }
+ node_interfaces = node_hash['network_adapters'].select{ |v|
+ v['interface'] == 'Ethernet' and
+ v['enabled'] == true and
+ (v['mounted'] == true or v['mountable'] == true) and
+ v['management'] == false
+ }
interfaces = {}
- interfaces['25g_count'] = node_interfaces.select { |k, v| v['rate'] == 25_000_000_000 }.count
- interfaces['10g_count'] = node_interfaces.select { |k, v| v['rate'] == 10_000_000_000 }.count
- interfaces['1g_count'] = node_interfaces.select { |k, v| v['rate'] == 1_000_000_000 }.count
- interfaces['details'] = node_interfaces.map{ |k, v| k + (v['name'].nil? ? '' : '/' + v['name']) + ' (' + G5K.get_rate(v['rate']) + ')' }.sort.join(', ')
+ interfaces['25g_count'] = node_interfaces.select { |v| v['rate'] == 25_000_000_000 }.count
+ interfaces['10g_count'] = node_interfaces.select { |v| v['rate'] == 10_000_000_000 }.count
+ interfaces['1g_count'] = node_interfaces.select { |v| v['rate'] == 1_000_000_000 }.count
+ interfaces['details'] = node_interfaces.map{ |v| v['device'] + (v['name'].nil? ? '' : '/' + v['name']) + ' (' + G5K.get_rate(v['rate']) + ')' }.sort.join(', ')
queues = cluster_hash['queues'] - ['admin', 'default', 'testing']
interfaces['queues'] = (queues.nil? || (queues.empty? ? '' : queues[0] + G5K.pluralize(queues.count, ' queue')))
interface_add(network_interfaces, node_uid, interfaces) if node_interfaces.count > 1
diff --git a/lib/refrepo/gen/wiki/generators/site_hardware.rb b/lib/refrepo/gen/wiki/generators/site_hardware.rb
index 6e5cd3d193e..aea21ae09cd 100644
--- a/lib/refrepo/gen/wiki/generators/site_hardware.rb
+++ b/lib/refrepo/gen/wiki/generators/site_hardware.rb
@@ -184,7 +184,6 @@ def get_hardware(sites)
cluster_hash.fetch('nodes').sort.each { |node_uid, node_hash|
next if node_hash['status'] == 'retired'
# map model to vendor (eg: {'SAS5484654' => 'Seagate', 'PX458' => 'Toshiba' ...}
- disk_model_vendor_mapping = global_hash['disk_vendor_model_mapping'].map{ |vdr, mdls| mdls.map{ |mdl| [mdl, vdr] } }.flatten(1).to_h
hard = {}
queue = cluster_hash['queues'] - ['admin', 'default']
hard['queue'] = (queue.nil? || queue.empty?) ? '' : queue[0]
@@ -204,17 +203,51 @@ def get_hardware(sites)
hard['num_processor_model'] = (hard['cpus_per_node'] == 1 ? '' : "#{hard['cpus_per_node']} x ") + hard['processor_model'].gsub(' ', ' ')
hard['processor_description'] = "#{hard['processor_model']} (#{hard['microarchitecture']}#{hard['processor_freq'] ? ', ' + hard['processor_freq'] : ''}, #{hard['cpus_per_node_str']}, #{hard['cores_per_cpu_str']})"
hard['ram_size'] = G5K.get_size(node_hash['main_memory']['ram_size'])
- storage = node_hash['storage_devices'].map{ |k, v| {'size' => v['size'], 'tech' => v['storage']} }
- hard['storage'] = storage.each_with_object(Hash.new(0)) { |data, counts| counts[data] += 1 }.to_a.sort_by { |e| e[0]['size'].to_f }.map{ |e| (e[1] == 1 ? '' : e[1].to_s + ' x ') + G5K.get_size(e[0]['size'],'metric') + ' ' + e[0]['tech'] }.join(' + ')
- hard['storage_size'] = storage.inject(0){|sum, v| sum + (v['size'].to_f / 2**30).floor }.to_s # round to GB to avoid small differences within a cluster
- storage_description = node_hash['storage_devices'].map { |k, v| { 'device' => v['device'], 'size' => v['size'], 'tech' => v['storage'], 'interface' => v['interface'], 'vendor' => disk_model_vendor_mapping[v['model']],'model' => v['model'], 'driver' => v['driver'], 'path' => v['by_path'] || v['by_id'], 'count' => node_hash['storage_devices'].count } }
+ storage = node_hash['storage_devices'].map { |i| { 'size' => i['size'], 'tech' => i['storage'] } }
+ hard['storage'] = storage.each_with_object(Hash.new(0)) { |data, counts|
+ counts[data] += 1
+ }.to_a.sort_by { |e|
+ e[0]['size'].to_f
+ }.map { |e|
+ (e[1] == 1 ? '' : e[1].to_s + ' x ') +
+ G5K.get_size(e[0]['size'], 'metric') +
+ ' ' +
+ e[0]['tech']
+ }.join(' + ')
- hard['storage_description'] = storage_description.map { |e| [ e['count'] > 1 ? "\n*" : '', G5K.get_size(e['size'],'metric'), e['tech'], e['interface'], e['vendor'], e['model'], ' (driver: ' + (e['driver'] || 'MISSING') + ', path: ' + (e['path'] || 'MISSING') + ')'].join(' ') }.join('<br />')
+ hard['storage_size'] = storage.inject(0){|sum, v| sum + (v['size'].to_f / 2**30).floor }.to_s # round to GB to avoid small differences within a cluster
+ storage_description = node_hash['storage_devices'].sort { |a,b|
+ a['device'] <=> b['device']
+ }.map do |v|
+ {
+ 'device' => v['device'],
+ 'size' => v['size'],
+ 'tech' => v['storage'],
+ 'interface' => v['interface'],
+ 'vendor' => v['vendor'],
+ 'model' => v['model'],
+ 'driver' => v['driver'],
+ 'path' => v['by_path'] || v['by_id'],
+ 'count' => node_hash['storage_devices'].count
+ }
+ end
- network = node_hash['network_adapters'].select { |k, v|
+ hard['storage_description'] = storage_description.map { |e|
+ [
+ e['count'] > 1 ? "\n*" : '',
+ G5K.get_size(e['size'],'metric'),
+ e['tech'],
+ e['interface'],
+ e['vendor'],
+ e['model'],
+ ' (driver: ' + (e['driver'] || 'MISSING') + ', path: ' + (e['path'] || 'MISSING') + ')'
+ ].join(' ')
+ }.join('<br />')
+
+ network = node_hash['network_adapters'].select { |v|
v['management'] == false &&
- (k =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
- }.map{|k, v| {
+ (v['device'] =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
+ }.map{|v| {
'rate' => v['rate'],
'interface' => v['interface'],
'used' => (v['enabled'] and (v['mounted'] or v['mountable']))
@@ -236,12 +269,12 @@ def get_hardware(sites)
sum + (v['rate'].to_f / 10**6).floor
}.to_s # round to Mbps
- network_description = node_hash['network_adapters'].select { |k, v|
+ network_description = node_hash['network_adapters'].select { |v|
v['management'] == false &&
- (k =~ /\./).nil? # exclude PKEY / VLAN interface see #9417
- }.map{ |k, v|
+ (v['device'] =~ /\./).nil? # exclude PKEY / VLAN interface see #9417
+ }.map{ |v|
{
- 'device' => k,
+ 'device' => v['device'],
'name' => v['name'],
'rate' => v['rate'],
'interface' => v['interface'],
diff --git a/lib/refrepo/gen/wiki/generators/site_network.rb b/lib/refrepo/gen/wiki/generators/site_network.rb
index c6ff45a39a6..56f373a6d01 100644
--- a/lib/refrepo/gen/wiki/generators/site_network.rb
+++ b/lib/refrepo/gen/wiki/generators/site_network.rb
@@ -23,7 +23,7 @@ class SiteNetworkGenerator < WikiGenerator
def generate_equipments
h = G5K::get_global_hash['sites'][@site]
- return h['networks'].to_a.map { |e| "* #{e[0]}: #{e[1]['model']}" }.sort.join("\n")
+ return h['network_equipments'].to_a.map { |e| "* #{e[0]}: #{e[1]['model']}" }.sort.join("\n")
end
def generate_content
diff --git a/lib/refrepo/gen/wiki/mw_utils.rb b/lib/refrepo/gen/wiki/mw_utils.rb
index 64011552c35..8124d729c05 100644
--- a/lib/refrepo/gen/wiki/mw_utils.rb
+++ b/lib/refrepo/gen/wiki/mw_utils.rb
@@ -21,7 +21,7 @@ module MediawikiApi
res = get_conn.send(:get, '', params)
res.body
end
-
+
def get_file_content(file_name)
get_conn = Faraday.new(url: MW::BASE_URL + "images/#{file_name}") do |faraday|
faraday.request :multipart
@@ -95,7 +95,7 @@ module G5K
end
end
end
-
+
def self.get_rate(x)
return '' if (x == 0 || x.nil?)
mbps = (x.to_f / 10**6).floor
@@ -109,11 +109,11 @@ module G5K
def self.pluralize(count, word)
return (count == 1 || word[-1] == 's') ? word : word + 's'
end
-
+
@@global_hash = nil
def self.get_global_hash
if @@global_hash.nil?
- @@global_hash = load_yaml_file_hierarchy
+ @@global_hash = load_data_hierarchy
end
# return a deep copy of global_hash
return Marshal.load(Marshal.dump(@@global_hash))
@@ -144,7 +144,7 @@ module MW
UNSORTED_INLINE_CELL = "!!"
UNSORTED_TABLE_CELL = "!"
-
+
LINE_FEED = "\n"
LIST_ITEM = "*"
@@ -212,7 +212,7 @@ module MW
def self.italic(text)
"''" + text + "''"
end
-
+
def self.bold(text)
"'''" + text + "'''"
end
@@ -223,6 +223,6 @@ module MW
def self.code(text)
"<code>" + text + "</code>"
- end
+ end
end
--
GitLab