[lib/refrepos/gen/wiki] use data instead of input

lib/refrepo/gen/wiki/generators/cpu_parameters.rb

@@ -28,7 +28,7 @@ class CPUParametersGenerator < WikiGenerator
 
         #One line per cluster
         table_data << [
-          DateTime.new(*cluster_hash["created_at"].to_s.scan(/\d+/).map {|i| i.to_i}).strftime("%Y-%m-%d"),
+          DateTime.parse(*cluster_hash["created_at"]).strftime("%Y-%m-%d"),
           site_uid,
           cluster_uid,
           cpu_family,

lib/refrepo/gen/wiki/generators/disk_reservation.rb

@@ -17,7 +17,7 @@ class DiskReservationGenerator < WikiGenerator
         disk_info = {}
         cluster_hash.fetch('nodes').sort.to_h.each { |node_uid, node_hash|
           next if node_hash['status'] == 'retired'
-           reservable_disks = node_hash['storage_devices'].select{ |k, v| v['reservation'] == true }.count
+           reservable_disks = node_hash['storage_devices'].select{ |v| v['reservation'] == true }.count
           add(disk_info, node_uid, reservable_disks)
         }
 

lib/refrepo/gen/wiki/generators/hardware.rb

@@ -42,10 +42,10 @@ class G5KHardwareGenerator < WikiGenerator
           if node_hash['gpu'] and node_hash['gpu']['gpu_count']
             gpus += node_hash['gpu']['gpu_count']
           end
-          ssds += node_hash['storage_devices'].values.select { |d| d['storage'] == 'SSD' }.length
-          hdds += node_hash['storage_devices'].values.select { |d| d['storage'] == 'HDD' }.length
-          node_hash['storage_devices'].each_pair do |k, e|
-            storage_space += e['size']
+          ssds += node_hash['storage_devices'].select { |d| d['storage'] == 'SSD' }.length
+          hdds += node_hash['storage_devices'].select { |d| d['storage'] == 'HDD' }.length
+          node_hash['storage_devices'].each do |i|
+            storage_space += i['size']
           end
         end
       end
@@ -108,12 +108,12 @@ class G5KHardwareGenerator < WikiGenerator
             data['ram_size'][key][site_uid] += 1
 
             # HPC Networks
-            interfaces = node_hash['network_adapters'].select{ |k, v|
+            interfaces = node_hash['network_adapters'].select{ |v|
               v['enabled'] and
                 (v['mounted'] or v['mountable']) and
                 not v['management'] and
-                (k =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
-            }.map{ |k, v|
+                (v['device'] =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
+            }.map{ |v|
               [
                 {
                   text: v['interface'] + ' ' + G5K.get_rate(v['rate']),
@@ -129,12 +129,12 @@ class G5KHardwareGenerator < WikiGenerator
             }
 
             # NIC models
-            interfaces = node_hash['network_adapters'].select{ |k, v|
+            interfaces = node_hash['network_adapters'].select{ |v|
               v['enabled'] and
                 (v['mounted'] or v['mountable']) and
                 not v['management'] and
-                (k =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
-            }.map{ |k, v|
+                (v['device'] =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
+            }.map{ |v|
               t = (v['vendor'] || 'N/A') + ' ' + (v['model'] || 'N/A');
               [
                 {
@@ -314,21 +314,21 @@ class G5KHardwareGenerator < WikiGenerator
         cluster_hash.fetch('nodes').sort.to_h.each do |node_uid, node_hash|
           next if node_hash['status'] == 'retired'
           sd = node_hash['storage_devices']
-          reservable_disks = sd.to_a.select{ |v| v[1]['reservation'] == true }.count > 0
-          maindisk = sd.to_a.select { |v| v[0] == 'sda' }.first[1]
+          reservable_disks = sd.select{ |v| v['reservation'] == true }.count > 0
+          maindisk = sd.select { |v| v['device'] == 'sda' }[0]
           maindisk_t = maindisk['storage'] + ' ' + G5K.get_size(maindisk['size'],'metric')
-          other = sd.to_a.select { |d| d[0] != 'sda' }
-          hdds = other.select { |d| d[1]['storage'] == 'HDD' }
+          other = sd.select { |d| d['device'] != 'sda' }
+          hdds = other.select { |d| d['storage'] == 'HDD' }
           if hdds.count == 0
             hdd_t = "0"
           else
-            hdd_t = hdds.count.to_s + " (" + hdds.map { |d| G5K.get_size(d[1]['size'],'metric') }.join(', ') + ")"
+            hdd_t = hdds.count.to_s + " (" + hdds.map { |d| G5K.get_size(d['size'],'metric') }.join(', ') + ")"
           end
-          ssds = other.select { |d| d[1]['storage'] == 'SSD' }
+          ssds = other.select { |d| d['storage'] == 'SSD' }
           if ssds.count == 0
             ssd_t = "0"
           else
-            ssd_t = ssds.count.to_s + " (" + ssds.map { |d| G5K.get_size(d[1]['size'],'metric') }.join(', ') + ")"
+            ssd_t = ssds.count.to_s + " (" + ssds.map { |d| G5K.get_size(d['size'],'metric') }.join(', ') + ")"
           end
           queues = cluster_hash['queues'] - ['admin', 'default']
           queue_t = (queues.nil? || (queues.empty? ? '' : "_.28" + queues[0].gsub(' ', '_') + ' queue.29'))
@@ -373,13 +373,18 @@ class G5KHardwareGenerator < WikiGenerator
         cluster_hash.fetch('nodes').sort.to_h.each { |node_uid, node_hash|
           next if node_hash['status'] == 'retired'
           if node_hash['network_adapters']
-            node_interfaces = node_hash['network_adapters'].select{ |k, v| v['interface'] == 'Ethernet' and v['enabled'] == true and (v['mounted'] == true or v['mountable'] == true) and v['management'] == false }
+            node_interfaces = node_hash['network_adapters'].select{ |v|
+              v['interface'] == 'Ethernet' and
+              v['enabled'] == true and
+              (v['mounted'] == true or v['mountable'] == true) and
+              v['management'] == false
+            }
 
             interfaces = {}
-            interfaces['25g_count'] = node_interfaces.select { |k, v| v['rate'] == 25_000_000_000 }.count
-            interfaces['10g_count'] = node_interfaces.select { |k, v| v['rate'] == 10_000_000_000 }.count
-            interfaces['1g_count'] = node_interfaces.select { |k, v| v['rate'] == 1_000_000_000 }.count
-            interfaces['details'] = node_interfaces.map{ |k, v| k + (v['name'].nil? ? '' : '/' + v['name'])  + ' (' + G5K.get_rate(v['rate']) + ')' }.sort.join(', ')
+            interfaces['25g_count'] = node_interfaces.select { |v| v['rate'] == 25_000_000_000 }.count
+            interfaces['10g_count'] = node_interfaces.select { |v| v['rate'] == 10_000_000_000 }.count
+            interfaces['1g_count'] = node_interfaces.select { |v| v['rate'] == 1_000_000_000 }.count
+            interfaces['details'] = node_interfaces.map{ |v| v['device'] + (v['name'].nil? ? '' : '/' + v['name'])  + ' (' + G5K.get_rate(v['rate']) + ')' }.sort.join(', ')
             queues = cluster_hash['queues'] - ['admin', 'default', 'testing']
             interfaces['queues'] = (queues.nil? || (queues.empty? ? '' : queues[0] + G5K.pluralize(queues.count, ' queue')))
             interface_add(network_interfaces, node_uid, interfaces) if node_interfaces.count > 1

lib/refrepo/gen/wiki/generators/site_hardware.rb

@@ -184,7 +184,6 @@ def get_hardware(sites)
       cluster_hash.fetch('nodes').sort.each { |node_uid, node_hash|
         next if node_hash['status'] == 'retired'
         # map model to vendor (eg: {'SAS5484654' => 'Seagate', 'PX458' => 'Toshiba' ...}
-        disk_model_vendor_mapping =  global_hash['disk_vendor_model_mapping'].map{ |vdr, mdls| mdls.map{ |mdl| [mdl, vdr] } }.flatten(1).to_h
         hard = {}
         queue = cluster_hash['queues'] - ['admin', 'default']
         hard['queue'] = (queue.nil? || queue.empty?) ? '' : queue[0]
@@ -204,17 +203,51 @@ def get_hardware(sites)
         hard['num_processor_model'] = (hard['cpus_per_node'] == 1 ? '' : "#{hard['cpus_per_node']} x ") + hard['processor_model'].gsub(' ', ' ')
         hard['processor_description'] = "#{hard['processor_model']} (#{hard['microarchitecture']}#{hard['processor_freq'] ?  ', ' + hard['processor_freq'] : ''}, #{hard['cpus_per_node_str']}, #{hard['cores_per_cpu_str']})"
         hard['ram_size'] = G5K.get_size(node_hash['main_memory']['ram_size'])
-        storage = node_hash['storage_devices'].map{ |k, v| {'size' => v['size'],  'tech' => v['storage']} }
-        hard['storage'] = storage.each_with_object(Hash.new(0)) { |data, counts| counts[data] += 1 }.to_a.sort_by { |e| e[0]['size'].to_f }.map{ |e| (e[1] == 1 ? '' : e[1].to_s + ' x ') + G5K.get_size(e[0]['size'],'metric') + ' ' + e[0]['tech'] }.join(' + ')
-        hard['storage_size'] = storage.inject(0){|sum, v| sum + (v['size'].to_f / 2**30).floor }.to_s # round to GB to avoid small differences within a cluster
-        storage_description = node_hash['storage_devices'].map { |k, v| { 'device' => v['device'], 'size' => v['size'], 'tech' => v['storage'], 'interface' => v['interface'], 'vendor' => disk_model_vendor_mapping[v['model']],'model' => v['model'], 'driver' => v['driver'], 'path' => v['by_path'] || v['by_id'],  'count' => node_hash['storage_devices'].count } }
+        storage = node_hash['storage_devices'].map { |i| { 'size' => i['size'], 'tech' => i['storage'] } }
+        hard['storage'] = storage.each_with_object(Hash.new(0)) { |data, counts|
+          counts[data] += 1
+        }.to_a.sort_by { |e|
+          e[0]['size'].to_f
+        }.map { |e|
+          (e[1] == 1 ? '' : e[1].to_s + ' x ') +
+          G5K.get_size(e[0]['size'], 'metric') +
+          ' ' +
+          e[0]['tech']
+        }.join(' + ')
 
-        hard['storage_description'] = storage_description.map { |e| [ e['count'] > 1 ? "\n*" : '', G5K.get_size(e['size'],'metric'), e['tech'], e['interface'], e['vendor'], e['model'], ' (driver: ' + (e['driver'] || 'MISSING') + ', path: ' + (e['path'] || 'MISSING') + ')'].join(' ') }.join('<br />')
+        hard['storage_size'] = storage.inject(0){|sum, v| sum + (v['size'].to_f / 2**30).floor }.to_s # round to GB to avoid small differences within a cluster
+        storage_description = node_hash['storage_devices'].sort { |a,b|
+          a['device'] <=> b['device']
+        }.map do |v|
+          {
+            'device' => v['device'],
+            'size' => v['size'],
+            'tech' => v['storage'],
+            'interface' => v['interface'],
+            'vendor' => v['vendor'],
+            'model' => v['model'],
+            'driver' => v['driver'],
+            'path' => v['by_path'] || v['by_id'],
+            'count' => node_hash['storage_devices'].count
+          }
+        end
 
-        network = node_hash['network_adapters'].select { |k, v|
+        hard['storage_description'] = storage_description.map { |e|
+          [
+            e['count'] > 1 ? "\n*" : '',
+            G5K.get_size(e['size'],'metric'),
+            e['tech'],
+            e['interface'],
+            e['vendor'],
+            e['model'],
+            ' (driver: ' + (e['driver'] || 'MISSING') + ', path: ' + (e['path'] || 'MISSING') + ')'
+          ].join(' ')
+        }.join('<br />')
+
+        network = node_hash['network_adapters'].select { |v|
           v['management'] == false &&
-          (k =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
-        }.map{|k, v| {
+          (v['device'] =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
+        }.map{|v| {
             'rate' => v['rate'],
             'interface' => v['interface'],
             'used' => (v['enabled'] and (v['mounted'] or v['mountable']))
@@ -236,12 +269,12 @@ def get_hardware(sites)
           sum + (v['rate'].to_f / 10**6).floor
         }.to_s # round to Mbps
 
-        network_description = node_hash['network_adapters'].select { |k, v|
+        network_description = node_hash['network_adapters'].select { |v|
           v['management'] == false &&
-          (k =~ /\./).nil? # exclude PKEY / VLAN interface see #9417
-        }.map{ |k, v|
+          (v['device'] =~ /\./).nil? # exclude PKEY / VLAN interface see #9417
+        }.map{ |v|
           {
-            'device' => k,
+            'device' => v['device'],
             'name' => v['name'],
             'rate' => v['rate'],
             'interface' => v['interface'],

lib/refrepo/gen/wiki/generators/site_network.rb

@@ -23,7 +23,7 @@ class SiteNetworkGenerator < WikiGenerator
 
   def generate_equipments
     h = G5K::get_global_hash['sites'][@site]
-    return h['networks'].to_a.map { |e| "* #{e[0]}: #{e[1]['model']}" }.sort.join("\n")
+    return h['network_equipments'].to_a.map { |e| "* #{e[0]}: #{e[1]['model']}" }.sort.join("\n")
   end
 
   def generate_content

lib/refrepo/gen/wiki/mw_utils.rb

@@ -21,7 +21,7 @@ module MediawikiApi
       res = get_conn.send(:get, '', params)
       res.body
     end
-    
+
     def get_file_content(file_name)
       get_conn = Faraday.new(url: MW::BASE_URL + "images/#{file_name}") do |faraday|
         faraday.request :multipart
@@ -95,7 +95,7 @@ module G5K
       end
     end
   end
-  
+
   def self.get_rate(x)
     return '' if (x == 0 || x.nil?)
     mbps = (x.to_f / 10**6).floor
@@ -109,11 +109,11 @@ module G5K
   def self.pluralize(count, word)
     return (count == 1 || word[-1] == 's') ? word : word + 's'
   end
-  
+
   @@global_hash = nil
   def self.get_global_hash
     if @@global_hash.nil?
-      @@global_hash = load_yaml_file_hierarchy
+      @@global_hash = load_data_hierarchy
     end
     # return a deep copy of global_hash
     return Marshal.load(Marshal.dump(@@global_hash))
@@ -144,7 +144,7 @@ module MW
   UNSORTED_INLINE_CELL = "!!"
 
   UNSORTED_TABLE_CELL = "!"
-  
+
   LINE_FEED = "\n"
 
   LIST_ITEM = "*"
@@ -212,7 +212,7 @@ module MW
   def self.italic(text)
     "''" + text + "''"
   end
- 
+
   def self.bold(text)
     "'''" + text + "'''"
   end
@@ -223,6 +223,6 @@ module MW
 
   def self.code(text)
     "<code>" + text + "</code>"
-  end   
+  end
 
 end