[dev] Sort network interfaces and storage devices, Manage Sensors,...

[dev] Sort network interfaces and storage devices, Manage Sensors, <cluster>.yaml generation, GPUs, new version of the code for getting network adapter info ...

generators/reference-api/reference-api.rb

@@ -5,6 +5,7 @@ require 'erb'
 require 'fileutils'
 require 'pathname'
 require 'json'
+require 'time'
 
 require '../lib/input_loader'
 
@@ -12,14 +13,22 @@ require '../lib/input_loader'
 refapi_path = "/tmp/data"
 
 #global_hash = JSON.parse(STDIN.read)
-global_hash = load_yaml_file_hierarchy("../../input/example/")
+#global_hash = load_yaml_file_hierarchy("../../input/example/")
+global_hash = load_yaml_file_hierarchy("../../input/grid5000/")
+
+class Hash
+  # sort a hash according to the position of the key in the array
+  def sort_by_array(array)
+    Hash[sort_by{|key, _| array.index(key) || length}] 
+  end
+end
 
 # Write pretty and sorted JSON files
 def write_json(filepath, data)
   def rec_sort(h)
     case h
     when Array
-      h.map{|v| rec_sort(v)}.sort_by!{|v| (v.to_s rescue nil) }
+      h.map{|v| rec_sort(v)}#.sort_by!{|v| (v.to_s rescue nil) }
     when Hash
       Hash[Hash[h.map{|k,v| [rec_sort(k),rec_sort(v)]}].sort_by{|k,v| [(k.to_s rescue nil), (v.to_s rescue nil)]}]
     else
@@ -35,7 +44,8 @@ end
 #  In the network description, if the node interface is given (using "port" attribute),
 #  the interface parameter must be used.
 def net_switch_port_lookup(site, node_uid, interface='')
-  site["network"].each do |switch_uid, switch|
+  site["net-links"].each do |switch_uid, switch| # TODO: rename net-links<->network
+    #pp switch_uid
     switch["linecards"].each do |lc_uid,lc|
       lc["ports"].each do |port_uid,port|
         if port.is_a?(Hash)
@@ -58,20 +68,46 @@ def net_switch_port_lookup(site, node_uid, interface='')
 end
 
 global_hash["sites"].each do |site_uid, site|
+  pp site_uid
+#  pp site
 
   site["clusters"].each do |cluster_uid, cluster|
+    pp cluster_uid
 
     cluster_path = Pathname.new(refapi_path).join("sites",site_uid,"clusters",cluster_uid)
     cluster_path.join("nodes").mkpath()
 
     # Write cluster info w/o nodes entries
+    cluster["type"] = "cluster"
+    cluster["uid"]  = cluster_uid
+    
+    # On the previous version of this script, cluster["created_ad"] was generated from a Ruby Time. cluster["created_ad"] is now a Ruby Date at JSON import.
+    # As Date.httpdate and Time.httpdate does not behave the same with timezone, it is converted here as a Ruby time.
+    cluster["created_at"] = Time.parse(cluster["created_at"].to_s).httpdate
+    
     write_json(cluster_path.join("#{cluster_uid}.json"),
                cluster.reject {|k, v| k == "nodes"})
-
+    
     # Write node info
     cluster["nodes"].each do |node_uid, node|
+      pp node_uid
 
+#     next unless node_uid == "griffon-1"
       node["uid"] = node_uid
+      node["type"] = "node"
+      if node.key?("processor")
+        node["processor"]["cache_l1"] = nil unless node["processor"].key?("cache_l1")
+      end
+      
+      # Add default keys
+      node["gpu"] = {} unless node.key?("gpu")
+      node["gpu"]["gpu"] = false unless node["gpu"].key?("gpu")
+      
+      node["main_memory"] = {} unless node.key?("main_memory")
+      node["main_memory"]["virtual_size"] = nil unless node["main_memory"].key?("virtual_size")
+
+#     node["monitoring"] = {}   unless node.key?("monitoring")
+#     node["monitoring"]["wattmeter"] = false unless node["monitoring"].key?("wattmeter")
 
       # Rename keys
       node["storage_devices"] = node.delete("block_devices")
@@ -81,31 +117,67 @@ global_hash["sites"].each do |site_uid, site|
         node["chassis"]["serial"] = node["chassis"].delete("serial_number")
       end
       
+      # Type conversion
+      node["network_adapters"].each { |key, hash| hash["rate"] = hash["rate"].to_i if hash["rate"].is_a?(Float) }
+
       # Convert hashes to arrays
-      node["storage_devices"] = node["storage_devices"].values
-      node["network_adapters"] = node["network_adapters"].values
+      node["storage_devices"].each { |key, hash| node["storage_devices"][key]["device"] = key; } # Add "device: sdX" within the hash
+      node["storage_devices"] = node["storage_devices"].sort_by_array(["sda", "sdb", "sdc", "sdd", "sde"]).values
+
+      node["network_adapters"].each { |key, hash| node["network_adapters"][key]["device"] = key; } # Add "device: ethX" within the hash
+      node["network_adapters"] = node["network_adapters"].sort_by_array(["eth0", "eth1", "eth2", "eth3", "ib0", "ib1", "ib2", "ib3", "bmc"]).values
 
-      # Populate "network_address", "switch" and "port" from the network equipment description for each network adapters
+      # Populate "network_address", "switch" and "switch_port" from the network equipment description for each network adapters
       node["network_adapters"].each { |network_adapter|
-        if network_adapter["enabled"]
-          if network_adapter["management"]
-            # Management network_adapter
-            network_adapter["network_address"] = "#{node_uid}-bmc.#{site_uid}.grid5000.fr"
-          elsif network_adapter["mounted"]
-            # Primary network_adapter
-            network_adapter["bridged"] = true
-            network_adapter["network_address"] = "#{node_uid}.#{site_uid}.grid5000.fr"
-            # Interface may not be specified in Network Reference for primary network_adapter
-            network_adapter["switch"], network_adapter["port"] = \
-              net_switch_port_lookup(site, node_uid) || net_switch_port_lookup(site, node_uid, network_adapter["device"])
-          else
-            # Secondary network_adapter(s)
-            network_adapter["network_address"] = "#{node_uid}-#{network_adapter["device"]}.#{site}.grid5000.fr"
-            network_adapter["switch"], network_adapter["port"] = net_switch_port_lookup(site, node_uid, network_adapter["device"])
-          end
+
+        # ib properties
+        network_adapter["ib_switch_card"]     = network_adapter.delete("line_card") if network_adapter.key?("line_card")
+        network_adapter["ib_switch_card_pos"] = network_adapter.delete("position")  if network_adapter.key?("position")
+
+        next unless network_adapter["enabled"]
+
+        # Management network_adapter (bmc)
+        if network_adapter["management"]
+          network_adapter["network_address"] = "#{node_uid}-bmc.#{site_uid}.grid5000.fr"
+          next
         end
+
+        # if network_adapter["network_address"] or network_adapter["switch"] or network_adapter["switch_port"]
+        #   pp "Warning: network_address, switch or switch_port defined manually for (#{node_uid}, #{network_adapter["device"]})"
+        #   pp "#{network_adapter["network_address"]}, #{network_adapter["switch"]} or #{network_adapter["switch_port"]}"
+        # end
+        
+        if network_adapter["mounted"] and /^eth[0-9]$/.match(network_adapter["device"])
+          # Primary network_adapter
+          network_adapter["network_address"] = "#{node_uid}.#{site_uid}.grid5000.fr"
+         
+          # Interface may not be specified in Network Reference for primary network_adapter
+          network_adapter["switch"], network_adapter["switch_port"] = net_switch_port_lookup(site, node_uid, network_adapter["device"]) || net_switch_port_lookup(site, node_uid)
+
+          # network_adapter["bridged"] = true # TODO?
+          next
+        end
+#        if network_adapter["bridged"]
+          # Secondary network_adapter(s)
+          network_adapter["network_address"] = "#{node_uid}-#{network_adapter["device"]}.#{site_uid}.grid5000.fr"
+          switch, port = net_switch_port_lookup(site, node_uid, network_adapter["device"])
+          network_adapter["switch"] = switch if switch
+          network_adapter["switch_port"] = port if port
+#        end
       }
 
+      if node.key?("sensors") and node.key?("pdu") and node["pdu"].key?("pdu_name")
+        node["sensors"]["power"] = {}                  unless node["sensors"].key?("power")
+        node["sensors"]["power"]["via"] = {}           unless node["sensors"]["power"].key?("via")
+        node["sensors"]["power"]["via"]["pdu"] = []    unless node["sensors"]["power"]["via"].key?("pdu")
+        node["sensors"]["power"]["via"]["pdu"][0] = {} unless node["sensors"]["power"]["via"]["pdu"].size > 0
+        
+        node["sensors"]["power"]["via"]["pdu"][0]["uid"]  = node["pdu"]["pdu_name"]
+        node["sensors"]["power"]["via"]["pdu"][0]["port"] = node["pdu"]["pdu_position"] if  node["pdu"].key?("pdu_position")
+
+        node.delete("pdu")
+      end
+
       #pp cluster_path.join("nodes","#{node_uid}.json")
       write_json(cluster_path.join("nodes","#{node_uid}.json"), node)
     end