[dev] Command line options + download 'oarnodes -Y' with ssh

generators/oar-properties/lib/lib-oar-properties.rb

@@ -92,7 +92,7 @@ def get_node_properties(cluster_uid, cluster, node_uid, node)
   end
   h['max_walltime'] = node['supported_job_types']['max_walltime'] || 0
   
-  h
+  return h
 end
   
 #
@@ -190,11 +190,42 @@ def oarcmd_set_node_properties(host, properties)
   
   return command
 end
+# '
+
+# Get the OAR properties from the OAR scheduler
+# This is only needed for the -d option
+def oarcmd_get_nodelist_properties(site_uid, filename=nil)
+  oarnodes_yaml = ""
+
+  if filename and File.exist?(filename)
+    # Read oar properties from file
+    puts "Read 'oarnodes -Y' from #{filename}"
+    oarnodes_yaml = File.open(filename, 'rb') { |f| f.read }
+  else
+    # Download the oar properties from the oar server
+    puts "Downloading 'oarnodes -Y' from oar.#{site_uid}.g5kadmin ..."
+
+    Net::SSH.start("oar.#{site_uid}.g5kadmin", 'g5kadmin', :keys => ['~/.ssh/id_rsa_g5k', '~/.ssh/id_rsa_g5kadmin.pub']) do |ssh|
+      # capture all stderr and stdout output from a remote process
+      oarnodes_yaml = ssh.exec!('oarnodes -Y')
+      
+      #  puts output  
+    end
+    puts "... done"
 
-def oarcmd_get_nodelist_properties()
-  # Get the current OAR properties from the OAR scheduler
-  h = YAML.load_file('../../JG/nancy-oarnodes.yaml')
+    if filename
+      # Cache the file
+      puts "Save 'oarnodes -Y' as #{filename}"
+      File.write(filename, oarnodes_yaml)
+    end
+  end
+
+  # Load the YAML file into an hashtable
+  h = YAML.load(oarnodes_yaml)
+
+  # Format convertion: use host as keys of the hash (instead of id)
   h = h.map {|k, v| v['type'] == 'default' ? [v['host'].split('.').first, v] : [nil, nil] }.to_h
+
   return h
 end
 

generators/oar-properties/oar-properties.rb

@@ -6,180 +6,127 @@ require 'fileutils'
 require 'pathname'
 require 'json'
 require 'time'
+require 'yaml'
+require 'hashdiff'
+require 'optparse'
+require 'net/ssh'
 
+require '../oar-properties/lib/lib-oar-properties'
 require '../lib/input_loader'
 
-# Output directory
-refapi_path = "/tmp/data"
+options = {}
+options[:sites] = %w{grenoble lille luxembourg lyon nantes reims rennes sophia}
+options[:diff]  = false
 
-#global_hash = JSON.parse(STDIN.read)
-#global_hash = load_yaml_file_hierarchy("../../input/example/")
-global_hash = load_yaml_file_hierarchy("../../input/grid5000/")
+OptionParser.new do |opts|
+  opts.banner = "Usage: oar-properties.rb [options]"
 
-class Hash
-  # sort a hash according to the position of the key in the array
-  def sort_by_array(array)
-    Hash[sort_by{|key, _| array.index(key) || length}] 
-  end
-end
+  opts.separator ""
+  opts.separator "Example: ruby oar-properties.rb -v -s nancy -d oarnodes-%s.yaml -o cmd-%s.sh"
+
+  opts.separator ""
+  opts.separator "Filters:"
+
+  ###
 
-# Write pretty and sorted JSON files
-def write_json(filepath, data)
-  def rec_sort(h)
-    case h
-    when Array
-      h.map{|v| rec_sort(v)}#.sort_by!{|v| (v.to_s rescue nil) }
-    when Hash
-      Hash[Hash[h.map{|k,v| [rec_sort(k),rec_sort(v)]}].sort_by{|k,v| [(k.to_s rescue nil), (v.to_s rescue nil)]}]
-    else
-      h
-    end
+  opts.on('-s', '--sites a,b,c', Array, 'Select site(s)',
+                                        "Default: "+options[:sites].join(", ")) do |s|
+    options[:sites] = s
   end
-  File.open(filepath, 'w') do |f|
-    f.write(JSON.pretty_generate(rec_sort(data)))
+
+  opts.on('-c', '--clusters a,b,c', Array, 'Select clusters(s). Default: all') do |s|
+    options[:clusters] = s
   end
-end
 
-# Parse network equipment description and return switch name and port connected to given node
-#  In the network description, if the node interface is given (using "port" attribute),
-#  the interface parameter must be used.
-def net_switch_port_lookup(site, node_uid, interface='')
-  site["net-links"].each do |switch_uid, switch| # TODO: rename net-links<->network
-    #pp switch_uid
-    switch["linecards"].each do |lc_uid,lc|
-      lc["ports"].each do |port_uid,port|
-        if port.is_a?(Hash)
-          switch_remote_port = port["port"] || lc["port"] || ""
-          switch_remote_uid = port["uid"]
-        else
-          switch_remote_port = lc["port"] || ""
-          switch_remote_uid = port
-        end
-        if switch_remote_uid == node_uid and switch_remote_port == interface
-            # Build port name from snmp_naming_pattern
-            # Example: '3 2 GigabitEthernet%LINECARD%/%PORT%' -> 'GigabitEthernet3/2'
-            port_name = lc["snmp_pattern"].sub("%LINECARD%",lc_uid.to_s).sub("%PORT%",port_uid.to_s)
-            return switch_uid, port_name
-        end
-      end
-    end
+  opts.on('-n', '--nodes a,b,c', Array, 'Select nodes(s). Default: all') do |n|
+    options[:nodes] = n
   end
-  return nil
-end
 
-global_hash["sites"].each do |site_uid, site|
-  pp site_uid
-#  pp site
+  ###
 
-  site["clusters"].each do |cluster_uid, cluster|
-    pp cluster_uid
+  opts.separator ""
+  opts.separator "Output options:"
 
-    cluster_path = Pathname.new(refapi_path).join("sites",site_uid,"clusters",cluster_uid)
-    cluster_path.join("nodes").mkpath()
+  opts.on('-o', '--output', 'Output oarnodesetting command into a file. Default: stdout') do |o|
+    options[:output] = o
+  end
 
-    # Write cluster info w/o nodes entries
-    cluster["type"] = "cluster"
-    cluster["uid"]  = cluster_uid
-    
-    # On the previous version of this script, cluster["created_ad"] was generated from a Ruby Time. cluster["created_ad"] is now a Ruby Date at JSON import.
-    # As Date.httpdate and Time.httpdate does not behave the same with timezone, it is converted here as a Ruby time.
-    cluster["created_at"] = Time.parse(cluster["created_at"].to_s).httpdate
-    
-    write_json(cluster_path.join("#{cluster_uid}.json"),
-               cluster.reject {|k, v| k == "nodes"})
+  opts.on('-e', '--exec', 'Directly apply the changes to the OAR server') do |e|
+    options[:exec] = e
+  end
+  
+  opts.on("-d", "--diff [YAML filename]", 
+          "Only generates the minimal list of commands needed to update the site configuration",
+          "The optional YAML file is suppose to be the output of the 'oarnodes -Y' command.",
+          "If the file does not exist, the script will get the data from the OAR server and save the result on disk for future use.",
+          "If no filename is specified, the script will simply connect to the OAR server.",
+          "You can use the '%s' placeholder for 'site'. Ex: oarnodes-%s.yaml") do |d|
+    d = true if d == nil
+    options[:diff] = d
+  end
     
-    # Write node info
-    cluster["nodes"].each do |node_uid, node|
-      pp node_uid
-
-#     next unless node_uid == "griffon-1"
-      node["uid"] = node_uid
-      node["type"] = "node"
-      if node.key?("processor")
-        node["processor"]["cache_l1"] = nil unless node["processor"].key?("cache_l1")
-      end
-      
-      # Add default keys
-      node["gpu"] = {} unless node.key?("gpu")
-      node["gpu"]["gpu"] = false unless node["gpu"].key?("gpu")
-      
-      node["main_memory"] = {} unless node.key?("main_memory")
-      node["main_memory"]["virtual_size"] = nil unless node["main_memory"].key?("virtual_size")
-
-#     node["monitoring"] = {}   unless node.key?("monitoring")
-#     node["monitoring"]["wattmeter"] = false unless node["monitoring"].key?("wattmeter")
-
-      # Rename keys
-      node["storage_devices"] = node.delete("block_devices")
-      node["network_adapters"] = node.delete("network_interfaces")
-      if node.key?("chassis")
-        node["chassis"]["name"]   = node["chassis"].delete("product_name")
-        node["chassis"]["serial"] = node["chassis"].delete("serial_number")
-      end
-      
-      # Type conversion
-      node["network_adapters"].each { |key, hash| hash["rate"] = hash["rate"].to_i if hash["rate"].is_a?(Float) }
-
-      # Convert hashes to arrays
-      node["storage_devices"].each { |key, hash| node["storage_devices"][key]["device"] = key; } # Add "device: sdX" within the hash
-      node["storage_devices"] = node["storage_devices"].sort_by_array(["sda", "sdb", "sdc", "sdd", "sde"]).values
-
-      node["network_adapters"].each { |key, hash| node["network_adapters"][key]["device"] = key; } # Add "device: ethX" within the hash
-      node["network_adapters"] = node["network_adapters"].sort_by_array(["eth0", "eth1", "eth2", "eth3", "ib0", "ib1", "ib2", "ib3", "bmc"]).values
-
-      # Populate "network_address", "switch" and "switch_port" from the network equipment description for each network adapters
-      node["network_adapters"].each { |network_adapter|
-
-        # ib properties
-        network_adapter["ib_switch_card"]     = network_adapter.delete("line_card") if network_adapter.key?("line_card")
-        network_adapter["ib_switch_card_pos"] = network_adapter.delete("position")  if network_adapter.key?("position")
-
-        next unless network_adapter["enabled"]
-
-        # Management network_adapter (bmc)
-        if network_adapter["management"]
-          network_adapter["network_address"] = "#{node_uid}-bmc.#{site_uid}.grid5000.fr"
-          next
-        end
-
-        # if network_adapter["network_address"] or network_adapter["switch"] or network_adapter["switch_port"]
-        #   pp "Warning: network_address, switch or switch_port defined manually for (#{node_uid}, #{network_adapter["device"]})"
-        #   pp "#{network_adapter["network_address"]}, #{network_adapter["switch"]} or #{network_adapter["switch_port"]}"
-        # end
-        
-        if network_adapter["mounted"] and /^eth[0-9]$/.match(network_adapter["device"])
-          # Primary network_adapter
-          network_adapter["network_address"] = "#{node_uid}.#{site_uid}.grid5000.fr"
-         
-          # Interface may not be specified in Network Reference for primary network_adapter
-          network_adapter["switch"], network_adapter["switch_port"] = net_switch_port_lookup(site, node_uid, network_adapter["device"]) || net_switch_port_lookup(site, node_uid)
-
-          # network_adapter["bridged"] = true # TODO?
-          next
-        end
-#        if network_adapter["bridged"]
-          # Secondary network_adapter(s)
-          network_adapter["network_address"] = "#{node_uid}-#{network_adapter["device"]}.#{site_uid}.grid5000.fr"
-          switch, port = net_switch_port_lookup(site, node_uid, network_adapter["device"])
-          network_adapter["switch"] = switch if switch
-          network_adapter["switch_port"] = port if port
-#        end
-      }
-
-      if node.key?("sensors") and node.key?("pdu") and node["pdu"].key?("pdu_name")
-        node["sensors"]["power"] = {}                  unless node["sensors"].key?("power")
-        node["sensors"]["power"]["via"] = {}           unless node["sensors"]["power"].key?("via")
-        node["sensors"]["power"]["via"]["pdu"] = []    unless node["sensors"]["power"]["via"].key?("pdu")
-        node["sensors"]["power"]["via"]["pdu"][0] = {} unless node["sensors"]["power"]["via"]["pdu"].size > 0
-        
-        node["sensors"]["power"]["via"]["pdu"][0]["uid"]  = node["pdu"]["pdu_name"]
-        node["sensors"]["power"]["via"]["pdu"][0]["port"] = node["pdu"]["pdu_position"] if  node["pdu"].key?("pdu_position")
-
-        node.delete("pdu")
-      end
-
-      #pp cluster_path.join("nodes","#{node_uid}.json")
-      write_json(cluster_path.join("nodes","#{node_uid}.json"), node)
-    end
+#  opts.on("-n", "--dry-run", "Perform a trial run with no changes made") do |n|
+#    options[:dryrun] = n
+#  end
+
+  ###
+
+  opts.separator ""
+  opts.separator "Common options:"
+
+  opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+    options[:verbose] = v
   end
+  
+  # Print an options summary.
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+pp options
+
+# Option
+site_uid = 'nancy'
+
+nodelist_properties = {}
+
+#
+# Get the OAR properties from the reference-repo
+#
+
+global_hash = load_yaml_file_hierarchy('../../input/grid5000/')
+nodelist_properties["ref"] = get_nodelist_properties(site_uid, global_hash["sites"][site_uid])
+
+#
+# Get the current OAR properties from the OAR scheduler
+# This is only needed for the -d option
+#
+
+nodelist_properties["oar"] = {}
+
+if options[:diff]
+  filename = options[:diff].is_a?(String) ? options[:diff].gsub("%s", site_uid) : nil
+  nodelist_properties["oar"] = oarcmd_get_nodelist_properties(site_uid, filename)
 end
+
+
+#
+# Diff
+#
+
+node_properties = {}
+node_properties["ref"] = nodelist_properties["ref"]["graphene-1"]
+node_properties["oar"] = nodelist_properties["oar"]["graphene-1"]
+
+diff      = diff_node_properties(node_properties["ref"], node_properties["oar"])
+diff_keys = diff.map{ |hashdiff_array| hashdiff_array[1] }
+
+node_properties["to_be_updated"] = node_properties["ref"].select { |key, value| diff_keys.include?(key) }
+
+#
+# Example
+#
+puts oarcmd_set_node_properties("graphene-1", node_properties["oar"])
+puts oarcmd_set_node_properties("graphene-1", node_properties["to_be_updated"])