diff --git a/generators/oar-properties/lib/lib-oar-properties.rb b/generators/oar-properties/lib/lib-oar-properties.rb
index 620c4847b8d3cff67310f1e793fe2dba68315f64..dec454aabdc669f24371bca36e9828951f5e2a11 100755
--- a/generators/oar-properties/lib/lib-oar-properties.rb
+++ b/generators/oar-properties/lib/lib-oar-properties.rb
@@ -92,7 +92,7 @@ def get_node_properties(cluster_uid, cluster, node_uid, node)
end
h['max_walltime'] = node['supported_job_types']['max_walltime'] || 0
- h
+ return h
end
#
@@ -190,11 +190,42 @@ def oarcmd_set_node_properties(host, properties)
return command
end
+# '
+
+# Get the OAR properties from the OAR scheduler
+# This is only needed for the -d option
+def oarcmd_get_nodelist_properties(site_uid, filename=nil)
+ oarnodes_yaml = ""
+
+ if filename and File.exist?(filename)
+ # Read oar properties from file
+ puts "Read 'oarnodes -Y' from #{filename}"
+ oarnodes_yaml = File.open(filename, 'rb') { |f| f.read }
+ else
+ # Download the oar properties from the oar server
+ puts "Downloading 'oarnodes -Y' from oar.#{site_uid}.g5kadmin ..."
+
+ Net::SSH.start("oar.#{site_uid}.g5kadmin", 'g5kadmin', :keys => ['~/.ssh/id_rsa_g5k', '~/.ssh/id_rsa_g5kadmin.pub']) do |ssh|
+ # capture all stderr and stdout output from a remote process
+ oarnodes_yaml = ssh.exec!('oarnodes -Y')
+
+ # puts output
+ end
+ puts "... done"
-def oarcmd_get_nodelist_properties()
- # Get the current OAR properties from the OAR scheduler
- h = YAML.load_file('../../JG/nancy-oarnodes.yaml')
+ if filename
+ # Cache the file
+ puts "Save 'oarnodes -Y' as #{filename}"
+ File.write(filename, oarnodes_yaml)
+ end
+ end
+
+ # Load the YAML file into an hashtable
+ h = YAML.load(oarnodes_yaml)
+
+ # Format convertion: use host as keys of the hash (instead of id)
h = h.map {|k, v| v['type'] == 'default' ? [v['host'].split('.').first, v] : [nil, nil] }.to_h
+
return h
end
diff --git a/generators/oar-properties/oar-properties.rb b/generators/oar-properties/oar-properties.rb
index 58806c96bbdb1dac8ceff01afbc584b91720dbeb..50ace66d44e1a2b8d6d41466350e749b70664ed0 100755
--- a/generators/oar-properties/oar-properties.rb
+++ b/generators/oar-properties/oar-properties.rb
@@ -6,180 +6,127 @@ require 'fileutils'
require 'pathname'
require 'json'
require 'time'
+require 'yaml'
+require 'hashdiff'
+require 'optparse'
+require 'net/ssh'
+require '../oar-properties/lib/lib-oar-properties'
require '../lib/input_loader'
-# Output directory
-refapi_path = "/tmp/data"
+options = {}
+options[:sites] = %w{grenoble lille luxembourg lyon nantes reims rennes sophia}
+options[:diff] = false
-#global_hash = JSON.parse(STDIN.read)
-#global_hash = load_yaml_file_hierarchy("../../input/example/")
-global_hash = load_yaml_file_hierarchy("../../input/grid5000/")
+OptionParser.new do |opts|
+ opts.banner = "Usage: oar-properties.rb [options]"
-class Hash
- # sort a hash according to the position of the key in the array
- def sort_by_array(array)
- Hash[sort_by{|key, _| array.index(key) || length}]
- end
-end
+ opts.separator ""
+ opts.separator "Example: ruby oar-properties.rb -v -s nancy -d oarnodes-%s.yaml -o cmd-%s.sh"
+
+ opts.separator ""
+ opts.separator "Filters:"
+
+ ###
-# Write pretty and sorted JSON files
-def write_json(filepath, data)
- def rec_sort(h)
- case h
- when Array
- h.map{|v| rec_sort(v)}#.sort_by!{|v| (v.to_s rescue nil) }
- when Hash
- Hash[Hash[h.map{|k,v| [rec_sort(k),rec_sort(v)]}].sort_by{|k,v| [(k.to_s rescue nil), (v.to_s rescue nil)]}]
- else
- h
- end
+ opts.on('-s', '--sites a,b,c', Array, 'Select site(s)',
+ "Default: "+options[:sites].join(", ")) do |s|
+ options[:sites] = s
end
- File.open(filepath, 'w') do |f|
- f.write(JSON.pretty_generate(rec_sort(data)))
+
+ opts.on('-c', '--clusters a,b,c', Array, 'Select clusters(s). Default: all') do |s|
+ options[:clusters] = s
end
-end
-# Parse network equipment description and return switch name and port connected to given node
-# In the network description, if the node interface is given (using "port" attribute),
-# the interface parameter must be used.
-def net_switch_port_lookup(site, node_uid, interface='')
- site["net-links"].each do |switch_uid, switch| # TODO: rename net-links<->network
- #pp switch_uid
- switch["linecards"].each do |lc_uid,lc|
- lc["ports"].each do |port_uid,port|
- if port.is_a?(Hash)
- switch_remote_port = port["port"] || lc["port"] || ""
- switch_remote_uid = port["uid"]
- else
- switch_remote_port = lc["port"] || ""
- switch_remote_uid = port
- end
- if switch_remote_uid == node_uid and switch_remote_port == interface
- # Build port name from snmp_naming_pattern
- # Example: '3 2 GigabitEthernet%LINECARD%/%PORT%' -> 'GigabitEthernet3/2'
- port_name = lc["snmp_pattern"].sub("%LINECARD%",lc_uid.to_s).sub("%PORT%",port_uid.to_s)
- return switch_uid, port_name
- end
- end
- end
+ opts.on('-n', '--nodes a,b,c', Array, 'Select nodes(s). Default: all') do |n|
+ options[:nodes] = n
end
- return nil
-end
-global_hash["sites"].each do |site_uid, site|
- pp site_uid
-# pp site
+ ###
- site["clusters"].each do |cluster_uid, cluster|
- pp cluster_uid
+ opts.separator ""
+ opts.separator "Output options:"
- cluster_path = Pathname.new(refapi_path).join("sites",site_uid,"clusters",cluster_uid)
- cluster_path.join("nodes").mkpath()
+ opts.on('-o', '--output', 'Output oarnodesetting command into a file. Default: stdout') do |o|
+ options[:output] = o
+ end
- # Write cluster info w/o nodes entries
- cluster["type"] = "cluster"
- cluster["uid"] = cluster_uid
-
- # On the previous version of this script, cluster["created_ad"] was generated from a Ruby Time. cluster["created_ad"] is now a Ruby Date at JSON import.
- # As Date.httpdate and Time.httpdate does not behave the same with timezone, it is converted here as a Ruby time.
- cluster["created_at"] = Time.parse(cluster["created_at"].to_s).httpdate
-
- write_json(cluster_path.join("#{cluster_uid}.json"),
- cluster.reject {|k, v| k == "nodes"})
+ opts.on('-e', '--exec', 'Directly apply the changes to the OAR server') do |e|
+ options[:exec] = e
+ end
+
+ opts.on("-d", "--diff [YAML filename]",
+ "Only generates the minimal list of commands needed to update the site configuration",
+ "The optional YAML file is suppose to be the output of the 'oarnodes -Y' command.",
+ "If the file does not exist, the script will get the data from the OAR server and save the result on disk for future use.",
+ "If no filename is specified, the script will simply connect to the OAR server.",
+ "You can use the '%s' placeholder for 'site'. Ex: oarnodes-%s.yaml") do |d|
+ d = true if d == nil
+ options[:diff] = d
+ end
- # Write node info
- cluster["nodes"].each do |node_uid, node|
- pp node_uid
-
-# next unless node_uid == "griffon-1"
- node["uid"] = node_uid
- node["type"] = "node"
- if node.key?("processor")
- node["processor"]["cache_l1"] = nil unless node["processor"].key?("cache_l1")
- end
-
- # Add default keys
- node["gpu"] = {} unless node.key?("gpu")
- node["gpu"]["gpu"] = false unless node["gpu"].key?("gpu")
-
- node["main_memory"] = {} unless node.key?("main_memory")
- node["main_memory"]["virtual_size"] = nil unless node["main_memory"].key?("virtual_size")
-
-# node["monitoring"] = {} unless node.key?("monitoring")
-# node["monitoring"]["wattmeter"] = false unless node["monitoring"].key?("wattmeter")
-
- # Rename keys
- node["storage_devices"] = node.delete("block_devices")
- node["network_adapters"] = node.delete("network_interfaces")
- if node.key?("chassis")
- node["chassis"]["name"] = node["chassis"].delete("product_name")
- node["chassis"]["serial"] = node["chassis"].delete("serial_number")
- end
-
- # Type conversion
- node["network_adapters"].each { |key, hash| hash["rate"] = hash["rate"].to_i if hash["rate"].is_a?(Float) }
-
- # Convert hashes to arrays
- node["storage_devices"].each { |key, hash| node["storage_devices"][key]["device"] = key; } # Add "device: sdX" within the hash
- node["storage_devices"] = node["storage_devices"].sort_by_array(["sda", "sdb", "sdc", "sdd", "sde"]).values
-
- node["network_adapters"].each { |key, hash| node["network_adapters"][key]["device"] = key; } # Add "device: ethX" within the hash
- node["network_adapters"] = node["network_adapters"].sort_by_array(["eth0", "eth1", "eth2", "eth3", "ib0", "ib1", "ib2", "ib3", "bmc"]).values
-
- # Populate "network_address", "switch" and "switch_port" from the network equipment description for each network adapters
- node["network_adapters"].each { |network_adapter|
-
- # ib properties
- network_adapter["ib_switch_card"] = network_adapter.delete("line_card") if network_adapter.key?("line_card")
- network_adapter["ib_switch_card_pos"] = network_adapter.delete("position") if network_adapter.key?("position")
-
- next unless network_adapter["enabled"]
-
- # Management network_adapter (bmc)
- if network_adapter["management"]
- network_adapter["network_address"] = "#{node_uid}-bmc.#{site_uid}.grid5000.fr"
- next
- end
-
- # if network_adapter["network_address"] or network_adapter["switch"] or network_adapter["switch_port"]
- # pp "Warning: network_address, switch or switch_port defined manually for (#{node_uid}, #{network_adapter["device"]})"
- # pp "#{network_adapter["network_address"]}, #{network_adapter["switch"]} or #{network_adapter["switch_port"]}"
- # end
-
- if network_adapter["mounted"] and /^eth[0-9]$/.match(network_adapter["device"])
- # Primary network_adapter
- network_adapter["network_address"] = "#{node_uid}.#{site_uid}.grid5000.fr"
-
- # Interface may not be specified in Network Reference for primary network_adapter
- network_adapter["switch"], network_adapter["switch_port"] = net_switch_port_lookup(site, node_uid, network_adapter["device"]) || net_switch_port_lookup(site, node_uid)
-
- # network_adapter["bridged"] = true # TODO?
- next
- end
-# if network_adapter["bridged"]
- # Secondary network_adapter(s)
- network_adapter["network_address"] = "#{node_uid}-#{network_adapter["device"]}.#{site_uid}.grid5000.fr"
- switch, port = net_switch_port_lookup(site, node_uid, network_adapter["device"])
- network_adapter["switch"] = switch if switch
- network_adapter["switch_port"] = port if port
-# end
- }
-
- if node.key?("sensors") and node.key?("pdu") and node["pdu"].key?("pdu_name")
- node["sensors"]["power"] = {} unless node["sensors"].key?("power")
- node["sensors"]["power"]["via"] = {} unless node["sensors"]["power"].key?("via")
- node["sensors"]["power"]["via"]["pdu"] = [] unless node["sensors"]["power"]["via"].key?("pdu")
- node["sensors"]["power"]["via"]["pdu"][0] = {} unless node["sensors"]["power"]["via"]["pdu"].size > 0
-
- node["sensors"]["power"]["via"]["pdu"][0]["uid"] = node["pdu"]["pdu_name"]
- node["sensors"]["power"]["via"]["pdu"][0]["port"] = node["pdu"]["pdu_position"] if node["pdu"].key?("pdu_position")
-
- node.delete("pdu")
- end
-
- #pp cluster_path.join("nodes","#{node_uid}.json")
- write_json(cluster_path.join("nodes","#{node_uid}.json"), node)
- end
+# opts.on("-n", "--dry-run", "Perform a trial run with no changes made") do |n|
+# options[:dryrun] = n
+# end
+
+ ###
+
+ opts.separator ""
+ opts.separator "Common options:"
+
+ opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+ options[:verbose] = v
end
+
+ # Print an options summary.
+ opts.on_tail("-h", "--help", "Show this message") do
+ puts opts
+ exit
+ end
+end.parse!
+
+pp options
+
+# Option
+site_uid = 'nancy'
+
+nodelist_properties = {}
+
+#
+# Get the OAR properties from the reference-repo
+#
+
+global_hash = load_yaml_file_hierarchy('../../input/grid5000/')
+nodelist_properties["ref"] = get_nodelist_properties(site_uid, global_hash["sites"][site_uid])
+
+#
+# Get the current OAR properties from the OAR scheduler
+# This is only needed for the -d option
+#
+
+nodelist_properties["oar"] = {}
+
+if options[:diff]
+ filename = options[:diff].is_a?(String) ? options[:diff].gsub("%s", site_uid) : nil
+ nodelist_properties["oar"] = oarcmd_get_nodelist_properties(site_uid, filename)
end
+
+
+#
+# Diff
+#
+
+node_properties = {}
+node_properties["ref"] = nodelist_properties["ref"]["graphene-1"]
+node_properties["oar"] = nodelist_properties["oar"]["graphene-1"]
+
+diff = diff_node_properties(node_properties["ref"], node_properties["oar"])
+diff_keys = diff.map{ |hashdiff_array| hashdiff_array[1] }
+
+node_properties["to_be_updated"] = node_properties["ref"].select { |key, value| diff_keys.include?(key) }
+
+#
+# Example
+#
+puts oarcmd_set_node_properties("graphene-1", node_properties["oar"])
+puts oarcmd_set_node_properties("graphene-1", node_properties["to_be_updated"])