Commit dfdcf2a2 authored by Jérémie Gaidamour's avatar Jérémie Gaidamour
Browse files

[dev] Sort network interfaces and storage devices, Manage Sensors,...

[dev] Sort network interfaces and storage devices, Manage Sensors, <cluster>.yaml generation, GPUs, new version of the code for getting network adapter info ...
parent 5438fa9d
......@@ -5,6 +5,7 @@ require 'erb'
require 'fileutils'
require 'pathname'
require 'json'
require 'time'
require '../lib/input_loader'
......@@ -12,14 +13,22 @@ require '../lib/input_loader'
refapi_path = "/tmp/data"
#global_hash = JSON.parse(STDIN.read)
global_hash = load_yaml_file_hierarchy("../../input/example/")
#global_hash = load_yaml_file_hierarchy("../../input/example/")
global_hash = load_yaml_file_hierarchy("../../input/grid5000/")
class Hash
# sort a hash according to the position of the key in the array
def sort_by_array(array)
Hash[sort_by{|key, _| array.index(key) || length}]
end
end
# Write pretty and sorted JSON files
def write_json(filepath, data)
def rec_sort(h)
case h
when Array
h.map{|v| rec_sort(v)}.sort_by!{|v| (v.to_s rescue nil) }
h.map{|v| rec_sort(v)}#.sort_by!{|v| (v.to_s rescue nil) }
when Hash
Hash[Hash[h.map{|k,v| [rec_sort(k),rec_sort(v)]}].sort_by{|k,v| [(k.to_s rescue nil), (v.to_s rescue nil)]}]
else
......@@ -35,7 +44,8 @@ end
# In the network description, if the node interface is given (using "port" attribute),
# the interface parameter must be used.
def net_switch_port_lookup(site, node_uid, interface='')
site["network"].each do |switch_uid, switch|
site["net-links"].each do |switch_uid, switch| # TODO: rename net-links<->network
#pp switch_uid
switch["linecards"].each do |lc_uid,lc|
lc["ports"].each do |port_uid,port|
if port.is_a?(Hash)
......@@ -58,20 +68,46 @@ def net_switch_port_lookup(site, node_uid, interface='')
end
global_hash["sites"].each do |site_uid, site|
pp site_uid
# pp site
site["clusters"].each do |cluster_uid, cluster|
pp cluster_uid
cluster_path = Pathname.new(refapi_path).join("sites",site_uid,"clusters",cluster_uid)
cluster_path.join("nodes").mkpath()
# Write cluster info w/o nodes entries
cluster["type"] = "cluster"
cluster["uid"] = cluster_uid
# On the previous version of this script, cluster["created_ad"] was generated from a Ruby Time. cluster["created_ad"] is now a Ruby Date at JSON import.
# As Date.httpdate and Time.httpdate does not behave the same with timezone, it is converted here as a Ruby time.
cluster["created_at"] = Time.parse(cluster["created_at"].to_s).httpdate
write_json(cluster_path.join("#{cluster_uid}.json"),
cluster.reject {|k, v| k == "nodes"})
# Write node info
cluster["nodes"].each do |node_uid, node|
pp node_uid
# next unless node_uid == "griffon-1"
node["uid"] = node_uid
node["type"] = "node"
if node.key?("processor")
node["processor"]["cache_l1"] = nil unless node["processor"].key?("cache_l1")
end
# Add default keys
node["gpu"] = {} unless node.key?("gpu")
node["gpu"]["gpu"] = false unless node["gpu"].key?("gpu")
node["main_memory"] = {} unless node.key?("main_memory")
node["main_memory"]["virtual_size"] = nil unless node["main_memory"].key?("virtual_size")
# node["monitoring"] = {} unless node.key?("monitoring")
# node["monitoring"]["wattmeter"] = false unless node["monitoring"].key?("wattmeter")
# Rename keys
node["storage_devices"] = node.delete("block_devices")
......@@ -81,31 +117,67 @@ global_hash["sites"].each do |site_uid, site|
node["chassis"]["serial"] = node["chassis"].delete("serial_number")
end
# Type conversion
node["network_adapters"].each { |key, hash| hash["rate"] = hash["rate"].to_i if hash["rate"].is_a?(Float) }
# Convert hashes to arrays
node["storage_devices"] = node["storage_devices"].values
node["network_adapters"] = node["network_adapters"].values
node["storage_devices"].each { |key, hash| node["storage_devices"][key]["device"] = key; } # Add "device: sdX" within the hash
node["storage_devices"] = node["storage_devices"].sort_by_array(["sda", "sdb", "sdc", "sdd", "sde"]).values
node["network_adapters"].each { |key, hash| node["network_adapters"][key]["device"] = key; } # Add "device: ethX" within the hash
node["network_adapters"] = node["network_adapters"].sort_by_array(["eth0", "eth1", "eth2", "eth3", "ib0", "ib1", "ib2", "ib3", "bmc"]).values
# Populate "network_address", "switch" and "port" from the network equipment description for each network adapters
# Populate "network_address", "switch" and "switch_port" from the network equipment description for each network adapters
node["network_adapters"].each { |network_adapter|
if network_adapter["enabled"]
if network_adapter["management"]
# Management network_adapter
network_adapter["network_address"] = "#{node_uid}-bmc.#{site_uid}.grid5000.fr"
elsif network_adapter["mounted"]
# Primary network_adapter
network_adapter["bridged"] = true
network_adapter["network_address"] = "#{node_uid}.#{site_uid}.grid5000.fr"
# Interface may not be specified in Network Reference for primary network_adapter
network_adapter["switch"], network_adapter["port"] = \
net_switch_port_lookup(site, node_uid) || net_switch_port_lookup(site, node_uid, network_adapter["device"])
else
# Secondary network_adapter(s)
network_adapter["network_address"] = "#{node_uid}-#{network_adapter["device"]}.#{site}.grid5000.fr"
network_adapter["switch"], network_adapter["port"] = net_switch_port_lookup(site, node_uid, network_adapter["device"])
end
# ib properties
network_adapter["ib_switch_card"] = network_adapter.delete("line_card") if network_adapter.key?("line_card")
network_adapter["ib_switch_card_pos"] = network_adapter.delete("position") if network_adapter.key?("position")
next unless network_adapter["enabled"]
# Management network_adapter (bmc)
if network_adapter["management"]
network_adapter["network_address"] = "#{node_uid}-bmc.#{site_uid}.grid5000.fr"
next
end
# if network_adapter["network_address"] or network_adapter["switch"] or network_adapter["switch_port"]
# pp "Warning: network_address, switch or switch_port defined manually for (#{node_uid}, #{network_adapter["device"]})"
# pp "#{network_adapter["network_address"]}, #{network_adapter["switch"]} or #{network_adapter["switch_port"]}"
# end
if network_adapter["mounted"] and /^eth[0-9]$/.match(network_adapter["device"])
# Primary network_adapter
network_adapter["network_address"] = "#{node_uid}.#{site_uid}.grid5000.fr"
# Interface may not be specified in Network Reference for primary network_adapter
network_adapter["switch"], network_adapter["switch_port"] = net_switch_port_lookup(site, node_uid, network_adapter["device"]) || net_switch_port_lookup(site, node_uid)
# network_adapter["bridged"] = true # TODO?
next
end
# if network_adapter["bridged"]
# Secondary network_adapter(s)
network_adapter["network_address"] = "#{node_uid}-#{network_adapter["device"]}.#{site_uid}.grid5000.fr"
switch, port = net_switch_port_lookup(site, node_uid, network_adapter["device"])
network_adapter["switch"] = switch if switch
network_adapter["switch_port"] = port if port
# end
}
if node.key?("sensors") and node.key?("pdu") and node["pdu"].key?("pdu_name")
node["sensors"]["power"] = {} unless node["sensors"].key?("power")
node["sensors"]["power"]["via"] = {} unless node["sensors"]["power"].key?("via")
node["sensors"]["power"]["via"]["pdu"] = [] unless node["sensors"]["power"]["via"].key?("pdu")
node["sensors"]["power"]["via"]["pdu"][0] = {} unless node["sensors"]["power"]["via"]["pdu"].size > 0
node["sensors"]["power"]["via"]["pdu"][0]["uid"] = node["pdu"]["pdu_name"]
node["sensors"]["power"]["via"]["pdu"][0]["port"] = node["pdu"]["pdu_position"] if node["pdu"].key?("pdu_position")
node.delete("pdu")
end
#pp cluster_path.join("nodes","#{node_uid}.json")
write_json(cluster_path.join("nodes","#{node_uid}.json"), node)
end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment