oar-properties.rb 7.56 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
#!/usr/bin/ruby

require 'pp'
require 'erb'
require 'fileutils'
require 'pathname'
require 'json'
require 'time'

require '../lib/input_loader'

# Output directory
refapi_path = "/tmp/data"

#global_hash = JSON.parse(STDIN.read)
#global_hash = load_yaml_file_hierarchy("../../input/example/")
global_hash = load_yaml_file_hierarchy("../../input/grid5000/")

class Hash
  # sort a hash according to the position of the key in the array
  def sort_by_array(array)
    Hash[sort_by{|key, _| array.index(key) || length}] 
  end
end

# Write pretty and sorted JSON files
def write_json(filepath, data)
  def rec_sort(h)
    case h
    when Array
      h.map{|v| rec_sort(v)}#.sort_by!{|v| (v.to_s rescue nil) }
    when Hash
      Hash[Hash[h.map{|k,v| [rec_sort(k),rec_sort(v)]}].sort_by{|k,v| [(k.to_s rescue nil), (v.to_s rescue nil)]}]
    else
      h
    end
  end
  File.open(filepath, 'w') do |f|
    f.write(JSON.pretty_generate(rec_sort(data)))
  end
end

# Parse network equipment description and return switch name and port connected to given node
#  In the network description, if the node interface is given (using "port" attribute),
#  the interface parameter must be used.
def net_switch_port_lookup(site, node_uid, interface='')
  site["net-links"].each do |switch_uid, switch| # TODO: rename net-links<->network
    #pp switch_uid
    switch["linecards"].each do |lc_uid,lc|
      lc["ports"].each do |port_uid,port|
        if port.is_a?(Hash)
          switch_remote_port = port["port"] || lc["port"] || ""
          switch_remote_uid = port["uid"]
        else
          switch_remote_port = lc["port"] || ""
          switch_remote_uid = port
        end
        if switch_remote_uid == node_uid and switch_remote_port == interface
            # Build port name from snmp_naming_pattern
            # Example: '3 2 GigabitEthernet%LINECARD%/%PORT%' -> 'GigabitEthernet3/2'
            port_name = lc["snmp_pattern"].sub("%LINECARD%",lc_uid.to_s).sub("%PORT%",port_uid.to_s)
            return switch_uid, port_name
        end
      end
    end
  end
  return nil
end

global_hash["sites"].each do |site_uid, site|
  pp site_uid
#  pp site

  site["clusters"].each do |cluster_uid, cluster|
    pp cluster_uid

    cluster_path = Pathname.new(refapi_path).join("sites",site_uid,"clusters",cluster_uid)
    cluster_path.join("nodes").mkpath()

    # Write cluster info w/o nodes entries
    cluster["type"] = "cluster"
    cluster["uid"]  = cluster_uid
    
    # On the previous version of this script, cluster["created_ad"] was generated from a Ruby Time. cluster["created_ad"] is now a Ruby Date at JSON import.
    # As Date.httpdate and Time.httpdate does not behave the same with timezone, it is converted here as a Ruby time.
    cluster["created_at"] = Time.parse(cluster["created_at"].to_s).httpdate
    
    write_json(cluster_path.join("#{cluster_uid}.json"),
               cluster.reject {|k, v| k == "nodes"})
    
    # Write node info
    cluster["nodes"].each do |node_uid, node|
      pp node_uid

#     next unless node_uid == "griffon-1"
      node["uid"] = node_uid
      node["type"] = "node"
      if node.key?("processor")
        node["processor"]["cache_l1"] = nil unless node["processor"].key?("cache_l1")
      end
      
      # Add default keys
      node["gpu"] = {} unless node.key?("gpu")
      node["gpu"]["gpu"] = false unless node["gpu"].key?("gpu")
      
      node["main_memory"] = {} unless node.key?("main_memory")
      node["main_memory"]["virtual_size"] = nil unless node["main_memory"].key?("virtual_size")

#     node["monitoring"] = {}   unless node.key?("monitoring")
#     node["monitoring"]["wattmeter"] = false unless node["monitoring"].key?("wattmeter")

      # Rename keys
      node["storage_devices"] = node.delete("block_devices")
      node["network_adapters"] = node.delete("network_interfaces")
      if node.key?("chassis")
        node["chassis"]["name"]   = node["chassis"].delete("product_name")
        node["chassis"]["serial"] = node["chassis"].delete("serial_number")
      end
      
      # Type conversion
      node["network_adapters"].each { |key, hash| hash["rate"] = hash["rate"].to_i if hash["rate"].is_a?(Float) }

      # Convert hashes to arrays
      node["storage_devices"].each { |key, hash| node["storage_devices"][key]["device"] = key; } # Add "device: sdX" within the hash
      node["storage_devices"] = node["storage_devices"].sort_by_array(["sda", "sdb", "sdc", "sdd", "sde"]).values

      node["network_adapters"].each { |key, hash| node["network_adapters"][key]["device"] = key; } # Add "device: ethX" within the hash
      node["network_adapters"] = node["network_adapters"].sort_by_array(["eth0", "eth1", "eth2", "eth3", "ib0", "ib1", "ib2", "ib3", "bmc"]).values

      # Populate "network_address", "switch" and "switch_port" from the network equipment description for each network adapters
      node["network_adapters"].each { |network_adapter|

        # ib properties
        network_adapter["ib_switch_card"]     = network_adapter.delete("line_card") if network_adapter.key?("line_card")
        network_adapter["ib_switch_card_pos"] = network_adapter.delete("position")  if network_adapter.key?("position")

        next unless network_adapter["enabled"]

        # Management network_adapter (bmc)
        if network_adapter["management"]
          network_adapter["network_address"] = "#{node_uid}-bmc.#{site_uid}.grid5000.fr"
          next
        end

        # if network_adapter["network_address"] or network_adapter["switch"] or network_adapter["switch_port"]
        #   pp "Warning: network_address, switch or switch_port defined manually for (#{node_uid}, #{network_adapter["device"]})"
        #   pp "#{network_adapter["network_address"]}, #{network_adapter["switch"]} or #{network_adapter["switch_port"]}"
        # end
        
        if network_adapter["mounted"] and /^eth[0-9]$/.match(network_adapter["device"])
          # Primary network_adapter
          network_adapter["network_address"] = "#{node_uid}.#{site_uid}.grid5000.fr"
         
          # Interface may not be specified in Network Reference for primary network_adapter
          network_adapter["switch"], network_adapter["switch_port"] = net_switch_port_lookup(site, node_uid, network_adapter["device"]) || net_switch_port_lookup(site, node_uid)

          # network_adapter["bridged"] = true # TODO?
          next
        end
#        if network_adapter["bridged"]
          # Secondary network_adapter(s)
          network_adapter["network_address"] = "#{node_uid}-#{network_adapter["device"]}.#{site_uid}.grid5000.fr"
          switch, port = net_switch_port_lookup(site, node_uid, network_adapter["device"])
          network_adapter["switch"] = switch if switch
          network_adapter["switch_port"] = port if port
#        end
      }

      if node.key?("sensors") and node.key?("pdu") and node["pdu"].key?("pdu_name")
        node["sensors"]["power"] = {}                  unless node["sensors"].key?("power")
        node["sensors"]["power"]["via"] = {}           unless node["sensors"]["power"].key?("via")
        node["sensors"]["power"]["via"]["pdu"] = []    unless node["sensors"]["power"]["via"].key?("pdu")
        node["sensors"]["power"]["via"]["pdu"][0] = {} unless node["sensors"]["power"]["via"]["pdu"].size > 0
        
        node["sensors"]["power"]["via"]["pdu"][0]["uid"]  = node["pdu"]["pdu_name"]
        node["sensors"]["power"]["via"]["pdu"][0]["port"] = node["pdu"]["pdu_position"] if  node["pdu"].key?("pdu_position")

        node.delete("pdu")
      end

      #pp cluster_path.join("nodes","#{node_uid}.json")
      write_json(cluster_path.join("nodes","#{node_uid}.json"), node)
    end
  end
end