[dev] Prototype written by Simon for the new version of the generator

README

+To get JSON input data:
+  ./generator/input_loader.rb
+
+To generate Ref API data in /tmp/data:
+  ./generator/input_loader.rb | ./generators/reference-api/reference-api.rb

generators/input_loader.rb

+#!/usr/bin/ruby
+
+require 'pathname'
+require 'yaml'
+require 'json'
+
+# Extend Hash with helper methods needed to convert input data files to ruby Hash
+class ::Hash
+
+  # Recursively merge this Hash with another
+  def deep_merge(second)
+    merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
+    self.merge(second, &merger)
+  end
+
+  # Merge keys that match "PREFIX-<a-b>" with others keys that begins by
+  #   "PREFIX-" and that ends with x, where a<=x<=b
+  # a and/or b may be ommited, meaning that there are no lower and/or upper bound for x
+  # This is done recursively (for this Hash and every Hashes it contains)
+  #  i.e.:
+  # {"foo-1": {a: 0}, "foo-2": {a: 0}, "foo-3": {a: 0}, "foo-<2->": {b: 1}}.kbracket_merge()
+  #  -> {"foo-2": {a: 0, b:1},  "foo-3": {a: 0, b: 0}}
+  # TODO: only "<->" suffix is currently implemented
+  def kbracket_merge()
+    self.each { |k_all, v_all|
+      if k_all.to_s.end_with?('-<->') and v_all.is_a?(Hash)
+        key_prefix_to_merge = k_all.to_s.sub(/-<->$/,'')
+        self.each { |k, v|
+          if k.to_s.start_with?(key_prefix_to_merge)
+            self[k] = v_all.deep_merge(v)
+          end
+        }
+        self.delete(k_all)
+      end
+    }
+    self.each { |k, v|
+      if v.is_a?(Hash)
+        v.kbracket_merge()
+      end
+    }
+  end
+
+  # Add an element composed of nested Hashes made from elements found in "array" argument
+  # i.e.: from_array([a, b, c],"foo") -> {a: {b: {c: "foo"}}}
+  def self.from_array(array, value)
+    return array.reverse.inject(value) { |a, n| { n => a } }
+  end
+end
+
+#TODO: Ensure that deepest elements in input files hierarchy have lowest priority
+data = Hash.new
+
+Dir.chdir("input")
+Dir['**/*.y*ml'].each { |f|
+   node_path = Pathname.new(f)
+   node_dir, _ = node_path.split()
+
+   node_hierarchy = node_dir.to_s.split('/')
+   node_value = YAML::load_file(node_path)
+
+   node_data = Hash.from_array(node_hierarchy,node_value)
+   data = data.deep_merge(node_data)
+}
+data.kbracket_merge()
+
+puts JSON.generate(data)
+

generators/reference-api/reference-api.rb

+#!/usr/bin/ruby
+
+require 'pathname'
+require 'json'
+
+refapi_path = "/tmp/data"
+data = JSON.parse(STDIN.read)
+
+# Write pretty and sorted JSON files
+def write_json(filepath, data)
+  def rec_sort(h)
+    case h
+    when Array
+      h.map{|v| rec_sort(v)}.sort_by!{|v| (v.to_s rescue nil) }
+    when Hash
+      Hash[Hash[h.map{|k,v| [rec_sort(k),rec_sort(v)]}].sort_by{|k,v| [(k.to_s rescue nil), (v.to_s rescue nil)]}]
+    else
+      h
+    end
+  end
+  File.open(filepath, 'w') do |f|
+    f.write(JSON.pretty_generate(rec_sort(data)))
+  end
+end
+
+# Parse network equipment description and return switch name and port connected to given node
+#  In the network description, if the node interface is given (using "port" attribute),
+#  the interface parameter must be used.
+def net_switch_port_lookup(site, node_uid, interface='')
+  site["network"].each do |switch_uid, switch|
+    switch["linecards"].each do |lc_uid,lc|
+      lc["ports"].each do |port_uid,port|
+        if port.is_a?(Hash)
+          switch_remote_port = port["port"] || lc["port"] || ""
+          switch_remote_uid = port["uid"]
+        else
+          switch_remote_port = lc["port"] || ""
+          switch_remote_uid = port
+        end
+        if switch_remote_uid == node_uid and switch_remote_port == interface
+            # Build port name from snmp_naming_pattern
+            # Example: '3 2 GigabitEthernet%LINECARD%/%PORT%' -> 'GigabitEthernet3/2'
+            port_name = lc["snmp_pattern"].sub("%LINECARD%",lc_uid.to_s).sub("%PORT%",port_uid.to_s)
+            return switch_uid, port_name
+        end
+      end
+    end
+  end
+  return nil
+end
+
+data["sites"].each do |site_uid, site|
+
+  site["clusters"].each do |cluster_uid, cluster|
+
+    cluster_path = Pathname.new(refapi_path).join("sites",site_uid,"clusters",cluster_uid)
+    cluster_path.join("nodes").mkpath()
+
+    # Write cluster info w/o nodes entries
+    write_json(cluster_path.join("#{cluster_uid}.json"),
+               cluster.reject {|k, v| k == "nodes"})
+
+    cluster["nodes"].each do |node_uid, node|
+
+      node["uid"] = node_uid
+
+      # TODO: fix inconsistency?
+      node["storage_devices"].each {|s| s.update(node["block_devices"].values.at(node["storage_devices"].index(s)))}
+      node.delete("block_devices")
+
+      # TODO: fix inconsistency?
+      node["chassis"]["name"] = node["chassis"]["product_name"]
+      node["chassis"]["serial"] = node["chassis"]["serial_number"]
+      node["chassis"].delete("product_name")
+      node["chassis"].delete("serial_number")
+
+      # TODO: fix inconsistency?
+      node["network_adapters"].each do |iface|
+        node["network_interfaces"].select {|k| k.to_s == iface["device"]}.each do |k, v|
+          iface.update(v)
+        end
+      end
+      node.delete("network_interfaces")
+
+      # Network Reference
+      node["network_adapters"].each do |iface|
+        if iface["enabled"]
+          if iface["management"]
+            # Managment iface
+            iface["network_address"] = "#{node_uid}-bmc.#{site_uid}.grid5000.fr"
+          elsif iface["mounted"]
+            # Primary iface
+            iface["bridged"] = true
+            iface["network_address"] = "#{node_uid}.#{site_uid}.grid5000.fr"
+            # Interface may not be specified in Network Reference for primary iface
+            iface["switch"], iface["port"] = \
+              net_switch_port_lookup(site, node_uid) || net_switch_port_lookup(site, node_uid, iface["device"])
+          else
+            # Secondary iface(s)
+            iface["network_address"] = "#{node_uid}-#{iface["device"]}.#{site}.grid5000.fr"
+            iface["switch"], iface["port"] = net_switch_port_lookup(site, node_uid, iface["device"])
+          end
+        end
+      end
+
+      write_json(cluster_path.join("nodes","#{node_uid}.json"), node)
+    end
+  end
+end

input/example/input/sites/rouen/clusters/pomme/nodes/pomme-1.yml

+---
+pomme-1:
+  bios:
+    version: V1.35.2.2
+    release_date: 04/25/2006
+    vendor: Phoenix Technologies Ltd.
+  network_interfaces:
+    eth0:
+      mounted: false
+      mac: 00:09:3d:12:9e:13
+      management: false
+      enabled: false
+      interface: Ethernet
+      driver: tg3
+      mountable: false
+      ip: 172.16.49.101
+    eth1:
+      mounted: true
+      switch_port: "1:1"
+      rate: 1000000000
+      mac: 00:09:3d:12:9e:14
+      ip6: fe80::209:3dff:fe12:9e14
+      management: false
+      enabled: true
+      interface: Ethernet
+      driver: tg3
+      mountable: true
+      ip: 172.16.49.1
+    bmc:
+      mac: 00:09:3d:12:9e:15
+      ip: 172.17.49.1
+  block_devices:
+    sda:
+      model: MAT3073NC
+      device: sda
+      size: 73543163904
+      rev: 104
+  supported_job_types:
+    virtual: false
+  chassis:
+    serial_number: XG052536201
+    manufacturer: Sun Microsystems
+    product_name: Sun Fire V20z
+  main_memory:
+    ram_size: 2146435072
+  processor:
+    model: AMD Opteron
+    cache_l1d: 65536
+    clock_speed: 2400000000
+    other_description: AMD Opteron(tm) Processor 250
+    version: 250
+    cache_l2: 1048576
+    cache_l3: 0
+    vendor: AMD
+    instruction_set: x86-64
+    cache_l1i: 65536
+  architecture:
+    platform_type: x86_64
+    smt_size: 2
+    smp_size: 2

input/example/input/sites/rouen/clusters/pomme/nodes/pomme-2.yml

+---
+pomme-2:
+  bios:
+    version: V1.35.2.2
+    release_date: 04/25/2006
+    vendor: Phoenix Technologies Ltd.
+  network_interfaces:
+    eth0:
+      mounted: false
+      mac: 00:09:3d:12:9e:13
+      management: false
+      enabled: false
+      interface: Ethernet
+      driver: tg3
+      mountable: false
+      ip: 172.16.49.101
+    eth1:
+      mounted: true
+      switch_port: "1:1"
+      rate: 1000000000
+      mac: 00:09:3d:12:9e:14
+      ip6: fe80::209:3dff:fe12:9e14
+      management: false
+      enabled: true
+      interface: Ethernet
+      driver: tg3
+      mountable: true
+      ip: 172.16.49.1
+    bmc:
+      mac: 00:09:3d:12:9e:15
+      ip: 172.17.49.1
+  block_devices:
+    sda:
+      model: MAT3073NC
+      device: sda
+      size: 73543163904
+      rev: 104
+  supported_job_types:
+    virtual: false
+  chassis:
+    serial_number: XG052536201
+    manufacturer: Sun Microsystems
+    product_name: Sun Fire V20z
+  main_memory:
+    ram_size: 2146435072
+  processor:
+    model: AMD Opteron
+    cache_l1d: 65536
+    clock_speed: 2400000000
+    other_description: AMD Opteron(tm) Processor 250
+    version: 250
+    cache_l2: 1048576
+    cache_l3: 0
+    vendor: AMD
+    instruction_set: x86-64
+    cache_l1i: 65536
+  architecture:
+    platform_type: x86_64
+    smt_size: 2
+    smp_size: 2

input/example/input/sites/rouen/clusters/pomme/pomme.yml

+---
+model: Sun Fire V20z
+created_at: "Sat, 01 Jul 2006 12:00:00 GMT"
+kavlan: true
+production: true
+type: cluster
+uid: pomme
+
+nodes:
+  pomme-<->:
+    type: node
+    performance:
+      core_flops: 3929000000
+      node_flops: 7440000000
+    main_memory:
+      virtual_size: null
+    processor:
+      cache_l1: null
+    supported_job_types:
+      deploy: true
+      besteffort: true
+      virtual: true
+    storage_devices:
+      - interface: SCSI
+        driver: mptspi
+        storage: HDD
+    network_adapters:
+      - rate: 1000000000
+        bridged: false
+        device: eth0
+        vendor: Broadcom
+        model: BCM5704
+      - rate: 1000000000
+        bridged: false
+        device: eth1
+        vendor: Broadcom
+        model: BCM5704
+      - rate: 100000000
+        device: bmc
+        enabled: true
+        management: true
+        mountable: false
+        mounted: false
+        interface: Ethernet
+        driver: bnx2
+    gpu:
+      gpu: false
+    monitoring:
+      wattmeter: true
+    sensors:
+      power:
+        available: true
+        per_outlets: true
+        via:
+          api:
+            metric: pdu
+          www:
+            url: http://wattmetre.lyon.grid5000.fr/GetWatts-json.php
+
+  pomme-2:
+    sensors:
+      power:
+        available: false

input/example/input/sites/rouen/network/gw-production.yaml

+---
+gw-production:
+  model: Extreme Networks version 12.1.3.14
+  kind: router
+  site: lyon
+  snmp_community: public
+  vlans:
+    naming_pattern: Vlan%VLANID%
+    "vlan100":
+      administrative: yes
+      addresses:
+        - 172.16.63.254
+    "vlan101":
+      addresses:
+        - 172.17.63.254
+    "vlan500":
+      addresses:
+        - 192.168.4.13
+    "vlan701":
+      name: kavlan-1
+      addresses:
+        - 192.168.192.0/20
+    "vlan702":
+      name: kavlan-2
+      addresses:
+        - 192.168.208.0/20
+    "vlan703":
+      name: kavlan-3
+      addresses:
+        - 192.168.224.0/20
+    "vlan704":
+      name: kavlan-4
+      addresses:
+        - 10.12.0.0/18
+    "vlan705":
+      name: kavlan-5
+      addresses:
+        - 10.12.64.0/18	
+    "vlan706":
+      name: kavlan-6
+      addresses:
+        - 10.12.128.0/18
+    "vlan707":
+      name: kavlan-7
+      addresses:
+        - 10.12.192.0/18	
+    "vlan708":
+      name: kavlan-8
+      addresses:
+        - 10.13.0.0/18	
+    "vlan709":
+      name: kavlan-9
+      addresses:
+        - 10.13.64.0/18
+    "vlan713":
+      name: kavlan-13
+      addresses:
+        - 10.15.192.0/18
+  routes: {}
+  backplane_bps: 800000000000
+  mtu: 9216
+  linecards:
+    2:
+      naming_pattern: "%LINECARD%:%PORT%"
+      snmp_pattern: "BD-8810 Port %LINECARD%:%PORT%"
+      backplane_bps: 48000000000,
+      kind: node
+      rate: 1000000000
+      port: eth1
+      ports:
+        1: pomme-1
+        2: pomme-2
+    9:
+      naming_pattern: "%LINECARD%:%PORT%"
+      snmp_pattern: "BD-8810 Port %LINECARD%:%PORT%"
+      backplane_bps: 48000000000,
+      kind: switch
+      rate: 10000000000
+      ports:
+        2:
+          uid: force10
+          port: "0:42"
+        3:
+          uid: salome
+          port: "0:49"
+        4:
+          uid: renater-lyon
+          site: renater
+          kind: virtual
+  channels:
+    naming_pattern: Po%CHANNELID%