From 678fa670743bab42a90e6e054cea1aac9a518e9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Gaidamour?= <jeremie.gaidamour@inria.fr>
Date: Mon, 19 Oct 2015 13:59:36 +0200
Subject: [PATCH] [dev] Prototype written by Simon for the new version of the
generator
---
README | 5 +
generators/input_loader.rb | 67 +++++++++++
generators/reference-api/reference-api.rb | 109 ++++++++++++++++++
.../rouen/clusters/pomme/nodes/pomme-1.yml | 60 ++++++++++
.../rouen/clusters/pomme/nodes/pomme-2.yml | 60 ++++++++++
.../sites/rouen/clusters/pomme/pomme.yml | 63 ++++++++++
.../sites/rouen/network/gw-production.yaml | 91 +++++++++++++++
7 files changed, 455 insertions(+)
create mode 100644 README
create mode 100755 generators/input_loader.rb
create mode 100755 generators/reference-api/reference-api.rb
create mode 100644 input/example/input/sites/rouen/clusters/pomme/nodes/pomme-1.yml
create mode 100644 input/example/input/sites/rouen/clusters/pomme/nodes/pomme-2.yml
create mode 100644 input/example/input/sites/rouen/clusters/pomme/pomme.yml
create mode 100644 input/example/input/sites/rouen/network/gw-production.yaml
diff --git a/README b/README
new file mode 100644
index 00000000000..d5f60efcf0c
--- /dev/null
+++ b/README
@@ -0,0 +1,5 @@
+To get JSON input data:
+ ./generator/input_loader.rb
+
+To generate Ref API data in /tmp/data:
+ ./generator/input_loader.rb | ./generators/reference-api/reference-api.rb
diff --git a/generators/input_loader.rb b/generators/input_loader.rb
new file mode 100755
index 00000000000..006a528c5d4
--- /dev/null
+++ b/generators/input_loader.rb
@@ -0,0 +1,67 @@
+#!/usr/bin/ruby
+
+require 'pathname'
+require 'yaml'
+require 'json'
+
+# Extend Hash with helper methods needed to convert input data files to ruby Hash
+class ::Hash
+
+ # Recursively merge this Hash with another
+ def deep_merge(second)
+ merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
+ self.merge(second, &merger)
+ end
+
+ # Merge keys that match "PREFIX-<a-b>" with others keys that begins by
+ # "PREFIX-" and that ends with x, where a<=x<=b
+ # a and/or b may be ommited, meaning that there are no lower and/or upper bound for x
+ # This is done recursively (for this Hash and every Hashes it contains)
+ # i.e.:
+ # {"foo-1": {a: 0}, "foo-2": {a: 0}, "foo-3": {a: 0}, "foo-<2->": {b: 1}}.kbracket_merge()
+ # -> {"foo-2": {a: 0, b:1}, "foo-3": {a: 0, b: 0}}
+ # TODO: only "<->" suffix is currently implemented
+ def kbracket_merge()
+ self.each { |k_all, v_all|
+ if k_all.to_s.end_with?('-<->') and v_all.is_a?(Hash)
+ key_prefix_to_merge = k_all.to_s.sub(/-<->$/,'')
+ self.each { |k, v|
+ if k.to_s.start_with?(key_prefix_to_merge)
+ self[k] = v_all.deep_merge(v)
+ end
+ }
+ self.delete(k_all)
+ end
+ }
+ self.each { |k, v|
+ if v.is_a?(Hash)
+ v.kbracket_merge()
+ end
+ }
+ end
+
+ # Add an element composed of nested Hashes made from elements found in "array" argument
+ # i.e.: from_array([a, b, c],"foo") -> {a: {b: {c: "foo"}}}
+ def self.from_array(array, value)
+ return array.reverse.inject(value) { |a, n| { n => a } }
+ end
+end
+
+#TODO: Ensure that deepest elements in input files hierarchy have lowest priority
+data = Hash.new
+
+Dir.chdir("input")
+Dir['**/*.y*ml'].each { |f|
+ node_path = Pathname.new(f)
+ node_dir, _ = node_path.split()
+
+ node_hierarchy = node_dir.to_s.split('/')
+ node_value = YAML::load_file(node_path)
+
+ node_data = Hash.from_array(node_hierarchy,node_value)
+ data = data.deep_merge(node_data)
+}
+data.kbracket_merge()
+
+puts JSON.generate(data)
+
diff --git a/generators/reference-api/reference-api.rb b/generators/reference-api/reference-api.rb
new file mode 100755
index 00000000000..157d09e01f4
--- /dev/null
+++ b/generators/reference-api/reference-api.rb
@@ -0,0 +1,109 @@
+#!/usr/bin/ruby
+
+require 'pathname'
+require 'json'
+
+refapi_path = "/tmp/data"
+data = JSON.parse(STDIN.read)
+
+# Write pretty and sorted JSON files
+def write_json(filepath, data)
+ def rec_sort(h)
+ case h
+ when Array
+ h.map{|v| rec_sort(v)}.sort_by!{|v| (v.to_s rescue nil) }
+ when Hash
+ Hash[Hash[h.map{|k,v| [rec_sort(k),rec_sort(v)]}].sort_by{|k,v| [(k.to_s rescue nil), (v.to_s rescue nil)]}]
+ else
+ h
+ end
+ end
+ File.open(filepath, 'w') do |f|
+ f.write(JSON.pretty_generate(rec_sort(data)))
+ end
+end
+
+# Parse network equipment description and return switch name and port connected to given node
+# In the network description, if the node interface is given (using "port" attribute),
+# the interface parameter must be used.
+def net_switch_port_lookup(site, node_uid, interface='')
+ site["network"].each do |switch_uid, switch|
+ switch["linecards"].each do |lc_uid,lc|
+ lc["ports"].each do |port_uid,port|
+ if port.is_a?(Hash)
+ switch_remote_port = port["port"] || lc["port"] || ""
+ switch_remote_uid = port["uid"]
+ else
+ switch_remote_port = lc["port"] || ""
+ switch_remote_uid = port
+ end
+ if switch_remote_uid == node_uid and switch_remote_port == interface
+ # Build port name from snmp_naming_pattern
+ # Example: '3 2 GigabitEthernet%LINECARD%/%PORT%' -> 'GigabitEthernet3/2'
+ port_name = lc["snmp_pattern"].sub("%LINECARD%",lc_uid.to_s).sub("%PORT%",port_uid.to_s)
+ return switch_uid, port_name
+ end
+ end
+ end
+ end
+ return nil
+end
+
+data["sites"].each do |site_uid, site|
+
+ site["clusters"].each do |cluster_uid, cluster|
+
+ cluster_path = Pathname.new(refapi_path).join("sites",site_uid,"clusters",cluster_uid)
+ cluster_path.join("nodes").mkpath()
+
+ # Write cluster info w/o nodes entries
+ write_json(cluster_path.join("#{cluster_uid}.json"),
+ cluster.reject {|k, v| k == "nodes"})
+
+ cluster["nodes"].each do |node_uid, node|
+
+ node["uid"] = node_uid
+
+ # TODO: fix inconsistency?
+ node["storage_devices"].each {|s| s.update(node["block_devices"].values.at(node["storage_devices"].index(s)))}
+ node.delete("block_devices")
+
+ # TODO: fix inconsistency?
+ node["chassis"]["name"] = node["chassis"]["product_name"]
+ node["chassis"]["serial"] = node["chassis"]["serial_number"]
+ node["chassis"].delete("product_name")
+ node["chassis"].delete("serial_number")
+
+ # TODO: fix inconsistency?
+ node["network_adapters"].each do |iface|
+ node["network_interfaces"].select {|k| k.to_s == iface["device"]}.each do |k, v|
+ iface.update(v)
+ end
+ end
+ node.delete("network_interfaces")
+
+ # Network Reference
+ node["network_adapters"].each do |iface|
+ if iface["enabled"]
+ if iface["management"]
+ # Managment iface
+ iface["network_address"] = "#{node_uid}-bmc.#{site_uid}.grid5000.fr"
+ elsif iface["mounted"]
+ # Primary iface
+ iface["bridged"] = true
+ iface["network_address"] = "#{node_uid}.#{site_uid}.grid5000.fr"
+ # Interface may not be specified in Network Reference for primary iface
+ iface["switch"], iface["port"] = \
+ net_switch_port_lookup(site, node_uid) || net_switch_port_lookup(site, node_uid, iface["device"])
+ else
+ # Secondary iface(s)
+ iface["network_address"] = "#{node_uid}-#{iface["device"]}.#{site}.grid5000.fr"
+ iface["switch"], iface["port"] = net_switch_port_lookup(site, node_uid, iface["device"])
+ end
+ end
+ end
+
+ write_json(cluster_path.join("nodes","#{node_uid}.json"), node)
+ end
+ end
+end
diff --git a/input/example/input/sites/rouen/clusters/pomme/nodes/pomme-1.yml b/input/example/input/sites/rouen/clusters/pomme/nodes/pomme-1.yml
new file mode 100644
index 00000000000..a78dfa53d3c
--- /dev/null
+++ b/input/example/input/sites/rouen/clusters/pomme/nodes/pomme-1.yml
@@ -0,0 +1,60 @@
+---
+pomme-1:
+ bios:
+ version: V1.35.2.2
+ release_date: 04/25/2006
+ vendor: Phoenix Technologies Ltd.
+ network_interfaces:
+ eth0:
+ mounted: false
+ mac: 00:09:3d:12:9e:13
+ management: false
+ enabled: false
+ interface: Ethernet
+ driver: tg3
+ mountable: false
+ ip: 172.16.49.101
+ eth1:
+ mounted: true
+ switch_port: "1:1"
+ rate: 1000000000
+ mac: 00:09:3d:12:9e:14
+ ip6: fe80::209:3dff:fe12:9e14
+ management: false
+ enabled: true
+ interface: Ethernet
+ driver: tg3
+ mountable: true
+ ip: 172.16.49.1
+ bmc:
+ mac: 00:09:3d:12:9e:15
+ ip: 172.17.49.1
+ block_devices:
+ sda:
+ model: MAT3073NC
+ device: sda
+ size: 73543163904
+ rev: 104
+ supported_job_types:
+ virtual: false
+ chassis:
+ serial_number: XG052536201
+ manufacturer: Sun Microsystems
+ product_name: Sun Fire V20z
+ main_memory:
+ ram_size: 2146435072
+ processor:
+ model: AMD Opteron
+ cache_l1d: 65536
+ clock_speed: 2400000000
+ other_description: AMD Opteron(tm) Processor 250
+ version: 250
+ cache_l2: 1048576
+ cache_l3: 0
+ vendor: AMD
+ instruction_set: x86-64
+ cache_l1i: 65536
+ architecture:
+ platform_type: x86_64
+ smt_size: 2
+ smp_size: 2
diff --git a/input/example/input/sites/rouen/clusters/pomme/nodes/pomme-2.yml b/input/example/input/sites/rouen/clusters/pomme/nodes/pomme-2.yml
new file mode 100644
index 00000000000..369263b53e9
--- /dev/null
+++ b/input/example/input/sites/rouen/clusters/pomme/nodes/pomme-2.yml
@@ -0,0 +1,60 @@
+---
+pomme-2:
+ bios:
+ version: V1.35.2.2
+ release_date: 04/25/2006
+ vendor: Phoenix Technologies Ltd.
+ network_interfaces:
+ eth0:
+ mounted: false
+ mac: 00:09:3d:12:9e:13
+ management: false
+ enabled: false
+ interface: Ethernet
+ driver: tg3
+ mountable: false
+ ip: 172.16.49.101
+ eth1:
+ mounted: true
+ switch_port: "1:1"
+ rate: 1000000000
+ mac: 00:09:3d:12:9e:14
+ ip6: fe80::209:3dff:fe12:9e14
+ management: false
+ enabled: true
+ interface: Ethernet
+ driver: tg3
+ mountable: true
+ ip: 172.16.49.1
+ bmc:
+ mac: 00:09:3d:12:9e:15
+ ip: 172.17.49.1
+ block_devices:
+ sda:
+ model: MAT3073NC
+ device: sda
+ size: 73543163904
+ rev: 104
+ supported_job_types:
+ virtual: false
+ chassis:
+ serial_number: XG052536201
+ manufacturer: Sun Microsystems
+ product_name: Sun Fire V20z
+ main_memory:
+ ram_size: 2146435072
+ processor:
+ model: AMD Opteron
+ cache_l1d: 65536
+ clock_speed: 2400000000
+ other_description: AMD Opteron(tm) Processor 250
+ version: 250
+ cache_l2: 1048576
+ cache_l3: 0
+ vendor: AMD
+ instruction_set: x86-64
+ cache_l1i: 65536
+ architecture:
+ platform_type: x86_64
+ smt_size: 2
+ smp_size: 2
diff --git a/input/example/input/sites/rouen/clusters/pomme/pomme.yml b/input/example/input/sites/rouen/clusters/pomme/pomme.yml
new file mode 100644
index 00000000000..2fd8e4b5a23
--- /dev/null
+++ b/input/example/input/sites/rouen/clusters/pomme/pomme.yml
@@ -0,0 +1,63 @@
+---
+model: Sun Fire V20z
+created_at: "Sat, 01 Jul 2006 12:00:00 GMT"
+kavlan: true
+production: true
+type: cluster
+uid: pomme
+
+nodes:
+ pomme-<->:
+ type: node
+ performance:
+ core_flops: 3929000000
+ node_flops: 7440000000
+ main_memory:
+ virtual_size: null
+ processor:
+ cache_l1: null
+ supported_job_types:
+ deploy: true
+ besteffort: true
+ virtual: true
+ storage_devices:
+ - interface: SCSI
+ driver: mptspi
+ storage: HDD
+ network_adapters:
+ - rate: 1000000000
+ bridged: false
+ device: eth0
+ vendor: Broadcom
+ model: BCM5704
+ - rate: 1000000000
+ bridged: false
+ device: eth1
+ vendor: Broadcom
+ model: BCM5704
+ - rate: 100000000
+ device: bmc
+ enabled: true
+ management: true
+ mountable: false
+ mounted: false
+ interface: Ethernet
+ driver: bnx2
+ gpu:
+ gpu: false
+ monitoring:
+ wattmeter: true
+ sensors:
+ power:
+ available: true
+ per_outlets: true
+ via:
+ api:
+ metric: pdu
+ www:
+ url: http://wattmetre.lyon.grid5000.fr/GetWatts-json.php
+
+ pomme-2:
+ sensors:
+ power:
+ available: false
diff --git a/input/example/input/sites/rouen/network/gw-production.yaml b/input/example/input/sites/rouen/network/gw-production.yaml
new file mode 100644
index 00000000000..fed978d49b9
--- /dev/null
+++ b/input/example/input/sites/rouen/network/gw-production.yaml
@@ -0,0 +1,91 @@
+---
+gw-production:
+ model: Extreme Networks version 12.1.3.14
+ kind: router
+ site: lyon
+ snmp_community: public
+ vlans:
+ naming_pattern: Vlan%VLANID%
+ "vlan100":
+ administrative: yes
+ addresses:
+ - 172.16.63.254
+ "vlan101":
+ addresses:
+ - 172.17.63.254
+ "vlan500":
+ addresses:
+ - 192.168.4.13
+ "vlan701":
+ name: kavlan-1
+ addresses:
+ - 192.168.192.0/20
+ "vlan702":
+ name: kavlan-2
+ addresses:
+ - 192.168.208.0/20
+ "vlan703":
+ name: kavlan-3
+ addresses:
+ - 192.168.224.0/20
+ "vlan704":
+ name: kavlan-4
+ addresses:
+ - 10.12.0.0/18
+ "vlan705":
+ name: kavlan-5
+ addresses:
+ - 10.12.64.0/18
+ "vlan706":
+ name: kavlan-6
+ addresses:
+ - 10.12.128.0/18
+ "vlan707":
+ name: kavlan-7
+ addresses:
+ - 10.12.192.0/18
+ "vlan708":
+ name: kavlan-8
+ addresses:
+ - 10.13.0.0/18
+ "vlan709":
+ name: kavlan-9
+ addresses:
+ - 10.13.64.0/18
+ "vlan713":
+ name: kavlan-13
+ addresses:
+ - 10.15.192.0/18
+ routes: {}
+ backplane_bps: 800000000000
+ mtu: 9216
+ linecards:
+ 2:
+ naming_pattern: "%LINECARD%:%PORT%"
+ snmp_pattern: "BD-8810 Port %LINECARD%:%PORT%"
+ backplane_bps: 48000000000,
+ kind: node
+ rate: 1000000000
+ port: eth1
+ ports:
+ 1: pomme-1
+ 2: pomme-2
+ 9:
+ naming_pattern: "%LINECARD%:%PORT%"
+ snmp_pattern: "BD-8810 Port %LINECARD%:%PORT%"
+ backplane_bps: 48000000000,
+ kind: switch
+ rate: 10000000000
+ ports:
+ 2:
+ uid: force10
+ port: "0:42"
+ 3:
+ uid: salome
+ port: "0:49"
+ 4:
+ uid: renater-lyon
+ site: renater
+ kind: virtual
+ channels:
+ naming_pattern: Po%CHANNELID%
--
GitLab