[dev] Integration of check-cluster-homogeneity into reference-api.rb

generators/input-validators/check-cluster-homogeneity.rb

+#!/usr/bin/ruby
+
+# This script checks the cluster homogeneity
+
+if RUBY_VERSION < "2.1"
+  puts "This script requires ruby >= 2.1"
+  exit
+end
+
+require 'pp'
+require 'fileutils'
+require 'pathname'
+require 'hashdiff'
+
+dir = Pathname(__FILE__).parent
+
+require "#{dir}/../lib/input_loader"
+
+def cluster_homogeneity(refapi_hash, verbose=false)
+
+  ignore_keys = %w(
+    chassis.serial
+  
+    network_adapters.bmc.ip
+    network_adapters.bmc.mac
+    network_adapters.bmc.network_address
+    network_adapters.bmc.switch
+    network_adapters.bmc.switch_port
+  
+    network_adapters.ib0.guid
+    network_adapters.ib0.hwid
+    network_adapters.ib0.ip
+    network_adapters.ib0.ip6
+    network_adapters.ib0.line_card
+    network_adapters.ib0.position
+    network_adapters.ib1.guid
+  
+    network_adapters.myri0.ip
+    network_adapters.myri0.ip6
+    network_adapters.myri0.mac
+    network_adapters.myri0.network_address
+  
+    pdu
+    pdu.port
+    pdu.uid
+  
+    supported_job_types.max_walltime
+  )
+
+  ignore_netkeys = <<-eos
+    network_adapters.eth.ip
+    network_adapters.eth.ip6
+    network_adapters.eth.mac
+    network_adapters.eth.network_address
+    network_adapters.eth.switch
+    network_adapters.eth.switch_port
+    network_adapters.eth.ip
+    network_adapters.eth.ip6
+    network_adapters.eth.mac
+    network_adapters.eth.switch_port
+    network_adapters.eth.ip
+    network_adapters.eth.ip6
+    network_adapters.eth.mac
+    network_adapters.eth.switch_port
+    network_adapters.eth.ip
+    network_adapters.eth.mac
+    network_adapters.eth.mac
+    network_adapters.eth.mac
+eos
+
+  ignore_stokeys = <<-eos
+    storage_devices.sd.model
+    storage_devices.sd.rev
+    storage_devices.sd.size
+    storage_devices.sd.timeread
+    storage_devices.sd.timewrite
+    storage_devices.sd.vendor
+eos
+
+  (0..5).each { |eth| 
+    keys = ignore_netkeys.gsub('.eth.', ".eth#{eth}.").gsub("\n", " ").split(" ")
+    ignore_keys.push(* keys)
+
+    (1..21).each { |kavlan|
+      ignore_keys << "kavlan.eth#{eth}.kavlan-#{kavlan}"
+    }
+  }
+
+  ('a'..'d').each { |sd| 
+    keys = ignore_stokeys.gsub('.sd.', ".sd#{sd}.").gsub("\n", " ").split(" ")
+    ignore_keys.push(* keys)
+  }
+
+  refapi_hash = load_yaml_file_hierarchy("../../input/grid5000/")
+  count = {}
+  
+  refapi_hash["sites"].sort.each do |site_uid, site|
+    count[site_uid] = {}
+
+    site["clusters"].sort.each do |cluster_uid, cluster|
+      count[site_uid][cluster_uid] = 0
+
+      refnode_uid = cluster['nodes'].keys.sort.first
+      refnode = cluster['nodes'][refnode_uid]
+      
+      cluster["nodes"].each_sort_by_node_uid do |node_uid, node|
+        #next if node_uid != 'graphene-2'
+        
+        diffs = HashDiff.diff(refnode, node)
+        
+        # Remove keys that are specific to each nodes (ip, mac etc.)
+        diffs.clone.each { |diff|
+          diffs.delete(diff) if diff[0] == '~' && ignore_keys.include?(diff[1])
+        }
+
+        if verbose && !diffs.empty?
+            puts "Differences between #{refnode_uid} and #{node_uid}:"
+            pp diffs
+        end
+
+        count[site_uid][cluster_uid] += diffs.size
+
+        # Remove the following line if you want to compare each nodes to the first cluster node
+        refnode_uid = node_uid
+        refnode = node
+
+      end
+      
+    end
+  end
+
+  return count
+end
+
+def check_cluster_homogeneity(refapi_hash, verbose=false)
+  puts "Differences found between successive nodes, per cluster:\n\n"
+
+  count = cluster_homogeneity(refapi_hash, verbose)
+  puts "\n" if verbose
+
+  puts count.to_yaml unless verbose
+
+  puts "\nUse '../input-validators/check-cluster-homogeneity.rb -v' for details." unless verbose
+  
+  return count
+end
+
+if __FILE__ == $0
+  require 'optparse'
+
+  options = {}
+
+  OptionParser.new do |opts|
+    opts.banner = "Usage: check-cluster-homogeneity.rb [options]"
+
+    opts.separator ""
+    opts.separator "Example: ruby check-cluster-homogeneity.rb -v"
+    opts.separator ""
+
+    opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+      options[:verbose] ||= 0
+      options[:verbose] = options[:verbose] + 1
+    end
+  
+    # Print an options summary.
+    opts.on_tail("-h", "--help", "Show this message") do
+      puts opts
+      exit
+    end
+  end.parse!
+
+  refapi_hash = load_yaml_file_hierarchy("#{dir}/../../input/grid5000/")
+  check_cluster_homogeneity(refapi_hash, options.key?(:verbose))
+end

generators/input-validators/cluster-homogeneity.rb

-#!/usr/bin/ruby
-
-# This script checks the cluste homogeneity
-
-require 'pp'
-require 'fileutils'
-require 'pathname'
-require 'hashdiff'
-
-require '../lib/input_loader'
-
-ignore_keys = %w(
-  chassis.serial
-
-  kavlan.kavlan-1
-  kavlan.kavlan-2
-  kavlan.kavlan-3
-  kavlan.kavlan-4
-  kavlan.kavlan-5
-  kavlan.kavlan-6
-  kavlan.kavlan-7
-  kavlan.kavlan-8
-  kavlan.kavlan-9
-  kavlan.kavlan-10
-  kavlan.kavlan-11
-  kavlan.kavlan-12
-  kavlan.kavlan-13
-  kavlan.kavlan-14
-  kavlan.kavlan-15
-  kavlan.kavlan-16
-  kavlan.kavlan-17
-  kavlan.kavlan-18
-  kavlan.kavlan-19
-  kavlan.kavlan-20
-  kavlan.kavlan-21
-
-  network_adapters.bmc.ip
-  network_adapters.bmc.mac
-  network_adapters.bmc.network_address
-  network_adapters.bmc.switch
-  network_adapters.bmc.switch_port
-
-  network_adapters.eth0.ip
-  network_adapters.eth0.ip6
-  network_adapters.eth0.mac
-  network_adapters.eth0.network_address
-  network_adapters.eth0.switch
-  network_adapters.eth0.switch_port
-
-  network_adapters.eth1.ip
-  network_adapters.eth1.ip6
-  network_adapters.eth1.mac
-  network_adapters.eth1.switch_port
-
-  network_adapters.eth2.ip
-  network_adapters.eth2.ip6
-  network_adapters.eth2.mac
-  network_adapters.eth2.switch_port
-
-  network_adapters.eth3.ip
-  network_adapters.eth3.mac
-
-  network_adapters.eth4.mac
-
-  network_adapters.eth5.mac
-
-  network_adapters.ib0.guid
-  network_adapters.ib0.hwid
-  network_adapters.ib0.ip
-  network_adapters.ib0.ip6
-  network_adapters.ib0.line_card
-  network_adapters.ib0.position
-
-  network_adapters.ib1.guid
-
-  network_adapters.myri0.ip
-  network_adapters.myri0.ip6
-  network_adapters.myri0.mac
-  network_adapters.myri0.network_address
-
-  pdu.port
-  pdu.uid
-
-  storage_devices.sda.model
-  storage_devices.sda.rev
-  storage_devices.sda.size
-  storage_devices.sda.timeread
-  storage_devices.sda.timewrite
-  storage_devices.sda.vendor
-
-  storage_devices.sdc.model
-  storage_devices.sdc.rev
-
-  storage_devices.sde.model
-  storage_devices.sde.rev
-
-  supported_job_types.max_walltime
-)
-
-refapi_hash = load_yaml_file_hierarchy("../../input/grid5000/")
-
-refapi_hash["sites"].each do |site_uid, site|
-  site["clusters"].each do |cluster_uid, cluster|
-
-    refnode_uid = cluster['nodes'].keys.sort.first
-    refnode = cluster['nodes'][refnode_uid]
-     
-    cluster["nodes"].each_sort_by_node_uid do |node_uid, node|
-      #next if node_uid != 'graphene-2'
-      
-      diffs = HashDiff.diff(refnode, node)
- 
-      # Remove keys that are specific to each nodes (ip, mac etc.)
-      diffs.clone.each { |diff|
-        diffs.delete(diff) if diff[0] == '~' && ignore_keys.include?(diff[1])
-      }
-
-      if !diffs.empty?
-        puts "Differences between #{refnode_uid} and #{node_uid}:"
-        pp diffs
-      end
-
-      # Remove the following line if you want to compare each nodes to the first cluster node
-      refnode_uid = node_uid
-      refnode = node
-
-    end
-  end
-end

generators/input-validators/yaml-input-find-duplicates.rb

 #!/usr/bin/ruby
 
+if RUBY_VERSION < "2.1"
+  puts "This script requires ruby >= 2.1"
+  exit
+end
+
 require 'pp'
 require '../lib/input_loader'
 

generators/input-validators/yaml-input-schema-validator.rb

 #!/usr/bin/ruby
 
+if RUBY_VERSION < "2.1"
+  puts "This script requires ruby >= 2.1"
+  exit
+end
+
 require 'fileutils'
 require 'pathname'
 
@@ -13,7 +18,9 @@ def run_validator(uid, data, schema)
   if ! validator.valid?
     errors = {uid => validator.errors}
     puts errors.to_yaml
+    return false
   end
+  return true
 end
 
 def yaml_input_schema_validator(global_hash)
@@ -24,26 +31,29 @@ def yaml_input_schema_validator(global_hash)
   schema_cluster = load_yaml_schema("#{dir}/schema-cluster.yaml")
   schema_node    = load_yaml_schema("#{dir}/schema-node.yaml")
   
-  run_validator('global', global_hash, schema_global) #
+  r &= run_validator('global', global_hash, schema_global) #
   
   global_hash["sites"].each do |site_uid, site|
     
-    run_validator(site_uid, site, schema_site) #
+    r &= run_validator(site_uid, site, schema_site) #
     
     site["clusters"].each do |cluster_uid, cluster|
       
-      run_validator(cluster_uid, cluster, schema_cluster) #
+      r &= run_validator(cluster_uid, cluster, schema_cluster) #
       
       cluster["nodes"].each do |node_uid, node|
         
-        run_validator(node_uid, node, schema_node) #
+        r &= run_validator(node_uid, node, schema_node) #
         
       end
     end
   end
+  return r
 end
 
 if __FILE__ == $0
+  puts "Checking input data:\n\n"
   global_hash = load_yaml_file_hierarchy("#{dir}/../../input/grid5000/")
-  yaml_input_schema_validator(global_hash)
+  r = yaml_input_schema_validator(global_hash)
+  puts 'OK' if r
 end

generators/oar-properties/oar-properties.rb

@@ -2,6 +2,11 @@
 
 # Generator for the OAR properties
 
+if RUBY_VERSION < "2.1"
+  puts "This script requires ruby >= 2.1"
+  exit
+end
+
 require 'pp'
 require 'erb'
 require 'fileutils'

generators/puppet/kadeployg5k.rb

 #!/usr/bin/ruby
 
+# This script generates:
+# - kadeployg5k/files/<site_uid>/server_conf[_dev]/clusters.conf from input/
+# - kadeployg5k/files/<site_uid>/server_conf[_dev]/<cluster_uid>-cluster.conf from kadeployg5k[-dev].yaml and template/kadeployg5k.yaml.erb
+
 if RUBY_VERSION < "2.1"
   puts "This script requires ruby >= 2.1"
   exit
 end
 
-# This script generates:
-# - kadeployg5k/files/<site_uid>/server_conf[_dev]/clusters.conf from input/
-# - kadeployg5k/files/<site_uid>/server_conf[_dev]/<cluster_uid>-cluster.conf from kadeployg5k[-dev].yaml and template/kadeployg5k.yaml.erb
-
 require 'pp'
 require 'erb'
 require 'pathname'

generators/reference-api/reference-api.rb

 #!/usr/bin/ruby
 
+if RUBY_VERSION < "2.1"
+  puts "This script requires ruby >= 2.1"
+  exit
+end
+
 require 'pp'
 require 'optparse'
 require 'erb'
@@ -10,6 +15,7 @@ require 'time'
 
 require '../lib/input_loader'
 require '../input-validators/yaml-input-schema-validator'
+require '../input-validators/check-cluster-homogeneity'
 require '../input-validators/check-monitoring-properties'
 
 # Output directory
@@ -20,8 +26,6 @@ refapi_path = "../../data/grid5000"
 #global_hash = load_yaml_file_hierarchy("../../input/example/")
 global_hash = load_yaml_file_hierarchy("../../input/grid5000/")
 
-yaml_input_schema_validator(global_hash)
-
 # Parse network equipment description and return switch name and port connected to given node
 #  In the network description, if the node interface is given (using "port" attribute),
 #  the interface parameter must be used.
@@ -64,6 +68,25 @@ OptionParser.new do |opts|
   end
 end.parse!
 
+
+#
+# input-validators
+#
+
+puts "Checking input data:\n\n"
+r = yaml_input_schema_validator(global_hash)
+puts 'OK' if r
+
+puts "\n"
+puts "#" * 80
+puts "\n"
+
+check_cluster_homogeneity(global_hash)
+puts "\n"
+puts "#" * 80
+puts "\n"
+
+
 #
 # Write grid info
 #
@@ -77,8 +100,10 @@ if global_hash['uid']
              global_hash.reject {|k, v| k == "sites"})
 end
 
+puts "Generating the reference api:\n\n"
+
 global_hash["sites"].each do |site_uid, site|
-  puts site_uid
+  puts "#{site_uid}:"
 
   #
   # Write site info
@@ -110,7 +135,7 @@ global_hash["sites"].each do |site_uid, site|
   # Write network info
   #
 
-  site["networks"].each do |network_uid, network|
+  site["networks"].sort.each do |network_uid, network|
     network["type"] = "network_equipment"
     network["uid"]  = network_uid
 
@@ -140,7 +165,7 @@ global_hash["sites"].each do |site_uid, site|
     network["linecards"] = linecards_tmp # restore
   end
 
-  site["clusters"].each do |cluster_uid, cluster|
+  site["clusters"].sort.each do |cluster_uid, cluster|
     puts "  #{cluster_uid}"
 
     #
@@ -281,4 +306,4 @@ global_hash["sites"].each do |site_uid, site|
 
 end
 
-check_monitoring_properties(global_hash)
+#check_monitoring_properties(global_hash)