Bug12144 positionnement des propriétés sur les ressources disk comme sur les ressources default

Cette MR fix le bug 12144 : https://intranet.grid5000.fr/bugzilla/show_bug.cgi?id=12144

Avec le commit 4f716650, les propriétés des ressources "default" sont aussi positionnées sur les ressources "disk". Certaines propriétés sont ensuite surchargées (ex: network_address = '') ou crées (ex: diskpath) spécifiquement pour les ressources disk.

Le commit bdfec996 met à jour les sorties du générateur attendues par rspec pour que le CI passe.

Le diff de rake gen:oar-properties sur un cluster sans, puis avec disques réservables. Il faudrait un relecteur pour vérifier que je n'oublie aucune propriétés.

➜ reference-repository(bug12144): rake gen:oar-properties DO=diff VERBOSE=2 SITE=lyon CLUSTER=nova   
Downloading resources properties from https://api.grid5000.fr/stable/sites/lyon/internal/oarapi/resources/details.json?limit=999999 ...
... done
Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
  nova-1: OK
  nova-2: OK
  nova-3: OK
  nova-4: OK
  nova-5: OK
  nova-6: OK
  nova-7: OK
  nova-8: OK
  nova-9: OK
  nova-10: OK
  nova-11: OK
  nova-12: OK
  nova-13: OK
  nova-14: OK
  nova-15: OK
  nova-16: OK
  nova-17: OK
  nova-18: OK
  nova-19: OK
  nova-20: OK
  nova-21: OK
  nova-22: OK
  nova-23: OK

➜ reference-repository(bug12144): rake gen:oar-properties DO=diff VERBOSE=2 SITE=grenoble CLUSTER=yeti
Downloading resources properties from https://api.grid5000.fr/stable/sites/grenoble/internal/oarapi/resources/details.json?limit=999999 ...
... done
Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
  yeti-1: OK
  yeti-2: OK
  yeti-3: OK
  yeti-4: OK
  ["yeti-1", "sdb.yeti-1"]:
    ["~", "chassis", nil, "Dell Inc. PowerEdge R940 6FGQSM2"]
    ["~", "cluster_priority", nil, 201801]
    ["~", "cpuarch", nil, "x86_64"]
    ["~", "cpucore", nil, 16]
    ["~", "cpufreq", nil, "2.1"]
    ["~", "cputype", nil, "Intel Xeon Gold 6130"]
    ["~", "disk_reservation_count", nil, 3]
    ["~", "disktype", nil, "SAS"]
    ["~", "eth_count", nil, 1]
    ["~", "eth_rate", nil, 10]
    ["~", "gpu_count", nil, 0]
    ["~", "ib", nil, "NO"]
    ["~", "ib_count", nil, 0]
    ["~", "ib_rate", nil, 0]
    ["~", "ip", nil, "172.16.19.1"]
    ["~", "max_walltime", nil, 0]
    ["~", "memcore", nil, 12288]
    ["~", "memcpu", nil, 196608]
    ["~", "memnode", nil, 786432]
    ["~", "mic", nil, "NO"]
    ["~", "myri", nil, "NO"]
    ["~", "myri_count", nil, 0]
    ["~", "myri_rate", nil, 0]
    ["~", "nodemodel", nil, "Dell PowerEdge R940"]
    ["~", "opa_count", nil, 1]
    ["~", "opa_rate", nil, 100]
    ["~", "switch", nil, "gw"]
    ["~", "virtual", nil, "ivt"]
    ["~", "wattmeter", nil, "YES"]
  ["yeti-1", "sdc.yeti-1"]: same modifications as above
  ["yeti-1", "sdd.yeti-1"]: same modifications as above
  ["yeti-2", "sdb.yeti-2"]:
    ["~", "chassis", nil, "Dell Inc. PowerEdge R940 6FGRSM2"]
    ["~", "cluster_priority", nil, 201801]
    ["~", "cpuarch", nil, "x86_64"]
    ["~", "cpucore", nil, 16]
    ["~", "cpufreq", nil, "2.1"]
    ["~", "cputype", nil, "Intel Xeon Gold 6130"]
    ["~", "disk_reservation_count", nil, 3]
    ["~", "disktype", nil, "SAS"]
    ["~", "eth_count", nil, 1]
    ["~", "eth_rate", nil, 10]
    ["~", "gpu_count", nil, 0]
    ["~", "ib", nil, "NO"]
    ["~", "ib_count", nil, 0]
    ["~", "ib_rate", nil, 0]
    ["~", "ip", nil, "172.16.19.2"]
    ["~", "max_walltime", nil, 0]
    ["~", "memcore", nil, 12288]
    ["~", "memcpu", nil, 196608]
    ["~", "memnode", nil, 786432]
    ["~", "mic", nil, "NO"]
    ["~", "myri", nil, "NO"]
    ["~", "myri_count", nil, 0]
    ["~", "myri_rate", nil, 0]
    ["~", "nodemodel", nil, "Dell PowerEdge R940"]
    ["~", "opa_count", nil, 1]
    ["~", "opa_rate", nil, 100]
    ["~", "switch", nil, "gw"]
    ["~", "virtual", nil, "ivt"]
    ["~", "wattmeter", nil, "YES"]
  ["yeti-2", "sdc.yeti-2"]: same modifications as above
  ["yeti-2", "sdd.yeti-2"]: same modifications as above
  ["yeti-3", "sdb.yeti-3"]:
    ["~", "chassis", nil, "Dell Inc. PowerEdge R940 6FGPSM2"]
    ["~", "cluster_priority", nil, 201801]
    ["~", "cpuarch", nil, "x86_64"]
    ["~", "cpucore", nil, 16]
    ["~", "cpufreq", nil, "2.1"]
    ["~", "cputype", nil, "Intel Xeon Gold 6130"]
    ["~", "disk_reservation_count", nil, 3]
    ["~", "disktype", nil, "SAS"]
    ["~", "eth_count", nil, 1]
    ["~", "eth_rate", nil, 10]
    ["~", "gpu_count", nil, 0]
    ["~", "ib", nil, "NO"]
    ["~", "ib_count", nil, 0]
    ["~", "ib_rate", nil, 0]
    ["~", "ip", nil, "172.16.19.3"]
    ["~", "max_walltime", nil, 0]
    ["~", "memcore", nil, 12288]
    ["~", "memcpu", nil, 196608]
    ["~", "memnode", nil, 786432]
    ["~", "mic", nil, "NO"]
    ["~", "myri", nil, "NO"]
    ["~", "myri_count", nil, 0]
    ["~", "myri_rate", nil, 0]
    ["~", "nodemodel", nil, "Dell PowerEdge R940"]
    ["~", "opa_count", nil, 1]
    ["~", "opa_rate", nil, 100]
    ["~", "switch", nil, "gw"]
    ["~", "virtual", nil, "ivt"]
    ["~", "wattmeter", nil, "YES"]
  ["yeti-3", "sdc.yeti-3"]: same modifications as above
  ["yeti-3", "sdd.yeti-3"]: same modifications as above
  ["yeti-4", "sdb.yeti-4"]:
    ["~", "chassis", nil, "Dell Inc. PowerEdge R940 6FGMSM2"]
    ["~", "cluster_priority", nil, 201801]
    ["~", "cpuarch", nil, "x86_64"]
    ["~", "cpucore", nil, 16]
    ["~", "cpufreq", nil, "2.1"]
    ["~", "cputype", nil, "Intel Xeon Gold 6130"]
    ["~", "disk_reservation_count", nil, 3]
    ["~", "disktype", nil, "SAS"]
    ["~", "eth_count", nil, 1]
    ["~", "eth_rate", nil, 10]
    ["~", "gpu_count", nil, 0]
    ["~", "ib", nil, "NO"]
    ["~", "ib_count", nil, 0]
    ["~", "ib_rate", nil, 0]
    ["~", "ip", nil, "172.16.19.4"]
    ["~", "max_walltime", nil, 0]
    ["~", "memcore", nil, 12288]
    ["~", "memcpu", nil, 196608]
    ["~", "memnode", nil, 786432]
    ["~", "mic", nil, "NO"]
    ["~", "myri", nil, "NO"]
    ["~", "myri_count", nil, 0]
    ["~", "myri_rate", nil, 0]
    ["~", "nodemodel", nil, "Dell PowerEdge R940"]
    ["~", "opa_count", nil, 1]
    ["~", "opa_rate", nil, 100]
    ["~", "switch", nil, "gw"]
    ["~", "virtual", nil, "ivt"]
    ["~", "wattmeter", nil, "YES"]
  ["yeti-4", "sdc.yeti-4"]: same modifications as above
  ["yeti-4", "sdd.yeti-4"]: same modifications as above

lib/refrepo/gen/oar-properties.rb

@@ -369,7 +369,7 @@ def get_ref_disk_properties(site_uid, site)
   site['clusters'].each do |cluster_uid, cluster|
     cluster['nodes'].each do |node_uid, node|
       begin
-        properties.merge!(get_ref_disk_properties_internal(site_uid, cluster_uid, node_uid, node))
+        properties.merge!(get_ref_disk_properties_internal(site_uid, cluster_uid, cluster, node_uid, node))
       rescue MissingProperty => e
         puts "Error while processing node #{node_uid}: #{e}"
       end
@@ -576,24 +576,28 @@ end
 #  "diskpath"=>"/dev/disk/by-path/pci-0000:02:00.0-scsi-0:0:1:0",
 #  "cpuset"=>-1},
 #  ["grimoire-1", "sdc.grimoire-1"]=> ...
-def get_ref_disk_properties_internal(site_uid, cluster_uid, node_uid, node)
+def get_ref_disk_properties_internal(site_uid, cluster_uid, cluster, node_uid, node)
   properties = {}
   node['storage_devices'].each_with_index do |device, index|
     disk = [device['device'], node_uid].join('.')
     if index > 0 && device['reservation'] # index > 0 is used to exclude sda
       key = [node_uid, disk]
-      h = {}
-      node_address = [node_uid, site_uid, 'grid5000.fr'].join('.')
-      h['cluster'] = cluster_uid
-      h['host'] = node_address
+      # Start by copying the properties of the resource of type default,
+      # because to reserve both the resources of type disk and of type default
+      # for a same host, almost all properties must have the same value.
+      # In particular, max_walltime must be identic, but mind that that
+      # max_walltime properties is not related to the max duration of a job
+      # containing only disk resources (14days), it is not used for that.
+      h = get_ref_node_properties_internal(cluster_uid, cluster, node_uid, node)
+      # We set the host property because it's not done in what we got above
+      h['host'] = [node_uid, site_uid, 'grid5000.fr'].join('.')
+      # We do not set network_address and available_upto for disk resources
+      # (FIXME: recall why)
       h['network_address'] = ''
       h['available_upto'] = 0
-      h['deploy'] = 'YES'
-      h['production'] = get_production_property(node)
-      h['maintenance'] = get_maintenance_property(node)
+      # We set the disk specific property values
       h['disk'] = disk
       h['diskpath'] = device['by_path']
-      h['exotic'] = node['exotic'] ? 'YES' : 'NO'
       h['cpuset'] = -1
       properties[key] = h
     end
@@ -685,13 +689,12 @@ def diff_properties(type, properties_oar, properties_ref)
 
   if type == 'default'
     ignore_keys = ignore_keys()
-    ignore_keys.each { |key| properties_oar.delete(key) }
-    ignore_keys.each { |key| properties_ref.delete(key) }
   elsif type == 'disk'
     check_keys = %w(cluster host network_address available_upto deploy production maintenance disk diskpath cpuset)
-    properties_oar.select! { |k, _v| check_keys.include?(k) }
-    properties_ref.select! { |k, _v| check_keys.include?(k) }
+    ignore_keys = ignore_keys() - check_keys #Some key must be ignored for default but not for disks, ex: available_upto
   end
+  ignore_keys.each { |key| properties_oar.delete(key) }
+  ignore_keys.each { |key| properties_ref.delete(key) }
 
   # Ignore the 'state' property only if the node is not 'Dead' according to
   # the reference-repo.
@@ -998,7 +1001,7 @@ def do_diff(options, generated_hierarchy, refrepo_properties)
         end
       end
 
-      diagnostic_msgs.push( "Properties that need to be created on the #{site_uid} server: #{properties_keys['diff'][site_uid].keys.to_a.delete_if { |e| ignore_default_keys.include?(e) }.join(', ')}") if options[:verbose] && properties_keys['diff'][site_uid].keys.to_a.delete_if { |e| ignore_default_keys.include?(e) }.size > 0
+      diagnostic_msgs.push( "Properties that need to be created on the #{site_uid} server: #{properties_keys['diff'][site_uid].keys.to_a.delete_if { |e| ignore_keys.include?(e) }.join(', ')}") if options[:verbose] && properties_keys['diff'][site_uid].keys.to_a.delete_if { |e| ignore_keys.include?(e) }.size > 0
 
       # Detect unknown properties
       unknown_properties = properties_keys['oar'][site_uid].keys.to_set - properties_keys['ref'][site_uid].keys.to_set