[gen:oar-propeties] avoid the duplicate calls to the API + no downloading message in verbose = 0

lib/refrepo/gen/oar-properties.rb

@@ -15,6 +15,9 @@ MiB = 1024**2
 
 module OarProperties
 
+# OAR API data cache
+@@oar_data = {}
+
 def export_rows_as_formated_line(generated_hierarchy)
   # Display header
   puts "+#{'-' * 10} + #{'-' * 20} + #{'-' * 5} + #{'-' * 5} + #{'-' * 8} + #{'-' * 4} + #{'-' * 20} + #{'-' * 30} + #{'-' * 30}+"
@@ -612,43 +615,48 @@ end
 
 # Get all data from the OAR database
 def get_oar_data(site_uid, options)
+  data_key = [site_uid,options].to_s
+  # is data already in cache ?
+  if @@oar_data[data_key].nil?
+    # If no API URL is given, set a default URL on https://api.grid5000.fr
+    if not options[:api][:uri]
+      options[:api][:uri] = "https://api.grid5000.fr"
+    end
 
-  # If no API URL is given, set a default URL on https://api.grid5000.fr
-  if not options[:api][:uri]
-    options[:api][:uri] = "https://api.grid5000.fr"
-  end
-
-  # Preparing the URL that will be used to fetch OAR resources
-  if options[:api][:uri].include? "api.grid5000.fr"
-    api_uri = URI.parse('https://api.grid5000.fr/stable/sites/' + site_uid  + '/internal/oarapi/resources/details.json?limit=999999')
-  elsif options[:api][:uri].include? "api-ext.grid5000.fr"
-    api_uri = URI.parse('https://api-ext.grid5000.fr/stable/sites/' + site_uid  + '/internal/oarapi/resources/details.json?limit=999999')
-  else
-    api_uri = URI.parse(options[:api][:uri]+'/oarapi/resources/details.json?limit=999999')
-  end
+    # Preparing the URL that will be used to fetch OAR resources
+    if options[:api][:uri].include? "api.grid5000.fr"
+      api_uri = URI.parse('https://api.grid5000.fr/stable/sites/' + site_uid  + '/internal/oarapi/resources/details.json?limit=999999')
+    elsif options[:api][:uri].include? "api-ext.grid5000.fr"
+      api_uri = URI.parse('https://api-ext.grid5000.fr/stable/sites/' + site_uid  + '/internal/oarapi/resources/details.json?limit=999999')
+    else
+      api_uri = URI.parse(options[:api][:uri]+'/oarapi/resources/details.json?limit=999999')
+    end
 
-  # Download the OAR properties from the OAR API (through G5K API)
-  puts "Downloading resources properties from #{api_uri} ..." if options[:verbose]
-  http = Net::HTTP.new(api_uri.host, api_uri.port)
-  if api_uri.scheme == "https"
-    http.use_ssl = true
-  end
-  request = Net::HTTP::Get.new(api_uri.request_uri, {'User-Agent' => 'reference-repository/gen/oar-properties'})
+    # Download the OAR properties from the OAR API (through G5K API)
+    puts "Downloading resources properties from #{api_uri} ..." if options[:verbose] and options[:verbose] > 0
+    http = Net::HTTP.new(api_uri.host, api_uri.port)
+    if api_uri.scheme == "https"
+      http.use_ssl = true
+    end
+    request = Net::HTTP::Get.new(api_uri.request_uri, {'User-Agent' => 'reference-repository/gen/oar-properties'})
 
-  # For outside g5k network access
-  if options[:api][:user] && options[:api][:pwd]
-    request.basic_auth(options[:api][:user], options[:api][:pwd])
-  end
+    # For outside g5k network access
+    if options[:api][:user] && options[:api][:pwd]
+      request.basic_auth(options[:api][:user], options[:api][:pwd])
+    end
 
-  response = http.request(request)
-  raise "Failed to fetch resources properties from API: \n#{response.body}\n" unless response.code.to_i == 200
-  puts '... done' if options[:verbose]
+    response = http.request(request)
+    raise "Failed to fetch resources properties from API: \n#{response.body}\n" unless response.code.to_i == 200
+    puts '... done' if options[:verbose] and options[:verbose] > 0
 
-  oarnodes = JSON.parse(response.body)
+    oarnodes = JSON.parse(response.body)
 
-  # Adapt from the format of the OAR API
-  oarnodes = oarnodes['items'] if oarnodes.key?('items')
-  return oarnodes
+    # Adapt from the format of the OAR API
+    oarnodes = oarnodes['items'] if oarnodes.key?('items')
+    @@oar_data[data_key] = oarnodes
+  end
+  # Return a deep copy of the cached value as it will be modified in place
+  return Marshal.load(Marshal.dump(@@oar_data[data_key]))
 end
 
 def get_property_keys_internal(_type, type_properties)
@@ -1470,6 +1478,10 @@ end
 
 def generate_oar_properties(options)
 
+  # Reset the OAR API cache, because the rpec tests change the data in our back
+  # while calling multiple times this function.
+  @@oar_data = {}
+
   options[:api] ||= {}
   conf = RefRepo::Utils.get_api_config
   options[:api][:user] = conf['username']

spec/output/graffiti_empty_diff_stdout.txt

 Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
 ... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
 Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
   clustera-1: new node !
     ["+", "besteffort", "YES"]
@@ -461,5 +457,3 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
     ["+", "virtual", "ivt"]
     ["+", "wattmeter", "MULTIPLE"]
 Properties that need to be created on the fakesite server: ip, cluster, nodemodel, switch, besteffort, deploy, virtual, cpuarch, cpucore, cputype, cpufreq, disktype, eth_count, eth_rate, ib_count, ib_rate, ib, opa_count, opa_rate, myri_count, myri_rate, myri, memcore, memcpu, memnode, gpu_count, exotic, mic, wattmeter, cluster_priority, max_walltime, production, maintenance, disk_reservation_count
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done

spec/output/graphite_empty_diff_stdout.txt

 Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
 ... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
 Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
   clustera-1: new node !
     ["+", "besteffort", "YES"]
@@ -146,5 +142,3 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
     ["+", "virtual", "ivt"]
     ["+", "wattmeter", "MULTIPLE"]
 Properties that need to be created on the fakesite server: ip, cluster, nodemodel, switch, besteffort, deploy, virtual, cpuarch, cpucore, cputype, cpufreq, disktype, eth_count, eth_rate, ib_count, ib_rate, ib, opa_count, opa_rate, myri_count, myri_rate, myri, memcore, memcpu, memnode, gpu_count, exotic, mic, wattmeter, cluster_priority, max_walltime, production, maintenance, disk_reservation_count
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done

spec/output/grimoire_empty_diff_stdout.txt

 Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
 ... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
 Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
   clustera-1: new node !
     ["+", "besteffort", "YES"]
@@ -726,5 +722,3 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
     ["+", "network_address", ""]
     ["+", "production", "NO"]
 Properties that need to be created on the fakesite server: ip, cluster, nodemodel, switch, besteffort, deploy, virtual, cpuarch, cpucore, cputype, cpufreq, disktype, eth_count, eth_rate, ib_count, ib_rate, ib, opa_count, opa_rate, myri_count, myri_rate, myri, memcore, memcpu, memnode, gpu_count, exotic, mic, wattmeter, cluster_priority, max_walltime, production, maintenance, disk_reservation_count, disk, diskpath
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done

spec/output/grue_empty_diff_stdout.txt

 Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
 ... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
 Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
   clustera-1: new node !
     ["+", "besteffort", "YES"]
@@ -181,5 +177,3 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
     ["+", "virtual", "amd-v"]
     ["+", "wattmeter", "NO"]
 Properties that need to be created on the fakesite server: ip, cluster, nodemodel, switch, besteffort, deploy, virtual, cpuarch, cpucore, cputype, cpufreq, disktype, eth_count, eth_rate, ib_count, ib_rate, ib, opa_count, opa_rate, myri_count, myri_rate, myri, memcore, memcpu, memnode, gpu_count, exotic, mic, wattmeter, cluster_priority, max_walltime, production, maintenance, disk_reservation_count
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done

spec/output/grue_nogpus_diff_stdout.txt

 Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
 ... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
 Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
   clustera-1:
     ["-", "opa", "NO"]
@@ -19,8 +15,6 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
 Properties that need to be created on the fakesite server: exotic
 Properties existing on the fakesite server but not managed/known by the generator: opa, disk, diskpath.
 Hint: you can delete properties with 'oarproperty -d <property>' or add them to the ignore list in lib/lib-oar-properties.rb.
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
 # Error: Resource 12215 (host=clustera-1.nancy.grid5000.fr cpu=1765 core=10967 cpuset=0 gpu= gpudevice=) has a mismatch for ressource GPU: OAR API gives , generator wants 105.
 # Error: Resource 12215 (host=clustera-1.nancy.grid5000.fr cpu=1765 core=10967 cpuset=0 gpu= gpudevice=) has a mismatch for ressource GPUDEVICE: OAR API gives , generator wants 0.
 # Error: Resource 12216 (host=clustera-1.nancy.grid5000.fr cpu=1765 core=10968 cpuset=2 gpu= gpudevice=) has a mismatch for ressource GPU: OAR API gives , generator wants 106.

spec/output/yeti_empty_diff_stdout.txt

 Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
 ... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done
 Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
   clustera-1: new node !
     ["+", "besteffort", "YES"]
@@ -278,5 +274,3 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
     ["+", "network_address", ""]
     ["+", "production", "NO"]
 Properties that need to be created on the fakesite server: ip, cluster, nodemodel, switch, besteffort, deploy, virtual, cpuarch, cpucore, cputype, cpufreq, disktype, eth_count, eth_rate, ib_count, ib_rate, ib, opa_count, opa_rate, myri_count, myri_rate, myri, memcore, memcpu, memnode, gpu_count, exotic, mic, wattmeter, cluster_priority, max_walltime, production, maintenance, disk_reservation_count, disk, diskpath
-Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
-... done