Commit 48145ca5 authored by Pierre Neyron's avatar Pierre Neyron 🚴 Committed by Pierre Neyron

[gen:oar-propeties] avoid the duplicate calls to the API + no downloading message in verbose = 0

parent bc17b23d
Pipeline #161083 failed with stages
in 4 minutes and 48 seconds
......@@ -15,6 +15,9 @@ MiB = 1024**2
module OarProperties
# OAR API data cache
@@oar_data = {}
def export_rows_as_formated_line(generated_hierarchy)
# Display header
puts "+#{'-' * 10} + #{'-' * 20} + #{'-' * 5} + #{'-' * 5} + #{'-' * 8} + #{'-' * 4} + #{'-' * 20} + #{'-' * 30} + #{'-' * 30}+"
......@@ -612,43 +615,48 @@ end
# Get all data from the OAR database
def get_oar_data(site_uid, options)
data_key = [site_uid,options].to_s
# is data already in cache ?
if @@oar_data[data_key].nil?
# If no API URL is given, set a default URL on https://api.grid5000.fr
if not options[:api][:uri]
options[:api][:uri] = "https://api.grid5000.fr"
end
# If no API URL is given, set a default URL on https://api.grid5000.fr
if not options[:api][:uri]
options[:api][:uri] = "https://api.grid5000.fr"
end
# Preparing the URL that will be used to fetch OAR resources
if options[:api][:uri].include? "api.grid5000.fr"
api_uri = URI.parse('https://api.grid5000.fr/stable/sites/' + site_uid + '/internal/oarapi/resources/details.json?limit=999999')
elsif options[:api][:uri].include? "api-ext.grid5000.fr"
api_uri = URI.parse('https://api-ext.grid5000.fr/stable/sites/' + site_uid + '/internal/oarapi/resources/details.json?limit=999999')
else
api_uri = URI.parse(options[:api][:uri]+'/oarapi/resources/details.json?limit=999999')
end
# Preparing the URL that will be used to fetch OAR resources
if options[:api][:uri].include? "api.grid5000.fr"
api_uri = URI.parse('https://api.grid5000.fr/stable/sites/' + site_uid + '/internal/oarapi/resources/details.json?limit=999999')
elsif options[:api][:uri].include? "api-ext.grid5000.fr"
api_uri = URI.parse('https://api-ext.grid5000.fr/stable/sites/' + site_uid + '/internal/oarapi/resources/details.json?limit=999999')
else
api_uri = URI.parse(options[:api][:uri]+'/oarapi/resources/details.json?limit=999999')
end
# Download the OAR properties from the OAR API (through G5K API)
puts "Downloading resources properties from #{api_uri} ..." if options[:verbose]
http = Net::HTTP.new(api_uri.host, api_uri.port)
if api_uri.scheme == "https"
http.use_ssl = true
end
request = Net::HTTP::Get.new(api_uri.request_uri, {'User-Agent' => 'reference-repository/gen/oar-properties'})
# Download the OAR properties from the OAR API (through G5K API)
puts "Downloading resources properties from #{api_uri} ..." if options[:verbose] and options[:verbose] > 0
http = Net::HTTP.new(api_uri.host, api_uri.port)
if api_uri.scheme == "https"
http.use_ssl = true
end
request = Net::HTTP::Get.new(api_uri.request_uri, {'User-Agent' => 'reference-repository/gen/oar-properties'})
# For outside g5k network access
if options[:api][:user] && options[:api][:pwd]
request.basic_auth(options[:api][:user], options[:api][:pwd])
end
# For outside g5k network access
if options[:api][:user] && options[:api][:pwd]
request.basic_auth(options[:api][:user], options[:api][:pwd])
end
response = http.request(request)
raise "Failed to fetch resources properties from API: \n#{response.body}\n" unless response.code.to_i == 200
puts '... done' if options[:verbose]
response = http.request(request)
raise "Failed to fetch resources properties from API: \n#{response.body}\n" unless response.code.to_i == 200
puts '... done' if options[:verbose] and options[:verbose] > 0
oarnodes = JSON.parse(response.body)
oarnodes = JSON.parse(response.body)
# Adapt from the format of the OAR API
oarnodes = oarnodes['items'] if oarnodes.key?('items')
return oarnodes
# Adapt from the format of the OAR API
oarnodes = oarnodes['items'] if oarnodes.key?('items')
@@oar_data[data_key] = oarnodes
end
# Return a deep copy of the cached value as it will be modified in place
return Marshal.load(Marshal.dump(@@oar_data[data_key]))
end
def get_property_keys_internal(_type, type_properties)
......@@ -1470,6 +1478,10 @@ end
def generate_oar_properties(options)
# Reset the OAR API cache, because the rpec tests change the data in our back
# while calling multiple times this function.
@@oar_data = {}
options[:api] ||= {}
conf = RefRepo::Utils.get_api_config
options[:api][:user] = conf['username']
......
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
clustera-1: new node !
["+", "besteffort", "YES"]
......@@ -461,5 +457,3 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
["+", "virtual", "ivt"]
["+", "wattmeter", "MULTIPLE"]
Properties that need to be created on the fakesite server: ip, cluster, nodemodel, switch, besteffort, deploy, virtual, cpuarch, cpucore, cputype, cpufreq, disktype, eth_count, eth_rate, ib_count, ib_rate, ib, opa_count, opa_rate, myri_count, myri_rate, myri, memcore, memcpu, memnode, gpu_count, exotic, mic, wattmeter, cluster_priority, max_walltime, production, maintenance, disk_reservation_count
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
clustera-1: new node !
["+", "besteffort", "YES"]
......@@ -146,5 +142,3 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
["+", "virtual", "ivt"]
["+", "wattmeter", "MULTIPLE"]
Properties that need to be created on the fakesite server: ip, cluster, nodemodel, switch, besteffort, deploy, virtual, cpuarch, cpucore, cputype, cpufreq, disktype, eth_count, eth_rate, ib_count, ib_rate, ib, opa_count, opa_rate, myri_count, myri_rate, myri, memcore, memcpu, memnode, gpu_count, exotic, mic, wattmeter, cluster_priority, max_walltime, production, maintenance, disk_reservation_count
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
clustera-1: new node !
["+", "besteffort", "YES"]
......@@ -726,5 +722,3 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
["+", "network_address", ""]
["+", "production", "NO"]
Properties that need to be created on the fakesite server: ip, cluster, nodemodel, switch, besteffort, deploy, virtual, cpuarch, cpucore, cputype, cpufreq, disktype, eth_count, eth_rate, ib_count, ib_rate, ib, opa_count, opa_rate, myri_count, myri_rate, myri, memcore, memcpu, memnode, gpu_count, exotic, mic, wattmeter, cluster_priority, max_walltime, production, maintenance, disk_reservation_count, disk, diskpath
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
clustera-1: new node !
["+", "besteffort", "YES"]
......@@ -181,5 +177,3 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
["+", "virtual", "amd-v"]
["+", "wattmeter", "NO"]
Properties that need to be created on the fakesite server: ip, cluster, nodemodel, switch, besteffort, deploy, virtual, cpuarch, cpucore, cputype, cpufreq, disktype, eth_count, eth_rate, ib_count, ib_rate, ib, opa_count, opa_rate, myri_count, myri_rate, myri, memcore, memcpu, memnode, gpu_count, exotic, mic, wattmeter, cluster_priority, max_walltime, production, maintenance, disk_reservation_count
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
clustera-1:
["-", "opa", "NO"]
......@@ -19,8 +15,6 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
Properties that need to be created on the fakesite server: exotic
Properties existing on the fakesite server but not managed/known by the generator: opa, disk, diskpath.
Hint: you can delete properties with 'oarproperty -d <property>' or add them to the ignore list in lib/lib-oar-properties.rb.
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
# Error: Resource 12215 (host=clustera-1.nancy.grid5000.fr cpu=1765 core=10967 cpuset=0 gpu= gpudevice=) has a mismatch for ressource GPU: OAR API gives , generator wants 105.
# Error: Resource 12215 (host=clustera-1.nancy.grid5000.fr cpu=1765 core=10967 cpuset=0 gpu= gpudevice=) has a mismatch for ressource GPUDEVICE: OAR API gives , generator wants 0.
# Error: Resource 12216 (host=clustera-1.nancy.grid5000.fr cpu=1765 core=10968 cpuset=2 gpu= gpudevice=) has a mismatch for ressource GPU: OAR API gives , generator wants 106.
......
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
clustera-1: new node !
["+", "besteffort", "YES"]
......@@ -278,5 +274,3 @@ Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for ad
["+", "network_address", ""]
["+", "production", "NO"]
Properties that need to be created on the fakesite server: ip, cluster, nodemodel, switch, besteffort, deploy, virtual, cpuarch, cpucore, cputype, cpufreq, disktype, eth_count, eth_rate, ib_count, ib_rate, ib, opa_count, opa_rate, myri_count, myri_rate, myri, memcore, memcpu, memnode, gpu_count, exotic, mic, wattmeter, cluster_priority, max_walltime, production, maintenance, disk_reservation_count, disk, diskpath
Downloading resources properties from https://api.grid5000.fr/stable/sites/fakesite/internal/oarapi/resources/details.json?limit=999999 ...
... done
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment