Attention une mise à jour du service Gitlab va être effectuée le mardi 14 décembre entre 13h30 et 14h00. Cette mise à jour va générer une interruption du service dont nous ne maîtrisons pas complètement la durée mais qui ne devrait pas excéder quelques minutes.

oar-properties.rb 28.9 KB
Newer Older
1
# coding: utf-8
2 3

require 'hashdiff'
4
require 'refrepo/data_loader'
5
require 'net/ssh'
6

7
class MissingProperty < StandardError; end
8 9 10

MiB = 1024**2

11 12 13 14 15 16 17 18
def get_ids(host)
  node_uid, site_uid, grid_uid, _tdl = host.split('.')
  cluster_uid, node_num = node_uid.split('-')
  ids = { 'node_uid' => node_uid, 'site_uid' => site_uid, 'grid_uid' => grid_uid, 'cluster_uid' => cluster_uid, 'node_num' => node_num }
  return ids
end

# Get all node properties of a given site from the reference repo hash
19
# See also: https://www.grid5000.fr/mediawiki/index.php/Reference_Repository
20 21 22 23 24 25 26
def get_ref_default_properties(_site_uid, site)
  properties = {}
  site['clusters'].each do |cluster_uid, cluster|
    cluster['nodes'].each do |node_uid, node|
      begin
        properties[node_uid] = get_ref_node_properties_internal(cluster_uid, cluster, node_uid, node)
      rescue MissingProperty => e
27 28 29 30 31 32
        puts "Error (missing property) while processing node #{node_uid}: #{e}"
      rescue Exception => e
        puts "FATAL ERROR while processing node #{node_uid}: #{e}"
        puts "Description of the node is:"
        pp node
        raise
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
      end
    end
  end
  return properties
end

def get_ref_disk_properties(site_uid, site)
  properties = {}
  site['clusters'].each do |cluster_uid, cluster|
    cluster['nodes'].each do |node_uid, node|
      begin
        properties.merge!(get_ref_disk_properties_internal(site_uid, cluster_uid, node_uid, node))
      rescue MissingProperty => e
        puts "Error while processing node #{node_uid}: #{e}"
      end
    end
  end
  return properties
end

# Generates the properties of a single node
def get_ref_node_properties_internal(cluster_uid, cluster, node_uid, node)
  h = {}
56

57
  if node['status'] == 'retired'
58 59
    # For dead nodes, additional information is most likely missing
    # from the ref-repository: just return the state
60
    h['state'] = 'Dead'
61
    return h
62 63
  end

64 65
  main_network_adapter = node['network_adapters'].find { |na| /^eth[0-9]*$/.match(na['device']) && na['enabled'] && na['mounted'] && !na['management'] }

66
  raise MissingProperty, "Node #{node_uid} does not have a main network_adapter (ie. an ethernet interface with enabled=true && mounted==true && management==false)" unless main_network_adapter
67

68
  h['ip'] = main_network_adapter['ip']
69
  raise MissingProperty, "Node #{node_uid} has no IP" unless h['ip']
70 71
  h['cluster'] = cluster_uid
  h['nodemodel'] = cluster['model']
72
  h['switch'] = main_network_adapter['switch']
73 74 75 76 77 78 79
  h['besteffort'] = node['supported_job_types']['besteffort']
  h['deploy'] = node['supported_job_types']['deploy']
  h['virtual'] = node['supported_job_types']['virtual']
  h['cpuarch'] = node['architecture']['platform_type']
  h['cpucore'] = node['architecture']['nb_cores'] / node['architecture']['nb_procs']
  h['cputype'] = [node['processor']['model'], node['processor']['version']].join(' ')
  h['cpufreq'] = node['processor']['clock_speed'] / 1_000_000_000.0
80
  h['disktype'] = (node['storage_devices'].first || {})['interface']
81

82
  # ETH
83
  ni_mountable = node['network_adapters'].select { |na| /^eth[0-9]*$/.match(na['device']) && (na['enabled'] == true && (na['mounted'] == true || na['mountable'] == true)) }
84
  ni_fastest   = ni_mountable.max_by { |na| na['rate'] || 0 }
85

86 87
  h['eth_count'] = ni_mountable.length
  h['eth_rate']  = ni_fastest['rate'] / 1_000_000_000
88

89 90
  puts "#{node_uid}: Warning - no rate info for the eth interface" if h['eth_count'] > 0 && h['eth_rate'] == 0

91
  # INFINIBAND
92
  ni_mountable = node['network_adapters'].select { |na| /^ib[0-9]*(\.[0-9]*)?$/.match(na['device']) && (na['interface'] == 'InfiniBand' and na['enabled'] == true && (na['mounted'] == true || na['mountable'] == true)) }
93
  ni_fastest   = ni_mountable.max_by { |na| na['rate'] || 0 }
94
  ib_map = { 0 => 'NO', 10 => 'SDR', 20 => 'DDR', 40 => 'QDR', 56 => 'FDR' }
95 96 97

  h['ib_count'] = ni_mountable.length
  h['ib_rate']  = ni_mountable.length > 0 ? ni_fastest['rate'] / 1_000_000_000 : 0
98
  h['ib'] = ib_map[h['ib_rate']]
99

100
  puts "#{node_uid}: Warning - no rate info for the ib interface" if h['ib_count'] > 0 && h['ib_rate'] == 0
Lucas Nussbaum's avatar
Lucas Nussbaum committed
101 102
  
  # OMNIPATH
103
  ni_mountable = node['network_adapters'].select { |na| /^ib[0-9]*(\.[0-9]*)?$/.match(na['device']) && (na['interface'] == 'Omni-Path' and na['enabled'] == true && (na['mounted'] == true || na['mountable'] == true)) }
Lucas Nussbaum's avatar
Lucas Nussbaum committed
104 105 106 107 108 109 110 111
  ni_fastest   = ni_mountable.max_by { |na| na['rate'] || 0 }

  h['opa_count'] = ni_mountable.length
  h['opa_rate']  = ni_mountable.length > 0 ? ni_fastest['rate'] / 1_000_000_000 : 0
  h['opa'] = h['opa_count'] > 0

  puts "#{node_uid}: Warning - no rate info for the opa interface" if h['opa_count'] > 0 && h['opa_rate'] == 0

112

113
  # MYRINET
114
  ni_mountable = node['network_adapters'].select { |na| /^myri[0-9]*$/.match(na['device']) && (na['enabled'] == true && (na['mounted'] == true || na['mountable'] == true)) }
115
  ni_fastest   = ni_mountable.max_by { |na| na['rate'] || 0 }
116
  myri_map = { 0 => 'NO', 2 => 'Myrinet-2000', 10 => 'Myri-10G' }
117 118 119

  h['myri_count'] = ni_mountable.length
  h['myri_rate']  = ni_mountable.length > 0 ? ni_fastest['rate'] / 1_000_000_000 : 0
120
  h['myri'] = myri_map[h['myri_rate']]
121

122
  puts "#{node_uid}: Warning - no rate info for the myri interface" if h['myri_count'] > 0 && h['myri_rate'] == 0
123

124 125 126
  h['memcore'] = node['main_memory']['ram_size'] / node['architecture']['nb_cores']/MiB
  h['memcpu'] = node['main_memory']['ram_size'] / node['architecture']['nb_procs']/MiB
  h['memnode'] = node['main_memory']['ram_size'] / MiB
127

128
  if node.key?('gpu') && node['gpu']['gpu'] == true
129
    h['gpu'] = node['gpu']['gpu_model']
130 131 132 133 134
    h['gpu_count'] = node['gpu']['gpu_count']
  else
    h['gpu'] = false
    h['gpu_count'] = 0
  end
135

136 137 138 139 140
  h['mic'] = if node['mic']
               'YES'
             else
               'NO'
             end
141

142
  node['monitoring'] ||= {}
143
  h['wattmeter'] = case node['monitoring']['wattmeter']
144 145
                   when "true" then true
                   when "false" then false
146 147 148
                   when nil then false
                   else node['monitoring']['wattmeter'].upcase
                   end
149 150

  h['cluster_priority'] = (cluster['priority'] || Time.parse(cluster['created_at'].to_s).strftime('%Y%m')).to_i
151

152 153
  h['max_walltime'] = 0 # default
  h['max_walltime'] = node['supported_job_types']['max_walltime'] if node['supported_job_types'] && node['supported_job_types'].has_key?('max_walltime')
154

155 156
  h['production'] = get_production_property(node)
  h['maintenance'] = get_maintenance_property(node)
157 158

  # Disk reservation
159
  h['disk_reservation_count'] = node['storage_devices'].select { |v| v['reservation'] }.length
160

161
  # convert booleans to YES/NO string
162
  h.each do |k, v|
163 164 165 166 167
    if v == true
      h[k] = 'YES'
    elsif v == false
      h[k] = 'NO'
    elsif v.is_a? Float
168
      h[k] = v.to_s
169
    end
170
  end
171

172
  return h
173
end
174

175 176 177
def get_production_property(node)
  production = false # default
  production = node['supported_job_types']['queues'].include?('production') if node['supported_job_types'] && node['supported_job_types'].has_key?('queues')
178
  production = production == true ? 'YES' : 'NO'
179 180 181 182 183 184
  return production
end

def get_maintenance_property(node)
  maintenance = false # default
  maintenance = node['supported_job_types']['queues'].include?('testing') if node['supported_job_types'] && node['supported_job_types'].has_key?('queues')
185
  maintenance = maintenance == true ? 'YES' : 'NO'
186 187 188
  return maintenance
end

189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
# Returns the expected properties of the reservable disks. These
# properties are then compared with the values in OAR database, to
# generate a diff.
# The key is of the form [node, disk]. In the following example
# we list the different disks (from sdb to sdf) of node grimoire-1.
# {["grimoire-1", "sdb.grimoire-1"]=>
#  {"cluster"=>"grimoire",
#  "host"=>"grimoire-1.nancy.grid5000.fr",
#  "network_address"=>"",
#  "available_upto"=>0,
#  "deploy"=>"YES",
#  "production"=>"NO",
#  "maintenance"=>"NO",
#  "disk"=>"sdb.grimoire-1",
#  "diskpath"=>"/dev/disk/by-path/pci-0000:02:00.0-scsi-0:0:1:0",
#  "cpuset"=>-1},
#  ["grimoire-1", "sdc.grimoire-1"]=> ...
206 207
def get_ref_disk_properties_internal(site_uid, cluster_uid, node_uid, node)
  properties = {}
208 209
  node['storage_devices'].each_with_index do |device, index|
    disk = [device['device'], node_uid].join('.')
210
    if index > 0 && device['reservation'] # index > 0 is used to exclude sda
211
      key = [node_uid, disk]
212 213 214 215
      h = {}
      node_address = [node_uid, site_uid, 'grid5000.fr'].join('.')
      h['cluster'] = cluster_uid
      h['host'] = node_address
Florent Didier's avatar
Florent Didier committed
216
      h['network_address'] = ''
217 218 219 220
      h['available_upto'] = 0
      h['deploy'] = 'YES'
      h['production'] = get_production_property(node)
      h['maintenance'] = get_maintenance_property(node)
221
      h['disk'] = disk
222
      h['diskpath'] = device['by_path']
223
      h['cpuset'] = -1
224 225 226 227 228
      properties[key] = h
    end
  end
  properties
end
229

230
def get_oar_default_properties(site_uid, filename, options)
Florent Didier's avatar
Florent Didier committed
231
  oarnodes = get_oar_data(site_uid, filename, options)
232

233 234 235
  # Handle the two possible input format from oarnodes -Y:
  # given by a file, and from the OAR API
  if oarnodes.is_a?(Hash)
Florent Didier's avatar
Florent Didier committed
236 237
    oarnodes = oarnodes.map { |_k, v| v['type'] == 'default' ? [get_ids(v['host'])['node_uid'], v] : [nil, nil] }.to_h
    oarnodes.delete(nil)
238 239 240 241 242 243 244 245 246 247 248 249 250 251
  elsif oarnodes.is_a?(Array)
    oarnodes = oarnodes.select { |v| v['type'] == 'default' }.map { |v| [get_ids(v['host'])['node_uid'], v] }.to_h
  else
    raise 'Invalid input format for OAR properties'
  end
  return oarnodes
end

def get_oar_disk_properties(site_uid, filename, options)
  oarnodes = get_oar_data(site_uid, filename, options)

  # Handle the two possible input format from oarnodes -Y:
  # given by a file, and from the OAR API
  if oarnodes.is_a?(Hash)
Florent Didier's avatar
Florent Didier committed
252 253
    oarnodes = oarnodes.map { |_k, v|  v['type'] == 'disk' ? [[get_ids(v['host'])['node_uid'], v['disk']], v] : [nil, nil] }.to_h
    oarnodes.delete(nil)
254
  elsif oarnodes.is_a?(Array)
Florent Didier's avatar
Florent Didier committed
255
    oarnodes = oarnodes.select { |v| v['type'] == 'disk' }.map { |v| [[get_ids(v['host'])['node_uid'], v['disk']], v] }.to_h
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
  else
    raise 'Invalid input format for OAR properties'
  end
  return oarnodes
end

# Get all data from the OAR database
def get_oar_data(site_uid, filename, options)
  oarnodes = ''
  if filename && File.exist?(filename)
    # Read OAR properties from file
    puts "Reading OAR resources properties from file #{filename}" if options[:verbose]
    oarnodes = YAML.load(File.open(filename, 'rb') { |f| f.read })
  else
    api_uri = URI.parse('https://api.grid5000.fr/stable/sites/' + site_uid  + '/internal/oarapi/resources/details.json?limit=999999')

    # Download the OAR properties from the OAR API (through G5K API)
    puts "Downloading resources properties from #{api_uri} ..." if options[:verbose]
    http = Net::HTTP.new(api_uri.host, Net::HTTP.https_default_port)
    http.use_ssl = true
    request = Net::HTTP::Get.new(api_uri.request_uri)
277

278 279 280 281 282 283 284 285 286 287 288 289 290
    # For outside g5k network access
    if options[:api][:user] && options[:api][:pwd]
      request.basic_auth(options[:api][:user], options[:api][:pwd])
    end

    response = http.request(request)
    raise "Failed to fetch resources properties from API: \n#{response.body}\n" unless response.code.to_i == 200
    puts '... done' if options[:verbose]

    oarnodes = JSON.parse(response.body)
    if filename
      puts "Saving OAR resources properties as #{filename}" if options[:verbose]
      File.write(filename, YAML.dump(oarnodes))
291 292 293
    end
  end

294 295 296
  # Adapt from the format of the OAR API
  oarnodes = oarnodes['items'] if oarnodes.key?('items')
  return oarnodes
297 298
end

299 300 301 302
# Return a list of properties as a hash: { property1 => String, property2 => Fixnum, ... }
# We detect the type of the property (Fixnum/String) by looking at the existing values
def get_property_keys(properties)
  properties_keys = {}
303 304
  properties.each do |type, type_properties|
    properties_keys.merge!(get_property_keys_internal(type, type_properties))
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
  end
  return properties_keys
end

def get_property_keys_internal(_type, type_properties)
  properties_keys = {}
  type_properties.each do |key, node_properties|
    # Differenciate between 'default' type (key = node_uid)
    # and 'disk' type (key = [node_uid, disk_id])
    node_uid, = key
    next if node_uid.nil?
    node_properties.each do |k, v|
      next if properties_keys.key?(k)
      next if NilClass === v
      # also skip detection if 'v == false' because it seems that if a varchar property
      # only as 'NO' values, it might be interpreted as a boolean
      # (see the ib property at nantes: ib: NO in the YAML instead of ib: 'NO')
      next if v == false
      properties_keys[k] = v.class
    end
  end
  return properties_keys
end

def diff_properties(type, properties_oar, properties_ref)
  properties_oar ||= {}
  properties_ref ||= {}

Florent Didier's avatar
Florent Didier committed
333 334 335 336 337
  if type == 'default'
    ignore_keys = ignore_keys()
    ignore_keys.each { |key| properties_oar.delete(key) }
    ignore_keys.each { |key| properties_ref.delete(key) }
  elsif type == 'disk'
338 339 340
    check_keys = %w(cluster host network_address available_upto deploy production maintenance disk diskpath cpuset)
    properties_oar.select! { |k, _v| check_keys.include?(k) }
    properties_ref.select! { |k, _v| check_keys.include?(k) }
Florent Didier's avatar
Florent Didier committed
341
  end
342 343 344 345 346 347 348 349 350 351

  # Ignore the 'state' property only if the node is not 'Dead' according to
  # the reference-repo.
  # Otherwise, we must enforce that the node state is also 'Dead' on the OAR server.
  # On the OAR server, the 'state' property can be modified by phoenix. We ignore that.
  if type == 'default' && properties_ref['state'] != 'Dead'
    properties_oar.delete('state')
    properties_ref.delete('state')
  elsif type == 'default' && properties_ref.size == 1
    # For dead nodes, when information is missing from the reference-repo, only enforce the 'state' property and ignore other differences.
352
    return HashDiff.diff({'state' => properties_oar['state']}, {'state' => properties_ref['state']})
353 354 355 356
  end

  return HashDiff.diff(properties_oar, properties_ref)
end
357

358
# These keys will not be created neither compared with the -d option
359 360
# ignore_default_keys is only applied to resources of type 'default'
def ignore_default_keys()
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
  # default OAR at resource creation:
  #  available_upto: '2147483647'
  #  besteffort: 'YES'
  #  core: ~
  #  cpu: ~
  #  cpuset: 0
  #  deploy: 'NO'
  #  desktop_computing: 'NO'
  #  drain: 'NO'
  #  expiry_date: 0
  #  finaud_decision: 'YES'
  #  host: ~
  #  last_available_upto: 0
  #  last_job_date: 0
  #  network_address: server
  #  next_finaud_decision: 'NO'
  #  next_state: UnChanged
  #  resource_id: 9
  #  scheduler_priority: 0
  #  state: Suspected
  #  state_num: 3
  #  suspended_jobs: 'NO'
  #  type: default
384
  ignore_default_keys = [
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
    "chassis",
    "slash_16",
    "slash_17",
    "slash_18",
    "slash_19",
    "slash_20",
    "slash_21",
    "slash_22",
    "available_upto",
    "chunks",
    "comment", # TODO
    "core",
    "cpu",
    "cpuset",
    "desktop_computing",
    "drain",
    "expiry_date",
    "finaud_decision",
    "grub",
    "host", # TODO
    "jobs", # This property exists when a job is running
    "last_available_upto",
    "last_job_date",
    "network_address", # TODO
    "next_finaud_decision",
    "next_state",
    "rconsole", # TODO
    "resource_id",
    "scheduler_priority",
    "state",
    "state_num",
    "subnet_address",
    "subnet_prefix",
    "suspended_jobs",
    "thread",
    "type", # TODO
    "vlan",
    "pdu",
    "id", # id from API (= resource_id from oarnodes)
    "api_timestamp", # from API
425 426 427 428 429 430 431 432 433
    "links" # from API
  ]
  return ignore_default_keys
end

# Properties of resources of type 'disk' to ignore (for example, when
# comparing resources of type 'default' with the -d option)
def ignore_disk_keys()
  ignore_disk_keys = [
Florent Didier's avatar
Florent Didier committed
434 435
    "disk",
    "diskpath"
436
  ]
437 438 439 440 441
  return ignore_disk_keys
end

def ignore_keys()
  return ignore_default_keys() + ignore_disk_keys()
442 443
end

444 445
def oarcmd_script_header()
  return <<EOF
446 447
#! /usr/bin/env bash

448
set -eu
449
set -o pipefail
450 451 452 453 454 455

EOF
end

def oarcmd_create_node_header()
  return <<EOF
456 457 458
node_exist () {
  [[ $(oarnodes --sql "host='$1' and type='default'") ]]
}
459

460
disk_exist () {
461
  [[ $(oarnodes --sql "host='$1' and type='disk' and disk='$2'") ]]
462 463 464 465 466
}

EOF
end

467 468 469
def oarcmd_separator
  return "echo '" + '=' * 80 + "'\n\n"
end
470

471 472 473 474 475 476 477 478 479 480 481 482 483
def oarcmd_create_properties(properties_keys)
  command = ''
  properties_keys.each do |key, key_type|
    if key_type == Fixnum
      command += "oarproperty -a #{key} || true\n"
    elsif key_type == String
      command += "oarproperty -a #{key} --varchar || true\n"
    else
      raise "Error: the type of the '#{key}' property is unknown (Integer/String). Cannot generate the corresponding 'oarproperty' command. You must create this property manually ('oarproperty -a #{key} [--varchar]')"
    end
  end
  return command
end
484

485 486 487
def oarcmd_create_node(host, default_properties, node_hash)
  id = get_ids(host)
  node_exist = "node_exist '#{host}'"
488
  command  = "echo; echo 'Adding host #{host}:'\n"
489 490
  command += "#{node_exist} && echo '=> host already exists'\n"
  command += "#{node_exist} || oar_resources_add -a --hosts 1 --host0 #{id['node_num']} --host-prefix #{id['cluster_uid']}- --host-suffix .#{id['site_uid']}.#{id['grid_uid']}.fr --cpus #{node_hash['architecture']['nb_procs']} --cores #{default_properties['cpucore']}"
491
  command += ' | bash'
492
  return command + "\n\n"
493
end
494

495 496
def oarcmd_set_node_properties(host, default_properties)
  return '' if default_properties.size == 0
497
  command  = "echo; echo 'Setting properties for #{host}:'; echo\n"
Florent Didier's avatar
Florent Didier committed
498
  command += "oarnodesetting --sql \"host='#{host}' and type='default'\" -p "
499 500 501 502 503 504
  command += properties_internal(default_properties)
  return command + "\n\n"
end

def properties_internal(properties)
  str = properties.to_a.map do |(k, v)|
505 506
    v = "YES" if v == true
    v = "NO"  if v == false
507 508 509
    !v.nil? ? "#{k}=#{v.inspect.gsub("'", "\\'").gsub("\"", "'")}" : nil
  end.compact.join(' -p ')
  return str
510
end
511

Florent Didier's avatar
Florent Didier committed
512
def oarcmd_create_disk(host, disk)
513 514 515
  disk_exist = "disk_exist '#{host}' '#{disk}'"
  command = "echo; echo 'Adding disk #{disk} on host #{host}:'\n"
  command += "#{disk_exist} && echo '=> disk already exists'\n"
516
  command += "#{disk_exist} || oarnodesetting -a -h '' -p host='#{host}' -p network_address='' -p type='disk' -p disk='#{disk}'"
517
  return command + "\n\n"
518 519
end

Florent Didier's avatar
Florent Didier committed
520
def oarcmd_set_disk_properties(host, disk, disk_properties)
521 522
  return '' if disk_properties.size == 0
  command = "echo; echo 'Setting properties for disk #{disk} on host #{host}:'; echo\n"
523
  command += "oarnodesetting --sql \"host='#{host}' and type='disk' and disk='#{disk}'\" -p "
524 525
  command += properties_internal(disk_properties)
  return command + "\n\n"
526
end
527

528
# sudo exec
529 530 531
def ssh_exec(site_uid, cmds, options)
  # The following is equivalent to : "cat cmds | bash"
  #res = ""
532
  c = Net::SSH.start("oar.#{site_uid}.g5kadmin", "g5kadmin")
533
  c.open_channel { |channel|
534
    channel.exec('sudo bash') { |ch, success|
535
      # stdout
536 537 538
      channel.on_data { |ch, data|
        puts data #if options[:verbose] # ssh cmd output
      }
539 540 541 542 543
      # stderr
      channel.on_extended_data do |ch, type, data|
        puts data
      end

544
      cmds.each { |cmd|
545 546 547 548 549 550 551
        channel.send_data cmd 
      }
      channel.eof!
    }
  }
  c.loop
end
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591

# Get the properties of each node
def get_oar_properties_from_the_ref_repo(global_hash, options)
  properties = {}
  sites = options[:sites]
  sites.each do |site_uid|
    properties[site_uid] = {}
    properties[site_uid]['default'] = get_ref_default_properties(site_uid, global_hash['sites'][site_uid])
    properties[site_uid]['disk'] = get_ref_disk_properties(site_uid, global_hash['sites'][site_uid])
  end
  return properties
end

def get_oar_properties_from_oar(options)
  properties = {}
  sites = options[:sites]
  diff = options[:diff]
  sites.each do |site_uid|
    filename = diff.is_a?(String) ? diff.gsub('%s', site_uid) : nil
    properties[site_uid] = {}
    properties[site_uid]['default'] = get_oar_default_properties(site_uid, filename, options)
    properties[site_uid]['disk'] = get_oar_disk_properties(site_uid, filename, options)
  end
  return properties
end

# Main program
# properties['ref'] = properties from the reference-repo
# properties['oar'] = properties from the OAR server
# properties['diff'] = diff between "ref" and "oar"

def generate_oar_properties(options)
  options[:api] = {}
  conf = RefRepo::Utils.get_api_config
  options[:api][:user] = conf['username']
  options[:api][:pwd] = conf['password']
  options[:ssh] = {}
  options[:ssh][:user] = 'g5kadmin'
  options[:ssh][:host] = 'oar.%s.g5kadmin'
  ret = true
592
  global_hash = load_data_hierarchy
593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807

  properties = {}
  properties['ref'] = get_oar_properties_from_the_ref_repo(global_hash, options)
  properties['oar'] = get_oar_properties_from_oar(options)

  # Get the list of property keys from the reference-repo (['ref'])
  properties_keys = {
    'ref' => {},
    'oar' => {},
    'diff' => {}
  }
  options[:sites].each do |site_uid|
    properties_keys['ref'][site_uid] = get_property_keys(properties['ref'][site_uid])
  end
  ignore_default_keys = ignore_default_keys()

  # Diff
  if options[:diff]
    # Build the list of nodes that are listed in properties['oar'],
    # but does not exist in properties['ref']
    # We distinguish 'Dead' nodes and 'Alive'/'Absent'/etc. nodes
    missings_alive = []
    missings_dead = []
    properties['oar'].each do |site_uid, site_properties|
      site_properties['default'].each_filtered_node_uid(options[:clusters], options[:nodes]) do |node_uid, node_properties_oar|
        unless properties['ref'][site_uid]['default'][node_uid]
          node_properties_oar['state'] != 'Dead' ? missings_alive << node_uid : missings_dead << node_uid
        end
      end
    end

    if missings_alive.size > 0
      puts "*** Error: The following nodes exist in the OAR server but are missing in the reference-repo: #{missings_alive.join(', ')}.\n"
      ret = false unless options[:exec] || options[:output]
    end

    skipped_nodes = []
    prev_diff = {}
    properties['diff'] = {}

    header = false
    properties['ref'].each do |site_uid, site_properties|
      properties['diff'][site_uid] = {}
      site_properties.each do |type, type_properties|
        properties['diff'][site_uid][type] = {}
        type_properties.each_filtered_node_uid(options[:clusters], options[:nodes]) do |key, properties_ref|
          # As an example, key can be equal to 'grimoire-1' for default resources or
          # ['grimoire-1', 1] for disk resources (disk n°1 of grimoire-1)
          node_uid, = key

          if properties_ref['state'] == 'Dead'
            skipped_nodes << node_uid
            next
          end

          properties_oar = properties['oar'][site_uid][type][key]

          diff = diff_properties(type, properties_oar, properties_ref) # Note: this deletes some properties from the input parameters
          diff_keys = diff.map { |hashdiff_array| hashdiff_array[1] }
          properties['diff'][site_uid][type][key] = properties_ref.select { |k, _v| diff_keys.include?(k) }

          # Verbose output
          info = type == 'default' ? ' new node !' : ' new disk !' if properties['oar'][site_uid][type][key].nil?
          case options[:verbose]
          when 1
            puts "#{key}:#{info}" if info != ''
            puts "#{key}:#{diff_keys}" if diff.size != 0
          when 2
            # Give more details
            if header == false
              puts "Output format: ['~', 'key', 'old value', 'new value']"
              header = true
            end
            if diff.empty?
              puts "  #{key}: OK#{info}"
            elsif diff == prev_diff
              puts "  #{key}:#{info} same modifications as above"
            else
              puts "  #{key}:#{info}"
              diff.each { |d| puts "    #{d}" }
            end
            prev_diff = diff
          when 3
            # Even more details
            puts "#{key}:#{info}" if info != ''
            puts JSON.pretty_generate(key => { 'old values' => properties_oar, 'new values' => properties_ref })
          end
          if diff.size != 0
            ret = false unless options[:exec] || options[:output]
          end
        end
      end

      # Get the list of property keys from the OAR scheduler (['oar'])   
      properties_keys['oar'][site_uid] = get_property_keys(properties['oar'][site_uid])

      # Build the list of properties that must be created in the OAR server
      properties_keys['diff'][site_uid] = {}
      properties_keys['ref'][site_uid].each do |k, v_ref|
        v_oar = properties_keys['oar'][site_uid][k]
        properties_keys['diff'][site_uid][k] = v_ref unless v_oar
        if v_oar && v_oar != v_ref && v_ref != NilClass && v_oar != NilClass
          # Detect inconsistency between the type (String/Fixnum) of properties generated by this script and the existing values on the server.
          puts "Error: the OAR property '#{k}' is a '#{v_oar}' on the #{site_uid} server and this script uses '#{v_ref}' for this property."
          ret = false unless options[:exec] || options[:output]
        end
      end

      puts "Properties that need to be created on the #{site_uid} server: #{properties_keys['diff'][site_uid].keys.to_a.delete_if { |e| ignore_default_keys.include?(e) }.join(', ')}" if options[:verbose] && properties_keys['diff'][site_uid].keys.to_a.delete_if { |e| ignore_default_keys.include?(e) }.size > 0

      # Detect unknown properties
      unknown_properties = properties_keys['oar'][site_uid].keys.to_set - properties_keys['ref'][site_uid].keys.to_set
      ignore_default_keys.each do |key|
        unknown_properties.delete(key)
      end

      if options[:verbose] && unknown_properties.size > 0
        puts "Properties existing on the #{site_uid} server but not managed/known by the generator: #{unknown_properties.to_a.join(', ')}."
        puts "Hint: you can delete properties with 'oarproperty -d <property>' or add them to the ignore list in lib/lib-oar-properties.rb."
        ret = false unless options[:exec] || options[:output]
      end
      puts "Skipped retired nodes: #{skipped_nodes}" if skipped_nodes.any?
    end # if options[:diff]
  end

  # Build and execute commands
  if options[:output] || options[:exec]
    skipped_nodes = [] unless options[:diff]
    opt = options[:diff] ? 'diff' : 'ref'

    properties[opt].each do |site_uid, site_properties|
      options[:output].is_a?(String) ? o = File.open(options[:output].gsub('%s', site_uid), 'w') : o = $stdout.dup

      ssh_cmd = []
      cmd = []
      cmd << oarcmd_script_header
      cmd << oarcmd_separator

      # Create properties keys
      properties_keys[opt][site_uid].delete_if { |k, _v| ignore_default_keys.include?(k) }
      unless properties_keys[opt][site_uid].empty?
        cmd << oarcmd_create_properties(properties_keys[opt][site_uid]) + "\n"
        cmd << oarcmd_separator
      end
      cmd << oarcmd_create_node_header
      cmd << oarcmd_separator

      # Build and output node commands
      site_properties['default'].each_filtered_node_uid(options[:clusters], options[:nodes]) do |node_uid, node_properties|
        cluster_uid = node_uid.split('-')[0]
        node_address = [node_uid, site_uid, 'grid5000.fr'].join('.')

        if node_properties['state'] == 'Dead'
          # Do not log node skipping twice if we just did a diff
          skipped_nodes << node_uid unless options[:diff]
          next
        end

        # Create new nodes
        if opt == 'ref' || properties['oar'][site_uid]['default'][node_uid].nil?
          node_hash = global_hash['sites'][site_uid]['clusters'][cluster_uid]['nodes'][node_uid]
          cmd << oarcmd_create_node(node_address, node_properties, node_hash)
        end

        # Update properties
        unless node_properties.empty?
          cmd << oarcmd_set_node_properties(node_address, node_properties)
          cmd << oarcmd_separator
        end
        ssh_cmd += cmd if options[:exec]
        o.write(cmd.join('')) if options[:output]
        cmd = []
      end

      # Build and output disk commands
      site_properties['disk'].each_filtered_node_uid(options[:clusters], options[:nodes]) do |key, disk_properties|
        # As an example, key can be equal to 'grimoire-1' for default resources or
        # ['grimoire-1', 'sdb.grimoire-1'] for disk resources (disk sdb of grimoire-1)
        node_uid, disk = key
        host = [node_uid, site_uid, 'grid5000.fr'].join('.')

        next if skipped_nodes.include?(node_uid)

        # Create a new disk
        if opt == 'ref' || properties['oar'][site_uid]['disk'][key].nil?
          cmd << oarcmd_create_disk(host, disk)
        end

        # Update the disk properties
        unless disk_properties.empty?
          cmd << oarcmd_set_disk_properties(host, disk, disk_properties)
          cmd << oarcmd_separator
        end

        ssh_cmd += cmd if options[:exec]
        o.write(cmd.join('')) if options[:output]
        cmd = []
      end
      o.close

      # Execute commands
      if options[:exec]
        printf 'Apply changes to the OAR server ' + options[:ssh][:host].gsub('%s', site_uid) + ' ? (y/N) '
        prompt = STDIN.gets.chomp
        ssh_exec(site_uid, ssh_cmd, options) if prompt == 'y'
      end
    end # site loop

    if skipped_nodes.any?
      puts "Skipped retired nodes: #{skipped_nodes}" unless options[:diff]
    end
  end # if options[:output] || options[:exec]

  return ret
end