Maj terminée. Pour consulter la release notes associée voici le lien :
https://about.gitlab.com/releases/2021/07/07/critical-security-release-gitlab-14-0-4-released/

Commit 27e7f2bb authored by Jérémie Gaidamour's avatar Jérémie Gaidamour
Browse files

[dev] The oar-properties generator can now add new nodes to the OAR configuration

Also:
* Added support for using a vagrant box and setting SSH options
* The script prevents from adding the same host twice
* Refactoring of the NetSSH code
parent 79fa0f50
......@@ -14,6 +14,7 @@ class MissingProperty < StandardError; end
MiB = 1024**2
# Get node properties from the reference repo hash
# See also: https://www.grid5000.fr/mediawiki/index.php/Reference_Repository
def get_node_properties(cluster_uid, cluster, node_uid, node)
h = {} # ouput
......@@ -117,6 +118,32 @@ def get_nodelist_properties(site_uid, site)
end
def diff_node_properties(a, b)
a ||= {}
b ||= {}
# default OAR at resource creation:
# available_upto: '2147483647'
# besteffort: 'YES'
# core: ~
# cpu: ~
# cpuset: 0
# deploy: 'NO'
# desktop_computing: 'NO'
# drain: 'NO'
# expiry_date: 0
# finaud_decision: 'YES'
# host: ~
# last_available_upto: 0
# last_job_date: 0
# network_address: server
# next_finaud_decision: 'NO'
# next_state: UnChanged
# resource_id: 9
# scheduler_priority: 0
# state: Suspected
# state_num: 3
# suspended_jobs: 'NO'
# type: default
ignore_keys = [
"slash_16",
......@@ -168,17 +195,47 @@ def diff_node_properties(a, b)
end
#def cmd_set_oarnodesetting(properties)
# properties.each
#end
def oarcmd_script_header()
return <<EOF
set -eu
echo '================================================================================'
EOF
end
def oarcmd_create_node_header()
return <<EOF
nodelist=$(oarnodes -l)
list_contains () {
[[ "$1" =~ (^|[[:space:]])"$2"($|[[:space:]]) ]] && return 0 || return 1
}
EOF
end
def oarcmd_create_node(host, properties, node_hash) # host = grifffon-1.nancy.grid5000.fr; properties, node_hash: input of the reference API for the node
node_uid, site_uid, grid_uid = host.split(".")
cluster_uid, node_number = node_uid.split("-")
command = "echo; echo 'Adding host #{host}:'\n"
command += 'list_contains "$nodelist" "' + host + '" && '
command += "echo '=> host already exist'\n"
command += 'list_contains "$nodelist" "' + host + '" || '
command += "sudo oar_resources_add -a --hosts 1 --host0 #{node_number} --host-prefix #{cluster_uid}- --host-suffix .#{site_uid}.#{grid_uid}.fr --cpus #{node_hash['architecture']['smp_size']} --cores #{properties['cpucore']}"
command += ' | sudo bash'
return command + "\n"
end
def oarcmd_set_node_properties(host, properties)
#return "# #{host}: OK" if properties.size == 0
return "" if properties.size == 0
# command = "# #{host}:\n"
# command += "#{ENV["SUDO"]} oarnodesetting -h #{host} -p "
command = "oarnodesetting -h #{host} -p "
command = "echo; echo 'Setting properties for #{host}:'; echo\n"
command += "sudo oarnodesetting -h #{host} -p "
command +=
properties.to_a.map{ |(k,v)|
......@@ -188,32 +245,32 @@ def oarcmd_set_node_properties(host, properties)
! v.nil? ? "#{k}=#{v.inspect.gsub("'", "\\'").gsub("\"", "'")}" : nil
}.compact.join(' -p ')
return command
return command + "\n"
end
# '
# Get the OAR properties from the OAR scheduler
# This is only needed for the -d option
def oarcmd_get_nodelist_properties(site_uid, filename=nil, sshkeys=[])
def oarcmd_get_nodelist_properties(site_uid, filename=nil, options)
oarnodes_yaml = ""
if filename and File.exist?(filename)
# Read oar properties from file
puts "Read 'oarnodes -Y' from #{filename}"
puts "Read 'oarnodes -Y' from #{filename}" if options[:verbose]
oarnodes_yaml = File.open(filename, 'rb') { |f| f.read }
else
# Download the oar properties from the oar server
puts "Downloading 'oarnodes -Y' from oar.#{site_uid}.g5kadmin ..."
puts "Downloading 'oarnodes -Y' from " + options[:ssh][:host].gsub("%s", site_uid) + "..." if options[:verbose]
Net::SSH.start("oar.#{site_uid}.g5kadmin", 'g5kadmin', :keys => sshkeys) { |ssh|
Net::SSH.start(options[:ssh][:host].gsub("%s", site_uid), options[:ssh][:user], options[:ssh][:params]) { |ssh|
# capture all stderr and stdout output from a remote process
oarnodes_yaml = ssh.exec!('oarnodes -Y')
}
puts "... done"
puts "... done" if options[:verbose]
if filename
# Cache the file
puts "Save 'oarnodes -Y' as #{filename}"
puts "Save 'oarnodes -Y' as #{filename}" if options[:verbose]
File.write(filename, oarnodes_yaml)
end
end
......@@ -227,3 +284,22 @@ def oarcmd_get_nodelist_properties(site_uid, filename=nil, sshkeys=[])
return h
end
def ssh_exec(site_uid, cmds, options)
# The following is equivalent to : "cat cmds | bash"
#res = ""
c = Net::SSH.start(options[:ssh][:host].gsub("%s", site_uid), options[:ssh][:user], options[:ssh][:params])
c.open_channel { |channel|
channel.exec('bash') { |ch, success|
channel.on_data { |ch, data|
puts data #if options[:verbose] # ssh cmd output
}
cmds.each { |cmd|
channel.send_data cmd
}
channel.eof!
}
}
c.loop
end
......@@ -18,8 +18,6 @@ require '../lib/input_loader'
options = {}
options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes reims rennes sophia}
options[:diff] = false
options[:sshkeys] = []
OptionParser.new do |opts|
opts.banner = "Usage: oar-properties.rb [options]"
......@@ -60,8 +58,21 @@ OptionParser.new do |opts|
options[:exec] = e
end
opts.on('-k', '--ssh-keys k1,k2,k3', Array, 'SSH keys') do |k|
options[:sshkeys] = k
opts.on('--ssh-keys k1,k2,k3', Array, 'SSH keys') do |k|
options[:ssh] ||= {}
options[:ssh][:params] ||= {}
options[:ssh][:params][:keys] ||= []
options[:ssh][:params][:keys] << k
end
opts.on('--vagrant', 'This option modifies the SSH parameters to use a vagrant box instead of Grid5000 servers.') do |v|
options[:ssh] ||= {}
options[:ssh][:host] = '127.0.0.1' unless options[:ssh][:host]
options[:ssh][:user] = 'vagrant' unless options[:ssh][:user]
options[:ssh][:params] ||= {}
options[:ssh][:params][:keys] ||= []
options[:ssh][:params][:keys] << '~/.vagrant.d/insecure_private_key'
options[:ssh][:params][:port] = 2222 unless options[:ssh][:params][:port]
end
opts.on("-d", "--diff [YAML filename]",
......@@ -91,6 +102,11 @@ OptionParser.new do |opts|
end
end.parse!
options[:ssh] ||= {}
options[:ssh][:host] = 'oar.%s.g5kadmin' unless options[:ssh][:host]
options[:ssh][:user] = 'g5kadmin' unless options[:ssh][:user]
options[:diff] = false unless options[:diff]
puts "Options: #{options}" if options[:verbose]
nodelist_properties = {} # ["ref"] : properties from the reference-repo
......@@ -117,15 +133,15 @@ if options[:diff]
options[:sites].each { |site_uid|
nodelist_properties["oar"][site_uid] = {}
filename = options[:diff].is_a?(String) ? options[:diff].gsub("%s", site_uid) : nil
nodelist_properties["oar"][site_uid] = oarcmd_get_nodelist_properties(site_uid, filename, options[:sshkeys])
nodelist_properties["oar"][site_uid] = oarcmd_get_nodelist_properties(site_uid, filename, options)
}
end
#
# Diff
# Diff (-d option)
#
if options[:diff]
header ||= false
header = false
prev_diff = {}
nodelist_properties["diff"] = {}
......@@ -142,27 +158,30 @@ if options[:diff]
nodelist_properties["diff"][site_uid][node_uid] = node_properties_ref.select { |key, value| diff_keys.include?(key) }
info = (nodelist_properties["oar"][site_uid][node_uid] == nil ? " new node !" : "")
case options[:verbose]
when 1
puts "#{node_uid}:#{info}" if info != ""
puts "#{node_uid}: #{diff_keys}"
when 2
# Give more details
# puts "#{node_uid}: #{diff}"
if !header
header=true
header = true
puts "Output format: ['~', 'key', 'old value', 'new value']"
end
if diff.size==0
puts " #{node_uid}: OK"
puts " #{node_uid}: OK#{info}"
elsif diff == prev_diff
puts " #{node_uid}: same as above"
puts " #{node_uid}:#{info} same modifications as above"
else
puts " #{node_uid}:"
puts " #{node_uid}:#{info}"
diff.each { |d| puts " #{d}" }
end
prev_diff = diff
when 3
# Even more details
puts "#{node_uid}:#{info}" if info != ""
puts JSON.pretty_generate({node_uid => {"old values" => node_properties_oar, "new values" => node_properties_ref}})
end
}
......@@ -171,45 +190,55 @@ if options[:diff]
end # if options[:diff]
#
# Output commands
# Build and execute commands
#
if options[:output]
if options[:output] || options[:exec]
opt = options[:diff] ? 'diff' : 'ref'
nodelist_properties[opt].each { |site_uid, site_properties|
# Init
options[:output].is_a?(String) ? o = File.open(options[:output].gsub("%s", site_uid),'w') : o = $stdout.dup
ssh_cmd = []
create_node_header = false
cmd = []
cmd << oarcmd_script_header()
#
# Build and output commands
#
site_properties.each_filtered_node_uid(options[:clusters], options[:nodes]) { |node_uid, node_properties|
o.write(oarcmd_set_node_properties(node_uid + "." + site_uid + ".grid5000.fr", node_properties) + "\n")
}
# Create new nodes
if (opt == 'ref' || nodelist_properties['oar'][site_uid][node_uid] == nil)
if !create_node_header
create_node_header = true
cmd << oarcmd_create_node_header()
end
o.close
cluster_uid = node_uid.split('-')[0]
node_hash = global_hash['sites'][site_uid]['clusters'][cluster_uid]['nodes'][node_uid]
cmd << oarcmd_create_node(node_uid + '.' + site_uid + '.grid5000.fr', node_properties, node_hash) + "\n"
end
}
end
# Update properties
cmd << oarcmd_set_node_properties(node_uid + '.' + site_uid + '.grid5000.fr', node_properties) + "\n"
#
# Execute commands
#
if options[:exec]
printf "Apply changes to the OAR servers ? (y/n)"
prompt = STDIN.gets.chomp
exit unless prompt == 'y'
opt = options[:diff] ? 'diff' : 'ref'
nodelist_properties[opt].each { |site_uid, site_properties|
cmd << "echo '================================================================================'\n\n"
ssh_cmd += cmd if options[:exec]
o.write(cmd.join('')) if options[:output]
cmd = []
}
puts "Connecting #{site_uid} ..."
Net::SSH.start("oar.#{site_uid}.g5kadmin", 'g5kadmin', :keys => options[:sshkeys]) { |ssh|
o.close
site_properties.each_filtered_node_uid(options[:clusters], options[:nodes]) { |node_uid, node_properties|
cmd = oarcmd_set_node_properties(node_uid + "." + site_uid + ".grid5000.fr", node_properties)
if cmd.size>0
puts "#{cmd}" if options[:verbose]
ssh_output = ssh.exec!('sudo ' + cmd)
puts "#{ssh_output}\n" if options[:verbose]
#
# Execute commands
#
if options[:exec]
printf "Apply changes to the OAR server " + options[:ssh][:host].gsub("%s", site_uid) + " ? (y/N) "
prompt = STDIN.gets.chomp
ssh_exec(site_uid, ssh_cmd, options) if prompt == 'y'
end
}
}
}
end
} # site loop
end # if options[:output] || options[:exec]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment