oar-properties.rb 7.69 KB
Newer Older
1 2
#!/usr/bin/ruby

3 4
# Generator for the OAR properties

5 6 7 8 9 10
require 'pp'
require 'erb'
require 'fileutils'
require 'pathname'
require 'json'
require 'time'
11 12 13 14
require 'yaml'
require 'hashdiff'
require 'optparse'
require 'net/ssh'
15

16
require '../oar-properties/lib/lib-oar-properties'
17 18
require '../lib/input_loader'

19
options = {}
Jérémie Gaidamour's avatar
Jérémie Gaidamour committed
20
options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes reims rennes sophia}
21

22 23
OptionParser.new do |opts|
  opts.banner = "Usage: oar-properties.rb [options]"
24

25 26 27 28 29 30 31
  opts.separator ""
  opts.separator "Example: ruby oar-properties.rb -v -s nancy -d oarnodes-%s.yaml -o cmd-%s.sh"

  opts.separator ""
  opts.separator "Filters:"

  ###
32

33 34
  opts.on('-s', '--sites a,b,c', Array, 'Select site(s)',
                                        "Default: "+options[:sites].join(", ")) do |s|
Jérémie Gaidamour's avatar
Jérémie Gaidamour committed
35
    raise "Wrong argument for -s option." unless (s - options[:sites]).empty?
36
    options[:sites] = s
37
  end
38 39 40

  opts.on('-c', '--clusters a,b,c', Array, 'Select clusters(s). Default: all') do |s|
    options[:clusters] = s
41 42
  end

43 44
  opts.on('-n', '--nodes a,b,c', Array, 'Select nodes(s). Default: all') do |n|
    options[:nodes] = n
45 46
  end

47
  ###
48

49 50
  opts.separator ""
  opts.separator "Output options:"
51

52 53
  opts.on('-o', '--output [FILE]', 'Output oarnodesetting commands to a file. Default FILE is stdout.') do |o|
    o = true if o == nil
54 55
    options[:output] = o
  end
56

57 58 59
  opts.on('-e', '--exec', 'Directly apply the changes to the OAR server') do |e|
    options[:exec] = e
  end
60

61 62 63 64 65
  opts.on('--ssh-keys k1,k2,k3', Array, 'SSH keys') do |k|
    options[:ssh] ||= {}
    options[:ssh][:params] ||= {}
    options[:ssh][:params][:keys] ||= []
    options[:ssh][:params][:keys] << k
66
  end
67
  
68 69 70 71 72 73 74 75 76 77
  opts.on('--vagrant', 'This option modifies the SSH parameters to use a vagrant box instead of Grid5000 servers.') do |v|
    options[:ssh] ||= {}
    options[:ssh][:host] = '127.0.0.1' unless options[:ssh][:host]
    options[:ssh][:user] = 'vagrant'   unless options[:ssh][:user]
    options[:ssh][:params] ||= {}
    options[:ssh][:params][:keys] ||= []
    options[:ssh][:params][:keys] << '~/.vagrant.d/insecure_private_key'
    options[:ssh][:params][:port] = 2222 unless options[:ssh][:params][:port]
  end

78 79 80 81 82 83 84 85 86
  opts.on("-d", "--diff [YAML filename]", 
          "Only generates the minimal list of commands needed to update the site configuration",
          "The optional YAML file is suppose to be the output of the 'oarnodes -Y' command.",
          "If the file does not exist, the script will get the data from the OAR server and save the result on disk for future use.",
          "If no filename is specified, the script will simply connect to the OAR server.",
          "You can use the '%s' placeholder for 'site'. Ex: oarnodes-%s.yaml") do |d|
    d = true if d == nil
    options[:diff] = d
  end
87
  
88 89 90 91 92
  ###

  opts.separator ""
  opts.separator "Common options:"

Jérémie Gaidamour's avatar
Jérémie Gaidamour committed
93 94 95
  opts.on("-v", "--[no-]verbose", "Run verbosely", "Multiple -v options increase the verbosity. The maximum is 3.") do |v|
    options[:verbose] ||= 0
    options[:verbose] = options[:verbose] + 1
96
  end
97 98 99 100 101 102 103 104
  
  # Print an options summary.
  opts.on_tail("-h", "--help", "Show this message") do
    puts opts
    exit
  end
end.parse!

105 106 107 108 109
options[:ssh] ||= {}
options[:ssh][:host] = 'oar.%s.g5kadmin' unless options[:ssh][:host]
options[:ssh][:user] = 'g5kadmin'        unless options[:ssh][:user]
options[:diff] = false unless options[:diff]

Jérémie Gaidamour's avatar
Jérémie Gaidamour committed
110
puts "Options: #{options}" if options[:verbose]
111

112 113 114
nodelist_properties = {} # ["ref"]  : properties from the reference-repo
                         # ["oar"]  : properties from the OAR server
                         # ["diff"] : diff between "ref" and "oar"
115 116

#
117
# Get the OAR properties from the reference-repo (["ref"])
118 119
#

120
nodelist_properties["ref"] = {}
121
global_hash = load_yaml_file_hierarchy('../../input/grid5000/')
122 123 124
options[:sites].each { |site_uid| 
  nodelist_properties["ref"][site_uid] = get_nodelist_properties(site_uid, global_hash["sites"][site_uid]) 
}
125 126

#
127
# Get the current OAR properties from the OAR scheduler (["oar"])
128 129
#

130 131 132 133 134
# This is only needed for the -d option  
if options[:diff]
  nodelist_properties["oar"] = {}
  options[:sites].each { |site_uid| 
    nodelist_properties["oar"][site_uid] = {}
135
    filename = options[:diff].is_a?(String) ? options[:diff].gsub("%s", site_uid) : nil
136
    nodelist_properties["oar"][site_uid] = oarcmd_get_nodelist_properties(site_uid, filename, options)
137 138
  }
end
139 140

#
141
# Diff (-d option)
142
#
143
if options[:diff]
144
  header = false
145
  prev_diff = {}
146
  
147 148 149 150 151 152
  nodelist_properties["diff"] = {}
  
  nodelist_properties["ref"].each { |site_uid, site_properties| 
    nodelist_properties["diff"][site_uid] = {}

    site_properties.each_filtered_node_uid(options[:clusters], options[:nodes]) { |node_uid, node_properties_ref|
153
      
154
      node_properties_oar = nodelist_properties["oar"][site_uid][node_uid]
155

156 157 158 159
      diff      = diff_node_properties(node_properties_oar, node_properties_ref)
      diff_keys = diff.map{ |hashdiff_array| hashdiff_array[1] }
      
      nodelist_properties["diff"][site_uid][node_uid] = node_properties_ref.select { |key, value| diff_keys.include?(key) }
160 161

      info = (nodelist_properties["oar"][site_uid][node_uid] == nil ? " new node !" : "")      
162 163
      case options[:verbose]
      when 1
164
        puts "#{node_uid}:#{info}" if info != ""
165 166 167 168 169
        puts "#{node_uid}: #{diff_keys}"
      when 2
        # Give more details
        # puts "#{node_uid}: #{diff}"
        if !header
170
          header = true
171 172 173
          puts "Output format: ['~', 'key', 'old value', 'new value']"
        end
        if diff.size==0
174
          puts "  #{node_uid}: OK#{info}"
175
        elsif diff == prev_diff
176
          puts "  #{node_uid}:#{info} same modifications as above"
177
        else
178
          puts "  #{node_uid}:#{info}"
179 180 181 182 183
          diff.each { |d| puts "    #{d}" } 
        end
        prev_diff = diff
      when 3
        # Even more details
184
        puts "#{node_uid}:#{info}" if info != ""
185
        puts JSON.pretty_generate({node_uid => {"old values" => node_properties_oar, "new values" => node_properties_ref}})
Jérémie Gaidamour's avatar
Jérémie Gaidamour committed
186
      end
187 188
    }
    
Jérémie Gaidamour's avatar
Jérémie Gaidamour committed
189
  }
190
end # if options[:diff]
191 192

#
193
# Build and execute commands
194
#
195
if options[:output] || options[:exec]
196 197 198
  opt = options[:diff] ? 'diff' : 'ref'
  nodelist_properties[opt].each { |site_uid, site_properties| 
    
199
    # Init
200
    options[:output].is_a?(String) ? o = File.open(options[:output].gsub("%s", site_uid),'w') : o = $stdout.dup
201
    ssh_cmd = []
202

203 204 205 206 207 208 209
    create_node_header = false
    cmd = []
    cmd << oarcmd_script_header()

    #
    # Build and output commands
    #
210
    site_properties.each_filtered_node_uid(options[:clusters], options[:nodes]) { |node_uid, node_properties|
211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
      # Create new nodes
      if (opt == 'ref' || nodelist_properties['oar'][site_uid][node_uid] == nil)
        if !create_node_header
          create_node_header = true
          cmd << oarcmd_create_node_header()
        end

        cluster_uid = node_uid.split('-')[0]
        node_hash = global_hash['sites'][site_uid]['clusters'][cluster_uid]['nodes'][node_uid]
        cmd << oarcmd_create_node(node_uid + '.' + site_uid + '.grid5000.fr', node_properties, node_hash) + "\n"
      end

      # Update properties
      cmd << oarcmd_set_node_properties(node_uid + '.' + site_uid + '.grid5000.fr', node_properties) + "\n"

      cmd << "echo '================================================================================'\n\n"
      ssh_cmd += cmd        if options[:exec]
      o.write(cmd.join('')) if options[:output]
      cmd = []
230 231 232 233
    }
    
    o.close
    
234 235 236 237 238 239 240 241 242 243
    #
    # Execute commands
    #
    if options[:exec]
      printf "Apply changes to the OAR server " + options[:ssh][:host].gsub("%s", site_uid) + " ? (y/N) "
      prompt = STDIN.gets.chomp
      ssh_exec(site_uid, ssh_cmd, options) if prompt == 'y'
    end
  } # site loop
end # if options[:output] || options[:exec]
Jérémie Gaidamour's avatar
Jérémie Gaidamour committed
244