Rakefile 9.97 KB
Newer Older
1
#
2
3
4
require 'fileutils'
require 'json'
require 'logger'
5
require 'restfully'
6
7
8
9
10
11
12
13
14
15
16
17

ROOT_DIR = File.expand_path File.dirname(__FILE__)
LIB_DIR = File.join(ROOT_DIR, "generators", "lib")
$LOAD_PATH.unshift(LIB_DIR) unless $LOAD_PATH.include?(LIB_DIR)

require 'grid5000'

task :environment do
  Dir.chdir(ROOT_DIR)
  @logger = Logger.new(STDERR)
  @logger.level = Logger.const_get((ENV['DEBUG'] || 'INFO').upcase)
end
18
19
20
21

task :api_sites  do
  api_logger = Logger.new("/dev/null")
  api_logger.level = Logger::FATAL
22
  @api = Restfully::Session.new(:configuration_file => File.expand_path("~/.restfully/api.grid5000.fr.yml"),:logger => api_logger,:base_uri => 'https://api.grid5000.fr/sid')
23
24
25
26
27
28
29
  @api_sites = if ENV['SITE']
    [@api.root.sites[ENV['SITE'].to_sym]]
  else
    @api.root.sites
  end
end

30
task :hosts do
31
  # HOSTS=gw.lille
32
33
34
35
36
  # HOSTS=*.lille
  # SITES=lille => HOSTS=*.lille
  # SITES=* => HOSTS=*.*
  site = ENV['SITE']
  host = ENV['HOST']
37
  if site != nil
38
39
40
41
42
43
44
45
46
    host = "*.#{site}"
  elsif host != nil
    abort "HOST must be on the form <hostname>.<site>. You provided '#{host}'." if host.scan(/^(\S+)\.(\S+)$/).empty?
  else
    abort "You must provide SITE= , (SITE=lille, or SITE=*), or a HOST, (HOST=gw.lille, or HOST=*.lille, or HOST=sw-*.lille)"
  end
  abort "You must provide HOST= , (HOST=gw.lille, or HOST=*.lille, or HOST=sw-*.lille)" if host.nil?
  @host = host
end
47

48
49

namespace :g5k do
50
  desc "Generates the JSON files based on the generators, for all sites.\nUse SITE=<SITE-NAME> if you wish to restrict the generation to a specific site.\nUse DRY=yes to simulate the execution."
51
  task :generate => [:environment,:hosts] do
52
    host,site = @host.scan(/(\S+)\.(\S+)/).flatten
53
54
55
56
57
58
59
60
61
62
63
    root_dir_input = "#{ROOT_DIR}/generators/input/sites"
    command = File.join(ROOT_DIR, "generators", "grid5000")
    command += " " + File.join(root_dir_input, site,"#{site}.rb")
    command += " " + File.join(root_dir_input, site,"clusters","#{host}.rb")
    command += " " + File.join(root_dir_input, site,"clusters","#{host}.yaml")

    command << " -s" if ENV['DRY'] == "yes"
#    puts command
    sh command
  end
end
64

Pascal Morillon's avatar
Pascal Morillon committed
65
66
67
68
69
70
71
72
73
74
75
76
namespace :pdus do
  desc "Generates the JSON files for PDUs informations"
  task :generate do
    raise "Need SITE=" unless site = ENV['SITE']
    root_dir_input = "#{ROOT_DIR}/generators/input/sites"
    command = File.join(ROOT_DIR, "generators", "grid5000")
    command += " " + File.join(root_dir_input, site,"pdus.rb")
    command << " -s" if ENV['DRY'] == "yes"
    sh command
  end
end

77
78
79
# rake deadnodes:reasons
# rake deadnodes:tofix
namespace :deadnodes do
80

81
82
83
84
85
  desc "List all dead nodes and the reason why they are dead. (SITE=)"
  task :reasons => [:environment,:api_sites] do
    @logger.level = Logger::INFO
    @reasons = true
    Rake::Task["deadnodes:browse"].execute
Gaetan SIMO's avatar
Gaetan SIMO committed
86
  end
87
88
89
90
91
  desc "List all nodes which have they state not in synch with they comment. (SITE=)"
  task :tofix => [:environment,:api_sites] do
    @logger.level = Logger::ERROR
    @tofix = true
    Rake::Task["deadnodes:browse"].execute
Gaetan SIMO's avatar
Gaetan SIMO committed
92
93
  end

94
95
  task :browse do
    def comment_ok?(comment)
96
      comment == "OK"
97
    end
98
    phoenix = []
99
    @api_sites.each do |site|
100
      reg = /^([^-]+)-(\d+)/
101
      site.status["nodes"].sort{|a,b|
102
103
104
105
        a_cluster,a_id = a[0].scan(reg).flatten
        b_cluster,b_id = b[0].scan(reg).flatten
        [a_cluster,a_id.to_i] <=> [b_cluster,b_id.to_i]
      }.each do |uid,status|
106
107
108
109
110
        comment = status["comment"]
        state = status["hard"].downcase
        if comment_ok?(comment)
          if state == "dead"
            @logger.error "Node '#{uid}' of state '#{state}' should not have comment '#{comment}'" if @tofix
Gaetan SIMO's avatar
Gaetan SIMO committed
111
          else
112
            # nothing, good state
Gaetan SIMO's avatar
Gaetan SIMO committed
113
          end
114
115
116
117
118
        else
          if state == "dead"
            @logger.info "Node '#{uid}' is dead because '#{comment}'" if @reasons
          else
            @logger.error "Node '#{uid}' should have the not-dead-comment 'OK', since its state is '#{state}'. Instead, it has comment '#{comment}'." if @tofix
119
            phoenix.push uid if comment.match(/^\[phoenix\]/) != nil
120
121
122
123
          end
        end
      end
    end
124
    puts phoenix if ENV["PHOENIX"] == "yes"
125
126
127
128
  end
end

# TESTS
129
# Deletion:
130
131
#   rake -s oar:generate FROM=4cfebf92e9cce05315782b51e05eded4ab4f0e7e TO=7d2648eaad7dbbc6f1fdb9c0279f73d374ccd47a
#
132
# Update:
133
134
# rake -s oar:generate FROM=7d2648eaad7dbbc6f1fdb9c0279f73d374ccd47a TO=bb528643003757942521942eaeab74b15aaa976d
#
135
# Add:
136
137
#   rake -s oar:generate FROM=be9f7338b9750ce675447c13d172157992041ec1 TO=7dc3a4101a657230b7ad0534025a7ca93c905411
#
138
# All:
139
#   rake -s oar:generate FROM=be9f7338b9750ce675447c13d172157992041ec1 TO=7d2648eaad7dbbc6f1fdb9c0279f73d374ccd47a
140
#
141
142
143
144
namespace :oar do
  desc "Generates the oaradmin lines to update the OAR database after a change in the reference repository.\nUse FROM=<SHA-ID> and TO=<SHA-ID> to specify the starting and ending commits.\nUse -s to suppress the 'in directory' announcement."
  task :generate => :environment do
    if ENV['FROM'].nil? || ENV['FROM'].empty?
145
      @logger.fatal "You MUST specify a commit id from where to start using the FROM=<SHA-ID> argument. Ex: rake -s oar:generate FROM=be9f7338b9750ce675447c13d172157992041ec1 TO=7dc3a4101a657230b7ad0534025a7ca93c905411 2> /dev/null"
146
147
148
149
      exit(1)
    end
    ENV['TO'] ||= 'HEAD'
    @logger.info "Analysing changes between #{ENV['FROM']}..#{ENV['TO']}..."
150

151
152
    commands = []
    diff = `git diff --name-status #{ENV['FROM']}..#{ENV['TO']}`
153

154
155
156
    diff.split("\n").each do |line|
      action, filename = line.split("\t")
      next unless filename =~ %r{.+/nodes/.+}
157

158
159
160
      node_uid, site_uid, grid_uid = filename.gsub(/\.json/,'').split("/").values_at(-1, -5, -7)
      cluster_uid = node_uid.split("-")[0]
      host = [node_uid, site_uid, grid_uid, "fr"].flatten.join(".")
161

162
163
164
165
      if ENV['SITE'] && !ENV['SITE'].split(",").include?(site_uid)
        @logger.info "Skipping #{host} since you only want changes that occured on #{ENV['SITE'].inspect}"
        next
      end
166

167
      command = "oaradmin resources"
168

169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
      case action
      when "A", "C", "M"
        node_properties = JSON.parse(File.read(filename))
        cluster_properties = JSON.parse(File.read(filename.gsub(%r{/nodes.*}, "/#{cluster_uid}.json")))
        cluster = Grid5000::Cluster.new(cluster_properties)
        node = Grid5000::Node.new(cluster, node_properties)
        begin
          export = node.export("oar-2.4")
        rescue Grid5000::MissingProperty => e
          @logger.warn "Error when exporting #{host}: #{e.message}. Skipped."
          next
        end
        if action == "M"  # modification of a file
          command.concat(" -s node=#{host} ")
        else              # new file
184
185
186
187
188
189
190
191
          command.concat(" -a /node=#{host}/cpu={#{node.properties['architecture']['smp_size']}}/core={#{export['cpucore']}}")
          command.concat(" --auto-offset")
          if ENV['MAINTENANCE'] && ENV['MAINTENANCE']=='NO'
            command.concat(' -p maintenance="NO"')
          else
            # by default, maintenance is YES when creating new resources
            command.concat(' -p maintenance="YES"')
          end
192
193
          # by default, an Alive node has comment "OK"
          command.concat(' -p comment="OK"')
194
195
196
197
198
199
200
        end
        command.concat(" -p ").concat( export.to_a.map{|(k,v)|
          if v.nil?
            nil
          else
            [k, v.inspect].join("=")
          end
201
        }.compact.join(" -p ") )
202
203
      when "D"            # deletion of a file
        command.concat(" -d node=#{host}")
204
      else
205
206
207
        @logger.warn "Don't know what to do with #{line.inspect}. Ignoring."
        next
      end
208

209
210
211
      if ENV['COMMIT'] && ENV['COMMIT']=='YES'
        command.concat(' -c')
      end
212

213
214
215
216
217
218
219
      commands << command
    end
    commands.each do |command|
      puts command
    end
  end
end
220
221
222
223
224

namespace :netlinks do
  desc "Generates network API JSON files based on net-links yaml files.\nUse DRY=yes to simulate the execution. "
  task :generate => [:environment,:hosts] do
    host,site = @host.scan(/(\S+)\.(\S+)/).flatten
225
    root_dir_input = File.join(ROOT_DIR, "generators","input")
226
227
    command = File.join(ROOT_DIR, "generators", "grid5000")
    command += " " + File.join(root_dir_input, "net-links.rb")
228
229
    command += " " + File.join(root_dir_input,"sites", site,"#{site}.rb")
    command += " " + File.join(root_dir_input,"sites", site,"net-links","#{host}.yaml")
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248

    command << " -s" if ENV['DRY'] == "yes"
#    puts command
    sh command
  end
end
namespace :env do
  desc "Generates environment JSON files .\nUse DRY=yes to simulate the execution. "
  task :generate => [:environment] do
    env_name = ENV["ENV_NAME"]
    abort "You must provide ENV_NAME=" if env_name.nil?
    root_dir_input = "#{ROOT_DIR}/generators/input"
    command = File.join(ROOT_DIR, "generators", "grid5000")
    command += " " + File.join(root_dir_input, "environments","#{env_name}")

    command << " -s" if ENV['DRY'] == "yes"
    sh command
  end
end
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
namespace :weathermap do
  @weathermap_options = ""
  task :hosts do
    @weathermap_site = ENV['SITE']
    @weathermap_host = ENV['HOST']
    abort "You must provide the SITE= " if @weathermap_site.nil?
    abort "You must provide the HOST= name (uid) in its site " if @weathermap_host.nil? or @weathermap_host.match(/\.grid5000\.fr/) != nil
  end
  task :execute => ["weathermap:hosts"]  do
    cmd = "bundle exec weathermap"
    cmd += " --site '#{@weathermap_site}' --host '#{@weathermap_host}' --api-path #{ROOT_DIR} #{@weathermap_options}"
    sh cmd
  end
  desc "Create weathermaps for host HOST without data."
  task :testing => ["weathermap:hosts"]  do
    @weathermap_options.replace("--action write --use-cacti no")
    Rake::Task['weathermap:execute'].invoke
  end
  desc "Create weathermaps for host HOST with RRD from cacti."
  task :production => ["weathermap:hosts"]  do
    @weathermap_options.replace("--action write --use-cacti yes")
    Rake::Task['weathermap:execute'].invoke
  end
  desc "Display network links description amongst network equipments."
  task :display do
    @weathermap_options.replace("--action display")
    Rake::Task['weathermap:execute'].invoke
276
277
  end
end
278