check-cluster-homogeneity.rb 7.45 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
#!/usr/bin/ruby

# This script checks the cluster homogeneity

if RUBY_VERSION < "2.1"
  puts "This script requires ruby >= 2.1"
  exit
end

require 'pp'
require 'fileutils'
require 'pathname'
require 'hashdiff'

15
require_relative "../lib/input_loader"
16

17 18 19 20 21
def global_ignore_keys()

  #
  # Global ignore keys
  #
22 23

  ignore_keys = %w(
24
    ~chassis.serial
25
  
26 27 28 29 30
    ~network_adapters.bmc.ip
    ~network_adapters.bmc.mac
    ~network_adapters.bmc.network_address
    ~network_adapters.bmc.switch
    ~network_adapters.bmc.switch_port
31
  
32 33 34 35
    ~network_adapters.myri0.ip
    ~network_adapters.myri0.ip6
    ~network_adapters.myri0.mac
    ~network_adapters.myri0.network_address
36
  
37 38 39 40 41
    ~pdu
    ~pdu.port
    ~pdu.uid
    ~pdu[0]
    ~pdu[1]
42

43
    ~supported_job_types.max_walltime
44

45 46
    ~mic.ip
    ~mic.mac
47

48 49
    +status
    -status
50 51 52
  )

  ignore_netkeys = <<-eos
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
    ~network_adapters.eth.ip
    ~network_adapters.eth.ip6
    ~network_adapters.eth.mac
    ~network_adapters.eth.network_address
    ~network_adapters.eth.switch
    ~network_adapters.eth.switch_port
    ~network_adapters.eth.ip
    ~network_adapters.eth.ip6
    ~network_adapters.eth.mac
    ~network_adapters.eth.switch_port
    ~network_adapters.eth.ip
    ~network_adapters.eth.ip6
    ~network_adapters.eth.mac
    ~network_adapters.eth.switch_port
    ~network_adapters.eth.ip
    ~network_adapters.eth.mac
    ~network_adapters.eth.mac
    ~network_adapters.eth.mac
71 72 73
eos

  ignore_stokeys = <<-eos
74 75 76 77 78 79
    ~storage_devices.sd.model
    ~storage_devices.sd.rev
    ~storage_devices.sd.size
    ~storage_devices.sd.timeread
    ~storage_devices.sd.timewrite
    ~storage_devices.sd.vendor
80 81
    ~storage_devices.sd.by_id
    ~storage_devices.sd.by_path
82 83 84 85 86 87 88
eos

  (0..5).each { |eth| 
    keys = ignore_netkeys.gsub('.eth.', ".eth#{eth}.").gsub("\n", " ").split(" ")
    ignore_keys.push(* keys)

    (1..21).each { |kavlan|
89
      ignore_keys << "~kavlan.eth#{eth}.kavlan-#{kavlan}"
90 91 92
    }
  }

93
  ('a'..'f').each { |sd| 
94 95 96 97
    keys = ignore_stokeys.gsub('.sd.', ".sd#{sd}.").gsub("\n", " ").split(" ")
    ignore_keys.push(* keys)
  }

98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
  ignore_ibkeys = <<-eos
    ~network_adapters.IB_IF.guid
    ~network_adapters.IB_IF.hwid
    ~network_adapters.IB_IF.ip
    ~network_adapters.IB_IF.ip6
    ~network_adapters.IB_IF.line_card
    ~network_adapters.IB_IF.position
eos

  ib_interfaces = [
    'ib0',
    'ib1',
    'ib0.8100'
  ]

  ib_interfaces.each { |ib_if|
    keys = ignore_ibkeys.gsub('IB_IF', "#{ib_if}").gsub("\n", " ").split(" ")
    ignore_keys.push(* keys)
  }

118 119 120
  return ignore_keys
end

121 122 123 124 125 126
def cluster_ignore_keys(filename)
  file_hash = YAML::load(ERB.new(File.read(filename)).result(binding))
  file_hash.expand_square_brackets() if file_hash
  return file_hash
end

127 128 129
def cluster_homogeneity(refapi_hash, options = {:verbose => false})
  verbose = options[:verbose]

130 131 132 133 134 135 136 137
  if verbose
    puts "The change set is represented using the following syntax:"
    puts '  [["+", "path.to.key1", value],          # new key'
    puts '   ["-", "path.to.key2", value],          # missing key'
    puts '   ["~", "path.to.key3", value1, value2]] # modified value'
    puts ''
  end

138
  ignore_keys  = global_ignore_keys()
139
  cignore_keys = cluster_ignore_keys(File.expand_path("../input-validators/check-cluster-homogeneity.yaml.erb", File.dirname(__FILE__)))
140

141 142
  input_data_dir = "../../input/grid5000/"
  refapi_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
143
  count = {}
144 145
  total_count = 0

146
  refapi_hash["sites"].sort.each do |site_uid, site|
147 148
    next if options.key?(:sites) && !options[:sites].include?(site_uid)

149 150 151
    count[site_uid] = {}

    site["clusters"].sort.each do |cluster_uid, cluster|
152 153
      next if options.key?(:clusters) && !options[:clusters].include?(cluster_uid)

154 155
      count[site_uid][cluster_uid] = 0

156 157
      refnode_uid = nil
      refnode = nil
158

159
      cluster["nodes"].each_sort_by_node_uid do |node_uid, node|
160

161
        next if node['status'] == 'retired'
162

163 164 165 166 167 168
        if !refnode
          refnode = node
          refnode_uid = node_uid
          next
        end

169
        diffs = HashDiff.diff(refnode, node)
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192

        # Hack HashDiff output for arrays:
        #[["-", "pdu[1]", {"uid"=>"graphene-pdu9", "port"=>24}],
        # ["-", "pdu[0]", {"uid"=>"graphene-pdu9", "port"=>23}],
        # ["+", "pdu[0]", {"uid"=>"graphene-pdu9", "port"=>21}],
        # ["+", "pdu[1]", {"uid"=>"graphene-pdu9", "port"=>22}]]
        # => should be something like this:
        # [["~", "pdu[0]", {"uid"=>"graphene-pdu9", "port"=>23}, {"uid"=>"graphene-pdu9", "port"=>22},
        #  ["~", "pdu[1]", {"uid"=>"graphene-pdu9", "port"=>24}, {"uid"=>"graphene-pdu9", "port"=>23}}
        d = diffs.select{|x| x[0] != '~' }.group_by{ |x| x[1] }
        d.each { |k, v|
          d[k] = v.group_by{ |x| x[0] }
        }
        d.each { |k,v|
          if v.key?('-') && v.key?('+')
            #puts "Warning: #{node_uid}: convert +/- -> ~ for #{k}"
            diffs.delete(["-", k, v['-'][0][2]])
            diffs.delete(["+", k, v['+'][0][2]])
            diffs << ["~", k, v['-'][0][2], v['+'][0][2] ]
          end
        }
        # end of hack

193
        # Remove keys that are specific to each nodes (ip, mac etc.)
194
        ikeys = cignore_keys[site_uid][node_uid] rescue nil
195
        diffs.clone.each { |diff|
196 197
          diffs.delete(diff) if ignore_keys.include?(diff[0] + diff[1])
          diffs.delete(diff) if ikeys && ikeys.include?(diff[0] + diff[1])
198 199 200
        }

        if verbose && !diffs.empty?
201 202
          puts "Differences between #{refnode_uid} and #{node_uid}:"
          pp diffs
203 204
        end

205
        total_count += diffs.size
206 207 208 209 210 211 212 213 214
        count[site_uid][cluster_uid] += diffs.size

        # Remove the following line if you want to compare each nodes to the first cluster node
        refnode_uid = node_uid
        refnode = node
      end
    end
  end

215
  return [total_count, count]
216 217
end

218 219
def check_cluster_homogeneity(refapi_hash, options = {:verbose => false})
  verbose = options[:verbose]
220 221
  puts "Differences found between successive nodes, per cluster:\n\n"

222
  total_count, count = cluster_homogeneity(refapi_hash, options)
223 224 225 226
  puts "\n" if verbose

  puts count.to_yaml unless verbose

227
  puts "\nUse '-v' option for details." unless verbose
228 229

  return total_count
230 231 232 233 234 235
end

if __FILE__ == $0
  require 'optparse'

  options = {}
Arthur Garnier's avatar
Arthur Garnier committed
236
  options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
237 238 239

  OptionParser.new do |opts|
    opts.banner = "Usage: check-cluster-homogeneity.rb [options]"
240

241 242
    opts.separator ""
    opts.separator "Example: ruby check-cluster-homogeneity.rb -v"
243 244

    ###
245

246
    opts.separator ""
247
    opts.separator "Filters:"
248

249 250 251 252 253
    opts.on('-s', '--sites a,b,c', Array, 'Select site(s)',
            "Default: "+options[:sites].join(", ")) do |s|
      raise "Wrong argument for -s option." unless (s - options[:sites]).empty?
      options[:sites] = s
    end
254

255 256 257
    opts.on('-c', '--clusters a,b,c', Array, 'Select clusters(s). Default: all') do |s|
      options[:clusters] = s
    end
258

259 260
    opts.separator ""
    opts.separator "Common options:"
261 262 263 264 265

    opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
      options[:verbose] ||= 0
      options[:verbose] = options[:verbose] + 1
    end
266

267 268 269 270 271 272
    # Print an options summary.
    opts.on_tail("-h", "--help", "Show this message") do
      puts opts
      exit
    end
  end.parse!
273

274
  refapi_hash = load_yaml_file_hierarchy(File.expand_path("../../input/grid5000/", File.dirname(__FILE__)))
275 276 277 278 279
  total_count = check_cluster_homogeneity(refapi_hash, options)

  # return 0 if all nodes are homogeneous, 1 otherwise
  exit total_count == 0

280
end