input_loader.rb 11.9 KB
Newer Older
1
2
# Load a hierarchy of YAML file into a Ruby hash

Lucas Nussbaum's avatar
Lucas Nussbaum committed
3
require 'refrepo/hash/hash'
4
require 'refrepo/gen/reference-api'
5

6
def load_yaml_file_hierarchy(directory = File.expand_path("../../input/grid5000/", File.dirname(__FILE__)))
7
8

  global_hash = {} # the global data structure
9

10
  Dir.chdir(directory) {
11

12
13
14
    # Recursively list the .yaml files.
    # The order in which the results are returned depends on the system (http://ruby-doc.org/core-2.2.3/Dir.html).
    # => List deepest files first as they have lowest priority when hash keys are duplicated.
15
    list_of_yaml_files = Dir['**/*.y*ml', '**/*.y*ml.erb'].sort_by { |x| -x.count('/') }
16

17
    list_of_yaml_files.each { |filename|
18
19
20
21
22
23
24
25
      begin
        # Load YAML
        if /\.y.*ml\.erb$/.match(filename)
          # For files with .erb.yaml extensions, process the template before loading the YAML.
          file_hash = YAML::load(ERB.new(File.read(filename)).result(binding))
        else
          file_hash = YAML::load_file(filename)
        end
26
      if not file_hash
27
        raise Exception.new("loaded hash is empty")
28
      end
29
      # YAML::Psych raises an exception if the file cannot be loaded.
Lucas Nussbaum's avatar
Lucas Nussbaum committed
30
      rescue StandardError => e
31
32
33
        puts "Error loading '#{filename}', #{e.message}"
      end

34
35
      # Inject the file content into the global_hash, at the right place
      path_hierarchy = File.dirname(filename).split('/')     # Split the file path (path relative to input/)
36
37
      path_hierarchy = [] if path_hierarchy == ['.']

38
39
      file_hash = Hash.from_array(path_hierarchy, file_hash) # Build the nested hash hierarchy according to the file path
      global_hash = global_hash.deep_merge(file_hash)        # Merge global_hash and file_hash. The value for entries with duplicate keys will be that of file_hash
40
41
42

      # Expand the hash. Done at each iteration for enforcing priorities between duplicate entries:
      # ie. keys to be expanded have lowest priority on existing entries but higher priority on the entries found in the next files
43
      global_hash.expand_square_brackets(file_hash)
44

45
46
47
48
    }

  }

Jérémie Gaidamour's avatar
Jérémie Gaidamour committed
49
50
#  pp global_hash

51
52
53
  # populate each node with its IPv6
  add_ipv6(global_hash)

54
55
56
57
  # populate each node with its IPv4 addresses
  add_ipv4(global_hash)

  # populate each node with its kavlan IPv4 IPs
58
  add_kavlan_ips(global_hash)
59
  add_kavlan_ipv6s(global_hash)
60

61
62
63
  # populate each node with software informations
  add_software(global_hash)

64
65
66
  # populate each cluster with metrics network information
  add_network_metrics(global_hash)

67
68
69
  # populate each node with theorical flops
  add_theorical_flops(global_hash)

70
71
  return global_hash
end
72
73

def sorted_vlan_offsets
74
75
  offsets = load_yaml_file_hierarchy['vlans']['offsets'].split("\n").
    map { |l| l = l.split(/\s+/) ; (4..7).each { |e| l[e] = l[e].to_i } ; l }
76
77
78
79
80
81
82
83
84
85
86
87
88
89
  # for local VLANs, we include the site when we sort
  puts offsets.select { |l| l[0] == 'local' }.
   sort_by { |l| [l[0], l[1] ] + l[4..-1] }.
   map { |l| l.join(' ') }.
   join("\n")
  puts offsets.select { |l| l[0] != 'local' }.
   sort_by { |l| [l[0] ] + l[4..-1] }.
   map { |l| l.join(' ') }.
   join("\n")

end


def add_kavlan_ips(h)
90
  allocated = {}
91
92
  vlan_base = h['vlans']['base']
  vlan_offset = h['vlans']['offsets'].split("\n").map { |l| l = l.split(/\s+/) ; [ l[0..3], l[4..-1].inject(0) { |a, b| (a << 8) + b.to_i } ] }.to_h
93
  h['sites'].each_pair do |site_uid, hs|
94
    # forget about allocated ips for local vlans, since we are starting a new site
95
    allocated.delete_if { |k, v| v[3] == 'local' }
96
    hs['clusters'].each_pair do |cluster_uid, hc|
97
      next if !hc['kavlan'] # skip clusters where kavlan is globally set to false (used for initial cluster installation)
98
      hc['nodes'].each_pair do |node_uid, hn|
99
100
        raise "Node hash for #{node_uid} is nil" if hn.nil?
        raise "Old kavlan data in input/ for #{node_uid}" if hn.has_key?('kavlan')
101
102
103
104
        node_id = node_uid.split('-')[1].to_i
        hn['kavlan'] = {}
        hn['network_adapters'].to_a.select { |i| i[1]['mountable'] and (i[1]['kavlan'] or not i[1].has_key?('kavlan')) and i[1]['interface'] == 'Ethernet' }.map { |e| e[0] }.each do |iface|
          hn['kavlan'][iface] = {}
105
106
107
          vlan_base.each do |vlan, v|
            type = v['type']
            base = IPAddress::IPv4::new(v['address']).to_u32
108
            k = [type, site_uid, cluster_uid, iface]
109
            if not vlan_offset.has_key?(k)
110
              raise "Missing VLAN offset for #{k}"
111
            end
112
113
            ip = IPAddress::IPv4::parse_u32(base + vlan_offset[k] + node_id).to_s
            a = [ site_uid, node_uid, iface, type, vlan ]
114
115
            raise "IP already allocated: #{ip} (trying to add it to #{a} ; allocated to #{allocated[ip]})" if allocated[ip]
            allocated[ip] = a
116
117
118
119
120
            hn['kavlan'][iface]["kavlan-#{vlan}"] = ip
          end
        end
      end
    end
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
  end
end

def add_ipv4(h)
  allocated = {}
  base = IPAddress::IPv4::new(h['ipv4']['base']).to_u32
  sites_offsets = h['ipv4']['sites_offsets'].split("\n").map { |l| l = l.split(/\s+/) ; [ l[0], l[1..-1].inject(0) { |a, b| (a << 8) + b.to_i } ] }.to_h
  iface_offsets = h['ipv4']['iface_offsets'].split("\n").map { |l| l = l.split(/\s+/) ; [ l[0..2], l[3..-1].inject(0) { |a, b| (a << 8) + b.to_i } ] }.to_h
  h['sites'].each_pair do |site_uid, hs|
    hs['clusters'].each_pair do |cluster_uid, hc|
      hc['nodes'].each_pair do |node_uid, hn|
        raise "Node hash for #{node_uid} is nil" if hn.nil?
        node_id = node_uid.split('-')[1].to_i
        hn['network_adapters'].each_pair do |iface, v|
          # only allocate mountable ethernet interfaces
          next if not (v['mountable'] and v['interface'] == 'Ethernet')
          k = [site_uid, cluster_uid, iface]
          if not iface_offsets.has_key?(k)
            raise "Missing IPv4 information for #{k}"
          end
          ip = IPAddress::IPv4::parse_u32(base + sites_offsets[site_uid] + iface_offsets[k] + node_id).to_s
          a = [ site_uid, node_uid, iface ]
          raise "IP already allocated: #{ip} (trying to add it to #{a} ; allocated to #{allocated[ip]})" if allocated[ip]
          allocated[ip] = a
          v['ip'] = ip
        end
      end
    end
149
150
  end
end
151
152
153
154
155
156

def add_ipv6(h)
  # for each node
  h['sites'].each_pair do |site_uid, hs|
    hs['clusters'].each_pair do |cluster_uid, hc|
      hc['nodes'].each_pair do |node_uid, hn|
157
        ipv6_adapters = hn['network_adapters'].select { |_k,v| v['mountable'] and v['interface'] == 'Ethernet' }
158
159
160
161
162
        if ipv6_adapters.length > 0
          if not ipv6_adapters.values[0]['mounted']
            raise "#{node_uid}: inconsistency: this code assumes first mountable ethernet adapter should be mounted: #{hn}"
          end
          ip4 = ipv6_adapters.values[0]['ip']
163
          ipv6_adapters.each_with_index do |(_iface, nah), idx|
164
            # compute and assign IPv6 based on IPv4 of the first adapter
165
            ip6 = h['ipv6']['prefix'] + ':'
166
167
            ip6 += '%x' % h['ipv6']['site-indexes'][site_uid]
            ip6 += '00:'
168
169
170
171
172
173
            ip6 += '%x' % ((ip4.split('.')[2].to_i & 0b1111) + 1)
            if idx > 0
              ip6 += ':%x::' % idx
            else
              ip6 += '::'
            end
174
175
176
177
178
179
180
181
182
183
184
            ip6 += '%x' % (ip4.split('.')[3].to_i)
            nah['ip6'] = ip6
          end
        end
      end
    end
  end
end

def add_kavlan_ipv6s(h)
  h['sites'].each_pair do |site_uid, hs|
185
    hs['clusters'].each_pair do |_cluster_uid, hc|
186
      next if !hc['kavlan'] # skip clusters where kavlan is globally set to false (used for initial cluster installation)
187
      hc['nodes'].each_pair do |node_uid, hn|
188
        kvl_adapters = hn['network_adapters'].select { |_k,v| v['mountable'] and (v['kavlan'] or not v.has_key?('kavlan')) and v['interface'] == 'Ethernet' }
189
190
191
192
193
194
195
196
197
        if kvl_adapters.length > 0
          if kvl_adapters.length != hn['kavlan'].length
            raise "#{node_uid}: inconsistency: num kvl_adapters = #{kvl_adapters.length}, num kavlan entries = #{hn['kavlan'].length}"
          end
          if not kvl_adapters.values[0]['mounted']
            raise "#{node_uid}: inconsistency: this code assumes first kvl_adapters should be mounted: #{hn}"
          end
          ip4 = kvl_adapters.values[0]['ip']
          hn['kavlan6'] = {}
198
          kvl_adapters.each_with_index do |(iface, _nah), idx|
199
200
201
            hn['kavlan6'][iface] = {}
            hn['kavlan'][iface].each_key do |kvl|
              kvl_id = kvl.split('-')[1].to_i
202
              ip6 = h['ipv6']['prefix'] + ':'
203
              ip6 += '%x' % h['ipv6']['site-indexes'][site_uid]
204
              ip6 += '%x:' % (kvl_id + 0x80)
205
206
207
208
209
210
              ip6 += '%x' % ((ip4.split('.')[2].to_i & 0b1111) + 1)
              if idx > 0
                ip6 += ':%x::' % idx
              else
                ip6 += '::'
              end
211
              ip6 += '%x' % (ip4.split('.')[3].to_i)
212
              hn['kavlan6'][iface]["kavlan-#{kvl_id}"] = ip6
213
214
            end
          end
215
        end
216
217
218
219
      end
    end
  end
end
220
221
222
223
224
225

def add_software(h)
  # for each node
  h['sites'].each_pair do |site_uid, hs|
    hs['clusters'].each_pair do |cluster_uid, hc|
      hc['nodes'].each_pair do |node_uid, hn|
226
227
228
        if not hn.key?('software')
          hn['software'] = {}
        end
229
230
231
232
233
234
        hn['software']['postinstall-version'] = h['software']['postinstall-version']
        hn['software']['forced-deployment-timestamp'] = h['software']['forced-deployment-timestamp']
      end
    end
  end
end
235
236
237
238
239
240
241
242
243
244

def add_network_metrics(h)
  # for each cluster
  h['sites'].each_pair do |site_uid, site|
    site['clusters'].each_pair do |cluster_uid, cluster|

      # remove any network metrics defined in cluster
      cluster['metrics'] = cluster.fetch('metrics', []).reject {|m| m['name'] =~ /network_.*_bytes_total/}

      # for each interface of a cluster's node
245
      node_uid, node = cluster['nodes'].select { |k, v| v['status'] != 'retired' }.sort_by{ |k, v| k }.first
246
247
248
249
250
251
252
253
254
255
256
257
258
      node["network_adapters"].each do |iface_uid, iface|

        # get switch attached to interface
        if iface['mounted'] and not iface['management'] and iface['interface'] == 'Ethernet'
          switch, _ = net_switch_port_lookup(site, node_uid, iface_uid) || net_switch_port_lookup(site, node_uid)
        else
          switch, _ = net_switch_port_lookup(site, node_uid, iface_uid)
        end

        # for each network metric declared for the switch
        site.fetch('networks', {}).fetch(switch, {}).fetch('metrics', []).select{|m| m['name'] =~ /network_.*_bytes_total/}.each do |metric|

          # add this metric to cluster's list of available metrics, associated to node interface
259
260
          new_metric = metric.merge({"labels" => {"interface" => iface_uid}})
          new_metric["source"] = {"protocol" => "network_equipment"}
261
          cluster['metrics'].push(new_metric)
262
263
264
265
266
        end
      end
    end
  end
end
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288

def get_flops_per_cycle(microarch, cpu_name)
  # Double precision operations each cycle, sources:
  # https://en.wikipedia.org/wiki/FLOPS
  # https://en.wikichip.org/wiki/WikiChip
  # https://ark.intel.com/
  case microarch
  when "K8"
    return 2
  when "Clovertown", "Nehalem", "Westmere", "K10"
    return 4
  when "Sandy Bridge", "Zen", "Vulcan"
    return 8
  when "Haswell", "Broadwell"
    return 16
  when "Cascade Lake-SP", "Skylake"
    case cpu_name
    when /Silver 4110/, /Gold 5218/, /Gold 5220/
      return 16
    when /Gold 6126/, /Gold 6130/
      return 32
    end
289
290
291
  # 4 64-bit FPUs, x2 for Fused Multiply-Add
  when /POWER8/
    return 8
292
293
294
295
296
297
298
299
  end
  raise "Error: Unknown CPU architecture, cannot compute flops"
end

def add_theorical_flops(h)
  h['sites'].each_pair do |site_uid, site|
    site['clusters'].each_pair do |cluster_uid, cluster|
      cluster['nodes'].select { |k, v| v['status'] != 'retired' }.each_pair do |node_uid, node|
300
        node['performance'] = {}
301
302
        node['performance']['core_flops'] =  node['processor']['clock_speed'].to_i * get_flops_per_cycle(node['processor']['microarchitecture'], node['processor']['other_description'])
        node['performance']['node_flops'] = node['architecture']['nb_cores'].to_i * node['performance']['core_flops'].to_i
303
304
305
306
      end
    end
  end
end