hardware.rb 16.7 KB
Newer Older
1
# coding: utf-8
2 3
$LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), 'lib')))
require 'wiki_generator'
4 5 6 7 8 9 10 11 12
require_relative './site_hardware.rb'

class G5KHardwareGenerator < WikiGenerator

  def initialize(page_name)
    super(page_name)
  end

  def generate_content
13
    @global_hash = get_global_hash
14
    @site_uids = G5K::SITES
15

16
    @generated_content = "__NOEDITSECTION__\n"
17
    @generated_content += "<div class=\"sitelink\">[[Hardware|Global]] | " + G5K::SITES.map { |e| "[[#{e.capitalize}:Hardware|#{e.capitalize}]]" }.join(" | ") + "</div>\n"
18 19 20
    @generated_content += "\n= Clusters =\n"
    @generated_content += SiteHardwareGenerator.generate_all_clusters
    @generated_content += generate_totals
21
    @generated_content += MW.italic(MW.small(generated_date_string))
22 23 24 25 26 27 28 29 30 31
    @generated_content += MW::LINE_FEED
  end

  def generate_totals
    data = {
      'proc_families' => {},
      'proc_models' => {},
      'core_models' => {},
      'ram_size' => {},
      'net_interconnects' => {},
32
      'net_models' => {},
33 34 35 36 37
      'acc_families' => {},
      'acc_models' => {},
      'acc_cores' => {},
      'node_models' => {}
    }
38

39 40 41
    @global_hash['sites'].sort.to_h.each { |site_uid, site_hash|
      site_hash['clusters'].sort.to_h.each { |cluster_uid, cluster_hash|
        cluster_hash['nodes'].sort.to_h.each { |node_uid, node_hash|
42 43
          next if node_hash['status'] == 'retired'
          @node = node_uid
44

45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
          # Processors
          model = node_hash['processor']['model']
          version = "#{model} #{node_hash['processor']['version']}"
          microarchitecture = node_hash['processor']['microarchitecture']

          cluster_procs = node_hash['architecture']['nb_procs']
          cluster_cores = node_hash['architecture']['nb_cores']

          key = [model]
          init(data, 'proc_families', key)
          data['proc_families'][key][site_uid] += cluster_procs

          key = [{text: microarchitecture || ' ', sort: get_date(microarchitecture) + ', ' + microarchitecture.to_s}, {text: version, sort: get_date(microarchitecture) + ', ' + version.to_s}]
          init(data, 'proc_models', key)
          data['proc_models'][key][site_uid] += cluster_procs

          init(data, 'core_models', key)
          data['core_models'][key][site_uid] += cluster_cores
63

64 65 66 67 68 69 70
          # RAM size
          ram_size = node_hash['main_memory']['ram_size']
          key = [{ text: G5K.get_size(ram_size), sort: (ram_size / 2**30).to_s.rjust(6, '0') + ' GB' }]
          init(data, 'ram_size', key)
          data['ram_size'][key][site_uid] += 1

          # HPC Networks
71 72 73
          interfaces = node_hash['network_adapters'].select{ |k, v|
            v['enabled'] and
            (v['mounted'] or v['mountable']) and
74 75
            not v['management'] and
            (k =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
76 77 78 79 80 81 82 83
          }.map{ |k, v|
            [
              {
                text: v['interface'] + ' ' + G5K.get_rate(v['rate']),
                sort: ((v['rate'])/10**6).to_s.rjust(6, '0') + ' Gbps, ' + v['interface']
              }
            ]
          }.uniq
84 85

          net_interconnects = interfaces.inject(Hash.new(0)){ |h, v| h[v] += 1; h }
86
          net_interconnects.sort_by { |k, v|  k.first[:sort] }.each { |k, v|
87 88 89
            init(data, 'net_interconnects', k)
            data['net_interconnects'][k][site_uid] += v
          }
90

91
          # NIC models
92 93 94
          interfaces = node_hash['network_adapters'].select{ |k, v|
            v['enabled'] and
            (v['mounted'] or v['mountable']) and
95 96
            not v['management'] and
            (k =~ /\./).nil? # exclude PKEY / VLAN interfaces see #9417
97 98 99 100 101 102 103 104 105 106 107 108
          }.map{ |k, v|
            t = (v['vendor'] || 'N/A') + ' ' + (v['model'] || 'N/A');
            [
              {
                text: v['interface'],
                sort: v['interface']
              },
              {
                text: t, sort: t
              }
            ]
          }.uniq
109 110 111 112 113 114

          net_models = interfaces.inject(Hash.new(0)){ |h, v| h[v] += 1; h }
          net_models.sort_by { |k, v|  k.first[:sort] }.each { |k, v|
            init(data, 'net_models', k)
            data['net_models'][k][site_uid] += v
          }
115 116 117

          # Accelerators
          g = node_hash['gpu']
118
          m = node_hash['mic']
119 120 121 122 123

          gpu_families = {}
          gpu_families[[g['gpu_vendor']]] = g['gpu_count'] if g and g['gpu']
          mic_families = {}
          mic_families[[m['mic_vendor']]] = m['mic_count'] if m and m['mic']
124
          gpu_families.merge(mic_families).sort.to_h.each { |k, v|
125
            init(data, 'acc_families', k)
126 127 128 129 130 131 132
            data['acc_families'][k][site_uid] += v
          }

          gpu_details = {}
          gpu_details[["#{g['gpu_vendor']} #{g['gpu_model']}"]] = [g['gpu_count'], g['gpu_cores']] if g and g['gpu']
          mic_details = {}
          mic_details[["#{m['mic_vendor']} #{m['mic_model']}"]] = [m['mic_count'], m['mic_cores']] if m and m['mic']
133

134
          gpu_details.merge(mic_details).sort.to_h.each { |k, v|
135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
            init(data, 'acc_models', k)
            data['acc_models'][k][site_uid] += v[0]

            init(data, 'acc_cores', k)
            data['acc_cores'][k][site_uid] += v[1]
          }

          # Nodes
          key = [cluster_hash['model']]
          init(data, 'node_models', key)
          data['node_models'][key][site_uid] += 1
        }
      }
    }

    # Table construction
    generated_content = "= Processors ="
    generated_content += "\n== Processors counts per families ==\n"
    sites = @site_uids.map{ |e| "[[#{e.capitalize}:Hardware|#{e.capitalize}]]" }
    table_options = 'class="wikitable sortable" style="text-align: center;"'
    table_columns = ['Processor family'] + sites + ['Processors total']
    generated_content += MW.generate_table(table_options, table_columns, get_table_data(data, 'proc_families'))
    generated_content += "\n== Processors counts per models ==\n"
    table_columns = ['Microarchitecture', 'Processor model'] + sites + ['Processors total']
    generated_content += MW.generate_table(table_options, table_columns, get_table_data(data, 'proc_models'))
    generated_content += "\n== Cores counts per models ==\n"
    table_columns =  ['Microarchitecture', 'Core model'] + sites + ['Cores total']
    generated_content += MW.generate_table(table_options, table_columns, get_table_data(data, 'core_models'))

    generated_content += "\n= RAM size per node =\n"
    table_columns = ['RAM size'] + sites + ['Nodes total']
    generated_content += MW.generate_table(table_options, table_columns, get_table_data(data, 'ram_size'))

168 169
    generated_content += "\n= Networking =\n"
    generated_content += "\n== Network interconnects ==\n"
170 171 172
    table_columns = ['Interconnect'] + sites + ['Cards total']
    generated_content += MW.generate_table(table_options, table_columns, get_table_data(data, 'net_interconnects'))

173
    generated_content += "\n== Nodes with several Ethernet interfaces ==\n"
174
    generated_content +=  generate_interfaces
175 176

    generated_content += "\n== Network interface models ==\n"
177
    table_columns = ['Type', 'Model'] + sites + ['Cards total']
178
    generated_content += MW.generate_table(table_options, table_columns, get_table_data(data, 'net_models'))
179

180 181 182 183
    generated_content += "\n= Storage ="
    generated_content += "\n== Nodes with several disks ==\n"
    generated_content +=  generate_storage

184 185 186 187 188 189 190 191 192 193
    generated_content += "\n= Accelerators (GPU, Xeon Phi) ="
    generated_content += "\n== Accelerator families ==\n"
    table_columns = ['Accelerator family'] + sites + ['Accelerators total']
    generated_content += MW.generate_table(table_options, table_columns, get_table_data(data, 'acc_families'))
    table_columns = ['Accelerator model'] + sites + ['Accelerators total']
    generated_content += "\n== Accelerator models ==\n"
    generated_content += MW.generate_table(table_options, table_columns, get_table_data(data, 'acc_models'))
    generated_content += "\n== Accelerator cores ==\n"
    table_columns = ['Accelerator model'] + sites + ['Cores total']
    generated_content += MW.generate_table(table_options, table_columns, get_table_data(data, 'acc_cores'))
194

195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
    generated_content += "\n= Nodes models =\n"
    table_columns = ['Nodes model'] + sites + ['Nodes total']
    generated_content += MW.generate_table(table_options, table_columns, get_table_data(data, 'node_models'))
  end

  def init(data, key1, key2)
    if not data[key1].key?(key2)
      data[key1][key2] = {}
      @site_uids.each { |s| data[key1][key2][s] = 0 }
    end
  end

  # This method generates a wiki table from data[key] values, sorted by key
  # values in first column.
  def get_table_data(data, key)
    raw_data = []
    table_data = []
    index = 0
    k0 = 0
    data[key].sort_by{
215 216 217 218
      # Sort the table by the identifiers (e.g. Microarchitecture, or Microarchitecture + CPU name).
      # This colum is either just a text field, or a more complex hash with a :sort key that should be
      # used for sorting.
      |k, v| k.map { |c| c.kind_of?(Hash) ? c[:sort] : c }
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
    }.to_h.each { |k, v|
      k0 = k if index == 0
      index += 1
      elts = v.sort.to_h.values
      raw_data << elts
      table_data << k.map{ |e| e.kind_of?(Hash) ? "data-sort-value=\"#{e[:sort]}\"|#{e[:text]}" : "data-sort-value=\"#{index.to_s.rjust(3, '0')}\"|#{e}" } +
        elts.map{ |e| e.kind_of?(Hash) ? "data-sort-value=\"#{e[:sort]}\"|#{e[:text]}" : e }
        .map{ |e| e == 0 ? '' : e  } + ["'''#{elts.reduce(:+)}'''"]
    }
    elts = raw_data.transpose.map{ |e| e.reduce(:+)}
    table_data << {columns: ["'''Sites total'''"] +
                   [' '] * (k0.length - 1) +
                   (elts + [elts.reduce(:+)]).map{ |e| e == 0 ? '' : "'''#{e}'''" },
                   sort: false}
  end

  # See: https://en.wikipedia.org/wiki/List_of_Intel_Xeon_microprocessors
  # For a correct sort of the column, all dates must be in the same
  # format (same number of digits)
  def get_date(microarchitecture)
239
    return 'MISSING' if microarchitecture.nil?
240 241 242 243 244 245 246 247 248 249 250 251
    release_dates = {
      'K8' => '2003',
      'K10' => '2007',
      'Clovertown' => '2006',
      'Harpertown' => '2007',
      'Dunnington' => '2008',
      'Lynnfield' => '2009',
      'Nehalem' => '2010',
      'Westmere' => '2011',
      'Sandy Bridge' => '2012',
      'Haswell' => '2013',
      'Broadwell' => '2015',
252
      'Skylake' => '2016'
253 254
    }
    date = release_dates[microarchitecture.to_s]
255
    raise "ERROR: microarchitecture not found: '#{microarchitecture.to_s}'. Add in hardware.rb" if date.nil?
256 257
    date
  end
258 259 260 261 262 263 264 265 266 267

  def storage_size_to_text(s)
    if s > 1000*1000*1000*1000 # 1 TB
      return sprintf("%.1f", s/(1000*1000*1000*1000)) + ' TB'
    else
      return sprintf("%d", s/(1000*1000*1000)) + ' GB'
    end
  end

  def generate_storage
268
    table_columns = ["Site", "Cluster", "Number of nodes", "Main disk", "Additional HDDs", "Additional SSDs", "[[Disk_reservation|Disk reservation]]"]
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
    table_data = []
    global_hash = get_global_hash

    # Loop over Grid'5000 sites
    global_hash["sites"].sort.to_h.each do |site_uid, site_hash|
      site_hash.fetch("clusters").sort.to_h.each do |cluster_uid, cluster_hash|
        nodes_data = []
        cluster_hash.fetch('nodes').sort.to_h.each do |node_uid, node_hash|
          next if node_hash['status'] == 'retired'
          sd = node_hash['storage_devices']
          reservable_disks = sd.to_a.select{ |v| v[1]['reservation'] == true }.count > 0
          maindisk = sd.to_a.select { |v| v[0] == 'sda' }.first[1]
          maindisk_t = maindisk['storage'] + ' ' + storage_size_to_text(maindisk['size'])
          other = sd.to_a.select { |d| d[0] != 'sda' }
          hdds = other.select { |d| d[1]['storage'] == 'HDD' }
          if hdds.count == 0
            hdd_t = "0"
          else
            hdd_t = hdds.count.to_s + " (" + hdds.map { |d| storage_size_to_text(d[1]['size']) }.join(', ') + ")"
          end
          ssds = other.select { |d| d[1]['storage'] == 'SSD' }
          if ssds.count == 0
            ssd_t = "0"
          else
            ssd_t = ssds.count.to_s + " (" + ssds.map { |d| storage_size_to_text(d[1]['size']) }.join(', ') + ")"
          end
295 296 297
          queues = cluster_hash['queues'] - ['admin', 'default']
          queue_t = (queues.nil? || (queues.empty? ? '' : "_.28" + queues[0].gsub(' ', '_') + ' queue.29'))
          nodes_data << { 'uid' => node_uid, 'data' => { 'main' => maindisk_t, 'hdd' => hdd_t, 'ssd' => ssd_t, 'reservation' => reservable_disks, 'queue' => queue_t } }
298 299 300 301 302 303 304 305 306 307 308 309
        end
        nd = nodes_data.group_by { |d| d['data'] }
        nd.each do |data, nodes|
          # only keep nodes with more than one disk
          next if data['hdd'] == "0" and data['ssd'] == "0"
          if nd.length == 1
            nodesetname = cluster_uid
          else
            nodesetname = G5K.nodeset(nodes.map { |n| n['uid'].split('-')[1].to_i })
          end
          table_data << [
            "[[#{site_uid.capitalize}:Hardware|#{site_uid.capitalize}]]",
310
              "[[#{site_uid.capitalize}:Hardware##{cluster_uid}#{data['queue']}|#{nodesetname}]]",
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
              nodes.length,
              data['main'],
              data['hdd'],
              data['ssd'],
              data['reservation'] ? 'yes' : 'no'
          ]
        end
      end
    end
    # Sort by site and cluster name
    table_data.sort_by! { |row|
      [row[0], row[1]]
    }

    # Table construction
Lucas Nussbaum's avatar
Lucas Nussbaum committed
326
    table_options = 'class="wikitable sortable" style="text-align: center;"'
327 328 329
    return MW.generate_table(table_options, table_columns, table_data)
  end

330 331
  def generate_interfaces
    table_data = []
332 333
    @global_hash["sites"].sort.to_h.each { |site_uid, site_hash|
      site_hash.fetch("clusters").sort.to_h.each { |cluster_uid, cluster_hash|
334
        network_interfaces = {}
335
        cluster_hash.fetch('nodes').sort.to_h.each { |node_uid, node_hash|
336 337 338 339 340 341 342 343 344 345
          next if node_hash['status'] == 'retired'
          if node_hash['network_adapters']
            node_interfaces = node_hash['network_adapters'].select{ |k, v| v['interface'] == 'Ethernet' and v['enabled'] == true and (v['mounted'] == true or v['mountable'] == true) and v['management'] == false }

            interfaces = {}
            interfaces['10g_count'] = node_interfaces.select { |k, v| v['rate'] == 10_000_000_000 }.count
            interfaces['1g_count'] = node_interfaces.select { |k, v| v['rate'] == 1_000_000_000 }.count
            interfaces['details'] = node_interfaces.map{ |k, v| k + (v['name'].nil? ? '' : '/' + v['name'])  + ' (' + G5K.get_rate(v['rate']) + ')' }.sort.join(', ')
            queues = cluster_hash['queues'] - ['admin', 'default']
            interfaces['queues'] = (queues.nil? || (queues.empty? ? '' : queues[0] + G5K.pluralize(queues.count, ' queue')))
346
            interface_add(network_interfaces, node_uid, interfaces) if node_interfaces.count > 1
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
          end
        }

        # One line for each group of nodes with the same interfaces
        network_interfaces.sort.to_h.each { |num, interfaces|
          table_data << [
            "[[#{site_uid.capitalize}:Network|#{site_uid.capitalize}]]",
            "[[#{site_uid.capitalize}:Hardware##{cluster_uid}" + (interfaces['queues'] == '' ? '' : "_.28#{queues.gsub(' ', '_')}.29") + "|#{cluster_uid}" + (network_interfaces.size==1 ? '' : '-' + G5K.nodeset(num)) + "]]",
            num.count,
            interfaces['10g_count'].zero? ? '' : interfaces['10g_count'],
            interfaces['1g_count'].zero? ? '' : interfaces['1g_count'],
            interfaces['details']
          ]
        }
      }
    }
    # Sort by site and cluster name
    table_data.sort_by! { |row|
      [row[0], row[1]]
    }

    table_options = 'class="wikitable sortable" style="text-align: center;"'
    table_columns = ["Site", "Cluster", "Nodes", "10G interfaces", "1G interfaces", "Interfaces (throughput)"]
370
    MW.generate_table(table_options, table_columns, table_data)
371 372 373 374 375 376
  end

  # This methods adds the array interfaces to the hash
  # network_interfaces. If nodes 2,3,7 have the same interfaces, they
  # will be gathered in the same key and we will have
  # network_interfaces[[2,3,7]] = interfaces
377
  def interface_add(network_interfaces, node_uid, interfaces)
378 379 380 381 382 383 384 385
    num1 = node_uid.split('-')[1].to_i
    if network_interfaces.has_value?(interfaces) == false
      network_interfaces[[num1]] = interfaces
    else
      num2 = network_interfaces.key(interfaces)
      network_interfaces.delete(num2)
      network_interfaces[num2.push(num1)] = interfaces
    end
386
  end
387 388
end

Lucas Nussbaum's avatar
Lucas Nussbaum committed
389 390
if __FILE__ == $0
  generator = G5KHardwareGenerator.new("Hardware")
391

Lucas Nussbaum's avatar
Lucas Nussbaum committed
392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
  options = WikiGenerator::parse_options
  if (options)
    ret = 2
    begin
      ret = generator.exec(options)
    rescue MediawikiApi::ApiError => e
      puts e, e.backtrace
      ret = 3
    rescue StandardError => e
      puts "Error with node: #{generator.instance_variable_get(:@node)}"
      puts e, e.backtrace
      ret = 4
    ensure
      exit(ret)
    end
407 408
  end
end