oar_properties.rb 8.62 KB
Newer Older
1
# coding: utf-8
2
require 'refrepo/gen/wiki/wiki_generator'
3

4 5
$LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), '../../../../../generators/oar-properties/lib')))
require 'lib-oar-properties'
6 7 8 9 10 11 12 13 14

class OarPropertiesGenerator < WikiGenerator

  def initialize(page_name)
    super(page_name)
  end

  #Static information about properties that cannot be infered from ref-api data
  @@properties = {
15 16 17
    "max_walltime" => {
      "description" => "Maximum walltime in seconds allowed on this node (for the production queue only)"
    },
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48
    "cluster_priority" => {
      "description" => "The priority of this resource for job scheduling"
    },
    "besteffort" => {
      "description" => "Can the resource be reserved in besteffort mode ?",
      "value_type" => "Boolean"
    },
    "deploy" => {
      "description" => "Can the resource be used for a deployment ?",
      "value_type" => "Boolean"
    },
    "virtual" => {
      "description" => "Node virtualization capacity"
    },
    "production" => {
      "description" => "Is this resource available in production queue ?",
      "value_type" => "Boolean"
    },
    "cluster" => {
      "description" => "The name of the cluster the resource is part of"
    },
    "core" => {
      "description" => "The ID of the CPU core the resource is part of. The unique scope is the OAR server.",
      "possible_values" => "1, 2, 3, ..."
    },
    "cpu" => {
      "description" => "The ID of the CPU the resource is part of. The unique scope is the OAR server. ",
      "possible_values" => "1, 2, 3, ..."
    },
    "host" => {
      "description" => "A user-friendly name for the network_address property",
49
      "possible_values" => "dahu-1.grenoble.grid5000.fr, ..."
50 51 52 53 54 55
    },
    "ip" => {
      "description" => "The IPv4 address of the node the resource is part of"
    },
    "network_address" => {
      "description" => "The full hostname of the node the resource is part of",
56
      "possible_values" => "dahu-1.grenoble.grid5000.fr, ..."
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
    },
    "switch" => {
      "description" => "On what switch the resource is directly connected ?"
    },
    "nodemodel" => {
      "description" => "The type of the chassis"
    },
    "cpuarch" => {
      "description" => "What CPU architecture resource's CPU is member of ?"
    },
    "cpucore" => {
      "description" => "The number of core per CPU"
    },
    "cputype" => {
      "description" => "What processor's family resource's CPU is member of"
    },
    "cpufreq" => {
      "description" => "The frequency in GHz of resource's CPU"
    },
76 77 78
    # "cpucount" => { #Unused ?
    #   "description" => "The number of CPUs"
    # },
79 80 81 82
    "eth_count" => {
      "description" => "The number of Ethernet interface the node has"
    },
    "eth_rate" => {
83
      "description" => "the maximum rate of connected network interfaces in Gbps"
84 85
    },
    "ib" => {
Lucas Nussbaum's avatar
Lucas Nussbaum committed
86
      "description" => "The technology of the Infiniband interface"
87 88 89 90 91
    },
    "ib_count" => {
      "description" => "The number of Infiniband interfaces available"
    },
    "ib_rate" => {
Lucas Nussbaum's avatar
Lucas Nussbaum committed
92 93 94 95 96 97 98 99 100 101
      "description" => "The rate of the connected Infiniband interface in Gbps"
    },
    "opa" => {
      "description" => "Whether an Omni-Path interface is available"
    },
    "opa_count" => {
      "description" => "The number of Omni-Path interfaces available"
    },
    "opa_rate" => {
      "description" => "The rate of the connected Omni-Path interface in Gbps"
102 103 104 105 106 107 108 109
    },
    "myri" => {
      "description" => "The type of Myrinet interfaces available"
    },
    "myri_count" => {
      "description" => "The number of Myrinet interfaces available"
    },
    "myri_rate" => {
110
      "description" => "The rate of the connected Myrinet interface in Gbps"
111 112 113 114
    },
    "disktype" => {
      "description" => "What disk's interface family node's disk is member of ?"
    },
115 116 117
    "disk_reservation_count" => {
      "description" => "The number of reservable disks"
    },
118 119 120 121 122 123 124
    "memcpu" => {
      "description" => "The amount of memory in MB per CPU"
    },
    "memcore" => {
      "description" => "The amount of memory in MB per CPU core"
    },
    "memnode" => {
125
      "description" => "The total amount of memory in MB of the node"
126 127 128 129 130 131 132 133 134
    },
    "gpu" => {
      "description" => "The type of GPU available"
    },
    "gpu_count" => {
      "description" => "The number of GPU cards available"
    },
    "wattmeter" => {
      "description" => "The type of wattmeter available"
135 136 137 138
    },
    "mic" => {
      "description" => "Intel many integrated core architecture support",
      "value_type" => "Boolean"
139 140 141 142 143
    }
  }

  #Group properties by categories
  @@categories = {
144
    "Job-related properties" => ["besteffort", "deploy", "production", "cluster_priority", "max_walltime"],
145
    "Hierarchy" => ["cluster", "cpu", "core", "host", "network_address", "ip", "switch"],
Lucas Nussbaum's avatar
Lucas Nussbaum committed
146
    "Hardware" => ["gpu", "gpu_count", "memnode", "memcore", "memcpu", "disktype", "disk_reservation_count", "myri_rate", "myri_count", "myri", "ib_rate", "ib_count", "ib", "opa", "opa_rate", "opa_count", "eth_rate", "eth_count", "cpufreq", "cputype", "cpucore", "cpuarch", "virtual", "mic"],
147 148 149 150
    "Miscellaneous" => ["wattmeter", "nodemodel"]
  }

  #Existing properties that won't be documented
151
  @@ignored_properties = ["maintenance", "state", "ip_virtual"]
152 153 154

  def get_nodes_properties(site_uid, site)
    properties = {}
155 156
    site['clusters'].sort.to_h.each do |cluster_uid, cluster|
      cluster['nodes'].sort.to_h.each do |node_uid, node|
157
        begin
158
          properties[node_uid] = get_ref_node_properties_internal(cluster_uid, cluster, node_uid, node)
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
        rescue MissingProperty => e
          puts "Error while processing node #{node_uid}: #{e}"
        end
      end
    end
    return properties
  end

  def get_value_type(prop, values)
    if (@@properties[prop]["value_type"])
      return @@properties[prop]["value_type"]
    end
    if (!values.empty?)
      case values[0].class.name
      when "Fixnum"
        return "Integer"
      when "TrueClass", "FalseClass"
        return "Boolean"
      end
    end
    "String"
  end

  def generate_content
183
    refapi = get_global_hash
184 185 186 187 188 189 190 191
    #Properties generated from oar-properties generator
    props = {}
    G5K::SITES.each{ |site_uid|
      props[site_uid] = get_nodes_properties(site_uid, refapi["sites"][site_uid])
    }

    #Compiled properties used to generate page
    oar_properties = {}
192 193 194
    props.sort.to_h.each { |site, site_props|
      site_props.sort.to_h.each { |node_uid, node_props|
        node_props.sort.to_h.each { |property, value|
195 196 197 198 199 200 201
          next if @@ignored_properties.include?(property)

          oar_properties[property] ||= {}
          oar_properties[property]["values"] ||= []
          oar_properties[property]["values"] << value unless value.nil?
          oar_properties[property]["values"].uniq!
          oar_properties[property]["values"].sort!{ |a, b|
202
            (a && a.to_s || "") <=> (b && b.to_s || "")
203 204 205 206
          }
        }
      }
    }
207
    oar_properties.sort.to_h.each { |prop, prop_hash|
Lucas Nussbaum's avatar
Lucas Nussbaum committed
208
      prop_hash["values"].sort!
209 210 211 212 213 214
      if (prop_hash["values"].length > 20)
        #Limit possible values to 20 elements and mark the list as truncated
        prop_hash["values"].slice!(0...-20)
        prop_hash["values"].push("...")
      end
      @@properties[prop]["possible_values"] ||= prop_hash["values"].join(", ")
215 216 217 218 219
    }

    @generated_content = "Properties on resources managed by OAR allow users to select them according to their experiment's characteristics." + MW::LINE_FEED
    @generated_content += MW::heading("OAR Properties", 1) + MW::LINE_FEED

220
    @@categories.sort.to_h.each { |cat, cat_properties|
221
      @generated_content += MW::heading(cat, 2) + MW::LINE_FEED
222
      cat_properties.sort.each{ |property|
223 224 225 226 227
        values = oar_properties[property]["values"] rescue []
        @generated_content += MW::heading(MW::code(property), 3) + MW::LINE_FEED
        @generated_content += @@properties[property]["description"] + MW::LINE_FEED
        value_type = get_value_type(property, values)
        @generated_content += MW::LIST_ITEM + " Value type: " + MW::code(value_type) + MW::LINE_FEED
228
        @generated_content += MW::LIST_ITEM + " Possible values: " + MW::code(@@properties[property]["possible_values"]) + MW::LINE_FEED
229 230
      }
    }
231 232
    @generated_content += MW.italic(MW.small(generated_date_string))
    @generated_content += MW::LINE_FEED
233 234 235
  end
end

Lucas Nussbaum's avatar
Lucas Nussbaum committed
236
if __FILE__ == $0
237
  generator = OarPropertiesGenerator.new("OAR_Properties")
Lucas Nussbaum's avatar
Lucas Nussbaum committed
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252

  options = WikiGenerator::parse_options
  if (options)
    ret = 2
    begin
      ret = generator.exec(options)
    rescue MediawikiApi::ApiError => e
      puts e, e.backtrace
      ret = 3
    rescue StandardError => e
      puts e, e.backtrace
      ret = 4
    ensure
      exit(ret)
    end
253
  end
254
end