Attention une mise à jour du serveur va être effectuée le lundi 17 mai entre 13h et 13h30. Cette mise à jour va générer une interruption du service de quelques minutes.

oar_properties.rb 8.52 KB
Newer Older
1
# coding: utf-8
2

Lucas Nussbaum's avatar
Lucas Nussbaum committed
3
require 'refrepo/gen/oar-properties'
4
require 'refrepo/data_loader'
5 6 7 8 9 10 11 12 13

class OarPropertiesGenerator < WikiGenerator

  def initialize(page_name)
    super(page_name)
  end

  #Static information about properties that cannot be infered from ref-api data
  @@properties = {
14 15 16
    "max_walltime" => {
      "description" => "Maximum walltime in seconds allowed on this node (for the production queue only)"
    },
17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
    "cluster_priority" => {
      "description" => "The priority of this resource for job scheduling"
    },
    "besteffort" => {
      "description" => "Can the resource be reserved in besteffort mode ?",
      "value_type" => "Boolean"
    },
    "deploy" => {
      "description" => "Can the resource be used for a deployment ?",
      "value_type" => "Boolean"
    },
    "virtual" => {
      "description" => "Node virtualization capacity"
    },
    "production" => {
      "description" => "Is this resource available in production queue ?",
      "value_type" => "Boolean"
    },
    "cluster" => {
      "description" => "The name of the cluster the resource is part of"
    },
    "core" => {
      "description" => "The ID of the CPU core the resource is part of. The unique scope is the OAR server.",
      "possible_values" => "1, 2, 3, ..."
    },
    "cpu" => {
      "description" => "The ID of the CPU the resource is part of. The unique scope is the OAR server. ",
      "possible_values" => "1, 2, 3, ..."
    },
    "host" => {
      "description" => "A user-friendly name for the network_address property",
48
      "possible_values" => "dahu-1.grenoble.grid5000.fr, ..."
49 50 51 52 53 54
    },
    "ip" => {
      "description" => "The IPv4 address of the node the resource is part of"
    },
    "network_address" => {
      "description" => "The full hostname of the node the resource is part of",
55
      "possible_values" => "dahu-1.grenoble.grid5000.fr, ..."
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
    },
    "switch" => {
      "description" => "On what switch the resource is directly connected ?"
    },
    "nodemodel" => {
      "description" => "The type of the chassis"
    },
    "cpuarch" => {
      "description" => "What CPU architecture resource's CPU is member of ?"
    },
    "cpucore" => {
      "description" => "The number of core per CPU"
    },
    "cputype" => {
      "description" => "What processor's family resource's CPU is member of"
    },
    "cpufreq" => {
      "description" => "The frequency in GHz of resource's CPU"
    },
75 76 77
    # "cpucount" => { #Unused ?
    #   "description" => "The number of CPUs"
    # },
78 79 80 81
    "eth_count" => {
      "description" => "The number of Ethernet interface the node has"
    },
    "eth_rate" => {
82
      "description" => "the maximum rate of connected network interfaces in Gbps"
83 84
    },
    "ib" => {
Lucas Nussbaum's avatar
Lucas Nussbaum committed
85
      "description" => "The technology of the Infiniband interface"
86 87 88 89 90
    },
    "ib_count" => {
      "description" => "The number of Infiniband interfaces available"
    },
    "ib_rate" => {
Lucas Nussbaum's avatar
Lucas Nussbaum committed
91 92 93 94 95 96 97 98 99 100
      "description" => "The rate of the connected Infiniband interface in Gbps"
    },
    "opa" => {
      "description" => "Whether an Omni-Path interface is available"
    },
    "opa_count" => {
      "description" => "The number of Omni-Path interfaces available"
    },
    "opa_rate" => {
      "description" => "The rate of the connected Omni-Path interface in Gbps"
101 102 103 104 105 106 107 108
    },
    "myri" => {
      "description" => "The type of Myrinet interfaces available"
    },
    "myri_count" => {
      "description" => "The number of Myrinet interfaces available"
    },
    "myri_rate" => {
109
      "description" => "The rate of the connected Myrinet interface in Gbps"
110 111 112 113
    },
    "disktype" => {
      "description" => "What disk's interface family node's disk is member of ?"
    },
114 115 116
    "disk_reservation_count" => {
      "description" => "The number of reservable disks"
    },
117 118 119 120 121 122 123
    "memcpu" => {
      "description" => "The amount of memory in MB per CPU"
    },
    "memcore" => {
      "description" => "The amount of memory in MB per CPU core"
    },
    "memnode" => {
124
      "description" => "The total amount of memory in MB of the node"
125 126 127 128 129 130 131 132 133
    },
    "gpu" => {
      "description" => "The type of GPU available"
    },
    "gpu_count" => {
      "description" => "The number of GPU cards available"
    },
    "wattmeter" => {
      "description" => "The type of wattmeter available"
134 135 136 137
    },
    "mic" => {
      "description" => "Intel many integrated core architecture support",
      "value_type" => "Boolean"
138 139 140 141 142
    }
  }

  #Group properties by categories
  @@categories = {
143
    "Job-related properties" => ["besteffort", "deploy", "production", "cluster_priority", "max_walltime"],
144
    "Hierarchy" => ["cluster", "cpu", "core", "host", "network_address", "ip", "switch"],
Lucas Nussbaum's avatar
Lucas Nussbaum committed
145
    "Hardware" => ["gpu", "gpu_count", "memnode", "memcore", "memcpu", "disktype", "disk_reservation_count", "myri_rate", "myri_count", "myri", "ib_rate", "ib_count", "ib", "opa", "opa_rate", "opa_count", "eth_rate", "eth_count", "cpufreq", "cputype", "cpucore", "cpuarch", "virtual", "mic"],
146 147 148 149
    "Miscellaneous" => ["wattmeter", "nodemodel"]
  }

  #Existing properties that won't be documented
150
  @@ignored_properties = ["maintenance", "state", "ip_virtual"]
151

Lucas Nussbaum's avatar
Lucas Nussbaum committed
152
  def get_nodes_properties(_site_uid, site)
153
    properties = {}
154 155
    site['clusters'].sort.to_h.each do |cluster_uid, cluster|
      cluster['nodes'].sort.to_h.each do |node_uid, node|
156
        begin
157
          properties[node_uid] = get_ref_node_properties_internal(cluster_uid, cluster, node_uid, node)
158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
        rescue MissingProperty => e
          puts "Error while processing node #{node_uid}: #{e}"
        end
      end
    end
    return properties
  end

  def get_value_type(prop, values)
    if (@@properties[prop]["value_type"])
      return @@properties[prop]["value_type"]
    end
    if (!values.empty?)
      case values[0].class.name
      when "Fixnum"
        return "Integer"
      when "TrueClass", "FalseClass"
        return "Boolean"
      end
    end
    "String"
  end

  def generate_content
182
    refapi = load_data_hierarchy
183 184 185 186 187 188 189 190
    #Properties generated from oar-properties generator
    props = {}
    G5K::SITES.each{ |site_uid|
      props[site_uid] = get_nodes_properties(site_uid, refapi["sites"][site_uid])
    }

    #Compiled properties used to generate page
    oar_properties = {}
191 192 193
    props.sort.to_h.each { |site, site_props|
      site_props.sort.to_h.each { |node_uid, node_props|
        node_props.sort.to_h.each { |property, value|
194 195 196 197 198 199 200
          next if @@ignored_properties.include?(property)

          oar_properties[property] ||= {}
          oar_properties[property]["values"] ||= []
          oar_properties[property]["values"] << value unless value.nil?
          oar_properties[property]["values"].uniq!
          oar_properties[property]["values"].sort!{ |a, b|
201
            (a && a.to_s || "") <=> (b && b.to_s || "")
202 203 204 205
          }
        }
      }
    }
206
    oar_properties.sort.to_h.each { |prop, prop_hash|
Lucas Nussbaum's avatar
Lucas Nussbaum committed
207
      prop_hash["values"].sort!
208 209 210 211 212 213
      if (prop_hash["values"].length > 20)
        #Limit possible values to 20 elements and mark the list as truncated
        prop_hash["values"].slice!(0...-20)
        prop_hash["values"].push("...")
      end
      @@properties[prop]["possible_values"] ||= prop_hash["values"].join(", ")
214 215
    }

216
    @generated_content = "{{Portal|User}}\nProperties on resources managed by OAR allow users to select them according to their experiment's characteristics." + MW::LINE_FEED
217 218
    @generated_content += MW::heading("OAR Properties", 1) + MW::LINE_FEED

219
    @@categories.sort.to_h.each { |cat, cat_properties|
220
      @generated_content += MW::heading(cat, 2) + MW::LINE_FEED
221
      cat_properties.sort.each{ |property|
222 223 224 225 226
        values = oar_properties[property]["values"] rescue []
        @generated_content += MW::heading(MW::code(property), 3) + MW::LINE_FEED
        @generated_content += @@properties[property]["description"] + MW::LINE_FEED
        value_type = get_value_type(property, values)
        @generated_content += MW::LIST_ITEM + " Value type: " + MW::code(value_type) + MW::LINE_FEED
227
        @generated_content += MW::LIST_ITEM + " Possible values: " + MW::code(@@properties[property]["possible_values"]) + MW::LINE_FEED
228 229
      }
    }
230 231
    @generated_content += MW.italic(MW.small(generated_date_string))
    @generated_content += MW::LINE_FEED
232 233 234
  end
end

Lucas Nussbaum's avatar
Lucas Nussbaum committed
235
if __FILE__ == $0
236
  generator = OarPropertiesGenerator.new("OAR_Properties")
Lucas Nussbaum's avatar
Lucas Nussbaum committed
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251

  options = WikiGenerator::parse_options
  if (options)
    ret = 2
    begin
      ret = generator.exec(options)
    rescue MediawikiApi::ApiError => e
      puts e, e.backtrace
      ret = 3
    rescue StandardError => e
      puts e, e.backtrace
      ret = 4
    ensure
      exit(ret)
    end
252
  end
253
end