From 9a11923bc4b3acacee06ed72df3ef4940ca4886c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Gaidamour?= <jeremie.gaidamour@inria.fr>
Date: Thu, 31 Mar 2016 17:35:18 +0200
Subject: [PATCH] [dev] Documentation
---
dev/README.md | 147 ++++++++++++++++++
dev/input/grid5000/sites/lyon/pdus.yaml | 18 ---
.../nancy/clusters/grigri/grigri.yaml.erb | 10 --
dev/input/grid5000/sites/nancy/retired.yaml | 7 -
dev/input/grid5000/sites/reims/pdus.yaml | 21 ---
dev/input/grid5000/sites/rennes/pdus.yaml | 82 ----------
dev/oar-properties/lib/lib-oar-properties.rb | 2 +-
dev/reference-api/reference-api.rb | 6 +-
dev/run-g5kchecks/postprocessing.rb | 6 +-
dev/run-g5kchecks/run-g5kchecks.rb | 10 ++
10 files changed, 165 insertions(+), 144 deletions(-)
create mode 100644 dev/README.md
delete mode 100644 dev/input/grid5000/sites/lyon/pdus.yaml
delete mode 100644 dev/input/grid5000/sites/nancy/clusters/grigri/grigri.yaml.erb
delete mode 100644 dev/input/grid5000/sites/nancy/retired.yaml
delete mode 100644 dev/input/grid5000/sites/reims/pdus.yaml
delete mode 100644 dev/input/grid5000/sites/rennes/pdus.yaml
diff --git a/dev/README.md b/dev/README.md
new file mode 100644
index 00000000000..8f458758bd2
--- /dev/null
+++ b/dev/README.md
@@ -0,0 +1,147 @@
+Reference repository generators
+===============================
+
+This directory contains the input files and scripts needed for generating:
+
+- The Reference API ie. the JSON files that are served by the Grid'5000 API for describing nodes, network equipments, topology ...
+
+ Ex: $ curl -k https://api.grid5000.fr/sid/sites/nancy/clusters/graoully/nodes/graoully-1.json?pretty
+ See also: https://www.grid5000.fr/mediawiki/index.php/API_all_in_one_Tutorial
+
+- The OAR properties ie. the node information that is registered in OAR databases and
+ allows user to select resources matching their experiment requirements.
+
+ Ex: $ oarsub -p "wattmeter=’YES’ and gpu=’YES’ and eth10g=’Y’"
+
+- ...
+
+See also: https://www.grid5000.fr/mediawiki/index.php/Reference_Repository (parts of this page is outdated).
+
+General Design
+--------------
+
+For the general design discussion, see:
+* https://www.grid5000.fr/mediawiki/index.php/CT-114_DesignRepoAPI (25/10/2015)
+* [CT-grid5000] Proposition de réorganisation des outils autour de la ref API (16/10/2015)
+* git clone git@gitolite.g5kadmin:slide -> 2016-03-29-IJD-Seminar-JG/
+
+Requirements
+------------
+Ruby 2.2 (+ HashDiff and Net/SSH for the oar-generator)
+
+Example using RVM (https://rvm.io/):
+$ \curl -sSL https://get.rvm.io | bash -s stable --ruby
+$ source ~/.rvm/scripts/rvm
+$ rvm install 2.2
+$ gem install hashdiff
+$ gem install net-ssh
+
+Input files
+-----------
+
+Input files are stored in the input/ directory.
+
+A ruby script loads the YAML files in a global hashtable. The file paths are used as entry points to the hashtable.
+# sites/nancy/clusters/pomme/pomme.yaml.erb
+nodes:
+ pomme-1:
+ ib0:
+ ip: 172.18.70.11
+
+becomes:
+
+hash = {"sites"=>
+ {"nancy"=>
+ {"clusters"=>
+ {"pomme"=>
+ {"nodes"=>
+ {"pomme-1"=>
+ {"ib0"=> {"ip"=> "172.18.70.11"} }
+ } } } } } }
+
+The input file loader (lib/input_loader.rb) also supports coalesced key names:
+
+pomme-[1-4]:
+ performance:
+ core_flops: 3929000000
+ node_flops: 7440000000
+
+and for more complex use case, YAML can be generated with ERB:
+
+# input/sites/nancy/clusters/pomme.yaml.erb
+nodes:
+ <% (1..16).each { |i| %>
+ pomme-<%= i %>:
+ ib0:
+ ip: 172.18.70.<%= i + 10 %>
+ <% } %>
+
+Updating the node input files using g5k-checks
+----------------------------------------------
+
+Node files (input/grid5000/sites/*/clusters/*/nodes/*.yaml) are generated by g5k-checks and should not be edited manually.
+- You can use run-g5kchecks/run-g5kchecks.rb for updating those files.
+- Node files of input/ are slightly edited compared to the raw output of `g5k-checks -m api`.
+ => Use run-g5kchecks/postprocessing.rb for applying the mandatory modifications to the g5k-checks ouput.
+
+See also:
+* https://www.grid5000.fr/mediawiki/index.php/G5k-checks
+* https://github.com/grid5000/g5k-checks/
+
+Reference API Generator
+-----------------------
+
+The Reference API generator reads the input/ YAML files and generates the data/ JSON files.
+
+Usage: cd reference-api/; ruby reference-api.rb
+
+Update OAR properties
+---------------------
+
+The generator can show the differences between the reference-repo and the OAR servers databases:
+
+workstation$ ruby oar-properties.rb -d -n graphene-105 -vv
+
+Output format: [’~’, ’key’, ’old value’, ’new value’]
+ graphene-105:
+ ["~", "disktype", "sata", "SATA II"]
+ ["~", "eth10g", nil, false]
+ ["~", "ib10g", false, true]
+ ["~", "ib10gmodel", "none", "MT26418"]
+ ["~", "ib40g", nil, false]
+ ["~", "ib40gmodel", nil, "none"]
+
+You can run the script per site, cluster or node using the -s, -c and -n options. There are several verbose levels (-v, -vv, -vvv).
+After reviewing the changes, you can update the configuration of the OAR server:
+
+workstation$$ ruby oar-properties.rb -d -n graphene-105 -e
+
+The previous command will *execute* the oarnodesetting/oar_resources_add/oarproperty commands needed for updating the OAR database via SSH. For example, it will execute:
+g5kadmin@oar.nancy$ oarnodesetting -h graphene-104 -p disktype='SATA II' -p eth10g='NO' -p ib10gmodel='MT26418'
+
+If you do not want to run the script directly on the OAR server, you can also print the commands out by using the -o (--output) option in place of -e (--execute).
+
+With the -d (--diff) option, the script executes 'oarnodes -Y' on the OAR servers for retrieving the OAR configurations. If you remove this option, the generator
+simply creates a script for setting up every node properties (even those that might already be set correctly). There is a safeguard for not adding a node twice to an OAR database.
+You can also use cache files (-d oarnodes-%s.yaml) if you do not want to retrieve the OAR configuration each time you run the generator.
+
+See ruby oar-properties.rb --help for more information.
+
+If you need to add new OAR properties, see `get_node_properties` and `diff_node_properties` in oar-properties/lib/lib-oar-properties.rb.
+This script can be tested with the oar-vagrant box (https://github.com/oar-team/oar-vagrant).
+
+See also:
+* https://www.grid5000.fr/mediawiki/index.php/OAR_properties
+* https://www.grid5000.fr/mediawiki/index.php/OAR_properties_2.0
+
+How to add a new cluster
+------------------------
+Steps:
+- Manually get the list of node MAC adresses (QR code scan)
+- DHCP, Kadeploy, Conman, Lanpower configuration
+- First boot!
+- Retrieve hardware information using g5kcheck and add them to the reference repository
+- Add information not provided by g5kcheck (ex: PDU)
+- Boot and check nodes with g5k-checks
+
+See sites/nancy/clusters/graoully/graoully_ip.yaml.erb as an example for bootstrapping a cluster configuration.
diff --git a/dev/input/grid5000/sites/lyon/pdus.yaml b/dev/input/grid5000/sites/lyon/pdus.yaml
deleted file mode 100644
index 76bf1fe5df6..00000000000
--- a/dev/input/grid5000/sites/lyon/pdus.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-# sitelyon do |site_uid|
-# pdu wattmetre do |pdu_uid|
-# vendor OmegaWatt
-# model
-# sensors [
-# {
-# power: {
-# per_outlets: true
-# wattmetre: {
-# available: true
-# www: {url: http://wattmetre.lyon.grid5000.fr/GetWatts-json.php
-# unit: W
-
-
-
-# ]
-# end
-# end
diff --git a/dev/input/grid5000/sites/nancy/clusters/grigri/grigri.yaml.erb b/dev/input/grid5000/sites/nancy/clusters/grigri/grigri.yaml.erb
deleted file mode 100644
index 2cec932adf6..00000000000
--- a/dev/input/grid5000/sites/nancy/clusters/grigri/grigri.yaml.erb
+++ /dev/null
@@ -1,10 +0,0 @@
-# nodes:
-# grigri
-# network_adapters:
-# eth0:
-# mounted: true
-# ip:
-# mac: 24:6E:96:02:A7:B8
-# bmc:
-# ip:
-# mac: 64:00:6A:C0:F0:8C
diff --git a/dev/input/grid5000/sites/nancy/retired.yaml b/dev/input/grid5000/sites/nancy/retired.yaml
deleted file mode 100644
index 4f36641fee1..00000000000
--- a/dev/input/grid5000/sites/nancy/retired.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-# ---
-# # Retired nodes
-# clusters:
-# mbi:
-# nodes:
-# mbi-[1-32]:
-# status: retired
diff --git a/dev/input/grid5000/sites/reims/pdus.yaml b/dev/input/grid5000/sites/reims/pdus.yaml
deleted file mode 100644
index 46ff8cf8360..00000000000
--- a/dev/input/grid5000/sites/reims/pdus.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-# sitereims do |site_uid|
-# 3.times do |i|
-# pdu stremi-pdu#{i+1 do |pdu_uid|
-# vendor Raritan
-# model
-# sensors [
-# {
-# power: {
-# per_outlets: true
-# snmp: {
-# available: true
-# total_oids: iso.3.6.1.4.1.13742.4.1.3.1.3.0
-# unit: W
-# outlet_prefix_oid: iso.3.6.1.4.1.13742.4.1.2.2.1.7
-
-
-
-# ]
-# end
-# end
-# end
diff --git a/dev/input/grid5000/sites/rennes/pdus.yaml b/dev/input/grid5000/sites/rennes/pdus.yaml
deleted file mode 100644
index 4bce3c2c83f..00000000000
--- a/dev/input/grid5000/sites/rennes/pdus.yaml
+++ /dev/null
@@ -1,82 +0,0 @@
-# siterennes do |site_uid|
-
-# 4.times do |i|
-# pdu parapide-pdu-#{i+1 do |pdu_uid|
-# vendor APC
-# model AP7851
-# sensors [
-# {
-# power: {
-# per_outlets: false
-# snmp: {
-# available: true
-# total_oids: [iso.3.6.1.4.1.318.1.1.12.1.16.0]
-# unit: W
-
-
-
-# ]
-# end
-# end
-
-# 6.times do |i|
-# pdu parapluie-pdu-#{i+1 do |pdu_uid|
-# vendor Eaton Corporation
-# model
-# sensors [
-# {
-# power: {
-# per_outlets: true
-# snmp: {
-# available: true
-# total_oids: [iso.3.6.1.4.1.534.6.6.7.5.5.1.3.0.1 iso.3.6.1.4.1.534.6.6.7.5.5.1.3.0.2]
-# unit: W
-# outlet_prefix_oid: iso.3.6.1.4.1.534.6.6.7.6.5.1.3.0
-
-
-
-# ]
-# end
-# end
-
-# 8.times do |i|
-# pdu paravance-pdu-#{i+1 do |pdu_uid|
-# vendor APC
-# model AP8659
-# sensors [
-# {
-# power: {
-# per_outlets: true
-# snmp: {
-# available: true
-# total_oids: [iso.3.6.1.4.1.318.1.1.12.1.16.0]
-# unit: W
-# outlet_prefix_oid: iso.3.6.1.4.1.318.1.1.26.9.4.3.1.7
-
-
-
-# ]
-# end
-# end
-
-# 4.times do |i|
-# pdu parasilo-pdu-#{i+1 do |pdu_uid|
-# vendor APC
-# model AP8659
-# sensors [
-# {
-# power: {
-# per_outlets: true
-# snmp: {
-# available: true
-# total_oids: [iso.3.6.1.4.1.318.1.1.12.1.16.0]
-# unit: W
-# outlet_prefix_oid: iso.3.6.1.4.1.318.1.1.26.9.4.3.1.7
-
-
-
-# ]
-# end
-# end
-
-# end
diff --git a/dev/oar-properties/lib/lib-oar-properties.rb b/dev/oar-properties/lib/lib-oar-properties.rb
index 23bfb355ebe..55fe328f658 100755
--- a/dev/oar-properties/lib/lib-oar-properties.rb
+++ b/dev/oar-properties/lib/lib-oar-properties.rb
@@ -44,7 +44,7 @@ def get_node_properties(cluster_uid, cluster, node_uid, node)
h['cpucore'] = node['architecture']['smt_size']/node['architecture']['smp_size']
h['cputype'] = [node['processor']['model'], node['processor']['version']].join(' ')
h['cpufreq'] = node['processor']['clock_speed']/1_000_000_000.0
- h['disktype'] = (node['block_devices'].first[1] || {})['interface']
+ h['disktype'] = (node['storage_devices'].first[1] || {})['interface']
h['ethnb'] = node['network_adapters'].values.select{|na| na['interface'] =~ /ethernet/i}.select{|nb| nb['mounted'] == true || nb['mountable'] == true}.length
eth10g = node['network_adapters'].values.select{|na| na['interface'] =~ /ethernet/i}.select{|nb| nb['mounted'] == true || nb['mountable'] == true}
diff --git a/dev/reference-api/reference-api.rb b/dev/reference-api/reference-api.rb
index 4f4a8a3d4db..73363364aad 100644
--- a/dev/reference-api/reference-api.rb
+++ b/dev/reference-api/reference-api.rb
@@ -45,10 +45,10 @@ def net_switch_port_lookup(site, node_uid, interface='')
end
global_hash["sites"].each do |site_uid, site|
- pp site_uid
+ puts site_uid
site["clusters"].each do |cluster_uid, cluster|
- pp cluster_uid
+ puts " #{cluster_uid}"
cluster_path = Pathname.new(refapi_path).join("sites",site_uid,"clusters",cluster_uid)
cluster_path.join("nodes").mkpath()
@@ -66,7 +66,7 @@ global_hash["sites"].each do |site_uid, site|
# Write node info
cluster["nodes"].each do |node_uid, node|# _sort_by_node_uid
- pp node_uid
+ #puts node_uid
#pp node if node_uid == "graoully-1"
#next unless node_uid == "griffon-1"
diff --git a/dev/run-g5kchecks/postprocessing.rb b/dev/run-g5kchecks/postprocessing.rb
index 370929ed599..60d724703a2 100644
--- a/dev/run-g5kchecks/postprocessing.rb
+++ b/dev/run-g5kchecks/postprocessing.rb
@@ -1,5 +1,9 @@
#!/usr/bin/ruby
+# This script does minor edits on the ouput of g5k-checks YAML files and moves the files to the right place in the input/ directory
+#
+# Usage: cd run-g5kcheck; ruby run-g5kchecks.rb; ruby postprocessing.rb
+
require 'pp'
require 'erb'
require 'fileutils'
@@ -7,8 +11,6 @@ require 'pathname'
require 'yaml'
require '../lib/hash/hash'
-####
-
list_of_yaml_files = Dir['output/*.y*ml'].sort_by { |x| -x.count('/') }
list_of_yaml_files.each { |filename|
file = filename.split("/")[1]
diff --git a/dev/run-g5kchecks/run-g5kchecks.rb b/dev/run-g5kchecks/run-g5kchecks.rb
index a198f336dfe..a15ff3d664f 100644
--- a/dev/run-g5kchecks/run-g5kchecks.rb
+++ b/dev/run-g5kchecks/run-g5kchecks.rb
@@ -1,3 +1,13 @@
+# This script reserves nodes and then runs g5k-check as root using the g5kadmin credentials.
+#
+# Usage: cd run-g5kcheck; ruby run-g5kchecks.rb; ruby postprocessing.rb
+#
+# - You can edit the node reservation at the beginning of the script (or create reservation manually).
+# - The script will run g5k-checks on every nodes that have been reserved.
+# - Output YAML files of g5k-checks are stored in output/
+# - If an output YAML file already exist in ouput/, the execution of g5k-check on the corresponding node is skipped.
+# - Use postprocessing.rb for moving the file in th input/ directory. This script also edits some keys of the YAML files.
+
require 'cute'
require 'peach'
require 'fileutils'
--
GitLab