From cc506fcab0bd56c515484b37c0a300f14f6f55f2 Mon Sep 17 00:00:00 2001
From: Lucas Nussbaum <lucas.nussbaum@loria.fr>
Date: Fri, 22 Sep 2023 18:03:10 +0200
Subject: [PATCH] [gen:oar-prop] those cases no longer fail and are handled
 transparently

---
 ...d-OAR-updated-gpu-re-added_diff_stdout.txt |   1 -
 ...-OAR-updated-gpu-re-added_print_stdout.txt |   1 -
 ...-OAR-updated-gpu-re-added_table_stdout.txt |   1 -
 ...with_too_few_oar_resources_diff_stdout.txt |  22 ++-
 ...ith_too_few_oar_resources_print_stdout.txt | 142 +++++++++++++++++-
 ...ith_too_few_oar_resources_table_stdout.txt |  38 ++++-
 6 files changed, 196 insertions(+), 9 deletions(-)

diff --git a/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_diff_stdout.txt b/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_diff_stdout.txt
index 52715b941d..bbf7aa3d08 100644
--- a/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_diff_stdout.txt
+++ b/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_diff_stdout.txt
@@ -1,4 +1,3 @@
-GPU has an unexpected number of resources (current:51 vs expected:52).
 Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
   clustera-1:
     ["~", "gpu_model", "RTX 2080 Ti", "GeForce RTX 2080 Ti"]
diff --git a/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_print_stdout.txt b/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_print_stdout.txt
index e79a6d915a..2434d82fc3 100644
--- a/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_print_stdout.txt
+++ b/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_print_stdout.txt
@@ -1,4 +1,3 @@
-GPU has an unexpected number of resources (current:51 vs expected:52).
 
 #! /usr/bin/env bash
 
diff --git a/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_table_stdout.txt b/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_table_stdout.txt
index 2da72708a8..5b7e29d640 100644
--- a/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_table_stdout.txt
+++ b/spec/output/graffiti-gpu-removed-OAR-updated-gpu-re-added_table_stdout.txt
@@ -1,4 +1,3 @@
-GPU has an unexpected number of resources (current:51 vs expected:52).
 +---------- + -------------------- + ----- + ----- + -------- + ---- + -------------------- + ------------------------------ + ------------------------------+
 |   cluster | host                 | cpu   | core  | cpuset   | gpu  | gpudevice            | cpumodel                       | gpumodel                      |
 +---------- + -------------------- + ----- + ----- + -------- + ---- + -------------------- + ------------------------------ + ------------------------------+
diff --git a/spec/output/with_too_few_oar_resources_diff_stdout.txt b/spec/output/with_too_few_oar_resources_diff_stdout.txt
index 6001108a83..204e2f18bf 100644
--- a/spec/output/with_too_few_oar_resources_diff_stdout.txt
+++ b/spec/output/with_too_few_oar_resources_diff_stdout.txt
@@ -1,2 +1,20 @@
-CORE has an unexpected number of resources (current:31 vs expected:32).
-unexpected number (current:31 vs expected:32) of resources for cluster clustera
+Output format: [ '-', 'key', 'value'] for missing, [ '+', 'key', 'value'] for added, ['~', 'key', 'old value', 'new value'] for changed
+  clustera-1:
+    ["~", "deploy", "NO", "YES"]
+    ["~", "wattmeter", "MULTIPLE", "NO"]
+    ["+", "chassis", "Dell Inc. PowerEdge T640 FL1CBX2"]
+    ["+", "core_count", 16]
+    ["+", "cpu_count", 2]
+    ["+", "eth_kavlan_count", 0]
+    ["+", "exotic", "NO"]
+    ["+", "thread_count", 32]
+  clustera-2:
+    ["~", "deploy", "NO", "YES"]
+    ["~", "wattmeter", "MULTIPLE", "NO"]
+    ["+", "chassis", "Dell Inc. PowerEdge T640 9L1CBX2"]
+    ["+", "core_count", 16]
+    ["+", "cpu_count", 2]
+    ["+", "eth_kavlan_count", 0]
+    ["+", "exotic", "NO"]
+    ["+", "thread_count", 32]
+Properties that need to be created on the fakesite server: cpu_count, core_count, thread_count, chassis, eth_kavlan_count, exotic
diff --git a/spec/output/with_too_few_oar_resources_print_stdout.txt b/spec/output/with_too_few_oar_resources_print_stdout.txt
index 6001108a83..7bae3e4584 100644
--- a/spec/output/with_too_few_oar_resources_print_stdout.txt
+++ b/spec/output/with_too_few_oar_resources_print_stdout.txt
@@ -1,2 +1,140 @@
-CORE has an unexpected number of resources (current:31 vs expected:32).
-unexpected number (current:31 vs expected:32) of resources for cluster clustera
+
+#! /usr/bin/env bash
+
+set -eu
+set -x
+set -o pipefail
+
+echo '================================================================================'
+
+property_exist () {
+  [[ $(oarproperty -l | grep -e "^$1$") ]]
+}
+
+node_exist () {
+  [[ $(oarnodes --sql "host='$1' and type='default'") ]]
+}
+
+disk_exist () {
+  [[ $(oarnodes --sql "host='$1' and type='disk' and disk='$2'") ]]
+}
+
+
+# if [ $(oarnodes -Y | grep " cpu:" | awk '{print $2}' | sort -nr | wc -c) == "0" ]; then
+#   NEXT_AVAILABLE_CPU_ID=0
+# else
+#   MAX_CPU_ID=$(oarnodes -Y | grep " cpu:" | awk '{print $2}' | sort -nr | head -n1)
+#   let "NEXT_AVAILABLE_CPU_ID=MAX_CPU_ID+1"
+# fi
+#
+# if [ $(oarnodes -Y | grep " core:" | awk '{print $2}' | sort -nr | wc -c) == "0" ]; then
+#   NEXT_AVAILABLE_CORE_ID=0
+# else
+#   MAX_CORE_ID=$(oarnodes -Y | grep " core:" | awk '{print $2}' | sort -nr | head -n1)
+#   let "NEXT_AVAILABLE_CORE_ID=MAX_CORE_ID+1"
+# fi
+
+#############################################
+# Create OAR properties that were created by 'oar_resources_add'
+#############################################
+property_exist 'host' || oarproperty -a host --varchar
+property_exist 'cpu' || oarproperty -a cpu
+property_exist 'core' || oarproperty -a core
+property_exist 'gpudevice' || oarproperty -a gpudevice
+property_exist 'gpu' || oarproperty -a gpu
+property_exist 'disk' || oarproperty -a disk --varchar
+property_exist 'diskpath' || oarproperty -a diskpath --varchar
+
+
+#############################################
+# Create OAR properties if they don't exist
+#############################################
+
+property_exist 'ip' || oarproperty -a ip --varchar
+property_exist 'cluster' || oarproperty -a cluster --varchar
+property_exist 'nodemodel' || oarproperty -a nodemodel --varchar
+property_exist 'switch' || oarproperty -a switch --varchar
+property_exist 'virtual' || oarproperty -a virtual --varchar
+property_exist 'cpuarch' || oarproperty -a cpuarch --varchar
+property_exist 'cpucore' || oarproperty -a cpucore
+property_exist 'cpu_count' || oarproperty -a cpu_count
+property_exist 'core_count' || oarproperty -a core_count
+property_exist 'thread_count' || oarproperty -a thread_count
+property_exist 'cputype' || oarproperty -a cputype --varchar
+property_exist 'cpufreq' || oarproperty -a cpufreq --varchar
+property_exist 'disktype' || oarproperty -a disktype --varchar
+property_exist 'chassis' || oarproperty -a chassis --varchar
+property_exist 'eth_count' || oarproperty -a eth_count
+property_exist 'eth_kavlan_count' || oarproperty -a eth_kavlan_count
+property_exist 'eth_rate' || oarproperty -a eth_rate
+property_exist 'ib_count' || oarproperty -a ib_count
+property_exist 'ib_rate' || oarproperty -a ib_rate
+property_exist 'ib' || oarproperty -a ib --varchar
+property_exist 'opa_count' || oarproperty -a opa_count
+property_exist 'opa_rate' || oarproperty -a opa_rate
+property_exist 'myri_count' || oarproperty -a myri_count
+property_exist 'myri_rate' || oarproperty -a myri_rate
+property_exist 'myri' || oarproperty -a myri --varchar
+property_exist 'memcore' || oarproperty -a memcore
+property_exist 'memcpu' || oarproperty -a memcpu
+property_exist 'memnode' || oarproperty -a memnode
+property_exist 'gpu_model' || oarproperty -a gpu_model --varchar
+property_exist 'gpu_count' || oarproperty -a gpu_count
+property_exist 'exotic' || oarproperty -a exotic --varchar
+property_exist 'mic' || oarproperty -a mic --varchar
+property_exist 'wattmeter' || oarproperty -a wattmeter --varchar
+property_exist 'cluster_priority' || oarproperty -a cluster_priority
+property_exist 'max_walltime' || oarproperty -a max_walltime
+property_exist 'production' || oarproperty -a production --varchar
+property_exist 'maintenance' || oarproperty -a maintenance --varchar
+property_exist 'disk_reservation_count' || oarproperty -a disk_reservation_count
+
+
+###################################
+# clustera-1.fakesite.grid5000.fr
+###################################
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='1' AND type='default'" -p cpu=1 -p core=1 -p cpuset=0 -p gpu=1 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=0 # This GPU is mapped on /dev/nvidia0
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='2' AND type='default'" -p cpu=1 -p core=2 -p cpuset=1 -p gpu=1 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=0 # This GPU is mapped on /dev/nvidia0
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='3' AND type='default'" -p cpu=1 -p core=3 -p cpuset=2 -p gpu=1 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=0 # This GPU is mapped on /dev/nvidia0
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='4' AND type='default'" -p cpu=1 -p core=4 -p cpuset=3 -p gpu=1 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=0 # This GPU is mapped on /dev/nvidia0
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='5' AND type='default'" -p cpu=1 -p core=5 -p cpuset=4 -p gpu=2 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=1 # This GPU is mapped on /dev/nvidia1
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='6' AND type='default'" -p cpu=1 -p core=6 -p cpuset=5 -p gpu=2 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=1 # This GPU is mapped on /dev/nvidia1
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='7' AND type='default'" -p cpu=1 -p core=7 -p cpuset=6 -p gpu=2 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=1 # This GPU is mapped on /dev/nvidia1
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='8' AND type='default'" -p cpu=1 -p core=8 -p cpuset=7 -p gpu=2 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=1 # This GPU is mapped on /dev/nvidia1
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='9' AND type='default'" -p cpu=2 -p core=9 -p cpuset=8 -p gpu=3 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=2 # This GPU is mapped on /dev/nvidia2
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='10' AND type='default'" -p cpu=2 -p core=10 -p cpuset=9 -p gpu=3 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=2 # This GPU is mapped on /dev/nvidia2
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='11' AND type='default'" -p cpu=2 -p core=11 -p cpuset=10 -p gpu=3 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=2 # This GPU is mapped on /dev/nvidia2
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='12' AND type='default'" -p cpu=2 -p core=12 -p cpuset=11 -p gpu=3 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=2 # This GPU is mapped on /dev/nvidia2
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='13' AND type='default'" -p cpu=2 -p core=13 -p cpuset=12 -p gpu=4 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=3 # This GPU is mapped on /dev/nvidia3
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='14' AND type='default'" -p cpu=2 -p core=14 -p cpuset=13 -p gpu=4 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=3 # This GPU is mapped on /dev/nvidia3
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='15' AND type='default'" -p cpu=2 -p core=15 -p cpuset=14 -p gpu=4 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=3 # This GPU is mapped on /dev/nvidia3
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' AND resource_id='16' AND type='default'" -p cpu=2 -p core=16 -p cpuset=15 -p gpu=4 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=3 # This GPU is mapped on /dev/nvidia3
+echo; echo 'Setting properties for clustera-1.fakesite.grid5000.fr:'; echo
+oarnodesetting --sql "host='clustera-1.fakesite.grid5000.fr' and type='default'" -p ip='172.16.64.1' -p cluster='clustera' -p nodemodel='Dell PowerEdge T640' -p switch='gw-fakesite' -p besteffort='YES' -p deploy='YES' -p virtual='ivt' -p cpuarch='x86_64' -p cpucore=8 -p cpu_count=2 -p core_count=16 -p thread_count=32 -p cputype='Intel Xeon Silver 4110' -p cpufreq='2.1' -p disktype='SATA/SSD' -p chassis='Dell Inc. PowerEdge T640 FL1CBX2' -p eth_count=1 -p eth_kavlan_count=0 -p eth_rate=10 -p ib_count=0 -p ib_rate=0 -p ib='NO' -p opa_count=0 -p opa_rate=0 -p myri_count=0 -p myri_rate=0 -p myri='NO' -p memcore=8192 -p memcpu=65536 -p memnode=131072 -p gpu_model='GeForce RTX 2080 Ti' -p gpu_count=4 -p exotic='NO' -p mic='NO' -p wattmeter='NO' -p cluster_priority=201906 -p max_walltime=86400 -p production='YES' -p maintenance='NO' -p disk_reservation_count=0
+
+echo '================================================================================'
+
+
+###################################
+# clustera-2.fakesite.grid5000.fr
+###################################
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='17' AND type='default'" -p cpu=3 -p core=17 -p cpuset=0 -p gpu=5 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=0 # This GPU is mapped on /dev/nvidia0
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='18' AND type='default'" -p cpu=3 -p core=18 -p cpuset=1 -p gpu=5 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=0 # This GPU is mapped on /dev/nvidia0
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='19' AND type='default'" -p cpu=3 -p core=19 -p cpuset=2 -p gpu=5 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=0 # This GPU is mapped on /dev/nvidia0
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='20' AND type='default'" -p cpu=3 -p core=20 -p cpuset=3 -p gpu=5 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=0 # This GPU is mapped on /dev/nvidia0
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='21' AND type='default'" -p cpu=3 -p core=21 -p cpuset=4 -p gpu=6 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=1 # This GPU is mapped on /dev/nvidia1
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='22' AND type='default'" -p cpu=3 -p core=22 -p cpuset=5 -p gpu=6 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=1 # This GPU is mapped on /dev/nvidia1
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='23' AND type='default'" -p cpu=3 -p core=23 -p cpuset=6 -p gpu=6 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=1 # This GPU is mapped on /dev/nvidia1
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='24' AND type='default'" -p cpu=3 -p core=24 -p cpuset=7 -p gpu=6 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=1 # This GPU is mapped on /dev/nvidia1
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='25' AND type='default'" -p cpu=4 -p core=25 -p cpuset=8 -p gpu=7 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=2 # This GPU is mapped on /dev/nvidia2
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='26' AND type='default'" -p cpu=4 -p core=26 -p cpuset=9 -p gpu=7 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=2 # This GPU is mapped on /dev/nvidia2
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='27' AND type='default'" -p cpu=4 -p core=27 -p cpuset=10 -p gpu=7 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=2 # This GPU is mapped on /dev/nvidia2
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='28' AND type='default'" -p cpu=4 -p core=28 -p cpuset=11 -p gpu=7 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=2 # This GPU is mapped on /dev/nvidia2
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='29' AND type='default'" -p cpu=4 -p core=29 -p cpuset=12 -p gpu=8 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=3 # This GPU is mapped on /dev/nvidia3
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='30' AND type='default'" -p cpu=4 -p core=30 -p cpuset=13 -p gpu=8 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=3 # This GPU is mapped on /dev/nvidia3
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' AND resource_id='31' AND type='default'" -p cpu=4 -p core=31 -p cpuset=14 -p gpu=8 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=3 # This GPU is mapped on /dev/nvidia3
+oarnodesetting -a -h 'clustera-2.fakesite.grid5000.fr' -s Absent -p host='clustera-2.fakesite.grid5000.fr' -p cpu=4 -p core=32 -p cpuset=15 -p gpu=8 -p gpu_model='GeForce RTX 2080 Ti' -p gpudevice=3 # This GPU is mapped on /dev/nvidia3
+echo; echo 'Setting properties for clustera-2.fakesite.grid5000.fr:'; echo
+oarnodesetting --sql "host='clustera-2.fakesite.grid5000.fr' and type='default'" -p ip='172.16.64.2' -p cluster='clustera' -p nodemodel='Dell PowerEdge T640' -p switch='gw-fakesite' -p besteffort='YES' -p deploy='YES' -p virtual='ivt' -p cpuarch='x86_64' -p cpucore=8 -p cpu_count=2 -p core_count=16 -p thread_count=32 -p cputype='Intel Xeon Silver 4110' -p cpufreq='2.1' -p disktype='SATA/SSD' -p chassis='Dell Inc. PowerEdge T640 9L1CBX2' -p eth_count=1 -p eth_kavlan_count=0 -p eth_rate=10 -p ib_count=0 -p ib_rate=0 -p ib='NO' -p opa_count=0 -p opa_rate=0 -p myri_count=0 -p myri_rate=0 -p myri='NO' -p memcore=8192 -p memcpu=65536 -p memnode=131072 -p gpu_model='GeForce RTX 2080 Ti' -p gpu_count=4 -p exotic='NO' -p mic='NO' -p wattmeter='NO' -p cluster_priority=201906 -p max_walltime=86400 -p production='YES' -p maintenance='NO' -p disk_reservation_count=0
+
+echo '================================================================================'
diff --git a/spec/output/with_too_few_oar_resources_table_stdout.txt b/spec/output/with_too_few_oar_resources_table_stdout.txt
index 6001108a83..ff22c49a17 100644
--- a/spec/output/with_too_few_oar_resources_table_stdout.txt
+++ b/spec/output/with_too_few_oar_resources_table_stdout.txt
@@ -1,2 +1,36 @@
-CORE has an unexpected number of resources (current:31 vs expected:32).
-unexpected number (current:31 vs expected:32) of resources for cluster clustera
++---------- + -------------------- + ----- + ----- + -------- + ---- + -------------------- + ------------------------------ + ------------------------------+
+|   cluster | host                 | cpu   | core  | cpuset   | gpu  | gpudevice            | cpumodel                       | gpumodel                      |
++---------- + -------------------- + ----- + ----- + -------- + ---- + -------------------- + ------------------------------ + ------------------------------+
+|  clustera | clustera-1           | 1     | 1     | 0        | 1    | 0                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 1     | 2     | 1        | 1    | 0                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 1     | 3     | 2        | 1    | 0                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 1     | 4     | 3        | 1    | 0                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 1     | 5     | 4        | 2    | 1                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 1     | 6     | 5        | 2    | 1                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 1     | 7     | 6        | 2    | 1                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 1     | 8     | 7        | 2    | 1                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 2     | 9     | 8        | 3    | 2                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 2     | 10    | 9        | 3    | 2                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 2     | 11    | 10       | 3    | 2                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 2     | 12    | 11       | 3    | 2                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 2     | 13    | 12       | 4    | 3                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 2     | 14    | 13       | 4    | 3                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 2     | 15    | 14       | 4    | 3                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-1           | 2     | 16    | 15       | 4    | 3                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 3     | 17    | 0        | 5    | 0                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 3     | 18    | 1        | 5    | 0                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 3     | 19    | 2        | 5    | 0                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 3     | 20    | 3        | 5    | 0                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 3     | 21    | 4        | 6    | 1                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 3     | 22    | 5        | 6    | 1                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 3     | 23    | 6        | 6    | 1                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 3     | 24    | 7        | 6    | 1                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 4     | 25    | 8        | 7    | 2                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 4     | 26    | 9        | 7    | 2                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 4     | 27    | 10       | 7    | 2                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 4     | 28    | 11       | 7    | 2                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 4     | 29    | 12       | 8    | 3                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 4     | 30    | 13       | 8    | 3                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 4     | 31    | 14       | 8    | 3                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
+|  clustera | clustera-2           | 4     | 32    | 15       | 8    | 3                    | Intel Xeon Silver 4110         | GeForce RTX 2080 Ti           |
++---------- + -------------------- + ----- + ----- + -------- + ---- + -------------------- + ------------------------------ + ------------------------------+
-- 
GitLab