Merge branch 'disk_ignore_homogeneity' into 'master'

Disk ignore homogeneity

See merge request !299

Gemfile

@@ -15,3 +15,4 @@ gem "rspec", :group => [:development]
 gem "webmock", :group => [:development]
 gem "simplecov", :group => [:development]
 gem "code-explorer", :group => [:development]
+gem "byebug", "~> 11.1", :group => :development

Gemfile.lock

@@ -2,12 +2,18 @@ GEM
   remote: https://rubygems.org/
   specs:
     abstract_method (1.2.1)
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     addressable (2.6.0)
       public_suffix (>= 2.0.2, < 4.0)
     ast (2.4.0)
-    backports (3.15.0)
     bcrypt_pbkdf (1.0.1)
-    bond (0.4.3)
+    bond (0.5.1)
+    byebug (11.1.3)
     cheetah (0.5.0)
       abstract_method (~> 1.2)
     code-explorer (0.3.0)
@@ -16,6 +22,7 @@ GEM
       parser (~> 2.0)
       sinatra (~> 1)
     coderay (1.1.2)
+    concurrent-ruby (1.1.9)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     diff-lcs (1.3)
@@ -36,6 +43,8 @@ GEM
     hashdiff (1.0.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
     ipaddress (0.8.3)
     json (2.5.1)
     mediawiki_api (0.7.1)
@@ -43,6 +52,7 @@ GEM
       faraday-cookie_jar (~> 0.0, >= 0.0.6)
       faraday_middleware (~> 0.10, >= 0.10.0)
     mime-types (2.99.3)
+    minitest (5.14.4)
     multipart-post (2.1.1)
     net-ssh (4.2.0)
     net-ssh-gateway (2.0.0)
@@ -56,7 +66,7 @@ GEM
     peach (0.5.1)
     public_suffix (3.1.1)
     rack (1.6.13)
-    rack-cache (1.9.0)
+    rack-cache (1.13.0)
       rack (>= 0.4)
     rack-protection (1.5.5)
       rack
@@ -70,21 +80,24 @@ GEM
     rest-client-components (1.5.0)
       rack (>= 1.0.1)
       rest-client (>= 1.6.0)
-    restfully (1.2.0)
+    restfully (1.3.0)
+      activesupport
       addressable
-      backports
-      json (~> 1.5)
+      json
+      mime-types
+      public_suffix
+      rack
       rack-cache
       rb-readline
-      rest-client (~> 1.6)
+      rest-client
       rest-client-components
-      ripl (= 0.6.1)
+      ripl
       ripl-color_streams
       ripl-multi_line
-      ripl-play (~> 0.2.1)
+      ripl-play
       ripl-short_errors
-    ripl (0.6.1)
-      bond (~> 0.4.0)
+    ripl (0.7.1)
+      bond (~> 0.5.1)
     ripl-color_streams (0.1.2)
       ripl (>= 0.2.8)
     ripl-multi_line (0.3.1)
@@ -122,6 +135,8 @@ GEM
       rack-protection (~> 1.4)
       tilt (>= 1.3, < 3)
     tilt (2.0.10)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.6)
@@ -129,12 +144,14 @@ GEM
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bcrypt_pbkdf (~> 1.0)
+  byebug (~> 11.1)
   code-explorer
   diffy
   dns-zone

lib/refrepo/valid/data/homogeneity.yaml.erb

@@ -16,7 +16,36 @@
 #   griffon-[11-14]: additional myrinet interface
 #
 ---
+grenoble:
+  drac-[8-9]:
+    - ~storage_devices.pci-0009:04:00.0-ata-1.firmware_version
+  dahu-[7-8]:
+    - ~storage_devices.pci-0000:00:11.5-ata-5.firmware_version
+  yeti-[3-4]:
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:0:0.firmware_version
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:0:0.model
 lille:
+  chetemi-[10-11]:
+    - ~storage_devices.pci-0000:03:00.0-scsi-0:0:0:0.firmware_version
+    - ~storage_devices.pci-0000:03:00.0-scsi-0:0:0:0.model
+    - ~storage_devices.pci-0000:03:00.0-scsi-0:0:0:0.size
+  chifflot-2:
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:3:0.firmware_version
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:3:0.model
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:3:0.vendor
+  chifflot-[3-4]:
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:2:0.firmware_version
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:2:0.model
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:2:0.vendor
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:3:0.firmware_version
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:3:0.model
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:3:0.vendor
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:4:0.firmware_version
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:4:0.model
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:4:0.vendor
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:5:0.firmware_version
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:5:0.model
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:5:0.vendor
   chifflot-[6-7]:
     - ~gpu.gpu_model
     - ~gpu.gpu_cores
@@ -26,6 +55,18 @@ lille:
     - ~gpu_devices.nvidia1.vbios_version
     - ~gpu_devices.nvidia1.memory
     - ~gpu_devices.nvidia1.model
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:2:0.firmware_version
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:2:0.model
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:2:0.vendor
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:3:0.firmware_version
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:3:0.model
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:3:0.vendor
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:4:0.firmware_version
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:4:0.model
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:4:0.vendor
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:5:0.firmware_version
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:5:0.model
+    - ~storage_devices.pci-0000:18:00.0-scsi-0:0:5:0.vendor
 lyon:
   nova-[6-7]:
     - ~bios.release_date
@@ -73,6 +114,17 @@ lyon:
     - ~network_adapters.eth3.firmware_version
     - ~network_adapters.eth4.firmware_version
     - ~network_adapters.eth5.firmware_version
+  sagittaire-11:
+    - ~storage_devices.pci-0000:02:04.0-scsi-0:0:1:0.firmware_version
+    - ~storage_devices.pci-0000:02:04.0-scsi-0:0:1:0.model
+  sagittaire-13:
+    - ~storage_devices.pci-0000:02:04.0-scsi-0:0:1:0.firmware_version
+    - ~storage_devices.pci-0000:02:04.0-scsi-0:0:1:0.model
+  sagittaire-[17-18]:
+    - ~storage_devices.pci-0000:02:04.0-scsi-0:0:1:0.model
+    - ~storage_devices.pci-0000:02:04.0-scsi-0:0:1:0.size
+    - ~storage_devices.pci-0000:02:04.0-scsi-0:0:1:0.vendor
+    - ~storage_devices.pci-0000:02:04.0-scsi-0:0:1:0.firmware_version
 
 nancy:
   graffiti-[1-3]:
@@ -137,6 +189,34 @@ nancy:
     - ~network_adapters.eth2.name
     - ~network_adapters.eth3.name
     - +network_adapters.ib0
+  gros-[68-69]:
+    - ~storage_devices.pci-0000:17:00.0-scsi-0:0:1:0.vendor
+    - ~storage_devices.pci-0000:17:00.0-scsi-0:0:1:0.model
+    - ~storage_devices.pci-0000:17:00.0-scsi-0:0:1:0.firmware_version
+  grcinq-2:
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.vendor
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.model
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.firmware_version
+  grcinq-[5-6]:
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.vendor
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.model
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.firmware_version
+  grcinq-[8-9]:
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.vendor
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.model
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.firmware_version
+  grcinq-[18-19]:
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.vendor
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.model
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.firmware_version
+  grcinq-[30-31]:
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.vendor
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.model
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.firmware_version
+  grcinq-[46-47]:
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.vendor
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.model
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.firmware_version
 nantes:
   econome-[1-22]:
     - +network_adapters.eth2.rate
@@ -144,12 +224,21 @@ nantes:
   econome-19:
     - ~network_adapters.eth0.firmware_version
     - ~network_adapters.eth1.firmware_version
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.firmware_version
 rennes:
-  parasilo-[1-28]:
-    - ~storage_devices.sdf.model
-    - ~storage_devices.sdf.rev
-    - ~storage_devices.sd[a-e].by_id
-    - ~storage_devices.sd[a-e].by_path
+  paranoia-[3-5]:
+    - ~storage_devices.pci-0000:82:00.0-sas-phy0-lun-0.firmware_version
+    - ~storage_devices.pci-0000:82:00.0-sas-phy0-lun-0.model
+    - ~storage_devices.pci-0000:82:00.0-sas-phy4-lun-0.firmware_version
+    - ~storage_devices.pci-0000:82:00.0-sas-phy4-lun-0.model
+  parapide-[5-8]:
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.firmware_version
+  parapide-16:
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.firmware_version
+  parapide-18:
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.firmware_version
+  parapide-24:
+    - ~storage_devices.pci-0000:00:1f.2-ata-1.firmware_version
   parasilo-8:
     - -pdu
   parasilo-10:
@@ -168,20 +257,17 @@ rennes:
     - +pdu
   parasilo-22:
     - +pdu
-  paranoia-[1-8]:
-    - ~storage_devices.sd[b-e].model
-    - ~storage_devices.sd[b-e].rev
-    - ~storage_devices.sd[a-e].by_id
-    - ~storage_devices.sd[a-e].by_path
-  parapide-[1-25]:
-    - ~storage_devices.sd[a-e].by_id
-    - ~storage_devices.sd[a-e].by_path
-  parapluie-[1-40]:
-    - ~storage_devices.sd[a-e].by_id
-    - ~storage_devices.sd[a-e].by_path
+  parasilo-[6-7]:
+    - ~storage_devices.pci-0000:03:00.0-scsi-0:0:5:0.firmware_version
+    - ~storage_devices.pci-0000:03:00.0-scsi-0:0:5:0.model
+  parasilo-[13-14]:
+    - ~storage_devices.pci-0000:03:00.0-scsi-0:0:4:0.firmware_version
+    - ~storage_devices.pci-0000:03:00.0-scsi-0:0:4:0.model
+    - ~storage_devices.pci-0000:03:00.0-scsi-0:0:4:0.vendor
+  parasilo-[25-26]:
+    - ~storage_devices.pci-0000:03:00.0-scsi-0:0:2:0.firmware_version
+    - ~storage_devices.pci-0000:03:00.0-scsi-0:0:2:0.model
   paravance-[1-72]:
-    - ~storage_devices.sd[a-e].by_id
-    - ~storage_devices.sd[a-e].by_path
     - ~network_adapters.eth0.firmware_version
     - ~network_adapters.eth1.firmware_version
     - ~network_adapters.eth2.firmware_version

lib/refrepo/valid/homogeneity.rb

@@ -57,22 +57,9 @@ def global_ignore_keys
     ~network_adapters.eth.switch_port
 eos
 
-  ignore_stokeys = <<-eos
-    ~storage_devices.sd.model
-    ~storage_devices.sd.firmware_version
-    ~storage_devices.sd.rev
-    -storage_devices.sd.rev
-    ~storage_devices.sd.size
-    ~storage_devices.sd.timeread
-    ~storage_devices.sd.timewrite
-    ~storage_devices.sd.vendor
-    ~storage_devices.sd.by_id
-    ~storage_devices.sd.by_path
-eos
-
-  ignore_nvmekeys = <<-eos
-    ~storage_devices.nvme.by_id
-eos
+  ignore_regex = [
+    ['~', /storage_devices\..*\.by_id/],
+  ]
 
   (0..5).each { |eth|
     keys = ignore_netkeys.gsub('.eth.', ".eth#{eth}.").gsub("\n", " ").split(" ")
@@ -84,18 +71,6 @@ eos
     }
   }
 
-  ('a'..'f').each { |sd|
-    keys = ignore_stokeys.gsub('.sd.', ".sd#{sd}.").gsub("\n", " ").split(" ")
-    ignore_keys.push(* keys)
-  }
-
-  (0..2).each { |i|
-    (0..2).each { |j|
-      keys = ignore_nvmekeys.gsub('.nvme.', ".nvme#{i}n#{j}.").gsub("\n", " ").split(" ")
-      ignore_keys.push(* keys)
-    }
-  }
-
   ignore_ibkeys = <<-eos
     ~network_adapters.IB_IF.guid
     ~network_adapters.IB_IF.hwid
@@ -119,7 +94,7 @@ eos
     keys = ignore_ibkeys.gsub('IB_IF', "#{ib_if}").gsub("\n", " ").split(" ")
     ignore_keys.push(* keys)
   }
-  return ignore_keys
+  return [ignore_keys, ignore_regex]
 end
 
 def cluster_ignore_keys(filename)
@@ -139,7 +114,7 @@ def cluster_homogeneity(refapi_hash, options = {:verbose => false})
     puts ''
   end
 
-  ignore_keys  = global_ignore_keys()
+  ignore_keys, ignore_regex  = global_ignore_keys()
   cignore_keys = cluster_ignore_keys(File.expand_path("data/homogeneity.yaml.erb", File.dirname(__FILE__)))
 
   count = {}
@@ -193,10 +168,25 @@ def cluster_homogeneity(refapi_hash, options = {:verbose => false})
 
         # Remove keys that are specific to each nodes (ip, mac etc.)
         ikeys = cignore_keys[site_uid][node_uid] rescue nil
-        diffs.clone.each { |diff|
-          diffs.delete(diff) if ignore_keys.include?(diff[0] + diff[1])
-          diffs.delete(diff) if ikeys && ikeys.include?(diff[0] + diff[1])
-        }
+
+        diffs.clone.each do |diff|
+          if ignore_keys.include?(diff[0] + diff[1])
+            diffs.delete(diff)
+            next
+          end
+
+          if ikeys && ikeys.include?(diff[0] + diff[1])
+            diffs.delete(diff)
+            next
+          end
+
+          ignore_regex.each do |e|
+            if e[0] == diff[0] && e[1].match?(diff[1])
+              diffs.delete(diff)
+              break
+            end
+          end
+        end
 
         if verbose && !diffs.empty?
           puts "Differences between #{refnode_uid} and #{node_uid}:"