Attention une mise à jour du service Gitlab va être effectuée le mardi 30 novembre entre 17h30 et 18h00. Cette mise à jour va générer une interruption du service dont nous ne maîtrisons pas complètement la durée mais qui ne devrait pas excéder quelques minutes. Cette mise à jour intermédiaire en version 14.0.12 nous permettra de rapidement pouvoir mettre à votre disposition une version plus récente.

Commit e87e6bbd authored by Jérémie Gaidamour's avatar Jérémie Gaidamour
Browse files

[dev] First implementation of expand_angle_brackets() that supports the <a-b> syntax.

Also:
* simplified the processing of the YAML file hierarchy
* implemented priorities between files, hash entries
* unit testing of expand_angle_brackets()
* added documentation
parent 678fa670
#!/usr/bin/ruby
require 'pp'
require 'pathname'
require 'yaml'
require 'json'
# Merge a and b. If a and b are hashes, they are recursively merged.
# - a and b might be strings or nil.
# - b values have the highest priority (if not nil).
def deep_merge_entries(a, b)
if b.is_a?(Hash)
a.is_a?(Hash) ? a.deep_merge(b) : b
else
b.nil? ? a : b
end
end
# Extend Hash with helper methods needed to convert input data files to ruby Hash
class ::Hash
# Recursively merge this Hash with another
def deep_merge(second)
# Recursively merge this Hash with another (ie. merge nested hash)
# Returns a new hash containing the contents of other_hash and the contents of hash. The value for entries with duplicate keys will be that of other_hash:
# a = {"key": "value_a"}
# b = {"key": "value_b"}
# pp a.deep_merge(b) => {:key=>"value_b"}
# pp b.deep_merge(a) => {:key=>"value_a"}
def deep_merge(other_hash)
merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
self.merge(second, &merger)
self.merge(other_hash, &merger)
end
# Merge keys that match "PREFIX-<a-b>" with others keys that begins by
# "PREFIX-" and that ends with x, where a<=x<=b
# a and/or b may be ommited, meaning that there are no lower and/or upper bound for x
# This is done recursively (for this Hash and every Hashes it contains)
# i.e.:
# {"foo-1": {a: 0}, "foo-2": {a: 0}, "foo-3": {a: 0}, "foo-<2->": {b: 1}}.kbracket_merge()
# -> {"foo-2": {a: 0, b:1}, "foo-3": {a: 0, b: 0}}
# TODO: only "<->" suffix is currently implemented
def kbracket_merge()
self.each { |k_all, v_all|
if k_all.to_s.end_with?('-<->') and v_all.is_a?(Hash)
key_prefix_to_merge = k_all.to_s.sub(/-<->$/,'')
self.each { |k, v|
if k.to_s.start_with?(key_prefix_to_merge)
self[k] = v_all.deep_merge(v)
end
# Merge keys that match "PREFIX-<a-b>" with others keys that begins by "PREFIX-"
# and that ends with x, where a<=x<=b.
# - This is done recursively (for this Hash and every Hashes it may contain).
# - PREFIX-<a-b> values have lower priority on existing PREFIX-x keys.
# - "a" and/or "b" may be omited (ie. "PREFIX-<a->", "PREFIX-<-b>" or "PREFIX-<->"), meaning that there are no lower and/or upper bound for x.
# * If only a is omited, a == 1.
# * If b is omited, only existing keys are modified (no keys are created). Otherwise, PREFIX-<a> to PREFIX-<b> entries are created (if missing).
# Example:
# {"foo-1": {a: 0}, "foo-2": {a: 0}, "foo-3": {a: 0}, "foo-<2->": {b: 1}}.expand_angle_brackets()
# -> {"foo-1": {a: 0}, "foo-2": {a: 0, b:1}, "foo-3": {a: 0, b: 0}}
def expand_angle_brackets()
dup = self.clone # because can't add a new key into hash during iteration
# Looking up for PREFIX-<a-b> keys
dup.each { |key_ab, value_ab|
prefix, a, b = key_ab.to_s.scan(/^(.*)-<(\d*)-(\d*)>$/).first
next if not a and not b # not found
a != "" ? a = a.to_i : a = 1
b != "" ? b = b.to_i : b
if b != ""
# Merge keys, creating missing entries if needed.
(a..b).each { |x|
key = "#{prefix}-#{x}"
key = key.to_sym if key_ab.is_a?(Symbol)
# For duplicate entries, the value of PREFIX-x is kept.
self[key] = deep_merge_entries(value_ab, self[key])
}
else
# Modify only existing keys. Looking up for PREFIX-x keys.
dup.each { |key_x, value_x|
next if key_x.class != key_ab.class
x = key_x.to_s.scan(/^#{prefix}-(\d*)$/).first
x = x.first if x
next if not x or x.to_i < a
# For duplicate entries, the value of PREFIX-x is kept.
self[key_x] = deep_merge_entries(value_ab, value_x)
}
self.delete(k_all)
end
# Delete entry "PREFIX-<a-b>"
self.delete(key_ab)
}
self.each { |k, v|
if v.is_a?(Hash)
v.kbracket_merge()
# Do it recursivly
self.each { |key, value|
if value.is_a?(Hash)
value.expand_angle_brackets()
end
}
end
......@@ -47,21 +93,37 @@ class ::Hash
end
end
#TODO: Ensure that deepest elements in input files hierarchy have lowest priority
data = Hash.new
global_hash = {} # the global data structure
Dir.chdir("../input/grid5000/")
Dir.chdir("input")
Dir['**/*.y*ml'].each { |f|
node_path = Pathname.new(f)
node_dir, _ = node_path.split()
# Recursively list the .yaml files.
# The order in which the results are returned depends on the system (http://ruby-doc.org/core-2.2.3/Dir.html).
# => List deepest files first as they have lowest priority when hash keys are duplicated.
list_of_yaml_files = Dir['**/*.y*ml'].sort_by { |x| -x.count('/') }
node_hierarchy = node_dir.to_s.split('/')
node_value = YAML::load_file(node_path)
list_of_yaml_files.each { |filename|
node_data = Hash.from_array(node_hierarchy,node_value)
data = data.deep_merge(node_data)
# Load YAML
file_hash = YAML::load_file(filename)
if not file_hash
puts "Error loading '#{filename}'"
next
end
# Expand the hash
file_hash.expand_angle_brackets()
# Inject the file content into the global_hash, at the right place
path_hierarchy = File.dirname(filename).split('/') # Split the file path (path relative to input/)
file_hash = Hash.from_array(path_hierarchy, file_hash) # Build the nested hash hierarchy according to the file path
global_hash = global_hash.deep_merge(file_hash) # Merge global_hash and file_hash. The value for entries with duplicate keys will be that of file_hash
}
data.kbracket_merge()
puts JSON.generate(data)
#pp global_hash
#pp data
#puts JSON.generate(data)
require_relative "input_loader"
require "test/unit"
require "hashdiff"
# Helper test routine. Test if the hashes are equal. Also do the test for symbolized keys.
def assert_equal_hash(hash1, hash2)
diff = HashDiff.diff(hash1, hash2)
assert_equal([], diff)
end
# Helper test routine. Test if the hashes are equal after having been expanded.
def assert_equal_expanded_hash(hash1, hash2)
expanded_hash1 = hash1.clone.expand_angle_brackets()
expanded_hash2 = hash2.clone.expand_angle_brackets()
assert_equal_hash(expanded_hash1, expanded_hash2)
end
class TestInputLoader < Test::Unit::TestCase
def test__deep_merge_entries
a, b = 1, 2
ha = {key: "value_a"}
hb = {key: "value_b"}
# Check
assert_equal_hash(deep_merge_entries(a, b), b) # numeric - numeric
assert_equal_hash(deep_merge_entries(a, hb), hb) # numeric - hash
assert_equal_hash(deep_merge_entries(ha, b), b) # hash - numeric
assert_equal_hash(deep_merge_entries(ha, hb), hb) # hash - hash
assert_equal_hash(deep_merge_entries(a, nil), a) # numeric - nil
assert_equal_hash(deep_merge_entries(nil, b), b) # nil - numeric
assert_equal_hash(deep_merge_entries(ha, nil), ha) # hash - nil
assert_equal_hash(deep_merge_entries(nil, hb), hb) # nil - hash
assert_equal_hash(deep_merge_entries(nil, nil), nil) # nil - nil
# Check recursivity
# At the last level
a = {"c-1": {"c-2": {"a-3": 0}}}
b = {"c-1": {"c-2": {"b-3": 0}}}
assert_equal_hash(deep_merge_entries(a, b), {"c-1": {"c-2": {"a-3": 0, "b-3": 0}}})
# At an intermediate level
a = {"c-1": {"a-2": {"a-3": 0}}}
b = {"c-1": {"b-2": {"b-3": 0}}}
assert_equal_hash(deep_merge_entries(a, b), {"c-1": {
"a-2": {"a-3": 0},
"b-2": {"b-3": 0}
}
})
end
# Test the example given in documentation
def test__expand_angle_brackets__doc_example
hash = {
"foo-1": {a: 0},
"foo-2": {a: 0},
"foo-3": {a: 0},
"foo-<2->": {b: 1}
}
expected_expanded_hash = {
"foo-1": {a: 0},
"foo-2": {a: 0, b: 1},
"foo-3": {a: 0, b: 1}
}
assert_equal_expanded_hash(hash, expected_expanded_hash)
end
# The 'a' parameter
def test__expand_angle_brackets__a_values
assert_equal_expanded_hash({"foo-<-3>": 0}, {"foo-1": 0, "foo-2": 0, "foo-3": 0}) # Default 'a' value is 1
assert_equal_expanded_hash({"foo-<2-3>": 0}, {"foo-2": 0, "foo-3": 0}) # Simply check if the value of 'a' is taken into account
end
def test__expand_angle_brackets__create_keys
#
# If 'b' is given, create missing keys. Also, the keys must be of the same type (Symbol or String).
#
# With symbol keys and numeric values
assert_equal_expanded_hash({"foo-<2-3>": 0}, {"foo-2": 0, "foo-3": 0})
assert_equal_expanded_hash({"foo-<-2>": 0}, {"foo-1": 0, "foo-2": 0})
# With symbol keys and hash values
assert_equal_expanded_hash({"foo-<2-3>": {a: 0}}, {"foo-2": {a: 0}, "foo-3": {a: 0}})
assert_equal_expanded_hash({"foo-<-2>": {a: 0}}, {"foo-1": {a: 0}, "foo-2": {a: 0}})
# With string keys and numeric values
assert_equal_expanded_hash({"foo-<2-3>" => 0}, {"foo-2" => 0, "foo-3" => 0})
assert_equal_expanded_hash({"foo-<-2>" => 0}, {"foo-1" => 0, "foo-2" => 0})
# With string keys and hash values
assert_equal_expanded_hash({"foo-<2-3>" => {a: 0}}, {"foo-2" => {a: 0}, "foo-3" => {a: 0}})
assert_equal_expanded_hash({"foo-<-2>" => {a: 0}}, {"foo-1" => {a: 0}, "foo-2" => {a: 0}})
#
# If 'b' is not given, do not create any new key
#
assert_equal_expanded_hash({"foo-<->": 0}, {}) # All
assert_equal_expanded_hash({"foo-<2->": 0}, {})
end
def test__expand_angle_brackets__keep_existing_keys
#
# Do not modify existing keys
#
[0, {h: 0}].each { |v|
assert_equal_expanded_hash({"foo-<1-3>": v, "foo-2": 1}, {"foo-1": v, "foo-2": 1, "foo-3": v}) # b given
assert_equal_expanded_hash({"foo-<2->": v, "foo-2": 1}, {"foo-2": 1}) # b not given
}
end
# Some tests with nil values (=> nil values are created, existing nil values are overriden)
def test__expand_angle_brackets__nil
assert_equal_expanded_hash({"foo-<2-3>": nil}, {"foo-2": nil, "foo-3": nil}) # PREFIX-<a-b> is nil
assert_equal_expanded_hash({"foo-<2-3>": 0, "foo-2": nil}, {"foo-2": 0, "foo-3": 0}) # PREFIX-x is nil
assert_equal_expanded_hash({"foo-<2->": nil, "foo-3": 0}, {"foo-3": 0}) # PREFIX-<a-> is nil
assert_equal_expanded_hash({"foo-<2->": 0, "foo-3": nil}, {"foo-3": 0}) # PREFIX-x is nil
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment