diff --git a/Rakefile b/Rakefile index 3325b2481cdb9f08872b685963bb86d69a8a2bcf..91f41906472b1b0f941e91dd4edb63610c24039b 100644 --- a/Rakefile +++ b/Rakefile @@ -205,9 +205,21 @@ namespace :gen do exit(ret) end + desc 'Generate accesses json' + task :accesses do + require 'refrepo/gen/accesses' + generate_accesses_ir(%i[json]) + end + + desc 'Generate accesses mode history' + task 'accesses-history' do + require 'refrepo/gen/accesses' + generate_accesses_ir(%i[json history]) + end + namespace :puppet do - all_puppet_tasks = [:bindg5k, :conmang5k, :dhcpg5k, :kadeployg5k, :lanpowerg5k, :kavlang5k, :kwollectg5k, :network_monitoring, :'refapi-subset', :oxidizedg5k, :'oarsub-simplifier-aliases', :accesses, :kavlanngg5k, :stitcherg5k, :clusters, :webfish] + all_puppet_tasks = [:bindg5k, :conmang5k, :dhcpg5k, :kadeployg5k, :lanpowerg5k, :kavlang5k, :kwollectg5k, :network_monitoring, :'refapi-subset', :oxidizedg5k, :'oarsub-simplifier-aliases', :kavlanngg5k, :stitcherg5k, :clusters, :webfish] all_puppet_tasks.each { |t| generated_desc = (t == :'refapi-subset') ? 'description' : 'configuration' diff --git a/lib/refrepo/accesses.rb b/lib/refrepo/accesses.rb index 45af893f59fefa05bab447e56e98fe1d7013ecb5..636d5d7f14741aa3b5a25c208771237f30db9ac4 100644 --- a/lib/refrepo/accesses.rb +++ b/lib/refrepo/accesses.rb @@ -3,18 +3,82 @@ require 'refrepo/data_loader' require 'git' -$group_of_gga = {} - -ALL_GGAS_AND_SITES = RefRepo::Utils.get_public_api('users/ggas_and_sites') -ALL_GGAS = ALL_GGAS_AND_SITES['ggas'] -ALL_SITES = ALL_GGAS_AND_SITES['sites'] +PRIOLEVEL = { + 'p1' => 40, + 'p2' => 30, + 'p3' => 20, + 'p4' => 10, + 'besteffort' => 0 +}.freeze INPUT_FOLDER = 'input/grid5000/access' -IGNORE_SITES = %w[strasbourg] +YAML_LOAD_ARGS = { aliases: true }.freeze + +# Parses and stores expanded group structures +# also epands any list given to expand(...) +class GgaGroups + def initialize(filepath = nil) + @groups = {} + load_file(filepath) if filepath + end + + def get(name) + @groups[name] + end + + def expand(list) + parse_def(list) + end + + def load_file(filepath) + if File.exist?(filepath) + YAML.load_file(filepath).each do |name, v| + groupdef = parse_def(v) + set(name, groupdef) + rescue RuntimeError => e + raise e.exception("#{e.message}, when parsing group #{v} in #{filepath}") + end + else + # FIXME: put some "skip" in the yaml, and remove useless yamls. + puts 'Warning: Skipping group configuration since there is no file' + end + end + + def set(name, groupdef) + raise "Trying to redefine existing access group #{name}" if @groups.key? name + raise "Bad group definition for #{name}: #{groupdef.pretty_inspect}" unless check_def(groupdef) + + @groups[name] = groupdef + end + + private -$yaml_load_args = {} -#FIXME We cannot drop ruby 2.7 support until jenkins is on debian 11 -$yaml_load_args[:aliases] = true if ::Gem::Version.new(RUBY_VERSION) >= ::Gem::Version.new('3.1.0') + def check_def(groupdef) + %w[ggas sites].all? { |k| groupdef.key? k } + end + def parse_def(group_list) + definition = {} + %w[ggas sites].each { |k| definition[k] = [] } + + group_list.each do |element| + case element[0] + when '@' + other_group = get(element[1..]) + raise "Couldn't find find group #{element}" if other_group.nil? + + definition['sites'] |= other_group['sites'] + definition['ggas'] |= other_group['ggas'] + when '%' + definition['sites'].push(element[1..]) + else + definition['ggas'].push(element) + end + end + definition['ggas'].uniq + definition['sites'].uniq + definition + end +end # Ulgy function to order hash since order is different on ruby 2.7 and 3.x def deep_sort_hash(hash) @@ -25,8 +89,6 @@ def deep_sort_hash(hash) sorted_hash end - - def generate_accesses_yaml(output_path, data) output_file = File.new(output_path, 'w') output_file.write(deep_sort_hash(data).to_yaml) @@ -37,6 +99,10 @@ def generate_accesses_json(output_path, data) output_file.write(JSON.pretty_generate(deep_sort_hash(data))) end +def drop(str, num = 1) + str[num..] +end + ########################################## # nodeset mode history generation # ########################################## @@ -75,7 +141,7 @@ end def load_yaml_from_git(git_repo, sha, yaml_path) relative_path = yaml_path.sub(git_repo.repo.path.gsub(/\.git$/, ''), '') - YAML.load(git_repo.show("#{sha}:#{relative_path}"), **$yaml_load_args) || {} + YAML.load(git_repo.show("#{sha}:#{relative_path}"), **YAML_LOAD_ARGS) || {} end # Update history only if the mode changed, if so we terminate the last entry and @@ -91,7 +157,7 @@ end def generate_nodeset_mode_history site_data_hierarchy = load_data_hierarchy nodeset_history = {} - git_repo = Git.open(".") + git_repo = Git.open('.') diff = git_repo.diff.name_status.keys.select { |x| x.start_with?(INPUT_FOLDER) } unless diff.empty? abort "Please commit your changed on: #{diff.join(',')}. This generator use the git history to build history of the access mode of the nodes" @@ -112,120 +178,49 @@ end # access level generation # ########################################## -# Helper function -def value_and_tail_iterator(array) - Enumerator.new do |yielder| - array.each_with_index do |value, index| - yielder.yield [value, array[(index + 1)..-1]] - end - end -end - -def priority_to_level(priority) - case priority - when 'p1' - 40 - when 'p2' - 30 - when 'p3' - 20 - when 'p4' - 10 - when 'besteffort' - 0 - end -end +def expand_nodeset_lists + group_config_path = File.join(INPUT_FOLDER, 'group.yaml') + groups = GgaGroups.new(group_config_path) -def determine_access_level(expanded_ggas, gga) - value_and_tail_iterator(%w[p1 p2 p3 p4 besteffort]).each do |level, lower_levels| - next unless expanded_ggas[level]&.delete(gga) + inputs = load_nodeset_inputs - lower_levels.each { |l| expanded_ggas[l]&.delete(gga) } - return { 'label' => level, 'level' => priority_to_level(level) } + unspecified_nodesets = all_nodesets - inputs.keys + overspecified_nodesets = inputs.keys - all_nodesets + abort "Some nodeset are not configure: #{unspecified_nodesets.join(', ')}" unless unspecified_nodesets.empty? + unless overspecified_nodesets.empty? + puts "Warning: Some unkown (or not production) nodeset ARE configured : #{overspecified_nodesets.join(', ')}" end - { 'label' => 'no-access', 'level' => -1 } -end -def create_access(prio, nodeset) - expanded_ggas = prio.transform_values { |x| expand_ggas(x) } - puts "Warning: No prio defined for #{nodeset}" if expanded_ggas.values.flatten.empty? - h = ALL_GGAS.map { |x| x['name'] }.sort.map do |gga| - level_info = determine_access_level(expanded_ggas, gga) - [gga, level_info] - end.to_h - expanded_ggas.each do |_, remanding_gga| - unless remanding_gga.empty? - puts "Warning: Some gga specified for the #{nodeset} nodeset do not exist: #{remanding_gga.join(',')}" - end + outputs = inputs.each_with_object({}) do |(nodeset, access_hash), output| + output[nodeset] = expands_acces_hashes(access_hash, groups) + rescue RuntimeError => e + raise e.exception "#{e.message} when processing input nodeset #{nodeset}" end - h -end - -def expand_ggas(ggas) - return [] if ggas.nil? - expanded_ggas = [] - ggas.each do |group| - to_remove = group.start_with?('-') - group = group[1..-1] if to_remove + outputs +end - if group.start_with?('%') - site = group[1..-1] - abort "Error: Unable to expand %#{site}: no site of that name" unless ALL_SITES.include?(site) - site_gga = ALL_GGAS.select { |x| x['site'] == site }.map { |x| x['name'] } - if site_gga.empty? - puts "Warning: expanding %#{site} gave no gga" - end - expanded_ggas = to_remove ? expanded_ggas - site_gga : expanded_ggas + site_gga - elsif group.start_with?('@') - group_gga = group[1..-1] - unless $group_of_gga.key?(group_gga) - abort "Error: Unable to expand @#{group_gga}: group of gga is not not defined?" - end - expanded_ggas = to_remove ? expanded_ggas - $group_of_gga[group_gga] : expanded_ggas + $group_of_gga[group_gga] - elsif to_remove - expanded_ggas.delete(group) - else - expanded_ggas << group - end - end - expanded_ggas.uniq +# Eats a {p1 => [@group,%site,gga], p2=> ... } hash of lists +# Outputs a { p1 => {ggas => [...], sites => [...]}, p2 => ...} unaliased hash of hashes +def expands_acces_hashes(hash, ggagroups) + hash.transform_values { |list| ggagroups.expand(list) } end -def generate_access_level +def load_nodeset_inputs + access_hash = {} site_data_hierarchy = load_data_hierarchy - group_config_path = File.join(INPUT_FOLDER, 'group.yaml') - if File.exist?(group_config_path) - YAML.load_file(group_config_path).each do |k, v| - $group_of_gga[k] = expand_ggas(v) - end - else - # FIXME: put some "skip" in the yaml, and remove useless yamls. - puts 'Warning: Skipping group configuration since there is no file' - end - - nodesets = {} site_data_hierarchy['sites'].each_key do |site| site_config_path = File.join(INPUT_FOLDER, "#{site}.yaml") - if File.exist?(site_config_path) - yaml_access_file = YAML.load_file(site_config_path, **$yaml_load_args) - unless yaml_access_file - puts "Warning: #{site} configuration is present but empty" - next - end - nodesets.update(yaml_access_file) unless IGNORE_SITES.include?(site) - end - end - unspecified_nodesets = all_nodesets - nodesets.keys - overspecified_nodesets = nodesets.keys - all_nodesets - abort "Some nodeset are not configure: #{unspecified_nodesets.join(', ')}" unless unspecified_nodesets.empty? - puts "Warning: Some unkown (or not production) nodeset ARE configured : #{overspecified_nodesets.join(', ')}" unless overspecified_nodesets.empty? - nodesets.each_with_object({}) do |(nodeset, prio_input), acc| - create_access(prio_input, nodeset).each do |gga, prio| - acc[gga] = {} unless acc.key?(gga) - acc[gga][nodeset] = prio + next unless File.exist?(site_config_path) + + yaml_access_file = YAML.load_file(site_config_path, **YAML_LOAD_ARGS) + unless yaml_access_file + puts "Warning: #{site} configuration is present but empty" + next end + access_hash.update(yaml_access_file) end + access_hash end def all_nodesets diff --git a/lib/refrepo/gen/accesses.rb b/lib/refrepo/gen/accesses.rb new file mode 100644 index 0000000000000000000000000000000000000000..5a9e97a953d33ad0a5220b00cae3cff58d024fcb --- /dev/null +++ b/lib/refrepo/gen/accesses.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'refrepo/accesses' + +VALID_TARGETS = %i[json history human].freeze +ACCESS_TARGETS = %i[json human].freeze + +def generate_accesses_ir(list) + targets = list.uniq & VALID_TARGETS + + if targets.include? :history + history_data = generate_nodeset_mode_history + write_mode_history(history_data) + end + + return unless ACCESS_TARGETS.any? { |t| targets.include? t } + + access_data = expand_nodeset_lists + write_access_json(access_data) if targets.include? :json + write_human_readables(access_data) if targets.include? :human +end + +def write_access_json(data) + output_path = Pathname.new(output_dir).join('nodesets.json') + File.delete(output_path) if File.exist?(output_path) + generate_accesses_json(output_path, data) +end + +def write_human_readables(data) + output_path = Pathname.new(output_dir).join('nodesets.yaml') + File.delete(output_path) if File.exist?(output_path) + generate_accesses_yaml(output_path, data) +end + +def write_mode_history(data) + output_path = Pathname.new(output_dir).join('accesses_mode_history.yaml') + File.delete(output_path) if File.exist?(output_path) + generate_accesses_yaml(output_path, data) +end + +def output_dir + output_data_dir = '../../../data/grid5000/' + refapi_path = File.expand_path(output_data_dir, File.dirname(__FILE__)) + Pathname.new(refapi_path).join('accesses') +end diff --git a/lib/refrepo/gen/reference-api.rb b/lib/refrepo/gen/reference-api.rb index 6b1b8065833e2cdb13239676a2ef77e95f42dc89..77ba5bd29058c6f6602b94f19a97302aa1f55b4d 100644 --- a/lib/refrepo/gen/reference-api.rb +++ b/lib/refrepo/gen/reference-api.rb @@ -162,8 +162,8 @@ def generate_reference_api # Generate the json containing all accesses level. accesses_path.mkpath() generate_accesses_json( - accesses_path.join("all.json"), - generate_access_level + accesses_path.join('nodesets.json'), + expand_nodeset_lists )