diff --git a/README.md b/README.md index 5ca1c53d110c0555bc360b21d2157e280c3cfe07..b68042ae66f28e477300d2b2dba2c7ce562c74db 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,18 @@ The reference API data of Grid'5000 is stored in a git repository as JSON files. The git repository also includes scripts for generating: * The OAR properties ie. the node information that is registered in OAR databases. OAR properties allow user to select resources matching their experiment requirements. * The configuration files of the following puppet modules: bindg5kb, conmang5k, dhcpg5k, kadeployg5k and lanpowerg5k. + +Automated tasks are provided to validate and generate data: + +rake oar:properties # Generate oar properties +rake puppet:all # Launch all puppet generators +rake puppet:bindg5k # Generate bindg5k configuration +rake puppet:conmang5k # Generate conmang5k configuration +rake puppet:dhcpg5k # Generate dhcpg5k configuration +rake puppet:kadeployg5k # Generate kadeployg5k configuration +rake puppet:lanpowerg5k # Generate lanpowerg5k configuration +rake reference-api # Creates json data from inputs +rake validators:homogeneity # Check homogeneity of clusters +rake validators:schema # Check input data schema validity + +For more information about generators and validators, please see generators/README.md. diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000000000000000000000000000000000000..90d170baeaa56b29c43b178dcbf800cc20a0c19f --- /dev/null +++ b/Rakefile @@ -0,0 +1,65 @@ + +REFAPI_DIR = "./generators/reference-api" +PUPPET_DIR = "./generators/puppet" +OAR_DIR = "./generators/oar-properties" +VALIDATORS_DIR = "./generators/input-validators" + +namespace :puppet do + + all_puppet_tasks = [:bindg5k, :conmang5k, :dhcpg5k, :kadeployg5k, :lanpowerg5k] + + all_puppet_tasks.each { |t| + desc "Generate #{t} configuration" + task t do + puts "#{PUPPET_DIR}/#{t}.rb #{$CMD_ARGS}" + ruby "#{PUPPET_DIR}/#{t}.rb #{$CMD_ARGS}" + end + } + + desc "Launch all puppet generators" + task :all => all_puppet_tasks + +end + +namespace :oar do + + desc "Generate oar properties" + task :properties do + puts "#{OAR_DIR}/oar-properties.rb #{$CMD_ARGS}" + ruby "#{OAR_DIR}/oar-properties.rb #{$CMD_ARGS}" + end + +end + +namespace :validators do + + desc "Check homogeneity of clusters" + task "homogeneity" do + puts "ruby #{VALIDATORS_DIR}/check-cluster-homogeneity.rb #{$CMD_ARGS}" + system ("ruby #{VALIDATORS_DIR}/check-cluster-homogeneity.rb #{$CMD_ARGS}") + end + + desc "Check input data schema validity" + task "schema" do + puts "ruby #{VALIDATORS_DIR}/yaml-input-schema-validator.rb #{$CMD_ARGS}" + system ("ruby #{VALIDATORS_DIR}/yaml-input-schema-validator.rb #{$CMD_ARGS}") + end +end + +desc "Creates json data from inputs" +task "reference-api" => ["validators:homogeneity", "validators:schema"] do + #no args needed here + puts "#{REFAPI_DIR}/reference-api.rb" + ruby "#{REFAPI_DIR}/reference-api.rb" +end + +#Hack rake: call only the first task and consider the rest as arguments to this task +currentTask = Rake.application.top_level_tasks.first +taskNames = Rake.application.tasks().map { |task| task.name() } +if (taskNames.include?(currentTask)) + Rake.application.instance_variable_set(:@top_level_tasks, [currentTask]) + ARGV.shift(ARGV.index(currentTask) + 1) + $CMD_ARGS = ARGV.map{|arg| "'#{arg}'"}.join(' ') +else + #Not running any task, maybe rake options, invalid, etc... +end diff --git a/generators/input-validators/check-cluster-homogeneity.rb b/generators/input-validators/check-cluster-homogeneity.rb index ba341c2ebd02123acc3c47aac9b55fb7d040c176..905cdb8828509c3f826e5547af86a3cb2d8012f7 100644 --- a/generators/input-validators/check-cluster-homogeneity.rb +++ b/generators/input-validators/check-cluster-homogeneity.rb @@ -12,9 +12,7 @@ require 'fileutils' require 'pathname' require 'hashdiff' -dir = Pathname(__FILE__).parent - -require "#{dir}/../lib/input_loader" +require_relative "../lib/input_loader" def global_ignore_keys() @@ -138,9 +136,10 @@ def cluster_homogeneity(refapi_hash, options = {:verbose => false}) end ignore_keys = global_ignore_keys() - cignore_keys = cluster_ignore_keys("../input-validators/check-cluster-homogeneity.yaml.erb") + cignore_keys = cluster_ignore_keys(File.expand_path("../input-validators/check-cluster-homogeneity.yaml.erb", File.dirname(__FILE__))) - refapi_hash = load_yaml_file_hierarchy("../../input/grid5000/") + input_data_dir = "../../input/grid5000/" + refapi_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__))) count = {} total_count = 0 @@ -218,7 +217,7 @@ def check_cluster_homogeneity(refapi_hash, options = {:verbose => false}) puts count.to_yaml unless verbose - puts "\nUse '../input-validators/check-cluster-homogeneity.rb -v' for details." unless verbose + puts "\nUse '-v' option for details." unless verbose return total_count end @@ -265,7 +264,7 @@ if __FILE__ == $0 end end.parse! - refapi_hash = load_yaml_file_hierarchy("#{dir}/../../input/grid5000/") + refapi_hash = load_yaml_file_hierarchy(File.expand_path("../../input/grid5000/", File.dirname(__FILE__))) total_count = check_cluster_homogeneity(refapi_hash, options) # return 0 if all nodes are homogeneous, 1 otherwise diff --git a/generators/input-validators/yaml-input-schema-validator.rb b/generators/input-validators/yaml-input-schema-validator.rb index 868a9adbb4a0152466ddbb3859e6b63c38f94136..60939a53314186a5719acbccad162fc94d34ea03 100644 --- a/generators/input-validators/yaml-input-schema-validator.rb +++ b/generators/input-validators/yaml-input-schema-validator.rb @@ -8,10 +8,8 @@ end require 'fileutils' require 'pathname' -dir = Pathname(__FILE__).parent - -require "#{dir}/../lib/input_loader" -require "#{dir}/lib/schema_validator" +require_relative "../lib/input_loader" +require_relative "./lib/schema_validator" def run_validator(uid, data, schema) validator = HashValidator.validate(data, schema, strict = true) @@ -24,13 +22,11 @@ def run_validator(uid, data, schema) end def yaml_input_schema_validator(global_hash, sites = nil, clusters = nil) - dir = Pathname(__FILE__).parent - - schema_global = load_yaml_schema("#{dir}/schema-global.yaml") - schema_site = load_yaml_schema("#{dir}/schema-site.yaml") - schema_cluster = load_yaml_schema("#{dir}/schema-cluster.yaml") - schema_node = load_yaml_schema("#{dir}/schema-node.yaml") - schema_network_equipments = load_yaml_schema("#{dir}/schema-network_equipments.yaml") + schema_global = load_yaml_schema(File.expand_path("./schema-global.yaml", File.dirname(__FILE__))) + schema_site = load_yaml_schema(File.expand_path("./schema-site.yaml", File.dirname(__FILE__))) + schema_cluster = load_yaml_schema(File.expand_path("./schema-cluster.yaml", File.dirname(__FILE__))) + schema_node = load_yaml_schema(File.expand_path("./schema-node.yaml", File.dirname(__FILE__))) + schema_network_equipments = load_yaml_schema(File.expand_path("./schema-network_equipments.yaml", File.dirname(__FILE__))) r = true @@ -90,7 +86,7 @@ if __FILE__ == $0 end.parse! puts "Checking input data:\n\n" - global_hash = load_yaml_file_hierarchy("#{dir}/../../input/grid5000/") + global_hash = load_yaml_file_hierarchy(File.expand_path("../../input/grid5000/", File.dirname(__FILE__))) r = yaml_input_schema_validator(global_hash, options[:sites], options[:clusters]) puts 'OK' if r exit r diff --git a/generators/lib/input_loader.rb b/generators/lib/input_loader.rb index ef1aeff8961a33ce70e3894195c68126e8db9c65..6a0cf2ec4a6e4c36a5f5e673f1a968db2e2e2a9b 100755 --- a/generators/lib/input_loader.rb +++ b/generators/lib/input_loader.rb @@ -2,7 +2,7 @@ require 'yaml' require 'erb' -require '../lib/hash/hash.rb' +require_relative '../lib/hash/hash.rb' def load_yaml_file_hierarchy(directory) diff --git a/generators/puppet/conmang5k.rb b/generators/puppet/conmang5k.rb index b552b5632ab177a279c4379ce1b89bd2e6daf9b2..eb5128d5f2821e898c0773936e053f1222748804 100644 --- a/generators/puppet/conmang5k.rb +++ b/generators/puppet/conmang5k.rb @@ -11,13 +11,13 @@ require 'pp' require 'erb' require 'pathname' require 'optparse' -require '../lib/input_loader' -require '../lib/hash/hash.rb' +require_relative '../lib/input_loader' +require_relative '../lib/hash/hash.rb' options = {} options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia} options[:output_dir] = "/tmp/puppet-repo" -options[:conf_dir] = "./conf-examples/" +options[:conf_dir] = File.expand_path("conf-examples/", File.dirname(__FILE__)) OptionParser.new do |opts| opts.banner = "Usage: conmang5k.rb [options]" @@ -30,7 +30,7 @@ OptionParser.new do |opts| options[:conf_dir] = "#{options[:output_dir]}/modules/lanpowerg5k/generators/" end - opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: ./conf-examples/") do |d| + opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: #{options[:conf_dir]}") do |d| options[:conf_dir] = d end @@ -54,14 +54,16 @@ puts "Writing Conman configuration files to: #{options[:output_dir]}" puts "Using configuration directory: #{options[:conf_dir]}" puts "For site(s): #{options[:sites].join(', ')}" -# Input -refapi = load_yaml_file_hierarchy("../../input/grid5000/") -config = YAML::load_file(options[:conf_dir] + 'console.yaml') -credentials = YAML::load_file(options[:conf_dir] + 'console-password.yaml') +#Input +input_data_dir = "../../input/grid5000/" +refapi = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__))) + +config = YAML::load_file(options[:conf_dir] + '/console.yaml') +credentials = YAML::load_file(options[:conf_dir] + '/console-password.yaml') # Apply ERB template and save result to file def write_conman_file(site_uid, site_refapi, site_config, site_credentials, options) - output = ERB.new(File.read("templates/conman.erb")).result(binding) + output = ERB.new(File.read(File.expand_path('templates/conman.erb', File.dirname(__FILE__)))).result(binding) output_file = Pathname("#{options[:output_dir]}/modules/conmang5k/files/#{site_uid}/conman.conf") output_file.dirname.mkpath() diff --git a/generators/puppet/dhcpg5k.rb b/generators/puppet/dhcpg5k.rb index 692845abfe5fb202cd9310bd9bf466430512a447..13b11f827446f4ecad850a69ddba45ee9a009a73 100644 --- a/generators/puppet/dhcpg5k.rb +++ b/generators/puppet/dhcpg5k.rb @@ -9,9 +9,11 @@ require 'pp' require 'erb' require 'pathname' require 'optparse' -require '../lib/input_loader' +require_relative '../lib/input_loader' -global_hash = load_yaml_file_hierarchy("../../input/grid5000/") +input_data_dir = "../../input/grid5000/" + +global_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__))) options = {} options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia} @@ -72,7 +74,7 @@ def write_dhcp_file(data, options) return "" end - output = ERB.new(File.read('templates/dhcp.erb')).result(binding) + output = ERB.new(File.read(File.expand_path('templates/dhcp.erb', File.dirname(__FILE__)))).result(binding) output_file = Pathname("#{options[:output_dir]}/modules/dhcpg5k/files/#{data.fetch("site_uid")}/dhcpd.conf.d/#{data.fetch('filename')}") output_file.dirname.mkpath() File.write(output_file, output) diff --git a/generators/puppet/kadeployg5k.rb b/generators/puppet/kadeployg5k.rb index 9c5ed20c7d9cdc474904595eb75971e83f424259..2d9ef8914673d101d4c1330f6894641547823a6d 100644 --- a/generators/puppet/kadeployg5k.rb +++ b/generators/puppet/kadeployg5k.rb @@ -13,15 +13,16 @@ require 'pp' require 'erb' require 'pathname' require 'optparse' -require '../lib/input_loader' -require '../lib/hash/hash.rb' +require_relative '../lib/input_loader' +require_relative '../lib/hash/hash.rb' -global_hash = load_yaml_file_hierarchy("../../input/grid5000/") +input_data_dir = "../../input/grid5000/" +global_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__))) options = {} options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia} options[:output_dir] = "/tmp/puppet-repo" -options[:conf_dir] = "./conf-examples/" +options[:conf_dir] = File.expand_path("conf-examples/", File.dirname(__FILE__)) OptionParser.new do |opts| opts.banner = "Usage: kadeployg5k.rb [options]" @@ -34,7 +35,7 @@ OptionParser.new do |opts| options[:conf_dir] = "#{options[:output_dir]}/modules/kadeployg5k/generators/" end - opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: ./conf-examples") do |d| + opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: #{options[:conf_dir]}") do |d| options[:conf_dir] = d end @@ -184,8 +185,8 @@ end next end - output = ERB.new(File.read('templates/kadeployg5k.conf.erb')).result(binding) - + output = ERB.new(File.read(File.expand_path('templates/kadeployg5k.conf.erb', File.dirname(__FILE__)))).result(binding) + output_file = Pathname("#{options[:output_dir]}/modules/kadeployg5k/files/#{site_uid}/server_conf#{suffix.tr('-', '_')}/#{cluster_uid}-cluster.conf") output_file.dirname.mkpath() File.write(output_file, output) diff --git a/generators/puppet/lanpowerg5k.rb b/generators/puppet/lanpowerg5k.rb index 008171a9a24ff83e637362f68a0a7c1a9282b02f..ba4c43e99de392ae2d0322801348d6a53573954c 100644 --- a/generators/puppet/lanpowerg5k.rb +++ b/generators/puppet/lanpowerg5k.rb @@ -11,13 +11,13 @@ require 'pp' require 'yaml' require 'pathname' require 'optparse' -require '../lib/input_loader' -require '../lib/hash/hash.rb' +require_relative '../lib/input_loader' +require_relative '../lib/hash/hash.rb' options = {} options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia} options[:output_dir] = "/tmp/puppet-repo" -options[:conf_dir] = "./conf-examples/" +options[:conf_dir] = File.expand_path("conf-examples/", File.dirname(__FILE__)) OptionParser.new do |opts| opts.banner = "Usage: lanpowerg5k.rb [options]" @@ -30,7 +30,7 @@ OptionParser.new do |opts| options[:conf_dir] = "#{options[:output_dir]}/modules/lanpowerg5k/generators/" end - opts.on('-c', '--conf-dir dir', String, 'Select the lanpower module configuration path', "Default: ./conf-examples") do |d| + opts.on('-c', '--conf-dir dir', String, 'Select the lanpower module configuration path', "Default: #{options[:conf_dir]}") do |d| options[:conf_dir] = d end @@ -54,9 +54,11 @@ puts "Writing lanpower configuration files to: #{options[:output_dir]}" puts "Using configuration directory: #{options[:conf_dir]}" puts "For site(s): #{options[:sites].join(', ')}" -config = YAML::load_file("#{options[:conf_dir]}console.yaml") -credentials = YAML::load_file("#{options[:conf_dir]}console-password.yaml") -refapi = load_yaml_file_hierarchy("../../input/grid5000/") +config = YAML::load_file("#{options[:conf_dir]}/console.yaml") +credentials = YAML::load_file("#{options[:conf_dir]}/console-password.yaml") +input_data_dir = "../../input/grid5000/" + +refapi = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__))) refapi['sites'].each { |site_uid, site_refapi| diff --git a/generators/puppet/templates/bind-global-site.conf.erb b/generators/puppet/templates/bind-global-site.conf.erb index 970e435239d422b0c7181c953017c50c5bc80397..f4ab07e98e2107be9b67e24e2399b4297842f252 100644 --- a/generators/puppet/templates/bind-global-site.conf.erb +++ b/generators/puppet/templates/bind-global-site.conf.erb @@ -11,7 +11,8 @@ Dir.entries(zones_dir).sort.each { |file| next unless /.*.db$/.match(file) next if /.*-manual\.db/.match(file) # Those files are included in the generated version of the file - next if local_reverse_list.include?(file) # Local reverse zone files are not be included on the global-<site_uid>.conf files (see #7170 -- fixed) + next if /^local\/.*/.match(file) # Local reverse zone files are not be included on the global-<site_uid>.conf files (see #7170 -- fixed) + next if /#{site_uid}-/.match(file) #Do not include site_uid-{servers, pdus} etc comment = '' zone = '' @@ -24,11 +25,10 @@ Dir.entries(zones_dir).sort.each { |file| puts "Error: unknown zone for '#{file}' in '#{zones_dir}'" next end - %> +%> zone "<%= zone %>" { type <%= type %>; file "/etc/bind/zones/<%= site_uid %>/<%= file %>"; <%= type == 'slave' ? 'masters { master-ns-global; };' : '' %> }; <% } %> - diff --git a/generators/reference-api/reference-api.rb b/generators/reference-api/reference-api.rb index 849a913cf5a4d3582bcd3b8cef94a16f7c988d59..ef33fca54438d3711241942a3ec5214684215e19 100644 --- a/generators/reference-api/reference-api.rb +++ b/generators/reference-api/reference-api.rb @@ -13,18 +13,17 @@ require 'pathname' require 'json' require 'time' -require '../lib/input_loader' -require '../input-validators/yaml-input-schema-validator' -require '../input-validators/check-cluster-homogeneity' -require '../input-validators/check-monitoring-properties' +require_relative '../lib/input_loader' +require_relative '../input-validators/yaml-input-schema-validator' +require_relative '../input-validators/check-cluster-homogeneity' +require_relative '../input-validators/check-monitoring-properties' # Output directory -#refapi_path = "/tmp/data" -refapi_path = "../../data/grid5000" +input_data_dir = "../../input/grid5000/" +output_data_dir = "../../data/grid5000/" -#global_hash = JSON.parse(STDIN.read) -#global_hash = load_yaml_file_hierarchy("../../input/example/") -global_hash = load_yaml_file_hierarchy("../../input/grid5000/") +refapi_path = File.expand_path(output_data_dir, File.dirname(__FILE__)) +global_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__))) # Parse network equipment description and return switch name and port connected to given node # In the network description, if the node interface is given (using "port" attribute), @@ -104,24 +103,6 @@ OptionParser.new do |opts| end end.parse! - -# -# input-validators -# - -puts "Checking input data:\n\n" -r = yaml_input_schema_validator(global_hash) -puts 'OK' if r - -puts "\n" -puts "#" * 80 -puts "\n" - -check_cluster_homogeneity(global_hash) -puts "\n" -puts "#" * 80 -puts "\n" - # # Write grid info # @@ -341,10 +322,5 @@ global_hash["sites"].each do |site_uid, site| site["network_equipments"] = site.delete("networks") end -write_json(grid_path.join("../../#{global_hash['uid']}-all.json"), global_hash) - -# -# -# - -#check_monitoring_properties(global_hash) +#Write global json file +write_json(grid_path.join(File.expand_path("../../#{global_hash['uid']}-all.json", File.dirname(__FILE__))), global_hash)