Commit 4e7d4751 authored by MICHON Nicolas's avatar MICHON Nicolas
Browse files

[dev] Added global Rakefile and required adaptations of existing scripts.

parent 2bbb00a1
......@@ -3,3 +3,18 @@ The reference API data of Grid'5000 is stored in a git repository as JSON files.
The git repository also includes scripts for generating:
* The OAR properties ie. the node information that is registered in OAR databases. OAR properties allow user to select resources matching their experiment requirements.
* The configuration files of the following puppet modules: bindg5kb, conmang5k, dhcpg5k, kadeployg5k and lanpowerg5k.
Automated tasks are provided to validate and generate data:
rake oar:properties # Generate oar properties
rake puppet:all # Launch all puppet generators
rake puppet:bindg5k # Generate bindg5k configuration
rake puppet:conmang5k # Generate conmang5k configuration
rake puppet:dhcpg5k # Generate dhcpg5k configuration
rake puppet:kadeployg5k # Generate kadeployg5k configuration
rake puppet:lanpowerg5k # Generate lanpowerg5k configuration
rake reference-api # Creates json data from inputs
rake validators:homogeneity # Check homogeneity of clusters
rake validators:schema # Check input data schema validity
For more information about generators and validators, please see generators/README.md.
REFAPI_DIR = "./generators/reference-api"
PUPPET_DIR = "./generators/puppet"
OAR_DIR = "./generators/oar-properties"
VALIDATORS_DIR = "./generators/input-validators"
namespace :puppet do
all_puppet_tasks = [:bindg5k, :conmang5k, :dhcpg5k, :kadeployg5k, :lanpowerg5k]
all_puppet_tasks.each { |t|
desc "Generate #{t} configuration"
task t do
puts "#{PUPPET_DIR}/#{t}.rb #{$CMD_ARGS}"
ruby "#{PUPPET_DIR}/#{t}.rb #{$CMD_ARGS}"
end
}
desc "Launch all puppet generators"
task :all => all_puppet_tasks
end
namespace :oar do
desc "Generate oar properties"
task :properties do
puts "#{OAR_DIR}/oar-properties.rb #{$CMD_ARGS}"
ruby "#{OAR_DIR}/oar-properties.rb #{$CMD_ARGS}"
end
end
namespace :validators do
desc "Check homogeneity of clusters"
task "homogeneity" do
puts "ruby #{VALIDATORS_DIR}/check-cluster-homogeneity.rb #{$CMD_ARGS}"
system ("ruby #{VALIDATORS_DIR}/check-cluster-homogeneity.rb #{$CMD_ARGS}")
end
desc "Check input data schema validity"
task "schema" do
puts "ruby #{VALIDATORS_DIR}/yaml-input-schema-validator.rb #{$CMD_ARGS}"
system ("ruby #{VALIDATORS_DIR}/yaml-input-schema-validator.rb #{$CMD_ARGS}")
end
end
desc "Creates json data from inputs"
task "reference-api" => ["validators:homogeneity", "validators:schema"] do
#no args needed here
puts "#{REFAPI_DIR}/reference-api.rb"
ruby "#{REFAPI_DIR}/reference-api.rb"
end
#Hack rake: call only the first task and consider the rest as arguments to this task
currentTask = Rake.application.top_level_tasks.first
taskNames = Rake.application.tasks().map { |task| task.name() }
if (taskNames.include?(currentTask))
Rake.application.instance_variable_set(:@top_level_tasks, [currentTask])
ARGV.shift(ARGV.index(currentTask) + 1)
$CMD_ARGS = ARGV.map{|arg| "'#{arg}'"}.join(' ')
else
#Not running any task, maybe rake options, invalid, etc...
end
......@@ -12,9 +12,7 @@ require 'fileutils'
require 'pathname'
require 'hashdiff'
dir = Pathname(__FILE__).parent
require "#{dir}/../lib/input_loader"
require_relative "../lib/input_loader"
def global_ignore_keys()
......@@ -138,9 +136,10 @@ def cluster_homogeneity(refapi_hash, options = {:verbose => false})
end
ignore_keys = global_ignore_keys()
cignore_keys = cluster_ignore_keys("../input-validators/check-cluster-homogeneity.yaml.erb")
cignore_keys = cluster_ignore_keys(File.expand_path("../input-validators/check-cluster-homogeneity.yaml.erb", File.dirname(__FILE__)))
refapi_hash = load_yaml_file_hierarchy("../../input/grid5000/")
input_data_dir = "../../input/grid5000/"
refapi_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
count = {}
total_count = 0
......@@ -218,7 +217,7 @@ def check_cluster_homogeneity(refapi_hash, options = {:verbose => false})
puts count.to_yaml unless verbose
puts "\nUse '../input-validators/check-cluster-homogeneity.rb -v' for details." unless verbose
puts "\nUse '-v' option for details." unless verbose
return total_count
end
......@@ -265,7 +264,7 @@ if __FILE__ == $0
end
end.parse!
refapi_hash = load_yaml_file_hierarchy("#{dir}/../../input/grid5000/")
refapi_hash = load_yaml_file_hierarchy(File.expand_path("../../input/grid5000/", File.dirname(__FILE__)))
total_count = check_cluster_homogeneity(refapi_hash, options)
# return 0 if all nodes are homogeneous, 1 otherwise
......
......@@ -8,10 +8,8 @@ end
require 'fileutils'
require 'pathname'
dir = Pathname(__FILE__).parent
require "#{dir}/../lib/input_loader"
require "#{dir}/lib/schema_validator"
require_relative "../lib/input_loader"
require_relative "./lib/schema_validator"
def run_validator(uid, data, schema)
validator = HashValidator.validate(data, schema, strict = true)
......@@ -24,13 +22,11 @@ def run_validator(uid, data, schema)
end
def yaml_input_schema_validator(global_hash, sites = nil, clusters = nil)
dir = Pathname(__FILE__).parent
schema_global = load_yaml_schema("#{dir}/schema-global.yaml")
schema_site = load_yaml_schema("#{dir}/schema-site.yaml")
schema_cluster = load_yaml_schema("#{dir}/schema-cluster.yaml")
schema_node = load_yaml_schema("#{dir}/schema-node.yaml")
schema_network_equipments = load_yaml_schema("#{dir}/schema-network_equipments.yaml")
schema_global = load_yaml_schema(File.expand_path("./schema-global.yaml", File.dirname(__FILE__)))
schema_site = load_yaml_schema(File.expand_path("./schema-site.yaml", File.dirname(__FILE__)))
schema_cluster = load_yaml_schema(File.expand_path("./schema-cluster.yaml", File.dirname(__FILE__)))
schema_node = load_yaml_schema(File.expand_path("./schema-node.yaml", File.dirname(__FILE__)))
schema_network_equipments = load_yaml_schema(File.expand_path("./schema-network_equipments.yaml", File.dirname(__FILE__)))
r = true
......@@ -90,7 +86,7 @@ if __FILE__ == $0
end.parse!
puts "Checking input data:\n\n"
global_hash = load_yaml_file_hierarchy("#{dir}/../../input/grid5000/")
global_hash = load_yaml_file_hierarchy(File.expand_path("../../input/grid5000/", File.dirname(__FILE__)))
r = yaml_input_schema_validator(global_hash, options[:sites], options[:clusters])
puts 'OK' if r
exit r
......
......@@ -2,7 +2,7 @@
require 'yaml'
require 'erb'
require '../lib/hash/hash.rb'
require_relative '../lib/hash/hash.rb'
def load_yaml_file_hierarchy(directory)
......
......@@ -11,13 +11,13 @@ require 'pp'
require 'erb'
require 'pathname'
require 'optparse'
require '../lib/input_loader'
require '../lib/hash/hash.rb'
require_relative '../lib/input_loader'
require_relative '../lib/hash/hash.rb'
options = {}
options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
options[:output_dir] = "/tmp/puppet-repo"
options[:conf_dir] = "./conf-examples/"
options[:conf_dir] = File.expand_path("conf-examples/", File.dirname(__FILE__))
OptionParser.new do |opts|
opts.banner = "Usage: conmang5k.rb [options]"
......@@ -30,7 +30,7 @@ OptionParser.new do |opts|
options[:conf_dir] = "#{options[:output_dir]}/modules/lanpowerg5k/generators/"
end
opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: ./conf-examples/") do |d|
opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: #{options[:conf_dir]}") do |d|
options[:conf_dir] = d
end
......@@ -54,14 +54,16 @@ puts "Writing Conman configuration files to: #{options[:output_dir]}"
puts "Using configuration directory: #{options[:conf_dir]}"
puts "For site(s): #{options[:sites].join(', ')}"
# Input
refapi = load_yaml_file_hierarchy("../../input/grid5000/")
config = YAML::load_file(options[:conf_dir] + 'console.yaml')
credentials = YAML::load_file(options[:conf_dir] + 'console-password.yaml')
#Input
input_data_dir = "../../input/grid5000/"
refapi = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
config = YAML::load_file(options[:conf_dir] + '/console.yaml')
credentials = YAML::load_file(options[:conf_dir] + '/console-password.yaml')
# Apply ERB template and save result to file
def write_conman_file(site_uid, site_refapi, site_config, site_credentials, options)
output = ERB.new(File.read("templates/conman.erb")).result(binding)
output = ERB.new(File.read(File.expand_path('templates/conman.erb', File.dirname(__FILE__)))).result(binding)
output_file = Pathname("#{options[:output_dir]}/modules/conmang5k/files/#{site_uid}/conman.conf")
output_file.dirname.mkpath()
......
......@@ -9,9 +9,11 @@ require 'pp'
require 'erb'
require 'pathname'
require 'optparse'
require '../lib/input_loader'
require_relative '../lib/input_loader'
global_hash = load_yaml_file_hierarchy("../../input/grid5000/")
input_data_dir = "../../input/grid5000/"
global_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
options = {}
options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
......@@ -72,7 +74,7 @@ def write_dhcp_file(data, options)
return ""
end
output = ERB.new(File.read('templates/dhcp.erb')).result(binding)
output = ERB.new(File.read(File.expand_path('templates/dhcp.erb', File.dirname(__FILE__)))).result(binding)
output_file = Pathname("#{options[:output_dir]}/modules/dhcpg5k/files/#{data.fetch("site_uid")}/dhcpd.conf.d/#{data.fetch('filename')}")
output_file.dirname.mkpath()
File.write(output_file, output)
......
......@@ -13,15 +13,16 @@ require 'pp'
require 'erb'
require 'pathname'
require 'optparse'
require '../lib/input_loader'
require '../lib/hash/hash.rb'
require_relative '../lib/input_loader'
require_relative '../lib/hash/hash.rb'
global_hash = load_yaml_file_hierarchy("../../input/grid5000/")
input_data_dir = "../../input/grid5000/"
global_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
options = {}
options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
options[:output_dir] = "/tmp/puppet-repo"
options[:conf_dir] = "./conf-examples/"
options[:conf_dir] = File.expand_path("conf-examples/", File.dirname(__FILE__))
OptionParser.new do |opts|
opts.banner = "Usage: kadeployg5k.rb [options]"
......@@ -34,7 +35,7 @@ OptionParser.new do |opts|
options[:conf_dir] = "#{options[:output_dir]}/modules/kadeployg5k/generators/"
end
opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: ./conf-examples") do |d|
opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: #{options[:conf_dir]}") do |d|
options[:conf_dir] = d
end
......@@ -184,8 +185,8 @@ end
next
end
output = ERB.new(File.read('templates/kadeployg5k.conf.erb')).result(binding)
output = ERB.new(File.read(File.expand_path('templates/kadeployg5k.conf.erb', File.dirname(__FILE__)))).result(binding)
output_file = Pathname("#{options[:output_dir]}/modules/kadeployg5k/files/#{site_uid}/server_conf#{suffix.tr('-', '_')}/#{cluster_uid}-cluster.conf")
output_file.dirname.mkpath()
File.write(output_file, output)
......
......@@ -11,13 +11,13 @@ require 'pp'
require 'yaml'
require 'pathname'
require 'optparse'
require '../lib/input_loader'
require '../lib/hash/hash.rb'
require_relative '../lib/input_loader'
require_relative '../lib/hash/hash.rb'
options = {}
options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
options[:output_dir] = "/tmp/puppet-repo"
options[:conf_dir] = "./conf-examples/"
options[:conf_dir] = File.expand_path("conf-examples/", File.dirname(__FILE__))
OptionParser.new do |opts|
opts.banner = "Usage: lanpowerg5k.rb [options]"
......@@ -30,7 +30,7 @@ OptionParser.new do |opts|
options[:conf_dir] = "#{options[:output_dir]}/modules/lanpowerg5k/generators/"
end
opts.on('-c', '--conf-dir dir', String, 'Select the lanpower module configuration path', "Default: ./conf-examples") do |d|
opts.on('-c', '--conf-dir dir', String, 'Select the lanpower module configuration path', "Default: #{options[:conf_dir]}") do |d|
options[:conf_dir] = d
end
......@@ -54,9 +54,11 @@ puts "Writing lanpower configuration files to: #{options[:output_dir]}"
puts "Using configuration directory: #{options[:conf_dir]}"
puts "For site(s): #{options[:sites].join(', ')}"
config = YAML::load_file("#{options[:conf_dir]}console.yaml")
credentials = YAML::load_file("#{options[:conf_dir]}console-password.yaml")
refapi = load_yaml_file_hierarchy("../../input/grid5000/")
config = YAML::load_file("#{options[:conf_dir]}/console.yaml")
credentials = YAML::load_file("#{options[:conf_dir]}/console-password.yaml")
input_data_dir = "../../input/grid5000/"
refapi = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
refapi['sites'].each { |site_uid, site_refapi|
......
......@@ -11,7 +11,8 @@
Dir.entries(zones_dir).sort.each { |file|
next unless /.*.db$/.match(file)
next if /.*-manual\.db/.match(file) # Those files are included in the generated version of the file
next if local_reverse_list.include?(file) # Local reverse zone files are not be included on the global-<site_uid>.conf files (see #7170 -- fixed)
next if /^local\/.*/.match(file) # Local reverse zone files are not be included on the global-<site_uid>.conf files (see #7170 -- fixed)
next if /#{site_uid}-/.match(file) #Do not include site_uid-{servers, pdus} etc
comment = ''
zone = ''
......@@ -24,11 +25,10 @@ Dir.entries(zones_dir).sort.each { |file|
puts "Error: unknown zone for '#{file}' in '#{zones_dir}'"
next
end
%>
%>
zone "<%= zone %>" {
type <%= type %>;
file "/etc/bind/zones/<%= site_uid %>/<%= file %>";
<%= type == 'slave' ? 'masters { master-ns-global; };' : '' %>
};
<% } %>
......@@ -13,18 +13,17 @@ require 'pathname'
require 'json'
require 'time'
require '../lib/input_loader'
require '../input-validators/yaml-input-schema-validator'
require '../input-validators/check-cluster-homogeneity'
require '../input-validators/check-monitoring-properties'
require_relative '../lib/input_loader'
require_relative '../input-validators/yaml-input-schema-validator'
require_relative '../input-validators/check-cluster-homogeneity'
require_relative '../input-validators/check-monitoring-properties'
# Output directory
#refapi_path = "/tmp/data"
refapi_path = "../../data/grid5000"
input_data_dir = "../../input/grid5000/"
output_data_dir = "../../data/grid5000/"
#global_hash = JSON.parse(STDIN.read)
#global_hash = load_yaml_file_hierarchy("../../input/example/")
global_hash = load_yaml_file_hierarchy("../../input/grid5000/")
refapi_path = File.expand_path(output_data_dir, File.dirname(__FILE__))
global_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
# Parse network equipment description and return switch name and port connected to given node
# In the network description, if the node interface is given (using "port" attribute),
......@@ -104,24 +103,6 @@ OptionParser.new do |opts|
end
end.parse!
#
# input-validators
#
puts "Checking input data:\n\n"
r = yaml_input_schema_validator(global_hash)
puts 'OK' if r
puts "\n"
puts "#" * 80
puts "\n"
check_cluster_homogeneity(global_hash)
puts "\n"
puts "#" * 80
puts "\n"
#
# Write grid info
#
......@@ -341,10 +322,5 @@ global_hash["sites"].each do |site_uid, site|
site["network_equipments"] = site.delete("networks")
end
write_json(grid_path.join("../../#{global_hash['uid']}-all.json"), global_hash)
#
#
#
#check_monitoring_properties(global_hash)
#Write global json file
write_json(grid_path.join(File.expand_path("../../#{global_hash['uid']}-all.json", File.dirname(__FILE__))), global_hash)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment