kadeployg5k.rb 6.78 KB
Newer Older
1 2
#!/usr/bin/ruby

3 4 5 6
# This script generates:
# - kadeployg5k/files/<site_uid>/server_conf[_dev]/clusters.conf from input/
# - kadeployg5k/files/<site_uid>/server_conf[_dev]/<cluster_uid>-cluster.conf from kadeployg5k[-dev].yaml and template/kadeployg5k.yaml.erb

Jérémie Gaidamour's avatar
Jérémie Gaidamour committed
7 8 9 10 11
if RUBY_VERSION < "2.1"
  puts "This script requires ruby >= 2.1"
  exit
end

12 13
require 'pp'
require 'erb'
14
require 'pathname'
15
require 'optparse'
16 17
require_relative '../lib/input_loader'
require_relative '../lib/hash/hash.rb'
18

19 20
input_data_dir = "../../input/grid5000/"
global_hash = load_yaml_file_hierarchy(File.expand_path(input_data_dir, File.dirname(__FILE__)))
21 22 23 24

options = {}
options[:sites] = %w{grenoble lille luxembourg lyon nancy nantes rennes sophia}
options[:output_dir] = "/tmp/puppet-repo"
25
options[:conf_dir] = File.expand_path("conf-examples/", File.dirname(__FILE__))
26 27 28 29 30

OptionParser.new do |opts|
  opts.banner = "Usage: kadeployg5k.rb [options]"

  opts.separator ""
31
  opts.separator "Example: ruby kadeployg5k.rb -s nancy -o /tmp/puppet-repo -c $(dirname path_to_kadeployg5k.yaml)"
32 33 34

  opts.on('-o', '--output-dir dir', String, 'Select the puppet repo path', "Default: " + options[:output_dir]) do |d|
    options[:output_dir] = d
35
    options[:conf_dir] = "#{options[:output_dir]}/platforms/production/generators/kadeploy"
36 37
  end

38
  opts.on('-c', '--conf-dir dir', String, 'Select the conman configuration path', "Default: #{options[:conf_dir]}") do |d|
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
    options[:conf_dir] = d
  end

  opts.separator ""
  opts.separator "Filters:"

  opts.on('-s', '--sites a,b,c', Array, 'Select site(s)', "Default: " + options[:sites].join(", ")) do |s|
    raise "Wrong argument for -s option." unless (s - options[:sites]).empty?
    options[:sites] = s
  end

  opts.on_tail("-h", "--help", "Show this message") do
    puts opts
    exit
  end
end.parse!

56
raise("Error: #{options[:conf_dir]} does not exist. The given configuration path is incorrect") unless Pathname(options[:conf_dir].to_s).exist?
57 58 59 60

puts "Writing Kadeploy configuration files to: #{options[:output_dir]}"
puts "Using configuration directory: #{options[:conf_dir]}"
puts "For site(s): #{options[:sites].join(', ')}"
61 62 63 64 65 66 67 68

# Compute cluster prefix
# input: cluster_list = ['graoully', 'graphene', 'griffon', ...]
# output: prefix_hash = {'grao' => 'graoully', 'graphe' => 'graphene', ...}
def cluster_prefix(cluster_list)

  # Shrink cluster names. Start with 3 characters
  # prefix_hash = {'gra' => ['graoully', 'graphene', ...], 'gri' => ['griffon']}
69
  prefix_hash = cluster_list.group_by { |x| x[0, 3] }
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85

  # Add characters until each prefix is unique
  loop do
    prefix_hash.clone.each { |k, v|
      next if v.size == 1
      prefix_hash.merge!(v.group_by { |x| x[0, k.length+1] })
      prefix_hash.delete(k)
    }
    break if prefix_hash.keys.size == cluster_list.size # no prefix duplicates
  end

  # Inverse key <=> value
  prefix_hash = Hash[prefix_hash.map {|k, v| [v[0], k] }]

end

86
# Extract the node ip from the node hash
87 88 89 90 91 92 93 94
def get_ip(node)
  node['network_adapters'].each { |device, network_adapter|
    if network_adapter['mounted'] && /^eth[0-9]$/.match(device)
      return network_adapter['ip']
    end
  }
end

95
# There is two kadeploy servers : kadeploy and kadeploy-dev
96
['', '-dev'].each {|suffix|
97

98 99
  global_hash['sites'].each { |site_uid, site|

100 101
    next unless options[:sites].include?(site_uid)

102 103 104
    #
    # Generate site/<site_uid>/servers_conf[_dev]/clusters.conf
    #
105

106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
    clusters_conf = { 'clusters'=> [] } # output clusters.conf
    prefix = cluster_prefix(site['clusters'].keys)

    site['clusters'].sort.each { |cluster_uid, cluster|

      # clusters:
      # - name: griffon
      #   prefix: gri
      #   conf_file: /etc/kadeploy3/griffon-cluster.conf
      #   nodes:
      #   - name: griffon-[1-92].nancy.grid5000.fr
      #     address: 172.16.65.[1-92]

      cluster_conf = {}
      cluster_conf['name']      = cluster_uid
      cluster_conf['prefix']    = prefix[cluster_uid]
      cluster_conf['conf_file'] = "/etc/kadeploy3#{suffix}/#{cluster_uid}-cluster.conf"
      cluster_conf['nodes']     = []

      # init
      first = last = c_uid = -1
      first_ip = ['0', '0', '0', 0]

      # group nodes by range (griffon-[1-92] -> 172.16.65.[1-92])
      cluster['nodes'].each_sort_by_node_uid { |node_uid, node|
131 132 133

        next if node == nil || (node['status'] && node['status'] == 'retired')

134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
        c, id = node_uid.scan(/^([^\d]*)(\d*)$/).first
        id = id.to_i
        ip = get_ip(node).split('.')
        ip[3] = ip[3].to_i

        if c == c_uid && id == last + 1 && ip[0..2] == first_ip[0..2] && ip[3] == first_ip[3] + id - first
          # extend range
          last = id
        else
          if c_uid != -1
            node = {}
            node['name']    = "#{c_uid}[#{first}-#{last}].#{site_uid}.grid5000.fr"
            node['address'] = "#{first_ip[0..2].join('.')}.[#{first_ip[3]}-#{first_ip[3]+last-first}]"
            cluster_conf['nodes'] << node
          end

          # new range
          first = last = id
          first_ip = ip
          c_uid = c
        end
      }
      # last range
      if c_uid != -1
        node = {}
        node['name']    = "#{c_uid}[#{first}-#{last}].#{site_uid}.grid5000.fr"
        node['address'] = "#{first_ip[0..2].join('.')}.[#{first_ip[3]}-#{first_ip[3]+last-first}]"
        cluster_conf['nodes'] << node
      end

      clusters_conf['clusters'] << cluster_conf

    } # site['clusters'].each

168 169
    output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kadeploy/server#{suffix.tr('-', '_')}/#{site_uid}/clusters.conf")

170
    output_file.dirname.mkpath()
171
    write_yaml(output_file, clusters_conf)
172 173
    add_header(output_file)

174 175 176 177 178
    #
    # Generate <cluster_uid>-cluster.conf files
    #

    # Load 'conf/kadeployg5k.yaml' data and fill up the kadeployg5k.conf.erb template for each cluster
179

180
    conf = YAML::load(ERB.new(File.read("#{options[:conf_dir]}/kadeployg5k#{suffix}.yaml")).result(binding))
181 182

    site['clusters'].each { |cluster_uid, cluster|
183 184 185 186 187 188 189
      defaults = conf['defaults']
      overrides = conf[site_uid][cluster_uid]
      dupes = (defaults.to_a & overrides.to_a)
      if not dupes.empty?
        puts "Warning: cluster-specific configuration for #{cluster_uid} overrides default values: #{dupes.to_s}"
      end
      data = data = defaults.merge(overrides)
190
      if data.nil?
191
        puts "Warning: configuration not found in #{options[:conf_dir]}/kadeployg5k#{suffix}.yaml for #{cluster_uid}. Skipped"
192 193 194
        next
      end

195
      output = ERB.new(File.read(File.expand_path('templates/kadeployg5k.conf.erb', File.dirname(__FILE__)))).result(binding)
196

197
      output_file = Pathname("#{options[:output_dir]}//platforms/production/modules/generated/files/grid5000/kadeploy/server#{suffix.tr('-', '_')}/#{site_uid}/#{cluster_uid}-cluster.conf")
198

199 200
      output_file.dirname.mkpath()
      File.write(output_file, output)
201

202
    }
203

204 205
  }
}