diff --git a/lib/refrepo/gen/puppet/bindg5k.rb b/lib/refrepo/gen/puppet/bindg5k.rb index e60c255c4639e9733305c085654f97dfcd166a71..c5a45e108064efc865c8d285fa6fb74bbf431894 100644 --- a/lib/refrepo/gen/puppet/bindg5k.rb +++ b/lib/refrepo/gen/puppet/bindg5k.rb @@ -61,7 +61,7 @@ class DNS::Zone attr_accessor :site_uid attr_accessor :header attr_accessor :soa - attr_accessor :ns + attr_accessor :ns_list attr_accessor :mx attr_accessor :at attr_accessor :include @@ -71,11 +71,15 @@ class DNS::Zone if header content += "$TTL 3h\n" content += soa.dump + "\n" - content += ns.dump + "\n" + for ns in ns_list + content += ns.dump + "\n" + end if at content += at.dump + "\n" end - content += mx.dump + "\n" + if mx + content += mx.dump + "\n" + end end return content end @@ -241,14 +245,18 @@ def get_node_records(cluster_uid, node_uid, network_adapters) if net_hash['mounted'] && /^eth[0-9]$/.match(net_uid) # CNAME enabled for primary interface (node-id-iface cname node-id) - cname_record = DNS::Zone::RR::CNAME.new - cname_record.label = "#{cluster_uid}-#{node_id}-#{net_uid}" - cname_record.domainname = "#{cluster_uid}-#{node_id}" - records << cname_record - cname_record_ipv6 = DNS::Zone::RR::CNAME.new - cname_record_ipv6.label = "#{cluster_uid}-#{node_id}-#{net_uid}-ipv6" - cname_record_ipv6.domainname = "#{cluster_uid}-#{node_id}-ipv6" - records << cname_record_ipv6 + if net_hash['ip'] + cname_record = DNS::Zone::RR::CNAME.new + cname_record.label = "#{cluster_uid}-#{node_id}-#{net_uid}" + cname_record.domainname = "#{cluster_uid}-#{node_id}" + records << cname_record + end + if net_hash['ip6'] + cname_record_ipv6 = DNS::Zone::RR::CNAME.new + cname_record_ipv6.label = "#{cluster_uid}-#{node_id}-#{net_uid}-ipv6" + cname_record_ipv6.domainname = "#{cluster_uid}-#{node_id}-ipv6" + records << cname_record_ipv6 + end end #Handle interface aliases @@ -297,11 +305,13 @@ def get_node_kavlan_records(_cluster_uid, node_uid, network_adapters, kavlan_ada end # CNAME only for primary interface kavlan - if net_primaries.include?(net_uid_eth) + if net_primaries.include?(net_uid_eth) and net_hash['ip'] cname_record = DNS::Zone::RR::CNAME.new cname_record.label = "#{node_uid}-#{net_uid_kavlan}" cname_record.domainname = "#{node_uid}-#{net_uid}" #sol-23-eth0-kavlan-1 records << cname_record + end + if net_primaries.include?(net_uid_eth) and net_hash['ip6'] cname_record_ipv6 = DNS::Zone::RR::CNAME.new cname_record_ipv6.label = "#{node_uid}-#{net_uid_kavlan}-ipv6" cname_record_ipv6.domainname = "#{node_uid}-#{net_uid}-ipv6" #sol-23-eth0-kavlan-1 @@ -411,7 +421,7 @@ def include_manual_file(zone) return '' end -# +# Header can be :internal, :external, or :none def load_zone(zone_file_path, site_uid, site, header) if File.exist?(zone_file_path) zone = DNS::Zone.load(File.read(zone_file_path)) @@ -421,21 +431,23 @@ def load_zone(zone_file_path, site_uid, site, header) zone.site_uid = site_uid zone.file_path = zone_file_path #If the target file will have a header or not (manage included files without records duplication) - zone.header = header + zone.header = (header != :none) zone.soa = zone.records[0] if zone.records.any? && zone.records[0].type == 'SOA' #We only want the zone to manage these records, we will manage SOA, MX, NS, @s manually zone.records.reject! { |rec| !( rec.is_a?(DNS::Zone::RR::A) || rec.is_a?(DNS::Zone::RR::CNAME) || rec.is_a?(DNS::Zone::RR::PTR)) || rec.label == "@" } - if header - set_zone_header_records(zone, site) + if header == :internal + set_internal_zone_header_records(zone, site) + elsif header == :external + set_external_zone_header_records(zone, site) end zone.include = include_manual_file(zone) return zone end -def set_zone_header_records(zone, site) +def set_internal_zone_header_records(zone, site) if zone.soa.nil? soa = DNS::Zone::RR::SOA.new soa.serial = Time.now.utc.strftime("%Y%m%d00") @@ -447,8 +459,9 @@ def set_zone_header_records(zone, site) soa.email = "nsmaster.dns.grid5000.fr." zone.soa = soa end - zone.ns = DNS::Zone::RR::NS.new - zone.ns.nameserver = "dns.grid5000.fr." + ns = DNS::Zone::RR::NS.new + ns.nameserver = "dns.grid5000.fr." + zone.ns_list = [ns] zone.mx = DNS::Zone::RR::MX.new zone.mx.priority = 10 zone.mx.exchange = "mail.#{zone.site_uid}.grid5000.fr." @@ -458,6 +471,27 @@ def set_zone_header_records(zone, site) end end +def set_external_zone_header_records(zone, _site) + if zone.soa.nil? + soa = DNS::Zone::RR::SOA.new + soa.serial = Time.now.utc.strftime("%Y%m%d00") + soa.refresh_ttl = "4h" + soa.retry_ttl = "1h" + soa.expiry_ttl = "1w" + soa.minimum_ttl = "1h" + soa.nameserver = "ns1.grid5000.fr." + soa.email = "network-staff.lists.grid5000.fr." + zone.soa = soa + end + ns1 = DNS::Zone::RR::NS.new + ns1.nameserver = "ns1.grid5000.fr." + ns2 = DNS::Zone::RR::NS.new + ns2.nameserver = "ns2.grid5000.fr." + ns3 = DNS::Zone::RR::NS.new + ns3.nameserver = "ns-ext1.grid5000.fr." + zone.ns_list = [ns1, ns2, ns3] +end + def update_serial(serial) date_serial = DateTime.strptime(serial.to_s, "%Y%m%d%S") now_date = DateTime.now @@ -512,6 +546,13 @@ def write_site_local_conf(site_uid, dest_dir, zones_dir) File.write(conf_file, conf_content) end +def write_site_external_conf(site_uid, dest_dir, zones_dir) + conf_file = File.join(dest_dir, "#{site_uid}-zones.conf") + FileUtils.mkdir_p(File.dirname(conf_file)) + conf_content = ERB.new(File.read(File.expand_path('templates/bind-site-external.conf.erb', File.dirname(__FILE__)))).result(binding) + File.write(conf_file, conf_content) +end + def write_zone(zone) FileUtils.mkdir_p(File.dirname(zone.file_path)) File.write(zone.file_path, zone.dump) @@ -519,39 +560,13 @@ end CLEAN_OLD_ZONE_FILES = false -# main method -def generate_puppet_bindg5k(options) - $options = options - puts "Writing DNS configuration files to: #{$options[:output_dir]}" - puts "For site(s): #{$options[:sites].join(', ')}" - - puts "Note: if you modify *-manual.db files you will have to manually update the serial in managed db file for changes to be applied" - - $written_files = [] - - refapi = load_data_hierarchy - - # Loop over Grid'5000 sites - refapi["sites"].each { |site_uid, site| - - next unless $options[:sites].include?(site_uid) - - dest_dir = "#{$options[:output_dir]}/platforms/production/modules/generated/files/bind/" - zones_dir = File.join(dest_dir, "zones/#{site_uid}") - - if CLEAN_OLD_ZONE_FILES and File::exist?(zones_dir) - # Cleanup of old zone files - Find.find(zones_dir) do |path| - next if not File::file?(path) - next if path =~ /manual/ # skip *manual* files - # FIXME those files are not named *manual*, but should not be removed - next if ['nancy-laptops.db', 'toulouse-servers.db', 'toulouse.db'].include?(File::basename(path)) - FileUtils::rm(path) - end - end - - site_records = {} +# Type can be :internal (internal DNS data served to Grid5000) or +# :external (external DNS data served to the Internet) +def fetch_site_records(site, type) + site_records = {} + # This makes no sense (currently) for external DNS + if type == :internal # Servers site_records['servers'] = get_servers_records(site) unless site['servers'].nil? @@ -561,155 +576,268 @@ def generate_puppet_bindg5k(options) # Networks and laptops (same input format) site_records['networks'] = get_networks_records(site, 'network_equipments') unless site['network_equipments'].nil? site_records['laptops'] = get_networks_records(site, 'laptops') unless site['laptops'].nil? + end - site.fetch("clusters", []).sort.each { |cluster_uid, cluster| + site.fetch("clusters", []).sort.each { |cluster_uid, cluster| - cluster.fetch('nodes').select { |node_uid, node| - node != nil && node["status"] != "retired" && node.has_key?('network_adapters') - }.each_sort_by_node_uid { |node_uid, node| + cluster.fetch('nodes').select { |node_uid, node| + node != nil && node["status"] != "retired" && node.has_key?('network_adapters') + }.each_sort_by_node_uid { |node_uid, node| - network_adapters = {} + network_adapters = {} - # Nodes - node.fetch('network_adapters').each { |net| - network_adapters[net['device']] = { - "ip" => net["ip"], - "ip6" => net["ip6"], - "mounted" => net["mounted"], - 'alias' => net['alias'], - 'pname' => net['name'], - } + # Nodes + node.fetch('network_adapters').each { |net| + network_adapters[net['device']] = { + "ip6" => net["ip6"], + "mounted" => net["mounted"], + 'alias' => net['alias'], + 'pname' => net['name'], } + network_adapters[net['device']]["ip"] = net["ip"] if type == :internal + } - # Mic - if node['mic'] && (node['mic']['ip'] || node['mic']['ip6']) - network_adapters['mic0'] = {"ip" => node['mic']['ip'], "ip6" => node['mic']['ip6']} - end + # Mic + if node['mic'] && (node['mic']['ip'] || node['mic']['ip6']) + network_adapters['mic0'] = {"ip" => node['mic']['ip'], "ip6" => node['mic']['ip6']} + end - site_records[cluster_uid] ||= [] - site_records[cluster_uid] += get_node_records(cluster_uid, node_uid, network_adapters) - - # Kavlan - kavlan_adapters = {} - ['kavlan', 'kavlan6'].each { |kavlan_kind| - if node[kavlan_kind] - node.fetch(kavlan_kind).each { |net_uid, net_hash| - net_hash.each { |kavlan_net_uid, ip| - kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"] ||= {} - kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"]['mounted'] = node['network_adapters'].select { |n| - n['device'] == net_uid - }[0]['mounted'] - kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"]['pname'] = node['network_adapters'].select { |n| - n['device'] == net_uid - }.first['name'] + '-' + kavlan_net_uid - if kavlan_kind == 'kavlan6' - kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"]['ip6'] = ip - else - kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"]['ip'] = ip - end - } + site_records[cluster_uid] ||= [] + site_records[cluster_uid] += get_node_records(cluster_uid, node_uid, network_adapters) + + # Kavlan + kavlan_adapters = {} + kavlan_kinds = ['kavlan6'] + kavlan_kinds << 'kavlan' if type == :internal + kavlan_kinds.each { |kavlan_kind| + if node[kavlan_kind] + node.fetch(kavlan_kind).each { |net_uid, net_hash| + net_hash.each { |kavlan_net_uid, ip| + kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"] ||= {} + kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"]['mounted'] = node['network_adapters'].select { |n| + n['device'] == net_uid + }[0]['mounted'] + kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"]['pname'] = node['network_adapters'].select { |n| + n['device'] == net_uid + }.first['name'] + '-' + kavlan_net_uid + if kavlan_kind == 'kavlan6' + kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"]['ip6'] = ip + else + kavlan_adapters["#{net_uid}-#{kavlan_net_uid}"]['ip'] = ip + end } + } + end + } + if kavlan_adapters.length > 0 + key_sr = "#{cluster_uid}-kavlan" + site_records[key_sr] ||= [] + site_records[key_sr] += get_node_kavlan_records(cluster_uid, node_uid, network_adapters, kavlan_adapters) + end + } # each nodes + } # each cluster + + site_records +end + +# Returns one hash entry per reverse dns file +def compute_reverse_records(site_uid, site_records) + reverse_records = {} + + site_records.each { |zone, records| + # Sort records + site_records[zone] = sort_records(records) + + records.each{ |record| + # Get reverse records + reverse_file_name, reverse_record = get_reverse_record(record, site_uid) + if reverse_file_name != nil + reverse_records[reverse_file_name] ||= [] + reverse_records[reverse_file_name].each {|r| + if r.label == reverse_record.label + puts "Warning: reverse entry with address #{reverse_record.label} already exists in #{reverse_file_name}, #{reverse_record.name} is duplicate" end } - if kavlan_adapters.length > 0 - key_sr = "#{cluster_uid}-kavlan" - site_records[key_sr] ||= [] - site_records[key_sr] += get_node_kavlan_records(cluster_uid, node_uid, network_adapters, kavlan_adapters) - end - } # each nodes - } # each cluster + reverse_records[reverse_file_name] << reverse_record + end + } + } - reverse_records = {} # one hash entry per reverse dns file + reverse_records +end - site_records.each { |zone, records| +def generate_internal_site_data(site_uid, site, dest_dir, zones_dir) + if CLEAN_OLD_ZONE_FILES and File::exist?(zones_dir) + # Cleanup of old zone files + Find.find(zones_dir) do |path| + next if not File::file?(path) + next if path =~ /manual/ # skip *manual* files + # FIXME those files are not named *manual*, but should not be removed + next if ['nancy-laptops.db', 'toulouse-servers.db', 'toulouse.db'].include?(File::basename(path)) + FileUtils::rm(path) + end + end - #Sort records - site_records[zone] = sort_records(records) + site_records = fetch_site_records(site, :internal) - records.each{ |record| - #get Reverse records - reverse_file_name, reverse_record = get_reverse_record(record, site_uid) - if reverse_file_name != nil - reverse_records[reverse_file_name] ||= [] - reverse_records[reverse_file_name].each {|r| - if r.label == reverse_record.label - puts "Warning: reverse entry with address #{reverse_record.label} already exists in #{reverse_file_name}, #{reverse_record.name} is duplicate" - end - } - reverse_records[reverse_file_name] << reverse_record - end + # One hash entry per reverse dns file + reverse_records = compute_reverse_records(site_uid, site_records) + + # Build all internal DNS zones + internal_zones = [] + + # Sort reverse records and create reverse zone from files + reverse_records.each{ |file_name, records| + if file_name.start_with?('reverse6') + records.sort!{ |a, b| + a.label.gsub('.','').reverse <=> b.label.gsub('.','').reverse } - } + else + records.sort_by!{ |r| + [r.label.to_i, r.name] + } + end - zones = [] + reverse_file_path = File.join(zones_dir, file_name) + zone = load_zone(reverse_file_path, site_uid, site, :internal) + if diff_zone_file(zone, records) + zone.soa.serial = update_serial(zone.soa.serial) + end + zone.records = records; + internal_zones << zone + } - #Sort reverse records and create reverse zone from files - reverse_records.each{ |file_name, records| - if file_name.start_with?('reverse6') - records.sort!{ |a, b| - a.label.gsub('.','').reverse <=> b.label.gsub('.','').reverse - } - else - records.sort_by!{ |r| - [r.label.to_i, r.name] - } - end + # Manage site zone (SITE.db file) + # It only contains header and inclusion of other db files + # Check modification in included files and update serial accordingly + site_zone_path = File.join(zones_dir, site_uid + ".db") + site_zone = load_zone(site_zone_path, site_uid, site, :internal) + site_zone_changed = false + + site_records.each{ |type, records| + next if records.empty? + zone_file_path = File.join(zones_dir, site_uid + "-" + type + ".db") + zone = load_zone(zone_file_path, site_uid, site, :none) + if diff_zone_file(zone, records) + puts "Internal zone file changed: #{zone.file_path}" if $options[:verbose] + site_zone_changed = true + end + zone.records = records + site_zone.include += "$INCLUDE /etc/bind/zones/#{site_uid}/#{File.basename(zone_file_path)}\n" + internal_zones << zone + } - reverse_file_path = File.join(zones_dir, file_name) - zone = load_zone(reverse_file_path, site_uid, site, true) - if diff_zone_file(zone, records) - zone.soa.serial = update_serial(zone.soa.serial) - end - zone.records = records; - zones << zone - } + if (site_zone_changed) + site_zone.soa.serial = update_serial(site_zone.soa.serial) + end - #Manage site zone (SITE.db file) - #It only contains header and inclusion of other db files - #Check modification in included files and update serial accordingly - site_zone_path = File.join(zones_dir, site_uid + ".db") - site_zone = load_zone(site_zone_path, site_uid, site, true) - site_zone_changed = false - - site_records.each{ |type, records| - next if records.empty? - zone_file_path = File.join(zones_dir, site_uid + "-" + type + ".db") - zone = load_zone(zone_file_path, site_uid, site, false) - if diff_zone_file(zone, records) - puts "Zone file changed: #{zone.file_path}" if $options[:verbose] - site_zone_changed = true - end - zone.records = records - site_zone.include += "$INCLUDE /etc/bind/zones/#{site_uid}/#{File.basename(zone_file_path)}\n" - zones << zone + internal_zones << site_zone + + # zones that are already known and going to be written + future_zones = internal_zones.map { |z| File::basename(z.file_path) } + + # Create reverse-*.db files for each reverse-*-manual.db that do not have (yet) a corresponding file. + Dir.glob(File.join(zones_dir, "reverse-*-manual.db")).each { |reverse_manual_file| + # FIXME: need to be adapted for IPv6 at some point + output_file = reverse_manual_file.sub("-manual.db", ".db") + next if future_zones.include?(File::basename(output_file)) # the zone is already going to be written + puts "Creating file for orphan reverse manual file: #{output_file}" if $options[:verbose] + # Creating the zone will include automatically the manual file + zone = load_zone(output_file, site_uid, site, :internal) + internal_zones << zone + } + + internal_zones.each{ |zone| + write_zone(zone) + } + + write_site_conf(site_uid, dest_dir, zones_dir) + write_site_local_conf(site_uid, dest_dir, zones_dir) +end + +def generate_external_site_data(site_uid, site, dest_dir, zones_dir) + site_records = fetch_site_records(site, :external) + + # One hash entry per reverse dns file + reverse_records = compute_reverse_records(site_uid, site_records) + + # Build all external DNS zones (IPv6 only) + external_zones = [] + + # Sort reverse records and create reverse zone from files + reverse_records.each{ |file_name, records| + # Only keep IPv6 reverse zones + next if not file_name.start_with?('reverse6') + # Sort + records.sort!{ |a, b| + a.label.gsub('.','').reverse <=> b.label.gsub('.','').reverse } - if (site_zone_changed) - site_zone.soa.serial = update_serial(site_zone.soa.serial) + reverse_file_path = File.join(zones_dir, file_name) + zone = load_zone(reverse_file_path, site_uid, site, :external) + if diff_zone_file(zone, records) + zone.soa.serial = update_serial(zone.soa.serial) end + zone.records = records; + external_zones << zone + } - zones << site_zone + # Manage site zone (SITE.db file) + # It only contains header and inclusion of other db files + # Check modification in included files and update serial accordingly + site_zone_path = File.join(zones_dir, site_uid + ".db") + site_zone = load_zone(site_zone_path, site_uid, site, :external) + site_zone_changed = false + + site_records.each{ |type, records| + next if records.empty? + zone_file_path = File.join(zones_dir, site_uid + "-" + type + ".db") + zone = load_zone(zone_file_path, site_uid, site, :none) + if diff_zone_file(zone, records) + puts "External zone file changed: #{zone.file_path}" if $options[:verbose] + site_zone_changed = true + end + zone.records = records + site_zone.include += "$INCLUDE /etc/bind/zones/external/#{site_uid}/#{File.basename(zone_file_path)}\n" + external_zones << zone + } - # zones that are already known and going to be written - future_zones = zones.map { |z| File::basename(z.file_path) } + if (site_zone_changed) + site_zone.soa.serial = update_serial(site_zone.soa.serial) + end - # Create reverse-*.db files for each reverse-*-manual.db that do not have (yet) a corresponding file. - Dir.glob(File.join(zones_dir, "reverse-*-manual.db")).each { |reverse_manual_file| - #FIXME: need to be adapted for IPv6 at some point - output_file = reverse_manual_file.sub("-manual.db", ".db") - next if future_zones.include?(File::basename(output_file)) # the zone is already going to be written - puts "Creating file for orphan reverse manual file: #{output_file}" if $options[:verbose] - #Creating the zone will include automatically the manual file - zone = load_zone(output_file, site_uid, site, true) - zones << zone - } + external_zones << site_zone - zones.each{ |zone| - write_zone(zone) - } + external_zones.each{ |zone| + write_zone(zone) + } + + write_site_external_conf(site_uid, dest_dir, zones_dir) +end + +# main method +def generate_puppet_bindg5k(options) + $options = options + puts "Writing DNS configuration files to: #{$options[:output_dir]}" + puts "For site(s): #{$options[:sites].join(', ')}" + + puts "Note: if you modify *-manual.db files you will have to manually update the serial in managed db file for changes to be applied" + + $written_files = [] + + refapi = load_data_hierarchy + + # Loop over Grid'5000 sites + refapi["sites"].each { |site_uid, site| + + next unless $options[:sites].include?(site_uid) - write_site_conf(site_uid, dest_dir, zones_dir) - write_site_local_conf(site_uid, dest_dir, zones_dir) + internal_dest_dir = "#{$options[:output_dir]}/platforms/production/modules/generated/files/bind/" + internal_zones_dir = File.join(internal_dest_dir, "zones/#{site_uid}") + generate_internal_site_data(site_uid, site, internal_dest_dir, internal_zones_dir) + external_dest_dir = "#{$options[:output_dir]}/platforms/production/modules/generated/files/bind_external/" + external_zones_dir = File.join(external_dest_dir, "zones/#{site_uid}") + generate_external_site_data(site_uid, site, external_dest_dir, external_zones_dir) } # each sites end diff --git a/lib/refrepo/gen/puppet/templates/bind-site-external.conf.erb b/lib/refrepo/gen/puppet/templates/bind-site-external.conf.erb new file mode 100644 index 0000000000000000000000000000000000000000..c3c518eb1587cee914465052c15cd5ceece0a72d --- /dev/null +++ b/lib/refrepo/gen/puppet/templates/bind-site-external.conf.erb @@ -0,0 +1,25 @@ +<% +# List file in the directory instead of using the 'reverse' variable as some files might be set manually +Dir.entries(zones_dir).sort.each { |file| + next unless /.*.db$/.match(file) + next if /#{site_uid}-/.match(file) #Do not include site_uid-{servers, pdus} etc + + comment = '' + zone = '' + + if file == "#{site_uid}.db" + zone = "#{site_uid}.grid5000.fr" + elsif /^reverse6-(.*).db$/.match(file) + zone = "#{$1}.ip6.arpa" + else + puts "Error: unknown zone for '#{file}' in '#{zones_dir}'" + next + end + +%> + zone "<%= zone %>" { + type master; + allow-query { any; }; + file "/etc/bind/zones/external/<%= site_uid %>/<%= file %>"; + }; +<% } %>