Commit 9e5fda6a authored by Gaetan SIMO's avatar Gaetan SIMO
Browse files

Added net-links

parent 8e12373e
require 'rubygems'
require 'fileutils'
require 'json'
require 'logger'
require 'restfully'
require 'yaml'
require 'pp'
ROOT_DIR = File.expand_path File.dirname(__FILE__)
LIB_DIR = File.join(ROOT_DIR, "generators", "lib")
$LOAD_PATH.unshift(LIB_DIR) unless $LOAD_PATH.include?(LIB_DIR)
EXTRA_DIR = File.join(ROOT_DIR, "extra", "lib")
$LOAD_PATH.unshift(EXTRA_DIR) unless $LOAD_PATH.include?(EXTRA_DIR)
Rake.application.options.trace = true
require 'grid5000'
require 'naming-pattern'
task :environment do
Dir.chdir(ROOT_DIR)
......@@ -15,14 +24,133 @@ task :environment do
@logger.level = Logger.const_get((ENV['DEBUG'] || 'INFO').upcase)
end
task :api_sites do
api_logger = Logger.new("/dev/null")
api_logger.level = Logger::FATAL
@api = Restfully::Session.new(:configuration_file => File.expand_path("~/.restfully/api.grid5000.fr.yml"),:logger => api_logger)
@api_sites = if ENV['SITE']
[@api.root.sites[ENV['SITE'].to_sym]]
else
@api.root.sites
namespace :netlinks do
desc "Probe a remote network equipement and retrieve its neighbors in a yaml file. HOST is its grid5000 FQDN."
task :probe do
host = ENV['HOST']
abort "You must provide HOST= the network equipment FQDN within grid5000" if host.nil?
scan = host.scan(/^([^.]+)\.([^.]+)$/)
scan = host.scan(/^([^.]+)\.([^.]+)\.grid5000\.fr$/) if scan.empty?
abort "HOST is on the form 'gw.lille' or 'gw.lille.grid5000.fr'" if scan.empty?
uid,site = scan.flatten
# copy the 'extra' dir on the site,
# execute the probing
# retrieve the result
prober = "weathermap.#{site}.grid5000.fr"
sh "rsync -av extra #{prober}:"
sh "ssh #{prober} 'cd extra && bundle install && ./bin/net-links.rb --host #{uid} --community public'"
sh "rsync -av #{prober}:/tmp/#{uid}.yaml generators/input/#{site}/net-links/"
message = "Network links saved into file generators/input/#{site}/net-links/#{uid}.yaml"
puts "+-#{"-" * message.size}-+"
puts "| #{message} |"
puts "+-#{"-" * message.size}-+"
end
def format_vlan(coord,raw_port,linecards)
puts "Vlan : #{coord.inspect} #{raw_port.inspect}"
end
def format_channel(coord,raw_port,linecards)
# puts "Channel : #{coord.inspect} #{raw_port.inspect}"
end
def format_port(coord,raw_port,linecards)
# puts "#{coord.inspect} #{raw_port.inspect}"
return unless raw_port.has_key? :fqdn
neighbor,site = raw_port[:fqdn].scan(/([^.]+)\.([^.]+)\.grid5000\.fr/).flatten
return if neighbor.nil?
l = coord["linecard"]
p = coord["port"]
linecards[l] = {"ports"=>{}} unless linecards.has_key? l
ports = linecards[l]["ports"]
formated_port = ports[p]
if formated_port.nil?
ports[p] = neighbor
elsif formated_port.is_a? Hash
ports[p]["uid"] = neighbor
else
ports[p] = neighbor
end
end
def browse_naming_patterns(dico,cb,patterns)
dico.each do |key,value|
if value.is_a? Hash
browse_naming_patterns(value,cb,patterns)
else
if key == "naming_pattern"
pattern = value
if patterns.has_key? pattern
@logger.warn "Naming Pattern already defined '#{pattern}'. No redefinition possible."
else
patterns[pattern] = cb
end
end
end
end
end
desc "Updated formated net-links with the raw information gathered from network equipments."
task :format => :environment do
# Read the network links fresh out of network equipment
# Read the configuration formated
# update the formated configuration with raw information from the net-links yaml file
site = ENV['SITE']
abort "You must provide SITE= " if site.nil?
formated_file = "generators/input/#{site}/net-links.yaml"
formated_all = YAML::load_file(formated_file)
Dir.glob("generators/input/#{site}/net-links/*.yaml").each do |raw_file|
uid = File.basename(raw_file).scan(/(\S+).yaml$/).first.first
next unless uid == "gw"
formated = formated_all[uid]
if formated.nil?
@logger.warn "Network Equipment '#{uid}' was not found in formated network links file '#{formated_file}'. Skiping"
next
end
raw = YAML::load_file(raw_file)
# Go through all formated config, and register a call back that will encode the naming_pattern
naming_patterns = {}
vlans = formated["vlans"]
vlans_cb = proc { |dict,raw_port|
format_vlan(dict,raw_port,vlans)
}
browse_naming_patterns(vlans,vlans_cb,naming_patterns)
channels = formated["channels"]
channels_cb = proc { |dict,raw_port|
format_channel(dict,raw_port,channels)
}
browse_naming_patterns(channels,channels_cb,naming_patterns)
linecards = formated["linecards"]
linecards_cb = proc { |dict,raw_port|
format_port(dict,raw_port,linecards)
}
browse_naming_patterns(linecards,linecards_cb,naming_patterns)
# Go through all ports to find they linecard and port index,
raw.each do |port|
ifname = port[:ifname]
next if ifname.nil?
# find the naming_pattern that correspond to this ifname
naming_patterns.each do |np,cb|
# scan the ifname with each naming_pattern
dict = NamingPattern.encode(np,ifname)
unless dict.empty?
# Found the naming_pattern for this ifname
# So Place it where it belongs
cb.call(dict,port)
break
end
end
#
end
end
File.open(formated_file,'w'){|f| YAML::dump(formated_all,f)}
# pp formated_all
message = "Formated Network links saved into file generators/input/#{site}/net-links.yaml"
puts "+-#{"-" * message.size}-+"
puts "| #{message} |"
puts "+-#{"-" * message.size}-+"
end
end
......@@ -41,50 +169,6 @@ namespace :g5k do
end
end
# rake deadnodes:reasons
# rake deadnodes:tofix
namespace :deadnodes do
desc "List all dead nodes and the reason why they are dead. (SITE=)"
task :reasons => [:environment,:api_sites] do
@logger.level = Logger::INFO
@reasons = true
Rake::Task["deadnodes:browse"].execute
end
desc "List all nodes which have they state not in synch with they comment. (SITE=)"
task :tofix => [:environment,:api_sites] do
@logger.level = Logger::ERROR
@tofix = true
Rake::Task["deadnodes:browse"].execute
end
task :browse do
def comment_ok?(comment)
# comment.nil? or comment == "OK"
comment == "OK"
end
@api_sites.each do |site|
site.status["nodes"].each do |uid,status|
comment = status["comment"]
state = status["hard"].downcase
if comment_ok?(comment)
if state == "dead"
@logger.error "Node '#{uid}' of state '#{state}' should not have comment '#{comment}'" if @tofix
else
# nothing, good state
end
else
if state == "dead"
@logger.info "Node '#{uid}' is dead because '#{comment}'" if @reasons
else
@logger.error "Node '#{uid}' should have the not-dead-comment 'OK', since its state is '#{state}'. Instead, it has comment '#{comment}'." if @tofix
end
end
end
end
end
end
# TESTS
# Deletion:
# rake -s oar:generate FROM=4cfebf92e9cce05315782b51e05eded4ab4f0e7e TO=7d2648eaad7dbbc6f1fdb9c0279f73d374ccd47a
......
source :rubygems
gem "snmp", "1.1.0"
GEM
remote: http://rubygems.org/
specs:
snmp (1.1.0)
PLATFORMS
ruby
DEPENDENCIES
snmp (= 1.1.0)
#!/usr/bin/env ruby
ROOT_DIR = File.expand_path('../..', __FILE__)
LIB_DIR = File.join(ROOT_DIR, "lib")
$LOAD_PATH.unshift LIB_DIR unless $LOAD_PATH.include?(LIB_DIR)
require 'naming-pattern'
#naming_pattern = "Vlan%VLANID%"
#naming_pattern = "Po%CHANNELID%"
#naming_pattern = "Te%LINECARD%/%PORT%"
#naming_pattern = "Gi%LINECARD%/%PORT%"
naming_pattern = "Gi%LINECARD%/%PORT%"
#naming_pattern = "Gi%UNO:sd:1%/%DOS:1%"
naming_pattern = "%LINECARD:A%%PORT%"
#ifname = "Gi2/7"
ifname = "D4"
def usage
puts "Usage : #{ENV["_"]} naming_pattern interface"
end
if ARGV.size != 2
usage
exit!
end
naming_pattern,ifname = ARGV
dict = NamingPattern.encode(naming_pattern,ifname)
puts "#{naming_pattern} + #{ifname} \t => \t #{dict.inspect}"
ifname = NamingPattern.decode(naming_pattern,dict)
puts "#{naming_pattern} + #{dict.inspect} \t => \t #{ifname.inspect}"
#!/usr/bin/env ruby
ROOT_DIR = File.expand_path('../..', __FILE__)
LIB_DIR = File.join(ROOT_DIR, "lib")
$LOAD_PATH.unshift LIB_DIR unless $LOAD_PATH.include?(LIB_DIR)
require 'net-links-engine'
netlinks = NetLinksEngine.new
netlinks.parse!(ARGV)
netlinks.run!
require 'optparse'
require 'logger'
class CmdLineScript
def initialize
@version = "0.1"
end
def sanitize(str)
str.strip.downcase.gsub(/\s+/,"-")
end
def parse!(args,params=[])
args.push("-h") if args.empty?
# Save the user options
user_args = args.dup
args.clear
@parsing_args = args
@prev_parsing_args_size = @parsing_args.size
@min_opt_index = 0
@options_loaded = []
@opt_parser = OptionParser.new do |opts|
opts.banner = "Usage: #{File.basename(ENV["_"])} [options]"
opts.on_tail("--version", "Display the version.") do |v|
puts @version
exit(0)
end
opts.on_tail("-h", "--help", "You are looking at it.") do
puts opts.help
exit(0)
end
end
@options = {}
if block_given?
yield
elsif params.is_a? Array
until params.empty?
k = params.shift
v = params.shift
add_option k,v
end
else
fail "This method must receive a block or an Array "
end
add_option :logger
# Merge first options with user's options (priority to user options)
merge_options user_args
@prev_parsing_args_size = @parsing_args.size
@opt_parser.parse!(@parsing_args)
end
def add_option(opt_sym,opt_dft=nil)
current_parsing_args_size = @parsing_args.size
parser_opts_nb = @prev_parsing_args_size - current_parsing_args_size
@min_opt_index -= parser_opts_nb
unless @options_loaded.include? opt_sym
opt_str = opt_sym_to_str opt_sym
dft_aft = send("option_#{opt_sym.to_s}",opt_sym,opt_str,opt_dft)
opt_dft = dft_aft unless dft_aft.nil? or dft_aft.kind_of? OptionParser
default_args opt_str, opt_dft
@options_loaded.push opt_sym
end
@prev_parsing_args_size = @parsing_args.size
end
# Find the index of a given option within the arguments that are being parsed
# Return -1 if the argument is not found.
def option_index(opt)
@parsing_args.each_with_index{|o,idx|
if o == opt
return idx
end
}
return -1
end
def opt_sym_to_str(opt_sym)
"--"+opt_sym.to_s.gsub(/_/,'-')
end
def opt_str_to_sym(opt_str)
opt_str.gsub(/^--/,"").gsub(/-/,"_").to_sym
end
# Set the argument for an option.
# If the option has already been parsed, set its value,
# Otherwise, set its value within the parsing arguments.
def set_arg(opt,arg,params={})
params[:force] = true unless params.has_key? :force
#puts "opt = #{opt}"
opt_idx = option_index(opt.to_s)
if opt_idx > -1 and params[:force]
@parsing_args[opt_idx+1].replace(arg)
else
opt_sym = opt_str_to_sym opt
unless @options.has_key? opt_sym and !params[:force]
@parsing_args.concat([opt,arg])
end
end
end
def get_arg(opt_sym)
opt_str = opt_sym_to_str opt_sym
opt_idx = option_index(opt_str.to_s)
(opt_idx > -1 ) ? @parsing_args[opt_idx+1] : nil
end
# Set the default option argument.
# If the user issued an argument, leave that as the default argument for the option
# The option and argument are both appended at the end of argument array, even if they were already within the array.
# This make sure options and arguments are parsed in the order they are declared, not in the order user supplied them.
def default_args(opt,arg)
opt_idx = option_index(opt.to_s)
#~ puts "opt = #{opt} opt_idx=#{opt_idx} @min_opt_index = #{@min_opt_index}"
if opt_idx < 0
# If option not found yet, place it at the min_index position
insert_arg(@min_opt_index,opt,arg)
@min_opt_index += 2
else
# If option found, it may be before or after the min_index
if opt_idx < @min_opt_index
# if before the min_index, do nothing since this option was already computed
arg.replace @parsing_args[opt_idx+1]
elsif opt_idx == @min_opt_index
# if equal to min_index, unshift the min_index value
@min_opt_index += 2
arg.replace @parsing_args[opt_idx+1]
else
# if after the min_index, remove it and insert it at the min_index position
opt.replace(@parsing_args.slice!(opt_idx))
arg.replace(@parsing_args.slice!(opt_idx))
insert_arg(@min_opt_index,opt,arg)
@min_opt_index += 2
end
end
end
def insert_arg(index,opt,arg)
@parsing_args.insert(index,arg)
@parsing_args.insert(index,opt)
end
def merge_options(before)
help = []
until before.empty?
opt = before.shift
unless /^-/.match(opt).nil?
if opt == "--help" or opt == "-h" or opt == "--version"
help.push opt
else
set_arg opt,before.shift
end
end
end
@parsing_args.concat(help)
end
# Mandatory options (as an example too)
def option_logger(opt_sym,opt_str,opt_dft)
opt_dft ||= "stdout:info"
@opt_parser.on("#{opt_str}=", "The logger. Can be [none, path:level]. [default=#{opt_dft}].") do |log|
levels = {:fatal=>Logger::FATAL,
:error=>Logger::ERROR,
:warn=>Logger::WARN,
:info=>Logger::INFO,
:debug=>Logger::DEBUG
}
logger = nil
if log.empty? or log == "none"
logger = Logger.new("/dev/null")
logger.level = levels[:fatal]
else
type,level = log.split(/:/,2)
logger = case type
when "stdout";Logger.new(STDOUT)
when "stderr";Logger.new(STDERR)
else;Logger.new(File.expand_path(type))
end
level = (level || "info").to_sym
level = :info unless levels.has_key? level
logger.level = levels[level]
end
@options[opt_sym] = logger unless logger.nil?
end
opt_dft
end
end
class NamingPattern
# Encode a naming pattern into a hash containing the keys and decoded values
#
# Vlan%VLANID% + Vlan101 => {"vlanid"=>101}
# Po%CHANNELID% + Po2 => {"channelid"=>2}
# Gi%LINECARD%/%PORT% + Gi2/7 => {"linecard"=>2, "port"=>7}
# Te%LINECARD%/%PORT% + Te2/7 => {"linecard"=>2, "port"=>7}
# Gi%STACK%/%LINECARD%/%PORT% + Gi2/4/5 => {"stack"=>2, "linecard"=>4, "port"=>5}
# Gi2/%LINECARD%/%PORT% + Gi2/4/5 => {"linecard"=>4, "port"=>5}
# Te2/%LINECARD%/%PORT% + Te2/4/5 => {"linecard"=>4, "port"=>5}
# %LINECARD:A%%PORT% + D5 => {"linecard"=>3, "port"=>5}
# %LINECARD%%PORT:a% + 3d => {"linecard"=>3, "port"=>3}
def self.encode(pattern,name)
keys = pattern.scan(/%([^%]+)%/).flatten
return {} if keys.nil?
# protect the naming pattern from regexp interpretation
reg = pattern.gsub('/','\/')
keys.each do |key_type|
key,type = key_type.split(/:([^:]+$)/)
type = '1' if type.nil?
reg_key_type = Regexp.new("%#{key_type}%")
reg_type = case type
when '1'; '(\d+)';
when 'a'; '([a-z]+)';
when 'A'; '([A-Z]+)';
else
abort "naming pattern type '#{type}' is unknown."
end
reg.gsub!(reg_key_type,reg_type)
key_type.replace(key)
end
reg = Regexp.new(reg)
values = name.scan(reg).flatten
return {} if values.nil? or values.size != keys.size
keys.map!{|key| key.downcase}
values.map!{|value|
if value.match(/\d+/).nil?
if value.size != 1
abort "Naming pattern does not deal with port number with more than on letter. You supplied '#{value}'"
end
value = value.ord
if ( 64 < value and value < 91 ) # [A-Z]
value - 65
elsif ( 96 < value and value < 123 ) # [a-z]
value - 97
else
abort "Naming pattern port number is not within range = '#{value.chr}'"
end
else
value.to_i
end
}
Hash[keys.zip(values)]
end
# Decode naming pattern and return a string corresponding to the given hash containing keys and values within the naming pattern
#
# Vlan%VLANID% + {"vlanid"=>101} => "Vlan101"
# Po%CHANNELID% + {"channelid"=>2} => "Po2"
# Gi%LINECARD%/%PORT% + {"linecard"=>2, "port"=>7} => "Gi2/7"
# Te%LINECARD%/%PORT% + {"linecard"=>2, "port"=>7} => "Te2/7"
# Gi%STACK%/%LINECARD%/%PORT% + {"stack"=>2, "linecard"=>4, "port"=>5} => "Gi2/4/5"
# Te%STACK%/%LINECARD%/%PORT% + {"stack"=>2, "linecard"=>4, "port"=>5} => "Te2/4/5"
# Gi2/%LINECARD%/%PORT% + {"linecard"=>4, "port"=>5} => "Gi2/4/5"
# Te2/%LINECARD%/%PORT% + {"linecard"=>4, "port"=>5} => "Te2/4/5"
# %LINECARD:A%%PORT% + {"linecard"=>3, "port"=>5} => "D5"
# %LINECARD%%PORT:a% + {"linecard"=>3, "port"=>3} => "3d"
def self.decode(pattern,dict)
keys = pattern.scan(/%([^%]+)%/).flatten
return "" if keys.nil? or dict.size != keys.size
name = pattern.dup
keys.each do |key_type|
key,type = key_type.split(/:([^:]+$)/)
type = '1' if type.nil?
value = dict[key.downcase]
return "" if value.nil?
reg_key_type = Regexp.new("%#{key_type}%")
port = case type
when '1'; value.to_s;
when 'a'; (value.to_i + 97).chr
when 'A'; (value.to_i + 65).chr
else
abort "naming pattern type '#{type}' is unknown."
end
name.gsub!(reg_key_type,port)
end
name
end
end
require 'rubygems'
require 'cmd-line-script'
require 'snmp'
require 'resolv'
#host = "gw"
#community = "public@600"
#mac_port_oids = ["1.3.6.1.2.1.17.4.3.1.1", "1.3.6.1.2.1.17.4.3.1.2"]
#bridge_oid = "1.3.6.1.2.1.17.1.4.1.2"
#ifname_oid = "1.3.6.1.2.1.31.1.1.1.1"
#
#ports = Hash.new
#
#SNMP::Manager.open(:Host => host, :Community => community) do |manager|
# manager.walk(mac_port_oids) do |mac, port|
# bridge = String.new
# ifname = String.new
# mac_addr = mac.value.unpack("H2H2H2H2H2H2").join(":")
# res = manager.get(bridge_oid + "." + port.value.to_s)
# res.each_varbind do |var|
# bridge = var.value.to_s
# end
# unless bridge == "noSuchInstance"
# res = manager.get(ifname_oid + "." + bridge)
# res.each_varbind do |var|
# ifname = var.value.to_s
# end
# end
# #ifname = manager.get(ifname_oid + "." + bridge.value.to_s)
# ports[mac_addr] = ifname
# end
#end
module MIB
module Cisco
def reload
@vlans = "1.3.6.1.4.1.9.9.46.1.3.1.1.2"
@macs = "1.3.6.1.2.1.17.4.3.1.1"
@bridges = "1.3.6.1.2.1.17.4.3.1.2"
@ifindexes = "1.3.6.1.2.1.17.1.4.1.2"
@ifnames = "1.3.6.1.2.1.31.1.1.1.1"
@arp_table = "1.3.6.1.2.1.4.22.1.2"
@mac_port_oids = ["1.3.6.1.2.1.17.4.3.1.1", "1.3.6.1.2.1.17.4.3.1.2"]
@bridge_oid = "1.3.6.1.2.1.17.1.4.1.2"
@ifname_oid = "1.3.6.1.2.1.31.1.1.1.1"
@no_such_instance = "noSuchInstance"
end
def wrong_response?(var)
var.value.to_s == @no_such_instance
end