From 52cb73bea7c1f1a21212d0058439adb4ae47f7e7 Mon Sep 17 00:00:00 2001
From: Kevin Pouget <kevin.pouget@imag.fr>
Date: Tue, 24 Jan 2017 14:43:24 +0100
Subject: [PATCH] add pagemap in the git tree
---
model/numa/__init__.py | 2 +-
model/numa/pagemap | 479 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 480 insertions(+), 1 deletion(-)
create mode 100755 model/numa/pagemap
diff --git a/model/numa/__init__.py b/model/numa/__init__.py
index 4cdcaa8..7b78f12 100644
--- a/model/numa/__init__.py
+++ b/model/numa/__init__.py
@@ -16,7 +16,7 @@ except ImportError:
numa_loaded = None
-PAGEMAP_PATH = "/home/videau/"
+PAGEMAP_PATH = "./"
PAGEMAP = "pagemap"
pagemap_bin = None
diff --git a/model/numa/pagemap b/model/numa/pagemap
new file mode 100755
index 0000000..fe1696c
--- /dev/null
+++ b/model/numa/pagemap
@@ -0,0 +1,479 @@
+#!/usr/bin/ruby
+
+=begin
+Copyright (c) 2013, Brice Videau <brice.videau@imag.fr>
+Copyright (c) 2013, Vincent Danjean <Vincent.Danjean@ens-lyon.org>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+=end
+
+PAGEMAP_VERSION="1.1"
+
+require 'optparse'
+
+class MemoryArchitecture
+ NODE_PATH = "/sys/devices/system/node/"
+ MEMORY_PATH = "/sys/devices/system/memory/"
+ def initialize
+ @nodes = {}
+ block_size_bytes = File::read(MEMORY_PATH+"block_size_bytes")
+ block_size_bytes = block_size_bytes.to_i(16)
+ nodes = Dir.entries(NODE_PATH)
+ nodes = nodes.select { |entry| entry.match(/node\d+/) }
+ @memories = Hash::new { |hash,key| hash[key] = [] }
+ nodes.each { |node|
+ memories = Dir.entries(NODE_PATH+node)
+ memories = memories.select { |entry| entry.match(/memory\d+/) }
+ ranges = []
+ number = node.scan(/(\d+)/).first[0]
+ number = number.to_i
+ memories.each { |memory|
+ phys_index = File::read(MEMORY_PATH+memory+"/phys_index")
+ phys_index = phys_index.to_i(16)
+ end_phys_index = File::read(MEMORY_PATH+memory+"/end_phys_index")
+ end_phys_index = end_phys_index.to_i(16)
+ start_address = phys_index*block_size_bytes
+ end_address = (end_phys_index-phys_index+1)*block_size_bytes+start_address
+ ranges.push(start_address...end_address)
+ @memories[start_address...end_address].push(number)
+ }
+ @nodes[number] = ranges
+ }
+ @memories = @memories.to_a
+ @memories.sort! { |a,b| a[0].end <= b[0].begin ? -1 : b[0].end <= a[0].begin ? +1 : raise( "Overlapping memory ranges!" ) }
+ end
+
+ def binary_search(address, from, to)
+ low = from
+ high = to
+ while (low <= high) do
+ split = (low+high)/2
+ if address < @memories[split][0].begin then
+ high = split - 1
+ elsif address > @memories[split][0].end then
+ low = split + 1
+ else
+ return split
+ end
+ end
+ raise "Could not find address #{address}!"
+ end
+
+ def find_node(address)
+ index = binary_search(address, 0, @memories.length - 1)
+ return [] if not @memories[index][0].include?(address)
+ result = @memories[index][1]
+ return result
+ end
+
+end
+
+class Flags
+ attr_accessor :value
+ HACK_FLAGS = [ ["RESERVED",32,"r"], ["MLOCKED",33,"m"], ["MAPPEDTODISK",34,"d"], ["PRIVATE",35,"P"], ["PRIVATE_2",36,"p"], ["OWNER_PRIVATE",37,"O"], ["ARCH",38,"h"], ["UNCACHED",39,"c"]]
+ FLAGS = [["LOCKED",0,"L"], ["ERROR",1,"E"], ["REFERENCED",2,"R"], ["UPTODATE",3,"U"], ["DIRTY",4,"D"], ["LRU",5,"l"], ["ACTIVE",6,"A"], ["SLAB",7,"S"], ["WRITEBACK",8,"W"], ["RECLAIM",9,"I"], ["BUDDY",10,"B"], ["MMAP",11,"M"], ["ANON",12,"a"], ["SWAPCACHE",13,"s"], ["SWAPBACKED",14,"b"], ["COMPOUND_HEAD",15,"H"], ["COMPOUND_TAIL",16,"T"], ["HUGE",17,"G"], ["UNEVICTABLE",18,"u"], ["HWPOISON",19,"X"], ["NOPAGE",20,"n"], ["KSM",21,"x"], ["THP",22,"t"]]
+ def initialize(value)
+ @value = value
+ end
+ def Flags.doc
+ s = ""
+ print_flag = Proc.new { |flag, val, sym| s += "#{sym}:#{flag} (#{val})\n" }
+ FLAGS.each( &print_flag )
+ HACK_FLAGS.each( &print_flag ) if $options[:hack]
+ return s
+ end
+ def to_s
+ flags = []
+ join_char = ""
+ join_char = " " if not $options[:symbols]
+ if $options[:symbols] then
+ process_flag = Proc.new { |flag, val, sym|
+ if ( (@value >> val) & 1) == 1 then
+ flags.push(sym)
+ else
+ flags.push("_")
+ end
+ }
+ else
+ process_flag = Proc.new { |flag, val| flags.push(flag) if ( (@value >> val) & 1) == 1 }
+ end
+ FLAGS.each(&process_flag)
+ HACK_FLAGS.each(&process_flag) if $options[:hack]
+ return flags.join(join_char)
+ end
+end
+
+class Page
+ attr_accessor :pfn, :swap_type, :swap_offset, :page_shift, :reserved, :swapped, :present, :count, :flags, :nodes
+ def Page.decode(code)
+ present = ((code >> 63) & 1) == 1
+ swapped = ((code >> 62) & 1) == 1
+ reserved = ((code >> 61) & 1) == 1
+ pfn = code & 0x7fffffffffffff
+ swap_type = code & 0x1f
+ swap_offset = (code & 0x7fffffffffffe0) >> 5
+ page_shift = $page_shift
+ return Page::new(pfn, swap_type, swap_offset, page_shift, reserved, swapped, present)
+ end
+ def initialize(pfn, swap_type, swap_offset, page_shift, reserved, swapped, present)
+ @pfn, @swap_type, @swap_offset, @page_shift, @reserved, @swapped, @present = pfn, swap_type, swap_offset, page_shift, reserved, swapped, present
+ @count = nil
+ if $kpagecount and @present then
+ $kpagecount.seek(@pfn*8)
+ str = $kpagecount.read(8)
+ if str then
+ @count = str.unpack('Q').first
+ end
+ end
+ @flags = nil
+ if $kpageflags and @present then
+ $kpageflags.seek(@pfn*8)
+ str = $kpageflags.read(8)
+ if str then
+ @flags = Flags::new(str.unpack('Q').first)
+ end
+ end
+ if $memory_architecture and @present then
+ @nodes = $memory_architecture.find_node(@pfn << @page_shift)
+ end
+ end
+ def to_s
+ s = ""
+ if @present then
+ s += (@pfn << @page_shift).to_s(16)
+ s += " " + @count.to_s if @count
+ s += " " + @flags.to_s if @flags
+ if @nodes then
+ @nodes.each { |node|
+ s += " N#{node}"
+ }
+ end
+ return s
+ end
+ return "swapped" if @swapped
+ return "absent"
+ end
+ def absent
+ return ((!@present) && (!@swapped))
+ end
+end
+
+class PageRange < Range
+ def initialize(address)
+ a_start, a_end = address.split("-")
+ a_start = a_start.to_i(16)
+ a_end = a_end.to_i(16)
+ super(a_start, a_end, true)
+ end
+ def to_s
+ return self.begin.to_s(16)+"-"+self.end.to_s(16)
+ end
+ def size
+ return self.end - self.begin
+ end
+ def number
+ return (self.end - self.begin)/$page_size
+ end
+ def each_page(&proc)
+ self.step($page_size, &proc)
+ end
+end
+
+class Map
+ attr_accessor :address
+ attr_accessor :perms
+ attr_accessor :offset
+ attr_accessor :device
+ attr_accessor :inode
+ attr_accessor :pathname
+ attr_accessor :pages
+ attr_accessor :numa_policy
+ attr_accessor :numa_informations
+
+ def Map.convert(line,match="")
+ address, perms, offset, device, inode, pathname = line.scan(/([0-9a-fA-F]+-[0-9a-fA-F]+)\s+([rwxsp-]+)\s+([0-9a-fA-F]+)\s+(\S+)\s+([0-9a-fA-F]+)\s+(\S*)/).first
+ address = PageRange::new(address)
+ offset = offset.to_i(16)
+ inode = inode.to_i
+ return Map::new(address, perms, offset, device, inode, pathname) if pathname.match(match)
+ return nil
+ end
+ def initialize(address, perms, offset, device, inode, pathname)
+ @address, @perms, @offset, @device, @inode, @pathname = address, perms, offset, device, inode, pathname
+ @pages = {}
+ @address.each_page{ |page|
+ $pagemap.seek((page/$page_size)*8)
+ str = $pagemap.read(8)
+ if str then
+ @pages[page] = Page::decode(str.unpack('Q').first)
+ end
+ }
+ end
+ def to_s
+ return @address.to_s + " " + @perms + " " + @offset.to_s(16) + " " + @device + " " + @inode.to_s + " " + @pathname
+ end
+ def add_numa(numa_map)
+ @numa_policy = numa_map.policy
+ @numa_informations = numa_map.informations
+ end
+end
+class NumaInformations < Hash
+ attr_accessor :nodes
+ def initialize(informations)
+ infos = informations.scan(/\S+/)
+ @nodes = {}
+ infos.each { |info|
+ key,value = info.split("=")
+ if num = key.scan(/N(\d+)/).first then
+ @nodes[num.pop] = value
+ else
+ self[key] = value
+ end
+ }
+ end
+ def to_s
+ s = []
+ self.each { |key, value|
+ str = "#{key}"
+ str += "=#{value}" if value
+ s.push( str )
+ }
+ @nodes.each { |key, value|
+ str = "N#{key}=#{value}"
+ s.push( str )
+ }
+ return s.join(" ")
+ end
+ def stack?
+ self.each_key { |key|
+ return true if key.match(/stack/)
+ }
+ return false
+ end
+end
+class NumaMap
+ attr_accessor :address
+ attr_accessor :policy
+ attr_accessor :informations
+ def NumaMap.convert(line)
+ address, policy, informations = line.scan(/([0-9a-fA-F]+)\s+(\w+)\s*(.*)/).first
+ address = address.to_i(16)
+ informations = NumaInformations::new(informations)
+ address += $page_size if informations.stack? #remove stack guard page
+ return NumaMap::new(address, policy, informations)
+ end
+ def initialize(address, policy, informations)
+ @address, @policy, @informations = address, policy, informations
+ end
+end
+
+def usage(msg=nil)
+ puts msg if msg
+ puts $parser.to_s
+ exit(0)
+end
+
+$options = {:match => "", :base => 16}
+
+$parser = OptionParser::new do |opts|
+ opts.banner = "Usage: pagemap [options] [pid [address[-address][,address[-address]...]]]"
+ opts.on("-b", "--base [BASE]", Integer, "Address base") do |base|
+ $options[:base] = base
+ end
+ opts.on("-y", "--[no-]yaml","YAML output") do |yaml|
+ $options[:yaml] = yaml
+ end
+ opts.on("-m", "--match [MATCH]", "Pathname match") do |match|
+ $options[:match] = match
+ end
+ opts.on("-a", "--[no-]all", "List absent pages") do |v|
+ $options[:all] = v
+ end
+ opts.on("-k", "--[no-]hack", "Use kernel hack flags (unreliable)") do |v|
+ $options[:hack] = v
+ end
+ opts.on("-s", "--[no-]symbols", "Use flags symbols instead of name") do |v|
+ $options[:symbols] = v
+ end
+ opts.on("-n", "--[no-]numa", "Add numa information") do |v|
+ $options[:numa] = v
+ end
+ opts.on("-r", "--ranges [address[-address][,address[-address]...]]", "Specify address ranges") do |range|
+ $options[:ranges] = range
+ end
+ opts.on("-d", "--[no-]debug", "Print debug messages") do |v|
+ $options[:debug] = v
+ end
+ opts.on("-h", "--help", "Show this message") do
+ puts <<EOF
+pagemap is a simple command line tool to analyze and print the physical memory
+layout of a Linux process. It is used to debug and interpret performances of
+standard or HPC applications.
+
+EOF
+ puts opts
+ exit
+ end
+ opts.on("--version", "Display version") do
+ puts "pagemap #{PAGEMAP_VERSION}"
+ exit
+ end
+ opts.on("--help-symbols", "Show flags symbols") do
+ puts opts
+ puts "Flags symbols:"
+ Flags.doc.each_line { |line| puts "\t#{line}" }
+ exit
+ end
+ opts.parse!
+end
+addresses = []
+
+if ARGV.length > 2 then
+ usage("Invalid number of arguments (#{ARGV.length})!")
+end
+
+if ARGV.length == 0 then
+ pid = Process.ppid
+else
+ pid = ARGV[0]
+ pid = pid.to_i
+ if pid == 0 then
+ usage("Invalid pid!")
+ end
+end
+
+addresses = ARGV[1].split(",") if ARGV.length == 2
+addresses += $options[:ranges].split(",") if $options[:ranges]
+
+
+$page_size = `getconf PAGESIZE`.to_i
+ps = $page_size
+$page_shift = 0
+while ps & 1 == 0 do
+ $page_shift += 1
+ ps >>= 1
+end
+$pagemap = File::open("/proc/#{pid}/pagemap")
+numa_maps_lines = nil
+$memory_architecture = nil
+if $options[:numa] then
+ begin
+ numa_maps_lines = File::open("/proc/#{pid}/numa_maps")
+ rescue Exception => e
+ $stderr.puts e if $options[:debug]
+ numa_maps_lines = nil
+ end
+ begin
+ $memory_architecture = MemoryArchitecture::new
+ rescue Exception => e
+ $stderr.puts e if $options[:debug]
+ $memory_architecture = nil
+ end
+end
+
+begin
+ $kpagecount = File::open("/proc/kpagecount")
+rescue Exception => e
+ $stderr.puts e if $options[:debug]
+ $kpagecount = nil
+end
+
+begin
+ $kpageflags = File::open("/proc/kpageflags")
+rescue Exception => e
+ $stderr.puts e if $options[:debug]
+ $kpageflags = nil
+end
+
+
+if addresses.length != 0 then
+ pages = {}
+ addresses_unranged = []
+ addresses.each { |range|
+ addrs = range.split("-")
+ addrs.collect! { |addr| addr.to_i($options[:base]) }
+ find_and_decode = lambda { |address|
+ page = nil
+ $pagemap.seek((address/$page_size)*8)
+ str = $pagemap.read(8)
+ page = Page::decode(str.unpack('Q').first) if str
+ pages[address] = page
+ addresses_unranged.push(address)
+ }
+ if addrs.length == 2 then
+ if addrs[0] % $page_size != 0 then
+ find_and_decode.call(addrs[0]) if addrs[0] % $page_size != 0
+ addrs[0] += $page_size - (addrs[0] % $page_size)
+ end
+ (addrs[0] - ( addrs[0] % $page_size )).step(addrs[1],$page_size,&find_and_decode)
+ else
+ find_and_decode.call(addrs[0])
+ end
+ }
+ if $options[:yaml] then
+ require 'yaml'
+ puts YAML::dump(pages)
+ else
+ addresses_unranged.each { |address|
+ p = pages[address]
+ if p && (!p.absent || $options[:all]) then
+ puts address.to_s(16) + " " + p.to_s
+ end
+ }
+ end
+else
+ numa_maps = {}
+ if numa_maps_lines then
+ numa_maps_lines.each { |line|
+ numa_map = NumaMap::convert(line)
+ numa_maps[numa_map.address] = numa_map if numa_map
+ }
+ end
+ maps_lines = File::open("/proc/#{pid}/maps")
+ maps = []
+ maps_lines.each { |line|
+ map = Map::convert(line,$options[:match])
+ if map then
+ numa_maps.each { |key, value|
+ map.add_numa(value) if map.address.include?(key)
+ }
+ maps.push(map)
+ end
+ }
+ if $options[:yaml] then
+ require 'yaml'
+ puts YAML::dump(maps)
+ else
+ maps.each { |map|
+ puts '## mapping: ' + map.to_s
+ puts '## mapping size: ' + (map.address.size/1024).to_s + "kB" +
+ ' / number of pages: ' + map.address.number.to_s
+ puts '## numa policy: ' + map.numa_policy.to_s + ' / numa informations: ' + map.numa_informations.to_s if $options[:numa]
+ map.address.each_page {|x|
+ p = map.pages[x]
+ if p && (!p.absent || $options[:all]) then
+ puts x.to_s(16) + " " + p.to_s
+ end
+ }
+ }
+ end
+end
--
GitLab