From 3f918f55e5598f2b6afae4e8ddb0f9cbe2a29c34 Mon Sep 17 00:00:00 2001
From: Kevin Pouget <kevin.pouget@imag.fr>
Date: Tue, 27 Sep 2016 14:13:37 +0200
Subject: [PATCH] work on omp profiling
---
model/__init__.py | 2 +-
model/numa/__init__.py | 52 +++++++++----
model/profiling/info/__init__.py | 2 +-
model/profiling/info/perf_standalone.py | 2 -
model/profiling/interaction/graph.py | 74 +++++++++++++++++--
model/profiling/interaction/summary.py | 6 +-
.../openmp/capture/iomp/kmpc_for_static.py | 20 +++--
.../environment/openmp/interaction/loop.py | 14 ++--
.../environment/openmp/interaction/profile.py | 6 +-
9 files changed, 136 insertions(+), 42 deletions(-)
diff --git a/model/__init__.py b/model/__init__.py
index 5827848..5891cdd 100644
--- a/model/__init__.py
+++ b/model/__init__.py
@@ -1,2 +1,2 @@
-from . import gpu, task, profiling
+from . import gpu, task, profiling, numa
diff --git a/model/numa/__init__.py b/model/numa/__init__.py
index 45ac7d8..07a9fd1 100644
--- a/model/numa/__init__.py
+++ b/model/numa/__init__.py
@@ -7,13 +7,14 @@ import logging; log = logging.getLogger(__name__)
log_user = logging.getLogger("mcgdb.log.user.numa")
import os
+import subprocess
try:
import shutil
except ImportError:
shutil = None
-numa_loaded = False
+numa_loaded = None
PAGEMAP_PATH = "/home/videau/"
PAGEMAP = "pagemap"
@@ -22,14 +23,22 @@ pagemap_bin = None
class cmd_numa_pagemap(gdb.Command):
def __init__ (self):
- gdb.Command.__init__ (self, "pagemap", gdb.COMMAND_NONE)
+ gdb.Command.__init__ (self, "numa pagemap", gdb.COMMAND_NONE)
global pagemap_bin
- pagemap_bin = PAGEMAP
-
- if (shutil and shutil.which(PAGEMAP) # > py3.3
- or "/{}".format(PAGEMAP) in os.system("which {}").format(PAGEMAP))):
- # nothing to do, we found it
- else:
+
+ has_pagepage = False
+ try:
+ has_pagepage = shutil.which(PAGEMAP) is not None # > py3.3
+ except AttributeError: # py2: shutil doesn't have which
+ pass
+
+ if not has_pagepage:
+ try:
+ has_pagepage = "/{}".format(PAGEMAP) in subprocess.check_output("which {}".format(PAGEMAP), shell=True, stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError: # which returned false
+ pass
+
+ if not has_pagepage:
pagemap_path = os.path.join(PAGEMAP_PATH, PAGEMAP)
if not os.path.isfile(pagemap_path):
@@ -39,9 +48,21 @@ class cmd_numa_pagemap(gdb.Command):
pagemap_bin = pagemap_path
def invoke (self, args, from_tty):
- addr = gdb.parse_and_eval(args)
- os.system("{} -n {} {}".format(pagemap_bin, gdb.selected_inferior().pid, addr))
+ try:
+ addr = gdb.parse_and_eval(args)
+ except gdb.error as e:
+ log_user.error("Cannot evaluate '{}': {}".format(args, e))
+ return
+
+ cmd = "{} -n {} {}".format(pagemap_bin, gdb.selected_inferior().pid, addr)
+ res = subprocess.check_output(cmd, shell=True)
+ try:
+ pm_addr, _, node = res[:-1].split(" ")
+ log_user.info("Address 0x{} is located on node {}".format(pm_addr, node))
+ except ValueError: # couldnt split correctly
+ log_user.warn("Unexpected response from pagemap: '{}'".format(res[:-1]))
+
class cmd_numa_current_node(gdb.Command):
def __init__ (self):
gdb.Command.__init__ (self, "numa current_node", gdb.COMMAND_NONE)
@@ -49,12 +70,13 @@ class cmd_numa_current_node(gdb.Command):
def invoke (self, args, from_tty):
for symb in ("numa_node_of_cpu", "sched_getcpu"):
if not gdb.lookup_symbol(symb):
- log_user.error("Symbol '{}' not found, cannot get current node.".format(symb)
- return
+ log_user.error("Symbol '{}' not found, cannot get current node.".format(symb))
+ return
with my_gdb.set_parameter("scheduler-locking", "on"):
- gdb.execute("p numa_node_of_cpu(sched_getcpu())")
-
+ node = gdb.execute("p numa_node_of_cpu(sched_getcpu())", to_string=True)
+ node = int(node.partition("= ")[-1])
+ log_user.info("Current node is N{}".format(node))
class cmd_numa(gdb.Command):
def __init__(self, limited=False):
@@ -69,7 +91,7 @@ class cmd_numa(gdb.Command):
on_activated()
numa_loaded = True
-
+ log_user.info("Numa module successfully loaded.")
if args:
gdb.execute("numa {}".format(args))
diff --git a/model/profiling/info/__init__.py b/model/profiling/info/__init__.py
index c8d8b6c..90beb36 100644
--- a/model/profiling/info/__init__.py
+++ b/model/profiling/info/__init__.py
@@ -6,7 +6,7 @@ log_user = logging.getLogger("mcgdb.log.user")
from mcgdb.toolbox import my_gdb
# will be added to global scope
-info_counters = "perf_standalone", #"proc", "hit", "counter", "perf", "gprof", "memleaks"
+info_counters = "perf_standalone", "omp" #"proc", "hit", "counter", "perf", "gprof", "memleaks"
info_counter_classes = None # will be populated in initialize()
disabled_counters = set()
diff --git a/model/profiling/info/perf_standalone.py b/model/profiling/info/perf_standalone.py
index b131a48..dba3ef9 100644
--- a/model/profiling/info/perf_standalone.py
+++ b/model/profiling/info/perf_standalone.py
@@ -41,7 +41,6 @@ class perf_info_standalone():
import signal, time, os
old = signal.signal(signal.SIGUSR1, handler)
- log.error("start perf")
command = ['perf', 'stat', '-x,',
'-e', counters.param_perf_counters.get()
]
@@ -56,7 +55,6 @@ class perf_info_standalone():
self.perf = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.res = "running"
assert self.perf.pid > 1
- log.error("perf pid is {}".format(self.perf.pid))
if not DISABLE_STARTUP_SYNC:
cnt = MAX_WAIT
diff --git a/model/profiling/interaction/graph.py b/model/profiling/interaction/graph.py
index 17ed168..4d6de6b 100644
--- a/model/profiling/interaction/graph.py
+++ b/model/profiling/interaction/graph.py
@@ -49,10 +49,8 @@ def comparison(record, name):
plotly.offline.plot(fig, filename='comparison.html')
def sequential(record, name):
- print('chart.add("{}", {})'.format(name, record))
- print()
- with open("data.gplt", "w") as out:
- out.write("")
+ print('{} | {}'.format(name, " ".join(map(str, record))))
+
return
data = [plotly.graph_objs.Bar(
@@ -152,7 +150,7 @@ class cmd_profile_graph_plot (gdb.Command):
def invoke (self, args, from_tty, show_data=False, show_keys=False):
prof_id, _, key = args.partition(" ")
- prof_type = "first"
+ prof_type = "first last"
if prof_id.isdigit():
prof_id = int(prof_id)
@@ -170,6 +168,10 @@ class cmd_profile_graph_plot (gdb.Command):
if prof_id != "all":
prof_id = int(prof_id)
+
+ if not key:
+ raise Exception("no key ...")
+
except Exception:
log_user.error("Expected [first|soustractive|function] <profile id|all> <key> arguments.")
return
@@ -183,11 +185,14 @@ class cmd_profile_graph_plot (gdb.Command):
continue
found = True
-
+
+ if prof.running:
+ log_user.warn("Skipping {} #{}, it's running".format(prof, prof.numbers[profile.Profile]))
+ continue
+
thread_key = prof.thread_key
to_graph = defaultdict(list)
-
if show_keys:
per_thread = prof.all_per_thead_infos[0][thread_key]
log_user.info("Profile id={}".format(prof_id))
@@ -210,6 +215,9 @@ class cmd_profile_graph_plot (gdb.Command):
continue
if prof_id == "all":
+ if to_graph[prof_type][0] is None:
+ pass # what to do here ??
+
to_all_graph.append(to_graph[prof_type][0])
continue
@@ -281,3 +289,55 @@ def initialize():
from .. import profile
cmd_profile_graph()
+ cmd_profile_offline_graph()
+
+class cmd_profile_offline_graph(gdb.Command):
+ def __init__ (self):
+ gdb.Command.__init__(self, "profile graph offline", gdb.COMMAND_OBSCURE, prefix=True)
+
+ def invoke (self, args, from_tty):
+ import pygal
+
+ import fileinput
+
+ chart = pygal.Line()
+ names = []
+ do_div = False
+
+ finput = fileinput.input()
+ for line in finput:
+ line = line[:-1]
+
+ if not line:
+ finput.close()
+ break
+
+ if line.endswith(" y2"):
+ secondary = True
+ line = line.replace(" y2", "")
+ elif line.endswith(" /"):
+ do_div = True
+ line = line.replace(" /", "")
+ else:
+ secondary = False
+
+ try:
+ name, values = line.split(" | ")
+
+ cast = float if "." in values else int
+ values = list(map(cast, values.split(" ")))
+ except Exception as e:
+ log_user.warn("Parsing failed, the line above will be ignored ({})".format(e))
+ continue
+ if do_div is True:
+ do_div = values
+ continue
+ elif do_div:
+ values = [a/b for a, b in zip(values, do_div)]
+
+ chart.add(name, values, secondary=secondary)
+ names.append(name)
+
+ TARGET = '/tmp/chart.png'
+ chart.render_to_png(TARGET)
+ log_user.info("Chart of {} plotted into {}".format("/".join(names), TARGET))
diff --git a/model/profiling/interaction/summary.py b/model/profiling/interaction/summary.py
index 69799d2..75ec962 100644
--- a/model/profiling/interaction/summary.py
+++ b/model/profiling/interaction/summary.py
@@ -231,16 +231,18 @@ class InfoEntrySummary():
def avg(self, value, prof):
if isinstance(value, str): return
+
try:
- prev, it = self.values["avg"]
+ (prev, _what_), it = self.values["avg"]
it += 1
# iterative mean http://www.heikohoffmann.de/htmlthesis/node134.html
new = prev + float(value) - prev*1/it
except KeyError: # first pass
new = value
it = 0
- except:
+ except Exception as e:
import pdb;pdb.set_trace()
+ return
self.values["avg"] = new, it
class cmd_profile_summ (gdb.Command):
diff --git a/model/task/environment/openmp/capture/iomp/kmpc_for_static.py b/model/task/environment/openmp/capture/iomp/kmpc_for_static.py
index 16a397c..b799239 100644
--- a/model/task/environment/openmp/capture/iomp/kmpc_for_static.py
+++ b/model/task/environment/openmp/capture/iomp/kmpc_for_static.py
@@ -50,11 +50,11 @@ class kmpc_for_static_fini_Breakpoint(OmpFunctionBreakpoint):
def prepare_before (self):
data = {}
- worker = current_worker()
+ data["worker"] = worker = current_worker()
data["loop"], data["iteration"] = loop, iteration \
= representation.ForLoopJob.get_current_loop_of(worker)
-
+
if not loop:
log.warn("spurious end")
return
@@ -63,14 +63,20 @@ class kmpc_for_static_fini_Breakpoint(OmpFunctionBreakpoint):
stop_after = loop.break_after_next
loop.break_after_next = False
-
+
+ data["old-scheduler-locking"] = gdb.parameter("scheduler-locking")
+ if stop_after:
+ gdb.execute("set scheduler-locking on")
+
return False, stop_after, data
def prepare_after(self, data):
- iteration = loop.working
- log_user.info("Stopped after loop {} iteration {}-{}.".format(data["loop"],
- data["iteration"][0],
- data["iteration"][1]))
+ gdb.execute("set scheduler-locking {}".format(data["old-scheduler-locking"]))
+ log.warn("Stopped in worker {} after loop {} iteration {}-{}.".format(
+ data["worker"],
+ data["loop"],
+ data["iteration"][0],
+ data["iteration"][1]))
return True
diff --git a/model/task/environment/openmp/interaction/loop.py b/model/task/environment/openmp/interaction/loop.py
index 6771e4c..eb8f1f9 100644
--- a/model/task/environment/openmp/interaction/loop.py
+++ b/model/task/environment/openmp/interaction/loop.py
@@ -50,7 +50,7 @@ class cmd_omp_loop_break (gdb.Command):
class cmd_omp_loop_break_after_next (gdb.Command):
def __init__ (self):
gdb.Command.__init__(self, "omp loop break after_next", gdb.COMMAND_OBSCURE, prefix=1)
-
+
def invoke (self, args, from_tty):
last_loop = representation.ForLoopJob.get_last_loop()
@@ -58,17 +58,19 @@ class cmd_omp_loop_break_after_next (gdb.Command):
log_user.warn("Could not find any last loop")
return
- action = "Enable" if not self.break_after_next else "Disable"
-
- log_user.info("{} break-after-next breakpoint on loop {}.".format(action, last_loop))
- self.break_after_next = not self.break_after_next
+ action = "Enable" if not last_loop.break_after_next else "Disable"
-iteration_profiling_enabled = False
+ log_user.info("{} break-after-next breakpoint on loop {}.".format(action, repr(last_loop)))
+ last_loop.break_after_next = not last_loop.break_after_next
+iteration_profiling_enabled = False
class cmd_omp_loop_profile (gdb.Command):
def __init__ (self):
gdb.Command.__init__(self, "omp loop profile", gdb.COMMAND_OBSCURE, prefix=True)
+
+ def invoke (self, arg, from_tty):
+ log_user.warn("Please specify what kind of profiling you want to do.")
class cmd_omp_loop_profile_iterations (gdb.Command):
def __init__ (self):
diff --git a/model/task/environment/openmp/interaction/profile.py b/model/task/environment/openmp/interaction/profile.py
index 13e5b45..c6674e2 100644
--- a/model/task/environment/openmp/interaction/profile.py
+++ b/model/task/environment/openmp/interaction/profile.py
@@ -1,3 +1,6 @@
+import logging; log = logging.getLogger(__name__)
+log_user = logging.getLogger("mcgdb.log.user.openmp.profile")
+
import gdb
from mcgdb.toolbox import my_gdb
@@ -15,4 +18,5 @@ class cmd_omp_profile (gdb.Command):
def __init__ (self):
gdb.Command.__init__(self, "omp profile", gdb.COMMAND_OBSCURE, prefix=True)
-
+ def invoke (self, arg, from_tty):
+ log_user.warn("Please specify what kind of profiling you want to do.")
--
GitLab