Verified Commit ff701e59 authored by Raphaël Bleuse's avatar Raphaël Bleuse
Browse files

Remove pybatsim.batsim.cmds module

The module provided two scripts to launch schedulers:
- `pybatsim.batsim.cmds.launcher:main` has been replaced by
  `pybatsim.cmdline:main`
- `pybatsim.batsim.cmds.experiments:main` is an experiments orchestrator
  and is out of the scope of this project: see [1] for an alternative

[1]: https://gitlab.inria.fr/batsim/batexpe
parent b687a650
......@@ -21,8 +21,6 @@ let
"^src/pybatsim/.\+\.py$"
"^src/pybatsim/batsim$"
"^src/pybatsim/batsim/.\+\.py$"
"^src/pybatsim/batsim/cmds$"
"^src/pybatsim/batsim/cmds/.\+\.py$"
"^src/pybatsim/batsim/tools$"
"^src/pybatsim/batsim/tools/.\+\.py$"
"^src/pybatsim/schedulers$"
......
......@@ -66,14 +66,6 @@ python-versions = ">=3.6"
[package.extras]
toml = ["tomli"]
[[package]]
name = "docopt"
version = "0.6.2"
description = "Pythonic argument parser, that will make you smile"
category = "main"
optional = false
python-versions = "*"
[[package]]
name = "docutils"
version = "0.17.1"
......@@ -386,7 +378,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes
[metadata]
lock-version = "1.1"
python-versions = "^3.7.1"
content-hash = "b86a81693c2375c86690763221982508ddd6a5da7d46a8a901e5e3c7c6d78890"
content-hash = "c6a54fb8c088b78420b5f97d2ba0ba20238e7be75d94e6e94d81e369d996d6aa"
[metadata.files]
alabaster = [
......@@ -510,9 +502,6 @@ coverage = [
{file = "coverage-6.2-pp36.pp37.pp38-none-any.whl", hash = "sha256:5829192582c0ec8ca4a2532407bc14c2f338d9878a10442f5d03804a95fac9de"},
{file = "coverage-6.2.tar.gz", hash = "sha256:e2cad8093172b7d1595b4ad66f24270808658e11acf43a8f95b41276162eb5b8"},
]
docopt = [
{file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"},
]
docutils = [
{file = "docutils-0.17.1-py2.py3-none-any.whl", hash = "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61"},
{file = "docutils-0.17.1.tar.gz", hash = "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125"},
......
......@@ -43,7 +43,6 @@ python = "^3.7.1"
procset = "^1.0"
sortedcontainers = "^2.3.0"
pyzmq = "^22.0.3"
docopt = "^0.6.2"
importlib-metadata = {version = ">=3.6", python = "<3.10"}
[tool.poetry.dev-dependencies]
......@@ -52,8 +51,6 @@ Sphinx = "^4.3.1"
[tool.poetry.scripts]
pybatsim = "pybatsim.cmdline:main"
pybatsim-legacy = "pybatsim.batsim.cmds.launcher:main"
pybatsim-experiment = "pybatsim.batsim.cmds.experiments:main"
# definition of officialy maintained schedulers
[tool.poetry.plugins."pybatsim.schedulers"]
......
"""
batsim.cmds
~~~~~~~~~~~
Additional tools installed in the path.
"""
'''
Run PyBatsim Experiments.
Usage:
pybatsim-experiment <experiment> [options]
Options:
--version Print the version of pybatsim and exit
-h --help Show this help message and exit.
-q --quiet Silent experiment output.
-d --debug Print additional debug messages.
'''
import sys
import json
from docopt import docopt
from pybatsim.batsim.tools.experiments import launch_experiment
from pybatsim import __version__
def main():
arguments = docopt(__doc__, version=__version__)
verbose = not bool(arguments["--quiet"])
debug = bool(arguments["--debug"])
options_file = arguments["<experiment>"]
try:
with open(options_file) as f:
options = json.loads(f.read())
except FileNotFoundError:
if debug:
raise
print("Experiment file does not exist: {}".format(
options_file), file=sys.stderr)
sys.exit(1)
except Exception:
if debug:
raise
print("Error in json file: {}".format(options_file), file=sys.stderr)
sys.exit(1)
options["options-file"] = options_file
if verbose:
print("Running experiment: {}".format(options_file))
return launch_experiment(options, verbose=verbose)
if __name__ == "__main__":
sys.exit(main())
'''
Run PyBatsim Schedulers.
Usage:
pybatsim <scheduler> [-o <options_string>] [options]
Options:
--version Print the version of pybatsim and exit
-h --help Show this help message and exit.
-v --verbosity=<verbosity-level> Sets the verbosity level. Available
values are {debug, info, warning, error, critical}
Default: info
-s --socket-endpoint=<endpoint> Batsim socket endpoint to use [default: tcp://*:28000]
-e --event-socket-endpoint=<endpoint> Socket endpoint to use to publish scheduler events
-o --options=<options_string> A Json string to pass to the scheduler [default: {}]
-O --options-file=<options_file> A file containing the json options
-t --timeout=<timeout> How long to wait for responses from Batsim [default: 2000]
'''
import sys
import json
import logging
from docopt import docopt
from pybatsim.batsim.tools.launcher import launch_scheduler, instanciate_scheduler
from pybatsim import __version__
def main():
arguments = docopt(__doc__, version=__version__)
loglevel = logging.WARNING
if not arguments['--verbosity']:
loglevel = logging.INFO
else:
loglevel = logging.getLevelName(arguments['--verbosity'].upper())
FORMAT = '[pybatsim - %(asctime)s - %(name)s - %(levelname)s] %(message)s'
logging.basicConfig(format=FORMAT, level=loglevel)
timeout = int(arguments['--timeout'] or float("inf"))
if arguments["--options-file"]:
with open(arguments["--options-file"]) as options_file:
options = json.load(options_file)
elif arguments["--options"]:
options = json.loads(arguments['--options'])
else:
options = {}
scheduler_filename = arguments['<scheduler>']
socket_endpoint = arguments['--socket-endpoint']
event_socket_endpoint = arguments['--event-socket-endpoint']
scheduler = instanciate_scheduler(scheduler_filename, options=options)
return launch_scheduler(scheduler,
socket_endpoint,
event_socket_endpoint,
options,
timeout)
if __name__ == "__main__":
sys.exit(main())
"""
batsim.tools.experiments
~~~~~~~~~~~~~~~~~~~~~~~~
Tools to launch experiments.
"""
import subprocess
import os
import os.path
import sys
import json
import time
import signal
import functools
from pybatsim.batsim.network import NetworkHandler
def is_executable(file):
return os.access(file, os.X_OK)
def get_value(options, keys, fallback_keys=None, default=None):
original_options = options
if not isinstance(keys, list):
keys = [keys]
try:
for k in keys:
options = options[k]
except KeyError:
if fallback_keys:
return get_value(original_options, fallback_keys, default=default)
elif default is not None:
return default
else:
print(
"Option is missing in experiment settings ({})".format(
".".join(keys)),
file=sys.stderr)
sys.exit(1)
return options
def delete_key(options, keys):
if not isinstance(keys, list):
keys = [keys]
try:
for k in keys[:-1]:
options = options[k]
del options[keys[-1]]
except KeyError:
pass
def tail(f, n):
return subprocess.check_output(["tail", "-n", str(n), f]).decode("utf-8")
def truncate_string(s, len_s=30):
if len(s) > len_s:
return "..." + s[len(s) - len_s:]
return s
def get_terminal_size():
try:
rows, columns = subprocess.check_output(["stty", "size"]).split()
return int(rows), int(columns)
except Exception:
return None, None
def print_separator(header=None):
rows, columns = get_terminal_size()
columns = columns or 30
if header:
header = truncate_string(header, columns // 3)
header = " " + header + " "
rem_line_len_part = (columns - len(header)) // 2
line = ("".ljust(rem_line_len_part, "=") +
header +
"".ljust(rem_line_len_part, "="))
else:
line = "".ljust(columns, "=")
print(line)
def check_print(header, s):
if s:
print_separator(header)
print(s)
def execute_cl(
name,
cl,
stdout=None,
stderr=None,
on_failure=None,
verbose=False):
if verbose:
print("Starting: {}".format(" ".join(cl)), end="")
if stdout:
print(" 1>{}".format(stdout.name), end="")
if stderr:
print(" 2>{}".format(stderr.name), end="")
print()
exec = subprocess.Popen(
cl, stdout=stdout, stderr=stderr)
exec.name = name
return exec
def terminate_cl(p, terminate=False):
try:
print("Terminating {}".format(p.name), file=sys.stderr)
except AttributeError:
print("Terminating subprocess", file=sys.stderr)
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
p.wait()
if terminate:
sys.exit(3)
def run_workload_script(options, verbose):
script = options["batsim"]["workload-script"]["path"]
interpreter = options["batsim"]["workload-script"].get("interpreter", None)
args = [str(s) for s in options["batsim"]
["workload-script"].get("args", [])]
def do_run_script(cmds):
out_workload_file_path = os.path.join(
options["output-dir"], "workload.json")
with open(out_workload_file_path, "w") as f:
script_exec = execute_cl(script, cmds, stdout=f, verbose=verbose)
ret = script_exec.wait()
if ret != 0:
raise ValueError(
"Workload script {} failed with return code {}".format(
script, ret))
return out_workload_file_path
if not os.path.exists(script):
raise ValueError("Workload script {} does not exist".format(script))
if interpreter:
return do_run_script([interpreter, script] + args)
else:
if is_executable(script):
return do_run_script(["./" + script] + args)
elif script.endswith(".py"):
return do_run_script(["python3", script] + args)
else:
raise ValueError(
"Workload script {} is not executable but also does not seem to be a python script.".format(script))
def generate_config(options):
out_config_file_path = os.path.join(
options["output-dir"], "config.json")
with open(out_config_file_path, "w") as f:
f.write(json.dumps(options["batsim"]["config"], indent=4))
return out_config_file_path
def prepare_batsim_cl(options, verbose):
if "batsim" not in options:
print("batsim section is missing in experiment settings", file=sys.stderr)
sys.exit(1)
batsim_cl = [
get_value(
options, [
"batsim", "executable", "path"], [
"batsim", "bin"], default="batsim")]
batsim_cl += get_value(options, ["batsim",
"executable", "args"], default=[])
delete_key(options, ["batsim", "executable"])
delete_key(options, ["batsim", "bin"])
batsim_cl.append(
'--export=' +
os.path.join(
options["output-dir"],
options.get("export", "out")))
if "workload-script" in options["batsim"]:
options["batsim"]["workload"] = run_workload_script(options, verbose)
delete_key(options, ["batsim", "workload-script"])
if "config" in options["batsim"]:
options["batsim"]["config-file"] = generate_config(options)
delete_key(options, ["batsim", "config"])
for key, val in options.get("batsim", {}).items():
if not key.startswith("_"):
if isinstance(val, bool):
if not val:
continue
val = ""
else:
val = "=" + str(val)
batsim_cl.append("--" + key + val)
return batsim_cl
def prepare_scheduler_cl(options, verbose):
if "scheduler" not in options:
print(
"scheduler section is missing in experiment settings",
file=sys.stderr)
sys.exit(1)
sched_cl = []
if 'interpreter' in options["scheduler"]:
if options["scheduler"]["interpreter"] == "coverage":
interpreter = ["python3", "-m", "coverage", "run", "-a"]
elif options["scheduler"]["interpreter"] == "pypy":
interpreter = ["pypy", "-OOO"]
elif options["scheduler"]["interpreter"] == "profile":
interpreter = ["python3", "-m", "cProfile", "-o", "simul.cprof"]
else:
assert False, "Unknown interpreter"
sched_cl += interpreter
launcher_path = "launcher.py"
if 'srcdir' in options["scheduler"]:
launcher_path = os.path.join(
options["scheduler"].get(
"srcdir", "."), launcher_path)
sched_cl.append(launcher_path)
else:
sched_cl.append("pybatsim")
sched_cl.append(options["scheduler"]["name"])
try:
sched_options = options["scheduler"]["options"]
except KeyError:
sched_options = {}
sched_options["export-prefix"] = os.path.join(
options["output-dir"], options.get("export", "out"))
sched_cl.append("-o")
sched_cl.append(json.dumps(sched_options))
if options["scheduler"].get("verbose", False):
sched_cl.append('-v')
if "socket-endpoint" in options["scheduler"]:
sched_cl.append('-s')
sched_cl.append(options["scheduler"]["socket-endpoint"])
if "event-socket-endpoint" in options["scheduler"]:
sched_cl.append('-e')
sched_cl.append(options["scheduler"]["event-socket-endpoint"])
if "timeout" in options["scheduler"]:
sched_cl.append('-t')
sched_cl.append(options["scheduler"]["timeout"])
return sched_cl
def prepare_exec_cl(options, name):
path = options[name].get("path")
args = options[name].get("args", [])
if path:
return [path] + args
def launch_experiment(options, verbose=True):
if options.get("output-dir", "SELF") == "SELF":
options["output-dir"] = os.path.dirname("./" + options["options-file"])
if not os.path.exists(options["output-dir"]):
os.makedirs(options["output-dir"])
batsim_cl = prepare_batsim_cl(options, verbose)
sched_cl = prepare_scheduler_cl(options, verbose)
if "pre" in options:
pre_cl = prepare_exec_cl(options, "pre")
if verbose:
print()
pre_exec = execute_cl("pre", pre_cl, verbose=verbose)
pre_exec.wait()
if pre_exec.returncode != 0:
sys.exit(pre_exec.returncode)
with open(os.path.join(options["output-dir"], "batsim.stdout"), "w") as batsim_stdout_file, \
open(os.path.join(options["output-dir"], "batsim.stderr"), "w") as batsim_stderr_file, \
open(os.path.join(options["output-dir"], "sched.stdout"), "w") as sched_stdout_file, \
open(os.path.join(options["output-dir"], "sched.stderr"), "w") as sched_stderr_file:
if verbose:
print()
batsim_exec = execute_cl(
batsim_cl[0],
batsim_cl,
stdout=batsim_stdout_file,
stderr=batsim_stderr_file,
verbose=verbose)
if verbose:
print()
sched_exec = execute_cl(
sched_cl[0],
sched_cl,
stdout=sched_stdout_file,
stderr=sched_stderr_file,
on_failure=functools.partial(
terminate_cl,
batsim_exec,
terminate=True),
verbose=verbose)
event_socket_connect = options.get(
"event-socket-connect", "tcp://localhost:28001")
network_client = NetworkHandler(
socket_endpoint=event_socket_connect, timeout=100)
network_client.subscribe()
try:
if verbose:
print("\nSimulation is in progress:")
while True:
if batsim_exec.poll() is not None:
if verbose:
print()
break
elif sched_exec.poll() is not None:
if verbose:
print()
break
time.sleep(0.5)
if verbose:
while True:
status = network_client.recv_string()
if status is None:
break
rows, columns = get_terminal_size()
if columns:
if len(status) > columns:
status = status[:columns - 3] + "..."
status = status.ljust(columns)
print(status, end='\r', flush=True)
except KeyboardInterrupt:
print("\nSimulation was aborted => Terminating batsim and the scheduler")
terminate_cl(batsim_exec)
terminate_cl(sched_exec)
if sched_exec.poll() is not None and sched_exec.returncode != 0 and batsim_exec.poll() is None:
print("Scheduler has died => Terminating batsim")
terminate_cl(batsim_exec)
if batsim_exec.poll() is not None and batsim_exec.returncode != 0 and sched_exec.poll() is None:
print("Batsim has died => Terminating the scheduler")
terminate_cl(sched_exec)
sched_exec.wait()
batsim_exec.wait()
ret_code = abs(batsim_exec.returncode) + abs(sched_exec.returncode)
if verbose or ret_code != 0:
print()
check_print(
"Excerpt of log: " +
batsim_stdout_file.name,
tail(
batsim_stdout_file.name,
5))
check_print(
"Excerpt of log: " +
batsim_stderr_file.name,
tail(
batsim_stderr_file.name,
5))
check_print(
"Excerpt of log: " +
sched_stdout_file.name,
tail(
sched_stdout_file.name,
5))
check_print(
"Excerpt of log: " +
sched_stderr_file.name,
tail(
sched_stderr_file.name,
5))
print("Scheduler return code: " + str(sched_exec.returncode))
print("Batsim return code: " + str(batsim_exec.returncode))
if ret_code == 0:
if "post" in options:
post_cl = prepare_exec_cl(options, "post")
if verbose:
print()