runtest.py

import argparse
import subprocess
import os
import json
import signal
import re
from datetime import datetime
from pathlib import Path

parser = argparse.ArgumentParser(description="A description")
parser.add_argument(
    "pkg", help="The package which is tested. Should be packages/<pkg_name>.py e.g. packages/algpath.py")
parser.add_argument(
    "data", help="The system which is tested. Shoud be data/<test_name>.json e.g. data/linear_dense-10_all-paths_1.json")
parser.add_argument("-p", "--perf", action='store_true', default=False,
                    help="perf stat the benchmark and put the result in perflog.txt")
parser.add_argument("-t", "--timeout", nargs="?", const="300",
                    help="To run the benchmark with a timeout. Should be written as <number><unit> where <unit> may be nothing (this implicitely means seconds), s, m or h, e.g. 1h")
parser.add_argument("-m", "--mem", nargs="?", const="8G",
                    help="Maximum amount of memory used. Should be written as <number><unit> where <unit> may be nothing, K, M or G, e.g. 100M")
parser.add_argument("-n", "--norun", action='store_true',
                    default=False, help="To only generate script and command")
parser.add_argument("-c", "--core", nargs="?", const="1",
                    help="Core on which the job should be ran")
args = parser.parse_args()

pkg_path = Path(args.pkg)
data_path = Path(args.data)
tmp_data_str = f"file {data_path} does not exist"
tmp_pkg_str = f"file {pkg_path} does not exist"

assert data_path.exists(), tmp_data_str
assert pkg_path.exists(), tmp_pkg_str

data = data_path.stem
pkg = pkg_path.stem

pkg_binary_dict = {"homotopycontinuation": "julia", "sirocco": "sage",
                   "macaulay2": "M2", "algpath": "algpath", "adaptive": "packages/adaptive"}

print(f"Benchmarking {pkg} on {data_path}" + args.perf*" (with perf stat)")

print("Creating benchmark folder...")
bench_path = (Path("benchmarks").joinpath(
    *data_path.with_suffix("").parts[1:]) / pkg)
bench_path.mkdir(parents=True, exist_ok=True)

print("Generating script and command...")

auxi = Path("/".join([".." for _ in bench_path.parts]))

if pkg in ["adaptive"]:
    pkg_binary_path = auxi / pkg_binary_dict[pkg]
else:
    pkg_binary_path = pkg_binary_dict[pkg]

os.chdir(bench_path)
subprocess.run(["python3", f"{auxi / pkg_path}",
               f"{auxi / data_path}", f"{pkg_binary_path}"])
subprocess.run(["chmod", "+x", str("command.sh")])
# assert 1 == 0

if not args.norun:
    print(f"Timeout: {args.timeout}")
    print(f"Maximum amount of memory used: {args.mem}")
    info_file = open("info.json", "w")

    # Functions to parse input
    def memory_parsing(s):
        auxi = re.split('(\d+)', s)
        val = int(auxi[1])
        try:
            unit = auxi[2]
        except:
            return val
        else:
            match unit:
                case "K":
                    return val*(2**10)
                case "M":
                    return val*(2**20)
                case "G":
                    return val*(2**30)
                case _:
                    return val

    def timeout_parsing(s):
        auxi = re.split('(\d+)', s)
        val = int(auxi[1])
        try:
            unit = auxi[2]
        except:
            return val
        else:
            match unit:
                case "s":
                    return val
                case "m":
                    return val*60
                case "h":
                    return val*3600
                case _:
                    return val

    if args.timeout != None:
        timeout_s = timeout_parsing(args.timeout)
    else:
        timeout_s = None

    if args.mem != None:
        memory_b = memory_parsing(args.mem)
    else:
        memory_b = None

    memory_max = str(args.mem)
    info_dict = {
        "datetime": str(datetime.now()),
        "return code": None,
        "timeout": timeout_s,
        "timeout error": False,
        "memory": memory_b,
        "memory error": False,
        "script error": False,
        "killed manually": True,
    }

    json.dump(info_dict, info_file, indent=2)
    info_file.close()

    out_file = open("out.json", "w")
    log_file = open("log.txt", "w")

    print("Running script...")

    cmd = ["systemd-run", "--scope", "--user", "-p", f"MemoryMax={memory_b}", "-p", "MemorySwapMax=0"]*(
        memory_b != None) + ["perf", "stat", "-o", "perflog.txt"]*args.perf + ["taskset", args.core, "./command.sh"]

    p = subprocess.Popen(cmd, start_new_session=True,
                         stdout=out_file, stderr=log_file)
    try:
        _, _ = p.communicate(timeout=timeout_s)
        log_file.close()
        log_file = open("log.txt", "r")
        log = log_file.read()
        try:
            user_unit = re.findall(r"run.*scope", log)[0]
            log_systemd = subprocess.run(
                ["journalctl", "--user-unit", user_unit], capture_output=True).stdout
        except:
            log_systemd = ""

        if "A process of this unit has been killed by the OOM killer" in str(log_systemd):
            info_dict["memory error"] = True
        elif p.returncode != 0:
            info_dict["script error"] = True

    except subprocess.TimeoutExpired:
        print(f'Timeout ({args.timeout})',)
        print('Terminating the whole process group...')
        os.killpg(os.getpgid(p.pid), signal.SIGTERM)
        _, _ = p.communicate()
        info_dict["timeout error"] = True
    except:
        os.killpg(os.getpgid(p.pid), signal.SIGTERM)
        _, _ = p.communicate()
        

    out_file.close()
    log_file.close()

    info_dict["killed manually"] = False
    info_dict["return code"] = p.returncode
    info_file = open("info.json", "w")
    json.dump(info_dict, info_file, indent=2)
    info_file.close()

    out_file = open("out.json", "r")
    try:
        result = json.load(out_file)
    except:
        out_file.close()
    else:
        out_file.close()
        out_file = open("out.json", "w")
        json.dump(result, out_file, indent=2)