reserve_and_test.py 3.17 KB
Newer Older
1 2 3
#!/usr/bin/env python
import logging

Maverick Chardet's avatar
Maverick Chardet committed
4 5
from experiment_utilities.remote_host import RemoteHost
from experiment_utilities.reserve_g5k import G5kReservation
6 7 8 9 10 11 12 13 14 15 16 17 18

CONCERTO_GIT = 'https://gitlab.inria.fr/mchardet/madpp.git'
CONCERTO_DIR_IN_GIT = 'madpp'
EXP_DIR_IN_GIT = CONCERTO_DIR_IN_GIT + '/tests/ssh_scalability'
ROOT_DIR = '~/concertonode'
PYTHON_FILE = 'assembly.py'

CONCERTO_DIR = '%s/%s' % (ROOT_DIR, CONCERTO_DIR_IN_GIT)
EXP_DIR = '%s/%s' % (ROOT_DIR, EXP_DIR_IN_GIT)

DEFAULT_WORKING_DIRECTORY = '.'


Maverick Chardet's avatar
Maverick Chardet committed
19
def run_experiment(list_nb_remote_ssh, nb_repeats, conf, working_directory=DEFAULT_WORKING_DIRECTORY,
20 21 22 23 24 25 26 27 28 29 30 31
                   force_deployment=True, destroy=False):
    from json import dump

    with G5kReservation(conf, force_deployment, destroy) as g5k:
        remote_machines = g5k.get_hosts_info(role='remote')
        concerto_machine = g5k.get_hosts_info(role='concerto')[0]
        print("Remote: %s" % str(remote_machines))
        print("Concerto: %s" % str(concerto_machine))
        concerto_config = {
            "remote_hosts": remote_machines,
            "concerto_host": concerto_machine,
            "list_nb_remote_ssh": list_nb_remote_ssh,
Maverick Chardet's avatar
Maverick Chardet committed
32
            "nb_repeats": nb_repeats
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
        }
        with open(working_directory + "/concerto_config.json", "w") as concerto_config_file:
            dump(concerto_config, concerto_config_file)

        with RemoteHost(concerto_machine["address"], remote_user="root") as concerto_host:
            run_cmd = "mkdir -p %s;" % ROOT_DIR + \
                      "cd %s;" % ROOT_DIR + \
                      "git clone %s;" % CONCERTO_GIT
            print("Executing commands: %s" % run_cmd)
            concerto_host.run(run_cmd)
            concerto_host.send_files(
                local_files=[working_directory + "/concerto_config.json"],
                remote_location=EXP_DIR
            )
            concerto_host.send_files(
                local_files=["~/.ssh/id_rsa", "~/.ssh/id_rsa.pub"],
                remote_location="~/.ssh"
            )
            run_cmd = "cd %s;" % CONCERTO_DIR + \
                      "source source_dir.sh;" + \
                      "cd %s;" % EXP_DIR + \
                      "timeout -k 10s 30m python3 %s >stdout 2>stderr" % PYTHON_FILE
            print("Executing commands: %s" % run_cmd)
            concerto_host.run(run_cmd)
            files_to_get_names = ['stdout', 'stderr', 'results.gpl', 'results.json', 'times.json']
            files_to_get = ["%s/%s" % (EXP_DIR, fn) for fn in files_to_get_names]
            concerto_host.get_files(
                remote_files=files_to_get,
                local_location=working_directory
            )


Maverick Chardet's avatar
Maverick Chardet committed
65
def perform_experiment(list_nb_remote_ssh, nb_repeats):
66 67 68 69 70 71 72 73 74 75 76 77 78
    import yaml
    from os import makedirs

    with open("conf.yaml") as f:
        conf = yaml.load(f)

    conf['g5k']['resources']['machines'][0]['nodes'] = max(list_nb_remote_ssh)

    wd = "exp"
    makedirs(wd, exist_ok=True)
    with open(wd + '/g5k_config.yaml', 'w') as g5k_config_file:
        yaml.dump(conf, g5k_config_file)

Maverick Chardet's avatar
Maverick Chardet committed
79
    run_experiment(list_nb_remote_ssh, nb_repeats, conf, wd)
80 81 82 83 84


if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    perform_experiment([1, 5, 10, 15, 20], 5)