From 0a3173ec863d239cf1e03fb26811dfa22b9f6823 Mon Sep 17 00:00:00 2001
From: tbadts <thomas.badts@inria.fr>
Date: Tue, 26 Nov 2024 10:24:29 +0100
Subject: [PATCH] Small reformating
---
e2clab/app.py | 16 +++++------
e2clab/config.py | 2 +-
e2clab/constants/__init__.py | 22 ++++++---------
e2clab/experiment.py | 34 +++++++++++++----------
e2clab/managers/monitoring_kwollect.py | 3 +-
e2clab/probe.py | 38 ++++++++++++++------------
e2clab/tests/unit/test_constants.py | 13 ---------
e2clab/tests/unit/test_experiment.py | 12 ++------
e2clab/tests/unit/test_probe.py | 22 +++++++++++++--
e2clab/utils.py | 3 ++
10 files changed, 82 insertions(+), 83 deletions(-)
diff --git a/e2clab/app.py b/e2clab/app.py
index a95f2529..ae5a2ec5 100644
--- a/e2clab/app.py
+++ b/e2clab/app.py
@@ -7,7 +7,7 @@ import copy
import json
import logging
from pathlib import Path
-from typing import Union
+from typing import Optional, Union
from enoslib.api import Results, run_play
from enoslib.enos_inventory import EnosInventory
@@ -48,8 +48,8 @@ class App:
artifacts_dir: Path,
roles: Roles,
all_serv_extra_inf: dict,
- app_conf: str = None,
- env_config: Path = None,
+ app_conf: Optional[str] = None,
+ env_config: Optional[Path] = None,
optimization_config: dict[str, any] = None,
) -> None:
"""Create an application for the experiment
@@ -121,7 +121,7 @@ class App:
def run_task(
self,
task: WorkflowTasks,
- current_repeat: int = None,
+ current_repeat: Optional[int] = None,
) -> None:
"""Run application task
@@ -131,10 +131,10 @@ class App:
"""
# TODO: propagate enum use to the rest of the module
- task = task.value
+ task_name = task.value
# Define our working directory
- self.working_dir = self._prepare_working_dir(task=task)
+ self.working_dir = self._prepare_working_dir(task=task_name)
self.current_repeat = current_repeat
# Get task run relevant additional ansible variables
self.ansible_vars = self._vars_to_inject_ansible()
@@ -144,7 +144,7 @@ class App:
self.logger.debug(f"[ANSIBLE VARS] {self.ansible_vars}")
# filtered_config is a copy
- filtered_config = self.config.get_task_filtered_host_config(task=task)
+ filtered_config = self.config.get_task_filtered_host_config(task=task_name)
self.logger.debug(f"[TASK CONFIG] {filtered_config}")
@@ -157,7 +157,7 @@ class App:
self._print_workflow_validate(
host_conf=host_task_conf,
roles_validate=ansible_roles,
- task=task,
+ task=task_name,
ret=ansible_play_return,
)
diff --git a/e2clab/config.py b/e2clab/config.py
index 4a7080dc..92713a5f 100644
--- a/e2clab/config.py
+++ b/e2clab/config.py
@@ -223,7 +223,7 @@ class InfrastructureConfig(dict, E2clabConfig):
"""
# dict type ensured by schema
manager_key = manager.value.get_config_key()
- manager_conf: dict = self.get_manager_conf(manager)
+ manager_conf = self.get_manager_conf(manager)
if manager_conf is None:
return
# TODO: change managers API
diff --git a/e2clab/constants/__init__.py b/e2clab/constants/__init__.py
index 4acc9d13..d03e4006 100644
--- a/e2clab/constants/__init__.py
+++ b/e2clab/constants/__init__.py
@@ -13,12 +13,6 @@ PATH_ROOT_E2CLAB = Path(__file__).parent.parent.resolve()
PATH_SERVICES_PLUGINS = PATH_ROOT_E2CLAB / "services" / "plugins"
-class ExtEnum(Enum):
- @classmethod
- def value_list(cls):
- return [e.value for e in cls]
-
-
class ConfFiles:
LAYERS_SERVICES = "layers_services.yaml"
NETWORK = "network.yaml"
@@ -38,14 +32,14 @@ CONF_FILES_LIST = [
"""
-class Environment(ExtEnum):
+class Environment(Enum):
G5K: str = "g5k"
IOT_LAB: str = "iotlab"
CHAMELEON_CLOUD: str = "chameleoncloud"
CHAMELEON_EDGE: str = "chameleonedge"
-SUPPORTED_ENVIRONMENTS = Environment.value_list()
+SUPPORTED_ENVIRONMENTS = [e.value for e in Environment]
"""
CLI constants
@@ -57,7 +51,7 @@ ENV_ARTIFACTS_DIR = "E2C_ARTIFACTS_DIR"
ENV_AUTO_PREFIX = "E2C"
-class Command(ExtEnum):
+class Command(Enum):
DEPLOY: str = "deploy"
LYR_SVC: str = "layers-services"
NETWORK: str = "network"
@@ -65,20 +59,20 @@ class Command(ExtEnum):
FINALIZE: str = "finalize"
-COMMAND_RUN_LIST = Command.value_list()
+COMMAND_RUN_LIST = [e.value for e in Command]
"""
Workflow tasks
"""
-class WorkflowTasks(ExtEnum):
+class WorkflowTasks(Enum):
PREPARE: str = "prepare"
LAUNCH: str = "launch"
FINALIZE: str = "finalize"
-WORKFLOW_TASKS = WorkflowTasks.value_list()
+WORKFLOW_TASKS = [e.value for e in WorkflowTasks]
"""
@@ -86,13 +80,13 @@ WORKFLOW_TASKS = WorkflowTasks.value_list()
"""
-class ManagerSvcs(ExtEnum):
+class ManagerSvcs(Enum):
PROVENANCE: str = layers_services.PROVENANCE_SVC
MONITORING: str = layers_services.MONITORING_SVC
MONITORING_IOT: str = layers_services.MONITORING_IOT_SVC
-class MonitoringType(ExtEnum):
+class MonitoringType(Enum):
TIG: str = layers_services.MONITORING_SVC_TIG
TPG: str = layers_services.MONITORING_SVC_TPG
DSTAT: str = layers_services.MONITORING_SVC_DSTAT
diff --git a/e2clab/experiment.py b/e2clab/experiment.py
index 8f6eed7b..31460690 100644
--- a/e2clab/experiment.py
+++ b/e2clab/experiment.py
@@ -42,7 +42,7 @@ class Experiment:
self,
scenario_dir: Path,
artifacts_dir: Path,
- repeat: int = None,
+ repeat: Optional[int] = None,
app_conf_list: list[str] = [],
optimization_config=None,
optimization_id: Optional[UUID] = None,
@@ -62,14 +62,12 @@ class Experiment:
self.logger = get_logger(__name__, ["EXP"])
# Experiment components
- self.infra = None
- self.net = None
- self.app = None
+ self.infra: Optional[Infrastructure] = None
+ self.net: Optional[Network] = None
+ self.app: Optional[App] = None
self.probe = TaskProbe()
- self.experiment_dir = None
-
def __setstate__(self, state):
"""Ran when unpickling"""
self.__dict__.update(state)
@@ -149,7 +147,7 @@ class Experiment:
self.logger.info("Experiment network deployed")
- def application(self, task: WorkflowTasks, app_conf: str = None) -> None:
+ def application(self, task: WorkflowTasks, app_conf: Optional[str] = None) -> None:
"""
Enforce workflow definition
"""
@@ -176,18 +174,19 @@ class Experiment:
)
# Enforce task
- self.logger.info(f"Enforcing workflow:{task}")
- # self.app.run_task(task=task, current_repeat=self.repeat)
+ self.logger.info(f"Enforcing workflow:{task.value}")
self.run_task(task=task, current_repeat=self.repeat)
- self.logger.info(f"Done enforcing workflow:{task}")
+ self.logger.info(f"Done enforcing workflow:{task.value}")
- def run_task(self, task: str, current_repeat: int):
+ def run_task(self, task: WorkflowTasks, current_repeat: Optional[int] = None):
"""Wrapper for application run_task"""
- self.probe.set_start(task_name=task)
+ if not self.app:
+ raise E2clabError("Failed initializing App")
+ self.probe.set_start(record_name=task)
self.app.run_task(task=task, current_repeat=current_repeat)
- self.probe.set_end(task_name=task)
+ self.probe.set_end(record_name=task)
- def finalize(self, app_conf: str = None, destroy: bool = False) -> None:
+ def finalize(self, app_conf: Optional[str] = None, destroy: bool = False) -> None:
"""
Finalize experiment
"""
@@ -260,6 +259,11 @@ class Experiment:
"""
Release (free) computing resources, e.g. kill G5k oar jobs
"""
+ if not self.infra:
+ raise E2clabError(
+ "Can't destroy an uninstantiated infrastructure."
+ "Have you run `e2clab layers_services` ?"
+ )
self.logger.info("Destroying provider computing resource")
self.infra.destroy()
self.logger.info("Destroyed computing resources")
@@ -297,7 +301,7 @@ class Experiment:
"""Prints experiment directory to stdout"""
print(self.get_exp_dir())
- def get_exp_id(self) -> UUID:
+ def get_exp_id(self) -> str:
return self.id
def get_exp_dir(self) -> Path:
diff --git a/e2clab/managers/monitoring_kwollect.py b/e2clab/managers/monitoring_kwollect.py
index 274c9d1e..257b2e60 100644
--- a/e2clab/managers/monitoring_kwollect.py
+++ b/e2clab/managers/monitoring_kwollect.py
@@ -76,13 +76,12 @@ class MonitoringKwollectManager(Manager):
kwollect_output_dir.mkdir(exist_ok=True)
task_probe = TaskProbe.get_probe()
- records = task_probe.get_records()
metrics = self._get_metrics_str()
step = self.config.get(STEP, WorkflowTasks.LAUNCH.value)
- rec = records.get(step)
+ rec = task_probe.get_task_record(WorkflowTasks(step))
if not rec:
self.logger.warning(f"No timestamps found for {step}. Did you run it ?")
return
diff --git a/e2clab/probe.py b/e2clab/probe.py
index d2740aa5..dcff374a 100644
--- a/e2clab/probe.py
+++ b/e2clab/probe.py
@@ -4,14 +4,15 @@ Probe singletons module
from dataclasses import dataclass
from datetime import datetime
+from typing import Hashable, Optional
from e2clab.log import get_logger
@dataclass
class Record:
- start: datetime
- end: datetime
+ start: Optional[datetime] = None
+ end: Optional[datetime] = None
class TaskProbe:
@@ -23,10 +24,10 @@ class TaskProbe:
_instance = None
- def __init__(self):
+ def __init__(self) -> None:
self.logger = get_logger(__name__, ["PROBE"])
- self._records: dict[str, Record] = {}
+ self._records: dict[Hashable, Record] = {}
def __new__(cls):
if cls._instance is None:
@@ -41,37 +42,38 @@ class TaskProbe:
# restore unpickled probe as singleton
type(self)._instance = self
- def set_start(self, task_name: str) -> None:
- self._check_record(task_name)
+ def set_start(self, record_name: Hashable) -> None:
+ self._check_record(record_name)
timestmp = self._get_now()
- self._records[task_name].start = timestmp
+ self._records[record_name].start = timestmp
- self.logger.debug(f"Set start timestamp for {task_name}")
+ self.logger.debug(f"Set start timestamp for {record_name}")
- def set_end(self, task_name: str) -> None:
- self._check_record(task_name)
+ def set_end(self, record_name: Hashable) -> None:
+ self._check_record(record_name)
timestmp = self._get_now()
- self._records[task_name].end = timestmp
+ self._records[record_name].end = timestmp
- self.logger.debug(f"Set start timestamp for {task_name}")
+ self.logger.debug(f"Set end timestamp for {record_name}")
@staticmethod
def _get_now() -> datetime:
return datetime.now().astimezone()
- def _check_record(self, task_name: str):
+ def _check_record(self, record_name: Hashable):
"""Init record if doesn't exist"""
- if task_name not in self._records:
- self._records[task_name] = Record(start=None, end=None)
+ if record_name not in self._records:
+ self._records[record_name] = Record()
- def get_task_record(self, task_name):
- return self._records.get(task_name, None)
+ def get_task_record(self, record_name: Hashable):
+ return self._records.get(record_name, None)
- def get_records(self) -> dict[str, Record]:
+ def get_records(self) -> dict[Hashable, Record]:
return self._records
@classmethod
def get_probe(cls):
+ """Get timestamp probe object (singleton)"""
if not cls._instance:
cls._instance = cls()
return cls._instance
diff --git a/e2clab/tests/unit/test_constants.py b/e2clab/tests/unit/test_constants.py
index c02f69ed..eab10c99 100644
--- a/e2clab/tests/unit/test_constants.py
+++ b/e2clab/tests/unit/test_constants.py
@@ -1,23 +1,10 @@
from unittest.mock import patch
-import e2clab.constants as e2cconst
from e2clab.constants.default import load_def, prefix
from e2clab.tests.unit import TestE2cLab
class TestConstants(TestE2cLab):
- def test_enum(self):
- class TestClass(e2cconst.ExtEnum):
- A = 1
- B = 2
- C = "test"
- D = 4
- E = 5
-
- expect = [1, 2, "test", 4, 5]
- res = TestClass.value_list()
- self.assertEqual(expect, res)
-
def test_prefix(self):
self.assertEqual("E2C_TEST", prefix("TEST"))
diff --git a/e2clab/tests/unit/test_experiment.py b/e2clab/tests/unit/test_experiment.py
index 51a293d1..db9495f5 100644
--- a/e2clab/tests/unit/test_experiment.py
+++ b/e2clab/tests/unit/test_experiment.py
@@ -30,6 +30,7 @@ class TestExperiment(TestE2cLab):
scenario_dir=self.test_folder,
artifacts_dir=self.test_folder,
)
+ self.def_exp.initiate()
def tearDown(self) -> None:
# Removing output folder
@@ -44,7 +45,7 @@ class TestExperiment(TestE2cLab):
self.assertIsInstance(self.def_exp.id, str)
def test_initiate(self):
- with self.assertLogs("e2clab.experiment", level="INFO") as _:
+ with self.assertLogs("e2clab.experiment", level="INFO"):
self.def_exp.initiate()
self.assertTrue(Path(self.def_exp.experiment_dir).exists())
@@ -61,7 +62,6 @@ class TestExperiment(TestE2cLab):
infra = MockInfra.return_value
infra.deploy.return_value = (Roles({}), Networks({}))
- self.def_exp.initiate()
self.def_exp.infrastructure()
infra.deploy.assert_called_once_with(
@@ -73,14 +73,12 @@ class TestExperiment(TestE2cLab):
self.assertIsInstance(self.def_exp.infra, Infrastructure)
def test_network_no_infra(self):
- self.def_exp.initiate()
with self.assertRaises(E2clabError):
self.def_exp.network()
@patch("e2clab.experiment.Network", autospec=True)
def test_network(self, MockNetwork: MagicMock):
# Prepare test
- self.def_exp.initiate()
self.def_exp.infra = True
self.def_exp.roles = Roles({})
self.def_exp.networks = Networks({})
@@ -94,7 +92,6 @@ class TestExperiment(TestE2cLab):
self.assertIsInstance(self.def_exp.net, Network)
def test_application_no_infra(self):
- self.def_exp.initiate()
with self.assertRaises(E2clabError):
self.def_exp.application("prepare")
@@ -105,7 +102,6 @@ class TestExperiment(TestE2cLab):
infra = MockInfra.return_value
infra.all_serv_extra_inf.return_value = {}
- self.def_exp.initiate()
self.def_exp.infra = infra
self.def_exp.roles = Roles({})
self.def_exp.networks = Networks({})
@@ -115,7 +111,6 @@ class TestExperiment(TestE2cLab):
app.run_task.assert_called_once_with(task="prepare", current_repeat=None)
def test_finalize_no_infra(self):
- self.def_exp.initiate()
with self.assertRaises(E2clabError):
self.def_exp.finalize()
@@ -126,7 +121,6 @@ class TestExperiment(TestE2cLab):
app = MockApp.return_value
infra = MockInfra.return_value
- self.def_exp.initiate()
self.def_exp.infra = infra
self.def_exp.app = app
self.def_exp.roles = Roles({})
@@ -149,7 +143,6 @@ class TestExperiment(TestE2cLab):
app = MockApp.return_value
infra = MockInfra.return_value
- self.def_exp.initiate()
self.def_exp.infra = infra
self.def_exp.app = app
self.def_exp.roles = Roles({})
@@ -169,7 +162,6 @@ class TestExperiment(TestE2cLab):
app = MockApp.return_value
infra = MockInfra.return_value
- self.def_exp.initiate()
self.def_exp.infra = infra
self.def_exp.app = app
self.def_exp.roles = Roles({})
diff --git a/e2clab/tests/unit/test_probe.py b/e2clab/tests/unit/test_probe.py
index 1542232a..b77c8078 100644
--- a/e2clab/tests/unit/test_probe.py
+++ b/e2clab/tests/unit/test_probe.py
@@ -4,6 +4,7 @@ Testing the probe.py module
import shutil
+from e2clab.constants import WorkflowTasks
from e2clab.probe import TaskProbe
from . import TestE2cLab
@@ -20,10 +21,27 @@ class TestTaskProbe(TestE2cLab):
def tearDownClass(cls):
shutil.rmtree(cls.env)
+ def setUp(self):
+ self.probe = TaskProbe()
+
def test_singleton(self):
- probe = TaskProbe()
probe2 = TaskProbe()
- self.assertEqual(id(probe2), id(probe))
+ self.assertEqual(id(probe2), id(self.probe))
+
+ def test_set_start(self):
+ self.probe.set_start(WorkflowTasks.LAUNCH)
+ rec = self.probe.get_task_record(WorkflowTasks.LAUNCH)
+ self.assertIsNotNone(rec.start)
+ self.assertIsNone(rec.end)
+
+ def test_set_end(self):
+ self.probe.set_end(WorkflowTasks.LAUNCH)
+ rec = self.probe.get_task_record(WorkflowTasks.LAUNCH)
+ self.assertIsNone(rec.start)
+ self.assertIsNotNone(rec.end)
+
+ def test_check_record(self):
+ pass
# TODO: test __setstate__
def test_setstate(self):
diff --git a/e2clab/utils.py b/e2clab/utils.py
index 08609127..304b862d 100644
--- a/e2clab/utils.py
+++ b/e2clab/utils.py
@@ -144,3 +144,6 @@ def write_dot_param(file, envname: str, value, comment: bool = False):
if comment:
line = "# " + line
file.write(line)
+
+
+load_yaml_file("notafile")
--
GitLab