Mentions légales du service

Skip to content
Snippets Groups Projects
Commit 075df469 authored by SIMONIN Matthieu's avatar SIMONIN Matthieu
Browse files

api and dstat/tcpdump updates

More friendly CM.
Support various signals in tmux_start, tmux_stop
parent 29eca5d1
No related branches found
No related tags found
No related merge requests found
from enoslib.infra.enos_g5k.provider import G5k
from enoslib.infra.enos_g5k.configuration import (Configuration,
NetworkConfiguration)
from enoslib.service import Dstat
import logging
import os
import time
logging.basicConfig(level=logging.DEBUG)
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
# path to the inventory
inventory = os.path.join(os.getcwd(), "hosts")
import enoslib as en
logging.basicConfig(level=logging.DEBUG)
# claim the resources
conf = Configuration.from_settings(job_type="allow_classic_ssh",
conf = en.G5kConf.from_settings(job_type="allow_classic_ssh",
job_name="test-non-deploy")
network = NetworkConfiguration(id="n1",
network = en.G5kNetworkConf(id="n1",
type="prod",
roles=["my_network"],
site="nancy")
......@@ -26,17 +24,30 @@ conf.add_network_conf(network)\
primary_network=network)\
.finalize()
provider = G5k(conf)
provider = en.G5k(conf)
roles, networks = provider.init()
m = Dstat(nodes=roles["control"])
m.deploy()
time.sleep(10)
m.destroy()
m.backup()
# destroy the boxes
# provider.destroy()
with en.actions(roles["control"]) as a:
a.apt(name="stress", state="present")
# Start a capture
# - for the duration of the commands
with en.Dstat(nodes=roles["control"]) as d:
time.sleep(5)
en.run("stress --cpu 4 --timeout 10", roles["control"])
time.sleep(5)
backup_dir = d.backup_dir
# Create a dictionnary of (alias) -> list of pandas df
result = pd.DataFrame()
for host in roles["control"]:
host_dir = backup_dir / host.alias
csvs = host_dir.rglob("*.csv")
for csv in csvs:
df = pd.read_csv(csv, skiprows=5, index_col=False)
df["host"] = host.alias
df["csv"] = csv
result = pd.concat([result, df], axis=0)
sns.lineplot(data=result, x="epoch", y="usr", hue="host", markers=True, style="host")
plt.show()
\ No newline at end of file
......@@ -8,13 +8,14 @@ from scapy.all import rdpcap
logging.basicConfig(level=logging.INFO)
CLUSTER = "paravance"
CLUSTER = "parasilo"
SITE = en.g5k_api_utils.get_cluster_site(CLUSTER)
job_name = Path(__file__).name
# claim the resources
conf = en.G5kConf.from_settings(
job_type="allow_classic_ssh", job_name="test-non-deploy"
job_type="allow_classic_ssh", job_name=job_name
)
network = en.G5kNetworkConf(id="n1", type="prod", roles=["my_network"], site=SITE)
conf.add_network_conf(network).add_machine(
......@@ -33,8 +34,9 @@ roles = en.sync_info(roles, networks)
# - we dump icmp traffic only
# - for the duration of the commands (here a client is pigging the server)
with en.TCPDump(
hosts=roles["control"], networks=networks["my_network"], options="-U icmp"
):
hosts=roles["control"], networks=networks["my_network"], options="icmp"
) as t:
backup_dir = t.backup_dir
_ = en.run(f"ping -c10 {roles['server'][0].address}", roles["client"])
# pcap files are retrieved in the __enoslib__tcpdump__ directory
......@@ -44,10 +46,9 @@ with en.TCPDump(
# Examples:
# create a dictionnary of (alias, if) -> list of decoded packets by scapy
base_dir = Path("__enoslib_tcpdump__")
decoded_pcaps = dict()
for host in roles["control"]:
host_dir = base_dir / host.alias
host_dir = backup_dir / host.alias
t = tarfile.open(host_dir / "tcpdump.tar.gz")
t.extractall(host_dir / "extracted")
# get all extracted pcap for this host
......
......@@ -23,6 +23,8 @@ import time
from collections import namedtuple
from typing import Any, Dict, List, Mapping, Optional, Set, Union, Tuple
from pathlib import Path
import signal
from cryptography.hazmat.primitives import serialization as crypto_serialization
from cryptography.hazmat.primitives.asymmetric import rsa
......@@ -1074,7 +1076,7 @@ def tmux_start(key: str, cmd: str) -> str:
return "(tmux ls | grep %s) ||tmux new-session -s %s -d '%s'" % (key, key, cmd)
def tmux_stop(key: str) -> str:
def tmux_stop(key: str, num: int = signal.SIGINT) -> str:
"""Stop a command that runs in the background.
Generate the command that will stop a previously started command in the
......@@ -1086,7 +1088,17 @@ def tmux_stop(key: str) -> str:
Returns:
command that will stop a tmux session
"""
return f"tmux kill-session -t {key} || true"
if num == signal.SIGHUP:
# default tmux termination signal
# This will send SIGHUP to all the encapsulated processes
return f"tmux kill-session -t {key} || true"
else:
# We prefer send a sigint to all the encapsulated processes
cmd = (
"(tmux list-panes -t %s -F '#{pane_pid}' | "
"xargs -n1 kill -%s) || true"
) % (key, int(num))
return cmd
# Private zone
......
from pathlib import Path
import os
from typing import Dict, List, Optional
from time import time
from typing import Dict, List
from enoslib.api import play_on, __python3__
from enoslib.api import play_on, tmux_start, tmux_stop
from enoslib.objects import Host, Roles
from ..service import Service
from ..utils import _check_path
......@@ -13,10 +14,12 @@ TMUX_SESSION = "__enoslib_dstat__"
DOOL_URL = (
"https://raw.githubusercontent.com/"
"scottchiefbaker/dool/6b89f2d0b6e38e1c8d706e88a12e020367f5100d/dool"
"scottchiefbaker/dool/02b1c69d441764b030db5e78b4b6fb231c29f8f1/dool"
)
DOOL_DIR = Path("/opt/enoslib_dool")
DOOL_PATH = DOOL_DIR / "dool"
REMOTE_OUTPUT_DIR = Path("/tmp/__enoslib_dstat__")
DOOL_PATH = REMOTE_OUTPUT_DIR / "dool"
LOCAL_OUTPUT_DIR = Path("__enoslib_dstat__")
class Dstat(Service):
......@@ -24,9 +27,8 @@ class Dstat(Service):
self,
*,
nodes: List[Host],
options: str = "",
remote_working_dir: str = "/builds/dstat",
priors: List[play_on] = [__python3__],
options: str = "-aT",
backup_dir: Path = LOCAL_OUTPUT_DIR,
extra_vars: Dict = None,
):
"""Deploy dstat on all hosts.
......@@ -56,33 +58,38 @@ class Dstat(Service):
"""
self.nodes = nodes
self.options = options
self.priors = priors
self.remote_working_dir = remote_working_dir
self.remote_working_dir = Path(REMOTE_OUTPUT_DIR)
self._roles = Roles(all=self.nodes)
self.backup_dir = Path(backup_dir)
self.backup_dir = _check_path(self.backup_dir)
# generate output file name by default dstat append to the existing file
# which isn't very convenient at parsing time we could also add some
# prefix to differentiate from different calls to this service in the
# same xp
self.output_file = f"{time()}-{OUTPUT_FILE}"
# We force python3
extra_vars = extra_vars if extra_vars is not None else {}
self.extra_vars = {"ansible_python_interpreter": "/usr/bin/python3"}
self.extra_vars.update(extra_vars)
def deploy(self):
def deploy(self, force=False):
"""Deploy the dstat monitoring stack."""
if force:
self.destroy()
with play_on(
roles=self._roles, priors=self.priors, extra_vars=self.extra_vars
roles=self._roles, extra_vars=self.extra_vars
) as p:
p.apt(name=["tmux"], state="present")
# install dool
p.file(path=str(DOOL_DIR), state="directory")
p.file(path=str(self.remote_working_dir), state="directory", recurse="yes")
p.get_url(url=DOOL_URL, dest=str(DOOL_PATH), mode="0755")
p.file(path=self.remote_working_dir, recurse="yes", state="directory")
options = f"{self.options} -o {OUTPUT_FILE}"
options = f"{self.options} -o {self.output_file}"
p.shell(
(
f"(tmux ls | grep {TMUX_SESSION}) || "
f"tmux new-session -s {TMUX_SESSION} "
f"-d 'exec {DOOL_PATH} {options}'"
),
chdir=self.remote_working_dir,
tmux_start(TMUX_SESSION, f"python3 {str(DOOL_PATH)} {options}"),
chdir=str(self.remote_working_dir),
display_name=f"Running dstat with the options {options}",
)
......@@ -93,10 +100,10 @@ class Dstat(Service):
Metric files survive to destroy.
"""
with play_on(roles=self._roles, extra_vars=self.extra_vars) as p:
kill_cmd = f"tmux kill-session -t {TMUX_SESSION}"
kill_cmd = tmux_stop(TMUX_SESSION)
p.shell(kill_cmd)
def backup(self, backup_dir: Optional[str] = None):
def backup(self):
"""Backup the dstat monitoring stack.
This fetches all the remote dstat csv files under the backup_dir.
......@@ -104,18 +111,15 @@ class Dstat(Service):
Args:
backup_dir (str): path of the backup directory to use.
"""
if backup_dir is None:
_backup_dir = Path.cwd()
else:
_backup_dir = Path(backup_dir)
_backup_dir = _check_path(_backup_dir)
with play_on(roles=self._roles, extra_vars=self.extra_vars) as p:
backup_path = os.path.join(self.remote_working_dir, OUTPUT_FILE)
backup_path = os.path.join(self.remote_working_dir, self.output_file)
p.fetch(
display_name="Fetching the dstat output",
src=backup_path,
dest=str(Path(_backup_dir, "dstat")),
dest=str(self.backup_dir),
flat=False,
)
def __enter__(self):
self.deploy(force=True)
return self
\ No newline at end of file
......@@ -24,3 +24,11 @@ class Service:
def backup(self):
"""(abstract) Backup the service."""
pass
def __enter__(self):
self.deploy()
return self
def __exit__(self, *args):
self.destroy()
self.backup()
\ No newline at end of file
......@@ -115,7 +115,4 @@ class TCPDump(Service):
def __enter__(self):
self.deploy(force=True)
def __exit__(self, *args):
self.backup()
self.destroy()
return self
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment