up

e67d8671 · SIMONIN Matthieu · 53e1ab2d · e67d8671 · e67d8671 · e67d8671
Commit e67d8671 authored 2 years ago by SIMONIN Matthieu
--- a/fit_and_g5k/01_networking.ipynb
+++ b/fit_and_g5k/01_networking.ipynb
@@ -152,7 +152,7 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "fit = en.Iotlab(fit_conf)"
+    "fit = en.Iotlab(fit_conf, name=\"Iotlab\")"
   ]
  },
  {
@@ -205,7 +205,7 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "g5k = en.G5k(conf)"
+    "g5k = en.G5k(conf, name=\"G5k\")"
   ]
  },
  {
@@ -230,10 +230,31 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "roles, networks = providers.init(time_window=86400)\n",
+    "roles, networks = providers.init()\n",
    "fit, g5k = providers.providers"
   ]
  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13657ac1-6666-42d8-9fcc-27bb0e1391bd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Get all the nodes and networks reserved for this experiment\n",
+    "roles"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8c41ddaa-0f50-42da-a9db-fabc993ea3f0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "networks"
+   ]
+  },
  {
   "cell_type": "code",
   "execution_count": null,
@@ -304,8 +325,8 @@
   "outputs": [],
   "source": [
    "# CHANGE ME!\n",
-    "tap = \"tap1\"\n",
-    "prefix = \"2001:660:5307:3110\""
+    "tap = \"tap2\"\n",
+    "prefix = \"2001:660:5307:3101\""
   ]
  },
  {
@@ -424,7 +445,7 @@
   "outputs": [],
   "source": [
    "# Change me! (manual change)\n",
-    "G5K_IPv6 = \"2001:660:4406:700:1::38\""
+    "G5K_IPv6 = \"2001:660:4406:700:1::e\""
   ]
  },
  {
@@ -487,7 +508,7 @@
    "# Getting the IP automatically using the library\n",
    "# First, do a sync of the description\n",
    "# Second, filter the desired IP\n",
-    "roles = en.sync_info(roles,networks)\n",
+    "roles = en.sync_info(roles, networks)\n",
    "roles[\"server\"][0]"
   ]
  },

 %% Cell type:markdown id:b8ec4017 tags:

 # Networking between FIT and Grid'5000

 Revisiting https://www.iot-lab.info/learn/tutorials/riot/riot-public-ipv6-m3/ in the context of mixed G5K/FIT resources.

 ---

 - Website: https://discovery.gitlabpages.inria.fr/enoslib/index.html
 - Instant chat: https://framateam.org/enoslib
 - Source code: https://gitlab.inria.fr/discovery/enoslib
 ---

 ## Prerequisites:

 <div class="alert alert-block alert-warning">
    Make sure you've run the one time setup for your environment (e.g one time setup for https://labs.iot-lab.info)
 </div>

 ## Overview


 ![01_networking.png](attachment:cd3f5af7-fa96-4e29-b432-e08e1e7b9f68.png)

 %% Cell type:markdown id:243d7bf5-812d-4acb-a900-df7d72cfd155 tags:

 FIT side

 ### Firmware compilations

 %% Cell type:code id:16f2af92-e5fc-404f-a43f-63a990075263 tags:

 ``` python
 import os,binascii,random
 pan_id = binascii.b2a_hex(os.urandom(2)).decode()
 channel = random.randint(11, 26)
 fit_site = random.choice(["grenoble", "lille"])

 print(f"Use FIT_SITE={fit_site}, CHANNEL={channel}, PAN_ID=0x{pan_id}")
 ```

 %% Cell type:code id:a3a1abf3-dacc-4cd6-8ba4-3eec6a4c2af8 tags:

 ``` python
 # save the random parameters (allow to restart from here)
 FIT_SITE = "grenoble"
 %env CHANNEL = 26
 %env PAN_ID = 0xd0fb
 %env ETHOS_BAUDRATE = 500000
 ```

 %% Cell type:code id:a6cb1167-7525-48e8-8e7d-40dece549842 tags:

 ``` python
 %env APP_DIR=../../riot/RIOT/examples/gnrc_border_router
 !make -C $APP_DIR BOARD=iotlab-m3 DEFAULT_CHANNEL=$CHANNEL DEFAULT_PAN_ID=$PAN_ID && cp $APP_DIR/bin/iotlab-m3/*.elf .
 ```

 %% Cell type:code id:923be597-18db-422a-8260-90de84da0577 tags:

 ``` python
 %env APP_DIR=../../riot/RIOT/examples/gnrc_networking
 !make -C $APP_DIR BOARD=iotlab-m3 DEFAULT_CHANNEL=$CHANNEL DEFAULT_PAN_ID=$PAN_ID && cp $APP_DIR/bin/iotlab-m3/*.elf .
 ```

 %% Cell type:code id:2a17ee75-4745-4bbf-a50f-0886fe263309 tags:

 ``` python
 import enoslib as en

 _ = en.init_logging()
 ```

 %% Cell type:markdown id:a74c6fdb-bd1f-483f-8fbd-fd99cd0e98e2 tags:

 ### Get some resources

 Check the testbed status: https://www.iot-lab.info/testbed/status

 %% Cell type:code id:4907ee08-cb8c-4b25-8f77-3d271fc10d60 tags:

 ``` python
 fit_conf = (
    en.IotlabConf.from_settings(job_name="riot_m3", walltime="01:00")
    .add_machine(roles=["border_router"], archi="m3:at86rf231", site=FIT_SITE, number=1, image="gnrc_border_router.elf")
    .add_machine(roles=["other"], archi="m3:at86rf231", site=FIT_SITE, number=1, image="gnrc_networking.elf")
 )
 fit_conf

 # Hostname based
 # fit_conf = (
 #  en.IotlabConf.from_settings(job_name="riot_m3", walltime="02:00")
 #  .add_machine(roles=["border_router"], hostname=["m3-62.grenoble.iot-lab.info"], image="gnrc_border_router.bin")
 #  .add_machine(roles=["other"], hostname=["m3-63.grenoble.iot-lab.info"], image="gnrc_border_router.bin")
 # ).finalize()
 # fit_conf
 ```

 %% Cell type:code id:a1ab91de-9e0e-4eb6-b64e-604a3ee8c7a1 tags:

 ``` python
-fit = en.Iotlab(fit_conf)
+fit = en.Iotlab(fit_conf, name="Iotlab")
 ```

 %% Cell type:code id:3a2536a0-0a8d-43ae-9fe2-dfb6e5db4d70 tags:

 ``` python
 G5K_SITE = "rennes"
 G5K_CLUSTER = "paravance"
 ```

 %% Cell type:code id:dc7c38a8-e03b-47d8-9c3b-66b1e21ab371 tags:

 ``` python
 network = en.G5kNetworkConf(type="prod", roles=["my_network"], site=G5K_SITE)
 ```

 %% Cell type:code id:7669ecfb-dcc2-42e2-8517-30e41f307171 tags:

 ``` python
 conf = (
    en.G5kConf.from_settings(
        job_type="allow_classic_ssh",
        job_name="fit_g5k_01_networking",
        walltime="01:00:00",
    )
    .add_network_conf(network)
    .add_machine(
        roles=["server"], cluster=G5K_CLUSTER, nodes=1, primary_network=network
    )
    .finalize()
 )
 conf
 ```

 %% Cell type:code id:025af2b4-2d07-4e9c-9e3a-9a83914163c4 tags:

 ``` python
-g5k = en.G5k(conf)
+g5k = en.G5k(conf, name="G5k")
 ```

 %% Cell type:code id:708de76c-97bf-4aad-a5ed-c1b308c56533 tags:

 ``` python
 # Here we set up a special provider that will try to reserve both G5k and Iotlab configuration simultaneously

 from enoslib.infra.providers import Providers


 providers = Providers([fit, g5k])
 ```

 %% Cell type:code id:0929ce4f-5292-4dd5-8ecb-471a88b5ba97 tags:

 ``` python
-roles, networks = providers.init(time_window=86400)
+roles, networks = providers.init()
 fit, g5k = providers.providers
 ```

+%% Cell type:code id:13657ac1-6666-42d8-9fcc-27bb0e1391bd tags:
+
+``` python
+# Get all the nodes and networks reserved for this experiment
+roles
+```
+
+%% Cell type:code id:8c41ddaa-0f50-42da-a9db-fabc993ea3f0 tags:
+
+``` python
+networks
+```
+
 %% Cell type:code id:123d7bc5-d60c-4c6a-9b3c-ebc6f5de9e90 tags:

 ``` python
 # keep track of the border router
 br = roles["border_router"][0]
 br
 ```

 %% Cell type:code id:46fa11ae-405f-48dc-8ed5-e309d431781a tags:

 ``` python
 # This is useless at this point but practical in case you want to reflash your nodes at some point
 fit.reset()
 ```

 %% Cell type:markdown id:82ef229a-533a-435b-980f-87e0317e49ac tags:

 ### Setting up IPv6

 %% Cell type:code id:645f4642-ab5e-47b1-8a3a-0d31663af443 tags:

 ``` python
 import iotlabcli
 iotlab_user, _ = iotlabcli.auth.get_user_credentials()
 fit_frontend = en.Host("%s.iot-lab.info" % FIT_SITE, user=iotlab_user)
 ```

 %% Cell type:markdown id:2c22b0ad-c188-494c-99c0-18a4f69de82f tags:

 At this point you need to pick a free tap number and a [free subnet](https://www.iot-lab.info/legacy/tutorials/understand-ipv6-subnetting-on-the-fit-iot-lab-testbed/index.html)

 %% Cell type:code id:f041a67f-65e9-4d41-88e0-3d8d63f3bf7b tags:

 ``` python
 r = en.run(f"ip -6 route", roles=fit_frontend)
 print(r[0].stdout)
 ```

 %% Cell type:code id:7aac9ab7-22ab-460c-b42e-c04cab19ee29 tags:

 ``` python
 # CHANGE ME!
-tap = "tap1"
-prefix = "2001:660:5307:3110"
+tap = "tap2"
+prefix = "2001:660:5307:3101"
 ```

 %% Cell type:code id:00474406-f270-486a-b026-5165c91ea204 tags:

 ``` python
 en.run(f"ps aux | grep {tap} | grep {iotlab_user}| grep -v grep | awk '{{print $2}}' | xargs -r kill", roles=fit_frontend)
 r = en.run(f"sudo ethos_uhcpd.py {br.alias} {tap} {prefix}::1/64 2>&1 > ethos_uhcpd.out", roles=fit_frontend, background=True)
 ```

 %% Cell type:code id:087454e4-c074-4cfd-bfe2-aa11ced1a67a tags:

 ``` python
 r = en.run(f"ip -6 route", roles=fit_frontend)
 print(r[0].stdout)
 ```

 %% Cell type:markdown id:52206ed1-ebd2-4977-94db-7cf9afc9eb83 tags:

 The other M3 device should get its IPv6 in the chosen after few seconds.
 Retry the following until you get this IPv6 (or until you get bored)

 Note that the command sends a command (`ifconfig`) to the other M3 device after SSH-ing to the frontend.

 %% Cell type:code id:d7700cf9-3997-4813-bf18-656d304f5aac tags:

 ``` python
 other = roles["other"][0]
 r = en.run(f"echo ifconfig | nc -w1  {other.alias} 20000", fit_frontend)
 print(r[0].stdout)
 ```

 %% Cell type:code id:e1a16d39-dd14-4d59-8032-a12beb89251c tags:

 ``` python
 # record this for future use
 import re
 m = re.findall(f'inet6 addr: (.*)  scope: global', r[0].stdout)
 FIT_IPv6 = m[0]
 FIT_IPv6
 ```

 %% Cell type:code id:f380d20b-a9cb-4abc-b0ef-478af6538591 tags:

 ``` python
 # connectivity test with ... google
 r = en.run(f"echo ping6 2001:4860:4860::8888 | nc -w5  {other.alias} 20000", fit_frontend)
 print(r[0].stdout)
 ```

 %% Cell type:markdown id:b336117c-1e36-4fbe-91c4-1bcf7c8923b7 tags:

 ## Grid'5000 side

 ### Get some resources

 You can check what's available here: https://www.grid5000.fr/w/Status

 Some direct links:
 - Gantt for nancy: https://intranet.grid5000.fr/oar/Nancy/drawgantt-svg/
 - Gantt for rennes: https://intranet.grid5000.fr/oar/Rennes/drawgantt-svg/
 - ...

 %% Cell type:markdown id:86e23aab-c592-48d0-9946-5c956ecec03b tags:

 ### Setting up IPv6

 %% Cell type:code id:e242da2b-edae-42fc-8b5b-256f3ba43199 tags:

 ``` python
 en.run("dhclient -6 br0", roles=roles["G5k"])
 r = en.run("ip a", roles["G5k"])
 print(r[0].stdout)
 ```

 %% Cell type:code id:1dae6906-d715-43fc-bc73-b486286986a4 tags:

 ``` python
 # Change me! (manual change)
-G5K_IPv6 = "2001:660:4406:700:1::38"
+G5K_IPv6 = "2001:660:4406:700:1::e"
 ```

 %% Cell type:markdown id:95c8ba2e-31e3-439f-802e-0c03f79ea7a3 tags:

 There are ways to retrieve the IP address automatically, let's review two of them:
 - using the REST API (work for static IPs)
 - using the library (cover use cases where IP are dynamically assigned)

 %% Cell type:code id:219d7bcd-cfcc-40ec-8587-1a82a3de5f2a tags:

 ``` python
 # The ip address can be retrieved from the REST API
 from enoslib.infra.enos_g5k.g5k_api_utils import get_api_client

 gk = get_api_client()
 # grisou-1.nancy.grid5000.fr -> grisou-1
 node_uid = roles["server"][0].alias.split(".")[0]
 node = gk.sites[G5K_SITE].clusters[G5K_CLUSTER].nodes[node_uid]
 print(node)
 ```

 %% Cell type:code id:416ccd7f-40bc-4e7c-af12-67c055597ce6 tags:

 ``` python
 # eth0 is the primary interface;
 device = [ n for n in node.network_adapters if n["device"] == "eth0"][0]
 device
 ```

 %% Cell type:code id:9c0454e3-ac92-4b58-8db1-862d7147bdc0 tags:

 ``` python
 G5K_IPv6 = device["ip6"]
 G5K_IPv6
 ```

 %% Cell type:code id:c7938ddc-989c-4510-b626-2e36451c38d2 tags:

 ``` python
 # Getting the IP automatically using the library
 # First, do a sync of the description
 # Second, filter the desired IP
-roles = en.sync_info(roles,networks)
+roles = en.sync_info(roles, networks)
 roles["server"][0]
 ```

 %% Cell type:code id:d43fd024-0573-4f0f-be72-df5b27443219 tags:

 ``` python
 g5k_server = roles["server"][0]
 addresses = g5k_server.filter_addresses(networks=networks["my_network"])
 addresses
 ```

 %% Cell type:code id:2d0ab76c-df08-46ee-92d4-38cdd333bcda tags:

 ``` python
 from ipaddress import IPv6Interface
 G5K_IPv6 = [str(a.ip.ip) for a in addresses if isinstance(a.ip, IPv6Interface)][0]
 G5K_IPv6
 ```

 %% Cell type:markdown id:cc02dce1-23e8-4324-8ea1-06e45211e3fc tags:

 ## Cross traffic connectivity

 ### Grid'5000 -> FIT

 %% Cell type:code id:2c0c5091-ff79-46b7-a943-b64410106487 tags:

 ``` python
 r = en.run(f"ping -c 5 {FIT_IPv6}", roles=roles["G5k"])
 print(r[0].stdout)
 ```

 %% Cell type:markdown id:84dde85b-58ee-40a8-b434-791ce175e120 tags:

 ### FIT -> Grid5000

 %% Cell type:code id:792912e9-159a-46c4-b1f9-be894edd4d75 tags:

 ``` python
 # open the firewall for the duration of the inner command
 # opening a custom port will allow icmp traffic to flow
 with g5k.firewall(proto="tcp+udp", port=80):
    # fails if the firewall isn't opened
    r = en.run(f"echo ping6 {G5K_IPv6} | nc -w5  {other.alias} 20000", fit_frontend)
    print(r[0].stdout)

 ```

 %% Cell type:markdown id:62a5f0a4-4ed2-455b-8cb3-377ca568d5e3 tags:

 ## Clean up

 %% Cell type:code id:07b08249-3188-4f69-beae-0689a04f1986 tags:

 ``` python
 providers.destroy()
 ```

--- a/fit_and_g5k/02_monitoring_stack.ipynb
+++ b/fit_and_g5k/02_monitoring_stack.ipynb
@@ -312,12 +312,12 @@
    "\n",
    "from enoslib.infra.providers import Providers\n",
    "\n",
-    "iotlab_provider = en.Iotlab(fit_conf)\n",
-    "g5k_provider = en.G5k(g5k_conf)\n",
+    "iotlab_provider = en.Iotlab(fit_conf, name=\"Iotlab\")\n",
+    "g5k_provider = en.G5k(g5k_conf, name=\"G5k\")\n",
    "\n",
    "providers = Providers([iotlab_provider,g5k_provider])\n",
    "\n",
-    "roles,networks = providers.init(86400)\n",
+    "roles, networks = providers.init(86400)\n",
    "iotlab_provider, g5k_provider = providers.providers"
   ]
  },

 %% Cell type:markdown id:2b44eb9a-c604-4e6f-9fef-ea20e90e1ae1 tags:

 # Smoothing heterogeneity: the monitoring case

 Revisiting here the <a href="../g5k/02_observability.ipynb">"observability notebook"</a> in the context of mixed G5K/FIT resources.

 ---

 - Website: https://discovery.gitlabpages.inria.fr/enoslib/index.html
 - Instant chat: https://framateam.org/enoslib
 - Source code: https://gitlab.inria.fr/discovery/enoslib
 ---


 ## Prerequisites

 <div class="alert alert-block alert-warning">
    Make sure you've run the one time setup for your environment (e.g one time setup for https://labs.iot-lab.info)
 </div>


 %% Cell type:markdown id:f9f007ba tags:

 ## Monitoring options

 Experimenters rely on monitoring to get insight on their deployments.
 Fit and G5k provide their own job monitoring infrastructure. An thus an experimenter have different choices:
 - Get metrics from the infrastructure (G5K REST API / FIT oml files). This is especially interesting for environnemental metrics (power ...)
 - Deploy their own monitoring tool


 %% Cell type:markdown id:f2541874 tags:


 ### On Grid'5000

 There are also different options to interact with the REST API (see https://www.grid5000.fr/w/Grid5000:Software#Experiment_management_tools) and of course `curl`. REST API specification is available here: https://api.grid5000.fr/doc/3.0/


 From python you can use [`python-grid5000`](https://pypi.org/project/python-grid5000/).

 %% Cell type:code id:064c1ed1 tags:

 ``` python
 from grid5000 import Grid5000
 from pathlib import Path

 conf = Path.home() / ".python-grid5000.yaml"

 gk = Grid5000.from_yaml(conf)
 ```

 %% Cell type:code id:508cc5e1 tags:

 ``` python
 # get the list of the available metrics for a given cluster
 import json

 metrics = gk.sites["lyon"].clusters["nova"].metrics
 print(json.dumps(metrics, indent=4))
 ```

 %% Cell type:code id:f846652e tags:

 ``` python
 [m["name"] for m in metrics]
 ```

 %% Cell type:code id:0acd8e0c tags:

 ``` python
 import time
 metric = "wattmetre_power_watt"
 now = time.time()
 measurements = gk.sites["lyon"].metrics.list(nodes="nova-1,nova-2,nova-3", start_time=now - 1000, metrics=metric)

 # alternatively one can pass a job_id
 # measurements = gk.sites["lyon"].metrics.list(job_id=1307628, metrics=metric)
 measurements[:10]
 ```

 %% Cell type:code id:8ab58c55 tags:

 ``` python
 import pandas as pd

 df = pd.DataFrame([m.to_dict() for m in measurements])
 df["timestamp"] = pd.to_datetime(df["timestamp"])
 import seaborn as sns

 sns.relplot(data=df, x="timestamp", y="value", hue="device_id", alpha=0.7)
 ```

 %% Cell type:markdown id:0e12b1d0 tags:

 ### On IOT-LAB

 One need to attach a `profile` to the experiment (either at reservation time or dynamically)

 %% Cell type:code id:f8a458b9 tags:

 ``` python
 !ls tutorial_m3.elf || wget -q https://raw.githubusercontent.com/wiki/iot-lab/iot-lab/firmwares/tutorial_m3.elf
 ```

 %% Cell type:code id:b712f822-e282-42f4-b10e-16b30c4d32ef tags:

 ``` python
 import enoslib as en
 from enoslib.infra.enos_iotlab.configuration import ConsumptionConfiguration

 en.init_logging()

 FIT_SITE="grenoble"

 fit_conf = (
    en.IotlabConf.from_settings(job_name="tutorial_m3", walltime="02:00")
    .add_machine(roles=["xp_fit"], archi="m3:at86rf231", site=FIT_SITE, number=1, image="tutorial_m3.elf", profile="test_profile")
    .add_profile(name="test_profile", archi="m3", consumption=ConsumptionConfiguration(current=True, power=True, voltage=True, period=8244, average=4))
 )
 fit_conf

 fit = en.Iotlab(fit_conf)
 fit_roles, _ = fit.init()
 ```

 %% Cell type:code id:847d83ae tags:

 ``` python
 # wait a bit for data to be collected and flushed
 import time
 time.sleep(20)
 ```

 %% Cell type:code id:d9905e90-9222-4661-85fb-8ab07e9053b9 tags:

 ``` python
 fit.collect_data_experiment()
 ```

 %% Cell type:code id:76d76a3e tags:

 ``` python
 import tarfile
 job_id = fit.client.get_job_id()
 tar = tarfile.open("%s.tar.gz" % (job_id)) # tar = tarfile.open("%s-%s.iot-lab.info.tar.gz" % (job_id, FIT_SITE))
 tar.extractall(path=".")
 tar.close()
 ```

 %% Cell type:code id:e5728ad8 tags:

 ``` python
 %matplotlib widget
 from oml_plot_tools import consum

 from pathlib import Path
 # iterate over all the *.om found
 consuption_dir = Path(str(job_id)) / "consumption"
 for consumption in consuption_dir.glob("*.oml"):
  print(consumption)
  data = consum.oml_load(consumption)
  data = data[0:1000]
  consum.consumption_plot(data, 'consumption', ('power'))
 ```

 %% Cell type:code id:1916ed43 tags:

 ``` python
 fit.destroy()
 ```

 %% Cell type:markdown id:6a84fb84 tags:

 ## User defined monitoring


 The user deploys its own monitoring solution
 EnOSlib provides different ways of doing that:
 - a lightweight monitoring: based on independant monitoring processes running on each host
 - an heavyweight monitoring: based on a classical monitoring stack (Telegraf + InfluxBD/Prometheus + Grafana)

 Monitoring stacks are exposed using EnOSlib's `Service`. We show that these service can mix G5K + FIT nodes.

 We reserve:
 - 2 nodes on G5K
 - 1 A8 nodes on FIT

 %% Cell type:code id:695b7ee7-c8c2-4785-8679-9237f1163c25 tags:

 ``` python
 import enoslib as en

 en.init_logging()

 network = en.G5kNetworkConf(type="prod", roles=["my_network"], site="rennes")

 g5k_conf = (
    en.G5kConf.from_settings(job_type="allow_classic_ssh", job_name="fit_g5k_monitoring")
    .add_network_conf(network)
    .add_machine(
        roles=["xp", "collector"], cluster="paravance", nodes=1, primary_network=network
    )
    .add_machine(
        roles=["xp", "agent"], cluster="paravance", nodes=1, primary_network=network
    )
    .finalize()
 )
 g5k_conf
 ```

 %% Cell type:code id:97b67dc9 tags:

 ``` python
 FIT_SITE="grenoble"

 fit_conf = (
    en.IotlabConf.from_settings(job_name="riot_a8", walltime="02:00")
    .add_machine(roles=["agent"], archi="a8:at86rf231", site=FIT_SITE, number=1)
 )
 fit_conf
 ```

 %% Cell type:code id:093b1a48-3338-4b1e-a64a-cd2dd68fb952 tags:

 ``` python
 # Here we set up a special provider that will try to reserve both G5k and Iotlab configuration simultaneously

 from enoslib.infra.providers import Providers

-iotlab_provider = en.Iotlab(fit_conf)
-g5k_provider = en.G5k(g5k_conf)
+iotlab_provider = en.Iotlab(fit_conf, name="Iotlab")
+g5k_provider = en.G5k(g5k_conf, name="G5k")

 providers = Providers([iotlab_provider,g5k_provider])

-roles,networks = providers.init(86400)
+roles, networks = providers.init(86400)
 iotlab_provider, g5k_provider = providers.providers
 ```

 %% Cell type:markdown id:55acb50a-8a55-4e6e-ba60-5b2e257736aa tags:

 ### Lightweight monitoring
 EnOSlib has a lighweight monitoring service based on Dstat.

 ```
 Dstat service anatomy:
 ----------------------
 .deploy: deploys a monitoring process in the background on each targetted nodes and store metric in a file
 .destroy: stop the monitoring process on each node
 .backup: retrieve all the monitoring files back to your control node for post-mortem analysis
 ```

 %% Cell type:code id:89bb0eba-60d1-4046-a943-6764cefe54e1 tags:

 ``` python
 # a context manager that do deploy when entering and destroy + backup when exiting
 with en.Dstat(nodes=roles["agent"]) as d:
    import time
    time.sleep(5)
    en.run_command("apt install -y stress && stress --timeout 10 --cpu 8", roles=roles["G5k"] & roles["agent"])
    time.sleep(5)
 ```

 %% Cell type:code id:9f1dfd8b-0321-450e-88d6-84837a0b1c36 tags:

 ``` python
 import seaborn as sns

 sns.relplot(data=en.Dstat.to_pandas(d.backup_dir), x="epoch", y="idl", hue="host")
 ```

 %% Cell type:markdown id:da2a6712-59fc-4062-8918-d08cb0ea8505 tags:

 ### Heavyweight monitoring stack

 ![monitoring.png](attachment:cde7736f-c3de-435e-8778-aab716158721.png)

 %% Cell type:code id:bde3efe4-05bb-4fb0-89c6-bedf4c624e66 tags:

 ``` python
 en.run_command("dhclient -6 br0", roles=roles["G5k"])

 # mandatory for the following
 roles = en.sync_info(roles,networks)
 ```

 %% Cell type:markdown id:1171991d-f0a2-4afd-a8fd-338eea1a1323 tags:

 A good practice to deploy docker on G5k is to the EnOSlib service to
 - install docker agent on all nodes
 - configure the deamon to use the Grid'5000 registry cache
 - bind the docker state directory to a place with some space

 %% Cell type:code id:fb7a2d49-4680-4005-a067-8cf744d5675b tags:

 ``` python
 docker = en.Docker(agent=roles["xp"],
                 bind_var_docker="/tmp/docker",
                 registry_opts=dict(type="external", ip="docker-cache.grid5000.fr", port=80))
 docker.deploy()
 ```

 %% Cell type:markdown id:283d49b3-fea8-4f4d-bfd8-53e6170c2606 tags:

 We're now ready to instantiate the Monitoring Service,

 We only need to make sure to bind the various client on the IPv6 networks and make the client use these networks.

 %% Cell type:code id:814f4e07-942d-49a8-9137-6b220e8fb366 tags:

 ``` python
 def get_nets(networks, net_type):
    """ Aux method to filter networks from roles """
    return set([
        n for net_list in networks.values() for n in net_list
        if isinstance(n.network, net_type)
    ])
 ```

 %% Cell type:code id:adfdb69d-3956-4f70-b4c9-e2e756f985fc tags:

 ``` python
 from ipaddress import IPv6Network
 get_nets(networks, IPv6Network)
 ```

 %% Cell type:code id:d482a38e-7b19-4c9e-b80a-2a6a6c202bb8 tags:

 ``` python
 m = en.TIGMonitoring(
    # collector is the node where the DB will be deployed and
    # where the monitoring data will be sent
    collector=roles["collector"][0],
    # agent are the nodes where the monitoring agent will be deployed
    # on g5k we deploy those using docker
    # while on fit we deploy the agent from the binary
    agent=roles["agent"],
    # ui is the node where the dashboard will be deployed
    ui=roles["collector"][0],
    # networks represent the network to use (agent <-> collector communication)
    networks=get_nets(networks, IPv6Network)
 )
 m.deploy()
 ```

 %% Cell type:code id:bcf0086a-44ba-417a-9afb-22e2e694a385 tags:

 ``` python
 print(f"""
 Access the UI at {m.ui.address}:3000 (admin/admin)")
 ---
 tip1: create a ssh port forwarding -> ssh -NL 3000:{m.ui.address}:3000 access.grid5000.fr (and point your browser to http://localhost:3000)
 tip2: use a proxy socks -> ssh -ND 2100 access.grid5000.fr (and point your browser to http://{m.ui.address}:3000)
 tip3: use the G5K vpn
 """)
 ```

 %% Cell type:code id:568744ad-d833-49fa-ba8f-084a2b3aff9c tags:

 ``` python
 g5k_provider.fw_create(proto="tcp+udp", port=8086)
 ```

 %% Cell type:code id:0b446e85-395d-4498-a637-717813af1eb3 tags:

 ``` python
 proc = (roles["G5k"] & roles["agent"])[0].processor
 cpu_stress = proc.cores * proc.count

 en.run_command(f"stress --cpu {cpu_stress} --timeout 30", roles=[host for host in roles["G5k"] if host in roles["agent"]])
 ```

 %% Cell type:markdown id:31d2f076-1a96-488b-bb75-03c00b94b7e2 tags:

 This is an example of the outcome in the dashboard

 ![image.png](attachment:bfb85aed-068f-40b7-81da-48a4a23ff2fc.png)

 %% Cell type:markdown id:5a39b968-9c0b-4223-b0bd-11fff63df6e5 tags:

 ## Clean up

 %% Cell type:code id:02329f1c-e61f-41b1-8435-a7a98784670c tags:

 ``` python
 providers.destroy()
 ```

--- a/setup_for_use_in_labs.iot-lab.info.ipynb
+++ b/setup_for_use_in_labs.iot-lab.info.ipynb
@@ -66,7 +66,7 @@
    "# !pip install -U --user --pre  enoslib[silecs]\n",
    "\n",
    "# master version (might be wild)\n",
-    "!pip install  -U --user enoslib[silecs]@git+https://gitlab.inria.fr/discovery/enoslib"
+    "!pip install  -U --user enoslib[silecs]@git+https://gitlab.inria.fr/discovery/enoslib@dev/hostsview"
   ]
  },
  {
@@ -319,14 +319,6 @@
   "source": [
    "en.check()"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3baa0a7b-6319-414d-8f5c-8c40922d6a41",
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {

 %% Cell type:markdown id:b46992a6-b75c-45d8-bacb-a8d54ea8de44 tags:
  
 # One time setup for use in labs.iot-lab.info
  
 ---
  
 - Website: https://discovery.gitlabpages.inria.fr/enoslib/index.html
 - Instant chat: https://framateam.org/enoslib
 - Source code: https://gitlab.inria.fr/discovery/enoslib
  
 ---
  
  
 Prerequisites:
  
 - Connect to this Jupyter lab instance https://labs.iot-lab.info
 - Clone the enoslib tutorias source: `git clone https://gitlab.inria.fr/msimonin/enoslib-tutorials` at the root of the jupyter lab instance
 - Open this notebook at `enoslib-tutorials/`
  
 ![overview.png](attachment:f8a0b9c9-9ffe-470a-bd8b-83cd3b209be0.png)
  
 In this setup the control machine is the jupyter instance running at  https://labs.iot-lab.info
  
 %% Cell type:markdown id:9499ca23-b2c7-4b52-a5f2-6c661e8e3497 tags:
  
 ## Software dependencies
  
 First things first, you'll need EnOSlib library to go through this tutorial.
  
 - Launch the following
 - Restart the kernel
  
 %% Cell type:code id:f78e397c-52bd-4c78-94ef-eacc16170832 tags:
  
 ``` python
 !pip install -U pip
  
 # pick one
 # latest stable
 # !pip install -U --user enoslib[silecs]
  
 # latest alpha version
 # !pip install -U --user --pre  enoslib[silecs]
  
 # master version (might be wild)
-!pip install  -U --user enoslib[silecs]@git+https://gitlab.inria.fr/discovery/enoslib
+!pip install  -U --user enoslib[silecs]@git+https://gitlab.inria.fr/discovery/enoslib@dev/hostsview
 ```
  
 %% Cell type:code id:c1f7a79e-9d32-44b1-a805-88cbdcea174c tags:
  
 ``` python
 # This must not fail (restart the kernel after installing the library)
 import enoslib as en
 ```
  
 %% Cell type:markdown id:840f2c4c-0e54-4b8f-ad5b-e10644dc0b4e tags:
  
 ### Setup access to Grid'5000
  
 We need two kind of accesses to the Grid'5000 platform:
 - REST API Access to interact with  Grid'5000 exposed resources (jobs, nodes, networks, firewall ....)
 - SSH Access to control the remote machine once acquired
  
 ---
 #### REST API Access
 The REST API access is performed using HTTP request using BasicAuth Authentication mecanism through the [`python-grid5000`](https://pypi.org/project/python-grid5000/) wrapper.
 This requires to store the username/password in a file located in your home directory.
  
 %% Cell type:code id:71b8d3a8-a8bb-4d49-83fe-b2334aa29167 tags:
  
 ``` python
 from grid5000.cli import auth
 # CHANGE ME!
 G5K_USER = "msimonin"
 # will prompt for the password and write the authentication file
 auth(G5K_USER)
 ```
  
 %% Cell type:code id:3320e287-270f-4122-a7cb-688e2705bf13 tags:
  
 ``` python
 # Test the api access
 from grid5000 import Grid5000
 from pathlib import Path
 conf_file = Path.home() / ".python-grid5000.yaml"
 gk = Grid5000.from_yaml(conf_file)
 gk.sites.list()
 ```
  
 %% Cell type:code id:241809aa-41bc-4171-a0f5-874f82425aaf tags:
  
 ``` python
 # have some fun with the g5k python client ...
 gk.sites["rennes"].clusters.list()
 ```
  
 %% Cell type:code id:6b246f33-2a09-456d-9330-58d8c48d7375 tags:
  
 ``` python
 print(gk.sites["rennes"].clusters["paravance"].nodes.list()[0])
 ```
  
 %% Cell type:markdown id:3f0a7522-7c68-4c8f-8197-002b7959c9d6 tags:
  
 ---
 #### SSH access
 The SSH access relies on SSH key authentication.
 Add the content of the public key in the Grid'5000 interface in [your Grid'5000 account](https://api.grid5000.fr/stable/users/) > ssh_keys
  
 %% Cell type:code id:3e3564b4-b675-4e37-a731-16e0d274f1ad tags:
  
 ``` python
 !cat ~/.ssh/id_rsa.pub
 ```
  
 %% Cell type:markdown id:5b86261e-357a-4c5f-aeaa-f26c26fdb3ef tags:
  
 <div class="alert alert-block alert-warning">
    Note that you don't own the corresponding private key, so that's probably better to remove it from <a href="https://api.grid5000.fr/stable/users/">your Grid'5000 account</a> as soon as possible.
 </div>
  
 %% Cell type:markdown id:cf150071-950b-44e4-a2f2-499d00fe1a46 tags:
  
 #### Set a SSH config to ease the connection to G5K
  
 For instance this eases the direct connection to unconstrained node (A8, Rpi ...) that run a SSH server.
  
 %% Cell type:code id:0e4d6cf8-65ae-474e-8549-0a7709133770 tags:
  
 ``` python
 from pathlib import Path
  
 config = Path.home() / ".ssh/config"
  
 # Access to machines inside G5K
 config_g5k = f"""Host !access.grid5000.fr *.grid5000.fr
   User {G5K_USER}
   ProxyJump {G5K_USER}@access.grid5000.fr
   StrictHostKeyChecking no
   UserKnownHostsFile /dev/null
   ForwardAgent yes
  
   Host access.grid5000.fr
   User {G5K_USER}
   StrictHostKeyChecking no
   UserKnownHostsFile /dev/null
   ForwardAgent yes
 """
  
  
 FIT_USER = "simonin"
 configs_iot = []
 for site in ["grenoble", "lille", "saclay", "strasbourg"]:
    configs_iot.append(f"""Host !{site}.iot-lab.info *.{site}.iot-lab.info
   User {FIT_USER}
   ProxyJump {FIT_USER}@{site}.iot-lab.info
   StrictHostKeyChecking no
   UserKnownHostsFile /dev/null
   ForwardAgent yes
  
   Host {site}.iot-lab.info
   User {FIT_USER}
   StrictHostKeyChecking no
   UserKnownHostsFile /dev/null
   ForwardAgent yes
 """
 )
  
 configs = [config_g5k] + configs_iot
 config.write_text("\n".join(configs))
 ```
  
 %% Cell type:code id:b0d7da19-ea57-40e4-9314-ad628ffbd6ff tags:
  
 ``` python
 # This must work
 !ssh rennes.grid5000.fr hostname
 ```
  
 %% Cell type:code id:c49e26a9-965b-4501-afa2-e689ef88e2c3 tags:
  
 ``` python
 # Testing the ssh connection to g5k using the  lib
 import enoslib as en
  
 en.init_logging()
  
 frontend = en.Host("rennes.grid5000.fr", user=G5K_USER)
  
 r = en.run_command("hostname", roles=frontend, raw=True)
  
 print(f"We've succesfully reached {r[0].stdout}")
 ```
  
 %% Cell type:code id:5794c7a6-2b8e-4dd2-bd6f-2e6b08b0ba14 tags:
  
 ``` python
 # This also must work
 !ssh grenoble.iot-lab.info hostname
 ```
  
 %% Cell type:code id:e8dd01c1-586b-47a6-96b5-c488a01b4696 tags:
  
 ``` python
 fit_grenoble = en.Host("grenoble.iot-lab.info", user=FIT_USER)
  
 r = en.run_command("hostname", roles=fit_grenoble, raw=True)
  
 print(f"We've succesfully reached {r[0].stdout}")
 ```
  
 %% Cell type:code id:f4ff0662-03b0-407f-95a4-66f453b11905 tags:
  
 ``` python
 # you're ready !
 ```
  
 %% Cell type:markdown id:5e3e183e-4043-4569-b960-039b5977ed45 tags:
  
 ## Check
  
 %% Cell type:code id:725cf9e2-971a-4ec8-aa69-72e0a6242281 tags:
  
 ``` python
 en.check()
 ```
-
-%% Cell type:code id:3baa0a7b-6319-414d-8f5c-8c40922d6a41 tags:
-
-``` python
-```