Mentions légales du service

Skip to content
Snippets Groups Projects
Commit e9668b6f authored by SIMONIN Matthieu's avatar SIMONIN Matthieu
Browse files

refacto netem

parent da60e6b5
No related branches found
No related tags found
No related merge requests found
Network Emulation
=================
Network Emulation API
=====================
Introduction
------------
.. automodule:: enoslib.service.emul
.. _emul:
Netem Emulation
Netem emulation
---------------
.. automodule:: enoslib.service.emul.netem
......
......@@ -7,24 +7,39 @@ Network Emulation
:depth: 2
This tutorial illustrates how network constraints can be enforced using |enoslib|.
Another resources can be found in the :ref:`netem`.
Another resources can be found in the :ref:`emul`.
Setting up homogeneous constraints
-----------------------------------
When all your nodes share the same network limitations you can use the
`SimpleNetem` service.
`Netem` service.
.. literalinclude:: network_emulation/tuto_simple_netem.py
.. literalinclude:: network_emulation/tuto_svc_netem.py
:language: python
:linenos:
Setting up heterogeneous constraints
-------------------------------------
You can use the Netem service for this purpose. The example is based on the
You can use the HTBNetem service for this purpose. The example is based on the
G5K provider, but can be adapted to another one if desired.
.. literalinclude:: network_emulation/tuto_svc_htb.py
:language: python
:linenos:
Working at the network device level
-----------------------------------
If you know the device on which limitations will be applied you can use the
functions ``netem`` or ``netem_htb``.
.. literalinclude:: network_emulation/tuto_netem.py
:language: python
:linenos:
.. literalinclude:: network_emulation/tuto_netem_htb.py
:language: python
:linenos:
\ No newline at end of file
......@@ -21,7 +21,7 @@ from enoslib.service.locust.locust import Locust
from enoslib.service.k3s.k3s import K3s
from enoslib.service.monitoring.monitoring import TIGMonitoring, TPGMonitoring
from enoslib.service.emul.netem import Netem, netem, NetemOutConstraint, NetemInOutSource, NetemInConstraint
from enoslib.service.emul.htb import NetemHTB, HTBConstraint, HTBSource
from enoslib.service.emul.htb import netem_htb, NetemHTB, HTBConstraint, HTBSource
from enoslib.service.skydive.skydive import Skydive
......
......@@ -42,4 +42,4 @@ function.
.. [#3] https://wiki.linuxfoundation.org/networking/netem
.. [#4] https://tldp.org/HOWTO/Traffic-Control-HOWTO/classful-qdiscs.html
"""
pass
\ No newline at end of file
pass
......@@ -3,15 +3,14 @@ from enoslib.utils import _check_tmpdir
import logging
import os
from dataclasses import dataclass, field
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Set
from typing import Any, Dict, Iterable, List, Mapping, Optional, Set, Tuple
from enoslib.api import play_on, run_ansible
from enoslib.constants import TMP_DIRNAME
from enoslib.objects import Networks, Roles
from enoslib.objects import Host, Networks, Roles
from jsonschema import validate
from ..service import Service
from .objects import Constraint, Source
from .schema import SCHEMA
from .utils import (
_build_commands,
......@@ -84,7 +83,7 @@ def _build_ip_constraints(
@dataclass(eq=True, frozen=True)
class HTBConstraint(Constraint):
class HTBConstraint(object):
"""An HTB constraint.
.. note ::
......@@ -159,7 +158,7 @@ class HTBConstraint(Constraint):
@dataclass
class HTBSource(Source):
class HTBSource(object):
"""Model a host and all the htb constraints.
Args
......@@ -172,6 +171,7 @@ class HTBSource(Source):
constraints are left to the application.
"""
host: Host
constraints: Set[HTBConstraint] = field(default_factory=set)
def add_constraint(self, *args, **kwargs):
......@@ -182,13 +182,7 @@ class HTBSource(Source):
"""
self.constraints.add(HTBConstraint(*args, **kwargs))
# type: ignore[override]
def add_constraints(self, constraints: Iterable[HTBConstraint]):
"""Add some constraints.
constraints: iterable of
:py:class:`enoslib.service.netem.netem.HTBConstraint`
"""
self.constraints = self.constraints.union(set(constraints))
def add_commands(self) -> List[str]:
......@@ -210,6 +204,9 @@ class HTBSource(Source):
htb_cmds.extend(tc.commands(idx))
return htb_cmds
def all_commands(self) -> Tuple[List[str], List[str], List[str]]:
return self.remove_commands(), self.add_commands(), self.commands()
def netem_htb(
htb_hosts: List[HTBSource],
......@@ -366,7 +363,7 @@ class NetemHTB(Service):
:language: python
:linenos:
"""
"""
NetemHTB.is_valid(network_constraints)
self.network_constraints = network_constraints
self.roles = roles if roles is not None else []
......
import logging
from dataclasses import dataclass, field
from typing import List, Mapping, Optional
from typing import Iterable, List, Mapping, Optional, Set, Tuple
from enoslib.api import play_on
from enoslib.objects import Host, Network
from enoslib.service.service import Service
from .objects import Constraint, Source
from .utils import _build_commands, _build_options, _combine, _validate
logger = logging.getLogger(__name__)
class NetemConstraint(object):
pass
@dataclass(eq=True, frozen=True)
class NetemOutConstraint(Constraint):
class NetemOutConstraint(NetemConstraint):
"""A Constraint on the egress part of a device.
Args:
......@@ -44,25 +47,10 @@ class NetemInConstraint(NetemOutConstraint):
"""A Constraint on the ingress part of a device.
Inbound limitations works differently.
Quoted from https://wiki.linuxfoundation.org/networking/netem
> How can I use netem on incoming traffic?
> You need to use the Intermediate Functional Block pseudo-device IFB .
This network device allows attaching queuing discplines to incoming
packets.
.. code-block:: bash
modprobe ifb
ip link set dev ifb0 up
tc qdisc add dev eth0 ingress
tc filter add dev eth0 parent ffff: \
protocol ip u32 match u32 0 0 flowid 1:1 action mirred egress redirect dev ifb0
tc qdisc add dev ifb0 root netem delay 750ms
see https://wiki.linuxfoundation.org/networking/netem
Assuming that the modprobe is already done this is exactly what we'll do
to enforce the inbound constraints.
We'll create an ifb, redirect incoming traffic to it and apply some
queuing discipline using netem.
Args:
ifb: the ifb name (e.g. ifb0) that will be used. That means that the
......@@ -70,12 +58,15 @@ class NetemInConstraint(NetemOutConstraint):
"""
def add_commands(self, ifb: str) -> List[str]:
"""Return the commands that adds the ifb."""
return [f"ip link set dev {ifb} up", f"tc qdisc add dev {self.device} ingress"]
def remove_commands(self, ifb: str) -> List[str]:
"""Return the commands that remove the qdisc from the ifb and the net device."""
return super().remove_commands(ifb) + [f"tc qdisc del dev {ifb} root"]
def commands(self, ifb: str) -> List[str]:
"""Return the commands that redirect and apply netem constraints on the ifb."""
return [
(
f"tc filter add dev {self.device} parent ffff: "
......@@ -87,8 +78,8 @@ class NetemInConstraint(NetemOutConstraint):
@dataclass
class NetemInOutSource(Source):
"""Model a host and the constraints on its networkd devices.
class NetemInOutSource(object):
"""Model a host and the constraints on its network devices.
Args:
inbound : The constraints to set on the ingress part of the host
......@@ -96,16 +87,27 @@ class NetemInOutSource(Source):
outbound: The constraints to set on the egress part of the host
devices
"""
inbounds: List[NetemOutConstraint] = field(default_factory=list)
outbounds: List[NetemInConstraint] = field(default_factory=list)
host: Host
constraints: Set[NetemConstraint] = field(default_factory=set)
def _commands(self, _c: str) -> List[str]:
cmds = []
for xgress in [self.inbounds, self.outbounds]:
for idx, constraint in enumerate(xgress):
cmds.extend(getattr(constraint, _c)(f"ifb{idx}"))
for idx, constraint in enumerate(self.constraints):
cmds.extend(getattr(constraint, _c)(f"ifb{idx}"))
return cmds
@property
def inbounds(self) -> List[NetemInConstraint]:
return [c for c in self.constraints if isinstance(c, NetemInConstraint)]
@property
def outbounds(self) -> List[NetemOutConstraint]:
return [c for c in self.constraints if isinstance(c, NetemOutConstraint)]
def add_constraints(self, constraints: Iterable[NetemInConstraint]):
self.constraints = self.constraints.union(set(constraints))
def add_commands(self) -> List[str]:
return self._commands("add_commands")
......@@ -115,13 +117,8 @@ class NetemInOutSource(Source):
def commands(self) -> List[str]:
return self._commands("commands")
# type: ignore[override]
def add_constraints(self, constraints: List[Constraint]):
if len(constraints) == 0 or len(constraints) > 2:
raise ValueError("One or two constraints must be passed")
self.inbounds.append(constraints[0])
if len(constraints) == 2:
self.outbounds.append(constraints[1])
def all_commands(self) -> Tuple[List[str], List[str], List[str]]:
return self.remove_commands(), self.add_commands(), self.commands()
def netem(
......@@ -167,8 +164,7 @@ def netem(
# by default modprobe provision 2, so we default to this value
numifbs = 2
for source in sources:
numifbs = max(numifbs, len(source.inbounds), len(source.outbounds))
numifbs = max(numifbs, len(source.constraints))
# Run the commands on the remote hosts (only those involved)
# First allocate a sufficient number of ifbs
with play_on(roles=roles, extra_vars=options) as p:
......@@ -245,7 +241,6 @@ class Netem(Service):
source.add_constraints(constraints)
total_ifbs = max(total_ifbs, len(interfaces))
sources.append(source)
netem(sources, self.extra_vars, chunk_size)
def backup(self):
......
from dataclasses import dataclass
from typing import Iterable, List
from enoslib.objects import Host
class Constraint(object):
pass
@dataclass
class Source(object):
"""Base class for our limitations.
Constraint are applied on a host so, keeping this as an attribute.
Args:
host: the host on which the constraint will be applied
"""
host: Host
def add_constraints(self, constraints: Iterable[Constraint]):
"""Subclass specific, because that depends how constraints can be combined."""
pass
def all_commands(self):
"""all in one."""
r = self.remove_commands()
a = self.add_commands()
h = self.commands()
return r, a, h
import copy
from typing import Dict, List
from enoslib.api import run_ansible
from enoslib.utils import _check_tmpdir
from enoslib.constants import TMP_DIRNAME
......@@ -8,10 +9,8 @@ from itertools import groupby
from operator import attrgetter
import os
import re
from typing import Dict, List, Sequence
from enoslib.objects import Roles
from .objects import Source
SERVICE_PATH = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
PLAYBOOK = os.path.join(SERVICE_PATH, "netem.yml")
......@@ -48,7 +47,7 @@ def _build_options(extra_vars, options):
return _options
def _build_commands(sources: Sequence[Source]):
def _build_commands(sources):
"""Source agnostic way of recombining the list of constraints."""
_remove = defaultdict(list)
_add = defaultdict(list)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment