WiP Introduce VMonG5k provider

It's a new provider that provides virtual machines on Grid'5000.

Example:

```
conf = Configuration.from_settings(job_name="tuto-vmong5k")\
    .add_machine(roles=["control"],
                 cluster="parapluie",
                 number=3,
                 flavour="large")\
    .add_machine(roles=["compute"],
                 cluster="parapluie",
                 number=100,
                 flavour="tiny")\
    .finalize()
provider = VMonG5k(conf)

roles, networks = provider.init()

```

It adds a new `enos_vmong5k` package alongside new modules:

- `enos_vmong5k.configuration`
- `enos_vmong5k.schema`
- `enos_vmong5k.constants`
- `enos_vmong5k.provider`

and also new unit tests and documentation (functionnal tests in
docs/tutorials/vmong5k)

The VmonG5K provider internally uses the G5k provider. In particular it sets
the job_type to allow_classic_ssh and claim an extra slash_22 subnet.

docs/apidoc/infra.rst

@@ -51,6 +51,22 @@ G5k API
     :undoc-members:
     :show-inheritance:
 
+Virtual Machines on Grid5000 (VMonG5k)
+--------------------------------------
+
+VMonG5k Provider Class
+^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: enoslib.infra.enos_vmong5k.provider
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+VMonG5k Schema
+^^^^^^^^^^^^^^^
+
+.. literalinclude:: ../../enoslib/infra/enos_vmong5k/schema.py
+
 Static
 ------
 

docs/tutorials/grid5000.rst

@@ -117,4 +117,5 @@ You can found it `here
 It starts reserve non-deploy nodes and a subnet and start virtual machines on
 them.
 
-
+.. hint::
+  Note that it is now possible to use the ``VMonG5k`` provider directly.

docs/tutorials/index.rst

@@ -16,3 +16,4 @@ Tutorials
     using-tasks
     ansible-integration
     chameleon
+    vmong5k

docs/tutorials/vmong5k.rst

+Tutorial 6 - Working with virtual machines on Grid'5000
+=======================================================
+
+This tutorial leverages the ``VmonG5k`` provider: a provider that provisions
+virtual machines for you on Grid'5000. 
+
+Installation
+------------
+
+
+On Grid'5000, you can go with a virtualenv :
+
+.. code-block:: bash
+
+    $ virtualenv venv
+    $ source venv/bin/activate
+    $ pip install -U pip
+
+    $ pip install enoslib
+
+Basic example
+-------------
+
+We'll imagine a system that requires 100 compute machines and 3 controller machines.
+We express this using the ~VmonG5K~ provider:
+
+.. literalinclude:: vmong5k/tuto_vmong5k.py
+   :language: python
+   :linenos:
+
+
+- You can launch the script using :
+
+    .. code-block:: bash
+
+        $ python tuto_vmg5k.py
+
+- The raw data structures of EnOSlib will be displayed and you should be able to
+  connect to any machine using SSH and the root account.
+
+Notes
+-----
+
+* The ``VmonG5K`` provider internally uses the ``G5k`` provider. In particular
+  it sets the ``job_type`` to ``allow_classic_ssh`` and claim an extra
+  ``slash_22`` subnet.
+
+* SSH access will be granted to the VMs using the ``~/.ssh/id_rsa | ~/.ssh/id_rsa.pub`` keypair.
+  So these files should be present in your home directory.

docs/tutorials/vmong5k/tuto_vmong5k.py

+from enoslib.infra.enos_vmong5k.provider import VMonG5k
+from enoslib.infra.enos_vmong5k.configuration import Configuration
+
+import logging
+import os
+
+logging.basicConfig(level=logging.INFO)
+
+# path to the inventory
+inventory = os.path.join(os.getcwd(), "hosts")
+
+# claim the resources
+conf = Configuration.from_settings(job_name="tuto-vmong5k")\
+    .add_machine(roles=["control"],
+                 cluster="parapluie",
+                 number=3,
+                 flavour="large")\
+    .add_machine(roles=["compute"],
+                 cluster="parapluie",
+                 number=100,
+                 flavour="tiny")\
+    .finalize()
+provider = VMonG5k(conf)
+
+roles, networks = provider.init()
+print(roles)
+print(networks)

enoslib/host.py

 # -*- coding: utf-8 -*-
+import copy
 
 
 class Host(object):
 
     def __init__(
             self,
-            address,
+            address, *,
             alias=None,
             user=None,
             keyfile=None,
@@ -20,6 +21,9 @@ class Host(object):
         self.port = port
         self.extra = extra or {}
 
+    def to_dict(self):
+        return copy.deepcopy(self.__dict__)
+
     def __repr__(self):
         args = [self.alias, "address=%s" % self.address]
         return "Host(%s)" % ", ".join(args)

enoslib/infra/enos_vmong5k/__init__.py

+# init

enoslib/infra/enos_vmong5k/ansible/cloud-init-data/meta-data

+#instance-id: iid-local01
+#local-hostname: example-vm
+public-keys:
+  - "{{ pubkey }}" 

enoslib/infra/enos_vmong5k/ansible/cloud-init-data/user-data

+#cloud-config
+disable_root: false
+bootcmd:
+  - sed -i "/127.0.0.1/d" /etc/hosts
+  - echo "127.0.0.1 localhost" >> /etc/hosts
+  - echo "10.158.0.2   vm_control-0  vm_control-0.grid5000.fr" >> /etc/hosts
+  - echo "10.158.0.3   vm_control-1  vm_control-1.grid5000.fr" >> /etc/hosts
+  - echo "10.158.0.4   vm_control-2  vm_control-2.grid5000.fr" >> /etc/hosts
+  - echo "10.158.0.5   vm_network-0  vm_network-0.grid5000.fr" >> /etc/hosts
+  - echo "10.158.0.6   vm_compute-0  vm_compute-0.grid5000.fr" >> /etc/hosts

enoslib/infra/enos_vmong5k/ansible/domain.xml.j2

+<domain type='kvm'>
+ <cpu mode='host-passthrough'></cpu>
+ <name>{{ item.alias }}</name>
+ <memory>{{ item.mem }}</memory>
+ <vcpu>{{ item.core }}</vcpu>
+ <features>
+   <acpi/>
+ </features>
+ <os>
+   <type arch="x86_64">hvm</type>
+ </os>
+ <clock offset="localtime"/>
+ <on_poweroff>destroy</on_poweroff>
+ <on_reboot>restart</on_reboot>
+ <on_crash>destroy</on_crash>
+ <devices>
+   <emulator>/usr/bin/kvm</emulator>
+   <disk type='file' device='disk'>
+     <driver name='qemu' type='qcow2'/>
+     <source file='/tmp/disks/{{ item.alias}}'/>
+     <target dev='vda' bus='virtio'/>
+   </disk>
+   <disk type='file' device='cdrom'>
+     <source file='/tmp/cloud-init-data-{{ item.alias }}.iso'/>
+     <target dev='vdb' bus='virtio'/>
+     <readonly/>
+   </disk>
+   <interface type='bridge'>
+     <source bridge='br0'/>
+     <mac address='{{ item.eui }}'/>
+   </interface>
+   <serial type='pty'>
+     <source path='/dev/ttyS0'/>
+     <target port='0'/>
+   </serial>
+   <console type='pty'>
+     <source path='/dev/ttyS0'/>
+     <target port='0'/>
+   </console>
+ </devices>
+</domain>

enoslib/infra/enos_vmong5k/ansible/meta-data.j2

+---

enoslib/infra/enos_vmong5k/ansible/site.yaml

+---
+- name: This is a play for virtual machines on Grid'5000
+  hosts: all
+  vars:
+    pubkey: "{{lookup('file', '~/.ssh/id_rsa.pub')}}"
+  tasks:
+
+    - name: Destroy running virtual machines (vm -1 / 1)
+      virt:
+        name: "{{ item.alias }}"
+        state: destroyed
+      ignore_errors: yes
+      with_items: "{{ vms[inventory_hostname] }}"
+
+    - name: Unregister existing virtual machines (vm -0 / 1)
+      virt:
+        name: "{{ item.alias }}"
+        command: undefine
+      ignore_errors: yes
+      with_items: "{{ vms[inventory_hostname] }}"
+
+    - name: Enable nested virtualization
+      shell: |
+        modprobe -r kvm_intel
+        modprobe kvm_intel nested=1
+
+    - name: Unmount the tmpfs
+      mount:
+        path: /tmp/disks
+        state: unmounted
+      when:
+        - tmpfs is defined
+        - tmpfs
+
+    - name: Remove a tmpfs for the vms
+      file:
+        path: /tmp/disks
+        state: absent
+      when:
+        - tmpfs is defined
+        - tmpfs
+
+    - name: Create a directory for hosting the virtual disks
+      file:
+        path: /tmp/disks
+        state: directory
+        mode: 777
+
+    - name: Mount the tmpfs
+      shell: "mount -t tmpfs -o size={{ tmpfs }} tmpfs /tmp/disks"
+      when:
+        - tmpfs is defined
+        - tmpfs
+
+    - name: Removing previous cloud init data
+      file:
+        path: "/tmp/cloud-init-data-{{ item.alias }}"
+        state: absent
+      loop: "{{ vms[inventory_hostname] }}"
+
+    - name: Removing previous cloud init data iso
+      file:
+        path: "/tmp/cloud-init-data-{{ item.alias }}.iso"
+        state: absent
+      loop: "{{ vms[inventory_hostname] }}"
+
+    - name: Creating cloud init data directory
+      file:
+        path: "/tmp/cloud-init-data-{{ item.alias }}"
+        state: directory
+      loop: "{{ vms[inventory_hostname] }}"
+
+    - name: Generate meta-data for cloud-init
+      template:
+        src: meta-data.j2
+        dest: "/tmp/cloud-init-data-{{ item.alias }}/meta-data"
+      loop: "{{ vms[inventory_hostname] }}"
+
+    - name: Generate user data for cloud-init
+      template:
+        src: user-data.j2
+        dest: "/tmp/cloud-init-data-{{ item.alias }}/user-data"
+      loop: "{{ vms[inventory_hostname] }}"
+
+    #  Create one iso per vm
+    - name: Create the iso for cloud-init
+      shell: "cd /tmp &&  genisoimage -output cloud-init-data-{{ item.alias }}.iso -volid cidata -joliet -rock cloud-init-data-{{ item.alias }}/user-data cloud-init-data-{{ item.alias }}/meta-data"
+      loop: "{{ vms[inventory_hostname] }}"
+
+    - name: Check base image
+      stat:
+        path: "{{ base_image }}"
+      register: p
+
+    - name: Verify base image accessibility
+      fail:
+        msg: "Base image does not exist. Verify this path is valid: {{ base_image }}"
+      when: p.stat.exists == False
+
+      # NOTE(msimonin): We don't copy in the ramfs in a first iteration
+    - name: Copy base image
+      shell: "cp {{ base_image }} /tmp/kenan-base-image.qcow2"
+
+    - name: Link virtual image to base image
+      shell: "qemu-img create -f qcow2 -o backing_file=/tmp/kenan-base-image.qcow2 /tmp/disks/{{ item.alias }}"
+      with_items: "{{ vms[inventory_hostname] }}"
+
+    - name: Define virtual machines (vm 0 / 1)
+      virt:
+        name: "{{ item.alias }}"
+        command: define
+        xml: "{{ lookup('template', 'domain.xml.j2') }}"
+      with_items: "{{ vms[inventory_hostname] }}"
+
+    - name: Launch virtual machines (vm 1 / 1)
+      virt:
+        name: "{{ item.alias }}"
+        state: running
+      with_items: "{{ vms[inventory_hostname] }}"

enoslib/infra/enos_vmong5k/ansible/user-data.j2

+#cloud-config
+hostname: {{ item.alias }}
+fqdn: {{ item.alias }}.grid5000.fr
+
+disable_root: false
+bootcmd:
+  - mkdir -p /root/.ssh
+  - echo "{{ pubkey }}" > /root/.ssh/authorized_keys
+  - sed -i "/127.0.0.1/d" /etc/hosts
+  - echo "127.0.0.1 localhost" >> /etc/hosts
+{% for _vms in vms.values() %}
+{% for vm in _vms %}
+  - echo "{{ vm.address }}   {{ vm.alias }}  {{ vm.alias }}.grid5000.fr" >> /etc/hosts
+{% endfor %}
+{% endfor %}

enoslib/infra/enos_vmong5k/configuration.py

+from ..configuration import BaseConfiguration
+from .constants import (DEFAULT_FLAVOUR, DEFAULT_IMAGE, DEFAULT_JOB_NAME,
+                        DEFAULT_NETWORKS, DEFAULT_NUMBER, DEFAULT_QUEUE,
+                        DEFAULT_WALLTIME, FLAVOURS)
+from .schema import SCHEMA
+
+import uuid
+
+
+class Configuration(BaseConfiguration):
+
+    _SCHEMA = SCHEMA
+
+    def __init__(self):
+        super().__init__()
+        self.job_name = DEFAULT_JOB_NAME
+        self.queue = DEFAULT_QUEUE
+        self.walltime = DEFAULT_WALLTIME
+        self.image = DEFAULT_IMAGE
+
+        self._machine_cls = MachineConfiguration
+        self._network_cls = str
+
+        self.networks = DEFAULT_NETWORKS
+
+    @classmethod
+    def from_dictionnary(cls, dictionnary, validate=True):
+        if validate:
+            cls.validate(dictionnary)
+
+        self = cls()
+        for k in self.__dict__.keys():
+            v = dictionnary.get(k)
+            if v is not None:
+                setattr(self, k, v)
+
+        _resources = dictionnary["resources"]
+        _machines = _resources["machines"]
+        _networks = _resources["networks"]
+        self.networks = [NetworkConfiguration.from_dictionnary(n) for n in
+                         _networks]
+        self.machines = [MachineConfiguration.from_dictionnary(m) for m in
+                         _machines]
+
+        self.finalize()
+        return self
+
+    def to_dict(self):
+        d = {}
+        for k, v in self.__dict__.items():
+            if v is None or k in ["machines", "networks", "_machine_cls",
+                                  "_network_cls"]:
+                continue
+            d.update({k: v})
+
+        d.update(resources={
+            "machines": [m.to_dict() for m in self.machines],
+            "networks": self.networks
+        })
+        return d
+
+
+class MachineConfiguration:
+
+    def __init__(self, *,
+               roles=None,
+               cluster=None,
+               flavour=None,
+               number=DEFAULT_NUMBER):
+        self.roles = roles
+
+        if flavour is None:
+            self.flavour = DEFAULT_FLAVOUR
+        if isinstance(flavour, dict):
+            self.flavour = flavour
+        elif isinstance(flavour, str):
+            self.flavour = FLAVOURS[flavour]
+        else:
+            self.flavour = DEFAULT_FLAVOUR
+
+        self.number = number
+        self.cluster = cluster
+
+        # a cookie to identify uniquely the group of machine this is used when
+        # redistributing the vms to pms in the provider. I've the feeling that
+        # this could be used to express some affinity between vms
+        self.cookie = uuid.uuid4().hex
+
+    @classmethod
+    def from_dictionnary(cls, dictionnary):
+        kwargs = {}
+        roles = dictionnary["roles"]
+        kwargs.update(roles=roles)
+        flavour = dictionnary.get("flavour")
+        if flavour is not None:
+            # The flavour name is used in the dictionnary
+            # This makes a diff with the constructor where
+            # A dict describing the flavour is given
+            kwargs.update(flavour=FLAVOURS[flavour])
+        number = dictionnary.get("number")
+        if number is not None:
+            kwargs.update(number=number)
+
+        cluster = dictionnary["cluster"]
+        if cluster is not None:
+            kwargs.update(cluster=cluster)
+
+        return cls(**kwargs)
+
+    def to_dict(self):
+        d = {}
+        d.update(roles=self.roles, flavour=self.flavour, number=self.number,
+                 cluster=self.cluster)
+        return d

enoslib/infra/enos_vmong5k/constants.py

+import os
+
+DEFAULT_JOB_NAME = "EnOslib-vmong5k"
+DEFAULT_QUEUE = "default"
+DEFAULT_WALLTIME = "02:00:00"
+DEFAULT_IMAGE = "/grid5000/virt-images/debian9-x64-base.qcow2"
+PROVIDER_PATH = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+PLAYBOOK_PATH = os.path.join(PROVIDER_PATH, "ansible", "site.yaml")
+
+
+#: Sizes of the machines available for the configuration
+FLAVOURS = {
+    "tiny": {
+        "core": 1,
+        "mem": 512
+    },
+    "small": {
+        "core": 1,
+        "mem": 1024
+    },
+    "medium": {
+        "core": 2,
+        "mem": 2048
+    },
+    "big": {
+        "core": 3,
+        "mem": 3072,
+    },
+    "large": {
+        "core": 4,
+        "mem": 4096
+    },
+    "extra-large": {
+        "core": 6,
+        "mem": 6144
+    }
+}
+
+DEFAULT_FLAVOUR = FLAVOURS["tiny"]
+
+DEFAULT_NETWORKS = ["enos_network"]
+
+DEFAULT_NUMBER = 1

enoslib/infra/enos_vmong5k/provider.py

+# -*- coding: utf-8 -*-
+from .constants import PLAYBOOK_PATH
+from ..provider import Provider
+
+
+from enoslib.api import generate_inventory, run_ansible
+from enoslib.host import Host
+import enoslib.infra.enos_g5k.api as enoslib
+import enoslib.infra.enos_g5k.configuration as g5kconf
+import enoslib.infra.enos_g5k.provider as g5kprovider
+
+
+from collections import defaultdict
+import execo_g5k.api_utils as utils
+from ipaddress import IPv4Address
+import itertools
+import logging
+from netaddr import EUI, mac_unix_expanded
+import os
+
+
+logger = logging.getLogger(__name__)
+
+
+def _get_clusters_site(clusters):
+    sites = enoslib.get_clusters_sites(clusters)
+    site_names = set(sites.values())
+    if len(site_names) > 1:
+        raise Exception("Multi-site deployment is not supported yet")
+
+    return site_names.pop()
+
+
+def get_subnet_ip(mac):
+    # This is the format allowed on G5K for subnets
+    address = ['10'] + [str(int(i, 2)) for i in mac.bits().split('-')[-3:]]
+    return IPv4Address('.'.join(address))
+
+
+def mac_range(start, stop, step=1):
+    for item in range(int(start) + 1, int(stop), step):
+        yield EUI(item, dialect=mac_unix_expanded)
+
+
+def get_host_cores(cluster):
+    hosts = utils.get_cluster_hosts(cluster)
+    # (suppose) all hosts are the same in a cluster
+    attributes = utils.get_host_attributes(hosts[-1])
+    processors = attributes['architecture']['nb_procs']
+    cores = attributes['architecture']['nb_cores']
+
+    # number of cores as reported in the Website
+    return cores * processors
+
+
+def find_nodes_number(machine):
+    cores = get_host_cores(machine.cluster)
+    return - ((-1 * machine.number * machine.flavour["core"]) // cores)
+
+
+def _do_build_g5k_conf(vmong5k_conf, site):
+    g5k_conf = g5kconf.Configuration.from_settings(
+        job_name=vmong5k_conf.job_name,
+        walltime=vmong5k_conf.walltime,
+        queue=vmong5k_conf.queue,
+        job_type="allow_classic_ssh")
+    prod_network = g5kconf.NetworkConfiguration(roles=["prod"],
+                                           id="prod",
+                                           type="prod",
+                                           site=site)
+    subnet_roles = vmong5k_conf.networks
+    subnet_roles.append("__subnet__")
+    subnet = g5kconf.NetworkConfiguration(roles=subnet_roles,
+                                          id="subnet",
+                                          type="slash_22",
+                                          site=site)
+    # let's start by adding the networks
+    g5k_conf.add_network_conf(prod_network)\
+           .add_network_conf(subnet)
+
+    for _, machine in enumerate(vmong5k_conf.machines):
+        # we hide a descriptor of group in the original machines
+        roles = machine.roles
+        roles.append(machine.cookie)
+        g5k_conf.add_machine(roles=roles,
+                            cluster=machine.cluster,
+                            nodes=find_nodes_number(machine),
+                            primary_network=prod_network)
+    return g5k_conf
+
+
+def _build_g5k_conf(vmong5k_conf):
+    """Build the conf of the g5k provider from the vmong5k conf."""
+    clusters = [m.cluster for m in vmong5k_conf.machines]
+    site = _get_clusters_site(clusters)
+
+    return _do_build_g5k_conf(vmong5k_conf, site)
+
+
+def _distribute(machines, g5k_roles, g5k_subnet):
+    vmong5k_roles = defaultdict(list)
+    euis = mac_range(EUI(g5k_subnet["mac_start"]), EUI(g5k_subnet["mac_end"]))
+    for machine in machines:
+        pms = g5k_roles[machine.cookie]
+        pms_it = itertools.cycle(pms)
+        for idx in range(machine.number):
+            name = "vm-{}".format(idx)
+            pm = next(pms_it)
+            vm = VirtualMachine(name, next(euis), machine.flavour, pm)
+
+            for role in machine.roles:
+                vmong5k_roles[role].append(vm)
+    return dict(vmong5k_roles)
+
+
+def _index_by_host(roles):
+    virtual_machines_by_host = defaultdict(set)
+    for vms in roles.values():
+        for virtual_machine in vms:
+            host = virtual_machine.pm
+            # Two vms are equal if they have the same euis
+            virtual_machines_by_host[host.alias].add(
+                virtual_machine)
+    # now serialize all the thing
+    vms_by_host = defaultdict(list)
+    for host, vms in virtual_machines_by_host.items():
+        for virtual_machine in vms:
+            vms_by_host[host].append(virtual_machine.to_dict())
+
+    return dict(vms_by_host)
+
+
+def start_virtualmachines(provider_conf, g5k_init, vmong5k_roles):
+    vms_by_host = _index_by_host(vmong5k_roles)
+
+    extra_vars = {'vms': vms_by_host,
+                  'base_image': provider_conf.image}
+    pm_inventory_path = os.path.join(os.getcwd(), "pm_hosts")
+    generate_inventory(*g5k_init, pm_inventory_path)
+    # deploy virtual machines with ansible playbook
+    run_ansible([PLAYBOOK_PATH], pm_inventory_path, extra_vars)
+
+
+class VirtualMachine(Host):
+
+    def __init__(self, name, eui, flavour, pm):
+        super().__init__(str(get_subnet_ip(eui)), alias=name)
+        self.core = flavour["core"]
+        # libvirt uses kiB by default
+        self.mem = int(flavour["mem"]) * 1024
+        self.eui = eui
+        self.pm = pm
+
+    def to_dict(self):
+        d = super().to_dict()
+        d.update(core=self.core, mem=self.mem, eui=str(self.eui),
+                 pm=self.pm.to_dict())
+        return d
+
+    def __hash__(self):
+        return int(self.eui)
+
+    def __eq__(self, other):
+        return int(self.eui) == int(other.eui)
+
+
+class VMonG5k(Provider):
+    """The provider to use when deploying virtual machines on Grid'5000."""
+
+    def init(self, force_deploy=False):
+        g5k_conf = _build_g5k_conf(self.provider_conf)
+        g5k_provider = g5kprovider.G5k(g5k_conf)
+        g5k_roles, g5k_networks = g5k_provider.init()
+        g5k_subnet = [n for n in g5k_networks if "__subnet__" in n["roles"]][0]
+        vmong5k_roles = _distribute(self.provider_conf.machines,
+                                    g5k_roles,
+                                    g5k_subnet)
+
+        start_virtualmachines(self.provider_conf,
+                              (g5k_roles, g5k_networks),
+                              vmong5k_roles)
+
+        return vmong5k_roles, [g5k_subnet]
+
+    def destroy(self):
+        pass
+
+    def __str__(self):
+        return 'VMonG5k'

enoslib/infra/enos_vmong5k/schema.py

+from .constants import FLAVOURS
+
+QUEUE_TYPES = ["default", "testing", "production"]
+
+SCHEMA = {
+    "type": "object",
+    "properties": {
+        "resources": {"$ref": "#/resources"},
+        "job_name": {"type": "string"},
+        "queue": {"type": "string", "enum": QUEUE_TYPES},
+        "walltime": {"type": "string"},
+        "image": {"type": "string"}
+    },
+    "additionalProperties": False,
+    "required": ["resources"],
+    "resources": {
+        "title": "Resource",
+
+        "type": "object",
+        "properties": {
+            "machines": {
+                "type": "array",
+                "items": {"$ref": "#/machine"}
+            },
+            "networks": {"type": "array", "items": {"type": "string"}}
+        },
+        "additionalProperties": False,
+        "required": ["machines", "networks"],
+    },
+
+    "machine": {
+        "title": "Compute",
+        "type": "object",
+        "properties": {
+            "roles": {"type": "array", "items": {"type": "string"}},
+            "cluster": {"type": "string"},
+            "number": {"type": "number"},
+            "oneOf": [
+                {"flavour": {"type": "string", "enum": list(FLAVOURS.keys())}},
+                {"flavour_desc": {"$ref": "#/flavour_desc"}}
+            ],
+
+        },
+        "required": ["roles", "cluster"]
+    },
+}

enoslib/tests/unit/infra/enos_vmong5k/__init__.py

+# init

enoslib/tests/unit/infra/enos_vmong5k/test_configuration.py

+from enoslib.infra.enos_vmong5k.configuration import Configuration, MachineConfiguration
+import enoslib.infra.enos_vmong5k.constants as constants
+
+from ... import EnosTest
+
+import jsonschema
+
+
+class TestConfiguration(EnosTest):
+
+    def test_from_dictionnary_minimal(self):
+        d = {
+            "resources": {
+                "machines": [],
+                "networks": []
+            }
+        }
+        conf = Configuration.from_dictionnary(d)
+        self.assertEqual(constants.DEFAULT_JOB_NAME, conf.job_name)
+        self.assertEqual([], conf.machines)
+        self.assertEqual([], conf.machines)
+
+    def test_from_dictionnary_custom_backend(self):
+        d = {
+            "job_name": "test-job",
+            "walltime": "12:34:56",
+            "resources": {
+                "machines": [],
+                "networks": []
+            }
+        }
+        conf = Configuration.from_dictionnary(d)
+        self.assertEqual("test-job", conf.job_name)
+        self.assertEqual("12:34:56", conf.walltime)
+
+    def test_programmatic(self):
+        conf = Configuration()
+        conf.add_machine_conf(MachineConfiguration(roles=["r1"],
+                                                   flavour=constants.FLAVOURS["large"],
+                                                   number=10,
+                                                   cluster="test-cluster"
+                                                   ))
+        conf.finalize()
+        self.assertEqual(1, len(conf.machines))
+        # default networks
+        self.assertEqual(constants.DEFAULT_NETWORKS, conf.networks)
+
+    def test_programmatic_missing_keys(self):
+        conf = Configuration()
+        conf.add_machine_conf(MachineConfiguration())
+        with self.assertRaises(jsonschema.exceptions.ValidationError) as _:
+            conf.finalize()
+
+
+class TestMachineConfiguration(EnosTest):
+
+    def test_from_dictionnary_minimal(self):
+        d = {
+            "roles": ["r1"],
+            "cluster": "test-cluster"
+        }
+        conf = MachineConfiguration.from_dictionnary(d)
+        self.assertEqual(constants.DEFAULT_FLAVOUR, conf.flavour)
+
+    def test_from_dictionnary(self):
+        d = {
+            "roles": ["r1"],
+            "flavour": "large",
+            "number": 2,
+            "cluster": "test-cluster"
+        }
+        conf = MachineConfiguration.from_dictionnary(d)
+        self.assertEqual(constants.FLAVOURS["large"], conf.flavour)
+        self.assertEqual(2, conf.number)

enoslib/tests/unit/infra/enos_vmong5k/test_provider.py

+from enoslib.host import Host
+from enoslib.infra.enos_vmong5k.configuration import Configuration, MachineConfiguration
+from enoslib.infra.enos_vmong5k.provider import (_do_build_g5k_conf,
+                                                 _distribute, _index_by_host,
+                                                 VirtualMachine)
+from enoslib.tests.unit import EnosTest
+
+from netaddr import EUI
+
+import mock
+
+
+class TestBuildG5kConf(EnosTest):
+
+    @mock.patch("enoslib.infra.enos_vmong5k.provider.find_nodes_number", return_value=2)
+    def test_do_build_g5k_conf(self, mock_find_node_number):
+        conf = Configuration()
+        conf.add_machine(roles=["r1"],
+                         cluster="cluster1",
+                         number=10,
+                         flavour="tiny")
+        conf.finalize()
+        g5k_conf = _do_build_g5k_conf(conf, "rennes")
+        # it's valid
+        g5k_conf.finalize()
+
+
+        # machines
+        self.assertEqual(1, len(g5k_conf.machines))
+        machine = g5k_conf.machines[0]
+        self.assertEqual("cluster1", machine.cluster)
+        self.assertEqual(2, machine.nodes)
+        # role have been expanded with the unique cookie
+        self.assertEqual(2, len(machine.roles))
+
+        # networks
+        self.assertEqual(2, len(g5k_conf.networks))
+        self.assertTrue(g5k_conf.networks[0].type in ["prod", "slash_22"])
+        self.assertTrue(g5k_conf.networks[1].type in ["prod", "slash_22"])
+
+
+class TestDistribute(EnosTest):
+
+    def test_distribute_minimal(self):
+        machine = MachineConfiguration(roles=["r1"],
+                                         flavour="tiny",
+                                         cluster="paravance",
+                                         number=1)
+        machines = [machine]
+        host = Host("paravance-1")
+
+        g5k_roles = {
+            "r1": [host],
+            machine.cookie: [host]
+        }
+
+        g5k_subnet = {
+            "mac_start": "00:16:3E:9E:44:00",
+            "mac_end": "00:16:3E:9E:47:FE"
+        }
+
+        vmong5k_roles = _distribute(machines, g5k_roles, g5k_subnet)
+        self.assertEqual(1, len(vmong5k_roles["r1"]))
+        vm = vmong5k_roles["r1"][0]
+        # we skip the first mac 
+        self.assertEqual(EUI(int(EUI(g5k_subnet['mac_start'])) + 1), vm.eui)
+        self.assertEqual(host, vm.pm)
+
+
+    def test_distribute_2_vms_1_host(self):
+        machine = MachineConfiguration(roles=["r1"],
+                                         flavour="tiny",
+                                         cluster="paravance",
+                                         number=2)
+        machines = [machine]
+        host = Host("paravance-1")
+
+        g5k_roles = {
+            "r1": [host],
+            machine.cookie: [host]
+        }
+
+        g5k_subnet = {
+            "mac_start": "00:16:3E:9E:44:00",
+            "mac_end": "00:16:3E:9E:47:FE"
+        }
+
+        vmong5k_roles = _distribute(machines, g5k_roles, g5k_subnet)
+        self.assertEqual(2, len(vmong5k_roles["r1"]))
+        vm = vmong5k_roles["r1"][0]
+        # we skip the first mac 
+        self.assertEqual(EUI(int(EUI(g5k_subnet['mac_start'])) + 1), vm.eui)
+        self.assertEqual(host, vm.pm)
+
+        vm = vmong5k_roles["r1"][1]
+        self.assertEqual(EUI(int(EUI(g5k_subnet['mac_start'])) + 2), vm.eui)
+        self.assertEqual(host, vm.pm)
+
+    def test_distribute_2_vms_2_hosts(self):
+        machine = MachineConfiguration(roles=["r1"],
+                                         flavour="tiny",
+                                         cluster="paravance",
+                                         number=2)
+        machines = [machine]
+        host0 = Host("paravance-1")
+        host1 = Host("paravance-2")
+
+        g5k_roles = {
+            "r1": [host0, host1],
+            machine.cookie: [host0, host1]
+        }
+
+        g5k_subnet = {
+            "mac_start": EUI("00:16:3E:9E:44:00"),
+            "mac_end": EUI("00:16:3E:9E:47:FE")
+        }
+
+        vmong5k_roles = _distribute(machines, g5k_roles, g5k_subnet)
+        self.assertEqual(2, len(vmong5k_roles["r1"]))
+        vm = vmong5k_roles["r1"][0]
+        # we skip the first mac
+        self.assertEqual(EUI(int(g5k_subnet['mac_start']) + 1), vm.eui)
+        self.assertEqual(host0, vm.pm)
+
+        vm = vmong5k_roles["r1"][1]
+        self.assertEqual(EUI(int(g5k_subnet['mac_start']) + 2), vm.eui)
+        self.assertEqual(host1, vm.pm)
+
+
+class TestIndexByHost(EnosTest):
+
+    def test_index_by_host(self):
+        host = Host("paravance-1")
+        machine = VirtualMachine("vm-test",
+                                 EUI("00:16:3E:9E:44:01"),
+                                 {"core": 1, "mem": 512},
+                                 host)
+        roles = {"r1": [machine]}
+        vms_by_host = _index_by_host(roles)
+        self.assertTrue(host.alias in vms_by_host)
+        self.assertEqual(1, len(vms_by_host[host.alias]))