Commit 795bebb6 authored by SIMONIN Matthieu's avatar SIMONIN Matthieu
Browse files

Merge branch 'dev/static_ips' into 'master'

Dev/static ips

See merge request !5
parents a713b8fe e78d5fe6
Pipeline #194258 passed with stages
in 26 minutes and 33 seconds
---
variables:
GIT_SUBMODULE_STRATEGY: recursive
# https://docs.gitlab.com/ee/ci/runners/README.html#git-submodule-strategy
# normal -> only top level are fetched before the job starts
GIT_SUBMODULE_STRATEGY: normal
LD_PRELOAD: ""
RUST_VERSION: 1.46.0
......@@ -12,113 +14,64 @@ workflow:
when: never
- when: always
# unstable, yeah !
image: simgrid/unstable:latest
stages:
- build1
- build2
- test
###############################################################################
#
# Build Qemu patched for Tansiv
# -- test locally:
# $) gitlab-runner exec docker qemu
#
###############################################################################
qemu:
stage: build1
script:
#
# Some missing build tools
#
- apt-get update
- apt-get install -y build-essential git pkg-config libglib2.0-dev libpixman-1-dev
#
- git clone --depth 1 https://gitlab.inria.fr/msimonin/qemu.git -b tantap src/qemu
- cd src/qemu
- ./configure && make config-host.h
artifacts:
paths:
- src/qemu
###############################################################################
#
# Build tansiv and some cppunit stuffs
# -- test locally:
# $) gitlab-runner exec docker tansiv
#
###############################################################################
tansiv:
stage: build2
script:
#
# Some missing build tools
#
- apt-get update
- apt-get install -y curl git build-essential pkg-config libboost-dev cmake libcppunit-dev libglib2.0-dev clang libclang-dev
- curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup-init
- sh rustup-init -y --profile minimal --default-toolchain $RUST_VERSION
- export PATH=$HOME/.cargo/bin:$PATH
#
# Build tests
# TODO(msimonin): make all (but we'll need to adapt all vsg clients ...)
#
- mkdir -p build
- cd build
- cmake .. && make
# unittests
# NOTE(msimonin): as long as https://gitlab.inria.fr/quinson/2018-vsg/-/issues/5 is around
# we run our tests isolated in different process.
# This is far from ideal since reports will be spread in the output of each process
# But at least catch2 lets use specify this easily...
# But ...
# for some reason this fails: ./tests --list-test-names-only | xargs -d "\n" -n 1 ./tests
# grrr!
- ./tests "VSG receive one message"
- ./tests "VSG deliver one message"
- ./tests "VSG send piggyback port"
- ./tests "piggyback port"
artifacts:
paths:
- build
# expected /opt structure:
# /opt
# examples/ -> compiled examples
# platform/ -> some simgrid plaform files
# tansiv/ -> result of tansiv make install (include + lib + bin)
# qemu/ -> result of qemu make install (include + lib + bin)
- build
- publish
# unstable, yeah !
image: simgrid/unstable:latest
###############################################################################
#
# Build fake-vm
# -- test locally:
# $) gitlab-runner exec docker fake-vm --docker-volumes $(pwd)/opt:/opt
#
###############################################################################
fake-vm:
stage: build2
image: rust:${RUST_VERSION}-slim-buster
build:
stage: build
tags:
- large
script:
#
# Some missing build tools
#
- export PROJECT_DIR=$(pwd)
- apt-get update
- apt-get install -y build-essential pkg-config libglib2.0-dev clang
- cd src/fake-vm
- apt-get install -y curl git build-essential pkg-config libboost-dev cmake libcppunit-dev libglib2.0-dev clang libclang-dev libpixman-1-dev flex bison
- curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup-init
- sh rustup-init -y --profile minimal --default-toolchain $RUST_VERSION
- export PATH=$HOME/.cargo/bin:$PATH
- mkdir -p build
# build test
- cd build
- cmake .. && make config-host.h
# fake-vm tests
- cd ${PROJECT_DIR}/src/fake-vm
- make build
- make test
- cd ${PROJECT_DIR}/build
- make
# NOTE(msimonin): as long as https://gitlab.inria.fr/quinson/2018-vsg/-/issues/5 is around
# we run our tests isolated in different process.
# This is far from ideal since reports will be spread in the output of each process
# But at least catch2 lets use specify this easily...
# But ...
# for some reason this fails: ./tests --list-test-names-only | xargs -d "\n" -n 1 ./tests
# grrr!
- ./tests "VSG receive one message"
- ./tests "VSG deliver one message"
- ./tests "VSG send piggyback port"
- ./tests "piggyback port"
###############################################################################
#
# Dummy ping/pong example
# inputs:
# opt/ -> artifact of the build phase
#
###############################################################################
send:
stage: test
docker:
image: docker:latest
stage: publish
tags:
- large
script:
- cd build
- ./tansiv examples/send/nova_cluster.xml examples/send/deployment.xml
# login iff we're on gitlab ci
- |
if [[ -n "${CI_JOB_TOKEN}" ]]
then
docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
export DOCKER_IMAGE=${CI_REGISTRY_IMAGE}/tansiv:${CI_COMMIT_SHORT_SHA}
docker build -t ${DOCKER_IMAGE} .
docker push ${DOCKER_IMAGE}
else
export DOCKER_IMAGE=tansiv:latest
docker build -t ${DOCKER_IMAGE} .
fi
\ No newline at end of file
[submodule "src/slirp"]
path = src/slirp
url = https://gitlab.inria.fr/msimonin/libslirp.git
[submodule "src/crossbeam"]
path = src/crossbeam
url = https://gitlab.inria.fr/lrilling/crossbeam.git
branch = dev/waitfree
[submodule "src/qemu"]
path = src/qemu
url = https://gitlab.inria.fr/msimonin/qemu
......@@ -76,5 +76,5 @@
},
"editor.formatOnSave": true,
"clang-format.assumeFilename": "${workspaceRoot}/.clang-format",
"editor.tabSize": 2,
"python.formatting.provider": "black",
}
\ No newline at end of file
......@@ -24,10 +24,16 @@ find_package(SimGrid REQUIRED)
include_directories("${SimGrid_INCLUDE_DIR}" "${FAKEVM_INCLUDE_DIR}" "${VSG_INCLUDE_DIR}" SYSTEM)
# To build fake-vm/qemu implementation we need to generate the config-host.h header
add_custom_target(config-host.h COMMAND ./configure --target-list=x86_64-softmmu && make -j config-host.h WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/qemu)
# fake-vm client lib (rust implementation)
# installed in /opt/fake-vm
# compiled in RELEASE mode (I've got the power !)
add_custom_target(fake-vm ALL COMMAND make PREFIX=/opt/fake-vm RELEASE=0 install WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/fake-vm)
add_dependencies(fake-vm config-host.h)
# build the qemu base image
add_custom_target(packer COMMAND packer build debian-10.3.0-x86_64.json WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/packer)
# vsg lib (actor side)
add_library(vsg STATIC src/vsg/vsg.c src/vsg/log.c)
......@@ -56,7 +62,7 @@ configure_file(examples/benchs/nova_cluster.xml examples/benchs/nova_cluster.xml
add_dependencies(gettimeofday fake-vm)
# Qemus
configure_file(examples/qemus/boot_and_log.sh examples/qemus/boot_and_log.sh COPYONLY)
configure_file(examples/qemus/boot.py examples/qemus/boot.py COPYONLY)
configure_file(examples/qemus/deployment.xml examples/qemus/deployment.xml COPYONLY)
configure_file(examples/qemus/nova_cluster.xml examples/qemus/nova_cluster.xml COPYONLY)
......
# tansiv in docker :)
FROM simgrid/unstable:latest
WORKDIR /app
COPY . /app
RUN apt-get update
RUN apt-get install -y build-essential \
libboost-dev \
cmake \
libcppunit-dev \
libglib2.0-dev \
clang \
libclang-dev \
curl \
git \
pkg-config \
libglib2.0-dev \
libpixman-1-dev \
flex \
bison \
genisoimage
# a cup of rust
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup-init
RUN sh rustup-init -y --profile minimal --default-toolchain 1.46.0
ENV PATH=/root/.cargo/bin:$PATH
RUN cargo --help
WORKDIR /app/build
RUN cmake .. && make && make DESTDIR=/opt/tansiv install
# run some tests about the rust part
# WORKDIR /app/src/fake-vm
# RUN make && make test
# run some tests about the c/c++ part
WORKDIR /app/build
RUN ./tests --list-test-names-only | xargs -d "\n" -n1 ./tests
# build qemu with the new network backend (tantap)
WORKDIR /app/src/qemu
RUN make -j && make install
# make some room
RUN rm -rf /app
ENV PATH=/opt/tansiv/usr/local/bin:$PATH
ENV QEMU=/usr/local/bin/qemu-system-x86_64
# create an ssh key (not really usefull, we'd want our local key to be pushed
# inside the vm anyway)
RUN mkdir -p /root/.ssh
RUN ssh-keygen -t rsa -P '' -f /root/.ssh/id_rsa
WORKDIR /srv
ENTRYPOINT ["tansiv"]
\ No newline at end of file
......@@ -11,24 +11,76 @@ mkdir build
cd build
cmake .. && make
```
TODO c'est pas tout à fait ça :( (voir la chaîne de build dans le ci ou l'image docker).
# Example jouets
# Tests unitaires
... TEC: Travail En Cours...
Il y a des tests à différents niveau:
- `dummy_ping/pong`: Implementation d'échange de messages au niveau du
protocole vsg. Les messages transitent par simgrid qui gère le temps de
propagation en fonction du fichier de plateform. Le déploiement
- 1 processus ping: envoie un ping (à travers la socket vsg) aux deux processus pong
- 2 prorcessus pong: lorsqu'ils reçoivent le ping, décodent la source
(entête du protocole vsg) et renvoie un pong.
- code rust: `cargo test` dans les sous répertoire
- tests unitaires de l'implémentation cliente (sans simgrid)
- code c++: `./tests`
- tests des bindings c du code rust (le code qui sera embarqué dans les
applis c/c++)
- à cause de https://gitlab.inria.fr/quinson/2018-vsg/-/issues/5 on doit les lancer individuellement:
`./tests --list-test-names-only | xargs -d "\n" -n1 ./tests`
# Tests fonctionnels
Cette fois Simgrid est impliqué.
- `send`: échange d'un message entre deux processus (utilise l'implémentation sur process de fake-vm)
```
./tansiv examples/send/nova_cluster.xml examples/send/deployment.xml --log=vm_interface.threshold:debug --log=vm_coordinator.threshold:debug
```
- `qemus`: Lance des machines virtuelles dont les communications passent sur
simgrid. Il faut:
- le programme `genisoimage` (pour générer l'iso cloud-init), `qemu-img` (pour créer les disques des VMs à la volée)
- une image de base compatible: par exemple celle construite à l'aide de `packer`.
- notre backend réseau `tantap` (un backend `tap` modifié qui
intercepte/réinjecte les communications vers/en provenance de simgrid)
```
cd examples/qemus
../../tansiv nova_cluster.xml deployment.xml --log=vm_interface.threshold:debug --log=vm_coordinator.threshold:debug
```
# Docker (par exemple sur g5k)
```
g5k-setup-docker -t
cd examples/qemus
(get the debian-10.3.0-x86_64.qcow2 image)
docker run \
--network host \
--device /dev/net/tun \
--cap-add=NET_ADMIN \
-e AUTOCONFIG_NET=1 \
-e IMAGE=debian-10.3.0-x86_64.qcow2 \
-v /home/msimonin/.ssh/id_rsa.pub:/root/.ssh/id_rsa.pub \
-v $(pwd):/srv \
-ti registry.gitlab.inria.fr/msimonin/2018-vsg/tansiv:ad782302 nova_cluster.xml deployment.xml
```
ensuite (ça met un peu de temps à arriver)
```
# connection using the management interface should be able after some time
$) ssh -l root 172.16.0.10
# test d'un ping à travers vsg
root@tansiv-192-168-120-10:~# ping t11
PING t11 (192.168.120.11) 56(84) bytes of data.
64 bytes from t11 (192.168.120.11): icmp_seq=1 ttl=64 time=400 ms
64 bytes from t11 (192.168.120.11): icmp_seq=2 ttl=64 time=400 ms
```
- `qemu_sink`: illustre la modification de SLIRP (backend réseau de QEMU) pour faire
transiter les messages sortants (UDP seulement pour l'instant) à travers
vsg.
- `constant_rate`: 2 processes s'envoie des messages à vitesse constantes
(mesurée en nombre de messages par seconde). C'est pratique pour avoir un
comportement déterministe dans les échanges des messages.
\ No newline at end of file
#!/usr/bin/env python3
import argparse
from ipaddress import IPv4Interface
import logging
from pathlib import Path
import os
from subprocess import check_call
import tempfile
from typing import Any, Dict, List, Optional
import yaml
# some env variable supported by the program
ENV_QEMU = "QEMU"
ENV_IMAGE = "IMAGE"
ENV_AUTOCONFIG_NET = "AUTOCONFIG_NET"
LOGGER = logging.getLogger(__name__)
def from_env(key: str, default: Optional[Any] = None) -> str:
value = os.environ.get(key, default)
if value is None:
raise ValueError(f"Missing {key} in the environment")
return value
class TansivVM(object):
"""
TODO autogenerate this based on the cli
"""
def __init__(
self,
ip_tantap: IPv4Interface,
ip_management: IPv4Interface,
qemu_cmd: str,
qemu_image: Path(),
qemu_args: Optional[str] = None,
hostname: Optional[str] = None,
public_key: Optional[str] = None,
autoconfig_net: bool = False,
):
self.tantap = ip_tantap
self.management = ip_management
self.qemu_cmd = qemu_cmd
self.qemu_image = qemu_image.resolve()
self._hostname = hostname
self.public_key = public_key
self.autoconfig_net = autoconfig_net
if qemu_args is None:
# Tansiv profile ?
self.qemu_args = (
" --icount shift=1,sleep=on"
" -rtc clock=vm"
f" -m 1g --vsg mynet0,src={ip_tantap.ip}"
)
else:
self.qemu_args = qemu_args
@property
def tantap_id(self):
_, _, _, t = self.tantap.ip.packed
return t
@property
def management_id(self):
_, _, _, m = self.management.ip.packed
return m
@property
def hostname(self) -> str:
if self._hostname is None:
return f"tansiv-{str(self.tantap.ip).replace('.', '-')}"
return self._hostname
@property
def tapname(self) -> List[str]:
t = self.tantap_id
m = self.management_id
return [f"tantap{t}", f"mantap{m}"]
@property
def bridgename(self) -> List[str]:
return ["tantap-br", "mantap-br"]
@property
def gateway(self) -> List[IPv4Interface]:
t_cidr = str(self.tantap).split("/")[1]
m_cidr = str(self.management).split("/")[1]
return [
IPv4Interface(f"{str(next(self.tantap.network.hosts()))}/{t_cidr}"),
IPv4Interface(f"{str(next(self.management.network.hosts()))}/{m_cidr}"),
]
@property
def mac(self) -> List[str]:
t = self.tantap_id
m = self.management_id
return [
f"02:ca:fe:f0:0d:{hex(t).lstrip('0x').rjust(2, '0')}",
f"54:52:fe:f0:0d:{hex(m).lstrip('0x').rjust(2, '0')}",
]
def ci_meta_data(self) -> Dict:
meta_data = dict()
meta_data.update({"instance-id": self.hostname})
if self.public_key is not None:
meta_data.update({"public-keys": self.public_key})
return meta_data
def ci_network_config(self) -> Dict:
# yes the nic names are hardcoded...
def ethernet_config(interface: IPv4Interface):
return dict(
addresses=[f"{interface.ip}"],
gateway4=str(next(interface.network.hosts())),
dhcp4=False,
dhcp6=False,
)
ens3 = ethernet_config(self.tantap)
ens4 = ethernet_config(self.management)
network_config = dict(version=2, ethernets=dict(ens3=ens3, ens4=ens4))
return network_config
def ci_user_data(self) -> Dict:
bootcmd = ["----> START OF TANTAP CLOUD INIT <----------------"]
# we generate all the possible mapping for the different network
def _mapping(interface: IPv4Interface, prefix: str):
host_entries = []
for _ip in interface.network.hosts():
_, _, _, d = _ip.exploded.split(".")
# 192.168.0.123 m123
host_entries.append((_ip, f"{prefix}{d}"))
return host_entries
t_entries = [
f'echo "{ip} {alias}" >> /etc/hosts'
for ip, alias in _mapping(self.tantap, "t")
]
m_entries = [
f'echo "{ip} {alias}" >> /etc/hosts'
for ip, alias in _mapping(self.management, "m")
]
bootcmd.extend(t_entries)
bootcmd.extend(m_entries)
bootcmd.append(f'echo "127.0.0.1 {self.hostname}" >> /etc/hosts')
# - echo 127.0.0.1 $VM_NAME >> /etc/hosts
user_data = dict(hostname=self.hostname, disable_root=False, bootcmd=bootcmd)
# non python compatible key
user_data.update({"local-hostname": self.hostname})
return user_data
def prepare_cloud_init(self, working_dir: Path) -> Path:
# generate cloud init files
with open(working_dir / "user-data", "w") as f:
# yes, this is mandatory to prepend this at the beginning of this
# file
f.write("#cloud-config\n")
with open(working_dir / "user-data", "a") as f:
yaml.dump(self.ci_user_data(), f)
with open(working_dir / "meta-data", "w") as f:
yaml.dump(self.ci_meta_data(), f)
with open(working_dir / "network-config", "w") as f:
yaml.dump(self.ci_network_config(), f)
# generate the iso
iso = (working_dir / "cloud-init.iso").resolve()
check_call(
f"genisoimage -output {iso} -volid cidata -joliet -rock user-data meta-data network-config",
shell=True,
cwd=working_dir,
)
return iso
def prepare_image(self, working_dir: Path) -> Path:
qemu_image = (working_dir / "image.qcow2").resolve()
check_call(
f"qemu-img create -f qcow2 -o backing_file={self.qemu_image} {qemu_image}",
shell=True,
)
return qemu_image
def prepare_net(self):
"""Create the bridges, the tap if needed."""
def br_tap(br: str, ip: IPv4Interface, tap: str):
"""Create a bridge and a tap attached."""
check_call(
f"""
ip link show dev {br} || ip link add name {br} type bridge
ip link set {br} up
(ip addr show dev {br} | grep {ip}) || ip addr add {ip} dev {br}
ip link show dev {tap} || ip tuntap add {tap} mode tap
ip link set {tap} master {br}
ip link set {tap} up
""",
shell=True,
)
if not self.autoconfig_net:
return
br_tap(self.bridgename[0], self.gateway[0], self.tapname[0])
br_tap(
self.bridgename[1],
self.gateway[1],
self.tapname[1],
)
def start(self, working_dir: Path) -> Path:
working_dir.mkdir(parents=True, exist_ok=True)
# generate cloud_init
iso = self.prepare_cloud_init(working_dir)
# generate the base image
image = self.prepare_image(working_dir)
self.prepare_net()
# boot
cmd = (
f"{self.qemu_cmd}"
f" {self.qemu_args}"
f" -drive file={image} "
f" -cdrom {iso}"
f" -netdev tantap,id=mynet0,ifname={self.tapname[0]},script=no,downscript=no"
f" -device e1000,netdev=mynet0,mac={self.mac[0]}"
f" -netdev tap,id=mynet1,ifname={self.tapname[1]},script=no,downscript=no"
f" -device e1000,netdev=mynet1,mac={self.mac[1]}"
)
LOGGER.info(cmd)
check_call(
cmd,
shell=True,
)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="""
Boots a VM using qemu, icount mode ...
Use two nics: one for tantap the other for a regular tap (management interface).
Assumptions:
IP addresses are part of a /24 network whose gateway is set to the first possible IP
ip: 192.168.0.42/24 -> gateway: 192.168.0.1
So the bridge IP must/will be set accordingly
Some environment Variables (MANDATORY):
QEMU: path to the qemu binary (useful to test a modified version)
IMAGE: path to a qcow2 or raw image disk (to serve as backing file for the disk images)