Commit 227d9381 authored by SIMONIN Matthieu's avatar SIMONIN Matthieu
Browse files

ci: some fixes

- recover LD_PRELOAD trick
- sync qemu
parent 5823c4c2
......@@ -3,6 +3,7 @@ variables:
# https://docs.gitlab.com/ee/ci/runners/README.html#git-submodule-strategy
# normal -> only top level are fetched before the job starts
GIT_SUBMODULE_STRATEGY: normal
# Was here to fix a TLS proxy issue we had (this disable the custom hook implemented in the CI infrastructure)
LD_PRELOAD: ""
RUST_VERSION: 1.46.0
......@@ -15,8 +16,8 @@ workflow:
- when: always
stages:
- build
- publish
- build
- publish
# unstable, yeah !
image: simgrid/unstable:latest
......@@ -24,57 +25,59 @@ image: simgrid/unstable:latest
build:
stage: build
tags:
- large
- large
script:
#
# Some missing build tools
#
- export PROJECT_DIR=$(pwd)
- apt-get update
- apt-get install -y curl git build-essential pkg-config libboost-dev cmake libcppunit-dev libglib2.0-dev clang libclang-dev libpixman-1-dev flex bison
- curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup-init
- sh rustup-init -y --profile minimal --default-toolchain $RUST_VERSION
- export PATH=$HOME/.cargo/bin:$PATH
- mkdir -p build
# build test
- cd build
- cmake .. && make config-host.h
# fake-vm tests
- cd ${PROJECT_DIR}/src/fake-vm
- make build
- make test
- cd ${PROJECT_DIR}/build
- make
# NOTE(msimonin): as long as https://gitlab.inria.fr/quinson/2018-vsg/-/issues/5 is around
# we run our tests isolated in different process.
# This is far from ideal since reports will be spread in the output of each process
# But at least catch2 lets use specify this easily...
# But ...
# for some reason this fails: ./tests --list-test-names-only | xargs -d "\n" -n 1 ./tests
# grrr!
- ./tests "VSG receive one message"
- ./tests "VSG deliver one message"
- ./tests "VSG send piggyback port"
- ./tests "piggyback port"
#
# Some missing build tools
#
- export PROJECT_DIR=$(pwd)
- apt-get update
- apt-get install -y curl git build-essential pkg-config libboost-dev cmake libcppunit-dev libglib2.0-dev clang libclang-dev libpixman-1-dev flex bison
- curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup-init
- sh rustup-init -y --profile minimal --default-toolchain $RUST_VERSION
- export PATH=$HOME/.cargo/bin:$PATH
- mkdir -p build
# build test
- cd build
- cmake .. && make config-host.h
# fake-vm tests
- cd ${PROJECT_DIR}/src/fake-vm
- make build
- make test
- cd ${PROJECT_DIR}/build
- make
# NOTE(msimonin): as long as https://gitlab.inria.fr/quinson/2018-vsg/-/issues/5 is around
# we run our tests isolated in different process.
# This is far from ideal since reports will be spread in the output of each process
# But at least catch2 lets use specify this easily...
# But ...
# for some reason this fails: ./tests --list-test-names-only | xargs -d "\n" -n 1 ./tests
# grrr!
- ./tests "VSG receive one message"
- ./tests "VSG deliver one message"
- ./tests "VSG send piggyback port"
- ./tests "piggyback port"
# for now
allow_failure: true
docker:
image: docker:latest
stage: publish
tags:
- large
- large
script:
# login iff we're on gitlab ci
- |
if [[ -n "${CI_JOB_TOKEN}" ]]
then
docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
export DOCKER_IMAGE=${CI_REGISTRY_IMAGE}/tansiv:${CI_COMMIT_SHORT_SHA}
export DOCKER_IMAGE_LATEST=${CI_REGISTRY_IMAGE}/tansiv:latest
docker build -t ${DOCKER_IMAGE} .
docker push ${DOCKER_IMAGE}
docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_LATEST}
docker push ${DOCKER_IMAGE_LATEST}
else
export DOCKER_IMAGE=tansiv:latest
docker build -t ${DOCKER_IMAGE} .
fi
\ No newline at end of file
- |
if [[ -n "${CI_JOB_TOKEN}" ]]
then
docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
export DOCKER_IMAGE=${CI_REGISTRY_IMAGE}/tansiv:${CI_COMMIT_SHORT_SHA}
export DOCKER_IMAGE_LATEST=${CI_REGISTRY_IMAGE}/tansiv:latest
docker build -t ${DOCKER_IMAGE} .
docker push ${DOCKER_IMAGE}
docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_LATEST}
docker push ${DOCKER_IMAGE_LATEST}
else
export DOCKER_IMAGE=tansiv:latest
docker build -t ${DOCKER_IMAGE} .
fi
......@@ -4,4 +4,4 @@
branch = dev/waitfree
[submodule "src/qemu"]
path = src/qemu
url = https://gitlab.inria.fr/lrilling/qemu
url = https://gitlab.inria.fr/msimonin/qemu
# tansiv in docker :)
FROM simgrid/unstable:latest
ENV LD_PRELOAD ""
WORKDIR /app
COPY . /app
RUN apt-get update
RUN apt-get install -y build-essential \
libboost-dev \
cmake \
libcppunit-dev \
libglib2.0-dev \
clang \
libclang-dev \
curl \
git \
pkg-config \
libglib2.0-dev \
libpixman-1-dev \
flex \
bison \
genisoimage
libboost-dev \
cmake \
libcppunit-dev \
libglib2.0-dev \
clang \
libclang-dev \
curl \
git \
pkg-config \
libglib2.0-dev \
libpixman-1-dev \
flex \
bison \
genisoimage
# a cup of rust
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup-init
......
......@@ -82,32 +82,17 @@ def generate_deployment(args) -> str:
)
ET.SubElement(vm, "argument", dict(value=f"192.168.120.{descriptor}"))
ET.SubElement(vm, "argument", dict(value="./boot.py"))
ET.SubElement(vm, "argument", dict(value=f"--mode"))
ET.SubElement(vm, "argument", dict(value=f"tantap"))
ET.SubElement(vm, "argument", dict(value=f"--autoconfig_net"))
ET.SubElement(vm, "argument", dict(value=f"192.168.120.{descriptor}/24"))
ET.SubElement(vm, "argument", dict(value=f"10.0.0.{descriptor}/24"))
element_tree = ElementTree(platform)
with open(out, "w") as f:
f.write('<!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">')
f.write(ET.tostring(platform, encoding="unicode"))
def start_tansiv(docker_image: str, p: en.play_on):
p.docker_container(
state="started",
network_mode="host",
name="tansiv",
image=docker_image,
command="platform.xml deployment.xml",
volumes=["/tmp/id_rsa.pub:/root/.ssh/id_rsa.pub", "/tmp/tansiv:/srv"],
env={
"AUTOCONFIG_NET": "true",
"IMAGE": "image.qcow2",
},
capabilities=["NET_ADMIN"],
devices=["/dev/net/tun"],
)
@en.enostask(new=True)
def deploy(args, env=None):
"""Deploy tansiv and the associated VMs.
......@@ -121,6 +106,7 @@ def deploy(args, env=None):
walltime = args.walltime
queue = args.queue
docker_image = args.docker_image
qemu_args = args.qemu_args
prod = en.G5kNetworkConf(
id="id",
roles=["prod"],
......@@ -176,7 +162,25 @@ def deploy(args, env=None):
display_name="copying deployment file",
)
# finally start the container
start_tansiv(docker_image, p)
environment = {
"AUTOCONFIG_NET": "true",
"IMAGE": "image.qcow2",
}
if qemu_args is not None:
environment.update(QEMU_ARGS=qemu_args)
p.docker_container(
state="started",
network_mode="host",
name="tansiv",
image=docker_image,
command="platform.xml deployment.xml --log=vm_interface.threshold:debug --log=vm_coordinator.threshold:debug",
volumes=["/tmp/id_rsa.pub:/root/.ssh/id_rsa.pub", "/tmp/tansiv:/srv"],
env=environment,
capabilities=["NET_ADMIN"],
devices=["/dev/net/tun"],
)
# by default packets that needs to be forwarded by the bridge are sent to iptables
# iptables will most likely drop them.
# we can disabled this behaviour by bypassing iptables
......@@ -224,9 +228,8 @@ def fping(args, env=None):
]
print(hostnames)
result = en.run_command(
f'fping -q -C 10 -s -e {" ".join(hostnames)}',
f'fping -q -C 30 -s -e {" ".join(hostnames)}',
roles=tansiv_roles,
on_error_continue=True,
)
# displayng the output (the result datastructure is a bit painful to parse...ask enoslib maintainer)
......@@ -257,7 +260,9 @@ def flent(args, env=None):
)
with en.play_on(roles=dict(all=workers)) as p:
p.shell("flent udp_flood -p totals -l 10 -H {{ flent_server }} -o filename.png")
p.shell(
"flent tcp_download -p totals -l 30 -H {{ flent_server }} -o filename.png"
)
p.fetch(src="filename.png", dest="result")
......@@ -311,6 +316,7 @@ if __name__ == "__main__":
help="Tansiv docker image to use",
default=DEFAULT_DOCKER_IMAGE,
)
parser_deploy.add_argument("--qemu_args", help="Some qemu_args")
parser_deploy.set_defaults(func=deploy)
# --------------------------------------------------------------------------
......
Subproject commit 64d6ac77697099481e60d4d866a8362caf0c496d
Subproject commit 1e8d4f36181b5adf21fd75cd470356b2d8a59deb
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment