Commit bb71d99f authored by Pierre Neyron's avatar Pierre Neyron 🚴
Browse files

Initial commit

parents
*~
.*.swp
kameleon/build
About
=====
This directory contains the recipe to built the operating system to be used in this tutorial (IA stack included).
The recipe is to be used with the kameleon tool (http://kameleon.imag.fr). It builds a new Grid'5000 kadeploy environment, customized for the need.
Recipe details
==============
This recipe extends a kameleon recipe which allows creating a new kadeploy environment based on an existing other kadeploy environment (see in the kameleon-recipes/from_kadeploy_environment/ directory). Here we customize the `debian-x64-big` Grid'5000 environment to add the IA stack.
The recipe adds some steps (defined in the steps directory) in its setup section, in order to install miniconda system-wide, and then the following conda packages:
- ipython
- pytorch
How to build
============
First reserve a node where to run the build:
```
frontend$ oarsub "sleep 2h" -l nodes=1,walltime=2
```
Once the job is running, connect to your node and install the required components:
```
node$ gem install --user-install kameleon-builder
```
Make the `kameleon` command reachable with the PATH environment variable:
```
node$ export PATH=$HOME/.gem/ruby/2.3.0/bin:$PATH
```
Then go to the kameleon directory (the directory which contains this file) and start the **build**:
```
node$ kameleon build debian9-x64-ia-tutorial
```
#==============================================================================
# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
#==============================================================================
#
# DESCRIPTION: Debian9 system built for the BigData-tutorial
#
#==============================================================================
---
extend: kameleon-recipes/from_kadeploy_environment/debian9-x64-custom.yaml
global:
# Base environment on Grid'5000 debian9-x64-big
kadeploy_environment_import_name: "debian9-x64-big"
# URL of miniconda installer
miniconda_url: https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
# We do a system-wide installation of conda in /opt
miniconda_dir: /opt/miniconda
# conda install command adapted to the installed python and cuda toolkit
# versions
pytorch_conda_install_command: conda install pytorch torchvision cudatoolkit=9.0 -c pytorch
# kameleon VM disk size
image_size: 20G
bootstrap:
- "@base"
setup:
- install_miniconda:
- install:
- download_file_in:
- $${miniconda_url}
- miniconda.sh
- exec_in: bash miniconda.sh -b -p $${miniconda_dir}
- exec_in: rm -v miniconda.sh
- setup:
- exec_in: echo 'export PATH=$${miniconda_dir}/bin:$PATH' >> /etc/profile
- install_ipython_in_conda:
- install:
- exec_in: |
export PATH=/opt/miniconda/bin:$PATH
conda install -y ipython
- install_pytorch_in_conda:
- install:
- exec_in: |
export PATH=/opt/miniconda/bin:$PATH
$${pytorch_conda_install_command}
export:
- "@base"
#==============================================================================
# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
#==============================================================================
#
# DESCRIPTION: base recipe for a customization of a Grid'5000 environment
#
#==============================================================================
---
extend: ../steps/backend/$${backend}.yaml
# Loads some helpful aliases (this files are located in steps/aliases/ directory)
aliases: defaults.yaml
# Custom shell environement (this files are located in steps/env/ directory)
env:
- bashrc
- functions.sh
# Global variables use by Kameleon engine and the steps
global:
# Architecture for the target system
arch: x86_64
# Distribution
distrib: debian
release: 9
# Kadeploy environment
kadeploy_environment_import_name: "debian9-x64-min"
kadeploy_environment_import_user: "deploy"
kadeploy_environment_import_version: ""
kadeploy_frontend: "frontend"
kadeploy_environment_export_name: "$${kameleon_recipe_name}"
kadeploy_environment_export_format: "tar.gz"
kadeploy_environment_export_description: "Customized $${kadeploy_environment_import_name}"
kadeploy_environment_export_dir: "$HOME/public/"
kadeploy_environment_export_baseurl: "local://$HOME/public/"
# output appliance options
filesystem_type: ext4
image_size: 10G
image_disk: $${kameleon_cwd}/base_$${kameleon_recipe_name}
# Allowed formats are: tar.gz, tar.bz2, tar.xz, tar.lzo, qcow, qcow2, qed, vdi, raw, vmdk
appliance_formats: "$${kadeploy_environment_export_format}"
appliance_tar_compression_level: "9"
appliance_filename: "$${kameleon_cwd}/$${kameleon_recipe_name}"
appliance_tar_excludes: >-
./etc/fstab ./root/.bash_history ./root/kameleon_workdir ./root/.ssh
./var/tmp/* ./tmp/* ./var/log/* ./dev/* ./proc/* ./run/*
./sys/*
# rootfs options
rootfs: $${kameleon_cwd}/rootfs
# enable workaround for the login delay due to ldap and dns
enable_workaround_login_delay: true
# kernel boot parameters: use legacy network interfaces
kernel_args: "quiet net.ifnames=0 biosdevname=0"
ssh_config_file: $${kameleon_cwd}/ssh_config
local_ip: 10.0.2.2
## Select backend for in context isolation
backend: qemu
## System variables. Required by kameleon engine
# Include specific steps
include_steps:
- $${distrib}/$${release}
- $${distrib}
bootstrap:
- "@base"
setup:
# Add steps here
export:
- "@base"
- create_kadeploy_environment
#==============================================================================
# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
#==============================================================================
#
# DESCRIPTION: base recipe for a customization of a Grid'5000 environment
#
#==============================================================================
---
extend: ../steps/backend/$${backend}.yaml
# Loads some helpful aliases (this files are located in steps/aliases/ directory)
aliases: defaults.yaml
# Custom shell environement (this files are located in steps/env/ directory)
env:
- bashrc
- functions.sh
# Global variables use by Kameleon engine and the steps
global:
# Architecture for the target system
arch: x86_64
# Distribution
distrib: debian
release: 9
# Kadeploy environment
kadeploy_environment_import_name: "debian9-x64-min"
kadeploy_environment_import_user: "deploy"
kadeploy_environment_import_version: ""
kadeploy_frontend: "frontend"
kadeploy_environment_export_name: "$${kameleon_recipe_name}"
kadeploy_environment_export_format: "tar.gz"
kadeploy_environment_export_description: "Customized $${kadeploy_environment_import_name}"
kadeploy_environment_export_dir: "$HOME/public/"
kadeploy_environment_export_baseurl: "local://$HOME/public/"
# output appliance options
filesystem_type: ext4
image_size: 10G
image_disk: $${kameleon_cwd}/base_$${kameleon_recipe_name}
# Allowed formats are: tar.gz, tar.bz2, tar.xz, tar.lzo, qcow, qcow2, qed, vdi, raw, vmdk
appliance_formats: "$${kadeploy_environment_export_format}"
appliance_tar_compression_level: "9"
appliance_filename: "$${kameleon_cwd}/$${kameleon_recipe_name}"
appliance_tar_excludes: >-
./etc/fstab ./root/.bash_history ./root/kameleon_workdir ./root/.ssh
./var/tmp/* ./tmp/* ./var/log/* ./dev/* ./proc/* ./run/*
./sys/*
# rootfs options
rootfs: $${kameleon_cwd}/rootfs
# kernel boot parameters: use legacy network interfaces
kernel_args: "quiet net.ifnames=0 biosdevname=0"
ssh_config_file: $${kameleon_cwd}/ssh_config
local_ip: 10.0.2.2
## Select backend for in context isolation
backend: qemu
## System variables. Required by kameleon engine
# Include specific steps
include_steps:
- $${distrib}/$${release}
- $${distrib}
bootstrap:
- "@base"
setup:
# Add steps here
export:
- "@base"
- create_kadeploy_environment
write_local:
- exec_local: |
mkdir -p $(dirname @1);
cat >@1 <<EOF_KAMELEON_INTERNAL
@2
EOF_KAMELEON_INTERNAL
write_in:
- exec_in: |
mkdir -p $(dirname @1);
cat >@1 <<EOF_KAMELEON_INTERNAL
@2
EOF_KAMELEON_INTERNAL
write_out:
- exec_out: |
mkdir -p $(dirname @1);
cat >@1 <<EOF_KAMELEON_INTERNAL
@2
EOF_KAMELEON_INTERNAL
append_local:
- exec_local: |
mkdir -p $(dirname @1);
cat >>@1 <<EOF_KAMELEON_INTERNAL
@2
EOF_KAMELEON_INTERNAL
append_in:
- exec_in: |
mkdir -p $(dirname @1);
cat >>@1 <<EOF_KAMELEON_INTERNAL
@2
EOF_KAMELEON_INTERNAL
append_out:
- exec_out: |
mkdir -p $(dirname @1);
cat >>@1 <<EOF_KAMELEON_INTERNAL
@2
EOF_KAMELEON_INTERNAL
local2out:
- exec_out: |
mkdir -p $(dirname @2)
- pipe:
- exec_local: cat @1
- exec_out: cat > @2
local2in:
- exec_in: mkdir -p $(dirname @2)
- pipe:
- exec_local: cat @1
- exec_in: cat > @2
out2local:
- exec_local: mkdir -p $(dirname @2)
- pipe:
- exec_out: cat @1
- exec_local: cat > @2
out2in:
- exec_in: mkdir -p $(dirname @2)
- pipe:
- exec_out: cat @1
- exec_in: cat > @2
in2local:
- exec_local: mkdir -p $(dirname @2)
- pipe:
- exec_in: cat @1
- exec_local: cat > @2
in2out:
- exec_out: mkdir -p $(dirname @2)
- pipe:
- exec_in: cat @1
- exec_out: cat > @2
check_cmd_out:
- rescue:
- exec_out: command -V @1 2> /dev/null
- breakpoint: "@1 is missing from out_context"
check_cmd_local:
- on_bootstrap_init:
- rescue:
- exec_local: command -V @1 2> /dev/null
- breakpoint: "@1 is missing from local_context"
check_cmd_in:
- rescue:
- exec_in: command -V @1 2> /dev/null
- breakpoint: "@1 is missing from in_context"
umount_out:
- exec_out: |
echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true
umount_local:
- exec_local: |
echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true
umount_in:
- exec_in: |
echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true
download_file_in:
- exec_in: __download "@1" "@2"
download_file_out:
- exec_out: __download "@1" "@2"
download_file_local:
- exec_local: __download "@1" "@2"
download_recipe_build_local:
- exec_local: __download_recipe_build "@1" "@2" "@3" "@4" "@5" "@6" "@7"
download_kadeploy_environment_image_local:
- exec_local: __download_kadeploy_environment_image "@1" "@2" "@3" "@4" "@5"
# Generic Virtual Machine backend recipe
---
global:
ssh_config_file: $${kameleon_cwd}/ssh_config
out_context:
cmd: ssh -F $${ssh_config_file} $${kameleon_recipe_name} -t /bin/bash
workdir: /root/kameleon_workdir
proxy_cache: $${local_ip}
in_context:
cmd: ssh -F $${ssh_config_file} $${kameleon_recipe_name} -t /bin/bash
workdir: /root/kameleon_workdir
proxy_cache: $${local_ip}
bootstrap:
- download_upstream_tarball
- create_appliance
- prepare_appliance
setup:
- "@base"
export:
- disable_checkpoint
- save_appliance_VM
# Qemu backend recipe
---
extend: VM.yaml
checkpoint: qemu.yaml
global:
image_format: qcow2
qemu_enable_kvm: "$(egrep '(vmx|svm)' /proc/cpuinfo > /dev/null && echo true)"
qemu_cpu: "$(($(nproc)-1))"
qemu_memory_size: 768
qemu_monitor_socket: $${kameleon_cwd}/qemu_monitor.socket
qemu_arch: $${arch}
qemu_pidfile: $${kameleon_cwd}/qemu.pid
bootstrap:
- prepare_ssh_to_out_context
- "@base"
- start_qemu
setup:
- "@base"
export:
- "@base"
- create_appliance_script: $${kameleon_data_dir}/helpers/create_appliance.py
- create_appliance:
- check_cmd_local: virt-make-fs
- exec_local: |
rm -f $${image_disk}.$${image_format}
[ -n "$UPSTREAM_TARBALL" ] || fail "UPSTREAM_TARBALL is not defined"
[ -r "$UPSTREAM_TARBALL" ] || fail "UPSTREAM_TARBALL ($UPSTREAM_TARBALL) is not readable"
# Create an appliance from rootfs directory
python $${create_appliance_script} $UPSTREAM_TARBALL \
--size $${image_size} \
-o $${image_disk} \
--filesystem $${filesystem_type} \
--format $${image_format} \
--append "$${kernel_args}"
- delete_initial_image_at_the_end:
- on_checkpoint: skip
- on_export_clean:
- exec_local: rm -f $${image_disk}.$${image_format}
- upstream_store_dir: "$${kameleon_cwd}/upstream_build/"
- upstream_tarball: ""
- upstream_recipe: ""
- upstream_version: ""
- upstream_checksum: ""
- upstream_checksign: ""
- upstream_cache: ""
- upstream_url: ""
- kadeploy_environment_import_name: ""
- kadeploy_environment_import_user: ""
- kadeploy_environment_import_version: ""
- kadeploy_frontend: ""
- download:
- on_bootstrap_clean:
- exec_local: rm -rf $${upstream_store_dir}
- test:
- exec_local: test -n "$${upstream_tarball}"
- group:
- exec_local: echo "Download tarball..."
- download_file_local:
- $${upstream_url}/$${upstream_tarball}
- $${upstream_store_dir}/$${upstream_tarball}
- exec_local: export UPSTREAM_TARBALL="$${upstream_store_dir}/$${upstream_tarball}"
- test:
- exec_local: test -n "$${upstream_recipe}"
- download_recipe_build_local:
- $${upstream_recipe}
- $${upstream_version}
- $${upstream_checksum}
- $${upstream_checksign}
- $${upstream_cache}
- $${upstream_url}
- $${upstream_store_dir}
- test:
- exec_local: test -n "$${kadeploy_environment_import_name}"
- download_kadeploy_environment_image_local:
- $${kadeploy_environment_import_name}
- $${kadeploy_environment_import_user}
- $${kadeploy_environment_import_version}
- $${kadeploy_frontend}
- $${upstream_store_dir}
- exec_local: fail "Nothing to download ???"
- insecure_ssh_key: $${kameleon_cwd}/insecure_ssh_key
- enable_workaround_login_delay: false
- generate_ssh_keys:
- check_cmd_local: ssh-keygen
- exec_local: echo -e 'y\n' | ssh-keygen -q -t rsa -b 4096 -f $${insecure_ssh_key} -N ''
- exec_local: cat $${insecure_ssh_key}
- inject_ssh_private_key:
- check_cmd_local: virt-customize
- exec_local: |
virt-customize \
-a $${image_disk}.$${image_format} \
--run-command 'mkdir -p /root/.ssh' \
--upload $${insecure_ssh_key}.pub:/root/.ssh/.kameleon_authorized_keys \
--run-command 'touch /root/.ssh/authorized_keys' \
--run-command 'cp /root/.ssh/authorized_keys /root/.ssh/authorized_keys.bak' \
--run-command 'cat /root/.ssh/.kameleon_authorized_keys >> /root/.ssh/authorized_keys' \
--run-command 'chmod 700 /root/.ssh' \
--run-command 'chmod -R go-rw /root/.ssh' \
--run-command 'chown -R root:root /root/.ssh'
- on_export_init:
- exec_local: |
virt-customize \
-a $${image_disk}.$${image_format} \
--run-command 'mv /root/.ssh/authorized_keys.bak /root/.ssh/authorized_keys' \
--delete /root/.ssh/.kameleon_authorized_keys
- add_insecure_key_to_ssh_config:
- on_checkpoint: redo
- exec_local: |
cat <<EOF >> $${ssh_config_file}
IdentityFile $${insecure_ssh_key}
EOF
- workaround_login_delay_due_to_ldap_and_dns:
- test:
- exec_local: test "$${enable_workaround_login_delay}" = "true"
- exec_local: |
virt-customize \
-a $${image_disk}.$${image_format} \
--run-command 'mv /etc/nslcd.conf /etc/nslcd.conf.orig-g5k' \
--run-command 'sed -i.orig-g5k "s/ ldap//" /etc/nsswitch.conf' \
--run-command 'echo > /etc/resolv.conf'
- exec_local: echo "Workaround is not enabled"
- on_setup_clean:
- test:
- exec_local: test "$${enable_workaround_login_delay}" = "true"
- exec_local: |
virt-customize \
-a $${image_disk}.$${image_format} \
--run-command 'mv /etc/nslcd.conf.orig-g5k /etc/nslcd.conf' \
--run-command 'mv /etc/nsswitch.conf.orig-g5k /etc/nsswitch.conf'
- insecure_ssh_key: $${kameleon_cwd}/insecure_ssh_key
- enable_workaround_login_delay: false
- generate_ssh_keys:
- check_cmd_local: ssh-keygen
- exec_local: echo -e 'y\n' | ssh-keygen -q -t rsa -b 4096 -f $${insecure_ssh_key} -N ''
- exec_local: cat $${insecure_ssh_key}
- inject_ssh_private_key:
- check_cmd_local: virt-customize
- exec_local: |
virt-customize \
-a $${image_disk}.$${image_format} \
--run-command 'mkdir -p /root/.ssh' \
--upload $${insecure_ssh_key}.pub:/root/.ssh/.kameleon_authorized_keys \
--run-command 'touch /root/.ssh/authorized_keys' \
--run-command 'cp /root/.ssh/authorized_keys /root/.ssh/authorized_keys.bak' \
--run-command 'cat /root/.ssh/.kameleon_authorized_keys >> /root/.ssh/authorized_keys' \
--run-command 'chmod 700 /root/.ssh' \
--run-command 'chmod -R go-rw /root/.ssh' \
--run-command 'chown -R root:root /root/.ssh'
- on_export_init:
- exec_local: |
virt-customize \
-a $${image_disk}.$${image_format} \
--run-command 'mv /root/.ssh/authorized_keys.bak /root/.ssh/authorized_keys' \
--delete /root/.ssh/.kameleon_authorized_keys
- add_insecure_key_to_ssh_config:
- on_checkpoint: redo
- exec_local: |
cat <<EOF >> $${ssh_config_file}
IdentityFile $${insecure_ssh_key}
EOF
- workaround_login_delay_due_to_ldap_and_dns:
- test:
- exec_local: test "$${enable_workaround_login_delay}" = "true"
- exec_local: |
virt-customize \
-a $${image_disk}.$${image_format} \
--run-command 'mv /etc/nslcd.conf /etc/nslcd.conf.orig-g5k' \
--run-command 'sed -i.orig-g5k "s/ ldap//" /etc/nsswitch.conf' \
--run-command 'echo > /etc/resolv.conf'
- exec_local: echo "Workaround is not enabled"
- on_setup_clean:
- test:
- exec_local: test "$${enable_workaround_login_delay}" = "true"
- exec_local: |
virt-customize \
-a $${image_disk}.$${image_format} \
--run-command 'mv /etc/nslcd.conf.orig-g5k /etc/nslcd.conf' \
--run-command 'mv /etc/nsswitch.conf.orig-g5k /etc/nsswitch.conf'
- select_empty_port:
- on_checkpoint: redo
- exec_local: |