Initial commit

.gitignore

+*~
+.*.swp
+kameleon/build

kameleon/README.md

+About
+=====
+
+This directory contains the recipe to built the operating system to be used in this tutorial (IA stack included).
+The recipe is to be used with the kameleon tool (http://kameleon.imag.fr). It builds a new Grid'5000 kadeploy environment, customized for the need.
+
+Recipe details
+==============
+
+This recipe extends a kameleon recipe which allows creating a new kadeploy environment based on an existing other kadeploy environment (see in the kameleon-recipes/from_kadeploy_environment/ directory). Here we customize the `debian-x64-big` Grid'5000 environment to add the IA stack.
+
+The recipe adds some steps (defined in the steps directory) in its setup section, in order to install miniconda system-wide, and then the following conda packages:
+- ipython
+- pytorch
+
+How to build
+============
+
+First reserve a node where to run the build:
+
+```
+frontend$ oarsub "sleep 2h" -l nodes=1,walltime=2
+```
+
+Once the job is running, connect to your node and install the required components:
+
+```
+node$ gem install --user-install kameleon-builder
+```
+
+Make the `kameleon` command reachable with the PATH environment variable:
+
+```
+node$ export PATH=$HOME/.gem/ruby/2.3.0/bin:$PATH
+```
+
+Then go to the kameleon directory (the directory which contains this file) and start the **build**:
+
+```
+node$ kameleon build debian9-x64-ia-tutorial
+```

kameleon/debian9-x64-ia-tutorial.yaml

+#==============================================================================
+# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
+#==============================================================================
+#
+# DESCRIPTION: Debian9 system built for the BigData-tutorial
+#
+#==============================================================================
+---
+extend: kameleon-recipes/from_kadeploy_environment/debian9-x64-custom.yaml
+
+global:
+  # Base environment on Grid'5000 debian9-x64-big
+  kadeploy_environment_import_name: "debian9-x64-big"
+  # URL of miniconda installer
+  miniconda_url: https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+  # We do a system-wide installation of conda in /opt
+  miniconda_dir: /opt/miniconda 
+  # conda install command adapted to the installed python and cuda toolkit
+  # versions
+  pytorch_conda_install_command: conda install pytorch torchvision cudatoolkit=9.0 -c pytorch
+
+  # kameleon VM disk size
+  image_size: 20G
+
+bootstrap:
+  - "@base"
+
+setup:
+  - install_miniconda:
+    - install:
+      - download_file_in:
+        - $${miniconda_url}
+        - miniconda.sh
+      - exec_in: bash miniconda.sh -b -p $${miniconda_dir}
+      - exec_in: rm -v miniconda.sh
+
+    - setup:
+      - exec_in: echo 'export PATH=$${miniconda_dir}/bin:$PATH' >> /etc/profile
+  - install_ipython_in_conda:
+    - install:
+      - exec_in: |
+          export PATH=/opt/miniconda/bin:$PATH
+          conda install -y ipython
+  - install_pytorch_in_conda:
+    - install:
+      - exec_in: |
+          export PATH=/opt/miniconda/bin:$PATH
+          $${pytorch_conda_install_command}
+export:
+  - "@base"

kameleon/kameleon-recipes/from_kadeploy_environment/debian9-x64-custom.yaml

+#==============================================================================
+# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
+#==============================================================================
+#
+# DESCRIPTION: base recipe for a customization of a Grid'5000 environment
+#
+#==============================================================================
+---
+extend: ../steps/backend/$${backend}.yaml
+
+# Loads some helpful aliases (this files are located in steps/aliases/ directory)
+aliases: defaults.yaml
+
+# Custom shell environement (this files are located in steps/env/ directory)
+env:
+  - bashrc
+  - functions.sh
+
+# Global variables use by Kameleon engine and the steps
+global:
+  # Architecture for the target system
+  arch: x86_64
+  # Distribution
+  distrib: debian
+  release: 9
+  # Kadeploy environment
+  kadeploy_environment_import_name: "debian9-x64-min"
+  kadeploy_environment_import_user: "deploy"
+  kadeploy_environment_import_version: ""
+  kadeploy_frontend: "frontend"
+  kadeploy_environment_export_name: "$${kameleon_recipe_name}"
+  kadeploy_environment_export_format: "tar.gz"
+  kadeploy_environment_export_description: "Customized $${kadeploy_environment_import_name}"
+  kadeploy_environment_export_dir: "$HOME/public/"
+  kadeploy_environment_export_baseurl: "local://$HOME/public/"
+
+  # output appliance options
+  filesystem_type: ext4
+  image_size: 10G
+  image_disk: $${kameleon_cwd}/base_$${kameleon_recipe_name}
+  # Allowed formats are: tar.gz, tar.bz2, tar.xz, tar.lzo, qcow, qcow2, qed, vdi, raw, vmdk
+  appliance_formats: "$${kadeploy_environment_export_format}"
+  appliance_tar_compression_level: "9"
+  appliance_filename: "$${kameleon_cwd}/$${kameleon_recipe_name}"
+  appliance_tar_excludes: >-
+    ./etc/fstab ./root/.bash_history ./root/kameleon_workdir ./root/.ssh
+    ./var/tmp/* ./tmp/* ./var/log/* ./dev/* ./proc/* ./run/*
+    ./sys/*
+  # rootfs options
+  rootfs: $${kameleon_cwd}/rootfs
+
+  # enable workaround for the login delay due to ldap and dns
+  enable_workaround_login_delay: true
+
+  # kernel boot parameters: use legacy network interfaces
+  kernel_args: "quiet net.ifnames=0 biosdevname=0"
+
+  ssh_config_file: $${kameleon_cwd}/ssh_config
+  local_ip: 10.0.2.2
+
+  ## Select backend for in context isolation
+  backend: qemu
+
+  ## System variables. Required by kameleon engine
+  # Include specific steps
+  include_steps:
+    - $${distrib}/$${release}
+    - $${distrib}
+bootstrap:
+  - "@base"
+
+setup:
+  # Add steps here
+
+export:
+  - "@base"
+  - create_kadeploy_environment

kameleon/kameleon-recipes/from_kadeploy_environment/debian9-x64-custom.yaml~

+#==============================================================================
+# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
+#==============================================================================
+#
+# DESCRIPTION: base recipe for a customization of a Grid'5000 environment
+#
+#==============================================================================
+---
+extend: ../steps/backend/$${backend}.yaml
+
+# Loads some helpful aliases (this files are located in steps/aliases/ directory)
+aliases: defaults.yaml
+
+# Custom shell environement (this files are located in steps/env/ directory)
+env:
+  - bashrc
+  - functions.sh
+
+# Global variables use by Kameleon engine and the steps
+global:
+  # Architecture for the target system
+  arch: x86_64
+  # Distribution
+  distrib: debian
+  release: 9
+  # Kadeploy environment
+  kadeploy_environment_import_name: "debian9-x64-min"
+  kadeploy_environment_import_user: "deploy"
+  kadeploy_environment_import_version: ""
+  kadeploy_frontend: "frontend"
+  kadeploy_environment_export_name: "$${kameleon_recipe_name}"
+  kadeploy_environment_export_format: "tar.gz"
+  kadeploy_environment_export_description: "Customized $${kadeploy_environment_import_name}"
+  kadeploy_environment_export_dir: "$HOME/public/"
+  kadeploy_environment_export_baseurl: "local://$HOME/public/"
+
+  # output appliance options
+  filesystem_type: ext4
+  image_size: 10G
+  image_disk: $${kameleon_cwd}/base_$${kameleon_recipe_name}
+  # Allowed formats are: tar.gz, tar.bz2, tar.xz, tar.lzo, qcow, qcow2, qed, vdi, raw, vmdk
+  appliance_formats: "$${kadeploy_environment_export_format}"
+  appliance_tar_compression_level: "9"
+  appliance_filename: "$${kameleon_cwd}/$${kameleon_recipe_name}"
+  appliance_tar_excludes: >-
+    ./etc/fstab ./root/.bash_history ./root/kameleon_workdir ./root/.ssh
+    ./var/tmp/* ./tmp/* ./var/log/* ./dev/* ./proc/* ./run/*
+    ./sys/*
+  # rootfs options
+  rootfs: $${kameleon_cwd}/rootfs
+
+  # kernel boot parameters: use legacy network interfaces
+  kernel_args: "quiet net.ifnames=0 biosdevname=0"
+
+  ssh_config_file: $${kameleon_cwd}/ssh_config
+  local_ip: 10.0.2.2
+
+  ## Select backend for in context isolation
+  backend: qemu
+
+  ## System variables. Required by kameleon engine
+  # Include specific steps
+  include_steps:
+    - $${distrib}/$${release}
+    - $${distrib}
+bootstrap:
+  - "@base"
+
+setup:
+  # Add steps here
+
+export:
+  - "@base"
+  - create_kadeploy_environment

kameleon/kameleon-recipes/steps/aliases/defaults.yaml

+write_local:
+  - exec_local: |
+      mkdir -p $(dirname @1);
+      cat >@1 <<EOF_KAMELEON_INTERNAL
+      @2
+      EOF_KAMELEON_INTERNAL
+
+write_in:
+  - exec_in: |
+      mkdir -p $(dirname @1);
+      cat >@1 <<EOF_KAMELEON_INTERNAL
+      @2
+      EOF_KAMELEON_INTERNAL
+
+write_out:
+  - exec_out: |
+      mkdir -p $(dirname @1);
+      cat >@1 <<EOF_KAMELEON_INTERNAL
+      @2
+      EOF_KAMELEON_INTERNAL
+
+append_local:
+  - exec_local: |
+      mkdir -p $(dirname @1);
+      cat >>@1 <<EOF_KAMELEON_INTERNAL
+      @2
+      EOF_KAMELEON_INTERNAL
+
+append_in:
+  - exec_in: |
+      mkdir -p $(dirname @1);
+      cat >>@1 <<EOF_KAMELEON_INTERNAL
+      @2
+      EOF_KAMELEON_INTERNAL
+
+append_out:
+  - exec_out: |
+      mkdir -p $(dirname @1);
+      cat >>@1 <<EOF_KAMELEON_INTERNAL
+      @2
+      EOF_KAMELEON_INTERNAL
+
+local2out:
+  - exec_out: |
+      mkdir -p $(dirname @2)
+  - pipe:
+      - exec_local: cat @1
+      - exec_out: cat > @2
+
+local2in:
+  - exec_in: mkdir -p $(dirname @2)
+  - pipe:
+      - exec_local: cat @1
+      - exec_in: cat > @2
+
+out2local:
+  - exec_local: mkdir -p $(dirname @2)
+  - pipe:
+      - exec_out: cat @1
+      - exec_local: cat > @2
+
+out2in:
+  - exec_in: mkdir -p $(dirname @2)
+  - pipe:
+      - exec_out: cat @1
+      - exec_in: cat > @2
+
+in2local:
+  - exec_local: mkdir -p $(dirname @2)
+  - pipe:
+      - exec_in: cat @1
+      - exec_local: cat > @2
+
+in2out:
+  - exec_out: mkdir -p $(dirname @2)
+  - pipe:
+      - exec_in: cat @1
+      - exec_out: cat > @2
+
+check_cmd_out:
+  - rescue:
+    - exec_out: command -V @1 2> /dev/null
+    - breakpoint: "@1 is missing from out_context"
+
+check_cmd_local:
+  - on_bootstrap_init:
+    - rescue:
+      - exec_local: command -V @1 2> /dev/null
+      - breakpoint: "@1 is missing from local_context"
+
+check_cmd_in:
+  - rescue:
+    - exec_in: command -V @1 2> /dev/null
+    - breakpoint: "@1 is missing from in_context"
+
+umount_out:
+  - exec_out: |
+      echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true
+
+umount_local:
+  - exec_local: |
+      echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true
+
+umount_in:
+  - exec_in: |
+      echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true
+
+download_file_in:
+  - exec_in: __download "@1" "@2"
+
+download_file_out:
+  - exec_out: __download "@1" "@2"
+
+download_file_local:
+  - exec_local: __download "@1" "@2"
+
+download_recipe_build_local:
+  - exec_local: __download_recipe_build "@1" "@2" "@3" "@4" "@5" "@6" "@7"
+
+download_kadeploy_environment_image_local:
+  - exec_local: __download_kadeploy_environment_image "@1" "@2" "@3" "@4" "@5"

kameleon/kameleon-recipes/steps/backend/VM.yaml

+# Generic Virtual Machine backend recipe
+---
+global:
+  ssh_config_file: $${kameleon_cwd}/ssh_config
+  out_context:
+    cmd: ssh -F $${ssh_config_file} $${kameleon_recipe_name} -t /bin/bash
+    workdir: /root/kameleon_workdir
+    proxy_cache: $${local_ip}
+
+  in_context:
+    cmd: ssh -F $${ssh_config_file} $${kameleon_recipe_name} -t /bin/bash
+    workdir: /root/kameleon_workdir
+    proxy_cache: $${local_ip}
+
+bootstrap:
+  - download_upstream_tarball
+  - create_appliance
+  - prepare_appliance
+
+setup:
+  - "@base"
+
+export:
+  - disable_checkpoint
+  - save_appliance_VM

kameleon/kameleon-recipes/steps/backend/qemu.yaml

+# Qemu backend recipe
+---
+extend: VM.yaml
+
+checkpoint: qemu.yaml
+
+global:
+  image_format: qcow2
+
+  qemu_enable_kvm: "$(egrep '(vmx|svm)' /proc/cpuinfo > /dev/null && echo true)"
+  qemu_cpu: "$(($(nproc)-1))"
+  qemu_memory_size: 768
+  qemu_monitor_socket: $${kameleon_cwd}/qemu_monitor.socket
+  qemu_arch: $${arch}
+  qemu_pidfile: $${kameleon_cwd}/qemu.pid
+
+bootstrap:
+  - prepare_ssh_to_out_context
+  - "@base"
+  - start_qemu
+
+setup:
+  - "@base"
+
+export:
+  - "@base"

kameleon/kameleon-recipes/steps/bootstrap/create_appliance.yaml

+- create_appliance_script: $${kameleon_data_dir}/helpers/create_appliance.py
+
+- create_appliance:
+  - check_cmd_local: virt-make-fs
+  - exec_local: |
+      rm -f $${image_disk}.$${image_format}
+      [ -n "$UPSTREAM_TARBALL" ] || fail "UPSTREAM_TARBALL is not defined"
+      [ -r "$UPSTREAM_TARBALL" ] || fail "UPSTREAM_TARBALL ($UPSTREAM_TARBALL) is not readable"
+      # Create an appliance from rootfs directory
+      python $${create_appliance_script} $UPSTREAM_TARBALL \
+        --size $${image_size} \
+        -o $${image_disk} \
+        --filesystem $${filesystem_type} \
+        --format $${image_format} \
+        --append "$${kernel_args}"
+
+- delete_initial_image_at_the_end:
+  - on_checkpoint: skip
+  - on_export_clean:
+    - exec_local: rm -f $${image_disk}.$${image_format}

kameleon/kameleon-recipes/steps/bootstrap/download_upstream_tarball.yaml

+- upstream_store_dir: "$${kameleon_cwd}/upstream_build/"
+- upstream_tarball: ""
+- upstream_recipe: ""
+- upstream_version: ""
+- upstream_checksum: ""
+- upstream_checksign: ""
+- upstream_cache: ""
+- upstream_url: ""
+- kadeploy_environment_import_name: ""
+- kadeploy_environment_import_user: ""
+- kadeploy_environment_import_version: ""
+- kadeploy_frontend: ""
+
+- download:
+  - on_bootstrap_clean:
+    - exec_local: rm -rf $${upstream_store_dir}
+  - test:
+    - exec_local: test -n "$${upstream_tarball}"
+    - group:
+      - exec_local: echo "Download tarball..."
+      - download_file_local:
+        - $${upstream_url}/$${upstream_tarball}
+        - $${upstream_store_dir}/$${upstream_tarball}
+      - exec_local: export UPSTREAM_TARBALL="$${upstream_store_dir}/$${upstream_tarball}"
+    - test:
+      - exec_local: test -n "$${upstream_recipe}"
+      - download_recipe_build_local:
+        - $${upstream_recipe}
+        - $${upstream_version}
+        - $${upstream_checksum}
+        - $${upstream_checksign}
+        - $${upstream_cache}
+        - $${upstream_url}
+        - $${upstream_store_dir}
+      - test:
+        - exec_local: test -n "$${kadeploy_environment_import_name}"
+        - download_kadeploy_environment_image_local:
+          - $${kadeploy_environment_import_name}
+          - $${kadeploy_environment_import_user}
+          - $${kadeploy_environment_import_version}
+          - $${kadeploy_frontend}
+          - $${upstream_store_dir}
+        - exec_local: fail "Nothing to download ???"

kameleon/kameleon-recipes/steps/bootstrap/prepare_appliance.yaml

+- insecure_ssh_key: $${kameleon_cwd}/insecure_ssh_key
+- enable_workaround_login_delay: false
+
+- generate_ssh_keys:
+  - check_cmd_local: ssh-keygen
+  - exec_local: echo -e  'y\n' | ssh-keygen -q -t rsa -b 4096 -f $${insecure_ssh_key} -N ''
+  - exec_local: cat $${insecure_ssh_key}
+
+- inject_ssh_private_key:
+  - check_cmd_local: virt-customize
+  - exec_local: |
+      virt-customize \
+        -a $${image_disk}.$${image_format} \
+        --run-command 'mkdir -p /root/.ssh' \
+        --upload $${insecure_ssh_key}.pub:/root/.ssh/.kameleon_authorized_keys \
+        --run-command 'touch /root/.ssh/authorized_keys' \
+        --run-command 'cp /root/.ssh/authorized_keys /root/.ssh/authorized_keys.bak' \
+        --run-command 'cat /root/.ssh/.kameleon_authorized_keys >> /root/.ssh/authorized_keys' \
+        --run-command 'chmod 700 /root/.ssh' \
+        --run-command 'chmod -R go-rw /root/.ssh' \
+        --run-command 'chown -R root:root /root/.ssh'
+  - on_export_init:
+    - exec_local: |
+        virt-customize \
+          -a $${image_disk}.$${image_format} \
+          --run-command 'mv /root/.ssh/authorized_keys.bak /root/.ssh/authorized_keys' \
+          --delete /root/.ssh/.kameleon_authorized_keys
+
+- add_insecure_key_to_ssh_config:
+  - on_checkpoint: redo
+  - exec_local: |
+      cat <<EOF >> $${ssh_config_file}
+      IdentityFile $${insecure_ssh_key}
+      EOF
+
+- workaround_login_delay_due_to_ldap_and_dns:
+  - test:
+    - exec_local: test "$${enable_workaround_login_delay}" = "true"
+    - exec_local: |
+        virt-customize \
+          -a $${image_disk}.$${image_format} \
+          --run-command 'mv /etc/nslcd.conf /etc/nslcd.conf.orig-g5k' \
+          --run-command 'sed -i.orig-g5k "s/ ldap//" /etc/nsswitch.conf' \
+          --run-command 'echo > /etc/resolv.conf'
+    - exec_local: echo "Workaround is not enabled"
+  - on_setup_clean:
+    - test:
+      - exec_local: test "$${enable_workaround_login_delay}" = "true"
+      - exec_local: |
+          virt-customize \
+            -a $${image_disk}.$${image_format} \
+            --run-command 'mv /etc/nslcd.conf.orig-g5k /etc/nslcd.conf' \
+            --run-command 'mv /etc/nsswitch.conf.orig-g5k /etc/nsswitch.conf'

kameleon/kameleon-recipes/steps/bootstrap/prepare_appliance.yaml~

+- insecure_ssh_key: $${kameleon_cwd}/insecure_ssh_key
+- enable_workaround_login_delay: false
+
+- generate_ssh_keys:
+  - check_cmd_local: ssh-keygen
+  - exec_local: echo -e  'y\n' | ssh-keygen -q -t rsa -b 4096 -f $${insecure_ssh_key} -N ''
+  - exec_local: cat $${insecure_ssh_key}
+
+- inject_ssh_private_key:
+  - check_cmd_local: virt-customize
+  - exec_local: |
+      virt-customize \
+        -a $${image_disk}.$${image_format} \
+        --run-command 'mkdir -p /root/.ssh' \
+        --upload $${insecure_ssh_key}.pub:/root/.ssh/.kameleon_authorized_keys \
+        --run-command 'touch /root/.ssh/authorized_keys' \
+        --run-command 'cp /root/.ssh/authorized_keys /root/.ssh/authorized_keys.bak' \
+        --run-command 'cat /root/.ssh/.kameleon_authorized_keys >> /root/.ssh/authorized_keys' \
+        --run-command 'chmod 700 /root/.ssh' \
+        --run-command 'chmod -R go-rw /root/.ssh' \
+        --run-command 'chown -R root:root /root/.ssh'
+  - on_export_init:
+    - exec_local: |
+        virt-customize \
+          -a $${image_disk}.$${image_format} \
+          --run-command 'mv /root/.ssh/authorized_keys.bak /root/.ssh/authorized_keys' \
+          --delete /root/.ssh/.kameleon_authorized_keys
+
+- add_insecure_key_to_ssh_config:
+  - on_checkpoint: redo
+  - exec_local: |
+      cat <<EOF >> $${ssh_config_file}
+      IdentityFile $${insecure_ssh_key}
+      EOF
+
+- workaround_login_delay_due_to_ldap_and_dns:
+  - test:
+    - exec_local: test "$${enable_workaround_login_delay}" = "true"
+    - exec_local: |
+        virt-customize \
+          -a $${image_disk}.$${image_format} \
+          --run-command 'mv /etc/nslcd.conf /etc/nslcd.conf.orig-g5k' \
+          --run-command 'sed -i.orig-g5k "s/ ldap//" /etc/nsswitch.conf' \
+          --run-command 'echo > /etc/resolv.conf'
+    - exec_local: echo "Workaround is not enabled"
+  - on_setup_clean:
+    - test:
+      - exec_local: test "$${enable_workaround_login_delay}" = "true"
+      - exec_local: |
+          virt-customize \
+            -a $${image_disk}.$${image_format} \
+            --run-command 'mv /etc/nslcd.conf.orig-g5k /etc/nslcd.conf' \
+            --run-command 'mv /etc/nsswitch.conf.orig-g5k /etc/nsswitch.conf'

kameleon/kameleon-recipes/steps/bootstrap/prepare_ssh_to_out_context.yaml

+- select_empty_port:
+  - on_checkpoint: redo
+  - exec_local: |
+      # Find empty SSH forwarding port
+      SSH_FWD_PORT=$(__find_free_port 50000 60000)
+      echo "SSH forwarding port: $SSH_FWD_PORT"
+- prepare_ssh_config:
+  - on_checkpoint: redo
+  - write_local:
+    - $${ssh_config_file}
+    - |
+      Host $${kameleon_recipe_name}
+      HostName 127.0.0.1
+      Port ${SSH_FWD_PORT}
+      User root
+      UserKnownHostsFile /dev/null
+      StrictHostKeyChecking no
+      PasswordAuthentication no
+      IdentitiesOnly yes
+      LogLevel FATAL
+      ForwardAgent yes
+      Compression yes
+      Protocol 2

kameleon/kameleon-recipes/steps/bootstrap/start_qemu.yaml

+# Require SSH_FWD_PORT bash environment variable to be set
+- disk_cache: unsafe
+# This must be set if you want to boot an ISO image instead of a disk
+- iso_path: ""
+- iso_boot: "yes"
+- vm_expected_service: ssh
+- boot_timeout: 100
+- shutdown_timeout: 100
+- debug: false
+- telnet_port: ""
+- no_reboot: true
+- socat_monitor: socat - UNIX-CONNECT:$${qemu_monitor_socket}
+- qemu_sendkeys_script: $${kameleon_data_dir}/qemu-sendkeys.rb
+- qemu_sendkeys_commands: 
+- vm_expected_service: ssh
+- vm_cleanup_section: setup
+- shutdown_vm_immediately: false
+- force_vm_shutdown: true
+- qemu_enable_kvm: "$(egrep '(vmx|svm)' /proc/cpuinfo > /dev/null && echo true)"
+- qemu_cpu: 2
+- qemu_memory_size: 768
+- qemu_monitor_socket: $${kameleon_cwd}/qemu_monitor.socket
+- qemu_arch: $${arch}
+- qemu_image_size: 10G
+- qemu_pidfile: $${kameleon_cwd}/qemu.pid
+
+- start_vm:
+  - on_checkpoint: redo
+  - check_cmd_local: qemu-system-$${qemu_arch}
+  - check_cmd_local: socat
+  - on_bootstrap_clean:
+    - test:
+      - exec_local: test "$${shutdown_vm_immediately}" == "false" -a "$${vm_cleanup_section}" == "bootstrap"
+      - group: 
+        - exec_local: &1 |
+            if [ -f $${qemu_pidfile} ]; then
+              _QEMU_PID=$(< $${qemu_pidfile})
+              if ps -p $_QEMU_PID > /dev/null; then
+                if [ "$${force_vm_shutdown}" == "true" ]; then
+                  if [ -S $${qemu_monitor_socket} ]; then
+                    echo "Executing a graceful shutdown of the qemu VM via the monitor socket..."
+                    NEXT_WAIT_TIME=0
+                    echo system_powerdown | socat - UNIX-CONNECT:$${qemu_monitor_socket} || true
+                    while ps -p $_QEMU_PID > /dev/null && [ $NEXT_WAIT_TIME -lt $${shutdown_timeout} ];
+                    do
+                      sleep 1
+                      echo -en "\rWaiting for qemu virtual machine to shutdown...($(( $${shutdown_timeout} - 1 - NEXT_WAIT_TIME++ ))s)"
+                    done
+                  fi
+                else
+                  echo "Waiting for the VM to shutdown"
+                  echo "Run 'vncviewer :$VNC_PORT' to see what's happening in the VM"
+                  while ps -p $_QEMU_PID > /dev/null;
+                  do
+                    sleep 2
+                  done
+                fi
+              fi
+            fi
+        - exec_local: &2 |
+            if [ -f $${qemu_pidfile} ]; then
+              _QEMU_PID=$(< $${qemu_pidfile})
+              if ps -p $_QEMU_PID > /dev/null; then
+                if [ -S $${qemu_monitor_socket} ]; then
+                  echo "The graceful shutdown of the qemu VM should have failed (monitor socket is there)..."
+                fi
+                echo "Killing qemu (pid: $_QEMU_PID)."
+                kill -9 "$_QEMU_PID"
+              fi
+              rm -f $${qemu_pidfile}
+            fi
+            rm -f $${qemu_monitor_socket}
+  - on_setup_clean:
+    - test:
+      - exec_local: test "$${shutdown_vm_immediately}" == "false" -a "$${vm_cleanup_section}" == "setup"
+      - group:
+        - exec_local: *1
+        - exec_local: *2
+  - on_export_clean:
+    - test:
+      - exec_local: test "$${shutdown_vm_immediately}" == "false" -a "$${vm_cleanup_section}" == "export"
+      - group:
+        - exec_local: *1
+        - exec_local: *2
+  - exec_local: |
+      if [ "$${shutdown_vm_immediately}" == "true" ]; then
+        echo "Qemu VM shutdown: immediately"
+      else
+        echo "Qemu VM shutdown: in $${vm_cleanup_section} section cleaning"
+      fi
+  - exec_local: |
+      if [ -r $${qemu_pidfile} ] && pgrep -F $${qemu_pidfile} > /dev/null; then
+        echo "Qemu pid file found, with process running: killing it !" 1>&2
+        pkill -F $${qemu_pidfile}
+        sleep 0.5
+        if pgrep -F $${qemu_pidfile} > /dev/null; then
+          echo "Failed to kill qemu process." 1>&2
+          exit 1
+        fi
+      fi
+  - exec_local: |
+      echo "Starting qemu..."
+      if [ "$${qemu_enable_kvm}" == "true" ]; then
+        ENABLE_KVM="-enable-kvm -cpu host"
+        BOOT_TIMEOUT=$${boot_timeout}
+      else
+        BOOT_TIMEOUT=$(($${boot_timeout}*2))
+      fi
+      if [ -f "vm_state_to_load.txt" ]
+      then
+          SAVED_STATE="$(< vm_state_to_load.txt)"
+          LOADVM="-loadvm $SAVED_STATE"
+          rm -f vm_state_to_load.txt
+      fi
+      if [ "$${debug}" == "true" ]; then
+        VNC_OPT=""
+      else
+        # Find empty VNC port
+        VNC_PORT=$(( $(__find_free_port 5900 5910) - 5900 ))
+        echo "VNC port: $VNC_PORT"
+        VNC_OPT="-vnc :$VNC_PORT"
+      fi
+      if [ -n "$${telnet_port}" ]; then
+        SERIAL_TELNET="telnet:localhost:$${telnet_port},server"
+      fi
+      # Select disk
+      QEMU_DRIVES="-drive file=$${image_disk}.$${image_format},cache=$${disk_cache},media=disk,if=virtio,id=drive0"
+      QENU_BOOT=
+      if [ -n "$${iso_path}" ]; then
+        QEMU_DRIVES="-drive file=$${iso_path},readonly,media=cdrom $QEMU_DRIVES"
+        if [ "$${iso_boot}" == "yes" ]; then
+          QEMU_BOOT="-boot order=d"
+        fi
+      fi
+      if [ "$${no_reboot}" == "true" ]; then
+        NO_REBOOT="-no-reboot"
+      fi
+      if [ -n "${SSH_FWD_PORT}" ]; then
+        HOSTFWD=",hostfwd=tcp::${SSH_FWD_PORT}-:22"
+      fi
+      qemu-system-$${qemu_arch} $ENABLE_KVM -smp $${qemu_cpu} -m $${qemu_memory_size} -rtc base=localtime \
+        -net nic,model=virtio -net user${HOSTFWD} \
+        $QEMU_DRIVES \
+        -monitor unix:$${qemu_monitor_socket},server,nowait -pidfile $${qemu_pidfile} -daemonize \
+        $QEMU_BOOT $NO_REBOOT \
+        $VNC_OPT $SERIAL_TELNET\
+        $LOADVM
+  - exec_local: |
+      VM_AVAILABLE=0
+      if [ "$${vm_expected_service}" == "ssh" ]; then
+        TIMEOUT=$(( $(date +%s) + $BOOT_TIMEOUT ))
+        until timeout 5 ssh -q -F $${ssh_config_file} -o ConnectionAttempts=1  $${kameleon_recipe_name} -t true && VM_AVAILABLE=1 || [ $(date +%s) -gt $TIMEOUT ];
+        do
+          echo -en "\rWaiting for SSH to become available in VM for out_context...($(( TIMEOUT - $(date +%s) ))s)"
+          sleep 1
+        done
+        echo
+      else
+        TIMEOUT=$(( $(date +%s) + $BOOT_TIMEOUT ))
+        until timeout 1 [ $(date +%s) -gt $TIMEOUT ];
+        do
+          echo -en "\rWaiting for VM to become available : ($(( TIMEOUT - $(date +%s) ))s)"
+          sleep 1
+        done
+        echo
+        VM_AVAILABLE=1
+      fi
+  - rescue:
+    - exec_local: test $VM_AVAILABLE -eq 1
+    - breakpoint: |
+        Failed to get VM up and running (expected service: $${vm_expected_service}). Please verify the VM successfully booted with a vnc client.
+  - test:
+    - exec_local: test -e "$${qemu_sendkeys_commands}" -a -s "$${qemu_sendkeys_commands}"
+    - exec_local: |
+        echo "Sending keyboard commands to the VM: $${qemu_sendkeys_commands}"
+        echo "(Local httpd server url: http://$${local_ip}:$HTTP_PORT)"
+        ruby $${qemu_sendkeys_script} -d 0.05 "$(sed -e s/%LOCAL_IP%/$${local_ip}/g -e s/%HTTP_PORT%/$HTTP_PORT/g $${qemu_sendkeys_commands})" | $${socat_monitor} > /dev/null
+    - exec_local: echo "No keyboard commands to send"
+
+- shutdown_vm:
+  - on_checkpoint: redo
+  - on_clean:
+    - test:
+      - exec_local: test "$${shutdown_vm_immediately}" == "true"
+      - exec_local: *2
+  - test:
+    - exec_local: test "$${shutdown_vm_immediately}" == "true"
+    - exec_local: *1

kameleon/kameleon-recipes/steps/checkpoints/qemu.yaml

+enabled?:
+  - exec_local: test -f $${kameleon_cwd}/checkpoint_enabled
+  - exec_local: test -S $${qemu_monitor_socket}
+
+create:
+  - exec_local: echo -e "savevm @microstep_id" | socat - UNIX-CONNECT:$${qemu_monitor_socket}
+
+apply:
+  - exec_local: |
+      [[ ! -f "$${kameleon_cwd}/vm_state_to_load.txt" ]] || rm -f $${kameleon_cwd}/vm_state_to_load.txt
+      if [ -f "$${image_disk}" ] ; then
+        _checkpoints=$(qemu-img snapshot -l $${image_disk} | tail -n +3 | awk '{print $2}')
+        _checkpoints=($_checkpoints)
+        for checkpoint in "${_checkpoints[@]}"; do
+          if [ -f "$${kameleon_cwd}/vm_state_to_load.txt" ]; then
+            echo "Removing old checkpoint $checkpoint"
+            qemu-img snapshot -d $checkpoint $${image_disk}
+          fi
+          if [[ "$checkpoint" == "@microstep_id" ]]; then
+              echo "@microstep_id" > $${kameleon_cwd}/vm_state_to_load.txt
+              qemu-img snapshot -a $checkpoint $${image_disk}
+          fi
+      done
+      fi
+
+clear:
+  - exec_local: rm -f $${image_disk}
+
+list:
+  - exec_local: |
+      if [ -f "$${image_disk}" ] ; then
+          qemu-img snapshot -l $${image_disk} | tail -n +3 | awk '{print $2}'
+      fi

kameleon/kameleon-recipes/steps/data/helpers/create_appliance.py

+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""Convert an rootfs archive to a bootable disk image with guestfish."""
+
+import os
+import os.path as op
+import sys
+import subprocess
+import argparse
+import logging
+import tempfile
+import shutil
+import contextlib
+
+
+logger = logging.getLogger(__name__)
+
+
+tar_options = ["--selinux",
+               "--xattrs",
+               "--xattrs-include='*'",
+               "--numeric-owner",
+               "--one-file-system"]
+
+
+@contextlib.contextmanager
+def temporary_directory():
+    """Context manager for tempfile.mkdtemp()."""
+    name = tempfile.mkdtemp()
+    try:
+        yield name
+    finally:
+        shutil.rmtree(name)
+
+
+def which(command):
+    """Locate a command.
+    Snippet from: http://stackoverflow.com/a/377028
+    """
+    def is_exe(fpath):
+        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+    fpath, fname = os.path.split(command)
+    if fpath:
+        if is_exe(command):
+            return command
+    else:
+        for path in os.environ["PATH"].split(os.pathsep):
+            path = path.strip('"')
+            exe_file = os.path.join(path, command)
+            if is_exe(exe_file):
+                return exe_file
+
+    raise ValueError("Command '%s' not found" % command)
+
+
+def file_type(path):
+    """Get file type."""
+    if not op.exists(path):
+        raise Exception("cannot open '%s' (No such file or directory)" % path)
+    cmd = [which("file"), path]
+    proc = subprocess.Popen(cmd,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT,
+                            env=os.environ.copy(),
+                            shell=False)
+    output, _ = proc.communicate()
+    if proc.returncode:
+        raise subprocess.CalledProcessError(proc.returncode, ' '.join(cmd))
+    return output.decode('utf-8').split(':')[1].strip()
+
+
+def qemu_convert(disk, output_fmt, output_filename):
+    """Convert the disk image filename to disk image output_filename."""
+    binary = which("qemu-img")
+    cmd = [binary, "convert", "-p", "-O", output_fmt, disk, output_filename]
+    if output_fmt in ("qcow", "qcow2"):
+        cmd.insert(2, "-c")
+    proc = subprocess.Popen(cmd, env=os.environ.copy(), shell=False)
+    proc.communicate()
+    if proc.returncode:
+        raise subprocess.CalledProcessError(proc.returncode, ' '.join(cmd))
+
+
+def run_guestfish_script(disk, script, mount=True, piped_output=False):
+    """Run guestfish script."""
+    args = [which("guestfish"), '-a', disk]
+    if mount:
+        script = "run\nmount /dev/sda1 /\n%s" % script
+    else:
+        script = "run\n%s" % script
+    if piped_output:
+        proc = subprocess.Popen(args,
+                                stdin=subprocess.PIPE,
+                                stdout=subprocess.PIPE,
+                                env=os.environ.copy())
+
+        stdout, _ = proc.communicate(input=script.encode('utf-8'))
+    else:
+        proc = subprocess.Popen(args,
+                                stdin=subprocess.PIPE,
+                                env=os.environ.copy())
+        proc.communicate(input=script.encode('utf-8'))
+    if proc.returncode:
+        raise subprocess.CalledProcessError(proc.returncode, ' '.join(args))
+
+    if piped_output:
+        return stdout.decode('utf-8')
+
+
+def find_mbr():
+    """ ..."""
+    search_paths = (
+        "/usr/share/syslinux/mbr.bin",
+        "/usr/lib/bios/syslinux/mbr.bin",
+        "/usr/lib/syslinux/bios/mbr.bin",
+        "/usr/lib/extlinux/mbr.bin",
+        "/usr/lib/syslinux/mbr.bin",
+        "/usr/lib/syslinux/mbr/mbr.bin",
+        "/usr/lib/EXTLINUX/mbr.bin"
+    )
+    for path in search_paths:
+        if op.exists(path):
+            return path
+    raise Exception("syslinux MBR not found")
+
+
+def get_boot_information(disk):
+    """Looking for boot information"""
+    script_1 = """
+blkid /dev/sda1 | grep ^UUID: | awk '{print $2}'
+ls /boot/ | grep ^vmlinuz | head -n 1
+ls /boot/ | grep ^init | grep -v fallback | head -n 1
+ls /boot/ | grep ^init | grep fallback | head -n 1"""
+    logger.info(get_boot_information.__doc__)
+    output_1 = run_guestfish_script(disk, script_1, piped_output=True)
+    try:
+        infos = output_1.strip().split('\n')
+        if len(infos) == 4:
+            uuid, vmlinuz, initrd, initrd_fallback = infos
+            if initrd:
+                return uuid, vmlinuz, initrd
+            else:
+                return uuid, vmlinuz, initrd_fallback
+        else:
+            uuid, vmlinuz, initrd = infos
+            return uuid, vmlinuz, initrd
+    except:
+        raise Exception("Invalid boot information (missing kernel ?)")
+
+
+def generate_fstab(disk, uuid, filesystem_type):
+    """Generate /etc/fstab file"""
+    logger.info("Generating /etc/fstab")
+    script = """
+write /etc/fstab "# /etc/fstab: static file system information.\\n"
+write-append  /etc/fstab "# Generated by kameleon-helpers.\\n\\n"
+write-append /etc/fstab "UUID=%s\\t/\\t%s\\tdefaults\\t0\\t1\\n"
+""" % (uuid, filesystem_type)
+    run_guestfish_script(disk, script)
+
+
+def install_bootloader(disk, mbr, append):
+    """Install a bootloader"""
+    mbr_path = mbr or find_mbr()
+    mbr_path = op.abspath(mbr_path)
+    uuid, vmlinuz, initrd = get_boot_information(disk)
+    logger.info("Root partition UUID: %s" % uuid)
+    logger.info("Kernel image: /boot/%s" % vmlinuz)
+    logger.info("Initrd image: /boot/%s" % initrd)
+    script = """
+echo "[guestfish] Upload the master boot record"
+upload %s /boot/mbr.bin
+
+echo "[guestfish] Generate /boot/syslinux.cfg"
+write /boot/syslinux.cfg "DEFAULT linux\\n"
+write-append /boot/syslinux.cfg "LABEL linux\\n"
+write-append /boot/syslinux.cfg "SAY Booting the kernel\\n"
+write-append /boot/syslinux.cfg "KERNEL /boot/%s\\n"
+write-append /boot/syslinux.cfg "INITRD /boot/%s\\n"
+write-append /boot/syslinux.cfg "APPEND ro root=UUID=%s %s\\n"
+
+echo "[guestfish] Put the MBR into the boot sector"
+copy-file-to-device /boot/mbr.bin /dev/sda size:440
+
+echo "[guestfish] Install extlinux on the first partition"
+extlinux /boot
+
+echo "[guestfish] Set the first partition as bootable"
+part-set-bootable /dev/sda 1 true
+
+echo "[guestfish] Generate empty fstab"
+write /etc/fstab "# UNCONFIGURED FSTAB FOR BASE SYSTEM\\n"
+
+echo "[guestfish] Set / permissions to '0755'"
+chmod 0755 /
+""" % (mbr_path, vmlinuz, initrd, uuid, append)
+    run_guestfish_script(disk, script)
+    return uuid, vmlinuz, initrd
+
+
+def create_disk(input_, output_filename, fmt, size, filesystem, verbose):
+    """Make a disk image from a tar archive or files."""
+    # create empty disk
+    logger.info("Creating an empty disk image")
+    qemu_img = which("qemu-img")
+    cmd = [qemu_img, "create", "-f", fmt, output_filename, size, "-q"]
+    proc = subprocess.Popen(cmd, env=os.environ.copy(), shell=False)
+    proc.communicate()
+    if proc.returncode:
+        raise subprocess.CalledProcessError(proc.returncode, ' '.join(cmd))
+
+    mkfs_options = ""
+    # Fix syslinux 6.03 compat for ext4 64bit, see:
+    # http://www.syslinux.org/wiki/index.php?title=Filesystem#ext
+    if "ext4" in filesystem:
+        mkfs_options = "features:^64bit"
+    # parition disk and create the filesystem
+    script = """
+echo "[guestfish] Create new MBR partition table on /dev/sda"
+part-disk /dev/sda mbr
+echo "[guestfish] Create {filesystem} filesystem on /dev/sda1"
+mkfs {filesystem} /dev/sda1 {mkfs_options}
+""".format(filesystem=filesystem,
+           mkfs_options=mkfs_options)
+    run_guestfish_script(output_filename, script, mount=False)
+
+    # Fill disk with our data
+    logger.info("Copying data in the disk image")
+    input_type = file_type(input_).lower()
+
+    make_tar_cmd = ""
+    if "xz compressed data" in input_type:
+        make_tar_cmd = "%s %s" % (which("xzcat"), input_)
+    elif "bzip2 compressed data" in input_type:
+        make_tar_cmd = "%s %s" % (which("bzcat"), input_)
+    elif "gzip compressed data" in input_type:
+        make_tar_cmd = "%s %s" % (which("zcat"), input_)
+    elif "directory" in input_type:
+        excludes = ['dev/*', 'proc/*', 'sys/*', 'tmp/*', 'run/*',
+                    '/mnt/*']
+        tar_options_str = ' '.join(tar_options +
+                                   ['--exclude="%s"' % s for s in excludes])
+        make_tar_cmd = '%s -cf - %s -C %s $(cd %s; ls -A)' % \
+            (which("tar"), tar_options_str, input_, input_)
+
+    if make_tar_cmd:
+        cmd = "%s | %s -a %s -m /dev/sda1:/ tar-in - /" % \
+            (make_tar_cmd, which("guestfish"), output_filename)
+    else:
+        cmd = "%s -a %s -m /dev/sda1:/ tar-in %s /" % \
+            (which("guestfish"), output_filename, input_)
+    proc = subprocess.Popen(cmd, env=os.environ.copy(), shell=True)
+    proc.communicate()
+    if proc.returncode:
+        raise subprocess.CalledProcessError(proc.returncode, cmd)
+
+
+def create_appliance(args):
+    """Convert disk to another format."""
+    input_ = op.abspath(args.input)
+    output = op.abspath(args.output)
+    temp_filename = next(tempfile._get_candidate_names())
+    temp_file = op.abspath(".%s" % temp_filename)
+    output_fmt = args.format.lower()
+    output_filename = "%s.%s" % (output, output_fmt)
+
+    os.environ['LIBGUESTFS_CACHEDIR'] = os.getcwd()
+    if args.verbose:
+        os.environ['LIBGUESTFS_DEBUG'] = '1'
+
+    create_disk(input_,
+                temp_file,
+                args.format,
+                args.size,
+                args.filesystem,
+                args.verbose)
+    logger.info("Installing bootloader")
+    uuid, _, _ = install_bootloader(temp_file,
+                                    args.extlinux_mbr,
+                                    args.append)
+    generate_fstab(temp_file, uuid, args.filesystem)
+
+    logger.info("Exporting appliance to %s" % output_filename)
+    if output_fmt == "qcow2" and args.compress:
+        qemu_convert(temp_file, output_fmt, output_filename)
+        os.remove(temp_file) if os.path.exists(temp_file) else None
+    else:
+        shutil.move(temp_file, output_filename)
+
+
+if __name__ == '__main__':
+    allowed_formats = ('qcow', 'qcow2', 'qed', 'vdi', 'raw', 'vmdk')
+    allowed_formats_help = 'Allowed values are ' + ', '.join(allowed_formats)
+
+    allowed_levels = ["%d" % i for i in range(1, 10)] + ["best", "fast"]
+    allowed_levels_helps = 'Allowed values are ' + ', '.join(allowed_levels)
+
+    parser = argparse.ArgumentParser(
+        description=sys.modules[__name__].__doc__,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument('input', action="store",
+                        help='input root filesystem in directory (chroot) '
+                             'or a tarball (conpressed or not)')
+    parser.add_argument('-F', '--format', action="store", type=str,
+                        help=('Choose the output disk image format. %s' %
+                              allowed_formats_help), default='qcow2')
+    parser.add_argument('-t', '--filesystem', action="store", type=str,
+                        help='Choose the output filesystem type.',
+                        default="ext2")
+    parser.add_argument('-s', '--size', action="store", type=str,
+                        help='choose the size of the output image',
+                        default="10G")
+    parser.add_argument('-o', '--output', action="store", type=str,
+                        help='Output filename (without file extension)',
+                        required=True, metavar='filename')
+    parser.add_argument('--extlinux-mbr', action="store", type=str,
+                        help='Extlinux MBR', metavar='')
+    parser.add_argument('--append', action="store", type=str,
+                        default="",
+                        help='Additional kernel args', metavar='')
+    parser.add_argument('--verbose', action="store_true", default=False,
+                        help='Enable very verbose messages')
+    parser.add_argument('--compress', action="store_true", default=False,
+                        help='compress the output file (Only qcow2).')
+    log_format = '%(levelname)s: %(message)s'
+    level = logging.INFO
+    try:
+        args = parser.parse_args()
+        if args.verbose:
+            level = logging.DEBUG
+
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(logging.Formatter(log_format))
+
+        logger.setLevel(level)
+        logger.addHandler(handler)
+        create_appliance(args)
+    except Exception as exc:
+        sys.stderr.write(u"\nError: %s\nTry to execute this script with "
+                         "the '--verbose' option to get more details" % exc)
+        sys.exit(1)

kameleon/kameleon-recipes/steps/data/helpers/export_appliance.py

+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""Convert a disk image to many others formats with guestfish."""
+from __future__ import division, unicode_literals
+
+import os
+# import time
+import os.path as op
+import sys
+import subprocess
+import argparse
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+tar_formats = ('tar', 'tar.gz', 'tgz', 'tar.bz2', 'tbz', 'tar.xz', 'txz',
+               'tar.lzo', 'tzo')
+
+tar_options = ["--selinux", "--xattrs", "--xattrs-include='*'", "--numeric-owner", "--one-file-system"] 
+
+disk_formats = ('qcow', 'qcow2', 'qed', 'vdi', 'raw', 'vmdk')
+
+
+def which(command):
+    """Locate a command.
+    Snippet from: http://stackoverflow.com/a/377028
+    """
+    def is_exe(fpath):
+        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+    fpath, fname = os.path.split(command)
+    if fpath:
+        if is_exe(command):
+            return command
+    else:
+        for path in os.environ["PATH"].split(os.pathsep):
+            path = path.strip('"')
+            exe_file = os.path.join(path, command)
+            if is_exe(exe_file):
+                return exe_file
+
+    raise ValueError("Command '%s' not found" % command)
+
+
+def tar_convert(disk, output, excludes, compression_level):
+    """Convert image to a tar rootfs archive."""
+    if compression_level in ("best", "fast"):
+        compression_level_opt = "--%s" % compression_level
+    else:
+        compression_level_opt = "-%s" % compression_level
+
+    compr = ""
+    if output.endswith(('tar.gz', 'tgz')):
+        try:
+            compr = "| %s %s" % (which("pigz"), compression_level_opt)
+        except:
+            compr = "| %s %s" % (which("gzip"), compression_level_opt)
+    elif output.endswith(('tar.bz2', 'tbz')):
+        compr = "| %s %s" % (which("bzip2"), compression_level_opt)
+    elif output.endswith(('tar.xz', 'txz')):
+        compr = "| {} {} -c --threads=0 -".format(
+            which("xz"), compression_level_opt)
+    elif output.endswith(('tar.lzo', 'tzo')):
+        compr = "| %s %s -c -" % (which("lzop"), compression_level_opt)
+
+    # NB: guestfish version >= 1.32 supports the special tar options, but not available in Debian stable (jessie): do not use for now
+    #tar_options_list = ["selinux:true", "acls:true", "xattrs:true",
+    #                    "numericowner:true",
+    #                    "excludes:\"%s\"" % ' '.join(excludes)]
+    #tar_options_str = ' '.join(tar_options_list)
+    #cmd = which("guestfish") + \
+    #    " --ro -i tar-out -a %s / - %s %s > %s"
+    #cmd = cmd % (disk, tar_options_str, compr, output)
+    #proc = subprocess.Popen(cmd_mount_tar, env=os.environ.copy(), shell=True)
+    #proc.communicate()
+    #if proc.returncode:
+    #    raise subprocess.CalledProcessError(proc.returncode, cmd)
+
+    tar_options_str = ' '.join(tar_options + ['--exclude="%s"' % s for s in excludes])
+    directory = dir_path = os.path.dirname(os.path.realpath(disk))
+    cmds = [
+        which("mkdir") + " %s/.mnt" % directory,
+        which("guestmount") + " --ro -i -a %s %s/.mnt" % (disk, directory),
+        which("tar") + " -c %s -C %s/.mnt . %s > %s" % (tar_options_str, directory, compr, output)
+        ]
+    cmd_mount_tar = " && ".join(cmds)
+    proc = subprocess.Popen(cmd_mount_tar, env=os.environ.copy(), shell=True)
+    proc.communicate()
+    returncode_mount_tar = proc.returncode
+
+    # try to umount even if the previous command failed
+    cmds = [
+        which("guestunmount") + " %s/.mnt" % directory,
+        which("rmdir") + " %s/.mnt" % directory
+        ]
+    cmd_umount = " && ".join(cmds)
+    proc = subprocess.Popen(cmd_umount, env=os.environ.copy(), shell=True)
+    proc.communicate()
+    returncode_umount = proc.returncode
+
+    if returncode_mount_tar:
+        raise subprocess.CalledProcessError(returncode_mount_tar, cmd_mount_tar)
+    elif returncode_umount:
+        raise subprocess.CalledProcessError(returncode_umount, cmd_umount)
+
+
+def qemu_convert(disk, output_fmt, output_filename):
+    """Convert the disk image filename to disk image output_filename."""
+    binary = which("qemu-img")
+    cmd = [binary, "convert", "-O", output_fmt, disk, output_filename]
+    if output_fmt in ("qcow", "qcow2"):
+        cmd.insert(2, "-c")
+    proc = subprocess.Popen(cmd, env=os.environ.copy(), shell=False)
+    proc.communicate()
+    if proc.returncode:
+        raise subprocess.CalledProcessError(proc.returncode, ' '.join(cmd))
+
+
+def run_guestfish_script(disk, script, mount=""):
+    """
+    Run guestfish script.
+    Mount should be in ("read_only", "read_write", "ro", "rw")
+    """
+    args = [which("guestfish"), '-a', disk]
+    if mount in ("read_only", "read_write", "ro", "rw"):
+        args.append('-i')
+        if mount in mount in ("read_only", "ro"):
+            args.append('--ro')
+        else:
+            args.append('--rw')
+    else:
+        script = "run\n%s" % script
+    proc = subprocess.Popen(args,
+                            stdin=subprocess.PIPE,
+                            env=os.environ.copy())
+    proc.communicate(input=script.encode('utf-8'))
+    if proc.returncode:
+        raise subprocess.CalledProcessError(proc.returncode, ' '.join(args))
+
+
+def guestfish_zerofree(filename):
+    """Fill free space with zero"""
+    logger.info(guestfish_zerofree.__doc__)
+    cmd = "virt-filesystems -a %s" % filename
+    fs = subprocess.check_output(cmd.encode('utf-8'),
+                                 stderr=subprocess.STDOUT,
+                                 shell=True,
+                                 env=os.environ.copy())
+    list_fs = fs.decode('utf-8').split()
+    logger.info('\n'.join(('  `--> %s' % i for i in list_fs)))
+    script = '\n'.join(('zerofree %s' % i for i in list_fs))
+    run_guestfish_script(filename, script, mount="read_only")
+
+
+def convert_disk_image(args):
+    """Convert disk to another format."""
+    filename = op.abspath(args.file.name)
+    output = op.abspath(args.output)
+
+    os.environ['LIBGUESTFS_CACHEDIR'] = os.getcwd()
+    if args.verbose:
+        os.environ['LIBGUESTFS_DEBUG'] = '1'
+
+    # sometimes guestfish fails because of other virtualization tools are
+    # still running use a test and retry to wait for availability
+    # attempts = 0
+    # while attempts < 3:
+    #    try:
+    #        logger.info("Waiting for virtualisation to be available...")
+    #        run_guestfish_script(filename, "cat /etc/hostname", mount='ro')
+    #        break
+    #    except:
+    #        attempts += 1
+    #        time.sleep(1)
+
+    if args.zerofree:
+        guestfish_zerofree(filename)
+
+    for fmt in args.formats:
+        if fmt in (tar_formats + disk_formats):
+            output_filename = "%s.%s" % (output, fmt)
+            if output_filename == filename:
+                continue
+            logger.info("Creating %s" % output_filename)
+            try:
+                if fmt in tar_formats:
+                    tar_convert(filename, output_filename,
+                                args.tar_excludes,
+                                args.tar_compression_level)
+                else:
+                    qemu_convert(filename, fmt, output_filename)
+            except ValueError as exp:
+                logger.error("Error: %s" % exp)
+
+
+if __name__ == '__main__':
+    allowed_formats = tar_formats + disk_formats
+    allowed_formats_help = 'Allowed values are ' + ', '.join(allowed_formats)
+
+    allowed_levels = ["%d" % i for i in range(1, 10)] + ["best", "fast"]
+    allowed_levels_helps = 'Allowed values are ' + ', '.join(allowed_levels)
+
+    parser = argparse.ArgumentParser(
+        description=sys.modules[__name__].__doc__,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument('file', action="store", type=argparse.FileType('r'),
+                        help='Disk image filename')
+    parser.add_argument('-F', '--formats', action="store", type=str, nargs='+',
+                        help='Output format. ' + allowed_formats_help,
+                        choices=allowed_formats, metavar='fmt', required=True)
+    parser.add_argument('-o', '--output', action="store", type=str,
+                        help='Output filename (without file extension)',
+                        required=True, metavar='filename')
+    parser.add_argument('--tar-compression-level', action="store", type=str,
+                        default="9", choices=allowed_levels, metavar='lvl',
+                        help="Compression level. " + allowed_levels_helps)
+    parser.add_argument('--tar-excludes', action="store", type=str, nargs='+',
+                        help="Files to excluded from archive",
+                        metavar='pattern', default=[])
+    parser.add_argument('--zerofree', action="store_true", default=False,
+                        help='Zero free unallocated blocks from ext2/3 '
+                             'file-systems before export to reduce image size')
+    parser.add_argument('--verbose', action="store_true", default=False,
+                        help='Enable very verbose messages')
+    log_format = '%(levelname)s: %(message)s'
+    level = logging.INFO
+    args = parser.parse_args()
+    if args.verbose:
+        level = logging.DEBUG
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setLevel(level)
+    handler.setFormatter(logging.Formatter(log_format))
+
+    logger.setLevel(level)
+    logger.addHandler(handler)
+
+    convert_disk_image(args)

kameleon/kameleon-recipes/steps/data/helpers/kaenv-customize.py

+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""Kadeploy helper functions"""
+
+import re
+import sys
+import argparse
+import logging
+import yaml
+from execo import Process, SshProcess, Host
+from pprint import pprint
+
+class Environment:
+    desc = None
+    def __init__(self):
+        """initialize an Environment object"""
+        
+    def import_from_kaenv(self, env, remote = None):
+        """import a kadeploy environment from	a kadeploy database"""
+        env = re.match(r"^(?P<name>[-_.\w]+)(?:@(?P<user>[_.\w]+))?(:?:(?P<version>[_.\w]+))?$", env).groupdict("")
+        if env['user']:
+            env['user'] = " -u " + env['user']
+        if env['version']:
+            env['version'] = " --env-version " + env['version']
+        kaenv_cmd = "kaenv3{user}{version} -p {name}".format(**env)
+        if remote:
+            remote = re.match(r"^(?:(?P<user>[-_.\w]+)@)?(?P<address>[-_.\w]+)(?::(?P<port>\d{1,5}))?$", remote).groupdict()
+            p = SshProcess(kaenv_cmd, Host(**remote))
+        else:
+            p = Process(kaenv_cmd, shell = True)
+        p.run()
+        self.desc = yaml.load(p.stdout)
+        return self
+    
+    def modify(self, **kwargs):
+        for key, value in kwargs.items():
+            if key in ['author', 'description', 'filesystem', 'name', 'os', 'visibility']:
+                self.desc[key] = str(value)
+            elif key in ['destructive', 'multipart', 'visibility']:
+                self.desc[key] = bool(value)
+            elif key in ['partition_type', 'version']:
+                self.desc[key] = int(value)
+            else:
+                try:
+                    key1, key2 = key.split('_')
+                except Exception as exc:
+                    raise Exception("Unknown description field '{}'".format(key))
+                if ((key1 == 'boot' and key2 in ['initrd', 'kernel', 'kernel_params']) or
+                    (key1 == 'image' and key2 in ['compression', 'file', 'kind'])):
+                    self.desc[key1][key2] = value
+                elif (key1 == 'postinstalls' and key2[-1:].isdigit() and 
+                      key2[:-1] in ['archive', 'compression', 'script']):
+                    if self.desc[key1][int(key2[-1:])]:
+                      self.desc[key1][int(key2[-1:])][key2[:-1]] = value
+                    else:
+                        raise Exception("No postinstall index {}".format(key2[:-1]))
+                else:
+                    raise Exception("Unknown description field '{}'".format(key))
+        return self
+
+    def add_postinstall(self, archive, compression, script):
+        self.desc['postinstalls'].append({'archive': archive, 'compression': compression, 'script': script})
+        return self
+
+    def del_postinstall(self, index):
+        del(self.desv['postinstall'][index])
+        return self
+ 
+    def export(self):
+        print("---\n" + yaml.dump(self.desc, default_flow_style=False))
+		
+logger = logging.getLogger(__name__)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('environment', metavar="ENVIRONMENT", help='Environment to import')
+    parser.add_argument('--remote', metavar="REMOTE", help='Remote kadeploy frontend')
+    parser.add_argument('--modify', metavar="KEY=VALUE", action='append', type=lambda kv: kv.split("=",1), help='modify description')
+    parser.add_argument('--info', action="store_true", default=False, help='print info messages')
+    parser.add_argument('--debug', action="store_true", default=False, help='print debug messages')
+    args = parser.parse_args()
+
+    handler = logging.StreamHandler()
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
+        handler.setLevel(logging.DEBUG)
+    elif args.info:
+        logger.setLevel(logging.INFO)
+        handler.setLevel(logging.INFO)
+    else:
+        logger.setLevel(logging.WARNING)
+        handler.setLevel(logging.WARNING)
+    handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
+    logger.addHandler(handler)
+
+    try:
+        if any(len(kv) < 2 for kv in args.modify):
+            raise ValueError('--modify take an argument in the form "key=value"')
+        Environment().import_from_kaenv(args.environment, args.remote).modify(**dict(args.modify)).export()
+    except Exception as exc:
+        sys.stderr.write("Error:  {}\n".format(exc))
+        sys.exit(1)

kameleon/kameleon-recipes/steps/data/qemu-sendkeys.rb

+#!/usr/bin/env ruby
+# Translate a string to "sendkey" commands for QEMU.
+# Martin Vidner, MIT License
+
+# https://en.wikibooks.org/wiki/QEMU/Monitor#sendkey_keys
+# sendkey keys
+#
+# You can emulate keyboard events through sendkey command. The syntax is: sendkey keys. To get a list of keys, type sendkey [tab]. Examples:
+#
+#     sendkey a
+#     sendkey shift-a
+#     sendkey ctrl-u
+#     sendkey ctrl-alt-f1
+#
+# As of QEMU 0.12.5 there are:
+# shift     shift_r     alt     alt_r   altgr   altgr_r
+# ctrl  ctrl_r  menu    esc     1   2
+# 3     4   5   6   7   8
+# 9     0   minus   equal   backspace   tab
+# q     w   e   r   t   y
+# u     i   o   p   ret     a
+# s     d   f   g   h   j
+# k     l   z   x   c   v
+# b     n   m   comma   dot     slash
+# asterisk  spc     caps_lock   f1  f2  f3
+# f4    f5  f6  f7  f8  f9
+# f10   num_lock    scroll_lock     kp_divide   kp_multiply     kp_subtract
+# kp_add    kp_enter    kp_decimal  sysrq   kp_0    kp_1
+# kp_2  kp_3    kp_4    kp_5    kp_6    kp_7
+# kp_8  kp_9    <   f11     f12     print
+# home  pgup    pgdn    end     left    up
+# down  right   insert  delete
+
+require "optparse"
+
+# incomplete! only what I need now.
+KEYS = {
+  "%" => "shift-5",
+  "/" => "slash",
+  ":" => "shift-semicolon",
+  "=" => "equal",
+  "." => "dot",
+  " " => "spc",
+  "-" => "minus",
+  "_" => "shift-minus",
+  "*" => "asterisk",
+  "," => "comma",
+  "+" => "shift-equal",
+  "|" => "shift-backslash",
+  "\\" => "backslash",
+}
+
+class Main
+  attr_accessor :command
+  attr_accessor :delay_s
+  attr_accessor :keystring
+
+  def initialize
+    self.command = nil
+    self.delay_s = 0.1
+
+    OptionParser.new do |opts|
+      opts.banner = "Usage: sendkeys [-c command_to_pipe_to] STRING\n" +
+        "Where STRING can be 'ls<enter>ls<gt>/dev/null<enter>'"
+
+      opts.on("-c", "--command COMMAND",
+              "Pipe sendkeys to this commands, individually") do |v|
+        self.command = v
+      end
+      opts.on("-d", "--delay SECONDS", Float,
+              "Delay SECONDS after each key (default: 0.1)") do |v|
+        self.delay_s = v
+      end
+    end.parse!
+    self.keystring = ARGV[0]
+  end
+
+  def sendkey(qemu_key_name)
+    if qemu_key_name == "wait"
+      sleep 1
+    else
+      if qemu_key_name =~ /[A-Za-z]/ && qemu_key_name == qemu_key_name.upcase
+        key = "shift-#{qemu_key_name.downcase}"
+      else
+        key = qemu_key_name
+      end
+      qemu_cmd = "sendkey #{key}"
+      if command
+        system "echo '#{qemu_cmd}' | #{command}"
+      else
+        puts qemu_cmd
+        $stdout.flush             # important when we are piped
+      end
+      sleep delay_s
+    end
+  end
+
+  PATTERN = /
+              \G  # where last match ended
+              < [^>]+ >
+            |
+              \G
+              .
+            /x
+  def run
+    keystring.scan(PATTERN) do |match|
+      if match[0] == "<"
+        key_name = match.slice(1..-2)
+        sendkey case key_name
+                when "lt" then "shift-comma"
+                when "gt" then "shift-dot"
+                else key_name
+                end
+      else
+        sendkey KEYS.fetch(match, match)
+      end
+    end
+  end
+end
+
+Main.new.run

kameleon/kameleon-recipes/steps/disable_checkpoint.yaml

+- disable_checkpoint:
+  - on_checkpoint: redo
+  - exec_local: rm -f $${kameleon_cwd}/checkpoint_enabled