diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f11592437a95f518602a344441532fd6d5473802..e197afc9190b5b657f7aae31968f34210fcc4b22 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,53 +1,55 @@
-# Official language image, in version 3.8(.latest)
-image: python:3.8
-
-# Set up a virtual environment.
-before_script:
- - python -m venv venv
- - source venv/bin/activate
-
-# Run the test suite using tox.
-# This job is called when commits are pushed to the main or a release branch.
-test:
- script:
+# Shared configuration elements between test jobs.
+.config:
+ # Official Python language image, in version 3.8(.latest)
+ image: python:3.8
+ # Set up a virtual environment, with latest pip and tox installed.
+ before_script:
+ - python -m venv venv
+ - source venv/bin/activate
+ - pip install -U pip
- pip install -U tox
- - tox -e py38
+ # Configure coverage export and collection.
artifacts:
reports:
coverage_report:
coverage_format: cobertura
path: coverage.xml
coverage: /(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
- rules:
- - if: ($CI_PIPELINE_SOURCE == "push") &&
- (($CI_COMMIT_BRANCH == "develop") || ($CI_COMMIT_BRANCH =~ "/^r\d\.\d+/"))
+ # Run on a shared Gitlab CI Runner.
tags:
- ci.inria.fr
- - small
+ - medium
-# Run the test suite using tox, with --fulltest option.
-# This job is called on creation and pushes to non-Draft MRs.
-# It may also be launched manually upon pushes to Draft MRs or main/dev branch.
-test-full:
+# Basic test suite: skip GPU use and a few extra integration tests scenarios.
+# This job is called when commits are pushed to a non-draft MR branch.
+# It may also be called manually on commits to draft MR branches.
+test-minimal:
+ extends:
+ .config
script:
- - pip install -U tox
- - tox -e py38 -- --fulltest
- artifacts:
- reports:
- coverage_report:
- coverage_format: cobertura
- path: coverage.xml
- coverage: /(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
+ - tox -e py38 -- --cpu-only
rules:
- if: ($CI_PIPELINE_SOURCE == "merge_request_event") &&
($CI_MERGE_REQUEST_TITLE !~ /^Draft:.*/)
- if: ($CI_PIPELINE_SOURCE == "merge_request_event") &&
($CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/)
when: manual
+ allow_failure: true
+
+# Exhaustive test suite: lint the code, run all tests, use GPU when available.
+# This job is called when commits are pushed to a main branch.
+# It may also be called manually on commits to MR branches.
+test-maximal:
+ extends:
+ .config
+ script:
+ - tox -e py38 -- --fulltest
+ rules:
- if: ($CI_PIPELINE_SOURCE == "push") &&
- (($CI_COMMIT_BRANCH == "develop") || ($CI_COMMIT_BRANCH =~ "/^r\d\.\d+/"))
+ (
+ ($CI_COMMIT_BRANCH == "develop") ||
+ ($CI_COMMIT_BRANCH =~ "/^r\d\.\d+/")
+ )
+ - if: ($CI_PIPELINE_SOURCE == "merge_request_event")
when: manual
- allow_failure: true # do not block the base "test" pipeline
- tags:
- - ci.inria.fr
- - small
+ allow_failure: true
diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
new file mode 100644
index 0000000000000000000000000000000000000000..189986b730d1ee3e75ccd723bc49697598371879
--- /dev/null
+++ b/scripts/run_tests.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+
+# Copyright 2023 Inria (Institut National de Recherche en Informatique
+# et Automatique)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+: '
+Bash script to launch test pipelines for declearn.
+'
+
+run_command(){
+ : '
+ Run a given command, echoing it and being verbose about success/failure.
+
+ Return:
+ 1 if the command failed, 0 if it was successful.
+ '
+ cmd=$@
+ name="\e[34m$(echo $cmd | cut -d' ' -f1)\e[0m"
+ echo -e "\nRunning command: \e[34m$cmd\e[0m"
+ if $cmd; then
+ echo -e "$name command was \e[32msuccessful\e[0m"
+ return 0
+ else
+ echo -e "$name command \e[31mfailed\e[0m"
+ return 1
+ fi
+}
+
+
+run_commands() {
+ : '
+ Verbosely run a set of commands and report whether all were a success.
+
+ Syntax:
+ run_commands "context string" ("command")+
+
+ Return:
+ The number of sub-commands that failed (0 if successful).
+ '
+ context=$1; shift; commands=( "$@" )
+ n_cmd=${#commands[@]}
+ declare -i failed
+ echo "Running $context ($n_cmd commands)."
+ for ((i = 0; i < $n_cmd ; i++)); do
+ run_command ${commands[i]}
+ if [[ $? -ne 0 ]]; then failed+=1; fi
+ done
+ if [[ $failed -eq 0 ]]; then
+ echo -e "\n\e[32mAll $n_cmd $context commands were successful.\e[0m"
+ else
+ echo -e "\n\e[31m$failed/$n_cmd $context commands failed.\e[0m"
+ fi
+ return $failed
+}
+
+
+lint_declearn_code() {
+ : '
+ Verbosely run linters on the declearn package source code.
+
+ Return:
+ The number of sub-commands that failed (0 if successful).
+ '
+ commands=(
+ "pylint declearn"
+ "mypy --install-types --non-interactive declearn"
+ "black --check declearn"
+ )
+ run_commands "declearn code static analysis" "${commands[@]}"
+}
+
+
+lint_declearn_tests() {
+ : '
+ Verbosely run linters on the declearn test suite source code.
+
+ Return:
+ The number of sub-commands that failed (0 if successful).
+ '
+ commands=(
+ "pylint --recursive=y test"
+ "mypy --install-types --non-interactive --exclude=conftest.py declearn"
+ "black --check test"
+ )
+ run_commands "declearn test code static analysis" "${commands[@]}"
+}
+
+
+run_declearn_tests() {
+ : '
+ Verbosely run the declearn test suite and export coverage results.
+
+ Syntax:
+ run_tests [optional_pytest_flags]
+
+ Return:
+ The number of sub-commands that failed (0 if successful).
+ '
+ # Remove any pre-existing coverage file.
+ if [[ -f .coverage ]]; then rm .coverage; fi
+ # Run the various sets of tests.
+ commands=(
+ "run_unit_tests $@"
+ "run_integration_tests $@"
+ )
+ run_commands "declearn test suite" "${commands[@]}"
+ status=$?
+ # Display and export the cumulated coverage.
+ coverage report
+ coverage xml
+ # Return the success/failure status of tests.
+ return $status
+}
+
+
+run_unit_tests() {
+ : '
+ Verbosely run the declearn unit tests (excluding integration ones).
+ '
+ echo "Running DecLearn unit tests."
+ command="pytest $@
+ --cov --cov-append --cov-report=
+ --ignore=test/functional/
+ test
+ "
+ run_command $command
+}
+
+
+run_integration_tests() {
+ : '
+ Verbosely run the declearn integration tests (skipping unit ones).
+ '
+ echo "Running DecLearn integration tests."
+ command="pytest $@
+ --cov --cov-append --cov-report=
+ test/functional/
+ "
+ run_command $command
+}
+
+
+main() {
+ if [[ $# -eq 0 ]]; then
+ echo "Missing required positional argument."
+ echo "Usage: {lint_code|lint_tests|run_tests} [PYTEST_FLAGS]*"
+ exit 1
+ fi
+ action=$1; shift
+ case $action in
+ lc | lint_code)
+ lint_declearn_code
+ exit $?
+ ;;
+ lt | lint_tests)
+ lint_declearn_tests
+ exit $?
+ ;;
+ rt | run_tests)
+ run_declearn_tests "$@"
+ exit $?
+ ;;
+ *)
+ echo "Usage: {lint_code|lint_tests|run_tests} [PYTEST_FLAGS]*"
+ echo "Aliases for the action parameter values: {lc|lt|rt}."
+ exit 1
+ ;;
+ esac
+}
+
+
+main $@
diff --git a/tox.ini b/tox.ini
index 1e962ca49d9252c275ee526d2c4860cb580c25d7..a66fd5864cc019f59c9fe79f4c9dfe1728b7628a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -2,32 +2,52 @@
envlist = py38
minversion = 3.18.0
-[testenv]
-recreate = True
+
+[testenv:py{38,39,310,311,312}]
+description =
+ "Default job, running all kinds of tests."
extras =
all,tests
allowlist_externals =
- openssl
+ bash, openssl
commands=
- # run unit tests first
- pytest {posargs} \
- --cov --cov-report= \ # reset then accumulate coverage quietly
- --ignore=test/functional/ \
- test
- # verify code acceptance by pylint
- pylint declearn
- pylint --recursive=y test
- # verify code typing using mypy
- mypy --install-types --non-interactive declearn
- # verify code formatting
- black --check declearn
- black --check test
- # run functional ~ integration tests (that build on unit ones)
- pytest {posargs} \
- --cov --cov-append --cov-report=term \ # acc. and display coverage
- test/functional/
- # export the finalized coverage report to xml
- coverage xml
+ bash scripts/run_tests.sh lint_code
+ bash scripts/run_tests.sh lint_tests
+ bash scripts/run_tests.sh run_tests {posargs}
+
+
+[testenv:py{38,39,310,311,312}-tests]
+description =
+ "Individualized job to run unit and integration tests."
+extras =
+ all,tests
+allowlist_externals =
+ bash, openssl
+commands=
+ bash scripts/run_tests.sh run_tests {posargs}
+
+
+[testenv:py{38,39,310,311,312}-lint_code]
+description =
+ "Individualized job to run source code static analysis."
+extras =
+ all,tests
+allowlist_externals =
+ bash
+commands =
+ bash scripts/run_tests.sh lint_code
+
+
+[testenv:py{38,39,310,311,312}-lint_tests]
+description =
+ "Individualized job to run test code static analysis."
+extras =
+ all,tests
+allowlist_externals =
+ bash
+commands =
+ bash scripts/run_tests.sh lint_tests
+
[pytest]
addopts = --full-trace