before_script: - make demo - make data - make germline - make -C browser sha1 - cp -p doc/analysis-example.vidjil browser/ stages: - test_tools - test_quality - test_unit - deploy_review - test_germlines - test_functional - test_shouldvdj - test_functional_external - coverage - prepare_release - tgz_release - valgrind_unit - valgrind_functional - multiple_tests - benchmark - publish_release - deploy_prod # Anchors .coverage_dependency: artifacts: &lcov_artifact paths: - algo/lcov_test_*.filtered expire_in: 1 week # Tools test_tools: stage: test_tools script: make -C tools/tests # Germlines test_germlines: stage: test_germlines script: - make -C germline get-all-data - make -C germline tests only: - /^feature-.*g.*\/.*$/ - schedules # Doc include: - local: '/doc/.gitlab-ci.yml' - local: 'algo/.gitlab-ci-compilers.yml' # Stage multiple_tests # Algorithm test_algo_unit: stage: test_unit script: make -C algo unit_coverage artifacts: <<: *lcov_artifact only: - /^feature-.*a.*\/.*$/ - schedules test_algo_should: stage: test_functional script: make -C algo should_coverage artifacts: <<: *lcov_artifact reports: junit: algo/tests/should.xml only: - /^feature-.*a.*\/.*$/ - schedules test_algo_shouldvdj: stage: test_shouldvdj script: make -C algo shouldvdj_coverage artifacts: <<: *lcov_artifact only: - /^feature-.*a.*\/.*$/ - schedules algo_coverage: stage: coverage coverage: /^\s*lines\.*:\s+([0-9.]+\%)/ script: - make -C algo lcov_reports - mv algo/reports/ coverage artifacts: paths: - coverage/ expire_in: 1 week only: - /^feature-.*a.*\/.*$/ - schedules algo_valgrind_unit: stage: valgrind_unit script: make -C algo valgrind_unit only: - /^feature-.*a.*\/.*$/ - schedules tags: - valgrind .snip_valgrind_functional: &valgrind_functional stage: valgrind_functional script: make -C algo valgrind_should tags: - valgrind algo_valgrind_functional: <<: *valgrind_functional when: manual only: - /^feature-.*a.*\/.*$/ - schedules release_valgrind_functional: <<: *valgrind_functional only: - /^feature-.*a.*\/release$/ prepare_release: stage: prepare_release script: make -C algo release RELEASE_TAG='rc' when: manual only: - /^feature-.*a.*\/.*$/ tgz_release: stage: tgz_release script: make -C algo release RELEASE_TAG=`cat algo/release` only: - /^feature-.*a.*\/release$/ artifacts: paths: - algo/releases/vidjil-algo-`cat algo/release`.tar.gz - algo/releases/vidjil-algo-`cat algo/release`/vidjil-algo-`cat algo/release`_`uname -m` expire_in: 10 years allow_failure: false tags: - ubuntu-16.04-amd64 copy_release: stage: publish_release script: - | for repo in $ALGO_REPOSITORIES; do echo "Copying release to $repo" scp algo/releases/vidjil-algo*.tar.gz algo/releases/vidjil-algo*/vidjil-algo-*_* $repo done release_name=$(cat algo/release) ssh $VIDJIL_WWW "cd /var/www/html/releases; ln -sf vidjil-algo-${release_name}.tar.gz vidjil-latest.tgz; ln -sf vidjil-algo-${release_name}_x86_64 vidjil-latest_x86_64" ssh $VIDJIL_BONSAI "cd /bio1/www/html/vidjil; scp vidjil-algo-${release_name}* $VIDJIL_BONSAI_PROD:/bio1/www/html/vidjil" when: manual only: - /^feature-.*a.*\/release$/ deploy_release_prod: stage: deploy_prod when: manual only: - /^feature-.*a.*\/release$/ script: - | release_name=$(cat algo/release) ssh $DEPLOY_SERVER "cd releases/vidjil;\ tar xvzf vidjil-algo-${release_name}.tar.gz;\ make -C vidjil-algo-${release_name};\ ln -sfT vidjil-algo-${release_name} next" # Client test_browser_unit: stage: test_unit script: make unit_browser artifacts: paths: - browser/ only: - /^feature-.*c.*\/.*$/ - /^hotfix-.*c.*\/.*$/ - prod-client - dev - schedules tags: - web .browser-functional: stage: test_functional retry: 2 script: - make -C browser - source /etc/profile.d/rvm.sh - rvm use 2.6.1 - HEADLESS=1 make -C browser/test functional artifacts: paths: - browser/ reports: junit: browser/test/test/TEST-*.xml only: - /^feature-.*c.*\/.*$/ - /^hotfix-.*c.*\/.*$/ - prod-client - schedules tags: - web .browser-functional-chrome: extends: .browser-functional variables: WATIR_CHROME: "1" .browser-functional-external: stage: test_functional_external retry: 2 script: - make -C browser - source /etc/profile.d/rvm.sh - rvm use 2.6.1 - HEADLESS=1 make -C browser/test external artifacts: paths: - browser/ reports: junit: browser/test/test/TEST-*.xml only: - /^feature-.*c.*\/.*$/ - /^hotfix-.*c.*\/.*$/ - prod-client - schedules tags: - web .browser-functional-external-chrome: extends: .browser-functional-external variables: WATIR_CHROME: "1" ff32-browser-functional: extends: .browser-functional variables: FIREFOX_VERSION: 32 ff45-browser-functional: extends: .browser-functional variables: FIREFOX_VERSION: 45 chrome-browser-functional: extends: .browser-functional-chrome old-chrome-browser-functional: extends: .browser-functional-chrome tags: - legacy ff32-browser-functional-external: extends: .browser-functional-external variables: FIREFOX_VERSION: 32 ff45-browser-functional-external: extends: .browser-functional-external variables: FIREFOX_VERSION: 45 chrome-browser-functional-external: extends: .browser-functional-external-chrome old-chrome-browser-functional-external-chrome: extends: .browser-functional-external-chrome tags: - legacy code_quality: stage: test_quality script: make -C browser quality only: - /^feature-.*c.*\/.*$/ - /^hotfix-.*c.*\/.*$/ - prod-client - dev - schedules tags: - web # Server test_server_unit: stage: test_unit script: - virtualenv $CI_BUILD_REF_SLUG - source $CI_BUILD_REF_SLUG/bin/activate - pip install -r requirements.txt - make -C server install_web2py_standalone - make -C server launch_fuse_server - make unit_server || (make -C server kill_fuse_server; deactivate; false) - make -C server kill_fuse_server - deactivate artifacts: reports: junit: server/web2py/test-reports/TEST*xml only: - /^feature-.*s.*\/.*$/ - /^hotfix-.*s.*\/.*$/ - prod-server - dev - schedules tags: - web .test_server_functional: stage: test_functional script: - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client - sed -i '/\:\/etc\/nginx\/ssl/d' ./docker/docker-compose.yml - sed -i '/\/var\/lib\/mysql/d' ./docker/docker-compose.yml - sed -i 's/\/opt\/vidjil\/databases/databases/' ./docker/docker-compose.yml - echo -e "volumes:\n databases:" >> ./docker/docker-compose.yml - sed -i 's/\:latest/\:test/g' ./docker/docker-compose.yml - cd docker/vidjil-server/conf/ && mv defs.py defs_https.py && mv defs_http.py defs.py && cd ../../.. - make germline && cp browser/js/germline.js docker/vidjil-client/conf - mkdir -p docker/postfix/ssl - cd docker && docker-compose up -d && cd .. - sed -i "s/^python\ \.\.\/\.\.\/\.\./docker\ exec\ docker_uwsgi_1\ python\ \/usr\/share\/vidjil\/server\/web2py/" server/web2py/applications/vidjil/tests/init_tests.sh - docker exec docker_uwsgi_1 sed -i "s/^\(FILE_SOURCE .*\)/FILE_SOURCE = '\/usr\/share\/vidjil\/demo'/" /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py - docker exec docker_nginx_1 make -C /usr/share/vidjil browser - source /etc/profile.d/rvm.sh - rvm use 2.6.1 - HEADLESS=1 make functional_server after_script: - cd docker && docker-compose down -v - docker rmi "vidjil/server:test" "vidjil/client:test" reporter:test artifacts: reports: junit: server/web2py/applications/vidjil/tests/test/TEST-*.xml only: - /^feature-.*s.*\/.*$/ - /^hotfix-.*s.*\/.*$/ - prod-server - dev - schedules tags: - x86_64 - docker ff32-server-functional: extends: .test_server_functional variables: FIREFOX_VERSION: 32 ff45-server-functional: extends: .test_server_functional variables: FIREFOX_VERSION: 45 # Benchmark benchmark_algo: image: gcc:6.3 before_script: - apt-get update - apt-get install -y time valgrind python3 wget tar stage: benchmark script: - cd algo/tests ; python3 benchmark-releases.py -r 3 -bIc artifacts: paths: - algo/tests/benchmark.log when: manual only: - /^feature-.*a.*\/.*$/ tags: - cidocker # Deployment deploy_review: stage: deploy_review script: - make -C browser icons - echo "$REVIEW_CONFIG_JS" > browser/js/conf.js - rsync -av --delete browser $REVIEW_SERVER:$REVIEW_SERVER_PATH$CI_BUILD_REF_SLUG environment: name: review/$CI_BUILD_REF_NAME url: http://$CI_BUILD_REF_SLUG.ci.vidjil.org/?data=analysis-example.vidjil on_stop: stop_deploy_review only: - /^feature-.*c.*\/.*$/ - /^hotfix-.*c.*\/.*$/ tags: - web deploy_review_server: stage: deploy_review script: - make -C browser icons - cp ./docker/vidjil-client/conf/conf.js ./browser/js/conf.js - sed -i -r "s/localhost/${CI_BUILD_REF_SLUG}.server.ci.vidjil.org/g" ./browser/js/conf.js - sed -r 's/^.*FILE_SOURCE.*$/FILE_SOURCE="\/usr\/share\/vidjil\/demo"/' ./docker/vidjil-server/conf/defs.py > ./server/web2py/applications/vidjil/modules/defs.py - sed -i -e "/EMAIL_SUBJECT_START/c EMAIL_SUBJECT_START='[Vidjil/CI][$CI_COMMIT_REF_NAME]'" -e '/FROM_EMAIL/c FROM_EMAIL="server.ci@vidjil.org"' ./server/web2py/applications/vidjil/modules/defs.py - mv docker docker_${CI_BUILD_REF_SLUG} - cd docker_${CI_BUILD_REF_SLUG} - TMP_DOCKER=$(tempfile) - cp docker-compose.yml $TMP_DOCKER - awk -f ci/for-review.awk $TMP_DOCKER > docker-compose.yml - rm -f $TMP_DOCKER - sed -i -r "s/WEB2PY_ADMIN_PASSWORD=/WEB2PY_ADMIN_PASSWORD=$REVIEW_SERVER_ADMIN_PASSWORD/" docker-compose.yml - cd .. - rsync -av --exclude .git . $REVIEW_SERVER:$REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG || $(case "$?" in 0|23) exit 0 ;; *) exit $?; esac) - ssh $REVIEW_SERVER "cd $REVIEW_SERVER_PATH_SERVER; ./install_certs.sh ${CI_BUILD_REF_SLUG}; cd ${CI_BUILD_REF_SLUG}/docker_${CI_BUILD_REF_SLUG}; mkdir -p postfix/ssl; docker-compose up -d mysql fuse uwsgi nginx workers" environment: name: review_server/$CI_BUILD_REF_NAME url: http://$CI_BUILD_REF_SLUG.server.ci.vidjil.org/browser/ on_stop: stop_deploy_review_server auto_stop_in: 2 months only: - /^feature-.*s.*\/.*$/ - /^hotfix-.*s.*\/.*$/ deploy_docker: stage: deploy_prod script: - export CUR_DATE=`date +%Y-%m-%d` - export TYPE=`echo $CI_COMMIT_REF_NAME | awk '{ gsub(/prod-/, "") }{ print $1 }'` - export SHA=`echo $CI_COMMIT_SHA | awk '{ print substr($0, 0, 8) }'` - export TAG="vidjil/$TYPE:$CUR_DATE-$SHA" - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=PRODUCTION -t $TAG docker/vidjil-$TYPE - docker push $TAG - docker tag $TAG "vidjil/$TYPE:latest" - docker push "vidjil/$TYPE:latest" - docker rmi $TAG "vidjil/$TYPE:latest" when: manual only: - prod-server - prod-client tags: - web - docker deploy_prod: stage: deploy_prod script: - ssh $PROD_CLIENT_SERVER " cd $PROD_CLIENT_PATH; git fetch && git reset --hard origin/prod-client && make -C browser && make -C browser sha1 && cp doc/analysis-example.vidjil browser/" environment: name: production url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil only: - prod-client tags: - web deploy_germlines: stage: deploy_prod script: - make -C germline get-all-data - make -C germline js - date=$(date +%Y-%m-%d) - tar cvzf germline-$date.tar.gz germline/*/*.fa germline/IMGT_RELEASE browser/js/germline.js - scp germline-$date.tar.gz $VIDJIL_WWW:$REMOTE_GERMLINE_DIR only: - manual - /^feature-.*g.*\/.*$/ stop_deploy_review: stage: deploy_review variables: GIT_STRATEGY: none script: - ssh $REVIEW_SERVER "rm -rf $REVIEW_SERVER_PATH$CI_BUILD_REF_SLUG" when: manual environment: name: review/$CI_BUILD_REF_NAME action: stop only: - /^feature-.*c.*\/.*$/ - /^hotfix-.*c.*\/.*$/ tags: - web stop_deploy_review_server: stage: deploy_review script: - ssh $REVIEW_SERVER "set -o xtrace; docker-compose -f $REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG/docker_$CI_BUILD_REF_SLUG/docker-compose.yml down -v ; rm -rf $REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG; $REVIEW_SERVER_PATH_SERVER/uninstall_certs.sh $CI_BUILD_REF_SLUG" when: manual environment: name: review_server/$CI_BUILD_REF_NAME action: stop only: - /^feature-.*s.*\/.*$/ - /^hotfix-.*s.*\/.*$/