From ce612d5113a79e622c13e1180d1c2a3b48acabaf Mon Sep 17 00:00:00 2001 From: PRUVOST Florent <florent.pruvost@inria.fr> Date: Thu, 9 Mar 2023 07:40:33 +0000 Subject: [PATCH] Update gitlab-ci (factorize), fix coverage report, and sonarqube parameters --- .gitlab-ci.yml | 3 -- .gitlab/build.yml | 56 ++++++--------------------------- .gitlab/common.yml | 51 +++++++++++++++++++----------- .gitlab/coverage.yml | 17 +++++++--- .gitlab/coverity.yml | 11 +++---- .gitlab/pages.yml | 4 +-- .gitlab/preliminary.yml | 2 +- .gitlab/release.yml | 3 +- .gitlab/sonarqube.yml | 27 +++++++++------- .gitlab/test_hmat.yml | 12 +------ .gitlab/test_openmp.yml | 12 +------ .gitlab/test_parsec.yml | 12 +------ .gitlab/test_quark.yml | 12 +------ .gitlab/test_starpu.yml | 30 ++++++------------ .gitlab/test_starpu_simgrid.yml | 12 +------ tools/analysis.sh | 32 ++++++++++--------- tools/build_macosx.sh | 2 +- tools/coverage.sh | 2 ++ tools/find_sources.sh | 4 +++ 19 files changed, 118 insertions(+), 186 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90eadbeb6..b8567fc1f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,6 +34,3 @@ include: - .gitlab/bench_plafrim.yml - .gitlab/pages.yml - .gitlab/release.yml - - - diff --git a/.gitlab/build.yml b/.gitlab/build.yml index 163ce7258..ad45a43e9 100644 --- a/.gitlab/build.yml +++ b/.gitlab/build.yml @@ -1,60 +1,36 @@ --- +build_hmat: + extends: .build_script_template + variables: + BUILD_OPTIONS: "-DCHAMELEON_USE_CUDA=OFF -DCHAMELEON_USE_MPI=ON -DCHAMELEON_USE_HMAT=ON" + VERSION: hmat + build_openmp: extends: .build_script_template - artifacts: - name: build_openmp - expire_in: 48 hours - paths: - - build-openmp - - chameleon_openmp.log variables: BUILD_OPTIONS: "-DCHAMELEON_SCHED=OPENMP" VERSION: openmp build_parsec: extends: .build_script_template - artifacts: - name: build_parsec - expire_in: 48 hours - paths: - - build-parsec - - chameleon_parsec.log variables: BUILD_OPTIONS: "-DCHAMELEON_SCHED=PARSEC" VERSION: parsec build_quark: extends: .build_script_template - artifacts: - name: build_quark - expire_in: 48 hours - paths: - - build-quark - - chameleon_quark.log variables: BUILD_OPTIONS: "-DCHAMELEON_SCHED=QUARK" VERSION: quark build_starpu: extends: .build_script_template - artifacts: - name: build_starpu - expire_in: 48 hours - paths: - - build-starpu - - chameleon_starpu.log variables: BUILD_OPTIONS: "-DCHAMELEON_USE_CUDA=ON -DCHAMELEON_USE_MPI=ON" VERSION: starpu build_starpu_simgrid: extends: .build_script_template - artifacts: - name: build_starpu_simgrid - expire_in: 48 hours - paths: - - build-starpu_simgrid - - chameleon_starpu_simgrid.log variables: SIMU: simu BUILD_OPTIONS: "-DCHAMELEON_SIMULATION=ON -DCHAMELEON_USE_CUDA=ON -DCHAMELEON_USE_MPI=OFF" @@ -64,24 +40,12 @@ build_starpu_macosx: extends: .only-master stage: build tags: ['macosx'] - artifacts: - untracked: true - name: build_starpu_macosx - expire_in: 48 hours variables: VERSION: starpu script: - tools/build_macosx.sh - -build_hmat: - extends: .build_script_template artifacts: - name: build_hmat - expire_in: 48 hours - paths: - - build-hmat - - chameleon_hmat.log - variables: - BUILD_OPTIONS: "-DCHAMELEON_USE_CUDA=OFF -DCHAMELEON_USE_MPI=ON -DCHAMELEON_USE_HMAT=ON" - VERSION: hmat - + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + expire_in: 2 days + untracked: true + when: always diff --git a/.gitlab/common.yml b/.gitlab/common.yml index 2c597c65f..996dfa73b 100644 --- a/.gitlab/common.yml +++ b/.gitlab/common.yml @@ -2,29 +2,28 @@ # This file describes the common rules for the stages # --- -.only-master: +default: image: registry.gitlab.inria.fr/solverstack/docker/distrib + tags: ['ci.inria.fr', 'linux', 'large'] + +.only-master: + except: ['schedule'] interruptible: true only: - master@solverstack/chameleon - /^ci-.*$/ - except: - - schedules .only-branches: - image: registry.gitlab.inria.fr/solverstack/docker/distrib + except: ['schedule'] interruptible: true only: - merge_requests - master@solverstack/chameleon - /^ci-.*$/ - except: - - schedules .build_script_template: - extends: .only-branches stage: build - tags: ["linux"] + extends: .only-branches script: - export LOGNAME=chameleon_${VERSION}.log - echo $LOGNAME @@ -35,18 +34,20 @@ scan-build -plist --intercept-first --exclude CMakeFiles --analyze-headers -o analyzer_reports cmake -C ../cmake_modules/gitlab-ci-initial-cache.cmake .. $BUILD_OPTIONS && scan-build -plist --intercept-first --exclude CMakeFiles --analyze-headers -o analyzer_reports - ctest --no-compress-output -V -j 5 + ctest --no-compress-output -j 5 -V -T Build | tee ../${LOGNAME}) - (cd build-$VERSION && make install | tee -a ../${LOGNAME} && rm install/ -r) - except: - - schedules + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + expire_in: 2 days + untracked: true + when: always .test_script_template: stage: test - tags: ["linux"] script: - export LOGNAME=chameleon_${VERSION}_${CATEGORY}_${PRECISION} - echo $LOGNAME @@ -56,17 +57,29 @@ - echo test PRECISION $PRECISION | tee -a ${LOGNAME}.log - ls -l *.log - (cd build-$VERSION && - eval "ctest --no-compress-output -V + eval "ctest --no-compress-output $TESTS_RESTRICTION -T Test - -T Coverage + --output-junit ../${LOGNAME}.junit | tee -a ../${LOGNAME}.log") - - lcov --directory build-$VERSION --capture --output-file ./${LOGNAME}.lcov - - xsltproc -o report.xml /home/gitlab/ctest-to-junit.xsl build-$VERSION/Testing/**/Test.xml + - gcovr --xml-pretty --exclude-unreachable-branches --print-summary -o ${LOGNAME}.cov --root ${CI_PROJECT_DIR} + - lcov --directory build-$VERSION --capture --output-file ${LOGNAME}.lcov + - cp ${LOGNAME}.junit junit.xml + - cp ${LOGNAME}.cov coverage.xml + coverage: /^\s*lines:\s*\d+.\d+\%/ + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + expire_in: 2 days + untracked: true + when: always + reports: + coverage_report: + coverage_format: cobertura + path: coverage.xml + junit: junit.xml .bench_plafrim_common: - only: - - schedules + only: ['schedule'] stage: test tags: ['plafrim'] timeout: 1 week @@ -75,7 +88,7 @@ script: - ./tools/bench/plafrim/run.sh artifacts: - name: "$CI_JOB_NAME" + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" expire_in: 1 week paths: - "chameleon-$NODE-$MPI-$SLURM_NP.err" diff --git a/.gitlab/coverage.yml b/.gitlab/coverage.yml index 42cbf2a86..44d5576ae 100644 --- a/.gitlab/coverage.yml +++ b/.gitlab/coverage.yml @@ -6,10 +6,19 @@ # --- coverage: - image: registry.gitlab.inria.fr/solverstack/docker/distrib + only: + - merge_requests + interruptible: true stage: analyse - tags: ["docker"] script: - tools/coverage.sh - only: - - merge_requests + coverage: /^\s*lines:\s*\d+.\d+\%/ + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + expire_in: 2 days + untracked: true + when: always + reports: + coverage_report: + coverage_format: cobertura + path: coverage.xml diff --git a/.gitlab/coverity.yml b/.gitlab/coverity.yml index f9f7b7256..593d3e3f1 100644 --- a/.gitlab/coverity.yml +++ b/.gitlab/coverity.yml @@ -2,13 +2,7 @@ coverity: extends: .only-master stage: analyse - tags: ["linux"] dependencies: [] - artifacts: - name: chameleon_coverity - expire_in: 1 week - paths: - - chameleon.tgz variables: VERSION: coverity script: @@ -23,3 +17,8 @@ coverity: --form version="`git rev-parse --short HEAD`" --form description="" https://scan.coverity.com/builds?project=Chameleon + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + expire_in: 1 week + paths: + - chameleon.tgz diff --git a/.gitlab/pages.yml b/.gitlab/pages.yml index bf18172b2..41aec70ac 100644 --- a/.gitlab/pages.yml +++ b/.gitlab/pages.yml @@ -1,10 +1,9 @@ --- pages: - image: registry.gitlab.inria.fr/solverstack/docker/distrib stage: deploy dependencies: [] artifacts: - name: chameleon_doc + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" expire_in: 1 week paths: - public @@ -15,3 +14,4 @@ pages: only: - master@solverstack/chameleon - /^ci.*$/ + interruptible: true diff --git a/.gitlab/preliminary.yml b/.gitlab/preliminary.yml index 331c55b37..a1e137cb5 100644 --- a/.gitlab/preliminary.yml +++ b/.gitlab/preliminary.yml @@ -3,7 +3,6 @@ # --- preliminary_checks: - image: registry.gitlab.inria.fr/solverstack/docker/distrib stage: .pre parallel: matrix: @@ -12,4 +11,5 @@ preliminary_checks: - .gitlab/check_ci.sh $TEST only: - merge_requests + interruptible: true diff --git a/.gitlab/release.yml b/.gitlab/release.yml index c12bdd30a..5ed711f56 100644 --- a/.gitlab/release.yml +++ b/.gitlab/release.yml @@ -1,10 +1,9 @@ --- release: - image: registry.gitlab.inria.fr/solverstack/docker/distrib stage: deploy dependencies: [] artifacts: - name: chameleon_release + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" expire_in: 1 week paths: - chameleon-*.tar.gz diff --git a/.gitlab/sonarqube.yml b/.gitlab/sonarqube.yml index b4d5f8fd7..c8372797a 100644 --- a/.gitlab/sonarqube.yml +++ b/.gitlab/sonarqube.yml @@ -1,19 +1,24 @@ --- sonarqube: stage: analyse - tags: ["docker","linux"] - artifacts: - name: chameleon_analysis - expire_in: 1 week - paths: - - chameleon_coverage.xml - - chameleon_cppcheck.xml - - chameleon_rats.xml - - sonar.log + extends: + - .only-master variables: VERSION: sonarqube script: - cat *.log > chameleon_build.log - ./tools/analysis.sh - extends: - - .only-master + coverage: /^\s*lines:\s*\d+.\d+\%/ + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + expire_in: 2 days + when: always + paths: + - coverage.xml + - chameleon_coverage.xml + - chameleon_cppcheck.xml + - sonar.log + reports: + coverage_report: + coverage_format: cobertura + path: coverage.xml diff --git a/.gitlab/test_hmat.yml b/.gitlab/test_hmat.yml index 085778b7e..894dc50f1 100644 --- a/.gitlab/test_hmat.yml +++ b/.gitlab/test_hmat.yml @@ -2,17 +2,7 @@ .test_hmat_template: extends: .test_script_template needs: [build_hmat] - dependencies: - - build_hmat - artifacts: - name: test_hmat - expire_in: 48 hours - paths: - - build_hmat - - chameleon_hmat_*.log - - chameleon_hmat_*.lcov - reports: - junit: report.xml + dependencies: [build_hmat] .test_hmat_template_master: &test_hmat_master extends: diff --git a/.gitlab/test_openmp.yml b/.gitlab/test_openmp.yml index f48abad06..b026ae451 100644 --- a/.gitlab/test_openmp.yml +++ b/.gitlab/test_openmp.yml @@ -2,17 +2,7 @@ .test_openmp_template: &test_openmp extends: .test_script_template needs: [build_openmp] - dependencies: - - build_openmp - artifacts: - name: test_openmp - expire_in: 48 hours - paths: - - build_openmp - - chameleon_openmp_*.log - - chameleon_openmp_*.lcov - reports: - junit: report.xml + dependencies: [build_openmp] .test_openmp_template_master: &test_openmp_master extends: diff --git a/.gitlab/test_parsec.yml b/.gitlab/test_parsec.yml index 149856adc..ae137fafe 100644 --- a/.gitlab/test_parsec.yml +++ b/.gitlab/test_parsec.yml @@ -2,17 +2,7 @@ .test_parsec_template: &test_parsec extends: .test_script_template needs: [build_parsec] - dependencies: - - build_parsec - artifacts: - name: test_parsec - expire_in: 48 hours - paths: - - build_parsec - - chameleon_parsec_*.log - - chameleon_parsec_*.lcov - reports: - junit: report.xml + dependencies: [build_parsec] .test_parsec_template_master: &test_parsec_master extends: diff --git a/.gitlab/test_quark.yml b/.gitlab/test_quark.yml index 21c0e541a..98d27b947 100644 --- a/.gitlab/test_quark.yml +++ b/.gitlab/test_quark.yml @@ -2,17 +2,7 @@ .test_quark_template: &test_quark extends: .test_script_template needs: [build_quark] - dependencies: - - build_quark - artifacts: - name: test_quark - expire_in: 48 hours - paths: - - build_quark - - chameleon_quark_*.log - - chameleon_quark_*.lcov - reports: - junit: report.xml + dependencies: [build_quark] .test_quark_template_master: &test_quark_master extends: diff --git a/.gitlab/test_starpu.yml b/.gitlab/test_starpu.yml index e42561d49..45dd5bfb5 100644 --- a/.gitlab/test_starpu.yml +++ b/.gitlab/test_starpu.yml @@ -2,17 +2,7 @@ .test_starpu_template: extends: .test_script_template needs: [build_starpu] - dependencies: - - build_starpu - artifacts: - name: test_starpu - expire_in: 48 hours - paths: - - build_starpu - - chameleon_starpu_*.log - - chameleon_starpu_*.lcov - reports: - junit: report.xml + dependencies: [build_starpu] .test_starpu_template_master: &test_starpu_master extends: @@ -38,7 +28,7 @@ test_starpu_shm_d: VERSION: starpu CATEGORY: shm PRECISION: d - TESTS_RESTRICTION: "-R \"_${CATEGORY}_${PRECISION}\"" + TESTS_RESTRICTION: "-R \"_${CATEGORY}_${PRECISION}|example\"" test_starpu_shm_c: <<: *test_starpu_master @@ -93,15 +83,13 @@ test_starpu_shm_s_macosx: stage: test tags: ['macosx'] needs: [build_starpu_macosx] - dependencies: - - build_starpu_macosx - artifacts: - untracked: true - name: build_starpu_macosx - expire_in: 48 hours + dependencies: [build_starpu_macosx] variables: VERSION: starpu script: - - cd build-starpu && ctest -V -R test_shm_s - -T Test - -T Coverage + - cd build-starpu && ctest -R test_shm_s -T Test | tee -a ../chameleon_macosx.log + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + untracked: true + expire_in: 2 days + when: always diff --git a/.gitlab/test_starpu_simgrid.yml b/.gitlab/test_starpu_simgrid.yml index ff0745267..5c688dc94 100644 --- a/.gitlab/test_starpu_simgrid.yml +++ b/.gitlab/test_starpu_simgrid.yml @@ -2,17 +2,7 @@ .test_starpu_simgrid_template: &test_starpu_simgrid extends: .test_script_template needs: [build_starpu_simgrid] - dependencies: - - build_starpu_simgrid - artifacts: - name: test_starpu_simgrid - expire_in: 48 hours - paths: - - build_starpu_simgrid - - chameleon_starpu_simgrid_*.log - - chameleon_starpu_simgrid_*.lcov - reports: - junit: report.xml + dependencies: [build_starpu_simgrid] .test_starpu_simgrid_template_master: &test_starpu_simgrid_master extends: diff --git a/tools/analysis.sh b/tools/analysis.sh index 0baef9732..8fe79adcf 100755 --- a/tools/analysis.sh +++ b/tools/analysis.sh @@ -33,7 +33,6 @@ $TOOLSDIR/find_sources.sh # Generate coverage xml output $TOOLSDIR/coverage.sh -python3 /usr/local/lib/python3.8/dist-packages/lcov_cobertura.py chameleon.lcov --output chameleon_coverage.xml # Undefine this because not relevant in our configuration export UNDEFINITIONS="-UCHAMELEON_USE_OPENCL -UWIN32 -UWIN64 -U_MSC_EXTENSIONS -U_MSC_VER -U__SUNPRO_C -U__SUNPRO_CC -U__sun -Usun -U__cplusplus" @@ -47,7 +46,7 @@ cppcheck $CPPCHECK_OPT -DPRECISION_c -UPRECISION_s -UPRECISION_d -UPRECISION_z - cppcheck $CPPCHECK_OPT -DPRECISION_z -UPRECISION_s -UPRECISION_d -UPRECISION_c --file-list=./filelist_z.txt 2>> chameleon_cppcheck.xml # Set the default for the project key -SONARQUBE_PROJECTKEY=${SONARQUBE_PROJECTKEY:-hiepacs:chameleon:gitlab:$CI_PROJECT_NAMESPACE:$CI_COMMIT_REF_NAME} +SONARQUBE_PROJECTKEY=${SONARQUBE_PROJECTKEY:-hiepacs:chameleon:gitlab:$CI_PROJECT_NAMESPACE} # create the sonarqube config file cat > sonar-project.properties << EOF @@ -61,23 +60,26 @@ sonar.links.issue=$CI_PROJECT_URL/issues sonar.projectKey=$SONARQUBE_PROJECTKEY sonar.projectDescription=Dense linear algebra subroutines for heterogeneous and distributed architectures -sonar.projectVersion=master +sonar.projectVersion=1.1.2 + +sonar.scm.disabled=false +sonar.scm.provider=git +sonar.scm.exclusions.disabled=true sonar.sources=build-openmp/runtime/openmp, build-parsec/runtime/parsec, build-quark/runtime/quark, build-starpu, compute, control, coreblas, example, include, runtime, testing sonar.inclusions=`cat filelist.txt | sed ':a;N;$!ba;s/\n/, /g'` -sonar.c.includeDirectories=$(echo | gcc -E -Wp,-v - 2>&1 | grep "^ " | tr '\n' ',').,$(find . -type f -name '*.h' | sed -r 's|/[^/]+$||' |sort |uniq | xargs echo | sed -e 's/ /,/g'),$PARSEC_DIR/include,$QUARK_DIR/include,$STARPU_DIR/include/starpu/1.2,$SIMGRID_DIR/include sonar.sourceEncoding=UTF-8 -sonar.c.errorRecoveryEnabled=true -sonar.c.gcc.charset=UTF-8 -sonar.c.gcc.regex=(?<file>.*):(?<line>[0-9]+):[0-9]+:\\\x20warning:\\\x20(?<message>.*)\\\x20\\\[(?<id>.*)\\\] -sonar.c.gcc.reportPath=chameleon_build.log -sonar.c.coverage.reportPath=chameleon_coverage.xml -sonar.c.cppcheck.reportPath=chameleon_cppcheck.xml -sonar.c.clangsa.reportPath=build-openmp/analyzer_reports/*/*.plist, build-parsec/analyzer_reports/*/*.plist, build-quark/analyzer_reports/*/*.plist, build-starpu/analyzer_reports/*/*.plist, build-starpu_simgrid/analyzer_reports/*/*.plist -sonar.c.jsonCompilationDatabase=build-openmp/compile_commands.json, build-parsec/compile_commands.json, build-quark/compile_commands.json, build-starpu/compile_commands.json, build-starpu_simgrid/compile_commands.json -sonar.lang.patterns.c++: **/*.cxx,**/*.cpp,**/*.cc,**/*.hxx,**/*.hpp,**/*.hh -sonar.lang.patterns.c: **/*.c,**/*.h -sonar.lang.patterns.python: **/*.py +sonar.cxx.includeDirectories=$(echo | gcc -E -Wp,-v - 2>&1 | grep "^ " | tr '\n' ',').,$(find . -type f -name '*.h' | sed -r 's|/[^/]+$||' |sort |uniq | xargs echo | sed -e 's/ /,/g'),$PARSEC_DIR/include,$QUARK_DIR/include,$STARPU_DIR/include/starpu/1.3,$SIMGRID_DIR/include +sonar.cxx.file.suffixes=.h,.c +sonar.cxx.errorRecoveryEnabled=true +sonar.cxx.gcc.encoding=UTF-8 +sonar.cxx.gcc.regex=(?<file>.*):(?<line>[0-9]+):[0-9]+:\\\x20warning:\\\x20(?<message>.*)\\\x20\\\[(?<id>.*)\\\] +sonar.cxx.gcc.reportPaths=chameleon_build.log +sonar.cxx.xunit.reportPaths=*.junit +sonar.cxx.cobertura.reportPaths=*.cov +sonar.cxx.cppcheck.reportPaths=chameleon_cppcheck.xml +sonar.cxx.clangsa.reportPaths=build-openmp/analyzer_reports/*/*.plist, build-parsec/analyzer_reports/*/*.plist, build-quark/analyzer_reports/*/*.plist, build-starpu/analyzer_reports/*/*.plist, build-starpu_simgrid/analyzer_reports/*/*.plist +sonar.cxx.jsonCompilationDatabase=build-openmp/compile_commands.json, build-parsec/compile_commands.json, build-quark/compile_commands.json, build-starpu/compile_commands.json, build-starpu_simgrid/compile_commands.json EOF # run sonar analysis + publish on sonarqube-dev diff --git a/tools/build_macosx.sh b/tools/build_macosx.sh index 08743e444..6203017b2 100755 --- a/tools/build_macosx.sh +++ b/tools/build_macosx.sh @@ -15,5 +15,5 @@ cd build-starpu cmake .. -DCMAKE_INSTALL_PREFIX=$PWD/../install -DCHAMELEON_PREC_D=OFF -DCHAMELEON_PREC_C=OFF -DCHAMELEON_PREC_Z=OFF -DBLA_PREFER_PKGCONFIG=ON -DBUILD_SHARED_LIBS=ON -make -j5 +make -j5 | tee ../chameleon_macosx.log make install diff --git a/tools/coverage.sh b/tools/coverage.sh index 704177e35..d1344a854 100755 --- a/tools/coverage.sh +++ b/tools/coverage.sh @@ -22,3 +22,5 @@ do done lcov $INPUT_FILES -o chameleon.lcov lcov --summary chameleon.lcov +lcov_cobertura chameleon.lcov --output chameleon_coverage.xml +gcovr --xml-pretty --exclude-unreachable-branches --print-summary -o coverage.xml --root ${CI_PROJECT_DIR} diff --git a/tools/find_sources.sh b/tools/find_sources.sh index fe75a73e6..c37572683 100755 --- a/tools/find_sources.sh +++ b/tools/find_sources.sh @@ -27,6 +27,10 @@ sed -i '/CMakeFiles/d' filelist.txt # Remove installed files sed -i '/build.*\/install.*/d' filelist.txt +# Remove all hmat files (external package) +sed -i '/coreblas\/hmat-oss/d' filelist.txt +sed -i '/testing\/test_fembem/d' filelist.txt + # Remove original files used for precision generation for file in `git grep "@precisions" | awk -F ":" '{ print $1 }'` do -- GitLab