diff --git a/.gitlab-ci-env.sh b/.gitlab-ci-env.sh index e055861482b5c1614a142cd3457cb9fce253d4ba..d615bff8669efbc3606e3c367e75882f841788d9 100755 --- a/.gitlab-ci-env.sh +++ b/.gitlab-ci-env.sh @@ -26,32 +26,11 @@ if [[ "$SYSTEM" == "linux" ]]; then fi export STARPU_DIR=/home/gitlab/install/starpu${STARPU_VARIANT} - # add additional env. var. depending on the starpu variant - case $STARPU_VARIANT in - -hip ) - export CMAKE_PREFIX_PATH=$STARPU_DIR:/opt/rocm - export LD_LIBRARY_PATH=/opt/rocm/lib - ;; - -hipcuda ) - export CMAKE_PREFIX_PATH=$STARPU_DIR:$HIPCUDA_DIR - export LD_LIBRARY_PATH=$HIPCUDA_DIR/lib - export HIP_PLATFORM=nvidia - export HIP_PATH=$HIPCUDA_DIR - ;; - * ) - ;; - esac - # for build: better to rely on pkg-config than to guess libraries with the env. var. - export PKG_CONFIG_PATH=$PARSEC_DIR/lib/pkgconfig:$PKG_CONFIG_PATH export PKG_CONFIG_PATH=$STARPU_DIR/lib/pkgconfig:$PKG_CONFIG_PATH - export PKG_CONFIG_PATH=$SIMGRID_DIR/lib/pkgconfig:$PKG_CONFIG_PATH # for ctest: we need this at runtime - export LD_LIBRARY_PATH=$PARSEC_DIR/lib:$LD_LIBRARY_PATH - export LD_LIBRARY_PATH=$QUARK_DIR/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$STARPU_DIR/lib:$LD_LIBRARY_PATH - export LD_LIBRARY_PATH=$SIMGRID_DIR/lib:$LD_LIBRARY_PATH elif [[ "$SYSTEM" == "windows" ]]; then diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f73b5ad321c2ee8a762613053e0e6c0a09d77f33..e01e26e32b69683fa226bb64ddc2b5e840a9fa28 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,6 @@ --- stages: + - docker - pre - build - test @@ -7,20 +8,10 @@ stages: - analyze - deploy -# git config --global ahev been added to get around the issue related in -# https://github.com/actions/checkout/issues/760 for github -# Does not seem to be fixed for gitlab-runner yet -before_script: - - git config --global --add safe.directory $CI_PROJECT_DIR - - git config --global --add safe.directory $CI_PROJECT_DIR/cmake_modules/morse_cmake - - git config --global --add safe.directory $CI_PROJECT_DIR/coreblas/hmat-oss - - git config --global --add safe.directory $CI_PROJECT_DIR/hqr - - git config --global --add safe.directory $CI_PROJECT_DIR/testing/test_fembem - - git submodule update --init --recursive - include: - - .gitlab/preliminary.yml - .gitlab/common.yml + - .gitlab/preliminary.yml + - .gitlab/docker.yml - .gitlab/build.yml - .gitlab/test_starpu.yml - .gitlab/test_starpu_simgrid.yml diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..c4799ec2ff1d0e8fff0768b429403f6c709c9fcd --- /dev/null +++ b/.gitlab/Dockerfile @@ -0,0 +1,88 @@ +# +# @file Dockerfile +# +# @copyright 2024-2024 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, +# Univ. Bordeaux. All rights reserved. +# +# @version 1.3.0 +# @author Florent Pruvost +# @date 2024-11-22 +# +# This docker image is used to test the package during gitlab-ci pipelines. +# It should be stored in the gitlab's project container registry: +# https://gitlab.inria.fr/solverstack/chameleon/container_registry +# +FROM registry.gitlab.inria.fr/solverstack/docker/analysis + +USER root + +# Chameleon dependencies: cpu kernels +RUN apt-get -y upgrade --no-install-recommends \ + libblas-dev \ + liblapack-dev \ + liblapacke-dev +RUN update-alternatives --set libblas.so-x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/blas/libblas.so +RUN update-alternatives --set libblas.so.3-x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/blas/libblas.so.3 +RUN update-alternatives --set liblapack.so-x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/lapack/liblapack.so +RUN update-alternatives --set liblapack.so.3-x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/lapack/liblapack.so.3 + +# Starpu dependencies +RUN apt-get -y upgrade --no-install-recommends \ + autoconf \ + automake \ + libtool-bin \ + libfxt-dev \ + libhwloc-dev \ + libopenmpi-dev \ + libsimgrid-dev + +RUN apt-get autoremove -y +RUN apt-get autoclean -y +RUN apt-get purge -y + +RUN update-ca-certificates + +# Install PARSEC +RUN git clone https://bitbucket.org/mfaverge/parsec.git && \ + cd parsec && \ + git checkout mymaster && \ + git checkout 6022a61dc96c25f11dd2aeabff2a5b3d7bce867d && \ + git submodule update && \ + cmake -B build -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=ON -DPARSEC_GPU_WITH_CUDA=OFF -DPARSEC_DIST_WITH_MPI=OFF && \ + cmake --build build -j5 && \ + cmake --install build && \ + cd .. && \ + rm parsec/ -rf + +# Install QUARK +RUN git clone https://github.com/ecrc/quark && \ + cd quark/ && \ + git checkout db4aef9a66a00487d849cf8591927dcebe18ef2f && \ + sed -i -e "s#prefix=.*#prefix=/usr#g" make.inc && \ + sed -i -e "s#CFLAGS=.*#CFLAGS= -g -O2 -DADD_ -fPIC#g" make.inc && \ + make && \ + make install && \ + cd .. && \ + rm quark/ -rf + +# Install StarPU +RUN wget https://files.inria.fr/starpu/starpu-1.4.7/starpu-1.4.7.tar.gz > /dev/null 2>&1 && \ + tar xzf starpu-1.4.7.tar.gz && \ + cd starpu-1.4.7/ && \ + ./configure --prefix=/home/gitlab/install/starpu --enable-debug --disable-build-doc --with-fxt --disable-simgrid && \ + make -j5 install + +# Install StarPU with Simgrid +RUN cd starpu-1.4.7/ && \ + ./configure --prefix=/home/gitlab/install/starpu-simgrid --enable-debug --disable-build-doc --with-fxt --enable-simgrid && \ + make -j5 install && \ + cd ../ && \ + rm starpu-1.4.7/ -rf && \ + rm starpu-1.4.7.tar.gz + +USER gitlab + +# Set some default environment variables +ENV LIBRARY_PATH=/usr/local/lib +ENV LD_LIBRARY_PATH=/usr/local/lib +ENV PKG_CONFIG_PATH=/home/gitlab/install/starpu/lib/pkgconfig diff --git a/.gitlab/Dockerfile-pages b/.gitlab/Dockerfile-pages new file mode 100644 index 0000000000000000000000000000000000000000..441685f584274e724e21468fced26638beb8d239 --- /dev/null +++ b/.gitlab/Dockerfile-pages @@ -0,0 +1,40 @@ +# +# @file Dockerfile-pages +# +# @copyright 2024-2024 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, +# Univ. Bordeaux. All rights reserved. +# +# @version 1.3.0 +# @author Florent Pruvost +# @date 2024-12-02 +# +# This docker image is used to test the package during gitlab-ci pipelines. +# It should be stored in the gitlab's project container registry: +# https://gitlab.inria.fr/solverstack/chameleon/container_registry +# +FROM registry.gitlab.inria.fr/solverstack/chameleon + +USER root + +# Chameleon tools/pages.sh script additional dependencies +RUN apt-get -y upgrade --no-install-recommends \ + emacs \ + emacs-goodies-el \ + elpa-htmlize \ + python3-click \ + python3-elasticsearch \ + python3-pandas \ + r-cran-ggplot2 \ + r-cran-plyr \ + r-cran-reshape2 + +RUN apt-get autoremove -y +RUN apt-get autoclean -y +RUN apt-get purge -y + +USER gitlab + +# Set some default environment variables +ENV LIBRARY_PATH=/usr/local/lib +ENV LD_LIBRARY_PATH=/usr/local/lib +ENV PKG_CONFIG_PATH=/home/gitlab/install/starpu/lib/pkgconfig diff --git a/.gitlab/build.sh b/.gitlab/build.sh index de06d28ccbb1424496d8169a107b7c404ae247bb..370b00f1e86c64478defcbc2309b3b20cc2be9c9 100755 --- a/.gitlab/build.sh +++ b/.gitlab/build.sh @@ -3,8 +3,25 @@ set -ex SCAN="" -if [[ "$SYSTEM" != "windows" ]]; then - if [[ "$SYSTEM" == "macosx" ]]; then +# Configure with CMake +case $SYSTEM in + + guix) + echo "build on guix" + cmake -B build-${VERSION} -S . -C cmake_modules/gitlab-ci-initial-cache.cmake $BUILD_OPTIONS + ;; + + linux) + echo "build on linux" + source .gitlab-ci-env.sh $CHAM_CI_ENV_ARG + if [[ $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH ]]; then + SCAN="scan-build -plist --intercept-first --exclude CMakeFiles --analyze-headers -o analyzer_reports " + fi + eval '${SCAN}cmake -B build-${VERSION} -S . -C cmake_modules/gitlab-ci-initial-cache.cmake $BUILD_OPTIONS' + ;; + + macosx) + echo "build on macosx" # check starpu is already installed and install it if necessary DEP_INSTALLED=`brew ls --versions starpu | cut -d " " -f 2` if [[ -z "${DEP_INSTALLED}" ]]; then @@ -24,23 +41,27 @@ if [[ "$SYSTEM" != "windows" ]]; then cmake -B build-${VERSION} -S . \ -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$PWD/install-${VERSION} \ -DMORSE_ENABLE_COVERAGE=OFF -DBLA_PREFER_PKGCONFIG=ON - else - source .gitlab-ci-env.sh $CHAM_CI_ENV_ARG - if [[ $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH ]] - then - SCAN="scan-build -plist --intercept-first --exclude CMakeFiles --analyze-headers -o analyzer_reports " - fi - eval '${SCAN}cmake -B build-${VERSION} -S . -C cmake_modules/gitlab-ci-initial-cache.cmake $BUILD_OPTIONS' - fi -else - # on windows the mpi_f08 interface is missing, see https://www.scivision.dev/windows-mpi-msys2/ - # do not use static libraries because executables are too large and the build - # directory can reach more than 10Go - cmake -GNinja -B build-${VERSION} -S . \ - -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$PWD/install-${VERSION} \ - -DCHAMELEON_USE_MPI=OFF -fi -eval '${SCAN}cmake --build build-${VERSION} -j 4' + ;; + + windows) + echo "build on windows" + # on windows the mpi_f08 interface is missing, see https://www.scivision.dev/windows-mpi-msys2/ + # do not use static libraries because executables are too large and the build + # directory can reach more than 10Go + cmake -GNinja -B build-${VERSION} -S . \ + -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$PWD/install-${VERSION} \ + -DCHAMELEON_USE_MPI=OFF + ;; + *) + echo "The SYSTEM environment variable is $SYSTEM. It is not one of : guix, linux, macosx, windows -> exit 1." + exit 1 + ;; +esac + +# Compile +eval '${SCAN}cmake --build build-${VERSION} -j 4 > /dev/null' + +# Install cmake --install build-${VERSION} # @@ -58,7 +79,10 @@ fi export FC=gfortran # Set the path variables -if [[ "$SYSTEM" == "linux" ]]; then +if [[ "$SYSTEM" == "guix" ]]; then + export LIBRARY_PATH=$PWD/../../install-${VERSION}/lib:$LIBRARY_PATH + export LD_LIBRARY_PATH=$PWD/../../install-${VERSION}/lib:$LD_LIBRARY_PATH +elif [[ "$SYSTEM" == "linux" ]]; then export LIBRARY_PATH=$PWD/../../install-${VERSION}/lib:$LIBRARY_PATH export LD_LIBRARY_PATH=$PWD/../../install-${VERSION}/lib:$LD_LIBRARY_PATH elif [[ "$SYSTEM" == "macosx" ]]; then diff --git a/.gitlab/build.yml b/.gitlab/build.yml index b79afef185608c55a760b731963fd3fd2db321f9..e5d6743322aec15b494fc88129fb6f2672b926b3 100644 --- a/.gitlab/build.yml +++ b/.gitlab/build.yml @@ -1,16 +1,18 @@ --- -.build_script_linux: +.build_script_common: stage: build - tags: ["docker", "large"] extends: .only-master-mr - script: - - bash .gitlab/build.sh | tee ${LOGNAME}.log artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - expire_in: 180 minutes + expire_in: 1 week untracked: true when: always +.build_script_linux: + extends: .build_script_common + script: + - bash .gitlab/build.sh | tee ${LOGNAME}.log + build_hmat: extends: .build_script_linux variables: @@ -27,7 +29,6 @@ build_openmp: LOGNAME: "chameleon-build-${SYSTEM}-${VERSION}" BUILD_OPTIONS: "-DCHAMELEON_SCHED=OPENMP" - build_parsec: extends: .build_script_linux variables: @@ -50,25 +51,34 @@ build_starpu: SYSTEM: linux VERSION: starpu LOGNAME: "chameleon-build-${SYSTEM}-${VERSION}" - BUILD_OPTIONS: "-DCHAMELEON_USE_CUDA=ON -DCHAMELEON_USE_MPI=ON" + BUILD_OPTIONS: "-DCHAMELEON_USE_CUDA=OFF -DCHAMELEON_USE_MPI=ON" -build_starpu_hip: - extends: .build_script_linux +.build_script_guix: + tags: ['plafrim'] + extends: .build_script_common + script: + - guix describe -f channels + - guix shell --pure --preserve=SYSTEM --preserve=VERSION --preserve=LOGNAME --preserve=BUILD_OPTIONS + -D chameleon-${GUIX_CHAMELEON_VARIANT} coreutils bash-minimal -- + bash .gitlab/build.sh | tee ${LOGNAME}.log + +build_starpu_cuda: + extends: .build_script_guix variables: - SYSTEM: linux - VERSION: starpu_hip + SYSTEM: guix + VERSION: starpu_cuda + GUIX_CHAMELEON_VARIANT: cuda LOGNAME: "chameleon-build-${SYSTEM}-${VERSION}" - CHAM_CI_ENV_ARG: hip - BUILD_OPTIONS: "-DCHAMELEON_USE_HIP_ROC=ON -DCHAMELEON_USE_MPI=ON" + BUILD_OPTIONS: "-DCHAMELEON_USE_CUDA=ON -DCHAMELEON_USE_MPI=ON -DBLA_VENDOR=OpenBLAS" -build_starpu_hipcuda: - extends: .build_script_linux +build_starpu_hip: + extends: .build_script_guix variables: - SYSTEM: linux - VERSION: starpu_hipcuda + SYSTEM: guix + VERSION: starpu_hip + GUIX_CHAMELEON_VARIANT: hip LOGNAME: "chameleon-build-${SYSTEM}-${VERSION}" - CHAM_CI_ENV_ARG: hipcuda - BUILD_OPTIONS: "-DCHAMELEON_USE_HIP_CUDA=ON -DCHAMELEON_HIPBLAS_PATH=/home/gitlab/hipcuda/hipblas -DCHAMELEON_USE_MPI=ON" + BUILD_OPTIONS: "-DCHAMELEON_USE_HIP_ROC=ON -DCHAMELEON_USE_MPI=ON -DBLA_VENDOR=OpenBLAS -DCMAKE_C_COMPILER=gcc -DCMAKE_Fortran_COMPILER=gfortran" build_starpu_simgrid: extends: .build_script_linux @@ -91,7 +101,7 @@ build_starpu_macosx: - bash .gitlab/build.sh | tee ${LOGNAME}.log artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - expire_in: 180 minutes + expire_in: 1 week paths: - ${LOGNAME}.log cache: @@ -113,7 +123,7 @@ build_starpu_macosx: - bash -lc .gitlab/build.sh | tee "$env:LOGNAME.log" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - expire_in: 180 minutes + expire_in: 1 week paths: - ${LOGNAME}.log cache: diff --git a/.gitlab/check_link/link_cmake.sh b/.gitlab/check_link/link_cmake.sh index 613218956c689e3187ae72e99207f2d51a2b2305..3b995b52b32a4c3702af05adb3e557494efedc94 100755 --- a/.gitlab/check_link/link_cmake.sh +++ b/.gitlab/check_link/link_cmake.sh @@ -21,7 +21,7 @@ then exit 1 fi cmake -B build -DCMAKE_PREFIX_PATH=$1 -cmake --build build --verbose +cmake --build build --verbose if [[ "$VERSION" != "starpu_simgrid" ]]; then ctest --test-dir build --verbose fi diff --git a/.gitlab/common.yml b/.gitlab/common.yml index fd00a2649d9cf269fc7870f27ac79dc5c848e235..9e679f1c4e1b4eabe97865e7d1ff3a9fbaa57244 100644 --- a/.gitlab/common.yml +++ b/.gitlab/common.yml @@ -3,13 +3,16 @@ # --- default: - image: registry.gitlab.inria.fr/solverstack/docker/distrib - tags: ['ci.inria.fr', 'linux', 'large'] + image: $CI_REGISTRY_IMAGE + tags: ['ci.inria.fr', 'linux', 'large'] # gitlab instance runner (i.e. docker on linux) + interruptible: true + +variables: + GIT_SUBMODULE_STRATEGY: recursive # push event: executed only if the source branch name is master or starts with ci- # schedule event: not executed .only-master: - interruptible: true rules: - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /^ci-.*$/) && $CI_PIPELINE_SOURCE != "schedule" @@ -17,14 +20,12 @@ default: # merge request event: executed if the source branch name does not start with notest- # schedule event: not executed .only-master-mr: - interruptible: true rules: - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /^ci-.*$/) && $CI_PIPELINE_SOURCE != "schedule" - if: ($CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^notest-.*$/) # merge request event: executed if the source branch name does not start with notest- .only-mr: - interruptible: true rules: - if: ($CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^notest-.*$/) @@ -34,7 +35,7 @@ default: - bash .gitlab/test.sh | tee ${LOGNAME}.log artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - expire_in: 180 minutes + expire_in: 1 week untracked: true reports: junit: ${LOGNAME}-junit.xml diff --git a/.gitlab/coverage.yml b/.gitlab/coverage.yml index 5f9db28dde0f2fdeb490c172ded7229b175c451e..cbe0a2aca08937a32668022ecbeadb170354d3aa 100644 --- a/.gitlab/coverage.yml +++ b/.gitlab/coverage.yml @@ -12,7 +12,7 @@ coverage: /^\s*lines......:\s*\d+.\d+\%/ artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - expire_in: 180 minutes + expire_in: 1 week paths: - chameleon-*.lcov - chameleon.lcov @@ -32,7 +32,6 @@ coverage_master: coverage_mr: extends: .coverage - interruptible: true rules: - if: ($CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^notest-.*$/) needs: [test_openmp_mr,test_parsec_mr,test_quark_mr,test_starpu_mr,test_starpu_simgrid_simu_all] diff --git a/.gitlab/coverity.yml b/.gitlab/coverity.yml index ddb92221249d3f1225d79820cc95c0bc08b96450..8a2fdd0b38e079147a159df48e58088090f276a1 100644 --- a/.gitlab/coverity.yml +++ b/.gitlab/coverity.yml @@ -1,8 +1,6 @@ --- coverity: stage: analyze - tags: ["docker", "large"] - interruptible: true rules: - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /^ci-.*$/) && $CI_PIPELINE_SOURCE != "schedule" - if: ($CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^notest-.*$/) @@ -12,17 +10,7 @@ coverity: variables: VERSION: coverity script: - - (mkdir -p build-$VERSION && cd build-$VERSION && - cmake .. -C ../cmake_modules/gitlab-ci-initial-cache.cmake - -DCHAMELEON_USE_CUDA=ON -DCHAMELEON_USE_MPI=ON && - cov-build --dir ../cov-int make -j 4) - - tar czvf chameleon.tgz cov-int - - curl --form token=$COVERITY_TOKEN - --form email=florent.pruvost@inria.fr - --form file=@chameleon.tgz - --form version="`git rev-parse --short HEAD`" - --form description="" - https://scan.coverity.com/builds?project=Chameleon + - ./tools/coverity.sh artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" expire_in: 1 week diff --git a/.gitlab/docker.yml b/.gitlab/docker.yml new file mode 100644 index 0000000000000000000000000000000000000000..f2510aee3c5aac35a648b465ee552377bd2c3517 --- /dev/null +++ b/.gitlab/docker.yml @@ -0,0 +1,22 @@ +--- +.docker_script_common: + stage: docker + image: docker + when: manual + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + script: + - docker build -f $IMAGE_FILE -t $IMAGE_NAME $PWD + - docker push $IMAGE_NAME + +docker-tests: + extends: .docker_script_common + variables: + IMAGE_FILE: .gitlab/Dockerfile + IMAGE_NAME: $CI_REGISTRY_IMAGE + +docker-pages: + extends: .docker_script_common + variables: + IMAGE_FILE: .gitlab/Dockerfile-pages + IMAGE_NAME: $CI_REGISTRY_IMAGE/pages diff --git a/.gitlab/pages.yml b/.gitlab/pages.yml index 5299509e39c5c001ca1f3afc67a7bfc7f608eb7b..3e533f37978f158b241fa90a2aab0178c021d65e 100644 --- a/.gitlab/pages.yml +++ b/.gitlab/pages.yml @@ -1,14 +1,15 @@ --- pages: stage: deploy + image: $CI_REGISTRY_IMAGE/pages extends: .only-master needs: [coverage_master] + variables: + VERSION: pages + script: + - ./tools/pages.sh artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" expire_in: 1 week paths: - public - variables: - VERSION: pages - script: - - ./tools/pages.sh diff --git a/.gitlab/preliminary.yml b/.gitlab/preliminary.yml index 5377e9ea1dcf1a3469f10b60a6d31f1283b983ef..8d6a909755522eaaac4ed8dfa1b0d98e57fd1591 100644 --- a/.gitlab/preliminary.yml +++ b/.gitlab/preliminary.yml @@ -4,7 +4,6 @@ --- preliminary_checks: stage: pre - interruptible: true rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" parallel: diff --git a/.gitlab/sonarqube.yml b/.gitlab/sonarqube.yml index 88ef0b06883dd2df45b59b068a19b3a942f3e50f..e9d4d1ab108d5a261ad3645012cb904d9798a9b6 100644 --- a/.gitlab/sonarqube.yml +++ b/.gitlab/sonarqube.yml @@ -1,7 +1,6 @@ --- .sonarqube: stage: analyze - tags: ["docker", "large"] variables: GIT_DEPTH: "0" VERSION: sonarqube @@ -14,7 +13,7 @@ - ./tools/analysis.sh artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - expire_in: 180 minutes + expire_in: 1 week when: always paths: - filelist.txt @@ -31,7 +30,6 @@ sonarqube_master: sonarqube_mr: extends: .sonarqube - interruptible: true rules: - if: ($CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^notest-.*$/) when: manual diff --git a/.gitlab/test.sh b/.gitlab/test.sh index 7b0e4326886ff57af177b3b9d2f4a8fba7cfe413..0a4a8639fa4a54d1121f149e57264ed3ad2338fa 100755 --- a/.gitlab/test.sh +++ b/.gitlab/test.sh @@ -1,16 +1,11 @@ #!/usr/bin/env bash -fatal() { - echo "$0: error occurred, exit" - exit 1 -} - +set -e set -x cd build-$VERSION -source ../.gitlab-ci-env.sh $CHAM_CI_ENV_ARG || fatal -CTESTCOMMAND=`echo "ctest --output-on-failure --no-compress-output $TESTS_RESTRICTION -T Test --output-junit ../${LOGNAME}-junit.xml"` -$CTESTCOMMAND || fatal +source ../.gitlab-ci-env.sh $CHAM_CI_ENV_ARG +ctest --output-on-failure --no-compress-output $TESTS_RESTRICTION --output-junit ../${LOGNAME}-junit.xml if [[ "$SYSTEM" == "linux" ]]; then # clang is used on macosx and it is not compatible with MORSE_ENABLE_COVERAGE=ON # so that we can only make the coverage report on the linux runner with gcc diff --git a/.gitlab/test_openmp.yml b/.gitlab/test_openmp.yml index cfbb14f3cde0d5a7dffd93417df7b709ca8abaa9..923e7ebb378818a00a6fdf7f4ca1fdde61560db1 100644 --- a/.gitlab/test_openmp.yml +++ b/.gitlab/test_openmp.yml @@ -2,7 +2,6 @@ .test_openmp: extends: .test_script_linux needs: [build_openmp] - interruptible: true variables: SYSTEM: linux VERSION: openmp diff --git a/.gitlab/test_parsec.yml b/.gitlab/test_parsec.yml index 52764d27b18f585eae8584a4a39da80b864b5042..b131e8b6924e3fcf92c71d5275fd907c233df255 100644 --- a/.gitlab/test_parsec.yml +++ b/.gitlab/test_parsec.yml @@ -2,7 +2,6 @@ .test_parsec: extends: .test_script_linux needs: [build_parsec] - interruptible: true variables: SYSTEM: linux VERSION: parsec diff --git a/.gitlab/test_quark.yml b/.gitlab/test_quark.yml index d1ede5b5853e264d0afea65ae4a50ed76c7d743e..0bd998d9b9850dbe80fea058d16a731cd5ccd7b3 100644 --- a/.gitlab/test_quark.yml +++ b/.gitlab/test_quark.yml @@ -2,7 +2,6 @@ .test_quark: extends: .test_script_linux needs: [build_quark] - interruptible: true variables: SYSTEM: linux VERSION: quark diff --git a/.gitlab/test_starpu.yml b/.gitlab/test_starpu.yml index bbb562a6389c95c409c7e50ece69d63d57548679..efc3bd783aa0b64355500d6bb8a5fa4b233f5e59 100644 --- a/.gitlab/test_starpu.yml +++ b/.gitlab/test_starpu.yml @@ -2,7 +2,6 @@ .test_starpu: extends: .test_script_linux needs: [build_starpu] - interruptible: true variables: SYSTEM: linux VERSION: starpu @@ -43,7 +42,7 @@ test_starpu_macosx: - bash .gitlab/test.sh | tee ${LOGNAME}.log artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - expire_in: 180 minutes + expire_in: 1 week paths: - ${LOGNAME}.log reports: @@ -69,7 +68,7 @@ test_starpu_macosx: - bash -lc .gitlab/test.sh | tee "$env:LOGNAME.log" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - expire_in: 180 minutes + expire_in: 1 week paths: - ${LOGNAME}.log reports: diff --git a/CMakeLists.txt b/CMakeLists.txt index 54275bb054632c724345c421450a8063c599cbf0..15f757a9e8721c2f39ab2bb62fdcf6b6347f5ccb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -766,7 +766,7 @@ add_custom_target(chameleon_all_sources ALL DEPENDS ${CHAMELEON_SOURCES_TARGETS} if(CHAMELEON_ENABLE_DOC) add_subdirectory(doc) - if (DOXYGEN_EXECUTABLE AND EMACS_COMPILER AND TEX_COMPILER) + if (DOXYGEN_EXECUTABLE AND EMACS_COMPILER) add_custom_target(doc ALL DEPENDS doc-homepage doxygen-out) endif() endif() diff --git a/cmake_modules/ChameleonPkgConfig.cmake b/cmake_modules/ChameleonPkgConfig.cmake index 15df826032481a17049eba238abf49f3b0061bb6..0c1f9df0f080dc281cffd747a5cff712c94bdcf0 100644 --- a/cmake_modules/ChameleonPkgConfig.cmake +++ b/cmake_modules/ChameleonPkgConfig.cmake @@ -8,7 +8,7 @@ ### # # @project Chameleon -# @version 1.2.0 +# @version 1.3.0 # @author Mathieu Faverge # @author Florent Pruvost # @author Abel Calluaud @@ -93,7 +93,6 @@ macro(chameleon_generate_pkgconfig_files) if(CHAMELEON_USE_CUDA) list(APPEND GPUCUBLAS_PKGCONFIG_LIBS_PRIVATE ${CUDA_CUBLAS_LIBRARIES}) - list(APPEND GPUCUBLAS_PKGCONFIG_REQUIRED "cuda") list(APPEND CHAMELEON_PKGCONFIG_REQUIRED "gpucublas") endif() @@ -110,7 +109,28 @@ macro(chameleon_generate_pkgconfig_files) ) if(CHAMELEON_USE_MPI) - list(APPEND CHAMELEON_PKGCONFIG_REQUIRED "mpi") + if(${MPI_C_LIBRARIES} MATCHES "mpich") + set(MPI_NAME "mpich") + elseif(${MPI_C_LIBRARIES} MATCHES "mvapich2") + set(MPI_NAME "mvapich2") + elseif(${MPI_C_LIBRARIES} MATCHES "madmpi" OR ${MPI_C_LIBRARIES} MATCHES "nmad") + set(MPI_NAME "nmad") + elseif(${MPI_C_LIBRARIES} MATCHES "openmpi") + set(MPI_NAME "ompi") + else() + set(MPI_NAME "mpi") + endif() + find_file(MPIPC_PATH "${MPI_NAME}.pc" + HINTS ${CMAKE_C_IMPLICIT_LINK_DIRECTORIES} + ENV PKG_CONFIG_PATH + PATH_SUFFIXES pkgconfig + NO_CACHE) + if(MPIPC_PATH) + list(APPEND CHAMELEON_PKGCONFIG_REQUIRED ${MPI_NAME}) + else() + list(APPEND CHAMELEON_PKGCONFIG_INCS "${MPI_C_INCLUDE_DIRS}") + list(APPEND CHAMELEON_PKGCONFIG_LIBS "${MPI_C_LIBRARIES}") + endif() endif() generate_pkgconfig_files( diff --git a/doc/dev/CMakeLists.txt b/doc/dev/CMakeLists.txt index d086e2bce51cc66f8313a3b966370320f0074f8e..4f94bec91cabe32e835589ea711b8c8ddd2b43d2 100644 --- a/doc/dev/CMakeLists.txt +++ b/doc/dev/CMakeLists.txt @@ -30,7 +30,7 @@ cmake_minimum_required(VERSION 3.5) # Looking for doxygen # ------------------- -find_package(Doxygen) +find_package(Doxygen REQUIRED) # Generate documentation # ---------------------- diff --git a/doc/user/CMakeLists.txt b/doc/user/CMakeLists.txt index 635d662e6a93db6b329f1348445e555483ad3e28..34c0f8d8fa687de36c9b6239c0c737316c6815cf 100644 --- a/doc/user/CMakeLists.txt +++ b/doc/user/CMakeLists.txt @@ -68,10 +68,10 @@ endforeach() # Looking for emacs # ----------------- -FIND_PROGRAM(EMACS_COMPILER emacs) -FIND_PROGRAM(TEX_COMPILER tex) +find_program(EMACS_COMPILER emacs REQUIRED) if(EMACS_COMPILER) + message(STATUS "Looking for emacs - found") # Add target for homepage # ----------------------- diff --git a/doc/user/homepage.org b/doc/user/homepage.org index 19c8ce6fd9f9e418af577fd87000c721b8f0c246..2bf41a44e83fbc7bf4051e1d25517dc9d947fbab 100644 --- a/doc/user/homepage.org +++ b/doc/user/homepage.org @@ -6,7 +6,7 @@ #+OPTIONS: H:3 num:t \n:nil @:t ::t |:t _:nil ^:nil -:t f:t *:t <:t #+OPTIONS: TeX:t LaTeX:t skip:nil d:nil pri:nil tags:not-in-toc html-style:nil # #+HTML_LINK_HOME: https://solverstack.gitlabpages.inria.fr/chameleon/ -#+SETUPFILE: https://mfelsoci.gitlabpages.inria.fr/inria-org-html-themes/online-theme-readtheorginria.setup +#+SETUPFILE: https://solverstack.gitlabpages.inria.fr/inria-org-html-themes/online-theme-readtheorginria.setup * Overview :PROPERTIES: diff --git a/doc/user/publish.el b/doc/user/publish.el index 7a2325217e6f73b15790b74a0cf8521e72d0f750..2ecd431c003c99776af9da567d1c55fa80f71a1f 100644 --- a/doc/user/publish.el +++ b/doc/user/publish.el @@ -9,6 +9,8 @@ (require 'org) (require 'htmlize) +(setq org-safe-remote-resources '("https://solverstack.gitlabpages.inria.fr/inria-org-html-themes/online-theme-readtheorginria.setup")) + (setq org-html-htmlize-output-type 'css) (setq org-src-fontify-natively t) (org-babel-do-load-languages diff --git a/tools/analysis.sh b/tools/analysis.sh index 999f6a0d340f160e9cbd060329bee144014d9d6e..a5dd554540679b26b2644fe6109d30da271f909c 100755 --- a/tools/analysis.sh +++ b/tools/analysis.sh @@ -23,7 +23,8 @@ # - consider generated files in ${BUILDDIR} # - exclude base *z* files to avoid duplication # - exclude cblas.h and lapacke-.h because not really part of chameleon and make cppcheck analysis too long - +set -e +set -x if [ $# -gt 0 ] then BUILDDIR=$1 @@ -37,7 +38,7 @@ $TOOLSDIR/find_sources.sh export UNDEFINITIONS="-UCHAMELEON_USE_OPENCL -UWIN32 -UWIN64 -U_MSC_EXTENSIONS -U_MSC_VER -U__SUNPRO_C -U__SUNPRO_CC -U__sun -Usun -U__cplusplus" # run cppcheck analysis -CPPCHECK_OPT=" -v -f --language=c --platform=unix64 --enable=all --xml --xml-version=2 --suppress=missingInclude ${UNDEFINITIONS}" +CPPCHECK_OPT=" -v -f --language=c --platform=unix64 --enable=all --xml --xml-version=2 --suppress=missingIncludeSystem ${UNDEFINITIONS}" cppcheck $CPPCHECK_OPT --file-list=./filelist_none.txt 2> chameleon_cppcheck.xml cppcheck $CPPCHECK_OPT -DPRECISION_s -UPRECISION_d -UPRECISION_c -UPRECISION_z --file-list=./filelist_s.txt 2>> chameleon_cppcheck.xml cppcheck $CPPCHECK_OPT -DPRECISION_d -UPRECISION_s -UPRECISION_c -UPRECISION_z --file-list=./filelist_d.txt 2>> chameleon_cppcheck.xml @@ -68,6 +69,7 @@ sonar.scm.exclusions.disabled=true sonar.sources=build-openmp/runtime/openmp, build-parsec/runtime/parsec, build-quark/runtime/quark, build-starpu, compute, control, coreblas, example, include, runtime, testing sonar.inclusions=`cat filelist.txt | sed ':a;N;$!ba;s/\n/, /g'` +sonar.coverage.exclusions=build-starpu/gpucublas/**/*,build-starpu/gpuhipblas/**/* sonar.sourceEncoding=UTF-8 sonar.cxx.file.suffixes=.h,.c sonar.cxx.errorRecoveryEnabled=true diff --git a/tools/bench/jube/get_result.py b/tools/bench/jube/get_result.py index a80831b241dead44ddd62669bc66f165eda90050..017f23fff6f73c8f562e7c1afd69c8bf0abfeda7 100755 --- a/tools/bench/jube/get_result.py +++ b/tools/bench/jube/get_result.py @@ -11,7 +11,6 @@ # import pandas import click -import csv from elasticsearch import Elasticsearch @click.command() @@ -57,7 +56,7 @@ def main( doc_data = doc_data.drop(labels=['Commit_date_chameleon', 'Commit_sha_chameleon', 'Commit_sha_guix', 'Commit_sha_guix_hpc', 'Commit_sha_guix_hpcnonfree']) # append the Series object to the DataFrame object - docs = docs.append(doc_data) + docs = pandas.concat([docs, doc_data.to_frame().T]) docs = docs.astype({"Nmpi": int, "Nthread": int, "Ngpu": int, "P": int, "Q": int, "M": int, "N": int, "K": int}) docs = docs.rename(columns=str.lower) diff --git a/tools/coverity.sh b/tools/coverity.sh new file mode 100755 index 0000000000000000000000000000000000000000..bd2323509683acbba3b86fe758c2b1a24ae1edc3 --- /dev/null +++ b/tools/coverity.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# @file coverity.sh +# +# @copyright 2024-2024 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, +# Univ. Bordeaux. All rights reserved. +# +# @version 1.3.0 +# @author Florent Pruvost +# @date 2024-11-27 +# +set -e +set -x +CHAMELEON_SRC_DIR=${CHAMELEON_SRC_DIR:-$PWD} + +# install plotting dependencies in the testing docker image +CURRENTUSER=`whoami` +if [[ -f /.dockerenv && $CURRENTUSER == "gitlab" ]]; then + export PKG_CONFIG_PATH=/home/gitlab/install/starpu/lib/pkgconfig:$PKG_CONFIG_PATH +fi + +cmake -B build-$VERSION -C cmake_modules/gitlab-ci-initial-cache.cmake -DCHAMELEON_USE_CUDA=OFF -DCHAMELEON_USE_MPI=ON +cov-build --dir cov-int/ cmake --build build-$VERSION -j4 +tar czvf chameleon.tgz cov-int/ +curl --form token=$COVERITY_TOKEN \ + --form email=florent.pruvost@inria.fr \ + --form file=@chameleon.tgz \ + --form version="`git rev-parse --short HEAD`" \ + --form description="" \ + https://scan.coverity.com/builds?project=Chameleon diff --git a/tools/find_sources.sh b/tools/find_sources.sh index 1119c0b5da566ee545fc07b8c91ead691fe97447..7731851cbb5c10ae8bd85ad9fd1f2c5a13faaa84 100755 --- a/tools/find_sources.sh +++ b/tools/find_sources.sh @@ -12,7 +12,7 @@ # #set -x -SRCDIR_TO_ANALYZE="build-openmp/runtime/openmp build-parsec/runtime/parsec build-quark/runtime/quark build-starpu build compute control coreblas example include runtime testing" +SRCDIR_TO_ANALYZE="build-openmp/runtime/openmp build-parsec/runtime/parsec build-quark/runtime/quark build-starpu compute control coreblas example include runtime testing" echo $PWD rm -f filelist.txt diff --git a/tools/homebrew/starpu.rb b/tools/homebrew/starpu.rb index a6b99fc77c6bd7d9e1ed0895aa17baf1e40db3eb..8d44b1c6293827d2467b459281e6cf75711b28da 100644 --- a/tools/homebrew/starpu.rb +++ b/tools/homebrew/starpu.rb @@ -13,8 +13,8 @@ class Starpu < Formula desc "StarPU is a task programming library for hybrid architectures" homepage "https://starpu.gitlabpages.inria.fr/" - url "https://files.inria.fr/starpu/starpu-1.4.6/starpu-1.4.6.tar.gz" - sha256 "579f38e150ab78c9ab299649d8649a043dbf49786e54acbf3dad90c160672cc7" + url "https://files.inria.fr/starpu/starpu-1.4.7/starpu-1.4.7.tar.gz" + sha256 "1eb3df551089153fe6e0b172ad95118434b4a81ea9eaa5a2c38725d0db53b13e" license "GNU GPL v2.1" depends_on "autoconf" => :build diff --git a/tools/pages.sh b/tools/pages.sh index df0023083225f5bf5e43cc0eef816d9931fa1753..98f170243fc5472b637cd913bd59feff9f7b9a97 100755 --- a/tools/pages.sh +++ b/tools/pages.sh @@ -5,13 +5,15 @@ # @copyright 2020-2024 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, # Univ. Bordeaux. All rights reserved. # -# @version 1.2.0 +# @version 1.3.0 # @author Florent Pruvost # @date 2022-02-22 # +set -e +set -x CHAMELEON_SRC_DIR=${CHAMELEON_SRC_DIR:-$PWD} -mkdir tmp_fig +mkdir -p tmp_fig cd tmp_fig ## need to generate figures from last benchmarks @@ -49,22 +51,23 @@ cd .. ## Build the doc VERSION=${VERSION:-pages} -mkdir -p build-$VERSION -cd build-$VERSION - -cmake $CHAMELEON_SRC_DIR -DCHAMELEON_ENABLE_DOC=ON -make doc -j5 +cmake -S ${CHAMELEON_SRC_DIR} -B build-$VERSION -DCHAMELEON_ENABLE_DOC=ON +cmake --build build-$VERSION --target doc --verbose -j5 ## Copy files in public/ used as an artefact (zip archive) to upload on gitlab pages, see -## Homepage: https://solverstack.gitlabpages.inria.fr/chameleon/index.html -## API: https://solverstack.gitlabpages.inria.fr/chameleon/dev/index.html -cd .. -mkdir public/ -mkdir public/dev/ +mkdir -p public/dev/ + +## Homepage (user's guide): https://solverstack.gitlabpages.inria.fr/chameleon/index.html cp build-$VERSION/doc/user/*.html public/ cp build-$VERSION/doc/user/*.png public/ cp build-$VERSION/doc/user/*.jpg public/ cp build-$VERSION/doc/user/*.svg public/ + +## API (doxygen): https://solverstack.gitlabpages.inria.fr/chameleon/dev/index.html cp -r build-$VERSION/doc/dev/html/* public/dev/ + +## Images such as last performances on plafrim cp tmp_fig/* public/ + +## lcov code coverage: https://solverstack.gitlabpages.inria.fr/chameleon/coverage/ cp -r coverage public/