From f8ce2cc942146b664d6584fa5d2b3ed1a47cc8bb Mon Sep 17 00:00:00 2001
From: Florent Pruvost <florent.pruvost@inria.fr>
Date: Thu, 6 Mar 2025 16:44:32 +0100
Subject: [PATCH] benchs: push first version of jube scripts
---
.gitlab-ci.yml | 13 +++
guix-tools/scalfmm-manifest-gcc-mkl-bench.scm | 33 +++++++
scripts/plafrim_level1.sh | 11 +++
scripts/plafrim_level2.sh | 20 +++++
scripts/plafrim_level3.sh | 85 ++++++++++++++++++
scripts/scalfmm-accuracy.xml | 77 ++++++++++++++++
scripts/scalfmm-time-omp.xml | 86 ++++++++++++++++++
scripts/scalfmm-time-seq.xml | 88 +++++++++++++++++++
8 files changed, 413 insertions(+)
create mode 100644 guix-tools/scalfmm-manifest-gcc-mkl-bench.scm
create mode 100755 scripts/plafrim_level1.sh
create mode 100755 scripts/plafrim_level2.sh
create mode 100755 scripts/plafrim_level3.sh
create mode 100644 scripts/scalfmm-accuracy.xml
create mode 100644 scripts/scalfmm-time-omp.xml
create mode 100644 scripts/scalfmm-time-seq.xml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8cdec8f3a..40a9a9996 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -78,6 +78,19 @@ test:
# coverage_format: cobertura
# path: coverage.xml
+benchmark:
+ stage: test
+ tags: ['plafrim']
+ #extends: .only-main-mr
+ needs: []
+ script:
+ - ./scripts/plafrim_level1.sh
+ artifacts:
+ when: always
+ paths:
+ - bench.sqlite3
+ - ./*.csv
+
pages:
stage: deploy
when: manual
diff --git a/guix-tools/scalfmm-manifest-gcc-mkl-bench.scm b/guix-tools/scalfmm-manifest-gcc-mkl-bench.scm
new file mode 100644
index 000000000..7fa7b2a66
--- /dev/null
+++ b/guix-tools/scalfmm-manifest-gcc-mkl-bench.scm
@@ -0,0 +1,33 @@
+;; What follows is a "manifest" equivalent to the command line you gave.
+;; You can store it in a file that you may then pass to any 'guix' command
+;; that accepts a '--manifest' (or '-m') option.
+
+(specifications->manifest
+ (list "bash"
+ "cmake"
+ "coreutils"
+ "curl"
+ "findutils"
+ "gcc-toolchain@11"
+ "git"
+ "grep"
+ "inetutils"
+ "intel-oneapi-mkl"
+ "jq"
+ "jube-with-yaml"
+ "make"
+ "ncurses"
+ "nss-certs"
+ "python"
+ "python-certifi"
+ "python-click"
+ "python-colorama"
+ "python-elasticsearch"
+ "python-matplotlib"
+ "python-pandas"
+ "python-pyyaml"
+ "python-setuptools"
+ "sed"
+ "slurm"
+ "sqlite"))
+
diff --git a/scripts/plafrim_level1.sh b/scripts/plafrim_level1.sh
new file mode 100755
index 000000000..391c279fe
--- /dev/null
+++ b/scripts/plafrim_level1.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -x
+
+# Submit slurm jobs
+sbatch --wait --job-name=scalfmm --output=scalfmm.out --nodes=1 --time=01:00:00 --constraint=suroit --exclusive --ntasks-per-node=1 ./scripts/plafrim_level2.sh
+err=$?
+
+cat scalfmm.out
+
+# exit with error code from the sbatch command
+exit $err
diff --git a/scripts/plafrim_level2.sh b/scripts/plafrim_level2.sh
new file mode 100755
index 000000000..d8cd9781d
--- /dev/null
+++ b/scripts/plafrim_level2.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -x
+
+# to avoid a lock during fetching git branches in parallel
+export XDG_CACHE_HOME=/tmp/guix-$$
+
+# save guix commits
+guix time-machine -C .guix/scalfmm-channels.scm -- describe --format=json > guix.json
+
+exec guix time-machine -C .guix/scalfmm-channels.scm -- shell \
+ --pure --preserve="^CI|proxy$|^SLURM" \
+ -m .guix/scalfmm-manifest-gcc-mkl-bench.scm \
+ -- /bin/bash --norc ./scripts/plafrim_level3.sh
+err=$?
+
+# clean tmp
+rm -rf /tmp/guix-$$
+
+# exit with error code from the guix command
+exit $err
diff --git a/scripts/plafrim_level3.sh b/scripts/plafrim_level3.sh
new file mode 100755
index 000000000..c4179019e
--- /dev/null
+++ b/scripts/plafrim_level3.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+set -ex
+
+export HFI_NO_CPUAFFINITY=1
+export MKLROOT=$GUIX_ENVIRONMENT
+
+if [[ -z "${CI_COMMIT_SHA}" ]]; then
+ export GIT_COMMIT=`git rev-parse HEAD`
+else
+ export GIT_COMMIT=$CI_COMMIT_SHA
+ export GIT_CONFIG_COUNT=1
+ export GIT_CONFIG_KEY_0=safe.directory
+ export GIT_CONFIG_VALUE_0=*
+fi
+export GIT_COMMIT_DATE=`git show --no-patch --format=%ci $GIT_COMMIT`
+
+if [[ -z "${CI_HOSTNAME}" ]]; then
+ export CI_HOSTNAME=`hostname`
+fi
+
+# configuration and compilation
+#if [ -d build ]; then
+# rm build -r
+#fi
+cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-O3 -march=native" -Dscalfmm_BUILD_BENCH=ON -Dscalfmm_USE_MKL=ON
+cmake --build build --target fmm-computation-seq
+cmake --build build --target fmm-computation-omp
+export SCALFMM_EXE_DIR=$PWD/build/bench/Release
+
+# clean old benchmarks
+if [ -d scripts/results ]; then
+ rm scripts/results -r
+fi
+# Execute benchmark commands
+jube run scripts/scalfmm-accuracy.xml --id 1
+jube run scripts/scalfmm-time-seq.xml --id 2
+jube run scripts/scalfmm-time-omp.xml --id 3
+
+# parse log files
+jube analyse scripts/results --id 1
+jube analyse scripts/results --id 2
+jube analyse scripts/results --id 3
+
+# reports in csv
+jube result scripts/results --id 1 -o accuracy_csv > scalfmm_accuracy.csv
+jube result scripts/results --id 2 -o timeseq_csv > scalfmm_timeseq.csv
+jube result scripts/results --id 3 -o timeomp_csv > scalfmm_timeomp.csv
+cat scalfmm_accuracy.csv
+cat scalfmm_timeseq.csv
+cat scalfmm_timeomp.csv
+
+# get current database file scalfmm.sqlite3 stored on gitlab
+if [[ ! -z "${CI_JOB_TOKEN}" ]]; then
+ export PACKAGEID=`curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages" |jq '.[0].id'`
+ if [[ "${PACKAGEID}" != "null" ]]; then
+ export FILEID=`curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages/$PACKAGEID/package_files" |jq '.[0].id'`
+ if [[ "${FILEID}" != "null" ]]; then
+ curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages/generic/benchmark/latest/scalfmm.sqlite3" -o scalfmm.sqlite3
+ fi
+ fi
+fi
+
+# update scalfmm.sqlite3 database
+jube result scripts/results --id 1 -o accuracy
+jube result scripts/results --id 2 -o timeseq
+jube result scripts/results --id 3 -o timeomp
+
+# upload updated scalfmm.sqlite3 on gitlab
+if [[ ! -z "${CI_JOB_TOKEN}" ]]; then
+ # get package id of the database stored on gitlab
+ export PACKAGEID=`curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages" |jq '.[0].id'`
+ if [[ "${PACKAGEID}" != "null" ]]; then
+ # get file id of the database on gitlab
+ export FILEID=`curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages/$PACKAGEID/package_files" |jq '.[0].id'`
+ if [[ "${FILEID}" != "null" ]]; then
+ # delete previous database version on gitlab if exists
+ curl --request DELETE --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages/$PACKAGEID/package_files/$FILEID"
+ fi
+ fi
+ #curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ./scalfmm.sqlite3 "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages/generic/benchmark/latest/scalfmm.sqlite3"
+fi
+
+# send results to the elasticsearch server
+#ls guix.json
+#python3 script/add_result.py -e https://elasticsearch.bordeaux.inria.fr -t concace -p "scalfmm" scalfmm.csv
diff --git a/scripts/scalfmm-accuracy.xml b/scripts/scalfmm-accuracy.xml
new file mode 100644
index 000000000..af35c8993
--- /dev/null
+++ b/scripts/scalfmm-accuracy.xml
@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<jube>
+ <benchmark name="scalfmm_accuracy" outpath="results">
+ <comment>scalfmm accuracy tests</comment>
+
+ <!-- Cartesian product of input parameters -->
+ <parameterset name="param">
+ <parameter name="ndim" type="int" >1, 2, 3</parameter>
+ <!-- <parameter name="ndim" type="int" >2, 3</parameter> -->
+ <parameter name="kernel_type" type="int" >0, 6, 7</parameter>
+ <!-- <parameter name="kernel_type" type="int" >0, 7</parameter> -->
+ <parameter name="interp_type" type="int" >0, 1, 2, 3, 4</parameter>
+ <!-- <parameter name="interp_type" type="int" >2, 3</parameter> -->
+ <parameter name="nrun" type="int" >1</parameter>
+ <parameter name="nthread" type="int" >4</parameter>
+ <parameter name="groupsize" type="int" >1</parameter>
+ <parameter name="tree_height" type="int" >3, 4, 5</parameter>
+ <parameter name="interp_order" type="int" >3, 4, 5, 6, 7, 8, 9, 10, 11, 12</parameter>
+ <!-- <parameter name="interp_order" type="int" >3, 4</parameter> -->
+ <parameter name="command" type="string">$SCALFMM_EXE_DIR/fmm-computation-seq --direct-computation --fmm-computation --dimension ${ndim} --group-size ${groupsize} --interp-settings ${interp_type} --kernel ${kernel_type} --order ${interp_order} --nb-runs ${nrun} --threads ${nthread} --tree-height ${tree_height}</parameter>
+ </parameterset>
+
+ <!-- Execution commands -->
+ <step name="run">
+ <use>param</use>
+ <do>echo "Commit = $GIT_COMMIT"</do>
+ <do>echo "Date = $GIT_COMMIT_DATE"</do>
+ <do>echo "Hostname = $CI_HOSTNAME"</do>
+ <do>$command</do>
+ </step>
+
+ <!-- Regex pattern -->
+ <patternset name="regexlog">
+ <pattern name="gitcommit" type="string">Commit = ([0-9a-f]{40}).*</pattern>
+ <pattern name="gitcommitdate" type="string">Date = (\d{4}\-\d{2}\-\d{2}\s\d{2}:\d{2}:\d{2}\s\+\d{4}).*</pattern>
+ <pattern name="hostname" type="string">Hostname = $jube_pat_wrd</pattern>
+ <pattern name="error" type="float">Relative L2-norm = $jube_pat_fp</pattern>
+ </patternset>
+
+ <!-- Analyse -->
+ <analyser name="parse">
+ <!-- use a pattern set -->
+ <use>regexlog</use>
+ <analyse step="run">
+ <file>stdout</file> <!-- file which should be scanned -->
+ </analyse>
+ </analyser>
+
+ <!-- Create result table -->
+ <result>
+ <use>parse</use> <!-- use existing analyser -->
+ <table name="accuracy_csv" style="csv">
+ <column>gitcommit</column>
+ <column>gitcommitdate</column>
+ <column>hostname</column>
+ <column>ndim</column>
+ <column>kernel_type</column>
+ <column>interp_type</column>
+ <column>tree_height</column>
+ <column>interp_order</column>
+ <column>error</column>
+ </table>
+ <database name="accuracy" primekeys="gitcommit,gitcommitdate,hostname,ndim,kernel_type,interp_type,tree_height,interp_order,error" file="scalfmm.sqlite3">
+ <key>gitcommit</key>
+ <key>gitcommitdate</key>
+ <key>hostname</key>
+ <key>ndim</key>
+ <key>kernel_type</key>
+ <key>interp_type</key>
+ <key>tree_height</key>
+ <key>interp_order</key>
+ <key>error</key>
+ </database>
+ </result>
+
+ </benchmark>
+</jube>
diff --git a/scripts/scalfmm-time-omp.xml b/scripts/scalfmm-time-omp.xml
new file mode 100644
index 000000000..9395523ee
--- /dev/null
+++ b/scripts/scalfmm-time-omp.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<jube>
+ <benchmark name="scalfmm_time_seq" outpath="results">
+ <comment>scalfmm OMP-tasks timing tests</comment>
+
+ <!-- Cartesian product of input parameters -->
+ <parameterset name="param">
+ <parameter name="ndim" type="int" >1, 2, 3</parameter>
+ <parameter name="ndim" type="int" >2</parameter>
+ <parameter name="kernel_type" type="int" >0</parameter>
+ <parameter name="interp_type" type="int" >2</parameter>
+ <parameter name="nrun" type="int" >1</parameter>
+ <parameter name="tree_height" type="int" >3, 4, 5</parameter>
+ <!-- <parameter name="tree_height" type="int" >5</parameter> -->
+ <parameter name="interp_order" type="int" >5</parameter>
+ <parameter name="size" type="int" >100000</parameter>
+ <parameter name="nthread" type="int" >1, 2, 4, 8, 16</parameter>
+ <!-- <parameter name="nthread" type="int" >1, 2, 4</parameter> -->
+ <parameter name="groupsize" type="int" >1, 5, 10, 100, 1000</parameter>
+ <!-- <parameter name="groupsize" type="int" >1, 5, 10, 100</parameter> -->
+ <parameter name="command" type="string">$SCALFMM_EXE_DIR/fmm-computation-omp --fmm-computation --dimension ${ndim} --group-size ${groupsize} --interp-settings ${interp_type} --kernel ${kernel_type} --order ${interp_order} --nb-runs ${nrun} --threads ${nthread} --size ${size} --tree-height ${tree_height}</parameter>
+ </parameterset>
+
+ <!-- Execution commands -->
+ <step name="run">
+ <use>param</use>
+ <do>echo "Commit = $GIT_COMMIT"</do>
+ <do>echo "Date = $GIT_COMMIT_DATE"</do>
+ <do>echo "Hostname = $CI_HOSTNAME"</do>
+ <do>$command</do>
+ </step>
+
+ <!-- Regex pattern -->
+ <patternset name="regexlog">
+ <pattern name="gitcommit" type="string">Commit = ([0-9a-f]{40}).*</pattern>
+ <pattern name="gitcommitdate" type="string">Date = (\d{4}\-\d{2}\-\d{2}\s\d{2}:\d{2}:\d{2}\s\+\d{4}).*</pattern>
+ <pattern name="hostname" type="string">Hostname = $jube_pat_wrd</pattern>
+ <pattern name="timefull" type="float">Full algo : $jube_pat_fp</pattern>
+ </patternset>
+
+ <!-- Analyse -->
+ <analyser name="parse">
+ <!-- use a pattern set -->
+ <use>regexlog</use>
+ <analyse step="run">
+ <file>stdout</file> <!-- file which should be scanned -->
+ </analyse>
+ </analyser>
+
+ <!-- Create result table -->
+ <result>
+ <use>parse</use> <!-- use existing analyser -->
+ <table name="timeomp_csv" style="csv">
+ <column>gitcommit</column>
+ <column>gitcommitdate</column>
+ <column>hostname</column>
+ <column>ndim</column>
+ <column>kernel_type</column>
+ <column>interp_type</column>
+ <column>nrun</column>
+ <column>tree_height</column>
+ <column>interp_order</column>
+ <column>size</column>
+ <column>nthread</column>
+ <column>groupsize</column>
+ <column>timefull_avg</column>
+ </table>
+ <database name="timeomp" primekeys="gitcommit,gitcommitdate,hostname,ndim,kernel_type,interp_type,nrun,tree_height,interp_order,size,nthread,groupsize,timefull_avg" file="scalfmm.sqlite3">
+ <key>gitcommit</key>
+ <key>gitcommitdate</key>
+ <key>hostname</key>
+ <key>ndim</key>
+ <key>kernel_type</key>
+ <key>interp_type</key>
+ <key>nrun</key>
+ <key>tree_height</key>
+ <key>interp_order</key>
+ <key>size</key>
+ <key>nthread</key>
+ <key>groupsize</key>
+ <key>timefull_avg</key>
+ </database>
+ </result>
+
+ </benchmark>
+</jube>
diff --git a/scripts/scalfmm-time-seq.xml b/scripts/scalfmm-time-seq.xml
new file mode 100644
index 000000000..43092c6f1
--- /dev/null
+++ b/scripts/scalfmm-time-seq.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<jube>
+ <benchmark name="scalfmm_time_seq" outpath="results">
+ <comment>scalfmm sequential timing tests</comment>
+
+ <!-- Cartesian product of input parameters -->
+ <parameterset name="param">
+ <parameter name="ndim" type="int" >1, 2, 3</parameter>
+ <!-- <parameter name="ndim" type="int" >2</parameter> -->
+ <parameter name="kernel_type" type="int" >0</parameter>
+ <parameter name="interp_type" type="int" >1, 2, 4</parameter>
+ <!-- <parameter name="interp_type" type="int" >1</parameter> -->
+ <parameter name="nrun" type="int" >1</parameter>
+ <parameter name="nthread" type="int" >1</parameter>
+ <parameter name="groupsize" type="int" >1</parameter>
+ <parameter name="tree_height" type="int" >3, 4, 5</parameter>
+ <!-- <parameter name="tree_height" type="int" >3</parameter> -->
+ <parameter name="interp_order" type="int" >5</parameter>
+ <parameter name="size" type="int" >1000, 5000, 10000, 50000, 100000, 500000, 1000000</parameter>
+ <!-- <parameter name="size" type="int" >1000, 5000, 10000</parameter> -->
+ <parameter name="command" type="string">$SCALFMM_EXE_DIR/fmm-computation-seq --fmm-computation --dimension ${ndim} --group-size ${groupsize} --interp-settings ${interp_type} --kernel ${kernel_type} --order ${interp_order} --nb-runs ${nrun} --threads ${nthread} --size ${size} --tree-height ${tree_height}</parameter>
+ </parameterset>
+
+ <!-- Execution commands -->
+ <step name="run">
+ <use>param</use>
+ <do>echo "Commit = $GIT_COMMIT"</do>
+ <do>echo "Date = $GIT_COMMIT_DATE"</do>
+ <do>echo "Hostname = $CI_HOSTNAME"</do>
+ <do>$command</do>
+ </step>
+
+ <!-- Regex pattern -->
+ <patternset name="regexlog">
+ <pattern name="gitcommit" type="string">Commit = ([0-9a-f]{40}).*</pattern>
+ <pattern name="gitcommitdate" type="string">Date = (\d{4}\-\d{2}\-\d{2}\s\d{2}:\d{2}:\d{2}\s\+\d{4}).*</pattern>
+ <pattern name="hostname" type="string">Hostname = $jube_pat_wrd</pattern>
+ <pattern name="timefar" type="float">\[time\]\[far time\] : $jube_pat_fp</pattern>
+ <pattern name="timenear" type="float">\[time\]\[near time\] : $jube_pat_fp</pattern>
+ <pattern name="timefull" type="float">\[time\]\[full algo\] : $jube_pat_fp</pattern>
+ </patternset>
+
+ <!-- Analyse -->
+ <analyser name="parse">
+ <!-- use a pattern set -->
+ <use>regexlog</use>
+ <analyse step="run">
+ <file>stdout</file> <!-- file which should be scanned -->
+ </analyse>
+ </analyser>
+
+ <!-- Create result table -->
+ <result>
+ <use>parse</use> <!-- use existing analyser -->
+ <table name="timeseq_csv" style="csv">
+ <column>gitcommit</column>
+ <column>gitcommitdate</column>
+ <column>hostname</column>
+ <column>ndim</column>
+ <column>kernel_type</column>
+ <column>interp_type</column>
+ <column>nrun</column>
+ <column>tree_height</column>
+ <column>interp_order</column>
+ <column>size</column>
+ <column>timefar_avg</column>
+ <column>timenear_avg</column>
+ <column>timefull_avg</column>
+ </table>
+ <database name="timeseq" primekeys="gitcommit,gitcommitdate,hostname,ndim,kernel_type,interp_type,nrun,tree_height,interp_order,size,timefar_avg,timenear_avg,timefull_avg" file="scalfmm.sqlite3">
+ <key>gitcommit</key>
+ <key>gitcommitdate</key>
+ <key>hostname</key>
+ <key>ndim</key>
+ <key>kernel_type</key>
+ <key>interp_type</key>
+ <key>nrun</key>
+ <key>tree_height</key>
+ <key>interp_order</key>
+ <key>size</key>
+ <key>timefar_avg</key>
+ <key>timenear_avg</key>
+ <key>timefull_avg</key>
+ </database>
+ </result>
+
+ </benchmark>
+</jube>
--
GitLab