diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8cdec8f3a6a1233bbe83576450e3919127ac9beb..73c869b682e941d2d2a4d3c2eaed0b4e19d47747 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,6 +78,46 @@ test: # coverage_format: cobertura # path: coverage.xml +benchmark: + stage: test + tags: ['plafrim'] + #extends: .only-main-mr + needs: [] + variables: + CI_HOSTNAME: zonda + cache: + key: "$CI_JOB_NAME_SLUG-$CI_COMMIT_REF_SLUG" + paths: + - build + parallel: + matrix: + - CI_BENCHMARK: ['accuracy', 'timeseq', 'timeomp'] + script: + - ./scripts/plafrim_level1.sh + artifacts: + when: always + paths: + - ./*.csv + - ./*.log + - ./scripts/results/ + +database: + stage: deploy + tags: ['plafrim'] + #extends: .only-main-mr + needs: ['benchmark'] + script: + - guix time-machine -C .guix/scalfmm-channels.scm -- shell --pure --preserve="^CI|proxy$" + -m .guix/scalfmm-manifest-gcc-mkl-bench.scm + -- /bin/bash --norc ./scripts/database.sh + artifacts: + when: always + paths: + - ./*.csv + - ./*.png + - ./*.sqlite3 + - ./scripts/results/ + pages: stage: deploy when: manual diff --git a/CMakePresets.json b/CMakePresets.json index 9d8a6ce00f33f4670544fec76653005b2f7b420a..213d24b58186aacd41b2d0278d565321cc8f1d33 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -59,6 +59,13 @@ "description": "Build all targets in Release mode with OpenBLAS.", "configurePreset": "default" }, + { + "name": "build-mkl", + "inherits": "base", + "displayName": "Default build (MKL).", + "description": "Build all targets in Release mode with MKL.", + "configurePreset": "mkl" + }, { "name": "build-bench-default", "inherits": "bench", diff --git a/guix-tools/scalfmm-manifest-gcc-mkl-bench.scm b/guix-tools/scalfmm-manifest-gcc-mkl-bench.scm new file mode 100644 index 0000000000000000000000000000000000000000..7fa7b2a66ec172845f0a9009d802218caab230d6 --- /dev/null +++ b/guix-tools/scalfmm-manifest-gcc-mkl-bench.scm @@ -0,0 +1,33 @@ +;; What follows is a "manifest" equivalent to the command line you gave. +;; You can store it in a file that you may then pass to any 'guix' command +;; that accepts a '--manifest' (or '-m') option. + +(specifications->manifest + (list "bash" + "cmake" + "coreutils" + "curl" + "findutils" + "gcc-toolchain@11" + "git" + "grep" + "inetutils" + "intel-oneapi-mkl" + "jq" + "jube-with-yaml" + "make" + "ncurses" + "nss-certs" + "python" + "python-certifi" + "python-click" + "python-colorama" + "python-elasticsearch" + "python-matplotlib" + "python-pandas" + "python-pyyaml" + "python-setuptools" + "sed" + "slurm" + "sqlite")) + diff --git a/scripts/add_result.py b/scripts/add_result.py new file mode 100755 index 0000000000000000000000000000000000000000..87b68fa45080fe1928470655f405c26307ef3ed9 --- /dev/null +++ b/scripts/add_result.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python +from typing import Any, Dict, List, Union +import click +import csv +from elasticsearch import Elasticsearch + + +Row = Dict[str, Union[str, float]] + + +def open_csv(filename: str) -> List[Dict[str, str]]: + """ + Open a csv file a return it as dictionary. + First row is titles. + """ + csv_rows = [] + with open(filename) as csv_data: + reader = csv.DictReader(csv_data) + titles = reader.fieldnames + for row in reader: + csv_rows.append( + { + title: row[title] + for title in titles + } + ) + return csv_rows + +def format_entry_accuracy(row: Row) -> Dict[str, Any]: + """"format a result""" + commit_sha = str(row.pop('gitcommit')) + commit_date = str(row.pop('gitcommitdate')) + hostname = str(row.pop('hostname')) + ndim = int(row.pop('ndim')) + kernel_type = int(row.pop('kernel_type')) + interp_type = int(row.pop('interp_type')) + tree_height = int(row.pop('tree_height')) + interp_order = int(row.pop('interp_order')) + error = float(row.pop('error')) + result = { + "Commit_date": commit_date, + "Commit_sha": commit_sha, + "Hostname": hostname, + "Ndim": ndim, + "Kernel_type": kernel_type, + "Interp_type": interp_type, + "Tree_height": tree_height, + "Interp_order": interp_order, + "Error": error + } + return result + +def format_entry_timeseq(row: Row) -> Dict[str, Any]: + """"format a result""" + commit_sha = str(row.pop('gitcommit')) + commit_date = str(row.pop('gitcommitdate')) + hostname = str(row.pop('hostname')) + ndim = int(row.pop('ndim')) + kernel_type = int(row.pop('kernel_type')) + interp_type = int(row.pop('interp_type')) + nrun = int(row.pop('nrun')) + tree_height = int(row.pop('tree_height')) + interp_order = int(row.pop('interp_order')) + size = int(row.pop('size')) + timefar_avg = float(row.pop('timefar_avg')) + timenear_avg = float(row.pop('timenear_avg')) + timefull_avg = float(row.pop('timefull_avg')) + + result = { + "Commit_date": commit_date, + "Commit_sha": commit_sha, + "Hostname": hostname, + "Ndim": ndim, + "Kernel_type": kernel_type, + "Interp_type": interp_type, + "Nrun": nrun, + "Tree_height": tree_height, + "Interp_order": interp_order, + "Size": size, + "Timefar_avg": timefar_avg, + "Timenear_avg": timenear_avg, + "Timefull_avg": timefull_avg + } + return result + +def format_entry_timeomp(row: Row) -> Dict[str, Any]: + """"format a result""" + commit_sha = str(row.pop('gitcommit')) + commit_date = str(row.pop('gitcommitdate')) + hostname = str(row.pop('hostname')) + ndim = int(row.pop('ndim')) + kernel_type = int(row.pop('kernel_type')) + interp_type = int(row.pop('interp_type')) + nrun = int(row.pop('nrun')) + tree_height = int(row.pop('tree_height')) + interp_order = int(row.pop('interp_order')) + size = int(row.pop('size')) + nthread = int(row.pop('nthread')) + groupsize = int(row.pop('groupsize')) + timefull_avg = float(row.pop('timefull_avg')) + + result = { + "Commit_date": commit_date, + "Commit_sha": commit_sha, + "Hostname": hostname, + "Ndim": ndim, + "Kernel_type": kernel_type, + "Interp_type": interp_type, + "Nrun": nrun, + "Tree_height": tree_height, + "Interp_order": interp_order, + "Size": size, + "Nthread": nthread, + "Groupsize": groupsize, + "Timefull_avg": timefull_avg + } + return result + +@click.command() +@click.option("-e", "--elastic-url", default="http://localhost:9200", help="elasticsearch instance url") +@click.option("-t", "--team", required=True, help="team name") +@click.option("-p", "--project", required=True, help="project name") +@click.option("-n", "--name", required=True, help="Table name") +@click.argument("csv-files", nargs=-1) +def main( + elastic_url: str, + team: str, + project: str, + name: str, + csv_files: str, +): + """Add a result to an elasticsearch database.""" + es = Elasticsearch(elastic_url) + info = es.info() + print("Elasticsearch version:", info["version"]["number"]) + + es_index = team + "-" + project + "-" + name + if not es.indices.exists(index=es_index): + es.indices.create(index=es_index) + + mapping_input_accuracy = { + "properties": { + "Commit_sha": {"type": "keyword"}, + "Commit_date": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss Z"}, + "Hostname": {"type": "keyword"}, + "Ndim": {"type": "integer"}, + "Kernel_type": {"type": "integer"}, + "Interp_type": {"type": "integer"}, + "Tree_height": {"type": "integer"}, + "Interp_order": {"type": "integer"}, + "Error": {"type": "float"} + } + } + mapping_input_timeseq = { + "properties": { + "Commit_sha": {"type": "keyword"}, + "Commit_date": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss Z"}, + "Hostname": {"type": "keyword"}, + "Ndim": {"type": "integer"}, + "Kernel_type": {"type": "integer"}, + "Interp_type": {"type": "integer"}, + "Nrun": {"type": "integer"}, + "Tree_height": {"type": "integer"}, + "Interp_order": {"type": "integer"}, + "Size": {"type": "integer"}, + "Timefar_avg": {"type": "float"}, + "Timenear_avg": {"type": "float"}, + "Timefull_avg": {"type": "float"} + } + } + mapping_input_timeomp = { + "properties": { + "Commit_sha": {"type": "keyword"}, + "Commit_date": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss Z"}, + "Hostname": {"type": "keyword"}, + "Ndim": {"type": "integer"}, + "Kernel_type": {"type": "integer"}, + "Interp_type": {"type": "integer"}, + "Nrun": {"type": "integer"}, + "Tree_height": {"type": "integer"}, + "Interp_order": {"type": "integer"}, + "Size": {"type": "integer"}, + "Nthread": {"type": "integer"}, + "Groupsize": {"type": "integer"}, + "Timefull_avg": {"type": "float"} + } + } + + if name == "accuracy": + es.indices.put_mapping(index=es_index, body=mapping_input_accuracy) + elif name == "timeseq": + es.indices.put_mapping(index=es_index, body=mapping_input_timeseq) + elif name == "timeomp": + es.indices.put_mapping(index=es_index, body=mapping_input_timeomp) + + requests = [ + request + for file in csv_files + for request in map( + lambda row: format_entry_accuracy(row) if name == "accuracy" else format_entry_timeseq(row) if name == "timeseq" else format_entry_timeomp(row), + open_csv(file) + ) + ] + for request in requests: + es.index(index=es_index.lower(), body=request) + + +if __name__ == "__main__": + main() diff --git a/scripts/bench-time-seq.sh b/scripts/bench-time-seq.sh index 53aecf977577af3e4319efce39c1f2760fb70d3a..dae3ae5d7ef77f101fc374f4e1fe40a2c5a8c3ea 100644 --- a/scripts/bench-time-seq.sh +++ b/scripts/bench-time-seq.sh @@ -1,24 +1,26 @@ #!/usr/bin/env bash +export HFI_NO_CPUAFFINITY=1 + suffix="$1" # configuration and compilation -build_dir="build${suffix}" -rm -rf "${build_dir}" # start from scratch +build_dir="build-${suffix}" +#rm -rf "${build_dir}" # start from scratch cmake -B "${build_dir}" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-O3 -march=native" -Dscalfmm_BUILD_BENCH=ON cmake --build "${build_dir}" --target fmm-computation-seq exec_file="${build_dir}/bench/Release/fmm-computation-seq" # changing parameters +dimension=(1 2 3) order_values=(5) interp_settings_values=(1 2 4) -size_values=(1000 5000 10000 50000 100000) # 500000 1000000) +size_values=(1000 5000 10000 50000 100000 500000 1000000) tree_height_values=(3 4 5) kernel_values=(0) # fixed parameters log_file="tmp-log-buffer.txt" -dimension=2 nb_threads=1 # fully sequential ! group_size=1 nb_runs=5 @@ -59,7 +61,6 @@ do --group-size "${group_size}" \ --nb-runs "${nb_runs}" \ --kernel "${kernel}" \ - --input-file "${input_file}" \ --threads "${nb_threads}" \ --interp-settings "${interp_settings}" | tee "${log_file}" diff --git a/scripts/bench-time-task-dep.sh b/scripts/bench-time-task-dep.sh index 330e7e4acd7160bedb2111157c0ff48ce83003eb..d55ae5fde02734697a264041a1ecbcd8e6bbe8e4 100644 --- a/scripts/bench-time-task-dep.sh +++ b/scripts/bench-time-task-dep.sh @@ -1,22 +1,24 @@ #!/usr/bin/env bash +export HFI_NO_CPUAFFINITY=1 + suffix="$1" # configuration and compilation -build_dir="build${suffix}" -rm -rf "${build_dir}" # start from scratch +build_dir="build-${suffix}" +#rm -rf "${build_dir}" # start from scratch cmake -B "${build_dir}" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-O3 -march=native" -Dscalfmm_BUILD_BENCH=ON cmake --build "${build_dir}" --target fmm-computation-omp exec_file="${build_dir}/bench/Release/fmm-computation-omp" # changing parameters +dimension=(1 2 3) nb_threads_values=(1 2 4 8 16) -group_size_values=(1 5 10 50 100) +group_size_values=(1 5 10 100 1000) tree_height_values=(3 4 5) # fixed parameters log_file="tmp-log-buffer.txt" -dimension=2 kernel=0 order=5 nb_runs=5 @@ -55,7 +57,6 @@ do --group-size "${group_size}" \ --nb-runs "${nb_runs}" \ --kernel "${kernel}" \ - --input-file "${input_file}" \ --threads "${nb_threads}" \ --interp-settings "${interp_settings}" | tee "${log_file}" diff --git a/scripts/database.sh b/scripts/database.sh new file mode 100755 index 0000000000000000000000000000000000000000..53d5741efeeec01830e93abc7dae75780a17afd3 --- /dev/null +++ b/scripts/database.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -ex + +# get current database file scalfmm.sqlite3 stored on gitlab +if [[ ! -z "${CI_JOB_TOKEN}" ]]; then + export PACKAGEID=`curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages" |jq '.[0].id'` + if [[ "${PACKAGEID}" != "null" ]]; then + export FILEID=`curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages/$PACKAGEID/package_files" |jq '.[0].id'` + if [[ "${FILEID}" != "null" ]]; then + curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages/generic/benchmark/latest/scalfmm.sqlite3" -o scalfmm.sqlite3 + fi + fi +fi + +# update scalfmm.sqlite3 database, tables : accuracy, timeseq, timeomp +jube result scripts/results --id 1 -o accuracy +jube result scripts/results --id 2 -o timeseq +jube result scripts/results --id 3 -o timeomp + +# upload updated scalfmm.sqlite3 on gitlab +if [[ ! -z "${CI_JOB_TOKEN}" ]]; then + # get package id of the database stored on gitlab + export PACKAGEID=`curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages" |jq '.[0].id'` + if [[ "${PACKAGEID}" != "null" ]]; then + # get file id of the database on gitlab + export FILEID=`curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages/$PACKAGEID/package_files" |jq '.[0].id'` + if [[ "${FILEID}" != "null" ]]; then + # delete previous database version on gitlab if exists + curl --request DELETE --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages/$PACKAGEID/package_files/$FILEID" + fi + fi + curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ./scalfmm.sqlite3 "https://gitlab.inria.fr/api/v4/projects/$CI_PROJECT_ID/packages/generic/benchmark/latest/scalfmm.sqlite3" +fi + +# upload results to the elasticsearch server +python3 ./scripts/add_result.py -e https://elasticsearch.bordeaux.inria.fr -t concace -p scalfmm -n accuracy scalfmm_accuracy.csv +python3 ./scripts/add_result.py -e https://elasticsearch.bordeaux.inria.fr -t concace -p scalfmm -n timeseq scalfmm_timeseq.csv +python3 ./scripts/add_result.py -e https://elasticsearch.bordeaux.inria.fr -t concace -p scalfmm -n timeomp scalfmm_timeomp.csv + +# plot some figures at this precise commit +python3 ./scripts/plot.py -f scalfmm_accuracy.csv -n accuracy +python3 ./scripts/plot.py -f scalfmm_timeseq.csv -n timeseq +python3 ./scripts/plot.py -f scalfmm_timeomp.csv -n timeomp diff --git a/scripts/experiment-bora-accuracy.slurm b/scripts/experiment-suroit-accuracy.slurm similarity index 62% rename from scripts/experiment-bora-accuracy.slurm rename to scripts/experiment-suroit-accuracy.slurm index 2e41beb3a15b745de2a67c18477259d64e51482c..b5ab2a5f46c963fcc439af56a018d121083eeb35 100644 --- a/scripts/experiment-bora-accuracy.slurm +++ b/scripts/experiment-suroit-accuracy.slurm @@ -1,10 +1,10 @@ #!/usr/bin/env bash -#SBATCH --job-name=experiment-bora-accuracy +#SBATCH --job-name=scalfmm-suroit-accuracy #SBATCH --nodes=1 -#SBATCH --time=3:00:00 -#SBATCH -C "bora" -#SBATCH --output=out-experiment-bora-accuracy-%j.txt -#SBATCH --error=err-experiment-bora-accuracy-%j.txt +#SBATCH --exclusive +#SBATCH --time=00:30:00 +#SBATCH -C "suroit" +#SBATCH --output=scalfmm-suroit-accuracy.log manifest_file=".guix/scalfmm-manifest-gcc11-openblas.scm" channels_file=".guix/scalfmm-channels.scm" diff --git a/scripts/experiment-bora-bench-time-seq.slurm b/scripts/experiment-suroit-bench-time-seq.slurm similarity index 64% rename from scripts/experiment-bora-bench-time-seq.slurm rename to scripts/experiment-suroit-bench-time-seq.slurm index a0373c90bd4ebee4bf9ea02b7649ef46a8e4ad3a..e77457f926a5548a965043f452405915b401e2fc 100644 --- a/scripts/experiment-bora-bench-time-seq.slurm +++ b/scripts/experiment-suroit-bench-time-seq.slurm @@ -1,10 +1,10 @@ #!/usr/bin/env bash -#SBATCH --job-name=experiment-bora-bench-time-seq +#SBATCH --job-name=scalfmm-suroit-time-seq #SBATCH --nodes=1 +#SBATCH --exclusive #SBATCH --time=3:00:00 -#SBATCH -C "bora" -#SBATCH --output=out-experiment-bora-bench-time-seq-%j.txt -#SBATCH --error=err-experiment-bora-bench-time-seq-%j.txt +#SBATCH -C "suroit" +#SBATCH --output=scalfmm-suroit-time-seq.log manifest_file=".guix/scalfmm-manifest-gcc11-openblas.scm" channels_file=".guix/scalfmm-channels.scm" diff --git a/scripts/experiment-bora-bench-time-task-dep.slurm b/scripts/experiment-suroit-bench-time-task-dep.slurm similarity index 62% rename from scripts/experiment-bora-bench-time-task-dep.slurm rename to scripts/experiment-suroit-bench-time-task-dep.slurm index dbb52a5f6e552b2ee970cddccb376bf79ee0fcab..21931e8d5b37951861eb0fe393b4f1f52665688f 100644 --- a/scripts/experiment-bora-bench-time-task-dep.slurm +++ b/scripts/experiment-suroit-bench-time-task-dep.slurm @@ -1,10 +1,10 @@ #!/usr/bin/env bash -#SBATCH --job-name=experiment-bora-bench-time-task-dep +#SBATCH --job-name=scalfmm-suroit-time-task-dep #SBATCH --nodes=1 +#SBATCH --exclusive #SBATCH --time=3:00:00 -#SBATCH -C "bora" -#SBATCH --output=out-experiment-bora-bench-time-task-dep-%j.txt -#SBATCH --error=err-experiment-bora-bench-time-task-dep-%j.txt +#SBATCH -C "suroit" +#SBATCH --output=scalfmm-suroit-time-task-dep.log manifest_file=".guix/scalfmm-manifest-gcc11-openblas.scm" channels_file=".guix/scalfmm-channels.scm" diff --git a/scripts/get_result.py b/scripts/get_result.py new file mode 100755 index 0000000000000000000000000000000000000000..023080ceda706514be8d13bc793a6ce70040955d --- /dev/null +++ b/scripts/get_result.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +import pandas +import click +import csv +from elasticsearch import Elasticsearch + +@click.command() +@click.option("-e", "--elastic-url", default="http://localhost:9200", help="elasticsearch instance url") +@click.option("-t", "--team", required=True, help="team name") +@click.option("-p", "--project", required=True, help="project name") +@click.option("-n", "--name", required=True, help="Table name") +@click.option("-c", "--commit", required=True, help="project commit") +def main( + elastic_url: str, + team: str, + project: str, + name: str, + commit: str +): + """Get a result from an elasticsearch database, e.g. + https://elasticsearch.bordeaux.inria.fr.""" + es = Elasticsearch(elastic_url) + es_index = team + "-" + project + "-" + name + + search_param = { + "query": { + "bool": { + "must": [ + {"term": {"Commit_sha": {"value": commit}}} + ] + } + }, + "size": 1000 + } + response = es.search(index=es_index, body=search_param) + elastic_docs = response["hits"]["hits"] + + docs = pandas.DataFrame() + for num, doc in enumerate(elastic_docs): + + # get _source data dict from document + source_data = doc["_source"] + + # get _id from document + _id = doc["_id"] + + # create a Series object from doc dict object + doc_data = pandas.Series(source_data, name = _id) + doc_data = doc_data.drop(labels=['Commit_date', 'Commit_sha']) + + # append the Series object to the DataFrame object + docs = pandas.concat([docs, doc_data.to_frame().T]) + + if name == 'accuracy': + docs = docs.astype({"Hostname": str, "Ndim": int, "Kernel_type": int, "Interp_type": int, "Tree_height": int, "Interp_order": int, "Error": float}) + elif name == 'timeseq': + docs = docs.astype({"Hostname": str, "Ndim": int, "Kernel_type": int, "Interp_type": int, "Tree_height": int, "Interp_order": int, "Size": int, "Timefar_avg": float, "Timenear_avg": float, "Timefull_avg": float}) + elif name == 'timeomp': + docs = docs.astype({"Hostname": str, "Ndim": int, "Kernel_type": int, "Interp_type": int, "Nrun": int, "Tree_height": int, "Interp_order": int, "Size": int, "Nthread": int, "Groupsize": int, "Timefull_avg": float}) + + docs = docs.rename(columns=str.lower) + docs.to_csv("scalfmm_" + name + ".csv", ",", index=False) + +if __name__ == "__main__": + main() diff --git a/scripts/plafrim_level1.sh b/scripts/plafrim_level1.sh new file mode 100755 index 0000000000000000000000000000000000000000..243c56fab8da67a6cb4a06d3945dd595a92916d2 --- /dev/null +++ b/scripts/plafrim_level1.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -x + +# Submit slurm jobs + +# number of machine nodes +SLURM_NNODES=1 +# number of MPI tasks +SLURM_NTASKS=1 +# number of CPU cores per task +SLURM_NCORES=1 +SLURM_TIME=02:00:00 + +# to change defaults slurm parameters if needed +case "${CI_BENCHMARK}" in + accuracy) + SLURM_NAME=scalfmm_accuracy + ;; + timeseq) + SLURM_NAME=scalfmm_timeseq + ;; + timeomp) + SLURM_NAME=scalfmm_timeomp + SLURM_NCORES=16 + ;; + *) + echo "CI_BENCHMARK is set to an unknown value." + exit 1 + ;; +esac + +sbatch --wait --job-name=${SLURM_NAME} --output=${SLURM_NAME}.log -N ${SLURM_NNODES} -n ${SLURM_NTASKS} -c ${SLURM_NCORES} \ + --time=${SLURM_TIME} --constraint=${CI_HOSTNAME} ./scripts/plafrim_level2.sh +err=$? + +cat scalfmm.log + +# exit with error code from the sbatch command +exit $err diff --git a/scripts/plafrim_level2.sh b/scripts/plafrim_level2.sh new file mode 100755 index 0000000000000000000000000000000000000000..c54923fd5432f42f4ad976b68294d884ec0fcae4 --- /dev/null +++ b/scripts/plafrim_level2.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -x + +exec guix time-machine -C .guix/scalfmm-channels.scm -- shell --pure --preserve="^CI|proxy$|^SLURM" \ + -m .guix/scalfmm-manifest-gcc-mkl-bench.scm \ + -- /bin/bash --norc ./scripts/plafrim_level3.sh +err=$? + +# exit with error code from the guix command +exit $err diff --git a/scripts/plafrim_level3.sh b/scripts/plafrim_level3.sh new file mode 100755 index 0000000000000000000000000000000000000000..bd3e8dbcc8037ca0e96fdc11081151ff133e22a0 --- /dev/null +++ b/scripts/plafrim_level3.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -ex + +export HFI_NO_CPUAFFINITY=1 +export MKLROOT=$GUIX_ENVIRONMENT + +if [[ -z "${CI_COMMIT_SHA}" ]]; then + export GIT_COMMIT=`git rev-parse HEAD` +else + export GIT_COMMIT=$CI_COMMIT_SHA + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0=safe.directory + export GIT_CONFIG_VALUE_0=* +fi +export GIT_COMMIT_DATE=`git show --no-patch --format=%ci $GIT_COMMIT` + +if [[ -z "${CI_HOSTNAME}" ]]; then + export CI_HOSTNAME=`hostname` +fi + +export SCALFMM_EXE_DIR=$PWD/build/bench/Release + +# clean old benchmarks +if [ -d scripts/results ]; then + rm scripts/results -r +fi + +# configuration +cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-O3 -march=native" -Dscalfmm_BUILD_BENCH=ON -Dscalfmm_USE_MKL=ON + +# compilation and run benchmarks +case "${CI_BENCHMARK}" in + accuracy) + echo "CI_BENCHMARK is set to accuracy." + # compilation + cmake --build build --target fmm-computation-seq + # run benchmarks + jube run scripts/scalfmm-accuracy.xml --id 1 + # parse result log files + jube analyse scripts/results --id 1 + # generate csv file + jube result scripts/results --id 1 -o accuracy_csv > scalfmm_accuracy.csv + cat scalfmm_accuracy.csv + # update scalfmm.sqlite3 database, table accuracy + #jube result scripts/results --id 1 -o accuracy + ;; + timeseq) + echo "CI_BENCHMARK is set to timeseq." + cmake --build build --target fmm-computation-seq + jube run scripts/scalfmm-time-seq.xml --id 2 + jube analyse scripts/results --id 2 + jube result scripts/results --id 2 -o timeseq_csv > scalfmm_timeseq.csv + cat scalfmm_timeseq.csv + #jube result scripts/results --id 2 -o timeseq + ;; + timeomp) + echo "CI_BENCHMARK is set to timeomp." + cmake --build build --target fmm-computation-omp + jube run scripts/scalfmm-time-omp.xml --id 3 + jube analyse scripts/results --id 3 + jube result scripts/results --id 3 -o timeomp_csv > scalfmm_timeomp.csv + cat scalfmm_timeomp.csv + #jube result scripts/results --id 3 -o timeomp + ;; + *) + echo "CI_BENCHMARK is set to an unknown value." + exit 1 + ;; +esac diff --git a/scripts/plot.py b/scripts/plot.py new file mode 100644 index 0000000000000000000000000000000000000000..d56a62c9a21092e823958e1114345efd2ddec329 --- /dev/null +++ b/scripts/plot.py @@ -0,0 +1,52 @@ +import click +import pandas as pd +import matplotlib.pyplot as plt + +@click.command() +@click.option("-f", "--file", required=True, help="input csv file") +@click.option("-n", "--name", required=True, help="benchmark name") +def main( + file: str, + name: str, +): + """Generate figure from a csv file""" + + df = pd.read_csv(file) + + if name == "accuracy": + for (ndim, kernel_type, interp_type, tree_height), group in df.groupby(['ndim', 'kernel_type', 'interp_type', 'tree_height']): + x = group['interp_order'] + y = group['error'] + plt.plot(x, y, label=f"{ndim}-{kernel_type}-{interp_type}-{tree_height}") + plt.yscale('log') + + plt.xlabel('Order') + plt.ylabel('Relative norm-2 error') + plt.title('Error vs. Order') + plt.legend() + plt.savefig('scalfmm_accuracy.png', bbox_inches='tight') + elif name == "timeseq": + for (ndim, kernel_type, interp_type, tree_height, interp_order), group in df.groupby(['ndim', 'kernel_type', 'interp_type', 'tree_height', 'interp_order']): + x = group['size'] + y = group['timefull_avg'] + plt.plot(x, y, label=f"{ndim}-{kernel_type}-{interp_type}-{tree_height}-{interp_order}") + + plt.xlabel('Size') + plt.ylabel('Time (s)') + plt.title('Time vs. Size') + plt.legend() + plt.savefig('scalfmm_timeseq.png', bbox_inches='tight') + elif name == "timeomp": + for (ndim, kernel_type, interp_type, tree_height, interp_order, size, groupsize), group in df.groupby(['ndim', 'kernel_type', 'interp_type', 'tree_height', 'interp_order', 'size', 'groupsize']): + x = group['nthread'] + y = group['timefull_avg'] + plt.plot(x, y, label=f"{ndim}-{kernel_type}-{interp_type}-{tree_height}-{interp_order}-{size}-{groupsize}") + + plt.xlabel('Number of threads') + plt.ylabel('Time (s)') + plt.title('Time vs. Number of threads') + plt.legend() + plt.savefig('scalfmm_timeomp.png', bbox_inches='tight') + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/scalfmm-accuracy.xml b/scripts/scalfmm-accuracy.xml new file mode 100644 index 0000000000000000000000000000000000000000..77073acf27337fc7b44bf0eef907efeeb677caa9 --- /dev/null +++ b/scripts/scalfmm-accuracy.xml @@ -0,0 +1,77 @@ +<?xml version="1.0" encoding="UTF-8"?> +<jube> + <benchmark name="scalfmm_accuracy" outpath="results"> + <comment>scalfmm accuracy tests</comment> + + <!-- Cartesian product of input parameters --> + <parameterset name="param"> + <parameter name="ndim" type="int" >1, 2, 3</parameter> + <!-- <parameter name="ndim" type="int" >2, 3</parameter> --> + <parameter name="kernel_type" type="int" >0, 6, 7</parameter> + <!-- <parameter name="kernel_type" type="int" >0</parameter> --> + <parameter name="interp_type" type="int" >0, 1, 2, 3, 4</parameter> + <!-- <parameter name="interp_type" type="int" >3</parameter> --> + <parameter name="nrun" type="int" >1</parameter> + <parameter name="nthread" type="int" >4</parameter> + <parameter name="groupsize" type="int" >1</parameter> + <parameter name="tree_height" type="int" >3, 4, 5</parameter> + <parameter name="interp_order" type="int" >3, 4, 5, 6, 7, 8, 9, 10, 11, 12</parameter> + <!-- <parameter name="interp_order" type="int" >3, 4</parameter> --> + <parameter name="command" type="string">$SCALFMM_EXE_DIR/fmm-computation-seq --direct-computation --fmm-computation --dimension ${ndim} --group-size ${groupsize} --interp-settings ${interp_type} --kernel ${kernel_type} --order ${interp_order} --nb-runs ${nrun} --threads ${nthread} --tree-height ${tree_height}</parameter> + </parameterset> + + <!-- Execution commands --> + <step name="run"> + <use>param</use> + <do>echo "Commit = $GIT_COMMIT"</do> + <do>echo "Date = $GIT_COMMIT_DATE"</do> + <do>echo "Hostname = $CI_HOSTNAME"</do> + <do>$command</do> + </step> + + <!-- Regex pattern --> + <patternset name="regexlog"> + <pattern name="gitcommit" type="string">Commit = ([0-9a-f]{40}).*</pattern> + <pattern name="gitcommitdate" type="string">Date = (\d{4}\-\d{2}\-\d{2}\s\d{2}:\d{2}:\d{2}\s\+\d{4}).*</pattern> + <pattern name="hostname" type="string">Hostname = $jube_pat_wrd</pattern> + <pattern name="error" type="float">Relative L2-norm = $jube_pat_fp</pattern> + </patternset> + + <!-- Analyse --> + <analyser name="parse"> + <!-- use a pattern set --> + <use>regexlog</use> + <analyse step="run"> + <file>stdout</file> <!-- file which should be scanned --> + </analyse> + </analyser> + + <!-- Create result table --> + <result> + <use>parse</use> <!-- use existing analyser --> + <table name="accuracy_csv" style="csv"> + <column>gitcommit</column> + <column>gitcommitdate</column> + <column>hostname</column> + <column>ndim</column> + <column>kernel_type</column> + <column>interp_type</column> + <column>tree_height</column> + <column>interp_order</column> + <column>error</column> + </table> + <database name="accuracy" primekeys="gitcommit,gitcommitdate,hostname,ndim,kernel_type,interp_type,tree_height,interp_order,error" file="scalfmm.sqlite3"> + <key>gitcommit</key> + <key>gitcommitdate</key> + <key>hostname</key> + <key>ndim</key> + <key>kernel_type</key> + <key>interp_type</key> + <key>tree_height</key> + <key>interp_order</key> + <key>error</key> + </database> + </result> + + </benchmark> +</jube> diff --git a/scripts/scalfmm-time-omp.xml b/scripts/scalfmm-time-omp.xml new file mode 100644 index 0000000000000000000000000000000000000000..276c4602210cd21a529de9a4eb764b316f439e3c --- /dev/null +++ b/scripts/scalfmm-time-omp.xml @@ -0,0 +1,86 @@ +<?xml version="1.0" encoding="UTF-8"?> +<jube> + <benchmark name="scalfmm_time_seq" outpath="results"> + <comment>scalfmm OMP-tasks timing tests</comment> + + <!-- Cartesian product of input parameters --> + <parameterset name="param"> + <parameter name="ndim" type="int" >1, 2, 3</parameter> + <parameter name="ndim" type="int" >2</parameter> + <parameter name="kernel_type" type="int" >0</parameter> + <parameter name="interp_type" type="int" >2</parameter> + <parameter name="nrun" type="int" >1</parameter> + <parameter name="tree_height" type="int" >3, 4, 5</parameter> + <!-- <parameter name="tree_height" type="int" >5</parameter> --> + <parameter name="interp_order" type="int" >5</parameter> + <parameter name="size" type="int" >100000</parameter> + <parameter name="nthread" type="int" >1, 2, 4, 8, 16</parameter> + <!-- <parameter name="nthread" type="int" >1, 2, 4</parameter> --> + <parameter name="groupsize" type="int" >1, 5, 10, 100, 1000</parameter> + <!-- <parameter name="groupsize" type="int" >10, 100</parameter> --> + <parameter name="command" type="string">$SCALFMM_EXE_DIR/fmm-computation-omp --fmm-computation --dimension ${ndim} --group-size ${groupsize} --interp-settings ${interp_type} --kernel ${kernel_type} --order ${interp_order} --nb-runs ${nrun} --threads ${nthread} --size ${size} --tree-height ${tree_height}</parameter> + </parameterset> + + <!-- Execution commands --> + <step name="run"> + <use>param</use> + <do>echo "Commit = $GIT_COMMIT"</do> + <do>echo "Date = $GIT_COMMIT_DATE"</do> + <do>echo "Hostname = $CI_HOSTNAME"</do> + <do>$command</do> + </step> + + <!-- Regex pattern --> + <patternset name="regexlog"> + <pattern name="gitcommit" type="string">Commit = ([0-9a-f]{40}).*</pattern> + <pattern name="gitcommitdate" type="string">Date = (\d{4}\-\d{2}\-\d{2}\s\d{2}:\d{2}:\d{2}\s\+\d{4}).*</pattern> + <pattern name="hostname" type="string">Hostname = $jube_pat_wrd</pattern> + <pattern name="timefull" type="float">Full algo : $jube_pat_fp</pattern> + </patternset> + + <!-- Analyse --> + <analyser name="parse"> + <!-- use a pattern set --> + <use>regexlog</use> + <analyse step="run"> + <file>stdout</file> <!-- file which should be scanned --> + </analyse> + </analyser> + + <!-- Create result table --> + <result> + <use>parse</use> <!-- use existing analyser --> + <table name="timeomp_csv" style="csv"> + <column>gitcommit</column> + <column>gitcommitdate</column> + <column>hostname</column> + <column>ndim</column> + <column>kernel_type</column> + <column>interp_type</column> + <column>nrun</column> + <column>tree_height</column> + <column>interp_order</column> + <column>size</column> + <column>nthread</column> + <column>groupsize</column> + <column>timefull_avg</column> + </table> + <database name="timeomp" primekeys="gitcommit,gitcommitdate,hostname,ndim,kernel_type,interp_type,nrun,tree_height,interp_order,size,nthread,groupsize,timefull_avg" file="scalfmm.sqlite3"> + <key>gitcommit</key> + <key>gitcommitdate</key> + <key>hostname</key> + <key>ndim</key> + <key>kernel_type</key> + <key>interp_type</key> + <key>nrun</key> + <key>tree_height</key> + <key>interp_order</key> + <key>size</key> + <key>nthread</key> + <key>groupsize</key> + <key>timefull_avg</key> + </database> + </result> + + </benchmark> +</jube> diff --git a/scripts/scalfmm-time-seq.xml b/scripts/scalfmm-time-seq.xml new file mode 100644 index 0000000000000000000000000000000000000000..e6c6629ed20411f38cc49e01c3a11831720ff434 --- /dev/null +++ b/scripts/scalfmm-time-seq.xml @@ -0,0 +1,88 @@ +<?xml version="1.0" encoding="UTF-8"?> +<jube> + <benchmark name="scalfmm_time_seq" outpath="results"> + <comment>scalfmm sequential timing tests</comment> + + <!-- Cartesian product of input parameters --> + <parameterset name="param"> + <parameter name="ndim" type="int" >1, 2, 3</parameter> + <!-- <parameter name="ndim" type="int" >2</parameter> --> + <parameter name="kernel_type" type="int" >0</parameter> + <parameter name="interp_type" type="int" >1, 2, 4</parameter> + <!-- <parameter name="interp_type" type="int" >1</parameter> --> + <parameter name="nrun" type="int" >1</parameter> + <parameter name="nthread" type="int" >1</parameter> + <parameter name="groupsize" type="int" >1</parameter> + <parameter name="tree_height" type="int" >3, 4, 5</parameter> + <!-- <parameter name="tree_height" type="int" >3</parameter> --> + <parameter name="interp_order" type="int" >5</parameter> + <parameter name="size" type="int" >1000, 5000, 10000, 50000, 100000, 500000, 1000000</parameter> + <!-- <parameter name="size" type="int" >1000, 5000, 10000, 50000</parameter> --> + <parameter name="command" type="string">$SCALFMM_EXE_DIR/fmm-computation-seq --fmm-computation --dimension ${ndim} --group-size ${groupsize} --interp-settings ${interp_type} --kernel ${kernel_type} --order ${interp_order} --nb-runs ${nrun} --threads ${nthread} --size ${size} --tree-height ${tree_height}</parameter> + </parameterset> + + <!-- Execution commands --> + <step name="run"> + <use>param</use> + <do>echo "Commit = $GIT_COMMIT"</do> + <do>echo "Date = $GIT_COMMIT_DATE"</do> + <do>echo "Hostname = $CI_HOSTNAME"</do> + <do>$command</do> + </step> + + <!-- Regex pattern --> + <patternset name="regexlog"> + <pattern name="gitcommit" type="string">Commit = ([0-9a-f]{40}).*</pattern> + <pattern name="gitcommitdate" type="string">Date = (\d{4}\-\d{2}\-\d{2}\s\d{2}:\d{2}:\d{2}\s\+\d{4}).*</pattern> + <pattern name="hostname" type="string">Hostname = $jube_pat_wrd</pattern> + <pattern name="timefar" type="float">\[time\]\[far time\] : $jube_pat_fp</pattern> + <pattern name="timenear" type="float">\[time\]\[near time\] : $jube_pat_fp</pattern> + <pattern name="timefull" type="float">\[time\]\[full algo\] : $jube_pat_fp</pattern> + </patternset> + + <!-- Analyse --> + <analyser name="parse"> + <!-- use a pattern set --> + <use>regexlog</use> + <analyse step="run"> + <file>stdout</file> <!-- file which should be scanned --> + </analyse> + </analyser> + + <!-- Create result table --> + <result> + <use>parse</use> <!-- use existing analyser --> + <table name="timeseq_csv" style="csv"> + <column>gitcommit</column> + <column>gitcommitdate</column> + <column>hostname</column> + <column>ndim</column> + <column>kernel_type</column> + <column>interp_type</column> + <column>nrun</column> + <column>tree_height</column> + <column>interp_order</column> + <column>size</column> + <column>timefar_avg</column> + <column>timenear_avg</column> + <column>timefull_avg</column> + </table> + <database name="timeseq" primekeys="gitcommit,gitcommitdate,hostname,ndim,kernel_type,interp_type,nrun,tree_height,interp_order,size,timefar_avg,timenear_avg,timefull_avg" file="scalfmm.sqlite3"> + <key>gitcommit</key> + <key>gitcommitdate</key> + <key>hostname</key> + <key>ndim</key> + <key>kernel_type</key> + <key>interp_type</key> + <key>nrun</key> + <key>tree_height</key> + <key>interp_order</key> + <key>size</key> + <key>timefar_avg</key> + <key>timenear_avg</key> + <key>timefull_avg</key> + </database> + </result> + + </benchmark> +</jube> diff --git a/scripts/test-accuracy.sh b/scripts/test-accuracy.sh index 9960340798b68f76531002821c84d358cf407cb3..50dac45fdb04bbe7e9cb980c56aa886034d78825 100644 --- a/scripts/test-accuracy.sh +++ b/scripts/test-accuracy.sh @@ -1,24 +1,26 @@ #!/usr/bin/env bash +export HFI_NO_CPUAFFINITY=1 + suffix="$1" # configuration and compilation -build_dir="build${suffix}" -rm -rf "${build_dir}" # start from scratch +build_dir="build-${suffix}" +#rm -rf "${build_dir}" # start from scratch cmake -B "${build_dir}" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-O3 -march=native" -Dscalfmm_BUILD_BENCH=ON cmake --build "${build_dir}" --target fmm-computation-seq exec_file="${build_dir}/bench/Release/fmm-computation-seq" # changing parameters +dimension=(1 2 3) order_values=(3 4 5 6 7 8 9 10 11 12) +tree_height=(3 4 5) interp_settings_values=(0 1 2 3 4) kernel_values=(0 6 7) # fixed parameters log_file="tmp-log-buffer.txt" input_file="data/cuboid11-dim2-in-val1-center00-N10000.fma" -dimension=2 -tree_height=4 group_size=1 nb_threads=4 # for direct computation nb_runs=1 @@ -53,7 +55,6 @@ do --group-size "${group_size}" \ --nb-runs "${nb_runs}" \ --kernel "${kernel}" \ - --input-file "${input_file}" \ --threads "${nb_threads}" \ --interp-settings "${interp_settings}" | tee "${log_file}"