diff --git a/.gitmodules b/.gitmodules
index 0ab797b12027a56786a5464f6376ccec9350719c..5f0b612967c16d216bf3c4c816e9f9b431819f04 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,15 +1,12 @@
[submodule "cmake_modules/morse_cmake"]
- path = cmake_modules/morse_cmake
- url = https://gitlab.inria.fr/solverstack/morse_cmake.git
+ path = cmake_modules/morse_cmake
+ url = https://gitlab.inria.fr/solverstack/morse_cmake.git
[submodule "hqr"]
- path = hqr
- url = https://gitlab.inria.fr/solverstack/hqr.git
-[submodule "doc/orgmode/org-html-themes"]
- path = doc/orgmode/org-html-themes
- url = https://github.com/fniessen/org-html-themes.git
+ path = hqr
+ url = https://gitlab.inria.fr/solverstack/hqr.git
[submodule "coreblas/hmat-oss"]
- path = coreblas/hmat-oss
- url = https://github.com/jeromerobert/hmat-oss.git
+ path = coreblas/hmat-oss
+ url = https://github.com/jeromerobert/hmat-oss.git
[submodule "testing/test_fembem"]
- path = testing/test_fembem
- url = https://gitlab.inria.fr/solverstack/test_fembem.git
+ path = testing/test_fembem
+ url = https://gitlab.inria.fr/solverstack/test_fembem.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8dbc5f4b3fda1685968154087aa656d7ea21f88d..ddad6ab292083638bf3a78953b238d43812c91e0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -569,7 +569,7 @@ add_custom_target(chameleon_all_sources ALL DEPENDS ${CHAMELEON_SOURCES_TARGETS}
if(CHAMELEON_ENABLE_DOC)
add_subdirectory(doc)
if (DOXYGEN_EXECUTABLE AND EMACS_COMPILER AND TEX_COMPILER)
- add_custom_target(doc ALL DEPENDS doxygen-out doc-html-users_guide doc-pdf-users_guide)
+ add_custom_target(doc ALL DEPENDS doc-homepage doxygen-out)
endif()
endif()
#------------------------------------------------------------------------------
diff --git a/CONTRIBUTING.org b/CONTRIBUTING.org
index 68757f7c27e5e9fcd85d2b181aa76384b8feb09f..debbd85d269b9d448c381a23fbe96baed711dee8 100644
--- a/CONTRIBUTING.org
+++ b/CONTRIBUTING.org
@@ -1,170 +1,177 @@
-* To contribute to the project, you need to do it through merge request
-** Regular / Inria contributors
-*** Create a fork
- First you need to fork the repository into your own account. You can
- do that simply by clicking the fork button on the gitlab interface.
-
- https://gitlab.inria.fr/solverstack/chameleon/forks/new
-
- Then, clone the repository on your laptop:
- #+begin_src sh
- git clone git@gitlab.inria.fr:username/forkname.git
- #+end_src
-
- Once this is done, you can setup the chameleon repository as the
- upstream of your clone to simplify the update of your fork
- repository.
- #+begin_src sh
- git remote add upstream git@gitlab.inria.fr:solverstack/chameleon.git
- #+end_src
-
- To update your fork with the upstream chameleon's state:
- #+begin_src sh
- git pull upstream master
- git push -u origin master
- #+end_src
-
-*** Create a "Feature" branch in your fork
-
- To add a new feature, fix a bug, and so on, you need to create a
- new branch from the last state of the master branch
- #+begin_src sh
- git branch your_branch_name
- git checkout your_branch_name
- #+end_src
-
- Apply your modifications in that "Feature" branch. Then, you need
- to push this branch on your online repository
- #+begin_src sh
- git push origin your_branch_name
- #+end_src
-
-*** Merge request
- Once your branch is online, on the gitlab interface, go to the
- branches webpage, select the branch you want to push as a merge
- request, and push the button !!!
-
- *Be careful to check the 'close after merge' check box, and to push
- to the solverstack/chameleon repository*. By default the checkbox
- may not be checked, and the default repository is your fork.
-
- If the pull request is made to fix an issue, please name the branch
- "issueXX" so it is automatically linked to the issue. In addition,
- please add "fix issue #xx" in the comment of the pull request to
- automatically close the issue when the PR is merged.
-
-*** Rebase on top of 'master'
- In some cases your "feature" branch you want to merge into "master"
- has a long life span so that your branch and the master branch
- could make some conflicts. To avoid having to handle the possible
- conflicts at *merge request* time, please rebase your "feature" on
- top of "master" before pushing the button *merge request*.
-
- To do that, just go at the HEAD of your "feature" branch and rebase
- #+begin_src sh
- git checkout feature
- git rebase master
- #+end_src
-
- Then force to push on your origin
- #+begin_src sh
- git push --force origin feature
- #+end_src
-
- Then push the button *merge request*.
-
-** Occasional / external contributors
-*** Create a gitlab account
-Whereas [[https://gitlab.inria.fr/solverstack/chameleon][Chameleon]] is a public project and does not require an authentication
-to access it, a gitlab account is necessary to contribute. If you do not
-already have one, this is the first step to do. Note that Inria members can
-login directly with their Inria login in the *iLDAP* tab of the [[https://gitlab.inria.fr/users/sign_in][sign_in]] page.
-External users need to first [[https://gitlab-account.inria.fr/][register]] and can then login in the *Standard*
-tab of the [[https://gitlab.inria.fr/users/sign_in][sign_in]] page.
-*** Post an issue
-Create a new issue (see [[https://gitlab.inria.fr/solverstack/chameleon/issues][issues]]) presenting your contribution proposal (feature,
-fix, ...). The Chameleon team will set up a contribution branch for you. You can
-attach a patch to the issue, which we will use in this case to initiate the
-branch. In any case, we will then provide you with further instructions to work
-on the branch and eventually perform your merge request.
-
-* Configure a runner to test your branch
- To be effectively merged, your branch must be tested through the
- [[https://gitlab.inria.fr/help/ci/README.md][gitlab-ci]] mechanism.
-
- In order to execute the tests the contributor should define his own
- /gitlab runner/, /e.g/. his laptop or any other remote machine. To avoid
- having to install the proper dependencies in every runners we use
- the [[https://www.docker.com/][Docker]] image /hpclib/hiepacs/ whose recipe is defined
- [[https://gitlab.inria.fr/sed-bso/hpclib/blob/master/tools/dockerfiles/hiepacs/Dockerfile][here]]. Consequently, to register a compatible runner the requirements
- on the system are :
- * OS must be Linux
- * Docker must be installed, e.g.
+** To contribute to the project, you need to do it through merge request
+*** Regular / Inria contributors
+**** Create a fork
+ First you need to fork the repository into your own account. You can
+ do that simply by clicking the fork button on the gitlab interface.
+
+ https://gitlab.inria.fr/solverstack/chameleon/forks/new
+
+ Then, clone the repository on your laptop:
+ #+begin_src sh
+ git clone git@gitlab.inria.fr:username/forkname.git
+ #+end_src
+
+ Once this is done, you can setup the chameleon repository as the
+ upstream of your clone to simplify the update of your fork
+ repository.
+ #+begin_src sh
+ git remote add upstream git@gitlab.inria.fr:solverstack/chameleon.git
+ #+end_src
+
+ To update your fork with the upstream chameleon's state:
+ #+begin_src sh
+ git pull upstream master
+ git push -u origin master
+ #+end_src
+
+**** Create a "Feature" branch in your fork
+
+ To add a new feature, fix a bug, and so on, you need to create a
+ new branch from the last state of the master branch
+ #+begin_src sh
+ git branch your_branch_name
+ git checkout your_branch_name
+ #+end_src
+
+ Apply your modifications in that "Feature" branch. Then, you need
+ to push this branch on your online repository
+ #+begin_src sh
+ git push origin your_branch_name
+ #+end_src
+
+**** Merge request
+ Once your branch is online, on the gitlab interface, go to the
+ branches webpage, select the branch you want to push as a merge
+ request, and push the button !!!
+
+ *Be careful to check the 'close after merge' check box, and to push
+ to the solverstack/chameleon repository*. By default the checkbox
+ may not be checked, and the default repository is your fork.
+
+ If the pull request is made to fix an issue, please name the branch
+ "issueXX" so it is automatically linked to the issue. In addition,
+ please add "fix issue #xx" in the comment of the pull request to
+ automatically close the issue when the PR is merged.
+
+**** Rebase on top of 'master'
+ In some cases your "feature" branch you want to merge into "master"
+ has a long life span so that your branch and the master branch
+ could make some conflicts. To avoid having to handle the possible
+ conflicts at *merge request* time, please rebase your "feature" on
+ top of "master" before pushing the button *merge request*.
+
+ To do that, just go at the HEAD of your "feature" branch and rebase
+ #+begin_src sh
+ git checkout feature
+ git rebase master
+ #+end_src
+
+ Then force to push on your origin
+ #+begin_src sh
+ git push --force origin feature
+ #+end_src
+
+ Then push the button *merge request*.
+
+*** Occasional / external contributors
+**** Create a gitlab account
+ Whereas [[https://gitlab.inria.fr/solverstack/chameleon][Chameleon]] is a public project and does not require an
+ authentication to access it, a gitlab account is necessary to
+ contribute. If you do not already have one, this is the first
+ step to do.
+
+ Inria members can login directly with their Inria login in the
+ *iLDAP* tab of the [[https://gitlab.inria.fr/users/sign_in][sign_in]] page.
+
+ External users need to ask for an [[https://external-account.inria.fr/][external account]], send an email
+ to [[mailto:mathieu.faverge@inria.fr][mathieu.faverge@inria.fr]]. Then login in the *Standard* tab of
+ the [[https://gitlab.inria.fr/users/sign_in][sign_in]] page.
+**** Post an issue
+ Create a new issue (see [[https://gitlab.inria.fr/solverstack/chameleon/issues][issues]]) presenting your contribution
+ proposal (feature, fix, ...). The Chameleon team will set up a
+ contribution branch for you. You can attach a patch to the issue,
+ which we will use in this case to initiate the branch. In any
+ case, we will then provide you with further instructions to work
+ on the branch and eventually perform your merge request.
+
+** Configure a runner to test your branch
+ To be effectively merged, your branch must be tested through the
+ [[https://gitlab.inria.fr/help/ci/README.md][gitlab-ci]] mechanism.
+
+ In order to execute the tests the contributor should define his own
+ /gitlab runner/, /e.g/. his laptop or any other remote machine. To
+ avoid having to install the proper dependencies in every runners we
+ use the [[https://www.docker.com/][Docker]] image
+ *registry.gitlab.inria.fr/solverstack/docker/distrib* whose recipe is
+ defined [[https://gitlab.inria.fr/solverstack/docker/-/blob/master/dockerfile-distrib][here]]. Consequently, to register a compatible runner the
+ requirements on the system are :
+ * OS must be Linux
+ * Docker must be installed, e.g.
+ #+begin_src sh
+ sudo apt-get update && sudo apt-get install -y curl
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+ sudo apt install -y software-properties-common
+ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+ sudo apt-get update
+ sudo apt install -y docker-ce
+ sudo usermod -aG docker ${USER}
+ newgrp docker
+ #+end_src
+
+**** Register your runner
+ Please read first the [[https://gitlab.inria.fr/help/ci/runners/README.md][Gitlab documentation]] for general information
+ about runners registration.
+
+ Three steps are required:
+ 1) install the gitlab-runner program
+ 2) register your runner to your project (your fork of Chameleon)
+ 3) start gitlab-runner as a service
+ #+begin_src sh
+ # install gitlab-runner
+ sudo wget -O /usr/local/bin/gitlab-runner https://gitlab-ci-multi-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-ci-multi-runner-linux-amd64
+ sudo chmod +x /usr/local/bin/gitlab-runner
+ sudo useradd --comment 'GitLab Runner' --create-home gitlab-runner --shell /bin/bash
+
+ # register runner to https://gitlab.inria.fr/
+ sudo gitlab-runner register # see just after for an example
+
+ # install and run as a service
+ sudo gitlab-runner install --user=gitlab-runner --working-directory=/home/gitlab-runner
+ sudo gitlab-runner start
+ #+end_src
+
+ Example of registering sequence:
+ #+begin_example
+ sudo gitlab-runner register
+ Please enter the gitlab-ci coordinator URL (e.g. https://gitlab.com/):
+ https://gitlab.inria.fr/
+ Please enter the gitlab-ci token for this runner:
+ # copy/paste the project's secret token here
+ Please enter the gitlab-ci description for this runner:
+ [ubuntu1604]:
+ Please enter the gitlab-ci tags for this runner (comma separated):
+ linux, ubuntu
+ Whether to run untagged builds [true/false]:
+ [false]: true
+ Whether to lock Runner to current project [true/false]:
+ [false]:
+ Registering runner... succeeded runner=4jknGvoz
+ Please enter the executor: shell, ssh, docker+machine, docker-ssh+machine, kubernetes, docker, parallels, virtualbox, docker-ssh:
+ docker
+ Please enter the default Docker image (e.g. ruby:2.1):
+ ubuntu
+ Runner registered successfully. Feel free to start it, but if it's running already the config should be automatically reloaded!
+ #+end_example
+
+** To review locally a private pull request submitted by someone else
+
+ Get the patch from the pull request (Need to update that !!!!
+ Coming from bitbucket)
#+begin_src sh
- sudo apt-get update && sudo apt-get install -y curl
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
- sudo apt install -y software-properties-common
- sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
- sudo apt-get update
- sudo apt install -y docker-ce
- sudo usermod -aG docker ${USER}
- newgrp docker
+ curl https://bitbucket.org/api/2.0/repositories/icldistcomp/parsec/pullrequests/#PR/patch > pr#PR.patch
#+end_src
-*** Register your runner
- Please read first the [[https://gitlab.inria.fr/help/ci/runners/README.md][Gitlab documentation]] for general information
- about runners registration.
-
- Three steps are required:
- 1) install the gitlab-runner program
- 2) register your runner to your project (your fork of Chameleon)
- 3) start gitlab-runner as a service
+ Then apply the patch on your local copy
#+begin_src sh
- # install gitlab-runner
- sudo wget -O /usr/local/bin/gitlab-runner https://gitlab-ci-multi-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-ci-multi-runner-linux-amd64
- sudo chmod +x /usr/local/bin/gitlab-runner
- sudo useradd --comment 'GitLab Runner' --create-home gitlab-runner --shell /bin/bash
-
- # register runner to https://gitlab.inria.fr/
- sudo gitlab-runner register # see just after for an example
-
- # install and run as a service
- sudo gitlab-runner install --user=gitlab-runner --working-directory=/home/gitlab-runner
- sudo gitlab-runner start
+ git apply pr#PR.patch
#+end_src
-
- Example of registering sequence:
- #+begin_example
- sudo gitlab-runner register
- Please enter the gitlab-ci coordinator URL (e.g. https://gitlab.com/):
- https://gitlab.inria.fr/
- Please enter the gitlab-ci token for this runner:
- # copy/paste the project's secret token here
- Please enter the gitlab-ci description for this runner:
- [ubuntu1604]:
- Please enter the gitlab-ci tags for this runner (comma separated):
- linux, ubuntu
- Whether to run untagged builds [true/false]:
- [false]: true
- Whether to lock Runner to current project [true/false]:
- [false]:
- Registering runner... succeeded runner=4jknGvoz
- Please enter the executor: shell, ssh, docker+machine, docker-ssh+machine, kubernetes, docker, parallels, virtualbox, docker-ssh:
- docker
- Please enter the default Docker image (e.g. ruby:2.1):
- ubuntu
- Runner registered successfully. Feel free to start it, but if it's running already the config should be automatically reloaded!
- #+end_example
-
-* To review locally a private pull request submitted by someone else
-
- Get the patch from the pull request (Need to update that !!!!
- Coming from bitbucket)
- #+begin_src sh
- curl https://bitbucket.org/api/2.0/repositories/icldistcomp/parsec/pullrequests/#PR/patch > pr#PR.patch
- #+end_src
-
- Then apply the patch on your local copy
- #+begin_src sh
- git apply pr#PR.patch
- #+end_src
diff --git a/README.org b/README.org
index 7463d62c11591794b59d249dc0c689a965c9b5b3..2907869a6a40f351bfa073e37ddbf829b9b7240c 100644
--- a/README.org
+++ b/README.org
@@ -12,21 +12,6 @@
Chameleon is a C library providing parallel algorithms to perform
BLAS/LAPACK operations exploiting fully modern architectures.
-Chameleon dense linear algebra software relies on sequential
-task-based algorithms where sub-tasks of the overall algorithms are
-submitted to a Runtime system. Such a system is a layer between the
-application and the hardware which handles the scheduling and the
-effective execution of tasks on the processing units. A Runtime system
-such as [[http://starpu.gforge.inria.fr/][StarPU]] is able to manage automatically data transfers between
-not shared memory area (CPUs-GPUs, distributed nodes).
-
-This kind of implementation paradigm allows to design high performing
-linear algebra algorithms on very different type of architecture:
-laptop, many-core nodes, CPUs-GPUs, multiple nodes. For example,
-Chameleon is able to perform a Cholesky factorization
-(double-precision) at 80 TFlop/s on a dense matrix of order 400 000
-(i.e. 4 min). Chameleon is a sub-project of [[http://icl.cs.utk.edu/morse/][MORSE]] specifically
-dedicated to dense linear algebra.
* Get Chameleon
@@ -35,49 +20,38 @@ dedicated to dense linear algebra.
To get sources please use these commands:
#+begin_src sh
- # if git version >= 1.9
- git clone --recursive git@gitlab.inria.fr:solverstack/chameleon.git
- cd chameleon
- # else
- git clone git@gitlab.inria.fr:solverstack/chameleon.git
- cd chameleon
- git submodule init
- git submodule update
+ # if git version >= 1.9
+ git clone --recursive git@gitlab.inria.fr:solverstack/chameleon.git
+ cd chameleon
+ # else
+ git clone git@gitlab.inria.fr:solverstack/chameleon.git
+ cd chameleon
+ git submodule init
+ git submodule update
#+end_src
- Last releases of Chameleon are hosted on the [[https://gforge.inria.fr/frs/?group_id=2884][gforge.inria.fr]] for
- now. Future releases will be available on this gitlab project.
-
-* Documentation
-** User guide
-
- Please refer to the [[https://solverstack.gitlabpages.inria.fr/chameleon/users_guide.html][User guide]] to learn how to install and use
- Chameleon.
-
- The user guide is also available directly in the sources as emacs
- orgmode files, see :
- 1) [[file:doc/orgmode/chapters/introduction.org][Introduction]] : description of the scientific context
- 2) [[file:doc/orgmode/chapters/installing.org][Installing]] :
- * Getting Chameleon
- * Prerequisites for installing Chameleon
- * Distribution of Chameleon using Spack
- * Build and install Chameleon with CMake
- 3) [[file:doc/orgmode/chapters/using.org][Using]] :
- * Linking an external application with Chameleon libraries
- * Using Chameleon executables
- * Chameleon API
-
- This documentation could also be generated in html and/or pdf :
+ Last releases of Chameleon are hosted on the [[https://gitlab.inria.fr/solverstack/chameleon/-/releases][releases page]].
+
+* Documentations
+** Homepage and user's guide
+ Please visit our [[https://solverstack.gitlabpages.inria.fr/chameleon/][Homepage]] to get:
+ * download links,
+ * quick start guides,
+ * installation instructions,
+ * tutorials,
+ * bencmarks.
+
+ This documentation could also be generated in html:
#+begin_src
# build the doc with cmake (emacs with orgmode and latex are required), e.g.
cmake .. -DCHAMELEON_ENABLE_DOC=ON
make doc
#+end_src
- see the ~doc/orgmode~ directory.
+ see the ~doc/user~ directory.
-** Source code documentation (doxygen)
+** Source code documentation, API (doxygen)
- Please refer to the [[https://solverstack.gitlabpages.inria.fr/chameleon/doxygen/index.html][doxygen documentation]] to get more precise
+ Please refer to the [[https://solverstack.gitlabpages.inria.fr/chameleon/dev/index.html][doxygen documentation]] to get more precise
information about the API, the public and internal functions
prototypes and the data structures.
@@ -87,71 +61,18 @@ dedicated to dense linear algebra.
** For developers
Please refer to the [[file:READMEDEV.org][READMEDEV]] page.
-* Get involved!
-** Mailing list
+* Contact
+ If you have an account on [[https://gitlab.inria.fr/][gitlab inria]] please submit a [[https://gitlab.inria.fr/solverstack/chameleon/-/issues][new issue]].
- To contact the developers send an email to
- [[mailto:morse-devel@lists.gforge.inria.fr][morse-devel@lists.gforge.inria.fr]]
+ If you don't have an account on [[https://gitlab.inria.fr][gitlab inria]] you can send emails to
+ [[mailto:chameleon-issues@inria.fr][chameleon-issues@inria.fr]].
-** Contributions
+ To get the news, register to the mailing list
+ [[https://sympa.inria.fr/sympa/info/chameleon-announce][chameleon-announce@inria.fr]] (click on "S'abonner" in the left
+ panel).
+* Get involved!
[[file:CONTRIBUTING.org][CONTRIBUTING]]
-* Authors
-
- First, since the Chameleon library started as an extension of the
- PLASMA library to support multiple runtime systems, all developpers
- of the PLASMA library are developpers of the Chameleon library.
-
- The following people contributed to the development of Chameleon:
- * Emmanuel Agullo, PI
- * Olivier Aumage
- * Cedric Castagnede
- * Terry Cojean
- * Mathieu Faverge, PI
- * Nathalie Furmento
- * Reazul Hoque
- * Hatem Ltaief
- * Gregoire Pichon
- * Florent Pruvost, PI
- * Marc Sergent
- * Guillaume Sylvand
- * Samuel Thibault
- * Stanimire Tomov
- * Omar Zenati
-
- If we forgot your name, please let us know that we can fix that mistake.
-
-* Citing Chameleon
-
-Feel free to use the following publications to reference Chameleon:
-
- * Original paper that initiated Chameleon and the principles:
- - Agullo, Emmanuel and Augonnet, Cédric and Dongarra, Jack and
- Ltaief, Hatem and Namyst, Raymond and Thibault, Samuel and Tomov,
- Stanimire, *Faster, Cheaper, Better -- a Hybridization Methodology
- to Develop Linear Algebra Software for GPUs*, /GPU Computing Gems/,
- [[https://hal.inria.fr/inria-00547847][First Online: 17 December 2010]].
- * Design of the QR algorithms:
- - Agullo, Emmanuel and Augonnet, Cédric and Dongarra, Jack and
- Faverge, Mathieu and Ltaief, Hatem and Thibault, Samuel an
- Tomov, Stanimire, *QR Factorization on a Multicore Node Enhanced
- with Multiple GPU Accelerators*, /25th IEEE International Parallel
- & Distributed Processing Symposium/, [[https://hal.inria.fr/inria-00547614][First Online: 16 December
- 2010]].
- * Design of the LU algorithms:
- - Agullo, Emmanuel and Augonnet, Cédric and Dongarra, Jack and
- Faverge, Mathieu and Langou, Julien and Ltaief, Hatem and Tomov,
- Stanimire, *LU Factorization for Accelerator-based Systems*,
- /9th ACS/IEEE International Conference on Computer Systems and
- Applications (AICCSA 11)/, [[https://hal.inria.fr/hal-00654193][First Online: 21 December 2011]].
- * Regarding distributed memory:
- - Agullo, Emmanuel and Aumage, Olivier and Faverge, Mathieu and
- Furmento, Nathalie and Pruvost, Florent and Sergent, Marc and
- Thibault, Samuel, *Achieving High Performance on Supercomputers
- with a Sequential Task-based Programming Model*, /Research Report/,
- [[https://hal.inria.fr/hal-01332774][First Online: 16 June 2016]].
-
* Licence
-
[[file:LICENCE.txt][LICENCE]]
diff --git a/compute/zcesca.c b/compute/zcesca.c
index 35f2839a72d3c1d3a97c555e3747cf2435f5bec2..822858dad6683f5748cb7073b6bfcbd4c2286491 100644
--- a/compute/zcesca.c
+++ b/compute/zcesca.c
@@ -142,8 +142,8 @@ void CHAMELEON_zcesca_WS_Free( void *user_ws )
* A bicentered gives \f[\bar{A} = (\bar{a}_{i,j})_{1 \leq i \leq m, 1 \leq j \leq n}\f] such that
* \f[ \bar{a}_{i,j} = a_{i,j} - g_i - g_j + g \f]
* Lets
- * \f[d_i = || a_{i*} || = \sqrt{ \sum_j a_{ij}²} \\
- * d_j = || a_{*j} || = \sqrt{ \sum_i a_{ij}²} \f]
+ * \f[d_i = || a_{i*} || = \sqrt{ \sum_j a_{ij}^2} \\
+ * d_j = || a_{*j} || = \sqrt{ \sum_i a_{ij}^2} \f]
* A scaled rowwise gives \f[A' = (a_{i,j}')_{1 \leq i \leq m, 1 \leq j \leq n}\f] such that
* \f[ a_{i*}' = \frac{a_{i*}}{d_i} \f]
* A scaled columnwise gives \f[A' = (a_{i,j}')_{1 \leq i \leq m, 1 \leq j \leq n}\f] such that
diff --git a/coreblas/compute/core_zcesca.c b/coreblas/compute/core_zcesca.c
index 8bdc87dac306a36aecd8ba527832af7516d240c2..989e48b82ad718b41d0975628368df85ba39c232 100644
--- a/coreblas/compute/core_zcesca.c
+++ b/coreblas/compute/core_zcesca.c
@@ -36,8 +36,8 @@
* A bicentered gives \f[\bar{A} = (\bar{a}_{i,j})_{1 \leq i \leq m, 1 \leq j \leq n}\f] such that
* \f[ \bar{a}_{i,j} = a_{i,j} - g_i - g_j + g \f]
* Lets
- * \f[d_i = || a_{i*} || = \sqrt{ \sum_j a_{ij}²} \\
- * d_j = || a_{*j} || = \sqrt{ \sum_i a_{ij}²} \f]
+ * \f[d_i = || a_{i*} || = \sqrt{ \sum_j a_{ij}^2} \\
+ * d_j = || a_{*j} || = \sqrt{ \sum_i a_{ij}^2} \f]
* A scaled rowwise gives \f[A' = (a_{i,j}')_{1 \leq i \leq m, 1 \leq j \leq n}\f] such that
* \f[ a_{i*}' = \frac{a_{i*}}{d_i} \f]
* A scaled columnwise gives \f[A' = (a_{i,j}')_{1 \leq i \leq m, 1 \leq j \leq n}\f] such that
diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt
index 98e18955dc5ea0db2eb00b092cf0e2374dec85eb..d7077e2a524f2abce42eed968c1dfc76223aef81 100644
--- a/doc/CMakeLists.txt
+++ b/doc/CMakeLists.txt
@@ -33,8 +33,8 @@ cmake_minimum_required(VERSION 3.1)
# Compilation of documentation #
# #
#############################################
-add_subdirectory(doxygen)
-add_subdirectory(orgmode)
+add_subdirectory(dev)
+add_subdirectory(user)
###
### END CMakeLists.txt
###
diff --git a/doc/doxygen/CMakeLists.txt b/doc/dev/CMakeLists.txt
similarity index 87%
rename from doc/doxygen/CMakeLists.txt
rename to doc/dev/CMakeLists.txt
index 3663c7a62a4ee501da415e945fa2e6ae364e5fab..03e930ef1e699317eb9e236f30b0d508bf194f7a 100644
--- a/doc/doxygen/CMakeLists.txt
+++ b/doc/dev/CMakeLists.txt
@@ -48,21 +48,21 @@ if(DOXYGEN_EXECUTABLE)
# Doxygen documentation
# ---------------------
- add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/out/index.html
+ add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/index.html
COMMAND ${DOXYGEN_EXECUTABLE}
ARGS ${CMAKE_CURRENT_BINARY_DIR}/chameleon.dox
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/chameleon.dox
${CMAKE_CURRENT_SOURCE_DIR}/main.dox
)
add_custom_target(doxygen-out ALL
- DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/out/index.html
+ DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/index.html
chameleon_all_sources
)
# Installation for html version
# -----------------------------
- install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/out/html
- DESTINATION share/chameleon/doxygen/dev)
+ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html
+ DESTINATION share/chameleon/doc/dev)
else()
message(STATUS "Looking for doxygen - not found")
diff --git a/doc/doxygen/README.org b/doc/dev/README.org
similarity index 100%
rename from doc/doxygen/README.org
rename to doc/dev/README.org
diff --git a/doc/dev/conf.dox.in b/doc/dev/conf.dox.in
new file mode 100644
index 0000000000000000000000000000000000000000..e3ce59be0e5afd58ae1d311055437a120620d6fd
--- /dev/null
+++ b/doc/dev/conf.dox.in
@@ -0,0 +1,2526 @@
+# Doxyfile 1.8.17
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project.
+#
+# All text after a double hash (##) is considered a comment and is placed in
+# front of the TAG it is preceding.
+#
+# All text after a single hash (#) is considered a comment and will be ignored.
+# The format is:
+# TAG = value [value, ...]
+# For lists, items can also be appended using:
+# TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (\" \").
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+
+# This tag specifies the encoding used for all characters in the configuration
+# file that follow. The default is UTF-8 which is also the encoding used for all
+# text before the first occurrence of this tag. Doxygen uses libiconv (or the
+# iconv built into libc) for the transcoding. See
+# https://www.gnu.org/software/libiconv/ for the list of possible encodings.
+# The default value is: UTF-8.
+
+DOXYFILE_ENCODING = UTF-8
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME = CHAMELEON
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER = @CHAMELEON_VERSION_MAJOR@.@CHAMELEON_VERSION_MINOR@.@CHAMELEON_VERSION_MICRO@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF =
+
+# With the PROJECT_LOGO tag one can specify a logo or an icon that is included
+# in the documentation. The maximum height of the logo should not exceed 55
+# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
+# the logo to the output directory.
+
+PROJECT_LOGO =
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
+# into which the generated documentation will be written. If a relative path is
+# entered, it will be relative to the location where doxygen was started. If
+# left blank the current directory will be used.
+
+OUTPUT_DIRECTORY = @CMAKE_CURRENT_BINARY_DIR@
+
+# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub-
+# directories (in 2 levels) under the output directory of each output format and
+# will distribute the generated files over these directories. Enabling this
+# option can be useful when feeding doxygen a huge amount of source files, where
+# putting all generated files in the same directory would otherwise causes
+# performance problems for the file system.
+# The default value is: NO.
+
+CREATE_SUBDIRS = NO
+
+# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII
+# characters to appear in the names of generated files. If set to NO, non-ASCII
+# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode
+# U+3044.
+# The default value is: NO.
+
+ALLOW_UNICODE_NAMES = NO
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all constant output in the proper language.
+# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
+# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
+# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
+# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
+# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
+# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
+# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
+# Ukrainian and Vietnamese.
+# The default value is: English.
+
+OUTPUT_LANGUAGE = English
+
+# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all generated output in the proper direction.
+# Possible values are: None, LTR, RTL and Context.
+# The default value is: None.
+
+OUTPUT_TEXT_DIRECTION = None
+
+# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
+# descriptions after the members that are listed in the file and class
+# documentation (similar to Javadoc). Set to NO to disable this.
+# The default value is: YES.
+
+BRIEF_MEMBER_DESC = YES
+
+# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief
+# description of a member or function before the detailed description
+#
+# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
+# brief descriptions will be completely suppressed.
+# The default value is: YES.
+
+REPEAT_BRIEF = YES
+
+# This tag implements a quasi-intelligent brief description abbreviator that is
+# used to form the text in various listings. Each string in this list, if found
+# as the leading text of the brief description, will be stripped from the text
+# and the result, after processing the whole list, is used as the annotated
+# text. Otherwise, the brief description is used as-is. If left blank, the
+# following values are used ($name is automatically replaced with the name of
+# the entity):The $name class, The $name widget, The $name file, is, provides,
+# specifies, contains, represents, a, an and the.
+
+ABBREVIATE_BRIEF =
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
+# doxygen will generate a detailed section even if there is only a brief
+# description.
+# The default value is: NO.
+
+ALWAYS_DETAILED_SEC = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
+# inherited members of a class in the documentation of that class as if those
+# members were ordinary class members. Constructors, destructors and assignment
+# operators of the base classes will not be shown.
+# The default value is: NO.
+
+INLINE_INHERITED_MEMB = NO
+
+# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path
+# before files name in the file list and in the header files. If set to NO the
+# shortest path that makes the file name unique will be used
+# The default value is: YES.
+
+FULL_PATH_NAMES = NO
+
+# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
+# Stripping is only done if one of the specified strings matches the left-hand
+# part of the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the path to
+# strip.
+#
+# Note that you can specify absolute paths here, but also relative paths, which
+# will be relative from the directory where doxygen is started.
+# This tag requires that the tag FULL_PATH_NAMES is set to YES.
+
+STRIP_FROM_PATH =
+
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
+# path mentioned in the documentation of a class, which tells the reader which
+# header file to include in order to use a class. If left blank only the name of
+# the header file containing the class definition is used. Otherwise one should
+# specify the list of include paths that are normally passed to the compiler
+# using the -I flag.
+
+STRIP_FROM_INC_PATH =
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
+# less readable) file names. This can be useful is your file systems doesn't
+# support long names like on DOS, Mac, or CD-ROM.
+# The default value is: NO.
+
+SHORT_NAMES = NO
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
+# first line (until the first dot) of a Javadoc-style comment as the brief
+# description. If set to NO, the Javadoc-style will behave just like regular Qt-
+# style comments (thus requiring an explicit @brief command for a brief
+# description.)
+# The default value is: NO.
+
+JAVADOC_AUTOBRIEF = NO
+
+# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line
+# such as
+# /***************
+# as being the beginning of a Javadoc-style comment "banner". If set to NO, the
+# Javadoc-style will behave just like regular comments and it will not be
+# interpreted by doxygen.
+# The default value is: NO.
+
+JAVADOC_BANNER = NO
+
+# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
+# line (until the first dot) of a Qt-style comment as the brief description. If
+# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
+# requiring an explicit \brief command for a brief description.)
+# The default value is: NO.
+
+QT_AUTOBRIEF = NO
+
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
+# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
+# a brief description. This used to be the default behavior. The new default is
+# to treat a multi-line C++ comment block as a detailed description. Set this
+# tag to YES if you prefer the old behavior instead.
+#
+# Note that setting this tag to YES also means that rational rose comments are
+# not recognized any more.
+# The default value is: NO.
+
+MULTILINE_CPP_IS_BRIEF = NO
+
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
+
+INHERIT_DOCS = YES
+
+# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new
+# page for each member. If set to NO, the documentation of a member will be part
+# of the file/class/namespace that contains it.
+# The default value is: NO.
+
+SEPARATE_MEMBER_PAGES = YES
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
+# uses this value to replace tabs by spaces in code fragments.
+# Minimum value: 1, maximum value: 16, default value: 4.
+
+TAB_SIZE = 4
+
+# This tag can be used to specify a number of aliases that act as commands in
+# the documentation. An alias has the form:
+# name=value
+# For example adding
+# "sideeffect=@par Side Effects:\n"
+# will allow you to put the command \sideeffect (or @sideeffect) in the
+# documentation, which will result in a user-defined paragraph with heading
+# "Side Effects:". You can put \n's in the value part of an alias to insert
+# newlines (in the resulting output). You can put ^^ in the value part of an
+# alias to insert a newline as if a physical newline was in the original file.
+# When you need a literal { or } or , in the value part of an alias you have to
+# escape them by means of a backslash (\), this can lead to conflicts with the
+# commands \{ and \} for these it is advised to use the version @{ and @} or use
+# a double escape (\\{ and \\})
+
+ALIASES =
+
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST =
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
+# only. Doxygen will then generate output that is more tailored for C. For
+# instance, some of the names that are used will be different. The list of all
+# members will be omitted, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_FOR_C = YES
+
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
+# Python sources only. Doxygen will then generate output that is more tailored
+# for that language. For instance, namespaces will be presented as packages,
+# qualified scopes will look different, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_JAVA = NO
+
+# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
+# sources. Doxygen will then generate output that is tailored for Fortran.
+# The default value is: NO.
+
+OPTIMIZE_FOR_FORTRAN = YES
+
+# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
+# sources. Doxygen will then generate output that is tailored for VHDL.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_VHDL = NO
+
+# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice
+# sources only. Doxygen will then generate output that is more tailored for that
+# language. For instance, namespaces will be presented as modules, types will be
+# separated into more groups, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_SLICE = NO
+
+# Doxygen selects the parser to use depending on the extension of the files it
+# parses. With this tag you can assign which parser to use for a given
+# extension. Doxygen has a built-in mapping, but you can override or extend it
+# using this tag. The format is ext=language, where ext is a file extension, and
+# language is one of the parsers supported by doxygen: IDL, Java, JavaScript,
+# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice,
+# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran:
+# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser
+# tries to guess whether the code is fixed or free formatted code, this is the
+# default for Fortran type files), VHDL, tcl. For instance to make doxygen treat
+# .inc files as Fortran files (default is PHP), and .f files as C (default is
+# Fortran), use: inc=Fortran f=C.
+#
+# Note: For files without extension you can use no_extension as a placeholder.
+#
+# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
+# the files are not read by doxygen.
+
+EXTENSION_MAPPING =
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See https://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT = YES
+
+# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up
+# to that level are automatically included in the table of contents, even if
+# they do not have an id attribute.
+# Note: This feature currently applies only to Markdown headings.
+# Minimum value: 0, maximum value: 99, default value: 5.
+# This tag requires that the tag MARKDOWN_SUPPORT is set to YES.
+
+TOC_INCLUDE_HEADINGS = 5
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by putting a % sign in front of the word or
+# globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT = YES
+
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
+# to include (a tag file for) the STL sources as input, then you should set this
+# tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string);
+# versus func(std::string) {}). This also make the inheritance and collaboration
+# diagrams that involve STL classes more complete and accurate.
+# The default value is: NO.
+
+BUILTIN_STL_SUPPORT = NO
+
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+# The default value is: NO.
+
+CPP_CLI_SUPPORT = NO
+
+# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
+# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen
+# will parse them like normal C++ but will assume all classes use public instead
+# of private inheritance when no explicit protection keyword is present.
+# The default value is: NO.
+
+SIP_SUPPORT = NO
+
+# For Microsoft's IDL there are propget and propput attributes to indicate
+# getter and setter methods for a property. Setting this option to YES will make
+# doxygen to replace the get and set methods by a property in the documentation.
+# This will only work if the methods are indeed getting or setting a simple
+# type. If this is not the case, or you want to show the methods anyway, you
+# should set this option to NO.
+# The default value is: YES.
+
+IDL_PROPERTY_SUPPORT = YES
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
+# tag is set to YES then doxygen will reuse the documentation of the first
+# member in the group (if any) for the other members of the group. By default
+# all members of a group must be documented explicitly.
+# The default value is: NO.
+
+DISTRIBUTE_GROUP_DOC = NO
+
+# If one adds a struct or class to a group and this option is enabled, then also
+# any nested class or struct is added to the same group. By default this option
+# is disabled and one has to add nested compounds explicitly via \ingroup.
+# The default value is: NO.
+
+GROUP_NESTED_COMPOUNDS = NO
+
+# Set the SUBGROUPING tag to YES to allow class member groups of the same type
+# (for instance a group of public functions) to be put as a subgroup of that
+# type (e.g. under the Public Functions section). Set it to NO to prevent
+# subgrouping. Alternatively, this can be done per class using the
+# \nosubgrouping command.
+# The default value is: YES.
+
+SUBGROUPING = YES
+
+# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
+# are shown inside the group in which they are included (e.g. using \ingroup)
+# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
+# and RTF).
+#
+# Note that this feature does not work in combination with
+# SEPARATE_MEMBER_PAGES.
+# The default value is: NO.
+
+INLINE_GROUPED_CLASSES = NO
+
+# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
+# with only public data fields or simple typedef fields will be shown inline in
+# the documentation of the scope in which they are defined (i.e. file,
+# namespace, or group documentation), provided this scope is documented. If set
+# to NO, structs, classes, and unions are shown on a separate page (for HTML and
+# Man pages) or section (for LaTeX and RTF).
+# The default value is: NO.
+
+INLINE_SIMPLE_STRUCTS = NO
+
+# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
+# enum is documented as struct, union, or enum with the name of the typedef. So
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
+# with name TypeT. When disabled the typedef will appear as a member of a file,
+# namespace, or class. And the struct will be named TypeS. This can typically be
+# useful for C code in case the coding convention dictates that all compound
+# types are typedef'ed and only the typedef is referenced, never the tag name.
+# The default value is: NO.
+
+TYPEDEF_HIDES_STRUCT = NO
+
+# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
+# cache is used to resolve symbols given their name and scope. Since this can be
+# an expensive process and often the same symbol appears multiple times in the
+# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
+# doxygen will become slower. If the cache is too large, memory is wasted. The
+# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
+# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
+# symbols. At the end of a run doxygen will report the cache usage and suggest
+# the optimal cache size from a speed point of view.
+# Minimum value: 0, maximum value: 9, default value: 0.
+
+LOOKUP_CACHE_SIZE = 0
+
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+
+# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
+
+EXTRACT_ALL = NO
+
+# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will
+# be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIVATE = YES
+
+# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual
+# methods of a class will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIV_VIRTUAL = NO
+
+# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
+# scope will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PACKAGE = NO
+
+# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be
+# included in the documentation.
+# The default value is: NO.
+
+EXTRACT_STATIC = YES
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined
+# locally in source files will be included in the documentation. If set to NO,
+# only classes defined in header files are included. Does not have any effect
+# for Java sources.
+# The default value is: YES.
+
+EXTRACT_LOCAL_CLASSES = YES
+
+# This flag is only useful for Objective-C code. If set to YES, local methods,
+# which are defined in the implementation section but not in the interface are
+# included in the documentation. If set to NO, only methods in the interface are
+# included.
+# The default value is: NO.
+
+EXTRACT_LOCAL_METHODS = NO
+
+# If this flag is set to YES, the members of anonymous namespaces will be
+# extracted and appear in the documentation as a namespace called
+# 'anonymous_namespace{file}', where file will be replaced with the base name of
+# the file that contains the anonymous namespace. By default anonymous namespace
+# are hidden.
+# The default value is: NO.
+
+EXTRACT_ANON_NSPACES = NO
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_MEMBERS = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO, these classes will be included in the various overviews. This option
+# has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_CLASSES = NO
+
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
+# declarations. If set to NO, these declarations will be included in the
+# documentation.
+# The default value is: NO.
+
+HIDE_FRIEND_COMPOUNDS = NO
+
+# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
+# documentation blocks found inside the body of a function. If set to NO, these
+# blocks will be appended to the function's detailed documentation block.
+# The default value is: NO.
+
+HIDE_IN_BODY_DOCS = NO
+
+# The INTERNAL_DOCS tag determines if documentation that is typed after a
+# \internal command is included. If the tag is set to NO then the documentation
+# will be excluded. Set it to YES to include the internal documentation.
+# The default value is: NO.
+
+INTERNAL_DOCS = YES
+
+# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
+# names in lower-case letters. If set to YES, upper-case letters are also
+# allowed. This is useful if you have classes or files whose names only differ
+# in case and if your file system supports case sensitive file names. Windows
+# (including Cygwin) ands Mac users are advised to set this option to NO.
+# The default value is: system dependent.
+
+CASE_SENSE_NAMES = YES
+
+# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
+# their full class and namespace scopes in the documentation. If set to YES, the
+# scope will be hidden.
+# The default value is: NO.
+
+HIDE_SCOPE_NAMES = NO
+
+# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will
+# append additional text to a page's title, such as Class Reference. If set to
+# YES the compound reference will be hidden.
+# The default value is: NO.
+
+HIDE_COMPOUND_REFERENCE= NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
+# the files that are included by a file in the documentation of that file.
+# The default value is: YES.
+
+SHOW_INCLUDE_FILES = YES
+
+# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each
+# grouped member an include statement to the documentation, telling the reader
+# which file to include in order to use the member.
+# The default value is: NO.
+
+SHOW_GROUPED_MEMB_INC = NO
+
+# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
+# files with double quotes in the documentation rather than with sharp brackets.
+# The default value is: NO.
+
+FORCE_LOCAL_INCLUDES = NO
+
+# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
+# documentation for inline members.
+# The default value is: YES.
+
+INLINE_INFO = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
+# (detailed) documentation of file and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order.
+# The default value is: YES.
+
+SORT_MEMBER_DOCS = YES
+
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
+# descriptions of file, namespace and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order. Note that
+# this will also influence the order of the classes in the class list.
+# The default value is: NO.
+
+SORT_BRIEF_DOCS = NO
+
+# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
+# (brief and detailed) documentation of class members so that constructors and
+# destructors are listed first. If set to NO the constructors will appear in the
+# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
+# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
+# member documentation.
+# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
+# detailed member documentation.
+# The default value is: NO.
+
+SORT_MEMBERS_CTORS_1ST = NO
+
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
+# of group names into alphabetical order. If set to NO the group names will
+# appear in their defined order.
+# The default value is: NO.
+
+SORT_GROUP_NAMES = YES
+
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
+# fully-qualified names, including namespaces. If set to NO, the class list will
+# be sorted only by class name, not including the namespace part.
+# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
+# Note: This option applies only to the class list, not to the alphabetical
+# list.
+# The default value is: NO.
+
+SORT_BY_SCOPE_NAME = NO
+
+# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
+# type resolution of all parameters of a function it will reject a match between
+# the prototype and the implementation of a member function even if there is
+# only one candidate or it is obvious which candidate to choose by doing a
+# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
+# accept a match between prototype and implementation in such cases.
+# The default value is: NO.
+
+STRICT_PROTO_MATCHING = NO
+
+# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo
+# list. This list is created by putting \todo commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TODOLIST = YES
+
+# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test
+# list. This list is created by putting \test commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TESTLIST = YES
+
+# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug
+# list. This list is created by putting \bug commands in the documentation.
+# The default value is: YES.
+
+GENERATE_BUGLIST = YES
+
+# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO)
+# the deprecated list. This list is created by putting \deprecated commands in
+# the documentation.
+# The default value is: YES.
+
+GENERATE_DEPRECATEDLIST= YES
+
+# The ENABLED_SECTIONS tag can be used to enable conditional documentation
+# sections, marked by \if <section_label> ... \endif and \cond <section_label>
+# ... \endcond blocks.
+
+ENABLED_SECTIONS =
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
+# initial value of a variable or macro / define can have for it to appear in the
+# documentation. If the initializer consists of more lines than specified here
+# it will be hidden. Use a value of 0 to hide initializers completely. The
+# appearance of the value of individual variables and macros / defines can be
+# controlled using \showinitializer or \hideinitializer command in the
+# documentation regardless of this setting.
+# Minimum value: 0, maximum value: 10000, default value: 30.
+
+MAX_INITIALIZER_LINES = 30
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
+# the bottom of the documentation of classes and structs. If set to YES, the
+# list will mention the files that were used to generate the documentation.
+# The default value is: YES.
+
+SHOW_USED_FILES = YES
+
+# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
+# will remove the Files entry from the Quick Index and from the Folder Tree View
+# (if specified).
+# The default value is: YES.
+
+SHOW_FILES = YES
+
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
+# page. This will remove the Namespaces entry from the Quick Index and from the
+# Folder Tree View (if specified).
+# The default value is: YES.
+
+SHOW_NAMESPACES = YES
+
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that
+# doxygen should invoke to get the current version for each file (typically from
+# the version control system). Doxygen will invoke the program by executing (via
+# popen()) the command command input-file, where command is the value of the
+# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
+# by doxygen. Whatever the program writes to standard output is used as the file
+# version. For an example see the documentation.
+
+FILE_VERSION_FILTER =
+
+# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
+# by doxygen. The layout file controls the global structure of the generated
+# output files in an output format independent way. To create the layout file
+# that represents doxygen's defaults, run doxygen with the -l option. You can
+# optionally specify a file name after the option, if omitted DoxygenLayout.xml
+# will be used as the name of the layout file.
+#
+# Note that if you run doxygen from a directory containing a file called
+# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
+# tag is left empty.
+
+LAYOUT_FILE =
+
+# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
+# the reference definitions. This must be a list of .bib files. The .bib
+# extension is automatically appended if omitted. This requires the bibtex tool
+# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info.
+# For LaTeX the style of the bibliography can be controlled using
+# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
+# search path. See also \cite for info how to create references.
+
+CITE_BIB_FILES =
+
+#---------------------------------------------------------------------------
+# Configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated to
+# standard output by doxygen. If QUIET is set to YES this implies that the
+# messages are off.
+# The default value is: NO.
+
+QUIET = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are
+# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES
+# this implies that the warnings are on.
+#
+# Tip: Turn warnings on while writing the documentation.
+# The default value is: YES.
+
+WARNINGS = YES
+
+# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate
+# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
+# will automatically be disabled.
+# The default value is: YES.
+
+WARN_IF_UNDOCUMENTED = YES
+
+# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some parameters
+# in a documented function, or documenting parameters that don't exist or using
+# markup commands wrongly.
+# The default value is: YES.
+
+WARN_IF_DOC_ERROR = YES
+
+# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
+# are documented, but have no documentation for their parameters or return
+# value. If set to NO, doxygen will only warn about wrong or incomplete
+# parameter documentation, but not about the absence of documentation. If
+# EXTRACT_ALL is set to YES then this flag will automatically be disabled.
+# The default value is: NO.
+
+WARN_NO_PARAMDOC = YES
+
+# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when
+# a warning is encountered.
+# The default value is: NO.
+
+WARN_AS_ERROR = NO
+
+# The WARN_FORMAT tag determines the format of the warning messages that doxygen
+# can produce. The string should contain the $file, $line, and $text tags, which
+# will be replaced by the file and line number from which the warning originated
+# and the warning text. Optionally the format may contain $version, which will
+# be replaced by the version of the file (if it could be obtained via
+# FILE_VERSION_FILTER)
+# The default value is: $file:$line: $text.
+
+WARN_FORMAT = "$file:$line: $text"
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning and error
+# messages should be written. If left blank the output is written to standard
+# error (stderr).
+
+WARN_LOGFILE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+INPUT = @CMAKE_CURRENT_SOURCE_DIR@/main.dox \
+ @CMAKE_CURRENT_SOURCE_DIR@/../../include \
+ @CMAKE_CURRENT_SOURCE_DIR@/../../compute \
+ @CMAKE_CURRENT_SOURCE_DIR@/../../control
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: https://www.gnu.org/software/libiconv/) for the list of
+# possible encodings.
+# The default value is: UTF-8.
+
+INPUT_ENCODING = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# read by doxygen.
+#
+# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
+# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
+# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
+# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment),
+# *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen
+# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f, *.for, *.tcl, *.vhd,
+# *.vhdl, *.ucf, *.qsf and *.ice.
+
+FILE_PATTERNS = *.c
+
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
+
+RECURSIVE = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should be
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
+
+EXCLUDE =
+
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
+# from the input.
+# The default value is: NO.
+
+EXCLUDE_SYMLINKS = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
+
+EXCLUDE_PATTERNS =
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
+
+EXCLUDE_SYMBOLS =
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or directories
+# that contain example code fragments that are included (see the \include
+# command).
+
+EXAMPLE_PATH =
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank all
+# files are included.
+
+EXAMPLE_PATTERNS =
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
+# searched for input files to be used with the \include or \dontinclude commands
+# irrespective of the value of the RECURSIVE tag.
+# The default value is: NO.
+
+EXAMPLE_RECURSIVE = NO
+
+# The IMAGE_PATH tag can be used to specify one or more files or directories
+# that contain images that are to be included in the documentation (see the
+# \image command).
+
+IMAGE_PATH =
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should
+# invoke to filter for each input file. Doxygen will invoke the filter program
+# by executing (via popen()) the command:
+#
+# <filter> <input-file>
+#
+# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the
+# name of an input file. Doxygen will then use the output that the filter
+# program writes to standard output. If FILTER_PATTERNS is specified, this tag
+# will be ignored.
+#
+# Note that the filter must not add or remove lines; it is applied before the
+# code is scanned, but not when the output code is generated. If lines are added
+# or removed, the anchors will not be placed correctly.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+INPUT_FILTER =
+
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
+# basis. Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match. The filters are a list of the form: pattern=filter
+# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
+# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
+# patterns match the file name, INPUT_FILTER is applied.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+FILTER_PATTERNS =
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
+# INPUT_FILTER) will also be used to filter the input files that are used for
+# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
+# The default value is: NO.
+
+FILTER_SOURCE_FILES = NO
+
+# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
+# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
+# it is also possible to disable source filtering for a specific pattern using
+# *.ext= (so without naming a filter).
+# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
+
+FILTER_SOURCE_PATTERNS =
+
+# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
+# is part of the input, its contents will be placed on the main page
+# (index.html). This can be useful if you have a project on for instance GitHub
+# and want to reuse the introduction page also for the doxygen output.
+
+USE_MDFILE_AS_MAINPAGE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
+# generated. Documented entities will be cross-referenced with these sources.
+#
+# Note: To get rid of all source code in the generated output, make sure that
+# also VERBATIM_HEADERS is set to NO.
+# The default value is: NO.
+
+SOURCE_BROWSER = YES
+
+# Setting the INLINE_SOURCES tag to YES will include the body of functions,
+# classes and enums directly into the documentation.
+# The default value is: NO.
+
+INLINE_SOURCES = NO
+
+# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
+# special comment blocks from generated source code fragments. Normal C, C++ and
+# Fortran comments will always remain visible.
+# The default value is: YES.
+
+STRIP_CODE_COMMENTS = YES
+
+# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
+# entity all documented functions referencing it will be listed.
+# The default value is: NO.
+
+REFERENCED_BY_RELATION = YES
+
+# If the REFERENCES_RELATION tag is set to YES then for each documented function
+# all documented entities called/used by that function will be listed.
+# The default value is: NO.
+
+REFERENCES_RELATION = YES
+
+# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
+# to YES then the hyperlinks from functions in REFERENCES_RELATION and
+# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
+# link to the documentation.
+# The default value is: YES.
+
+REFERENCES_LINK_SOURCE = YES
+
+# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
+# source code will show a tooltip with additional information such as prototype,
+# brief description and links to the definition and documentation. Since this
+# will make the HTML file larger and loading of large files a bit slower, you
+# can opt to disable this feature.
+# The default value is: YES.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+SOURCE_TOOLTIPS = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code will
+# point to the HTML generated by the htags(1) tool instead of doxygen built-in
+# source browser. The htags tool is part of GNU's global source tagging system
+# (see https://www.gnu.org/software/global/global.html). You will need version
+# 4.8.6 or higher.
+#
+# To use it do the following:
+# - Install the latest version of global
+# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file
+# - Make sure the INPUT points to the root of the source tree
+# - Run doxygen as normal
+#
+# Doxygen will invoke htags (and that will in turn invoke gtags), so these
+# tools must be available from the command line (i.e. in the search path).
+#
+# The result: instead of the source browser generated by doxygen, the links to
+# source code will now point to the output of htags.
+# The default value is: NO.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+USE_HTAGS = NO
+
+# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
+# verbatim copy of the header file for each class for which an include is
+# specified. Set to NO to disable this.
+# See also: Section \class.
+# The default value is: YES.
+
+VERBATIM_HEADERS = YES
+
+# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the
+# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the
+# cost of reduced performance. This can be particularly helpful with template
+# rich C++ code for which doxygen's built-in parser lacks the necessary type
+# information.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse_libclang=ON option for CMake.
+# The default value is: NO.
+
+CLANG_ASSISTED_PARSING = NO
+
+# If clang assisted parsing is enabled you can provide the compiler with command
+# line options that you would normally use when invoking the compiler. Note that
+# the include paths will already be set by doxygen for the files and directories
+# specified with INPUT and INCLUDE_PATH.
+# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
+
+CLANG_OPTIONS =
+
+# If clang assisted parsing is enabled you can provide the clang parser with the
+# path to the compilation database (see:
+# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) used when the files
+# were built. This is equivalent to specifying the "-p" option to a clang tool,
+# such as clang-check. These options will then be passed to the parser.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse_libclang=ON option for CMake.
+
+CLANG_DATABASE_PATH =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
+# compounds will be generated. Enable this if the project contains a lot of
+# classes, structs, unions or interfaces.
+# The default value is: YES.
+
+ALPHABETICAL_INDEX = YES
+
+# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
+# which the alphabetical index list will be split.
+# Minimum value: 1, maximum value: 20, default value: 5.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+COLS_IN_ALPHA_INDEX = 5
+
+# In case all classes in a project start with a common prefix, all classes will
+# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
+# can be used to specify a prefix (or a list of prefixes) that should be ignored
+# while generating the index headers.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+IGNORE_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
+# The default value is: YES.
+
+GENERATE_HTML = YES
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_OUTPUT = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
+# generated HTML page (for example: .htm, .php, .asp).
+# The default value is: .html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FILE_EXTENSION = .html
+
+# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
+# each generated HTML page. If the tag is left blank doxygen will generate a
+# standard header.
+#
+# To get valid HTML the header file that includes any scripts and style sheets
+# that doxygen needs, which is dependent on the configuration options used (e.g.
+# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
+# default header using
+# doxygen -w html new_header.html new_footer.html new_stylesheet.css
+# YourConfigFile
+# and then modify the file new_header.html. See also section "Doxygen usage"
+# for information on how to generate the default header that doxygen normally
+# uses.
+# Note: The header is subject to change so you typically have to regenerate the
+# default header when upgrading to a newer version of doxygen. For a description
+# of the possible markers and block names see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_HEADER =
+
+# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
+# generated HTML page. If the tag is left blank doxygen will generate a standard
+# footer. See HTML_HEADER for more information on how to generate a default
+# footer and what special commands can be used inside the footer. See also
+# section "Doxygen usage" for information on how to generate the default footer
+# that doxygen normally uses.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FOOTER =
+
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
+# sheet that is used by each HTML page. It can be used to fine-tune the look of
+# the HTML output. If left blank doxygen will generate a default style sheet.
+# See also section "Doxygen usage" for information on how to generate the style
+# sheet that doxygen normally uses.
+# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
+# it is more robust and this tag (HTML_STYLESHEET) will in the future become
+# obsolete.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_STYLESHEET =
+
+# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# cascading style sheets that are included after the standard style sheets
+# created by doxygen. Using this option one can overrule certain style aspects.
+# This is preferred over using HTML_STYLESHEET since it does not replace the
+# standard style sheet and is therefore more robust against future updates.
+# Doxygen will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list). For an example see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_STYLESHEET =
+
+# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the HTML output directory. Note
+# that these files will be copied to the base HTML output directory. Use the
+# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
+# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
+# files will be copied as-is; there are no commands or markers available.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_FILES =
+
+# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
+# will adjust the colors in the style sheet and background images according to
+# this color. Hue is specified as an angle on a colorwheel, see
+# https://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
+# purple, and 360 is red again.
+# Minimum value: 0, maximum value: 359, default value: 220.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_HUE = 220
+
+# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
+# in the HTML output. For a value of 0 the output will use grayscales only. A
+# value of 255 will produce the most vivid colors.
+# Minimum value: 0, maximum value: 255, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_SAT = 100
+
+# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
+# luminance component of the colors in the HTML output. Values below 100
+# gradually make the output lighter, whereas values above 100 make the output
+# darker. The value divided by 100 is the actual gamma applied, so 80 represents
+# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
+# change the gamma.
+# Minimum value: 40, maximum value: 240, default value: 80.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_GAMMA = 80
+
+# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
+# page will contain the date and time when the page was generated. Setting this
+# to YES can help to show when doxygen was last run and thus if the
+# documentation is up to date.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_TIMESTAMP = NO
+
+# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML
+# documentation will contain a main index with vertical navigation menus that
+# are dynamically created via JavaScript. If disabled, the navigation index will
+# consists of multiple levels of tabs that are statically embedded in every HTML
+# page. Disable this option to support browsers that do not have JavaScript,
+# like the Qt help browser.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_MENUS = YES
+
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_SECTIONS = NO
+
+# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
+# shown in the various tree structured indices initially; the user can expand
+# and collapse entries dynamically later on. Doxygen will expand the tree to
+# such a level that at most the specified number of entries are visible (unless
+# a fully collapsed tree already exceeds this amount). So setting the number of
+# entries 1 will produce a full collapsed tree by default. 0 is a special value
+# representing an infinite number of entries and will result in a full expanded
+# tree by default.
+# Minimum value: 0, maximum value: 9999, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_INDEX_NUM_ENTRIES = 100
+
+# If the GENERATE_DOCSET tag is set to YES, additional index files will be
+# generated that can be used as input for Apple's Xcode 3 integrated development
+# environment (see: https://developer.apple.com/xcode/), introduced with OSX
+# 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# Makefile in the HTML output directory. Running make will produce the docset in
+# that directory and running make install will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
+# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy
+# genXcode/_index.html for more information.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_DOCSET = NO
+
+# This tag determines the name of the docset feed. A documentation feed provides
+# an umbrella under which multiple documentation sets from a single provider
+# (such as a company or product suite) can be grouped.
+# The default value is: Doxygen generated docs.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_FEEDNAME = "Doxygen generated docs"
+
+# This tag specifies a string that should uniquely identify the documentation
+# set bundle. This should be a reverse domain-name style string, e.g.
+# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_BUNDLE_ID = fr.inria.chameleon
+
+# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
+# the documentation publisher. This should be a reverse domain-name style
+# string, e.g. com.mycompany.MyDocSet.documentation.
+# The default value is: org.doxygen.Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_ID = fr.inria.chameleon
+
+# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
+# The default value is: Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_NAME = BordeauxINP,CNRS(LaBRIUMR5800),Inria,Univ.Bordeaux,ICL,KAUST.
+
+# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
+# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
+# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
+# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# Windows.
+#
+# The HTML Help Workshop contains a compiler that can convert all HTML output
+# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
+# files are now used as the Windows 98 help format, and will replace the old
+# Windows help format (.hlp) on all Windows platforms in the future. Compressed
+# HTML files also contain an index, a table of contents, and you can search for
+# words in the documentation. The HTML workshop also contains a viewer for
+# compressed HTML files.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_HTMLHELP = NO
+
+# The CHM_FILE tag can be used to specify the file name of the resulting .chm
+# file. You can add a path in front of the file if the result should not be
+# written to the html output directory.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_FILE =
+
+# The HHC_LOCATION tag can be used to specify the location (absolute path
+# including file name) of the HTML help compiler (hhc.exe). If non-empty,
+# doxygen will try to run the HTML help compiler on the generated index.hhp.
+# The file has to be specified with full path.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+HHC_LOCATION =
+
+# The GENERATE_CHI flag controls if a separate .chi index file is generated
+# (YES) or that it should be included in the master .chm file (NO).
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+GENERATE_CHI = NO
+
+# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc)
+# and project file content.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_INDEX_ENCODING =
+
+# The BINARY_TOC flag controls whether a binary table of contents is generated
+# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it
+# enables the Previous and Next buttons.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+BINARY_TOC = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members to
+# the table of contents of the HTML help documentation and to the tree view.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+TOC_EXPAND = NO
+
+# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
+# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
+# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
+# (.qch) of the generated HTML documentation.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_QHP = NO
+
+# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
+# the file name of the resulting .qch file. The path specified is relative to
+# the HTML output folder.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QCH_FILE =
+
+# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
+# Project output. For more information please see Qt Help Project / Namespace
+# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace).
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_NAMESPACE = edu.utk.eecs.icl.Plasma
+
+# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
+# Help Project output. For more information please see Qt Help Project / Virtual
+# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-
+# folders).
+# The default value is: doc.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_VIRTUAL_FOLDER = doc
+
+# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
+# filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_NAME =
+
+# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
+# custom filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_ATTRS =
+
+# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
+# project's filter section matches. Qt Help Project / Filter Attributes (see:
+# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_SECT_FILTER_ATTRS =
+
+# The QHG_LOCATION tag can be used to specify the location of Qt's
+# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
+# generated .qhp file.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHG_LOCATION =
+
+# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
+# generated, together with the HTML files, they form an Eclipse help plugin. To
+# install this plugin and make it available under the help contents menu in
+# Eclipse, the contents of the directory containing the HTML and XML files needs
+# to be copied into the plugins directory of eclipse. The name of the directory
+# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
+# After copying Eclipse needs to be restarted before the help appears.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_ECLIPSEHELP = NO
+
+# A unique identifier for the Eclipse help plugin. When installing the plugin
+# the directory name containing the HTML and XML files should also have this
+# name. Each documentation set should have its own identifier.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
+
+ECLIPSE_DOC_ID = edu.utk.eecs.icl.Plasma
+
+# If you want full control over the layout of the generated HTML pages it might
+# be necessary to disable the index and replace it with your own. The
+# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
+# of each HTML page. A value of NO enables the index and the value YES disables
+# it. Since the tabs in the index contain the same information as the navigation
+# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+DISABLE_INDEX = NO
+
+# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
+# structure should be generated to display hierarchical information. If the tag
+# value is set to YES, a side panel will be generated containing a tree-like
+# index structure (just like the one that is generated for HTML Help). For this
+# to work a browser that supports JavaScript, DHTML, CSS and frames is required
+# (i.e. any modern browser). Windows users are probably better off using the
+# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can
+# further fine-tune the look of the index. As an example, the default style
+# sheet generated by doxygen has an example that shows how to put an image at
+# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
+# the same information as the tab index, you could consider setting
+# DISABLE_INDEX to YES when enabling this option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_TREEVIEW = YES
+
+# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
+# doxygen will group on one line in the generated HTML documentation.
+#
+# Note that a value of 0 will completely suppress the enum values from appearing
+# in the overview section.
+# Minimum value: 0, maximum value: 20, default value: 4.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+ENUM_VALUES_PER_LINE = 4
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
+# to set the initial width (in pixels) of the frame in which the tree is shown.
+# Minimum value: 0, maximum value: 1500, default value: 250.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+TREEVIEW_WIDTH = 250
+
+# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to
+# external symbols imported via tag files in a separate window.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+EXT_LINKS_IN_WINDOW = NO
+
+# Use this tag to change the font size of LaTeX formulas included as images in
+# the HTML documentation. When you change the font size after a successful
+# doxygen run you need to manually remove any form_*.png images from the HTML
+# output directory to force them to be regenerated.
+# Minimum value: 8, maximum value: 50, default value: 10.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_FONTSIZE = 10
+
+# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
+# generated for formulas are transparent PNGs. Transparent PNGs are not
+# supported properly for IE 6.0, but are supported on all modern browsers.
+#
+# Note that when changing this option you need to delete any form_*.png files in
+# the HTML output directory before the changes have effect.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_TRANSPARENT = YES
+
+# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands
+# to create new LaTeX commands to be used in formulas as building blocks. See
+# the section "Including formulas" for details.
+
+FORMULA_MACROFILE =
+
+# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
+# https://www.mathjax.org) which uses client side JavaScript for the rendering
+# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
+# installed or if you want to formulas look prettier in the HTML output. When
+# enabled you may also need to install MathJax separately and configure the path
+# to it using the MATHJAX_RELPATH option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+USE_MATHJAX = NO
+
+# When MathJax is enabled you can set the default output format to be used for
+# the MathJax output. See the MathJax site (see:
+# http://docs.mathjax.org/en/latest/output.html) for more details.
+# Possible values are: HTML-CSS (which is slower, but has the best
+# compatibility), NativeMML (i.e. MathML) and SVG.
+# The default value is: HTML-CSS.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_FORMAT = HTML-CSS
+
+# When MathJax is enabled you need to specify the location relative to the HTML
+# output directory using the MATHJAX_RELPATH option. The destination directory
+# should contain the MathJax.js script. For instance, if the mathjax directory
+# is located at the same level as the HTML output directory, then
+# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
+# Content Delivery Network so you can quickly see the result without installing
+# MathJax. However, it is strongly recommended to install a local copy of
+# MathJax from https://www.mathjax.org before deployment.
+# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_RELPATH = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/
+
+# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
+# extension names that should be enabled during MathJax rendering. For example
+# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_EXTENSIONS =
+
+# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
+# of code that will be used on startup of the MathJax code. See the MathJax site
+# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
+# example see the documentation.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_CODEFILE =
+
+# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
+# the HTML output. The underlying search engine uses javascript and DHTML and
+# should work on any modern browser. Note that when using HTML help
+# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
+# there is already a search function so this one should typically be disabled.
+# For large projects the javascript based search engine can be slow, then
+# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
+# search using the keyboard; to jump to the search box use <access key> + S
+# (what the <access key> is depends on the OS and browser, but it is typically
+# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down
+# key> to jump into the search results window, the results can be navigated
+# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel
+# the search. The filter options can be selected when the cursor is inside the
+# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>
+# to select a filter and <Enter> or <escape> to activate or cancel the filter
+# option.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+SEARCHENGINE = YES
+
+# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
+# implemented using a web server instead of a web client using JavaScript. There
+# are two flavors of web server based searching depending on the EXTERNAL_SEARCH
+# setting. When disabled, doxygen will generate a PHP script for searching and
+# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
+# and searching needs to be provided by external tools. See the section
+# "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SERVER_BASED_SEARCH = NO
+
+# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
+# script for searching. Instead the search results are written to an XML file
+# which needs to be processed by an external indexer. Doxygen will invoke an
+# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
+# search results.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: https://xapian.org/).
+#
+# See the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH = NO
+
+# The SEARCHENGINE_URL should point to a search engine hosted by a web server
+# which will return the search results when EXTERNAL_SEARCH is enabled.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: https://xapian.org/). See the section "External Indexing and
+# Searching" for details.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHENGINE_URL =
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
+# search data is written to a file for indexing by an external tool. With the
+# SEARCHDATA_FILE tag the name of this file can be specified.
+# The default file is: searchdata.xml.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHDATA_FILE = searchdata.xml
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
+# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
+# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
+# projects and redirect the results back to the right project.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH_ID =
+
+# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
+# projects other than the one defined by this configuration file, but that are
+# all added to the same external search index. Each project needs to have a
+# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
+# to a relative location where the documentation can be found. The format is:
+# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTRA_SEARCH_MAPPINGS =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX = NO
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_OUTPUT = latex
+
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
+# invoked.
+#
+# Note that when not enabling USE_PDFLATEX the default is latex when enabling
+# USE_PDFLATEX the default is pdflatex and when in the later case latex is
+# chosen this is overwritten by pdflatex. For specific output languages the
+# default can have been set differently, this depends on the implementation of
+# the output language.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_CMD_NAME = latex
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
+# index for LaTeX.
+# Note: This tag is used in the Makefile / make.bat.
+# See also: LATEX_MAKEINDEX_CMD for the part in the generated output file
+# (.tex).
+# The default file is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+MAKEINDEX_CMD_NAME = makeindex
+
+# The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to
+# generate index for LaTeX. In case there is no backslash (\) as first character
+# it will be automatically added in the LaTeX code.
+# Note: This tag is used in the generated output file (.tex).
+# See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat.
+# The default value is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_MAKEINDEX_CMD = makeindex
+
+# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+COMPACT_LATEX = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used by the
+# printer.
+# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
+# 14 inches) and executive (7.25 x 10.5 inches).
+# The default value is: a4.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PAPER_TYPE = a4wide
+
+# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
+# that should be included in the LaTeX output. The package can be specified just
+# by its name or with the correct syntax as to be used with the LaTeX
+# \usepackage command. To get the times font for instance you can specify :
+# EXTRA_PACKAGES=times or EXTRA_PACKAGES={times}
+# To use the option intlimits with the amsmath package you can specify:
+# EXTRA_PACKAGES=[intlimits]{amsmath}
+# If left blank no extra packages will be included.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+EXTRA_PACKAGES =
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
+# generated LaTeX document. The header should contain everything until the first
+# chapter. If it is left blank doxygen will generate a standard header. See
+# section "Doxygen usage" for information on how to let doxygen write the
+# default header to a separate file.
+#
+# Note: Only use a user-defined header if you know what you are doing! The
+# following commands have a special meaning inside the header: $title,
+# $datetime, $date, $doxygenversion, $projectname, $projectnumber,
+# $projectbrief, $projectlogo. Doxygen will replace $title with the empty
+# string, for the replacement values of the other commands the user is referred
+# to HTML_HEADER.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HEADER =
+
+# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
+# generated LaTeX document. The footer should contain everything after the last
+# chapter. If it is left blank doxygen will generate a standard footer. See
+# LATEX_HEADER for more information on how to generate a default footer and what
+# special commands can be used inside the footer.
+#
+# Note: Only use a user-defined footer if you know what you are doing!
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_FOOTER =
+
+# The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# LaTeX style sheets that are included after the standard style sheets created
+# by doxygen. Using this option one can overrule certain style aspects. Doxygen
+# will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list).
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_STYLESHEET =
+
+# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the LATEX_OUTPUT output
+# directory. Note that the files will be copied as-is; there are no commands or
+# markers available.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_FILES =
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
+# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
+# contain links (just like the HTML output) instead of page references. This
+# makes the output suitable for online browsing using a PDF viewer.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PDF_HYPERLINKS = YES
+
+# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
+# the PDF file directly from the LaTeX files. Set this option to YES, to get a
+# higher quality PDF documentation.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+USE_PDFLATEX = YES
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
+# command to the generated LaTeX files. This will instruct LaTeX to keep running
+# if errors occur, instead of asking the user for help. This option is also used
+# when generating formulas in HTML.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BATCHMODE = NO
+
+# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
+# index chapters (such as File Index, Compound Index, etc.) in the output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HIDE_INDICES = NO
+
+# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
+# code with syntax highlighting in the LaTeX output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_SOURCE_CODE = NO
+
+# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
+# bibliography, e.g. plainnat, or ieeetr. See
+# https://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# The default value is: plain.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BIB_STYLE = plain
+
+# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated
+# page will contain the date and time when the page was generated. Setting this
+# to NO can help when comparing the output of multiple runs.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_TIMESTAMP = NO
+
+# The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute)
+# path from which the emoji images will be read. If a relative path is entered,
+# it will be relative to the LATEX_OUTPUT directory. If left blank the
+# LATEX_OUTPUT directory will be used.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EMOJI_DIRECTORY =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The
+# RTF output is optimized for Word 97 and may not look too pretty with other RTF
+# readers/editors.
+# The default value is: NO.
+
+GENERATE_RTF = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: rtf.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_OUTPUT = rtf
+
+# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+COMPACT_RTF = NO
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
+# contain hyperlink fields. The RTF file will contain links (just like the HTML
+# output) instead of page references. This makes the output suitable for online
+# browsing using Word or some other Word compatible readers that support those
+# fields.
+#
+# Note: WordPad (write) and others do not support links.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_HYPERLINKS = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's
+# configuration file, i.e. a series of assignments. You only have to provide
+# replacements, missing definitions are set to their default value.
+#
+# See also section "Doxygen usage" for information on how to generate the
+# default style sheet that doxygen normally uses.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_STYLESHEET_FILE =
+
+# Set optional variables used in the generation of an RTF document. Syntax is
+# similar to doxygen's configuration file. A template extensions file can be
+# generated using doxygen -e rtf extensionFile.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_EXTENSIONS_FILE =
+
+# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
+# with syntax highlighting in the RTF output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_SOURCE_CODE = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for
+# classes and files.
+# The default value is: NO.
+
+GENERATE_MAN = NO
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it. A directory man3 will be created inside the directory specified by
+# MAN_OUTPUT.
+# The default directory is: man.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_OUTPUT = man
+
+# The MAN_EXTENSION tag determines the extension that is added to the generated
+# man pages. In case the manual section does not start with a number, the number
+# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
+# optional.
+# The default value is: .3.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_EXTENSION = .3
+
+# The MAN_SUBDIR tag determines the name of the directory created within
+# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by
+# MAN_EXTENSION with the initial . removed.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_SUBDIR =
+
+# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
+# will generate one additional man file for each entity documented in the real
+# man page(s). These additional files only source the real man page, but without
+# them the man command would be unable to find the correct page.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_LINKS = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
+
+GENERATE_XML = NO
+
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: xml.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_OUTPUT = xml
+
+# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program
+# listings (including syntax highlighting and cross-referencing information) to
+# the XML output. Note that enabling this will significantly increase the size
+# of the XML output.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_PROGRAMLISTING = YES
+
+# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include
+# namespace members in file scope as well, matching the HTML output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_NS_MEMB_FILE_SCOPE = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files
+# that can be used to generate PDF.
+# The default value is: NO.
+
+GENERATE_DOCBOOK = NO
+
+# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
+# front of it.
+# The default directory is: docbook.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_OUTPUT = docbook
+
+# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the
+# program listings (including syntax highlighting and cross-referencing
+# information) to the DOCBOOK output. Note that enabling this will significantly
+# increase the size of the DOCBOOK output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_PROGRAMLISTING = NO
+
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
+# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures
+# the structure of the code including all documentation. Note that this feature
+# is still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_AUTOGEN_DEF = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module
+# file that captures the structure of the code including all documentation.
+#
+# Note that this feature is still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_PERLMOD = NO
+
+# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary
+# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
+# output from the Perl module output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_LATEX = NO
+
+# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely
+# formatted so it can be parsed by a human reader. This is useful if you want to
+# understand what is going on. On the other hand, if this tag is set to NO, the
+# size of the Perl module output will be much smaller and Perl will parse it
+# just the same.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_PRETTY = YES
+
+# The names of the make variables in the generated doxyrules.make file are
+# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
+# so different doxyrules.make files included by the same Makefile don't
+# overwrite each other's variables.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_MAKEVAR_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
+
+ENABLE_PREPROCESSING = YES
+
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION = NO
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF = NO
+
+# If the SEARCH_INCLUDES tag is set to YES, the include files in the
+# INCLUDE_PATH will be searched if a #include is found.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SEARCH_INCLUDES = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH =
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
+# patterns (like *.h and *.hpp) to filter out the header-files in the
+# directories. If left blank, the patterns specified with FILE_PATTERNS will be
+# used.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+INCLUDE_FILE_PATTERNS =
+
+# The PREDEFINED tag can be used to specify one or more macro names that are
+# defined before the preprocessor is started (similar to the -D option of e.g.
+# gcc). The argument of the tag is a list of macros of the form: name or
+# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
+# is assumed. To prevent a macro definition from being undefined via #undef or
+# recursively expanded use the := operator instead of the = operator.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+PREDEFINED =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED =
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
+# remove all references to function-like macros that are alone on a line, have
+# an all uppercase name, and do not end with a semicolon. Such function macros
+# are typically used for boiler-plate code, and will confuse the parser if not
+# removed.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SKIP_FUNCTION_MACROS = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to external references
+#---------------------------------------------------------------------------
+
+# The TAGFILES tag can be used to specify one or more tag files. For each tag
+# file the location of the external documentation should be added. The format of
+# a tag file without this location is as follows:
+# TAGFILES = file1 file2 ...
+# Adding location for the tag files is done as follows:
+# TAGFILES = file1=loc1 "file2 = loc2" ...
+# where loc1 and loc2 can be relative or absolute paths or URLs. See the
+# section "Linking to external documentation" for more information about the use
+# of tag files.
+# Note: Each tag file must have a unique name (where the name does NOT include
+# the path). If a tag file is not located in the directory in which doxygen is
+# run, you must also specify the path to the tagfile here.
+
+TAGFILES =
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
+# tag file that is based on the input files it reads. See section "Linking to
+# external documentation" for more information about the usage of tag files.
+
+GENERATE_TAGFILE =
+
+# If the ALLEXTERNALS tag is set to YES, all external class will be listed in
+# the class index. If set to NO, only the inherited external classes will be
+# listed.
+# The default value is: NO.
+
+ALLEXTERNALS = NO
+
+# If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed
+# in the modules index. If set to NO, only the current project's groups will be
+# listed.
+# The default value is: YES.
+
+EXTERNAL_GROUPS = YES
+
+# If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in
+# the related pages index. If set to NO, only the current project's pages will
+# be listed.
+# The default value is: YES.
+
+EXTERNAL_PAGES = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram
+# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
+# NO turns the diagrams off. Note that this option also works with HAVE_DOT
+# disabled, but it is recommended to install and use dot, since it yields more
+# powerful graphs.
+# The default value is: YES.
+
+CLASS_DIAGRAMS = YES
+
+# You can include diagrams made with dia in doxygen documentation. Doxygen will
+# then run dia to produce the diagram and insert it in the documentation. The
+# DIA_PATH tag allows you to specify the directory where the dia binary resides.
+# If left empty dia is assumed to be found in the default search path.
+
+DIA_PATH =
+
+# If set to YES the inheritance and collaboration graphs will hide inheritance
+# and usage relations if the target is undocumented or is not a class.
+# The default value is: YES.
+
+HIDE_UNDOC_RELATIONS = YES
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
+# available from the path. This tool is part of Graphviz (see:
+# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
+# Bell Labs. The other options in this section have no effect if this option is
+# set to NO
+# The default value is: YES.
+
+HAVE_DOT = NO
+
+# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
+# to run in parallel. When set to 0 doxygen will base this on the number of
+# processors available in the system. You can set it explicitly to a value
+# larger than 0 to get control over the balance between CPU load and processing
+# speed.
+# Minimum value: 0, maximum value: 32, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_NUM_THREADS = 0
+
+# When you want a differently looking font in the dot files that doxygen
+# generates you can specify the font name using DOT_FONTNAME. You need to make
+# sure dot is able to find the font, which can be done by putting it in a
+# standard location or by setting the DOTFONTPATH environment variable or by
+# setting DOT_FONTPATH to the directory containing the font.
+# The default value is: Helvetica.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTNAME = FreeSans.ttf
+
+# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
+# dot graphs.
+# Minimum value: 4, maximum value: 24, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTSIZE = 10
+
+# By default doxygen will tell dot to use the default font as specified with
+# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
+# the path where dot can find it using this tag.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTPATH =
+
+# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
+# each documented class showing the direct and indirect inheritance relations.
+# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CLASS_GRAPH = YES
+
+# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
+# graph for each documented class showing the direct and indirect implementation
+# dependencies (inheritance, containment, and class references variables) of the
+# class with other documented classes.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+COLLABORATION_GRAPH = YES
+
+# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
+# groups, showing the direct groups dependencies.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GROUP_GRAPHS = YES
+
+# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and
+# collaboration diagrams in a style similar to the OMG's Unified Modeling
+# Language.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LOOK = NO
+
+# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
+# class node. If there are many fields or methods and many nodes the graph may
+# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
+# number of items for each type to make the size more manageable. Set this to 0
+# for no limit. Note that the threshold may be exceeded by 50% before the limit
+# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
+# but if the number exceeds 15, the total amount of fields shown is limited to
+# 10.
+# Minimum value: 0, maximum value: 100, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LIMIT_NUM_FIELDS = 10
+
+# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
+# collaboration graphs will show the relations between templates and their
+# instances.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+TEMPLATE_RELATIONS = NO
+
+# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
+# YES then doxygen will generate a graph for each documented file showing the
+# direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDE_GRAPH = YES
+
+# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
+# set to YES then doxygen will generate a graph for each documented file showing
+# the direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDED_BY_GRAPH = YES
+
+# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable call graphs for selected
+# functions only using the \callgraph command. Disabling a call graph can be
+# accomplished by means of the command \hidecallgraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALL_GRAPH = NO
+
+# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable caller graphs for selected
+# functions only using the \callergraph command. Disabling a caller graph can be
+# accomplished by means of the command \hidecallergraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALLER_GRAPH = NO
+
+# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
+# hierarchy of all classes instead of a textual one.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GRAPHICAL_HIERARCHY = YES
+
+# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
+# dependencies a directory has on other directories in a graphical way. The
+# dependency relations are determined by the #include relations between the
+# files in the directories.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DIRECTORY_GRAPH = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
+# generated by dot. For an explanation of the image formats see the section
+# output formats in the documentation of the dot tool (Graphviz (see:
+# http://www.graphviz.org/)).
+# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
+# to make the SVG files visible in IE 9+ (other browsers do not have this
+# requirement).
+# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd,
+# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo,
+# gif:cairo:gd, gif:gd, gif:gd:gd, svg, png:gd, png:gd:gd, png:cairo,
+# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and
+# png:gdiplus:gdiplus.
+# The default value is: png.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_IMAGE_FORMAT = png
+
+# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
+# enable generation of interactive SVG images that allow zooming and panning.
+#
+# Note that this requires a modern browser other than Internet Explorer. Tested
+# and working are Firefox, Chrome, Safari, and Opera.
+# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
+# the SVG files visible. Older versions of IE do not have SVG support.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INTERACTIVE_SVG = NO
+
+# The DOT_PATH tag can be used to specify the path where the dot tool can be
+# found. If left blank, it is assumed the dot tool can be found in the path.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_PATH =
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that
+# contain dot files that are included in the documentation (see the \dotfile
+# command).
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOTFILE_DIRS =
+
+# The MSCFILE_DIRS tag can be used to specify one or more directories that
+# contain msc files that are included in the documentation (see the \mscfile
+# command).
+
+MSCFILE_DIRS =
+
+# The DIAFILE_DIRS tag can be used to specify one or more directories that
+# contain dia files that are included in the documentation (see the \diafile
+# command).
+
+DIAFILE_DIRS =
+
+# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
+# path where java can find the plantuml.jar file. If left blank, it is assumed
+# PlantUML is not used or called during a preprocessing step. Doxygen will
+# generate a warning when it encounters a \startuml command in this case and
+# will not generate output for the diagram.
+
+PLANTUML_JAR_PATH =
+
+# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a
+# configuration file for plantuml.
+
+PLANTUML_CFG_FILE =
+
+# When using plantuml, the specified paths are searched for files specified by
+# the !include statement in a plantuml block.
+
+PLANTUML_INCLUDE_PATH =
+
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
+# that will be shown in the graph. If the number of nodes in a graph becomes
+# larger than this value, doxygen will truncate the graph, which is visualized
+# by representing a node as a red box. Note that doxygen if the number of direct
+# children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
+# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+# Minimum value: 0, maximum value: 10000, default value: 50.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_GRAPH_MAX_NODES = 50
+
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
+# generated by dot. A depth value of 3 means that only nodes reachable from the
+# root by following a path via at most 3 edges will be shown. Nodes that lay
+# further from the root node will be omitted. Note that setting this option to 1
+# or 2 may greatly reduce the computation time needed for large code bases. Also
+# note that the size of a graph can be further restricted by
+# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+# Minimum value: 0, maximum value: 1000, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+MAX_DOT_GRAPH_DEPTH = 0
+
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
+# background. This is disabled by default, because dot on Windows does not seem
+# to support this out of the box.
+#
+# Warning: Depending on the platform used, enabling this option may lead to
+# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
+# read).
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_TRANSPARENT = NO
+
+# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output
+# files in one run (i.e. multiple -o and -T options on the command line). This
+# makes dot run faster, but since only newer versions of dot (>1.8.10) support
+# this, this feature is disabled by default.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_MULTI_TARGETS = YES
+
+# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
+# explaining the meaning of the various boxes and arrows in the dot generated
+# graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GENERATE_LEGEND = YES
+
+# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot
+# files that are used to generate the various graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_CLEANUP = YES
diff --git a/doc/doxygen/main.dox b/doc/dev/main.dox
similarity index 98%
rename from doc/doxygen/main.dox
rename to doc/dev/main.dox
index 12edd52c1eeb12f176220318f4ec5453d1df1200..e4ee8ecbd42362f581169c95db3623d3ae2f39c6 100644
--- a/doc/doxygen/main.dox
+++ b/doc/dev/main.dox
@@ -7,7 +7,7 @@ as well as the source code organization.
To get information about the Chameleon project, installation guide and
usage examples please refer to the [user's
-guide](https://solverstack.gitlabpages.inria.fr/chameleon/users_guide.html).
+guide](https://solverstack.gitlabpages.inria.fr/chameleon/index.html).
Chameleon user's API is mostly composed of linear algebra routines of
the form **CHAMELEON_name[_Tile[_Async]]** where *name* follows the
diff --git a/doc/doxygen/conf.dox.in b/doc/doxygen/conf.dox.in
deleted file mode 100644
index fbd4553685ffdbe9b076727347841d46001b1400..0000000000000000000000000000000000000000
--- a/doc/doxygen/conf.dox.in
+++ /dev/null
@@ -1,1639 +0,0 @@
-# Doxyfile 1.7.1
-
-# This file describes the settings to be used by the documentation system
-# doxygen (www.doxygen.org) for a project
-#
-# All text after a hash (#) is considered a comment and will be ignored
-# The format is:
-# TAG = value [value, ...]
-# For lists items can also be appended using:
-# TAG += value [value, ...]
-# Values that contain spaces should be placed between quotes (" ")
-
-#---------------------------------------------------------------------------
-# Project related configuration options
-#---------------------------------------------------------------------------
-
-# This tag specifies the encoding used for all characters in the config file
-# that follow. The default is UTF-8 which is also the encoding used for all
-# text before the first occurrence of this tag. Doxygen uses libiconv (or the
-# iconv built into libc) for the transcoding. See
-# http://www.gnu.org/software/libiconv for the list of possible encodings.
-
-DOXYFILE_ENCODING = UTF-8
-
-# The PROJECT_NAME tag is a single word (or a sequence of words surrounded
-# by quotes) that should identify the project.
-
-PROJECT_NAME = CHAMELEON
-
-# The PROJECT_NUMBER tag can be used to enter a project or revision number.
-# This could be handy for archiving the generated documentation or
-# if some version control system is used.
-
-PROJECT_NUMBER = @CHAMELEON_VERSION_MAJOR@.@CHAMELEON_VERSION_MINOR@.@CHAMELEON_VERSION_MICRO@
-
-# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute)
-# base path where the generated documentation will be put.
-# If a relative path is entered, it will be relative to the location
-# where doxygen was started. If left blank the current directory will be used.
-
-OUTPUT_DIRECTORY = @CMAKE_CURRENT_BINARY_DIR@/out
-
-# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create
-# 4096 sub-directories (in 2 levels) under the output directory of each output
-# format and will distribute the generated files over these directories.
-# Enabling this option can be useful when feeding doxygen a huge amount of
-# source files, where putting all generated files in the same directory would
-# otherwise cause performance problems for the file system.
-
-CREATE_SUBDIRS = NO
-
-# The OUTPUT_LANGUAGE tag is used to specify the language in which all
-# documentation generated by doxygen is written. Doxygen will use this
-# information to generate all constant output in the proper language.
-# The default language is English, other supported languages are:
-# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional,
-# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German,
-# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English
-# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian,
-# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak,
-# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese.
-
-OUTPUT_LANGUAGE = English
-
-# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will
-# include brief member descriptions after the members that are listed in
-# the file and class documentation (similar to JavaDoc).
-# Set to NO to disable this.
-
-BRIEF_MEMBER_DESC = YES
-
-# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend
-# the brief description of a member or function before the detailed description.
-# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
-# brief descriptions will be completely suppressed.
-
-REPEAT_BRIEF = YES
-
-# This tag implements a quasi-intelligent brief description abbreviator
-# that is used to form the text in various listings. Each string
-# in this list, if found as the leading text of the brief description, will be
-# stripped from the text and the result after processing the whole list, is
-# used as the annotated text. Otherwise, the brief description is used as-is.
-# If left blank, the following values are used ("$name" is automatically
-# replaced with the name of the entity): "The $name class" "The $name widget"
-# "The $name file" "is" "provides" "specifies" "contains"
-# "represents" "a" "an" "the"
-
-ABBREVIATE_BRIEF =
-
-# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
-# Doxygen will generate a detailed section even if there is only a brief
-# description.
-
-ALWAYS_DETAILED_SEC = NO
-
-# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
-# inherited members of a class in the documentation of that class as if those
-# members were ordinary class members. Constructors, destructors and assignment
-# operators of the base classes will not be shown.
-
-INLINE_INHERITED_MEMB = NO
-
-# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full
-# path before files name in the file list and in the header files. If set
-# to NO the shortest path that makes the file name unique will be used.
-
-FULL_PATH_NAMES = NO
-
-# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag
-# can be used to strip a user-defined part of the path. Stripping is
-# only done if one of the specified strings matches the left-hand part of
-# the path. The tag can be used to show relative paths in the file list.
-# If left blank the directory from which doxygen is run is used as the
-# path to strip.
-
-STRIP_FROM_PATH =
-
-# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of
-# the path mentioned in the documentation of a class, which tells
-# the reader which header file to include in order to use a class.
-# If left blank only the name of the header file containing the class
-# definition is used. Otherwise one should specify the include paths that
-# are normally passed to the compiler using the -I flag.
-
-STRIP_FROM_INC_PATH =
-
-# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter
-# (but less readable) file names. This can be useful is your file systems
-# doesn't support long names like on DOS, Mac, or CD-ROM.
-
-SHORT_NAMES = NO
-
-# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen
-# will interpret the first line (until the first dot) of a JavaDoc-style
-# comment as the brief description. If set to NO, the JavaDoc
-# comments will behave just like regular Qt-style comments
-# (thus requiring an explicit @brief command for a brief description.)
-
-JAVADOC_AUTOBRIEF = NO
-
-# If the QT_AUTOBRIEF tag is set to YES then Doxygen will
-# interpret the first line (until the first dot) of a Qt-style
-# comment as the brief description. If set to NO, the comments
-# will behave just like regular Qt-style comments (thus requiring
-# an explicit \brief command for a brief description.)
-
-QT_AUTOBRIEF = NO
-
-# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen
-# treat a multi-line C++ special comment block (i.e. a block of //! or ///
-# comments) as a brief description. This used to be the default behaviour.
-# The new default is to treat a multi-line C++ comment block as a detailed
-# description. Set this tag to YES if you prefer the old behaviour instead.
-
-MULTILINE_CPP_IS_BRIEF = NO
-
-# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented
-# member inherits the documentation from any documented member that it
-# re-implements.
-
-INHERIT_DOCS = YES
-
-# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce
-# a new page for each member. If set to NO, the documentation of a member will
-# be part of the file/class/namespace that contains it.
-
-SEPARATE_MEMBER_PAGES = YES
-
-# The TAB_SIZE tag can be used to set the number of spaces in a tab.
-# Doxygen uses this value to replace tabs by spaces in code fragments.
-
-TAB_SIZE = 4
-
-# This tag can be used to specify a number of aliases that acts
-# as commands in the documentation. An alias has the form "name=value".
-# For example adding "sideeffect=\par Side Effects:\n" will allow you to
-# put the command \sideeffect (or @sideeffect) in the documentation, which
-# will result in a user-defined paragraph with heading "Side Effects:".
-# You can put \n's in the value part of an alias to insert newlines.
-
-ALIASES =
-
-# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C
-# sources only. Doxygen will then generate output that is more tailored for C.
-# For instance, some of the names that are used will be different. The list
-# of all members will be omitted, etc.
-
-OPTIMIZE_OUTPUT_FOR_C = YES
-
-# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java
-# sources only. Doxygen will then generate output that is more tailored for
-# Java. For instance, namespaces will be presented as packages, qualified
-# scopes will look different, etc.
-
-OPTIMIZE_OUTPUT_JAVA = NO
-
-# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
-# sources only. Doxygen will then generate output that is more tailored for
-# Fortran.
-
-OPTIMIZE_FOR_FORTRAN = YES
-
-# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
-# sources. Doxygen will then generate output that is tailored for
-# VHDL.
-
-OPTIMIZE_OUTPUT_VHDL = NO
-
-# Doxygen selects the parser to use depending on the extension of the files it
-# parses. With this tag you can assign which parser to use for a given extension.
-# Doxygen has a built-in mapping, but you can override or extend it using this
-# tag. The format is ext=language, where ext is a file extension, and language
-# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C,
-# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make
-# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C
-# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions
-# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen.
-
-EXTENSION_MAPPING =
-
-# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
-# to include (a tag file for) the STL sources as input, then you should
-# set this tag to YES in order to let doxygen match functions declarations and
-# definitions whose arguments contain STL classes (e.g. func(std::string); v.s.
-# func(std::string) {}). This also make the inheritance and collaboration
-# diagrams that involve STL classes more complete and accurate.
-
-BUILTIN_STL_SUPPORT = NO
-
-# If you use Microsoft's C++/CLI language, you should set this option to YES to
-# enable parsing support.
-
-CPP_CLI_SUPPORT = NO
-
-# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only.
-# Doxygen will parse them like normal C++ but will assume all classes use public
-# instead of private inheritance when no explicit protection keyword is present.
-
-SIP_SUPPORT = NO
-
-# For Microsoft's IDL there are propget and propput attributes to indicate getter
-# and setter methods for a property. Setting this option to YES (the default)
-# will make doxygen to replace the get and set methods by a property in the
-# documentation. This will only work if the methods are indeed getting or
-# setting a simple type. If this is not the case, or you want to show the
-# methods anyway, you should set this option to NO.
-
-IDL_PROPERTY_SUPPORT = YES
-
-# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
-# tag is set to YES, then doxygen will reuse the documentation of the first
-# member in the group (if any) for the other members of the group. By default
-# all members of a group must be documented explicitly.
-
-DISTRIBUTE_GROUP_DOC = NO
-
-# Set the SUBGROUPING tag to YES (the default) to allow class member groups of
-# the same type (for instance a group of public functions) to be put as a
-# subgroup of that type (e.g. under the Public Functions section). Set it to
-# NO to prevent subgrouping. Alternatively, this can be done per class using
-# the \nosubgrouping command.
-
-SUBGROUPING = YES
-
-# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum
-# is documented as struct, union, or enum with the name of the typedef. So
-# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
-# with name TypeT. When disabled the typedef will appear as a member of a file,
-# namespace, or class. And the struct will be named TypeS. This can typically
-# be useful for C code in case the coding convention dictates that all compound
-# types are typedef'ed and only the typedef is referenced, never the tag name.
-
-TYPEDEF_HIDES_STRUCT = NO
-
-# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to
-# determine which symbols to keep in memory and which to flush to disk.
-# When the cache is full, less often used symbols will be written to disk.
-# For small to medium size projects (<1000 input files) the default value is
-# probably good enough. For larger projects a too small cache size can cause
-# doxygen to be busy swapping symbols to and from disk most of the time
-# causing a significant performance penality.
-# If the system has enough physical memory increasing the cache will improve the
-# performance by keeping more symbols in memory. Note that the value works on
-# a logarithmic scale so increasing the size by one will rougly double the
-# memory usage. The cache size is given by this formula:
-# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0,
-# corresponding to a cache size of 2^16 = 65536 symbols
-
-SYMBOL_CACHE_SIZE = 0
-
-#---------------------------------------------------------------------------
-# Build related configuration options
-#---------------------------------------------------------------------------
-
-# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in
-# documentation are documented, even if no documentation was available.
-# Private class members and static file members will be hidden unless
-# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES
-
-EXTRACT_ALL = NO
-
-# If the EXTRACT_PRIVATE tag is set to YES all private members of a class
-# will be included in the documentation.
-
-EXTRACT_PRIVATE = YES
-
-# If the EXTRACT_STATIC tag is set to YES all static members of a file
-# will be included in the documentation.
-
-EXTRACT_STATIC = YES
-
-# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs)
-# defined locally in source files will be included in the documentation.
-# If set to NO only classes defined in header files are included.
-
-EXTRACT_LOCAL_CLASSES = YES
-
-# This flag is only useful for Objective-C code. When set to YES local
-# methods, which are defined in the implementation section but not in
-# the interface are included in the documentation.
-# If set to NO (the default) only methods in the interface are included.
-
-EXTRACT_LOCAL_METHODS = NO
-
-# If this flag is set to YES, the members of anonymous namespaces will be
-# extracted and appear in the documentation as a namespace called
-# 'anonymous_namespace{file}', where file will be replaced with the base
-# name of the file that contains the anonymous namespace. By default
-# anonymous namespace are hidden.
-
-EXTRACT_ANON_NSPACES = NO
-
-# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all
-# undocumented members of documented classes, files or namespaces.
-# If set to NO (the default) these members will be included in the
-# various overviews, but no documentation section is generated.
-# This option has no effect if EXTRACT_ALL is enabled.
-
-HIDE_UNDOC_MEMBERS = NO
-
-# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all
-# undocumented classes that are normally visible in the class hierarchy.
-# If set to NO (the default) these classes will be included in the various
-# overviews. This option has no effect if EXTRACT_ALL is enabled.
-
-HIDE_UNDOC_CLASSES = NO
-
-# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all
-# friend (class|struct|union) declarations.
-# If set to NO (the default) these declarations will be included in the
-# documentation.
-
-HIDE_FRIEND_COMPOUNDS = NO
-
-# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any
-# documentation blocks found inside the body of a function.
-# If set to NO (the default) these blocks will be appended to the
-# function's detailed documentation block.
-
-HIDE_IN_BODY_DOCS = NO
-
-# The INTERNAL_DOCS tag determines if documentation
-# that is typed after a \internal command is included. If the tag is set
-# to NO (the default) then the documentation will be excluded.
-# Set it to YES to include the internal documentation.
-
-INTERNAL_DOCS = YES
-
-# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate
-# file names in lower-case letters. If set to YES upper-case letters are also
-# allowed. This is useful if you have classes or files whose names only differ
-# in case and if your file system supports case sensitive file names. Windows
-# and Mac users are advised to set this option to NO.
-
-CASE_SENSE_NAMES = YES
-
-# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen
-# will show members with their full class and namespace scopes in the
-# documentation. If set to YES the scope will be hidden.
-
-HIDE_SCOPE_NAMES = NO
-
-# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen
-# will put a list of the files that are included by a file in the documentation
-# of that file.
-
-SHOW_INCLUDE_FILES = YES
-
-# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen
-# will list include files with double quotes in the documentation
-# rather than with sharp brackets.
-
-FORCE_LOCAL_INCLUDES = NO
-
-# If the INLINE_INFO tag is set to YES (the default) then a tag [inline]
-# is inserted in the documentation for inline members.
-
-INLINE_INFO = YES
-
-# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen
-# will sort the (detailed) documentation of file and class members
-# alphabetically by member name. If set to NO the members will appear in
-# declaration order.
-
-SORT_MEMBER_DOCS = YES
-
-# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the
-# brief documentation of file, namespace and class members alphabetically
-# by member name. If set to NO (the default) the members will appear in
-# declaration order.
-
-SORT_BRIEF_DOCS = NO
-
-# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen
-# will sort the (brief and detailed) documentation of class members so that
-# constructors and destructors are listed first. If set to NO (the default)
-# the constructors will appear in the respective orders defined by
-# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS.
-# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO
-# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO.
-
-SORT_MEMBERS_CTORS_1ST = NO
-
-# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the
-# hierarchy of group names into alphabetical order. If set to NO (the default)
-# the group names will appear in their defined order.
-
-SORT_GROUP_NAMES = YES
-
-# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be
-# sorted by fully-qualified names, including namespaces. If set to
-# NO (the default), the class list will be sorted only by class name,
-# not including the namespace part.
-# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
-# Note: This option applies only to the class list, not to the
-# alphabetical list.
-
-SORT_BY_SCOPE_NAME = NO
-
-# The GENERATE_TODOLIST tag can be used to enable (YES) or
-# disable (NO) the todo list. This list is created by putting \todo
-# commands in the documentation.
-
-GENERATE_TODOLIST = YES
-
-# The GENERATE_TESTLIST tag can be used to enable (YES) or
-# disable (NO) the test list. This list is created by putting \test
-# commands in the documentation.
-
-GENERATE_TESTLIST = YES
-
-# The GENERATE_BUGLIST tag can be used to enable (YES) or
-# disable (NO) the bug list. This list is created by putting \bug
-# commands in the documentation.
-
-GENERATE_BUGLIST = YES
-
-# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or
-# disable (NO) the deprecated list. This list is created by putting
-# \deprecated commands in the documentation.
-
-GENERATE_DEPRECATEDLIST= YES
-
-# The ENABLED_SECTIONS tag can be used to enable conditional
-# documentation sections, marked by \if sectionname ... \endif.
-
-ENABLED_SECTIONS =
-
-# The MAX_INITIALIZER_LINES tag determines the maximum number of lines
-# the initial value of a variable or define consists of for it to appear in
-# the documentation. If the initializer consists of more lines than specified
-# here it will be hidden. Use a value of 0 to hide initializers completely.
-# The appearance of the initializer of individual variables and defines in the
-# documentation can be controlled using \showinitializer or \hideinitializer
-# command in the documentation regardless of this setting.
-
-MAX_INITIALIZER_LINES = 30
-
-# Set the SHOW_USED_FILES tag to NO to disable the list of files generated
-# at the bottom of the documentation of classes and structs. If set to YES the
-# list will mention the files that were used to generate the documentation.
-
-SHOW_USED_FILES = YES
-
-# If the sources in your project are distributed over multiple directories
-# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy
-# in the documentation. The default is NO.
-
-SHOW_DIRECTORIES = YES
-
-# Set the SHOW_FILES tag to NO to disable the generation of the Files page.
-# This will remove the Files entry from the Quick Index and from the
-# Folder Tree View (if specified). The default is YES.
-
-SHOW_FILES = YES
-
-# Set the SHOW_NAMESPACES tag to NO to disable the generation of the
-# Namespaces page.
-# This will remove the Namespaces entry from the Quick Index
-# and from the Folder Tree View (if specified). The default is YES.
-
-SHOW_NAMESPACES = YES
-
-# The FILE_VERSION_FILTER tag can be used to specify a program or script that
-# doxygen should invoke to get the current version for each file (typically from
-# the version control system). Doxygen will invoke the program by executing (via
-# popen()) the command <command> <input-file>, where <command> is the value of
-# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file
-# provided by doxygen. Whatever the program writes to standard output
-# is used as the file version. See the manual for examples.
-
-FILE_VERSION_FILTER =
-
-# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
-# by doxygen. The layout file controls the global structure of the generated
-# output files in an output format independent way. The create the layout file
-# that represents doxygen's defaults, run doxygen with the -l option.
-# You can optionally specify a file name after the option, if omitted
-# DoxygenLayout.xml will be used as the name of the layout file.
-
-LAYOUT_FILE =
-
-#---------------------------------------------------------------------------
-# configuration options related to warning and progress messages
-#---------------------------------------------------------------------------
-
-# The QUIET tag can be used to turn on/off the messages that are generated
-# by doxygen. Possible values are YES and NO. If left blank NO is used.
-
-QUIET = NO
-
-# The WARNINGS tag can be used to turn on/off the warning messages that are
-# generated by doxygen. Possible values are YES and NO. If left blank
-# NO is used.
-
-WARNINGS = YES
-
-# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings
-# for undocumented members. If EXTRACT_ALL is set to YES then this flag will
-# automatically be disabled.
-
-WARN_IF_UNDOCUMENTED = YES
-
-# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for
-# potential errors in the documentation, such as not documenting some
-# parameters in a documented function, or documenting parameters that
-# don't exist or using markup commands wrongly.
-
-WARN_IF_DOC_ERROR = YES
-
-# This WARN_NO_PARAMDOC option can be abled to get warnings for
-# functions that are documented, but have no documentation for their parameters
-# or return value. If set to NO (the default) doxygen will only warn about
-# wrong or incomplete parameter documentation, but not about the absence of
-# documentation.
-
-WARN_NO_PARAMDOC = YES
-
-# The WARN_FORMAT tag determines the format of the warning messages that
-# doxygen can produce. The string should contain the $file, $line, and $text
-# tags, which will be replaced by the file and line number from which the
-# warning originated and the warning text. Optionally the format may contain
-# $version, which will be replaced by the version of the file (if it could
-# be obtained via FILE_VERSION_FILTER)
-
-WARN_FORMAT = "$file:$line: $text"
-
-# The WARN_LOGFILE tag can be used to specify a file to which warning
-# and error messages should be written. If left blank the output is written
-# to stderr.
-
-WARN_LOGFILE =
-
-#---------------------------------------------------------------------------
-# configuration options related to the input files
-#---------------------------------------------------------------------------
-
-# The INPUT tag can be used to specify the files and/or directories that contain
-# documented source files. You may enter file names like "myfile.cpp" or
-# directories like "/usr/src/myproject". Separate the files or directories
-# with spaces.
-
-INPUT = @CMAKE_CURRENT_SOURCE_DIR@/main.dox \
- @CMAKE_CURRENT_SOURCE_DIR@/../../include \
- @CMAKE_CURRENT_SOURCE_DIR@/../../compute \
- @CMAKE_CURRENT_SOURCE_DIR@/../../control
-# @CMAKE_CURRENT_SOURCE_DIR@/../../coreblas \
-# @CMAKE_CURRENT_SOURCE_DIR@/../../cudablas \
-# @CMAKE_CURRENT_SOURCE_DIR@/../../runtime
-# @CMAKE_CURRENT_BINARY_DIR@/../../include \
-# @CMAKE_CURRENT_BINARY_DIR@/../../compute \
-# @CMAKE_CURRENT_BINARY_DIR@/../../coreblas/ \
-
-# This tag can be used to specify the character encoding of the source files
-# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is
-# also the default input encoding. Doxygen uses libiconv (or the iconv built
-# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for
-# the list of possible encodings.
-
-INPUT_ENCODING = UTF-8
-
-# If the value of the INPUT tag contains directories, you can use the
-# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp
-# and *.h) to filter out the source-files in the directories. If left
-# blank the following patterns are tested:
-# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx
-# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90
-
-FILE_PATTERNS = *.c
-
-# The RECURSIVE tag can be used to turn specify whether or not subdirectories
-# should be searched for input files as well. Possible values are YES and NO.
-# If left blank NO is used.
-
-RECURSIVE = YES
-
-# The EXCLUDE tag can be used to specify files and/or directories that should
-# excluded from the INPUT source files. This way you can easily exclude a
-# subdirectory from a directory tree whose root is specified with the INPUT tag.
-
-EXCLUDE =
-
-# The EXCLUDE_SYMLINKS tag can be used select whether or not files or
-# directories that are symbolic links (a Unix filesystem feature) are excluded
-# from the input.
-
-EXCLUDE_SYMLINKS = NO
-
-# If the value of the INPUT tag contains directories, you can use the
-# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
-# certain files from those directories. Note that the wildcards are matched
-# against the file with absolute path, so to exclude all test directories
-# for example use the pattern */test/*
-
-EXCLUDE_PATTERNS =
-
-# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
-# (namespaces, classes, functions, etc.) that should be excluded from the
-# output. The symbol name can be a fully qualified name, a word, or if the
-# wildcard * is used, a substring. Examples: ANamespace, AClass,
-# AClass::ANamespace, ANamespace::*Test
-
-EXCLUDE_SYMBOLS =
-
-# The EXAMPLE_PATH tag can be used to specify one or more files or
-# directories that contain example code fragments that are included (see
-# the \include command).
-
-EXAMPLE_PATH =
-
-# If the value of the EXAMPLE_PATH tag contains directories, you can use the
-# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp
-# and *.h) to filter out the source-files in the directories. If left
-# blank all files are included.
-
-EXAMPLE_PATTERNS =
-
-# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
-# searched for input files to be used with the \include or \dontinclude
-# commands irrespective of the value of the RECURSIVE tag.
-# Possible values are YES and NO. If left blank NO is used.
-
-EXAMPLE_RECURSIVE = NO
-
-# The IMAGE_PATH tag can be used to specify one or more files or
-# directories that contain image that are included in the documentation (see
-# the \image command).
-
-IMAGE_PATH =
-
-# The INPUT_FILTER tag can be used to specify a program that doxygen should
-# invoke to filter for each input file. Doxygen will invoke the filter program
-# by executing (via popen()) the command <filter> <input-file>, where <filter>
-# is the value of the INPUT_FILTER tag, and <input-file> is the name of an
-# input file. Doxygen will then use the output that the filter program writes
-# to standard output.
-# If FILTER_PATTERNS is specified, this tag will be
-# ignored.
-
-INPUT_FILTER =
-
-# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
-# basis.
-# Doxygen will compare the file name with each pattern and apply the
-# filter if there is a match.
-# The filters are a list of the form:
-# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further
-# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER
-# is applied to all files.
-
-FILTER_PATTERNS =
-
-# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
-# INPUT_FILTER) will be used to filter the input files when producing source
-# files to browse (i.e. when SOURCE_BROWSER is set to YES).
-
-FILTER_SOURCE_FILES = NO
-
-#---------------------------------------------------------------------------
-# configuration options related to source browsing
-#---------------------------------------------------------------------------
-
-# If the SOURCE_BROWSER tag is set to YES then a list of source files will
-# be generated. Documented entities will be cross-referenced with these sources.
-# Note: To get rid of all source code in the generated output, make sure also
-# VERBATIM_HEADERS is set to NO.
-
-SOURCE_BROWSER = YES
-
-# Setting the INLINE_SOURCES tag to YES will include the body
-# of functions and classes directly in the documentation.
-
-INLINE_SOURCES = NO
-
-# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct
-# doxygen to hide any special comment blocks from generated source code
-# fragments. Normal C and C++ comments will always remain visible.
-
-STRIP_CODE_COMMENTS = YES
-
-# If the REFERENCED_BY_RELATION tag is set to YES
-# then for each documented function all documented
-# functions referencing it will be listed.
-
-REFERENCED_BY_RELATION = YES
-
-# If the REFERENCES_RELATION tag is set to YES
-# then for each documented function all documented entities
-# called/used by that function will be listed.
-
-REFERENCES_RELATION = YES
-
-# If the REFERENCES_LINK_SOURCE tag is set to YES (the default)
-# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from
-# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will
-# link to the source code.
-# Otherwise they will link to the documentation.
-
-REFERENCES_LINK_SOURCE = YES
-
-# If the USE_HTAGS tag is set to YES then the references to source code
-# will point to the HTML generated by the htags(1) tool instead of doxygen
-# built-in source browser. The htags tool is part of GNU's global source
-# tagging system (see http://www.gnu.org/software/global/global.html). You
-# will need version 1.0.0 or higher.
-
-USE_HTAGS = NO
-
-# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen
-# will generate a verbatim copy of the header file for each class for
-# which an include is specified. Set to NO to disable this.
-
-VERBATIM_HEADERS = YES
-
-#---------------------------------------------------------------------------
-# configuration options related to the alphabetical class index
-#---------------------------------------------------------------------------
-
-# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index
-# of all compounds will be generated. Enable this if the project
-# contains a lot of classes, structs, unions or interfaces.
-
-ALPHABETICAL_INDEX = YES
-
-# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then
-# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns
-# in which this list will be split (can be a number in the range [1..20])
-
-COLS_IN_ALPHA_INDEX = 5
-
-# In case all classes in a project start with a common prefix, all
-# classes will be put under the same header in the alphabetical index.
-# The IGNORE_PREFIX tag can be used to specify one or more prefixes that
-# should be ignored while generating the index headers.
-
-IGNORE_PREFIX =
-
-#---------------------------------------------------------------------------
-# configuration options related to the HTML output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_HTML tag is set to YES (the default) Doxygen will
-# generate HTML output.
-
-GENERATE_HTML = YES
-
-# The HTML_OUTPUT tag is used to specify where the HTML docs will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `html' will be used as the default path.
-
-HTML_OUTPUT = html
-
-# The HTML_FILE_EXTENSION tag can be used to specify the file extension for
-# each generated HTML page (for example: .htm,.php,.asp). If it is left blank
-# doxygen will generate files with .html extension.
-
-HTML_FILE_EXTENSION = .html
-
-# The HTML_HEADER tag can be used to specify a personal HTML header for
-# each generated HTML page. If it is left blank doxygen will generate a
-# standard header.
-
-HTML_HEADER =
-
-# The HTML_FOOTER tag can be used to specify a personal HTML footer for
-# each generated HTML page. If it is left blank doxygen will generate a
-# standard footer.
-
-HTML_FOOTER =
-
-# The HTML_STYLESHEET tag can be used to specify a user-defined cascading
-# style sheet that is used by each HTML page. It can be used to
-# fine-tune the look of the HTML output. If the tag is left blank doxygen
-# will generate a default style sheet. Note that doxygen will try to copy
-# the style sheet file to the HTML output directory, so don't put your own
-# stylesheet in the HTML output directory as well, or it will be erased!
-
-HTML_STYLESHEET =
-
-# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output.
-# Doxygen will adjust the colors in the stylesheet and background images
-# according to this color. Hue is specified as an angle on a colorwheel,
-# see http://en.wikipedia.org/wiki/Hue for more information.
-# For instance the value 0 represents red, 60 is yellow, 120 is green,
-# 180 is cyan, 240 is blue, 300 purple, and 360 is red again.
-# The allowed range is 0 to 359.
-
-HTML_COLORSTYLE_HUE = 220
-
-# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of
-# the colors in the HTML output. For a value of 0 the output will use
-# grayscales only. A value of 255 will produce the most vivid colors.
-
-HTML_COLORSTYLE_SAT = 100
-
-# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to
-# the luminance component of the colors in the HTML output. Values below
-# 100 gradually make the output lighter, whereas values above 100 make
-# the output darker. The value divided by 100 is the actual gamma applied,
-# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2,
-# and 100 does not change the gamma.
-
-HTML_COLORSTYLE_GAMMA = 80
-
-# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
-# page will contain the date and time when the page was generated. Setting
-# this to NO can help when comparing the output of multiple runs.
-
-HTML_TIMESTAMP = NO
-
-# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes,
-# files or namespaces will be aligned in HTML using tables. If set to
-# NO a bullet list will be used.
-
-HTML_ALIGN_MEMBERS = YES
-
-# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
-# documentation will contain sections that can be hidden and shown after the
-# page has loaded. For this to work a browser that supports
-# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox
-# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari).
-
-HTML_DYNAMIC_SECTIONS = NO
-
-# If the GENERATE_DOCSET tag is set to YES, additional index files
-# will be generated that can be used as input for Apple's Xcode 3
-# integrated development environment, introduced with OSX 10.5 (Leopard).
-# To create a documentation set, doxygen will generate a Makefile in the
-# HTML output directory. Running make will produce the docset in that
-# directory and running "make install" will install the docset in
-# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find
-# it at startup.
-# See http:/developer.apple.com/tools/creatingdocsetswithdoxygen.html
-# for more information.
-
-GENERATE_DOCSET = NO
-
-# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the
-# feed. A documentation feed provides an umbrella under which multiple
-# documentation sets from a single provider (such as a company or product suite)
-# can be grouped.
-
-DOCSET_FEEDNAME = "Doxygen generated docs"
-
-# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that
-# should uniquely identify the documentation set bundle. This should be a
-# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen
-# will append .docset to the name.
-
-DOCSET_BUNDLE_ID = fr.inria.chameleon
-
-# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify
-# the documentation publisher. This should be a reverse domain-name style
-# string, e.g. com.mycompany.MyDocSet.documentation.
-
-DOCSET_PUBLISHER_ID = fr.inria.chameleon
-
-# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher.
-
-DOCSET_PUBLISHER_NAME = Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux, ICL, KAUST.
-
-# If the GENERATE_HTMLHELP tag is set to YES, additional index files
-# will be generated that can be used as input for tools like the
-# Microsoft HTML help workshop to generate a compiled HTML help file (.chm)
-# of the generated HTML documentation.
-
-GENERATE_HTMLHELP = NO
-
-# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can
-# be used to specify the file name of the resulting .chm file. You
-# can add a path in front of the file if the result should not be
-# written to the html output directory.
-
-CHM_FILE =
-
-# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can
-# be used to specify the location (absolute path including file name) of
-# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run
-# the HTML help compiler on the generated index.hhp.
-
-HHC_LOCATION =
-
-# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag
-# controls if a separate .chi index file is generated (YES) or that
-# it should be included in the master .chm file (NO).
-
-GENERATE_CHI = NO
-
-# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING
-# is used to encode HtmlHelp index (hhk), content (hhc) and project file
-# content.
-
-CHM_INDEX_ENCODING =
-
-# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag
-# controls whether a binary table of contents is generated (YES) or a
-# normal table of contents (NO) in the .chm file.
-
-BINARY_TOC = NO
-
-# The TOC_EXPAND flag can be set to YES to add extra items for group members
-# to the contents of the HTML help documentation and to the tree view.
-
-TOC_EXPAND = NO
-
-# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
-# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated
-# that can be used as input for Qt's qhelpgenerator to generate a
-# Qt Compressed Help (.qch) of the generated HTML documentation.
-
-GENERATE_QHP = NO
-
-# If the QHG_LOCATION tag is specified, the QCH_FILE tag can
-# be used to specify the file name of the resulting .qch file.
-# The path specified is relative to the HTML output folder.
-
-QCH_FILE =
-
-# The QHP_NAMESPACE tag specifies the namespace to use when generating
-# Qt Help Project output. For more information please see
-# http://doc.trolltech.com/qthelpproject.html#namespace
-
-QHP_NAMESPACE = edu.utk.eecs.icl.Plasma
-
-# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating
-# Qt Help Project output. For more information please see
-# http://doc.trolltech.com/qthelpproject.html#virtual-folders
-
-QHP_VIRTUAL_FOLDER = doc
-
-# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to
-# add. For more information please see
-# http://doc.trolltech.com/qthelpproject.html#custom-filters
-
-QHP_CUST_FILTER_NAME =
-
-# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the
-# custom filter to add. For more information please see
-# <a href="http://doc.trolltech.com/qthelpproject.html#custom-filters">
-# Qt Help Project / Custom Filters</a>.
-
-QHP_CUST_FILTER_ATTRS =
-
-# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
-# project's
-# filter section matches.
-# <a href="http://doc.trolltech.com/qthelpproject.html#filter-attributes">
-# Qt Help Project / Filter Attributes</a>.
-
-QHP_SECT_FILTER_ATTRS =
-
-# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can
-# be used to specify the location of Qt's qhelpgenerator.
-# If non-empty doxygen will try to run qhelpgenerator on the generated
-# .qhp file.
-
-QHG_LOCATION =
-
-# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files
-# will be generated, which together with the HTML files, form an Eclipse help
-# plugin. To install this plugin and make it available under the help contents
-# menu in Eclipse, the contents of the directory containing the HTML and XML
-# files needs to be copied into the plugins directory of eclipse. The name of
-# the directory within the plugins directory should be the same as
-# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before
-# the help appears.
-
-GENERATE_ECLIPSEHELP = NO
-
-# A unique identifier for the eclipse help plugin. When installing the plugin
-# the directory name containing the HTML and XML files should also have
-# this name.
-
-ECLIPSE_DOC_ID = edu.utk.eecs.icl.Plasma
-
-# The DISABLE_INDEX tag can be used to turn on/off the condensed index at
-# top of each HTML page. The value NO (the default) enables the index and
-# the value YES disables it.
-
-DISABLE_INDEX = NO
-
-# This tag can be used to set the number of enum values (range [1..20])
-# that doxygen will group on one line in the generated HTML documentation.
-
-ENUM_VALUES_PER_LINE = 4
-
-# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
-# structure should be generated to display hierarchical information.
-# If the tag value is set to YES, a side panel will be generated
-# containing a tree-like index structure (just like the one that
-# is generated for HTML Help). For this to work a browser that supports
-# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser).
-# Windows users are probably better off using the HTML help feature.
-
-GENERATE_TREEVIEW = YES
-
-# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories,
-# and Class Hierarchy pages using a tree view instead of an ordered list.
-
-USE_INLINE_TREES = YES
-
-# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be
-# used to set the initial width (in pixels) of the frame in which the tree
-# is shown.
-
-TREEVIEW_WIDTH = 250
-
-# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open
-# links to external symbols imported via tag files in a separate window.
-
-EXT_LINKS_IN_WINDOW = NO
-
-# Use this tag to change the font size of Latex formulas included
-# as images in the HTML documentation. The default is 10. Note that
-# when you change the font size after a successful doxygen run you need
-# to manually remove any form_*.png images from the HTML output directory
-# to force them to be regenerated.
-
-FORMULA_FONTSIZE = 10
-
-# Use the FORMULA_TRANPARENT tag to determine whether or not the images
-# generated for formulas are transparent PNGs. Transparent PNGs are
-# not supported properly for IE 6.0, but are supported on all modern browsers.
-# Note that when changing this option you need to delete any form_*.png files
-# in the HTML output before the changes have effect.
-
-FORMULA_TRANSPARENT = YES
-
-# When the SEARCHENGINE tag is enabled doxygen will generate a search box
-# for the HTML output. The underlying search engine uses javascript
-# and DHTML and should work on any modern browser. Note that when using
-# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets
-# (GENERATE_DOCSET) there is already a search function so this one should
-# typically be disabled. For large projects the javascript based search engine
-# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution.
-
-SEARCHENGINE = YES
-
-# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
-# implemented using a PHP enabled web server instead of at the web client
-# using Javascript. Doxygen will generate the search PHP script and index
-# file to put on the web server. The advantage of the server
-# based approach is that it scales better to large projects and allows
-# full text search. The disadvances is that it is more difficult to setup
-# and does not have live searching capabilities.
-
-SERVER_BASED_SEARCH = NO
-
-#---------------------------------------------------------------------------
-# configuration options related to the LaTeX output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will
-# generate Latex output.
-
-GENERATE_LATEX = NO
-
-# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `latex' will be used as the default path.
-
-LATEX_OUTPUT = latex
-
-# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
-# invoked. If left blank `latex' will be used as the default command name.
-# Note that when enabling USE_PDFLATEX this option is only used for
-# generating bitmaps for formulas in the HTML output, but not in the
-# Makefile that is written to the output directory.
-
-LATEX_CMD_NAME = latex
-
-# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to
-# generate index for LaTeX. If left blank `makeindex' will be used as the
-# default command name.
-
-MAKEINDEX_CMD_NAME = makeindex
-
-# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact
-# LaTeX documents. This may be useful for small projects and may help to
-# save some trees in general.
-
-COMPACT_LATEX = NO
-
-# The PAPER_TYPE tag can be used to set the paper type that is used
-# by the printer. Possible values are: a4, a4wide, letter, legal and
-# executive. If left blank a4wide will be used.
-
-PAPER_TYPE = a4wide
-
-# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX
-# packages that should be included in the LaTeX output.
-
-EXTRA_PACKAGES =
-
-# The LATEX_HEADER tag can be used to specify a personal LaTeX header for
-# the generated latex document. The header should contain everything until
-# the first chapter. If it is left blank doxygen will generate a
-# standard header. Notice: only use this tag if you know what you are doing!
-
-LATEX_HEADER =
-
-# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated
-# is prepared for conversion to pdf (using ps2pdf). The pdf file will
-# contain links (just like the HTML output) instead of page references
-# This makes the output suitable for online browsing using a pdf viewer.
-
-PDF_HYPERLINKS = YES
-
-# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of
-# plain latex in the generated Makefile. Set this option to YES to get a
-# higher quality PDF documentation.
-
-USE_PDFLATEX = YES
-
-# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode.
-# command to the generated LaTeX files. This will instruct LaTeX to keep
-# running if errors occur, instead of asking the user for help.
-# This option is also used when generating formulas in HTML.
-
-LATEX_BATCHMODE = NO
-
-# If LATEX_HIDE_INDICES is set to YES then doxygen will not
-# include the index chapters (such as File Index, Compound Index, etc.)
-# in the output.
-
-LATEX_HIDE_INDICES = NO
-
-# If LATEX_SOURCE_CODE is set to YES then doxygen will include
-# source code with syntax highlighting in the LaTeX output.
-# Note that which sources are shown also depends on other settings
-# such as SOURCE_BROWSER.
-
-LATEX_SOURCE_CODE = NO
-
-#---------------------------------------------------------------------------
-# configuration options related to the RTF output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output
-# The RTF output is optimized for Word 97 and may not look very pretty with
-# other RTF readers or editors.
-
-GENERATE_RTF = NO
-
-# The RTF_OUTPUT tag is used to specify where the RTF docs will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `rtf' will be used as the default path.
-
-RTF_OUTPUT = rtf
-
-# If the COMPACT_RTF tag is set to YES Doxygen generates more compact
-# RTF documents. This may be useful for small projects and may help to
-# save some trees in general.
-
-COMPACT_RTF = NO
-
-# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated
-# will contain hyperlink fields. The RTF file will
-# contain links (just like the HTML output) instead of page references.
-# This makes the output suitable for online browsing using WORD or other
-# programs which support those fields.
-# Note: wordpad (write) and others do not support links.
-
-RTF_HYPERLINKS = NO
-
-# Load stylesheet definitions from file. Syntax is similar to doxygen's
-# config file, i.e. a series of assignments. You only have to provide
-# replacements, missing definitions are set to their default value.
-
-RTF_STYLESHEET_FILE =
-
-# Set optional variables used in the generation of an rtf document.
-# Syntax is similar to doxygen's config file.
-
-RTF_EXTENSIONS_FILE =
-
-#---------------------------------------------------------------------------
-# configuration options related to the man page output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_MAN tag is set to YES (the default) Doxygen will
-# generate man pages
-
-GENERATE_MAN = NO
-
-# The MAN_OUTPUT tag is used to specify where the man pages will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `man' will be used as the default path.
-
-MAN_OUTPUT = man
-
-# The MAN_EXTENSION tag determines the extension that is added to
-# the generated man pages (default is the subroutine's section .3)
-
-MAN_EXTENSION = .3
-
-# If the MAN_LINKS tag is set to YES and Doxygen generates man output,
-# then it will generate one additional man file for each entity
-# documented in the real man page(s). These additional files
-# only source the real man page, but without them the man command
-# would be unable to find the correct page. The default is NO.
-
-MAN_LINKS = NO
-
-#---------------------------------------------------------------------------
-# configuration options related to the XML output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_XML tag is set to YES Doxygen will
-# generate an XML file that captures the structure of
-# the code including all documentation.
-
-GENERATE_XML = NO
-
-# The XML_OUTPUT tag is used to specify where the XML pages will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `xml' will be used as the default path.
-
-XML_OUTPUT = xml
-
-# The XML_SCHEMA tag can be used to specify an XML schema,
-# which can be used by a validating XML parser to check the
-# syntax of the XML files.
-
-XML_SCHEMA =
-
-# The XML_DTD tag can be used to specify an XML DTD,
-# which can be used by a validating XML parser to check the
-# syntax of the XML files.
-
-XML_DTD =
-
-# If the XML_PROGRAMLISTING tag is set to YES Doxygen will
-# dump the program listings (including syntax highlighting
-# and cross-referencing information) to the XML output. Note that
-# enabling this will significantly increase the size of the XML output.
-
-XML_PROGRAMLISTING = YES
-
-#---------------------------------------------------------------------------
-# configuration options for the AutoGen Definitions output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will
-# generate an AutoGen Definitions (see autogen.sf.net) file
-# that captures the structure of the code including all
-# documentation. Note that this feature is still experimental
-# and incomplete at the moment.
-
-GENERATE_AUTOGEN_DEF = NO
-
-#---------------------------------------------------------------------------
-# configuration options related to the Perl module output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_PERLMOD tag is set to YES Doxygen will
-# generate a Perl module file that captures the structure of
-# the code including all documentation. Note that this
-# feature is still experimental and incomplete at the
-# moment.
-
-GENERATE_PERLMOD = NO
-
-# If the PERLMOD_LATEX tag is set to YES Doxygen will generate
-# the necessary Makefile rules, Perl scripts and LaTeX code to be able
-# to generate PDF and DVI output from the Perl module output.
-
-PERLMOD_LATEX = NO
-
-# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be
-# nicely formatted so it can be parsed by a human reader.
-# This is useful
-# if you want to understand what is going on.
-# On the other hand, if this
-# tag is set to NO the size of the Perl module output will be much smaller
-# and Perl will parse it just the same.
-
-PERLMOD_PRETTY = YES
-
-# The names of the make variables in the generated doxyrules.make file
-# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX.
-# This is useful so different doxyrules.make files included by the same
-# Makefile don't overwrite each other's variables.
-
-PERLMOD_MAKEVAR_PREFIX =
-
-#---------------------------------------------------------------------------
-# Configuration options related to the preprocessor
-#---------------------------------------------------------------------------
-
-# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will
-# evaluate all C-preprocessor directives found in the sources and include
-# files.
-
-ENABLE_PREPROCESSING = YES
-
-# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro
-# names in the source code. If set to NO (the default) only conditional
-# compilation will be performed. Macro expansion can be done in a controlled
-# way by setting EXPAND_ONLY_PREDEF to YES.
-
-MACRO_EXPANSION = NO
-
-# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES
-# then the macro expansion is limited to the macros specified with the
-# PREDEFINED and EXPAND_AS_DEFINED tags.
-
-EXPAND_ONLY_PREDEF = NO
-
-# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files
-# in the INCLUDE_PATH (see below) will be search if a #include is found.
-
-SEARCH_INCLUDES = YES
-
-# The INCLUDE_PATH tag can be used to specify one or more directories that
-# contain include files that are not input files but should be processed by
-# the preprocessor.
-
-INCLUDE_PATH =
-
-# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
-# patterns (like *.h and *.hpp) to filter out the header-files in the
-# directories. If left blank, the patterns specified with FILE_PATTERNS will
-# be used.
-
-INCLUDE_FILE_PATTERNS =
-
-# The PREDEFINED tag can be used to specify one or more macro names that
-# are defined before the preprocessor is started (similar to the -D option of
-# gcc). The argument of the tag is a list of macros of the form: name
-# or name=definition (no spaces). If the definition and the = are
-# omitted =1 is assumed. To prevent a macro definition from being
-# undefined via #undef or recursively expanded use the := operator
-# instead of the = operator.
-
-PREDEFINED =
-
-# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then
-# this tag can be used to specify a list of macro names that should be expanded.
-# The macro definition that is found in the sources will be used.
-# Use the PREDEFINED tag if you want to use a different macro definition.
-
-EXPAND_AS_DEFINED =
-
-# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then
-# doxygen's preprocessor will remove all function-like macros that are alone
-# on a line, have an all uppercase name, and do not end with a semicolon. Such
-# function macros are typically used for boiler-plate code, and will confuse
-# the parser if not removed.
-
-SKIP_FUNCTION_MACROS = YES
-
-#---------------------------------------------------------------------------
-# Configuration::additions related to external references
-#---------------------------------------------------------------------------
-
-# The TAGFILES option can be used to specify one or more tagfiles.
-# Optionally an initial location of the external documentation
-# can be added for each tagfile. The format of a tag file without
-# this location is as follows:
-#
-# TAGFILES = file1 file2 ...
-# Adding location for the tag files is done as follows:
-#
-# TAGFILES = file1=loc1 "file2 = loc2" ...
-# where "loc1" and "loc2" can be relative or absolute paths or
-# URLs. If a location is present for each tag, the installdox tool
-# does not have to be run to correct the links.
-# Note that each tag file must have a unique name
-# (where the name does NOT include the path)
-# If a tag file is not located in the directory in which doxygen
-# is run, you must also specify the path to the tagfile here.
-
-TAGFILES =
-
-# When a file name is specified after GENERATE_TAGFILE, doxygen will create
-# a tag file that is based on the input files it reads.
-
-GENERATE_TAGFILE =
-
-# If the ALLEXTERNALS tag is set to YES all external classes will be listed
-# in the class index. If set to NO only the inherited external classes
-# will be listed.
-
-ALLEXTERNALS = NO
-
-# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed
-# in the modules index. If set to NO, only the current project's groups will
-# be listed.
-
-EXTERNAL_GROUPS = YES
-
-# The PERL_PATH should be the absolute path and name of the perl script
-# interpreter (i.e. the result of `which perl').
-
-PERL_PATH = /usr/bin/perl
-
-#---------------------------------------------------------------------------
-# Configuration options related to the dot tool
-#---------------------------------------------------------------------------
-
-# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will
-# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base
-# or super classes. Setting the tag to NO turns the diagrams off. Note that
-# this option is superseded by the HAVE_DOT option below. This is only a
-# fallback. It is recommended to install and use dot, since it yields more
-# powerful graphs.
-
-CLASS_DIAGRAMS = YES
-
-# You can define message sequence charts within doxygen comments using the \msc
-# command. Doxygen will then run the mscgen tool (see
-# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the
-# documentation. The MSCGEN_PATH tag allows you to specify the directory where
-# the mscgen tool resides. If left empty the tool is assumed to be found in the
-# default search path.
-
-MSCGEN_PATH =
-
-# If set to YES, the inheritance and collaboration graphs will hide
-# inheritance and usage relations if the target is undocumented
-# or is not a class.
-
-HIDE_UNDOC_RELATIONS = YES
-
-# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
-# available from the path. This tool is part of Graphviz, a graph visualization
-# toolkit from AT&T and Lucent Bell Labs. The other options in this section
-# have no effect if this option is set to NO (the default)
-
-HAVE_DOT = NO
-
-# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is
-# allowed to run in parallel. When set to 0 (the default) doxygen will
-# base this on the number of processors available in the system. You can set it
-# explicitly to a value larger than 0 to get control over the balance
-# between CPU load and processing speed.
-
-DOT_NUM_THREADS = 0
-
-# By default doxygen will write a font called FreeSans.ttf to the output
-# directory and reference it in all dot files that doxygen generates. This
-# font does not include all possible unicode characters however, so when you need
-# these (or just want a differently looking font) you can specify the font name
-# using DOT_FONTNAME. You need need to make sure dot is able to find the font,
-# which can be done by putting it in a standard location or by setting the
-# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory
-# containing the font.
-
-DOT_FONTNAME = FreeSans.ttf
-
-# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs.
-# The default size is 10pt.
-
-DOT_FONTSIZE = 10
-
-# By default doxygen will tell dot to use the output directory to look for the
-# FreeSans.ttf font (which doxygen will put there itself). If you specify a
-# different font using DOT_FONTNAME you can set the path where dot
-# can find it using this tag.
-
-DOT_FONTPATH =
-
-# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen
-# will generate a graph for each documented class showing the direct and
-# indirect inheritance relations. Setting this tag to YES will force the
-# the CLASS_DIAGRAMS tag to NO.
-
-CLASS_GRAPH = YES
-
-# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen
-# will generate a graph for each documented class showing the direct and
-# indirect implementation dependencies (inheritance, containment, and
-# class references variables) of the class with other documented classes.
-
-COLLABORATION_GRAPH = YES
-
-# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen
-# will generate a graph for groups, showing the direct groups dependencies
-
-GROUP_GRAPHS = YES
-
-# If the UML_LOOK tag is set to YES doxygen will generate inheritance and
-# collaboration diagrams in a style similar to the OMG's Unified Modeling
-# Language.
-
-UML_LOOK = NO
-
-# If set to YES, the inheritance and collaboration graphs will show the
-# relations between templates and their instances.
-
-TEMPLATE_RELATIONS = NO
-
-# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT
-# tags are set to YES then doxygen will generate a graph for each documented
-# file showing the direct and indirect include dependencies of the file with
-# other documented files.
-
-INCLUDE_GRAPH = YES
-
-# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and
-# HAVE_DOT tags are set to YES then doxygen will generate a graph for each
-# documented header file showing the documented files that directly or
-# indirectly include this file.
-
-INCLUDED_BY_GRAPH = YES
-
-# If the CALL_GRAPH and HAVE_DOT options are set to YES then
-# doxygen will generate a call dependency graph for every global function
-# or class method. Note that enabling this option will significantly increase
-# the time of a run. So in most cases it will be better to enable call graphs
-# for selected functions only using the \callgraph command.
-
-CALL_GRAPH = NO
-
-# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then
-# doxygen will generate a caller dependency graph for every global function
-# or class method. Note that enabling this option will significantly increase
-# the time of a run. So in most cases it will be better to enable caller
-# graphs for selected functions only using the \callergraph command.
-
-CALLER_GRAPH = NO
-
-# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen
-# will graphical hierarchy of all classes instead of a textual one.
-
-GRAPHICAL_HIERARCHY = YES
-
-# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES
-# then doxygen will show the dependencies a directory has on other directories
-# in a graphical way. The dependency relations are determined by the #include
-# relations between the files in the directories.
-
-DIRECTORY_GRAPH = YES
-
-# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
-# generated by dot. Possible values are png, jpg, or gif
-# If left blank png will be used.
-
-DOT_IMAGE_FORMAT = png
-
-# The tag DOT_PATH can be used to specify the path where the dot tool can be
-# found. If left blank, it is assumed the dot tool can be found in the path.
-
-DOT_PATH =
-
-# The DOTFILE_DIRS tag can be used to specify one or more directories that
-# contain dot files that are included in the documentation (see the
-# \dotfile command).
-
-DOTFILE_DIRS =
-
-# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of
-# nodes that will be shown in the graph. If the number of nodes in a graph
-# becomes larger than this value, doxygen will truncate the graph, which is
-# visualized by representing a node as a red box. Note that doxygen if the
-# number of direct children of the root node in a graph is already larger than
-# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note
-# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
-
-DOT_GRAPH_MAX_NODES = 50
-
-# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the
-# graphs generated by dot. A depth value of 3 means that only nodes reachable
-# from the root by following a path via at most 3 edges will be shown. Nodes
-# that lay further from the root node will be omitted. Note that setting this
-# option to 1 or 2 may greatly reduce the computation time needed for large
-# code bases. Also note that the size of a graph can be further restricted by
-# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
-
-MAX_DOT_GRAPH_DEPTH = 0
-
-# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
-# background. This is disabled by default, because dot on Windows does not
-# seem to support this out of the box. Warning: Depending on the platform used,
-# enabling this option may lead to badly anti-aliased labels on the edges of
-# a graph (i.e. they become hard to read).
-
-DOT_TRANSPARENT = NO
-
-# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output
-# files in one run (i.e. multiple -o and -T options on the command line). This
-# makes dot run faster, but since only newer versions of dot (>1.8.10)
-# support this, this feature is disabled by default.
-
-DOT_MULTI_TARGETS = YES
-
-# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will
-# generate a legend page explaining the meaning of the various boxes and
-# arrows in the dot generated graphs.
-
-GENERATE_LEGEND = YES
-
-# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will
-# remove the intermediate dot files that are used to generate
-# the various graphs.
-
-DOT_CLEANUP = YES
diff --git a/doc/orgmode/CMakeLists.txt b/doc/orgmode/CMakeLists.txt
deleted file mode 100644
index 32da258dd0d203f1ffd79f6f75ebf0ceeac8ddc5..0000000000000000000000000000000000000000
--- a/doc/orgmode/CMakeLists.txt
+++ /dev/null
@@ -1,115 +0,0 @@
-###
-#
-# @file CMakeLists.txt
-#
-# @copyright (c) 2017 Inria. All rights reserved.
-#
-###
-#
-# @project CHAMELEON
-# CHAMELEON is a software package provided by:
-# Inria Bordeaux - Sud-Ouest,
-# Univ. of Tennessee,
-# King Abdullah Univesity of Science and Technology
-# Univ. of California Berkeley,
-# Univ. of Colorado Denver.
-#
-# @version 1.1.0
-# @author Florent Pruvost
-# @author Mathieu Faverge
-# @date 2021-01-04
-#
-###
-
-cmake_minimum_required(VERSION 3.1)
-
-# Create file version.org
-# -----------------------
-configure_file("version.org.in"
- "version.org"
- @ONLY)
-configure_file("users_guide.org.in"
- "users_guide.org"
- @ONLY)
-
-set(FIGURES
- tile_lu.pdf
- tile_lu.jpg
- tile_layout.pdf
- tile_layout.jpg
- trace_qr.pdf
- trace_qr.jpg
- potri_async.png
- chameleon_header.png
- )
-set(FIGURES_HTML
- tile_lu.jpg
- tile_layout.jpg
- trace_qr.jpg
- potri_async.png
- chameleon_header.png
- )
-
-foreach(_fig ${FIGURES})
- configure_file(${CMAKE_CURRENT_SOURCE_DIR}/chapters/${_fig}
- ${CMAKE_CURRENT_BINARY_DIR}/${_fig}
- COPYONLY)
-endforeach()
-
-# Looking for emacs
-# -----------------
-FIND_PROGRAM(EMACS_COMPILER emacs)
-FIND_PROGRAM(TEX_COMPILER tex)
-
-if(EMACS_COMPILER)
-
- # org-html-themes directory is required for the html output
- add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/org-html-themes
- COMMAND ${CMAKE_COMMAND} -E copy_directory
- ${CMAKE_CURRENT_SOURCE_DIR}/org-html-themes
- ${CMAKE_CURRENT_BINARY_DIR}/org-html-themes
- DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/org-html-themes)
-
- # Add target
- # ----------
- add_custom_command(OUTPUT users_guide.html
- COMMAND ${EMACS_COMPILER}
- ARGS ${CMAKE_CURRENT_BINARY_DIR}/users_guide.org
- --batch
- -f
- org-html-export-to-html
- --kill
- DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/users_guide.org
- ${CMAKE_CURRENT_BINARY_DIR}/version.org
- ${CMAKE_CURRENT_BINARY_DIR}/org-html-themes)
- add_custom_command(OUTPUT users_guide.pdf
- COMMAND ${EMACS_COMPILER}
- ARGS ${CMAKE_CURRENT_BINARY_DIR}/users_guide.org
- --batch
- -f
- org-latex-export-to-pdf
- --kill
- DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/users_guide.org
- ${CMAKE_CURRENT_BINARY_DIR}/version.org)
-
- add_custom_target(doc-html-users_guide ALL DEPENDS users_guide.html)
- add_custom_target(doc-pdf-users_guide ALL DEPENDS users_guide.pdf)
-
- # Installation
- # ------------
- install(FILES ${CMAKE_CURRENT_BINARY_DIR}/users_guide.html
- DESTINATION share/chameleon/html)
- install(FILES ${CMAKE_CURRENT_BINARY_DIR}/users_guide.pdf
- DESTINATION share/chameleon/pdf)
- foreach(_fig ${FIGURES_HTML})
- install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${_fig}
- DESTINATION share/chameleon/html)
- endforeach()
-
-else(EMACS_COMPILER)
- message(STATUS "Looking for emacs - not found")
-endif(EMACS_COMPILER)
-
-###
-### END CMakeLists.txt
-###
diff --git a/doc/orgmode/chapters/installing.org b/doc/orgmode/chapters/installing.org
deleted file mode 100644
index bbeb9fe7a891ce1b52670f1940557225a9c0154d..0000000000000000000000000000000000000000
--- a/doc/orgmode/chapters/installing.org
+++ /dev/null
@@ -1,813 +0,0 @@
-# This file is part of the Chameleon User's Guide.
-# Copyright (C) 2017 Inria
-# See the file ../users_guide.org for copying conditions.
-
-Chameleon is written in C and depends on a couple of external
-libraries that must be installed on the system.
-# , it provides an interface to be called from Fortran
-
-Chameleon can be built and installed on UNIX systems (Linux) by the standard
-means of [[http://www.cmake.org/][CMake]]. General information about CMake, as well as
-installation binaries and CMake source code are available from [[http://www.cmake.org/cmake/resources/software.html][here]].
-
-To get support to install a full distribution Chameleon + dependencies
-we encourage users to use [[sec:spack][Spack]].
-
-
-** Getting Chameleon
-
- The latest official release tarballs of Chameleon sources are
- available for download from the [[https://gitlab.inria.fr/solverstack/chameleon/tags][gitlab tags page]].
-
- The latest development state is available on [[https://gitlab.inria.fr/solverstack/chameleon][gitlab]]. You need [[https://git-scm.com/downloads][Git]]
- #+begin_src
- git clone --recursive https://gitlab.inria.fr/solverstack/chameleon.git
- #+end_src
-
-** Prerequisites for installing Chameleon
-
- To install Chameleon's libraries, header files, and executables, one
- needs:
- - CMake (version 2.8 minimum): the build system
- - C and Fortran compilers: GNU compiler suite, Clang, Intel or IBM
- can be used
- - python: to generate files in the different precisions
- - external libraries: this depends on the configuration, by default
- the required libraries are
- - runtimes: [[http://runtime.bordeaux.inria.fr/StarPU/][StarPU]] or [[http://icl.utk.edu/parsec/][PaRSEC]] or [[http://icl.cs.utk.edu/quark/][QUARK]] or [[https://www.openmp.org/][OpenMP]]
- - kernels : CBLAS, LAPACKE (with TMG). These are C interfaces to
- Fortran kernels BLAS and LAPACK. There exist several providers
- that can be used with Chameleon (Intel MKL, Netlib, OpenBLAS,
- BLIS/FLAME)
-
- Optional libraries:
- - cuda: [[https://developer.nvidia.com/cuda-downloads][cuda]], [[http://docs.nvidia.com/cuda/cublas/][cublas]] (comes with cuda)
- - mpi: [[http://www.open-mpi.org/][openmpi]], [[https://www.mpich.org/][mpich]], [[https://software.intel.com/en-us/mpi-library][intelmpi]]
-
- These packages must be installed on the system before trying to
- configure/build chameleon. Please look at the distrib/ directory
- which gives some hints for the installation of dependencies for
- Unix systems.
-
- We give here some examples for a Debian system:
- #+begin_src
-
- # Update Debian packages list
- sudo apt-get update
- # Install BLAS/LAPACK, can be OpenBLAS, Intel MKL, Netlib LAPACK
- sudo apt-get install -y libopenblas-dev liblapacke-dev
- # or sudo apt-get install -y libmkl-dev
- # or sudo apt-get install -y liblapack-dev liblapacke-dev
- # Install OpenMPI
- sudo apt-get install -y libopenmpi-dev
- # Install StarPU
- sudo apt-get install libstarpu-dev
-
- # Optionnaly to make some specific developments, the following may be installed
- # Install hwloc (used by StarPU or QUARK, already a dependency of OpenMPI)
- sudo apt-get install -y libhwloc-dev
- # install EZTrace, usefull to export some nice execution traces
- with all runtimes
- sudo apt-get install -y libeztrace-dev
- # install FxT, usefull to export some nice execution traces with StarPU
- sudo apt-get install -y libfxt-dev
- # Install cuda and cuBLAS: only if you have a GPU cuda compatible
- sudo apt-get install -y nvidia-cuda-toolkit nvidia-cuda-dev
-
- # If you prefer a specific version of StarPU, install it yourself, e.g.
- # Install StarPU (with MPI and FxT enabled)
- mkdir -p $HOME/install
- cd $HOME/install
- wget https://files.inria.fr/starpu/starpu-1.3.7/starpu-1.3.7.tar.gz
- tar xvzf starpu-1.3.7.tar.gz
- cd starpu-1.3.7/
- ./configure --prefix=/usr/local --with-fxt=/usr/lib/x86_64-linux-gnu/
- make -j5
- sudo make install
-
- # Install PaRSEC: to be used in place of StarPU
- mkdir -p $HOME/install
- cd $HOME/install
- git clone https://bitbucket.org/mfaverge/parsec.git
- cd parsec
- git checkout mymaster
- git submodule update
- mkdir -p build
- cd build
- cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_SHARED_LIBS=ON
- make -j5
- sudo make install
-
- # Install QUARK: to be used in place of StarPU
- mkdir -p $HOME/install
- cd $HOME/install
- git clone https://github.com/ecrc/quark
- cd quark/
- sed -i -e "s#prefix=.*#prefix=/usr/local#g" make.inc
- sed -i -e "s#CFLAGS=.*#CFLAGS= -O2 -DADD_ -fPIC#g" make.inc
- make
- sudo make install
- #+end_src
-
- See also our script example in the [[https://gitlab.inria.fr/solverstack/chameleon/-/blob/master/distrib/debian/install_dependencies.sh][distrib/debian]] sub-directory.
-
-*** Known issues
- - we need the lapacke interface to tmg routines and symbol like
- ~LAPACKE_dlatms_work~ should be defined in the lapacke
- library. Make sure the Debian packages /libopenblas-dev/ and
- /liblapacke-dev/ (no problem with Intel MKL) do provide the tmg
- interface. If not you can possibly update your distribution or
- install the lapacke interface library in another way, by
- yourself from source or with [[https://gitlab.inria.fr/solverstack/spack-repo][Spack]], or with [[https://gitlab.inria.fr/guix-hpc/guix-hpc-non-free][Guix-HPC]],...
-*** Some details about dependencies
-**** BLAS implementation
- [[http://www.netlib.org/blas/][BLAS]] (Basic Linear Algebra Subprograms), are a de facto standard
- for basic linear algebra operations such as vector and matrix
- multiplication. FORTRAN implementation of BLAS is available from
- Netlib. Also, C implementation of BLAS is included in GSL (GNU
- Scientific Library). Both these implementations are reference
- implementation of BLAS, are not optimized for modern processor
- architectures and provide an order of magnitude lower performance
- than optimized implementations. Highly optimized implementations
- of BLAS are available from many hardware vendors, such as Intel
- MKL, IBM ESSL and AMD ACML. Fast implementations are also
- available as academic packages, such as ATLAS and OpenBLAS. The
- standard interface to BLAS is the FORTRAN interface.
-
- *Caution about the compatibility:* Chameleon has been mainly tested
- with the reference BLAS from NETLIB, OpenBLAS and Intel MKL.
-**** CBLAS
- [[http://www.netlib.org/blas/#_cblas][CBLAS]] is a C language interface to BLAS. Most commercial and
- academic implementations of BLAS also provide CBLAS. Netlib
- provides a reference implementation of CBLAS on top of FORTRAN
- BLAS (Netlib CBLAS). Since GSL is implemented in C, it naturally
- provides CBLAS.
-
- *Caution about the compatibility:* Chameleon has been mainly tested with
- the reference CBLAS from NETLIB, OpenBLAS and Intel MKL.
-**** LAPACK implementation
- [[http://www.netlib.org/lapack/][LAPACK]] (Linear Algebra PACKage) is a software library for
- numerical linear algebra, a successor of LINPACK and EISPACK and
- a predecessor of Chameleon. LAPACK provides routines for solving
- linear systems of equations, linear least square problems,
- eigenvalue problems and singular value problems. Most commercial
- and academic BLAS packages also provide some LAPACK routines.
-
- *Caution about the compatibility:* Chameleon has been mainly tested
- with the reference LAPACK from NETLIB, OpenBLAS and Intel MKL.
-**** LAPACKE
- [[http://www.netlib.org/lapack/][LAPACKE]] is a C language interface to LAPACK (or CLAPACK). It is
- produced by Intel in coordination with the LAPACK team and is
- available in source code from Netlib in its original version
- (Netlib LAPACKE) and from Chameleon website in an extended
- version (LAPACKE for Chameleon). In addition to implementing the
- C interface, LAPACKE also provides routines which automatically
- handle workspace allocation, making the use of LAPACK much more
- convenient.
-
- *Caution about the compatibility:* Chameleon has been mainly tested
- with the reference LAPACKE from NETLIB, OpenBLAS and Intel
- MKL. In addition the LAPACKE library *must* be configured to
- provide the interface with the TMG routines and symbols like
- ~LAPACKE_dlatms_work~ should be defined.
-**** libtmg
- [[http://www.netlib.org/lapack/][libtmg]] is a component of the LAPACK library, containing routines
- for generation of input matrices for testing and timing of
- LAPACK. The testing and timing suites of LAPACK require libtmg,
- but not the library itself. Note that the LAPACK library can be
- built and used without libtmg.
-
- *Caution about the compatibility:* Chameleon has been mainly tested
- with the reference TMGLIB from NETLIB, OpenBLAS and Intel MKL.
-**** StarPU
- [[http://runtime.bordeaux.inria.fr/StarPU/][StarPU]] is a task programming library for hybrid architectures.
- StarPU handles run-time concerns such as:
- * Task dependencies
- * Optimized heterogeneous scheduling
- * Optimized data transfers and replication between main memory
- and discrete memories
- * Optimized cluster communications
-
- StarPU can be used to benefit from GPUs and distributed-memory
- environment. Note StarPU is enabled by default.
-
- *Caution about the compatibility:* Chameleon has been mainly tested
- with StarPU-1.1, 1.2 and 1.3 releases.
-**** PaRSEC
- [[http://icl.utk.edu/parsec/][PaRSEC]] is a generic framework for architecture aware scheduling
- and management of micro-tasks on distributed many-core
- heterogeneous architectures. It can be used with MPI and Cuda.
-
- *Caution about the compatibility:* Chameleon is compatible with
- this version
- https://bitbucket.org/mfaverge/parsec/branch/mymaster.
-**** QUARK
- [[http://icl.cs.utk.edu/quark/][QUARK]] (QUeuing And Runtime for Kernels) provides a library that
- enables the dynamic execution of tasks with data dependencies in
- a multi-core, multi-socket, shared-memory environment. When
- Chameleon is linked with QUARK, it is not possible to exploit
- neither CUDA (for GPUs) nor MPI (distributed-memory environment).
- You can use PaRSEC or StarPU to do so.
-
- *Caution about the compatibility:* Chameleon has been mainly tested
- with the QUARK library coming from https://github.com/ecrc/quark.
-**** EZTrace
- This library provides efficient modules for recording
- traces. Chameleon can trace kernels execution on CPU workers
- thanks to EZTrace and produce .paje files. EZTrace also provides
- integrated modules to trace MPI calls and/or memory usage. See
- how to use this feature here [[sec:trace_ezt][Execution trace using EZTrace]]. To
- trace kernels execution on all kind of workers, such as CUDA, We
- recommend to use the internal tracing support of the runtime
- system used done by the underlying runtime. See how to use this
- feature here [[sec:trace_fxt][Execution trace using StarPU/FxT]].
-**** hwloc
- [[http://www.open-mpi.org/projects/hwloc/][hwloc]] (Portable Hardware Locality) is a software package for
- accessing the topology of a multicore system including components
- like: cores, sockets, caches and NUMA nodes. The topology
- discovery library, ~hwloc~, is strongly recommended to be used
- through the runtime system. It allows to increase performance,
- and to perform some topology aware scheduling. ~hwloc~ is available
- in major distributions and for most OSes and can be downloaded
- from http://www.open-mpi.org/software/hwloc.
-
- *Caution about the compatibility:* hwloc should be compatible with
- the runtime system used.
-**** OpenMPI
- [[http://www.open-mpi.org/][OpenMPI]] is an open source Message Passing Interface
- implementation for execution on multiple nodes with
- distributed-memory environment. MPI can be enabled only if the
- runtime system chosen is StarPU (default). To use MPI through
- StarPU, it is necessary to compile StarPU with MPI enabled.
-
- *Caution about the compatibility:* OpenMPI should be built with the
- --enable-mpi-thread-multiple option.
-**** Nvidia CUDA Toolkit
- [[https://developer.nvidia.com/cuda-toolkit][Nvidia CUDA Toolkit]] provides a comprehensive development
- environment for C and C++ developers building GPU-accelerated
- applications. Chameleon can use a set of low level optimized
- kernels coming from cuBLAS to accelerate computations on GPUs.
- The [[http://docs.nvidia.com/cuda/cublas/][cuBLAS]] library is an implementation of BLAS (Basic Linear
- Algebra Subprograms) on top of the Nvidia CUDA runtime. cuBLAS
- is normaly distributed with Nvidia CUDA Toolkit. CUDA/cuBLAS can
- be enabled in Chameleon only if the runtime system chosen is
- StarPU (default). To use CUDA through StarPU, it is necessary to
- compile StarPU with CUDA enabled.
-
- *Caution about the compatibility:* your compiler must be compatible
- with CUDA.
-** Distribution Debian
- Download one of the available package for your distribution here
- https://gitlab.inria.fr/solverstack/chameleon/-/packages, then
- install as follows
- #+begin_src sh
- sudo apt-get install ./chameleon_1.1.0-1_amd64.deb -y
- #+end_src
- Chameleon will be installed on your system meaning you can use
- drivers for performance tests
- #+begin_src sh
- mpiexec -n 2 chameleon_stesting -t 2 -o gemm -n 1000
- #+end_src
- and use Chameleon library in your own project
- #+begin_src sh
- # example usage: use chameleon library in your own cmake project (we provide a CHAMELEONConfig.cmake)
- git clone https://gitlab.inria.fr/solverstack/distrib.git
- cd distrib/cmake/test/chameleon && mkdir build && cd build && cmake .. && make && ./test_chameleon
-
- # example usage: use chameleon library in your own not cmake project
- # use pkg-config to get compiler flags and linking
- pkg-config --cflags chameleon
- pkg-config --libs chameleon
- # if there are static libraries use the --static option of pkg-config
- #+end_src
-
- Do not hesitate to send an [[mailto:florent.pruvost@inria.fr][email]] if you need a package for your
- Debian distribution.
-** Distribution of Chameleon using GNU Guix
- <<sec:guix>>
-
- We provide [[http://guix.gnu.org/][Guix]] packages to install Chameleon with its dependencies
- in a reproducible way on GNU/Linux systems. For MacOSX please refer
- to the next sections about Brew or Spack packaging.
-
- If you are "root" on the system you can install Guix and directly
- use it to install the libraries. On supercomputers your are not
- root on you may still be able to use it if Docker or Singularity
- are available on the machine because Chameleon can be packaged as
- Docker/Singularity images with Guix.
-
-*** Installing Guix
- Guix requires a running GNU/Linux system, GNU tar and Xz.
-
- #+begin_src sh
- gpg --keyserver pgp.mit.edu --recv-keys 3CE464558A84FDC69DB40CFB090B11993D9AEBB5
- wget https://git.savannah.gnu.org/cgit/guix.git/plain/etc/guix-install.sh
- chmod +x guix-install.sh
- sudo ./guix-install.sh
- #+end_src
-
- The Chameleon packages are not official Guix packages. It is then
- necessary to add a channel to get additional packages. Create a
- ~/.config/guix/channels.scm file with the following snippet:
- #+begin_example
- (cons (channel
- (name 'guix-hpc-non-free)
- (url "https://gitlab.inria.fr/guix-hpc/guix-hpc-non-free.git"))
- %default-channels)
- #+end_example
-
- Update guix package definition
- #+begin_src sh
- guix pull
- #+end_src
-
- Update new guix in the path
- #+begin_src sh
- PATH="$HOME/.config/guix/current/bin${PATH:+:}$PATH"
- hash guix
- #+end_src
-
- For further shell sessions, add this to the ~/.bash_profile file
- #+begin_example
- export PATH="$HOME/.config/guix/current/bin${PATH:+:}$PATH"
- export GUIX_LOCPATH="$HOME/.guix-profile/lib/locale"
- #+end_example
-
- Chameleon packages are now available
- #+begin_src sh
- guix search ^chameleon
- #+end_src
-
- Refer to the [[https://guix.gnu.org/manual/en/guix.html#Invoking-guix-package][official documentation of Guix]] to learn the basic
- commands.
-
-*** Installing Chameleon with Guix
-
- Standard Chameleon, last release
- #+begin_src sh
- guix install chameleon
- #+end_src
-
- Notice that there exist several build variants
- - chameleon (default) : with starpu - with mpi - with OpenBlas
- - chameleon-mkl-mt : default version but with Intel MKL multithreaded to replace OpenBlas
- - chameleon-cuda : with starpu - with mpi - with cuda
- - chameleon-cuda-mkl-mt : with starpu - with mpi - with cuda - with Intel MKL multithreaded to replace OpenBlas
- - chameleon-simgrid : with starpu - with mpi - with simgrid
- - chameleon-openmp : with openmp - without mpi
- - chameleon-parsec : with parsec - without mpi
- - chameleon-quark : with quark - without mpi
-
-
- Change the version
- #+begin_src sh
- guix install chameleon --with-branch=chameleon=master
- guix install chameleon --with-commit=chameleon=b31d7575fb7d9c0e1ba2d8ec633e16cb83778e8b
- guix install chameleon --with-git-url=chameleon=https://gitlab.inria.fr/fpruvost/chameleon.git
- guix install chameleon --with-git-url=chameleon=$HOME/git/chameleon
- #+end_src
-
- Notice also that default mpi is OpenMPI and default blas/lapack is
- Openblas. This can be changed with a [[https://guix.gnu.org/manual/en/guix.html#Package-Transformation-Options][transformation option]].
-
- Change some dependencies
- #+begin_src sh
- # install chameleon with intel mkl to replace openblas, nmad to replace openmpi and starpu with fxt
- guix install chameleon --with-input=openblas=mkl --with-input=openmpi=nmad --with-input=starpu=starpu-fxt
- #+end_src
-
-*** Generate a Chameleon Docker image with Guix
-
- To install Chameleon and its dependencies within a docker image
- (OpenMPI stack)
- #+begin_src sh
- docker_chameleon=`guix pack -f docker chameleon chameleon --with-branch=chameleon=master --with-input=openblas=mkl mkl starpu hwloc openmpi openssh slurm bash coreutils inetutils util-linux procps git grep tar sed gzip which gawk perl emacs-minimal vim gcc-toolchain make cmake pkg-config -S /bin=bin --entry-point=/bin/bash`
- # Load the generated tarball as a docker image
- docker_chameleon_tag=`docker load --input $docker_chameleon | grep "Loaded image: " | cut -d " " -f 3-`
- # Change tag name, see the existing image name with "docker images" command, then change to a more simple name
- docker tag $docker_chameleon_tag guix/chameleon-tmp
- #+end_src
-
- Create a Dockerfile inheriting from the image (renamed
- =guix/chameleon= here):
-
- #+begin_src sh :eval no :tangle Dockerfile :export none
- FROM guix/chameleon-tmp
-
- # Create a directory for user 1000
- RUN mkdir -p /builds
- RUN chown -R 1000 /builds
-
- ENTRYPOINT ["/bin/bash", "-l"]
-
- # Enter the image as user 1000 in /builds
- USER 1000
- WORKDIR /builds
- ENV HOME /builds
- #+end_src
-
- Then create the final docker image from this docker file.
-
- #+begin_src sh
- docker build -t guix/chameleon .
- #+end_src
-
- Test the image
- #+begin_src sh
- docker run -it guix/chameleon
- # test starpu
- STARPU=`pkg-config --variable=prefix libstarpu`
- mpiexec -np 4 $STARPU/lib/starpu/mpi/comm
- # test chameleon
- CHAMELEON=`pkg-config --variable=prefix chameleon`
- mpiexec -np 2 $CHAMELEON/bin/chameleon_stesting -H -o gemm -P 2 -t 2 -m 2000 -n 2000 -k 2000
- #+end_src
-
-*** Generate a Chameleon Singularity image with Guix
-
- To package Chameleon and its dependencies within a singularity image
- (OpenMPI stack)
- #+begin_src sh
- singularity_chameleon=`guix pack -f squashfs chameleon --with-branch=chameleon=master --with-input=openblas=mkl mkl starpu hwloc openmpi openssh slurm hdf5 zlib bash coreutils inetutils util-linux procps git grep tar sed gzip which gawk perl emacs-minimal vim gcc-toolchain make cmake pkg-config -S /bin=bin --entry-point=/bin/bash`
- cp $singularity_chameleon chameleon-pack.gz.squashfs
- # copy the singularity image on the supercomputer, e.g. 'supercomputer'
- scp chameleon-pack.gz.squashfs supercomputer:
- #+end_src
-
- On a machine where Singularity is installed Chameleon can then be
- called as follows
- #+begin_src sh
- # at least openmpi and singularity are required here, e.g. module add openmpi singularity
- mpiexec -np 2 singularity exec chameleon-pack.gz.squashfs /bin/chameleon_stesting -H -o gemm -P 2 -t 2 -m 2000 -n 2000 -k 2000
- #+end_src
-
-** Distribution of Chameleon using Spack
- <<sec:spack>>
-
-*** Installing Spack
-
- To get support to install a full distribution on Linux or MacOS X,
- Chameleon plus dependencies, we encourage users to use
- [[https://spack.io/][Spack]]. Please refer to our [[https://gitlab.inria.fr/solverstack/spack-repo/blob/master/README.org][Spack Repository]].
-
- #+begin_src sh
- git clone https://github.com/llnl/spack.git
- export SPACK_ROOT=$PWD/spack
- cd spack/
- git checkout v0.16.0
- . $SPACK_ROOT/share/spack/setup-env.sh
- git clone https://gitlab.inria.fr/solverstack/spack-repo.git ./var/spack/repos/solverstack
- spack repo add ./var/spack/repos/solverstack
- #+end_src
-
- Chameleon is then available
- #+begin_src sh
- spack info chameleon
- spack spec chameleon
- #+end_src
-
- Refer to te [[https://spack.readthedocs.io/en/latest/basic_usage.html][official documentation of Spack]] to learn the basic
- commands.
-
-*** Installing Chameleon with Spack
-
- Standard Chameleon, last state on the 'master' branch
- #+begin_src sh
- spack install -v chameleon
- # chameleon is installed here:
- spack location -i chameleon
- #+end_src
-
- Notice that there exist several build variants
- - chameleon (default) : with starpu - with mpi
- - tune the build type (CMake) with build_type=RelWithDebInfo|Debug|Release
- - enable/disable shared libraries with +/- shared
- - enable/disable mpi with +/- mpi
- - enable/disable cuda with +/- cuda
- - enable/disable fxt with +/- fxt
- - enable/disable simgrid with +/- simgrid
- - +openmp~starpu : with openmp - without starpu
- - +quark~starpu : with quark - without starpu
-
- Change the version
- #+begin_src sh
- spack install -v chameleon@1.0.0
- #+end_src
-
- Notice also that default mpi is OpenMPI and default blas/lapack is
- Openblas. This can be changed by adding some [[https://spack.readthedocs.io/en/latest/basic_usage.html#constraining-virtual-packages][constraints on
- virtual packages]].
-
- Change some dependencies
- #+begin_src sh
- # see lapack providers
- spack providers lapack
- # see mpi providers
- spack providers mpi
- # install chameleon with intel mkl to replace openblas
- spack install -v chameleon ^intel-mkl
- # install chameleon with nmad to replace openmpi
- spack install -v chameleon ^nmad
- #+end_src
-** Distribution Brew for Mac OS X
- We provide some [[https://brew.sh/][brew]] packages here
- https://gitlab.inria.fr/solverstack/brew-repo (under construction).
-** Build and install Chameleon with CMake
- Compilation of Chameleon libraries and executables are done with
- CMake (http://www.cmake.org/). This version has been tested with
- CMake 3.10.2 but any version superior to 2.8 should be fine.
-
- Here the steps to configure, build, test and install
- 1. configure:
- #+begin_src
- cmake path/to/chameleon -DOPTION1= -DOPTION2= ...
- # see the "Configuration options" section to get list of options
- # see the "Dependencies detection" for details about libraries detection
- #+end_src
- 2. build:
- #+begin_src
- make
- # do not hesitate to use -j[ncores] option to speedup the compilation
- #+end_src
- 3. test (optional, required CHAMELEON_ENABLE_TESTING=ON):
- #+begin_src
- make test
- # or
- ctest
- #+end_src
- 4. install (optional):
- #+begin_src
- make install
- #+end_src
- Do not forget to specify the install directory with
- *-DCMAKE_INSTALL_PREFIX* at configure.
- #+begin_example
- cmake /home/jdoe/chameleon -DCMAKE_INSTALL_PREFIX=/home/jdoe/install/chameleon
- #+end_example
- Note that the install process is optional. You are free to use
- Chameleon binaries compiled in the build directory.
-*** Configuration options
- You can optionally activate some options at cmake configure (like CUDA, MPI, ...)
- invoking ~cmake path/to/your/CMakeLists.txt -DOPTION1= -DOPTION2= ...~
- #+begin_src
- cmake /home/jdoe/chameleon/ -DCMAKE_BUILD_TYPE=Debug \
- -DCMAKE_INSTALL_PREFIX=/home/jdoe/install/ \
- -DCHAMELEON_USE_CUDA=ON \
- -DCHAMELEON_USE_MPI=ON \
- -DBLA_VENDOR=Intel10_64lp \
- -DCHAMELEON_ENABLE_TRACING=ON
- #+end_src
-
- You can get the full list of options with *-L[A][H]* options of cmake command
- #+begin_src
- cmake -LH /home/jdoe/chameleon/
- #+end_src
-
- You can also set the options thanks to the *ccmake* interface.
-
-**** Native CMake options (non-exhaustive list)
- * *CMAKE_BUILD_TYPE=Debug|Release|RelWithDebInfo|MinSizeRel*:
- level of compiler optimization, enable/disable debug
- information
- * *CMAKE_INSTALL_PREFIX=path/to/your/install/dir*: where headers,
- libraries, executables, etc, will be copied when invoking make
- install
- * *BUILD_SHARED_LIBS=ON|OFF*: indicate wether or not CMake has to
- build Chameleon static (~OFF~) or shared (~ON~) libraries.
- * *CMAKE_C_COMPILER=gcc|icc|...*: to choose the C compilers
- if several exist in the environment
- * *CMAKE_Fortran_COMPILER=gfortran|ifort|...*: to choose the
- Fortran compilers if several exist in the environment
-
-**** Related to specific modules (find_package) to find external libraries
- * *BLA_VENDOR=All|OpenBLAS|Generic|Intel10_64lp|Intel10_64lp_seq|FLAME*:
- to use intel mkl for example, see the list of BLA_VENDOR in
- FindBLAS.cmake in cmake_modules/morse_cmake/modules/find
-
- Libraries detected with an official cmake module (see module
- files in CMAKE_ROOT/Modules/): BLAS - LAPACK - CUDA - MPI -
- OpenMP - Threads.
-
- Libraries detected with our cmake modules (see module files in
- cmake_modules/morse_cmake/modules/find/ directory of Chameleon
- sources): CBLAS - EZTRACE - FXT - HWLOC - LAPACKE - PARSEC -
- QUARK - SIMGRID - STARPU.
-
-**** Chameleon specific options
- * *CHAMELEON_SCHED=STARPU|PARSEC|QUARK|OPENMP* (default STARPU): to
- link respectively with StarPU, PaRSEC, Quark, OpenMP library
- (runtime system)
- * *CHAMELEON_USE_MPI=ON|OFF* (default OFF): to link with MPI
- library (message passing implementation for use of multiple
- nodes with distributed memory), can only be used with StarPU
- and PaRSEC
- * *CHAMELEON_USE_CUDA=ON|OFF* (default OFF): to link with CUDA
- runtime (implementation paradigm for accelerated codes on GPUs)
- and cuBLAS library (optimized BLAS kernels on GPUs), can only
- be used with StarPU and PaRSEC
- * *CHAMELEON_ENABLE_DOC=ON|OFF* (default OFF): to control build of
- the documentation contained in doc/ sub-directory
- * *CHAMELEON_ENABLE_EXAMPLE=ON|OFF* (default ON): to control build
- of the examples executables (API usage) contained in example/
- sub-directory
- * *CHAMELEON_ENABLE_PRUNING_STATS=ON|OFF* (default OFF)
- * *CHAMELEON_ENABLE_TESTING=ON|OFF* (default ON): to control build
- of testing executables (timer and numerical check) contained in testing/
- sub-directory
- * *CHAMELEON_SIMULATION=ON|OFF* (default OFF): to enable
- simulation mode, means Chameleon will not really execute tasks,
- see details in section [[sec:simu][Use simulation mode with
- StarPU-SimGrid]]. This option must be used with StarPU compiled
- with [[http://simgrid.gforge.inria.fr/][SimGrid]] allowing to guess the execution time on any
- architecture. This feature should be used to make experiments
- on the scheduler behaviors and performances not to produce
- solutions of linear systems.
- * *CHAMELEON_USE_MIGRATE=ON|OFF* (default OFF): enables the data
- migration in QR algorithms
-
-*** Dependencies detection
- <<sec:depdet>>
-
- You have different choices to detect dependencies on your system,
- either by setting some environment variables containing paths to
- the libs and headers or by specifying them directly at cmake
- configure. Different cases:
-
- 1) detection of dependencies through environment variables:
- - LD_LIBRARY_PATH (DYLD_LIBRARY_PATH on Mac OSX) should contain
- the list of paths where to find the libraries:
- #+begin_src
- export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:install/path/to/your/lib
- #+end_src
- - INCLUDE (or CPATH, or C_INCLUDE_PATH)should contain the list
- of paths where to find the header files of libraries
- #+begin_src
- export INCLUDE=$INCLUDE:install/path/to/your/headers
- #+end_src
- 2) detection with user's given paths:
- - you can specify the path at cmake configure by invoking
- #+begin_example
- cmake path/to/your/CMakeLists.txt -DLIB_DIR=path/to/your/lib
- #+end_example
- where LIB stands for the name of the lib to look for, /e.g./
- #+begin_src
- cmake path/to/your/CMakeLists.txt -DQUARK_DIR=path/to/quarkdir \
- -DCBLAS_DIR= ...
- #+end_src
- it is also possible to specify headers and library directories
- separately
- #+begin_src
- cmake path/to/your/CMakeLists.txt \
- -DQUARK_INCDIR=path/to/quark/include \
- -DQUARK_LIBDIR=path/to/quark/lib
- #+end_src
- 3) detection with custom environment variables: all variables like
- _DIR, _INCDIR, _LIBDIR can be set as environment variables
- instead of CMake options, there will be read
- 4) using [[https://www.freedesktop.org/wiki/Software/pkg-config/][pkg-config]] for libraries that provide .pc files
- - update your *PKG_CONFIG_PATH* to the paths where to find .pc
- files of installed external libraries like hwloc, starpu, some
- blas/lapack, etc
-
- Note that PaRSEC and StarPU are only detected with pkg-config
- mechanism because it is always provided and this avoids errors.
-
-** Linking an external application with Chameleon libraries
- Compilation and link with Chameleon libraries have been tested with
- the GNU compiler suite ~gcc/gfortran~ and the Intel compiler suite
- ~icc/ifort~.
-
-*** For CMake projects
- A CHAMELEONConfig.cmake file is provided at installation, stored
- in <prefix>/lib/cmake/chameleon, so that users in cmake project
- can use through the variable CHAMELEON_ROOT (set it as environment
- or CMake variable).
-
- #+begin_src
- sudo apt-get update
- sudo apt-get install -y libopenblas-dev liblapacke-dev libstarpu-dev
- git clone --recursive https://gitlab.inria.fr/solverstack/chameleon.git
- cd chameleon && mkdir -p build && cd build
- CHAMELEON_ROOT=$PWD/install
- cmake .. -DCMAKE_INSTALL_PREFIX=$CHAMELEON_ROOT && make -j5 install
- # chameleon is installed in $CHAMELEON_ROOT
-
- # if your work in a cmake project you can use the CHAMELEONConfig.cmake file
- # installed under <prefix>/lib/cmake/chameleon/ by setting your
- # CMAKE_PREFIX_PATH with the path of installation. In your cmake project, use
- # find_package(CHAMELEON) and link your libraries and/or executables with the
- # library target MORSE::CHAMELEON
- cmake . -DCMAKE_PREFIX_PATH=$CHAMELEON_ROOT
- #+end_src
-*** For non CMake projects
- The compiler, linker flags that are necessary to build an
- application using Chameleon are given through the [[https://www.freedesktop.org/wiki/Software/pkg-config/][pkg-config]]
- mechanism.
- #+begin_src
- sudo apt-get update
- sudo apt-get install -y libopenblas-dev liblapacke-dev libstarpu-dev
- git clone --recursive https://gitlab.inria.fr/solverstack/chameleon.git
- cd chameleon && mkdir -p build && cd build
- CHAMELEON_ROOT=$PWD/install
- cmake .. -DCMAKE_INSTALL_PREFIX=$CHAMELEON_ROOT && make -j5 install
- # chameleon is installed in $CHAMELEON_ROOT
-
- export PKG_CONFIG_PATH=$CHAMELEON_ROOT/lib/pkgconfig:$PKG_CONFIG_PATH
- pkg-config --cflags chameleon
- pkg-config --libs chameleon
- pkg-config --libs --static chameleon
-
- # use it in your configure/make
- #+end_src
- The .pc files required are located in the sub-directory
- ~lib/pkgconfig~ of your Chameleon install directory.
-
-*** Static linking in C
- Lets imagine you have a file ~main.c~ that you want to link with
- Chameleon static libraries. Lets consider
- ~/home/yourname/install/chameleon~ is the install directory
- of Chameleon containing sub-directories ~include/~ and
- ~lib/~. Here could be your compilation command with gcc
- compiler:
- #+begin_src
- gcc -I/home/yourname/install/chameleon/include -o main.o -c main.c
- #+end_src
- Now if you want to link your application with Chameleon static libraries, you
- could do:
- #+begin_src
- gcc main.o -o main \
- /home/yourname/install/chameleon/lib/libchameleon.a \
- /home/yourname/install/chameleon/lib/libchameleon_starpu.a \
- /home/yourname/install/chameleon/lib/libcoreblas.a \
- -lstarpu-1.2 -Wl,--no-as-needed -lmkl_intel_lp64 \
- -lmkl_sequential -lmkl_core -lpthread -lm -lrt
- #+end_src
- As you can see in this example, we also link with some dynamic
- libraries *starpu-1.2*, *Intel MKL* libraries (for
- BLAS/LAPACK/CBLAS/LAPACKE), *pthread*, *m* (math) and *rt*. These
- libraries will depend on the configuration of your Chameleon
- build. You can find these dependencies in .pc files we generate
- during compilation and that are installed in the sub-directory
- ~lib/pkgconfig~ of your Chameleon install directory. Note also that
- you could need to specify where to find these libraries with *-L*
- option of your compiler/linker.
-
- Before to run your program, make sure that all shared libraries
- paths your executable depends on are known. Enter ~ldd main~
- to check. If some shared libraries paths are missing append them
- in the LD_LIBRARY_PATH (for Linux systems) environment
- variable (DYLD_LIBRARY_PATH on Mac).
-
-*** Dynamic linking in C
- For dynamic linking (need to build Chameleon with CMake option
- BUILD_SHARED_LIBS=ON) it is similar to static compilation/link but
- instead of specifying path to your static libraries you indicate
- the path to dynamic libraries with *-L* option and you give
- the name of libraries with *-l* option like this:
- #+begin_src
- gcc main.o -o main \
- -L/home/yourname/install/chameleon/lib \
- -lchameleon -lchameleon_starpu -lcoreblas \
- -lstarpu-1.2 -Wl,--no-as-needed -lmkl_intel_lp64 \
- -lmkl_sequential -lmkl_core -lpthread -lm -lrt
- #+end_src
- Note that an update of your environment variable LD_LIBRARY_PATH
- (DYLD_LIBRARY_PATH on Mac) with the path of the libraries could be
- required before executing
- #+begin_src
- export LD_LIBRARY_PATH=path/to/libs:path/to/chameleon/lib
- #+end_src
-
-# # *** Build a Fortran program with Chameleon :noexport:
-# #
-# # Chameleon provides a Fortran interface to user functions. Example:
-# # #+begin_src
-# # call chameleon_version(major, minor, patch) !or
-# # call CHAMELEON_VERSION(major, minor, patch)
-# # #+end_src
-# #
-# # Build and link are very similar to the C case.
-# #
-# # Compilation example:
-# # #+begin_src
-# # gfortran -o main.o -c main.f90
-# # #+end_src
-# #
-# # Static linking example:
-# # #+begin_src
-# # gfortran main.o -o main \
-# # /home/yourname/install/chameleon/lib/libchameleon.a \
-# # /home/yourname/install/chameleon/lib/libchameleon_starpu.a \
-# # /home/yourname/install/chameleon/lib/libcoreblas.a \
-# # -lstarpu-1.2 -Wl,--no-as-needed -lmkl_intel_lp64 \
-# # -lmkl_sequential -lmkl_core -lpthread -lm -lrt
-# # #+end_src
-# #
-# # Dynamic linking example:
-# # #+begin_src
-# # gfortran main.o -o main \
-# # -L/home/yourname/install/chameleon/lib \
-# # -lchameleon -lchameleon_starpu -lcoreblas \
-# # -lstarpu-1.2 -Wl,--no-as-needed -lmkl_intel_lp64 \
-# # -lmkl_sequential -lmkl_core -lpthread -lm -lrt
-# # #+end_src
diff --git a/doc/orgmode/chapters/introduction.org b/doc/orgmode/chapters/introduction.org
deleted file mode 100644
index ba95de2cd854d94e857832b03fae7cba03687fcc..0000000000000000000000000000000000000000
--- a/doc/orgmode/chapters/introduction.org
+++ /dev/null
@@ -1,310 +0,0 @@
-# This file is part of the Chameleon User's Guide.
-# Copyright (C) 2018 Inria
-# See the file ../users_guide.org for copying conditions.
-** MORSE project
- #+NAME: fig:chameleon_header
- #+ATTR_HTML: :align center
- [[file:chameleon_header.png]]
-
- Chameleon is a linear algebra software created jointly by several
- research teams as part of the MORSE associate team: [[http://www.icl.utk.edu/][ICL]], [[https://www.inria.fr/en/][Inria]],
- [[https://www.kaust.edu.sa/en][KAUST]], [[http://www.ucdenver.edu/pages/ucdwelcomepage.aspx][The University of Colorado Denver]].
-
-*** MORSE Objectives
- When processor clock speeds flatlined in 2004, after more than
- fifteen years of exponential increases, the era of near automatic
- performance improvements that the HPC application community had
- previously enjoyed came to an abrupt end. To develop software that
- will perform well on petascale and exascale systems with thousands
- of nodes and millions of cores, the list of major challenges that
- must now be confronted is formidable:
- 1) dramatic escalation in the costs of intrasystem communication
- between processors and/or levels of memory hierarchy;
- 2) increased heterogeneity of the processing units (mixing CPUs,
- GPUs, etc. in varying and unexpected design combinations);
- 3) high levels of parallelism and more complex constraints means
- that cooperating processes must be dynamically and unpredictably
- scheduled for asynchronous execution;
- 4) software will not run at scale without much better resilience to
- faults and far more robustness; and
- 5) new levels of self-adaptivity will be required to enable
- software to modulate process speed in order to satisfy limited
- energy budgets.
- The MORSE associate team will tackle the first three challenges in
- a orchestrating work between research groups respectively
- specialized in sparse linear algebra, dense linear algebra and
- runtime systems. The overall objective is to develop robust
- linear algebra libraries relying on innovative runtime systems
- that can fully benefit from the potential of those future
- large-scale complex machines. Challenges 4) and 5) will also be
- investigated by the different teams in the context of other
- partnerships, but they will not be the main focus of the associate
- team as they are much more prospective.
-
-*** Research fields
- The overall goal of the MORSE associate team is to enable advanced
- numerical algorithms to be executed on a scalable unified runtime
- system for exploiting the full potential of future exascale
- machines. We expect advances in three directions based first on
- strong and closed interactions between the runtime and numerical
- linear algebra communities. This initial activity will then
- naturally expand to more focused but still joint research in both
- fields.
-
-**** Fine interaction between linear algebra and runtime systems
- On parallel machines, HPC applications need to take care of data
- movement and consistency, which can be either explicitly managed
- at the level of the application itself or delegated to a runtime
- system. We adopt the latter approach in order to better keep up
- with hardware trends whose complexity is growing exponentially.
- One major task in this project is to define a proper interface
- between HPC applications and runtime systems in order to maximize
- productivity and expressivity. As mentioned in the next section,
- a widely used approach consists in abstracting the application as
- a DAG that the runtime system is in charge of scheduling.
- Scheduling such a DAG over a set of heterogeneous processing
- units introduces a lot of new challenges, such as predicting
- accurately the execution time of each type of task over each kind
- of unit, minimizing data transfers between memory banks,
- performing data prefetching, etc. Expected advances: In a
- nutshell, a new runtime system API will be designed to allow
- applications to provide scheduling hints to the runtime system
- and to get real-time feedback about the consequences of
- scheduling decisions.
-
-**** Runtime systems
- A runtime environment is an intermediate layer between the system
- and the application. It provides low-level functionality not
- provided by the system (such as scheduling or management of the
- heterogeneity) and high-level features (such as performance
- portability). In the framework of this proposal, we will work on
- the scalability of runtime environment. To achieve scalability it
- is required to avoid all centralization. Here, the main problem
- is the scheduling of the tasks. In many task-based runtime
- environments the scheduler is centralized and becomes a
- bottleneck as soon as too many cores are involved. It is
- therefore required to distribute the scheduling decision or to
- compute a data distribution that impose the mapping of task
- using, for instance the so-called ``owner-compute'' rule.
- Expected advances: We will design runtime systems that enable an
- efficient and scalable use of thousands of distributed multicore
- nodes enhanced with accelerators.
-
-**** Linear algebra
- Because of its central position in HPC and of the well understood
- structure of its algorithms, dense linear algebra has often
- pioneered new challenges that HPC had to face. Again, dense
- linear algebra has been in the vanguard of the new era of
- petascale computing with the design of new algorithms that can
- efficiently run on a multicore node with GPU accelerators. These
- algorithms are called ``communication-avoiding'' since they have
- been redesigned to limit the amount of communication between
- processing units (and between the different levels of memory
- hierarchy). They are expressed through Direct Acyclic Graphs
- (DAG) of fine-grained tasks that are dynamically
- scheduled. Expected advances: First, we plan to investigate the
- impact of these principles in the case of sparse applications
- (whose algorithms are slightly more complicated but often rely on
- dense kernels). Furthermore, both in the dense and sparse cases,
- the scalability on thousands of nodes is still limited; new
- numerical approaches need to be found. We will specifically
- design sparse hybrid direct/iterative methods that represent a
- promising approach.
-
-*** Research papers
- Research papers about MORSE can be found [[http://icl.cs.utk.edu/projectsdev/morse/pubs/index.html][here]].
-
-** Chameleon
-*** Chameleon software
- The main purpose is to address the performance shortcomings of the
- [[http://www.netlib.org/lapack/][LAPACK]] and [[http://www.netlib.org/scalapack/][ScaLAPACK]] libraries on multicore processors and
- multi-socket systems of multicore processors and their inability
- to efficiently utilize accelerators such as Graphics Processing
- Units (GPUs).
-
- Chameleon is a framework written in C which provides routines to
- solve dense general systems of linear equations, symmetric
- positive definite systems of linear equations and linear least
- squares problems, using LU, Cholesky, QR and LQ factorizations.
- Real arithmetic and complex arithmetic are supported in both
- single precision and double precision. It supports Linux and Mac
- OS/X machines (mainly tested on Intel x86-64 and IBM Power
- architectures).
-
- Chameleon is based on the [[http://icl.cs.utk.edu/plasma/][PLASMA]] source code but is not limited to
- shared-memory environment and can exploit multiple GPUs.
- Chameleon is interfaced in a generic way with [[http://runtime.bordeaux.inria.fr/StarPU/][StarPU]], [[http://icl.utk.edu/parsec/][PaRSEC]],
- [[http://icl.cs.utk.edu/quark/][QUARK]] runtime systems. This feature allows to analyze in a
- unified framework how sequential task-based algorithms behave
- regarding different runtime systems implementations. Using
- Chameleon with *StarPU* or *PaRSEC* runtime systems allows to exploit
- GPUs through kernels provided by [[https://developer.nvidia.com/cublas][cuBLAS]] and clusters of
- interconnected nodes with distributed memory (using [[http://www.open-mpi.org/][MPI]]).
- Computation of very large systems with dense matrices on a cluster
- of nodes is still being experimented and stabilized. It is not
- expected to get stable performances with the current version using
- MPI.
-
-*** PLASMA's design principles
- Chameleon is originally based on [[http://icl.cs.utk.edu/plasma/][PLASMA]] so that design principles
- are very similar. The content of this section PLASMA's design
- principles has been copied from the /Design principles/ section of
- the PLASMA User's Guide.
-
-**** Tile Algorithms
- Tile algorithms are based on the idea of processing the matrix by
- square tiles of relatively small size, such that a tile fits
- entirely in one of the cache levels associated with one core.
- This way a tile can be loaded to the cache and processed
- completely before being evicted back to the main memory. Of the
- three types of cache misses, *compulsory*, *capacity* and *conflict*,
- the use of tile algorithms minimizes the number of capacity
- misses, since each operation loads the amount of data that does
- not ``overflow'' the cache.
-
- For some operations such as matrix multiplication and Cholesky
- factorization, translating the classic algorithm to the tile
- algorithm is trivial. In the case of matrix multiplication, the
- tile algorithm is simply a product of applying the technique of
- *loop tiling* to the canonical definition of three nested loops.
- It is very similar for the Cholesky factorization. The
- *left-looking* definition of Cholesky factorization from LAPACK is
- a loop with a sequence of calls to four routines: xSYRK
- (symmetric *rank-k* update), xPOTRF (Cholesky factorization of a
- small block on the diagonal), xGEMM (matrix multiplication) and
- xTRSM (triangular solve). If the xSYRK, xGEMM and xTRSM
- operations are expressed with the canonical definition of three
- nested loops and the technique of loop tiling is applied, the
- tile algorithm results. Since the algorithm is produced by
- simple reordering of operations, neither the number of operations
- nor numerical stability of the algorithm are affected.
-
- The situation becomes slightly more complicated for LU and QR
- factorizations, where the classic algorithms factorize an entire
- panel of the matrix (a block of columns) at every step of the
- algorithm. One can observe, however, that the process of matrix
- factorization is synonymous with introducing zeros in approproate
- places and a tile algorithm can be fought of as one that zeroes
- one tile of the matrix at a time. This process is referred to as
- updating of a factorization or *incremental factorization*. The
- process is equivalent to factorizing the top tile of a panel,
- then placing the upper triangle of the result on top of the tile
- blow and factorizing again, then moving to the next tile and so
- on. Here, the tile LU and QR algorithms perform slightly more
- floating point operations and require slightly more memory for
- auxiliary data. Also, the tile LU factorization applies a
- different pivoting pattern and, as a result, is less numerically
- stable than classic LU with full pivoting. Numerical stability
- is not an issue in case of the tile QR, which relies on
- orthogonal transformations (Householder reflections), which are
- numerically stable.
-
- #+CAPTION: Schematic illustration of the tile LU factorization (kernel names for real arithmetics in double precision), courtesey of the [[http://icl.cs.utk.edu/plasma/][PLASMA]] team.
- #+NAME: fig:tile_lu
- #+ATTR_HTML: :width 640px :align center
- [[file:tile_lu.jpg]]
-
-**** Tile Data Layout
- <<sec:tile>>
-
- Tile layout is based on the idea of storing the matrix by square
- tiles of relatively small size, such that each tile occupies a
- continuous memory region. This way a tile can be loaded to the
- cache memory efficiently and the risk of evicting it from the
- cache memory before it is completely processed is minimized. Of
- the three types of cache misses, *compulsory*, *capacity* and
- *conflict*, the use of tile layout minimizes the number of conflict
- misses, since a continuous region of memory will completely fill
- out a /set-associative/ cache memory before an eviction can happen.
- Also, from the standpoint of multithreaded execution, the
- probability of *false sharing* is minimized. It can only affect
- the cache lines containing the beginning and the ending of a
- tile.
-
- In standard *cache-based* architecture, tiles continously laid out
- in memory maximize the profit from automatic prefetching. Tile
- layout is also beneficial in situations involving the use of
- accelerators, where explicit communication of tiles through DMA
- transfers is required, such as moving tiles between the system
- memory and the local store in Cell B. E. or moving tiles between
- the host memory and the device memory in GPUs. In most
- circumstances tile layout also minimizes the number of TLB misses
- and conflicts to memory banks or partitions. With the standard
- (*column-major*) layout, access to each column of a tile is much
- more likely to cause a conflict miss, a false sharing miss, a TLB
- miss or a bank or partition conflict. The use of the standard
- layout for dense matrix operations is a performance minefield.
- Although occasionally one can pass through it unscathed, the risk
- of hitting a spot deadly to performance is very high.
-
- Another property of the layout utilized in PLASMA is that it is
- ``flat'', meaning that it does not involve a level of
- indirection. Each tile stores a small square submatrix of the
- main matrix in a *column-major* layout. In turn, the main matrix is
- an arrangement of tiles immediately following one another in a
- *column-major* layout. The offset of each tile can be calculated
- through address arithmetics and does not involve pointer
- indirection. Alternatively, a matrix could be represented as an
- array of pointers to tiles, located anywhere in memory. Such
- layout would be a radical and unjustifiable departure from LAPACK
- and ScaLAPACK. Flat tile layout is a natural progression from
- LAPACK's *column-major* layout and ScaLAPACK's /block-cyclic/ layout.
-
- Another related property of PLASMA's tile layout is that it
- includes provisions for padding of tiles, i.e., the actual region
- of memory designated for a tile can be larger than the memory
- occupied by the actual data. This allows to force a certain
- alignment of tile boundaries, while using the flat organization
- described in the previous paragraph. The motivation is that, at
- the price of small memory overhead, alignment of tile boundaries
- may prove benefivial in multiple scenarios involving memory
- systems of standard multicore processors, as well as
- accelerators. The issues that come into play are, again, the use
- of TLBs and memory banks or partitions.
-
- #+CAPTION: Schematic illustration of the tile layout with *column-major* order of tiles, *column-major* order of elements within tiles and (optional) padding for enforcing a certain alighment of tile bondaries, courtesey of the [[http://icl.cs.utk.edu/plasma/][PLASMA]] team.
- #+NAME: fig:tile_layout
- #+ATTR_HTML: :width 640px :align center
- [[file:tile_layout.jpg]]
-
-**** Dynamic Task Scheduling
-
- Dynamic scheduling is the idea of assigning work to cores based
- on the availability of data for processing at any given point in
- time and is also referred to as *data-driven* scheduling. The
- concept is related closely to the idea of expressing computation
- through a task graph, often referred to as the DAG (*Direct
- Acyclic Graph*), and the flexibility exploring the DAG at runtime.
- Thus, to a large extent, dynamic scheduling is synonymous with
- *runtime scheduling*. An important concept here is the one of
- the *critical path*, which defines the upper bound on the
- achievable parallelism, and needs to be pursued at the maximum
- speed. This is in direct opposition to the *fork-and-join* or
- *data-parallel* programming models, where artificial
- synchronization points expose serial sections of the code, where
- multiple cores are idle, while sequential processing takes place.
- The use of dynamic scheduling introduces a *trade-off*, though.
- The more dynamic (flexible) scheduling is, the more centralized
- (and less scalable) the scheduling mechanism is. For that
- reason, currently PLASMA uses two scheduling mechanisms, one
- which is fully dynamic and one where work is assigned statically
- and dependency checks are done at runtime.
-
- The first scheduling mechanism relies on unfolding a *sliding
- window* of the task graph at runtime and scheduling work by
- resolving data hazards: *Read After Write(RAW)*, *Write After Read
- (WAR)* and *Write After Write (WAW)*, a technique analogous to
- instruction scheduling in superscalar processors. It also relies
- on *work-stealing* for balanding the load among all multiple cores.
- The second scheduling mechanism relies on statically designating
- a path through the execution space of the algorithm to each core
- and following a cycle: transition to a task, wait for its
- dependencies, execute it, update the overall progress. Task are
- identified by tuples and task transitions are done through
- locally evaluated formulas. Progress information can be
- centralized, replicated or distributed (currently centralized).
-
- #+CAPTION: A trace of the tile QR factorization executing on eight cores without any global synchronization points (kernel names for real arithmetics in single precision), courtesey of the [[http://icl.cs.utk.edu/plasma/][PLASMA]] team.
- #+NAME: fig:trace_qr
- #+ATTR_HTML: :width 640px :align center
- [[file:trace_qr.jpg]]
diff --git a/doc/orgmode/chapters/performances.org b/doc/orgmode/chapters/performances.org
deleted file mode 100644
index 860f52346f2568f6c82d04dcc408413b7f24af5a..0000000000000000000000000000000000000000
--- a/doc/orgmode/chapters/performances.org
+++ /dev/null
@@ -1,65 +0,0 @@
-# This file is part of the Chameleon User's Guide.
-# Copyright (C) 2020 Inria
-# See the file ../users_guide.org for copying conditions.
-
-Show performances on [[https://www.plafrim.fr/][PlaFRIM]] supercomputer.
-
-See [[https://www.plafrim.fr/hardware-documentation/][characteristics]] to get details about the hardwares.
-
-See script ~tools/bench/plafrim/run.sh~ to get details about the environment (Guix, Slurm,
-etc) and the build.
-
-Chameleon is run this way:
-#+begin_src sh
-mpiexec -np $nmpi $CHAMELEON_BUILD/testing/chameleon_${precision}testing -o ${algorithm} -P $p -t $nthr -g $ngpu -m $m -n $n -k $k -b $b
-#+end_src
-- runtime : *starpu*
-- precision : *s* or *d* for simple or double precision
-- algorithm : *gemm* or *potrf* or *geqrf_hqr*
-- nmpi = p x p
-- nthr : depends on the node
-- ngpu : depends on the node
-- m = n = k
-- b : depends on the node
-** bora (36 CPUs) nodes
- - nmpi = *1*, *4*, *9*
- - 2D block cyclic parameters : PxQ = 1x1, 2x2 and 3x3
- - Number of threads (t) = *34*, one CPU being dedicated for the
- scheduler and one other for MPI communications
- - Number of GPUs = *0*
- - Tile Size (b) = *280*
-*** CPU times
- #+CAPTION: Performances in CPU time of GEMM, POTRF and QR on bora nodes
- #+NAME: fig:chameleon_plafrim_bora_time_openmpi
- #+ATTR_HTML: :align center :width 75%
- [[file:chameleon_plafrim_bora_time_openmpi.png]]
-
-*** GFLOP/s
- #+CAPTION: Performances in GFlop/s of GEMM, POTRF and QR on bora nodes
- #+NAME: fig:chameleon_plafrim_bora_perf_openmpi
- #+ATTR_HTML: :align center :width 75%
- [[file:chameleon_plafrim_bora_perf_openmpi.png]]
-
-** sirocco [14-17] (32 CPUs + 2 GPUs V100) nodes
- - nmpi = *1*
- - 2D block cyclic parameters : PxQ = 1x1
- - Number of threads (t) = *29*, one CPU being dedicated for the
- scheduler and two others for the 2 GPUs
- - Number of GPUs = *2*
- - Tile Size (b) = *1600*
- #+CAPTION: Performances in CPU time of GEMM, POTRF and QR on sirocco nodes
- #+NAME: fig:chameleon_plafrim_sirocco
- #+ATTR_HTML: :align center :width 75%
- [[file:chameleon_plafrim_sirocco.png]]
-
-# *** CPU times
-# #+CAPTION: Performances in CPU time of GEMM, POTRF and QR on sirocco nodes
-# #+NAME: fig:chameleon_plafrim_sirocco_openmpi_time
-# #+ATTR_HTML: :align center :height 1024
-# [[file:chameleon_plafrim_sirocco_openmpi_time.png]]
-#
-# *** GFLOP/s
-# #+CAPTION: Performances in GFlop/s of GEMM, POTRF and QR on sirocco nodes
-# #+NAME: fig:chameleon_plafrim_sirocco_openmpi_perf
-# #+ATTR_HTML: :align center :height 1024
-# [[file:chameleon_plafrim_sirocco_openmpi_perf.png]]
diff --git a/doc/orgmode/org-html-themes b/doc/orgmode/org-html-themes
deleted file mode 160000
index 7e4e2ec56069dae3b40e2afdc0174bec368609d8..0000000000000000000000000000000000000000
--- a/doc/orgmode/org-html-themes
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 7e4e2ec56069dae3b40e2afdc0174bec368609d8
diff --git a/doc/orgmode/users_guide.org.in b/doc/orgmode/users_guide.org.in
deleted file mode 100644
index 226ca624c719ca1b6fcc10c15ccd23301176163d..0000000000000000000000000000000000000000
--- a/doc/orgmode/users_guide.org.in
+++ /dev/null
@@ -1,67 +0,0 @@
-#+TITLE: CHAMELEON User's Guide
-
-#+SUBTITLE: A dense linear algebra software for heterogeneous architectures
-#+LANGUAGE: en
-#+OPTIONS: H:3 num:t \n:nil @:t ::t |:t _:nil ^:nil -:t f:t *:t <:t
-#+OPTIONS: TeX:t LaTeX:t skip:nil d:nil pri:nil tags:not-in-toc html-style:nil
-#+BEAMER_THEME: Rochester
-
-#+HTML_HEAD: <link rel="stylesheet" type="text/css" href="org-html-themes/styles/readtheorg/css/htmlize.css"/>
-#+HTML_HEAD: <link rel="stylesheet" type="text/css" href="org-html-themes/styles/readtheorg/css/readtheorg.css"/>
-#+HTML_HEAD: <script type="text/javascript" src="org-html-themes/styles/readtheorg/js/jquery.min.js"></script>
-#+HTML_HEAD: <script type="text/javascript" src="org-html-themes/styles/readtheorg/js/bootstrap.min.js"></script>
-#+HTML_HEAD: <script type="text/javascript" src="org-html-themes/styles/readtheorg/js/jquery.stickytableheaders.min.js"></script>
-#+HTML_HEAD: <script type="text/javascript" src="org-html-themes/styles/readtheorg/js/readtheorg.js"></script>
-
-#+INCLUDE: "./version.org"
-
-This is the users guide to Chameleon. The software ecosystem will be
-presented, the installation instructions detailed and some usage
-examples are presented. To get more information about the application
-programming interface, please refer to the [[https://solverstack.gitlabpages.inria.fr/chameleon/doxygen/index.html][doxygen documentation]].
-* Version
- This manual documents the usage of Chameleon *version {{{VERSION}}}*.
- It was last updated on {{{UPDATED}}}.
-* Authors
- * Inria
- * University of Tennessee
- * University of Colorado Denver
- * King Abdullah University of Science and Technology
-* Copying
-
- * Copyright \copy {{{UPDATED-YEAR}}} Inria
- * Copyright \copy 2014 The University of Tennessee
- * Copyright \copy 2014 King Abdullah University of Science and Technology
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- - Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- - Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer listed
- in this license in the documentation and/or other materials provided
- with the distribution.
- - Neither the name of the copyright holders nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- This software is provided by the copyright holders and contributors
- "as is" and any express or implied warranties, including, but not
- limited to, the implied warranties of merchantability and fitness for
- a particular purpose are disclaimed. In no event shall the copyright
- owner or contributors be liable for any direct, indirect, incidental,
- special, exemplary, or consequential damages (including, but not
- limited to, procurement of substitute goods or services; loss of use,
- data, or profits; or business interruption) however caused and on any
- theory of liability, whether in contract, strict liability, or tort
- (including negligence or otherwise) arising in any way out of the use
- of this software, even if advised of the possibility of such damage.
-
-* Introduction to Chameleon
-#+INCLUDE: @CMAKE_CURRENT_SOURCE_DIR@/chapters/introduction.org
-* Installing Chameleon
-#+INCLUDE: @CMAKE_CURRENT_SOURCE_DIR@/chapters/installing.org
-* Using Chameleon
-#+INCLUDE: @CMAKE_CURRENT_SOURCE_DIR@/chapters/using.org
diff --git a/doc/user/CMakeLists.txt b/doc/user/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a28e97cfa713ce2f3fc8bbace6cf6208728a423d
--- /dev/null
+++ b/doc/user/CMakeLists.txt
@@ -0,0 +1,105 @@
+###
+#
+# @file CMakeLists.txt
+#
+# @copyright (c) 2017 Inria. All rights reserved.
+#
+###
+#
+# @project CHAMELEON
+# CHAMELEON is a software package provided by:
+# Inria Bordeaux - Sud-Ouest,
+# Univ. of Tennessee,
+# King Abdullah Univesity of Science and Technology
+# Univ. of California Berkeley,
+# Univ. of Colorado Denver.
+#
+# @version 1.1.0
+# @author Florent Pruvost
+# @author Mathieu Faverge
+# @date 2021-01-04
+#
+###
+
+cmake_minimum_required(VERSION 3.1)
+
+# Create files in binary dir
+# --------------------------
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/homepage.org
+ ${CMAKE_CURRENT_BINARY_DIR}/homepage.org
+ COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/news.org
+ ${CMAKE_CURRENT_BINARY_DIR}/news.org
+ COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/publish.el
+ ${CMAKE_CURRENT_BINARY_DIR}/publish.el
+ COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version.org.in
+ ${CMAKE_CURRENT_BINARY_DIR}/version.org
+ @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/users_guide.org.in
+ ${CMAKE_CURRENT_BINARY_DIR}/users_guide.org
+ @ONLY)
+configure_file(${CMAKE_SOURCE_DIR}/CONTRIBUTING.org
+ ${CMAKE_CURRENT_BINARY_DIR}/CONTRIBUTING.org
+ COPYONLY)
+set(FIGURES_HOMEPAGE
+ chameleon.svg
+ )
+foreach(_fig ${FIGURES_HOMEPAGE})
+ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${_fig}
+ ${CMAKE_CURRENT_BINARY_DIR}/${_fig}
+ COPYONLY)
+endforeach()
+
+set(FIGURES_USERGUIDE
+ tile_lu.jpg
+ tile_layout.jpg
+ trace_qr.jpg
+ potri_async.png
+ chameleon_header.png
+ )
+foreach(_fig ${FIGURES_USERGUIDE})
+ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/chapters/${_fig}
+ ${CMAKE_CURRENT_BINARY_DIR}/${_fig}
+ COPYONLY)
+endforeach()
+
+# Looking for emacs
+# -----------------
+FIND_PROGRAM(EMACS_COMPILER emacs)
+FIND_PROGRAM(TEX_COMPILER tex)
+
+if(EMACS_COMPILER)
+
+ # Add target for homepage
+ # -----------------------
+ add_custom_command(OUTPUT index.html
+ COMMAND ${EMACS_COMPILER}
+ ARGS --batch --no-init-file --load publish.el --funcall org-publish-all
+ DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/homepage.org
+ ${CMAKE_CURRENT_BINARY_DIR}/news.org
+ ${CMAKE_CURRENT_BINARY_DIR}/version.org
+ ${CMAKE_CURRENT_BINARY_DIR}/users_guide.org
+ ${CMAKE_CURRENT_BINARY_DIR}/CONTRIBUTING.org
+ ${CMAKE_CURRENT_BINARY_DIR}/publish.el
+ ${CMAKE_CURRENT_BINARY_DIR}/chameleon.svg)
+
+ add_custom_target(doc-homepage ALL DEPENDS index.html)
+
+ # Installation
+ # ------------
+ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/index.html
+ DESTINATION share/chameleon/doc/)
+ foreach(_fig ${FIGURES_HOMEPAGE} ${FIGURES_USERGUIDE})
+ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${_fig}
+ DESTINATION share/chameleon/doc)
+ endforeach()
+
+else(EMACS_COMPILER)
+ message(STATUS "Looking for emacs - not found")
+endif(EMACS_COMPILER)
+
+###
+### END CMakeLists.txt
+###
diff --git a/doc/user/chameleon.dot b/doc/user/chameleon.dot
new file mode 100644
index 0000000000000000000000000000000000000000..6cbc54d892d922b74c23952fe953084a9fe6f03c
--- /dev/null
+++ b/doc/user/chameleon.dot
@@ -0,0 +1,56 @@
+digraph G{
+
+# Pre-Depends: purple, bold
+# Depends: black
+# Recommends: grey
+# Suggests: black, dotted
+# Conflicts: red
+
+compound=true;
+bgcolor=transparent
+
+node [shape=box];
+
+chameleon [style="filled,bold",fillcolor=lightgrey,color=blue,label=<<B>Chameleon</B>>]
+
+subgraph cluster_runtime {
+ label = "Runtime Systems"
+ style = rounded
+ color = black
+ sched [
+ shape = "record"
+ label = "<sched_openmp> OpenMP | <sched_parsec> PaRSEC | <sched_quark> QUARK | <sched_starpu> StarPU"
+ color=green
+ ]
+ }
+chameleon -> sched [lhead=cluster_runtime]
+sched:sched_starpu -> cuda [color=dimgrey]
+sched:sched_starpu -> mpi [color=dimgrey]
+sched:sched_parsec -> cuda [color=dimgrey]
+sched:sched_parsec -> mpi [color=dimgrey]
+
+subgraph cluster_paradigm {
+ label = "Paradigms"
+ style = rounded
+ color = black
+ cuda [color=gold, style=bold,label=<<B>CUDA</B>>]
+ mpi [color=gold, style=bold,label=<<B>MPI</B>>]
+ }
+
+subgraph cluster_kernel {
+ label = "Kernels"
+ style = rounded
+ color = black
+ cublas [color=red, style=bold,label=<<B>cuBLAS</B>>]
+ lapacke [color=red, style=bold,label=<<B>LAPACKE</B>>]
+ cblas [color=red, style=bold,label=<<B>CBLAS</B>>]
+ lapacke -> cblas
+ cublas -> cuda
+ }
+
+chameleon -> cblas
+chameleon -> lapacke
+chameleon -> mpi [color=dimgrey,lhead=cluster_paradigm]
+chameleon -> cublas
+
+}
\ No newline at end of file
diff --git a/doc/user/chameleon.svg b/doc/user/chameleon.svg
new file mode 100644
index 0000000000000000000000000000000000000000..4a51dd62fcb01bd34df3e301af8ce6c7c6d61016
--- /dev/null
+++ b/doc/user/chameleon.svg
@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.43.0 (0)
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="579pt" height="219pt"
+ viewBox="0.00 0.00 579.35 219.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 215)">
+<title>G</title>
+<g id="clust1" class="cluster">
+<title>cluster_runtime</title>
+<path fill="transparent" stroke="black" d="M20,-91C20,-91 302,-91 302,-91 308,-91 314,-97 314,-103 314,-103 314,-155 314,-155 314,-161 308,-167 302,-167 302,-167 20,-167 20,-167 14,-167 8,-161 8,-155 8,-155 8,-103 8,-103 8,-97 14,-91 20,-91"/>
+<text text-anchor="middle" x="161" y="-151.8" font-family="Times,serif" font-size="14.00">Runtime Systems</text>
+</g>
+<g id="clust2" class="cluster">
+<title>cluster_paradigm</title>
+<path fill="transparent" stroke="black" d="M191,-8C191,-8 317,-8 317,-8 323,-8 329,-14 329,-20 329,-20 329,-71 329,-71 329,-77 323,-83 317,-83 317,-83 191,-83 191,-83 185,-83 179,-77 179,-71 179,-71 179,-20 179,-20 179,-14 185,-8 191,-8"/>
+<text text-anchor="middle" x="254" y="-67.8" font-family="Times,serif" font-size="14.00">Paradigms</text>
+</g>
+<g id="clust3" class="cluster">
+<title>cluster_kernel</title>
+<path fill="transparent" stroke="black" d="M366,-8C366,-8 545,-8 545,-8 551,-8 557,-14 557,-20 557,-20 557,-154.5 557,-154.5 557,-160.5 551,-166.5 545,-166.5 545,-166.5 366,-166.5 366,-166.5 360,-166.5 354,-160.5 354,-154.5 354,-154.5 354,-20 354,-20 354,-14 360,-8 366,-8"/>
+<text text-anchor="middle" x="455.5" y="-151.3" font-family="Times,serif" font-size="14.00">Kernels</text>
+</g>
+<!-- chameleon -->
+<g id="node1" class="node">
+<title>chameleon</title>
+<polygon fill="lightgrey" stroke="blue" stroke-width="2" points="453.5,-211 348.5,-211 348.5,-175 453.5,-175 453.5,-211"/>
+<text text-anchor="start" x="356.5" y="-190.3" font-family="Times,serif" font-weight="bold" font-size="14.00">Chameleon</text>
+</g>
+<!-- sched -->
+<g id="node2" class="node">
+<title>sched</title>
+<polygon fill="none" stroke="green" points="15.5,-99.5 15.5,-135.5 306.5,-135.5 306.5,-99.5 15.5,-99.5"/>
+<text text-anchor="middle" x="54.5" y="-113.8" font-family="Times,serif" font-size="14.00">OpenMP</text>
+<polyline fill="none" stroke="green" points="93.5,-99.5 93.5,-135.5 "/>
+<text text-anchor="middle" x="131" y="-113.8" font-family="Times,serif" font-size="14.00">PaRSEC</text>
+<polyline fill="none" stroke="green" points="168.5,-99.5 168.5,-135.5 "/>
+<text text-anchor="middle" x="203.5" y="-113.8" font-family="Times,serif" font-size="14.00">QUARK</text>
+<polyline fill="none" stroke="green" points="238.5,-99.5 238.5,-135.5 "/>
+<text text-anchor="middle" x="272.5" y="-113.8" font-family="Times,serif" font-size="14.00">StarPU</text>
+</g>
+<!-- chameleon->sched -->
+<g id="edge1" class="edge">
+<title>chameleon->sched</title>
+<path fill="none" stroke="black" d="M348.33,-175.87C340.64,-173.51 332.39,-170.99 323.81,-168.36"/>
+<polygon fill="black" stroke="black" points="324.58,-164.94 314,-165.36 322.53,-171.63 324.58,-164.94"/>
+</g>
+<!-- mpi -->
+<g id="node4" class="node">
+<title>mpi</title>
+<polygon fill="none" stroke="gold" stroke-width="2" points="241,-52 187,-52 187,-16 241,-16 241,-52"/>
+<text text-anchor="start" x="197" y="-31.3" font-family="Times,serif" font-weight="bold" font-size="14.00">MPI</text>
+</g>
+<!-- chameleon->mpi -->
+<g id="edge10" class="edge">
+<title>chameleon->mpi</title>
+<path fill="none" stroke="dimgrey" d="M359.38,-174.94C355.97,-172.58 352.78,-169.94 350,-167 324.84,-140.35 347.29,-113.02 318,-91 296.72,-75 280.81,-92.5 259.5,-86.95"/>
+<polygon fill="dimgrey" stroke="dimgrey" points="260.58,-83.61 250,-83 257.89,-90.07 260.58,-83.61"/>
+</g>
+<!-- cublas -->
+<g id="node5" class="node">
+<title>cublas</title>
+<polygon fill="none" stroke="red" stroke-width="2" points="439.5,-135.5 362.5,-135.5 362.5,-99.5 439.5,-99.5 439.5,-135.5"/>
+<text text-anchor="start" x="370.5" y="-114.8" font-family="Times,serif" font-weight="bold" font-size="14.00">cuBLAS</text>
+</g>
+<!-- chameleon->cublas -->
+<g id="edge11" class="edge">
+<title>chameleon->cublas</title>
+<path fill="none" stroke="black" d="M401,-174.95C401,-166.3 401,-155.57 401,-145.79"/>
+<polygon fill="black" stroke="black" points="404.5,-145.71 401,-135.71 397.5,-145.71 404.5,-145.71"/>
+</g>
+<!-- lapacke -->
+<g id="node6" class="node">
+<title>lapacke</title>
+<polygon fill="none" stroke="red" stroke-width="2" points="548.5,-135.5 457.5,-135.5 457.5,-99.5 548.5,-99.5 548.5,-135.5"/>
+<text text-anchor="start" x="465.5" y="-114.8" font-family="Times,serif" font-weight="bold" font-size="14.00">LAPACKE</text>
+</g>
+<!-- chameleon->lapacke -->
+<g id="edge9" class="edge">
+<title>chameleon->lapacke</title>
+<path fill="none" stroke="black" d="M436.21,-174.98C440.62,-172.47 444.98,-169.78 449,-167 459.25,-159.89 469.65,-151.03 478.55,-142.83"/>
+<polygon fill="black" stroke="black" points="481.07,-145.27 485.94,-135.86 476.27,-140.18 481.07,-145.27"/>
+</g>
+<!-- cblas -->
+<g id="node7" class="node">
+<title>cblas</title>
+<polygon fill="none" stroke="red" stroke-width="2" points="543,-52 473,-52 473,-16 543,-16 543,-52"/>
+<text text-anchor="start" x="481" y="-31.3" font-family="Times,serif" font-weight="bold" font-size="14.00">CBLAS</text>
+</g>
+<!-- chameleon->cblas -->
+<g id="edge8" class="edge">
+<title>chameleon->cblas</title>
+<path fill="none" stroke="black" d="M453.77,-190.72C494,-188.5 544.7,-182.59 558,-167 579.93,-141.31 570.04,-122.56 558,-91 553.46,-79.11 545.11,-68.14 536.52,-59.12"/>
+<polygon fill="black" stroke="black" points="538.9,-56.55 529.32,-52.03 533.98,-61.54 538.9,-56.55"/>
+</g>
+<!-- cuda -->
+<g id="node3" class="node">
+<title>cuda</title>
+<polygon fill="none" stroke="gold" stroke-width="2" points="321,-52 259,-52 259,-16 321,-16 321,-52"/>
+<text text-anchor="start" x="267" y="-31.3" font-family="Times,serif" font-weight="bold" font-size="14.00">CUDA</text>
+</g>
+<!-- sched->cuda -->
+<g id="edge2" class="edge">
+<title>sched:sched_starpu->cuda</title>
+<path fill="none" stroke="dimgrey" d="M273,-98.5C273,-86.18 276.02,-72.96 279.52,-61.79"/>
+<polygon fill="dimgrey" stroke="dimgrey" points="282.88,-62.78 282.79,-52.19 276.25,-60.52 282.88,-62.78"/>
+</g>
+<!-- sched->cuda -->
+<g id="edge4" class="edge">
+<title>sched:sched_parsec->cuda</title>
+<path fill="none" stroke="dimgrey" d="M131,-98.5C131,-71.83 226.63,-95.85 250,-83 259.7,-77.67 268.01,-68.97 274.52,-60.42"/>
+<polygon fill="dimgrey" stroke="dimgrey" points="277.5,-62.26 280.39,-52.07 271.78,-58.23 277.5,-62.26"/>
+</g>
+<!-- sched->mpi -->
+<g id="edge3" class="edge">
+<title>sched:sched_starpu->mpi</title>
+<path fill="none" stroke="dimgrey" d="M273,-98.5C273,-86.17 258.93,-91.49 250,-83 243.05,-76.4 236.45,-68.34 230.84,-60.73"/>
+<polygon fill="dimgrey" stroke="dimgrey" points="233.38,-58.26 224.75,-52.11 227.66,-62.3 233.38,-58.26"/>
+</g>
+<!-- sched->mpi -->
+<g id="edge5" class="edge">
+<title>sched:sched_parsec->mpi</title>
+<path fill="none" stroke="dimgrey" d="M131,-98.5C131,-73.42 155.21,-56.91 177.47,-47.02"/>
+<polygon fill="dimgrey" stroke="dimgrey" points="178.9,-50.22 186.82,-43.19 176.24,-43.75 178.9,-50.22"/>
+</g>
+<!-- cublas->cuda -->
+<g id="edge7" class="edge">
+<title>cublas->cuda</title>
+<path fill="none" stroke="black" d="M373.95,-99.4C366.12,-94.28 357.63,-88.54 350,-83 339.46,-75.35 328.23,-66.55 318.37,-58.59"/>
+<polygon fill="black" stroke="black" points="320.42,-55.75 310.45,-52.15 316,-61.18 320.42,-55.75"/>
+</g>
+<!-- lapacke->cblas -->
+<g id="edge6" class="edge">
+<title>lapacke->cblas</title>
+<path fill="none" stroke="black" d="M504.06,-99.22C504.72,-88.52 505.57,-74.55 506.32,-62.37"/>
+<polygon fill="black" stroke="black" points="509.82,-62.45 506.94,-52.26 502.84,-62.03 509.82,-62.45"/>
+</g>
+</g>
+</svg>
diff --git a/doc/orgmode/chapters/chameleon_header.png b/doc/user/chapters/chameleon_header.png
similarity index 100%
rename from doc/orgmode/chapters/chameleon_header.png
rename to doc/user/chapters/chameleon_header.png
diff --git a/doc/user/chapters/installing.org b/doc/user/chapters/installing.org
new file mode 100644
index 0000000000000000000000000000000000000000..4640ded4dd70e44dd94dfdf4c931c96ffc6a0318
--- /dev/null
+++ b/doc/user/chapters/installing.org
@@ -0,0 +1,821 @@
+# This file is part of the Chameleon User's Guide.
+# Copyright (C) 2017 Inria
+# See the file ../users_guide.org for copying conditions.
+
+Chameleon is written in C and depends on a couple of external
+libraries that must be installed on the system.
+# , it provides an interface to be called from Fortran
+
+Chameleon can be built and installed on UNIX systems (Linux) by the
+standard means of [[http://www.cmake.org/][CMake]]. General information about CMake, as well as
+installation binaries and CMake source code are available from [[http://www.cmake.org/cmake/resources/software.html][here]].
+
+To get support to install a full distribution Chameleon + dependencies
+we encourage users to use [[sec:ug:guix][GNU Guix]] or [[sec:ug:spack][Spack]].
+
+Finally some packages or also available for [[sec:ug:debian][Debian/Ubuntu]] and [[sec:ug:brew][MacOSX]].
+
+*** Getting Chameleon
+ <<sec:ug:git>>
+
+ The latest official release tarballs of Chameleon sources are
+ available for download from the [[https://gitlab.inria.fr/solverstack/chameleon/tags][gitlab tags page]].
+
+ The latest development state is available on [[https://gitlab.inria.fr/solverstack/chameleon][gitlab]]. You need [[https://git-scm.com/downloads][Git]]
+ #+begin_src
+ git clone --recursive https://gitlab.inria.fr/solverstack/chameleon.git
+ #+end_src
+
+*** Prerequisites for installing Chameleon
+ <<sec:ug:prereq>>
+
+ To install Chameleon's libraries, header files, and executables, one
+ needs:
+ - CMake (version 2.8 minimum): the build system
+ - C and Fortran compilers: GNU compiler suite, Clang, Intel or IBM
+ can be used
+ - python: to generate files in the different precisions
+ - external libraries: this depends on the configuration, by default
+ the required libraries are
+ - runtimes: [[http://runtime.bordeaux.inria.fr/StarPU/][StarPU]] or [[http://icl.utk.edu/parsec/][PaRSEC]] or [[http://icl.cs.utk.edu/quark/][QUARK]] or [[https://www.openmp.org/][OpenMP]]
+ - kernels : CBLAS, LAPACKE (with TMG). These are C interfaces to
+ Fortran kernels BLAS and LAPACK. There exist several providers
+ that can be used with Chameleon (Intel MKL, Netlib, OpenBLAS,
+ BLIS/FLAME)
+
+ Optional libraries:
+ - cuda: [[https://developer.nvidia.com/cuda-downloads][cuda]], [[http://docs.nvidia.com/cuda/cublas/][cublas]] (comes with cuda)
+ - mpi: [[http://www.open-mpi.org/][openmpi]], [[https://www.mpich.org/][mpich]], [[https://software.intel.com/en-us/mpi-library][intelmpi]]
+
+ These packages must be installed on the system before trying to
+ configure/build chameleon. Please look at the distrib/ directory
+ which gives some hints for the installation of dependencies for
+ Unix systems.
+
+ We give here some examples for a Debian system:
+ #+begin_src
+
+ # Update Debian packages list
+ sudo apt-get update
+ # Install BLAS/LAPACK, can be OpenBLAS, Intel MKL, Netlib LAPACK
+ sudo apt-get install -y libopenblas-dev liblapacke-dev
+ # or sudo apt-get install -y libmkl-dev
+ # or sudo apt-get install -y liblapack-dev liblapacke-dev
+ # Install OpenMPI
+ sudo apt-get install -y libopenmpi-dev
+ # Install StarPU
+ sudo apt-get install libstarpu-dev
+
+ # Optionnaly to make some specific developments, the following may be installed
+ # Install hwloc (used by StarPU or QUARK, already a dependency of OpenMPI)
+ sudo apt-get install -y libhwloc-dev
+ # install EZTrace, usefull to export some nice execution traces
+ with all runtimes
+ sudo apt-get install -y libeztrace-dev
+ # install FxT, usefull to export some nice execution traces with StarPU
+ sudo apt-get install -y libfxt-dev
+ # Install cuda and cuBLAS: only if you have a GPU cuda compatible
+ sudo apt-get install -y nvidia-cuda-toolkit nvidia-cuda-dev
+
+ # If you prefer a specific version of StarPU, install it yourself, e.g.
+ # Install StarPU (with MPI and FxT enabled)
+ mkdir -p $HOME/install
+ cd $HOME/install
+ wget https://files.inria.fr/starpu/starpu-1.3.7/starpu-1.3.7.tar.gz
+ tar xvzf starpu-1.3.7.tar.gz
+ cd starpu-1.3.7/
+ ./configure --prefix=/usr/local --with-fxt=/usr/lib/x86_64-linux-gnu/
+ make -j5
+ sudo make install
+
+ # Install PaRSEC: to be used in place of StarPU
+ mkdir -p $HOME/install
+ cd $HOME/install
+ git clone https://bitbucket.org/mfaverge/parsec.git
+ cd parsec
+ git checkout mymaster
+ git submodule update
+ mkdir -p build
+ cd build
+ cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_SHARED_LIBS=ON
+ make -j5
+ sudo make install
+
+ # Install QUARK: to be used in place of StarPU
+ mkdir -p $HOME/install
+ cd $HOME/install
+ git clone https://github.com/ecrc/quark
+ cd quark/
+ sed -i -e "s#prefix=.*#prefix=/usr/local#g" make.inc
+ sed -i -e "s#CFLAGS=.*#CFLAGS= -O2 -DADD_ -fPIC#g" make.inc
+ make
+ sudo make install
+ #+end_src
+
+ See also our script example in the [[https://gitlab.inria.fr/solverstack/chameleon/-/blob/master/distrib/debian/install_dependencies.sh][distrib/debian]] sub-directory.
+
+**** Known issues
+ - we need the lapacke interface to tmg routines and symbol like
+ ~LAPACKE_dlatms_work~ should be defined in the lapacke
+ library. Make sure the Debian packages /libopenblas-dev/ and
+ /liblapacke-dev/ (no problem with Intel MKL) do provide the tmg
+ interface. If not you can possibly update your distribution or
+ install the lapacke interface library in another way, by
+ yourself from source or with [[https://gitlab.inria.fr/solverstack/spack-repo][Spack]], or with [[https://gitlab.inria.fr/guix-hpc/guix-hpc-non-free][Guix-HPC]],...
+**** Some details about dependencies
+***** BLAS implementation
+ [[http://www.netlib.org/blas/][BLAS]] (Basic Linear Algebra Subprograms), are a de facto standard
+ for basic linear algebra operations such as vector and matrix
+ multiplication. FORTRAN implementation of BLAS is available from
+ Netlib. Also, C implementation of BLAS is included in GSL (GNU
+ Scientific Library). Both these implementations are reference
+ implementation of BLAS, are not optimized for modern processor
+ architectures and provide an order of magnitude lower performance
+ than optimized implementations. Highly optimized implementations
+ of BLAS are available from many hardware vendors, such as Intel
+ MKL, IBM ESSL and AMD ACML. Fast implementations are also
+ available as academic packages, such as ATLAS and OpenBLAS. The
+ standard interface to BLAS is the FORTRAN interface.
+
+ *Caution about the compatibility:* Chameleon has been mainly tested
+ with the reference BLAS from NETLIB, OpenBLAS and Intel MKL.
+***** CBLAS
+ [[http://www.netlib.org/blas/#_cblas][CBLAS]] is a C language interface to BLAS. Most commercial and
+ academic implementations of BLAS also provide CBLAS. Netlib
+ provides a reference implementation of CBLAS on top of FORTRAN
+ BLAS (Netlib CBLAS). Since GSL is implemented in C, it naturally
+ provides CBLAS.
+
+ *Caution about the compatibility:* Chameleon has been mainly tested with
+ the reference CBLAS from NETLIB, OpenBLAS and Intel MKL.
+***** LAPACK implementation
+ [[http://www.netlib.org/lapack/][LAPACK]] (Linear Algebra PACKage) is a software library for
+ numerical linear algebra, a successor of LINPACK and EISPACK and
+ a predecessor of Chameleon. LAPACK provides routines for solving
+ linear systems of equations, linear least square problems,
+ eigenvalue problems and singular value problems. Most commercial
+ and academic BLAS packages also provide some LAPACK routines.
+
+ *Caution about the compatibility:* Chameleon has been mainly tested
+ with the reference LAPACK from NETLIB, OpenBLAS and Intel MKL.
+***** LAPACKE
+ [[http://www.netlib.org/lapack/][LAPACKE]] is a C language interface to LAPACK (or CLAPACK). It is
+ produced by Intel in coordination with the LAPACK team and is
+ available in source code from Netlib in its original version
+ (Netlib LAPACKE) and from Chameleon website in an extended
+ version (LAPACKE for Chameleon). In addition to implementing the
+ C interface, LAPACKE also provides routines which automatically
+ handle workspace allocation, making the use of LAPACK much more
+ convenient.
+
+ *Caution about the compatibility:* Chameleon has been mainly tested
+ with the reference LAPACKE from NETLIB, OpenBLAS and Intel
+ MKL. In addition the LAPACKE library *must* be configured to
+ provide the interface with the TMG routines and symbols like
+ ~LAPACKE_dlatms_work~ should be defined.
+***** libtmg
+ [[http://www.netlib.org/lapack/][libtmg]] is a component of the LAPACK library, containing routines
+ for generation of input matrices for testing and timing of
+ LAPACK. The testing and timing suites of LAPACK require libtmg,
+ but not the library itself. Note that the LAPACK library can be
+ built and used without libtmg.
+
+ *Caution about the compatibility:* Chameleon has been mainly tested
+ with the reference TMGLIB from NETLIB, OpenBLAS and Intel MKL.
+***** StarPU
+ [[http://runtime.bordeaux.inria.fr/StarPU/][StarPU]] is a task programming library for hybrid architectures.
+ StarPU handles run-time concerns such as:
+ * Task dependencies
+ * Optimized heterogeneous scheduling
+ * Optimized data transfers and replication between main memory
+ and discrete memories
+ * Optimized cluster communications
+
+ StarPU can be used to benefit from GPUs and distributed-memory
+ environment. Note StarPU is enabled by default.
+
+ *Caution about the compatibility:* Chameleon has been mainly tested
+ with StarPU-1.1, 1.2 and 1.3 releases.
+***** PaRSEC
+ [[http://icl.utk.edu/parsec/][PaRSEC]] is a generic framework for architecture aware scheduling
+ and management of micro-tasks on distributed many-core
+ heterogeneous architectures. It can be used with MPI and Cuda.
+
+ *Caution about the compatibility:* Chameleon is compatible with
+ this version
+ https://bitbucket.org/mfaverge/parsec/branch/mymaster.
+***** QUARK
+ [[http://icl.cs.utk.edu/quark/][QUARK]] (QUeuing And Runtime for Kernels) provides a library that
+ enables the dynamic execution of tasks with data dependencies in
+ a multi-core, multi-socket, shared-memory environment. When
+ Chameleon is linked with QUARK, it is not possible to exploit
+ neither CUDA (for GPUs) nor MPI (distributed-memory environment).
+ You can use PaRSEC or StarPU to do so.
+
+ *Caution about the compatibility:* Chameleon has been mainly tested
+ with the QUARK library coming from https://github.com/ecrc/quark.
+***** EZTrace
+ This library provides efficient modules for recording
+ traces. Chameleon can trace kernels execution on CPU workers
+ thanks to EZTrace and produce .paje files. EZTrace also provides
+ integrated modules to trace MPI calls and/or memory usage. See
+ how to use this feature here [[sec:trace_ezt][Execution trace using EZTrace]]. To
+ trace kernels execution on all kind of workers, such as CUDA, We
+ recommend to use the internal tracing support of the runtime
+ system used done by the underlying runtime. See how to use this
+ feature here [[sec:trace_fxt][Execution trace using StarPU/FxT]].
+***** hwloc
+ [[http://www.open-mpi.org/projects/hwloc/][hwloc]] (Portable Hardware Locality) is a software package for
+ accessing the topology of a multicore system including components
+ like: cores, sockets, caches and NUMA nodes. The topology
+ discovery library, ~hwloc~, is strongly recommended to be used
+ through the runtime system. It allows to increase performance,
+ and to perform some topology aware scheduling. ~hwloc~ is available
+ in major distributions and for most OSes and can be downloaded
+ from http://www.open-mpi.org/software/hwloc.
+
+ *Caution about the compatibility:* hwloc should be compatible with
+ the runtime system used.
+***** OpenMPI
+ [[http://www.open-mpi.org/][OpenMPI]] is an open source Message Passing Interface
+ implementation for execution on multiple nodes with
+ distributed-memory environment. MPI can be enabled only if the
+ runtime system chosen is StarPU (default). To use MPI through
+ StarPU, it is necessary to compile StarPU with MPI enabled.
+
+ *Caution about the compatibility:* OpenMPI should be built with the
+ --enable-mpi-thread-multiple option.
+***** Nvidia CUDA Toolkit
+ [[https://developer.nvidia.com/cuda-toolkit][Nvidia CUDA Toolkit]] provides a comprehensive development
+ environment for C and C++ developers building GPU-accelerated
+ applications. Chameleon can use a set of low level optimized
+ kernels coming from cuBLAS to accelerate computations on GPUs.
+ The [[http://docs.nvidia.com/cuda/cublas/][cuBLAS]] library is an implementation of BLAS (Basic Linear
+ Algebra Subprograms) on top of the Nvidia CUDA runtime. cuBLAS
+ is normaly distributed with Nvidia CUDA Toolkit. CUDA/cuBLAS can
+ be enabled in Chameleon only if the runtime system chosen is
+ StarPU (default). To use CUDA through StarPU, it is necessary to
+ compile StarPU with CUDA enabled.
+
+ *Caution about the compatibility:* your compiler must be compatible
+ with CUDA.
+*** Build and install Chameleon with CMake
+ <<sec:ug:cmake>>
+ Compilation of Chameleon libraries and executables are done with
+ CMake (http://www.cmake.org/). This version has been tested with
+ CMake 3.10.2 but any version superior to 2.8 should be fine.
+
+ Here the steps to configure, build, test and install
+ 1. configure:
+ #+begin_src
+ cmake path/to/chameleon -DOPTION1= -DOPTION2= ...
+ # see the "Configuration options" section to get list of options
+ # see the "Dependencies detection" for details about libraries detection
+ #+end_src
+ 2. build:
+ #+begin_src
+ make
+ # do not hesitate to use -j[ncores] option to speedup the compilation
+ #+end_src
+ 3. test (optional, required CHAMELEON_ENABLE_TESTING=ON):
+ #+begin_src
+ make test
+ # or
+ ctest
+ #+end_src
+ 4. install (optional):
+ #+begin_src
+ make install
+ #+end_src
+ Do not forget to specify the install directory with
+ *-DCMAKE_INSTALL_PREFIX* at configure.
+ #+begin_example
+ cmake /home/jdoe/chameleon -DCMAKE_INSTALL_PREFIX=/home/jdoe/install/chameleon
+ #+end_example
+ Note that the install process is optional. You are free to use
+ Chameleon binaries compiled in the build directory.
+**** Configuration options
+ You can optionally activate some options at cmake configure (like CUDA, MPI, ...)
+ invoking ~cmake path/to/your/CMakeLists.txt -DOPTION1= -DOPTION2= ...~
+ #+begin_src
+ cmake /home/jdoe/chameleon/ -DCMAKE_BUILD_TYPE=Debug \
+ -DCMAKE_INSTALL_PREFIX=/home/jdoe/install/ \
+ -DCHAMELEON_USE_CUDA=ON \
+ -DCHAMELEON_USE_MPI=ON \
+ -DBLA_VENDOR=Intel10_64lp \
+ -DCHAMELEON_ENABLE_TRACING=ON
+ #+end_src
+
+ You can get the full list of options with *-L[A][H]* options of cmake command
+ #+begin_src
+ cmake -LH /home/jdoe/chameleon/
+ #+end_src
+
+ You can also set the options thanks to the *ccmake* interface.
+
+***** Native CMake options (non-exhaustive list)
+ * *CMAKE_BUILD_TYPE=Debug|Release|RelWithDebInfo|MinSizeRel*:
+ level of compiler optimization, enable/disable debug
+ information
+ * *CMAKE_INSTALL_PREFIX=path/to/your/install/dir*: where headers,
+ libraries, executables, etc, will be copied when invoking make
+ install
+ * *BUILD_SHARED_LIBS=ON|OFF*: indicate wether or not CMake has to
+ build Chameleon static (~OFF~) or shared (~ON~) libraries.
+ * *CMAKE_C_COMPILER=gcc|icc|...*: to choose the C compilers
+ if several exist in the environment
+ * *CMAKE_Fortran_COMPILER=gfortran|ifort|...*: to choose the
+ Fortran compilers if several exist in the environment
+
+***** Related to specific modules (find_package) to find external libraries
+ * *BLA_VENDOR=All|OpenBLAS|Generic|Intel10_64lp|Intel10_64lp_seq|FLAME*:
+ to use intel mkl for example, see the list of BLA_VENDOR in
+ FindBLAS.cmake in cmake_modules/morse_cmake/modules/find
+
+ Libraries detected with an official cmake module (see module
+ files in CMAKE_ROOT/Modules/): BLAS - LAPACK - CUDA - MPI -
+ OpenMP - Threads.
+
+ Libraries detected with our cmake modules (see module files in
+ cmake_modules/morse_cmake/modules/find/ directory of Chameleon
+ sources): CBLAS - EZTRACE - FXT - HWLOC - LAPACKE - PARSEC -
+ QUARK - SIMGRID - STARPU.
+
+***** Chameleon specific options
+ * *CHAMELEON_SCHED=STARPU|PARSEC|QUARK|OPENMP* (default STARPU): to
+ link respectively with StarPU, PaRSEC, Quark, OpenMP library
+ (runtime system)
+ * *CHAMELEON_USE_MPI=ON|OFF* (default OFF): to link with MPI
+ library (message passing implementation for use of multiple
+ nodes with distributed memory), can only be used with StarPU
+ and PaRSEC
+ * *CHAMELEON_USE_CUDA=ON|OFF* (default OFF): to link with CUDA
+ runtime (implementation paradigm for accelerated codes on GPUs)
+ and cuBLAS library (optimized BLAS kernels on GPUs), can only
+ be used with StarPU and PaRSEC
+ * *CHAMELEON_ENABLE_DOC=ON|OFF* (default OFF): to control build of
+ the documentation contained in doc/ sub-directory
+ * *CHAMELEON_ENABLE_EXAMPLE=ON|OFF* (default ON): to control build
+ of the examples executables (API usage) contained in example/
+ sub-directory
+ * *CHAMELEON_ENABLE_PRUNING_STATS=ON|OFF* (default OFF)
+ * *CHAMELEON_ENABLE_TESTING=ON|OFF* (default ON): to control build
+ of testing executables (timer and numerical check) contained in testing/
+ sub-directory
+ * *CHAMELEON_SIMULATION=ON|OFF* (default OFF): to enable
+ simulation mode, means Chameleon will not really execute tasks,
+ see details in section [[sec:simu][Use simulation mode with
+ StarPU-SimGrid]]. This option must be used with StarPU compiled
+ with [[http://simgrid.gforge.inria.fr/][SimGrid]] allowing to guess the execution time on any
+ architecture. This feature should be used to make experiments
+ on the scheduler behaviors and performances not to produce
+ solutions of linear systems.
+ * *CHAMELEON_USE_MIGRATE=ON|OFF* (default OFF): enables the data
+ migration in QR algorithms
+
+**** Dependencies detection
+ <<sec:depdet>>
+
+ You have different choices to detect dependencies on your system,
+ either by setting some environment variables containing paths to
+ the libs and headers or by specifying them directly at cmake
+ configure. Different cases:
+
+ 1) detection of dependencies through environment variables:
+ - LD_LIBRARY_PATH (DYLD_LIBRARY_PATH on Mac OSX) should contain
+ the list of paths where to find the libraries:
+ #+begin_src
+ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:install/path/to/your/lib
+ #+end_src
+ - INCLUDE (or CPATH, or C_INCLUDE_PATH)should contain the list
+ of paths where to find the header files of libraries
+ #+begin_src
+ export INCLUDE=$INCLUDE:install/path/to/your/headers
+ #+end_src
+ 2) detection with user's given paths:
+ - you can specify the path at cmake configure by invoking
+ #+begin_example
+ cmake path/to/your/CMakeLists.txt -DLIB_DIR=path/to/your/lib
+ #+end_example
+ where LIB stands for the name of the lib to look for, /e.g./
+ #+begin_src
+ cmake path/to/your/CMakeLists.txt -DQUARK_DIR=path/to/quarkdir \
+ -DCBLAS_DIR= ...
+ #+end_src
+ it is also possible to specify headers and library directories
+ separately
+ #+begin_src
+ cmake path/to/your/CMakeLists.txt \
+ -DQUARK_INCDIR=path/to/quark/include \
+ -DQUARK_LIBDIR=path/to/quark/lib
+ #+end_src
+ 3) detection with custom environment variables: all variables like
+ _DIR, _INCDIR, _LIBDIR can be set as environment variables
+ instead of CMake options, there will be read
+ 4) using [[https://www.freedesktop.org/wiki/Software/pkg-config/][pkg-config]] for libraries that provide .pc files
+ - update your *PKG_CONFIG_PATH* to the paths where to find .pc
+ files of installed external libraries like hwloc, starpu, some
+ blas/lapack, etc
+
+ Note that PaRSEC and StarPU are only detected with pkg-config
+ mechanism because it is always provided and this avoids errors.
+*** Distribution Debian
+ <<sec:ug:debian>>
+ Download one of the available package for your distribution here
+ https://gitlab.inria.fr/solverstack/chameleon/-/packages, then
+ install as follows
+ #+begin_src sh
+ sudo apt-get install ./chameleon_1.1.0-1_amd64.deb -y
+ #+end_src
+ Chameleon will be installed on your system meaning you can use
+ drivers for performance tests
+ #+begin_src sh
+ mpiexec -n 2 chameleon_stesting -t 2 -o gemm -n 1000
+ #+end_src
+ and use Chameleon library in your own project
+ #+begin_src sh
+ # example usage: use chameleon library in your own cmake project (we provide a CHAMELEONConfig.cmake)
+ git clone https://gitlab.inria.fr/solverstack/distrib.git
+ cd distrib/cmake/test/chameleon && mkdir build && cd build && cmake .. && make && ./test_chameleon
+
+ # example usage: use chameleon library in your own not cmake project
+ # use pkg-config to get compiler flags and linking
+ pkg-config --cflags chameleon
+ pkg-config --libs chameleon
+ # if there are static libraries use the --static option of pkg-config
+ #+end_src
+
+ Do not hesitate to send an [[mailto:florent.pruvost@inria.fr][email]] if you need a package for your
+ Debian distribution.
+*** Distribution of Chameleon using GNU Guix
+ <<sec:ug:guix>>
+
+ We provide [[http://guix.gnu.org/][Guix]] packages to install Chameleon with its dependencies
+ in a reproducible way on GNU/Linux systems. For MacOSX please refer
+ to the next sections about Brew or Spack packaging.
+
+ If you are "root" on the system you can install Guix and directly
+ use it to install the libraries. On supercomputers your are not
+ root on you may still be able to use it if Docker or Singularity
+ are available on the machine because Chameleon can be packaged as
+ Docker/Singularity images with Guix.
+
+**** Installing Guix
+ Guix requires a running GNU/Linux system, GNU tar and Xz.
+
+ #+begin_src sh
+ gpg --keyserver pgp.mit.edu --recv-keys 3CE464558A84FDC69DB40CFB090B11993D9AEBB5
+ wget https://git.savannah.gnu.org/cgit/guix.git/plain/etc/guix-install.sh
+ chmod +x guix-install.sh
+ sudo ./guix-install.sh
+ #+end_src
+
+ The Chameleon packages are not official Guix packages. It is then
+ necessary to add a channel to get additional packages. Create a
+ ~/.config/guix/channels.scm file with the following snippet:
+ #+begin_example
+ (cons (channel
+ (name 'guix-hpc-non-free)
+ (url "https://gitlab.inria.fr/guix-hpc/guix-hpc-non-free.git"))
+ %default-channels)
+ #+end_example
+
+ Update guix package definition
+ #+begin_src sh
+ guix pull
+ #+end_src
+
+ Update new guix in the path
+ #+begin_src sh
+ PATH="$HOME/.config/guix/current/bin${PATH:+:}$PATH"
+ hash guix
+ #+end_src
+
+ For further shell sessions, add this to the ~/.bash_profile file
+ #+begin_example
+ export PATH="$HOME/.config/guix/current/bin${PATH:+:}$PATH"
+ export GUIX_LOCPATH="$HOME/.guix-profile/lib/locale"
+ #+end_example
+
+ Chameleon packages are now available
+ #+begin_src sh
+ guix search ^chameleon
+ #+end_src
+
+ Refer to the [[https://guix.gnu.org/manual/en/guix.html#Invoking-guix-package][official documentation of Guix]] to learn the basic
+ commands.
+
+**** Installing Chameleon with Guix
+
+ Standard Chameleon, last release
+ #+begin_src sh
+ guix install chameleon
+ #+end_src
+
+ Notice that there exist several build variants
+ - chameleon (default) : with starpu - with mpi - with OpenBlas
+ - chameleon-mkl-mt : default version but with Intel MKL multithreaded to replace OpenBlas
+ - chameleon-cuda : with starpu - with mpi - with cuda
+ - chameleon-cuda-mkl-mt : with starpu - with mpi - with cuda - with Intel MKL multithreaded to replace OpenBlas
+ - chameleon-simgrid : with starpu - with mpi - with simgrid
+ - chameleon-openmp : with openmp - without mpi
+ - chameleon-parsec : with parsec - without mpi
+ - chameleon-quark : with quark - without mpi
+
+
+ Change the version
+ #+begin_src sh
+ guix install chameleon --with-branch=chameleon=master
+ guix install chameleon --with-commit=chameleon=b31d7575fb7d9c0e1ba2d8ec633e16cb83778e8b
+ guix install chameleon --with-git-url=chameleon=https://gitlab.inria.fr/fpruvost/chameleon.git
+ guix install chameleon --with-git-url=chameleon=$HOME/git/chameleon
+ #+end_src
+
+ Notice also that default mpi is OpenMPI and default blas/lapack is
+ Openblas. This can be changed with a [[https://guix.gnu.org/manual/en/guix.html#Package-Transformation-Options][transformation option]].
+
+ Change some dependencies
+ #+begin_src sh
+ # install chameleon with intel mkl to replace openblas, nmad to replace openmpi and starpu with fxt
+ guix install chameleon --with-input=openblas=mkl --with-input=openmpi=nmad --with-input=starpu=starpu-fxt
+ #+end_src
+
+**** Generate a Chameleon Docker image with Guix
+
+ To install Chameleon and its dependencies within a docker image
+ (OpenMPI stack)
+ #+begin_src sh
+ docker_chameleon=`guix pack -f docker chameleon chameleon --with-branch=chameleon=master --with-input=openblas=mkl mkl starpu hwloc openmpi openssh slurm bash coreutils inetutils util-linux procps git grep tar sed gzip which gawk perl emacs-minimal vim gcc-toolchain make cmake pkg-config -S /bin=bin --entry-point=/bin/bash`
+ # Load the generated tarball as a docker image
+ docker_chameleon_tag=`docker load --input $docker_chameleon | grep "Loaded image: " | cut -d " " -f 3-`
+ # Change tag name, see the existing image name with "docker images" command, then change to a more simple name
+ docker tag $docker_chameleon_tag guix/chameleon-tmp
+ #+end_src
+
+ Create a Dockerfile inheriting from the image (renamed
+ =guix/chameleon= here):
+
+ #+begin_src sh :eval no :tangle Dockerfile :export none
+ FROM guix/chameleon-tmp
+
+ # Create a directory for user 1000
+ RUN mkdir -p /builds
+ RUN chown -R 1000 /builds
+
+ ENTRYPOINT ["/bin/bash", "-l"]
+
+ # Enter the image as user 1000 in /builds
+ USER 1000
+ WORKDIR /builds
+ ENV HOME /builds
+ #+end_src
+
+ Then create the final docker image from this docker file.
+
+ #+begin_src sh
+ docker build -t guix/chameleon .
+ #+end_src
+
+ Test the image
+ #+begin_src sh
+ docker run -it guix/chameleon
+ # test starpu
+ STARPU=`pkg-config --variable=prefix libstarpu`
+ mpiexec -np 4 $STARPU/lib/starpu/mpi/comm
+ # test chameleon
+ CHAMELEON=`pkg-config --variable=prefix chameleon`
+ mpiexec -np 2 $CHAMELEON/bin/chameleon_stesting -H -o gemm -P 2 -t 2 -m 2000 -n 2000 -k 2000
+ #+end_src
+
+**** Generate a Chameleon Singularity image with Guix
+
+ To package Chameleon and its dependencies within a singularity image
+ (OpenMPI stack)
+ #+begin_src sh
+ singularity_chameleon=`guix pack -f squashfs chameleon --with-branch=chameleon=master --with-input=openblas=mkl mkl starpu hwloc openmpi openssh slurm hdf5 zlib bash coreutils inetutils util-linux procps git grep tar sed gzip which gawk perl emacs-minimal vim gcc-toolchain make cmake pkg-config -S /bin=bin --entry-point=/bin/bash`
+ cp $singularity_chameleon chameleon-pack.gz.squashfs
+ # copy the singularity image on the supercomputer, e.g. 'supercomputer'
+ scp chameleon-pack.gz.squashfs supercomputer:
+ #+end_src
+
+ On a machine where Singularity is installed Chameleon can then be
+ called as follows
+ #+begin_src sh
+ # at least openmpi and singularity are required here, e.g. module add openmpi singularity
+ mpiexec -np 2 singularity exec chameleon-pack.gz.squashfs /bin/chameleon_stesting -H -o gemm -P 2 -t 2 -m 2000 -n 2000 -k 2000
+ #+end_src
+
+*** Distribution of Chameleon using Spack
+ <<sec:ug:spack>>
+
+**** Installing Spack
+
+ To get support to install a full distribution on Linux or MacOS X,
+ Chameleon plus dependencies, we encourage users to use
+ [[https://spack.io/][Spack]]. Please refer to our [[https://gitlab.inria.fr/solverstack/spack-repo/blob/master/README.org][Spack Repository]].
+
+ #+begin_src sh
+ git clone https://github.com/llnl/spack.git
+ export SPACK_ROOT=$PWD/spack
+ cd spack/
+ git checkout v0.16.0
+ . $SPACK_ROOT/share/spack/setup-env.sh
+ git clone https://gitlab.inria.fr/solverstack/spack-repo.git ./var/spack/repos/solverstack
+ spack repo add ./var/spack/repos/solverstack
+ #+end_src
+
+ Chameleon is then available
+ #+begin_src sh
+ spack info chameleon
+ spack spec chameleon
+ #+end_src
+
+ Refer to te [[https://spack.readthedocs.io/en/latest/basic_usage.html][official documentation of Spack]] to learn the basic
+ commands.
+
+**** Installing Chameleon with Spack
+
+ Standard Chameleon, last state on the 'master' branch
+ #+begin_src sh
+ spack install -v chameleon
+ # chameleon is installed here:
+ spack location -i chameleon
+ #+end_src
+
+ Notice that there exist several build variants
+ - chameleon (default) : with starpu - with mpi
+ - tune the build type (CMake) with build_type=RelWithDebInfo|Debug|Release
+ - enable/disable shared libraries with +/- shared
+ - enable/disable mpi with +/- mpi
+ - enable/disable cuda with +/- cuda
+ - enable/disable fxt with +/- fxt
+ - enable/disable simgrid with +/- simgrid
+ - +openmp~starpu : with openmp - without starpu
+ - +quark~starpu : with quark - without starpu
+
+ Change the version
+ #+begin_src sh
+ spack install -v chameleon@1.0.0
+ #+end_src
+
+ Notice also that default mpi is OpenMPI and default blas/lapack is
+ Openblas. This can be changed by adding some [[https://spack.readthedocs.io/en/latest/basic_usage.html#constraining-virtual-packages][constraints on
+ virtual packages]].
+
+ Change some dependencies
+ #+begin_src sh
+ # see lapack providers
+ spack providers lapack
+ # see mpi providers
+ spack providers mpi
+ # install chameleon with intel mkl to replace openblas
+ spack install -v chameleon ^intel-mkl
+ # install chameleon with nmad to replace openmpi
+ spack install -v chameleon ^nmad
+ #+end_src
+*** Distribution Brew for Mac OS X
+ <<sec:ug:brew>>
+
+ We provide some [[https://brew.sh/][brew]] packages here
+ https://gitlab.inria.fr/solverstack/brew-repo (under construction).
+
+*** Linking an external application with Chameleon libraries
+ <<sec:ug:link>>
+ Compilation and link with Chameleon libraries have been tested with
+ the GNU compiler suite ~gcc/gfortran~ and the Intel compiler suite
+ ~icc/ifort~.
+
+**** For CMake projects
+ A CHAMELEONConfig.cmake file is provided at installation, stored
+ in <prefix>/lib/cmake/chameleon, so that users in cmake project
+ can use through the variable CHAMELEON_ROOT (set it as environment
+ or CMake variable).
+
+ #+begin_src
+ sudo apt-get update
+ sudo apt-get install -y libopenblas-dev liblapacke-dev libstarpu-dev
+ git clone --recursive https://gitlab.inria.fr/solverstack/chameleon.git
+ cd chameleon && mkdir -p build && cd build
+ CHAMELEON_ROOT=$PWD/install
+ cmake .. -DCMAKE_INSTALL_PREFIX=$CHAMELEON_ROOT && make -j5 install
+ # chameleon is installed in $CHAMELEON_ROOT
+
+ # if your work in a cmake project you can use the CHAMELEONConfig.cmake file
+ # installed under <prefix>/lib/cmake/chameleon/ by setting your
+ # CMAKE_PREFIX_PATH with the path of installation. In your cmake project, use
+ # find_package(CHAMELEON) and link your libraries and/or executables with the
+ # library target MORSE::CHAMELEON
+ cmake . -DCMAKE_PREFIX_PATH=$CHAMELEON_ROOT
+ #+end_src
+**** For non CMake projects
+ The compiler, linker flags that are necessary to build an
+ application using Chameleon are given through the [[https://www.freedesktop.org/wiki/Software/pkg-config/][pkg-config]]
+ mechanism.
+ #+begin_src
+ sudo apt-get update
+ sudo apt-get install -y libopenblas-dev liblapacke-dev libstarpu-dev
+ git clone --recursive https://gitlab.inria.fr/solverstack/chameleon.git
+ cd chameleon && mkdir -p build && cd build
+ CHAMELEON_ROOT=$PWD/install
+ cmake .. -DCMAKE_INSTALL_PREFIX=$CHAMELEON_ROOT && make -j5 install
+ # chameleon is installed in $CHAMELEON_ROOT
+
+ export PKG_CONFIG_PATH=$CHAMELEON_ROOT/lib/pkgconfig:$PKG_CONFIG_PATH
+ pkg-config --cflags chameleon
+ pkg-config --libs chameleon
+ pkg-config --libs --static chameleon
+
+ # use it in your configure/make
+ #+end_src
+ The .pc files required are located in the sub-directory
+ ~lib/pkgconfig~ of your Chameleon install directory.
+
+**** Static linking in C
+ Lets imagine you have a file ~main.c~ that you want to link with
+ Chameleon static libraries. Lets consider
+ ~/home/yourname/install/chameleon~ is the install directory
+ of Chameleon containing sub-directories ~include/~ and
+ ~lib/~. Here could be your compilation command with gcc
+ compiler:
+ #+begin_src
+ gcc -I/home/yourname/install/chameleon/include -o main.o -c main.c
+ #+end_src
+ Now if you want to link your application with Chameleon static libraries, you
+ could do:
+ #+begin_src
+ gcc main.o -o main \
+ /home/yourname/install/chameleon/lib/libchameleon.a \
+ /home/yourname/install/chameleon/lib/libchameleon_starpu.a \
+ /home/yourname/install/chameleon/lib/libcoreblas.a \
+ -lstarpu-1.2 -Wl,--no-as-needed -lmkl_intel_lp64 \
+ -lmkl_sequential -lmkl_core -lpthread -lm -lrt
+ #+end_src
+ As you can see in this example, we also link with some dynamic
+ libraries *starpu-1.2*, *Intel MKL* libraries (for
+ BLAS/LAPACK/CBLAS/LAPACKE), *pthread*, *m* (math) and *rt*. These
+ libraries will depend on the configuration of your Chameleon
+ build. You can find these dependencies in .pc files we generate
+ during compilation and that are installed in the sub-directory
+ ~lib/pkgconfig~ of your Chameleon install directory. Note also that
+ you could need to specify where to find these libraries with *-L*
+ option of your compiler/linker.
+
+ Before to run your program, make sure that all shared libraries
+ paths your executable depends on are known. Enter ~ldd main~
+ to check. If some shared libraries paths are missing append them
+ in the LD_LIBRARY_PATH (for Linux systems) environment
+ variable (DYLD_LIBRARY_PATH on Mac).
+
+**** Dynamic linking in C
+ For dynamic linking (need to build Chameleon with CMake option
+ BUILD_SHARED_LIBS=ON) it is similar to static compilation/link but
+ instead of specifying path to your static libraries you indicate
+ the path to dynamic libraries with *-L* option and you give
+ the name of libraries with *-l* option like this:
+ #+begin_src
+ gcc main.o -o main \
+ -L/home/yourname/install/chameleon/lib \
+ -lchameleon -lchameleon_starpu -lcoreblas \
+ -lstarpu-1.2 -Wl,--no-as-needed -lmkl_intel_lp64 \
+ -lmkl_sequential -lmkl_core -lpthread -lm -lrt
+ #+end_src
+ Note that an update of your environment variable LD_LIBRARY_PATH
+ (DYLD_LIBRARY_PATH on Mac) with the path of the libraries could be
+ required before executing
+ #+begin_src
+ export LD_LIBRARY_PATH=path/to/libs:path/to/chameleon/lib
+ #+end_src
+
+ # # *** Build a Fortran program with Chameleon :noexport:
+ # #
+ # # Chameleon provides a Fortran interface to user functions. Example:
+ # # #+begin_src
+ # # call chameleon_version(major, minor, patch) !or
+ # # call CHAMELEON_VERSION(major, minor, patch)
+ # # #+end_src
+ # #
+ # # Build and link are very similar to the C case.
+ # #
+ # # Compilation example:
+ # # #+begin_src
+ # # gfortran -o main.o -c main.f90
+ # # #+end_src
+ # #
+ # # Static linking example:
+ # # #+begin_src
+ # # gfortran main.o -o main \
+ # # /home/yourname/install/chameleon/lib/libchameleon.a \
+ # # /home/yourname/install/chameleon/lib/libchameleon_starpu.a \
+ # # /home/yourname/install/chameleon/lib/libcoreblas.a \
+ # # -lstarpu-1.2 -Wl,--no-as-needed -lmkl_intel_lp64 \
+ # # -lmkl_sequential -lmkl_core -lpthread -lm -lrt
+ # # #+end_src
+ # #
+ # # Dynamic linking example:
+ # # #+begin_src
+ # # gfortran main.o -o main \
+ # # -L/home/yourname/install/chameleon/lib \
+ # # -lchameleon -lchameleon_starpu -lcoreblas \
+ # # -lstarpu-1.2 -Wl,--no-as-needed -lmkl_intel_lp64 \
+ # # -lmkl_sequential -lmkl_core -lpthread -lm -lrt
+ # # #+end_src
diff --git a/doc/user/chapters/introduction.org b/doc/user/chapters/introduction.org
new file mode 100644
index 0000000000000000000000000000000000000000..a1fbc023b9e9b934b45ea5a51a4d19c33cb8ea99
--- /dev/null
+++ b/doc/user/chapters/introduction.org
@@ -0,0 +1,310 @@
+# This file is part of the Chameleon User's Guide.
+# Copyright (C) 2018 Inria
+# See the file ../users_guide.org for copying conditions.
+*** MORSE project
+ #+NAME: fig:chameleon_header
+ #+ATTR_HTML: :align center
+ [[file:chameleon_header.png]]
+
+ Chameleon is a linear algebra software created jointly by several
+ research teams as part of the MORSE associate team: [[http://www.icl.utk.edu/][ICL]], [[https://www.inria.fr/en/][Inria]],
+ [[https://www.kaust.edu.sa/en][KAUST]], [[http://www.ucdenver.edu/pages/ucdwelcomepage.aspx][The University of Colorado Denver]].
+
+**** MORSE Objectives
+ When processor clock speeds flatlined in 2004, after more than
+ fifteen years of exponential increases, the era of near automatic
+ performance improvements that the HPC application community had
+ previously enjoyed came to an abrupt end. To develop software that
+ will perform well on petascale and exascale systems with thousands
+ of nodes and millions of cores, the list of major challenges that
+ must now be confronted is formidable:
+ 1) dramatic escalation in the costs of intrasystem communication
+ between processors and/or levels of memory hierarchy;
+ 2) increased heterogeneity of the processing units (mixing CPUs,
+ GPUs, etc. in varying and unexpected design combinations);
+ 3) high levels of parallelism and more complex constraints means
+ that cooperating processes must be dynamically and unpredictably
+ scheduled for asynchronous execution;
+ 4) software will not run at scale without much better resilience to
+ faults and far more robustness; and
+ 5) new levels of self-adaptivity will be required to enable
+ software to modulate process speed in order to satisfy limited
+ energy budgets.
+ The MORSE associate team will tackle the first three challenges in
+ a orchestrating work between research groups respectively
+ specialized in sparse linear algebra, dense linear algebra and
+ runtime systems. The overall objective is to develop robust
+ linear algebra libraries relying on innovative runtime systems
+ that can fully benefit from the potential of those future
+ large-scale complex machines. Challenges 4) and 5) will also be
+ investigated by the different teams in the context of other
+ partnerships, but they will not be the main focus of the associate
+ team as they are much more prospective.
+
+**** Research fields
+ The overall goal of the MORSE associate team is to enable advanced
+ numerical algorithms to be executed on a scalable unified runtime
+ system for exploiting the full potential of future exascale
+ machines. We expect advances in three directions based first on
+ strong and closed interactions between the runtime and numerical
+ linear algebra communities. This initial activity will then
+ naturally expand to more focused but still joint research in both
+ fields.
+
+***** Fine interaction between linear algebra and runtime systems
+ On parallel machines, HPC applications need to take care of data
+ movement and consistency, which can be either explicitly managed
+ at the level of the application itself or delegated to a runtime
+ system. We adopt the latter approach in order to better keep up
+ with hardware trends whose complexity is growing exponentially.
+ One major task in this project is to define a proper interface
+ between HPC applications and runtime systems in order to maximize
+ productivity and expressivity. As mentioned in the next section,
+ a widely used approach consists in abstracting the application as
+ a DAG that the runtime system is in charge of scheduling.
+ Scheduling such a DAG over a set of heterogeneous processing
+ units introduces a lot of new challenges, such as predicting
+ accurately the execution time of each type of task over each kind
+ of unit, minimizing data transfers between memory banks,
+ performing data prefetching, etc. Expected advances: In a
+ nutshell, a new runtime system API will be designed to allow
+ applications to provide scheduling hints to the runtime system
+ and to get real-time feedback about the consequences of
+ scheduling decisions.
+
+***** Runtime systems
+ A runtime environment is an intermediate layer between the system
+ and the application. It provides low-level functionality not
+ provided by the system (such as scheduling or management of the
+ heterogeneity) and high-level features (such as performance
+ portability). In the framework of this proposal, we will work on
+ the scalability of runtime environment. To achieve scalability it
+ is required to avoid all centralization. Here, the main problem
+ is the scheduling of the tasks. In many task-based runtime
+ environments the scheduler is centralized and becomes a
+ bottleneck as soon as too many cores are involved. It is
+ therefore required to distribute the scheduling decision or to
+ compute a data distribution that impose the mapping of task
+ using, for instance the so-called ``owner-compute'' rule.
+ Expected advances: We will design runtime systems that enable an
+ efficient and scalable use of thousands of distributed multicore
+ nodes enhanced with accelerators.
+
+***** Linear algebra
+ Because of its central position in HPC and of the well understood
+ structure of its algorithms, dense linear algebra has often
+ pioneered new challenges that HPC had to face. Again, dense
+ linear algebra has been in the vanguard of the new era of
+ petascale computing with the design of new algorithms that can
+ efficiently run on a multicore node with GPU accelerators. These
+ algorithms are called ``communication-avoiding'' since they have
+ been redesigned to limit the amount of communication between
+ processing units (and between the different levels of memory
+ hierarchy). They are expressed through Direct Acyclic Graphs
+ (DAG) of fine-grained tasks that are dynamically
+ scheduled. Expected advances: First, we plan to investigate the
+ impact of these principles in the case of sparse applications
+ (whose algorithms are slightly more complicated but often rely on
+ dense kernels). Furthermore, both in the dense and sparse cases,
+ the scalability on thousands of nodes is still limited; new
+ numerical approaches need to be found. We will specifically
+ design sparse hybrid direct/iterative methods that represent a
+ promising approach.
+
+**** Research papers
+ Research papers about MORSE can be found [[http://icl.cs.utk.edu/projectsdev/morse/pubs/index.html][here]].
+
+*** Chameleon
+**** Chameleon software
+ The main purpose is to address the performance shortcomings of the
+ [[http://www.netlib.org/lapack/][LAPACK]] and [[http://www.netlib.org/scalapack/][ScaLAPACK]] libraries on multicore processors and
+ multi-socket systems of multicore processors and their inability
+ to efficiently utilize accelerators such as Graphics Processing
+ Units (GPUs).
+
+ Chameleon is a framework written in C which provides routines to
+ solve dense general systems of linear equations, symmetric
+ positive definite systems of linear equations and linear least
+ squares problems, using LU, Cholesky, QR and LQ factorizations.
+ Real arithmetic and complex arithmetic are supported in both
+ single precision and double precision. It supports Linux and Mac
+ OS/X machines (mainly tested on Intel x86-64 and IBM Power
+ architectures).
+
+ Chameleon is based on the [[http://icl.cs.utk.edu/plasma/][PLASMA]] source code but is not limited to
+ shared-memory environment and can exploit multiple GPUs.
+ Chameleon is interfaced in a generic way with [[http://runtime.bordeaux.inria.fr/StarPU/][StarPU]], [[http://icl.utk.edu/parsec/][PaRSEC]],
+ [[http://icl.cs.utk.edu/quark/][QUARK]] runtime systems. This feature allows to analyze in a
+ unified framework how sequential task-based algorithms behave
+ regarding different runtime systems implementations. Using
+ Chameleon with *StarPU* or *PaRSEC* runtime systems allows to exploit
+ GPUs through kernels provided by [[https://developer.nvidia.com/cublas][cuBLAS]] and clusters of
+ interconnected nodes with distributed memory (using [[http://www.open-mpi.org/][MPI]]).
+ Computation of very large systems with dense matrices on a cluster
+ of nodes is still being experimented and stabilized. It is not
+ expected to get stable performances with the current version using
+ MPI.
+
+**** PLASMA's design principles
+ Chameleon is originally based on [[http://icl.cs.utk.edu/plasma/][PLASMA]] so that design principles
+ are very similar. The content of this section PLASMA's design
+ principles has been copied from the /Design principles/ section of
+ the PLASMA User's Guide.
+
+***** Tile Algorithms
+ Tile algorithms are based on the idea of processing the matrix by
+ square tiles of relatively small size, such that a tile fits
+ entirely in one of the cache levels associated with one core.
+ This way a tile can be loaded to the cache and processed
+ completely before being evicted back to the main memory. Of the
+ three types of cache misses, *compulsory*, *capacity* and *conflict*,
+ the use of tile algorithms minimizes the number of capacity
+ misses, since each operation loads the amount of data that does
+ not ``overflow'' the cache.
+
+ For some operations such as matrix multiplication and Cholesky
+ factorization, translating the classic algorithm to the tile
+ algorithm is trivial. In the case of matrix multiplication, the
+ tile algorithm is simply a product of applying the technique of
+ *loop tiling* to the canonical definition of three nested loops.
+ It is very similar for the Cholesky factorization. The
+ *left-looking* definition of Cholesky factorization from LAPACK is
+ a loop with a sequence of calls to four routines: xSYRK
+ (symmetric *rank-k* update), xPOTRF (Cholesky factorization of a
+ small block on the diagonal), xGEMM (matrix multiplication) and
+ xTRSM (triangular solve). If the xSYRK, xGEMM and xTRSM
+ operations are expressed with the canonical definition of three
+ nested loops and the technique of loop tiling is applied, the
+ tile algorithm results. Since the algorithm is produced by
+ simple reordering of operations, neither the number of operations
+ nor numerical stability of the algorithm are affected.
+
+ The situation becomes slightly more complicated for LU and QR
+ factorizations, where the classic algorithms factorize an entire
+ panel of the matrix (a block of columns) at every step of the
+ algorithm. One can observe, however, that the process of matrix
+ factorization is synonymous with introducing zeros in approproate
+ places and a tile algorithm can be fought of as one that zeroes
+ one tile of the matrix at a time. This process is referred to as
+ updating of a factorization or *incremental factorization*. The
+ process is equivalent to factorizing the top tile of a panel,
+ then placing the upper triangle of the result on top of the tile
+ blow and factorizing again, then moving to the next tile and so
+ on. Here, the tile LU and QR algorithms perform slightly more
+ floating point operations and require slightly more memory for
+ auxiliary data. Also, the tile LU factorization applies a
+ different pivoting pattern and, as a result, is less numerically
+ stable than classic LU with full pivoting. Numerical stability
+ is not an issue in case of the tile QR, which relies on
+ orthogonal transformations (Householder reflections), which are
+ numerically stable.
+
+ #+CAPTION: Schematic illustration of the tile LU factorization (kernel names for real arithmetics in double precision), courtesey of the [[http://icl.cs.utk.edu/plasma/][PLASMA]] team.
+ #+NAME: fig:tile_lu
+ #+ATTR_HTML: :width 640px :align center
+ [[file:tile_lu.jpg]]
+
+***** Tile Data Layout
+ <<sec:tile>>
+
+ Tile layout is based on the idea of storing the matrix by square
+ tiles of relatively small size, such that each tile occupies a
+ continuous memory region. This way a tile can be loaded to the
+ cache memory efficiently and the risk of evicting it from the
+ cache memory before it is completely processed is minimized. Of
+ the three types of cache misses, *compulsory*, *capacity* and
+ *conflict*, the use of tile layout minimizes the number of conflict
+ misses, since a continuous region of memory will completely fill
+ out a /set-associative/ cache memory before an eviction can happen.
+ Also, from the standpoint of multithreaded execution, the
+ probability of *false sharing* is minimized. It can only affect
+ the cache lines containing the beginning and the ending of a
+ tile.
+
+ In standard *cache-based* architecture, tiles continously laid out
+ in memory maximize the profit from automatic prefetching. Tile
+ layout is also beneficial in situations involving the use of
+ accelerators, where explicit communication of tiles through DMA
+ transfers is required, such as moving tiles between the system
+ memory and the local store in Cell B. E. or moving tiles between
+ the host memory and the device memory in GPUs. In most
+ circumstances tile layout also minimizes the number of TLB misses
+ and conflicts to memory banks or partitions. With the standard
+ (*column-major*) layout, access to each column of a tile is much
+ more likely to cause a conflict miss, a false sharing miss, a TLB
+ miss or a bank or partition conflict. The use of the standard
+ layout for dense matrix operations is a performance minefield.
+ Although occasionally one can pass through it unscathed, the risk
+ of hitting a spot deadly to performance is very high.
+
+ Another property of the layout utilized in PLASMA is that it is
+ ``flat'', meaning that it does not involve a level of
+ indirection. Each tile stores a small square submatrix of the
+ main matrix in a *column-major* layout. In turn, the main matrix is
+ an arrangement of tiles immediately following one another in a
+ *column-major* layout. The offset of each tile can be calculated
+ through address arithmetics and does not involve pointer
+ indirection. Alternatively, a matrix could be represented as an
+ array of pointers to tiles, located anywhere in memory. Such
+ layout would be a radical and unjustifiable departure from LAPACK
+ and ScaLAPACK. Flat tile layout is a natural progression from
+ LAPACK's *column-major* layout and ScaLAPACK's /block-cyclic/ layout.
+
+ Another related property of PLASMA's tile layout is that it
+ includes provisions for padding of tiles, i.e., the actual region
+ of memory designated for a tile can be larger than the memory
+ occupied by the actual data. This allows to force a certain
+ alignment of tile boundaries, while using the flat organization
+ described in the previous paragraph. The motivation is that, at
+ the price of small memory overhead, alignment of tile boundaries
+ may prove benefivial in multiple scenarios involving memory
+ systems of standard multicore processors, as well as
+ accelerators. The issues that come into play are, again, the use
+ of TLBs and memory banks or partitions.
+
+ #+CAPTION: Schematic illustration of the tile layout with *column-major* order of tiles, *column-major* order of elements within tiles and (optional) padding for enforcing a certain alighment of tile bondaries, courtesey of the [[http://icl.cs.utk.edu/plasma/][PLASMA]] team.
+ #+NAME: fig:tile_layout
+ #+ATTR_HTML: :width 640px :align center
+ [[file:tile_layout.jpg]]
+
+***** Dynamic Task Scheduling
+
+ Dynamic scheduling is the idea of assigning work to cores based
+ on the availability of data for processing at any given point in
+ time and is also referred to as *data-driven* scheduling. The
+ concept is related closely to the idea of expressing computation
+ through a task graph, often referred to as the DAG (*Direct
+ Acyclic Graph*), and the flexibility exploring the DAG at runtime.
+ Thus, to a large extent, dynamic scheduling is synonymous with
+ *runtime scheduling*. An important concept here is the one of
+ the *critical path*, which defines the upper bound on the
+ achievable parallelism, and needs to be pursued at the maximum
+ speed. This is in direct opposition to the *fork-and-join* or
+ *data-parallel* programming models, where artificial
+ synchronization points expose serial sections of the code, where
+ multiple cores are idle, while sequential processing takes place.
+ The use of dynamic scheduling introduces a *trade-off*, though.
+ The more dynamic (flexible) scheduling is, the more centralized
+ (and less scalable) the scheduling mechanism is. For that
+ reason, currently PLASMA uses two scheduling mechanisms, one
+ which is fully dynamic and one where work is assigned statically
+ and dependency checks are done at runtime.
+
+ The first scheduling mechanism relies on unfolding a *sliding
+ window* of the task graph at runtime and scheduling work by
+ resolving data hazards: *Read After Write(RAW)*, *Write After Read
+ (WAR)* and *Write After Write (WAW)*, a technique analogous to
+ instruction scheduling in superscalar processors. It also relies
+ on *work-stealing* for balanding the load among all multiple cores.
+ The second scheduling mechanism relies on statically designating
+ a path through the execution space of the algorithm to each core
+ and following a cycle: transition to a task, wait for its
+ dependencies, execute it, update the overall progress. Task are
+ identified by tuples and task transitions are done through
+ locally evaluated formulas. Progress information can be
+ centralized, replicated or distributed (currently centralized).
+
+ #+CAPTION: A trace of the tile QR factorization executing on eight cores without any global synchronization points (kernel names for real arithmetics in single precision), courtesey of the [[http://icl.cs.utk.edu/plasma/][PLASMA]] team.
+ #+NAME: fig:trace_qr
+ #+ATTR_HTML: :width 640px :align center
+ [[file:trace_qr.jpg]]
diff --git a/doc/user/chapters/performances.org b/doc/user/chapters/performances.org
new file mode 100644
index 0000000000000000000000000000000000000000..cbf157482270056b9c66d3cba0ec35dd4475c911
--- /dev/null
+++ b/doc/user/chapters/performances.org
@@ -0,0 +1,65 @@
+# This file is part of the Chameleon User's Guide.
+# Copyright (C) 2020 Inria
+# See the file ../users_guide.org for copying conditions.
+
+Show performances on [[https://www.plafrim.fr/][PlaFRIM]] supercomputer.
+
+See [[https://www.plafrim.fr/hardware-documentation/][characteristics]] to get details about the hardwares.
+
+See script ~tools/bench/plafrim/run.sh~ to get details about the environment (Guix, Slurm,
+etc) and the build.
+
+Chameleon is run this way:
+#+begin_src sh
+mpiexec -np $nmpi $CHAMELEON_BUILD/testing/chameleon_${precision}testing -o ${algorithm} -P $p -t $nthr -g $ngpu -m $m -n $n -k $k -b $b
+#+end_src
+- runtime : *starpu*
+- precision : *s* or *d* for simple or double precision
+- algorithm : *gemm* or *potrf* or *geqrf_hqr*
+- nmpi = p x p
+- nthr : depends on the node
+- ngpu : depends on the node
+- m = n = k
+- b : depends on the node
+*** bora (36 CPUs) nodes
+ - nmpi = *1*, *4*, *9*
+ - 2D block cyclic parameters : PxQ = 1x1, 2x2 and 3x3
+ - Number of threads (t) = *34*, one CPU being dedicated for the
+ scheduler and one other for MPI communications
+ - Number of GPUs = *0*
+ - Tile Size (b) = *280*
+**** CPU times
+ #+CAPTION: Performances in CPU time of GEMM, POTRF and QR on bora nodes
+ #+NAME: fig:chameleon_plafrim_bora_time_openmpi
+ #+ATTR_HTML: :align center :width 75%
+ [[file:chameleon_plafrim_bora_time_openmpi.png]]
+
+**** GFLOP/s
+ #+CAPTION: Performances in GFlop/s of GEMM, POTRF and QR on bora nodes
+ #+NAME: fig:chameleon_plafrim_bora_perf_openmpi
+ #+ATTR_HTML: :align center :width 75%
+ [[file:chameleon_plafrim_bora_perf_openmpi.png]]
+
+*** sirocco [14-17] (32 CPUs + 2 GPUs V100) nodes
+ - nmpi = *1*
+ - 2D block cyclic parameters : PxQ = 1x1
+ - Number of threads (t) = *29*, one CPU being dedicated for the
+ scheduler and two others for the 2 GPUs
+ - Number of GPUs = *2*
+ - Tile Size (b) = *1600*
+ #+CAPTION: Performances in CPU time of GEMM, POTRF and QR on sirocco nodes
+ #+NAME: fig:chameleon_plafrim_sirocco
+ #+ATTR_HTML: :align center :width 75%
+ [[file:chameleon_plafrim_sirocco.png]]
+
+ # *** CPU times
+ # #+CAPTION: Performances in CPU time of GEMM, POTRF and QR on sirocco nodes
+ # #+NAME: fig:chameleon_plafrim_sirocco_openmpi_time
+ # #+ATTR_HTML: :align center :height 1024
+ # [[file:chameleon_plafrim_sirocco_openmpi_time.png]]
+ #
+ # *** GFLOP/s
+ # #+CAPTION: Performances in GFlop/s of GEMM, POTRF and QR on sirocco nodes
+ # #+NAME: fig:chameleon_plafrim_sirocco_openmpi_perf
+ # #+ATTR_HTML: :align center :height 1024
+ # [[file:chameleon_plafrim_sirocco_openmpi_perf.png]]
diff --git a/doc/orgmode/chapters/potri_async.png b/doc/user/chapters/potri_async.png
similarity index 100%
rename from doc/orgmode/chapters/potri_async.png
rename to doc/user/chapters/potri_async.png
diff --git a/doc/orgmode/chapters/tile_layout.jpg b/doc/user/chapters/tile_layout.jpg
similarity index 100%
rename from doc/orgmode/chapters/tile_layout.jpg
rename to doc/user/chapters/tile_layout.jpg
diff --git a/doc/orgmode/chapters/tile_layout.pdf b/doc/user/chapters/tile_layout.pdf
similarity index 100%
rename from doc/orgmode/chapters/tile_layout.pdf
rename to doc/user/chapters/tile_layout.pdf
diff --git a/doc/orgmode/chapters/tile_lu.jpg b/doc/user/chapters/tile_lu.jpg
similarity index 100%
rename from doc/orgmode/chapters/tile_lu.jpg
rename to doc/user/chapters/tile_lu.jpg
diff --git a/doc/orgmode/chapters/tile_lu.pdf b/doc/user/chapters/tile_lu.pdf
similarity index 100%
rename from doc/orgmode/chapters/tile_lu.pdf
rename to doc/user/chapters/tile_lu.pdf
diff --git a/doc/orgmode/chapters/trace_qr.jpg b/doc/user/chapters/trace_qr.jpg
similarity index 100%
rename from doc/orgmode/chapters/trace_qr.jpg
rename to doc/user/chapters/trace_qr.jpg
diff --git a/doc/orgmode/chapters/trace_qr.pdf b/doc/user/chapters/trace_qr.pdf
similarity index 100%
rename from doc/orgmode/chapters/trace_qr.pdf
rename to doc/user/chapters/trace_qr.pdf
diff --git a/doc/orgmode/chapters/using.org b/doc/user/chapters/using.org
similarity index 73%
rename from doc/orgmode/chapters/using.org
rename to doc/user/chapters/using.org
index 74be7d6be8537a9d3f0cafb28f42c7f32c73d6e1..6e640c2bda2405fb4b8634173df121250ec5635c 100644
--- a/doc/orgmode/chapters/using.org
+++ b/doc/user/chapters/using.org
@@ -1,253 +1,252 @@
# This file is part of the Chameleon User's Guide.
# Copyright (C) 2017 Inria
# See the file ../users_guide.org for copying conditions.
-** Using Chameleon executables
-
- Chameleon provides several test executables that are compiled and
- linked with Chameleon's dependencies. Instructions about the
- arguments to give to executables are accessible thanks to the
- option ~-[-]help~ or ~-[-]h~. This set of binaries are separated into
- three categories and can be found in three different directories:
- * *example*: contains examples of API usage and more specifically the
- sub-directory ~lapack_to_chameleon/~ provides a tutorial that explains
- how to use Chameleon functionalities starting from a full LAPACK
- code, see [[sec:tuto][Tutorial LAPACK to Chameleon]]
- * *testing*: contains testing drivers to check numerical
- correctness and assess performance of Chameleon linear algebra
- routines with a wide range of parameters
- #+begin_src
- ./testing/chameleon_stesting -H -o gemm -t 2 -m 2000 -n 2000 -k 2000
- #+end_src
-
- To get the list of parameters, use the ~-h~ or ~--help~ option.
- #+begin_src
- ./testing/chameleon_stesting -h
- #+end_src
-
- Available algorithms for testing are:
- * gels_hqr: Linear least squares with general matrix using hierarchical reduction trees
- * ormlq_hqr: Q application with hierarchical reduction trees (LQ)
- * orglq_hqr: Q generation with hierarchical reduction trees (LQ)
- * gelqf_hqr: General LQ factorization with hierachical reduction trees
- * ormqr_hqr: Q application with hierarchical reduction trees (QR)
- * orgqr_hqr: Q generation with hierarchical reduction trees (QR)
- * geqrf_hqr: General QR factorization with hierachical reduction trees
- * gels: Linear least squares with general matrix
- * ormlq: Q application (LQ)
- * orglq: Q generation (LQ)
- * gelqf: General LQ factorization
- * ormqr: Q application (QR)
- * orgqr: Q generation (QR)
- * geqrf: General QR factorization
- * gesv: General linear system solve (LU without pivoting)
- * getrs: General triangular solve (LU without pivoting)
- * getrf: General factorization (LU without pivoting)
- * potri: Symmetric positive definite matrix inversion
- * lauum: Trianguilar in-place matrix-matrix computation for Cholesky inversion
- * trtri: Triangular matrix inversion
- * posv: Symmetric positive definite linear system solve (Cholesky)
- * potrs: Symmetric positive definite solve (Cholesky)
- * potrf: Symmetric positive definite factorization (Cholesky)
- * trsm: Triangular matrix solve
- * trmm: Triangular matrix-matrix multiply
- * syr2k: Symmetrix matrix-matrix rank 2k update
- * syrk: Symmetrix matrix-matrix rank k update
- * symm: Symmetric matrix-matrix multiply
- * gemm: General matrix-matrix multiply
- * lascal: General matrix scaling
- * tradd: Triangular matrix-matrix addition
- * geadd: General matrix-matrix addition
- * lantr: Triangular matrix norm
- * lansy: Symmetric matrix norm
- * lange: General matrix norm
- * lacpy: General matrix copy
-
-*** Execution trace using EZTrace
- <<sec:trace_ezt>>
-
- [[http://eztrace.gforge.inria.fr/support.html][EZTrace]] can be used by chameleon to generate traces. Two modules
- are automatically generated as soon as EZTrace is detected on the
- system. The first one (which is recommended) is the
- ~chameleon_tcore~ module. It traces all the ~TCORE_...()~ functions
- that are called by the codelets of all the runtime but PaRSEC. The
- second one is the ~chameleon_core~ module which traces the lower
- level ~CORE_...()~ functions. If using PaRSEC, you need to use this
- module to generate the traces.
-
- To generate traces with EZTrace, you need first to compile with
- *-DBUILD_SHARED_LIBS=ON*. EZTrace is using weak symbols to overload
- function calls with ld_preload and enable trace generation. Then,
- either you install the ~libeztrace-*.so~ files into the EZTrace
- install directory, or you can add the path of the modules to your
- environement
- #+begin_src
- export EZTRACE_LIBRARY_PATH=/path/to/your/modules
- #+end_src
+*** Using Chameleon executables
+ <<sec:usetesting>>
+
+ Chameleon provides several test executables that are compiled and
+ linked with Chameleon's dependencies. Instructions about the
+ arguments to give to executables are accessible thanks to the
+ option ~-[-]help~ or ~-[-]h~. This set of binaries are separated into
+ three categories and can be found in three different directories:
+ * *example*: contains examples of API usage and more specifically the
+ sub-directory ~lapack_to_chameleon/~ provides a tutorial that explains
+ how to use Chameleon functionalities starting from a full LAPACK
+ code, see [[sec:tuto][Tutorial LAPACK to Chameleon]]
+ * *testing*: contains testing drivers to check numerical
+ correctness and assess performance of Chameleon linear algebra
+ routines with a wide range of parameters
+ #+begin_src
+ ./testing/chameleon_stesting -H -o gemm -t 2 -m 2000 -n 2000 -k 2000
+ #+end_src
+
+ To get the list of parameters, use the ~-h~ or ~--help~ option.
+ #+begin_src
+ ./testing/chameleon_stesting -h
+ #+end_src
+
+ Available algorithms for testing are:
+ * gels_hqr: Linear least squares with general matrix using hierarchical reduction trees
+ * ormlq_hqr: Q application with hierarchical reduction trees (LQ)
+ * orglq_hqr: Q generation with hierarchical reduction trees (LQ)
+ * gelqf_hqr: General LQ factorization with hierachical reduction trees
+ * ormqr_hqr: Q application with hierarchical reduction trees (QR)
+ * orgqr_hqr: Q generation with hierarchical reduction trees (QR)
+ * geqrf_hqr: General QR factorization with hierachical reduction trees
+ * gels: Linear least squares with general matrix
+ * ormlq: Q application (LQ)
+ * orglq: Q generation (LQ)
+ * gelqf: General LQ factorization
+ * ormqr: Q application (QR)
+ * orgqr: Q generation (QR)
+ * geqrf: General QR factorization
+ * gesv: General linear system solve (LU without pivoting)
+ * getrs: General triangular solve (LU without pivoting)
+ * getrf: General factorization (LU without pivoting)
+ * potri: Symmetric positive definite matrix inversion
+ * lauum: Trianguilar in-place matrix-matrix computation for Cholesky inversion
+ * trtri: Triangular matrix inversion
+ * posv: Symmetric positive definite linear system solve (Cholesky)
+ * potrs: Symmetric positive definite solve (Cholesky)
+ * potrf: Symmetric positive definite factorization (Cholesky)
+ * trsm: Triangular matrix solve
+ * trmm: Triangular matrix-matrix multiply
+ * syr2k: Symmetrix matrix-matrix rank 2k update
+ * syrk: Symmetrix matrix-matrix rank k update
+ * symm: Symmetric matrix-matrix multiply
+ * gemm: General matrix-matrix multiply
+ * lascal: General matrix scaling
+ * tradd: Triangular matrix-matrix addition
+ * geadd: General matrix-matrix addition
+ * lantr: Triangular matrix norm
+ * lansy: Symmetric matrix norm
+ * lange: General matrix norm
+ * lacpy: General matrix copy
+
+**** Execution trace using EZTrace
+ <<sec:trace_ezt>>
+
+ [[http://eztrace.gforge.inria.fr/support.html][EZTrace]] can be used by chameleon to generate traces. Two modules
+ are automatically generated as soon as EZTrace is detected on the
+ system. The first one (which is recommended) is the
+ ~chameleon_tcore~ module. It traces all the ~TCORE_...()~ functions
+ that are called by the codelets of all the runtime but PaRSEC. The
+ second one is the ~chameleon_core~ module which traces the lower
+ level ~CORE_...()~ functions. If using PaRSEC, you need to use this
+ module to generate the traces.
+
+ To generate traces with EZTrace, you need first to compile with
+ *-DBUILD_SHARED_LIBS=ON*. EZTrace is using weak symbols to overload
+ function calls with ld_preload and enable trace generation. Then,
+ either you install the ~libeztrace-*.so~ files into the EZTrace
+ install directory, or you can add the path of the modules to your
+ environement
+ #+begin_src
+ export EZTRACE_LIBRARY_PATH=/path/to/your/modules
+ #+end_src
+
+ To check if the modules are available you should have
+ #+begin_src
+ $ eztrace_avail
+ 1 omp Module for OpenMP parallel regions
+ 2 pthread Module for PThread synchronization functions (mutex, semaphore, spinlock, etc.)
+ 3 stdio Module for stdio functions (read, write, select, poll, etc.)
+ 4 mpi Module for MPI functions
+ 5 memory Module for memory functions (malloc, free, etc.)
+ 6 papi Module for PAPI Performance counters
+ 128 chameleon_core Module for Chameleon CORE functions
+ 129 chameleon_tcore Module for Chameleon TCORE functions
+ #+end_src
+
+ Then, you can restrict the modules used during the execution
+ #+begin_src
+ export EZTRACE_TRACE="mpi chameleon_tcore"
+ #+end_src
+
+ _The module ~mpi~ is required if you want to run in distributed._
+
+ The setup can be checked with ~eztrace_loaded~
+ #+begin_src
+ $ eztrace_loaded
+ 4 mpi Module for MPI functions
+ 129 chameleon_tcore Module for Chameleon TCORE functions
+ #+end_src
+
+ To generate the traces, you need to run your binary through
+ eztrace:
+ #+begin_src
+ eztrace ./chameleon_dtesting -o gemm -n 1000 -b 200
+ mpirun -np 4 eztrace ./chameleon_dtesting -o gemm -n 1000 -b 200 -P 2
+ #+end_src
+
+ Convert the binary files into a ~.trace~ file, and visualize it.
+ #+begin_src
+ eztrace_convert <username>_eztrace_log_rank_<[0-9]*>
+ vite eztrace_output.trace
+ #+end_src
+
+ For more information on EZTrace, you can follow the [[http://eztrace.gforge.inria.fr/support.html][support page]].
+
+**** Execution trace using StarPU/FxT
+ <<sec:trace_fxt>>
+
+ StarPU can generate its own trace log files by compiling it with
+ the ~--with-fxt~ option at the configure step (you can have to
+ specify the directory where you installed FxT by giving
+ ~--with-fxt=...~ instead of ~--with-fxt~ alone). By doing so, traces
+ are generated after each execution of a program which uses StarPU
+ in the directory pointed by the [[http://starpu.gforge.inria.fr/doc/html/ExecutionConfigurationThroughEnvironmentVariables.html][STARPU_FXT_PREFIX]] environment
+ variable.
+ #+begin_example
+ export STARPU_FXT_PREFIX=/home/jdoe/fxt_files/
+ #+end_example
+ When executing a ~./testing/...~ Chameleon program, if it has been
+ enabled (StarPU compiled with FxT), the program will generate
+ trace files in the directory $STARPU_FXT_PREFIX.
+
+ Finally, to generate the trace file which can be opened with [[http://vite.gforge.inria.fr/][Vite]]
+ program, you can use the *starpu_fxt_tool* executable of StarPU.
+ This tool should be in the bin directory of StarPU's installation.
+ You can use it to generate the trace file like this:
+ #+begin_src
+ path/to/your/install/starpu/bin/starpu_fxt_tool -i prof_filename
+ #+end_src
+ There is one file per mpi processus (prof_filename_0,
+ prof_filename_1 ...). To generate a trace of mpi programs you can
+ call it like this:
+ #+begin_src
+ path/to/your/install/starpu/bin/starpu_fxt_tool -i prof_filename*
+ #+end_src
+ The trace file will be named paje.trace (use -o option to specify
+ an output name). Alternatively, for non mpi execution (only one
+ processus and profiling file), you can set the environment
+ variable *STARPU_GENERATE_TRACE=1* to automatically generate the
+ paje trace file.
+
+**** Use simulation mode with StarPU-SimGrid
+ <<sec:simu>>
+
+ Simulation mode can be activated by setting the cmake option
+ CHAMELEON_SIMULATION to ON. This mode allows you to simulate
+ execution of algorithms with StarPU compiled with [[http://simgrid.gforge.inria.fr/][SimGrid]]. To do
+ so, we provide some perfmodels in the simucore/perfmodels/
+ directory of Chameleon sources. To use these perfmodels, please
+ set your *STARPU_HOME* environment variable to
+ ~path/to/your/chameleon_sources/simucore/perfmodels~. Finally, you
+ need to set your *STARPU_HOSTNAME* environment variable to the name
+ of the machine to simulate. Note that only *potrf* algorithm,
+ simple and double precisions, on /mirage/ and /sirocco/ machines, with
+ or without gpus, and for the following tile sizes (choose a size *N*
+ multiple of the tile size) are available for now
+ - /mirage/: 320, 960
+ - /sirocco/: 80, 440, 960, 1440, 1920
+
+ Database of models is subject to change.
- To check if the modules are available you should have
- #+begin_src
- $ eztrace_avail
- 1 omp Module for OpenMP parallel regions
- 2 pthread Module for PThread synchronization functions (mutex, semaphore, spinlock, etc.)
- 3 stdio Module for stdio functions (read, write, select, poll, etc.)
- 4 mpi Module for MPI functions
- 5 memory Module for memory functions (malloc, free, etc.)
- 6 papi Module for PAPI Performance counters
- 128 chameleon_core Module for Chameleon CORE functions
- 129 chameleon_tcore Module for Chameleon TCORE functions
- #+end_src
+ #+begin_example
+ export STARPU_HOME=/tmp/chameleon/simucore/perfmodels/
+ export STARPU_HOSTNAME=sirocco
+ ./testing/chameleon_dtesting -o potrf -t 22 -g 2 -n 14400 -b 1440
+ Id;Function;threads;gpus;P;Q;mtxfmt;nb;uplo;n;lda;seedA;time;gflops
+ 0;dpotrf;22;2;1;1;0;1440;121;14400;14400;846930886;7.814116e-01;1.273889e+03
+ #+end_example
- Then, you can restrict the modules used during the execution
- #+begin_src
- export EZTRACE_TRACE="mpi chameleon_tcore"
- #+end_src
+**** Use out of core support with StarPU
+ <<sec:ooc>>
- _The module ~mpi~ is required if you want to run in distributed._
+ If the matrix can not fit in the main memory, StarPU can automatically evict
+ tiles to the disk. The following variables need to be set:
+ * *STARPU_DISK_SWAP* environment variable to a place where to store
+ evicted tiles, for example: ~STARPU_DISK_SWAP=/tmp~
+ * *STARPU_DISK_SWAP_BACKEND* environment variable to the I/O method,
+ for example: ~STARPU_DISK_SWAP_BACKEND=unistd_o_direct~
+ * *STARPU_LIMIT_CPU_MEM* environment variable to the amount of memory
+ that can be used in MBytes, for example: ~STARPU_LIMIT_CPU_MEM=1000~
- The setup can be checked with ~eztrace_loaded~
- #+begin_src
- $ eztrace_loaded
- 4 mpi Module for MPI functions
- 129 chameleon_tcore Module for Chameleon TCORE functions
- #+end_src
-
- To generate the traces, you need to run your binary through
- eztrace:
- #+begin_src
- eztrace ./chameleon_dtesting -o gemm -n 1000 -b 200
- mpirun -np 4 eztrace ./chameleon_dtesting -o gemm -n 1000 -b 200 -P 2
- #+end_src
+*** Tutorial LAPACK to Chameleon
+ <<sec:tuto>>
- Convert the binary files into a ~.trace~ file, and visualize it.
+ Chameleon provides routines to solve dense general systems of
+ linear equations, symmetric positive definite systems of linear
+ equations and linear least squares problems, using LU, Cholesky, QR
+ and LQ factorizations. Real arithmetic and complex arithmetic are
+ supported in both single precision and double precision. Routines
+ that compute linear algebra are of the following form:
#+begin_src
- eztrace_convert <username>_eztrace_log_rank_<[0-9]*>
- vite eztrace_output.trace
+ CHAMELEON_name[_Tile[_Async]]
#+end_src
-
- For more information on EZTrace, you can follow the [[http://eztrace.gforge.inria.fr/support.html][support page]].
-
-*** Execution trace using StarPU/FxT
- <<sec:trace_fxt>>
-
- StarPU can generate its own trace log files by compiling it with
- the ~--with-fxt~ option at the configure step (you can have to
- specify the directory where you installed FxT by giving
- ~--with-fxt=...~ instead of ~--with-fxt~ alone). By doing so, traces
- are generated after each execution of a program which uses StarPU
- in the directory pointed by the [[http://starpu.gforge.inria.fr/doc/html/ExecutionConfigurationThroughEnvironmentVariables.html][STARPU_FXT_PREFIX]] environment
- variable.
- #+begin_example
- export STARPU_FXT_PREFIX=/home/jdoe/fxt_files/
- #+end_example
- When executing a ~./testing/...~ Chameleon program, if it has been
- enabled (StarPU compiled with FxT), the program will generate
- trace files in the directory $STARPU_FXT_PREFIX.
-
- Finally, to generate the trace file which can be opened with [[http://vite.gforge.inria.fr/][Vite]]
- program, you can use the *starpu_fxt_tool* executable of StarPU.
- This tool should be in the bin directory of StarPU's installation.
- You can use it to generate the trace file like this:
+ * all user routines are prefixed with *CHAMELEON*
+ * in the pattern *CHAMELEON_name[_Tile[_Async]]*, /name/ follows the
+ BLAS/LAPACK naming scheme for algorithms (/e.g./ sgemm for general
+ matrix-matrix multiply simple precision)
+ * Chameleon provides three interface levels
+ * *CHAMELEON_name*: simplest interface, very close to CBLAS and
+ LAPACKE, matrices are given following the LAPACK data layout
+ (1-D array column-major). It involves copy of data from LAPACK
+ layout to tile layout and conversely (to update LAPACK data),
+ see [[sec:tuto_step1][Step1]].
+ * *CHAMELEON_name_Tile*: the tile interface avoid copies between LAPACK
+ and tile layouts. It is the standard interface of Chameleon and
+ it should achieved better performance than the previous
+ simplest interface. The data are given through a specific
+ structure called a descriptor, see [[sec:tuto_step2][Step2]].
+ * *CHAMELEON_name_Tile_Async*: similar to the tile interface, it avoids
+ synchonization barrier normally called between *Tile* routines.
+ At the end of an *Async* function, completion of tasks is not
+ guaranteed and data are not necessarily up-to-date. To ensure
+ that tasks have been all executed, a synchronization function
+ has to be called after the sequence of *Async* functions, see
+ [[sec:tuto_step4][Step4]].
+
+ CHAMELEON routine calls have to be preceded from
#+begin_src
- path/to/your/install/starpu/bin/starpu_fxt_tool -i prof_filename
+ CHAMELEON_Init( NCPU, NGPU );
#+end_src
- There is one file per mpi processus (prof_filename_0,
- prof_filename_1 ...). To generate a trace of mpi programs you can
- call it like this:
+ to initialize CHAMELEON and the runtime system and followed by
#+begin_src
- path/to/your/install/starpu/bin/starpu_fxt_tool -i prof_filename*
+ CHAMELEON_Finalize();
#+end_src
- The trace file will be named paje.trace (use -o option to specify
- an output name). Alternatively, for non mpi execution (only one
- processus and profiling file), you can set the environment
- variable *STARPU_GENERATE_TRACE=1* to automatically generate the
- paje trace file.
-
-*** Use simulation mode with StarPU-SimGrid
- <<sec:simu>>
-
- Simulation mode can be activated by setting the cmake option
- CHAMELEON_SIMULATION to ON. This mode allows you to simulate
- execution of algorithms with StarPU compiled with [[http://simgrid.gforge.inria.fr/][SimGrid]]. To do
- so, we provide some perfmodels in the simucore/perfmodels/
- directory of Chameleon sources. To use these perfmodels, please
- set your *STARPU_HOME* environment variable to
- ~path/to/your/chameleon_sources/simucore/perfmodels~. Finally, you
- need to set your *STARPU_HOSTNAME* environment variable to the name
- of the machine to simulate. Note that only *potrf* algorithm,
- simple and double precisions, on /mirage/ and /sirocco/ machines, with
- or without gpus, and for the following tile sizes (choose a size *N*
- multiple of the tile size) are available for now
- - /mirage/: 320, 960
- - /sirocco/: 80, 440, 960, 1440, 1920
-
- Database of models is subject to change.
-
- #+begin_example
- export STARPU_HOME=/tmp/chameleon/simucore/perfmodels/
- export STARPU_HOSTNAME=sirocco
- ./testing/chameleon_dtesting -o potrf -t 22 -g 2 -n 14400 -b 1440
- Id;Function;threads;gpus;P;Q;mtxfmt;nb;uplo;n;lda;seedA;time;gflops
- 0;dpotrf;22;2;1;1;0;1440;121;14400;14400;846930886;7.814116e-01;1.273889e+03
- #+end_example
-
-*** Use out of core support with StarPU
- <<sec:ooc>>
-
- If the matrix can not fit in the main memory, StarPU can automatically evict
- tiles to the disk. The following variables need to be set:
- * *STARPU_DISK_SWAP* environment variable to a place where to store
- evicted tiles, for example: ~STARPU_DISK_SWAP=/tmp~
- * *STARPU_DISK_SWAP_BACKEND* environment variable to the I/O method,
- for example: ~STARPU_DISK_SWAP_BACKEND=unistd_o_direct~
- * *STARPU_LIMIT_CPU_MEM* environment variable to the amount of memory
- that can be used in MBytes, for example: ~STARPU_LIMIT_CPU_MEM=1000~
-
-** Chameleon API
-
- Chameleon provides routines to solve dense general systems of
- linear equations, symmetric positive definite systems of linear
- equations and linear least squares problems, using LU, Cholesky, QR
- and LQ factorizations. Real arithmetic and complex arithmetic are
- supported in both single precision and double precision. Routines
- that compute linear algebra are of the following form:
- #+begin_src
- CHAMELEON_name[_Tile[_Async]]
- #+end_src
- * all user routines are prefixed with *CHAMELEON*
- * in the pattern *CHAMELEON_name[_Tile[_Async]]*, /name/ follows the
- BLAS/LAPACK naming scheme for algorithms (/e.g./ sgemm for general
- matrix-matrix multiply simple precision)
- * Chameleon provides three interface levels
- * *CHAMELEON_name*: simplest interface, very close to CBLAS and
- LAPACKE, matrices are given following the LAPACK data layout
- (1-D array column-major). It involves copy of data from LAPACK
- layout to tile layout and conversely (to update LAPACK data),
- see [[sec:tuto_step1][Step1]].
- * *CHAMELEON_name_Tile*: the tile interface avoid copies between LAPACK
- and tile layouts. It is the standard interface of Chameleon and
- it should achieved better performance than the previous
- simplest interface. The data are given through a specific
- structure called a descriptor, see [[sec:tuto_step2][Step2]].
- * *CHAMELEON_name_Tile_Async*: similar to the tile interface, it avoids
- synchonization barrier normally called between *Tile* routines.
- At the end of an *Async* function, completion of tasks is not
- guaranteed and data are not necessarily up-to-date. To ensure
- that tasks have been all executed, a synchronization function
- has to be called after the sequence of *Async* functions, see
- [[sec:tuto_step4][Step4]].
-
- CHAMELEON routine calls have to be preceded from
- #+begin_src
- CHAMELEON_Init( NCPU, NGPU );
- #+end_src
- to initialize CHAMELEON and the runtime system and followed by
- #+begin_src
- CHAMELEON_Finalize();
- #+end_src
- to free some data and finalize the runtime and/or MPI.
-
-*** Tutorial LAPACK to Chameleon
- <<sec:tuto>>
+ to free some data and finalize the runtime and/or MPI.
This tutorial is dedicated to the API usage of Chameleon. The
idea is to start from a simple code and step by step explain how
diff --git a/doc/user/homepage.org b/doc/user/homepage.org
new file mode 100644
index 0000000000000000000000000000000000000000..4604ba0f1440cdb66cc0772e8d35b749cb44b758
--- /dev/null
+++ b/doc/user/homepage.org
@@ -0,0 +1,578 @@
+#+TITLE: Chameleon
+#+SUBTITLE: A dense linear algebra software for heterogeneous architectures
+#+AUTHOR: HiePACS
+#+EXPORT_FILE_NAME: index.html
+#+LANGUAGE: en
+#+OPTIONS: H:3 num:t \n:nil @:t ::t |:t _:nil ^:nil -:t f:t *:t <:t
+#+OPTIONS: TeX:t LaTeX:t skip:nil d:nil pri:nil tags:not-in-toc html-style:nil
+# #+HTML_LINK_HOME: https://solverstack.gitlabpages.inria.fr/chameleon/
+#+SETUPFILE: https://mfelsoci.gitlabpages.inria.fr/inria-org-html-themes/online-theme-readtheorginria.setup
+
+* Overview
+ <<sec:overview>>
+
+ Chameleon is a framework written in C which provides routines to
+ solve dense general systems of linear equations, symmetric positive
+ definite systems of linear equations and linear least squares
+ problems, using LU, Cholesky, QR and LQ factorizations. Real
+ arithmetic and complex arithmetic are supported in both single
+ precision and double precision. It supports Linux and Mac OS/X
+ machines (mainly tested on Intel x86-64 and IBM Power
+ architectures). Chameleon is based on the PLASMA source code but
+ is not limited to shared-memory environment and can exploit
+ multiple GPUs. Chameleon is interfaced in a generic way with
+ StarPU, PaRSEC, QUARK, OpenMP runtime systems. This feature allows
+ to analyze in a unified framework how sequential task-based
+ algorithms behave regarding different runtime systems
+ implementations. Using Chameleon with StarPU or PaRSEC runtime
+ systems allows to exploit GPUs through kernels provided by cuBLAS
+ and clusters of interconnected nodes with distributed memory (using
+ MPI).
+
+ Main features:
+ * Written in C, Fortran interface, CMake build system
+ * Algorithms: GEMM, POTRF, GETRF, GEQRF, GESVD, ...
+ * Matrices forms: general, symmetric, triangular
+ * Precisions: simple, double, complex, double complex
+
+ #+ATTR_HTML: :alt DAG of Chameleon image :title DAG of Chameleon :align center :width 600px
+ [[./chameleon.svg]]
+
+* News
+ <<sec:news>>
+#+INCLUDE: "news.org"
+
+* Download
+ <<sec:download>>
+
+ Depending on how much you need to tune the library installation we
+ propose several solutions.
+ 1) You just want to have a try, to see if it can be installed well
+ on your system, what are the performances on simple cases, run
+ the examples, or simply use the last stable version: we recommand
+ to use one of our packages, .deb ones for those who work on a
+ Linux Debian or Ubuntu distribution, Guix or Spack on other Linux
+ systems, Brew on macOS.
+ 2) You want to use it but you need a change somewhere in the stack
+ like considering another version (git branch), change the default
+ BLAS/LAPACK or MPI, use your favorite compiler, modify some
+ sources: you may try with Guix or Spack because these package
+ managers allow to build from sources and thus many options can be
+ changed dynamically (command line), or directly build from source
+ with the native build system of the library (Makefile, GNU
+ autotools, CMake) following the procedures described in the
+ installation guide of the library, cf. [[sec:quickstartguide]].
+ 3) You need a tool for reproducibility of your experiments: Guix is
+ recommended.
+
+ #+ATTR_HTML: :alt Table of download materials image :title Table of download materials :align center
+ | Git | Release source | Debian/Ubuntu | [[https://brew.sh/][Brew]] (Mac) | [[https://guix.gnu.org/en/][Guix]] (Linux) | [[https://spack.readthedocs.io/en/latest/][Spack]] (Linux/Mac) |
+ |-----------+----------------+---------------+------------+--------------+-------------------|
+ | [[https://gitlab.inria.fr/solverstack/chameleon][Chameleon]] | [[https://gitlab.inria.fr/solverstack/chameleon/uploads/b299d6037d7636c6be16108c89bc2aab/chameleon-1.1.0.tar.gz][1.1.0]] | [[https://gitlab.inria.fr/solverstack/chameleon/-/packages][packages]] | [[https://gitlab.inria.fr/solverstack/brew-repo][brew-repo]] | [[https://gitlab.inria.fr/guix-hpc/guix-hpc-non-free][guix-repo]] | [[https://gitlab.inria.fr/solverstack/spack-repo][spack-repo]] |
+
+ Some packages are part of the official distribution and we just
+ provide the package name. For others we provide links where you can
+ find either a file to install or a package recipe that can be used
+ with Brew, Guix, Spack. If there are no package available for your
+ distribution please [[sec:contact][contact us]] and we will try to find a solution.
+
+ All these packages have been successfully installed and tested on
+ Unix systems: *Linux* (Debian testing, Ubuntu 20.04 LTS) and *macOS*
+ (Catalina).
+
+* Quick start guide
+ <<sec:quickstartguide>>
+
+ Here a quick starting guide for using Chameleon. For more
+ information please refer to the [[sec:documentation][full documentation]].
+
+** Install
+ In the following we present quick examples of installation of the
+ [[sec:download][packages]].
+*** Release source installation with CMake
+**** Linux Ubuntu 20.04
+ Start by installing common development tools
+ #+begin_src sh :eval never-export
+ sudo apt-get update -y
+ sudo apt-get install -y git cmake build-essential gfortran python wget tar curl pkg-config
+ #+end_src
+ - CBLAS/LAPACKE is required (OpenBLAS, Intel MKL, BLIS/FLAME, IBM
+ ESSL + Reference LAPACK for cblas/lapacke interface)
+ - we recommend to install StarPU as runtime system with MPI
+ enabled and optionally CUDA/cuBLAS if enabled on your system
+ #+begin_src sh :eval never-export
+ sudo apt-get install -y libopenblas-dev liblapacke-dev libhwloc-dev libopenmpi-dev libstarpu-dev
+ #+end_src
+ Remarks:
+ - The pair ~libopenblas-dev liblapacke-dev~ can be replaced by
+ ~libmkl-dev~.
+ - One can also use lib blis and flame but be sure to install a
+ cblas and lapacke, from the Reference LAPACK (with CBLAS=ON,
+ LAPACKE=ON), linked to blis/flame.
+ - The lib essl (IBM) can also be used as BLAS/LAPACK with the
+ Reference LAPACK providing cblas/lapacke.
+
+ Then to install Chameleon from sources with CMake, proceed as follows
+ #+begin_src sh :eval never-export
+ wget https://gitlab.inria.fr/solverstack/chameleon/uploads/b299d6037d7636c6be16108c89bc2aab/chameleon-1.1.0.tar.gz
+ tar xvf chameleon-1.1.0.tar.gz
+ cd chameleon-1.1.0
+ # or clone from git repository
+ # git clone --recursive https://gitlab.inria.fr/solverstack/chameleon.git && cd chameleon
+ mkdir build && cd build
+ cmake .. -DCHAMELEON_USE_MPI=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$PWD/install
+ make -j5 install
+ #+end_src
+
+ See more examples in the [[sec:ug:cmake][user's guide]].
+
+**** macOS
+ Start by installing common development tools
+ #+begin_src sh :eval never-export
+ # install Homebrew if not already available
+ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+
+ # install compiler and tools
+ brew install htop tmux gcc automake autoconf libtool doxygen emacs zlib bzip2 bison hwloc pkgconfig openblas openmpi
+
+ # gcc and g++ are missing (avoid to use clang version in /usr/bin)
+ ln -sf /usr/local/bin/gcc-11 /usr/local/bin/gcc
+ ln -sf /usr/local/bin/g++-11 /usr/local/bin/g++
+
+ # use pkg-config .pc files to detect some dependencies
+ export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/local/opt/openblas/lib/pkgconfig
+
+ # cmake checks blas.pc not openblas.pc
+ sudo cp /usr/local/opt/openblas/lib/pkgconfig/openblas.pc /usr/local/opt/openblas/lib/pkgconfig/blas.pc
+ #+end_src
+ - CBLAS/LAPACKE is required (OpenBLAS or Intel MKL)
+ - we recommend to install StarPU as runtime system with MPI
+ enabled
+ #+begin_src sh :eval never-export
+ # install last starpu release
+ cd $HOME
+ wget https://files.inria.fr/starpu/starpu-1.3.8/starpu-1.3.8.tar.gz
+ tar xvf starpu-1.3.8.tar.gz
+ cd starpu-1.3.8
+ ./configure
+ make -j5
+ sudo make install
+ #+end_src
+
+ Then to install Chameleon from sources with CMake, proceed as follows
+ #+begin_src sh :eval never-export
+ wget https://gitlab.inria.fr/solverstack/chameleon/uploads/b299d6037d7636c6be16108c89bc2aab/chameleon-1.1.0.tar.gz
+ tar xvf chameleon-1.1.0.tar.gz
+ cd chameleon-1.1.0
+ # or clone from git repository
+ # git clone --recursive https://gitlab.inria.fr/solverstack/chameleon.git && cd chameleon
+ mkdir build && cd build
+ cmake .. -DCHAMELEON_USE_MPI=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$PWD/install
+ make -j5 install
+ #+end_src
+*** Debian/Ubuntu packages
+ Visit the [[https://gitlab.inria.fr/solverstack/chameleon/-/packages][packages]] page and download the appropriate file for your
+ system. Then in a shell terminal
+ #+begin_src sh :eval never-export
+ # to install one of our non offical package, for example on Ubuntu 20.04
+ curl https://gitlab.inria.fr/solverstack/chameleon/-/package_files/15259/download -o chameleon_1.1.0-1_amd64.deb
+ sudo apt-get install -y ./chameleon_1.1.0-1_amd64.deb
+ #+end_src
+ If there are no package available for your Debian/Ubuntu
+ distribution please [[sec:contact][contact us]] and we will try to provide one.
+
+*** Brew packages
+ Brew packages for macOS are stored in our [[https://gitlab.inria.fr/solverstack/brew-repo][brew-repo]] git
+ repository. Please refer to the [[https://gitlab.inria.fr/solverstack/brew-repo/-/blob/master/README.org][README]] for installation
+ instructions.
+
+ Examples:
+ #+begin_src sh :eval never-export
+ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+ git clone https://gitlab.inria.fr/solverstack/brew-repo.git
+ brew install --build-from-source ./brew-repo/starpu.rb
+ brew install --build-from-source ./brew-repo/chameleon.rb
+ #+end_src
+
+*** Guix packages
+ [[https://guix.gnu.org/][Guix]] requires a running GNU/Linux system, GNU tar and Xz.
+ #+begin_src sh :eval never-export
+ gpg --keyserver pgp.mit.edu --recv-keys 3CE464558A84FDC69DB40CFB090B11993D9AEBB5
+ wget https://git.savannah.gnu.org/cgit/guix.git/plain/etc/guix-install.sh
+ chmod +x guix-install.sh
+ sudo ./guix-install.sh
+ #+end_src
+
+ Guix packages are stored in our [[https://gitlab.inria.fr/guix-hpc/guix-hpc][guix-hpc]], and [[https://gitlab.inria.fr/guix-hpc/guix-hpc-non-free][guix-hpc-non-free]]
+ (for versions with Intel MKL and/or CUDA) git repositories. Please
+ refer to the README to see how to add our package to the list of
+ Guix available packages (/i.e./ add a channel).
+
+ The package definitions in this [[https://gitlab.inria.fr/guix-hpc/guix-hpc-non-free][repo]] extend those that come with
+ Guix. To make them visible to the guix command-line tools, create
+ the ~~/.config/guix/channels.scm~ file with the following snippet to
+ request the guix-hpc channel:
+ #+begin_src :eval never-export
+ (cons (channel
+ (name 'guix-hpc-non-free)
+ (url "https://gitlab.inria.fr/guix-hpc/guix-hpc-non-free.git"))
+ %default-channels)
+ #+end_src
+ That way, ~guix pull~ will systematically pull not only Guix, but
+ also Guix-HPC-non-free and Guix-HPC.
+ #+begin_src sh :eval never-export
+ guix pull
+ #+end_src
+
+ Then to install Chameleon last release
+ #+begin_src sh
+ guix install chameleon
+ #+end_src
+
+ See more examples in the [[sec:ug:guix][user's guide]].
+
+*** Spack packages
+ Spack packages for Linux or macOS are stored in our [[https://gitlab.inria.fr/solverstack/spack-repo][spack-repo]] git
+ repository. Please refer to the [[https://gitlab.inria.fr/solverstack/spack-repo/-/blob/master/README.org][README]] for installation
+ instructions.
+
+ Examples:
+ #+begin_src sh :eval never-export
+ # please read https://spack.readthedocs.io/en/latest/getting_started.html
+ git clone https://github.com/llnl/spack.git
+ cd spack
+ git checkout v0.16.2
+ . share/spack/setup-env.sh
+ # Currently spack provides openmpi v3 as default openmpi. Add the
+ # following in your etc/spack/defaults/packages.yaml in order to set
+ # openmpi 4.0.5 to be the default:
+ # openmpi:
+ # version: [4.0.5]
+ git clone https://gitlab.inria.fr/solverstack/spack-repo.git
+ spack repo add spack-repo
+ spack install chameleon
+ # chameleon is installed here:
+ spack location -i chameleon
+ #+end_src
+ Spack allows to expose many build variants so that it is difficult
+ to ensure that all installations will succeed.
+
+ See more examples in the [[sec:ug:spack][user's guide]].
+
+** Linking
+ If you do not use CMake we provide a pkg-config file at
+ installation in the subdirectory ~lib/pkgconfig~.
+ #+begin_src sh :eval never-export
+ # lets CHAMELEON_ROOT be the installation path
+ export PKG_CONFIG_PATH=$CHAMELEON_ROOT/lib/pkgconfig:$PKG_CONFIG_PATH
+ pkg-config --cflags chameleon
+ pkg-config --libs chameleon
+ pkg-config --libs --static chameleon
+
+ export LD_LIBRARY_PATH=$CHAMELEON_ROOT/lib:$LD_LIBRARY_PATH
+ #+end_src
+
+ If you build your project with CMake we provide a
+ ~CHAMELEONConfig.cmake~ file at installation, in the subdirectory
+ ~lib/cmake/~ of the installation. Example of ~CMakeLists.txt~ for
+ Chameleon
+ #+begin_src :eval never-export
+ project(CHAMELEON_EXAMPLE C Fortran)
+
+ # to be able to use CHAMELEON_ROOT env. var.
+ cmake_policy(SET CMP0074 NEW)
+
+ # look for CHAMELEON on the system
+ # Hint: use CHAMELEON_ROOT (env. var. or cmake var.) to the installation directory of
+ # CHAMELEON if not installed in a standard path
+
+ find_package(CHAMELEON REQUIRED)
+
+ # compile your example
+
+ add_executable(chameleon_example chameleon_example.c)
+
+ # link to chameleon
+
+ target_link_libraries(chameleon_example PRIVATE CHAMELEON::chameleon)
+ #+end_src
+** Using
+ Considering that the ~bin/~ directory of the Chameleon installation
+ is in the ~PATH~, the testing executables can be used to check main
+ linear algebra operations such as 'gemm', 'potrf', 'getrf',
+ 'geqrf', 'gels', etc
+ #+begin_src sh :eval never-export
+ chameleon_stesting -H -o gemm -t 2 -m 2000 -n 2000 -k 2000
+ #+end_src
+
+ See the options with
+ #+begin_src sh :eval never-export
+ chameleon_stesting -h
+ #+end_src
+
+ See the available linear algebra operations with
+ #+begin_src sh :eval never-export
+ chameleon_stesting -o help
+ #+end_src
+
+ Remarks:
+ - If using OpenBLAS multithreaded ensure to set
+ ~OPENBLAS_NUM_THREADS=1~ because Chameleon handles mutithreading
+ directly
+ - same for Intel MKL, ensure to set ~MKL_NUM_THREADS=1~
+
+ Here an example of linear system solving written in C through a
+ Cholesky factorization on a SPD matrix with *LAPACK format*
+ #+begin_src C :eval never-export
+ #include <chameleon.h>
+ #include <stdlib.h>
+ int main(void)
+ {
+ size_t N; // matrix order
+ size_t NRHS; // number of RHS vectors
+ int NCPU; // number of cores to use
+ int NGPU; // number of gpus (cuda devices) to use
+ int UPLO = ChamUpper; // where is stored L
+
+ int major, minor, patch;
+ CHAMELEON_Version(&major, &minor, &patch);
+
+ /* Linear system parameters */
+ N = 1000;
+ NRHS = 10;
+
+ /* Initialize the number of CPUs to be used with threads */
+ NCPU = 2;
+ NGPU = 0;
+
+ /* Initialize CHAMELEON with main parameters */
+ CHAMELEON_Init( NCPU, NGPU );
+
+ /*
+ * allocate memory for our data
+ * - matrix A : size N x N
+ * - set of RHS vectors B : size N x NRHS
+ * - set of solutions vectors X : size N x NRHS
+ */
+ double *A = malloc( N * N * sizeof(double) );
+ double *B = malloc( N * NRHS * sizeof(double) );
+ double *X = malloc( N * NRHS * sizeof(double) );
+
+ /* generate A matrix with random values such that it is spd */
+ CHAMELEON_dplgsy( (double)N, ChamUpperLower, N, A, N, 51 );
+
+ /* generate RHS */
+ CHAMELEON_dplrnt( N, NRHS, B, N, 5673 );
+
+ /* copy B in X before solving */
+ memcpy(X, B, N*NRHS*sizeof(double));
+
+ /************************************************************/
+ /* solve the system AX = B using the Cholesky factorization */
+ /************************************************************/
+
+ /* Cholesky facorization:
+ * A is replaced by its factorization L or L^T depending on uplo */
+ CHAMELEON_dpotrf( UPLO, N, A, N );
+
+ /* Solve:
+ * B is stored in X on entry, X contains the result on exit.
+ * Forward and back substitutions
+ */
+ CHAMELEON_dpotrs(UPLO, N, NRHS, A, N, X, N);
+
+ /* deallocate data */
+ free(A);
+ free(B);
+ free(X);
+
+ /* Finalize CHAMELEON */
+ CHAMELEON_Finalize();
+
+ return EXIT_SUCCESS;
+ }
+ #+end_src
+ In this example the LAPACK matrix is internally converted into
+ Chameleon /tiled/ matrix format then task-based algorithms can be
+ called. The copy operation can be costly. Please consider learning
+ how to work directly with the Chameleon /tiled/ matrix format to get
+ faster executions and the ability to handle distributed matrices
+ over several machines. The user's data can be given in several way
+ to fill the Chameleon /tiled/ matrix, see [[sec:tuto]].
+
+ Here a simple example of linear system solving written in C through
+ a Cholesky factorization on a SPD matrix with *Chameleon format*
+ #+begin_src C :eval never-export
+ #include <chameleon.h>
+ #include <stdlib.h>
+ int main(void)
+ {
+ size_t N; // matrix order
+ size_t NRHS; // number of RHS vectors
+ int NCPU; // number of cores to use
+ int NGPU; // number of gpus (cuda devices) to use
+ int UPLO = ChamUpper; // where is stored L
+
+ /* descriptors necessary for calling CHAMELEON tile interface */
+ CHAM_desc_t *descA = NULL, *descB = NULL, *descX = NULL;
+
+ int major, minor, patch;
+ CHAMELEON_Version(&major, &minor, &patch);
+
+ /* Linear system parameters */
+ N = 1000;
+ NRHS = 10;
+
+ /* Initialize the number of CPUs to be used with threads */
+ NCPU = 2;
+ NGPU = 0;
+
+ /* Initialize CHAMELEON with main parameters */
+ CHAMELEON_Init( NCPU, NGPU );
+
+ /*
+ * Initialize the structure required for CHAMELEON tile interface
+ * CHAM_desc_t is a structure wrapping your data allowing CHAMELEON to get
+ * pointers to tiles. A tile is a data subset of your matrix on which we
+ * apply some optimized CPU/GPU kernels.
+ * Notice that this routine suppose your matrix is a contiguous vector of
+ * data (1D array), as a data you would give to BLAS/LAPACK.
+ * Main arguments:
+ * - descA is a pointer to a descriptor, you need to give the address
+ * of this pointer
+ * - if you want to give your allocated matrix give its address,
+ * if not give a NULL pointer, the routine will allocate the memory
+ * and you access the matrix data with descA->mat
+ * - give the data type (ChamByte, ChamInteger, ChamRealFloat,
+ * ChamRealDouble, ChamComplexFloat, ChamComplexDouble)
+ * - number of rows in a block (tile)
+ * - number of columns in a block (tile)
+ * - number of elements in a block (tile)
+ * The other parameters are specific, use:
+ * CHAMELEON_Desc_Create( ... , 0, 0, number of rows, number of columns, 1, 1);
+ * Have a look to the documentation for details about these parameters.
+ */
+ CHAMELEON_Desc_Create(&descA, NULL, ChamRealDouble,
+ NB, NB, NB*NB, N, N, 0, 0, N, N, 1, 1);
+ CHAMELEON_Desc_Create(&descB, NULL, ChamRealDouble,
+ NB, NB, NB*NB, N, NRHS, 0, 0, N, NRHS, 1, 1);
+ CHAMELEON_Desc_Create(&descX, NULL, ChamRealDouble,
+ NB, NB, NB*NB, N, NRHS, 0, 0, N, NRHS, 1, 1);
+
+ /* generate A matrix with random values such that it is spd */
+ CHAMELEON_dplgsy_Tile( (double)N, ChamUpperLower, descA, 51 );
+
+ /* generate RHS */
+ CHAMELEON_dplrnt_Tile( descB, 5673 );
+
+ /* copy B in X before solving */
+ CHAMELEON_dlacpy_Tile(ChamUpperLower, descB, descX);
+
+ /************************************************************/
+ /* solve the system AX = B using the Cholesky factorization */
+ /************************************************************/
+
+ /* Cholesky facorization:
+ * A is replaced by its factorization L or L^T depending on uplo */
+ CHAMELEON_dpotrf_Tile( UPLO, descA );
+
+ /* Solve:
+ * B is stored in X on entry, X contains the result on exit.
+ * Forward and back substitutions
+ */
+ CHAMELEON_dpotrs_Tile( UPLO, descA, descX );
+
+ /* deallocate data */
+ CHAMELEON_Desc_Destroy( &descA );
+ CHAMELEON_Desc_Destroy( &descB );
+ CHAMELEON_Desc_Destroy( &descX );
+
+ /* Finalize CHAMELEON */
+ CHAMELEON_Finalize();
+
+ return EXIT_SUCCESS;
+ }
+ #+end_src
+
+* Documentation
+ <<sec:documentation>>
+#+INCLUDE: "users_guide.org"
+* Tutorials
+ <<sec:tutorials>>
+
+ * [[sec:usetesting][Using Chameleon executables]]
+ * [[sec:tuto][Tutorial LAPACK to Chameleon]]
+ * [[https://fpruvost.gitlabpages.inria.fr/exa2pro/][Exa2pro: Chameleon usage on a supercomputer]]
+
+* Contact
+ <<sec:contact>>
+
+ If you have an account on [[https://gitlab.inria.fr/][gitlab inria]] please submit a [[https://gitlab.inria.fr/solverstack/chameleon/-/issues][new issue]].
+
+ If you don't have an account on [[https://gitlab.inria.fr/solverstack/chameleon/-/issues][gitlab inria]] you can send emails to
+ [[mailto:chameleon-issues@inria.fr][chameleon-issues@inria.fr]].
+
+ To get the news, register to the mailing list
+ [[https://sympa.inria.fr/sympa/info/chameleon-announce][chameleon-announce@inria.fr]] (click on "S'abonner" on the left
+ panel).
+
+* Get involved!
+#+INCLUDE: "CONTRIBUTING.org"
+
+* Authors
+
+ First, since the Chameleon library started as an extension of the
+ PLASMA library to support multiple runtime systems, all developpers
+ of the PLASMA library are developpers of the Chameleon library.
+
+ The following people contributed to the development of Chameleon:
+ * Emmanuel Agullo, PI
+ * Olivier Aumage
+ * Cedric Castagnede
+ * Terry Cojean
+ * Mathieu Faverge, PI
+ * Nathalie Furmento
+ * Reazul Hoque
+ * Hatem Ltaief
+ * Gregoire Pichon
+ * Florent Pruvost, PI
+ * Marc Sergent
+ * Guillaume Sylvand
+ * Samuel Thibault
+ * Stanimire Tomov
+ * Omar Zenati
+
+ If we forgot your name, please let us know that we can fix that mistake.
+
+* Citing Chameleon
+
+Feel free to use the following publications to reference Chameleon:
+
+ * Original paper that initiated Chameleon and the principles:
+ - Agullo, Emmanuel and Augonnet, Cédric and Dongarra, Jack and
+ Ltaief, Hatem and Namyst, Raymond and Thibault, Samuel and Tomov,
+ Stanimire, *Faster, Cheaper, Better -- a Hybridization Methodology
+ to Develop Linear Algebra Software for GPUs*, /GPU Computing Gems/,
+ [[https://hal.inria.fr/inria-00547847][First Online: 17 December 2010]].
+ * Design of the QR algorithms:
+ - Agullo, Emmanuel and Augonnet, Cédric and Dongarra, Jack and
+ Faverge, Mathieu and Ltaief, Hatem and Thibault, Samuel an
+ Tomov, Stanimire, *QR Factorization on a Multicore Node Enhanced
+ with Multiple GPU Accelerators*, /25th IEEE International Parallel
+ & Distributed Processing Symposium/, [[https://hal.inria.fr/inria-00547614][First Online: 16 December
+ 2010]].
+ * Design of the LU algorithms:
+ - Agullo, Emmanuel and Augonnet, Cédric and Dongarra, Jack and
+ Faverge, Mathieu and Langou, Julien and Ltaief, Hatem and Tomov,
+ Stanimire, *LU Factorization for Accelerator-based Systems*,
+ /9th ACS/IEEE International Conference on Computer Systems and
+ Applications (AICCSA 11)/, [[https://hal.inria.fr/hal-00654193][First Online: 21 December 2011]].
+ * Regarding distributed memory:
+ - Agullo, Emmanuel and Aumage, Olivier and Faverge, Mathieu and
+ Furmento, Nathalie and Pruvost, Florent and Sergent, Marc and
+ Thibault, Samuel, *Achieving High Performance on Supercomputers
+ with a Sequential Task-based Programming Model*, /Research Report/,
+ [[https://hal.inria.fr/hal-01332774][First Online: 16 June 2016]].
+
+* Licence
+ [[https://gitlab.inria.fr/solverstack/chameleon/-/blob/master/LICENCE.txt][LICENCE]]
diff --git a/doc/user/news.org b/doc/user/news.org
new file mode 100644
index 0000000000000000000000000000000000000000..488f4af81b1ac75014db45fcc45b3e54cbc41e7d
--- /dev/null
+++ b/doc/user/news.org
@@ -0,0 +1,6 @@
+** [2021-04-21] *Release 1.1.0* available
+ * See [[https://gitlab.inria.fr/solverstack/chameleon/-/releases/v1.1.0][changes]]
+ * [[https://gitlab.inria.fr/solverstack/chameleon/uploads/b299d6037d7636c6be16108c89bc2aab/chameleon-1.1.0.tar.gz][Download source tarball]]
+** [2021-04-06] Chameleon integration into FMR
+ Chameleon has been successfully integrated into the C++ Randomized
+ SVD library [[https://gitlab.inria.fr/compose/oldstack/fmr][FMR]].
diff --git a/doc/user/publish.el b/doc/user/publish.el
new file mode 100644
index 0000000000000000000000000000000000000000..7a2325217e6f73b15790b74a0cf8521e72d0f750
--- /dev/null
+++ b/doc/user/publish.el
@@ -0,0 +1,29 @@
+;; publish.el
+;; Emacs publish file for project.
+;; Run the following command to execute:
+;; emacs --batch --no-init-file --load publish.el --funcall org-publish-all
+
+;; Packages:
+(require 'package)
+(require 'ox-publish)
+(require 'org)
+(require 'htmlize)
+
+(setq org-html-htmlize-output-type 'css)
+(setq org-src-fontify-natively t)
+(org-babel-do-load-languages
+ 'org-babel-load-languages
+ '(
+ (C . t)
+ (fortran . t)
+ (python . t)
+ (shell . t)
+ ))
+
+(setq org-publish-project-alist
+ `(("homepage"
+ :base-directory "."
+ :base-extension "org"
+ :publishing-directory "."
+ :publishing-function org-html-publish-to-html
+ )))
diff --git a/doc/user/users_guide.org.in b/doc/user/users_guide.org.in
new file mode 100644
index 0000000000000000000000000000000000000000..106be47a5d30f8b5dbcea05d44ab830de9fa85b9
--- /dev/null
+++ b/doc/user/users_guide.org.in
@@ -0,0 +1,55 @@
+#+INCLUDE: "version.org"
+ This is the users guide to Chameleon. The software ecosystem will be
+ presented, the installation instructions detailed and some usage
+ examples are presented. To get more information about the
+ application programming interface, please refer to the [[https://solverstack.gitlabpages.inria.fr/chameleon/dev/index.html][doxygen
+ documentation]].
+
+** Version
+ This manual documents the usage of Chameleon *version {{{VERSION}}}*.
+ It was last updated on {{{UPDATED}}}.
+** Authors
+ * Inria
+ * University of Tennessee
+ * University of Colorado Denver
+ * King Abdullah University of Science and Technology
+** Copying
+
+ * Copyright \copy {{{UPDATED-YEAR}}} Inria
+ * Copyright \copy 2014 The University of Tennessee
+ * Copyright \copy 2014 King Abdullah University of Science and Technology
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer listed
+ in this license in the documentation and/or other materials provided
+ with the distribution.
+ - Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ This software is provided by the copyright holders and contributors
+ "as is" and any express or implied warranties, including, but not
+ limited to, the implied warranties of merchantability and fitness for
+ a particular purpose are disclaimed. In no event shall the copyright
+ owner or contributors be liable for any direct, indirect, incidental,
+ special, exemplary, or consequential damages (including, but not
+ limited to, procurement of substitute goods or services; loss of use,
+ data, or profits; or business interruption) however caused and on any
+ theory of liability, whether in contract, strict liability, or tort
+ (including negligence or otherwise) arising in any way out of the use
+ of this software, even if advised of the possibility of such damage.
+
+** Introduction to Chameleon
+ #+INCLUDE: @CMAKE_CURRENT_SOURCE_DIR@/chapters/introduction.org
+** Installing Chameleon
+ <<sec:ug:install>>
+ #+INCLUDE: @CMAKE_CURRENT_SOURCE_DIR@/chapters/installing.org
+** Using Chameleon
+ <<sec:ug:using>>
+ #+INCLUDE: @CMAKE_CURRENT_SOURCE_DIR@/chapters/using.org
diff --git a/doc/orgmode/version.org.in b/doc/user/version.org.in
similarity index 100%
rename from doc/orgmode/version.org.in
rename to doc/user/version.org.in
diff --git a/tools/pages.sh b/tools/pages.sh
index f7be64c651dc28f187569243170dfabfafb8cd97..251e472a33ba9c53c0ede3d3bbf2402851d405a7 100755
--- a/tools/pages.sh
+++ b/tools/pages.sh
@@ -27,8 +27,8 @@ Rscript ${CHAMELEON_SRC_DIR}/tools/bench/jube/GenFigures.R
# add the performance chapter to the doc. we need the performances files to add
# this chapter that are not commited in the sources so that this chapter is not
# here by default
-cat >> ${CHAMELEON_SRC_DIR}/doc/orgmode/users_guide.org.in <<EOF
-* Chameleon Performances on PlaFRIM
+cat >> ${CHAMELEON_SRC_DIR}/doc/user/users_guide.org.in <<EOF
+** Chameleon Performances on PlaFRIM
Chameleon commit: *$commit_sha*.
#+INCLUDE: @CMAKE_CURRENT_SOURCE_DIR@/chapters/performances.org
EOF
@@ -43,19 +43,15 @@ cd build-$VERSION
cmake $CHAMELEON_SRC_DIR -DCHAMELEON_ENABLE_DOC=ON
make doc -j5
+## Copy files in public/ used as an artefact (zip archive) to upload on gitlab pages, see
+## Homepage: https://solverstack.gitlabpages.inria.fr/chameleon/index.html
+## API: https://solverstack.gitlabpages.inria.fr/chameleon/dev/index.html
cd ..
-mv build-$VERSION/doc/orgmode public/
-mv build-$VERSION/doc/doxygen/out/html public/doxygen
-mv tmp_fig/*.png public/
-
-cd public/
-if [ -f users_guide.html ]
-then
- ln -sfn users_guide.html index.html
-else
- echo -e "ERROR: missing users_guide.html file"
- exit 1
-fi
-
-# Change the width of the page in the CSS file
-sed -i -e "s#max-width:800px#max-width:1800px#" org-html-themes/styles/readtheorg/css/readtheorg.css
+mkdir public/
+mkdir public/dev/
+cp build-$VERSION/doc/user/*.html public/
+cp build-$VERSION/doc/user/*.png public/
+cp build-$VERSION/doc/user/*.jpg public/
+cp build-$VERSION/doc/user/*.svg public/
+cp -r build-$VERSION/doc/dev/html/* public/dev/
+cp tmp_fig/* public/