debug mpi from tinker branch feedback (it does not include the tinker part)

Examples/RotationMPIFMM.cpp

@@ -164,6 +164,8 @@ int main(int argc, char* argv[])
                                                                     tree.getBoxWidth(),
                                                                     tree.getHeight(), &finalParticles,&balancer);
 
+        std::cout << "Local nb particles after sort "
+                  << finalParticles.getSize() << std::endl;
         for(FSize idx = 0 ; idx < finalParticles.getSize(); ++idx){
             tree.insert(finalParticles[idx].position,finalParticles[idx].indexInFile,finalParticles[idx].physicalValue);
         }

Licence.txt

@@ -4,6 +4,9 @@ the more restrictive has to be used.
 ScalFmm est régi par la licence CeCILL-C & LGPL, en cas de conflit
 la plus restrictive prime.
 
+Folders under Addons might have separate Licence, in such case
+one can find a dedicated Licence file where appropriate.
+
 //////////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////////
 

Src/BalanceTree/FEqualize.hpp

@@ -81,7 +81,10 @@ public:
             pack.elementTo   = Min(allObjectives[idxProc].second , myCurrentInterval.second) - myCurrentInterval.first;
             // Next time give from the previous end
             currentElement   = pack.elementTo;
-            packToSend.push_back(pack);
+
+            if(pack.elementTo - pack.elementFrom != 0){
+                packToSend.push_back(pack);
+            }
             // Progress
             idxProc += 1;
         }

Src/Containers/FOctree.hpp

@@ -1685,6 +1685,10 @@ public:
      * @param function
      */
     void forEachLeaf(std::function<void(LeafClass*)> function){
+        if(isEmpty()){
+            return;
+        }
+
         Iterator octreeIterator(this);
         octreeIterator.gotoBottomLeft();
 
@@ -1698,6 +1702,10 @@ public:
      * @param function
      */
     void forEachCell(std::function<void(CellClass*)> function){
+        if(isEmpty()){
+            return;
+        }
+
         Iterator octreeIterator(this);
         octreeIterator.gotoBottomLeft();
 
@@ -1717,6 +1725,10 @@ public:
      * @param function
      */
     void forEachCellWithLevel(std::function<void(CellClass*,const int)> function){
+        if(isEmpty()){
+            return;
+        }
+
         Iterator octreeIterator(this);
         octreeIterator.gotoBottomLeft();
 
@@ -1736,6 +1748,10 @@ public:
      * @param function
      */
     void forEachCellLeaf(std::function<void(CellClass*,LeafClass*)> function){
+        if(isEmpty()){
+            return;
+        }
+
         Iterator octreeIterator(this);
         octreeIterator.gotoBottomLeft();
 

Src/Utils/FMpi.hpp

@@ -107,7 +107,7 @@ public:
      * This class is used to gather the usual methods related to identifying an
      * MPI communicator.
      */
-    class FComm : public FNoCopyable {
+    class FComm {
         int rank;   ///< rank related to the comm
         int nbProc; ///< nb proc in this group
 
@@ -130,6 +130,26 @@ public:
             reset();
         }
 
+        /// Constructor : duplicates the given communicator
+        FComm(const FComm& inCommunicator ) {
+            FMpi::Assert( MPI_Comm_dup(inCommunicator.communicator, &communicator),  __LINE__ , "comm dup");
+            FMpi::Assert( MPI_Comm_group(communicator, &group),  __LINE__ , "comm group");
+
+            reset();
+        }
+
+        FComm& operator=(const FComm& inCommunicator ) {
+            FMpi::Assert( MPI_Comm_free(&communicator),  __LINE__ );
+            FMpi::Assert( MPI_Group_free(&group),  __LINE__ );
+
+            FMpi::Assert( MPI_Comm_dup(inCommunicator.communicator, &communicator),  __LINE__ , "comm dup");
+            FMpi::Assert( MPI_Comm_group(communicator, &group),  __LINE__ , "comm group");
+
+            reset();
+
+            return *this;
+        }
+
         /// Frees communicator and group
         virtual ~FComm(){
             FMpi::Assert( MPI_Comm_free(&communicator),  __LINE__ );
@@ -248,6 +268,35 @@ public:
             delete[]  procsIdArray ;
         }
 
+        /** Change the group, create one groupd where processInGroup[i] != 0
+          * and another where processInGroup[i] == 0
+          */
+        void groupReduce(const int processInGroup[]){
+            int * procsIdArray = new int [nbProc];
+            int counterNewGroup = 0;
+            for(int idxProc = 0 ;idxProc < nbProc ; ++idxProc){
+                if(processInGroup[rank] && processInGroup[idxProc]){
+                    procsIdArray[counterNewGroup++] = idxProc;
+                }
+                else if(!processInGroup[rank] && !processInGroup[idxProc]){
+                    procsIdArray[counterNewGroup++] = idxProc;
+                }
+            }
+
+            MPI_Group previousGroup = group;
+            FMpi::Assert( MPI_Group_incl(previousGroup, counterNewGroup , procsIdArray, &group),  __LINE__ );
+
+            MPI_Comm previousComm = communicator;
+            FMpi::Assert( MPI_Comm_create(previousComm, group, &communicator),  __LINE__ );
+
+            MPI_Comm_free(&previousComm);
+            MPI_Group_free(&previousGroup);
+
+            reset();
+            FAssertLF(nbProc == counterNewGroup);
+            delete[]  procsIdArray ;
+        }
+
         void barrier() const {
             FMpi::Assert(MPI_Barrier(getComm()), __LINE__);
         }

Src/Utils/FQuickSortMpi.hpp

@@ -93,14 +93,18 @@ class FQuickSortMpi : public FQuickSort< SortType, IndexType> {
         {
             // Get the number of elements each proc should recv
             IndexType totalRemainingElements = totalElements;
+            IndexType totalAvailableElements = totalElementToProceed;
+
 
             for(int idxProc = firstProcToRecv; idxProc < lastProcToRecv ; ++idxProc){
                 const IndexType nbElementsAlreadyOwned = (inFromRightToLeft ? globalElementBalance[idxProc].lowerPart : globalElementBalance[idxProc].greaterPart);
                 const IndexType averageNbElementForRemainingProc = (totalRemainingElements)/(lastProcToRecv-idxProc);
                 totalRemainingElements -= nbElementsAlreadyOwned;
                 FAssertLF(totalRemainingElements >= 0);
-                if(nbElementsAlreadyOwned < averageNbElementForRemainingProc){
-                    nbElementsToRecvPerProc[idxProc - firstProcToRecv] = (averageNbElementForRemainingProc - nbElementsAlreadyOwned);
+                if(nbElementsAlreadyOwned < averageNbElementForRemainingProc && totalAvailableElements){
+                    nbElementsToRecvPerProc[idxProc - firstProcToRecv] = FMath::Min(totalAvailableElements,
+                                                                                    averageNbElementForRemainingProc - nbElementsAlreadyOwned);
+                    totalAvailableElements -= nbElementsToRecvPerProc[idxProc - firstProcToRecv];
                     totalRemainingElements -= nbElementsToRecvPerProc[idxProc - firstProcToRecv];
                 }
                 else{
@@ -308,6 +312,10 @@ class FQuickSortMpi : public FQuickSort< SortType, IndexType> {
                     counterValuesInPivot += 1;
                 }
             }
+            if(counterValuesInPivot <= 1){
+                (*shouldStop) = true;
+                return globalPivot;
+            }
             (*shouldStop) = false;
             return globalPivot/counterValuesInPivot;
         }
@@ -336,7 +344,7 @@ public:
                 break;
             }
 
-            FLOG(if(VerboseLog)  FLog::Controller << "SCALFMM-DEBUG ["  << currentComm.processId() << "] globalPivot = " << globalPivot << "\n" );
+            FLOG(if(VerboseLog)  FLog::Controller << "SCALFMM-DEBUG ["  << currentComm.processId() << "] globalPivot = " << globalPivot << " for " << currentComm.processCount() << "\n" );
             FLOG(if(VerboseLog)  FLog::Controller.flush());
 
             // Split the array in two parts lower equal to pivot and greater than pivot

Tests/Utils/testPartitionsMapping.cpp

+// ===================================================================================
+// Copyright ScalFmm 2011 INRIA, Olivier Coulaud, Berenger Bramas
+// olivier.coulaud@inria.fr, berenger.bramas@inria.fr
+// This software is a computer program whose purpose is to compute the FMM.
+//
+// This software is governed by the CeCILL-C and LGPL licenses and
+// abiding by the rules of distribution of free software.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public and CeCILL-C Licenses for more details.
+// "http://www.cecill.info".
+// "http://www.gnu.org/licenses".
+// ===================================================================================
+
+// ==== CMAKE =====
+// @FUSE_MPI
+// ================
+
+#include <iostream>
+
+#include <cstdio>
+#include <cstdlib>
+
+
+#include "../../Src/Kernels/Rotation/FRotationCell.hpp"
+#include "../../Src/Kernels/Rotation/FRotationKernel.hpp"
+
+#include "../../Src/Components/FSimpleLeaf.hpp"
+#include "../../Src/Kernels/P2P/FP2PParticleContainerIndexed.hpp"
+
+#include "../../Src/Utils/FParameters.hpp"
+#include "../../Src/Utils/FMemUtils.hpp"
+#include "../../Src/BalanceTree/FPartitionsMapping.hpp"
+
+#include "../../Src/Containers/FOctree.hpp"
+#include "../../Src/Containers/FVector.hpp"
+
+#include "../../Src/Files/FRandomLoader.hpp"
+#include "../../Src/Files/FMpiTreeBuilder.hpp"
+
+#include "../../Src/Core/FFmmAlgorithm.hpp"
+#include "../../Src/Core/FFmmAlgorithmThread.hpp"
+#include "../../Src/Core/FFmmAlgorithmThreadProc.hpp"
+
+#include "../../Src/BalanceTree/FLeafBalance.hpp"
+
+#include "../../Src/Utils/FParameterNames.hpp"
+
+/**
+ * This program runs the FMM Algorithm Distributed with the Rotation kernel
+ */
+
+// Simply create particles and try the kernels
+int main(int argc, char* argv[])
+{
+    FHelpDescribeAndExit(argc, argv,
+                         "Test with MPI the chebyshev FMM and compare it to the direct computation for debugging purpose.",
+                         FParameterDefinitions::NbParticles, FParameterDefinitions::OctreeHeight,
+                         FParameterDefinitions::OctreeSubHeight, FParameterDefinitions::NbThreads);
+
+    typedef double FReal;
+
+    FMpi app(argc,argv);
+
+    const FSize nbParticles       = FParameters::getValue(argc,argv, FParameterDefinitions::NbParticles.options, 10000000ULL);
+    const unsigned int TreeHeight    = FParameters::getValue(argc, argv, FParameterDefinitions::OctreeHeight.options, 5);
+    FTic time;
+
+    std::cout << ">> This executable has to be used to test Proc Rotation Algorithm. \n";
+
+    // init particles position and physical value
+    struct TestParticle{
+        FPoint<FReal> position;
+        const FPoint<FReal>& getPosition(){
+            return position;
+        }
+    };
+
+    // open particle file
+    std::cout << "Creating : " << nbParticles << "\n" << std::endl;
+    FRandomLoader<FReal> loader(nbParticles, 1.0, FPoint<FReal>(0,0,0), app.global().processId());
+
+    time.tic();
+    std::unique_ptr<TestParticle[]> particles(new TestParticle[loader.getNumberOfParticles()]);
+    for(FSize idxPart = 0 ; idxPart < loader.getNumberOfParticles() ; ++idxPart){
+        loader.fillParticle(&particles[idxPart].position);
+    }
+
+    FPartitionsMapping<FReal> map(app.global());
+
+    FVector<FPartitionsMapping<double>::TestParticle<0> > finalParticles = map.distributeParticles<0>(loader.getNumberOfParticles(),
+                                                                   loader.getCenterOfBox(),
+                                                                   loader.getBoxWidth(), TreeHeight,
+                                                                   [&](const int idx, FPoint<FReal>* position, std::array<FReal, 0>* /*val*/){
+        position->setPosition(particles[idx].position.getX(),
+                              particles[idx].position.getY(),
+                              particles[idx].position.getZ());
+    });
+
+    // Test every particles exist
+    {
+        std::unique_ptr<int[]> count(new int[app.global().processCount() * nbParticles]);
+        memset(count.get(), 0, sizeof(int) * app.global().processCount() * nbParticles);
+        for(FSize part = 0 ; part < finalParticles.getSize() ; ++part){
+            const FSize idx = finalParticles[part].initialProcOwner*nbParticles + finalParticles[part].localIndex;
+            FAssertLF(count[idx] == 0)
+            count[idx] = 1;
+        }
+
+        FMpi::Assert( MPI_Allreduce(MPI_IN_PLACE, count.get(), app.global().processCount()*nbParticles, MPI_INT, MPI_SUM,
+                      app.global().getComm()), __LINE__);
+
+        for(FSize part = 0 ; part < app.global().processCount()*nbParticles ; ++part){
+            FAssertLF(count[part] == 1);
+        }
+    }
+
+    // Test to send data
+    {
+        std::unique_ptr<std::array<FReal, 2>[]> toDistr(new std::array<FReal, 2>[nbParticles]);
+        for(FSize idxPart = 0 ; idxPart < loader.getNumberOfParticles() ; ++idxPart){
+            toDistr[idxPart][0] = app.global().processId();
+            toDistr[idxPart][1] = FReal(idxPart);
+        }
+
+        std::unique_ptr<std::array<FReal, 2>[]> res = map.distributeData<2>(toDistr.get());
+
+        for(FSize part = 0 ; part < finalParticles.getSize() ; ++part){
+            if(int(res[part][0]) != finalParticles[part].initialProcOwner)
+                    std::cout << "[" << app.global().processId() << "]Res proc is " << res[part][0] << " should be " << finalParticles[part].initialProcOwner << std::endl;
+            if(int(res[part][1]) != finalParticles[part].localIndex)
+                    std::cout << "[" << app.global().processId() << "]Res localidx is " << res[part][1] << " should be " << finalParticles[part].localIndex << std::endl;
+        }
+
+        std::unique_ptr<std::array<FReal, 2>[]> resBack = map.getResultingData<2>(res.get());
+
+        for(FSize part = 0 ; part < loader.getNumberOfParticles() ; ++part){
+            if(int(resBack[part][0]) != app.global().processId())
+                    std::cout << "[" << app.global().processId() << "]ResBack proc is " << resBack[part][0] << " should be " << app.global().processId() << std::endl;
+            if(int(resBack[part][1]) != part)
+                    std::cout << "[" << app.global().processId() << "]ResBack localidx is " << resBack[part][1] << " should be " << part << std::endl;
+        }
+    }
+
+    return 0;
+}
+