Commit a67c5340 authored by berenger-bramas's avatar berenger-bramas
Browse files

Work in progress on the parallel loading using quick sort.

git-svn-id: svn+ssh://scm.gforge.inria.fr/svn/scalfmm/scalfmm/trunk@155 2616d619-271b-44dc-8df4-d4a8f33a7222
parent 51ca3ed6
This diff is collapsed.
......@@ -12,6 +12,7 @@
#include "../Src/Utils/F3DPosition.hpp"
#include "../Src/Utils/FAbstractSendable.hpp"
#include "../Src/Utils/FQuickSort.hpp"
#include "../Src/Components/FFmaParticle.hpp"
#include "../Src/Components/FTestParticle.hpp"
......@@ -265,6 +266,10 @@ typedef FFmmAlgorithmThreadProc<OctreeClass, ParticleClass, CellClass, Container
struct IndexedParticle{
MortonIndex index;
ParticleClass particle;
operator MortonIndex(){
return this->index;
}
};
long getTreeCoordinate(const FReal inRelativePosition, const FReal boxWidthAtLeafLevel) {
......@@ -276,34 +281,6 @@ long getTreeCoordinate(const FReal inRelativePosition, const FReal boxWidthAtLea
return index;
}
long partition(IndexedParticle arr[], const long left, const long right) {
long i = left, j = right;
IndexedParticle tmp;
IndexedParticle pivot = arr[(left + right) / 2];
/* partition */
while (i <= j) {
while (arr[i].index < pivot.index)
i++;
while (arr[j].index > pivot.index)
j--;
if (i <= j) {
tmp = arr[i];
arr[i] = arr[j];
arr[j] = tmp;
i++;
j--;
}
}
return j;
}
void quick_sort(IndexedParticle arr[], const long left, const long right){
/* recursion */
long part_index = partition(arr, left, right);
if (left < part_index)
quick_sort(arr, left, part_index);
if (part_index + 1 < right)
quick_sort(arr, part_index + 1, right);
}
// Simply create particles and try the kernels
......@@ -347,22 +324,21 @@ int main(int argc, char ** argv){
// We sort our particles
//////////////////////////////////////////////////////////////////////////////////
{
FVector<ParticlesGroup> groups;
ParticleClass*const realParticles = reinterpret_cast<ParticleClass*>(new char[loader.getNumberOfParticles() * sizeof(ParticleClass)]);
//////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////
std::cout << "Inserting & Sorting my particles ..." << std::endl;
counter.tic();
//////////////////////////////////////////////////////////////////////////////////
// Quick sort
//////////////////////////////////////////////////////////////////////////////////
if(true){
IndexedParticle* outputArray = 0;
long outputSize = 0;
{
IndexedParticle*const realParticlesIndexed = reinterpret_cast<IndexedParticle*>(new char[loader.getNumberOfParticles() * sizeof(ParticleClass)]);
// create particles
IndexedParticle*const realParticlesIndexed = reinterpret_cast<IndexedParticle*>(new char[loader.getNumberOfParticles() * sizeof(IndexedParticle)]);
F3DPosition boxCorner(loader.getCenterOfBox() - (loader.getBoxWidth()/2));
FTreeCoordinate host;
const FReal boxWidthAtLeafLevel = loader.getBoxWidth() / (2 << (NbLevels - 1) );
const FReal boxWidthAtLeafLevel = loader.getBoxWidth() / (1 << (NbLevels - 1) );
for(long idxPart = 0 ; idxPart < loader.getNumberOfParticles() ; ++idxPart){
loader.fillParticle(realParticlesIndexed[idxPart].particle);
host.setX( getTreeCoordinate( realParticlesIndexed[idxPart].particle.getPosition().getX() - boxCorner.getX(), boxWidthAtLeafLevel ));
......@@ -371,12 +347,97 @@ int main(int argc, char ** argv){
realParticlesIndexed[idxPart].index = host.getMortonIndex(NbLevels - 1);
}
quick_sort(realParticlesIndexed,0,loader.getNumberOfParticles()-1);
std::cout << "Start to sort " << loader.getNumberOfParticles() << " particles..." << std::endl;
// sort particles
FQuickSort::QsMpi<IndexedParticle,MortonIndex>(realParticlesIndexed, loader.getNumberOfParticles(),outputArray,outputSize);
delete [] reinterpret_cast<char*>(realParticlesIndexed);
std::cout << "Sorted "<< outputSize << " particles..." << std::endl;
}
// be sure there is no splited leaves
{
MortonIndex otherFirstIndex = -1;
{
FMpi::Request req[2];
int reqiter = 0;
if( 0 < app.processId() && outputSize){
std::cout << "I send to left that my first index is " << outputArray[0].index << std::endl;
MPI_Isend( &outputArray[0].index, 1, MPI_LONG_LONG, app.processId() - 1, 0, MPI_COMM_WORLD, &req[reqiter++]);
}
if( app.processId() != app.processCount() - 1){
MPI_Irecv(&otherFirstIndex, 1, MPI_LONG_LONG, app.processId() + 1, 0, MPI_COMM_WORLD, &req[reqiter++]);
std::cout << "I receive index from right " << otherFirstIndex << std::endl;
}
/*{
MPI_Waitall(reqiter,req,0);
if( 0 < app.processId() && !outputSize){
std::cout << "I send to left the first index is from right " << otherFirstIndex << std::endl;
MPI_Send( &otherFirstIndex, 1, MPI_LONG_LONG, app.processId() - 1, 0, MPI_COMM_WORLD);
}
}
// at this point every one know the first index of his right neighbors
const bool needToRecvBeforeSend = (app.processId() != 0 && ((outputSize && outputArray[0].index == otherFirstIndex) || !outputSize));
if( needToRecvBeforeSend || (app.processId() == app.processCount() - 1) ){
long sendByOther = 0;
MPI_Recv(&sendByOther, 1, MPI_LONG, app.processId() - 1, 0, MPI_COMM_WORLD,0);
std::cout << "I will receive " << sendByOther << std::endl;
if(sendByOther){
const IndexedParticle* const reallocOutputArray = outputArray;
const long reallocOutputSize = outputSize;
outputSize += sendByOther;
outputArray = new IndexedParticle[outputSize];
memcpy(&outputArray[sendByOther], reallocOutputArray, reallocOutputSize * sizeof(IndexedParticle));
delete[] reallocOutputArray;
MPI_Recv(outputArray, sizeof(IndexedParticle) * sendByOther, MPI_BYTE, app.processId() - 1, 0, MPI_COMM_WORLD, 0);
}
}
if(app.processId() != app.processCount() - 1){
long idxPart = outputSize - 1 ;
while(idxPart >= 0 && outputArray[idxPart].index == otherFirstIndex){
--idxPart;
}
long toSend = outputSize - 1 - idxPart;
MPI_Send( &toSend, 1, MPI_LONG, app.processId() + 1, 0, MPI_COMM_WORLD);
std::cout << "send to other " << toSend << std::endl;
if(toSend){
MPI_Send( &outputArray[idxPart + 1], toSend * sizeof(IndexedParticle), MPI_BYTE, app.processId() + 1, 0, MPI_COMM_WORLD);
}
if( app.processId() != 0 && !needToRecvBeforeSend ){
long sendByOther = 0;
MPI_Recv(&sendByOther, 1, MPI_LONG, app.processId() - 1, 0, MPI_COMM_WORLD, 0);
std::cout << "I will receive " << sendByOther << std::endl;
if(sendByOther){
const IndexedParticle* const reallocOutputArray = outputArray;
const long reallocOutputSize = outputSize;
outputSize += sendByOther;
outputArray = new IndexedParticle[outputSize];
memcpy(&outputArray[sendByOther], reallocOutputArray, reallocOutputSize * sizeof(IndexedParticle));
delete[] reallocOutputArray;
MPI_Recv(outputArray, sizeof(IndexedParticle) * sendByOther, MPI_BYTE, app.processId() - 1, 0, MPI_COMM_WORLD,0);
}
}
}
}
// we can insert into the tree
for(int idxPart = 0 ; idxPart < outputSize ; ++idxPart){
treeInterval.insert(outputArray[idxPart].particle);
}
myNbParticlesCounter = outputSize;
}
else{
//////////////////////////////////////////////////////////////////////////////////
// My method
//////////////////////////////////////////////////////////////////////////////////
FVector<ParticlesGroup> groups;
ParticleClass*const realParticles = reinterpret_cast<ParticleClass*>(new char[loader.getNumberOfParticles() * sizeof(ParticleClass)]);
{
OctreeClass sortingTree(NbLevels, SizeSubLevels,loader.getBoxWidth(),loader.getCenterOfBox());
ParticleClass particle;
......@@ -411,7 +472,7 @@ int main(int argc, char ** argv){
}
} while(octreeIterator.moveRight());
}*/
}
counter.tac();
std::cout << "Done " << "(@Creating and Inserting Temporary Particles = " << counter.elapsed() << "s)." << std::endl;
......@@ -531,6 +592,7 @@ int main(int argc, char ** argv){
delete [] reinterpret_cast<char*>(realParticles);
delete [] needToReceive;
}
}
//////////////////////////////////////////////////////////////////////////////////
// Now we can build the real tree
......@@ -607,7 +669,7 @@ int main(int argc, char ** argv){
}
}
printf("%d I have %d leafs on the right and %d on the left\n",app.processId(), rightLeafs, leftLeafs);
printf("%d I have %d leafs on the left and %d on the right\n",app.processId(), leftLeafs, rightLeafs);
//////////////////////////////////////////////////////////////////////////////////
// We balance the data
......@@ -641,16 +703,17 @@ int main(int argc, char ** argv){
while(idxSend < iNeedToSend && idxSend < iCanSend){
app.sendData(app.processId() + 1, sizeof(int),&groups[nbLeafs - idxSend - 1].number,0);
app.sendData(app.processId() + 1, sizeof(ParticleClass) * groups[nbLeafs - idxSend - 1].number,
&realParticles[groups[nbLeafs - idxSend - 1].positionInArray],0);
&realParticles[groups[nbLeafs - idxSend - 1].positionInArray], 0);
++idxSend;
}
// I need to wait (idxSend == iCanSend && idxSend < iNeedToSend)
if( iDoNotHaveEnoughtToSendRight ){ // I need to wait
int nbLeafsToRead(0);
app.receiveDataFromTag(sizeof(int), 1, &nbLeafsToRead);
app.receiveDataFromTag(sizeof(int), 0, &nbLeafsToRead);
printf("Receive %d from left to send to right\n", nbLeafsToRead);
for(int idxToRead = 0 ; idxToRead < nbLeafsToRead ; ++idxToRead){
int nbPartToRead(0);
app.receiveDataFromTag(sizeof(int), 1, &nbPartToRead);
app.receiveDataFromTag(sizeof(int), 0, &nbPartToRead);
if(rpartSize < nbPartToRead){
rpartSize = nbPartToRead;
......@@ -668,8 +731,6 @@ int main(int argc, char ** argv){
//insert into tree
for(int idxPart = 0 ; idxPart < nbPartToRead ; ++idxPart){
realTree.insert(rpart[idxPart]);
//const F3DPosition& particlePosition = rpart[idxPart].getPosition();
//std::cout << "\t I received (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
}
}
}
......@@ -681,7 +742,7 @@ int main(int argc, char ** argv){
int iCanSend = nbLeafs;
int idxSend (0);
printf("%d I need to send %d to left",app.processId(), iNeedToSend);
printf("%d I need to send %d to left\n",app.processId(), iNeedToSend);
app.sendData( app.processId() - 1, sizeof(int), &iNeedToSend, 1);
while(idxSend < iNeedToSend && idxSend < iCanSend){
......@@ -694,6 +755,7 @@ int main(int argc, char ** argv){
if( iDoNotHaveEnoughtToSendLeft ){
int nbLeafsToRead(0);
app.receiveDataFromTag(sizeof(int), 1, &nbLeafsToRead);
printf("Receive %d from right to send to left\n", nbLeafsToRead);
for(int idxToRead = 0 ; idxToRead < nbLeafsToRead ; ++idxToRead){
int nbPartToRead(0);
app.receiveDataFromTag(sizeof(int), 1, &nbPartToRead);
......@@ -713,8 +775,6 @@ int main(int argc, char ** argv){
else{
for(int idxPart = 0 ; idxPart < nbPartToRead ; ++idxPart){
realTree.insert(rpart[idxPart]);
//const F3DPosition& particlePosition = rpart[idxPart].getPosition();
//std::cout << "\t I receive (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
}
}
}
......@@ -740,8 +800,6 @@ int main(int argc, char ** argv){
app.receiveDataFromTag(nbPartToRead*sizeof(ParticleClass), 0, rpart);
for(int idxPart = 0 ; idxPart < nbPartToRead ; ++idxPart){
realTree.insert(rpart[idxPart]);
//const F3DPosition& particlePosition = rpart[idxPart].getPosition();
//std::cout << "\t I received (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
}
}
}
......@@ -750,11 +808,11 @@ int main(int argc, char ** argv){
printf("%d I need to receive from right\n",app.processId());
int nbLeafsToRead(0);
app.receiveDataFromTag(sizeof(int), 1, &nbLeafsToRead);
//printf("%d I will receive from right %d\n",app.processId(), nbLeafsToRead);
printf("%d I will receive from right %d\n",app.processId(), nbLeafsToRead);
for(int idxToRead = 0 ; idxToRead < nbLeafsToRead ; ++idxToRead){
int nbPartToRead(0);
app.receiveDataFromTag(sizeof(int), 1, &nbPartToRead);
printf("%d I will receive %d particles\n",app.processId(), nbPartToRead);
//printf("%d I will receive %d particles\n",app.processId(), nbPartToRead);
if(rpartSize < nbPartToRead){
rpartSize = nbPartToRead;
delete [] (reinterpret_cast<char*>(rpart));
......@@ -764,19 +822,18 @@ int main(int argc, char ** argv){
app.receiveDataFromTag(nbPartToRead*sizeof(ParticleClass), 1, rpart);
for(int idxPart = 0 ; idxPart < nbPartToRead ; ++idxPart){
realTree.insert(rpart[idxPart]);
//const F3DPosition& particlePosition = rpart[idxPart].getPosition();
//std::cout << "\t I received (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
}
}
}
printf("Will now take my own particles from %d to %d\n",FMath::Max(myLeftLeaf-leftLeafs,0) , FMath::Min(myRightLeaf,totalNbLeafs- rightLeafs) - myLeftLeaf);
printf("Will now take my own particles from %d to %d\n",FMath::Max(myLeftLeaf-leftLeafs,0) , FMath::Max(myLeftLeaf-leftLeafs,0) + FMath::Min(myRightLeaf,totalNbLeafs - rightLeafs) - myLeftLeaf);
printf("myLeftLeaf %d leftLeafs %d myRightLeaf %d rightLeafs %d totalNbLeafs %d\n",myLeftLeaf,leftLeafs, myRightLeaf, rightLeafs, totalNbLeafs);
// insert the particles we already have
for(int idxLeafInsert = FMath::Max(myLeftLeaf-leftLeafs,0) ; idxLeafInsert < FMath::Min(myRightLeaf,totalNbLeafs- rightLeafs) - myLeftLeaf ; ++idxLeafInsert){
if(leftLeafs != totalNbLeafs){
for(int idxLeafInsert = FMath::Max(myLeftLeaf-leftLeafs,0) ; idxLeafInsert < FMath::Max(myLeftLeaf-leftLeafs,0) + FMath::Min(myRightLeaf,totalNbLeafs- rightLeafs) - myLeftLeaf ; ++idxLeafInsert){
for(int idxPart = 0 ; idxPart < groups[idxLeafInsert].number ; ++idxPart){
realTree.insert(realParticles[groups[idxLeafInsert].positionInArray + idxPart]);
//const F3DPosition& particlePosition = realParticles[groups[idxLeafInsert].positionInArray + idxPart].getPosition();
//std::cout << "\t Position is (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
}
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment