Commit 495487dc authored by berenger-bramas's avatar berenger-bramas
Browse files

First draft version of a parallel loading

git-svn-id: svn+ssh://scm.gforge.inria.fr/svn/scalfmm/scalfmm/trunk@148 2616d619-271b-44dc-8df4-d4a8f33a7222
parent 0f663c97
......@@ -27,6 +27,7 @@ public:
////////////////////////////////////////////////////////
// Use MPI
////////////////////////////////////////////////////////
typedef MPI_Request Request;
FMpi(int inArgc, char ** inArgv ) {
MPI_Init(&inArgc,&inArgv);
......@@ -36,12 +37,35 @@ public:
MPI_Finalize();
}
void allgather(void* const sendbuf, const int sendcount, void* const recvbuf, const int recvcount, MPI_Datatype datatype = MPI_INT){
MPI_Allgather( sendbuf, sendcount, datatype, recvbuf, recvcount, datatype, MPI_COMM_WORLD);
}
void sendData(const int inReceiver, const int inSize, void* const inData, const int inTag){
//MPI_Request request;
//MPI_Isend(inData, inSize, MPI_CHAR , inReceiver, inTag, MPI_COMM_WORLD, &request);
MPI_Send(inData, inSize, MPI_CHAR , inReceiver, inTag, MPI_COMM_WORLD);
}
void isendData(const int inReceiver, const int inSize, void* const inData, const int inTag, MPI_Request*const request){
MPI_Isend(inData, inSize, MPI_CHAR , inReceiver, inTag, MPI_COMM_WORLD, request);
}
void iWait( MPI_Request*const request ){
MPI_Status status;
MPI_Wait(request, &status);
}
void iWaitall(MPI_Request requests[], const int count){
MPI_Status status[count];
MPI_Waitall(count, requests, status);
}
void waitMessage(int*const sender, int*const tag = 0, const int restrictsender = MPI_ANY_SOURCE, const int restricttag = MPI_ANY_TAG){
MPI_Status status;
MPI_Probe( restrictsender, restricttag, MPI_COMM_WORLD, &status );
if(sender) *sender = status.MPI_SOURCE;
if(tag) *tag = status.MPI_TAG;
}
void receiveData(const int inSize, void* const inData, int* const inSource = 0, int* const inTag = 0, int* const inFilledSize = 0){
MPI_Status status;
MPI_Recv(inData, inSize, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG,MPI_COMM_WORLD, &status);
......@@ -133,11 +157,21 @@ public:
////////////////////////////////////////////////////////
// Without MPI
////////////////////////////////////////////////////////
typedef int Request;
FMpi(int inArgc, char ** inArgv ) {}
void allgather(void* const , const int , void* const , const int, MPI_Datatype = 0){
}
void sendData(const int, const int, void* const, const int ){}
void isendData(const int , const int , void* const , const int , Request*const ){
}
void iWait( Request*const ){
}
void receiveData(const int, void* const, int* const inSource = 0, int* const inTag = 0, int* const inFilledSize = 0){
if(inSource) *inSource = 0;
if(inTag) *inTag = 0;
......
......@@ -301,7 +301,7 @@ int main(int argc, char ** argv){
//////////////////////////////////////////////////////////////////////////////////
{
FVector<ParticlesGroup> groups;
ParticleClass*const realParticles = new ParticleClass[loader.getNumberOfParticles()];
ParticleClass*const realParticles = reinterpret_cast<ParticleClass*>(new char[loader.getNumberOfParticles() * sizeof(ParticleClass)]);
OctreeClass sortingTree(NbLevels, SizeSubLevels,loader.getBoxWidth(),loader.getCenterOfBox());
......@@ -337,10 +337,10 @@ int main(int argc, char ** argv){
while( iter.hasNotFinished() ){
realParticles[indexPart] = iter.data();
std::cout << "Particles with index " << indexPart << " has a morton index of " << indexAtThisLeaf << std::endl;
//std::cout << "Particles with index " << indexPart << " has a morton index of " << indexAtThisLeaf << std::endl;
const F3DPosition& particlePosition = realParticles[indexPart].getPosition();
std::cout << "\t The real position of this particle is (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
//const F3DPosition& particlePosition = realParticles[indexPart].getPosition();
//std::cout << "\t The real position of this particle is (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
++indexPart;
iter.gotoNext();
......@@ -362,75 +362,105 @@ int main(int argc, char ** argv){
printf("%d I will go from %lld to %lld\n",app.processId(), startIndex, endIndex);
printf("There is actually %d leafs\n", groups.getSize());
MortonIndex rightMortonIndex = min;
int groudIndex = 0;
for(int idxProc = 0 ; idxProc < app.processCount() ; ++idxProc){
const MortonIndex leftMortonIndex = rightMortonIndex;
rightMortonIndex = app.getOtherRight(max - min + 1, idxProc) + min;
printf("Working with %d, he goes from %lld to %lld \n",idxProc, leftMortonIndex,rightMortonIndex);
if(idxProc != app.processId()){
int size = 0;
int currentGroupIndex = groudIndex;
while(groudIndex < groups.getSize() && groups[groudIndex].index < rightMortonIndex){
size += groups[groudIndex].number;
++groudIndex;
}
printf("%d Send %d to %d (group index from %d, \n",app.processId(), size, idxProc);
if(size){
//send particles
app.sendData(idxProc,sizeof(int),&size,0);
app.sendData(idxProc,sizeof(ParticleClass) * size, &realParticles[groups[groudIndex].positionInArray],1);
int*const needToReceive = new int[app.processCount() * app.processCount()];
memset(needToReceive,0,app.processCount() * app.processCount() * sizeof(int));
FMpi::Request requests[app.processCount()];
{
int needToSend[app.processCount()];
memset(needToSend, 0, sizeof(int) * app.processCount());
MortonIndex rightMortonIndex = min;
int groudIndex = 0;
for(int idxProc = 0 ; idxProc < app.processCount() && groudIndex < groups.getSize() ; ++idxProc){
rightMortonIndex = app.getOtherRight(max - min + 1, idxProc) + min;
printf("Working with %d, he goes to %lld \n",idxProc,rightMortonIndex);
if(idxProc != app.processId()){
int size = 0;
int currentGroupIndex = groudIndex;
while(groudIndex < groups.getSize() && groups[groudIndex].index < rightMortonIndex){
size += groups[groudIndex].number;
++groudIndex;
}
needToSend[idxProc] = size;
printf("%d Send %d to %d\n",app.processId(), size, idxProc);
app.isendData( idxProc, sizeof(ParticleClass) * size, &realParticles[groups[currentGroupIndex].positionInArray], 1, &requests[idxProc]);
}
else{
//send empty message
int zeros(0);
app.sendData(idxProc,sizeof(int),&zeros,0);
needToSend[idxProc] = 0;
while(groudIndex < groups.getSize() && groups[groudIndex].index < rightMortonIndex){
const int end = groups[groudIndex].positionInArray + groups[groudIndex].number;
for(int idxPart = groups[groudIndex].positionInArray ; idxPart < end ; ++idxPart){
//std::cout << "\t I keep (" << realParticles[idxPart].getPosition().getX() << ";" << realParticles[idxPart].getPosition().getY() << ";" << realParticles[idxPart].getPosition().getZ() << ")" << std::endl;
treeInterval.insert(realParticles[idxPart]);
++myNbParticlesCounter;
}
++groudIndex;
}
}
}
else{
while(groudIndex < groups.getSize() && groups[groudIndex].index < rightMortonIndex){
const int end = groups[groudIndex].positionInArray + groups[groudIndex].number;
for(int idxPart = groups[groudIndex].positionInArray ; idxPart < end ; ++idxPart){
treeInterval.insert(realParticles[idxPart]);
++myNbParticlesCounter;
}
++groudIndex;
app.allgather(needToSend, app.processCount(), needToReceive, app.processCount());
for(int idxSrc = 0 ; idxSrc < app.processCount() ; ++idxSrc){
for(int idxTest = 0 ; idxTest < app.processCount() ; ++idxTest){
printf("[%d][%d] = %d\n", idxSrc, idxTest, needToReceive[idxSrc * app.processCount() + idxTest]);
}
}
}
delete [] realParticles;
}
//////////////////////////////////////////////////////////////////////////////////
// We receive others particles and insert them in the tree
//////////////////////////////////////////////////////////////////////////////////
int CounterProcToReceive(0);
int maxPartToReceive(0);
for(int idxProc = 0 ; idxProc < app.processCount() ; ++idxProc){
if(idxProc != app.processId() && needToReceive[app.processCount() * idxProc + app.processId()]){
++CounterProcToReceive;
if(maxPartToReceive < needToReceive[app.processCount() * idxProc + app.processId()]){
maxPartToReceive = needToReceive[app.processCount() * idxProc + app.processId()];
}
printf("needToReceive[idxProc][app.processId()] %d",needToReceive[app.processCount() * idxProc + app.processId()]);
}
}
//////////////////////////////////////////////////////////////////////////////////
// We receive others particles and insert them in the tree
//////////////////////////////////////////////////////////////////////////////////
{
char* buffer(0);
int currentBufferCapacity(0);
printf("maxPartToReceive %d \n",maxPartToReceive);
ParticleClass*const iterParticles = reinterpret_cast<ParticleClass*>(new char[maxPartToReceive * sizeof(ParticleClass)]);
// we receive message from nb proc - 1 (from every other procs
for(int idxProc = 1 ; idxProc < app.processCount() ; ++idxProc){
int nbPartFromProc(0);
for(int idxProc = 0 ; idxProc < CounterProcToReceive ; ++idxProc){
int source(0);
app.receiveDataFromTag(sizeof(int), 0, &nbPartFromProc, &source);
printf("%d receive %d from %d\n",app.processId(),nbPartFromProc,source);
if(nbPartFromProc){
if(currentBufferCapacity < int(nbPartFromProc * sizeof(ParticleClass)) ){
delete [] buffer;
currentBufferCapacity = nbPartFromProc * sizeof(ParticleClass);
buffer = new char[currentBufferCapacity];
}
app.receiveDataFromTagAndSource(sizeof(ParticleClass), 1, source, buffer);
printf("Wait data to receive\n");
app.waitMessage(&source);
ParticleClass* iterParticles = reinterpret_cast<ParticleClass*>(buffer);
for(int idxPart = 0 ; idxPart < nbPartFromProc ; ++idxPart, ++iterParticles){
std::cout << "\t We receive a new particle (" << (*iterParticles).getPosition().getX() << ";" << (*iterParticles).getPosition().getY() << ";" << (*iterParticles).getPosition().getZ() << ")" << std::endl;
treeInterval.insert(*iterParticles);
++myNbParticlesCounter;
}
const int nbPartFromProc = needToReceive[app.processCount() * source + app.processId()];
int received(0);
printf("%d receive %d\n",source,nbPartFromProc);
app.receiveDataFromTagAndSource(sizeof(ParticleClass) * nbPartFromProc, 1, source, iterParticles,&received);
printf("Received %d part*partcileSize %d \n",received,sizeof(ParticleClass) * nbPartFromProc);
printf("Insert into tree\n");
for(int idxPart = 0 ; idxPart < nbPartFromProc ; ++idxPart){
//std::cout << "\t We receive a new particle (" << (*iterParticles).getPosition().getX() << ";" << (*iterParticles).getPosition().getY() << ";" << (*iterParticles).getPosition().getZ() << ")" << std::endl;
treeInterval.insert(iterParticles[idxPart]);
++myNbParticlesCounter;
}
}
printf("Wait all send\n");
for(int idxProc = 0 ; idxProc < app.processCount() ; ++idxProc){
if(idxProc != app.processId() && needToReceive[app.processCount() * app.processId() + idxProc ]){
app.iWait(&requests[idxProc]);
}
}
printf("Delete particle array\n");
delete [] reinterpret_cast<char*>(realParticles);
delete [] needToReceive;
}
//////////////////////////////////////////////////////////////////////////////////
......@@ -442,17 +472,18 @@ int main(int argc, char ** argv){
//////////////////////////////////////////////////////////////////////////////////
// We inform the master proc about the data we have
//////////////////////////////////////////////////////////////////////////////////
printf("Inform other about leaves we have\n");
FVector<ParticlesGroup> groups;
ParticleClass*const realParticles = new ParticleClass[myNbParticlesCounter];
ParticleClass*const realParticles = myNbParticlesCounter?new ParticleClass[myNbParticlesCounter]:0;
int nbLeafs = 0;
int indexPart = 0;
// we might now have any particles in our interval
if(myNbParticlesCounter){
OctreeClass::Iterator octreeIterator(&treeInterval);
octreeIterator.gotoBottomLeft();
int indexPart = 0;
do{
ContainerClass::ConstBasicIterator iter(*octreeIterator.getCurrentListTargets());
const MortonIndex indexAtThisLeaf = octreeIterator.getCurrentGlobalIndex();
......@@ -568,8 +599,8 @@ int main(int argc, char ** argv){
//insert into tree
for(int idxPart = 0 ; idxPart < nbPartToRead ; ++idxPart){
realTree.insert(rpart[idxPart]);
const F3DPosition& particlePosition = rpart[idxPart].getPosition();
std::cout << "\t I received (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
//const F3DPosition& particlePosition = rpart[idxPart].getPosition();
//std::cout << "\t I received (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
}
}
}
......@@ -613,8 +644,8 @@ int main(int argc, char ** argv){
else{
for(int idxPart = 0 ; idxPart < nbPartToRead ; ++idxPart){
realTree.insert(rpart[idxPart]);
const F3DPosition& particlePosition = rpart[idxPart].getPosition();
std::cout << "\t I receive (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
//const F3DPosition& particlePosition = rpart[idxPart].getPosition();
//std::cout << "\t I receive (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
}
}
}
......@@ -630,7 +661,7 @@ int main(int argc, char ** argv){
for(int idxToRead = 0 ; idxToRead < nbLeafsToRead ; ++idxToRead){
int nbPartToRead(0);
app.receiveDataFromTag(sizeof(int), 0, &nbPartToRead);
printf("%d I will receive %d particles\n",app.processId(), nbPartToRead);
//printf("%d I will receive %d particles\n",app.processId(), nbPartToRead);
if(rpartSize < nbPartToRead){
rpartSize = nbPartToRead;
delete [] (reinterpret_cast<char*>(rpart));
......@@ -640,8 +671,8 @@ int main(int argc, char ** argv){
app.receiveDataFromTag(nbPartToRead*sizeof(ParticleClass), 0, rpart);
for(int idxPart = 0 ; idxPart < nbPartToRead ; ++idxPart){
realTree.insert(rpart[idxPart]);
const F3DPosition& particlePosition = rpart[idxPart].getPosition();
std::cout << "\t I received (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
//const F3DPosition& particlePosition = rpart[idxPart].getPosition();
//std::cout << "\t I received (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
}
}
}
......@@ -650,7 +681,7 @@ int main(int argc, char ** argv){
printf("%d I need to receive from right\n",app.processId());
int nbLeafsToRead(0);
app.receiveDataFromTag(sizeof(int), 1, &nbLeafsToRead);
printf("%d I will receive from right %d\n",app.processId(), nbLeafsToRead);
//printf("%d I will receive from right %d\n",app.processId(), nbLeafsToRead);
for(int idxToRead = 0 ; idxToRead < nbLeafsToRead ; ++idxToRead){
int nbPartToRead(0);
app.receiveDataFromTag(sizeof(int), 1, &nbPartToRead);
......@@ -664,27 +695,25 @@ int main(int argc, char ** argv){
app.receiveDataFromTag(nbPartToRead*sizeof(ParticleClass), 1, rpart);
for(int idxPart = 0 ; idxPart < nbPartToRead ; ++idxPart){
realTree.insert(rpart[idxPart]);
const F3DPosition& particlePosition = rpart[idxPart].getPosition();
std::cout << "\t I received (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
//const F3DPosition& particlePosition = rpart[idxPart].getPosition();
//std::cout << "\t I received (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
}
}
}
printf("Will now take my own particles from %d to %d\n",FMath::Max(myLeftLeaf,leftLeafs) - myLeftLeaf , FMath::Min(myRightLeaf,totalNbLeafs- rightLeafs) - myLeftLeaf);
printf("Will now take my own particles from %d to %d\n",FMath::Max(myLeftLeaf-leftLeafs,0) , FMath::Min(myRightLeaf,totalNbLeafs- rightLeafs) - myLeftLeaf);
// insert the particles we already have
for(int idxLeafInsert = FMath::Max(myLeftLeaf,leftLeafs) - myLeftLeaf ; idxLeafInsert < FMath::Min(myRightLeaf,totalNbLeafs- rightLeafs) - myLeftLeaf ; ++idxLeafInsert){
for(int idxLeafInsert = FMath::Max(myLeftLeaf-leftLeafs,0) ; idxLeafInsert < FMath::Min(myRightLeaf,totalNbLeafs- rightLeafs) - myLeftLeaf ; ++idxLeafInsert){
for(int idxPart = 0 ; idxPart < groups[idxLeafInsert].number ; ++idxPart){
realTree.insert(realParticles[groups[idxLeafInsert].positionInArray + idxPart]);
const F3DPosition& particlePosition = realParticles[groups[idxLeafInsert].positionInArray + idxPart].getPosition();
std::cout << "\t Position is (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
//const F3DPosition& particlePosition = realParticles[groups[idxLeafInsert].positionInArray + idxPart].getPosition();
//std::cout << "\t Position is (" << particlePosition.getX() << ";" << particlePosition.getY() << ";" << particlePosition.getZ() << ")" << std::endl;
}
}
delete [] reinterpret_cast<char*>(rpart);
}
app.processBarrier();
//////////////////////////////////////////////////////////////////////////////////
......@@ -747,14 +776,15 @@ int main(int argc, char ** argv){
octreeIteratorValide.getCurrentListSrc()->getSize(), octreeIterator.getCurrentListSrc()->getSize() );
}
printf("index %lld with %d particles\n", octreeIteratorValide.getCurrentGlobalIndex(), octreeIteratorValide.getCurrentListSrc()->getSize());
//printf("index %lld with %d particles\n", octreeIteratorValide.getCurrentGlobalIndex(), octreeIteratorValide.getCurrentListSrc()->getSize());
if(!octreeIterator.moveRight()){
printf("Error cannot test tree end to early\n");
if(!octreeIteratorValide.moveRight() && idxLeaf != myRightLeaf - 1){
printf("Error cannot valide tree end to early, idxLeaf %d myRightLeaf %d\n", idxLeaf, myRightLeaf);
break;
}
if(!octreeIteratorValide.moveRight()){
printf("Error cannot valide tree end to early\n");
if(!octreeIterator.moveRight() && idxLeaf != myRightLeaf - 1){
printf("Error cannot test tree end to early, idxLeaf %d myRightLeaf %d\n", idxLeaf, myRightLeaf);
break;
}
}
......
......@@ -25,7 +25,7 @@ int main(int argc, char ** argv){
//////////////////////////////////////////////////////////////
// Nb of particles
const long NbParticles = FParameters::getValue(argc,argv,"-nb", 50000L);
const long NbParticles = FParameters::getValue(argc,argv,"-nb", 20000L);
// Center of the box
const FReal XCenter = 0.5;
......@@ -74,13 +74,13 @@ int main(int argc, char ** argv){
data[3] = 0.1;
// Generate particles
for( long idx = 0 ; idx < NbParticles ; ++idx ){
/*data[0] = ((FReal(rand())/RAND_MAX) * BoxWidth * 2) + XCenter - BoxWidth;
data[0] = ((FReal(rand())/RAND_MAX) * BoxWidth * 2) + XCenter - BoxWidth;
data[1] = ((FReal(rand())/RAND_MAX) * BoxWidth * 2) + YCenter - BoxWidth;
data[2] = ((FReal(rand())/RAND_MAX) * BoxWidth * 2) + ZCenter - BoxWidth;*/
data[2] = ((FReal(rand())/RAND_MAX) * BoxWidth * 2) + ZCenter - BoxWidth;
data[0] = ((FReal(idx)/NbParticles) * BoxWidth * 2) + XCenter - BoxWidth;
/*data[0] = ((FReal(idx)/NbParticles) * BoxWidth * 2) + XCenter - BoxWidth;
data[1] = ((FReal(idx)/NbParticles) * BoxWidth * 2) + YCenter - BoxWidth;
data[2] = ((FReal(idx)/NbParticles) * BoxWidth * 2) + ZCenter - BoxWidth;
data[2] = ((FReal(idx)/NbParticles) * BoxWidth * 2) + ZCenter - BoxWidth;*/
fwrite(&data, sizeof(FReal), 4, myfile);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment