Skip to content
GitLab
Commit 57601dcd authored by COULAUD Olivier's avatar COULAUD Olivier
Browse files

Start moving the particles according to a given morton distribution

parent d8032aea
Hide whitespace changes
Inline Side-by-side
experimental/examples/test_l2p.cpp
View file @ 57601dcd
......@@ -198,7 +198,7 @@ auto main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) -> int
interpolator_type2 cheb(matrix_kernel_type{}, order,
static_cast<std::size_t>(tree_height),
box.width(0));
std::clog << " warn 1 \n";
auto roots2_1d = cheb.roots(static_cast<std::size_t>(order));
std::cout << "roots: " << roots2_1d << std::endl;
......
experimental/include/scalfmm/tree/group_let.hpp
View file @ 57601dcd
......@@ -150,6 +150,51 @@ namespace scalfmm::tree
}
return std::make_tuple(buff_p, buff_n);
}
void exchange_particles()
{
#ifdef COMM_PART
/// Send the number
/// send to left and right
int nb_elt_from_left{0}, nb_elt_from_right{0};
/// receive from left right
{
// compute the buffer size
inria::mpi::request tab_mpi_status[2];
auto mpi_type = inria::mpi::get_datatype<int>();
// if i'm not the last proc
const int to_right = (rank + 1 == nb_proc) ? MPI_PROC_NULL : rank + 1;
const int to_left = (rank == 0) ? MPI_PROC_NULL : rank - 1;
if(tosendL[rank] > 0)
{
manager.comm.isend(&nb_part_to_left, 1, mpi_type, to_left, 1);
}
if(tosendR[rank] > 0)
manager.comm.isend(&nb_part_to_right, 1, mpi_type, to_right, 1);
int idx = 0;
if(toReceivL > 0)
{
tab_mpi_status[idx] = manager.comm.irecv(&nb_elt_from_left, 1, mpi_type, to_left, 1);
idx++;
}
if(toReceivR > 0)
tab_mpi_status[idx] = manager.comm.irecv(&nb_elt_from_right, 1, mpi_type, to_right, 1);
// Waiting for the result of the request, if my rank is 0
// I don't need to wait
if(toReceivL + toReceivR > 0)
{
inria::mpi::request::waitall(toReceivL + toReceivR, tab_mpi_status);
}
std::cout << rank << " nb Part left: " << nb_elt_from_left << " nb Part right: " << nb_elt_from_right
<< std::endl;
}
/// Second send the particles
#endif
}
///
/// \brief balanced_leaves
///
......@@ -315,139 +360,191 @@ namespace scalfmm::tree
}
print("rank(" + std::to_string(rank) + ") initial tomove: ", tomove);
// if(rank == 0)
for(int i = 0; i < nb_proc - 1; ++i)
{
for(int i = 0; i < nb_proc - 1; ++i)
if(tomove[i] < 0)
{
if(tomove[i] < 0)
{
tosendL[i + 1] = -tomove[i];
tomove[i + 1] += tomove[i];
tomove[i] = 0;
}
else if(tomove[i] > 0)
{
tosendR[i + 1] = tomove[i];
tomove[i] = 0;
tomove[i + 1] += tomove[i];
}
// print(" end (" + std::to_string(i) + ")
// tomove: ", tomove); print(" end (" +
// std::to_string(i) + ") tosendR: ",
// tosendR); print(" end (" +
// std::to_string(i) + ") tosendL: ",
// tosendL);
tosendL[i + 1] = -tomove[i];
tomove[i + 1] += tomove[i];
tomove[i] = 0;
}
tosendR[nb_proc - 1] = 0;
print("rank(" + std::to_string(rank) + ") tomove: ", tomove);
print("rank(" + std::to_string(rank) + ") tosendR: ", tosendR);
print("rank(" + std::to_string(rank) + ") tosendRL: ", tosendL);
///
std::cout << "tosendL(" + std::to_string(rank) + "): " << tosendL[rank] << std::endl;
std::cout << "tosendR(" + std::to_string(rank) + "): " << tosendR[rank] << std::endl;
if(rank > 0)
std::cout << "toReceivL(" + std::to_string(rank) + "): " << tosendR[rank - 1] << std::endl;
if(rank < nb_proc - 1)
std::cout << "toReceivR(" + std::to_string(rank) + "): " << tosendL[rank + 1] << std::endl;
int toReceivL, toReceivR;
toReceivL = tosendR[rank - 1] > 0 ? 1 : 0;
toReceivR = tosendL[rank + 1] > 0 ? 1 : 0;
/// proceed the communications
/// first send the number particles inside leaves to send
///
int nb_leaf_to_left{0}, nb_leaf_to_right{0}, nb_part_to_left{0}, nb_part_to_right{0};
if(tosendL[rank] > 0)
else if(tomove[i] > 0)
{
int leaf_idx = 0;
nb_part_to_left = weight[leaf_idx][1];
std::cout << " tosendL leaf_idx " << leaf_idx << " " << nb_part_to_left << std::endl;
while(nb_part_to_left <= tosendL[rank])
{
leaf_idx++;
nb_part_to_left += weight[leaf_idx][1];
std::cout << " tosendL new pos " << leaf_idx << " " << nb_part_to_left << std::endl;
}
nb_leaf_to_left = leaf_idx + 1;
std::cout << rank << "send " << nb_leaf_to_left << " leaf to left - nb part " << nb_part_to_left
<< std::endl;
tosendR[i + 1] = tomove[i];
tomove[i] = 0;
tomove[i + 1] += tomove[i];
}
// print(" end (" + std::to_string(i) + ")
// tomove: ", tomove); print(" end (" +
// std::to_string(i) + ") tosendR: ",
// tosendR); print(" end (" +
// std::to_string(i) + ") tosendL: ",
// tosendL);
}
tosendR[nb_proc - 1] = 0;
if(tosendR[rank] > 0)
print("rank(" + std::to_string(rank) + ") tomove: ", tomove);
print("rank(" + std::to_string(rank) + ") tosendR: ", tosendR);
print("rank(" + std::to_string(rank) + ") tosendRL: ", tosendL);
///
std::cout << "tosendL(" + std::to_string(rank) + "): " << tosendL[rank] << std::endl;
std::cout << "tosendR(" + std::to_string(rank) + "): " << tosendR[rank] << std::endl;
if(rank > 0)
std::cout << "toReceivL(" + std::to_string(rank) + "): " << tosendR[rank - 1] << std::endl;
if(rank < nb_proc - 1)
std::cout << "toReceivR(" + std::to_string(rank) + "): " << tosendL[rank + 1] << std::endl;
int toReceivL, toReceivR;
toReceivL = tosendR[rank - 1] > 0 ? 1 : 0;
toReceivR = tosendL[rank + 1] > 0 ? 1 : 0;
/// proceed the communications
/// first send the number particles inside leaves to send
///
int nb_leaf_to_left{0}, nb_leaf_to_right{0}, nb_part_to_left{0}, nb_part_to_right{0};
Morton_type morton_to_left{0}, morton_to_right{0};
MortonWeight_type MortonPart_to_left{0}, MortonPart_to_right{0};
if(tosendL[rank] > 0)
{
int leaf_idx = 0;
nb_part_to_left = weight[leaf_idx][1];
std::cout << " tosendL leaf_idx " << leaf_idx << " " << nb_part_to_left << std::endl;
while(nb_part_to_left <= tosendL[rank])
{
int leaf_idx = 0;
nb_part_to_right = weight[leaf_idx][1];
std::cout << "tosendR leaf_idx " << leaf_idx << " " << nb_part_to_right << std::endl;
leaf_idx++;
nb_part_to_left += weight[leaf_idx][1];
std::cout << " tosendL new pos " << leaf_idx << " " << nb_part_to_left << std::endl;
}
nb_leaf_to_left = leaf_idx + 1;
morton_to_left = weight[leaf_idx][0];
MortonPart_to_left = {weight[leaf_idx][0], nb_leaf_to_left};
// New starting Morton index for the local distribution
local[0] = weight[leaf_idx + 1][0];
while(nb_part_to_right <= tosendL[rank])
{
leaf_idx++;
nb_part_to_right += weight[leaf_idx][1];
std::cout << " - tosendR new pos " << leaf_idx << " " << nb_part_to_right << std::endl;
}
nb_leaf_to_right = leaf_idx + 1;
std::cout << rank << "send morton_to_left" << morton_to_left << std::endl;
}
if(tosendR[rank] > 0)
{
int leaf_idx = weight.size() - 1;
nb_part_to_right = weight[leaf_idx][1];
std::cout << "tosendR leaf_idx " << leaf_idx << " " << nb_part_to_right << std::endl;
std::cout << rank << " send " << nb_leaf_to_right << " leaf to right - nb part "
<< nb_part_to_right << std::endl;
while(nb_part_to_right <= tosendL[rank])
{
leaf_idx--;
nb_part_to_right += weight[leaf_idx][1];
std::cout << " - tosendR new pos " << leaf_idx << " " << nb_part_to_right << std::endl;
}
/// Send the number
/// send to left and right
int nb_elt_from_left{0}, nb_elt_from_right{0};
nb_leaf_to_right = leaf_idx + 1;
morton_to_right = weight[leaf_idx][0];
MortonPart_to_right = {weight[leaf_idx][0], nb_leaf_to_left};
// New starting Morton index for the local distribution
local[1] = weight[leaf_idx][0];
// std::cout << rank << " send " << nb_leaf_to_right << " leaf to right - nb
// part "
// << nb_part_to_right << " " << MortonPart_to_right[0] << std::endl;
std::cout << rank << "send morton_to_right " << morton_to_right << std::endl;
}
local[3] = 0;
std::cout << rank << " local partition [ " << local[0] << ", " << local[1] << "]" << std::endl;
/// Send the number
/// send to left and right
int nb_elt_from_left{0}, nb_elt_from_right{0};
Morton_type min_idx{part_distrib[rank][0]}, max_idx{part_distrib[rank][1]};
Morton_type morton_from_left{local[0]}, morton_from_right{local[1]};
/// receive from left right
// auto exchange_val = [&manager, &rank,
// &nb_proc, &tosendL, &tosendR, &toReceivL,
// &toReceivR](const auto&
// nb_part_to_left, const
// auto& nb_part_to_right,
// auto&
// nb_elt_from_left,
// auto&
// nb_elt_from_right)
{
// compute the buffer size
/// receive from left right
inria::mpi::request tab_mpi_status[2];
auto mpi_type = inria::mpi::get_datatype<decltype(morton_from_left)>();
// if i'm not the last proc
const int to_right = (rank + 1 == nb_proc) ? MPI_PROC_NULL : rank + 1;
const int to_left = (rank == 0) ? MPI_PROC_NULL : rank - 1;
if(tosendL[rank] > 0)
{
// compute the buffer size
inria::mpi::request tab_mpi_status[2];
auto mpi_type = inria::mpi::get_datatype<int>();
// if i'm not the last proc
const int to_right = (rank + 1 == nb_proc) ? MPI_PROC_NULL : rank + 1;
const int to_left = (rank == 0) ? MPI_PROC_NULL : rank - 1;
if(tosendL[rank] > 0)
{
manager.comm.isend(&nb_part_to_left, 1, mpi_type, to_left, 1);
}
if(tosendR[rank] > 0)
manager.comm.isend(&nb_part_to_right, 1, mpi_type, to_right, 1);
int idx = 0;
if(toReceivL > 0)
{
tab_mpi_status[idx] = manager.comm.irecv(&nb_elt_from_left, 1, mpi_type, to_left, 1);
idx++;
}
if(toReceivR > 0)
tab_mpi_status[idx] = manager.comm.irecv(&nb_elt_from_right, 1, mpi_type, to_right, 1);
// Waiting for the result of the request, if my rank is 0
// I don't need to wait
manager.comm.isend(&morton_to_left, 1, mpi_type, to_left, 1);
}
if(tosendR[rank] > 0)
manager.comm.isend(&morton_to_right, 1, mpi_type, to_right, 1);
int idx = 0;
if(toReceivL > 0)
{
tab_mpi_status[idx] = manager.comm.irecv(&morton_from_left, 1, mpi_type, to_left, 1);
local[0] = morton_from_left;
if(toReceivL + toReceivR > 0)
{
inria::mpi::request::waitall(toReceivL + toReceivR, tab_mpi_status);
}
idx++;
}
if(toReceivR > 0)
{
tab_mpi_status[idx] = manager.comm.irecv(&morton_from_right, 1, mpi_type, to_right, 1);
local[1] = morton_from_right;
}
// Waiting for the result of the request, if my rank is 0
// I don't need to wait
std::cout << rank << " nb Part left: " << nb_elt_from_left
<< " nb Part right: " << nb_elt_from_right << std::endl;
if(toReceivL + toReceivR > 0)
{
inria::mpi::request::waitall(toReceivL + toReceivR, tab_mpi_status);
}
/// Second send the particles
std::cout << rank << " Morton Left: " << morton_from_left << " Morton right: " << morton_from_right
<< std::endl;
}
// try
// {
// inria::distribute(manager, begin(mortonArray), end(mortonArray), morton_distrib,
// inria::uniform_distribution{manager, mortonArray});
// }
// catch(std::out_of_range& e)
// {
// std::cerr << e.what() << '\n';
// }
// print("rank(" + std::to_string(rank) + ") morton_distrib ", morton_distrib);
// manager.comm.barrier();
// std::cout << "rank(" + std::to_string(rank) + ") Mordon distrib [" << morton_distrib[0]
// << ",
// "
// << morton_distrib[morton_distrib.size() - 1] << "]\n";
return part_distrib;
// exchange_val(nb_part_to_left, nb_part_to_right,
// nb_elt_from_left, nb_elt_from_right);
// std::array<Morton_type, 3> local{weight[0][0], weight[weight.size() - 1][0], nb_part};
std::vector<std::array<Morton_type, 2>> morton_distrib(nb_proc);
std::array<Morton_type, 2> local1 = {morton_from_left, morton_from_right};
std::cout << rank << " final local 1 [ " << local1[0] << ", " << local1[1] << "]" << std::endl;
morton_distrib[0] = local1;
// print("rank(" + std::to_string(rank) + ") Distrib cells Index: ", part_distrib);
// std::cout << "rank(" << rank << ") Distrib Leaf Index: " <<
// nb_part << std::endl;
/// share the distribution on all processors
nb_elt = sizeof(local1);
manager.comm.allgather(local1.data(), nb_elt, MPI_CHAR, morton_distrib[0].data(), nb_elt, MPI_CHAR /*, 0*/);
print("rank(" + std::to_string(rank) + ") morton distrib final: ", morton_distrib);
return morton_distrib;
}
template<typename ParticlesArray_type, typename MortonArray_type>
void fit_particles_in_distrib(parallel_manager& manager, ParticlesArray_type& particles,
const MortonArray_type& morton_dist)
{
auto comm = manager.get_communicator();
auto my_rank = manager.get_process_id();
// get the min and the max morton index of the particles own by the
// process
// send the number of communication we will receive
auto mortonMin = morton_dist[my_rank][0];
auto mortonMax = morton_dist[my_rank][1];
// Count the number of particles to move and where
// Send these numbers
// Send the particles
}
} // namespace distrib
namespace let
......@@ -554,6 +651,7 @@ namespace scalfmm::tree
/// Add the leaves/cells used in the algorithm + Let
distrib::fit_particles_in_distrib(manager, particle_container, particles_distrib);
/// construct the local tree based on the let
/// return the tree
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment