Commit 7da032d1 authored by GILLES Sebastien's avatar GILLES Sebastien
Browse files

#820 - #9 - #859 Many slight improvements and some comments added while...

#820 - #9 - #859 Many slight improvements and some comments added while tracking a potential bug in parallel (which in all likelyhood doesn't exist: the culprit seems really to be an interpolation matrix...).
parent 73dad330
......@@ -108,7 +108,7 @@ namespace HappyHeart
// rely on the automatic deletion of unused shared pointers and storing more here would harm the process.
dof_list_per_felt_space = ComputeDofIndexListPerFEltSpace(felt_space_list);
auto& node_bearer_list = GetNonCstNodeBearerList();
auto& node_bearer_list = GetNonCstProcessorWiseNodeBearerList();
// Compute the required elements to perform the partition. Parmetis call for instance is done there.
// Node bearer list is still the same length in output (reduction not yet done) but its elements are sort
......@@ -162,13 +162,14 @@ namespace HappyHeart
const auto& mpi = MpiHappyHeart();
const auto rank = mpi.GetRank<unsigned int>();
// Reduce the list of node bearers.
Internal::FEltSpaceNS::ReduceNodeBearerList(rank, GetNonCstNodeBearerList());
// Reduce the list of node bearers to the ones present on processor.
// Pattern computation will correctly address those not in the list as it will query finite elements to find
// the adequate columns to fill with non-zero.
Internal::FEltSpaceNS::ReduceNodeBearerList(rank, GetNonCstProcessorWiseNodeBearerList());
// Compute the pattern of the global matrix.
const auto& felt_space_list = GetFEltSpaceList();
const auto& node_bearer_list = GetNodeBearerList();
const auto& node_bearer_list = GetProcessorWiseNodeBearerList();
const auto& numbering_subset_list = GetNumberingSubsetList();
matrix_pattern_per_numbering_subset_ =
......@@ -177,7 +178,7 @@ namespace HappyHeart
node_bearer_list,
numbering_subset_list,
GetNdofHolder());
// - Reduce the list of finite elements in each finite element space to the processor-wise ones only.
// - Compute the ghost node bearers for each finite element space.
// - Compute the reduced geometric mesh region, limited to processor-wise geometric elements.
......@@ -264,7 +265,9 @@ namespace HappyHeart
{
const auto& boundary_condition_list = DirichletBoundaryConditionManager::GetInstance().GetList();
Internal::FEltSpaceNS::CreateNodeListHelper node_helper(node_bearer_list_);
auto& processor_wise_node_bearer_list = GetNonCstProcessorWiseNodeBearerList();
Internal::FEltSpaceNS::CreateNodeListHelper node_helper(processor_wise_node_bearer_list);
#ifdef HAPPY_HEART_DEBUG_FILES
const int rank = MpiHappyHeart().Rank<int>();
......@@ -323,7 +326,7 @@ namespace HappyHeart
void GodOfDof::ComputeProcessorWiseAndGhostDofIndex()
{
auto& node_bearer_list = GetNonCstNodeBearerList();
auto& node_bearer_list = GetNonCstProcessorWiseNodeBearerList();
auto& ghost_node_bearer_list = GetNonCstGhostNodeBearerList();
// Sort both node_bearer list and ghost node_bearer list per increasing index.
......@@ -546,7 +549,7 @@ namespace HappyHeart
void GodOfDof::PrintDofInformation(const NumberingSubset& numbering_subset,
std::ostream& out) const
{
const auto& node_bearer_list = GetNodeBearerList();
const auto& node_bearer_list = GetProcessorWiseNodeBearerList();
out << "Ndof (processor_wise) = " << NprocessorWiseDof(numbering_subset) << std::endl;
out << "# First column: program-wise index." << std::endl;
......
......@@ -165,8 +165,8 @@ namespace HappyHeart
//! Get the list of FEltSpace.
const FEltSpace::vector_unique_ptr& GetFEltSpaceList() const;
//! Returns the list of node bearers.
const NodeBearer::vector_shared_ptr& GetNodeBearerList() const;
//! Returns the list of node bearers present on local processor.
const NodeBearer::vector_shared_ptr& GetProcessorWiseNodeBearerList() const;
//! Returns the list of ghost node bearers.
const NodeBearer::vector_shared_ptr& GetGhostNodeBearerList() const;
......@@ -365,7 +365,7 @@ namespace HappyHeart
private:
//! Non constant access to the list of node bearers.
NodeBearer::vector_shared_ptr& GetNonCstNodeBearerList();
NodeBearer::vector_shared_ptr& GetNonCstProcessorWiseNodeBearerList();
//! Non constant access to the list of ghost node bearers.
NodeBearer::vector_shared_ptr& GetNonCstGhostNodeBearerList();
......@@ -413,7 +413,7 @@ namespace HappyHeart
void ReduceToProcessorWise();
//! Returns the number of node bearers.
unsigned int NnodeBearer() const;
unsigned int NprocessorWiseNodeBearer() const;
/*!
* \brief Set for each concerned dof the associated value from a Dirichlet boundary condition.
......@@ -549,7 +549,7 @@ namespace HappyHeart
GeometricMeshRegion& mesh_;
//! List of all node bearers. Each one is present only once here; ghost aren't present in this one.
NodeBearer::vector_shared_ptr node_bearer_list_;
NodeBearer::vector_shared_ptr processor_wise_node_bearer_list_;
//! List of all ghosted node bearers, that is all nodes required for calculation but not hosted by local processor.
NodeBearer::vector_shared_ptr ghost_node_bearer_list_;
......
......@@ -59,15 +59,15 @@ namespace HappyHeart
}
inline const NodeBearer::vector_shared_ptr& GodOfDof::GetNodeBearerList() const
inline const NodeBearer::vector_shared_ptr& GodOfDof::GetProcessorWiseNodeBearerList() const
{
return node_bearer_list_;
return processor_wise_node_bearer_list_;
}
inline NodeBearer::vector_shared_ptr& GodOfDof::GetNonCstNodeBearerList()
inline NodeBearer::vector_shared_ptr& GodOfDof::GetNonCstProcessorWiseNodeBearerList()
{
return node_bearer_list_;
return processor_wise_node_bearer_list_;
}
......@@ -121,9 +121,9 @@ namespace HappyHeart
}
inline unsigned int GodOfDof::NnodeBearer() const
inline unsigned int GodOfDof::NprocessorWiseNodeBearer() const
{
return static_cast<unsigned int>(node_bearer_list_.size());
return static_cast<unsigned int>(processor_wise_node_bearer_list_.size());
}
......
......@@ -28,7 +28,7 @@ namespace HappyHeart
void ComputeNodeBearerConnectivityInFEltSpace(FEltSpace& felt_space,
void ComputeNodeBearerConnectivityInFEltSpace(const FEltSpace& felt_space,
connectivity_type& connectivity,
const unsigned int mpi_rank);
......@@ -71,19 +71,12 @@ namespace HappyHeart
return connectivity;
}
namespace // anonymous
{
void ComputeNodeBearerConnectivityInFEltSpace(FEltSpace& felt_space,
void ComputeNodeBearerConnectivityInFEltSpace(const FEltSpace& felt_space,
connectivity_type& connectivity,
const unsigned int mpi_rank)
{
......@@ -106,8 +99,8 @@ namespace HappyHeart
{
const auto& local_felt_space_ptr = local_felt_space_pair.second;
assert(!(!local_felt_space_ptr));
const NodeBearer::vector_shared_ptr& node_bearers_in_current_finite_element =
local_felt_space_ptr->GetNodeBearerList();
const auto& node_bearers_in_current_finite_element =
local_felt_space_ptr->GetNodeBearerList();
for (const auto& node_bearer_ptr : node_bearers_in_current_finite_element)
{
......
......@@ -78,23 +78,25 @@ namespace HappyHeart
{
assert(!(!dof_ptr));
auto& dof = *dof_ptr;
# ifndef NDEBUG
dof_ptr->numbering_subset_index_list_ = debug_numbering_subset_index_list;
dof.numbering_subset_index_list_ = debug_numbering_subset_index_list;
# endif // NDEBUG
switch(TypeDofIndexT)
{
case TypeDofIndex::program_wise_per_numbering_subset:
assert(!(!numbering_subset_ptr));
dof_ptr->SetProgramWiseIndex(*numbering_subset_ptr, current_dof_index);
dof.SetProgramWiseIndex(*numbering_subset_ptr, current_dof_index);
break;
case TypeDofIndex::processor_wise_per_numbering_subset:
assert(!(!numbering_subset_ptr));
dof_ptr->SetProcessorWiseOrGhostIndex(*numbering_subset_ptr, current_dof_index);
dof.SetProcessorWiseOrGhostIndex(*numbering_subset_ptr, current_dof_index);
break;
case TypeDofIndex::processor_wise:
assert(!numbering_subset_ptr && "No numbering subset are expected for this case!");
dof_ptr->SetInternalProcessorWiseOrGhostIndex(current_dof_index);
dof.SetInternalProcessorWiseOrGhostIndex(current_dof_index);
break;
}
......
......@@ -44,13 +44,33 @@ namespace HappyHeart
std::size_t buf_dof_index = 0;
assert(processor_wise_node_bearer_list.size() == connectivity.size());
assert(std::is_sorted(processor_wise_node_bearer_list.cbegin(),
processor_wise_node_bearer_list.cend(),
Utilities::PointerComparison::Less<NodeBearer::shared_ptr>()));
#ifndef NDEBUG
{
NodeBearer::vector_shared_ptr keys;
keys.reserve(connectivity.size());
for (const auto& item : connectivity)
keys.push_back(item.first);
std::sort(keys.begin(), keys.end(), Utilities::PointerComparison::Less<NodeBearer::shared_ptr>());
assert(keys == processor_wise_node_bearer_list);
}
#endif // NDEBUG
for (const auto& node_bearer_ptr : processor_wise_node_bearer_list)
{
assert(!(!node_bearer_ptr));
auto it = connectivity.find(node_bearer_ptr);
assert(it != connectivity.cend() && "This method is called after reduction to processor-wise of the node bearer list.");
assert(it != connectivity.cend() && "This method is called after reduction to processor-wise of the "
"node bearer list.");
// Compute the list of dofs that are connected to the current dof. As node_bearer is itself in
// connected list, dofs on the same node_bearer are counted as well.
......
......@@ -94,7 +94,9 @@ namespace HappyHeart
*
* \param[in] connectivity For each node bearer, the list of connected node bearers. Computed by
* ComputeNodeBearerConnectivity().
* \param[in] processor_wise_node_bearer_list Node bearer list on the local processor.
* \param[in] processor_wise_node_bearer_list Node bearer list on the local processor. It is somewhat
* redundant with keys of \a connectivity; it is there only because it has the advantage of being already
* sort.
* \param[in] Ndof_holder Object which knows number of dofs in each numbering subset.
* \param[in] row_numbering_subset Numbering subset over which all dofs considered in the rows of the
* matrix are defined.
......
......@@ -44,7 +44,7 @@ namespace HappyHeart
const auto& mesh = god_of_dof.GetGeometricMeshRegion();
const auto Ncoords = static_cast<std::size_t>(mesh.Ncoord());
const auto& node_bearer_list = god_of_dof.GetNodeBearerList();
const auto& node_bearer_list = god_of_dof.GetProcessorWiseNodeBearerList();
dof_index_list_per_coord_.resize(static_cast<std::size_t>(Ncoords));
initial_position_per_coord_.resize(static_cast<std::size_t>(Ncoords));
......
......@@ -64,7 +64,11 @@ namespace HappyHeart
/*!
* \brief Reduce the node bearers to the processor-wise values.
* \brief Reduce the node bearers to the processor-wise values in the list.
*
* \internal <b><tt>[internal]</tt></b> NodeBearer objects aren't destroyed at this stage: they are still
* needed. The ones not useful on current processor will be destroyed shortly after this call, before the
* end of GodOfDof::Init().
*
* \param[in] mpi_rank Rank of the current processor.
* \param[in,out] node_bearer_list In input, the full list of node bearers. In output, the processor-wise
......
......@@ -166,10 +166,10 @@ namespace HappyHeart
}
#ifndef NDEBUG
#ifndef NDEBUG
for (const auto& pair_in_shrunk : shrunk_list)
assert(pair_in_shrunk.first != ref_felt_space_ptr);
#endif // NDEBUG
#endif // NDEBUG
auto&& copy_ref_felt_space_ptr = std::make_unique<Internal::RefFEltNS::RefFEltSpace>(*ref_felt_space_ptr);
......
......@@ -105,11 +105,25 @@ namespace HappyHeart
decltype(auto) ghosted_dof_list = felt_space.GetGhostedDofList();
decltype(auto) fluid_mass_numbering_subset =
god_of_dof.GetNumberingSubset(EnumUnderlyingType(NumberingSubsetIndex::fluid_mass));
god_of_dof.GetNumberingSubset(EnumUnderlyingType(NumberingSubsetIndex::fluid_mass));
std::unordered_map<unsigned int, unsigned int> fluid_mass_monolithic_matching;
fluid_mass_monolithic_matching.max_load_factor(Utilities::DefaultMaxLoadFactor());
decltype(auto) pattern = god_of_dof.GetMatrixPattern(fluid_mass_numbering_subset,
fluid_mass_numbering_subset);
decltype(auto) icsr = pattern.GetICsr();
decltype(auto) jcsr = pattern.GetJCsr();
{
std::ostringstream oconv;
oconv << MpiHappyHeart().GetRankPreffix();
Utilities::PrintContainer(icsr, oconv, ", ", "iCSR = [");
Utilities::PrintContainer(jcsr, oconv, ", ", "jCSR = [");
std::cout << oconv.str();
}
for (const auto& dof_ptr : dof_list)
{
assert(!(!dof_ptr));
......@@ -126,6 +140,11 @@ namespace HappyHeart
static_cast<void>(check);
}
Utilities::PrintAssociativeContainer(fluid_mass_monolithic_matching,
std::cout,
", ",
MpiHappyHeart().GetRankPreffix() + " ");
std::vector<unsigned int> relevant_program_wise_index_list(fluid_mass_monolithic_matching.size());
std::transform(fluid_mass_monolithic_matching.cbegin(),
......@@ -136,7 +155,6 @@ namespace HappyHeart
return pair.first;
});
for (const auto& dof_ptr : ghosted_dof_list)
{
assert(!(!dof_ptr));
......@@ -153,10 +171,15 @@ namespace HappyHeart
static_cast<void>(check);
}
//Utilities::PrintAssociativeContainer(fluid_mass_monolithic_matching);
// Utilities::PrintAssociativeContainer(fluid_mass_monolithic_matching);
const auto end_matching = fluid_mass_monolithic_matching.cend();
const auto& t11 = GetFluidMassMatrixOnFluidMesh();
t11.View(MpiHappyHeart(), __FILE__, __LINE__);
for (auto row_index : relevant_program_wise_index_list)
......@@ -164,26 +187,50 @@ namespace HappyHeart
std::vector<std::pair<PetscInt, PetscScalar>> row_content;
t11.GetRow(static_cast<PetscInt>(row_index), row_content, __FILE__, __LINE__);
if (row_index == 3)
{
std::ostringstream oconv;
oconv << MpiHappyHeart().GetRankPreffix() << " Small row = ";
Utilities::PrintKeys(row_content, oconv);
std::cout << oconv.str();
}
const auto it_row = fluid_mass_monolithic_matching.find(row_index);
assert(it_row != fluid_mass_monolithic_matching.cend());
assert(it_row != end_matching);
const auto row_index_in_monolithic = static_cast<PetscInt>(it_row->second);
for (const auto& content : row_content)
{
const auto it_col = fluid_mass_monolithic_matching.find(static_cast<unsigned int>(content.first));
assert(it_col != fluid_mass_monolithic_matching.cend());
// Might happen for ghost values!
if (it_col == end_matching)
continue;
system_matrix.SetValue(row_index_in_monolithic,
static_cast<PetscInt>(it_col->second),
content.second,
ADD_VALUES,
__FILE__, __LINE__);
std::vector<std::pair<PetscInt, PetscScalar>> bar;
system_matrix.GetRow(static_cast<PetscInt>(row_index_in_monolithic), bar, __FILE__, __LINE__);
if (row_index == 3)
{
std::ostringstream oconv;
oconv << MpiHappyHeart().GetRankPreffix() << " ROW SMALL = " << row_index << " ROW MONO = "
<< row_index_in_monolithic << " CONTENT SMALL = " << content.first << " PROJECTED = "
<< it_col->second << " and poss = ";
Utilities::PrintKeys(bar, oconv);
std::cout << oconv.str();
}
// system_matrix.SetValue(row_index_in_monolithic,
// static_cast<PetscInt>(it_col->second),
// content.second,
// ADD_VALUES,
// __FILE__, __LINE__);
}
}
system_matrix.Assembly( __FILE__, __LINE__);
......
......@@ -56,7 +56,7 @@ namespace HappyHeart
auto& global_vector = GetNonCstGlobalVector();
FiberNS::Private::FillGlobalVector<TypeT>(value_list_per_coord_index,
god_of_dof.GetNodeBearerList(),
god_of_dof.GetProcessorWiseNodeBearerList(),
unknown,
numbering_subset,
global_vector);
......
......@@ -18,7 +18,6 @@
#include "ThirdParty/Wrappers/Petsc/Exceptions/Petsc.hpp"
namespace HappyHeart
{
......@@ -274,8 +273,6 @@ namespace HappyHeart
throw ExceptionNS::Exception(error_code, "MatSetValuesBlocked", invoking_file, invoking_line);
}
void Matrix::SetValuesRow(PetscInt row_index,
const PetscScalar* values,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment