From 0d0695fac159949f49cef96d7722a1e8847744c0 Mon Sep 17 00:00:00 2001
From: Sebastien Gilles <sebastien.gilles@inria.fr>
Date: Wed, 7 Sep 2016 18:00:02 +0200
Subject: [PATCH] #976 Add the automatic update of ghosts to more Petsc
 functions and methods (this can be opted out through a dedicated argument,
 which by default is set to yes).

---
 .../Petsc/Matrix/MatrixOperations.hpp         | 16 ++++++--
 .../Petsc/Matrix/MatrixOperations.hxx         | 18 +++++++--
 .../Wrappers/Petsc/Vector/Vector.cpp          | 28 +++++---------
 .../Wrappers/Petsc/Vector/Vector.hpp          | 37 +++++++++++++++----
 .../Wrappers/Petsc/Vector/Vector.hxx          | 15 ++++++++
 5 files changed, 80 insertions(+), 34 deletions(-)

diff --git a/Sources/ThirdParty/Wrappers/Petsc/Matrix/MatrixOperations.hpp b/Sources/ThirdParty/Wrappers/Petsc/Matrix/MatrixOperations.hpp
index 71d79a8d68..d957f4355d 100644
--- a/Sources/ThirdParty/Wrappers/Petsc/Matrix/MatrixOperations.hpp
+++ b/Sources/ThirdParty/Wrappers/Petsc/Matrix/MatrixOperations.hpp
@@ -112,11 +112,13 @@ namespace HappyHeart
              *
              * \param[in] invoking_file File that invoked the function or class; usually __FILE__.
              * \param[in] invoking_line File that invoked the function or class; usually __LINE__.
+             * \copydoc doxygen_hide_do_update_ghost_arg
              */
             template<class MatrixT>
             std::enable_if_t<std::is_base_of<Private::BaseMatrix, MatrixT>::value, void>
             MatMult(const MatrixT& matrix, const Vector& v1, Vector& v2,
-                    const char* invoking_file, int invoking_line);
+                    const char* invoking_file, int invoking_line,
+                    update_ghost do_update_ghost = update_ghost::yes);
             
             
             /*!
@@ -124,11 +126,13 @@ namespace HappyHeart
              *
              * \param[in] invoking_file File that invoked the function or class; usually __FILE__.
              * \param[in] invoking_line File that invoked the function or class; usually __LINE__.
+             * \copydoc doxygen_hide_do_update_ghost_arg
              */
             template<class MatrixT>
             std::enable_if_t<std::is_base_of<Private::BaseMatrix, MatrixT>::value, void>
             MatMultAdd(const MatrixT& matrix, const Vector& v1, const Vector& v2, Vector& v3,
-                       const char* invoking_file, int invoking_line);
+                       const char* invoking_file, int invoking_line,
+                       update_ghost do_update_ghost = update_ghost::yes);
             
             
             
@@ -137,11 +141,13 @@ namespace HappyHeart
              *
              * \param[in] invoking_file File that invoked the function or class; usually __FILE__.
              * \param[in] invoking_line File that invoked the function or class; usually __LINE__.
+             * \copydoc doxygen_hide_do_update_ghost_arg
              */
             template<class MatrixT>
             std::enable_if_t<std::is_base_of<Private::BaseMatrix, MatrixT>::value, void>
             MatMultTranspose(const MatrixT& matrix, const Vector& v1, Vector& v2,
-                             const char* invoking_file, int invoking_line);
+                             const char* invoking_file, int invoking_line,
+                             update_ghost do_update_ghost = update_ghost::yes);
             
             
             /*!
@@ -149,11 +155,13 @@ namespace HappyHeart
              *
              * \param[in] invoking_file File that invoked the function or class; usually __FILE__.
              * \param[in] invoking_line File that invoked the function or class; usually __LINE__.
+             * \copydoc doxygen_hide_do_update_ghost_arg
              */
             template<class MatrixT>
             std::enable_if_t<std::is_base_of<Private::BaseMatrix, MatrixT>::value, void>
             MatMultTransposeAdd(const MatrixT& matrix, const Vector& v1, const Vector& v2, Vector& v3,
-                                const char* invoking_file, int invoking_line);
+                                const char* invoking_file, int invoking_line,
+                                update_ghost do_update_ghost = update_ghost::yes);
             
             /*!
              * \class doxygen_hide_matmatmult_warning
diff --git a/Sources/ThirdParty/Wrappers/Petsc/Matrix/MatrixOperations.hxx b/Sources/ThirdParty/Wrappers/Petsc/Matrix/MatrixOperations.hxx
index 2673605323..a911bdb580 100644
--- a/Sources/ThirdParty/Wrappers/Petsc/Matrix/MatrixOperations.hxx
+++ b/Sources/ThirdParty/Wrappers/Petsc/Matrix/MatrixOperations.hxx
@@ -29,11 +29,14 @@ namespace HappyHeart
             MatMultTranspose(const MatrixT& matrix,
                              const Vector& v1,
                              Vector& v2,
-                             const char* invoking_file, int invoking_line)
+                             const char* invoking_file, int invoking_line,
+                             update_ghost do_update_ghost)
             {
                 int error_code = ::MatMultTranspose(matrix.Internal(), v1.Internal(), v2.Internal());
                 if (error_code)
                     throw ExceptionNS::Exception(error_code, "MatMultTranspose", invoking_file, invoking_line);
+                
+                v2.UpdateGhosts(invoking_file, invoking_line, do_update_ghost);
             }
             
             
@@ -43,11 +46,14 @@ namespace HappyHeart
                                 const Vector& v1,
                                 const Vector& v2,
                                 Vector& v3,
-                                const char* invoking_file, int invoking_line)
+                                const char* invoking_file, int invoking_line,
+                                update_ghost do_update_ghost)
             {
                 int error_code = ::MatMultTransposeAdd(matrix.Internal(), v1.Internal(), v2.Internal(), v3.Internal());
                 if (error_code)
                     throw ExceptionNS::Exception(error_code, "MatMultTransposeAdd", invoking_file, invoking_line);
+                
+                v3.UpdateGhosts(invoking_file, invoking_line, do_update_ghost);
             }
             
             
@@ -246,11 +252,14 @@ namespace HappyHeart
             MatMult(const MatrixT& matrix,
                     const Vector& v1,
                     Vector& v2,
-                    const char* invoking_file, int invoking_line)
+                    const char* invoking_file, int invoking_line,
+                    update_ghost do_update_ghost)
             {
                 int error_code = ::MatMult(matrix.Internal(), v1.Internal(), v2.Internal());
                 if (error_code)
                     throw ExceptionNS::Exception(error_code, "MatMult", invoking_file, invoking_line);
+                
+                v2.UpdateGhosts(invoking_file, invoking_line, do_update_ghost);
             }
             
             
@@ -260,7 +269,8 @@ namespace HappyHeart
                        const Vector& v1,
                        const Vector& v2,
                        Vector& v3,
-                       const char* invoking_file, int invoking_line)
+                       const char* invoking_file, int invoking_line,
+                       update_ghost do_update_ghost)
             {
                 int error_code = ::MatMultAdd(matrix.Internal(), v1.Internal(), v2.Internal(), v3.Internal());
                 if (error_code)
diff --git a/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.cpp b/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.cpp
index f9bffb6949..21ae835e32 100644
--- a/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.cpp
+++ b/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.cpp
@@ -299,7 +299,7 @@ namespace HappyHeart
             
             
             void Vector::Assembly(const char* invoking_file, int invoking_line,
-                                  DoUpdateGhosts do_update_ghosts)
+                                  update_ghost do_update_ghost)
             {
                 int error_code = VecAssemblyBegin(Internal());
                 if (error_code)
@@ -310,14 +310,7 @@ namespace HappyHeart
                 if (error_code)
                     throw ExceptionNS::Exception(error_code, "VecAssemblyEnd", invoking_file, invoking_line);
                 
-                switch(do_update_ghosts)
-                {
-                    case DoUpdateGhosts::yes:
-                        UpdateGhosts(invoking_file, invoking_line);
-                        break;
-                    case DoUpdateGhosts::no:
-                        break;
-                }
+                UpdateGhosts(invoking_file, invoking_line, do_update_ghost);
             }
             
             
@@ -329,14 +322,7 @@ namespace HappyHeart
                 if (error_code)
                     throw ExceptionNS::Exception(error_code, "VecCopy", invoking_file, invoking_line);
                 
-                switch(do_update_ghost)
-                {
-                    case update_ghost::yes:
-                        UpdateGhosts(invoking_file, invoking_line);
-                        break;
-                    case update_ghost::no:
-                        break;
-                }
+                UpdateGhosts(invoking_file, invoking_line, do_update_ghost);
             }
             
             
@@ -435,11 +421,17 @@ namespace HappyHeart
             }
             
             
-            void AXPY(PetscScalar alpha, const Vector& x, Vector& y, const char* invoking_file, int invoking_line)
+            void AXPY(PetscScalar alpha,
+                      const Vector& x,
+                      Vector& y,
+                      const char* invoking_file, int invoking_line,
+                      update_ghost do_update_ghost)
             {
                 int error_code = VecAXPY(y.Internal(), alpha, x.Internal());
                 if (error_code)
                     throw ExceptionNS::Exception(error_code, "VecAXPY", invoking_file, invoking_line);
+                
+                y.UpdateGhosts(invoking_file, invoking_line, do_update_ghost);
             }
             
             
diff --git a/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.hpp b/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.hpp
index 89944acb76..4221068e1c 100644
--- a/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.hpp
+++ b/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.hpp
@@ -289,21 +289,16 @@ namespace HappyHeart
                 void Set(const PetscScalar alpha, const char* invoking_file, int invoking_line);
                 
                 
-                //! Convenient enum class for Assembly().
-                enum class DoUpdateGhosts { yes, no };
-                
                 /*!
                  * \brief Petsc Assembling of the vector.
                  *
                  * \param[in] invoking_file File that invoked the function or class; usually __FILE__.
                  * \param[in] invoking_line File that invoked the function or class; usually __LINE__.
-                 * \param[in] do_update_ghosts If 'yes', update the values of ghosts just after the assembling is done.
-                 * 'yes' is therefore more secure but lead to slower code if you don't actually need the updated values
-                 * of the ghosts. 'yes' is set by default.
+                 * \copydoc doxygen_hide_do_update_ghost_arg
                  *
                  */
                 void Assembly(const char* invoking_file, int invoking_line,
-                              DoUpdateGhosts do_update_ghosts = DoUpdateGhosts::yes);
+                              update_ghost do_update_ghost = update_ghost::yes);
                                 
                 /*!
                  * \brief Add or modify values inside a Petsc vector.
@@ -541,6 +536,27 @@ namespace HappyHeart
                  */
                 void UpdateGhosts(const char* invoking_file, int invoking_line);
                 
+                
+                /*!
+                 * \brief Update the ghost values if do_update_ghost is set to yes.
+                 *
+                 * This convenient method should be used only in Vector or Matrix related functions that 
+                 * provides the possibility to automatically update ghosts through an ad hoc argument (see
+                 * for instance Copy() method).
+                 *
+                 * \param[in] invoking_file File that invoked the function or class; usually __FILE__.
+                 * \param[in] invoking_line File that invoked the function or class; usually __LINE__.
+                 * Behaviour when the vector is without ghost is up to Petsc (if it results in an error
+                 * an exception will be thrown).
+                 *
+                 * Beware: all processor must make the call to this method! If for instance you're in a loop
+                 * and one processor leaves it before the other, the code will be deadlocked...
+                 *
+                 * \copydoc doxygen_hide_do_update_ghost_arg
+                 */
+                void UpdateGhosts(const char* invoking_file, int invoking_line,
+                                  update_ghost do_update_ghost);
+                
                 //! Number of ghosts.
                 unsigned int Nghost() const;
                 
@@ -631,8 +647,13 @@ namespace HappyHeart
             /*!
              * \brief Wrapper over VecAXPY, that performs Y = alpha * X + Y.
              *
+             * \copydoc doxygen_hide_do_update_ghost_arg
              */
-            void AXPY(PetscScalar alpha, const Vector& x, Vector& y, const char* invoking_file, int invoking_line);
+            void AXPY(PetscScalar alpha,
+                      const Vector& x,
+                      Vector& y,
+                      const char* invoking_file, int invoking_line,
+                      update_ghost do_update_ghost = update_ghost::yes);
             
             
             /*!
diff --git a/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.hxx b/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.hxx
index a4306f8b3b..2c79ef1aa2 100644
--- a/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.hxx
+++ b/Sources/ThirdParty/Wrappers/Petsc/Vector/Vector.hxx
@@ -61,6 +61,21 @@ namespace HappyHeart
                         break;
                 }
             }
+            
+            
+            inline void Vector::UpdateGhosts(const char* invoking_file, int invoking_line,
+                                             update_ghost do_update_ghost)
+            {
+                switch(do_update_ghost)
+                {
+                    case update_ghost::yes:
+                        UpdateGhosts(invoking_file, invoking_line);
+                        break;
+                    case update_ghost::no:
+                        break;
+                }
+            }
+
 
           
             
-- 
GitLab