Langjian/mpi finalize (#2884)

* Rewrite the MPI_Finalize

* Change destructor to virtual finalize

* Test

* More bug fix

* Delete space

* Add variable to control initialization and finalization

* Add a bool variable to control initialization and finalization for openmpi and mlsl

src/ngraph/distributed/mlsl.hpp

@@ -33,17 +33,19 @@ namespace ngraph
             MLSLDistributedInterface(const std::string& name = "MLSL")
                 : m_name(name)
             {
-                if (!MLSL::Environment::GetEnv().IsInitialized())
+                if (!MLSL::Environment::GetEnv().IsInitialized() && !m_initialized_mlsl)
                 {
                     MLSL::Environment::GetEnv().Init(nullptr, nullptr);
+                    m_initialized_mlsl = true;
                 }
             }
 
             ~MLSLDistributedInterface() override
             {
-                if (MLSL::Environment::GetEnv().IsInitialized())
+                if (MLSL::Environment::GetEnv().IsInitialized() && m_initialized_mlsl)
                 {
                     MLSL::Environment::GetEnv().Finalize();
+                    m_initialized_mlsl = false;
                 }
             }
 
@@ -107,6 +109,7 @@ namespace ngraph
 
         protected:
             std::string m_name{"MLSL"};
+            bool m_initialized_mlsl = false;
         };
     }
 }

src/ngraph/distributed/open_mpi.hpp

@@ -37,20 +37,21 @@ namespace ngraph
             {
                 int flag = 0;
                 MPI_Initialized(&flag);
-                if (!flag)
+                if (!flag && !m_initialized_mpi)
                 {
                     MPI_Init(NULL, NULL);
+                    m_initialized_mpi = true;
                 }
             }
 
             ~OpenMPIDistributedInterface() override
             {
-                int flag = 0;
-                MPI_Initialized(&flag);
-
-                if (flag)
+                int is_mpi_finalized = 0;
+                MPI_Finalized(&is_mpi_finalized);
+                if (!is_mpi_finalized && m_initialized_mpi)
                 {
                     MPI_Finalize();
+                    m_initialized_mpi = false;
                 }
             }
 
@@ -108,6 +109,7 @@ namespace ngraph
 
         protected:
             std::string m_name;
+            bool m_initialized_mpi = false;
         };
     }
 }