Merge branch 'master' into tfl/gpu_broadcast

doc/sphinx/source/ops/allreduce.rst

+.. allreduce.rst:
+
+###
+AllReduce
+###
+
+.. code-block:: cpp
+
+   AllReduce // Collective operation
+
+
+Description
+===========
+
+Combines values from all processes or devices and distributes the result back
+to all processes or devices.
+
+
+Inputs
+------
+
++-----------------+-------------------------+--------------------------------+
+| Name            | Element Type            | Shape                          |
++=================+=========================+================================+
+| ``arg``         | ``element::f32``        | Any                            |
+|                 | ``element::f64``        |                                |
++-----------------+-------------------------+--------------------------------+
+
+
+Outputs
+-------
+
++-----------------+-------------------------+--------------------------------+
+| Name            | Element Type            | Shape                          |
++=================+=========================+================================+
+| ``output``      | ``element::f32``        | Same as ``arg``                |
+|                 | ``element::f64``        |                                |
++-----------------+-------------------------+--------------------------------+
+
+
+C++ Interface
+=============
+
+.. doxygenclass:: ngraph::op::AllReduce
+   :project: ngraph
+   :members:

doc/sphinx/source/ops/index.rst

@@ -52,6 +52,7 @@ Not currently a comprehensive list.
    abs.rst
    acos.rst
    add.rst
+   allreduce.rst
    asin.rst
    atan.rst
    avg_pool.rst

src/ngraph/CMakeLists.txt

@@ -34,6 +34,7 @@ set (SRC
     node.cpp
     ops/abs.cpp
     ops/add.cpp
+    ops/allreduce.cpp
     ops/avg_pool.cpp
     ops/batch_norm.cpp
     ops/broadcast.cpp
@@ -215,11 +216,6 @@ if(NGRAPH_DISTRIBUTED_ENABLE AND MPI_CXX_INCLUDE_PATH)
     include_directories(SYSTEM ${MPI_C_INCLUDE_PATH} ${MPI_CXX_INCLUDE_PATH})
     link_directories(${MPI_C_LIBRARIES} ${MPI_CXX_LIBRARIES})
 
-    # Add sources for distributed ngraph
-    # and all its dependencies
-    set(SRC ${SRC}
-        ops/allreduce.cpp
-    )
     set_property(SOURCE codegen/compiler.cpp APPEND PROPERTY COMPILE_DEFINITIONS
         "MPI_HEADER_PATH=\"${MPI_C_INCLUDE_PATH}\";")
 endif()

src/ngraph/axis_set.hpp

@@ -21,7 +21,6 @@
 
 namespace ngraph
 {
-#ifdef NO_GLOBAL_TYPE_ALIASES
     /// \brief A set of axes.
     class AxisSet : public std::set<size_t>
     {
@@ -54,8 +53,4 @@ namespace ngraph
             return *this;
         }
     };
-#else
-    // Deprecated definition
-    using AxisSet = std::set<size_t>;
-#endif
 }

src/ngraph/axis_vector.hpp

@@ -21,7 +21,6 @@
 
 namespace ngraph
 {
-#ifdef NO_GLOBAL_TYPE_ALIASES
     /// \brief A vector of axes.
     class AxisVector : public std::vector<size_t>
     {
@@ -64,8 +63,4 @@ namespace ngraph
             return *this;
         }
     };
-#else
-    // Deprecated definition
-    using AxisVector = std::vector<size_t>;
-#endif
 }

src/ngraph/coordinate.hpp

@@ -24,7 +24,6 @@
 
 namespace ngraph
 {
-#ifdef NO_GLOBAL_TYPE_ALIASES
     /// \brief Coordinates for a tensor element
     class Coordinate : public std::vector<size_t>
     {
@@ -73,10 +72,6 @@ namespace ngraph
             return *this;
         }
     };
-#else
-    // Deprecated definition
-    using Coordinate = std::vector<size_t>;
-#endif
 
     // Removes some values from a vector of axis values
     template <typename AXIS_VALUES>

src/ngraph/coordinate_diff.hpp

@@ -21,7 +21,6 @@
 
 namespace ngraph
 {
-#ifdef NO_GLOBAL_TYPE_ALIASES
     /// \brief A difference (signed) of tensor element coordinates.
     class CoordinateDiff : public std::vector<std::ptrdiff_t>
     {
@@ -64,8 +63,4 @@ namespace ngraph
             return *this;
         }
     };
-#else
-    // Deprecated definition
-    using CoordinateDiff = std::vector<std::ptrdiff_t>;
-#endif
 }

src/ngraph/ngraph.hpp

@@ -67,6 +67,7 @@
 #include "ngraph/ops/abs.hpp"
 #include "ngraph/ops/acos.hpp"
 #include "ngraph/ops/add.hpp"
+#include "ngraph/ops/allreduce.hpp"
 #include "ngraph/ops/asin.hpp"
 #include "ngraph/ops/atan.hpp"
 #include "ngraph/ops/avg_pool.hpp"
@@ -130,7 +131,3 @@
 #include "ngraph/shape.hpp"
 #include "ngraph/types/element_type.hpp"
 #include "ngraph/types/type.hpp"
-
-#ifdef NGRAPH_DISTRIBUTED
-#include "ngraph/ops/allreduce.hpp"
-#endif

src/ngraph/node_vector.hpp

@@ -23,8 +23,6 @@ namespace ngraph
 {
     class Node;
 
-#ifdef NO_GLOBAL_TYPE_ALIASES
-
     /// \brief Zero or more nodes.
     class NodeVector : public std::vector<std::shared_ptr<Node>>
     {
@@ -46,10 +44,4 @@ namespace ngraph
 
         NodeVector() {}
     };
-
-#else
-    // Deprecated definitions
-    using NodeVector = std::vector<std::shared_ptr<Node>>;
-    using Nodes = NodeVector;
-#endif
 }

src/ngraph/ops/allreduce.cpp

@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // ----------------------------------------------------------------------------
 
-#ifdef NGRAPH_DISTRIBUTED
-
 #include "ngraph/ops/allreduce.hpp"
 
 using namespace std;
@@ -31,5 +29,3 @@ op::AllReduce::AllReduce(const std::shared_ptr<Node>& arg)
         throw ngraph_error("Unsupported data type for AllReduce");
     }
 }
-
-#endif

src/ngraph/ops/allreduce.hpp

@@ -14,8 +14,6 @@
 
 #pragma once
 
-#ifdef NGRAPH_DISTRIBUTED
-
 #include <memory>
 #include "ngraph/ops/util/requires_tensor_view_args.hpp"
 
@@ -40,5 +38,3 @@ namespace ngraph
         };
     }
 }
-
-#endif

src/ngraph/ops/parameter_vector.hpp

@@ -24,8 +24,6 @@ namespace ngraph
     {
         class Parameter;
 
-#ifdef NO_GLOBAL_TYPE_ALIASES
-
         /// \brief Zero or more nodes.
         class ParameterVector : public std::vector<std::shared_ptr<op::Parameter>>
         {
@@ -47,11 +45,5 @@ namespace ngraph
 
             ParameterVector() {}
         };
-
-#else
-        // Deprecated definitions
-        using ParameterVector = std::vector<std::shared_ptr<op::Parameter>>;
-        using Parameters = ParameterVector;
-#endif
     }
 }

src/ngraph/runtime/cpu/cpu_external_function.cpp

@@ -249,6 +249,10 @@ runtime::cpu::CPU_ExternalFunction::CPU_ExternalFunction(
 {
 }
 
+runtime::cpu::CPU_ExternalFunction::~CPU_ExternalFunction()
+{
+}
+
 void runtime::cpu::CPU_ExternalFunction::compile()
 {
     if (m_is_compiled)
@@ -256,14 +260,14 @@ void runtime::cpu::CPU_ExternalFunction::compile()
         return;
     }
 
-    m_mkldnn_emitter.reset(new MKLDNNEmitter(shared_from_this()));
+    m_mkldnn_emitter.reset(new MKLDNNEmitter());
 
     ngraph::pass::Manager pass_manager;
 
     pass_manager.register_pass<ngraph::pass::CoreFusion>();
     pass_manager.register_pass<runtime::cpu::pass::CPUFusion>();
-    pass_manager.register_pass<runtime::cpu::pass::CPUAssignment>(shared_from_this());
-    pass_manager.register_pass<runtime::cpu::pass::CPULayout>(shared_from_this());
+    pass_manager.register_pass<runtime::cpu::pass::CPUAssignment>(this);
+    pass_manager.register_pass<runtime::cpu::pass::CPULayout>(this);
     pass_manager.register_pass<ngraph::pass::Liveness>();
     pass_manager.register_pass<ngraph::pass::MemoryLayout>(s_memory_pool_alignment);
 

src/ngraph/runtime/cpu/cpu_external_function.hpp

@@ -75,6 +75,7 @@ namespace ngraph
             public:
                 CPU_ExternalFunction(const std::shared_ptr<ngraph::Function>& function,
                                      bool release_function = true);
+                ~CPU_ExternalFunction();
                 std::shared_ptr<ngraph::runtime::CallFrame> make_call_frame();
 
                 const LayoutDescriptorPtrs& get_parameter_layout_descriptors();

src/ngraph/runtime/cpu/cpu_manager.cpp

@@ -30,7 +30,8 @@ std::shared_ptr<ngraph::runtime::Backend> runtime::cpu::CPU_Manager::allocate_ba
 std::shared_ptr<ngraph::runtime::ExternalFunction>
     runtime::cpu::CPU_Manager::compile(const std::shared_ptr<ngraph::Function>& fun)
 {
-    return std::make_shared<CPU_ExternalFunction>(fun);
+    auto rc = std::make_shared<CPU_ExternalFunction>(fun);
+    return rc;
 }
 
 ngraph::runtime::Manager::Factory runtime::cpu::CPU_Manager::factory =

src/ngraph/runtime/cpu/mkldnn_emitter.hpp

@@ -23,6 +23,7 @@
 #include <mkldnn.hpp>
 
 #include "ngraph/coordinate_diff.hpp"
+#include "ngraph/shape.hpp"
 #include "ngraph/strides.hpp"
 
 namespace ngraph
@@ -37,11 +38,7 @@ namespace ngraph
             class MKLDNNEmitter
             {
             public:
-                MKLDNNEmitter(std::shared_ptr<CPU_ExternalFunction> ef)
-                    : external_function(ef)
-                {
-                }
-
+                MKLDNNEmitter() {}
                 const std::vector<mkldnn::primitive*>& get_mkldnn_primitives() const;
 
                 size_t insert_primitive(mkldnn::primitive* primitive);
@@ -69,6 +66,37 @@ namespace ngraph
                                                  const ngraph::CoordinateDiff& padding_below,
                                                  const ngraph::CoordinateDiff& padding_above);
 
+                size_t
+                    build_convolution_backward_weights(const mkldnn::memory::desc& input_desc,
+                                                       const mkldnn::memory::desc& delta_desc,
+                                                       const mkldnn::memory::desc& result_desc,
+                                                       const ngraph::Strides& strides,
+                                                       const ngraph::Strides& dilation_strides,
+                                                       const ngraph::CoordinateDiff& padding_below,
+                                                       const ngraph::CoordinateDiff& padding_above);
+
+                size_t build_convolution_backward_data(const mkldnn::memory::desc& weights_desc,
+                                                       const mkldnn::memory::desc& delta_desc,
+                                                       const mkldnn::memory::desc& result_desc,
+                                                       const ngraph::Strides& strides,
+                                                       const ngraph::Strides& dilation_strides,
+                                                       const ngraph::CoordinateDiff& padding_below,
+                                                       const ngraph::CoordinateDiff& padding_above);
+
+                size_t build_pooling_forward(mkldnn::algorithm pooling_algorithm,
+                                             const mkldnn::memory::desc& input_desc,
+                                             const mkldnn::memory::desc& result_desc,
+                                             const ngraph::Strides& window_strides,
+                                             const ngraph::Shape& window_shape,
+                                             const ngraph::Shape& padding_below,
+                                             const ngraph::Shape& padding_above);
+
+                size_t build_reorder(const mkldnn::memory::desc& input_desc,
+                                     const mkldnn::memory::desc& result_desc);
+
+                size_t build_relu_forward(const mkldnn::memory::desc& input_desc,
+                                          const mkldnn::memory::desc& result_desc);
+
                 size_t build_elementwise_add(
                     const mkldnn::memory::desc& input0_data_desc,
                     const mkldnn::memory::desc& input1_data_desc,
@@ -77,10 +105,9 @@ namespace ngraph
                     const std::vector<mkldnn::memory::primitive_desc>& input_pd);
 
             private:
-                std::shared_ptr<CPU_ExternalFunction> external_function;
-                std::vector<mkldnn::primitive*> mkldnn_primitives;
-                std::vector<mkldnn::stream> mkldnn_streams;
-                std::unordered_map<size_t, std::vector<size_t>> primitive_deps;
+                std::vector<mkldnn::primitive*> m_mkldnn_primitives;
+                std::vector<mkldnn::stream> m_mkldnn_streams;
+                std::unordered_map<size_t, std::vector<size_t>> m_primitive_deps;
             };
         }
     }

src/ngraph/runtime/cpu/pass/cpu_assignment.cpp

@@ -256,7 +256,7 @@ bool runtime::cpu::pass::CPUAssignment::run_on_call_graph(
         auto handler = s_dispatcher.find(TI(n));
         if (handler != s_dispatcher.end())
         {
-            handler->second(m_external_function.get(), node.get());
+            handler->second(m_external_function, node.get());
         }
     }
 

src/ngraph/runtime/cpu/pass/cpu_assignment.hpp

@@ -39,7 +39,7 @@ namespace ngraph
                 class CPUAssignment : public ngraph::pass::CallGraphPass
                 {
                 public:
-                    CPUAssignment(std::shared_ptr<CPU_ExternalFunction> external_function)
+                    CPUAssignment(CPU_ExternalFunction* external_function)
                         : m_external_function(external_function)
                     {
                     }
@@ -56,7 +56,7 @@ namespace ngraph
                     }
 
                 private:
-                    std::shared_ptr<CPU_ExternalFunction> m_external_function;
+                    CPU_ExternalFunction* m_external_function;
                 };
             }
         }

src/ngraph/runtime/cpu/pass/cpu_layout.cpp

@@ -719,11 +719,11 @@ bool runtime::cpu::pass::CPULayout::run_on_call_graph(const std::list<std::share
         auto handler = s_dispatcher.find(TI(n));
         if (handler != s_dispatcher.end())
         {
-            handler->second(m_external_function.get(), node);
+            handler->second(m_external_function, node);
         }
         else
         {
-            set_default_layouts(m_external_function.get(), node);
+            set_default_layouts(m_external_function, node);
         }
     }
 

src/ngraph/runtime/cpu/pass/cpu_layout.hpp

@@ -39,7 +39,7 @@ namespace ngraph
                 class CPULayout : public ngraph::pass::CallGraphPass
                 {
                 public:
-                    CPULayout(std::shared_ptr<CPU_ExternalFunction> external_function)
+                    CPULayout(CPU_ExternalFunction* external_function)
                         : m_external_function(external_function)
                     {
                     }
@@ -52,7 +52,7 @@ namespace ngraph
                                std::shared_ptr<ngraph::Node> node);
 
                 private:
-                    std::shared_ptr<CPU_ExternalFunction> m_external_function;
+                    CPU_ExternalFunction* m_external_function;
                     static std::shared_ptr<Node> insert_input_conversions(
                         CPU_ExternalFunction* external_function,
                         std::shared_ptr<Node>& node,

src/ngraph/serializer.cpp

@@ -19,6 +19,7 @@
 #include "ngraph/ops/abs.hpp"
 #include "ngraph/ops/acos.hpp"
 #include "ngraph/ops/add.hpp"
+#include "ngraph/ops/allreduce.hpp"
 #include "ngraph/ops/asin.hpp"
 #include "ngraph/ops/atan.hpp"
 #include "ngraph/ops/avg_pool.hpp"
@@ -77,10 +78,6 @@
 #include "ngraph/util.hpp"
 #include "nlohmann/json.hpp"
 
-#ifdef NGRAPH_DISTRIBUTED
-#include "ngraph/ops/allreduce.hpp"
-#endif
-
 using namespace ngraph;
 using namespace std;
 using json = nlohmann::json;
@@ -279,12 +276,10 @@ static shared_ptr<ngraph::Function>
         {
             node = make_shared<op::Add>(args[0], args[1]);
         }
-#ifdef NGRAPH_DISTRIBUTED
         else if (node_op == "AllReduce")
         {
             node = make_shared<op::AllReduce>(args[0]);
         }
-#endif
         else if (node_op == "Asin")
         {
             node = make_shared<op::Asin>(args[0]);

src/ngraph/shape.hpp

@@ -25,7 +25,6 @@
 
 namespace ngraph
 {
-#ifdef NO_GLOBAL_TYPE_ALIASES
     /// \brief Shape for a tensor.
     class Shape : public std::vector<size_t>
     {
@@ -68,10 +67,6 @@ namespace ngraph
             return *this;
         }
     };
-#else
-    // Deprecated definition
-    using Shape = std::vector<size_t>;
-#endif
 
     /// Number of elements in spanned by a shape
     size_t shape_size(const Shape& shape);

src/ngraph/strides.hpp

@@ -20,7 +20,6 @@
 
 namespace ngraph
 {
-#ifdef NO_GLOBAL_TYPE_ALIASES
     /// \brief Strides for a tensor.
     class Strides : public std::vector<size_t>
     {
@@ -63,8 +62,4 @@ namespace ngraph
             return *this;
         }
     };
-#else
-    // Deprecated definition
-    using Strides = std::vector<size_t>;
-#endif
 }

test/backend_test.in.cpp

@@ -43,6 +43,29 @@ static const vector<element::Type> s_known_element_types = {element::from<float>
                                                             element::from<uint32_t>(),
                                                             element::from<uint64_t>()};
 
+TEST(${BACKEND_NAME}, component_cleanup)
+{
+    shared_ptr<runtime::Backend> backend;
+    shared_ptr<runtime::ExternalFunction> external;
+    shared_ptr<runtime::CallFrame> cf;
+    {
+        Shape shape{2, 2};
+        auto A = make_shared<op::Parameter>(element::f32, shape);
+        auto B = make_shared<op::Parameter>(element::f32, shape);
+        auto f = make_shared<Function>(A + B, op::ParameterVector{A, B});
+
+        auto manager = runtime::Manager::get("${BACKEND_NAME}");
+        external = manager->compile(f);
+        backend = manager->allocate_backend();
+        cf = backend->make_call_frame(external);
+    }
+    EXPECT_EQ(cf.use_count(), 1);
+    cf = nullptr;
+    EXPECT_EQ(backend.use_count(), 1);
+    backend = nullptr;
+    EXPECT_EQ(external.use_count(), 1);
+}
+
 TEST(${BACKEND_NAME}, aliased_output)
 {
     Shape shape{2, 2};
@@ -138,7 +161,7 @@ TEST(${BACKEND_NAME}, abc)
     auto f = make_shared<Function>((A + B) * C, op::ParameterVector{A, B, C});
 
     auto manager = runtime::Manager::get("${BACKEND_NAME}");
-    auto external = manager->compile(f);
+    shared_ptr<runtime::ExternalFunction> external = manager->compile(f);
     auto backend = manager->allocate_backend();
     auto cf = backend->make_call_frame(external);