Copy params used as output from input

src/ngraph/autodiff/numeric_derivative.cpp

@@ -37,13 +37,16 @@ std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>
     Shape y_shape =
         std::dynamic_pointer_cast<const TensorViewType>(y->get_value_type())->get_shape();
 
+    auto params = f->get_parameters();
+
     // Results for each derivative, shape Y|X_i
     std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> results;
-    for (size_t i = 0; i < args.size(); i++)
+    for (auto param : params)
     {
         Shape s = y_shape;
-        auto arg_shape = args[i]->get_shape();
-        s.insert(s.end(), arg_shape.begin(), arg_shape.end());
+        auto param_shape =
+            std::dynamic_pointer_cast<const TensorViewType>(param->get_value_type())->get_shape();
+        s.insert(s.end(), param_shape.begin(), param_shape.end());
         results.push_back(backend->make_parameterized_tensor_view<ET>(s));
     }
 

src/ngraph/ops/parameter.cpp

@@ -32,3 +32,7 @@ Parameter::Parameter(const ngraph::element::Type& element_type, const Shape& sha
 void Parameter::propagate_types()
 {
 }
+
+void Parameter::generate_adjoints(autodiff::Adjoints& adjoints, const std::shared_ptr<Node>& delta)
+{
+}

src/ngraph/ops/parameter.hpp

@@ -49,9 +49,7 @@ namespace ngraph
         {
         protected:
             virtual void generate_adjoints(autodiff::Adjoints& adjoints,
-                                           const std::shared_ptr<Node>& delta) override
-            {
-            }
+                                           const std::shared_ptr<Node>& delta) override;
 
         public:
             /// \brief Constructions a parameter node.

src/ngraph/runtime/ngvm/call_frame.cpp

@@ -22,13 +22,15 @@ using namespace ngraph::runtime::ngvm;
 
 CallFrame::CallFrame(size_t n_inputs,
                      size_t n_outputs,
+                     size_t frame_size,
                      const TensorViewPtrs& temps,
                      size_t initial_pc,
                      const shared_ptr<vector<shared_ptr<Instruction>>>& instructions)
 
     : m_n_inputs(n_inputs)
     , m_n_outputs(n_outputs)
-    , m_tensor_views(n_inputs + n_outputs + temps.size())
+    , m_frame_size(frame_size)
+    , m_tensor_views(m_frame_size)
     , m_initial_pc(initial_pc)
     , m_instructions(instructions)
 {

src/ngraph/runtime/ngvm/call_frame.hpp

@@ -38,6 +38,7 @@ namespace ngraph
                 CallFrame(
                     size_t n_inputs,
                     size_t n_outputs,
+                    size_t frame_size,
                     const TensorViewPtrs& temps,
                     size_t initial_pc,
                     const std::shared_ptr<std::vector<std::shared_ptr<Instruction>>>& instructions);
@@ -69,6 +70,7 @@ namespace ngraph
             protected:
                 size_t m_n_inputs;
                 size_t m_n_outputs;
+                size_t m_frame_size;
                 TensorViewPtrs m_tensor_views;
                 size_t m_initial_pc;
                 std::shared_ptr<std::vector<std::shared_ptr<Instruction>>> m_instructions;

src/ngraph/runtime/ngvm/external_function.cpp

@@ -1053,20 +1053,38 @@ void ExternalFunction::compile(FunctionMap& function_map)
         for (const descriptor::Output& output : param->get_outputs())
         {
             auto tv = output.get_tensor_view();
-            size_t index = tensor_index.size();
+            size_t index = m_frame_size++;
             tensor_index[tv] = index;
         }
     }
-    m_n_inputs = tensor_index.size();
+    m_n_inputs = m_frame_size;
 
-    // Next are the function outputs
     for (const descriptor::Output& output : m_function->get_result()->get_outputs())
     {
         auto tv = output.get_tensor_view();
-        size_t index = tensor_index.size();
+        size_t index = m_frame_size++;
+        auto prev_index_it = tensor_index.find(tv);
+        if (prev_index_it != tensor_index.end())
+        {
+            auto result_tensor_type =
+                dynamic_pointer_cast<const TensorViewType>(tv->get_value_type());
+            assert(nullptr != result_tensor_type);
+            auto& result_element_type = result_tensor_type->get_element_type();
+            auto ef = this;
+            PUSH_POLYMORPHIC_INSTRUCTION(result_element_type,
+                                         "Copy has unhandled element type",
+                                         eigen::CopyInstruction,
+                                         prev_index_it->second,
+                                         index);
+        }
+        else
+        {
             tensor_index[tv] = index;
         }
-    m_n_outputs = tensor_index.size() - m_n_inputs;
+    }
+    m_n_outputs = m_frame_size - m_n_inputs;
+    vector<shared_ptr<Instruction>> input_output_copies;
+    swap(*m_instructions, input_output_copies);
 
     // All remaining tensor views
     for (shared_ptr<Node> node : m_function->get_ordered_ops())
@@ -1076,7 +1094,7 @@ void ExternalFunction::compile(FunctionMap& function_map)
             auto tv = output.get_tensor_view();
             if (0 == tensor_index.count(tv))
             {
-                size_t index = tensor_index.size();
+                size_t index = m_frame_size++;
                 tensor_index[tv] = index;
                 m_temp_views.push_back(tv);
             }
@@ -1109,6 +1127,8 @@ void ExternalFunction::compile(FunctionMap& function_map)
         }
         handler_it->second(node.get(), this, function_map, in, out);
     }
+    m_instructions->insert(
+        m_instructions->end(), input_output_copies.begin(), input_output_copies.end());
     m_instructions->push_back(make_shared<eigen::ReturnInstruction>());
     m_is_compiled = true;
     if (m_release_function)
@@ -1141,5 +1161,5 @@ shared_ptr<ngraph::runtime::CallFrame> ExternalFunction::make_call_frame(Functio
 #undef M
     }
     return make_shared<ngraph::runtime::ngvm::CallFrame>(
-        m_n_inputs, m_n_outputs, temps, 0, m_instructions);
+        m_n_inputs, m_n_outputs, m_frame_size, temps, 0, m_instructions);
 }

src/ngraph/runtime/ngvm/external_function.hpp

@@ -62,6 +62,7 @@ namespace ngraph
 
                 size_t m_n_inputs;
                 size_t m_n_outputs;
+                size_t m_frame_size{0};
                 std::shared_ptr<std::vector<std::shared_ptr<Instruction>>> m_instructions;
                 ngraph::descriptor::TensorViewPtrs m_temp_views;
 

src/ngraph/test/all_close.cpp

@@ -19,6 +19,43 @@
 #include "ngraph/except.hpp"
 #include "ngraph/test/all_close.hpp"
 
+template <typename ET>
+bool ngraph::test::all_close(
+    const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>& as,
+    const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>& bs,
+    typename ET::type rtol,
+    typename ET::type atol)
+{
+    if (as.size() != bs.size())
+    {
+        return false;
+    }
+    for (size_t i = 0; i < as.size(); ++i)
+    {
+        if (!all_close(as[i], bs[i], rtol, atol))
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+template bool ngraph::test::all_close<ngraph::element::Float32>(
+    const std::vector<
+        std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>>& as,
+    const std::vector<
+        std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>>& bs,
+    ngraph::element::Float32::type rtol,
+    ngraph::element::Float32::type atol);
+
+template bool ngraph::test::all_close<ngraph::element::Float64>(
+    const std::vector<
+        std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>>& as,
+    const std::vector<
+        std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>>& bs,
+    ngraph::element::Float64::type rtol,
+    ngraph::element::Float64::type atol);
+
 template <typename ET>
 bool ngraph::test::all_close(const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& a,
                              const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& b,

src/ngraph/test/all_close.hpp

@@ -15,6 +15,7 @@
 #pragma once
 
 #include <memory>
+#include <vector>
 
 #include "ngraph/runtime/parameterized_tensor_view.hpp"
 #include "ngraph/types/element_type.hpp"
@@ -23,6 +24,35 @@ namespace ngraph
 {
     namespace test
     {
+        /// @brief Same as numpy.allclose
+        /// @param as First tensors to compare
+        /// @param bs Second tensors to compare
+        /// @param rtol Relative tolerance
+        /// @param atol Absolute tolerance
+        /// Returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
+        template <typename ET>
+        bool all_close(
+            const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>& as,
+            const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>& bs,
+            typename ET::type rtol,
+            typename ET::type atol);
+
+        extern template bool ngraph::test::all_close<ngraph::element::Float32>(
+            const std::vector<std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>>& as,
+            const std::vector<std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>>& bs,
+            ngraph::element::Float32::type rtol,
+            ngraph::element::Float32::type atol);
+
+        extern template bool ngraph::test::all_close<ngraph::element::Float64>(
+            const std::vector<std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>>& as,
+            const std::vector<std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>>& bs,
+            ngraph::element::Float64::type rtol,
+            ngraph::element::Float64::type atol);
+
         /// @brief Same as numpy.allclose
         /// @param a First tensor to compare
         /// @param b Second tensor to compare

test/autodiff.cpp

@@ -13,6 +13,7 @@
 // ----------------------------------------------------------------------------
 
 #include <algorithm>
+#include <functional>
 #include <memory>
 #include <tuple>
 
@@ -28,9 +29,37 @@
 using namespace std;
 using namespace ngraph;
 
+template <typename ET>
+bool autodiff_numeric_compare(
+    const std::shared_ptr<runtime::Manager>& manager,
+    const std::shared_ptr<runtime::Backend>& backend,
+    std::function<std::shared_ptr<Function>()> make_graph,
+    const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args,
+    typename ET::type rtol,
+    typename ET::type atol)
+{
+    auto results_num =
+        autodiff::numeric_derivative<element::Float32>(manager, backend, make_graph(), args, .001f);
+    auto results_sym =
+        autodiff::backprop_derivative<element::Float32>(manager, backend, make_graph(), args);
+    return test::all_close(results_num, results_sym, .01f, .01f);
+}
+
 TEST(backwards, parameter)
 {
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
     auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(X0, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0});
+    };
+    EXPECT_TRUE(
+        autodiff_numeric_compare<element::Float32>(manager, backend, make_graph, {x0}, .01f, .01f));
+
     auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
     auto Y = X0;
     auto C = make_shared<op::Parameter>(element::Float32::element_type(), shape);
@@ -59,13 +88,7 @@ TEST(backwards, add)
         manager, backend, make_graph(), {x0, x1}, .001f);
     auto results_sym =
         autodiff::backprop_derivative<element::Float32>(manager, backend, make_graph(), {x0, x1});
-    for (size_t i = 0; i < results_num.size(); ++i)
-    {
-        auto result_num = results_num[i];
-        auto result_sym = results_sym[i];
-        bool ac = test::all_close(result_num, result_sym, .01f, .01f);
-        EXPECT_TRUE(ac);
-    }
+    EXPECT_TRUE(test::all_close(results_num, results_sym, .01f, .01f));
 }
 
 TEST(backwards, multiply)
@@ -84,16 +107,6 @@ TEST(backwards, multiply)
         return make_shared<Function>(
             X0 * X1, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
     };
-
-    auto results_num = autodiff::numeric_derivative<element::Float32>(
-        manager, backend, make_graph(), {x0, x1}, .001f);
-    auto results_sym =
-        autodiff::backprop_derivative<element::Float32>(manager, backend, make_graph(), {x0, x1});
-    for (size_t i = 0; i < results_num.size(); ++i)
-    {
-        auto result_num = results_num[i];
-        auto result_sym = results_sym[i];
-        bool ac = test::all_close(result_num, result_sym, .01f, .01f);
-        EXPECT_TRUE(ac);
-    }
+    EXPECT_TRUE(autodiff_numeric_compare<element::Float32>(
+        manager, backend, make_graph, {x0, x1}, .01f, .01f));
 }