Numeric derivative, reorg mult test for numeric derivative

src/ngraph/runtime/utils.cpp

@@ -12,9 +12,12 @@
 // See the License for the specific language governing permissions and
 // ----------------------------------------------------------------------------
 
+#include <algorithm>
 #include <cassert>
 #include <cmath>
 
+#include "ngraph/function.hpp"
+#include "ngraph/runtime/call_frame.hpp"
 #include "ngraph/runtime/utils.hpp"
 
 std::shared_ptr<ngraph::runtime::Tuple> ngraph::runtime::make_tuple(
@@ -77,3 +80,92 @@ template bool ngraph::runtime::all_close<double>(const std::vector<double>& a,
                                                  const std::vector<double>& b,
                                                  double rtol,
                                                  double atol);
+
+template <typename ET>
+std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>
+    ngraph::runtime::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<ngraph::Function>& f,
+        const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>& args,
+        typename ET::type delta)
+{
+    auto y = f->get_result();
+
+    Shape y_shape =
+        std::dynamic_pointer_cast<const ngraph::descriptor::TensorView>(y->get_value_type())
+            ->get_tensor_view_type()
+            ->get_shape();
+
+    // Check all the shapes
+    std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>> results;
+    for (size_t i = 0; i < args.size(); i++)
+    {
+        Shape s = y_shape;
+        auto arg_shape = args[i]->get_shape();
+        s.insert(s.end(), arg_shape.begin(), arg_shape.end());
+        results.push_back(backend->make_parameterized_tensor_view<ET>(s));
+    }
+
+    auto external = manager->compile(f);
+    auto cf = backend->make_call_frame(external);
+
+    // ref_y is the function evaluated at the args
+    auto ref_y = backend->make_parameterized_tensor_view<ET>(y_shape);
+
+    ngraph::runtime::TensorViewPtrs args_tv;
+    args_tv.insert(args_tv.begin(), args.begin(), args.end());
+
+    cf->tensor_call(args_tv, TensorViewPtrs{ref_y});
+    auto ref_vec = ref_y->get_vector();
+
+    // inc_y will hold f(x+dx) values
+    auto inc_y = backend->make_parameterized_tensor_view<ET>(y_shape);
+    auto inc_vec = inc_y->get_vector();
+
+    // Assuming vars, y, and results are row-major
+
+    typename ET::type inv_delta = 1 / delta;
+    for (size_t i = 0; i < args.size(); ++i)
+    {
+        auto arg = args[i];
+        auto df_darg = results[i];
+        auto df_darg_it = df_darg->get_vector().begin();
+        std::vector<typename ET::type>& vec = arg->get_vector();
+        for (size_t j = 0; j < vec.size(); i++)
+        {
+            auto old_val = vec[j];
+            vec[j] += delta;
+            cf->tensor_call(args_tv, {inc_y});
+            vec[j] = old_val;
+            df_darg_it = std::transform(inc_vec.begin(),
+                                        inc_vec.end(),
+                                        ref_vec.begin(),
+                                        df_darg_it,
+                                        [inv_delta](typename ET::type y1, typename ET::type y0) {
+                                            return inv_delta * (y1 - y0);
+                                        });
+        }
+    }
+    return results;
+}
+
+template std::vector<
+    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+    ngraph::runtime::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<ngraph::Function>& f,
+        const std::vector<
+            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float32>>>& args,
+        element::Float32::type delta);
+
+template std::vector<
+    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+    ngraph::runtime::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<ngraph::Function>& f,
+        const std::vector<
+            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>& args,
+        element::Float64::type delta);

src/ngraph/runtime/utils.hpp

@@ -17,6 +17,8 @@
 #include <memory>
 #include <vector>
 
+#include "ngraph/runtime/backend.hpp"
+#include "ngraph/runtime/manager.hpp"
 #include "ngraph/runtime/parameterized_tensor_view.hpp"
 #include "ngraph/runtime/tuple.hpp"
 #include "ngraph/runtime/value.hpp"
@@ -66,6 +68,12 @@ namespace ngraph
             ngraph::element::Float64::type rtol,
             ngraph::element::Float64::type atol);
 
+        /// @brief Same as numpy.allclose
+        /// @param a First tensor to compare
+        /// @param b Second tensor to compare
+        /// @param rtol Relative tolerance
+        /// @param atol Absolute tolerance
+        /// @returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
         template <typename T>
         bool all_close(const std::vector<T>& a,
                        const std::vector<T>& b,
@@ -81,5 +89,42 @@ namespace ngraph
                                                                 const std::vector<double>& b,
                                                                 double rtol,
                                                                 double atol);
+
+        /// @brief numeric approximation of the derivative
+        /// @param f A function
+        /// @param args Values for the arguments (the independent variables)
+        /// @param delta increment for the variables
+        /// @returns vector of dy/dvar, where each dy/dvar's shape is concat(y.shape(), var.shape())
+        template <typename ET>
+        std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>
+            numeric_derivative(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>&
+                    args,
+                typename ET::type delta);
+
+        extern template std::vector<
+            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float32>>>
+            ngraph::runtime::numeric_derivative(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<ngraph::Function>& f,
+                const std::vector<
+                    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float32>>>&
+                    args,
+                element::Float32::type delta);
+
+        extern template std::vector<
+            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>
+            ngraph::runtime::numeric_derivative(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<ngraph::Function>& f,
+                const std::vector<
+                    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>&
+                    args,
+                element::Float64::type delta);
     }
 }

test/autodiff.cpp

@@ -14,6 +14,7 @@
 
 #include <algorithm>
 #include <memory>
+#include <tuple>
 
 #include "gtest/gtest.h"
 
@@ -65,22 +66,27 @@ shared_ptr<Function> derivative(const std::shared_ptr<Node>& Y,
 TEST(backwards, multiply)
 {
     auto shape = Shape{2, 3};
-    auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
-    auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
-    auto Y = X0 * X1;
-    auto f = derivative(Y, {X0, X1});
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return std::make_tuple(X0 * X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
+    };
+    auto val_and_vars = make_graph();
+    auto Y = std::get<0>(val_and_vars);
+    auto vars = std::get<1>(val_and_vars);
+    auto f = derivative(Y, vars);
 
     auto manager = runtime::Manager::get("NGVM");
     auto external = manager->compile(f);
     auto backend = manager->allocate_backend();
     auto cf = backend->make_call_frame(external);
 
-    auto x0 = backend->make_parameterized_tensor_view<element::Float32>(shape);
-    *x0 = vector<float>{1, 3, 5, 7, 9, 11};
-    auto x1 = backend->make_parameterized_tensor_view<element::Float32>(shape);
-    *x1 = vector<float>{0, 2, 4, 6, 8, 10};
-    auto c = backend->make_parameterized_tensor_view<element::Float32>(shape);
-    *c = vector<float>{0, 0, 0, 0, 0, 0};
+    auto x0 = backend->make_parameterized_tensor_view<element::Float32>(
+        runtime::NDArray<float, 2>({{1, 3, 5}, {7, 9, 11}}));
+    auto x1 = backend->make_parameterized_tensor_view<element::Float32>(
+        runtime::NDArray<float, 2>({{0, 2, 4}, {6, 8, 10}}));
+    auto c = backend->make_parameterized_tensor_view<element::Float32>(
+        runtime::NDArray<float, 2>({{0, 0, 0}, {0, 0, 0}}));
 
     auto dx0 = backend->make_parameterized_tensor_view<element::Float32>(shape);
     auto dx1 = backend->make_parameterized_tensor_view<element::Float32>(shape);