Housekeeping

src/ngraph/CMakeLists.txt

@@ -13,6 +13,9 @@
 
 set (SRC
     autodiff/adjoints.cpp
+    autodiff/backprop_derivative.cpp
+    autodiff/backprop_function.cpp
+    autodiff/numeric_derivative.cpp
     descriptor/input.cpp
     descriptor/layout/dense_tensor_view_layout.cpp
     descriptor/layout/tensor_view_layout.cpp
@@ -67,6 +70,7 @@ set (SRC
     runtime/tensor_view.cpp
     runtime/tuple.cpp
     runtime/utils.cpp
+    test/all_close.cpp
     shape.cpp
     types/element_type.cpp
     types/type.cpp

src/ngraph/autodiff/adjoints.cpp

@@ -19,6 +19,7 @@
 #include <unordered_set>
 
 #include "ngraph/autodiff/adjoints.hpp"
+#include "ngraph/function.hpp"
 #include "ngraph/node.hpp"
 #include "ngraph/ops/add.hpp"
 #include "ngraph/ops/broadcast.hpp"

src/ngraph/autodiff/adjoints.hpp

@@ -17,9 +17,18 @@
 #include <memory>
 #include <unordered_map>
 
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+
 namespace ngraph
 {
     class Node;
+    class Function;
+
+    namespace runtime
+    {
+        class Backend;
+        class Manager;
+    }
 
     namespace autodiff
     {
@@ -50,5 +59,35 @@ namespace ngraph
         protected:
             std::unordered_map<Node*, std::shared_ptr<Node>> m_adjoint_map;
         };
+
+        /// @brief Returns a FunctionSpec for the backprop derivative of its argument.
+        /// @param f is f(X_i...)
+        /// @returns f'(X_i..., c) where f'(x_i, ..., c)_j is backprop for X_j
+        std::shared_ptr<Function> backprop_function(const std::shared_ptr<Function>& f);
+
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> backprop_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+            backprop_derivative<ngraph::element::Float32>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+            backprop_derivative<ngraph::element::Float64>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args);
     }
 }

src/ngraph/autodiff/backprop_derivative.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <memory>
+#include <vector>
+
+#include "ngraph/autodiff/backprop_derivative.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/runtime/backend.hpp"
+#include "ngraph/runtime/call_frame.hpp"
+#include "ngraph/runtime/manager.hpp"
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+#include "ngraph/types/type.hpp"
+
+using namespace ngraph;
+
+template <typename ET>
+std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>
+    autodiff::backprop_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args)
+{
+    auto y = f->get_result();
+    Shape y_shape =
+        std::dynamic_pointer_cast<const TensorViewType>(y->get_value_type())->get_shape();
+
+    auto c_param = std::make_shared<op::Parameter>(ET::element_type(), y_shape);
+    auto c_arg = backend->make_parameterized_tensor_view<ET>(y_shape);
+    auto params = f->get_parameters();
+
+    std::vector<std::shared_ptr<Node>> deriv_nodes;
+    std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> bprops;
+    std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> results;
+    for (auto param : params)
+    {
+        Shape s = y_shape;
+        auto param_shape =
+            std::dynamic_pointer_cast<const TensorViewType>(param->get_value_type())->get_shape();
+        s.insert(s.end(), param_shape.begin(), param_shape.end());
+        results.push_back(backend->make_parameterized_tensor_view<ET>(s));
+        bprops.push_back(backend->make_parameterized_tensor_view<ET>(param_shape));
+        deriv_nodes.push_back(y->backprop_node(param, c_param));
+    }
+
+    std::vector<std::shared_ptr<op::Parameter>> df_params = params;
+    df_params.push_back(c_param);
+    auto df_result = std::make_shared<op::Tuple>(deriv_nodes);
+    auto df = std::make_shared<Function>(df_result, df_result->get_value_type(), df_params);
+
+    auto external = manager->compile(df);
+    auto cf = backend->make_call_frame(external);
+
+    // We compute the derivatives chunk by chunk
+    std::vector<typename std::vector<typename ET::type>::iterator> result_pos;
+    for (auto result : results)
+    {
+        result_pos.push_back(result->get_vector().begin());
+    }
+
+    ngraph::runtime::TensorViewPtrs args_tv;
+    args_tv.insert(args_tv.begin(), args.begin(), args.end());
+    args_tv.push_back(c_arg);
+
+    runtime::TensorViewPtrs bprops_tv;
+    bprops_tv.insert(bprops_tv.begin(), bprops.begin(), bprops.end());
+
+    auto& c_vec = c_arg->get_vector();
+    for (size_t i = 0; i < c_vec.size(); i++)
+    {
+        c_vec[i] = 1;
+        cf->tensor_call(args_tv, bprops_tv);
+        c_vec[i] = 0;
+        for (size_t j = 0; j < results.size(); j++)
+        {
+            auto& bprop_vec = bprops[j]->get_vector();
+            result_pos[j] = std::copy(bprop_vec.begin(), bprop_vec.end(), result_pos[j]);
+        }
+    }
+
+    return results;
+}
+
+template std::vector<
+    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+    autodiff::backprop_derivative<ngraph::element::Float32>(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<
+            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+template std::vector<
+    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+    autodiff::backprop_derivative<ngraph::element::Float64>(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<
+            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>& args);

src/ngraph/autodiff/backprop_derivative.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+#include "ngraph/types/element_type.hpp"
+
+namespace ngraph
+{
+    class Node;
+    class Function;
+
+    namespace runtime
+    {
+        class Backend;
+        class Manager;
+    }
+
+    namespace autodiff
+    {
+        /// @brief Returns a FunctionSpec for the backprop derivative of its argument.
+        /// @param f is f(X_i...)
+        /// @returns f'(X_i..., c) where f'(x_i, ..., c)_j is backprop for X_j
+        std::shared_ptr<Function> backprop_function(const std::shared_ptr<Function>& f);
+
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> backprop_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+            backprop_derivative<ngraph::element::Float32>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+            backprop_derivative<ngraph::element::Float64>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args);
+    }
+}

src/ngraph/autodiff/backprop_function.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <cassert>
+#include <list>
+#include <memory>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "ngraph/autodiff/adjoints.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/node.hpp"
+#include "ngraph/ops/add.hpp"
+#include "ngraph/ops/broadcast.hpp"
+#include "ngraph/ops/constant.hpp"
+#include "ngraph/ops/convert.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/types/type.hpp"
+
+using namespace ngraph;
+
+std::shared_ptr<Function> autodiff::backprop_function(const std::shared_ptr<Function>& f)
+{
+    auto Y = f->get_result();
+    auto Xs = f->get_parameters();
+    auto C = std::make_shared<op::Parameter>(Y->get_value_type());
+    std::vector<std::shared_ptr<Node>> dYdXs(Xs.size());
+    transform(Xs.begin(), Xs.end(), dYdXs.begin(), [C, Y](const std::shared_ptr<Node>& X) {
+        return Y->backprop_node(X, C);
+    });
+    auto result = std::make_shared<op::Tuple>(dYdXs);
+    std::vector<std::shared_ptr<op::Parameter>> params(Xs);
+    params.push_back(C);
+    return std::make_shared<Function>(result, result->get_value_type(), params);
+}

src/ngraph/autodiff/backprop_function.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+#include <unordered_map>
+
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+
+namespace ngraph
+{
+    class Node;
+    class Function;
+
+    namespace runtime
+    {
+        class Backend;
+        class Manager;
+    }
+
+    namespace autodiff
+    {
+        /// @brief Returns a FunctionSpec for the backprop derivative of its argument.
+        /// @param f is f(X_i...)
+        /// @returns f'(X_i..., c) where f'(x_i, ..., c)_j is backprop for X_j
+        std::shared_ptr<Function> backprop_function(const std::shared_ptr<Function>& f);
+
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> backprop_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+            backprop_derivative<ngraph::element::Float32>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+            backprop_derivative<ngraph::element::Float64>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args);
+    }
+}

src/ngraph/autodiff/numeric_derivative.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+
+#include "ngraph/autodiff/numeric_derivative.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/runtime/call_frame.hpp"
+
+using namespace ngraph;
+
+template <typename ET>
+std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>
+    autodiff::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args,
+        typename ET::type delta)
+{
+    auto y = f->get_result();
+
+    Shape y_shape =
+        std::dynamic_pointer_cast<const TensorViewType>(y->get_value_type())->get_shape();
+
+    // Results for each derivative, shape Y|X_i
+    std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> results;
+    for (size_t i = 0; i < args.size(); i++)
+    {
+        Shape s = y_shape;
+        auto arg_shape = args[i]->get_shape();
+        s.insert(s.end(), arg_shape.begin(), arg_shape.end());
+        results.push_back(backend->make_parameterized_tensor_view<ET>(s));
+    }
+
+    auto external = manager->compile(f);
+    auto cf = backend->make_call_frame(external);
+
+    // ref_y is the function evaluated at the args
+    auto ref_y = backend->make_parameterized_tensor_view<ET>(y_shape);
+
+    ngraph::runtime::TensorViewPtrs args_tv;
+    args_tv.insert(args_tv.begin(), args.begin(), args.end());
+
+    cf->tensor_call(args_tv, runtime::TensorViewPtrs{ref_y});
+    auto& ref_vec = ref_y->get_vector();
+
+    // inc_y will hold f(x+dx) values
+    auto inc_y = backend->make_parameterized_tensor_view<ET>(y_shape);
+    auto& inc_vec = inc_y->get_vector();
+
+    // Assuming vars, y, and results are row-major
+
+    typename ET::type inv_delta = 1 / delta;
+    for (size_t i = 0; i < args.size(); ++i)
+    {
+        auto arg = args[i];
+        auto df_darg = results[i];
+        auto df_darg_it = df_darg->get_vector().begin();
+        auto& vec = arg->get_vector();
+        for (size_t j = 0; j < vec.size(); j++)
+        {
+            auto old_val = vec[j];
+            vec[j] += delta;
+            cf->tensor_call(args_tv, {inc_y});
+            vec[j] = old_val;
+            df_darg_it = std::transform(inc_vec.begin(),
+                                        inc_vec.end(),
+                                        ref_vec.begin(),
+                                        df_darg_it,
+                                        [inv_delta](typename ET::type y1, typename ET::type y0) {
+                                            return inv_delta * (y1 - y0);
+                                        });
+        }
+    }
+    return results;
+}
+
+template std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>
+    autodiff::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>&
+            args,
+        element::Float32::type delta);
+
+template std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>
+    autodiff::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>&
+            args,
+        element::Float64::type delta);

src/ngraph/autodiff/numeric_derivative.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+#include <vector>
+
+#include "ngraph/runtime/backend.hpp"
+#include "ngraph/runtime/manager.hpp"
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+#include "ngraph/runtime/tuple.hpp"
+#include "ngraph/runtime/value.hpp"
+#include "ngraph/types/element_type.hpp"
+
+namespace ngraph
+{
+    namespace autodiff
+    {
+        /// @brief numeric approximation of the derivative
+        /// @param f A function
+        /// @param args Values for the arguments (the independent variables)
+        /// @param delta increment for the variables
+        /// @returns vector of dy/dvar, where each dy/dvar's shape is concat(y.shape(), var.shape())
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> numeric_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args,
+            typename ET::type delta);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>
+            numeric_derivative(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args,
+                element::Float32::type delta);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>
+            numeric_derivative(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args,
+                element::Float64::type delta);
+    }
+}

src/ngraph/function.cpp

@@ -33,12 +33,6 @@ Function::Function(const std::shared_ptr<Node>& result,
     , m_ordered_ops_valid(false)
     , m_instance_id(m_next_instance_id.fetch_add(1))
 {
-    size_t i = 0;
-    for (auto parameter : parameters)
-    {
-        parameter->assign_function(this, i++);
-    }
-
     traverse_nodes(result, [&](shared_ptr<Node> node) { m_ops.push_back(node); });
 }
 

src/ngraph/node.cpp

@@ -20,7 +20,7 @@ using namespace ngraph;
 
 atomic<size_t> Node::m_next_instance_id(0);
 
-Node::Node(const std::vector<shared_ptr<Node>>& arguments, shared_ptr<ValueType> value_type)
+Node::Node(const std::vector<shared_ptr<Node>>& arguments, shared_ptr<const ValueType> value_type)
     : m_arguments(arguments)
     , m_value_type(value_type)
     , m_instance_id(m_next_instance_id.fetch_add(1))
@@ -38,7 +38,7 @@ Node::Node()
 {
 }
 
-Node::Node(std::shared_ptr<ValueType> value_type)
+Node::Node(std::shared_ptr<const ValueType> value_type)
     : Node({}, value_type)
 {
 }
@@ -159,7 +159,7 @@ void Node::set_name(const string& name)
     }
 }
 
-std::shared_ptr<Node> Node::backwards_derivative(const std::shared_ptr<Node>& x,
+std::shared_ptr<Node> Node::backprop_node(const std::shared_ptr<Node>& x,
                                           const std::shared_ptr<Node>& c)
 {
     auto adjoints_it = m_adjoint_map.find(c.get());

src/ngraph/node.hpp

@@ -43,9 +43,9 @@ namespace ngraph
         friend class autodiff::Adjoints;
 
     protected:
-        Node(const Nodes& arguments, std::shared_ptr<ValueType> value_type = nullptr);
+        Node(const Nodes& arguments, std::shared_ptr<const ValueType> value_type = nullptr);
         Node();
-        Node(std::shared_ptr<ValueType> value_type);
+        Node(std::shared_ptr<const ValueType> value_type);
 
         virtual ~Node();
 
@@ -114,7 +114,7 @@ namespace ngraph
         std::unordered_set<descriptor::Tensor*> liveness_new_list;
         std::unordered_set<descriptor::Tensor*> liveness_free_list;
 
-        std::shared_ptr<Node> backwards_derivative(const std::shared_ptr<Node>& x,
+        std::shared_ptr<Node> backprop_node(const std::shared_ptr<Node>& x,
                                             const std::shared_ptr<Node>& c);
 
     protected:

src/ngraph/ops/parameter.cpp

@@ -19,7 +19,7 @@
 using namespace std;
 using namespace ngraph::op;
 
-Parameter::Parameter(const std::shared_ptr<ValueType>& value_type)
+Parameter::Parameter(const std::shared_ptr<const ValueType>& value_type)
     : Node(value_type)
     , m_function(nullptr)
     , m_index(0)
@@ -31,16 +31,6 @@ Parameter::Parameter(const ngraph::element::Type& element_type, const Shape& sha
 {
 }
 
-void Parameter::assign_function(Function* function, size_t index)
-{
-    if (nullptr != m_function)
-    {
-        throw ngraph_error("Re-assigning function to a parameter.");
-    }
-    m_function = function;
-    m_index = index;
-}
-
 void Parameter::propagate_types()
 {
 }

src/ngraph/ops/parameter.hpp

@@ -42,7 +42,7 @@ namespace ngraph
             }
 
         public:
-            Parameter(const std::shared_ptr<ValueType>& value_type = nullptr);
+            Parameter(const std::shared_ptr<const ValueType>& value_type = nullptr);
             Parameter(const ngraph::element::Type& element_type, const Shape& shape);
 
             std::string description() const override { return "Parameter"; }

src/ngraph/runtime/utils.hpp

@@ -17,8 +17,6 @@
 #include <memory>
 #include <vector>
 
-#include "ngraph/runtime/backend.hpp"
-#include "ngraph/runtime/manager.hpp"
 #include "ngraph/runtime/parameterized_tensor_view.hpp"
 #include "ngraph/runtime/tuple.hpp"
 #include "ngraph/runtime/value.hpp"
@@ -39,163 +37,5 @@ namespace ngraph
         /// @brief Framework constructor of a tuple from a sequence of values.
         std::shared_ptr<ngraph::runtime::Tuple>
             make_tuple(const std::vector<std::shared_ptr<ngraph::runtime::Value>>& elements);
-
-        /// @brief Same as numpy.allclose
-        /// @param a First tensor to compare
-        /// @param b Second tensor to compare
-        /// @param rtol Relative tolerance
-        /// @param atol Absolute tolerance
-        /// Returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
-        template <typename ET>
-        bool all_close(const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& a,
-                       const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& b,
-                       typename ET::type rtol = 1e-5f,
-                       typename ET::type atol = 1e-8f);
-
-        extern template bool ngraph::runtime::all_close<ngraph::element::Float32>(
-            const std::shared_ptr<
-                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>& a,
-            const std::shared_ptr<
-                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>& b,
-            ngraph::element::Float32::type rtol,
-            ngraph::element::Float32::type atol);
-
-        extern template bool ngraph::runtime::all_close<ngraph::element::Float64>(
-            const std::shared_ptr<
-                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>& a,
-            const std::shared_ptr<
-                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>& b,
-            ngraph::element::Float64::type rtol,
-            ngraph::element::Float64::type atol);
-
-        /// @brief Same as numpy.allclose
-        /// @param a First tensor to compare
-        /// @param b Second tensor to compare
-        /// @param rtol Relative tolerance
-        /// @param atol Absolute tolerance
-        /// @returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
-        template <typename T>
-        bool all_close(const std::vector<T>& a,
-                       const std::vector<T>& b,
-                       T rtol = 1e-5f,
-                       T atol = 1e-8f);
-
-        extern template bool ngraph::runtime::all_close<float>(const std::vector<float>& a,
-                                                               const std::vector<float>& b,
-                                                               float rtol,
-                                                               float atol);
-
-        extern template bool ngraph::runtime::all_close<double>(const std::vector<double>& a,
-                                                                const std::vector<double>& b,
-                                                                double rtol,
-                                                                double atol);
-
-        /// @brief Contains the information in a Function, but can be used to construct derived functions such as derivatives.
-        class FunctionSpec
-        {
-        public:
-            FunctionSpec(const std::shared_ptr<Node>& result,
-                         const std::shared_ptr<const ValueType>& result_type,
-                         const std::vector<std::shared_ptr<op::Parameter>>& parameters)
-                : m_result(result)
-                , m_result_type(result_type)
-                , m_parameters(parameters)
-            {
-            }
-
-            FunctionSpec(const std::shared_ptr<Node>& result,
-                         const std::vector<std::shared_ptr<op::Parameter>>& parameters)
-                : m_result(result)
-                , m_result_type(result->get_value_type())
-                , m_parameters(parameters)
-            {
-            }
-
-            const std::shared_ptr<const ValueType> get_result_type() const { return m_result_type; }
-            std::shared_ptr<Node> get_result() { return m_result; }
-            const std::vector<std::shared_ptr<op::Parameter>> get_parameters() const
-            {
-                return m_parameters;
-            }
-
-            operator std::shared_ptr<Function>() const;
-
-        protected:
-            std::shared_ptr<Node> m_result;
-            std::shared_ptr<const ValueType> m_result_type;
-            std::vector<std::shared_ptr<op::Parameter>> m_parameters;
-        };
-
-        /// @brief Returns a FunctionSpec for the backprop derivative of its argument.
-        /// @param f is f(X_i...)
-        /// @returns f'(c, X_i...) -> tuple of tensors in same order as in X_i
-        std::shared_ptr<ngraph::runtime::FunctionSpec>
-            derivative(const std::shared_ptr<ngraph::runtime::FunctionSpec>& f);
-
-        /// @brief numeric approximation of the derivative
-        /// @param f A function
-        /// @param args Values for the arguments (the independent variables)
-        /// @param delta increment for the variables
-        /// @returns vector of dy/dvar, where each dy/dvar's shape is concat(y.shape(), var.shape())
-        template <typename ET>
-        std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>
-            numeric_derivative(
-                const std::shared_ptr<runtime::Manager>& manager,
-                const std::shared_ptr<runtime::Backend>& backend,
-                const std::shared_ptr<FunctionSpec>& f,
-                const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>&
-                    args,
-                typename ET::type delta);
-
-        extern template std::vector<
-            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float32>>>
-            ngraph::runtime::numeric_derivative(
-                const std::shared_ptr<runtime::Manager>& manager,
-                const std::shared_ptr<runtime::Backend>& backend,
-                const std::shared_ptr<FunctionSpec>& f,
-                const std::vector<
-                    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float32>>>&
-                    args,
-                element::Float32::type delta);
-
-        extern template std::vector<
-            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>
-            ngraph::runtime::numeric_derivative(
-                const std::shared_ptr<runtime::Manager>& manager,
-                const std::shared_ptr<runtime::Backend>& backend,
-                const std::shared_ptr<FunctionSpec>& f,
-                const std::vector<
-                    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>&
-                    args,
-                element::Float64::type delta);
-
-        template <typename ET>
-        std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>
-            backwards_derivative(
-                const std::shared_ptr<runtime::Manager>& manager,
-                const std::shared_ptr<runtime::Backend>& backend,
-                const std::shared_ptr<FunctionSpec>& f,
-                const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>&
-                    args);
-
-        extern template std::vector<
-            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>>
-            ngraph::runtime::backwards_derivative<ngraph::element::Float32>(
-                const std::shared_ptr<runtime::Manager>& manager,
-                const std::shared_ptr<runtime::Backend>& backend,
-                const std::shared_ptr<FunctionSpec>& f,
-                const std::vector<
-                    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float32>>>&
-                    args);
-
-        extern template std::vector<
-            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>>
-            ngraph::runtime::backwards_derivative<ngraph::element::Float64>(
-                const std::shared_ptr<runtime::Manager>& manager,
-                const std::shared_ptr<runtime::Backend>& backend,
-                const std::shared_ptr<FunctionSpec>& f,
-                const std::vector<
-                    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>&
-                    args);
     }
 }

src/ngraph/test/all_close.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <cmath>
+#include <memory>
+#include <vector>
+
+#include "ngraph/except.hpp"
+#include "ngraph/test/all_close.hpp"
+
+template <typename ET>
+bool ngraph::test::all_close(const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& a,
+                             const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& b,
+                             typename ET::type rtol,
+                             typename ET::type atol)
+{
+    // Check that the layouts are compatible
+    if (*a->get_tensor_view_layout() != *b->get_tensor_view_layout())
+    {
+        throw ngraph_error("Cannot compare tensors with different layouts");
+    }
+
+    if (a->get_shape() != b->get_shape())
+        return false;
+
+    return all_close(a->get_vector(), b->get_vector(), rtol, atol);
+}
+
+template bool ngraph::test::all_close<ngraph::element::Float32>(
+    const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>& a,
+    const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>& b,
+    ngraph::element::Float32::type rtol,
+    ngraph::element::Float32::type atol);
+
+template bool ngraph::test::all_close<ngraph::element::Float64>(
+    const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>& a,
+    const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>& b,
+    ngraph::element::Float64::type rtol,
+    ngraph::element::Float64::type atol);
+
+template <typename T>
+bool ngraph::test::all_close(const std::vector<T>& a, const std::vector<T>& b, T rtol, T atol)
+{
+    assert(a.size() == b.size());
+    for (size_t i = 0; i < a.size(); ++i)
+    {
+        if (std::abs(a[i] - b[i]) > atol + rtol * std::abs(b[i]))
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+template bool ngraph::test::all_close<float>(const std::vector<float>& a,
+                                             const std::vector<float>& b,
+                                             float rtol,
+                                             float atol);
+
+template bool ngraph::test::all_close<double>(const std::vector<double>& a,
+                                              const std::vector<double>& b,
+                                              double rtol,
+                                              double atol);

src/ngraph/test/all_close.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+#include "ngraph/types/element_type.hpp"
+
+namespace ngraph
+{
+    namespace test
+    {
+        /// @brief Same as numpy.allclose
+        /// @param a First tensor to compare
+        /// @param b Second tensor to compare
+        /// @param rtol Relative tolerance
+        /// @param atol Absolute tolerance
+        /// Returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
+        template <typename ET>
+        bool all_close(const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& a,
+                       const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& b,
+                       typename ET::type rtol = 1e-5f,
+                       typename ET::type atol = 1e-8f);
+
+        extern template bool all_close<ngraph::element::Float32>(
+            const std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>& a,
+            const std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>& b,
+            ngraph::element::Float32::type rtol,
+            ngraph::element::Float32::type atol);
+
+        extern template bool all_close<ngraph::element::Float64>(
+            const std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>& a,
+            const std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>& b,
+            ngraph::element::Float64::type rtol,
+            ngraph::element::Float64::type atol);
+
+        /// @brief Same as numpy.allclose
+        /// @param a First tensor to compare
+        /// @param b Second tensor to compare
+        /// @param rtol Relative tolerance
+        /// @param atol Absolute tolerance
+        /// @returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
+        template <typename T>
+        bool all_close(const std::vector<T>& a,
+                       const std::vector<T>& b,
+                       T rtol = 1e-5f,
+                       T atol = 1e-8f);
+
+        extern template bool all_close<float>(const std::vector<float>& a,
+                                              const std::vector<float>& b,
+                                              float rtol,
+                                              float atol);
+
+        extern template bool all_close<double>(const std::vector<double>& a,
+                                               const std::vector<double>& b,
+                                               double rtol,
+                                               double atol);
+    }
+}

test/autodiff.cpp

@@ -18,7 +18,11 @@
 
 #include "gtest/gtest.h"
 
+#include "ngraph/autodiff/backprop_derivative.hpp"
+#include "ngraph/autodiff/backprop_function.hpp"
+#include "ngraph/autodiff/numeric_derivative.hpp"
 #include "ngraph/ngraph.hpp"
+#include "ngraph/test/all_close.hpp"
 
 using namespace std;
 using namespace ngraph;
@@ -29,7 +33,7 @@ TEST(backwards, parameter)
     auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
     auto Y = X0;
     auto C = make_shared<op::Parameter>(element::Float32::element_type(), shape);
-    auto DYDX0 = Y->backwards_derivative(X0, C);
+    auto DYDX0 = Y->backprop_node(X0, C);
     ASSERT_EQ(DYDX0, C);
 }
 
@@ -40,8 +44,8 @@ TEST(backwards, add)
     auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
     auto Y = X0 + X1;
     auto C = make_shared<op::Parameter>(element::Float32::element_type(), shape);
-    auto DYDX0 = Y->backwards_derivative(X0, C);
-    auto DYDX1 = Y->backwards_derivative(X1, C);
+    auto DYDX0 = Y->backprop_node(X0, C);
+    auto DYDX1 = Y->backprop_node(X1, C);
     ASSERT_EQ(DYDX0, C);
     ASSERT_EQ(DYDX1, C);
 }
@@ -52,14 +56,14 @@ TEST(backwards, multiply)
     auto make_graph = [shape]() {
         auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
         auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
-        return make_shared<ngraph::runtime::FunctionSpec>(
-            X0 * X1, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
+        return make_shared<Function>(
+            X0 * X1, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
     };
 
     auto manager = runtime::Manager::get("NGVM");
     auto backend = manager->allocate_backend();
 
-    auto external = manager->compile(*derivative(make_graph()));
+    auto external = manager->compile(ngraph::autodiff::backprop_function(make_graph()));
     auto cf = backend->make_call_frame(external);
 
     auto x0 = backend->make_parameterized_tensor_view<element::Float32>(
@@ -80,7 +84,7 @@ TEST(backwards, multiply)
     {
         c->get_vector().assign(n, 0);
         c->get_vector()[i] = 1;
-        (*cf)({c, x0, x1}, {dx});
+        (*cf)({x0, x1, c}, {dx});
         dx0_correct.assign(n, 0);
         dx1_correct.assign(n, 0);
         dx0_correct[i] = x1->get_vector()[i];
@@ -91,15 +95,15 @@ TEST(backwards, multiply)
 
     auto f_num = make_graph();
     auto results_num =
-        runtime::numeric_derivative<element::Float32>(manager, backend, f_num, {x0, x1}, .001f);
+        autodiff::numeric_derivative<element::Float32>(manager, backend, f_num, {x0, x1}, .001f);
     auto f_sym = make_graph();
     auto results_sym =
-        runtime::backwards_derivative<element::Float32>(manager, backend, f_sym, {x0, x1});
+        autodiff::backprop_derivative<element::Float32>(manager, backend, f_sym, {x0, x1});
     for (size_t i = 0; i < results_num.size(); ++i)
     {
         auto result_num = results_num[i];
         auto result_sym = results_sym[i];
-        bool ac = all_close(result_num, result_sym, .01f, .01f);
+        bool ac = test::all_close(result_num, result_sym, .01f, .01f);
         EXPECT_TRUE(ac);
     }
 }

test/util.cpp

@@ -19,7 +19,7 @@
 #include "gtest/gtest.h"
 
 #include "ngraph/ngraph.hpp"
-#include "ngraph/runtime/utils.hpp"
+#include "ngraph/test/all_close.hpp"
 #include "ngraph/util.hpp"
 
 using namespace std;
@@ -183,13 +183,13 @@ TEST(util, all_close)
     auto b = backend->make_parameterized_tensor_view<element::Float32>(
         runtime::NDArray<float, 2>({{1, 2, 3}, {3, 4, 5}}));
 
-    EXPECT_TRUE(ngraph::runtime::all_close(a, b));
+    EXPECT_TRUE(ngraph::test::all_close(a, b));
 
     auto c = backend->make_parameterized_tensor_view<element::Float32>(
         runtime::NDArray<float, 2>({{1.1f, 2, 3}, {3, 4, 5}}));
-    EXPECT_FALSE(ngraph::runtime::all_close(c, a, 0, .05f));
-    EXPECT_TRUE(ngraph::runtime::all_close(c, a, 0, .11f));
+    EXPECT_FALSE(ngraph::test::all_close(c, a, 0, .05f));
+    EXPECT_TRUE(ngraph::test::all_close(c, a, 0, .11f));
 
-    EXPECT_FALSE(ngraph::runtime::all_close(c, a, .05f, 0));
-    EXPECT_TRUE(ngraph::runtime::all_close(c, a, .11f, 0));
+    EXPECT_FALSE(ngraph::test::all_close(c, a, .05f, 0));
+    EXPECT_TRUE(ngraph::test::all_close(c, a, .11f, 0));
 }