Merge pull request #192 from NervanaSystems/cyphers/autodiff

Basic autodiff for +, -, *, /, exp, log, plus numeric testing support

cmake/clang_4_0_flags.cmake

@@ -13,6 +13,7 @@
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=return-type")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=inconsistent-missing-override")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic-errors")
 
  # whitelist errors here
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything")

src/ngraph/CMakeLists.txt

@@ -12,6 +12,10 @@
 # limitations under the License.
 
 set (SRC
+    autodiff/adjoints.cpp
+    autodiff/backprop_derivative.cpp
+    autodiff/backprop_function.cpp
+    autodiff/numeric_derivative.cpp
     descriptor/input.cpp
     descriptor/layout/dense_tensor_view_layout.cpp
     descriptor/layout/tensor_view_layout.cpp
@@ -23,6 +27,7 @@ set (SRC
     function.cpp
     log.cpp
     node.cpp
+    ops/add.cpp
     ops/binary_elementwise_arithmetic.cpp
     ops/binary_elementwise_builtin.cpp
     ops/binary_elementwise_comparison.cpp
@@ -30,15 +35,21 @@ set (SRC
     ops/concatenate.cpp
     ops/constant.cpp
     ops/convert.cpp
+    ops/divide.cpp
     ops/dot.cpp
+    ops/exp.cpp
     ops/function_call.cpp
     ops/get_tuple_element.cpp
+    ops/log.cpp
+    ops/multiply.cpp
+    ops/negative.cpp
     ops/op.cpp
     ops/parameter.cpp
     ops/reduce.cpp
     ops/reshape.cpp
     ops/select.cpp
     ops/slice.cpp
+    ops/subtract.cpp
     ops/sum.cpp
     ops/tuple.cpp
     ops/unary_elementwise_arithmetic.cpp
@@ -64,6 +75,7 @@ set (SRC
     runtime/tensor_view.cpp
     runtime/tuple.cpp
     runtime/utils.cpp
+    test/all_close.cpp
     shape.cpp
     types/element_type.cpp
     types/type.cpp

src/ngraph/autodiff/adjoints.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <cassert>
+#include <list>
+#include <memory>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "ngraph/autodiff/adjoints.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/node.hpp"
+#include "ngraph/ops/add.hpp"
+#include "ngraph/ops/broadcast.hpp"
+#include "ngraph/ops/constant.hpp"
+#include "ngraph/ops/convert.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/types/type.hpp"
+
+using namespace ngraph;
+
+/// @brief Make a zero matching a value type.
+std::shared_ptr<Node> make_zero(const std::shared_ptr<const ValueType>& value_type);
+
+std::shared_ptr<Node> make_zero(const std::shared_ptr<const TensorViewType>& tensor_view_type)
+{
+    std::shared_ptr<Node> zero =
+        std::make_shared<op::Constant>(tensor_view_type->get_element_type(), Shape{}, "0");
+    const Shape& shape = tensor_view_type->get_shape();
+    if (shape.size() > 0)
+    {
+        AxisSet axes;
+        for (size_t i = 0; i < shape.size(); i++)
+        {
+            axes.insert(i);
+        }
+        zero = std::make_shared<op::Broadcast>(zero, shape, axes);
+    }
+    return zero;
+}
+
+std::shared_ptr<Node> make_zero(const std::shared_ptr<const TupleType>& tuple_type)
+{
+    std::vector<std::shared_ptr<Node>> elements;
+    for (auto& value_type : tuple_type->get_element_types())
+    {
+        elements.push_back(make_zero(value_type));
+    }
+    return std::make_shared<op::Tuple>(elements);
+}
+
+std::shared_ptr<Node> make_zero(const std::shared_ptr<const ValueType>& value_type)
+{
+    std::shared_ptr<const TensorViewType> tensor_view_type =
+        std::dynamic_pointer_cast<const TensorViewType>(value_type);
+    if (nullptr != tensor_view_type)
+    {
+        return (make_zero(tensor_view_type));
+    }
+    std::shared_ptr<const TupleType> tuple_type =
+        std::dynamic_pointer_cast<const TupleType>(value_type);
+    if (nullptr != tuple_type)
+    {
+        return make_zero(tuple_type);
+    }
+    // Should be impossible
+    throw ngraph_error("Unknown value type");
+}
+
+autodiff::Adjoints::Adjoints(const std::shared_ptr<Node>& y, const std::shared_ptr<Node>& c)
+{
+    // Pass 1 determines which nodes contribute to y as well as setting up a reverse
+    // topological sort.
+
+    // Number of nodes that use the node's value
+    std::unordered_map<std::shared_ptr<Node>, size_t> parent_counts;
+
+    // Nodes that have been processed
+    std::unordered_set<std::shared_ptr<Node>> visited_nodes;
+
+    // Nodes we should check
+    std::list<std::shared_ptr<Node>> nodes_to_check;
+    nodes_to_check.push_front(y);
+    while (nodes_to_check.size() > 0)
+    {
+        auto node = nodes_to_check.front();
+        nodes_to_check.pop_front();
+        if (visited_nodes.count(node) != 0)
+        {
+            continue;
+        }
+        for (auto arg : node->get_arguments())
+        {
+            auto count_it = parent_counts.find(arg);
+            if (count_it == parent_counts.end())
+            {
+                parent_counts[arg] = 1;
+                nodes_to_check.push_front(arg);
+            }
+            else
+            {
+                parent_counts[arg]++;
+            }
+        }
+        visited_nodes.insert(node);
+    }
+
+    // Second pass visits the nodes so that all users of a node's value are visited
+    // before a node is visited.
+    m_adjoint_map[y.get()] = c;
+    nodes_to_check.push_front(y);
+    while (nodes_to_check.size() > 0)
+    {
+        auto node = nodes_to_check.front();
+        nodes_to_check.pop_front();
+        // Look for nodes that will be available when this node is done
+        for (auto arg : node->get_arguments())
+        {
+            auto count_it = parent_counts.find(arg);
+            count_it->second--;
+            if (0 == count_it->second)
+            {
+                nodes_to_check.push_front(arg);
+            }
+        }
+        node->generate_adjoints(*this, m_adjoint_map.at(node.get()));
+    }
+}
+
+std::shared_ptr<Node> autodiff::Adjoints::get(const std::shared_ptr<Node>& x)
+{
+    auto adjoint_it = m_adjoint_map.find(x.get());
+    if (m_adjoint_map.end() == adjoint_it)
+    {
+        auto result = make_zero(x->get_value_type());
+        adjoint_it = m_adjoint_map.insert({x.get(), result}).first;
+    }
+    return adjoint_it->second;
+}
+
+void autodiff::Adjoints::add_delta(const std::shared_ptr<Node>& x,
+                                   const std::shared_ptr<Node>& delta)
+{
+    assert(*x->get_value_type() == *delta->get_value_type());
+    auto adjoint_it = m_adjoint_map.find(x.get());
+    if (m_adjoint_map.end() == adjoint_it)
+    {
+        m_adjoint_map.insert({x.get(), delta});
+    }
+    else
+    {
+        m_adjoint_map.insert({x.get(), std::make_shared<op::Add>(adjoint_it->second, delta)});
+    }
+}

src/ngraph/autodiff/adjoints.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+#include <unordered_map>
+
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+
+namespace ngraph
+{
+    class Node;
+    class Function;
+
+    namespace runtime
+    {
+        class Backend;
+        class Manager;
+    }
+
+    namespace autodiff
+    {
+        class Adjoints
+        {
+        public:
+            /// @brief (dy/dx)(c) for all x used to compute y
+            ///
+            /// @param y The dependent value
+            /// @param c An expression for where to evaluate the derivatives
+            Adjoints(const std::shared_ptr<Node>& y, const std::shared_ptr<Node>& c);
+
+            Adjoints(const Adjoints& adjoints) = default;
+            Adjoints& operator=(const Adjoints& adjoints) = default;
+            Adjoints() = default;
+
+            /// @brief (dy/dx)(c)
+            ///
+            /// @param x The node whose adjoint is desired.
+            std::shared_ptr<Node> get(const std::shared_ptr<Node>& x);
+
+            /// @brief Add a backprop contribution to x's adjoint
+            ///
+            /// @param x The adjoint node
+            /// @param delta A backprop contribution
+            void add_delta(const std::shared_ptr<Node>& x, const std::shared_ptr<Node>& delta);
+
+        protected:
+            std::unordered_map<Node*, std::shared_ptr<Node>> m_adjoint_map;
+        };
+
+        /// @brief Returns a FunctionSpec for the backprop derivative of its argument.
+        /// @param f is f(X_i...)
+        /// @returns f'(X_i..., c) where f'(x_i, ..., c)_j is backprop for X_j
+        std::shared_ptr<Function> backprop_function(const std::shared_ptr<Function>& f);
+
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> backprop_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+            backprop_derivative<ngraph::element::Float32>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+            backprop_derivative<ngraph::element::Float64>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args);
+    }
+}

src/ngraph/autodiff/backprop_derivative.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <memory>
+#include <vector>
+
+#include "ngraph/autodiff/backprop_derivative.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/runtime/backend.hpp"
+#include "ngraph/runtime/call_frame.hpp"
+#include "ngraph/runtime/manager.hpp"
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+#include "ngraph/types/type.hpp"
+
+using namespace ngraph;
+
+template <typename ET>
+std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>
+    autodiff::backprop_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args)
+{
+    auto y = f->get_result();
+    Shape y_shape =
+        std::dynamic_pointer_cast<const TensorViewType>(y->get_value_type())->get_shape();
+
+    auto c_param = std::make_shared<op::Parameter>(ET::element_type(), y_shape);
+    auto c_arg = backend->make_parameterized_tensor_view<ET>(y_shape);
+    auto params = f->get_parameters();
+
+    std::vector<std::shared_ptr<Node>> deriv_nodes;
+    std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> bprops;
+    std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> results;
+    for (auto param : params)
+    {
+        Shape s = y_shape;
+        auto param_shape =
+            std::dynamic_pointer_cast<const TensorViewType>(param->get_value_type())->get_shape();
+        s.insert(s.end(), param_shape.begin(), param_shape.end());
+        results.push_back(backend->make_parameterized_tensor_view<ET>(s));
+        bprops.push_back(backend->make_parameterized_tensor_view<ET>(param_shape));
+        deriv_nodes.push_back(y->backprop_node(param, c_param));
+    }
+
+    std::vector<std::shared_ptr<op::Parameter>> df_params = params;
+    df_params.push_back(c_param);
+    auto df_result = std::make_shared<op::Tuple>(deriv_nodes);
+    auto df = std::make_shared<Function>(df_result, df_result->get_value_type(), df_params);
+
+    auto external = manager->compile(df);
+    auto cf = backend->make_call_frame(external);
+
+    // We compute the derivatives chunk by chunk
+    std::vector<typename std::vector<typename ET::type>::iterator> result_pos;
+    for (auto result : results)
+    {
+        result_pos.push_back(result->get_vector().begin());
+    }
+
+    ngraph::runtime::TensorViewPtrs args_tv;
+    args_tv.insert(args_tv.begin(), args.begin(), args.end());
+    args_tv.push_back(c_arg);
+
+    runtime::TensorViewPtrs bprops_tv;
+    bprops_tv.insert(bprops_tv.begin(), bprops.begin(), bprops.end());
+
+    auto& c_vec = c_arg->get_vector();
+    for (size_t i = 0; i < c_vec.size(); i++)
+    {
+        c_vec[i] = 1;
+        cf->tensor_call(args_tv, bprops_tv);
+        c_vec[i] = 0;
+        for (size_t j = 0; j < results.size(); j++)
+        {
+            auto& bprop_vec = bprops[j]->get_vector();
+            result_pos[j] = std::copy(bprop_vec.begin(), bprop_vec.end(), result_pos[j]);
+        }
+    }
+
+    return results;
+}
+
+template std::vector<
+    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+    autodiff::backprop_derivative<ngraph::element::Float32>(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<
+            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+template std::vector<
+    std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+    autodiff::backprop_derivative<ngraph::element::Float64>(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<
+            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>& args);

src/ngraph/autodiff/backprop_derivative.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+#include "ngraph/types/element_type.hpp"
+
+namespace ngraph
+{
+    class Node;
+    class Function;
+
+    namespace runtime
+    {
+        class Backend;
+        class Manager;
+    }
+
+    namespace autodiff
+    {
+        /// @brief Returns a FunctionSpec for the backprop derivative of its argument.
+        /// @param f is f(X_i...)
+        /// @returns f'(X_i..., c) where f'(x_i, ..., c)_j is backprop for X_j
+        std::shared_ptr<Function> backprop_function(const std::shared_ptr<Function>& f);
+
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> backprop_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+            backprop_derivative<ngraph::element::Float32>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+            backprop_derivative<ngraph::element::Float64>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args);
+    }
+}

src/ngraph/autodiff/backprop_function.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <cassert>
+#include <list>
+#include <memory>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "ngraph/autodiff/adjoints.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/node.hpp"
+#include "ngraph/ops/add.hpp"
+#include "ngraph/ops/broadcast.hpp"
+#include "ngraph/ops/constant.hpp"
+#include "ngraph/ops/convert.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/types/type.hpp"
+
+using namespace ngraph;
+
+std::shared_ptr<Function> autodiff::backprop_function(const std::shared_ptr<Function>& f)
+{
+    auto Y = f->get_result();
+    auto Xs = f->get_parameters();
+    auto C = std::make_shared<op::Parameter>(Y->get_value_type());
+    std::vector<std::shared_ptr<Node>> dYdXs(Xs.size());
+    transform(Xs.begin(), Xs.end(), dYdXs.begin(), [C, Y](const std::shared_ptr<Node>& X) {
+        return Y->backprop_node(X, C);
+    });
+    auto result = std::make_shared<op::Tuple>(dYdXs);
+    std::vector<std::shared_ptr<op::Parameter>> params(Xs);
+    params.push_back(C);
+    return std::make_shared<Function>(result, result->get_value_type(), params);
+}

src/ngraph/autodiff/backprop_function.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+#include <unordered_map>
+
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+
+namespace ngraph
+{
+    class Node;
+    class Function;
+
+    namespace runtime
+    {
+        class Backend;
+        class Manager;
+    }
+
+    namespace autodiff
+    {
+        /// @brief Returns a FunctionSpec for the backprop derivative of its argument.
+        /// @param f is f(X_i...)
+        /// @returns f'(X_i..., c) where f'(x_i, ..., c)_j is backprop for X_j
+        std::shared_ptr<Function> backprop_function(const std::shared_ptr<Function>& f);
+
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> backprop_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float32>>>
+            backprop_derivative<ngraph::element::Float32>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<ngraph::element::Float64>>>
+            backprop_derivative<ngraph::element::Float64>(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args);
+    }
+}

src/ngraph/autodiff/numeric_derivative.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+
+#include "ngraph/autodiff/numeric_derivative.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/runtime/call_frame.hpp"
+
+using namespace ngraph;
+
+template <typename ET>
+std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>
+    autodiff::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args,
+        typename ET::type delta)
+{
+    auto y = f->get_result();
+
+    Shape y_shape =
+        std::dynamic_pointer_cast<const TensorViewType>(y->get_value_type())->get_shape();
+
+    auto params = f->get_parameters();
+
+    // Results for each derivative, shape Y|X_i
+    std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> results;
+    for (auto param : params)
+    {
+        Shape s = y_shape;
+        auto param_shape =
+            std::dynamic_pointer_cast<const TensorViewType>(param->get_value_type())->get_shape();
+        s.insert(s.end(), param_shape.begin(), param_shape.end());
+        results.push_back(backend->make_parameterized_tensor_view<ET>(s));
+    }
+
+    auto external = manager->compile(f);
+    auto cf = backend->make_call_frame(external);
+
+    // ref_y is the function evaluated at the args
+    auto ref_y = backend->make_parameterized_tensor_view<ET>(y_shape);
+
+    ngraph::runtime::TensorViewPtrs args_tv;
+    args_tv.insert(args_tv.begin(), args.begin(), args.end());
+
+    cf->tensor_call(args_tv, runtime::TensorViewPtrs{ref_y});
+    auto& ref_vec = ref_y->get_vector();
+
+    // inc_y will hold f(x+dx) values
+    auto inc_y = backend->make_parameterized_tensor_view<ET>(y_shape);
+    auto& inc_vec = inc_y->get_vector();
+
+    // Assuming vars, y, and results are row-major
+
+    typename ET::type inv_delta = 1 / delta;
+    for (size_t i = 0; i < args.size(); ++i)
+    {
+        auto arg = args[i];
+        auto df_darg = results[i];
+        auto df_darg_it = df_darg->get_vector().begin();
+        auto& vec = arg->get_vector();
+        for (size_t j = 0; j < vec.size(); j++)
+        {
+            auto old_val = vec[j];
+            vec[j] += delta;
+            cf->tensor_call(args_tv, {inc_y});
+            vec[j] = old_val;
+            df_darg_it = std::transform(inc_vec.begin(),
+                                        inc_vec.end(),
+                                        ref_vec.begin(),
+                                        df_darg_it,
+                                        [inv_delta](typename ET::type y1, typename ET::type y0) {
+                                            return inv_delta * (y1 - y0);
+                                        });
+        }
+    }
+    return results;
+}
+
+template std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>
+    autodiff::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>&
+            args,
+        element::Float32::type delta);
+
+template std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<element::Float64>>>
+    autodiff::numeric_derivative(
+        const std::shared_ptr<runtime::Manager>& manager,
+        const std::shared_ptr<runtime::Backend>& backend,
+        const std::shared_ptr<Function>& f,
+        const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>&
+            args,
+        element::Float64::type delta);

src/ngraph/autodiff/numeric_derivative.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+#include <vector>
+
+#include "ngraph/runtime/backend.hpp"
+#include "ngraph/runtime/manager.hpp"
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+#include "ngraph/runtime/tuple.hpp"
+#include "ngraph/runtime/value.hpp"
+#include "ngraph/types/element_type.hpp"
+
+namespace ngraph
+{
+    namespace autodiff
+    {
+        /// @brief numeric approximation of the derivative
+        /// @param f A function
+        /// @param args Values for the arguments (the independent variables)
+        /// @param delta increment for the variables
+        /// @returns vector of dy/dvar, where each dy/dvar's shape is concat(y.shape(), var.shape())
+        template <typename ET>
+        std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>> numeric_derivative(
+            const std::shared_ptr<runtime::Manager>& manager,
+            const std::shared_ptr<runtime::Backend>& backend,
+            const std::shared_ptr<Function>& f,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args,
+            typename ET::type delta);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>
+            numeric_derivative(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>& args,
+                element::Float32::type delta);
+
+        extern template std::vector<
+            std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>
+            numeric_derivative(
+                const std::shared_ptr<runtime::Manager>& manager,
+                const std::shared_ptr<runtime::Backend>& backend,
+                const std::shared_ptr<Function>& f,
+                const std::vector<
+                    std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>& args,
+                element::Float64::type delta);
+    }
+}

src/ngraph/descriptor/layout/dense_tensor_view_layout.cpp

@@ -44,3 +44,21 @@ size_t DenseTensorViewLayout::get_index_offset(const std::vector<size_t>& indice
     }
     return result;
 }
+
+bool DenseTensorViewLayout::operator==(const TensorViewLayout& other) const
+{
+    const DenseTensorViewLayout* p_other = dynamic_cast<const DenseTensorViewLayout*>(&other);
+    if (nullptr == p_other)
+        return false;
+
+    if (get_element_type() != p_other->get_element_type())
+        return false;
+
+    if (m_strides != p_other->m_strides)
+        return false;
+
+    if (m_offset != p_other->m_offset)
+        return false;
+
+    return true;
+}

src/ngraph/descriptor/layout/dense_tensor_view_layout.hpp

@@ -41,9 +41,11 @@ namespace ngraph
                 virtual size_t get_index_offset(const std::vector<size_t>& indices) override;
 
                 const Strides& get_strides() const { return m_strides; }
+                virtual bool operator==(const TensorViewLayout& other) const override;
+
             protected:
                 Strides m_strides;
-                size_t m_offset;
+                size_t m_offset{0};
                 size_t m_size;
             };
         }

src/ngraph/descriptor/layout/tensor_view_layout.hpp

@@ -59,6 +59,9 @@ namespace ngraph
                 /// Where this view is located in the buffer.
                 const BufferPos& get_buffer_pos() const { return m_buffer_pos; }
                 BufferPos& get_buffer_pos() { return m_buffer_pos; }
+                /// @brief Return true if this and other have the same element interpretation
+                virtual bool operator==(const TensorViewLayout& other) const = 0;
+                bool operator!=(const TensorViewLayout& other) const { return !(*this == other); }
             protected:
                 std::shared_ptr<const TensorViewType> m_tensor_view_type;
                 BufferPos m_buffer_pos;

src/ngraph/function.cpp

@@ -23,7 +23,7 @@ using namespace ngraph;
 atomic<size_t> Function::m_next_instance_id(0);
 
 Function::Function(const std::shared_ptr<Node>& result,
-                   const std::shared_ptr<ValueType>& result_type,
+                   const std::shared_ptr<const ValueType>& result_type,
                    const std::vector<std::shared_ptr<op::Parameter>>& parameters,
                    const std::string& name)
     : m_result(result)
@@ -33,12 +33,6 @@ Function::Function(const std::shared_ptr<Node>& result,
     , m_ordered_ops_valid(false)
     , m_instance_id(m_next_instance_id.fetch_add(1))
 {
-    size_t i = 0;
-    for (auto parameter : parameters)
-    {
-        parameter->assign_function(this, i++);
-    }
-
     traverse_nodes(this, [&](shared_ptr<Node> node) { m_ops.push_back(node); });
 }
 

src/ngraph/function.hpp

@@ -35,16 +35,16 @@ namespace ngraph
     {
     public:
         Function(const std::shared_ptr<Node>& result,
-                 const std::shared_ptr<ValueType>& result_type,
+                 const std::shared_ptr<const ValueType>& result_type,
                  const std::vector<std::shared_ptr<op::Parameter>>& parameters,
                  const std::string& name = "");
 
         std::shared_ptr<Node> get_result() { return m_result; }
-        const std::vector<std::shared_ptr<op::Parameter>> get_parameters() const
+        const std::vector<std::shared_ptr<op::Parameter>>& get_parameters() const
         {
             return m_parameters;
         }
-        const std::shared_ptr<ValueType> get_result_type() const { return m_result_type; }
+        std::shared_ptr<const ValueType> get_result_type() const { return m_result_type; }
         std::string get_name() const;
         void set_name(const std::string& name);
         std::list<std::shared_ptr<Node>>& get_ops();
@@ -60,7 +60,7 @@ namespace ngraph
         std::shared_ptr<Node> m_result;
         std::vector<std::shared_ptr<ngraph::op::Parameter>> m_parameters;
         std::string m_name;
-        std::shared_ptr<ValueType> m_result_type;
+        std::shared_ptr<const ValueType> m_result_type;
         bool m_ordered_ops_valid;
         std::list<std::shared_ptr<Node>> m_ordered_ops;
         std::list<std::shared_ptr<Node>> m_ops;

src/ngraph/node.cpp

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // ----------------------------------------------------------------------------
 
+#include "ngraph/autodiff/adjoints.hpp"
 #include "ngraph/ngraph.hpp"
 
 using namespace std;
@@ -19,7 +20,7 @@ using namespace ngraph;
 
 atomic<size_t> Node::m_next_instance_id(0);
 
-Node::Node(const std::vector<shared_ptr<Node>>& arguments, shared_ptr<ValueType> value_type)
+Node::Node(const std::vector<shared_ptr<Node>>& arguments, shared_ptr<const ValueType> value_type)
     : m_arguments(arguments)
     , m_value_type(value_type)
     , m_instance_id(m_next_instance_id.fetch_add(1))
@@ -32,6 +33,16 @@ Node::Node(const std::vector<shared_ptr<Node>>& arguments, shared_ptr<ValueType>
     }
 }
 
+Node::Node()
+    : Node({}, nullptr)
+{
+}
+
+Node::Node(std::shared_ptr<const ValueType> value_type)
+    : Node({}, value_type)
+{
+}
+
 Node::~Node()
 {
 }
@@ -51,6 +62,24 @@ void Node::set_value_type_checked(const shared_ptr<const ValueType>& value_type)
     }
 }
 
+std::shared_ptr<const ValueType> Node::get_value_type()
+{
+    if (nullptr == m_value_type)
+    {
+        propagate_types();
+    }
+    return m_value_type;
+}
+
+const std::shared_ptr<const ValueType> Node::get_value_type() const
+{
+    if (nullptr == m_value_type)
+    {
+        const_cast<Node*>(this)->propagate_types();
+    }
+    return m_value_type;
+}
+
 void Node::assign_tensors()
 {
     vector<std::shared_ptr<const TensorViewType>> tensor_view_types;
@@ -130,6 +159,18 @@ void Node::set_name(const string& name)
     }
 }
 
+std::shared_ptr<Node> Node::backprop_node(const std::shared_ptr<Node>& x,
+                                          const std::shared_ptr<Node>& c)
+{
+    auto adjoints_it = m_adjoint_map.find(c.get());
+    if (adjoints_it == m_adjoint_map.end())
+    {
+        adjoints_it =
+            m_adjoint_map.insert({c.get(), autodiff::Adjoints(shared_from_this(), c)}).first;
+    }
+    return adjoints_it->second.get(x);
+}
+
 namespace ngraph
 {
     ostream& operator<<(ostream& out, const Node& node)

src/ngraph/node.hpp

@@ -15,13 +15,17 @@
 #pragma once
 
 #include <atomic>
+#include <memory>
 #include <set>
 #include <string>
+#include <typeindex>
+#include <unordered_map>
 #include <unordered_set>
 #include <vector>
 
 #include <iostream>
 
+#include "ngraph/autodiff/adjoints.hpp"
 #include "ngraph/common.hpp"
 #include "ngraph/descriptor/input.hpp"
 #include "ngraph/descriptor/output.hpp"
@@ -35,20 +39,21 @@ namespace ngraph
     /// view or a (possibly empty) tuple of values.
     class Node : public std::enable_shared_from_this<Node>
     {
+        // So Adjoints can call generate_adjoints
+        friend class autodiff::Adjoints;
+
     protected:
-        Node(const Nodes& arguments, std::shared_ptr<ValueType> value_type = nullptr);
-        Node()
-            : Node({}, nullptr)
-        {
-        }
+        Node(const Nodes& arguments, std::shared_ptr<const ValueType> value_type = nullptr);
+        Node();
+        Node(std::shared_ptr<const ValueType> value_type);
+
+        virtual ~Node();
 
-        Node(std::shared_ptr<ValueType> value_type)
-            : Node({}, value_type)
+        virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                       const std::shared_ptr<Node>& delta)
         {
         }
 
-        virtual ~Node();
-
     public:
         /// The class name, must not contain spaces
         virtual std::string description() const = 0;
@@ -73,11 +78,11 @@ namespace ngraph
         bool is_same_op_type(const std::shared_ptr<Node>& node) const
         {
             Node* n = node.get();
-            return typeid(*this) == typeid(*n);
+            return std::type_index(typeid(*this)) == std::type_index(typeid(*n));
         }
 
-        std::shared_ptr<const ValueType> get_value_type() { return m_value_type; }
-        const std::shared_ptr<const ValueType> get_value_type() const { return m_value_type; }
+        std::shared_ptr<const ValueType> get_value_type();
+        const std::shared_ptr<const ValueType> get_value_type() const;
         void set_value_type(const element::Type& element_type, const Shape& shape)
         {
             m_value_type = std::make_shared<TensorViewType>(element_type, shape);
@@ -109,6 +114,9 @@ namespace ngraph
         std::unordered_set<descriptor::Tensor*> liveness_new_list;
         std::unordered_set<descriptor::Tensor*> liveness_free_list;
 
+        std::shared_ptr<Node> backprop_node(const std::shared_ptr<Node>& x,
+                                            const std::shared_ptr<Node>& c);
+
     protected:
         Nodes m_arguments;
         std::shared_ptr<const ValueType> m_value_type;
@@ -119,5 +127,6 @@ namespace ngraph
         std::deque<descriptor::Input> m_inputs;
         std::deque<descriptor::Output> m_outputs;
         bool m_is_output;
+        std::unordered_map<Node*, autodiff::Adjoints> m_adjoint_map;
     };
 }

src/ngraph/ops/add.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/add.hpp"
+
+void ngraph::op::Add::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+    auto y = m_arguments[1];
+
+    adjoints.add_delta(x, delta);
+    adjoints.add_delta(y, delta);
+}

src/ngraph/ops/add.hpp

@@ -52,6 +52,9 @@ namespace ngraph
             {
             }
             virtual std::string description() const override { return "Add"; }
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
     }
 

src/ngraph/ops/divide.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/divide.hpp"
+#include "ngraph/ops/multiply.hpp"
+#include "ngraph/ops/negative.hpp"
+
+void ngraph::op::Divide::generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+    auto y = m_arguments[1];
+
+    adjoints.add_delta(x, delta * shared_from_this() / x);
+    adjoints.add_delta(y, -delta * shared_from_this() / y);
+}

src/ngraph/ops/divide.hpp

@@ -52,6 +52,9 @@ namespace ngraph
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Divide"; }
         };
     }

src/ngraph/ops/exp.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/exp.hpp"
+#include "ngraph/ops/multiply.hpp"
+
+void ngraph::op::Exp::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, delta * shared_from_this());
+}

src/ngraph/ops/exp.hpp

@@ -50,6 +50,9 @@ namespace ngraph
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Exp"; }
         };
     }

src/ngraph/ops/log.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/log.hpp"
+#include "ngraph/ops/divide.hpp"
+
+void ngraph::op::Log::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, delta / x);
+}

src/ngraph/ops/log.hpp

@@ -50,6 +50,9 @@ namespace ngraph
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Log"; }
         };
     }

src/ngraph/ops/multiply.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/multiply.hpp"
+
+void ngraph::op::Multiply::generate_adjoints(autodiff::Adjoints& adjoints,
+                                             const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+    auto y = m_arguments[1];
+
+    adjoints.add_delta(x, delta * y);
+    adjoints.add_delta(y, x * delta);
+}

src/ngraph/ops/multiply.hpp

@@ -53,8 +53,11 @@ namespace ngraph
             }
 
             virtual std::string description() const override { return "Multiply"; }
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+        };
     };
-    }
 
     inline std::shared_ptr<ngraph::Node> operator*(const std::shared_ptr<ngraph::Node> arg0,
                                                    const std::shared_ptr<ngraph::Node> arg1)

src/ngraph/ops/negative.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/negative.hpp"
+
+void ngraph::op::Negative::generate_adjoints(autodiff::Adjoints& adjoints,
+                                             const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, -delta);
+}

src/ngraph/ops/negative.hpp

@@ -50,6 +50,9 @@ namespace ngraph
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Negative"; }
         };
     }

src/ngraph/ops/parameter.cpp

@@ -19,10 +19,8 @@
 using namespace std;
 using namespace ngraph::op;
 
-Parameter::Parameter(const std::shared_ptr<ValueType>& value_type)
+Parameter::Parameter(const std::shared_ptr<const ValueType>& value_type)
     : Node(value_type)
-    , m_function(nullptr)
-    , m_index(0)
 {
 }
 
@@ -31,16 +29,10 @@ Parameter::Parameter(const ngraph::element::Type& element_type, const Shape& sha
 {
 }
 
-void Parameter::assign_function(Function* function, size_t index)
+void Parameter::propagate_types()
 {
-    if (nullptr != m_function)
-    {
-        throw ngraph_error("Re-assigning function to a parameter.");
-    }
-    m_function = function;
-    m_index = index;
 }
 
-void Parameter::propagate_types()
+void Parameter::generate_adjoints(autodiff::Adjoints& adjoints, const std::shared_ptr<Node>& delta)
 {
 }

src/ngraph/ops/parameter.hpp

@@ -47,18 +47,15 @@ namespace ngraph
         /// | NGVM    | Fully implemented. |
         class Parameter : public Node
         {
-            friend class ngraph::Function;
-
         protected:
-            // Called by the Function constructor to associate this parameter with the function.
-            // It is an error to try to associate a parameter with more than one function.
-            void assign_function(Function* function, size_t index);
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
 
         public:
             /// \brief Constructions a parameter node.
             ///
             /// \param value_type The type of the parameter.
-            Parameter(const std::shared_ptr<ValueType>& value_type = nullptr);
+            Parameter(const std::shared_ptr<const ValueType>& value_type = nullptr);
             /// \brief Constructions a tensor view-typed parameter node.
             ///
             /// \param element_type The element type of the parameter.
@@ -67,10 +64,6 @@ namespace ngraph
 
             std::string description() const override { return "Parameter"; }
             virtual void propagate_types() override;
-
-        protected:
-            Function* m_function;
-            size_t m_index;
         };
     }
 }

src/ngraph/ops/subtract.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/subtract.hpp"
+#include "ngraph/ops/negative.hpp"
+
+void ngraph::op::Subtract::generate_adjoints(autodiff::Adjoints& adjoints,
+                                             const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+    auto y = m_arguments[1];
+
+    adjoints.add_delta(x, delta);
+    adjoints.add_delta(y, -delta);
+}

src/ngraph/ops/subtract.hpp

@@ -52,6 +52,9 @@ namespace ngraph
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Subtract"; }
         };
     }

src/ngraph/runtime/backend.hpp

@@ -17,6 +17,7 @@
 #include <memory>
 
 #include "ngraph/common.hpp"
+#include "ngraph/runtime/ndarray.hpp"
 
 namespace ngraph
 {
@@ -62,6 +63,17 @@ namespace ngraph
                     make_primary_tensor_view(ET::element_type(), shape));
             }
 
+            template <typename ET>
+            std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>
+                make_parameterized_tensor_view(const NDArrayBase<typename ET::type>& ndarray)
+            {
+                auto result =
+                    std::dynamic_pointer_cast<ngraph::runtime::ParameterizedTensorView<ET>>(
+                        make_primary_tensor_view(ET::element_type(), ndarray.get_shape()));
+                *result = ndarray;
+                return result;
+            }
+
             /// @brief Construct a tuple handle from a sequence of values.
             virtual std::shared_ptr<ngraph::runtime::Tuple>
                 make_tuple(const std::vector<std::shared_ptr<ngraph::runtime::Value>>& elements);

src/ngraph/runtime/ndarray.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+// Based on the Matrix class in
+// The C++ Programming Language
+// Fourth edition
+// Bjarne Stroustrup
+// Addison-Wesley, Boston, 2013.
+
+#pragma once
+
+#include <algorithm>
+#include <cassert>
+#include <cstring>
+#include <memory>
+#include <type_traits>
+#include <vector>
+
+#include "ngraph/shape.hpp"
+
+namespace ngraph
+{
+    namespace runtime
+    {
+        namespace init
+        {
+            // Recursively define types for N-deep initializer lists
+            template <typename T, size_t N>
+            struct NestedInitializerListWrapper
+            {
+                using type =
+                    std::initializer_list<typename NestedInitializerListWrapper<T, N - 1>::type>;
+            };
+
+            // 1-deep is a plain initializer_list
+            template <typename T>
+            struct NestedInitializerListWrapper<T, 1>
+            {
+                using type = std::initializer_list<T>;
+            };
+
+            // Scalar case is just the element type
+            template <typename T>
+            struct NestedInitializerListWrapper<T, 0>
+            {
+                using type = T;
+            };
+
+            // Convenience type name for N-deep initializer lists of Ts
+            template <typename T, size_t N>
+            using NestedInitializerList = typename NestedInitializerListWrapper<T, N>::type;
+
+            // Fill in a shape from a nested initializer list
+            // For a scalar, nothing to do.
+            template <typename T, size_t N>
+            typename std::enable_if<(N == 0), void>::type
+                fill_shape(Shape& shape, const NestedInitializerList<T, N>& inits)
+            {
+            }
+
+            // Check that the inits match the shape
+            template <typename T, size_t N>
+            typename std::enable_if<(N == 0), void>::type
+                check_shape(const Shape& shape, const NestedInitializerList<T, N>& inits)
+            {
+                assert(shape.size() == 0);
+            }
+
+            // For a plain initializer list, the shape is the length of the list.
+            template <typename T, size_t N>
+            typename std::enable_if<(N == 1)>::type
+                fill_shape(Shape& shape, const NestedInitializerList<T, N>& inits)
+            {
+                shape.push_back(inits.size());
+            }
+
+            template <typename T, size_t N>
+            typename std::enable_if<(N == 1)>::type
+                check_shape(const Shape& shape, const NestedInitializerList<T, N>& inits)
+            {
+                assert(shape.at(shape.size() - N) == inits.size());
+            }
+
+            // In the general case, we append our level's length and recurse.
+            template <typename T, size_t N>
+            typename std::enable_if<(N > 1), void>::type
+                fill_shape(Shape& shape, const NestedInitializerList<T, N>& inits)
+            {
+                shape.push_back(inits.size());
+                fill_shape<T, N - 1>(shape, *inits.begin());
+            }
+
+            template <typename T, size_t N>
+            typename std::enable_if<(N > 1), void>::type
+                check_shape(const Shape& shape, const NestedInitializerList<T, N>& inits)
+            {
+                assert(shape.at(shape.size() - N) == inits.size());
+                for (auto it : inits)
+                {
+                    check_shape<T, N - 1>(shape, it);
+                }
+            }
+
+            // Get the shape of inits.
+            template <typename T, size_t N>
+            Shape get_shape(const NestedInitializerList<T, N>& inits)
+            {
+                Shape shape;
+                fill_shape<T, N>(shape, inits);
+                check_shape<T, N>(shape, inits);
+                return shape;
+            }
+
+            template <typename IT, typename T, size_t N>
+            typename std::enable_if<(N == 1), IT>::type
+                flatten(IT it, const Shape& shape, const NestedInitializerList<T, N>& inits)
+            {
+                assert(inits.size() == shape.at(shape.size() - N));
+                for (auto it1 : inits)
+                {
+                    *(it++) = it1;
+                }
+                return it;
+            }
+
+            template <typename IT, typename T, size_t N>
+            typename std::enable_if<(N > 1), IT>::type
+                flatten(IT it, const Shape& shape, const NestedInitializerList<T, N>& inits)
+            {
+                assert(inits.size() == shape.at(shape.size() - N));
+                for (auto it1 : inits)
+                {
+                    it = flatten<IT, T, N - 1>(it, shape, it1);
+                }
+                return it;
+            }
+
+            template <typename IT, typename T, size_t N>
+            typename std::enable_if<(N == 0), IT>::type
+                flatten(IT it, const Shape& shape, const NestedInitializerList<T, 0>& init)
+            {
+                assert(shape.size() == 0);
+                *(it++) = init;
+                return it;
+            }
+        }
+
+        template <typename T>
+        class NDArrayBase
+        {
+            using vtype = std::vector<T>;
+
+        public:
+            using type = T;
+            using iterator = typename vtype::iterator;
+            using const_iterator = typename vtype::const_iterator;
+
+            NDArrayBase(const Shape& shape)
+                : m_shape(shape)
+                , m_elements(shape_size(m_shape))
+            {
+            }
+
+            const Shape& get_shape() const { return m_shape; }
+            const_iterator begin() const { return m_elements.begin(); }
+            const_iterator end() const { return m_elements.end(); }
+            vtype get_vector() { return m_elements; }
+            const vtype get_vector() const { return m_elements; }
+            bool operator==(const NDArrayBase<T>& other) const
+            {
+                return m_shape == other.m_shape && m_elements == other.m_elements;
+            }
+
+        protected:
+            Shape m_shape;
+            vtype m_elements;
+        };
+
+        /// An N dimensional array of elements of type T
+        template <typename T, size_t N>
+        class NDArray : public NDArrayBase<T>
+        {
+        public:
+            NDArray(const init::NestedInitializerList<T, N>& initial_value)
+                : NDArrayBase<T>(init::get_shape<T, N>(initial_value))
+            {
+                init::flatten<typename std::vector<T>::iterator, T, N>(
+                    NDArrayBase<T>::m_elements.begin(), NDArrayBase<T>::m_shape, initial_value);
+            }
+        };
+    }
+}

src/ngraph/runtime/ngvm/call_frame.cpp

@@ -22,13 +22,15 @@ using namespace ngraph::runtime::ngvm;
 
 CallFrame::CallFrame(size_t n_inputs,
                      size_t n_outputs,
+                     size_t frame_size,
                      const TensorViewPtrs& temps,
                      size_t initial_pc,
                      const shared_ptr<vector<shared_ptr<Instruction>>>& instructions)
 
     : m_n_inputs(n_inputs)
     , m_n_outputs(n_outputs)
-    , m_tensor_views(n_inputs + n_outputs + temps.size())
+    , m_frame_size(frame_size)
+    , m_tensor_views(m_frame_size)
     , m_initial_pc(initial_pc)
     , m_instructions(instructions)
 {
@@ -39,7 +41,15 @@ void CallFrame::tensor_call(
     const std::vector<std::shared_ptr<ngraph::runtime::TensorView>>& inputs,
     const std::vector<std::shared_ptr<ngraph::runtime::TensorView>>& outputs)
 {
+    if (inputs.size() != m_n_inputs)
+    {
+        throw ngraph_error("Incorrect number of inputs");
+    }
     copy(inputs.begin(), inputs.end(), m_tensor_views.begin());
+    if (outputs.size() != m_n_outputs)
+    {
+        throw ngraph_error("Incorrect number of outputs");
+    }
     copy(outputs.begin(), outputs.end(), m_tensor_views.begin() + m_n_inputs);
     m_next_pc = m_initial_pc;
     m_return = false;

src/ngraph/runtime/ngvm/call_frame.hpp

@@ -38,6 +38,7 @@ namespace ngraph
                 CallFrame(
                     size_t n_inputs,
                     size_t n_outputs,
+                    size_t frame_size,
                     const TensorViewPtrs& temps,
                     size_t initial_pc,
                     const std::shared_ptr<std::vector<std::shared_ptr<Instruction>>>& instructions);
@@ -69,6 +70,7 @@ namespace ngraph
             protected:
                 size_t m_n_inputs;
                 size_t m_n_outputs;
+                size_t m_frame_size;
                 TensorViewPtrs m_tensor_views;
                 size_t m_initial_pc;
                 std::shared_ptr<std::vector<std::shared_ptr<Instruction>>> m_instructions;

src/ngraph/runtime/ngvm/external_function.cpp

@@ -1048,20 +1048,38 @@ void ExternalFunction::compile(FunctionMap& function_map)
         for (const descriptor::Output& output : param->get_outputs())
         {
             auto tv = output.get_tensor_view();
-            size_t index = tensor_index.size();
+            size_t index = m_frame_size++;
             tensor_index[tv] = index;
         }
     }
-    m_n_inputs = tensor_index.size();
+    m_n_inputs = m_frame_size;
 
-    // Next are the function outputs
     for (const descriptor::Output& output : m_function->get_result()->get_outputs())
     {
         auto tv = output.get_tensor_view();
-        size_t index = tensor_index.size();
+        size_t index = m_frame_size++;
+        auto prev_index_it = tensor_index.find(tv);
+        if (prev_index_it != tensor_index.end())
+        {
+            auto result_tensor_type =
+                dynamic_pointer_cast<const TensorViewType>(tv->get_value_type());
+            assert(nullptr != result_tensor_type);
+            auto& result_element_type = result_tensor_type->get_element_type();
+            auto ef = this;
+            PUSH_POLYMORPHIC_INSTRUCTION(result_element_type,
+                                         "Copy has unhandled element type",
+                                         eigen::CopyInstruction,
+                                         prev_index_it->second,
+                                         index);
+        }
+        else
+        {
             tensor_index[tv] = index;
         }
-    m_n_outputs = tensor_index.size() - m_n_inputs;
+    }
+    m_n_outputs = m_frame_size - m_n_inputs;
+    vector<shared_ptr<Instruction>> input_output_copies;
+    swap(*m_instructions, input_output_copies);
 
     // All remaining tensor views
     for (shared_ptr<Node> node : m_function->get_ordered_ops())
@@ -1071,7 +1089,7 @@ void ExternalFunction::compile(FunctionMap& function_map)
             auto tv = output.get_tensor_view();
             if (0 == tensor_index.count(tv))
             {
-                size_t index = tensor_index.size();
+                size_t index = m_frame_size++;
                 tensor_index[tv] = index;
                 m_temp_views.push_back(tv);
             }
@@ -1104,6 +1122,8 @@ void ExternalFunction::compile(FunctionMap& function_map)
         }
         handler_it->second(node.get(), this, function_map, in, out);
     }
+    m_instructions->insert(
+        m_instructions->end(), input_output_copies.begin(), input_output_copies.end());
     m_instructions->push_back(make_shared<eigen::ReturnInstruction>());
     m_is_compiled = true;
     if (m_release_function)
@@ -1136,5 +1156,5 @@ shared_ptr<ngraph::runtime::CallFrame> ExternalFunction::make_call_frame(Functio
 #undef M
     }
     return make_shared<ngraph::runtime::ngvm::CallFrame>(
-        m_n_inputs, m_n_outputs, temps, 0, m_instructions);
+        m_n_inputs, m_n_outputs, m_frame_size, temps, 0, m_instructions);
 }

src/ngraph/runtime/ngvm/external_function.hpp

@@ -62,6 +62,7 @@ namespace ngraph
 
                 size_t m_n_inputs;
                 size_t m_n_outputs;
+                size_t m_frame_size{0};
                 std::shared_ptr<std::vector<std::shared_ptr<Instruction>>> m_instructions;
                 ngraph::descriptor::TensorViewPtrs m_temp_views;
 

src/ngraph/runtime/parameterized_tensor_view.hpp

@@ -14,12 +14,16 @@
 
 #pragma once
 
+#include <algorithm>
+#include <cassert>
 #include <cstring>
 #include <memory>
+#include <type_traits>
 #include <vector>
 
 #include "ngraph/descriptor/layout/dense_tensor_view_layout.hpp"
 #include "ngraph/descriptor/primary_tensor_view.hpp"
+#include "ngraph/runtime/ndarray.hpp"
 #include "ngraph/runtime/tensor_view.hpp"
 #include "ngraph/shape.hpp"
 #include "ngraph/types/element_type.hpp"
@@ -49,6 +53,12 @@ namespace ngraph
             ParameterizedTensorView(
                 const std::shared_ptr<ngraph::descriptor::TensorView>& descriptor);
 
+            ParameterizedTensorView(const NDArrayBase<typename ET::type>& initializer)
+                : ParameterizedTensorView(initializer.get_shape())
+            {
+                m_vector = initializer.get_vector();
+            }
+
             using element_type = ET;
             using value_type = typename ET::type;
             using storage_type = std::vector<value_type>;
@@ -60,8 +70,17 @@ namespace ngraph
                 return *this;
             }
 
+            template <typename T>
+            ParameterizedTensorView<ET>& operator=(const NDArrayBase<T>& ndarray)
+            {
+                assert(ndarray.get_shape() == get_shape());
+                std::copy(ndarray.begin(), ndarray.end(), m_vector.begin());
+                return *this;
+            }
+
             // For getting the data out
             storage_type& get_vector() { return m_vector; }
+            const storage_type& get_vector() const { return m_vector; }
             virtual void write(const void* p, size_t tensor_offset, size_t n) override
             {
                 size_t elt_offset = tensor_offset / sizeof(typename ET::type);
@@ -104,6 +123,11 @@ namespace ngraph
                 std::memcpy(p, &m_vector[elt_offset], n);
             }
 
+            bool operator==(const NDArrayBase<typename ET::type>& ndarray) const
+            {
+                return get_shape() == ndarray.get_shape() && get_vector() == ndarray.get_vector();
+            }
+
         protected:
             storage_type m_vector;
         };

src/ngraph/runtime/tensor_view.cpp

@@ -38,3 +38,9 @@ const ngraph::Shape& TensorView::get_shape() const
 {
     return m_descriptor->get_tensor_view_type()->get_shape();
 }
+
+std::shared_ptr<ngraph::descriptor::layout::TensorViewLayout>
+    TensorView::get_tensor_view_layout() const
+{
+    return m_descriptor->get_tensor_view_layout();
+}

src/ngraph/runtime/tensor_view.hpp

@@ -60,6 +60,9 @@ namespace ngraph
 
             const ngraph::Shape& get_shape() const;
 
+            std::shared_ptr<ngraph::descriptor::layout::TensorViewLayout>
+                get_tensor_view_layout() const;
+
             /// @brief Write bytes directly into the tensor
             /// @param p Pointer to source of data
             /// @param tensor_offset Offset into tensor storage to begin writing. Must be element-aligned.

src/ngraph/runtime/utils.cpp

@@ -12,6 +12,13 @@
 // See the License for the specific language governing permissions and
 // ----------------------------------------------------------------------------
 
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+
+#include "ngraph/function.hpp"
+#include "ngraph/ops/tuple.hpp"
+#include "ngraph/runtime/call_frame.hpp"
 #include "ngraph/runtime/utils.hpp"
 
 std::shared_ptr<ngraph::runtime::Tuple> ngraph::runtime::make_tuple(

src/ngraph/test/all_close.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <cmath>
+#include <memory>
+#include <vector>
+
+#include "ngraph/except.hpp"
+#include "ngraph/test/all_close.hpp"
+
+template <typename ET>
+bool ngraph::test::all_close(
+    const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>& as,
+    const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>& bs,
+    typename ET::type rtol,
+    typename ET::type atol)
+{
+    if (as.size() != bs.size())
+    {
+        return false;
+    }
+    for (size_t i = 0; i < as.size(); ++i)
+    {
+        if (!all_close(as[i], bs[i], rtol, atol))
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+template bool ngraph::test::all_close<ngraph::element::Float32>(
+    const std::vector<
+        std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>>& as,
+    const std::vector<
+        std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>>& bs,
+    ngraph::element::Float32::type rtol,
+    ngraph::element::Float32::type atol);
+
+template bool ngraph::test::all_close<ngraph::element::Float64>(
+    const std::vector<
+        std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>>& as,
+    const std::vector<
+        std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>>& bs,
+    ngraph::element::Float64::type rtol,
+    ngraph::element::Float64::type atol);
+
+template <typename ET>
+bool ngraph::test::all_close(const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& a,
+                             const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& b,
+                             typename ET::type rtol,
+                             typename ET::type atol)
+{
+    // Check that the layouts are compatible
+    if (*a->get_tensor_view_layout() != *b->get_tensor_view_layout())
+    {
+        throw ngraph_error("Cannot compare tensors with different layouts");
+    }
+
+    if (a->get_shape() != b->get_shape())
+        return false;
+
+    return all_close(a->get_vector(), b->get_vector(), rtol, atol);
+}
+
+template bool ngraph::test::all_close<ngraph::element::Float32>(
+    const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>& a,
+    const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>& b,
+    ngraph::element::Float32::type rtol,
+    ngraph::element::Float32::type atol);
+
+template bool ngraph::test::all_close<ngraph::element::Float64>(
+    const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>& a,
+    const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>& b,
+    ngraph::element::Float64::type rtol,
+    ngraph::element::Float64::type atol);
+
+template <typename T>
+bool ngraph::test::all_close(const std::vector<T>& a, const std::vector<T>& b, T rtol, T atol)
+{
+    assert(a.size() == b.size());
+    for (size_t i = 0; i < a.size(); ++i)
+    {
+        if (std::abs(a[i] - b[i]) > atol + rtol * std::abs(b[i]))
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+template bool ngraph::test::all_close<float>(const std::vector<float>& a,
+                                             const std::vector<float>& b,
+                                             float rtol,
+                                             float atol);
+
+template bool ngraph::test::all_close<double>(const std::vector<double>& a,
+                                              const std::vector<double>& b,
+                                              double rtol,
+                                              double atol);

src/ngraph/test/all_close.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <memory>
+#include <vector>
+
+#include "ngraph/runtime/parameterized_tensor_view.hpp"
+#include "ngraph/types/element_type.hpp"
+
+namespace ngraph
+{
+    namespace test
+    {
+        /// @brief Same as numpy.allclose
+        /// @param as First tensors to compare
+        /// @param bs Second tensors to compare
+        /// @param rtol Relative tolerance
+        /// @param atol Absolute tolerance
+        /// Returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
+        template <typename ET>
+        bool all_close(
+            const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>& as,
+            const std::vector<std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>>& bs,
+            typename ET::type rtol,
+            typename ET::type atol);
+
+        extern template bool all_close<element::Float32>(
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>&
+                as,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float32>>>&
+                bs,
+            element::Float32::type rtol,
+            element::Float32::type atol);
+
+        extern template bool all_close<element::Float64>(
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>&
+                as,
+            const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<element::Float64>>>&
+                bs,
+            element::Float64::type rtol,
+            element::Float64::type atol);
+
+        /// @brief Same as numpy.allclose
+        /// @param a First tensor to compare
+        /// @param b Second tensor to compare
+        /// @param rtol Relative tolerance
+        /// @param atol Absolute tolerance
+        /// Returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
+        template <typename ET>
+        bool all_close(const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& a,
+                       const std::shared_ptr<ngraph::runtime::ParameterizedTensorView<ET>>& b,
+                       typename ET::type rtol = 1e-5f,
+                       typename ET::type atol = 1e-8f);
+
+        extern template bool all_close<ngraph::element::Float32>(
+            const std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>& a,
+            const std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float32>>& b,
+            ngraph::element::Float32::type rtol,
+            ngraph::element::Float32::type atol);
+
+        extern template bool all_close<ngraph::element::Float64>(
+            const std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>& a,
+            const std::shared_ptr<
+                ngraph::runtime::ParameterizedTensorView<ngraph::element::Float64>>& b,
+            ngraph::element::Float64::type rtol,
+            ngraph::element::Float64::type atol);
+
+        /// @brief Same as numpy.allclose
+        /// @param a First tensor to compare
+        /// @param b Second tensor to compare
+        /// @param rtol Relative tolerance
+        /// @param atol Absolute tolerance
+        /// @returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
+        template <typename T>
+        bool all_close(const std::vector<T>& a,
+                       const std::vector<T>& b,
+                       T rtol = 1e-5f,
+                       T atol = 1e-8f);
+
+        extern template bool all_close<float>(const std::vector<float>& a,
+                                              const std::vector<float>& b,
+                                              float rtol,
+                                              float atol);
+
+        extern template bool all_close<double>(const std::vector<double>& a,
+                                               const std::vector<double>& b,
+                                               double rtol,
+                                               double atol);
+    }
+}

src/ngraph/test/random.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include <functional>
+#include <random>
+
+#include "ngraph/types/element_type.hpp"
+
+namespace ngraph
+{
+    namespace test
+    {
+        /// @brief A predictable pseudo-random number generator
+        /// The seed is initialized so that we get repeatable pseudo-random numbers for tests
+        template <typename ET>
+        class Uniform
+        {
+        public:
+            using type = typename ET::type;
+
+            Uniform(type min, type max, type seed = 0)
+                : m_engine(seed)
+                , m_distribution(min, max)
+                , m_r(std::bind(m_distribution, m_engine))
+            {
+            }
+
+            /// @brief Randomly initialize a tensor
+            /// @param ptv The tensor to initialize
+            const std::shared_ptr<runtime::ParameterizedTensorView<ET>>
+                initialize(const std::shared_ptr<runtime::ParameterizedTensorView<ET>>& ptv)
+            {
+                for (auto& elt : ptv->get_vector())
+                {
+                    elt = m_r();
+                }
+                return ptv;
+            }
+
+        protected:
+            std::default_random_engine m_engine;
+            std::uniform_real_distribution<type> m_distribution;
+            std::function<type()> m_r;
+        };
+    }
+}

src/ngraph/types/element_type.hpp

@@ -155,6 +155,9 @@ namespace ngraph
         NGRAPH_DEFINE_TRAITED_TYPE_NAME(float)
         using Float32 = TraitedType<float>;
 
+        NGRAPH_DEFINE_TRAITED_TYPE_NAME(double)
+        using Float64 = TraitedType<double>;
+
         NGRAPH_DEFINE_TRAITED_TYPE_NAME(int8_t)
         using Int8 = TraitedType<int8_t>;
 

test/CMakeLists.txt

@@ -22,6 +22,7 @@ include_directories(
     )
 
 set (SRC
+    autodiff.cpp
     build_graph.cpp
     eigen.cpp
     element_type.cpp

test/autodiff.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <algorithm>
+#include <functional>
+#include <memory>
+#include <tuple>
+
+#include "gtest/gtest.h"
+
+#include "ngraph/autodiff/backprop_derivative.hpp"
+#include "ngraph/autodiff/backprop_function.hpp"
+#include "ngraph/autodiff/numeric_derivative.hpp"
+#include "ngraph/ngraph.hpp"
+#include "ngraph/test/all_close.hpp"
+#include "ngraph/test/random.hpp"
+
+using namespace std;
+using namespace ngraph;
+
+template <typename ET>
+bool autodiff_numeric_compare(
+    const std::shared_ptr<runtime::Manager>& manager,
+    const std::shared_ptr<runtime::Backend>& backend,
+    std::function<std::shared_ptr<Function>()> make_graph,
+    const std::vector<std::shared_ptr<runtime::ParameterizedTensorView<ET>>>& args,
+    typename ET::type rtol,
+    typename ET::type atol)
+{
+    auto results_num =
+        autodiff::numeric_derivative<element::Float32>(manager, backend, make_graph(), args, .001f);
+    auto results_sym =
+        autodiff::backprop_derivative<element::Float32>(manager, backend, make_graph(), args);
+    return test::all_close(results_num, results_sym, .01f, .01f);
+}
+
+TEST(backwards, add)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto x1 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(
+            X0 + X1, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
+    };
+    EXPECT_TRUE(autodiff_numeric_compare<element::Float32>(
+        manager, backend, make_graph, {x0, x1}, .01f, .01f));
+}
+
+TEST(backwards, divide)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    test::Uniform<element::Float32> rng1(1.0f, 2.0f);
+    test::Uniform<element::Float32> rng2(-2.0f, -1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto x1 = rng1.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto x2 = rng2.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(
+            X0 / X1, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
+    };
+    EXPECT_TRUE(autodiff_numeric_compare<element::Float32>(
+        manager, backend, make_graph, {x0, x1}, .01f, .01f));
+    EXPECT_TRUE(autodiff_numeric_compare<element::Float32>(
+        manager, backend, make_graph, {x0, x2}, .01f, .01f));
+}
+
+TEST(backwards, exp)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(
+            make_shared<op::Exp>(X0), nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0});
+    };
+    EXPECT_TRUE(
+        autodiff_numeric_compare<element::Float32>(manager, backend, make_graph, {x0}, .01f, .01f));
+}
+
+TEST(backwards, log)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(1.0f, 2.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(
+            make_shared<op::Log>(X0), nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0});
+    };
+    EXPECT_TRUE(
+        autodiff_numeric_compare<element::Float32>(manager, backend, make_graph, {x0}, .01f, .01f));
+}
+
+TEST(backwards, multiply)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto x1 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(
+            X0 * X1, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
+    };
+    EXPECT_TRUE(autodiff_numeric_compare<element::Float32>(
+        manager, backend, make_graph, {x0, x1}, .01f, .01f));
+}
+
+TEST(backwards, negative)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(-X0, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0});
+    };
+    EXPECT_TRUE(
+        autodiff_numeric_compare<element::Float32>(manager, backend, make_graph, {x0}, .01f, .01f));
+}
+
+TEST(backwards, parameter)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(X0, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0});
+    };
+    EXPECT_TRUE(
+        autodiff_numeric_compare<element::Float32>(manager, backend, make_graph, {x0}, .01f, .01f));
+}
+
+TEST(backwards, subtract)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto x1 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(
+            X0 - X1, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
+    };
+    EXPECT_TRUE(autodiff_numeric_compare<element::Float32>(
+        manager, backend, make_graph, {x0, x1}, .01f, .01f));
+}
+
+TEST(backwards, abc)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto x1 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto x2 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        auto X2 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(
+            (X0 + X1) * X2, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0, X1, X2});
+    };
+    EXPECT_TRUE(autodiff_numeric_compare<element::Float32>(
+        manager, backend, make_graph, {x0, x1, x2}, .01f, .01f));
+}

test/execute.cpp

@@ -36,22 +36,22 @@ TEST(execute, abc)
     auto cf = backend->make_call_frame(external);
 
     // Create some tensors for input/output
-    auto a = backend->make_parameterized_tensor_view<element::Float32>(shape);
-    *a = vector<float>{1, 2, 3, 4};
-    auto b = backend->make_parameterized_tensor_view<element::Float32>(shape);
-    *b = vector<float>{5, 6, 7, 8};
-    auto c = backend->make_parameterized_tensor_view<element::Float32>(shape);
-    *c = vector<float>{9, 10, 11, 12};
+    auto a = backend->make_parameterized_tensor_view<element::Float32>(
+        runtime::NDArray<float, 2>({{1, 2}, {3, 4}}));
+    auto b = backend->make_parameterized_tensor_view<element::Float32>(
+        runtime::NDArray<float, 2>({{5, 6}, {7, 8}}));
+    auto c = backend->make_parameterized_tensor_view<element::Float32>(
+        runtime::NDArray<float, 2>({{9, 10}, {11, 12}}));
     auto result = backend->make_parameterized_tensor_view<element::Float32>(shape);
 
     (*cf)({a, b, c}, {result});
-    ASSERT_EQ((vector<float>{54, 80, 110, 144}), result->get_vector());
+    ASSERT_EQ(*result, (runtime::NDArray<float, 2>({{54, 80}, {110, 144}})));
 
     (*cf)({b, a, c}, {result});
-    ASSERT_EQ((vector<float>{54, 80, 110, 144}), result->get_vector());
+    ASSERT_EQ(*result, (runtime::NDArray<float, 2>({{54, 80}, {110, 144}})));
 
     (*cf)({a, c, b}, {result});
-    ASSERT_EQ((vector<float>{50, 72, 98, 128}), result->get_vector());
+    ASSERT_EQ(*result, (runtime::NDArray<float, 2>({{50, 72}, {98, 128}})));
 }
 
 TEST(execute, abc_int64)

test/util.cpp

@@ -18,6 +18,8 @@
 
 #include "gtest/gtest.h"
 
+#include "ngraph/ngraph.hpp"
+#include "ngraph/test/all_close.hpp"
 #include "ngraph/util.hpp"
 
 using namespace std;
@@ -169,3 +171,25 @@ TEST(util, reduce)
         EXPECT_EQ(actual, 720);
     }
 }
+
+TEST(util, all_close)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    // Create some tensors for input/output
+    auto a = backend->make_parameterized_tensor_view<element::Float32>(
+        runtime::NDArray<float, 2>({{1, 2, 3}, {3, 4, 5}}));
+    auto b = backend->make_parameterized_tensor_view<element::Float32>(
+        runtime::NDArray<float, 2>({{1, 2, 3}, {3, 4, 5}}));
+
+    EXPECT_TRUE(ngraph::test::all_close(a, b));
+
+    auto c = backend->make_parameterized_tensor_view<element::Float32>(
+        runtime::NDArray<float, 2>({{1.1f, 2, 3}, {3, 4, 5}}));
+    EXPECT_FALSE(ngraph::test::all_close(c, a, 0, .05f));
+    EXPECT_TRUE(ngraph::test::all_close(c, a, 0, .11f));
+
+    EXPECT_FALSE(ngraph::test::all_close(c, a, .05f, 0));
+    EXPECT_TRUE(ngraph::test::all_close(c, a, .11f, 0));
+}