Backprop for some ops (#257)

* Autodiff for abs

* Formatting, more tests for abs

* Cos autodiff; also a clarifying comment in the abs test

* Forgot cos.cpp

* Sin autodiff

* Again, forgot to add sin.cpp  :/

* Tan autodiff

* Minor formatting tweak

* Commit partial work on select backprop so I can run valgrind on the server  :/

* Fix boolean thingy so it works on Linux

* Autodiff for ceiling, convert (untested), floor, sign. Fix unit test for tan. Implement ceiling, floor in VM.

* Fix bug in abs/sign unit test ranges

* Add multiplicative inverse and square-root ops (needed for hyperbolic trig autodiff)

* Better formula for sqrt adjoints

* Autodiff for hyperbolic trig ops

* Forgot to add cpp files for hyperbolics

* Remove inv (don't need it after all); also formatting (oops)

* fix bug with Convert autodiff

* Autodiff for concat

* Restore the accidentally-commented-out unit test for abs

* Formatting

* Fix 'unordered_map.at' exception when Adjoints::Adjoints visit a node that has never been add_delta'd; remove workarounds for that bug

* Fix erroneous 'not implemented' docstring for op::Not

* Autodiff for sum

* Checking in broken support for replace_slice so I can test it with valgrind on the dev box

* Fix unit test bug with tensor initialization; formatting

* Implement replace-slice in CPU backend

* Autodiff for Slice and ReplaceSlice

* Tweak docs for ReplaceSlice

* Remove no-longer-needed cast of arg list to runtime::TensorView

src/ngraph/CMakeLists.txt

@@ -26,6 +26,7 @@ set (SRC
     function.cpp
     log.cpp
     node.cpp
+    ops/abs.cpp
     ops/add.cpp
     ops/binary_elementwise_arithmetic.cpp
     ops/binary_elementwise_comparison.cpp
@@ -34,6 +35,8 @@ set (SRC
     ops/concatenate.cpp
     ops/constant.cpp
     ops/convert.cpp
+    ops/cos.cpp
+    ops/cosh.cpp
     ops/divide.cpp
     ops/dot.cpp
     ops/exp.cpp
@@ -44,15 +47,22 @@ set (SRC
     ops/minimum.cpp
     ops/multiply.cpp
     ops/negative.cpp
+    ops/not.cpp
     ops/op.cpp
     ops/parameter.cpp
     ops/power.cpp
     ops/reduce.cpp
+    ops/replace_slice.cpp
     ops/reshape.cpp
     ops/select.cpp
+    ops/sin.cpp
+    ops/sinh.cpp
     ops/slice.cpp
+    ops/sqrt.cpp
     ops/subtract.cpp
     ops/sum.cpp
+    ops/tan.cpp
+    ops/tanh.cpp
     ops/tuple.cpp
     ops/unary_elementwise_arithmetic.cpp
     ops/unary_elementwise.cpp

src/ngraph/autodiff/adjoints.cpp

@@ -25,6 +25,8 @@
 #include "ngraph/ops/broadcast.hpp"
 #include "ngraph/ops/constant.hpp"
 #include "ngraph/ops/convert.hpp"
+#include "ngraph/ops/replace_slice.hpp"
+#include "ngraph/ops/slice.hpp"
 #include "ngraph/ops/tuple.hpp"
 #include "ngraph/types/type.hpp"
 
@@ -103,7 +105,7 @@ autodiff::Adjoints::Adjoints(const std::shared_ptr<Node>& y, const std::shared_p
                 nodes_to_check.push_front(arg);
             }
         }
-        node->generate_adjoints(*this, m_adjoint_map.at(node.get()));
+        node->generate_adjoints(*this, get(node));
     }
 }
 
@@ -132,3 +134,37 @@ void autodiff::Adjoints::add_delta(const std::shared_ptr<Node>& x,
         adjoint_it->second = std::make_shared<op::Add>(adjoint_it->second, delta);
     }
 }
+
+void autodiff::Adjoints::add_delta_to_slice(const std::shared_ptr<Node>& x,
+                                            const std::shared_ptr<Node>& delta,
+                                            const Coordinate& lower_bounds,
+                                            const Coordinate& upper_bounds,
+                                            const Shape& step)
+{
+    auto x_tensor_view_type = std::dynamic_pointer_cast<const TensorViewType>(x->get_value_type());
+    auto delta_tensor_view_type =
+        std::dynamic_pointer_cast<const TensorViewType>(delta->get_value_type());
+    assert(nullptr != x_tensor_view_type);
+    assert(nullptr != delta_tensor_view_type);
+    assert(x_tensor_view_type->get_element_type() == delta_tensor_view_type->get_element_type());
+    assert(x_tensor_view_type->get_shape().size() == delta_tensor_view_type->get_shape().size());
+
+    auto adjoint_it = m_adjoint_map.find(x.get());
+    if (m_adjoint_map.end() == adjoint_it)
+    {
+        auto zeros = make_zero(x->get_outputs().at(0).get_tensor_view_type());
+        m_adjoint_map.insert(
+            {x.get(),
+             std::make_shared<op::ReplaceSlice>(zeros, delta, lower_bounds, upper_bounds, step)});
+    }
+    else
+    {
+        adjoint_it->second = std::make_shared<op::ReplaceSlice>(
+            adjoint_it->second,
+            std::make_shared<op::Slice>(adjoint_it->second, lower_bounds, upper_bounds, step) +
+                delta,
+            lower_bounds,
+            upper_bounds,
+            step);
+    }
+}

src/ngraph/autodiff/adjoints.hpp

@@ -17,6 +17,8 @@
 #include <memory>
 #include <unordered_map>
 
+#include "ngraph/common.hpp"
+
 namespace ngraph
 {
     class Node;
@@ -54,6 +56,19 @@ namespace ngraph
             /// @param delta A backprop contribution
             void add_delta(const std::shared_ptr<Node>& x, const std::shared_ptr<Node>& delta);
 
+            /// @brief Add a backprop contribution to a slice of x's adjoint
+            ///
+            /// @param x The adjoint node
+            /// @param delta A backprop contribution
+            /// @param lower_bounds Lower bounds of slice to add to
+            /// @param upper_bounds Upper bounds of slice to add to
+            /// @param step Step (or stride) of slice to add to
+            void add_delta_to_slice(const std::shared_ptr<Node>& x,
+                                    const std::shared_ptr<Node>& delta,
+                                    const Coordinate& lower_bounds,
+                                    const Coordinate& upper_bounds,
+                                    const Shape& step);
+
         protected:
             std::unordered_map<Node*, std::shared_ptr<Node>> m_adjoint_map;
         };

src/ngraph/ngraph.hpp

@@ -84,18 +84,21 @@
 #include "ngraph/ops/minimum.hpp"
 #include "ngraph/ops/multiply.hpp"
 #include "ngraph/ops/negative.hpp"
+#include "ngraph/ops/not.hpp"
 #include "ngraph/ops/not_equal.hpp"
 #include "ngraph/ops/op.hpp"
 #include "ngraph/ops/parameter.hpp"
 #include "ngraph/ops/power.hpp"
 #include "ngraph/ops/reduce.hpp"
 #include "ngraph/ops/remainder.hpp"
+#include "ngraph/ops/replace_slice.hpp"
 #include "ngraph/ops/reshape.hpp"
 #include "ngraph/ops/select.hpp"
 #include "ngraph/ops/sign.hpp"
 #include "ngraph/ops/sin.hpp"
 #include "ngraph/ops/sinh.hpp"
 #include "ngraph/ops/slice.hpp"
+#include "ngraph/ops/sqrt.hpp"
 #include "ngraph/ops/subtract.hpp"
 #include "ngraph/ops/sum.hpp"
 #include "ngraph/ops/tan.hpp"

src/ngraph/ops/abs.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/abs.hpp"
+#include "ngraph/ops/multiply.hpp"
+#include "ngraph/ops/sign.hpp"
+
+void ngraph::op::Abs::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = get_inputs().at(0).get_output().get_node();
+
+    adjoints.add_delta(x, delta * std::make_shared<op::Sign>(x));
+}

src/ngraph/ops/abs.hpp

@@ -59,6 +59,10 @@ namespace ngraph
                     throw ngraph_error("Incorrect number of new arguments");
                 return std::make_shared<Abs>(new_args.at(0));
             }
+
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
     }
 }

src/ngraph/ops/add.cpp

@@ -17,8 +17,8 @@
 void ngraph::op::Add::generate_adjoints(autodiff::Adjoints& adjoints,
                                         const std::shared_ptr<Node>& delta)
 {
-    auto x = m_arguments[0];
-    auto y = m_arguments[1];
+    auto x = get_inputs().at(0).get_output().get_node();
+    auto y = get_inputs().at(1).get_output().get_node();
 
     adjoints.add_delta(x, delta);
     adjoints.add_delta(y, delta);

src/ngraph/ops/ceiling.hpp

@@ -36,9 +36,9 @@ namespace ngraph
         ///
         /// ## Implementation Status
         ///
-        /// | Backend | Status           |
-        /// | ------- | ---------------- |
-        /// | NGVM    | Not implemented. |
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Ceiling : public UnaryElementwiseArithmetic
         {
         public:

src/ngraph/ops/concatenate.cpp

@@ -12,9 +12,11 @@
 // See the License for the specific language governing permissions and
 // ----------------------------------------------------------------------------
 
+#include <cassert>
 #include <memory>
 
 #include "ngraph/ops/concatenate.hpp"
+#include "ngraph/ops/slice.hpp"
 
 using namespace std;
 using namespace ngraph;
@@ -70,3 +72,41 @@ op::Concat::Concat(const Nodes& args, size_t concatenation_axis)
 
     set_value_type_checked(make_shared<TensorViewType>(arg0_element_type, concatenated_shape));
 }
+
+void op::Concat::generate_adjoints(autodiff::Adjoints& adjoints, const std::shared_ptr<Node>& delta)
+{
+    auto value_type = get_value_type();
+    auto tensor_view_type = std::dynamic_pointer_cast<const TensorViewType>(value_type);
+
+    assert(nullptr != tensor_view_type);
+
+    auto concat_result_shape = tensor_view_type->get_shape();
+
+    Coordinate arg_delta_slice_lower = Coordinate(concat_result_shape.size(), 0);
+    Coordinate arg_delta_slice_upper = concat_result_shape;
+    Coordinate arg_delta_slice_step = Coordinate(concat_result_shape.size(), 1);
+
+    size_t pos = 0;
+
+    for (auto arg : m_arguments)
+    {
+        auto arg_value_type = arg->get_value_type();
+        auto arg_tensor_view_type = std::dynamic_pointer_cast<const TensorViewType>(arg_value_type);
+        assert(nullptr != arg_tensor_view_type);
+        auto arg_shape = arg_tensor_view_type->get_shape();
+
+        auto slice_width = arg_shape[m_concatenation_axis];
+
+        size_t next_pos = pos + slice_width;
+
+        arg_delta_slice_lower[m_concatenation_axis] = pos;
+        arg_delta_slice_upper[m_concatenation_axis] = next_pos;
+
+        adjoints.add_delta(
+            arg,
+            make_shared<op::Slice>(
+                delta, arg_delta_slice_lower, arg_delta_slice_upper, arg_delta_slice_step));
+
+        pos = next_pos;
+    }
+}

src/ngraph/ops/concatenate.hpp

@@ -81,6 +81,8 @@ namespace ngraph
             /// \return The concatenation axis.
             size_t get_concatenation_axis() const { return m_concatenation_axis; }
         protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
             const size_t m_concatenation_axis;
         };
     }

src/ngraph/ops/convert.cpp

@@ -28,3 +28,11 @@ op::Convert::Convert(const std::shared_ptr<Node>& arg, const ngraph::element::Ty
     , m_element_type(element_type)
 {
 }
+
+void ngraph::op::Convert::generate_adjoints(autodiff::Adjoints& adjoints,
+                                            const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, std::make_shared<op::Convert>(delta, x->get_element_type()));
+}

src/ngraph/ops/convert.hpp

@@ -68,6 +68,8 @@ namespace ngraph
             const element::Type& get_convert_element_type() const { return m_element_type; }
         protected:
             const ngraph::element::Type& m_element_type;
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
     }
 }

src/ngraph/ops/cos.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/cos.hpp"
+#include "ngraph/ops/multiply.hpp"
+#include "ngraph/ops/negative.hpp"
+#include "ngraph/ops/sin.hpp"
+
+void ngraph::op::Cos::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, -delta * (std::make_shared<op::Sin>(x)));
+}

src/ngraph/ops/cos.hpp

@@ -57,6 +57,10 @@ namespace ngraph
                     throw ngraph_error("Incorrect number of new arguments");
                 return std::make_shared<Cos>(new_args.at(0));
             }
+
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
     }
 }

src/ngraph/ops/cosh.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/cosh.hpp"
+#include "ngraph/ops/multiply.hpp"
+#include "ngraph/ops/sinh.hpp"
+
+void ngraph::op::Cosh::generate_adjoints(autodiff::Adjoints& adjoints,
+                                         const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, delta * (std::make_shared<op::Sinh>(x)));
+}

src/ngraph/ops/cosh.hpp

@@ -57,6 +57,10 @@ namespace ngraph
                     throw ngraph_error("Incorrect number of new arguments");
                 return std::make_shared<Cosh>(new_args.at(0));
             }
+
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
     }
 }

src/ngraph/ops/floor.hpp

@@ -36,9 +36,9 @@ namespace ngraph
         ///
         /// ## Implementation Status
         ///
-        /// | Backend | Status           |
-        /// | ------- | ---------------- |
-        /// | NGVM    | Not implemented. |
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
         class Floor : public UnaryElementwiseArithmetic
         {
         public:

src/ngraph/ops/not.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/not.hpp"
+#include "ngraph/ops/op.hpp"
+
+using namespace ngraph;
+using namespace ngraph::op;
+
+op::Not::Not(const std::shared_ptr<Node>& arg)
+    : UnaryElementwise(
+          "Not",
+          [](const ngraph::element::Type& arg_element_type) -> const ngraph::element::Type& {
+              if (arg_element_type != element::Bool::element_type())
+              {
+                  throw ngraph_error(
+                      "Operands for logical operators must have boolean element "
+                      "type");
+              }
+
+              return arg_element_type;
+          },
+          arg)
+{
+}

src/ngraph/ops/not.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise logical negation operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                                          | Description                                       |
+        /// | ----- | --------------------------------------------- | ------------------------------------------------- |
+        /// | `arg` | \f$\texttt{bool}[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape, with boolean element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                               | Description                                                                                                    |
+        /// | ---------------------------------- | -------------------------------------------------------------------------------------------------------------- |
+        /// | \f$\texttt{bool}[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = 1\text{ if }\texttt{arg}[i_1,\dots,i_n] = 0\text{, else } 0\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Not : public UnaryElementwise
+        {
+        public:
+            /// \brief Constructs a logical negation operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Not(const std::shared_ptr<Node>& arg);
+
+            virtual std::shared_ptr<Node> copy_with_new_args(
+                const std::vector<std::shared_ptr<Node>>& new_args) const override
+            {
+                if (new_args.size() != 1)
+                    throw ngraph_error("Incorrect number of new arguments");
+                return std::make_shared<Not>(new_args.at(0));
+            }
+        };
+    }
+}

src/ngraph/ops/replace_slice.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/replace_slice.hpp"
+#include "ngraph/ops/constant.hpp"
+#include "ngraph/ops/slice.hpp"
+
+using namespace std;
+using namespace ngraph;
+
+op::ReplaceSlice::ReplaceSlice(const std::shared_ptr<Node>& arg0,
+                               const std::shared_ptr<Node>& arg1,
+                               const Coordinate& lower_bounds,
+                               const Coordinate& upper_bounds,
+                               const Shape& step)
+    : RequiresTensorViewArgs("ReplaceSlice", {arg0, arg1})
+    , m_lower_bounds(lower_bounds)
+    , m_upper_bounds(upper_bounds)
+    , m_step(step)
+{
+    check_args();
+}
+
+op::ReplaceSlice::ReplaceSlice(const std::shared_ptr<Node>& arg0,
+                               const std::shared_ptr<Node>& arg1,
+                               const Coordinate& lower_bounds,
+                               const Coordinate& upper_bounds)
+    : RequiresTensorViewArgs("ReplaceSlice", {arg0, arg1})
+    , m_lower_bounds(lower_bounds)
+    , m_upper_bounds(upper_bounds)
+    , m_step(Shape(lower_bounds.size(), 1))
+{
+    check_args();
+}
+
+void op::ReplaceSlice::check_args()
+{
+    auto arg0_tensor_view_type = get_inputs().at(0).get_tensor_view_type();
+    auto& arg0_shape = arg0_tensor_view_type->get_shape();
+    auto& arg0_element_type = arg0_tensor_view_type->get_element_type();
+
+    auto arg1_tensor_view_type = get_inputs().at(1).get_tensor_view_type();
+    auto& arg1_shape = arg1_tensor_view_type->get_shape();
+    auto& arg1_element_type = arg1_tensor_view_type->get_element_type();
+
+    if (arg0_shape.size() != arg1_shape.size())
+    {
+        throw ngraph_error("Replace-slice argument ranks do not match");
+    }
+
+    if (arg0_element_type != arg1_element_type)
+    {
+        throw ngraph_error("Element types for replace-slice arguments do not match");
+    }
+
+    if (m_lower_bounds.size() != arg0_shape.size())
+    {
+        throw ngraph_error(
+            "Number of lower bounds provided for slice does not match number of input axes");
+    }
+
+    if (m_upper_bounds.size() != arg0_shape.size())
+    {
+        throw ngraph_error(
+            "Number of upper bounds provided for slice does not match number of input axes");
+    }
+
+    if (m_step.size() != arg0_shape.size())
+    {
+        throw ngraph_error(
+            "Number of step axes provided for slice does not match number of input axes");
+    }
+
+    Shape slice_shape;
+
+    for (size_t i = 0; i < arg0_shape.size(); i++)
+    {
+        if (m_upper_bounds[i] > arg0_shape[i])
+        {
+            throw ngraph_error("Upper bound for slice is out of range");
+        }
+
+        if (m_lower_bounds[i] > m_upper_bounds[i])
+        {
+            throw ngraph_error("Lower bound for slice is greater than upper bound");
+        }
+
+        if (0 == m_step[i])
+        {
+            throw ngraph_error("Step distance for slice is zero");
+        }
+
+        size_t slice_axis_size = m_upper_bounds[i] - m_lower_bounds[i];
+        slice_axis_size =
+            slice_axis_size / m_step[i] + ((slice_axis_size % m_step[i] == 0) ? 0 : 1);
+        slice_shape.push_back(slice_axis_size);
+    }
+
+    if (arg1_shape != slice_shape)
+    {
+        throw ngraph_error("Shape of replacement tensor does not match slice shape");
+    }
+
+    set_value_type_checked(arg0_tensor_view_type);
+}
+
+void op::ReplaceSlice::generate_adjoints(autodiff::Adjoints& adjoints,
+                                         const std::shared_ptr<Node>& delta)
+{
+    auto x = get_inputs().at(0).get_output().get_node();
+    auto& y_input = get_inputs().at(1);
+    auto y = y_input.get_output().get_node();
+    auto& y_element_type = y_input.get_tensor_view_type()->get_element_type();
+    auto y_shape = y_input.get_tensor_view_type()->get_shape();
+
+    auto zeros_shaped_like_y = std::make_shared<op::Constant>(y_element_type, y_shape, "0");
+
+    adjoints.add_delta(x,
+                       std::make_shared<op::ReplaceSlice>(
+                           delta, zeros_shaped_like_y, m_lower_bounds, m_upper_bounds, m_step));
+    adjoints.add_delta(y,
+                       std::make_shared<op::Slice>(delta, m_lower_bounds, m_upper_bounds, m_step));
+}

src/ngraph/ops/replace_slice.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Takes two input tensors of identical rank, with the second tensor no larger than the first in any dimension, and returns a copy of
+        ///        the first input tensor with the specified slice overwritten by the second input tensor.
+        ///
+        /// ## Parameters
+        ///
+        /// |                | Description                                                                                                                                                                                  |
+        /// | -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+        /// | `lower_bounds` | The (inclusive) lower-bound coordinates \f$l_i\f$ for the tensor slice to be overwritten. For example, a lower-bound of \f$(1,2)\f$ means to start the slice at row 1 and column 2.          |
+        /// | `upper_bounds` | The (non-inclusive) upper-bound coordinates \f$u_i\f$ for the tensor slice to be overwritten. For example, an upper-bound of \f$(5,4)\f$ means to end the slice before row 4 and column 3.   |
+        /// | `step`         | The "step" or "stride" \f$s_i\f$ for the tensor slice to be overwritten. For example, a stride of \f$(1,3)\f$ means to take every row, and every third column (starting at the lower bound). |
+        ///
+        /// ## Inputs
+        ///
+        /// |        | Type                                                                           | Description                                                                                                                         |
+        /// | ------ | ------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------- |
+        /// | `arg0` | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$                                              | A tensor of any shape and element type.                                                                                             |
+        /// | `arg1` | \f$E[d'_1,\dots,d'_n]\f$ where \f$(d'_i = \lceil(u_i - l_i)\, /\, s_i\rceil\f$ | A tensor of the same element type and rank as `arg0`, whose shape is determined by the lower and upper slice bounds and slice step. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                                                                                                                                                                 |
+        /// | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+        /// | \f$E[d_1,\dots,d_n]\f$ | The tensor \f$T\f$ where \f$T[i_1,\dots,i_n] = \texttt{arg1}[j_1,\dots,j_n]\f$ if \f$j_1,\dots,j_n\f$ is in bounds for `arg1` and for all \f$m\f$, \f$i_m = l_m + j_m s_m\f$, otherwise \f$\texttt{arg0}[i_1,\dots,i_n]\f$. |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status                                          |
+        /// | ------- | ----------------------------------------------- |
+        /// | NGVM    | Implemented for scalars, matrices, and vectors. |
+        class ReplaceSlice : public RequiresTensorViewArgs
+        {
+        public:
+            /// \brief Constructs a tensor slice replacement operation.
+            ///
+            /// \param arg0 The tensor view to overwrite into.
+            /// \param arg1 The tensor view to write into `arg0`.
+            /// \param lower_bounds The axiswise lower bounds of the slice (inclusive).
+            /// \param upper_bounds The axiswise upper bounds of the slice (exclusive).
+            /// \param step The slicing step; for example, step of `{n,m}` means to take
+            ///             every nth row and every mth column of `arg0` as part of the
+            ///             slice to be replaced.
+            ReplaceSlice(const std::shared_ptr<Node>& arg0,
+                         const std::shared_ptr<Node>& arg1,
+                         const Coordinate& lower_bounds,
+                         const Coordinate& upper_bounds,
+                         const Shape& step);
+
+            /// \brief Constructs a tensor slice replacement operation with unit step; i.e., every element inside the bounding box will be overwritten.
+            ///
+            /// \param arg0 The tensor view to overwrite into.
+            /// \param arg1 The tensor view to write into `arg0`.
+            /// \param lower_bounds The axiswise lower bounds of the slice (inclusive).
+            /// \param upper_bounds The axiswise upper bounds of the slice (exclusive).
+            ReplaceSlice(const std::shared_ptr<Node>& arg0,
+                         const std::shared_ptr<Node>& arg1,
+                         const Coordinate& lower_bounds,
+                         const Coordinate& upper_bounds);
+
+            virtual std::shared_ptr<Node> copy_with_new_args(
+                const std::vector<std::shared_ptr<Node>>& new_args) const override
+            {
+                if (new_args.size() != 2)
+                    throw ngraph_error("Incorrect number of new arguments");
+                return std::make_shared<ReplaceSlice>(
+                    new_args.at(0), new_args.at(1), m_lower_bounds, m_upper_bounds, m_step);
+            }
+
+            /// \return The inclusive lower-bound coordinates.
+            const Coordinate& get_lower_bounds() const { return m_lower_bounds; }
+            /// \return The exclusive upper-bound coordinates.
+            const Coordinate& get_upper_bounds() const { return m_upper_bounds; }
+            /// \return The slicing step.
+            const Shape& get_step() const { return m_step; }
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+            void check_args();
+
+            const Coordinate m_lower_bounds;
+            const Coordinate m_upper_bounds;
+            const Shape m_step;
+        };
+    }
+}

src/ngraph/ops/select.cpp

@@ -15,6 +15,10 @@
 #include <memory>
 
 #include "ngraph/log.hpp"
+#include "ngraph/ops/constant.hpp"
+#include "ngraph/ops/convert.hpp"
+#include "ngraph/ops/multiply.hpp"
+#include "ngraph/ops/not.hpp"
 #include "ngraph/ops/select.hpp"
 
 using namespace std;
@@ -45,3 +49,18 @@ op::Select::Select(const std::shared_ptr<Node>& arg0,
 
     set_value_type_checked(arg1_tensor_type);
 }
+
+void ngraph::op::Select::generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta)
+{
+    auto p = get_inputs().at(0).get_output().get_node();
+    auto x = get_inputs().at(1).get_output().get_node();
+    auto y = get_inputs().at(2).get_output().get_node();
+
+    auto p_as_float = std::make_shared<op::Convert>(p, element::Float32::element_type());
+    auto not_p_as_float = std::make_shared<op::Convert>(std::make_shared<op::Not>(p),
+                                                        element::Float32::element_type());
+
+    adjoints.add_delta(x, delta * p_as_float);
+    adjoints.add_delta(y, delta * not_p_as_float);
+}

src/ngraph/ops/select.hpp

@@ -60,6 +60,10 @@ namespace ngraph
                     throw ngraph_error("Incorrect number of new arguments");
                 return std::make_shared<Select>(new_args.at(0), new_args.at(1), new_args.at(2));
             }
+
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
     }
 }

src/ngraph/ops/sin.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/sin.hpp"
+#include "ngraph/ops/cos.hpp"
+#include "ngraph/ops/multiply.hpp"
+
+void ngraph::op::Sin::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, delta * (std::make_shared<op::Cos>(x)));
+}

src/ngraph/ops/sin.hpp

@@ -57,6 +57,10 @@ namespace ngraph
                     throw ngraph_error("Incorrect number of new arguments");
                 return std::make_shared<Sin>(new_args.at(0));
             }
+
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
     }
 }

src/ngraph/ops/sinh.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/sinh.hpp"
+#include "ngraph/ops/cosh.hpp"
+#include "ngraph/ops/multiply.hpp"
+
+void ngraph::op::Sinh::generate_adjoints(autodiff::Adjoints& adjoints,
+                                         const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, delta * (std::make_shared<op::Cosh>(x)));
+}

src/ngraph/ops/sinh.hpp

@@ -57,6 +57,10 @@ namespace ngraph
                     throw ngraph_error("Incorrect number of new arguments");
                 return std::make_shared<Sinh>(new_args.at(0));
             }
+
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
     }
 }

src/ngraph/ops/slice.cpp

@@ -91,3 +91,10 @@ void op::Slice::check_args()
     set_value_type_checked(
         make_shared<TensorViewType>(arg_tensor_view_type->get_element_type(), result_shape));
 }
+
+void op::Slice::generate_adjoints(autodiff::Adjoints& adjoints, const std::shared_ptr<Node>& delta)
+{
+    auto x = get_inputs().at(0).get_output().get_node();
+
+    adjoints.add_delta_to_slice(x, delta, m_lower_bounds, m_upper_bounds, m_step);
+}

src/ngraph/ops/slice.hpp

@@ -92,6 +92,8 @@ namespace ngraph
             /// \return The slicing step.
             const Shape& get_step() const { return m_step; }
         protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
             void check_args();
 
             const Coordinate m_lower_bounds;

src/ngraph/ops/sqrt.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/sqrt.hpp"
+#include "ngraph/ops/add.hpp"
+#include "ngraph/ops/divide.hpp"
+
+void ngraph::op::Sqrt::generate_adjoints(autodiff::Adjoints& adjoints,
+                                         const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, delta / (shared_from_this() + shared_from_this()));
+}

src/ngraph/ops/sqrt.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/ops/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Elementwise square root operation.
+        ///
+        /// ## Inputs
+        ///
+        /// |       | Type                              | Description                                     |
+        /// | ----- | --------------------------------- | ----------------------------------------------- |
+        /// | `arg` | \f$N[d_1,\dots,d_n]~(n \geq 0)\f$ | A tensor of any shape and numeric element type. |
+        ///
+        /// ## Output
+        ///
+        /// | Type                   | Description                                                                           |
+        /// | ---------------------- | ------------------------------------------------------------------------------------- |
+        /// | \f$N[d_1,\dots,d_n]\f$ | The tensor \f$T\f$, where \f$T[i_1,\dots,i_n] = \sqrt{\texttt{arg}[i_1,\dots,i_n]}\f$ |
+        ///
+        /// ## Implementation Status
+        ///
+        /// | Backend | Status             |
+        /// | ------- | ------------------ |
+        /// | NGVM    | Fully implemented. |
+        class Sqrt : public UnaryElementwiseArithmetic
+        {
+        public:
+            /// \brief Constructs a square operation.
+            ///
+            /// \param arg Node that produces the input tensor.
+            Sqrt(const std::shared_ptr<Node>& arg)
+                : UnaryElementwiseArithmetic("Sqrt", arg)
+            {
+            }
+
+            virtual std::shared_ptr<Node> copy_with_new_args(
+                const std::vector<std::shared_ptr<Node>>& new_args) const override
+            {
+                if (new_args.size() != 1)
+                    throw ngraph_error("Incorrect number of new arguments");
+                return std::make_shared<Sqrt>(new_args.at(0));
+            }
+
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+        };
+    }
+}

src/ngraph/ops/sum.cpp

@@ -14,6 +14,7 @@
 
 #include "ngraph/ops/sum.hpp"
 #include "ngraph/function.hpp"
+#include "ngraph/ops/broadcast.hpp"
 
 using namespace std;
 using namespace ngraph;
@@ -52,3 +53,11 @@ op::Sum::Sum(const std::shared_ptr<Node>& arg, const AxisSet& reduction_axes)
     set_value_type_checked(
         make_shared<TensorViewType>(arg_tensor_view_type->get_element_type(), result_shape));
 }
+
+void op::Sum::generate_adjoints(autodiff::Adjoints& adjoints, const std::shared_ptr<Node>& delta)
+{
+    auto x = get_inputs().at(0).get_output().get_node();
+    auto& x_shape = get_inputs().at(0).get_tensor_view_type()->get_shape();
+
+    adjoints.add_delta(x, make_shared<op::Broadcast>(delta, x_shape, m_reduction_axes));
+}

src/ngraph/ops/sum.hpp

@@ -100,6 +100,9 @@ namespace ngraph
             /// \return The axis positions (0-based) to be eliminated through summation.
             const AxisSet& get_reduction_axes() const { return m_reduction_axes; }
         protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             AxisSet m_reduction_axes;
         };
     }

src/ngraph/ops/tan.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/tan.hpp"
+#include "ngraph/ops/cos.hpp"
+#include "ngraph/ops/divide.hpp"
+#include "ngraph/ops/multiply.hpp"
+
+void ngraph::op::Tan::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    auto c = std::make_shared<op::Cos>(x);
+
+    adjoints.add_delta(x, delta / (c * c));
+}

src/ngraph/ops/tan.hpp

@@ -57,6 +57,10 @@ namespace ngraph
                     throw ngraph_error("Incorrect number of new arguments");
                 return std::make_shared<Tan>(new_args.at(0));
             }
+
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
     }
 }

src/ngraph/ops/tanh.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/tanh.hpp"
+#include "ngraph/ops/cosh.hpp"
+#include "ngraph/ops/divide.hpp"
+#include "ngraph/ops/multiply.hpp"
+
+void ngraph::op::Tanh::generate_adjoints(autodiff::Adjoints& adjoints,
+                                         const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    auto c = std::make_shared<op::Cosh>(x);
+
+    adjoints.add_delta(x, delta / (c * c));
+}

src/ngraph/ops/tanh.hpp

@@ -57,6 +57,10 @@ namespace ngraph
                     throw ngraph_error("Incorrect number of new arguments");
                 return std::make_shared<Tanh>(new_args.at(0));
             }
+
+        protected:
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
         };
     }
 }

src/ngraph/runtime/cpu/emitter.cpp

@@ -28,6 +28,7 @@
 #include "ngraph/ops/function_call.hpp"
 #include "ngraph/ops/get_tuple_element.hpp"
 #include "ngraph/ops/reduce.hpp"
+#include "ngraph/ops/replace_slice.hpp"
 #include "ngraph/ops/reshape.hpp"
 #include "ngraph/ops/slice.hpp"
 #include "ngraph/ops/sum.hpp"
@@ -1461,6 +1462,86 @@ void Emitter::EmitPower(const ngraph::Node* n,
     TU << "}\n";
 }
 
+void Emitter::EmitReplaceSlice(const ngraph::Node* n,
+                               const std::vector<TensorViewInfo>& inputs,
+                               const std::vector<TensorViewInfo>& outputs)
+{
+    auto replace_slice = static_cast<const op::Slice*>(n);
+
+    for (auto d : replace_slice->get_step())
+    {
+        if (1 != d)
+        {
+            throw ngraph_error("Replace-slice does not support non-unit step yet");
+        }
+    }
+
+    auto arg0_type = replace_slice->get_arguments().at(0)->get_value_type();
+    auto arg0_tensor_view_type = dynamic_pointer_cast<const TensorViewType>(arg0_type);
+    assert(arg0_tensor_view_type);
+    auto arg0_shape = arg0_tensor_view_type->get_shape();
+    auto arg0_rank = arg0_shape.size();
+
+    auto arg1_type = replace_slice->get_arguments().at(1)->get_value_type();
+    auto arg1_tensor_view_type = dynamic_pointer_cast<const TensorViewType>(arg1_type);
+    assert(arg1_tensor_view_type);
+    auto arg1_shape = arg1_tensor_view_type->get_shape();
+    auto arg1_rank = arg1_shape.size();
+
+    auto& lower_bounds = replace_slice->get_lower_bounds();
+    auto& upper_bounds = replace_slice->get_upper_bounds();
+
+    // Scalar slice is necessarily just a copy.
+    if (arg0_rank == 0)
+    {
+        TU << "{   // " << n->get_name() << " 1\n";
+        TU.indent++;
+        TU << "memcpy(" << outputs[0].get_tensor().get_name() << ", "
+           << inputs[1].get_tensor().get_name() << ", "
+           << outputs[0].get_tensor_view_layout()->get_size() *
+                  outputs[0].get_tensor_view_layout()->get_element_type().size()
+           << ");\n";
+        TU.indent--;
+        TU << "}\n";
+    }
+    else if (arg0_rank == 1)
+    {
+        TU << "{   // " << n->get_name() << " 2\n";
+        TU.indent++;
+        TU << "" << emit_vector(outputs[0]) << " =\n"
+           << "    " << emit_vector(inputs[0]) << ";\n"
+           << "" << emit_vector(outputs[0]) << ".segment(\n"
+           << "    " << to_string(lower_bounds[0]) << ", "
+           << to_string(upper_bounds[0] - lower_bounds[0]) << ") =\n"
+           << "    " << emit_vector(inputs[1]) << ";\n";
+        TU.indent--;
+        TU << "}\n";
+    }
+    else if (arg0_rank == 2)
+    {
+        auto arg0_layout = inputs[0].get_layout<DenseTensorViewLayout>();
+        auto out_layout = outputs[0].get_layout<DenseTensorViewLayout>();
+
+        TU << "{   // " << n->get_name() << " 3\n";
+        TU.indent++;
+        TU << "" << emit_matrix(outputs[0]) << " =\n"
+           << "    " << emit_matrix(inputs[0]) << ";\n"
+           << "" << emit_matrix(outputs[0]) << ".block(\n"
+           << "        " << to_string(lower_bounds[0]) << ",\n"
+           << "        " << to_string(lower_bounds[1]) << ",\n"
+           << "        " << to_string(upper_bounds[0] - lower_bounds[0]) << ",\n"
+           << "        " << to_string(upper_bounds[1] - lower_bounds[1]) << ") =\n"
+           << "    " << emit_matrix(inputs[1]) << ";\n";
+        TU.indent--;
+        TU << "}\n";
+    }
+    // Other cases (reordering of axes for tensors with rank>2) are not handled yet.
+    else
+    {
+        throw ngraph_error("Replace-slice is not implemented yet for tensors with rank>2");
+    }
+}
+
 //------------------------------------------------------------------------------------------------
 // Utility methods
 //------------------------------------------------------------------------------------------------

src/ngraph/runtime/cpu/emitter.hpp

@@ -94,6 +94,7 @@ namespace ngraph
                 void EMITTER_DECL(EmitAcos);
                 void EMITTER_DECL(EmitAtan);
                 void EMITTER_DECL(EmitPower);
+                void EMITTER_DECL(EmitReplaceSlice);
 
             private:
                 void generate_call(const std::vector<TensorViewInfo>& inputs,

src/ngraph/runtime/cpu/external_function.cpp

@@ -58,6 +58,7 @@
 #include "ngraph/ops/not_equal.hpp"
 #include "ngraph/ops/power.hpp"
 #include "ngraph/ops/reduce.hpp"
+#include "ngraph/ops/replace_slice.hpp"
 #include "ngraph/ops/reshape.hpp"
 #include "ngraph/ops/select.hpp"
 #include "ngraph/ops/sign.hpp"
@@ -156,6 +157,7 @@ static const OpMap dispatcher{
     {TI(ngraph::op::Asin), &Emitter::EmitAsin},
     {TI(ngraph::op::Acos), &Emitter::EmitAcos},
     {TI(ngraph::op::Atan), &Emitter::EmitAtan},
+    {TI(ngraph::op::ReplaceSlice), &Emitter::EmitReplaceSlice},
 };
 
 ExternalFunction::ExternalFunction(const std::shared_ptr<ngraph::Function>& function,

src/ngraph/runtime/ngvm/eigen/ceiling.hpp

+
+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/runtime/ngvm/call_frame.hpp"
+#include "ngraph/runtime/ngvm/eigen/utils.hpp"
+#include "ngraph/runtime/ngvm/instruction.hpp"
+#include "ngraph/runtime/tensor_view.hpp"
+
+namespace ngraph
+{
+    namespace runtime
+    {
+        namespace ngvm
+        {
+            namespace eigen
+            {
+                template <typename ET>
+                class CeilingInstruction : public Instruction
+                {
+                public:
+                    CeilingInstruction(TensorViewInfo arg, TensorViewInfo out)
+                        : m_arg(arg)
+                        , m_out(out)
+                    {
+                    }
+
+                    virtual void execute(CallFrame& call_frame) const override
+                    {
+                        EigenArray1d<ET, fmt::V>(call_frame, m_out) =
+                            EigenArray1d<ET, fmt::V>(call_frame, m_arg).ceil();
+                    }
+
+                protected:
+                    TensorViewInfo m_arg;
+                    TensorViewInfo m_out;
+                };
+            }
+        }
+    }
+}

src/ngraph/runtime/ngvm/eigen/floor.hpp

+
+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/runtime/ngvm/call_frame.hpp"
+#include "ngraph/runtime/ngvm/eigen/utils.hpp"
+#include "ngraph/runtime/ngvm/instruction.hpp"
+#include "ngraph/runtime/tensor_view.hpp"
+
+namespace ngraph
+{
+    namespace runtime
+    {
+        namespace ngvm
+        {
+            namespace eigen
+            {
+                template <typename ET>
+                class FloorInstruction : public Instruction
+                {
+                public:
+                    FloorInstruction(TensorViewInfo arg, TensorViewInfo out)
+                        : m_arg(arg)
+                        , m_out(out)
+                    {
+                    }
+
+                    virtual void execute(CallFrame& call_frame) const override
+                    {
+                        EigenArray1d<ET, fmt::V>(call_frame, m_out) =
+                            EigenArray1d<ET, fmt::V>(call_frame, m_arg).floor();
+                    }
+
+                protected:
+                    TensorViewInfo m_arg;
+                    TensorViewInfo m_out;
+                };
+            }
+        }
+    }
+}

src/ngraph/runtime/ngvm/eigen/not.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/runtime/ngvm/call_frame.hpp"
+#include "ngraph/runtime/ngvm/eigen/utils.hpp"
+#include "ngraph/runtime/ngvm/instruction.hpp"
+#include "ngraph/runtime/tensor_view.hpp"
+
+namespace ngraph
+{
+    namespace runtime
+    {
+        namespace ngvm
+        {
+            namespace eigen
+            {
+                class NotInstruction : public Instruction
+                {
+                public:
+                    NotInstruction(TensorViewInfo arg, TensorViewInfo out)
+                        : m_arg(arg)
+                        , m_out(out)
+                    {
+                    }
+
+                    virtual void execute(CallFrame& call_frame) const override
+                    {
+                        // This is a bit frustrating. We have to cast the Eigen
+                        // matrix to a real bool, negate that, then cast that
+                        // back to our storage representation (ultimately char).
+                        EigenArray1d<element::Bool>(call_frame, m_out) =
+                            (!(EigenArray1d<element::Bool>(call_frame, m_arg)
+                                   .template cast<bool>()))
+                                .template cast<element::Bool::type>();
+                    }
+
+                protected:
+                    TensorViewInfo m_arg;
+                    TensorViewInfo m_out;
+                };
+            }
+        }
+    }
+}

src/ngraph/runtime/ngvm/eigen/replace_matrix_slice.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/runtime/ngvm/call_frame.hpp"
+#include "ngraph/runtime/ngvm/eigen/utils.hpp"
+#include "ngraph/runtime/ngvm/instruction.hpp"
+#include "ngraph/runtime/tensor_view.hpp"
+
+namespace ngraph
+{
+    namespace runtime
+    {
+        namespace ngvm
+        {
+            namespace eigen
+            {
+                template <typename ET>
+                class ReplaceMatrixSliceInstruction : public Instruction
+                {
+                public:
+                    ReplaceMatrixSliceInstruction(const TensorViewInfo& arg0,
+                                                  const TensorViewInfo& arg1,
+                                                  const TensorViewInfo& out,
+                                                  size_t lower_row,
+                                                  size_t lower_col,
+                                                  size_t upper_row,
+                                                  size_t upper_col)
+                        : m_arg0(arg0)
+                        , m_arg1(arg1)
+                        , m_out(out)
+                        , m_lower_row(lower_row)
+                        , m_lower_col(lower_col)
+                        , m_upper_row(upper_row)
+                        , m_upper_col(upper_col)
+                    {
+                    }
+
+                    virtual void execute(CallFrame& call_frame) const override
+                    {
+                        EigenMatrix<ET>(call_frame, m_out) = EigenMatrix<ET>(call_frame, m_arg0);
+                        EigenMatrix<ET>(call_frame, m_out)
+                            .block(m_lower_row,
+                                   m_lower_col,
+                                   m_upper_row - m_lower_row,
+                                   m_upper_col - m_lower_col) = EigenMatrix<ET>(call_frame, m_arg1);
+                    }
+
+                protected:
+                    TensorViewInfo m_arg0;
+                    TensorViewInfo m_arg1;
+                    TensorViewInfo m_out;
+                    size_t m_lower_row;
+                    size_t m_lower_col;
+                    size_t m_upper_row;
+                    size_t m_upper_col;
+                };
+            }
+        }
+    }
+}

src/ngraph/runtime/ngvm/eigen/replace_vector_slice.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/runtime/ngvm/call_frame.hpp"
+#include "ngraph/runtime/ngvm/eigen/utils.hpp"
+#include "ngraph/runtime/ngvm/instruction.hpp"
+#include "ngraph/runtime/tensor_view.hpp"
+
+namespace ngraph
+{
+    namespace runtime
+    {
+        namespace ngvm
+        {
+            namespace eigen
+            {
+                template <typename ET>
+                class ReplaceVectorSliceInstruction : public Instruction
+                {
+                public:
+                    ReplaceVectorSliceInstruction(const TensorViewInfo& arg0,
+                                                  const TensorViewInfo& arg1,
+                                                  const TensorViewInfo& out,
+                                                  size_t lower,
+                                                  size_t upper)
+                        : m_arg0(arg0)
+                        , m_arg1(arg1)
+                        , m_out(out)
+                        , m_lower(lower)
+                        , m_upper(upper)
+                    {
+                    }
+
+                    virtual void execute(CallFrame& call_frame) const override
+                    {
+                        EigenVector<ET>(call_frame, m_out) = EigenVector<ET>(call_frame, m_arg0);
+                        EigenVector<ET>(call_frame, m_out).segment(m_lower, m_upper - m_lower) =
+                            EigenVector<ET>(call_frame, m_arg1);
+                    }
+
+                protected:
+                    TensorViewInfo m_arg0;
+                    TensorViewInfo m_arg1;
+                    TensorViewInfo m_out;
+                    size_t m_lower;
+                    size_t m_upper;
+                };
+            }
+        }
+    }
+}

src/ngraph/runtime/ngvm/eigen/sqrt.hpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#pragma once
+
+#include "ngraph/runtime/ngvm/call_frame.hpp"
+#include "ngraph/runtime/ngvm/eigen/utils.hpp"
+#include "ngraph/runtime/ngvm/instruction.hpp"
+#include "ngraph/runtime/tensor_view.hpp"
+#include "ngraph/runtime/tensor_view_info.hpp"
+
+namespace ngraph
+{
+    namespace runtime
+    {
+        namespace ngvm
+        {
+            namespace eigen
+            {
+                template <typename ET>
+                class SqrtInstruction : public Instruction
+                {
+                public:
+                    SqrtInstruction(const TensorViewInfo& arg, const TensorViewInfo& out)
+                        : m_arg(arg)
+                        , m_out(out)
+                    {
+                    }
+
+                    virtual void execute(CallFrame& call_frame) const override
+                    {
+                        EigenArray1d<ET>(call_frame, m_out) =
+                            Eigen::sqrt(EigenArray1d<ET>(call_frame, m_arg));
+                    }
+
+                protected:
+                    TensorViewInfo m_arg;
+                    TensorViewInfo m_out;
+                };
+            }
+        }
+    }
+}

test/backend_test.in.cpp

@@ -253,6 +253,27 @@ TEST(${BACKEND_NAME}, abs)
     ASSERT_EQ((vector<float>{1, 2, 0, 4.8f}), result->get_vector<float>());
 }
 
+TEST(${BACKEND_NAME}, ceiling)
+{
+    auto shape = Shape{2, 2};
+    auto A = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+    auto result_type = make_shared<TensorViewType>(element::Float32::element_type(), shape);
+    auto f = make_shared<Function>(make_shared<op::Ceiling>(A), result_type, op::Parameters{A});
+
+    auto manager = runtime::Manager::get("NGVM");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::Float32::element_type(), shape);
+    copy_data(a, vector<float>{-2.5f, -2.0f, 0.3f, 4.8f});
+    auto result = backend->make_primary_tensor_view(element::Float32::element_type(), shape);
+
+    cf->call({a}, {result});
+    ASSERT_EQ((vector<float>{-2.0f, -2.0f, 1.0f, 5.0f}), result->get_vector<float>());
+}
+
 TEST(${BACKEND_NAME}, concat_matrix_colwise)
 {
     auto shape_a = Shape{2, 2};
@@ -435,6 +456,27 @@ TEST(${BACKEND_NAME}, equal)
     ASSERT_EQ((vector<char>{1, 1, 0, 0, 0, 1, 1, 0}), result->get_vector<char>());
 }
 
+TEST(${BACKEND_NAME}, floor)
+{
+    auto shape = Shape{2, 2};
+    auto A = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+    auto result_type = make_shared<TensorViewType>(element::Float32::element_type(), shape);
+    auto f = make_shared<Function>(make_shared<op::Floor>(A), result_type, op::Parameters{A});
+
+    auto manager = runtime::Manager::get("NGVM");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::Float32::element_type(), shape);
+    copy_data(a, vector<float>{-2.5f, -2.0f, 0.3f, 4.8f});
+    auto result = backend->make_primary_tensor_view(element::Float32::element_type(), shape);
+
+    cf->call({a}, {result});
+    ASSERT_EQ((vector<float>{-3.0f, -2.0f, 0.0f, 4.0f}), result->get_vector<float>());
+}
+
 TEST(${BACKEND_NAME}, dot_0_0)
 {
     auto shape = Shape{0};
@@ -2354,7 +2396,7 @@ TEST(${BACKEND_NAME}, slice_matrix)
     auto shape_a = Shape{4, 4};
     auto A = make_shared<op::Parameter>(
         make_shared<TensorViewType>(element::Float32::element_type(), shape_a));
-    auto shape_r = Shape{2, 3};
+    auto shape_r = Shape{3, 2};
     auto rt = make_shared<TensorViewType>(element::Float32::element_type(), shape_r);
     auto r = make_shared<op::Slice>(A, Coordinate{0, 1}, Coordinate{3, 3});
     auto f = make_shared<Function>(r, rt, op::Parameters{A});
@@ -2759,3 +2801,114 @@ TEST(${BACKEND_NAME}, constant_equality_bool)
     cf->call({}, {result});
     ASSERT_EQ((vector<char>{true, false, true, false}), result->get_vector<char>());
 }
+
+TEST(${BACKEND_NAME}, sqrt)
+{
+    auto shape = Shape{2, 3};
+    auto A = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+    auto result_type = make_shared<TensorViewType>(element::Float32::element_type(), shape);
+    auto f = make_shared<Function>(make_shared<op::Sqrt>(A), result_type, op::Parameters{A});
+
+    auto manager = runtime::Manager::get("NGVM");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::Float32::element_type(), shape);
+    copy_data(a, vector<float>{16, 4, 81, 100, 10000, 0});
+    auto result = backend->make_primary_tensor_view(element::Float32::element_type(), shape);
+
+    cf->call({a}, {result});
+    ASSERT_EQ((vector<float>{4, 2, 9, 10, 100, 0}), result->get_vector<float>());
+}
+
+TEST(${BACKEND_NAME}, replace_slice_scalar)
+{
+    auto shape_a = Shape{};
+    auto A = make_shared<op::Parameter>(
+        make_shared<TensorViewType>(element::Float32::element_type(), shape_a));
+    auto shape_b = Shape{};
+    auto B = make_shared<op::Parameter>(
+        make_shared<TensorViewType>(element::Float32::element_type(), shape_b));
+    auto shape_r = Shape{};
+    auto rt = make_shared<TensorViewType>(element::Float32::element_type(), shape_r);
+    auto r = make_shared<op::ReplaceSlice>(A, B, Coordinate{}, Coordinate{});
+    auto f = make_shared<Function>(r, rt, op::Parameters{A, B});
+
+    auto manager = runtime::Manager::get("${BACKEND_NAME}");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::Float32::element_type(), shape_a);
+    copy_data(a, vector<float>{312});
+    auto b = backend->make_primary_tensor_view(element::Float32::element_type(), shape_b);
+    copy_data(b, vector<float>{808});
+    auto result = backend->make_primary_tensor_view(element::Float32::element_type(), shape_r);
+
+    cf->call({a, b}, {result});
+    ASSERT_EQ((vector<float>{808}), result->get_vector<float>());
+}
+
+TEST(${BACKEND_NAME}, replace_slice_matrix)
+{
+    auto shape_a = Shape{4, 4};
+    auto A = make_shared<op::Parameter>(
+        make_shared<TensorViewType>(element::Float32::element_type(), shape_a));
+    auto shape_b = Shape{3, 2};
+    auto B = make_shared<op::Parameter>(
+        make_shared<TensorViewType>(element::Float32::element_type(), shape_b));
+    auto shape_r = Shape{4, 4};
+    auto rt = make_shared<TensorViewType>(element::Float32::element_type(), shape_r);
+    auto r = make_shared<op::ReplaceSlice>(A, B, Coordinate{0, 1}, Coordinate{3, 3});
+    auto f = make_shared<Function>(r, rt, op::Parameters{A, B});
+
+    auto manager = runtime::Manager::get("${BACKEND_NAME}");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::Float32::element_type(), shape_a);
+    copy_data(a, vector<float>{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
+    auto b = backend->make_primary_tensor_view(element::Float32::element_type(), shape_b);
+    copy_data(b, vector<float>{102, 103, 106, 107, 110, 111});
+    auto result = backend->make_primary_tensor_view(element::Float32::element_type(), shape_r);
+
+    cf->call({a, b}, {result});
+    ASSERT_EQ((vector<float>{1, 102, 103, 4, 5, 106, 107, 8, 9, 110, 111, 12, 13, 14, 15, 16}),
+              result->get_vector<float>());
+}
+
+TEST(${BACKEND_NAME}, replace_slice_vector)
+{
+    auto shape_a = Shape{16};
+    auto A = make_shared<op::Parameter>(
+        make_shared<TensorViewType>(element::Float32::element_type(), shape_a));
+    auto shape_b = Shape{12};
+    auto B = make_shared<op::Parameter>(
+        make_shared<TensorViewType>(element::Float32::element_type(), shape_b));
+    auto shape_r = Shape{16};
+    auto rt = make_shared<TensorViewType>(element::Float32::element_type(), shape_r);
+    auto r = make_shared<op::ReplaceSlice>(A, B, Coordinate{2}, Coordinate{14});
+    auto f = make_shared<Function>(r, rt, op::Parameters{A, B});
+
+    auto manager = runtime::Manager::get("${BACKEND_NAME}");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::Float32::element_type(), shape_a);
+    copy_data(a, vector<float>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15});
+    auto b = backend->make_primary_tensor_view(element::Float32::element_type(), shape_b);
+    copy_data(b, vector<float>{102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113});
+    auto result = backend->make_primary_tensor_view(element::Float32::element_type(), shape_r);
+
+    cf->call({a, b}, {result});
+    ASSERT_EQ(
+        (vector<float>{0, 1, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 14, 15}),
+        result->get_vector<float>());
+}

test/util/autodiff/backprop_derivative.hpp

@@ -44,7 +44,8 @@ namespace ngraph
             backprop_derivative(const std::shared_ptr<runtime::Manager>& manager,
                                 const std::shared_ptr<runtime::Backend>& backend,
                                 const std::shared_ptr<Function>& f,
-                                const std::vector<std::shared_ptr<runtime::TensorView>>& args)
+                                const std::vector<std::shared_ptr<runtime::TensorView>>& args,
+                                const std::vector<std::shared_ptr<op::Parameter>>& indep_params)
         {
             auto y = f->get_result();
             Shape y_shape =
@@ -57,7 +58,8 @@ namespace ngraph
             std::vector<std::shared_ptr<Node>> deriv_nodes;
             std::vector<std::shared_ptr<runtime::TensorView>> bprops;
             std::vector<std::shared_ptr<runtime::TensorView>> results;
-            for (auto param : params)
+
+            for (auto param : indep_params)
             {
                 Shape s = y_shape;
                 auto param_shape =

test/util/autodiff/numeric_derivative.hpp

@@ -32,6 +32,7 @@ namespace ngraph
         /// @param f A function
         /// @param args Values for the arguments (the independent variables)
         /// @param delta increment for the variables
+        /// @param indep_params parameters with respect to which to compute derivatives
         /// @returns vector of dy/dvar, where each dy/dvar's shape is concat(y.shape(), var.shape())
         template <typename T>
         std::vector<std::shared_ptr<runtime::TensorView>>
@@ -39,7 +40,8 @@ namespace ngraph
                                const std::shared_ptr<runtime::Backend>& backend,
                                const std::shared_ptr<Function>& f,
                                const std::vector<std::shared_ptr<runtime::TensorView>>& args,
-                               T delta)
+                               T delta,
+                               const std::vector<std::shared_ptr<op::Parameter>>& indep_params)
         {
             auto y = f->get_result();
 
@@ -50,7 +52,8 @@ namespace ngraph
 
             // Results for each derivative, shape Y|X_i
             std::vector<std::shared_ptr<runtime::TensorView>> results;
-            for (auto param : params)
+
+            for (auto param : indep_params)
             {
                 Shape s = y_shape;
                 auto param_shape =
@@ -75,30 +78,38 @@ namespace ngraph
             // Assuming vars, y, and results are row-major
 
             T inv_delta = 1 / delta;
+
+            size_t pos = 0;
+
             for (size_t i = 0; i < args.size(); ++i)
             {
-                auto arg = args[i];
-                auto res = results[i]->get_vector<T>();
-                auto vec = arg->get_vector<T>();
-                for (size_t j = 0; j < vec.size(); j++)
+                if (std::find(indep_params.begin(), indep_params.end(), params[i]) !=
+                    indep_params.end())
                 {
-                    auto old_val = vec[j];
-                    vec[j] += delta;
-                    arg->write(vec);
-                    cf->tensor_call(args, {inc_y});
-                    auto inc_vec = inc_y->template get_vector<T>();
-                    vec[j] = old_val;
-                    arg->write(vec);
-                    size_t res_k = j;
-                    for (size_t k = 0; k < inc_vec.size(); k++)
+                    auto arg = args[i];
+                    auto res = results[pos]->get_vector<T>();
+                    auto vec = arg->get_vector<T>();
+                    for (size_t j = 0; j < vec.size(); j++)
                     {
-                        auto y1 = inc_vec[k];
-                        auto y0 = ref_vec[k];
-                        res[res_k] = inv_delta * (y1 - y0);
-                        res_k += vec.size();
+                        auto old_val = vec[j];
+                        vec[j] += delta;
+                        arg->write(vec);
+                        cf->tensor_call(args, {inc_y});
+                        auto inc_vec = inc_y->template get_vector<T>();
+                        vec[j] = old_val;
+                        arg->write(vec);
+                        size_t res_k = j;
+                        for (size_t k = 0; k < inc_vec.size(); k++)
+                        {
+                            auto y1 = inc_vec[k];
+                            auto y0 = ref_vec[k];
+                            res[res_k] = inv_delta * (y1 - y0);
+                            res_k += vec.size();
+                        }
                     }
+                    results[pos]->write(res);
+                    pos++;
                 }
-                results[i]->write(res);
             }
             return results;
         }