More operations

src/ngraph/CMakeLists.txt

@@ -35,16 +35,21 @@ set (SRC
     ops/concatenate.cpp
     ops/constant.cpp
     ops/convert.cpp
+    ops/divide.cpp
     ops/dot.cpp
+    ops/exp.cpp
     ops/function_call.cpp
     ops/get_tuple_element.cpp
+    ops/log.cpp
     ops/multiply.cpp
+    ops/negative.cpp
     ops/op.cpp
     ops/parameter.cpp
     ops/reduce.cpp
     ops/reshape.cpp
     ops/select.cpp
     ops/slice.cpp
+    ops/subtract.cpp
     ops/sum.cpp
     ops/tuple.cpp
     ops/unary_elementwise_arithmetic.cpp

src/ngraph/ops/divide.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/divide.hpp"
+#include "ngraph/ops/multiply.hpp"
+#include "ngraph/ops/negative.hpp"
+
+void ngraph::op::Divide::generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+    auto y = m_arguments[1];
+
+    adjoints.add_delta(x, delta * shared_from_this() / x);
+    adjoints.add_delta(y, -delta * shared_from_this() / y);
+}

src/ngraph/ops/divide.hpp

@@ -52,6 +52,9 @@ namespace ngraph
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Divide"; }
         };
     }

src/ngraph/ops/exp.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/exp.hpp"
+#include "ngraph/ops/multiply.hpp"
+
+void ngraph::op::Exp::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, delta * shared_from_this());
+}

src/ngraph/ops/exp.hpp

@@ -50,6 +50,9 @@ namespace ngraph
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Exp"; }
         };
     }

src/ngraph/ops/log.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/log.hpp"
+#include "ngraph/ops/divide.hpp"
+
+void ngraph::op::Log::generate_adjoints(autodiff::Adjoints& adjoints,
+                                        const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, delta / x);
+}

src/ngraph/ops/log.hpp

@@ -50,6 +50,9 @@ namespace ngraph
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Log"; }
         };
     }

src/ngraph/ops/negative.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/negative.hpp"
+
+void ngraph::op::Negative::generate_adjoints(autodiff::Adjoints& adjoints,
+                                             const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+
+    adjoints.add_delta(x, -delta);
+}

src/ngraph/ops/negative.hpp

@@ -50,6 +50,9 @@ namespace ngraph
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Negative"; }
         };
     }

src/ngraph/ops/subtract.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include "ngraph/ops/subtract.hpp"
+#include "ngraph/ops/negative.hpp"
+
+void ngraph::op::Subtract::generate_adjoints(autodiff::Adjoints& adjoints,
+                                             const std::shared_ptr<Node>& delta)
+{
+    auto x = m_arguments[0];
+    auto y = m_arguments[1];
+
+    adjoints.add_delta(x, delta);
+    adjoints.add_delta(y, -delta);
+}

src/ngraph/ops/subtract.hpp

@@ -52,6 +52,9 @@ namespace ngraph
             {
             }
 
+            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
+                                           const std::shared_ptr<Node>& delta) override;
+
             virtual std::string description() const override { return "Subtract"; }
         };
     }

test/autodiff.cpp

@@ -45,7 +45,7 @@ bool autodiff_numeric_compare(
     return test::all_close(results_num, results_sym, .01f, .01f);
 }
 
-TEST(backwards, parameter)
+TEST(backwards, add)
 {
     auto manager = runtime::Manager::get("NGVM");
     auto backend = manager->allocate_backend();
@@ -53,42 +53,81 @@ TEST(backwards, parameter)
     test::Uniform<element::Float32> rng(-1.0f, 1.0f);
     auto shape = Shape{2, 3};
     auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto x1 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
     auto make_graph = [shape]() {
         auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
-        return make_shared<Function>(X0, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0});
+        auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(
+            X0 + X1, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
     };
-    EXPECT_TRUE(
-        autodiff_numeric_compare<element::Float32>(manager, backend, make_graph, {x0}, .01f, .01f));
 
-    auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
-    auto Y = X0;
-    auto C = make_shared<op::Parameter>(element::Float32::element_type(), shape);
-    auto DYDX0 = Y->backprop_node(X0, C);
-    ASSERT_EQ(DYDX0, C);
+    auto results_num = autodiff::numeric_derivative<element::Float32>(
+        manager, backend, make_graph(), {x0, x1}, .001f);
+    auto results_sym =
+        autodiff::backprop_derivative<element::Float32>(manager, backend, make_graph(), {x0, x1});
+    EXPECT_TRUE(test::all_close(results_num, results_sym, .01f, .01f));
 }
 
-TEST(backwards, add)
+TEST(backwards, divide)
 {
     auto manager = runtime::Manager::get("NGVM");
     auto backend = manager->allocate_backend();
 
     test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    test::Uniform<element::Float32> rng1(1.0f, 2.0f);
+    test::Uniform<element::Float32> rng2(-2.0f, -1.0f);
     auto shape = Shape{2, 3};
     auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
-    auto x1 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto x1 = rng1.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto x2 = rng2.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
 
     auto make_graph = [shape]() {
         auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
         auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
         return make_shared<Function>(
-            X0 + X1, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
+            X0 / X1, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
     };
+    EXPECT_TRUE(autodiff_numeric_compare<element::Float32>(
+        manager, backend, make_graph, {x0, x1}, .01f, .01f));
+    EXPECT_TRUE(autodiff_numeric_compare<element::Float32>(
+        manager, backend, make_graph, {x0, x2}, .01f, .01f));
+}
 
-    auto results_num = autodiff::numeric_derivative<element::Float32>(
-        manager, backend, make_graph(), {x0, x1}, .001f);
-    auto results_sym =
-        autodiff::backprop_derivative<element::Float32>(manager, backend, make_graph(), {x0, x1});
-    EXPECT_TRUE(test::all_close(results_num, results_sym, .01f, .01f));
+TEST(backwards, exp)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(
+            make_shared<op::Exp>(X0), nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0});
+    };
+    EXPECT_TRUE(
+        autodiff_numeric_compare<element::Float32>(manager, backend, make_graph, {x0}, .01f, .01f));
+}
+
+TEST(backwards, log)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(1.0f, 2.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(
+            make_shared<op::Log>(X0), nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0});
+    };
+    EXPECT_TRUE(
+        autodiff_numeric_compare<element::Float32>(manager, backend, make_graph, {x0}, .01f, .01f));
 }
 
 TEST(backwards, multiply)
@@ -110,3 +149,66 @@ TEST(backwards, multiply)
     EXPECT_TRUE(autodiff_numeric_compare<element::Float32>(
         manager, backend, make_graph, {x0, x1}, .01f, .01f));
 }
+
+TEST(backwards, negative)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(-X0, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0});
+    };
+    EXPECT_TRUE(
+        autodiff_numeric_compare<element::Float32>(manager, backend, make_graph, {x0}, .01f, .01f));
+}
+
+TEST(backwards, parameter)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(X0, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0});
+    };
+    EXPECT_TRUE(
+        autodiff_numeric_compare<element::Float32>(manager, backend, make_graph, {x0}, .01f, .01f));
+
+    auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+    auto Y = X0;
+    auto C = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+    auto DYDX0 = Y->backprop_node(X0, C);
+    ASSERT_EQ(DYDX0, C);
+}
+
+TEST(backwards, subtract)
+{
+    auto manager = runtime::Manager::get("NGVM");
+    auto backend = manager->allocate_backend();
+
+    test::Uniform<element::Float32> rng(-1.0f, 1.0f);
+    auto shape = Shape{2, 3};
+    auto x0 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+    auto x1 = rng.initialize(backend->make_parameterized_tensor_view<element::Float32>(shape));
+
+    auto make_graph = [shape]() {
+        auto X0 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        auto X1 = make_shared<op::Parameter>(element::Float32::element_type(), shape);
+        return make_shared<Function>(
+            X0 - X1, nullptr, std::vector<std::shared_ptr<op::Parameter>>{X0, X1});
+    };
+
+    auto results_num = autodiff::numeric_derivative<element::Float32>(
+        manager, backend, make_graph(), {x0, x1}, .001f);
+    auto results_sym =
+        autodiff::backprop_derivative<element::Float32>(manager, backend, make_graph(), {x0, x1});
+    EXPECT_TRUE(test::all_close(results_num, results_sym, .01f, .01f));
+}