Reduce Ops in the Builder (#252)

* First compiling pass, doesn't link tests, looks like templates aren't properly initialized?

* Make reduce inline, compiles, sum passes test

* Initial working reduce op builder

* Add Std dev, Variance, L2Norm. Tests fail because Power isn't actually implemented in ngraph++ yet

* Add Doc Strings, slight refactor

* fix tests after master merge

* Fix style issues raised in PR review

* Switch to Sum-based ops instead of Reduce-based ops for simpler autodiff.

* Add requested TODO comments

* Use all_close for tests

* Remove product op

src/ngraph/CMakeLists.txt

@@ -13,6 +13,7 @@
 
 set (SRC
     autodiff/adjoints.cpp
+    builder/reduce_ops.cpp
     descriptor/input.cpp
     descriptor/layout/dense_tensor_view_layout.cpp
     descriptor/layout/tensor_view_layout.cpp

src/ngraph/builder/reduce_ops.cpp

+/*
+ Copyright 2017 Nervana Systems Inc.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#include "ngraph/builder/reduce_ops.hpp"
+#include "ngraph/ops/add.hpp"
+#include "ngraph/ops/divide.hpp"
+#include "ngraph/ops/multiply.hpp"
+#include "ngraph/ops/power.hpp"
+#include "ngraph/ops/subtract.hpp"
+#include "ngraph/ops/sum.hpp"
+
+namespace ngraph
+{
+    namespace builder
+    {
+        size_t get_num_elements(const Shape& shape, const AxisSet& reduction_axes)
+        {
+            size_t N = 1;
+            for (auto a : reduction_axes)
+            {
+                N *= shape[a];
+            }
+            return N;
+        }
+
+        std::shared_ptr<Node> l2_norm(const std::shared_ptr<Node>& node,
+                                      const AxisSet& reduction_axes)
+        {
+            const auto& et = node->get_element_type();
+            auto x2 = node * node;
+            auto x2sum = std::make_shared<op::Sum>(x2, reduction_axes);
+
+            // TODO(mbrookhart): Use Sqrt instead of Power
+            auto half = std::make_shared<op::Constant>(et, x2sum->get_shape(), "0.5");
+            return std::make_shared<op::Power>(x2sum, half);
+        }
+
+        std::shared_ptr<Node> mean(const std::shared_ptr<Node>& node, const AxisSet& reduction_axes)
+        {
+            auto xsum = std::make_shared<op::Sum>(node, reduction_axes);
+
+            auto N = get_num_elements(node->get_shape(), reduction_axes);
+            const auto& et = node->get_element_type();
+
+            auto divisor = std::make_shared<op::Constant>(et, xsum->get_shape(), std::to_string(N));
+
+            return xsum / divisor;
+        }
+
+        std::shared_ptr<Node> std_dev(const std::shared_ptr<Node>& node,
+                                      const AxisSet& reduction_axes,
+                                      const bool bessel_correction)
+        {
+            auto var = variance(node, reduction_axes, bessel_correction);
+
+            const auto& et = node->get_element_type();
+            // TODO(mbrookhart): Use Sqrt instead of Power
+            auto half = std::make_shared<op::Constant>(et, var->get_shape(), "0.5");
+            return std::make_shared<op::Power>(var, half);
+        }
+
+        // This currently calculates [E[X^2] - E[X]^2] instead of [E[(X-\mu)^2]]
+        // The second might be more numerically stable/easier to pattern match
+        // It also requires adding a broadcast op, and would probably be slower
+        // TODO(mbrookhart): Switch to E[(X-\mu)^2]?
+        std::shared_ptr<Node> variance(const std::shared_ptr<Node>& node,
+                                       const AxisSet& reduction_axes,
+                                       const bool bessel_correction)
+        {
+            auto xsum = std::make_shared<op::Sum>(node, reduction_axes);
+
+            auto x2 = node * node;
+
+            auto x2sum = std::make_shared<op::Sum>(x2, reduction_axes);
+
+            const auto& et = node->get_element_type();
+            auto N = get_num_elements(node->get_shape(), reduction_axes);
+
+            auto Nconst = std::make_shared<op::Constant>(et, xsum->get_shape(), std::to_string(N));
+            auto xbar2 = (xsum * xsum) / Nconst;
+
+            auto diff = x2sum - xbar2;
+
+            if (bessel_correction)
+            {
+                auto N1const =
+                    std::make_shared<op::Constant>(et, xsum->get_shape(), std::to_string(N - 1));
+                return diff / N1const;
+            }
+            else
+            {
+                return diff / Nconst;
+            }
+        }
+
+    } // namespace builder
+} // namespace ngraph

src/ngraph/builder/reduce_ops.hpp

+/*
+ Copyright 2017 Nervana Systems Inc.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#pragma once
+
+#include "ngraph/common.hpp"
+#include "ngraph/function.hpp"
+#include "ngraph/node.hpp"
+#include "ngraph/ops/constant.hpp"
+#include "ngraph/ops/parameter.hpp"
+#include "ngraph/ops/reduce.hpp"
+#include "ngraph/types/type.hpp"
+
+namespace ngraph
+{
+    namespace builder
+    {
+        /// \brief Sum-based L2 Norm of a Tensor.
+        ///
+        /// Calculates
+        ///
+        /// \f$\left(\sum_{i=1}^{N} x_i^2\right)^{0.5}\f$
+        ///
+        /// Where `i` traverses all of the axes provided in `reduction_axes`
+        ///
+        /// ## Inputs
+        ///
+        /// |                  | Type                              | Description                                                                                           |
+        /// | ---------------- | --------------------------------- | ----------------------------------------------------------------------------------------------------- |
+        /// | `node`           | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | An input tensor of any shape
+        /// | `reduction_axes` | AxesSet                           | The axes to eliminate through reduction (0 indexed).                                                                                  |
+        ///
+        /// ## Output
+        ///
+        /// | Type                                      | Description                                                                                                      |
+        /// | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+        /// | \f$E[\textit{delete}(A,d_1,\dots,d_n)]\f$ | The tensor \f$T\f$, where \f$T\f$ is the input tensor with the `reduction_axes` \f$A\f$ eliminated by reduction. |
+        std::shared_ptr<Node> l2_norm(const std::shared_ptr<Node>& node,
+                                      const AxisSet& reduction_axes);
+
+        /// \brief Sum-based Mean of a Tensor.
+        ///
+        /// Calculates
+        ///
+        /// \f$\sum_{i=1}^{N} \frac{x_i}{N}\f$
+        ///
+        /// Where `i` traverses all of the axes provided in `reduction_axes`
+        ///
+        /// ## Inputs
+        ///
+        /// |                  | Type                              | Description                                                                                           |
+        /// | ---------------- | --------------------------------- | ----------------------------------------------------------------------------------------------------- |
+        /// | `node`           | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | An input tensor of any shape
+        /// | `reduction_axes` | AxesSet                           | The axes to eliminate through reduction (0 indexed).                                                                                  |
+        ///
+        /// ## Output
+        ///
+        /// | Type                                      | Description                                                                                                      |
+        /// | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+        /// | \f$E[\textit{delete}(A,d_1,\dots,d_n)]\f$ | The tensor \f$T\f$, where \f$T\f$ is the input tensor with the `reduction_axes` \f$A\f$ eliminated by reduction. |
+        std::shared_ptr<Node> mean(const std::shared_ptr<Node>& node,
+                                   const AxisSet& reduction_axes);
+
+        /// \brief Sum-based Standard Deviation of a Tensor.
+        ///
+        /// If bessel_correct is true, calculates
+        ///
+        /// \f$\sqrt{\frac{\sum_{i=1}^{N}\left(x_i-\bar{x}\right)^2}{N-1}}\f$
+        ///
+        /// else, calculates
+        ///
+        /// \f$\sqrt{\frac{\sum_{i=1}^{N}\left(x_i-\bar{x}\right)^2}{N}}\f$
+        ///
+        /// Where `i` traverses all of the axes provided in `reduction_axes` and \f$\bar{x} = \sum_{i=1}^{N} \frac{x_i}{N}\f$
+        ///
+        /// ## Inputs
+        ///
+        /// |                     | Type                              | Description                                                                                           |
+        /// | ------------------- | --------------------------------- | ----------------------------------------------------------------------------------------------------- |
+        /// | `node`              | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | An input tensor of any shape
+        /// | `reduction_axes`    | AxesSet                           | The axes to eliminate through reduction (0 indexed).                                                                                  |
+        /// | `bessel_correction` | bool (default = false)            | Enable Bessel's correction to std_dev for Small sample sizes                                                                                  |
+        ///
+        /// ## Output
+        ///
+        /// | Type                                      | Description                                                                                                      |
+        /// | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+        /// | \f$E[\textit{delete}(A,d_1,\dots,d_n)]\f$ | The tensor \f$T\f$, where \f$T\f$ is the input tensor with the `reduction_axes` \f$A\f$ eliminated by reduction. |
+        std::shared_ptr<Node> std_dev(const std::shared_ptr<Node>& node,
+                                      const AxisSet& reduction_axes,
+                                      const bool bessel_correction = false);
+
+        /// \brief Sum-based Variance of a Tensor.
+        ///
+        /// If bessel_correct is true, calculates
+        ///
+        /// \f$\frac{\sum_{i=1}^{N}\left(x_i-\bar{x}\right)^2}{N-1}\f$
+        ///
+        /// else, calculates
+        ///
+        /// \f$\frac{\sum_{i=1}^{N}\left(x_i-\bar{x}\right)^2}{N}\f$
+        ///
+        /// Where `i` traverses all of the axes provided in `reduction_axes` and \f$\bar{x} = \sum_{i=1}^{N} \frac{x_i}{N}\f$
+        ///
+        /// ## Inputs
+        ///
+        /// |                     | Type                              | Description                                                                                           |
+        /// | ------------------- | --------------------------------- | ----------------------------------------------------------------------------------------------------- |
+        /// | `node`              | \f$E[d_1,\dots,d_n]~(n \geq 0)\f$ | An input tensor of any shape
+        /// | `reduction_axes`    | AxesSet                           | The axes to eliminate through reduction (0 indexed).                                                                                  |
+        /// | `bessel_correction` | bool (default = false)            | Enable Bessel's correction to std_dev for Small sample sizes                                                                                  |
+        ///
+        /// ## Output
+        ///
+        /// | Type                                      | Description                                                                                                      |
+        /// | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+        /// | \f$E[\textit{delete}(A,d_1,\dots,d_n)]\f$ | The tensor \f$T\f$, where \f$T\f$ is the input tensor with the `reduction_axes` \f$A\f$ eliminated by reduction. |
+        std::shared_ptr<Node> variance(const std::shared_ptr<Node>& node,
+                                       const AxisSet& reduction_axes,
+                                       const bool bessel_correction = false);
+
+    } // namespace builder
+} // namespace ngraph

src/ngraph/ngraph.hpp

@@ -37,6 +37,11 @@
 /// @namespace ngraph::runtime
 /// @brief The objects used for executing the graph.
 
+/// @namespace ngraph::builder
+/// @brief Convenience functions that create addional graph nodes to implement commonly-used
+///        recipes, for example auto-broadcast.
+
+#include "ngraph/builder/reduce_ops.hpp"
 #include "ngraph/common.hpp"
 #include "ngraph/descriptor/buffer.hpp"
 #include "ngraph/descriptor/input.hpp"

test/CMakeLists.txt

@@ -22,6 +22,7 @@ include_directories(
     )
 
 set (SRC
+    builder_reduce_ops.cpp
     autodiff.cpp
     build_graph.cpp
     copy.cpp

test/builder_reduce_ops.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+#include "gtest/gtest.h"
+
+#include "ngraph/ngraph.hpp"
+#include "util/all_close.hpp"
+
+using namespace ngraph;
+using namespace ngraph::test;
+using namespace std;
+
+template <typename T>
+static void copy_data(shared_ptr<runtime::TensorView> tv, const vector<T>& data)
+{
+    size_t data_size = data.size() * sizeof(T);
+    tv->write(data.data(), 0, data_size);
+}
+
+std::shared_ptr<ngraph::runtime::TensorView> make_reduce_result(
+    std::function<std::shared_ptr<Node>(const std::shared_ptr<Node>&, const AxisSet&)> func)
+{
+    auto shape_a = Shape{3, 2};
+    auto A = make_shared<op::Parameter>(element::Float32::element_type(), shape_a);
+    auto shape_rt = Shape{2};
+    auto rt = make_shared<TensorViewType>(element::Float32::element_type(), shape_rt);
+    auto f = make_shared<Function>(func(A, {0}), rt, op::Parameters{A});
+    auto manager = runtime::Manager::get("NGVM");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::Float32::element_type(), shape_a);
+    copy_data(a, vector<float>{1, 2, 3, 4, 5, 6});
+    auto result = backend->make_primary_tensor_view(element::Float32::element_type(), shape_rt);
+    cf->call({a}, {result});
+
+    return result;
+}
+
+std::shared_ptr<ngraph::runtime::TensorView> make_reduce_result_true(
+    std::function<std::shared_ptr<Node>(const std::shared_ptr<Node>&, const AxisSet&, bool)> func)
+{
+    auto shape_a = Shape{3, 2};
+    auto A = make_shared<op::Parameter>(element::Float32::element_type(), shape_a);
+    auto shape_rt = Shape{2};
+    auto rt = make_shared<TensorViewType>(element::Float32::element_type(), shape_rt);
+    auto f = make_shared<Function>(func(A, {0}, true), rt, op::Parameters{A});
+    auto manager = runtime::Manager::get("NGVM");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::Float32::element_type(), shape_a);
+    copy_data(a, vector<float>{1, 2, 3, 4, 5, 6});
+    auto result = backend->make_primary_tensor_view(element::Float32::element_type(), shape_rt);
+    cf->call({a}, {result});
+
+    return result;
+}
+
+std::shared_ptr<ngraph::runtime::TensorView> make_reduce_result_false(
+    std::function<std::shared_ptr<Node>(const std::shared_ptr<Node>&, const AxisSet&, bool)> func)
+{
+    auto shape_a = Shape{3, 2};
+    auto A = make_shared<op::Parameter>(element::Float32::element_type(), shape_a);
+    auto shape_rt = Shape{2};
+    auto rt = make_shared<TensorViewType>(element::Float32::element_type(), shape_rt);
+    auto f = make_shared<Function>(func(A, {0}, false), rt, op::Parameters{A});
+    auto manager = runtime::Manager::get("NGVM");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::Float32::element_type(), shape_a);
+    copy_data(a, vector<float>{1, 2, 3, 4, 5, 6});
+    auto result = backend->make_primary_tensor_view(element::Float32::element_type(), shape_rt);
+    cf->call({a}, {result});
+
+    return result;
+}
+
+TEST(builder_reduce_ops, l2_norm)
+{
+    auto result = make_reduce_result(builder::l2_norm);
+    ASSERT_TRUE(
+        all_close((vector<float>{5.9160797831f, 7.48331477355f}), result->get_vector<float>()));
+}
+
+TEST(builder_reduce_ops, mean)
+{
+    auto result = make_reduce_result(builder::mean);
+    ASSERT_TRUE(all_close((vector<float>{3, 4}), result->get_vector<float>()));
+}
+
+TEST(builder_reduce_ops, std_dev)
+{
+    auto result = make_reduce_result_false(builder::std_dev);
+    ASSERT_TRUE(
+        all_close((vector<float>{1.63299316186f, 1.63299316186f}), result->get_vector<float>()));
+    result = make_reduce_result_true(builder::std_dev);
+    ASSERT_TRUE(all_close((vector<float>{2, 2}), result->get_vector<float>()));
+}
+
+TEST(builder_reduce_ops, variance)
+{
+    auto result = make_reduce_result_false(builder::variance);
+    ASSERT_TRUE(
+        all_close((vector<float>{2.66666666666f, 2.66666666666f}), result->get_vector<float>()));
+    result = make_reduce_result_true(builder::variance);
+    ASSERT_TRUE(all_close((vector<float>{4, 4}), result->get_vector<float>()));
+}