Merge pull request #3302 from NervanaSystems/etusien/gelu

[Py] Added gelu operator to Python API.

doc/sphinx/source/python_api/_autosummary/ngraph.ops.rst

@@ -37,6 +37,7 @@ ngraph.ops
       equal
       exp
       floor
+      gelu
       get_output_element
       greater
       greater_eq

python/ngraph/__init__.py

@@ -50,6 +50,7 @@ from ngraph.ops import elu
 from ngraph.ops import equal
 from ngraph.ops import exp
 from ngraph.ops import floor
+from ngraph.ops import gelu
 from ngraph.ops import get_output_element
 from ngraph.ops import greater
 from ngraph.ops import greater_eq

python/ngraph/impl/op/__init__.py

@@ -74,6 +74,7 @@ from _pyngraph.op import Elu
 from _pyngraph.op import Equal
 from _pyngraph.op import Exp
 from _pyngraph.op import Floor
+from _pyngraph.op import Gelu
 from _pyngraph.op import GetOutputElement
 from _pyngraph.op import Greater
 from _pyngraph.op import GreaterEq

python/ngraph/ops.py

@@ -23,7 +23,7 @@ from ngraph.impl import AxisSet, AxisVector, Coordinate, CoordinateDiff, Functio
 from ngraph.impl.op import Abs, Acos, Add, And, Asin, ArgMax, ArgMin, Atan, AvgPool, \
     BatchNormTraining, BatchNormInference, Broadcast, Ceiling, Clamp, Concat, Constant, Convert, \
     Convolution, ConvolutionBackpropData, Cos, Cosh, Divide, Dot, Elu, Equal, Exp, Floor, \
-    GetOutputElement, Greater, GreaterEq, Less, LessEq, Log, LRN, Max, Maximum, MaxPool, \
+    Gelu, GetOutputElement, Greater, GreaterEq, Less, LessEq, Log, LRN, Max, Maximum, MaxPool, \
     Min, Minimum, Multiply, Negative, Not, NotEqual, OneHot, Or, Pad, Parameter, Product, \
     Power, Relu, ReplaceSlice, Reshape, Reverse, Select, Sign, Sin, Sinh, Slice, Softmax, \
     Sqrt, Subtract, Sum, Tan, Tanh, TopK
@@ -527,6 +527,24 @@ def convert(node, new_type, name=None):  # type: (Node, NumericType, str) -> Nod
     return Convert(node, new_element_type)
 
 
+@nameable_op
+def gelu(node, name=None):  # type: (NodeInput, str) -> Node
+    r"""Perform Gaussian Error Linear Unit operation element-wise on data from input node.
+
+    Computes GELU function:
+
+    .. math:: f(x) = 0.5\cdot x\cdot(1 + erf( \dfrac{x}{\sqrt{2}})
+
+    For more information refer to:
+    `Gaussian Error Linear Unit (GELU) <https://arxiv.org/pdf/1606.08415.pdf>`_
+
+    :param node: Input tensor. One of: input node, array or scalar.
+    :param name: Optional output node name.
+    :return: The new node performing a GELU operation on its input data element-wise.
+    """
+    return Gelu(as_node(node))
+
+
 @nameable_op
 def select(selection_node, input_node1, input_node2, name=None):
     # type: (Node, Node, Node, str) -> Node

python/pyngraph/ops/fused/gelu.cpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+#include "ngraph/op/fused/gelu.hpp"
+#include "pyngraph/ops/fused/gelu.hpp"
+
+namespace py = pybind11;
+
+void regclass_pyngraph_op_Gelu(py::module m)
+{
+    py::class_<ngraph::op::Gelu, std::shared_ptr<ngraph::op::Gelu>, ngraph::op::Op> gelu(m, "Gelu");
+    gelu.doc() = "ngraph.impl.op.Gelu wraps ngraph::op::Gelu";
+    gelu.def(py::init<const std::shared_ptr<ngraph::Node>&>());
+}

python/pyngraph/ops/fused/gelu.hpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#pragma once
+
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
+
+void regclass_pyngraph_op_Gelu(py::module m);

python/pyngraph/ops/regmodule_pyngraph_op.cpp

@@ -54,6 +54,7 @@ void regmodule_pyngraph_op(py::module m_op)
     regclass_pyngraph_op_Equal(m_op);
     regclass_pyngraph_op_Exp(m_op);
     regclass_pyngraph_op_Floor(m_op);
+    regclass_pyngraph_op_Gelu(m_op);
     regclass_pyngraph_op_GetOutputElement(m_op);
     regclass_pyngraph_op_Greater(m_op);
     regclass_pyngraph_op_GreaterEq(m_op);

python/pyngraph/ops/regmodule_pyngraph_op.hpp

@@ -44,6 +44,7 @@
 #include "pyngraph/ops/floor.hpp"
 #include "pyngraph/ops/fused/clamp.hpp"
 #include "pyngraph/ops/fused/elu.hpp"
+#include "pyngraph/ops/fused/gelu.hpp"
 #include "pyngraph/ops/get_output_element.hpp"
 #include "pyngraph/ops/greater.hpp"
 #include "pyngraph/ops/greater_eq.hpp"

python/setup.py

@@ -184,6 +184,7 @@ sources = [
     'pyngraph/ops/equal.cpp',
     'pyngraph/ops/exp.cpp',
     'pyngraph/ops/floor.cpp',
+    'pyngraph/ops/fused/gelu.cpp',
     'pyngraph/ops/greater.cpp',
     'pyngraph/ops/greater_eq.cpp',
     'pyngraph/ops/less.cpp',

python/test/ngraph/test_ops_fused.py

@@ -19,7 +19,7 @@ import ngraph as ng
 from test.ngraph.util import get_runtime
 
 
-def test_elu_operator():
+def test_elu_operator_with_parameters():
     runtime = get_runtime()
 
     data_shape = [2, 2]
@@ -69,6 +69,38 @@ def test_elu_operator_with_scalar():
     assert np.allclose(result, expected)
 
 
+def test_gelu_operator_with_parameters():
+    runtime = get_runtime()
+
+    data_value = np.array([[-5, 1], [-2, 3]], dtype=np.float32)
+
+    data_shape = [2, 2]
+    parameter_data = ng.parameter(data_shape, name='Data', dtype=np.float32)
+
+    model = ng.gelu(parameter_data)
+    computation = runtime.computation(model, parameter_data)
+
+    result = computation(data_value)
+    expected = np.array([[-1.4901161e-06, 8.4134471e-01], [-4.5500278e-02, 2.9959502]],
+                        dtype=np.float32)
+    assert np.allclose(result, expected)
+
+
+def test_gelu_operator_with_array():
+    runtime = get_runtime()
+
+    data_value = np.array([[-5, 1], [-2, 3]], dtype=np.float32)
+
+    model = ng.gelu(data_value)
+    computation = runtime.computation(model)
+
+    result = computation()
+    expected = np.array([[-1.4901161e-06, 8.4134471e-01], [-4.5500278e-02, 2.9959502]],
+                        dtype=np.float32)
+
+    assert np.allclose(result, expected)
+
+
 def test_clamp_operator():
     runtime = get_runtime()
 
@@ -99,4 +131,5 @@ def test_clamp_operator_with_array():
 
     result = computation()
     expected = np.clip(data_value, min_value, max_value)
+
     assert np.allclose(result, expected)