Skip to content

N
ngraph
Unverified Commit 1e358743 authored by Scott Cyphers's avatar Scott Cyphers Committed by GitHub
Browse files
Options

Merge pull request #2983 from NervanaSystems/arogowiec/fused_rnn_gru_cells 

[Fused] RNN and  GRU cells
parents c36e4980 a997ccad
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 886 additions and 78 deletions
src/ngraph/CMakeLists.txt
View file @ 1e358743
......@@ -318,6 +318,8 @@ set (SRC
op/fused/group_conv.cpp
op/fused/group_conv_transpose.hpp
op/fused/group_conv_transpose.cpp
op/fused/gru_cell.cpp
op/fused/gru_cell.hpp
op/fused/leaky_relu.cpp
op/fused/leaky_relu.hpp
op/fused/lstm_cell.cpp
......@@ -328,6 +330,8 @@ set (SRC
op/fused/normalize.hpp
op/fused/prelu.cpp
op/fused/prelu.hpp
op/fused/rnn_cell.cpp
op/fused/rnn_cell.hpp
op/fused/scale_shift.cpp
op/fused/scale_shift.hpp
op/fused/shuffle_channels.cpp
......
src/ngraph/frontend/onnx_import/utils/reshape.cpp
View file @ 1e358743
......@@ -25,7 +25,6 @@
#include "exceptions.hpp"
#include "ngraph/builder/reshape.hpp"
#include "ngraph/op/reshape.hpp"
#include "ngraph/op/slice.hpp"
#include "utils/common.hpp"
#include "utils/reshape.hpp"
......
src/ngraph/ngraph.hpp
View file @ 1e358743
......@@ -105,12 +105,14 @@
#include "ngraph/op/fused/grn.hpp"
#include "ngraph/op/fused/group_conv.hpp"
#include "ngraph/op/fused/group_conv_transpose.hpp"
#include "ngraph/op/fused/gru_cell.hpp"
#include "ngraph/op/fused/hard_sigmoid.hpp"
#include "ngraph/op/fused/leaky_relu.hpp"
#include "ngraph/op/fused/lstm_cell.hpp"
#include "ngraph/op/fused/mvn.hpp"
#include "ngraph/op/fused/normalize.hpp"
#include "ngraph/op/fused/prelu.hpp"
#include "ngraph/op/fused/rnn_cell.hpp"
#include "ngraph/op/fused/scale_shift.hpp"
#include "ngraph/op/fused/shuffle_channels.hpp"
#include "ngraph/op/fused/space_to_depth.hpp"
......
src/ngraph/op/fused/gru_cell.cpp 0 → 100644
View file @ 1e358743
This diff is collapsed.
src/ngraph/op/fused/gru_cell.hpp 0 → 100644
View file @ 1e358743
//*****************************************************************************
// Copyright 2017-2019 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************
#pragma once
#include <cstddef>
#include <memory>
#include <string>
#include <vector>
#include "ngraph/node.hpp"
#include "ngraph/op/util/activation_functions.hpp"
#include "ngraph/op/util/fused_op.hpp"
#include "ngraph/op/util/rnn_cell_base.hpp"
namespace ngraph
{
namespace op
{
///
/// \brief Class for GRU cell node.
///
/// \note It follows notation and equations defined as in ONNX standard:
/// https://github.com/onnx/onnx/blob/master/docs/Operators.md#GRU
///
/// Note this class represents only single *cell* and not whole GRU *layer*.
///
class GRUCell : public util::FusedOp, public util::RNNCellBase
{
public:
///
/// \brief Constructs GRUCell node.
///
/// \param[in] X The input tensor with shape: [batch_size, input_size].
/// \param[in] W The weight tensor with shape:
/// [gates_count * hidden_size, input_size].
/// \param[in] R The recurrence weight tensor with shape:
/// [gates_count * hidden_size, hidden_size].
/// \param[in] H_t The hidden state tensor at current time step with
/// shape: [batch_size, hidden_size].
/// \param[in] hidden_size The number of hidden units for recurrent cell.
///
GRUCell(const std::shared_ptr<Node>& X,
const std::shared_ptr<Node>& W,
const std::shared_ptr<Node>& R,
const std::shared_ptr<Node>& H_t,
std::size_t hidden_size);
///
/// \brief Constructs GRUCell node.
///
/// \param[in] X The input tensor with shape: [batch_size, input_size].
/// \param[in] W The weight tensor with shape:
/// [gates_count * hidden_size, input_size].
/// \param[in] R The recurrence weight tensor with shape:
/// [gates_count * hidden_size, hidden_size].
/// \param[in] H_t The hidden state tensor at current time step with
/// shape: [batch_size, hidden_size].
/// \param[in] hidden_size The number of hidden units for recurrent cell.
/// \param[in] activations The vector of activation functions used inside
/// recurrent cell.
/// \param[in] activation_alpha The vector of alpha parameters for activation
/// functions in order respective to activation list.
/// \param[in] activation_beta The vector of beta parameters for activation functions
/// in order respective to activation list.
/// \param[in] clip The value defining clipping range [-clip, clip] on
/// input of activation functions.
///
GRUCell(const std::shared_ptr<Node>& X,
const std::shared_ptr<Node>& W,
const std::shared_ptr<Node>& R,
const std::shared_ptr<Node>& H_t,
std::size_t hidden_size,
const std::vector<std::string>& activations,
const std::vector<float>& activation_alpha,
const std::vector<float>& activation_beta,
float clip,
bool linear_before_reset);
///
/// \brief Constructs GRUCell node.
///
/// \param[in] X The input tensor with shape: [batch_size, input_size].
/// \param[in] W The weight tensor with shape:
/// [gates_count * hidden_size, input_size].
/// \param[in] R The recurrence weight tensor with shape:
/// [gates_count * hidden_size, hidden_size].
/// \param[in] H_t The hidden state tensor at current time step with
/// shape: [batch_size, hidden_size].
/// \param[in] hidden_size The number of hidden units for recurrent cell.
/// \param[in] B The bias tensor for input gate with shape:
/// [2 * gates_count * hidden_size].
/// \param[in] activations The vector of activation functions used inside
/// recurrent cell.
/// \param[in] activation_alpha The vector of alpha parameters for activation
/// functions in order respective to activation list.
/// \param[in] activation_beta The vector of beta parameters for activation functions
/// in order respective to activation list.
/// \param[in] clip The value defining clipping range [-clip, clip] on
/// input of activation functions.
///
GRUCell(const std::shared_ptr<Node>& X,
const std::shared_ptr<Node>& W,
const std::shared_ptr<Node>& R,
const std::shared_ptr<Node>& H_t,
std::size_t hidden_size,
const std::shared_ptr<Node>& B,
const std::vector<std::string>& activations =
std::vector<std::string>{"sigmoid", "tanh"},
const std::vector<float>& activation_alpha = {},
const std::vector<float>& activation_beta = {},
float clip = 0.f,
bool linear_before_reset = false);
virtual void pre_validate_and_infer_types() override;
virtual NodeVector decompose_op() const override;
virtual std::shared_ptr<Node>
copy_with_new_args(const NodeVector& new_args) const override;
bool get_linear_before_reset() const { return m_linear_before_reset; }
private:
/// brief Add and initialize bias input to all zeros.
void add_default_bias_input();
///
/// \brief The Activation function f.
///
util::ActivationFunction m_activation_f;
///
/// \brief The Activation function g.
///
util::ActivationFunction m_activation_g;
static constexpr std::size_t s_gates_count{3};
///
/// \brief Control whether or not apply the linear transformation.
///
/// \note The linear transformation may be applied when computing the output of hidden gate.
/// It's done before multiplying by the output of the reset gate.
///
bool m_linear_before_reset;
};
}
}
src/ngraph/op/fused/lstm_cell.cpp
View file @ 1e358743
......@@ -24,11 +24,6 @@
#include "ngraph/op/constant.hpp"
#include "ngraph/op/dot.hpp"
#include "ngraph/op/fused/lstm_cell.hpp"
#include "ngraph/op/maximum.hpp"
#include "ngraph/op/minimum.hpp"
#include "ngraph/op/multiply.hpp"
#include "ngraph/op/subtract.hpp"
#include "ngraph/op/util/broadcasting.hpp"
#include "ngraph/shape.hpp"
#include "ngraph/type/element_type.hpp"
#include "ngraph/util.hpp"
......@@ -36,46 +31,6 @@
using namespace std;
using namespace ngraph;
// ------------- HELPER FUNCTIONS ---------------------------------------------
static shared_ptr<Node> add(const shared_ptr<Node>& lhs, const shared_ptr<Node>& rhs)
{
auto args = op::numpy_style_broadcast({lhs, rhs});
return {make_shared<op::Add>(args.at(0), args.at(1))};
}
static shared_ptr<Node> sub(const shared_ptr<Node>& lhs, const shared_ptr<Node>& rhs)
{
auto args = op::numpy_style_broadcast({lhs, rhs});
return {make_shared<op::Subtract>(args.at(0), args.at(1))};
}
static shared_ptr<Node> mul(const shared_ptr<Node>& lhs, const shared_ptr<Node>& rhs)
{
auto args = op::numpy_style_broadcast({lhs, rhs});
return {make_shared<op::Multiply>(args.at(0), args.at(1))};
}
static shared_ptr<Node> clip(const shared_ptr<Node>& data, float threshold)
{
if (threshold == 0.f)
{
return data;
}
float min_val = -threshold;
float max_val = threshold;
size_t size = shape_size(data->get_shape());
const shared_ptr<Node> min_val_node = op::Constant::create(
data->get_element_type(), data->get_shape(), vector<float>(size, min_val));
const shared_ptr<Node> max_val_node = op::Constant::create(
data->get_element_type(), data->get_shape(), vector<float>(size, max_val));
return make_shared<op::Minimum>(max_val_node, make_shared<op::Maximum>(data, min_val_node));
}
// ------------- LSTM_CELL ----------------------------------------------------
op::LSTMCell::LSTMCell(const shared_ptr<Node>& X,
const shared_ptr<Node>& W,
const shared_ptr<Node>& R,
......@@ -166,18 +121,18 @@ void op::LSTMCell::pre_validate_and_infer_types()
const Shape& ct_shape{ct_pshape.to_shape()};
NODE_VALIDATION_CHECK(this,
(w_shape == Shape{m_gates_count * get_hidden_size(), input_size}),
(w_shape == Shape{s_gates_count * get_hidden_size(), input_size}),
"Input tensor W must have shape (",
m_gates_count * get_hidden_size(),
s_gates_count * get_hidden_size(),
", ",
input_size,
"). Actual shape is:",
w_shape,
".");
NODE_VALIDATION_CHECK(this,
(r_shape == Shape{m_gates_count * get_hidden_size(), get_hidden_size()}),
(r_shape == Shape{s_gates_count * get_hidden_size(), get_hidden_size()}),
"Input tensor R must have shape (",
m_gates_count * get_hidden_size(),
s_gates_count * get_hidden_size(),
", ",
get_hidden_size(),
"). Actual shape is:",
......@@ -213,7 +168,7 @@ void op::LSTMCell::pre_validate_and_infer_types()
const Shape& p_shape{p_pshape.to_shape()};
NODE_VALIDATION_CHECK(this,
(b_shape == Shape{2 * m_gates_count * get_hidden_size()}),
(b_shape == Shape{2 * s_gates_count * get_hidden_size()}),
"Input tensor B must have shape (",
8 * get_hidden_size(),
"). Actual shape is:",
......@@ -221,9 +176,9 @@ void op::LSTMCell::pre_validate_and_infer_types()
".");
NODE_VALIDATION_CHECK(this,
(p_shape == Shape{m_peepholes_count * get_hidden_size()}),
(p_shape == Shape{s_peepholes_count * get_hidden_size()}),
"Input tensor P must have shape (",
m_peepholes_count * get_hidden_size(),
s_peepholes_count * get_hidden_size(),
"). Actual shape is:",
p_shape,
".");
......@@ -295,7 +250,7 @@ NodeVector op::LSTMCell::decompose_op() const
auto c_t = split_gates.at(3);
// f(Xt*(Wi^T) + Ht-1*(Ri^T) + Pi (.) Ct-1 + Wbi + Rbi)
i_t = m_activation_f(clip(add(i_t, mul(p_i, C_t)), get_clip()));
i_t = m_activation_f(clip(add(i_t, mul(p_i, C_t))));
if (m_input_forget)
{
// Couple input with forget gate: 1 - i_t
......@@ -307,14 +262,14 @@ NodeVector op::LSTMCell::decompose_op() const
else
{
// f(Xt*(Wf^T) + Ht-1*(Rf^T) + Pf (.) Ct-1 + Wbf + Rbf)
f_t = m_activation_f(clip(add(f_t, mul(p_f, C_t)), get_clip()));
f_t = m_activation_f(clip(add(f_t, mul(p_f, C_t))));
}
// ft (.) Ct-1 + it (.) ct
auto C = add(mul(f_t, C_t), mul(i_t, m_activation_g(clip(c_t, get_clip()))));
auto C = add(mul(f_t, C_t), mul(i_t, m_activation_g(clip(c_t))));
// f(Xt*(Wo^T) + Ht-1*(Ro^T) + Po (.) Ct + Wbo + Rbo)
o_t = m_activation_f(clip(add(o_t, mul(p_o, C)), get_clip()));
o_t = m_activation_f(clip(add(o_t, mul(p_o, C))));
// ot (.) h(Ct)
auto H = mul(o_t, m_activation_h(clip(C, get_clip())));
auto H = mul(o_t, m_activation_h(clip(C)));
return {H, C};
}
......@@ -332,15 +287,15 @@ NodeVector op::LSTMCell::get_peephole_weigths() const
{
shared_ptr<Node> P;
P = get_argument(6);
return builder::split(P, m_peepholes_count);
return builder::split(P, s_peepholes_count);
}
void op::LSTMCell::add_default_bias_input()
{
shared_ptr<Node> B =
op::Constant::create(input(0).get_element_type(),
Shape{2 * m_gates_count * get_hidden_size()},
vector<float>(2 * m_gates_count * get_hidden_size(), 0.f));
Shape{2 * s_gates_count * get_hidden_size()},
vector<float>(2 * s_gates_count * get_hidden_size(), 0.f));
set_argument(5, B->output(0));
}
......@@ -348,8 +303,8 @@ void op::LSTMCell::add_default_peepholes_input()
{
shared_ptr<Node> P =
op::Constant::create(input(0).get_element_type(),
Shape{m_peepholes_count * get_hidden_size()},
vector<float>(m_peepholes_count * get_hidden_size(), 0.f));
Shape{s_peepholes_count * get_hidden_size()},
vector<float>(s_peepholes_count * get_hidden_size(), 0.f));
set_argument(6, P->output(0));
}
......
src/ngraph/op/fused/lstm_cell.hpp
View file @ 1e358743
......@@ -168,8 +168,8 @@ namespace ngraph
///
bool m_input_forget = false;
static constexpr std::size_t m_gates_count{4};
static constexpr std::size_t m_peepholes_count{3};
static constexpr std::size_t s_gates_count{4};
static constexpr std::size_t s_peepholes_count{3};
};
}
}
src/ngraph/op/fused/rnn_cell.cpp 0 → 100644
View file @ 1e358743
//*****************************************************************************
// Copyright 2017-2019 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************
#include <algorithm>
#include <cmath>
#include <functional>
#include "ngraph/builder/reshape.hpp"
#include "ngraph/builder/split.hpp"
#include "ngraph/op/add.hpp"
#include "ngraph/op/constant.hpp"
#include "ngraph/op/dot.hpp"
#include "ngraph/op/fused/rnn_cell.hpp"
#include "ngraph/shape.hpp"
#include "ngraph/type/element_type.hpp"
#include "ngraph/util.hpp"
using namespace std;
using namespace ngraph;
op::RNNCell::RNNCell(const shared_ptr<Node>& X,
const shared_ptr<Node>& W,
const shared_ptr<Node>& R,
const shared_ptr<Node>& H_t,
size_t hidden_size)
: RNNCell(
X, W, R, H_t, hidden_size, vector<string>{"tanh"}, vector<float>{}, vector<float>{}, 0.f)
{
}
op::RNNCell::RNNCell(const shared_ptr<Node>& X,
const shared_ptr<Node>& W,
const shared_ptr<Node>& R,
const shared_ptr<Node>& H_t,
size_t hidden_size,
const vector<string>& activations,
const vector<float>& activation_alpha,
const vector<float>& activation_beta,
float clip)
: FusedOp("RNNCell", {X, W, R, H_t})
, RNNCellBase(hidden_size, clip, activations, activation_alpha, activation_beta)
, m_activation_f{get_activation_function(0)}
{
add_default_bias_input();
constructor_validate_and_infer_types();
}
op::RNNCell::RNNCell(const shared_ptr<Node>& X,
const shared_ptr<Node>& W,
const shared_ptr<Node>& R,
const shared_ptr<Node>& H_t,
size_t hidden_size,
const shared_ptr<Node>& B,
const vector<string>& activations,
const vector<float>& activation_alpha,
const vector<float>& activation_beta,
float clip)
: FusedOp("RNNCell", {X, W, R, H_t, B})
, RNNCellBase(hidden_size, clip, activations, activation_alpha, activation_beta)
, m_activation_f{get_activation_function(0)}
{
constructor_validate_and_infer_types();
}
void op::RNNCell::pre_validate_and_infer_types()
{
const auto& x_pshape = get_input_partial_shape(0);
const auto& w_pshape = get_input_partial_shape(1);
const auto& r_pshape = get_input_partial_shape(2);
const auto& ht_pshape = get_input_partial_shape(3);
NODE_VALIDATION_CHECK(this,
(x_pshape.is_static() || w_pshape.is_static() || r_pshape.is_static() ||
ht_pshape.is_static()),
"RNNCell supports only static input tensors.");
const Shape& x_shape{x_pshape.to_shape()};
const size_t batch_size = x_shape.at(0);
const size_t input_size = x_shape.at(1);
const Shape& w_shape{w_pshape.to_shape()};
const Shape& r_shape{r_pshape.to_shape()};
const Shape& ht_shape{ht_pshape.to_shape()};
NODE_VALIDATION_CHECK(this,
(w_shape == Shape{get_hidden_size(), input_size}),
"Input tensor W must have shape (",
get_hidden_size(),
", ",
input_size,
"). Actual shape is:",
w_shape,
".");
NODE_VALIDATION_CHECK(this,
(r_shape == Shape{get_hidden_size(), get_hidden_size()}),
"Input tensor R must have shape (",
get_hidden_size(),
", ",
get_hidden_size(),
"). Actual shape is:",
w_shape,
".");
NODE_VALIDATION_CHECK(this,
(ht_shape == Shape{batch_size, get_hidden_size()}),
"Input tensor H_t must have shape (",
batch_size,
", ",
get_hidden_size(),
"). Actual shape is:",
w_shape,
".");
const auto& b_pshape = get_input_partial_shape(4);
NODE_VALIDATION_CHECK(
this, b_pshape.is_static(), "RNNCell supports only static input tensors.");
const Shape& b_shape{b_pshape.to_shape()};
NODE_VALIDATION_CHECK(this,
(b_shape == Shape{2 * get_hidden_size()}),
"Input tensor B must have shape (",
2 * get_hidden_size(),
"). Actual shape is:",
b_shape,
".");
}
NodeVector op::RNNCell::decompose_op() const
{
// ------ VARIABLE'S NAMES AND ACRONYM DEFINITIONS ------
// The names used below are analogous to the one used in ONNX documentation.
//
// ------ ACRONYMS ------
// i_t - input gate at current time step
// t - time step (t-1 means previous time step)
// X - The input data tensor. Shape: [batch_size, input_size].
// W - The weight tensor for input gate. Shape: [hidden_size, input_size].
// R - The recurrence weight tensor for input gate. Shape: [hidden_size, hidden_size].
// H_t - The hidden state tensor at current time step. Shape: [batch_size, hidden_size].
// B - The bias tensor for the input gate. Shape: [2 * hidden_size] Concatenation of `[Wb, Rb]`.
// Wb - W bias vectors for input gate.
// Rb - R bias vectors for input gate.
// ------ VARIABLE NAMES ------
// Xt_W - Input sequence multiplied by weights tensor at current time step.
// Ht_R - Hidden state multiplied by weights tensor at current time step.
// (.) - Denotes element-wise multiplication.
// * - Denotes dot product.
// ---- Equations ----
// f - is activation functions.
// Ht = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi)
// --------------------
std::shared_ptr<Node> X = get_argument(0);
std::shared_ptr<Node> W = get_argument(1);
std::shared_ptr<Node> R = get_argument(2);
std::shared_ptr<Node> H_t = get_argument(3);
std::shared_ptr<Node> bias = get_bias();
// Xt*(W^T)
auto Xt_W = std::make_shared<op::Dot>(X, builder::transpose(W));
// Ht-1*(R^T)
auto Ht_R = std::make_shared<op::Dot>(H_t, builder::transpose(R));
// Xt*(W^T) + Ht-1*(R^T) + Wb + Rb
auto i_t = add(Xt_W, add(Ht_R, bias));
// f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi)
i_t = m_activation_f(clip(i_t));
return {i_t};
}
shared_ptr<Node> op::RNNCell::get_bias() const
{
shared_ptr<Node> bias;
// Split B onto Wb an Rb and add them.
NodeVector b_W_R = builder::split(get_argument(4), 2);
bias = b_W_R.at(0) + b_W_R.at(1);
return bias;
}
void op::RNNCell::add_default_bias_input()
{
shared_ptr<Node> B =
op::Constant::create(input(0).get_element_type(),
Shape{2 * s_gates_count * get_hidden_size()},
vector<float>(2 * s_gates_count * get_hidden_size(), 0.f));
set_argument(4, B->output(0));
}
shared_ptr<Node> op::RNNCell::copy_with_new_args(const NodeVector& new_args) const
{
check_new_args_count(this, new_args);
if (new_args.size() == 4)
{
return make_shared<RNNCell>(new_args.at(0),
new_args.at(1),
new_args.at(2),
new_args.at(3),
get_hidden_size(),
get_activations(),
get_activation_alpha(),
get_activation_beta(),
get_clip());
}
else if (new_args.size() == 5)
{
return make_shared<RNNCell>(new_args.at(0),
new_args.at(1),
new_args.at(2),
new_args.at(3),
get_hidden_size(),
new_args.at(4),
get_activations(),
get_activation_alpha(),
get_activation_beta(),
get_clip());
}
else
{
throw ngraph_error("Incorrect number of new arguments");
}
}
src/ngraph/op/fused/rnn_cell.hpp 0 → 100644
View file @ 1e358743
//*****************************************************************************
// Copyright 2017-2019 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************
#pragma once
#include <cstddef>
#include <memory>
#include <string>
#include <vector>
#include "ngraph/node.hpp"
#include "ngraph/op/util/activation_functions.hpp"
#include "ngraph/op/util/fused_op.hpp"
#include "ngraph/op/util/rnn_cell_base.hpp"
namespace ngraph
{
namespace op
{
///
/// \brief Class for RNN cell node.
///
/// \note It follows notation and equations defined as in ONNX standard:
/// https://github.com/onnx/onnx/blob/master/docs/Operators.md#RNN
///
/// Note this class represents only single *cell* and not whole RNN *layer*.
///
class RNNCell : public util::FusedOp, public util::RNNCellBase
{
public:
///
/// \brief Constructs RNNCell node.
///
/// \param[in] X The input tensor with shape: [batch_size, input_size].
/// \param[in] W The weight tensor with shape: [hidden_size, input_size].
/// \param[in] R The recurrence weight tensor with shape:
/// [hidden_size, hidden_size].
/// \param[in] H_t The hidden state tensor at current time step with shape:
/// [batch_size, hidden_size].
/// \param[in] hidden_size The number of hidden units for recurrent cell.
///
RNNCell(const std::shared_ptr<Node>& X,
const std::shared_ptr<Node>& W,
const std::shared_ptr<Node>& R,
const std::shared_ptr<Node>& H_t,
std::size_t hidden_size);
///
/// \brief Constructs RNNCell node.
///
/// \param[in] X The input tensor with shape: [batch_size, input_size].
/// \param[in] W The weight tensor with shape: [hidden_size, input_size].
/// \param[in] R The recurrence weight tensor with shape:
/// [hidden_size, hidden_size].
/// \param[in] H_t The hidden state tensor at current time step with
/// shape: [batch_size, hidden_size].
/// \param[in] hidden_size The number of hidden units for recurrent cell.
/// \param[in] activations The vector of activation functions used inside
/// recurrent cell.
/// \param[in] activation_alpha The vector of alpha parameters for activation
/// functions in order respective to activation list.
/// \param[in] activation_beta The vector of beta parameters for activation functions
/// in order respective to activation list.
/// \param[in] clip The value defining clipping range [-clip, clip] on
/// input of activation functions.
///
RNNCell(const std::shared_ptr<Node>& X,
const std::shared_ptr<Node>& W,
const std::shared_ptr<Node>& R,
const std::shared_ptr<Node>& H_t,
std::size_t hidden_size,
const std::vector<std::string>& activations,
const std::vector<float>& activation_alpha,
const std::vector<float>& activation_beta,
float clip);
///
/// \brief Constructs RNNCell node.
///
/// \param[in] X The input tensor with shape: [batch_size, input_size].
/// \param[in] W The weight tensor with shape: [hidden_size, input_size].
/// \param[in] R The recurrence weight tensor with shape:
/// [hidden_size, hidden_size].
/// \param[in] H_t The hidden state tensor at current time step with
/// shape: [batch_size, hidden_size].
/// \param[in] hidden_size The number of hidden units for recurrent cell.
/// \param[in] B The bias tensor for input gate with shape: [2*hidden_size].
/// \param[in] activations The vector of activation functions used inside
/// recurrent cell.
/// \param[in] activation_alpha The vector of alpha parameters for activation
/// functions in order respective to activation list.
/// \param[in] activation_beta The vector of beta parameters for activation functions
/// in order respective to activation list.
/// \param[in] clip The value defining clipping range [-clip, clip] on
/// input of activation functions.
///
RNNCell(const std::shared_ptr<Node>& X,
const std::shared_ptr<Node>& W,
const std::shared_ptr<Node>& R,
const std::shared_ptr<Node>& H_t,
std::size_t hidden_size,
const std::shared_ptr<Node>& B,
const std::vector<std::string>& activations = std::vector<std::string>{"tanh"},
const std::vector<float>& activation_alpha = {},
const std::vector<float>& activation_beta = {},
float clip = 0.f);
virtual void pre_validate_and_infer_types() override;
virtual NodeVector decompose_op() const override;
virtual std::shared_ptr<Node>
copy_with_new_args(const NodeVector& new_args) const override;
private:
std::shared_ptr<Node> get_bias() const;
/// brief Add and initialize bias input to all zeros.
void add_default_bias_input();
///
/// \brief The Activation function f.
///
util::ActivationFunction m_activation_f;
static constexpr std::size_t s_gates_count{1};
};
}
}
src/ngraph/op/fused_op_tbl.hpp
View file @ 1e358743
......@@ -28,12 +28,14 @@ NGRAPH_OP(Gemm, ngraph::op)
NGRAPH_OP(GRN, ngraph::op)
NGRAPH_OP(GroupConvolution, ngraph::op)
NGRAPH_OP(GroupConvolutionTranspose, ngraph::op)
NGRAPH_OP(GRUCell, ngraph::op)
NGRAPH_OP(HardSigmoid, ngraph::op)
NGRAPH_OP(LeakyRelu, ngraph::op)
NGRAPH_OP(LSTMCell, ngraph::op)
NGRAPH_OP(MVN, ngraph::op)
NGRAPH_OP(Normalize, ngraph::op)
NGRAPH_OP(PRelu, ngraph::op)
NGRAPH_OP(RNNCell, ngraph::op)
NGRAPH_OP(ScaleShift, ngraph::op)
NGRAPH_OP(ShuffleChannels, ngraph::op)
NGRAPH_OP(SpaceToDepth, ngraph::op)
......
src/ngraph/op/util/rnn_cell_base.cpp
View file @ 1e358743
......@@ -17,6 +17,14 @@
#include <algorithm>
#include <iterator>
#include "ngraph/op/add.hpp"
#include "ngraph/op/constant.hpp"
#include "ngraph/op/fused/clamp.hpp"
#include "ngraph/op/maximum.hpp"
#include "ngraph/op/minimum.hpp"
#include "ngraph/op/multiply.hpp"
#include "ngraph/op/subtract.hpp"
#include "ngraph/op/util/broadcasting.hpp"
#include "ngraph/op/util/rnn_cell_base.hpp"
#include "ngraph/util.hpp"
......@@ -60,3 +68,34 @@ op::util::ActivationFunction op::util::RNNCellBase::get_activation_function(size
return afunc;
}
shared_ptr<Node> op::util::RNNCellBase::add(const shared_ptr<Node>& lhs,
const shared_ptr<Node>& rhs)
{
auto args = op::numpy_style_broadcast({lhs, rhs});
return {make_shared<op::Add>(args.at(0), args.at(1))};
}
shared_ptr<Node> op::util::RNNCellBase::sub(const shared_ptr<Node>& lhs,
const shared_ptr<Node>& rhs)
{
auto args = op::numpy_style_broadcast({lhs, rhs});
return {make_shared<op::Subtract>(args.at(0), args.at(1))};
}
shared_ptr<Node> op::util::RNNCellBase::mul(const shared_ptr<Node>& lhs,
const shared_ptr<Node>& rhs)
{
auto args = op::numpy_style_broadcast({lhs, rhs});
return {make_shared<op::Multiply>(args.at(0), args.at(1))};
}
shared_ptr<Node> op::util::RNNCellBase::clip(const shared_ptr<Node>& data) const
{
if (m_clip == 0.f)
{
return data;
}
return make_shared<op::Clamp>(data, -m_clip, m_clip);
}
src/ngraph/op/util/rnn_cell_base.hpp
View file @ 1e358743
......@@ -17,9 +17,11 @@
#pragma once
#include <cstddef>
#include <memory>
#include <string>
#include <vector>
#include "ngraph/node.hpp"
#include "ngraph/op/util/activation_functions.hpp"
namespace ngraph
......@@ -71,10 +73,48 @@ namespace ngraph
/// \return The object representing activation function.
///
ActivationFunction get_activation_function(std::size_t idx) const;
///
/// \brief Creates node with element-wise add operation with numpy broadcasting.
///
/// \param[in] lhs The left hand side argument node.
/// \param[in] rhs The right hand side argument node.
///
/// \return Node with element-wise add operation.
///
static std::shared_ptr<Node> add(const std::shared_ptr<Node>& lhs,
const std::shared_ptr<Node>& rhs);
///
/// \brief Creates node with element-wise subtract operation with numpy broadcasting.
///
/// \param[in] lhs The left hand side argument node.
/// \param[in] rhs The right hand side argument node.
///
/// \return Node with element-wise subtract operation.
///
static std::shared_ptr<Node> sub(const std::shared_ptr<Node>& lhs,
const std::shared_ptr<Node>& rhs);
///
/// \brief Creates node with element-wise multiply operation with numpy broadcasting.
///
/// \param[in] lhs The left hand side argument node.
/// \param[in] rhs The right hand side argument node.
///
/// \return Node with element-wise multiply operation.
///
static std::shared_ptr<Node> mul(const std::shared_ptr<Node>& lhs,
const std::shared_ptr<Node>& rhs);
///
/// \brief Creates node with element-wise clip operation with numpy broadcasting.
///
/// \param[in] data The input tensor for clipping.
///
/// \return Node with element-wise clip operation.
///
std::shared_ptr<Node> clip(const std::shared_ptr<Node>& data) const;
private:
std::size_t m_hidden_size = 0.f;
float m_clip = 0.f;
const std::size_t m_hidden_size;
const float m_clip;
const std::vector<std::string> m_activations;
const std::vector<float> m_activation_alpha;
const std::vector<float> m_activation_beta;
......
src/ngraph/runtime/intelgpu/intelgpu_backend.cpp
View file @ 1e358743
......@@ -87,11 +87,13 @@
#include "ngraph/op/fused/grn.hpp"
#include "ngraph/op/fused/group_conv.hpp"
#include "ngraph/op/fused/group_conv_transpose.hpp"
#include "ngraph/op/fused/gru_cell.hpp"
#include "ngraph/op/fused/hard_sigmoid.hpp"
#include "ngraph/op/fused/leaky_relu.hpp"
#include "ngraph/op/fused/lstm_cell.hpp"
#include "ngraph/op/fused/mvn.hpp"
#include "ngraph/op/fused/normalize.hpp"
#include "ngraph/op/fused/rnn_cell.hpp"
#include "ngraph/op/fused/scale_shift.hpp"
#include "ngraph/op/fused/shuffle_channels.hpp"
#include "ngraph/op/fused/space_to_depth.hpp"
......@@ -2070,6 +2072,7 @@ shared_ptr<runtime::Executable>
case OP_TYPEID::GenerateMask:
case OP_TYPEID::GRN:
case OP_TYPEID::GroupConvolutionTranspose:
case OP_TYPEID::GRUCell:
case OP_TYPEID::HardSigmoid:
case OP_TYPEID::LeakyRelu:
case OP_TYPEID::LSTMCell:
......@@ -2077,6 +2080,7 @@ shared_ptr<runtime::Executable>
case OP_TYPEID::Normalize:
case OP_TYPEID::PRelu:
case OP_TYPEID::Passthrough:
case OP_TYPEID::RNNCell:
case OP_TYPEID::QuantizedAvgPool:
case OP_TYPEID::QuantizedConvolution:
case OP_TYPEID::QuantizedConvolutionBias:
......@@ -2195,11 +2199,13 @@ bool runtime::intelgpu::IntelGPUBackend::is_supported_impl(const Node& node)
case OP_TYPEID::Gemm:
case OP_TYPEID::GRN:
case OP_TYPEID::GroupConvolutionTranspose:
case OP_TYPEID::GRUCell:
case OP_TYPEID::LeakyRelu:
case OP_TYPEID::LSTMCell:
case OP_TYPEID::MVN:
case OP_TYPEID::Normalize:
case OP_TYPEID::PRelu:
case OP_TYPEID::RNNCell:
case OP_TYPEID::ScaleShift:
case OP_TYPEID::ShuffleChannels:
case OP_TYPEID::SpaceToDepth:
......
src/ngraph/serializer.cpp
View file @ 1e358743
......@@ -79,12 +79,14 @@
#include "ngraph/op/fused/grn.hpp"
#include "ngraph/op/fused/group_conv.hpp"
#include "ngraph/op/fused/group_conv_transpose.hpp"
#include "ngraph/op/fused/gru_cell.hpp"
#include "ngraph/op/fused/hard_sigmoid.hpp"
#include "ngraph/op/fused/leaky_relu.hpp"
#include "ngraph/op/fused/lstm_cell.hpp"
#include "ngraph/op/fused/mvn.hpp"
#include "ngraph/op/fused/normalize.hpp"
#include "ngraph/op/fused/prelu.hpp"
#include "ngraph/op/fused/rnn_cell.hpp"
#include "ngraph/op/fused/scale_shift.hpp"
#include "ngraph/op/fused/shuffle_channels.hpp"
#include "ngraph/op/fused/space_to_depth.hpp"
......@@ -1230,13 +1232,6 @@ shared_ptr<Node> JSONDeserializer::deserialize_node(json node_js)
node = make_shared<op::GRN>(args[0], bias);
break;
}
case OP_TYPEID::HardSigmoid:
{
auto alpha = node_js.at("alpha").get<float>();
auto beta = node_js.at("beta").get<float>();
node = make_shared<op::HardSigmoid>(args[0], alpha, beta);
break;
}
case OP_TYPEID::GroupConvolution:
{
auto window_movement_strides =
......@@ -1283,6 +1278,35 @@ shared_ptr<Node> JSONDeserializer::deserialize_node(json node_js)
output_shape);
break;
}
case OP_TYPEID::GRUCell:
{
auto hidden_size = node_js.at("hidden_size").get<size_t>();
auto clip = node_js.at("clip").get<float>();
auto activations = node_js.at("activations").get<vector<string>>();
auto activation_alpha = node_js.at("activation_alpha").get<vector<float>>();
auto activation_beta = node_js.at("activation_beta").get<vector<float>>();
auto linear_before_reset = node_js.at("linear_before_reset").get<bool>();
node = make_shared<op::GRUCell>(args[0],
args[1],
args[2],
args[3],
hidden_size,
args[4],
activations,
activation_alpha,
activation_beta,
clip,
linear_before_reset);
break;
}
case OP_TYPEID::HardSigmoid:
{
auto alpha = node_js.at("alpha").get<float>();
auto beta = node_js.at("beta").get<float>();
node = make_shared<op::HardSigmoid>(args[0], alpha, beta);
break;
}
case OP_TYPEID::LeakyRelu:
{
node = make_shared<op::LeakyRelu>(args[0], args[1]);
......@@ -1664,6 +1688,25 @@ shared_ptr<Node> JSONDeserializer::deserialize_node(json node_js)
node = make_shared<op::ReverseSequence>(args[0], args[1], batch_axis, sequence_axis);
break;
}
case OP_TYPEID::RNNCell:
{
auto hidden_size = node_js.at("hidden_size").get<size_t>();
auto clip = node_js.at("clip").get<float>();
auto activations = node_js.at("activations").get<vector<string>>();
auto activation_alpha = node_js.at("activation_alpha").get<vector<float>>();
auto activation_beta = node_js.at("activation_beta").get<vector<float>>();
node = make_shared<op::RNNCell>(args[0],
args[1],
args[2],
args[3],
hidden_size,
args[4],
activations,
activation_alpha,
activation_beta,
clip);
break;
}
case OP_TYPEID::ScalarConstantLike:
{
double value = node_js.at("value").get<double>();
......@@ -2340,11 +2383,15 @@ json JSONSerializer::serialize_node(const Node& n)
node["bias"] = tmp->get_bias();
break;
}
case OP_TYPEID::HardSigmoid:
case OP_TYPEID::GRUCell:
{
auto tmp = dynamic_cast<const op::HardSigmoid*>(&n);
node["alpha"] = tmp->get_alpha();
node["beta"] = tmp->get_beta();
auto tmp = dynamic_cast<const op::GRUCell*>(&n);
node["hidden_size"] = tmp->get_hidden_size();
node["clip"] = tmp->get_clip();
node["activations"] = tmp->get_activations();
node["activation_alpha"] = tmp->get_activation_alpha();
node["activation_beta"] = tmp->get_activation_beta();
node["linear_before_reset"] = tmp->get_linear_before_reset();
break;
}
case OP_TYPEID::GroupConvolution:
......@@ -2372,6 +2419,13 @@ json JSONSerializer::serialize_node(const Node& n)
node["output_shape"] = tmp->get_output_shape();
break;
}
case OP_TYPEID::HardSigmoid:
{
auto tmp = dynamic_cast<const op::HardSigmoid*>(&n);
node["alpha"] = tmp->get_alpha();
node["beta"] = tmp->get_beta();
break;
}
case OP_TYPEID::LeakyRelu: { break;
}
case OP_TYPEID::Less:
......@@ -2662,6 +2716,16 @@ json JSONSerializer::serialize_node(const Node& n)
node["sequence_axis"] = tmp->get_sequence_axis();
break;
}
case OP_TYPEID::RNNCell:
{
auto tmp = dynamic_cast<const op::RNNCell*>(&n);
node["hidden_size"] = tmp->get_hidden_size();
node["clip"] = tmp->get_clip();
node["activations"] = tmp->get_activations();
node["activation_alpha"] = tmp->get_activation_alpha();
node["activation_beta"] = tmp->get_activation_beta();
break;
}
case OP_TYPEID::ScalarConstantLike:
{
auto tmp = dynamic_cast<const op::ScalarConstantLikeBase*>(&n);
......
test/backend_fused_op.in.cpp
View file @ 1e358743
This diff is collapsed.
test/type_prop.cpp
View file @ 1e358743
......@@ -15763,3 +15763,164 @@ INSTANTIATE_TEST_CASE_P(type_prop,
::testing::Values(RangeParams{0, 1, 0.25, PartialShape{4}},
RangeParams{-1, 1, 0.25, PartialShape{8}},
RangeParams{-1, 0.875, 0.25, PartialShape{8}}));
TEST(type_prop, rnn_cell)
{
const size_t batch_size = 2;
const size_t input_size = 3;
const size_t hidden_size = 3;
const auto X = make_shared<op::Parameter>(element::f32, Shape{batch_size, input_size});
const auto W = make_shared<op::Parameter>(element::f32, Shape{hidden_size, input_size});
const auto R = make_shared<op::Parameter>(element::f32, Shape{hidden_size, hidden_size});
const auto H_t = make_shared<op::Parameter>(element::f32, Shape{batch_size, hidden_size});
const auto rnn_cell = make_shared<op::RNNCell>(X, W, R, H_t, hidden_size);
EXPECT_EQ(rnn_cell->output(0).get_element_type(), element::f32);
EXPECT_EQ(rnn_cell->output(0).get_shape(), (Shape{batch_size, hidden_size}));
}
TEST(type_prop, rnn_cell_invalid_input)
{
const size_t batch_size = 2;
const size_t input_size = 3;
const size_t hidden_size = 3;
auto X = make_shared<op::Parameter>(element::f32, Shape{batch_size, input_size});
auto R = make_shared<op::Parameter>(element::f32, Shape{hidden_size, hidden_size});
auto H_t = make_shared<op::Parameter>(element::f32, Shape{batch_size, hidden_size});
// Invalid W tensor shape.
auto W = make_shared<op::Parameter>(element::f32, Shape{2 * hidden_size, input_size});
try
{
const auto rnn_cell = make_shared<op::RNNCell>(X, W, R, H_t, hidden_size);
FAIL() << "RNNCell node was created with invalid data.";
}
catch (const NodeValidationFailure& error)
{
EXPECT_HAS_SUBSTRING(error.what(), std::string("Input tensor W must have shape"));
}
// Invalid R tensor shape.
W = make_shared<op::Parameter>(element::f32, Shape{hidden_size, input_size});
R = make_shared<op::Parameter>(element::f32, Shape{hidden_size, 1});
try
{
const auto rnn_cell = make_shared<op::RNNCell>(X, W, R, H_t, hidden_size);
FAIL() << "RNNCell node was created with invalid data.";
}
catch (const NodeValidationFailure& error)
{
EXPECT_HAS_SUBSTRING(error.what(), std::string("Input tensor R must have shape"));
}
// Invalid H_t tensor shape.
R = make_shared<op::Parameter>(element::f32, Shape{hidden_size, hidden_size});
H_t = make_shared<op::Parameter>(element::f32, Shape{4, hidden_size});
try
{
const auto rnn_cell = make_shared<op::RNNCell>(X, W, R, H_t, hidden_size);
FAIL() << "RNNCell node was created with invalid data.";
}
catch (const NodeValidationFailure& error)
{
EXPECT_HAS_SUBSTRING(error.what(), std::string("Input tensor H_t must have shape"));
}
// Invalid B tensor shape.
H_t = make_shared<op::Parameter>(element::f32, Shape{batch_size, hidden_size});
auto B = make_shared<op::Parameter>(element::f32, Shape{hidden_size});
try
{
const auto rnn_cell = make_shared<op::RNNCell>(X, W, R, H_t, hidden_size, B);
FAIL() << "RNNCell node was created with invalid data.";
}
catch (const NodeValidationFailure& error)
{
EXPECT_HAS_SUBSTRING(error.what(), std::string("Input tensor B must have shape"));
}
}
TEST(type_prop, gru_cell)
{
const size_t batch_size = 2;
const size_t input_size = 3;
const size_t hidden_size = 3;
const size_t gates_count = 3;
const auto X = make_shared<op::Parameter>(element::f32, Shape{batch_size, input_size});
const auto W =
make_shared<op::Parameter>(element::f32, Shape{gates_count * hidden_size, input_size});
const auto R =
make_shared<op::Parameter>(element::f32, Shape{gates_count * hidden_size, hidden_size});
const auto H_t = make_shared<op::Parameter>(element::f32, Shape{batch_size, hidden_size});
const auto gru_cell = make_shared<op::GRUCell>(X, W, R, H_t, hidden_size);
EXPECT_EQ(gru_cell->output(0).get_element_type(), element::f32);
EXPECT_EQ(gru_cell->output(0).get_shape(), (Shape{batch_size, hidden_size}));
}
TEST(type_prop, gru_cell_invalid_input)
{
const size_t batch_size = 2;
const size_t input_size = 3;
const size_t hidden_size = 3;
const size_t gates_count = 3;
const auto X = make_shared<op::Parameter>(element::f32, Shape{batch_size, input_size});
auto R =
make_shared<op::Parameter>(element::f32, Shape{gates_count * hidden_size, hidden_size});
auto H_t = make_shared<op::Parameter>(element::f32, Shape{batch_size, hidden_size});
// Invalid W tensor shape.
auto W = make_shared<op::Parameter>(element::f32, Shape{hidden_size, input_size});
try
{
const auto gru_cell = make_shared<op::GRUCell>(X, W, R, H_t, hidden_size);
FAIL() << "GRUCell node was created with invalid data.";
}
catch (const NodeValidationFailure& error)
{
EXPECT_HAS_SUBSTRING(error.what(), std::string("Input tensor W must have shape"));
}
// Invalid R tensor shape.
W = make_shared<op::Parameter>(element::f32, Shape{gates_count * hidden_size, input_size});
R = make_shared<op::Parameter>(element::f32, Shape{hidden_size, 1});
try
{
const auto gru_cell = make_shared<op::GRUCell>(X, W, R, H_t, hidden_size);
FAIL() << "GRUCell node was created with invalid data.";
}
catch (const NodeValidationFailure& error)
{
EXPECT_HAS_SUBSTRING(error.what(), std::string("Input tensor R must have shape"));
}
// Invalid H_t tensor shape.
R = make_shared<op::Parameter>(element::f32, Shape{gates_count * hidden_size, hidden_size});
H_t = make_shared<op::Parameter>(element::f32, Shape{4, hidden_size});
try
{
const auto gru_cell = make_shared<op::GRUCell>(X, W, R, H_t, hidden_size);
FAIL() << "GRUCell node was created with invalid data.";
}
catch (const NodeValidationFailure& error)
{
EXPECT_HAS_SUBSTRING(error.what(), std::string("Input tensor H_t must have shape"));
}
// Invalid B tensor shape.
H_t = make_shared<op::Parameter>(element::f32, Shape{batch_size, hidden_size});
auto B = make_shared<op::Parameter>(element::f32, Shape{hidden_size});
try
{
const auto gru_cell = make_shared<op::GRUCell>(X, W, R, H_t, hidden_size, B);
FAIL() << "GRUCell node was created with invalid data.";
}
catch (const NodeValidationFailure& error)
{
EXPECT_HAS_SUBSTRING(error.what(), std::string("Input tensor B must have shape"));
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment