Skip to content

N
ngraph
Commit 5b459fd9 authored by Adam Rogowiec's avatar Adam Rogowiec
Browse files
Options

Store original Bias input as RNNCell member.

parent 835dd295
Hide whitespace changes
Inline Side-by-side
Showing with 112 additions and 105 deletions
src/ngraph/op/fused/rnn_cell.cpp
View file @ 5b459fd9
...@@ -32,31 +32,24 @@ using namespace std; ...@@ -32,31 +32,24 @@ using namespace std;
using namespace ngraph; using namespace ngraph;
op::RNNCell::RNNCell(const shared_ptr<Node>& X, op::RNNCell::RNNCell(const shared_ptr<Node>& X,
const shared_ptr<Node>& W, const shared_ptr<Node>& W,
const shared_ptr<Node>& R, const shared_ptr<Node>& R,
const shared_ptr<Node>& H_t, const shared_ptr<Node>& H_t,
size_t hidden_size) size_t hidden_size)
: RNNCell(X, : RNNCell(
W, X, W, R, H_t, hidden_size, vector<string>{"tanh"}, vector<float>{}, vector<float>{}, 0.f)
R,
H_t,
hidden_size,
vector<string>{"tanh"},
vector<float>{},
vector<float>{},
0.f)
{ {
} }
op::RNNCell::RNNCell(const shared_ptr<Node>& X, op::RNNCell::RNNCell(const shared_ptr<Node>& X,
const shared_ptr<Node>& W, const shared_ptr<Node>& W,
const shared_ptr<Node>& R, const shared_ptr<Node>& R,
const shared_ptr<Node>& H_t, const shared_ptr<Node>& H_t,
size_t hidden_size, size_t hidden_size,
const vector<string>& activations, const vector<string>& activations,
const vector<float>& activation_alpha, const vector<float>& activation_alpha,
const vector<float>& activation_beta, const vector<float>& activation_beta,
float clip) float clip)
: FusedOp("RNNCell", {X, W, R, H_t}) : FusedOp("RNNCell", {X, W, R, H_t})
, RNNCellBase(hidden_size, clip, activations, activation_alpha, activation_beta) , RNNCellBase(hidden_size, clip, activations, activation_alpha, activation_beta)
, m_X{X} , m_X{X}
...@@ -65,58 +58,56 @@ op::RNNCell::RNNCell(const shared_ptr<Node>& X, ...@@ -65,58 +58,56 @@ op::RNNCell::RNNCell(const shared_ptr<Node>& X,
, m_H_t{H_t} , m_H_t{H_t}
, m_activation_f{get_activation_function(0)} , m_activation_f{get_activation_function(0)}
{ {
// Normally we would split B onto Wb an Rb and add them, however here they are all zeros, // As default bias is all zeros, thus just initialize it with appropriate shape and zeros.
// thus just initialize bias with appropriate shape and zeros. m_B = op::Constant::create(m_X->get_element_type(),
m_bias = ngraph::op::Constant::create(element::f32, Shape{2 * get_hidden_size()},
Shape{m_gates_count * get_hidden_size()}, vector<float>(2 * get_hidden_size(), 0.f));
vector<float>(m_gates_count * get_hidden_size(), 0.f));
constructor_validate_and_infer_types(); constructor_validate_and_infer_types();
} }
op::RNNCell::RNNCell(const shared_ptr<Node>& X, op::RNNCell::RNNCell(const shared_ptr<Node>& X,
const shared_ptr<Node>& W, const shared_ptr<Node>& W,
const shared_ptr<Node>& R, const shared_ptr<Node>& R,
const shared_ptr<Node>& H_t, const shared_ptr<Node>& H_t,
size_t hidden_size, size_t hidden_size,
const shared_ptr<Node>& B, const shared_ptr<Node>& B,
const vector<string>& activations, const vector<string>& activations,
const vector<float>& activation_alpha, const vector<float>& activation_alpha,
const vector<float>& activation_beta, const vector<float>& activation_beta,
float clip) float clip)
: FusedOp("RNNCell", {X, W, R, H_t, B}) : FusedOp("RNNCell", {X, W, R, H_t, B})
, RNNCellBase(hidden_size, clip, activations, activation_alpha, activation_beta) , RNNCellBase(hidden_size, clip, activations, activation_alpha, activation_beta)
, m_X{X} , m_X{X}
, m_W{W} , m_W{W}
, m_R{R} , m_R{R}
, m_H_t{H_t} , m_H_t{H_t}
, m_B{B}
, m_activation_f{get_activation_function(0)} , m_activation_f{get_activation_function(0)}
{ {
// Split B onto Wb and Rb and add them.
NODE_VALIDATION_CHECK(this,
(B->get_shape() == Shape{2 * m_gates_count * get_hidden_size()}),
"Input tensor B must have shape (",
8 * get_hidden_size(),
"). Actual shape is:",
B->get_shape(),
".");
NodeVector b_W_R = builder::split(B, 2);
m_bias = b_W_R.at(0) + b_W_R.at(1);
constructor_validate_and_infer_types(); constructor_validate_and_infer_types();
} }
void op::RNNCell::pre_validate_and_infer_types() void op::RNNCell::pre_validate_and_infer_types()
{ {
const auto& x_shape = input(0).get_shape(); const auto& x_pshape = get_input_partial_shape(0);
const auto& w_pshape = get_input_partial_shape(1);
const auto& r_pshape = get_input_partial_shape(2);
const auto& ht_pshape = get_input_partial_shape(3);
NODE_VALIDATION_CHECK(this,
(x_pshape.is_static() || w_pshape.is_static() || r_pshape.is_static() ||
ht_pshape.is_static()),
"GRUCell supports only static input tensors.");
const Shape& x_shape{x_pshape.to_shape()};
const size_t batch_size = x_shape.at(0); const size_t batch_size = x_shape.at(0);
const size_t input_size = x_shape.at(1); const size_t input_size = x_shape.at(1);
const auto& w_shape = input(1).get_shape(); const Shape& w_shape{w_pshape.to_shape()};
const auto& r_shape = input(2).get_shape(); const Shape& r_shape{r_pshape.to_shape()};
const auto& ht_shape = input(3).get_shape(); const Shape& ht_shape{ht_pshape.to_shape()};
NODE_VALIDATION_CHECK(this, NODE_VALIDATION_CHECK(this,
(w_shape == Shape{get_hidden_size(), input_size}), (w_shape == Shape{get_hidden_size(), input_size}),
...@@ -145,6 +136,24 @@ void op::RNNCell::pre_validate_and_infer_types() ...@@ -145,6 +136,24 @@ void op::RNNCell::pre_validate_and_infer_types()
"). Actual shape is:", "). Actual shape is:",
w_shape, w_shape,
"."); ".");
if (get_input_size() > 4)
{
const auto& b_pshape = get_input_partial_shape(4);
NODE_VALIDATION_CHECK(
this, b_pshape.is_static(), "GRUCell supports only static input tensors.");
const Shape& b_shape{b_pshape.to_shape()};
NODE_VALIDATION_CHECK(this,
(b_shape == Shape{2 * get_hidden_size()}),
"Input tensor B must have shape (",
2 * get_hidden_size(),
"). Actual shape is:",
b_shape,
".");
}
} }
NodeVector op::RNNCell::decompose_op() const NodeVector op::RNNCell::decompose_op() const
...@@ -155,15 +164,12 @@ NodeVector op::RNNCell::decompose_op() const ...@@ -155,15 +164,12 @@ NodeVector op::RNNCell::decompose_op() const
// ------ ACRONYMS ------ // ------ ACRONYMS ------
// i_t - input gate at current time step // i_t - input gate at current time step
// t - time step (t-1 means previous time step) // t - time step (t-1 means previous time step)
// W - W parameter weight matrix for input, output, forget, and // W - W parameter weight matrix for input gate.
// cell gates. // R - R recurrence weight matrix for input gate.
// R - R recurrence weight matrix for input, output, forget, and // Wb - W bias vectors for input gate.
// cell gates. // Rb - R bias vectors for input gate.
// Wb - W bias vectors for input, output, forget, and cell gates.
// Rb - R bias vectors for input, output, forget, and cell gates.
// ------ VARIABLE NAMES ------ // ------ VARIABLE NAMES ------
// Xt_W - Input sequence multiplied by weights tensor at current time // Xt_W - Input sequence multiplied by weights tensor at current time step.
// step.
// Ht_R - Hidden state multiplied by weights tensor at current time step. // Ht_R - Hidden state multiplied by weights tensor at current time step.
// (.) - Denotes element-wise multiplication. // (.) - Denotes element-wise multiplication.
...@@ -174,12 +180,15 @@ NodeVector op::RNNCell::decompose_op() const ...@@ -174,12 +180,15 @@ NodeVector op::RNNCell::decompose_op() const
// Ht = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi) // Ht = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi)
// -------------------- // --------------------
NodeVector b_W_R = builder::split(m_B, 2);
auto bias = b_W_R.at(0) + b_W_R.at(1);
// Xt*(W^T) // Xt*(W^T)
auto Xt_W = std::make_shared<ngraph::op::Dot>(m_X, ngraph::op::util::transpose(m_W)); auto Xt_W = std::make_shared<ngraph::op::Dot>(m_X, ngraph::op::util::transpose(m_W));
// Ht-1*(R^T) // Ht-1*(R^T)
auto Ht_R = std::make_shared<ngraph::op::Dot>(m_H_t, ngraph::op::util::transpose(m_R)); auto Ht_R = std::make_shared<ngraph::op::Dot>(m_H_t, ngraph::op::util::transpose(m_R));
// Xt*(W^T) + Ht-1*(R^T) + Wb + Rb // Xt*(W^T) + Ht-1*(R^T) + Wb + Rb
auto i_t = add(Xt_W, add(Ht_R, m_bias)); auto i_t = add(Xt_W, add(Ht_R, bias));
// f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi) // f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi)
i_t = m_activation_f(clip(i_t)); i_t = m_activation_f(clip(i_t));
...@@ -193,27 +202,27 @@ shared_ptr<Node> op::RNNCell::copy_with_new_args(const NodeVector& new_args) con ...@@ -193,27 +202,27 @@ shared_ptr<Node> op::RNNCell::copy_with_new_args(const NodeVector& new_args) con
if (new_args.size() == 4) if (new_args.size() == 4)
{ {
return make_shared<RNNCell>(new_args.at(0), return make_shared<RNNCell>(new_args.at(0),
new_args.at(1), new_args.at(1),
new_args.at(2), new_args.at(2),
new_args.at(3), new_args.at(3),
get_hidden_size(), get_hidden_size(),
get_activations(), get_activations(),
get_activation_alpha(), get_activation_alpha(),
get_activation_beta(), get_activation_beta(),
get_clip()); get_clip());
} }
else if (new_args.size() == 5) else if (new_args.size() == 5)
{ {
return make_shared<RNNCell>(new_args.at(0), return make_shared<RNNCell>(new_args.at(0),
new_args.at(1), new_args.at(1),
new_args.at(2), new_args.at(2),
new_args.at(3), new_args.at(3),
get_hidden_size(), get_hidden_size(),
new_args.at(4), new_args.at(4),
get_activations(), get_activations(),
get_activation_alpha(), get_activation_alpha(),
get_activation_beta(), get_activation_beta(),
get_clip()); get_clip());
} }
else else
{ {
......
src/ngraph/op/fused/rnn_cell.hpp
View file @ 5b459fd9
...@@ -53,10 +53,10 @@ namespace ngraph ...@@ -53,10 +53,10 @@ namespace ngraph
/// \param[in] hidden_size The number of hidden units for recurrent cell. /// \param[in] hidden_size The number of hidden units for recurrent cell.
/// ///
RNNCell(const std::shared_ptr<Node>& X, RNNCell(const std::shared_ptr<Node>& X,
const std::shared_ptr<Node>& W, const std::shared_ptr<Node>& W,
const std::shared_ptr<Node>& R, const std::shared_ptr<Node>& R,
const std::shared_ptr<Node>& H_t, const std::shared_ptr<Node>& H_t,
std::size_t hidden_size); std::size_t hidden_size);
/// ///
/// \brief Constructs RNNCell node. /// \brief Constructs RNNCell node.
...@@ -78,14 +78,14 @@ namespace ngraph ...@@ -78,14 +78,14 @@ namespace ngraph
/// input of activation functions. /// input of activation functions.
/// ///
RNNCell(const std::shared_ptr<Node>& X, RNNCell(const std::shared_ptr<Node>& X,
const std::shared_ptr<Node>& W, const std::shared_ptr<Node>& W,
const std::shared_ptr<Node>& R, const std::shared_ptr<Node>& R,
const std::shared_ptr<Node>& H_t, const std::shared_ptr<Node>& H_t,
std::size_t hidden_size, std::size_t hidden_size,
const std::vector<std::string>& activations, const std::vector<std::string>& activations,
const std::vector<float>& activation_alpha, const std::vector<float>& activation_alpha,
const std::vector<float>& activation_beta, const std::vector<float>& activation_beta,
float clip); float clip);
/// ///
/// \brief Constructs RNNCell node. /// \brief Constructs RNNCell node.
...@@ -108,16 +108,15 @@ namespace ngraph ...@@ -108,16 +108,15 @@ namespace ngraph
/// input of activation functions. /// input of activation functions.
/// ///
RNNCell(const std::shared_ptr<Node>& X, RNNCell(const std::shared_ptr<Node>& X,
const std::shared_ptr<Node>& W, const std::shared_ptr<Node>& W,
const std::shared_ptr<Node>& R, const std::shared_ptr<Node>& R,
const std::shared_ptr<Node>& H_t, const std::shared_ptr<Node>& H_t,
std::size_t hidden_size, std::size_t hidden_size,
const std::shared_ptr<Node>& B, const std::shared_ptr<Node>& B,
const std::vector<std::string>& activations = const std::vector<std::string>& activations = std::vector<std::string>{"tanh"},
std::vector<std::string>{"tanh"}, const std::vector<float>& activation_alpha = {},
const std::vector<float>& activation_alpha = {}, const std::vector<float>& activation_beta = {},
const std::vector<float>& activation_beta = {}, float clip = 0.f);
float clip = 0.f);
virtual void pre_validate_and_infer_types() override; virtual void pre_validate_and_infer_types() override;
virtual NodeVector decompose_op() const override; virtual NodeVector decompose_op() const override;
...@@ -142,17 +141,16 @@ namespace ngraph ...@@ -142,17 +141,16 @@ namespace ngraph
/// ///
std::shared_ptr<Node> m_H_t; std::shared_ptr<Node> m_H_t;
/// ///
/// \brief The bias tensor for the gates. Shape: [2 * gates_count * hidden_size].
/// \note Concatenation of `[Wb[i], Rb[i]]`.
///
std::shared_ptr<Node> m_B;
///
/// \brief The Activation function f. /// \brief The Activation function f.
/// ///
ActivationFunction m_activation_f; ActivationFunction m_activation_f;
static constexpr std::size_t m_gates_count{1}; static constexpr std::size_t m_gates_count{1};
///
/// \brief Sum of biases (weight and recurrence) for input gate.
///
/// Sum of `[Wb, Rb]`.
///
std::shared_ptr<Node> m_bias;
}; };
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment