Commit b466027e authored by Diego Caballero's avatar Diego Caballero Committed by Scott Cyphers

[CPU] Fix ambiguous 'op' namespace. (#2683)

parent 105f03bc
...@@ -38,7 +38,7 @@ namespace ngraph ...@@ -38,7 +38,7 @@ namespace ngraph
auto& out_tensor = external_function->get_tensor_data(out[0].get_name()); auto& out_tensor = external_function->get_tensor_data(out[0].get_name());
size_t count = out[0].get_size(); size_t count = out[0].get_size();
auto alpha = static_cast<const op::BoundedRelu*>(node)->get_alpha(); auto alpha = static_cast<const ngraph::op::BoundedRelu*>(node)->get_alpha();
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
......
...@@ -38,7 +38,7 @@ namespace ngraph ...@@ -38,7 +38,7 @@ namespace ngraph
auto& out_tensor = external_function->get_tensor_data(out[0].get_name()); auto& out_tensor = external_function->get_tensor_data(out[0].get_name());
size_t count = out[0].get_size(); size_t count = out[0].get_size();
auto alpha = static_cast<const op::LeakyRelu*>(node)->get_alpha(); auto alpha = static_cast<const ngraph::op::LeakyRelu*>(node)->get_alpha();
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
......
...@@ -313,7 +313,7 @@ namespace ngraph ...@@ -313,7 +313,7 @@ namespace ngraph
auto arg0_shape = args[0].get_shape(); auto arg0_shape = args[0].get_shape();
auto arg1_shape = args[1].get_shape(); auto arg1_shape = args[1].get_shape();
auto daxes = quantize->get_axes(); auto daxes = quantize->get_axes();
op::Quantize::RoundMode round_mode = quantize->get_round_mode(); ngraph::op::Quantize::RoundMode round_mode = quantize->get_round_mode();
if (args[0].get_element_type() == element::f32) if (args[0].get_element_type() == element::f32)
{ {
......
...@@ -705,7 +705,7 @@ bool runtime::cpu::mkldnn_utils::use_mkldnn_kernel(const ngraph::Node* node) ...@@ -705,7 +705,7 @@ bool runtime::cpu::mkldnn_utils::use_mkldnn_kernel(const ngraph::Node* node)
void runtime::cpu::mkldnn_utils::assign_mkldnn_kernel(Node* node) void runtime::cpu::mkldnn_utils::assign_mkldnn_kernel(Node* node)
{ {
auto ngraph_op = static_cast<op::Op*>(node); auto ngraph_op = static_cast<ngraph::op::Op*>(node);
auto op_annotations = std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>(); auto op_annotations = std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>();
op_annotations->set_mkldnn_op(true); op_annotations->set_mkldnn_op(true);
ngraph_op->set_op_annotations(op_annotations); ngraph_op->set_op_annotations(op_annotations);
......
...@@ -126,7 +126,7 @@ namespace ngraph ...@@ -126,7 +126,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::QuantizedConcat) void CPUAssignment::ASSIGN_DECL(ngraph::op::QuantizedConcat)
{ {
auto quantized_concat = static_cast<op::QuantizedConcat*>(node); auto quantized_concat = static_cast<ngraph::op::QuantizedConcat*>(node);
if ((node->get_input_element_type(0) == element::i8 || if ((node->get_input_element_type(0) == element::i8 ||
node->get_input_element_type(0) == element::u8) && node->get_input_element_type(0) == element::u8) &&
...@@ -195,7 +195,7 @@ namespace ngraph ...@@ -195,7 +195,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionBiasAdd) void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionBiasAdd)
{ {
auto convolution = static_cast<op::ConvolutionBiasAdd*>(node); auto convolution = static_cast<ngraph::op::ConvolutionBiasAdd*>(node);
if (mkldnn_utils::can_use_mkldnn_conv<ngraph::op::ConvolutionBiasAdd>(node)) if (mkldnn_utils::can_use_mkldnn_conv<ngraph::op::ConvolutionBiasAdd>(node))
{ {
...@@ -212,7 +212,7 @@ namespace ngraph ...@@ -212,7 +212,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::GetOutputElement) void CPUAssignment::ASSIGN_DECL(ngraph::op::GetOutputElement)
{ {
auto goe = static_cast<op::GetOutputElement*>(node); auto goe = static_cast<ngraph::op::GetOutputElement*>(node);
auto op_annotations = auto op_annotations =
std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>(); std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>();
op_annotations->add_in_place_oi_pair({0, goe->get_n(), false}); op_annotations->add_in_place_oi_pair({0, goe->get_n(), false});
...@@ -222,7 +222,7 @@ namespace ngraph ...@@ -222,7 +222,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionAdd) void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionAdd)
{ {
auto convolution = static_cast<op::ConvolutionAdd*>(node); auto convolution = static_cast<ngraph::op::ConvolutionAdd*>(node);
if (mkldnn_utils::can_use_mkldnn_conv<ngraph::op::ConvolutionAdd>(node)) if (mkldnn_utils::can_use_mkldnn_conv<ngraph::op::ConvolutionAdd>(node))
{ {
...@@ -257,7 +257,7 @@ namespace ngraph ...@@ -257,7 +257,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionBackpropData) void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionBackpropData)
{ {
auto convolution = static_cast<op::ConvolutionBackpropData*>(node); auto convolution = static_cast<ngraph::op::ConvolutionBackpropData*>(node);
auto arg0_shape = node->get_input_shape(0); auto arg0_shape = node->get_input_shape(0);
auto arg1_shape = node->get_input_shape(1); auto arg1_shape = node->get_input_shape(1);
...@@ -282,7 +282,7 @@ namespace ngraph ...@@ -282,7 +282,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionBackpropFilters) void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionBackpropFilters)
{ {
auto convolution = static_cast<op::ConvolutionBackpropFilters*>(node); auto convolution = static_cast<ngraph::op::ConvolutionBackpropFilters*>(node);
auto arg0_shape = node->get_input_shape(0); auto arg0_shape = node->get_input_shape(0);
auto arg1_shape = node->get_input_shape(1); auto arg1_shape = node->get_input_shape(1);
...@@ -316,7 +316,8 @@ namespace ngraph ...@@ -316,7 +316,8 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionBiasBackpropFiltersBias) void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionBiasBackpropFiltersBias)
{ {
auto convolution = static_cast<op::ConvolutionBiasBackpropFiltersBias*>(node); auto convolution =
static_cast<ngraph::op::ConvolutionBiasBackpropFiltersBias*>(node);
auto data_shape = node->get_input_shape(0); auto data_shape = node->get_input_shape(0);
auto delta_shape = node->get_input_shape(1); auto delta_shape = node->get_input_shape(1);
...@@ -340,7 +341,7 @@ namespace ngraph ...@@ -340,7 +341,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::AvgPool) void CPUAssignment::ASSIGN_DECL(ngraph::op::AvgPool)
{ {
auto avg_pool = static_cast<op::AvgPool*>(node); auto avg_pool = static_cast<ngraph::op::AvgPool*>(node);
auto arg0_shape = node->get_input_shape(0); auto arg0_shape = node->get_input_shape(0);
auto arg0_rank = arg0_shape.size(); auto arg0_rank = arg0_shape.size();
...@@ -357,7 +358,7 @@ namespace ngraph ...@@ -357,7 +358,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::AvgPoolBackprop) void CPUAssignment::ASSIGN_DECL(ngraph::op::AvgPoolBackprop)
{ {
auto avg_pool = static_cast<op::AvgPoolBackprop*>(node); auto avg_pool = static_cast<ngraph::op::AvgPoolBackprop*>(node);
auto arg0_shape = node->get_input_shape(0); auto arg0_shape = node->get_input_shape(0);
auto arg0_rank = arg0_shape.size(); auto arg0_rank = arg0_shape.size();
...@@ -374,7 +375,7 @@ namespace ngraph ...@@ -374,7 +375,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::MaxPool) void CPUAssignment::ASSIGN_DECL(ngraph::op::MaxPool)
{ {
auto max_pool = static_cast<op::MaxPool*>(node); auto max_pool = static_cast<ngraph::op::MaxPool*>(node);
auto arg0_shape = node->get_input_shape(0); auto arg0_shape = node->get_input_shape(0);
auto arg0_rank = arg0_shape.size(); auto arg0_rank = arg0_shape.size();
...@@ -391,7 +392,7 @@ namespace ngraph ...@@ -391,7 +392,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::MaxPoolWithIndices) void CPUAssignment::ASSIGN_DECL(ngraph::op::MaxPoolWithIndices)
{ {
auto max_pool = static_cast<op::MaxPoolWithIndices*>(node); auto max_pool = static_cast<ngraph::op::MaxPoolWithIndices*>(node);
auto arg0_shape = node->get_input_shape(0); auto arg0_shape = node->get_input_shape(0);
auto arg0_rank = arg0_shape.size(); auto arg0_rank = arg0_shape.size();
...@@ -407,7 +408,7 @@ namespace ngraph ...@@ -407,7 +408,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::MaxPoolBackprop) void CPUAssignment::ASSIGN_DECL(ngraph::op::MaxPoolBackprop)
{ {
auto max_pool = static_cast<op::MaxPoolBackprop*>(node); auto max_pool = static_cast<ngraph::op::MaxPoolBackprop*>(node);
auto arg1_shape = node->get_input_shape(1); auto arg1_shape = node->get_input_shape(1);
auto arg1_rank = arg1_shape.size(); auto arg1_rank = arg1_shape.size();
...@@ -424,7 +425,7 @@ namespace ngraph ...@@ -424,7 +425,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::MaxPoolWithIndicesBackprop) void CPUAssignment::ASSIGN_DECL(ngraph::op::MaxPoolWithIndicesBackprop)
{ {
auto max_pool = static_cast<op::MaxPoolWithIndicesBackprop*>(node); auto max_pool = static_cast<ngraph::op::MaxPoolWithIndicesBackprop*>(node);
auto arg1_shape = node->get_input_shape(1); auto arg1_shape = node->get_input_shape(1);
auto arg1_rank = arg1_shape.size(); auto arg1_rank = arg1_shape.size();
...@@ -440,7 +441,7 @@ namespace ngraph ...@@ -440,7 +441,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::Relu) void CPUAssignment::ASSIGN_DECL(ngraph::op::Relu)
{ {
auto relu = static_cast<op::Relu*>(node); auto relu = static_cast<ngraph::op::Relu*>(node);
auto arg0_shape = node->get_input_shape(0); auto arg0_shape = node->get_input_shape(0);
auto arg0_rank = arg0_shape.size(); auto arg0_rank = arg0_shape.size();
...@@ -464,7 +465,7 @@ namespace ngraph ...@@ -464,7 +465,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::ReplaceSlice) void CPUAssignment::ASSIGN_DECL(ngraph::op::ReplaceSlice)
{ {
auto replace_slice = static_cast<op::ReplaceSlice*>(node); auto replace_slice = static_cast<ngraph::op::ReplaceSlice*>(node);
// ReplaceSlice is independent of data type. Hence not checking type // ReplaceSlice is independent of data type. Hence not checking type
auto op_annotations = auto op_annotations =
...@@ -480,7 +481,7 @@ namespace ngraph ...@@ -480,7 +481,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::UpdateSlice) void CPUAssignment::ASSIGN_DECL(ngraph::op::UpdateSlice)
{ {
auto update_slice = static_cast<op::UpdateSlice*>(node); auto update_slice = static_cast<ngraph::op::UpdateSlice*>(node);
auto op_annotations = auto op_annotations =
std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>(); std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>();
...@@ -601,7 +602,7 @@ namespace ngraph ...@@ -601,7 +602,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::Softmax) void CPUAssignment::ASSIGN_DECL(ngraph::op::Softmax)
{ {
auto softmax = static_cast<op::Softmax*>(node); auto softmax = static_cast<ngraph::op::Softmax*>(node);
auto arg0_shape = node->get_input_shape(0); auto arg0_shape = node->get_input_shape(0);
auto arg0_rank = arg0_shape.size(); auto arg0_rank = arg0_shape.size();
...@@ -618,7 +619,7 @@ namespace ngraph ...@@ -618,7 +619,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::Slice) void CPUAssignment::ASSIGN_DECL(ngraph::op::Slice)
{ {
auto slice = static_cast<op::Slice*>(node); auto slice = static_cast<ngraph::op::Slice*>(node);
auto strides = slice->get_strides(); auto strides = slice->get_strides();
if (!is_strided(strides) && node->get_input_element_type(0) == element::f32) if (!is_strided(strides) && node->get_input_element_type(0) == element::f32)
{ {
...@@ -649,7 +650,7 @@ namespace ngraph ...@@ -649,7 +650,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::BoundedRelu) void CPUAssignment::ASSIGN_DECL(ngraph::op::BoundedRelu)
{ {
auto bounded_relu = static_cast<op::BoundedRelu*>(node); auto bounded_relu = static_cast<ngraph::op::BoundedRelu*>(node);
auto arg0_shape = node->get_input_shape(0); auto arg0_shape = node->get_input_shape(0);
auto arg0_rank = arg0_shape.size(); auto arg0_rank = arg0_shape.size();
...@@ -673,7 +674,7 @@ namespace ngraph ...@@ -673,7 +674,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::LeakyRelu) void CPUAssignment::ASSIGN_DECL(ngraph::op::LeakyRelu)
{ {
auto leaky_relu = static_cast<op::LeakyRelu*>(node); auto leaky_relu = static_cast<ngraph::op::LeakyRelu*>(node);
auto arg0_shape = node->get_input_shape(0); auto arg0_shape = node->get_input_shape(0);
auto arg0_rank = arg0_shape.size(); auto arg0_rank = arg0_shape.size();
...@@ -719,7 +720,8 @@ namespace ngraph ...@@ -719,7 +720,8 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::QuantizedConvolutionBiasAdd) void CPUAssignment::ASSIGN_DECL(ngraph::op::QuantizedConvolutionBiasAdd)
{ {
auto quantized_conv_bias = static_cast<op::QuantizedConvolutionBiasAdd*>(node); auto quantized_conv_bias =
static_cast<ngraph::op::QuantizedConvolutionBiasAdd*>(node);
auto op_annotations = auto op_annotations =
std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>(); std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>();
op_annotations->set_mkldnn_op(true); op_annotations->set_mkldnn_op(true);
...@@ -733,7 +735,7 @@ namespace ngraph ...@@ -733,7 +735,7 @@ namespace ngraph
void CPUAssignment::ASSIGN_DECL(ngraph::op::QuantizedConvolutionBiasSignedAdd) void CPUAssignment::ASSIGN_DECL(ngraph::op::QuantizedConvolutionBiasSignedAdd)
{ {
auto quantized_conv_bias = auto quantized_conv_bias =
static_cast<op::QuantizedConvolutionBiasSignedAdd*>(node); static_cast<ngraph::op::QuantizedConvolutionBiasSignedAdd*>(node);
auto op_annotations = auto op_annotations =
std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>(); std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>();
op_annotations->set_mkldnn_op(true); op_annotations->set_mkldnn_op(true);
...@@ -758,7 +760,7 @@ namespace ngraph ...@@ -758,7 +760,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::Dequantize) void CPUAssignment::ASSIGN_DECL(ngraph::op::Dequantize)
{ {
auto dequantize = static_cast<op::Dequantize*>(node); auto dequantize = static_cast<ngraph::op::Dequantize*>(node);
// TODO(nbpatel): Support dynamic offset via mkldnn // TODO(nbpatel): Support dynamic offset via mkldnn
// Go through reference if the offset is not a constant // Go through reference if the offset is not a constant
if (!dequantize->get_argument(2)->is_constant()) if (!dequantize->get_argument(2)->is_constant())
...@@ -796,7 +798,7 @@ namespace ngraph ...@@ -796,7 +798,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::Quantize) void CPUAssignment::ASSIGN_DECL(ngraph::op::Quantize)
{ {
auto quantize = static_cast<op::Quantize*>(node); auto quantize = static_cast<ngraph::op::Quantize*>(node);
// TODO(nbpatel): Support dynamic offset via mkldnn // TODO(nbpatel): Support dynamic offset via mkldnn
// Go through reference if the offset is not a constant // Go through reference if the offset is not a constant
if (!quantize->get_argument(2)->is_constant()) if (!quantize->get_argument(2)->is_constant())
...@@ -805,8 +807,8 @@ namespace ngraph ...@@ -805,8 +807,8 @@ namespace ngraph
} }
auto offset_const_op = auto offset_const_op =
std::static_pointer_cast<ngraph::op::Constant>(quantize->get_argument(2)); std::static_pointer_cast<ngraph::op::Constant>(quantize->get_argument(2));
op::Quantize::RoundMode round_mode = quantize->get_round_mode(); ngraph::op::Quantize::RoundMode round_mode = quantize->get_round_mode();
if (round_mode != op::Quantize::RoundMode::ROUND_NEAREST_TOWARD_EVEN) if (round_mode != ngraph::op::Quantize::RoundMode::ROUND_NEAREST_TOWARD_EVEN)
{ {
return; return;
} }
...@@ -845,7 +847,7 @@ namespace ngraph ...@@ -845,7 +847,7 @@ namespace ngraph
template <> template <>
void CPUAssignment::ASSIGN_DECL(ngraph::op::Convert) void CPUAssignment::ASSIGN_DECL(ngraph::op::Convert)
{ {
auto convert = static_cast<op::Convert*>(node); auto convert = static_cast<ngraph::op::Convert*>(node);
if ((node->get_input_element_type(0) == element::i8 && if ((node->get_input_element_type(0) == element::i8 &&
node->get_output_element_type(0) == element::u8) || node->get_output_element_type(0) == element::u8) ||
(node->get_input_element_type(0) == element::u8 && (node->get_input_element_type(0) == element::u8 &&
......
...@@ -144,9 +144,9 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_matmulbias() ...@@ -144,9 +144,9 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_matmulbias()
auto x = std::make_shared<pattern::op::Label>(element::f32, shape_x); auto x = std::make_shared<pattern::op::Label>(element::f32, shape_x);
auto b = std::make_shared<pattern::op::Label>(element::f32, shape_b); auto b = std::make_shared<pattern::op::Label>(element::f32, shape_b);
auto pmmb = std::make_shared<op::MatmulBias>( auto pmmb = std::make_shared<ngraph::op::MatmulBias>(
W, x, nullptr, W->get_shape(), x->get_shape(), false, false); W, x, nullptr, W->get_shape(), x->get_shape(), false, false);
auto pbroadcast = std::make_shared<op::Broadcast>(b, pmmb->get_shape(), AxisSet{0}); auto pbroadcast = std::make_shared<ngraph::op::Broadcast>(b, pmmb->get_shape(), AxisSet{0});
auto padd = pmmb + pbroadcast; auto padd = pmmb + pbroadcast;
ngraph::pattern::graph_rewrite_callback callback = [W, x](pattern::Matcher& m) { ngraph::pattern::graph_rewrite_callback callback = [W, x](pattern::Matcher& m) {
...@@ -154,12 +154,12 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_matmulbias() ...@@ -154,12 +154,12 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_matmulbias()
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto mpattern = m.get_match_root(); // add auto mpattern = m.get_match_root(); // add
auto m_matmul = ngraph::pattern::Matcher::unique_match<op::MatmulBias>(mpattern); auto m_matmul = ngraph::pattern::Matcher::unique_match<ngraph::op::MatmulBias>(mpattern);
auto m_broadcast = ngraph::pattern::Matcher::unique_match<op::Broadcast>(mpattern); auto m_broadcast = ngraph::pattern::Matcher::unique_match<ngraph::op::Broadcast>(mpattern);
auto m_bias = m_broadcast->get_argument(0); auto m_bias = m_broadcast->get_argument(0);
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto mmb = std::make_shared<op::MatmulBias>(pattern_map[W], auto mmb = std::make_shared<ngraph::op::MatmulBias>(pattern_map[W],
pattern_map[x], pattern_map[x],
m_bias, m_bias,
m_matmul->get_a_shape(), m_matmul->get_a_shape(),
...@@ -186,12 +186,12 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_matmul() ...@@ -186,12 +186,12 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_matmul()
auto W = std::make_shared<pattern::op::Label>(element::f32, shape_w); auto W = std::make_shared<pattern::op::Label>(element::f32, shape_w);
auto x = std::make_shared<pattern::op::Label>(element::f32, shape_x); auto x = std::make_shared<pattern::op::Label>(element::f32, shape_x);
auto reshape_pred = pattern::has_class<op::Reshape>(); auto reshape_pred = pattern::has_class<ngraph::op::Reshape>();
auto skip_w = std::make_shared<pattern::op::Skip>(W, reshape_pred); auto skip_w = std::make_shared<pattern::op::Skip>(W, reshape_pred);
auto skip_x = std::make_shared<pattern::op::Skip>(x, reshape_pred); auto skip_x = std::make_shared<pattern::op::Skip>(x, reshape_pred);
auto pdot = std::make_shared<op::Dot>(skip_w, skip_x); auto pdot = std::make_shared<ngraph::op::Dot>(skip_w, skip_x);
ngraph::pattern::graph_rewrite_callback callback = [W, x](pattern::Matcher& m) { ngraph::pattern::graph_rewrite_callback callback = [W, x](pattern::Matcher& m) {
NGRAPH_DEBUG << "In callback for construct_matmul_pattern against node = " NGRAPH_DEBUG << "In callback for construct_matmul_pattern against node = "
...@@ -233,7 +233,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_matmul() ...@@ -233,7 +233,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_matmul()
return false; return false;
} }
auto cg = std::shared_ptr<Node>(new op::MatmulBias(pattern_map[W], auto cg = std::shared_ptr<Node>(new ngraph::op::MatmulBias(pattern_map[W],
pattern_map[x], pattern_map[x],
nullptr, nullptr,
shape_arg0, shape_arg0,
...@@ -252,47 +252,51 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_matmul() ...@@ -252,47 +252,51 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_matmul()
void ngraph::runtime::cpu::pass::CPUFusion::construct_fprop_bn() void ngraph::runtime::cpu::pass::CPUFusion::construct_fprop_bn()
{ {
// construct varaiance // construct varaiance
auto N = op::Constant::create(element::f32, Shape{3}, {2, 2, 2}); auto N = ngraph::op::Constant::create(element::f32, Shape{3}, {2, 2, 2});
auto input = std::make_shared<pattern::op::Label>(element::f32, Shape{2, 3}); auto input = std::make_shared<pattern::op::Label>(element::f32, Shape{2, 3});
auto input_sq = std::make_shared<op::Multiply>(input, input); auto input_sq = std::make_shared<ngraph::op::Multiply>(input, input);
auto sum_input = std::make_shared<op::Sum>(input, AxisSet{0}); auto sum_input = std::make_shared<ngraph::op::Sum>(input, AxisSet{0});
auto square_sumed_input = std::make_shared<op::Multiply>(sum_input, sum_input); auto square_sumed_input = std::make_shared<ngraph::op::Multiply>(sum_input, sum_input);
auto sum_squared_input = std::make_shared<op::Sum>(input_sq, AxisSet{0}); auto sum_squared_input = std::make_shared<ngraph::op::Sum>(input_sq, AxisSet{0});
auto avg_input_sum_sq = std::make_shared<op::Divide>(square_sumed_input, N); auto avg_input_sum_sq = std::make_shared<ngraph::op::Divide>(square_sumed_input, N);
auto xmu = std::make_shared<op::Subtract>(sum_squared_input, avg_input_sum_sq); auto xmu = std::make_shared<ngraph::op::Subtract>(sum_squared_input, avg_input_sum_sq);
auto variance = std::make_shared<op::Divide>(xmu, N); auto variance = std::make_shared<ngraph::op::Divide>(xmu, N);
auto variance_label = auto variance_label =
std::make_shared<pattern::op::Label>(variance, nullptr, NodeVector{variance}); std::make_shared<pattern::op::Label>(variance, nullptr, NodeVector{variance});
auto variance_with_broadcast = auto variance_with_broadcast =
std::make_shared<op::Broadcast>(variance_label, Shape{2, 3}, AxisSet{0}); std::make_shared<ngraph::op::Broadcast>(variance_label, Shape{2, 3}, AxisSet{0});
// construct mean // construct mean
auto sum_input1 = std::make_shared<op::Sum>(input, AxisSet{0}); auto sum_input1 = std::make_shared<ngraph::op::Sum>(input, AxisSet{0});
auto mean = std::make_shared<op::Divide>(sum_input1, N); auto mean = std::make_shared<ngraph::op::Divide>(sum_input1, N);
auto mean_label = std::make_shared<pattern::op::Label>(mean, nullptr, NodeVector{mean}); auto mean_label = std::make_shared<pattern::op::Label>(mean, nullptr, NodeVector{mean});
auto mean_with_broadcast = std::make_shared<op::Broadcast>(mean_label, Shape{2, 3}, AxisSet{0}); auto mean_with_broadcast =
auto input_diff_mean = std::make_shared<op::Subtract>(input, mean_with_broadcast); std::make_shared<ngraph::op::Broadcast>(mean_label, Shape{2, 3}, AxisSet{0});
auto input_diff_mean = std::make_shared<ngraph::op::Subtract>(input, mean_with_broadcast);
// Eps // Eps
auto eps_label = std::make_shared<pattern::op::Label>(element::f32, Shape{3}); auto eps_label = std::make_shared<pattern::op::Label>(element::f32, Shape{3});
auto eps_with_broadcast = std::make_shared<op::Broadcast>(eps_label, Shape{2, 3}, AxisSet{0}); auto eps_with_broadcast =
std::make_shared<ngraph::op::Broadcast>(eps_label, Shape{2, 3}, AxisSet{0});
auto add1 = std::make_shared<op::Add>(eps_with_broadcast, variance_with_broadcast); auto add1 = std::make_shared<ngraph::op::Add>(eps_with_broadcast, variance_with_broadcast);
auto sqrt_variance_eps = std::make_shared<op::Sqrt>(add1); auto sqrt_variance_eps = std::make_shared<ngraph::op::Sqrt>(add1);
auto divide_mean_variance = std::make_shared<op::Divide>(input_diff_mean, sqrt_variance_eps); auto divide_mean_variance =
std::make_shared<ngraph::op::Divide>(input_diff_mean, sqrt_variance_eps);
// Gamma // Gamma
auto gamma_label = std::make_shared<pattern::op::Label>(element::f32, Shape{3}); auto gamma_label = std::make_shared<pattern::op::Label>(element::f32, Shape{3});
auto gamma_with_broadcast = auto gamma_with_broadcast =
std::make_shared<op::Broadcast>(gamma_label, Shape{2, 3}, AxisSet{0}); std::make_shared<ngraph::op::Broadcast>(gamma_label, Shape{2, 3}, AxisSet{0});
auto multiply_gamma = auto multiply_gamma =
std::make_shared<op::Multiply>(gamma_with_broadcast, divide_mean_variance); std::make_shared<ngraph::op::Multiply>(gamma_with_broadcast, divide_mean_variance);
// Beta // Beta
auto beta_label = std::make_shared<pattern::op::Label>(element::f32, Shape{3}); auto beta_label = std::make_shared<pattern::op::Label>(element::f32, Shape{3});
auto beta_with_broadcast = std::make_shared<op::Broadcast>(beta_label, Shape{2, 3}, AxisSet{0}); auto beta_with_broadcast =
std::make_shared<ngraph::op::Broadcast>(beta_label, Shape{2, 3}, AxisSet{0});
auto add_beta = std::make_shared<op::Add>(beta_with_broadcast, multiply_gamma); auto add_beta = std::make_shared<ngraph::op::Add>(beta_with_broadcast, multiply_gamma);
// This completes fprop bn pattern // This completes fprop bn pattern
// Define a call back that needs to called once the DFG matches the pattern // Define a call back that needs to called once the DFG matches the pattern
...@@ -322,21 +326,22 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_fprop_bn() ...@@ -322,21 +326,22 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_fprop_bn()
Shape m_bn_variance_shape{pattern_map[variance_label]->get_shape()}; Shape m_bn_variance_shape{pattern_map[variance_label]->get_shape()};
// get epsilon value // get epsilon value
auto eps_ptr = std::dynamic_pointer_cast<op::Constant>(pattern_map[eps_label]); auto eps_ptr = std::dynamic_pointer_cast<ngraph::op::Constant>(pattern_map[eps_label]);
if (!eps_ptr) if (!eps_ptr)
{ {
NGRAPH_DEBUG << "Eps must be a constant"; NGRAPH_DEBUG << "Eps must be a constant";
return false; return false;
} }
double epsilon = *(reinterpret_cast<const double*>(eps_ptr->get_data_ptr())); double epsilon = *(reinterpret_cast<const double*>(eps_ptr->get_data_ptr()));
auto bn_node = std::make_shared<op::BatchNormTraining>( auto bn_node = std::make_shared<ngraph::op::BatchNormTraining>(
epsilon, pattern_map[gamma_label], pattern_map[beta_label], pattern_map[input]); epsilon, pattern_map[gamma_label], pattern_map[beta_label], pattern_map[input]);
if (!mkldnn_utils::can_use_mkldnn_batchnorm_fprop(bn_node.get())) if (!mkldnn_utils::can_use_mkldnn_batchnorm_fprop(bn_node.get()))
{ {
return false; return false;
} }
auto normalized_output = std::shared_ptr<Node>(new op::GetOutputElement(bn_node, 0)); auto normalized_output =
std::shared_ptr<Node>(new ngraph::op::GetOutputElement(bn_node, 0));
ngraph::replace_node(m.get_match_root(), normalized_output); ngraph::replace_node(m.get_match_root(), normalized_output);
return true; return true;
...@@ -407,16 +412,18 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_reshaped_conv( ...@@ -407,16 +412,18 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_reshaped_conv(
{ {
auto pad_input = std::make_shared<pattern::op::Label>(element::f32, Shape{}); auto pad_input = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto pad_value = std::make_shared<pattern::op::Label>(element::f32, Shape{}); auto pad_value = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto pad = std::make_shared<op::Pad>(pad_input, pad_value, CoordinateDiff{}, CoordinateDiff{}); auto pad =
std::make_shared<ngraph::op::Pad>(pad_input, pad_value, CoordinateDiff{}, CoordinateDiff{});
auto pad_label = std::make_shared<pattern::op::Label>(pad, nullptr, NodeVector{pad}); auto pad_label = std::make_shared<pattern::op::Label>(pad, nullptr, NodeVector{pad});
auto reshape = std::make_shared<op::Reshape>(pad_label, AxisVector{}, Shape{1, 1, 1, 1}); auto reshape =
std::make_shared<ngraph::op::Reshape>(pad_label, AxisVector{}, Shape{1, 1, 1, 1});
auto reshape_label = auto reshape_label =
std::make_shared<pattern::op::Label>(reshape, nullptr, NodeVector{reshape}); std::make_shared<pattern::op::Label>(reshape, nullptr, NodeVector{reshape});
auto conv_filter = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 1, 1}); auto conv_filter = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 1, 1});
auto conv = std::make_shared<op::Convolution>(reshape_label, auto conv = std::make_shared<ngraph::op::Convolution>(reshape_label,
conv_filter, conv_filter,
Strides{1, 1}, Strides{1, 1},
Strides{1, 1}, Strides{1, 1},
...@@ -425,12 +432,15 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_reshaped_conv( ...@@ -425,12 +432,15 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_reshaped_conv(
Strides{1, 1}); Strides{1, 1});
auto conv_label = std::make_shared<pattern::op::Label>(conv, nullptr, NodeVector{conv}); auto conv_label = std::make_shared<pattern::op::Label>(conv, nullptr, NodeVector{conv});
ngraph::pattern::graph_rewrite_callback callback = ngraph::pattern::graph_rewrite_callback callback = [pad_input,
[pad_input, pad_value, pad_label, reshape_label, conv_filter, conv_label]( pad_value,
pattern::Matcher& m) { pad_label,
reshape_label,
conv_filter,
conv_label](pattern::Matcher& m) {
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto pad_value_op = std::dynamic_pointer_cast<op::Constant>(pattern_map[pad_value]); auto pad_value_op = std::dynamic_pointer_cast<ngraph::op::Constant>(pattern_map[pad_value]);
if (!pad_value_op) if (!pad_value_op)
{ {
NGRAPH_DEBUG << "Pad value must be a constant"; NGRAPH_DEBUG << "Pad value must be a constant";
...@@ -438,16 +448,16 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_reshaped_conv( ...@@ -438,16 +448,16 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_reshaped_conv(
} }
const auto& matched_conv = const auto& matched_conv =
std::static_pointer_cast<op::Convolution>(pattern_map[conv_label]); std::static_pointer_cast<ngraph::op::Convolution>(pattern_map[conv_label]);
const auto& matched_pad = std::static_pointer_cast<op::Pad>(pattern_map[pad_label]); const auto& matched_pad = std::static_pointer_cast<ngraph::op::Pad>(pattern_map[pad_label]);
const auto& matched_reshape = const auto& matched_reshape =
std::static_pointer_cast<op::Reshape>(pattern_map[reshape_label]); std::static_pointer_cast<ngraph::op::Reshape>(pattern_map[reshape_label]);
const auto& input_order = matched_reshape->get_input_order(); const auto& input_order = matched_reshape->get_input_order();
auto hoisted_reshape_output_shape = auto hoisted_reshape_output_shape =
ngraph::apply_permutation<Shape>(pattern_map[pad_input]->get_shape(), input_order); ngraph::apply_permutation<Shape>(pattern_map[pad_input]->get_shape(), input_order);
auto hoisted_reshape = std::make_shared<op::Reshape>( auto hoisted_reshape = std::make_shared<ngraph::op::Reshape>(
pattern_map[pad_input], pattern_map[pad_input],
input_order, input_order,
Shape(hoisted_reshape_output_shape.begin(), hoisted_reshape_output_shape.end())); Shape(hoisted_reshape_output_shape.begin(), hoisted_reshape_output_shape.end()));
...@@ -474,7 +484,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_reshaped_conv( ...@@ -474,7 +484,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_reshaped_conv(
matched_pad->get_padding_above().at(input_order[3]))}; matched_pad->get_padding_above().at(input_order[3]))};
auto zero_padded_conv = auto zero_padded_conv =
std::make_shared<op::Convolution>(hoisted_reshape, std::make_shared<ngraph::op::Convolution>(hoisted_reshape,
pattern_map[conv_filter], pattern_map[conv_filter],
matched_conv->get_window_movement_strides(), matched_conv->get_window_movement_strides(),
matched_conv->get_window_dilation_strides(), matched_conv->get_window_dilation_strides(),
...@@ -494,13 +504,13 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv() ...@@ -494,13 +504,13 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv()
{ {
auto pad_input = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 1, 1}); auto pad_input = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 1, 1});
auto pad_value = std::make_shared<pattern::op::Label>(element::f32, Shape{}); auto pad_value = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto pad = std::make_shared<op::Pad>( auto pad = std::make_shared<ngraph::op::Pad>(
pad_input, pad_value, CoordinateDiff{0, 0, 0, 0}, CoordinateDiff{0, 0, 0, 0}); pad_input, pad_value, CoordinateDiff{0, 0, 0, 0}, CoordinateDiff{0, 0, 0, 0});
auto pad_label = std::make_shared<pattern::op::Label>(pad, nullptr, NodeVector{pad}); auto pad_label = std::make_shared<pattern::op::Label>(pad, nullptr, NodeVector{pad});
auto conv_filter = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 1, 1}); auto conv_filter = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 1, 1});
auto conv = std::make_shared<op::Convolution>(pad_label, auto conv = std::make_shared<ngraph::op::Convolution>(pad_label,
conv_filter, conv_filter,
Strides{1, 1}, Strides{1, 1},
Strides{1, 1}, Strides{1, 1},
...@@ -509,11 +519,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv() ...@@ -509,11 +519,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv()
Strides{1, 1}); Strides{1, 1});
auto conv_label = std::make_shared<pattern::op::Label>(conv, nullptr, NodeVector{conv}); auto conv_label = std::make_shared<pattern::op::Label>(conv, nullptr, NodeVector{conv});
ngraph::pattern::graph_rewrite_callback callback = ngraph::pattern::graph_rewrite_callback callback = [pad_input,
[pad_input, pad_value, pad_label, conv_filter, conv_label](pattern::Matcher& m) { pad_value,
pad_label,
conv_filter,
conv_label](pattern::Matcher& m) {
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto pad_value_op = std::dynamic_pointer_cast<op::Constant>(pattern_map[pad_value]); auto pad_value_op = std::dynamic_pointer_cast<ngraph::op::Constant>(pattern_map[pad_value]);
if (!pad_value_op) if (!pad_value_op)
{ {
NGRAPH_DEBUG << "Pad value must be a constant"; NGRAPH_DEBUG << "Pad value must be a constant";
...@@ -521,8 +534,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv() ...@@ -521,8 +534,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv()
} }
const auto& matched_conv = const auto& matched_conv =
std::static_pointer_cast<op::Convolution>(pattern_map[conv_label]); std::static_pointer_cast<ngraph::op::Convolution>(pattern_map[conv_label]);
const auto& matched_pad = std::static_pointer_cast<op::Pad>(pattern_map[pad_label]); const auto& matched_pad = std::static_pointer_cast<ngraph::op::Pad>(pattern_map[pad_label]);
if (!zero_padded_conv_consistency_check(m.get_match_root(), if (!zero_padded_conv_consistency_check(m.get_match_root(),
pad_value_op, pad_value_op,
...@@ -544,7 +557,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv() ...@@ -544,7 +557,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv()
static_cast<CoordinateDiff::value_type>(matched_pad->get_padding_above().at(3))}; static_cast<CoordinateDiff::value_type>(matched_pad->get_padding_above().at(3))};
auto zero_padded_conv = auto zero_padded_conv =
std::make_shared<op::Convolution>(pattern_map[pad_input], std::make_shared<ngraph::op::Convolution>(pattern_map[pad_input],
pattern_map[conv_filter], pattern_map[conv_filter],
matched_conv->get_window_movement_strides(), matched_conv->get_window_movement_strides(),
matched_conv->get_window_dilation_strides(), matched_conv->get_window_dilation_strides(),
...@@ -564,13 +577,13 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv_backprop_ ...@@ -564,13 +577,13 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv_backprop_
{ {
auto pad_input = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 1, 1}); auto pad_input = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 1, 1});
auto pad_value = std::make_shared<pattern::op::Label>(element::f32, Shape{}); auto pad_value = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto pad = std::make_shared<op::Pad>( auto pad = std::make_shared<ngraph::op::Pad>(
pad_input, pad_value, CoordinateDiff{0, 0, 0, 0}, CoordinateDiff{0, 0, 0, 0}); pad_input, pad_value, CoordinateDiff{0, 0, 0, 0}, CoordinateDiff{0, 0, 0, 0});
auto pad_label = std::make_shared<pattern::op::Label>(pad, nullptr, NodeVector{pad}); auto pad_label = std::make_shared<pattern::op::Label>(pad, nullptr, NodeVector{pad});
auto output_delta = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 1, 1}); auto output_delta = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 1, 1});
auto conv = std::make_shared<op::ConvolutionBackpropFilters>(pad_label, auto conv = std::make_shared<ngraph::op::ConvolutionBackpropFilters>(pad_label,
Shape{1, 1, 3, 3}, Shape{1, 1, 3, 3},
output_delta, output_delta,
Strides{1, 1}, Strides{1, 1},
...@@ -580,20 +593,23 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv_backprop_ ...@@ -580,20 +593,23 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv_backprop_
Strides{1, 1}); Strides{1, 1});
auto conv_label = std::make_shared<pattern::op::Label>(conv, nullptr, NodeVector{conv}); auto conv_label = std::make_shared<pattern::op::Label>(conv, nullptr, NodeVector{conv});
ngraph::pattern::graph_rewrite_callback callback = ngraph::pattern::graph_rewrite_callback callback = [pad_input,
[pad_input, pad_value, pad_label, output_delta, conv_label](pattern::Matcher& m) { pad_value,
pad_label,
output_delta,
conv_label](pattern::Matcher& m) {
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto pad_value_op = std::dynamic_pointer_cast<op::Constant>(pattern_map[pad_value]); auto pad_value_op = std::dynamic_pointer_cast<ngraph::op::Constant>(pattern_map[pad_value]);
if (!pad_value_op) if (!pad_value_op)
{ {
NGRAPH_DEBUG << "Pad value must be a constant"; NGRAPH_DEBUG << "Pad value must be a constant";
return false; return false;
} }
const auto& matched_conv = const auto& matched_conv = std::static_pointer_cast<ngraph::op::ConvolutionBackpropFilters>(
std::static_pointer_cast<op::ConvolutionBackpropFilters>(pattern_map[conv_label]); pattern_map[conv_label]);
const auto& matched_pad = std::static_pointer_cast<op::Pad>(pattern_map[pad_label]); const auto& matched_pad = std::static_pointer_cast<ngraph::op::Pad>(pattern_map[pad_label]);
if (!zero_padded_conv_consistency_check(m.get_match_root(), if (!zero_padded_conv_consistency_check(m.get_match_root(),
pad_value_op, pad_value_op,
...@@ -615,7 +631,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv_backprop_ ...@@ -615,7 +631,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_zero_padded_conv_backprop_
static_cast<CoordinateDiff::value_type>(matched_pad->get_padding_above().at(3))}; static_cast<CoordinateDiff::value_type>(matched_pad->get_padding_above().at(3))};
auto zero_padded_conv_backprop_filters = auto zero_padded_conv_backprop_filters =
std::make_shared<op::ConvolutionBackpropFilters>( std::make_shared<ngraph::op::ConvolutionBackpropFilters>(
pattern_map[pad_input], pattern_map[pad_input],
matched_conv->get_filters_shape(), matched_conv->get_filters_shape(),
pattern_map[output_delta], pattern_map[output_delta],
...@@ -640,9 +656,9 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias() ...@@ -640,9 +656,9 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias()
auto filters = std::make_shared<pattern::op::Label>(element::f32, shape); auto filters = std::make_shared<pattern::op::Label>(element::f32, shape);
auto pbias = std::make_shared<pattern::op::Label>(element::f32, Shape{}); auto pbias = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto pbroadcast = std::make_shared<op::Broadcast>(pbias, shape, AxisSet{0, 1, 2, 3}); auto pbroadcast = std::make_shared<ngraph::op::Broadcast>(pbias, shape, AxisSet{0, 1, 2, 3});
auto pconv1 = std::make_shared<op::Convolution>(data_batch, auto pconv1 = std::make_shared<ngraph::op::Convolution>(data_batch,
filters, filters,
Strides{1, 1}, Strides{1, 1},
Strides{1, 1}, Strides{1, 1},
...@@ -656,9 +672,10 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias() ...@@ -656,9 +672,10 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias()
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto conv = std::static_pointer_cast<op::Convolution>(m.get_match_root()->get_argument(0)); auto conv =
std::static_pointer_cast<ngraph::op::Convolution>(m.get_match_root()->get_argument(0));
if (!runtime::cpu::mkldnn_utils::can_use_mkldnn_conv<op::Convolution>(conv.get())) if (!runtime::cpu::mkldnn_utils::can_use_mkldnn_conv<ngraph::op::Convolution>(conv.get()))
{ {
NGRAPH_DEBUG << "Convolution not supported by MKLDNN"; NGRAPH_DEBUG << "Convolution not supported by MKLDNN";
return false; return false;
...@@ -671,14 +688,15 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias() ...@@ -671,14 +688,15 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias()
NGRAPH_DEBUG << "mpattern = " << m.get_match_root()->get_name() NGRAPH_DEBUG << "mpattern = " << m.get_match_root()->get_name()
<< "conv_bias bias shape != 1, requires reshape to match filter count."; << "conv_bias bias shape != 1, requires reshape to match filter count.";
auto order = ngraph::get_default_order(bias_shape); auto order = ngraph::get_default_order(bias_shape);
auto bias_reshape = auto bias_reshape = std::make_shared<ngraph::op::Reshape>(
std::make_shared<op::Reshape>(bias, order, Shape{conv->get_input_shape(1)[0]}); bias, order, Shape{conv->get_input_shape(1)[0]});
auto conv_bias = std::shared_ptr<Node>(new op::ConvolutionBias(conv, bias_reshape)); auto conv_bias =
std::shared_ptr<Node>(new ngraph::op::ConvolutionBias(conv, bias_reshape));
ngraph::replace_node(m.get_match_root(), conv_bias); ngraph::replace_node(m.get_match_root(), conv_bias);
} }
else else
{ {
auto conv_bias = std::shared_ptr<Node>(new op::ConvolutionBias(conv, bias)); auto conv_bias = std::shared_ptr<Node>(new ngraph::op::ConvolutionBias(conv, bias));
ngraph::replace_node(m.get_match_root(), conv_bias); ngraph::replace_node(m.get_match_root(), conv_bias);
} }
return true; return true;
...@@ -694,7 +712,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop() ...@@ -694,7 +712,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop()
Shape shape{2, 2, 1, 1}; Shape shape{2, 2, 1, 1};
auto data_batch = std::make_shared<pattern::op::Label>(element::f32, shape); auto data_batch = std::make_shared<pattern::op::Label>(element::f32, shape);
auto delta = std::make_shared<pattern::op::Label>(element::f32, shape); auto delta = std::make_shared<pattern::op::Label>(element::f32, shape);
auto conv_bprop_filter = std::make_shared<op::ConvolutionBackpropFilters>(data_batch, auto conv_bprop_filter =
std::make_shared<ngraph::op::ConvolutionBackpropFilters>(data_batch,
shape, shape,
delta, delta,
Strides{1, 1}, Strides{1, 1},
...@@ -709,7 +728,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop() ...@@ -709,7 +728,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop()
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto conv_bprop = auto conv_bprop =
std::static_pointer_cast<op::ConvolutionBackpropFilters>(m.get_match_root()); std::static_pointer_cast<ngraph::op::ConvolutionBackpropFilters>(m.get_match_root());
if (conv_bprop->get_input_shape(0).size() == 4 && if (conv_bprop->get_input_shape(0).size() == 4 &&
conv_bprop->get_input_shape(1).size() == 4 && conv_bprop->get_input_shape(1).size() == 4 &&
...@@ -717,9 +736,9 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop() ...@@ -717,9 +736,9 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop()
{ {
for (auto delta_user : pattern_map[delta]->get_users()) for (auto delta_user : pattern_map[delta]->get_users())
{ {
if (std::dynamic_pointer_cast<op::Sum>(delta_user)) if (std::dynamic_pointer_cast<ngraph::op::Sum>(delta_user))
{ {
auto bias = std::dynamic_pointer_cast<op::Sum>(delta_user); auto bias = std::dynamic_pointer_cast<ngraph::op::Sum>(delta_user);
auto bias_shape = bias->get_shape(); auto bias_shape = bias->get_shape();
bool flag = false; bool flag = false;
if (bias_shape.size() > 1) if (bias_shape.size() > 1)
...@@ -728,12 +747,13 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop() ...@@ -728,12 +747,13 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop()
<< "mpattern = " << m.get_match_root()->get_name() << "mpattern = " << m.get_match_root()->get_name()
<< "conv_bias bias shape != 1, requires reshape to match filter count."; << "conv_bias bias shape != 1, requires reshape to match filter count.";
auto order = ngraph::get_default_order(bias_shape); auto order = ngraph::get_default_order(bias_shape);
auto bias_reshape = std::make_shared<op::Reshape>( auto bias_reshape = std::make_shared<ngraph::op::Reshape>(
bias, order, Shape{conv_bprop->get_filters_shape()[0]}); bias, order, Shape{conv_bprop->get_filters_shape()[0]});
bias_shape = bias_reshape->get_shape(); bias_shape = bias_reshape->get_shape();
flag = true; flag = true;
} }
auto conv_bias_bprop = std::make_shared<op::ConvolutionBiasBackpropFiltersBias>( auto conv_bias_bprop =
std::make_shared<ngraph::op::ConvolutionBiasBackpropFiltersBias>(
pattern_map[data_batch], pattern_map[data_batch],
conv_bprop->get_filters_shape(), conv_bprop->get_filters_shape(),
bias_shape, bias_shape,
...@@ -743,8 +763,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop() ...@@ -743,8 +763,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop()
conv_bprop->get_padding_below_forward(), conv_bprop->get_padding_below_forward(),
conv_bprop->get_padding_above_forward(), conv_bprop->get_padding_above_forward(),
conv_bprop->get_data_dilation_strides_forward()); conv_bprop->get_data_dilation_strides_forward());
auto goe1 = std::make_shared<op::GetOutputElement>(conv_bias_bprop, 0); auto goe1 = std::make_shared<ngraph::op::GetOutputElement>(conv_bias_bprop, 0);
auto goe2 = std::make_shared<op::GetOutputElement>(conv_bias_bprop, 1); auto goe2 = std::make_shared<ngraph::op::GetOutputElement>(conv_bias_bprop, 1);
NGRAPH_DEBUG << "Replacing " << m.get_match_root()->get_name() NGRAPH_DEBUG << "Replacing " << m.get_match_root()->get_name()
<< "with ConvolutionBiasBackpropFiltersBias"; << "with ConvolutionBiasBackpropFiltersBias";
ngraph::replace_node(m.get_match_root(), goe1); ngraph::replace_node(m.get_match_root(), goe1);
...@@ -752,7 +772,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop() ...@@ -752,7 +772,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_bprop()
"ConvolutionBiasBackpropFiltersBias"; "ConvolutionBiasBackpropFiltersBias";
if (flag) if (flag)
{ {
auto goe2_reshape = std::make_shared<op::Reshape>( auto goe2_reshape = std::make_shared<ngraph::op::Reshape>(
goe2, AxisVector{0}, delta_user->get_shape()); goe2, AxisVector{0}, delta_user->get_shape());
ngraph::replace_node(delta_user, goe2_reshape); ngraph::replace_node(delta_user, goe2_reshape);
} }
...@@ -783,16 +803,16 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_batch_norm_relu() ...@@ -783,16 +803,16 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_batch_norm_relu()
auto beta_shape = Shape{2}; auto beta_shape = Shape{2};
auto beta = std::make_shared<pattern::op::Label>(element::f32, beta_shape); auto beta = std::make_shared<pattern::op::Label>(element::f32, beta_shape);
double eps = 0.001; double eps = 0.001;
auto bn = std::make_shared<op::BatchNormTraining>(eps, gamma, beta, input); auto bn = std::make_shared<ngraph::op::BatchNormTraining>(eps, gamma, beta, input);
auto goe = std::make_shared<op::GetOutputElement>(bn, 0); auto goe = std::make_shared<ngraph::op::GetOutputElement>(bn, 0);
auto prelu = std::make_shared<op::Relu>(goe); auto prelu = std::make_shared<ngraph::op::Relu>(goe);
ngraph::pattern::graph_rewrite_callback callback = [input, gamma, beta](pattern::Matcher& m) { ngraph::pattern::graph_rewrite_callback callback = [input, gamma, beta](pattern::Matcher& m) {
NGRAPH_DEBUG << "In callback for construct_batch_norm_relu against node = " NGRAPH_DEBUG << "In callback for construct_batch_norm_relu against node = "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto m_bn = std::static_pointer_cast<op::BatchNormTraining>( auto m_bn = std::static_pointer_cast<ngraph::op::BatchNormTraining>(
m.get_match_root()->get_argument(0)->get_inputs().at(0).get_output().get_node()); m.get_match_root()->get_argument(0)->get_inputs().at(0).get_output().get_node());
if (!mkldnn_utils::can_use_mkldnn_batchnorm_fprop(m_bn.get())) if (!mkldnn_utils::can_use_mkldnn_batchnorm_fprop(m_bn.get()))
...@@ -802,7 +822,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_batch_norm_relu() ...@@ -802,7 +822,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_batch_norm_relu()
std::vector<std::shared_ptr<Node>> mgoes(m_bn->get_outputs().size()); std::vector<std::shared_ptr<Node>> mgoes(m_bn->get_outputs().size());
for (auto bn_in : m_bn->get_output_inputs(0)) for (auto bn_in : m_bn->get_output_inputs(0))
{ {
auto mgoe = std::dynamic_pointer_cast<op::GetOutputElement>(bn_in->get_node()); auto mgoe = std::dynamic_pointer_cast<ngraph::op::GetOutputElement>(bn_in->get_node());
NGRAPH_ASSERT(mgoe); NGRAPH_ASSERT(mgoe);
mgoes[mgoe->get_n()] = mgoe; mgoes[mgoe->get_n()] = mgoe;
} }
...@@ -815,12 +835,12 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_batch_norm_relu() ...@@ -815,12 +835,12 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_batch_norm_relu()
mgoes[0] = m.get_match_root(); // replace relu instead of its GetOutputElement mgoes[0] = m.get_match_root(); // replace relu instead of its GetOutputElement
auto bn_relu = std::make_shared<op::BatchNormTrainingRelu>( auto bn_relu = std::make_shared<ngraph::op::BatchNormTrainingRelu>(
m_bn->get_eps_value(), pattern_map[gamma], pattern_map[beta], pattern_map[input]); m_bn->get_eps_value(), pattern_map[gamma], pattern_map[beta], pattern_map[input]);
auto bn_relu_output = std::make_shared<op::GetOutputElement>(bn_relu, 0); auto bn_relu_output = std::make_shared<ngraph::op::GetOutputElement>(bn_relu, 0);
auto bn_relu_mean = std::make_shared<op::GetOutputElement>(bn_relu, 1); auto bn_relu_mean = std::make_shared<ngraph::op::GetOutputElement>(bn_relu, 1);
auto bn_relu_var = std::make_shared<op::GetOutputElement>(bn_relu, 2); auto bn_relu_var = std::make_shared<ngraph::op::GetOutputElement>(bn_relu, 2);
std::shared_ptr<Node> new_nodes[] = {bn_relu_output, bn_relu_mean, bn_relu_var}; std::shared_ptr<Node> new_nodes[] = {bn_relu_output, bn_relu_mean, bn_relu_var};
...@@ -848,12 +868,12 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_batch_norm_relu_global_sta ...@@ -848,12 +868,12 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_batch_norm_relu_global_sta
auto beta_shape = Shape{2}; auto beta_shape = Shape{2};
auto beta = std::make_shared<pattern::op::Label>(element::f32, beta_shape); auto beta = std::make_shared<pattern::op::Label>(element::f32, beta_shape);
auto bn_pred = [](std::shared_ptr<Node> node) { auto bn_pred = [](std::shared_ptr<Node> node) {
return pattern::has_class<op::BatchNormInference>()(node) || return pattern::has_class<ngraph::op::BatchNormInference>()(node) ||
pattern::has_class<op::BatchNormTraining>()(node); pattern::has_class<ngraph::op::BatchNormTraining>()(node);
}; };
auto bn = std::make_shared<pattern::op::Any>( auto bn = std::make_shared<pattern::op::Any>(
input, bn_pred, NodeVector{gamma, beta, input, mean, var}); input, bn_pred, NodeVector{gamma, beta, input, mean, var});
auto prelu = std::make_shared<op::Relu>(bn); auto prelu = std::make_shared<ngraph::op::Relu>(bn);
ngraph::pattern::graph_rewrite_callback callback = [input, mean, var, gamma, beta]( ngraph::pattern::graph_rewrite_callback callback = [input, mean, var, gamma, beta](
pattern::Matcher& m) { pattern::Matcher& m) {
...@@ -870,13 +890,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_batch_norm_relu_global_sta ...@@ -870,13 +890,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_batch_norm_relu_global_sta
} }
std::shared_ptr<Node> bn_relu; std::shared_ptr<Node> bn_relu;
if (auto bn_inference = std::dynamic_pointer_cast<op::BatchNormInference>(bn_match)) if (auto bn_inference = std::dynamic_pointer_cast<ngraph::op::BatchNormInference>(bn_match))
{ {
if (!mkldnn_utils::can_use_mkldnn_batchnorm_fprop(bn_inference.get())) if (!mkldnn_utils::can_use_mkldnn_batchnorm_fprop(bn_inference.get()))
{ {
return false; return false;
} }
bn_relu = std::make_shared<op::BatchNormInferenceRelu>(bn_inference->get_eps_value(), bn_relu =
std::make_shared<ngraph::op::BatchNormInferenceRelu>(bn_inference->get_eps_value(),
pattern_map[gamma], pattern_map[gamma],
pattern_map[beta], pattern_map[beta],
pattern_map[input], pattern_map[input],
...@@ -904,7 +925,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_relu() ...@@ -904,7 +925,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_relu()
auto data_batch = std::make_shared<pattern::op::Label>(element::f32, shape); auto data_batch = std::make_shared<pattern::op::Label>(element::f32, shape);
auto filters = std::make_shared<pattern::op::Label>(element::f32, shape); auto filters = std::make_shared<pattern::op::Label>(element::f32, shape);
auto pconv = std::make_shared<op::Convolution>(data_batch, auto pconv = std::make_shared<ngraph::op::Convolution>(data_batch,
filters, filters,
Strides{1, 1}, Strides{1, 1},
Strides{1, 1}, Strides{1, 1},
...@@ -912,15 +933,16 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_relu() ...@@ -912,15 +933,16 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_relu()
CoordinateDiff{0, 0}, CoordinateDiff{0, 0},
Strides{1, 1}); Strides{1, 1});
auto prelu = std::make_shared<op::Relu>(pconv); auto prelu = std::make_shared<ngraph::op::Relu>(pconv);
pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_conv_relu against " NGRAPH_DEBUG << "In a callback for construct_conv_relu against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto conv = std::static_pointer_cast<op::Convolution>(m.get_match_root()->get_argument(0)); auto conv =
std::static_pointer_cast<ngraph::op::Convolution>(m.get_match_root()->get_argument(0));
if (!runtime::cpu::mkldnn_utils::can_use_mkldnn_conv<op::Convolution>(conv.get())) if (!runtime::cpu::mkldnn_utils::can_use_mkldnn_conv<ngraph::op::Convolution>(conv.get()))
{ {
NGRAPH_DEBUG << "Convolution not supported by MKLDNN"; NGRAPH_DEBUG << "Convolution not supported by MKLDNN";
return false; return false;
...@@ -932,7 +954,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_relu() ...@@ -932,7 +954,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_relu()
return false; return false;
} }
auto conv_relu = std::shared_ptr<Node>(new op::ConvolutionRelu(conv)); auto conv_relu = std::shared_ptr<Node>(new ngraph::op::ConvolutionRelu(conv));
ngraph::replace_node(m.get_match_root(), conv_relu); ngraph::replace_node(m.get_match_root(), conv_relu);
return true; return true;
}; };
...@@ -948,7 +970,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_relu() ...@@ -948,7 +970,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_relu()
auto filters = std::make_shared<pattern::op::Label>(element::f32, shape); auto filters = std::make_shared<pattern::op::Label>(element::f32, shape);
auto bias = std::make_shared<pattern::op::Label>(element::f32, Shape{shape[0]}); auto bias = std::make_shared<pattern::op::Label>(element::f32, Shape{shape[0]});
auto conv_bias = std::make_shared<op::ConvolutionBias>(data_batch, auto conv_bias = std::make_shared<ngraph::op::ConvolutionBias>(data_batch,
filters, filters,
bias, bias,
Strides{1, 1}, Strides{1, 1},
...@@ -957,14 +979,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_relu() ...@@ -957,14 +979,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_relu()
CoordinateDiff{0, 0}, CoordinateDiff{0, 0},
Strides{1, 1}); Strides{1, 1});
auto prelu = std::make_shared<op::Relu>(conv_bias); auto prelu = std::make_shared<ngraph::op::Relu>(conv_bias);
pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_conv_relu against " NGRAPH_DEBUG << "In a callback for construct_conv_relu against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto conv = auto conv = std::static_pointer_cast<ngraph::op::ConvolutionBias>(
std::static_pointer_cast<op::ConvolutionBias>(m.get_match_root()->get_argument(0)); m.get_match_root()->get_argument(0));
if (conv->get_users().size() > 1) if (conv->get_users().size() > 1)
{ {
...@@ -974,7 +996,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_relu() ...@@ -974,7 +996,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_relu()
// ConvolutionBias created only if it can run with MKLDNN. // ConvolutionBias created only if it can run with MKLDNN.
// No further checks needed. // No further checks needed.
auto conv_relu = std::make_shared<op::ConvolutionBias>(conv->get_argument(0), auto conv_relu =
std::make_shared<ngraph::op::ConvolutionBias>(conv->get_argument(0),
conv->get_argument(1), conv->get_argument(1),
conv->get_argument(2), conv->get_argument(2),
conv->get_window_movement_strides(), conv->get_window_movement_strides(),
...@@ -997,7 +1020,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add() ...@@ -997,7 +1020,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add()
auto data_batch = std::make_shared<pattern::op::Label>(element::f32, shape); auto data_batch = std::make_shared<pattern::op::Label>(element::f32, shape);
auto filters = std::make_shared<pattern::op::Label>(element::f32, shape); auto filters = std::make_shared<pattern::op::Label>(element::f32, shape);
auto pconv = std::make_shared<op::Convolution>(data_batch, auto pconv = std::make_shared<ngraph::op::Convolution>(data_batch,
filters, filters,
Strides{1, 1}, Strides{1, 1},
Strides{1, 1}, Strides{1, 1},
...@@ -1005,7 +1028,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add() ...@@ -1005,7 +1028,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add()
CoordinateDiff{0, 0}, CoordinateDiff{0, 0},
Strides{1, 1}); Strides{1, 1});
auto add_input = std::make_shared<pattern::op::Label>(element::f32, pconv->get_shape()); auto add_input = std::make_shared<pattern::op::Label>(element::f32, pconv->get_shape());
auto padd = std::make_shared<op::Add>(add_input, pconv); auto padd = std::make_shared<ngraph::op::Add>(add_input, pconv);
pattern::graph_rewrite_callback callback = [data_batch, filters](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [data_batch, filters](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_conv_add against " NGRAPH_DEBUG << "In a callback for construct_conv_add against "
...@@ -1013,16 +1036,16 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add() ...@@ -1013,16 +1036,16 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add()
auto add_m = m.get_match_root(); auto add_m = m.get_match_root();
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto conv_m = std::dynamic_pointer_cast<op::Convolution>(add_m->get_argument(1)); auto conv_m = std::dynamic_pointer_cast<ngraph::op::Convolution>(add_m->get_argument(1));
auto inplace_input = add_m->get_argument(0); auto inplace_input = add_m->get_argument(0);
if (!conv_m) if (!conv_m)
{ {
conv_m = std::dynamic_pointer_cast<op::Convolution>(add_m->get_argument(0)); conv_m = std::dynamic_pointer_cast<ngraph::op::Convolution>(add_m->get_argument(0));
inplace_input = add_m->get_argument(1); inplace_input = add_m->get_argument(1);
} }
if (!runtime::cpu::mkldnn_utils::can_use_mkldnn_conv<op::Convolution>(conv_m.get())) if (!runtime::cpu::mkldnn_utils::can_use_mkldnn_conv<ngraph::op::Convolution>(conv_m.get()))
{ {
NGRAPH_DEBUG << "Convolution not supported by MKLDNN"; NGRAPH_DEBUG << "Convolution not supported by MKLDNN";
return false; return false;
...@@ -1048,7 +1071,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add() ...@@ -1048,7 +1071,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add()
return false; return false;
} }
auto conv_add = std::shared_ptr<Node>(new op::ConvolutionAdd(conv_m, inplace_input, false)); auto conv_add =
std::shared_ptr<Node>(new ngraph::op::ConvolutionAdd(conv_m, inplace_input, false));
ngraph::replace_node(m.get_match_root(), conv_add); ngraph::replace_node(m.get_match_root(), conv_add);
return true; return true;
}; };
...@@ -1064,7 +1088,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add_relu() ...@@ -1064,7 +1088,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add_relu()
auto filters = std::make_shared<pattern::op::Label>(element::f32, shape); auto filters = std::make_shared<pattern::op::Label>(element::f32, shape);
auto add_input = std::make_shared<pattern::op::Label>(element::f32, shape); auto add_input = std::make_shared<pattern::op::Label>(element::f32, shape);
auto pconv = std::make_shared<op::ConvolutionAdd>(data_batch, auto pconv = std::make_shared<ngraph::op::ConvolutionAdd>(data_batch,
filters, filters,
add_input, add_input,
Strides{1, 1}, Strides{1, 1},
...@@ -1073,14 +1097,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add_relu() ...@@ -1073,14 +1097,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add_relu()
CoordinateDiff{0, 0}, CoordinateDiff{0, 0},
Strides{1, 1}, Strides{1, 1},
false); false);
auto prelu = std::make_shared<op::Relu>(pconv); auto prelu = std::make_shared<ngraph::op::Relu>(pconv);
pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_conv_add_relu against " NGRAPH_DEBUG << "In a callback for construct_conv_add_relu against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto conv_m = auto conv_m = std::static_pointer_cast<ngraph::op::ConvolutionAdd>(
std::static_pointer_cast<op::ConvolutionAdd>(m.get_match_root()->get_argument(0)); m.get_match_root()->get_argument(0));
if (conv_m->get_users().size() > 1) if (conv_m->get_users().size() > 1)
{ {
NGRAPH_DEBUG << "Convolution has more than one user"; NGRAPH_DEBUG << "Convolution has more than one user";
...@@ -1089,7 +1113,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add_relu() ...@@ -1089,7 +1113,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_add_relu()
// ConvolutionAdd created only if it can run with MKLDNN. // ConvolutionAdd created only if it can run with MKLDNN.
// No further checks needed. // No further checks needed.
auto conv_n = std::make_shared<op::ConvolutionAdd>(conv_m->get_argument(0), auto conv_n =
std::make_shared<ngraph::op::ConvolutionAdd>(conv_m->get_argument(0),
conv_m->get_argument(1), conv_m->get_argument(1),
conv_m->get_argument(2), conv_m->get_argument(2),
conv_m->get_window_movement_strides(), conv_m->get_window_movement_strides(),
...@@ -1113,7 +1138,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add() ...@@ -1113,7 +1138,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add()
auto filters = std::make_shared<pattern::op::Label>(element::f32, shape); auto filters = std::make_shared<pattern::op::Label>(element::f32, shape);
auto bias = std::make_shared<pattern::op::Label>(element::f32, Shape{shape[0]}); auto bias = std::make_shared<pattern::op::Label>(element::f32, Shape{shape[0]});
auto pconv = std::make_shared<op::ConvolutionBias>(data_batch, auto pconv = std::make_shared<ngraph::op::ConvolutionBias>(data_batch,
filters, filters,
bias, bias,
Strides{1, 1}, Strides{1, 1},
...@@ -1122,7 +1147,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add() ...@@ -1122,7 +1147,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add()
CoordinateDiff{0, 0}, CoordinateDiff{0, 0},
Strides{1, 1}); Strides{1, 1});
auto add_input = std::make_shared<pattern::op::Label>(element::f32, pconv->get_shape()); auto add_input = std::make_shared<pattern::op::Label>(element::f32, pconv->get_shape());
auto padd = std::make_shared<op::Add>(add_input, pconv); auto padd = std::make_shared<ngraph::op::Add>(add_input, pconv);
pattern::graph_rewrite_callback callback = [data_batch, filters](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [data_batch, filters](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_conv_sum against " NGRAPH_DEBUG << "In a callback for construct_conv_sum against "
...@@ -1130,16 +1155,18 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add() ...@@ -1130,16 +1155,18 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add()
auto add_m = m.get_match_root(); auto add_m = m.get_match_root();
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto conv_m = std::dynamic_pointer_cast<op::ConvolutionBias>(add_m->get_argument(1)); auto conv_m =
std::dynamic_pointer_cast<ngraph::op::ConvolutionBias>(add_m->get_argument(1));
auto inplace_input = add_m->get_argument(0); auto inplace_input = add_m->get_argument(0);
if (!conv_m) if (!conv_m)
{ {
conv_m = std::dynamic_pointer_cast<op::ConvolutionBias>(add_m->get_argument(0)); conv_m = std::dynamic_pointer_cast<ngraph::op::ConvolutionBias>(add_m->get_argument(0));
inplace_input = add_m->get_argument(1); inplace_input = add_m->get_argument(1);
} }
if (!runtime::cpu::mkldnn_utils::can_use_mkldnn_conv<op::ConvolutionBias>(conv_m.get())) if (!runtime::cpu::mkldnn_utils::can_use_mkldnn_conv<ngraph::op::ConvolutionBias>(
conv_m.get()))
{ {
NGRAPH_DEBUG << "Convolution not supported by MKLDNN"; NGRAPH_DEBUG << "Convolution not supported by MKLDNN";
return false; return false;
...@@ -1166,7 +1193,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add() ...@@ -1166,7 +1193,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add()
} }
auto conv_add = auto conv_add =
std::shared_ptr<Node>(new op::ConvolutionBiasAdd(conv_m, inplace_input, false)); std::shared_ptr<Node>(new ngraph::op::ConvolutionBiasAdd(conv_m, inplace_input, false));
ngraph::replace_node(m.get_match_root(), conv_add); ngraph::replace_node(m.get_match_root(), conv_add);
return true; return true;
}; };
...@@ -1183,7 +1210,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add_relu() ...@@ -1183,7 +1210,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add_relu()
auto bias = std::make_shared<pattern::op::Label>(element::f32, Shape{shape[0]}); auto bias = std::make_shared<pattern::op::Label>(element::f32, Shape{shape[0]});
auto add_input = std::make_shared<pattern::op::Label>(element::f32, shape); auto add_input = std::make_shared<pattern::op::Label>(element::f32, shape);
auto pconv = std::make_shared<op::ConvolutionBiasAdd>(data_batch, auto pconv = std::make_shared<ngraph::op::ConvolutionBiasAdd>(data_batch,
filters, filters,
bias, bias,
add_input, add_input,
...@@ -1193,14 +1220,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add_relu() ...@@ -1193,14 +1220,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add_relu()
CoordinateDiff{0, 0}, CoordinateDiff{0, 0},
Strides{1, 1}, Strides{1, 1},
false); false);
auto prelu = std::make_shared<op::Relu>(pconv); auto prelu = std::make_shared<ngraph::op::Relu>(pconv);
pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_conv_sum against " NGRAPH_DEBUG << "In a callback for construct_conv_sum against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto conv_m = auto conv_m = std::static_pointer_cast<ngraph::op::ConvolutionBiasAdd>(
std::static_pointer_cast<op::ConvolutionBiasAdd>(m.get_match_root()->get_argument(0)); m.get_match_root()->get_argument(0));
if (conv_m->get_users().size() > 1) if (conv_m->get_users().size() > 1)
{ {
NGRAPH_DEBUG << "Convolution has more than one user"; NGRAPH_DEBUG << "Convolution has more than one user";
...@@ -1220,7 +1247,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add_relu() ...@@ -1220,7 +1247,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_add_relu()
// ConvolutionBiasAdd created only if it can run with MKLDNN. // ConvolutionBiasAdd created only if it can run with MKLDNN.
// No further checks needed. // No further checks needed.
auto conv_n = auto conv_n =
std::make_shared<op::ConvolutionBiasAdd>(conv_m->get_argument(0), std::make_shared<ngraph::op::ConvolutionBiasAdd>(conv_m->get_argument(0),
conv_m->get_argument(1), conv_m->get_argument(1),
conv_m->get_argument(2), conv_m->get_argument(2),
conv_m->get_argument(3), conv_m->get_argument(3),
...@@ -1242,19 +1269,19 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_sigmoid_multiply() ...@@ -1242,19 +1269,19 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_sigmoid_multiply()
{ {
// Construct predicate to match sigmoid and tanh // Construct predicate to match sigmoid and tanh
auto sigmoid_pred = [](std::shared_ptr<Node> n) { auto sigmoid_pred = [](std::shared_ptr<Node> n) {
return (std::dynamic_pointer_cast<op::Sigmoid>(n) != nullptr) || return (std::dynamic_pointer_cast<ngraph::op::Sigmoid>(n) != nullptr) ||
(std::dynamic_pointer_cast<op::Tanh>(n) != nullptr); (std::dynamic_pointer_cast<ngraph::op::Tanh>(n) != nullptr);
}; };
// Construct predicate to match other valid nodes // Construct predicate to match other valid nodes
auto other_pred = [](std::shared_ptr<Node> n) { auto other_pred = [](std::shared_ptr<Node> n) {
return (std::dynamic_pointer_cast<op::Sigmoid>(n) != nullptr) || return (std::dynamic_pointer_cast<ngraph::op::Sigmoid>(n) != nullptr) ||
(std::dynamic_pointer_cast<op::Tanh>(n) != nullptr) || (std::dynamic_pointer_cast<ngraph::op::Tanh>(n) != nullptr) ||
(std::dynamic_pointer_cast<op::Add>(n) != nullptr) || (std::dynamic_pointer_cast<ngraph::op::Add>(n) != nullptr) ||
(std::dynamic_pointer_cast<op::Broadcast>(n) != nullptr); (std::dynamic_pointer_cast<ngraph::op::Broadcast>(n) != nullptr);
}; };
auto sigmoid_0 = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1}, sigmoid_pred); auto sigmoid_0 = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1}, sigmoid_pred);
auto sigmoid_1 = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1}, other_pred); auto sigmoid_1 = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1}, other_pred);
auto elem_mul = std::make_shared<op::Multiply>(sigmoid_0, sigmoid_1); auto elem_mul = std::make_shared<ngraph::op::Multiply>(sigmoid_0, sigmoid_1);
ngraph::pattern::graph_rewrite_callback callback = [sigmoid_0, sigmoid_1](pattern::Matcher& m) { ngraph::pattern::graph_rewrite_callback callback = [sigmoid_0, sigmoid_1](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_sigmoid_multiply pattern against " NGRAPH_DEBUG << "In a callback for construct_sigmoid_multiply pattern against "
...@@ -1268,7 +1295,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_sigmoid_multiply() ...@@ -1268,7 +1295,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_sigmoid_multiply()
return false; return false;
} }
using FunctionType = op::SigmoidMultiply::FunctionType; using FunctionType = ngraph::op::SigmoidMultiply::FunctionType;
const int max_inputs{2}; const int max_inputs{2};
std::array<std::shared_ptr<ngraph::Node>, max_inputs> match_nodes{ std::array<std::shared_ptr<ngraph::Node>, max_inputs> match_nodes{
{pattern_map[sigmoid_0], pattern_map[sigmoid_1]}}; {pattern_map[sigmoid_0], pattern_map[sigmoid_1]}};
...@@ -1276,7 +1303,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_sigmoid_multiply() ...@@ -1276,7 +1303,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_sigmoid_multiply()
std::array<FunctionType, max_inputs> input_type; std::array<FunctionType, max_inputs> input_type;
for (int i = 0; i < max_inputs; ++i) for (int i = 0; i < max_inputs; ++i)
{ {
input_type[i] = op::SigmoidMultiply::identify_node_type(match_nodes[i]); input_type[i] = ngraph::op::SigmoidMultiply::identify_node_type(match_nodes[i]);
if (input_type[i] != FunctionType::Identity) if (input_type[i] != FunctionType::Identity)
{ {
if (match_nodes[i]->get_users().size() > 1) if (match_nodes[i]->get_users().size() > 1)
...@@ -1291,7 +1318,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_sigmoid_multiply() ...@@ -1291,7 +1318,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_sigmoid_multiply()
input_nodes[i] = match_nodes[i]; input_nodes[i] = match_nodes[i];
} }
} }
auto sigmoid_mul_node = std::make_shared<op::SigmoidMultiply>( auto sigmoid_mul_node = std::make_shared<ngraph::op::SigmoidMultiply>(
input_nodes[0], input_nodes[1], input_type[0], input_type[1]); input_nodes[0], input_nodes[1], input_type[0], input_type[1]);
ngraph::replace_node(m.get_match_root(), sigmoid_mul_node); ngraph::replace_node(m.get_match_root(), sigmoid_mul_node);
return true; return true;
...@@ -1304,21 +1331,21 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_sigmoid_multiply() ...@@ -1304,21 +1331,21 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_sigmoid_multiply()
void ngraph::runtime::cpu::pass::CPUFusion::construct_leaky_relu() void ngraph::runtime::cpu::pass::CPUFusion::construct_leaky_relu()
{ {
auto input = std::make_shared<pattern::op::Label>(element::f32, Shape{}); auto input = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto iconst1 = op::Constant::create(element::f32, Shape{}, {1}); auto iconst1 = ngraph::op::Constant::create(element::f32, Shape{}, {1});
auto alpha = std::make_shared<pattern::op::Label>(iconst1); auto alpha = std::make_shared<pattern::op::Label>(iconst1);
auto broadcast_pred = [](std::shared_ptr<Node> n) { auto broadcast_pred = [](std::shared_ptr<Node> n) {
return (std::dynamic_pointer_cast<op::Broadcast>(n) != nullptr); return (std::dynamic_pointer_cast<ngraph::op::Broadcast>(n) != nullptr);
}; };
auto skip_broadcast = std::make_shared<pattern::op::Skip>(alpha, broadcast_pred); auto skip_broadcast = std::make_shared<pattern::op::Skip>(alpha, broadcast_pred);
auto leaky_relu = auto leaky_relu = std::make_shared<ngraph::op::Maximum>(
std::make_shared<op::Maximum>(input, std::make_shared<op::Multiply>(input, skip_broadcast)); input, std::make_shared<ngraph::op::Multiply>(input, skip_broadcast));
pattern::graph_rewrite_callback callback = [input, alpha](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [input, alpha](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_leaky_relu against " NGRAPH_DEBUG << "In a callback for construct_leaky_relu against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
if (!std::dynamic_pointer_cast<op::Constant>(pattern_map[alpha])) if (!std::dynamic_pointer_cast<ngraph::op::Constant>(pattern_map[alpha]))
{ {
NGRAPH_DEBUG << "alpha must be constant for leaky relu"; NGRAPH_DEBUG << "alpha must be constant for leaky relu";
return false; return false;
...@@ -1330,7 +1357,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_leaky_relu() ...@@ -1330,7 +1357,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_leaky_relu()
return false; return false;
} }
auto alpha_const_op = std::static_pointer_cast<op::Constant>(pattern_map[alpha]); auto alpha_const_op = std::static_pointer_cast<ngraph::op::Constant>(pattern_map[alpha]);
auto alpha_vec = alpha_const_op->get_vector<float>(); auto alpha_vec = alpha_const_op->get_vector<float>();
for (auto val : alpha_vec) for (auto val : alpha_vec)
{ {
...@@ -1350,7 +1377,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_leaky_relu() ...@@ -1350,7 +1377,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_leaky_relu()
return false; return false;
} }
auto cg = std::shared_ptr<Node>(new op::LeakyRelu(pattern_map[input], alpha_vec[0])); auto cg =
std::shared_ptr<Node>(new ngraph::op::LeakyRelu(pattern_map[input], alpha_vec[0]));
ngraph::replace_node(m.get_match_root(), cg); ngraph::replace_node(m.get_match_root(), cg);
return true; return true;
}; };
...@@ -1361,14 +1389,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_leaky_relu() ...@@ -1361,14 +1389,14 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_leaky_relu()
void ngraph::runtime::cpu::pass::CPUFusion::construct_bounded_relu() void ngraph::runtime::cpu::pass::CPUFusion::construct_bounded_relu()
{ {
auto relu_input = std::make_shared<pattern::op::Label>(element::f32, Shape{}); auto relu_input = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto relu = std::make_shared<op::Relu>(relu_input); auto relu = std::make_shared<ngraph::op::Relu>(relu_input);
auto iconst1 = op::Constant::create(element::f32, Shape{}, {1}); auto iconst1 = ngraph::op::Constant::create(element::f32, Shape{}, {1});
auto alpha = std::make_shared<pattern::op::Label>(iconst1); auto alpha = std::make_shared<pattern::op::Label>(iconst1);
auto broadcast_pred = [](std::shared_ptr<Node> n) { auto broadcast_pred = [](std::shared_ptr<Node> n) {
return (std::dynamic_pointer_cast<op::Broadcast>(n) != nullptr); return (std::dynamic_pointer_cast<ngraph::op::Broadcast>(n) != nullptr);
}; };
auto skip_broadcast = std::make_shared<pattern::op::Skip>(alpha, broadcast_pred); auto skip_broadcast = std::make_shared<pattern::op::Skip>(alpha, broadcast_pred);
auto min = std::make_shared<op::Minimum>(relu, skip_broadcast); auto min = std::make_shared<ngraph::op::Minimum>(relu, skip_broadcast);
pattern::graph_rewrite_callback callback = [relu_input, alpha](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [relu_input, alpha](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_bounded_relu against " NGRAPH_DEBUG << "In a callback for construct_bounded_relu against "
...@@ -1381,7 +1409,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_bounded_relu() ...@@ -1381,7 +1409,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_bounded_relu()
return false; return false;
} }
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
if (!std::dynamic_pointer_cast<op::Constant>(pattern_map[alpha])) if (!std::dynamic_pointer_cast<ngraph::op::Constant>(pattern_map[alpha]))
{ {
NGRAPH_DEBUG << "alpha must be constant for bounded relu"; NGRAPH_DEBUG << "alpha must be constant for bounded relu";
return false; return false;
...@@ -1397,12 +1425,13 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_bounded_relu() ...@@ -1397,12 +1425,13 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_bounded_relu()
return false; return false;
} }
auto alpha_const_op = std::static_pointer_cast<op::Constant>(pattern_map[alpha]); auto alpha_const_op = std::static_pointer_cast<ngraph::op::Constant>(pattern_map[alpha]);
float alpha_val = *(static_cast<float const*>(alpha_const_op->get_data_ptr())); float alpha_val = *(static_cast<float const*>(alpha_const_op->get_data_ptr()));
NGRAPH_DEBUG << "relu_input: " << pattern_map[relu_input] << " min_val: " NGRAPH_DEBUG << "relu_input: " << pattern_map[relu_input] << " min_val: "
<< *(static_cast<float const*>(alpha_const_op->get_data_ptr())); << *(static_cast<float const*>(alpha_const_op->get_data_ptr()));
auto cg = std::shared_ptr<Node>(new op::BoundedRelu(pattern_map[relu_input], alpha_val)); auto cg =
std::shared_ptr<Node>(new ngraph::op::BoundedRelu(pattern_map[relu_input], alpha_val));
ngraph::replace_node(m.get_match_root(), cg); ngraph::replace_node(m.get_match_root(), cg);
return true; return true;
}; };
...@@ -1417,7 +1446,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_folded_batch_nor ...@@ -1417,7 +1446,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_folded_batch_nor
auto filters = std::make_shared<pattern::op::Label>(element::f32, Shape{2, 2, 1, 1}); auto filters = std::make_shared<pattern::op::Label>(element::f32, Shape{2, 2, 1, 1});
auto bias = std::make_shared<pattern::op::Label>(element::f32, Shape{2}); auto bias = std::make_shared<pattern::op::Label>(element::f32, Shape{2});
auto pconv = std::make_shared<op::ConvolutionBias>(input, auto pconv = std::make_shared<ngraph::op::ConvolutionBias>(input,
filters, filters,
bias, bias,
Strides{1, 1}, Strides{1, 1},
...@@ -1431,7 +1460,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_folded_batch_nor ...@@ -1431,7 +1460,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_folded_batch_nor
auto gamma = std::make_shared<pattern::op::Label>(element::f32, Shape{2}); auto gamma = std::make_shared<pattern::op::Label>(element::f32, Shape{2});
auto beta = std::make_shared<pattern::op::Label>(element::f32, Shape{2}); auto beta = std::make_shared<pattern::op::Label>(element::f32, Shape{2});
double eps = 0.001; double eps = 0.001;
auto bn = std::make_shared<op::BatchNormInference>(eps, gamma, beta, pconv, mean, var); auto bn = std::make_shared<ngraph::op::BatchNormInference>(eps, gamma, beta, pconv, mean, var);
ngraph::pattern::graph_rewrite_callback callback = ngraph::pattern::graph_rewrite_callback callback =
[input, filters, bias, mean, var, gamma, beta](pattern::Matcher& m) { [input, filters, bias, mean, var, gamma, beta](pattern::Matcher& m) {
...@@ -1439,8 +1468,10 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_folded_batch_nor ...@@ -1439,8 +1468,10 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_folded_batch_nor
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto m_bn = std::static_pointer_cast<op::BatchNormInference>(m.get_match_root()); auto m_bn =
auto m_conv = std::static_pointer_cast<op::ConvolutionBias>(m_bn->get_argument(2)); std::static_pointer_cast<ngraph::op::BatchNormInference>(m.get_match_root());
auto m_conv =
std::static_pointer_cast<ngraph::op::ConvolutionBias>(m_bn->get_argument(2));
if (m_conv->get_users().size() > 1) if (m_conv->get_users().size() > 1)
{ {
...@@ -1455,25 +1486,28 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_folded_batch_nor ...@@ -1455,25 +1486,28 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_folded_batch_nor
// new weights = old weights * gamma / sqrt(variance + epsilon) // new weights = old weights * gamma / sqrt(variance + epsilon)
// new biases = (old_bias-mean) * gamma / sqrt(variance + epsilon) + beta // new biases = (old_bias-mean) * gamma / sqrt(variance + epsilon) + beta
auto bn_eps = op::Constant::create(element::f32, Shape{}, {m_bn->get_eps_value()}); auto bn_eps =
auto var_eps = std::make_shared<op::Add>( ngraph::op::Constant::create(element::f32, Shape{}, {m_bn->get_eps_value()});
auto var_eps = std::make_shared<ngraph::op::Add>(
pattern_map[var], pattern_map[var],
std::make_shared<op::Broadcast>(bn_eps, pattern_map[var]->get_shape(), AxisSet{0})); std::make_shared<ngraph::op::Broadcast>(
auto sqrt_var_eps = std::make_shared<op::Sqrt>(var_eps); bn_eps, pattern_map[var]->get_shape(), AxisSet{0}));
auto sqrt_var_eps = std::make_shared<ngraph::op::Sqrt>(var_eps);
auto mean_gamma = std::make_shared<op::Multiply>( auto mean_gamma = std::make_shared<ngraph::op::Multiply>(
std::make_shared<op::Subtract>(pattern_map[bias], pattern_map[mean]), std::make_shared<ngraph::op::Subtract>(pattern_map[bias], pattern_map[mean]),
pattern_map[gamma]); pattern_map[gamma]);
auto new_biases = std::make_shared<op::Add>( auto new_biases = std::make_shared<ngraph::op::Add>(
pattern_map[beta], std::make_shared<op::Divide>(mean_gamma, sqrt_var_eps)); pattern_map[beta], std::make_shared<ngraph::op::Divide>(mean_gamma, sqrt_var_eps));
auto weight_scaling = std::make_shared<op::Divide>(pattern_map[gamma], sqrt_var_eps); auto weight_scaling =
auto new_weights = std::make_shared<op::Multiply>( std::make_shared<ngraph::op::Divide>(pattern_map[gamma], sqrt_var_eps);
auto new_weights = std::make_shared<ngraph::op::Multiply>(
pattern_map[filters], pattern_map[filters],
std::make_shared<op::Broadcast>( std::make_shared<ngraph::op::Broadcast>(
weight_scaling, pattern_map[filters]->get_shape(), AxisSet{1, 2, 3})); weight_scaling, pattern_map[filters]->get_shape(), AxisSet{1, 2, 3}));
auto conv_bias = auto conv_bias =
std::make_shared<op::ConvolutionBias>(pattern_map[input], std::make_shared<ngraph::op::ConvolutionBias>(pattern_map[input],
new_weights, new_weights,
new_biases, new_biases,
m_conv->get_window_movement_strides(), m_conv->get_window_movement_strides(),
...@@ -1500,7 +1534,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding() ...@@ -1500,7 +1534,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding()
auto filters = std::make_shared<pattern::op::Label>(element::f32, shape); auto filters = std::make_shared<pattern::op::Label>(element::f32, shape);
auto bias = std::make_shared<pattern::op::Label>(element::f32, Shape{2}); auto bias = std::make_shared<pattern::op::Label>(element::f32, Shape{2});
auto conv = std::make_shared<op::ConvolutionBias>(input, auto conv = std::make_shared<ngraph::op::ConvolutionBias>(input,
filters, filters,
bias, bias,
Strides{1, 1}, Strides{1, 1},
...@@ -1511,9 +1545,9 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding() ...@@ -1511,9 +1545,9 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding()
auto conv_label = std::make_shared<pattern::op::Label>(conv, nullptr, NodeVector{conv}); auto conv_label = std::make_shared<pattern::op::Label>(conv, nullptr, NodeVector{conv});
auto Ac = std::make_shared<pattern::op::Label>(element::f32, Shape{2}); auto Ac = std::make_shared<pattern::op::Label>(element::f32, Shape{2});
auto A = std::make_shared<op::Broadcast>(Ac, Shape{2, 2, 1, 1}, AxisSet{0, 2, 3}); auto A = std::make_shared<ngraph::op::Broadcast>(Ac, Shape{2, 2, 1, 1}, AxisSet{0, 2, 3});
auto A_label = std::make_shared<pattern::op::Label>(A, nullptr, NodeVector{A}); auto A_label = std::make_shared<pattern::op::Label>(A, nullptr, NodeVector{A});
auto multiply = std::make_shared<op::Multiply>(conv_label, A_label); auto multiply = std::make_shared<ngraph::op::Multiply>(conv_label, A_label);
ngraph::pattern::graph_rewrite_callback callback = [input, filters, bias, conv_label, A_label]( ngraph::pattern::graph_rewrite_callback callback = [input, filters, bias, conv_label, A_label](
pattern::Matcher& m) { pattern::Matcher& m) {
...@@ -1521,7 +1555,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding() ...@@ -1521,7 +1555,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding()
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto conv_m = std::static_pointer_cast<op::ConvolutionBias>(pattern_map[conv_label]); auto conv_m =
std::static_pointer_cast<ngraph::op::ConvolutionBias>(pattern_map[conv_label]);
if (conv_m->get_users().size() > 1) if (conv_m->get_users().size() > 1)
{ {
...@@ -1538,10 +1573,10 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding() ...@@ -1538,10 +1573,10 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding()
return false; return false;
} }
auto A_m = std::static_pointer_cast<op::Broadcast>(pattern_map[A_label]); auto A_m = std::static_pointer_cast<ngraph::op::Broadcast>(pattern_map[A_label]);
// Check if values are being broadcast along channel (2nd) dimension // Check if values are being broadcast along channel (2nd) dimension
auto is_channel_bcast = [](const std::shared_ptr<op::Broadcast>& bcast) { auto is_channel_bcast = [](const std::shared_ptr<ngraph::op::Broadcast>& bcast) {
auto input_shape = bcast->get_argument(0)->get_shape(); auto input_shape = bcast->get_argument(0)->get_shape();
if (input_shape.size() == 0 || shape_size(input_shape) == 1) if (input_shape.size() == 0 || shape_size(input_shape) == 1)
{ {
...@@ -1566,19 +1601,21 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding() ...@@ -1566,19 +1601,21 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding()
return false; return false;
} }
auto get_bcast_input = [](const std::shared_ptr<op::Broadcast>& bcast) { auto get_bcast_input = [](const std::shared_ptr<ngraph::op::Broadcast>& bcast) {
auto input_shape = bcast->get_argument(0)->get_shape(); auto input_shape = bcast->get_argument(0)->get_shape();
if (input_shape.size() == 0) if (input_shape.size() == 0)
{ {
Shape bshape{bcast->get_shape()[1]}; Shape bshape{bcast->get_shape()[1]};
return std::static_pointer_cast<ngraph::Node>( return std::static_pointer_cast<ngraph::Node>(
std::make_shared<op::Broadcast>(bcast->get_argument(0), bshape, AxisSet{0})); std::make_shared<ngraph::op::Broadcast>(
bcast->get_argument(0), bshape, AxisSet{0}));
} }
if (shape_size(input_shape) == 1) if (shape_size(input_shape) == 1)
{ {
Shape bshape{bcast->get_shape()[1]}; Shape bshape{bcast->get_shape()[1]};
return std::static_pointer_cast<ngraph::Node>(std::make_shared<op::Broadcast>( return std::static_pointer_cast<ngraph::Node>(
std::make_shared<op::Reshape>( std::make_shared<ngraph::op::Broadcast>(
std::make_shared<ngraph::op::Reshape>(
bcast->get_argument(0), get_default_order(input_shape), Shape{}), bcast->get_argument(0), get_default_order(input_shape), Shape{}),
bshape, bshape,
AxisSet{0})); AxisSet{0}));
...@@ -1590,7 +1627,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding() ...@@ -1590,7 +1627,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding()
if (input_shape.size() == 2) if (input_shape.size() == 2)
{ {
Shape bshape{input_shape[1]}; Shape bshape{input_shape[1]};
return std::static_pointer_cast<ngraph::Node>(std::make_shared<op::Reshape>( return std::static_pointer_cast<ngraph::Node>(std::make_shared<ngraph::op::Reshape>(
bcast->get_argument(0), AxisVector{0, 1}, bshape)); bcast->get_argument(0), AxisVector{0, 1}, bshape));
} }
throw ngraph_error("Unexpected shape for bcast input"); throw ngraph_error("Unexpected shape for bcast input");
...@@ -1601,15 +1638,15 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding() ...@@ -1601,15 +1638,15 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias_affine_folding()
// new weights = old weights * Ac_m // new weights = old weights * Ac_m
// new_bias = old_bias * Ac_m; // new_bias = old_bias * Ac_m;
auto filters_n = std::make_shared<op::Multiply>( auto filters_n = std::make_shared<ngraph::op::Multiply>(
pattern_map[filters], pattern_map[filters],
std::make_shared<op::Broadcast>( std::make_shared<ngraph::op::Broadcast>(
Ac_m, pattern_map[filters]->get_shape(), AxisSet{1, 2, 3})); Ac_m, pattern_map[filters]->get_shape(), AxisSet{1, 2, 3}));
auto bias_n = std::make_shared<op::Multiply>(pattern_map[bias], Ac_m); auto bias_n = std::make_shared<ngraph::op::Multiply>(pattern_map[bias], Ac_m);
auto convbias_n = auto convbias_n =
std::make_shared<op::ConvolutionBias>(pattern_map[input], std::make_shared<ngraph::op::ConvolutionBias>(pattern_map[input],
filters_n, filters_n,
bias_n, bias_n,
conv_m->get_window_movement_strides(), conv_m->get_window_movement_strides(),
...@@ -1638,7 +1675,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_groupconv_batchnorm_global ...@@ -1638,7 +1675,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_groupconv_batchnorm_global
auto filters = std::make_shared<pattern::op::Label>(element::f32, shape_b); auto filters = std::make_shared<pattern::op::Label>(element::f32, shape_b);
auto resShape = std::make_shared<pattern::op::Label>(element::f32, shape_r); auto resShape = std::make_shared<pattern::op::Label>(element::f32, shape_r);
auto conv = std::make_shared<op::GroupConvolution>(input, auto conv = std::make_shared<ngraph::op::GroupConvolution>(input,
filters, filters,
Strides{1, 1}, Strides{1, 1},
Strides{1, 1}, Strides{1, 1},
...@@ -1654,7 +1691,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_groupconv_batchnorm_global ...@@ -1654,7 +1691,8 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_groupconv_batchnorm_global
auto gamma = std::make_shared<pattern::op::Label>(element::f32, Shape{32}); auto gamma = std::make_shared<pattern::op::Label>(element::f32, Shape{32});
auto beta = std::make_shared<pattern::op::Label>(element::f32, Shape{32}); auto beta = std::make_shared<pattern::op::Label>(element::f32, Shape{32});
double eps = 0.001; double eps = 0.001;
auto bn = std::make_shared<op::BatchNormInference>(eps, gamma, beta, conv_label, mean, var); auto bn =
std::make_shared<ngraph::op::BatchNormInference>(eps, gamma, beta, conv_label, mean, var);
ngraph::pattern::graph_rewrite_callback callback = ngraph::pattern::graph_rewrite_callback callback =
[input, filters, conv_label, mean, var, gamma, beta, eps](pattern::Matcher& m) { [input, filters, conv_label, mean, var, gamma, beta, eps](pattern::Matcher& m) {
...@@ -1663,8 +1701,10 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_groupconv_batchnorm_global ...@@ -1663,8 +1701,10 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_groupconv_batchnorm_global
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto m_bn = std::static_pointer_cast<op::BatchNormInference>(m.get_match_root()); auto m_bn =
auto conv_m = std::static_pointer_cast<op::GroupConvolution>(pattern_map[conv_label]); std::static_pointer_cast<ngraph::op::BatchNormInference>(m.get_match_root());
auto conv_m =
std::static_pointer_cast<ngraph::op::GroupConvolution>(pattern_map[conv_label]);
if (conv_m->get_users().size() > 1) if (conv_m->get_users().size() > 1)
{ {
...@@ -1684,25 +1724,29 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_groupconv_batchnorm_global ...@@ -1684,25 +1724,29 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_groupconv_batchnorm_global
// new weights = old weights * gamma / sqrt(variance + epsilon) // new weights = old weights * gamma / sqrt(variance + epsilon)
// new biases = (-mean) * gamma / sqrt(variance + epsilon) + beta // new biases = (-mean) * gamma / sqrt(variance + epsilon) + beta
auto bn_eps = op::Constant::create(element::f32, Shape{}, {m_bn->get_eps_value()}); auto bn_eps =
ngraph::op::Constant::create(element::f32, Shape{}, {m_bn->get_eps_value()});
auto var_eps = std::make_shared<op::Add>( auto var_eps = std::make_shared<ngraph::op::Add>(
pattern_map[var], pattern_map[var],
std::make_shared<op::Broadcast>(bn_eps, pattern_map[var]->get_shape(), AxisSet{0})); std::make_shared<ngraph::op::Broadcast>(
auto sqrt_var_eps = std::make_shared<op::Sqrt>(var_eps); bn_eps, pattern_map[var]->get_shape(), AxisSet{0}));
auto sqrt_var_eps = std::make_shared<ngraph::op::Sqrt>(var_eps);
auto weight_scaling = std::make_shared<op::Divide>(pattern_map[gamma], sqrt_var_eps); auto weight_scaling =
std::make_shared<ngraph::op::Divide>(pattern_map[gamma], sqrt_var_eps);
auto weight_scaling_bcast = std::make_shared<op::Broadcast>( auto weight_scaling_bcast = std::make_shared<ngraph::op::Broadcast>(
weight_scaling, pattern_map[filters]->get_shape(), AxisSet{1, 2, 3}); weight_scaling, pattern_map[filters]->get_shape(), AxisSet{1, 2, 3});
auto new_weights = auto new_weights =
std::make_shared<op::Multiply>(pattern_map[filters], weight_scaling_bcast); std::make_shared<ngraph::op::Multiply>(pattern_map[filters], weight_scaling_bcast);
auto mean_gamma = std::make_shared<op::Multiply>(pattern_map[mean], weight_scaling); auto mean_gamma =
auto new_biases = std::make_shared<op::Subtract>(pattern_map[beta], mean_gamma); std::make_shared<ngraph::op::Multiply>(pattern_map[mean], weight_scaling);
auto new_biases = std::make_shared<ngraph::op::Subtract>(pattern_map[beta], mean_gamma);
auto g_conv_bias = auto g_conv_bias = std::make_shared<ngraph::op::GroupConvolutionBias>(
std::make_shared<op::GroupConvolutionBias>(pattern_map[input], pattern_map[input],
new_weights, new_weights,
new_biases, new_biases,
conv_m->get_window_movement_strides(), conv_m->get_window_movement_strides(),
...@@ -1738,7 +1782,7 @@ void ngraph::runtime::cpu::pass::CPUFusion:: ...@@ -1738,7 +1782,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::
auto bias = std::make_shared<pattern::op::Label>(element::f32, shape_bias); auto bias = std::make_shared<pattern::op::Label>(element::f32, shape_bias);
auto num = std::make_shared<pattern::op::Label>(element::f32, shape_num); auto num = std::make_shared<pattern::op::Label>(element::f32, shape_num);
auto conv = std::make_shared<op::GroupConvolutionBias>(input, auto conv = std::make_shared<ngraph::op::GroupConvolutionBias>(input,
filters, filters,
bias, bias,
Strides{1, 1}, Strides{1, 1},
...@@ -1753,7 +1797,7 @@ void ngraph::runtime::cpu::pass::CPUFusion:: ...@@ -1753,7 +1797,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::
auto conv_label = std::make_shared<pattern::op::Label>(conv, nullptr, NodeVector{conv}); auto conv_label = std::make_shared<pattern::op::Label>(conv, nullptr, NodeVector{conv});
// GroupConv + BatchNorm + Relu -> GroupConvBias // GroupConv + BatchNorm + Relu -> GroupConvBias
auto prelu = std::make_shared<op::Relu>(conv_label); auto prelu = std::make_shared<ngraph::op::Relu>(conv_label);
ngraph::pattern::graph_rewrite_callback callback = ngraph::pattern::graph_rewrite_callback callback =
[input, filters, bias, num, conv_label, prelu](pattern::Matcher& m) { [input, filters, bias, num, conv_label, prelu](pattern::Matcher& m) {
...@@ -1763,11 +1807,11 @@ void ngraph::runtime::cpu::pass::CPUFusion:: ...@@ -1763,11 +1807,11 @@ void ngraph::runtime::cpu::pass::CPUFusion::
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto conv_m = auto conv_m =
std::static_pointer_cast<op::GroupConvolutionBias>(pattern_map[conv_label]); std::static_pointer_cast<ngraph::op::GroupConvolutionBias>(pattern_map[conv_label]);
auto relu_m = std::dynamic_pointer_cast<op::Relu>(m.get_match_root()); auto relu_m = std::dynamic_pointer_cast<ngraph::op::Relu>(m.get_match_root());
auto g_conv_bias_relu = auto g_conv_bias_relu = std::make_shared<ngraph::op::GroupConvolutionBias>(
std::make_shared<op::GroupConvolutionBias>(conv_m->get_argument(0), conv_m->get_argument(0),
conv_m->get_argument(1), conv_m->get_argument(1),
conv_m->get_argument(2), conv_m->get_argument(2),
conv_m->get_window_movement_strides(), conv_m->get_window_movement_strides(),
...@@ -1796,25 +1840,26 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_fuse_lstm_recurrent_state( ...@@ -1796,25 +1840,26 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_fuse_lstm_recurrent_state(
auto bias_label = std::make_shared<pattern::op::Label>(element::f32, Shape{400}); auto bias_label = std::make_shared<pattern::op::Label>(element::f32, Shape{400});
ngraph::runtime::cpu::rnn_utils::rnntype rnn_type = ngraph::runtime::cpu::rnn_utils::rnntype rnn_type =
ngraph::runtime::cpu::rnn_utils::rnntype::vanilla_lstm; ngraph::runtime::cpu::rnn_utils::rnntype::vanilla_lstm;
auto lstm1 = std::make_shared<op::Lstm>(src_layer_label, auto lstm1 = std::make_shared<ngraph::op::Lstm>(src_layer_label,
src_iter_label, src_iter_label,
weights_layer_label, weights_layer_label,
weights_iter_label, weights_iter_label,
bias_label, bias_label,
rnn_type); rnn_type);
auto lstm1_goe0 = std::make_shared<op::GetOutputElement>(lstm1, 0); auto lstm1_goe0 = std::make_shared<ngraph::op::GetOutputElement>(lstm1, 0);
auto lstm1_goe1 = std::make_shared<op::GetOutputElement>(lstm1, 1); auto lstm1_goe1 = std::make_shared<ngraph::op::GetOutputElement>(lstm1, 1);
auto lstm1_goe0_label = auto lstm1_goe0_label =
std::make_shared<pattern::op::Label>(lstm1_goe0, nullptr, NodeVector{lstm1_goe0}); std::make_shared<pattern::op::Label>(lstm1_goe0, nullptr, NodeVector{lstm1_goe0});
auto lstm1_goe1_label = auto lstm1_goe1_label =
std::make_shared<pattern::op::Label>(lstm1_goe1, nullptr, NodeVector{lstm1_goe1}); std::make_shared<pattern::op::Label>(lstm1_goe1, nullptr, NodeVector{lstm1_goe1});
auto lstm1_goe0_slice = auto lstm1_goe0_slice = std::make_shared<ngraph::op::Slice>(
std::make_shared<op::Slice>(lstm1_goe0_label, Coordinate{0, 0}, Coordinate{10, 100}); lstm1_goe0_label, Coordinate{0, 0}, Coordinate{10, 100});
auto lstm1_goe1_slice = auto lstm1_goe1_slice = std::make_shared<ngraph::op::Slice>(
std::make_shared<op::Slice>(lstm1_goe1_label, Coordinate{10, 0}, Coordinate{20, 100}); lstm1_goe1_label, Coordinate{10, 0}, Coordinate{20, 100});
auto concat = std::make_shared<op::Concat>(NodeVector{lstm1_goe0_slice, lstm1_goe1_slice}, 0); auto concat =
std::make_shared<ngraph::op::Concat>(NodeVector{lstm1_goe0_slice, lstm1_goe1_slice}, 0);
auto concat_label = std::make_shared<pattern::op::Label>(concat, nullptr, NodeVector{concat}); auto concat_label = std::make_shared<pattern::op::Label>(concat, nullptr, NodeVector{concat});
ngraph::pattern::graph_rewrite_callback callback = ngraph::pattern::graph_rewrite_callback callback =
...@@ -1841,19 +1886,20 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_update_slice() ...@@ -1841,19 +1886,20 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_update_slice()
Shape shape_b{1, 32, 2}; Shape shape_b{1, 32, 2};
auto input = std::make_shared<pattern::op::Label>(element::f32, shape_a); auto input = std::make_shared<pattern::op::Label>(element::f32, shape_a);
auto slice = std::make_shared<op::Slice>(input, Coordinate{1, 0, 0}, Coordinate{2, 32, 2}); auto slice =
std::make_shared<ngraph::op::Slice>(input, Coordinate{1, 0, 0}, Coordinate{2, 32, 2});
auto slice_label = std::make_shared<pattern::op::Label>(slice, nullptr, NodeVector{slice}); auto slice_label = std::make_shared<pattern::op::Label>(slice, nullptr, NodeVector{slice});
auto update_input = std::make_shared<pattern::op::Label>(element::f32, shape_b); auto update_input = std::make_shared<pattern::op::Label>(element::f32, shape_b);
auto update = std::make_shared<op::Add>(update_input, slice_label); auto update = std::make_shared<ngraph::op::Add>(update_input, slice_label);
auto replace_slice = std::make_shared<op::ReplaceSlice>( auto replace_slice = std::make_shared<ngraph::op::ReplaceSlice>(
input, update, Coordinate{1, 0, 0}, Coordinate{2, 32, 2}); input, update, Coordinate{1, 0, 0}, Coordinate{2, 32, 2});
ngraph::pattern::graph_rewrite_callback callback = [input, update_input, slice_label]( ngraph::pattern::graph_rewrite_callback callback = [input, update_input, slice_label](
pattern::Matcher& m) { pattern::Matcher& m) {
NGRAPH_DEBUG << "In callback for update_slice = " << m.get_match_root()->get_name(); NGRAPH_DEBUG << "In callback for update_slice = " << m.get_match_root()->get_name();
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto slice_m = std::static_pointer_cast<op::Slice>(pattern_map[slice_label]); auto slice_m = std::static_pointer_cast<ngraph::op::Slice>(pattern_map[slice_label]);
auto replace_m = std::static_pointer_cast<op::ReplaceSlice>(m.get_match_root()); auto replace_m = std::static_pointer_cast<ngraph::op::ReplaceSlice>(m.get_match_root());
if (replace_m->get_lower_bounds() != slice_m->get_lower_bounds() || if (replace_m->get_lower_bounds() != slice_m->get_lower_bounds() ||
replace_m->get_upper_bounds() != slice_m->get_upper_bounds() || replace_m->get_upper_bounds() != slice_m->get_upper_bounds() ||
replace_m->get_strides() != slice_m->get_strides()) replace_m->get_strides() != slice_m->get_strides())
...@@ -1869,7 +1915,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_update_slice() ...@@ -1869,7 +1915,7 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_update_slice()
return false; return false;
} }
auto update_slice = std::make_shared<op::UpdateSlice>(pattern_map[input], auto update_slice = std::make_shared<ngraph::op::UpdateSlice>(pattern_map[input],
pattern_map[update_input], pattern_map[update_input],
replace_m->get_lower_bounds(), replace_m->get_lower_bounds(),
replace_m->get_upper_bounds(), replace_m->get_upper_bounds(),
...@@ -1897,7 +1943,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_ ...@@ -1897,7 +1943,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_
if (with_bias) if (with_bias)
{ {
auto bias = std::make_shared<pattern::op::Label>(element::i32, Shape{shape[0]}); auto bias = std::make_shared<pattern::op::Label>(element::i32, Shape{shape[0]});
qconv = std::make_shared<op::QuantizedConvolutionBias>(data_batch, qconv = std::make_shared<ngraph::op::QuantizedConvolutionBias>(data_batch,
filters, filters,
bias, bias,
Strides{1, 1}, Strides{1, 1},
...@@ -1910,7 +1956,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_ ...@@ -1910,7 +1956,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_
} }
else else
{ {
qconv = std::make_shared<op::QuantizedConvolution>(data_batch, qconv = std::make_shared<ngraph::op::QuantizedConvolution>(data_batch,
filters, filters,
Strides{1, 1}, Strides{1, 1},
Strides{1, 1}, Strides{1, 1},
...@@ -1919,14 +1965,16 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_ ...@@ -1919,14 +1965,16 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_
Strides{1, 1}, Strides{1, 1},
requantization_scale); requantization_scale);
} }
auto dq = std::make_shared<op::Dequantize>(qconv, dq_scale, dq_zp, element::f32, AxisSet{}); auto dq =
auto relu = std::make_shared<op::Relu>(dq); std::make_shared<ngraph::op::Dequantize>(qconv, dq_scale, dq_zp, element::f32, AxisSet{});
auto relu = std::make_shared<ngraph::op::Relu>(dq);
pattern::graph_rewrite_callback callback = [with_bias](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [with_bias](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_qconv_relu against " NGRAPH_DEBUG << "In a callback for construct_qconv_relu against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto dq_m = std::static_pointer_cast<op::Dequantize>(m.get_match_root()->get_argument(0)); auto dq_m =
std::static_pointer_cast<ngraph::op::Dequantize>(m.get_match_root()->get_argument(0));
if (!(ngraph::is_zero(dq_m->get_argument(2)))) if (!(ngraph::is_zero(dq_m->get_argument(2))))
{ {
...@@ -1942,7 +1990,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_ ...@@ -1942,7 +1990,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_
if (!with_bias) if (!with_bias)
{ {
if (!runtime::cpu::mkldnn_utils::can_use_mkldnn_conv<op::QuantizedConvolution>( if (!runtime::cpu::mkldnn_utils::can_use_mkldnn_conv<ngraph::op::QuantizedConvolution>(
dq_m->get_argument(0).get())) dq_m->get_argument(0).get()))
{ {
NGRAPH_DEBUG << "Quantized Convolution not supported by MKLDNN"; NGRAPH_DEBUG << "Quantized Convolution not supported by MKLDNN";
...@@ -1953,9 +2001,9 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_ ...@@ -1953,9 +2001,9 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_
std::shared_ptr<ngraph::op::Op> qconv_n; std::shared_ptr<ngraph::op::Op> qconv_n;
if (with_bias) if (with_bias)
{ {
auto qconv_m = auto qconv_m = std::static_pointer_cast<ngraph::op::QuantizedConvolutionBias>(
std::static_pointer_cast<op::QuantizedConvolutionBias>(dq_m->get_argument(0)); dq_m->get_argument(0));
qconv_n = std::make_shared<op::QuantizedConvolutionBias>( qconv_n = std::make_shared<ngraph::op::QuantizedConvolutionBias>(
qconv_m->get_argument(0), qconv_m->get_argument(0),
qconv_m->get_argument(1), qconv_m->get_argument(1),
qconv_m->get_argument(2), qconv_m->get_argument(2),
...@@ -1970,8 +2018,8 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_ ...@@ -1970,8 +2018,8 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_
else else
{ {
auto qconv_m = auto qconv_m =
std::static_pointer_cast<op::QuantizedConvolution>(dq_m->get_argument(0)); std::static_pointer_cast<ngraph::op::QuantizedConvolution>(dq_m->get_argument(0));
qconv_n = std::make_shared<op::QuantizedConvolutionRelu>( qconv_n = std::make_shared<ngraph::op::QuantizedConvolutionRelu>(
qconv_m->get_argument(0), qconv_m->get_argument(0),
qconv_m->get_argument(1), qconv_m->get_argument(1),
qconv_m->get_window_movement_strides(), qconv_m->get_window_movement_strides(),
...@@ -1983,7 +2031,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_ ...@@ -1983,7 +2031,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconv_relu(bool with_
} }
auto zp = auto zp =
builder::make_constant<uint8_t>(element::u8, dq_m->get_argument(1)->get_shape(), 0); builder::make_constant<uint8_t>(element::u8, dq_m->get_argument(1)->get_shape(), 0);
auto dq_n = std::make_shared<op::Dequantize>( auto dq_n = std::make_shared<ngraph::op::Dequantize>(
qconv_n, dq_m->get_argument(1), zp, dq_m->get_output_element_type(0), dq_m->get_axes()); qconv_n, dq_m->get_argument(1), zp, dq_m->get_output_element_type(0), dq_m->get_axes());
ngraph::replace_node(m.get_match_root(), dq_n); ngraph::replace_node(m.get_match_root(), dq_n);
return true; return true;
...@@ -2008,24 +2056,25 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qavg_pool() ...@@ -2008,24 +2056,25 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qavg_pool()
auto input = std::make_shared<pattern::op::Label>(element::i8, shape); auto input = std::make_shared<pattern::op::Label>(element::i8, shape);
auto dq_scale = std::make_shared<pattern::op::Label>(element::f32, Shape{}); auto dq_scale = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto dq_zp = std::make_shared<pattern::op::Label>(element::i8, Shape{}); auto dq_zp = std::make_shared<pattern::op::Label>(element::i8, Shape{});
auto dq = std::make_shared<op::Dequantize>(input, dq_scale, dq_zp, element::f32, AxisSet{}); auto dq =
auto avg_pool = std::make_shared<op::AvgPool>(dq, Shape{1, 1}); std::make_shared<ngraph::op::Dequantize>(input, dq_scale, dq_zp, element::f32, AxisSet{});
auto avg_pool = std::make_shared<ngraph::op::AvgPool>(dq, Shape{1, 1});
pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_qavg_pool against " NGRAPH_DEBUG << "In a callback for construct_qavg_pool against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto avg_pool_m = std::static_pointer_cast<op::AvgPool>(m.get_match_root()); auto avg_pool_m = std::static_pointer_cast<ngraph::op::AvgPool>(m.get_match_root());
auto dq_m = std::static_pointer_cast<op::Dequantize>(avg_pool_m->get_argument(0)); auto dq_m = std::static_pointer_cast<ngraph::op::Dequantize>(avg_pool_m->get_argument(0));
auto qavg_pool_n = std::make_shared<op::QuantizedAvgPool>( auto qavg_pool_n = std::make_shared<ngraph::op::QuantizedAvgPool>(
dq_m->get_argument(0), dq_m->get_argument(0),
avg_pool_m->get_window_shape(), avg_pool_m->get_window_shape(),
avg_pool_m->get_window_movement_strides(), avg_pool_m->get_window_movement_strides(),
avg_pool_m->get_padding_below(), avg_pool_m->get_padding_below(),
avg_pool_m->get_padding_above(), avg_pool_m->get_padding_above(),
avg_pool_m->get_include_padding_in_avg_computation()); avg_pool_m->get_include_padding_in_avg_computation());
auto dq_n = std::make_shared<op::Dequantize>(qavg_pool_n, auto dq_n = std::make_shared<ngraph::op::Dequantize>(qavg_pool_n,
dq_m->get_argument(1), dq_m->get_argument(1),
dq_m->get_argument(2), dq_m->get_argument(2),
dq_m->get_output_element_type(0), dq_m->get_output_element_type(0),
...@@ -2045,23 +2094,24 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qmax_pool() ...@@ -2045,23 +2094,24 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qmax_pool()
auto input = std::make_shared<pattern::op::Label>(element::i8, shape); auto input = std::make_shared<pattern::op::Label>(element::i8, shape);
auto dq_scale = std::make_shared<pattern::op::Label>(element::f32, Shape{}); auto dq_scale = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto dq_zp = std::make_shared<pattern::op::Label>(element::i8, Shape{}); auto dq_zp = std::make_shared<pattern::op::Label>(element::i8, Shape{});
auto dq = std::make_shared<op::Dequantize>(input, dq_scale, dq_zp, element::f32, AxisSet{}); auto dq =
auto max_pool = std::make_shared<op::MaxPool>(dq, Shape{1, 1}); std::make_shared<ngraph::op::Dequantize>(input, dq_scale, dq_zp, element::f32, AxisSet{});
auto max_pool = std::make_shared<ngraph::op::MaxPool>(dq, Shape{1, 1});
pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_qmax_pool against " NGRAPH_DEBUG << "In a callback for construct_qmax_pool against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto max_pool_m = std::static_pointer_cast<op::MaxPool>(m.get_match_root()); auto max_pool_m = std::static_pointer_cast<ngraph::op::MaxPool>(m.get_match_root());
auto dq_m = std::static_pointer_cast<op::Dequantize>(max_pool_m->get_argument(0)); auto dq_m = std::static_pointer_cast<ngraph::op::Dequantize>(max_pool_m->get_argument(0));
auto qmax_pool_n = auto qmax_pool_n = std::make_shared<ngraph::op::QuantizedMaxPool>(
std::make_shared<op::QuantizedMaxPool>(dq_m->get_argument(0), dq_m->get_argument(0),
max_pool_m->get_window_shape(), max_pool_m->get_window_shape(),
max_pool_m->get_window_movement_strides(), max_pool_m->get_window_movement_strides(),
max_pool_m->get_padding_below(), max_pool_m->get_padding_below(),
max_pool_m->get_padding_above()); max_pool_m->get_padding_above());
auto dq_n = std::make_shared<op::Dequantize>(qmax_pool_n, auto dq_n = std::make_shared<ngraph::op::Dequantize>(qmax_pool_n,
dq_m->get_argument(1), dq_m->get_argument(1),
dq_m->get_argument(2), dq_m->get_argument(2),
dq_m->get_output_element_type(0), dq_m->get_output_element_type(0),
...@@ -2086,15 +2136,15 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconcat() ...@@ -2086,15 +2136,15 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconcat()
for (size_t i = 0; i < 5; i++) for (size_t i = 0; i < 5; i++)
{ {
inputs.push_back(std::make_shared<pattern::op::Label>(element::f32, shape)); inputs.push_back(std::make_shared<pattern::op::Label>(element::f32, shape));
concats.push_back(std::make_shared<op::Concat>(inputs, 0)); concats.push_back(std::make_shared<ngraph::op::Concat>(inputs, 0));
} }
pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_qconcat against " NGRAPH_DEBUG << "In a callback for construct_qconcat against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto concat_m = std::static_pointer_cast<op::Concat>(m.get_match_root()); auto concat_m = std::static_pointer_cast<ngraph::op::Concat>(m.get_match_root());
auto dq_m = std::static_pointer_cast<op::Dequantize>(concat_m->get_argument(0)); auto dq_m = std::static_pointer_cast<ngraph::op::Dequantize>(concat_m->get_argument(0));
NodeVector new_args; NodeVector new_args;
for (auto arg : concat_m->get_arguments()) for (auto arg : concat_m->get_arguments())
{ {
...@@ -2112,9 +2162,9 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconcat() ...@@ -2112,9 +2162,9 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconcat()
new_args.push_back(arg->get_argument(0)); new_args.push_back(arg->get_argument(0));
} }
auto concat_n = auto concat_n = std::make_shared<ngraph::op::QuantizedConcat>(
std::make_shared<op::QuantizedConcat>(new_args, concat_m->get_concatenation_axis()); new_args, concat_m->get_concatenation_axis());
auto dq_n = std::make_shared<op::Dequantize>(concat_n, auto dq_n = std::make_shared<ngraph::op::Dequantize>(concat_n,
dq_m->get_argument(1), dq_m->get_argument(1),
dq_m->get_argument(2), dq_m->get_argument(2),
dq_m->get_element_type(), dq_m->get_element_type(),
...@@ -2141,16 +2191,19 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_dq_q() ...@@ -2141,16 +2191,19 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_dq_q()
auto q_scale = std::make_shared<pattern::op::Label>(element::f32, Shape{}); auto q_scale = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto q_zp = std::make_shared<pattern::op::Label>(element::i8, Shape{}); auto q_zp = std::make_shared<pattern::op::Label>(element::i8, Shape{});
auto dq = std::make_shared<op::Dequantize>(input, dq_scale, dq_zp, element::f32, AxisSet{}); auto dq =
op::Quantize::RoundMode round_mode = op::Quantize::RoundMode::ROUND_NEAREST_TOWARD_EVEN; std::make_shared<ngraph::op::Dequantize>(input, dq_scale, dq_zp, element::f32, AxisSet{});
auto q = std::make_shared<op::Quantize>(dq, q_scale, q_zp, element::i8, AxisSet{}, round_mode); ngraph::op::Quantize::RoundMode round_mode =
ngraph::op::Quantize::RoundMode::ROUND_NEAREST_TOWARD_EVEN;
auto q = std::make_shared<ngraph::op::Quantize>(
dq, q_scale, q_zp, element::i8, AxisSet{}, round_mode);
pattern::graph_rewrite_callback callback = [input](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [input](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_dq_q against " NGRAPH_DEBUG << "In a callback for construct_dq_q against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto q_m = std::static_pointer_cast<op::Quantize>(m.get_match_root()); auto q_m = std::static_pointer_cast<ngraph::op::Quantize>(m.get_match_root());
auto dq_m = std::static_pointer_cast<op::Dequantize>(q_m->get_argument(0)); auto dq_m = std::static_pointer_cast<ngraph::op::Dequantize>(q_m->get_argument(0));
if (!(ngraph::is_zero(q_m->get_argument(2)) && ngraph::is_zero(dq_m->get_argument(2)))) if (!(ngraph::is_zero(q_m->get_argument(2)) && ngraph::is_zero(dq_m->get_argument(2))))
{ {
NGRAPH_DEBUG << "Non-zero zero points"; NGRAPH_DEBUG << "Non-zero zero points";
...@@ -2195,7 +2248,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add() ...@@ -2195,7 +2248,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add()
auto dq_zp2 = std::make_shared<pattern::op::Label>(element::i8, Shape{}); auto dq_zp2 = std::make_shared<pattern::op::Label>(element::i8, Shape{});
// Left Graph // Left Graph
auto qconvb = std::make_shared<op::QuantizedConvolutionBias>(data_batch, auto qconvb = std::make_shared<ngraph::op::QuantizedConvolutionBias>(data_batch,
filters, filters,
bias, bias,
Strides{1, 1}, Strides{1, 1},
...@@ -2206,8 +2259,8 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add() ...@@ -2206,8 +2259,8 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add()
requantization_scale, requantization_scale,
false); false);
auto qconvb_label = std::make_shared<pattern::op::Label>(qconvb, nullptr, NodeVector{qconvb}); auto qconvb_label = std::make_shared<pattern::op::Label>(qconvb, nullptr, NodeVector{qconvb});
auto dq_l = auto dq_l = std::make_shared<ngraph::op::Dequantize>(
std::make_shared<op::Dequantize>(qconvb_label, dq_scale1, dq_zp1, element::f32, AxisSet{}); qconvb_label, dq_scale1, dq_zp1, element::f32, AxisSet{});
auto dq_l_label = std::make_shared<pattern::op::Label>(dq_l, nullptr, NodeVector{dq_l}); auto dq_l_label = std::make_shared<pattern::op::Label>(dq_l, nullptr, NodeVector{dq_l});
auto skipr_l = std::make_shared<pattern::op::Skip>( auto skipr_l = std::make_shared<pattern::op::Skip>(
dq_l_label, [](std::shared_ptr<Node> n) { return n->description() == "Reshape"; }); dq_l_label, [](std::shared_ptr<Node> n) { return n->description() == "Reshape"; });
...@@ -2216,8 +2269,8 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add() ...@@ -2216,8 +2269,8 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add()
//Right Graph //Right Graph
auto summand = std::make_shared<pattern::op::Label>(element::i8, qconvb->get_shape()); auto summand = std::make_shared<pattern::op::Label>(element::i8, qconvb->get_shape());
auto dq_r = auto dq_r = std::make_shared<ngraph::op::Dequantize>(
std::make_shared<op::Dequantize>(summand, dq_scale2, dq_zp2, element::f32, AxisSet{}); summand, dq_scale2, dq_zp2, element::f32, AxisSet{});
auto dq_r_label = std::make_shared<pattern::op::Label>(dq_r, nullptr, NodeVector{dq_r}); auto dq_r_label = std::make_shared<pattern::op::Label>(dq_r, nullptr, NodeVector{dq_r});
auto skipr_r = std::make_shared<pattern::op::Skip>( auto skipr_r = std::make_shared<pattern::op::Skip>(
dq_r_label, [](std::shared_ptr<Node> n) { return n->description() == "Reshape"; }); dq_r_label, [](std::shared_ptr<Node> n) { return n->description() == "Reshape"; });
...@@ -2226,17 +2279,18 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add() ...@@ -2226,17 +2279,18 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add()
//Add left + right //Add left + right
auto add = skipb_l + skipb_r; auto add = skipb_l + skipb_r;
auto prelu = std::make_shared<op::Relu>(add); auto prelu = std::make_shared<ngraph::op::Relu>(add);
pattern::graph_rewrite_callback callback = [dq_l_label, dq_r_label](pattern::Matcher& m) { pattern::graph_rewrite_callback callback = [dq_l_label, dq_r_label](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_qconvb_dq_add_relu against " NGRAPH_DEBUG << "In a callback for construct_qconvb_dq_add_relu against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto add_m = std::dynamic_pointer_cast<op::Add>(m.get_match_root()->get_argument(0)); auto add_m =
auto dq_l_m = std::dynamic_pointer_cast<op::Dequantize>(pattern_map[dq_l_label]); std::dynamic_pointer_cast<ngraph::op::Add>(m.get_match_root()->get_argument(0));
auto dq_r_m = std::dynamic_pointer_cast<op::Dequantize>(pattern_map[dq_r_label]); auto dq_l_m = std::dynamic_pointer_cast<ngraph::op::Dequantize>(pattern_map[dq_l_label]);
auto dq_r_m = std::dynamic_pointer_cast<ngraph::op::Dequantize>(pattern_map[dq_r_label]);
auto qconv = auto qconv =
std::static_pointer_cast<op::QuantizedConvolutionBias>(dq_l_m->get_argument(0)); std::static_pointer_cast<ngraph::op::QuantizedConvolutionBias>(dq_l_m->get_argument(0));
auto inplace_input = dq_r_m->get_argument(0); auto inplace_input = dq_r_m->get_argument(0);
if (!(ngraph::is_zero(dq_l_m->get_argument(2)) && ngraph::is_zero(dq_r_m->get_argument(2)))) if (!(ngraph::is_zero(dq_l_m->get_argument(2)) && ngraph::is_zero(dq_r_m->get_argument(2))))
...@@ -2289,8 +2343,8 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add() ...@@ -2289,8 +2343,8 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add()
if (dq_r_m->get_input_element_type(2) == element::i8) if (dq_r_m->get_input_element_type(2) == element::i8)
{ {
// TODO (jbobba): Investigate the need for Convert op // TODO (jbobba): Investigate the need for Convert op
qconvba = std::make_shared<op::Convert>( qconvba = std::make_shared<ngraph::op::Convert>(
std::make_shared<op::QuantizedConvolutionBiasSignedAdd>( std::make_shared<ngraph::op::QuantizedConvolutionBiasSignedAdd>(
qconv->get_argument(0), qconv->get_argument(0),
qconv->get_argument(1), qconv->get_argument(1),
qconv->get_argument(2), qconv->get_argument(2),
...@@ -2307,7 +2361,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add() ...@@ -2307,7 +2361,7 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add()
} }
else else
{ {
qconvba = std::make_shared<op::QuantizedConvolutionBiasAdd>( qconvba = std::make_shared<ngraph::op::QuantizedConvolutionBiasAdd>(
qconv->get_argument(0), qconv->get_argument(0),
qconv->get_argument(1), qconv->get_argument(1),
qconv->get_argument(2), qconv->get_argument(2),
...@@ -2321,9 +2375,9 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add() ...@@ -2321,9 +2375,9 @@ void ngraph::runtime::cpu::pass::CPUQuantFusion::construct_qconvb_add()
sum_scale, sum_scale,
true); true);
} }
auto zp = op::Constant::create(element::u8, Shape{}, {0}); auto zp = ngraph::op::Constant::create(element::u8, Shape{}, {0});
auto DQ = auto DQ = std::make_shared<ngraph::op::Dequantize>(
std::make_shared<op::Dequantize>(qconvba, dq_l_scale, zp, element::f32, AxisSet{}); qconvba, dq_l_scale, zp, element::f32, AxisSet{});
ngraph::replace_node(m.get_match_root(), DQ); ngraph::replace_node(m.get_match_root(), DQ);
return true; return true;
......
...@@ -64,7 +64,7 @@ bool runtime::cpu::pass::CPUMemoryOptimization::run_on_function(std::shared_ptr< ...@@ -64,7 +64,7 @@ bool runtime::cpu::pass::CPUMemoryOptimization::run_on_function(std::shared_ptr<
{ {
if (n->description() == "Concat") if (n->description() == "Concat")
{ {
auto concat = std::static_pointer_cast<op::Concat>(n); auto concat = std::static_pointer_cast<ngraph::op::Concat>(n);
auto shape = concat->get_input_shape(0); auto shape = concat->get_input_shape(0);
auto axis = concat->get_concatenation_axis(); auto axis = concat->get_concatenation_axis();
auto product = 1; auto product = 1;
...@@ -134,7 +134,7 @@ bool runtime::cpu::pass::CPUMemoryOptimization::run_on_function(std::shared_ptr< ...@@ -134,7 +134,7 @@ bool runtime::cpu::pass::CPUMemoryOptimization::run_on_function(std::shared_ptr<
{ {
if (arg->is_op()) if (arg->is_op())
{ {
auto op = std::static_pointer_cast<op::Op>(arg); auto op = std::static_pointer_cast<ngraph::op::Op>(arg);
auto annotation = op->get_op_annotations(); auto annotation = op->get_op_annotations();
if (annotation && annotation->get_in_place_oi_pairs().size() > 0) if (annotation && annotation->get_in_place_oi_pairs().size() > 0)
...@@ -177,7 +177,7 @@ bool runtime::cpu::pass::CPUMemoryOptimization::run_on_function(std::shared_ptr< ...@@ -177,7 +177,7 @@ bool runtime::cpu::pass::CPUMemoryOptimization::run_on_function(std::shared_ptr<
{ {
if (user->is_op()) if (user->is_op())
{ {
auto op = std::static_pointer_cast<op::Op>(user); auto op = std::static_pointer_cast<ngraph::op::Op>(user);
if (auto op_annotations = op->get_op_annotations()) if (auto op_annotations = op->get_op_annotations())
{ {
if (op_annotations->get_in_place_oi_pairs().size() > 0) if (op_annotations->get_in_place_oi_pairs().size() > 0)
...@@ -227,7 +227,7 @@ bool runtime::cpu::pass::CPUMemoryOptimization::run_on_function(std::shared_ptr< ...@@ -227,7 +227,7 @@ bool runtime::cpu::pass::CPUMemoryOptimization::run_on_function(std::shared_ptr<
{ {
if (n->description() == "Slice") if (n->description() == "Slice")
{ {
auto slice = std::static_pointer_cast<op::Slice>(n); auto slice = std::static_pointer_cast<ngraph::op::Slice>(n);
auto in_shape = slice->get_input_shape(0); auto in_shape = slice->get_input_shape(0);
auto out_shape = slice->get_output_shape(0); auto out_shape = slice->get_output_shape(0);
auto strides = slice->get_strides(); auto strides = slice->get_strides();
......
...@@ -66,15 +66,16 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_sigmoid() ...@@ -66,15 +66,16 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_sigmoid()
{ {
// construct variance // construct variance
auto input = std::make_shared<pattern::op::Label>(element::f32, Shape{3, 4}); auto input = std::make_shared<pattern::op::Label>(element::f32, Shape{3, 4});
auto neg_input = std::make_shared<op::Negative>(input); auto neg_input = std::make_shared<ngraph::op::Negative>(input);
auto exp_neg_input = std::make_shared<op::Exp>(neg_input); auto exp_neg_input = std::make_shared<ngraph::op::Exp>(neg_input);
// broadcast input // broadcast input
auto constant = std::make_shared<pattern::op::Label>(element::f32, Shape{}); auto constant = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto broadcast_constant = std::make_shared<op::Broadcast>(constant, Shape{3, 4}, AxisSet{0, 1}); auto broadcast_constant =
std::make_shared<ngraph::op::Broadcast>(constant, Shape{3, 4}, AxisSet{0, 1});
auto add_exp = std::make_shared<op::Add>(exp_neg_input, broadcast_constant); auto add_exp = std::make_shared<ngraph::op::Add>(exp_neg_input, broadcast_constant);
auto divide_1_over_exp = std::make_shared<op::Divide>(broadcast_constant, add_exp); auto divide_1_over_exp = std::make_shared<ngraph::op::Divide>(broadcast_constant, add_exp);
// Define a call back that needs to called once the DFG matches the pattern // Define a call back that needs to called once the DFG matches the pattern
ngraph::pattern::graph_rewrite_callback callback = [input](pattern::Matcher& m) { ngraph::pattern::graph_rewrite_callback callback = [input](pattern::Matcher& m) {
...@@ -96,7 +97,7 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_sigmoid() ...@@ -96,7 +97,7 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_sigmoid()
return false; return false;
} }
auto sigmoid_node = std::make_shared<op::Sigmoid>(pattern_map[input]); auto sigmoid_node = std::make_shared<ngraph::op::Sigmoid>(pattern_map[input]);
ngraph::replace_node(m.get_match_root(), sigmoid_node); ngraph::replace_node(m.get_match_root(), sigmoid_node);
return true; return true;
}; };
...@@ -147,43 +148,50 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop() ...@@ -147,43 +148,50 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop()
auto ct_1 = std::make_shared<pattern::op::Label>(element::f32, Shape{10, 100}); auto ct_1 = std::make_shared<pattern::op::Label>(element::f32, Shape{10, 100});
auto broadcast_pred = [](std::shared_ptr<Node> n) { auto broadcast_pred = [](std::shared_ptr<Node> n) {
return ((std::dynamic_pointer_cast<op::Broadcast>(n) != nullptr) || return ((std::dynamic_pointer_cast<ngraph::op::Broadcast>(n) != nullptr) ||
(std::dynamic_pointer_cast<op::Reshape>(n) != nullptr)); (std::dynamic_pointer_cast<ngraph::op::Reshape>(n) != nullptr));
}; };
// Fused MatMuls // Fused MatMuls
// (W_{ii} | (W_{if} | W_{ig} | W_{io}) * x_t + (b_{ii} | b_{if} | b_{ig} | b_{io}) // (W_{ii} | (W_{if} | W_{ig} | W_{io}) * x_t + (b_{ii} | b_{if} | b_{ig} | b_{io})
auto dot1 = std::make_shared<op::Dot>(xt, w_i2h); auto dot1 = std::make_shared<ngraph::op::Dot>(xt, w_i2h);
auto add1 = std::make_shared<op::Add>( auto add1 = std::make_shared<ngraph::op::Add>(
dot1, std::make_shared<pattern::op::Skip>(bias_i2h, broadcast_pred)); dot1, std::make_shared<pattern::op::Skip>(bias_i2h, broadcast_pred));
// (W_{hi} | (W_{hf} | W_{hg} | W_{ho}) * h_{(t-1)} + (b_{hi} | b_{hf} | b_{hg} | b_{ho}) // (W_{hi} | (W_{hf} | W_{hg} | W_{ho}) * h_{(t-1)} + (b_{hi} | b_{hf} | b_{hg} | b_{ho})
auto dot2 = std::make_shared<op::Dot>(ht_1, w_h2h); auto dot2 = std::make_shared<ngraph::op::Dot>(ht_1, w_h2h);
auto add2 = std::make_shared<op::Add>( auto add2 = std::make_shared<ngraph::op::Add>(
dot2, std::make_shared<pattern::op::Skip>(bias_h2h, broadcast_pred)); dot2, std::make_shared<pattern::op::Skip>(bias_h2h, broadcast_pred));
auto X = std::make_shared<op::Add>(add2, add1); auto X = std::make_shared<ngraph::op::Add>(add2, add1);
// construct gates // construct gates
auto it = std::make_shared<op::Sigmoid>( auto it = std::make_shared<ngraph::op::Sigmoid>(
std::make_shared<op::Slice>(X, Coordinate{0, 0}, Coordinate{10, 100})); std::make_shared<ngraph::op::Slice>(X, Coordinate{0, 0}, Coordinate{10, 100}));
auto ft = std::make_shared<op::Sigmoid>( auto ft = std::make_shared<ngraph::op::Sigmoid>(
std::make_shared<op::Slice>(X, Coordinate{0, 100}, Coordinate{10, 200})); std::make_shared<ngraph::op::Slice>(X, Coordinate{0, 100}, Coordinate{10, 200}));
auto gt = std::make_shared<op::Tanh>( auto gt = std::make_shared<ngraph::op::Tanh>(
std::make_shared<op::Slice>(X, Coordinate{0, 200}, Coordinate{10, 300})); std::make_shared<ngraph::op::Slice>(X, Coordinate{0, 200}, Coordinate{10, 300}));
auto ot = std::make_shared<op::Sigmoid>( auto ot = std::make_shared<ngraph::op::Sigmoid>(
std::make_shared<op::Slice>(X, Coordinate{0, 300}, Coordinate{10, 400})); std::make_shared<ngraph::op::Slice>(X, Coordinate{0, 300}, Coordinate{10, 400}));
// construct (c_t) cell state // construct (c_t) cell state
auto ct = std::make_shared<op::Add>(std::make_shared<op::Multiply>(ft, ct_1), auto ct = std::make_shared<ngraph::op::Add>(std::make_shared<ngraph::op::Multiply>(ft, ct_1),
std::make_shared<op::Multiply>(it, gt)); std::make_shared<ngraph::op::Multiply>(it, gt));
auto ct_label = std::make_shared<pattern::op::Label>(ct, nullptr, NodeVector{ct}); auto ct_label = std::make_shared<pattern::op::Label>(ct, nullptr, NodeVector{ct});
// construct (h_t) // construct (h_t)
auto ht = std::make_shared<op::Multiply>(ot, std::make_shared<op::Tanh>(ct_label)); auto ht =
std::make_shared<ngraph::op::Multiply>(ot, std::make_shared<ngraph::op::Tanh>(ct_label));
// Define a call back that needs to called once the DFG matches the pattern // Define a call back that needs to called once the DFG matches the pattern
pattern::graph_rewrite_callback callback = pattern::graph_rewrite_callback callback = [ct_label,
[ct_label, w_i2h, bias_i2h, w_h2h, bias_h2h, xt, ht_1, ct_1](pattern::Matcher& m) { w_i2h,
bias_i2h,
w_h2h,
bias_h2h,
xt,
ht_1,
ct_1](pattern::Matcher& m) {
NGRAPH_DEBUG << "In a callback for construct_fprop_lstm pattern against " NGRAPH_DEBUG << "In a callback for construct_fprop_lstm pattern against "
<< m.get_match_root()->get_name(); << m.get_match_root()->get_name();
...@@ -213,10 +221,10 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop() ...@@ -213,10 +221,10 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop()
// this checks avoids fusing of LSTM cells if its a part of decoder, we // this checks avoids fusing of LSTM cells if its a part of decoder, we
// will remove this once mkldnn optimizes individual LSTM cell or once // will remove this once mkldnn optimizes individual LSTM cell or once
// we have decoder pattern for GNMT. // we have decoder pattern for GNMT.
if (!(std::dynamic_pointer_cast<op::Broadcast>(cell_state) && if (!(std::dynamic_pointer_cast<ngraph::op::Broadcast>(cell_state) &&
std::dynamic_pointer_cast<op::Constant>(cell_state->get_argument(0))) && std::dynamic_pointer_cast<ngraph::op::Constant>(cell_state->get_argument(0))) &&
!(std::dynamic_pointer_cast<op::Slice>(cell_state) && !(std::dynamic_pointer_cast<ngraph::op::Slice>(cell_state) &&
std::dynamic_pointer_cast<op::GetOutputElement>(cell_state->get_argument(0)))) std::dynamic_pointer_cast<ngraph::op::GetOutputElement>(cell_state->get_argument(0))))
{ {
return false; return false;
} }
...@@ -232,8 +240,8 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop() ...@@ -232,8 +240,8 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop()
// pattern matcher cannot guarantee this since the computations are // pattern matcher cannot guarantee this since the computations are
// symmetric around x_t and ht_1. Use heuristics to swap the matched // symmetric around x_t and ht_1. Use heuristics to swap the matched
// labels // labels
if (std::dynamic_pointer_cast<op::Broadcast>(src_layer) && if (std::dynamic_pointer_cast<ngraph::op::Broadcast>(src_layer) &&
std::dynamic_pointer_cast<op::Constant>(src_layer->get_argument(0))) std::dynamic_pointer_cast<ngraph::op::Constant>(src_layer->get_argument(0)))
{ {
// First timestep of an RNN layer // First timestep of an RNN layer
swap_lstm_inputs(); swap_lstm_inputs();
...@@ -242,7 +250,8 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop() ...@@ -242,7 +250,8 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop()
{ {
swap_lstm_inputs(); swap_lstm_inputs();
} }
else if (std::dynamic_pointer_cast<op::GetOutputElement>(cell_state->get_argument(0))) else if (std::dynamic_pointer_cast<ngraph::op::GetOutputElement>(
cell_state->get_argument(0)))
{ {
// swap the inputs if the cell_state and hidden state does not // swap the inputs if the cell_state and hidden state does not
// belong to the same Lstm // belong to the same Lstm
...@@ -256,8 +265,7 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop() ...@@ -256,8 +265,7 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop()
if (hidden_state->get_shape() != cell_state->get_shape()) if (hidden_state->get_shape() != cell_state->get_shape())
{ {
NGRAPH_DEBUG NGRAPH_DEBUG << "Lstm MKLDNN kernel requires recurrent output hidden states to match ";
<< "Lstm MKLDNN kernel requires recurrent output hidden states to match ";
return false; return false;
} }
...@@ -281,26 +289,26 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop() ...@@ -281,26 +289,26 @@ void ngraph::runtime::cpu::pass::LSTMFusion::construct_lstm_fprop()
} }
std::shared_ptr<Node> src_iter = std::shared_ptr<Node> src_iter =
std::make_shared<op::Concat>(NodeVector{hidden_state, cell_state}, 0); std::make_shared<ngraph::op::Concat>(NodeVector{hidden_state, cell_state}, 0);
if (src_layer->get_shape()[1] != slc || src_iter->get_shape()[1] != sic) if (src_layer->get_shape()[1] != slc || src_iter->get_shape()[1] != sic)
{ {
NGRAPH_DEBUG << "Feature size mismatch between weights and input tensors"; NGRAPH_DEBUG << "Feature size mismatch between weights and input tensors";
return false; return false;
} }
auto bias = std::make_shared<op::Add>(pattern_map[bias_i2h], pattern_map[bias_h2h]); auto bias = std::make_shared<ngraph::op::Add>(pattern_map[bias_i2h], pattern_map[bias_h2h]);
auto lstm_node = std::make_shared<op::Lstm>( auto lstm_node = std::make_shared<ngraph::op::Lstm>(
src_layer, src_iter, weights_layer, weights_iter, bias, rnn_type); src_layer, src_iter, weights_layer, weights_iter, bias, rnn_type);
auto lstm_ht_output = std::make_shared<op::GetOutputElement>(lstm_node, 0); auto lstm_ht_output = std::make_shared<ngraph::op::GetOutputElement>(lstm_node, 0);
auto lstm_ht_ct_output = std::make_shared<op::GetOutputElement>(lstm_node, 1); auto lstm_ht_ct_output = std::make_shared<ngraph::op::GetOutputElement>(lstm_node, 1);
// dst_iter of lstm mkldnn output holds the results of both recurrent state // dst_iter of lstm mkldnn output holds the results of both recurrent state
// tensor outputs. we need to slice the ct. // tensor outputs. we need to slice the ct.
auto ht_slice = std::make_shared<op::Slice>( auto ht_slice = std::make_shared<ngraph::op::Slice>(
lstm_ht_output, Coordinate{0, 0}, Coordinate{batch_size, dlc}); lstm_ht_output, Coordinate{0, 0}, Coordinate{batch_size, dlc});
auto ct_slice = std::make_shared<op::Slice>( auto ct_slice = std::make_shared<ngraph::op::Slice>(
lstm_ht_ct_output, Coordinate{batch_size, 0}, Coordinate{(2 * batch_size), dic}); lstm_ht_ct_output, Coordinate{batch_size, 0}, Coordinate{(2 * batch_size), dic});
if (lstm_node->get_outputs().at(0).get_inputs().size() != 2) if (lstm_node->get_outputs().at(0).get_inputs().size() != 2)
...@@ -330,44 +338,45 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop() ...@@ -330,44 +338,45 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop()
auto lstm_ht = std::make_shared<pattern::op::Label>(element::f32, Shape{10, 100}); auto lstm_ht = std::make_shared<pattern::op::Label>(element::f32, Shape{10, 100});
auto lstm_ct = std::make_shared<pattern::op::Label>(element::f32, Shape{10, 100}); auto lstm_ct = std::make_shared<pattern::op::Label>(element::f32, Shape{10, 100});
auto lstm_src_iter = std::make_shared<op::Concat>(NodeVector{lstm_ht, lstm_ct}, 0); auto lstm_src_iter = std::make_shared<ngraph::op::Concat>(NodeVector{lstm_ht, lstm_ct}, 0);
auto lstm_src_iter_label = auto lstm_src_iter_label =
std::make_shared<pattern::op::Label>(lstm_src_iter, nullptr, NodeVector{lstm_src_iter}); std::make_shared<pattern::op::Label>(lstm_src_iter, nullptr, NodeVector{lstm_src_iter});
auto lstm_weights_layer_shared = std::make_shared<pattern::op::Label>( auto lstm_weights_layer_shared = std::make_shared<pattern::op::Label>(
element::f32, Shape{400, 100}, pattern::has_class<op::Parameter>()); element::f32, Shape{400, 100}, pattern::has_class<ngraph::op::Parameter>());
auto lstm_weights_layer = auto lstm_weights_layer = std::make_shared<ngraph::op::Reshape>(
std::make_shared<op::Reshape>(lstm_weights_layer_shared, AxisVector{1, 0}, Shape{100, 400}); lstm_weights_layer_shared, AxisVector{1, 0}, Shape{100, 400});
auto lstm_weights_layer_label = std::make_shared<pattern::op::Label>( auto lstm_weights_layer_label = std::make_shared<pattern::op::Label>(
lstm_weights_layer, nullptr, NodeVector{lstm_weights_layer}); lstm_weights_layer, nullptr, NodeVector{lstm_weights_layer});
auto lstm_weights_iter_shared = std::make_shared<pattern::op::Label>( auto lstm_weights_iter_shared = std::make_shared<pattern::op::Label>(
element::f32, Shape{400, 100}, pattern::has_class<op::Parameter>()); element::f32, Shape{400, 100}, pattern::has_class<ngraph::op::Parameter>());
auto lstm_weights_iter = auto lstm_weights_iter = std::make_shared<ngraph::op::Reshape>(
std::make_shared<op::Reshape>(lstm_weights_iter_shared, AxisVector{1, 0}, Shape{100, 400}); lstm_weights_iter_shared, AxisVector{1, 0}, Shape{100, 400});
auto lstm_weights_iter_label = std::make_shared<pattern::op::Label>( auto lstm_weights_iter_label = std::make_shared<pattern::op::Label>(
lstm_weights_iter, nullptr, NodeVector{lstm_weights_iter}); lstm_weights_iter, nullptr, NodeVector{lstm_weights_iter});
auto lstm_bias_layer_shared = std::make_shared<pattern::op::Label>(element::f32, Shape{400}); auto lstm_bias_layer_shared = std::make_shared<pattern::op::Label>(element::f32, Shape{400});
auto lstm_bias_iter_shared = std::make_shared<pattern::op::Label>(element::f32, Shape{400}); auto lstm_bias_iter_shared = std::make_shared<pattern::op::Label>(element::f32, Shape{400});
auto lstm_bias = std::make_shared<op::Add>(lstm_bias_layer_shared, lstm_bias_iter_shared); auto lstm_bias =
std::make_shared<ngraph::op::Add>(lstm_bias_layer_shared, lstm_bias_iter_shared);
auto lstm_bias_label = auto lstm_bias_label =
std::make_shared<pattern::op::Label>(lstm_bias, nullptr, NodeVector{lstm_bias}); std::make_shared<pattern::op::Label>(lstm_bias, nullptr, NodeVector{lstm_bias});
ngraph::runtime::cpu::rnn_utils::rnntype ref_rnn_type = ngraph::runtime::cpu::rnn_utils::rnntype ref_rnn_type =
ngraph::runtime::cpu::rnn_utils::rnntype::vanilla_lstm; ngraph::runtime::cpu::rnn_utils::rnntype::vanilla_lstm;
auto lstm = std::make_shared<op::Lstm>(lstm_src_layer, auto lstm = std::make_shared<ngraph::op::Lstm>(lstm_src_layer,
lstm_src_iter_label, lstm_src_iter_label,
lstm_weights_layer_label, lstm_weights_layer_label,
lstm_weights_iter_label, lstm_weights_iter_label,
lstm_bias_label, lstm_bias_label,
ref_rnn_type); ref_rnn_type);
auto lstm_goe = std::make_shared<op::GetOutputElement>(lstm, 1); auto lstm_goe = std::make_shared<ngraph::op::GetOutputElement>(lstm, 1);
// We cannot attach labels to multi-output nodes, so we attach a label to the goe instead // We cannot attach labels to multi-output nodes, so we attach a label to the goe instead
auto lstm_goe_label = auto lstm_goe_label =
std::make_shared<pattern::op::Label>(lstm_goe, nullptr, NodeVector{lstm_goe}); std::make_shared<pattern::op::Label>(lstm_goe, nullptr, NodeVector{lstm_goe});
auto lstm_goe_slice = auto lstm_goe_slice =
std::make_shared<op::Slice>(lstm_goe_label, Coordinate{10, 0}, Coordinate{20, 100}); std::make_shared<ngraph::op::Slice>(lstm_goe_label, Coordinate{10, 0}, Coordinate{20, 100});
pattern::recurrent_graph_rewrite_callback callback = [lstm_goe_label, pattern::recurrent_graph_rewrite_callback callback = [lstm_goe_label,
lstm_src_layer, lstm_src_layer,
...@@ -387,7 +396,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop() ...@@ -387,7 +396,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop()
{ {
auto node_labels = m.get_bound_nodes_for_pattern(input_label); auto node_labels = m.get_bound_nodes_for_pattern(input_label);
std::reverse(node_labels.begin(), node_labels.end()); std::reverse(node_labels.begin(), node_labels.end());
return std::make_shared<op::Concat>(node_labels, 0); return std::make_shared<ngraph::op::Concat>(node_labels, 0);
} }
}; };
...@@ -429,9 +438,9 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop() ...@@ -429,9 +438,9 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop()
} }
auto check_const_input = [&](std::shared_ptr<Node> n) { auto check_const_input = [&](std::shared_ptr<Node> n) {
if (std::dynamic_pointer_cast<op::Constant>(n) || if (std::dynamic_pointer_cast<ngraph::op::Constant>(n) ||
(std::dynamic_pointer_cast<op::Broadcast>(n) && (std::dynamic_pointer_cast<ngraph::op::Broadcast>(n) &&
std::dynamic_pointer_cast<op::Constant>(n->get_argument(0)))) std::dynamic_pointer_cast<ngraph::op::Constant>(n->get_argument(0))))
{ {
return true; return true;
} }
...@@ -460,7 +469,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop() ...@@ -460,7 +469,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop()
return false; return false;
} }
auto rnn = std::make_shared<op::Rnn>(rnn_src_layer, auto rnn = std::make_shared<ngraph::op::Rnn>(rnn_src_layer,
rnn_src_iter, rnn_src_iter,
rnn_weights_layer, rnn_weights_layer,
rnn_weights_iter, rnn_weights_iter,
...@@ -473,13 +482,14 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop() ...@@ -473,13 +482,14 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop()
num_fused_rnn_layers, num_fused_rnn_layers,
rnn_type); rnn_type);
std::vector<std::shared_ptr<op::Slice>> ht_slice_per_timestep(sequence_len, nullptr); std::vector<std::shared_ptr<ngraph::op::Slice>> ht_slice_per_timestep(sequence_len,
auto rnn_ht_goe = std::make_shared<op::GetOutputElement>(rnn, 0); nullptr);
auto rnn_ht_ct_goe = std::make_shared<op::GetOutputElement>(rnn, 1); auto rnn_ht_goe = std::make_shared<ngraph::op::GetOutputElement>(rnn, 0);
auto rnn_ht_ct_goe = std::make_shared<ngraph::op::GetOutputElement>(rnn, 1);
for (size_t i = 0, start_index = 0; i < sequence_len; i++, start_index += batch_size) for (size_t i = 0, start_index = 0; i < sequence_len; i++, start_index += batch_size)
{ {
ht_slice_per_timestep[i] = (std::make_shared<op::Slice>( ht_slice_per_timestep[i] = (std::make_shared<ngraph::op::Slice>(
rnn_ht_goe, rnn_ht_goe,
Coordinate{start_index, 0}, Coordinate{start_index, 0},
Coordinate{start_index + batch_size, src_iter_feature_size})); Coordinate{start_index + batch_size, src_iter_feature_size}));
...@@ -503,7 +513,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop() ...@@ -503,7 +513,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop()
for (size_t index = 0; index < sequence_len; index++) for (size_t index = 0; index < sequence_len; index++)
{ {
auto goe_nodes = op::get_output_elements(lstm_nodes[index]); auto goe_nodes = ngraph::op::get_output_elements(lstm_nodes[index]);
// if there is no GOE followed by the Lstm, their might be pattern match error // if there is no GOE followed by the Lstm, their might be pattern match error
// we will return safely // we will return safely
...@@ -521,7 +531,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop() ...@@ -521,7 +531,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop()
{ {
if (ngraph::is_used(goe0_user.get())) if (ngraph::is_used(goe0_user.get()))
{ {
if (!std::dynamic_pointer_cast<op::Slice>(goe0_user)) if (!std::dynamic_pointer_cast<ngraph::op::Slice>(goe0_user))
{ {
NGRAPH_DEBUG << "Did not find LSTM slice to replace with RNN slice"; NGRAPH_DEBUG << "Did not find LSTM slice to replace with RNN slice";
return false; return false;
...@@ -536,7 +546,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop() ...@@ -536,7 +546,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop()
} }
} }
auto rnn_ct_goe = op::get_output_elements(lstm_nodes[sequence_len - 1])[1]; auto rnn_ct_goe = ngraph::op::get_output_elements(lstm_nodes[sequence_len - 1])[1];
if (rnn_ct_goe) if (rnn_ct_goe)
{ {
replace_collapse_node_user(rnn_ct_goe, rnn_ht_ct_goe->get_outputs().at(0)); replace_collapse_node_user(rnn_ct_goe, rnn_ht_ct_goe->get_outputs().at(0));
...@@ -566,7 +576,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop() ...@@ -566,7 +576,7 @@ void ngraph::runtime::cpu::pass::RNNFusion::construct_rnn_lstm_fprop()
static std::shared_ptr<Node> stack_rnn_inputs(NodeVector rnn_input_nodes) static std::shared_ptr<Node> stack_rnn_inputs(NodeVector rnn_input_nodes)
{ {
std::reverse(rnn_input_nodes.begin(), rnn_input_nodes.end()); std::reverse(rnn_input_nodes.begin(), rnn_input_nodes.end());
return std::make_shared<op::Concat>(rnn_input_nodes, 0); return std::make_shared<ngraph::op::Concat>(rnn_input_nodes, 0);
} }
void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_fusion_fprop() void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_fusion_fprop()
...@@ -585,7 +595,7 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_ ...@@ -585,7 +595,7 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_
ngraph::runtime::cpu::rnn_utils::rnntype ref_rnn_type = ngraph::runtime::cpu::rnn_utils::rnntype ref_rnn_type =
ngraph::runtime::cpu::rnn_utils::rnntype::vanilla_lstm; ngraph::runtime::cpu::rnn_utils::rnntype::vanilla_lstm;
auto ref_rnn_node = std::make_shared<op::Rnn>(rnn_src_layer, auto ref_rnn_node = std::make_shared<ngraph::op::Rnn>(rnn_src_layer,
rnn_src_iter, rnn_src_iter,
rnn_weights_layer, rnn_weights_layer,
rnn_weights_iter, rnn_weights_iter,
...@@ -598,7 +608,7 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_ ...@@ -598,7 +608,7 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_
ref_num_of_rnn_fused_layer, ref_num_of_rnn_fused_layer,
ref_rnn_type); ref_rnn_type);
auto rnn_goe0 = std::make_shared<op::GetOutputElement>(ref_rnn_node, 0); auto rnn_goe0 = std::make_shared<ngraph::op::GetOutputElement>(ref_rnn_node, 0);
auto rnn_goe0_label = auto rnn_goe0_label =
std::make_shared<pattern::op::Label>(rnn_goe0, nullptr, NodeVector{rnn_goe0}); std::make_shared<pattern::op::Label>(rnn_goe0, nullptr, NodeVector{rnn_goe0});
...@@ -622,10 +632,11 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_ ...@@ -622,10 +632,11 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_
auto rnn_goe0_bounded_nodes = m.get_bound_nodes_for_pattern(rnn_goe0_label); auto rnn_goe0_bounded_nodes = m.get_bound_nodes_for_pattern(rnn_goe0_label);
std::vector<std::shared_ptr<op::Rnn>> rnn_nodes; std::vector<std::shared_ptr<ngraph::op::Rnn>> rnn_nodes;
for (auto rnn_goe : m.get_bound_nodes_for_pattern(rnn_goe0_label)) for (auto rnn_goe : m.get_bound_nodes_for_pattern(rnn_goe0_label))
{ {
if (auto rnn_op = std::dynamic_pointer_cast<op::Rnn>(rnn_goe->get_arguments()[0])) if (auto rnn_op =
std::dynamic_pointer_cast<ngraph::op::Rnn>(rnn_goe->get_arguments()[0]))
{ {
rnn_nodes.push_back(rnn_op); rnn_nodes.push_back(rnn_op);
} }
...@@ -695,7 +706,7 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_ ...@@ -695,7 +706,7 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_
"layer"); "layer");
} }
auto rnn = std::make_shared<op::Rnn>(mrnn_src_layer, auto rnn = std::make_shared<ngraph::op::Rnn>(mrnn_src_layer,
mrnn_src_iter, mrnn_src_iter,
mrnn_weights_layer, mrnn_weights_layer,
mrnn_weights_iter, mrnn_weights_iter,
...@@ -708,8 +719,8 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_ ...@@ -708,8 +719,8 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_
num_fused_rnn_layers, num_fused_rnn_layers,
rnn_type); rnn_type);
auto mrnn_ht = std::make_shared<op::GetOutputElement>(rnn, 0); auto mrnn_ht = std::make_shared<ngraph::op::GetOutputElement>(rnn, 0);
auto mrnn_ht_ct = std::make_shared<op::GetOutputElement>(rnn, 1); auto mrnn_ht_ct = std::make_shared<ngraph::op::GetOutputElement>(rnn, 1);
// Replace all the users of RNN cell state {ct} across different user. // Replace all the users of RNN cell state {ct} across different user.
auto replace_rnn_output_cellstate = [&](std::shared_ptr<Node> rnn_ct_goe1, size_t layer) { auto replace_rnn_output_cellstate = [&](std::shared_ptr<Node> rnn_ct_goe1, size_t layer) {
...@@ -718,7 +729,7 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_ ...@@ -718,7 +729,7 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_
// of all the layers, {{ht_1 | ct_1} || {ht2 |ct2} || ....{htn | ctn}} // of all the layers, {{ht_1 | ct_1} || {ht2 |ct2} || ....{htn | ctn}}
// we will slice the cell state output tensor {ct_*} from the fused RNN kerenel output and feeds // we will slice the cell state output tensor {ct_*} from the fused RNN kerenel output and feeds
// {ct_*} consumer if any // {ct_*} consumer if any
auto ct_slice = std::make_shared<op::Slice>( auto ct_slice = std::make_shared<ngraph::op::Slice>(
mrnn_ht_ct, mrnn_ht_ct,
Coordinate{((layer - 1) * batch_size * num_rnn_cell_states) + batch_size, 0}, Coordinate{((layer - 1) * batch_size * num_rnn_cell_states) + batch_size, 0},
Coordinate{layer * batch_size * num_rnn_cell_states, src_iter_feature_size}); Coordinate{layer * batch_size * num_rnn_cell_states, src_iter_feature_size});
...@@ -732,7 +743,7 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_ ...@@ -732,7 +743,7 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_
// i.e {RNN7, RNN6, RNN5.... RNN0} // i.e {RNN7, RNN6, RNN5.... RNN0}
for (size_t index = 0; index < rnn_nodes.size(); index++) for (size_t index = 0; index < rnn_nodes.size(); index++)
{ {
auto goe_nodes = op::get_output_elements(rnn_nodes[index]); auto goe_nodes = ngraph::op::get_output_elements(rnn_nodes[index]);
// if there is no GOE followed by the Lstm, their might be pattern match error // if there is no GOE followed by the Lstm, their might be pattern match error
// we will return safely // we will return safely
if (goe_nodes.size() != 2) if (goe_nodes.size() != 2)
...@@ -771,15 +782,17 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_ ...@@ -771,15 +782,17 @@ void ngraph::runtime::cpu::pass::MultiLayerRNNFusion::construct_multi_layer_rnn_
void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn() void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn()
{ {
auto rnn_left_to_right = std::make_shared<pattern::op::Label>( auto rnn_left_to_right = std::make_shared<pattern::op::Label>(
element::f32, Shape{1, 256}, pattern::has_class<op::Rnn>()); element::f32, Shape{1, 256}, pattern::has_class<ngraph::op::Rnn>());
auto rnn_right_to_left = std::make_shared<pattern::op::Label>( auto rnn_right_to_left = std::make_shared<pattern::op::Label>(
element::f32, Shape{1, 256}, pattern::has_class<op::Rnn>()); element::f32, Shape{1, 256}, pattern::has_class<ngraph::op::Rnn>());
auto reshape_pred = [](std::shared_ptr<Node> n) { auto reshape_pred = [](std::shared_ptr<Node> n) {
return (std::dynamic_pointer_cast<op::Reshape>(n) != nullptr); return (std::dynamic_pointer_cast<ngraph::op::Reshape>(n) != nullptr);
}; };
auto rnn_left_to_right_goe0 = std::make_shared<op::GetOutputElement>(rnn_left_to_right, 0); auto rnn_left_to_right_goe0 =
auto rnn_right_to_left_goe0 = std::make_shared<op::GetOutputElement>(rnn_right_to_left, 0); std::make_shared<ngraph::op::GetOutputElement>(rnn_left_to_right, 0);
auto rnn_right_to_left_goe0 =
std::make_shared<ngraph::op::GetOutputElement>(rnn_right_to_left, 0);
auto rnn_rtol_goe0_reshape_ntc = auto rnn_rtol_goe0_reshape_ntc =
std::make_shared<pattern::op::Skip>(rnn_right_to_left_goe0, reshape_pred); std::make_shared<pattern::op::Skip>(rnn_right_to_left_goe0, reshape_pred);
...@@ -791,21 +804,23 @@ void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn() ...@@ -791,21 +804,23 @@ void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn()
std::make_shared<pattern::op::Skip>(rnn_ltor_goe0_reshape_ntc, reshape_pred); std::make_shared<pattern::op::Skip>(rnn_ltor_goe0_reshape_ntc, reshape_pred);
auto reverse_seq_predicate = [](std::shared_ptr<Node> node) { auto reverse_seq_predicate = [](std::shared_ptr<Node> node) {
return pattern::has_class<op::ReverseSequence>()(node) || return pattern::has_class<ngraph::op::ReverseSequence>()(node) ||
pattern::has_class<op::Reverse>()(node); pattern::has_class<ngraph::op::Reverse>()(node);
}; };
auto skip_reverse_seq = auto skip_reverse_seq =
std::make_shared<pattern::op::Skip>(rnn_rtol_goe0_reshape_tnc, reverse_seq_predicate); std::make_shared<pattern::op::Skip>(rnn_rtol_goe0_reshape_tnc, reverse_seq_predicate);
auto concat = auto concat = std::make_shared<ngraph::op::Concat>(
std::make_shared<op::Concat>(NodeVector{rnn_ltor_goe0_reshape_tnc, skip_reverse_seq}, 0); NodeVector{rnn_ltor_goe0_reshape_tnc, skip_reverse_seq}, 0);
// Define a call back that needs to called once the DFG matches the pattern // Define a call back that needs to called once the DFG matches the pattern
ngraph::pattern::graph_rewrite_callback callback = [rnn_left_to_right, ngraph::pattern::graph_rewrite_callback callback = [rnn_left_to_right,
rnn_right_to_left](pattern::Matcher& m) { rnn_right_to_left](pattern::Matcher& m) {
auto pattern_map = m.get_pattern_map(); auto pattern_map = m.get_pattern_map();
auto rnn_ltor_node = std::static_pointer_cast<op::Rnn>(pattern_map[rnn_left_to_right]); auto rnn_ltor_node =
auto rnn_rtol_node = std::static_pointer_cast<op::Rnn>(pattern_map[rnn_right_to_left]); std::static_pointer_cast<ngraph::op::Rnn>(pattern_map[rnn_left_to_right]);
auto rnn_rtol_node =
std::static_pointer_cast<ngraph::op::Rnn>(pattern_map[rnn_right_to_left]);
if (rnn_ltor_node->get_src_sequence_length() != rnn_rtol_node->get_src_sequence_length()) if (rnn_ltor_node->get_src_sequence_length() != rnn_rtol_node->get_src_sequence_length())
{ {
...@@ -852,7 +867,7 @@ void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn() ...@@ -852,7 +867,7 @@ void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn()
auto nodes = auto nodes =
NodeVector{rnn_ltor_node->get_argument(index), rnn_rtol_node->get_argument(index)}; NodeVector{rnn_ltor_node->get_argument(index), rnn_rtol_node->get_argument(index)};
return std::make_shared<op::Concat>(nodes, 0); return std::make_shared<ngraph::op::Concat>(nodes, 0);
}; };
auto src_layer = rnn_ltor_node->get_arguments()[0]; auto src_layer = rnn_ltor_node->get_arguments()[0];
...@@ -861,7 +876,7 @@ void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn() ...@@ -861,7 +876,7 @@ void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn()
auto weights_iter = construct_birnn_inputs(3); auto weights_iter = construct_birnn_inputs(3);
auto bias = construct_birnn_inputs(4); auto bias = construct_birnn_inputs(4);
auto rnn = std::make_shared<op::Rnn>(src_layer, auto rnn = std::make_shared<ngraph::op::Rnn>(src_layer,
src_iter, src_iter,
weights_layer, weights_layer,
weights_iter, weights_iter,
...@@ -874,7 +889,7 @@ void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn() ...@@ -874,7 +889,7 @@ void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn()
num_fused_rnn_layers, num_fused_rnn_layers,
rnn_type); rnn_type);
auto layer_rnn_ht = std::make_shared<op::GetOutputElement>(rnn, 0); auto layer_rnn_ht = std::make_shared<ngraph::op::GetOutputElement>(rnn, 0);
size_t batch_size = layer_rnn_ht->get_shape()[0] / num_time_steps; size_t batch_size = layer_rnn_ht->get_shape()[0] / num_time_steps;
size_t feature_size = layer_rnn_ht->get_shape()[1]; size_t feature_size = layer_rnn_ht->get_shape()[1];
...@@ -882,15 +897,15 @@ void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn() ...@@ -882,15 +897,15 @@ void ngraph::runtime::cpu::pass::BiDirectionalRnn::construct_bidirectional_rnn()
std::shared_ptr<Node> layer_rnn_ht_reshape = layer_rnn_ht; std::shared_ptr<Node> layer_rnn_ht_reshape = layer_rnn_ht;
if (m.get_match_root()->get_shape() != layer_rnn_ht->get_shape()) if (m.get_match_root()->get_shape() != layer_rnn_ht->get_shape())
{ {
layer_rnn_ht_reshape = std::make_shared<op::Reshape>( layer_rnn_ht_reshape = std::make_shared<ngraph::op::Reshape>(
layer_rnn_ht, AxisVector{0, 1}, Shape{num_time_steps, batch_size, feature_size}); layer_rnn_ht, AxisVector{0, 1}, Shape{num_time_steps, batch_size, feature_size});
} }
// we will check if the node being replaced is in Shape{n, t, c}, if so we will transpose // we will check if the node being replaced is in Shape{n, t, c}, if so we will transpose
if (m.get_match_root()->get_shape() == Shape{batch_size, num_time_steps, feature_size}) if (m.get_match_root()->get_shape() == Shape{batch_size, num_time_steps, feature_size})
{ {
layer_rnn_ht_reshape = layer_rnn_ht_reshape = std::make_shared<ngraph::op::Reshape>(
std::make_shared<op::Reshape>(layer_rnn_ht_reshape, layer_rnn_ht_reshape,
AxisVector{1, 0, 2}, AxisVector{1, 0, 2},
Shape{batch_size, num_time_steps, feature_size}); Shape{batch_size, num_time_steps, feature_size});
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment