Skip to content

N
ngraph
Commit 9335e41c authored by Amy Zhuang's avatar Amy Zhuang Committed by Scott Cyphers
Browse files
Options

Create mkldnn primitives at first iteration for codegen - part2 (#2859) 

* Create mkldnn primitives at first iteration for CODEGEN.

 OPs: add, lstm, and rnn.

*  OPs: batchnorm.

*  OPs: concat and lrn.

Remove dead code.

* Skip in place concat, relu, reshape, and slice when building node_primitive_string_deps_index map.

* Change NGRAPH_ASSERT to NGRAPH_CHECK.

* Address PR Feedback.

* Create mkldnn primitives at first iteration for CODEGEN.
 OPs: convertlayout, relu, leakyrelu, boundedrelu, sigmoid, softmax, slice.

* Fix bugs.

*  OPs: quantizedconcat.

Check if there are descriptors before emitting code to read desc_file.

*  OPs: convolution backward.

Use macro to write mkldnn memory dims to generated file.

*  OPs: MaxPoolWithIndices and MaxPoolWithIndicesBackprop.

Add unit tests for MaxPoolWithIndices, MaxPoolWithIndicesBackprop, and MaxPoolBackprop.

* Fix style error.

*  OPs: AvgPoolBackprop and MaxPoolBackprop.

Add unit test for AvgPoolBackprop.

*  OPs: DeconvolutionBias.

*  OPs: Quantize and Dequantize.

*  OPs: QuantizedDot and QuantizedDotBias.

* Use reference kernel for QuantizedConvolution for CODEGEN when mkldnn does not support the parameter types.
Get scales for quantization ops in cpu_emitter.

* Fix Windows build error: add CPU_BACKEND_API.

* Use template for quantization ops.

*  OPs: QuantizedMatmul.

Emit referece kernel for QuantizedDot in CODEGEN.

* Remove QuantizedDot from get_scale_index.

* Address PR feedback.
parent 30f3634e
Hide whitespace changes
Inline Side-by-side
Showing with 868 additions and 1195 deletions
src/ngraph/runtime/cpu/builder/convolution.cpp
View file @ 9335e41c
...@@ -402,8 +402,8 @@ namespace ngraph ...@@ -402,8 +402,8 @@ namespace ngraph
ngraph::op::ConvolutionBackpropData>(node); ngraph::op::ConvolutionBackpropData>(node);
auto fwd_desc = mkldnn_emitter->get_convolution_forward_desc_for_backward_op< auto fwd_desc = mkldnn_emitter->get_convolution_forward_desc_for_backward_op<
ngraph::op::ConvolutionBackpropData>(node); ngraph::op::ConvolutionBackpropData>(node);
// ConvolutionBackpropData needs 4 primitives: weights, delta, result, // ConvolutionBackpropData needs 4 primitives: weights, diff_dst, diff_src,
// and convolution_backward. // and convolution_backward_data.
auto conv_index = mkldnn_emitter->reserve_primitive_space(4); auto conv_index = mkldnn_emitter->reserve_primitive_space(4);
auto& deps = mkldnn_emitter->get_primitive_deps(conv_index); auto& deps = mkldnn_emitter->get_primitive_deps(conv_index);
...@@ -502,7 +502,7 @@ namespace ngraph ...@@ -502,7 +502,7 @@ namespace ngraph
ngraph::op::ConvolutionBackpropFilters>(node); ngraph::op::ConvolutionBackpropFilters>(node);
auto fwd_desc = mkldnn_emitter->get_convolution_forward_desc_for_backward_op< auto fwd_desc = mkldnn_emitter->get_convolution_forward_desc_for_backward_op<
ngraph::op::ConvolutionBackpropFilters>(node); ngraph::op::ConvolutionBackpropFilters>(node);
// ConvolutionBackpropFilter needs 4 primitives: input, delta, weights_delta, // ConvolutionBackpropFilter needs 4 primitives: src, diff_dst, diff_weights,
// and convolution_backward_weights. // and convolution_backward_weights.
auto conv_index = mkldnn_emitter->reserve_primitive_space(4); auto conv_index = mkldnn_emitter->reserve_primitive_space(4);
auto& deps = mkldnn_emitter->get_primitive_deps(conv_index); auto& deps = mkldnn_emitter->get_primitive_deps(conv_index);
...@@ -598,8 +598,8 @@ namespace ngraph ...@@ -598,8 +598,8 @@ namespace ngraph
ngraph::op::ConvolutionBiasBackpropFiltersBias>(node); ngraph::op::ConvolutionBiasBackpropFiltersBias>(node);
auto fwd_desc = mkldnn_emitter->get_convolution_forward_desc_for_backward_op< auto fwd_desc = mkldnn_emitter->get_convolution_forward_desc_for_backward_op<
ngraph::op::ConvolutionBiasBackpropFiltersBias>(node); ngraph::op::ConvolutionBiasBackpropFiltersBias>(node);
// ConvolutionBiasBackpropFilter needs 5 primitives: input, delta, weights_delta, // ConvolutionBackpropFiltersBias needs 5 primitives: src, diff_dst, diff_weights,
// bias_delta, and convolution_backward_weights. // diff_bias, and convolution_backward_weights.
auto conv_index = mkldnn_emitter->reserve_primitive_space(5); auto conv_index = mkldnn_emitter->reserve_primitive_space(5);
auto& deps = mkldnn_emitter->get_primitive_deps(conv_index); auto& deps = mkldnn_emitter->get_primitive_deps(conv_index);
......
src/ngraph/runtime/cpu/builder/max_pool.cpp
View file @ 9335e41c
...@@ -301,7 +301,7 @@ namespace ngraph ...@@ -301,7 +301,7 @@ namespace ngraph
->get_max_pooling_backward_desc<ngraph::op::MaxPoolWithIndicesBackprop>( ->get_max_pooling_backward_desc<ngraph::op::MaxPoolWithIndicesBackprop>(
node); node);
// MaxPoolWithIndicesBackprop needs 4 primitives: diff_dst, fprop_workspace, // MaxPoolWithIndicesBackprop needs 4 primitives: diff_dst, fprop_workspace,
// diff_dst, and pooling_backward. // diff_src, and pooling_backward.
size_t max_pool_index = mkldnn_emitter->reserve_primitive_space(4); size_t max_pool_index = mkldnn_emitter->reserve_primitive_space(4);
auto& deps = mkldnn_emitter->get_primitive_deps(max_pool_index); auto& deps = mkldnn_emitter->get_primitive_deps(max_pool_index);
......
src/ngraph/runtime/cpu/cpu_emitter.cpp
View file @ 9335e41c
...@@ -174,6 +174,59 @@ namespace ngraph ...@@ -174,6 +174,59 @@ namespace ngraph
index = get<2>(external_function->get_primitive_build_tuple(node)); index = get<2>(external_function->get_primitive_build_tuple(node));
} }
template <typename OP>
static void emit_build_primitives(CPU_ExternalFunction* external_function,
const ngraph::Node* node,
CodeWriter& writer,
size_t& index,
std::vector<std::size_t>& deps,
const std::vector<TensorViewWrapper>& args)
{
writer << "if (ctx->first_iteration)\n";
writer.block_begin();
auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
auto scale_index = mkldnn_emitter->get_scale_index<OP>();
auto scales_size = shape_size(node->get_input_shape(scale_index));
writer << "std::vector<float> dyn_scales;\n";
writer << "dyn_scales.assign(" << args[scale_index].get_name() << ", "
<< args[scale_index].get_name() << " + " << std::to_string(scales_size)
<< ");\n";
// for Quantize
if (is_same<OP, ngraph::op::Quantize>())
{
writer << "for (size_t i = 0; i < " << std::to_string(scales_size)
<< "; i++)\n";
writer.block_begin();
writer << "dyn_scales[i] = 1.0 / dyn_scales[i];\n";
writer.block_end();
}
// QuantizedConvolutionBiasAdd and QuantizedConvolutionBiasSignedAdd
if (is_same<OP, ngraph::op::QuantizedConvolutionBiasAdd>() ||
is_same<OP, ngraph::op::QuantizedConvolutionBiasSignedAdd>())
{
auto sum_scale_index = 5;
auto sum_scales_size = shape_size(node->get_input_shape(sum_scale_index));
writer << "std::vector<float> dyn_post_op_scales;\n";
writer << "dyn_post_op_scales.assign(" << args[sum_scale_index].get_name()
<< ", " << args[sum_scale_index].get_name() << " + "
<< std::to_string(sum_scales_size) << ");\n";
}
writer << "// quantize across first dim (mask=2^0) if dyn_scales is a "
"vector \n";
writer << "const int mask = " << std::to_string(scales_size) << " == 1 ? 0 : 1;\n";
// get the string, deps, and index from the map
writer << get<0>(external_function->get_primitive_build_tuple(node));
writer.block_end();
deps = get<1>(external_function->get_primitive_build_tuple(node));
index = get<2>(external_function->get_primitive_build_tuple(node));
}
template <> template <>
void CPU_Emitter::EMITTER_DECL(ngraph::op::Add) void CPU_Emitter::EMITTER_DECL(ngraph::op::Add)
{ {
...@@ -1434,22 +1487,18 @@ namespace ngraph ...@@ -1434,22 +1487,18 @@ namespace ngraph
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto out_shape = out[0].get_shape(); size_t slice_index;
std::vector<std::size_t> deps;
auto lower_bounds = slice->get_lower_bounds(); emit_build_primitives(external_function, node, writer, slice_index, deps);
auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
auto slice_index = external_function->get_primitive_index(node);
auto& deps = mkldnn_emitter->get_primitive_deps(slice_index);
writer.block_begin(); writer.block_begin();
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(slice_index)
<< to_string(slice_index) << ");\n"; << ");\n";
writer.block_end(); writer.block_end();
return; return;
} }
...@@ -1553,12 +1602,24 @@ namespace ngraph ...@@ -1553,12 +1602,24 @@ namespace ngraph
auto index_type_name = embed->get_argument(0)->get_element_type().c_type_string(); auto index_type_name = embed->get_argument(0)->get_element_type().c_type_string();
auto type_name = embed->get_element_type().c_type_string(); auto type_name = embed->get_element_type().c_type_string();
auto element_count = shape_size(embed->get_argument(0)->get_shape()); auto element_count = shape_size(embed->get_argument(0)->get_shape());
// FIXME
// clang generates 16 bytes aligned store with unaligned address,
// which results in segmentation fault.
// Workaround for now: Use push_back to avoid generating such store.
auto arg1_shape = args[1].get_shape();
writer << "ngraph::Shape shape;\n";
for (auto i = 0; i < arg1_shape.size(); i++)
{
writer << "shape.push_back(" << std::to_string(arg1_shape[i]) << ");\n";
}
writer << "reference::embedding<" << type_name << "," << index_type_name << ">("; writer << "reference::embedding<" << type_name << "," << index_type_name << ">(";
writer << " " << args[0].get_name() << ",\n"; writer << " " << args[0].get_name() << ",\n";
writer << " " << args[1].get_name() << ",\n"; writer << " " << args[1].get_name() << ",\n";
writer << " " << out[0].get_name() << ",\n"; writer << " " << out[0].get_name() << ",\n";
writer << " " << element_count << ",\n"; writer << " " << element_count << ",\n";
writer << " {" << join(args[1].get_shape()) << "});\n"; writer << " shape);\n";
writer.block_end(); writer.block_end();
} }
...@@ -2056,7 +2117,8 @@ namespace ngraph ...@@ -2056,7 +2117,8 @@ namespace ngraph
{ {
size_t conv_index; size_t conv_index;
std::vector<std::size_t> deps; std::vector<std::size_t> deps;
emit_build_primitives(external_function, node, writer, conv_index, deps); emit_build_primitives<ngraph::op::QuantizedConvolutionRelu>(
external_function, node, writer, conv_index, deps, args);
writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", " writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
...@@ -2079,7 +2141,8 @@ namespace ngraph ...@@ -2079,7 +2141,8 @@ namespace ngraph
{ {
size_t conv_index; size_t conv_index;
std::vector<std::size_t> deps; std::vector<std::size_t> deps;
emit_build_primitives(external_function, node, writer, conv_index, deps); emit_build_primitives<ngraph::op::QuantizedConvolution>(
external_function, node, writer, conv_index, deps, args);
writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", " writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
...@@ -2091,7 +2154,35 @@ namespace ngraph ...@@ -2091,7 +2154,35 @@ namespace ngraph
} }
else else
{ {
throw ngraph_error("unsupported parameters for QuantizedConvolution"); auto convolution = static_cast<const ngraph::op::QuantizedConvolution*>(node);
auto arg0_shape = args[0].get_shape();
auto arg1_shape = args[1].get_shape();
auto result_shape = out[0].get_shape();
auto scales_size = shape_size(node->get_input_shape(2));
writer << "std::vector<float> dyn_scales;\n";
writer << "dyn_scales.assign(" << args[2].get_name() << ", "
<< args[2].get_name() << " + " << std::to_string(scales_size) << ");\n";
writer << "reference::convolution<" << out[0].get_type() << ">("
<< args[0].get_name() << ",\n";
writer << " " << args[1].get_name() << ",\n";
writer << " " << out[0].get_name() << ",\n";
writer << " {" << join(arg0_shape) << "},\n";
writer << " {" << join(arg1_shape) << "},\n";
writer << " {" << join(result_shape) << "},\n";
writer << " {"
<< join(convolution->get_window_movement_strides()) << "},\n";
writer << " {"
<< join(convolution->get_window_dilation_strides()) << "},\n";
writer << " {" << join(convolution->get_padding_below())
<< "},\n";
writer << " {" << join(convolution->get_padding_above())
<< "},\n";
writer << " {"
<< join(convolution->get_data_dilation_strides()) << "}, \n";
writer << " dyn_scales[0]);\n";
} }
} }
...@@ -2201,19 +2292,18 @@ namespace ngraph ...@@ -2201,19 +2292,18 @@ namespace ngraph
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t conv_index;
auto conv_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& deps = mkldnn_emitter->get_primitive_deps(conv_index); emit_build_primitives(external_function, node, writer, conv_index, deps);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << args[1].get_name() << ");\n"; << args[1].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[2]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(conv_index) << ");\n";
<< to_string(conv_index) << ");\n";
} }
else else
{ {
...@@ -2247,23 +2337,20 @@ namespace ngraph ...@@ -2247,23 +2337,20 @@ namespace ngraph
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t conv_index;
auto conv_index = std::vector<std::size_t> deps;
mkldnn_emitter->build_deconvolution<ngraph::op::DeconvolutionBias>( emit_build_primitives(external_function, node, writer, conv_index, deps);
node, args, out);
auto& deps = mkldnn_emitter->get_primitive_deps(conv_index);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << args[1].get_name() << ");\n"; << args[1].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[2]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
<< ", " << args[2].get_name() << ");\n"; << args[2].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[3]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[3]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(conv_index) << ");\n";
<< to_string(conv_index) << ");\n";
} }
else else
{ {
...@@ -2282,19 +2369,18 @@ namespace ngraph ...@@ -2282,19 +2369,18 @@ namespace ngraph
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t conv_index;
auto conv_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& deps = mkldnn_emitter->get_primitive_deps(conv_index); emit_build_primitives(external_function, node, writer, conv_index, deps);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << args[1].get_name() << ");\n"; << args[1].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[2]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(conv_index) << ");\n";
<< to_string(conv_index) << ");\n";
} }
else else
{ {
...@@ -2326,7 +2412,8 @@ namespace ngraph ...@@ -2326,7 +2412,8 @@ namespace ngraph
{ {
size_t conv_index; size_t conv_index;
std::vector<std::size_t> deps; std::vector<std::size_t> deps;
emit_build_primitives(external_function, node, writer, conv_index, deps); emit_build_primitives<ngraph::op::QuantizedConvolutionBias>(
external_function, node, writer, conv_index, deps, args);
writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", " writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
...@@ -2352,7 +2439,8 @@ namespace ngraph ...@@ -2352,7 +2439,8 @@ namespace ngraph
{ {
size_t conv_index; size_t conv_index;
std::vector<std::size_t> deps; std::vector<std::size_t> deps;
emit_build_primitives(external_function, node, writer, conv_index, deps); emit_build_primitives<ngraph::op::QuantizedConvolutionBiasAdd>(
external_function, node, writer, conv_index, deps, args);
writer << "if (" << out[0].get_name() << " != " << args[3].get_name() << ")\n"; writer << "if (" << out[0].get_name() << " != " << args[3].get_name() << ")\n";
writer.block_begin(); writer.block_begin();
...@@ -2383,7 +2471,8 @@ namespace ngraph ...@@ -2383,7 +2471,8 @@ namespace ngraph
{ {
size_t conv_index; size_t conv_index;
std::vector<std::size_t> deps; std::vector<std::size_t> deps;
emit_build_primitives(external_function, node, writer, conv_index, deps); emit_build_primitives<ngraph::op::QuantizedConvolutionBiasSignedAdd>(
external_function, node, writer, conv_index, deps, args);
writer << "if (" << out[0].get_name() << " != " << args[3].get_name() << ")\n"; writer << "if (" << out[0].get_name() << " != " << args[3].get_name() << ")\n";
writer.block_begin(); writer.block_begin();
...@@ -2412,23 +2501,21 @@ namespace ngraph ...@@ -2412,23 +2501,21 @@ namespace ngraph
{ {
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t qip_index;
auto qip_index = std::vector<std::size_t> deps;
mkldnn_emitter->build_inner_product<ngraph::op::QuantizedDotBias>( emit_build_primitives<ngraph::op::QuantizedDotBias>(
node, args, out); external_function, node, writer, qip_index, deps, args);
auto& deps = mkldnn_emitter->get_primitive_deps(qip_index);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << args[1].get_name() << ");\n"; << args[1].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[2]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[3]) << ", "
<< ", " << args[2].get_name() << ");\n"; << args[2].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[3]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(qip_index) << ");\n";
<< to_string(qip_index) << ");\n";
} }
else else
{ {
...@@ -2438,33 +2525,46 @@ namespace ngraph ...@@ -2438,33 +2525,46 @@ namespace ngraph
template <> template <>
void CPU_Emitter::EMITTER_DECL(ngraph::op::QuantizedDot) void CPU_Emitter::EMITTER_DECL(ngraph::op::QuantizedDot)
{
if (shape_size(args[2].get_shape()) != 1)
{
throw ngraph_error("Scale size should be 1 for QuantizedDot");
}
writer << "float dyn_scale = *(static_cast<float*>(" << args[2].get_name()
<< "));\n";
writer << "reference::dot(" << args[0].get_name() << ",\n";
writer << " " << args[1].get_name() << ",\n";
writer << " " << out[0].get_name() << ",\n";
writer << " {" << join(args[0].get_shape()) << "},\n";
writer << " {" << join(args[1].get_shape()) << "},\n";
writer << " {" << join(out[0].get_shape()) << "},\n";
writer << " 1,\n";
writer << " dyn_scale);\n";
}
template <>
void CPU_Emitter::EMITTER_DECL(ngraph::op::QuantizedMatmul)
{ {
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
if (node->get_input_element_type(0) == element::u8 && size_t qip_index;
node->get_input_element_type(1) == element::u8) std::vector<std::size_t> deps;
{ emit_build_primitives<ngraph::op::QuantizedMatmul>(
throw ngraph_error( external_function, node, writer, qip_index, deps, args);
"Unsupported data types for QuantizedDot MKLDNN kernel.");
}
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
auto qip_index = mkldnn_emitter->build_inner_product<ngraph::op::QuantizedDot>( << args[0].get_name() << ");\n";
node, args, out); writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
auto& deps = mkldnn_emitter->get_primitive_deps(qip_index); << args[1].get_name() << ");\n";
writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
<< out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(qip_index) << ");\n";
<< ", " << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1])
<< ", " << args[1].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[2])
<< ", " << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, "
<< to_string(qip_index) << ");\n";
} }
else else
{ {
throw ngraph_error("unsupported parameters for QuantizedDot"); throw ngraph_error("QuantizedMatmul is only supported with MKLDNN kernel.");
} }
} }
...@@ -2556,21 +2656,20 @@ namespace ngraph ...@@ -2556,21 +2656,20 @@ namespace ngraph
{ {
if (mkldnn_utils::use_mkldnn_kernel(node)) if (mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t conv_index;
auto conv_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& deps = mkldnn_emitter->get_primitive_deps(conv_index); emit_build_primitives(external_function, node, writer, conv_index, deps);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << args[1].get_name() << ");\n"; << args[1].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[2]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[3]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[3]) << ", "
<< ", " << out[1].get_name() << ");\n"; << out[1].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(conv_index) << ");\n";
<< to_string(conv_index) << ");\n";
} }
else else
{ {
...@@ -2672,19 +2771,19 @@ namespace ngraph ...@@ -2672,19 +2771,19 @@ namespace ngraph
{ {
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t max_pool_index;
size_t max_pool_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& deps = mkldnn_emitter->get_primitive_deps(max_pool_index); emit_build_primitives(external_function, node, writer, max_pool_index, deps);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[2]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
<< ", " << out[1].get_name() << ");\n"; << out[1].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(max_pool_index)
<< to_string(max_pool_index) << ");\n"; << ");\n";
} }
else else
{ {
...@@ -2883,17 +2982,17 @@ namespace ngraph ...@@ -2883,17 +2982,17 @@ namespace ngraph
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t avg_pool_index;
size_t avg_pool_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& deps = mkldnn_emitter->get_primitive_deps(avg_pool_index); emit_build_primitives(external_function, node, writer, avg_pool_index, deps);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(avg_pool_index)
<< to_string(avg_pool_index) << ");\n"; << ");\n";
} }
else else
{ {
...@@ -2925,29 +3024,27 @@ namespace ngraph ...@@ -2925,29 +3024,27 @@ namespace ngraph
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t max_pool_index;
size_t max_pool_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& fdeps = mkldnn_emitter->get_primitive_deps(max_pool_index - 1); emit_build_primitives(external_function, node, writer, max_pool_index, deps);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(fdeps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(fdeps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(fdeps[2]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[3])
<< ", ctx->mkldnn_workspaces[" << fdeps[3] << "]);\n"; << ", cg_ctx->mkldnn_workspaces[" << deps[5] << "]);\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(deps[4]) << ");\n";
<< to_string(max_pool_index - 1) << ");\n";
writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
auto& bdeps = mkldnn_emitter->get_primitive_deps(max_pool_index); << args[1].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(bdeps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[3])
<< ", " << args[1].get_name() << ");\n"; << ", cg_ctx->mkldnn_workspaces[" << deps[5] << "]);\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(bdeps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
<< ", ctx->mkldnn_workspaces[" << bdeps[3] << "]);\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(bdeps[2])
<< ", " << out[0].get_name() << ");\n"; writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(max_pool_index)
<< ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, "
<< to_string(max_pool_index) << ");\n";
} }
else else
{ {
...@@ -2971,19 +3068,19 @@ namespace ngraph ...@@ -2971,19 +3068,19 @@ namespace ngraph
{ {
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t max_pool_index;
size_t max_pool_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& bdeps = mkldnn_emitter->get_primitive_deps(max_pool_index); emit_build_primitives(external_function, node, writer, max_pool_index, deps);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(bdeps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[1].get_name() << ");\n"; << args[1].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(bdeps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
<< ", " << args[2].get_name() << ");\n"; << args[2].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(bdeps[2]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(max_pool_index)
<< to_string(max_pool_index) << ");\n"; << ");\n";
} }
else else
{ {
...@@ -3076,55 +3173,24 @@ namespace ngraph ...@@ -3076,55 +3173,24 @@ namespace ngraph
template <> template <>
void CPU_Emitter::EMITTER_DECL(ngraph::runtime::cpu::op::ConvertLayout) void CPU_Emitter::EMITTER_DECL(ngraph::runtime::cpu::op::ConvertLayout)
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{
size_t reorder_index;
std::vector<std::size_t> deps;
emit_build_primitives(external_function, node, writer, reorder_index, deps);
auto input_desc = mkldnn_utils::get_input_mkldnn_md(node, 0); writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
auto result_desc = mkldnn_utils::get_output_mkldnn_md(node, 0); << args[0].get_name() << ");\n";
writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< out[0].get_name() << ");\n";
//this is a special case to handle nchw(oihw) to goihw/Goihw16g/Goihw8g for GroupConvolution's weights writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(reorder_index)
if (input_desc.data.format == mkldnn_nchw && << ");\n";
result_desc.data.format == mkldnn_goihw)
{
input_desc = result_desc;
} }
else if (input_desc.data.format == mkldnn_nchw && else
input_desc.data.ndims == 4 /*nchw*/ &&
result_desc.data.ndims == 5 /*Goihw16g/Goihw8g/etc*/ &&
node->get_users().size() == 1)
{ {
Shape weights_shape_groups; throw ngraph_error("ConvertLayout is only supported with MKLDNN kernel.");
if (auto gconv = std::dynamic_pointer_cast<ngraph::op::GroupConvolution>(
node->get_users()[0]))
{
weights_shape_groups = gconv->get_weights_dimensions();
}
else if (auto gconvb =
std::dynamic_pointer_cast<ngraph::op::GroupConvolutionBias>(
node->get_users()[0]))
{
weights_shape_groups = gconvb->get_weights_dimensions();
}
else
{
throw ngraph_error("Incompatible input/output shape in ConvertLayout op");
}
input_desc = mkldnn::memory::desc(
mkldnn::memory::dims(weights_shape_groups.begin(),
weights_shape_groups.end()),
mkldnn_utils::get_mkldnn_data_type(args[0].get_element_type()),
mkldnn::memory::format::goihw);
} }
size_t reorder_index = external_function->get_primitive_index(node);
auto& deps = mkldnn_emitter->get_primitive_deps(reorder_index);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) << ", "
<< args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) << ", "
<< out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, "
<< to_string(reorder_index) << ");\n";
} }
template <> template <>
...@@ -3132,19 +3198,18 @@ namespace ngraph ...@@ -3132,19 +3198,18 @@ namespace ngraph
{ {
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t relu_index;
size_t relu_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
emit_build_primitives(external_function, node, writer, relu_index, deps);
auto& deps = mkldnn_emitter->get_primitive_deps(relu_index); writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) << args[0].get_name() << ");\n";
<< ", " << args[0].get_name() << ");\n"; writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) << args[1].get_name() << ");\n";
<< ", " << args[1].get_name() << ");\n"; writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[2]) << out[0].get_name() << ");\n";
<< ", " << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(relu_index) << ");\n";
<< to_string(relu_index) << ");\n";
} }
else else
{ {
...@@ -3162,17 +3227,16 @@ namespace ngraph ...@@ -3162,17 +3227,16 @@ namespace ngraph
{ {
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t relu_index;
size_t relu_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& deps = mkldnn_emitter->get_primitive_deps(relu_index); emit_build_primitives(external_function, node, writer, relu_index, deps);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(relu_index) << ");\n";
<< to_string(relu_index) << ");\n";
} }
else else
{ {
...@@ -3188,23 +3252,24 @@ namespace ngraph ...@@ -3188,23 +3252,24 @@ namespace ngraph
template <> template <>
void CPU_Emitter::EMITTER_DECL(ngraph::op::LeakyRelu) void CPU_Emitter::EMITTER_DECL(ngraph::op::LeakyRelu)
{ {
auto leaky_relu_node = static_cast<const ngraph::op::LeakyRelu*>(node);
float alpha = leaky_relu_node->get_alpha();
auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto leaky_relu_index = external_function->get_primitive_index(node); size_t leaky_relu_index;
auto& deps = mkldnn_emitter->get_primitive_deps(leaky_relu_index); std::vector<std::size_t> deps;
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) emit_build_primitives(external_function, node, writer, leaky_relu_index, deps);
<< ", " << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << out[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(leaky_relu_index)
<< to_string(leaky_relu_index) << ");\n"; << ");\n";
} }
else else
{ {
auto leaky_relu_node = static_cast<const ngraph::op::LeakyRelu*>(node);
float alpha = leaky_relu_node->get_alpha();
writer << "#pragma omp parallel for\n"; writer << "#pragma omp parallel for\n";
writer << "for (size_t i = 0; i < " << out[0].get_size() << "; i++)\n"; writer << "for (size_t i = 0; i < " << out[0].get_size() << "; i++)\n";
writer.block_begin(); writer.block_begin();
...@@ -3218,23 +3283,25 @@ namespace ngraph ...@@ -3218,23 +3283,25 @@ namespace ngraph
template <> template <>
void CPU_Emitter::EMITTER_DECL(ngraph::op::BoundedRelu) void CPU_Emitter::EMITTER_DECL(ngraph::op::BoundedRelu)
{ {
auto bounded_relu_node = static_cast<const ngraph::op::BoundedRelu*>(node);
float alpha = bounded_relu_node->get_alpha();
auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto bounded_relu_index = external_function->get_primitive_index(node); size_t bounded_relu_index;
auto& deps = mkldnn_emitter->get_primitive_deps(bounded_relu_index); std::vector<std::size_t> deps;
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) emit_build_primitives(
<< ", " << args[0].get_name() << ");\n"; external_function, node, writer, bounded_relu_index, deps);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1])
<< ", " << out[0].get_name() << ");\n"; writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< args[0].get_name() << ");\n";
writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(bounded_relu_index)
<< to_string(bounded_relu_index) << ");\n"; << ");\n";
} }
else else
{ {
auto bounded_relu_node = static_cast<const ngraph::op::BoundedRelu*>(node);
float alpha = bounded_relu_node->get_alpha();
writer << "#pragma omp parallel for\n"; writer << "#pragma omp parallel for\n";
writer << "for (size_t i = 0; i < " << out[0].get_size() << "; i++)\n"; writer << "for (size_t i = 0; i < " << out[0].get_size() << "; i++)\n";
writer.block_begin(); writer.block_begin();
...@@ -3249,42 +3316,49 @@ namespace ngraph ...@@ -3249,42 +3316,49 @@ namespace ngraph
template <> template <>
void CPU_Emitter::EMITTER_DECL(ngraph::op::Sigmoid) void CPU_Emitter::EMITTER_DECL(ngraph::op::Sigmoid)
{ {
auto input_shape = args[0].get_shape(); if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
auto result_shape = out[0].get_shape(); {
size_t sigmoid_index;
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); std::vector<std::size_t> deps;
size_t sigmoid_index = external_function->get_primitive_index(node); emit_build_primitives(external_function, node, writer, sigmoid_index, deps);
auto& deps = mkldnn_emitter->get_primitive_deps(sigmoid_index);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) << ", " writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) << ", " writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(sigmoid_index)
<< to_string(sigmoid_index) << ");\n"; << ");\n";
}
else
{
throw ngraph_error("Sigmoid is only supported with MKLDNN kernel.");
}
} }
template <> template <>
void CPU_Emitter::EMITTER_DECL(ngraph::op::SigmoidBackprop) void CPU_Emitter::EMITTER_DECL(ngraph::op::SigmoidBackprop)
{ {
auto input_shape = args[0].get_shape(); if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
auto delta_shape = args[1].get_shape(); {
auto result_shape = out[0].get_shape(); size_t sigmoid_index;
std::vector<std::size_t> deps;
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); emit_build_primitives(external_function, node, writer, sigmoid_index, deps);
size_t sigmoid_index = external_function->get_primitive_index(node);
auto& deps = mkldnn_emitter->get_primitive_deps(sigmoid_index);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) << ", " writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) << ", " writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< args[1].get_name() << ");\n"; << args[1].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[2]) << ", " writer << "cg_ctx->set_memory_ptr(" << to_string(deps[2]) << ", "
<< out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(sigmoid_index)
<< to_string(sigmoid_index) << ");\n"; << ");\n";
}
else
{
throw ngraph_error("SigmoidBackprop is only supported with MKLDNN kernel.");
}
} }
std::string std::string
...@@ -3445,17 +3519,17 @@ namespace ngraph ...@@ -3445,17 +3519,17 @@ namespace ngraph
{ {
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t softmax_index;
size_t softmax_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& deps = mkldnn_emitter->get_primitive_deps(softmax_index); emit_build_primitives(external_function, node, writer, softmax_index, deps);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(softmax_index)
<< to_string(softmax_index) << ");\n"; << ");\n";
} }
else else
{ {
...@@ -3854,16 +3928,17 @@ namespace ngraph ...@@ -3854,16 +3928,17 @@ namespace ngraph
{ {
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t dequantize_index;
size_t dequantize_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& deps = mkldnn_emitter->get_primitive_deps(dequantize_index); emit_build_primitives<ngraph::op::Dequantize>(
external_function, node, writer, dequantize_index, deps, args);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(dequantize_index)
<< to_string(dequantize_index) << ");\n"; << ");\n";
} }
else else
{ {
...@@ -3885,16 +3960,17 @@ namespace ngraph ...@@ -3885,16 +3960,17 @@ namespace ngraph
auto quantize = static_cast<const ngraph::op::Quantize*>(node); auto quantize = static_cast<const ngraph::op::Quantize*>(node);
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t quantize_index;
size_t quantize_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& deps = mkldnn_emitter->get_primitive_deps(quantize_index); emit_build_primitives<ngraph::op::Quantize>(
external_function, node, writer, quantize_index, deps, args);
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[0]) << ", "
<< ", " << args[0].get_name() << ");\n"; << args[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[1]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(quantize_index)
<< to_string(quantize_index) << ");\n"; << ");\n";
} }
else else
{ {
...@@ -3916,21 +3992,21 @@ namespace ngraph ...@@ -3916,21 +3992,21 @@ namespace ngraph
{ {
if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node)) if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node))
{ {
auto& mkldnn_emitter = external_function->get_mkldnn_emitter(); size_t concat_index;
size_t concat_index = external_function->get_primitive_index(node); std::vector<std::size_t> deps;
auto& deps = mkldnn_emitter->get_primitive_deps(concat_index); emit_build_primitives(external_function, node, writer, concat_index, deps);
size_t i; size_t i;
for (i = 0; i < args.size(); i++) for (i = 0; i < args.size(); i++)
{ {
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[i]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[i]) << ", "
<< ", " << args[i].get_name() << ");\n"; << args[i].get_name() << ");\n";
} }
writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[i]) writer << "cg_ctx->set_memory_ptr(" << to_string(deps[i]) << ", "
<< ", " << out[0].get_name() << ");\n"; << out[0].get_name() << ");\n";
writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, " writer << "cg_ctx->mkldnn_invoke_primitive(" << to_string(concat_index)
<< to_string(concat_index) << ");\n"; << ");\n";
} }
else else
{ {
......
src/ngraph/runtime/cpu/cpu_external_function.cpp
View file @ 9335e41c
...@@ -480,10 +480,7 @@ void runtime::cpu::CPU_ExternalFunction::compile(ngraph::pass::PassConfig& pass_ ...@@ -480,10 +480,7 @@ void runtime::cpu::CPU_ExternalFunction::compile(ngraph::pass::PassConfig& pass_
// Build mkldnn primitives for codegen. // Build mkldnn primitives for codegen.
pass_manager.register_pass<runtime::cpu::pass::MKLDNNPrimitiveBuildPass>( pass_manager.register_pass<runtime::cpu::pass::MKLDNNPrimitiveBuildPass>(
m_desc_filename, m_desc_filename, *m_mkldnn_emitter, m_node_primitive_string_deps_index_map);
*m_mkldnn_emitter,
m_node_primitive_idx_map,
m_node_primitive_string_deps_index_map);
unordered_map<Node*, Node*> node_function_map; unordered_map<Node*, Node*> node_function_map;
string common_function_string; string common_function_string;
...@@ -746,16 +743,20 @@ using namespace ngraph::runtime; ...@@ -746,16 +743,20 @@ using namespace ngraph::runtime;
writer << "extern \"C\" void " << current_function->get_name() << func_params << "\n"; writer << "extern \"C\" void " << current_function->get_name() << func_params << "\n";
writer << "{\n"; writer << "{\n";
writer.indent++; writer.indent++;
writer << "std::ifstream desc_file (\"" << m_desc_filename << "\", std::ios::binary);\n";
//deserialize and build mkldnn primitives //deserialize and build mkldnn primitives
writer << "if (ctx->first_iteration)\n"; if (m_mkldnn_emitter->get_mkldnn_descriptors_size() > 0)
writer.block_begin(); {
writer << "// read in memory descriptors and build mkldnn primitives\n"; writer << "if (ctx->first_iteration)\n";
writer << "deserialize_memory_descs_and_build_memory_primitives(" << m_desc_filename writer.block_begin();
<< ", cg_ctx, " << to_string(m_mkldnn_emitter->get_mkldnn_descriptors_size()) writer << "// read in memory descriptors and build mkldnn primitives\n";
<< ");\n"; writer << "std::ifstream desc_file (\"" << m_desc_filename
writer.block_end(); << "\", std::ios::binary);\n";
writer << "deserialize_memory_descs_and_build_memory_primitives(" << m_desc_filename
<< ", cg_ctx, " << to_string(m_mkldnn_emitter->get_mkldnn_descriptors_size())
<< ");\n";
writer.block_end();
}
// Execution tracing support // Execution tracing support
if (runtime::cpu::IsTracingEnabled() && current_function->get_name() == m_function_name) if (runtime::cpu::IsTracingEnabled() && current_function->get_name() == m_function_name)
......
src/ngraph/runtime/cpu/cpu_external_function.hpp
View file @ 9335e41c
...@@ -114,17 +114,6 @@ namespace ngraph ...@@ -114,17 +114,6 @@ namespace ngraph
return m_mkldnn_emitter; return m_mkldnn_emitter;
} }
/// Returns the index of the mkldnn primitive previously created for \p node.
size_t get_primitive_index(const Node* node) const
{
auto it = m_node_primitive_idx_map.find(node);
NGRAPH_CHECK(it != m_node_primitive_idx_map.end(),
"Primitive not found for node ",
node->description());
return it->second;
}
// Return the tuple including the string to create mkldnn primitive, the deps and the index in CODEGEN // Return the tuple including the string to create mkldnn primitive, the deps and the index in CODEGEN
const std::tuple<std::string, std::vector<size_t>, size_t>& const std::tuple<std::string, std::vector<size_t>, size_t>&
get_primitive_build_tuple(const Node* node) const get_primitive_build_tuple(const Node* node) const
...@@ -328,8 +317,6 @@ namespace ngraph ...@@ -328,8 +317,6 @@ namespace ngraph
std::unordered_map<std::string, size_t> subgraph_param_indices; std::unordered_map<std::string, size_t> subgraph_param_indices;
#endif #endif
/// Map each node with mkldnn implementation to its mkldnn primitive index.
std::unordered_map<const Node*, size_t> m_node_primitive_idx_map;
/// Map each node with mkldnn implementation to its mkldnn primitive creating string, deps, and mkldnn primitive index. /// Map each node with mkldnn implementation to its mkldnn primitive creating string, deps, and mkldnn primitive index.
std::map<const Node*, std::tuple<std::string, std::vector<size_t>, size_t>> std::map<const Node*, std::tuple<std::string, std::vector<size_t>, size_t>>
m_node_primitive_string_deps_index_map; m_node_primitive_string_deps_index_map;
......
src/ngraph/runtime/cpu/mkldnn_emitter.cpp
View file @ 9335e41c
...@@ -24,19 +24,15 @@ ...@@ -24,19 +24,15 @@
#include "ngraph/op/batch_norm.hpp" #include "ngraph/op/batch_norm.hpp"
#include "ngraph/op/concat.hpp" #include "ngraph/op/concat.hpp"
#include "ngraph/op/constant.hpp" #include "ngraph/op/constant.hpp"
#include "ngraph/op/constant.hpp"
#include "ngraph/op/convert.hpp" #include "ngraph/op/convert.hpp"
#include "ngraph/op/convolution.hpp" #include "ngraph/op/convolution.hpp"
#include "ngraph/op/dequantize.hpp" #include "ngraph/op/dequantize.hpp"
#include "ngraph/op/dequantize.hpp"
#include "ngraph/op/experimental/quantized_avg_pool.hpp"
#include "ngraph/op/experimental/quantized_avg_pool.hpp" #include "ngraph/op/experimental/quantized_avg_pool.hpp"
#include "ngraph/op/experimental/quantized_concat.hpp" #include "ngraph/op/experimental/quantized_concat.hpp"
#include "ngraph/op/experimental/quantized_conv.hpp" #include "ngraph/op/experimental/quantized_conv.hpp"
#include "ngraph/op/experimental/quantized_conv_bias.hpp" #include "ngraph/op/experimental/quantized_conv_bias.hpp"
#include "ngraph/op/experimental/quantized_conv_relu.hpp" #include "ngraph/op/experimental/quantized_conv_relu.hpp"
#include "ngraph/op/experimental/quantized_max_pool.hpp" #include "ngraph/op/experimental/quantized_max_pool.hpp"
#include "ngraph/op/experimental/quantized_max_pool.hpp"
#include "ngraph/op/get_output_element.hpp" #include "ngraph/op/get_output_element.hpp"
#include "ngraph/op/lrn.hpp" #include "ngraph/op/lrn.hpp"
#include "ngraph/op/max_pool.hpp" #include "ngraph/op/max_pool.hpp"
...@@ -342,119 +338,6 @@ void MKLDNNEmitter::build_deconvolutionbias_forward( ...@@ -342,119 +338,6 @@ void MKLDNNEmitter::build_deconvolutionbias_forward(
} }
} }
size_t MKLDNNEmitter::build_deconvolutionbias_forward(const mkldnn::memory::desc& input_data_desc,
const mkldnn::memory::desc& weights_desc,
const mkldnn::memory::desc& bias_desc,
const mkldnn::memory::desc& result_desc,
const ngraph::Strides& strides,
const ngraph::Strides& dilation_strides,
const ngraph::CoordinateDiff& padding_below,
const ngraph::CoordinateDiff& padding_above,
const mkldnn::post_ops& pops)
{
size_t input_data_index = build_memory_primitive(input_data_desc);
size_t weights_index = build_memory_primitive(weights_desc);
size_t bias_index = build_memory_primitive(bias_desc);
size_t result_index = build_memory_primitive(result_desc);
mkldnn::primitive_attr conv_attr;
conv_attr.set_post_ops(pops);
size_t conv_index = 0;
try
{
auto conv_prim = new mkldnn::deconvolution_forward(
{{mkldnn::prop_kind::forward,
mkldnn::algorithm::deconvolution_direct,
input_data_desc,
weights_desc,
bias_desc,
result_desc,
mkldnn::memory::dims(strides.begin(), strides.end()),
mkldnn::memory::dims(dilation_strides.begin(), dilation_strides.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
conv_attr,
executor::global_cpu_engine},
*m_mkldnn_primitives[input_data_index],
*m_mkldnn_primitives[weights_index],
*m_mkldnn_primitives[bias_index],
*m_mkldnn_primitives[result_index]);
conv_index = insert_primitive(conv_prim);
m_primitive_deps[conv_index] = {weights_index, input_data_index, bias_index, result_index};
}
catch (const mkldnn::error& e)
{
throw ngraph_error("Could not create mkldnn deconvolution_forward " + e.message);
}
return conv_index;
}
size_t MKLDNNEmitter::build_convolution_backward_weights_bias(
const mkldnn::memory::desc& in_data_desc,
const mkldnn::memory::desc& in_delta_desc,
const mkldnn::memory::desc& out_weights_delta_desc,
const mkldnn::memory::desc& out_bias_delta_desc,
const ngraph::Strides& ng_strides,
const ngraph::Strides& ng_dilation_strides,
const ngraph::CoordinateDiff& ng_padding_below,
const ngraph::CoordinateDiff& ng_padding_above)
{
const size_t in_data_index = build_memory_primitive(in_data_desc);
const size_t in_delta_index = build_memory_primitive(in_delta_desc);
const size_t out_weights_delta_index = build_memory_primitive(out_weights_delta_desc);
const size_t out_bias_delta_index = build_memory_primitive(out_bias_delta_desc);
mkldnn::memory::dims strides(ng_strides.begin(), ng_strides.end());
mkldnn::memory::dims dilation(ng_dilation_strides.begin(), ng_dilation_strides.end());
mkldnn::memory::dims padding_l(ng_padding_below.begin(), ng_padding_below.end());
mkldnn::memory::dims padding_r(ng_padding_above.begin(), ng_padding_above.end());
mkldnn::algorithm convolution_algo = mkldnn_utils::get_conv_algo();
mkldnn::convolution_forward::primitive_desc fwd_pd{{mkldnn::prop_kind::forward,
convolution_algo,
in_data_desc,
out_weights_delta_desc,
out_bias_delta_desc,
in_delta_desc,
strides,
dilation,
padding_l,
padding_r,
mkldnn::padding_kind::zero},
executor::global_cpu_engine};
mkldnn::convolution_backward_weights::primitive_desc bwd_pd{{convolution_algo,
in_data_desc,
out_weights_delta_desc,
out_bias_delta_desc,
in_delta_desc,
strides,
dilation,
padding_l,
padding_r,
mkldnn::padding_kind::zero},
executor::global_cpu_engine,
fwd_pd};
const size_t conv_index = insert_primitive(
new mkldnn::convolution_backward_weights(bwd_pd,
*m_mkldnn_primitives[in_data_index],
*m_mkldnn_primitives[in_delta_index],
*m_mkldnn_primitives[out_weights_delta_index],
*m_mkldnn_primitives[out_bias_delta_index]));
NGRAPH_CHECK(m_primitive_deps.find(conv_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[conv_index] = {
in_data_index, in_delta_index, out_weights_delta_index, out_bias_delta_index};
return conv_index;
}
void MKLDNNEmitter::build_convolution_backward_weights_bias( void MKLDNNEmitter::build_convolution_backward_weights_bias(
std::vector<mkldnn::primitive*>& mkldnn_primitives, std::vector<mkldnn::primitive*>& mkldnn_primitives,
const mkldnn::convolution_backward_weights::desc& bwd_desc, const mkldnn::convolution_backward_weights::desc& bwd_desc,
...@@ -462,15 +345,14 @@ void MKLDNNEmitter::build_convolution_backward_weights_bias( ...@@ -462,15 +345,14 @@ void MKLDNNEmitter::build_convolution_backward_weights_bias(
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t conv_index) size_t conv_index)
{ {
size_t in_data_index = deps[0]; size_t src_index = deps[0];
build_memory_primitive(mkldnn_primitives, bwd_desc.data.src_desc, in_data_index); build_memory_primitive(mkldnn_primitives, bwd_desc.data.src_desc, src_index);
size_t in_delta_index = deps[1]; size_t diff_dst_index = deps[1];
build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_dst_desc, in_delta_index); build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_dst_desc, diff_dst_index);
size_t out_weights_delta_index = deps[2]; size_t diff_weights_index = deps[2];
build_memory_primitive( build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_weights_desc, diff_weights_index);
mkldnn_primitives, bwd_desc.data.diff_weights_desc, out_weights_delta_index); size_t diff_bias_index = deps[3];
size_t out_bias_delta_index = deps[3]; build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_bias_desc, diff_bias_index);
build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_bias_desc, out_bias_delta_index);
mkldnn::convolution_forward::primitive_desc fwd_pd{fwd_desc, executor::global_cpu_engine}; mkldnn::convolution_forward::primitive_desc fwd_pd{fwd_desc, executor::global_cpu_engine};
...@@ -479,58 +361,10 @@ void MKLDNNEmitter::build_convolution_backward_weights_bias( ...@@ -479,58 +361,10 @@ void MKLDNNEmitter::build_convolution_backward_weights_bias(
mkldnn_primitives[conv_index] = mkldnn_primitives[conv_index] =
new mkldnn::convolution_backward_weights(bwd_pd, new mkldnn::convolution_backward_weights(bwd_pd,
*mkldnn_primitives[in_data_index], *mkldnn_primitives[src_index],
*mkldnn_primitives[in_delta_index], *mkldnn_primitives[diff_dst_index],
*mkldnn_primitives[out_weights_delta_index], *mkldnn_primitives[diff_weights_index],
*mkldnn_primitives[out_bias_delta_index]); *mkldnn_primitives[diff_bias_index]);
}
size_t
MKLDNNEmitter::build_convolution_backward_weights(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& delta_desc,
const mkldnn::memory::desc& result_desc,
const ngraph::Strides& strides,
const ngraph::Strides& dilation_strides,
const ngraph::CoordinateDiff& padding_below,
const ngraph::CoordinateDiff& padding_above)
{
size_t input_index = build_memory_primitive(input_desc);
size_t delta_index = build_memory_primitive(delta_desc);
size_t result_index = build_memory_primitive(result_desc);
mkldnn::algorithm convolution_algo = mkldnn_utils::get_conv_algo();
size_t primitive_index = insert_primitive(new mkldnn::convolution_backward_weights(
{{convolution_algo,
input_desc,
result_desc,
delta_desc,
mkldnn::memory::dims(strides.begin(), strides.end()),
mkldnn::memory::dims(dilation_strides.begin(), dilation_strides.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
executor::global_cpu_engine,
// Forward primitive descriptor corresponding to this backward weights descriptor
{{mkldnn::prop_kind::forward,
convolution_algo,
input_desc,
result_desc,
delta_desc,
mkldnn::memory::dims(strides.begin(), strides.end()),
mkldnn::memory::dims(dilation_strides.begin(), dilation_strides.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
executor::global_cpu_engine}},
*m_mkldnn_primitives[input_index],
*m_mkldnn_primitives[delta_index],
*m_mkldnn_primitives[result_index]));
NGRAPH_CHECK(m_primitive_deps.find(primitive_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[primitive_index] = {input_index, delta_index, result_index};
return primitive_index;
} }
void MKLDNNEmitter::build_convolution_backward_weights( void MKLDNNEmitter::build_convolution_backward_weights(
...@@ -540,66 +374,21 @@ void MKLDNNEmitter::build_convolution_backward_weights( ...@@ -540,66 +374,21 @@ void MKLDNNEmitter::build_convolution_backward_weights(
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t conv_index) size_t conv_index)
{ {
size_t in_data_index = deps[0]; size_t src_index = deps[0];
build_memory_primitive(mkldnn_primitives, bwd_desc.data.src_desc, in_data_index); build_memory_primitive(mkldnn_primitives, bwd_desc.data.src_desc, src_index);
size_t in_delta_index = deps[1]; size_t diff_dst_index = deps[1];
build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_dst_desc, in_delta_index); build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_dst_desc, diff_dst_index);
size_t out_weights_delta_index = deps[2]; size_t diff_weights_index = deps[2];
build_memory_primitive( build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_weights_desc, diff_weights_index);
mkldnn_primitives, bwd_desc.data.diff_weights_desc, out_weights_delta_index);
mkldnn_primitives[conv_index] = new mkldnn::convolution_backward_weights( mkldnn_primitives[conv_index] = new mkldnn::convolution_backward_weights(
{bwd_desc, {bwd_desc,
executor::global_cpu_engine, executor::global_cpu_engine,
// Forward primitive descriptor corresponding to this backward weights descriptor // Forward primitive descriptor corresponding to this backward weights descriptor
{fwd_desc, executor::global_cpu_engine}}, {fwd_desc, executor::global_cpu_engine}},
*mkldnn_primitives[in_data_index], *mkldnn_primitives[src_index],
*mkldnn_primitives[in_delta_index], *mkldnn_primitives[diff_dst_index],
*mkldnn_primitives[out_weights_delta_index]); *mkldnn_primitives[diff_weights_index]);
}
size_t MKLDNNEmitter::build_convolution_backward_data(const mkldnn::memory::desc& weights_desc,
const mkldnn::memory::desc& delta_desc,
const mkldnn::memory::desc& result_desc,
const ngraph::Strides& strides,
const ngraph::Strides& dilation_strides,
const ngraph::CoordinateDiff& padding_below,
const ngraph::CoordinateDiff& padding_above)
{
size_t weights_index = build_memory_primitive(weights_desc);
size_t delta_index = build_memory_primitive(delta_desc);
size_t result_index = build_memory_primitive(result_desc);
mkldnn::algorithm convolution_algo = mkldnn_utils::get_conv_algo();
size_t primitive_index = insert_primitive(new mkldnn::convolution_backward_data(
{{convolution_algo,
result_desc,
weights_desc,
delta_desc,
mkldnn::memory::dims(strides.begin(), strides.end()),
mkldnn::memory::dims(dilation_strides.begin(), dilation_strides.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
executor::global_cpu_engine,
// Forward primitive descriptor corresponding to this backward data descriptor
{{mkldnn::prop_kind::forward,
convolution_algo,
result_desc,
weights_desc,
delta_desc,
mkldnn::memory::dims(strides.begin(), strides.end()),
mkldnn::memory::dims(dilation_strides.begin(), dilation_strides.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
executor::global_cpu_engine}},
*m_mkldnn_primitives[delta_index],
*m_mkldnn_primitives[weights_index],
*m_mkldnn_primitives[result_index]));
m_primitive_deps[primitive_index] = {weights_index, delta_index, result_index};
return primitive_index;
} }
void MKLDNNEmitter::build_convolution_backward_data( void MKLDNNEmitter::build_convolution_backward_data(
...@@ -611,19 +400,19 @@ void MKLDNNEmitter::build_convolution_backward_data( ...@@ -611,19 +400,19 @@ void MKLDNNEmitter::build_convolution_backward_data(
{ {
size_t weights_index = deps[0]; size_t weights_index = deps[0];
build_memory_primitive(mkldnn_primitives, bwd_desc.data.weights_desc, weights_index); build_memory_primitive(mkldnn_primitives, bwd_desc.data.weights_desc, weights_index);
size_t delta_index = deps[1]; size_t diff_dst_index = deps[1];
build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_dst_desc, delta_index); build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_dst_desc, diff_dst_index);
size_t result_index = deps[2]; size_t diff_src_index = deps[2];
build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_src_desc, result_index); build_memory_primitive(mkldnn_primitives, bwd_desc.data.diff_src_desc, diff_src_index);
mkldnn_primitives[conv_index] = new mkldnn::convolution_backward_data( mkldnn_primitives[conv_index] = new mkldnn::convolution_backward_data(
{bwd_desc, {bwd_desc,
executor::global_cpu_engine, executor::global_cpu_engine,
// Forward primitive descriptor corresponding to this backward data descriptor // Forward primitive descriptor corresponding to this backward data descriptor
{fwd_desc, executor::global_cpu_engine}}, {fwd_desc, executor::global_cpu_engine}},
*mkldnn_primitives[delta_index], *mkldnn_primitives[diff_dst_index],
*mkldnn_primitives[weights_index], *mkldnn_primitives[weights_index],
*mkldnn_primitives[result_index]); *mkldnn_primitives[diff_src_index]);
} }
void MKLDNNEmitter::build_pooling_forward(std::vector<mkldnn::primitive*>& mkldnn_primitives, void MKLDNNEmitter::build_pooling_forward(std::vector<mkldnn::primitive*>& mkldnn_primitives,
...@@ -642,47 +431,6 @@ void MKLDNNEmitter::build_pooling_forward(std::vector<mkldnn::primitive*>& mkldn ...@@ -642,47 +431,6 @@ void MKLDNNEmitter::build_pooling_forward(std::vector<mkldnn::primitive*>& mkldn
*mkldnn_primitives[result_index]); *mkldnn_primitives[result_index]);
} }
size_t MKLDNNEmitter::build_pooling_backward(mkldnn::algorithm pooling_algorithm,
const mkldnn::memory::desc& diff_dst_desc,
const mkldnn::memory::desc& diff_src_desc,
const ngraph::Strides& window_strides,
const ngraph::Shape& window_shape,
const ngraph::Shape& padding_below,
const ngraph::Shape& padding_above)
{
size_t input_index = build_memory_primitive(diff_dst_desc);
size_t result_index = build_memory_primitive(diff_src_desc);
size_t primitive_index = insert_primitive(new mkldnn::pooling_backward(
{{pooling_algorithm,
diff_src_desc,
diff_dst_desc,
mkldnn::memory::dims(window_strides.begin(), window_strides.end()),
mkldnn::memory::dims(window_shape.begin(), window_shape.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
executor::global_cpu_engine,
{{mkldnn::prop_kind::forward_training,
pooling_algorithm,
diff_src_desc,
diff_dst_desc,
mkldnn::memory::dims(window_strides.begin(), window_strides.end()),
mkldnn::memory::dims(window_shape.begin(), window_shape.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
executor::global_cpu_engine}},
*m_mkldnn_primitives[input_index],
*m_mkldnn_primitives[result_index]));
NGRAPH_CHECK(m_primitive_deps.find(primitive_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[primitive_index] = {input_index, result_index};
return primitive_index;
}
void MKLDNNEmitter::build_pooling_backward(std::vector<mkldnn::primitive*>& mkldnn_primitives, void MKLDNNEmitter::build_pooling_backward(std::vector<mkldnn::primitive*>& mkldnn_primitives,
const mkldnn::pooling_backward::desc& pool_desc, const mkldnn::pooling_backward::desc& pool_desc,
const mkldnn::pooling_forward::desc& pool_fwd_desc, const mkldnn::pooling_forward::desc& pool_fwd_desc,
...@@ -703,71 +451,6 @@ void MKLDNNEmitter::build_pooling_backward(std::vector<mkldnn::primitive*>& mkld ...@@ -703,71 +451,6 @@ void MKLDNNEmitter::build_pooling_backward(std::vector<mkldnn::primitive*>& mkld
pool_pd, *mkldnn_primitives[input_index], *mkldnn_primitives[result_index]); pool_pd, *mkldnn_primitives[input_index], *mkldnn_primitives[result_index]);
} }
size_t MKLDNNEmitter::build_max_pooling_backward(mkldnn::algorithm pooling_algorithm,
const mkldnn::memory::desc& fprop_src_desc,
const mkldnn::memory::desc& diff_dst_desc,
const mkldnn::memory::desc& diff_src_desc,
const ngraph::Strides& window_strides,
const ngraph::Shape& window_shape,
const ngraph::Shape& padding_below,
const ngraph::Shape& padding_above)
{
size_t fprop_src_index = build_memory_primitive(fprop_src_desc);
size_t diff_dst_index = build_memory_primitive(diff_dst_desc);
size_t diff_src_index = build_memory_primitive(diff_src_desc);
mkldnn::pooling_forward::primitive_desc fwd_pd{
{mkldnn::prop_kind::forward_training,
pooling_algorithm,
diff_src_desc,
diff_dst_desc,
mkldnn::memory::dims(window_strides.begin(), window_strides.end()),
mkldnn::memory::dims(window_shape.begin(), window_shape.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
executor::global_cpu_engine};
auto ws_index = build_memory_primitive(fwd_pd.workspace_primitive_desc().desc());
// Allocate workspace
// TODO (jbobba): Might need to align memory
auto ws = std::unique_ptr<MKLDNNWorkspace>(
new MKLDNNWorkspace(fwd_pd.workspace_primitive_desc().get_size()));
auto ws_buf_index = insert_workspace(ws);
size_t fwd_primitive_index = insert_primitive(new mkldnn::pooling_forward(
fwd_pd,
*m_mkldnn_primitives[fprop_src_index],
*m_mkldnn_primitives
[diff_src_index], // HACK - Uses diff_src buffer. Safe since diff_src > fprop_dst
*m_mkldnn_primitives[ws_index]));
size_t bwd_primitive_index = insert_primitive(new mkldnn::pooling_backward(
{{pooling_algorithm,
diff_src_desc,
diff_dst_desc,
mkldnn::memory::dims(window_strides.begin(), window_strides.end()),
mkldnn::memory::dims(window_shape.begin(), window_shape.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
executor::global_cpu_engine,
fwd_pd},
*m_mkldnn_primitives[diff_dst_index],
*m_mkldnn_primitives[ws_index],
*m_mkldnn_primitives[diff_src_index]));
NGRAPH_CHECK(m_primitive_deps.find(fwd_primitive_index) == m_primitive_deps.end() &&
m_primitive_deps.find(bwd_primitive_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[fwd_primitive_index] = {
fprop_src_index, diff_src_index, ws_index, ws_buf_index};
m_primitive_deps[bwd_primitive_index] = {
diff_dst_index, ws_index, diff_src_index, ws_buf_index};
return bwd_primitive_index;
}
void MKLDNNEmitter::build_max_pooling_backward(std::vector<mkldnn::primitive*>& mkldnn_primitives, void MKLDNNEmitter::build_max_pooling_backward(std::vector<mkldnn::primitive*>& mkldnn_primitives,
std::vector<char*>& mkldnn_workspaces, std::vector<char*>& mkldnn_workspaces,
const mkldnn::pooling_backward::desc& bwd_pool_desc, const mkldnn::pooling_backward::desc& bwd_pool_desc,
...@@ -814,44 +497,6 @@ void MKLDNNEmitter::build_max_pooling_backward(std::vector<mkldnn::primitive*>& ...@@ -814,44 +497,6 @@ void MKLDNNEmitter::build_max_pooling_backward(std::vector<mkldnn::primitive*>&
*mkldnn_primitives[diff_src_index]); *mkldnn_primitives[diff_src_index]);
} }
size_t MKLDNNEmitter::build_max_pooling_with_indices_forward(mkldnn::algorithm pooling_algorithm,
const mkldnn::memory::desc& src_desc,
const mkldnn::memory::desc& dst_desc,
const ngraph::Strides& window_strides,
const ngraph::Shape& window_shape,
const ngraph::Shape& padding_below,
const ngraph::Shape& padding_above)
{
size_t src_index = build_memory_primitive(src_desc);
size_t dst_index = build_memory_primitive(dst_desc);
mkldnn::pooling_forward::primitive_desc fwd_pd{
{mkldnn::prop_kind::forward_training,
pooling_algorithm,
src_desc,
dst_desc,
mkldnn::memory::dims(window_strides.begin(), window_strides.end()),
mkldnn::memory::dims(window_shape.begin(), window_shape.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
executor::global_cpu_engine};
auto ws_index = build_memory_primitive(fwd_pd.workspace_primitive_desc().desc());
size_t fwd_primitive_index =
insert_primitive(new mkldnn::pooling_forward(fwd_pd,
*m_mkldnn_primitives[src_index],
*m_mkldnn_primitives[dst_index],
*m_mkldnn_primitives[ws_index]));
NGRAPH_CHECK(m_primitive_deps.find(fwd_primitive_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[fwd_primitive_index] = {src_index, dst_index, ws_index};
return fwd_primitive_index;
}
void MKLDNNEmitter::build_max_pooling_with_indices_forward( void MKLDNNEmitter::build_max_pooling_with_indices_forward(
std::vector<mkldnn::primitive*>& mkldnn_primitives, std::vector<mkldnn::primitive*>& mkldnn_primitives,
const mkldnn::pooling_forward::desc& max_pool_desc, const mkldnn::pooling_forward::desc& max_pool_desc,
...@@ -874,54 +519,6 @@ void MKLDNNEmitter::build_max_pooling_with_indices_forward( ...@@ -874,54 +519,6 @@ void MKLDNNEmitter::build_max_pooling_with_indices_forward(
*mkldnn_primitives[ws_index]); *mkldnn_primitives[ws_index]);
} }
size_t MKLDNNEmitter::build_max_pooling_with_indices_backward(
mkldnn::algorithm pooling_algorithm,
const mkldnn::memory::desc& diff_dst_desc,
const mkldnn::memory::desc& diff_src_desc,
const ngraph::Strides& window_strides,
const ngraph::Shape& window_shape,
const ngraph::Shape& padding_below,
const ngraph::Shape& padding_above)
{
size_t diff_dst_index = build_memory_primitive(diff_dst_desc);
size_t diff_src_index = build_memory_primitive(diff_src_desc);
mkldnn::pooling_forward::primitive_desc fwd_pd{
{mkldnn::prop_kind::forward_training,
pooling_algorithm,
diff_src_desc,
diff_dst_desc,
mkldnn::memory::dims(window_strides.begin(), window_strides.end()),
mkldnn::memory::dims(window_shape.begin(), window_shape.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
executor::global_cpu_engine};
auto fprop_ws_index = build_memory_primitive(fwd_pd.workspace_primitive_desc().desc());
size_t bwd_primitive_index = insert_primitive(new mkldnn::pooling_backward(
{{pooling_algorithm,
diff_src_desc,
diff_dst_desc,
mkldnn::memory::dims(window_strides.begin(), window_strides.end()),
mkldnn::memory::dims(window_shape.begin(), window_shape.end()),
mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
mkldnn::padding_kind::zero},
executor::global_cpu_engine,
fwd_pd},
*m_mkldnn_primitives[diff_dst_index],
*m_mkldnn_primitives[fprop_ws_index],
*m_mkldnn_primitives[diff_src_index]));
NGRAPH_CHECK(m_primitive_deps.find(bwd_primitive_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[bwd_primitive_index] = {diff_dst_index, fprop_ws_index, diff_src_index};
return bwd_primitive_index;
}
void MKLDNNEmitter::build_max_pooling_with_indices_backward( void MKLDNNEmitter::build_max_pooling_with_indices_backward(
std::vector<mkldnn::primitive*>& mkldnn_primitives, std::vector<mkldnn::primitive*>& mkldnn_primitives,
const mkldnn::pooling_backward::desc& bwd_pool_desc, const mkldnn::pooling_backward::desc& bwd_pool_desc,
...@@ -1023,27 +620,6 @@ void MKLDNNEmitter::build_lrn_forward(std::vector<mkldnn::primitive*>& mkldnn_pr ...@@ -1023,27 +620,6 @@ void MKLDNNEmitter::build_lrn_forward(std::vector<mkldnn::primitive*>& mkldnn_pr
lrn_prim_desc, *mkldnn_primitives[input_index], *mkldnn_primitives[result_index]); lrn_prim_desc, *mkldnn_primitives[input_index], *mkldnn_primitives[result_index]);
} }
size_t MKLDNNEmitter::build_relu_forward(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc)
{
size_t input_index = build_memory_primitive(input_desc);
size_t result_index = build_memory_primitive(result_desc);
const float negative_slope = 0.0f;
auto relu_desc = mkldnn::eltwise_forward::desc(
mkldnn::prop_kind::forward, mkldnn::algorithm::eltwise_relu, input_desc, negative_slope);
auto relu_pd = mkldnn::eltwise_forward::primitive_desc(relu_desc, executor::global_cpu_engine);
size_t primitive_index = insert_primitive(new mkldnn::eltwise_forward(
relu_pd, *m_mkldnn_primitives[input_index], *m_mkldnn_primitives[result_index]));
NGRAPH_CHECK(m_primitive_deps.find(primitive_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[primitive_index] = {input_index, result_index};
return primitive_index;
}
mkldnn::eltwise_forward::desc MKLDNNEmitter::get_relu_forward_desc(const ngraph::Node* node) mkldnn::eltwise_forward::desc MKLDNNEmitter::get_relu_forward_desc(const ngraph::Node* node)
{ {
const float negative_slope = 0.0f; const float negative_slope = 0.0f;
...@@ -1070,39 +646,6 @@ void MKLDNNEmitter::build_relu_forward(std::vector<mkldnn::primitive*>& mkldnn_p ...@@ -1070,39 +646,6 @@ void MKLDNNEmitter::build_relu_forward(std::vector<mkldnn::primitive*>& mkldnn_p
*mkldnn_primitives[result_index]); *mkldnn_primitives[result_index]);
} }
size_t MKLDNNEmitter::build_relu_backward(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& delta_desc,
const mkldnn::memory::desc& result_desc)
{
size_t input_index = build_memory_primitive(input_desc);
size_t delta_index = build_memory_primitive(delta_desc);
size_t result_index = build_memory_primitive(result_desc);
/* Backward relu */
const float negative_slope = 0.0f;
auto relu_desc = mkldnn::eltwise_forward::desc(
mkldnn::prop_kind::forward, mkldnn::algorithm::eltwise_relu, input_desc, negative_slope);
auto relu_pd = mkldnn::eltwise_forward::primitive_desc(relu_desc, executor::global_cpu_engine);
/* create backward relu primitive_descriptor */
auto relu_bwd_desc = mkldnn::eltwise_backward::desc(
mkldnn::algorithm::eltwise_relu, result_desc, input_desc, negative_slope);
auto relu_bwd_pd = mkldnn::eltwise_backward::primitive_desc(
relu_bwd_desc, executor::global_cpu_engine, relu_pd);
size_t primitive_index =
insert_primitive(new mkldnn::eltwise_backward(relu_bwd_pd,
*m_mkldnn_primitives[input_index],
*m_mkldnn_primitives[delta_index],
*m_mkldnn_primitives[result_index]));
NGRAPH_CHECK(m_primitive_deps.find(primitive_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[primitive_index] = {input_index, delta_index, result_index};
return primitive_index;
}
mkldnn::eltwise_backward::desc MKLDNNEmitter::get_relu_backward_desc(const ngraph::Node* node) mkldnn::eltwise_backward::desc MKLDNNEmitter::get_relu_backward_desc(const ngraph::Node* node)
{ {
auto input_desc = mkldnn_utils::get_input_mkldnn_md(node, 0); auto input_desc = mkldnn_utils::get_input_mkldnn_md(node, 0);
...@@ -1139,29 +682,6 @@ void MKLDNNEmitter::build_relu_backward(std::vector<mkldnn::primitive*>& mkldnn_ ...@@ -1139,29 +682,6 @@ void MKLDNNEmitter::build_relu_backward(std::vector<mkldnn::primitive*>& mkldnn_
*mkldnn_primitives[result_index]); *mkldnn_primitives[result_index]);
} }
size_t MKLDNNEmitter::build_sigmoid_forward(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc)
{
size_t input_index = build_memory_primitive(input_desc);
size_t result_index = build_memory_primitive(result_desc);
size_t primitive_index =
insert_primitive(new mkldnn::eltwise_forward({{mkldnn::prop_kind::forward_training,
mkldnn::algorithm::eltwise_logistic,
input_desc,
0,
0},
executor::global_cpu_engine},
*m_mkldnn_primitives[input_index],
*m_mkldnn_primitives[result_index]));
NGRAPH_CHECK(m_primitive_deps.find(primitive_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[primitive_index] = {input_index, result_index};
return primitive_index;
}
mkldnn::eltwise_forward::desc MKLDNNEmitter::get_sigmoid_forward_desc(const ngraph::Node* node, mkldnn::eltwise_forward::desc MKLDNNEmitter::get_sigmoid_forward_desc(const ngraph::Node* node,
bool backward_op) bool backward_op)
{ {
...@@ -1198,35 +718,6 @@ void MKLDNNEmitter::build_sigmoid_forward(std::vector<mkldnn::primitive*>& mkldn ...@@ -1198,35 +718,6 @@ void MKLDNNEmitter::build_sigmoid_forward(std::vector<mkldnn::primitive*>& mkldn
*mkldnn_primitives[result_index]); *mkldnn_primitives[result_index]);
} }
size_t MKLDNNEmitter::build_sigmoid_backward(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& delta_desc,
const mkldnn::memory::desc& result_desc)
{
size_t input_index = build_memory_primitive(input_desc);
size_t delta_index = build_memory_primitive(delta_desc);
size_t result_index = build_memory_primitive(result_desc);
// sigmoid forward primitive desc
mkldnn::eltwise_forward::primitive_desc sigmoid_fwd_pd =
mkldnn::eltwise_forward::primitive_desc(
{mkldnn::prop_kind::forward, mkldnn::algorithm::eltwise_logistic, input_desc, 0, 0},
executor::global_cpu_engine);
size_t primitive_index = insert_primitive(new mkldnn::eltwise_backward(
{{mkldnn::algorithm::eltwise_logistic, delta_desc, input_desc, 0, 0},
executor::global_cpu_engine,
sigmoid_fwd_pd},
*m_mkldnn_primitives[input_index],
*m_mkldnn_primitives[delta_index],
*m_mkldnn_primitives[result_index]));
NGRAPH_CHECK(m_primitive_deps.find(primitive_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[primitive_index] = {input_index, delta_index, result_index};
return primitive_index;
}
mkldnn::eltwise_backward::desc MKLDNNEmitter::get_sigmoid_backward_desc(const ngraph::Node* node) mkldnn::eltwise_backward::desc MKLDNNEmitter::get_sigmoid_backward_desc(const ngraph::Node* node)
{ {
auto input_desc = mkldnn_utils::get_input_mkldnn_md(node, 0); auto input_desc = mkldnn_utils::get_input_mkldnn_md(node, 0);
...@@ -1432,6 +923,7 @@ void MKLDNNEmitter::build_rnn_forward(std::vector<mkldnn::primitive*>& mkldnn_pr ...@@ -1432,6 +923,7 @@ void MKLDNNEmitter::build_rnn_forward(std::vector<mkldnn::primitive*>& mkldnn_pr
size_t weights_layer_index = deps[2]; size_t weights_layer_index = deps[2];
build_memory_primitive( build_memory_primitive(
mkldnn_primitives, rnn_desc.data.weights_layer_desc, weights_layer_index); mkldnn_primitives, rnn_desc.data.weights_layer_desc, weights_layer_index);
size_t weights_iter_index = m_primitive_deps[rnn_index][3]; size_t weights_iter_index = m_primitive_deps[rnn_index][3];
build_memory_primitive(mkldnn_primitives, rnn_desc.data.weights_iter_desc, weights_iter_index); build_memory_primitive(mkldnn_primitives, rnn_desc.data.weights_iter_desc, weights_iter_index);
size_t bias_index = deps[4]; size_t bias_index = deps[4];
...@@ -1463,48 +955,6 @@ void MKLDNNEmitter::build_rnn_forward(std::vector<mkldnn::primitive*>& mkldnn_pr ...@@ -1463,48 +955,6 @@ void MKLDNNEmitter::build_rnn_forward(std::vector<mkldnn::primitive*>& mkldnn_pr
static_cast<mkldnn::memory>(*mkldnn_primitives[workspace_index])); static_cast<mkldnn::memory>(*mkldnn_primitives[workspace_index]));
} }
size_t MKLDNNEmitter::build_concat(const std::vector<mkldnn::memory::desc>& inputs_data_desc,
const mkldnn::memory::desc& result_desc,
const size_t concat_dim)
{
std::vector<mkldnn::memory::primitive::at> inputs_primitive;
std::vector<size_t> inputs_data_index;
std::vector<size_t> in_out_index;
std::vector<mkldnn::memory::primitive_desc> inputs_pd;
for (size_t i = 0; i < inputs_data_desc.size(); i++)
{
inputs_pd.push_back(mkldnn::memory::primitive_desc(
inputs_data_desc[i], runtime::cpu::executor::global_cpu_engine));
}
for (size_t i = 0; i < inputs_data_desc.size(); i++)
{
inputs_data_index.push_back(build_memory_primitive(inputs_data_desc[i]));
inputs_primitive.push_back(*m_mkldnn_primitives[inputs_data_index[i]]);
}
size_t result_index = build_memory_primitive(result_desc);
// concat primtive descriptor
mkldnn::concat::primitive_desc concat_pd =
mkldnn::concat::primitive_desc(result_desc, static_cast<int>(concat_dim), inputs_pd);
// concat primitive
size_t concat_index = insert_primitive(
new mkldnn::concat(concat_pd, inputs_primitive, *m_mkldnn_primitives[result_index]));
for (size_t i = 0; i < inputs_data_index.size(); i++)
{
in_out_index.push_back(inputs_data_index[i]);
}
in_out_index.push_back(result_index);
NGRAPH_CHECK(m_primitive_deps.find(concat_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[concat_index] = in_out_index;
return concat_index;
}
void MKLDNNEmitter::build_concat(std::vector<mkldnn::primitive*>& mkldnn_primitives, void MKLDNNEmitter::build_concat(std::vector<mkldnn::primitive*>& mkldnn_primitives,
const mkldnn::concat::primitive_desc& concat_pd, const mkldnn::concat::primitive_desc& concat_pd,
const std::vector<mkldnn::memory::desc>& inputs_data_desc, const std::vector<mkldnn::memory::desc>& inputs_data_desc,
...@@ -1534,41 +984,6 @@ void MKLDNNEmitter::build_concat(std::vector<mkldnn::primitive*>& mkldnn_primiti ...@@ -1534,41 +984,6 @@ void MKLDNNEmitter::build_concat(std::vector<mkldnn::primitive*>& mkldnn_primiti
new mkldnn::concat(concat_pd, inputs_primitive, *mkldnn_primitives[result_index]); new mkldnn::concat(concat_pd, inputs_primitive, *mkldnn_primitives[result_index]);
} }
size_t MKLDNNEmitter::build_slice(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc,
const ngraph::Coordinate& lower_bounds,
const ngraph::Shape& result_shape)
{
std::vector<size_t> in_out_index;
mkldnn::memory::primitive_desc input_pd =
mkldnn::memory::primitive_desc(input_desc, runtime::cpu::executor::global_cpu_engine);
size_t input_index = build_memory_primitive(input_desc);
auto dims = mkldnn::memory::dims(result_shape.begin(), result_shape.end());
auto offsets = mkldnn::memory::dims(lower_bounds.begin(), lower_bounds.end());
auto view_pd = mkldnn::view::primitive_desc(input_pd, dims, offsets).dst_primitive_desc();
mkldnn::memory::primitive_desc result_pd =
mkldnn::memory::primitive_desc(result_desc, runtime::cpu::executor::global_cpu_engine);
size_t result_index = build_memory_primitive(result_desc);
// reorder primitive descriptor
mkldnn::reorder::primitive_desc reorder_pd =
mkldnn::reorder::primitive_desc(view_pd, result_pd);
// reorder primitive
size_t reorder_index = insert_primitive(new mkldnn::reorder(
reorder_pd, *m_mkldnn_primitives[input_index], *m_mkldnn_primitives[result_index]));
NGRAPH_CHECK(m_primitive_deps.find(reorder_index) == m_primitive_deps.end(),
"Dependencies already created for node");
in_out_index.push_back(input_index);
in_out_index.push_back(result_index);
m_primitive_deps[reorder_index] = in_out_index;
return reorder_index;
}
void MKLDNNEmitter::build_slice(std::vector<mkldnn::primitive*>& mkldnn_primitives, void MKLDNNEmitter::build_slice(std::vector<mkldnn::primitive*>& mkldnn_primitives,
const mkldnn::memory::desc& input_desc, const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc, const mkldnn::memory::desc& result_desc,
...@@ -1600,26 +1015,6 @@ void MKLDNNEmitter::build_slice(std::vector<mkldnn::primitive*>& mkldnn_primitiv ...@@ -1600,26 +1015,6 @@ void MKLDNNEmitter::build_slice(std::vector<mkldnn::primitive*>& mkldnn_primitiv
reorder_pd, *mkldnn_primitives[input_index], *mkldnn_primitives[result_index]); reorder_pd, *mkldnn_primitives[input_index], *mkldnn_primitives[result_index]);
} }
size_t MKLDNNEmitter::build_softmax_forward(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc,
int softmax_axis)
{
size_t input_index = build_memory_primitive(input_desc);
size_t result_index = build_memory_primitive(result_desc);
size_t primitive_index = insert_primitive(
new mkldnn::softmax_forward({{mkldnn::prop_kind::forward_scoring, input_desc, softmax_axis},
executor::global_cpu_engine},
*m_mkldnn_primitives[input_index],
*m_mkldnn_primitives[result_index]));
NGRAPH_CHECK(m_primitive_deps.find(primitive_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[primitive_index] = {input_index, result_index};
return primitive_index;
}
mkldnn::softmax_forward::desc MKLDNNEmitter::get_softmax_forward_desc(const ngraph::Node* node) mkldnn::softmax_forward::desc MKLDNNEmitter::get_softmax_forward_desc(const ngraph::Node* node)
{ {
auto softmax = static_cast<const ngraph::op::Softmax*>(node); auto softmax = static_cast<const ngraph::op::Softmax*>(node);
...@@ -1653,30 +1048,6 @@ void MKLDNNEmitter::build_softmax_forward(std::vector<mkldnn::primitive*>& mkldn ...@@ -1653,30 +1048,6 @@ void MKLDNNEmitter::build_softmax_forward(std::vector<mkldnn::primitive*>& mkldn
*mkldnn_primitives[result_index]); *mkldnn_primitives[result_index]);
} }
size_t MKLDNNEmitter::build_leaky_relu(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc,
float alpha)
{
size_t input_index = build_memory_primitive(input_desc);
size_t result_index = build_memory_primitive(result_desc);
size_t primitive_index =
insert_primitive(new mkldnn::eltwise_forward({{mkldnn::prop_kind::forward_training,
mkldnn::algorithm::eltwise_relu,
input_desc,
alpha,
0.0f},
executor::global_cpu_engine},
*m_mkldnn_primitives[input_index],
*m_mkldnn_primitives[result_index]));
NGRAPH_CHECK(m_primitive_deps.find(primitive_index) == m_primitive_deps.end(),
"Dependencies already created for node");
m_primitive_deps[primitive_index] = {input_index, result_index};
return primitive_index;
}
mkldnn::eltwise_forward::desc MKLDNNEmitter::get_leaky_relu_desc(const ngraph::Node* node) mkldnn::eltwise_forward::desc MKLDNNEmitter::get_leaky_relu_desc(const ngraph::Node* node)
{ {
auto alpha = static_cast<const ngraph::op::LeakyRelu*>(node)->get_alpha(); auto alpha = static_cast<const ngraph::op::LeakyRelu*>(node)->get_alpha();
...@@ -1706,27 +1077,6 @@ void MKLDNNEmitter::build_leaky_relu(std::vector<mkldnn::primitive*>& mkldnn_pri ...@@ -1706,27 +1077,6 @@ void MKLDNNEmitter::build_leaky_relu(std::vector<mkldnn::primitive*>& mkldnn_pri
*mkldnn_primitives[result_index]); *mkldnn_primitives[result_index]);
} }
size_t MKLDNNEmitter::build_bounded_relu(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc,
float alpha)
{
size_t input_index = build_memory_primitive(input_desc);
size_t result_index = build_memory_primitive(result_desc);
size_t primitive_index =
insert_primitive(new mkldnn::eltwise_forward({{mkldnn::prop_kind::forward_training,
mkldnn::algorithm::eltwise_bounded_relu,
input_desc,
alpha,
0.0f},
executor::global_cpu_engine},
*m_mkldnn_primitives[input_index],
*m_mkldnn_primitives[result_index]));
m_primitive_deps[primitive_index] = {input_index, result_index};
return primitive_index;
}
mkldnn::eltwise_forward::desc MKLDNNEmitter::get_bounded_relu_desc(const ngraph::Node* node) mkldnn::eltwise_forward::desc MKLDNNEmitter::get_bounded_relu_desc(const ngraph::Node* node)
{ {
auto alpha = static_cast<const ngraph::op::BoundedRelu*>(node)->get_alpha(); auto alpha = static_cast<const ngraph::op::BoundedRelu*>(node)->get_alpha();
......
src/ngraph/runtime/cpu/mkldnn_emitter.hpp
View file @ 9335e41c
...@@ -33,6 +33,7 @@ ...@@ -33,6 +33,7 @@
#include "ngraph/op/concat.hpp" #include "ngraph/op/concat.hpp"
#include "ngraph/op/constant.hpp" #include "ngraph/op/constant.hpp"
#include "ngraph/op/convolution.hpp" #include "ngraph/op/convolution.hpp"
#include "ngraph/op/dequantize.hpp"
#include "ngraph/op/experimental/quantized_avg_pool.hpp" #include "ngraph/op/experimental/quantized_avg_pool.hpp"
#include "ngraph/op/experimental/quantized_conv.hpp" #include "ngraph/op/experimental/quantized_conv.hpp"
#include "ngraph/op/experimental/quantized_conv_bias.hpp" #include "ngraph/op/experimental/quantized_conv_bias.hpp"
...@@ -44,6 +45,7 @@ ...@@ -44,6 +45,7 @@
#include "ngraph/op/fused/group_conv.hpp" #include "ngraph/op/fused/group_conv.hpp"
#include "ngraph/op/lrn.hpp" #include "ngraph/op/lrn.hpp"
#include "ngraph/op/max_pool.hpp" #include "ngraph/op/max_pool.hpp"
#include "ngraph/op/quantize.hpp"
#include "ngraph/op/softmax.hpp" #include "ngraph/op/softmax.hpp"
#include "ngraph/runtime/cpu/cpu_executor.hpp" #include "ngraph/runtime/cpu/cpu_executor.hpp"
#include "ngraph/runtime/cpu/cpu_tensor_view_wrapper.hpp" #include "ngraph/runtime/cpu/cpu_tensor_view_wrapper.hpp"
...@@ -186,17 +188,6 @@ namespace ngraph ...@@ -186,17 +188,6 @@ namespace ngraph
size_t conv_index, size_t conv_index,
const mkldnn::memory::desc& weights_desc); const mkldnn::memory::desc& weights_desc);
size_t build_deconvolutionbias_forward(
const mkldnn::memory::desc& input_data_desc,
const mkldnn::memory::desc& weights_desc,
const mkldnn::memory::desc& bias_desc,
const mkldnn::memory::desc& result_desc,
const ngraph::Strides& strides,
const ngraph::Strides& dilation_strides,
const ngraph::CoordinateDiff& padding_below,
const ngraph::CoordinateDiff& padding_above,
const mkldnn::post_ops& pops = mkldnn::post_ops());
template <typename OP> template <typename OP>
size_t build_deconvolution(const ngraph::Node* node, size_t build_deconvolution(const ngraph::Node* node,
const std::vector<TensorViewWrapper>& args, const std::vector<TensorViewWrapper>& args,
...@@ -323,15 +314,6 @@ namespace ngraph ...@@ -323,15 +314,6 @@ namespace ngraph
const ngraph::CoordinateDiff& padding_below, const ngraph::CoordinateDiff& padding_below,
const ngraph::CoordinateDiff& padding_above); const ngraph::CoordinateDiff& padding_above);
size_t
build_convolution_backward_weights(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& delta_desc,
const mkldnn::memory::desc& result_desc,
const ngraph::Strides& strides,
const ngraph::Strides& dilation_strides,
const ngraph::CoordinateDiff& padding_below,
const ngraph::CoordinateDiff& padding_above);
void build_convolution_backward_weights( void build_convolution_backward_weights(
std::vector<mkldnn::primitive*>& mkldnn_primitives, std::vector<mkldnn::primitive*>& mkldnn_primitives,
const mkldnn::convolution_backward_weights::desc& bwd_desc, const mkldnn::convolution_backward_weights::desc& bwd_desc,
...@@ -339,14 +321,6 @@ namespace ngraph ...@@ -339,14 +321,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t conv_index); size_t conv_index);
size_t build_convolution_backward_data(const mkldnn::memory::desc& weights_desc,
const mkldnn::memory::desc& delta_desc,
const mkldnn::memory::desc& result_desc,
const ngraph::Strides& strides,
const ngraph::Strides& dilation_strides,
const ngraph::CoordinateDiff& padding_below,
const ngraph::CoordinateDiff& padding_above);
void build_convolution_backward_data( void build_convolution_backward_data(
std::vector<mkldnn::primitive*>& mkldnn_primitives, std::vector<mkldnn::primitive*>& mkldnn_primitives,
const mkldnn::convolution_backward_data::desc& bwd_desc, const mkldnn::convolution_backward_data::desc& bwd_desc,
...@@ -357,16 +331,6 @@ namespace ngraph ...@@ -357,16 +331,6 @@ namespace ngraph
/** /**
* Convolution + bias backprop for weights and bias * Convolution + bias backprop for weights and bias
*/ */
size_t build_convolution_backward_weights_bias(
const mkldnn::memory::desc& in_data_desc,
const mkldnn::memory::desc& in_delta_desc,
const mkldnn::memory::desc& out_weights_delta_desc,
const mkldnn::memory::desc& out_bias_delta_desc,
const ngraph::Strides& ng_strides,
const ngraph::Strides& ng_dilation_strides,
const ngraph::CoordinateDiff& ng_padding_below,
const ngraph::CoordinateDiff& ng_padding_above);
void build_convolution_backward_weights_bias( void build_convolution_backward_weights_bias(
std::vector<mkldnn::primitive*>& mkldnn_primitives, std::vector<mkldnn::primitive*>& mkldnn_primitives,
const mkldnn::convolution_backward_weights::desc& bwd_desc, const mkldnn::convolution_backward_weights::desc& bwd_desc,
...@@ -472,14 +436,6 @@ namespace ngraph ...@@ -472,14 +436,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t pool_index); size_t pool_index);
size_t build_pooling_backward(mkldnn::algorithm pooling_algorithm,
const mkldnn::memory::desc& diff_dst_desc,
const mkldnn::memory::desc& diff_src_desc,
const ngraph::Strides& window_strides,
const ngraph::Shape& window_shape,
const ngraph::Shape& padding_below,
const ngraph::Shape& padding_above);
template <typename OP> template <typename OP>
mkldnn::pooling_backward::desc mkldnn::pooling_backward::desc
get_avg_pooling_backward_desc(const ngraph::Node* node) get_avg_pooling_backward_desc(const ngraph::Node* node)
...@@ -515,14 +471,6 @@ namespace ngraph ...@@ -515,14 +471,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t pool_index); size_t pool_index);
size_t build_max_pooling_with_indices_forward(mkldnn::algorithm pooling_algorithm,
const mkldnn::memory::desc& src_desc,
const mkldnn::memory::desc& dst_desc,
const ngraph::Strides& window_strides,
const ngraph::Shape& window_shape,
const ngraph::Shape& padding_below,
const ngraph::Shape& padding_above);
template <typename OP> template <typename OP>
mkldnn::pooling_forward::desc mkldnn::pooling_forward::desc
get_max_pooling_with_indices_forward_desc(const ngraph::Node* node) get_max_pooling_with_indices_forward_desc(const ngraph::Node* node)
...@@ -555,15 +503,6 @@ namespace ngraph ...@@ -555,15 +503,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t max_pool_index); size_t max_pool_index);
size_t build_max_pooling_backward(mkldnn::algorithm pooling_algorithm,
const mkldnn::memory::desc& fprop_src_desc,
const mkldnn::memory::desc& diff_dst_desc,
const mkldnn::memory::desc& diff_src_desc,
const ngraph::Strides& window_strides,
const ngraph::Shape& window_shape,
const ngraph::Shape& padding_below,
const ngraph::Shape& padding_above);
template <typename OP> template <typename OP>
mkldnn::pooling_backward::desc mkldnn::pooling_backward::desc
get_max_pooling_backward_desc(const ngraph::Node* node) get_max_pooling_backward_desc(const ngraph::Node* node)
...@@ -599,15 +538,6 @@ namespace ngraph ...@@ -599,15 +538,6 @@ namespace ngraph
size_t fwd_pool_index, size_t fwd_pool_index,
size_t bwd_pool_index); size_t bwd_pool_index);
size_t build_max_pooling_with_indices_backward(
mkldnn::algorithm pooling_algorithm,
const mkldnn::memory::desc& diff_dst_desc,
const mkldnn::memory::desc& diff_src_desc,
const ngraph::Strides& window_strides,
const ngraph::Shape& window_shape,
const ngraph::Shape& padding_below,
const ngraph::Shape& padding_above);
void build_max_pooling_with_indices_backward( void build_max_pooling_with_indices_backward(
std::vector<mkldnn::primitive*>& mkldnn_primitives, std::vector<mkldnn::primitive*>& mkldnn_primitives,
const mkldnn::pooling_backward::desc& bwd_pool_desc, const mkldnn::pooling_backward::desc& bwd_pool_desc,
...@@ -631,9 +561,6 @@ namespace ngraph ...@@ -631,9 +561,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t lrn_index); size_t lrn_index);
size_t build_relu_forward(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc);
mkldnn::eltwise_forward::desc get_relu_forward_desc(const ngraph::Node* node); mkldnn::eltwise_forward::desc get_relu_forward_desc(const ngraph::Node* node);
void build_relu_forward(std::vector<mkldnn::primitive*>& mkldnn_primitives, void build_relu_forward(std::vector<mkldnn::primitive*>& mkldnn_primitives,
...@@ -641,10 +568,6 @@ namespace ngraph ...@@ -641,10 +568,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t relu_index); size_t relu_index);
size_t build_relu_backward(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& delta_desc,
const mkldnn::memory::desc& result_desc);
mkldnn::eltwise_backward::desc get_relu_backward_desc(const ngraph::Node* node); mkldnn::eltwise_backward::desc get_relu_backward_desc(const ngraph::Node* node);
void build_relu_backward(std::vector<mkldnn::primitive*>& mkldnn_primitives, void build_relu_backward(std::vector<mkldnn::primitive*>& mkldnn_primitives,
...@@ -653,9 +576,6 @@ namespace ngraph ...@@ -653,9 +576,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t relu_index); size_t relu_index);
size_t build_sigmoid_forward(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc);
mkldnn::eltwise_forward::desc get_sigmoid_forward_desc(const ngraph::Node* node, mkldnn::eltwise_forward::desc get_sigmoid_forward_desc(const ngraph::Node* node,
bool backward_op); bool backward_op);
...@@ -664,10 +584,6 @@ namespace ngraph ...@@ -664,10 +584,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t sigmoid_index); size_t sigmoid_index);
size_t build_sigmoid_backward(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& delta_desc,
const mkldnn::memory::desc& result_desc);
mkldnn::eltwise_backward::desc get_sigmoid_backward_desc(const ngraph::Node* node); mkldnn::eltwise_backward::desc get_sigmoid_backward_desc(const ngraph::Node* node);
void build_sigmoid_backward(std::vector<mkldnn::primitive*>& mkldnn_primitives, void build_sigmoid_backward(std::vector<mkldnn::primitive*>& mkldnn_primitives,
...@@ -737,10 +653,6 @@ namespace ngraph ...@@ -737,10 +653,6 @@ namespace ngraph
std::vector<size_t>& deps, std::vector<size_t>& deps,
size_t rnn_idx); size_t rnn_idx);
size_t build_concat(const std::vector<mkldnn::memory::desc>& inputs_data_desc,
const mkldnn::memory::desc& result_desc,
const size_t concat_dim);
template <typename OP> template <typename OP>
mkldnn::concat::primitive_desc get_concat_desc(const ngraph::Node* node, mkldnn::concat::primitive_desc get_concat_desc(const ngraph::Node* node,
size_t nargs) size_t nargs)
...@@ -769,11 +681,6 @@ namespace ngraph ...@@ -769,11 +681,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t concat_index); size_t concat_index);
size_t build_slice(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc,
const ngraph::Coordinate& lower_bounds,
const ngraph::Shape& result_shape);
void build_slice(std::vector<mkldnn::primitive*>& mkldnn_primitives, void build_slice(std::vector<mkldnn::primitive*>& mkldnn_primitives,
const mkldnn::memory::desc& input_desc, const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc, const mkldnn::memory::desc& result_desc,
...@@ -782,10 +689,6 @@ namespace ngraph ...@@ -782,10 +689,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t slice_index); size_t slice_index);
size_t build_softmax_forward(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc,
int softmax_axis);
mkldnn::softmax_forward::desc get_softmax_forward_desc(const ngraph::Node* node); mkldnn::softmax_forward::desc get_softmax_forward_desc(const ngraph::Node* node);
void build_softmax_forward(std::vector<mkldnn::primitive*>& mkldnn_primitives, void build_softmax_forward(std::vector<mkldnn::primitive*>& mkldnn_primitives,
...@@ -793,10 +696,6 @@ namespace ngraph ...@@ -793,10 +696,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t softmax_index); size_t softmax_index);
size_t build_leaky_relu(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc,
float alpha);
mkldnn::eltwise_forward::desc get_leaky_relu_desc(const ngraph::Node* node); mkldnn::eltwise_forward::desc get_leaky_relu_desc(const ngraph::Node* node);
void build_leaky_relu(std::vector<mkldnn::primitive*>& mkldnn_primitives, void build_leaky_relu(std::vector<mkldnn::primitive*>& mkldnn_primitives,
...@@ -804,10 +703,6 @@ namespace ngraph ...@@ -804,10 +703,6 @@ namespace ngraph
const std::vector<size_t>& deps, const std::vector<size_t>& deps,
size_t leaky_relu_index); size_t leaky_relu_index);
size_t build_bounded_relu(const mkldnn::memory::desc& input_desc,
const mkldnn::memory::desc& result_desc,
float alpha);
mkldnn::eltwise_forward::desc get_bounded_relu_desc(const ngraph::Node* node); mkldnn::eltwise_forward::desc get_bounded_relu_desc(const ngraph::Node* node);
void build_bounded_relu(std::vector<mkldnn::primitive*>& mkldnn_primitives, void build_bounded_relu(std::vector<mkldnn::primitive*>& mkldnn_primitives,
...@@ -835,9 +730,14 @@ namespace ngraph ...@@ -835,9 +730,14 @@ namespace ngraph
size_t get_scale_index() size_t get_scale_index()
{ {
size_t index = 0; size_t index = 0;
if (std::is_same<OP, ngraph::op::QuantizedConvolution>() || if (std::is_same<OP, ngraph::op::Quantize>() ||
std::is_same<OP, ngraph::op::QuantizedMatmul>() || std::is_same<OP, ngraph::op::Dequantize>())
std::is_same<OP, ngraph::op::QuantizedConvolutionRelu>()) {
index = 1;
}
else if (std::is_same<OP, ngraph::op::QuantizedConvolution>() ||
std::is_same<OP, ngraph::op::QuantizedMatmul>() ||
std::is_same<OP, ngraph::op::QuantizedConvolutionRelu>())
{ {
index = 2; index = 2;
} }
...@@ -1354,15 +1254,15 @@ namespace ngraph ...@@ -1354,15 +1254,15 @@ namespace ngraph
{ {
weights_desc.data.format = mkldnn_oidhw; weights_desc.data.format = mkldnn_oidhw;
} }
auto delta_desc = mkldnn_utils::get_input_mkldnn_md(node, 1); auto diff_dst_desc = mkldnn_utils::get_input_mkldnn_md(node, 1);
auto result_desc = mkldnn_utils::get_output_mkldnn_md(node, 0); auto diff_src_desc = mkldnn_utils::get_output_mkldnn_md(node, 0);
mkldnn::algorithm convolution_algo = mkldnn_utils::get_conv_algo(); mkldnn::algorithm convolution_algo = mkldnn_utils::get_conv_algo();
return mkldnn::convolution_backward_data::desc( return mkldnn::convolution_backward_data::desc(
convolution_algo, convolution_algo,
result_desc, diff_src_desc,
weights_desc, weights_desc,
delta_desc, diff_dst_desc,
MKLDNN_DIMS(convolution->get_window_movement_strides_forward()), MKLDNN_DIMS(convolution->get_window_movement_strides_forward()),
MKLDNN_DIMS(window_dilation_strides_adjusted), MKLDNN_DIMS(window_dilation_strides_adjusted),
MKLDNN_DIMS(convolution->get_padding_below_forward()), MKLDNN_DIMS(convolution->get_padding_below_forward()),
...@@ -1384,20 +1284,20 @@ namespace ngraph ...@@ -1384,20 +1284,20 @@ namespace ngraph
window_dilation_strides_adjusted.push_back(s - 1); window_dilation_strides_adjusted.push_back(s - 1);
} }
auto in_data_desc = mkldnn_utils::get_input_mkldnn_md(node, 0); auto src_desc = mkldnn_utils::get_input_mkldnn_md(node, 0);
auto in_delta_desc = mkldnn_utils::get_input_mkldnn_md(node, 1); auto diff_dst_desc = mkldnn_utils::get_input_mkldnn_md(node, 1);
auto out_weights_delta_desc = mkldnn_utils::get_output_mkldnn_md(node, 0); auto diff_weights_desc = mkldnn_utils::get_output_mkldnn_md(node, 0);
mkldnn::algorithm convolution_algo = mkldnn_utils::get_conv_algo(); mkldnn::algorithm convolution_algo = mkldnn_utils::get_conv_algo();
if (has_bias<OP>()) if (has_bias<OP>())
{ {
auto out_bias_delta_desc = mkldnn_utils::get_output_mkldnn_md(node, 1); auto diff_bias_desc = mkldnn_utils::get_output_mkldnn_md(node, 1);
return mkldnn::convolution_backward_weights::desc( return mkldnn::convolution_backward_weights::desc(
convolution_algo, convolution_algo,
in_data_desc, src_desc,
out_weights_delta_desc, diff_weights_desc,
out_bias_delta_desc, diff_bias_desc,
in_delta_desc, diff_dst_desc,
MKLDNN_DIMS(convolution->get_window_movement_strides_forward()), MKLDNN_DIMS(convolution->get_window_movement_strides_forward()),
MKLDNN_DIMS(window_dilation_strides_adjusted), MKLDNN_DIMS(window_dilation_strides_adjusted),
MKLDNN_DIMS(convolution->get_padding_below_forward()), MKLDNN_DIMS(convolution->get_padding_below_forward()),
...@@ -1408,9 +1308,9 @@ namespace ngraph ...@@ -1408,9 +1308,9 @@ namespace ngraph
{ {
return mkldnn::convolution_backward_weights::desc( return mkldnn::convolution_backward_weights::desc(
convolution_algo, convolution_algo,
in_data_desc, src_desc,
out_weights_delta_desc, diff_weights_desc,
in_delta_desc, diff_dst_desc,
MKLDNN_DIMS(convolution->get_window_movement_strides_forward()), MKLDNN_DIMS(convolution->get_window_movement_strides_forward()),
MKLDNN_DIMS(window_dilation_strides_adjusted), MKLDNN_DIMS(window_dilation_strides_adjusted),
MKLDNN_DIMS(convolution->get_padding_below_forward()), MKLDNN_DIMS(convolution->get_padding_below_forward()),
...@@ -1446,15 +1346,15 @@ namespace ngraph ...@@ -1446,15 +1346,15 @@ namespace ngraph
{ {
weights_desc.data.format = mkldnn_oidhw; weights_desc.data.format = mkldnn_oidhw;
} }
auto delta_desc = mkldnn_utils::get_input_mkldnn_md(node, 1); auto diff_dst_desc = mkldnn_utils::get_input_mkldnn_md(node, 1);
auto result_desc = mkldnn_utils::get_output_mkldnn_md(node, 0); auto diff_src_desc = mkldnn_utils::get_output_mkldnn_md(node, 0);
return mkldnn::convolution_forward::desc( return mkldnn::convolution_forward::desc(
mkldnn::prop_kind::forward, mkldnn::prop_kind::forward,
convolution_algo, convolution_algo,
result_desc, diff_src_desc,
weights_desc, weights_desc,
delta_desc, diff_dst_desc,
MKLDNN_DIMS(convolution->get_window_movement_strides_forward()), MKLDNN_DIMS(convolution->get_window_movement_strides_forward()),
MKLDNN_DIMS(window_dilation_strides_adjusted), MKLDNN_DIMS(window_dilation_strides_adjusted),
MKLDNN_DIMS(convolution->get_padding_below_forward()), MKLDNN_DIMS(convolution->get_padding_below_forward()),
...@@ -1463,15 +1363,15 @@ namespace ngraph ...@@ -1463,15 +1363,15 @@ namespace ngraph
} }
else if (std::is_same<OP, ngraph::op::ConvolutionBackpropFilters>()) else if (std::is_same<OP, ngraph::op::ConvolutionBackpropFilters>())
{ {
auto in_data_desc = mkldnn_utils::get_input_mkldnn_md(node, 0); auto src_desc = mkldnn_utils::get_input_mkldnn_md(node, 0);
auto in_delta_desc = mkldnn_utils::get_input_mkldnn_md(node, 1); auto diff_dst_desc = mkldnn_utils::get_input_mkldnn_md(node, 1);
auto out_weights_delta_desc = mkldnn_utils::get_output_mkldnn_md(node, 0); auto diff_weights_desc = mkldnn_utils::get_output_mkldnn_md(node, 0);
return mkldnn::convolution_forward::desc( return mkldnn::convolution_forward::desc(
mkldnn::prop_kind::forward, mkldnn::prop_kind::forward,
convolution_algo, convolution_algo,
in_data_desc, src_desc,
out_weights_delta_desc, diff_weights_desc,
in_delta_desc, diff_dst_desc,
MKLDNN_DIMS(convolution->get_window_movement_strides_forward()), MKLDNN_DIMS(convolution->get_window_movement_strides_forward()),
MKLDNN_DIMS(window_dilation_strides_adjusted), MKLDNN_DIMS(window_dilation_strides_adjusted),
MKLDNN_DIMS(convolution->get_padding_below_forward()), MKLDNN_DIMS(convolution->get_padding_below_forward()),
...@@ -1480,18 +1380,18 @@ namespace ngraph ...@@ -1480,18 +1380,18 @@ namespace ngraph
} }
else else
{ {
auto in_data_desc = mkldnn_utils::get_input_mkldnn_md(node, 0); auto src_desc = mkldnn_utils::get_input_mkldnn_md(node, 0);
auto in_delta_desc = mkldnn_utils::get_input_mkldnn_md(node, 1); auto diff_dst_desc = mkldnn_utils::get_input_mkldnn_md(node, 1);
auto out_weights_delta_desc = mkldnn_utils::get_output_mkldnn_md(node, 0); auto diff_weights_desc = mkldnn_utils::get_output_mkldnn_md(node, 0);
auto out_bias_delta_desc = mkldnn_utils::get_output_mkldnn_md(node, 1); auto diff_bias_desc = mkldnn_utils::get_output_mkldnn_md(node, 1);
return mkldnn::convolution_forward::desc( return mkldnn::convolution_forward::desc(
mkldnn::prop_kind::forward, mkldnn::prop_kind::forward,
convolution_algo, convolution_algo,
in_data_desc, src_desc,
out_weights_delta_desc, diff_weights_desc,
out_bias_delta_desc, diff_bias_desc,
in_delta_desc, diff_dst_desc,
MKLDNN_DIMS(convolution->get_window_movement_strides_forward()), MKLDNN_DIMS(convolution->get_window_movement_strides_forward()),
MKLDNN_DIMS(window_dilation_strides_adjusted), MKLDNN_DIMS(window_dilation_strides_adjusted),
MKLDNN_DIMS(convolution->get_padding_below_forward()), MKLDNN_DIMS(convolution->get_padding_below_forward()),
......
src/ngraph/runtime/cpu/op/max_pool_with_indices.hpp
View file @ 9335e41c
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include "ngraph/graph_util.hpp" #include "ngraph/graph_util.hpp"
#include "ngraph/op/op.hpp" #include "ngraph/op/op.hpp"
#include "ngraph/runtime/cpu/cpu_backend_visibility.h"
namespace ngraph namespace ngraph
{ {
...@@ -31,11 +32,11 @@ namespace ngraph ...@@ -31,11 +32,11 @@ namespace ngraph
class MaxPoolWithIndices : public Op class MaxPoolWithIndices : public Op
{ {
public: public:
MaxPoolWithIndices(const std::shared_ptr<Node>& arg, CPU_BACKEND_API MaxPoolWithIndices(const std::shared_ptr<Node>& arg,
const Shape& window_shape, const Shape& window_shape,
const Strides& window_movement_strides, const Strides& window_movement_strides,
const Shape& padding_below, const Shape& padding_below,
const Shape& padding_above); const Shape& padding_above);
virtual std::shared_ptr<Node> virtual std::shared_ptr<Node>
copy_with_new_args(const NodeVector& new_args) const override; copy_with_new_args(const NodeVector& new_args) const override;
...@@ -64,13 +65,13 @@ namespace ngraph ...@@ -64,13 +65,13 @@ namespace ngraph
class MaxPoolWithIndicesBackprop : public Op class MaxPoolWithIndicesBackprop : public Op
{ {
public: public:
MaxPoolWithIndicesBackprop(const std::shared_ptr<Node>& arg_forward, CPU_BACKEND_API MaxPoolWithIndicesBackprop(const std::shared_ptr<Node>& arg_forward,
const std::shared_ptr<Node>& delta, const std::shared_ptr<Node>& delta,
const std::shared_ptr<Node>& indices, const std::shared_ptr<Node>& indices,
const Shape& window_shape, const Shape& window_shape,
const Strides& window_movement_strides, const Strides& window_movement_strides,
const Shape& padding_below, const Shape& padding_below,
const Shape& padding_above); const Shape& padding_above);
virtual std::shared_ptr<Node> virtual std::shared_ptr<Node>
copy_with_new_args(const NodeVector& new_args) const override; copy_with_new_args(const NodeVector& new_args) const override;
......
src/ngraph/runtime/cpu/pass/cpu_mkldnn_primitive_build.cpp
View file @ 9335e41c
This source diff could not be displayed because it is too large. You can view the blob instead.
src/ngraph/runtime/cpu/pass/cpu_mkldnn_primitive_build.hpp
View file @ 9335e41c
...@@ -23,10 +23,6 @@ ...@@ -23,10 +23,6 @@
#include <typeindex> #include <typeindex>
#include <unordered_map> #include <unordered_map>
#define BUILD_PRIMITIVE_DECL(op_name) \
build_primitive<op_name>(ngraph::runtime::cpu::MKLDNNEmitter & mkldnn_emitter, \
ngraph::Node * node)
#define CONSTRUCT_PRIMITIVE_BUILD_STRING_DECL(op_name) \ #define CONSTRUCT_PRIMITIVE_BUILD_STRING_DECL(op_name) \
construct_primitive_build_string<op_name>(ngraph::runtime::cpu::MKLDNNEmitter & \ construct_primitive_build_string<op_name>(ngraph::runtime::cpu::MKLDNNEmitter & \
mkldnn_emitter, \ mkldnn_emitter, \
...@@ -53,11 +49,6 @@ namespace ngraph ...@@ -53,11 +49,6 @@ namespace ngraph
namespace pass namespace pass
{ {
using PrimitiveBuildFunction =
std::function<size_t(ngraph::runtime::cpu::MKLDNNEmitter&, ngraph::Node*)>;
using PrimitiveBuildOpMap =
std::unordered_map<std::type_index, PrimitiveBuildFunction>;
using PrimitiveBuildStringConstructFunction = using PrimitiveBuildStringConstructFunction =
std::function<void(ngraph::runtime::cpu::MKLDNNEmitter&, std::function<void(ngraph::runtime::cpu::MKLDNNEmitter&,
ngraph::Node*, ngraph::Node*,
...@@ -77,10 +68,6 @@ namespace ngraph ...@@ -77,10 +68,6 @@ namespace ngraph
ngraph::runtime::cpu::MKLDNNEmitter& m_mkldnn_emitter; ngraph::runtime::cpu::MKLDNNEmitter& m_mkldnn_emitter;
/// External map to store each node with mkldnn implementation and its mkldnn
/// associated primitive index.
std::unordered_map<const Node*, size_t>& m_node_primitive_idx_map;
/// External map to store each node with mkldnn implementation and its mkldnn /// External map to store each node with mkldnn implementation and its mkldnn
/// creation string, deps, and mkldnn primitive index. /// creation string, deps, and mkldnn primitive index.
std::map<const Node*, std::tuple<std::string, std::vector<size_t>, size_t>>& std::map<const Node*, std::tuple<std::string, std::vector<size_t>, size_t>>&
...@@ -90,12 +77,10 @@ namespace ngraph ...@@ -90,12 +77,10 @@ namespace ngraph
MKLDNNPrimitiveBuildPass( MKLDNNPrimitiveBuildPass(
std::string filename, std::string filename,
ngraph::runtime::cpu::MKLDNNEmitter& mkldnn_emitter, ngraph::runtime::cpu::MKLDNNEmitter& mkldnn_emitter,
std::unordered_map<const Node*, size_t>& node_primitive_idx_map,
std::map<const Node*, std::tuple<std::string, std::vector<size_t>, size_t>>& std::map<const Node*, std::tuple<std::string, std::vector<size_t>, size_t>>&
node_primitive_string_deps_index_map) node_primitive_string_deps_index_map)
: m_desc_filename(filename) : m_desc_filename(filename)
, m_mkldnn_emitter(mkldnn_emitter) , m_mkldnn_emitter(mkldnn_emitter)
, m_node_primitive_idx_map(node_primitive_idx_map)
, m_node_primitive_string_deps_index_map( , m_node_primitive_string_deps_index_map(
node_primitive_string_deps_index_map) node_primitive_string_deps_index_map)
{ {
...@@ -103,15 +88,6 @@ namespace ngraph ...@@ -103,15 +88,6 @@ namespace ngraph
bool run_on_call_graph(const std::list<std::shared_ptr<Node>>& nodes) override; bool run_on_call_graph(const std::list<std::shared_ptr<Node>>& nodes) override;
template <typename OP>
static size_t
build_primitive(ngraph::runtime::cpu::MKLDNNEmitter& mkldnn_emitter,
ngraph::Node* node)
{
throw std::runtime_error("Unimplemented op '" + node->description() +
"' in MKLDNNPrimitiveBuildPass");
}
template <typename OP> template <typename OP>
static void construct_primitive_build_string( static void construct_primitive_build_string(
ngraph::runtime::cpu::MKLDNNEmitter& mkldnn_emitter, ngraph::runtime::cpu::MKLDNNEmitter& mkldnn_emitter,
......
test/cpu_test.cpp
View file @ 9335e41c
...@@ -40,6 +40,7 @@ ...@@ -40,6 +40,7 @@
#include "ngraph/runtime/cpu/cpu_builder.hpp" #include "ngraph/runtime/cpu/cpu_builder.hpp"
#include "ngraph/runtime/cpu/mkldnn_utils.hpp" #include "ngraph/runtime/cpu/mkldnn_utils.hpp"
#include "ngraph/runtime/cpu/op/convert_layout.hpp" #include "ngraph/runtime/cpu/op/convert_layout.hpp"
#include "ngraph/runtime/cpu/op/max_pool_with_indices.hpp"
#include "ngraph/serializer.hpp" #include "ngraph/serializer.hpp"
#include "ngraph/util.hpp" #include "ngraph/util.hpp"
#include "util/all_close.hpp" #include "util/all_close.hpp"
...@@ -1347,3 +1348,384 @@ TEST(cpu_test, gauss_error_function_erf_int32) ...@@ -1347,3 +1348,384 @@ TEST(cpu_test, gauss_error_function_erf_int32)
auto expected_values = expected_result_nd_array.get_vector(); auto expected_values = expected_result_nd_array.get_vector();
ASSERT_EQ(result_values, expected_values); ASSERT_EQ(result_values, expected_values);
} }
TEST(cpu_test, max_pool_with_indices_2d_2channel_2image)
{
Shape shape_a{2, 2, 5, 5};
Shape window_shape{2, 3};
auto window_movement_strides = Strides{1, 1};
Shape padding_below{0, 0};
Shape padding_above{0, 0};
auto A = make_shared<op::Parameter>(element::f32, shape_a);
auto max_pool = make_shared<op::MaxPoolWithIndices>(
A, window_shape, window_movement_strides, padding_below, padding_above);
Shape shape_r{2, 2, 4, 3};
auto data = make_shared<op::Result>(make_shared<op::GetOutputElement>(max_pool, 0));
auto indices = make_shared<op::Result>(make_shared<op::GetOutputElement>(max_pool, 1));
auto f = make_shared<Function>(ResultVector{data, indices}, ParameterVector{A});
auto backend = runtime::Backend::create("CPU");
// Create some tensors for input/output
auto a = backend->create_tensor(element::f32, shape_a);
copy_data(a,
test::NDArray<float, 4>({{{{0, 1, 0, 2, 1}, // img 0 chan 0
{0, 3, 2, 0, 0},
{2, 0, 0, 0, 1},
{2, 0, 1, 1, 2},
{0, 2, 1, 0, 0}},
{{0, 0, 0, 2, 0}, // img 0 chan 1
{0, 2, 3, 0, 1},
{2, 0, 1, 0, 2},
{3, 1, 0, 0, 0},
{2, 0, 0, 0, 0}}},
{{{0, 2, 1, 1, 0}, // img 1 chan 0
{0, 0, 2, 0, 1},
{0, 0, 1, 2, 3},
{2, 0, 0, 3, 0},
{0, 0, 0, 0, 0}},
{{2, 1, 0, 0, 1}, // img 1 chan 1
{0, 2, 0, 0, 0},
{1, 1, 2, 0, 2},
{1, 1, 1, 0, 1},
{1, 0, 0, 0, 2}}}})
.get_vector());
auto result_data = backend->create_tensor(element::f32, shape_r);
auto result_indices = backend->create_tensor(element::i32, shape_r);
auto handle = backend->compile(f);
handle->call_with_validate({result_data, result_indices}, {a});
EXPECT_TRUE(test::all_close_f((test::NDArray<float, 4>({{{{3, 3, 2}, // img 0 chan 0
{3, 3, 2},
{2, 1, 2},
{2, 2, 2}},
{{3, 3, 3}, // img 0 chan 1
{3, 3, 3},
{3, 1, 2},
{3, 1, 0}}},
{{{2, 2, 2}, // img 1 chan 0
{2, 2, 3},
{2, 3, 3},
{2, 3, 3}},
{{2, 2, 1}, // img 1 chan 1
{2, 2, 2},
{2, 2, 2},
{1, 1, 2}}}})
.get_vector()),
read_vector<float>(result_data),
MIN_FLOAT_TOLERANCE_BITS));
EXPECT_TRUE(test::all_close((test::NDArray<int, 4>({{{{4, 3, 1}, // img 0 chan 0
{1, 0, 0},
{0, 4, 5},
{0, 3, 2}},
{{5, 4, 3}, // img 0 chan 1
{2, 1, 0},
{3, 1, 2},
{0, 0, 0}}},
{{{1, 0, 3}, // img 1 chan 0
{2, 1, 5},
{3, 5, 2},
{0, 2, 1}},
{{0, 3, 2}, // img 1 chan 1
{1, 0, 3},
{2, 1, 0},
{0, 0, 5}}}})
.get_vector()),
read_vector<int>(result_indices)));
}
TEST(cpu_test, max_pool_with_indices_bprop_2d_2channel_2image)
{
Shape shape_a{2, 2, 5, 5};
Shape window_shape{2, 3};
auto window_movement_strides = Strides{1, 1};
Shape padding_below{0, 0};
Shape padding_above{0, 0};
auto A = make_shared<op::Parameter>(element::f32, shape_a);
Shape shape_i{2, 2, 4, 3};
auto indices = make_shared<op::Parameter>(element::i32, shape_i);
auto delta = make_shared<op::Parameter>(element::f32, shape_i);
auto max_pool_bprop = make_shared<op::MaxPoolWithIndicesBackprop>(
A, delta, indices, window_shape, window_movement_strides, padding_below, padding_above);
auto f = make_shared<Function>(max_pool_bprop, ParameterVector{A, delta, indices});
auto backend = runtime::Backend::create("CPU");
// Create some tensors for input/output
auto a = backend->create_tensor(element::f32, shape_a);
copy_data(a,
test::NDArray<float, 4>({{{{0, 1, 0, 2, 1}, // img 0 chan 0
{0, 3, 2, 0, 0},
{2, 0, 0, 0, 1},
{2, 0, 1, 1, 2},
{0, 2, 1, 0, 0}},
{{0, 0, 0, 2, 0}, // img 0 chan 1
{0, 2, 3, 0, 1},
{2, 0, 1, 0, 2},
{3, 1, 0, 0, 0},
{2, 0, 0, 0, 0}}},
{{{0, 2, 1, 1, 0}, // img 1 chan 0
{0, 0, 2, 0, 1},
{0, 0, 1, 2, 3},
{2, 0, 0, 3, 0},
{0, 0, 0, 0, 0}},
{{2, 1, 0, 0, 1}, // img 1 chan 1
{0, 2, 0, 0, 0},
{1, 1, 2, 0, 2},
{1, 1, 1, 0, 1},
{1, 0, 0, 0, 2}}}})
.get_vector());
auto i = backend->create_tensor(element::i32, shape_i);
copy_data(i,
test::NDArray<int, 4>({{{{4, 3, 1}, // img 0 chan 0
{1, 0, 0},
{0, 4, 5},
{0, 3, 2}},
{{5, 4, 3}, // img 0 chan 1
{2, 1, 0},
{3, 1, 2},
{0, 0, 0}}},
{{{1, 0, 3}, // img 1 chan 0
{2, 1, 5},
{3, 5, 2},
{0, 2, 1}},
{{0, 3, 2}, // img 1 chan 1
{1, 0, 3},
{2, 1, 0},
{0, 0, 5}}}})
.get_vector());
auto d = backend->create_tensor(element::f32, shape_i);
copy_data(d,
test::NDArray<float, 4>({{{{0.3, 0.3, 0.2}, // img 0 chan 0
{0.3, 0.3, 0.2},
{0.2, 0.1, 0.2},
{0.2, 0.2, 0.2}},
{{0.3, 0.3, 0.3}, // img 0 chan 1
{0.3, 0.3, 0.3},
{0.3, 0.1, 0.2},
{0.3, 0.1, 0.4}}},
{{{0.2, 0.2, 0.2}, // img 1 chan 0
{0.2, 0.2, 0.3},
{0.2, 0.3, 0.3},
{0.2, 0.3, 0.3}},
{{0.2, 0.2, 0.1}, // img 1 chan 1
{0.2, 0.2, 0.2},
{0.2, 0.2, 0.2},
{0.1, 0.1, 0.2}}}})
.get_vector());
auto result = backend->create_tensor(element::f32, shape_a);
auto handle = backend->compile(f);
handle->call_with_validate({result}, {a, d, i});
EXPECT_TRUE(test::all_close_f((test::NDArray<float, 4>({{{{0, 0, 0, 0.2, 0}, // img 0 chan 0
{0, 1.2, 0.2, 0, 0},
{0.2, 0, 0, 0, 0},
{0.2, 0, 0.1, 0, 0.4},
{0, 0.2, 0, 0, 0}},
{{0, 0, 0, 0, 0}, // img 0 chan 1
{0, 0, 1.8, 0, 0},
{0, 0, 0.1, 0, 0.2},
{0.6, 0.1, 0.4, 0, 0},
{0, 0, 0, 0, 0}}},
{{{0, 0.4, 0, 0, 0}, // img 1 chan 0
{0, 0, 0.6, 0, 0},
{0, 0, 0, 0, 0.6},
{0.4, 0, 0, 0.9, 0},
{0, 0, 0, 0, 0}},
{{0.2, 0, 0, 0, 0.1}, // img 1 chan 1
{0, 0.6, 0, 0, 0},
{0, 0, 0.8, 0, 0},
{0.1, 0.1, 0, 0, 0},
{0, 0, 0, 0, 0.2}}}})
.get_vector()),
read_vector<float>(result),
MIN_FLOAT_TOLERANCE_BITS));
}
TEST(cpu_test, max_pool_bprop_2d_2channel_2image)
{
Shape shape_a{2, 2, 5, 5};
Shape window_shape{2, 3};
auto window_movement_strides = Strides{1, 1};
Shape padding_below{0, 0};
Shape padding_above{0, 0};
auto A = make_shared<op::Parameter>(element::f32, shape_a);
Shape shape_i{2, 2, 4, 3};
auto delta = make_shared<op::Parameter>(element::f32, shape_i);
auto max_pool_bprop = make_shared<op::MaxPoolBackprop>(
A, delta, window_shape, window_movement_strides, padding_below, padding_above);
auto f = make_shared<Function>(max_pool_bprop, ParameterVector{A, delta});
auto backend = runtime::Backend::create("CPU");
// Create some tensors for input/output
auto a = backend->create_tensor(element::f32, shape_a);
copy_data(a,
test::NDArray<float, 4>({{{{0, 1, 0, 2, 1}, // img 0 chan 0
{0, 3, 2, 0, 0},
{2, 0, 0, 0, 1},
{2, 0, 1, 1, 2},
{0, 2, 1, 0, 0}},
{{0, 0, 0, 2, 0}, // img 0 chan 1
{0, 2, 3, 0, 1},
{2, 0, 1, 0, 2},
{3, 1, 0, 0, 0},
{2, 0, 0, 0, 0}}},
{{{0, 2, 1, 1, 0}, // img 1 chan 0
{0, 0, 2, 0, 1},
{0, 0, 1, 2, 3},
{2, 0, 0, 3, 0},
{0, 0, 0, 0, 0}},
{{2, 1, 0, 0, 1}, // img 1 chan 1
{0, 2, 0, 0, 0},
{1, 1, 2, 0, 2},
{1, 1, 1, 0, 1},
{1, 0, 0, 0, 2}}}})
.get_vector());
auto d = backend->create_tensor(element::f32, shape_i);
copy_data(d,
test::NDArray<float, 4>({{{{0.3, 0.3, 0.2}, // img 0 chan 0
{0.3, 0.3, 0.2},
{0.2, 0.1, 0.2},
{0.2, 0.2, 0.2}},
{{0.3, 0.3, 0.3}, // img 0 chan 1
{0.3, 0.3, 0.3},
{0.3, 0.1, 0.2},
{0.3, 0.1, 0.4}}},
{{{0.2, 0.2, 0.2}, // img 1 chan 0
{0.2, 0.2, 0.3},
{0.2, 0.3, 0.3},
{0.2, 0.3, 0.3}},
{{0.2, 0.2, 0.1}, // img 1 chan 1
{0.2, 0.2, 0.2},
{0.2, 0.2, 0.2},
{0.1, 0.1, 0.2}}}})
.get_vector());
auto result = backend->create_tensor(element::f32, shape_a);
auto handle = backend->compile(f);
handle->call_with_validate({result}, {a, d});
EXPECT_TRUE(test::all_close_f((test::NDArray<float, 4>({{{{0, 0, 0, 0.2, 0}, // img 0 chan 0
{0, 1.2, 0.2, 0, 0},
{0.2, 0, 0, 0, 0},
{0.2, 0, 0.1, 0, 0.4},
{0, 0.2, 0, 0, 0}},
{{0, 0, 0, 0, 0}, // img 0 chan 1
{0, 0, 1.8, 0, 0},
{0, 0, 0.1, 0, 0.2},
{0.6, 0.1, 0.4, 0, 0},
{0, 0, 0, 0, 0}}},
{{{0, 0.4, 0, 0, 0}, // img 1 chan 0
{0, 0, 0.6, 0, 0},
{0, 0, 0, 0, 0.6},
{0.4, 0, 0, 0.9, 0},
{0, 0, 0, 0, 0}},
{{0.2, 0, 0, 0, 0.1}, // img 1 chan 1
{0, 0.6, 0, 0, 0},
{0, 0, 0.8, 0, 0},
{0.1, 0.1, 0, 0, 0},
{0, 0, 0, 0, 0.2}}}})
.get_vector()),
read_vector<float>(result),
MIN_FLOAT_TOLERANCE_BITS));
}
TEST(cpu_test, avg_pool_bprop_2d_2channel_2image)
{
Shape shape_a{2, 2, 3, 3};
Shape window_shape{2, 2};
auto window_movement_strides = Strides{1, 1};
Shape padding_below{0, 0};
Shape padding_above{0, 0};
Shape shape_d{2, 2, 2, 2};
auto delta = make_shared<op::Parameter>(element::f32, shape_d);
auto avg_pool_bprop = make_shared<op::AvgPoolBackprop>(
shape_a, delta, window_shape, window_movement_strides, padding_below, padding_above, false);
auto f = make_shared<Function>(avg_pool_bprop, ParameterVector{delta});
auto backend = runtime::Backend::create("CPU");
// Create some tensors for input/output
auto d = backend->create_tensor(element::f32, shape_d);
copy_data(d,
test::NDArray<float, 4>({{{{0.3, 0.3}, // img 0 chan 0
{0.3, 0.3}},
{{0.2, 0.2}, // img 0 chan 1
{0.2, 0.2}}},
{{{0.1, 0.1}, // img 1 chan 0
{0.1, 0.1}},
{{0.4, 0.4}, // img 1 chan 1
{0.4, 0.4}}}})
.get_vector());
auto result = backend->create_tensor(element::f32, shape_a);
float denom = 2 * 2;
auto handle = backend->compile(f);
handle->call_with_validate({result}, {d});
EXPECT_TRUE(test::all_close_f(
(test::NDArray<float, 4>({{{{0.3f / denom, 0.6f / denom, 0.3f / denom}, // img 0 chan 0
{0.6f / denom, 1.2f / denom, 0.6f / denom},
{0.3f / denom, 0.6f / denom, 0.3f / denom}},
{{0.2f / denom, 0.4f / denom, 0.2f / denom}, // img 0 chan 1
{0.4f / denom, 0.8f / denom, 0.4f / denom},
{0.2f / denom, 0.4f / denom, 0.2f / denom}}},
{{{0.1f / denom, 0.2f / denom, 0.1f / denom}, // img 1 chan 0
{0.2f / denom, 0.4f / denom, 0.2f / denom},
{0.1f / denom, 0.2f / denom, 0.1f / denom}},
{{0.4f / denom, 0.8f / denom, 0.4f / denom}, // img 1 chan 1
{0.8f / denom, 1.6f / denom, 0.8f / denom},
{0.4f / denom, 0.8f / denom, 0.4f / denom}}}})
.get_vector()),
read_vector<float>(result),
MIN_FLOAT_TOLERANCE_BITS));
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment