Dynamically create and destroy mkldnn engine objects to work around function outlining

src/ngraph/runtime/cpu/cpu_emitter.cpp

@@ -90,8 +90,7 @@ static const string& get_mkldnn_data_type(const string& type)
 
 void runtime::cpu::CPU_Emitter::EmitMKLDNNPreamble(codegen::CodeWriter& writer)
 {
-    writer << "using namespace mkldnn;\n";
-    writer << "auto cpu_engine = engine(engine::cpu, 0);\n";
+    writer << "using namespace mkldnn;\n\n";
 }
 
 void runtime::cpu::CPU_Emitter::EmitNop(codegen::CodeWriter& writer,
@@ -1922,6 +1921,7 @@ void runtime::cpu::CPU_Emitter::EmitConvolution(codegen::CodeWriter& writer,
         writer << "{\n";
         writer.indent++;
 
+        writer << "auto cpu_engine = engine(engine::cpu, 0);\n";
         writer << "auto input_data_desc = memory::desc({" << join(arg0_shape) << "}, " << et
                << ", memory::format::nchw);\n";
         writer << "auto weights_desc = memory::desc({" << join(arg1_shape) << "}, " << et
@@ -1964,6 +1964,7 @@ void runtime::cpu::CPU_Emitter::EmitConvolution(codegen::CodeWriter& writer,
         writer << "{\n";
         writer.indent++;
 
+        writer << "auto cpu_engine = engine(engine::cpu, 0);\n";
         writer << "auto input_data_desc = memory::desc({" << join(arg0_shape) << "}, " << et
                << ", memory::format::nchw);\n";
         writer << "auto weights_desc = memory::desc({" << join(arg1_shape) << "}, " << et
@@ -2107,6 +2108,7 @@ void runtime::cpu::CPU_Emitter::EmitMaxPool(codegen::CodeWriter& writer,
         writer << "{\n";
         writer.indent++;
 
+        writer << "auto cpu_engine = engine(engine::cpu, 0);\n";
         writer << "auto input_data_desc = memory::desc({" << join(arg_shape) << "}, " << et
                << ", memory::format::nchw);\n";
         writer << "auto result_desc = memory::desc({" << join(result_shape) << "}, " << et
@@ -2284,6 +2286,7 @@ void runtime::cpu::CPU_Emitter::EmitAvgPool(codegen::CodeWriter& writer,
         writer << "{\n";
         writer.indent++;
 
+        writer << "auto cpu_engine = engine(engine::cpu, 0);\n";
         writer << "auto input_data_desc = memory::desc({" << join(arg_shape) << "}, " << et
                << ", memory::format::nchw);\n";
         writer << "auto result_desc = memory::desc({" << join(result_shape) << "}, " << et

src/ngraph/runtime/cpu/cpu_external_function.cpp

@@ -364,6 +364,8 @@ using namespace ngraph::runtime;
         }
     }
 
+    runtime::cpu::CPU_Emitter::EmitMKLDNNPreamble(writer);
+
     writer << "// Declare all functions\n";
     for (shared_ptr<Function> f : pass_manager.get_state().get_functions())
     {
@@ -456,8 +458,6 @@ using namespace ngraph::runtime;
             writer << "tbb::flow::graph G;\n\n";
         }
 
-        runtime::cpu::CPU_Emitter::EmitMKLDNNPreamble(writer);
-
         bool temporaries_used = false;
         size_t worst_case_tmp_size = 0;
         for (shared_ptr<Node> node : current_function->get_ordered_ops())

test/backend_test.in.cpp

@@ -5619,6 +5619,39 @@ TEST(${BACKEND_NAME}, zero_sized_subtract)
     make_binary_empty_test<op::Subtract>("${BACKEND_NAME}");
 }
 
+TEST (${BACKEND_NAME}, convolution_outlining)
+{
+    auto shape_a = Shape{1,2,2,2};
+    auto A = make_shared<op::Parameter>(element::f32, shape_a);
+    auto shape_b = Shape{2,2,1,1};
+    auto B = make_shared<op::Parameter>(element::f32, shape_b);
+    auto shape_r = Shape{1,2,2,2};
+    auto conv1 = make_shared<op::Convolution>(A, B, Strides{1,1}, Strides{1, 1},
+                                              CoordinateDiff{0,0}, CoordinateDiff{0,0}, 
+                                              Strides{1,1});
+    auto conv2 = make_shared<op::Convolution>(conv1, B, Strides{1,1}, Strides{1, 1},
+                                              CoordinateDiff{0,0}, CoordinateDiff{0,0}, 
+                                              Strides{1,1});
+    auto f = make_shared<Function>(conv2, op::Parameters{A,B}); 
+
+    auto manager = runtime::Manager::get("${BACKEND_NAME}");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::f32, shape_a);
+    copy_data(a, vector<float>{1.0f,1.0f,1.0f,1.0f,1.0f,1.0f,1.0f,1.0f});
+    auto b = backend->make_primary_tensor_view(element::f32, shape_b);
+    copy_data(b, vector<float>{1.0f,1.0f,1.0f,1.0f});
+    auto result = backend->make_primary_tensor_view(element::f32, shape_r);
+
+    vector<float> expected_result{4.0f,4.0f,4.0f,4.0f,4.0f,4.0f,4.0f,4.0f};
+
+    cf->call({a, b}, {result});
+    EXPECT_EQ(vector<float>{expected_result}, read_vector<float>(result));
+}
+
 TEST(${BACKEND_NAME}, avg_pool_1d_1channel_1image)
 {
     auto shape_a = Shape{1, 1, 14};