Updated nGraph backend unit tests to be compatible with OpenVINO plugins (#3756)

* Updated nGraph backend to be compatible with OpenVINO plugins

* Resolve issue with gtest

* Resolve comments in PR

* Resolve comments in PR

* Code style fix

* Rename runtime2 to ov_runtime

* Style check fix

test/CMakeLists.txt

@@ -187,36 +187,51 @@ endif()
 set_source_files_properties(includes.cpp PROPERTIES COMPILE_DEFINITIONS
     NGRAPH_INCLUDES="${PROJECT_SOURCE_DIR}/src/ngraph")
 
-if (NGRAPH_INTERPRETER_ENABLE)
-    list(APPEND SRC
-        backend_debug_api.cpp
-        builder.cpp
-        backend_api.cpp)
-    set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} INTERPRETER)
-endif()
+if (NGRAPH_UNIT_TEST_OPENVINO_ENABLE)
+    message(STATUS "NGRAPH_UNIT_TEST_OPENVINO_ENABLE = TRUE")
+    add_definitions(-DNGRAPH_UNIT_TEST_OPENVINO_ENABLE)
 
-if (NGRAPH_CPU_ENABLE)
-    list(APPEND SRC core_fusion.cpp builder_quantization.cpp)
-    list(APPEND SRC backend_performance.cpp cpu_fusion.cpp cpu_test.cpp cpu_debugger.cpp cpu_debug_tracer.cpp)
-    if (NOT NGRAPH_DEX_ONLY)
-        list(APPEND SRC cpu_codegen.cpp)
+    if (ENABLE_MKL_DNN)
+        message(STATUS "NGRAPH_TESTS: CPU enabled")
+        set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} CPU)
     endif()
-    if (NGRAPH_HALIDE)
-        list(APPEND SRC halide.cpp)
+
+    if (ENABLE_CLDNN)
+        message(STATUS "NGRAPH_TESTS: GPU enabled")
+        set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} GPU)
+    endif()
+else()
+    if (NGRAPH_INTERPRETER_ENABLE)
+        list(APPEND SRC
+            backend_debug_api.cpp
+            builder.cpp
+            backend_api.cpp)
+        set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} INTERPRETER)
     endif()
-    set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} CPU)
-endif()
 
-if (NGRAPH_GPUH_ENABLE)
-    set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} GPUH)
-endif()
+    if (NGRAPH_CPU_ENABLE)
+        list(APPEND SRC core_fusion.cpp builder_quantization.cpp)
+        list(APPEND SRC backend_performance.cpp cpu_fusion.cpp cpu_test.cpp cpu_debugger.cpp cpu_debug_tracer.cpp)
+        if (NOT NGRAPH_DEX_ONLY)
+            list(APPEND SRC cpu_codegen.cpp)
+        endif()
+        if (NGRAPH_HALIDE)
+            list(APPEND SRC halide.cpp)
+        endif()
+        set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} CPU)
+    endif()
 
-if (NGRAPH_PLAIDML_ENABLE)
-    set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} PlaidML)
-endif()
+    if (NGRAPH_GPUH_ENABLE)
+        set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} GPUH)
+    endif()
+
+    if (NGRAPH_PLAIDML_ENABLE)
+        set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} PlaidML)
+    endif()
 
-if (NGRAPH_GENERIC_CPU_ENABLE)
-    set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} GCPU)
+    if (NGRAPH_GENERIC_CPU_ENABLE)
+        set(ACTIVE_BACKEND_LIST ${ACTIVE_BACKEND_LIST} GCPU)
+    endif()
 endif()
 
 add_definitions("-DTEST_FILES=\"${CMAKE_CURRENT_SOURCE_DIR}/files\"")
@@ -230,6 +245,7 @@ add_subdirectory(util)
 # 5) Use the `NGRAPH_TEST` macro in place of `TEST`.
 # All such files are configured via cmake which replaces all instances of cmake variables
 # such as ${BACKEND_NAME} with their values, such as CPU, GPU, or INTERPRETER.
+
 set(MULTI_TEST_SRC
     backend/abc.in.cpp
     backend/abs.in.cpp
@@ -329,6 +345,116 @@ set(MULTI_TEST_SRC
     backend/zero_sized.in.cpp
 )
 
+if (NGRAPH_UNIT_TEST_OPENVINO_ENABLE)
+    set(MULTI_TEST_SRC
+            backend/abc.in.cpp
+            backend/abs.in.cpp
+            backend/acos.in.cpp
+            backend/add.in.cpp
+            backend/arg_reduce.in.cpp
+            backend/asin.in.cpp
+            backend/atan.in.cpp
+            backend/auto_broadcast.in.cpp
+            backend/batch_mat_mul.in.cpp
+            backend/builder_flatten.in.cpp
+            backend/comparison.in.cpp
+            backend/computation_reuse.in.cpp
+            backend/concat.in.cpp
+            backend/convert.in.cpp
+            backend/convolution.in.cpp
+            backend/cos.in.cpp
+            backend/cosh.in.cpp
+            backend/divide.in.cpp
+            backend/dyn_broadcast.in.cpp
+            backend/dyn_replace_slice_reference.in.cpp
+            backend/dyn_reshape.in.cpp
+            backend/dyn_slice_reference.in.cpp
+            backend/dynamic.in.cpp
+            backend/embedding_lookup.in.cpp
+            backend/erf.in.cpp
+            backend/exp.in.cpp
+            backend/floor.in.cpp
+            backend/function_name.in.cpp
+            backend/fused_op.in.cpp
+            backend/gather.in.cpp
+            backend/layer_norm.in.cpp
+            backend/log.in.cpp
+            backend/logical_and.in.cpp
+            backend/logical_or.in.cpp
+            backend/logical_xor.in.cpp
+            backend/lrn.in.cpp
+            backend/maximum.in.cpp
+            backend/minimum.in.cpp
+            backend/multiple_backends.in.cpp
+            backend/multiple_result.in.cpp
+            backend/multiply.in.cpp
+            backend/negative.in.cpp
+            backend/node_name.in.cpp
+            backend/not.in.cpp
+            backend/numeric.in.cpp
+            backend/one_hot.in.cpp
+            backend/pad.in.cpp
+            backend/pool.in.cpp
+            backend/power.in.cpp
+            backend/quantize_dequantize.in.cpp
+            backend/quantized_convolution.in.cpp
+            backend/quantized_dot.in.cpp
+            backend/random_uniform.in.cpp
+            backend/range.in.cpp
+            backend/relu.in.cpp
+            backend/reshape.in.cpp
+            backend/reverse_sequence.in.cpp
+            backend/reverse.in.cpp
+            backend/scatter.in.cpp
+            backend/select.in.cpp
+            backend/shape_of.in.cpp
+            backend/sigmoid.in.cpp
+            backend/sign.in.cpp
+            backend/sin.in.cpp
+            backend/sinh.in.cpp
+            backend/slice.in.cpp
+            backend/softmax.in.cpp
+            backend/sqrt.in.cpp
+            backend/subtract.in.cpp
+            backend/tan.in.cpp
+            backend/tanh.in.cpp
+            backend/tensorview_custom_mem.in.cpp
+            backend/transpose.in.cpp
+            backend/validate_call.in.cpp
+            backend/zero_sized.in.cpp
+
+#            This files should be commented,because they are not working for a while.
+#            Not v1 op:
+#            backend/all.in.cpp
+#            backend/any.in.cpp
+#            backend/broadcast.in.cpp
+#            backend/ceiling.in.cpp
+#            backend/dot.in.cpp
+#            backend/generate_mask.in.cpp
+#            backend/max.in.cpp
+#            backend/min.in.cpp
+#            backend/product.in.cpp
+#            backend/replace_slice.in.cpp
+#            backend/sum.in.cpp
+#            backend/unhandled_op.in.cpp
+
+#            Need to fix for IE plugin:
+#            backend/api.in.cpp
+#            backend/batch_norm.in.cpp
+#            backend/gelu.in.cpp
+#            backend/topk.in.cpp
+
+#            Other issues:
+#            backend/autodiff.in.cpp
+#            backend/constant.in.cpp
+#            backend/convolution_reference.in.cpp
+
+#            Segmentation fault. Need to fix on IE side:
+#            backend/parameter_as_output.in.cpp
+#            backend/aliased_output.in.cpp
+            )
+endif()
+
 if (NGRAPH_MLIR_ENABLE)
     list(APPEND MULTI_TEST_SRC backend/mlir.in.cpp)
 endif()
@@ -389,7 +515,12 @@ if(NGRAPH_ADDRESS_SANITIZER)
 endif()
 
 target_link_libraries(unit-test PRIVATE ngraph_test_util)
-target_link_libraries(unit-test PRIVATE ngraph libgtest)
+if(NGRAPH_UNIT_TEST_OPENVINO_ENABLE)
+    target_link_libraries(unit-test PRIVATE ngraph gtest)
+else()
+    target_link_libraries(unit-test PRIVATE ngraph libgtest)
+endif()
+
 target_compile_definitions(unit-test PRIVATE NGRAPH_VERSION_LABEL="${NGRAPH_VERSION_LABEL}")
 if (NGRAPH_JSON_ENABLE)
     target_link_libraries(unit-test PRIVATE libjson)

test/util/CMakeLists.txt

@@ -21,6 +21,7 @@ set (SRC
     test_tools.cpp
     test_control.cpp
     test_case.cpp
+    backend_utils.cpp
 )
 
 add_library(ngraph_test_util STATIC ${SRC})
@@ -29,6 +30,12 @@ if(NGRAPH_LIB_VERSIONING_ENABLE)
         VERSION ${NGRAPH_VERSION})
 endif()
 target_include_directories(ngraph_test_util PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/..)
-target_link_libraries(ngraph_test_util ngraph libgtest)
+target_link_libraries(ngraph_test_util PRIVATE ngraph libgtest)
+
+if (NGRAPH_UNIT_TEST_OPENVINO_ENABLE)
+    add_definitions(-DNGRAPH_UNIT_TEST_OPENVINO_ENABLE)
+    target_include_directories(ngraph_test_util PUBLIC ${IE_MAIN_SOURCE_DIR}/include)
+    target_link_libraries(ngraph_test_util PRIVATE inference_engine)
+endif()
 
 install(TARGETS ngraph_test_util DESTINATION ${NGRAPH_INSTALL_LIB})

test/util/backend_utils.cpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#ifdef NGRAPH_UNIT_TEST_OPENVINO_ENABLE
+#include "backend_utils.hpp"
+#include <ie_core.hpp>
+#include "ngraph/ngraph.hpp"
+
+using namespace std;
+using namespace ngraph;
+using namespace InferenceEngine;
+
+Blob::Ptr fill_blob(SizeVector shape, std::vector<float> data)
+{
+    Layout layout;
+    switch (shape.size())
+    {
+    case 1: layout = Layout::C; break;
+    case 2: layout = Layout::NC; break;
+    case 3: layout = Layout::CHW; break;
+    case 4: layout = Layout::NCHW; break;
+    case 5: layout = Layout::NCDHW; break;
+    default: THROW_IE_EXCEPTION << "Can't convert dims " << shape.size() << " to Layout!";
+    }
+    Blob::Ptr blob(new TBlob<float>({Precision::FP32, shape, layout}));
+    blob->allocate();
+    float* blob_ptr = (float*)(void*)blob->buffer();
+    for (int i = 0; i < data.size(); i++)
+    {
+        blob_ptr[i] = data[i];
+    }
+    return blob;
+}
+#endif
\ No newline at end of file

test/util/backend_utils.hpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#pragma once
+
+#ifdef NGRAPH_UNIT_TEST_OPENVINO_ENABLE
+#include <ie_core.hpp>
+#include <string>
+#include "ngraph/ngraph.hpp"
+
+using namespace std;
+using namespace ngraph;
+using namespace InferenceEngine;
+
+Blob::Ptr fill_blob(SizeVector shape, std::vector<float> data);
+
+class Handle;
+
+namespace ov_runtime
+{
+    class Backend;
+    class Tensor
+    {
+    public:
+        std::vector<float> data;
+        PartialShape shape;
+        ngraph::element::Type type;
+
+        Shape get_shape() { return shape.to_shape(); }
+        PartialShape get_partial_shape() { return shape; }
+        explicit Tensor(ngraph::element::Type type, PartialShape ps)
+            : type(type)
+            , shape(ps)
+        {
+        }
+        explicit Tensor(ngraph::element::Type type, Shape ps)
+            : type(type)
+            , shape(ps)
+        {
+        }
+
+        const element::Type& get_element_type() const { return type; }
+        size_t get_element_count() { return shape_size(get_shape()); }
+        void set_stale(bool flag) {}
+        void copy_from(ov_runtime::Tensor t)
+        {
+            data = t.data;
+            shape = t.shape;
+            type = t.type;
+        }
+    };
+}
+
+class Executable
+{
+private:
+    CNNNetwork network;
+    std::string device;
+
+public:
+    Executable(std::shared_ptr<Function> func, std::string _device)
+    {
+        network = CNNNetwork(func);
+        device = _device;
+    }
+
+    bool call_with_validate(const vector<shared_ptr<ov_runtime::Tensor>>& outputs,
+                            const vector<shared_ptr<ov_runtime::Tensor>>& inputs)
+    {
+        Core ie;
+
+        //  Loading model to the plugin (BACKEND_NAME)
+        ExecutableNetwork exeNetwork = ie.LoadNetwork(network, device);
+        //  Create infer request
+        InferRequest inferRequest = exeNetwork.CreateInferRequest();
+        //  Prepare input and output blobs
+        InputsDataMap inputInfo = network.getInputsInfo();
+
+        if (inputInfo.size() != inputs.size())
+        {
+            THROW_IE_EXCEPTION << "Function inputs number differ from number of given inputs";
+        }
+
+        size_t i = 0;
+        for (auto& it : inputInfo)
+        {
+            inferRequest.SetBlob(
+                it.first, fill_blob(it.second->getTensorDesc().getDims(), inputs[i++]->data));
+        }
+
+        //  Prepare output blobs
+        std::string output_name = network.getOutputsInfo().begin()->first;
+
+        inferRequest.Infer();
+        Blob::Ptr output = inferRequest.GetBlob(output_name);
+
+        float* output_ptr = output->buffer().as<float*>();
+        // TODO: how to get size without explicit calculation?
+        size_t size = 1;
+        for (const auto& dim : output->getTensorDesc().getDims())
+        {
+            size *= dim;
+        }
+        //  Vector initialization from pointer
+        std::vector<float> result(output_ptr, output_ptr + size);
+        outputs[0]->data = result;
+        return true;
+    }
+};
+
+template <class T>
+void copy_data(std::shared_ptr<ov_runtime::Tensor> t, const std::vector<T>& data);
+
+class ov_runtime::Backend
+{
+private:
+    string device;
+
+public:
+    static std::shared_ptr<ov_runtime::Backend> create(std::string device,
+                                                       bool must_support_dynamic = false)
+    {
+        return std::shared_ptr<Backend>(new Backend(device));
+    }
+
+    Backend(std::string _device)
+        : device(_device)
+    {
+    }
+
+    std::shared_ptr<ov_runtime::Tensor> create_tensor(ngraph::element::Type type,
+                                                      ngraph::Shape shape)
+    {
+        return std::shared_ptr<ov_runtime::Tensor>(new ov_runtime::Tensor(type, shape));
+    }
+
+    template <typename T>
+    std::shared_ptr<ov_runtime::Tensor>
+        create_tensor(ngraph::element::Type type, ngraph::Shape shape, T* data)
+    {
+        auto tensor = std::shared_ptr<ov_runtime::Tensor>(new ov_runtime::Tensor(type, shape));
+        size_t size = 1;
+        for (auto x : shape)
+        {
+            size *= x;
+        }
+        vector<T> v(data, data + size);
+        copy_data(tensor, v);
+        return tensor;
+    }
+
+    template <class T>
+    std::shared_ptr<ov_runtime::Tensor> create_tensor(ngraph::Shape shape)
+    {
+        return std::shared_ptr<ov_runtime::Tensor>(
+            new ov_runtime::Tensor(ngraph::element::from<T>(), shape));
+    }
+
+    std::shared_ptr<ov_runtime::Tensor> create_dynamic_tensor(ngraph::element::Type type,
+                                                              ngraph::PartialShape shape)
+    {
+        return std::shared_ptr<ov_runtime::Tensor>(new ov_runtime::Tensor(type, shape));
+    }
+
+    bool supports_dynamic_tensors() { return true; }
+    std::shared_ptr<Executable> compile(std::shared_ptr<Function> func)
+    {
+        return std::shared_ptr<Executable>(new Executable(func, device));
+    }
+};
+
+template <class T>
+std::vector<T> read_vector(std::shared_ptr<ov_runtime::Tensor> tv)
+{
+    std::vector<T> v(tv->data.size());
+    for (size_t i = 0; i < v.size(); ++i)
+    {
+        v[i] = tv->data[i];
+    }
+    return v;
+}
+
+template <class T>
+void copy_data(std::shared_ptr<ov_runtime::Tensor> t, const std::vector<T>& data)
+{
+    t->data.resize(data.size());
+    for (size_t i = 0; i < data.size(); ++i)
+    {
+        t->data[i] = data[i];
+    }
+}
+
+#endif
\ No newline at end of file

test/util/test_control.hpp

@@ -18,6 +18,11 @@
 
 #include "gtest/gtest.h"
 
+#ifdef NGRAPH_UNIT_TEST_OPENVINO_ENABLE
+#include "util/backend_utils.hpp"
+#define runtime ov_runtime
+#endif
+
 // Copied from gtest
 
 namespace ngraph