Merge pull request #12071 from l-bat/l-bat:onnx_parser

* Add Squeezenet support in ONNX

* Add AlexNet support in ONNX

* Add Googlenet support in ONNX

* Add CaffeNet and RCNN support in ONNX

* Add VGG16 and VGG16 with batch normalization support in ONNX

* Add RCNN, ZFNet, ResNet18v1 and ResNet50v1 support in ONNX

* Add ResNet101_DUC_HDC

* Add Tiny Yolov2

* Add CNN_MNIST, MobileNetv2 and LResNet100 support in ONNX

* Add ONNX models for emotion recognition

* Add DenseNet121 support in ONNX

* Add Inception v1 support in ONNX

* Refactoring

* Fix tests

* Fix tests

* Skip unstable test

* Modify Reshape operation

modules/dnn/CMakeLists.txt

@@ -67,13 +67,13 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS
 )
 
 if(PROTOBUF_UPDATE_FILES)
-  file(GLOB proto_files "${CMAKE_CURRENT_LIST_DIR}/src/tensorflow/*.proto" "${CMAKE_CURRENT_LIST_DIR}/src/caffe/opencv-caffe.proto")
+  file(GLOB proto_files "${CMAKE_CURRENT_LIST_DIR}/src/tensorflow/*.proto" "${CMAKE_CURRENT_LIST_DIR}/src/caffe/opencv-caffe.proto" "${CMAKE_CURRENT_LIST_DIR}/src/onnx/opencv-onnx.proto")
   set(PROTOBUF_GENERATE_CPP_APPEND_PATH ON) # required for tensorflow
   protobuf_generate_cpp(fw_srcs fw_hdrs ${proto_files})
 else()
-  file(GLOB fw_srcs "${CMAKE_CURRENT_LIST_DIR}/misc/tensorflow/*.cc" "${CMAKE_CURRENT_LIST_DIR}/misc/caffe/opencv-caffe.pb.cc")
-  file(GLOB fw_hdrs "${CMAKE_CURRENT_LIST_DIR}/misc/tensorflow/*.h" "${CMAKE_CURRENT_LIST_DIR}/misc/caffe/opencv-caffe.pb.h")
-  set(fw_inc "${CMAKE_CURRENT_LIST_DIR}/misc/caffe" "${CMAKE_CURRENT_LIST_DIR}/misc/tensorflow")
+  file(GLOB fw_srcs "${CMAKE_CURRENT_LIST_DIR}/misc/tensorflow/*.cc" "${CMAKE_CURRENT_LIST_DIR}/misc/caffe/opencv-caffe.pb.cc" "${CMAKE_CURRENT_LIST_DIR}/misc/onnx/opencv-onnx.pb.cc")
+  file(GLOB fw_hdrs "${CMAKE_CURRENT_LIST_DIR}/misc/tensorflow/*.h" "${CMAKE_CURRENT_LIST_DIR}/misc/caffe/opencv-caffe.pb.h" "${CMAKE_CURRENT_LIST_DIR}/misc/onnx/opencv-onnx.pb.h")
+  set(fw_inc "${CMAKE_CURRENT_LIST_DIR}/misc/caffe" "${CMAKE_CURRENT_LIST_DIR}/misc/tensorflow" "${CMAKE_CURRENT_LIST_DIR}/misc/onnx")
 endif()
 
 set(include_dirs ${fw_inc})

modules/dnn/include/opencv2/dnn/dict.hpp

@@ -141,6 +141,9 @@ public:
     template<typename T>
     const T &set(const String &key, const T &value);
 
+    //! Erase @p key from the dictionary.
+    void erase(const String &key);
+
     friend std::ostream &operator<<(std::ostream &stream, const Dict &dict);
 
     std::map<String, DictValue>::const_iterator begin() const;

modules/dnn/include/opencv2/dnn/dnn.hpp

@@ -814,6 +814,18 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
      */
     CV_EXPORTS_W Net readNetFromModelOptimizer(const String &xml, const String &bin);
 
+    /** @brief Reads a network model <a href="https://onnx.ai/">ONNX</a>.
+     *  @param onnxFile path to the .onnx file with text description of the network architecture.
+     *  @returns Network object that ready to do forward, throw an exception in failure cases.
+     */
+    CV_EXPORTS_W Net readNetFromONNX(const String &onnxFile);
+
+    /** @brief Creates blob from .pb file.
+     *  @param path to the .pb file with input tensor.
+     *  @returns Mat.
+     */
+    CV_EXPORTS_W Mat readTensorFromONNX(const String& path);
+
     /** @brief Creates 4-dimensional blob from image. Optionally resizes and crops @p image from center,
      *  subtract @p mean values, scales values by @p scalefactor, swap Blue and Red channels.
      *  @param image input image (with 1-, 3- or 4-channels).

modules/dnn/include/opencv2/dnn/dnn.inl.hpp

@@ -364,6 +364,11 @@ inline const T &Dict::set(const String &key, const T &value)
     return value;
 }
 
+inline void Dict::erase(const String &key)
+{
+    dict.erase(key);
+}
+
 inline std::ostream &operator<<(std::ostream &stream, const Dict &dict)
 {
     Dict::_Dict::const_iterator it;

modules/dnn/src/dnn.cpp

@@ -3462,6 +3462,10 @@ Net readNet(const String& _model, const String& _config, const String& _framewor
             std::swap(model, config);
         return readNetFromModelOptimizer(config, model);
     }
+    if (framework == "onnx" || modelExt == "onnx")
+    {
+        return readNetFromONNX(model);
+    }
     CV_Error(Error::StsError, "Cannot determine an origin framework of files: " +
                                       model + (config.empty() ? "" : ", " + config));
 }

modules/dnn/test/test_onnx_importer.cpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+// Copyright (C) 2018, Intel Corporation, all rights reserved.
+// Third party copyrights are property of their respective owners.
+
+
+#include "test_precomp.hpp"
+#include "npy_blob.hpp"
+#include <opencv2/dnn/shape_utils.hpp>
+
+namespace opencv_test { namespace {
+
+template<typename TString>
+static std::string _tf(TString filename)
+{
+    String rootFolder = "dnn/onnx/";
+    return findDataFile(rootFolder + filename, false);
+}
+
+class Test_ONNX_layers : public DNNTestLayer
+{
+public:
+    enum Extension
+    {
+        npy,
+        pb
+    };
+
+    void testONNXModels(const String& basename, const Extension ext = npy, const double l1 = 0, const float lInf = 0)
+    {
+        String onnxmodel = _tf("models/" + basename + ".onnx");
+        Mat inp, ref;
+        if (ext == npy) {
+            inp = blobFromNPY(_tf("data/input_" + basename + ".npy"));
+            ref = blobFromNPY(_tf("data/output_" + basename + ".npy"));
+        }
+        else if (ext == pb) {
+            inp = readTensorFromONNX(_tf("data/input_" + basename + ".pb"));
+            ref = readTensorFromONNX(_tf("data/output_" + basename + ".pb"));
+        }
+        else
+            CV_Error(Error::StsUnsupportedFormat, "Unsupported extension");
+
+        checkBackend(&inp, &ref);
+        Net net = readNetFromONNX(onnxmodel);
+        ASSERT_FALSE(net.empty());
+
+        net.setPreferableBackend(backend);
+        net.setPreferableTarget(target);
+
+        net.setInput(inp);
+        Mat out = net.forward();
+        normAssert(ref, out, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
+    }
+};
+
+TEST_P(Test_ONNX_layers, MaxPooling)
+{
+    testONNXModels("maxpooling");
+    testONNXModels("two_maxpooling");
+}
+
+TEST_P(Test_ONNX_layers, Convolution)
+{
+    testONNXModels("convolution");
+    testONNXModels("two_convolution");
+}
+
+TEST_P(Test_ONNX_layers, Dropout)
+{
+    testONNXModels("dropout");
+}
+
+TEST_P(Test_ONNX_layers, Linear)
+{
+    if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
+        throw SkipTestException("");
+    testONNXModels("linear");
+}
+
+TEST_P(Test_ONNX_layers, ReLU)
+{
+    testONNXModels("ReLU");
+}
+
+TEST_P(Test_ONNX_layers, MaxPooling_Sigmoid)
+{
+    testONNXModels("maxpooling_sigmoid");
+}
+
+TEST_P(Test_ONNX_layers, Concatenation)
+{
+    if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
+         (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_OPENCL || target == DNN_TARGET_MYRIAD))
+        throw SkipTestException("");
+    testONNXModels("concatenation");
+}
+
+TEST_P(Test_ONNX_layers, AveragePooling)
+{
+    testONNXModels("average_pooling");
+}
+
+TEST_P(Test_ONNX_layers, BatchNormalization)
+{
+    testONNXModels("batch_norm");
+}
+
+TEST_P(Test_ONNX_layers, Multiplication)
+{
+    if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16 ||
+        backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
+        throw SkipTestException("");
+    testONNXModels("mul");
+}
+
+TEST_P(Test_ONNX_layers, Constant)
+{
+    testONNXModels("constant");
+}
+
+TEST_P(Test_ONNX_layers, MultyInputs)
+{
+    const String model =  _tf("models/multy_inputs.onnx");
+
+    Net net = readNetFromONNX(model);
+    ASSERT_FALSE(net.empty());
+
+    net.setPreferableBackend(backend);
+    net.setPreferableTarget(target);
+
+    Mat inp1 = blobFromNPY(_tf("data/input_multy_inputs_0.npy"));
+    Mat inp2 = blobFromNPY(_tf("data/input_multy_inputs_1.npy"));
+    Mat ref  = blobFromNPY(_tf("data/output_multy_inputs.npy"));
+    checkBackend(&inp1, &ref);
+
+    net.setInput(inp1, "0");
+    net.setInput(inp2, "1");
+    Mat out = net.forward();
+
+    normAssert(ref, out, "", default_l1,  default_lInf);
+}
+
+
+INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_ONNX_layers, dnnBackendsAndTargets());
+
+class Test_ONNX_nets : public Test_ONNX_layers {};
+TEST_P(Test_ONNX_nets, Alexnet)
+{
+    const String model =  _tf("models/alexnet.onnx");
+
+    Net net = readNetFromONNX(model);
+    ASSERT_FALSE(net.empty());
+
+    net.setPreferableBackend(backend);
+    net.setPreferableTarget(target);
+
+    Mat inp = imread(_tf("../grace_hopper_227.png"));
+    Mat ref = blobFromNPY(_tf("../caffe_alexnet_prob.npy"));
+    checkBackend(&inp, &ref);
+
+    net.setInput(blobFromImage(inp, 1.0f, Size(227, 227), Scalar(), false));
+    ASSERT_FALSE(net.empty());
+    Mat out = net.forward();
+
+    normAssert(out, ref, "", default_l1,  default_lInf);
+}
+
+TEST_P(Test_ONNX_nets, Squeezenet)
+{
+    testONNXModels("squeezenet", pb);
+}
+
+TEST_P(Test_ONNX_nets, Googlenet)
+{
+    if (backend == DNN_BACKEND_INFERENCE_ENGINE)
+        throw SkipTestException("");
+
+    const String model = _tf("models/googlenet.onnx");
+
+    Net net = readNetFromONNX(model);
+    ASSERT_FALSE(net.empty());
+
+    net.setPreferableBackend(backend);
+    net.setPreferableTarget(target);
+
+    std::vector<Mat> images;
+    images.push_back( imread(_tf("../googlenet_0.png")) );
+    images.push_back( imread(_tf("../googlenet_1.png")) );
+    Mat inp = blobFromImages(images, 1.0f, Size(), Scalar(), false);
+    Mat ref = blobFromNPY(_tf("../googlenet_prob.npy"));
+    checkBackend(&inp, &ref);
+
+    net.setInput(inp);
+    ASSERT_FALSE(net.empty());
+    Mat out = net.forward();
+
+    normAssert(ref, out, "", default_l1,  default_lInf);
+}
+
+TEST_P(Test_ONNX_nets, CaffeNet)
+{
+    testONNXModels("caffenet", pb);
+}
+
+TEST_P(Test_ONNX_nets, RCNN_ILSVRC13)
+{
+    testONNXModels("rcnn_ilsvrc13", pb);
+}
+
+TEST_P(Test_ONNX_nets, VGG16)
+{
+    double l1 = default_l1;
+    double lInf = default_lInf;
+    // output range: [-69; 72]
+    if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) {
+        l1 = 0.087;
+        lInf = 0.585;
+    }
+    else if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL) {
+        lInf = 1.2e-4;
+    }
+    testONNXModels("vgg16", pb, l1, lInf);
+}
+
+TEST_P(Test_ONNX_nets, VGG16_bn)
+{
+    double l1 = default_l1;
+    double lInf = default_lInf;
+    // output range: [-16; 27]
+    if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) {
+        l1 = 0.0086;
+        lInf = 0.037;
+    }
+    else if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
+             (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)) {
+        l1 = 0.031;
+        lInf = 0.2;
+    }
+    testONNXModels("vgg16-bn", pb, l1, lInf);
+}
+
+TEST_P(Test_ONNX_nets, ZFNet)
+{
+    testONNXModels("zfnet512", pb);
+}
+
+TEST_P(Test_ONNX_nets, ResNet18v1)
+{
+    // output range: [-16; 22]
+    const double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.022 : default_l1;
+    const double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.12 : default_lInf;
+    testONNXModels("resnet18v1", pb, l1, lInf);
+}
+
+TEST_P(Test_ONNX_nets, ResNet50v1)
+{
+    // output range: [-67; 75]
+    const double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.6 : 1.25e-5;
+    const double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.51 : 1.2e-4;
+    testONNXModels("resnet50v1", pb, l1, lInf);
+}
+
+TEST_P(Test_ONNX_nets, ResNet101_DUC_HDC)
+{
+    if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_OPENCL
+                || target == DNN_TARGET_MYRIAD) {
+        throw SkipTestException("");
+    }
+    testONNXModels("resnet101_duc_hdc", pb);
+}
+
+TEST_P(Test_ONNX_nets, TinyYolov2)
+{
+    if (cvtest::skipUnstableTests ||
+        backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) {
+        throw SkipTestException("");
+    }
+    // output range: [-11; 8]
+    const double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.017 : default_l1;
+    const double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.14 : default_lInf;
+    testONNXModels("tiny_yolo2", pb, l1, lInf);
+}
+
+TEST_P(Test_ONNX_nets, CNN_MNIST)
+{
+    // output range: [-1952; 6574]
+    const double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 3.82 : 4.3e-4;
+    const double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 13.5 : 1e-3;
+
+    testONNXModels("cnn_mnist", pb, l1, lInf);
+}
+
+TEST_P(Test_ONNX_nets, MobileNet_v2)
+{
+    // output range: [-166; 317]
+    const double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.38 : 7e-5;
+    const double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 2.87 : 5e-4;
+    testONNXModels("mobilenetv2", pb, l1, lInf);
+}
+
+TEST_P(Test_ONNX_nets, LResNet100E_IR)
+{
+    if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
+         (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_OPENCL || target == DNN_TARGET_MYRIAD))
+        throw SkipTestException("");
+
+    double l1 = default_l1;
+    double lInf = default_lInf;
+    // output range: [-3; 3]
+    if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) {
+        l1 = 0.009;
+        lInf = 0.035;
+    }
+    testONNXModels("LResNet100E_IR", pb, l1, lInf);
+}
+
+TEST_P(Test_ONNX_nets, Emotion_ferplus)
+{
+    testONNXModels("emotion_ferplus", pb);
+}
+
+TEST_P(Test_ONNX_nets, Inception_v2)
+{
+    if (backend == DNN_BACKEND_INFERENCE_ENGINE)
+        throw SkipTestException("");
+
+    testONNXModels("inception_v2", pb);
+}
+
+TEST_P(Test_ONNX_nets, DenseNet121)
+{
+    // output range: [-87; 138]
+    const double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.12 : 1.88e-5;
+    const double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.74 : 1.23e-4;
+    testONNXModels("densenet121", pb, l1, lInf);
+}
+
+
+INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_nets, dnnBackendsAndTargets());
+
+}} // namespace