Merge pull request #14368 from dkurt:dnn_ie_tests

modules/dnn/src/layers/lrn_layer.cpp

@@ -91,7 +91,7 @@ public:
     virtual bool supportBackend(int backendId) CV_OVERRIDE
     {
         if (backendId == DNN_BACKEND_INFERENCE_ENGINE)
-            return bias == 1;
+            return bias == (int)bias;
         return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE;
     }
 

modules/dnn/test/test_backends.cpp

@@ -45,7 +45,7 @@ public:
         netDefault.setInput(inp);
         Mat outDefault = netDefault.forward(outputLayer).clone();
 
-        Net net = readNet(weights, proto);
+        net = readNet(weights, proto);
         net.setInput(inp);
         net.setPreferableBackend(backend);
         net.setPreferableTarget(target);
@@ -94,6 +94,8 @@ public:
         else
             normAssert(ref, out, msg, l1, lInf);
     }
+
+    Net net;
 };
 
 TEST_P(DNNTestNetwork, AlexNet)
@@ -103,6 +105,7 @@ TEST_P(DNNTestNetwork, AlexNet)
                Size(227, 227), "prob",
                target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_alexnet.yml" :
                                              "dnn/halide_scheduler_alexnet.yml");
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, ResNet_50)
@@ -112,6 +115,7 @@ TEST_P(DNNTestNetwork, ResNet_50)
                Size(224, 224), "prob",
                target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_resnet_50.yml" :
                                              "dnn/halide_scheduler_resnet_50.yml");
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, SqueezeNet_v1_1)
@@ -120,6 +124,7 @@ TEST_P(DNNTestNetwork, SqueezeNet_v1_1)
                Size(227, 227), "prob",
                target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_squeezenet_v1_1.yml" :
                                              "dnn/halide_scheduler_squeezenet_v1_1.yml");
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, GoogLeNet)
@@ -127,6 +132,7 @@ TEST_P(DNNTestNetwork, GoogLeNet)
     applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
     processNet("dnn/bvlc_googlenet.caffemodel", "dnn/bvlc_googlenet.prototxt",
                Size(224, 224), "prob");
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, Inception_5h)
@@ -142,6 +148,7 @@ TEST_P(DNNTestNetwork, Inception_5h)
                target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_inception_5h.yml" :
                                              "dnn/halide_scheduler_inception_5h.yml",
                l1, lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, ENet)
@@ -168,6 +175,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe)
     float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.252  : 0.0;
          processNet("dnn/MobileNetSSD_deploy.caffemodel", "dnn/MobileNetSSD_deploy.prototxt",
                     inp, "detection_out", "", diffScores, diffSquares, detectionConfThresh);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe_Different_Width_Height)
@@ -185,7 +193,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe_Different_Width_Height)
     float diffSquares = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09  : 0.0;
     processNet("dnn/MobileNetSSD_deploy.caffemodel", "dnn/MobileNetSSD_deploy.prototxt",
                 inp, "detection_out", "", diffScores, diffSquares);
-
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow)
@@ -200,6 +208,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow)
     float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.216 : 0.2;
     processNet("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", "dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt",
                inp, "detection_out", "", l1, lInf, detectionConfThresh);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow_Different_Width_Height)
@@ -217,6 +226,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow_Different_Width_Height)
     float lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.06 : 0.0;
     processNet("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", "dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt",
                inp, "detection_out", "", l1, lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow)
@@ -230,6 +240,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow)
     float lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.062 : 0.0;
     processNet("dnn/ssd_mobilenet_v2_coco_2018_03_29.pb", "dnn/ssd_mobilenet_v2_coco_2018_03_29.pbtxt",
                inp, "detection_out", "", l1, lInf, 0.25);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, SSD_VGG16)
@@ -244,6 +255,7 @@ TEST_P(DNNTestNetwork, SSD_VGG16)
     Mat inp = blobFromImage(sample, 1.0f, Size(300, 300), Scalar(), false);
     processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel",
                "dnn/ssd_vgg16.prototxt", inp, "detection_out", "", scoreThreshold, lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, OpenPose_pose_coco)
@@ -262,6 +274,7 @@ TEST_P(DNNTestNetwork, OpenPose_pose_coco)
     const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.072 : 0.0;
     processNet("dnn/openpose_pose_coco.caffemodel", "dnn/openpose_pose_coco.prototxt",
                Size(46, 46), "", "", l1, lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, OpenPose_pose_mpi)
@@ -280,6 +293,7 @@ TEST_P(DNNTestNetwork, OpenPose_pose_mpi)
     const float lInf = (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_OPENCL_FP16) ? 0.16 : 0.0;
     processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi.prototxt",
                Size(46, 46), "", "", l1, lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
@@ -296,6 +310,7 @@ TEST_P(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
     // See https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/pose/poseParameters.cpp
     processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi_faster_4_stages.prototxt",
                Size(46, 46));
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, OpenFace)
@@ -324,6 +339,7 @@ TEST_P(DNNTestNetwork, opencv_face_detector)
     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
     processNet("dnn/opencv_face_detector.caffemodel", "dnn/opencv_face_detector.prototxt",
                inp, "detection_out");
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
@@ -342,6 +358,7 @@ TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
     float lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0731 : 0.0;
     processNet("dnn/ssd_inception_v2_coco_2017_11_17.pb", "dnn/ssd_inception_v2_coco_2017_11_17.pbtxt",
                inp, "detection_out", "", l1, lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, DenseNet_121)
@@ -360,6 +377,7 @@ TEST_P(DNNTestNetwork, DenseNet_121)
         l1 = 0.1; lInf = 0.6;
     }
     processNet("dnn/DenseNet_121.caffemodel", "dnn/DenseNet_121.prototxt", Size(224, 224), "", "", l1, lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(DNNTestNetwork, FastNeuralStyle_eccv16)

modules/dnn/test/test_caffe_importer.cpp

@@ -417,6 +417,7 @@ TEST_P(Test_Caffe_nets, Colorization)
         l1 = 0.6; lInf = 15;
     }
     normAssert(out, ref, "", l1, lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(Test_Caffe_nets, DenseNet_121)
@@ -448,6 +449,7 @@ TEST_P(Test_Caffe_nets, DenseNet_121)
         l1 = 0.11; lInf = 0.5;
     }
     normAssert(out, ref, "", l1, lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST(Test_Caffe, multiple_inputs)

modules/dnn/test/test_common.hpp

@@ -110,6 +110,31 @@ public:
         }
     }
 
+    void expectNoFallbacks(Net& net)
+    {
+        // Check if all the layers are supported with current backend and target.
+        // Some layers might be fused so their timings equal to zero.
+        std::vector<double> timings;
+        net.getPerfProfile(timings);
+        std::vector<String> names = net.getLayerNames();
+        CV_Assert(names.size() == timings.size());
+
+        for (int i = 0; i < names.size(); ++i)
+        {
+            Ptr<dnn::Layer> l = net.getLayer(net.getLayerId(names[i]));
+            bool fused = !timings[i];
+            if ((!l->supportBackend(backend) || l->preferableTarget != target) && !fused)
+                CV_Error(Error::StsNotImplemented, "Layer [" + l->name + "] of type [" +
+                         l->type + "] is expected to has backend implementation");
+        }
+    }
+
+    void expectNoFallbacksFromIE(Net& net)
+    {
+        if (backend == DNN_BACKEND_INFERENCE_ENGINE)
+            expectNoFallbacks(net);
+    }
+
 protected:
     void checkBackend(Mat* inp = 0, Mat* ref = 0)
     {

modules/dnn/test/test_onnx_importer.cpp

@@ -28,7 +28,8 @@ public:
     };
 
     void testONNXModels(const String& basename, const Extension ext = npy,
-                        const double l1 = 0, const float lInf = 0, const bool useSoftmax = false)
+                        const double l1 = 0, const float lInf = 0, const bool useSoftmax = false,
+                        bool checkNoFallbacks = true)
     {
         String onnxmodel = _tf("models/" + basename + ".onnx");
         Mat inp, ref;
@@ -67,6 +68,8 @@ public:
             ref = netSoftmax.forward();
         }
         normAssert(ref, out, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
+        if (checkNoFallbacks)
+            expectNoFallbacksFromIE(net);
     }
 };
 
@@ -103,11 +106,11 @@ TEST_P(Test_ONNX_layers, Two_convolution)
 
 TEST_P(Test_ONNX_layers, Deconvolution)
 {
-    testONNXModels("deconvolution");
-    testONNXModels("two_deconvolution");
-    testONNXModels("deconvolution_group");
-    testONNXModels("deconvolution_output_shape");
-    testONNXModels("deconv_adjpad_2d");
+    testONNXModels("deconvolution", npy, 0, 0, false, false);
+    testONNXModels("two_deconvolution", npy, 0, 0, false, false);
+    testONNXModels("deconvolution_group", npy, 0, 0, false, false);
+    testONNXModels("deconvolution_output_shape", npy, 0, 0, false, false);
+    testONNXModels("deconv_adjpad_2d", npy, 0, 0, false, false);
 }
 
 TEST_P(Test_ONNX_layers, Dropout)
@@ -220,6 +223,7 @@ TEST_P(Test_ONNX_layers, MultyInputs)
     Mat out = net.forward();
 
     normAssert(ref, out, "", default_l1,  default_lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(Test_ONNX_layers, DynamicReshape)
@@ -257,6 +261,7 @@ TEST_P(Test_ONNX_nets, Alexnet)
     Mat out = net.forward();
 
     normAssert(out, ref, "", default_l1,  default_lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(Test_ONNX_nets, Squeezenet)
@@ -289,6 +294,7 @@ TEST_P(Test_ONNX_nets, Googlenet)
     Mat out = net.forward();
 
     normAssert(ref, out, "", default_l1,  default_lInf);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(Test_ONNX_nets, CaffeNet)

modules/dnn/test/test_tf_importer.cpp

@@ -356,6 +356,7 @@ TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
     normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
@@ -393,6 +394,7 @@ TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
     normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
@@ -423,6 +425,7 @@ TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
     float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 7e-3 : 1.5e-5;
     float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
     normAssertDetections(ref, out, "", 0.3, scoreDiff, iouDiff);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
@@ -481,6 +484,7 @@ TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
     normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
+    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
@@ -510,6 +514,7 @@ TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
     normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
+    expectNoFallbacksFromIE(net);
 }
 
 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
@@ -574,6 +579,7 @@ TEST_P(Test_TensorFlow_nets, EAST_text_detection)
     }
     normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
+    expectNoFallbacksFromIE(net);
 }
 
 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());