Merge pull request #9313 from dkurt:dnn_perf_test

doc/tutorials/dnn/dnn_halide/dnn_halide.markdown

@@ -8,20 +8,7 @@ according to specific device and evaluate it with a quite good efficiency.
 
 An official website of the Halide project: http://halide-lang.org/.
 
-## Efficiency comparison
-Measured on Intel® Core™ i7-6700K CPU @ 4.00GHz x 8.
-
-Single image forward pass (in milliseconds):
-
-|     Architecture | MKL backend | Halide backend | Speed Up ratio |
-|-----------------:|------------:|---------------:|---------------:|
-|          AlexNet |       16.55 |          22.38 |          x0.73 |
-|        ResNet-50 |       63.69 |          73.91 |          x0.86 |
-|  SqueezeNet v1.1 |       10.11 |           8.21 |          x1.23 |
-|     Inception-5h |       35.38 |          37.06 |          x0.95 |
-| ENet @ 3x512x256 |       82.26 |          41.21 |          x1.99 |
-
-Scheduling directives might be found @ [opencv_extra/testdata/dnn](https://github.com/opencv/opencv_extra/tree/master/testdata/dnn).
+An up to date efficiency comparison: https://github.com/opencv/opencv/wiki/DNN-Efficiency
 
 ## Requirements
 ### LLVM compiler
@@ -81,6 +68,8 @@ MSBuild.exe /m:4 /t:Build /p:Configuration=Release .\\ALL_BUILD.vcxproj
 ## Build OpenCV with Halide backend
 When you build OpenCV add the following configuration flags:
 
+- `ENABLE_CXX11` - enable C++11 standard
+
 - `WITH_HALIDE` - enable Halide linkage
 
 - `HALIDE_ROOT_DIR` - path to Halide build directory

modules/dnn/CMakeLists.txt

@@ -77,6 +77,24 @@ ocv_add_samples()
 ocv_add_accuracy_tests()
 ocv_add_perf_tests()
 
+ocv_option(${the_module}_PERF_CAFFE "Add performance tests of Caffe framework" OFF)
+ocv_option(${the_module}_PERF_CLCAFFE "Add performance tests of clCaffe framework" OFF)
+if(BUILD_PERF_TESTS)
+  if (${the_module}_PERF_CAFFE)
+    find_package(Caffe QUIET)
+    if (Caffe_FOUND)
+      add_definitions(-DHAVE_CAFFE=1)
+      ocv_target_link_libraries(opencv_perf_dnn caffe)
+    endif()
+  elseif(${the_module}_PERF_CLCAFFE)
+    find_package(Caffe QUIET)
+    if (Caffe_FOUND)
+      add_definitions(-DHAVE_CLCAFFE=1)
+      ocv_target_link_libraries(opencv_perf_dnn caffe)
+    endif()
+  endif()
+endif()
+
 # ----------------------------------------------------------------------------
 # Torch7 importer of blobs and models, produced by Torch.nn module
 # ----------------------------------------------------------------------------

modules/dnn/include/opencv2/dnn/dnn.hpp

@@ -433,21 +433,21 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
          * specific target. For layers that not represented in scheduling file
          * or if no manual scheduling used at all, automatic scheduling will be applied.
          */
-        void setHalideScheduler(const String& scheduler);
+        CV_WRAP void setHalideScheduler(const String& scheduler);
 
         /**
          * @brief Ask network to use specific computation backend where it supported.
          * @param[in] backendId backend identifier.
          * @see Backend
          */
-        void setPreferableBackend(int backendId);
+        CV_WRAP void setPreferableBackend(int backendId);
 
         /**
          * @brief Ask network to make computations on specific target device.
          * @param[in] targetId target identifier.
          * @see Target
          */
-        void setPreferableTarget(int targetId);
+        CV_WRAP void setPreferableTarget(int targetId);
 
         /** @brief Sets the new value for the layer output blob
          *  @param name descriptor of the updating layer output blob.

modules/dnn/perf/perf_caffe.cpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2017, Intel Corporation, all rights reserved.
+// Third party copyrights are property of their respective owners.
+
+// Recommends run this performance test via
+// ./bin/opencv_perf_dnn 2> /dev/null | grep "PERFSTAT" -A 3
+// because whole output includes Caffe's logs.
+//
+// Note: Be sure that interesting version of Caffe was linked.
+// Note: There is an impact on Halide performance. Comment this tests if you
+//       want to run the last one.
+//
+// How to build Intel-Caffe with MKLDNN backend
+// ============================================
+// mkdir build && cd build
+// cmake -DCMAKE_BUILD_TYPE=Release \
+//       -DUSE_MKLDNN_AS_DEFAULT_ENGINE=ON \
+//       -DUSE_MKL2017_AS_DEFAULT_ENGINE=OFF \
+//       -DCPU_ONLY=ON \
+//       -DCMAKE_INSTALL_PREFIX=/usr/local .. && make -j8
+// sudo make install
+//
+// In case of problems with cublas_v2.h at include/caffe/util/device_alternate.hpp: add line
+// #define CPU_ONLY
+// before the first line
+// #ifdef CPU_ONLY  // CPU-only Caffe.
+
+#if defined(HAVE_CAFFE) || defined(HAVE_CLCAFFE)
+
+#include "perf_precomp.hpp"
+#include <iostream>
+#include <caffe/caffe.hpp>
+
+namespace cvtest
+{
+
+static caffe::Net<float>* initNet(std::string proto, std::string weights)
+{
+    proto = findDataFile(proto, false);
+    weights = findDataFile(weights, false);
+
+#ifdef HAVE_CLCAFFE
+    caffe::Caffe::set_mode(caffe::Caffe::GPU);
+    caffe::Caffe::SetDevice(0);
+
+    caffe::Net<float>* net =
+        new caffe::Net<float>(proto, caffe::TEST, caffe::Caffe::GetDefaultDevice());
+#else
+    caffe::Caffe::set_mode(caffe::Caffe::CPU);
+
+    caffe::Net<float>* net = new caffe::Net<float>(proto, caffe::TEST);
+#endif
+
+    net->CopyTrainedLayersFrom(weights);
+
+    caffe::Blob<float>* input = net->input_blobs()[0];
+
+    CV_Assert(input->num() == 1);
+    CV_Assert(input->channels() == 3);
+
+    Mat inputMat(input->height(), input->width(), CV_32FC3, (char*)input->cpu_data());
+    randu(inputMat, 0.0f, 1.0f);
+
+    net->Forward();
+    return net;
+}
+
+PERF_TEST(GoogLeNet_caffe, CaffePerfTest)
+{
+    caffe::Net<float>* net = initNet("dnn/bvlc_googlenet.prototxt",
+                                     "dnn/bvlc_googlenet.caffemodel");
+    TEST_CYCLE() net->Forward();
+    SANITY_CHECK_NOTHING();
+}
+
+PERF_TEST(AlexNet_caffe, CaffePerfTest)
+{
+    caffe::Net<float>* net = initNet("dnn/bvlc_alexnet.prototxt",
+                                     "dnn/bvlc_alexnet.caffemodel");
+    TEST_CYCLE() net->Forward();
+    SANITY_CHECK_NOTHING();
+}
+
+PERF_TEST(ResNet50_caffe, CaffePerfTest)
+{
+    caffe::Net<float>* net = initNet("dnn/ResNet-50-deploy.prototxt",
+                                     "dnn/ResNet-50-model.caffemodel");
+    TEST_CYCLE() net->Forward();
+    SANITY_CHECK_NOTHING();
+}
+
+PERF_TEST(SqueezeNet_v1_1_caffe, CaffePerfTest)
+{
+    caffe::Net<float>* net = initNet("dnn/squeezenet_v1.1.prototxt",
+                                     "dnn/squeezenet_v1.1.caffemodel");
+    TEST_CYCLE() net->Forward();
+    SANITY_CHECK_NOTHING();
+}
+
+}  // namespace cvtest
+
+#endif  // HAVE_CAFFE

modules/dnn/perf/perf_halide_net.cpp

@@ -55,7 +55,7 @@ PERF_TEST(GoogLeNet, HalidePerfTest)
 {
     Net net;
     loadNet("dnn/bvlc_googlenet.caffemodel", "dnn/bvlc_googlenet.prototxt",
-            "", 227, 227, "prob", "caffe", DNN_TARGET_CPU, &net);
+            "", 224, 224, "prob", "caffe", DNN_TARGET_CPU, &net);
     TEST_CYCLE() net.forward();
     SANITY_CHECK_NOTHING();
 }

modules/dnn/test/test_halide_nets.cpp

@@ -99,7 +99,7 @@ TEST(Reproducibility_GoogLeNet_Halide, Accuracy)
 {
     test(findDataFile("dnn/bvlc_googlenet.caffemodel", false),
          findDataFile("dnn/bvlc_googlenet.prototxt", false),
-         "", 227, 227, "prob", "caffe", DNN_TARGET_CPU);
+         "", 224, 224, "prob", "caffe", DNN_TARGET_CPU);
 };
 
 TEST(Reproducibility_AlexNet_Halide, Accuracy)

samples/dnn/squeezenet_halide.cpp

-// This file is part of OpenCV project.
-// It is subject to the license terms in the LICENSE file found in the top-level directory
-// of this distribution and at http://opencv.org/license.html.
-//
-// Copyright (C) 2017, Intel Corporation, all rights reserved.
-// Third party copyrights are property of their respective owners.
-
 // Sample of using Halide backend in OpenCV deep learning module.
 // Based on caffe_googlenet.cpp.