uint8 inputs for deep learning networks

modules/dnn/include/opencv2/dnn/dnn.hpp

@@ -46,9 +46,9 @@
 #include <opencv2/core.hpp>
 
 #if !defined CV_DOXYGEN && !defined CV_DNN_DONT_ADD_EXPERIMENTAL_NS
-#define CV__DNN_EXPERIMENTAL_NS_BEGIN namespace experimental_dnn_v5 {
+#define CV__DNN_EXPERIMENTAL_NS_BEGIN namespace experimental_dnn_v6 {
 #define CV__DNN_EXPERIMENTAL_NS_END }
-namespace cv { namespace dnn { namespace experimental_dnn_v5 { } using namespace experimental_dnn_v5; }}
+namespace cv { namespace dnn { namespace experimental_dnn_v6 { } using namespace experimental_dnn_v6; }}
 #else
 #define CV__DNN_EXPERIMENTAL_NS_BEGIN
 #define CV__DNN_EXPERIMENTAL_NS_END
@@ -487,14 +487,19 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
          */
         CV_WRAP void setPreferableTarget(int targetId);
 
-        /** @brief Sets the new value for the layer output blob
-         *  @param name descriptor of the updating layer output blob.
-         *  @param blob new blob.
+        /** @brief Sets the new input value for the network
+         *  @param blob        A new blob. Should have CV_32F or CV_8U depth.
+         *  @param name        A name of input layer.
+         *  @param scalefactor An optional normalization scale.
+         *  @param mean        An optional mean subtraction values.
          *  @see connect(String, String) to know format of the descriptor.
-         *  @note If updating blob is not empty then @p blob must have the same shape,
-         *  because network reshaping is not implemented yet.
+         *
+         *  If scale or mean values are specified, a final input blob is computed
+         *  as:
+         * \f[input(n,c,h,w) = scalefactor \times (blob(n,c,h,w) - mean_c)\f]
          */
-        CV_WRAP void setInput(InputArray blob, const String& name = "");
+        CV_WRAP void setInput(InputArray blob, const String& name = "",
+                              double scalefactor = 1.0, const Scalar& mean = Scalar());
 
         /** @brief Sets the new value for the learned param of the layer.
          *  @param layer name or id of the layer.
@@ -805,13 +810,15 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
      *  @param swapRB flag which indicates that swap first and last channels
      *  in 3-channel image is necessary.
      *  @param crop flag which indicates whether image will be cropped after resize or not
+     *  @param ddepth Depth of output blob. Choose CV_32F or CV_8U.
      *  @details if @p crop is true, input image is resized so one side after resize is equal to corresponding
      *  dimension in @p size and another one is equal or larger. Then, crop from the center is performed.
      *  If @p crop is false, direct resize without cropping and preserving aspect ratio is performed.
      *  @returns 4-dimensional Mat with NCHW dimensions order.
      */
     CV_EXPORTS_W Mat blobFromImage(InputArray image, double scalefactor=1.0, const Size& size = Size(),
-                                   const Scalar& mean = Scalar(), bool swapRB=true, bool crop=true);
+                                   const Scalar& mean = Scalar(), bool swapRB=true, bool crop=true,
+                                   int ddepth=CV_32F);
 
     /** @brief Creates 4-dimensional blob from image.
      *  @details This is an overloaded member function, provided for convenience.
@@ -819,7 +826,7 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
      */
     CV_EXPORTS void blobFromImage(InputArray image, OutputArray blob, double scalefactor=1.0,
                                   const Size& size = Size(), const Scalar& mean = Scalar(),
-                                  bool swapRB=true, bool crop=true);
+                                  bool swapRB=true, bool crop=true, int ddepth=CV_32F);
 
 
     /** @brief Creates 4-dimensional blob from series of images. Optionally resizes and
@@ -833,13 +840,15 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
      *  @param swapRB flag which indicates that swap first and last channels
      *  in 3-channel image is necessary.
      *  @param crop flag which indicates whether image will be cropped after resize or not
+     *  @param ddepth Depth of output blob. Choose CV_32F or CV_8U.
      *  @details if @p crop is true, input image is resized so one side after resize is equal to corresponding
      *  dimension in @p size and another one is equal or larger. Then, crop from the center is performed.
      *  If @p crop is false, direct resize without cropping and preserving aspect ratio is performed.
-     *  @returns 4-dimansional Mat with NCHW dimensions order.
+     *  @returns 4-dimensional Mat with NCHW dimensions order.
      */
     CV_EXPORTS_W Mat blobFromImages(InputArrayOfArrays images, double scalefactor=1.0,
-                                    Size size = Size(), const Scalar& mean = Scalar(), bool swapRB=true, bool crop=true);
+                                    Size size = Size(), const Scalar& mean = Scalar(), bool swapRB=true, bool crop=true,
+                                    int ddepth=CV_32F);
 
     /** @brief Creates 4-dimensional blob from series of images.
      *  @details This is an overloaded member function, provided for convenience.
@@ -847,7 +856,8 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
      */
     CV_EXPORTS void blobFromImages(InputArrayOfArrays images, OutputArray blob,
                                    double scalefactor=1.0, Size size = Size(),
-                                   const Scalar& mean = Scalar(), bool swapRB=true, bool crop=true);
+                                   const Scalar& mean = Scalar(), bool swapRB=true, bool crop=true,
+                                   int ddepth=CV_32F);
 
     /** @brief Parse a 4D blob and output the images it contains as 2D arrays through a simpler data structure
      *  (std::vector<cv::Mat>).

modules/dnn/src/op_inf_engine.cpp

@@ -68,19 +68,32 @@ static InferenceEngine::DataPtr wrapToInfEngineDataNode(const Mat& m, const std:
 {
     std::vector<size_t> reversedShape(&m.size[0], &m.size[0] + m.dims);
     std::reverse(reversedShape.begin(), reversedShape.end());
-    return InferenceEngine::DataPtr(
-        new InferenceEngine::Data(name, reversedShape, InferenceEngine::Precision::FP32, estimateLayout(m))
-    );
+    if (m.type() == CV_32F)
+        return InferenceEngine::DataPtr(
+            new InferenceEngine::Data(name, reversedShape, InferenceEngine::Precision::FP32, estimateLayout(m))
+        );
+    else if (m.type() == CV_8U)
+        return InferenceEngine::DataPtr(
+            new InferenceEngine::Data(name, reversedShape, InferenceEngine::Precision::U8, estimateLayout(m))
+        );
+    else
+        CV_Error(Error::StsNotImplemented, format("Unsupported data type %d", m.type()));
 }
 
-InferenceEngine::TBlob<float>::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector<size_t>& shape,
-                                                       InferenceEngine::Layout layout)
+InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector<size_t>& shape,
+                                               InferenceEngine::Layout layout)
 {
-    return InferenceEngine::make_shared_blob<float>(InferenceEngine::Precision::FP32,
-                                                    layout, shape, (float*)m.data);
+    if (m.type() == CV_32F)
+        return InferenceEngine::make_shared_blob<float>(InferenceEngine::Precision::FP32,
+                                                        layout, shape, (float*)m.data);
+    else if (m.type() == CV_8U)
+        return InferenceEngine::make_shared_blob<uint8_t>(InferenceEngine::Precision::U8,
+                                                          layout, shape, (uint8_t*)m.data);
+    else
+        CV_Error(Error::StsNotImplemented, format("Unsupported data type %d", m.type()));
 }
 
-InferenceEngine::TBlob<float>::Ptr wrapToInfEngineBlob(const Mat& m, InferenceEngine::Layout layout)
+InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, InferenceEngine::Layout layout)
 {
     std::vector<size_t> reversedShape(&m.size[0], &m.size[0] + m.dims);
     std::reverse(reversedShape.begin(), reversedShape.end());
@@ -102,6 +115,24 @@ InfEngineBackendWrapper::InfEngineBackendWrapper(int targetId, const cv::Mat& m)
     blob = wrapToInfEngineBlob(m, estimateLayout(m));
 }
 
+InfEngineBackendWrapper::InfEngineBackendWrapper(Ptr<BackendWrapper> wrapper)
+    : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE, wrapper->targetId)
+{
+    Ptr<InfEngineBackendWrapper> ieWrapper = wrapper.dynamicCast<InfEngineBackendWrapper>();
+    CV_Assert(!ieWrapper.empty());
+    InferenceEngine::DataPtr srcData = ieWrapper->dataPtr;
+    dataPtr = InferenceEngine::DataPtr(
+        new InferenceEngine::Data(srcData->name, srcData->dims, srcData->precision,
+                                  srcData->layout)
+    );
+    blob = ieWrapper->blob;
+}
+
+Ptr<BackendWrapper> InfEngineBackendWrapper::create(Ptr<BackendWrapper> wrapper)
+{
+    return Ptr<BackendWrapper>(new InfEngineBackendWrapper(wrapper));
+}
+
 InfEngineBackendWrapper::~InfEngineBackendWrapper()
 {
 
@@ -329,6 +360,7 @@ void InfEngineBackendNet::init(int targetId)
     {
         CV_Assert(allBlobs.find(it.first) != allBlobs.end());
         inpBlobs[it.first] = allBlobs[it.first];
+        it.second->setPrecision(inpBlobs[it.first]->precision());
     }
 
     // Set up output blobs.
@@ -427,7 +459,7 @@ void InfEngineBackendNet::addBlobs(const std::vector<Ptr<BackendWrapper> >& ptrs
     auto wrappers = infEngineWrappers(ptrs);
     for (const auto& wrapper : wrappers)
     {
-        allBlobs[wrapper->dataPtr->name] = wrapper->blob;
+        allBlobs.insert({wrapper->dataPtr->name, wrapper->blob});
     }
 }
 

modules/dnn/src/op_inf_engine.hpp

@@ -115,19 +115,23 @@ class InfEngineBackendWrapper : public BackendWrapper
 public:
     InfEngineBackendWrapper(int targetId, const Mat& m);
 
+    InfEngineBackendWrapper(Ptr<BackendWrapper> wrapper);
+
     ~InfEngineBackendWrapper();
 
+    static Ptr<BackendWrapper> create(Ptr<BackendWrapper> wrapper);
+
     virtual void copyToHost() CV_OVERRIDE;
 
     virtual void setHostDirty() CV_OVERRIDE;
 
     InferenceEngine::DataPtr dataPtr;
-    InferenceEngine::TBlob<float>::Ptr blob;
+    InferenceEngine::Blob::Ptr blob;
 };
 
-InferenceEngine::TBlob<float>::Ptr wrapToInfEngineBlob(const Mat& m, InferenceEngine::Layout layout = InferenceEngine::Layout::ANY);
+InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, InferenceEngine::Layout layout = InferenceEngine::Layout::ANY);
 
-InferenceEngine::TBlob<float>::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector<size_t>& shape, InferenceEngine::Layout layout);
+InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector<size_t>& shape, InferenceEngine::Layout layout);
 
 InferenceEngine::DataPtr infEngineDataNode(const Ptr<BackendWrapper>& ptr);
 

modules/dnn/test/test_halide_layers.cpp

@@ -107,12 +107,10 @@ TEST_P(Convolution, Accuracy)
     if (backendId == DNN_BACKEND_INFERENCE_ENGINE && targetId == DNN_TARGET_MYRIAD)
         throw SkipTestException("");
 
-    // TODO: unstable test cases
-    if (backendId == DNN_BACKEND_OPENCV && (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16) &&
-        inChannels == 6 && outChannels == 9 && group == 1 && inSize == Size(5, 6) &&
-        kernel == Size(3, 1) && stride == Size(1, 1) && pad == Size(0, 1) && dilation == Size(1, 1) &&
-        hasBias)
-        throw SkipTestException("");
+    if (cvtest::skipUnstableTests && backendId == DNN_BACKEND_OPENCV &&
+        (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16) &&
+        kernel == Size(3, 1) && stride == Size(1, 1) && pad == Size(0, 1))
+        throw SkipTestException("Skip unstable test");
 
     int sz[] = {outChannels, inChannels / group, kernel.height, kernel.width};
     Mat weights(4, &sz[0], CV_32F);

modules/dnn/test/test_layers.cpp

@@ -291,7 +291,7 @@ TEST_P(Test_Caffe_layers, Fused_Concat)
 
 TEST_P(Test_Caffe_layers, Eltwise)
 {
-    if (backend == DNN_BACKEND_INFERENCE_ENGINE)
+    if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
         throw SkipTestException("");
     testLayerUsingCaffeModels("layer_eltwise");
 }
@@ -939,6 +939,25 @@ TEST(Layer_Test_Convolution_DLDT, Accuracy)
     ASSERT_EQ(net.getLayer(outLayers[0])->type, "Concat");
 }
 
+TEST(Layer_Test_Convolution_DLDT, setInput_uint8)
+{
+    Mat inp = blobFromNPY(_tf("blob.npy"));
+
+    Mat inputs[] = {Mat(inp.dims, inp.size, CV_8U), Mat()};
+    randu(inputs[0], 0, 255);
+    inputs[0].convertTo(inputs[1], CV_32F);
+
+    Mat outs[2];
+    for (int i = 0; i < 2; ++i)
+    {
+        Net net = readNet(_tf("layer_convolution.xml"), _tf("layer_convolution.bin"));
+        net.setInput(inputs[i]);
+        outs[i] = net.forward();
+        ASSERT_EQ(outs[i].type(), CV_32F);
+    }
+    normAssert(outs[0], outs[1]);
+}
+
 // 1. Create a .prototxt file with the following network:
 // layer {
 //   type: "Input" name: "data" top: "data"
@@ -961,22 +980,65 @@ TEST(Layer_Test_Convolution_DLDT, Accuracy)
 // net.save('/path/to/caffemodel')
 //
 // 3. Convert using ModelOptimizer.
-TEST(Test_DLDT, two_inputs)
+typedef testing::TestWithParam<tuple<int, int> > Test_DLDT_two_inputs;
+TEST_P(Test_DLDT_two_inputs, as_IR)
 {
+    int firstInpType = get<0>(GetParam());
+    int secondInpType = get<1>(GetParam());
+    // TODO: It looks like a bug in Inference Engine.
+    if (secondInpType == CV_8U)
+        throw SkipTestException("");
+
     Net net = readNet(_tf("net_two_inputs.xml"), _tf("net_two_inputs.bin"));
     int inpSize[] = {1, 2, 3};
-    Mat firstInp(3, &inpSize[0], CV_32F);
-    Mat secondInp(3, &inpSize[0], CV_32F);
-    randu(firstInp, -1, 1);
-    randu(secondInp, -1, 1);
+    Mat firstInp(3, &inpSize[0], firstInpType);
+    Mat secondInp(3, &inpSize[0], secondInpType);
+    randu(firstInp, 0, 255);
+    randu(secondInp, 0, 255);
 
     net.setInput(firstInp, "data");
     net.setInput(secondInp, "second_input");
     Mat out = net.forward();
 
-    normAssert(out, firstInp + secondInp);
+    Mat ref;
+    cv::add(firstInp, secondInp, ref, Mat(), CV_32F);
+    normAssert(out, ref);
 }
 
+TEST_P(Test_DLDT_two_inputs, as_backend)
+{
+    static const float kScale = 0.5f;
+    static const float kScaleInv = 1.0f / kScale;
+
+    Net net;
+    LayerParams lp;
+    lp.type = "Eltwise";
+    lp.name = "testLayer";
+    lp.set("operation", "sum");
+    int eltwiseId = net.addLayerToPrev(lp.name, lp.type, lp);  // connect to a first input
+    net.connect(0, 1, eltwiseId, 1);  // connect to a second input
+
+    int inpSize[] = {1, 2, 3};
+    Mat firstInp(3, &inpSize[0], get<0>(GetParam()));
+    Mat secondInp(3, &inpSize[0], get<1>(GetParam()));
+    randu(firstInp, 0, 255);
+    randu(secondInp, 0, 255);
+
+    net.setInputsNames({"data", "second_input"});
+    net.setInput(firstInp, "data", kScale);
+    net.setInput(secondInp, "second_input", kScaleInv);
+    net.setPreferableBackend(DNN_BACKEND_INFERENCE_ENGINE);
+    Mat out = net.forward();
+
+    Mat ref;
+    addWeighted(firstInp, kScale, secondInp, kScaleInv, 0, ref, CV_32F);
+    normAssert(out, ref);
+}
+
+INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_DLDT_two_inputs, Combine(
+  Values(CV_8U, CV_32F), Values(CV_8U, CV_32F)
+));
+
 class UnsupportedLayer : public Layer
 {
 public:

modules/dnn/test/test_misc.cpp

@@ -138,4 +138,44 @@ TEST(LayerFactory, custom_layers)
     LayerFactory::unregisterLayer("CustomType");
 }
 
+typedef testing::TestWithParam<tuple<float, Vec3f, int, tuple<Backend, Target> > > setInput;
+TEST_P(setInput, normalization)
+{
+    const float kScale = get<0>(GetParam());
+    const Scalar kMean = get<1>(GetParam());
+    const int dtype    = get<2>(GetParam());
+    const int backend  = get<0>(get<3>(GetParam()));
+    const int target   = get<1>(get<3>(GetParam()));
+    const bool kSwapRB = true;
+
+    if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD && !checkMyriadTarget())
+        throw SkipTestException("Myriad is not available/disabled in OpenCV");
+    if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16 && dtype != CV_32F)
+        throw SkipTestException("");
+
+    Mat inp(5, 5, CV_8UC3);
+    randu(inp, 0, 255);
+    Mat ref = blobFromImage(inp, kScale, Size(), kMean, kSwapRB, /*crop*/false);
+
+    LayerParams lp;
+    Net net;
+    net.addLayerToPrev("testLayer", "Identity", lp);
+    net.setPreferableBackend(backend);
+    net.setPreferableTarget(target);
+
+    Mat blob = blobFromImage(inp, 1.0, Size(), Scalar(), kSwapRB, /*crop*/false, dtype);
+    ASSERT_EQ(blob.type(), dtype);
+    net.setInput(blob, "", kScale, kMean);
+    Mat out = net.forward();
+    ASSERT_EQ(out.type(), CV_32F);
+    normAssert(ref, out, "", 4e-4, 1e-3);
+}
+
+INSTANTIATE_TEST_CASE_P(/**/, setInput, Combine(
+  Values(1.0f, 1.0 / 127.5),
+  Values(Vec3f(), Vec3f(50, 50, 50), Vec3f(10, 50, 140)),
+  Values(CV_32F, CV_8U),
+  dnnBackendsAndTargets()
+));
+
 }} // namespace