Adding of transperent Mat/UMat switching for Blob API

modules/dnn/include/opencv2/dnn/blob.hpp

@@ -144,19 +144,25 @@ namespace dnn
     class CV_EXPORTS_W Blob
     {
     public:
-        explicit Blob();
+        Blob();
 
         /** @brief Constructs blob with specified @p shape and @p type. */
-        explicit Blob(const BlobShape &shape, int type = CV_32F);
+        explicit Blob(const BlobShape &shape, int type = CV_32F, int allocFlags = ALLOC_MAT);
+
+        /** @brief Constructs Blob from existing Mat or UMat. */
+        Blob(InputArray data);
 
         /** @brief Constucts 4-dimensional blob (so-called batch) from image or array of images.
-         * @param image 2-dimensional multi-channel or 3-dimensional single-channel image (or array of images)
-         * @param dstCn specify size of second axis of ouptut blob
-        */
-        explicit Blob(InputArray image, int dstCn = -1);
+         * @param image 2-dimensional multi-channel or 3-dimensional single-channel image (or array of such images)
+         * @param dstCn specifies size of second axis of ouptut blob
+         */
+        static Blob fromImages(InputArray image, int dstCn = -1);
+
+        /** @brief Works like Blob::fromImages() but in-place. */
+        void batchFromImages(InputArray image, int dstCn = -1);
 
         /** @brief Creates blob with specified @p shape and @p type. */
-        void create(const BlobShape &shape, int type = CV_32F);
+        void create(const BlobShape &shape, int type = CV_32F, int allocFlags = ALLOC_MAT);
 
         /** @brief Creates blob from cv::Mat or cv::UMat without copying the data */
         void fill(InputArray in);
@@ -165,10 +171,19 @@ namespace dnn
          */
         void fill(const BlobShape &shape, int type, void *data, bool deepCopy = true);
 
-        Mat& matRef();                      //!< Returns reference to cv::Mat, containing blob data.
-        const Mat& matRefConst() const;     //!< Returns reference to cv::Mat, containing blob data, for read-only purposes.
-        UMat &umatRef();                    //!< Returns reference to cv::UMat, containing blob data (not implemented yet).
-        const UMat &umatRefConst() const;   //!< Returns reference to cv::UMat, containing blob data, for read-only purposes (not implemented yet).
+        Mat& matRef(bool writeOnly = true);     //!< Returns reference to cv::Mat, containing blob data.
+        const Mat& matRefConst() const;         //!< Returns reference to cv::Mat, containing blob data, for read-only purposes.
+        UMat &umatRef(bool writeOnly = true);   //!< Returns reference to cv::UMat, containing blob data.
+        const UMat &umatRefConst() const;       //!< Returns reference to cv::UMat, containing blob data, for read-only purposes.
+
+        template<typename XMat>
+        XMat &getRef(bool writeOnly = true);
+        template<typename XMat>
+        const XMat &getRefConst() const;
+
+        void updateMat(bool syncData = true) const;     //!< Actualizes data stored inside Mat of Blob; if @p syncData is false then only shape will be actualized.
+        void updateUMat(bool syncData = true) const;    //!< Actualizes data stored inside Mat of Blob; if @p syncData is false then only shape will be actualized.
+        void sync() const;                              //!< Updates Mat and UMat of Blob.
 
         /** @brief Returns number of blob dimensions. */
         int dims() const;
@@ -204,11 +219,6 @@ namespace dnn
         /** @brief Checks equality of two blobs shapes. */
         bool equalShape(const Blob &other) const;
 
-        /** @brief Returns slice of first two dimensions.
-         *  @details The behaviour is similar to the following numpy code: blob[n, cn, ...]
-         */
-        Mat getPlane(int n, int cn);
-
         /* Shape getters of 4-dimensional blobs. */
         int cols() const;       //!< Returns size of the fourth axis blob.
         int rows() const;       //!< Returns size of the thrid  axis blob.
@@ -236,12 +246,17 @@ namespace dnn
          */
         uchar *ptr(int n = 0, int cn = 0, int row = 0, int col = 0);
         /** @overload */
-        template<typename TFloat>
-        TFloat *ptr(int n = 0, int cn = 0, int row = 0, int col = 0);
+        template<typename Type>
+        Type *ptr(int n = 0, int cn = 0, int row = 0, int col = 0);
         /** @overload ptr<float>() */
         float *ptrf(int n = 0, int cn = 0, int row = 0, int col = 0);
         //TODO: add const ptr methods
 
+        /** @brief Returns slice of first two dimensions.
+         *  @details The behaviour is similar to the following numpy code: blob[n, cn, ...]
+         */
+        Mat getPlane(int n, int cn);
+
         /** @brief Shares data from other @p blob.
          * @returns *this
          */
@@ -257,13 +272,45 @@ namespace dnn
          */
         Blob reshaped(const BlobShape &newShape) const;
 
-        /** @brief Returns type of the blob. */
-        int type() const;
+        int type() const;       //!< Returns type of the blob.
+        int elemSize() const;   //!< Returns size of single element in bytes.
 
     private:
         const int *sizes() const;
 
+#   define CV_DNN_UMAT //DBG
+#ifdef HAVE_OPENCL
+#   define CV_DNN_UMAT
+#endif
+
+#ifdef CV_DNN_UMAT
+#   define CV_DNN_UMAT_ONLY(expr) (expr)
+#else
+#   define CV_DNN_UMAT_ONLY(expr)
+#endif
+
+#ifndef CV_DNN_UMAT
         Mat m;
+#else
+        mutable Mat m;
+        mutable UMat um;
+        mutable uchar state;
+#endif
+
+        enum DataState
+        {
+            UNINITIALIZED,
+            HEAD_AT_MAT,
+            HEAD_AT_UMAT,
+            SYNCED
+        };
+
+        enum AllocFlag
+        {
+            ALLOC_MAT  = 1,
+            ALLOC_UMAT = 2,
+            ALLOC_BOTH = 3
+        };
     };
 
 //! @}

modules/dnn/include/opencv2/dnn/blob.inl.hpp

@@ -247,19 +247,63 @@ inline bool BlobShape::isEmpty() const
     return dims() == 0;
 }
 
+inline BlobShape BlobShape::operator+(const BlobShape &r) const
+{
+    BlobShape newShape(this->dims() + r.dims(), (int*)NULL);
+    for (int i = 0; i < this->dims(); i++)
+        newShape[i] = (*this)[i];
+    for (int i = 0; i < r.dims(); i++)
+        newShape[this->dims() + i] = r[i];
+    return newShape;
+}
+
 CV_EXPORTS std::ostream &operator<< (std::ostream &stream, const BlobShape &shape);
 
 /////////////////////////////////////////////////////////////////////
 
-inline int Blob::canonicalAxis(int axis) const
+#ifndef CV_DNN_UMAT
+#   define CV_DNN_SWITCH_MU(cpu_expr, gpu_expr) (cpu_expr)
+#else
+#   define CV_DNN_SWITCH_MU(cpu_expr, gpu_expr) ((state == HEAD_AT_UMAT) ? (gpu_expr) : (cpu_expr))
+#endif
+
+
+inline int Blob::dims() const
 {
-    CV_Assert(-dims() <= axis && axis < dims());
-    return (axis < 0) ? axis + dims() : axis;
+    return CV_DNN_SWITCH_MU(m.dims, um.dims);
 }
 
-inline int Blob::dims() const
+inline const int * Blob::sizes() const
+{
+    return CV_DNN_SWITCH_MU((const int*)m.size, (const int*)um.size);
+}
+
+inline int Blob::type() const
 {
-    return m.dims;
+    return CV_DNN_SWITCH_MU(m.type(), um.type());
+}
+
+template<int n>
+inline size_t Blob::offset(const Vec<int, n> &pos) const
+{
+    const MatStep &step = CV_DNN_SWITCH_MU(m.step, um.step);
+    size_t ofs = 0;
+    int i;
+    for (i = 0; i < std::min(n, dims()); i++)
+    {
+        CV_DbgAssert(pos[i] >= 0 && pos[i] < size(i));
+        ofs += step[i] * pos[i];
+    }
+    for (; i < dims(); i++)
+        CV_DbgAssert(pos[i] == 0);
+    CV_DbgAssert(ofs % elemSize() == 0);
+    return ofs / elemSize();
+}
+
+inline int Blob::canonicalAxis(int axis) const
+{
+    CV_Assert(-dims() <= axis && axis < dims());
+    return (axis < 0) ? axis + dims() : axis;
 }
 
 inline int Blob::xsize(int axis) const
@@ -295,22 +339,6 @@ inline size_t Blob::total(int startAxis, int endAxis) const
     return cnt;
 }
 
-
-template<int n>
-inline size_t Blob::offset(const Vec<int, n> &pos) const
-{
-    size_t ofs = 0;
-    int i;
-    for (i = 0; i < std::min(n, dims()); i++)
-    {
-        CV_DbgAssert(pos[i] >= 0 && pos[i] < size(i));
-        ofs = ofs * (size_t)size(i) + pos[i];
-    }
-    for (; i < dims(); i++)
-        ofs *= (size_t)size(i);
-    return ofs;
-}
-
 inline size_t Blob::offset(int n, int cn, int row, int col) const
 {
     return offset(Vec4i(n, cn, row, col));
@@ -318,20 +346,20 @@ inline size_t Blob::offset(int n, int cn, int row, int col) const
 
 inline float *Blob::ptrf(int n, int cn, int row, int col)
 {
-    CV_Assert(type() == CV_32F);
-    return (float*)m.data + offset(n, cn, row, col);
+    return matRef().ptr<float>() + offset(n, cn, row, col);
 }
 
 inline uchar *Blob::ptr(int n, int cn, int row, int col)
 {
-    return m.data + m.elemSize() * offset(n, cn, row, col);
+    Mat &mat = matRef();
+    return mat.ptr() + mat.elemSize() * offset(n, cn, row, col);
 }
 
-template<typename TFloat>
-inline TFloat* Blob::ptr(int n, int cn, int row, int col)
+template<typename Dtype>
+inline Dtype* Blob::ptr(int n, int cn, int row, int col)
 {
-    CV_Assert(type() == cv::DataDepth<TFloat>::value);
-    return (TFloat*) ptr(n, cn, row, col);
+    CV_Assert(type() == cv::DataDepth<Dtype>::value);
+    return (Dtype*) ptr(n, cn, row, col);
 }
 
 inline BlobShape Blob::shape() const
@@ -339,17 +367,6 @@ inline BlobShape Blob::shape() const
     return BlobShape(dims(), sizes());
 }
 
-
-inline BlobShape BlobShape::operator+(const BlobShape &r) const
-{
-    BlobShape newShape(this->dims() + r.dims(), (int*)NULL);
-    for (int i = 0; i < this->dims(); i++)
-        newShape[i] = (*this)[i];
-    for (int i = 0; i < r.dims(); i++)
-        newShape[this->dims() + i] = r[i];
-    return newShape;
-}
-
 inline bool Blob::equalShape(const Blob &other) const
 {
     if (this->dims() != other.dims())
@@ -363,26 +380,69 @@ inline bool Blob::equalShape(const Blob &other) const
     return true;
 }
 
-inline Mat& Blob::matRef()
+inline Mat& Blob::matRef(bool writeOnly)
 {
+#ifdef CV_DNN_UMAT
+    updateMat(!writeOnly);
+    state = HEAD_AT_MAT;
+#else
+    (void)writeOnly;
+#endif
     return m;
 }
 
 inline const Mat& Blob::matRefConst() const
 {
+    CV_DNN_UMAT_ONLY( updateMat() );
     return m;
 }
 
-inline UMat &Blob::umatRef()
+inline UMat &Blob::umatRef(bool writeOnly)
 {
-    CV_Error(Error::StsNotImplemented, "");
+#ifndef CV_DNN_UMAT
+    CV_Error(Error::GpuNotSupported, "");
+    (void)writeOnly;
     return *(new UMat());
+#else
+    updateUMat(!writeOnly);
+    state = HEAD_AT_UMAT;
+    return um;
+#endif
 }
 
 inline const UMat &Blob::umatRefConst() const
 {
-    CV_Error(Error::StsNotImplemented, "");
+#ifndef CV_DNN_UMAT
+    CV_Error(Error::GpuNotSupported, "");
     return *(new UMat());
+#else
+    updateUMat();
+    return um;
+#endif
+}
+
+template<>
+inline Mat &Blob::getRef<Mat>(bool writeOnly)
+{
+    return matRef(writeOnly);
+}
+
+template<>
+inline UMat &Blob::getRef<UMat>(bool writeOnly)
+{
+    return umatRef(writeOnly);
+}
+
+template<>
+inline const Mat &Blob::getRefConst<Mat>() const
+{
+    return matRefConst();
+}
+
+template<>
+inline const UMat &Blob::getRefConst<UMat>() const
+{
+    return umatRefConst();
 }
 
 inline Mat Blob::getPlane(int n, int cn)
@@ -416,26 +476,22 @@ inline Size Blob::size2() const
     return Size(cols(), rows());
 }
 
-inline int Blob::type() const
-{
-    return m.depth();
-}
-
-inline const int * Blob::sizes() const
-{
-    return &m.size[0];
-}
-
-
 inline Blob &Blob::shareFrom(const Blob &blob)
 {
     this->m = blob.m;
+#ifdef CV_DNN_UMAT
+    this->um = blob.um;
+    this->state = blob.state;
+#endif
     return *this;
 }
 
 inline Blob &Blob::reshape(const BlobShape &newShape)
 {
-    m = m.reshape(1, newShape.dims(), newShape.ptr());
+    if (!m.empty()) m = m.reshape(1, newShape.dims(), newShape.ptr());
+#ifdef CV_DNN_UMAT
+    if (!um.empty()) um = um.reshape(1, newShape.dims(), newShape.ptr());
+#endif
     return *this;
 }
 
@@ -446,6 +502,11 @@ inline Blob Blob::reshaped(const BlobShape &newShape) const
     return res;
 }
 
+inline int Blob::elemSize() const
+{
+    return CV_ELEM_SIZE(type());
+}
+
 }
 }
 

modules/dnn/samples/caffe_googlenet.cpp

@@ -124,8 +124,8 @@ int main(int argc, char **argv)
         exit(-1);
     }
 
-    resize(img, img, Size(224, 224));       //GoogLeNet accepts only 224x224 RGB-images
-    dnn::Blob inputBlob = dnn::Blob(img);   //Convert Mat to dnn::Blob image batch
+    resize(img, img, Size(224, 224));                   //GoogLeNet accepts only 224x224 RGB-images
+    dnn::Blob inputBlob = dnn::Blob::fromImages(img);   //Convert Mat to dnn::Blob batch of images
     //! [Prepare blob]
 
     //! [Set input blob]

modules/dnn/src/blob.cpp

@@ -48,8 +48,57 @@ namespace dnn
 
     Blob::Blob()
     {
-        int zeros[4] = { 0, 0, 0, 0 };
-        m = Mat(4, zeros, CV_32F, NULL);
+        CV_DNN_UMAT_ONLY(state = UNINITIALIZED);
+    }
+
+    Blob::Blob(const BlobShape &shape, int type, int allocFlags)
+    {
+        CV_DNN_UMAT_ONLY(state = UNINITIALIZED);
+        this->create(shape, type, allocFlags);
+    }
+
+    Blob::Blob(InputArray data)
+    {
+#ifndef CV_DNN_UMAT
+        m = data.getMat();
+#else
+        CV_Assert(data.isMat() || data.isUMat());
+        if (data.isMat())
+        {
+            m = data.getMat();
+            state = HEAD_AT_MAT;
+        }
+        else
+        {
+            um = data.getUMat();
+            state = HEAD_AT_UMAT;
+        }
+#endif
+    }
+
+    void Blob::create(const BlobShape &shape, int type, int allocFlags)
+    {
+#ifndef CV_DNN_UMAT
+        CV_Assert(allocFlags & ALLOC_MAT);
+        m.create(shape.dims(), shape.ptr(), type);
+#else
+        CV_Assert(allocFlags & ALLOC_MAT || allocFlags & ALLOC_UMAT);
+
+        if (allocFlags & ALLOC_MAT)
+            m.create(shape.dims(), shape.ptr(), type);
+        if (allocFlags & ALLOC_UMAT)
+            um.create(shape.dims(), shape.ptr(), type);
+
+        if (state == UNINITIALIZED)
+        {
+            if (allocFlags & ALLOC_MAT && allocFlags & ALLOC_UMAT)
+                state = SYNCED;
+            else if (allocFlags & ALLOC_MAT)
+                state = HEAD_AT_MAT;
+            else
+                state = HEAD_AT_UMAT;
+        }
+#endif
     }
 
     static inline int getMatChannels(const Mat &mat)
@@ -119,83 +168,115 @@ namespace dnn
         }
     }
 
-    Blob::Blob(InputArray image, int dstCn)
+    void Blob::batchFromImages(InputArray image, int dstCn)
     {
         CV_Assert(dstCn == -1 || dstCn > 0);
         std::vector<Mat> inMats = extractMatVector(image);
         BlobShape dstShape = getBlobShape(inMats, dstCn);
 
-        m.create(dstShape.dims(), dstShape.ptr(), CV_32F);
+        int dtype = CV_32F;
+        this->create(dstShape, dtype, ALLOC_MAT);
+        uchar *dstPtr = this->matRef().ptr();
+        int elemSize = CV_ELEM_SIZE(dtype);
 
         std::vector<Mat> wrapBuf(dstShape[-3]);
-        int elemSize = (int)m.elemSize();
-        uchar *ptr = this->ptr();
         for (size_t i = 0; i < inMats.size(); i++)
         {
             Mat inMat = inMats[i];
 
             if (inMat.dims <= 2)
             {
-                inMat.convertTo(inMat, m.type());
+                inMat.convertTo(inMat, dtype);
 
                 wrapBuf.resize(0);
                 for (int cn = 0; cn < inMat.channels(); cn++)
                 {
-                    wrapBuf.push_back(Mat(inMat.rows, inMat.cols, m.type(), ptr));
-                    ptr += elemSize * inMat.total();
+                    wrapBuf.push_back(Mat(inMat.rows, inMat.cols, dtype, dstPtr));
+                    dstPtr += elemSize * inMat.total();
                 }
 
                 cv::split(inMat, wrapBuf);
             }
             else
             {
-                inMat.convertTo(Mat(inMat.dims, inMat.size, m.type(), ptr), m.type());
-                ptr += elemSize * inMat.total();
+                inMat.convertTo(Mat(inMat.dims, inMat.size, dtype, dstPtr), dtype);
+                dstPtr += elemSize * inMat.total();
             }
         }
     }
 
-    Blob::Blob(const BlobShape &shape, int type)
+    Blob Blob::fromImages(InputArray image, int dstCn)
     {
-        this->create(shape, type);
+        Blob res;
+        res.batchFromImages(image, dstCn);
+        return res;
     }
 
     void Blob::fill(const BlobShape &shape, int type, void *data, bool deepCopy)
     {
-        CV_Assert(type == CV_32F || type == CV_64F);
-
         if (deepCopy)
         {
-            m.create(shape.dims(), shape.ptr(), type);
-            memcpy(m.data, data, m.total() * m.elemSize());
+            create(shape, type);
+            memcpy(ptr(), data, this->total() * CV_ELEM_SIZE(type));
         }
         else
         {
             m = Mat(shape.dims(), shape.ptr(), type, data);
         }
+        CV_DNN_UMAT_ONLY(state = HEAD_AT_MAT);
     }
 
-    void Blob::create(const BlobShape &shape, int type)
+    void Blob::updateMat(bool syncData) const
     {
-        CV_Assert(type == CV_32F || type == CV_64F);
-        m.create(shape.dims(), shape.ptr(), type);
+#ifdef CV_DNN_UMAT
+        if (state == UNINITIALIZED || state == SYNCED || state == HEAD_AT_MAT)
+        {
+            return;
+        }
+        else if (state == HEAD_AT_UMAT)
+        {
+            if (syncData)
+                um.copyTo(m);
+            else
+                m.create(dims(), sizes(), type());
+            state = SYNCED;
+        }
+        else
+        {
+            CV_Error(Error::StsInternal, "");
+        }
+#else
+        (void)syncData;
+#endif
     }
 
-    inline void squeezeShape(const int srcDims, const int *srcSizes, const int dstDims, int *dstSizes)
+    void Blob::updateUMat(bool syncData) const
     {
-        const int m = std::min(dstDims, srcDims);
-
-        //copy common(last) dimensions
-        for (int i = 0; i < m; i++)
-            dstSizes[dstDims - 1 - i] = srcSizes[srcDims - 1 - i];
-
-        //either flatten extra dimensions
-        for (int i = m; i < srcDims; i++)
-            dstSizes[0] *= srcSizes[srcDims - 1 - i];
+#ifdef CV_DNN_UMAT
+        if (state == UNINITIALIZED || state == SYNCED || state == HEAD_AT_UMAT)
+        {
+            return;
+        }
+        else if (state == HEAD_AT_MAT)
+        {
+            if (syncData)
+                m.copyTo(um);
+            else
+                um.create(dims(), sizes(), type());
+        }
+        else
+        {
+            CV_Error(Error::StsInternal, "");
+        }
+#else
+        (void)syncData;
+#endif
+    }
 
-        //either fill gaps
-        for (int i = m; i < dstDims; i++)
-            dstSizes[dstDims - 1 - i] = 1;
+    void Blob::sync() const
+    {
+        updateMat();
+        updateUMat();
     }
 
     Vec4i Blob::shape4() const

modules/dnn/test/test_googlenet.cpp

@@ -69,7 +69,7 @@ TEST(Reproducibility_GoogLeNet, Accuracy)
     inpMats.push_back( imread(_tf("googlenet_1.jpg")) );
     ASSERT_TRUE(!inpMats[0].empty() && !inpMats[1].empty());
 
-    net.setBlob(".data", Blob(inpMats));
+    net.setBlob(".data", Blob::fromImages(inpMats));
     net.forward();
 
     Blob out = net.getBlob("prob");