core: rework getContinuousSize() for vector-col/row support

modules/core/src/arithm.cpp

@@ -199,11 +199,9 @@ static void binary_op( InputArray _src1, InputArray _src2, OutputArray _dst,
             func = tab[depth1];
 
         Mat src1 = psrc1->getMat(), src2 = psrc2->getMat(), dst = _dst.getMat();
-        if (_dst.isVector() && dst.size() != src1.size())  // https://github.com/opencv/opencv/pull/4159
-            dst = dst.reshape(0, (int)dst.total());
-        Size sz = getContinuousSize(src1, src2, dst);
+        Size sz = getContinuousSize2D(src1, src2, dst);
         size_t len = sz.width*(size_t)cn;
-        if( len == (size_t)(int)len )
+        if (len < INT_MAX)  // FIXIT similar code below doesn't have that check
         {
             sz.width = (int)len;
             func(src1.ptr(), src1.step, src2.ptr(), src2.step, dst.ptr(), dst.step, sz.width, sz.height, 0);
@@ -632,9 +630,7 @@ static void arithm_op(InputArray _src1, InputArray _src2, OutputArray _dst,
                           usrdata, oclop, false))
 
         Mat src1 = psrc1->getMat(), src2 = psrc2->getMat(), dst = _dst.getMat();
-        if (_dst.isVector() && dst.size() != src1.size())  // https://github.com/opencv/opencv/pull/4159
-            dst = dst.reshape(0, (int)dst.total());
-        Size sz = getContinuousSize(src1, src2, dst, src1.channels());
+        Size sz = getContinuousSize2D(src1, src2, dst, src1.channels());
         tab[depth1](src1.ptr(), src1.step, src2.ptr(), src2.step, dst.ptr(), dst.step, sz.width, sz.height, usrdata);
         return;
     }
@@ -1283,9 +1279,7 @@ void cv::compare(InputArray _src1, InputArray _src2, OutputArray _dst, int op)
         int cn = src1.channels();
         _dst.create(src1.size(), CV_8UC(cn));
         Mat dst = _dst.getMat();
-        if (_dst.isVector() && dst.size() != src1.size())  // https://github.com/opencv/opencv/pull/4159
-            dst = dst.reshape(0, (int)dst.total());
-        Size sz = getContinuousSize(src1, src2, dst, src1.channels());
+        Size sz = getContinuousSize2D(src1, src2, dst, src1.channels());
         getCmpFunc(src1.depth())(src1.ptr(), src1.step, src2.ptr(), src2.step, dst.ptr(), dst.step, sz.width, sz.height, &op);
         return;
     }

modules/core/src/convert.cpp

@@ -450,9 +450,7 @@ void cv::Mat::convertTo(OutputArray _dst, int _type, double alpha, double beta)
 
     if( dims <= 2 )
     {
-        if (_dst.isVector() && dst.size() != src.size())  // https://github.com/opencv/opencv/pull/4159
-            dst = dst.reshape(0, (int)dst.total());
-        Size sz = getContinuousSize(src, dst, cn);
+        Size sz = getContinuousSize2D(src, dst, cn);
         func( src.data, src.step, 0, 0, dst.data, dst.step, sz, scale );
     }
     else
@@ -513,9 +511,7 @@ void cv::convertFp16( InputArray _src, OutputArray _dst )
 
     if( src.dims <= 2 )
     {
-        if (_dst.isVector() && dst.size() != src.size())  // https://github.com/opencv/opencv/pull/4159
-            dst = dst.reshape(0, (int)dst.total());
-        Size sz = getContinuousSize(src, dst, cn);
+        Size sz = getContinuousSize2D(src, dst, cn);
         func( src.data, src.step, 0, 0, dst.data, dst.step, sz, 0);
     }
     else

modules/core/src/convert_scale.cpp

@@ -427,9 +427,7 @@ void cv::convertScaleAbs( InputArray _src, OutputArray _dst, double alpha, doubl
 
     if( src.dims <= 2 )
     {
-        if (_dst.isVector() && dst.size() != src.size())  // https://github.com/opencv/opencv/pull/4159
-            dst = dst.reshape(0, (int)dst.total());
-        Size sz = getContinuousSize(src, dst, cn);
+        Size sz = getContinuousSize2D(src, dst, cn);
         func( src.ptr(), src.step, 0, 0, dst.ptr(), dst.step, sz, scale );
     }
     else

modules/core/src/copy.cpp

@@ -287,23 +287,19 @@ void Mat::copyTo( OutputArray _dst ) const
 
         if( rows > 0 && cols > 0 )
         {
-            // For some cases (with vector) dst.size != src.size, so force to column-based form
-            // It prevents memory corruption in case of column-based src
-            if (_dst.isVector() && dst.size() != size())  // https://github.com/opencv/opencv/pull/4159
-                dst = dst.reshape(0, (int)dst.total());
+            Mat src = *this;
+            Size sz = getContinuousSize2D(src, dst, (int)elemSize());
+            CV_CheckGE(sz.width, 0, "");
 
-            const uchar* sptr = data;
+            const uchar* sptr = src.data;
             uchar* dptr = dst.data;
 
 #if IPP_VERSION_X100 >= 201700
-            CV_IPP_RUN_FAST(CV_INSTRUMENT_FUN_IPP(ippiCopy_8u_C1R_L, sptr, (int)step, dptr, (int)dst.step, ippiSizeL((int)(cols*elemSize()), rows)) >= 0)
+            CV_IPP_RUN_FAST(CV_INSTRUMENT_FUN_IPP(ippiCopy_8u_C1R_L, sptr, (int)src.step, dptr, (int)dst.step, ippiSizeL(sz.width, sz.height)) >= 0)
 #endif
 
-            Size sz = getContinuousSize(*this, dst);
-            size_t len = sz.width*elemSize();
-
-            for( ; sz.height--; sptr += step, dptr += dst.step )
-                memcpy( dptr, sptr, len );
+            for (; sz.height--; sptr += src.step, dptr += dst.step)
+                memcpy(dptr, sptr, sz.width);
         }
         return;
     }
@@ -403,10 +399,9 @@ void Mat::copyTo( OutputArray _dst, InputArray _mask ) const
 
     if( dims <= 2 )
     {
-        if (_dst.isVector() && dst.size() != size())  // https://github.com/opencv/opencv/pull/4159
-            dst = dst.reshape(0, (int)dst.total());
-        Size sz = getContinuousSize(*this, dst, mask, mcn);
-        copymask(data, step, mask.data, mask.step, dst.data, dst.step, sz, &esz);
+        Mat src = *this;
+        Size sz = getContinuousSize2D(src, dst, mask, mcn);
+        copymask(src.data, src.step, mask.data, mask.step, dst.data, dst.step, sz, &esz);
         return;
     }
 

modules/core/src/mathfuncs.cpp

@@ -1489,7 +1489,7 @@ bool checkRange(InputArray _src, bool quiet, Point* pt, double minVal, double ma
     {
         int i, loc = 0;
         int cn = src.channels();
-        Size size = getContinuousSize( src, cn );
+        Size size = getContinuousSize2D(src, cn);
 
         if( depth == CV_32F )
         {

modules/core/src/matrix.cpp

@@ -943,4 +943,77 @@ int Mat::checkVector(int _elemChannels, int _depth, bool _requireContinuous) con
     ? (int)(total()*channels()/_elemChannels) : -1;
 }
 
+
+static inline Size getContinuousSize_(int flags, int cols, int rows, int widthScale)
+{
+    int64 sz = (int64)cols * rows * widthScale;
+    bool has_int_overflow = sz >= INT_MAX;
+    bool isContiguous = (flags & Mat::CONTINUOUS_FLAG) != 0;
+    return (isContiguous && !has_int_overflow)
+            ? Size((int)sz, 1)
+            : Size(cols * widthScale, rows);
+}
+
+Size getContinuousSize2D(Mat& m1, int widthScale)
+{
+    CV_CheckLE(m1.dims, 2, "");
+    return getContinuousSize_(m1.flags,
+                              m1.cols, m1.rows, widthScale);
+}
+Size getContinuousSize2D(Mat& m1, Mat& m2, int widthScale)
+{
+    CV_CheckLE(m1.dims, 2, "");
+    CV_CheckLE(m2.dims, 2, "");
+    const Size sz1 = m1.size();
+    if (sz1 != m2.size())  // reshape all matrixes to the same size (#4159)
+    {
+        size_t total_sz = m1.total();
+        CV_CheckEQ(total_sz, m2.total(), "");
+        bool is_m1_vector = m1.cols == 1 || m1.rows == 1;
+        bool is_m2_vector = m2.cols == 1 || m2.rows == 1;
+        CV_Assert(is_m1_vector); CV_Assert(is_m2_vector);
+        int total = (int)total_sz;  // vector-column
+        bool isContiguous = ((m1.flags & m2.flags) & Mat::CONTINUOUS_FLAG) != 0;
+        bool has_int_overflow = ((int64)total_sz * widthScale) >= INT_MAX;
+        if (isContiguous && !has_int_overflow)
+            total = 1; // vector-row
+        m1 = m1.reshape(0, total);
+        m2 = m2.reshape(0, total);
+        CV_Assert(m1.cols == m2.cols && m1.rows == m2.rows);
+        return Size(m1.cols * widthScale, m1.rows);
+    }
+    return getContinuousSize_(m1.flags & m2.flags,
+                              m1.cols, m1.rows, widthScale);
+}
+
+Size getContinuousSize2D(Mat& m1, Mat& m2, Mat& m3, int widthScale)
+{
+    CV_CheckLE(m1.dims, 2, "");
+    CV_CheckLE(m2.dims, 2, "");
+    CV_CheckLE(m3.dims, 2, "");
+    const Size sz1 = m1.size();
+    if (sz1 != m2.size() || sz1 != m3.size())  // reshape all matrixes to the same size (#4159)
+    {
+        size_t total_sz = m1.total();
+        CV_CheckEQ(total_sz, m2.total(), "");
+        CV_CheckEQ(total_sz, m3.total(), "");
+        bool is_m1_vector = m1.cols == 1 || m1.rows == 1;
+        bool is_m2_vector = m2.cols == 1 || m2.rows == 1;
+        bool is_m3_vector = m3.cols == 1 || m3.rows == 1;
+        CV_Assert(is_m1_vector); CV_Assert(is_m2_vector); CV_Assert(is_m3_vector);
+        int total = (int)total_sz;  // vector-column
+        bool isContiguous = ((m1.flags & m2.flags & m3.flags) & Mat::CONTINUOUS_FLAG) != 0;
+        bool has_int_overflow = ((int64)total_sz * widthScale) >= INT_MAX;
+        if (isContiguous && !has_int_overflow)
+            total = 1; // vector-row
+        m1 = m1.reshape(0, total);
+        m2 = m2.reshape(0, total);
+        m3 = m3.reshape(0, total);
+        CV_Assert(m1.cols == m2.cols && m1.rows == m2.rows && m1.cols == m3.cols && m1.rows == m3.rows);
+        return Size(m1.cols * widthScale, m1.rows);
+    }
+    return getContinuousSize_(m1.flags & m2.flags & m3.flags,
+                              m1.cols, m1.rows, widthScale);
+}
+
 } // cv::

modules/core/src/precomp.hpp

@@ -248,34 +248,12 @@ BinaryFunc getCopyMaskFunc(size_t esz);
 /* maximal average node_count/hash_size ratio beyond which hash table is resized */
 #define  CV_SPARSE_HASH_RATIO    3
 
-inline Size getContinuousSize_( int flags, int cols, int rows, int widthScale )
-{
-    int64 sz = (int64)cols * rows * widthScale;
-    return (flags & Mat::CONTINUOUS_FLAG) != 0 &&
-        (int)sz == sz ? Size((int)sz, 1) : Size(cols * widthScale, rows);
-}
-
-inline Size getContinuousSize( const Mat& m1, int widthScale=1 )
-{
-    return getContinuousSize_(m1.flags,
-                              m1.cols, m1.rows, widthScale);
-}
-
-inline Size getContinuousSize( const Mat& m1, const Mat& m2, int widthScale=1 )
-{
-    CV_Assert(m1.size() == m2.size());
-    return getContinuousSize_(m1.flags & m2.flags,
-                              m1.cols, m1.rows, widthScale);
-}
-
-inline Size getContinuousSize( const Mat& m1, const Mat& m2,
-                               const Mat& m3, int widthScale=1 )
-{
-    CV_Assert(m1.size() == m2.size());
-    CV_Assert(m1.size() == m3.size());
-    return getContinuousSize_(m1.flags & m2.flags & m3.flags,
-                              m1.cols, m1.rows, widthScale);
-}
+// There is some mess in code with vectors representation.
+// Both vector-column / vector-rows are used with dims=2 (as Mat2D always).
+// Reshape matrices if neccessary (in case of vectors) and returns size with scaled width.
+Size getContinuousSize2D(Mat& m1, int widthScale=1);
+Size getContinuousSize2D(Mat& m1, Mat& m2, int widthScale=1);
+Size getContinuousSize2D(Mat& m1, Mat& m2, Mat& m3, int widthScale=1);
 
 void setSize( Mat& m, int _dims, const int* _sz, const size_t* _steps, bool autoSteps=false );
 void finalizeHdr(Mat& m);