fixed memory corruption when normal dist. params have less channels than target matrix; test added

modules/core/src/rand.cpp

@@ -624,7 +624,7 @@ void RNG::fill( InputOutputArray _mat, int disttype,
         int ptype = depth == CV_64F ? CV_64F : CV_32F;
         int esz = (int)CV_ELEM_SIZE(ptype);
 
-        if( _param1.isContinuous() && _param1.type() == ptype )
+        if( _param1.isContinuous() && _param1.type() == ptype && n1 >= cn)
             mean = _param1.ptr();
         else
         {
@@ -637,18 +637,18 @@ void RNG::fill( InputOutputArray _mat, int disttype,
             for( j = n1*esz; j < cn*esz; j++ )
                 mean[j] = mean[j - n1*esz];
 
-        if( _param2.isContinuous() && _param2.type() == ptype )
+        if( _param2.isContinuous() && _param2.type() == ptype && n2 >= cn)
             stddev = _param2.ptr();
         else
         {
-            Mat tmp(_param2.size(), ptype, parambuf + cn);
+            Mat tmp(_param2.size(), ptype, parambuf + MAX(n1, cn));
             _param2.convertTo(tmp, ptype);
-            stddev = (uchar*)(parambuf + cn);
+            stddev = (uchar*)(parambuf + MAX(n1, cn));
         }
 
-        if( n1 < cn )
-            for( j = n1*esz; j < cn*esz; j++ )
-                stddev[j] = stddev[j - n1*esz];
+        if( n2 < cn )
+            for( j = n2*esz; j < cn*esz; j++ )
+                stddev[j] = stddev[j - n2*esz];
 
         stdmtx = _param2.rows == cn && _param2.cols == cn;
         scaleFunc = randnScaleTab[depth];

modules/core/test/test_rand.cpp

@@ -365,3 +365,20 @@ TEST(Core_RNG_MT19937, regression)
         ASSERT_EQ(expected[i], actual[i]);
     }
 }
+
+
+TEST(Core_Rand, Regression_Stack_Corruption)
+{
+    int bufsz = 128; //enough for 14 doubles
+    AutoBuffer<uchar> buffer(bufsz);
+    size_t offset = 0;
+    cv::Mat_<cv::Point2d> x(2, 3, (cv::Point2d*)(buffer+offset)); offset += x.total()*x.elemSize();
+    double& param1 = *(double*)(buffer+offset); offset += sizeof(double);
+    double& param2 = *(double*)(buffer+offset); offset += sizeof(double);
+    param1 = -9; param2 = 2;
+
+    cv::theRNG().fill(x, cv::RNG::NORMAL, param1, param2);
+
+    ASSERT_EQ(param1, -9);
+    ASSERT_EQ(param2,  2);
+}