Merge pull request #9238 from alalek:valgrind_fixes

cmake/OpenCVCompilerOptions.cmake

@@ -203,7 +203,10 @@ if(CMAKE_COMPILER_IS_GNUCXX)
   endif()
 
   set(OPENCV_EXTRA_FLAGS_RELEASE "${OPENCV_EXTRA_FLAGS_RELEASE} -DNDEBUG")
-  set(OPENCV_EXTRA_FLAGS_DEBUG "${OPENCV_EXTRA_FLAGS_DEBUG} -O0 -DDEBUG -D_DEBUG")
+  if(NOT " ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_DEBUG} " MATCHES "-O")
+    set(OPENCV_EXTRA_FLAGS_DEBUG "${OPENCV_EXTRA_FLAGS_DEBUG} -O0")
+  endif()
+  set(OPENCV_EXTRA_FLAGS_DEBUG "${OPENCV_EXTRA_FLAGS_DEBUG} -DDEBUG -D_DEBUG")
 endif()
 
 if(MSVC)

cmake/OpenCVUtils.cmake

@@ -533,6 +533,12 @@ macro(ocv_finalize_status)
       execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different "${OPENCV_BUILD_INFO_FILE}" "${OPENCV_MODULE_opencv_core_BINARY_DIR}/version_string.inc" OUTPUT_QUIET)
     endif()
   endif()
+
+  if(UNIX)
+    install(FILES "${OpenCV_SOURCE_DIR}/platforms/scripts/valgrind.supp"
+                  "${OpenCV_SOURCE_DIR}/platforms/scripts/valgrind_3rdparty.supp"
+            DESTINATION "${OPENCV_OTHER_INSTALL_PATH}" COMPONENT "dev")
+  endif()
 endmacro()
 
 

modules/calib3d/src/stereobm.cpp

@@ -197,7 +197,7 @@ prefilterXSobel( const Mat& src, Mat& dst, int ftzero )
 {
     int x, y;
     const int OFS = 256*4, TABSZ = OFS*2 + 256;
-    uchar tab[TABSZ];
+    uchar tab[TABSZ] = { 0 };
     Size size = src.size();
 
     for( x = 0; x < TABSZ; x++ )
@@ -227,7 +227,7 @@ prefilterXSobel( const Mat& src, Mat& dst, int ftzero )
             v_int16x8 ftz2 = v_setall_s16((short)(ftzero*2));
             v_int16x8 z = v_setzero_s16();
 
-            for(; x <= size.width-8; x += 8 )
+            for(; x <= (size.width - 1) - 8; x += 8 )
             {
                 v_int16x8 s00 = v_reinterpret_as_s16(v_load_expand(srow0 + x + 1));
                 v_int16x8 s01 = v_reinterpret_as_s16(v_load_expand(srow0 + x - 1));

modules/core/src/stat.cpp

@@ -1547,7 +1547,7 @@ static bool ocl_meanStdDev( InputArray _src, OutputArray _mean, OutputArray _sdv
 
     bool haveMask = _mask.kind() != _InputArray::NONE;
     int nz = haveMask ? -1 : (int)_src.total();
-    Scalar mean, stddev;
+    Scalar mean(0), stddev(0);
     const int cn = _src.channels();
     if (cn > 4)
         return false;

modules/dnn/src/init.cpp

@@ -42,6 +42,8 @@
 #include "precomp.hpp"
 #include <opencv2/dnn/layer.details.hpp>
 
+#include <google/protobuf/stubs/common.h>
+
 namespace cv {
 namespace dnn {
 CV__DNN_EXPERIMENTAL_NS_BEGIN
@@ -56,11 +58,26 @@ Mutex& getInitializationMutex()
 // force initialization (single-threaded environment)
 Mutex* __initialization_mutex_initializer = &getInitializationMutex();
 
+namespace {
+using namespace google::protobuf;
+class ProtobufShutdown {
+public:
+    bool initialized;
+    ProtobufShutdown() : initialized(true) {}
+    ~ProtobufShutdown()
+    {
+        initialized = false;
+        google::protobuf::ShutdownProtobufLibrary();
+    }
+};
+} // namespace
 
 void initializeLayerFactory()
 {
     CV_TRACE_FUNCTION();
 
+    static ProtobufShutdown protobufShutdown; (void)protobufShutdown;
+
     CV_DNN_REGISTER_LAYER_CLASS(Slice,          SliceLayer);
     CV_DNN_REGISTER_LAYER_CLASS(Split,          SplitLayer);
     CV_DNN_REGISTER_LAYER_CLASS(Concat,         ConcatLayer);

modules/flann/include/opencv2/flann/lsh_table.h

@@ -147,6 +147,7 @@ public:
     LshTable()
     {
         key_size_ = 0;
+        feature_size_ = 0;
         speed_level_ = kArray;
     }
 
@@ -157,7 +158,7 @@ public:
      */
     LshTable(unsigned int feature_size, unsigned int key_size)
     {
-        (void)feature_size;
+        feature_size_ = feature_size;
         (void)key_size;
         std::cerr << "LSH is not implemented for that type" << std::endl;
         assert(0);
@@ -332,6 +333,8 @@ private:
      */
     unsigned int key_size_;
 
+    unsigned int feature_size_;
+
     // Members only used for the unsigned char specialization
     /** The mask to apply to a feature to get the hash key
      * Only used in the unsigned char case
@@ -345,9 +348,10 @@ private:
 template<>
 inline LshTable<unsigned char>::LshTable(unsigned int feature_size, unsigned int subsignature_size)
 {
+    feature_size_ = feature_size;
     initialize(subsignature_size);
     // Allocate the mask
-    mask_ = std::vector<size_t>((size_t)ceil((float)(feature_size * sizeof(char)) / (float)sizeof(size_t)), 0);
+    mask_ = std::vector<size_t>((feature_size * sizeof(char) + sizeof(size_t) - 1) / sizeof(size_t), 0);
 
     // A bit brutal but fast to code
     std::vector<size_t> indices(feature_size * CHAR_BIT);
@@ -392,6 +396,7 @@ inline size_t LshTable<unsigned char>::getKey(const unsigned char* feature) cons
 {
     // no need to check if T is dividable by sizeof(size_t) like in the Hamming
     // distance computation as we have a mask
+    // FIXIT: This is bad assumption, because we reading tail bytes after of the allocated features buffer
     const size_t* feature_block_ptr = reinterpret_cast<const size_t*> ((const void*)feature);
 
     // Figure out the subsignature of the feature
@@ -400,10 +405,20 @@ inline size_t LshTable<unsigned char>::getKey(const unsigned char* feature) cons
     size_t subsignature = 0;
     size_t bit_index = 1;
 
-    for (std::vector<size_t>::const_iterator pmask_block = mask_.begin(); pmask_block != mask_.end(); ++pmask_block) {
+    for (unsigned i = 0; i < feature_size_; i += sizeof(size_t)) {
         // get the mask and signature blocks
-        size_t feature_block = *feature_block_ptr;
-        size_t mask_block = *pmask_block;
+        size_t feature_block;
+        if (i <= feature_size_ - sizeof(size_t))
+        {
+            feature_block = *feature_block_ptr;
+        }
+        else
+        {
+            size_t tmp = 0;
+            memcpy(&tmp, feature_block_ptr, feature_size_ - i); // preserve bytes order
+            feature_block = tmp;
+        }
+        size_t mask_block = mask_[i / sizeof(size_t)];
         while (mask_block) {
             // Get the lowest set bit in the mask block
             size_t lowest_bit = mask_block & (-(ptrdiff_t)mask_block);

modules/ts/misc/run.py

@@ -37,7 +37,7 @@ if __name__ == "__main__":
 
     # Valgrind
     parser.add_argument("--valgrind", action="store_true", default=False, help="Run C++ tests in valgrind")
-    parser.add_argument("--valgrind_supp", metavar="FILE", help="Path to valgrind suppression file (example: --valgrind_supp opencv/platforms/scripts/valgrind.supp)")
+    parser.add_argument("--valgrind_supp", metavar="FILE", action='append', help="Path to valgrind suppression file (example: --valgrind_supp opencv/platforms/scripts/valgrind.supp)")
     parser.add_argument("--valgrind_opt", metavar="OPT", action="append", default=[], help="Add command line option to valgrind (example: --valgrind_opt=--leak-check=full)")
 
     # Android

modules/ts/misc/run_long.py

@@ -7,7 +7,10 @@ from pprint import PrettyPrinter as PP
 
 LONG_TESTS_DEBUG_VALGRIND = [
     ('calib3d', 'Calib3d_InitUndistortRectifyMap.accuracy', 2017.22),
+    ('dnn', 'Reproducibility*', 1000), # large DNN models
     ('features2d', 'Features2d_Feature2d.no_crash', 1235.68),
+    ('imgcodecs', 'Imgcodecs_Png.write_big', 1000), # memory limit
+    ('imgcodecs', 'Imgcodecs_Tiff.decode_tile16384x16384', 1000), # memory limit
     ('ml', 'ML_RTrees.regression', 1423.47),
     ('optflow', 'DenseOpticalFlow_DeepFlow.ReferenceAccuracy', 1360.95),
     ('optflow', 'DenseOpticalFlow_DeepFlow_perf.perf/0', 1881.59),
@@ -23,6 +26,7 @@ LONG_TESTS_DEBUG_VALGRIND = [
     ('shape', 'Shape_SCD.regression', 3311.46),
     ('tracking', 'AUKF.br_mean_squared_error', 10764.6),
     ('tracking', 'UKF.br_mean_squared_error', 5228.27),
+    ('videoio', 'Videoio_Video.ffmpeg_writebig', 1000),
     ('xfeatures2d', 'Features2d_RotationInvariance_Descriptor_BoostDesc_LBGM.regression', 1124.51),
     ('xfeatures2d', 'Features2d_RotationInvariance_Descriptor_VGG120.regression', 2198.1),
     ('xfeatures2d', 'Features2d_RotationInvariance_Descriptor_VGG48.regression', 1958.52),
@@ -43,10 +47,8 @@ LONG_TESTS_DEBUG_VALGRIND = [
 ]
 
 
-def longTestFilter(data):
-    res = ['*', '-']
-    for _, v, _ in data:
-        res.append(v)
+def longTestFilter(data, module = None):
+    res = ['*', '-'] + [v for _, v, m in data if module is None or m == module]
     return '--gtest_filter={}'.format(':'.join(res))
 
 

modules/ts/misc/run_suite.py

@@ -103,10 +103,15 @@ class TestSuite(object):
     def wrapInValgrind(self, cmd = []):
         if self.options.valgrind:
             res = ['valgrind']
-            if self.options.valgrind_supp:
-                res.append("--suppressions=%s" % self.options.valgrind_supp)
+            supp = self.options.valgrind_supp or []
+            for f in supp:
+                if os.path.isfile(f):
+                    res.append("--suppressions=%s" % f)
+                else:
+                    print("WARNING: Valgrind suppression file is missing, SKIP: %s" % f)
             res.extend(self.options.valgrind_opt)
-            return res + cmd + [longTestFilter(LONG_TESTS_DEBUG_VALGRIND)]
+            has_gtest_filter = next((True for x in cmd if x.startswith('--gtest_filter=')), False)
+            return res + cmd + ([longTestFilter(LONG_TESTS_DEBUG_VALGRIND)] if not has_gtest_filter else [])
         return cmd
 
     def tryCommand(self, cmd):

platforms/scripts/valgrind.supp

 {
-   IPP static init
+   OpenCV-IPP static init
    Memcheck:Cond
    fun:ippicvGetCpuFeatures
    fun:ippicvStaticInit
 }
 
 {
-   TBB - allocate_via_handler_v3 issue
+   OpenCV-getInitializationMutex
    Memcheck:Leak
-   fun:malloc
-   fun:_ZN3tbb8internal23allocate_via_handler_v3Em
+   ...
+   fun:_ZN2cv22getInitializationMutexEv
+}
+
+{
+   OpenCV-getStdAllocator
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv3Mat15getStdAllocatorEv
+}
+
+{
+   OpenCV-getOpenCLAllocator
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv3ocl18getOpenCLAllocatorEv
+}
+
+{
+   OpenCV-getCoreTlsData
+   Memcheck:Leak
+   fun:_Znwm
+   fun:_ZN2cv14getCoreTlsDataEv
+}
+
+{
+   OpenCV-TLS-getTlsStorage
+   Memcheck:Leak
+   ...
+   fun:_ZN2cvL13getTlsStorageEv
+}
+
+{
+   OpenCV-TLS-getData()
+   Memcheck:Leak
+   ...
+   fun:*setData*
+   fun:_ZNK2cv16TLSDataContainer7getDataEv
+}
+
+{
+   OpenCV-parallel_for
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv13ThreadManager8initPoolEv*
+}
+
+{
+   OpenCV-parallel_for
+   Memcheck:Leak
+   fun:_Znwm
+   fun:*instance*
+   fun:_ZN2cv21parallel_for_pthreadsERKNS_5RangeERKNS_16ParallelLoopBodyEd
+   fun:_ZN2cv13parallel_for_ERKNS_5RangeERKNS_16ParallelLoopBodyEd
+}
+
+{
+   OpenCV-parallel_for-ThreadManager::TLS
+   Memcheck:Leak
+   fun:_Znwm
+   fun:_ZNK2cv7TLSDataINS_13ThreadManager13work_thread_tEE18createDataInstanceEv
+}
+
+{
+   OpenCV-parallel_for-setNumThreads()
+   Memcheck:Leak
+   fun:_Znwm
+   fun:_ZN2cv13ThreadManager8instanceEv
+   fun:_ZN2cv33parallel_pthreads_set_threads_numEi
+   fun:_ZN2cv13setNumThreadsEi
+}
+
+{
+   OpenCV-parallel_for-getNumThreads()
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv13getNumThreadsEv
+}
+
+{
+   OpenCV-getIPPSingelton
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv3ippL15getIPPSingeltonEv
+}
+
+{
+   OpenCV-getGlobalMatOpInitializer
+   Memcheck:Leak
+   fun:_Znwm
+   fun:_ZN2cvL25getGlobalMatOpInitializerEv
+}
+
+{
+   OpenCV-CoreTLSData
+   Memcheck:Leak
+   ...
+   fun:_ZNK2cv7TLSDataINS_11CoreTLSDataEE3getEv
+}
+
+{
+   OpenCV-ThreadID
+   Memcheck:Leak
+   fun:_Znwm
+   fun:_ZNK2cv7TLSDataINS_12_GLOBAL__N_18ThreadIDEE18createDataInstanceEv
+}
+
+{
+   OpenCV-ThreadID-TLS
+   Memcheck:Leak
+   fun:_Znwm
+   fun:getThreadIDTLS
+}
+
+{
+   OpenCV-CoreTLS
+   Memcheck:Leak
+   fun:_Znwm
+   fun:_ZNK2cv7TLSDataINS_11CoreTLSDataEE18createDataInstanceEv
+}
+
+{
+   OpenCV-haveOpenCL
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv3ocl10haveOpenCLEv
+}
+
+{
+   OpenCV-DNN-getLayerFactoryMutex
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv3dnn*L20getLayerFactoryMutexEv
+}
+
+{
+   OpenCV-ocl::Context
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv3ocl7Context10getDefaultEb
+}
+
+{
+   OpenCV-ocl::Device
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv3ocl6Device10getDefaultEv
+}
+
+{
+   OpenCV-ocl::Queue
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv3ocl5Queue6createERKNS0_7ContextERKNS0_6DeviceE
+}
+
+{
+   OpenCV-ocl::Program
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv3ocl6Kernel6createEPKcRKNS0_7ProgramE
+}
+
+{
+   OpenCV-ocl::ProgramEntry
+   Memcheck:Leak
+   ...
+   fun:_ZNK2cv3ocl8internal12ProgramEntrycvRNS0_13ProgramSourceEEv
+}
+
+{
+   OpenCV-ocl::Context::getProg
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv3ocl7Context7getProgERKNS0_13ProgramSourceERKNS_6StringERS5_
+}
+
+
+{
+   OpenCV-ITT
+   Memcheck:Leak
+   ...
+   fun:__itt_*create*
+}
+
+{
+   OpenCV-FFmpeg-swsscale
+   Memcheck:Addr16
+   ...
+   fun:sws_scale
+   fun:_ZN20CvVideoWriter_FFMPEG10writeFrameEPKhiiiii
+   fun:cvWriteFrame_FFMPEG
 }

platforms/scripts/valgrind_3rdparty.supp

+{
+   IPP static init
+   Memcheck:Cond
+   fun:ippicvGetCpuFeatures
+   fun:ippicvStaticInit
+}
+
+{
+   TBB - allocate_via_handler_v3 issue
+   Memcheck:Leak
+   fun:malloc
+   fun:_ZN3tbb8internal23allocate_via_handler_v3Em
+}
+
+{
+   GTest
+   Memcheck:Cond
+   fun:_ZN7testing8internal11CmpHelperLEIddEENS_15AssertionResultEPKcS4_RKT_RKT0_
+}
+
+{
+   OpenCL
+   Memcheck:Cond
+   ...
+   obj:**/libOpenCL.so*
+}
+
+{
+   OpenCL-Intel
+   Memcheck:Cond
+   ...
+   obj:**/libigdrcl.so
+}
+
+{
+   OpenCL-Intel
+   Memcheck:Leak
+   ...
+   obj:*/libigdrcl.so*
+}
+
+{
+   OpenCL
+   Memcheck:Param
+   ioctl(generic)
+   ...
+   fun:clGetPlatformIDs
+}
+
+{
+   OpenCL-Init
+   Memcheck:Leak
+   ...
+   fun:clGetPlatformIDs
+}
+
+{
+   glib
+   Memcheck:Leak
+   fun:*alloc
+   obj:*/libglib*
+}
+
+{
+   gcrypt
+   Memcheck:Leak
+   ...
+   obj:*/libgcrypt*
+}
+
+{
+   p11-kit
+   Memcheck:Leak
+   fun:*alloc
+   obj:*/libp11-kit*
+}
+
+{
+   gobject
+   Memcheck:Leak
+   fun:*alloc
+   ...
+   obj:*/libgobject*
+}
+
+{
+   tasn
+   Memcheck:Leak
+   fun:*alloc
+   obj:*/libtasn*.so*
+}
+
+{
+   dl_init
+   Memcheck:Leak
+   ...
+   fun:_dl_init
+}
+
+{
+   dl_open
+   Memcheck:Leak
+   ...
+   fun:_dl_open
+}
+
+{
+   GDAL
+   Memcheck:Leak
+   fun:*alloc
+   ...
+   obj:/usr/lib/libgdal.so.1.17.1
+}