ts: test tags for flexible/reliable tests filtering

- added functionality to collect memory usage of OpenCL sybsystem
- memory usage of fastMalloc() (disabled by default):
  * It is not accurate sometimes - external memory profiler is required.
- specify common `CV_TEST_TAG_` macros
- added applyTestTag() function
- write memory usage / enabled tags into Google Tests output file (.xml)

modules/core/include/opencv2/core/private.hpp

@@ -141,13 +141,20 @@ namespace cv
 {
 CV_EXPORTS void scalarToRawData(const cv::Scalar& s, void* buf, int type, int unroll_to = 0);
 
-//! Allocate all memory buffers which will not be freed, ease filtering memcheck issues
+//! Allocate memory buffers which will not be freed, ease filtering memcheck issues. Uses fastMalloc() call.
 CV_EXPORTS void* allocSingletonBuffer(size_t size);
 
-//! Allocate all memory buffers which will not be freed, ease filtering memcheck issues
+//! Allocate memory buffers which will not be freed, ease filtering memcheck issues. Uses fastMalloc() call
 template <typename T> static inline
 T* allocSingleton(size_t count = 1) { return static_cast<T*>(allocSingletonBuffer(sizeof(T) * count)); }
 
+//! Allocate memory buffers which will not be freed, ease filtering memcheck issues. Uses generic malloc() call.
+CV_EXPORTS void* allocSingletonNewBuffer(size_t size);
+
+//! Allocate memory buffers which will not be freed, ease filtering memcheck issues.  Uses generic malloc() call.
+template <typename T> static inline
+T* allocSingletonNew() { return new(allocSingletonNewBuffer(sizeof(T))) T(); }
+
 } // namespace
 
 #if 1 // TODO: Remove in OpenCV 4.x

modules/core/include/opencv2/core/utils/allocator_stats.hpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef OPENCV_CORE_ALLOCATOR_STATS_HPP
+#define OPENCV_CORE_ALLOCATOR_STATS_HPP
+
+#include "../cvdef.h"
+
+namespace cv { namespace utils {
+
+class AllocatorStatisticsInterface
+{
+protected:
+    AllocatorStatisticsInterface() {}
+    virtual ~AllocatorStatisticsInterface() {}
+public:
+    virtual uint64_t getCurrentUsage() const = 0;
+    virtual uint64_t getTotalUsage() const = 0;
+    virtual uint64_t getNumberOfAllocations() const = 0;
+    virtual uint64_t getPeakUsage() const = 0;
+
+    /** set peak usage = current usage */
+    virtual void resetPeakUsage() = 0;
+};
+
+}} // namespace
+
+#endif // OPENCV_CORE_ALLOCATOR_STATS_HPP

modules/core/include/opencv2/core/utils/allocator_stats.impl.hpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef OPENCV_CORE_ALLOCATOR_STATS_IMPL_HPP
+#define OPENCV_CORE_ALLOCATOR_STATS_IMPL_HPP
+
+#include "./allocator_stats.hpp"
+
+#ifdef CV_CXX11
+#include <atomic>
+#endif
+
+namespace cv { namespace utils {
+
+#ifdef CV__ALLOCATOR_STATS_LOG
+namespace {
+#endif
+
+class AllocatorStatistics : public AllocatorStatisticsInterface
+{
+protected:
+#ifdef CV_CXX11
+    std::atomic<long long> curr, total, total_allocs, peak;
+#else
+    volatile long long curr, total, total_allocs, peak;  // overflow is possible, CV_XADD operates with 'int' only
+#endif
+
+public:
+    AllocatorStatistics()
+#ifndef CV_CXX11
+        : curr(0), total(0), total_allocs(0), peak(0)
+#endif
+    {}
+    ~AllocatorStatistics() CV_OVERRIDE {}
+
+    // AllocatorStatisticsInterface
+
+#ifdef CV_CXX11
+    uint64_t getCurrentUsage() const CV_OVERRIDE { return (uint64_t)curr.load(); }
+    uint64_t getTotalUsage() const CV_OVERRIDE { return (uint64_t)total.load(); }
+    uint64_t getNumberOfAllocations() const CV_OVERRIDE { return (uint64_t)total_allocs.load(); }
+    uint64_t getPeakUsage() const CV_OVERRIDE { return (uint64_t)peak.load(); }
+
+    /** set peak usage = current usage */
+    void resetPeakUsage() CV_OVERRIDE { peak.store(curr.load()); }
+
+    // Controller interface
+    void onAllocate(size_t sz)
+    {
+#ifdef CV__ALLOCATOR_STATS_LOG
+        CV__ALLOCATOR_STATS_LOG(cv::format("allocate: %lld (curr=%lld)", (long long int)sz, (long long int)curr.load()));
+#endif
+
+        long long new_curr = curr.fetch_add((long long)sz) + (long long)sz;
+
+        // peak = std::max((uint64_t)peak, new_curr);
+        auto prev_peak = peak.load();
+        while (prev_peak < new_curr)
+        {
+            if (peak.compare_exchange_weak(prev_peak, new_curr))
+                break;
+        }
+        // end of peak = max(...)
+
+        total += (long long)sz;
+        total_allocs++;
+    }
+    void onFree(size_t sz)
+    {
+#ifdef CV__ALLOCATOR_STATS_LOG
+        CV__ALLOCATOR_STATS_LOG(cv::format("free: %lld (curr=%lld)", (long long int)sz, (long long int)curr.load()));
+#endif
+        curr -= (long long)sz;
+    }
+
+#else
+    uint64_t getCurrentUsage() const CV_OVERRIDE { return (uint64_t)curr; }
+    uint64_t getTotalUsage() const CV_OVERRIDE { return (uint64_t)total; }
+    uint64_t getNumberOfAllocations() const CV_OVERRIDE { return (uint64_t)total_allocs; }
+    uint64_t getPeakUsage() const CV_OVERRIDE { return (uint64_t)peak; }
+
+    void resetPeakUsage() CV_OVERRIDE { peak = curr; }
+
+    // Controller interface
+    void onAllocate(size_t sz)
+    {
+#ifdef CV__ALLOCATOR_STATS_LOG
+        CV__ALLOCATOR_STATS_LOG(cv::format("allocate: %lld (curr=%lld)", (long long int)sz, (long long int)curr));
+#endif
+
+        uint64_t new_curr = (uint64_t)CV_XADD(&curr, (uint64_t)sz) + sz;
+
+        peak = std::max((uint64_t)peak, new_curr);  // non-thread safe
+
+        //CV_XADD(&total, (uint64_t)sz);  // overflow with int, non-reliable...
+        total += sz;
+
+        CV_XADD(&total_allocs, (uint64_t)1);
+    }
+    void onFree(size_t sz)
+    {
+#ifdef CV__ALLOCATOR_STATS_LOG
+        CV__ALLOCATOR_STATS_LOG(cv::format("free: %lld (curr=%lld)", (long long int)sz, (long long int)curr));
+#endif
+        CV_XADD(&curr, (uint64_t)-sz);
+    }
+#endif
+};
+
+#ifdef CV__ALLOCATOR_STATS_LOG
+} // namespace
+#endif
+
+}} // namespace
+
+#endif // OPENCV_CORE_ALLOCATOR_STATS_IMPL_HPP

modules/core/src/alloc.cpp

@@ -42,12 +42,29 @@
 
 #include "precomp.hpp"
 
+#include <opencv2/core/utils/logger.defines.hpp>
+#undef CV_LOG_STRIP_LEVEL
+#define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_VERBOSE + 1
+#include <opencv2/core/utils/logger.hpp>
+
+#define CV__ALLOCATOR_STATS_LOG(...) CV_LOG_VERBOSE(NULL, 0, "alloc.cpp: " << __VA_ARGS__)
+#include "opencv2/core/utils/allocator_stats.impl.hpp"
+#undef CV__ALLOCATOR_STATS_LOG
+
+//#define OPENCV_ALLOC_ENABLE_STATISTICS
+#define OPENCV_ALLOC_STATISTICS_LIMIT 4096  // don't track buffers less than N bytes
+
+
 #ifdef HAVE_POSIX_MEMALIGN
 #include <stdlib.h>
 #elif defined HAVE_MALLOC_H
 #include <malloc.h>
 #endif
 
+#ifdef OPENCV_ALLOC_ENABLE_STATISTICS
+#include <map>
+#endif
+
 namespace cv {
 
 static void* OutOfMemoryError(size_t size)
@@ -55,8 +72,21 @@ static void* OutOfMemoryError(size_t size)
     CV_Error_(CV_StsNoMem, ("Failed to allocate %llu bytes", (unsigned long long)size));
 }
 
+CV_EXPORTS cv::utils::AllocatorStatisticsInterface& getAllocatorStatistics();
+
+static cv::utils::AllocatorStatistics allocator_stats;
+
+cv::utils::AllocatorStatisticsInterface& getAllocatorStatistics()
+{
+    return allocator_stats;
+}
 
-void* fastMalloc( size_t size )
+#ifdef OPENCV_ALLOC_ENABLE_STATISTICS
+static inline
+void* fastMalloc_(size_t size)
+#else
+void* fastMalloc(size_t size)
+#endif
 {
 #ifdef HAVE_POSIX_MEMALIGN
     void* ptr = NULL;
@@ -80,7 +110,12 @@ void* fastMalloc( size_t size )
 #endif
 }
 
+#ifdef OPENCV_ALLOC_ENABLE_STATISTICS
+static inline
+void fastFree_(void* ptr)
+#else
 void fastFree(void* ptr)
+#endif
 {
 #if defined HAVE_POSIX_MEMALIGN || defined HAVE_MEMALIGN
     free(ptr);
@@ -95,6 +130,47 @@ void fastFree(void* ptr)
 #endif
 }
 
+#ifdef OPENCV_ALLOC_ENABLE_STATISTICS
+
+static
+Mutex& getAllocationStatisticsMutex()
+{
+    static Mutex* p_alloc_mutex = allocSingletonNew<Mutex>();
+    CV_Assert(p_alloc_mutex);
+    return *p_alloc_mutex;
+}
+
+static std::map<void*, size_t> allocated_buffers;  // guarded by getAllocationStatisticsMutex()
+
+void* fastMalloc(size_t size)
+{
+    void* res = fastMalloc_(size);
+    if (res && size >= OPENCV_ALLOC_STATISTICS_LIMIT)
+    {
+        cv::AutoLock lock(getAllocationStatisticsMutex());
+        allocated_buffers.insert(std::make_pair(res, size));
+        allocator_stats.onAllocate(size);
+    }
+    return res;
+}
+
+void fastFree(void* ptr)
+{
+    {
+        cv::AutoLock lock(getAllocationStatisticsMutex());
+        std::map<void*, size_t>::iterator i = allocated_buffers.find(ptr);
+        if (i != allocated_buffers.end())
+        {
+            size_t size = i->second;
+            allocator_stats.onFree(size);
+            allocated_buffers.erase(i);
+        }
+    }
+    fastFree_(ptr);
+}
+
+#endif // OPENCV_ALLOC_ENABLE_STATISTICS
+
 } // namespace
 
 CV_IMPL void* cvAlloc( size_t size )

modules/core/src/ocl.cpp

@@ -54,6 +54,9 @@
 
 #include <opencv2/core/utils/configuration.private.hpp>
 
+#include <opencv2/core/utils/logger.defines.hpp>
+#undef CV_LOG_STRIP_LEVEL
+#define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_DEBUG + 1
 #include <opencv2/core/utils/logger.hpp>
 
 #include "opencv2/core/ocl_genbase.hpp"
@@ -63,6 +66,10 @@
 #include "opencv2/core/utils/filesystem.hpp"
 #include "opencv2/core/utils/filesystem.private.hpp"
 
+#define CV__ALLOCATOR_STATS_LOG(...) CV_LOG_VERBOSE(NULL, 0, "OpenCL allocator: " << __VA_ARGS__)
+#include "opencv2/core/utils/allocator_stats.impl.hpp"
+#undef CV__ALLOCATOR_STATS_LOG
+
 #define CV_OPENCL_ALWAYS_SHOW_BUILD_LOG          0
 
 #define CV_OPENCL_SHOW_RUN_KERNELS               0
@@ -132,6 +139,14 @@ namespace cv { namespace ocl {
     void release() { if( CV_XADD(&refcount, -1) == 1 && !cv::__termination) delete this; } \
     int refcount
 
+static cv::utils::AllocatorStatistics opencl_allocator_stats;
+
+CV_EXPORTS cv::utils::AllocatorStatisticsInterface& getOpenCLAllocatorStatistics();
+cv::utils::AllocatorStatisticsInterface& getOpenCLAllocatorStatistics()
+{
+    return opencl_allocator_stats;
+}
+
 #ifndef HAVE_OPENCL
 #define CV_OPENCL_NO_SUPPORT() CV_Error(cv::Error::OpenCLApiCallError, "OpenCV build without OpenCL support")
 namespace {
@@ -4539,15 +4554,17 @@ class OpenCLAllocator CV_FINAL : public MatAllocator
     mutable OpenCLSVMBufferPoolImpl bufferPoolSVM;
 #endif
 
+public:
     enum AllocatorFlags
     {
         ALLOCATOR_FLAGS_BUFFER_POOL_USED = 1 << 0,
-        ALLOCATOR_FLAGS_BUFFER_POOL_HOST_PTR_USED = 1 << 1
+        ALLOCATOR_FLAGS_BUFFER_POOL_HOST_PTR_USED = 1 << 1,
 #ifdef HAVE_OPENCL_SVM
-        ,ALLOCATOR_FLAGS_BUFFER_POOL_SVM_USED = 1 << 2
+        ALLOCATOR_FLAGS_BUFFER_POOL_SVM_USED = 1 << 2,
 #endif
+        ALLOCATOR_FLAGS_EXTERNAL_BUFFER = 1 << 3  // convertFromBuffer()
     };
-public:
+
     OpenCLAllocator()
         : bufferPool(0),
           bufferPoolHostPtr(CL_MEM_ALLOC_HOST_PTR)
@@ -4652,6 +4669,7 @@ public:
         u->allocatorFlags_ = allocatorFlags;
         CV_DbgAssert(!u->tempUMat()); // for bufferPool.release() consistency in deallocate()
         u->markHostCopyObsolete(true);
+        opencl_allocator_stats.onAllocate(u->size);
         return u;
     }
 
@@ -4760,6 +4778,7 @@ public:
         }
         if(accessFlags & ACCESS_WRITE)
             u->markHostCopyObsolete(true);
+        opencl_allocator_stats.onAllocate(u->size);
         return true;
     }
 
@@ -4812,6 +4831,13 @@ public:
 
     void deallocate_(UMatData* u) const
     {
+        CV_Assert(u);
+        CV_Assert(u->handle);
+        if ((u->allocatorFlags_ & ALLOCATOR_FLAGS_EXTERNAL_BUFFER) == 0)
+        {
+            opencl_allocator_stats.onFree(u->size);
+        }
+
 #ifdef _WIN32
         if (cv::__termination)  // process is not in consistent state (after ExitProcess call) and terminating
             return;             // avoid any OpenCL calls
@@ -5793,7 +5819,7 @@ void convertFromBuffer(void* cl_mem_buffer, size_t step, int rows, int cols, int
     // attach clBuffer to UMatData
     dst.u = new UMatData(getOpenCLAllocator());
     dst.u->data            = 0;
-    dst.u->allocatorFlags_ = 0; // not allocated from any OpenCV buffer pool
+    dst.u->allocatorFlags_ = OpenCLAllocator::ALLOCATOR_FLAGS_EXTERNAL_BUFFER;  // not allocated from any OpenCV buffer pool
     dst.u->flags           = 0;
     dst.u->handle          = cl_mem_buffer;
     dst.u->origdata        = 0;

modules/core/src/system.cpp

@@ -71,6 +71,8 @@ static bool param_dumpErrors = utils::getConfigurationParameterBool("OPENCV_DUMP
 );
 
 void* allocSingletonBuffer(size_t size) { return fastMalloc(size); }
+void* allocSingletonNewBuffer(size_t size) { return malloc(size); }
+
 
 } // namespace cv
 

modules/ts/include/opencv2/ts.hpp

@@ -51,6 +51,44 @@
 # endif
 #endif
 
+
+
+// most part of OpenCV tests are fit into 200Mb limit, but some tests are not:
+// Note: due memory fragmentation real limits are usually lower on 20-25% (400Mb memory usage goes into mem_1Gb class)
+#define CV_TEST_TAG_MEMORY_512MB "mem_512mb"     // used memory: 200..512Mb - enabled by default
+#define CV_TEST_TAG_MEMORY_1GB "mem_1gb"         // used memory: 512Mb..1Gb - enabled by default
+#define CV_TEST_TAG_MEMORY_2GB "mem_2gb"         // used memory: 1..2Gb - enabled by default on 64-bit configuration (32-bit - disabled)
+#define CV_TEST_TAG_MEMORY_6GB "mem_6gb"         // used memory: 2..6Gb - disabled by default
+#define CV_TEST_TAG_MEMORY_14GB "mem_14gb"       // used memory: 6..14Gb - disabled by default
+
+// Large / huge video streams or complex workloads
+#define CV_TEST_TAG_LONG "long"                  // 5+ seconds on modern desktop machine (single thread)
+#define CV_TEST_TAG_VERYLONG "verylong"          // 20+ seconds on modern desktop machine (single thread)
+
+// Large / huge video streams or complex workloads for debug builds
+#define CV_TEST_TAG_DEBUG_LONG "debug_long"           // 10+ seconds on modern desktop machine (single thread)
+#define CV_TEST_TAG_DEBUG_VERYLONG "debug_verylong"   // 40+ seconds on modern desktop machine (single thread)
+
+// Lets skip processing of high resolution images via instrumentation tools (valgrind/coverage/sanitizers).
+// It is enough to run lower resolution (VGA: 640x480) tests.
+#define CV_TEST_TAG_SIZE_HD "size_hd"            // 720p+, enabled
+#define CV_TEST_TAG_SIZE_FULLHD "size_fullhd"    // 1080p+, enabled (disable these tests for valgrind/coverage run)
+#define CV_TEST_TAG_SIZE_4K "size_4k"            // 2160p+, enabled (disable these tests for valgrind/coverage run)
+
+// Other misc test tags
+#define CV_TEST_TAG_TYPE_64F "type_64f"          // CV_64F, enabled (disable these tests on low power embedded devices)
+
+// Kernel-based image processing
+#define CV_TEST_TAG_FILTER_SMALL "filter_small"       // Filtering with kernels <= 3x3
+#define CV_TEST_TAG_FILTER_MEDIUM "filter_medium"     // Filtering with kernels: 3x3 < kernel <= 5x5
+#define CV_TEST_TAG_FILTER_LARGE "filter_large"       // Filtering with kernels: 5x5 < kernel <= 9x9
+#define CV_TEST_TAG_FILTER_HUGE "filter_huge"         // Filtering with kernels: > 9x9
+
+// Other tests categories
+#define CV_TEST_TAG_OPENCL "opencl"              // Tests with OpenCL
+
+
+
 #ifdef WINRT
     #pragma warning(disable:4447) // Disable warning 'main' signature found without threading model
 #endif
@@ -150,6 +188,30 @@ public:
     SkipTestException(const cv::String& message) : dummy(0) { this->msg = message; }
 };
 
+/** Apply tag to the current test
+
+Automatically apply corresponding additional tags (for example, 4K => FHD => HD => VGA).
+
+If tag is in skip list, then SkipTestException is thrown
+*/
+void applyTestTag(const std::string& tag);
+
+/** Run postponed checks of applied test tags
+
+If tag is in skip list, then SkipTestException is thrown
+*/
+void checkTestTags();
+
+void applyTestTag_(const std::string& tag);
+
+static inline void applyTestTag(const std::string& tag1, const std::string& tag2)
+{ applyTestTag_(tag1); applyTestTag_(tag2); checkTestTags(); }
+static inline void applyTestTag(const std::string& tag1, const std::string& tag2, const std::string& tag3)
+{ applyTestTag_(tag1); applyTestTag_(tag2); applyTestTag_(tag3); checkTestTags(); }
+static inline void applyTestTag(const std::string& tag1, const std::string& tag2, const std::string& tag3, const std::string& tag4)
+{ applyTestTag_(tag1); applyTestTag_(tag2); applyTestTag_(tag3); applyTestTag_(tag4); checkTestTags(); }
+
+
 class TS;
 
 int64 readSeed(const char* str);

modules/ts/include/opencv2/ts/ts_ext.hpp

@@ -13,6 +13,9 @@ void checkIppStatus();
 extern bool skipUnstableTests;
 extern bool runBigDataTests;
 extern int testThreads;
+
+void testSetUp();
+void testTearDown();
 }
 
 // check for required "opencv_test" namespace
@@ -24,10 +27,8 @@ extern int testThreads;
 
 #define CV__TEST_INIT \
     CV__TEST_NAMESPACE_CHECK \
-    cv::ipp::setIppStatus(0); \
-    cv::theRNG().state = cvtest::param_seed; \
-    cv::setNumThreads(cvtest::testThreads);
-#define CV__TEST_CLEANUP ::cvtest::checkIppStatus();
+    ::cvtest::testSetUp();
+#define CV__TEST_CLEANUP ::cvtest::testTearDown();
 #define CV__TEST_BODY_IMPL(name) \
     { \
        CV__TRACE_APP_FUNCTION_NAME(name); \

modules/ts/include/opencv2/ts/ts_perf.hpp

@@ -527,7 +527,15 @@ void PrintTo(const Size& sz, ::std::ostream* os);
     { \
        CV__TEST_NAMESPACE_CHECK \
        CV__TRACE_APP_FUNCTION_NAME("PERF_TEST: " name); \
+       try { \
+       ::cvtest::testSetUp(); \
        RunPerfTestBody(); \
+       } \
+       catch (cvtest::SkipTestException& e) \
+       { \
+          printf("[     SKIP ] %s\n", e.what()); \
+       } \
+       ::cvtest::testTearDown(); \
     }
 
 #define PERF_PROXY_NAMESPACE_NAME_(test_case_name, test_name) \

modules/ts/misc/testlog_parser.py

@@ -51,6 +51,7 @@ class TestInfo(object):
         self.parseLongMetric(xmlnode, "stddev");
         self.parseFloatMetric(xmlnode, "gstddev");
         self.parseFloatMetric(xmlnode, "time");
+        self.parseLongMetric(xmlnode, "total_memory_usage");
 
     def parseLongMetric(self, xmlnode, name, default = 0):
         if name in self.properties:

modules/ts/src/precomp.hpp

 #include "opencv2/ts.hpp"
+#include <opencv2/core/utils/logger.hpp>
+#include "opencv2/core/utility.hpp"
 #include "opencv2/core/private.hpp"
 
 #ifdef GTEST_LINKED_AS_SHARED_LIBRARY

modules/ts/src/ts.cpp

@@ -91,11 +91,31 @@
 
 #include "opencv2/core/opencl/opencl_info.hpp"
 
+#include "opencv2/core/utils/allocator_stats.hpp"
+namespace cv { namespace ocl {
+cv::utils::AllocatorStatisticsInterface& getOpenCLAllocatorStatistics();
+}}
 #endif // HAVE_OPENCL
 
-#include "opencv2/core/utility.hpp"
+#include "opencv2/core/utils/allocator_stats.hpp"
+namespace cv {
+CV_EXPORTS cv::utils::AllocatorStatisticsInterface& getAllocatorStatistics();
+}
+
 #include "opencv_tests_config.hpp"
 
+#include "ts_tags.hpp"
+
+#if defined(__GNUC__) && defined(__linux__)
+extern "C" {
+size_t malloc_peak(void) __attribute__((weak));
+void malloc_reset_peak(void) __attribute__((weak));
+} // extern "C"
+#else // stubs
+static size_t (*malloc_peak)(void) = 0;
+static void (*malloc_reset_peak)(void) = 0;
+#endif
+
 namespace opencv_test {
 bool required_opencv_test_namespace = false;  // compilation check for non-refactored tests
 }
@@ -726,6 +746,85 @@ bool skipUnstableTests = false;
 bool runBigDataTests = false;
 int testThreads = 0;
 
+
+static size_t memory_usage_base = 0;
+static uint64_t memory_usage_base_opencv = 0;
+#ifdef HAVE_OPENCL
+static uint64_t memory_usage_base_opencl = 0;
+#endif
+
+void testSetUp()
+{
+    cv::ipp::setIppStatus(0);
+    cv::theRNG().state = cvtest::param_seed;
+    cv::setNumThreads(cvtest::testThreads);
+    if (malloc_peak)  // if memory profiler is available
+    {
+        malloc_reset_peak();
+        memory_usage_base = malloc_peak(); // equal to malloc_current()
+    }
+    {
+        cv::utils::AllocatorStatisticsInterface& ocv_stats = cv::getAllocatorStatistics();
+        ocv_stats.resetPeakUsage();
+        memory_usage_base_opencv = ocv_stats.getCurrentUsage();
+    }
+#ifdef HAVE_OPENCL
+    {
+        cv::utils::AllocatorStatisticsInterface& ocl_stats = cv::ocl::getOpenCLAllocatorStatistics();
+        ocl_stats.resetPeakUsage();
+        memory_usage_base_opencl = ocl_stats.getCurrentUsage();
+    }
+#endif
+    checkTestTags();
+}
+
+void testTearDown()
+{
+    ::cvtest::checkIppStatus();
+    uint64_t memory_usage = 0;
+    uint64_t ocv_memory_usage = 0, ocv_peak = 0;
+    if (malloc_peak)  // if memory profiler is available
+    {
+        size_t peak = malloc_peak();
+        memory_usage = peak - memory_usage_base;
+        CV_LOG_INFO(NULL, "Memory_usage (malloc): " << memory_usage << " (base=" << memory_usage_base << ")");
+    }
+    {
+        // core/src/alloc.cpp: #define OPENCV_ALLOC_ENABLE_STATISTICS
+        // handle large buffers via fastAlloc()
+        // (not always accurate on heavy 3rdparty usage, like protobuf)
+        cv::utils::AllocatorStatisticsInterface& ocv_stats = cv::getAllocatorStatistics();
+        ocv_peak = ocv_stats.getPeakUsage();
+        ocv_memory_usage = ocv_peak - memory_usage_base_opencv;
+        CV_LOG_INFO(NULL, "Memory_usage (OpenCV): " << ocv_memory_usage << " (base=" << memory_usage_base_opencv << "  current=" << ocv_stats.getCurrentUsage() << ")");
+        if (memory_usage == 0)  // external profiler has higher priority (and accuracy)
+            memory_usage = ocv_memory_usage;
+    }
+#ifdef HAVE_OPENCL
+    uint64_t ocl_memory_usage = 0, ocl_peak = 0;
+    {
+        cv::utils::AllocatorStatisticsInterface& ocl_stats = cv::ocl::getOpenCLAllocatorStatistics();
+        ocl_peak = ocl_stats.getPeakUsage();
+        ocl_memory_usage = ocl_peak - memory_usage_base_opencl;
+        CV_LOG_INFO(NULL, "Memory_usage (OpenCL): " << ocl_memory_usage << " (base=" << memory_usage_base_opencl << "  current=" << ocl_stats.getCurrentUsage() << ")");
+        ::testing::Test::RecordProperty("ocl_memory_usage",
+                cv::format("%llu", (unsigned long long)ocl_memory_usage));
+    }
+#else
+    uint64_t ocl_memory_usage = 0;
+#endif
+    if (malloc_peak      // external memory profiler is available
+        || ocv_peak > 0  // or enabled OpenCV builtin allocation statistics
+    )
+    {
+        CV_LOG_INFO(NULL, "Memory usage total: " << (memory_usage + ocl_memory_usage));
+        ::testing::Test::RecordProperty("memory_usage",
+                cv::format("%llu", (unsigned long long)memory_usage));
+        ::testing::Test::RecordProperty("total_memory_usage",
+                cv::format("%llu", (unsigned long long)(memory_usage + ocl_memory_usage)));
+    }
+}
+
 void parseCustomOptions(int argc, char **argv)
 {
     const char * const command_line_keys =
@@ -735,7 +834,9 @@ void parseCustomOptions(int argc, char **argv)
         "{ skip_unstable      |false    |skip unstable tests }"
         "{ test_bigdata       |false    |run BigData tests (>=2Gb) }"
         "{ test_require_data  |false    |fail on missing non-required test data instead of skip}"
-        "{ h   help           |false    |print help info                          }";
+        CV_TEST_TAGS_PARAMS
+        "{ h   help           |false    |print help info                          }"
+    ;
 
     cv::CommandLineParser parser(argc, argv, command_line_keys);
     if (parser.get<bool>("help"))
@@ -759,8 +860,9 @@ void parseCustomOptions(int argc, char **argv)
     skipUnstableTests = parser.get<bool>("skip_unstable");
     runBigDataTests = parser.get<bool>("test_bigdata");
     checkTestData = parser.get<bool>("test_require_data");
-}
 
+    activateTestTags(parser);
+}
 
 static bool isDirectory(const std::string& path)
 {

modules/ts/src/ts_perf.cpp

 #include "precomp.hpp"
 
+#include "ts_tags.hpp"
+
 #include <map>
 #include <iostream>
 #include <fstream>
@@ -999,6 +1001,8 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
         "{   perf_cuda_info_only         |false    |print an information about system and an available CUDA devices and then exit.}"
 #endif
         "{ skip_unstable                 |false    |skip unstable tests }"
+
+        CV_TEST_TAGS_PARAMS
     ;
 
     cv::CommandLineParser args(argc, argv, command_line_keys);
@@ -1145,6 +1149,8 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
         ::testing::AddGlobalTestEnvironment(new PerfValidationEnvironment());
     }
 
+    activateTestTags(args);
+
     if (!args.check())
     {
         args.printErrors();
@@ -1869,14 +1875,15 @@ void TestBase::SetUp()
     currentIter = (unsigned int)-1;
     timeLimit = timeLimitDefault;
     times.clear();
+    metrics.terminationReason = performance_metrics::TERM_SKIP_TEST;
 }
 
 void TestBase::TearDown()
 {
     if (metrics.terminationReason == performance_metrics::TERM_SKIP_TEST)
     {
-        LOGI("\tTest was skipped");
-        GTEST_SUCCEED() << "Test was skipped";
+        //LOGI("\tTest was skipped");
+        //GTEST_SUCCEED() << "Test was skipped";
     }
     else
     {
@@ -1975,6 +1982,7 @@ std::string TestBase::getDataPath(const std::string& relativePath)
 
 void TestBase::RunPerfTestBody()
 {
+    metrics.clear();
     try
     {
 #ifdef CV_COLLECT_IMPL_DATA
@@ -1990,7 +1998,7 @@ void TestBase::RunPerfTestBody()
     catch(const SkipTestException&)
     {
         metrics.terminationReason = performance_metrics::TERM_SKIP_TEST;
-        return;
+        throw;
     }
     catch(const PerfSkipTestException&)
     {

modules/ts/src/ts_tags.hpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef OPENCV_TS_SRC_TAGS_HPP
+#define OPENCV_TS_SRC_TAGS_HPP
+
+// [all | test_tag] - (test_tag_skip - test_tag_enable) + test_tag_force
+
+#define CV_TEST_TAGS_PARAMS \
+    "{ test_tag           |         |run tests with specified 'tag' markers only (comma ',' separated list) }" \
+    "{ test_tag_skip      |         |skip tests with 'tag' markers (comma ',' separated list) }" \
+    "{ test_tag_enable    |         |don't skip tests with 'tag' markers (comma ',' separated list) }" \
+    "{ test_tag_force     |         |force running of tests with 'tag' markers (comma ',' separated list) }" \
+    "{ test_tag_print     | false   |print assigned tags for each test }" \
+
+// TODO
+//  "{ test_tag_file      |         |read test tags assignment }" \
+
+namespace cvtest {
+
+void activateTestTags(const cv::CommandLineParser& parser);
+
+} // namespace
+
+#endif // OPENCV_TS_SRC_TAGS_HPP

platforms/scripts/valgrind.supp

@@ -19,6 +19,13 @@
    fun:_ZN2cv20allocSingletonBufferEm
 }
 
+{
+   OpenCV-SingletonNewBuffer
+   Memcheck:Leak
+   ...
+   fun:_ZN2cv23allocSingletonNewBufferEm
+}
+
 {
    OpenCV-getStdAllocator
    Memcheck:Leak