Skip to content

O
opencv
Commit 64629cb9 authored by Alexander Alekhin's avatar Alexander Alekhin
Browse files
Options

Merge pull request #12783 from alalek:test_tag

parents 5f500692 e0841f3d
Show whitespace changes
Inline Side-by-side
Showing with 1027 additions and 29 deletions
modules/core/include/opencv2/core/private.hpp
View file @ 64629cb9
......@@ -141,13 +141,20 @@ namespace cv
{
CV_EXPORTS void scalarToRawData(const cv::Scalar& s, void* buf, int type, int unroll_to = 0);
//! Allocate all memory buffers which will not be freed, ease filtering memcheck issues
//! Allocate memory buffers which will not be freed, ease filtering memcheck issues. Uses fastMalloc() call.
CV_EXPORTS void* allocSingletonBuffer(size_t size);
//! Allocate all memory buffers which will not be freed, ease filtering memcheck issues
//! Allocate memory buffers which will not be freed, ease filtering memcheck issues. Uses fastMalloc() call
template <typename T> static inline
T* allocSingleton(size_t count = 1) { return static_cast<T*>(allocSingletonBuffer(sizeof(T) * count)); }
//! Allocate memory buffers which will not be freed, ease filtering memcheck issues. Uses generic malloc() call.
CV_EXPORTS void* allocSingletonNewBuffer(size_t size);
//! Allocate memory buffers which will not be freed, ease filtering memcheck issues. Uses generic malloc() call.
template <typename T> static inline
T* allocSingletonNew() { return new(allocSingletonNewBuffer(sizeof(T))) T(); }
} // namespace
#if 1 // TODO: Remove in OpenCV 4.x
......
modules/core/include/opencv2/core/utils/allocator_stats.hpp 0 → 100644
View file @ 64629cb9
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef OPENCV_CORE_ALLOCATOR_STATS_HPP
#define OPENCV_CORE_ALLOCATOR_STATS_HPP
#include "../cvdef.h"
namespace cv { namespace utils {
class AllocatorStatisticsInterface
{
protected:
AllocatorStatisticsInterface() {}
virtual ~AllocatorStatisticsInterface() {}
public:
virtual uint64_t getCurrentUsage() const = 0;
virtual uint64_t getTotalUsage() const = 0;
virtual uint64_t getNumberOfAllocations() const = 0;
virtual uint64_t getPeakUsage() const = 0;
/** set peak usage = current usage */
virtual void resetPeakUsage() = 0;
};
}} // namespace
#endif // OPENCV_CORE_ALLOCATOR_STATS_HPP
modules/core/include/opencv2/core/utils/allocator_stats.impl.hpp 0 → 100644
View file @ 64629cb9
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef OPENCV_CORE_ALLOCATOR_STATS_IMPL_HPP
#define OPENCV_CORE_ALLOCATOR_STATS_IMPL_HPP
#include "./allocator_stats.hpp"
#ifdef CV_CXX11
#include <atomic>
#endif
namespace cv { namespace utils {
#ifdef CV__ALLOCATOR_STATS_LOG
namespace {
#endif
class AllocatorStatistics : public AllocatorStatisticsInterface
{
protected:
#ifdef CV_CXX11
std::atomic<long long> curr, total, total_allocs, peak;
#else
volatile long long curr, total, total_allocs, peak; // overflow is possible, CV_XADD operates with 'int' only
#endif
public:
AllocatorStatistics()
#ifndef CV_CXX11
: curr(0), total(0), total_allocs(0), peak(0)
#endif
{}
~AllocatorStatistics() CV_OVERRIDE {}
// AllocatorStatisticsInterface
#ifdef CV_CXX11
uint64_t getCurrentUsage() const CV_OVERRIDE { return (uint64_t)curr.load(); }
uint64_t getTotalUsage() const CV_OVERRIDE { return (uint64_t)total.load(); }
uint64_t getNumberOfAllocations() const CV_OVERRIDE { return (uint64_t)total_allocs.load(); }
uint64_t getPeakUsage() const CV_OVERRIDE { return (uint64_t)peak.load(); }
/** set peak usage = current usage */
void resetPeakUsage() CV_OVERRIDE { peak.store(curr.load()); }
// Controller interface
void onAllocate(size_t sz)
{
#ifdef CV__ALLOCATOR_STATS_LOG
CV__ALLOCATOR_STATS_LOG(cv::format("allocate: %lld (curr=%lld)", (long long int)sz, (long long int)curr.load()));
#endif
long long new_curr = curr.fetch_add((long long)sz) + (long long)sz;
// peak = std::max((uint64_t)peak, new_curr);
auto prev_peak = peak.load();
while (prev_peak < new_curr)
{
if (peak.compare_exchange_weak(prev_peak, new_curr))
break;
}
// end of peak = max(...)
total += (long long)sz;
total_allocs++;
}
void onFree(size_t sz)
{
#ifdef CV__ALLOCATOR_STATS_LOG
CV__ALLOCATOR_STATS_LOG(cv::format("free: %lld (curr=%lld)", (long long int)sz, (long long int)curr.load()));
#endif
curr -= (long long)sz;
}
#else
uint64_t getCurrentUsage() const CV_OVERRIDE { return (uint64_t)curr; }
uint64_t getTotalUsage() const CV_OVERRIDE { return (uint64_t)total; }
uint64_t getNumberOfAllocations() const CV_OVERRIDE { return (uint64_t)total_allocs; }
uint64_t getPeakUsage() const CV_OVERRIDE { return (uint64_t)peak; }
void resetPeakUsage() CV_OVERRIDE { peak = curr; }
// Controller interface
void onAllocate(size_t sz)
{
#ifdef CV__ALLOCATOR_STATS_LOG
CV__ALLOCATOR_STATS_LOG(cv::format("allocate: %lld (curr=%lld)", (long long int)sz, (long long int)curr));
#endif
uint64_t new_curr = (uint64_t)CV_XADD(&curr, (uint64_t)sz) + sz;
peak = std::max((uint64_t)peak, new_curr); // non-thread safe
//CV_XADD(&total, (uint64_t)sz); // overflow with int, non-reliable...
total += sz;
CV_XADD(&total_allocs, (uint64_t)1);
}
void onFree(size_t sz)
{
#ifdef CV__ALLOCATOR_STATS_LOG
CV__ALLOCATOR_STATS_LOG(cv::format("free: %lld (curr=%lld)", (long long int)sz, (long long int)curr));
#endif
CV_XADD(&curr, (uint64_t)-sz);
}
#endif
};
#ifdef CV__ALLOCATOR_STATS_LOG
} // namespace
#endif
}} // namespace
#endif // OPENCV_CORE_ALLOCATOR_STATS_IMPL_HPP
modules/core/src/alloc.cpp
View file @ 64629cb9
......@@ -42,12 +42,29 @@
#include "precomp.hpp"
#include <opencv2/core/utils/logger.defines.hpp>
#undef CV_LOG_STRIP_LEVEL
#define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_VERBOSE + 1
#include <opencv2/core/utils/logger.hpp>
#define CV__ALLOCATOR_STATS_LOG(...) CV_LOG_VERBOSE(NULL, 0, "alloc.cpp: " << __VA_ARGS__)
#include "opencv2/core/utils/allocator_stats.impl.hpp"
#undef CV__ALLOCATOR_STATS_LOG
//#define OPENCV_ALLOC_ENABLE_STATISTICS
#define OPENCV_ALLOC_STATISTICS_LIMIT 4096 // don't track buffers less than N bytes
#ifdef HAVE_POSIX_MEMALIGN
#include <stdlib.h>
#elif defined HAVE_MALLOC_H
#include <malloc.h>
#endif
#ifdef OPENCV_ALLOC_ENABLE_STATISTICS
#include <map>
#endif
namespace cv {
static void* OutOfMemoryError(size_t size)
......@@ -55,8 +72,21 @@ static void* OutOfMemoryError(size_t size)
CV_Error_(CV_StsNoMem, ("Failed to allocate %llu bytes", (unsigned long long)size));
}
CV_EXPORTS cv::utils::AllocatorStatisticsInterface& getAllocatorStatistics();
static cv::utils::AllocatorStatistics allocator_stats;
void* fastMalloc( size_t size )
cv::utils::AllocatorStatisticsInterface& getAllocatorStatistics()
{
return allocator_stats;
}
#ifdef OPENCV_ALLOC_ENABLE_STATISTICS
static inline
void* fastMalloc_(size_t size)
#else
void* fastMalloc(size_t size)
#endif
{
#ifdef HAVE_POSIX_MEMALIGN
void* ptr = NULL;
......@@ -80,7 +110,12 @@ void* fastMalloc( size_t size )
#endif
}
#ifdef OPENCV_ALLOC_ENABLE_STATISTICS
static inline
void fastFree_(void* ptr)
#else
void fastFree(void* ptr)
#endif
{
#if defined HAVE_POSIX_MEMALIGN || defined HAVE_MEMALIGN
free(ptr);
......@@ -95,6 +130,47 @@ void fastFree(void* ptr)
#endif
}
#ifdef OPENCV_ALLOC_ENABLE_STATISTICS
static
Mutex& getAllocationStatisticsMutex()
{
static Mutex* p_alloc_mutex = allocSingletonNew<Mutex>();
CV_Assert(p_alloc_mutex);
return *p_alloc_mutex;
}
static std::map<void*, size_t> allocated_buffers; // guarded by getAllocationStatisticsMutex()
void* fastMalloc(size_t size)
{
void* res = fastMalloc_(size);
if (res && size >= OPENCV_ALLOC_STATISTICS_LIMIT)
{
cv::AutoLock lock(getAllocationStatisticsMutex());
allocated_buffers.insert(std::make_pair(res, size));
allocator_stats.onAllocate(size);
}
return res;
}
void fastFree(void* ptr)
{
{
cv::AutoLock lock(getAllocationStatisticsMutex());
std::map<void*, size_t>::iterator i = allocated_buffers.find(ptr);
if (i != allocated_buffers.end())
{
size_t size = i->second;
allocator_stats.onFree(size);
allocated_buffers.erase(i);
}
}
fastFree_(ptr);
}
#endif // OPENCV_ALLOC_ENABLE_STATISTICS
} // namespace
CV_IMPL void* cvAlloc( size_t size )
......
modules/core/src/ocl.cpp
View file @ 64629cb9
......@@ -54,6 +54,9 @@
#include <opencv2/core/utils/configuration.private.hpp>
#include <opencv2/core/utils/logger.defines.hpp>
#undef CV_LOG_STRIP_LEVEL
#define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_DEBUG + 1
#include <opencv2/core/utils/logger.hpp>
#include "opencv2/core/ocl_genbase.hpp"
......@@ -63,6 +66,10 @@
#include "opencv2/core/utils/filesystem.hpp"
#include "opencv2/core/utils/filesystem.private.hpp"
#define CV__ALLOCATOR_STATS_LOG(...) CV_LOG_VERBOSE(NULL, 0, "OpenCL allocator: " << __VA_ARGS__)
#include "opencv2/core/utils/allocator_stats.impl.hpp"
#undef CV__ALLOCATOR_STATS_LOG
#define CV_OPENCL_ALWAYS_SHOW_BUILD_LOG 0
#define CV_OPENCL_SHOW_RUN_KERNELS 0
......@@ -132,6 +139,14 @@ namespace cv { namespace ocl {
void release() { if( CV_XADD(&refcount, -1) == 1 && !cv::__termination) delete this; } \
int refcount
static cv::utils::AllocatorStatistics opencl_allocator_stats;
CV_EXPORTS cv::utils::AllocatorStatisticsInterface& getOpenCLAllocatorStatistics();
cv::utils::AllocatorStatisticsInterface& getOpenCLAllocatorStatistics()
{
return opencl_allocator_stats;
}
#ifndef HAVE_OPENCL
#define CV_OPENCL_NO_SUPPORT() CV_Error(cv::Error::OpenCLApiCallError, "OpenCV build without OpenCL support")
namespace {
......@@ -4539,15 +4554,17 @@ class OpenCLAllocator CV_FINAL : public MatAllocator
mutable OpenCLSVMBufferPoolImpl bufferPoolSVM;
#endif
public:
enum AllocatorFlags
{
ALLOCATOR_FLAGS_BUFFER_POOL_USED = 1 << 0,
ALLOCATOR_FLAGS_BUFFER_POOL_HOST_PTR_USED = 1 << 1
ALLOCATOR_FLAGS_BUFFER_POOL_HOST_PTR_USED = 1 << 1,
#ifdef HAVE_OPENCL_SVM
,ALLOCATOR_FLAGS_BUFFER_POOL_SVM_USED = 1 << 2
ALLOCATOR_FLAGS_BUFFER_POOL_SVM_USED = 1 << 2,
#endif
ALLOCATOR_FLAGS_EXTERNAL_BUFFER = 1 << 3 // convertFromBuffer()
};
public:
OpenCLAllocator()
: bufferPool(0),
bufferPoolHostPtr(CL_MEM_ALLOC_HOST_PTR)
......@@ -4652,6 +4669,7 @@ public:
u->allocatorFlags_ = allocatorFlags;
CV_DbgAssert(!u->tempUMat()); // for bufferPool.release() consistency in deallocate()
u->markHostCopyObsolete(true);
opencl_allocator_stats.onAllocate(u->size);
return u;
}
......@@ -4760,6 +4778,7 @@ public:
}
if(accessFlags & ACCESS_WRITE)
u->markHostCopyObsolete(true);
opencl_allocator_stats.onAllocate(u->size);
return true;
}
......@@ -4812,6 +4831,13 @@ public:
void deallocate_(UMatData* u) const
{
CV_Assert(u);
CV_Assert(u->handle);
if ((u->allocatorFlags_ & ALLOCATOR_FLAGS_EXTERNAL_BUFFER) == 0)
{
opencl_allocator_stats.onFree(u->size);
}
#ifdef _WIN32
if (cv::__termination) // process is not in consistent state (after ExitProcess call) and terminating
return; // avoid any OpenCL calls
......@@ -5793,7 +5819,7 @@ void convertFromBuffer(void* cl_mem_buffer, size_t step, int rows, int cols, int
// attach clBuffer to UMatData
dst.u = new UMatData(getOpenCLAllocator());
dst.u->data = 0;
dst.u->allocatorFlags_ = 0; // not allocated from any OpenCV buffer pool
dst.u->allocatorFlags_ = OpenCLAllocator::ALLOCATOR_FLAGS_EXTERNAL_BUFFER; // not allocated from any OpenCV buffer pool
dst.u->flags = 0;
dst.u->handle = cl_mem_buffer;
dst.u->origdata = 0;
......
modules/core/src/system.cpp
View file @ 64629cb9
......@@ -71,6 +71,8 @@ static bool param_dumpErrors = utils::getConfigurationParameterBool("OPENCV_DUMP
);
void* allocSingletonBuffer(size_t size) { return fastMalloc(size); }
void* allocSingletonNewBuffer(size_t size) { return malloc(size); }
} // namespace cv
......
modules/dnn/test/test_backends.cpp
View file @ 64629cb9
......@@ -98,6 +98,7 @@ public:
TEST_P(DNNTestNetwork, AlexNet)
{
applyTestTag(CV_TEST_TAG_MEMORY_1GB);
processNet("dnn/bvlc_alexnet.caffemodel", "dnn/bvlc_alexnet.prototxt",
Size(227, 227), "prob",
target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_alexnet.yml" :
......@@ -106,6 +107,7 @@ TEST_P(DNNTestNetwork, AlexNet)
TEST_P(DNNTestNetwork, ResNet_50)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt",
Size(224, 224), "prob",
target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_resnet_50.yml" :
......@@ -122,12 +124,14 @@ TEST_P(DNNTestNetwork, SqueezeNet_v1_1)
TEST_P(DNNTestNetwork, GoogLeNet)
{
applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
processNet("dnn/bvlc_googlenet.caffemodel", "dnn/bvlc_googlenet.prototxt",
Size(224, 224), "prob");
}
TEST_P(DNNTestNetwork, Inception_5h)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
double l1 = default_l1, lInf = default_lInf;
if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_CPU || target == DNN_TARGET_OPENCL))
{
......@@ -142,6 +146,7 @@ TEST_P(DNNTestNetwork, Inception_5h)
TEST_P(DNNTestNetwork, ENet)
{
applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
if ((backend == DNN_BACKEND_INFERENCE_ENGINE) ||
(backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16))
throw SkipTestException("");
......@@ -153,6 +158,7 @@ TEST_P(DNNTestNetwork, ENet)
TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
Mat sample = imread(findDataFile("dnn/street.png", false));
......@@ -184,6 +190,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe_Different_Width_Height)
TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow)
{
applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
Mat sample = imread(findDataFile("dnn/street.png", false));
......@@ -214,6 +221,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow_Different_Width_Height)
TEST_P(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
Mat sample = imread(findDataFile("dnn/street.png", false));
......@@ -226,6 +234,8 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow)
TEST_P(DNNTestNetwork, SSD_VGG16)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
CV_TEST_TAG_DEBUG_VERYLONG);
if (backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU)
throw SkipTestException("");
double scoreThreshold = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0325 : 0.0;
......@@ -238,6 +248,8 @@ TEST_P(DNNTestNetwork, SSD_VGG16)
TEST_P(DNNTestNetwork, OpenPose_pose_coco)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
CV_TEST_TAG_DEBUG_VERYLONG);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000)
......@@ -254,6 +266,8 @@ TEST_P(DNNTestNetwork, OpenPose_pose_coco)
TEST_P(DNNTestNetwork, OpenPose_pose_mpi)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
CV_TEST_TAG_DEBUG_VERYLONG);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000)
......@@ -270,6 +284,7 @@ TEST_P(DNNTestNetwork, OpenPose_pose_mpi)
TEST_P(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
{
applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_1GB);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000)
......@@ -313,6 +328,7 @@ TEST_P(DNNTestNetwork, opencv_face_detector)
TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
&& getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
......@@ -330,6 +346,7 @@ TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
TEST_P(DNNTestNetwork, DenseNet_121)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
// Reference output values are in range [-3.807, 4.605]
......
modules/dnn/test/test_caffe_importer.cpp
View file @ 64629cb9
......@@ -112,6 +112,8 @@ TEST(Test_Caffe, read_googlenet)
typedef testing::TestWithParam<tuple<bool, Target> > Reproducibility_AlexNet;
TEST_P(Reproducibility_AlexNet, Accuracy)
{
Target targetId = get<1>(GetParam());
applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
bool readFromMemory = get<0>(GetParam());
Net net;
{
......@@ -132,7 +134,6 @@ TEST_P(Reproducibility_AlexNet, Accuracy)
ASSERT_FALSE(net.empty());
}
int targetId = get<1>(GetParam());
const float l1 = 1e-5;
const float lInf = (targetId == DNN_TARGET_OPENCL_FP16) ? 3e-3 : 1e-4;
......@@ -151,9 +152,9 @@ TEST_P(Reproducibility_AlexNet, Accuracy)
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_AlexNet, Combine(testing::Bool(),
Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16)));
#if !defined(_WIN32) || defined(_WIN64)
TEST(Reproducibility_FCN, Accuracy)
{
applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_2GB);
Net net;
{
const string proto = findDataFile("dnn/fcn8s-heavy-pascal.prototxt", false);
......@@ -179,10 +180,10 @@ TEST(Reproducibility_FCN, Accuracy)
normAssert(ref, out);
}
#endif
TEST(Reproducibility_SSD, Accuracy)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
Net net;
{
const string proto = findDataFile("dnn/ssd_vgg16.prototxt", false);
......@@ -264,10 +265,11 @@ INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_MobileNet_SSD,
typedef testing::TestWithParam<Target> Reproducibility_ResNet50;
TEST_P(Reproducibility_ResNet50, Accuracy)
{
Target targetId = GetParam();
applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
Net net = readNetFromCaffe(findDataFile("dnn/ResNet-50-deploy.prototxt", false),
findDataFile("dnn/ResNet-50-model.caffemodel", false));
int targetId = GetParam();
net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(targetId);
......@@ -330,6 +332,7 @@ INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_SqueezeNet_v1_1,
TEST(Reproducibility_AlexNet_fp16, Accuracy)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
const float l1 = 1e-5;
const float lInf = 3e-3;
......@@ -375,6 +378,7 @@ TEST(Reproducibility_GoogLeNet_fp16, Accuracy)
// https://github.com/richzhang/colorization
TEST_P(Test_Caffe_nets, Colorization)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
checkBackend();
Mat inp = blobFromNPY(_tf("colorization_inp.npy"));
......@@ -405,6 +409,7 @@ TEST_P(Test_Caffe_nets, Colorization)
TEST_P(Test_Caffe_nets, DenseNet_121)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
checkBackend();
const string proto = findDataFile("dnn/DenseNet_121.prototxt", false);
const string model = findDataFile("dnn/DenseNet_121.caffemodel", false);
......@@ -520,6 +525,8 @@ INSTANTIATE_TEST_CASE_P(Test_Caffe, opencv_face_detector,
TEST_P(Test_Caffe_nets, FasterRCNN_vgg16)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB));
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
throw SkipTestException("Test is disabled for DLIE OpenCL targets"); // very slow
......@@ -536,6 +543,7 @@ TEST_P(Test_Caffe_nets, FasterRCNN_vgg16)
TEST_P(Test_Caffe_nets, FasterRCNN_zf)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
if ((backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16) ||
(backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD))
throw SkipTestException("");
......@@ -547,6 +555,7 @@ TEST_P(Test_Caffe_nets, FasterRCNN_zf)
TEST_P(Test_Caffe_nets, RFCN)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_2GB));
if ((backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16) ||
(backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD))
throw SkipTestException("");
......
modules/dnn/test/test_darknet_importer.cpp
View file @ 64629cb9
......@@ -78,6 +78,7 @@ TEST(Test_Darknet, read_yolo_voc)
TEST(Test_Darknet, read_yolo_voc_stream)
{
applyTestTag(CV_TEST_TAG_MEMORY_1GB);
Mat ref;
Mat sample = imread(_tf("dog416.png"));
Mat inp = blobFromImage(sample, 1.0/255, Size(416, 416), Scalar(), true, false);
......@@ -267,6 +268,8 @@ public:
TEST_P(Test_Darknet_nets, YoloVoc)
{
applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_1GB);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16)
throw SkipTestException("Test is disabled");
......@@ -305,6 +308,8 @@ TEST_P(Test_Darknet_nets, YoloVoc)
TEST_P(Test_Darknet_nets, TinyYoloVoc)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
&& getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
......@@ -339,6 +344,8 @@ TEST_P(Test_Darknet_nets, TinyYoloVoc)
TEST_P(Test_Darknet_nets, YOLOv3)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB));
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
&& getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
......
modules/dnn/test/test_onnx_importer.cpp
View file @ 64629cb9
......@@ -217,6 +217,7 @@ INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_ONNX_layers, dnnBackendsAndTargets());
class Test_ONNX_nets : public Test_ONNX_layers {};
TEST_P(Test_ONNX_nets, Alexnet)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
const String model = _tf("models/alexnet.onnx");
Net net = readNetFromONNX(model);
......@@ -270,31 +271,30 @@ TEST_P(Test_ONNX_nets, Googlenet)
TEST_P(Test_ONNX_nets, CaffeNet)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
testONNXModels("caffenet", pb);
}
TEST_P(Test_ONNX_nets, RCNN_ILSVRC13)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
// Reference output values are in range [-4.992, -1.161]
testONNXModels("rcnn_ilsvrc13", pb, 0.0045);
}
#ifdef OPENCV_32BIT_CONFIGURATION
TEST_P(Test_ONNX_nets, DISABLED_VGG16) // memory usage >2Gb
#else
TEST_P(Test_ONNX_nets, VGG16)
#endif
{
applyTestTag(CV_TEST_TAG_MEMORY_6GB); // > 2.3Gb
// output range: [-69; 72], after Softmax [0; 0.96]
testONNXModels("vgg16", pb, default_l1, default_lInf, true);
}
#ifdef OPENCV_32BIT_CONFIGURATION
TEST_P(Test_ONNX_nets, DISABLED_VGG16_bn) // memory usage >2Gb
#else
TEST_P(Test_ONNX_nets, VGG16_bn)
#endif
{
applyTestTag(CV_TEST_TAG_MEMORY_6GB); // > 2.3Gb
// output range: [-16; 27], after Softmax [0; 0.67]
const double lInf = (target == DNN_TARGET_MYRIAD) ? 0.038 : default_lInf;
testONNXModels("vgg16-bn", pb, default_l1, lInf, true);
......@@ -302,23 +302,30 @@ TEST_P(Test_ONNX_nets, VGG16_bn)
TEST_P(Test_ONNX_nets, ZFNet)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
testONNXModels("zfnet512", pb);
}
TEST_P(Test_ONNX_nets, ResNet18v1)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
// output range: [-16; 22], after Softmax [0, 0.51]
testONNXModels("resnet18v1", pb, default_l1, default_lInf, true);
}
TEST_P(Test_ONNX_nets, ResNet50v1)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
// output range: [-67; 75], after Softmax [0, 0.98]
testONNXModels("resnet50v1", pb, default_l1, default_lInf, true);
}
TEST_P(Test_ONNX_nets, ResNet101_DUC_HDC)
{
applyTestTag(CV_TEST_TAG_VERYLONG);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
if (backend == DNN_BACKEND_INFERENCE_ENGINE)
throw SkipTestException("Test is disabled for DLIE targets");
......@@ -334,6 +341,8 @@ TEST_P(Test_ONNX_nets, ResNet101_DUC_HDC)
TEST_P(Test_ONNX_nets, TinyYolov2)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
if (cvtest::skipUnstableTests)
throw SkipTestException("Skip unstable test");
#if defined(INF_ENGINE_RELEASE)
......@@ -347,6 +356,7 @@ TEST_P(Test_ONNX_nets, TinyYolov2)
)
throw SkipTestException("Test is disabled for MyriadX");
#endif
// output range: [-11; 8]
double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.017 : default_l1;
double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.14 : default_lInf;
......@@ -367,6 +377,7 @@ TEST_P(Test_ONNX_nets, MobileNet_v2)
TEST_P(Test_ONNX_nets, LResNet100E_IR)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
(target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_OPENCL || target == DNN_TARGET_MYRIAD))
throw SkipTestException("");
......@@ -419,6 +430,8 @@ TEST_P(Test_ONNX_nets, Inception_v2)
TEST_P(Test_ONNX_nets, DenseNet121)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
// output range: [-87; 138], after Softmax [0; 1]
testONNXModels("densenet121", pb, default_l1, default_lInf, true);
}
......
modules/dnn/test/test_tf_importer.cpp
View file @ 64629cb9
......@@ -339,6 +339,8 @@ TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
&& getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
......@@ -404,6 +406,7 @@ TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
TEST_P(Test_TensorFlow_nets, Faster_RCNN)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); // FIXIT split test
static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
"faster_rcnn_resnet50_coco_2018_01_28"};
......@@ -499,6 +502,8 @@ TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
// np.save('east_text_detection.geometry.npy', geometry)
TEST_P(Test_TensorFlow_nets, EAST_text_detection)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
throw SkipTestException("Test is disabled for Myriad targets");
......@@ -669,6 +674,7 @@ TEST(Test_TensorFlow, two_inputs)
TEST(Test_TensorFlow, Mask_RCNN)
{
applyTestTag(CV_TEST_TAG_MEMORY_1GB);
std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt", false);
std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
......
modules/dnn/test/test_torch_importer.cpp
View file @ 64629cb9
......@@ -345,6 +345,7 @@ static void normAssertSegmentation(const Mat& ref, const Mat& test)
TEST_P(Test_Torch_nets, ENet_accuracy)
{
applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
checkBackend();
if (backend == DNN_BACKEND_INFERENCE_ENGINE ||
(backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16))
......
modules/ts/include/opencv2/ts.hpp
View file @ 64629cb9
......@@ -51,6 +51,44 @@
# endif
#endif
// most part of OpenCV tests are fit into 200Mb limit, but some tests are not:
// Note: due memory fragmentation real limits are usually lower on 20-25% (400Mb memory usage goes into mem_1Gb class)
#define CV_TEST_TAG_MEMORY_512MB "mem_512mb" // used memory: 200..512Mb - enabled by default
#define CV_TEST_TAG_MEMORY_1GB "mem_1gb" // used memory: 512Mb..1Gb - enabled by default
#define CV_TEST_TAG_MEMORY_2GB "mem_2gb" // used memory: 1..2Gb - enabled by default on 64-bit configuration (32-bit - disabled)
#define CV_TEST_TAG_MEMORY_6GB "mem_6gb" // used memory: 2..6Gb - disabled by default
#define CV_TEST_TAG_MEMORY_14GB "mem_14gb" // used memory: 6..14Gb - disabled by default
// Large / huge video streams or complex workloads
#define CV_TEST_TAG_LONG "long" // 5+ seconds on modern desktop machine (single thread)
#define CV_TEST_TAG_VERYLONG "verylong" // 20+ seconds on modern desktop machine (single thread)
// Large / huge video streams or complex workloads for debug builds
#define CV_TEST_TAG_DEBUG_LONG "debug_long" // 10+ seconds on modern desktop machine (single thread)
#define CV_TEST_TAG_DEBUG_VERYLONG "debug_verylong" // 40+ seconds on modern desktop machine (single thread)
// Lets skip processing of high resolution images via instrumentation tools (valgrind/coverage/sanitizers).
// It is enough to run lower resolution (VGA: 640x480) tests.
#define CV_TEST_TAG_SIZE_HD "size_hd" // 720p+, enabled
#define CV_TEST_TAG_SIZE_FULLHD "size_fullhd" // 1080p+, enabled (disable these tests for valgrind/coverage run)
#define CV_TEST_TAG_SIZE_4K "size_4k" // 2160p+, enabled (disable these tests for valgrind/coverage run)
// Other misc test tags
#define CV_TEST_TAG_TYPE_64F "type_64f" // CV_64F, enabled (disable these tests on low power embedded devices)
// Kernel-based image processing
#define CV_TEST_TAG_FILTER_SMALL "filter_small" // Filtering with kernels <= 3x3
#define CV_TEST_TAG_FILTER_MEDIUM "filter_medium" // Filtering with kernels: 3x3 < kernel <= 5x5
#define CV_TEST_TAG_FILTER_LARGE "filter_large" // Filtering with kernels: 5x5 < kernel <= 9x9
#define CV_TEST_TAG_FILTER_HUGE "filter_huge" // Filtering with kernels: > 9x9
// Other tests categories
#define CV_TEST_TAG_OPENCL "opencl" // Tests with OpenCL
#ifdef WINRT
#pragma warning(disable:4447) // Disable warning 'main' signature found without threading model
#endif
......@@ -150,6 +188,30 @@ public:
SkipTestException(const cv::String& message) : dummy(0) { this->msg = message; }
};
/** Apply tag to the current test
Automatically apply corresponding additional tags (for example, 4K => FHD => HD => VGA).
If tag is in skip list, then SkipTestException is thrown
*/
void applyTestTag(const std::string& tag);
/** Run postponed checks of applied test tags
If tag is in skip list, then SkipTestException is thrown
*/
void checkTestTags();
void applyTestTag_(const std::string& tag);
static inline void applyTestTag(const std::string& tag1, const std::string& tag2)
{ applyTestTag_(tag1); applyTestTag_(tag2); checkTestTags(); }
static inline void applyTestTag(const std::string& tag1, const std::string& tag2, const std::string& tag3)
{ applyTestTag_(tag1); applyTestTag_(tag2); applyTestTag_(tag3); checkTestTags(); }
static inline void applyTestTag(const std::string& tag1, const std::string& tag2, const std::string& tag3, const std::string& tag4)
{ applyTestTag_(tag1); applyTestTag_(tag2); applyTestTag_(tag3); applyTestTag_(tag4); checkTestTags(); }
class TS;
int64 readSeed(const char* str);
......
modules/ts/include/opencv2/ts/ts_ext.hpp
View file @ 64629cb9
......@@ -13,6 +13,9 @@ void checkIppStatus();
extern bool skipUnstableTests;
extern bool runBigDataTests;
extern int testThreads;
void testSetUp();
void testTearDown();
}
// check for required "opencv_test" namespace
......@@ -24,10 +27,8 @@ extern int testThreads;
#define CV__TEST_INIT \
CV__TEST_NAMESPACE_CHECK \
cv::ipp::setIppStatus(0); \
cv::theRNG().state = cvtest::param_seed; \
cv::setNumThreads(cvtest::testThreads);
#define CV__TEST_CLEANUP ::cvtest::checkIppStatus();
::cvtest::testSetUp();
#define CV__TEST_CLEANUP ::cvtest::testTearDown();
#define CV__TEST_BODY_IMPL(name) \
{ \
CV__TRACE_APP_FUNCTION_NAME(name); \
......
modules/ts/include/opencv2/ts/ts_perf.hpp
View file @ 64629cb9
......@@ -527,7 +527,15 @@ void PrintTo(const Size& sz, ::std::ostream* os);
{ \
CV__TEST_NAMESPACE_CHECK \
CV__TRACE_APP_FUNCTION_NAME("PERF_TEST: " name); \
try { \
::cvtest::testSetUp(); \
RunPerfTestBody(); \
} \
catch (cvtest::SkipTestException& e) \
{ \
printf("[ SKIP ] %s\n", e.what()); \
} \
::cvtest::testTearDown(); \
}
#define PERF_PROXY_NAMESPACE_NAME_(test_case_name, test_name) \
......
modules/ts/misc/testlog_parser.py
View file @ 64629cb9
......@@ -51,6 +51,7 @@ class TestInfo(object):
self.parseLongMetric(xmlnode, "stddev");
self.parseFloatMetric(xmlnode, "gstddev");
self.parseFloatMetric(xmlnode, "time");
self.parseLongMetric(xmlnode, "total_memory_usage");
def parseLongMetric(self, xmlnode, name, default = 0):
if name in self.properties:
......
modules/ts/src/precomp.hpp
View file @ 64629cb9
#include "opencv2/ts.hpp"
#include <opencv2/core/utils/logger.hpp>
#include "opencv2/core/utility.hpp"
#include "opencv2/core/private.hpp"
#ifdef GTEST_LINKED_AS_SHARED_LIBRARY
......
modules/ts/src/ts.cpp
View file @ 64629cb9
......@@ -91,11 +91,31 @@
#include "opencv2/core/opencl/opencl_info.hpp"
#include "opencv2/core/utils/allocator_stats.hpp"
namespace cv { namespace ocl {
cv::utils::AllocatorStatisticsInterface& getOpenCLAllocatorStatistics();
}}
#endif // HAVE_OPENCL
#include "opencv2/core/utility.hpp"
#include "opencv2/core/utils/allocator_stats.hpp"
namespace cv {
CV_EXPORTS cv::utils::AllocatorStatisticsInterface& getAllocatorStatistics();
}
#include "opencv_tests_config.hpp"
#include "ts_tags.hpp"
#if defined(__GNUC__) && defined(__linux__)
extern "C" {
size_t malloc_peak(void) __attribute__((weak));
void malloc_reset_peak(void) __attribute__((weak));
} // extern "C"
#else // stubs
static size_t (*malloc_peak)(void) = 0;
static void (*malloc_reset_peak)(void) = 0;
#endif
namespace opencv_test {
bool required_opencv_test_namespace = false; // compilation check for non-refactored tests
}
......@@ -726,6 +746,85 @@ bool skipUnstableTests = false;
bool runBigDataTests = false;
int testThreads = 0;
static size_t memory_usage_base = 0;
static uint64_t memory_usage_base_opencv = 0;
#ifdef HAVE_OPENCL
static uint64_t memory_usage_base_opencl = 0;
#endif
void testSetUp()
{
cv::ipp::setIppStatus(0);
cv::theRNG().state = cvtest::param_seed;
cv::setNumThreads(cvtest::testThreads);
if (malloc_peak) // if memory profiler is available
{
malloc_reset_peak();
memory_usage_base = malloc_peak(); // equal to malloc_current()
}
{
cv::utils::AllocatorStatisticsInterface& ocv_stats = cv::getAllocatorStatistics();
ocv_stats.resetPeakUsage();
memory_usage_base_opencv = ocv_stats.getCurrentUsage();
}
#ifdef HAVE_OPENCL
{
cv::utils::AllocatorStatisticsInterface& ocl_stats = cv::ocl::getOpenCLAllocatorStatistics();
ocl_stats.resetPeakUsage();
memory_usage_base_opencl = ocl_stats.getCurrentUsage();
}
#endif
checkTestTags();
}
void testTearDown()
{
::cvtest::checkIppStatus();
uint64_t memory_usage = 0;
uint64_t ocv_memory_usage = 0, ocv_peak = 0;
if (malloc_peak) // if memory profiler is available
{
size_t peak = malloc_peak();
memory_usage = peak - memory_usage_base;
CV_LOG_INFO(NULL, "Memory_usage (malloc): " << memory_usage << " (base=" << memory_usage_base << ")");
}
{
// core/src/alloc.cpp: #define OPENCV_ALLOC_ENABLE_STATISTICS
// handle large buffers via fastAlloc()
// (not always accurate on heavy 3rdparty usage, like protobuf)
cv::utils::AllocatorStatisticsInterface& ocv_stats = cv::getAllocatorStatistics();
ocv_peak = ocv_stats.getPeakUsage();
ocv_memory_usage = ocv_peak - memory_usage_base_opencv;
CV_LOG_INFO(NULL, "Memory_usage (OpenCV): " << ocv_memory_usage << " (base=" << memory_usage_base_opencv << " current=" << ocv_stats.getCurrentUsage() << ")");
if (memory_usage == 0) // external profiler has higher priority (and accuracy)
memory_usage = ocv_memory_usage;
}
#ifdef HAVE_OPENCL
uint64_t ocl_memory_usage = 0, ocl_peak = 0;
{
cv::utils::AllocatorStatisticsInterface& ocl_stats = cv::ocl::getOpenCLAllocatorStatistics();
ocl_peak = ocl_stats.getPeakUsage();
ocl_memory_usage = ocl_peak - memory_usage_base_opencl;
CV_LOG_INFO(NULL, "Memory_usage (OpenCL): " << ocl_memory_usage << " (base=" << memory_usage_base_opencl << " current=" << ocl_stats.getCurrentUsage() << ")");
::testing::Test::RecordProperty("ocl_memory_usage",
cv::format("%llu", (unsigned long long)ocl_memory_usage));
}
#else
uint64_t ocl_memory_usage = 0;
#endif
if (malloc_peak // external memory profiler is available
|| ocv_peak > 0 // or enabled OpenCV builtin allocation statistics
)
{
CV_LOG_INFO(NULL, "Memory usage total: " << (memory_usage + ocl_memory_usage));
::testing::Test::RecordProperty("memory_usage",
cv::format("%llu", (unsigned long long)memory_usage));
::testing::Test::RecordProperty("total_memory_usage",
cv::format("%llu", (unsigned long long)(memory_usage + ocl_memory_usage)));
}
}
void parseCustomOptions(int argc, char **argv)
{
const char * const command_line_keys =
......@@ -735,7 +834,9 @@ void parseCustomOptions(int argc, char **argv)
"{ skip_unstable |false |skip unstable tests }"
"{ test_bigdata |false |run BigData tests (>=2Gb) }"
"{ test_require_data |false |fail on missing non-required test data instead of skip}"
"{ h help |false |print help info }";
CV_TEST_TAGS_PARAMS
"{ h help |false |print help info }"
;
cv::CommandLineParser parser(argc, argv, command_line_keys);
if (parser.get<bool>("help"))
......@@ -759,8 +860,9 @@ void parseCustomOptions(int argc, char **argv)
skipUnstableTests = parser.get<bool>("skip_unstable");
runBigDataTests = parser.get<bool>("test_bigdata");
checkTestData = parser.get<bool>("test_require_data");
}
activateTestTags(parser);
}
static bool isDirectory(const std::string& path)
{
......
modules/ts/src/ts_perf.cpp
View file @ 64629cb9
#include "precomp.hpp"
#include "ts_tags.hpp"
#include <map>
#include <iostream>
#include <fstream>
......@@ -999,6 +1001,8 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
"{ perf_cuda_info_only |false |print an information about system and an available CUDA devices and then exit.}"
#endif
"{ skip_unstable |false |skip unstable tests }"
CV_TEST_TAGS_PARAMS
;
cv::CommandLineParser args(argc, argv, command_line_keys);
......@@ -1145,6 +1149,8 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
::testing::AddGlobalTestEnvironment(new PerfValidationEnvironment());
}
activateTestTags(args);
if (!args.check())
{
args.printErrors();
......@@ -1869,14 +1875,15 @@ void TestBase::SetUp()
currentIter = (unsigned int)-1;
timeLimit = timeLimitDefault;
times.clear();
metrics.terminationReason = performance_metrics::TERM_SKIP_TEST;
}
void TestBase::TearDown()
{
if (metrics.terminationReason == performance_metrics::TERM_SKIP_TEST)
{
LOGI("\tTest was skipped");
GTEST_SUCCEED() << "Test was skipped";
//LOGI("\tTest was skipped");
//GTEST_SUCCEED() << "Test was skipped";
}
else
{
......@@ -1975,6 +1982,7 @@ std::string TestBase::getDataPath(const std::string& relativePath)
void TestBase::RunPerfTestBody()
{
metrics.clear();
try
{
#ifdef CV_COLLECT_IMPL_DATA
......@@ -1990,7 +1998,7 @@ void TestBase::RunPerfTestBody()
catch(const SkipTestException&)
{
metrics.terminationReason = performance_metrics::TERM_SKIP_TEST;
return;
throw;
}
catch(const PerfSkipTestException&)
{
......
modules/ts/src/ts_tags.cpp 0 → 100644
View file @ 64629cb9
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "precomp.hpp"
#include "ts_tags.hpp"
namespace cvtest {
static bool printTestTag = false;
static std::vector<std::string> currentDirectTestTags, currentImpliedTestTags;
static std::vector<const ::testing::TestInfo*> skipped_tests;
static std::vector<std::string>& getTestTagsSkipList()
{
static std::vector<std::string> testSkipWithTags;
static bool initialized = false;
if (!initialized)
{
#if OPENCV_32BIT_CONFIGURATION
testSkipWithTags.push_back(CV_TEST_TAG_MEMORY_2GB);
#else
testSkipWithTags.push_back(CV_TEST_TAG_MEMORY_6GB);
#endif
testSkipWithTags.push_back(CV_TEST_TAG_VERYLONG);
#if defined(_DEBUG)
testSkipWithTags.push_back(CV_TEST_TAG_DEBUG_VERYLONG);
#endif
initialized = true;
}
return testSkipWithTags;
}
static std::vector<std::string>& getTestTagsForceList()
{
static std::vector<std::string> getTestTagsForceList;
return getTestTagsForceList;
}
static std::vector<std::string>& getTestTagsRequiredList()
{
static std::vector<std::string> getTestTagsRequiredList;
return getTestTagsRequiredList;
}
class TestTagsListener: public ::testing::EmptyTestEventListener
{
public:
void OnTestProgramStart(const ::testing::UnitTest& /*unit_test*/) CV_OVERRIDE
{
{
const std::vector<std::string>& tags = getTestTagsRequiredList();
std::ostringstream os, os_direct;
for (size_t i = 0; i < tags.size(); i++)
{
os << (i == 0 ? "'" : ", '") << tags[i] << "'";
os_direct << (i == 0 ? "" : ",") << tags[i];
}
std::string tags_str = os.str();
if (!tags.empty())
std::cout << "TEST: Run tests with tags: " << tags_str << std::endl;
::testing::Test::RecordProperty("test_tags", os_direct.str());
}
{
const std::vector<std::string>& tags = getTestTagsSkipList();
std::ostringstream os, os_direct;
for (size_t i = 0; i < tags.size(); i++)
{
os << (i == 0 ? "'" : ", '") << tags[i] << "'";
os_direct << (i == 0 ? "" : ",") << tags[i];
}
std::string tags_str = os.str();
if (!tags.empty())
std::cout << "TEST: Skip tests with tags: " << tags_str << std::endl;
::testing::Test::RecordProperty("test_tags_skip", os_direct.str());
}
{
const std::vector<std::string>& tags = getTestTagsForceList();
std::ostringstream os, os_direct;
for (size_t i = 0; i < tags.size(); i++)
{
os << (i == 0 ? "'" : ", '") << tags[i] << "'";
os_direct << (i == 0 ? "" : ",") << tags[i];
}
std::string tags_str = os.str();
if (!tags.empty())
std::cout << "TEST: Force tests with tags: " << tags_str << std::endl;
::testing::Test::RecordProperty("test_tags_force", os_direct.str());
}
}
void OnTestStart(const ::testing::TestInfo& test_info) CV_OVERRIDE
{
currentDirectTestTags.clear();
currentImpliedTestTags.clear();
const char* value_param_ = test_info.value_param();
if (value_param_)
{
std::string value_param(value_param_);
if (value_param.find("CV_64F") != std::string::npos
|| (value_param.find("64F") != std::string::npos
&& value_param.find(" 64F") != std::string::npos
&& value_param.find(",64F") != std::string::npos
&& value_param.find("(64F") != std::string::npos
)
)
applyTestTag_(CV_TEST_TAG_TYPE_64F);
if (value_param.find("1280x720") != std::string::npos)
applyTestTag_(CV_TEST_TAG_SIZE_HD);
if (value_param.find("1920x1080") != std::string::npos)
applyTestTag_(CV_TEST_TAG_SIZE_FULLHD);
if (value_param.find("3840x2160") != std::string::npos)
applyTestTag_(CV_TEST_TAG_SIZE_4K);
}
}
void OnTestEnd(const ::testing::TestInfo& /*test_info*/) CV_OVERRIDE
{
if (currentDirectTestTags.empty() && currentImpliedTestTags.empty())
{
if (printTestTag) std::cout << "[ TAGS ] No tags" << std::endl;
return;
}
std::ostringstream os;
std::ostringstream os_direct;
std::ostringstream os_implied;
{
const std::vector<std::string>& tags = currentDirectTestTags;
for (size_t i = 0; i < tags.size(); i++)
{
os << (i == 0 ? "" : ", ") << tags[i];
os_direct << (i == 0 ? "" : ",") << tags[i];
}
}
if (!currentImpliedTestTags.empty())
{
os << " (implied tags: ";
const std::vector<std::string>& tags = currentImpliedTestTags;
for (size_t i = 0; i < tags.size(); i++)
{
os << (i == 0 ? "" : ", ") << tags[i];
os_implied << (i == 0 ? "" : ",") << tags[i];
}
os << ")";
}
if (printTestTag) std::cout << "[ TAGS ] " << os.str() << std::endl;
::testing::Test::RecordProperty("tags", os_direct.str());
::testing::Test::RecordProperty("tags_implied", os_implied.str());
}
void OnTestIterationEnd(const ::testing::UnitTest& /*unit_test*/, int /*iteration*/) CV_OVERRIDE
{
if (!skipped_tests.empty())
{
std::cout << "[ SKIP ] " << skipped_tests.size() << " tests via tags" << std::endl;
}
skipped_tests.clear();
}
void OnTestProgramEnd(const ::testing::UnitTest& /*unit_test*/) CV_OVERRIDE
{
/*if (!skipped_tests.empty())
{
for (size_t i = 0; i < skipped_tests.size(); i++)
{
const ::testing::TestInfo* test_info = skipped_tests[i];
if (!test_info) continue;
std::cout << "- " << test_info->test_case_name() << "." << test_info->name() << std::endl;
}
}*/
}
};
static bool isTestTagForced(const std::string& testTag)
{
const std::vector<std::string>& forceTags = getTestTagsForceList();
for (size_t i = 0; i < forceTags.size(); ++i)
{
const std::string& forceTag = forceTags[i];
if (testTag == forceTag
|| (testTag.size() >= forceTag.size()
&& forceTag[forceTag.size() - 1] == '*'
&& forceTag.substr(0, forceTag.size() - 1) == testTag.substr(0, forceTag.size() - 1)
)
)
{
return true;
}
}
return false;
}
static bool isTestTagSkipped(const std::string& testTag, CV_OUT std::string& skippedByTag)
{
skippedByTag.clear();
const std::vector<std::string>& skipTags = getTestTagsSkipList();
for (size_t i = 0; i < skipTags.size(); ++i)
{
const std::string& skipTag = skipTags[i];
if (testTag == skipTag
|| (testTag.size() >= skipTag.size()
&& skipTag[skipTag.size() - 1] == '*'
&& skipTag.substr(0, skipTag.size() - 1) == testTag.substr(0, skipTag.size() - 1)
)
)
{
skippedByTag = skipTag;
return true;
}
}
return false;
}
void checkTestTags()
{
std::string skipTag;
const std::vector<std::string>& testTags = currentDirectTestTags;
{
const std::vector<std::string>& tags = getTestTagsRequiredList();
if (!tags.empty())
{
size_t found = 0;
for (size_t i = 0; i < tags.size(); ++i)
{
const std::string& tag = tags[i];
for (size_t j = 0; j < testTags.size(); ++j)
{
const std::string& testTag = testTags[i];
if (testTag == tag
|| (testTag.size() >= tag.size()
&& tag[tag.size() - 1] == '*'
&& tag.substr(0, tag.size() - 1) == testTag.substr(0, tag.size() - 1)
)
)
{
found++;
break;
}
}
}
if (found != tags.size())
{
skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
throw SkipTestException("Test tags don't pass required tags list (--test_tag parameter)");
}
}
}
for (size_t i = 0; i < testTags.size(); ++i)
{
const std::string& testTag = testTags[i];
if (isTestTagForced(testTag))
return;
}
for (size_t i = 0; i < testTags.size(); ++i)
{
const std::string& testTag = testTags[i];
if (isTestTagSkipped(testTag, skipTag))
{
skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
throw SkipTestException("Test with tag '" + testTag + "' is skipped ('" + skipTag + "' is in skip list)");
}
}
const std::vector<std::string>& testTagsImplied = currentImpliedTestTags;
for (size_t i = 0; i < testTagsImplied.size(); ++i)
{
const std::string& testTag = testTagsImplied[i];
if (isTestTagSkipped(testTag, skipTag))
{
skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
throw SkipTestException("Test with tag '" + testTag + "' is skipped ('" + skipTag + "' is in skip list)");
}
}
}
static bool applyTestTagImpl(const std::string& tag, bool direct = false)
{
CV_Assert(!tag.empty());
std::vector<std::string>& testTags = direct ? currentDirectTestTags : currentImpliedTestTags;
for (size_t i = 0; i < testTags.size(); ++i)
{
const std::string& testTag = testTags[i];
if (tag == testTag)
{
return false; // already exists, skip
}
}
testTags.push_back(tag);
// Tags implies logic
if (tag == CV_TEST_TAG_MEMORY_14GB)
applyTestTagImpl(CV_TEST_TAG_MEMORY_6GB);
if (tag == CV_TEST_TAG_MEMORY_6GB)
applyTestTagImpl(CV_TEST_TAG_MEMORY_2GB);
if (tag == CV_TEST_TAG_MEMORY_2GB)
applyTestTagImpl(CV_TEST_TAG_MEMORY_1GB);
if (tag == CV_TEST_TAG_MEMORY_1GB)
applyTestTagImpl(CV_TEST_TAG_MEMORY_512MB);
if (tag == CV_TEST_TAG_VERYLONG)
{
applyTestTagImpl(CV_TEST_TAG_DEBUG_VERYLONG);
applyTestTagImpl(CV_TEST_TAG_LONG);
}
else if (tag == CV_TEST_TAG_DEBUG_VERYLONG)
{
applyTestTagImpl(CV_TEST_TAG_DEBUG_LONG);
}
else if (tag == CV_TEST_TAG_LONG)
{
applyTestTagImpl(CV_TEST_TAG_DEBUG_LONG);
}
if (tag == CV_TEST_TAG_SIZE_4K)
applyTestTagImpl(CV_TEST_TAG_SIZE_FULLHD);
if (tag == CV_TEST_TAG_SIZE_FULLHD)
applyTestTagImpl(CV_TEST_TAG_SIZE_HD);
return true;
}
void applyTestTag(const std::string& tag)
{
if (tag.empty()) return;
if (!applyTestTagImpl(tag, true))
return;
checkTestTags();
}
void applyTestTag_(const std::string& tag)
{
if (tag.empty()) return;
if (!applyTestTagImpl(tag, true))
return;
}
static std::vector<std::string> parseStringList(const std::string& s)
{
std::vector<std::string> result;
size_t start_pos = 0;
while (start_pos != std::string::npos)
{
while (start_pos < s.size() && s[start_pos] == ' ')
start_pos++;
const size_t pos_ = s.find(',', start_pos);
size_t pos = (pos_ == std::string::npos ? s.size() : pos_);
while (pos > start_pos && s[pos - 1] == ' ')
pos--;
if (pos > start_pos)
{
const std::string one_piece(s, start_pos, pos - start_pos);
result.push_back(one_piece);
}
start_pos = (pos_ == std::string::npos ? pos_ : pos_ + 1);
}
return result;
}
void activateTestTags(const cv::CommandLineParser& parser)
{
std::string test_tag_skip = parser.get<std::string>("test_tag_skip");
if (!test_tag_skip.empty())
{
const std::vector<std::string> tag_list = parseStringList(test_tag_skip);
if (!tag_list.empty())
{
std::vector<std::string>& skipTags = getTestTagsSkipList();
for (size_t k = 0; k < tag_list.size(); ++k)
{
const std::string& tag = tag_list[k];
bool found = false;
for (size_t i = 0; i < skipTags.size(); ++i)
{
if (tag == skipTags[i])
{
found = true;
break;
}
}
if (!found)
skipTags.push_back(tag);
}
}
}
std::string test_tag_enable = parser.get<std::string>("test_tag_enable");
if (!test_tag_enable.empty())
{
const std::vector<std::string> tag_list = parseStringList(test_tag_enable);
if (!tag_list.empty())
{
std::vector<std::string>& skipTags = getTestTagsSkipList();
for (size_t k = 0; k < tag_list.size(); ++k)
{
const std::string& tag = tag_list[k];
bool found = false;
for (size_t i = 0; i < skipTags.size(); ++i)
{
if (tag == skipTags[i])
{
skipTags.erase(skipTags.begin() + i);
found = true;
}
}
if (!found)
{
std::cerr << "Can't re-enable tag '" << tag << "' - it is not in the skip list" << std::endl;
}
}
}
}
std::string test_tag_force = parser.get<std::string>("test_tag_force");
if (!test_tag_force.empty())
{
const std::vector<std::string> tag_list = parseStringList(test_tag_force);
if (!tag_list.empty())
{
std::vector<std::string>& forceTags = getTestTagsForceList();
for (size_t k = 0; k < tag_list.size(); ++k)
{
const std::string& tag = tag_list[k];
bool found = false;
for (size_t i = 0; i < forceTags.size(); ++i)
{
if (tag == forceTags[i])
{
found = true;
break;
}
}
if (!found)
forceTags.push_back(tag);
}
}
}
std::string test_tag = parser.get<std::string>("test_tag");
if (!test_tag.empty())
{
const std::vector<std::string> tag_list = parseStringList(test_tag);
if (!tag_list.empty())
{
std::vector<std::string>& requiredTags = getTestTagsRequiredList();
for (size_t k = 0; k < tag_list.size(); ++k)
{
const std::string& tag = tag_list[k];
bool found = false;
for (size_t i = 0; i < requiredTags.size(); ++i)
{
if (tag == requiredTags[i])
{
found = true;
break;
}
}
if (!found)
requiredTags.push_back(tag);
}
}
}
printTestTag = parser.get<bool>("test_tag_print");
::testing::UnitTest::GetInstance()->listeners().Append(new TestTagsListener());
}
} // namespace
modules/ts/src/ts_tags.hpp 0 → 100644
View file @ 64629cb9
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef OPENCV_TS_SRC_TAGS_HPP
#define OPENCV_TS_SRC_TAGS_HPP
// [all | test_tag] - (test_tag_skip - test_tag_enable) + test_tag_force
#define CV_TEST_TAGS_PARAMS \
"{ test_tag | |run tests with specified 'tag' markers only (comma ',' separated list) }" \
"{ test_tag_skip | |skip tests with 'tag' markers (comma ',' separated list) }" \
"{ test_tag_enable | |don't skip tests with 'tag' markers (comma ',' separated list) }" \
"{ test_tag_force | |force running of tests with 'tag' markers (comma ',' separated list) }" \
"{ test_tag_print | false |print assigned tags for each test }" \
// TODO
// "{ test_tag_file | |read test tags assignment }" \
namespace cvtest {
void activateTestTags(const cv::CommandLineParser& parser);
} // namespace
#endif // OPENCV_TS_SRC_TAGS_HPP
platforms/scripts/valgrind.supp
View file @ 64629cb9
......@@ -19,6 +19,13 @@
fun:_ZN2cv20allocSingletonBufferEm
}
{
OpenCV-SingletonNewBuffer
Memcheck:Leak
...
fun:_ZN2cv23allocSingletonNewBufferEm
}
{
OpenCV-getStdAllocator
Memcheck:Leak
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment