Skip to content

O
opencv
Commit 3bc9912f authored by Alexander Alekhin's avatar Alexander Alekhin
Browse files
Options

Merge pull request #13893 from alalek:core_dispatch_split

parents 4c94804b 91d152e2
Hide whitespace changes
Inline Side-by-side
Showing with 245 additions and 193 deletions
modules/core/CMakeLists.txt
View file @ 3bc9912f
......@@ -8,6 +8,7 @@ ocv_add_dispatched_file(convert_scale SSE2 AVX2)
ocv_add_dispatched_file(count_non_zero SSE2 AVX2)
ocv_add_dispatched_file(matmul SSE2 AVX2)
ocv_add_dispatched_file(mean SSE2 AVX2)
ocv_add_dispatched_file(split SSE2 AVX2)
ocv_add_dispatched_file(sum SSE2 AVX2)
# dispatching for accuracy tests
......
modules/core/src/split.cpp → modules/core/src/split.dispatch.cpp
View file @ 3bc9912f
......@@ -6,213 +6,44 @@
#include "precomp.hpp"
#include "opencl_kernels_core.hpp"
namespace cv { namespace hal {
#if CV_SIMD
// see the comments for vecmerge_ in merge.cpp
template<typename T, typename VecT> static void
vecsplit_( const T* src, T** dst, int len, int cn )
{
const int VECSZ = VecT::nlanes;
int i, i0 = 0;
T* dst0 = dst[0];
T* dst1 = dst[1];
int r0 = (int)((size_t)(void*)dst0 % (VECSZ*sizeof(T)));
int r1 = (int)((size_t)(void*)dst1 % (VECSZ*sizeof(T)));
int r2 = cn > 2 ? (int)((size_t)(void*)dst[2] % (VECSZ*sizeof(T))) : r0;
int r3 = cn > 3 ? (int)((size_t)(void*)dst[3] % (VECSZ*sizeof(T))) : r0;
hal::StoreMode mode = hal::STORE_ALIGNED_NOCACHE;
if( (r0|r1|r2|r3) != 0 )
{
mode = hal::STORE_UNALIGNED;
if( r0 == r1 && r0 == r2 && r0 == r3 && r0 % sizeof(T) == 0 && len > VECSZ*2 )
i0 = VECSZ - (r0 / sizeof(T));
}
if( cn == 2 )
{
for( i = 0; i < len; i += VECSZ )
{
if( i > len - VECSZ )
{
i = len - VECSZ;
mode = hal::STORE_UNALIGNED;
}
VecT a, b;
v_load_deinterleave(src + i*cn, a, b);
v_store(dst0 + i, a, mode);
v_store(dst1 + i, b, mode);
if( i < i0 )
{
i = i0 - VECSZ;
mode = hal::STORE_ALIGNED_NOCACHE;
}
}
}
else if( cn == 3 )
{
T* dst2 = dst[2];
for( i = 0; i < len; i += VECSZ )
{
if( i > len - VECSZ )
{
i = len - VECSZ;
mode = hal::STORE_UNALIGNED;
}
VecT a, b, c;
v_load_deinterleave(src + i*cn, a, b, c);
v_store(dst0 + i, a, mode);
v_store(dst1 + i, b, mode);
v_store(dst2 + i, c, mode);
if( i < i0 )
{
i = i0 - VECSZ;
mode = hal::STORE_ALIGNED_NOCACHE;
}
}
}
else
{
CV_Assert( cn == 4 );
T* dst2 = dst[2];
T* dst3 = dst[3];
for( i = 0; i < len; i += VECSZ )
{
if( i > len - VECSZ )
{
i = len - VECSZ;
mode = hal::STORE_UNALIGNED;
}
VecT a, b, c, d;
v_load_deinterleave(src + i*cn, a, b, c, d);
v_store(dst0 + i, a, mode);
v_store(dst1 + i, b, mode);
v_store(dst2 + i, c, mode);
v_store(dst3 + i, d, mode);
if( i < i0 )
{
i = i0 - VECSZ;
mode = hal::STORE_ALIGNED_NOCACHE;
}
}
}
vx_cleanup();
}
#endif
template<typename T> static void
split_( const T* src, T** dst, int len, int cn )
{
int k = cn % 4 ? cn % 4 : 4;
int i, j;
if( k == 1 )
{
T* dst0 = dst[0];
if(cn == 1)
{
memcpy(dst0, src, len * sizeof(T));
}
else
{
for( i = 0, j = 0 ; i < len; i++, j += cn )
dst0[i] = src[j];
}
}
else if( k == 2 )
{
T *dst0 = dst[0], *dst1 = dst[1];
i = j = 0;
#include "split.simd.hpp"
#include "split.simd_declarations.hpp" // defines CV_CPU_DISPATCH_MODES_ALL=AVX2,...,BASELINE based on CMakeLists.txt content
for( ; i < len; i++, j += cn )
{
dst0[i] = src[j];
dst1[i] = src[j+1];
}
}
else if( k == 3 )
{
T *dst0 = dst[0], *dst1 = dst[1], *dst2 = dst[2];
i = j = 0;
for( ; i < len; i++, j += cn )
{
dst0[i] = src[j];
dst1[i] = src[j+1];
dst2[i] = src[j+2];
}
}
else
{
T *dst0 = dst[0], *dst1 = dst[1], *dst2 = dst[2], *dst3 = dst[3];
i = j = 0;
for( ; i < len; i++, j += cn )
{
dst0[i] = src[j]; dst1[i] = src[j+1];
dst2[i] = src[j+2]; dst3[i] = src[j+3];
}
}
for( ; k < cn; k += 4 )
{
T *dst0 = dst[k], *dst1 = dst[k+1], *dst2 = dst[k+2], *dst3 = dst[k+3];
for( i = 0, j = k; i < len; i++, j += cn )
{
dst0[i] = src[j]; dst1[i] = src[j+1];
dst2[i] = src[j+2]; dst3[i] = src[j+3];
}
}
}
namespace cv { namespace hal {
void split8u(const uchar* src, uchar** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
CALL_HAL(split8u, cv_hal_split8u, src,dst, len, cn)
#if CV_SIMD
if( len >= v_uint8::nlanes && 2 <= cn && cn <= 4 )
vecsplit_<uchar, v_uint8>(src, dst, len, cn);
else
#endif
split_(src, dst, len, cn);
CV_CPU_DISPATCH(split8u, (src, dst, len, cn),
CV_CPU_DISPATCH_MODES_ALL);
}
void split16u(const ushort* src, ushort** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
CALL_HAL(split16u, cv_hal_split16u, src,dst, len, cn)
#if CV_SIMD
if( len >= v_uint16::nlanes && 2 <= cn && cn <= 4 )
vecsplit_<ushort, v_uint16>(src, dst, len, cn);
else
#endif
split_(src, dst, len, cn);
CV_CPU_DISPATCH(split16u, (src, dst, len, cn),
CV_CPU_DISPATCH_MODES_ALL);
}
void split32s(const int* src, int** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
CALL_HAL(split32s, cv_hal_split32s, src,dst, len, cn)
#if CV_SIMD
if( len >= v_uint32::nlanes && 2 <= cn && cn <= 4 )
vecsplit_<int, v_int32>(src, dst, len, cn);
else
#endif
split_(src, dst, len, cn);
CV_CPU_DISPATCH(split32s, (src, dst, len, cn),
CV_CPU_DISPATCH_MODES_ALL);
}
void split64s(const int64* src, int64** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
CALL_HAL(split64s, cv_hal_split64s, src,dst, len, cn)
#if CV_SIMD
if( len >= v_int64::nlanes && 2 <= cn && cn <= 4 )
vecsplit_<int64, v_int64>(src, dst, len, cn);
else
#endif
split_(src, dst, len, cn);
CV_CPU_DISPATCH(split64s, (src, dst, len, cn),
CV_CPU_DISPATCH_MODES_ALL);
}
}} // cv::hal::
} // namespace cv::hal::
/****************************************************************************************\
* split & merge *
......@@ -233,7 +64,6 @@ static SplitFunc getSplitFunc(int depth)
#ifdef HAVE_IPP
namespace cv {
static bool ipp_split(const Mat& src, Mat* mv, int channels)
{
#ifdef HAVE_IPP_IW_LL
......@@ -282,10 +112,9 @@ static bool ipp_split(const Mat& src, Mat* mv, int channels)
return false;
#endif
}
}
#endif
void cv::split(const Mat& src, Mat* mv)
void split(const Mat& src, Mat* mv)
{
CV_INSTRUMENT_REGION();
......@@ -341,8 +170,6 @@ void cv::split(const Mat& src, Mat* mv)
#ifdef HAVE_OPENCL
namespace cv {
static bool ocl_split( InputArray _m, OutputArrayOfArrays _mv )
{
int type = _m.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type),
......@@ -381,11 +208,9 @@ static bool ocl_split( InputArray _m, OutputArrayOfArrays _mv )
return k.run(2, globalsize, NULL, false);
}
}
#endif
void cv::split(InputArray _m, OutputArrayOfArrays _mv)
void split(InputArray _m, OutputArrayOfArrays _mv)
{
CV_INSTRUMENT_REGION();
......@@ -411,3 +236,5 @@ void cv::split(InputArray _m, OutputArrayOfArrays _mv)
split(m, &dst[0]);
}
} // namespace
modules/core/src/split.simd.hpp 0 → 100644
View file @ 3bc9912f
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html
#include "precomp.hpp"
namespace cv { namespace hal {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
void split8u(const uchar* src, uchar** dst, int len, int cn);
void split16u(const ushort* src, ushort** dst, int len, int cn);
void split32s(const int* src, int** dst, int len, int cn);
void split64s(const int64* src, int64** dst, int len, int cn);
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
#if CV_SIMD
// see the comments for vecmerge_ in merge.cpp
template<typename T, typename VecT> static void
vecsplit_( const T* src, T** dst, int len, int cn )
{
const int VECSZ = VecT::nlanes;
int i, i0 = 0;
T* dst0 = dst[0];
T* dst1 = dst[1];
int r0 = (int)((size_t)(void*)dst0 % (VECSZ*sizeof(T)));
int r1 = (int)((size_t)(void*)dst1 % (VECSZ*sizeof(T)));
int r2 = cn > 2 ? (int)((size_t)(void*)dst[2] % (VECSZ*sizeof(T))) : r0;
int r3 = cn > 3 ? (int)((size_t)(void*)dst[3] % (VECSZ*sizeof(T))) : r0;
hal::StoreMode mode = hal::STORE_ALIGNED_NOCACHE;
if( (r0|r1|r2|r3) != 0 )
{
mode = hal::STORE_UNALIGNED;
if( r0 == r1 && r0 == r2 && r0 == r3 && r0 % sizeof(T) == 0 && len > VECSZ*2 )
i0 = VECSZ - (r0 / sizeof(T));
}
if( cn == 2 )
{
for( i = 0; i < len; i += VECSZ )
{
if( i > len - VECSZ )
{
i = len - VECSZ;
mode = hal::STORE_UNALIGNED;
}
VecT a, b;
v_load_deinterleave(src + i*cn, a, b);
v_store(dst0 + i, a, mode);
v_store(dst1 + i, b, mode);
if( i < i0 )
{
i = i0 - VECSZ;
mode = hal::STORE_ALIGNED_NOCACHE;
}
}
}
else if( cn == 3 )
{
T* dst2 = dst[2];
for( i = 0; i < len; i += VECSZ )
{
if( i > len - VECSZ )
{
i = len - VECSZ;
mode = hal::STORE_UNALIGNED;
}
VecT a, b, c;
v_load_deinterleave(src + i*cn, a, b, c);
v_store(dst0 + i, a, mode);
v_store(dst1 + i, b, mode);
v_store(dst2 + i, c, mode);
if( i < i0 )
{
i = i0 - VECSZ;
mode = hal::STORE_ALIGNED_NOCACHE;
}
}
}
else
{
CV_Assert( cn == 4 );
T* dst2 = dst[2];
T* dst3 = dst[3];
for( i = 0; i < len; i += VECSZ )
{
if( i > len - VECSZ )
{
i = len - VECSZ;
mode = hal::STORE_UNALIGNED;
}
VecT a, b, c, d;
v_load_deinterleave(src + i*cn, a, b, c, d);
v_store(dst0 + i, a, mode);
v_store(dst1 + i, b, mode);
v_store(dst2 + i, c, mode);
v_store(dst3 + i, d, mode);
if( i < i0 )
{
i = i0 - VECSZ;
mode = hal::STORE_ALIGNED_NOCACHE;
}
}
}
vx_cleanup();
}
#endif
template<typename T> static void
split_( const T* src, T** dst, int len, int cn )
{
int k = cn % 4 ? cn % 4 : 4;
int i, j;
if( k == 1 )
{
T* dst0 = dst[0];
if(cn == 1)
{
memcpy(dst0, src, len * sizeof(T));
}
else
{
for( i = 0, j = 0 ; i < len; i++, j += cn )
dst0[i] = src[j];
}
}
else if( k == 2 )
{
T *dst0 = dst[0], *dst1 = dst[1];
i = j = 0;
for( ; i < len; i++, j += cn )
{
dst0[i] = src[j];
dst1[i] = src[j+1];
}
}
else if( k == 3 )
{
T *dst0 = dst[0], *dst1 = dst[1], *dst2 = dst[2];
i = j = 0;
for( ; i < len; i++, j += cn )
{
dst0[i] = src[j];
dst1[i] = src[j+1];
dst2[i] = src[j+2];
}
}
else
{
T *dst0 = dst[0], *dst1 = dst[1], *dst2 = dst[2], *dst3 = dst[3];
i = j = 0;
for( ; i < len; i++, j += cn )
{
dst0[i] = src[j]; dst1[i] = src[j+1];
dst2[i] = src[j+2]; dst3[i] = src[j+3];
}
}
for( ; k < cn; k += 4 )
{
T *dst0 = dst[k], *dst1 = dst[k+1], *dst2 = dst[k+2], *dst3 = dst[k+3];
for( i = 0, j = k; i < len; i++, j += cn )
{
dst0[i] = src[j]; dst1[i] = src[j+1];
dst2[i] = src[j+2]; dst3[i] = src[j+3];
}
}
}
void split8u(const uchar* src, uchar** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_uint8::nlanes && 2 <= cn && cn <= 4 )
vecsplit_<uchar, v_uint8>(src, dst, len, cn);
else
#endif
split_(src, dst, len, cn);
}
void split16u(const ushort* src, ushort** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_uint16::nlanes && 2 <= cn && cn <= 4 )
vecsplit_<ushort, v_uint16>(src, dst, len, cn);
else
#endif
split_(src, dst, len, cn);
}
void split32s(const int* src, int** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_uint32::nlanes && 2 <= cn && cn <= 4 )
vecsplit_<int, v_int32>(src, dst, len, cn);
else
#endif
split_(src, dst, len, cn);
}
void split64s(const int64* src, int64** dst, int len, int cn )
{
CV_INSTRUMENT_REGION();
#if CV_SIMD
if( len >= v_int64::nlanes && 2 <= cn && cn <= 4 )
vecsplit_<int64, v_int64>(src, dst, len, cn);
else
#endif
split_(src, dst, len, cn);
}
#endif
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // namespace
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment