Skip to content

O
opencv
Commit 913a2dbd authored by Vadim Pisarevsky's avatar Vadim Pisarevsky
Browse files
Options

Merge pull request #8399 from woodychow:filter_avx2

parents 31aa69f5 67fe820c
Hide whitespace changes
Inline Side-by-side
Showing with 129 additions and 17 deletions
modules/imgproc/src/filter.cpp
View file @ 913a2dbd
...@@ -1354,12 +1354,14 @@ struct RowVec_32f ...@@ -1354,12 +1354,14 @@ struct RowVec_32f
RowVec_32f() RowVec_32f()
{ {
haveSSE = checkHardwareSupport(CV_CPU_SSE); haveSSE = checkHardwareSupport(CV_CPU_SSE);
haveAVX2 = checkHardwareSupport(CV_CPU_AVX2);
} }
RowVec_32f( const Mat& _kernel ) RowVec_32f( const Mat& _kernel )
{ {
kernel = _kernel; kernel = _kernel;
haveSSE = checkHardwareSupport(CV_CPU_SSE); haveSSE = checkHardwareSupport(CV_CPU_SSE);
haveAVX2 = checkHardwareSupport(CV_CPU_AVX2);
#if defined USE_IPP_SEP_FILTERS #if defined USE_IPP_SEP_FILTERS
bufsz = -1; bufsz = -1;
#endif #endif
...@@ -1386,14 +1388,36 @@ struct RowVec_32f ...@@ -1386,14 +1388,36 @@ struct RowVec_32f
int i = 0, k; int i = 0, k;
width *= cn; width *= cn;
#if CV_AVX2
if ( haveAVX2 )
{
for( ; i <= width - 8; i += 8 )
{
const float* src = src0 + i;
__m256 f, x0;
__m256 s0 = _mm256_set1_ps(0.0f);
for( k = 0; k < _ksize; k++, src += cn )
{
f = _mm256_set1_ps(_kx[k]);
x0 = _mm256_loadu_ps(src);
#if CV_FMA3
s0 = _mm256_fmadd_ps(x0, f, s0);
#else
s0 = _mm256_add_ps(s0, _mm256_mul_ps(x0, f));
#endif
}
_mm256_storeu_ps(dst + i, s0);
}
return i;
}
#endif
for( ; i <= width - 8; i += 8 ) for( ; i <= width - 8; i += 8 )
{ {
const float* src = src0 + i; const float* src = src0 + i;
__m128 f, s0 = _mm_setzero_ps(), s1 = s0, x0, x1; __m128 f, s0 = _mm_setzero_ps(), s1 = s0, x0, x1;
for( k = 0; k < _ksize; k++, src += cn ) for( k = 0; k < _ksize; k++, src += cn )
{ {
f = _mm_load_ss(_kx+k); f = _mm_set1_ps(_kx[k]);
f = _mm_shuffle_ps(f, f, 0);
x0 = _mm_loadu_ps(src); x0 = _mm_loadu_ps(src);
x1 = _mm_loadu_ps(src + 4); x1 = _mm_loadu_ps(src + 4);
...@@ -1408,6 +1432,7 @@ struct RowVec_32f ...@@ -1408,6 +1432,7 @@ struct RowVec_32f
Mat kernel; Mat kernel;
bool haveSSE; bool haveSSE;
bool haveAVX2;
#if defined USE_IPP_SEP_FILTERS #if defined USE_IPP_SEP_FILTERS
private: private:
mutable int bufsz; mutable int bufsz;
...@@ -1646,18 +1671,24 @@ struct SymmRowSmallVec_32f ...@@ -1646,18 +1671,24 @@ struct SymmRowSmallVec_32f
struct SymmColumnVec_32f struct SymmColumnVec_32f
{ {
SymmColumnVec_32f() { symmetryType=0; } SymmColumnVec_32f() {
symmetryType=0;
haveSSE = checkHardwareSupport(CV_CPU_SSE);
haveAVX2 = checkHardwareSupport(CV_CPU_AVX2);
}
SymmColumnVec_32f(const Mat& _kernel, int _symmetryType, int, double _delta) SymmColumnVec_32f(const Mat& _kernel, int _symmetryType, int, double _delta)
{ {
symmetryType = _symmetryType; symmetryType = _symmetryType;
kernel = _kernel; kernel = _kernel;
delta = (float)_delta; delta = (float)_delta;
haveSSE = checkHardwareSupport(CV_CPU_SSE);
haveAVX2 = checkHardwareSupport(CV_CPU_AVX2);
CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 ); CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 );
} }
int operator()(const uchar** _src, uchar* _dst, int width) const int operator()(const uchar** _src, uchar* _dst, int width) const
{ {
if( !checkHardwareSupport(CV_CPU_SSE) ) if( !haveSSE )
return 0; return 0;
int ksize2 = (kernel.rows + kernel.cols - 1)/2; int ksize2 = (kernel.rows + kernel.cols - 1)/2;
...@@ -1667,14 +1698,64 @@ struct SymmColumnVec_32f ...@@ -1667,14 +1698,64 @@ struct SymmColumnVec_32f
const float** src = (const float**)_src; const float** src = (const float**)_src;
const float *S, *S2; const float *S, *S2;
float* dst = (float*)_dst; float* dst = (float*)_dst;
__m128 d4 = _mm_set1_ps(delta); const __m128 d4 = _mm_set1_ps(delta);
#if CV_AVX2
const __m256 d8 = _mm256_set1_ps(delta);
#endif
if( symmetrical ) if( symmetrical )
{ {
#if CV_AVX2
if ( haveAVX2 )
{
for( ; i <= width - 16; i += 16 ) for( ; i <= width - 16; i += 16 )
{ {
__m128 f = _mm_load_ss(ky); __m256 f = _mm256_set1_ps(ky[0]);
f = _mm_shuffle_ps(f, f, 0); __m256 s0, s1;
__m256 x0;
S = src[0] + i;
s0 = _mm256_loadu_ps(S);
#if CV_FMA3
s0 = _mm256_fmadd_ps(s0, f, d8);
#else
s0 = _mm256_add_ps(_mm256_mul_ps(s0, f), d8);
#endif
s1 = _mm256_loadu_ps(S+8);
#if CV_FMA3
s1 = _mm256_fmadd_ps(s1, f, d8);
#else
s1 = _mm256_add_ps(_mm256_mul_ps(s1, f), d8);
#endif
for( k = 1; k <= ksize2; k++ )
{
S = src[k] + i;
S2 = src[-k] + i;
f = _mm256_set1_ps(ky[k]);
x0 = _mm256_add_ps(_mm256_loadu_ps(S), _mm256_loadu_ps(S2));
#if CV_FMA3
s0 = _mm256_fmadd_ps(x0, f, s0);
#else
s0 = _mm256_add_ps(s0, _mm256_mul_ps(x0, f));
#endif
x0 = _mm256_add_ps(_mm256_loadu_ps(S+8), _mm256_loadu_ps(S2+8));
#if CV_FMA3
s1 = _mm256_fmadd_ps(x0, f, s1);
#else
s1 = _mm256_add_ps(s1, _mm256_mul_ps(x0, f));
#endif
}
_mm256_storeu_ps(dst + i, s0);
_mm256_storeu_ps(dst + i + 8, s1);
}
_mm256_zeroupper();
}
#endif
for( ; i <= width - 16; i += 16 )
{
__m128 f = _mm_set1_ps(ky[0]);
__m128 s0, s1, s2, s3; __m128 s0, s1, s2, s3;
__m128 x0, x1; __m128 x0, x1;
S = src[0] + i; S = src[0] + i;
...@@ -1691,8 +1772,7 @@ struct SymmColumnVec_32f ...@@ -1691,8 +1772,7 @@ struct SymmColumnVec_32f
{ {
S = src[k] + i; S = src[k] + i;
S2 = src[-k] + i; S2 = src[-k] + i;
f = _mm_load_ss(ky+k); f = _mm_set1_ps(ky[k]);
f = _mm_shuffle_ps(f, f, 0);
x0 = _mm_add_ps(_mm_load_ps(S), _mm_load_ps(S2)); x0 = _mm_add_ps(_mm_load_ps(S), _mm_load_ps(S2));
x1 = _mm_add_ps(_mm_load_ps(S+4), _mm_load_ps(S2+4)); x1 = _mm_add_ps(_mm_load_ps(S+4), _mm_load_ps(S2+4));
s0 = _mm_add_ps(s0, _mm_mul_ps(x0, f)); s0 = _mm_add_ps(s0, _mm_mul_ps(x0, f));
...@@ -1711,15 +1791,13 @@ struct SymmColumnVec_32f ...@@ -1711,15 +1791,13 @@ struct SymmColumnVec_32f
for( ; i <= width - 4; i += 4 ) for( ; i <= width - 4; i += 4 )
{ {
__m128 f = _mm_load_ss(ky); __m128 f = _mm_set1_ps(ky[0]);
f = _mm_shuffle_ps(f, f, 0);
__m128 x0, s0 = _mm_load_ps(src[0] + i); __m128 x0, s0 = _mm_load_ps(src[0] + i);
s0 = _mm_add_ps(_mm_mul_ps(s0, f), d4); s0 = _mm_add_ps(_mm_mul_ps(s0, f), d4);
for( k = 1; k <= ksize2; k++ ) for( k = 1; k <= ksize2; k++ )
{ {
f = _mm_load_ss(ky+k); f = _mm_set1_ps(ky[k]);
f = _mm_shuffle_ps(f, f, 0);
S = src[k] + i; S = src[k] + i;
S2 = src[-k] + i; S2 = src[-k] + i;
x0 = _mm_add_ps(_mm_load_ps(src[k]+i), _mm_load_ps(src[-k] + i)); x0 = _mm_add_ps(_mm_load_ps(src[k]+i), _mm_load_ps(src[-k] + i));
...@@ -1731,6 +1809,40 @@ struct SymmColumnVec_32f ...@@ -1731,6 +1809,40 @@ struct SymmColumnVec_32f
} }
else else
{ {
#if CV_AVX2
if ( haveAVX2 )
{
for( ; i <= width - 16; i += 16 )
{
__m256 f, s0 = d8, s1 = d8;
__m256 x0;
S = src[0] + i;
for( k = 1; k <= ksize2; k++ )
{
S = src[k] + i;
S2 = src[-k] + i;
f = _mm256_set1_ps(ky[k]);
x0 = _mm256_sub_ps(_mm256_loadu_ps(S), _mm256_loadu_ps(S2));
#if CV_FMA3
s0 = _mm256_fmadd_ps(x0, f, s0);
#else
s0 = _mm256_add_ps(s0, _mm256_mul_ps(x0, f));
#endif
x0 = _mm256_sub_ps(_mm256_loadu_ps(S+8), _mm256_loadu_ps(S2+8));
#if CV_FMA3
s1 = _mm256_fmadd_ps(x0, f, s1);
#else
s1 = _mm256_add_ps(s1, _mm256_mul_ps(x0, f));
#endif
}
_mm256_storeu_ps(dst + i, s0);
_mm256_storeu_ps(dst + i + 8, s1);
}
_mm256_zeroupper();
}
#endif
for( ; i <= width - 16; i += 16 ) for( ; i <= width - 16; i += 16 )
{ {
__m128 f, s0 = d4, s1 = d4, s2 = d4, s3 = d4; __m128 f, s0 = d4, s1 = d4, s2 = d4, s3 = d4;
...@@ -1741,8 +1853,7 @@ struct SymmColumnVec_32f ...@@ -1741,8 +1853,7 @@ struct SymmColumnVec_32f
{ {
S = src[k] + i; S = src[k] + i;
S2 = src[-k] + i; S2 = src[-k] + i;
f = _mm_load_ss(ky+k); f = _mm_set1_ps(ky[k]);
f = _mm_shuffle_ps(f, f, 0);
x0 = _mm_sub_ps(_mm_load_ps(S), _mm_load_ps(S2)); x0 = _mm_sub_ps(_mm_load_ps(S), _mm_load_ps(S2));
x1 = _mm_sub_ps(_mm_load_ps(S+4), _mm_load_ps(S2+4)); x1 = _mm_sub_ps(_mm_load_ps(S+4), _mm_load_ps(S2+4));
s0 = _mm_add_ps(s0, _mm_mul_ps(x0, f)); s0 = _mm_add_ps(s0, _mm_mul_ps(x0, f));
...@@ -1765,8 +1876,7 @@ struct SymmColumnVec_32f ...@@ -1765,8 +1876,7 @@ struct SymmColumnVec_32f
for( k = 1; k <= ksize2; k++ ) for( k = 1; k <= ksize2; k++ )
{ {
f = _mm_load_ss(ky+k); f = _mm_set1_ps(ky[k]);
f = _mm_shuffle_ps(f, f, 0);
x0 = _mm_sub_ps(_mm_load_ps(src[k]+i), _mm_load_ps(src[-k] + i)); x0 = _mm_sub_ps(_mm_load_ps(src[k]+i), _mm_load_ps(src[-k] + i));
s0 = _mm_add_ps(s0, _mm_mul_ps(x0, f)); s0 = _mm_add_ps(s0, _mm_mul_ps(x0, f));
} }
...@@ -1781,6 +1891,8 @@ struct SymmColumnVec_32f ...@@ -1781,6 +1891,8 @@ struct SymmColumnVec_32f
int symmetryType; int symmetryType;
float delta; float delta;
Mat kernel; Mat kernel;
bool haveSSE;
bool haveAVX2;
}; };
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment