Chiebot-Mirror
/
opencv
mirror of https://github.com/opencv/opencv.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

countNonZero function reworked to use wide universal intrinsics instead of SSE2 intrinsics

Browse Source
pull/12341/head
Vitaly Tuzov 6 years ago
parent 087e9308ec
commit c894fc5bae
1 changed files with 101 additions and 165 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 266
      modules/core/src/count_non_zero.cpp

266
modules/core/src/count_non_zero.cpp
Unescape View File

@ -25,51 +25,34 @@ static int countNonZero_(const T* src, int len )
static int countNonZero8u( const uchar* src, int len ) static int countNonZero8u( const uchar* src, int len )
{ {
int i=0, nz = 0; int i=0, nz = 0;
#if CV_SSE2 #if CV_SIMD
if(USE_SSE2)//5x-6x int len0 = len & -v_uint8::nlanes;
{ v_uint8 v_zero = vx_setzero_u8();
__m128i v_zero = _mm_setzero_si128(); v_uint8 v_one = vx_setall_u8(1);
__m128i sum = _mm_setzero_si128();
for (; i<=len-16; i+=16)
{
__m128i r0 = _mm_loadu_si128((const __m128i*)(src+i));
sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_cmpeq_epi8(r0, v_zero)), v_zero));
}
nz = i - _mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum)));
}
#elif CV_NEON
int len0 = len & -16, blockSize1 = (1 << 8) - 16, blockSize0 = blockSize1 << 6;
uint32x4_t v_nz = vdupq_n_u32(0u);
uint8x16_t v_zero = vdupq_n_u8(0), v_1 = vdupq_n_u8(1);
const uchar * src0 = src;
while( i < len0 ) v_uint32 v_sum32 = vx_setzero_u32();
while (i < len0)
{ {
int blockSizei = std::min(len0 - i, blockSize0), j = 0; v_uint16 v_sum16 = vx_setzero_u16();
int j = i;
while (j < blockSizei) while (j < std::min(len0, i + 65280 * v_uint16::nlanes))
{ {
int blockSizej = std::min(blockSizei - j, blockSize1), k = 0; v_uint8 v_sum8 = vx_setzero_u8();
uint8x16_t v_pz = v_zero; int k = j;
for (; k < std::min(len0, j + 255 * v_uint8::nlanes); k += v_uint8::nlanes)
for( ; k <= blockSizej - 16; k += 16 ) v_sum8 += v_one & (vx_load(src + k) == v_zero);
v_pz = vaddq_u8(v_pz, vandq_u8(vceqq_u8(vld1q_u8(src0 + k), v_zero), v_1)); v_uint16 part1, part2;
v_expand(v_sum8, part1, part2);
uint16x8_t v_p1 = vmovl_u8(vget_low_u8(v_pz)), v_p2 = vmovl_u8(vget_high_u8(v_pz)); v_sum16 += part1 + part2;
v_nz = vaddq_u32(vaddl_u16(vget_low_u16(v_p1), vget_high_u16(v_p1)), v_nz); j = k;
v_nz = vaddq_u32(vaddl_u16(vget_low_u16(v_p2), vget_high_u16(v_p2)), v_nz);
src0 += blockSizej;
j += blockSizej;
} }
v_uint32 part1, part2;
i += blockSizei; v_expand(v_sum16, part1, part2);
v_sum32 += part1 + part2;
i = j;
} }
nz = i - v_reduce_sum(v_sum32);
CV_DECL_ALIGNED(16) unsigned int buf[4]; v_cleanup();
vst1q_u32(buf, v_nz);
nz += i - saturate_cast<int>(buf[0] + buf[1] + buf[2] + buf[3]);
#endif #endif
for( ; i < len; i++ ) for( ; i < len; i++ )
nz += src[i] != 0; nz += src[i] != 0;
@ -79,159 +62,112 @@ static int countNonZero8u( const uchar* src, int len )
static int countNonZero16u( const ushort* src, int len ) static int countNonZero16u( const ushort* src, int len )
{ {
int i = 0, nz = 0; int i = 0, nz = 0;
#if CV_SSE2 #if CV_SIMD
if (USE_SSE2) int len0 = len & -v_int8::nlanes;
{ v_uint16 v_zero = vx_setzero_u16();
__m128i v_zero = _mm_setzero_si128 (); v_int8 v_one = vx_setall_s8(1);
__m128i sum = _mm_setzero_si128();
for ( ; i <= len - 8; i += 8) v_int32 v_sum32 = vx_setzero_s32();
{ while (i < len0)
__m128i r0 = _mm_loadu_si128((const __m128i*)(src + i));
sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_cmpeq_epi16(r0, v_zero)), v_zero));
}
nz = i - (_mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum))) >> 1);
src += i;
}
#elif CV_NEON
int len0 = len & -8, blockSize1 = (1 << 15), blockSize0 = blockSize1 << 6;
uint32x4_t v_nz = vdupq_n_u32(0u);
uint16x8_t v_zero = vdupq_n_u16(0), v_1 = vdupq_n_u16(1);
while( i < len0 )
{ {
int blockSizei = std::min(len0 - i, blockSize0), j = 0; v_int16 v_sum16 = vx_setzero_s16();
int j = i;
while (j < blockSizei) while (j < std::min(len0, i + 32766 * v_int16::nlanes))
{ {
int blockSizej = std::min(blockSizei - j, blockSize1), k = 0; v_int8 v_sum8 = vx_setzero_s8();
uint16x8_t v_pz = v_zero; int k = j;
for (; k < std::min(len0, j + 127 * v_int8::nlanes); k += v_int8::nlanes)
for( ; k <= blockSizej - 8; k += 8 ) v_sum8 += v_one & v_pack(v_reinterpret_as_s16(vx_load(src + k) == v_zero), v_reinterpret_as_s16(vx_load(src + k + v_uint16::nlanes) == v_zero));
v_pz = vaddq_u16(v_pz, vandq_u16(vceqq_u16(vld1q_u16(src + k), v_zero), v_1)); v_int16 part1, part2;
v_expand(v_sum8, part1, part2);
v_nz = vaddq_u32(vaddl_u16(vget_low_u16(v_pz), vget_high_u16(v_pz)), v_nz); v_sum16 += part1 + part2;
j = k;
src += blockSizej;
j += blockSizej;
} }
v_int32 part1, part2;
i += blockSizei; v_expand(v_sum16, part1, part2);
v_sum32 += part1 + part2;
i = j;
} }
nz = i - v_reduce_sum(v_sum32);
CV_DECL_ALIGNED(16) unsigned int buf[4]; v_cleanup();
vst1q_u32(buf, v_nz);
nz += i - saturate_cast<int>(buf[0] + buf[1] + buf[2] + buf[3]);
#endif #endif
return nz + countNonZero_(src, len - i); return nz + countNonZero_(src + i, len - i);
} }
static int countNonZero32s( const int* src, int len ) static int countNonZero32s( const int* src, int len )
{ {
int i = 0, nz = 0; int i = 0, nz = 0;
#if CV_SSE2 #if CV_SIMD
if (USE_SSE2) int len0 = len & -v_int8::nlanes;
{ v_int32 v_zero = vx_setzero_s32();
__m128i v_zero = _mm_setzero_si128 (); v_int8 v_one = vx_setall_s8(1);
__m128i sum = _mm_setzero_si128();
for ( ; i <= len - 4; i += 4) v_int32 v_sum32 = vx_setzero_s32();
{ while (i < len0)
__m128i r0 = _mm_loadu_si128((const __m128i*)(src + i));
sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_cmpeq_epi32(r0, v_zero)), v_zero));
}
nz = i - (_mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum))) >> 2);
src += i;
}
#elif CV_NEON
int len0 = len & -8, blockSize1 = (1 << 15), blockSize0 = blockSize1 << 6;
uint32x4_t v_nz = vdupq_n_u32(0u);
int32x4_t v_zero = vdupq_n_s32(0.0f);
uint16x8_t v_1 = vdupq_n_u16(1u), v_zerou = vdupq_n_u16(0u);
while( i < len0 )
{ {
int blockSizei = std::min(len0 - i, blockSize0), j = 0; v_int16 v_sum16 = vx_setzero_s16();
int j = i;
while (j < blockSizei) while (j < std::min(len0, i + 32766 * v_int16::nlanes))
{ {
int blockSizej = std::min(blockSizei - j, blockSize1), k = 0; v_int8 v_sum8 = vx_setzero_s8();
uint16x8_t v_pz = v_zerou; int k = j;
for (; k < std::min(len0, j + 127 * v_int8::nlanes); k += v_int8::nlanes)
for( ; k <= blockSizej - 8; k += 8 ) v_sum8 += v_one & v_pack(
v_pz = vaddq_u16(v_pz, vandq_u16(vcombine_u16(vmovn_u32(vceqq_s32(vld1q_s32(src + k), v_zero)), v_pack(vx_load(src + k ) == v_zero, vx_load(src + k + v_int32::nlanes) == v_zero),
vmovn_u32(vceqq_s32(vld1q_s32(src + k + 4), v_zero))), v_1)); v_pack(vx_load(src + k + 2*v_int32::nlanes) == v_zero, vx_load(src + k + 3*v_int32::nlanes) == v_zero)
);
v_nz = vaddq_u32(vaddl_u16(vget_low_u16(v_pz), vget_high_u16(v_pz)), v_nz); v_int16 part1, part2;
v_expand(v_sum8, part1, part2);
src += blockSizej; v_sum16 += part1 + part2;
j += blockSizej; j = k;
} }
v_int32 part1, part2;
i += blockSizei; v_expand(v_sum16, part1, part2);
v_sum32 += part1 + part2;
i = j;
} }
nz = i - v_reduce_sum(v_sum32);
CV_DECL_ALIGNED(16) unsigned int buf[4]; v_cleanup();
vst1q_u32(buf, v_nz);
nz += i - saturate_cast<int>(buf[0] + buf[1] + buf[2] + buf[3]);
#endif #endif
return nz + countNonZero_(src, len - i); return nz + countNonZero_(src + i, len - i);
} }
static int countNonZero32f( const float* src, int len ) static int countNonZero32f( const float* src, int len )
{ {
int i = 0, nz = 0; int i = 0, nz = 0;
#if CV_SSE2 #if CV_SIMD
if (USE_SSE2) int len0 = len & -v_int8::nlanes;
{ v_float32 v_zero = vx_setzero_f32();
__m128 v_zero_f = _mm_setzero_ps(); v_int8 v_one = vx_setall_s8(1);
__m128i v_zero = _mm_setzero_si128 ();
__m128i sum = _mm_setzero_si128();
for ( ; i <= len - 4; i += 4) v_int32 v_sum32 = vx_setzero_s32();
{ while (i < len0)
__m128 r0 = _mm_loadu_ps(src + i);
sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_castps_si128(_mm_cmpeq_ps(r0, v_zero_f))), v_zero));
}
nz = i - (_mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum))) >> 2);
src += i;
}
#elif CV_NEON
int len0 = len & -8, blockSize1 = (1 << 15), blockSize0 = blockSize1 << 6;
uint32x4_t v_nz = vdupq_n_u32(0u);
float32x4_t v_zero = vdupq_n_f32(0.0f);
uint16x8_t v_1 = vdupq_n_u16(1u), v_zerou = vdupq_n_u16(0u);
while( i < len0 )
{ {
int blockSizei = std::min(len0 - i, blockSize0), j = 0; v_int16 v_sum16 = vx_setzero_s16();
int j = i;
while (j < blockSizei) while (j < std::min(len0, i + 32766 * v_int16::nlanes))
{ {
int blockSizej = std::min(blockSizei - j, blockSize1), k = 0; v_int8 v_sum8 = vx_setzero_s8();
uint16x8_t v_pz = v_zerou; int k = j;
for (; k < std::min(len0, j + 127 * v_int8::nlanes); k += v_int8::nlanes)
for( ; k <= blockSizej - 8; k += 8 ) v_sum8 += v_one & v_pack(
v_pz = vaddq_u16(v_pz, vandq_u16(vcombine_u16(vmovn_u32(vceqq_f32(vld1q_f32(src + k), v_zero)), v_pack(v_reinterpret_as_s32(vx_load(src + k ) == v_zero), v_reinterpret_as_s32(vx_load(src + k + v_float32::nlanes) == v_zero)),
vmovn_u32(vceqq_f32(vld1q_f32(src + k + 4), v_zero))), v_1)); v_pack(v_reinterpret_as_s32(vx_load(src + k + 2*v_float32::nlanes) == v_zero), v_reinterpret_as_s32(vx_load(src + k + 3*v_float32::nlanes) == v_zero))
);
v_nz = vaddq_u32(vaddl_u16(vget_low_u16(v_pz), vget_high_u16(v_pz)), v_nz); v_int16 part1, part2;
v_expand(v_sum8, part1, part2);
src += blockSizej; v_sum16 += part1 + part2;
j += blockSizej; j = k;
} }
v_int32 part1, part2;
i += blockSizei; v_expand(v_sum16, part1, part2);
v_sum32 += part1 + part2;
i = j;
} }
nz = i - v_reduce_sum(v_sum32);
CV_DECL_ALIGNED(16) unsigned int buf[4]; v_cleanup();
vst1q_u32(buf, v_nz);
nz += i - saturate_cast<int>(buf[0] + buf[1] + buf[2] + buf[3]);
#endif #endif
return nz + countNonZero_(src, len - i); return nz + countNonZero_(src + i, len - i);
} }
static int countNonZero64f( const double* src, int len ) static int countNonZero64f( const double* src, int len )

Write Preview
Loading…
Cancel
Save