From 0f2b535fccd1143d542fd02424904bc9ab85a958 Mon Sep 17 00:00:00 2001 From: Vitaly Tuzov Date: Fri, 31 Aug 2018 17:05:00 +0300 Subject: [PATCH] Bit-exact GaussianBlur reworked to use wide intrinsics (#12073) * Bit-exact GaussianBlur reworked to use wide intrinsics * Added v_mul_hi universal intrinsic * Removed custom SSE2 branch from bit-exact GaussianBlur * Removed loop unrolling for gaussianBlur horizontal smoothing --- .../include/opencv2/core/hal/intrin_avx.hpp | 2 + .../include/opencv2/core/hal/intrin_cpp.hpp | 14 + .../include/opencv2/core/hal/intrin_neon.hpp | 15 + .../include/opencv2/core/hal/intrin_sse.hpp | 3 + .../include/opencv2/core/hal/intrin_vsx.hpp | 15 + modules/imgproc/src/smooth.cpp | 712 +++++++++--------- 6 files changed, 385 insertions(+), 376 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_avx.hpp b/modules/core/include/opencv2/core/hal/intrin_avx.hpp index 1c5ffbd1ca..36c7c0f1a1 100644 --- a/modules/core/include/opencv2/core/hal/intrin_avx.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_avx.hpp @@ -664,6 +664,8 @@ inline void v_mul_expand(const v_uint32x8& a, const v_uint32x8& b, v_zip(v_uint64x4(v0), v_uint64x4(v1), c, d); } +inline v_int16x16 v_mul_hi(const v_int16x16& a, const v_int16x16& b) { return v_int16x16(_mm256_mulhi_epi16(a.val, b.val)); } +inline v_uint16x16 v_mul_hi(const v_uint16x16& a, const v_uint16x16& b) { return v_uint16x16(_mm256_mulhi_epu16(a.val, b.val)); } /** Non-saturating arithmetics **/ #define OPENCV_HAL_IMPL_AVX_BIN_FUNC(func, _Tpvec, intrin) \ diff --git a/modules/core/include/opencv2/core/hal/intrin_cpp.hpp b/modules/core/include/opencv2/core/hal/intrin_cpp.hpp index 61d58dbb06..ccd317682d 100644 --- a/modules/core/include/opencv2/core/hal/intrin_cpp.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_cpp.hpp @@ -891,6 +891,20 @@ template inline void v_mul_expand(const v_reg<_Tp, n>& a, c } } +/** @brief Multiply and extract high part + +Multiply values two registers and store high part of the results. +Implemented only for 16-bit source types (v_int16x8, v_uint16x8). Returns \f$ a*b >> 16 \f$ +*/ +template inline v_reg<_Tp, n> v_mul_hi(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b) +{ + typedef typename V_TypeTraits<_Tp>::w_type w_type; + v_reg<_Tp, n> c; + for (int i = 0; i < n; i++) + c.s[i] = (_Tp)(((w_type)a.s[i] * b.s[i]) >> sizeof(_Tp)*8); + return c; +} + //! @cond IGNORED template inline void v_hsum(const v_reg<_Tp, n>& a, v_reg::w_type, n/2>& c) diff --git a/modules/core/include/opencv2/core/hal/intrin_neon.hpp b/modules/core/include/opencv2/core/hal/intrin_neon.hpp index 04b6ba2259..c017b075f1 100644 --- a/modules/core/include/opencv2/core/hal/intrin_neon.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_neon.hpp @@ -553,6 +553,21 @@ inline void v_mul_expand(const v_uint32x4& a, const v_uint32x4& b, d.val = vmull_u32(vget_high_u32(a.val), vget_high_u32(b.val)); } +inline v_int16x8 v_mul_hi(const v_int16x8& a, const v_int16x8& b) +{ + return v_int16x8(vcombine_s16( + vshrn_n_s32(vmull_s16( vget_low_s16(a.val), vget_low_s16(b.val)), 16), + vshrn_n_s32(vmull_s16(vget_high_s16(a.val), vget_high_s16(b.val)), 16) + )); +} +inline v_uint16x8 v_mul_hi(const v_uint16x8& a, const v_uint16x8& b) +{ + return v_uint16x8(vcombine_u16( + vshrn_n_u32(vmull_u16( vget_low_u16(a.val), vget_low_u16(b.val)), 16), + vshrn_n_u32(vmull_u16(vget_high_u16(a.val), vget_high_u16(b.val)), 16) + )); +} + inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) { int32x4_t c = vmull_s16(vget_low_s16(a.val), vget_low_s16(b.val)); diff --git a/modules/core/include/opencv2/core/hal/intrin_sse.hpp b/modules/core/include/opencv2/core/hal/intrin_sse.hpp index 42a39d07f9..159ef356b5 100644 --- a/modules/core/include/opencv2/core/hal/intrin_sse.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_sse.hpp @@ -737,6 +737,9 @@ inline void v_mul_expand(const v_uint32x4& a, const v_uint32x4& b, d.val = _mm_unpackhi_epi64(c0, c1); } +inline v_int16x8 v_mul_hi(const v_int16x8& a, const v_int16x8& b) { return v_int16x8(_mm_mulhi_epi16(a.val, b.val)); } +inline v_uint16x8 v_mul_hi(const v_uint16x8& a, const v_uint16x8& b) { return v_uint16x8(_mm_mulhi_epu16(a.val, b.val)); } + inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) { return v_int32x4(_mm_madd_epi16(a.val, b.val)); diff --git a/modules/core/include/opencv2/core/hal/intrin_vsx.hpp b/modules/core/include/opencv2/core/hal/intrin_vsx.hpp index 52bc2cc0ba..a45e7a875f 100644 --- a/modules/core/include/opencv2/core/hal/intrin_vsx.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_vsx.hpp @@ -457,6 +457,21 @@ inline void v_mul_expand(const v_uint32x4& a, const v_uint32x4& b, v_uint64x2& c d.val = vec_mul(vec_unpacklu(a.val), vec_unpacklu(b.val)); } +inline v_int16x8 v_mul_hi(const v_int16x8& a, const v_int16x8& b) +{ + return v_int16x8(vec_packs( + vec_sra(vec_mul(vec_unpackh(a.val), vec_unpackh(b.val)), vec_uint4_sp(16)), + vec_sra(vec_mul(vec_unpackl(a.val), vec_unpackl(b.val)), vec_uint4_sp(16)) + )); +} +inline v_uint16x8 v_mul_hi(const v_uint16x8& a, const v_uint16x8& b) +{ + return v_uint16x8(vec_packs( + vec_sr(vec_mul(vec_unpackhu(a.val), vec_unpackhu(b.val)), vec_uint4_sp(16)), + vec_sr(vec_mul(vec_unpacklu(a.val), vec_unpacklu(b.val)), vec_uint4_sp(16)) + )); +} + /** Non-saturating arithmetics **/ #define OPENCV_HAL_IMPL_VSX_BIN_FUNC(func, intrin) \ template \ diff --git a/modules/imgproc/src/smooth.cpp b/modules/imgproc/src/smooth.cpp index d54065f801..212698bce0 100644 --- a/modules/imgproc/src/smooth.cpp +++ b/modules/imgproc/src/smooth.cpp @@ -1820,22 +1820,13 @@ template <> void hlineSmooth1N(const uint8_t* src, int cn, const ufixedpoint16* m, int, ufixedpoint16* dst, int len, int) { int lencn = len*cn; - v_uint16x8 v_mul = v_setall_u16(*((uint16_t*)m)); int i = 0; - for (; i <= lencn - 16; i += 16) - { - v_uint8x16 v_src = v_load(src + i); - v_uint16x8 v_tmp0, v_tmp1; - v_expand(v_src, v_tmp0, v_tmp1); - v_store((uint16_t*)dst + i, v_mul*v_tmp0); - v_store((uint16_t*)dst + i + 8, v_mul*v_tmp1); - } - if (i <= lencn - 8) - { - v_uint16x8 v_src = v_load_expand(src + i); - v_store((uint16_t*)dst + i, v_mul*v_src); - i += 8; - } +#if CV_SIMD + const int VECSZ = v_uint16::nlanes; + v_uint16 v_mul = vx_setall_u16(*((uint16_t*)m)); + for (; i <= lencn - VECSZ; i += VECSZ) + v_store((uint16_t*)dst + i, v_mul*vx_load_expand(src + i)); +#endif for (; i < lencn; i++) dst[i] = m[0] * src[i]; } @@ -1850,20 +1841,11 @@ void hlineSmooth1N1(const uint8_t* src, int cn, const uf { int lencn = len*cn; int i = 0; - for (; i <= lencn - 16; i += 16) - { - v_uint8x16 v_src = v_load(src + i); - v_uint16x8 v_tmp0, v_tmp1; - v_expand(v_src, v_tmp0, v_tmp1); - v_store((uint16_t*)dst + i, v_shl<8>(v_tmp0)); - v_store((uint16_t*)dst + i + 8, v_shl<8>(v_tmp1)); - } - if (i <= lencn - 8) - { - v_uint16x8 v_src = v_load_expand(src + i); - v_store((uint16_t*)dst + i, v_shl<8>(v_src)); - i += 8; - } +#if CV_SIMD + const int VECSZ = v_uint16::nlanes; + for (; i <= lencn - VECSZ; i += VECSZ) + v_store((uint16_t*)dst + i, v_shl<8>(vx_load_expand(src + i))); +#endif for (; i < lencn; i++) dst[i] = src[i]; } @@ -1926,18 +1908,15 @@ void hlineSmooth3N(const uint8_t* src, int cn, const ufi src += cn; dst += cn; int i = cn, lencn = (len - 1)*cn; - v_uint16x8 v_mul0 = v_setall_u16(*((uint16_t*)m)); - v_uint16x8 v_mul1 = v_setall_u16(*((uint16_t*)(m + 1))); - v_uint16x8 v_mul2 = v_setall_u16(*((uint16_t*)(m + 2))); - for (; i <= lencn - 16; i += 16, src += 16, dst += 16) - { - v_uint16x8 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; - v_expand(v_load(src - cn), v_src00, v_src01); - v_expand(v_load(src), v_src10, v_src11); - v_expand(v_load(src + cn), v_src20, v_src21); - v_store((uint16_t*)dst, v_src00 * v_mul0 + v_src10 * v_mul1 + v_src20 * v_mul2); - v_store((uint16_t*)dst + 8, v_src01 * v_mul0 + v_src11 * v_mul1 + v_src21 * v_mul2); - } +#if CV_SIMD + const uint16_t* _m = (const uint16_t*)m; + const int VECSZ = v_uint16::nlanes; + v_uint16 v_mul0 = vx_setall_u16(_m[0]); + v_uint16 v_mul1 = vx_setall_u16(_m[1]); + v_uint16 v_mul2 = vx_setall_u16(_m[2]); + for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) + v_store((uint16_t*)dst, vx_load_expand(src - cn) * v_mul0 + vx_load_expand(src) * v_mul1 + vx_load_expand(src + cn) * v_mul2); +#endif for (; i < lencn; i++, src++, dst++) *dst = m[0] * src[-cn] + m[1] * src[0] + m[2] * src[cn]; @@ -2017,15 +1996,11 @@ void hlineSmooth3N121(const uint8_t* src, int cn, const src += cn; dst += cn; int i = cn, lencn = (len - 1)*cn; - for (; i <= lencn - 16; i += 16, src += 16, dst += 16) - { - v_uint16x8 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; - v_expand(v_load(src - cn), v_src00, v_src01); - v_expand(v_load(src), v_src10, v_src11); - v_expand(v_load(src + cn), v_src20, v_src21); - v_store((uint16_t*)dst, (v_src00 + v_src20 + (v_src10 << 1)) << 6); - v_store((uint16_t*)dst + 8, (v_src01 + v_src21 + (v_src11 << 1)) << 6); - } +#if CV_SIMD + const int VECSZ = v_uint16::nlanes; + for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) + v_store((uint16_t*)dst, (vx_load_expand(src - cn) + vx_load_expand(src + cn) + (vx_load_expand(src) << 1)) << 6); +#endif for (; i < lencn; i++, src++, dst++) *((uint16_t*)dst) = (uint16_t(src[-cn]) + uint16_t(src[cn]) + (uint16_t(src[0]) << 1)) << 6; @@ -2108,17 +2083,14 @@ void hlineSmooth3Naba(const uint8_t* src, int cn, const src += cn; dst += cn; int i = cn, lencn = (len - 1)*cn; - v_uint16x8 v_mul0 = v_setall_u16(*((uint16_t*)m)); - v_uint16x8 v_mul1 = v_setall_u16(*((uint16_t*)m+1)); - for (; i <= lencn - 16; i += 16, src += 16, dst += 16) - { - v_uint16x8 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; - v_expand(v_load(src - cn), v_src00, v_src01); - v_expand(v_load(src), v_src10, v_src11); - v_expand(v_load(src + cn), v_src20, v_src21); - v_store((uint16_t*)dst, (v_src00 + v_src20) * v_mul0 + v_src10 * v_mul1); - v_store((uint16_t*)dst + 8, (v_src01 + v_src21) * v_mul0 + v_src11 * v_mul1); - } +#if CV_SIMD + const uint16_t* _m = (const uint16_t*)m; + const int VECSZ = v_uint16::nlanes; + v_uint16 v_mul0 = vx_setall_u16(_m[0]); + v_uint16 v_mul1 = vx_setall_u16(_m[1]); + for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) + v_store((uint16_t*)dst, (vx_load_expand(src - cn) + vx_load_expand(src + cn)) * v_mul0 + vx_load_expand(src) * v_mul1); +#endif for (; i < lencn; i++, src++, dst++) *((uint16_t*)dst) = ((uint16_t*)m)[1] * src[0] + ((uint16_t*)m)[0] * ((uint16_t)(src[-cn]) + (uint16_t)(src[cn])); @@ -2304,22 +2276,17 @@ void hlineSmooth5N(const uint8_t* src, int cn, const ufi src += 2 * cn; dst += 2 * cn; int i = 2*cn, lencn = (len - 2)*cn; - v_uint16x8 v_mul0 = v_setall_u16(*((uint16_t*)m)); - v_uint16x8 v_mul1 = v_setall_u16(*((uint16_t*)(m + 1))); - v_uint16x8 v_mul2 = v_setall_u16(*((uint16_t*)(m + 2))); - v_uint16x8 v_mul3 = v_setall_u16(*((uint16_t*)(m + 3))); - v_uint16x8 v_mul4 = v_setall_u16(*((uint16_t*)(m + 4))); - for (; i <= lencn - 16; i += 16, src += 16, dst += 16) - { - v_uint16x8 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21, v_src30, v_src31, v_src40, v_src41; - v_expand(v_load(src - 2*cn), v_src00, v_src01); - v_expand(v_load(src - cn), v_src10, v_src11); - v_expand(v_load(src), v_src20, v_src21); - v_expand(v_load(src + cn), v_src30, v_src31); - v_expand(v_load(src + 2*cn), v_src40, v_src41); - v_store((uint16_t*)dst, v_src00 * v_mul0 + v_src10 * v_mul1 + v_src20 * v_mul2 + v_src30 * v_mul3 + v_src40 * v_mul4); - v_store((uint16_t*)dst + 8, v_src01 * v_mul0 + v_src11 * v_mul1 + v_src21 * v_mul2 + v_src31 * v_mul3 + v_src41 * v_mul4); - } +#if CV_SIMD + const uint16_t* _m = (const uint16_t*)m; + const int VECSZ = v_uint16::nlanes; + v_uint16 v_mul0 = vx_setall_u16(_m[0]); + v_uint16 v_mul1 = vx_setall_u16(_m[1]); + v_uint16 v_mul2 = vx_setall_u16(_m[2]); + v_uint16 v_mul3 = vx_setall_u16(_m[3]); + v_uint16 v_mul4 = vx_setall_u16(_m[4]); + for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) + v_store((uint16_t*)dst, vx_load_expand(src - 2 * cn) * v_mul0 + vx_load_expand(src - cn) * v_mul1 + vx_load_expand(src) * v_mul2 + vx_load_expand(src + cn) * v_mul3 + vx_load_expand(src + 2 * cn) * v_mul4); +#endif for (; i < lencn; i++, src++, dst++) *dst = m[0] * src[-2*cn] + m[1] * src[-cn] + m[2] * src[0] + m[3] * src[cn] + m[4] * src[2*cn]; @@ -2517,18 +2484,12 @@ void hlineSmooth5N14641(const uint8_t* src, int cn, cons src += 2 * cn; dst += 2 * cn; int i = 2 * cn, lencn = (len - 2)*cn; - v_uint16x8 v_6 = v_setall_u16(6); - for (; i <= lencn - 16; i += 16, src += 16, dst += 16) - { - v_uint16x8 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21, v_src30, v_src31, v_src40, v_src41; - v_expand(v_load(src - 2*cn), v_src00, v_src01); - v_expand(v_load(src - cn), v_src10, v_src11); - v_expand(v_load(src), v_src20, v_src21); - v_expand(v_load(src + cn), v_src30, v_src31); - v_expand(v_load(src + 2*cn), v_src40, v_src41); - v_store((uint16_t*)dst, (v_src20 * v_6 + ((v_src10 + v_src30) << 2) + v_src00 + v_src40) << 4); - v_store((uint16_t*)dst + 8, (v_src21 * v_6 + ((v_src11 + v_src31) << 2) + v_src01 + v_src41) << 4); - } +#if CV_SIMD + const int VECSZ = v_uint16::nlanes; + v_uint16 v_6 = vx_setall_u16(6); + for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) + v_store((uint16_t*)dst, (vx_load_expand(src) * v_6 + ((vx_load_expand(src - cn) + vx_load_expand(src + cn)) << 2) + vx_load_expand(src - 2 * cn) + vx_load_expand(src + 2 * cn)) << 4); +#endif for (; i < lencn; i++, src++, dst++) *((uint16_t*)dst) = (uint16_t(src[0]) * 6 + ((uint16_t(src[-cn]) + uint16_t(src[cn])) << 2) + uint16_t(src[-2 * cn]) + uint16_t(src[2 * cn])) << 4; @@ -2721,20 +2682,15 @@ void hlineSmooth5Nabcba(const uint8_t* src, int cn, cons src += 2 * cn; dst += 2 * cn; int i = 2 * cn, lencn = (len - 2)*cn; - v_uint16x8 v_mul0 = v_setall_u16(*((uint16_t*)m)); - v_uint16x8 v_mul1 = v_setall_u16(*((uint16_t*)(m + 1))); - v_uint16x8 v_mul2 = v_setall_u16(*((uint16_t*)(m + 2))); - for (; i <= lencn - 16; i += 16, src += 16, dst += 16) - { - v_uint16x8 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21, v_src30, v_src31, v_src40, v_src41; - v_expand(v_load(src - 2 * cn), v_src00, v_src01); - v_expand(v_load(src - cn), v_src10, v_src11); - v_expand(v_load(src), v_src20, v_src21); - v_expand(v_load(src + cn), v_src30, v_src31); - v_expand(v_load(src + 2 * cn), v_src40, v_src41); - v_store((uint16_t*)dst, (v_src00 + v_src40) * v_mul0 + (v_src10 + v_src30)* v_mul1 + v_src20 * v_mul2); - v_store((uint16_t*)dst + 8, (v_src01 + v_src41) * v_mul0 + (v_src11 + v_src31) * v_mul1 + v_src21 * v_mul2); - } +#if CV_SIMD + const uint16_t* _m = (const uint16_t*)m; + const int VECSZ = v_uint16::nlanes; + v_uint16 v_mul0 = vx_setall_u16(_m[0]); + v_uint16 v_mul1 = vx_setall_u16(_m[1]); + v_uint16 v_mul2 = vx_setall_u16(_m[2]); + for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) + v_store((uint16_t*)dst, (vx_load_expand(src - 2 * cn) + vx_load_expand(src + 2 * cn)) * v_mul0 + (vx_load_expand(src - cn) + vx_load_expand(src + cn))* v_mul1 + vx_load_expand(src) * v_mul2); +#endif for (; i < lencn; i++, src++, dst++) *((uint16_t*)dst) = ((uint16_t*)m)[0] * ((uint16_t)(src[-2 * cn]) + (uint16_t)(src[2 * cn])) + ((uint16_t*)m)[1] * ((uint16_t)(src[-cn]) + (uint16_t)(src[cn])) + ((uint16_t*)m)[2] * src[0]; @@ -2844,23 +2800,16 @@ void hlineSmooth(const uint8_t* src, int cn, const ufixe } i *= cn; int lencn = (len - post_shift + 1)*cn; - for (; i <= lencn - 16; i+=16, src+=16, dst+=16) +#if CV_SIMD + const int VECSZ = v_uint16::nlanes; + for (; i <= lencn - VECSZ; i+=VECSZ, src+=VECSZ, dst+=VECSZ) { - v_uint16x8 v_src0, v_src1; - v_uint16x8 v_mul = v_setall_u16(*((uint16_t*)m)); - v_expand(v_load(src), v_src0, v_src1); - v_uint16x8 v_res0 = v_src0 * v_mul; - v_uint16x8 v_res1 = v_src1 * v_mul; + v_uint16 v_res0 = vx_load_expand(src) * vx_setall_u16(*((uint16_t*)m)); for (int j = 1; j < n; j++) - { - v_mul = v_setall_u16(*((uint16_t*)(m + j))); - v_expand(v_load(src + j * cn), v_src0, v_src1); - v_res0 += v_src0 * v_mul; - v_res1 += v_src1 * v_mul; - } + v_res0 += vx_load_expand(src + j * cn) * vx_setall_u16(*((uint16_t*)(m + j))); v_store((uint16_t*)dst, v_res0); - v_store((uint16_t*)dst+8, v_res1); } +#endif for (; i < lencn; i++, src++, dst++) { *dst = m[0] * src[0]; @@ -2970,26 +2919,16 @@ void hlineSmoothONa_yzy_a(const uint8_t* src, int cn, co } i *= cn; int lencn = (len - post_shift + 1)*cn; - for (; i <= lencn - 16; i += 16, src += 16, dst += 16) +#if CV_SIMD + const int VECSZ = v_uint16::nlanes; + for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) { - v_uint16x8 v_src00, v_src01, v_srcN00, v_srcN01; - - v_uint16x8 v_mul = v_setall_u16(*((uint16_t*)(m + pre_shift))); - v_expand(v_load(src + pre_shift * cn), v_src00, v_src01); - v_uint16x8 v_res0 = v_src00 * v_mul; - v_uint16x8 v_res1 = v_src01 * v_mul; + v_uint16 v_res0 = vx_load_expand(src + pre_shift * cn) * vx_setall_u16(*((uint16_t*)(m + pre_shift))); for (int j = 0; j < pre_shift; j ++) - { - v_mul = v_setall_u16(*((uint16_t*)(m + j))); - v_expand(v_load(src + j * cn), v_src00, v_src01); - v_expand(v_load(src + (n - 1 - j)*cn), v_srcN00, v_srcN01); - v_res0 += (v_src00 + v_srcN00) * v_mul; - v_res1 += (v_src01 + v_srcN01) * v_mul; - } - + v_res0 += (vx_load_expand(src + j * cn) + vx_load_expand(src + (n - 1 - j)*cn)) * vx_setall_u16(*((uint16_t*)(m + j))); v_store((uint16_t*)dst, v_res0); - v_store((uint16_t*)dst + 8, v_res1); } +#endif for (; i < lencn; i++, src++, dst++) { *dst = m[pre_shift] * src[pre_shift*cn]; @@ -3025,28 +2964,13 @@ template <> void vlineSmooth1N(const ufixedpoint16* const * src, const ufixedpoint16* m, int, uint8_t* dst, int len) { const ufixedpoint16* src0 = src[0]; - v_uint16x8 v_mul = v_setall_u16(*((uint16_t*)m)); -#if CV_SSE2 - v_uint16x8 v_1 = v_setall_u16(1); - v_mul += v_mul; -#endif int i = 0; - for (; i <= len - 16; i += 16) - { - v_uint16x8 v_src0 = v_load((uint16_t*)src0 + i); - v_uint16x8 v_src1 = v_load((uint16_t*)src0 + i + 8); - v_uint8x16 v_res; -#if CV_SSE2 - v_res.val = _mm_packus_epi16(_mm_srli_epi16(_mm_add_epi16(v_1.val, _mm_mulhi_epu16(v_src0.val, v_mul.val)),1), - _mm_srli_epi16(_mm_add_epi16(v_1.val, _mm_mulhi_epu16(v_src1.val, v_mul.val)),1)); -#else - v_uint32x4 v_res0, v_res1, v_res2, v_res3; - v_mul_expand(v_src0, v_mul, v_res0, v_res1); - v_mul_expand(v_src1, v_mul, v_res2, v_res3); - v_res = v_pack(v_rshr_pack<16>(v_res0, v_res1), v_rshr_pack<16>(v_res2, v_res3)); +#if CV_SIMD + const int VECSZ = v_uint16::nlanes; + v_uint16 v_mul = vx_setall_u16(*((uint16_t*)m)<<1); + for (; i <= len - VECSZ; i += VECSZ) + v_rshr_pack_store<1>(dst + i, v_mul_hi(vx_load((uint16_t*)src0 + i), v_mul)); #endif - v_store(dst + i, v_res); - } for (; i < len; i++) dst[i] = m[0] * src0[i]; } @@ -3062,8 +2986,11 @@ void vlineSmooth1N1(const ufixedpoint16* const * src, co { const ufixedpoint16* src0 = src[0]; int i = 0; - for (; i <= len - 8; i += 8) - v_rshr_pack_store<8>(dst + i, v_load((uint16_t*)(src0 + i))); +#if CV_SIMD + const int VECSZ = v_uint16::nlanes; + for (; i <= len - VECSZ; i += VECSZ) + v_rshr_pack_store<8>(dst + i, vx_load((uint16_t*)(src0 + i))); +#endif for (; i < len; i++) dst[i] = src0[i]; } @@ -3077,46 +3004,51 @@ template <> void vlineSmooth3N(const ufixedpoint16* const * src, const ufixedpoint16* m, int, uint8_t* dst, int len) { int i = 0; - static const v_int16x8 v_128 = v_reinterpret_as_s16(v_setall_u16((uint16_t)1 << 15)); - v_int32x4 v_128_4 = v_setall_s32(128 << 16); - if (len > 7) +#if CV_SIMD + static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); + v_int32 v_128_4 = vx_setall_s32(128 << 16); + const int VECSZ = v_uint16::nlanes; + if (len >= VECSZ) { ufixedpoint32 val[] = { (m[0] + m[1] + m[2]) * ufixedpoint16((uint8_t)128) }; - v_128_4 = v_setall_s32(*((int32_t*)val)); - } - v_int16x8 v_mul01 = v_reinterpret_as_s16(v_setall_u32(*((uint32_t*)m))); - v_int16x8 v_mul2 = v_reinterpret_as_s16(v_setall_u16(*((uint16_t*)(m + 2)))); - for (; i <= len - 32; i += 32) - { - v_int16x8 v_src00, v_src10, v_src01, v_src11, v_src02, v_src12, v_src03, v_src13; - v_int16x8 v_tmp0, v_tmp1; - - v_src00 = v_load((int16_t*)(src[0]) + i); - v_src01 = v_load((int16_t*)(src[0]) + i + 8); - v_src02 = v_load((int16_t*)(src[0]) + i + 16); - v_src03 = v_load((int16_t*)(src[0]) + i + 24); - v_src10 = v_load((int16_t*)(src[1]) + i); - v_src11 = v_load((int16_t*)(src[1]) + i + 8); - v_src12 = v_load((int16_t*)(src[1]) + i + 16); - v_src13 = v_load((int16_t*)(src[1]) + i + 24); + v_128_4 = vx_setall_s32(*((int32_t*)val)); + } + v_int16 v_mul01 = v_reinterpret_as_s16(vx_setall_u32(*((uint32_t*)m))); + v_int16 v_mul2 = v_reinterpret_as_s16(vx_setall_u16(*((uint16_t*)(m + 2)))); + for (; i <= len - 4*VECSZ; i += 4*VECSZ) + { + v_int16 v_src00, v_src10, v_src01, v_src11, v_src02, v_src12, v_src03, v_src13; + v_int16 v_tmp0, v_tmp1; + + const int16_t* src0 = (const int16_t*)src[0] + i; + const int16_t* src1 = (const int16_t*)src[1] + i; + v_src00 = vx_load(src0); + v_src01 = vx_load(src0 + VECSZ); + v_src02 = vx_load(src0 + 2*VECSZ); + v_src03 = vx_load(src0 + 3*VECSZ); + v_src10 = vx_load(src1); + v_src11 = vx_load(src1 + VECSZ); + v_src12 = vx_load(src1 + 2*VECSZ); + v_src13 = vx_load(src1 + 3*VECSZ); v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src10, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res0 = v_dotprod(v_tmp0, v_mul01); - v_int32x4 v_res1 = v_dotprod(v_tmp1, v_mul01); + v_int32 v_res0 = v_dotprod(v_tmp0, v_mul01); + v_int32 v_res1 = v_dotprod(v_tmp1, v_mul01); v_zip(v_add_wrap(v_src01, v_128), v_add_wrap(v_src11, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res2 = v_dotprod(v_tmp0, v_mul01); - v_int32x4 v_res3 = v_dotprod(v_tmp1, v_mul01); + v_int32 v_res2 = v_dotprod(v_tmp0, v_mul01); + v_int32 v_res3 = v_dotprod(v_tmp1, v_mul01); v_zip(v_add_wrap(v_src02, v_128), v_add_wrap(v_src12, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res4 = v_dotprod(v_tmp0, v_mul01); - v_int32x4 v_res5 = v_dotprod(v_tmp1, v_mul01); + v_int32 v_res4 = v_dotprod(v_tmp0, v_mul01); + v_int32 v_res5 = v_dotprod(v_tmp1, v_mul01); v_zip(v_add_wrap(v_src03, v_128), v_add_wrap(v_src13, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res6 = v_dotprod(v_tmp0, v_mul01); - v_int32x4 v_res7 = v_dotprod(v_tmp1, v_mul01); - - v_int32x4 v_resj0, v_resj1; - v_src00 = v_load((int16_t*)(src[2]) + i); - v_src01 = v_load((int16_t*)(src[2]) + i + 8); - v_src02 = v_load((int16_t*)(src[2]) + i + 16); - v_src03 = v_load((int16_t*)(src[2]) + i + 24); + v_int32 v_res6 = v_dotprod(v_tmp0, v_mul01); + v_int32 v_res7 = v_dotprod(v_tmp1, v_mul01); + + v_int32 v_resj0, v_resj1; + const int16_t* src2 = (const int16_t*)src[2] + i; + v_src00 = vx_load(src2); + v_src01 = vx_load(src2 + VECSZ); + v_src02 = vx_load(src2 + 2*VECSZ); + v_src03 = vx_load(src2 + 3*VECSZ); v_mul_expand(v_add_wrap(v_src00, v_128), v_mul2, v_resj0, v_resj1); v_res0 += v_resj0; v_res1 += v_resj1; @@ -3139,11 +3071,12 @@ void vlineSmooth3N(const ufixedpoint16* const * src, con v_res6 += v_128_4; v_res7 += v_128_4; - v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), - v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); - v_store(dst + i + 16, v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res4, v_res5)), - v_reinterpret_as_u16(v_rshr_pack<16>(v_res6, v_res7)))); + v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), + v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); + v_store(dst + i + 2*VECSZ, v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res4, v_res5)), + v_reinterpret_as_u16(v_rshr_pack<16>(v_res6, v_res7)))); } +#endif for (; i < len; i++) dst[i] = m[0] * src[0][i] + m[1] * src[1][i] + m[2] * src[2][i]; } @@ -3157,18 +3090,21 @@ template <> void vlineSmooth3N121(const ufixedpoint16* const * src, const ufixedpoint16*, int, uint8_t* dst, int len) { int i = 0; - for (; i <= len - 16; i += 16) - { - v_uint32x4 v_src00, v_src01, v_src02, v_src03, v_src10, v_src11, v_src12, v_src13, v_src20, v_src21, v_src22, v_src23; - v_expand(v_load((uint16_t*)(src[0]) + i), v_src00, v_src01); - v_expand(v_load((uint16_t*)(src[0]) + i + 8), v_src02, v_src03); - v_expand(v_load((uint16_t*)(src[1]) + i), v_src10, v_src11); - v_expand(v_load((uint16_t*)(src[1]) + i + 8), v_src12, v_src13); - v_expand(v_load((uint16_t*)(src[2]) + i), v_src20, v_src21); - v_expand(v_load((uint16_t*)(src[2]) + i + 8), v_src22, v_src23); +#if CV_SIMD + const int VECSZ = v_uint16::nlanes; + for (; i <= len - 2*VECSZ; i += 2*VECSZ) + { + v_uint32 v_src00, v_src01, v_src02, v_src03, v_src10, v_src11, v_src12, v_src13, v_src20, v_src21, v_src22, v_src23; + v_expand(vx_load((uint16_t*)(src[0]) + i), v_src00, v_src01); + v_expand(vx_load((uint16_t*)(src[0]) + i + VECSZ), v_src02, v_src03); + v_expand(vx_load((uint16_t*)(src[1]) + i), v_src10, v_src11); + v_expand(vx_load((uint16_t*)(src[1]) + i + VECSZ), v_src12, v_src13); + v_expand(vx_load((uint16_t*)(src[2]) + i), v_src20, v_src21); + v_expand(vx_load((uint16_t*)(src[2]) + i + VECSZ), v_src22, v_src23); v_store(dst + i, v_pack(v_rshr_pack<10>(v_src00 + v_src20 + (v_src10 + v_src10), v_src01 + v_src21 + (v_src11 + v_src11)), v_rshr_pack<10>(v_src02 + v_src22 + (v_src12 + v_src12), v_src03 + v_src23 + (v_src13 + v_src13)))); } +#endif for (; i < len; i++) dst[i] = (((uint32_t)(((uint16_t*)(src[0]))[i]) + (uint32_t)(((uint16_t*)(src[2]))[i]) + ((uint32_t)(((uint16_t*)(src[1]))[i]) << 1)) + (1 << 9)) >> 10; } @@ -3182,95 +3118,102 @@ template <> void vlineSmooth5N(const ufixedpoint16* const * src, const ufixedpoint16* m, int, uint8_t* dst, int len) { int i = 0; - static const v_int16x8 v_128 = v_reinterpret_as_s16(v_setall_u16((uint16_t)1 << 15)); - v_int32x4 v_128_4 = v_setall_s32(128 << 16); - if (len > 7) +#if CV_SIMD + const int VECSZ = v_uint16::nlanes; + if (len >= 4 * VECSZ) { ufixedpoint32 val[] = { (m[0] + m[1] + m[2] + m[3] + m[4]) * ufixedpoint16((uint8_t)128) }; - v_128_4 = v_setall_s32(*((int32_t*)val)); - } - v_int16x8 v_mul01 = v_reinterpret_as_s16(v_setall_u32(*((uint32_t*)m))); - v_int16x8 v_mul23 = v_reinterpret_as_s16(v_setall_u32(*((uint32_t*)(m + 2)))); - v_int16x8 v_mul4 = v_reinterpret_as_s16(v_setall_u16(*((uint16_t*)(m + 4)))); - for (; i <= len - 32; i += 32) - { - v_int16x8 v_src00, v_src10, v_src01, v_src11, v_src02, v_src12, v_src03, v_src13; - v_int16x8 v_tmp0, v_tmp1; - - v_src00 = v_load((int16_t*)(src[0]) + i); - v_src01 = v_load((int16_t*)(src[0]) + i + 8); - v_src02 = v_load((int16_t*)(src[0]) + i + 16); - v_src03 = v_load((int16_t*)(src[0]) + i + 24); - v_src10 = v_load((int16_t*)(src[1]) + i); - v_src11 = v_load((int16_t*)(src[1]) + i + 8); - v_src12 = v_load((int16_t*)(src[1]) + i + 16); - v_src13 = v_load((int16_t*)(src[1]) + i + 24); - v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src10, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res0 = v_dotprod(v_tmp0, v_mul01); - v_int32x4 v_res1 = v_dotprod(v_tmp1, v_mul01); - v_zip(v_add_wrap(v_src01, v_128), v_add_wrap(v_src11, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res2 = v_dotprod(v_tmp0, v_mul01); - v_int32x4 v_res3 = v_dotprod(v_tmp1, v_mul01); - v_zip(v_add_wrap(v_src02, v_128), v_add_wrap(v_src12, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res4 = v_dotprod(v_tmp0, v_mul01); - v_int32x4 v_res5 = v_dotprod(v_tmp1, v_mul01); - v_zip(v_add_wrap(v_src03, v_128), v_add_wrap(v_src13, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res6 = v_dotprod(v_tmp0, v_mul01); - v_int32x4 v_res7 = v_dotprod(v_tmp1, v_mul01); - - v_src00 = v_load((int16_t*)(src[2]) + i); - v_src01 = v_load((int16_t*)(src[2]) + i + 8); - v_src02 = v_load((int16_t*)(src[2]) + i + 16); - v_src03 = v_load((int16_t*)(src[2]) + i + 24); - v_src10 = v_load((int16_t*)(src[3]) + i); - v_src11 = v_load((int16_t*)(src[3]) + i + 8); - v_src12 = v_load((int16_t*)(src[3]) + i + 16); - v_src13 = v_load((int16_t*)(src[3]) + i + 24); - v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src10, v_128), v_tmp0, v_tmp1); - v_res0 += v_dotprod(v_tmp0, v_mul23); - v_res1 += v_dotprod(v_tmp1, v_mul23); - v_zip(v_add_wrap(v_src01, v_128), v_add_wrap(v_src11, v_128), v_tmp0, v_tmp1); - v_res2 += v_dotprod(v_tmp0, v_mul23); - v_res3 += v_dotprod(v_tmp1, v_mul23); - v_zip(v_add_wrap(v_src02, v_128), v_add_wrap(v_src12, v_128), v_tmp0, v_tmp1); - v_res4 += v_dotprod(v_tmp0, v_mul23); - v_res5 += v_dotprod(v_tmp1, v_mul23); - v_zip(v_add_wrap(v_src03, v_128), v_add_wrap(v_src13, v_128), v_tmp0, v_tmp1); - v_res6 += v_dotprod(v_tmp0, v_mul23); - v_res7 += v_dotprod(v_tmp1, v_mul23); - - v_int32x4 v_resj0, v_resj1; - v_src00 = v_load((int16_t*)(src[4]) + i); - v_src01 = v_load((int16_t*)(src[4]) + i + 8); - v_src02 = v_load((int16_t*)(src[4]) + i + 16); - v_src03 = v_load((int16_t*)(src[4]) + i + 24); - v_mul_expand(v_add_wrap(v_src00, v_128), v_mul4, v_resj0, v_resj1); - v_res0 += v_resj0; - v_res1 += v_resj1; - v_mul_expand(v_add_wrap(v_src01, v_128), v_mul4, v_resj0, v_resj1); - v_res2 += v_resj0; - v_res3 += v_resj1; - v_mul_expand(v_add_wrap(v_src02, v_128), v_mul4, v_resj0, v_resj1); - v_res4 += v_resj0; - v_res5 += v_resj1; - v_mul_expand(v_add_wrap(v_src03, v_128), v_mul4, v_resj0, v_resj1); - v_res6 += v_resj0; - v_res7 += v_resj1; + v_int32 v_128_4 = vx_setall_s32(*((int32_t*)val)); + static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); + v_int16 v_mul01 = v_reinterpret_as_s16(vx_setall_u32(*((uint32_t*)m))); + v_int16 v_mul23 = v_reinterpret_as_s16(vx_setall_u32(*((uint32_t*)(m + 2)))); + v_int16 v_mul4 = v_reinterpret_as_s16(vx_setall_u16(*((uint16_t*)(m + 4)))); + for (; i <= len - 4*VECSZ; i += 4*VECSZ) + { + v_int16 v_src00, v_src10, v_src01, v_src11, v_src02, v_src12, v_src03, v_src13; + v_int16 v_tmp0, v_tmp1; + + const int16_t* src0 = (const int16_t*)src[0] + i; + const int16_t* src1 = (const int16_t*)src[1] + i; + v_src00 = vx_load(src0); + v_src01 = vx_load(src0 + VECSZ); + v_src02 = vx_load(src0 + 2*VECSZ); + v_src03 = vx_load(src0 + 3*VECSZ); + v_src10 = vx_load(src1); + v_src11 = vx_load(src1 + VECSZ); + v_src12 = vx_load(src1 + 2*VECSZ); + v_src13 = vx_load(src1 + 3*VECSZ); + v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src10, v_128), v_tmp0, v_tmp1); + v_int32 v_res0 = v_dotprod(v_tmp0, v_mul01); + v_int32 v_res1 = v_dotprod(v_tmp1, v_mul01); + v_zip(v_add_wrap(v_src01, v_128), v_add_wrap(v_src11, v_128), v_tmp0, v_tmp1); + v_int32 v_res2 = v_dotprod(v_tmp0, v_mul01); + v_int32 v_res3 = v_dotprod(v_tmp1, v_mul01); + v_zip(v_add_wrap(v_src02, v_128), v_add_wrap(v_src12, v_128), v_tmp0, v_tmp1); + v_int32 v_res4 = v_dotprod(v_tmp0, v_mul01); + v_int32 v_res5 = v_dotprod(v_tmp1, v_mul01); + v_zip(v_add_wrap(v_src03, v_128), v_add_wrap(v_src13, v_128), v_tmp0, v_tmp1); + v_int32 v_res6 = v_dotprod(v_tmp0, v_mul01); + v_int32 v_res7 = v_dotprod(v_tmp1, v_mul01); + + const int16_t* src2 = (const int16_t*)src[2] + i; + const int16_t* src3 = (const int16_t*)src[3] + i; + v_src00 = vx_load(src2); + v_src01 = vx_load(src2 + VECSZ); + v_src02 = vx_load(src2 + 2*VECSZ); + v_src03 = vx_load(src2 + 3*VECSZ); + v_src10 = vx_load(src3); + v_src11 = vx_load(src3 + VECSZ); + v_src12 = vx_load(src3 + 2*VECSZ); + v_src13 = vx_load(src3 + 3*VECSZ); + v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src10, v_128), v_tmp0, v_tmp1); + v_res0 += v_dotprod(v_tmp0, v_mul23); + v_res1 += v_dotprod(v_tmp1, v_mul23); + v_zip(v_add_wrap(v_src01, v_128), v_add_wrap(v_src11, v_128), v_tmp0, v_tmp1); + v_res2 += v_dotprod(v_tmp0, v_mul23); + v_res3 += v_dotprod(v_tmp1, v_mul23); + v_zip(v_add_wrap(v_src02, v_128), v_add_wrap(v_src12, v_128), v_tmp0, v_tmp1); + v_res4 += v_dotprod(v_tmp0, v_mul23); + v_res5 += v_dotprod(v_tmp1, v_mul23); + v_zip(v_add_wrap(v_src03, v_128), v_add_wrap(v_src13, v_128), v_tmp0, v_tmp1); + v_res6 += v_dotprod(v_tmp0, v_mul23); + v_res7 += v_dotprod(v_tmp1, v_mul23); + + v_int32 v_resj0, v_resj1; + const int16_t* src4 = (const int16_t*)src[4] + i; + v_src00 = vx_load(src4); + v_src01 = vx_load(src4 + VECSZ); + v_src02 = vx_load(src4 + 2*VECSZ); + v_src03 = vx_load(src4 + 3*VECSZ); + v_mul_expand(v_add_wrap(v_src00, v_128), v_mul4, v_resj0, v_resj1); + v_res0 += v_resj0; + v_res1 += v_resj1; + v_mul_expand(v_add_wrap(v_src01, v_128), v_mul4, v_resj0, v_resj1); + v_res2 += v_resj0; + v_res3 += v_resj1; + v_mul_expand(v_add_wrap(v_src02, v_128), v_mul4, v_resj0, v_resj1); + v_res4 += v_resj0; + v_res5 += v_resj1; + v_mul_expand(v_add_wrap(v_src03, v_128), v_mul4, v_resj0, v_resj1); + v_res6 += v_resj0; + v_res7 += v_resj1; - v_res0 += v_128_4; - v_res1 += v_128_4; - v_res2 += v_128_4; - v_res3 += v_128_4; - v_res4 += v_128_4; - v_res5 += v_128_4; - v_res6 += v_128_4; - v_res7 += v_128_4; + v_res0 += v_128_4; + v_res1 += v_128_4; + v_res2 += v_128_4; + v_res3 += v_128_4; + v_res4 += v_128_4; + v_res5 += v_128_4; + v_res6 += v_128_4; + v_res7 += v_128_4; - v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), - v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); - v_store(dst + i + 16, v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res4, v_res5)), - v_reinterpret_as_u16(v_rshr_pack<16>(v_res6, v_res7)))); + v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), + v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); + v_store(dst + i + 2*VECSZ, v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res4, v_res5)), + v_reinterpret_as_u16(v_rshr_pack<16>(v_res6, v_res7)))); + } } +#endif for (; i < len; i++) dst[i] = m[0] * src[0][i] + m[1] * src[1][i] + m[2] * src[2][i] + m[3] * src[3][i] + m[4] * src[4][i]; } @@ -3284,28 +3227,31 @@ template <> void vlineSmooth5N14641(const ufixedpoint16* const * src, const ufixedpoint16*, int, uint8_t* dst, int len) { int i = 0; - v_uint32x4 v_6 = v_setall_u32(6); - for (; i <= len - 16; i += 16) - { - v_uint32x4 v_src00, v_src10, v_src20, v_src30, v_src40; - v_uint32x4 v_src01, v_src11, v_src21, v_src31, v_src41; - v_uint32x4 v_src02, v_src12, v_src22, v_src32, v_src42; - v_uint32x4 v_src03, v_src13, v_src23, v_src33, v_src43; - v_expand(v_load((uint16_t*)(src[0]) + i), v_src00, v_src01); - v_expand(v_load((uint16_t*)(src[0]) + i + 8), v_src02, v_src03); - v_expand(v_load((uint16_t*)(src[1]) + i), v_src10, v_src11); - v_expand(v_load((uint16_t*)(src[1]) + i + 8), v_src12, v_src13); - v_expand(v_load((uint16_t*)(src[2]) + i), v_src20, v_src21); - v_expand(v_load((uint16_t*)(src[2]) + i + 8), v_src22, v_src23); - v_expand(v_load((uint16_t*)(src[3]) + i), v_src30, v_src31); - v_expand(v_load((uint16_t*)(src[3]) + i + 8), v_src32, v_src33); - v_expand(v_load((uint16_t*)(src[4]) + i), v_src40, v_src41); - v_expand(v_load((uint16_t*)(src[4]) + i + 8), v_src42, v_src43); +#if CV_SIMD + v_uint32 v_6 = vx_setall_u32(6); + const int VECSZ = v_uint16::nlanes; + for (; i <= len - 2*VECSZ; i += 2*VECSZ) + { + v_uint32 v_src00, v_src10, v_src20, v_src30, v_src40; + v_uint32 v_src01, v_src11, v_src21, v_src31, v_src41; + v_uint32 v_src02, v_src12, v_src22, v_src32, v_src42; + v_uint32 v_src03, v_src13, v_src23, v_src33, v_src43; + v_expand(vx_load((uint16_t*)(src[0]) + i), v_src00, v_src01); + v_expand(vx_load((uint16_t*)(src[0]) + i + VECSZ), v_src02, v_src03); + v_expand(vx_load((uint16_t*)(src[1]) + i), v_src10, v_src11); + v_expand(vx_load((uint16_t*)(src[1]) + i + VECSZ), v_src12, v_src13); + v_expand(vx_load((uint16_t*)(src[2]) + i), v_src20, v_src21); + v_expand(vx_load((uint16_t*)(src[2]) + i + VECSZ), v_src22, v_src23); + v_expand(vx_load((uint16_t*)(src[3]) + i), v_src30, v_src31); + v_expand(vx_load((uint16_t*)(src[3]) + i + VECSZ), v_src32, v_src33); + v_expand(vx_load((uint16_t*)(src[4]) + i), v_src40, v_src41); + v_expand(vx_load((uint16_t*)(src[4]) + i + VECSZ), v_src42, v_src43); v_store(dst + i, v_pack(v_rshr_pack<12>(v_src20*v_6 + ((v_src10 + v_src30) << 2) + v_src00 + v_src40, v_src21*v_6 + ((v_src11 + v_src31) << 2) + v_src01 + v_src41), v_rshr_pack<12>(v_src22*v_6 + ((v_src12 + v_src32) << 2) + v_src02 + v_src42, v_src23*v_6 + ((v_src13 + v_src33) << 2) + v_src03 + v_src43))); } +#endif for (; i < len; i++) dst[i] = ((uint32_t)(((uint16_t*)(src[2]))[i]) * 6 + (((uint32_t)(((uint16_t*)(src[1]))[i]) + (uint32_t)(((uint16_t*)(src[3]))[i])) << 2) + @@ -3326,57 +3272,63 @@ template <> void vlineSmooth(const ufixedpoint16* const * src, const ufixedpoint16* m, int n, uint8_t* dst, int len) { int i = 0; - static const v_int16x8 v_128 = v_reinterpret_as_s16(v_setall_u16((uint16_t)1 << 15)); - v_int32x4 v_128_4 = v_setall_s32(128 << 16); - if (len > 7) +#if CV_SIMD + static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); + v_int32 v_128_4 = vx_setall_s32(128 << 16); + const int VECSZ = v_uint16::nlanes; + if (len >= VECSZ) { ufixedpoint16 msum = m[0] + m[1]; for (int j = 2; j < n; j++) msum = msum + m[j]; ufixedpoint32 val[] = { msum * ufixedpoint16((uint8_t)128) }; - v_128_4 = v_setall_s32(*((int32_t*)val)); + v_128_4 = vx_setall_s32(*((int32_t*)val)); } - for (; i <= len - 32; i += 32) + for (; i <= len - 4*VECSZ; i += 4*VECSZ) { - v_int16x8 v_src00, v_src10, v_src01, v_src11, v_src02, v_src12, v_src03, v_src13; - v_int16x8 v_tmp0, v_tmp1; + v_int16 v_src00, v_src10, v_src01, v_src11, v_src02, v_src12, v_src03, v_src13; + v_int16 v_tmp0, v_tmp1; - v_int16x8 v_mul = v_reinterpret_as_s16(v_setall_u32(*((uint32_t*)m))); + v_int16 v_mul = v_reinterpret_as_s16(vx_setall_u32(*((uint32_t*)m))); - v_src00 = v_load((int16_t*)(src[0]) + i); - v_src01 = v_load((int16_t*)(src[0]) + i + 8); - v_src02 = v_load((int16_t*)(src[0]) + i + 16); - v_src03 = v_load((int16_t*)(src[0]) + i + 24); - v_src10 = v_load((int16_t*)(src[1]) + i); - v_src11 = v_load((int16_t*)(src[1]) + i + 8); - v_src12 = v_load((int16_t*)(src[1]) + i + 16); - v_src13 = v_load((int16_t*)(src[1]) + i + 24); + const int16_t* src0 = (const int16_t*)src[0] + i; + const int16_t* src1 = (const int16_t*)src[1] + i; + v_src00 = vx_load(src0); + v_src01 = vx_load(src0 + VECSZ); + v_src02 = vx_load(src0 + 2*VECSZ); + v_src03 = vx_load(src0 + 3*VECSZ); + v_src10 = vx_load(src1); + v_src11 = vx_load(src1 + VECSZ); + v_src12 = vx_load(src1 + 2*VECSZ); + v_src13 = vx_load(src1 + 3*VECSZ); v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src10, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res0 = v_dotprod(v_tmp0, v_mul); - v_int32x4 v_res1 = v_dotprod(v_tmp1, v_mul); + v_int32 v_res0 = v_dotprod(v_tmp0, v_mul); + v_int32 v_res1 = v_dotprod(v_tmp1, v_mul); v_zip(v_add_wrap(v_src01, v_128), v_add_wrap(v_src11, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res2 = v_dotprod(v_tmp0, v_mul); - v_int32x4 v_res3 = v_dotprod(v_tmp1, v_mul); + v_int32 v_res2 = v_dotprod(v_tmp0, v_mul); + v_int32 v_res3 = v_dotprod(v_tmp1, v_mul); v_zip(v_add_wrap(v_src02, v_128), v_add_wrap(v_src12, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res4 = v_dotprod(v_tmp0, v_mul); - v_int32x4 v_res5 = v_dotprod(v_tmp1, v_mul); + v_int32 v_res4 = v_dotprod(v_tmp0, v_mul); + v_int32 v_res5 = v_dotprod(v_tmp1, v_mul); v_zip(v_add_wrap(v_src03, v_128), v_add_wrap(v_src13, v_128), v_tmp0, v_tmp1); - v_int32x4 v_res6 = v_dotprod(v_tmp0, v_mul); - v_int32x4 v_res7 = v_dotprod(v_tmp1, v_mul); + v_int32 v_res6 = v_dotprod(v_tmp0, v_mul); + v_int32 v_res7 = v_dotprod(v_tmp1, v_mul); int j = 2; for (; j < n - 1; j+=2) { - v_mul = v_reinterpret_as_s16(v_setall_u32(*((uint32_t*)(m+j)))); - - v_src00 = v_load((int16_t*)(src[j]) + i); - v_src01 = v_load((int16_t*)(src[j]) + i + 8); - v_src02 = v_load((int16_t*)(src[j]) + i + 16); - v_src03 = v_load((int16_t*)(src[j]) + i + 24); - v_src10 = v_load((int16_t*)(src[j+1]) + i); - v_src11 = v_load((int16_t*)(src[j+1]) + i + 8); - v_src12 = v_load((int16_t*)(src[j+1]) + i + 16); - v_src13 = v_load((int16_t*)(src[j+1]) + i + 24); + v_mul = v_reinterpret_as_s16(vx_setall_u32(*((uint32_t*)(m+j)))); + + const int16_t* srcj0 = (const int16_t*)src[j] + i; + const int16_t* srcj1 = (const int16_t*)src[j + 1] + i; + v_src00 = vx_load(srcj0); + v_src01 = vx_load(srcj0 + VECSZ); + v_src02 = vx_load(srcj0 + 2*VECSZ); + v_src03 = vx_load(srcj0 + 3*VECSZ); + v_src10 = vx_load(srcj1); + v_src11 = vx_load(srcj1 + VECSZ); + v_src12 = vx_load(srcj1 + 2*VECSZ); + v_src13 = vx_load(srcj1 + 3*VECSZ); v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src10, v_128), v_tmp0, v_tmp1); v_res0 += v_dotprod(v_tmp0, v_mul); v_res1 += v_dotprod(v_tmp1, v_mul); @@ -3392,12 +3344,13 @@ void vlineSmooth(const ufixedpoint16* const * src, const } if(j < n) { - v_int32x4 v_resj0, v_resj1; - v_mul = v_reinterpret_as_s16(v_setall_u16(*((uint16_t*)(m + j)))); - v_src00 = v_load((int16_t*)(src[j]) + i); - v_src01 = v_load((int16_t*)(src[j]) + i + 8); - v_src02 = v_load((int16_t*)(src[j]) + i + 16); - v_src03 = v_load((int16_t*)(src[j]) + i + 24); + v_int32 v_resj0, v_resj1; + v_mul = v_reinterpret_as_s16(vx_setall_u16(*((uint16_t*)(m + j)))); + const int16_t* srcj = (const int16_t*)src[j] + i; + v_src00 = vx_load(srcj); + v_src01 = vx_load(srcj + VECSZ); + v_src02 = vx_load(srcj + 2*VECSZ); + v_src03 = vx_load(srcj + 3*VECSZ); v_mul_expand(v_add_wrap(v_src00, v_128), v_mul, v_resj0, v_resj1); v_res0 += v_resj0; v_res1 += v_resj1; @@ -3420,11 +3373,12 @@ void vlineSmooth(const ufixedpoint16* const * src, const v_res6 += v_128_4; v_res7 += v_128_4; - v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), - v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); - v_store(dst + i + 16, v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res4, v_res5)), - v_reinterpret_as_u16(v_rshr_pack<16>(v_res6, v_res7)))); + v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), + v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); + v_store(dst + i + 2*VECSZ, v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res4, v_res5)), + v_reinterpret_as_u16(v_rshr_pack<16>(v_res6, v_res7)))); } +#endif for (; i < len; i++) { ufixedpoint32 val = m[0] * src[0][i]; @@ -3450,29 +3404,32 @@ void vlineSmoothONa_yzy_a(const FT* const * src, const FT* m, int n, ET* dst, in template <> void vlineSmoothONa_yzy_a(const ufixedpoint16* const * src, const ufixedpoint16* m, int n, uint8_t* dst, int len) { - int pre_shift = n / 2; int i = 0; - static const v_int16x8 v_128 = v_reinterpret_as_s16(v_setall_u16((uint16_t)1 << 15)); - v_int32x4 v_128_4 = v_setall_s32(128 << 16); - if (len > 7) +#if CV_SIMD + int pre_shift = n / 2; + static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); + v_int32 v_128_4 = vx_setall_s32(128 << 16); + const int VECSZ = v_uint16::nlanes; + if (len >= VECSZ) { ufixedpoint16 msum = m[0] + m[pre_shift] + m[n - 1]; for (int j = 1; j < pre_shift; j++) msum = msum + m[j] + m[n - 1 - j]; ufixedpoint32 val[] = { msum * ufixedpoint16((uint8_t)128) }; - v_128_4 = v_setall_s32(*((int32_t*)val)); + v_128_4 = vx_setall_s32(*((int32_t*)val)); } - for (; i <= len - 32; i += 32) + for (; i <= len - 4*VECSZ; i += 4*VECSZ) { - v_int16x8 v_src00, v_src10, v_src20, v_src30, v_src01, v_src11, v_src21, v_src31; - v_int32x4 v_res0, v_res1, v_res2, v_res3, v_res4, v_res5, v_res6, v_res7; - v_int16x8 v_tmp0, v_tmp1, v_tmp2, v_tmp3, v_tmp4, v_tmp5, v_tmp6, v_tmp7; + v_int16 v_src00, v_src10, v_src20, v_src30, v_src01, v_src11, v_src21, v_src31; + v_int32 v_res0, v_res1, v_res2, v_res3, v_res4, v_res5, v_res6, v_res7; + v_int16 v_tmp0, v_tmp1, v_tmp2, v_tmp3, v_tmp4, v_tmp5, v_tmp6, v_tmp7; - v_int16x8 v_mul = v_reinterpret_as_s16(v_setall_u16(*((uint16_t*)(m + pre_shift)))); - v_src00 = v_load((int16_t*)(src[pre_shift]) + i); - v_src10 = v_load((int16_t*)(src[pre_shift]) + i + 8); - v_src20 = v_load((int16_t*)(src[pre_shift]) + i + 16); - v_src30 = v_load((int16_t*)(src[pre_shift]) + i + 24); + v_int16 v_mul = v_reinterpret_as_s16(vx_setall_u16(*((uint16_t*)(m + pre_shift)))); + const int16_t* srcp = (const int16_t*)src[pre_shift] + i; + v_src00 = vx_load(srcp); + v_src10 = vx_load(srcp + VECSZ); + v_src20 = vx_load(srcp + 2*VECSZ); + v_src30 = vx_load(srcp + 3*VECSZ); v_mul_expand(v_add_wrap(v_src00, v_128), v_mul, v_res0, v_res1); v_mul_expand(v_add_wrap(v_src10, v_128), v_mul, v_res2, v_res3); v_mul_expand(v_add_wrap(v_src20, v_128), v_mul, v_res4, v_res5); @@ -3481,16 +3438,18 @@ void vlineSmoothONa_yzy_a(const ufixedpoint16* const * s int j = 0; for (; j < pre_shift; j++) { - v_mul = v_reinterpret_as_s16(v_setall_u16(*((uint16_t*)(m + j)))); - - v_src00 = v_load((int16_t*)(src[j]) + i); - v_src10 = v_load((int16_t*)(src[j]) + i + 8); - v_src20 = v_load((int16_t*)(src[j]) + i + 16); - v_src30 = v_load((int16_t*)(src[j]) + i + 24); - v_src01 = v_load((int16_t*)(src[n - 1 - j]) + i); - v_src11 = v_load((int16_t*)(src[n - 1 - j]) + i + 8); - v_src21 = v_load((int16_t*)(src[n - 1 - j]) + i + 16); - v_src31 = v_load((int16_t*)(src[n - 1 - j]) + i + 24); + v_mul = v_reinterpret_as_s16(vx_setall_u16(*((uint16_t*)(m + j)))); + + const int16_t* srcj0 = (const int16_t*)src[j] + i; + const int16_t* srcj1 = (const int16_t*)src[n - 1 - j] + i; + v_src00 = vx_load(srcj0); + v_src10 = vx_load(srcj0 + VECSZ); + v_src20 = vx_load(srcj0 + 2*VECSZ); + v_src30 = vx_load(srcj0 + 3*VECSZ); + v_src01 = vx_load(srcj1); + v_src11 = vx_load(srcj1 + VECSZ); + v_src21 = vx_load(srcj1 + 2*VECSZ); + v_src31 = vx_load(srcj1 + 3*VECSZ); v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src01, v_128), v_tmp0, v_tmp1); v_res0 += v_dotprod(v_tmp0, v_mul); v_res1 += v_dotprod(v_tmp1, v_mul); @@ -3514,11 +3473,12 @@ void vlineSmoothONa_yzy_a(const ufixedpoint16* const * s v_res6 += v_128_4; v_res7 += v_128_4; - v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), - v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); - v_store(dst + i + 16, v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res4, v_res5)), - v_reinterpret_as_u16(v_rshr_pack<16>(v_res6, v_res7)))); + v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), + v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); + v_store(dst + i + 2*VECSZ, v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res4, v_res5)), + v_reinterpret_as_u16(v_rshr_pack<16>(v_res6, v_res7)))); } +#endif for (; i < len; i++) { ufixedpoint32 val = m[0] * src[0][i]; @@ -3816,8 +3776,8 @@ static void createGaussianKernels( T & kx, T & ky, int type, Size &ksize, if( ksize.height <= 0 && sigma2 > 0 ) ksize.height = cvRound(sigma2*(depth == CV_8U ? 3 : 4)*2 + 1)|1; - CV_Assert( ksize.width > 0 && ksize.width % 2 == 1 && - ksize.height > 0 && ksize.height % 2 == 1 ); + CV_Assert( ksize.width > 0 && ksize.width % 2 == 1 && + ksize.height > 0 && ksize.height % 2 == 1 ); sigma1 = std::max( sigma1, 0. ); sigma2 = std::max( sigma2, 0. );