Chiebot-Mirror
/
opencv
mirror of https://github.com/opencv/opencv.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #20287 from hanliutong:dev-rvv-0.10

Browse Source 
Optimization of DNN using native RISC-V vector intrinsics.

* Use RVV to optimize fastGEMM (FP32) in DNN.

* Use RVV to optimize fastGEMM1T in DNN.

* Use RVV to optimize fastConv in DNN.

* Use RVV to optimize fastDepthwiseConv in DNN.

* Vectorize tails using vl.

* Use "vl" instead of scalar to handle small block in fastConv.

* Fix memory access out of bound in "fastGEMM1T".

* Remove setvl.

* Remove useless initialization.

* Use loop unrolling to handle tail part instead of switch.
pull/20536/head
HAN Liutong 3 years ago committed by GitHub
parent 5e7f06397f
commit aaca4987c9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 582 additions and 3 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      modules/dnn/CMakeLists.txt
  2. 25
      modules/dnn/src/layers/convolution_layer.cpp
  3. 9
      modules/dnn/src/layers/fully_connected_layer.cpp
  4. 549
      modules/dnn/src/layers/layers_common.simd.hpp

2
modules/dnn/CMakeLists.txt
Unescape View File

@ -8,7 +8,7 @@ endif()
set(the_description "Deep neural network module. It allows to load models from different frameworks and to make forward pass")
ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX)
ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX RVV)
ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js)

25
modules/dnn/src/layers/convolution_layer.cpp
Unescape View File

@ -914,11 +914,12 @@ public:
bool useAVX;
bool useAVX2;
bool useAVX512;
bool useRVV;
int blk_size_cn;
ParallelConv()
: input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0),
biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false)
biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false), useRVV(false)
, blk_size_cn(0)
{}
@ -976,6 +977,7 @@ public:
p.useAVX = checkHardwareSupport(CPU_AVX) && isConv2D;
p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D;
p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D;
p.useRVV = checkHardwareSupport(CPU_RVV) && isConv2D;
int kernel_d = isConv3D? kernel_size[0] : 1;
int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2];
@ -1176,6 +1178,13 @@ public:
stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l,
biasptr, relu, inptr_, height, width, outptr_, out_d, outH, outW);
else
#endif
#if CV_TRY_RVV
if(useRVV)
opt_RVV::fastDepthwiseConv(wptr, kernel_h, kernel_w,
stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l,
biasptr, relu, inptr_, height, width, outptr_, out_d, outH, outW);
else
#endif
{
const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2],
@ -1546,6 +1555,12 @@ public:
opt_AVX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
else
#endif
#if CV_TRY_RVV
if(useRVV)
opt_RVV::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
else
#endif
for( int i = 0; i < outCn; i += 2 )
{
@ -2297,6 +2312,7 @@ public:
useAVX = checkHardwareSupport(CPU_AVX);
useAVX2 = checkHardwareSupport(CPU_AVX2);
useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX;
useRVV = checkHardwareSupport(CPU_RVV);
}
void operator()(const Range& range_) const CV_OVERRIDE
@ -2328,6 +2344,12 @@ public:
if( useAVX )
opt_AVX::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
else
#endif
#if CV_TRY_RVV
if( useRVV ) {
opt_RVV::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
}
else
#endif
for( m = 0; m < mmax; m += 2 )
{
@ -2427,6 +2449,7 @@ public:
bool useAVX;
bool useAVX2;
bool useAVX512;
bool useRVV;
};
class Col2ImInvoker : public cv::ParallelLoopBody

9
modules/dnn/src/layers/fully_connected_layer.cpp
Unescape View File

@ -168,7 +168,7 @@ public:
class FullyConnected : public ParallelLoopBody
{
public:
FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false), useAVX512(false) {}
FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false), useAVX512(false), useRVV(false) {}
static void run(const Mat& srcMat, const Mat& weights, const Mat& biasMat,
Mat& dstMat, const ActivationLayer* activ, int nstripes)
@ -191,6 +191,7 @@ public:
p.useAVX = checkHardwareSupport(CPU_AVX);
p.useAVX2 = checkHardwareSupport(CPU_AVX2);
p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX;
p.useRVV = checkHardwareSupport(CPU_RVV);
parallel_for_(Range(0, nstripes), p, nstripes);
}
@ -239,6 +240,11 @@ public:
if( useAVX )
opt_AVX::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize);
else
#endif
#if CV_TRY_RVV
if( useRVV )
opt_RVV::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize);
else
#endif
{
int i = 0;
@ -293,6 +299,7 @@ public:
bool useAVX;
bool useAVX2;
bool useAVX512;
bool useRVV;
};
#ifdef HAVE_OPENCL

549
modules/dnn/src/layers/layers_common.simd.hpp
Unescape View File

@ -737,5 +737,554 @@ void fastGEMM( const float* aptr, size_t astep, const float* bptr,
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_RVV
void fastGEMM( const float* aptr, size_t astep, const float* bptr,
size_t bstep, float* cptr, size_t cstep,
int ma, int na, int nb )
{
int n = 0;
size_t vl = 8;
size_t mvl0 = 8;
size_t mvl1 = 8;
for( ; n < nb; n += 16 )
{
if ( n + 16 > nb) {
mvl0 = nb - n;
mvl1 = (nb - n -8) > 0 ? (nb - n -8) : 0;
}
for( int m = 0; m < ma; m += 4 )
{
const float* aptr0 = aptr + astep*m;
const float* aptr1 = aptr + astep*std::min(m+1, ma-1);
const float* aptr2 = aptr + astep*std::min(m+2, ma-1);
const float* aptr3 = aptr + astep*std::min(m+3, ma-1);
float* cptr0 = cptr + cstep*m;
float* cptr1 = cptr + cstep*std::min(m+1, ma-1);
float* cptr2 = cptr + cstep*std::min(m+2, ma-1);
float* cptr3 = cptr + cstep*std::min(m+3, ma-1);
vfloat32m2_t d00 = vfmv_v_f_f32m2(0, vl), d01 = vfmv_v_f_f32m2(0, vl);
vfloat32m2_t d10 = vfmv_v_f_f32m2(0, vl), d11 = vfmv_v_f_f32m2(0, vl);
vfloat32m2_t d20 = vfmv_v_f_f32m2(0, vl), d21 = vfmv_v_f_f32m2(0, vl);
vfloat32m2_t d30 = vfmv_v_f_f32m2(0, vl), d31 = vfmv_v_f_f32m2(0, vl);
for( int k = 0; k < na; k++ )
{
vfloat32m2_t a0 = vfmv_v_f_f32m2(aptr0[k], vl);
vfloat32m2_t a1 = vfmv_v_f_f32m2(aptr1[k], vl);
vfloat32m2_t a2 = vfmv_v_f_f32m2(aptr2[k], vl);
vfloat32m2_t a3 = vfmv_v_f_f32m2(aptr3[k], vl);
vfloat32m2_t b0 = vle32_v_f32m2(bptr + k*bstep + n, mvl0);
vfloat32m2_t b1 = vle32_v_f32m2(bptr + k*bstep + n + 8, mvl1);
d00 = vfmacc_vv_f32m2(d00, a0, b0, mvl0);
d01 = vfmacc_vv_f32m2(d01, a0, b1, mvl1);
d10 = vfmacc_vv_f32m2(d10, a1, b0, mvl0);
d11 = vfmacc_vv_f32m2(d11, a1, b1, mvl1);
d20 = vfmacc_vv_f32m2(d20, a2, b0, mvl0);
d21 = vfmacc_vv_f32m2(d21, a2, b1, mvl1);
d30 = vfmacc_vv_f32m2(d30, a3, b0, mvl0);
d31 = vfmacc_vv_f32m2(d31, a3, b1, mvl1);
}
vse32_v_f32m2(cptr0 + n, d00, mvl0);
vse32_v_f32m2(cptr1 + n, d10, mvl0);
vse32_v_f32m2(cptr2 + n, d20, mvl0);
vse32_v_f32m2(cptr3 + n, d30, mvl0);
vse32_v_f32m2(cptr0 + n + 8, d01, mvl1);
vse32_v_f32m2(cptr1 + n + 8, d11, mvl1);
vse32_v_f32m2(cptr2 + n + 8, d21, mvl1);
vse32_v_f32m2(cptr3 + n + 8, d31, mvl1);
}
}
}
void fastGEMM1T( const float* vec, const float* weights,
size_t wstep, const float* bias,
float* dst, int nvecs, int vecsize )
{
int i = 0;
size_t vl = 8;
for( ; i <= nvecs - 8; i += 8 )
{
const float* wptr = weights + i*wstep;
vfloat32m2_t vs0 = vfmv_v_f_f32m2(0, vl), vs1 = vfmv_v_f_f32m2(0, vl),
vs2 = vfmv_v_f_f32m2(0, vl), vs3 = vfmv_v_f_f32m2(0, vl),
vs4 = vfmv_v_f_f32m2(0, vl), vs5 = vfmv_v_f_f32m2(0, vl),
vs6 = vfmv_v_f_f32m2(0, vl), vs7 = vfmv_v_f_f32m2(0, vl);
for( int k = 0; k < vecsize; k += 8, wptr += 8 )
{
vfloat32m2_t v = vle32_v_f32m2(vec + k, vl);
vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl);
vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep, vl), v, vl);
vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, vl), v, vl);
vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, vl), v, vl);
vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, vl), v, vl);
vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, vl), v, vl);
vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, vl), v, vl);
vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, vl), v, vl);
}
// Calculate the sum of each vector
vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl);
vfloat32m1_t temp0 = vfredsum_vs_f32m2_f32m1(temp0, vs0, zero, vl);
vfloat32m1_t temp1 = vfredsum_vs_f32m2_f32m1(temp1, vs1, zero, vl);
vfloat32m1_t temp2 = vfredsum_vs_f32m2_f32m1(temp2, vs2, zero, vl);
vfloat32m1_t temp3 = vfredsum_vs_f32m2_f32m1(temp3, vs3, zero, vl);
vfloat32m1_t temp4 = vfredsum_vs_f32m2_f32m1(temp4, vs4, zero, vl);
vfloat32m1_t temp5 = vfredsum_vs_f32m2_f32m1(temp5, vs5, zero, vl);
vfloat32m1_t temp6 = vfredsum_vs_f32m2_f32m1(temp6, vs6, zero, vl);
vfloat32m1_t temp7 = vfredsum_vs_f32m2_f32m1(temp7, vs7, zero, vl);
float32_t sum[8];
sum[0] = vfmv_f_s_f32m1_f32(temp0);
sum[1] = vfmv_f_s_f32m1_f32(temp1);
sum[2] = vfmv_f_s_f32m1_f32(temp2);
sum[3] = vfmv_f_s_f32m1_f32(temp3);
sum[4] = vfmv_f_s_f32m1_f32(temp4);
sum[5] = vfmv_f_s_f32m1_f32(temp5);
sum[6] = vfmv_f_s_f32m1_f32(temp6);
sum[7] = vfmv_f_s_f32m1_f32(temp7);
vfloat32m2_t s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum, vl), vle32_v_f32m2(bias + i, vl), vl);
vse32_v_f32m2(dst + i, s0, vl);
}
int mvl = nvecs - i;
if (mvl > 0)
{
const float* wptr = weights + i*wstep;
vfloat32m2_t vs0 = vfmv_v_f_f32m2(0, vl), vs1 = vfmv_v_f_f32m2(0, vl),
vs2 = vfmv_v_f_f32m2(0, vl), vs3 = vfmv_v_f_f32m2(0, vl),
vs4 = vfmv_v_f_f32m2(0, vl), vs5 = vfmv_v_f_f32m2(0, vl),
vs6 = vfmv_v_f_f32m2(0, vl), vs7 = vfmv_v_f_f32m2(0, vl);
int k = 0;
for( ; k <= vecsize - 8; k += 8, wptr += 8 )
{
vfloat32m2_t v = vle32_v_f32m2(vec + k, vl);
vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl);
vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), vl), v, vl);
vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), vl), v, vl);
vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), vl), v, vl);
vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), vl), v, vl);
vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), vl), v, vl);
vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), vl), v, vl);
}
int kvl = vecsize - k;
if (kvl > 0) {
vfloat32m2_t v = vle32_v_f32m2(vec + k, kvl);
vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, kvl), v, kvl);
vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), kvl), v, kvl);
vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), kvl), v, kvl);
vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), kvl), v, kvl);
vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), kvl), v, kvl);
vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), kvl), v, kvl);
vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), kvl), v, kvl);
}
// Calculate the sum of each vector
vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl);
vfloat32m1_t temp0 = vfmv_v_f_f32m1(0, 4), temp1 = vfmv_v_f_f32m1(0, 4),
temp2 = vfmv_v_f_f32m1(0, 4), temp3 = vfmv_v_f_f32m1(0, 4),
temp4 = vfmv_v_f_f32m1(0, 4), temp5 = vfmv_v_f_f32m1(0, 4),
temp6 = vfmv_v_f_f32m1(0, 4), temp7 = vfmv_v_f_f32m1(0, 4);
temp0 = vfredsum_vs_f32m2_f32m1(temp0, vs0, zero, vl);
temp1 = vfredsum_vs_f32m2_f32m1(temp1, vs1, zero, vl);
temp2 = vfredsum_vs_f32m2_f32m1(temp2, vs2, zero, vl);
temp3 = vfredsum_vs_f32m2_f32m1(temp3, vs3, zero, vl);
temp4 = vfredsum_vs_f32m2_f32m1(temp4, vs4, zero, vl);
temp5 = vfredsum_vs_f32m2_f32m1(temp5, vs5, zero, vl);
temp6 = vfredsum_vs_f32m2_f32m1(temp6, vs6, zero, vl);
temp7 = vfredsum_vs_f32m2_f32m1(temp7, vs7, zero, vl);
float32_t sum[8];
sum[0] = vfmv_f_s_f32m1_f32(temp0);
sum[1] = vfmv_f_s_f32m1_f32(temp1);
sum[2] = vfmv_f_s_f32m1_f32(temp2);
sum[3] = vfmv_f_s_f32m1_f32(temp3);
sum[4] = vfmv_f_s_f32m1_f32(temp4);
sum[5] = vfmv_f_s_f32m1_f32(temp5);
sum[6] = vfmv_f_s_f32m1_f32(temp6);
sum[7] = vfmv_f_s_f32m1_f32(temp7);
vfloat32m2_t s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum, mvl), vle32_v_f32m2(bias + i, mvl), mvl);
vse32_v_f32m2(dst + i, s0, mvl);
}
}
enum { FASCONV_BASE_VECSZ = 4 }; // TODO: Large base size.
void fastConv( const float* weights, size_t wstep, const float* bias,
const float* rowbuf, float* output, const int* outShape,
int blockSize, int vecsize, int vecsize_aligned,
const float* relu, bool initOutput )
{
int vl = 4;
int outCn = outShape[1];
size_t outPlaneSize = outShape[2]*outShape[3];
float r0 = 1.f, r1 = 1.f, r2 = 1.f;
vfloat32m1_t vr0 = vfmv_v_f_f32m1(1, vl), vr1 = vfmv_v_f_f32m1(1, vl), vr2 = vfmv_v_f_f32m1(1, vl);
int maskbuf[FASCONV_BASE_VECSZ] = {0};
int rsz = blockSize % FASCONV_BASE_VECSZ;
for( int i = 0; i < rsz; i++ )
maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1;
vint32m1_t vmaskbuf = vle32_v_i32m1(maskbuf ,vl);
vbool32_t mask = vmslt_vx_i32m1_b32(vmaskbuf, 0, vl); // mask for tail
// now compute dot product of the weights
// and im2row-transformed part of the tensor
for( int i = 0; i < outCn; i += 3 )
{
const float* wptr0 = weights + i*wstep;
const float* wptr1 = wptr0 + wstep;
const float* wptr2 = wptr1 + wstep;
float* outptr0 = output + i*outPlaneSize;
float* outptr1 = outptr0 + outPlaneSize;
float* outptr2 = outptr1 + outPlaneSize;
float bias0 = bias[i], bias1 = bias[i+1], bias2 = bias[i+2];
if( i+2 >= outCn )
{
wptr2 = wptr1;
outptr2 = outptr1;
bias2 = bias1;
if( i+1 >= outCn )
{
wptr2 = wptr1 = wptr0;
outptr2 = outptr1 = outptr0;
bias2 = bias1 = bias0;
}
}
if( relu )
{
r0 = relu[i]; r1 = relu[i+1]; r2 = relu[i+2];
if( i+2 >= outCn )
{
r2 = r1;
if( i+1 >= outCn )
r2 = r1 = r0;
}
vr0 = vfmv_v_f_f32m1(r0, vl);
vr1 = vfmv_v_f_f32m1(r1, vl);
vr2 = vfmv_v_f_f32m1(r2, vl);
}
int j = 0;
for( ; j < blockSize; j += FASCONV_BASE_VECSZ )
{
bool tail = false;
if (j + FASCONV_BASE_VECSZ > blockSize)
{
if (j == 0) {
vl = blockSize;
}
else {
j = blockSize - FASCONV_BASE_VECSZ;
tail = true;
}
}
int k = 0;
const float* rptr = rowbuf + j*vecsize_aligned;
int vlm2 = 8;
vfloat32m2_t vs00 = vfmv_v_f_f32m2(0, vlm2), vs01 = vfmv_v_f_f32m2(0, vlm2),
vs02 = vfmv_v_f_f32m2(0, vlm2), vs03 = vfmv_v_f_f32m2(0, vlm2),
vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2),
vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2),
vs20 = vfmv_v_f_f32m2(0, vlm2), vs21 = vfmv_v_f_f32m2(0, vlm2),
vs22 = vfmv_v_f_f32m2(0, vlm2), vs23 = vfmv_v_f_f32m2(0, vlm2);
for (; k < vecsize; k += 8, rptr += 8 )
{
if (k+8 >= vecsize) {
vlm2 = vecsize - k;
}
vfloat32m2_t w0 = vle32_v_f32m2(wptr0 + k, vlm2);
vfloat32m2_t w1 = vle32_v_f32m2(wptr1 + k, vlm2);
vfloat32m2_t w2 = vle32_v_f32m2(wptr2 + k, vlm2);
vfloat32m2_t r0 = vle32_v_f32m2(rptr, vlm2);
vs00 = vfmacc_vv_f32m2(vs00, w0, r0, vlm2);
vs10 = vfmacc_vv_f32m2(vs10, w1, r0, vlm2);
vs20 = vfmacc_vv_f32m2(vs20, w2, r0, vlm2);
r0 = vle32_v_f32m2(rptr + vecsize_aligned, vlm2);
vs01 = vfmacc_vv_f32m2(vs01, w0, r0, vlm2);
vs11 = vfmacc_vv_f32m2(vs11, w1, r0, vlm2);
vs21 = vfmacc_vv_f32m2(vs21, w2, r0, vlm2);
r0 = vle32_v_f32m2(rptr + vecsize_aligned*2, vlm2);
vs02 = vfmacc_vv_f32m2(vs02, w0, r0, vlm2);
vs12 = vfmacc_vv_f32m2(vs12, w1, r0, vlm2);
vs22 = vfmacc_vv_f32m2(vs22, w2, r0, vlm2);
r0 = vle32_v_f32m2(rptr + vecsize_aligned*3, vlm2);
vs03 = vfmacc_vv_f32m2(vs03, w0, r0, vlm2);
vs13 = vfmacc_vv_f32m2(vs13, w1, r0, vlm2);
vs23 = vfmacc_vv_f32m2(vs23, w2, r0, vlm2);
}
vfloat32m1_t s0, s1, s2;
if( initOutput )
{
s0 = vfmv_v_f_f32m1(bias0, vl);
s1 = vfmv_v_f_f32m1(bias1, vl);
s2 = vfmv_v_f_f32m1(bias2, vl);
}
else
{
s0 = vle32_v_f32m1(outptr0 + j, vl);
s1 = vle32_v_f32m1(outptr1 + j, vl);
s2 = vle32_v_f32m1(outptr2 + j, vl);
}
// compute sum of each vs
vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl);
vfloat32m1_t temp00 = vfredsum_vs_f32m2_f32m1(temp00, vs00, zero, 8);
vfloat32m1_t temp01 = vfredsum_vs_f32m2_f32m1(temp01, vs01, zero, 8);
vfloat32m1_t temp02 = vfredsum_vs_f32m2_f32m1(temp02, vs02, zero, 8);
vfloat32m1_t temp03 = vfredsum_vs_f32m2_f32m1(temp03, vs03, zero, 8);
vfloat32m1_t temp10 = vfredsum_vs_f32m2_f32m1(temp10, vs10, zero, 8);
vfloat32m1_t temp11 = vfredsum_vs_f32m2_f32m1(temp11, vs11, zero, 8);
vfloat32m1_t temp12 = vfredsum_vs_f32m2_f32m1(temp12, vs12, zero, 8);
vfloat32m1_t temp13 = vfredsum_vs_f32m2_f32m1(temp13, vs13, zero, 8);
vfloat32m1_t temp20 = vfredsum_vs_f32m2_f32m1(temp20, vs20, zero, 8);
vfloat32m1_t temp21 = vfredsum_vs_f32m2_f32m1(temp21, vs21, zero, 8);
vfloat32m1_t temp22 = vfredsum_vs_f32m2_f32m1(temp22, vs22, zero, 8);
vfloat32m1_t temp23 = vfredsum_vs_f32m2_f32m1(temp23, vs23, zero, 8);
float32_t sum0[4], sum1[4], sum2[4];
sum0[0] = vfmv_f_s_f32m1_f32(temp00);
sum0[1] = vfmv_f_s_f32m1_f32(temp01);
sum0[2] = vfmv_f_s_f32m1_f32(temp02);
sum0[3] = vfmv_f_s_f32m1_f32(temp03);
sum1[0] = vfmv_f_s_f32m1_f32(temp10);
sum1[1] = vfmv_f_s_f32m1_f32(temp11);
sum1[2] = vfmv_f_s_f32m1_f32(temp12);
sum1[3] = vfmv_f_s_f32m1_f32(temp13);
sum2[0] = vfmv_f_s_f32m1_f32(temp20);
sum2[1] = vfmv_f_s_f32m1_f32(temp21);
sum2[2] = vfmv_f_s_f32m1_f32(temp22);
sum2[3] = vfmv_f_s_f32m1_f32(temp23);
s0 = vfadd_vv_f32m1(vle32_v_f32m1(sum0, vl), s0, vl);
s1 = vfadd_vv_f32m1(vle32_v_f32m1(sum1, vl), s1, vl);
s2 = vfadd_vv_f32m1(vle32_v_f32m1(sum2, vl), s2, vl);
if( relu )
{
vbool32_t m0 = vmfgt_vf_f32m1_b32(s0, 0, vl);
vbool32_t m1 = vmfgt_vf_f32m1_b32(s1, 0, vl);
vbool32_t m2 = vmfgt_vf_f32m1_b32(s2, 0, vl);
s0 = vmerge_vvm_f32m1(m0, vfmul_vv_f32m1(s0, vr0, vl), s0, vl);
s1 = vmerge_vvm_f32m1(m1, vfmul_vv_f32m1(s1, vr1, vl), s1, vl);
s2 = vmerge_vvm_f32m1(m2, vfmul_vv_f32m1(s2, vr2, vl), s2, vl);
}
if( tail )
{
s0 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr0 + j, vl), s0, vl);
s1 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr1 + j, vl), s1, vl);
s2 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr2 + j, vl), s2, vl);
}
vse32_v_f32m1(outptr0 + j, s0, vl);
vse32_v_f32m1(outptr1 + j, s1, vl);
vse32_v_f32m1(outptr2 + j, s2, vl);
}
}
}
/*
Example for load_deinterleave:
input: ptr[16] = {1,2,3, ... ,14,15,16}
output: a = {1, 3, 5, 7, 9, 11, 13, 15}
output: b = {2, 4, 6, 8,10, 12, 14, 16}
*/
static inline void vfloat32m2_load_deinterleave(const float* ptr, vfloat32m2_t& a, vfloat32m2_t& b)
{
int vl = 8;
uint32_t masks[] = {1,1,1,1,0,0,0,0};
vuint32m2_t vm = vle32_v_u32m2(masks,vl);
vbool16_t mask01 = vmseq_vx_u32m2_b16 (vm, 0, vl);
vbool16_t mask10 = vmseq_vx_u32m2_b16 (vm, 1, vl);
vfloat32m2_t ta = vle32_v_f32m2(ptr, vl), tb = vle32_v_f32m2(ptr+8, vl);
uint idx[] = {0,2,4,6,1,3,5,7};
uint idxa[] = {0,0,0,0,0,1,2,3}, idxb[] = {4,5,6,7,0,0,0,0};
vuint32m2_t vidxa = vle32_v_u32m2(idxa, 8), vidxb = vle32_v_u32m2(idxb, 8);
vuint32m2_t vidx = vle32_v_u32m2(idx, 8);
vfloat32m2_t high = vfmv_v_f_f32m2(0, 8), low = vfmv_v_f_f32m2(0, 8);
high = vrgather_vv_f32m2(ta, vidx, 8);
low = vrgather_vv_f32m2(tb, vidx, 8);
a = vrgather_vv_f32m2_m(mask01, high, low, vidxa, 8);
b = vrgather_vv_f32m2_m(mask10, low, high, vidxb, 8);
}
void fastDepthwiseConv( const float* wptr,
int kernel_h, int kernel_w,
int stride_h, int stride_w,
int dilation_h, int dilation_w,
int pad_t, int pad_l,
const float* biasptr, const float* relu,
const float* inptr_,
int height, int width,
float* outptr_,
int out_d, int outH, int outW )
{
int vl = 8;
const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2],
w10 = wptr[3], w11 = wptr[4], w12 = wptr[5],
w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8];
int outW1 = std::min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w);
float relu_coeff = relu ? relu[out_d] : 1.f, bias = biasptr[out_d];
for (int out_i = 0; out_i < outH; out_i++)
{
int in_i = out_i * stride_h - pad_t, out_j = 0;
const float* imgptr0 = inptr_ + in_i*width;
const float* imgptr1 = imgptr0 + dilation_h*width;
const float* imgptr2 = imgptr0 + (dilation_h*2)*width;
float out, w00 = w00_, w01 = w01_, w02 = w02_;
float w20 = w20_, w21 = w21_, w22 = w22_;
if (in_i < 0)
{
w00 = w01 = w02 = 0.f;
imgptr0 = imgptr1;
}
else if (in_i + dilation_h*(kernel_h-1) >= height)
{
w20 = w21 = w22 = 0.f;
imgptr2 = imgptr1;
}
float* outptr = outptr_ + out_i*outW;
if (pad_l > 0)
{
out = imgptr0[0]*w01 + imgptr0[dilation_w]*w02 +
imgptr1[0]*w11 + imgptr1[dilation_w]*w12 +
imgptr2[0]*w21 + imgptr2[dilation_w]*w22 + bias;
if (relu)
out = out > 0.f ? out : out*relu_coeff;
outptr[0] = out;
out_j = 1;
}
if (stride_w == 1 || (stride_w == 2 && dilation_w == 1))
{
const int VECSZ = 8;
vfloat32m2_t vw00 = vfmv_v_f_f32m2(w00, vl), vw01 = vfmv_v_f_f32m2(w01, vl), vw02 = vfmv_v_f_f32m2(w02, vl),
vw10 = vfmv_v_f_f32m2(w10, vl), vw11 = vfmv_v_f_f32m2(w11, vl), vw12 = vfmv_v_f_f32m2(w12, vl),
vw20 = vfmv_v_f_f32m2(w20, vl), vw21 = vfmv_v_f_f32m2(w21, vl), vw22 = vfmv_v_f_f32m2(w22, vl);
vfloat32m2_t vbias = vfmv_v_f_f32m2(bias, vl), vrc = vfmv_v_f_f32m2(relu_coeff, vl);
if( stride_w == 1 )
for( ; out_j < outW1; out_j += VECSZ )
{
if (out_j + VECSZ > outW1 && out_j > pad_l)
out_j = outW1 - VECSZ;
int in_j = out_j * stride_w - pad_l;
vfloat32m2_t v00 = vle32_v_f32m2(imgptr0 + in_j, vl),
v01 = vle32_v_f32m2(imgptr0 + in_j + dilation_w, vl),
v02 = vle32_v_f32m2(imgptr0 + in_j + dilation_w*2, vl),
v10 = vle32_v_f32m2(imgptr1 + in_j, vl),
v11 = vle32_v_f32m2(imgptr1 + in_j + dilation_w, vl),
v12 = vle32_v_f32m2(imgptr1 + in_j + dilation_w*2, vl),
v20 = vle32_v_f32m2(imgptr2 + in_j, vl),
v21 = vle32_v_f32m2(imgptr2 + in_j + dilation_w, vl),
v22 = vle32_v_f32m2(imgptr2 + in_j + dilation_w*2, vl);
vfloat32m2_t vout0 = vfmacc_vv_f32m2(vbias, v00, vw00, vl);
vfloat32m2_t vout1 = vfmul_vv_f32m2(v01, vw01, vl);
vfloat32m2_t vout2 = vfmul_vv_f32m2(v02, vw02, vl);
vout0 = vfmacc_vv_f32m2(vout0, v10, vw10, vl);
vout1 = vfmacc_vv_f32m2(vout1, v11, vw11, vl);
vout2 = vfmacc_vv_f32m2(vout2, v12, vw12, vl);
vout0 = vfmacc_vv_f32m2(vout0, v20, vw20, vl);
vout1 = vfmacc_vv_f32m2(vout1, v21, vw21, vl);
vout2 = vfmacc_vv_f32m2(vout2, v22, vw22, vl);
vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl);
if (relu)
{
vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl);
vout0 = vmerge_vvm_f32m2(m, vfmul_vv_f32m2(vout0, vrc, vl), vout0, vl);
}
vse32_v_f32m2(outptr + out_j, vout0, vl);
}
else
for( ; out_j < outW1; out_j += VECSZ )
{
if (out_j + VECSZ > outW1 && out_j > pad_l)
out_j = outW1 - VECSZ;
int in_j = out_j * stride_w - pad_l;
vfloat32m2_t v00, v01, v02, v10, v11, v12, v20, v21, v22, unused;
vfloat32m2_load_deinterleave(imgptr0 + in_j, v00, v01);
vfloat32m2_load_deinterleave(imgptr0 + in_j + 2, v02, unused);
vfloat32m2_load_deinterleave(imgptr1 + in_j, v10, v11);
vfloat32m2_load_deinterleave(imgptr1 + in_j + 2, v12, unused);
vfloat32m2_load_deinterleave(imgptr2 + in_j, v20, v21);
vfloat32m2_load_deinterleave(imgptr2 + in_j + 2, v22, unused);
vfloat32m2_t vout0 = vfmacc_vv_f32m2(vbias, v00, vw00, vl);
vfloat32m2_t vout1 = vfmul_vv_f32m2(v01, vw01, vl);
vfloat32m2_t vout2 = vfmul_vv_f32m2(v02, vw02, vl);
vout0 = vfmacc_vv_f32m2(vout0, v10, vw10, vl);
vout1 = vfmacc_vv_f32m2(vout1, v11, vw11, vl);
vout2 = vfmacc_vv_f32m2(vout2, v12, vw12, vl);
vout0 = vfmacc_vv_f32m2(vout0, v20, vw20, vl);
vout1 = vfmacc_vv_f32m2(vout1, v21, vw21, vl);
vout2 = vfmacc_vv_f32m2(vout2, v22, vw22, vl);
vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl);
if (relu)
{
vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl);
vout0 = vmerge_vvm_f32m2(m, vfmul_vv_f32m2(vout0, vrc, vl), vout0, vl);
}
vse32_v_f32m2(outptr + out_j, vout0, vl);
}
}
for (; out_j < outW1; out_j++)
{
int in_j = out_j * stride_w - pad_l;
out = imgptr0[in_j]*w00 + imgptr0[in_j + dilation_w]*w01 + imgptr0[in_j + dilation_w*2]*w02 +
imgptr1[in_j]*w10 + imgptr1[in_j + dilation_w]*w11 + imgptr1[in_j + dilation_w*2]*w12 +
imgptr2[in_j]*w20 + imgptr2[in_j + dilation_w]*w21 + imgptr2[in_j + dilation_w*2]*w22 + bias;
if (relu)
out = out > 0.f ? out : out*relu_coeff;
outptr[out_j] = out;
}
for (; out_j < outW; out_j++ )
{
int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2;
float s0 = 1.f, s1 = 1.f, s2 = 1.f;
if (in_j0 >= width)
{
in_j0 = 0;
s0 = 0.f;
}
if (in_j1 >= width)
{
in_j1 = 0;
s1 = 0.f;
}
if (in_j2 >= width)
{
in_j2 = 0;
s2 = 0.f;
}
out = imgptr0[in_j0]*w00*s0 + imgptr0[in_j1]*w01*s1 + imgptr0[in_j2]*w02*s2 +
imgptr1[in_j0]*w10*s0 + imgptr1[in_j1]*w11*s1 + imgptr1[in_j2]*w12*s2 +
imgptr2[in_j0]*w20*s0 + imgptr2[in_j1]*w21*s1 + imgptr2[in_j2]*w22*s2 + bias;
if (relu)
out = out > 0.f ? out : out*relu_coeff;
outptr[out_j] = out;
}
}
}
#endif // CV_RVV
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // namespace

Write Preview
Loading…
Cancel
Save