|
|
@ -82,25 +82,43 @@ kernel void integral_sum_cols(__global uchar4 *src, __global uchar *sum_ptr, |
|
|
|
__local sumT* sum_p; |
|
|
|
__local sumT* sum_p; |
|
|
|
src_step = src_step >> 2; |
|
|
|
src_step = src_step >> 2; |
|
|
|
gid = gid << 1; |
|
|
|
gid = gid << 1; |
|
|
|
for (int i = 0; i < rows; i =i + LSIZE_1) |
|
|
|
int lid_prim = ((lid & 127) << 1) + 1; |
|
|
|
|
|
|
|
for (int i = 0; i < rows; i += LSIZE_1) |
|
|
|
{ |
|
|
|
{ |
|
|
|
src_t[0] = (i + lid < rows ? convertToSum4(src[mad24((lid+i), src_step, src_offset + gid)]) : (vecSumT)0); |
|
|
|
if (i + lid < rows) |
|
|
|
src_t[1] = (i + lid < rows ? convertToSum4(src[mad24((lid+i), src_step, src_offset + gid + 1)]) : (vecSumT)0); |
|
|
|
{ |
|
|
|
|
|
|
|
int src_index = mad24((lid+i), src_step, gid + src_offset); |
|
|
|
|
|
|
|
src_t[0] = convertToSum4(src[src_index]); |
|
|
|
|
|
|
|
src_t[1] = convertToSum4(src[src_index + 1]); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
else |
|
|
|
|
|
|
|
{ |
|
|
|
|
|
|
|
src_t[0] = (vecSumT)0; |
|
|
|
|
|
|
|
src_t[1] = (vecSumT)0; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
sum_t[0] = (i == 0 ? (vecSumT)0 : lm_sum[0][LSIZE_2 + LOG_LSIZE]); |
|
|
|
if (i == 0) |
|
|
|
sum_t[1] = (i == 0 ? (vecSumT)0 : lm_sum[1][LSIZE_2 + LOG_LSIZE]); |
|
|
|
{ |
|
|
|
|
|
|
|
sum_t[0] = (vecSumT)0; |
|
|
|
|
|
|
|
sum_t[1] = (vecSumT)0; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
else |
|
|
|
|
|
|
|
{ |
|
|
|
|
|
|
|
sum_t[0] = lm_sum[0][LSIZE_2 + LOG_LSIZE]; |
|
|
|
|
|
|
|
sum_t[1] = lm_sum[1][LSIZE_2 + LOG_LSIZE]; |
|
|
|
|
|
|
|
} |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
|
|
|
|
|
|
|
|
int bf_loc = lid + GET_CONFLICT_OFFSET(lid); |
|
|
|
int bf_loc = lid + GET_CONFLICT_OFFSET(lid); |
|
|
|
lm_sum[0][bf_loc] = src_t[0]; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lm_sum[0][bf_loc] = src_t[0]; |
|
|
|
lm_sum[1][bf_loc] = src_t[1]; |
|
|
|
lm_sum[1][bf_loc] = src_t[1]; |
|
|
|
|
|
|
|
|
|
|
|
int offset = 1; |
|
|
|
int offset = 1; |
|
|
|
for (int d = LSIZE >> 1 ; d > 0; d>>=1) |
|
|
|
for (int d = LSIZE >> 1 ; d > 0; d>>=1) |
|
|
|
{ |
|
|
|
{ |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; |
|
|
|
int ai = offset * lid_prim - 1,bi = ai + offset; |
|
|
|
ai += GET_CONFLICT_OFFSET(ai); |
|
|
|
ai += GET_CONFLICT_OFFSET(ai); |
|
|
|
bi += GET_CONFLICT_OFFSET(bi); |
|
|
|
bi += GET_CONFLICT_OFFSET(bi); |
|
|
|
|
|
|
|
|
|
|
@ -119,7 +137,7 @@ kernel void integral_sum_cols(__global uchar4 *src, __global uchar *sum_ptr, |
|
|
|
{ |
|
|
|
{ |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
offset >>= 1; |
|
|
|
offset >>= 1; |
|
|
|
int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; |
|
|
|
int ai = offset * lid_prim - 1,bi = ai + offset; |
|
|
|
ai += GET_CONFLICT_OFFSET(ai); |
|
|
|
ai += GET_CONFLICT_OFFSET(ai); |
|
|
|
bi += GET_CONFLICT_OFFSET(bi); |
|
|
|
bi += GET_CONFLICT_OFFSET(bi); |
|
|
|
|
|
|
|
|
|
|
@ -138,13 +156,15 @@ kernel void integral_sum_cols(__global uchar4 *src, __global uchar *sum_ptr, |
|
|
|
sum_p = (__local sumT*)(&(lm_sum[0][bf_loc])); |
|
|
|
sum_p = (__local sumT*)(&(lm_sum[0][bf_loc])); |
|
|
|
for (int k = 0; k < 4; k++) |
|
|
|
for (int k = 0; k < 4; k++) |
|
|
|
{ |
|
|
|
{ |
|
|
|
if(gid * 4 + k >= cols) continue; |
|
|
|
if (gid * 4 + k >= cols) |
|
|
|
|
|
|
|
break; |
|
|
|
sum[loc_s0 + k * dst_step / 4] = sum_p[k]; |
|
|
|
sum[loc_s0 + k * dst_step / 4] = sum_p[k]; |
|
|
|
} |
|
|
|
} |
|
|
|
sum_p = (__local sumT*)(&(lm_sum[1][bf_loc])); |
|
|
|
sum_p = (__local sumT*)(&(lm_sum[1][bf_loc])); |
|
|
|
for (int k = 0; k < 4; k++) |
|
|
|
for (int k = 0; k < 4; k++) |
|
|
|
{ |
|
|
|
{ |
|
|
|
if(gid * 4 + k + 4 >= cols) break; |
|
|
|
if (gid * 4 + k + 4 >= cols) |
|
|
|
|
|
|
|
break; |
|
|
|
sum[loc_s1 + k * dst_step / 4] = sum_p[k]; |
|
|
|
sum[loc_s1 + k * dst_step / 4] = sum_p[k]; |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
@ -152,6 +172,7 @@ kernel void integral_sum_cols(__global uchar4 *src, __global uchar *sum_ptr, |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
kernel void integral_sum_rows(__global uchar *srcsum_ptr, __global uchar *sum_ptr, |
|
|
|
kernel void integral_sum_rows(__global uchar *srcsum_ptr, __global uchar *sum_ptr, |
|
|
|
int rows, int cols, int src_step, int sum_step, int sum_offset) |
|
|
|
int rows, int cols, int src_step, int sum_step, int sum_offset) |
|
|
|
{ |
|
|
|
{ |
|
|
@ -163,25 +184,42 @@ kernel void integral_sum_rows(__global uchar *srcsum_ptr, __global uchar *sum_pt |
|
|
|
__local vecSumT lm_sum[2][LSIZE + LOG_LSIZE]; |
|
|
|
__local vecSumT lm_sum[2][LSIZE + LOG_LSIZE]; |
|
|
|
__local sumT *sum_p; |
|
|
|
__local sumT *sum_p; |
|
|
|
src_step = src_step >> 4; |
|
|
|
src_step = src_step >> 4; |
|
|
|
for (int i = 0; i < rows; i =i + LSIZE_1) |
|
|
|
int lid_prim = ((lid & 127) << 1) + 1; |
|
|
|
|
|
|
|
for (int i = 0; i < rows; i += LSIZE_1) |
|
|
|
{ |
|
|
|
{ |
|
|
|
src_t[0] = i + lid < rows ? srcsum[mad24((lid+i), src_step, gid * 2)] : 0; |
|
|
|
if (i + lid < rows) |
|
|
|
src_t[1] = i + lid < rows ? srcsum[mad24((lid+i), src_step, gid * 2 + 1)] : 0; |
|
|
|
{ |
|
|
|
|
|
|
|
int sum_idx = mad24(lid + i, src_step, gid * 2); |
|
|
|
sum_t[0] = (i == 0 ? 0 : lm_sum[0][LSIZE_2 + LOG_LSIZE]); |
|
|
|
src_t[0] = srcsum[sum_idx]; |
|
|
|
sum_t[1] = (i == 0 ? 0 : lm_sum[1][LSIZE_2 + LOG_LSIZE]); |
|
|
|
src_t[1] = srcsum[sum_idx + 1]; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
else |
|
|
|
|
|
|
|
{ |
|
|
|
|
|
|
|
src_t[0] = 0; |
|
|
|
|
|
|
|
src_t[1] = 0; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
if (i == 0) |
|
|
|
|
|
|
|
{ |
|
|
|
|
|
|
|
sum_t[0] = 0; |
|
|
|
|
|
|
|
sum_t[1] = 0; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
else |
|
|
|
|
|
|
|
{ |
|
|
|
|
|
|
|
sum_t[0] = lm_sum[0][LSIZE_2 + LOG_LSIZE]; |
|
|
|
|
|
|
|
sum_t[1] = lm_sum[1][LSIZE_2 + LOG_LSIZE]; |
|
|
|
|
|
|
|
} |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
|
|
|
|
|
|
|
|
int bf_loc = lid + GET_CONFLICT_OFFSET(lid); |
|
|
|
int bf_loc = lid + GET_CONFLICT_OFFSET(lid); |
|
|
|
lm_sum[0][bf_loc] = src_t[0]; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lm_sum[0][bf_loc] = src_t[0]; |
|
|
|
lm_sum[1][bf_loc] = src_t[1]; |
|
|
|
lm_sum[1][bf_loc] = src_t[1]; |
|
|
|
|
|
|
|
|
|
|
|
int offset = 1; |
|
|
|
int offset = 1; |
|
|
|
for (int d = LSIZE >> 1 ; d > 0; d>>=1) |
|
|
|
for (int d = LSIZE >> 1 ; d > 0; d>>=1) |
|
|
|
{ |
|
|
|
{ |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; |
|
|
|
int ai = offset * lid_prim - 1, bi = ai + offset; |
|
|
|
ai += GET_CONFLICT_OFFSET(ai); |
|
|
|
ai += GET_CONFLICT_OFFSET(ai); |
|
|
|
bi += GET_CONFLICT_OFFSET(bi); |
|
|
|
bi += GET_CONFLICT_OFFSET(bi); |
|
|
|
|
|
|
|
|
|
|
@ -200,11 +238,11 @@ kernel void integral_sum_rows(__global uchar *srcsum_ptr, __global uchar *sum_pt |
|
|
|
{ |
|
|
|
{ |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
barrier(CLK_LOCAL_MEM_FENCE); |
|
|
|
offset >>= 1; |
|
|
|
offset >>= 1; |
|
|
|
int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; |
|
|
|
int ai = offset * lid_prim - 1,bi = ai + offset; |
|
|
|
ai += GET_CONFLICT_OFFSET(ai); |
|
|
|
ai += GET_CONFLICT_OFFSET(ai); |
|
|
|
bi += GET_CONFLICT_OFFSET(bi); |
|
|
|
bi += GET_CONFLICT_OFFSET(bi); |
|
|
|
|
|
|
|
|
|
|
|
if((lid & 127) < d) |
|
|
|
if ((lid & 127) < d) |
|
|
|
{ |
|
|
|
{ |
|
|
|
lm_sum[lid >> 7][bi] += lm_sum[lid >> 7][ai]; |
|
|
|
lm_sum[lid >> 7][bi] += lm_sum[lid >> 7][ai]; |
|
|
|
lm_sum[lid >> 7][ai] = lm_sum[lid >> 7][bi] - lm_sum[lid >> 7][ai]; |
|
|
|
lm_sum[lid >> 7][ai] = lm_sum[lid >> 7][bi] - lm_sum[lid >> 7][ai]; |
|
|
@ -220,7 +258,8 @@ kernel void integral_sum_rows(__global uchar *srcsum_ptr, __global uchar *sum_pt |
|
|
|
int loc0 = gid * 2 * sum_step; |
|
|
|
int loc0 = gid * 2 * sum_step; |
|
|
|
for(int k = 1; k <= 8; k++) |
|
|
|
for(int k = 1; k <= 8; k++) |
|
|
|
{ |
|
|
|
{ |
|
|
|
if(gid * 8 + k > cols) break; |
|
|
|
if (gid * 8 + k > cols) |
|
|
|
|
|
|
|
break; |
|
|
|
sum[sum_offset + loc0 + k * sum_step / 4] = 0; |
|
|
|
sum[sum_offset + loc0 + k * sum_step / 4] = 0; |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
@ -233,13 +272,15 @@ kernel void integral_sum_rows(__global uchar *srcsum_ptr, __global uchar *sum_pt |
|
|
|
sum_p = (__local sumT*)(&(lm_sum[0][bf_loc])); |
|
|
|
sum_p = (__local sumT*)(&(lm_sum[0][bf_loc])); |
|
|
|
for(int k = 0; k < 4; k++) |
|
|
|
for(int k = 0; k < 4; k++) |
|
|
|
{ |
|
|
|
{ |
|
|
|
if(gid * 8 + k >= cols) break; |
|
|
|
if (gid * 8 + k >= cols) |
|
|
|
|
|
|
|
break; |
|
|
|
sum[loc_s0 + k * sum_step / 4] = sum_p[k]; |
|
|
|
sum[loc_s0 + k * sum_step / 4] = sum_p[k]; |
|
|
|
} |
|
|
|
} |
|
|
|
sum_p = (__local sumT*)(&(lm_sum[1][bf_loc])); |
|
|
|
sum_p = (__local sumT*)(&(lm_sum[1][bf_loc])); |
|
|
|
for(int k = 0; k < 4; k++) |
|
|
|
for(int k = 0; k < 4; k++) |
|
|
|
{ |
|
|
|
{ |
|
|
|
if(gid * 8 + 4 + k >= cols) break; |
|
|
|
if (gid * 8 + 4 + k >= cols) |
|
|
|
|
|
|
|
break; |
|
|
|
sum[loc_s1 + k * sum_step / 4] = sum_p[k]; |
|
|
|
sum[loc_s1 + k * sum_step / 4] = sum_p[k]; |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|