|
|
|
@ -34,14 +34,17 @@ |
|
|
|
|
|
|
|
|
|
void ff_get_pixels_mmx(int16_t *block, const uint8_t *pixels, int line_size); |
|
|
|
|
void ff_get_pixels_sse2(int16_t *block, const uint8_t *pixels, int line_size); |
|
|
|
|
void ff_diff_pixels_mmx(int16_t *block, const uint8_t *s1, const uint8_t *s2, int stride); |
|
|
|
|
void ff_diff_pixels_mmx(int16_t *block, const uint8_t *s1, const uint8_t *s2, |
|
|
|
|
int stride); |
|
|
|
|
int ff_pix_sum16_mmx(uint8_t *pix, int line_size); |
|
|
|
|
int ff_pix_norm1_mmx(uint8_t *pix, int line_size); |
|
|
|
|
|
|
|
|
|
#if HAVE_INLINE_ASM |
|
|
|
|
|
|
|
|
|
static int sse8_mmx(void *v, uint8_t * pix1, uint8_t * pix2, int line_size, int h) { |
|
|
|
|
static int sse8_mmx(void *v, uint8_t *pix1, uint8_t *pix2, int line_size, int h) |
|
|
|
|
{ |
|
|
|
|
int tmp; |
|
|
|
|
|
|
|
|
|
__asm__ volatile ( |
|
|
|
|
"movl %4, %%ecx \n" |
|
|
|
|
"shr $1, %%ecx \n" |
|
|
|
@ -98,11 +101,15 @@ static int sse8_mmx(void *v, uint8_t * pix1, uint8_t * pix2, int line_size, int |
|
|
|
|
: "+r" (pix1), "+r" (pix2), "=r" (tmp) |
|
|
|
|
: "r" ((x86_reg) line_size), "m" (h) |
|
|
|
|
: "%ecx"); |
|
|
|
|
|
|
|
|
|
return tmp; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static int sse16_mmx(void *v, uint8_t * pix1, uint8_t * pix2, int line_size, int h) { |
|
|
|
|
static int sse16_mmx(void *v, uint8_t *pix1, uint8_t *pix2, |
|
|
|
|
int line_size, int h) |
|
|
|
|
{ |
|
|
|
|
int tmp; |
|
|
|
|
|
|
|
|
|
__asm__ volatile ( |
|
|
|
|
"movl %4, %%ecx\n" |
|
|
|
|
"pxor %%mm0, %%mm0\n" /* mm0 = 0 */ |
|
|
|
@ -158,11 +165,14 @@ static int sse16_mmx(void *v, uint8_t * pix1, uint8_t * pix2, int line_size, int |
|
|
|
|
: "+r" (pix1), "+r" (pix2), "=r" (tmp) |
|
|
|
|
: "r" ((x86_reg) line_size), "m" (h) |
|
|
|
|
: "%ecx"); |
|
|
|
|
|
|
|
|
|
return tmp; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static int hf_noise8_mmx(uint8_t * pix1, int line_size, int h) { |
|
|
|
|
static int hf_noise8_mmx(uint8_t *pix1, int line_size, int h) |
|
|
|
|
{ |
|
|
|
|
int tmp; |
|
|
|
|
|
|
|
|
|
__asm__ volatile ( |
|
|
|
|
"movl %3, %%ecx\n" |
|
|
|
|
"pxor %%mm7, %%mm7\n" |
|
|
|
@ -283,12 +293,15 @@ static int hf_noise8_mmx(uint8_t * pix1, int line_size, int h) { |
|
|
|
|
: "+r" (pix1), "=r" (tmp) |
|
|
|
|
: "r" ((x86_reg) line_size), "g" (h - 2) |
|
|
|
|
: "%ecx"); |
|
|
|
|
|
|
|
|
|
return tmp; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static int hf_noise16_mmx(uint8_t * pix1, int line_size, int h) { |
|
|
|
|
static int hf_noise16_mmx(uint8_t *pix1, int line_size, int h) |
|
|
|
|
{ |
|
|
|
|
int tmp; |
|
|
|
|
uint8_t *pix = pix1; |
|
|
|
|
|
|
|
|
|
__asm__ volatile ( |
|
|
|
|
"movl %3, %%ecx\n" |
|
|
|
|
"pxor %%mm7, %%mm7\n" |
|
|
|
@ -397,31 +410,46 @@ static int hf_noise16_mmx(uint8_t * pix1, int line_size, int h) { |
|
|
|
|
: "+r" (pix1), "=r" (tmp) |
|
|
|
|
: "r" ((x86_reg) line_size), "g" (h - 2) |
|
|
|
|
: "%ecx"); |
|
|
|
|
|
|
|
|
|
return tmp + hf_noise8_mmx(pix + 8, line_size, h); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static int nsse16_mmx(void *p, uint8_t * pix1, uint8_t * pix2, int line_size, int h) { |
|
|
|
|
static int nsse16_mmx(void *p, uint8_t *pix1, uint8_t *pix2, |
|
|
|
|
int line_size, int h) |
|
|
|
|
{ |
|
|
|
|
MpegEncContext *c = p; |
|
|
|
|
int score1, score2; |
|
|
|
|
|
|
|
|
|
if(c) score1 = c->dsp.sse[0](c, pix1, pix2, line_size, h); |
|
|
|
|
else score1 = sse16_mmx(c, pix1, pix2, line_size, h); |
|
|
|
|
score2= hf_noise16_mmx(pix1, line_size, h) - hf_noise16_mmx(pix2, line_size, h); |
|
|
|
|
|
|
|
|
|
if(c) return score1 + FFABS(score2)*c->avctx->nsse_weight; |
|
|
|
|
else return score1 + FFABS(score2)*8; |
|
|
|
|
if (c) |
|
|
|
|
score1 = c->dsp.sse[0](c, pix1, pix2, line_size, h); |
|
|
|
|
else |
|
|
|
|
score1 = sse16_mmx(c, pix1, pix2, line_size, h); |
|
|
|
|
score2 = hf_noise16_mmx(pix1, line_size, h) - |
|
|
|
|
hf_noise16_mmx(pix2, line_size, h); |
|
|
|
|
|
|
|
|
|
if (c) |
|
|
|
|
return score1 + FFABS(score2) * c->avctx->nsse_weight; |
|
|
|
|
else |
|
|
|
|
return score1 + FFABS(score2) * 8; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static int nsse8_mmx(void *p, uint8_t * pix1, uint8_t * pix2, int line_size, int h) { |
|
|
|
|
static int nsse8_mmx(void *p, uint8_t *pix1, uint8_t *pix2, |
|
|
|
|
int line_size, int h) |
|
|
|
|
{ |
|
|
|
|
MpegEncContext *c = p; |
|
|
|
|
int score1 = sse8_mmx(c, pix1, pix2, line_size, h); |
|
|
|
|
int score2= hf_noise8_mmx(pix1, line_size, h) - hf_noise8_mmx(pix2, line_size, h); |
|
|
|
|
int score2 = hf_noise8_mmx(pix1, line_size, h) - |
|
|
|
|
hf_noise8_mmx(pix2, line_size, h); |
|
|
|
|
|
|
|
|
|
if(c) return score1 + FFABS(score2)*c->avctx->nsse_weight; |
|
|
|
|
else return score1 + FFABS(score2)*8; |
|
|
|
|
if (c) |
|
|
|
|
return score1 + FFABS(score2) * c->avctx->nsse_weight; |
|
|
|
|
else |
|
|
|
|
return score1 + FFABS(score2) * 8; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static int vsad_intra16_mmx(void *v, uint8_t * pix, uint8_t * dummy, int line_size, int h) { |
|
|
|
|
static int vsad_intra16_mmx(void *v, uint8_t *pix, uint8_t *dummy, |
|
|
|
|
int line_size, int h) |
|
|
|
|
{ |
|
|
|
|
int tmp; |
|
|
|
|
|
|
|
|
|
av_assert2((((int) pix) & 7) == 0); |
|
|
|
@ -478,6 +506,7 @@ static int vsad_intra16_mmx(void *v, uint8_t * pix, uint8_t * dummy, int line_si |
|
|
|
|
: "+r" (pix), "=r" (tmp) |
|
|
|
|
: "r" ((x86_reg) line_size), "m" (h) |
|
|
|
|
: "%ecx"); |
|
|
|
|
|
|
|
|
|
return tmp & 0xFFFF; |
|
|
|
|
} |
|
|
|
|
#undef SUM |
|
|
|
@ -520,11 +549,14 @@ static int vsad_intra16_mmxext(void *v, uint8_t *pix, uint8_t *dummy, |
|
|
|
|
: "+r" (pix), "=r" (tmp) |
|
|
|
|
: "r" ((x86_reg) line_size), "m" (h) |
|
|
|
|
: "%ecx"); |
|
|
|
|
|
|
|
|
|
return tmp; |
|
|
|
|
} |
|
|
|
|
#undef SUM |
|
|
|
|
|
|
|
|
|
static int vsad16_mmx(void *v, uint8_t * pix1, uint8_t * pix2, int line_size, int h) { |
|
|
|
|
static int vsad16_mmx(void *v, uint8_t *pix1, uint8_t *pix2, |
|
|
|
|
int line_size, int h) |
|
|
|
|
{ |
|
|
|
|
int tmp; |
|
|
|
|
|
|
|
|
|
av_assert2((((int) pix1) & 7) == 0); |
|
|
|
@ -598,6 +630,7 @@ static int vsad16_mmx(void *v, uint8_t * pix1, uint8_t * pix2, int line_size, in |
|
|
|
|
: "+r" (pix1), "+r" (pix2), "=r" (tmp) |
|
|
|
|
: "r" ((x86_reg) line_size), "m" (h) |
|
|
|
|
: "%ecx"); |
|
|
|
|
|
|
|
|
|
return tmp & 0x7FFF; |
|
|
|
|
} |
|
|
|
|
#undef SUM |
|
|
|
@ -657,12 +690,15 @@ static int vsad16_mmxext(void *v, uint8_t *pix1, uint8_t *pix2, |
|
|
|
|
: "+r" (pix1), "+r" (pix2), "=r" (tmp) |
|
|
|
|
: "r" ((x86_reg) line_size), "m" (h) |
|
|
|
|
: "%ecx"); |
|
|
|
|
|
|
|
|
|
return tmp; |
|
|
|
|
} |
|
|
|
|
#undef SUM |
|
|
|
|
|
|
|
|
|
static void diff_bytes_mmx(uint8_t *dst, const uint8_t *src1, const uint8_t *src2, int w){ |
|
|
|
|
static void diff_bytes_mmx(uint8_t *dst, const uint8_t *src1, const uint8_t *src2, int w) |
|
|
|
|
{ |
|
|
|
|
x86_reg i = 0; |
|
|
|
|
|
|
|
|
|
if (w >= 16) |
|
|
|
|
__asm__ volatile ( |
|
|
|
|
"1: \n\t" |
|
|
|
@ -678,8 +714,8 @@ static void diff_bytes_mmx(uint8_t *dst, const uint8_t *src1, const uint8_t *src |
|
|
|
|
"cmp %4, %0 \n\t" |
|
|
|
|
" jb 1b \n\t" |
|
|
|
|
: "+r" (i) |
|
|
|
|
: "r"(src1), "r"(src2), "r"(dst), "r"((x86_reg)w-15) |
|
|
|
|
); |
|
|
|
|
: "r" (src1), "r" (src2), "r" (dst), "r" ((x86_reg) w - 15)); |
|
|
|
|
|
|
|
|
|
for (; i < w; i++) |
|
|
|
|
dst[i + 0] = src1[i + 0] - src2[i + 0]; |
|
|
|
|
} |
|
|
|
@ -713,8 +749,7 @@ static void sub_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *src1, |
|
|
|
|
"cmp %4, %0 \n\t" |
|
|
|
|
" jb 1b \n\t" |
|
|
|
|
: "+r" (i) |
|
|
|
|
: "r"(src1), "r"(src2), "r"(dst), "r"((x86_reg)w) |
|
|
|
|
); |
|
|
|
|
: "r" (src1), "r" (src2), "r" (dst), "r" ((x86_reg) w)); |
|
|
|
|
|
|
|
|
|
l = *left; |
|
|
|
|
lt = *left_top; |
|
|
|
@ -743,9 +778,10 @@ static void sub_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *src1, |
|
|
|
|
MMABS(a,z) \
|
|
|
|
|
"paddusw " #a ", " #sum " \n\t" |
|
|
|
|
|
|
|
|
|
/* FIXME: HSUM_* saturates at 64k, while an 8x8 hadamard or dct block can get up to
|
|
|
|
|
* about 100k on extreme inputs. But that's very unlikely to occur in natural video, |
|
|
|
|
* and it's even more unlikely to not have any alternative mvs/modes with lower cost. */ |
|
|
|
|
/* FIXME: HSUM_* saturates at 64k, while an 8x8 hadamard or dct block can get
|
|
|
|
|
* up to about 100k on extreme inputs. But that's very unlikely to occur in |
|
|
|
|
* natural video, and it's even more unlikely to not have any alternative |
|
|
|
|
* mvs/modes with lower cost. */ |
|
|
|
|
#define HSUM_MMX(a, t, dst) \ |
|
|
|
|
"movq " #a ", " #t " \n\t" \
|
|
|
|
|
"psrlq $32, " #a " \n\t" \
|
|
|
|
@ -800,13 +836,13 @@ static void sub_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *src1, |
|
|
|
|
HSUM(%%xmm0, %%xmm1, %0) |
|
|
|
|
|
|
|
|
|
#define DCT_SAD_FUNC(cpu) \ |
|
|
|
|
static int sum_abs_dctelem_##cpu(int16_t *block){\
|
|
|
|
|
static int sum_abs_dctelem_ ## cpu(int16_t *block) \
|
|
|
|
|
{ \
|
|
|
|
|
int sum; \
|
|
|
|
|
__asm__ volatile ( \
|
|
|
|
|
DCT_SAD \
|
|
|
|
|
:"=r"(sum) \
|
|
|
|
|
:"r"(block)\
|
|
|
|
|
);\
|
|
|
|
|
:"r"(block)); \
|
|
|
|
|
return sum & 0xFFFF; \
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -836,9 +872,12 @@ DCT_SAD_FUNC(ssse3) |
|
|
|
|
#undef HSUM |
|
|
|
|
#undef DCT_SAD |
|
|
|
|
|
|
|
|
|
static int ssd_int8_vs_int16_mmx(const int8_t *pix1, const int16_t *pix2, int size){ |
|
|
|
|
static int ssd_int8_vs_int16_mmx(const int8_t *pix1, const int16_t *pix2, |
|
|
|
|
int size) |
|
|
|
|
{ |
|
|
|
|
int sum; |
|
|
|
|
x86_reg i = size; |
|
|
|
|
|
|
|
|
|
__asm__ volatile ( |
|
|
|
|
"pxor %%mm4, %%mm4 \n" |
|
|
|
|
"1: \n" |
|
|
|
@ -862,8 +901,8 @@ static int ssd_int8_vs_int16_mmx(const int8_t *pix1, const int16_t *pix2, int si |
|
|
|
|
"paddd %%mm3, %%mm4 \n" |
|
|
|
|
"movd %%mm4, %1 \n" |
|
|
|
|
: "+r" (i), "=r" (sum) |
|
|
|
|
:"r"(pix1), "r"(pix2) |
|
|
|
|
); |
|
|
|
|
: "r" (pix1), "r" (pix2)); |
|
|
|
|
|
|
|
|
|
return sum; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -871,10 +910,11 @@ static int ssd_int8_vs_int16_mmx(const int8_t *pix1, const int16_t *pix2, int si |
|
|
|
|
"movq " #a ", " #t " \n\t" \
|
|
|
|
|
"psrlq $32, " #a " \n\t" \
|
|
|
|
|
"paddd " #t ", " #a " \n\t" |
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
pmulhw: dst[0-15]=(src[0-15]*dst[0-15])[16-31] |
|
|
|
|
pmulhrw: dst[0-15]=(src[0-15]*dst[0-15] + 0x8000)[16-31] |
|
|
|
|
pmulhrsw: dst[0-15]=(src[0-15]*dst[0-15] + 0x4000)[15-30] |
|
|
|
|
* pmulhw: dst[0 - 15] = (src[0 - 15] * dst[0 - 15])[16 - 31] |
|
|
|
|
* pmulhrw: dst[0 - 15] = (src[0 - 15] * dst[0 - 15] + 0x8000)[16 - 31] |
|
|
|
|
* pmulhrsw: dst[0 - 15] = (src[0 - 15] * dst[0 - 15] + 0x4000)[15 - 30] |
|
|
|
|
*/ |
|
|
|
|
#define PMULHRW(x, y, s, o) \ |
|
|
|
|
"pmulhw " #s ", " #x " \n\t" \
|
|
|
|
@ -913,9 +953,12 @@ static int ssd_int8_vs_int16_mmx(const int8_t *pix1, const int16_t *pix2, int si |
|
|
|
|
#define DEF(x) x ## _ssse3 |
|
|
|
|
#define SET_RND(x) |
|
|
|
|
#define SCALE_OFFSET -1 |
|
|
|
|
|
|
|
|
|
#define PHADDD(a, t) \ |
|
|
|
|
"pshufw $0x0E, " #a ", " #t " \n\t" \
|
|
|
|
|
"paddd "#t", "#a" \n\t" /* faster than phaddd on core2 */ |
|
|
|
|
/* faster than phaddd on core2 */ \
|
|
|
|
|
"paddd " #t ", " #a " \n\t" |
|
|
|
|
|
|
|
|
|
#define PMULHRW(x, y, s, o) \ |
|
|
|
|
"pmulhrsw " #s ", " #x " \n\t" \
|
|
|
|
|
"pmulhrsw " #s ", " #y " \n\t" |
|
|
|
@ -957,9 +1000,9 @@ av_cold void ff_dsputilenc_init_mmx(DSPContext *c, AVCodecContext *avctx) |
|
|
|
|
c->get_pixels = ff_get_pixels_mmx; |
|
|
|
|
c->diff_pixels = ff_diff_pixels_mmx; |
|
|
|
|
c->pix_sum = ff_pix_sum16_mmx; |
|
|
|
|
|
|
|
|
|
c->pix_norm1 = ff_pix_norm1_mmx; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (EXTERNAL_SSE2(cpu_flags)) |
|
|
|
|
if (bit_depth <= 8) |
|
|
|
|
c->get_pixels = ff_get_pixels_sse2; |
|
|
|
|