From 2025d8026fe507c3f82c3440ecdf09641c577163 Mon Sep 17 00:00:00 2001 From: James Almer Date: Thu, 20 Mar 2014 15:37:54 -0300 Subject: [PATCH 1/3] x86/synth_filter: add synth_filter_sse Build only on x86_32 targets. Signed-off-by: James Almer Signed-off-by: Anton Khirnov --- libavcodec/x86/dcadsp.asm | 55 +++++++++++++++++++++++++++--------- libavcodec/x86/dcadsp_init.c | 45 ++++++++++++++++++----------- 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/libavcodec/x86/dcadsp.asm b/libavcodec/x86/dcadsp.asm index 56039baa6f..970ec3dacb 100644 --- a/libavcodec/x86/dcadsp.asm +++ b/libavcodec/x86/dcadsp.asm @@ -199,15 +199,31 @@ INIT_XMM sse DCA_LFE_FIR 0 DCA_LFE_FIR 1 -INIT_XMM sse2 +%macro SETZERO 1 +%if cpuflag(sse2) + pxor %1, %1 +%else + xorps %1, %1, %1 +%endif +%endmacro + +%macro SHUF 2 +%if cpuflag(sse2) + pshufd %1, %2, q0123 +%else + mova %1, %2 + shufps %1, %1, q0123 +%endif +%endmacro + %macro INNER_LOOP 1 ; reading backwards: ptr1 = synth_buf + j + i; ptr2 = synth_buf + j - i ;~ a += window[i + j] * (-synth_buf[15 - i + j]) ;~ b += window[i + j + 16] * (synth_buf[i + j]) - pshufd m5, [ptr2 + j + (15 - 3) * 4], q0123 + SHUF m5, [ptr2 + j + (15 - 3) * 4] mova m6, [ptr1 + j] %if ARCH_X86_64 - pshufd m11, [ptr2 + j + (15 - 3) * 4 - mmsize], q0123 + SHUF m11, [ptr2 + j + (15 - 3) * 4 - mmsize] mova m12, [ptr1 + j + mmsize] %endif mulps m6, [win + %1 + j + 16 * 4] @@ -224,10 +240,10 @@ INIT_XMM sse2 %endif ;~ c += window[i + j + 32] * (synth_buf[16 + i + j]) ;~ d += window[i + j + 48] * (synth_buf[31 - i + j]) - pshufd m6, [ptr2 + j + (31 - 3) * 4], q0123 + SHUF m6, [ptr2 + j + (31 - 3) * 4] mova m5, [ptr1 + j + 16 * 4] %if ARCH_X86_64 - pshufd m12, [ptr2 + j + (31 - 3) * 4 - mmsize], q0123 + SHUF m12, [ptr2 + j + (31 - 3) * 4 - mmsize] mova m11, [ptr1 + j + mmsize + 16 * 4] %endif mulps m5, [win + %1 + j + 32 * 4] @@ -245,20 +261,25 @@ INIT_XMM sse2 sub j, 64 * 4 %endmacro -; void ff_synth_filter_inner_sse2(float *synth_buf, float synth_buf2[32], -; const float window[512], float out[32], -; intptr_t offset, float scale) +; void ff_synth_filter_inner_(float *synth_buf, float synth_buf2[32], +; const float window[512], float out[32], +; intptr_t offset, float scale) +%macro SYNTH_FILTER 0 cglobal synth_filter_inner, 0, 6 + 4 * ARCH_X86_64, 7 + 6 * ARCH_X86_64, \ synth_buf, synth_buf2, window, out, off, scale %define scale m0 %if ARCH_X86_32 || WIN64 +%if cpuflag(sse2) movd scale, scalem +%else + movss scale, scalem +%endif ; Make sure offset is in a register and not on the stack %define OFFQ r4q %else %define OFFQ offq %endif - pshufd m0, m0, 0 + SPLATD m0 ; prepare inner counter limit 1 mov r5q, 480 sub r5q, offmp @@ -274,8 +295,8 @@ cglobal synth_filter_inner, 0, 6 + 4 * ARCH_X86_64, 7 + 6 * ARCH_X86_64, \ %endif .mainloop ; m1 = a m2 = b m3 = c m4 = d - pxor m3, m3 - pxor m4, m4 + SETZERO m3 + SETZERO m4 mova m1, [buf2 + i] mova m2, [buf2 + i + 16 * 4] %if ARCH_X86_32 @@ -292,8 +313,8 @@ cglobal synth_filter_inner, 0, 6 + 4 * ARCH_X86_64, 7 + 6 * ARCH_X86_64, \ %define ptr2 r7q ; must be loaded %define win r8q %define j r9q - pxor m9, m9 - pxor m10, m10 + SETZERO m9 + SETZERO m10 mova m7, [buf2 + i + mmsize] mova m8, [buf2 + i + mmsize + 16 * 4] lea win, [windowq + i] @@ -350,3 +371,11 @@ cglobal synth_filter_inner, 0, 6 + 4 * ARCH_X86_64, 7 + 6 * ARCH_X86_64, \ sub i, (ARCH_X86_64 + 1) * mmsize jge .mainloop RET +%endmacro + +%if ARCH_X86_32 +INIT_XMM sse +SYNTH_FILTER +%endif +INIT_XMM sse2 +SYNTH_FILTER diff --git a/libavcodec/x86/dcadsp_init.c b/libavcodec/x86/dcadsp_init.c index 65e3db5c6c..5b77985569 100644 --- a/libavcodec/x86/dcadsp_init.c +++ b/libavcodec/x86/dcadsp_init.c @@ -56,25 +56,31 @@ av_cold void ff_dcadsp_init_x86(DCADSPContext *s) } } -void ff_synth_filter_inner_sse2(float *synth_buf_ptr, float synth_buf2[32], - const float window[512], - float out[32], intptr_t offset, float scale); -#if HAVE_YASM -static void synth_filter_sse2(FFTContext *imdct, - float *synth_buf_ptr, int *synth_buf_offset, - float synth_buf2[32], const float window[512], - float out[32], const float in[32], float scale) -{ - float *synth_buf= synth_buf_ptr + *synth_buf_offset; - - imdct->imdct_half(imdct, synth_buf, in); +#define SYNTH_FILTER_FUNC(opt) \ +void ff_synth_filter_inner_##opt(float *synth_buf_ptr, float synth_buf2[32], \ + const float window[512], \ + float out[32], intptr_t offset, float scale); \ +static void synth_filter_##opt(FFTContext *imdct, \ + float *synth_buf_ptr, int *synth_buf_offset, \ + float synth_buf2[32], const float window[512], \ + float out[32], const float in[32], float scale) \ +{ \ + float *synth_buf= synth_buf_ptr + *synth_buf_offset; \ + \ + imdct->imdct_half(imdct, synth_buf, in); \ + \ + ff_synth_filter_inner_##opt(synth_buf, synth_buf2, window, \ + out, *synth_buf_offset, scale); \ + \ + *synth_buf_offset = (*synth_buf_offset - 32) & 511; \ +} \ - ff_synth_filter_inner_sse2(synth_buf, synth_buf2, window, - out, *synth_buf_offset, scale); - - *synth_buf_offset = (*synth_buf_offset - 32) & 511; -} +#if HAVE_YASM +#if ARCH_X86_32 +SYNTH_FILTER_FUNC(sse) +#endif +SYNTH_FILTER_FUNC(sse2) #endif /* HAVE_YASM */ av_cold void ff_synth_filter_init_x86(SynthFilterContext *s) @@ -82,6 +88,11 @@ av_cold void ff_synth_filter_init_x86(SynthFilterContext *s) #if HAVE_YASM int cpu_flags = av_get_cpu_flags(); +#if ARCH_X86_32 + if (EXTERNAL_SSE(cpu_flags)) { + s->synth_filter_float = synth_filter_sse; + } +#endif if (EXTERNAL_SSE2(cpu_flags)) { s->synth_filter_float = synth_filter_sse2; } From 81e02fae6ec12ae85f052bd74aa9506cbebe4517 Mon Sep 17 00:00:00 2001 From: James Almer Date: Thu, 20 Mar 2014 15:37:55 -0300 Subject: [PATCH 2/3] x86/synth_filter: add synth_filter_avx Sandy Bridge Win64: 180 cycles in ff_synth_filter_inner_sse2 150 cycles in ff_synth_filter_inner_avx Also switch some instructions to a three operand format to avoid assembly errors with Yasm 1.1.0 or older. Signed-off-by: James Almer Signed-off-by: Anton Khirnov --- libavcodec/x86/dcadsp.asm | 85 ++++++++++++++++++++++-------------- libavcodec/x86/dcadsp_init.c | 4 ++ 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/libavcodec/x86/dcadsp.asm b/libavcodec/x86/dcadsp.asm index 970ec3dacb..662cb96f66 100644 --- a/libavcodec/x86/dcadsp.asm +++ b/libavcodec/x86/dcadsp.asm @@ -200,18 +200,22 @@ DCA_LFE_FIR 0 DCA_LFE_FIR 1 %macro SETZERO 1 -%if cpuflag(sse2) +%if cpuflag(sse2) && notcpuflag(avx) pxor %1, %1 %else xorps %1, %1, %1 %endif %endmacro -%macro SHUF 2 -%if cpuflag(sse2) - pshufd %1, %2, q0123 +%macro SHUF 3 +%if cpuflag(avx) + mova %3, [%2 - 16] + vperm2f128 %1, %3, %3, 1 + vshufps %1, %1, %1, q0123 +%elif cpuflag(sse2) + pshufd %1, [%2], q0123 %else - mova %1, %2 + mova %1, [%2] shufps %1, %1, q0123 %endif %endmacro @@ -220,43 +224,43 @@ DCA_LFE_FIR 1 ; reading backwards: ptr1 = synth_buf + j + i; ptr2 = synth_buf + j - i ;~ a += window[i + j] * (-synth_buf[15 - i + j]) ;~ b += window[i + j + 16] * (synth_buf[i + j]) - SHUF m5, [ptr2 + j + (15 - 3) * 4] + SHUF m5, ptr2 + j + (15 - 3) * 4, m6 mova m6, [ptr1 + j] %if ARCH_X86_64 - SHUF m11, [ptr2 + j + (15 - 3) * 4 - mmsize] + SHUF m11, ptr2 + j + (15 - 3) * 4 - mmsize, m12 mova m12, [ptr1 + j + mmsize] %endif - mulps m6, [win + %1 + j + 16 * 4] - mulps m5, [win + %1 + j] + mulps m6, m6, [win + %1 + j + 16 * 4] + mulps m5, m5, [win + %1 + j] %if ARCH_X86_64 - mulps m12, [win + %1 + j + mmsize + 16 * 4] - mulps m11, [win + %1 + j + mmsize] + mulps m12, m12, [win + %1 + j + mmsize + 16 * 4] + mulps m11, m11, [win + %1 + j + mmsize] %endif - addps m2, m6 - subps m1, m5 + addps m2, m2, m6 + subps m1, m1, m5 %if ARCH_X86_64 - addps m8, m12 - subps m7, m11 + addps m8, m8, m12 + subps m7, m7, m11 %endif ;~ c += window[i + j + 32] * (synth_buf[16 + i + j]) ;~ d += window[i + j + 48] * (synth_buf[31 - i + j]) - SHUF m6, [ptr2 + j + (31 - 3) * 4] + SHUF m6, ptr2 + j + (31 - 3) * 4, m5 mova m5, [ptr1 + j + 16 * 4] %if ARCH_X86_64 - SHUF m12, [ptr2 + j + (31 - 3) * 4 - mmsize] + SHUF m12, ptr2 + j + (31 - 3) * 4 - mmsize, m11 mova m11, [ptr1 + j + mmsize + 16 * 4] %endif - mulps m5, [win + %1 + j + 32 * 4] - mulps m6, [win + %1 + j + 48 * 4] + mulps m5, m5, [win + %1 + j + 32 * 4] + mulps m6, m6, [win + %1 + j + 48 * 4] %if ARCH_X86_64 - mulps m11, [win + %1 + j + mmsize + 32 * 4] - mulps m12, [win + %1 + j + mmsize + 48 * 4] + mulps m11, m11, [win + %1 + j + mmsize + 32 * 4] + mulps m12, m12, [win + %1 + j + mmsize + 48 * 4] %endif - addps m3, m5 - addps m4, m6 + addps m3, m3, m5 + addps m4, m4, m6 %if ARCH_X86_64 - addps m9, m11 - addps m10, m12 + addps m9, m9, m11 + addps m10, m10, m12 %endif sub j, 64 * 4 %endmacro @@ -269,25 +273,34 @@ cglobal synth_filter_inner, 0, 6 + 4 * ARCH_X86_64, 7 + 6 * ARCH_X86_64, \ synth_buf, synth_buf2, window, out, off, scale %define scale m0 %if ARCH_X86_32 || WIN64 -%if cpuflag(sse2) +%if cpuflag(sse2) && notcpuflag(avx) movd scale, scalem + SPLATD m0 %else - movss scale, scalem + VBROADCASTSS m0, scalem %endif ; Make sure offset is in a register and not on the stack %define OFFQ r4q %else + SPLATD xmm0 +%if cpuflag(avx) + vinsertf128 m0, m0, xmm0, 1 +%endif %define OFFQ offq %endif - SPLATD m0 ; prepare inner counter limit 1 mov r5q, 480 sub r5q, offmp and r5q, -64 shl r5q, 2 +%if ARCH_X86_32 || notcpuflag(avx) mov OFFQ, r5q %define i r5q mov i, 16 * 4 - (ARCH_X86_64 + 1) * mmsize ; main loop counter +%else +%define i 0 +%define OFFQ r5q +%endif %define buf2 synth_buf2q %if ARCH_X86_32 @@ -306,8 +319,10 @@ cglobal synth_filter_inner, 0, 6 + 4 * ARCH_X86_64, 7 + 6 * ARCH_X86_64, \ %define j r3q mov win, windowm mov ptr1, synth_bufm +%if ARCH_X86_32 || notcpuflag(avx) add win, i add ptr1, i +%endif %else ; ARCH_X86_64 %define ptr1 r6q %define ptr2 r7q ; must be loaded @@ -323,7 +338,9 @@ cglobal synth_filter_inner, 0, 6 + 4 * ARCH_X86_64, 7 + 6 * ARCH_X86_64, \ mov ptr2, synth_bufmp ; prepare the inner loop counter mov j, OFFQ +%if ARCH_X86_32 || notcpuflag(avx) sub ptr2, i +%endif .loop1: INNER_LOOP 0 jge .loop1 @@ -346,11 +363,11 @@ cglobal synth_filter_inner, 0, 6 + 4 * ARCH_X86_64, 7 + 6 * ARCH_X86_64, \ %endif ;~ out[i] = a * scale; ;~ out[i + 16] = b * scale; - mulps m1, scale - mulps m2, scale + mulps m1, m1, scale + mulps m2, m2, scale %if ARCH_X86_64 - mulps m7, scale - mulps m8, scale + mulps m7, m7, scale + mulps m8, m8, scale %endif ;~ synth_buf2[i] = c; ;~ synth_buf2[i + 16] = d; @@ -368,8 +385,10 @@ cglobal synth_filter_inner, 0, 6 + 4 * ARCH_X86_64, 7 + 6 * ARCH_X86_64, \ mova [outq + i + 0 * 4 + mmsize], m7 mova [outq + i + 16 * 4 + mmsize], m8 %endif +%if ARCH_X86_32 || notcpuflag(avx) sub i, (ARCH_X86_64 + 1) * mmsize jge .mainloop +%endif RET %endmacro @@ -379,3 +398,5 @@ SYNTH_FILTER %endif INIT_XMM sse2 SYNTH_FILTER +INIT_YMM avx +SYNTH_FILTER diff --git a/libavcodec/x86/dcadsp_init.c b/libavcodec/x86/dcadsp_init.c index 5b77985569..d7e0d65917 100644 --- a/libavcodec/x86/dcadsp_init.c +++ b/libavcodec/x86/dcadsp_init.c @@ -81,6 +81,7 @@ static void synth_filter_##opt(FFTContext *imdct, \ SYNTH_FILTER_FUNC(sse) #endif SYNTH_FILTER_FUNC(sse2) +SYNTH_FILTER_FUNC(avx) #endif /* HAVE_YASM */ av_cold void ff_synth_filter_init_x86(SynthFilterContext *s) @@ -96,5 +97,8 @@ av_cold void ff_synth_filter_init_x86(SynthFilterContext *s) if (EXTERNAL_SSE2(cpu_flags)) { s->synth_filter_float = synth_filter_sse2; } + if (EXTERNAL_AVX(cpu_flags)) { + s->synth_filter_float = synth_filter_avx; + } #endif /* HAVE_YASM */ } From c74b86699c86bdf62e8570f41d8a38be5710baa3 Mon Sep 17 00:00:00 2001 From: James Almer Date: Thu, 20 Mar 2014 15:37:56 -0300 Subject: [PATCH 3/3] x86/synth_filter: add synth_filter_fma3 Signed-off-by: James Almer Signed-off-by: Anton Khirnov --- libavcodec/x86/dcadsp.asm | 22 ++++++++++++++++++++++ libavcodec/x86/dcadsp_init.c | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/libavcodec/x86/dcadsp.asm b/libavcodec/x86/dcadsp.asm index 662cb96f66..59d96bf241 100644 --- a/libavcodec/x86/dcadsp.asm +++ b/libavcodec/x86/dcadsp.asm @@ -230,6 +230,14 @@ DCA_LFE_FIR 1 SHUF m11, ptr2 + j + (15 - 3) * 4 - mmsize, m12 mova m12, [ptr1 + j + mmsize] %endif +%if cpuflag(fma3) + fmaddps m2, m6, [win + %1 + j + 16 * 4], m2 + fnmaddps m1, m5, [win + %1 + j], m1 +%if ARCH_X86_64 + fmaddps m8, m12, [win + %1 + j + mmsize + 16 * 4], m8 + fnmaddps m7, m11, [win + %1 + j + mmsize], m7 +%endif +%else ; non-FMA mulps m6, m6, [win + %1 + j + 16 * 4] mulps m5, m5, [win + %1 + j] %if ARCH_X86_64 @@ -242,6 +250,7 @@ DCA_LFE_FIR 1 addps m8, m8, m12 subps m7, m7, m11 %endif +%endif ; cpuflag(fma3) ;~ c += window[i + j + 32] * (synth_buf[16 + i + j]) ;~ d += window[i + j + 48] * (synth_buf[31 - i + j]) SHUF m6, ptr2 + j + (31 - 3) * 4, m5 @@ -250,6 +259,14 @@ DCA_LFE_FIR 1 SHUF m12, ptr2 + j + (31 - 3) * 4 - mmsize, m11 mova m11, [ptr1 + j + mmsize + 16 * 4] %endif +%if cpuflag(fma3) + fmaddps m3, m5, [win + %1 + j + 32 * 4], m3 + fmaddps m4, m6, [win + %1 + j + 48 * 4], m4 +%if ARCH_X86_64 + fmaddps m9, m11, [win + %1 + j + mmsize + 32 * 4], m9 + fmaddps m10, m12, [win + %1 + j + mmsize + 48 * 4], m10 +%endif +%else ; non-FMA mulps m5, m5, [win + %1 + j + 32 * 4] mulps m6, m6, [win + %1 + j + 48 * 4] %if ARCH_X86_64 @@ -262,6 +279,7 @@ DCA_LFE_FIR 1 addps m9, m9, m11 addps m10, m10, m12 %endif +%endif ; cpuflag(fma3) sub j, 64 * 4 %endmacro @@ -400,3 +418,7 @@ INIT_XMM sse2 SYNTH_FILTER INIT_YMM avx SYNTH_FILTER +%if HAVE_FMA3_EXTERNAL +INIT_YMM fma3 +SYNTH_FILTER +%endif diff --git a/libavcodec/x86/dcadsp_init.c b/libavcodec/x86/dcadsp_init.c index d7e0d65917..beef288b13 100644 --- a/libavcodec/x86/dcadsp_init.c +++ b/libavcodec/x86/dcadsp_init.c @@ -82,6 +82,9 @@ SYNTH_FILTER_FUNC(sse) #endif SYNTH_FILTER_FUNC(sse2) SYNTH_FILTER_FUNC(avx) +#if HAVE_FMA3_EXTERNAL +SYNTH_FILTER_FUNC(fma3) +#endif #endif /* HAVE_YASM */ av_cold void ff_synth_filter_init_x86(SynthFilterContext *s) @@ -100,5 +103,8 @@ av_cold void ff_synth_filter_init_x86(SynthFilterContext *s) if (EXTERNAL_AVX(cpu_flags)) { s->synth_filter_float = synth_filter_avx; } + if (EXTERNAL_FMA3(cpu_flags)) { + s->synth_filter_float = synth_filter_fma3; + } #endif /* HAVE_YASM */ }