From 70eb77b34e9f08cd5e431921a2792f31e4b5265c Mon Sep 17 00:00:00 2001 From: Rostislav Pehlivanov Date: Sat, 29 Jul 2017 21:27:01 +0100 Subject: [PATCH] mdct15: add inverse transform postrotation SIMD 2.5ms frames: Before (c): 2638 decicycles in postrotate, 2097040 runs, 112 skips After (sse3): 1467 decicycles in postrotate, 2097083 runs, 69 skips After (avx2): 1244 decicycles in postrotate, 2097085 runs, 67 skips 5ms frames: Before (c): 4987 decicycles in postrotate, 1048371 runs, 205 skips After (sse3): 2644 decicycles in postrotate, 1048509 runs, 67 skips After (avx2): 2031 decicycles in postrotate, 1048523 runs, 53 skips 10ms frames: Before (c): 9153 decicycles in postrotate, 523575 runs, 713 skips After (sse3): 5110 decicycles in postrotate, 523726 runs, 562 skips After (avx2): 3738 decicycles in postrotate, 524223 runs, 65 skips 20ms frames: Before (c): 17857 decicycles in postrotate, 261866 runs, 278 skips After (sse3): 10041 decicycles in postrotate, 261746 runs, 398 skips After (avx2): 7050 decicycles in postrotate, 262116 runs, 28 skips Improves total decoding performance for real world content by 9% with avx2. Signed-off-by: Rostislav Pehlivanov --- libavcodec/mdct15.c | 38 +++++++++------ libavcodec/mdct15.h | 3 ++ libavcodec/x86/mdct15.asm | 93 ++++++++++++++++++++++++++++++++++-- libavcodec/x86/mdct15_init.c | 9 ++++ 4 files changed, 123 insertions(+), 20 deletions(-) diff --git a/libavcodec/mdct15.c b/libavcodec/mdct15.c index d68372c344..6f35059bfe 100644 --- a/libavcodec/mdct15.c +++ b/libavcodec/mdct15.c @@ -65,11 +65,11 @@ static inline int init_pfa_reindex_tabs(MDCT15Context *s) const int inv_1 = l_ptwo << ((4 - b_ptwo) & 3); /* (2^b_ptwo)^-1 mod 15 */ const int inv_2 = 0xeeeeeeef & ((1U << b_ptwo) - 1); /* 15^-1 mod 2^b_ptwo */ - s->pfa_prereindex = av_malloc(15 * l_ptwo * sizeof(*s->pfa_prereindex)); + s->pfa_prereindex = av_malloc_array(15 * l_ptwo, sizeof(*s->pfa_prereindex)); if (!s->pfa_prereindex) return 1; - s->pfa_postreindex = av_malloc(15 * l_ptwo * sizeof(*s->pfa_postreindex)); + s->pfa_postreindex = av_malloc_array(15 * l_ptwo, sizeof(*s->pfa_postreindex)); if (!s->pfa_postreindex) return 1; @@ -225,15 +225,22 @@ static void imdct15_half(MDCT15Context *s, float *dst, const float *src, for (i = 0; i < 15; i++) s->ptwo_fft.fft_calc(&s->ptwo_fft, s->tmp + l_ptwo*i); + /* Reindex again, apply twiddles and output */ + s->postreindex(z, s->tmp, s->twiddle_exptab, s->pfa_postreindex, len8); +} + +static void postrotate_c(FFTComplex *out, FFTComplex *in, FFTComplex *exp, + int *lut, ptrdiff_t len8) +{ + int i; + /* Reindex again, apply twiddles and output */ for (i = 0; i < len8; i++) { const int i0 = len8 + i, i1 = len8 - i - 1; - const int s0 = s->pfa_postreindex[i0], s1 = s->pfa_postreindex[i1]; + const int s0 = lut[i0], s1 = lut[i1]; - CMUL(z[i1].re, z[i0].im, s->tmp[s1].im, s->tmp[s1].re, - s->twiddle_exptab[i1].im, s->twiddle_exptab[i1].re); - CMUL(z[i0].re, z[i1].im, s->tmp[s0].im, s->tmp[s0].re, - s->twiddle_exptab[i0].im, s->twiddle_exptab[i0].re); + CMUL(out[i1].re, out[i0].im, in[s1].im, in[s1].re, exp[i1].im, exp[i1].re); + CMUL(out[i0].re, out[i1].im, in[s0].im, in[s0].re, exp[i0].im, exp[i0].re); } } @@ -253,13 +260,14 @@ av_cold int ff_mdct15_init(MDCT15Context **ps, int inverse, int N, double scale) if (!s) return AVERROR(ENOMEM); - s->fft_n = N - 1; - s->len4 = len2 / 2; - s->len2 = len2; - s->inverse = inverse; - s->fft15 = fft15_c; - s->mdct = mdct15; - s->imdct_half = imdct15_half; + s->fft_n = N - 1; + s->len4 = len2 / 2; + s->len2 = len2; + s->inverse = inverse; + s->fft15 = fft15_c; + s->mdct = mdct15; + s->imdct_half = imdct15_half; + s->postreindex = postrotate_c; if (ff_fft_init(&s->ptwo_fft, N - 1, s->inverse) < 0) goto fail; @@ -271,7 +279,7 @@ av_cold int ff_mdct15_init(MDCT15Context **ps, int inverse, int N, double scale) if (!s->tmp) goto fail; - s->twiddle_exptab = av_malloc_array(s->len4, sizeof(*s->twiddle_exptab)); + s->twiddle_exptab = av_malloc_array(s->len4, sizeof(*s->twiddle_exptab)); if (!s->twiddle_exptab) goto fail; diff --git a/libavcodec/mdct15.h b/libavcodec/mdct15.h index 1c2149d436..42e60f3e10 100644 --- a/libavcodec/mdct15.h +++ b/libavcodec/mdct15.h @@ -42,6 +42,9 @@ typedef struct MDCT15Context { /* 15-point FFT */ void (*fft15)(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride); + /* PFA postrotate and exptab */ + void (*postreindex)(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8); + /* Calculate a full 2N -> N MDCT */ void (*mdct)(struct MDCT15Context *s, float *dst, const float *src, ptrdiff_t stride); diff --git a/libavcodec/x86/mdct15.asm b/libavcodec/x86/mdct15.asm index f8b895944d..0309112538 100644 --- a/libavcodec/x86/mdct15.asm +++ b/libavcodec/x86/mdct15.asm @@ -22,14 +22,21 @@ %include "libavutil/x86/x86util.asm" -%if ARCH_X86_64 +SECTION_RODATA 32 -SECTION_RODATA +perm_neg: dd 2, 5, 3, 4, 6, 1, 7, 0 +perm_pos: dd 0, 7, 1, 6, 4, 3, 5, 2 +sign_adjust_r: times 4 dd 0x80000000, 0x00000000 sign_adjust_5: dd 0x00000000, 0x80000000, 0x80000000, 0x00000000 SECTION .text +%if ARCH_X86_64 + +;***************************************************************************************** +;void ff_fft15_avx(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride); +;***************************************************************************************** %macro FFT5 3 ; %1 - in_offset, %2 - dst1 (64bit used), %3 - dst2 VBROADCASTSD m0, [inq + %1] ; in[ 0].re, in[ 0].im, in[ 0].re, in[ 0].im movsd xm1, [inq + 1*16 + 8 + %1] ; in[ 3].re, in[ 3].im, 0, 0 @@ -103,9 +110,6 @@ SECTION .text movhps [%2q + strideq*4], xm1 %endmacro -;***************************************************************************************** -;void ff_fft15_avx(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride); -;***************************************************************************************** INIT_YMM avx cglobal fft15, 4, 6, 14, out, in, exptab, stride, stride3, stride5 %define out0q inq @@ -138,4 +142,83 @@ cglobal fft15, 4, 6, 14, out, in, exptab, stride, stride3, stride5 RET +%endif ; ARCH_X86_64 + +;******************************************************************************************************* +;void ff_mdct15_postreindex(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8); +;******************************************************************************************************* +%macro LUT_LOAD_4D 3 + mov r4d, [lutq + %3q*4 + 0] + movsd xmm%1, [inq + r4q*8] + mov r4d, [lutq + %3q*4 + 4] + movhps xmm%1, [inq + r4q*8] +%if cpuflag(avx2) + mov r4d, [lutq + %3q*4 + 8] + movsd %2, [inq + r4q*8] + mov r4d, [lutq + %3q*4 + 12] + movhps %2, [inq + r4q*8] + vinsertf128 %1, %1, %2, 1 +%endif +%endmacro + +%macro POSTROTATE_FN 1 +cglobal mdct15_postreindex, 5, 7, 8 + cpuflag(avx2)*2, out, in, exp, lut, len8, offset_p, offset_n + + xor offset_nq, offset_nq + lea offset_pq, [len8q*2 - %1] + + movaps m7, [sign_adjust_r] + +%if cpuflag(avx2) + movaps m8, [perm_pos] + movaps m9, [perm_neg] +%endif + +.loop: + movups m0, [expq + offset_pq*8] ; exp[p0].re, exp[p0].im, exp[p1].re, exp[p1].im, exp[p2].re, exp[p2].im, exp[p3].re, exp[p3].im + movups m1, [expq + offset_nq*8] ; exp[n3].re, exp[n3].im, exp[n2].re, exp[n2].im, exp[n1].re, exp[n1].im, exp[n0].re, exp[n0].im + + LUT_LOAD_4D m3, xm4, offset_p ; in[p0].re, in[p0].im, in[p1].re, in[p1].im, in[p2].re, in[p2].im, in[p3].re, in[p3].im + LUT_LOAD_4D m4, xm5, offset_n ; in[n3].re, in[n3].im, in[n2].re, in[n2].im, in[n1].re, in[n1].im, in[n0].re, in[n0].im + + mulps m5, m3, m0 ; in[p].reim * exp[p].reim + mulps m6, m4, m1 ; in[n].reim * exp[n].reim + + xorps m5, m7 ; in[p].re *= -1, in[p].im *= 1 + xorps m6, m7 ; in[n].re *= -1, in[n].im *= 1 + + shufps m3, m3, m3, q2301 ; in[p].imre + shufps m4, m4, m4, q2301 ; in[n].imre + + mulps m3, m0 ; in[p].imre * exp[p].reim + mulps m4, m1 ; in[n].imre * exp[n].reim + + haddps m3, m6 ; out[n0].im, out[n1].im, out[n3].re, out[n2].re, out[n2].im, out[n3].im, out[n1].re, out[n0].re + haddps m5, m4 ; out[p0].re, out[p1].re, out[p3].im, out[p2].im, out[p2].re, out[p3].re, out[p1].im, out[p0].im + +%if cpuflag(avx2) + vpermps m3, m9, m3 ; out[n3].im, out[n3].re, out[n2].im, out[n2].re, out[n1].im, out[n1].re, out[n0].im, out[n0].re + vpermps m5, m8, m5 ; out[p0].re, out[p0].im, out[p1].re, out[p1].im, out[p2].re, out[p2].im, out[p3].re, out[p3].im +%else + shufps m3, m3, m3, q0312 + shufps m5, m5, m5, q2130 +%endif + + movups [outq + offset_nq*8], m3 + movups [outq + offset_pq*8], m5 + + sub offset_pq, %1 + add offset_nq, %1 + cmp offset_nq, offset_pq + jle .loop + + REP_RET +%endmacro + +INIT_XMM sse3 +POSTROTATE_FN 2 + +%if ARCH_X86_64 && HAVE_AVX2_EXTERNAL +INIT_YMM avx2 +POSTROTATE_FN 4 %endif diff --git a/libavcodec/x86/mdct15_init.c b/libavcodec/x86/mdct15_init.c index ba3d94c2ec..45b91b7e21 100644 --- a/libavcodec/x86/mdct15_init.c +++ b/libavcodec/x86/mdct15_init.c @@ -25,6 +25,9 @@ #include "libavutil/x86/cpu.h" #include "libavcodec/mdct15.h" +void ff_mdct15_postreindex_sse3(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8); +void ff_mdct15_postreindex_avx2(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8); + void ff_fft15_avx(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride); static void perm_twiddles(MDCT15Context *s) @@ -85,11 +88,17 @@ av_cold void ff_mdct15_init_x86(MDCT15Context *s) int adjust_twiddles = 0; int cpu_flags = av_get_cpu_flags(); + if (EXTERNAL_SSE3(cpu_flags)) + s->postreindex = ff_mdct15_postreindex_sse3; + if (ARCH_X86_64 && EXTERNAL_AVX(cpu_flags)) { s->fft15 = ff_fft15_avx; adjust_twiddles = 1; } + if (ARCH_X86_64 && EXTERNAL_AVX2_FAST(cpu_flags)) + s->postreindex = ff_mdct15_postreindex_avx2; + if (adjust_twiddles) perm_twiddles(s); }