Chiebot-Mirror
/
FFmpeg
mirror of https://github.com/FFmpeg/FFmpeg.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

ff_fft_calc_3dn/3dn2/sse: convert intrinsics to inline asm.

Browse Source 
2.5% faster fft, 0.5% faster vorbis.

Originally committed as revision 6023 to svn://svn.ffmpeg.org/ffmpeg/trunk
pull/126/head
Loren Merritt 19 years ago
parent 57060f8936
commit 1e4ecf26f5
4 changed files with 233 additions and 285 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      libavcodec/fft.c
  2. 181
      libavcodec/i386/fft_3dn.c
  3. 177
      libavcodec/i386/fft_3dn2.c
  4. 154
      libavcodec/i386/fft_sse.c

6
libavcodec/fft.c
Unescape View File

@ -58,7 +58,7 @@ int ff_fft_init(FFTContext *s, int nbits, int inverse)
s->exptab1 = NULL;
/* compute constant table for HAVE_SSE version */
#if (defined(HAVE_MMX) && (defined(HAVE_BUILTIN_VECTOR) || defined(HAVE_MM3DNOW))) \
#if defined(HAVE_MMX) \
|| (defined(HAVE_ALTIVEC) && !defined(ALTIVEC_USE_REFERENCE_C_CODE))
{
int has_vectors = mm_support();
@ -67,15 +67,12 @@ int ff_fft_init(FFTContext *s, int nbits, int inverse)
#if defined(HAVE_MMX)
if (has_vectors & MM_3DNOWEXT)
s->imdct_calc = ff_imdct_calc_3dn2;
#ifdef HAVE_MM3DNOW
if (has_vectors & MM_3DNOWEXT)
/* 3DNowEx for Athlon(XP) */
s->fft_calc = ff_fft_calc_3dn2;
else if (has_vectors & MM_3DNOW)
/* 3DNow! for K6-2/3 */
s->fft_calc = ff_fft_calc_3dn;
#endif
#ifdef HAVE_BUILTIN_VECTOR
if (has_vectors & MM_SSE2)
/* SSE for P4/K8 */
s->fft_calc = ff_fft_calc_sse;
@ -83,7 +80,6 @@ int ff_fft_init(FFTContext *s, int nbits, int inverse)
s->fft_calc == ff_fft_calc_c)
/* SSE for P3 */
s->fft_calc = ff_fft_calc_sse;
#endif
#else /* HAVE_MMX */
if (has_vectors & MM_ALTIVEC)
s->fft_calc = ff_fft_calc_altivec;

181
libavcodec/i386/fft_3dn.c
Unescape View File

@ -1,6 +1,6 @@
/*
* FFT/MDCT transform with 3DNow! optimizations
* Copyright (c) 2006 Zuxy MENG Jie.
* Copyright (c) 2006 Zuxy MENG Jie, Loren Merritt
* Based on fft_sse.c copyright (c) 2002 Fabrice Bellard.
*
* This library is free software; you can redistribute it and/or
@ -18,11 +18,6 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "../dsputil.h"
#include <math.h>
#ifdef HAVE_MM3DNOW
#include <mm3dnow.h>
static const int p1m1[2] __attribute__((aligned(8))) =
{ 0, 1 << 31 };
@ -33,104 +28,96 @@ static const int m1p1[2] __attribute__((aligned(8))) =
void ff_fft_calc_3dn(FFTContext *s, FFTComplex *z)
{
int ln = s->nbits;
int j, np, np2;
int nblocks, nloops;
register FFTComplex *p, *q;
FFTComplex *cptr, *cptr1;
int k;
np = 1 << ln;
/* FEMMS not a must here but recommended by AMD */
_m_femms();
{
__m64 *r, a0, a1, b0, b1, tmp, c;
r = (__m64 *)&z[0];
if (s->inverse)
c = *(__m64 *)m1p1;
else
c = *(__m64 *)p1m1;
j = (np >> 2);
do {
/* do the pass 0 butterfly */
a0 = _m_pfadd(r[0], r[1]);
a1 = _m_pfsub(r[0], r[1]);
/* do the pass 0 butterfly */
b0 = _m_pfadd(r[2], r[3]);
b1 = _m_pfsub(r[2], r[3]);
/* multiply third by -i */
tmp = _m_punpckhdq(b1, b1);
b1 = _m_punpckldq(b1, b1);
b1 = _m_punpckldq(tmp, b1);
b1 = _m_pxor(b1, c);
/* do the pass 1 butterfly */
r[0] = _m_pfadd(a0, b0);
r[1] = _m_pfadd(a1, b1);
r[2] = _m_pfsub(a0, b0);
r[3] = _m_pfsub(a1, b1);
r += 4;
} while (--j != 0);
}
long i, j;
long nblocks, nloops;
FFTComplex *p, *cptr;
asm volatile(
/* FEMMS is not a must here but recommended by AMD */
"femms \n\t"
"movq %0, %%mm7 \n\t"
::"m"(*(s->inverse ? m1p1 : p1m1))
);
i = 8 << ln;
asm volatile(
"1: \n\t"
"sub $32, %0 \n\t"
"movq (%0,%1), %%mm0 \n\t"
"movq 16(%0,%1), %%mm1 \n\t"
"movq 8(%0,%1), %%mm2 \n\t"
"movq 24(%0,%1), %%mm3 \n\t"
"movq %%mm0, %%mm4 \n\t"
"movq %%mm1, %%mm5 \n\t"
"pfadd %%mm2, %%mm0 \n\t"
"pfadd %%mm3, %%mm1 \n\t"
"pfsub %%mm2, %%mm4 \n\t"
"pfsub %%mm3, %%mm5 \n\t"
"movq %%mm0, %%mm2 \n\t"
"punpckldq %%mm5, %%mm6 \n\t"
"punpckhdq %%mm6, %%mm5 \n\t"
"movq %%mm4, %%mm3 \n\t"
"pxor %%mm7, %%mm5 \n\t"
"pfadd %%mm1, %%mm0 \n\t"
"pfadd %%mm5, %%mm4 \n\t"
"pfsub %%mm1, %%mm2 \n\t"
"pfsub %%mm5, %%mm3 \n\t"
"movq %%mm0, (%0,%1) \n\t"
"movq %%mm4, 8(%0,%1) \n\t"
"movq %%mm2, 16(%0,%1) \n\t"
"movq %%mm3, 24(%0,%1) \n\t"
"jg 1b \n\t"
:"+r"(i)
:"r"(z)
);
/* pass 2 .. ln-1 */
nblocks = np >> 3;
nblocks = 1 << (ln-3);
nloops = 1 << 2;
np2 = np >> 1;
cptr1 = s->exptab1;
cptr = s->exptab1;
do {
p = z;
q = z + nloops;
j = nblocks;
do {
cptr = cptr1;
k = nloops >> 1;
do {
__m64 a0, a1, b0, b1, c0, c1, t10, t11, t20, t21;
a0 = *(__m64 *)&p[0];
a1 = *(__m64 *)&p[1];
b0 = *(__m64 *)&q[0];
b1 = *(__m64 *)&q[1];
/* complex mul */
c0 = *(__m64 *)&cptr[0];
c1 = *(__m64 *)&cptr[1];
/* cre*re cim*re */
t10 = _m_pfmul(c0, _m_punpckldq(b0, b0));
t11 = _m_pfmul(c1, _m_punpckldq(b1, b1));
c0 = *(__m64 *)&cptr[2];
c1 = *(__m64 *)&cptr[3];
/* -cim*im cre*im */
t20 = _m_pfmul(c0, _m_punpckhdq(b0, b0));
t21 = _m_pfmul(c1, _m_punpckhdq(b1, b1));
b0 = _m_pfadd(t10, t20);
b1 = _m_pfadd(t11, t21);
/* butterfly */
*(__m64 *)&p[0] = _m_pfadd(a0, b0);
*(__m64 *)&p[1] = _m_pfadd(a1, b1);
*(__m64 *)&q[0] = _m_pfsub(a0, b0);
*(__m64 *)&q[1] = _m_pfsub(a1, b1);
p += 2;
q += 2;
cptr += 4;
} while (--k);
p += nloops;
q += nloops;
i = nloops*8;
asm volatile(
"1: \n\t"
"sub $16, %0 \n\t"
"movq (%1,%0), %%mm0 \n\t"
"movq 8(%1,%0), %%mm1 \n\t"
"movq (%2,%0), %%mm2 \n\t"
"movq 8(%2,%0), %%mm3 \n\t"
"movq %%mm2, %%mm4 \n\t"
"movq %%mm3, %%mm5 \n\t"
"punpckldq %%mm2, %%mm2 \n\t"
"punpckldq %%mm3, %%mm3 \n\t"
"punpckhdq %%mm4, %%mm4 \n\t"
"punpckhdq %%mm5, %%mm5 \n\t"
"pfmul (%3,%0,2), %%mm2 \n\t" // cre*re cim*re
"pfmul 8(%3,%0,2), %%mm3 \n\t"
"pfmul 16(%3,%0,2), %%mm4 \n\t" // -cim*im cre*im
"pfmul 24(%3,%0,2), %%mm5 \n\t"
"pfadd %%mm2, %%mm4 \n\t" // cre*re-cim*im cim*re+cre*im
"pfadd %%mm3, %%mm5 \n\t"
"movq %%mm0, %%mm2 \n\t"
"movq %%mm1, %%mm3 \n\t"
"pfadd %%mm4, %%mm0 \n\t"
"pfadd %%mm5, %%mm1 \n\t"
"pfsub %%mm4, %%mm2 \n\t"
"pfsub %%mm5, %%mm3 \n\t"
"movq %%mm0, (%1,%0) \n\t"
"movq %%mm1, 8(%1,%0) \n\t"
"movq %%mm2, (%2,%0) \n\t"
"movq %%mm3, 8(%2,%0) \n\t"
"jg 1b \n\t"
:"+r"(i)
:"r"(p), "r"(p + nloops), "r"(cptr)
);
p += nloops*2;
} while (--j);
cptr1 += nloops * 2;
nblocks = nblocks >> 1;
nloops = nloops << 1;
cptr += nloops*2;
nblocks >>= 1;
nloops <<= 1;
} while (nblocks != 0);
_m_femms();
asm volatile("femms");
}
#endif

177
libavcodec/i386/fft_3dn2.c
Unescape View File

@ -18,11 +18,6 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "../dsputil.h"
#include <math.h>
#ifdef HAVE_MM3DNOW
#include <mm3dnow.h>
static const int p1m1[2] __attribute__((aligned(8))) =
{ 0, 1 << 31 };
@ -33,108 +28,97 @@ static const int m1p1[2] __attribute__((aligned(8))) =
void ff_fft_calc_3dn2(FFTContext *s, FFTComplex *z)
{
int ln = s->nbits;
int j, np, np2;
int nblocks, nloops;
register FFTComplex *p, *q;
FFTComplex *cptr, *cptr1;
int k;
np = 1 << ln;
/* FEMMS is not a must here but recommended by AMD */
_m_femms();
{
__m64 *r, a0, a1, b0, b1, c;
r = (__m64 *)&z[0];
if (s->inverse)
c = *(__m64 *)m1p1;
else
c = *(__m64 *)p1m1;
j = (np >> 2);
do {
/* do the pass 0 butterfly */
a0 = _m_pfadd(r[0], r[1]);
a1 = _m_pfsub(r[0], r[1]);
/* do the pass 0 butterfly */
b0 = _m_pfadd(r[2], r[3]);
b1 = _m_pfsub(r[2], r[3]);
/* multiply third by -i */
b1 = _m_pswapd(b1);
b1 = _m_pxor(b1, c);
r[0] = _m_pfadd(a0, b0);
r[1] = _m_pfadd(a1, b1);
r[2] = _m_pfsub(a0, b0);
r[3] = _m_pfsub(a1, b1);
r += 4;
} while (--j != 0);
}
long i, j;
long nblocks, nloops;
FFTComplex *p, *cptr;
asm volatile(
/* FEMMS is not a must here but recommended by AMD */
"femms \n\t"
"movq %0, %%mm7 \n\t"
::"m"(*(s->inverse ? m1p1 : p1m1))
);
i = 8 << ln;
asm volatile(
"1: \n\t"
"sub $32, %0 \n\t"
"movq (%0,%1), %%mm0 \n\t"
"movq 16(%0,%1), %%mm1 \n\t"
"movq 8(%0,%1), %%mm2 \n\t"
"movq 24(%0,%1), %%mm3 \n\t"
"movq %%mm0, %%mm4 \n\t"
"movq %%mm1, %%mm5 \n\t"
"pfadd %%mm2, %%mm0 \n\t"
"pfadd %%mm3, %%mm1 \n\t"
"pfsub %%mm2, %%mm4 \n\t"
"pfsub %%mm3, %%mm5 \n\t"
"movq %%mm0, %%mm2 \n\t"
"pswapd %%mm5, %%mm5 \n\t"
"movq %%mm4, %%mm3 \n\t"
"pxor %%mm7, %%mm5 \n\t"
"pfadd %%mm1, %%mm0 \n\t"
"pfadd %%mm5, %%mm4 \n\t"
"pfsub %%mm1, %%mm2 \n\t"
"pfsub %%mm5, %%mm3 \n\t"
"movq %%mm0, (%0,%1) \n\t"
"movq %%mm4, 8(%0,%1) \n\t"
"movq %%mm2, 16(%0,%1) \n\t"
"movq %%mm3, 24(%0,%1) \n\t"
"jg 1b \n\t"
:"+r"(i)
:"r"(z)
);
/* pass 2 .. ln-1 */
nblocks = np >> 3;
nblocks = 1 << (ln-3);
nloops = 1 << 2;
np2 = np >> 1;
cptr1 = s->exptab1;
cptr = s->exptab1;
do {
p = z;
q = z + nloops;
j = nblocks;
do {
cptr = cptr1;
k = nloops >> 1;
do {
__m64 a0, a1, b0, b1, c0, c1, t10, t11, t20, t21;
a0 = *(__m64 *)&p[0];
a1 = *(__m64 *)&p[1];
b0 = *(__m64 *)&q[0];
b1 = *(__m64 *)&q[1];
/* complex mul */
c0 = *(__m64 *)&cptr[0];
c1 = *(__m64 *)&cptr[1];
/* cre*re cim*im */
t10 = _m_pfmul(c0, b0);
t11 = _m_pfmul(c1, b1);
/* no need to access cptr[2] & cptr[3] */
c0 = _m_pswapd(c0);
c1 = _m_pswapd(c1);
/* cim*re cre*im */
t20 = _m_pfmul(c0, b0);
t21 = _m_pfmul(c1, b1);
/* cre*re-cim*im cim*re+cre*im */
b0 = _m_pfpnacc(t10, t20);
b1 = _m_pfpnacc(t11, t21);
/* butterfly */
*(__m64 *)&p[0] = _m_pfadd(a0, b0);
*(__m64 *)&p[1] = _m_pfadd(a1, b1);
*(__m64 *)&q[0] = _m_pfsub(a0, b0);
*(__m64 *)&q[1] = _m_pfsub(a1, b1);
p += 2;
q += 2;
cptr += 4;
} while (--k);
p += nloops;
q += nloops;
i = nloops*8;
asm volatile(
"1: \n\t"
"sub $16, %0 \n\t"
"movq (%1,%0), %%mm0 \n\t"
"movq 8(%1,%0), %%mm1 \n\t"
"movq (%2,%0), %%mm2 \n\t"
"movq 8(%2,%0), %%mm3 \n\t"
"movq (%3,%0,2), %%mm4 \n\t"
"movq 8(%3,%0,2), %%mm5 \n\t"
"pswapd %%mm4, %%mm6 \n\t" // no need for cptr[2] & cptr[3]
"pswapd %%mm5, %%mm7 \n\t"
"pfmul %%mm2, %%mm4 \n\t" // cre*re cim*im
"pfmul %%mm3, %%mm5 \n\t"
"pfmul %%mm2, %%mm6 \n\t" // cim*re cre*im
"pfmul %%mm3, %%mm7 \n\t"
"pfpnacc %%mm6, %%mm4 \n\t" // cre*re-cim*im cim*re+cre*im
"pfpnacc %%mm7, %%mm5 \n\t"
"movq %%mm0, %%mm2 \n\t"
"movq %%mm1, %%mm3 \n\t"
"pfadd %%mm4, %%mm0 \n\t"
"pfadd %%mm5, %%mm1 \n\t"
"pfsub %%mm4, %%mm2 \n\t"
"pfsub %%mm5, %%mm3 \n\t"
"movq %%mm0, (%1,%0) \n\t"
"movq %%mm1, 8(%1,%0) \n\t"
"movq %%mm2, (%2,%0) \n\t"
"movq %%mm3, 8(%2,%0) \n\t"
"jg 1b \n\t"
:"+r"(i)
:"r"(p), "r"(p + nloops), "r"(cptr)
);
p += nloops*2;
} while (--j);
cptr1 += nloops * 2;
nblocks = nblocks >> 1;
nloops = nloops << 1;
cptr += nloops*2;
nblocks >>= 1;
nloops <<= 1;
} while (nblocks != 0);
_m_femms();
asm volatile("femms");
}
#endif
void ff_imdct_calc_3dn2(MDCTContext *s, FFTSample *output,
const FFTSample *input, FFTSample *tmp)
{
@ -221,3 +205,4 @@ void ff_imdct_calc_3dn2(MDCTContext *s, FFTSample *output,
}
asm volatile("femms");
}

154
libavcodec/i386/fft_sse.c
Unescape View File

@ -17,11 +17,6 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "../dsputil.h"
#include <math.h>
#ifdef HAVE_BUILTIN_VECTOR
#include <xmmintrin.h>
static const int p1p1p1m1[4] __attribute__((aligned(16))) =
{ 0, 0, 0, 1 << 31 };
@ -45,96 +40,81 @@ static void print_v4sf(const char *str, __m128 a)
void ff_fft_calc_sse(FFTContext *s, FFTComplex *z)
{
int ln = s->nbits;
int j, np, np2;
int nblocks, nloops;
register FFTComplex *p, *q;
FFTComplex *cptr, *cptr1;
int k;
np = 1 << ln;
{
__m128 *r, a, b, a1, c1, c2;
r = (__m128 *)&z[0];
c1 = *(__m128 *)p1p1m1m1;
if (s->inverse)
c2 = *(__m128 *)p1p1m1p1;
else
c2 = *(__m128 *)p1p1p1m1;
j = (np >> 2);
do {
a = r[0];
b = _mm_shuffle_ps(a, a, _MM_SHUFFLE(1, 0, 3, 2));
a = _mm_xor_ps(a, c1);
/* do the pass 0 butterfly */
a = _mm_add_ps(a, b);
a1 = r[1];
b = _mm_shuffle_ps(a1, a1, _MM_SHUFFLE(1, 0, 3, 2));
a1 = _mm_xor_ps(a1, c1);
/* do the pass 0 butterfly */
b = _mm_add_ps(a1, b);
/* multiply third by -i */
/* by toggling the sign bit */
b = _mm_shuffle_ps(b, b, _MM_SHUFFLE(2, 3, 1, 0));
b = _mm_xor_ps(b, c2);
/* do the pass 1 butterfly */
r[0] = _mm_add_ps(a, b);
r[1] = _mm_sub_ps(a, b);
r += 2;
} while (--j != 0);
}
long i, j;
long nblocks, nloops;
FFTComplex *p, *cptr;
asm volatile(
"movaps %0, %%xmm4 \n\t"
"movaps %1, %%xmm5 \n\t"
::"m"(*p1p1m1m1),
"m"(*(s->inverse ? p1p1m1p1 : p1p1p1m1))
);
i = 8 << ln;
asm volatile(
"1: \n\t"
"sub $32, %0 \n\t"
/* do the pass 0 butterfly */
"movaps (%0,%1), %%xmm0 \n\t"
"movaps %%xmm0, %%xmm1 \n\t"
"shufps $0x4E, %%xmm0, %%xmm0 \n\t"
"xorps %%xmm4, %%xmm1 \n\t"
"addps %%xmm1, %%xmm0 \n\t"
"movaps 16(%0,%1), %%xmm2 \n\t"
"movaps %%xmm2, %%xmm3 \n\t"
"shufps $0x4E, %%xmm2, %%xmm2 \n\t"
"xorps %%xmm4, %%xmm3 \n\t"
"addps %%xmm3, %%xmm2 \n\t"
/* multiply third by -i */
/* by toggling the sign bit */
"shufps $0xB4, %%xmm2, %%xmm2 \n\t"
"xorps %%xmm5, %%xmm2 \n\t"
/* do the pass 1 butterfly */
"movaps %%xmm0, %%xmm1 \n\t"
"addps %%xmm2, %%xmm0 \n\t"
"subps %%xmm2, %%xmm1 \n\t"
"movaps %%xmm0, (%0,%1) \n\t"
"movaps %%xmm1, 16(%0,%1) \n\t"
"jg 1b \n\t"
:"+r"(i)
:"r"(z)
);
/* pass 2 .. ln-1 */
nblocks = np >> 3;
nblocks = 1 << (ln-3);
nloops = 1 << 2;
np2 = np >> 1;
cptr1 = s->exptab1;
cptr = s->exptab1;
do {
p = z;
q = z + nloops;
j = nblocks;
do {
cptr = cptr1;
k = nloops >> 1;
do {
__m128 a, b, c, t1, t2;
a = *(__m128 *)p;
b = *(__m128 *)q;
/* complex mul */
c = *(__m128 *)cptr;
/* cre*re cim*re */
t1 = _mm_mul_ps(c,
_mm_shuffle_ps(b, b, _MM_SHUFFLE(2, 2, 0, 0)));
c = *(__m128 *)(cptr + 2);
/* -cim*im cre*im */
t2 = _mm_mul_ps(c,
_mm_shuffle_ps(b, b, _MM_SHUFFLE(3, 3, 1, 1)));
b = _mm_add_ps(t1, t2);
/* butterfly */
*(__m128 *)p = _mm_add_ps(a, b);
*(__m128 *)q = _mm_sub_ps(a, b);
p += 2;
q += 2;
cptr += 4;
} while (--k);
p += nloops;
q += nloops;
i = nloops*8;
asm volatile(
"1: \n\t"
"sub $16, %0 \n\t"
"movaps (%2,%0), %%xmm1 \n\t"
"movaps (%1,%0), %%xmm0 \n\t"
"movaps %%xmm1, %%xmm2 \n\t"
"shufps $0xA0, %%xmm1, %%xmm1 \n\t"
"shufps $0xF5, %%xmm2, %%xmm2 \n\t"
"mulps (%3,%0,2), %%xmm1 \n\t" // cre*re cim*re
"mulps 16(%3,%0,2), %%xmm2 \n\t" // -cim*im cre*im
"addps %%xmm2, %%xmm1 \n\t"
"movaps %%xmm0, %%xmm3 \n\t"
"addps %%xmm1, %%xmm0 \n\t"
"subps %%xmm1, %%xmm3 \n\t"
"movaps %%xmm0, (%1,%0) \n\t"
"movaps %%xmm3, (%2,%0) \n\t"
"jg 1b \n\t"
:"+r"(i)
:"r"(p), "r"(p + nloops), "r"(cptr)
);
p += nloops*2;
} while (--j);
cptr1 += nloops * 2;
nblocks = nblocks >> 1;
nloops = nloops << 1;
cptr += nloops*2;
nblocks >>= 1;
nloops <<= 1;
} while (nblocks != 0);
}
#endif

Write Preview
Loading…
Cancel
Save