FFmpeg/libavcodec/x86/v210enc.asm

;******************************************************************************
;* V210 SIMD pack
;* Copyright (c) 2014 Kieran Kunhya <kierank@obe.tv>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************

%include "libavutil/x86/x86util.asm"

SECTION_RODATA 64

cextern pw_4
%define v210_enc_min_10 pw_4
v210_enc_max_10: times 16 dw 0x3fb

v210_enc_luma_mult_10: times 2 dw 4,1,16,4,1,16,0,0
v210_enc_luma_shuf_10: times 2 db -1,0,1,-1,2,3,4,5,-1,6,7,-1,8,9,10,11

v210_enc_chroma_mult_10: times 2 dw 1,4,16,0,16,1,4,0
v210_enc_chroma_shuf_10: times 2 db 0,1,8,9,-1,2,3,-1,10,11,4,5,-1,12,13,-1

cextern pb_1
%define v210_enc_min_8 pb_1
cextern pb_FE
%define v210_enc_max_8 pb_FE

v210_enc_luma_shuf_8: times 2 db 6,-1,7,-1,8,-1,9,-1,10,-1,11,-1,-1,-1,-1,-1
v210_enc_luma_mult_8: times 2 dw 16,4,64,16,4,64,0,0

v210_enc_chroma_shuf1_8: times 2 db 0,-1,1,-1,2,-1,3,-1,8,-1,9,-1,10,-1,11,-1
v210_enc_chroma_shuf2_8: times 2 db 3,-1,4,-1,5,-1,7,-1,11,-1,12,-1,13,-1,15,-1

v210_enc_chroma_mult_8: times 2 dw 4,16,64,0,64,4,16,0

v210enc_8_permb: db 32, 0,48,-1 ,  1,33, 2,-1 , 49, 3,34,-1 ,  4,50, 5,-1
                 db 35, 6,51,-1 ,  7,36, 8,-1 , 52, 9,37,-1 , 10,53,11,-1
                 db 38,12,54,-1 , 13,39,14,-1 , 55,15,40,-1 , 16,56,17,-1
                 db 41,18,57,-1 , 19,42,20,-1 , 58,21,43,-1 , 22,59,23,-1
v210enc_8_shufb: db  0, 8, 1,-1 ,  9, 2,10,-1 ,  3,11, 4,-1 , 12, 5,13,-1
                 db  2,10, 3,-1 , 11, 4,12,-1 ,  5,13, 6,-1 , 14, 7,15,-1
v210enc_8_permd: dd 0,1,4,5, 1,2,5,6
v210enc_8_mult: db 4, 0, 64, 0
v210enc_8_mask: dd 255<<12

icl_perm_y: ; vpermb does not set bytes to zero when the high bit is set unlike pshufb
%assign i 0
%rep 8
    db -1,i+0,i+1,-1 , i+2,i+3,i+4,i+5
    %assign i i+6
%endrep

icl_perm_uv: ; vpermb does not set bytes to zero when the high bit is set unlike pshufb
%assign i 0
%rep 4
    db i+0,i+1,i+32,i+33 , -1,i+2,i+3,-1 , i+34,i+35,i+4,i+5 , -1,i+36,i+37,-1
    %assign i i+6
%endrep

icl_perm_y_kmask:  times 8 db 1111_0110b
icl_perm_uv_kmask: times 8 db 0110_1111b

icl_shift_y:  times 10 dw 2,0,4
              times 4 db 0 ; padding to 64 bytes
icl_shift_uv: times 5 dw 0,2,4
              times 2 db 0 ; padding to 32 bytes
              times 5 dw 4,0,2
              times 2 db 0 ; padding to 32 bytes

v210enc_10_permd_y:  dd 0,1,2,-1 , 3,4,5,-1
v210enc_10_shufb_y:  db -1,0,1,-1 , 2,3,4,5 , -1,6,7,-1 , 8,9,10,11
v210enc_10_permd_uv: dd 0,1,4,5 , 1,2,5,6
v210enc_10_shufb_uv: db 0,1, 8, 9 , -1,2,3,-1 , 10,11,4,5 , -1,12,13,-1
                     db 2,3,10,11 , -1,4,5,-1 , 12,13,6,7 , -1,14,15,-1

SECTION .text

%macro v210_planar_pack_10 0

; v210_planar_pack_10(const uint16_t *y, const uint16_t *u, const uint16_t *v, uint8_t *dst, ptrdiff_t width)
cglobal v210_planar_pack_10, 5, 5, 4+cpuflag(avx2), y, u, v, dst, width
    lea     yq, [yq+2*widthq]
    add     uq, widthq
    add     vq, widthq
    neg     widthq

    mova    m2, [v210_enc_min_10]
    mova    m3, [v210_enc_max_10]

.loop:
    movu        xm0, [yq+2*widthq]
%if cpuflag(avx2)
    vinserti128 m0,   m0, [yq+widthq*2+12], 1
%endif
    CLIPW   m0, m2, m3

    movq         xm1, [uq+widthq]
    movhps       xm1, [vq+widthq]
%if cpuflag(avx2)
    movq         xm4, [uq+widthq+6]
    movhps       xm4, [vq+widthq+6]
    vinserti128  m1,   m1, xm4, 1
%endif
    CLIPW   m1, m2, m3

    pmullw  m0, [v210_enc_luma_mult_10]
    pshufb  m0, [v210_enc_luma_shuf_10]

    pmullw  m1, [v210_enc_chroma_mult_10]
    pshufb  m1, [v210_enc_chroma_shuf_10]

    por     m0, m1

    movu    [dstq], m0

    add     dstq, mmsize
    add     widthq, (mmsize*3)/8
    jl .loop

    RET
%endmacro

%if HAVE_SSSE3_EXTERNAL
INIT_XMM ssse3
v210_planar_pack_10
%endif

%if HAVE_AVX2_EXTERNAL
INIT_YMM avx2
v210_planar_pack_10
%endif

%macro v210_planar_pack_10_new 0

cglobal v210_planar_pack_10, 5, 5, 8+2*notcpuflag(avx512icl), y, u, v, dst, width
    lea     yq, [yq+2*widthq]
    add     uq, widthq
    add     vq, widthq
    neg     widthq

    %if cpuflag(avx512icl)
        movu  m6, [icl_perm_y]
        movu  m7, [icl_perm_uv]
        kmovq k1, [icl_perm_y_kmask]
        kmovq k2, [icl_perm_uv_kmask]
    %else
        movu           m6, [v210enc_10_permd_y]
        VBROADCASTI128 m7, [v210enc_10_shufb_y]
        movu           m8, [v210enc_10_permd_uv]
        movu           m9, [v210enc_10_shufb_uv]
    %endif
    movu  m2, [icl_shift_y]
    movu  m3, [icl_shift_uv]
    VBROADCASTI128 m4, [v210_enc_min_10] ; only ymm sized
    VBROADCASTI128 m5, [v210_enc_max_10] ; only ymm sized

    .loop:
        movu m0, [yq + widthq*2]
        %if cpuflag(avx512icl)
            movu         ym1, [uq + widthq*1]
            vinserti32x8 zm1, [vq + widthq*1], 1
        %else
            movu         xm1, [uq + widthq*1]
            vinserti128  ym1, [vq + widthq*1], 1
        %endif
        CLIPW m0, m4, m5
        CLIPW m1, m4, m5

        vpsllvw m0, m2
        vpsllvw m1, m3
        %if cpuflag(avx512icl)
            vpermb  m0{k1}{z}, m6, m0 ; make space for uv where the k-mask sets to zero
            vpermb  m1{k2}{z}, m7, m1 ; interleave uv and make space for y where the k-mask sets to zero
        %else
            vpermd m0, m6, m0
            pshufb m0, m7
            vpermd m1, m8, m1
            pshufb m1, m9
        %endif
        por     m0, m1

        movu  [dstq], m0
        add     dstq, mmsize
        add   widthq, (mmsize*3)/8
    jl .loop
RET

%endmacro

%if ARCH_X86_64
%if HAVE_AVX512_EXTERNAL
INIT_YMM avx512
v210_planar_pack_10_new
%endif
%endif

%if HAVE_AVX512ICL_EXTERNAL
INIT_ZMM avx512icl
v210_planar_pack_10_new
%endif

%macro v210_planar_pack_8 0

; v210_planar_pack_8(const uint8_t *y, const uint8_t *u, const uint8_t *v, uint8_t *dst, ptrdiff_t width)
cglobal v210_planar_pack_8, 5, 5, 7, y, u, v, dst, width
    add     yq, widthq
    shr     widthq, 1
    add     uq, widthq
    add     vq, widthq
    neg     widthq

    mova    m4, [v210_enc_min_8]
    mova    m5, [v210_enc_max_8]
    pxor    m6, m6

.loop:
    movu        xm1, [yq+widthq*2]
%if cpuflag(avx2)
    vinserti128 m1,   m1, [yq+widthq*2+12], 1
%endif
    CLIPUB  m1, m4, m5

    punpcklbw m0, m1, m6
    ; can't unpack high bytes in the same way because we process
    ; only six bytes at a time
    pshufb  m1, [v210_enc_luma_shuf_8]

    pmullw  m0, [v210_enc_luma_mult_8]
    pmullw  m1, [v210_enc_luma_mult_8]
    pshufb  m0, [v210_enc_luma_shuf_10]
    pshufb  m1, [v210_enc_luma_shuf_10]

    movq         xm3, [uq+widthq]
    movhps       xm3, [vq+widthq]
%if cpuflag(avx2)
    movq         xm2, [uq+widthq+6]
    movhps       xm2, [vq+widthq+6]
    vinserti128  m3,   m3, xm2, 1
%endif
    CLIPUB  m3, m4, m5

    ; shuffle and multiply to get the same packing as in 10-bit
    pshufb  m2, m3, [v210_enc_chroma_shuf1_8]
    pshufb  m3, [v210_enc_chroma_shuf2_8]

    pmullw  m2, [v210_enc_chroma_mult_8]
    pmullw  m3, [v210_enc_chroma_mult_8]
    pshufb  m2, [v210_enc_chroma_shuf_10]
    pshufb  m3, [v210_enc_chroma_shuf_10]

    por     m0, m2
    por     m1, m3

    movu         [dstq],    xm0
    movu         [dstq+16], xm1
%if cpuflag(avx2)
    vextracti128 [dstq+32], m0, 1
    vextracti128 [dstq+48], m1, 1
%endif

    add     dstq, 2*mmsize
    add     widthq, (mmsize*3)/8
    jl .loop

    RET
%endmacro

%if HAVE_SSSE3_EXTERNAL
INIT_XMM ssse3
v210_planar_pack_8
%endif
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
v210_planar_pack_8
%endif

%macro v210_planar_pack_8_new 0

cglobal v210_planar_pack_8, 5, 5, 7+notcpuflag(avx512icl), y, u, v, dst, width
    add     yq, widthq
    shr     widthq, 1
    add     uq, widthq
    add     vq, widthq
    neg     widthq

    %if cpuflag(avx512icl)
        mova m2, [v210enc_8_permb]
    %else
        mova m2, [v210enc_8_permd]
    %endif
    vpbroadcastd   m3, [v210enc_8_mult]
    VBROADCASTI128 m4, [v210_enc_min_8] ; only ymm sized
    VBROADCASTI128 m5, [v210_enc_max_8] ; only ymm sized
    vpbroadcastd   m6, [v210enc_8_mask]
    %if notcpuflag(avx512icl)
        movu m7, [v210enc_8_shufb]
    %endif

    .loop:
        %if cpuflag(avx512icl)
            movu         ym1, [yq + 2*widthq]
            vinserti32x4  m1, [uq + 1*widthq], 2
            vinserti32x4  m1, [vq + 1*widthq], 3
            vpermb        m1, m2, m1                 ; uyvx yuyx vyux yvyx
        %else
            movq         xm0, [uq + 1*widthq]        ; uuuu uuxx
            movq         xm1, [vq + 1*widthq]        ; vvvv vvxx
            punpcklbw    xm1, xm0, xm1               ; uvuv uvuv uvuv xxxx
            vinserti128   m1, m1, [yq + 2*widthq], 1 ; uvuv uvuv uvuv xxxx yyyy yyyy yyyy xxxx
            vpermd        m1, m2, m1                 ; uvuv uvxx yyyy yyxx xxuv uvuv xxyy yyyy
            pshufb        m1, m7                     ; uyv0 yuy0 vyu0 yvy0
        %endif
        CLIPUB       m1, m4, m5

        pmaddubsw  m0, m1, m3 ; shift high and low samples of each dword and mask out other bits
        pslld      m1,  4     ; shift center sample of each dword
        %if cpuflag(avx512)
            vpternlogd m0, m1, m6, 0xd8 ; C?B:A ; merge and mask out bad bits from B
        %else
            pand       m1, m6, m1
            por        m0, m0, m1
        %endif

        movu  [dstq], m0
        add     dstq, mmsize
        add   widthq, (mmsize*3)/16
    jl .loop
RET

%endmacro

%if HAVE_AVX2_EXTERNAL
INIT_YMM avx2
v210_planar_pack_8_new
%endif

%if HAVE_AVX512_EXTERNAL
INIT_YMM avx512
v210_planar_pack_8_new
%endif

%if HAVE_AVX512ICL_EXTERNAL
INIT_ZMM avx512icl
v210_planar_pack_8_new
%endif