664 lines
17 KiB
NASM
664 lines
17 KiB
NASM
|
;******************************************************************************
|
||
|
|
;* MMX/SSSE3-optimized functions for H.264 chroma MC
|
||
|
|
;* Copyright (c) 2005 Zoltan Hidvegi <hzoli -a- hzoli -d- com>,
|
||
|
|
;* 2005-2008 Loren Merritt
|
||
|
|
;*
|
||
|
|
;* This file is part of FFmpeg.
|
||
|
|
;*
|
||
|
|
;* FFmpeg is free software; you can redistribute it and/or
|
||
|
|
;* modify it under the terms of the GNU Lesser General Public
|
||
|
|
;* License as published by the Free Software Foundation; either
|
||
|
|
;* version 2.1 of the License, or (at your option) any later version.
|
||
|
|
;*
|
||
|
|
;* FFmpeg is distributed in the hope that it will be useful,
|
||
|
|
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||
|
|
;* Lesser General Public License for more details.
|
||
|
|
;*
|
||
|
|
;* You should have received a copy of the GNU Lesser General Public
|
||
|
|
;* License along with FFmpeg; if not, write to the Free Software
|
||
|
|
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||
|
|
;******************************************************************************
|
||
|
|
|
||
|
|
%include "libavutil/x86/x86util.asm"
|
||
|
|
|
||
|
|
SECTION_RODATA
|
||
|
|
|
||
|
|
rnd_rv40_2d_tbl: times 4 dw 0
|
||
|
|
times 4 dw 16
|
||
|
|
times 4 dw 32
|
||
|
|
times 4 dw 16
|
||
|
|
times 4 dw 32
|
||
|
|
times 4 dw 28
|
||
|
|
times 4 dw 32
|
||
|
|
times 4 dw 28
|
||
|
|
times 4 dw 0
|
||
|
|
times 4 dw 32
|
||
|
|
times 4 dw 16
|
||
|
|
times 4 dw 32
|
||
|
|
times 4 dw 32
|
||
|
|
times 4 dw 28
|
||
|
|
times 4 dw 32
|
||
|
|
times 4 dw 28
|
||
|
|
rnd_rv40_1d_tbl: times 4 dw 0
|
||
|
|
times 4 dw 2
|
||
|
|
times 4 dw 4
|
||
|
|
times 4 dw 2
|
||
|
|
times 4 dw 4
|
||
|
|
times 4 dw 3
|
||
|
|
times 4 dw 4
|
||
|
|
times 4 dw 3
|
||
|
|
times 4 dw 0
|
||
|
|
times 4 dw 4
|
||
|
|
times 4 dw 2
|
||
|
|
times 4 dw 4
|
||
|
|
times 4 dw 4
|
||
|
|
times 4 dw 3
|
||
|
|
times 4 dw 4
|
||
|
|
times 4 dw 3
|
||
|
|
|
||
|
|
cextern pw_3
|
||
|
|
cextern pw_4
|
||
|
|
cextern pw_8
|
||
|
|
pw_28: times 8 dw 28
|
||
|
|
cextern pw_32
|
||
|
|
cextern pw_64
|
||
|
|
|
||
|
|
SECTION .text
|
||
|
|
|
||
|
|
%macro mv0_pixels_mc8 0
|
||
|
|
lea r4, [r2*2 ]
|
||
|
|
.next4rows:
|
||
|
|
movq mm0, [r1 ]
|
||
|
|
movq mm1, [r1+r2]
|
||
|
|
add r1, r4
|
||
|
|
CHROMAMC_AVG mm0, [r0 ]
|
||
|
|
CHROMAMC_AVG mm1, [r0+r2]
|
||
|
|
movq [r0 ], mm0
|
||
|
|
movq [r0+r2], mm1
|
||
|
|
add r0, r4
|
||
|
|
movq mm0, [r1 ]
|
||
|
|
movq mm1, [r1+r2]
|
||
|
|
add r1, r4
|
||
|
|
CHROMAMC_AVG mm0, [r0 ]
|
||
|
|
CHROMAMC_AVG mm1, [r0+r2]
|
||
|
|
movq [r0 ], mm0
|
||
|
|
movq [r0+r2], mm1
|
||
|
|
add r0, r4
|
||
|
|
sub r3d, 4
|
||
|
|
jne .next4rows
|
||
|
|
%endmacro
|
||
|
|
|
||
|
|
%macro chroma_mc8_mmx_func 2-3
|
||
|
|
%ifidn %2, rv40
|
||
|
|
%ifdef PIC
|
||
|
|
%define rnd_1d_rv40 r8
|
||
|
|
%define rnd_2d_rv40 r8
|
||
|
|
%define extra_regs 2
|
||
|
|
%else ; no-PIC
|
||
|
|
%define rnd_1d_rv40 rnd_rv40_1d_tbl
|
||
|
|
%define rnd_2d_rv40 rnd_rv40_2d_tbl
|
||
|
|
%define extra_regs 1
|
||
|
|
%endif ; PIC
|
||
|
|
%else
|
||
|
|
%define extra_regs 0
|
||
|
|
%endif ; rv40
|
||
|
|
; void ff_put/avg_h264_chroma_mc8_*(uint8_t *dst /* align 8 */,
|
||
|
|
; uint8_t *src /* align 1 */,
|
||
|
|
; ptrdiff_t stride, int h, int mx, int my)
|
||
|
|
cglobal %1_%2_chroma_mc8%3, 6, 7 + extra_regs, 0
|
||
|
|
mov r6d, r5d
|
||
|
|
or r6d, r4d
|
||
|
|
jne .at_least_one_non_zero
|
||
|
|
; mx == 0 AND my == 0 - no filter needed
|
||
|
|
mv0_pixels_mc8
|
||
|
|
REP_RET
|
||
|
|
|
||
|
|
.at_least_one_non_zero:
|
||
|
|
%ifidn %2, rv40
|
||
|
|
%if ARCH_X86_64
|
||
|
|
mov r7, r5
|
||
|
|
and r7, 6 ; &~1 for mx/my=[0,7]
|
||
|
|
lea r7, [r7*4+r4]
|
||
|
|
sar r7d, 1
|
||
|
|
%define rnd_bias r7
|
||
|
|
%define dest_reg r0
|
||
|
|
%else ; x86-32
|
||
|
|
mov r0, r5
|
||
|
|
and r0, 6 ; &~1 for mx/my=[0,7]
|
||
|
|
lea r0, [r0*4+r4]
|
||
|
|
sar r0d, 1
|
||
|
|
%define rnd_bias r0
|
||
|
|
%define dest_reg r5
|
||
|
|
%endif
|
||
|
|
%else ; vc1, h264
|
||
|
|
%define rnd_bias 0
|
||
|
|
%define dest_reg r0
|
||
|
|
%endif
|
||
|
|
|
||
|
|
test r5d, r5d
|
||
|
|
mov r6, 1
|
||
|
|
je .my_is_zero
|
||
|
|
test r4d, r4d
|
||
|
|
mov r6, r2 ; dxy = x ? 1 : stride
|
||
|
|
jne .both_non_zero
|
||
|
|
.my_is_zero:
|
||
|
|
; mx == 0 XOR my == 0 - 1 dimensional filter only
|
||
|
|
or r4d, r5d ; x + y
|
||
|
|
|
||
|
|
%ifidn %2, rv40
|
||
|
|
%ifdef PIC
|
||
|
|
lea r8, [rnd_rv40_1d_tbl]
|
||
|
|
%endif
|
||
|
|
%if ARCH_X86_64 == 0
|
||
|
|
mov r5, r0m
|
||
|
|
%endif
|
||
|
|
%endif
|
||
|
|
|
||
|
|
movd m5, r4d
|
||
|
|
movq m4, [pw_8]
|
||
|
|
movq m6, [rnd_1d_%2+rnd_bias*8] ; mm6 = rnd >> 3
|
||
|
|
punpcklwd m5, m5
|
||
|
|
punpckldq m5, m5 ; mm5 = B = x
|
||
|
|
pxor m7, m7
|
||
|
|
psubw m4, m5 ; mm4 = A = 8-x
|
||
|
|
|
||
|
|
.next1drow:
|
||
|
|
movq m0, [r1 ] ; mm0 = src[0..7]
|
||
|
|
movq m2, [r1+r6] ; mm1 = src[1..8]
|
||
|
|
|
||
|
|
movq m1, m0
|
||
|
|
movq m3, m2
|
||
|
|
punpcklbw m0, m7
|
||
|
|
punpckhbw m1, m7
|
||
|
|
punpcklbw m2, m7
|
||
|
|
punpckhbw m3, m7
|
||
|
|
pmullw m0, m4 ; [mm0,mm1] = A * src[0..7]
|
||
|
|
pmullw m1, m4
|
||
|
|
pmullw m2, m5 ; [mm2,mm3] = B * src[1..8]
|
||
|
|
pmullw m3, m5
|
||
|
|
|
||
|
|
paddw m0, m6
|
||
|
|
paddw m1, m6
|
||
|
|
paddw m0, m2
|
||
|
|
paddw m1, m3
|
||
|
|
psrlw m0, 3
|
||
|
|
psrlw m1, 3
|
||
|
|
packuswb m0, m1
|
||
|
|
CHROMAMC_AVG m0, [dest_reg]
|
||
|
|
movq [dest_reg], m0 ; dst[0..7] = (A * src[0..7] + B * src[1..8] + (rnd >> 3)) >> 3
|
||
|
|
|
||
|
|
add dest_reg, r2
|
||
|
|
add r1, r2
|
||
|
|
dec r3d
|
||
|
|
jne .next1drow
|
||
|
|
REP_RET
|
||
|
|
|
||
|
|
.both_non_zero: ; general case, bilinear
|
||
|
|
movd m4, r4d ; x
|
||
|
|
movd m6, r5d ; y
|
||
|
|
%ifidn %2, rv40
|
||
|
|
%ifdef PIC
|
||
|
|
lea r8, [rnd_rv40_2d_tbl]
|
||
|
|
%endif
|
||
|
|
%if ARCH_X86_64 == 0
|
||
|
|
mov r5, r0m
|
||
|
|
%endif
|
||
|
|
%endif
|
||
|
|
mov r6, rsp ; backup stack pointer
|
||
|
|
and rsp, ~(mmsize-1) ; align stack
|
||
|
|
sub rsp, 16 ; AA and DD
|
||
|
|
|
||
|
|
punpcklwd m4, m4
|
||
|
|
punpcklwd m6, m6
|
||
|
|
punpckldq m4, m4 ; mm4 = x words
|
||
|
|
punpckldq m6, m6 ; mm6 = y words
|
||
|
|
movq m5, m4
|
||
|
|
pmullw m4, m6 ; mm4 = x * y
|
||
|
|
psllw m5, 3
|
||
|
|
psllw m6, 3
|
||
|
|
movq m7, m5
|
||
|
|
paddw m7, m6
|
||
|
|
movq [rsp+8], m4 ; DD = x * y
|
||
|
|
psubw m5, m4 ; mm5 = B = 8x - xy
|
||
|
|
psubw m6, m4 ; mm6 = C = 8y - xy
|
||
|
|
paddw m4, [pw_64]
|
||
|
|
psubw m4, m7 ; mm4 = A = xy - (8x+8y) + 64
|
||
|
|
pxor m7, m7
|
||
|
|
movq [rsp ], m4
|
||
|
|
|
||
|
|
movq m0, [r1 ] ; mm0 = src[0..7]
|
||
|
|
movq m1, [r1+1] ; mm1 = src[1..8]
|
||
|
|
.next2drow:
|
||
|
|
add r1, r2
|
||
|
|
|
||
|
|
movq m2, m0
|
||
|
|
movq m3, m1
|
||
|
|
punpckhbw m0, m7
|
||
|
|
punpcklbw m1, m7
|
||
|
|
punpcklbw m2, m7
|
||
|
|
punpckhbw m3, m7
|
||
|
|
pmullw m0, [rsp]
|
||
|
|
pmullw m2, [rsp]
|
||
|
|
pmullw m1, m5
|
||
|
|
pmullw m3, m5
|
||
|
|
paddw m2, m1 ; mm2 = A * src[0..3] + B * src[1..4]
|
||
|
|
paddw m3, m0 ; mm3 = A * src[4..7] + B * src[5..8]
|
||
|
|
|
||
|
|
movq m0, [r1]
|
||
|
|
movq m1, m0
|
||
|
|
punpcklbw m0, m7
|
||
|
|
punpckhbw m1, m7
|
||
|
|
pmullw m0, m6
|
||
|
|
pmullw m1, m6
|
||
|
|
paddw m2, m0
|
||
|
|
paddw m3, m1 ; [mm2,mm3] += C * src[0..7]
|
||
|
|
|
||
|
|
movq m1, [r1+1]
|
||
|
|
movq m0, m1
|
||
|
|
movq m4, m1
|
||
|
|
punpcklbw m0, m7
|
||
|
|
punpckhbw m4, m7
|
||
|
|
pmullw m0, [rsp+8]
|
||
|
|
pmullw m4, [rsp+8]
|
||
|
|
paddw m2, m0
|
||
|
|
paddw m3, m4 ; [mm2,mm3] += D * src[1..8]
|
||
|
|
movq m0, [r1]
|
||
|
|
|
||
|
|
paddw m2, [rnd_2d_%2+rnd_bias*8]
|
||
|
|
paddw m3, [rnd_2d_%2+rnd_bias*8]
|
||
|
|
psrlw m2, 6
|
||
|
|
psrlw m3, 6
|
||
|
|
packuswb m2, m3
|
||
|
|
CHROMAMC_AVG m2, [dest_reg]
|
||
|
|
movq [dest_reg], m2 ; dst[0..7] = ([mm2,mm3] + rnd) >> 6
|
||
|
|
|
||
|
|
add dest_reg, r2
|
||
|
|
dec r3d
|
||
|
|
jne .next2drow
|
||
|
|
mov rsp, r6 ; restore stack pointer
|
||
|
|
RET
|
||
|
|
%endmacro
|
||
|
|
|
||
|
|
%macro chroma_mc4_mmx_func 2
|
||
|
|
%define extra_regs 0
|
||
|
|
%ifidn %2, rv40
|
||
|
|
%ifdef PIC
|
||
|
|
%define extra_regs 1
|
||
|
|
%endif ; PIC
|
||
|
|
%endif ; rv40
|
||
|
|
cglobal %1_%2_chroma_mc4, 6, 6 + extra_regs, 0
|
||
|
|
pxor m7, m7
|
||
|
|
movd m2, r4d ; x
|
||
|
|
movd m3, r5d ; y
|
||
|
|
movq m4, [pw_8]
|
||
|
|
movq m5, [pw_8]
|
||
|
|
punpcklwd m2, m2
|
||
|
|
punpcklwd m3, m3
|
||
|
|
punpcklwd m2, m2
|
||
|
|
punpcklwd m3, m3
|
||
|
|
psubw m4, m2
|
||
|
|
psubw m5, m3
|
||
|
|
|
||
|
|
%ifidn %2, rv40
|
||
|
|
%ifdef PIC
|
||
|
|
lea r6, [rnd_rv40_2d_tbl]
|
||
|
|
%define rnd_2d_rv40 r6
|
||
|
|
%else
|
||
|
|
%define rnd_2d_rv40 rnd_rv40_2d_tbl
|
||
|
|
%endif
|
||
|
|
and r5, 6 ; &~1 for mx/my=[0,7]
|
||
|
|
lea r5, [r5*4+r4]
|
||
|
|
sar r5d, 1
|
||
|
|
%define rnd_bias r5
|
||
|
|
%else ; vc1, h264
|
||
|
|
%define rnd_bias 0
|
||
|
|
%endif
|
||
|
|
|
||
|
|
movd m0, [r1 ]
|
||
|
|
movd m6, [r1+1]
|
||
|
|
add r1, r2
|
||
|
|
punpcklbw m0, m7
|
||
|
|
punpcklbw m6, m7
|
||
|
|
pmullw m0, m4
|
||
|
|
pmullw m6, m2
|
||
|
|
paddw m6, m0
|
||
|
|
|
||
|
|
.next2rows:
|
||
|
|
movd m0, [r1 ]
|
||
|
|
movd m1, [r1+1]
|
||
|
|
add r1, r2
|
||
|
|
punpcklbw m0, m7
|
||
|
|
punpcklbw m1, m7
|
||
|
|
pmullw m0, m4
|
||
|
|
pmullw m1, m2
|
||
|
|
paddw m1, m0
|
||
|
|
movq m0, m1
|
||
|
|
|
||
|
|
pmullw m6, m5
|
||
|
|
pmullw m1, m3
|
||
|
|
paddw m6, [rnd_2d_%2+rnd_bias*8]
|
||
|
|
paddw m1, m6
|
||
|
|
psrlw m1, 6
|
||
|
|
packuswb m1, m1
|
||
|
|
CHROMAMC_AVG4 m1, m6, [r0]
|
||
|
|
movd [r0], m1
|
||
|
|
add r0, r2
|
||
|
|
|
||
|
|
movd m6, [r1 ]
|
||
|
|
movd m1, [r1+1]
|
||
|
|
add r1, r2
|
||
|
|
punpcklbw m6, m7
|
||
|
|
punpcklbw m1, m7
|
||
|
|
pmullw m6, m4
|
||
|
|
pmullw m1, m2
|
||
|
|
paddw m1, m6
|
||
|
|
movq m6, m1
|
||
|
|
pmullw m0, m5
|
||
|
|
pmullw m1, m3
|
||
|
|
paddw m0, [rnd_2d_%2+rnd_bias*8]
|
||
|
|
paddw m1, m0
|
||
|
|
psrlw m1, 6
|
||
|
|
packuswb m1, m1
|
||
|
|
CHROMAMC_AVG4 m1, m0, [r0]
|
||
|
|
movd [r0], m1
|
||
|
|
add r0, r2
|
||
|
|
sub r3d, 2
|
||
|
|
jnz .next2rows
|
||
|
|
REP_RET
|
||
|
|
%endmacro
|
||
|
|
|
||
|
|
%macro chroma_mc2_mmx_func 2
|
||
|
|
cglobal %1_%2_chroma_mc2, 6, 7, 0
|
||
|
|
mov r6d, r4d
|
||
|
|
shl r4d, 16
|
||
|
|
sub r4d, r6d
|
||
|
|
add r4d, 8
|
||
|
|
imul r5d, r4d ; x*y<<16 | y*(8-x)
|
||
|
|
shl r4d, 3
|
||
|
|
sub r4d, r5d ; x*(8-y)<<16 | (8-x)*(8-y)
|
||
|
|
|
||
|
|
movd m5, r4d
|
||
|
|
movd m6, r5d
|
||
|
|
punpckldq m5, m5 ; mm5 = {A,B,A,B}
|
||
|
|
punpckldq m6, m6 ; mm6 = {C,D,C,D}
|
||
|
|
pxor m7, m7
|
||
|
|
movd m2, [r1]
|
||
|
|
punpcklbw m2, m7
|
||
|
|
pshufw m2, m2, 0x94 ; mm0 = src[0,1,1,2]
|
||
|
|
|
||
|
|
.nextrow:
|
||
|
|
add r1, r2
|
||
|
|
movq m1, m2
|
||
|
|
pmaddwd m1, m5 ; mm1 = A * src[0,1] + B * src[1,2]
|
||
|
|
movd m0, [r1]
|
||
|
|
punpcklbw m0, m7
|
||
|
|
pshufw m0, m0, 0x94 ; mm0 = src[0,1,1,2]
|
||
|
|
movq m2, m0
|
||
|
|
pmaddwd m0, m6
|
||
|
|
paddw m1, [rnd_2d_%2]
|
||
|
|
paddw m1, m0 ; mm1 += C * src[0,1] + D * src[1,2]
|
||
|
|
psrlw m1, 6
|
||
|
|
packssdw m1, m7
|
||
|
|
packuswb m1, m7
|
||
|
|
CHROMAMC_AVG4 m1, m3, [r0]
|
||
|
|
movd r5d, m1
|
||
|
|
mov [r0], r5w
|
||
|
|
add r0, r2
|
||
|
|
sub r3d, 1
|
||
|
|
jnz .nextrow
|
||
|
|
REP_RET
|
||
|
|
%endmacro
|
||
|
|
|
||
|
|
%define rnd_1d_h264 pw_4
|
||
|
|
%define rnd_2d_h264 pw_32
|
||
|
|
%define rnd_1d_vc1 pw_3
|
||
|
|
%define rnd_2d_vc1 pw_28
|
||
|
|
|
||
|
|
%macro NOTHING 2-3
|
||
|
|
%endmacro
|
||
|
|
%macro DIRECT_AVG 2
|
||
|
|
PAVGB %1, %2
|
||
|
|
%endmacro
|
||
|
|
%macro COPY_AVG 3
|
||
|
|
movd %2, %3
|
||
|
|
PAVGB %1, %2
|
||
|
|
%endmacro
|
||
|
|
|
||
|
|
INIT_MMX mmx
|
||
|
|
%define CHROMAMC_AVG NOTHING
|
||
|
|
%define CHROMAMC_AVG4 NOTHING
|
||
|
|
chroma_mc8_mmx_func put, h264, _rnd
|
||
|
|
chroma_mc8_mmx_func put, vc1, _nornd
|
||
|
|
chroma_mc8_mmx_func put, rv40
|
||
|
|
chroma_mc4_mmx_func put, h264
|
||
|
|
chroma_mc4_mmx_func put, rv40
|
||
|
|
|
||
|
|
INIT_MMX mmxext
|
||
|
|
chroma_mc2_mmx_func put, h264
|
||
|
|
|
||
|
|
%define CHROMAMC_AVG DIRECT_AVG
|
||
|
|
%define CHROMAMC_AVG4 COPY_AVG
|
||
|
|
chroma_mc8_mmx_func avg, h264, _rnd
|
||
|
|
chroma_mc8_mmx_func avg, vc1, _nornd
|
||
|
|
chroma_mc8_mmx_func avg, rv40
|
||
|
|
chroma_mc4_mmx_func avg, h264
|
||
|
|
chroma_mc4_mmx_func avg, rv40
|
||
|
|
chroma_mc2_mmx_func avg, h264
|
||
|
|
|
||
|
|
INIT_MMX 3dnow
|
||
|
|
chroma_mc8_mmx_func avg, h264, _rnd
|
||
|
|
chroma_mc8_mmx_func avg, vc1, _nornd
|
||
|
|
chroma_mc8_mmx_func avg, rv40
|
||
|
|
chroma_mc4_mmx_func avg, h264
|
||
|
|
chroma_mc4_mmx_func avg, rv40
|
||
|
|
|
||
|
|
%macro chroma_mc8_ssse3_func 2-3
|
||
|
|
cglobal %1_%2_chroma_mc8%3, 6, 7, 8
|
||
|
|
mov r6d, r5d
|
||
|
|
or r6d, r4d
|
||
|
|
jne .at_least_one_non_zero
|
||
|
|
; mx == 0 AND my == 0 - no filter needed
|
||
|
|
mv0_pixels_mc8
|
||
|
|
REP_RET
|
||
|
|
|
||
|
|
.at_least_one_non_zero:
|
||
|
|
test r5d, r5d
|
||
|
|
je .my_is_zero
|
||
|
|
test r4d, r4d
|
||
|
|
je .mx_is_zero
|
||
|
|
|
||
|
|
; general case, bilinear
|
||
|
|
mov r6d, r4d
|
||
|
|
shl r4d, 8
|
||
|
|
sub r4, r6
|
||
|
|
mov r6, 8
|
||
|
|
add r4, 8 ; x*288+8 = x<<8 | (8-x)
|
||
|
|
sub r6d, r5d
|
||
|
|
imul r6, r4 ; (8-y)*(x*255+8) = (8-y)*x<<8 | (8-y)*(8-x)
|
||
|
|
imul r4d, r5d ; y *(x*255+8) = y *x<<8 | y *(8-x)
|
||
|
|
|
||
|
|
movd m7, r6d
|
||
|
|
movd m6, r4d
|
||
|
|
movdqa m5, [rnd_2d_%2]
|
||
|
|
movq m0, [r1 ]
|
||
|
|
movq m1, [r1+1]
|
||
|
|
pshuflw m7, m7, 0
|
||
|
|
pshuflw m6, m6, 0
|
||
|
|
punpcklbw m0, m1
|
||
|
|
movlhps m7, m7
|
||
|
|
movlhps m6, m6
|
||
|
|
|
||
|
|
.next2rows:
|
||
|
|
movq m1, [r1+r2*1 ]
|
||
|
|
movq m2, [r1+r2*1+1]
|
||
|
|
movq m3, [r1+r2*2 ]
|
||
|
|
movq m4, [r1+r2*2+1]
|
||
|
|
lea r1, [r1+r2*2]
|
||
|
|
punpcklbw m1, m2
|
||
|
|
movdqa m2, m1
|
||
|
|
punpcklbw m3, m4
|
||
|
|
movdqa m4, m3
|
||
|
|
pmaddubsw m0, m7
|
||
|
|
pmaddubsw m1, m6
|
||
|
|
pmaddubsw m2, m7
|
||
|
|
pmaddubsw m3, m6
|
||
|
|
paddw m0, m5
|
||
|
|
paddw m2, m5
|
||
|
|
paddw m1, m0
|
||
|
|
paddw m3, m2
|
||
|
|
psrlw m1, 6
|
||
|
|
movdqa m0, m4
|
||
|
|
psrlw m3, 6
|
||
|
|
%ifidn %1, avg
|
||
|
|
movq m2, [r0 ]
|
||
|
|
movhps m2, [r0+r2]
|
||
|
|
%endif
|
||
|
|
packuswb m1, m3
|
||
|
|
CHROMAMC_AVG m1, m2
|
||
|
|
movq [r0 ], m1
|
||
|
|
movhps [r0+r2], m1
|
||
|
|
sub r3d, 2
|
||
|
|
lea r0, [r0+r2*2]
|
||
|
|
jg .next2rows
|
||
|
|
REP_RET
|
||
|
|
|
||
|
|
.my_is_zero:
|
||
|
|
mov r5d, r4d
|
||
|
|
shl r4d, 8
|
||
|
|
add r4, 8
|
||
|
|
sub r4, r5 ; 255*x+8 = x<<8 | (8-x)
|
||
|
|
movd m7, r4d
|
||
|
|
movdqa m6, [rnd_1d_%2]
|
||
|
|
pshuflw m7, m7, 0
|
||
|
|
movlhps m7, m7
|
||
|
|
|
||
|
|
.next2xrows:
|
||
|
|
movq m0, [r1 ]
|
||
|
|
movq m1, [r1 +1]
|
||
|
|
movq m2, [r1+r2 ]
|
||
|
|
movq m3, [r1+r2+1]
|
||
|
|
punpcklbw m0, m1
|
||
|
|
punpcklbw m2, m3
|
||
|
|
pmaddubsw m0, m7
|
||
|
|
pmaddubsw m2, m7
|
||
|
|
%ifidn %1, avg
|
||
|
|
movq m4, [r0 ]
|
||
|
|
movhps m4, [r0+r2]
|
||
|
|
%endif
|
||
|
|
paddw m0, m6
|
||
|
|
paddw m2, m6
|
||
|
|
psrlw m0, 3
|
||
|
|
psrlw m2, 3
|
||
|
|
packuswb m0, m2
|
||
|
|
CHROMAMC_AVG m0, m4
|
||
|
|
movq [r0 ], m0
|
||
|
|
movhps [r0+r2], m0
|
||
|
|
sub r3d, 2
|
||
|
|
lea r0, [r0+r2*2]
|
||
|
|
lea r1, [r1+r2*2]
|
||
|
|
jg .next2xrows
|
||
|
|
REP_RET
|
||
|
|
|
||
|
|
.mx_is_zero:
|
||
|
|
mov r4d, r5d
|
||
|
|
shl r5d, 8
|
||
|
|
add r5, 8
|
||
|
|
sub r5, r4 ; 255*y+8 = y<<8 | (8-y)
|
||
|
|
movd m7, r5d
|
||
|
|
movdqa m6, [rnd_1d_%2]
|
||
|
|
pshuflw m7, m7, 0
|
||
|
|
movlhps m7, m7
|
||
|
|
|
||
|
|
.next2yrows:
|
||
|
|
movq m0, [r1 ]
|
||
|
|
movq m1, [r1+r2 ]
|
||
|
|
movdqa m2, m1
|
||
|
|
movq m3, [r1+r2*2]
|
||
|
|
lea r1, [r1+r2*2]
|
||
|
|
punpcklbw m0, m1
|
||
|
|
punpcklbw m2, m3
|
||
|
|
pmaddubsw m0, m7
|
||
|
|
pmaddubsw m2, m7
|
||
|
|
%ifidn %1, avg
|
||
|
|
movq m4, [r0 ]
|
||
|
|
movhps m4, [r0+r2]
|
||
|
|
%endif
|
||
|
|
paddw m0, m6
|
||
|
|
paddw m2, m6
|
||
|
|
psrlw m0, 3
|
||
|
|
psrlw m2, 3
|
||
|
|
packuswb m0, m2
|
||
|
|
CHROMAMC_AVG m0, m4
|
||
|
|
movq [r0 ], m0
|
||
|
|
movhps [r0+r2], m0
|
||
|
|
sub r3d, 2
|
||
|
|
lea r0, [r0+r2*2]
|
||
|
|
jg .next2yrows
|
||
|
|
REP_RET
|
||
|
|
%endmacro
|
||
|
|
|
||
|
|
%macro chroma_mc4_ssse3_func 2
|
||
|
|
cglobal %1_%2_chroma_mc4, 6, 7, 0
|
||
|
|
mov r6, r4
|
||
|
|
shl r4d, 8
|
||
|
|
sub r4d, r6d
|
||
|
|
mov r6, 8
|
||
|
|
add r4d, 8 ; x*288+8
|
||
|
|
sub r6d, r5d
|
||
|
|
imul r6d, r4d ; (8-y)*(x*255+8) = (8-y)*x<<8 | (8-y)*(8-x)
|
||
|
|
imul r4d, r5d ; y *(x*255+8) = y *x<<8 | y *(8-x)
|
||
|
|
|
||
|
|
movd m7, r6d
|
||
|
|
movd m6, r4d
|
||
|
|
movq m5, [pw_32]
|
||
|
|
movd m0, [r1 ]
|
||
|
|
pshufw m7, m7, 0
|
||
|
|
punpcklbw m0, [r1+1]
|
||
|
|
pshufw m6, m6, 0
|
||
|
|
|
||
|
|
.next2rows:
|
||
|
|
movd m1, [r1+r2*1 ]
|
||
|
|
movd m3, [r1+r2*2 ]
|
||
|
|
punpcklbw m1, [r1+r2*1+1]
|
||
|
|
punpcklbw m3, [r1+r2*2+1]
|
||
|
|
lea r1, [r1+r2*2]
|
||
|
|
movq m2, m1
|
||
|
|
movq m4, m3
|
||
|
|
pmaddubsw m0, m7
|
||
|
|
pmaddubsw m1, m6
|
||
|
|
pmaddubsw m2, m7
|
||
|
|
pmaddubsw m3, m6
|
||
|
|
paddw m0, m5
|
||
|
|
paddw m2, m5
|
||
|
|
paddw m1, m0
|
||
|
|
paddw m3, m2
|
||
|
|
psrlw m1, 6
|
||
|
|
movq m0, m4
|
||
|
|
psrlw m3, 6
|
||
|
|
packuswb m1, m1
|
||
|
|
packuswb m3, m3
|
||
|
|
CHROMAMC_AVG m1, [r0 ]
|
||
|
|
CHROMAMC_AVG m3, [r0+r2]
|
||
|
|
movd [r0 ], m1
|
||
|
|
movd [r0+r2], m3
|
||
|
|
sub r3d, 2
|
||
|
|
lea r0, [r0+r2*2]
|
||
|
|
jg .next2rows
|
||
|
|
REP_RET
|
||
|
|
%endmacro
|
||
|
|
|
||
|
|
%define CHROMAMC_AVG NOTHING
|
||
|
|
INIT_XMM ssse3
|
||
|
|
chroma_mc8_ssse3_func put, h264, _rnd
|
||
|
|
chroma_mc8_ssse3_func put, vc1, _nornd
|
||
|
|
INIT_MMX ssse3
|
||
|
|
chroma_mc4_ssse3_func put, h264
|
||
|
|
|
||
|
|
%define CHROMAMC_AVG DIRECT_AVG
|
||
|
|
INIT_XMM ssse3
|
||
|
|
chroma_mc8_ssse3_func avg, h264, _rnd
|
||
|
|
chroma_mc8_ssse3_func avg, vc1, _nornd
|
||
|
|
INIT_MMX ssse3
|
||
|
|
chroma_mc4_ssse3_func avg, h264
|