| 1 |
/* |
| 2 |
dct64_x86_64_float: SSE optimized dct64 for x86-64 (float output version) |
| 3 |
|
| 4 |
copyright 1995-2026 by the mpg123 project - free software under the terms of the LGPL 2.1 |
| 5 |
see COPYING and AUTHORS files in distribution or http://mpg123.org |
| 6 |
initially written by Taihei Monma |
| 7 |
*/ |
| 8 |
|
| 9 |
#include "x86_64_defs.h" |
| 10 |
#include "mangle.h" |
| 11 |
|
| 12 |
#define samples %rdx |
| 13 |
#define costab %rcx |
| 14 |
#define out0 %rdi |
| 15 |
#define out1 %rsi |
| 16 |
|
| 17 |
/* |
| 18 |
void dct64_real_avx(real *out0, real *out1, real *samples); |
| 19 |
*/ |
| 20 |
|
| 21 |
RODATA |
| 22 |
ALIGN32 |
| 23 |
costab_avx: |
| 24 |
.long 1056974725 |
| 25 |
.long 1057056395 |
| 26 |
.long 1057223771 |
| 27 |
.long 1057485416 |
| 28 |
.long 1057855544 |
| 29 |
.long 1058356026 |
| 30 |
.long 1059019886 |
| 31 |
.long 1059897405 |
| 32 |
.long 1061067246 |
| 33 |
.long 1062657950 |
| 34 |
.long 1064892987 |
| 35 |
.long 1066774581 |
| 36 |
.long 1069414683 |
| 37 |
.long 1073984175 |
| 38 |
.long 1079645762 |
| 39 |
.long 1092815430 |
| 40 |
.long 1057005197 |
| 41 |
.long 1057342072 |
| 42 |
.long 1058087743 |
| 43 |
.long 1059427869 |
| 44 |
.long 1061799040 |
| 45 |
.long 1065862217 |
| 46 |
.long 1071413542 |
| 47 |
.long 1084439708 |
| 48 |
.long 1057128951 |
| 49 |
.long 1058664893 |
| 50 |
.long 1063675095 |
| 51 |
.long 1076102863 |
| 52 |
.long 1057655764 |
| 53 |
.long 1067924853 |
| 54 |
.long 1060439283 |
| 55 |
.long 0 |
| 56 |
.text |
| 57 |
ALIGN16 |
| 58 |
.globl ASM_NAME(INT123_dct64_real_avx) |
| 59 |
ASM_NAME(INT123_dct64_real_avx): |
| 60 |
#ifdef IS_MSABI |
| 61 |
push %rbp |
| 62 |
mov %rsp, %rbp |
| 63 |
sub $112, %rsp |
| 64 |
movaps %xmm6, (%rsp) |
| 65 |
movaps %xmm7, 16(%rsp) |
| 66 |
movaps %xmm8, 32(%rsp) |
| 67 |
movaps %xmm9, 48(%rsp) |
| 68 |
movaps %xmm10, 64(%rsp) |
| 69 |
movaps %xmm11, 80(%rsp) |
| 70 |
movaps %xmm12, 96(%rsp) |
| 71 |
push %rdi |
| 72 |
push %rsi |
| 73 |
mov %rcx, %rdi |
| 74 |
mov %rdx, %rsi |
| 75 |
mov %r8, %rdx |
| 76 |
#endif |
| 77 |
leaq costab_avx(%rip), costab |
| 78 |
|
| 79 |
vmovups (samples), %ymm0 # input[0,1,2,3,4,5,6,7] |
| 80 |
vmovups 32(samples), %ymm1 # input[8,9,10,11,12,13,14,15] |
| 81 |
vperm2f128 $0x23, 64(samples), %ymm2, %ymm2 |
| 82 |
vperm2f128 $0x23, 96(samples), %ymm3, %ymm3 |
| 83 |
vshufps $0x1b, %ymm2, %ymm2, %ymm2 # input[23,22,21,20,19,18,17,16] |
| 84 |
vshufps $0x1b, %ymm3, %ymm3, %ymm3 # input[31,30,29,28,27,26,25,24] |
| 85 |
vsubps %ymm2, %ymm1, %ymm6 |
| 86 |
vsubps %ymm3, %ymm0, %ymm7 |
| 87 |
vaddps %ymm0, %ymm3, %ymm4 # bufs[0,1,2,3,4,5,6,7] |
| 88 |
vaddps %ymm1, %ymm2, %ymm5 # bufs[8,9,10,11,12,13,14,15] |
| 89 |
vmulps (costab), %ymm7, %ymm7 # bufs[31,30,29,28,27,26,25,24] cos64[0,1,2,3,4,5,6,7] |
| 90 |
vmulps 32(costab), %ymm6, %ymm6 # bufs[23,22,21,20,19,18,17,16] cos64[8,9,10,11,12,13,14,15] |
| 91 |
|
| 92 |
vmovaps 64(costab), %ymm8 # cos32[0,1,2,3,4,5,6,7] |
| 93 |
|
| 94 |
vshufps $0x1b, %ymm5, %ymm5, %ymm5 |
| 95 |
vshufps $0x1b, %ymm6, %ymm6, %ymm6 |
| 96 |
vperm2f128 $0x01, %ymm5, %ymm5, %ymm5 # bufs[15,14,13,12,11,10,9,8] |
| 97 |
vperm2f128 $0x01, %ymm6, %ymm6, %ymm6 # bufs[16,17,18,19,20,21,22,23] |
| 98 |
vsubps %ymm5, %ymm4, %ymm1 |
| 99 |
vsubps %ymm6, %ymm7, %ymm3 |
| 100 |
vaddps %ymm5, %ymm4, %ymm0 # bufs[32,33,34,35,36,37,38,39] |
| 101 |
vaddps %ymm6, %ymm7, %ymm2 # bufs[48,49,50,51,52,53,54,55] |
| 102 |
vmulps %ymm1, %ymm8, %ymm1 # bufs[47,46,45,44,43,42,41,40] |
| 103 |
vmulps %ymm3, %ymm8, %ymm3 # bufs[63,62,61,60,59,58,57,56] |
| 104 |
|
| 105 |
vmovaps 96(costab), %ymm8 # cos16[0,1,2,3]:cos8[0,1]:cos4[0]:- |
| 106 |
vperm2f128 $0x00, %ymm8, %ymm8, %ymm9 # cos16[0,1,2,3,0,1,2,3] |
| 107 |
|
| 108 |
vperm2f128 $0x20, %ymm1, %ymm0, %ymm4 # bufs[32,33,34,35,47,46,45,44] |
| 109 |
vperm2f128 $0x31, %ymm1, %ymm0, %ymm5 |
| 110 |
vshufps $0x1b, %ymm5, %ymm5, %ymm5 # bufs[39,38,37,36,40,41,42,43] |
| 111 |
vperm2f128 $0x20, %ymm3, %ymm2, %ymm6 # bufs[48,49,50,51,63,62,61,60] |
| 112 |
vperm2f128 $0x31, %ymm3, %ymm2, %ymm7 |
| 113 |
vshufps $0x1b, %ymm7, %ymm7, %ymm7 # bufs[55,54,53,52,56,57,58,59] |
| 114 |
vsubps %ymm5, %ymm4, %ymm1 |
| 115 |
vsubps %ymm7, %ymm6, %ymm3 |
| 116 |
vaddps %ymm5, %ymm4, %ymm0 # bufs[0,1,2,3,8,9,10,11] |
| 117 |
vaddps %ymm7, %ymm6, %ymm2 # bufs[16,17,18,19,24,25,26,27] |
| 118 |
vmulps %ymm1, %ymm9, %ymm1 # bufs[7,6,5,4,15,14,13,12] |
| 119 |
vmulps %ymm3, %ymm9, %ymm3 # bufs[23,22,21,20,31,30,29,28] |
| 120 |
|
| 121 |
vperm2f128 $0x11, %ymm8, %ymm8, %ymm8 # cos8[0,1]:cos4[0]:-:cos8[0,1]:cos4[0]:- |
| 122 |
vmovddup %ymm8, %ymm9 # cos8[0,1,0,1,0,1,0,1] |
| 123 |
|
| 124 |
vunpcklps %ymm1, %ymm0, %ymm4 # bufs[0,7,1,6,8,15,9,14] |
| 125 |
vunpckhps %ymm1, %ymm0, %ymm5 # bufs[2,5,3,4,10,13,11,12] |
| 126 |
vunpcklps %ymm3, %ymm2, %ymm6 # bufs[16,23,17,22,24,31,25,30] |
| 127 |
vunpckhps %ymm3, %ymm2, %ymm7 # bufs[18,21,19,20,26,29,27,28] |
| 128 |
vshufps $0xd8, %ymm4, %ymm4, %ymm4 # bufs[0,1,7,6,8,9,15,14] |
| 129 |
vshufps $0x72, %ymm5, %ymm5, %ymm5 # bufs[3,2,4,5,11,10,12,13] |
| 130 |
vshufps $0xd8, %ymm6, %ymm6, %ymm6 # bufs[16,17,23,22,24,25,31,30] |
| 131 |
vshufps $0x72, %ymm7, %ymm7, %ymm7 # bufs[19,18,20,21,27,26,28,29] |
| 132 |
vsubps %ymm5, %ymm4, %ymm1 |
| 133 |
vsubps %ymm7, %ymm6, %ymm3 |
| 134 |
vaddps %ymm5, %ymm4, %ymm0 # bufs[32,33,36,37,40,41,44,45] |
| 135 |
vaddps %ymm7, %ymm6, %ymm2 # bufs[48,49,52,53,56,57,60,61] |
| 136 |
vmulps %ymm1, %ymm9, %ymm1 # bufs[35,34,39,38,43,42,47,46] |
| 137 |
vmulps %ymm3, %ymm9, %ymm3 # bufs[51,50,55,54,59,58,63,62] |
| 138 |
|
| 139 |
vpermilps $0xaa, %ymm8, %ymm8 # cos4[0,0,0,0,0,0,0,0] |
| 140 |
|
| 141 |
vshufps $0xd8, %ymm0, %ymm0, %ymm0 # bufs[32,36,33,37,40,44,41,45] |
| 142 |
vshufps $0xd8, %ymm1, %ymm1, %ymm1 # bufs[35,39,34,38,43,47,42,46] |
| 143 |
vshufps $0xd8, %ymm2, %ymm2, %ymm2 # bufs[48,52,49,53,56,60,57,61] |
| 144 |
vshufps $0xd8, %ymm3, %ymm3, %ymm3 # bufs[51,55,50,54,59,63,58,62] |
| 145 |
vunpcklps %ymm1, %ymm0, %ymm4 # bufs[32,35,36,39,40,43,44,47] |
| 146 |
vunpckhps %ymm1, %ymm0, %ymm5 # bufs[33,34,37,38,41,42,45,46] |
| 147 |
vunpcklps %ymm3, %ymm2, %ymm6 # bufs[48,51,52,55,56,59,60,63] |
| 148 |
vunpckhps %ymm3, %ymm2, %ymm7 # bufs[49,50,53,54,57,58,61,62] |
| 149 |
vsubps %ymm5, %ymm4, %ymm1 |
| 150 |
vsubps %ymm7, %ymm6, %ymm3 |
| 151 |
vaddps %ymm5, %ymm4, %ymm0 # bufs[0,2,4,6,8,10,12,14] |
| 152 |
vaddps %ymm7, %ymm6, %ymm2 # bufs[16,18,20,22,24,26,28,30] |
| 153 |
vmulps %ymm1, %ymm8, %ymm1 # bufs[1,3,5,7,9,11,13,15] |
| 154 |
vmulps %ymm3, %ymm8, %ymm3 # bufs[17,19,21,23,25,27,29,31] |
| 155 |
|
| 156 |
vxorps %ymm8, %ymm8, %ymm8 |
| 157 |
vblendps $0xaa, %ymm1, %ymm8, %ymm5 |
| 158 |
vblendps $0xaa, %ymm3, %ymm8, %ymm6 |
| 159 |
vaddps %ymm5, %ymm0, %ymm0 |
| 160 |
vaddps %ymm6, %ymm2, %ymm2 |
| 161 |
vunpcklps %ymm1, %ymm0, %ymm4 # bufs[0,1,2,3,8,9,10,11] |
| 162 |
vunpckhps %ymm1, %ymm0, %ymm5 # bufs[4,5,6,7,12,13,14,15] |
| 163 |
vunpcklps %ymm3, %ymm2, %ymm6 # bufs[16,17,18,19,24,25,26,27] |
| 164 |
vunpckhps %ymm3, %ymm2, %ymm7 # bufs[20,21,22,23,28,29,30,31] |
| 165 |
|
| 166 |
vextractf128 $0x1, %ymm4, %xmm0 # bufs[8,9,10,11] |
| 167 |
vextractf128 $0x1, %ymm5, %xmm1 # bufs[12,13,14,15] |
| 168 |
vextractf128 $0x1, %ymm6, %xmm2 # bufs[24,25,26,27] |
| 169 |
vextractf128 $0x1, %ymm7, %xmm3 # bufs[28,29,30,31] |
| 170 |
|
| 171 |
vshufps $0x1e, %xmm5, %xmm5, %xmm9 # bufs[6,7,5,4] |
| 172 |
vshufps $0x1e, %xmm1, %xmm1, %xmm10 # bufs[14,15,13,12] |
| 173 |
vshufps $0x1e, %xmm7, %xmm7, %xmm11 # bufs[22,23,21,20] |
| 174 |
vshufps $0x1e, %xmm3, %xmm3, %xmm12 # bufs[30,31,29,28] |
| 175 |
vblendps $0x7, %xmm9, %xmm8, %xmm9 # bufs[6,7,5,-] |
| 176 |
vblendps $0x7, %xmm10, %xmm8, %xmm10 # bufs[14,15,13,-] |
| 177 |
vblendps $0x7, %xmm11, %xmm8, %xmm11 # bufs[22,23,21,-] |
| 178 |
vblendps $0x7, %xmm12, %xmm8, %xmm12 # bufs[30,31,29,-] |
| 179 |
vaddps %xmm5, %xmm9, %xmm5 |
| 180 |
vaddps %xmm1, %xmm10, %xmm1 |
| 181 |
vaddps %xmm7, %xmm11, %xmm7 |
| 182 |
vaddps %xmm3, %xmm12, %xmm3 |
| 183 |
|
| 184 |
prefetcht0 1024(out0) |
| 185 |
|
| 186 |
vshufps $0x1e, %xmm0, %xmm0, %xmm9 # bufs[10,11,9,8] |
| 187 |
vshufps $0x1e, %xmm2, %xmm2, %xmm10 # bufs[26,27,25,24] |
| 188 |
vaddps %xmm1, %xmm0, %xmm0 |
| 189 |
vaddps %xmm3, %xmm2, %xmm2 |
| 190 |
vblendps $0x7, %xmm9, %xmm8, %xmm9 # bufs[10,11,9,-] |
| 191 |
vblendps $0x7, %xmm10, %xmm8, %xmm10 # bufs[26,27,25,-] |
| 192 |
vaddps %xmm1, %xmm9, %xmm1 |
| 193 |
vaddps %xmm3, %xmm10, %xmm3 |
| 194 |
|
| 195 |
vzeroupper |
| 196 |
prefetcht0 1024(out1) |
| 197 |
|
| 198 |
addq $1024, out0 |
| 199 |
movq $-128, %rax |
| 200 |
movss %xmm4, (out0) |
| 201 |
movss %xmm0, (out0,%rax,1) |
| 202 |
movss %xmm5, (out0,%rax,2) |
| 203 |
movss %xmm1, -128(out0,%rax,2) |
| 204 |
leaq (out0,%rax,4), out0 |
| 205 |
movhlps %xmm4, %xmm9 |
| 206 |
movhlps %xmm0, %xmm10 |
| 207 |
movhlps %xmm5, %xmm11 |
| 208 |
movhlps %xmm1, %xmm12 |
| 209 |
vmovss %xmm9, (out0) |
| 210 |
vmovss %xmm10, (out0,%rax,1) |
| 211 |
vmovss %xmm11, (out0,%rax,2) |
| 212 |
vmovss %xmm12, -128(out0,%rax,2) |
| 213 |
leaq (out0,%rax,4), out0 |
| 214 |
negq %rax |
| 215 |
shufps $0xb1, %xmm4, %xmm4 |
| 216 |
shufps $0xb1, %xmm0, %xmm0 |
| 217 |
shufps $0xb1, %xmm5, %xmm5 |
| 218 |
shufps $0xb1, %xmm1, %xmm1 |
| 219 |
movss %xmm4, (out0) |
| 220 |
movss %xmm4, (out1) |
| 221 |
leaq (out1,%rax,1), out1 |
| 222 |
movss %xmm0, (out1) |
| 223 |
movss %xmm5, (out1,%rax,1) |
| 224 |
movss %xmm1, (out1,%rax,2) |
| 225 |
leaq (out1,%rax,4), out1 |
| 226 |
movhlps %xmm4, %xmm4 |
| 227 |
movhlps %xmm0, %xmm0 |
| 228 |
movhlps %xmm5, %xmm5 |
| 229 |
movhlps %xmm1, %xmm1 |
| 230 |
movss %xmm4, -128(out1) |
| 231 |
movss %xmm0, (out1) |
| 232 |
movss %xmm5, (out1,%rax,1) |
| 233 |
movss %xmm1, (out1,%rax,2) |
| 234 |
|
| 235 |
leaq -64(out0,%rax,8), out0 |
| 236 |
negq %rax |
| 237 |
vshufps $0x1e, %xmm6, %xmm6, %xmm0 |
| 238 |
vblendps $0x7, %xmm0, %xmm8, %xmm0 |
| 239 |
addps %xmm2, %xmm6 |
| 240 |
addps %xmm7, %xmm2 |
| 241 |
addps %xmm3, %xmm7 |
| 242 |
addps %xmm0, %xmm3 |
| 243 |
movss %xmm6, (out0) |
| 244 |
movss %xmm2, (out0,%rax,1) |
| 245 |
movss %xmm7, (out0,%rax,2) |
| 246 |
movss %xmm3, -128(out0,%rax,2) |
| 247 |
leaq (out0,%rax,4), out0 |
| 248 |
movhlps %xmm6, %xmm0 |
| 249 |
movhlps %xmm2, %xmm1 |
| 250 |
movhlps %xmm7, %xmm4 |
| 251 |
movhlps %xmm3, %xmm5 |
| 252 |
movss %xmm0, (out0) |
| 253 |
movss %xmm1, (out0,%rax,1) |
| 254 |
movss %xmm4, (out0,%rax,2) |
| 255 |
movss %xmm5, -128(out0,%rax,2) |
| 256 |
leaq 64(out1,%rax,4), out1 |
| 257 |
negq %rax |
| 258 |
shufps $0xb1, %xmm6, %xmm6 |
| 259 |
shufps $0xb1, %xmm2, %xmm2 |
| 260 |
shufps $0xb1, %xmm7, %xmm7 |
| 261 |
shufps $0xb1, %xmm3, %xmm3 |
| 262 |
movss %xmm6, -128(out1) |
| 263 |
movss %xmm2, (out1) |
| 264 |
movss %xmm7, (out1,%rax,1) |
| 265 |
movss %xmm3, (out1,%rax,2) |
| 266 |
leaq (out1,%rax,4), out1 |
| 267 |
movhlps %xmm6, %xmm6 |
| 268 |
movhlps %xmm2, %xmm2 |
| 269 |
movhlps %xmm7, %xmm7 |
| 270 |
movhlps %xmm3, %xmm3 |
| 271 |
movss %xmm6, -128(out1) |
| 272 |
movss %xmm2, (out1) |
| 273 |
movss %xmm7, (out1,%rax,1) |
| 274 |
movss %xmm3, (out1,%rax,2) |
| 275 |
|
| 276 |
#ifdef IS_MSABI |
| 277 |
pop %rsi |
| 278 |
pop %rdi |
| 279 |
movaps (%rsp), %xmm6 |
| 280 |
movaps 16(%rsp), %xmm7 |
| 281 |
movaps 32(%rsp), %xmm8 |
| 282 |
movaps 48(%rsp), %xmm9 |
| 283 |
movaps 64(%rsp), %xmm10 |
| 284 |
movaps 80(%rsp), %xmm11 |
| 285 |
movaps 96(%rsp), %xmm12 |
| 286 |
mov %rbp, %rsp |
| 287 |
pop %rbp |
| 288 |
#endif |
| 289 |
ret |
| 290 |
|
| 291 |
NONEXEC_STACK |