| 1 |
/* |
| 2 |
dct64_neon64: NEON optimized dct64 for AArch64 |
| 3 |
|
| 4 |
copyright 1995-2014 by the mpg123 project - free software under the terms of the LGPL 2.1 |
| 5 |
see COPYING and AUTHORS files in distribution or http://mpg123.org |
| 6 |
initially written by Taihei Monma |
| 7 |
*/ |
| 8 |
|
| 9 |
#include "mangle.h" |
| 10 |
|
| 11 |
RODATA |
| 12 |
ALIGN16 |
| 13 |
costab_neon_aarch64: |
| 14 |
.word 1056974725 |
| 15 |
.word 1057056395 |
| 16 |
.word 1057223771 |
| 17 |
.word 1057485416 |
| 18 |
.word 1057855544 |
| 19 |
.word 1058356026 |
| 20 |
.word 1059019886 |
| 21 |
.word 1059897405 |
| 22 |
.word 1061067246 |
| 23 |
.word 1062657950 |
| 24 |
.word 1064892987 |
| 25 |
.word 1066774581 |
| 26 |
.word 1069414683 |
| 27 |
.word 1073984175 |
| 28 |
.word 1079645762 |
| 29 |
.word 1092815430 |
| 30 |
.word 1057005197 |
| 31 |
.word 1057342072 |
| 32 |
.word 1058087743 |
| 33 |
.word 1059427869 |
| 34 |
.word 1061799040 |
| 35 |
.word 1065862217 |
| 36 |
.word 1071413542 |
| 37 |
.word 1084439708 |
| 38 |
.word 1057128951 |
| 39 |
.word 1058664893 |
| 40 |
.word 1063675095 |
| 41 |
.word 1076102863 |
| 42 |
.word 1057655764 |
| 43 |
.word 1067924853 |
| 44 |
.word 1060439283 |
| 45 |
.word 1060439283 |
| 46 |
.text |
| 47 |
ALIGN4 |
| 48 |
.globl ASM_NAME(INT123_dct64_neon64) |
| 49 |
#ifdef __ELF__ |
| 50 |
.type ASM_NAME(INT123_dct64_neon64), %function |
| 51 |
#endif |
| 52 |
ASM_NAME(INT123_dct64_neon64): |
| 53 |
add x3, x2, #64 |
| 54 |
adrp x4, AARCH64_PCREL_HI(costab_neon_aarch64) |
| 55 |
add x4, x4, AARCH64_PCREL_LO(costab_neon_aarch64) |
| 56 |
ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x2] |
| 57 |
ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x3] |
| 58 |
ld1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x4], #64 |
| 59 |
|
| 60 |
rev64 v19.4s, v19.4s |
| 61 |
rev64 v18.4s, v18.4s |
| 62 |
rev64 v17.4s, v17.4s |
| 63 |
rev64 v16.4s, v16.4s |
| 64 |
ext v4.16b, v19.16b, v19.16b, #8 |
| 65 |
ext v5.16b, v18.16b, v18.16b, #8 |
| 66 |
ext v6.16b, v17.16b, v17.16b, #8 |
| 67 |
ext v7.16b, v16.16b, v16.16b, #8 |
| 68 |
|
| 69 |
fsub v16.4s, v3.4s, v7.4s |
| 70 |
fsub v17.4s, v2.4s, v6.4s |
| 71 |
fsub v18.4s, v1.4s, v5.4s |
| 72 |
fsub v19.4s, v0.4s, v4.4s |
| 73 |
fadd v0.4s, v0.4s, v4.4s /* bs[0,1,2,3] */ |
| 74 |
fadd v1.4s, v1.4s, v5.4s /* bs[4,5,6,7] */ |
| 75 |
fadd v2.4s, v2.4s, v6.4s /* bs[8,9,10,11] */ |
| 76 |
fadd v3.4s, v3.4s, v7.4s /* bs[12,13,14,15] */ |
| 77 |
fmul v16.4s, v16.4s, v23.4s /* bs[19,18,17,16] */ |
| 78 |
fmul v17.4s, v17.4s, v22.4s /* bs[23,22,21,20] */ |
| 79 |
fmul v18.4s, v18.4s, v21.4s /* bs[27,26,25,24] */ |
| 80 |
fmul v19.4s, v19.4s, v20.4s /* bs[31,30,29,28] */ |
| 81 |
|
| 82 |
ld1 {v20.4s, v21.4s}, [x4], #32 |
| 83 |
rev64 v22.4s, v3.4s |
| 84 |
rev64 v23.4s, v2.4s |
| 85 |
rev64 v24.4s, v16.4s |
| 86 |
rev64 v25.4s, v17.4s |
| 87 |
ext v4.16b, v22.16b, v22.16b, #8 /* bs[15,14,13,12] */ |
| 88 |
ext v5.16b, v23.16b, v23.16b, #8 /* bs[11,10,9,8] */ |
| 89 |
ext v6.16b, v24.16b, v24.16b, #8 /* bs[16,17,18,19] */ |
| 90 |
ext v7.16b, v25.16b, v25.16b, #8 /* bs[20,21,22,23] */ |
| 91 |
|
| 92 |
fsub v26.4s, v1.4s, v5.4s |
| 93 |
fsub v27.4s, v0.4s, v4.4s |
| 94 |
fsub v28.4s, v18.4s, v7.4s |
| 95 |
fsub v29.4s, v19.4s, v6.4s |
| 96 |
fadd v4.4s, v0.4s, v4.4s /* bs[32,33,34,35] */ |
| 97 |
fadd v5.4s, v1.4s, v5.4s /* bs[36,37,38,39] */ |
| 98 |
fadd v6.4s, v6.4s, v19.4s /* bs[48,49,50,51] */ |
| 99 |
fadd v7.4s, v7.4s, v18.4s /* bs[52,53,54,55] */ |
| 100 |
fmul v26.4s, v26.4s, v21.4s /* bs[43,42,41,40] */ |
| 101 |
fmul v27.4s, v27.4s, v20.4s /* bs[47,46,45,44] */ |
| 102 |
fmul v28.4s, v28.4s, v21.4s /* bs[59,58,57,56] */ |
| 103 |
fmul v29.4s, v29.4s, v20.4s /* bs[63,62,61,60] */ |
| 104 |
|
| 105 |
ld1 {v20.4s}, [x4], #16 |
| 106 |
rev64 v16.4s, v5.4s |
| 107 |
rev64 v17.4s, v26.4s |
| 108 |
rev64 v18.4s, v7.4s |
| 109 |
rev64 v19.4s, v28.4s |
| 110 |
ext v0.16b, v16.16b, v16.16b, #8 /* bs[39,38,37,36] */ |
| 111 |
ext v1.16b, v17.16b, v17.16b, #8 /* bs[40,41,42,43] */ |
| 112 |
ext v2.16b, v18.16b, v18.16b, #8 /* bs[55,54,53,52] */ |
| 113 |
ext v3.16b, v19.16b, v19.16b, #8 /* bs[56,57,58,59] */ |
| 114 |
|
| 115 |
fsub v16.4s, v4.4s, v0.4s |
| 116 |
fsub v17.4s, v27.4s, v1.4s |
| 117 |
fsub v18.4s, v6.4s, v2.4s |
| 118 |
fsub v19.4s, v29.4s, v3.4s |
| 119 |
fadd v0.4s, v4.4s, v0.4s /* bs[0,1,2,3] */ |
| 120 |
fadd v1.4s, v1.4s, v27.4s /* bs[8,9,10,11] */ |
| 121 |
fadd v2.4s, v6.4s, v2.4s /* bs[16,17,18,19] */ |
| 122 |
fadd v3.4s, v3.4s, v29.4s /* bs[24,25,26,27] */ |
| 123 |
fmul v16.4s, v16.4s, v20.4s /* bs[7,6,5,4] */ |
| 124 |
fmul v17.4s, v17.4s, v20.4s /* bs[15,14,13,12] */ |
| 125 |
fmul v18.4s, v18.4s, v20.4s /* bs[23,22,21,20] */ |
| 126 |
fmul v19.4s, v19.4s, v20.4s /* bs[31,30,29,28] */ |
| 127 |
|
| 128 |
ld1 {v28.4s}, [x4] |
| 129 |
zip1 v4.2d, v0.2d, v16.2d /* bs[0,1,7,6] */ |
| 130 |
zip2 v5.2d, v0.2d, v16.2d /* bs[2,3,5,4] */ |
| 131 |
zip1 v6.2d, v1.2d, v17.2d /* bs[8,9,15,14] */ |
| 132 |
zip2 v7.2d, v1.2d, v17.2d /* bs[10,11,13,12] */ |
| 133 |
zip1 v20.2d, v2.2d, v18.2d /* bs[16,17,23,22] */ |
| 134 |
zip2 v21.2d, v2.2d, v18.2d /* bs[18,19,21,20] */ |
| 135 |
zip1 v22.2d, v3.2d, v19.2d /* bs[24,25,31,30] */ |
| 136 |
zip2 v23.2d, v3.2d, v19.2d /* bs[26,27,29,28] */ |
| 137 |
rev64 v5.4s, v5.4s /* bs[3,2,4,5] */ |
| 138 |
rev64 v7.4s, v7.4s /* bs[11,10,12,13] */ |
| 139 |
rev64 v21.4s, v21.4s /* bs[19,18,20,21] */ |
| 140 |
rev64 v23.4s, v23.4s /* bs[27,26,28,29] */ |
| 141 |
AARCH64_DUP_2D(v29, v28, 0) |
| 142 |
AARCH64_DUP_4S(v28, v28, 2) |
| 143 |
|
| 144 |
fsub v16.4s, v4.4s, v5.4s |
| 145 |
fsub v17.4s, v6.4s, v7.4s |
| 146 |
fsub v18.4s, v20.4s, v21.4s |
| 147 |
fsub v19.4s, v22.4s, v23.4s |
| 148 |
fadd v0.4s, v4.4s, v5.4s /* bs[32,33,36,37] */ |
| 149 |
fadd v1.4s, v6.4s, v7.4s /* bs[40,41,44,45] */ |
| 150 |
fadd v2.4s, v20.4s, v21.4s /* bs[48,49,52,53] */ |
| 151 |
fadd v3.4s, v22.4s, v23.4s /* bs[56,57,60,61] */ |
| 152 |
fmul v16.4s, v16.4s, v29.4s /* bs[35,34,39,38] */ |
| 153 |
fmul v17.4s, v17.4s, v29.4s /* bs[43,42,47,46] */ |
| 154 |
fmul v18.4s, v18.4s, v29.4s /* bs[51,50,55,54] */ |
| 155 |
fmul v19.4s, v19.4s, v29.4s /* bs[59,58,63,62] */ |
| 156 |
|
| 157 |
uzp1 v4.4s, v0.4s, v16.4s /* bs[32,36,35,39] */ |
| 158 |
uzp2 v5.4s, v0.4s, v16.4s /* bs[33,37,34,38] */ |
| 159 |
uzp1 v6.4s, v1.4s, v17.4s /* bs[40,44,43,47] */ |
| 160 |
uzp2 v7.4s, v1.4s, v17.4s /* bs[41,45,42,46] */ |
| 161 |
uzp1 v20.4s, v2.4s, v18.4s /* bs[48,52,51,55] */ |
| 162 |
uzp2 v21.4s, v2.4s, v18.4s /* bs[49,53,50,54] */ |
| 163 |
uzp1 v22.4s, v3.4s, v19.4s /* bs[56,60,59,63] */ |
| 164 |
uzp2 v23.4s, v3.4s, v19.4s /* bs[57,61,58,62] */ |
| 165 |
|
| 166 |
fsub v16.4s, v4.4s, v5.4s |
| 167 |
fsub v17.4s, v6.4s, v7.4s |
| 168 |
fsub v18.4s, v20.4s, v21.4s |
| 169 |
fsub v19.4s, v22.4s, v23.4s |
| 170 |
fadd v0.4s, v4.4s, v5.4s /* bs[0,4,2,6] */ |
| 171 |
fadd v1.4s, v6.4s, v7.4s /* bs[8,12,10,14] */ |
| 172 |
fadd v2.4s, v20.4s, v21.4s /* bs[16,20,18,22] */ |
| 173 |
fadd v3.4s, v22.4s, v23.4s /* bs[24,28,26,30] */ |
| 174 |
fmul v16.4s, v16.4s, v28.4s /* bs[1,5,3,7] */ |
| 175 |
fmul v17.4s, v17.4s, v28.4s /* bs[9,13,11,15] */ |
| 176 |
fmul v18.4s, v18.4s, v28.4s /* bs[17,21,19,23] */ |
| 177 |
fmul v19.4s, v19.4s, v28.4s /* bs[25,29,27,31] */ |
| 178 |
|
| 179 |
zip2 v4.2d, v0.2d, v1.2d /* bs[2,6,10,14] */ |
| 180 |
zip2 v5.2d, v16.2d, v17.2d /* bs[3,7,11,15] */ |
| 181 |
zip2 v6.2d, v2.2d, v3.2d /* bs[18,22,26,30] */ |
| 182 |
zip2 v7.2d, v18.2d, v19.2d /* bs[19,23,27,31] */ |
| 183 |
fadd v4.4s, v4.4s, v5.4s /* bs[2,6,10,14] */ |
| 184 |
fadd v6.4s, v6.4s, v7.4s /* bs[18,22,26,30] */ |
| 185 |
ins v0.d[1], v4.d[0] /* bs[0,4,2,6] */ |
| 186 |
ins v1.d[1], v4.d[1] /* bs[8,12,10,14] */ |
| 187 |
ins v2.d[1], v6.d[0] /* bs[16,20,18,22] */ |
| 188 |
ins v3.d[1], v6.d[1] /* bs[24,28,26,30] */ |
| 189 |
|
| 190 |
eor v31.16b, v31.16b, v31.16b |
| 191 |
zip1 v4.4s, v0.4s, v16.4s /* bs[0,1,4,5] */ |
| 192 |
zip2 v5.4s, v0.4s, v16.4s /* bs[2,3,6,7] */ |
| 193 |
zip1 v6.4s, v1.4s, v17.4s /* bs[8,9,12,13] */ |
| 194 |
zip2 v7.4s, v1.4s, v17.4s /* bs[10,11,14,15] */ |
| 195 |
zip1 v20.4s, v2.4s, v18.4s /* bs[16,17,20,21] */ |
| 196 |
zip2 v21.4s, v2.4s, v18.4s /* bs[18,19,22,23] */ |
| 197 |
zip1 v22.4s, v3.4s, v19.4s /* bs[24,25,28,29] */ |
| 198 |
zip2 v23.4s, v3.4s, v19.4s /* bs[26,27,30,31] */ |
| 199 |
zip1 v0.2d, v4.2d, v5.2d /* bs[0,1,2,3] */ |
| 200 |
zip2 v1.2d, v4.2d, v5.2d /* bs[4,5,6,7] */ |
| 201 |
zip1 v2.2d, v6.2d, v7.2d /* bs[8,9,10,11] */ |
| 202 |
zip2 v3.2d, v6.2d, v7.2d /* bs[12,13,14,15] */ |
| 203 |
rev64 v16.4s, v4.4s |
| 204 |
rev64 v17.4s, v6.4s |
| 205 |
zip1 v24.2d, v7.2d, v17.2d |
| 206 |
zip2 v16.2d, v5.2d, v16.2d |
| 207 |
zip2 v17.2d, v7.2d, v17.2d |
| 208 |
zip1 v4.2d, v20.2d, v21.2d /* bs[16,17,18,19] */ |
| 209 |
zip2 v5.2d, v20.2d, v21.2d /* bs[20,21,22,23] */ |
| 210 |
zip1 v6.2d, v22.2d, v23.2d /* bs[24,25,26,27] */ |
| 211 |
zip2 v7.2d, v22.2d, v23.2d /* bs[28,29,30,31] */ |
| 212 |
rev64 v18.4s, v20.4s |
| 213 |
rev64 v19.4s, v22.4s |
| 214 |
zip1 v25.2d, v23.2d, v19.2d |
| 215 |
zip1 v26.2d, v21.2d, v18.2d |
| 216 |
zip2 v18.2d, v21.2d, v18.2d |
| 217 |
zip2 v19.2d, v23.2d, v19.2d |
| 218 |
ins v16.s[3], v31.s[0] /* bs[6,7,5,-] */ |
| 219 |
ins v17.s[3], v31.s[0] /* bs[14,15,13,-] */ |
| 220 |
ins v18.s[3], v31.s[0] /* bs[22,23,21,-] */ |
| 221 |
ins v19.s[3], v31.s[0] /* bs[30,31,29,-] */ |
| 222 |
ins v24.s[3], v31.s[0] /* bs[10,11,9,-] */ |
| 223 |
ins v25.s[3], v31.s[0] /* bs[26,27,25,-] */ |
| 224 |
ins v26.s[3], v31.s[0] /* bs[18,19,17,-] */ |
| 225 |
|
| 226 |
fadd v1.4s, v1.4s, v16.4s |
| 227 |
fadd v3.4s, v3.4s, v17.4s |
| 228 |
fadd v5.4s, v5.4s, v18.4s |
| 229 |
fadd v7.4s, v7.4s, v19.4s |
| 230 |
|
| 231 |
fadd v2.4s, v2.4s, v3.4s |
| 232 |
fadd v3.4s, v3.4s, v24.4s |
| 233 |
fadd v6.4s, v6.4s, v7.4s |
| 234 |
fadd v7.4s, v7.4s, v25.4s |
| 235 |
|
| 236 |
fadd v4.4s, v4.4s, v6.4s |
| 237 |
fadd v6.4s, v6.4s, v5.4s |
| 238 |
fadd v5.4s, v5.4s, v7.4s |
| 239 |
fadd v7.4s, v7.4s, v26.4s |
| 240 |
|
| 241 |
fcvtns v0.4s, v0.4s |
| 242 |
fcvtns v1.4s, v1.4s |
| 243 |
fcvtns v2.4s, v2.4s |
| 244 |
fcvtns v3.4s, v3.4s |
| 245 |
fcvtns v4.4s, v4.4s |
| 246 |
fcvtns v5.4s, v5.4s |
| 247 |
fcvtns v6.4s, v6.4s |
| 248 |
fcvtns v7.4s, v7.4s |
| 249 |
sqxtn v0.4h, v0.4s |
| 250 |
sqxtn v1.4h, v1.4s |
| 251 |
sqxtn v2.4h, v2.4s |
| 252 |
sqxtn v3.4h, v3.4s |
| 253 |
sqxtn v4.4h, v4.4s |
| 254 |
sqxtn v5.4h, v5.4s |
| 255 |
sqxtn v6.4h, v6.4s |
| 256 |
sqxtn v7.4h, v7.4s |
| 257 |
|
| 258 |
mov x3, #32 |
| 259 |
st1 {v0.h}[1], [x0], x3 |
| 260 |
st1 {v7.h}[2], [x0], x3 |
| 261 |
st1 {v3.h}[2], [x0], x3 |
| 262 |
st1 {v5.h}[2], [x0], x3 |
| 263 |
st1 {v1.h}[2], [x0], x3 |
| 264 |
st1 {v6.h}[2], [x0], x3 |
| 265 |
st1 {v2.h}[2], [x0], x3 |
| 266 |
st1 {v4.h}[2], [x0], x3 |
| 267 |
st1 {v0.h}[2], [x0], x3 |
| 268 |
st1 {v7.h}[0], [x0], x3 |
| 269 |
st1 {v3.h}[0], [x0], x3 |
| 270 |
st1 {v5.h}[0], [x0], x3 |
| 271 |
st1 {v1.h}[0], [x0], x3 |
| 272 |
st1 {v6.h}[0], [x0], x3 |
| 273 |
st1 {v2.h}[0], [x0], x3 |
| 274 |
st1 {v4.h}[0], [x0], x3 |
| 275 |
st1 {v0.h}[0], [x0] |
| 276 |
st1 {v0.h}[1], [x1], x3 |
| 277 |
st1 {v4.h}[1], [x1], x3 |
| 278 |
st1 {v2.h}[1], [x1], x3 |
| 279 |
st1 {v6.h}[1], [x1], x3 |
| 280 |
st1 {v1.h}[1], [x1], x3 |
| 281 |
st1 {v5.h}[1], [x1], x3 |
| 282 |
st1 {v3.h}[1], [x1], x3 |
| 283 |
st1 {v7.h}[1], [x1], x3 |
| 284 |
st1 {v0.h}[3], [x1], x3 |
| 285 |
st1 {v4.h}[3], [x1], x3 |
| 286 |
st1 {v2.h}[3], [x1], x3 |
| 287 |
st1 {v6.h}[3], [x1], x3 |
| 288 |
st1 {v1.h}[3], [x1], x3 |
| 289 |
st1 {v5.h}[3], [x1], x3 |
| 290 |
st1 {v3.h}[3], [x1], x3 |
| 291 |
st1 {v7.h}[3], [x1] |
| 292 |
|
| 293 |
ret |
| 294 |
|
| 295 |
NONEXEC_STACK |