| 1 |
// Auto-generated file. Do not edit! |
| 2 |
// Template: src/f32-ppmm/8x8-aarch64-neonfma-ld128.S.in |
| 3 |
// Generator: tools/xngen |
| 4 |
// |
| 5 |
// Copyright 2023 Google LLC |
| 6 |
// |
| 7 |
// This source code is licensed under the BSD-style license found in the |
| 8 |
// LICENSE file in the root directory of this source tree. |
| 9 |
|
| 10 |
#include "xnnpack/assembly.h" |
| 11 |
|
| 12 |
# void xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_ld128( |
| 13 |
# size_t mr, x0 |
| 14 |
# size_t nc, x1 |
| 15 |
# size_t kc, x2 / x0 |
| 16 |
# const uint8_t* restrict a, x3 |
| 17 |
# const void* restrict w, x4 |
| 18 |
# uint8_t* restrict c, x5 |
| 19 |
# size_t cm_stride, x6 |
| 20 |
# size_t cn_stride, x7 |
| 21 |
# const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) [sp] -> x8 |
| 22 |
|
| 23 |
# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS. |
| 24 |
|
| 25 |
# Vector register usage |
| 26 |
# A x3 v0 v1 |
| 27 |
# B x4 v4 v5 |
| 28 |
# C0 x5 v16 v17 |
| 29 |
# C1 x12 v18 v19 |
| 30 |
# C2 x13 v20 v21 |
| 31 |
# C3 x14 v22 v23 |
| 32 |
# C4 x15 v24 v25 |
| 33 |
# C5 x16 v26 v27 |
| 34 |
# C6 x17 v28 v29 |
| 35 |
# C7 x6 v30 v31 |
| 36 |
# Clamp v6 v7 |
| 37 |
|
| 38 |
BEGIN_FUNCTION xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_ld128 |
| 39 |
|
| 40 |
# Load params pointer |
| 41 |
LDR x8, [sp] |
| 42 |
|
| 43 |
# Clamp A and C pointers |
| 44 |
CMP x0, 2 // if mr < 2 |
| 45 |
ADD x12, x5, x6 // c1 = c0 + cm_stride |
| 46 |
CSEL x12, x5, x12, LO // c1 = c0 |
| 47 |
ADD x13, x12, x6 // c2 = c1 + cm_stride |
| 48 |
// if mr <= 2 |
| 49 |
CSEL x13, x12, x13, LS // c2 = c1 |
| 50 |
CMP x0, 4 // if mr < 4 |
| 51 |
ADD x14, x13, x6 // c3 = c2 + cm_stride |
| 52 |
CSEL x14, x13, x14, LO // c3 = c2 |
| 53 |
|
| 54 |
# Load min/max values |
| 55 |
LD2R {v6.4s, v7.4s}, [x8] |
| 56 |
|
| 57 |
ADD x15, x14, x6 // c4 = c3 + cm_stride |
| 58 |
// if mr <= 4 |
| 59 |
CSEL x15, x14, x15, LS // c4 = c3 |
| 60 |
CMP x0, 6 // if mr < 6 |
| 61 |
ADD x16, x15, x6 // c5 = c4 + cm_stride |
| 62 |
CSEL x16, x15, x16, LO // c5 = c4 |
| 63 |
ADD x17, x16, x6 // c6 = c5 + cm_stride |
| 64 |
// if mr <= 4 |
| 65 |
CSEL x17, x16, x17, LS // c6 = c5 |
| 66 |
CMP x0, 8 // if mr < 8 |
| 67 |
ADD x6, x17, x6 // c7 = c6 + cm_stride |
| 68 |
CSEL x6, x17, x6, LO // c7 = c6 |
| 69 |
|
| 70 |
|
| 71 |
0: |
| 72 |
# Load initial bias from w into accumulators |
| 73 |
LDR q16, [x4], 16 |
| 74 |
LDR q17, [x4], 16 |
| 75 |
SUB x0, x2, 4 // k = kc - 4 |
| 76 |
MOV v18.16b, v16.16b |
| 77 |
MOV v20.16b, v16.16b |
| 78 |
MOV v22.16b, v16.16b |
| 79 |
MOV v24.16b, v16.16b |
| 80 |
MOV v26.16b, v16.16b |
| 81 |
MOV v28.16b, v16.16b |
| 82 |
MOV v30.16b, v16.16b |
| 83 |
MOV v19.16b, v17.16b |
| 84 |
MOV v21.16b, v17.16b |
| 85 |
MOV v23.16b, v17.16b |
| 86 |
MOV v25.16b, v17.16b |
| 87 |
MOV v27.16b, v17.16b |
| 88 |
MOV v29.16b, v17.16b |
| 89 |
MOV v31.16b, v17.16b |
| 90 |
|
| 91 |
# Main loop - 1 float of A (4 bytes) |
| 92 |
1: |
| 93 |
LDR q0, [x3], 16 |
| 94 |
LDR q4, [x4], 16 |
| 95 |
LDR q1, [x3], 16 |
| 96 |
LDR q5, [x4], 16 |
| 97 |
FMLA v16.4s, v4.4s, v0.s[0] |
| 98 |
FMLA v18.4s, v4.4s, v0.s[1] |
| 99 |
FMLA v20.4s, v4.4s, v0.s[2] |
| 100 |
FMLA v22.4s, v4.4s, v0.s[3] |
| 101 |
FMLA v24.4s, v4.4s, v1.s[0] |
| 102 |
FMLA v26.4s, v4.4s, v1.s[1] |
| 103 |
FMLA v28.4s, v4.4s, v1.s[2] |
| 104 |
FMLA v30.4s, v4.4s, v1.s[3] |
| 105 |
FMLA v17.4s, v5.4s, v0.s[0] |
| 106 |
FMLA v19.4s, v5.4s, v0.s[1] |
| 107 |
FMLA v21.4s, v5.4s, v0.s[2] |
| 108 |
FMLA v23.4s, v5.4s, v0.s[3] |
| 109 |
SUBS x0, x0, 4 |
| 110 |
FMLA v25.4s, v5.4s, v1.s[0] |
| 111 |
FMLA v27.4s, v5.4s, v1.s[1] |
| 112 |
FMLA v29.4s, v5.4s, v1.s[2] |
| 113 |
FMLA v31.4s, v5.4s, v1.s[3] |
| 114 |
B.HS 1b |
| 115 |
|
| 116 |
# Clamp |
| 117 |
FMAX v16.4s, v16.4s, v6.4s |
| 118 |
FMAX v17.4s, v17.4s, v6.4s |
| 119 |
FMAX v18.4s, v18.4s, v6.4s |
| 120 |
FMAX v19.4s, v19.4s, v6.4s |
| 121 |
FMAX v20.4s, v20.4s, v6.4s |
| 122 |
FMAX v21.4s, v21.4s, v6.4s |
| 123 |
FMAX v22.4s, v22.4s, v6.4s |
| 124 |
FMAX v23.4s, v23.4s, v6.4s |
| 125 |
FMAX v24.4s, v24.4s, v6.4s |
| 126 |
FMAX v25.4s, v25.4s, v6.4s |
| 127 |
FMAX v26.4s, v26.4s, v6.4s |
| 128 |
FMAX v27.4s, v27.4s, v6.4s |
| 129 |
FMAX v28.4s, v28.4s, v6.4s |
| 130 |
FMAX v29.4s, v29.4s, v6.4s |
| 131 |
FMAX v30.4s, v30.4s, v6.4s |
| 132 |
FMAX v31.4s, v31.4s, v6.4s |
| 133 |
SUBS x1, x1, 8 |
| 134 |
FMIN v16.4s, v16.4s, v7.4s |
| 135 |
FMIN v17.4s, v17.4s, v7.4s |
| 136 |
FMIN v18.4s, v18.4s, v7.4s |
| 137 |
FMIN v19.4s, v19.4s, v7.4s |
| 138 |
FMIN v20.4s, v20.4s, v7.4s |
| 139 |
FMIN v21.4s, v21.4s, v7.4s |
| 140 |
FMIN v22.4s, v22.4s, v7.4s |
| 141 |
FMIN v23.4s, v23.4s, v7.4s |
| 142 |
FMIN v24.4s, v24.4s, v7.4s |
| 143 |
FMIN v25.4s, v25.4s, v7.4s |
| 144 |
FMIN v26.4s, v26.4s, v7.4s |
| 145 |
FMIN v27.4s, v27.4s, v7.4s |
| 146 |
FMIN v28.4s, v28.4s, v7.4s |
| 147 |
FMIN v29.4s, v29.4s, v7.4s |
| 148 |
FMIN v30.4s, v30.4s, v7.4s |
| 149 |
FMIN v31.4s, v31.4s, v7.4s |
| 150 |
|
| 151 |
# Store full 8 x 8 |
| 152 |
B.LO 2f |
| 153 |
|
| 154 |
ST1 {v16.16b, v17.16b}, [x5], x7 |
| 155 |
ST1 {v18.16b, v19.16b}, [x12], x7 |
| 156 |
ST1 {v20.16b, v21.16b}, [x13], x7 |
| 157 |
ST1 {v22.16b, v23.16b}, [x14], x7 |
| 158 |
SUB x3, x3, x2, lsl #3 // a0 -= kc * 8 |
| 159 |
ST1 {v24.16b, v25.16b}, [x15], x7 |
| 160 |
ST1 {v26.16b, v27.16b}, [x16], x7 |
| 161 |
ST1 {v28.16b, v29.16b}, [x17], x7 |
| 162 |
ST1 {v30.16b, v31.16b}, [x6], x7 |
| 163 |
B.HI 0b |
| 164 |
RET |
| 165 |
|
| 166 |
# Store odd width |
| 167 |
2: |
| 168 |
TBZ x1, 2, 3f |
| 169 |
STR q16, [x5], 16 |
| 170 |
STR q18, [x12], 16 |
| 171 |
MOV v16.16b, v17.16b |
| 172 |
MOV v18.16b, v19.16b |
| 173 |
STR q20, [x13], 16 |
| 174 |
STR q22, [x14], 16 |
| 175 |
MOV v20.16b, v21.16b |
| 176 |
MOV v22.16b, v23.16b |
| 177 |
STR q24, [x15], 16 |
| 178 |
STR q26, [x16], 16 |
| 179 |
MOV v24.16b, v25.16b |
| 180 |
MOV v26.16b, v27.16b |
| 181 |
STR q28, [x17], 16 |
| 182 |
STR q30, [x6], 16 |
| 183 |
MOV v28.16b, v29.16b |
| 184 |
MOV v30.16b, v31.16b |
| 185 |
3: |
| 186 |
TBZ x1, 1, 4f |
| 187 |
STR d16, [x5], 8 |
| 188 |
STR d18, [x12], 8 |
| 189 |
DUP d16, v16.d[1] |
| 190 |
DUP d18, v18.d[1] |
| 191 |
STR d20, [x13], 8 |
| 192 |
STR d22, [x14], 8 |
| 193 |
DUP d20, v20.d[1] |
| 194 |
DUP d22, v22.d[1] |
| 195 |
STR d24, [x15], 8 |
| 196 |
STR d26, [x16], 8 |
| 197 |
DUP d24, v24.d[1] |
| 198 |
DUP d26, v26.d[1] |
| 199 |
STR d28, [x17], 8 |
| 200 |
STR d30, [x6], 8 |
| 201 |
DUP d28, v28.d[1] |
| 202 |
DUP d30, v30.d[1] |
| 203 |
|
| 204 |
4: |
| 205 |
TBZ x1, 0, 5f |
| 206 |
STR s16, [x5] |
| 207 |
STR s18, [x12] |
| 208 |
STR s20, [x13] |
| 209 |
STR s22, [x14] |
| 210 |
STR s24, [x15] |
| 211 |
STR s26, [x16] |
| 212 |
STR s28, [x17] |
| 213 |
STR s30, [x6] |
| 214 |
|
| 215 |
5: |
| 216 |
RET |
| 217 |
|
| 218 |
END_FUNCTION xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_ld128 |
| 219 |
|
| 220 |
#ifdef __ELF__ |
| 221 |
.section ".note.GNU-stack","",%progbits |
| 222 |
#endif |