Debian Codegraph @

src/f32-ppmm/gen/f32-ppmm-4x8-minmax-asm-aarch64-neonfma-cortex-a75-prfm.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/f32-ppmm/4x8-aarch64-neonfma-cortex-a75.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include "xnnpack/assembly.h"
11
12 # void xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a75_prfm(
13 # size_t mr, x0
14 # size_t nc, x1
15 # size_t kc, x2 / x0
16 # const uint8_t* restrict a, x3
17 # const void* restrict w, x4
18 # uint8_t* restrict c, x5
19 # size_t cm_stride, x6
20 # size_t cn_stride, x7
21 # const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) [sp] -> (x8)
22
23 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
24
25 # Register usage
26 # A x3 v0 v1 v2 v3
27 # B x4 v20 v21
28 # C0 x5 v16 v17
29 # C1 x9 v18 v19
30 # C2 x10 v28 v29
31 # C3 x6 v30 v31
32 # Clamp v4 v5
33
34
35 BEGIN_FUNCTION xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a75_prfm
36
37 # Load params pointer
38 LDR x8, [sp]
39
40 # Clamp A and C pointers
41 CMP x0, 2 // if mr < 2
42 ADD x9, x5, x6 // c1 = c0 + cm_stride
43 CSEL x9, x5, x9, LO // c1 = c0
44
45 ADD x10, x9, x6 // c2 = c1 + cm_stride
46 // if mr <= 2
47 CSEL x10, x9, x10, LS // c2 = c1
48
49 # Load min/max values
50 LD2R {v4.4s, v5.4s}, [x8]
51
52 CMP x0, 4 // if mr < 4
53 ADD x6, x10, x6 // c3 = c2 + cm_stride
54 CSEL x6, x10, x6, LO // c3 = c2
55
56 0:
57 # Load initial bias from w into accumulators
58 LDR q16, [x4], 16
59 LDR q17, [x4], 16
60 SUBS x0, x2, 16 // k = kc - 16
61 LDR q0, [x3], 16 // Preload A
62 MOV v18.16b, v16.16b
63 MOV v28.16b, v16.16b
64 LDR q20, [x4], 16 // Preload B0
65 MOV v30.16b, v16.16b
66 MOV v19.16b, v17.16b
67 LDR q21, [x4], 16 // Preload B1
68 MOV v29.16b, v17.16b
69 MOV v31.16b, v17.16b
70 PRFM PLDL1KEEP, [x4, 64] // Prefetch B
71
72 # Is there at least 4 floats (16 bytes) for epilogue?
73 B.LO 5f
74
75 # Is there at least 4 floats (16 bytes) for main loop?
76 SUBS x0, x0, 16 // k -= 16
77 B.LO 2f
78
79 # Main loop - 4 floats of A (16 bytes)
80 1:
81 LDR q22, [x4], 16
82 FMLA v16.4s, v20.4s, v0.s[0]
83 FMLA v18.4s, v20.4s, v0.s[1]
84 LDR q1, [x3], 16
85 FMLA v28.4s, v20.4s, v0.s[2]
86 FMLA v30.4s, v20.4s, v0.s[3]
87 LDR q23, [x4], 16
88 FMLA v17.4s, v21.4s, v0.s[0]
89 FMLA v19.4s, v21.4s, v0.s[1]
90 LDR q24, [x4], 16
91 FMLA v29.4s, v21.4s, v0.s[2]
92 FMLA v31.4s, v21.4s, v0.s[3]
93 LDR q2, [x3], 16
94 FMLA v16.4s, v22.4s, v1.s[0]
95 FMLA v18.4s, v22.4s, v1.s[1]
96 LDR q25, [x4], 16
97 FMLA v28.4s, v22.4s, v1.s[2]
98 FMLA v30.4s, v22.4s, v1.s[3]
99 LDR q26, [x4], 16
100 FMLA v17.4s, v23.4s, v1.s[0]
101 FMLA v19.4s, v23.4s, v1.s[1]
102 LDR q3, [x3], 16
103 FMLA v29.4s, v23.4s, v1.s[2]
104 FMLA v31.4s, v23.4s, v1.s[3]
105 LDR q27, [x4], 16
106 FMLA v16.4s, v24.4s, v2.s[0]
107 FMLA v18.4s, v24.4s, v2.s[1]
108 LDR q0, [x3], 16
109 FMLA v28.4s, v24.4s, v2.s[2]
110 FMLA v30.4s, v24.4s, v2.s[3]
111 LDR q20, [x4], 16
112 FMLA v17.4s, v25.4s, v2.s[0]
113 FMLA v19.4s, v25.4s, v2.s[1]
114 LDR q21, [x4], 16
115 FMLA v29.4s, v25.4s, v2.s[2]
116 FMLA v31.4s, v25.4s, v2.s[3]
117 PRFM PLDL1KEEP, [x4, 128] // Prefetch B0
118 FMLA v16.4s, v26.4s, v3.s[0]
119 FMLA v18.4s, v26.4s, v3.s[1]
120 PRFM PLDL1KEEP, [x4, 192] // Prefetch B1
121 FMLA v28.4s, v26.4s, v3.s[2]
122 FMLA v30.4s, v26.4s, v3.s[3]
123 SUBS x0, x0, 16
124 FMLA v17.4s, v27.4s, v3.s[0]
125 FMLA v19.4s, v27.4s, v3.s[1]
126 FMLA v29.4s, v27.4s, v3.s[2]
127 FMLA v31.4s, v27.4s, v3.s[3]
128 B.HS 1b
129
130 # Epilogue
131 2:
132 LDR q22, [x4], 16
133 FMLA v16.4s, v20.4s, v0.s[0]
134 FMLA v18.4s, v20.4s, v0.s[1]
135 LDR q1, [x3], 16
136 FMLA v28.4s, v20.4s, v0.s[2]
137 FMLA v30.4s, v20.4s, v0.s[3]
138 LDR q23, [x4], 16
139 FMLA v17.4s, v21.4s, v0.s[0]
140 FMLA v19.4s, v21.4s, v0.s[1]
141 LDR q24, [x4], 16
142 FMLA v29.4s, v21.4s, v0.s[2]
143 FMLA v31.4s, v21.4s, v0.s[3]
144 LDR q2, [x3], 16
145 FMLA v16.4s, v22.4s, v1.s[0]
146 FMLA v18.4s, v22.4s, v1.s[1]
147 LDR q25, [x4], 16
148 FMLA v28.4s, v22.4s, v1.s[2]
149 FMLA v30.4s, v22.4s, v1.s[3]
150 LDR q26, [x4], 16
151 FMLA v17.4s, v23.4s, v1.s[0]
152 FMLA v19.4s, v23.4s, v1.s[1]
153 LDR q3, [x3], 16
154 FMLA v29.4s, v23.4s, v1.s[2]
155 FMLA v31.4s, v23.4s, v1.s[3]
156 LDR q27, [x4], 16
157 FMLA v16.4s, v24.4s, v2.s[0]
158 FMLA v18.4s, v24.4s, v2.s[1]
159 FMLA v28.4s, v24.4s, v2.s[2]
160 FMLA v30.4s, v24.4s, v2.s[3]
161 FMLA v17.4s, v25.4s, v2.s[0]
162 FMLA v19.4s, v25.4s, v2.s[1]
163 FMLA v29.4s, v25.4s, v2.s[2]
164 FMLA v31.4s, v25.4s, v2.s[3]
165 FMLA v16.4s, v26.4s, v3.s[0]
166 FMLA v18.4s, v26.4s, v3.s[1]
167 FMLA v28.4s, v26.4s, v3.s[2]
168 FMLA v30.4s, v26.4s, v3.s[3]
169 TST x0, 15
170 FMLA v17.4s, v27.4s, v3.s[0]
171 FMLA v19.4s, v27.4s, v3.s[1]
172 FMLA v29.4s, v27.4s, v3.s[2]
173 FMLA v31.4s, v27.4s, v3.s[3]
174 B.NE 4f
175
176 3:
177 # Clamp
178 FMAX v16.4s, v16.4s, v4.4s
179 SUBS x1, x1, 8
180 FMAX v17.4s, v17.4s, v4.4s
181 FMAX v18.4s, v18.4s, v4.4s
182 FMAX v19.4s, v19.4s, v4.4s
183 FMAX v28.4s, v28.4s, v4.4s
184 FMAX v29.4s, v29.4s, v4.4s
185 FMAX v30.4s, v30.4s, v4.4s
186 FMAX v31.4s, v31.4s, v4.4s
187 FMIN v16.4s, v16.4s, v5.4s
188 FMIN v17.4s, v17.4s, v5.4s
189 FMIN v18.4s, v18.4s, v5.4s
190 FMIN v19.4s, v19.4s, v5.4s
191 FMIN v28.4s, v28.4s, v5.4s
192 FMIN v29.4s, v29.4s, v5.4s
193 FMIN v30.4s, v30.4s, v5.4s
194 FMIN v31.4s, v31.4s, v5.4s
195
196 # Store full 4 x 8
197 B.LO 7f
198
199 ST1 {v16.16b, v17.16b}, [x5], x7
200 ST1 {v18.16b, v19.16b}, [x9], x7
201 SUB x3, x3, x2, lsl #2 // a0 -= kc * 4
202 ST1 {v28.16b, v29.16b}, [x10], x7
203 ST1 {v30.16b, v31.16b}, [x6], x7
204
205 B.HI 0b
206 RET
207
208 # Remainder of 1..7
209 4:
210 LDR q0, [x3], 16
211 LDR q20, [x4], 16
212 LDR q21, [x4], 16
213
214 5:
215 # Is there a remainder?- 2 floats of A (8 bytes)
216 TBZ x0, 3, 6f
217
218 # Remainder- 2 floats of A (8 bytes)
219 LDR q22, [x4], 16
220 FMLA v16.4s, v20.4s, v0.s[0]
221 FMLA v18.4s, v20.4s, v0.s[1]
222 LDR q1, [x3], 16
223 FMLA v28.4s, v20.4s, v0.s[2]
224 FMLA v30.4s, v20.4s, v0.s[3]
225 LDR q23, [x4], 16
226 FMLA v17.4s, v21.4s, v0.s[0]
227 FMLA v19.4s, v21.4s, v0.s[1]
228 FMLA v29.4s, v21.4s, v0.s[2]
229 FMLA v31.4s, v21.4s, v0.s[3]
230 FMLA v16.4s, v22.4s, v1.s[0]
231 FMLA v18.4s, v22.4s, v1.s[1]
232 FMLA v28.4s, v22.4s, v1.s[2]
233 FMLA v30.4s, v22.4s, v1.s[3]
234 FMLA v17.4s, v23.4s, v1.s[0]
235 FMLA v19.4s, v23.4s, v1.s[1]
236 FMLA v29.4s, v23.4s, v1.s[2]
237 FMLA v31.4s, v23.4s, v1.s[3]
238
239 # Is there a remainder?- 1 float of A (4 bytes)
240 TBZ x0, 2, 3b
241
242 LDR q0, [x3], 16
243 LDR q20, [x4], 16
244 LDR q21, [x4], 16
245
246 # Remainder- 1 float of A (4 bytes)
247 6:
248 FMLA v16.4s, v20.4s, v0.s[0]
249 FMLA v18.4s, v20.4s, v0.s[1]
250 FMLA v28.4s, v20.4s, v0.s[2]
251 FMLA v30.4s, v20.4s, v0.s[3]
252 FMLA v17.4s, v21.4s, v0.s[0]
253 FMLA v19.4s, v21.4s, v0.s[1]
254 FMLA v29.4s, v21.4s, v0.s[2]
255 FMLA v31.4s, v21.4s, v0.s[3]
256 B 3b
257
258 # Store odd width
259 7:
260 TBZ x1, 2, 8f
261 STR q16, [x5], 16
262 STR q18, [x9], 16
263 MOV v16.16b, v17.16b
264 MOV v18.16b, v19.16b
265 STR q28, [x10], 16
266 STR q30, [x6], 16
267 MOV v28.16b, v29.16b
268 MOV v30.16b, v31.16b
269
270 8:
271 TBZ x1, 1, 9f
272 STR d16, [x5], 8
273 STR d18, [x9], 8
274 DUP d16, v16.d[1]
275 DUP d18, v18.d[1]
276 STR d28, [x10], 8
277 STR d30, [x6], 8
278 DUP d28, v28.d[1]
279 DUP d30, v30.d[1]
280
281 9:
282 TBZ x1, 0, 10f
283 STR s16, [x5]
284 STR s18, [x9]
285 STR s28, [x10]
286 STR s30, [x6]
287
288 10:
289 RET
290
291 END_FUNCTION xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a75_prfm
292
293 #ifdef __ELF__
294 .section ".note.GNU-stack","",%progbits
295 #endif