Debian Codegraph @

src/f32-ppmm/gen/f32-ppmm-4x8-minmax-asm-aarch64-neonfma-cortex-a75.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/f32-ppmm/4x8-aarch64-neonfma-cortex-a75.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include "xnnpack/assembly.h"
11
12 # void xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a75(
13 # size_t mr, x0
14 # size_t nc, x1
15 # size_t kc, x2 / x0
16 # const uint8_t* restrict a, x3
17 # const void* restrict w, x4
18 # uint8_t* restrict c, x5
19 # size_t cm_stride, x6
20 # size_t cn_stride, x7
21 # const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) [sp] -> (x8)
22
23 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
24
25 # Register usage
26 # A x3 v0 v1 v2 v3
27 # B x4 v20 v21
28 # C0 x5 v16 v17
29 # C1 x9 v18 v19
30 # C2 x10 v28 v29
31 # C3 x6 v30 v31
32 # Clamp v4 v5
33
34
35 BEGIN_FUNCTION xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a75
36
37 # Load params pointer
38 LDR x8, [sp]
39
40 # Clamp A and C pointers
41 CMP x0, 2 // if mr < 2
42 ADD x9, x5, x6 // c1 = c0 + cm_stride
43 CSEL x9, x5, x9, LO // c1 = c0
44
45 ADD x10, x9, x6 // c2 = c1 + cm_stride
46 // if mr <= 2
47 CSEL x10, x9, x10, LS // c2 = c1
48
49 # Load min/max values
50 LD2R {v4.4s, v5.4s}, [x8]
51
52 CMP x0, 4 // if mr < 4
53 ADD x6, x10, x6 // c3 = c2 + cm_stride
54 CSEL x6, x10, x6, LO // c3 = c2
55
56 0:
57 # Load initial bias from w into accumulators
58 LDR q16, [x4], 16
59 LDR q17, [x4], 16
60 SUBS x0, x2, 16 // k = kc - 16
61 LDR q0, [x3], 16 // Preload A
62 MOV v18.16b, v16.16b
63 MOV v28.16b, v16.16b
64 LDR q20, [x4], 16 // Preload B0
65 MOV v30.16b, v16.16b
66 MOV v19.16b, v17.16b
67 LDR q21, [x4], 16 // Preload B1
68 MOV v29.16b, v17.16b
69 MOV v31.16b, v17.16b
70
71 # Is there at least 4 floats (16 bytes) for epilogue?
72 B.LO 5f
73
74 # Is there at least 4 floats (16 bytes) for main loop?
75 SUBS x0, x0, 16 // k -= 16
76 B.LO 2f
77
78 # Main loop - 4 floats of A (16 bytes)
79 1:
80 LDR q22, [x4], 16
81 FMLA v16.4s, v20.4s, v0.s[0]
82 FMLA v18.4s, v20.4s, v0.s[1]
83 LDR q1, [x3], 16
84 FMLA v28.4s, v20.4s, v0.s[2]
85 FMLA v30.4s, v20.4s, v0.s[3]
86 LDR q23, [x4], 16
87 FMLA v17.4s, v21.4s, v0.s[0]
88 FMLA v19.4s, v21.4s, v0.s[1]
89 LDR q24, [x4], 16
90 FMLA v29.4s, v21.4s, v0.s[2]
91 FMLA v31.4s, v21.4s, v0.s[3]
92 LDR q2, [x3], 16
93 FMLA v16.4s, v22.4s, v1.s[0]
94 FMLA v18.4s, v22.4s, v1.s[1]
95 LDR q25, [x4], 16
96 FMLA v28.4s, v22.4s, v1.s[2]
97 FMLA v30.4s, v22.4s, v1.s[3]
98 LDR q26, [x4], 16
99 FMLA v17.4s, v23.4s, v1.s[0]
100 FMLA v19.4s, v23.4s, v1.s[1]
101 LDR q3, [x3], 16
102 FMLA v29.4s, v23.4s, v1.s[2]
103 FMLA v31.4s, v23.4s, v1.s[3]
104 LDR q27, [x4], 16
105 FMLA v16.4s, v24.4s, v2.s[0]
106 FMLA v18.4s, v24.4s, v2.s[1]
107 LDR q0, [x3], 16
108 FMLA v28.4s, v24.4s, v2.s[2]
109 FMLA v30.4s, v24.4s, v2.s[3]
110 LDR q20, [x4], 16
111 FMLA v17.4s, v25.4s, v2.s[0]
112 FMLA v19.4s, v25.4s, v2.s[1]
113 LDR q21, [x4], 16
114 FMLA v29.4s, v25.4s, v2.s[2]
115 FMLA v31.4s, v25.4s, v2.s[3]
116 FMLA v16.4s, v26.4s, v3.s[0]
117 FMLA v18.4s, v26.4s, v3.s[1]
118 FMLA v28.4s, v26.4s, v3.s[2]
119 FMLA v30.4s, v26.4s, v3.s[3]
120 SUBS x0, x0, 16
121 FMLA v17.4s, v27.4s, v3.s[0]
122 FMLA v19.4s, v27.4s, v3.s[1]
123 FMLA v29.4s, v27.4s, v3.s[2]
124 FMLA v31.4s, v27.4s, v3.s[3]
125 B.HS 1b
126
127 # Epilogue
128 2:
129 LDR q22, [x4], 16
130 FMLA v16.4s, v20.4s, v0.s[0]
131 FMLA v18.4s, v20.4s, v0.s[1]
132 LDR q1, [x3], 16
133 FMLA v28.4s, v20.4s, v0.s[2]
134 FMLA v30.4s, v20.4s, v0.s[3]
135 LDR q23, [x4], 16
136 FMLA v17.4s, v21.4s, v0.s[0]
137 FMLA v19.4s, v21.4s, v0.s[1]
138 LDR q24, [x4], 16
139 FMLA v29.4s, v21.4s, v0.s[2]
140 FMLA v31.4s, v21.4s, v0.s[3]
141 LDR q2, [x3], 16
142 FMLA v16.4s, v22.4s, v1.s[0]
143 FMLA v18.4s, v22.4s, v1.s[1]
144 LDR q25, [x4], 16
145 FMLA v28.4s, v22.4s, v1.s[2]
146 FMLA v30.4s, v22.4s, v1.s[3]
147 LDR q26, [x4], 16
148 FMLA v17.4s, v23.4s, v1.s[0]
149 FMLA v19.4s, v23.4s, v1.s[1]
150 LDR q3, [x3], 16
151 FMLA v29.4s, v23.4s, v1.s[2]
152 FMLA v31.4s, v23.4s, v1.s[3]
153 LDR q27, [x4], 16
154 FMLA v16.4s, v24.4s, v2.s[0]
155 FMLA v18.4s, v24.4s, v2.s[1]
156 FMLA v28.4s, v24.4s, v2.s[2]
157 FMLA v30.4s, v24.4s, v2.s[3]
158 FMLA v17.4s, v25.4s, v2.s[0]
159 FMLA v19.4s, v25.4s, v2.s[1]
160 FMLA v29.4s, v25.4s, v2.s[2]
161 FMLA v31.4s, v25.4s, v2.s[3]
162 FMLA v16.4s, v26.4s, v3.s[0]
163 FMLA v18.4s, v26.4s, v3.s[1]
164 FMLA v28.4s, v26.4s, v3.s[2]
165 FMLA v30.4s, v26.4s, v3.s[3]
166 TST x0, 15
167 FMLA v17.4s, v27.4s, v3.s[0]
168 FMLA v19.4s, v27.4s, v3.s[1]
169 FMLA v29.4s, v27.4s, v3.s[2]
170 FMLA v31.4s, v27.4s, v3.s[3]
171 B.NE 4f
172
173 3:
174 # Clamp
175 FMAX v16.4s, v16.4s, v4.4s
176 SUBS x1, x1, 8
177 FMAX v17.4s, v17.4s, v4.4s
178 FMAX v18.4s, v18.4s, v4.4s
179 FMAX v19.4s, v19.4s, v4.4s
180 FMAX v28.4s, v28.4s, v4.4s
181 FMAX v29.4s, v29.4s, v4.4s
182 FMAX v30.4s, v30.4s, v4.4s
183 FMAX v31.4s, v31.4s, v4.4s
184 FMIN v16.4s, v16.4s, v5.4s
185 FMIN v17.4s, v17.4s, v5.4s
186 FMIN v18.4s, v18.4s, v5.4s
187 FMIN v19.4s, v19.4s, v5.4s
188 FMIN v28.4s, v28.4s, v5.4s
189 FMIN v29.4s, v29.4s, v5.4s
190 FMIN v30.4s, v30.4s, v5.4s
191 FMIN v31.4s, v31.4s, v5.4s
192
193 # Store full 4 x 8
194 B.LO 7f
195
196 ST1 {v16.16b, v17.16b}, [x5], x7
197 ST1 {v18.16b, v19.16b}, [x9], x7
198 SUB x3, x3, x2, lsl #2 // a0 -= kc * 4
199 ST1 {v28.16b, v29.16b}, [x10], x7
200 ST1 {v30.16b, v31.16b}, [x6], x7
201
202 B.HI 0b
203 RET
204
205 # Remainder of 1..7
206 4:
207 LDR q0, [x3], 16
208 LDR q20, [x4], 16
209 LDR q21, [x4], 16
210
211 5:
212 # Is there a remainder?- 2 floats of A (8 bytes)
213 TBZ x0, 3, 6f
214
215 # Remainder- 2 floats of A (8 bytes)
216 LDR q22, [x4], 16
217 FMLA v16.4s, v20.4s, v0.s[0]
218 FMLA v18.4s, v20.4s, v0.s[1]
219 LDR q1, [x3], 16
220 FMLA v28.4s, v20.4s, v0.s[2]
221 FMLA v30.4s, v20.4s, v0.s[3]
222 LDR q23, [x4], 16
223 FMLA v17.4s, v21.4s, v0.s[0]
224 FMLA v19.4s, v21.4s, v0.s[1]
225 FMLA v29.4s, v21.4s, v0.s[2]
226 FMLA v31.4s, v21.4s, v0.s[3]
227 FMLA v16.4s, v22.4s, v1.s[0]
228 FMLA v18.4s, v22.4s, v1.s[1]
229 FMLA v28.4s, v22.4s, v1.s[2]
230 FMLA v30.4s, v22.4s, v1.s[3]
231 FMLA v17.4s, v23.4s, v1.s[0]
232 FMLA v19.4s, v23.4s, v1.s[1]
233 FMLA v29.4s, v23.4s, v1.s[2]
234 FMLA v31.4s, v23.4s, v1.s[3]
235
236 # Is there a remainder?- 1 float of A (4 bytes)
237 TBZ x0, 2, 3b
238
239 LDR q0, [x3], 16
240 LDR q20, [x4], 16
241 LDR q21, [x4], 16
242
243 # Remainder- 1 float of A (4 bytes)
244 6:
245 FMLA v16.4s, v20.4s, v0.s[0]
246 FMLA v18.4s, v20.4s, v0.s[1]
247 FMLA v28.4s, v20.4s, v0.s[2]
248 FMLA v30.4s, v20.4s, v0.s[3]
249 FMLA v17.4s, v21.4s, v0.s[0]
250 FMLA v19.4s, v21.4s, v0.s[1]
251 FMLA v29.4s, v21.4s, v0.s[2]
252 FMLA v31.4s, v21.4s, v0.s[3]
253 B 3b
254
255 # Store odd width
256 7:
257 TBZ x1, 2, 8f
258 STR q16, [x5], 16
259 STR q18, [x9], 16
260 MOV v16.16b, v17.16b
261 MOV v18.16b, v19.16b
262 STR q28, [x10], 16
263 STR q30, [x6], 16
264 MOV v28.16b, v29.16b
265 MOV v30.16b, v31.16b
266
267 8:
268 TBZ x1, 1, 9f
269 STR d16, [x5], 8
270 STR d18, [x9], 8
271 DUP d16, v16.d[1]
272 DUP d18, v18.d[1]
273 STR d28, [x10], 8
274 STR d30, [x6], 8
275 DUP d28, v28.d[1]
276 DUP d30, v30.d[1]
277
278 9:
279 TBZ x1, 0, 10f
280 STR s16, [x5]
281 STR s18, [x9]
282 STR s28, [x10]
283 STR s30, [x6]
284
285 10:
286 RET
287
288 END_FUNCTION xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a75
289
290 #ifdef __ELF__
291 .section ".note.GNU-stack","",%progbits
292 #endif