Debian Codegraph @

src/f32-ppmm/gen/f32-ppmm-4x8-minmax-asm-aarch64-neonfma-ld128-prfm.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/f32-ppmm/4x8-aarch64-neonfma-ld128.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include "xnnpack/assembly.h"
11
12 # void xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_ld128_prfm(
13 # size_t mr, x0
14 # size_t nc, x1
15 # size_t kc, x2 / x0
16 # const uint8_t* restrict a, x3
17 # const void* restrict w, x4
18 # uint8_t* restrict c, x5
19 # size_t cm_stride, x6
20 # size_t cn_stride, x7
21 # const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) [sp] -> (x8)
22
23 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
24
25 # Register usage
26 # A x3 v0 v1 v2 v3
27 # B x4 v20 v21
28 # C0 x5 v16 v17
29 # C1 x9 v18 v19
30 # C2 x10 v28 v29
31 # C3 x6 v30 v31
32 # Clamp v4 v5
33
34
35 BEGIN_FUNCTION xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_ld128_prfm
36
37 # Load params pointer
38 LDR x8, [sp]
39
40 # Clamp A and C pointers
41 CMP x0, 2 // if mr < 2
42 ADD x9, x5, x6 // c1 = c0 + cm_stride
43 CSEL x9, x5, x9, LO // c1 = c0
44
45 ADD x10, x9, x6 // c2 = c1 + cm_stride
46 // if mr <= 2
47 CSEL x10, x9, x10, LS // c2 = c1
48
49 # Load min/max values
50 LD2R {v4.4s, v5.4s}, [x8]
51
52 CMP x0, 4 // if mr < 4
53 ADD x6, x10, x6 // c3 = c2 + cm_stride
54 CSEL x6, x10, x6, LO // c3 = c2
55
56
57 0:
58 # Load initial bias from w into accumulators
59 LDR q16, [x4], 16
60 LDR q17, [x4], 16
61 SUBS x0, x2, 16 // k = kc - 16
62 MOV v18.16b, v16.16b
63 MOV v28.16b, v16.16b
64 PRFM PLDL1KEEP, [x4, 0] // Prefetch B
65 MOV v30.16b, v16.16b
66 MOV v19.16b, v17.16b
67 PRFM PLDL1KEEP, [x4, 64]
68 MOV v29.16b, v17.16b
69 MOV v31.16b, v17.16b
70
71 # Is there at least 4 floats (16 bytes)?
72 B.LO 3f
73
74 # Main loop - 4 floats of A (16 bytes)
75 1:
76 LDR q0, [x3], 16
77 LDR q20, [x4], 16
78 LDR q21, [x4], 16
79 LDR q22, [x4], 16
80 FMLA v16.4s, v20.4s, v0.s[0]
81 FMLA v18.4s, v20.4s, v0.s[1]
82 LDR q1, [x3], 16
83 FMLA v28.4s, v20.4s, v0.s[2]
84 FMLA v30.4s, v20.4s, v0.s[3]
85 LDR q23, [x4], 16
86 FMLA v17.4s, v21.4s, v0.s[0]
87 FMLA v19.4s, v21.4s, v0.s[1]
88 LDR q24, [x4], 16
89 FMLA v29.4s, v21.4s, v0.s[2]
90 FMLA v31.4s, v21.4s, v0.s[3]
91 LDR q2, [x3], 16
92 FMLA v16.4s, v22.4s, v1.s[0]
93 FMLA v18.4s, v22.4s, v1.s[1]
94 LDR q25, [x4], 16
95 FMLA v28.4s, v22.4s, v1.s[2]
96 FMLA v30.4s, v22.4s, v1.s[3]
97 LDR q26, [x4], 16
98 FMLA v17.4s, v23.4s, v1.s[0]
99 FMLA v19.4s, v23.4s, v1.s[1]
100 LDR q3, [x3], 16
101 FMLA v29.4s, v23.4s, v1.s[2]
102 FMLA v31.4s, v23.4s, v1.s[3]
103 LDR q27, [x4], 16
104 FMLA v16.4s, v24.4s, v2.s[0]
105 FMLA v18.4s, v24.4s, v2.s[1]
106 FMLA v28.4s, v24.4s, v2.s[2]
107 FMLA v30.4s, v24.4s, v2.s[3]
108 PRFM PLDL1KEEP, [x4, 128] // Prefetch B0
109 FMLA v17.4s, v25.4s, v2.s[0]
110 FMLA v19.4s, v25.4s, v2.s[1]
111 FMLA v29.4s, v25.4s, v2.s[2]
112 FMLA v31.4s, v25.4s, v2.s[3]
113 PRFM PLDL1KEEP, [x4, 192] // Prefetch B1
114 FMLA v16.4s, v26.4s, v3.s[0]
115 FMLA v18.4s, v26.4s, v3.s[1]
116 FMLA v28.4s, v26.4s, v3.s[2]
117 FMLA v30.4s, v26.4s, v3.s[3]
118 SUBS x0, x0, 16
119 FMLA v17.4s, v27.4s, v3.s[0]
120 FMLA v19.4s, v27.4s, v3.s[1]
121 FMLA v29.4s, v27.4s, v3.s[2]
122 FMLA v31.4s, v27.4s, v3.s[3]
123 B.HS 1b
124
125 TST x0, 15
126 B.NE 3f
127
128 2:
129 # Clamp
130 FMAX v16.4s, v16.4s, v4.4s
131 SUBS x1, x1, 8
132 FMAX v17.4s, v17.4s, v4.4s
133 FMAX v18.4s, v18.4s, v4.4s
134 FMAX v19.4s, v19.4s, v4.4s
135 FMAX v28.4s, v28.4s, v4.4s
136 FMAX v29.4s, v29.4s, v4.4s
137 FMAX v30.4s, v30.4s, v4.4s
138 FMAX v31.4s, v31.4s, v4.4s
139 FMIN v16.4s, v16.4s, v5.4s
140 FMIN v17.4s, v17.4s, v5.4s
141 FMIN v18.4s, v18.4s, v5.4s
142 FMIN v19.4s, v19.4s, v5.4s
143 FMIN v28.4s, v28.4s, v5.4s
144 FMIN v29.4s, v29.4s, v5.4s
145 FMIN v30.4s, v30.4s, v5.4s
146 FMIN v31.4s, v31.4s, v5.4s
147
148 # Store full 4 x 8
149 B.LO 5f
150
151 ST1 {v16.16b, v17.16b}, [x5], x7
152 ST1 {v18.16b, v19.16b}, [x9], x7
153 SUB x3, x3, x2, lsl #2 // a0 -= kc * 4
154 ST1 {v28.16b, v29.16b}, [x10], x7
155 ST1 {v30.16b, v31.16b}, [x6], x7
156
157 B.HI 0b
158 RET
159
160 # Remainder- 2 floats of A (8 bytes)
161 3:
162 # Is there a remainder?- 2 floats of A (8 bytes)
163 TBZ x0, 3, 4f
164
165 # Remainder- 2 floats of A (8 bytes)
166 LDR q0, [x3], 16
167 LDR q20, [x4], 16
168 LDR q21, [x4], 16
169 LDR q22, [x4], 16
170 FMLA v16.4s, v20.4s, v0.s[0]
171 FMLA v18.4s, v20.4s, v0.s[1]
172 LDR q1, [x3], 16
173 FMLA v28.4s, v20.4s, v0.s[2]
174 FMLA v30.4s, v20.4s, v0.s[3]
175 LDR q23, [x4], 16
176 FMLA v17.4s, v21.4s, v0.s[0]
177 FMLA v19.4s, v21.4s, v0.s[1]
178 FMLA v29.4s, v21.4s, v0.s[2]
179 FMLA v31.4s, v21.4s, v0.s[3]
180 FMLA v16.4s, v22.4s, v1.s[0]
181 FMLA v18.4s, v22.4s, v1.s[1]
182 FMLA v28.4s, v22.4s, v1.s[2]
183 FMLA v30.4s, v22.4s, v1.s[3]
184 FMLA v17.4s, v23.4s, v1.s[0]
185 FMLA v19.4s, v23.4s, v1.s[1]
186 FMLA v29.4s, v23.4s, v1.s[2]
187 FMLA v31.4s, v23.4s, v1.s[3]
188
189 # Is there a remainder?- 1 float of A (4 bytes)
190 TBZ x0, 2, 2b
191
192 # Remainder- 1 float of A (4 bytes)
193 4:
194 LDR q0, [x3], 16
195 LDR q20, [x4], 16
196 LDR q21, [x4], 16
197 FMLA v16.4s, v20.4s, v0.s[0]
198 FMLA v18.4s, v20.4s, v0.s[1]
199 FMLA v28.4s, v20.4s, v0.s[2]
200 FMLA v30.4s, v20.4s, v0.s[3]
201 FMLA v17.4s, v21.4s, v0.s[0]
202 FMLA v19.4s, v21.4s, v0.s[1]
203 FMLA v29.4s, v21.4s, v0.s[2]
204 FMLA v31.4s, v21.4s, v0.s[3]
205 B 2b
206
207 # Store odd width
208 5:
209 TBZ x1, 2, 6f
210 STR q16, [x5], 16
211 MOV v16.16b, v17.16b
212 STR q18, [x9], 16
213 MOV v18.16b, v19.16b
214 STR q28, [x10], 16
215 MOV v28.16b, v29.16b
216 STR q30, [x6], 16
217 MOV v30.16b, v31.16b
218
219 6:
220 TBZ x1, 1, 7f
221 STR d16, [x5], 8
222 STR d18, [x9], 8
223 DUP d16, v16.d[1]
224 DUP d18, v18.d[1]
225 STR d28, [x10], 8
226 STR d30, [x6], 8
227 DUP d28, v28.d[1]
228 DUP d30, v30.d[1]
229
230 7:
231 TBZ x1, 0, 8f
232 STR s16, [x5]
233 STR s18, [x9]
234 STR s28, [x10]
235 STR s30, [x6]
236
237 8:
238 RET
239
240 END_FUNCTION xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_ld128_prfm
241
242 #ifdef __ELF__
243 .section ".note.GNU-stack","",%progbits
244 #endif