Debian Codegraph @

src/f32-ppmm/gen/f32-ppmm-8x8-minmax-asm-aarch64-neonfma-cortex-a75-prfm.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/f32-ppmm/8x8-aarch64-neonfma-cortex-a75.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include "xnnpack/assembly.h"
11
12 # void xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_cortex_a75_prfm(
13 # size_t mr, x0
14 # size_t nc, x1
15 # size_t kc, x2 / x0
16 # const uint8_t* restrict a, x3
17 # const void* restrict w, x4
18 # uint8_t* restrict c, x5
19 # size_t cm_stride, x6
20 # size_t cn_stride, x7
21 # const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) [sp] -> x8
22
23 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
24
25 # Vector register usage
26 # A x3 v0 v1
27 # B x4 v4 v5
28 # C0 x5 v16 v17
29 # C1 x12 v18 v19
30 # C2 x13 v20 v21
31 # C3 x14 v22 v23
32 # C4 x15 v24 v25
33 # C5 x16 v26 v27
34 # C6 x17 v28 v29
35 # C7 x6 v30 v31
36 # Clamp v6 v7
37
38 BEGIN_FUNCTION xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_cortex_a75_prfm
39
40 # Load params pointer
41 LDR x8, [sp]
42
43 # Clamp A and C pointers
44 CMP x0, 2 // if mr < 2
45 ADD x12, x5, x6 // c1 = c0 + cm_stride
46 CSEL x12, x5, x12, LO // c1 = c0
47 ADD x13, x12, x6 // c2 = c1 + cm_stride
48 // if mr <= 2
49 CSEL x13, x12, x13, LS // c2 = c1
50 CMP x0, 4 // if mr < 4
51 ADD x14, x13, x6 // c3 = c2 + cm_stride
52 CSEL x14, x13, x14, LO // c3 = c2
53
54 # Load min/max values
55 LD2R {v6.4s, v7.4s}, [x8]
56
57 ADD x15, x14, x6 // c4 = c3 + cm_stride
58 // if mr <= 4
59 CSEL x15, x14, x15, LS // c4 = c3
60 CMP x0, 6 // if mr < 6
61 ADD x16, x15, x6 // c5 = c4 + cm_stride
62 CSEL x16, x15, x16, LO // c5 = c4
63 ADD x17, x16, x6 // c6 = c5 + cm_stride
64 // if mr <= 4
65 CSEL x17, x16, x17, LS // c6 = c5
66 CMP x0, 8 // if mr < 8
67 ADD x6, x17, x6 // c7 = c6 + cm_stride
68 CSEL x6, x17, x6, LO // c7 = c6
69
70 0:
71 # Load initial bias from w into accumulators
72 LDR q16, [x4], 16
73 LDR q17, [x4], 16
74 MOV v18.16b, v16.16b
75 MOV v20.16b, v16.16b
76 LDR q0, [x3], 16 // Prologue - loads for main loop
77 MOV v22.16b, v16.16b
78 MOV v24.16b, v16.16b
79 LDR q4, [x4], 16
80 MOV v26.16b, v16.16b
81 MOV v28.16b, v16.16b
82 PRFM PLDL1KEEP, [x4, 64] // Prefetch B
83 MOV v30.16b, v16.16b
84 MOV v19.16b, v17.16b
85 MOV v21.16b, v17.16b
86 MOV v23.16b, v17.16b
87 SUBS x0, x2, 8 // k = kc - 8
88 MOV v25.16b, v17.16b
89 MOV v27.16b, v17.16b
90 MOV v29.16b, v17.16b
91 MOV v31.16b, v17.16b
92
93 # Is there at least 2 floats (8 bytes) for mainloop + epilogue?
94 B.LO 2f
95
96 # Main loop - 2 float of A (8 bytes)
97 1:
98 LDR q1, [x3], 16
99 FMLA v16.4s, v4.4s, v0.s[0]
100 LDR q5, [x4], 16
101 FMLA v18.4s, v4.4s, v0.s[1]
102 FMLA v20.4s, v4.4s, v0.s[2]
103 PRFM PLDL1KEEP, [x4, 128] // Prefetch B
104 FMLA v22.4s, v4.4s, v0.s[3]
105 FMLA v24.4s, v4.4s, v1.s[0]
106 FMLA v26.4s, v4.4s, v1.s[1]
107 FMLA v28.4s, v4.4s, v1.s[2]
108 FMLA v30.4s, v4.4s, v1.s[3]
109 FMLA v17.4s, v5.4s, v0.s[0]
110 LDR q4, [x4], 16
111 FMLA v19.4s, v5.4s, v0.s[1]
112 FMLA v21.4s, v5.4s, v0.s[2]
113 FMLA v23.4s, v5.4s, v0.s[3]
114 FMLA v25.4s, v5.4s, v1.s[0]
115 LDR q0, [x3], 16
116 FMLA v27.4s, v5.4s, v1.s[1]
117 SUBS x0, x0, 4
118 FMLA v29.4s, v5.4s, v1.s[2]
119 FMLA v31.4s, v5.4s, v1.s[3]
120 B.HS 1b
121
122 # Epilogue
123 2:
124 LDR q1, [x3], 16
125 FMLA v16.4s, v4.4s, v0.s[0]
126 LDR q5, [x4], 16
127 FMLA v18.4s, v4.4s, v0.s[1]
128 FMLA v20.4s, v4.4s, v0.s[2]
129 FMLA v22.4s, v4.4s, v0.s[3]
130 FMLA v24.4s, v4.4s, v1.s[0]
131 FMLA v26.4s, v4.4s, v1.s[1]
132 FMLA v28.4s, v4.4s, v1.s[2]
133 FMLA v30.4s, v4.4s, v1.s[3]
134 FMLA v17.4s, v5.4s, v0.s[0]
135 FMLA v19.4s, v5.4s, v0.s[1]
136 FMLA v21.4s, v5.4s, v0.s[2]
137 FMLA v23.4s, v5.4s, v0.s[3]
138 FMLA v25.4s, v5.4s, v1.s[0]
139 FMLA v27.4s, v5.4s, v1.s[1]
140 FMLA v29.4s, v5.4s, v1.s[2]
141 FMLA v31.4s, v5.4s, v1.s[3]
142
143 # Clamp
144 FMAX v16.4s, v16.4s, v6.4s
145 FMAX v17.4s, v17.4s, v6.4s
146 FMAX v18.4s, v18.4s, v6.4s
147 FMAX v19.4s, v19.4s, v6.4s
148 FMAX v20.4s, v20.4s, v6.4s
149 FMAX v21.4s, v21.4s, v6.4s
150 FMAX v22.4s, v22.4s, v6.4s
151 FMAX v23.4s, v23.4s, v6.4s
152 FMAX v24.4s, v24.4s, v6.4s
153 FMAX v25.4s, v25.4s, v6.4s
154 FMAX v26.4s, v26.4s, v6.4s
155 FMAX v27.4s, v27.4s, v6.4s
156 FMAX v28.4s, v28.4s, v6.4s
157 FMAX v29.4s, v29.4s, v6.4s
158 FMAX v30.4s, v30.4s, v6.4s
159 FMAX v31.4s, v31.4s, v6.4s
160 SUBS x1, x1, 8
161 FMIN v16.4s, v16.4s, v7.4s
162 FMIN v17.4s, v17.4s, v7.4s
163 FMIN v18.4s, v18.4s, v7.4s
164 FMIN v19.4s, v19.4s, v7.4s
165 FMIN v20.4s, v20.4s, v7.4s
166 FMIN v21.4s, v21.4s, v7.4s
167 FMIN v22.4s, v22.4s, v7.4s
168 FMIN v23.4s, v23.4s, v7.4s
169 FMIN v24.4s, v24.4s, v7.4s
170 FMIN v25.4s, v25.4s, v7.4s
171 FMIN v26.4s, v26.4s, v7.4s
172 FMIN v27.4s, v27.4s, v7.4s
173 FMIN v28.4s, v28.4s, v7.4s
174 FMIN v29.4s, v29.4s, v7.4s
175 FMIN v30.4s, v30.4s, v7.4s
176 FMIN v31.4s, v31.4s, v7.4s
177
178 # Store full 8 x 8
179 B.LO 3f
180
181 ST1 {v16.16b, v17.16b}, [x5], x7
182 ST1 {v18.16b, v19.16b}, [x12], x7
183 ST1 {v20.16b, v21.16b}, [x13], x7
184 ST1 {v22.16b, v23.16b}, [x14], x7
185 SUB x3, x3, x2, lsl #3 // a0 -= kc * 8
186 ST1 {v24.16b, v25.16b}, [x15], x7
187 ST1 {v26.16b, v27.16b}, [x16], x7
188 ST1 {v28.16b, v29.16b}, [x17], x7
189 ST1 {v30.16b, v31.16b}, [x6], x7
190 B.HI 0b
191 RET
192
193 # Store odd width
194 3:
195 TBZ x1, 2, 4f
196 STR q16, [x5], 16
197 STR q18, [x12], 16
198 MOV v16.16b, v17.16b
199 MOV v18.16b, v19.16b
200 STR q20, [x13], 16
201 STR q22, [x14], 16
202 MOV v20.16b, v21.16b
203 MOV v22.16b, v23.16b
204 STR q24, [x15], 16
205 STR q26, [x16], 16
206 MOV v24.16b, v25.16b
207 MOV v26.16b, v27.16b
208 STR q28, [x17], 16
209 STR q30, [x6], 16
210 MOV v28.16b, v29.16b
211 MOV v30.16b, v31.16b
212 4:
213 TBZ x1, 1, 5f
214 STR d16, [x5], 8
215 STR d18, [x12], 8
216 DUP d16, v16.d[1]
217 DUP d18, v18.d[1]
218 STR d20, [x13], 8
219 STR d22, [x14], 8
220 DUP d20, v20.d[1]
221 DUP d22, v22.d[1]
222 STR d24, [x15], 8
223 STR d26, [x16], 8
224 DUP d24, v24.d[1]
225 DUP d26, v26.d[1]
226 STR d28, [x17], 8
227 STR d30, [x6], 8
228 DUP d28, v28.d[1]
229 DUP d30, v30.d[1]
230
231 5:
232 TBZ x1, 0, 6f
233 STR s16, [x5]
234 STR s18, [x12]
235 STR s20, [x13]
236 STR s22, [x14]
237 STR s24, [x15]
238 STR s26, [x16]
239 STR s28, [x17]
240 STR s30, [x6]
241
242 6:
243 RET
244
245 END_FUNCTION xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_cortex_a75_prfm
246
247 #ifdef __ELF__
248 .section ".note.GNU-stack","",%progbits
249 #endif