Debian Codegraph @

src/f32-ppmm/gen/f32-ppmm-4x8-minmax-asm-aarch64-neonfma-ld128.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/f32-ppmm/4x8-aarch64-neonfma-ld128.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include "xnnpack/assembly.h"
11
12 # void xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_ld128(
13 # size_t mr, x0
14 # size_t nc, x1
15 # size_t kc, x2 / x0
16 # const uint8_t* restrict a, x3
17 # const void* restrict w, x4
18 # uint8_t* restrict c, x5
19 # size_t cm_stride, x6
20 # size_t cn_stride, x7
21 # const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) [sp] -> (x8)
22
23 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
24
25 # Register usage
26 # A x3 v0 v1 v2 v3
27 # B x4 v20 v21
28 # C0 x5 v16 v17
29 # C1 x9 v18 v19
30 # C2 x10 v28 v29
31 # C3 x6 v30 v31
32 # Clamp v4 v5
33
34
35 BEGIN_FUNCTION xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_ld128
36
37 # Load params pointer
38 LDR x8, [sp]
39
40 # Clamp A and C pointers
41 CMP x0, 2 // if mr < 2
42 ADD x9, x5, x6 // c1 = c0 + cm_stride
43 CSEL x9, x5, x9, LO // c1 = c0
44
45 ADD x10, x9, x6 // c2 = c1 + cm_stride
46 // if mr <= 2
47 CSEL x10, x9, x10, LS // c2 = c1
48
49 # Load min/max values
50 LD2R {v4.4s, v5.4s}, [x8]
51
52 CMP x0, 4 // if mr < 4
53 ADD x6, x10, x6 // c3 = c2 + cm_stride
54 CSEL x6, x10, x6, LO // c3 = c2
55
56
57 0:
58 # Load initial bias from w into accumulators
59 LDR q16, [x4], 16
60 LDR q17, [x4], 16
61 SUBS x0, x2, 16 // k = kc - 16
62 MOV v18.16b, v16.16b
63 MOV v28.16b, v16.16b
64 MOV v30.16b, v16.16b
65 MOV v19.16b, v17.16b
66 MOV v29.16b, v17.16b
67 MOV v31.16b, v17.16b
68
69 # Is there at least 4 floats (16 bytes)?
70 B.LO 3f
71
72 # Main loop - 4 floats of A (16 bytes)
73 1:
74 LDR q0, [x3], 16
75 LDR q20, [x4], 16
76 LDR q21, [x4], 16
77 LDR q22, [x4], 16
78 FMLA v16.4s, v20.4s, v0.s[0]
79 FMLA v18.4s, v20.4s, v0.s[1]
80 LDR q1, [x3], 16
81 FMLA v28.4s, v20.4s, v0.s[2]
82 FMLA v30.4s, v20.4s, v0.s[3]
83 LDR q23, [x4], 16
84 FMLA v17.4s, v21.4s, v0.s[0]
85 FMLA v19.4s, v21.4s, v0.s[1]
86 LDR q24, [x4], 16
87 FMLA v29.4s, v21.4s, v0.s[2]
88 FMLA v31.4s, v21.4s, v0.s[3]
89 LDR q2, [x3], 16
90 FMLA v16.4s, v22.4s, v1.s[0]
91 FMLA v18.4s, v22.4s, v1.s[1]
92 LDR q25, [x4], 16
93 FMLA v28.4s, v22.4s, v1.s[2]
94 FMLA v30.4s, v22.4s, v1.s[3]
95 LDR q26, [x4], 16
96 FMLA v17.4s, v23.4s, v1.s[0]
97 FMLA v19.4s, v23.4s, v1.s[1]
98 LDR q3, [x3], 16
99 FMLA v29.4s, v23.4s, v1.s[2]
100 FMLA v31.4s, v23.4s, v1.s[3]
101 LDR q27, [x4], 16
102 FMLA v16.4s, v24.4s, v2.s[0]
103 FMLA v18.4s, v24.4s, v2.s[1]
104 FMLA v28.4s, v24.4s, v2.s[2]
105 FMLA v30.4s, v24.4s, v2.s[3]
106 FMLA v17.4s, v25.4s, v2.s[0]
107 FMLA v19.4s, v25.4s, v2.s[1]
108 FMLA v29.4s, v25.4s, v2.s[2]
109 FMLA v31.4s, v25.4s, v2.s[3]
110 FMLA v16.4s, v26.4s, v3.s[0]
111 FMLA v18.4s, v26.4s, v3.s[1]
112 FMLA v28.4s, v26.4s, v3.s[2]
113 FMLA v30.4s, v26.4s, v3.s[3]
114 SUBS x0, x0, 16
115 FMLA v17.4s, v27.4s, v3.s[0]
116 FMLA v19.4s, v27.4s, v3.s[1]
117 FMLA v29.4s, v27.4s, v3.s[2]
118 FMLA v31.4s, v27.4s, v3.s[3]
119 B.HS 1b
120
121 TST x0, 15
122 B.NE 3f
123
124 2:
125 # Clamp
126 FMAX v16.4s, v16.4s, v4.4s
127 SUBS x1, x1, 8
128 FMAX v17.4s, v17.4s, v4.4s
129 FMAX v18.4s, v18.4s, v4.4s
130 FMAX v19.4s, v19.4s, v4.4s
131 FMAX v28.4s, v28.4s, v4.4s
132 FMAX v29.4s, v29.4s, v4.4s
133 FMAX v30.4s, v30.4s, v4.4s
134 FMAX v31.4s, v31.4s, v4.4s
135 FMIN v16.4s, v16.4s, v5.4s
136 FMIN v17.4s, v17.4s, v5.4s
137 FMIN v18.4s, v18.4s, v5.4s
138 FMIN v19.4s, v19.4s, v5.4s
139 FMIN v28.4s, v28.4s, v5.4s
140 FMIN v29.4s, v29.4s, v5.4s
141 FMIN v30.4s, v30.4s, v5.4s
142 FMIN v31.4s, v31.4s, v5.4s
143
144 # Store full 4 x 8
145 B.LO 5f
146
147 ST1 {v16.16b, v17.16b}, [x5], x7
148 ST1 {v18.16b, v19.16b}, [x9], x7
149 SUB x3, x3, x2, lsl #2 // a0 -= kc * 4
150 ST1 {v28.16b, v29.16b}, [x10], x7
151 ST1 {v30.16b, v31.16b}, [x6], x7
152
153 B.HI 0b
154 RET
155
156 # Remainder- 2 floats of A (8 bytes)
157 3:
158 # Is there a remainder?- 2 floats of A (8 bytes)
159 TBZ x0, 3, 4f
160
161 # Remainder- 2 floats of A (8 bytes)
162 LDR q0, [x3], 16
163 LDR q20, [x4], 16
164 LDR q21, [x4], 16
165 LDR q22, [x4], 16
166 FMLA v16.4s, v20.4s, v0.s[0]
167 FMLA v18.4s, v20.4s, v0.s[1]
168 LDR q1, [x3], 16
169 FMLA v28.4s, v20.4s, v0.s[2]
170 FMLA v30.4s, v20.4s, v0.s[3]
171 LDR q23, [x4], 16
172 FMLA v17.4s, v21.4s, v0.s[0]
173 FMLA v19.4s, v21.4s, v0.s[1]
174 FMLA v29.4s, v21.4s, v0.s[2]
175 FMLA v31.4s, v21.4s, v0.s[3]
176 FMLA v16.4s, v22.4s, v1.s[0]
177 FMLA v18.4s, v22.4s, v1.s[1]
178 FMLA v28.4s, v22.4s, v1.s[2]
179 FMLA v30.4s, v22.4s, v1.s[3]
180 FMLA v17.4s, v23.4s, v1.s[0]
181 FMLA v19.4s, v23.4s, v1.s[1]
182 FMLA v29.4s, v23.4s, v1.s[2]
183 FMLA v31.4s, v23.4s, v1.s[3]
184
185 # Is there a remainder?- 1 float of A (4 bytes)
186 TBZ x0, 2, 2b
187
188 # Remainder- 1 float of A (4 bytes)
189 4:
190 LDR q0, [x3], 16
191 LDR q20, [x4], 16
192 LDR q21, [x4], 16
193 FMLA v16.4s, v20.4s, v0.s[0]
194 FMLA v18.4s, v20.4s, v0.s[1]
195 FMLA v28.4s, v20.4s, v0.s[2]
196 FMLA v30.4s, v20.4s, v0.s[3]
197 FMLA v17.4s, v21.4s, v0.s[0]
198 FMLA v19.4s, v21.4s, v0.s[1]
199 FMLA v29.4s, v21.4s, v0.s[2]
200 FMLA v31.4s, v21.4s, v0.s[3]
201 B 2b
202
203 # Store odd width
204 5:
205 TBZ x1, 2, 6f
206 STR q16, [x5], 16
207 MOV v16.16b, v17.16b
208 STR q18, [x9], 16
209 MOV v18.16b, v19.16b
210 STR q28, [x10], 16
211 MOV v28.16b, v29.16b
212 STR q30, [x6], 16
213 MOV v30.16b, v31.16b
214
215 6:
216 TBZ x1, 1, 7f
217 STR d16, [x5], 8
218 STR d18, [x9], 8
219 DUP d16, v16.d[1]
220 DUP d18, v18.d[1]
221 STR d28, [x10], 8
222 STR d30, [x6], 8
223 DUP d28, v28.d[1]
224 DUP d30, v30.d[1]
225
226 7:
227 TBZ x1, 0, 8f
228 STR s16, [x5]
229 STR s18, [x9]
230 STR s28, [x10]
231 STR s30, [x6]
232
233 8:
234 RET
235
236 END_FUNCTION xnn_f32_ppmm_minmax_ukernel_4x8__asm_aarch64_neonfma_ld128
237
238 #ifdef __ELF__
239 .section ".note.GNU-stack","",%progbits
240 #endif