Debian Codegraph @

src/f32-ppmm/gen/f32-ppmm-8x8-minmax-asm-aarch64-neonfma-ld128-prfm.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/f32-ppmm/8x8-aarch64-neonfma-ld128.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include "xnnpack/assembly.h"
11
12 # void xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_ld128_prfm(
13 # size_t mr, x0
14 # size_t nc, x1
15 # size_t kc, x2 / x0
16 # const uint8_t* restrict a, x3
17 # const void* restrict w, x4
18 # uint8_t* restrict c, x5
19 # size_t cm_stride, x6
20 # size_t cn_stride, x7
21 # const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) [sp] -> x8
22
23 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
24
25 # Vector register usage
26 # A x3 v0 v1
27 # B x4 v4 v5
28 # C0 x5 v16 v17
29 # C1 x12 v18 v19
30 # C2 x13 v20 v21
31 # C3 x14 v22 v23
32 # C4 x15 v24 v25
33 # C5 x16 v26 v27
34 # C6 x17 v28 v29
35 # C7 x6 v30 v31
36 # Clamp v6 v7
37
38 BEGIN_FUNCTION xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_ld128_prfm
39
40 # Load params pointer
41 LDR x8, [sp]
42
43 # Clamp A and C pointers
44 CMP x0, 2 // if mr < 2
45 ADD x12, x5, x6 // c1 = c0 + cm_stride
46 CSEL x12, x5, x12, LO // c1 = c0
47 ADD x13, x12, x6 // c2 = c1 + cm_stride
48 // if mr <= 2
49 CSEL x13, x12, x13, LS // c2 = c1
50 CMP x0, 4 // if mr < 4
51 ADD x14, x13, x6 // c3 = c2 + cm_stride
52 CSEL x14, x13, x14, LO // c3 = c2
53
54 # Load min/max values
55 LD2R {v6.4s, v7.4s}, [x8]
56
57 ADD x15, x14, x6 // c4 = c3 + cm_stride
58 // if mr <= 4
59 CSEL x15, x14, x15, LS // c4 = c3
60 CMP x0, 6 // if mr < 6
61 ADD x16, x15, x6 // c5 = c4 + cm_stride
62 CSEL x16, x15, x16, LO // c5 = c4
63 ADD x17, x16, x6 // c6 = c5 + cm_stride
64 // if mr <= 4
65 CSEL x17, x16, x17, LS // c6 = c5
66 CMP x0, 8 // if mr < 8
67 ADD x6, x17, x6 // c7 = c6 + cm_stride
68 CSEL x6, x17, x6, LO // c7 = c6
69
70
71 0:
72 # Load initial bias from w into accumulators
73 LDR q16, [x4], 16
74 LDR q17, [x4], 16
75 SUB x0, x2, 4 // k = kc - 4
76 MOV v18.16b, v16.16b
77 MOV v20.16b, v16.16b
78 PRFM PLDL1KEEP, [x4, 0] // Prefetch B
79 MOV v22.16b, v16.16b
80 MOV v24.16b, v16.16b
81 PRFM PLDL1KEEP, [x4, 64]
82 MOV v26.16b, v16.16b
83 MOV v28.16b, v16.16b
84 MOV v30.16b, v16.16b
85 MOV v19.16b, v17.16b
86 MOV v21.16b, v17.16b
87 MOV v23.16b, v17.16b
88 MOV v25.16b, v17.16b
89 MOV v27.16b, v17.16b
90 MOV v29.16b, v17.16b
91 MOV v31.16b, v17.16b
92
93 # Main loop - 1 float of A (4 bytes)
94 1:
95 LDR q0, [x3], 16
96 LDR q4, [x4], 16
97 LDR q1, [x3], 16
98 LDR q5, [x4], 16
99 FMLA v16.4s, v4.4s, v0.s[0]
100 FMLA v18.4s, v4.4s, v0.s[1]
101 FMLA v20.4s, v4.4s, v0.s[2]
102 FMLA v22.4s, v4.4s, v0.s[3]
103 PRFM PLDL1KEEP, [x4, 128] // Prefetch B0
104 FMLA v24.4s, v4.4s, v1.s[0]
105 FMLA v26.4s, v4.4s, v1.s[1]
106 FMLA v28.4s, v4.4s, v1.s[2]
107 FMLA v30.4s, v4.4s, v1.s[3]
108 FMLA v17.4s, v5.4s, v0.s[0]
109 FMLA v19.4s, v5.4s, v0.s[1]
110 FMLA v21.4s, v5.4s, v0.s[2]
111 FMLA v23.4s, v5.4s, v0.s[3]
112 SUBS x0, x0, 4
113 FMLA v25.4s, v5.4s, v1.s[0]
114 FMLA v27.4s, v5.4s, v1.s[1]
115 FMLA v29.4s, v5.4s, v1.s[2]
116 FMLA v31.4s, v5.4s, v1.s[3]
117 B.HS 1b
118
119 # Clamp
120 FMAX v16.4s, v16.4s, v6.4s
121 FMAX v17.4s, v17.4s, v6.4s
122 FMAX v18.4s, v18.4s, v6.4s
123 FMAX v19.4s, v19.4s, v6.4s
124 FMAX v20.4s, v20.4s, v6.4s
125 FMAX v21.4s, v21.4s, v6.4s
126 FMAX v22.4s, v22.4s, v6.4s
127 FMAX v23.4s, v23.4s, v6.4s
128 FMAX v24.4s, v24.4s, v6.4s
129 FMAX v25.4s, v25.4s, v6.4s
130 FMAX v26.4s, v26.4s, v6.4s
131 FMAX v27.4s, v27.4s, v6.4s
132 FMAX v28.4s, v28.4s, v6.4s
133 FMAX v29.4s, v29.4s, v6.4s
134 FMAX v30.4s, v30.4s, v6.4s
135 FMAX v31.4s, v31.4s, v6.4s
136 SUBS x1, x1, 8
137 FMIN v16.4s, v16.4s, v7.4s
138 FMIN v17.4s, v17.4s, v7.4s
139 FMIN v18.4s, v18.4s, v7.4s
140 FMIN v19.4s, v19.4s, v7.4s
141 FMIN v20.4s, v20.4s, v7.4s
142 FMIN v21.4s, v21.4s, v7.4s
143 FMIN v22.4s, v22.4s, v7.4s
144 FMIN v23.4s, v23.4s, v7.4s
145 FMIN v24.4s, v24.4s, v7.4s
146 FMIN v25.4s, v25.4s, v7.4s
147 FMIN v26.4s, v26.4s, v7.4s
148 FMIN v27.4s, v27.4s, v7.4s
149 FMIN v28.4s, v28.4s, v7.4s
150 FMIN v29.4s, v29.4s, v7.4s
151 FMIN v30.4s, v30.4s, v7.4s
152 FMIN v31.4s, v31.4s, v7.4s
153
154 # Store full 8 x 8
155 B.LO 2f
156
157 ST1 {v16.16b, v17.16b}, [x5], x7
158 ST1 {v18.16b, v19.16b}, [x12], x7
159 ST1 {v20.16b, v21.16b}, [x13], x7
160 ST1 {v22.16b, v23.16b}, [x14], x7
161 SUB x3, x3, x2, lsl #3 // a0 -= kc * 8
162 ST1 {v24.16b, v25.16b}, [x15], x7
163 ST1 {v26.16b, v27.16b}, [x16], x7
164 ST1 {v28.16b, v29.16b}, [x17], x7
165 ST1 {v30.16b, v31.16b}, [x6], x7
166 B.HI 0b
167 RET
168
169 # Store odd width
170 2:
171 TBZ x1, 2, 3f
172 STR q16, [x5], 16
173 STR q18, [x12], 16
174 MOV v16.16b, v17.16b
175 MOV v18.16b, v19.16b
176 STR q20, [x13], 16
177 STR q22, [x14], 16
178 MOV v20.16b, v21.16b
179 MOV v22.16b, v23.16b
180 STR q24, [x15], 16
181 STR q26, [x16], 16
182 MOV v24.16b, v25.16b
183 MOV v26.16b, v27.16b
184 STR q28, [x17], 16
185 STR q30, [x6], 16
186 MOV v28.16b, v29.16b
187 MOV v30.16b, v31.16b
188 3:
189 TBZ x1, 1, 4f
190 STR d16, [x5], 8
191 STR d18, [x12], 8
192 DUP d16, v16.d[1]
193 DUP d18, v18.d[1]
194 STR d20, [x13], 8
195 STR d22, [x14], 8
196 DUP d20, v20.d[1]
197 DUP d22, v22.d[1]
198 STR d24, [x15], 8
199 STR d26, [x16], 8
200 DUP d24, v24.d[1]
201 DUP d26, v26.d[1]
202 STR d28, [x17], 8
203 STR d30, [x6], 8
204 DUP d28, v28.d[1]
205 DUP d30, v30.d[1]
206
207 4:
208 TBZ x1, 0, 5f
209 STR s16, [x5]
210 STR s18, [x12]
211 STR s20, [x13]
212 STR s22, [x14]
213 STR s24, [x15]
214 STR s26, [x16]
215 STR s28, [x17]
216 STR s30, [x6]
217
218 5:
219 RET
220
221 END_FUNCTION xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_ld128_prfm
222
223 #ifdef __ELF__
224 .section ".note.GNU-stack","",%progbits
225 #endif