Debian Codegraph @

src/f32-ppmm/gen/f32-ppmm-8x8-minmax-asm-aarch64-neonfma-ld128.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/f32-ppmm/8x8-aarch64-neonfma-ld128.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include "xnnpack/assembly.h"
11
12 # void xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_ld128(
13 # size_t mr, x0
14 # size_t nc, x1
15 # size_t kc, x2 / x0
16 # const uint8_t* restrict a, x3
17 # const void* restrict w, x4
18 # uint8_t* restrict c, x5
19 # size_t cm_stride, x6
20 # size_t cn_stride, x7
21 # const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) [sp] -> x8
22
23 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
24
25 # Vector register usage
26 # A x3 v0 v1
27 # B x4 v4 v5
28 # C0 x5 v16 v17
29 # C1 x12 v18 v19
30 # C2 x13 v20 v21
31 # C3 x14 v22 v23
32 # C4 x15 v24 v25
33 # C5 x16 v26 v27
34 # C6 x17 v28 v29
35 # C7 x6 v30 v31
36 # Clamp v6 v7
37
38 BEGIN_FUNCTION xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_ld128
39
40 # Load params pointer
41 LDR x8, [sp]
42
43 # Clamp A and C pointers
44 CMP x0, 2 // if mr < 2
45 ADD x12, x5, x6 // c1 = c0 + cm_stride
46 CSEL x12, x5, x12, LO // c1 = c0
47 ADD x13, x12, x6 // c2 = c1 + cm_stride
48 // if mr <= 2
49 CSEL x13, x12, x13, LS // c2 = c1
50 CMP x0, 4 // if mr < 4
51 ADD x14, x13, x6 // c3 = c2 + cm_stride
52 CSEL x14, x13, x14, LO // c3 = c2
53
54 # Load min/max values
55 LD2R {v6.4s, v7.4s}, [x8]
56
57 ADD x15, x14, x6 // c4 = c3 + cm_stride
58 // if mr <= 4
59 CSEL x15, x14, x15, LS // c4 = c3
60 CMP x0, 6 // if mr < 6
61 ADD x16, x15, x6 // c5 = c4 + cm_stride
62 CSEL x16, x15, x16, LO // c5 = c4
63 ADD x17, x16, x6 // c6 = c5 + cm_stride
64 // if mr <= 4
65 CSEL x17, x16, x17, LS // c6 = c5
66 CMP x0, 8 // if mr < 8
67 ADD x6, x17, x6 // c7 = c6 + cm_stride
68 CSEL x6, x17, x6, LO // c7 = c6
69
70
71 0:
72 # Load initial bias from w into accumulators
73 LDR q16, [x4], 16
74 LDR q17, [x4], 16
75 SUB x0, x2, 4 // k = kc - 4
76 MOV v18.16b, v16.16b
77 MOV v20.16b, v16.16b
78 MOV v22.16b, v16.16b
79 MOV v24.16b, v16.16b
80 MOV v26.16b, v16.16b
81 MOV v28.16b, v16.16b
82 MOV v30.16b, v16.16b
83 MOV v19.16b, v17.16b
84 MOV v21.16b, v17.16b
85 MOV v23.16b, v17.16b
86 MOV v25.16b, v17.16b
87 MOV v27.16b, v17.16b
88 MOV v29.16b, v17.16b
89 MOV v31.16b, v17.16b
90
91 # Main loop - 1 float of A (4 bytes)
92 1:
93 LDR q0, [x3], 16
94 LDR q4, [x4], 16
95 LDR q1, [x3], 16
96 LDR q5, [x4], 16
97 FMLA v16.4s, v4.4s, v0.s[0]
98 FMLA v18.4s, v4.4s, v0.s[1]
99 FMLA v20.4s, v4.4s, v0.s[2]
100 FMLA v22.4s, v4.4s, v0.s[3]
101 FMLA v24.4s, v4.4s, v1.s[0]
102 FMLA v26.4s, v4.4s, v1.s[1]
103 FMLA v28.4s, v4.4s, v1.s[2]
104 FMLA v30.4s, v4.4s, v1.s[3]
105 FMLA v17.4s, v5.4s, v0.s[0]
106 FMLA v19.4s, v5.4s, v0.s[1]
107 FMLA v21.4s, v5.4s, v0.s[2]
108 FMLA v23.4s, v5.4s, v0.s[3]
109 SUBS x0, x0, 4
110 FMLA v25.4s, v5.4s, v1.s[0]
111 FMLA v27.4s, v5.4s, v1.s[1]
112 FMLA v29.4s, v5.4s, v1.s[2]
113 FMLA v31.4s, v5.4s, v1.s[3]
114 B.HS 1b
115
116 # Clamp
117 FMAX v16.4s, v16.4s, v6.4s
118 FMAX v17.4s, v17.4s, v6.4s
119 FMAX v18.4s, v18.4s, v6.4s
120 FMAX v19.4s, v19.4s, v6.4s
121 FMAX v20.4s, v20.4s, v6.4s
122 FMAX v21.4s, v21.4s, v6.4s
123 FMAX v22.4s, v22.4s, v6.4s
124 FMAX v23.4s, v23.4s, v6.4s
125 FMAX v24.4s, v24.4s, v6.4s
126 FMAX v25.4s, v25.4s, v6.4s
127 FMAX v26.4s, v26.4s, v6.4s
128 FMAX v27.4s, v27.4s, v6.4s
129 FMAX v28.4s, v28.4s, v6.4s
130 FMAX v29.4s, v29.4s, v6.4s
131 FMAX v30.4s, v30.4s, v6.4s
132 FMAX v31.4s, v31.4s, v6.4s
133 SUBS x1, x1, 8
134 FMIN v16.4s, v16.4s, v7.4s
135 FMIN v17.4s, v17.4s, v7.4s
136 FMIN v18.4s, v18.4s, v7.4s
137 FMIN v19.4s, v19.4s, v7.4s
138 FMIN v20.4s, v20.4s, v7.4s
139 FMIN v21.4s, v21.4s, v7.4s
140 FMIN v22.4s, v22.4s, v7.4s
141 FMIN v23.4s, v23.4s, v7.4s
142 FMIN v24.4s, v24.4s, v7.4s
143 FMIN v25.4s, v25.4s, v7.4s
144 FMIN v26.4s, v26.4s, v7.4s
145 FMIN v27.4s, v27.4s, v7.4s
146 FMIN v28.4s, v28.4s, v7.4s
147 FMIN v29.4s, v29.4s, v7.4s
148 FMIN v30.4s, v30.4s, v7.4s
149 FMIN v31.4s, v31.4s, v7.4s
150
151 # Store full 8 x 8
152 B.LO 2f
153
154 ST1 {v16.16b, v17.16b}, [x5], x7
155 ST1 {v18.16b, v19.16b}, [x12], x7
156 ST1 {v20.16b, v21.16b}, [x13], x7
157 ST1 {v22.16b, v23.16b}, [x14], x7
158 SUB x3, x3, x2, lsl #3 // a0 -= kc * 8
159 ST1 {v24.16b, v25.16b}, [x15], x7
160 ST1 {v26.16b, v27.16b}, [x16], x7
161 ST1 {v28.16b, v29.16b}, [x17], x7
162 ST1 {v30.16b, v31.16b}, [x6], x7
163 B.HI 0b
164 RET
165
166 # Store odd width
167 2:
168 TBZ x1, 2, 3f
169 STR q16, [x5], 16
170 STR q18, [x12], 16
171 MOV v16.16b, v17.16b
172 MOV v18.16b, v19.16b
173 STR q20, [x13], 16
174 STR q22, [x14], 16
175 MOV v20.16b, v21.16b
176 MOV v22.16b, v23.16b
177 STR q24, [x15], 16
178 STR q26, [x16], 16
179 MOV v24.16b, v25.16b
180 MOV v26.16b, v27.16b
181 STR q28, [x17], 16
182 STR q30, [x6], 16
183 MOV v28.16b, v29.16b
184 MOV v30.16b, v31.16b
185 3:
186 TBZ x1, 1, 4f
187 STR d16, [x5], 8
188 STR d18, [x12], 8
189 DUP d16, v16.d[1]
190 DUP d18, v18.d[1]
191 STR d20, [x13], 8
192 STR d22, [x14], 8
193 DUP d20, v20.d[1]
194 DUP d22, v22.d[1]
195 STR d24, [x15], 8
196 STR d26, [x16], 8
197 DUP d24, v24.d[1]
198 DUP d26, v26.d[1]
199 STR d28, [x17], 8
200 STR d30, [x6], 8
201 DUP d28, v28.d[1]
202 DUP d30, v30.d[1]
203
204 4:
205 TBZ x1, 0, 5f
206 STR s16, [x5]
207 STR s18, [x12]
208 STR s20, [x13]
209 STR s22, [x14]
210 STR s24, [x15]
211 STR s26, [x16]
212 STR s28, [x17]
213 STR s30, [x6]
214
215 5:
216 RET
217
218 END_FUNCTION xnn_f32_ppmm_minmax_ukernel_8x8__asm_aarch64_neonfma_ld128
219
220 #ifdef __ELF__
221 .section ".note.GNU-stack","",%progbits
222 #endif