Debian Codegraph @

src/qd8-f16-qc8w-igemm/gen/qd8-f16-qc8w-igemm-4x32c4-minmax-neondotfp16arith.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/c4-neondot.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include "xnnpack/igemm.h"
15 #include "xnnpack/math.h"
16
17
18 void xnn_qd8_f16_qc8w_igemm_minmax_ukernel_4x32c4__neondotfp16arith(
19 size_t mr,
20 size_t nc,
21 size_t kc,
22 size_t ks,
23 const int8_t** restrict a,
24 const void* restrict w,
25 xnn_float16* restrict c,
26 size_t cm_stride,
27 size_t cn_stride,
28 size_t a_offset,
29 const int8_t* zero,
30 const int8_t* zero_data,
31 const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
32 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
33 {
34 assert(mr != 0);
35 assert(mr <= 4);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(ks != 0);
39 assert(ks % (4 * sizeof(void*)) == 0);
40 assert(a_offset % sizeof(int8_t) == 0);
41 assert(a != NULL);
42 assert(w != NULL);
43 assert(c != NULL);
44
45 kc = round_up_po2(kc, 4 * sizeof(int8_t));
46 uint16_t* c0 = (uint16_t*) c;
47 uint16_t* c1 = (uint16_t*) ((uintptr_t) c0 + cm_stride);
48 if XNN_UNPREDICTABLE(mr < 2) {
49 c1 = c0;
50 }
51 uint16_t* c2 = (uint16_t*) ((uintptr_t) c1 + cm_stride);
52 if XNN_UNPREDICTABLE(mr <= 2) {
53 c2 = c1;
54 }
55 uint16_t* c3 = (uint16_t*) ((uintptr_t) c2 + cm_stride);
56 if XNN_UNPREDICTABLE(mr != 4) {
57 c3 = c2;
58 }
59
60 do {
61 const int32x4_t vinput_zero_point = vld1q_dup_s32(&quantization_params->zero_point);
62 const int32x4_t vksum0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
63 int32x4_t vacc0x0123 = vmulq_s32(vksum0123, vinput_zero_point);
64 const int32x4_t vksum4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
65 int32x4_t vacc0x4567 = vmulq_s32(vksum4567, vinput_zero_point);
66 const int32x4_t vksum89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
67 int32x4_t vacc0x89AB = vmulq_s32(vksum89AB, vinput_zero_point);
68 const int32x4_t vksumCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
69 int32x4_t vacc0xCDEF = vmulq_s32(vksumCDEF, vinput_zero_point);
70 const int32x4_t vksumGHIJ = vld1q_s32(w); w = (const int32_t*) w + 4;
71 int32x4_t vacc0xGHIJ = vmulq_s32(vksumGHIJ, vinput_zero_point);
72 const int32x4_t vksumKLMN = vld1q_s32(w); w = (const int32_t*) w + 4;
73 int32x4_t vacc0xKLMN = vmulq_s32(vksumKLMN, vinput_zero_point);
74 const int32x4_t vksumOPQR = vld1q_s32(w); w = (const int32_t*) w + 4;
75 int32x4_t vacc0xOPQR = vmulq_s32(vksumOPQR, vinput_zero_point);
76 const int32x4_t vksumSTUV = vld1q_s32(w); w = (const int32_t*) w + 4;
77 int32x4_t vacc0xSTUV = vmulq_s32(vksumSTUV, vinput_zero_point);
78 int32x4_t vacc1x0123 = vacc0x0123;
79 int32x4_t vacc1x4567 = vacc0x4567;
80 int32x4_t vacc1x89AB = vacc0x89AB;
81 int32x4_t vacc1xCDEF = vacc0xCDEF;
82 int32x4_t vacc1xGHIJ = vacc0xGHIJ;
83 int32x4_t vacc1xKLMN = vacc0xKLMN;
84 int32x4_t vacc1xOPQR = vacc0xOPQR;
85 int32x4_t vacc1xSTUV = vacc0xSTUV;
86 int32x4_t vacc2x0123 = vacc0x0123;
87 int32x4_t vacc2x4567 = vacc0x4567;
88 int32x4_t vacc2x89AB = vacc0x89AB;
89 int32x4_t vacc2xCDEF = vacc0xCDEF;
90 int32x4_t vacc2xGHIJ = vacc0xGHIJ;
91 int32x4_t vacc2xKLMN = vacc0xKLMN;
92 int32x4_t vacc2xOPQR = vacc0xOPQR;
93 int32x4_t vacc2xSTUV = vacc0xSTUV;
94 int32x4_t vacc3x0123 = vacc0x0123;
95 int32x4_t vacc3x4567 = vacc0x4567;
96 int32x4_t vacc3x89AB = vacc0x89AB;
97 int32x4_t vacc3xCDEF = vacc0xCDEF;
98 int32x4_t vacc3xGHIJ = vacc0xGHIJ;
99 int32x4_t vacc3xKLMN = vacc0xKLMN;
100 int32x4_t vacc3xOPQR = vacc0xOPQR;
101 int32x4_t vacc3xSTUV = vacc0xSTUV;
102
103 size_t p = ks;
104 do {
105 const int8_t* restrict a0 = a[0];
106 if XNN_UNPREDICTABLE(a0 != zero) {
107 a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
108 } else {
109 a0 = zero_data;
110 }
111 const int8_t* restrict a1 = a[1];
112 if XNN_UNPREDICTABLE(a1 != zero) {
113 a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
114 } else {
115 a1 = zero_data;
116 }
117 const int8_t* restrict a2 = a[2];
118 if XNN_UNPREDICTABLE(a2 != zero) {
119 a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
120 } else {
121 a2 = zero_data;
122 }
123 const int8_t* restrict a3 = a[3];
124 if XNN_UNPREDICTABLE(a3 != zero) {
125 a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
126 } else {
127 a3 = zero_data;
128 }
129 a += 4;
130
131 // Inner accumulation loop along the 32 columns.
132 size_t k = kc;
133 // 2x partial unrolled loop to load 8 bytes at a time.
134 while (k >= 8 * sizeof(int8_t)) {
135 // Load a 4x8 block of activations.
136 const int8x8_t va0x01234567 = vld1_s8(a0); a0 += 8;
137 const int8x8_t va1x01234567 = vld1_s8(a1); a1 += 8;
138 const int8x8_t va2x01234567 = vld1_s8(a2); a2 += 8;
139 const int8x8_t va3x01234567 = vld1_s8(a3); a3 += 8;
140
141 // Load a 8x32 block of weights.
142 const int8x16_t vb0123x0123 = vld1q_s8(w); w = (const int8_t*) w + 16;
143 const int8x16_t vb0123x4567 = vld1q_s8(w); w = (const int8_t*) w + 16;
144 const int8x16_t vb0123x89AB = vld1q_s8(w); w = (const int8_t*) w + 16;
145 const int8x16_t vb0123xCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
146 const int8x16_t vb0123xGHIJ = vld1q_s8(w); w = (const int8_t*) w + 16;
147 const int8x16_t vb0123xKLMN = vld1q_s8(w); w = (const int8_t*) w + 16;
148 const int8x16_t vb0123xOPQR = vld1q_s8(w); w = (const int8_t*) w + 16;
149 const int8x16_t vb0123xSTUV = vld1q_s8(w); w = (const int8_t*) w + 16;
150 const int8x16_t vb4567x0123 = vld1q_s8(w); w = (const int8_t*) w + 16;
151 const int8x16_t vb4567x4567 = vld1q_s8(w); w = (const int8_t*) w + 16;
152 const int8x16_t vb4567x89AB = vld1q_s8(w); w = (const int8_t*) w + 16;
153 const int8x16_t vb4567xCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
154 const int8x16_t vb4567xGHIJ = vld1q_s8(w); w = (const int8_t*) w + 16;
155 const int8x16_t vb4567xKLMN = vld1q_s8(w); w = (const int8_t*) w + 16;
156 const int8x16_t vb4567xOPQR = vld1q_s8(w); w = (const int8_t*) w + 16;
157 const int8x16_t vb4567xSTUV = vld1q_s8(w); w = (const int8_t*) w + 16;
158
159 // Multiply-accumulate: 4x8 * 8x32 --> 4x32.
160 vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb0123x0123, va0x01234567, 0);
161 vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb0123x4567, va0x01234567, 0);
162 vacc0x89AB = vdotq_lane_s32(vacc0x89AB, vb0123x89AB, va0x01234567, 0);
163 vacc0xCDEF = vdotq_lane_s32(vacc0xCDEF, vb0123xCDEF, va0x01234567, 0);
164 vacc0xGHIJ = vdotq_lane_s32(vacc0xGHIJ, vb0123xGHIJ, va0x01234567, 0);
165 vacc0xKLMN = vdotq_lane_s32(vacc0xKLMN, vb0123xKLMN, va0x01234567, 0);
166 vacc0xOPQR = vdotq_lane_s32(vacc0xOPQR, vb0123xOPQR, va0x01234567, 0);
167 vacc0xSTUV = vdotq_lane_s32(vacc0xSTUV, vb0123xSTUV, va0x01234567, 0);
168 vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb0123x0123, va1x01234567, 0);
169 vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb0123x4567, va1x01234567, 0);
170 vacc1x89AB = vdotq_lane_s32(vacc1x89AB, vb0123x89AB, va1x01234567, 0);
171 vacc1xCDEF = vdotq_lane_s32(vacc1xCDEF, vb0123xCDEF, va1x01234567, 0);
172 vacc1xGHIJ = vdotq_lane_s32(vacc1xGHIJ, vb0123xGHIJ, va1x01234567, 0);
173 vacc1xKLMN = vdotq_lane_s32(vacc1xKLMN, vb0123xKLMN, va1x01234567, 0);
174 vacc1xOPQR = vdotq_lane_s32(vacc1xOPQR, vb0123xOPQR, va1x01234567, 0);
175 vacc1xSTUV = vdotq_lane_s32(vacc1xSTUV, vb0123xSTUV, va1x01234567, 0);
176 vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb0123x0123, va2x01234567, 0);
177 vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb0123x4567, va2x01234567, 0);
178 vacc2x89AB = vdotq_lane_s32(vacc2x89AB, vb0123x89AB, va2x01234567, 0);
179 vacc2xCDEF = vdotq_lane_s32(vacc2xCDEF, vb0123xCDEF, va2x01234567, 0);
180 vacc2xGHIJ = vdotq_lane_s32(vacc2xGHIJ, vb0123xGHIJ, va2x01234567, 0);
181 vacc2xKLMN = vdotq_lane_s32(vacc2xKLMN, vb0123xKLMN, va2x01234567, 0);
182 vacc2xOPQR = vdotq_lane_s32(vacc2xOPQR, vb0123xOPQR, va2x01234567, 0);
183 vacc2xSTUV = vdotq_lane_s32(vacc2xSTUV, vb0123xSTUV, va2x01234567, 0);
184 vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb0123x0123, va3x01234567, 0);
185 vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb0123x4567, va3x01234567, 0);
186 vacc3x89AB = vdotq_lane_s32(vacc3x89AB, vb0123x89AB, va3x01234567, 0);
187 vacc3xCDEF = vdotq_lane_s32(vacc3xCDEF, vb0123xCDEF, va3x01234567, 0);
188 vacc3xGHIJ = vdotq_lane_s32(vacc3xGHIJ, vb0123xGHIJ, va3x01234567, 0);
189 vacc3xKLMN = vdotq_lane_s32(vacc3xKLMN, vb0123xKLMN, va3x01234567, 0);
190 vacc3xOPQR = vdotq_lane_s32(vacc3xOPQR, vb0123xOPQR, va3x01234567, 0);
191 vacc3xSTUV = vdotq_lane_s32(vacc3xSTUV, vb0123xSTUV, va3x01234567, 0);
192 vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb4567x0123, va0x01234567, 1);
193 vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb4567x4567, va0x01234567, 1);
194 vacc0x89AB = vdotq_lane_s32(vacc0x89AB, vb4567x89AB, va0x01234567, 1);
195 vacc0xCDEF = vdotq_lane_s32(vacc0xCDEF, vb4567xCDEF, va0x01234567, 1);
196 vacc0xGHIJ = vdotq_lane_s32(vacc0xGHIJ, vb4567xGHIJ, va0x01234567, 1);
197 vacc0xKLMN = vdotq_lane_s32(vacc0xKLMN, vb4567xKLMN, va0x01234567, 1);
198 vacc0xOPQR = vdotq_lane_s32(vacc0xOPQR, vb4567xOPQR, va0x01234567, 1);
199 vacc0xSTUV = vdotq_lane_s32(vacc0xSTUV, vb4567xSTUV, va0x01234567, 1);
200 vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb4567x0123, va1x01234567, 1);
201 vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb4567x4567, va1x01234567, 1);
202 vacc1x89AB = vdotq_lane_s32(vacc1x89AB, vb4567x89AB, va1x01234567, 1);
203 vacc1xCDEF = vdotq_lane_s32(vacc1xCDEF, vb4567xCDEF, va1x01234567, 1);
204 vacc1xGHIJ = vdotq_lane_s32(vacc1xGHIJ, vb4567xGHIJ, va1x01234567, 1);
205 vacc1xKLMN = vdotq_lane_s32(vacc1xKLMN, vb4567xKLMN, va1x01234567, 1);
206 vacc1xOPQR = vdotq_lane_s32(vacc1xOPQR, vb4567xOPQR, va1x01234567, 1);
207 vacc1xSTUV = vdotq_lane_s32(vacc1xSTUV, vb4567xSTUV, va1x01234567, 1);
208 vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb4567x0123, va2x01234567, 1);
209 vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb4567x4567, va2x01234567, 1);
210 vacc2x89AB = vdotq_lane_s32(vacc2x89AB, vb4567x89AB, va2x01234567, 1);
211 vacc2xCDEF = vdotq_lane_s32(vacc2xCDEF, vb4567xCDEF, va2x01234567, 1);
212 vacc2xGHIJ = vdotq_lane_s32(vacc2xGHIJ, vb4567xGHIJ, va2x01234567, 1);
213 vacc2xKLMN = vdotq_lane_s32(vacc2xKLMN, vb4567xKLMN, va2x01234567, 1);
214 vacc2xOPQR = vdotq_lane_s32(vacc2xOPQR, vb4567xOPQR, va2x01234567, 1);
215 vacc2xSTUV = vdotq_lane_s32(vacc2xSTUV, vb4567xSTUV, va2x01234567, 1);
216 vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb4567x0123, va3x01234567, 1);
217 vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb4567x4567, va3x01234567, 1);
218 vacc3x89AB = vdotq_lane_s32(vacc3x89AB, vb4567x89AB, va3x01234567, 1);
219 vacc3xCDEF = vdotq_lane_s32(vacc3xCDEF, vb4567xCDEF, va3x01234567, 1);
220 vacc3xGHIJ = vdotq_lane_s32(vacc3xGHIJ, vb4567xGHIJ, va3x01234567, 1);
221 vacc3xKLMN = vdotq_lane_s32(vacc3xKLMN, vb4567xKLMN, va3x01234567, 1);
222 vacc3xOPQR = vdotq_lane_s32(vacc3xOPQR, vb4567xOPQR, va3x01234567, 1);
223 vacc3xSTUV = vdotq_lane_s32(vacc3xSTUV, vb4567xSTUV, va3x01234567, 1);
224
225 k -= 8 * sizeof(int8_t);
226 }
227 // Handle up to 4 final positions of `k`
228 if XNN_UNLIKELY(k != 0) {
229 // Load a 4x4 block of activations.
230 const int8x8_t va0x01234567 = vld1_s8(a0);
231 const int8x8_t va1x01234567 = vld1_s8(a1);
232 const int8x8_t va2x01234567 = vld1_s8(a2);
233 const int8x8_t va3x01234567 = vld1_s8(a3);
234
235 // Load a 4x32 block of weights.
236 const int8x16_t vb0123x0123 = vld1q_s8(w); w = (const int8_t*) w + 16;
237 const int8x16_t vb0123x4567 = vld1q_s8(w); w = (const int8_t*) w + 16;
238 const int8x16_t vb0123x89AB = vld1q_s8(w); w = (const int8_t*) w + 16;
239 const int8x16_t vb0123xCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
240 const int8x16_t vb0123xGHIJ = vld1q_s8(w); w = (const int8_t*) w + 16;
241 const int8x16_t vb0123xKLMN = vld1q_s8(w); w = (const int8_t*) w + 16;
242 const int8x16_t vb0123xOPQR = vld1q_s8(w); w = (const int8_t*) w + 16;
243 const int8x16_t vb0123xSTUV = vld1q_s8(w); w = (const int8_t*) w + 16;
244
245 // Multiply-accumulate: 4x4 * 4x32 --> 4x32.
246 vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb0123x0123, va0x01234567, 0);
247 vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb0123x4567, va0x01234567, 0);
248 vacc0x89AB = vdotq_lane_s32(vacc0x89AB, vb0123x89AB, va0x01234567, 0);
249 vacc0xCDEF = vdotq_lane_s32(vacc0xCDEF, vb0123xCDEF, va0x01234567, 0);
250 vacc0xGHIJ = vdotq_lane_s32(vacc0xGHIJ, vb0123xGHIJ, va0x01234567, 0);
251 vacc0xKLMN = vdotq_lane_s32(vacc0xKLMN, vb0123xKLMN, va0x01234567, 0);
252 vacc0xOPQR = vdotq_lane_s32(vacc0xOPQR, vb0123xOPQR, va0x01234567, 0);
253 vacc0xSTUV = vdotq_lane_s32(vacc0xSTUV, vb0123xSTUV, va0x01234567, 0);
254 vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb0123x0123, va1x01234567, 0);
255 vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb0123x4567, va1x01234567, 0);
256 vacc1x89AB = vdotq_lane_s32(vacc1x89AB, vb0123x89AB, va1x01234567, 0);
257 vacc1xCDEF = vdotq_lane_s32(vacc1xCDEF, vb0123xCDEF, va1x01234567, 0);
258 vacc1xGHIJ = vdotq_lane_s32(vacc1xGHIJ, vb0123xGHIJ, va1x01234567, 0);
259 vacc1xKLMN = vdotq_lane_s32(vacc1xKLMN, vb0123xKLMN, va1x01234567, 0);
260 vacc1xOPQR = vdotq_lane_s32(vacc1xOPQR, vb0123xOPQR, va1x01234567, 0);
261 vacc1xSTUV = vdotq_lane_s32(vacc1xSTUV, vb0123xSTUV, va1x01234567, 0);
262 vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb0123x0123, va2x01234567, 0);
263 vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb0123x4567, va2x01234567, 0);
264 vacc2x89AB = vdotq_lane_s32(vacc2x89AB, vb0123x89AB, va2x01234567, 0);
265 vacc2xCDEF = vdotq_lane_s32(vacc2xCDEF, vb0123xCDEF, va2x01234567, 0);
266 vacc2xGHIJ = vdotq_lane_s32(vacc2xGHIJ, vb0123xGHIJ, va2x01234567, 0);
267 vacc2xKLMN = vdotq_lane_s32(vacc2xKLMN, vb0123xKLMN, va2x01234567, 0);
268 vacc2xOPQR = vdotq_lane_s32(vacc2xOPQR, vb0123xOPQR, va2x01234567, 0);
269 vacc2xSTUV = vdotq_lane_s32(vacc2xSTUV, vb0123xSTUV, va2x01234567, 0);
270 vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb0123x0123, va3x01234567, 0);
271 vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb0123x4567, va3x01234567, 0);
272 vacc3x89AB = vdotq_lane_s32(vacc3x89AB, vb0123x89AB, va3x01234567, 0);
273 vacc3xCDEF = vdotq_lane_s32(vacc3xCDEF, vb0123xCDEF, va3x01234567, 0);
274 vacc3xGHIJ = vdotq_lane_s32(vacc3xGHIJ, vb0123xGHIJ, va3x01234567, 0);
275 vacc3xKLMN = vdotq_lane_s32(vacc3xKLMN, vb0123xKLMN, va3x01234567, 0);
276 vacc3xOPQR = vdotq_lane_s32(vacc3xOPQR, vb0123xOPQR, va3x01234567, 0);
277 vacc3xSTUV = vdotq_lane_s32(vacc3xSTUV, vb0123xSTUV, va3x01234567, 0);
278 }
279 p -= 4 * sizeof(void*);
280 } while (p != 0);
281
282 float32x4_t vout0x0123 = vcvtq_f32_s32(vacc0x0123);
283 float32x4_t vout0x4567 = vcvtq_f32_s32(vacc0x4567);
284 float32x4_t vout0x89AB = vcvtq_f32_s32(vacc0x89AB);
285 float32x4_t vout0xCDEF = vcvtq_f32_s32(vacc0xCDEF);
286 float32x4_t vout0xGHIJ = vcvtq_f32_s32(vacc0xGHIJ);
287 float32x4_t vout0xKLMN = vcvtq_f32_s32(vacc0xKLMN);
288 float32x4_t vout0xOPQR = vcvtq_f32_s32(vacc0xOPQR);
289 float32x4_t vout0xSTUV = vcvtq_f32_s32(vacc0xSTUV);
290 float32x4_t vout1x0123 = vcvtq_f32_s32(vacc1x0123);
291 float32x4_t vout1x4567 = vcvtq_f32_s32(vacc1x4567);
292 float32x4_t vout1x89AB = vcvtq_f32_s32(vacc1x89AB);
293 float32x4_t vout1xCDEF = vcvtq_f32_s32(vacc1xCDEF);
294 float32x4_t vout1xGHIJ = vcvtq_f32_s32(vacc1xGHIJ);
295 float32x4_t vout1xKLMN = vcvtq_f32_s32(vacc1xKLMN);
296 float32x4_t vout1xOPQR = vcvtq_f32_s32(vacc1xOPQR);
297 float32x4_t vout1xSTUV = vcvtq_f32_s32(vacc1xSTUV);
298 float32x4_t vout2x0123 = vcvtq_f32_s32(vacc2x0123);
299 float32x4_t vout2x4567 = vcvtq_f32_s32(vacc2x4567);
300 float32x4_t vout2x89AB = vcvtq_f32_s32(vacc2x89AB);
301 float32x4_t vout2xCDEF = vcvtq_f32_s32(vacc2xCDEF);
302 float32x4_t vout2xGHIJ = vcvtq_f32_s32(vacc2xGHIJ);
303 float32x4_t vout2xKLMN = vcvtq_f32_s32(vacc2xKLMN);
304 float32x4_t vout2xOPQR = vcvtq_f32_s32(vacc2xOPQR);
305 float32x4_t vout2xSTUV = vcvtq_f32_s32(vacc2xSTUV);
306 float32x4_t vout3x0123 = vcvtq_f32_s32(vacc3x0123);
307 float32x4_t vout3x4567 = vcvtq_f32_s32(vacc3x4567);
308 float32x4_t vout3x89AB = vcvtq_f32_s32(vacc3x89AB);
309 float32x4_t vout3xCDEF = vcvtq_f32_s32(vacc3xCDEF);
310 float32x4_t vout3xGHIJ = vcvtq_f32_s32(vacc3xGHIJ);
311 float32x4_t vout3xKLMN = vcvtq_f32_s32(vacc3xKLMN);
312 float32x4_t vout3xOPQR = vcvtq_f32_s32(vacc3xOPQR);
313 float32x4_t vout3xSTUV = vcvtq_f32_s32(vacc3xSTUV);
314 const float32x4_t vinput_scale = vld1q_dup_f32(&quantization_params->inv_scale);
315 vout0x0123 = vmulq_f32(vout0x0123, vinput_scale);
316 vout0x4567 = vmulq_f32(vout0x4567, vinput_scale);
317 vout0x89AB = vmulq_f32(vout0x89AB, vinput_scale);
318 vout0xCDEF = vmulq_f32(vout0xCDEF, vinput_scale);
319 vout0xGHIJ = vmulq_f32(vout0xGHIJ, vinput_scale);
320 vout0xKLMN = vmulq_f32(vout0xKLMN, vinput_scale);
321 vout0xOPQR = vmulq_f32(vout0xOPQR, vinput_scale);
322 vout0xSTUV = vmulq_f32(vout0xSTUV, vinput_scale);
323 vout1x0123 = vmulq_f32(vout1x0123, vinput_scale);
324 vout1x4567 = vmulq_f32(vout1x4567, vinput_scale);
325 vout1x89AB = vmulq_f32(vout1x89AB, vinput_scale);
326 vout1xCDEF = vmulq_f32(vout1xCDEF, vinput_scale);
327 vout1xGHIJ = vmulq_f32(vout1xGHIJ, vinput_scale);
328 vout1xKLMN = vmulq_f32(vout1xKLMN, vinput_scale);
329 vout1xOPQR = vmulq_f32(vout1xOPQR, vinput_scale);
330 vout1xSTUV = vmulq_f32(vout1xSTUV, vinput_scale);
331 vout2x0123 = vmulq_f32(vout2x0123, vinput_scale);
332 vout2x4567 = vmulq_f32(vout2x4567, vinput_scale);
333 vout2x89AB = vmulq_f32(vout2x89AB, vinput_scale);
334 vout2xCDEF = vmulq_f32(vout2xCDEF, vinput_scale);
335 vout2xGHIJ = vmulq_f32(vout2xGHIJ, vinput_scale);
336 vout2xKLMN = vmulq_f32(vout2xKLMN, vinput_scale);
337 vout2xOPQR = vmulq_f32(vout2xOPQR, vinput_scale);
338 vout2xSTUV = vmulq_f32(vout2xSTUV, vinput_scale);
339 vout3x0123 = vmulq_f32(vout3x0123, vinput_scale);
340 vout3x4567 = vmulq_f32(vout3x4567, vinput_scale);
341 vout3x89AB = vmulq_f32(vout3x89AB, vinput_scale);
342 vout3xCDEF = vmulq_f32(vout3xCDEF, vinput_scale);
343 vout3xGHIJ = vmulq_f32(vout3xGHIJ, vinput_scale);
344 vout3xKLMN = vmulq_f32(vout3xKLMN, vinput_scale);
345 vout3xOPQR = vmulq_f32(vout3xOPQR, vinput_scale);
346 vout3xSTUV = vmulq_f32(vout3xSTUV, vinput_scale);
347
348 const float32x4_t vfilter_output_scale0123 = vld1q_f32(w); w = (const float*) w + 4;
349 const float32x4_t vfilter_output_scale4567 = vld1q_f32(w); w = (const float*) w + 4;
350 const float32x4_t vfilter_output_scale89AB = vld1q_f32(w); w = (const float*) w + 4;
351 const float32x4_t vfilter_output_scaleCDEF = vld1q_f32(w); w = (const float*) w + 4;
352 const float32x4_t vfilter_output_scaleGHIJ = vld1q_f32(w); w = (const float*) w + 4;
353 const float32x4_t vfilter_output_scaleKLMN = vld1q_f32(w); w = (const float*) w + 4;
354 const float32x4_t vfilter_output_scaleOPQR = vld1q_f32(w); w = (const float*) w + 4;
355 const float32x4_t vfilter_output_scaleSTUV = vld1q_f32(w); w = (const float*) w + 4;
356
357 const float32x4_t vbias0123 = vld1q_f32(w); w = (const float*) w + 4;
358 #if XNN_ARCH_ARM64
359 vout0x0123 = vfmaq_f32(vbias0123, vout0x0123, vfilter_output_scale0123);
360 vout1x0123 = vfmaq_f32(vbias0123, vout1x0123, vfilter_output_scale0123);
361 vout2x0123 = vfmaq_f32(vbias0123, vout2x0123, vfilter_output_scale0123);
362 vout3x0123 = vfmaq_f32(vbias0123, vout3x0123, vfilter_output_scale0123);
363 #else
364 vout0x0123 = vmlaq_f32(vbias0123, vout0x0123, vfilter_output_scale0123);
365 vout1x0123 = vmlaq_f32(vbias0123, vout1x0123, vfilter_output_scale0123);
366 vout2x0123 = vmlaq_f32(vbias0123, vout2x0123, vfilter_output_scale0123);
367 vout3x0123 = vmlaq_f32(vbias0123, vout3x0123, vfilter_output_scale0123);
368 #endif
369 const float32x4_t vbias4567 = vld1q_f32(w); w = (const float*) w + 4;
370 #if XNN_ARCH_ARM64
371 vout0x4567 = vfmaq_f32(vbias4567, vout0x4567, vfilter_output_scale4567);
372 vout1x4567 = vfmaq_f32(vbias4567, vout1x4567, vfilter_output_scale4567);
373 vout2x4567 = vfmaq_f32(vbias4567, vout2x4567, vfilter_output_scale4567);
374 vout3x4567 = vfmaq_f32(vbias4567, vout3x4567, vfilter_output_scale4567);
375 #else
376 vout0x4567 = vmlaq_f32(vbias4567, vout0x4567, vfilter_output_scale4567);
377 vout1x4567 = vmlaq_f32(vbias4567, vout1x4567, vfilter_output_scale4567);
378 vout2x4567 = vmlaq_f32(vbias4567, vout2x4567, vfilter_output_scale4567);
379 vout3x4567 = vmlaq_f32(vbias4567, vout3x4567, vfilter_output_scale4567);
380 #endif
381 const float32x4_t vbias89AB = vld1q_f32(w); w = (const float*) w + 4;
382 #if XNN_ARCH_ARM64
383 vout0x89AB = vfmaq_f32(vbias89AB, vout0x89AB, vfilter_output_scale89AB);
384 vout1x89AB = vfmaq_f32(vbias89AB, vout1x89AB, vfilter_output_scale89AB);
385 vout2x89AB = vfmaq_f32(vbias89AB, vout2x89AB, vfilter_output_scale89AB);
386 vout3x89AB = vfmaq_f32(vbias89AB, vout3x89AB, vfilter_output_scale89AB);
387 #else
388 vout0x89AB = vmlaq_f32(vbias89AB, vout0x89AB, vfilter_output_scale89AB);
389 vout1x89AB = vmlaq_f32(vbias89AB, vout1x89AB, vfilter_output_scale89AB);
390 vout2x89AB = vmlaq_f32(vbias89AB, vout2x89AB, vfilter_output_scale89AB);
391 vout3x89AB = vmlaq_f32(vbias89AB, vout3x89AB, vfilter_output_scale89AB);
392 #endif
393 const float32x4_t vbiasCDEF = vld1q_f32(w); w = (const float*) w + 4;
394 #if XNN_ARCH_ARM64
395 vout0xCDEF = vfmaq_f32(vbiasCDEF, vout0xCDEF, vfilter_output_scaleCDEF);
396 vout1xCDEF = vfmaq_f32(vbiasCDEF, vout1xCDEF, vfilter_output_scaleCDEF);
397 vout2xCDEF = vfmaq_f32(vbiasCDEF, vout2xCDEF, vfilter_output_scaleCDEF);
398 vout3xCDEF = vfmaq_f32(vbiasCDEF, vout3xCDEF, vfilter_output_scaleCDEF);
399 #else
400 vout0xCDEF = vmlaq_f32(vbiasCDEF, vout0xCDEF, vfilter_output_scaleCDEF);
401 vout1xCDEF = vmlaq_f32(vbiasCDEF, vout1xCDEF, vfilter_output_scaleCDEF);
402 vout2xCDEF = vmlaq_f32(vbiasCDEF, vout2xCDEF, vfilter_output_scaleCDEF);
403 vout3xCDEF = vmlaq_f32(vbiasCDEF, vout3xCDEF, vfilter_output_scaleCDEF);
404 #endif
405 const float32x4_t vbiasGHIJ = vld1q_f32(w); w = (const float*) w + 4;
406 #if XNN_ARCH_ARM64
407 vout0xGHIJ = vfmaq_f32(vbiasGHIJ, vout0xGHIJ, vfilter_output_scaleGHIJ);
408 vout1xGHIJ = vfmaq_f32(vbiasGHIJ, vout1xGHIJ, vfilter_output_scaleGHIJ);
409 vout2xGHIJ = vfmaq_f32(vbiasGHIJ, vout2xGHIJ, vfilter_output_scaleGHIJ);
410 vout3xGHIJ = vfmaq_f32(vbiasGHIJ, vout3xGHIJ, vfilter_output_scaleGHIJ);
411 #else
412 vout0xGHIJ = vmlaq_f32(vbiasGHIJ, vout0xGHIJ, vfilter_output_scaleGHIJ);
413 vout1xGHIJ = vmlaq_f32(vbiasGHIJ, vout1xGHIJ, vfilter_output_scaleGHIJ);
414 vout2xGHIJ = vmlaq_f32(vbiasGHIJ, vout2xGHIJ, vfilter_output_scaleGHIJ);
415 vout3xGHIJ = vmlaq_f32(vbiasGHIJ, vout3xGHIJ, vfilter_output_scaleGHIJ);
416 #endif
417 const float32x4_t vbiasKLMN = vld1q_f32(w); w = (const float*) w + 4;
418 #if XNN_ARCH_ARM64
419 vout0xKLMN = vfmaq_f32(vbiasKLMN, vout0xKLMN, vfilter_output_scaleKLMN);
420 vout1xKLMN = vfmaq_f32(vbiasKLMN, vout1xKLMN, vfilter_output_scaleKLMN);
421 vout2xKLMN = vfmaq_f32(vbiasKLMN, vout2xKLMN, vfilter_output_scaleKLMN);
422 vout3xKLMN = vfmaq_f32(vbiasKLMN, vout3xKLMN, vfilter_output_scaleKLMN);
423 #else
424 vout0xKLMN = vmlaq_f32(vbiasKLMN, vout0xKLMN, vfilter_output_scaleKLMN);
425 vout1xKLMN = vmlaq_f32(vbiasKLMN, vout1xKLMN, vfilter_output_scaleKLMN);
426 vout2xKLMN = vmlaq_f32(vbiasKLMN, vout2xKLMN, vfilter_output_scaleKLMN);
427 vout3xKLMN = vmlaq_f32(vbiasKLMN, vout3xKLMN, vfilter_output_scaleKLMN);
428 #endif
429 const float32x4_t vbiasOPQR = vld1q_f32(w); w = (const float*) w + 4;
430 #if XNN_ARCH_ARM64
431 vout0xOPQR = vfmaq_f32(vbiasOPQR, vout0xOPQR, vfilter_output_scaleOPQR);
432 vout1xOPQR = vfmaq_f32(vbiasOPQR, vout1xOPQR, vfilter_output_scaleOPQR);
433 vout2xOPQR = vfmaq_f32(vbiasOPQR, vout2xOPQR, vfilter_output_scaleOPQR);
434 vout3xOPQR = vfmaq_f32(vbiasOPQR, vout3xOPQR, vfilter_output_scaleOPQR);
435 #else
436 vout0xOPQR = vmlaq_f32(vbiasOPQR, vout0xOPQR, vfilter_output_scaleOPQR);
437 vout1xOPQR = vmlaq_f32(vbiasOPQR, vout1xOPQR, vfilter_output_scaleOPQR);
438 vout2xOPQR = vmlaq_f32(vbiasOPQR, vout2xOPQR, vfilter_output_scaleOPQR);
439 vout3xOPQR = vmlaq_f32(vbiasOPQR, vout3xOPQR, vfilter_output_scaleOPQR);
440 #endif
441 const float32x4_t vbiasSTUV = vld1q_f32(w); w = (const float*) w + 4;
442 #if XNN_ARCH_ARM64
443 vout0xSTUV = vfmaq_f32(vbiasSTUV, vout0xSTUV, vfilter_output_scaleSTUV);
444 vout1xSTUV = vfmaq_f32(vbiasSTUV, vout1xSTUV, vfilter_output_scaleSTUV);
445 vout2xSTUV = vfmaq_f32(vbiasSTUV, vout2xSTUV, vfilter_output_scaleSTUV);
446 vout3xSTUV = vfmaq_f32(vbiasSTUV, vout3xSTUV, vfilter_output_scaleSTUV);
447 #else
448 vout0xSTUV = vmlaq_f32(vbiasSTUV, vout0xSTUV, vfilter_output_scaleSTUV);
449 vout1xSTUV = vmlaq_f32(vbiasSTUV, vout1xSTUV, vfilter_output_scaleSTUV);
450 vout2xSTUV = vmlaq_f32(vbiasSTUV, vout2xSTUV, vfilter_output_scaleSTUV);
451 vout3xSTUV = vmlaq_f32(vbiasSTUV, vout3xSTUV, vfilter_output_scaleSTUV);
452 #endif
453
454 float16x8_t vfp16out0x01234567 = vcombine_f16(vcvt_f16_f32(vout0x0123), vcvt_f16_f32(vout0x4567));
455 float16x8_t vfp16out0x89ABCDEF = vcombine_f16(vcvt_f16_f32(vout0x89AB), vcvt_f16_f32(vout0xCDEF));
456 float16x8_t vfp16out0xGHIJKLMN = vcombine_f16(vcvt_f16_f32(vout0xGHIJ), vcvt_f16_f32(vout0xKLMN));
457 float16x8_t vfp16out0xOPQRSTUV = vcombine_f16(vcvt_f16_f32(vout0xOPQR), vcvt_f16_f32(vout0xSTUV));
458 float16x8_t vfp16out1x01234567 = vcombine_f16(vcvt_f16_f32(vout1x0123), vcvt_f16_f32(vout1x4567));
459 float16x8_t vfp16out1x89ABCDEF = vcombine_f16(vcvt_f16_f32(vout1x89AB), vcvt_f16_f32(vout1xCDEF));
460 float16x8_t vfp16out1xGHIJKLMN = vcombine_f16(vcvt_f16_f32(vout1xGHIJ), vcvt_f16_f32(vout1xKLMN));
461 float16x8_t vfp16out1xOPQRSTUV = vcombine_f16(vcvt_f16_f32(vout1xOPQR), vcvt_f16_f32(vout1xSTUV));
462 float16x8_t vfp16out2x01234567 = vcombine_f16(vcvt_f16_f32(vout2x0123), vcvt_f16_f32(vout2x4567));
463 float16x8_t vfp16out2x89ABCDEF = vcombine_f16(vcvt_f16_f32(vout2x89AB), vcvt_f16_f32(vout2xCDEF));
464 float16x8_t vfp16out2xGHIJKLMN = vcombine_f16(vcvt_f16_f32(vout2xGHIJ), vcvt_f16_f32(vout2xKLMN));
465 float16x8_t vfp16out2xOPQRSTUV = vcombine_f16(vcvt_f16_f32(vout2xOPQR), vcvt_f16_f32(vout2xSTUV));
466 float16x8_t vfp16out3x01234567 = vcombine_f16(vcvt_f16_f32(vout3x0123), vcvt_f16_f32(vout3x4567));
467 float16x8_t vfp16out3x89ABCDEF = vcombine_f16(vcvt_f16_f32(vout3x89AB), vcvt_f16_f32(vout3xCDEF));
468 float16x8_t vfp16out3xGHIJKLMN = vcombine_f16(vcvt_f16_f32(vout3xGHIJ), vcvt_f16_f32(vout3xKLMN));
469 float16x8_t vfp16out3xOPQRSTUV = vcombine_f16(vcvt_f16_f32(vout3xOPQR), vcvt_f16_f32(vout3xSTUV));
470
471 const float16x8_t voutput_min = vreinterpretq_f16_u16(vld1q_dup_u16((const uint16_t*) &params->scalar.min));
472 vfp16out0x01234567 = vmaxq_f16(vfp16out0x01234567, voutput_min);
473 vfp16out0x89ABCDEF = vmaxq_f16(vfp16out0x89ABCDEF, voutput_min);
474 vfp16out0xGHIJKLMN = vmaxq_f16(vfp16out0xGHIJKLMN, voutput_min);
475 vfp16out0xOPQRSTUV = vmaxq_f16(vfp16out0xOPQRSTUV, voutput_min);
476 vfp16out1x01234567 = vmaxq_f16(vfp16out1x01234567, voutput_min);
477 vfp16out1x89ABCDEF = vmaxq_f16(vfp16out1x89ABCDEF, voutput_min);
478 vfp16out1xGHIJKLMN = vmaxq_f16(vfp16out1xGHIJKLMN, voutput_min);
479 vfp16out1xOPQRSTUV = vmaxq_f16(vfp16out1xOPQRSTUV, voutput_min);
480 vfp16out2x01234567 = vmaxq_f16(vfp16out2x01234567, voutput_min);
481 vfp16out2x89ABCDEF = vmaxq_f16(vfp16out2x89ABCDEF, voutput_min);
482 vfp16out2xGHIJKLMN = vmaxq_f16(vfp16out2xGHIJKLMN, voutput_min);
483 vfp16out2xOPQRSTUV = vmaxq_f16(vfp16out2xOPQRSTUV, voutput_min);
484 vfp16out3x01234567 = vmaxq_f16(vfp16out3x01234567, voutput_min);
485 vfp16out3x89ABCDEF = vmaxq_f16(vfp16out3x89ABCDEF, voutput_min);
486 vfp16out3xGHIJKLMN = vmaxq_f16(vfp16out3xGHIJKLMN, voutput_min);
487 vfp16out3xOPQRSTUV = vmaxq_f16(vfp16out3xOPQRSTUV, voutput_min);
488 const float16x8_t voutput_max = vreinterpretq_f16_u16(vld1q_dup_u16((const uint16_t*) &params->scalar.max));
489 vfp16out0x01234567 = vminq_f16(vfp16out0x01234567, voutput_max);
490 vfp16out0x89ABCDEF = vminq_f16(vfp16out0x89ABCDEF, voutput_max);
491 vfp16out0xGHIJKLMN = vminq_f16(vfp16out0xGHIJKLMN, voutput_max);
492 vfp16out0xOPQRSTUV = vminq_f16(vfp16out0xOPQRSTUV, voutput_max);
493 vfp16out1x01234567 = vminq_f16(vfp16out1x01234567, voutput_max);
494 vfp16out1x89ABCDEF = vminq_f16(vfp16out1x89ABCDEF, voutput_max);
495 vfp16out1xGHIJKLMN = vminq_f16(vfp16out1xGHIJKLMN, voutput_max);
496 vfp16out1xOPQRSTUV = vminq_f16(vfp16out1xOPQRSTUV, voutput_max);
497 vfp16out2x01234567 = vminq_f16(vfp16out2x01234567, voutput_max);
498 vfp16out2x89ABCDEF = vminq_f16(vfp16out2x89ABCDEF, voutput_max);
499 vfp16out2xGHIJKLMN = vminq_f16(vfp16out2xGHIJKLMN, voutput_max);
500 vfp16out2xOPQRSTUV = vminq_f16(vfp16out2xOPQRSTUV, voutput_max);
501 vfp16out3x01234567 = vminq_f16(vfp16out3x01234567, voutput_max);
502 vfp16out3x89ABCDEF = vminq_f16(vfp16out3x89ABCDEF, voutput_max);
503 vfp16out3xGHIJKLMN = vminq_f16(vfp16out3xGHIJKLMN, voutput_max);
504 vfp16out3xOPQRSTUV = vminq_f16(vfp16out3xOPQRSTUV, voutput_max);
505 if XNN_LIKELY(nc >= 32) {
506 vst1q_u16(c3, vreinterpretq_u16_f16(vfp16out3x01234567));
507 vst1q_u16(c3 + 8, vreinterpretq_u16_f16(vfp16out3x89ABCDEF));
508 vst1q_u16(c3 + 16, vreinterpretq_u16_f16(vfp16out3xGHIJKLMN));
509 vst1q_u16(c3 + 24, vreinterpretq_u16_f16(vfp16out3xOPQRSTUV));
510 vst1q_u16(c2, vreinterpretq_u16_f16(vfp16out2x01234567));
511 vst1q_u16(c2 + 8, vreinterpretq_u16_f16(vfp16out2x89ABCDEF));
512 vst1q_u16(c2 + 16, vreinterpretq_u16_f16(vfp16out2xGHIJKLMN));
513 vst1q_u16(c2 + 24, vreinterpretq_u16_f16(vfp16out2xOPQRSTUV));
514 vst1q_u16(c1, vreinterpretq_u16_f16(vfp16out1x01234567));
515 vst1q_u16(c1 + 8, vreinterpretq_u16_f16(vfp16out1x89ABCDEF));
516 vst1q_u16(c1 + 16, vreinterpretq_u16_f16(vfp16out1xGHIJKLMN));
517 vst1q_u16(c1 + 24, vreinterpretq_u16_f16(vfp16out1xOPQRSTUV));
518 vst1q_u16(c0, vreinterpretq_u16_f16(vfp16out0x01234567));
519 vst1q_u16(c0 + 8, vreinterpretq_u16_f16(vfp16out0x89ABCDEF));
520 vst1q_u16(c0 + 16, vreinterpretq_u16_f16(vfp16out0xGHIJKLMN));
521 vst1q_u16(c0 + 24, vreinterpretq_u16_f16(vfp16out0xOPQRSTUV));
522
523 a = (const int8_t**restrict) ((uintptr_t) a - ks);
524
525 c0 = (uint16_t*) ((uintptr_t) c0 + cn_stride);
526 c1 = (uint16_t*) ((uintptr_t) c1 + cn_stride);
527 c2 = (uint16_t*) ((uintptr_t) c2 + cn_stride);
528 c3 = (uint16_t*) ((uintptr_t) c3 + cn_stride);
529
530 nc -= 32;
531 } else {
532 if (nc & 16) {
533 vst1q_u16(c3, vreinterpretq_u16_f16(vfp16out3x01234567)); c3 += 8;
534 vfp16out3x01234567 = vfp16out3xGHIJKLMN;
535 vst1q_u16(c2, vreinterpretq_u16_f16(vfp16out2x01234567)); c2 += 8;
536 vfp16out2x01234567 = vfp16out2xGHIJKLMN;
537 vst1q_u16(c1, vreinterpretq_u16_f16(vfp16out1x01234567)); c1 += 8;
538 vfp16out1x01234567 = vfp16out1xGHIJKLMN;
539 vst1q_u16(c0, vreinterpretq_u16_f16(vfp16out0x01234567)); c0 += 8;
540 vfp16out0x01234567 = vfp16out0xGHIJKLMN;
541 vst1q_u16(c3, vreinterpretq_u16_f16(vfp16out3x89ABCDEF)); c3 += 8;
542 vfp16out3x89ABCDEF = vfp16out3xOPQRSTUV;
543 vst1q_u16(c2, vreinterpretq_u16_f16(vfp16out2x89ABCDEF)); c2 += 8;
544 vfp16out2x89ABCDEF = vfp16out2xOPQRSTUV;
545 vst1q_u16(c1, vreinterpretq_u16_f16(vfp16out1x89ABCDEF)); c1 += 8;
546 vfp16out1x89ABCDEF = vfp16out1xOPQRSTUV;
547 vst1q_u16(c0, vreinterpretq_u16_f16(vfp16out0x89ABCDEF)); c0 += 8;
548 vfp16out0x89ABCDEF = vfp16out0xOPQRSTUV;
549 }
550 if (nc & 8) {
551 vst1q_u16(c3, vreinterpretq_u16_f16(vfp16out3x01234567)); c3 += 8;
552 vfp16out3x01234567 = vfp16out3x89ABCDEF;
553 vst1q_u16(c2, vreinterpretq_u16_f16(vfp16out2x01234567)); c2 += 8;
554 vfp16out2x01234567 = vfp16out2x89ABCDEF;
555 vst1q_u16(c1, vreinterpretq_u16_f16(vfp16out1x01234567)); c1 += 8;
556 vfp16out1x01234567 = vfp16out1x89ABCDEF;
557 vst1q_u16(c0, vreinterpretq_u16_f16(vfp16out0x01234567)); c0 += 8;
558 vfp16out0x01234567 = vfp16out0x89ABCDEF;
559 }
560 float16x4_t vfp16out3x0123 = vget_low_f16(vfp16out3x01234567);
561 float16x4_t vfp16out2x0123 = vget_low_f16(vfp16out2x01234567);
562 float16x4_t vfp16out1x0123 = vget_low_f16(vfp16out1x01234567);
563 float16x4_t vfp16out0x0123 = vget_low_f16(vfp16out0x01234567);
564 if (nc & 4) {
565 vst1_u16(c3, vreinterpret_u16_f16(vfp16out3x0123)); c3 += 4;
566 vst1_u16(c2, vreinterpret_u16_f16(vfp16out2x0123)); c2 += 4;
567 vst1_u16(c1, vreinterpret_u16_f16(vfp16out1x0123)); c1 += 4;
568 vst1_u16(c0, vreinterpret_u16_f16(vfp16out0x0123)); c0 += 4;
569 vfp16out3x0123 = vget_high_f16(vfp16out3x01234567);
570 vfp16out2x0123 = vget_high_f16(vfp16out2x01234567);
571 vfp16out1x0123 = vget_high_f16(vfp16out1x01234567);
572 vfp16out0x0123 = vget_high_f16(vfp16out0x01234567);
573 }
574 if (nc & 2) {
575 vst1_lane_u32((void*) c3, vreinterpret_u32_f16(vfp16out3x0123), 0); c3 += 2;
576 vst1_lane_u32((void*) c2, vreinterpret_u32_f16(vfp16out2x0123), 0); c2 += 2;
577 vst1_lane_u32((void*) c1, vreinterpret_u32_f16(vfp16out1x0123), 0); c1 += 2;
578 vst1_lane_u32((void*) c0, vreinterpret_u32_f16(vfp16out0x0123), 0); c0 += 2;
579 vfp16out3x0123 = vext_f16(vfp16out3x0123, vfp16out3x0123, 2);
580 vfp16out2x0123 = vext_f16(vfp16out2x0123, vfp16out2x0123, 2);
581 vfp16out1x0123 = vext_f16(vfp16out1x0123, vfp16out1x0123, 2);
582 vfp16out0x0123 = vext_f16(vfp16out0x0123, vfp16out0x0123, 2);
583 }
584 if (nc & 1) {
585 vst1_lane_u16(c3, vreinterpret_u16_f16(vfp16out3x0123), 0);
586 vst1_lane_u16(c2, vreinterpret_u16_f16(vfp16out2x0123), 0);
587 vst1_lane_u16(c1, vreinterpret_u16_f16(vfp16out1x0123), 0);
588 vst1_lane_u16(c0, vreinterpret_u16_f16(vfp16out0x0123), 0);
589 }
590 nc = 0;
591 }
592 } while (nc != 0);
593 }