Debian Codegraph @

src/qd8-f32-qc8w-igemm/gen/qd8-f32-qc8w-igemm-6x16-minmax-neon-mlal-lane-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/neon-mlal-lane.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/common.h"
16 #include "xnnpack/igemm.h"
17 #include "xnnpack/prefetch.h"
18
19
20 void xnn_qd8_f32_qc8w_igemm_minmax_ukernel_6x16__neon_mlal_lane_prfm(
21 size_t mr,
22 size_t nc,
23 size_t kc,
24 size_t ks,
25 const int8_t** restrict a,
26 const void* restrict w,
27 float* restrict c,
28 size_t cm_stride,
29 size_t cn_stride,
30 size_t a_offset,
31 const int8_t* zero,
32 const int8_t* zero_data,
33 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
34 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
35 {
36 assert(mr != 0);
37 assert(mr <= 6);
38 assert(nc != 0);
39 assert(kc != 0);
40 assert(ks != 0);
41 assert(ks % (6 * sizeof(void*)) == 0);
42 assert(a_offset % sizeof(int8_t) == 0);
43 assert(a != NULL);
44 assert(w != NULL);
45 assert(c != NULL);
46
47 float* c0 = c;
48 float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
49 if XNN_UNPREDICTABLE(mr < 2) {
50 c1 = c0;
51 }
52 float* c2 = (float*) ((uintptr_t) c1 + cm_stride);
53 if XNN_UNPREDICTABLE(mr <= 2) {
54 c2 = c1;
55 }
56 float* c3 = (float*) ((uintptr_t) c2 + cm_stride);
57 if XNN_UNPREDICTABLE(mr < 4) {
58 c3 = c2;
59 }
60 float* c4 = (float*) ((uintptr_t) c3 + cm_stride);
61 if XNN_UNPREDICTABLE(mr <= 4) {
62 c4 = c3;
63 }
64 float* c5 = (float*) ((uintptr_t) c4 + cm_stride);
65 if XNN_UNPREDICTABLE(mr != 6) {
66 c5 = c4;
67 }
68
69 do {
70 int32x4_t vksum0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
71 int32x4_t vksum4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
72 int32x4_t vksum89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
73 int32x4_t vksumCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
74 const int32x4_t vinput_zero_point = vld1q_dup_s32(&quantization_params->zero_point);
75 int32x4_t vacc0x0123 = vmulq_s32(vksum0123, vinput_zero_point);
76 int32x4_t vacc0x4567 = vmulq_s32(vksum4567, vinput_zero_point);
77 int32x4_t vacc0x89AB = vmulq_s32(vksum89AB, vinput_zero_point);
78 int32x4_t vacc0xCDEF = vmulq_s32(vksumCDEF, vinput_zero_point);
79 int32x4_t vacc1x0123 = vacc0x0123;
80 int32x4_t vacc1x4567 = vacc0x4567;
81 int32x4_t vacc1x89AB = vacc0x89AB;
82 int32x4_t vacc1xCDEF = vacc0xCDEF;
83 int32x4_t vacc2x0123 = vacc0x0123;
84 int32x4_t vacc2x4567 = vacc0x4567;
85 int32x4_t vacc2x89AB = vacc0x89AB;
86 int32x4_t vacc2xCDEF = vacc0xCDEF;
87 int32x4_t vacc3x0123 = vacc0x0123;
88 int32x4_t vacc3x4567 = vacc0x4567;
89 int32x4_t vacc3x89AB = vacc0x89AB;
90 int32x4_t vacc3xCDEF = vacc0xCDEF;
91 int32x4_t vacc4x0123 = vacc0x0123;
92 int32x4_t vacc4x4567 = vacc0x4567;
93 int32x4_t vacc4x89AB = vacc0x89AB;
94 int32x4_t vacc4xCDEF = vacc0xCDEF;
95 int32x4_t vacc5x0123 = vacc0x0123;
96 int32x4_t vacc5x4567 = vacc0x4567;
97 int32x4_t vacc5x89AB = vacc0x89AB;
98 int32x4_t vacc5xCDEF = vacc0xCDEF;
99
100 size_t p = ks;
101 do {
102 const int8_t* restrict a0 = a[0];
103 if XNN_UNPREDICTABLE(a0 != zero) {
104 a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
105 } else {
106 a0 = zero_data;
107 }
108 const int8_t* restrict a1 = a[1];
109 if XNN_UNPREDICTABLE(a1 != zero) {
110 a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
111 } else {
112 a1 = zero_data;
113 }
114 const int8_t* restrict a2 = a[2];
115 if XNN_UNPREDICTABLE(a2 != zero) {
116 a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
117 } else {
118 a2 = zero_data;
119 }
120 const int8_t* restrict a3 = a[3];
121 if XNN_UNPREDICTABLE(a3 != zero) {
122 a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
123 } else {
124 a3 = zero_data;
125 }
126 const int8_t* restrict a4 = a[4];
127 if XNN_UNPREDICTABLE(a4 != zero) {
128 a4 = (const int8_t*) ((uintptr_t) a4 + a_offset);
129 } else {
130 a4 = zero_data;
131 }
132 const int8_t* restrict a5 = a[5];
133 if XNN_UNPREDICTABLE(a5 != zero) {
134 a5 = (const int8_t*) ((uintptr_t) a5 + a_offset);
135 } else {
136 a5 = zero_data;
137 }
138 a += 6;
139
140 size_t k = kc;
141 while (k >= 8 * sizeof(int8_t)) {
142 const int8x8_t va0 = vld1_s8(a0); a0 += 8;
143 const int16x8_t vxa0 = vmovl_s8(va0);
144 const int8x8_t va1 = vld1_s8(a1); a1 += 8;
145 const int16x8_t vxa1 = vmovl_s8(va1);
146 const int8x8_t va2 = vld1_s8(a2); a2 += 8;
147 const int16x8_t vxa2 = vmovl_s8(va2);
148 const int8x8_t va3 = vld1_s8(a3); a3 += 8;
149 const int16x8_t vxa3 = vmovl_s8(va3);
150 const int8x8_t va4 = vld1_s8(a4); a4 += 8;
151 const int16x8_t vxa4 = vmovl_s8(va4);
152 const int8x8_t va5 = vld1_s8(a5); a5 += 8;
153 const int16x8_t vxa5 = vmovl_s8(va5);
154
155 const int8x8_t vb01234567c0 = vld1_s8(w); w = (const int8_t*) w + 8;
156 const int16x8_t vxb01234567c0 = vmovl_s8(vb01234567c0);
157
158 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa0), 0);
159 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa0), 0);
160 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa1), 0);
161 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa1), 0);
162 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa2), 0);
163 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa2), 0);
164 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa3), 0);
165 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa3), 0);
166 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa4), 0);
167 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa4), 0);
168 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa5), 0);
169 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa5), 0);
170 const int8x8_t vb89ABCDEFc0 = vld1_s8(w); w = (const int8_t*) w + 8;
171 const int16x8_t vxb89ABCDEFc0 = vmovl_s8(vb89ABCDEFc0);
172
173 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa0), 0);
174 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa0), 0);
175 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa1), 0);
176 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa1), 0);
177 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa2), 0);
178 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa2), 0);
179 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa3), 0);
180 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa3), 0);
181 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa4), 0);
182 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa4), 0);
183 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa5), 0);
184 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa5), 0);
185 const int8x8_t vb01234567c1 = vld1_s8(w); w = (const int8_t*) w + 8;
186 const int16x8_t vxb01234567c1 = vmovl_s8(vb01234567c1);
187
188 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa0), 1);
189 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa0), 1);
190 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa1), 1);
191 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa1), 1);
192 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa2), 1);
193 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa2), 1);
194 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa3), 1);
195 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa3), 1);
196 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa4), 1);
197 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa4), 1);
198 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa5), 1);
199 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa5), 1);
200 const int8x8_t vb89ABCDEFc1 = vld1_s8(w); w = (const int8_t*) w + 8;
201 const int16x8_t vxb89ABCDEFc1 = vmovl_s8(vb89ABCDEFc1);
202
203 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa0), 1);
204 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa0), 1);
205 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa1), 1);
206 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa1), 1);
207 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa2), 1);
208 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa2), 1);
209 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa3), 1);
210 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa3), 1);
211 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa4), 1);
212 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa4), 1);
213 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa5), 1);
214 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa5), 1);
215 const int8x8_t vb01234567c2 = vld1_s8(w); w = (const int8_t*) w + 8;
216 const int16x8_t vxb01234567c2 = vmovl_s8(vb01234567c2);
217
218 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa0), 2);
219 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa0), 2);
220 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa1), 2);
221 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa1), 2);
222 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa2), 2);
223 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa2), 2);
224 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa3), 2);
225 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa3), 2);
226 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa4), 2);
227 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa4), 2);
228 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa5), 2);
229 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa5), 2);
230 const int8x8_t vb89ABCDEFc2 = vld1_s8(w); w = (const int8_t*) w + 8;
231 const int16x8_t vxb89ABCDEFc2 = vmovl_s8(vb89ABCDEFc2);
232
233 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa0), 2);
234 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa0), 2);
235 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa1), 2);
236 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa1), 2);
237 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa2), 2);
238 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa2), 2);
239 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa3), 2);
240 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa3), 2);
241 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa4), 2);
242 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa4), 2);
243 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa5), 2);
244 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa5), 2);
245 const int8x8_t vb01234567c3 = vld1_s8(w); w = (const int8_t*) w + 8;
246 const int16x8_t vxb01234567c3 = vmovl_s8(vb01234567c3);
247
248 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa0), 3);
249 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa0), 3);
250 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa1), 3);
251 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa1), 3);
252 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa2), 3);
253 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa2), 3);
254 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa3), 3);
255 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa3), 3);
256 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa4), 3);
257 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa4), 3);
258 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa5), 3);
259 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa5), 3);
260 const int8x8_t vb89ABCDEFc3 = vld1_s8(w); w = (const int8_t*) w + 8;
261 const int16x8_t vxb89ABCDEFc3 = vmovl_s8(vb89ABCDEFc3);
262
263 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa0), 3);
264 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa0), 3);
265 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa1), 3);
266 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa1), 3);
267 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa2), 3);
268 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa2), 3);
269 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa3), 3);
270 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa3), 3);
271 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa4), 3);
272 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa4), 3);
273 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa5), 3);
274 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa5), 3);
275
276 xnn_prefetch_to_l1((const int8_t*) w + 448);
277 xnn_prefetch_to_l1((const int8_t*) w + 512);
278
279 const int8x8_t vb01234567c4 = vld1_s8(w); w = (const int8_t*) w + 8;
280 const int16x8_t vxb01234567c4 = vmovl_s8(vb01234567c4);
281
282 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa0), 0);
283 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa0), 0);
284 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa1), 0);
285 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa1), 0);
286 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa2), 0);
287 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa2), 0);
288 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa3), 0);
289 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa3), 0);
290 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa4), 0);
291 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa4), 0);
292 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa5), 0);
293 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa5), 0);
294 const int8x8_t vb89ABCDEFc4 = vld1_s8(w); w = (const int8_t*) w + 8;
295 const int16x8_t vxb89ABCDEFc4 = vmovl_s8(vb89ABCDEFc4);
296
297 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa0), 0);
298 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa0), 0);
299 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa1), 0);
300 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa1), 0);
301 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa2), 0);
302 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa2), 0);
303 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa3), 0);
304 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa3), 0);
305 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa4), 0);
306 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa4), 0);
307 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa5), 0);
308 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa5), 0);
309 const int8x8_t vb01234567c5 = vld1_s8(w); w = (const int8_t*) w + 8;
310 const int16x8_t vxb01234567c5 = vmovl_s8(vb01234567c5);
311
312 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa0), 1);
313 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa0), 1);
314 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa1), 1);
315 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa1), 1);
316 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa2), 1);
317 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa2), 1);
318 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa3), 1);
319 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa3), 1);
320 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa4), 1);
321 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa4), 1);
322 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa5), 1);
323 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa5), 1);
324 const int8x8_t vb89ABCDEFc5 = vld1_s8(w); w = (const int8_t*) w + 8;
325 const int16x8_t vxb89ABCDEFc5 = vmovl_s8(vb89ABCDEFc5);
326
327 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa0), 1);
328 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa0), 1);
329 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa1), 1);
330 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa1), 1);
331 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa2), 1);
332 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa2), 1);
333 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa3), 1);
334 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa3), 1);
335 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa4), 1);
336 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa4), 1);
337 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa5), 1);
338 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa5), 1);
339 const int8x8_t vb01234567c6 = vld1_s8(w); w = (const int8_t*) w + 8;
340 const int16x8_t vxb01234567c6 = vmovl_s8(vb01234567c6);
341
342 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa0), 2);
343 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa0), 2);
344 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa1), 2);
345 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa1), 2);
346 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa2), 2);
347 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa2), 2);
348 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa3), 2);
349 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa3), 2);
350 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa4), 2);
351 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa4), 2);
352 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa5), 2);
353 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa5), 2);
354 const int8x8_t vb89ABCDEFc6 = vld1_s8(w); w = (const int8_t*) w + 8;
355 const int16x8_t vxb89ABCDEFc6 = vmovl_s8(vb89ABCDEFc6);
356
357 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa0), 2);
358 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa0), 2);
359 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa1), 2);
360 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa1), 2);
361 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa2), 2);
362 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa2), 2);
363 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa3), 2);
364 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa3), 2);
365 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa4), 2);
366 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa4), 2);
367 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa5), 2);
368 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa5), 2);
369 const int8x8_t vb01234567c7 = vld1_s8(w); w = (const int8_t*) w + 8;
370 const int16x8_t vxb01234567c7 = vmovl_s8(vb01234567c7);
371
372 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c7), vget_high_s16(vxa0), 3);
373 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c7), vget_high_s16(vxa0), 3);
374 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c7), vget_high_s16(vxa1), 3);
375 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c7), vget_high_s16(vxa1), 3);
376 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c7), vget_high_s16(vxa2), 3);
377 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c7), vget_high_s16(vxa2), 3);
378 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c7), vget_high_s16(vxa3), 3);
379 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c7), vget_high_s16(vxa3), 3);
380 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c7), vget_high_s16(vxa4), 3);
381 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c7), vget_high_s16(vxa4), 3);
382 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c7), vget_high_s16(vxa5), 3);
383 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c7), vget_high_s16(vxa5), 3);
384 const int8x8_t vb89ABCDEFc7 = vld1_s8(w); w = (const int8_t*) w + 8;
385 const int16x8_t vxb89ABCDEFc7 = vmovl_s8(vb89ABCDEFc7);
386
387 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc7), vget_high_s16(vxa0), 3);
388 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc7), vget_high_s16(vxa0), 3);
389 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc7), vget_high_s16(vxa1), 3);
390 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc7), vget_high_s16(vxa1), 3);
391 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc7), vget_high_s16(vxa2), 3);
392 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc7), vget_high_s16(vxa2), 3);
393 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc7), vget_high_s16(vxa3), 3);
394 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc7), vget_high_s16(vxa3), 3);
395 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc7), vget_high_s16(vxa4), 3);
396 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc7), vget_high_s16(vxa4), 3);
397 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc7), vget_high_s16(vxa5), 3);
398 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc7), vget_high_s16(vxa5), 3);
399
400 k -= 8 * sizeof(int8_t);
401 }
402 if XNN_UNLIKELY(k != 0) {
403 const int8x8_t va0 = vld1_s8(a0); a0 = (const int8_t*) ((uintptr_t) a0 + k);
404 const int16x8_t vxa0 = vmovl_s8(va0);
405 const int8x8_t va1 = vld1_s8(a1); a1 = (const int8_t*) ((uintptr_t) a1 + k);
406 const int16x8_t vxa1 = vmovl_s8(va1);
407 const int8x8_t va2 = vld1_s8(a2); a2 = (const int8_t*) ((uintptr_t) a2 + k);
408 const int16x8_t vxa2 = vmovl_s8(va2);
409 const int8x8_t va3 = vld1_s8(a3); a3 = (const int8_t*) ((uintptr_t) a3 + k);
410 const int16x8_t vxa3 = vmovl_s8(va3);
411 const int8x8_t va4 = vld1_s8(a4); a4 = (const int8_t*) ((uintptr_t) a4 + k);
412 const int16x8_t vxa4 = vmovl_s8(va4);
413 const int8x8_t va5 = vld1_s8(a5); a5 = (const int8_t*) ((uintptr_t) a5 + k);
414 const int16x8_t vxa5 = vmovl_s8(va5);
415
416 const int8x8_t vb01234567c0 = vld1_s8(w); w = (const int8_t*) w + 8;
417 const int16x8_t vxb01234567c0 = vmovl_s8(vb01234567c0);
418 const int8x8_t vb89ABCDEFc0 = vld1_s8(w); w = (const int8_t*) w + 8;
419 const int16x8_t vxb89ABCDEFc0 = vmovl_s8(vb89ABCDEFc0);
420
421 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa0), 0);
422 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa0), 0);
423 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa0), 0);
424 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa0), 0);
425 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa1), 0);
426 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa1), 0);
427 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa1), 0);
428 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa1), 0);
429 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa2), 0);
430 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa2), 0);
431 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa2), 0);
432 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa2), 0);
433 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa3), 0);
434 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa3), 0);
435 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa3), 0);
436 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa3), 0);
437 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa4), 0);
438 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa4), 0);
439 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa4), 0);
440 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa4), 0);
441 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c0), vget_low_s16(vxa5), 0);
442 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c0), vget_low_s16(vxa5), 0);
443 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc0), vget_low_s16(vxa5), 0);
444 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc0), vget_low_s16(vxa5), 0);
445
446 if (k >= 2 * sizeof(int8_t)) {
447 const int8x8_t vb01234567c1 = vld1_s8(w); w = (const int8_t*) w + 8;
448 const int16x8_t vxb01234567c1 = vmovl_s8(vb01234567c1);
449 const int8x8_t vb89ABCDEFc1 = vld1_s8(w); w = (const int8_t*) w + 8;
450 const int16x8_t vxb89ABCDEFc1 = vmovl_s8(vb89ABCDEFc1);
451
452 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa0), 1);
453 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa0), 1);
454 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa0), 1);
455 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa0), 1);
456 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa1), 1);
457 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa1), 1);
458 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa1), 1);
459 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa1), 1);
460 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa2), 1);
461 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa2), 1);
462 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa2), 1);
463 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa2), 1);
464 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa3), 1);
465 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa3), 1);
466 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa3), 1);
467 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa3), 1);
468 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa4), 1);
469 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa4), 1);
470 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa4), 1);
471 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa4), 1);
472 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c1), vget_low_s16(vxa5), 1);
473 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c1), vget_low_s16(vxa5), 1);
474 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc1), vget_low_s16(vxa5), 1);
475 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc1), vget_low_s16(vxa5), 1);
476
477 if (k > 2 * sizeof(int8_t)) {
478 const int8x8_t vb01234567c2 = vld1_s8(w); w = (const int8_t*) w + 8;
479 const int16x8_t vxb01234567c2 = vmovl_s8(vb01234567c2);
480 const int8x8_t vb89ABCDEFc2 = vld1_s8(w); w = (const int8_t*) w + 8;
481 const int16x8_t vxb89ABCDEFc2 = vmovl_s8(vb89ABCDEFc2);
482
483 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa0), 2);
484 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa0), 2);
485 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa0), 2);
486 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa0), 2);
487 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa1), 2);
488 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa1), 2);
489 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa1), 2);
490 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa1), 2);
491 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa2), 2);
492 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa2), 2);
493 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa2), 2);
494 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa2), 2);
495 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa3), 2);
496 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa3), 2);
497 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa3), 2);
498 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa3), 2);
499 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa4), 2);
500 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa4), 2);
501 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa4), 2);
502 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa4), 2);
503 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c2), vget_low_s16(vxa5), 2);
504 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c2), vget_low_s16(vxa5), 2);
505 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc2), vget_low_s16(vxa5), 2);
506 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc2), vget_low_s16(vxa5), 2);
507
508 if (k >= 4 * sizeof(int8_t)) {
509 const int8x8_t vb01234567c3 = vld1_s8(w); w = (const int8_t*) w + 8;
510 const int16x8_t vxb01234567c3 = vmovl_s8(vb01234567c3);
511 const int8x8_t vb89ABCDEFc3 = vld1_s8(w); w = (const int8_t*) w + 8;
512 const int16x8_t vxb89ABCDEFc3 = vmovl_s8(vb89ABCDEFc3);
513
514 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa0), 3);
515 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa0), 3);
516 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa0), 3);
517 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa0), 3);
518 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa1), 3);
519 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa1), 3);
520 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa1), 3);
521 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa1), 3);
522 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa2), 3);
523 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa2), 3);
524 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa2), 3);
525 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa2), 3);
526 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa3), 3);
527 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa3), 3);
528 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa3), 3);
529 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa3), 3);
530 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa4), 3);
531 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa4), 3);
532 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa4), 3);
533 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa4), 3);
534 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c3), vget_low_s16(vxa5), 3);
535 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c3), vget_low_s16(vxa5), 3);
536 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc3), vget_low_s16(vxa5), 3);
537 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc3), vget_low_s16(vxa5), 3);
538
539 if (k > 4 * sizeof(int8_t)) {
540 const int8x8_t vb01234567c4 = vld1_s8(w); w = (const int8_t*) w + 8;
541 const int16x8_t vxb01234567c4 = vmovl_s8(vb01234567c4);
542 const int8x8_t vb89ABCDEFc4 = vld1_s8(w); w = (const int8_t*) w + 8;
543 const int16x8_t vxb89ABCDEFc4 = vmovl_s8(vb89ABCDEFc4);
544
545 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa0), 0);
546 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa0), 0);
547 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa0), 0);
548 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa0), 0);
549 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa1), 0);
550 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa1), 0);
551 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa1), 0);
552 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa1), 0);
553 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa2), 0);
554 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa2), 0);
555 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa2), 0);
556 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa2), 0);
557 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa3), 0);
558 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa3), 0);
559 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa3), 0);
560 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa3), 0);
561 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa4), 0);
562 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa4), 0);
563 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa4), 0);
564 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa4), 0);
565 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c4), vget_high_s16(vxa5), 0);
566 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c4), vget_high_s16(vxa5), 0);
567 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc4), vget_high_s16(vxa5), 0);
568 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc4), vget_high_s16(vxa5), 0);
569
570 if (k >= 6 * sizeof(int8_t)) {
571 const int8x8_t vb01234567c5 = vld1_s8(w); w = (const int8_t*) w + 8;
572 const int16x8_t vxb01234567c5 = vmovl_s8(vb01234567c5);
573 const int8x8_t vb89ABCDEFc5 = vld1_s8(w); w = (const int8_t*) w + 8;
574 const int16x8_t vxb89ABCDEFc5 = vmovl_s8(vb89ABCDEFc5);
575
576 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa0), 1);
577 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa0), 1);
578 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa0), 1);
579 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa0), 1);
580 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa1), 1);
581 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa1), 1);
582 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa1), 1);
583 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa1), 1);
584 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa2), 1);
585 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa2), 1);
586 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa2), 1);
587 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa2), 1);
588 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa3), 1);
589 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa3), 1);
590 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa3), 1);
591 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa3), 1);
592 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa4), 1);
593 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa4), 1);
594 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa4), 1);
595 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa4), 1);
596 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c5), vget_high_s16(vxa5), 1);
597 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c5), vget_high_s16(vxa5), 1);
598 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc5), vget_high_s16(vxa5), 1);
599 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc5), vget_high_s16(vxa5), 1);
600
601 if (k > 6 * sizeof(int8_t)) {
602 const int8x8_t vb01234567c6 = vld1_s8(w); w = (const int8_t*) w + 8;
603 const int16x8_t vxb01234567c6 = vmovl_s8(vb01234567c6);
604 const int8x8_t vb89ABCDEFc6 = vld1_s8(w); w = (const int8_t*) w + 8;
605 const int16x8_t vxb89ABCDEFc6 = vmovl_s8(vb89ABCDEFc6);
606
607 vacc0x0123 = vmlal_lane_s16(vacc0x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa0), 2);
608 vacc0x4567 = vmlal_lane_s16(vacc0x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa0), 2);
609 vacc0x89AB = vmlal_lane_s16(vacc0x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa0), 2);
610 vacc0xCDEF = vmlal_lane_s16(vacc0xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa0), 2);
611 vacc1x0123 = vmlal_lane_s16(vacc1x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa1), 2);
612 vacc1x4567 = vmlal_lane_s16(vacc1x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa1), 2);
613 vacc1x89AB = vmlal_lane_s16(vacc1x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa1), 2);
614 vacc1xCDEF = vmlal_lane_s16(vacc1xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa1), 2);
615 vacc2x0123 = vmlal_lane_s16(vacc2x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa2), 2);
616 vacc2x4567 = vmlal_lane_s16(vacc2x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa2), 2);
617 vacc2x89AB = vmlal_lane_s16(vacc2x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa2), 2);
618 vacc2xCDEF = vmlal_lane_s16(vacc2xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa2), 2);
619 vacc3x0123 = vmlal_lane_s16(vacc3x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa3), 2);
620 vacc3x4567 = vmlal_lane_s16(vacc3x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa3), 2);
621 vacc3x89AB = vmlal_lane_s16(vacc3x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa3), 2);
622 vacc3xCDEF = vmlal_lane_s16(vacc3xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa3), 2);
623 vacc4x0123 = vmlal_lane_s16(vacc4x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa4), 2);
624 vacc4x4567 = vmlal_lane_s16(vacc4x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa4), 2);
625 vacc4x89AB = vmlal_lane_s16(vacc4x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa4), 2);
626 vacc4xCDEF = vmlal_lane_s16(vacc4xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa4), 2);
627 vacc5x0123 = vmlal_lane_s16(vacc5x0123, vget_low_s16(vxb01234567c6), vget_high_s16(vxa5), 2);
628 vacc5x4567 = vmlal_lane_s16(vacc5x4567, vget_high_s16(vxb01234567c6), vget_high_s16(vxa5), 2);
629 vacc5x89AB = vmlal_lane_s16(vacc5x89AB, vget_low_s16(vxb89ABCDEFc6), vget_high_s16(vxa5), 2);
630 vacc5xCDEF = vmlal_lane_s16(vacc5xCDEF, vget_high_s16(vxb89ABCDEFc6), vget_high_s16(vxa5), 2);
631 }
632 }
633 }
634 }
635 }
636 }
637 }
638 p -= 6 * sizeof(void*);
639 } while (p != 0);
640
641 float32x4_t vout0x0123 = vcvtq_f32_s32(vacc0x0123);
642 float32x4_t vout0x4567 = vcvtq_f32_s32(vacc0x4567);
643 float32x4_t vout0x89AB = vcvtq_f32_s32(vacc0x89AB);
644 float32x4_t vout0xCDEF = vcvtq_f32_s32(vacc0xCDEF);
645 float32x4_t vout1x0123 = vcvtq_f32_s32(vacc1x0123);
646 float32x4_t vout1x4567 = vcvtq_f32_s32(vacc1x4567);
647 float32x4_t vout1x89AB = vcvtq_f32_s32(vacc1x89AB);
648 float32x4_t vout1xCDEF = vcvtq_f32_s32(vacc1xCDEF);
649 float32x4_t vout2x0123 = vcvtq_f32_s32(vacc2x0123);
650 float32x4_t vout2x4567 = vcvtq_f32_s32(vacc2x4567);
651 float32x4_t vout2x89AB = vcvtq_f32_s32(vacc2x89AB);
652 float32x4_t vout2xCDEF = vcvtq_f32_s32(vacc2xCDEF);
653 float32x4_t vout3x0123 = vcvtq_f32_s32(vacc3x0123);
654 float32x4_t vout3x4567 = vcvtq_f32_s32(vacc3x4567);
655 float32x4_t vout3x89AB = vcvtq_f32_s32(vacc3x89AB);
656 float32x4_t vout3xCDEF = vcvtq_f32_s32(vacc3xCDEF);
657 float32x4_t vout4x0123 = vcvtq_f32_s32(vacc4x0123);
658 float32x4_t vout4x4567 = vcvtq_f32_s32(vacc4x4567);
659 float32x4_t vout4x89AB = vcvtq_f32_s32(vacc4x89AB);
660 float32x4_t vout4xCDEF = vcvtq_f32_s32(vacc4xCDEF);
661 float32x4_t vout5x0123 = vcvtq_f32_s32(vacc5x0123);
662 float32x4_t vout5x4567 = vcvtq_f32_s32(vacc5x4567);
663 float32x4_t vout5x89AB = vcvtq_f32_s32(vacc5x89AB);
664 float32x4_t vout5xCDEF = vcvtq_f32_s32(vacc5xCDEF);
665
666 const float32x4_t vinput_scale = vld1q_dup_f32(&quantization_params->inv_scale);
667 vout0x0123 = vmulq_f32(vout0x0123, vinput_scale);
668 vout0x4567 = vmulq_f32(vout0x4567, vinput_scale);
669 vout0x89AB = vmulq_f32(vout0x89AB, vinput_scale);
670 vout0xCDEF = vmulq_f32(vout0xCDEF, vinput_scale);
671 vout1x0123 = vmulq_f32(vout1x0123, vinput_scale);
672 vout1x4567 = vmulq_f32(vout1x4567, vinput_scale);
673 vout1x89AB = vmulq_f32(vout1x89AB, vinput_scale);
674 vout1xCDEF = vmulq_f32(vout1xCDEF, vinput_scale);
675 vout2x0123 = vmulq_f32(vout2x0123, vinput_scale);
676 vout2x4567 = vmulq_f32(vout2x4567, vinput_scale);
677 vout2x89AB = vmulq_f32(vout2x89AB, vinput_scale);
678 vout2xCDEF = vmulq_f32(vout2xCDEF, vinput_scale);
679 vout3x0123 = vmulq_f32(vout3x0123, vinput_scale);
680 vout3x4567 = vmulq_f32(vout3x4567, vinput_scale);
681 vout3x89AB = vmulq_f32(vout3x89AB, vinput_scale);
682 vout3xCDEF = vmulq_f32(vout3xCDEF, vinput_scale);
683 vout4x0123 = vmulq_f32(vout4x0123, vinput_scale);
684 vout4x4567 = vmulq_f32(vout4x4567, vinput_scale);
685 vout4x89AB = vmulq_f32(vout4x89AB, vinput_scale);
686 vout4xCDEF = vmulq_f32(vout4xCDEF, vinput_scale);
687 vout5x0123 = vmulq_f32(vout5x0123, vinput_scale);
688 vout5x4567 = vmulq_f32(vout5x4567, vinput_scale);
689 vout5x89AB = vmulq_f32(vout5x89AB, vinput_scale);
690 vout5xCDEF = vmulq_f32(vout5xCDEF, vinput_scale);
691
692 const float32x4_t vfilter_output_scale0123 = vld1q_f32(w); w = (const float*) w + 4;
693 const float32x4_t vfilter_output_scale4567 = vld1q_f32(w); w = (const float*) w + 4;
694 const float32x4_t vfilter_output_scale89AB = vld1q_f32(w); w = (const float*) w + 4;
695 const float32x4_t vfilter_output_scaleCDEF = vld1q_f32(w); w = (const float*) w + 4;
696
697 const float32x4_t vbias0123 = vld1q_f32(w); w = (const float*) w + 4;
698 #if XNN_ARCH_ARM64
699 vout0x0123 = vfmaq_f32(vbias0123, vout0x0123, vfilter_output_scale0123);
700 vout1x0123 = vfmaq_f32(vbias0123, vout1x0123, vfilter_output_scale0123);
701 vout2x0123 = vfmaq_f32(vbias0123, vout2x0123, vfilter_output_scale0123);
702 vout3x0123 = vfmaq_f32(vbias0123, vout3x0123, vfilter_output_scale0123);
703 vout4x0123 = vfmaq_f32(vbias0123, vout4x0123, vfilter_output_scale0123);
704 vout5x0123 = vfmaq_f32(vbias0123, vout5x0123, vfilter_output_scale0123);
705 #else
706 vout0x0123 = vmlaq_f32(vbias0123, vout0x0123, vfilter_output_scale0123);
707 vout1x0123 = vmlaq_f32(vbias0123, vout1x0123, vfilter_output_scale0123);
708 vout2x0123 = vmlaq_f32(vbias0123, vout2x0123, vfilter_output_scale0123);
709 vout3x0123 = vmlaq_f32(vbias0123, vout3x0123, vfilter_output_scale0123);
710 vout4x0123 = vmlaq_f32(vbias0123, vout4x0123, vfilter_output_scale0123);
711 vout5x0123 = vmlaq_f32(vbias0123, vout5x0123, vfilter_output_scale0123);
712 #endif
713 const float32x4_t vbias4567 = vld1q_f32(w); w = (const float*) w + 4;
714 #if XNN_ARCH_ARM64
715 vout0x4567 = vfmaq_f32(vbias4567, vout0x4567, vfilter_output_scale4567);
716 vout1x4567 = vfmaq_f32(vbias4567, vout1x4567, vfilter_output_scale4567);
717 vout2x4567 = vfmaq_f32(vbias4567, vout2x4567, vfilter_output_scale4567);
718 vout3x4567 = vfmaq_f32(vbias4567, vout3x4567, vfilter_output_scale4567);
719 vout4x4567 = vfmaq_f32(vbias4567, vout4x4567, vfilter_output_scale4567);
720 vout5x4567 = vfmaq_f32(vbias4567, vout5x4567, vfilter_output_scale4567);
721 #else
722 vout0x4567 = vmlaq_f32(vbias4567, vout0x4567, vfilter_output_scale4567);
723 vout1x4567 = vmlaq_f32(vbias4567, vout1x4567, vfilter_output_scale4567);
724 vout2x4567 = vmlaq_f32(vbias4567, vout2x4567, vfilter_output_scale4567);
725 vout3x4567 = vmlaq_f32(vbias4567, vout3x4567, vfilter_output_scale4567);
726 vout4x4567 = vmlaq_f32(vbias4567, vout4x4567, vfilter_output_scale4567);
727 vout5x4567 = vmlaq_f32(vbias4567, vout5x4567, vfilter_output_scale4567);
728 #endif
729 const float32x4_t vbias89AB = vld1q_f32(w); w = (const float*) w + 4;
730 #if XNN_ARCH_ARM64
731 vout0x89AB = vfmaq_f32(vbias89AB, vout0x89AB, vfilter_output_scale89AB);
732 vout1x89AB = vfmaq_f32(vbias89AB, vout1x89AB, vfilter_output_scale89AB);
733 vout2x89AB = vfmaq_f32(vbias89AB, vout2x89AB, vfilter_output_scale89AB);
734 vout3x89AB = vfmaq_f32(vbias89AB, vout3x89AB, vfilter_output_scale89AB);
735 vout4x89AB = vfmaq_f32(vbias89AB, vout4x89AB, vfilter_output_scale89AB);
736 vout5x89AB = vfmaq_f32(vbias89AB, vout5x89AB, vfilter_output_scale89AB);
737 #else
738 vout0x89AB = vmlaq_f32(vbias89AB, vout0x89AB, vfilter_output_scale89AB);
739 vout1x89AB = vmlaq_f32(vbias89AB, vout1x89AB, vfilter_output_scale89AB);
740 vout2x89AB = vmlaq_f32(vbias89AB, vout2x89AB, vfilter_output_scale89AB);
741 vout3x89AB = vmlaq_f32(vbias89AB, vout3x89AB, vfilter_output_scale89AB);
742 vout4x89AB = vmlaq_f32(vbias89AB, vout4x89AB, vfilter_output_scale89AB);
743 vout5x89AB = vmlaq_f32(vbias89AB, vout5x89AB, vfilter_output_scale89AB);
744 #endif
745 const float32x4_t vbiasCDEF = vld1q_f32(w); w = (const float*) w + 4;
746 #if XNN_ARCH_ARM64
747 vout0xCDEF = vfmaq_f32(vbiasCDEF, vout0xCDEF, vfilter_output_scaleCDEF);
748 vout1xCDEF = vfmaq_f32(vbiasCDEF, vout1xCDEF, vfilter_output_scaleCDEF);
749 vout2xCDEF = vfmaq_f32(vbiasCDEF, vout2xCDEF, vfilter_output_scaleCDEF);
750 vout3xCDEF = vfmaq_f32(vbiasCDEF, vout3xCDEF, vfilter_output_scaleCDEF);
751 vout4xCDEF = vfmaq_f32(vbiasCDEF, vout4xCDEF, vfilter_output_scaleCDEF);
752 vout5xCDEF = vfmaq_f32(vbiasCDEF, vout5xCDEF, vfilter_output_scaleCDEF);
753 #else
754 vout0xCDEF = vmlaq_f32(vbiasCDEF, vout0xCDEF, vfilter_output_scaleCDEF);
755 vout1xCDEF = vmlaq_f32(vbiasCDEF, vout1xCDEF, vfilter_output_scaleCDEF);
756 vout2xCDEF = vmlaq_f32(vbiasCDEF, vout2xCDEF, vfilter_output_scaleCDEF);
757 vout3xCDEF = vmlaq_f32(vbiasCDEF, vout3xCDEF, vfilter_output_scaleCDEF);
758 vout4xCDEF = vmlaq_f32(vbiasCDEF, vout4xCDEF, vfilter_output_scaleCDEF);
759 vout5xCDEF = vmlaq_f32(vbiasCDEF, vout5xCDEF, vfilter_output_scaleCDEF);
760 #endif
761
762 const float32x4_t voutput_min = vld1q_dup_f32(&params->scalar.min);
763 vout0x0123 = vmaxq_f32(vout0x0123, voutput_min);
764 vout0x4567 = vmaxq_f32(vout0x4567, voutput_min);
765 vout0x89AB = vmaxq_f32(vout0x89AB, voutput_min);
766 vout0xCDEF = vmaxq_f32(vout0xCDEF, voutput_min);
767 vout1x0123 = vmaxq_f32(vout1x0123, voutput_min);
768 vout1x4567 = vmaxq_f32(vout1x4567, voutput_min);
769 vout1x89AB = vmaxq_f32(vout1x89AB, voutput_min);
770 vout1xCDEF = vmaxq_f32(vout1xCDEF, voutput_min);
771 vout2x0123 = vmaxq_f32(vout2x0123, voutput_min);
772 vout2x4567 = vmaxq_f32(vout2x4567, voutput_min);
773 vout2x89AB = vmaxq_f32(vout2x89AB, voutput_min);
774 vout2xCDEF = vmaxq_f32(vout2xCDEF, voutput_min);
775 vout3x0123 = vmaxq_f32(vout3x0123, voutput_min);
776 vout3x4567 = vmaxq_f32(vout3x4567, voutput_min);
777 vout3x89AB = vmaxq_f32(vout3x89AB, voutput_min);
778 vout3xCDEF = vmaxq_f32(vout3xCDEF, voutput_min);
779 vout4x0123 = vmaxq_f32(vout4x0123, voutput_min);
780 vout4x4567 = vmaxq_f32(vout4x4567, voutput_min);
781 vout4x89AB = vmaxq_f32(vout4x89AB, voutput_min);
782 vout4xCDEF = vmaxq_f32(vout4xCDEF, voutput_min);
783 vout5x0123 = vmaxq_f32(vout5x0123, voutput_min);
784 vout5x4567 = vmaxq_f32(vout5x4567, voutput_min);
785 vout5x89AB = vmaxq_f32(vout5x89AB, voutput_min);
786 vout5xCDEF = vmaxq_f32(vout5xCDEF, voutput_min);
787
788 const float32x4_t voutput_max = vld1q_dup_f32(&params->scalar.max);
789 vout0x0123 = vminq_f32(vout0x0123, voutput_max);
790 vout0x4567 = vminq_f32(vout0x4567, voutput_max);
791 vout0x89AB = vminq_f32(vout0x89AB, voutput_max);
792 vout0xCDEF = vminq_f32(vout0xCDEF, voutput_max);
793 vout1x0123 = vminq_f32(vout1x0123, voutput_max);
794 vout1x4567 = vminq_f32(vout1x4567, voutput_max);
795 vout1x89AB = vminq_f32(vout1x89AB, voutput_max);
796 vout1xCDEF = vminq_f32(vout1xCDEF, voutput_max);
797 vout2x0123 = vminq_f32(vout2x0123, voutput_max);
798 vout2x4567 = vminq_f32(vout2x4567, voutput_max);
799 vout2x89AB = vminq_f32(vout2x89AB, voutput_max);
800 vout2xCDEF = vminq_f32(vout2xCDEF, voutput_max);
801 vout3x0123 = vminq_f32(vout3x0123, voutput_max);
802 vout3x4567 = vminq_f32(vout3x4567, voutput_max);
803 vout3x89AB = vminq_f32(vout3x89AB, voutput_max);
804 vout3xCDEF = vminq_f32(vout3xCDEF, voutput_max);
805 vout4x0123 = vminq_f32(vout4x0123, voutput_max);
806 vout4x4567 = vminq_f32(vout4x4567, voutput_max);
807 vout4x89AB = vminq_f32(vout4x89AB, voutput_max);
808 vout4xCDEF = vminq_f32(vout4xCDEF, voutput_max);
809 vout5x0123 = vminq_f32(vout5x0123, voutput_max);
810 vout5x4567 = vminq_f32(vout5x4567, voutput_max);
811 vout5x89AB = vminq_f32(vout5x89AB, voutput_max);
812 vout5xCDEF = vminq_f32(vout5xCDEF, voutput_max);
813
814 if XNN_LIKELY(nc >= 16) {
815 vst1q_f32(&c5[0], vout5x0123);
816 vst1q_f32(&c5[4], vout5x4567);
817 vst1q_f32(&c5[8], vout5x89AB);
818 vst1q_f32(&c5[12], vout5xCDEF);
819 vst1q_f32(&c4[0], vout4x0123);
820 vst1q_f32(&c4[4], vout4x4567);
821 vst1q_f32(&c4[8], vout4x89AB);
822 vst1q_f32(&c4[12], vout4xCDEF);
823 vst1q_f32(&c3[0], vout3x0123);
824 vst1q_f32(&c3[4], vout3x4567);
825 vst1q_f32(&c3[8], vout3x89AB);
826 vst1q_f32(&c3[12], vout3xCDEF);
827 vst1q_f32(&c2[0], vout2x0123);
828 vst1q_f32(&c2[4], vout2x4567);
829 vst1q_f32(&c2[8], vout2x89AB);
830 vst1q_f32(&c2[12], vout2xCDEF);
831 vst1q_f32(&c1[0], vout1x0123);
832 vst1q_f32(&c1[4], vout1x4567);
833 vst1q_f32(&c1[8], vout1x89AB);
834 vst1q_f32(&c1[12], vout1xCDEF);
835 vst1q_f32(&c0[0], vout0x0123);
836 vst1q_f32(&c0[4], vout0x4567);
837 vst1q_f32(&c0[8], vout0x89AB);
838 vst1q_f32(&c0[12], vout0xCDEF);
839
840 a = (const int8_t**restrict) ((uintptr_t) a - ks);
841
842 c0 = (float*) ((uintptr_t) c0 + cn_stride);
843 c1 = (float*) ((uintptr_t) c1 + cn_stride);
844 c2 = (float*) ((uintptr_t) c2 + cn_stride);
845 c3 = (float*) ((uintptr_t) c3 + cn_stride);
846 c4 = (float*) ((uintptr_t) c4 + cn_stride);
847 c5 = (float*) ((uintptr_t) c5 + cn_stride);
848
849 nc -= 16;
850 } else {
851 if (nc & 8) {
852 vst1q_f32(c5, vout5x0123); c5 += 4;
853 vout5x0123 = vout5x89AB;
854 vst1q_f32(c4, vout4x0123); c4 += 4;
855 vout4x0123 = vout4x89AB;
856 vst1q_f32(c3, vout3x0123); c3 += 4;
857 vout3x0123 = vout3x89AB;
858 vst1q_f32(c2, vout2x0123); c2 += 4;
859 vout2x0123 = vout2x89AB;
860 vst1q_f32(c1, vout1x0123); c1 += 4;
861 vout1x0123 = vout1x89AB;
862 vst1q_f32(c0, vout0x0123); c0 += 4;
863 vout0x0123 = vout0x89AB;
864 vst1q_f32(c5, vout5x4567); c5 += 4;
865 vout5x4567 = vout5xCDEF;
866 vst1q_f32(c4, vout4x4567); c4 += 4;
867 vout4x4567 = vout4xCDEF;
868 vst1q_f32(c3, vout3x4567); c3 += 4;
869 vout3x4567 = vout3xCDEF;
870 vst1q_f32(c2, vout2x4567); c2 += 4;
871 vout2x4567 = vout2xCDEF;
872 vst1q_f32(c1, vout1x4567); c1 += 4;
873 vout1x4567 = vout1xCDEF;
874 vst1q_f32(c0, vout0x4567); c0 += 4;
875 vout0x4567 = vout0xCDEF;
876 }
877 if (nc & 4) {
878 vst1q_f32(c5, vout5x0123); c5 += 4;
879 vout5x0123 = vout5x4567;
880 vst1q_f32(c4, vout4x0123); c4 += 4;
881 vout4x0123 = vout4x4567;
882 vst1q_f32(c3, vout3x0123); c3 += 4;
883 vout3x0123 = vout3x4567;
884 vst1q_f32(c2, vout2x0123); c2 += 4;
885 vout2x0123 = vout2x4567;
886 vst1q_f32(c1, vout1x0123); c1 += 4;
887 vout1x0123 = vout1x4567;
888 vst1q_f32(c0, vout0x0123); c0 += 4;
889 vout0x0123 = vout0x4567;
890 }
891 float32x2_t vout5x01 = vget_low_f32(vout5x0123);
892 float32x2_t vout4x01 = vget_low_f32(vout4x0123);
893 float32x2_t vout3x01 = vget_low_f32(vout3x0123);
894 float32x2_t vout2x01 = vget_low_f32(vout2x0123);
895 float32x2_t vout1x01 = vget_low_f32(vout1x0123);
896 float32x2_t vout0x01 = vget_low_f32(vout0x0123);
897 if (nc & 2) {
898 vst1_f32(c5, vout5x01); c5 += 2;
899 vst1_f32(c4, vout4x01); c4 += 2;
900 vst1_f32(c3, vout3x01); c3 += 2;
901 vst1_f32(c2, vout2x01); c2 += 2;
902 vst1_f32(c1, vout1x01); c1 += 2;
903 vst1_f32(c0, vout0x01); c0 += 2;
904 vout5x01 = vget_high_f32(vout5x0123);
905 vout4x01 = vget_high_f32(vout4x0123);
906 vout3x01 = vget_high_f32(vout3x0123);
907 vout2x01 = vget_high_f32(vout2x0123);
908 vout1x01 = vget_high_f32(vout1x0123);
909 vout0x01 = vget_high_f32(vout0x0123);
910 }
911 if (nc & 1) {
912 vst1_lane_f32(c5, vout5x01, 0);
913 vst1_lane_f32(c4, vout4x01, 0);
914 vst1_lane_f32(c3, vout3x01, 0);
915 vst1_lane_f32(c2, vout2x01, 0);
916 vst1_lane_f32(c1, vout1x01, 0);
917 vst1_lane_f32(c0, vout0x01, 0);
918 }
919 nc = 0;
920 }
921 } while (nc != 0);
922 }