Debian Codegraph @

src/x32-packw/gen/x32-packw-x16s4-gemm-goi-sse2-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x16s4__sse2_u8_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 16);
39 assert(kr == 1);
40 assert(sr == 4);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 const float* b = (const float*) bias;
45 float* packed_w = (float*) packed_weights;
46 do {
47 // NC main loop multiple of 16
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50
51 for (; n >= 16; n -= 16) {
52 if XNN_LIKELY(b != NULL) {
53 const __m128 vb0 = _mm_loadu_ps(b);
54 const __m128 vb4 = _mm_loadu_ps(b + 4);
55 const __m128 vb8 = _mm_loadu_ps(b + 8);
56 const __m128 vb12 = _mm_loadu_ps(b + 12);
57 _mm_store_ps(packed_w, vb0);
58 _mm_store_ps(packed_w + 4, vb4);
59 _mm_store_ps(packed_w + 8, vb8);
60 _mm_store_ps(packed_w + 12, vb12);
61 b += 16;
62 } else {
63 const __m128 vzero = _mm_setzero_ps();
64 _mm_store_ps(packed_w, vzero);
65 _mm_store_ps(packed_w + 4, vzero);
66 _mm_store_ps(packed_w + 8, vzero);
67 _mm_store_ps(packed_w + 12, vzero);
68 }
69 packed_w += 16;
70
71 const float* w1 = w0 + kc;
72 const float* w2 = w1 + kc;
73 const float* w3 = w2 + kc;
74 const float* w4 = w3 + kc;
75 const float* w5 = w4 + kc;
76 const float* w6 = w5 + kc;
77 const float* w7 = w6 + kc;
78 const float* w8 = w7 + kc;
79 const float* w9 = w8 + kc;
80 const float* w10 = w9 + kc;
81 const float* w11 = w10 + kc;
82 const float* w12 = w11 + kc;
83 const float* w13 = w12 + kc;
84 const float* w14 = w13 + kc;
85 const float* w15 = w14 + kc;
86 xnn_prefetch_to_l1((const int8_t*) w0);
87 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
88 xnn_prefetch_to_l1((const int8_t*) w1);
89 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
90 xnn_prefetch_to_l1((const int8_t*) w2);
91 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
92 xnn_prefetch_to_l1((const int8_t*) w3);
93 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
94 xnn_prefetch_to_l1((const int8_t*) w4);
95 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
96 xnn_prefetch_to_l1((const int8_t*) w5);
97 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
98 xnn_prefetch_to_l1((const int8_t*) w6);
99 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
100 xnn_prefetch_to_l1((const int8_t*) w7);
101 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
102 xnn_prefetch_to_l1((const int8_t*) w8);
103 xnn_prefetch_to_l1((const int8_t*) w8 + 64);
104 xnn_prefetch_to_l1((const int8_t*) w9);
105 xnn_prefetch_to_l1((const int8_t*) w9 + 64);
106 xnn_prefetch_to_l1((const int8_t*) w10);
107 xnn_prefetch_to_l1((const int8_t*) w10 + 64);
108 xnn_prefetch_to_l1((const int8_t*) w11);
109 xnn_prefetch_to_l1((const int8_t*) w11 + 64);
110 xnn_prefetch_to_l1((const int8_t*) w12);
111 xnn_prefetch_to_l1((const int8_t*) w12 + 64);
112 xnn_prefetch_to_l1((const int8_t*) w13);
113 xnn_prefetch_to_l1((const int8_t*) w13 + 64);
114 xnn_prefetch_to_l1((const int8_t*) w14);
115 xnn_prefetch_to_l1((const int8_t*) w14 + 64);
116 xnn_prefetch_to_l1((const int8_t*) w15);
117 xnn_prefetch_to_l1((const int8_t*) w15 + 64);
118
119 size_t k = kc;
120 // KC main loop multiple of 8
121 for (; k >= 8; k -= 8) {
122 // Read blocks of 4x4
123 // a b c d
124 // e f g h
125 // i j k l
126 // m n o p
127 __m128 v0x0123 = _mm_loadu_ps(w0);
128 __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
129 w0 += 8;
130 __m128 v1x0123 = _mm_loadu_ps(w1);
131 __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
132 w1 += 8;
133 __m128 v2x0123 = _mm_loadu_ps(w2);
134 __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
135 w2 += 8;
136 __m128 v3x0123 = _mm_loadu_ps(w3);
137 __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
138 w3 += 8;
139 __m128 v4x0123 = _mm_loadu_ps(w4);
140 __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
141 w4 += 8;
142 __m128 v5x0123 = _mm_loadu_ps(w5);
143 __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
144 w5 += 8;
145 __m128 v6x0123 = _mm_loadu_ps(w6);
146 __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
147 w6 += 8;
148 __m128 v7x0123 = _mm_loadu_ps(w7);
149 __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
150 w7 += 8;
151 __m128 v8x0123 = _mm_loadu_ps(w8);
152 __m128 v8x4567 = _mm_loadu_ps(w8 + 4);
153 w8 += 8;
154 __m128 v9x0123 = _mm_loadu_ps(w9);
155 __m128 v9x4567 = _mm_loadu_ps(w9 + 4);
156 w9 += 8;
157 __m128 v10x0123 = _mm_loadu_ps(w10);
158 __m128 v10x4567 = _mm_loadu_ps(w10 + 4);
159 w10 += 8;
160 __m128 v11x0123 = _mm_loadu_ps(w11);
161 __m128 v11x4567 = _mm_loadu_ps(w11 + 4);
162 w11 += 8;
163 __m128 v12x0123 = _mm_loadu_ps(w12);
164 __m128 v12x4567 = _mm_loadu_ps(w12 + 4);
165 w12 += 8;
166 __m128 v13x0123 = _mm_loadu_ps(w13);
167 __m128 v13x4567 = _mm_loadu_ps(w13 + 4);
168 w13 += 8;
169 __m128 v14x0123 = _mm_loadu_ps(w14);
170 __m128 v14x4567 = _mm_loadu_ps(w14 + 4);
171 w14 += 8;
172 __m128 v15x0123 = _mm_loadu_ps(w15);
173 __m128 v15x4567 = _mm_loadu_ps(w15 + 4);
174 w15 += 8;
175
176 // Apply SR4 shuffle
177 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
178 v1x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x4567), _MM_SHUFFLE(0, 3, 2, 1)));
179 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
180 v2x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x4567), _MM_SHUFFLE(1, 0, 3, 2)));
181 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
182 v3x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x4567), _MM_SHUFFLE(2, 1, 0, 3)));
183 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
184 v5x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x4567), _MM_SHUFFLE(0, 3, 2, 1)));
185 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
186 v6x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x4567), _MM_SHUFFLE(1, 0, 3, 2)));
187 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
188 v7x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x4567), _MM_SHUFFLE(2, 1, 0, 3)));
189 v9x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x0123), _MM_SHUFFLE(0, 3, 2, 1)));
190 v9x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x4567), _MM_SHUFFLE(0, 3, 2, 1)));
191 v10x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x0123), _MM_SHUFFLE(1, 0, 3, 2)));
192 v10x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x4567), _MM_SHUFFLE(1, 0, 3, 2)));
193 v11x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x0123), _MM_SHUFFLE(2, 1, 0, 3)));
194 v11x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x4567), _MM_SHUFFLE(2, 1, 0, 3)));
195 v13x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x0123), _MM_SHUFFLE(0, 3, 2, 1)));
196 v13x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x4567), _MM_SHUFFLE(0, 3, 2, 1)));
197 v14x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x0123), _MM_SHUFFLE(1, 0, 3, 2)));
198 v14x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x4567), _MM_SHUFFLE(1, 0, 3, 2)));
199 v15x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15x0123), _MM_SHUFFLE(2, 1, 0, 3)));
200 v15x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15x4567), _MM_SHUFFLE(2, 1, 0, 3)));
201 // Transpose 2x2
202 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
203 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
204 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
205 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
206 const __m128 vtmp0x4567 = _mm_unpacklo_ps(v0x4567, v1x4567); // a e b f from row 0, 1
207 const __m128 vtmp1x4567 = _mm_unpacklo_ps(v2x4567, v3x4567); // i m j n from row 2, 3
208 const __m128 vtmp2x4567 = _mm_unpackhi_ps(v0x4567, v1x4567); // c g d h from row 0, 1
209 const __m128 vtmp3x4567 = _mm_unpackhi_ps(v2x4567, v3x4567); // k o l p from row 2, 3
210 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
211 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
212 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
213 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
214 const __m128 vtmp4x4567 = _mm_unpacklo_ps(v4x4567, v5x4567); // a e b f from row 0, 1
215 const __m128 vtmp5x4567 = _mm_unpacklo_ps(v6x4567, v7x4567); // i m j n from row 2, 3
216 const __m128 vtmp6x4567 = _mm_unpackhi_ps(v4x4567, v5x4567); // c g d h from row 0, 1
217 const __m128 vtmp7x4567 = _mm_unpackhi_ps(v6x4567, v7x4567); // k o l p from row 2, 3
218 const __m128 vtmp8x0123 = _mm_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
219 const __m128 vtmp9x0123 = _mm_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
220 const __m128 vtmp10x0123 = _mm_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
221 const __m128 vtmp11x0123 = _mm_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
222 const __m128 vtmp8x4567 = _mm_unpacklo_ps(v8x4567, v9x4567); // a e b f from row 0, 1
223 const __m128 vtmp9x4567 = _mm_unpacklo_ps(v10x4567, v11x4567); // i m j n from row 2, 3
224 const __m128 vtmp10x4567 = _mm_unpackhi_ps(v8x4567, v9x4567); // c g d h from row 0, 1
225 const __m128 vtmp11x4567 = _mm_unpackhi_ps(v10x4567, v11x4567); // k o l p from row 2, 3
226 const __m128 vtmp12x0123 = _mm_unpacklo_ps(v12x0123, v13x0123); // a e b f from row 0, 1
227 const __m128 vtmp13x0123 = _mm_unpacklo_ps(v14x0123, v15x0123); // i m j n from row 2, 3
228 const __m128 vtmp14x0123 = _mm_unpackhi_ps(v12x0123, v13x0123); // c g d h from row 0, 1
229 const __m128 vtmp15x0123 = _mm_unpackhi_ps(v14x0123, v15x0123); // k o l p from row 2, 3
230 const __m128 vtmp12x4567 = _mm_unpacklo_ps(v12x4567, v13x4567); // a e b f from row 0, 1
231 const __m128 vtmp13x4567 = _mm_unpacklo_ps(v14x4567, v15x4567); // i m j n from row 2, 3
232 const __m128 vtmp14x4567 = _mm_unpackhi_ps(v12x4567, v13x4567); // c g d h from row 0, 1
233 const __m128 vtmp15x4567 = _mm_unpackhi_ps(v14x4567, v15x4567); // k o l p from row 2, 3
234 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
235 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
236 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
237 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
238 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
239 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
240 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
241 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
242 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
243 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
244 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
245 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
246 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
247 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
248 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
249 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
250 // Transpose 4x4
251 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
252 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
253 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
254 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
255 v0x4567 = _mm_movelh_ps(vtmp0x4567, vtmp1x4567); // a e i m from row 0, 1
256 v1x4567 = _mm_movehl_ps(vtmp1x4567, vtmp0x4567); // b f j n from row 0, 1
257 v2x4567 = _mm_movelh_ps(vtmp2x4567, vtmp3x4567); // c g k o from row 2, 3
258 v3x4567 = _mm_movehl_ps(vtmp3x4567, vtmp2x4567); // d h l p from row 2, 3
259 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
260 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
261 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
262 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
263 v4x4567 = _mm_movelh_ps(vtmp4x4567, vtmp5x4567); // a e i m from row 0, 1
264 v5x4567 = _mm_movehl_ps(vtmp5x4567, vtmp4x4567); // b f j n from row 0, 1
265 v6x4567 = _mm_movelh_ps(vtmp6x4567, vtmp7x4567); // c g k o from row 2, 3
266 v7x4567 = _mm_movehl_ps(vtmp7x4567, vtmp6x4567); // d h l p from row 2, 3
267 v8x0123 = _mm_movelh_ps(vtmp8x0123, vtmp9x0123); // a e i m from row 0, 1
268 v9x0123 = _mm_movehl_ps(vtmp9x0123, vtmp8x0123); // b f j n from row 0, 1
269 v10x0123 = _mm_movelh_ps(vtmp10x0123, vtmp11x0123); // c g k o from row 2, 3
270 v11x0123 = _mm_movehl_ps(vtmp11x0123, vtmp10x0123); // d h l p from row 2, 3
271 v8x4567 = _mm_movelh_ps(vtmp8x4567, vtmp9x4567); // a e i m from row 0, 1
272 v9x4567 = _mm_movehl_ps(vtmp9x4567, vtmp8x4567); // b f j n from row 0, 1
273 v10x4567 = _mm_movelh_ps(vtmp10x4567, vtmp11x4567); // c g k o from row 2, 3
274 v11x4567 = _mm_movehl_ps(vtmp11x4567, vtmp10x4567); // d h l p from row 2, 3
275 v12x0123 = _mm_movelh_ps(vtmp12x0123, vtmp13x0123); // a e i m from row 0, 1
276 v13x0123 = _mm_movehl_ps(vtmp13x0123, vtmp12x0123); // b f j n from row 0, 1
277 v14x0123 = _mm_movelh_ps(vtmp14x0123, vtmp15x0123); // c g k o from row 2, 3
278 v15x0123 = _mm_movehl_ps(vtmp15x0123, vtmp14x0123); // d h l p from row 2, 3
279 v12x4567 = _mm_movelh_ps(vtmp12x4567, vtmp13x4567); // a e i m from row 0, 1
280 v13x4567 = _mm_movehl_ps(vtmp13x4567, vtmp12x4567); // b f j n from row 0, 1
281 v14x4567 = _mm_movelh_ps(vtmp14x4567, vtmp15x4567); // c g k o from row 2, 3
282 v15x4567 = _mm_movehl_ps(vtmp15x4567, vtmp14x4567); // d h l p from row 2, 3
283
284 _mm_store_ps(packed_w, v0x0123);
285 _mm_store_ps(packed_w + 4, v4x0123);
286 _mm_store_ps(packed_w + 8, v8x0123);
287 _mm_store_ps(packed_w + 12, v12x0123);
288 _mm_store_ps(packed_w + 16, v1x0123);
289 _mm_store_ps(packed_w + 20, v5x0123);
290 _mm_store_ps(packed_w + 24, v9x0123);
291 _mm_store_ps(packed_w + 28, v13x0123);
292 _mm_store_ps(packed_w + 32, v2x0123);
293 _mm_store_ps(packed_w + 36, v6x0123);
294 _mm_store_ps(packed_w + 40, v10x0123);
295 _mm_store_ps(packed_w + 44, v14x0123);
296 _mm_store_ps(packed_w + 48, v3x0123);
297 _mm_store_ps(packed_w + 52, v7x0123);
298 _mm_store_ps(packed_w + 56, v11x0123);
299 _mm_store_ps(packed_w + 60, v15x0123);
300 _mm_store_ps(packed_w + 64, v0x4567);
301 _mm_store_ps(packed_w + 68, v4x4567);
302 _mm_store_ps(packed_w + 72, v8x4567);
303 _mm_store_ps(packed_w + 76, v12x4567);
304 _mm_store_ps(packed_w + 80, v1x4567);
305 _mm_store_ps(packed_w + 84, v5x4567);
306 _mm_store_ps(packed_w + 88, v9x4567);
307 _mm_store_ps(packed_w + 92, v13x4567);
308 _mm_store_ps(packed_w + 96, v2x4567);
309 _mm_store_ps(packed_w + 100, v6x4567);
310 _mm_store_ps(packed_w + 104, v10x4567);
311 _mm_store_ps(packed_w + 108, v14x4567);
312 _mm_store_ps(packed_w + 112, v3x4567);
313 _mm_store_ps(packed_w + 116, v7x4567);
314 _mm_store_ps(packed_w + 120, v11x4567);
315 _mm_store_ps(packed_w + 124, v15x4567);
316 packed_w += 128;
317 }
318
319 // KC multiple of 4
320 for (; k >= 4; k -= 4) {
321 // Read blocks of 4x4
322 // a b c d
323 // e f g h
324 // i j k l
325 // m n o p
326 __m128 v0x0123 = _mm_loadu_ps(w0);
327 w0 += 4;
328 __m128 v1x0123 = _mm_loadu_ps(w1);
329 w1 += 4;
330 __m128 v2x0123 = _mm_loadu_ps(w2);
331 w2 += 4;
332 __m128 v3x0123 = _mm_loadu_ps(w3);
333 w3 += 4;
334 __m128 v4x0123 = _mm_loadu_ps(w4);
335 w4 += 4;
336 __m128 v5x0123 = _mm_loadu_ps(w5);
337 w5 += 4;
338 __m128 v6x0123 = _mm_loadu_ps(w6);
339 w6 += 4;
340 __m128 v7x0123 = _mm_loadu_ps(w7);
341 w7 += 4;
342 __m128 v8x0123 = _mm_loadu_ps(w8);
343 w8 += 4;
344 __m128 v9x0123 = _mm_loadu_ps(w9);
345 w9 += 4;
346 __m128 v10x0123 = _mm_loadu_ps(w10);
347 w10 += 4;
348 __m128 v11x0123 = _mm_loadu_ps(w11);
349 w11 += 4;
350 __m128 v12x0123 = _mm_loadu_ps(w12);
351 w12 += 4;
352 __m128 v13x0123 = _mm_loadu_ps(w13);
353 w13 += 4;
354 __m128 v14x0123 = _mm_loadu_ps(w14);
355 w14 += 4;
356 __m128 v15x0123 = _mm_loadu_ps(w15);
357 w15 += 4;
358
359 // Apply SR4 shuffle
360 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
361 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
362 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
363 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
364 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
365 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
366 v9x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x0123), _MM_SHUFFLE(0, 3, 2, 1)));
367 v10x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x0123), _MM_SHUFFLE(1, 0, 3, 2)));
368 v11x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x0123), _MM_SHUFFLE(2, 1, 0, 3)));
369 v13x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x0123), _MM_SHUFFLE(0, 3, 2, 1)));
370 v14x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x0123), _MM_SHUFFLE(1, 0, 3, 2)));
371 v15x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15x0123), _MM_SHUFFLE(2, 1, 0, 3)));
372
373 // Transpose 2x2
374 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
375 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
376 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
377 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
378 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
379 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
380 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
381 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
382 const __m128 vtmp8x0123 = _mm_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
383 const __m128 vtmp9x0123 = _mm_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
384 const __m128 vtmp10x0123 = _mm_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
385 const __m128 vtmp11x0123 = _mm_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
386 const __m128 vtmp12x0123 = _mm_unpacklo_ps(v12x0123, v13x0123); // a e b f from row 0, 1
387 const __m128 vtmp13x0123 = _mm_unpacklo_ps(v14x0123, v15x0123); // i m j n from row 2, 3
388 const __m128 vtmp14x0123 = _mm_unpackhi_ps(v12x0123, v13x0123); // c g d h from row 0, 1
389 const __m128 vtmp15x0123 = _mm_unpackhi_ps(v14x0123, v15x0123); // k o l p from row 2, 3
390 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
391 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
392 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
393 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
394 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
395 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
396 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
397 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
398 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
399 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
400 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
401 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
402 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
403 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
404 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
405 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
406 // Transpose 4x4
407 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
408 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
409 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
410 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
411 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
412 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
413 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
414 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
415 v8x0123 = _mm_movelh_ps(vtmp8x0123, vtmp9x0123); // a e i m from row 0, 1
416 v9x0123 = _mm_movehl_ps(vtmp9x0123, vtmp8x0123); // b f j n from row 0, 1
417 v10x0123 = _mm_movelh_ps(vtmp10x0123, vtmp11x0123); // c g k o from row 2, 3
418 v11x0123 = _mm_movehl_ps(vtmp11x0123, vtmp10x0123); // d h l p from row 2, 3
419 v12x0123 = _mm_movelh_ps(vtmp12x0123, vtmp13x0123); // a e i m from row 0, 1
420 v13x0123 = _mm_movehl_ps(vtmp13x0123, vtmp12x0123); // b f j n from row 0, 1
421 v14x0123 = _mm_movelh_ps(vtmp14x0123, vtmp15x0123); // c g k o from row 2, 3
422 v15x0123 = _mm_movehl_ps(vtmp15x0123, vtmp14x0123); // d h l p from row 2, 3
423
424 _mm_store_ps(packed_w, v0x0123);
425 _mm_store_ps(packed_w + 4, v4x0123);
426 _mm_store_ps(packed_w + 8, v8x0123);
427 _mm_store_ps(packed_w + 12, v12x0123);
428 _mm_store_ps(packed_w + 16, v1x0123);
429 _mm_store_ps(packed_w + 20, v5x0123);
430 _mm_store_ps(packed_w + 24, v9x0123);
431 _mm_store_ps(packed_w + 28, v13x0123);
432 _mm_store_ps(packed_w + 32, v2x0123);
433 _mm_store_ps(packed_w + 36, v6x0123);
434 _mm_store_ps(packed_w + 40, v10x0123);
435 _mm_store_ps(packed_w + 44, v14x0123);
436 _mm_store_ps(packed_w + 48, v3x0123);
437 _mm_store_ps(packed_w + 52, v7x0123);
438 _mm_store_ps(packed_w + 56, v11x0123);
439 _mm_store_ps(packed_w + 60, v15x0123);
440 packed_w += 64;
441 }
442
443 // KC remainder (1..3)
444 if XNN_UNLIKELY(k != 0) {
445 assert(k >= 1);
446 assert(k <= 3);
447 __m128 v0 = _mm_undefined_ps();
448 __m128 v1 = _mm_undefined_ps();
449 __m128 v2 = _mm_undefined_ps();
450 __m128 v3 = _mm_undefined_ps();
451 __m128 v4 = _mm_undefined_ps();
452 __m128 v5 = _mm_undefined_ps();
453 __m128 v6 = _mm_undefined_ps();
454 __m128 v7 = _mm_undefined_ps();
455 __m128 v8 = _mm_undefined_ps();
456 __m128 v9 = _mm_undefined_ps();
457 __m128 v10 = _mm_undefined_ps();
458 __m128 v11 = _mm_undefined_ps();
459 __m128 v12 = _mm_undefined_ps();
460 __m128 v13 = _mm_undefined_ps();
461 __m128 v14 = _mm_undefined_ps();
462 __m128 v15 = _mm_undefined_ps();
463
464 switch (k) {
465 case 1:
466 // Read blocks of 4x1
467 // a
468 // e
469 // i
470 // m
471 v0 = _mm_load_ss(w0);
472 w0 += 1;
473 v1 = _mm_load_ss(w1);
474 w1 += 1;
475 v2 = _mm_load_ss(w2);
476 w2 += 1;
477 v3 = _mm_load_ss(w3);
478 w3 += 1;
479 v4 = _mm_load_ss(w4);
480 w4 += 1;
481 v5 = _mm_load_ss(w5);
482 w5 += 1;
483 v6 = _mm_load_ss(w6);
484 w6 += 1;
485 v7 = _mm_load_ss(w7);
486 w7 += 1;
487 v8 = _mm_load_ss(w8);
488 w8 += 1;
489 v9 = _mm_load_ss(w9);
490 w9 += 1;
491 v10 = _mm_load_ss(w10);
492 w10 += 1;
493 v11 = _mm_load_ss(w11);
494 w11 += 1;
495 v12 = _mm_load_ss(w12);
496 w12 += 1;
497 v13 = _mm_load_ss(w13);
498 w13 += 1;
499 v14 = _mm_load_ss(w14);
500 w14 += 1;
501 v15 = _mm_load_ss(w15);
502 w15 += 1;
503 break;
504 case 2:
505 // Read blocks of 4x2
506 // a b
507 // e f
508 // i j
509 // m n
510 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
511 w0 += 2;
512 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
513 w1 += 2;
514 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
515 w2 += 2;
516 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
517 w3 += 2;
518 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
519 w4 += 2;
520 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
521 w5 += 2;
522 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
523 w6 += 2;
524 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
525 w7 += 2;
526 v8 = _mm_castpd_ps(_mm_load_sd((const double*) w8));
527 w8 += 2;
528 v9 = _mm_castpd_ps(_mm_load_sd((const double*) w9));
529 w9 += 2;
530 v10 = _mm_castpd_ps(_mm_load_sd((const double*) w10));
531 w10 += 2;
532 v11 = _mm_castpd_ps(_mm_load_sd((const double*) w11));
533 w11 += 2;
534 v12 = _mm_castpd_ps(_mm_load_sd((const double*) w12));
535 w12 += 2;
536 v13 = _mm_castpd_ps(_mm_load_sd((const double*) w13));
537 w13 += 2;
538 v14 = _mm_castpd_ps(_mm_load_sd((const double*) w14));
539 w14 += 2;
540 v15 = _mm_castpd_ps(_mm_load_sd((const double*) w15));
541 w15 += 2;
542 break;
543 case 3:
544 {
545 // Read blocks of 4x3
546 // a b c
547 // e f g
548 // i j k
549 // m n o
550 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
551 const __m128 v0hi = _mm_load_ss(w0 + 2);
552 v0 = _mm_movelh_ps(v0lo, v0hi);
553 w0 += 3;
554 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
555 const __m128 v1hi = _mm_load_ss(w1 + 2);
556 v1 = _mm_movelh_ps(v1lo, v1hi);
557 w1 += 3;
558 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
559 const __m128 v2hi = _mm_load_ss(w2 + 2);
560 v2 = _mm_movelh_ps(v2lo, v2hi);
561 w2 += 3;
562 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
563 const __m128 v3hi = _mm_load_ss(w3 + 2);
564 v3 = _mm_movelh_ps(v3lo, v3hi);
565 w3 += 3;
566 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
567 const __m128 v4hi = _mm_load_ss(w4 + 2);
568 v4 = _mm_movelh_ps(v4lo, v4hi);
569 w4 += 3;
570 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
571 const __m128 v5hi = _mm_load_ss(w5 + 2);
572 v5 = _mm_movelh_ps(v5lo, v5hi);
573 w5 += 3;
574 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
575 const __m128 v6hi = _mm_load_ss(w6 + 2);
576 v6 = _mm_movelh_ps(v6lo, v6hi);
577 w6 += 3;
578 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
579 const __m128 v7hi = _mm_load_ss(w7 + 2);
580 v7 = _mm_movelh_ps(v7lo, v7hi);
581 w7 += 3;
582 const __m128 v8lo = _mm_castpd_ps(_mm_load_sd((const double*) w8));
583 const __m128 v8hi = _mm_load_ss(w8 + 2);
584 v8 = _mm_movelh_ps(v8lo, v8hi);
585 w8 += 3;
586 const __m128 v9lo = _mm_castpd_ps(_mm_load_sd((const double*) w9));
587 const __m128 v9hi = _mm_load_ss(w9 + 2);
588 v9 = _mm_movelh_ps(v9lo, v9hi);
589 w9 += 3;
590 const __m128 v10lo = _mm_castpd_ps(_mm_load_sd((const double*) w10));
591 const __m128 v10hi = _mm_load_ss(w10 + 2);
592 v10 = _mm_movelh_ps(v10lo, v10hi);
593 w10 += 3;
594 const __m128 v11lo = _mm_castpd_ps(_mm_load_sd((const double*) w11));
595 const __m128 v11hi = _mm_load_ss(w11 + 2);
596 v11 = _mm_movelh_ps(v11lo, v11hi);
597 w11 += 3;
598 const __m128 v12lo = _mm_castpd_ps(_mm_load_sd((const double*) w12));
599 const __m128 v12hi = _mm_load_ss(w12 + 2);
600 v12 = _mm_movelh_ps(v12lo, v12hi);
601 w12 += 3;
602 const __m128 v13lo = _mm_castpd_ps(_mm_load_sd((const double*) w13));
603 const __m128 v13hi = _mm_load_ss(w13 + 2);
604 v13 = _mm_movelh_ps(v13lo, v13hi);
605 w13 += 3;
606 const __m128 v14lo = _mm_castpd_ps(_mm_load_sd((const double*) w14));
607 const __m128 v14hi = _mm_load_ss(w14 + 2);
608 v14 = _mm_movelh_ps(v14lo, v14hi);
609 w14 += 3;
610 const __m128 v15lo = _mm_castpd_ps(_mm_load_sd((const double*) w15));
611 const __m128 v15hi = _mm_load_ss(w15 + 2);
612 v15 = _mm_movelh_ps(v15lo, v15hi);
613 w15 += 3;
614 break;
615 }
616 default:
617 XNN_UNREACHABLE;
618 }
619
620 // Apply SR4 shuffle
621 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
622 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
623 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
624 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
625 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
626 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
627 v9 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9), _MM_SHUFFLE(0, 3, 2, 1)));
628 v10 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10), _MM_SHUFFLE(1, 0, 3, 2)));
629 v11 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11), _MM_SHUFFLE(2, 1, 0, 3)));
630 v13 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13), _MM_SHUFFLE(0, 3, 2, 1)));
631 v14 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14), _MM_SHUFFLE(1, 0, 3, 2)));
632 v15 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15), _MM_SHUFFLE(2, 1, 0, 3)));
633 // Transpose 2x2
634 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
635 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
636 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
637 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
638 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
639 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
640 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
641 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
642 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a e b f from row 0, 1
643 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // i m j n from row 2, 3
644 const __m128 vtmp10 = _mm_unpackhi_ps(v8, v9); // c g d h from row 0, 1
645 const __m128 vtmp11 = _mm_unpackhi_ps(v10, v11); // k o l p from row 2, 3
646 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a e b f from row 0, 1
647 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // i m j n from row 2, 3
648 const __m128 vtmp14 = _mm_unpackhi_ps(v12, v13); // c g d h from row 0, 1
649 const __m128 vtmp15 = _mm_unpackhi_ps(v14, v15); // k o l p from row 2, 3
650 // Transpose 4x4
651 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
652 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
653 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
654 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
655 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
656 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
657 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
658 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
659 v8 = _mm_movelh_ps(vtmp8, vtmp9); // a e i m from row 0, 1
660 v9 = _mm_movehl_ps(vtmp9, vtmp8); // b f j n from row 0, 1
661 v10 = _mm_movelh_ps(vtmp10, vtmp11); // c g k o from row 2, 3
662 v11 = _mm_movehl_ps(vtmp11, vtmp10); // d h l p from row 2, 3
663 v12 = _mm_movelh_ps(vtmp12, vtmp13); // a e i m from row 0, 1
664 v13 = _mm_movehl_ps(vtmp13, vtmp12); // b f j n from row 0, 1
665 v14 = _mm_movelh_ps(vtmp14, vtmp15); // c g k o from row 2, 3
666 v15 = _mm_movehl_ps(vtmp15, vtmp14); // d h l p from row 2, 3
667 _mm_store_ps(packed_w, v0);
668 _mm_store_ps(packed_w + 4, v4);
669 _mm_store_ps(packed_w + 8, v8);
670 _mm_store_ps(packed_w + 12, v12);
671 _mm_store_ps(packed_w + 16, v1);
672 _mm_store_ps(packed_w + 20, v5);
673 _mm_store_ps(packed_w + 24, v9);
674 _mm_store_ps(packed_w + 28, v13);
675 _mm_store_ps(packed_w + 32, v2);
676 _mm_store_ps(packed_w + 36, v6);
677 _mm_store_ps(packed_w + 40, v10);
678 _mm_store_ps(packed_w + 44, v14);
679 _mm_store_ps(packed_w + 48, v3);
680 _mm_store_ps(packed_w + 52, v7);
681 _mm_store_ps(packed_w + 56, v11);
682 _mm_store_ps(packed_w + 60, v15);
683 packed_w += 64;
684 }
685 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
686 w0 = w15;
687 }
688
689 // NC remainder (1..15)
690 if XNN_UNLIKELY(n != 0) {
691 assert(n >= 1);
692 assert(n <= 15);
693 if XNN_LIKELY(b != NULL) {
694 size_t nb = n;
695 do {
696 *packed_w++ = *b++;
697 } while (--nb != 0);
698 packed_w += (16 - n);
699 } else {
700 const __m128 vzero = _mm_setzero_ps();
701 _mm_store_ps(packed_w, vzero);
702 _mm_store_ps(packed_w + 4, vzero);
703 _mm_store_ps(packed_w + 8, vzero);
704 _mm_store_ps(packed_w + 12, vzero);
705 packed_w += 16;
706 }
707
708 // NR remainder has less than 16 rows so last row is not loaded
709 const float* w1 = w0 + kc;
710 if XNN_UNPREDICTABLE(n < 2) {
711 w1 = w0;
712 }
713 const float* w2 = w1 + kc;
714 if XNN_UNPREDICTABLE(n <= 2) {
715 w2 = w1;
716 }
717 const float* w3 = w2 + kc;
718 if XNN_UNPREDICTABLE(n < 4) {
719 w3 = w2;
720 }
721 const float* w4 = w3 + kc;
722 if XNN_UNPREDICTABLE(n <= 4) {
723 w4 = w3;
724 }
725 const float* w5 = w4 + kc;
726 if XNN_UNPREDICTABLE(n < 6) {
727 w5 = w4;
728 }
729 const float* w6 = w5 + kc;
730 if XNN_UNPREDICTABLE(n <= 6) {
731 w6 = w5;
732 }
733 const float* w7 = w6 + kc;
734 if XNN_UNPREDICTABLE(n < 8) {
735 w7 = w6;
736 }
737 const float* w8 = w7 + kc;
738 if XNN_UNPREDICTABLE(n <= 8) {
739 w8 = w7;
740 }
741 const float* w9 = w8 + kc;
742 if XNN_UNPREDICTABLE(n < 10) {
743 w9 = w8;
744 }
745 const float* w10 = w9 + kc;
746 if XNN_UNPREDICTABLE(n <= 10) {
747 w10 = w9;
748 }
749 const float* w11 = w10 + kc;
750 if XNN_UNPREDICTABLE(n < 12) {
751 w11 = w10;
752 }
753 const float* w12 = w11 + kc;
754 if XNN_UNPREDICTABLE(n <= 12) {
755 w12 = w11;
756 }
757 const float* w13 = w12 + kc;
758 if XNN_UNPREDICTABLE(n < 14) {
759 w13 = w12;
760 }
761 const float* w14 = w13 + kc;
762 if XNN_UNPREDICTABLE(n <= 14) {
763 w14 = w13;
764 }
765
766 size_t k = kc;
767 // KC main loop multiple of 8
768 for (; k >= 8; k -= 8) {
769 // Read blocks of 4x4
770 // a b c d
771 // e f g h
772 // i j k l
773 // m n o p
774 __m128 v0x0123 = _mm_loadu_ps(w0);
775 __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
776 w0 += 8;
777 __m128 v1x0123 = _mm_loadu_ps(w1);
778 __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
779 w1 += 8;
780 __m128 v2x0123 = _mm_loadu_ps(w2);
781 __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
782 w2 += 8;
783 __m128 v3x0123 = _mm_loadu_ps(w3);
784 __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
785 w3 += 8;
786 __m128 v4x0123 = _mm_loadu_ps(w4);
787 __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
788 w4 += 8;
789 __m128 v5x0123 = _mm_loadu_ps(w5);
790 __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
791 w5 += 8;
792 __m128 v6x0123 = _mm_loadu_ps(w6);
793 __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
794 w6 += 8;
795 __m128 v7x0123 = _mm_loadu_ps(w7);
796 __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
797 w7 += 8;
798 __m128 v8x0123 = _mm_loadu_ps(w8);
799 __m128 v8x4567 = _mm_loadu_ps(w8 + 4);
800 w8 += 8;
801 __m128 v9x0123 = _mm_loadu_ps(w9);
802 __m128 v9x4567 = _mm_loadu_ps(w9 + 4);
803 w9 += 8;
804 __m128 v10x0123 = _mm_loadu_ps(w10);
805 __m128 v10x4567 = _mm_loadu_ps(w10 + 4);
806 w10 += 8;
807 __m128 v11x0123 = _mm_loadu_ps(w11);
808 __m128 v11x4567 = _mm_loadu_ps(w11 + 4);
809 w11 += 8;
810 __m128 v12x0123 = _mm_loadu_ps(w12);
811 __m128 v12x4567 = _mm_loadu_ps(w12 + 4);
812 w12 += 8;
813 __m128 v13x0123 = _mm_loadu_ps(w13);
814 __m128 v13x4567 = _mm_loadu_ps(w13 + 4);
815 w13 += 8;
816 __m128 v14x0123 = _mm_loadu_ps(w14);
817 __m128 v14x4567 = _mm_loadu_ps(w14 + 4);
818 w14 += 8;
819
820 __m128 v15x0123 = _mm_undefined_ps();
821 __m128 v15x4567 = _mm_undefined_ps();
822
823 // Apply SR4 shuffle
824 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
825 v1x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x4567), _MM_SHUFFLE(0, 3, 2, 1)));
826 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
827 v2x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x4567), _MM_SHUFFLE(1, 0, 3, 2)));
828 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
829 v3x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x4567), _MM_SHUFFLE(2, 1, 0, 3)));
830 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
831 v5x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x4567), _MM_SHUFFLE(0, 3, 2, 1)));
832 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
833 v6x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x4567), _MM_SHUFFLE(1, 0, 3, 2)));
834 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
835 v7x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x4567), _MM_SHUFFLE(2, 1, 0, 3)));
836 v9x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x0123), _MM_SHUFFLE(0, 3, 2, 1)));
837 v9x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x4567), _MM_SHUFFLE(0, 3, 2, 1)));
838 v10x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x0123), _MM_SHUFFLE(1, 0, 3, 2)));
839 v10x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x4567), _MM_SHUFFLE(1, 0, 3, 2)));
840 v11x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x0123), _MM_SHUFFLE(2, 1, 0, 3)));
841 v11x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x4567), _MM_SHUFFLE(2, 1, 0, 3)));
842 v13x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x0123), _MM_SHUFFLE(0, 3, 2, 1)));
843 v13x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x4567), _MM_SHUFFLE(0, 3, 2, 1)));
844 v14x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x0123), _MM_SHUFFLE(1, 0, 3, 2)));
845 v14x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x4567), _MM_SHUFFLE(1, 0, 3, 2)));
846
847 // Transpose 2x2
848 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
849 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
850 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
851 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
852 const __m128 vtmp0x4567 = _mm_unpacklo_ps(v0x4567, v1x4567); // a e b f from row 0, 1
853 const __m128 vtmp1x4567 = _mm_unpacklo_ps(v2x4567, v3x4567); // i m j n from row 2, 3
854 const __m128 vtmp2x4567 = _mm_unpackhi_ps(v0x4567, v1x4567); // c g d h from row 0, 1
855 const __m128 vtmp3x4567 = _mm_unpackhi_ps(v2x4567, v3x4567); // k o l p from row 2, 3
856 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
857 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
858 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
859 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
860 const __m128 vtmp4x4567 = _mm_unpacklo_ps(v4x4567, v5x4567); // a e b f from row 0, 1
861 const __m128 vtmp5x4567 = _mm_unpacklo_ps(v6x4567, v7x4567); // i m j n from row 2, 3
862 const __m128 vtmp6x4567 = _mm_unpackhi_ps(v4x4567, v5x4567); // c g d h from row 0, 1
863 const __m128 vtmp7x4567 = _mm_unpackhi_ps(v6x4567, v7x4567); // k o l p from row 2, 3
864 const __m128 vtmp8x0123 = _mm_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
865 const __m128 vtmp9x0123 = _mm_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
866 const __m128 vtmp10x0123 = _mm_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
867 const __m128 vtmp11x0123 = _mm_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
868 const __m128 vtmp8x4567 = _mm_unpacklo_ps(v8x4567, v9x4567); // a e b f from row 0, 1
869 const __m128 vtmp9x4567 = _mm_unpacklo_ps(v10x4567, v11x4567); // i m j n from row 2, 3
870 const __m128 vtmp10x4567 = _mm_unpackhi_ps(v8x4567, v9x4567); // c g d h from row 0, 1
871 const __m128 vtmp11x4567 = _mm_unpackhi_ps(v10x4567, v11x4567); // k o l p from row 2, 3
872 const __m128 vtmp12x0123 = _mm_unpacklo_ps(v12x0123, v13x0123); // a e b f from row 0, 1
873 const __m128 vtmp13x0123 = _mm_unpacklo_ps(v14x0123, v14x0123); // i m j n from row 2, 3
874 const __m128 vtmp14x0123 = _mm_unpackhi_ps(v12x0123, v13x0123); // c g d h from row 0, 1
875 const __m128 vtmp15x0123 = _mm_unpackhi_ps(v14x0123, v14x0123); // k o l p from row 2, 3
876 const __m128 vtmp12x4567 = _mm_unpacklo_ps(v12x4567, v13x4567); // a e b f from row 0, 1
877 const __m128 vtmp13x4567 = _mm_unpacklo_ps(v14x4567, v14x4567); // i m j n from row 2, 3
878 const __m128 vtmp14x4567 = _mm_unpackhi_ps(v12x4567, v13x4567); // c g d h from row 0, 1
879 const __m128 vtmp15x4567 = _mm_unpackhi_ps(v14x4567, v14x4567); // k o l p from row 2, 3
880 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
881 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
882 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
883 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
884 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
885 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
886 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
887 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
888 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
889 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
890 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
891 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
892 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
893 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
894 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
895 // Transpose 4x4
896 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
897 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
898 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
899 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
900 v0x4567 = _mm_movelh_ps(vtmp0x4567, vtmp1x4567); // a e i m from row 0, 1
901 v1x4567 = _mm_movehl_ps(vtmp1x4567, vtmp0x4567); // b f j n from row 0, 1
902 v2x4567 = _mm_movelh_ps(vtmp2x4567, vtmp3x4567); // c g k o from row 2, 3
903 v3x4567 = _mm_movehl_ps(vtmp3x4567, vtmp2x4567); // d h l p from row 2, 3
904 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
905 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
906 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
907 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
908 v4x4567 = _mm_movelh_ps(vtmp4x4567, vtmp5x4567); // a e i m from row 0, 1
909 v5x4567 = _mm_movehl_ps(vtmp5x4567, vtmp4x4567); // b f j n from row 0, 1
910 v6x4567 = _mm_movelh_ps(vtmp6x4567, vtmp7x4567); // c g k o from row 2, 3
911 v7x4567 = _mm_movehl_ps(vtmp7x4567, vtmp6x4567); // d h l p from row 2, 3
912 v8x0123 = _mm_movelh_ps(vtmp8x0123, vtmp9x0123); // a e i m from row 0, 1
913 v9x0123 = _mm_movehl_ps(vtmp9x0123, vtmp8x0123); // b f j n from row 0, 1
914 v10x0123 = _mm_movelh_ps(vtmp10x0123, vtmp11x0123); // c g k o from row 2, 3
915 v11x0123 = _mm_movehl_ps(vtmp11x0123, vtmp10x0123); // d h l p from row 2, 3
916 v8x4567 = _mm_movelh_ps(vtmp8x4567, vtmp9x4567); // a e i m from row 0, 1
917 v9x4567 = _mm_movehl_ps(vtmp9x4567, vtmp8x4567); // b f j n from row 0, 1
918 v10x4567 = _mm_movelh_ps(vtmp10x4567, vtmp11x4567); // c g k o from row 2, 3
919 v11x4567 = _mm_movehl_ps(vtmp11x4567, vtmp10x4567); // d h l p from row 2, 3
920 v12x0123 = _mm_movelh_ps(vtmp12x0123, vtmp13x0123); // a e i m from row 0, 1
921 v13x0123 = _mm_movehl_ps(vtmp13x0123, vtmp12x0123); // b f j n from row 0, 1
922 v14x0123 = _mm_movelh_ps(vtmp14x0123, vtmp15x0123); // c g k o from row 2, 3
923 v15x0123 = _mm_movehl_ps(vtmp15x0123, vtmp14x0123); // d h l p from row 2, 3
924 v12x4567 = _mm_movelh_ps(vtmp12x4567, vtmp13x4567); // a e i m from row 0, 1
925 v13x4567 = _mm_movehl_ps(vtmp13x4567, vtmp12x4567); // b f j n from row 0, 1
926 v14x4567 = _mm_movelh_ps(vtmp14x4567, vtmp15x4567); // c g k o from row 2, 3
927 v15x4567 = _mm_movehl_ps(vtmp15x4567, vtmp14x4567); // d h l p from row 2, 3
928
929 _mm_store_ps(packed_w, v0x0123);
930 _mm_store_ps(packed_w + 4, v4x0123);
931 _mm_store_ps(packed_w + 8, v8x0123);
932 _mm_store_ps(packed_w + 12, v12x0123);
933 _mm_store_ps(packed_w + 16, v1x0123);
934 _mm_store_ps(packed_w + 20, v5x0123);
935 _mm_store_ps(packed_w + 24, v9x0123);
936 _mm_store_ps(packed_w + 28, v13x0123);
937 _mm_store_ps(packed_w + 32, v2x0123);
938 _mm_store_ps(packed_w + 36, v6x0123);
939 _mm_store_ps(packed_w + 40, v10x0123);
940 _mm_store_ps(packed_w + 44, v14x0123);
941 _mm_store_ps(packed_w + 48, v3x0123);
942 _mm_store_ps(packed_w + 52, v7x0123);
943 _mm_store_ps(packed_w + 56, v11x0123);
944 _mm_store_ps(packed_w + 60, v15x0123);
945 _mm_store_ps(packed_w + 64, v0x4567);
946 _mm_store_ps(packed_w + 68, v4x4567);
947 _mm_store_ps(packed_w + 72, v8x4567);
948 _mm_store_ps(packed_w + 76, v12x4567);
949 _mm_store_ps(packed_w + 80, v1x4567);
950 _mm_store_ps(packed_w + 84, v5x4567);
951 _mm_store_ps(packed_w + 88, v9x4567);
952 _mm_store_ps(packed_w + 92, v13x4567);
953 _mm_store_ps(packed_w + 96, v2x4567);
954 _mm_store_ps(packed_w + 100, v6x4567);
955 _mm_store_ps(packed_w + 104, v10x4567);
956 _mm_store_ps(packed_w + 108, v14x4567);
957 _mm_store_ps(packed_w + 112, v3x4567);
958 _mm_store_ps(packed_w + 116, v7x4567);
959 _mm_store_ps(packed_w + 120, v11x4567);
960 _mm_store_ps(packed_w + 124, v15x4567);
961 packed_w += 128;
962 }
963
964 // KC multiple of 4
965 for (; k >= 4; k -= 4) {
966 // Read blocks of 4x4
967 // a b c d
968 // e f g h
969 // i j k l
970 // m n o p
971 __m128 v0x0123 = _mm_loadu_ps(w0);
972 w0 += 4;
973 __m128 v1x0123 = _mm_loadu_ps(w1);
974 w1 += 4;
975 __m128 v2x0123 = _mm_loadu_ps(w2);
976 w2 += 4;
977 __m128 v3x0123 = _mm_loadu_ps(w3);
978 w3 += 4;
979 __m128 v4x0123 = _mm_loadu_ps(w4);
980 w4 += 4;
981 __m128 v5x0123 = _mm_loadu_ps(w5);
982 w5 += 4;
983 __m128 v6x0123 = _mm_loadu_ps(w6);
984 w6 += 4;
985 __m128 v7x0123 = _mm_loadu_ps(w7);
986 w7 += 4;
987 __m128 v8x0123 = _mm_loadu_ps(w8);
988 w8 += 4;
989 __m128 v9x0123 = _mm_loadu_ps(w9);
990 w9 += 4;
991 __m128 v10x0123 = _mm_loadu_ps(w10);
992 w10 += 4;
993 __m128 v11x0123 = _mm_loadu_ps(w11);
994 w11 += 4;
995 __m128 v12x0123 = _mm_loadu_ps(w12);
996 w12 += 4;
997 __m128 v13x0123 = _mm_loadu_ps(w13);
998 w13 += 4;
999 __m128 v14x0123 = _mm_loadu_ps(w14);
1000 w14 += 4;
1001 __m128 v15x0123 = _mm_undefined_ps();
1002
1003 // Apply SR4 shuffle
1004 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
1005 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
1006 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
1007 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
1008 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
1009 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
1010 v9x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x0123), _MM_SHUFFLE(0, 3, 2, 1)));
1011 v10x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x0123), _MM_SHUFFLE(1, 0, 3, 2)));
1012 v11x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x0123), _MM_SHUFFLE(2, 1, 0, 3)));
1013 v13x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x0123), _MM_SHUFFLE(0, 3, 2, 1)));
1014 v14x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x0123), _MM_SHUFFLE(1, 0, 3, 2)));
1015
1016 // Transpose 2x2
1017 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
1018 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
1019 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
1020 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
1021 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
1022 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
1023 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
1024 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
1025 const __m128 vtmp8x0123 = _mm_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
1026 const __m128 vtmp9x0123 = _mm_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
1027 const __m128 vtmp10x0123 = _mm_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
1028 const __m128 vtmp11x0123 = _mm_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
1029 const __m128 vtmp12x0123 = _mm_unpacklo_ps(v12x0123, v13x0123); // a e b f from row 0, 1
1030 const __m128 vtmp13x0123 = _mm_unpacklo_ps(v14x0123, v14x0123); // i m j n from row 2, 3
1031 const __m128 vtmp14x0123 = _mm_unpackhi_ps(v12x0123, v13x0123); // c g d h from row 0, 1
1032 const __m128 vtmp15x0123 = _mm_unpackhi_ps(v14x0123, v14x0123); // k o l p from row 2, 3
1033 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
1034 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
1035 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
1036 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
1037 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
1038 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
1039 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
1040 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
1041 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
1042 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
1043 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
1044 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
1045 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
1046 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
1047 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
1048 // Transpose 4x4
1049 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
1050 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
1051 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
1052 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
1053 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
1054 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
1055 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
1056 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
1057 v8x0123 = _mm_movelh_ps(vtmp8x0123, vtmp9x0123); // a e i m from row 0, 1
1058 v9x0123 = _mm_movehl_ps(vtmp9x0123, vtmp8x0123); // b f j n from row 0, 1
1059 v10x0123 = _mm_movelh_ps(vtmp10x0123, vtmp11x0123); // c g k o from row 2, 3
1060 v11x0123 = _mm_movehl_ps(vtmp11x0123, vtmp10x0123); // d h l p from row 2, 3
1061 v12x0123 = _mm_movelh_ps(vtmp12x0123, vtmp13x0123); // a e i m from row 0, 1
1062 v13x0123 = _mm_movehl_ps(vtmp13x0123, vtmp12x0123); // b f j n from row 0, 1
1063 v14x0123 = _mm_movelh_ps(vtmp14x0123, vtmp15x0123); // c g k o from row 2, 3
1064 v15x0123 = _mm_movehl_ps(vtmp15x0123, vtmp14x0123); // d h l p from row 2, 3
1065
1066 _mm_store_ps(packed_w, v0x0123);
1067 _mm_store_ps(packed_w + 4, v4x0123);
1068 _mm_store_ps(packed_w + 8, v8x0123);
1069 _mm_store_ps(packed_w + 12, v12x0123);
1070 _mm_store_ps(packed_w + 16, v1x0123);
1071 _mm_store_ps(packed_w + 20, v5x0123);
1072 _mm_store_ps(packed_w + 24, v9x0123);
1073 _mm_store_ps(packed_w + 28, v13x0123);
1074 _mm_store_ps(packed_w + 32, v2x0123);
1075 _mm_store_ps(packed_w + 36, v6x0123);
1076 _mm_store_ps(packed_w + 40, v10x0123);
1077 _mm_store_ps(packed_w + 44, v14x0123);
1078 _mm_store_ps(packed_w + 48, v3x0123);
1079 _mm_store_ps(packed_w + 52, v7x0123);
1080 _mm_store_ps(packed_w + 56, v11x0123);
1081 _mm_store_ps(packed_w + 60, v15x0123);
1082 packed_w += 64;
1083 }
1084
1085 // KC remainder (1..3)
1086 if XNN_UNLIKELY(k != 0) {
1087 assert(k >= 1);
1088 assert(k <= 3);
1089 __m128 v0 = _mm_undefined_ps();
1090 __m128 v1 = _mm_undefined_ps();
1091 __m128 v2 = _mm_undefined_ps();
1092 __m128 v3 = _mm_undefined_ps();
1093 __m128 v4 = _mm_undefined_ps();
1094 __m128 v5 = _mm_undefined_ps();
1095 __m128 v6 = _mm_undefined_ps();
1096 __m128 v7 = _mm_undefined_ps();
1097 __m128 v8 = _mm_undefined_ps();
1098 __m128 v9 = _mm_undefined_ps();
1099 __m128 v10 = _mm_undefined_ps();
1100 __m128 v11 = _mm_undefined_ps();
1101 __m128 v12 = _mm_undefined_ps();
1102 __m128 v13 = _mm_undefined_ps();
1103 __m128 v14 = _mm_undefined_ps();
1104 __m128 v15 = _mm_undefined_ps();
1105
1106 switch (k) {
1107 case 1:
1108 // Read blocks of 4x1
1109 // a
1110 // e
1111 // i
1112 // m
1113 v0 = _mm_load_ss(w0);
1114 w0 += 1;
1115 v1 = _mm_load_ss(w1);
1116 w1 += 1;
1117 v2 = _mm_load_ss(w2);
1118 w2 += 1;
1119 v3 = _mm_load_ss(w3);
1120 w3 += 1;
1121 v4 = _mm_load_ss(w4);
1122 w4 += 1;
1123 v5 = _mm_load_ss(w5);
1124 w5 += 1;
1125 v6 = _mm_load_ss(w6);
1126 w6 += 1;
1127 v7 = _mm_load_ss(w7);
1128 w7 += 1;
1129 v8 = _mm_load_ss(w8);
1130 w8 += 1;
1131 v9 = _mm_load_ss(w9);
1132 w9 += 1;
1133 v10 = _mm_load_ss(w10);
1134 w10 += 1;
1135 v11 = _mm_load_ss(w11);
1136 w11 += 1;
1137 v12 = _mm_load_ss(w12);
1138 w12 += 1;
1139 v13 = _mm_load_ss(w13);
1140 w13 += 1;
1141 v14 = _mm_load_ss(w14);
1142 w14 += 1;
1143 break;
1144 case 2:
1145 // Read blocks of 4x2
1146 // a b
1147 // e f
1148 // i j
1149 // m n
1150 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
1151 w0 += 2;
1152 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
1153 w1 += 2;
1154 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
1155 w2 += 2;
1156 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
1157 w3 += 2;
1158 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
1159 w4 += 2;
1160 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
1161 w5 += 2;
1162 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
1163 w6 += 2;
1164 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
1165 w7 += 2;
1166 v8 = _mm_castpd_ps(_mm_load_sd((const double*) w8));
1167 w8 += 2;
1168 v9 = _mm_castpd_ps(_mm_load_sd((const double*) w9));
1169 w9 += 2;
1170 v10 = _mm_castpd_ps(_mm_load_sd((const double*) w10));
1171 w10 += 2;
1172 v11 = _mm_castpd_ps(_mm_load_sd((const double*) w11));
1173 w11 += 2;
1174 v12 = _mm_castpd_ps(_mm_load_sd((const double*) w12));
1175 w12 += 2;
1176 v13 = _mm_castpd_ps(_mm_load_sd((const double*) w13));
1177 w13 += 2;
1178 v14 = _mm_castpd_ps(_mm_load_sd((const double*) w14));
1179 w14 += 2;
1180 break;
1181 case 3:
1182 {
1183 // Read blocks of 4x3
1184 // a b c
1185 // e f g
1186 // i j k
1187 // m n o
1188 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
1189 const __m128 v0hi = _mm_load_ss(w0 + 2);
1190 v0 = _mm_movelh_ps(v0lo, v0hi);
1191 w0 += 3;
1192 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
1193 const __m128 v1hi = _mm_load_ss(w1 + 2);
1194 v1 = _mm_movelh_ps(v1lo, v1hi);
1195 w1 += 3;
1196 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
1197 const __m128 v2hi = _mm_load_ss(w2 + 2);
1198 v2 = _mm_movelh_ps(v2lo, v2hi);
1199 w2 += 3;
1200 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
1201 const __m128 v3hi = _mm_load_ss(w3 + 2);
1202 v3 = _mm_movelh_ps(v3lo, v3hi);
1203 w3 += 3;
1204 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
1205 const __m128 v4hi = _mm_load_ss(w4 + 2);
1206 v4 = _mm_movelh_ps(v4lo, v4hi);
1207 w4 += 3;
1208 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
1209 const __m128 v5hi = _mm_load_ss(w5 + 2);
1210 v5 = _mm_movelh_ps(v5lo, v5hi);
1211 w5 += 3;
1212 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
1213 const __m128 v6hi = _mm_load_ss(w6 + 2);
1214 v6 = _mm_movelh_ps(v6lo, v6hi);
1215 w6 += 3;
1216 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
1217 const __m128 v7hi = _mm_load_ss(w7 + 2);
1218 v7 = _mm_movelh_ps(v7lo, v7hi);
1219 w7 += 3;
1220 const __m128 v8lo = _mm_castpd_ps(_mm_load_sd((const double*) w8));
1221 const __m128 v8hi = _mm_load_ss(w8 + 2);
1222 v8 = _mm_movelh_ps(v8lo, v8hi);
1223 w8 += 3;
1224 const __m128 v9lo = _mm_castpd_ps(_mm_load_sd((const double*) w9));
1225 const __m128 v9hi = _mm_load_ss(w9 + 2);
1226 v9 = _mm_movelh_ps(v9lo, v9hi);
1227 w9 += 3;
1228 const __m128 v10lo = _mm_castpd_ps(_mm_load_sd((const double*) w10));
1229 const __m128 v10hi = _mm_load_ss(w10 + 2);
1230 v10 = _mm_movelh_ps(v10lo, v10hi);
1231 w10 += 3;
1232 const __m128 v11lo = _mm_castpd_ps(_mm_load_sd((const double*) w11));
1233 const __m128 v11hi = _mm_load_ss(w11 + 2);
1234 v11 = _mm_movelh_ps(v11lo, v11hi);
1235 w11 += 3;
1236 const __m128 v12lo = _mm_castpd_ps(_mm_load_sd((const double*) w12));
1237 const __m128 v12hi = _mm_load_ss(w12 + 2);
1238 v12 = _mm_movelh_ps(v12lo, v12hi);
1239 w12 += 3;
1240 const __m128 v13lo = _mm_castpd_ps(_mm_load_sd((const double*) w13));
1241 const __m128 v13hi = _mm_load_ss(w13 + 2);
1242 v13 = _mm_movelh_ps(v13lo, v13hi);
1243 w13 += 3;
1244 const __m128 v14lo = _mm_castpd_ps(_mm_load_sd((const double*) w14));
1245 const __m128 v14hi = _mm_load_ss(w14 + 2);
1246 v14 = _mm_movelh_ps(v14lo, v14hi);
1247 w14 += 3;
1248 break;
1249 }
1250 default:
1251 XNN_UNREACHABLE;
1252 }
1253
1254 // Apply SR4 shuffle
1255 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
1256 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
1257 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
1258 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
1259 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
1260 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
1261 v9 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9), _MM_SHUFFLE(0, 3, 2, 1)));
1262 v10 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10), _MM_SHUFFLE(1, 0, 3, 2)));
1263 v11 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11), _MM_SHUFFLE(2, 1, 0, 3)));
1264 v13 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13), _MM_SHUFFLE(0, 3, 2, 1)));
1265 v14 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14), _MM_SHUFFLE(1, 0, 3, 2)));
1266 // Transpose 2x2
1267 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
1268 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
1269 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
1270 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
1271 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
1272 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
1273 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
1274 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
1275 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a e b f from row 0, 1
1276 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // i m j n from row 2, 3
1277 const __m128 vtmp10 = _mm_unpackhi_ps(v8, v9); // c g d h from row 0, 1
1278 const __m128 vtmp11 = _mm_unpackhi_ps(v10, v11); // k o l p from row 2, 3
1279 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a e b f from row 0, 1
1280 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // i m j n from row 2, 3
1281 const __m128 vtmp14 = _mm_unpackhi_ps(v12, v13); // c g d h from row 0, 1
1282 const __m128 vtmp15 = _mm_unpackhi_ps(v14, v14); // k o l p from row 2, 3
1283 // Transpose 4x4
1284 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
1285 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
1286 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
1287 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
1288 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
1289 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
1290 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
1291 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
1292 v8 = _mm_movelh_ps(vtmp8, vtmp9); // a e i m from row 0, 1
1293 v9 = _mm_movehl_ps(vtmp9, vtmp8); // b f j n from row 0, 1
1294 v10 = _mm_movelh_ps(vtmp10, vtmp11); // c g k o from row 2, 3
1295 v11 = _mm_movehl_ps(vtmp11, vtmp10); // d h l p from row 2, 3
1296 v12 = _mm_movelh_ps(vtmp12, vtmp13); // a e i m from row 0, 1
1297 v13 = _mm_movehl_ps(vtmp13, vtmp12); // b f j n from row 0, 1
1298 v14 = _mm_movelh_ps(vtmp14, vtmp15); // c g k o from row 2, 3
1299 v15 = _mm_movehl_ps(vtmp15, vtmp14); // d h l p from row 2, 3
1300 _mm_store_ps(packed_w, v0);
1301 _mm_store_ps(packed_w + 4, v4);
1302 _mm_store_ps(packed_w + 8, v8);
1303 _mm_store_ps(packed_w + 12, v12);
1304 _mm_store_ps(packed_w + 16, v1);
1305 _mm_store_ps(packed_w + 20, v5);
1306 _mm_store_ps(packed_w + 24, v9);
1307 _mm_store_ps(packed_w + 28, v13);
1308 _mm_store_ps(packed_w + 32, v2);
1309 _mm_store_ps(packed_w + 36, v6);
1310 _mm_store_ps(packed_w + 40, v10);
1311 _mm_store_ps(packed_w + 44, v14);
1312 _mm_store_ps(packed_w + 48, v3);
1313 _mm_store_ps(packed_w + 52, v7);
1314 _mm_store_ps(packed_w + 56, v11);
1315 _mm_store_ps(packed_w + 60, v15);
1316 packed_w += 64;
1317 }
1318 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
1319 }
1320 weights += nc * kc;
1321 } while (--g != 0);
1322 }