Debian Codegraph @

src/x32-packw/gen/x32-packw-x16s4-gemm-goi-sse2-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x16s4__sse2_u8(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 16);
38 assert(kr == 1);
39 assert(sr == 4);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 const float* b = (const float*) bias;
44 float* packed_w = (float*) packed_weights;
45 do {
46 // NC main loop multiple of 16
47 const float* w0 = (const float*) weights;
48 size_t n = nc;
49
50 for (; n >= 16; n -= 16) {
51 if XNN_LIKELY(b != NULL) {
52 const __m128 vb0 = _mm_loadu_ps(b);
53 const __m128 vb4 = _mm_loadu_ps(b + 4);
54 const __m128 vb8 = _mm_loadu_ps(b + 8);
55 const __m128 vb12 = _mm_loadu_ps(b + 12);
56 _mm_store_ps(packed_w, vb0);
57 _mm_store_ps(packed_w + 4, vb4);
58 _mm_store_ps(packed_w + 8, vb8);
59 _mm_store_ps(packed_w + 12, vb12);
60 b += 16;
61 } else {
62 const __m128 vzero = _mm_setzero_ps();
63 _mm_store_ps(packed_w, vzero);
64 _mm_store_ps(packed_w + 4, vzero);
65 _mm_store_ps(packed_w + 8, vzero);
66 _mm_store_ps(packed_w + 12, vzero);
67 }
68 packed_w += 16;
69
70 const float* w1 = w0 + kc;
71 const float* w2 = w1 + kc;
72 const float* w3 = w2 + kc;
73 const float* w4 = w3 + kc;
74 const float* w5 = w4 + kc;
75 const float* w6 = w5 + kc;
76 const float* w7 = w6 + kc;
77 const float* w8 = w7 + kc;
78 const float* w9 = w8 + kc;
79 const float* w10 = w9 + kc;
80 const float* w11 = w10 + kc;
81 const float* w12 = w11 + kc;
82 const float* w13 = w12 + kc;
83 const float* w14 = w13 + kc;
84 const float* w15 = w14 + kc;
85
86 size_t k = kc;
87 // KC main loop multiple of 8
88 for (; k >= 8; k -= 8) {
89 // Read blocks of 4x4
90 // a b c d
91 // e f g h
92 // i j k l
93 // m n o p
94 __m128 v0x0123 = _mm_loadu_ps(w0);
95 __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
96 w0 += 8;
97 __m128 v1x0123 = _mm_loadu_ps(w1);
98 __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
99 w1 += 8;
100 __m128 v2x0123 = _mm_loadu_ps(w2);
101 __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
102 w2 += 8;
103 __m128 v3x0123 = _mm_loadu_ps(w3);
104 __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
105 w3 += 8;
106 __m128 v4x0123 = _mm_loadu_ps(w4);
107 __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
108 w4 += 8;
109 __m128 v5x0123 = _mm_loadu_ps(w5);
110 __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
111 w5 += 8;
112 __m128 v6x0123 = _mm_loadu_ps(w6);
113 __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
114 w6 += 8;
115 __m128 v7x0123 = _mm_loadu_ps(w7);
116 __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
117 w7 += 8;
118 __m128 v8x0123 = _mm_loadu_ps(w8);
119 __m128 v8x4567 = _mm_loadu_ps(w8 + 4);
120 w8 += 8;
121 __m128 v9x0123 = _mm_loadu_ps(w9);
122 __m128 v9x4567 = _mm_loadu_ps(w9 + 4);
123 w9 += 8;
124 __m128 v10x0123 = _mm_loadu_ps(w10);
125 __m128 v10x4567 = _mm_loadu_ps(w10 + 4);
126 w10 += 8;
127 __m128 v11x0123 = _mm_loadu_ps(w11);
128 __m128 v11x4567 = _mm_loadu_ps(w11 + 4);
129 w11 += 8;
130 __m128 v12x0123 = _mm_loadu_ps(w12);
131 __m128 v12x4567 = _mm_loadu_ps(w12 + 4);
132 w12 += 8;
133 __m128 v13x0123 = _mm_loadu_ps(w13);
134 __m128 v13x4567 = _mm_loadu_ps(w13 + 4);
135 w13 += 8;
136 __m128 v14x0123 = _mm_loadu_ps(w14);
137 __m128 v14x4567 = _mm_loadu_ps(w14 + 4);
138 w14 += 8;
139 __m128 v15x0123 = _mm_loadu_ps(w15);
140 __m128 v15x4567 = _mm_loadu_ps(w15 + 4);
141 w15 += 8;
142
143 // Apply SR4 shuffle
144 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
145 v1x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x4567), _MM_SHUFFLE(0, 3, 2, 1)));
146 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
147 v2x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x4567), _MM_SHUFFLE(1, 0, 3, 2)));
148 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
149 v3x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x4567), _MM_SHUFFLE(2, 1, 0, 3)));
150 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
151 v5x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x4567), _MM_SHUFFLE(0, 3, 2, 1)));
152 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
153 v6x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x4567), _MM_SHUFFLE(1, 0, 3, 2)));
154 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
155 v7x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x4567), _MM_SHUFFLE(2, 1, 0, 3)));
156 v9x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x0123), _MM_SHUFFLE(0, 3, 2, 1)));
157 v9x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x4567), _MM_SHUFFLE(0, 3, 2, 1)));
158 v10x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x0123), _MM_SHUFFLE(1, 0, 3, 2)));
159 v10x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x4567), _MM_SHUFFLE(1, 0, 3, 2)));
160 v11x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x0123), _MM_SHUFFLE(2, 1, 0, 3)));
161 v11x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x4567), _MM_SHUFFLE(2, 1, 0, 3)));
162 v13x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x0123), _MM_SHUFFLE(0, 3, 2, 1)));
163 v13x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x4567), _MM_SHUFFLE(0, 3, 2, 1)));
164 v14x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x0123), _MM_SHUFFLE(1, 0, 3, 2)));
165 v14x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x4567), _MM_SHUFFLE(1, 0, 3, 2)));
166 v15x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15x0123), _MM_SHUFFLE(2, 1, 0, 3)));
167 v15x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15x4567), _MM_SHUFFLE(2, 1, 0, 3)));
168 // Transpose 2x2
169 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
170 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
171 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
172 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
173 const __m128 vtmp0x4567 = _mm_unpacklo_ps(v0x4567, v1x4567); // a e b f from row 0, 1
174 const __m128 vtmp1x4567 = _mm_unpacklo_ps(v2x4567, v3x4567); // i m j n from row 2, 3
175 const __m128 vtmp2x4567 = _mm_unpackhi_ps(v0x4567, v1x4567); // c g d h from row 0, 1
176 const __m128 vtmp3x4567 = _mm_unpackhi_ps(v2x4567, v3x4567); // k o l p from row 2, 3
177 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
178 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
179 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
180 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
181 const __m128 vtmp4x4567 = _mm_unpacklo_ps(v4x4567, v5x4567); // a e b f from row 0, 1
182 const __m128 vtmp5x4567 = _mm_unpacklo_ps(v6x4567, v7x4567); // i m j n from row 2, 3
183 const __m128 vtmp6x4567 = _mm_unpackhi_ps(v4x4567, v5x4567); // c g d h from row 0, 1
184 const __m128 vtmp7x4567 = _mm_unpackhi_ps(v6x4567, v7x4567); // k o l p from row 2, 3
185 const __m128 vtmp8x0123 = _mm_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
186 const __m128 vtmp9x0123 = _mm_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
187 const __m128 vtmp10x0123 = _mm_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
188 const __m128 vtmp11x0123 = _mm_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
189 const __m128 vtmp8x4567 = _mm_unpacklo_ps(v8x4567, v9x4567); // a e b f from row 0, 1
190 const __m128 vtmp9x4567 = _mm_unpacklo_ps(v10x4567, v11x4567); // i m j n from row 2, 3
191 const __m128 vtmp10x4567 = _mm_unpackhi_ps(v8x4567, v9x4567); // c g d h from row 0, 1
192 const __m128 vtmp11x4567 = _mm_unpackhi_ps(v10x4567, v11x4567); // k o l p from row 2, 3
193 const __m128 vtmp12x0123 = _mm_unpacklo_ps(v12x0123, v13x0123); // a e b f from row 0, 1
194 const __m128 vtmp13x0123 = _mm_unpacklo_ps(v14x0123, v15x0123); // i m j n from row 2, 3
195 const __m128 vtmp14x0123 = _mm_unpackhi_ps(v12x0123, v13x0123); // c g d h from row 0, 1
196 const __m128 vtmp15x0123 = _mm_unpackhi_ps(v14x0123, v15x0123); // k o l p from row 2, 3
197 const __m128 vtmp12x4567 = _mm_unpacklo_ps(v12x4567, v13x4567); // a e b f from row 0, 1
198 const __m128 vtmp13x4567 = _mm_unpacklo_ps(v14x4567, v15x4567); // i m j n from row 2, 3
199 const __m128 vtmp14x4567 = _mm_unpackhi_ps(v12x4567, v13x4567); // c g d h from row 0, 1
200 const __m128 vtmp15x4567 = _mm_unpackhi_ps(v14x4567, v15x4567); // k o l p from row 2, 3
201 // Transpose 4x4
202 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
203 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
204 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
205 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
206 v0x4567 = _mm_movelh_ps(vtmp0x4567, vtmp1x4567); // a e i m from row 0, 1
207 v1x4567 = _mm_movehl_ps(vtmp1x4567, vtmp0x4567); // b f j n from row 0, 1
208 v2x4567 = _mm_movelh_ps(vtmp2x4567, vtmp3x4567); // c g k o from row 2, 3
209 v3x4567 = _mm_movehl_ps(vtmp3x4567, vtmp2x4567); // d h l p from row 2, 3
210 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
211 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
212 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
213 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
214 v4x4567 = _mm_movelh_ps(vtmp4x4567, vtmp5x4567); // a e i m from row 0, 1
215 v5x4567 = _mm_movehl_ps(vtmp5x4567, vtmp4x4567); // b f j n from row 0, 1
216 v6x4567 = _mm_movelh_ps(vtmp6x4567, vtmp7x4567); // c g k o from row 2, 3
217 v7x4567 = _mm_movehl_ps(vtmp7x4567, vtmp6x4567); // d h l p from row 2, 3
218 v8x0123 = _mm_movelh_ps(vtmp8x0123, vtmp9x0123); // a e i m from row 0, 1
219 v9x0123 = _mm_movehl_ps(vtmp9x0123, vtmp8x0123); // b f j n from row 0, 1
220 v10x0123 = _mm_movelh_ps(vtmp10x0123, vtmp11x0123); // c g k o from row 2, 3
221 v11x0123 = _mm_movehl_ps(vtmp11x0123, vtmp10x0123); // d h l p from row 2, 3
222 v8x4567 = _mm_movelh_ps(vtmp8x4567, vtmp9x4567); // a e i m from row 0, 1
223 v9x4567 = _mm_movehl_ps(vtmp9x4567, vtmp8x4567); // b f j n from row 0, 1
224 v10x4567 = _mm_movelh_ps(vtmp10x4567, vtmp11x4567); // c g k o from row 2, 3
225 v11x4567 = _mm_movehl_ps(vtmp11x4567, vtmp10x4567); // d h l p from row 2, 3
226 v12x0123 = _mm_movelh_ps(vtmp12x0123, vtmp13x0123); // a e i m from row 0, 1
227 v13x0123 = _mm_movehl_ps(vtmp13x0123, vtmp12x0123); // b f j n from row 0, 1
228 v14x0123 = _mm_movelh_ps(vtmp14x0123, vtmp15x0123); // c g k o from row 2, 3
229 v15x0123 = _mm_movehl_ps(vtmp15x0123, vtmp14x0123); // d h l p from row 2, 3
230 v12x4567 = _mm_movelh_ps(vtmp12x4567, vtmp13x4567); // a e i m from row 0, 1
231 v13x4567 = _mm_movehl_ps(vtmp13x4567, vtmp12x4567); // b f j n from row 0, 1
232 v14x4567 = _mm_movelh_ps(vtmp14x4567, vtmp15x4567); // c g k o from row 2, 3
233 v15x4567 = _mm_movehl_ps(vtmp15x4567, vtmp14x4567); // d h l p from row 2, 3
234
235 _mm_store_ps(packed_w, v0x0123);
236 _mm_store_ps(packed_w + 4, v4x0123);
237 _mm_store_ps(packed_w + 8, v8x0123);
238 _mm_store_ps(packed_w + 12, v12x0123);
239 _mm_store_ps(packed_w + 16, v1x0123);
240 _mm_store_ps(packed_w + 20, v5x0123);
241 _mm_store_ps(packed_w + 24, v9x0123);
242 _mm_store_ps(packed_w + 28, v13x0123);
243 _mm_store_ps(packed_w + 32, v2x0123);
244 _mm_store_ps(packed_w + 36, v6x0123);
245 _mm_store_ps(packed_w + 40, v10x0123);
246 _mm_store_ps(packed_w + 44, v14x0123);
247 _mm_store_ps(packed_w + 48, v3x0123);
248 _mm_store_ps(packed_w + 52, v7x0123);
249 _mm_store_ps(packed_w + 56, v11x0123);
250 _mm_store_ps(packed_w + 60, v15x0123);
251 _mm_store_ps(packed_w + 64, v0x4567);
252 _mm_store_ps(packed_w + 68, v4x4567);
253 _mm_store_ps(packed_w + 72, v8x4567);
254 _mm_store_ps(packed_w + 76, v12x4567);
255 _mm_store_ps(packed_w + 80, v1x4567);
256 _mm_store_ps(packed_w + 84, v5x4567);
257 _mm_store_ps(packed_w + 88, v9x4567);
258 _mm_store_ps(packed_w + 92, v13x4567);
259 _mm_store_ps(packed_w + 96, v2x4567);
260 _mm_store_ps(packed_w + 100, v6x4567);
261 _mm_store_ps(packed_w + 104, v10x4567);
262 _mm_store_ps(packed_w + 108, v14x4567);
263 _mm_store_ps(packed_w + 112, v3x4567);
264 _mm_store_ps(packed_w + 116, v7x4567);
265 _mm_store_ps(packed_w + 120, v11x4567);
266 _mm_store_ps(packed_w + 124, v15x4567);
267 packed_w += 128;
268 }
269
270 // KC multiple of 4
271 for (; k >= 4; k -= 4) {
272 // Read blocks of 4x4
273 // a b c d
274 // e f g h
275 // i j k l
276 // m n o p
277 __m128 v0x0123 = _mm_loadu_ps(w0);
278 w0 += 4;
279 __m128 v1x0123 = _mm_loadu_ps(w1);
280 w1 += 4;
281 __m128 v2x0123 = _mm_loadu_ps(w2);
282 w2 += 4;
283 __m128 v3x0123 = _mm_loadu_ps(w3);
284 w3 += 4;
285 __m128 v4x0123 = _mm_loadu_ps(w4);
286 w4 += 4;
287 __m128 v5x0123 = _mm_loadu_ps(w5);
288 w5 += 4;
289 __m128 v6x0123 = _mm_loadu_ps(w6);
290 w6 += 4;
291 __m128 v7x0123 = _mm_loadu_ps(w7);
292 w7 += 4;
293 __m128 v8x0123 = _mm_loadu_ps(w8);
294 w8 += 4;
295 __m128 v9x0123 = _mm_loadu_ps(w9);
296 w9 += 4;
297 __m128 v10x0123 = _mm_loadu_ps(w10);
298 w10 += 4;
299 __m128 v11x0123 = _mm_loadu_ps(w11);
300 w11 += 4;
301 __m128 v12x0123 = _mm_loadu_ps(w12);
302 w12 += 4;
303 __m128 v13x0123 = _mm_loadu_ps(w13);
304 w13 += 4;
305 __m128 v14x0123 = _mm_loadu_ps(w14);
306 w14 += 4;
307 __m128 v15x0123 = _mm_loadu_ps(w15);
308 w15 += 4;
309
310 // Apply SR4 shuffle
311 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
312 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
313 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
314 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
315 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
316 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
317 v9x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x0123), _MM_SHUFFLE(0, 3, 2, 1)));
318 v10x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x0123), _MM_SHUFFLE(1, 0, 3, 2)));
319 v11x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x0123), _MM_SHUFFLE(2, 1, 0, 3)));
320 v13x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x0123), _MM_SHUFFLE(0, 3, 2, 1)));
321 v14x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x0123), _MM_SHUFFLE(1, 0, 3, 2)));
322 v15x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15x0123), _MM_SHUFFLE(2, 1, 0, 3)));
323
324 // Transpose 2x2
325 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
326 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
327 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
328 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
329 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
330 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
331 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
332 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
333 const __m128 vtmp8x0123 = _mm_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
334 const __m128 vtmp9x0123 = _mm_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
335 const __m128 vtmp10x0123 = _mm_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
336 const __m128 vtmp11x0123 = _mm_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
337 const __m128 vtmp12x0123 = _mm_unpacklo_ps(v12x0123, v13x0123); // a e b f from row 0, 1
338 const __m128 vtmp13x0123 = _mm_unpacklo_ps(v14x0123, v15x0123); // i m j n from row 2, 3
339 const __m128 vtmp14x0123 = _mm_unpackhi_ps(v12x0123, v13x0123); // c g d h from row 0, 1
340 const __m128 vtmp15x0123 = _mm_unpackhi_ps(v14x0123, v15x0123); // k o l p from row 2, 3
341 // Transpose 4x4
342 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
343 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
344 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
345 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
346 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
347 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
348 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
349 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
350 v8x0123 = _mm_movelh_ps(vtmp8x0123, vtmp9x0123); // a e i m from row 0, 1
351 v9x0123 = _mm_movehl_ps(vtmp9x0123, vtmp8x0123); // b f j n from row 0, 1
352 v10x0123 = _mm_movelh_ps(vtmp10x0123, vtmp11x0123); // c g k o from row 2, 3
353 v11x0123 = _mm_movehl_ps(vtmp11x0123, vtmp10x0123); // d h l p from row 2, 3
354 v12x0123 = _mm_movelh_ps(vtmp12x0123, vtmp13x0123); // a e i m from row 0, 1
355 v13x0123 = _mm_movehl_ps(vtmp13x0123, vtmp12x0123); // b f j n from row 0, 1
356 v14x0123 = _mm_movelh_ps(vtmp14x0123, vtmp15x0123); // c g k o from row 2, 3
357 v15x0123 = _mm_movehl_ps(vtmp15x0123, vtmp14x0123); // d h l p from row 2, 3
358
359 _mm_store_ps(packed_w, v0x0123);
360 _mm_store_ps(packed_w + 4, v4x0123);
361 _mm_store_ps(packed_w + 8, v8x0123);
362 _mm_store_ps(packed_w + 12, v12x0123);
363 _mm_store_ps(packed_w + 16, v1x0123);
364 _mm_store_ps(packed_w + 20, v5x0123);
365 _mm_store_ps(packed_w + 24, v9x0123);
366 _mm_store_ps(packed_w + 28, v13x0123);
367 _mm_store_ps(packed_w + 32, v2x0123);
368 _mm_store_ps(packed_w + 36, v6x0123);
369 _mm_store_ps(packed_w + 40, v10x0123);
370 _mm_store_ps(packed_w + 44, v14x0123);
371 _mm_store_ps(packed_w + 48, v3x0123);
372 _mm_store_ps(packed_w + 52, v7x0123);
373 _mm_store_ps(packed_w + 56, v11x0123);
374 _mm_store_ps(packed_w + 60, v15x0123);
375 packed_w += 64;
376 }
377
378 // KC remainder (1..3)
379 if XNN_UNLIKELY(k != 0) {
380 assert(k >= 1);
381 assert(k <= 3);
382 __m128 v0 = _mm_undefined_ps();
383 __m128 v1 = _mm_undefined_ps();
384 __m128 v2 = _mm_undefined_ps();
385 __m128 v3 = _mm_undefined_ps();
386 __m128 v4 = _mm_undefined_ps();
387 __m128 v5 = _mm_undefined_ps();
388 __m128 v6 = _mm_undefined_ps();
389 __m128 v7 = _mm_undefined_ps();
390 __m128 v8 = _mm_undefined_ps();
391 __m128 v9 = _mm_undefined_ps();
392 __m128 v10 = _mm_undefined_ps();
393 __m128 v11 = _mm_undefined_ps();
394 __m128 v12 = _mm_undefined_ps();
395 __m128 v13 = _mm_undefined_ps();
396 __m128 v14 = _mm_undefined_ps();
397 __m128 v15 = _mm_undefined_ps();
398
399 switch (k) {
400 case 1:
401 // Read blocks of 4x1
402 // a
403 // e
404 // i
405 // m
406 v0 = _mm_load_ss(w0);
407 w0 += 1;
408 v1 = _mm_load_ss(w1);
409 w1 += 1;
410 v2 = _mm_load_ss(w2);
411 w2 += 1;
412 v3 = _mm_load_ss(w3);
413 w3 += 1;
414 v4 = _mm_load_ss(w4);
415 w4 += 1;
416 v5 = _mm_load_ss(w5);
417 w5 += 1;
418 v6 = _mm_load_ss(w6);
419 w6 += 1;
420 v7 = _mm_load_ss(w7);
421 w7 += 1;
422 v8 = _mm_load_ss(w8);
423 w8 += 1;
424 v9 = _mm_load_ss(w9);
425 w9 += 1;
426 v10 = _mm_load_ss(w10);
427 w10 += 1;
428 v11 = _mm_load_ss(w11);
429 w11 += 1;
430 v12 = _mm_load_ss(w12);
431 w12 += 1;
432 v13 = _mm_load_ss(w13);
433 w13 += 1;
434 v14 = _mm_load_ss(w14);
435 w14 += 1;
436 v15 = _mm_load_ss(w15);
437 w15 += 1;
438 break;
439 case 2:
440 // Read blocks of 4x2
441 // a b
442 // e f
443 // i j
444 // m n
445 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
446 w0 += 2;
447 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
448 w1 += 2;
449 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
450 w2 += 2;
451 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
452 w3 += 2;
453 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
454 w4 += 2;
455 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
456 w5 += 2;
457 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
458 w6 += 2;
459 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
460 w7 += 2;
461 v8 = _mm_castpd_ps(_mm_load_sd((const double*) w8));
462 w8 += 2;
463 v9 = _mm_castpd_ps(_mm_load_sd((const double*) w9));
464 w9 += 2;
465 v10 = _mm_castpd_ps(_mm_load_sd((const double*) w10));
466 w10 += 2;
467 v11 = _mm_castpd_ps(_mm_load_sd((const double*) w11));
468 w11 += 2;
469 v12 = _mm_castpd_ps(_mm_load_sd((const double*) w12));
470 w12 += 2;
471 v13 = _mm_castpd_ps(_mm_load_sd((const double*) w13));
472 w13 += 2;
473 v14 = _mm_castpd_ps(_mm_load_sd((const double*) w14));
474 w14 += 2;
475 v15 = _mm_castpd_ps(_mm_load_sd((const double*) w15));
476 w15 += 2;
477 break;
478 case 3:
479 {
480 // Read blocks of 4x3
481 // a b c
482 // e f g
483 // i j k
484 // m n o
485 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
486 const __m128 v0hi = _mm_load_ss(w0 + 2);
487 v0 = _mm_movelh_ps(v0lo, v0hi);
488 w0 += 3;
489 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
490 const __m128 v1hi = _mm_load_ss(w1 + 2);
491 v1 = _mm_movelh_ps(v1lo, v1hi);
492 w1 += 3;
493 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
494 const __m128 v2hi = _mm_load_ss(w2 + 2);
495 v2 = _mm_movelh_ps(v2lo, v2hi);
496 w2 += 3;
497 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
498 const __m128 v3hi = _mm_load_ss(w3 + 2);
499 v3 = _mm_movelh_ps(v3lo, v3hi);
500 w3 += 3;
501 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
502 const __m128 v4hi = _mm_load_ss(w4 + 2);
503 v4 = _mm_movelh_ps(v4lo, v4hi);
504 w4 += 3;
505 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
506 const __m128 v5hi = _mm_load_ss(w5 + 2);
507 v5 = _mm_movelh_ps(v5lo, v5hi);
508 w5 += 3;
509 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
510 const __m128 v6hi = _mm_load_ss(w6 + 2);
511 v6 = _mm_movelh_ps(v6lo, v6hi);
512 w6 += 3;
513 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
514 const __m128 v7hi = _mm_load_ss(w7 + 2);
515 v7 = _mm_movelh_ps(v7lo, v7hi);
516 w7 += 3;
517 const __m128 v8lo = _mm_castpd_ps(_mm_load_sd((const double*) w8));
518 const __m128 v8hi = _mm_load_ss(w8 + 2);
519 v8 = _mm_movelh_ps(v8lo, v8hi);
520 w8 += 3;
521 const __m128 v9lo = _mm_castpd_ps(_mm_load_sd((const double*) w9));
522 const __m128 v9hi = _mm_load_ss(w9 + 2);
523 v9 = _mm_movelh_ps(v9lo, v9hi);
524 w9 += 3;
525 const __m128 v10lo = _mm_castpd_ps(_mm_load_sd((const double*) w10));
526 const __m128 v10hi = _mm_load_ss(w10 + 2);
527 v10 = _mm_movelh_ps(v10lo, v10hi);
528 w10 += 3;
529 const __m128 v11lo = _mm_castpd_ps(_mm_load_sd((const double*) w11));
530 const __m128 v11hi = _mm_load_ss(w11 + 2);
531 v11 = _mm_movelh_ps(v11lo, v11hi);
532 w11 += 3;
533 const __m128 v12lo = _mm_castpd_ps(_mm_load_sd((const double*) w12));
534 const __m128 v12hi = _mm_load_ss(w12 + 2);
535 v12 = _mm_movelh_ps(v12lo, v12hi);
536 w12 += 3;
537 const __m128 v13lo = _mm_castpd_ps(_mm_load_sd((const double*) w13));
538 const __m128 v13hi = _mm_load_ss(w13 + 2);
539 v13 = _mm_movelh_ps(v13lo, v13hi);
540 w13 += 3;
541 const __m128 v14lo = _mm_castpd_ps(_mm_load_sd((const double*) w14));
542 const __m128 v14hi = _mm_load_ss(w14 + 2);
543 v14 = _mm_movelh_ps(v14lo, v14hi);
544 w14 += 3;
545 const __m128 v15lo = _mm_castpd_ps(_mm_load_sd((const double*) w15));
546 const __m128 v15hi = _mm_load_ss(w15 + 2);
547 v15 = _mm_movelh_ps(v15lo, v15hi);
548 w15 += 3;
549 break;
550 }
551 default:
552 XNN_UNREACHABLE;
553 }
554
555 // Apply SR4 shuffle
556 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
557 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
558 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
559 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
560 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
561 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
562 v9 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9), _MM_SHUFFLE(0, 3, 2, 1)));
563 v10 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10), _MM_SHUFFLE(1, 0, 3, 2)));
564 v11 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11), _MM_SHUFFLE(2, 1, 0, 3)));
565 v13 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13), _MM_SHUFFLE(0, 3, 2, 1)));
566 v14 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14), _MM_SHUFFLE(1, 0, 3, 2)));
567 v15 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15), _MM_SHUFFLE(2, 1, 0, 3)));
568 // Transpose 2x2
569 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
570 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
571 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
572 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
573 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
574 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
575 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
576 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
577 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a e b f from row 0, 1
578 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // i m j n from row 2, 3
579 const __m128 vtmp10 = _mm_unpackhi_ps(v8, v9); // c g d h from row 0, 1
580 const __m128 vtmp11 = _mm_unpackhi_ps(v10, v11); // k o l p from row 2, 3
581 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a e b f from row 0, 1
582 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // i m j n from row 2, 3
583 const __m128 vtmp14 = _mm_unpackhi_ps(v12, v13); // c g d h from row 0, 1
584 const __m128 vtmp15 = _mm_unpackhi_ps(v14, v15); // k o l p from row 2, 3
585 // Transpose 4x4
586 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
587 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
588 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
589 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
590 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
591 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
592 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
593 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
594 v8 = _mm_movelh_ps(vtmp8, vtmp9); // a e i m from row 0, 1
595 v9 = _mm_movehl_ps(vtmp9, vtmp8); // b f j n from row 0, 1
596 v10 = _mm_movelh_ps(vtmp10, vtmp11); // c g k o from row 2, 3
597 v11 = _mm_movehl_ps(vtmp11, vtmp10); // d h l p from row 2, 3
598 v12 = _mm_movelh_ps(vtmp12, vtmp13); // a e i m from row 0, 1
599 v13 = _mm_movehl_ps(vtmp13, vtmp12); // b f j n from row 0, 1
600 v14 = _mm_movelh_ps(vtmp14, vtmp15); // c g k o from row 2, 3
601 v15 = _mm_movehl_ps(vtmp15, vtmp14); // d h l p from row 2, 3
602 _mm_store_ps(packed_w, v0);
603 _mm_store_ps(packed_w + 4, v4);
604 _mm_store_ps(packed_w + 8, v8);
605 _mm_store_ps(packed_w + 12, v12);
606 _mm_store_ps(packed_w + 16, v1);
607 _mm_store_ps(packed_w + 20, v5);
608 _mm_store_ps(packed_w + 24, v9);
609 _mm_store_ps(packed_w + 28, v13);
610 _mm_store_ps(packed_w + 32, v2);
611 _mm_store_ps(packed_w + 36, v6);
612 _mm_store_ps(packed_w + 40, v10);
613 _mm_store_ps(packed_w + 44, v14);
614 _mm_store_ps(packed_w + 48, v3);
615 _mm_store_ps(packed_w + 52, v7);
616 _mm_store_ps(packed_w + 56, v11);
617 _mm_store_ps(packed_w + 60, v15);
618 packed_w += 64;
619 }
620 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
621 w0 = w15;
622 }
623
624 // NC remainder (1..15)
625 if XNN_UNLIKELY(n != 0) {
626 assert(n >= 1);
627 assert(n <= 15);
628 if XNN_LIKELY(b != NULL) {
629 size_t nb = n;
630 do {
631 *packed_w++ = *b++;
632 } while (--nb != 0);
633 packed_w += (16 - n);
634 } else {
635 const __m128 vzero = _mm_setzero_ps();
636 _mm_store_ps(packed_w, vzero);
637 _mm_store_ps(packed_w + 4, vzero);
638 _mm_store_ps(packed_w + 8, vzero);
639 _mm_store_ps(packed_w + 12, vzero);
640 packed_w += 16;
641 }
642
643 // NR remainder has less than 16 rows so last row is not loaded
644 const float* w1 = w0 + kc;
645 if XNN_UNPREDICTABLE(n < 2) {
646 w1 = w0;
647 }
648 const float* w2 = w1 + kc;
649 if XNN_UNPREDICTABLE(n <= 2) {
650 w2 = w1;
651 }
652 const float* w3 = w2 + kc;
653 if XNN_UNPREDICTABLE(n < 4) {
654 w3 = w2;
655 }
656 const float* w4 = w3 + kc;
657 if XNN_UNPREDICTABLE(n <= 4) {
658 w4 = w3;
659 }
660 const float* w5 = w4 + kc;
661 if XNN_UNPREDICTABLE(n < 6) {
662 w5 = w4;
663 }
664 const float* w6 = w5 + kc;
665 if XNN_UNPREDICTABLE(n <= 6) {
666 w6 = w5;
667 }
668 const float* w7 = w6 + kc;
669 if XNN_UNPREDICTABLE(n < 8) {
670 w7 = w6;
671 }
672 const float* w8 = w7 + kc;
673 if XNN_UNPREDICTABLE(n <= 8) {
674 w8 = w7;
675 }
676 const float* w9 = w8 + kc;
677 if XNN_UNPREDICTABLE(n < 10) {
678 w9 = w8;
679 }
680 const float* w10 = w9 + kc;
681 if XNN_UNPREDICTABLE(n <= 10) {
682 w10 = w9;
683 }
684 const float* w11 = w10 + kc;
685 if XNN_UNPREDICTABLE(n < 12) {
686 w11 = w10;
687 }
688 const float* w12 = w11 + kc;
689 if XNN_UNPREDICTABLE(n <= 12) {
690 w12 = w11;
691 }
692 const float* w13 = w12 + kc;
693 if XNN_UNPREDICTABLE(n < 14) {
694 w13 = w12;
695 }
696 const float* w14 = w13 + kc;
697 if XNN_UNPREDICTABLE(n <= 14) {
698 w14 = w13;
699 }
700
701 size_t k = kc;
702 // KC main loop multiple of 8
703 for (; k >= 8; k -= 8) {
704 // Read blocks of 4x4
705 // a b c d
706 // e f g h
707 // i j k l
708 // m n o p
709 __m128 v0x0123 = _mm_loadu_ps(w0);
710 __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
711 w0 += 8;
712 __m128 v1x0123 = _mm_loadu_ps(w1);
713 __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
714 w1 += 8;
715 __m128 v2x0123 = _mm_loadu_ps(w2);
716 __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
717 w2 += 8;
718 __m128 v3x0123 = _mm_loadu_ps(w3);
719 __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
720 w3 += 8;
721 __m128 v4x0123 = _mm_loadu_ps(w4);
722 __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
723 w4 += 8;
724 __m128 v5x0123 = _mm_loadu_ps(w5);
725 __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
726 w5 += 8;
727 __m128 v6x0123 = _mm_loadu_ps(w6);
728 __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
729 w6 += 8;
730 __m128 v7x0123 = _mm_loadu_ps(w7);
731 __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
732 w7 += 8;
733 __m128 v8x0123 = _mm_loadu_ps(w8);
734 __m128 v8x4567 = _mm_loadu_ps(w8 + 4);
735 w8 += 8;
736 __m128 v9x0123 = _mm_loadu_ps(w9);
737 __m128 v9x4567 = _mm_loadu_ps(w9 + 4);
738 w9 += 8;
739 __m128 v10x0123 = _mm_loadu_ps(w10);
740 __m128 v10x4567 = _mm_loadu_ps(w10 + 4);
741 w10 += 8;
742 __m128 v11x0123 = _mm_loadu_ps(w11);
743 __m128 v11x4567 = _mm_loadu_ps(w11 + 4);
744 w11 += 8;
745 __m128 v12x0123 = _mm_loadu_ps(w12);
746 __m128 v12x4567 = _mm_loadu_ps(w12 + 4);
747 w12 += 8;
748 __m128 v13x0123 = _mm_loadu_ps(w13);
749 __m128 v13x4567 = _mm_loadu_ps(w13 + 4);
750 w13 += 8;
751 __m128 v14x0123 = _mm_loadu_ps(w14);
752 __m128 v14x4567 = _mm_loadu_ps(w14 + 4);
753 w14 += 8;
754
755 __m128 v15x0123 = _mm_undefined_ps();
756 __m128 v15x4567 = _mm_undefined_ps();
757
758 // Apply SR4 shuffle
759 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
760 v1x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x4567), _MM_SHUFFLE(0, 3, 2, 1)));
761 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
762 v2x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x4567), _MM_SHUFFLE(1, 0, 3, 2)));
763 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
764 v3x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x4567), _MM_SHUFFLE(2, 1, 0, 3)));
765 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
766 v5x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x4567), _MM_SHUFFLE(0, 3, 2, 1)));
767 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
768 v6x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x4567), _MM_SHUFFLE(1, 0, 3, 2)));
769 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
770 v7x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x4567), _MM_SHUFFLE(2, 1, 0, 3)));
771 v9x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x0123), _MM_SHUFFLE(0, 3, 2, 1)));
772 v9x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x4567), _MM_SHUFFLE(0, 3, 2, 1)));
773 v10x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x0123), _MM_SHUFFLE(1, 0, 3, 2)));
774 v10x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x4567), _MM_SHUFFLE(1, 0, 3, 2)));
775 v11x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x0123), _MM_SHUFFLE(2, 1, 0, 3)));
776 v11x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x4567), _MM_SHUFFLE(2, 1, 0, 3)));
777 v13x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x0123), _MM_SHUFFLE(0, 3, 2, 1)));
778 v13x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x4567), _MM_SHUFFLE(0, 3, 2, 1)));
779 v14x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x0123), _MM_SHUFFLE(1, 0, 3, 2)));
780 v14x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x4567), _MM_SHUFFLE(1, 0, 3, 2)));
781
782 // Transpose 2x2
783 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
784 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
785 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
786 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
787 const __m128 vtmp0x4567 = _mm_unpacklo_ps(v0x4567, v1x4567); // a e b f from row 0, 1
788 const __m128 vtmp1x4567 = _mm_unpacklo_ps(v2x4567, v3x4567); // i m j n from row 2, 3
789 const __m128 vtmp2x4567 = _mm_unpackhi_ps(v0x4567, v1x4567); // c g d h from row 0, 1
790 const __m128 vtmp3x4567 = _mm_unpackhi_ps(v2x4567, v3x4567); // k o l p from row 2, 3
791 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
792 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
793 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
794 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
795 const __m128 vtmp4x4567 = _mm_unpacklo_ps(v4x4567, v5x4567); // a e b f from row 0, 1
796 const __m128 vtmp5x4567 = _mm_unpacklo_ps(v6x4567, v7x4567); // i m j n from row 2, 3
797 const __m128 vtmp6x4567 = _mm_unpackhi_ps(v4x4567, v5x4567); // c g d h from row 0, 1
798 const __m128 vtmp7x4567 = _mm_unpackhi_ps(v6x4567, v7x4567); // k o l p from row 2, 3
799 const __m128 vtmp8x0123 = _mm_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
800 const __m128 vtmp9x0123 = _mm_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
801 const __m128 vtmp10x0123 = _mm_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
802 const __m128 vtmp11x0123 = _mm_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
803 const __m128 vtmp8x4567 = _mm_unpacklo_ps(v8x4567, v9x4567); // a e b f from row 0, 1
804 const __m128 vtmp9x4567 = _mm_unpacklo_ps(v10x4567, v11x4567); // i m j n from row 2, 3
805 const __m128 vtmp10x4567 = _mm_unpackhi_ps(v8x4567, v9x4567); // c g d h from row 0, 1
806 const __m128 vtmp11x4567 = _mm_unpackhi_ps(v10x4567, v11x4567); // k o l p from row 2, 3
807 const __m128 vtmp12x0123 = _mm_unpacklo_ps(v12x0123, v13x0123); // a e b f from row 0, 1
808 const __m128 vtmp13x0123 = _mm_unpacklo_ps(v14x0123, v14x0123); // i m j n from row 2, 3
809 const __m128 vtmp14x0123 = _mm_unpackhi_ps(v12x0123, v13x0123); // c g d h from row 0, 1
810 const __m128 vtmp15x0123 = _mm_unpackhi_ps(v14x0123, v14x0123); // k o l p from row 2, 3
811 const __m128 vtmp12x4567 = _mm_unpacklo_ps(v12x4567, v13x4567); // a e b f from row 0, 1
812 const __m128 vtmp13x4567 = _mm_unpacklo_ps(v14x4567, v14x4567); // i m j n from row 2, 3
813 const __m128 vtmp14x4567 = _mm_unpackhi_ps(v12x4567, v13x4567); // c g d h from row 0, 1
814 const __m128 vtmp15x4567 = _mm_unpackhi_ps(v14x4567, v14x4567); // k o l p from row 2, 3
815 // Transpose 4x4
816 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
817 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
818 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
819 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
820 v0x4567 = _mm_movelh_ps(vtmp0x4567, vtmp1x4567); // a e i m from row 0, 1
821 v1x4567 = _mm_movehl_ps(vtmp1x4567, vtmp0x4567); // b f j n from row 0, 1
822 v2x4567 = _mm_movelh_ps(vtmp2x4567, vtmp3x4567); // c g k o from row 2, 3
823 v3x4567 = _mm_movehl_ps(vtmp3x4567, vtmp2x4567); // d h l p from row 2, 3
824 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
825 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
826 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
827 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
828 v4x4567 = _mm_movelh_ps(vtmp4x4567, vtmp5x4567); // a e i m from row 0, 1
829 v5x4567 = _mm_movehl_ps(vtmp5x4567, vtmp4x4567); // b f j n from row 0, 1
830 v6x4567 = _mm_movelh_ps(vtmp6x4567, vtmp7x4567); // c g k o from row 2, 3
831 v7x4567 = _mm_movehl_ps(vtmp7x4567, vtmp6x4567); // d h l p from row 2, 3
832 v8x0123 = _mm_movelh_ps(vtmp8x0123, vtmp9x0123); // a e i m from row 0, 1
833 v9x0123 = _mm_movehl_ps(vtmp9x0123, vtmp8x0123); // b f j n from row 0, 1
834 v10x0123 = _mm_movelh_ps(vtmp10x0123, vtmp11x0123); // c g k o from row 2, 3
835 v11x0123 = _mm_movehl_ps(vtmp11x0123, vtmp10x0123); // d h l p from row 2, 3
836 v8x4567 = _mm_movelh_ps(vtmp8x4567, vtmp9x4567); // a e i m from row 0, 1
837 v9x4567 = _mm_movehl_ps(vtmp9x4567, vtmp8x4567); // b f j n from row 0, 1
838 v10x4567 = _mm_movelh_ps(vtmp10x4567, vtmp11x4567); // c g k o from row 2, 3
839 v11x4567 = _mm_movehl_ps(vtmp11x4567, vtmp10x4567); // d h l p from row 2, 3
840 v12x0123 = _mm_movelh_ps(vtmp12x0123, vtmp13x0123); // a e i m from row 0, 1
841 v13x0123 = _mm_movehl_ps(vtmp13x0123, vtmp12x0123); // b f j n from row 0, 1
842 v14x0123 = _mm_movelh_ps(vtmp14x0123, vtmp15x0123); // c g k o from row 2, 3
843 v15x0123 = _mm_movehl_ps(vtmp15x0123, vtmp14x0123); // d h l p from row 2, 3
844 v12x4567 = _mm_movelh_ps(vtmp12x4567, vtmp13x4567); // a e i m from row 0, 1
845 v13x4567 = _mm_movehl_ps(vtmp13x4567, vtmp12x4567); // b f j n from row 0, 1
846 v14x4567 = _mm_movelh_ps(vtmp14x4567, vtmp15x4567); // c g k o from row 2, 3
847 v15x4567 = _mm_movehl_ps(vtmp15x4567, vtmp14x4567); // d h l p from row 2, 3
848
849 _mm_store_ps(packed_w, v0x0123);
850 _mm_store_ps(packed_w + 4, v4x0123);
851 _mm_store_ps(packed_w + 8, v8x0123);
852 _mm_store_ps(packed_w + 12, v12x0123);
853 _mm_store_ps(packed_w + 16, v1x0123);
854 _mm_store_ps(packed_w + 20, v5x0123);
855 _mm_store_ps(packed_w + 24, v9x0123);
856 _mm_store_ps(packed_w + 28, v13x0123);
857 _mm_store_ps(packed_w + 32, v2x0123);
858 _mm_store_ps(packed_w + 36, v6x0123);
859 _mm_store_ps(packed_w + 40, v10x0123);
860 _mm_store_ps(packed_w + 44, v14x0123);
861 _mm_store_ps(packed_w + 48, v3x0123);
862 _mm_store_ps(packed_w + 52, v7x0123);
863 _mm_store_ps(packed_w + 56, v11x0123);
864 _mm_store_ps(packed_w + 60, v15x0123);
865 _mm_store_ps(packed_w + 64, v0x4567);
866 _mm_store_ps(packed_w + 68, v4x4567);
867 _mm_store_ps(packed_w + 72, v8x4567);
868 _mm_store_ps(packed_w + 76, v12x4567);
869 _mm_store_ps(packed_w + 80, v1x4567);
870 _mm_store_ps(packed_w + 84, v5x4567);
871 _mm_store_ps(packed_w + 88, v9x4567);
872 _mm_store_ps(packed_w + 92, v13x4567);
873 _mm_store_ps(packed_w + 96, v2x4567);
874 _mm_store_ps(packed_w + 100, v6x4567);
875 _mm_store_ps(packed_w + 104, v10x4567);
876 _mm_store_ps(packed_w + 108, v14x4567);
877 _mm_store_ps(packed_w + 112, v3x4567);
878 _mm_store_ps(packed_w + 116, v7x4567);
879 _mm_store_ps(packed_w + 120, v11x4567);
880 _mm_store_ps(packed_w + 124, v15x4567);
881 packed_w += 128;
882 }
883
884 // KC multiple of 4
885 for (; k >= 4; k -= 4) {
886 // Read blocks of 4x4
887 // a b c d
888 // e f g h
889 // i j k l
890 // m n o p
891 __m128 v0x0123 = _mm_loadu_ps(w0);
892 w0 += 4;
893 __m128 v1x0123 = _mm_loadu_ps(w1);
894 w1 += 4;
895 __m128 v2x0123 = _mm_loadu_ps(w2);
896 w2 += 4;
897 __m128 v3x0123 = _mm_loadu_ps(w3);
898 w3 += 4;
899 __m128 v4x0123 = _mm_loadu_ps(w4);
900 w4 += 4;
901 __m128 v5x0123 = _mm_loadu_ps(w5);
902 w5 += 4;
903 __m128 v6x0123 = _mm_loadu_ps(w6);
904 w6 += 4;
905 __m128 v7x0123 = _mm_loadu_ps(w7);
906 w7 += 4;
907 __m128 v8x0123 = _mm_loadu_ps(w8);
908 w8 += 4;
909 __m128 v9x0123 = _mm_loadu_ps(w9);
910 w9 += 4;
911 __m128 v10x0123 = _mm_loadu_ps(w10);
912 w10 += 4;
913 __m128 v11x0123 = _mm_loadu_ps(w11);
914 w11 += 4;
915 __m128 v12x0123 = _mm_loadu_ps(w12);
916 w12 += 4;
917 __m128 v13x0123 = _mm_loadu_ps(w13);
918 w13 += 4;
919 __m128 v14x0123 = _mm_loadu_ps(w14);
920 w14 += 4;
921 __m128 v15x0123 = _mm_undefined_ps();
922
923 // Apply SR4 shuffle
924 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
925 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
926 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
927 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
928 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
929 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
930 v9x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x0123), _MM_SHUFFLE(0, 3, 2, 1)));
931 v10x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x0123), _MM_SHUFFLE(1, 0, 3, 2)));
932 v11x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x0123), _MM_SHUFFLE(2, 1, 0, 3)));
933 v13x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x0123), _MM_SHUFFLE(0, 3, 2, 1)));
934 v14x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x0123), _MM_SHUFFLE(1, 0, 3, 2)));
935
936 // Transpose 2x2
937 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
938 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
939 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
940 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
941 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
942 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
943 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
944 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
945 const __m128 vtmp8x0123 = _mm_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
946 const __m128 vtmp9x0123 = _mm_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
947 const __m128 vtmp10x0123 = _mm_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
948 const __m128 vtmp11x0123 = _mm_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
949 const __m128 vtmp12x0123 = _mm_unpacklo_ps(v12x0123, v13x0123); // a e b f from row 0, 1
950 const __m128 vtmp13x0123 = _mm_unpacklo_ps(v14x0123, v14x0123); // i m j n from row 2, 3
951 const __m128 vtmp14x0123 = _mm_unpackhi_ps(v12x0123, v13x0123); // c g d h from row 0, 1
952 const __m128 vtmp15x0123 = _mm_unpackhi_ps(v14x0123, v14x0123); // k o l p from row 2, 3
953 // Transpose 4x4
954 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
955 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
956 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
957 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
958 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
959 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
960 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
961 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
962 v8x0123 = _mm_movelh_ps(vtmp8x0123, vtmp9x0123); // a e i m from row 0, 1
963 v9x0123 = _mm_movehl_ps(vtmp9x0123, vtmp8x0123); // b f j n from row 0, 1
964 v10x0123 = _mm_movelh_ps(vtmp10x0123, vtmp11x0123); // c g k o from row 2, 3
965 v11x0123 = _mm_movehl_ps(vtmp11x0123, vtmp10x0123); // d h l p from row 2, 3
966 v12x0123 = _mm_movelh_ps(vtmp12x0123, vtmp13x0123); // a e i m from row 0, 1
967 v13x0123 = _mm_movehl_ps(vtmp13x0123, vtmp12x0123); // b f j n from row 0, 1
968 v14x0123 = _mm_movelh_ps(vtmp14x0123, vtmp15x0123); // c g k o from row 2, 3
969 v15x0123 = _mm_movehl_ps(vtmp15x0123, vtmp14x0123); // d h l p from row 2, 3
970
971 _mm_store_ps(packed_w, v0x0123);
972 _mm_store_ps(packed_w + 4, v4x0123);
973 _mm_store_ps(packed_w + 8, v8x0123);
974 _mm_store_ps(packed_w + 12, v12x0123);
975 _mm_store_ps(packed_w + 16, v1x0123);
976 _mm_store_ps(packed_w + 20, v5x0123);
977 _mm_store_ps(packed_w + 24, v9x0123);
978 _mm_store_ps(packed_w + 28, v13x0123);
979 _mm_store_ps(packed_w + 32, v2x0123);
980 _mm_store_ps(packed_w + 36, v6x0123);
981 _mm_store_ps(packed_w + 40, v10x0123);
982 _mm_store_ps(packed_w + 44, v14x0123);
983 _mm_store_ps(packed_w + 48, v3x0123);
984 _mm_store_ps(packed_w + 52, v7x0123);
985 _mm_store_ps(packed_w + 56, v11x0123);
986 _mm_store_ps(packed_w + 60, v15x0123);
987 packed_w += 64;
988 }
989
990 // KC remainder (1..3)
991 if XNN_UNLIKELY(k != 0) {
992 assert(k >= 1);
993 assert(k <= 3);
994 __m128 v0 = _mm_undefined_ps();
995 __m128 v1 = _mm_undefined_ps();
996 __m128 v2 = _mm_undefined_ps();
997 __m128 v3 = _mm_undefined_ps();
998 __m128 v4 = _mm_undefined_ps();
999 __m128 v5 = _mm_undefined_ps();
1000 __m128 v6 = _mm_undefined_ps();
1001 __m128 v7 = _mm_undefined_ps();
1002 __m128 v8 = _mm_undefined_ps();
1003 __m128 v9 = _mm_undefined_ps();
1004 __m128 v10 = _mm_undefined_ps();
1005 __m128 v11 = _mm_undefined_ps();
1006 __m128 v12 = _mm_undefined_ps();
1007 __m128 v13 = _mm_undefined_ps();
1008 __m128 v14 = _mm_undefined_ps();
1009 __m128 v15 = _mm_undefined_ps();
1010
1011 switch (k) {
1012 case 1:
1013 // Read blocks of 4x1
1014 // a
1015 // e
1016 // i
1017 // m
1018 v0 = _mm_load_ss(w0);
1019 w0 += 1;
1020 v1 = _mm_load_ss(w1);
1021 w1 += 1;
1022 v2 = _mm_load_ss(w2);
1023 w2 += 1;
1024 v3 = _mm_load_ss(w3);
1025 w3 += 1;
1026 v4 = _mm_load_ss(w4);
1027 w4 += 1;
1028 v5 = _mm_load_ss(w5);
1029 w5 += 1;
1030 v6 = _mm_load_ss(w6);
1031 w6 += 1;
1032 v7 = _mm_load_ss(w7);
1033 w7 += 1;
1034 v8 = _mm_load_ss(w8);
1035 w8 += 1;
1036 v9 = _mm_load_ss(w9);
1037 w9 += 1;
1038 v10 = _mm_load_ss(w10);
1039 w10 += 1;
1040 v11 = _mm_load_ss(w11);
1041 w11 += 1;
1042 v12 = _mm_load_ss(w12);
1043 w12 += 1;
1044 v13 = _mm_load_ss(w13);
1045 w13 += 1;
1046 v14 = _mm_load_ss(w14);
1047 w14 += 1;
1048 break;
1049 case 2:
1050 // Read blocks of 4x2
1051 // a b
1052 // e f
1053 // i j
1054 // m n
1055 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
1056 w0 += 2;
1057 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
1058 w1 += 2;
1059 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
1060 w2 += 2;
1061 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
1062 w3 += 2;
1063 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
1064 w4 += 2;
1065 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
1066 w5 += 2;
1067 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
1068 w6 += 2;
1069 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
1070 w7 += 2;
1071 v8 = _mm_castpd_ps(_mm_load_sd((const double*) w8));
1072 w8 += 2;
1073 v9 = _mm_castpd_ps(_mm_load_sd((const double*) w9));
1074 w9 += 2;
1075 v10 = _mm_castpd_ps(_mm_load_sd((const double*) w10));
1076 w10 += 2;
1077 v11 = _mm_castpd_ps(_mm_load_sd((const double*) w11));
1078 w11 += 2;
1079 v12 = _mm_castpd_ps(_mm_load_sd((const double*) w12));
1080 w12 += 2;
1081 v13 = _mm_castpd_ps(_mm_load_sd((const double*) w13));
1082 w13 += 2;
1083 v14 = _mm_castpd_ps(_mm_load_sd((const double*) w14));
1084 w14 += 2;
1085 break;
1086 case 3:
1087 {
1088 // Read blocks of 4x3
1089 // a b c
1090 // e f g
1091 // i j k
1092 // m n o
1093 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
1094 const __m128 v0hi = _mm_load_ss(w0 + 2);
1095 v0 = _mm_movelh_ps(v0lo, v0hi);
1096 w0 += 3;
1097 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
1098 const __m128 v1hi = _mm_load_ss(w1 + 2);
1099 v1 = _mm_movelh_ps(v1lo, v1hi);
1100 w1 += 3;
1101 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
1102 const __m128 v2hi = _mm_load_ss(w2 + 2);
1103 v2 = _mm_movelh_ps(v2lo, v2hi);
1104 w2 += 3;
1105 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
1106 const __m128 v3hi = _mm_load_ss(w3 + 2);
1107 v3 = _mm_movelh_ps(v3lo, v3hi);
1108 w3 += 3;
1109 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
1110 const __m128 v4hi = _mm_load_ss(w4 + 2);
1111 v4 = _mm_movelh_ps(v4lo, v4hi);
1112 w4 += 3;
1113 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
1114 const __m128 v5hi = _mm_load_ss(w5 + 2);
1115 v5 = _mm_movelh_ps(v5lo, v5hi);
1116 w5 += 3;
1117 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
1118 const __m128 v6hi = _mm_load_ss(w6 + 2);
1119 v6 = _mm_movelh_ps(v6lo, v6hi);
1120 w6 += 3;
1121 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
1122 const __m128 v7hi = _mm_load_ss(w7 + 2);
1123 v7 = _mm_movelh_ps(v7lo, v7hi);
1124 w7 += 3;
1125 const __m128 v8lo = _mm_castpd_ps(_mm_load_sd((const double*) w8));
1126 const __m128 v8hi = _mm_load_ss(w8 + 2);
1127 v8 = _mm_movelh_ps(v8lo, v8hi);
1128 w8 += 3;
1129 const __m128 v9lo = _mm_castpd_ps(_mm_load_sd((const double*) w9));
1130 const __m128 v9hi = _mm_load_ss(w9 + 2);
1131 v9 = _mm_movelh_ps(v9lo, v9hi);
1132 w9 += 3;
1133 const __m128 v10lo = _mm_castpd_ps(_mm_load_sd((const double*) w10));
1134 const __m128 v10hi = _mm_load_ss(w10 + 2);
1135 v10 = _mm_movelh_ps(v10lo, v10hi);
1136 w10 += 3;
1137 const __m128 v11lo = _mm_castpd_ps(_mm_load_sd((const double*) w11));
1138 const __m128 v11hi = _mm_load_ss(w11 + 2);
1139 v11 = _mm_movelh_ps(v11lo, v11hi);
1140 w11 += 3;
1141 const __m128 v12lo = _mm_castpd_ps(_mm_load_sd((const double*) w12));
1142 const __m128 v12hi = _mm_load_ss(w12 + 2);
1143 v12 = _mm_movelh_ps(v12lo, v12hi);
1144 w12 += 3;
1145 const __m128 v13lo = _mm_castpd_ps(_mm_load_sd((const double*) w13));
1146 const __m128 v13hi = _mm_load_ss(w13 + 2);
1147 v13 = _mm_movelh_ps(v13lo, v13hi);
1148 w13 += 3;
1149 const __m128 v14lo = _mm_castpd_ps(_mm_load_sd((const double*) w14));
1150 const __m128 v14hi = _mm_load_ss(w14 + 2);
1151 v14 = _mm_movelh_ps(v14lo, v14hi);
1152 w14 += 3;
1153 break;
1154 }
1155 default:
1156 XNN_UNREACHABLE;
1157 }
1158
1159 // Apply SR4 shuffle
1160 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
1161 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
1162 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
1163 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
1164 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
1165 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
1166 v9 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9), _MM_SHUFFLE(0, 3, 2, 1)));
1167 v10 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10), _MM_SHUFFLE(1, 0, 3, 2)));
1168 v11 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11), _MM_SHUFFLE(2, 1, 0, 3)));
1169 v13 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13), _MM_SHUFFLE(0, 3, 2, 1)));
1170 v14 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14), _MM_SHUFFLE(1, 0, 3, 2)));
1171 // Transpose 2x2
1172 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
1173 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
1174 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
1175 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
1176 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
1177 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
1178 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
1179 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
1180 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a e b f from row 0, 1
1181 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // i m j n from row 2, 3
1182 const __m128 vtmp10 = _mm_unpackhi_ps(v8, v9); // c g d h from row 0, 1
1183 const __m128 vtmp11 = _mm_unpackhi_ps(v10, v11); // k o l p from row 2, 3
1184 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a e b f from row 0, 1
1185 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // i m j n from row 2, 3
1186 const __m128 vtmp14 = _mm_unpackhi_ps(v12, v13); // c g d h from row 0, 1
1187 const __m128 vtmp15 = _mm_unpackhi_ps(v14, v14); // k o l p from row 2, 3
1188 // Transpose 4x4
1189 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
1190 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
1191 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
1192 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
1193 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
1194 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
1195 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
1196 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
1197 v8 = _mm_movelh_ps(vtmp8, vtmp9); // a e i m from row 0, 1
1198 v9 = _mm_movehl_ps(vtmp9, vtmp8); // b f j n from row 0, 1
1199 v10 = _mm_movelh_ps(vtmp10, vtmp11); // c g k o from row 2, 3
1200 v11 = _mm_movehl_ps(vtmp11, vtmp10); // d h l p from row 2, 3
1201 v12 = _mm_movelh_ps(vtmp12, vtmp13); // a e i m from row 0, 1
1202 v13 = _mm_movehl_ps(vtmp13, vtmp12); // b f j n from row 0, 1
1203 v14 = _mm_movelh_ps(vtmp14, vtmp15); // c g k o from row 2, 3
1204 v15 = _mm_movehl_ps(vtmp15, vtmp14); // d h l p from row 2, 3
1205 _mm_store_ps(packed_w, v0);
1206 _mm_store_ps(packed_w + 4, v4);
1207 _mm_store_ps(packed_w + 8, v8);
1208 _mm_store_ps(packed_w + 12, v12);
1209 _mm_store_ps(packed_w + 16, v1);
1210 _mm_store_ps(packed_w + 20, v5);
1211 _mm_store_ps(packed_w + 24, v9);
1212 _mm_store_ps(packed_w + 28, v13);
1213 _mm_store_ps(packed_w + 32, v2);
1214 _mm_store_ps(packed_w + 36, v6);
1215 _mm_store_ps(packed_w + 40, v10);
1216 _mm_store_ps(packed_w + 44, v14);
1217 _mm_store_ps(packed_w + 48, v3);
1218 _mm_store_ps(packed_w + 52, v7);
1219 _mm_store_ps(packed_w + 56, v11);
1220 _mm_store_ps(packed_w + 60, v15);
1221 packed_w += 64;
1222 }
1223 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
1224 }
1225 weights += nc * kc;
1226 } while (--g != 0);
1227 }