Debian Codegraph @

src/x32-packw/gen/x32-packw-x2c4-gemm-goi-sse2-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/c4-sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <emmintrin.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x2c4__sse2_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 2);
38 assert(kr == 4);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42 __m128 v0;
43 __m128 v1;
44
45 const float* b = (const float*) bias;
46 float* packed_w = (float*) packed_weights;
47 do {
48 // NC main loop multiple of 2
49 const float* w0 = (const float*) weights;
50 size_t n = nc;
51
52 for (; n >= 2; n -= 2) {
53 if XNN_LIKELY(b != NULL) {
54 packed_w[0] = b[0];
55 packed_w[1] = b[1];
56 b += 2;
57 } else {
58 packed_w[0] = 0.0f;
59 packed_w[1] = 0.0f;
60 }
61 packed_w += 2;
62
63 const float* w1 = w0 + kc;
64
65 // KC main loop multiple of 2x4
66 size_t k = kc;
67 for (; k >= 4; k -= 4) {
68 // Read blocks of 2x4
69 // a b c d
70 // e f g h
71 v0 = _mm_loadu_ps(w0);
72 w0 += 4;
73 v1 = _mm_loadu_ps(w1);
74 w1 += 4;
75 _mm_storeu_ps(packed_w, v0);
76 _mm_storeu_ps(packed_w + 4, v1);
77 packed_w += 8;
78 }
79
80 // KC remainder (1..3)
81 if XNN_UNLIKELY(k != 0) {
82 assert(k >= 1);
83 assert(k <= 3);
84 switch (k) {
85 case 1:
86 // Read blocks of 2x1
87 // a
88 // e
89 v0 = _mm_load_ss(w0);
90 w0 += 1;
91 v1 = _mm_load_ss(w1);
92 w1 += 1;
93 break;
94 case 2:
95 // Read blocks of 2x2
96 // a b
97 // e f
98 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
99 w0 += 2;
100 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
101 w1 += 2;
102 break;
103 case 3:
104 {
105 // Read blocks of 2x3
106 // a b c
107 // e f g
108 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
109 const __m128 v0hi = _mm_load_ss(w0 + 2);
110 v0 = _mm_movelh_ps(v0lo, v0hi);
111 w0 += 3;
112 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
113 const __m128 v1hi = _mm_load_ss(w1 + 2);
114 v1 = _mm_movelh_ps(v1lo, v1hi);
115 w1 += 3;
116 break;
117 }
118 default:
119 XNN_UNREACHABLE;
120 }
121 _mm_storeu_ps(packed_w, v0);
122 _mm_storeu_ps(packed_w + 4, v1);
123 packed_w += 8;
124 }
125 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
126 w0 = w1;
127 }
128
129 // NC remainder (1..1)
130 if XNN_UNLIKELY(n != 0) {
131 assert(n >= 1);
132 assert(n <= 1);
133 if XNN_LIKELY(b != NULL) {
134 size_t nb = n;
135 do {
136 *packed_w++ = *b++;
137 } while (--nb != 0);
138 packed_w += (2 - n);
139 } else {
140 packed_w[0] = 0.0f;
141 packed_w[1] = 0.0f;
142 packed_w += 2;
143 }
144
145 // NR remainder has less than 2 rows so last row is not loaded
146
147 // KC main loop multiple of 2x4
148 size_t k = kc;
149 for (; k >= 4; k -= 4) {
150 // Read blocks of 2x4
151 // a b c d
152 // e f g h
153 v0 = _mm_loadu_ps(w0);
154 w0 += 4;
155 _mm_storeu_ps(packed_w, v0);
156 _mm_storeu_ps(packed_w + 4, v0);
157 packed_w += 8;
158 }
159
160 // KC remainder (1..3)
161 if XNN_UNLIKELY(k != 0) {
162 assert(k >= 1);
163 assert(k <= 3);
164 switch (k) {
165 case 1:
166 // Read blocks of 1x1
167 // a
168 v0 = _mm_load_ss(w0);
169 w0 += 1;
170 break;
171 case 2:
172 // Read blocks of 1x2
173 // a b
174 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
175 w0 += 2;
176 break;
177 case 3:
178 {
179 // Read blocks of 1x3
180 // a b c
181 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
182 const __m128 v0hi = _mm_load_ss(w0 + 2);
183 v0 = _mm_movelh_ps(v0lo, v0hi);
184 w0 += 3;
185 break;
186 }
187 default:
188 XNN_UNREACHABLE;
189 }
190 _mm_storeu_ps(packed_w, v0);
191 _mm_storeu_ps(packed_w + 4, v0);
192 packed_w += 8;
193 }
194 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
195 }
196 weights += nc * kc;
197 } while (--g != 0);
198 }