Debian Codegraph @

src/x32-packw/gen/x32-packw-x2c4-gemm-goi-sse2-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/c4-sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <emmintrin.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x2c4__sse2_u4_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 2);
39 assert(kr == 4);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43 __m128 v0;
44 __m128 v1;
45
46 const float* b = (const float*) bias;
47 float* packed_w = (float*) packed_weights;
48 do {
49 // NC main loop multiple of 2
50 const float* w0 = (const float*) weights;
51 size_t n = nc;
52
53 for (; n >= 2; n -= 2) {
54 if XNN_LIKELY(b != NULL) {
55 packed_w[0] = b[0];
56 packed_w[1] = b[1];
57 b += 2;
58 } else {
59 packed_w[0] = 0.0f;
60 packed_w[1] = 0.0f;
61 }
62 packed_w += 2;
63
64 const float* w1 = w0 + kc;
65 xnn_prefetch_to_l1((const int8_t*) w0);
66 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
67 xnn_prefetch_to_l1((const int8_t*) w1);
68 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
69
70 // KC main loop multiple of 2x4
71 size_t k = kc;
72 for (; k >= 4; k -= 4) {
73 // Read blocks of 2x4
74 // a b c d
75 // e f g h
76 v0 = _mm_loadu_ps(w0);
77 w0 += 4;
78 v1 = _mm_loadu_ps(w1);
79 w1 += 4;
80 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
81 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
82 _mm_storeu_ps(packed_w, v0);
83 _mm_storeu_ps(packed_w + 4, v1);
84 packed_w += 8;
85 }
86
87 // KC remainder (1..3)
88 if XNN_UNLIKELY(k != 0) {
89 assert(k >= 1);
90 assert(k <= 3);
91 switch (k) {
92 case 1:
93 // Read blocks of 2x1
94 // a
95 // e
96 v0 = _mm_load_ss(w0);
97 w0 += 1;
98 v1 = _mm_load_ss(w1);
99 w1 += 1;
100 break;
101 case 2:
102 // Read blocks of 2x2
103 // a b
104 // e f
105 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
106 w0 += 2;
107 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
108 w1 += 2;
109 break;
110 case 3:
111 {
112 // Read blocks of 2x3
113 // a b c
114 // e f g
115 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
116 const __m128 v0hi = _mm_load_ss(w0 + 2);
117 v0 = _mm_movelh_ps(v0lo, v0hi);
118 w0 += 3;
119 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
120 const __m128 v1hi = _mm_load_ss(w1 + 2);
121 v1 = _mm_movelh_ps(v1lo, v1hi);
122 w1 += 3;
123 break;
124 }
125 default:
126 XNN_UNREACHABLE;
127 }
128 _mm_storeu_ps(packed_w, v0);
129 _mm_storeu_ps(packed_w + 4, v1);
130 packed_w += 8;
131 }
132 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
133 w0 = w1;
134 }
135
136 // NC remainder (1..1)
137 if XNN_UNLIKELY(n != 0) {
138 assert(n >= 1);
139 assert(n <= 1);
140 if XNN_LIKELY(b != NULL) {
141 size_t nb = n;
142 do {
143 *packed_w++ = *b++;
144 } while (--nb != 0);
145 packed_w += (2 - n);
146 } else {
147 packed_w[0] = 0.0f;
148 packed_w[1] = 0.0f;
149 packed_w += 2;
150 }
151
152 // NR remainder has less than 2 rows so last row is not loaded
153
154 // KC main loop multiple of 2x4
155 size_t k = kc;
156 for (; k >= 4; k -= 4) {
157 // Read blocks of 2x4
158 // a b c d
159 // e f g h
160 v0 = _mm_loadu_ps(w0);
161 w0 += 4;
162 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
163 _mm_storeu_ps(packed_w, v0);
164 _mm_storeu_ps(packed_w + 4, v0);
165 packed_w += 8;
166 }
167
168 // KC remainder (1..3)
169 if XNN_UNLIKELY(k != 0) {
170 assert(k >= 1);
171 assert(k <= 3);
172 switch (k) {
173 case 1:
174 // Read blocks of 1x1
175 // a
176 v0 = _mm_load_ss(w0);
177 w0 += 1;
178 break;
179 case 2:
180 // Read blocks of 1x2
181 // a b
182 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
183 w0 += 2;
184 break;
185 case 3:
186 {
187 // Read blocks of 1x3
188 // a b c
189 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
190 const __m128 v0hi = _mm_load_ss(w0 + 2);
191 v0 = _mm_movelh_ps(v0lo, v0hi);
192 w0 += 3;
193 break;
194 }
195 default:
196 XNN_UNREACHABLE;
197 }
198 _mm_storeu_ps(packed_w, v0);
199 _mm_storeu_ps(packed_w + 4, v0);
200 packed_w += 8;
201 }
202 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
203 }
204 weights += nc * kc;
205 } while (--g != 0);
206 }