Debian Codegraph @

src/x32-packw/gen/x32-packw-x32-gemm-gio-avx-u1-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/gio-avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include <immintrin.h>
15
16 #include "xnnpack/intrinsics-polyfill.h"
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_gio_ukernel_x32__avx_u1_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 size_t k_stride,
29 const uint32_t* weights,
30 const uint32_t* bias,
31 const void* scale,
32 uint32_t* packed_weights,
33 size_t extra_bytes,
34 const void* params)
35 {
36 assert(g != 0);
37 assert(nc != 0);
38 assert(kc != 0);
39 assert(nr == 32); // This kernel is for NR=32
40 assert(kr == 1);
41 assert(sr == 1);
42 assert(k_stride != 0);
43 assert(weights != NULL);
44 assert(packed_weights != NULL);
45 static const int32_t mask_table[64] = {
46 -1, -1, -1, -1, -1, -1, -1, -1,
47 -1, -1, -1, -1, -1, -1, -1, -1,
48 -1, -1, -1, -1, -1, -1, -1, -1,
49 -1, -1, -1, -1, -1, -1, -1, -1,
50 0, 0, 0, 0, 0, 0, 0, 0,
51 0, 0, 0, 0, 0, 0, 0, 0,
52 0, 0, 0, 0, 0, 0, 0, 0,
53 0, 0, 0, 0, 0, 0, 0, 0,
54 };
55
56 const __m256 vzero = _mm256_setzero_ps();
57 const float* b = (const float*) bias;
58 float* packed_w = (float*) packed_weights;
59 do {
60 // NC main loop multiple of 32
61 const float* w = (const float*) weights;
62 size_t n = nc;
63
64 for (; n >= 32; n -= 32) {
65 if XNN_LIKELY(b != NULL) {
66 const __m256 vb0 = _mm256_loadu_ps(b + 0);
67 const __m256 vb8 = _mm256_loadu_ps(b + 8);
68 const __m256 vb16 = _mm256_loadu_ps(b + 16);
69 const __m256 vb24 = _mm256_loadu_ps(b + 24);
70 _mm256_store_ps(packed_w + 0, vb0);
71 _mm256_store_ps(packed_w + 8, vb8);
72 _mm256_store_ps(packed_w + 16, vb16);
73 _mm256_store_ps(packed_w + 24, vb24);
74 b += 32;
75 } else {
76 _mm256_store_ps(packed_w + 0, vzero);
77 _mm256_store_ps(packed_w + 8, vzero);
78 _mm256_store_ps(packed_w + 16, vzero);
79 _mm256_store_ps(packed_w + 24, vzero);
80 }
81 packed_w += 32;
82
83 size_t k = kc;
84
85 // KC remainder loop
86 for (; k > 0; --k) {
87 const __m256 v0 = _mm256_loadu_ps(w + 0);
88 const __m256 v8 = _mm256_loadu_ps(w + 8);
89 const __m256 v16 = _mm256_loadu_ps(w + 16);
90 const __m256 v24 = _mm256_loadu_ps(w + 24);
91 xnn_prefetch_to_l1((const int8_t*) w + 960);
92 xnn_prefetch_to_l1((const int8_t*) w + 1024);
93 _mm256_store_ps(packed_w + 0, v0);
94 _mm256_store_ps(packed_w + 8, v8);
95 _mm256_store_ps(packed_w + 16, v16);
96 _mm256_store_ps(packed_w + 24, v24);
97 w += k_stride;
98 packed_w += 32;
99 }
100 w = w - kc * k_stride + 32; // Advance to next column of 32 floats
101 }
102
103 // NC remainder (1..31)
104 if XNN_UNLIKELY(n != 0) {
105 assert(n >= 1);
106 assert(n <= 31);
107 const __m256i vmask0 = _mm256_loadu_si256((const __m256i*) &mask_table[32 - n]);
108 const __m256i vmask8 = _mm256_loadu_si256((const __m256i*) &mask_table[40 - n]);
109 const __m256i vmask16 = _mm256_loadu_si256((const __m256i*) &mask_table[48 - n]);
110 const __m256i vmask24 = _mm256_loadu_si256((const __m256i*) &mask_table[56 - n]);
111
112 if XNN_LIKELY(b != NULL) {
113 const __m256 vb0 = _mm256_maskload_ps(b + 0, vmask0);
114 const __m256 vb8 = _mm256_maskload_ps(b + 8, vmask8);
115 const __m256 vb16 = _mm256_maskload_ps(b + 16, vmask16);
116 const __m256 vb24 = _mm256_maskload_ps(b + 24, vmask24);
117 _mm256_store_ps(packed_w + 0, vb0);
118 _mm256_store_ps(packed_w + 8, vb8);
119 _mm256_store_ps(packed_w + 16, vb16);
120 _mm256_store_ps(packed_w + 24, vb24);
121 b += n;
122 } else {
123 _mm256_store_ps(packed_w + 0, vzero);
124 _mm256_store_ps(packed_w + 8, vzero);
125 _mm256_store_ps(packed_w + 16, vzero);
126 _mm256_store_ps(packed_w + 24, vzero);
127 }
128 packed_w += 32;
129
130 // KC main loop
131 for (size_t k = kc; k > 0; --k) {
132 const __m256 v0 = _mm256_maskload_ps(w + 0, vmask0);
133 const __m256 v8 = _mm256_maskload_ps(w + 8, vmask8);
134 const __m256 v16 = _mm256_maskload_ps(w + 16, vmask16);
135 const __m256 v24 = _mm256_maskload_ps(w + 24, vmask24);
136 _mm256_store_ps(packed_w + 0, v0);
137 _mm256_store_ps(packed_w + 8, v8);
138 _mm256_store_ps(packed_w + 16, v16);
139 _mm256_store_ps(packed_w + 24, v24);
140 w += k_stride;
141 packed_w += 32;
142 }
143 }
144 weights += nc * kc;
145 } while (--g != 0);
146 }