Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-gio-avx-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/gio-avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include <immintrin.h>
15
16 #include "xnnpack/intrinsics-polyfill.h"
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_gio_ukernel_x16__avx_u8_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 size_t k_stride,
29 const uint32_t* weights,
30 const uint32_t* bias,
31 const void* scale,
32 uint32_t* packed_weights,
33 size_t extra_bytes,
34 const void* params)
35 {
36 assert(g != 0);
37 assert(nc != 0);
38 assert(kc != 0);
39 assert(nr == 16); // This kernel is for NR=16
40 assert(kr == 1);
41 assert(sr == 1);
42 assert(k_stride != 0);
43 assert(weights != NULL);
44 assert(packed_weights != NULL);
45 static const int32_t mask_table[32] = {
46 -1, -1, -1, -1, -1, -1, -1, -1,
47 -1, -1, -1, -1, -1, -1, -1, -1,
48 0, 0, 0, 0, 0, 0, 0, 0,
49 0, 0, 0, 0, 0, 0, 0, 0,
50 };
51
52 const __m256 vzero = _mm256_setzero_ps();
53 const float* b = (const float*) bias;
54 float* packed_w = (float*) packed_weights;
55 do {
56 // NC main loop multiple of 16
57 const float* w = (const float*) weights;
58 size_t n = nc;
59
60 for (; n >= 16; n -= 16) {
61 if XNN_LIKELY(b != NULL) {
62 const __m256 vb0 = _mm256_loadu_ps(b + 0);
63 const __m256 vb8 = _mm256_loadu_ps(b + 8);
64 _mm256_store_ps(packed_w + 0, vb0);
65 _mm256_store_ps(packed_w + 8, vb8);
66 b += 16;
67 } else {
68 _mm256_store_ps(packed_w + 0, vzero);
69 _mm256_store_ps(packed_w + 8, vzero);
70 }
71 packed_w += 16;
72
73 size_t k = kc;
74 // KC main loop 8x16
75 for (; k >= 8; k -= 8) {
76 const __m256 v0_0 = _mm256_loadu_ps(w + 0 + 0 * k_stride);
77 const __m256 v8_0 = _mm256_loadu_ps(w + 8 + 0 * k_stride);
78 const __m256 v0_1 = _mm256_loadu_ps(w + 0 + 1 * k_stride);
79 const __m256 v8_1 = _mm256_loadu_ps(w + 8 + 1 * k_stride);
80 const __m256 v0_2 = _mm256_loadu_ps(w + 0 + 2 * k_stride);
81 const __m256 v8_2 = _mm256_loadu_ps(w + 8 + 2 * k_stride);
82 const __m256 v0_3 = _mm256_loadu_ps(w + 0 + 3 * k_stride);
83 const __m256 v8_3 = _mm256_loadu_ps(w + 8 + 3 * k_stride);
84 const __m256 v0_4 = _mm256_loadu_ps(w + 0 + 4 * k_stride);
85 const __m256 v8_4 = _mm256_loadu_ps(w + 8 + 4 * k_stride);
86 const __m256 v0_5 = _mm256_loadu_ps(w + 0 + 5 * k_stride);
87 const __m256 v8_5 = _mm256_loadu_ps(w + 8 + 5 * k_stride);
88 const __m256 v0_6 = _mm256_loadu_ps(w + 0 + 6 * k_stride);
89 const __m256 v8_6 = _mm256_loadu_ps(w + 8 + 6 * k_stride);
90 const __m256 v0_7 = _mm256_loadu_ps(w + 0 + 7 * k_stride);
91 const __m256 v8_7 = _mm256_loadu_ps(w + 8 + 7 * k_stride);
92 xnn_prefetch_to_l1((const int8_t*) w + 960);
93 xnn_prefetch_to_l1((const int8_t*) w + 960);
94 xnn_prefetch_to_l1((const int8_t*) w + 960);
95 xnn_prefetch_to_l1((const int8_t*) w + 960);
96 xnn_prefetch_to_l1((const int8_t*) w + 960);
97 xnn_prefetch_to_l1((const int8_t*) w + 960);
98 xnn_prefetch_to_l1((const int8_t*) w + 960);
99 xnn_prefetch_to_l1((const int8_t*) w + 960);
100 _mm256_store_ps(packed_w + 0, v0_0);
101 _mm256_store_ps(packed_w + 8, v8_0);
102 _mm256_store_ps(packed_w + 16, v0_1);
103 _mm256_store_ps(packed_w + 24, v8_1);
104 _mm256_store_ps(packed_w + 32, v0_2);
105 _mm256_store_ps(packed_w + 40, v8_2);
106 _mm256_store_ps(packed_w + 48, v0_3);
107 _mm256_store_ps(packed_w + 56, v8_3);
108 _mm256_store_ps(packed_w + 64, v0_4);
109 _mm256_store_ps(packed_w + 72, v8_4);
110 _mm256_store_ps(packed_w + 80, v0_5);
111 _mm256_store_ps(packed_w + 88, v8_5);
112 _mm256_store_ps(packed_w + 96, v0_6);
113 _mm256_store_ps(packed_w + 104, v8_6);
114 _mm256_store_ps(packed_w + 112, v0_7);
115 _mm256_store_ps(packed_w + 120, v8_7);
116 w += k_stride * 8;
117 packed_w += 128;
118 }
119
120 // KC remainder loop
121 for (; k > 0; --k) {
122 const __m256 v0 = _mm256_loadu_ps(w + 0);
123 const __m256 v8 = _mm256_loadu_ps(w + 8);
124 xnn_prefetch_to_l1((const int8_t*) w + 960);
125 _mm256_store_ps(packed_w + 0, v0);
126 _mm256_store_ps(packed_w + 8, v8);
127 w += k_stride;
128 packed_w += 16;
129 }
130 w = w - kc * k_stride + 16; // Advance to next column of 16 floats
131 }
132
133 // NC remainder (1..15)
134 if XNN_UNLIKELY(n != 0) {
135 assert(n >= 1);
136 assert(n <= 15);
137 const __m256i vmask0 = _mm256_loadu_si256((const __m256i*) &mask_table[16 - n]);
138 const __m256i vmask8 = _mm256_loadu_si256((const __m256i*) &mask_table[24 - n]);
139
140 if XNN_LIKELY(b != NULL) {
141 const __m256 vb0 = _mm256_maskload_ps(b + 0, vmask0);
142 const __m256 vb8 = _mm256_maskload_ps(b + 8, vmask8);
143 _mm256_store_ps(packed_w + 0, vb0);
144 _mm256_store_ps(packed_w + 8, vb8);
145 b += n;
146 } else {
147 _mm256_store_ps(packed_w + 0, vzero);
148 _mm256_store_ps(packed_w + 8, vzero);
149 }
150 packed_w += 16;
151
152 // KC main loop
153 for (size_t k = kc; k > 0; --k) {
154 const __m256 v0 = _mm256_maskload_ps(w + 0, vmask0);
155 const __m256 v8 = _mm256_maskload_ps(w + 8, vmask8);
156 _mm256_store_ps(packed_w + 0, v0);
157 _mm256_store_ps(packed_w + 8, v8);
158 w += k_stride;
159 packed_w += 16;
160 }
161 }
162 weights += nc * kc;
163 } while (--g != 0);
164 }