Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-gio-avx-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/gio-avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include <immintrin.h>
15
16 #include "xnnpack/intrinsics-polyfill.h"
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_gio_ukernel_x8__avx_u8(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 size_t k_stride,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 8); // This kernel is for NR=8
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(k_stride != 0);
42 assert(weights != NULL);
43 assert(packed_weights != NULL);
44 static const int32_t mask_table[16] = {
45 -1, -1, -1, -1, -1, -1, -1, -1,
46 0, 0, 0, 0, 0, 0, 0, 0,
47 };
48
49 const __m256 vzero = _mm256_setzero_ps();
50 const float* b = (const float*) bias;
51 float* packed_w = (float*) packed_weights;
52 do {
53 // NC main loop multiple of 8
54 const float* w = (const float*) weights;
55 size_t n = nc;
56
57 for (; n >= 8; n -= 8) {
58 if XNN_LIKELY(b != NULL) {
59 const __m256 vb0 = _mm256_loadu_ps(b + 0);
60 _mm256_store_ps(packed_w + 0, vb0);
61 b += 8;
62 } else {
63 _mm256_store_ps(packed_w + 0, vzero);
64 }
65 packed_w += 8;
66
67 size_t k = kc;
68 // KC main loop 8x8
69 for (; k >= 8; k -= 8) {
70 const __m256 v0_0 = _mm256_loadu_ps(w + 0 + 0 * k_stride);
71 const __m256 v0_1 = _mm256_loadu_ps(w + 0 + 1 * k_stride);
72 const __m256 v0_2 = _mm256_loadu_ps(w + 0 + 2 * k_stride);
73 const __m256 v0_3 = _mm256_loadu_ps(w + 0 + 3 * k_stride);
74 const __m256 v0_4 = _mm256_loadu_ps(w + 0 + 4 * k_stride);
75 const __m256 v0_5 = _mm256_loadu_ps(w + 0 + 5 * k_stride);
76 const __m256 v0_6 = _mm256_loadu_ps(w + 0 + 6 * k_stride);
77 const __m256 v0_7 = _mm256_loadu_ps(w + 0 + 7 * k_stride);
78 _mm256_store_ps(packed_w + 0, v0_0);
79 _mm256_store_ps(packed_w + 8, v0_1);
80 _mm256_store_ps(packed_w + 16, v0_2);
81 _mm256_store_ps(packed_w + 24, v0_3);
82 _mm256_store_ps(packed_w + 32, v0_4);
83 _mm256_store_ps(packed_w + 40, v0_5);
84 _mm256_store_ps(packed_w + 48, v0_6);
85 _mm256_store_ps(packed_w + 56, v0_7);
86 w += k_stride * 8;
87 packed_w += 64;
88 }
89
90 // KC remainder loop
91 for (; k > 0; --k) {
92 const __m256 v0 = _mm256_loadu_ps(w + 0);
93 _mm256_store_ps(packed_w + 0, v0);
94 w += k_stride;
95 packed_w += 8;
96 }
97 w = w - kc * k_stride + 8; // Advance to next column of 8 floats
98 }
99
100 // NC remainder (1..7)
101 if XNN_UNLIKELY(n != 0) {
102 assert(n >= 1);
103 assert(n <= 7);
104 const __m256i vmask0 = _mm256_loadu_si256((const __m256i*) &mask_table[8 - n]);
105
106 if XNN_LIKELY(b != NULL) {
107 const __m256 vb0 = _mm256_maskload_ps(b + 0, vmask0);
108 _mm256_store_ps(packed_w + 0, vb0);
109 b += n;
110 } else {
111 _mm256_store_ps(packed_w + 0, vzero);
112 }
113 packed_w += 8;
114
115 // KC main loop
116 for (size_t k = kc; k > 0; --k) {
117 const __m256 v0 = _mm256_maskload_ps(w + 0, vmask0);
118 _mm256_store_ps(packed_w + 0, v0);
119 w += k_stride;
120 packed_w += 8;
121 }
122 }
123 weights += nc * kc;
124 } while (--g != 0);
125 }