Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-gio-scalar.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/gio-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/packw.h"
15
16
17 void xnn_x32_packw_gemm_gio_ukernel_x8__scalar(
18 size_t g,
19 size_t nc,
20 size_t kc,
21 size_t nr,
22 size_t kr,
23 size_t sr,
24 size_t k_stride,
25 const uint32_t* weights,
26 const uint32_t* bias,
27 const void* scale,
28 uint32_t* packed_weights,
29 size_t extra_bytes,
30 const void* params)
31 {
32 assert(g != 0);
33 assert(nc != 0);
34 assert(kc != 0);
35 assert(nr == 8); // This kernel is for NR=8
36 assert(kr == 1);
37 assert(sr == 1);
38 assert(k_stride != 0);
39 assert(weights != NULL);
40 assert(packed_weights != NULL);
41
42 const float* b = (const float*) bias;
43 float* packed_w = (float*) packed_weights;
44 do {
45 // NC main loop multiple of 8
46 const float* w = (const float*) weights;
47 size_t n = nc;
48
49 for (; n >= 8; n -= 8) {
50 if XNN_LIKELY(b != NULL) {
51 const uint64_t v0 = ((const uint64_t*)b)[0];
52 const uint64_t v1 = ((const uint64_t*)b)[1];
53 const uint64_t v2 = ((const uint64_t*)b)[2];
54 const uint64_t v3 = ((const uint64_t*)b)[3];
55 ((uint64_t*)packed_w)[0] = v0;
56 ((uint64_t*)packed_w)[1] = v1;
57 ((uint64_t*)packed_w)[2] = v2;
58 ((uint64_t*)packed_w)[3] = v3;
59 b += 8;
60 } else {
61 ((uint64_t*)packed_w)[0] = 0;
62 ((uint64_t*)packed_w)[1] = 0;
63 ((uint64_t*)packed_w)[2] = 0;
64 ((uint64_t*)packed_w)[3] = 0;
65 }
66 packed_w += 8;
67
68 // KC main loop
69 for (size_t k = kc; k > 0; --k) {
70 const uint64_t v0 = ((const uint64_t*)w)[0];
71 const uint64_t v1 = ((const uint64_t*)w)[1];
72 const uint64_t v2 = ((const uint64_t*)w)[2];
73 const uint64_t v3 = ((const uint64_t*)w)[3];
74 ((uint64_t*)packed_w)[0] = v0;
75 ((uint64_t*)packed_w)[1] = v1;
76 ((uint64_t*)packed_w)[2] = v2;
77 ((uint64_t*)packed_w)[3] = v3;
78 w += k_stride;
79 packed_w += 8;
80 }
81 w = w - kc * k_stride + 8; // Advance to next column of 8 floats
82 }
83
84 // NC remainder (1..7)
85 if XNN_UNLIKELY(n != 0) {
86 assert(n >= 1);
87 assert(n <= 7);
88
89 if XNN_LIKELY(b != NULL) {
90 for (size_t i = 0; i < n; ++i) {
91 packed_w[i] = b[i];
92 }
93 b += n;
94 } else {
95 ((uint64_t*)packed_w)[0] = 0;
96 ((uint64_t*)packed_w)[1] = 0;
97 ((uint64_t*)packed_w)[2] = 0;
98 ((uint64_t*)packed_w)[3] = 0;
99 }
100 packed_w += 8;
101
102 // KC main loop
103 for (size_t k = kc; k > 0; --k) {
104 for (size_t i = 0; i < n; ++i) {
105 packed_w[i] = w[i];
106 }
107 w += k_stride;
108 packed_w += 8;
109 }
110 }
111 weights += nc * kc;
112 } while (--g != 0);
113 }