Debian Codegraph @

src/x32-packw/gen/x32-packw-x4-gemm-gio-scalar.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/gio-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/packw.h"
15
16
17 void xnn_x32_packw_gemm_gio_ukernel_x4__scalar(
18 size_t g,
19 size_t nc,
20 size_t kc,
21 size_t nr,
22 size_t kr,
23 size_t sr,
24 size_t k_stride,
25 const uint32_t* weights,
26 const uint32_t* bias,
27 const void* scale,
28 uint32_t* packed_weights,
29 size_t extra_bytes,
30 const void* params)
31 {
32 assert(g != 0);
33 assert(nc != 0);
34 assert(kc != 0);
35 assert(nr == 4); // This kernel is for NR=4
36 assert(kr == 1);
37 assert(sr == 1);
38 assert(k_stride != 0);
39 assert(weights != NULL);
40 assert(packed_weights != NULL);
41
42 const float* b = (const float*) bias;
43 float* packed_w = (float*) packed_weights;
44 do {
45 // NC main loop multiple of 4
46 const float* w = (const float*) weights;
47 size_t n = nc;
48
49 for (; n >= 4; n -= 4) {
50 if XNN_LIKELY(b != NULL) {
51 const uint64_t v0 = ((const uint64_t*)b)[0];
52 const uint64_t v1 = ((const uint64_t*)b)[1];
53 ((uint64_t*)packed_w)[0] = v0;
54 ((uint64_t*)packed_w)[1] = v1;
55 b += 4;
56 } else {
57 ((uint64_t*)packed_w)[0] = 0;
58 ((uint64_t*)packed_w)[1] = 0;
59 }
60 packed_w += 4;
61
62 // KC main loop
63 for (size_t k = kc; k > 0; --k) {
64 const uint64_t v0 = ((const uint64_t*)w)[0];
65 const uint64_t v1 = ((const uint64_t*)w)[1];
66 ((uint64_t*)packed_w)[0] = v0;
67 ((uint64_t*)packed_w)[1] = v1;
68 w += k_stride;
69 packed_w += 4;
70 }
71 w = w - kc * k_stride + 4; // Advance to next column of 4 floats
72 }
73
74 // NC remainder (1..3)
75 if XNN_UNLIKELY(n != 0) {
76 assert(n >= 1);
77 assert(n <= 3);
78
79 if XNN_LIKELY(b != NULL) {
80 for (size_t i = 0; i < n; ++i) {
81 packed_w[i] = b[i];
82 }
83 b += n;
84 } else {
85 ((uint64_t*)packed_w)[0] = 0;
86 ((uint64_t*)packed_w)[1] = 0;
87 }
88 packed_w += 4;
89
90 // KC main loop
91 for (size_t k = kc; k > 0; --k) {
92 for (size_t i = 0; i < n; ++i) {
93 packed_w[i] = w[i];
94 }
95 w += k_stride;
96 packed_w += 4;
97 }
98 }
99 weights += nc * kc;
100 } while (--g != 0);
101 }