Debian Codegraph @

src/x32-packw/gen/x32-packw-x2-gemm-goi-neon-ld2lane-u2.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/NR2-neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x2__neon_ld2lane_u2(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 2);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 uint32x2x2_t v00;
44 v00.val[0] = vdup_n_u32(0);
45 v00.val[1] = vdup_n_u32(0);
46
47 do {
48 // NC main loop multiple of 2
49 const uint32_t* w0 = weights;
50 size_t n = nc;
51
52 for (; n >= 2; n -= 2) {
53 if XNN_LIKELY(bias != NULL) {
54 uint32x2_t vb0 = vld1_u32(bias); bias += 2;
55 vst1_u32(packed_weights, vb0); packed_weights += 2;
56 } else {
57 const uint32x2_t vzero = vmov_n_u32(0);
58 vst1_u32(packed_weights, vzero); packed_weights += 2;
59 }
60
61 const uint32_t* w1 = w0 + kc;
62
63 // KC main loop multiple of 2
64 size_t k = kc;
65 for (; k >= 2; k -= 2) {
66 v00 = vld2_lane_u32(w0, v00, 0); w0 += 2;
67 v00 = vld2_lane_u32(w1, v00, 1); w1 += 2;
68 vst1_u32(packed_weights + 0, v00.val[0]);
69 vst1_u32(packed_weights + 2, v00.val[1]);
70 packed_weights += 4;
71 }
72
73 // KC remainder
74 for (; k != 0; --k) {
75 v00.val[0] = vld1_lane_u32(w0, v00.val[0], 0); w0 += 1;
76 v00.val[0] = vld1_lane_u32(w1, v00.val[0], 1); w1 += 1;
77 vst1_u32(packed_weights + 0, v00.val[0]);
78 packed_weights += 2;
79 }
80 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
81 w0 = w1;
82 }
83
84 if XNN_UNLIKELY(n != 0) {
85 // NC remainder of 1
86 if XNN_LIKELY(bias != NULL) {
87 *packed_weights = *bias++;
88 } else {
89 const uint32x2_t vzero = vmov_n_u32(0);
90 vst1_u32(packed_weights + 0, vzero);
91 }
92 packed_weights += 2;
93 size_t k = kc;
94 do {
95 *packed_weights = *w0++;
96 packed_weights += 2;
97 } while (--k);
98 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
99 }
100 weights += nc * kc;
101 } while (--g != 0);
102 }