Debian Codegraph @

src/x32-packw/gen/x32-packw-x2-gemm-goi-neon-ld2lane-u2-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/NR2-neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x2__neon_ld2lane_u2_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 2);
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 uint32x2x2_t v00;
45 v00.val[0] = vdup_n_u32(0);
46 v00.val[1] = vdup_n_u32(0);
47
48 do {
49 // NC main loop multiple of 2
50 const uint32_t* w0 = weights;
51 size_t n = nc;
52
53 for (; n >= 2; n -= 2) {
54 if XNN_LIKELY(bias != NULL) {
55 uint32x2_t vb0 = vld1_u32(bias); bias += 2;
56 vst1_u32(packed_weights, vb0); packed_weights += 2;
57 } else {
58 const uint32x2_t vzero = vmov_n_u32(0);
59 vst1_u32(packed_weights, vzero); packed_weights += 2;
60 }
61
62 const uint32_t* w1 = w0 + kc;
63 xnn_prefetch_to_l1((const int8_t*) w0);
64 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
65 xnn_prefetch_to_l1((const int8_t*) w1);
66 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
67
68 // KC main loop multiple of 2
69 size_t k = kc;
70 for (; k >= 2; k -= 2) {
71 v00 = vld2_lane_u32(w0, v00, 0); w0 += 2;
72 v00 = vld2_lane_u32(w1, v00, 1); w1 += 2;
73 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
74 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
75 vst1_u32(packed_weights + 0, v00.val[0]);
76 vst1_u32(packed_weights + 2, v00.val[1]);
77 packed_weights += 4;
78 }
79
80 // KC remainder
81 for (; k != 0; --k) {
82 v00.val[0] = vld1_lane_u32(w0, v00.val[0], 0); w0 += 1;
83 v00.val[0] = vld1_lane_u32(w1, v00.val[0], 1); w1 += 1;
84 vst1_u32(packed_weights + 0, v00.val[0]);
85 packed_weights += 2;
86 }
87 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
88 w0 = w1;
89 }
90
91 if XNN_UNLIKELY(n != 0) {
92 // NC remainder of 1
93 if XNN_LIKELY(bias != NULL) {
94 *packed_weights = *bias++;
95 } else {
96 const uint32x2_t vzero = vmov_n_u32(0);
97 vst1_u32(packed_weights + 0, vzero);
98 }
99 packed_weights += 2;
100 size_t k = kc;
101 do {
102 *packed_weights = *w0++;
103 packed_weights += 2;
104 } while (--k);
105 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
106 }
107 weights += nc * kc;
108 } while (--g != 0);
109 }