Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l1c1s1r-minmax-fp32-scalar-fmagic.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/microparams.h"
18 #include "xnnpack/unaligned.h"
19
20
21 void xnn_qs8_dwconv_minmax_fp32_ukernel_6f6m7l1c1s1r__scalar_fmagic(
22 size_t channels,
23 size_t output_width,
24 const int8_t** input,
25 const void* weights,
26 int8_t* output,
27 intptr_t input_stride,
28 size_t output_increment,
29 size_t input_offset,
30 const int8_t* zero,
31 size_t kernel_size,
32 int32_t* buffer,
33 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
34 {
35 assert(channels != 0);
36 assert(output_width != 0);
37 assert(kernel_size > 6);
38
39 const float vscale = params->fp32_scalar.scale;
40 const float voutput_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
41 const float voutput_max_less_zero_point = (int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point;
42 const float vmagic_bias = 12582912.0f;
43 const int32_t vmagic_bias_less_output_zero_point = INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point;
44 do {
45 const void* w = weights;
46
47 // First pass to process 6 inputs.
48 {
49 int32_t* b = buffer;
50 const int8_t* i0 = input[0];
51 assert(i0 != NULL);
52 if XNN_UNPREDICTABLE(i0 != zero) {
53 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
54 }
55 const int8_t* i1 = input[1];
56 assert(i1 != NULL);
57 if XNN_UNPREDICTABLE(i1 != zero) {
58 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
59 }
60 const int8_t* i2 = input[2];
61 assert(i2 != NULL);
62 if XNN_UNPREDICTABLE(i2 != zero) {
63 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
64 }
65 const int8_t* i3 = input[3];
66 assert(i3 != NULL);
67 if XNN_UNPREDICTABLE(i3 != zero) {
68 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
69 }
70 const int8_t* i4 = input[4];
71 assert(i4 != NULL);
72 if XNN_UNPREDICTABLE(i4 != zero) {
73 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
74 }
75 const int8_t* i5 = input[5];
76 assert(i5 != NULL);
77 if XNN_UNPREDICTABLE(i5 != zero) {
78 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
79 }
80 input += 6;
81
82 size_t c = channels;
83 do {
84 int32_t vacc = unaligned_load_s32(w);
85 const int32_t vi0 = (int32_t) *i0++;
86 const int32_t vk0 = ((const int8_t*) ((uintptr_t) w + sizeof(int32_t)))[0];
87 vacc += vi0 * vk0;
88 const int32_t vi1 = (int32_t) *i1++;
89 const int32_t vk1 = ((const int8_t*) ((uintptr_t) w + sizeof(int32_t)))[1];
90 vacc += vi1 * vk1;
91 const int32_t vi2 = (int32_t) *i2++;
92 const int32_t vk2 = ((const int8_t*) ((uintptr_t) w + sizeof(int32_t)))[2];
93 vacc += vi2 * vk2;
94 const int32_t vi3 = (int32_t) *i3++;
95 const int32_t vk3 = ((const int8_t*) ((uintptr_t) w + sizeof(int32_t)))[3];
96 vacc += vi3 * vk3;
97 const int32_t vi4 = (int32_t) *i4++;
98 const int32_t vk4 = ((const int8_t*) ((uintptr_t) w + sizeof(int32_t)))[4];
99 vacc += vi4 * vk4;
100 const int32_t vi5 = (int32_t) *i5++;
101 const int32_t vk5 = ((const int8_t*) ((uintptr_t) w + sizeof(int32_t)))[5];
102 vacc += vi5 * vk5;
103
104 w = (const void*) ((uintptr_t) w + sizeof(int32_t) + 6 * sizeof(int8_t));
105 *b++ = vacc;
106 } while (--c != 0);
107 }
108
109 // Middle pass to process 6 inputs in each iteration.
110 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
111 int32_t* b = buffer;
112 const int8_t* i0 = input[0];
113 assert(i0 != NULL);
114 if XNN_UNPREDICTABLE(i0 != zero) {
115 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
116 }
117 const int8_t* i1 = input[1];
118 assert(i1 != NULL);
119 if XNN_UNPREDICTABLE(i1 != zero) {
120 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
121 }
122 const int8_t* i2 = input[2];
123 assert(i2 != NULL);
124 if XNN_UNPREDICTABLE(i2 != zero) {
125 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
126 }
127 const int8_t* i3 = input[3];
128 assert(i3 != NULL);
129 if XNN_UNPREDICTABLE(i3 != zero) {
130 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
131 }
132 const int8_t* i4 = input[4];
133 assert(i4 != NULL);
134 if XNN_UNPREDICTABLE(i4 != zero) {
135 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
136 }
137 const int8_t* i5 = input[5];
138 assert(i5 != NULL);
139 if XNN_UNPREDICTABLE(i5 != zero) {
140 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
141 }
142 input += 6;
143
144 size_t c = channels;
145 do {
146 int32_t vacc = *b;
147 const int32_t vi0 = (int32_t) *i0++;
148 const int32_t vk0 = ((const int8_t*) w)[0];
149 vacc += vi0 * vk0;
150 const int32_t vi1 = (int32_t) *i1++;
151 const int32_t vk1 = ((const int8_t*) w)[1];
152 vacc += vi1 * vk1;
153 const int32_t vi2 = (int32_t) *i2++;
154 const int32_t vk2 = ((const int8_t*) w)[2];
155 vacc += vi2 * vk2;
156 const int32_t vi3 = (int32_t) *i3++;
157 const int32_t vk3 = ((const int8_t*) w)[3];
158 vacc += vi3 * vk3;
159 const int32_t vi4 = (int32_t) *i4++;
160 const int32_t vk4 = ((const int8_t*) w)[4];
161 vacc += vi4 * vk4;
162 const int32_t vi5 = (int32_t) *i5++;
163 const int32_t vk5 = ((const int8_t*) w)[5];
164 vacc += vi5 * vk5;
165
166 w = (const void*) ((uintptr_t) w + 6 * sizeof(int8_t));
167 *b++ = vacc;
168 } while (--c != 0);
169 }
170
171 // Last pass to process up to 7 inputs.
172 {
173 const int32_t* b = buffer;
174 const int8_t* i0 = input[0];
175 assert(i0 != NULL);
176 if XNN_UNPREDICTABLE(i0 != zero) {
177 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
178 }
179 const int8_t* i1 = input[1];
180 assert(i1 != NULL);
181 if XNN_UNPREDICTABLE(i1 != zero) {
182 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
183 }
184 const int8_t* i2 = input[2];
185 assert(i2 != NULL);
186 if XNN_UNPREDICTABLE(i2 != zero) {
187 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
188 }
189 const int8_t* i3 = input[3];
190 assert(i3 != NULL);
191 if XNN_UNPREDICTABLE(i3 != zero) {
192 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
193 }
194 const int8_t* i4 = input[4];
195 assert(i4 != NULL);
196 if XNN_UNPREDICTABLE(i4 != zero) {
197 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
198 }
199 const int8_t* i5 = input[5];
200 assert(i5 != NULL);
201 if XNN_UNPREDICTABLE(i5 != zero) {
202 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
203 }
204 const int8_t* i6 = input[6];
205 assert(i6 != NULL);
206 if XNN_UNPREDICTABLE(i6 != zero) {
207 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
208 }
209
210 size_t c = channels;
211 do {
212 int32_t vacc = unaligned_load_s32(b++);
213 const int32_t vi0 = (int32_t) *i0++;
214 const int32_t vk0 = ((const int8_t*) w)[0];
215 vacc += vi0 * vk0;
216 const int32_t vi1 = (int32_t) *i1++;
217 const int32_t vk1 = ((const int8_t*) w)[1];
218 vacc += vi1 * vk1;
219 const int32_t vi2 = (int32_t) *i2++;
220 const int32_t vk2 = ((const int8_t*) w)[2];
221 vacc += vi2 * vk2;
222 const int32_t vi3 = (int32_t) *i3++;
223 const int32_t vk3 = ((const int8_t*) w)[3];
224 vacc += vi3 * vk3;
225 const int32_t vi4 = (int32_t) *i4++;
226 const int32_t vk4 = ((const int8_t*) w)[4];
227 vacc += vi4 * vk4;
228 const int32_t vi5 = (int32_t) *i5++;
229 const int32_t vk5 = ((const int8_t*) w)[5];
230 vacc += vi5 * vk5;
231 const int32_t vi6 = (int32_t) *i6++;
232 const int32_t vk6 = ((const int8_t*) w)[6];
233 vacc += vi6 * vk6;
234
235 w = (const void*) ((uintptr_t) w + 7 * sizeof(int8_t));
236
237 float vfpacc = (float) vacc * vscale;
238
239 vfpacc = math_max_f32(vfpacc, voutput_min_less_zero_point);
240 vfpacc = math_min_f32(vfpacc, voutput_max_less_zero_point);
241 vfpacc += vmagic_bias;
242 int32_t vout = (int32_t) float_as_uint32(vfpacc) - vmagic_bias_less_output_zero_point;
243
244 *output++ = (int8_t) vout;
245 } while (--c != 0);
246 }
247
248 input = (const int8_t**) ((uintptr_t) input + input_stride);
249 output = (int8_t*) ((uintptr_t) output + output_increment);
250 } while (--output_width != 0);
251 }