Debian Codegraph @

src/qu8-vadd/gen/qu8-vadd-minmax-wasmsimd-u32.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-vadd/wasmsimd.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/vbinary.h"
16
17
18 void xnn_qu8_vadd_minmax_ukernel__wasmsimd_u32(
19 size_t batch,
20 const uint8_t* input_a,
21 const uint8_t* input_b,
22 uint8_t* output,
23 const struct xnn_qu8_add_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
24 {
25 assert(batch != 0);
26 assert(batch % sizeof(uint8_t) == 0);
27 assert(input_a != NULL);
28 assert(input_b != NULL);
29 assert(output != NULL);
30
31 const v128_t vbias = wasm_v128_load32_splat(&params->scalar.bias);
32 const v128_t va_multiplier = wasm_v128_load32_splat(&params->scalar.a_multiplier);
33 const v128_t vb_multiplier = wasm_v128_load32_splat(&params->scalar.b_multiplier);
34 const uint32_t vshift = params->scalar.shift;
35 const v128_t voutput_zero_point = wasm_v128_load16_splat(&params->scalar.output_zero_point);
36 const v128_t voutput_min = wasm_v128_load8_splat(&params->scalar.output_min);
37 const v128_t voutput_max = wasm_v128_load8_splat(&params->scalar.output_max);
38
39 XNN_FORCE_REALIZATION(vbias);
40 XNN_FORCE_REALIZATION(va_multiplier);
41 XNN_FORCE_REALIZATION(vb_multiplier);
42 XNN_FORCE_REALIZATION(voutput_zero_point);
43 XNN_FORCE_REALIZATION(voutput_min);
44 XNN_FORCE_REALIZATION(voutput_max);
45
46 for (; batch >= 32 * sizeof(uint8_t); batch -= 32 * sizeof(uint8_t)) {
47 const v128_t va01234567 = wasm_u16x8_load8x8(input_a);
48 const v128_t vb01234567 = wasm_u16x8_load8x8(input_b);
49 const v128_t va89ABCDEF = wasm_u16x8_load8x8(input_a + 8);
50 const v128_t vb89ABCDEF = wasm_u16x8_load8x8(input_b + 8);
51 const v128_t vaGHIJKLMN = wasm_u16x8_load8x8(input_a + 16);
52 const v128_t vbGHIJKLMN = wasm_u16x8_load8x8(input_b + 16);
53 const v128_t vaOPQRSTUV = wasm_u16x8_load8x8(input_a + 24);
54 const v128_t vbOPQRSTUV = wasm_u16x8_load8x8(input_b + 24);
55 input_a += 32;
56 input_b += 32;
57
58 v128_t vacc0123 = wasm_i32x4_add(vbias, wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(va01234567), va_multiplier));
59 v128_t vacc4567 = wasm_i32x4_add(vbias, wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(va01234567), va_multiplier));
60 v128_t vacc89AB = wasm_i32x4_add(vbias, wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(va89ABCDEF), va_multiplier));
61 v128_t vaccCDEF = wasm_i32x4_add(vbias, wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(va89ABCDEF), va_multiplier));
62 v128_t vaccGHIJ = wasm_i32x4_add(vbias, wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(vaGHIJKLMN), va_multiplier));
63 v128_t vaccKLMN = wasm_i32x4_add(vbias, wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(vaGHIJKLMN), va_multiplier));
64 v128_t vaccOPQR = wasm_i32x4_add(vbias, wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(vaOPQRSTUV), va_multiplier));
65 v128_t vaccSTUV = wasm_i32x4_add(vbias, wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(vaOPQRSTUV), va_multiplier));
66
67 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(vb01234567), vb_multiplier));
68 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(vb01234567), vb_multiplier));
69 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(vb89ABCDEF), vb_multiplier));
70 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(vb89ABCDEF), vb_multiplier));
71 vaccGHIJ = wasm_i32x4_add(vaccGHIJ, wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(vbGHIJKLMN), vb_multiplier));
72 vaccKLMN = wasm_i32x4_add(vaccKLMN, wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(vbGHIJKLMN), vb_multiplier));
73 vaccOPQR = wasm_i32x4_add(vaccOPQR, wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(vbOPQRSTUV), vb_multiplier));
74 vaccSTUV = wasm_i32x4_add(vaccSTUV, wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(vbOPQRSTUV), vb_multiplier));
75
76 vacc0123 = wasm_i32x4_shr(vacc0123, vshift);
77 vacc4567 = wasm_i32x4_shr(vacc4567, vshift);
78 vacc89AB = wasm_i32x4_shr(vacc89AB, vshift);
79 vaccCDEF = wasm_i32x4_shr(vaccCDEF, vshift);
80 vaccGHIJ = wasm_i32x4_shr(vaccGHIJ, vshift);
81 vaccKLMN = wasm_i32x4_shr(vaccKLMN, vshift);
82 vaccOPQR = wasm_i32x4_shr(vaccOPQR, vshift);
83 vaccSTUV = wasm_i32x4_shr(vaccSTUV, vshift);
84
85 v128_t vout01234567 = wasm_i16x8_add_sat(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
86 v128_t vout89ABCDEF = wasm_i16x8_add_sat(wasm_i16x8_narrow_i32x4(vacc89AB, vaccCDEF), voutput_zero_point);
87 v128_t voutGHIJKLMN = wasm_i16x8_add_sat(wasm_i16x8_narrow_i32x4(vaccGHIJ, vaccKLMN), voutput_zero_point);
88 v128_t voutOPQRSTUV = wasm_i16x8_add_sat(wasm_i16x8_narrow_i32x4(vaccOPQR, vaccSTUV), voutput_zero_point);
89
90 v128_t vout0123456789ABCDEF = wasm_u8x16_narrow_i16x8(vout01234567, vout89ABCDEF);
91 v128_t voutGHIJKLMNOPQRSTUV = wasm_u8x16_narrow_i16x8(voutGHIJKLMN, voutOPQRSTUV);
92
93 vout0123456789ABCDEF = wasm_u8x16_max(vout0123456789ABCDEF, voutput_min);
94 voutGHIJKLMNOPQRSTUV = wasm_u8x16_max(voutGHIJKLMNOPQRSTUV, voutput_min);
95
96 vout0123456789ABCDEF = wasm_u8x16_min(vout0123456789ABCDEF, voutput_max);
97 voutGHIJKLMNOPQRSTUV = wasm_u8x16_min(voutGHIJKLMNOPQRSTUV, voutput_max);
98
99 wasm_v128_store(output, vout0123456789ABCDEF);
100 wasm_v128_store(output + 16, voutGHIJKLMNOPQRSTUV);
101 output += 32;
102 }
103 if XNN_UNLIKELY(batch != 0) {
104 do {
105 const v128_t va01234567 = wasm_u16x8_load8x8(input_a);
106 const v128_t vb01234567 = wasm_u16x8_load8x8(input_b);
107 input_a += 8;
108 input_b += 8;
109
110 v128_t vacc0123 = wasm_i32x4_add(vbias, wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(va01234567), va_multiplier));
111 v128_t vacc4567 = wasm_i32x4_add(vbias, wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(va01234567), va_multiplier));
112
113 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_mul(wasm_u32x4_extend_low_u16x8(vb01234567), vb_multiplier));
114 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_mul(wasm_u32x4_extend_high_u16x8(vb01234567), vb_multiplier));
115
116 vacc0123 = wasm_i32x4_shr(vacc0123, vshift);
117 vacc4567 = wasm_i32x4_shr(vacc4567, vshift);
118
119 v128_t vout01234567 = wasm_i16x8_add_sat(wasm_i16x8_narrow_i32x4(vacc0123, vacc4567), voutput_zero_point);
120
121 v128_t vout0123456701234567 = wasm_u8x16_narrow_i16x8(vout01234567, vout01234567);
122 vout0123456701234567 = wasm_u8x16_max(vout0123456701234567, voutput_min);
123 vout0123456701234567 = wasm_u8x16_min(vout0123456701234567, voutput_max);
124
125 if XNN_LIKELY(batch >= (8 * sizeof(uint8_t))) {
126 wasm_v128_store64_lane(output, vout0123456701234567, 0);
127 output += 8;
128 batch -= 8 * sizeof(uint8_t);
129 } else {
130 if (batch & (4 * sizeof(uint8_t))) {
131 wasm_v128_store32_lane(output, vout0123456701234567, 0);
132 vout0123456701234567 = wasm_u64x2_shr(vout0123456701234567, 32);
133 output += 4;
134 }
135 if (batch & (2 * sizeof(uint8_t))) {
136 wasm_v128_store16_lane(output, vout0123456701234567, 0);
137 vout0123456701234567 = wasm_u32x4_shr(vout0123456701234567, 16);
138 output += 2;
139 }
140 if (batch & (1 * sizeof(uint8_t))) {
141 wasm_v128_store8_lane(output, vout0123456701234567, 0);
142 }
143 batch = 0;
144 }
145 } while (batch != 0);
146 }
147 }