Debian Codegraph @

src/f32-ppmm/gen/f32-ppmm-4x8-minmax-wasmsimd-x86-splat.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/f32-ppmm/wasmsimd-splat.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include "xnnpack/ppmm.h"
15
16
17 void xnn_f32_ppmm_minmax_ukernel_4x8__wasmsimd_x86_splat(
18 size_t mr,
19 size_t nc,
20 size_t kc,
21 const float* restrict a,
22 const float* restrict w,
23 float* restrict c,
24 size_t cm_stride,
25 size_t cn_stride,
26 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28 assert(mr != 0);
29 assert(mr <= 4);
30 assert(nc != 0);
31 assert(kc != 0);
32 assert(kc % sizeof(float) == 0);
33
34 float* c0 = c;
35 float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
36 if XNN_UNPREDICTABLE(mr < 2) {
37 c1 = c0;
38 }
39 float* c2 = (float*) ((uintptr_t) c1 + cm_stride);
40 if XNN_UNPREDICTABLE(mr <= 2) {
41 c2 = c1;
42 }
43 float* c3 = (float*) ((uintptr_t) c2 + cm_stride);
44 if XNN_UNPREDICTABLE(mr != 4) {
45 c3 = c2;
46 }
47
48 const v128_t vmin = wasm_v128_load32_splat(&params->scalar.min);
49 const v128_t vmax = wasm_v128_load32_splat(&params->scalar.max);
50 XNN_FORCE_REALIZATION(vmin);
51 XNN_FORCE_REALIZATION(vmax);
52 do {
53 v128_t vacc0x0123 = wasm_v128_load(w);
54 v128_t vacc0x4567 = wasm_v128_load(w + 4);
55 v128_t vacc1x0123 = vacc0x0123;
56 v128_t vacc1x4567 = vacc0x4567;
57 v128_t vacc2x0123 = vacc0x0123;
58 v128_t vacc2x4567 = vacc0x4567;
59 v128_t vacc3x0123 = vacc0x0123;
60 v128_t vacc3x4567 = vacc0x4567;
61 w += 8;
62
63 size_t k = kc;
64 do {
65 const v128_t va0123 = wasm_v128_load(a);
66 a += 4;
67
68 const v128_t vb0123 = wasm_v128_load(w);
69 const v128_t vb4567 = wasm_v128_load(w + 4);
70 w += 8;
71
72 const v128_t va0000 = wasm_v32x4_shuffle(va0123, va0123, 0, 0, 0, 0);
73 const v128_t va1111 = wasm_v32x4_shuffle(va0123, va0123, 1, 1, 1, 1);
74 const v128_t va2222 = wasm_v32x4_shuffle(va0123, va0123, 2, 2, 2, 2);
75 const v128_t va3333 = wasm_v32x4_shuffle(va0123, va0123, 3, 3, 3, 3);
76
77 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(va0000, vb0123));
78 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(va1111, vb0123));
79 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(va2222, vb0123));
80 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(va3333, vb0123));
81 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(va0000, vb4567));
82 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(va1111, vb4567));
83 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(va2222, vb4567));
84 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(va3333, vb4567));
85
86 k -= sizeof(float);
87 } while (k != 0);
88
89 vacc0x0123 = wasm_f32x4_pmax(vmin, vacc0x0123);
90 vacc1x0123 = wasm_f32x4_pmax(vmin, vacc1x0123);
91 vacc2x0123 = wasm_f32x4_pmax(vmin, vacc2x0123);
92 vacc3x0123 = wasm_f32x4_pmax(vmin, vacc3x0123);
93 vacc0x4567 = wasm_f32x4_pmax(vmin, vacc0x4567);
94 vacc1x4567 = wasm_f32x4_pmax(vmin, vacc1x4567);
95 vacc2x4567 = wasm_f32x4_pmax(vmin, vacc2x4567);
96 vacc3x4567 = wasm_f32x4_pmax(vmin, vacc3x4567);
97
98 vacc0x0123 = wasm_f32x4_pmin(vmax, vacc0x0123);
99 vacc1x0123 = wasm_f32x4_pmin(vmax, vacc1x0123);
100 vacc2x0123 = wasm_f32x4_pmin(vmax, vacc2x0123);
101 vacc3x0123 = wasm_f32x4_pmin(vmax, vacc3x0123);
102 vacc0x4567 = wasm_f32x4_pmin(vmax, vacc0x4567);
103 vacc1x4567 = wasm_f32x4_pmin(vmax, vacc1x4567);
104 vacc2x4567 = wasm_f32x4_pmin(vmax, vacc2x4567);
105 vacc3x4567 = wasm_f32x4_pmin(vmax, vacc3x4567);
106
107 if XNN_LIKELY(nc >= 8) {
108 wasm_v128_store(c3, vacc3x0123);
109 wasm_v128_store(c3 + 4, vacc3x4567);
110 wasm_v128_store(c2, vacc2x0123);
111 wasm_v128_store(c2 + 4, vacc2x4567);
112 wasm_v128_store(c1, vacc1x0123);
113 wasm_v128_store(c1 + 4, vacc1x4567);
114 wasm_v128_store(c0, vacc0x0123);
115 wasm_v128_store(c0 + 4, vacc0x4567);
116
117 a = (const float*) ((uintptr_t) a - kc * 4);
118
119 c3 = (float*) ((uintptr_t) c3 + cn_stride);
120 c2 = (float*) ((uintptr_t) c2 + cn_stride);
121 c1 = (float*) ((uintptr_t) c1 + cn_stride);
122 c0 = (float*) ((uintptr_t) c0 + cn_stride);
123
124 nc -= 8;
125 } else {
126 if (nc & 4) {
127 wasm_v128_store(c3, vacc3x0123);
128 wasm_v128_store(c2, vacc2x0123);
129 wasm_v128_store(c1, vacc1x0123);
130 wasm_v128_store(c0, vacc0x0123);
131
132 vacc3x0123 = vacc3x4567;
133 vacc2x0123 = vacc2x4567;
134 vacc1x0123 = vacc1x4567;
135 vacc0x0123 = vacc0x4567;
136
137 c3 += 4;
138 c2 += 4;
139 c1 += 4;
140 c0 += 4;
141 }
142 if (nc & 2) {
143 wasm_v128_store64_lane(c3, vacc3x0123, 0);
144 wasm_v128_store64_lane(c2, vacc2x0123, 0);
145 wasm_v128_store64_lane(c1, vacc1x0123, 0);
146 wasm_v128_store64_lane(c0, vacc0x0123, 0);
147
148 vacc3x0123 = wasm_v64x2_shuffle(vacc3x0123, vacc3x0123, 1, 1);
149 vacc2x0123 = wasm_v64x2_shuffle(vacc2x0123, vacc2x0123, 1, 1);
150 vacc1x0123 = wasm_v64x2_shuffle(vacc1x0123, vacc1x0123, 1, 1);
151 vacc0x0123 = wasm_v64x2_shuffle(vacc0x0123, vacc0x0123, 1, 1);
152
153 c3 += 2;
154 c2 += 2;
155 c1 += 2;
156 c0 += 2;
157 }
158 if (nc & 1) {
159 wasm_v128_store32_lane(c3, vacc3x0123, 0);
160 wasm_v128_store32_lane(c2, vacc2x0123, 0);
161 wasm_v128_store32_lane(c1, vacc1x0123, 0);
162 wasm_v128_store32_lane(c0, vacc0x0123, 0);
163 }
164
165 nc = 0;
166 }
167 } while (nc != 0);
168 }