Debian Codegraph @

libclc/clc/lib/clspv/math/clc_sw_fma.cl

raw
1 //===----------------------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 // This version is derived from the generic fma software implementation
10 // (__clc_sw_fma), but avoids the use of ulong in favor of uint2. The logic has
11 // been updated as appropriate.
12
13 #include <clc/clc_as_type.h>
14 #include <clc/clcmacro.h>
15 #include <clc/float/definitions.h>
16 #include <clc/integer/clc_abs.h>
17 #include <clc/integer/clc_clz.h>
18 #include <clc/integer/clc_hadd.h>
19 #include <clc/integer/clc_mul_hi.h>
20 #include <clc/integer/definitions.h>
21 #include <clc/math/clc_mad.h>
22 #include <clc/math/math.h>
23 #include <clc/relational/clc_isinf.h>
24 #include <clc/relational/clc_isnan.h>
25 #include <clc/shared/clc_max.h>
26
27 struct fp {
28 uint2 mantissa;
29 int exponent;
30 uint sign;
31 };
32
33 static uint2 u2_set(uint hi, uint lo) {
34 uint2 res;
35 res.lo = lo;
36 res.hi = hi;
37 return res;
38 }
39
40 static uint2 u2_set_u(uint val) { return u2_set(0, val); }
41
42 static uint2 u2_mul(uint a, uint b) {
43 uint2 res;
44 res.hi = __clc_mul_hi(a, b);
45 res.lo = a * b;
46 return res;
47 }
48
49 static uint2 u2_sll(uint2 val, uint shift) {
50 if (shift == 0)
51 return val;
52 if (shift < 32) {
53 val.hi <<= shift;
54 val.hi |= val.lo >> (32 - shift);
55 val.lo <<= shift;
56 } else {
57 val.hi = val.lo << (shift - 32);
58 val.lo = 0;
59 }
60 return val;
61 }
62
63 static uint2 u2_srl(uint2 val, uint shift) {
64 if (shift == 0)
65 return val;
66 if (shift < 32) {
67 val.lo >>= shift;
68 val.lo |= val.hi << (32 - shift);
69 val.hi >>= shift;
70 } else {
71 val.lo = val.hi >> (shift - 32);
72 val.hi = 0;
73 }
74 return val;
75 }
76
77 static uint2 u2_or(uint2 a, uint b) {
78 a.lo |= b;
79 return a;
80 }
81
82 static uint2 u2_and(uint2 a, uint2 b) {
83 a.lo &= b.lo;
84 a.hi &= b.hi;
85 return a;
86 }
87
88 static uint2 u2_add(uint2 a, uint2 b) {
89 uint carry = (__clc_hadd(a.lo, b.lo) >> 31) & 0x1;
90 a.lo += b.lo;
91 a.hi += b.hi + carry;
92 return a;
93 }
94
95 static uint2 u2_add_u(uint2 a, uint b) { return u2_add(a, u2_set_u(b)); }
96
97 static uint2 u2_inv(uint2 a) {
98 a.lo = ~a.lo;
99 a.hi = ~a.hi;
100 return u2_add_u(a, 1);
101 }
102
103 static uint u2_clz(uint2 a) {
104 uint leading_zeroes = __clc_clz(a.hi);
105 if (leading_zeroes == 32) {
106 leading_zeroes += __clc_clz(a.lo);
107 }
108 return leading_zeroes;
109 }
110
111 static bool u2_eq(uint2 a, uint2 b) { return a.lo == b.lo && a.hi == b.hi; }
112
113 static bool u2_zero(uint2 a) { return u2_eq(a, u2_set_u(0)); }
114
115 static bool u2_gt(uint2 a, uint2 b) {
116 return a.hi > b.hi || (a.hi == b.hi && a.lo > b.lo);
117 }
118
119 _CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {
120 /* special cases */
121 if (__clc_isnan(a) || __clc_isnan(b) || __clc_isnan(c) || __clc_isinf(a) ||
122 __clc_isinf(b)) {
123 return __clc_mad(a, b, c);
124 }
125
126 /* If only c is inf, and both a,b are regular numbers, the result is c*/
127 if (__clc_isinf(c)) {
128 return c;
129 }
130
131 a = __clc_flush_denormal_if_not_supported(a);
132 b = __clc_flush_denormal_if_not_supported(b);
133 c = __clc_flush_denormal_if_not_supported(c);
134
135 if (a == 0.0f || b == 0.0f) {
136 return c;
137 }
138
139 if (c == 0) {
140 return a * b;
141 }
142
143 struct fp st_a, st_b, st_c;
144
145 st_a.exponent = a == .0f ? 0 : ((__clc_as_uint(a) & 0x7f800000) >> 23) - 127;
146 st_b.exponent = b == .0f ? 0 : ((__clc_as_uint(b) & 0x7f800000) >> 23) - 127;
147 st_c.exponent = c == .0f ? 0 : ((__clc_as_uint(c) & 0x7f800000) >> 23) - 127;
148
149 st_a.mantissa =
150 u2_set_u(a == .0f ? 0 : (__clc_as_uint(a) & 0x7fffff) | 0x800000);
151 st_b.mantissa =
152 u2_set_u(b == .0f ? 0 : (__clc_as_uint(b) & 0x7fffff) | 0x800000);
153 st_c.mantissa =
154 u2_set_u(c == .0f ? 0 : (__clc_as_uint(c) & 0x7fffff) | 0x800000);
155
156 st_a.sign = __clc_as_uint(a) & 0x80000000;
157 st_b.sign = __clc_as_uint(b) & 0x80000000;
158 st_c.sign = __clc_as_uint(c) & 0x80000000;
159
160 // Multiplication.
161 // Move the product to the highest bits to maximize precision
162 // mantissa is 24 bits => product is 48 bits, 2bits non-fraction.
163 // Add one bit for future addition overflow,
164 // add another bit to detect subtraction underflow
165 struct fp st_mul;
166 st_mul.sign = st_a.sign ^ st_b.sign;
167 st_mul.mantissa = u2_sll(u2_mul(st_a.mantissa.lo, st_b.mantissa.lo), 14);
168 st_mul.exponent =
169 !u2_zero(st_mul.mantissa) ? st_a.exponent + st_b.exponent : 0;
170
171 // FIXME: Detecting a == 0 || b == 0 above crashed GCN isel
172 if (st_mul.exponent == 0 && u2_zero(st_mul.mantissa))
173 return c;
174
175 // Mantissa is 23 fractional bits, shift it the same way as product mantissa
176 #define C_ADJUST 37ul
177
178 // both exponents are bias adjusted
179 int exp_diff = st_mul.exponent - st_c.exponent;
180
181 st_c.mantissa = u2_sll(st_c.mantissa, C_ADJUST);
182 uint2 cutoff_bits = u2_set_u(0);
183 uint2 cutoff_mask = u2_add(u2_sll(u2_set_u(1), __clc_abs(exp_diff)),
184 u2_set(0xffffffff, 0xffffffff));
185 if (exp_diff > 0) {
186 cutoff_bits =
187 exp_diff >= 64 ? st_c.mantissa : u2_and(st_c.mantissa, cutoff_mask);
188 st_c.mantissa =
189 exp_diff >= 64 ? u2_set_u(0) : u2_srl(st_c.mantissa, exp_diff);
190 } else {
191 cutoff_bits = -exp_diff >= 64 ? st_mul.mantissa
192 : u2_and(st_mul.mantissa, cutoff_mask);
193 st_mul.mantissa =
194 -exp_diff >= 64 ? u2_set_u(0) : u2_srl(st_mul.mantissa, -exp_diff);
195 }
196
197 struct fp st_fma;
198 st_fma.sign = st_mul.sign;
199 st_fma.exponent = __clc_max(st_mul.exponent, st_c.exponent);
200 if (st_c.sign == st_mul.sign) {
201 st_fma.mantissa = u2_add(st_mul.mantissa, st_c.mantissa);
202 } else {
203 // cutoff bits borrow one
204 st_fma.mantissa =
205 u2_add(u2_add(st_mul.mantissa, u2_inv(st_c.mantissa)),
206 (!u2_zero(cutoff_bits) && (st_mul.exponent > st_c.exponent)
207 ? u2_set(0xffffffff, 0xffffffff)
208 : u2_set_u(0)));
209 }
210
211 // underflow: st_c.sign != st_mul.sign, and magnitude switches the sign
212 if (u2_gt(st_fma.mantissa, u2_set(0x7fffffff, 0xffffffff))) {
213 st_fma.mantissa = u2_inv(st_fma.mantissa);
214 st_fma.sign = st_mul.sign ^ 0x80000000;
215 }
216
217 // detect overflow/underflow
218 int overflow_bits = 3 - u2_clz(st_fma.mantissa);
219
220 // adjust exponent
221 st_fma.exponent += overflow_bits;
222
223 // handle underflow
224 if (overflow_bits < 0) {
225 st_fma.mantissa = u2_sll(st_fma.mantissa, -overflow_bits);
226 overflow_bits = 0;
227 }
228
229 // rounding
230 uint2 trunc_mask = u2_add(u2_sll(u2_set_u(1), C_ADJUST + overflow_bits),
231 u2_set(0xffffffff, 0xffffffff));
232 uint2 trunc_bits =
233 u2_or(u2_and(st_fma.mantissa, trunc_mask), !u2_zero(cutoff_bits));
234 uint2 last_bit =
235 u2_and(st_fma.mantissa, u2_sll(u2_set_u(1), C_ADJUST + overflow_bits));
236 uint2 grs_bits = u2_sll(u2_set_u(4), C_ADJUST - 3 + overflow_bits);
237
238 // round to nearest even
239 if (u2_gt(trunc_bits, grs_bits) ||
240 (u2_eq(trunc_bits, grs_bits) && !u2_zero(last_bit))) {
241 st_fma.mantissa =
242 u2_add(st_fma.mantissa, u2_sll(u2_set_u(1), C_ADJUST + overflow_bits));
243 }
244
245 // Shift mantissa back to bit 23
246 st_fma.mantissa = u2_srl(st_fma.mantissa, C_ADJUST + overflow_bits);
247
248 // Detect rounding overflow
249 if (u2_gt(st_fma.mantissa, u2_set_u(0xffffff))) {
250 ++st_fma.exponent;
251 st_fma.mantissa = u2_srl(st_fma.mantissa, 1);
252 }
253
254 if (u2_zero(st_fma.mantissa)) {
255 return 0.0f;
256 }
257
258 // Flating point range limit
259 if (st_fma.exponent > 127) {
260 return __clc_as_float(__clc_as_uint(INFINITY) | st_fma.sign);
261 }
262
263 // Flush denormals
264 if (st_fma.exponent <= -127) {
265 return __clc_as_float(st_fma.sign);
266 }
267
268 return __clc_as_float(st_fma.sign | ((st_fma.exponent + 127) << 23) |
269 ((uint)st_fma.mantissa.lo & 0x7fffff));
270 }
271
272 #define __FLOAT_ONLY
273 #define FUNCTION __clc_sw_fma
274 #define __CLC_BODY <clc/shared/ternary_def_scalarize.inc>
275 #include <clc/math/gentype.inc>