Debian Codegraph @

libclc/clc/lib/clspv/math/clc_sw_fma.cl

raw
1 //===----------------------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 // This version is derived from the generic fma software implementation
10 // (__clc_sw_fma), but avoids the use of ulong in favor of uint2. The logic has
11 // been updated as appropriate.
12
13 #include <clc/clc_as_type.h>
14 #include <clc/float/definitions.h>
15 #include <clc/integer/clc_abs.h>
16 #include <clc/integer/clc_clz.h>
17 #include <clc/integer/clc_hadd.h>
18 #include <clc/integer/clc_mul_hi.h>
19 #include <clc/integer/definitions.h>
20 #include <clc/math/clc_mad.h>
21 #include <clc/math/math.h>
22 #include <clc/relational/clc_isinf.h>
23 #include <clc/relational/clc_isnan.h>
24 #include <clc/shared/clc_max.h>
25
26 struct fp {
27 uint2 mantissa;
28 int exponent;
29 uint sign;
30 };
31
32 static uint2 u2_set(uint hi, uint lo) {
33 uint2 res;
34 res.lo = lo;
35 res.hi = hi;
36 return res;
37 }
38
39 static uint2 u2_set_u(uint val) { return u2_set(0, val); }
40
41 static uint2 u2_mul(uint a, uint b) {
42 uint2 res;
43 res.hi = __clc_mul_hi(a, b);
44 res.lo = a * b;
45 return res;
46 }
47
48 static uint2 u2_sll(uint2 val, uint shift) {
49 if (shift == 0)
50 return val;
51 if (shift < 32) {
52 val.hi <<= shift;
53 val.hi |= val.lo >> (32 - shift);
54 val.lo <<= shift;
55 } else {
56 val.hi = val.lo << (shift - 32);
57 val.lo = 0;
58 }
59 return val;
60 }
61
62 static uint2 u2_srl(uint2 val, uint shift) {
63 if (shift == 0)
64 return val;
65 if (shift < 32) {
66 val.lo >>= shift;
67 val.lo |= val.hi << (32 - shift);
68 val.hi >>= shift;
69 } else {
70 val.lo = val.hi >> (shift - 32);
71 val.hi = 0;
72 }
73 return val;
74 }
75
76 static uint2 u2_or(uint2 a, uint b) {
77 a.lo |= b;
78 return a;
79 }
80
81 static uint2 u2_and(uint2 a, uint2 b) {
82 a.lo &= b.lo;
83 a.hi &= b.hi;
84 return a;
85 }
86
87 static uint2 u2_add(uint2 a, uint2 b) {
88 uint carry = (__clc_hadd(a.lo, b.lo) >> 31) & 0x1;
89 a.lo += b.lo;
90 a.hi += b.hi + carry;
91 return a;
92 }
93
94 static uint2 u2_add_u(uint2 a, uint b) { return u2_add(a, u2_set_u(b)); }
95
96 static uint2 u2_inv(uint2 a) {
97 a.lo = ~a.lo;
98 a.hi = ~a.hi;
99 return u2_add_u(a, 1);
100 }
101
102 static uint u2_clz(uint2 a) {
103 uint leading_zeroes = __clc_clz(a.hi);
104 if (leading_zeroes == 32) {
105 leading_zeroes += __clc_clz(a.lo);
106 }
107 return leading_zeroes;
108 }
109
110 static bool u2_eq(uint2 a, uint2 b) { return a.lo == b.lo && a.hi == b.hi; }
111
112 static bool u2_zero(uint2 a) { return u2_eq(a, u2_set_u(0)); }
113
114 static bool u2_gt(uint2 a, uint2 b) {
115 return a.hi > b.hi || (a.hi == b.hi && a.lo > b.lo);
116 }
117
118 _CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {
119 /* special cases */
120 if (__clc_isnan(a) || __clc_isnan(b) || __clc_isnan(c) || __clc_isinf(a) ||
121 __clc_isinf(b)) {
122 return __clc_mad(a, b, c);
123 }
124
125 /* If only c is inf, and both a,b are regular numbers, the result is c*/
126 if (__clc_isinf(c)) {
127 return c;
128 }
129
130 a = __clc_flush_denormal_if_not_supported(a);
131 b = __clc_flush_denormal_if_not_supported(b);
132 c = __clc_flush_denormal_if_not_supported(c);
133
134 if (a == 0.0f || b == 0.0f) {
135 return c;
136 }
137
138 if (c == 0) {
139 return a * b;
140 }
141
142 struct fp st_a, st_b, st_c;
143
144 st_a.exponent = a == .0f ? 0 : ((__clc_as_uint(a) & 0x7f800000) >> 23) - 127;
145 st_b.exponent = b == .0f ? 0 : ((__clc_as_uint(b) & 0x7f800000) >> 23) - 127;
146 st_c.exponent = c == .0f ? 0 : ((__clc_as_uint(c) & 0x7f800000) >> 23) - 127;
147
148 st_a.mantissa =
149 u2_set_u(a == .0f ? 0 : (__clc_as_uint(a) & 0x7fffff) | 0x800000);
150 st_b.mantissa =
151 u2_set_u(b == .0f ? 0 : (__clc_as_uint(b) & 0x7fffff) | 0x800000);
152 st_c.mantissa =
153 u2_set_u(c == .0f ? 0 : (__clc_as_uint(c) & 0x7fffff) | 0x800000);
154
155 st_a.sign = __clc_as_uint(a) & 0x80000000;
156 st_b.sign = __clc_as_uint(b) & 0x80000000;
157 st_c.sign = __clc_as_uint(c) & 0x80000000;
158
159 // Multiplication.
160 // Move the product to the highest bits to maximize precision
161 // mantissa is 24 bits => product is 48 bits, 2bits non-fraction.
162 // Add one bit for future addition overflow,
163 // add another bit to detect subtraction underflow
164 struct fp st_mul;
165 st_mul.sign = st_a.sign ^ st_b.sign;
166 st_mul.mantissa = u2_sll(u2_mul(st_a.mantissa.lo, st_b.mantissa.lo), 14);
167 st_mul.exponent =
168 !u2_zero(st_mul.mantissa) ? st_a.exponent + st_b.exponent : 0;
169
170 // FIXME: Detecting a == 0 || b == 0 above crashed GCN isel
171 if (st_mul.exponent == 0 && u2_zero(st_mul.mantissa))
172 return c;
173
174 // Mantissa is 23 fractional bits, shift it the same way as product mantissa
175 #define C_ADJUST 37ul
176
177 // both exponents are bias adjusted
178 int exp_diff = st_mul.exponent - st_c.exponent;
179
180 st_c.mantissa = u2_sll(st_c.mantissa, C_ADJUST);
181 uint2 cutoff_bits = u2_set_u(0);
182 uint2 cutoff_mask = u2_add(u2_sll(u2_set_u(1), __clc_abs(exp_diff)),
183 u2_set(0xffffffff, 0xffffffff));
184 if (exp_diff > 0) {
185 cutoff_bits =
186 exp_diff >= 64 ? st_c.mantissa : u2_and(st_c.mantissa, cutoff_mask);
187 st_c.mantissa =
188 exp_diff >= 64 ? u2_set_u(0) : u2_srl(st_c.mantissa, exp_diff);
189 } else {
190 cutoff_bits = -exp_diff >= 64 ? st_mul.mantissa
191 : u2_and(st_mul.mantissa, cutoff_mask);
192 st_mul.mantissa =
193 -exp_diff >= 64 ? u2_set_u(0) : u2_srl(st_mul.mantissa, -exp_diff);
194 }
195
196 struct fp st_fma;
197 st_fma.sign = st_mul.sign;
198 st_fma.exponent = __clc_max(st_mul.exponent, st_c.exponent);
199 if (st_c.sign == st_mul.sign) {
200 st_fma.mantissa = u2_add(st_mul.mantissa, st_c.mantissa);
201 } else {
202 // cutoff bits borrow one
203 st_fma.mantissa =
204 u2_add(u2_add(st_mul.mantissa, u2_inv(st_c.mantissa)),
205 (!u2_zero(cutoff_bits) && (st_mul.exponent > st_c.exponent)
206 ? u2_set(0xffffffff, 0xffffffff)
207 : u2_set_u(0)));
208 }
209
210 // underflow: st_c.sign != st_mul.sign, and magnitude switches the sign
211 if (u2_gt(st_fma.mantissa, u2_set(0x7fffffff, 0xffffffff))) {
212 st_fma.mantissa = u2_inv(st_fma.mantissa);
213 st_fma.sign = st_mul.sign ^ 0x80000000;
214 }
215
216 // detect overflow/underflow
217 int overflow_bits = 3 - u2_clz(st_fma.mantissa);
218
219 // adjust exponent
220 st_fma.exponent += overflow_bits;
221
222 // handle underflow
223 if (overflow_bits < 0) {
224 st_fma.mantissa = u2_sll(st_fma.mantissa, -overflow_bits);
225 overflow_bits = 0;
226 }
227
228 // rounding
229 uint2 trunc_mask = u2_add(u2_sll(u2_set_u(1), C_ADJUST + overflow_bits),
230 u2_set(0xffffffff, 0xffffffff));
231 uint2 trunc_bits =
232 u2_or(u2_and(st_fma.mantissa, trunc_mask), !u2_zero(cutoff_bits));
233 uint2 last_bit =
234 u2_and(st_fma.mantissa, u2_sll(u2_set_u(1), C_ADJUST + overflow_bits));
235 uint2 grs_bits = u2_sll(u2_set_u(4), C_ADJUST - 3 + overflow_bits);
236
237 // round to nearest even
238 if (u2_gt(trunc_bits, grs_bits) ||
239 (u2_eq(trunc_bits, grs_bits) && !u2_zero(last_bit))) {
240 st_fma.mantissa =
241 u2_add(st_fma.mantissa, u2_sll(u2_set_u(1), C_ADJUST + overflow_bits));
242 }
243
244 // Shift mantissa back to bit 23
245 st_fma.mantissa = u2_srl(st_fma.mantissa, C_ADJUST + overflow_bits);
246
247 // Detect rounding overflow
248 if (u2_gt(st_fma.mantissa, u2_set_u(0xffffff))) {
249 ++st_fma.exponent;
250 st_fma.mantissa = u2_srl(st_fma.mantissa, 1);
251 }
252
253 if (u2_zero(st_fma.mantissa)) {
254 return 0.0f;
255 }
256
257 // Flating point range limit
258 if (st_fma.exponent > 127) {
259 return __clc_as_float(__clc_as_uint(INFINITY) | st_fma.sign);
260 }
261
262 // Flush denormals
263 if (st_fma.exponent <= -127) {
264 return __clc_as_float(st_fma.sign);
265 }
266
267 return __clc_as_float(st_fma.sign | ((st_fma.exponent + 127) << 23) |
268 ((uint)st_fma.mantissa.lo & 0x7fffff));
269 }
270
271 #define __CLC_FLOAT_ONLY
272 #define __CLC_FUNCTION __clc_sw_fma
273 #define __CLC_BODY <clc/shared/ternary_def_scalarize.inc>
274 #include <clc/math/gentype.inc>