Debian Codegraph @

src/gallium/drivers/nouveau/codegen/lib/gm107.asm

raw
1 .section #gm107_builtin_code
2 // DIV U32
3 //
4 // UNR recurrence (q = a / b):
5 // look for z such that 2^32 - b <= b * z < 2^32
6 // then q - 1 <= (a * z) / 2^32 <= q
7 //
8 // INPUT: $r0: dividend, $r1: divisor
9 // OUTPUT: $r0: result, $r1: modulus
10 // CLOBBER: $r2 - $r3, $p0 - $p1
11 // SIZE: 22 / 14 * 8 bytes
12 //
13 gm107_div_u32:
14 sched (st 0xd wr 0x0 wt 0x3f) (st 0x1 wt 0x1) (st 0x6)
15 flo u32 $r2 $r1
16 lop xor 1 $r2 $r2 0x1f
17 mov $r3 0x1 0xf
18 sched (st 0x1) (st 0xf wr 0x0) (st 0x6 wr 0x0 wt 0x1)
19 shl $r2 $r3 $r2
20 i2i u32 u32 $r1 neg $r1
21 imul u32 u32 $r3 $r1 $r2
22 sched (st 0x6 wr 0x0 wt 0x1) (st 0x6 wr 0x0 wt 0x1) (st 0x6 wr 0x0 wt 0x1)
23 imad u32 u32 hi $r2 $r2 $r3 $r2
24 imul u32 u32 $r3 $r1 $r2
25 imad u32 u32 hi $r2 $r2 $r3 $r2
26 sched (st 0x6 wr 0x0 wt 0x1) (st 0x6 wr 0x0 wt 0x1) (st 0x6 wr 0x0 wt 0x1)
27 imul u32 u32 $r3 $r1 $r2
28 imad u32 u32 hi $r2 $r2 $r3 $r2
29 imul u32 u32 $r3 $r1 $r2
30 sched (st 0x6 wr 0x0 wt 0x1) (st 0x6 wr 0x0 wt 0x1) (st 0x6 wr 0x0 rd 0x1 wt 0x1)
31 imad u32 u32 hi $r2 $r2 $r3 $r2
32 imul u32 u32 $r3 $r1 $r2
33 imad u32 u32 hi $r2 $r2 $r3 $r2
34 sched (st 0x6 wt 0x2) (st 0x6 wr 0x0 rd 0x1 wt 0x1) (st 0xf wr 0x0 rd 0x1 wt 0x2)
35 mov $r3 $r0 0xf
36 imul u32 u32 hi $r0 $r0 $r2
37 i2i u32 u32 $r2 neg $r1
38 sched (st 0x6 wr 0x0 wt 0x3) (st 0xd wt 0x1) (st 0x1)
39 imad u32 u32 $r1 $r1 $r0 $r3
40 isetp ge u32 and $p0 1 $r1 $r2 1
41 $p0 iadd $r1 $r1 neg $r2
42 sched (st 0x5) (st 0xd) (st 0x1)
43 $p0 iadd $r0 $r0 0x1
44 $p0 isetp ge u32 and $p0 1 $r1 $r2 1
45 $p0 iadd $r1 $r1 neg $r2
46 sched (st 0x1) (st 0xf) (st 0xf)
47 $p0 iadd $r0 $r0 0x1
48 ret
49 nop 0
50
51 // DIV S32, like DIV U32 after taking ABS(inputs)
52 //
53 // INPUT: $r0: dividend, $r1: divisor
54 // OUTPUT: $r0: result, $r1: modulus
55 // CLOBBER: $r2 - $r3, $p0 - $p3
56 //
57 gm107_div_s32:
58 sched (st 0xd wt 0x3f) (st 0x1) (st 0x1 wr 0x0)
59 isetp lt and $p2 0x1 $r0 0 1
60 isetp lt xor $p3 1 $r1 0 $p2
61 i2i s32 s32 $r0 abs $r0
62 sched (st 0xf wr 0x1) (st 0xd wr 0x1 wt 0x2) (st 0x1 wt 0x2)
63 i2i s32 s32 $r1 abs $r1
64 flo u32 $r2 $r1
65 lop xor 1 $r2 $r2 0x1f
66 sched (st 0x6) (st 0x1) (st 0xf wr 0x1)
67 mov $r3 0x1 0xf
68 shl $r2 $r3 $r2
69 i2i u32 u32 $r1 neg $r1
70 sched (st 0x6 wr 0x1 wt 0x2) (st 0x6 wr 0x1 wt 0x2) (st 0x6 wr 0x1 wt 0x2)
71 imul u32 u32 $r3 $r1 $r2
72 imad u32 u32 hi $r2 $r2 $r3 $r2
73 imul u32 u32 $r3 $r1 $r2
74 sched (st 0x6 wr 0x1 wt 0x2) (st 0x6 wr 0x1 wt 0x2) (st 0x6 wr 0x1 wt 0x2)
75 imad u32 u32 hi $r2 $r2 $r3 $r2
76 imul u32 u32 $r3 $r1 $r2
77 imad u32 u32 hi $r2 $r2 $r3 $r2
78 sched (st 0x6 wr 0x1 wt 0x2) (st 0x6 wr 0x1 wt 0x2) (st 0x6 wr 0x1 wt 0x2)
79 imul u32 u32 $r3 $r1 $r2
80 imad u32 u32 hi $r2 $r2 $r3 $r2
81 imul u32 u32 $r3 $r1 $r2
82 sched (st 0x6 wr 0x1 rd 0x2 wt 0x2) (st 0x2 wt 0x5) (st 0x6 wr 0x0 rd 0x1 wt 0x2)
83 imad u32 u32 hi $r2 $r2 $r3 $r2
84 mov $r3 $r0 0xf
85 imul u32 u32 hi $r0 $r0 $r2
86 sched (st 0xf wr 0x1 rd 0x2 wt 0x2) (st 0x6 wr 0x0 wt 0x5) (st 0xd wt 0x3)
87 i2i u32 u32 $r2 neg $r1
88 imad u32 u32 $r1 $r1 $r0 $r3
89 isetp ge u32 and $p0 1 $r1 $r2 1
90 sched (st 0x1) (st 0x5) (st 0xd)
91 $p0 iadd $r1 $r1 neg $r2
92 $p0 iadd $r0 $r0 0x1
93 $p0 isetp ge u32 and $p0 1 $r1 $r2 1
94 sched (st 0x1) (st 0x2) (st 0xf wr 0x0)
95 $p0 iadd $r1 $r1 neg $r2
96 $p0 iadd $r0 $r0 0x1
97 $p3 i2i s32 s32 $r0 neg $r0
98 sched (st 0xf wr 0x1) (st 0xf wt 0x3) (st 0xf)
99 $p2 i2i s32 s32 $r1 neg $r1
100 ret
101 nop 0
102
103 // RCP F64
104 //
105 // INPUT: $r0d
106 // OUTPUT: $r0d
107 // CLOBBER: $r2 - $r9, $p0
108 //
109 // The core of RCP and RSQ implementation is Newton-Raphson step, which is
110 // used to find successively better approximation from an imprecise initial
111 // value (single precision rcp in RCP and rsqrt64h in RSQ).
112 //
113 gm107_rcp_f64:
114 // Step 1: classify input according to exponent and value, and calculate
115 // result for 0/inf/nan. $r2 holds the exponent value, which starts at
116 // bit 52 (bit 20 of the upper half) and is 11 bits in length
117 sched (st 0x0) (st 0x0) (st 0x0)
118 bfe u32 $r2 $r1 0xb14
119 iadd32i $r3 $r2 -1
120 ssy #rcp_rejoin
121 // We want to check whether the exponent is 0 or 0x7ff (i.e. NaN, inf,
122 // denorm, or 0). Do this by subtracting 1 from the exponent, which will
123 // mean that it's > 0x7fd in those cases when doing unsigned comparison
124 sched (st 0x0) (st 0x0) (st 0x0)
125 isetp gt u32 and $p0 1 $r3 0x7fd 1
126 // $r3: 0 for norms, 0x36 for denorms, -1 for others
127 mov $r3 0x0 0xf
128 not $p0 sync
129 // Process all special values: NaN, inf, denorm, 0
130 sched (st 0x0) (st 0x0) (st 0x0)
131 mov32i $r3 0xffffffff 0xf
132 // A number is NaN if its abs value is greater than or unordered with inf
133 dsetp gtu and $p0 1 abs $r0 0x7ff0000000000000 1
134 not $p0 bra #rcp_inf_or_denorm_or_zero
135 // NaN -> NaN, the next line sets the "quiet" bit of the result. This
136 // behavior is both seen on the CPU and the blob
137 sched (st 0x0) (st 0x0) (st 0x0)
138 lop32i or $r1 $r1 0x80000
139 sync
140 rcp_inf_or_denorm_or_zero:
141 lop32i and $r4 $r1 0x7ff00000
142 sched (st 0x0) (st 0x0) (st 0x0)
143 // Other values with nonzero in exponent field should be inf
144 isetp eq and $p0 1 $r4 0x0 1
145 $p0 bra #rcp_denorm_or_zero
146 // +/-Inf -> +/-0
147 lop32i xor $r1 $r1 0x7ff00000
148 sched (st 0x0) (st 0x0) (st 0x0)
149 mov $r0 0x0 0xf
150 sync
151 rcp_denorm_or_zero:
152 dsetp gtu and $p0 1 abs $r0 0x0 1
153 sched (st 0x0) (st 0x0) (st 0x0)
154 $p0 bra #rcp_denorm
155 // +/-0 -> +/-Inf
156 lop32i or $r1 $r1 0x7ff00000
157 sync
158 rcp_denorm:
159 // non-0 denorms: multiply with 2^54 (the 0x36 in $r3), join with norms
160 sched (st 0x0) (st 0x0) (st 0x0)
161 dmul $r0 $r0 0x4350000000000000
162 mov $r3 0x36 0xf
163 sync
164 rcp_rejoin:
165 // All numbers with -1 in $r3 have their result ready in $r0d, return them
166 // others need further calculation
167 sched (st 0x0) (st 0x0) (st 0x0)
168 isetp lt and $p0 1 $r3 0x0 1
169 $p0 bra #rcp_end
170 // Step 2: Before the real calculation goes on, renormalize the values to
171 // range [1, 2) by setting exponent field to 0x3ff (the exponent of 1)
172 // result in $r6d. The exponent will be recovered later.
173 bfe u32 $r2 $r1 0xb14
174 sched (st 0x0) (st 0x0) (st 0x0)
175 lop32i and $r7 $r1 0x800fffff
176 iadd32i $r7 $r7 0x3ff00000
177 mov $r6 $r0 0xf
178 // Step 3: Convert new value to float (no overflow will occur due to step
179 // 2), calculate rcp and do newton-raphson step once
180 sched (st 0x0) (st 0x0) (st 0x0)
181 f2f ftz f64 f32 $r5 $r6
182 mufu rcp $r4 $r5
183 mov32i $r0 0xbf800000 0xf
184 sched (st 0x0) (st 0x0) (st 0x0)
185 ffma $r5 $r4 $r5 $r0
186 ffma $r0 $r5 neg $r4 $r4
187 // Step 4: convert result $r0 back to double, do newton-raphson steps
188 f2f f32 f64 $r0 $r0
189 sched (st 0x0) (st 0x0) (st 0x0)
190 f2f f64 f64 $r6 neg $r6
191 f2f f32 f64 $r8 0x3f800000
192 // 4 Newton-Raphson Steps, tmp in $r4d, result in $r0d
193 // The formula used here (and above) is:
194 // RCP_{n + 1} = 2 * RCP_{n} - x * RCP_{n} * RCP_{n}
195 // The following code uses 2 FMAs for each step, and it will basically
196 // looks like:
197 // tmp = -src * RCP_{n} + 1
198 // RCP_{n + 1} = RCP_{n} * tmp + RCP_{n}
199 dfma $r4 $r6 $r0 $r8
200 sched (st 0x0) (st 0x0) (st 0x0)
201 dfma $r0 $r0 $r4 $r0
202 dfma $r4 $r6 $r0 $r8
203 dfma $r0 $r0 $r4 $r0
204 sched (st 0x0) (st 0x0) (st 0x0)
205 dfma $r4 $r6 $r0 $r8
206 dfma $r0 $r0 $r4 $r0
207 dfma $r4 $r6 $r0 $r8
208 sched (st 0x0) (st 0x0) (st 0x0)
209 dfma $r0 $r0 $r4 $r0
210 // Step 5: Exponent recovery and final processing
211 // The exponent is recovered by adding what we added to the exponent.
212 // Suppose we want to calculate rcp(x), but we have rcp(cx), then
213 // rcp(x) = c * rcp(cx)
214 // The delta in exponent comes from two sources:
215 // 1) The renormalization in step 2. The delta is:
216 // 0x3ff - $r2
217 // 2) (For the denorm input) The 2^54 we multiplied at rcp_denorm, stored
218 // in $r3
219 // These 2 sources are calculated in the first two lines below, and then
220 // added to the exponent extracted from the result above.
221 // Note that after processing, the new exponent may >= 0x7ff (inf)
222 // or <= 0 (denorm). Those cases will be handled respectively below
223 iadd $r2 neg $r2 0x3ff
224 iadd $r4 $r2 $r3
225 sched (st 0x0) (st 0x0) (st 0x0)
226 bfe u32 $r3 $r1 0xb14
227 // New exponent in $r3
228 iadd $r3 $r3 $r4
229 iadd32i $r2 $r3 -1
230 // (exponent-1) < 0x7fe (unsigned) means the result is in norm range
231 // (same logic as in step 1)
232 sched (st 0x0) (st 0x0) (st 0x0)
233 isetp lt u32 and $p0 1 $r2 0x7fe 1
234 not $p0 bra #rcp_result_inf_or_denorm
235 // Norms: convert exponents back and return
236 shl $r4 $r4 0x14
237 sched (st 0x0) (st 0x0) (st 0x0)
238 iadd $r1 $r4 $r1
239 bra #rcp_end
240 rcp_result_inf_or_denorm:
241 // New exponent >= 0x7ff means that result is inf
242 isetp ge and $p0 1 $r3 0x7ff 1
243 sched (st 0x0) (st 0x0) (st 0x0)
244 not $p0 bra #rcp_result_denorm
245 // Infinity
246 lop32i and $r1 $r1 0x80000000
247 mov $r0 0x0 0xf
248 sched (st 0x0) (st 0x0) (st 0x0)
249 iadd32i $r1 $r1 0x7ff00000
250 bra #rcp_end
251 rcp_result_denorm:
252 // Denorm result comes from huge input. The greatest possible fp64, i.e.
253 // 0x7fefffffffffffff's rcp is 0x0004000000000000, 1/4 of the smallest
254 // normal value. Other rcp result should be greater than that. If we
255 // set the exponent field to 1, we can recover the result by multiplying
256 // it with 1/2 or 1/4. 1/2 is used if the "exponent" $r3 is 0, otherwise
257 // 1/4 ($r3 should be -1 then). This is quite tricky but greatly simplifies
258 // the logic here.
259 isetp ne u32 and $p0 1 $r3 0x0 1
260 sched (st 0x0) (st 0x0) (st 0x0)
261 lop32i and $r1 $r1 0x800fffff
262 // 0x3e800000: 1/4
263 $p0 f2f f32 f64 $r6 0x3e800000
264 // 0x3f000000: 1/2
265 not $p0 f2f f32 f64 $r6 0x3f000000
266 sched (st 0x0) (st 0x0) (st 0x0)
267 iadd32i $r1 $r1 0x00100000
268 dmul $r0 $r0 $r6
269 rcp_end:
270 ret
271
272 // RSQ F64
273 //
274 // INPUT: $r0d
275 // OUTPUT: $r0d
276 // CLOBBER: $r2 - $r9, $p0 - $p1
277 //
278 gm107_rsq_f64:
279 // Before getting initial result rsqrt64h, two special cases should be
280 // handled first.
281 // 1. NaN: set the highest bit in mantissa so it'll be surely recognized
282 // as NaN in rsqrt64h
283 sched (st 0xd wr 0x0 wt 0x3f) (st 0xd wt 0x1) (st 0xd)
284 dsetp gtu and $p0 1 abs $r0 0x7ff0000000000000 1
285 $p0 lop32i or $r1 $r1 0x00080000
286 lop32i and $r2 $r1 0x7fffffff
287 // 2. denorms and small normal values: using their original value will
288 // lose precision either at rsqrt64h or the first step in newton-raphson
289 // steps below. Take 2 as a threshold in exponent field, and multiply
290 // with 2^54 if the exponent is smaller or equal. (will multiply 2^27
291 // to recover in the end)
292 sched (st 0xd) (st 0xd) (st 0xd)
293 bfe u32 $r3 $r1 0xb14
294 isetp le u32 and $p1 1 $r3 0x2 1
295 lop or 1 $r2 $r0 $r2
296 sched (st 0xd wr 0x0) (st 0xd wr 0x0 wt 0x1) (st 0xd)
297 $p1 dmul $r0 $r0 0x4350000000000000
298 mufu rsq64h $r5 $r1
299 // rsqrt64h will give correct result for 0/inf/nan, the following logic
300 // checks whether the input is one of those (exponent is 0x7ff or all 0
301 // except for the sign bit)
302 iset ne u32 and $r6 $r3 0x7ff 1
303 sched (st 0xd) (st 0xd) (st 0xd)
304 lop and 1 $r2 $r2 $r6
305 isetp ne u32 and $p0 1 $r2 0x0 1
306 $p0 bra #rsq_norm
307 // For 0/inf/nan, make sure the sign bit agrees with input and return
308 sched (st 0xd) (st 0xd) (st 0xd wt 0x1)
309 lop32i and $r1 $r1 0x80000000
310 mov $r0 0x0 0xf
311 lop or 1 $r1 $r1 $r5
312 sched (st 0xd) (st 0xf) (st 0xf)
313 ret
314 nop 0
315 nop 0
316 rsq_norm:
317 // For others, do 4 Newton-Raphson steps with the formula:
318 // RSQ_{n + 1} = RSQ_{n} * (1.5 - 0.5 * x * RSQ_{n} * RSQ_{n})
319 // In the code below, each step is written as:
320 // tmp1 = 0.5 * x * RSQ_{n}
321 // tmp2 = -RSQ_{n} * tmp1 + 0.5
322 // RSQ_{n + 1} = RSQ_{n} * tmp2 + RSQ_{n}
323 sched (st 0xd) (st 0xd wr 0x1) (st 0xd wr 0x1 rd 0x0 wt 0x3)
324 mov $r4 0x0 0xf
325 // 0x3f000000: 1/2
326 f2f f32 f64 $r8 0x3f000000
327 dmul $r2 $r0 $r8
328 sched (st 0xd wr 0x0 wt 0x3) (st 0xd wr 0x0 wt 0x1) (st 0xd wr 0x0 wt 0x1)
329 dmul $r0 $r2 $r4
330 dfma $r6 $r0 neg $r4 $r8
331 dfma $r4 $r4 $r6 $r4
332 sched (st 0xd wr 0x0 wt 0x1) (st 0xd wr 0x0 wt 0x1) (st 0xd wr 0x0 wt 0x1)
333 dmul $r0 $r2 $r4
334 dfma $r6 $r0 neg $r4 $r8
335 dfma $r4 $r4 $r6 $r4
336 sched (st 0xd wr 0x0 wt 0x1) (st 0xd wr 0x0 wt 0x1) (st 0xd wr 0x0 wt 0x1)
337 dmul $r0 $r2 $r4
338 dfma $r6 $r0 neg $r4 $r8
339 dfma $r4 $r4 $r6 $r4
340 sched (st 0xd wr 0x0 wt 0x1) (st 0xd wr 0x0 wt 0x1) (st 0xd wr 0x0 wt 0x1)
341 dmul $r0 $r2 $r4
342 dfma $r6 $r0 neg $r4 $r8
343 dfma $r4 $r4 $r6 $r4
344 // Multiply 2^27 to result for small inputs to recover
345 sched (st 0xd wr 0x0 wt 0x1) (st 0xd wt 0x1) (st 0xd)
346 $p1 dmul $r4 $r4 0x41a0000000000000
347 mov $r1 $r5 0xf
348 mov $r0 $r4 0xf
349 sched (st 0xd) (st 0xf) (st 0xf)
350 ret
351 nop 0
352 nop 0
353
354 .section #gm107_builtin_offsets
355 .b64 #gm107_div_u32
356 .b64 #gm107_div_s32
357 .b64 #gm107_rcp_f64
358 .b64 #gm107_rsq_f64