Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_InnerTrsm_Serial_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_INNER_TRSM_SERIAL_IMPL_HPP
17 #define KOKKOSBATCHED_INNER_TRSM_SERIAL_IMPL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include "KokkosBatched_InnerTrsm_Decl.hpp"
23
24 namespace KokkosBatched {
25
26 ///
27 /// Fixed size TRSM
28 /// ================
29 /// L(m x m) X(m x n) = B (m x n)
30
31 template <>
32 template <typename ValueType>
33 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerUnitDiag<5>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int n,
34 /**/ ValueType *KOKKOS_RESTRICT B) {
35 if (n <= 0) return 0;
36
37 const ValueType a_10 = A[1 * _as0 + 0 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1],
38 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1], a_32 = A[3 * _as0 + 2 * _as1],
39 a_40 = A[4 * _as0 + 0 * _as1], a_41 = A[4 * _as0 + 1 * _as1], a_42 = A[4 * _as0 + 2 * _as1],
40 a_43 = A[4 * _as0 + 3 * _as1];
41
42 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p, ValueType &b_3p, ValueType &b_4p) {
43 // load
44 b_0p = B[0 * _bs0 + p * _bs1];
45 b_1p = B[1 * _bs0 + p * _bs1];
46 b_2p = B[2 * _bs0 + p * _bs1];
47 b_3p = B[3 * _bs0 + p * _bs1];
48 b_4p = B[4 * _bs0 + p * _bs1];
49
50 // 0 iteration
51 b_1p -= a_10 * b_0p;
52 b_2p -= a_20 * b_0p;
53 b_3p -= a_30 * b_0p;
54 b_4p -= a_40 * b_0p;
55
56 // 1 iteration
57 b_2p -= a_21 * b_1p;
58 b_3p -= a_31 * b_1p;
59 b_4p -= a_41 * b_1p;
60
61 // 2 iteration
62 b_3p -= a_32 * b_2p;
63 b_4p -= a_42 * b_2p;
64
65 // 3 iteration
66 b_4p -= a_43 * b_3p;
67
68 // store
69 B[1 * _bs0 + p * _bs1] = b_1p;
70 B[2 * _bs0 + p * _bs1] = b_2p;
71 B[3 * _bs0 + p * _bs1] = b_3p;
72 B[4 * _bs0 + p * _bs1] = b_4p;
73 };
74
75 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
76 #pragma unroll
77 #endif
78 for (int p = 0; p < n; ++p) {
79 ValueType b_p[5];
80 trsv(p, b_p[0], b_p[1], b_p[2], b_p[3], b_p[4]);
81 }
82 return 0;
83 }
84
85 template <>
86 template <typename ValueType>
87 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerUnitDiag<4>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int n,
88 /**/ ValueType *KOKKOS_RESTRICT B) {
89 if (n <= 0) return 0;
90
91 const ValueType a_10 = A[1 * _as0 + 0 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1],
92 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1], a_32 = A[3 * _as0 + 2 * _as1];
93
94 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p, ValueType &b_3p) {
95 // load
96 b_0p = B[0 * _bs0 + p * _bs1];
97 b_1p = B[1 * _bs0 + p * _bs1];
98 b_2p = B[2 * _bs0 + p * _bs1];
99 b_3p = B[3 * _bs0 + p * _bs1];
100
101 // 0 iteration
102 b_1p -= a_10 * b_0p;
103 b_2p -= a_20 * b_0p;
104 b_3p -= a_30 * b_0p;
105
106 // 1 iteration
107 b_2p -= a_21 * b_1p;
108 b_3p -= a_31 * b_1p;
109
110 // 2 iteration
111 b_3p -= a_32 * b_2p;
112
113 // store
114 B[1 * _bs0 + p * _bs1] = b_1p;
115 B[2 * _bs0 + p * _bs1] = b_2p;
116 B[3 * _bs0 + p * _bs1] = b_3p;
117 };
118
119 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
120 #pragma unroll
121 #endif
122 for (int p = 0; p < n; ++p) {
123 ValueType b_p[4];
124 trsv(p, b_p[0], b_p[1], b_p[2], b_p[3]);
125 }
126 return 0;
127 }
128
129 template <>
130 template <typename ValueType>
131 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerUnitDiag<3>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int n,
132 /**/ ValueType *KOKKOS_RESTRICT B) {
133 if (n <= 0) return 0;
134
135 const ValueType a_10 = A[1 * _as0 + 0 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1];
136
137 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p) {
138 // load
139 b_0p = B[0 * _bs0 + p * _bs1];
140 b_1p = B[1 * _bs0 + p * _bs1];
141 b_2p = B[2 * _bs0 + p * _bs1];
142
143 // 0 iteration
144 b_1p -= a_10 * b_0p;
145 b_2p -= a_20 * b_0p;
146
147 // 1 iteration
148 b_2p -= a_21 * b_1p;
149
150 // store
151 B[1 * _bs0 + p * _bs1] = b_1p;
152 B[2 * _bs0 + p * _bs1] = b_2p;
153 };
154
155 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
156 #pragma unroll
157 #endif
158 for (int p = 0; p < n; ++p) {
159 ValueType b_p[3];
160 trsv(p, b_p[0], b_p[1], b_p[2]);
161 }
162 return 0;
163 }
164
165 template <>
166 template <typename ValueType>
167 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerUnitDiag<2>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int n,
168 /**/ ValueType *KOKKOS_RESTRICT B) {
169 if (n <= 0) return 0;
170
171 const ValueType a_10 = A[1 * _as0 + 0 * _as1];
172
173 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p) {
174 // load
175 b_0p = B[0 * _bs0 + p * _bs1];
176 b_1p = B[1 * _bs0 + p * _bs1];
177
178 // 0 iteration
179 b_1p -= a_10 * b_0p;
180
181 // store
182 B[1 * _bs0 + p * _bs1] = b_1p;
183 };
184
185 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
186 #pragma unroll
187 #endif
188 for (int p = 0; p < n; ++p) {
189 ValueType b_p[2];
190 trsv(p, b_p[0], b_p[1]);
191 }
192
193 return 0;
194 }
195
196 template <>
197 template <typename ValueType>
198 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerUnitDiag<1>::serial_invoke(const ValueType *KOKKOS_RESTRICT /* A */,
199 const int /* n */,
200 /**/ ValueType *KOKKOS_RESTRICT /* B */) {
201 return 0;
202 }
203
204 ///
205 /// TRSM
206 /// ====
207 /// L(m x m) X(m x n) = B (m x n)
208
209 template <>
210 template <typename ValueType>
211 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerUnitDiag<5>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int m,
212 const int n,
213 /**/ ValueType *KOKKOS_RESTRICT B) {
214 if (m > 5) Kokkos::abort("InnerTrsmLeftLowerUnitDiag<5>::serial_invoke, assert failure (m<=5)");
215 if (m <= 0 || n <= 0) return 0;
216 switch (m) {
217 case 5: {
218 InnerTrsmLeftLowerUnitDiag<5> inner(_as0, _as1, _bs0, _bs1);
219 inner.serial_invoke(A, n, B);
220 break;
221 }
222 case 4: {
223 InnerTrsmLeftLowerUnitDiag<4> inner(_as0, _as1, _bs0, _bs1);
224 inner.serial_invoke(A, n, B);
225 break;
226 }
227 case 3: {
228 InnerTrsmLeftLowerUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
229 inner.serial_invoke(A, n, B);
230 break;
231 }
232 case 2: {
233 InnerTrsmLeftLowerUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
234 inner.serial_invoke(A, n, B);
235 break;
236 }
237 case 1: {
238 InnerTrsmLeftLowerUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
239 inner.serial_invoke(A, n, B);
240 break;
241 }
242 }
243 return 0;
244 }
245 template <>
246 template <typename ValueType>
247 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerUnitDiag<4>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int m,
248 const int n,
249 /**/ ValueType *KOKKOS_RESTRICT B) {
250 if (m > 4) Kokkos::abort("InnerTrsmLeftLowerUnitDiag<4>::serial_invoke, assert failure (m<=4)");
251 if (m <= 0 || n <= 0) return 0;
252 switch (m) {
253 case 4: {
254 InnerTrsmLeftLowerUnitDiag<4> inner(_as0, _as1, _bs0, _bs1);
255 inner.serial_invoke(A, n, B);
256 break;
257 }
258 case 3: {
259 InnerTrsmLeftLowerUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
260 inner.serial_invoke(A, n, B);
261 break;
262 }
263 case 2: {
264 InnerTrsmLeftLowerUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
265 inner.serial_invoke(A, n, B);
266 break;
267 }
268 case 1: {
269 InnerTrsmLeftLowerUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
270 inner.serial_invoke(A, n, B);
271 break;
272 }
273 }
274 return 0;
275 }
276 template <>
277 template <typename ValueType>
278 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerUnitDiag<3>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int m,
279 const int n,
280 /**/ ValueType *KOKKOS_RESTRICT B) {
281 if (m > 3) Kokkos::abort("InnerTrsmLeftLowerUnitDiag<3>::serial_invoke, assert failure (m<=3)");
282 if (m <= 0 || n <= 0) return 0;
283 switch (m) {
284 case 3: {
285 InnerTrsmLeftLowerUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
286 inner.serial_invoke(A, n, B);
287 break;
288 }
289 case 2: {
290 InnerTrsmLeftLowerUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
291 inner.serial_invoke(A, n, B);
292 break;
293 }
294 case 1: {
295 InnerTrsmLeftLowerUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
296 inner.serial_invoke(A, n, B);
297 break;
298 }
299 }
300 return 0;
301 }
302 template <>
303 template <typename ValueType>
304 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerUnitDiag<2>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int m,
305 const int n,
306 /**/ ValueType *KOKKOS_RESTRICT B) {
307 if (m > 2) Kokkos::abort("InnerTrsmLeftLowerUnitDiag<2>::serial_invoke, assert failure (m<=2)");
308 if (m <= 0 || n <= 0) return 0;
309 switch (m) {
310 case 2: {
311 InnerTrsmLeftLowerUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
312 inner.serial_invoke(A, n, B);
313 break;
314 }
315 case 1: {
316 InnerTrsmLeftLowerUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
317 inner.serial_invoke(A, n, B);
318 break;
319 }
320 }
321 return 0;
322 }
323 template <>
324 template <typename ValueType>
325 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerUnitDiag<1>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int m,
326 const int n,
327 /**/ ValueType *KOKKOS_RESTRICT B) {
328 if (m > 1) Kokkos::abort("InnerTrsmLeftLowerUnitDiag<1>::serial_invoke, assert failure (m<=1)");
329 if (m <= 0 || n <= 0) return 0;
330 switch (m) {
331 case 1: {
332 InnerTrsmLeftLowerUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
333 inner.serial_invoke(A, n, B);
334 break;
335 }
336 }
337 return 0;
338 }
339
340 ///
341 /// Fixed size TRSM
342 /// ================
343 /// L(m x m) X(m x n) = B (m x n)
344
345 template <>
346 template <typename ValueType>
347 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerNonUnitDiag<5>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
348 const int n,
349 /**/ ValueType *KOKKOS_RESTRICT B) {
350 if (n <= 0) return 0;
351
352 const ValueType a_10 = A[1 * _as0 + 0 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1],
353 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1], a_32 = A[3 * _as0 + 2 * _as1],
354 a_40 = A[4 * _as0 + 0 * _as1], a_41 = A[4 * _as0 + 1 * _as1], a_42 = A[4 * _as0 + 2 * _as1],
355 a_43 = A[4 * _as0 + 3 * _as1];
356
357 // const ValueType
358 // a_00 = A[0*_as0+0*_as1],
359 // a_11 = A[1*_as0+1*_as1],
360 // a_22 = A[2*_as0+2*_as1],
361 // a_33 = A[3*_as0+3*_as1],
362 // a_44 = A[4*_as0+4*_as1];
363
364 const ValueType inv_a_00 = static_cast<ValueType>(1.0) / A[0 * _as0 + 0 * _as1],
365 inv_a_11 = static_cast<ValueType>(1.0) / A[1 * _as0 + 1 * _as1],
366 inv_a_22 = static_cast<ValueType>(1.0) / A[2 * _as0 + 2 * _as1],
367 inv_a_33 = static_cast<ValueType>(1.0) / A[3 * _as0 + 3 * _as1],
368 inv_a_44 = static_cast<ValueType>(1.0) / A[4 * _as0 + 4 * _as1];
369
370 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p, ValueType &b_3p, ValueType &b_4p) {
371 // load
372 b_0p = B[0 * _bs0 + p * _bs1];
373 b_1p = B[1 * _bs0 + p * _bs1];
374 b_2p = B[2 * _bs0 + p * _bs1];
375 b_3p = B[3 * _bs0 + p * _bs1];
376 b_4p = B[4 * _bs0 + p * _bs1];
377
378 // 0 iteration
379 b_0p *= inv_a_00; /* b_0p /= a_00;*/
380 b_1p -= a_10 * b_0p;
381 b_2p -= a_20 * b_0p;
382 b_3p -= a_30 * b_0p;
383 b_4p -= a_40 * b_0p;
384
385 // 1 iteration
386 b_1p *= inv_a_11; /* b_1p /= a_11; */
387 b_2p -= a_21 * b_1p;
388 b_3p -= a_31 * b_1p;
389 b_4p -= a_41 * b_1p;
390
391 // 2 iteration
392 b_2p *= inv_a_22; /* b_2p /= a_22; */
393 b_3p -= a_32 * b_2p;
394 b_4p -= a_42 * b_2p;
395
396 // 3 iteration
397 b_3p *= inv_a_33; /* b_3p /= a_33; */
398 b_4p -= a_43 * b_3p;
399
400 // 4 iteration
401 b_4p *= inv_a_44; /* b_4p /= a_44; */
402
403 // store
404 B[0 * _bs0 + p * _bs1] = b_0p;
405 B[1 * _bs0 + p * _bs1] = b_1p;
406 B[2 * _bs0 + p * _bs1] = b_2p;
407 B[3 * _bs0 + p * _bs1] = b_3p;
408 B[4 * _bs0 + p * _bs1] = b_4p;
409 };
410
411 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
412 #pragma unroll
413 #endif
414 for (int p = 0; p < n; ++p) {
415 ValueType b_p[5];
416 trsv(p, b_p[0], b_p[1], b_p[2], b_p[3], b_p[4]);
417 }
418
419 return 0;
420 }
421
422 template <>
423 template <typename ValueType>
424 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerNonUnitDiag<4>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
425 const int n,
426 /**/ ValueType *KOKKOS_RESTRICT B) {
427 if (n <= 0) return 0;
428
429 const ValueType a_10 = A[1 * _as0 + 0 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1],
430 a_30 = A[3 * _as0 + 0 * _as1], a_31 = A[3 * _as0 + 1 * _as1], a_32 = A[3 * _as0 + 2 * _as1];
431
432 // const ValueType
433 // a_00 = A[0*_as0+0*_as1],
434 // a_11 = A[1*_as0+1*_as1],
435 // a_22 = A[2*_as0+2*_as1],
436 // a_33 = A[3*_as0+3*_as1];
437
438 const ValueType inv_a_00 = static_cast<ValueType>(1.0) / A[0 * _as0 + 0 * _as1],
439 inv_a_11 = static_cast<ValueType>(1.0) / A[1 * _as0 + 1 * _as1],
440 inv_a_22 = static_cast<ValueType>(1.0) / A[2 * _as0 + 2 * _as1],
441 inv_a_33 = static_cast<ValueType>(1.0) / A[3 * _as0 + 3 * _as1];
442
443 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p, ValueType &b_3p) {
444 // load
445 b_0p = B[0 * _bs0 + p * _bs1];
446 b_1p = B[1 * _bs0 + p * _bs1];
447 b_2p = B[2 * _bs0 + p * _bs1];
448 b_3p = B[3 * _bs0 + p * _bs1];
449
450 // 0 iteration
451 b_0p *= inv_a_00; /* b_0p /= a_00;*/
452 b_1p -= a_10 * b_0p;
453 b_2p -= a_20 * b_0p;
454 b_3p -= a_30 * b_0p;
455
456 // 1 iteration
457 b_1p *= inv_a_11; /* b_1p /= a_11; */
458 b_2p -= a_21 * b_1p;
459 b_3p -= a_31 * b_1p;
460
461 // 2 iteration
462 b_2p *= inv_a_22; /* b_2p /= a_22; */
463 b_3p -= a_32 * b_2p;
464
465 // 3 iteration
466 b_3p *= inv_a_33; /* b_3p /= a_33; */
467
468 // store
469 B[0 * _bs0 + p * _bs1] = b_0p;
470 B[1 * _bs0 + p * _bs1] = b_1p;
471 B[2 * _bs0 + p * _bs1] = b_2p;
472 B[3 * _bs0 + p * _bs1] = b_3p;
473 };
474
475 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
476 #pragma unroll
477 #endif
478 for (int p = 0; p < n; ++p) {
479 ValueType b_p[4];
480 trsv(p, b_p[0], b_p[1], b_p[2], b_p[3]);
481 }
482
483 return 0;
484 }
485
486 template <>
487 template <typename ValueType>
488 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerNonUnitDiag<3>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
489 const int n,
490 /**/ ValueType *KOKKOS_RESTRICT B) {
491 if (n <= 0) return 0;
492
493 const ValueType a_10 = A[1 * _as0 + 0 * _as1], a_20 = A[2 * _as0 + 0 * _as1], a_21 = A[2 * _as0 + 1 * _as1];
494
495 // const ValueType
496 // a_00 = A[0*_as0+0*_as1],
497 // a_11 = A[1*_as0+1*_as1],
498 // a_22 = A[2*_as0+2*_as1];
499
500 const ValueType inv_a_00 = static_cast<ValueType>(1.0) / A[0 * _as0 + 0 * _as1],
501 inv_a_11 = static_cast<ValueType>(1.0) / A[1 * _as0 + 1 * _as1],
502 inv_a_22 = static_cast<ValueType>(1.0) / A[2 * _as0 + 2 * _as1];
503
504 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p) {
505 // load
506 b_0p = B[0 * _bs0 + p * _bs1];
507 b_1p = B[1 * _bs0 + p * _bs1];
508 b_2p = B[2 * _bs0 + p * _bs1];
509
510 // 0 iteration
511 b_0p *= inv_a_00; /* b_0p /= a_00;*/
512 b_1p -= a_10 * b_0p;
513 b_2p -= a_20 * b_0p;
514
515 // 1 iteration
516 b_1p *= inv_a_11; /* b_1p /= a_11; */
517 b_2p -= a_21 * b_1p;
518
519 // 2 iteration
520 b_2p *= inv_a_22; /* b_2p /= a_22; */
521
522 // store
523 B[0 * _bs0 + p * _bs1] = b_0p;
524 B[1 * _bs0 + p * _bs1] = b_1p;
525 B[2 * _bs0 + p * _bs1] = b_2p;
526 };
527
528 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
529 #pragma unroll
530 #endif
531 for (int p = 0; p < n; ++p) {
532 ValueType b_p[3];
533 trsv(p, b_p[0], b_p[1], b_p[2]);
534 }
535
536 return 0;
537 }
538
539 template <>
540 template <typename ValueType>
541 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerNonUnitDiag<2>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
542 const int n,
543 /**/ ValueType *KOKKOS_RESTRICT B) {
544 if (n <= 0) return 0;
545
546 const ValueType a_10 = A[1 * _as0 + 0 * _as1];
547
548 // const ValueType
549 // a_00 = A[0*_as0+0*_as1],
550 // a_11 = A[1*_as0+1*_as1];
551
552 const ValueType inv_a_00 = static_cast<ValueType>(1.0) / A[0 * _as0 + 0 * _as1],
553 inv_a_11 = static_cast<ValueType>(1.0) / A[1 * _as0 + 1 * _as1];
554
555 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p) {
556 // load
557 b_0p = B[0 * _bs0 + p * _bs1];
558 b_1p = B[1 * _bs0 + p * _bs1];
559
560 // 0 iteration
561 b_0p *= inv_a_00; /* b_0p /= a_00;*/
562 b_1p -= a_10 * b_0p;
563
564 // 1 iteration
565 b_1p *= inv_a_11; /* b_1p /= a_11; */
566
567 // store
568 B[0 * _bs0 + p * _bs1] = b_0p;
569 B[1 * _bs0 + p * _bs1] = b_1p;
570 };
571
572 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
573 #pragma unroll
574 #endif
575 for (int p = 0; p < n; ++p) {
576 ValueType b_p[2];
577 trsv(p, b_p[0], b_p[1]);
578 }
579
580 return 0;
581 }
582
583 template <>
584 template <typename ValueType>
585 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerNonUnitDiag<1>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
586 const int n,
587 /**/ ValueType *KOKKOS_RESTRICT B) {
588 if (n <= 0) return 0;
589
590 // const ValueType
591 // a_00 = A[0*_as0+0*_as1];
592
593 const ValueType inv_a_00 = static_cast<ValueType>(1.0) / A[0 * _as0 + 0 * _as1];
594
595 auto trsv = [&](const int p, ValueType & /* b_0p */) {
596 B[0 * _bs0 + p * _bs1] *= inv_a_00; /* b_0p /= a_00;*/
597 };
598
599 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
600 #pragma unroll
601 #endif
602 for (int p = 0; p < n; ++p) {
603 ValueType b_p;
604 trsv(p, b_p);
605 }
606
607 return 0;
608 }
609
610 ///
611 /// TRSM
612 /// ==============
613 /// L(m x m) X(m x n) = B (m x n)
614
615 template <>
616 template <typename ValueType>
617 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerNonUnitDiag<5>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
618 const int m, const int n,
619 /**/ ValueType *KOKKOS_RESTRICT B) {
620 if (m > 5)
621 Kokkos::abort(
622 "InnerTrsmLeftLowerNonUnitDiag<5>::serial_invoke, assert failure "
623 "(m<=5)");
624 if (m <= 0 || n <= 0) return 0;
625 switch (m) {
626 case 5: {
627 InnerTrsmLeftLowerNonUnitDiag<5> inner(_as0, _as1, _bs0, _bs1);
628 inner.serial_invoke(A, n, B);
629 break;
630 }
631 case 4: {
632 InnerTrsmLeftLowerNonUnitDiag<4> inner(_as0, _as1, _bs0, _bs1);
633 inner.serial_invoke(A, n, B);
634 break;
635 }
636 case 3: {
637 InnerTrsmLeftLowerNonUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
638 inner.serial_invoke(A, n, B);
639 break;
640 }
641 case 2: {
642 InnerTrsmLeftLowerNonUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
643 inner.serial_invoke(A, n, B);
644 break;
645 }
646 case 1: {
647 InnerTrsmLeftLowerNonUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
648 inner.serial_invoke(A, n, B);
649 break;
650 }
651 }
652 return 0;
653 }
654 template <>
655 template <typename ValueType>
656 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerNonUnitDiag<4>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
657 const int m, const int n,
658 /**/ ValueType *KOKKOS_RESTRICT B) {
659 if (m > 4)
660 Kokkos::abort(
661 "InnerTrsmLeftLowerNonUnitDiag<4>::serial_invoke, assert failure "
662 "(m<=4)");
663 if (m <= 0 || n <= 0) return 0;
664 switch (m) {
665 case 4: {
666 InnerTrsmLeftLowerNonUnitDiag<4> inner(_as0, _as1, _bs0, _bs1);
667 inner.serial_invoke(A, n, B);
668 break;
669 }
670 case 3: {
671 InnerTrsmLeftLowerNonUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
672 inner.serial_invoke(A, n, B);
673 break;
674 }
675 case 2: {
676 InnerTrsmLeftLowerNonUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
677 inner.serial_invoke(A, n, B);
678 break;
679 }
680 case 1: {
681 InnerTrsmLeftLowerNonUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
682 inner.serial_invoke(A, n, B);
683 break;
684 }
685 }
686 return 0;
687 }
688 template <>
689 template <typename ValueType>
690 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerNonUnitDiag<3>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
691 const int m, const int n,
692 /**/ ValueType *KOKKOS_RESTRICT B) {
693 if (m > 3)
694 Kokkos::abort(
695 "InnerTrsmLeftLowerNonUnitDiag<3>::serial_invoke, assert failure "
696 "(m<=3)");
697 if (m <= 0 || n <= 0) return 0;
698 switch (m) {
699 case 3: {
700 InnerTrsmLeftLowerNonUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
701 inner.serial_invoke(A, n, B);
702 break;
703 }
704 case 2: {
705 InnerTrsmLeftLowerNonUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
706 inner.serial_invoke(A, n, B);
707 break;
708 }
709 case 1: {
710 InnerTrsmLeftLowerNonUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
711 inner.serial_invoke(A, n, B);
712 break;
713 }
714 }
715 return 0;
716 }
717 template <>
718 template <typename ValueType>
719 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerNonUnitDiag<2>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
720 const int m, const int n,
721 /**/ ValueType *KOKKOS_RESTRICT B) {
722 if (m > 2)
723 Kokkos::abort(
724 "InnerTrsmLeftLowerNonUnitDiag<2>::serial_invoke, assert failure "
725 "(m<=2)");
726 if (m <= 0 || n <= 0) return 0;
727 switch (m) {
728 case 2: {
729 InnerTrsmLeftLowerNonUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
730 inner.serial_invoke(A, n, B);
731 break;
732 }
733 case 1: {
734 InnerTrsmLeftLowerNonUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
735 inner.serial_invoke(A, n, B);
736 break;
737 }
738 }
739 return 0;
740 }
741 template <>
742 template <typename ValueType>
743 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftLowerNonUnitDiag<1>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
744 const int m, const int n,
745 /**/ ValueType *KOKKOS_RESTRICT B) {
746 if (m > 1)
747 Kokkos::abort(
748 "InnerTrsmLeftLowerNonUnitDiag<1>::serial_invoke, assert failure "
749 "(m<=1)");
750 if (m <= 0 || n <= 0) return 0;
751 switch (m) {
752 case 1: {
753 InnerTrsmLeftLowerNonUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
754 inner.serial_invoke(A, n, B);
755 break;
756 }
757 }
758 return 0;
759 }
760
761 ///
762 /// Fixed size TRSM
763 /// ================
764 /// L(m x m) X(m x n) = B (m x n)
765
766 template <>
767 template <typename ValueType>
768 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperUnitDiag<5>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int n,
769 /**/ ValueType *KOKKOS_RESTRICT B) {
770 if (n <= 0) return 0;
771
772 const ValueType a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1], a_03 = A[0 * _as0 + 3 * _as1],
773 a_04 = A[0 * _as0 + 4 * _as1],
774 /**/ a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1], a_14 = A[1 * _as0 + 4 * _as1],
775 /**/ a_23 = A[2 * _as0 + 3 * _as1], a_24 = A[2 * _as0 + 4 * _as1],
776 /**/ a_34 = A[3 * _as0 + 4 * _as1];
777
778 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p, ValueType &b_3p, ValueType &b_4p) {
779 // load
780 b_0p = B[0 * _bs0 + p * _bs1];
781 b_1p = B[1 * _bs0 + p * _bs1];
782 b_2p = B[2 * _bs0 + p * _bs1];
783 b_3p = B[3 * _bs0 + p * _bs1];
784 b_4p = B[4 * _bs0 + p * _bs1];
785
786 // 0 iteration
787 b_0p -= a_04 * b_4p;
788 b_1p -= a_14 * b_4p;
789 b_2p -= a_24 * b_4p;
790 b_3p -= a_34 * b_4p;
791
792 // 1 iteration
793 b_0p -= a_03 * b_3p;
794 b_1p -= a_13 * b_3p;
795 b_2p -= a_23 * b_3p;
796
797 // 2 iteration
798 b_0p -= a_02 * b_2p;
799 b_1p -= a_12 * b_2p;
800
801 // 1 iteration
802 b_0p -= a_01 * b_1p;
803
804 // store
805 B[0 * _bs0 + p * _bs1] = b_0p;
806 B[1 * _bs0 + p * _bs1] = b_1p;
807 B[2 * _bs0 + p * _bs1] = b_2p;
808 B[3 * _bs0 + p * _bs1] = b_3p;
809 };
810
811 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
812 #pragma unroll
813 #endif
814 for (int p = 0; p < n; ++p) {
815 ValueType b_p[5];
816 trsv(p, b_p[0], b_p[1], b_p[2], b_p[3], b_p[4]);
817 }
818
819 return 0;
820 }
821
822 template <>
823 template <typename ValueType>
824 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperUnitDiag<4>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int n,
825 /**/ ValueType *KOKKOS_RESTRICT B) {
826 if (n <= 0) return 0;
827
828 const ValueType a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1], a_03 = A[0 * _as0 + 3 * _as1],
829 /**/ a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1],
830 /**/ a_23 = A[2 * _as0 + 3 * _as1];
831
832 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p, ValueType &b_3p) {
833 // load
834 b_0p = B[0 * _bs0 + p * _bs1];
835 b_1p = B[1 * _bs0 + p * _bs1];
836 b_2p = B[2 * _bs0 + p * _bs1];
837 b_3p = B[3 * _bs0 + p * _bs1];
838
839 // 0 iteration
840 b_0p -= a_03 * b_3p;
841 b_1p -= a_13 * b_3p;
842 b_2p -= a_23 * b_3p;
843
844 // 1 iteration
845 b_0p -= a_02 * b_2p;
846 b_1p -= a_12 * b_2p;
847
848 // 2 iteration
849 b_0p -= a_01 * b_1p;
850
851 // store
852 B[0 * _bs0 + p * _bs1] = b_0p;
853 B[1 * _bs0 + p * _bs1] = b_1p;
854 B[2 * _bs0 + p * _bs1] = b_2p;
855 };
856
857 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
858 #pragma unroll
859 #endif
860 for (int p = 0; p < n; ++p) {
861 ValueType b_p[4];
862 trsv(p, b_p[0], b_p[1], b_p[2], b_p[3]);
863 }
864
865 return 0;
866 }
867
868 template <>
869 template <typename ValueType>
870 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperUnitDiag<3>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int n,
871 /**/ ValueType *KOKKOS_RESTRICT B) {
872 if (n <= 0) return 0;
873
874 const ValueType a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
875 /**/ a_12 = A[1 * _as0 + 2 * _as1];
876
877 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p) {
878 // load
879 b_0p = B[0 * _bs0 + p * _bs1];
880 b_1p = B[1 * _bs0 + p * _bs1];
881 b_2p = B[2 * _bs0 + p * _bs1];
882
883 // 0 iteration
884 b_0p -= a_02 * b_2p;
885 b_1p -= a_12 * b_2p;
886
887 // 1 iteration
888 b_0p -= a_01 * b_1p;
889
890 // store
891 B[0 * _bs0 + p * _bs1] = b_0p;
892 B[1 * _bs0 + p * _bs1] = b_1p;
893 };
894
895 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
896 #pragma unroll
897 #endif
898 for (int p = 0; p < n; ++p) {
899 ValueType b_p[3];
900 trsv(p, b_p[0], b_p[1], b_p[2]);
901 }
902
903 return 0;
904 }
905
906 template <>
907 template <typename ValueType>
908 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperUnitDiag<2>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int n,
909 /**/ ValueType *KOKKOS_RESTRICT B) {
910 if (n <= 0) return 0;
911
912 const ValueType a_01 = A[0 * _as0 + 1 * _as1];
913
914 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p) {
915 // load
916 b_0p = B[0 * _bs0 + p * _bs1];
917 b_1p = B[1 * _bs0 + p * _bs1];
918
919 // 0 iteration
920 b_0p -= a_01 * b_1p;
921
922 // store
923 B[0 * _bs0 + p * _bs1] = b_0p;
924 };
925
926 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
927 #pragma unroll
928 #endif
929 for (int p = 0; p < n; ++p) {
930 ValueType b_p[2];
931 trsv(p, b_p[0], b_p[1]);
932 }
933
934 return 0;
935 }
936
937 template <>
938 template <typename ValueType>
939 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperUnitDiag<1>::serial_invoke(const ValueType *KOKKOS_RESTRICT /* A */,
940 const int /* n */,
941 /**/ ValueType *KOKKOS_RESTRICT /* B */) {
942 return 0;
943 }
944
945 ///
946 /// TRSM
947 /// ====
948 /// L(m x m) X(m x n) = B (m x n)
949
950 template <>
951 template <typename ValueType>
952 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperUnitDiag<5>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int m,
953 const int n,
954 /**/ ValueType *KOKKOS_RESTRICT B) {
955 if (m > 5) Kokkos::abort("InnerTrsmLeftUpperUnitDiag<5>::serial_invoke, assert failure (m<=5)");
956 if (m <= 0 || n <= 0) return 0;
957 switch (m) {
958 case 5: {
959 InnerTrsmLeftUpperUnitDiag<5> inner(_as0, _as1, _bs0, _bs1);
960 inner.serial_invoke(A, n, B);
961 break;
962 }
963 case 4: {
964 InnerTrsmLeftUpperUnitDiag<4> inner(_as0, _as1, _bs0, _bs1);
965 inner.serial_invoke(A, n, B);
966 break;
967 }
968 case 3: {
969 InnerTrsmLeftUpperUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
970 inner.serial_invoke(A, n, B);
971 break;
972 }
973 case 2: {
974 InnerTrsmLeftUpperUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
975 inner.serial_invoke(A, n, B);
976 break;
977 }
978 case 1: {
979 InnerTrsmLeftUpperUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
980 inner.serial_invoke(A, n, B);
981 break;
982 }
983 }
984 return 0;
985 }
986 template <>
987 template <typename ValueType>
988 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperUnitDiag<4>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int m,
989 const int n,
990 /**/ ValueType *KOKKOS_RESTRICT B) {
991 if (m > 4) Kokkos::abort("InnerTrsmLeftUpperUnitDiag<4>::serial_invoke, assert failure (m<=4)");
992 if (m <= 0 || n <= 0) return 0;
993 switch (m) {
994 case 4: {
995 InnerTrsmLeftUpperUnitDiag<4> inner(_as0, _as1, _bs0, _bs1);
996 inner.serial_invoke(A, n, B);
997 break;
998 }
999 case 3: {
1000 InnerTrsmLeftUpperUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
1001 inner.serial_invoke(A, n, B);
1002 break;
1003 }
1004 case 2: {
1005 InnerTrsmLeftUpperUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
1006 inner.serial_invoke(A, n, B);
1007 break;
1008 }
1009 case 1: {
1010 InnerTrsmLeftUpperUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
1011 inner.serial_invoke(A, n, B);
1012 break;
1013 }
1014 }
1015 return 0;
1016 }
1017 template <>
1018 template <typename ValueType>
1019 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperUnitDiag<3>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int m,
1020 const int n,
1021 /**/ ValueType *KOKKOS_RESTRICT B) {
1022 if (m > 3) Kokkos::abort("InnerTrsmLeftUpperUnitDiag<3>::serial_invoke, assert failure (m<=3)");
1023 if (m <= 0 || n <= 0) return 0;
1024 switch (m) {
1025 case 3: {
1026 InnerTrsmLeftUpperUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
1027 inner.serial_invoke(A, n, B);
1028 break;
1029 }
1030 case 2: {
1031 InnerTrsmLeftUpperUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
1032 inner.serial_invoke(A, n, B);
1033 break;
1034 }
1035 case 1: {
1036 InnerTrsmLeftUpperUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
1037 inner.serial_invoke(A, n, B);
1038 break;
1039 }
1040 }
1041 return 0;
1042 }
1043 template <>
1044 template <typename ValueType>
1045 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperUnitDiag<2>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int m,
1046 const int n,
1047 /**/ ValueType *KOKKOS_RESTRICT B) {
1048 if (m > 2) Kokkos::abort("InnerTrsmLeftUpperUnitDiag<2>::serial_invoke, assert failure (m<=2)");
1049 if (m <= 0 || n <= 0) return 0;
1050 switch (m) {
1051 case 2: {
1052 InnerTrsmLeftUpperUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
1053 inner.serial_invoke(A, n, B);
1054 break;
1055 }
1056 case 1: {
1057 InnerTrsmLeftUpperUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
1058 inner.serial_invoke(A, n, B);
1059 break;
1060 }
1061 }
1062 return 0;
1063 }
1064 template <>
1065 template <typename ValueType>
1066 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperUnitDiag<1>::serial_invoke(const ValueType *KOKKOS_RESTRICT A, const int m,
1067 const int n,
1068 /**/ ValueType *KOKKOS_RESTRICT B) {
1069 if (m > 1) Kokkos::abort("InnerTrsmLeftUpperUnitDiag<1>::serial_invoke, assert failure (m<=1)");
1070 if (m <= 0 || n <= 0) return 0;
1071 switch (m) {
1072 case 1: {
1073 InnerTrsmLeftUpperUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
1074 inner.serial_invoke(A, n, B);
1075 break;
1076 }
1077 }
1078 return 0;
1079 }
1080
1081 ///
1082 /// Fixed size TRSM
1083 /// ================
1084 /// L(m x m) X(m x n) = B (m x n)
1085
1086 template <>
1087 template <typename ValueType>
1088 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperNonUnitDiag<5>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
1089 const int n,
1090 /**/ ValueType *KOKKOS_RESTRICT B) {
1091 if (n <= 0) return 0;
1092
1093 const ValueType a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1], a_03 = A[0 * _as0 + 3 * _as1],
1094 a_04 = A[0 * _as0 + 4 * _as1],
1095 /**/ a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1], a_14 = A[1 * _as0 + 4 * _as1],
1096 /**/ a_23 = A[2 * _as0 + 3 * _as1], a_24 = A[2 * _as0 + 4 * _as1],
1097 /**/ a_34 = A[3 * _as0 + 4 * _as1];
1098
1099 // const ValueType
1100 // a_00 = A[0*_as0+0*_as1],
1101 // a_11 = A[1*_as0+1*_as1],
1102 // a_22 = A[2*_as0+2*_as1],
1103 // a_33 = A[3*_as0+3*_as1],
1104 // a_44 = A[4*_as0+4*_as1];
1105
1106 const ValueType inv_a_00 = static_cast<ValueType>(1.0) / A[0 * _as0 + 0 * _as1],
1107 inv_a_11 = static_cast<ValueType>(1.0) / A[1 * _as0 + 1 * _as1],
1108 inv_a_22 = static_cast<ValueType>(1.0) / A[2 * _as0 + 2 * _as1],
1109 inv_a_33 = static_cast<ValueType>(1.0) / A[3 * _as0 + 3 * _as1],
1110 inv_a_44 = static_cast<ValueType>(1.0) / A[4 * _as0 + 4 * _as1];
1111
1112 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p, ValueType &b_3p, ValueType &b_4p) {
1113 // load
1114 b_0p = B[0 * _bs0 + p * _bs1];
1115 b_1p = B[1 * _bs0 + p * _bs1];
1116 b_2p = B[2 * _bs0 + p * _bs1];
1117 b_3p = B[3 * _bs0 + p * _bs1];
1118 b_4p = B[4 * _bs0 + p * _bs1];
1119
1120 // 0 iteration
1121 b_4p *= inv_a_44; /* b_4p /= a_44;*/
1122 b_3p -= a_34 * b_4p;
1123 b_2p -= a_24 * b_4p;
1124 b_1p -= a_14 * b_4p;
1125 b_0p -= a_04 * b_4p;
1126
1127 // 1 iterationls
1128 b_3p *= inv_a_33; /* b_3p /= a_33;*/
1129 b_2p -= a_23 * b_3p;
1130 b_1p -= a_13 * b_3p;
1131 b_0p -= a_03 * b_3p;
1132
1133 // 2 iteration
1134 b_2p *= inv_a_22; /* b_2p /= a_22; */
1135 b_1p -= a_12 * b_2p;
1136 b_0p -= a_02 * b_2p;
1137
1138 // 3 iteration
1139 b_1p *= inv_a_11; /* b_1p /= a_11; */
1140 b_0p -= a_01 * b_1p;
1141
1142 // 4 iteration
1143 b_0p *= inv_a_00; /* b_0p /= a_00; */
1144
1145 // store
1146 B[0 * _bs0 + p * _bs1] = b_0p;
1147 B[1 * _bs0 + p * _bs1] = b_1p;
1148 B[2 * _bs0 + p * _bs1] = b_2p;
1149 B[3 * _bs0 + p * _bs1] = b_3p;
1150 B[4 * _bs0 + p * _bs1] = b_4p;
1151 };
1152
1153 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1154 #pragma unroll
1155 #endif
1156 for (int p = 0; p < n; ++p) {
1157 ValueType b_p[5];
1158 trsv(p, b_p[0], b_p[1], b_p[2], b_p[3], b_p[4]);
1159 }
1160
1161 return 0;
1162 }
1163
1164 template <>
1165 template <typename ValueType>
1166 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperNonUnitDiag<4>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
1167 const int n,
1168 /**/ ValueType *KOKKOS_RESTRICT B) {
1169 if (n <= 0) return 0;
1170
1171 const ValueType a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1], a_03 = A[0 * _as0 + 3 * _as1],
1172 /**/ a_12 = A[1 * _as0 + 2 * _as1], a_13 = A[1 * _as0 + 3 * _as1],
1173 /**/ a_23 = A[2 * _as0 + 3 * _as1];
1174
1175 // const ValueType
1176 // a_00 = A[0*_as0+0*_as1],
1177 // a_11 = A[1*_as0+1*_as1],
1178 // a_22 = A[2*_as0+2*_as1],
1179 // a_33 = A[3*_as0+3*_as1];
1180
1181 const ValueType inv_a_00 = static_cast<ValueType>(1.0) / A[0 * _as0 + 0 * _as1],
1182 inv_a_11 = static_cast<ValueType>(1.0) / A[1 * _as0 + 1 * _as1],
1183 inv_a_22 = static_cast<ValueType>(1.0) / A[2 * _as0 + 2 * _as1],
1184 inv_a_33 = static_cast<ValueType>(1.0) / A[3 * _as0 + 3 * _as1];
1185
1186 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p, ValueType &b_3p) {
1187 // load
1188 b_0p = B[0 * _bs0 + p * _bs1];
1189 b_1p = B[1 * _bs0 + p * _bs1];
1190 b_2p = B[2 * _bs0 + p * _bs1];
1191 b_3p = B[3 * _bs0 + p * _bs1];
1192
1193 // 0 iteration
1194 b_3p *= inv_a_33; /* b_3p /= a_33;*/
1195 b_2p -= a_23 * b_3p;
1196 b_1p -= a_13 * b_3p;
1197 b_0p -= a_03 * b_3p;
1198
1199 // 1 iteration
1200 b_2p *= inv_a_22; /* b_2p /= a_22; */
1201 b_1p -= a_12 * b_2p;
1202 b_0p -= a_02 * b_2p;
1203
1204 // 2 iteration
1205 b_1p *= inv_a_11; /* b_1p /= a_11; */
1206 b_0p -= a_01 * b_1p;
1207
1208 // 3 iteration
1209 b_0p *= inv_a_00; /* b_0p /= a_00; */
1210
1211 // store
1212 B[0 * _bs0 + p * _bs1] = b_0p;
1213 B[1 * _bs0 + p * _bs1] = b_1p;
1214 B[2 * _bs0 + p * _bs1] = b_2p;
1215 B[3 * _bs0 + p * _bs1] = b_3p;
1216 };
1217
1218 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1219 #pragma unroll
1220 #endif
1221 for (int p = 0; p < n; ++p) {
1222 ValueType b_p[4];
1223 trsv(p, b_p[0], b_p[1], b_p[2], b_p[3]);
1224 }
1225
1226 return 0;
1227 }
1228
1229 template <>
1230 template <typename ValueType>
1231 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperNonUnitDiag<3>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
1232 const int n,
1233 /**/ ValueType *KOKKOS_RESTRICT B) {
1234 if (n <= 0) return 0;
1235
1236 const ValueType a_01 = A[0 * _as0 + 1 * _as1], a_02 = A[0 * _as0 + 2 * _as1],
1237 /**/ a_12 = A[1 * _as0 + 2 * _as1];
1238
1239 // const ValueType
1240 // a_00 = A[0*_as0+0*_as1],
1241 // a_11 = A[1*_as0+1*_as1],
1242 // a_22 = A[2*_as0+2*_as1];
1243
1244 const ValueType inv_a_00 = static_cast<ValueType>(1.0) / A[0 * _as0 + 0 * _as1],
1245 inv_a_11 = static_cast<ValueType>(1.0) / A[1 * _as0 + 1 * _as1],
1246 inv_a_22 = static_cast<ValueType>(1.0) / A[2 * _as0 + 2 * _as1];
1247
1248 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p, ValueType &b_2p) {
1249 // load
1250 b_0p = B[0 * _bs0 + p * _bs1];
1251 b_1p = B[1 * _bs0 + p * _bs1];
1252 b_2p = B[2 * _bs0 + p * _bs1];
1253
1254 // 0 iteration
1255 b_2p *= inv_a_22; /* b_2p /= a_22; */
1256 b_1p -= a_12 * b_2p;
1257 b_0p -= a_02 * b_2p;
1258
1259 // 1 iteration
1260 b_1p *= inv_a_11; /* b_1p /= a_11; */
1261 b_0p -= a_01 * b_1p;
1262
1263 // 2 iteration
1264 b_0p *= inv_a_00; /* b_0p /= a_00; */
1265
1266 // store
1267 B[0 * _bs0 + p * _bs1] = b_0p;
1268 B[1 * _bs0 + p * _bs1] = b_1p;
1269 B[2 * _bs0 + p * _bs1] = b_2p;
1270 };
1271
1272 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1273 #pragma unroll
1274 #endif
1275 for (int p = 0; p < n; ++p) {
1276 ValueType b_p[3];
1277 trsv(p, b_p[0], b_p[1], b_p[2]);
1278 }
1279
1280 return 0;
1281 }
1282
1283 template <>
1284 template <typename ValueType>
1285 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperNonUnitDiag<2>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
1286 const int n,
1287 /**/ ValueType *KOKKOS_RESTRICT B) {
1288 if (n <= 0) return 0;
1289
1290 const ValueType a_01 = A[0 * _as0 + 1 * _as1];
1291
1292 // const ValueType
1293 // a_00 = A[0*_as0+0*_as1],
1294 // a_11 = A[1*_as0+1*_as1];
1295
1296 const ValueType inv_a_00 = static_cast<ValueType>(1.0) / A[0 * _as0 + 0 * _as1],
1297 inv_a_11 = static_cast<ValueType>(1.0) / A[1 * _as0 + 1 * _as1];
1298
1299 auto trsv = [&](const int p, ValueType &b_0p, ValueType &b_1p) {
1300 // load
1301 b_0p = B[0 * _bs0 + p * _bs1];
1302 b_1p = B[1 * _bs0 + p * _bs1];
1303
1304 // 2 iteration
1305 b_1p *= inv_a_11; /* b_1p /= a_11; */
1306 b_0p -= a_01 * b_1p;
1307
1308 // 3 iteration
1309 b_0p *= inv_a_00; /* b_0p /= a_00; */
1310
1311 // store
1312 B[0 * _bs0 + p * _bs1] = b_0p;
1313 B[1 * _bs0 + p * _bs1] = b_1p;
1314 };
1315
1316 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1317 #pragma unroll
1318 #endif
1319 for (int p = 0; p < n; ++p) {
1320 ValueType b_p[2];
1321 trsv(p, b_p[0], b_p[1]);
1322 }
1323
1324 return 0;
1325 }
1326
1327 template <>
1328 template <typename ValueType>
1329 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperNonUnitDiag<1>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
1330 const int n,
1331 /**/ ValueType *KOKKOS_RESTRICT B) {
1332 if (n <= 0) return 0;
1333
1334 // const ValueType
1335 // a_00 = A[0*_as0+0*_as1];
1336
1337 const ValueType inv_a_00 = static_cast<ValueType>(1.0) / A[0 * _as0 + 0 * _as1];
1338
1339 auto trsv = [&](const int p, ValueType & /* b_0p */) {
1340 // 0 iteration
1341 B[0 * _bs0 + p * _bs1] *= inv_a_00; /* b_0p /= a_00; */
1342 };
1343
1344 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
1345 #pragma unroll
1346 #endif
1347 for (int p = 0; p < n; ++p) {
1348 ValueType b_p;
1349 trsv(p, b_p);
1350 }
1351
1352 return 0;
1353 }
1354
1355 ///
1356 /// TRSM
1357 /// ====
1358 /// L(m x m) X(m x n) = B (m x n)
1359
1360 template <>
1361 template <typename ValueType>
1362 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperNonUnitDiag<5>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
1363 const int m, const int n,
1364 /**/ ValueType *KOKKOS_RESTRICT B) {
1365 if (m > 5)
1366 Kokkos::abort(
1367 "InnerTrsmLeftUpperNonUnitDiag<5>::serial_invoke, assert failure "
1368 "(m<=5)");
1369 if (m <= 0 || n <= 0) return 0;
1370 switch (m) {
1371 case 5: {
1372 InnerTrsmLeftUpperNonUnitDiag<5> inner(_as0, _as1, _bs0, _bs1);
1373 inner.serial_invoke(A, n, B);
1374 break;
1375 }
1376 case 4: {
1377 InnerTrsmLeftUpperNonUnitDiag<4> inner(_as0, _as1, _bs0, _bs1);
1378 inner.serial_invoke(A, n, B);
1379 break;
1380 }
1381 case 3: {
1382 InnerTrsmLeftUpperNonUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
1383 inner.serial_invoke(A, n, B);
1384 break;
1385 }
1386 case 2: {
1387 InnerTrsmLeftUpperNonUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
1388 inner.serial_invoke(A, n, B);
1389 break;
1390 }
1391 case 1: {
1392 InnerTrsmLeftUpperNonUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
1393 inner.serial_invoke(A, n, B);
1394 break;
1395 }
1396 }
1397 return 0;
1398 }
1399 template <>
1400 template <typename ValueType>
1401 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperNonUnitDiag<4>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
1402 const int m, const int n,
1403 /**/ ValueType *KOKKOS_RESTRICT B) {
1404 if (m > 4)
1405 Kokkos::abort(
1406 "InnerTrsmLeftUpperNonUnitDiag<4>::serial_invoke, assert failure "
1407 "(m<=4)");
1408 if (m <= 0 || n <= 0) return 0;
1409 switch (m) {
1410 case 4: {
1411 InnerTrsmLeftUpperNonUnitDiag<4> inner(_as0, _as1, _bs0, _bs1);
1412 inner.serial_invoke(A, n, B);
1413 break;
1414 }
1415 case 3: {
1416 InnerTrsmLeftUpperNonUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
1417 inner.serial_invoke(A, n, B);
1418 break;
1419 }
1420 case 2: {
1421 InnerTrsmLeftUpperNonUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
1422 inner.serial_invoke(A, n, B);
1423 break;
1424 }
1425 case 1: {
1426 InnerTrsmLeftUpperNonUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
1427 inner.serial_invoke(A, n, B);
1428 break;
1429 }
1430 }
1431 return 0;
1432 }
1433 template <>
1434 template <typename ValueType>
1435 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperNonUnitDiag<3>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
1436 const int m, const int n,
1437 /**/ ValueType *KOKKOS_RESTRICT B) {
1438 if (m > 3)
1439 Kokkos::abort(
1440 "InnerTrsmLeftUpperNonUnitDiag<3>::serial_invoke, assert failure "
1441 "(m<=3)");
1442 if (m <= 0 || n <= 0) return 0;
1443 switch (m) {
1444 case 3: {
1445 InnerTrsmLeftUpperNonUnitDiag<3> inner(_as0, _as1, _bs0, _bs1);
1446 inner.serial_invoke(A, n, B);
1447 break;
1448 }
1449 case 2: {
1450 InnerTrsmLeftUpperNonUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
1451 inner.serial_invoke(A, n, B);
1452 break;
1453 }
1454 case 1: {
1455 InnerTrsmLeftUpperNonUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
1456 inner.serial_invoke(A, n, B);
1457 break;
1458 }
1459 }
1460 return 0;
1461 }
1462 template <>
1463 template <typename ValueType>
1464 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperNonUnitDiag<2>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
1465 const int m, const int n,
1466 /**/ ValueType *KOKKOS_RESTRICT B) {
1467 if (m > 2)
1468 Kokkos::abort(
1469 "InnerTrsmLeftUpperNonUnitDiag<2>::serial_invoke, assert failure "
1470 "(m<=2)");
1471 if (m <= 0 || n <= 0) return 0;
1472 switch (m) {
1473 case 2: {
1474 InnerTrsmLeftUpperNonUnitDiag<2> inner(_as0, _as1, _bs0, _bs1);
1475 inner.serial_invoke(A, n, B);
1476 break;
1477 }
1478 case 1: {
1479 InnerTrsmLeftUpperNonUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
1480 inner.serial_invoke(A, n, B);
1481 break;
1482 }
1483 }
1484 return 0;
1485 }
1486 template <>
1487 template <typename ValueType>
1488 KOKKOS_INLINE_FUNCTION int InnerTrsmLeftUpperNonUnitDiag<1>::serial_invoke(const ValueType *KOKKOS_RESTRICT A,
1489 const int m, const int n,
1490 /**/ ValueType *KOKKOS_RESTRICT B) {
1491 if (m > 1)
1492 Kokkos::abort(
1493 "InnerTrsmLeftUpperNonUnitDiag<1>::serial_invoke, assert failure "
1494 "(m<=1)");
1495 if (m <= 0 || n <= 0) return 0;
1496 switch (m) {
1497 case 1: {
1498 InnerTrsmLeftUpperNonUnitDiag<1> inner(_as0, _as1, _bs0, _bs1);
1499 inner.serial_invoke(A, n, B);
1500 break;
1501 }
1502 }
1503 return 0;
1504 }
1505
1506 } // namespace KokkosBatched
1507
1508 #endif