Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Gesv_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_GESV_IMPL_HPP
17 #define KOKKOSBATCHED_GESV_IMPL_HPP
18
19 /// \author Kim Liegeois ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include <KokkosBatched_LU_Decl.hpp>
23 #include "KokkosBatched_Trsm_Decl.hpp"
24 #include "KokkosBatched_Copy_Decl.hpp"
25
26 namespace KokkosBatched {
27
28 struct SerialStaticPivoting {
29 template <class MatrixType1, class MatrixType2, class VectorType1, class VectorType2>
30 KOKKOS_INLINE_FUNCTION static int invoke(const MatrixType1 A, const MatrixType2 PDAD, const VectorType1 Y,
31 const VectorType2 PDY, const VectorType2 D2, const VectorType2 tmp_v_1,
32 const VectorType2 tmp_v_2);
33 };
34
35 template <typename MemberType>
36 struct TeamStaticPivoting {
37 template <class MatrixType1, class MatrixType2, class VectorType1, class VectorType2>
38 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const MatrixType1 A, const MatrixType2 PDAD,
39 const VectorType1 Y, const VectorType2 PDY, const VectorType2 D2,
40 const VectorType2 tmp_v_1, const VectorType2 tmp_v_2);
41 };
42
43 template <typename MemberType>
44 struct TeamVectorStaticPivoting {
45 template <class MatrixType1, class MatrixType2, class VectorType1, class VectorType2>
46 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const MatrixType1 A, const MatrixType2 PDAD,
47 const VectorType1 Y, const VectorType2 PDY, const VectorType2 D2,
48 const VectorType2 tmp_v_1, const VectorType2 tmp_v_2);
49 };
50
51 template <class MatrixType1, class MatrixType2, class VectorType1, class VectorType2>
52 KOKKOS_INLINE_FUNCTION int SerialStaticPivoting::invoke(const MatrixType1 A, const MatrixType2 PDAD,
53 const VectorType1 Y, const VectorType2 PDY,
54 const VectorType2 D2, const VectorType2 tmp_v_1,
55 const VectorType2 tmp_v_2) {
56 using value_type = typename MatrixType1::non_const_value_type;
57 const size_t n = A.extent(0);
58
59 // First, the algorithm loops over the rows and columns and search
60 // for the maximal absolute value per row and column.
61 for (size_t i = 0; i < n; ++i) {
62 D2(i) = Kokkos::ArithTraits<value_type>::zero();
63 tmp_v_1(i) = 0;
64 tmp_v_2(i) = 1.;
65 for (size_t j = 0; j < n; ++j) {
66 if (D2(i) < Kokkos::abs(A(j, i))) D2(i) = Kokkos::abs(A(j, i));
67 if (tmp_v_1(i) < Kokkos::abs(A(i, j))) tmp_v_1(i) = Kokkos::abs(A(i, j));
68 }
69 D2(i) = 1. / D2(i);
70 }
71
72 // Then, the inverse of the maximal value per column is used to scale
73 // A by the right.
74 for (size_t i = 0; i < n; ++i) {
75 for (size_t j = 0; j < n; ++j) {
76 A(i, j) *= D2(j);
77 }
78 }
79
80 // Once again, the algorithm loops over the rows and store the maximal
81 // absolute value per row but after the right scalling and do a left scalling
82 // of A and Y.
83 value_type D1_i;
84 for (size_t i = 0; i < n; ++i) {
85 D1_i = Kokkos::ArithTraits<value_type>::zero();
86 for (size_t j = 0; j < n; ++j) {
87 if (D1_i < Kokkos::abs(A(i, j))) D1_i = Kokkos::abs(A(i, j));
88 }
89 D1_i = 1. / D1_i;
90 for (size_t j = 0; j < n; ++j) {
91 A(i, j) *= D1_i;
92 }
93 Y(i) *= D1_i;
94 }
95
96 // Finally, the algorithm starts to loop over the rows in an order such that
97 // their initial maximal absolute value decrease (it uses the tmp_v_1 to do
98 // so), then for a given row, it finds the available column with the largest
99 // absolute value. If this value is zero, the algorithm failed to compute a
100 // good pivot, otherwise it puts the current row to the found column index and
101 // it labels the row and column index as unavailable and continue the loop
102 // over the rows.
103 //
104 for (size_t i = 0; i < n; ++i) {
105 int row_index = 0;
106 int col_index = 0;
107 value_type tmp_0 = Kokkos::ArithTraits<value_type>::zero();
108 value_type tmp_1 = Kokkos::ArithTraits<value_type>::zero();
109 for (size_t j = 0; j < n; ++j) {
110 if (tmp_0 < tmp_v_1(j)) {
111 tmp_0 = tmp_v_1(j);
112 row_index = j;
113 }
114 }
115 for (size_t j = 0; j < n; ++j) {
116 if (tmp_1 < Kokkos::abs(A(row_index, j) * tmp_v_2(j))) {
117 tmp_1 = Kokkos::abs(A(row_index, j) * tmp_v_2(j));
118 col_index = j;
119 }
120 }
121 if (tmp_1 == Kokkos::ArithTraits<value_type>::zero()) return 1;
122 tmp_v_1(row_index) = Kokkos::ArithTraits<value_type>::zero();
123 tmp_v_2(col_index) = Kokkos::ArithTraits<value_type>::zero();
124
125 for (size_t j = 0; j < n; ++j) {
126 PDAD(col_index, j) = A(row_index, j);
127 }
128 PDY(col_index) = Y(row_index);
129 }
130
131 return 0;
132 }
133
134 template <typename MemberType>
135 template <class MatrixType1, class MatrixType2, class VectorType1, class VectorType2>
136 KOKKOS_INLINE_FUNCTION int TeamStaticPivoting<MemberType>::invoke(const MemberType &member, const MatrixType1 A,
137 const MatrixType2 PDAD, const VectorType1 Y,
138 const VectorType2 PDY, const VectorType2 D2,
139 const VectorType2 tmp_v_1,
140 const VectorType2 tmp_v_2) {
141 using value_type = typename MatrixType1::non_const_value_type;
142 using reducer_value_type = typename Kokkos::MaxLoc<value_type, int>::value_type;
143 // This implementation follows the strategy of SerialStaticPivoting but uses
144 // an extra level of parallelism.
145
146 // Made this non-const in order to WORKAROUND issue #349 (Credit to C. Trott)
147 size_t n = A.extent(0);
148
149 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n), [&](const int &i) {
150 D2(i) = Kokkos::ArithTraits<value_type>::zero();
151 tmp_v_1(i) = 0;
152 tmp_v_2(i) = 1.;
153 for (size_t j = 0; j < n; ++j) {
154 if (D2(i) < Kokkos::abs(A(j, i))) D2(i) = Kokkos::abs(A(j, i));
155 if (tmp_v_1(i) < Kokkos::abs(A(i, j))) tmp_v_1(i) = Kokkos::abs(A(i, j));
156 }
157 D2(i) = 1. / D2(i);
158 });
159
160 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n), [&](const int &i) {
161 for (size_t j = 0; j < n; ++j) {
162 A(i, j) *= D2(j);
163 }
164 });
165
166 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n), [&](const int &i) {
167 value_type D1_i = Kokkos::ArithTraits<value_type>::zero();
168 for (size_t j = 0; j < n; ++j) {
169 if (D1_i < Kokkos::abs(A(i, j))) D1_i = Kokkos::abs(A(i, j));
170 }
171 D1_i = 1. / D1_i;
172 for (size_t j = 0; j < n; ++j) {
173 A(i, j) *= D1_i;
174 }
175 Y(i) *= D1_i;
176 });
177
178 for (size_t i = 0; i < n; ++i) {
179 int row_index, col_index;
180 reducer_value_type value;
181 Kokkos::MaxLoc<value_type, int> reducer_value(value);
182 Kokkos::parallel_reduce(
183 Kokkos::TeamThreadRange(member, n),
184 [&](const int &j, reducer_value_type &update) {
185 if (tmp_v_1(j) > update.val) {
186 update.val = tmp_v_1(j);
187 update.loc = j;
188 }
189 },
190 reducer_value);
191 row_index = value.loc;
192 value.loc = 0;
193 value.val = Kokkos::ArithTraits<value_type>::zero();
194 Kokkos::parallel_reduce(
195 Kokkos::TeamThreadRange(member, n),
196 [&](const int &j, reducer_value_type &update) {
197 if (Kokkos::abs(A(row_index, j) * tmp_v_2(j)) > update.val) {
198 update.val = Kokkos::abs(A(row_index, j) * tmp_v_2(j));
199 update.loc = j;
200 }
201 },
202 reducer_value);
203 col_index = value.loc;
204 if (value.val == Kokkos::ArithTraits<value_type>::zero()) return 1;
205 tmp_v_1(row_index) = Kokkos::ArithTraits<value_type>::zero();
206 tmp_v_2(col_index) = Kokkos::ArithTraits<value_type>::zero();
207
208 for (size_t j = 0; j < n; ++j) {
209 PDAD(col_index, j) = A(row_index, j);
210 }
211 PDY(col_index) = Y(row_index);
212 }
213 return 0;
214 }
215
216 template <typename MemberType>
217 template <class MatrixType1, class MatrixType2, class VectorType1, class VectorType2>
218 KOKKOS_INLINE_FUNCTION int TeamVectorStaticPivoting<MemberType>::invoke(const MemberType &member, const MatrixType1 A,
219 const MatrixType2 PDAD, const VectorType1 Y,
220 const VectorType2 PDY, const VectorType2 D2,
221 const VectorType2 tmp_v_1,
222 const VectorType2 tmp_v_2) {
223 using value_type = typename MatrixType1::non_const_value_type;
224 using reducer_value_type = typename Kokkos::MaxLoc<value_type, int>::value_type;
225 // This implementation follows the strategy of SerialStaticPivoting but uses
226 // two extra levels of parallelism.
227
228 const size_t n = A.extent(0);
229
230 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n), [&](const int &i) {
231 D2(i) = Kokkos::ArithTraits<value_type>::zero();
232 tmp_v_1(i) = 0;
233 tmp_v_2(i) = 1.;
234 reducer_value_type value;
235 Kokkos::MaxLoc<value_type, int> reducer_value(value);
236 Kokkos::parallel_reduce(
237 Kokkos::ThreadVectorRange(member, n),
238 [&](const int &j, reducer_value_type &update) {
239 if (Kokkos::abs(A(j, i)) > update.val) {
240 update.val = Kokkos::abs(A(j, i));
241 update.loc = j;
242 }
243 },
244 reducer_value);
245 D2(i) = 1. / value.val;
246 Kokkos::parallel_reduce(
247 Kokkos::ThreadVectorRange(member, n),
248 [&](const int &j, reducer_value_type &update) {
249 if (Kokkos::abs(A(i, j)) > update.val) {
250 update.val = Kokkos::abs(A(i, j));
251 update.loc = j;
252 }
253 },
254 reducer_value);
255 tmp_v_1(i) = value.val;
256 });
257
258 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n), [&](const int &i) {
259 Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, n), [&](const int &j) { A(i, j) *= D2(j); });
260 });
261
262 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n), [&](const int &i) {
263 value_type D1_i = Kokkos::ArithTraits<value_type>::zero();
264 reducer_value_type value;
265 Kokkos::MaxLoc<value_type, int> reducer_value(value);
266 Kokkos::parallel_reduce(
267 Kokkos::ThreadVectorRange(member, n),
268 [&](const int &j, reducer_value_type &update) {
269 if (Kokkos::abs(A(i, j)) > update.val) {
270 update.val = Kokkos::abs(A(i, j));
271 update.loc = j;
272 }
273 },
274 reducer_value);
275 D1_i = 1. / value.val;
276 Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, n), [&](const int &j) { A(i, j) *= D1_i; });
277 Y(i) *= D1_i;
278 });
279
280 for (size_t i = 0; i < n; ++i) {
281 int row_index, col_index;
282 reducer_value_type value{};
283 Kokkos::MaxLoc<value_type, int> reducer_value(value);
284 Kokkos::parallel_reduce(
285 Kokkos::TeamVectorRange(member, n),
286 [&](const int &j, reducer_value_type &update) {
287 if (tmp_v_1(j) > update.val) {
288 update.val = tmp_v_1(j);
289 update.loc = j;
290 }
291 },
292 reducer_value);
293 row_index = value.loc;
294 value.loc = 0;
295 value.val = Kokkos::ArithTraits<value_type>::zero();
296 Kokkos::parallel_reduce(
297 Kokkos::TeamVectorRange(member, n),
298 [&](const int &j, reducer_value_type &update) {
299 if (Kokkos::abs(A(row_index, j) * tmp_v_2(j)) > update.val) {
300 update.val = Kokkos::abs(A(row_index, j) * tmp_v_2(j));
301 update.loc = j;
302 }
303 },
304 reducer_value);
305 col_index = value.loc;
306 if (value.val == Kokkos::ArithTraits<value_type>::zero()) return 1;
307 tmp_v_1(row_index) = Kokkos::ArithTraits<value_type>::zero();
308 tmp_v_2(col_index) = Kokkos::ArithTraits<value_type>::zero();
309
310 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, n),
311 [&](const int &j) { PDAD(col_index, j) = A(row_index, j); });
312 PDY(col_index) = Y(row_index);
313 }
314 return 0;
315 }
316
317 template <class VectorType1, class VectorType2, class VectorType3>
318 KOKKOS_INLINE_FUNCTION void SerialHadamard1D(const VectorType1 X, const VectorType2 D, const VectorType3 DX) {
319 const size_t n = X.extent(0);
320
321 for (size_t i = 0; i < n; ++i) {
322 DX(i) = D(i) * X(i);
323 }
324 }
325
326 template <typename MemberType, class VectorType1, class VectorType2, class VectorType3>
327 KOKKOS_INLINE_FUNCTION void TeamHadamard1D(const MemberType &member, const VectorType1 X, const VectorType2 D,
328 const VectorType3 DX) {
329 const size_t n = X.extent(0);
330
331 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, n), [&](const size_t &i) { DX(i) = D(i) * X(i); });
332 }
333
334 template <typename MemberType, class VectorType1, class VectorType2, class VectorType3>
335 KOKKOS_INLINE_FUNCTION void TeamVectorHadamard1D(const MemberType &member, const VectorType1 X, const VectorType2 D,
336 const VectorType3 DX) {
337 const size_t n = X.extent(0);
338
339 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, n), [&](const size_t &i) { DX(i) = D(i) * X(i); });
340 }
341
342 ///
343 /// Serial Impl
344 /// ===========
345 template <>
346 struct SerialGesv<Gesv::StaticPivoting> {
347 template <typename MatrixType, typename XVectorType, typename YVectorType>
348 KOKKOS_INLINE_FUNCTION static int invoke(const MatrixType A, const XVectorType X, const YVectorType Y,
349 const MatrixType tmp) {
350 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
351 static_assert(Kokkos::is_view<MatrixType>::value, "KokkosBatched::gesv: MatrixType is not a Kokkos::View.");
352 static_assert(Kokkos::is_view<XVectorType>::value, "KokkosBatched::gesv: XVectorType is not a Kokkos::View.");
353 static_assert(Kokkos::is_view<YVectorType>::value, "KokkosBatched::gesv: YVectorType is not a Kokkos::View.");
354 static_assert(MatrixType::rank == 2, "KokkosBatched::gesv: MatrixType must have rank 2.");
355 static_assert(XVectorType::rank == 1, "KokkosBatched::gesv: XVectorType must have rank 1.");
356 static_assert(YVectorType::rank == 1, "KokkosBatched::gesv: YVectorType must have rank 1.");
357
358 // Check compatibility of dimensions at run time.
359
360 if (A.extent(0) != tmp.extent(0) || A.extent(1) + 4 != tmp.extent(1)) {
361 Kokkos::printf(
362 "KokkosBatched::gesv: dimensions of A and tmp do not match: A: "
363 "%d x %d, tmp (note: its second dimension should be the second "
364 "dimension of A + 4): %d x %d\n",
365 (int)A.extent(0), (int)A.extent(1), (int)tmp.extent(0), (int)tmp.extent(1));
366 return 1;
367 }
368
369 if (A.extent(0) != X.extent(0) || A.extent(1) != X.extent(0) || A.extent(0) != Y.extent(0)) {
370 Kokkos::printf(
371 "KokkosBatched::gesv: dimensions of A and X and Y do not match: A: "
372 "%d x %d, X: %d, Y: %d\n",
373 (int)A.extent(0), (int)A.extent(1), (int)X.extent(0), (int)Y.extent(0));
374 return 1;
375 }
376 #endif
377
378 const int n = A.extent(0);
379
380 auto PDAD = Kokkos::subview(tmp, Kokkos::ALL, Kokkos::make_pair(0, n));
381 auto PDY = Kokkos::subview(tmp, Kokkos::ALL, n);
382 auto D2 = Kokkos::subview(tmp, Kokkos::ALL, n + 1);
383 auto tmp_v_1 = Kokkos::subview(tmp, Kokkos::ALL, n + 2);
384 auto tmp_v_2 = Kokkos::subview(tmp, Kokkos::ALL, n + 3);
385
386 if (SerialStaticPivoting::invoke(A, PDAD, Y, PDY, D2, tmp_v_1, tmp_v_2) == 1) {
387 Kokkos::printf(
388 "KokkosBatched::gesv: the currently implemented static pivoting "
389 "failed.\n");
390 return 1;
391 }
392
393 int r_val = SerialLU<Algo::Level3::Unblocked>::invoke(PDAD);
394
395 if (r_val == 0)
396 r_val = SerialTrsm<Side::Left, Uplo::Lower, Trans::NoTranspose, Diag::Unit, Algo::Level3::Unblocked>::invoke(
397 1.0, PDAD, PDY);
398
399 if (r_val == 0)
400 r_val = SerialTrsm<Side::Left, Uplo::Upper, Trans::NoTranspose, Diag::NonUnit, Algo::Level3::Unblocked>::invoke(
401 1.0, PDAD, PDY);
402
403 if (r_val == 0) SerialHadamard1D(PDY, D2, X);
404 return r_val;
405 }
406 };
407
408 template <>
409 struct SerialGesv<Gesv::NoPivoting> {
410 template <typename MatrixType, typename XVectorType, typename YVectorType>
411 KOKKOS_INLINE_FUNCTION static int invoke(const MatrixType A, const XVectorType X, const YVectorType Y,
412 const MatrixType /*tmp*/) {
413 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
414 static_assert(Kokkos::is_view<MatrixType>::value, "KokkosBatched::gesv: MatrixType is not a Kokkos::View.");
415 static_assert(Kokkos::is_view<XVectorType>::value, "KokkosBatched::gesv: XVectorType is not a Kokkos::View.");
416 static_assert(Kokkos::is_view<YVectorType>::value, "KokkosBatched::gesv: YVectorType is not a Kokkos::View.");
417 static_assert(MatrixType::rank == 2, "KokkosBatched::gesv: MatrixType must have rank 2.");
418 static_assert(XVectorType::rank == 1, "KokkosBatched::gesv: XVectorType must have rank 1.");
419 static_assert(YVectorType::rank == 1, "KokkosBatched::gesv: YVectorType must have rank 1.");
420
421 // Check compatibility of dimensions at run time.
422
423 if (A.extent(0) != X.extent(0) || A.extent(1) != X.extent(0) || A.extent(0) != Y.extent(0)) {
424 Kokkos::printf(
425 "KokkosBatched::gesv: dimensions of A and X and Y do not match: A: "
426 "%d x %d, X: %d, Y: %d\n",
427 (int)A.extent(0), (int)A.extent(1), (int)X.extent(0), (int)Y.extent(0));
428 return 1;
429 }
430 #endif
431
432 int r_val = SerialLU<Algo::Level3::Unblocked>::invoke(A);
433
434 if (r_val == 0) r_val = SerialCopy<Trans::NoTranspose, 1>::invoke(Y, X);
435
436 if (r_val == 0)
437 r_val = SerialTrsm<Side::Left, Uplo::Lower, Trans::NoTranspose, Diag::Unit, Algo::Level3::Unblocked>::invoke(
438 1.0, A, X);
439
440 if (r_val == 0)
441 r_val = SerialTrsm<Side::Left, Uplo::Upper, Trans::NoTranspose, Diag::NonUnit, Algo::Level3::Unblocked>::invoke(
442 1.0, A, X);
443
444 return r_val;
445 }
446 };
447
448 ///
449 /// Team Impl
450 /// =========
451
452 template <typename MemberType>
453 struct TeamGesv<MemberType, Gesv::StaticPivoting> {
454 template <typename MatrixType, typename VectorType>
455 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const MatrixType A, const VectorType X,
456 const VectorType Y) {
457 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
458 static_assert(Kokkos::is_view<MatrixType>::value, "KokkosBatched::gesv: MatrixType is not a Kokkos::View.");
459 static_assert(Kokkos::is_view<VectorType>::value, "KokkosBatched::gesv: VectorType is not a Kokkos::View.");
460 static_assert(MatrixType::rank == 2, "KokkosBatched::gesv: MatrixType must have rank 2.");
461 static_assert(VectorType::rank == 1, "KokkosBatched::gesv: VectorType must have rank 1.");
462
463 // Check compatibility of dimensions at run time.
464 if (A.extent(0) != X.extent(0) || A.extent(1) != X.extent(0) || A.extent(0) != Y.extent(0)) {
465 Kokkos::printf(
466 "KokkosBatched::gesv: dimensions of A and X and Y do not match: A: "
467 "%d x %d, X: %d, Y: %d\n",
468 (int)A.extent(0), (int)A.extent(1), (int)X.extent(0), (int)Y.extent(0));
469 return 1;
470 }
471 #endif
472 using ScratchPadMatrixViewType = Kokkos::View<typename MatrixType::non_const_value_type **,
473 typename MatrixType::execution_space::scratch_memory_space>;
474
475 const int n = A.extent(0);
476
477 ScratchPadMatrixViewType tmp(member.team_scratch(0), n, n + 4);
478 auto PDAD = Kokkos::subview(tmp, Kokkos::ALL, Kokkos::make_pair(0, n));
479 auto PDY = Kokkos::subview(tmp, Kokkos::ALL, n);
480 auto D2 = Kokkos::subview(tmp, Kokkos::ALL, n + 1);
481 auto tmp_v_1 = Kokkos::subview(tmp, Kokkos::ALL, n + 2);
482 auto tmp_v_2 = Kokkos::subview(tmp, Kokkos::ALL, n + 3);
483
484 if (TeamStaticPivoting<MemberType>::invoke(member, A, PDAD, Y, PDY, D2, tmp_v_1, tmp_v_2) == 1) {
485 Kokkos::printf(
486 "KokkosBatched::gesv: the currently implemented static pivoting "
487 "failed.\n");
488 return 1;
489 }
490 member.team_barrier();
491
492 int r_val = TeamLU<MemberType, Algo::Level3::Unblocked>::invoke(member, PDAD);
493 member.team_barrier();
494
495 if (r_val == 0) {
496 r_val = TeamTrsm<MemberType, Side::Left, Uplo::Lower, Trans::NoTranspose, Diag::Unit,
497 Algo::Level3::Unblocked>::invoke(member, 1.0, PDAD, PDY);
498 member.team_barrier();
499 }
500
501 if (r_val == 0) {
502 r_val = TeamTrsm<MemberType, Side::Left, Uplo::Upper, Trans::NoTranspose, Diag::NonUnit,
503 Algo::Level3::Unblocked>::invoke(member, 1.0, PDAD, PDY);
504 member.team_barrier();
505 }
506
507 if (r_val == 0) {
508 TeamHadamard1D(member, PDY, D2, X);
509 member.team_barrier();
510 }
511
512 return r_val;
513 }
514 };
515
516 template <typename MemberType>
517 struct TeamGesv<MemberType, Gesv::NoPivoting> {
518 template <typename MatrixType, typename VectorType>
519 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const MatrixType A, const VectorType X,
520 const VectorType Y) {
521 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
522 static_assert(Kokkos::is_view<MatrixType>::value, "KokkosBatched::gesv: MatrixType is not a Kokkos::View.");
523 static_assert(Kokkos::is_view<VectorType>::value, "KokkosBatched::gesv: VectorType is not a Kokkos::View.");
524 static_assert(MatrixType::rank == 2, "KokkosBatched::gesv: MatrixType must have rank 2.");
525 static_assert(VectorType::rank == 1, "KokkosBatched::gesv: VectorType must have rank 1.");
526
527 // Check compatibility of dimensions at run time.
528 if (A.extent(0) != X.extent(0) || A.extent(1) != X.extent(0) || A.extent(0) != Y.extent(0)) {
529 Kokkos::printf(
530 "KokkosBatched::gesv: dimensions of A and X and Y do not match: A: "
531 "%d x %d, X: %d, Y: %d\n",
532 (int)A.extent(0), (int)A.extent(1), (int)X.extent(0), (int)Y.extent(0));
533 return 1;
534 }
535 #endif
536
537 int r_val = TeamLU<MemberType, Algo::Level3::Unblocked>::invoke(member, A);
538 member.team_barrier();
539
540 if (r_val == 0) {
541 TeamCopy<MemberType, Trans::NoTranspose, 1>::invoke(member, Y, X);
542 member.team_barrier();
543 }
544
545 if (r_val == 0) {
546 TeamTrsm<MemberType, Side::Left, Uplo::Lower, Trans::NoTranspose, Diag::Unit, Algo::Level3::Unblocked>::invoke(
547 member, 1.0, A, X);
548 member.team_barrier();
549 }
550
551 if (r_val == 0) {
552 TeamTrsm<MemberType, Side::Left, Uplo::Upper, Trans::NoTranspose, Diag::NonUnit, Algo::Level3::Unblocked>::invoke(
553 member, 1.0, A, X);
554 member.team_barrier();
555 }
556
557 return r_val;
558 }
559 };
560
561 ///
562 /// TeamVector Impl
563 /// =========
564
565 template <typename MemberType>
566 struct TeamVectorGesv<MemberType, Gesv::StaticPivoting> {
567 template <typename MatrixType, typename VectorType>
568 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const MatrixType A, const VectorType X,
569 const VectorType Y) {
570 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
571 static_assert(Kokkos::is_view<MatrixType>::value, "KokkosBatched::gesv: MatrixType is not a Kokkos::View.");
572 static_assert(Kokkos::is_view<VectorType>::value, "KokkosBatched::gesv: VectorType is not a Kokkos::View.");
573 static_assert(MatrixType::rank == 2, "KokkosBatched::gesv: MatrixType must have rank 2.");
574 static_assert(VectorType::rank == 1, "KokkosBatched::gesv: VectorType must have rank 1.");
575
576 // Check compatibility of dimensions at run time.
577 if (A.extent(0) != X.extent(0) || A.extent(1) != X.extent(0) || A.extent(0) != Y.extent(0)) {
578 Kokkos::printf(
579 "KokkosBatched::gesv: dimensions of A and X and Y do not match: A: "
580 "%d x %d, X: %d, Y: %d\n",
581 (int)A.extent(0), (int)A.extent(1), (int)X.extent(0), (int)Y.extent(0));
582 return 1;
583 }
584 #endif
585 using ScratchPadMatrixViewType = Kokkos::View<typename MatrixType::non_const_value_type **,
586 typename MatrixType::execution_space::scratch_memory_space>;
587
588 const int n = A.extent(0);
589
590 ScratchPadMatrixViewType tmp(member.team_scratch(0), n, n + 4);
591 auto PDAD = Kokkos::subview(tmp, Kokkos::ALL, Kokkos::make_pair(0, n));
592 auto PDY = Kokkos::subview(tmp, Kokkos::ALL, n);
593 auto D2 = Kokkos::subview(tmp, Kokkos::ALL, n + 1);
594 auto tmp_v_1 = Kokkos::subview(tmp, Kokkos::ALL, n + 2);
595 auto tmp_v_2 = Kokkos::subview(tmp, Kokkos::ALL, n + 3);
596
597 if (TeamVectorStaticPivoting<MemberType>::invoke(member, A, PDAD, Y, PDY, D2, tmp_v_1, tmp_v_2) == 1) {
598 Kokkos::printf(
599 "KokkosBatched::gesv: the currently implemented static pivoting "
600 "failed.\n");
601 return 1;
602 }
603
604 member.team_barrier();
605
606 int r_val = TeamLU<MemberType, Algo::Level3::Unblocked>::invoke(member, PDAD);
607 member.team_barrier();
608
609 if (r_val == 0) {
610 TeamVectorTrsm<MemberType, Side::Left, Uplo::Lower, Trans::NoTranspose, Diag::Unit,
611 Algo::Level3::Unblocked>::invoke(member, 1.0, PDAD, PDY);
612 member.team_barrier();
613 }
614
615 if (r_val == 0) {
616 TeamVectorTrsm<MemberType, Side::Left, Uplo::Upper, Trans::NoTranspose, Diag::NonUnit,
617 Algo::Level3::Unblocked>::invoke(member, 1.0, PDAD, PDY);
618 member.team_barrier();
619 }
620
621 if (r_val == 0) {
622 TeamVectorHadamard1D(member, PDY, D2, X);
623 member.team_barrier();
624 }
625
626 return r_val;
627 }
628 };
629
630 template <typename MemberType>
631 struct TeamVectorGesv<MemberType, Gesv::NoPivoting> {
632 template <typename MatrixType, typename VectorType>
633 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const MatrixType A, const VectorType X,
634 const VectorType Y) {
635 #if (KOKKOSKERNELS_DEBUG_LEVEL > 0)
636 static_assert(Kokkos::is_view<MatrixType>::value, "KokkosBatched::gesv: MatrixType is not a Kokkos::View.");
637 static_assert(Kokkos::is_view<VectorType>::value, "KokkosBatched::gesv: VectorType is not a Kokkos::View.");
638 static_assert(MatrixType::rank == 2, "KokkosBatched::gesv: MatrixType must have rank 2.");
639 static_assert(VectorType::rank == 1, "KokkosBatched::gesv: VectorType must have rank 1.");
640
641 // Check compatibility of dimensions at run time.
642 if (A.extent(0) != X.extent(0) || A.extent(1) != X.extent(0) || A.extent(0) != Y.extent(0)) {
643 Kokkos::printf(
644 "KokkosBatched::gesv: dimensions of A and X and Y do not match: A: "
645 "%d x %d, X: %d, Y: %d\n",
646 (int)A.extent(0), (int)A.extent(1), (int)X.extent(0), (int)Y.extent(0));
647 return 1;
648 }
649 #endif
650
651 int r_val = TeamLU<MemberType, Algo::Level3::Unblocked>::invoke(member, A);
652 member.team_barrier();
653
654 if (r_val == 0) {
655 TeamVectorCopy<MemberType, Trans::NoTranspose, 1>::invoke(member, Y, X);
656 member.team_barrier();
657 }
658
659 if (r_val == 0) {
660 TeamVectorTrsm<MemberType, Side::Left, Uplo::Lower, Trans::NoTranspose, Diag::Unit,
661 Algo::Level3::Unblocked>::invoke(member, 1.0, A, X);
662 member.team_barrier();
663 }
664
665 if (r_val == 0) {
666 TeamVectorTrsm<MemberType, Side::Left, Uplo::Upper, Trans::NoTranspose, Diag::NonUnit,
667 Algo::Level3::Unblocked>::invoke(member, 1.0, A, X);
668 member.team_barrier();
669 }
670
671 return r_val;
672 }
673 };
674
675 } // namespace KokkosBatched
676
677 #endif