Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_LU_Team_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_LU_TEAM_INTERNAL_HPP
17 #define KOKKOSBATCHED_LU_TEAM_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include "KokkosBatched_Vector.hpp"
23 #include "KokkosBatched_InnerLU_Serial_Impl.hpp"
24 #include "KokkosBatched_InnerTrsm_Serial_Impl.hpp"
25
26 #include "KokkosBatched_Trsm_Team_Internal.hpp"
27 #include "KokkosBatched_Gemm_Team_Internal.hpp"
28
29 namespace KokkosBatched {
30
31 ///
32 /// Team Internal Impl
33 /// ==================
34
35 template <typename AlgoType>
36 struct TeamLU_Internal {
37 template <typename MemberType, typename ValueType>
38 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const int m, const int n,
39 ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
40 const typename MagnitudeScalarType<ValueType>::type tiny);
41 };
42
43 template <>
44 template <typename MemberType, typename ValueType>
45 KOKKOS_INLINE_FUNCTION int TeamLU_Internal<Algo::LU::Unblocked>::invoke(
46 const MemberType &member, const int m, const int n, ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
47 const typename MagnitudeScalarType<ValueType>::type tiny) {
48 const int k = (m < n ? m : n);
49 if (k <= 0) return 0;
50
51 const auto abs_tiny = tiny > 0 ? tiny : -tiny;
52 const auto minus_abs_tiny = -abs_tiny;
53
54 for (int p = 0; p < k; ++p) {
55 // Made this non-const in order to WORKAROUND issue #349
56 int iend = m - p - 1;
57 int jend = n - p - 1;
58
59 const ValueType *KOKKOS_RESTRICT a12t = A + (p)*as0 + (p + 1) * as1;
60
61 ValueType *KOKKOS_RESTRICT a21 = A + (p + 1) * as0 + (p)*as1,
62 *KOKKOS_RESTRICT A22 = A + (p + 1) * as0 + (p + 1) * as1;
63
64 if (tiny != 0) {
65 if (member.team_rank() == 0) {
66 ValueType &alpha11_reference = A[p * as0 + p * as1];
67 const auto alpha11_real = Kokkos::ArithTraits<ValueType>::real(alpha11_reference);
68 alpha11_reference += minus_abs_tiny * ValueType(alpha11_real < 0);
69 alpha11_reference += abs_tiny * ValueType(alpha11_real >= 0);
70 }
71 }
72
73 member.team_barrier();
74 const ValueType alpha11 = A[p * as0 + p * as1];
75 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, iend), [&](const int &i) {
76 // a21[i*as0] *= inv_alpha11;
77 a21[i * as0] /= alpha11;
78 });
79
80 member.team_barrier();
81 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, iend * jend), [&](const int &ij) {
82 // assume layout right for batched computation
83 const int i = ij / jend, j = ij % jend;
84 A22[i * as0 + j * as1] -= a21[i * as0] * a12t[j * as1];
85 });
86 }
87 return 0;
88 }
89
90 template <>
91 template <typename MemberType, typename ValueType>
92 KOKKOS_INLINE_FUNCTION int TeamLU_Internal<Algo::LU::Blocked>::invoke(
93 const MemberType &member, const int m, const int n, ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
94 const typename MagnitudeScalarType<ValueType>::type /*tiny*/) {
95 constexpr int mbAlgo = Algo::LU::Blocked::mb();
96
97 const int k = (m < n ? m : n);
98 if (k <= 0) return 0;
99
100 const typename MagnitudeScalarType<ValueType>::type one(1.0), minus_one(-1.0);
101
102 InnerLU<mbAlgo> lu(as0, as1);
103
104 InnerTrsmLeftLowerUnitDiag<mbAlgo> trsm_llu(as0, as1, as0, as1);
105 InnerTrsmLeftLowerNonUnitDiag<mbAlgo> trsm_run(as1, as0, as1, as0);
106 auto lu_factorize = [&](const int ib, const int jb, ValueType *KOKKOS_RESTRICT AA) {
107 const int tsize = member.team_size();
108 // Made this non-const in order to WORKAROUND issue #349
109 int mb = mbAlgo;
110 int nb = ((jb - mb) + (ib - mb)) > 0 ? ((jb - mb) + (ib - mb)) / tsize + (((jb - mb) + (ib - mb)) % tsize > 0) : 1;
111 const int kb = ib < jb ? ib : jb;
112
113 for (int p = 0; p < kb; p += mb) {
114 const int pb = (p + mb) > kb ? (kb - p) : mb;
115
116 // diagonal block
117 ValueType *KOKKOS_RESTRICT Ap = AA + p * as0 + p * as1;
118
119 // lu on a block
120 member.team_barrier();
121 if (member.team_rank() == 0) lu.serial_invoke(pb, Ap);
122 member.team_barrier();
123
124 // Made this non-const in order to WORKAROUND issue #349
125 int m_abr = ib - p - mb, n_abr = jb - p - mb, mp_abr = m_abr % nb, np_abr = n_abr % nb,
126 mq_abr = (m_abr / nb) + (mp_abr > 0), nq_abr = (n_abr / nb) + (np_abr > 0);
127
128 // trsm update
129 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, mq_abr + nq_abr), [&](const int &ij) {
130 if (ij < nq_abr) {
131 const int j = (ij)*nb, qb = (j + nb) > n_abr ? np_abr : nb;
132 trsm_llu.serial_invoke(Ap, pb, qb, Ap + (j + mb) * as1);
133 } else {
134 const int i = (ij - nq_abr) * nb, qb = (i + nb) > m_abr ? mp_abr : nb;
135 trsm_run.serial_invoke(Ap, pb, qb, Ap + (i + mb) * as0);
136 }
137 });
138 member.team_barrier();
139
140 // gemm update
141 TeamGemmInternal<Algo::Gemm::Blocked>::invoke(member, m_abr, n_abr, pb, minus_one, Ap + mb * as0, as0, as1,
142 Ap + mb * as1, as0, as1, one, Ap + mb * as0 + mb * as1, as0, as1);
143 }
144 };
145
146 const bool is_small = true; //(m*n <= 64*64);
147 if (is_small) {
148 lu_factorize(m, n, A);
149 } else {
150 // // some cache blocking may need (not priority yet);
151 // lu_factorize(m, n, A);
152 }
153
154 return 0;
155 }
156 } // namespace KokkosBatched
157
158 #endif