Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Trsv_Team_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_TRSV_TEAM_INTERNAL_HPP
17 #define KOKKOSBATCHED_TRSV_TEAM_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22
23 #include "KokkosBlas1_set_impl.hpp"
24 #include "KokkosBlas1_team_scal_impl.hpp"
25 #include "KokkosBatched_InnerTrsm_Serial_Impl.hpp"
26 #include "KokkosBlas2_team_gemv_spec.hpp"
27
28 namespace KokkosBatched {
29
30 ///
31 /// Team Internal Impl
32 /// ====================
33
34 ///
35 /// Lower
36 ///
37
38 template <typename AlgoType>
39 struct TeamTrsvInternalLower {
40 template <typename MemberType, typename ScalarType, typename ValueType>
41 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType & /*member*/, const bool /*use_unit_diag*/, const int /*m*/,
42 const ScalarType /*alpha*/, const ValueType *KOKKOS_RESTRICT /*A*/,
43 const int /*as0*/, const int /*as1*/,
44 /**/ ValueType *KOKKOS_RESTRICT /*b*/, const int /*bs0*/) {
45 assert(false && "Error: encounter dummy impl");
46 return 0;
47 }
48 };
49
50 template <>
51 template <typename MemberType, typename ScalarType, typename ValueType>
52 KOKKOS_INLINE_FUNCTION int TeamTrsvInternalLower<Algo::Trsv::Unblocked>::invoke(
53 const MemberType &member, const bool use_unit_diag, const int m, const ScalarType alpha,
54 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
55 /**/ ValueType *KOKKOS_RESTRICT b, const int bs0) {
56 const ScalarType one(1.0), zero(0.0);
57
58 if (alpha == zero)
59 KokkosBlas::Impl::TeamSetInternal::invoke(member, m, zero, b, bs0);
60 else {
61 if (alpha != one) KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, alpha, b, bs0);
62 if (m <= 0) return 0;
63
64 for (int p = 0; p < m; ++p) {
65 const int iend = m - p - 1;
66
67 const ValueType *KOKKOS_RESTRICT a21 = iend ? A + (p + 1) * as0 + p * as1 : NULL;
68
69 ValueType *KOKKOS_RESTRICT beta1 = b + p * bs0, *KOKKOS_RESTRICT b2 = iend ? beta1 + bs0 : NULL;
70
71 member.team_barrier();
72 ValueType local_beta1 = *beta1;
73 if (!use_unit_diag) {
74 const ValueType alpha11 = A[p * as0 + p * as1];
75 local_beta1 = local_beta1 / alpha11;
76 /// make sure all local beta1 is same for threads
77 member.team_barrier();
78 if (member.team_rank() == 0) *beta1 = local_beta1;
79 }
80 /// member.team_barrier();
81 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, iend),
82 [&](const int &i) { b2[i * bs0] -= a21[i * as0] * local_beta1; });
83 }
84 }
85 return 0;
86 }
87
88 template <>
89 template <typename MemberType, typename ScalarType, typename ValueType>
90 KOKKOS_INLINE_FUNCTION int TeamTrsvInternalLower<Algo::Trsv::Blocked>::invoke(
91 const MemberType &member, const bool use_unit_diag, const int m, const ScalarType alpha,
92 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
93 /**/ ValueType *KOKKOS_RESTRICT b, const int bs0) {
94 const ScalarType one(1.0), zero(0.0), minus_one(-1.0);
95
96 constexpr int mbAlgo = Algo::Trsv::Blocked::mb();
97
98 if (alpha == zero)
99 KokkosBlas::Impl::TeamSetInternal::invoke(member, m, zero, b, bs0);
100 else {
101 if (alpha != one) KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, alpha, b, bs0);
102 if (m <= 0) return 0;
103
104 /// case GPU: team size is large and blocksize (mb,nb) is small
105 InnerTrsmLeftLowerUnitDiag<mbAlgo> trsm_u(as0, as1, bs0, 0);
106 InnerTrsmLeftLowerNonUnitDiag<mbAlgo> trsm_n(as0, as1, bs0, 0);
107
108 const int mb = mbAlgo;
109 // const int tsize = member.team_size();
110 for (int p = 0; p < m; p += mb) {
111 const int pb = ((p + mb) > m ? (m - p) : mb);
112
113 // trsm update
114 const ValueType *KOKKOS_RESTRICT Ap = A + p * as0 + p * as1;
115 /**/ ValueType *KOKKOS_RESTRICT bp = b + p * bs0;
116
117 member.team_barrier();
118 if (member.team_rank() == 0) {
119 if (use_unit_diag)
120 trsm_u.serial_invoke(Ap, pb, 1, bp);
121 else
122 trsm_n.serial_invoke(Ap, pb, 1, bp);
123 }
124
125 // gemv update
126 member.team_barrier();
127 KokkosBlas::Impl::TeamGemvInternal<Algo::Gemv::Blocked>::invoke(member, m - p - pb, pb, minus_one, Ap + pb * as0,
128 as0, as1, bp, 1, one, bp + pb * bs0, bs0);
129 }
130 }
131 return 0;
132 }
133
134 ///
135 /// Upper
136 ///
137
138 template <typename AlgoType>
139 struct TeamTrsvInternalUpper {
140 template <typename MemberType, typename ScalarType, typename ValueType>
141 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType & /*member*/, const bool /*use_unit_diag*/, const int /*m*/,
142 const ScalarType /*alpha*/, const ValueType *KOKKOS_RESTRICT /*A*/,
143 const int /*as0*/, const int /*as1*/,
144 /**/ ValueType *KOKKOS_RESTRICT /*b*/, const int /*bs0*/) {
145 assert(false && "Error: encounter dummy impl");
146 return 0;
147 }
148 };
149
150 template <>
151 template <typename MemberType, typename ScalarType, typename ValueType>
152 KOKKOS_INLINE_FUNCTION int TeamTrsvInternalUpper<Algo::Trsv::Unblocked>::invoke(
153 const MemberType &member, const bool use_unit_diag, const int m, const ScalarType alpha,
154 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
155 /**/ ValueType *KOKKOS_RESTRICT b, const int bs0) {
156 const ScalarType one(1.0), zero(0.0);
157
158 if (alpha == zero)
159 KokkosBlas::Impl::TeamSetInternal::invoke(member, m, zero, b, bs0);
160 else {
161 if (alpha != one) KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, alpha, b, bs0);
162 if (m <= 0) return 0;
163
164 ValueType *KOKKOS_RESTRICT b0 = b;
165 for (int p = (m - 1); p >= 0; --p) {
166 const int iend = p;
167
168 const ValueType *KOKKOS_RESTRICT a01 = A + p * as1;
169 /**/ ValueType *KOKKOS_RESTRICT beta1 = b + p * bs0;
170
171 member.team_barrier();
172 ValueType local_beta1 = *beta1;
173 if (!use_unit_diag) {
174 const ValueType alpha11 = A[p * as0 + p * as1];
175 local_beta1 = local_beta1 / alpha11;
176 /// make sure local beta is same for all threads
177 member.team_barrier();
178 if (member.team_rank() == 0) *beta1 = local_beta1;
179 }
180 // member.team_barrier();
181 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, iend),
182 [&](const int &i) { b0[i * bs0] -= a01[i * as0] * local_beta1; });
183 }
184 }
185 return 0;
186 }
187
188 template <>
189 template <typename MemberType, typename ScalarType, typename ValueType>
190 KOKKOS_INLINE_FUNCTION int TeamTrsvInternalUpper<Algo::Trsv::Blocked>::invoke(
191 const MemberType &member, const bool use_unit_diag, const int m, const ScalarType alpha,
192 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
193 /**/ ValueType *KOKKOS_RESTRICT b, const int bs0) {
194 const ScalarType one(1.0), zero(0.0), minus_one(-1.0);
195
196 constexpr int mbAlgo = Algo::Trsm::Blocked::mb();
197
198 // note that parallel range is different ( m*n vs m-1*n);
199 if (alpha == zero)
200 KokkosBlas::Impl::TeamSetInternal::invoke(member, m, zero, b, bs0);
201 else {
202 if (alpha != one) KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, alpha, b, bs0);
203 if (m <= 0) return 0;
204
205 InnerTrsmLeftUpperUnitDiag<mbAlgo> trsm_u(as0, as1, bs0, 0);
206 InnerTrsmLeftUpperNonUnitDiag<mbAlgo> trsm_n(as0, as1, bs0, 0);
207
208 const int mb = mbAlgo;
209 for (int pp = 0; pp < m; pp += mb) {
210 const int ptmp = (m - pp - mb), p = (ptmp < 0 ? 0 : ptmp), pb = (mb + (ptmp < 0) * ptmp);
211
212 // trsm update
213 const ValueType *KOKKOS_RESTRICT Ap = A + p * as0 + p * as1;
214 /**/ ValueType *KOKKOS_RESTRICT bp = b + p * bs0;
215
216 member.team_barrier();
217 if (member.team_rank() == 0) {
218 if (use_unit_diag)
219 trsm_u.serial_invoke(Ap, pb, 1, bp);
220 else
221 trsm_n.serial_invoke(Ap, pb, 1, bp);
222 }
223
224 // gemv update
225 member.team_barrier();
226 KokkosBlas::Impl::TeamGemvInternal<Algo::Gemv::Unblocked>::invoke(member, p, pb, minus_one, Ap - p * as0, as0,
227 as1, bp, 1, one, b, bs0);
228 }
229 }
230 return 0;
231 }
232 } // namespace KokkosBatched
233
234 #endif