Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Trsm_Team_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_TRSM_TEAM_INTERNAL_HPP
17 #define KOKKOSBATCHED_TRSM_TEAM_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include "KokkosKernels_ExecSpaceUtils.hpp"
23
24 #include "KokkosBlas1_set_impl.hpp"
25 #include "KokkosBlas1_team_scal_impl.hpp"
26 #include "KokkosBatched_InnerTrsm_Serial_Impl.hpp"
27 #include "KokkosBatched_Gemm_Team_Internal.hpp"
28
29 namespace KokkosBatched {
30
31 ///
32 /// Team Internal Impl
33 /// ====================
34
35 template <typename AlgoType>
36 struct TeamTrsmInternalLeftLower {
37 template <typename MemberType, typename ScalarType, typename ValueType>
38 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const bool use_unit_diag, const int m, const int n,
39 const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0,
40 const int as1,
41 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1);
42 };
43
44 template <>
45 template <typename MemberType, typename ScalarType, typename ValueType>
46 KOKKOS_INLINE_FUNCTION int TeamTrsmInternalLeftLower<Algo::Trsm::Unblocked>::invoke(
47 const MemberType &member, const bool use_unit_diag, const int m, const int n, const ScalarType alpha,
48 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
49 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
50 const ScalarType one(1.0), zero(0.0);
51
52 if (alpha == zero)
53 KokkosBlas::Impl::TeamSetInternal::invoke(member, m, n, zero, B, bs0, bs1);
54 else {
55 if (alpha != one) KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1);
56 if (m <= 0 || n <= 0) return 0;
57
58 for (int p = 0; p < m; ++p) {
59 // Made this non-const in order to WORKAROUND issue #349
60 int iend = m - p - 1;
61 int jend = n;
62
63 const ValueType *KOKKOS_RESTRICT a21 = iend ? A + (p + 1) * as0 + p * as1 : NULL;
64
65 ValueType *KOKKOS_RESTRICT b1t = B + p * bs0, *KOKKOS_RESTRICT B2 = iend ? B + (p + 1) * bs0 : NULL;
66
67 member.team_barrier();
68 if (!use_unit_diag) {
69 const ValueType alpha11 = A[p * as0 + p * as1];
70 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, jend),
71 [&](const int &j) { b1t[j * bs1] = b1t[j * bs1] / alpha11; });
72 member.team_barrier();
73 }
74 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, iend * jend), [&](const int &ij) {
75 // assume layout right for batched computation
76 const int i = ij / jend, j = ij % jend;
77 B2[i * bs0 + j * bs1] -= a21[i * as0] * b1t[j * bs1];
78 });
79 }
80 }
81 return 0;
82 }
83
84 template <>
85 template <typename MemberType, typename ScalarType, typename ValueType>
86 KOKKOS_INLINE_FUNCTION int TeamTrsmInternalLeftLower<Algo::Trsm::Blocked>::invoke(
87 const MemberType &member, const bool use_unit_diag, const int m, const int n, const ScalarType alpha,
88 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
89 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
90 constexpr int mbAlgo = Algo::Trsm::Blocked::mb();
91
92 const ScalarType one(1.0), zero(0.0), minus_one(-1.0);
93
94 // note that parallel range is different ( m*n vs m-1*n);
95 if (alpha == zero)
96 KokkosBlas::Impl::TeamSetInternal::invoke(member, m, n, zero, B, bs0, bs1);
97 else {
98 if (alpha != one) KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1);
99 if (m <= 0 || n <= 0) return 0;
100
101 ///
102 /// case host: team size is small and blocksize (mb,nb) is large
103
104 ///
105 /// case GPU: team size is large and blocksize (mb,nb) is small
106 InnerTrsmLeftLowerUnitDiag<mbAlgo> trsm_u(as0, as1, bs0, bs1);
107 InnerTrsmLeftLowerNonUnitDiag<mbAlgo> trsm_n(as0, as1, bs0, bs1);
108
109 auto trsm = [&](const int ib, const int jb, const ValueType *KOKKOS_RESTRICT AA,
110 /**/ ValueType *KOKKOS_RESTRICT BB) {
111 const int mb = mbAlgo;
112 const int tsize = member.team_size();
113 // Made this non-const in order to WORKAROUND issue #349
114 int nb = (jb / tsize + jb % tsize > 0);
115 int np = jb % nb;
116 for (int p = 0; p < ib; p += mb) {
117 // Made this non-const in order to WORKAROUND issue #349
118 int pb = ((p + mb) > ib ? (ib - p) : mb);
119
120 // trsm update
121 const ValueType *KOKKOS_RESTRICT Ap = AA + p * as0 + p * as1;
122 /**/ ValueType *KOKKOS_RESTRICT Bp = BB + p * bs0;
123
124 member.team_barrier();
125 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, (jb / nb) + (np > 0)), [&](const int jj) {
126 // Made this non-const in order to WORKAROUND issue #349
127 int j = jj * nb, qb = (j + nb) > jb ? np : nb;
128 if (use_unit_diag)
129 trsm_u.serial_invoke(Ap, pb, qb, Bp + j * bs1);
130 else
131 trsm_n.serial_invoke(Ap, pb, qb, Bp + j * bs1);
132 });
133 member.team_barrier();
134
135 // gemm update
136 TeamGemmInternal<Algo::Gemm::Blocked>::invoke(member, ib - p - pb, jb, pb, minus_one, Ap + pb * as0, as0, as1,
137 Bp, bs0, bs1, one, Bp + pb * bs0, bs0, bs1);
138 }
139 };
140
141 const bool is_small = true; //(m*n <= 64*64);
142 if (is_small) {
143 trsm(m, n, A, B);
144 } else {
145 // // some cache blocking may need (not priority yet);
146 // trsm(m, n, A, B);
147 }
148 }
149 return 0;
150 }
151
152 template <typename AlgoType>
153 struct TeamTrsmInternalLeftUpper {
154 template <typename MemberType, typename ScalarType, typename ValueType>
155 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const bool use_unit_diag, const int m, const int n,
156 const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0,
157 const int as1,
158 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1);
159 };
160
161 template <>
162 template <typename MemberType, typename ScalarType, typename ValueType>
163 KOKKOS_INLINE_FUNCTION int TeamTrsmInternalLeftUpper<Algo::Trsm::Unblocked>::invoke(
164 const MemberType &member, const bool use_unit_diag, const int m, const int n, const ScalarType alpha,
165 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
166 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
167 const ScalarType one(1.0), zero(0.0);
168
169 // note that parallel range is different ( m*n vs m-1*n);
170 if (alpha == zero)
171 KokkosBlas::Impl::TeamSetInternal::invoke(member, m, n, zero, B, bs0, bs1);
172 else {
173 if (alpha != one) KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1);
174 if (m <= 0 || n <= 0) return 0;
175
176 ValueType *KOKKOS_RESTRICT B0 = B;
177 for (int p = (m - 1); p >= 0; --p) {
178 // Made this non-const in order to WORKAROUND issue #349
179 int iend = p;
180 int jend = n;
181
182 const ValueType *KOKKOS_RESTRICT a01 = A + p * as1;
183 /**/ ValueType *KOKKOS_RESTRICT b1t = B + p * bs0;
184
185 member.team_barrier();
186 if (!use_unit_diag) {
187 const ValueType alpha11 = A[p * as0 + p * as1];
188 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, jend),
189 [&](const int &j) { b1t[j * bs1] = b1t[j * bs1] / alpha11; });
190 member.team_barrier();
191 }
192
193 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, iend * jend), [&](const int &ij) {
194 int i, j;
195 if (KokkosKernels::Impl::is_gpu_exec_space_v<typename MemberType::execution_space>) {
196 i = ij % iend;
197 j = ij / iend;
198 } else {
199 i = ij / jend;
200 j = ij % jend;
201 }
202 B0[i * bs0 + j * bs1] -= a01[i * as0] * b1t[j * bs1];
203 });
204 }
205 }
206 return 0;
207 }
208
209 template <>
210 template <typename MemberType, typename ScalarType, typename ValueType>
211 KOKKOS_INLINE_FUNCTION int TeamTrsmInternalLeftUpper<Algo::Trsm::Blocked>::invoke(
212 const MemberType &member, const bool use_unit_diag, const int m, const int n, const ScalarType alpha,
213 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
214 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
215 constexpr int mbAlgo = Algo::Trsm::Blocked::mb();
216
217 const ScalarType one(1.0), zero(0.0), minus_one(-1.0);
218
219 // note that parallel range is different ( m*n vs m-1*n);
220 if (alpha == zero)
221 KokkosBlas::Impl::TeamSetInternal::invoke(member, m, n, zero, B, bs0, bs1);
222 else {
223 if (alpha != one) KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1);
224 if (m <= 0 || n <= 0) return 0;
225
226 InnerTrsmLeftUpperUnitDiag<mbAlgo> trsm_u(as0, as1, bs0, bs1);
227 InnerTrsmLeftUpperNonUnitDiag<mbAlgo> trsm_n(as0, as1, bs0, bs1);
228
229 auto trsm = [&](const int ib, const int jb, const ValueType *KOKKOS_RESTRICT AA,
230 /**/ ValueType *KOKKOS_RESTRICT BB) {
231 const int mb = mbAlgo; //(ib <=5 ? ib : mbAlgo);
232 const int tsize = member.team_size();
233 // Made this non-const in order to WORKAROUND issue #349
234 int nb = (jb / tsize + jb % tsize > 0);
235 int np = jb % nb;
236 for (int pp = 0; pp < ib; pp += mb) {
237 const int ptmp = (ib - pp - mb), p = (ptmp < 0 ? 0 : ptmp), pb = (mb + (ptmp < 0) * ptmp);
238
239 // trsm update
240 const ValueType *KOKKOS_RESTRICT Ap = AA + p * as0 + p * as1;
241 /**/ ValueType *KOKKOS_RESTRICT Bp = BB + p * bs0;
242
243 member.team_barrier();
244 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, (jb / nb) + (np > 0)), [&](const int &jj) {
245 const int j = jj * nb, qb = (j + nb) > jb ? np : nb;
246 if (use_unit_diag)
247 trsm_u.serial_invoke(Ap, pb, qb, Bp + j * bs1);
248 else
249 trsm_n.serial_invoke(Ap, pb, qb, Bp + j * bs1);
250 });
251 member.team_barrier();
252
253 // gemm update
254 TeamGemmInternal<Algo::Gemm::Blocked>::invoke(member, p, jb, pb, minus_one, Ap - p * as0, as0, as1, Bp, bs0,
255 bs1, one, BB, bs0, bs1);
256 }
257 };
258
259 const bool is_small = true; //(m*n <= 64*64);
260 if (is_small) {
261 trsm(m, n, A, B);
262 } else {
263 // // some cache blocking may need (not priority yet);
264 // trsm(m, n, A, B);
265 }
266 }
267 return 0;
268 }
269
270 } // namespace KokkosBatched
271
272 #endif