Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Trsm_TeamVector_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_TRSM_TEAMVECTOR_INTERNAL_HPP
17 #define KOKKOSBATCHED_TRSM_TEAMVECTOR_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22
23 #include "KokkosBlas1_set_impl.hpp"
24 #include "KokkosBlas1_team_scal_impl.hpp"
25
26 namespace KokkosBatched {
27
28 ///
29 /// Team Internal Impl
30 /// ====================
31
32 template <typename AlgoType>
33 struct TeamVectorTrsmInternalLeftLower {
34 template <typename MemberType, typename ScalarType, typename ValueType>
35 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const bool use_unit_diag, const int m, const int n,
36 const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0,
37 const int as1,
38 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1);
39 };
40
41 template <>
42 template <typename MemberType, typename ScalarType, typename ValueType>
43 KOKKOS_INLINE_FUNCTION int TeamVectorTrsmInternalLeftLower<Algo::Trsm::Unblocked>::invoke(
44 const MemberType &member, const bool use_unit_diag, const int m, const int n, const ScalarType alpha,
45 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
46 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
47 const ScalarType one(1.0), zero(0.0);
48
49 if (alpha == zero)
50 KokkosBlas::Impl::TeamVectorSetInternal::invoke(member, m, n, zero, B, bs0, bs1);
51 else {
52 if (alpha != one) KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1);
53 if (m <= 0 || n <= 0) return 0;
54
55 for (int p = 0; p < m; ++p) {
56 // Made this non-const in order to WORKAROUND issue #349
57 int iend = m - p - 1;
58 int jend = n;
59
60 const ValueType *KOKKOS_RESTRICT a21 = iend ? A + (p + 1) * as0 + p * as1 : NULL;
61
62 ValueType *KOKKOS_RESTRICT b1t = B + p * bs0, *KOKKOS_RESTRICT B2 = iend ? B + (p + 1) * bs0 : NULL;
63
64 member.team_barrier();
65 if (!use_unit_diag) {
66 const ValueType alpha11 = A[p * as0 + p * as1];
67 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, 0, jend),
68 [&](const int &j) { b1t[j * bs1] = b1t[j * bs1] / alpha11; });
69 member.team_barrier();
70 }
71 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, iend), [&](const int &i) {
72 Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, jend), [&](const int &j) {
73 // assume layout right for batched computation
74 B2[i * bs0 + j * bs1] -= a21[i * as0] * b1t[j * bs1];
75 });
76 });
77 }
78 }
79 return 0;
80 }
81
82 template <typename AlgoType>
83 struct TeamVectorTrsmInternalLeftUpper {
84 template <typename MemberType, typename ScalarType, typename ValueType>
85 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const bool use_unit_diag, const int m, const int n,
86 const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0,
87 const int as1,
88 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1);
89 };
90
91 template <>
92 template <typename MemberType, typename ScalarType, typename ValueType>
93 KOKKOS_INLINE_FUNCTION int TeamVectorTrsmInternalLeftUpper<Algo::Trsm::Unblocked>::invoke(
94 const MemberType &member, const bool use_unit_diag, const int m, const int n, const ScalarType alpha,
95 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
96 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
97 const ScalarType one(1.0), zero(0.0);
98
99 // note that parallel range is different ( m*n vs m-1*n);
100 if (alpha == zero)
101 KokkosBlas::Impl::TeamVectorSetInternal::invoke(member, m, n, zero, B, bs0, bs1);
102 else {
103 if (alpha != one) KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1);
104 if (m <= 0 || n <= 0) return 0;
105
106 ValueType *KOKKOS_RESTRICT B0 = B;
107 for (int p = (m - 1); p >= 0; --p) {
108 // Made this non-const in order to WORKAROUND issue #349
109 int iend = p;
110 int jend = n;
111
112 const ValueType *KOKKOS_RESTRICT a01 = A + p * as1;
113 /**/ ValueType *KOKKOS_RESTRICT b1t = B + p * bs0;
114
115 member.team_barrier();
116 if (!use_unit_diag) {
117 const ValueType alpha11 = A[p * as0 + p * as1];
118 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, 0, jend),
119 [&](const int &j) { b1t[j * bs1] = b1t[j * bs1] / alpha11; });
120 member.team_barrier();
121 }
122
123 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, iend), [&](const int &i) {
124 Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, jend),
125 [&](const int &j) { B0[i * bs0 + j * bs1] -= a01[i * as0] * b1t[j * bs1]; });
126 });
127 }
128 }
129 return 0;
130 }
131
132 } // namespace KokkosBatched
133
134 #endif