Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Trsv_TeamVector_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_TRSV_TEAMVECTOR_INTERNAL_HPP
17 #define KOKKOSBATCHED_TRSV_TEAMVECTOR_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22
23 #include "KokkosBlas1_set_impl.hpp"
24 #include "KokkosBlas1_team_scal_impl.hpp"
25
26 namespace KokkosBatched {
27
28 ///
29 /// Team Internal Impl
30 /// ====================
31
32 ///
33 /// Lower
34 ///
35
36 template <typename AlgoType>
37 struct TeamVectorTrsvInternalLower {
38 template <typename MemberType, typename ScalarType, typename ValueType>
39 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType & /*member*/, const bool /*use_unit_diag*/, const int /*m*/,
40 const ScalarType /*alpha*/, const ValueType *KOKKOS_RESTRICT /*A*/,
41 const int /*as0*/, const int /*as1*/,
42 /**/ ValueType *KOKKOS_RESTRICT /*b*/, const int /*bs0*/) {
43 assert(false && "Error: encounter dummy impl");
44 return 0;
45 }
46 };
47
48 template <>
49 template <typename MemberType, typename ScalarType, typename ValueType>
50 KOKKOS_INLINE_FUNCTION int TeamVectorTrsvInternalLower<Algo::Trsv::Unblocked>::invoke(
51 const MemberType &member, const bool use_unit_diag, const int m, const ScalarType alpha,
52 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
53 /**/ ValueType *KOKKOS_RESTRICT b, const int bs0) {
54 const ScalarType one(1.0), zero(0.0);
55
56 if (alpha == zero)
57 KokkosBlas::Impl::TeamVectorSetInternal::invoke(member, m, zero, b, bs0);
58 else {
59 if (alpha != one) KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, alpha, b, bs0);
60 if (m <= 0) return 0;
61
62 for (int p = 0; p < m; ++p) {
63 const int iend = m - p - 1;
64
65 const ValueType *KOKKOS_RESTRICT a21 = iend ? A + (p + 1) * as0 + p * as1 : NULL;
66
67 ValueType *KOKKOS_RESTRICT beta1 = b + p * bs0, *KOKKOS_RESTRICT b2 = iend ? beta1 + bs0 : NULL;
68
69 member.team_barrier();
70 ValueType local_beta1 = *beta1;
71 if (!use_unit_diag) {
72 const ValueType alpha11 = A[p * as0 + p * as1];
73 local_beta1 = local_beta1 / alpha11;
74
75 member.team_barrier();
76 Kokkos::single(Kokkos::PerTeam(member), [&]() { *beta1 = local_beta1; });
77 }
78 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, 0, iend),
79 [&](const int &i) { b2[i * bs0] -= a21[i * as0] * local_beta1; });
80 }
81 }
82 return 0;
83 }
84
85 ///
86 /// Upper
87 ///
88
89 template <typename AlgoType>
90 struct TeamVectorTrsvInternalUpper {
91 template <typename MemberType, typename ScalarType, typename ValueType>
92 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType & /*member*/, const bool /*use_unit_diag*/, const int /*m*/,
93 const ScalarType /*alpha*/, const ValueType *KOKKOS_RESTRICT /*A*/,
94 const int /*as0*/, const int /*as1*/,
95 /**/ ValueType *KOKKOS_RESTRICT /*b*/, const int /*bs0*/) {
96 assert(false && "Error: encounter dummy impl");
97 return 0;
98 }
99 };
100
101 template <>
102 template <typename MemberType, typename ScalarType, typename ValueType>
103 KOKKOS_INLINE_FUNCTION int TeamVectorTrsvInternalUpper<Algo::Trsv::Unblocked>::invoke(
104 const MemberType &member, const bool use_unit_diag, const int m, const ScalarType alpha,
105 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
106 /**/ ValueType *KOKKOS_RESTRICT b, const int bs0) {
107 const ScalarType one(1.0), zero(0.0);
108
109 if (alpha == zero)
110 KokkosBlas::Impl::TeamVectorSetInternal::invoke(member, m, zero, b, bs0);
111 else {
112 if (alpha != one) KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, alpha, b, bs0);
113 if (m <= 0) return 0;
114
115 ValueType *KOKKOS_RESTRICT b0 = b;
116 for (int p = (m - 1); p >= 0; --p) {
117 const int iend = p;
118
119 const ValueType *KOKKOS_RESTRICT a01 = A + p * as1;
120 /**/ ValueType *KOKKOS_RESTRICT beta1 = b + p * bs0;
121
122 member.team_barrier();
123 ValueType local_beta1 = *beta1;
124 if (!use_unit_diag) {
125 const ValueType alpha11 = A[p * as0 + p * as1];
126 local_beta1 = local_beta1 / alpha11;
127
128 member.team_barrier();
129 Kokkos::single(Kokkos::PerTeam(member), [&]() { *beta1 = local_beta1; });
130 }
131 Kokkos::parallel_for(Kokkos::TeamVectorRange(member, 0, iend),
132 [&](const int &i) { b0[i * bs0] -= a01[i * as0] * local_beta1; });
133 }
134 }
135 return 0;
136 }
137 } // namespace KokkosBatched
138
139 #endif