Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Tbsv_Serial_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16
17 #ifndef KOKKOSBATCHED_TBSV_SERIAL_INTERNAL_HPP_
18 #define KOKKOSBATCHED_TBSV_SERIAL_INTERNAL_HPP_
19
20 /// \author Yuuichi Asahi ([email protected])
21
22 #include "KokkosBatched_Util.hpp"
23
24 namespace KokkosBatched {
25
26 ///
27 /// Serial Internal Impl
28 /// ====================
29
30 ///
31 /// Lower, Non-Transpose
32 ///
33
34 template <typename AlgoType>
35 struct SerialTbsvInternalLower {
36 template <typename ValueType>
37 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const int an, const ValueType *KOKKOS_RESTRICT A,
38 const int as0, const int as1,
39 /**/ ValueType *KOKKOS_RESTRICT x, const int xs0, const int k);
40 };
41
42 template <>
43 template <typename ValueType>
44 KOKKOS_INLINE_FUNCTION int SerialTbsvInternalLower<Algo::Tbsv::Unblocked>::invoke(
45 const bool use_unit_diag, const int an, const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
46 /**/ ValueType *KOKKOS_RESTRICT x, const int xs0, const int k) {
47 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
48 #pragma unroll
49 #endif
50 for (int j = 0; j < an; ++j) {
51 if (x[j * xs0] != static_cast<ValueType>(0)) {
52 if (!use_unit_diag) x[j * xs0] = x[j * xs0] / A[0 + j * as1];
53
54 auto temp = x[j * xs0];
55 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
56 #pragma unroll
57 #endif
58 for (int i = j + 1; i < Kokkos::min(an, j + k + 1); ++i) {
59 x[i * xs0] = x[i * xs0] - temp * A[(i - j) * as0 + j * as1];
60 }
61 }
62 }
63
64 return 0;
65 }
66
67 ///
68 /// Lower, Transpose
69 ///
70
71 template <typename AlgoType>
72 struct SerialTbsvInternalLowerTranspose {
73 template <typename ValueType>
74 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const bool do_conj, const int an,
75 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
76 /**/ ValueType *KOKKOS_RESTRICT x, const int xs0, const int k);
77 };
78
79 template <>
80 template <typename ValueType>
81 KOKKOS_INLINE_FUNCTION int SerialTbsvInternalLowerTranspose<Algo::Tbsv::Unblocked>::invoke(
82 const bool use_unit_diag, const bool do_conj, const int an, const ValueType *KOKKOS_RESTRICT A, const int as0,
83 const int as1,
84 /**/ ValueType *KOKKOS_RESTRICT x, const int xs0, const int k) {
85 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
86 #pragma unroll
87 #endif
88 for (int j = an - 1; j >= 0; --j) {
89 auto temp = x[j * xs0];
90
91 if (do_conj) {
92 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
93 #pragma unroll
94 #endif
95 for (int i = Kokkos::min(an - 1, j + k); i > j; --i) {
96 temp -= Kokkos::ArithTraits<ValueType>::conj(A[(i - j) * as0 + j * as1]) * x[i * xs0];
97 }
98 if (!use_unit_diag) temp = temp / Kokkos::ArithTraits<ValueType>::conj(A[0 + j * as1]);
99 } else {
100 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
101 #pragma unroll
102 #endif
103 for (int i = Kokkos::min(an - 1, j + k); i > j; --i) {
104 temp -= A[(i - j) * as0 + j * as1] * x[i * xs0];
105 }
106 if (!use_unit_diag) temp = temp / A[0 + j * as1];
107 }
108 x[j * xs0] = temp;
109 }
110
111 return 0;
112 }
113
114 ///
115 /// Upper, Non-Transpose
116 ///
117
118 template <typename AlgoType>
119 struct SerialTbsvInternalUpper {
120 template <typename ValueType>
121 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const int an, const ValueType *KOKKOS_RESTRICT A,
122 const int as0, const int as1,
123 /**/ ValueType *KOKKOS_RESTRICT x, const int xs0, const int k);
124 };
125
126 template <>
127 template <typename ValueType>
128 KOKKOS_INLINE_FUNCTION int SerialTbsvInternalUpper<Algo::Tbsv::Unblocked>::invoke(
129 const bool use_unit_diag, const int an, const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
130 /**/ ValueType *KOKKOS_RESTRICT x, const int xs0, const int k) {
131 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
132 #pragma unroll
133 #endif
134 for (int j = an - 1; j >= 0; --j) {
135 if (x[j * xs0] != 0) {
136 if (!use_unit_diag) x[j * xs0] = x[j * xs0] / A[k * as0 + j * as1];
137
138 auto temp = x[j * xs0];
139 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
140 #pragma unroll
141 #endif
142 for (int i = j - 1; i >= Kokkos::max(0, j - k); --i) {
143 x[i * xs0] = x[i * xs0] - temp * A[(k - j + i) * as0 + j * as1];
144 }
145 }
146 }
147
148 return 0;
149 }
150
151 ///
152 /// Upper, Transpose
153 ///
154
155 template <typename AlgoType>
156 struct SerialTbsvInternalUpperTranspose {
157 template <typename ValueType>
158 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const bool do_conj, const int an,
159 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
160 /**/ ValueType *KOKKOS_RESTRICT x, const int xs0, const int k);
161 };
162
163 template <>
164 template <typename ValueType>
165 KOKKOS_INLINE_FUNCTION int SerialTbsvInternalUpperTranspose<Algo::Tbsv::Unblocked>::invoke(
166 const bool use_unit_diag, const bool do_conj, const int an, const ValueType *KOKKOS_RESTRICT A, const int as0,
167 const int as1,
168 /**/ ValueType *KOKKOS_RESTRICT x, const int xs0, const int k) {
169 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
170 #pragma unroll
171 #endif
172 for (int j = 0; j < an; j++) {
173 auto temp = x[j * xs0];
174 if (do_conj) {
175 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
176 #pragma unroll
177 #endif
178 for (int i = Kokkos::max(0, j - k); i < j; ++i) {
179 temp -= Kokkos::ArithTraits<ValueType>::conj(A[(i + k - j) * as0 + j * as1]) * x[i * xs0];
180 }
181 if (!use_unit_diag) temp = temp / Kokkos::ArithTraits<ValueType>::conj(A[k * as0 + j * as1]);
182 } else {
183 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
184 #pragma unroll
185 #endif
186 for (int i = Kokkos::max(0, j - k); i < j; ++i) {
187 temp -= A[(i + k - j) * as0 + j * as1] * x[i * xs0];
188 }
189 if (!use_unit_diag) temp = temp / A[k * as0 + j * as1];
190 }
191 x[j * xs0] = temp;
192 }
193
194 return 0;
195 }
196
197 } // namespace KokkosBatched
198
199 #endif // KOKKOSBATCHED_TBSV_SERIAL_INTERNAL_HPP_