Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Trsv_Serial_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_TRSV_SERIAL_INTERNAL_HPP
17 #define KOKKOSBATCHED_TRSV_SERIAL_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22
23 #include "KokkosBlas1_set_impl.hpp"
24 #include "KokkosBlas1_serial_scal_impl.hpp"
25 #include "KokkosBatched_InnerTrsm_Serial_Impl.hpp"
26 #include "KokkosBlas2_serial_gemv_internal.hpp"
27
28 namespace KokkosBatched {
29
30 ///
31 /// Serial Internal Impl
32 /// ====================
33
34 ///
35 /// Lower
36 ///
37
38 template <typename AlgoType>
39 struct SerialTrsvInternalLower {
40 template <typename ScalarType, typename ValueType>
41 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const int m, const ScalarType alpha,
42 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
43 /**/ ValueType *KOKKOS_RESTRICT b, const int bs0);
44 };
45
46 template <>
47 template <typename ScalarType, typename ValueType>
48 KOKKOS_INLINE_FUNCTION int SerialTrsvInternalLower<Algo::Trsv::Unblocked>::invoke(const bool use_unit_diag, const int m,
49 const ScalarType alpha,
50 const ValueType *KOKKOS_RESTRICT A,
51 const int as0, const int as1,
52 /**/ ValueType *KOKKOS_RESTRICT b,
53 const int bs0) {
54 const ScalarType one(1.0), zero(0.0);
55
56 if (alpha == zero)
57 KokkosBlas::Impl::SerialSetInternal::invoke(m, zero, b, bs0);
58 else {
59 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(m, alpha, b, bs0);
60 if (m <= 0) return 0;
61
62 for (int p = 0; p < m; ++p) {
63 const int iend = m - p - 1;
64
65 const ValueType *KOKKOS_RESTRICT a21 = iend ? A + (p + 1) * as0 + p * as1 : NULL;
66
67 ValueType *KOKKOS_RESTRICT beta1 = b + p * bs0, *KOKKOS_RESTRICT b2 = iend ? beta1 + bs0 : NULL;
68
69 // with KOKKOS_RESTRICT a compiler assumes that the pointer is not
70 // accessed by others op(/=) uses this pointer and changes the associated
71 // values, which brings a compiler problem
72 if (!use_unit_diag) *beta1 = *beta1 / A[p * as0 + p * as1];
73
74 for (int i = 0; i < iend; ++i) b2[i * bs0] -= a21[i * as0] * (*beta1);
75 }
76 }
77 return 0;
78 }
79
80 template <>
81 template <typename ScalarType, typename ValueType>
82 KOKKOS_INLINE_FUNCTION int SerialTrsvInternalLower<Algo::Trsv::Blocked>::invoke(const bool use_unit_diag, const int m,
83 const ScalarType alpha,
84 const ValueType *KOKKOS_RESTRICT A,
85 const int as0, const int as1,
86 /**/ ValueType *KOKKOS_RESTRICT b,
87 const int bs0) {
88 const ScalarType one(1.0), zero(0.0), minus_one(-1.0);
89
90 constexpr int mbAlgo = Algo::Trsv::Blocked::mb();
91
92 if (alpha == zero)
93 KokkosBlas::Impl::SerialSetInternal::invoke(m, zero, b, bs0);
94 else {
95 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(m, alpha, b, bs0);
96 if (m <= 0) return 0;
97
98 /// case GPU: team size is large and blocksize (mb,nb) is small
99 InnerTrsmLeftLowerUnitDiag<mbAlgo> trsm_u(as0, as1, bs0, 0);
100 InnerTrsmLeftLowerNonUnitDiag<mbAlgo> trsm_n(as0, as1, bs0, 0);
101
102 const int mb = mbAlgo;
103 for (int p = 0; p < m; p += mb) {
104 const int pb = ((p + mb) > m ? (m - p) : mb);
105
106 // trsm update
107 const ValueType *KOKKOS_RESTRICT Ap = A + p * as0 + p * as1;
108 /**/ ValueType *KOKKOS_RESTRICT bp = b + p * bs0;
109
110 if (use_unit_diag)
111 trsm_u.serial_invoke(Ap, pb, 1, bp);
112 else
113 trsm_n.serial_invoke(Ap, pb, 1, bp);
114
115 // gemv update
116 KokkosBlas::Impl::SerialGemvInternal<Algo::Gemv::Blocked>::invoke(m - p - pb, pb, minus_one, Ap + pb * as0, as0,
117 as1, bp, bs0, one, bp + pb * bs0, bs0);
118 }
119 }
120 return 0;
121 }
122
123 ///
124 /// Upper
125 ///
126
127 template <typename AlgoType>
128 struct SerialTrsvInternalUpper {
129 template <typename ScalarType, typename ValueType>
130 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const int m, const ScalarType alpha,
131 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
132 /**/ ValueType *KOKKOS_RESTRICT b, const int bs0);
133 };
134
135 template <>
136 template <typename ScalarType, typename ValueType>
137 KOKKOS_INLINE_FUNCTION int SerialTrsvInternalUpper<Algo::Trsv::Unblocked>::invoke(const bool use_unit_diag, const int m,
138 const ScalarType alpha,
139 const ValueType *KOKKOS_RESTRICT A,
140 const int as0, const int as1,
141 /**/ ValueType *KOKKOS_RESTRICT b,
142 const int bs0) {
143 const ScalarType one(1.0), zero(0.0);
144
145 if (alpha == zero)
146 KokkosBlas::Impl::SerialSetInternal::invoke(m, zero, b, bs0);
147 else {
148 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(m, alpha, b, bs0);
149 if (m <= 0) return 0;
150
151 ValueType *KOKKOS_RESTRICT b0 = b;
152 for (int p = (m - 1); p >= 0; --p) {
153 const int iend = p;
154
155 const ValueType *KOKKOS_RESTRICT a01 = A + p * as1;
156 /**/ ValueType *KOKKOS_RESTRICT beta1 = b + p * bs0;
157
158 // with KOKKOS_RESTRICT a compiler assumes that the pointer is not
159 // accessed by others op(/=) uses this pointer and changes the associated
160 // values, which brings a compiler problem
161 if (!use_unit_diag) *beta1 = *beta1 / A[p * as0 + p * as1];
162
163 for (int i = 0; i < iend; ++i) b0[i * bs0] -= a01[i * as0] * (*beta1);
164 }
165 }
166 return 0;
167 }
168
169 template <>
170 template <typename ScalarType, typename ValueType>
171 KOKKOS_INLINE_FUNCTION int SerialTrsvInternalUpper<Algo::Trsv::Blocked>::invoke(const bool use_unit_diag, const int m,
172 const ScalarType alpha,
173 const ValueType *KOKKOS_RESTRICT A,
174 const int as0, const int as1,
175 /**/ ValueType *KOKKOS_RESTRICT b,
176 const int bs0) {
177 const ScalarType one(1.0), zero(0.0), minus_one(-1.0);
178
179 constexpr int mbAlgo = Algo::Trsm::Blocked::mb();
180
181 // note that parallel range is different ( m*n vs m-1*n);
182 if (alpha == zero)
183 KokkosBlas::Impl::SerialSetInternal::invoke(m, zero, b, bs0);
184 else {
185 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(m, alpha, b, bs0);
186 if (m <= 0) return 0;
187
188 InnerTrsmLeftUpperUnitDiag<mbAlgo> trsm_u(as0, as1, bs0, 0);
189 InnerTrsmLeftUpperNonUnitDiag<mbAlgo> trsm_n(as0, as1, bs0, 0);
190
191 const int mb = mbAlgo;
192 for (int pp = 0; pp < m; pp += mb) {
193 const int ptmp = (m - pp - mb), p = (ptmp < 0 ? 0 : ptmp), pb = (mb + (ptmp < 0) * ptmp);
194
195 // trsm update
196 const ValueType *KOKKOS_RESTRICT Ap = A + p * as0 + p * as1;
197 /**/ ValueType *KOKKOS_RESTRICT bp = b + p * bs0;
198
199 if (use_unit_diag)
200 trsm_u.serial_invoke(Ap, pb, 1, bp);
201 else
202 trsm_n.serial_invoke(Ap, pb, 1, bp);
203
204 // gemv update
205 KokkosBlas::Impl::SerialGemvInternal<Algo::Gemv::Blocked>::invoke(p, pb, minus_one, Ap - p * as0, as0, as1, bp,
206 bs0, one, b, bs0);
207 }
208 }
209 return 0;
210 }
211
212 } // namespace KokkosBatched
213
214 #endif