Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Trsm_Serial_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_TRSM_SERIAL_INTERNAL_HPP
17 #define KOKKOSBATCHED_TRSM_SERIAL_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22
23 #include "KokkosBlas1_set_impl.hpp"
24 #include "KokkosBlas1_serial_scal_impl.hpp"
25 #include "KokkosBatched_InnerGemmFixA_Serial_Impl.hpp"
26 #include "KokkosBatched_InnerTrsm_Serial_Impl.hpp"
27
28 namespace KokkosBatched {
29
30 ///
31 /// Serial Internal Impl
32 /// ====================
33
34 template <typename AlgoType>
35 struct SerialTrsmInternalLeftLower {
36 template <typename ScalarType, typename ValueType>
37 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const int m, const int n, const ScalarType alpha,
38 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
39 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1);
40 };
41
42 template <>
43 template <typename ScalarType, typename ValueType>
44 KOKKOS_INLINE_FUNCTION int SerialTrsmInternalLeftLower<Algo::Trsm::Unblocked>::invoke(
45 const bool use_unit_diag, const int m, const int n, const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A,
46 const int as0, const int as1,
47 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
48 const ScalarType one(1.0), zero(0.0);
49
50 if (alpha == zero)
51 KokkosBlas::Impl::SerialSetInternal::invoke(m, n, zero, B, bs0, bs1);
52 else {
53 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(m, n, alpha, B, bs0, bs1);
54 if (m <= 0 || n <= 0) return 0;
55
56 for (int p = 0; p < m; ++p) {
57 const int iend = m - p - 1, jend = n;
58
59 const ValueType *KOKKOS_RESTRICT a21 = iend ? A + (p + 1) * as0 + p * as1 : NULL;
60
61 ValueType *KOKKOS_RESTRICT b1t = B + p * bs0, *KOKKOS_RESTRICT B2 = iend ? B + (p + 1) * bs0 : NULL;
62
63 if (!use_unit_diag) {
64 const ValueType alpha11 = A[p * as0 + p * as1];
65
66 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
67 #pragma unroll
68 #endif
69 for (int j = 0; j < jend; ++j) b1t[j * bs1] = b1t[j * bs1] / alpha11;
70 }
71
72 for (int i = 0; i < iend; ++i)
73
74 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
75 #pragma unroll
76 #endif
77 for (int j = 0; j < jend; ++j) B2[i * bs0 + j * bs1] -= a21[i * as0] * b1t[j * bs1];
78 }
79 }
80 return 0;
81 }
82
83 template <>
84 template <typename ScalarType, typename ValueType>
85 KOKKOS_INLINE_FUNCTION int SerialTrsmInternalLeftLower<Algo::Trsm::Blocked>::invoke(
86 const bool use_unit_diag, const int m, const int n, const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A,
87 const int as0, const int as1,
88 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
89 constexpr int mbAlgo = Algo::Trsm::Blocked::mb();
90
91 const ScalarType one(1.0), zero(0.0), minus_one(-1.0);
92
93 if (alpha == zero)
94 KokkosBlas::Impl::SerialSetInternal::invoke(m, n, zero, B, bs0, bs1);
95 else {
96 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(m, n, alpha, B, bs0, bs1);
97 if (m <= 0 || n <= 0) return 0;
98
99 InnerTrsmLeftLowerUnitDiag<mbAlgo> trsm_u(as0, as1, bs0, bs1);
100 InnerTrsmLeftLowerNonUnitDiag<mbAlgo> trsm_n(as0, as1, bs0, bs1);
101
102 InnerGemmFixA<mbAlgo, mbAlgo> gemm(as0, as1, bs0, bs1, bs0, bs1);
103 auto trsm = [&](const int ib, const int jb, const ValueType *KOKKOS_RESTRICT AA,
104 /**/ ValueType *KOKKOS_RESTRICT BB) {
105 const int mb = mbAlgo;
106 for (int p = 0; p < ib; p += mb) {
107 const int pb = (p + mb) > ib ? (ib - p) : mb;
108
109 // trsm update
110 const ValueType *KOKKOS_RESTRICT Ap = AA + p * as0 + p * as1;
111 /**/ ValueType *KOKKOS_RESTRICT Bp = BB + p * bs0;
112
113 if (use_unit_diag)
114 trsm_u.serial_invoke(Ap, pb, jb, Bp);
115 else
116 trsm_n.serial_invoke(Ap, pb, jb, Bp);
117
118 // gemm update
119 for (int i = p + mb; i < ib; i += mb) {
120 const int mm = (i + mb) > ib ? (ib - i) : mb;
121 gemm.serial_invoke(minus_one, AA + i * as0 + p * as1, BB + p * bs0, mm, jb, pb, BB + i * bs0);
122 }
123 }
124 };
125
126 const bool is_small = true; //(m*n <= 64*64);
127 if (is_small) {
128 trsm(m, n, A, B);
129 } else {
130 // // some cache blocking may need (not priority yet);
131 // trsm(m, n, A, B);
132 }
133 }
134 return 0;
135 }
136
137 template <typename AlgoType>
138 struct SerialTrsmInternalLeftUpper {
139 template <typename ScalarType, typename ValueType>
140 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const int m, const int n, const ScalarType alpha,
141 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
142 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1);
143 };
144
145 template <>
146 template <typename ScalarType, typename ValueType>
147 KOKKOS_INLINE_FUNCTION int SerialTrsmInternalLeftUpper<Algo::Trsm::Unblocked>::invoke(
148 const bool use_unit_diag, const int m, const int n, const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A,
149 const int as0, const int as1,
150 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
151 const ScalarType one(1.0), zero(0.0);
152
153 if (alpha == zero)
154 KokkosBlas::Impl::SerialSetInternal::invoke(m, n, zero, B, bs0, bs1);
155 else {
156 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(m, n, alpha, B, bs0, bs1);
157 if (m <= 0 || n <= 0) return 0;
158
159 ValueType *KOKKOS_RESTRICT B0 = B;
160 for (int p = (m - 1); p >= 0; --p) {
161 const int iend = p, jend = n;
162
163 const ValueType *KOKKOS_RESTRICT a01 = A + p * as1;
164 ValueType *KOKKOS_RESTRICT b1t = B + p * bs0;
165
166 if (!use_unit_diag) {
167 const ValueType alpha11 = A[p * as0 + p * as1];
168
169 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
170 #pragma unroll
171 #endif
172 for (int j = 0; j < n; ++j) b1t[j * bs1] = b1t[j * bs1] / alpha11;
173 }
174
175 if (p > 0) { // Note: A workaround to produce correct results for
176 // complex<double> with Intel-18.2.199
177 for (int i = 0; i < iend; ++i)
178
179 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
180 #pragma unroll
181 #endif
182 for (int j = 0; j < jend; ++j) B0[i * bs0 + j * bs1] -= a01[i * as0] * b1t[j * bs1];
183 }
184 }
185 }
186 return 0;
187 }
188
189 template <>
190 template <typename ScalarType, typename ValueType>
191 KOKKOS_INLINE_FUNCTION int SerialTrsmInternalLeftUpper<Algo::Trsm::Blocked>::invoke(
192 const bool use_unit_diag, const int m, const int n, const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A,
193 const int as0, const int as1,
194 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
195 const ScalarType one(1.0), zero(0.0), minus_one(-1.0);
196
197 constexpr int mbAlgo = Algo::Trsm::Blocked::mb();
198
199 if (alpha == zero)
200 KokkosBlas::Impl::SerialSetInternal::invoke(m, n, zero, B, bs0, bs1);
201 else {
202 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(m, n, alpha, B, bs0, bs1);
203 if (m <= 0 || n <= 0) return 0;
204
205 InnerTrsmLeftUpperUnitDiag<mbAlgo> trsm_u(as0, as1, bs0, bs1);
206 InnerTrsmLeftUpperNonUnitDiag<mbAlgo> trsm_n(as0, as1, bs0, bs1);
207
208 InnerGemmFixA<mbAlgo, mbAlgo> gemm(as0, as1, bs0, bs1, bs0, bs1);
209
210 auto trsm = [&](const int ib, const int jb, const ValueType *KOKKOS_RESTRICT AA,
211 /**/ ValueType *KOKKOS_RESTRICT BB) {
212 const int mb = mbAlgo;
213 for (int pp = 0; pp < ib; pp += mb) {
214 const int ptmp = ib - pp - mb, p = ptmp < 0 ? 0 : ptmp, pb = mb + (ptmp < 0) * ptmp;
215
216 // trsm update
217 const ValueType *KOKKOS_RESTRICT Ap = AA + p * as0 + p * as1;
218 /**/ ValueType *KOKKOS_RESTRICT Bp = BB + p * bs0;
219
220 if (use_unit_diag)
221 trsm_u.serial_invoke(Ap, pb, jb, Bp);
222 else
223 trsm_n.serial_invoke(Ap, pb, jb, Bp);
224
225 // gemm update
226 for (int i = 0; i < p; i += mb) {
227 gemm.serial_invoke(minus_one, AA + i * as0 + p * as1, Bp, (i + mb) > p ? (p - i) : mb, jb, pb, BB + i * bs0);
228 }
229 }
230 };
231
232 const bool is_small = (m * n <= 64 * 64);
233 if (is_small) {
234 trsm(m, n, A, B);
235 } else {
236 // // some cache blocking may need (not priority yet);
237 // trsm(m, n, A, B);
238 }
239 }
240 return 0;
241 }
242
243 } // namespace KokkosBatched
244
245 #endif