Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Trmm_Serial_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16
17 #ifndef KOKKOSBATCHED_TRMM_SERIAL_INTERNAL_HPP
18 #define KOKKOSBATCHED_TRMM_SERIAL_INTERNAL_HPP
19
20 #include "KokkosBatched_Util.hpp"
21
22 #include "KokkosBlas1_set_impl.hpp"
23 #include "KokkosBlas1_serial_scal_impl.hpp"
24
25 namespace KokkosBatched {
26
27 template <typename AlgoType>
28 struct SerialTrmmInternalLeftLower {
29 template <typename ScalarType, typename ValueType>
30 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const bool do_conj, const int am, const int an,
31 const int bm, const int bn, const ScalarType alpha,
32 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
33 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1);
34 };
35
36 template <typename AlgoType>
37 struct SerialTrmmInternalLeftUpper {
38 template <typename ScalarType, typename ValueType>
39 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const bool do_conj, const int am, const int an,
40 const int bm, const int bn, const ScalarType alpha,
41 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
42 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1);
43 };
44
45 template <typename AlgoType>
46 struct SerialTrmmInternalRightLower {
47 template <typename ScalarType, typename ValueType>
48 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const bool do_conj, const int am, const int an,
49 const int bm, const int bn, const ScalarType alpha,
50 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
51 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1);
52 };
53
54 template <typename AlgoType>
55 struct SerialTrmmInternalRightUpper {
56 template <typename ScalarType, typename ValueType>
57 KOKKOS_INLINE_FUNCTION static int invoke(const bool use_unit_diag, const bool do_conj, const int am, const int an,
58 const int bm, const int bn, const ScalarType alpha,
59 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
60 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1);
61 };
62
63 // ech-note: use_unit_diag intentionally ignored for now. Compiler can optimize
64 // it out. Assuming that branching logic (especially on GPU) to handle
65 // use_unit_diag will use more cycles than simply doing 1.0*B[idx] for the copy
66 // if use_unit_diag.
67 template <>
68 template <typename ScalarType, typename ValueType>
69 KOKKOS_INLINE_FUNCTION int SerialTrmmInternalLeftLower<Algo::Trmm::Unblocked>::invoke(
70 const bool /*use_unit_diag*/, const bool do_conj, const int am, const int an, const int bm, const int bn,
71 const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
72 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
73 const ScalarType one(1.0), zero(0.0);
74 typedef Kokkos::ArithTraits<ValueType> AT;
75 int left_m = am;
76 int right_n = bn;
77 // echo-TODO: See about coniditionally setting conjOp at compile time.
78 // auto conjOp = noop;
79 // if (do_conj) {
80 // conjOp = AT::conj;
81 //}
82 // printf("SerialTrmmInternalLeftLower\n");
83
84 auto dotLowerLeftConj = [&](const ValueType *KOKKOS_RESTRICT __A, const int __as0, const int __as1,
85 const int __left_row, ValueType *KOKKOS_RESTRICT __B, const int __bs0, const int __bs1,
86 const int __right_col) {
87 auto B_elems = __left_row;
88 ScalarType sum = 0;
89 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
90 #pragma unroll
91 #endif
92 for (int i = 0; i <= B_elems; i++) {
93 // sum += A[left_row, i] * B[i, right_col]
94 sum += AT::conj(__A[__left_row * __as0 + i * __as1]) * __B[i * __bs0 + __bs1 * __right_col];
95 }
96 return sum;
97 };
98
99 auto dotLowerLeft = [&](const ValueType *KOKKOS_RESTRICT __A, const int __as0, const int __as1, const int __left_row,
100 ValueType *KOKKOS_RESTRICT __B, const int __bs0, const int __bs1, const int __right_col) {
101 auto B_elems = __left_row;
102 ScalarType sum = 0;
103 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
104 #pragma unroll
105 #endif
106 for (int i = 0; i <= B_elems; i++) {
107 // sum += A[left_row, i] * B[i, right_col]
108 sum += __A[__left_row * __as0 + i * __as1] * __B[i * __bs0 + __bs1 * __right_col];
109 }
110 return sum;
111 };
112
113 if (bm <= 0 || bn <= 0 || am <= 0 || an <= 0) return 0;
114
115 if (alpha == zero)
116 KokkosBlas::Impl::SerialSetInternal::invoke(bm, bn, zero, B, bs0, bs1);
117 else {
118 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(bm, bn, alpha, B, bs0, bs1);
119
120 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
121 #pragma unroll
122 #endif
123 for (int m = left_m - 1; m >= 0; m--) {
124 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
125 #pragma unroll
126 #endif
127 for (int n = 0; n < right_n; n++) {
128 if (do_conj) {
129 B[m * bs0 + n * bs1] = dotLowerLeftConj(A, as0, as1, m, B, bs0, bs1, n);
130 } else {
131 B[m * bs0 + n * bs1] = dotLowerLeft(A, as0, as1, m, B, bs0, bs1, n);
132 }
133 }
134 }
135 }
136 return 0;
137 }
138
139 // ech-note: use_unit_diag intentionally ignored for now. Compiler can optimize
140 // it out. Assuming that branching logic (especially on GPU) to handle
141 // use_unit_diag will use more cycles than simply doing 1.0*B[idx] for the copy
142 // if use_unit_diag.
143 template <>
144 template <typename ScalarType, typename ValueType>
145 KOKKOS_INLINE_FUNCTION int SerialTrmmInternalRightLower<Algo::Trmm::Unblocked>::invoke(
146 const bool /*use_unit_diag*/, const bool do_conj, const int am, const int an, const int bm, const int bn,
147 const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
148 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
149 const ScalarType one(1.0), zero(0.0);
150 typedef Kokkos::ArithTraits<ValueType> AT;
151 int left_m = bm;
152 int right_n = an;
153 // echo-TODO: See about coniditionally setting conjOp at compile time.
154 // auto conjOp = noop;
155 // if (do_conj) {
156 // conjOp = AT::conj;
157 //}
158
159 // Lower triangular matrix is on RHS with the base facing down.
160 // Everytime we compute a new output row of B, we must shift over to the
161 // right by one in A's column to ensure we skip the 0's.
162 auto dotLowerRightConj = [&](const ValueType *KOKKOS_RESTRICT __A, const int __as0, const int __as1, const int __am,
163 const int __left_row, ValueType *KOKKOS_RESTRICT __B, const int __bs0, const int __bs1,
164 const int __right_col) {
165 auto B_elems = __am - 1;
166 ScalarType sum = 0;
167 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
168 #pragma unroll
169 #endif
170 for (int i = __right_col; i <= B_elems; i++) {
171 // sum += B[left_row, i] * A[i, right_col]
172 sum += __B[__bs0 * __left_row + i * __bs1] * AT::conj(__A[i * __as0 + __right_col * __as1]);
173 }
174 return sum;
175 };
176
177 auto dotLowerRight = [&](const ValueType *KOKKOS_RESTRICT __A, const int __as0, const int __as1, const int __am,
178 const int __left_row, ValueType *KOKKOS_RESTRICT __B, const int __bs0, const int __bs1,
179 const int __right_col) {
180 auto B_elems = __am - 1;
181 ScalarType sum = 0;
182 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
183 #pragma unroll
184 #endif
185 for (int i = __right_col; i <= B_elems; i++) {
186 // sum += B[left_row, i] * A[i, right_col]
187 sum += __B[__bs0 * __left_row + i * __bs1] * __A[i * __as0 + __right_col * __as1];
188 }
189 return sum;
190 };
191
192 if (bm <= 0 || bn <= 0 || am <= 0 || an <= 0) return 0;
193
194 if (alpha == zero)
195 KokkosBlas::Impl::SerialSetInternal::invoke(bm, bn, zero, B, bs0, bs1);
196 else {
197 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(bm, bn, alpha, B, bs0, bs1);
198
199 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
200 #pragma unroll
201 #endif
202 for (int m = 0; m < left_m; m++) {
203 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
204 #pragma unroll
205 #endif
206 for (int n = 0; n < right_n; n++) {
207 if (do_conj) {
208 B[m * bs0 + n * bs1] = dotLowerRightConj(A, as0, as1, am, m, B, bs0, bs1, n);
209 } else {
210 B[m * bs0 + n * bs1] = dotLowerRight(A, as0, as1, am, m, B, bs0, bs1, n);
211 }
212 }
213 }
214 }
215 return 0;
216 }
217
218 template <>
219 template <typename ScalarType, typename ValueType>
220 KOKKOS_INLINE_FUNCTION int SerialTrmmInternalLeftUpper<Algo::Trmm::Unblocked>::invoke(
221 const bool /*use_unit_diag*/, const bool do_conj, const int am, const int an, const int bm, const int bn,
222 const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
223 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
224 const ScalarType one(1.0), zero(0.0);
225 typedef Kokkos::ArithTraits<ValueType> AT;
226 int left_m = am;
227 int right_n = bn;
228 // echo-TODO: See about coniditionally setting conjOp at compile time.
229 // auto conjOp = noop;
230 // if (do_conj) {
231 // conjOp = AT::conj;
232 //}
233
234 auto dotUpperLeftConj = [&](const ValueType *KOKKOS_RESTRICT __A, const int __as0, const int __as1, const int __an,
235 const int __left_row, ValueType *KOKKOS_RESTRICT __B, const int __bs0, const int __bs1,
236 const int __right_col) {
237 auto B_elems = __an - __left_row - 1;
238 ScalarType sum = 0;
239 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
240 #pragma unroll
241 #endif
242 for (int i = 0; i <= B_elems; i++) {
243 // sum += A[left_row, i+left_row] * B[i+left_row, right_col]
244 sum += AT::conj(__A[__left_row * __as0 + (i + __left_row) * __as1]) *
245 __B[(i + __left_row) * __bs0 + __bs1 * __right_col];
246 }
247 return sum;
248 };
249
250 auto dotUpperLeft = [&](const ValueType *KOKKOS_RESTRICT __A, const int __as0, const int __as1, const int __an,
251 const int __left_row, ValueType *KOKKOS_RESTRICT __B, const int __bs0, const int __bs1,
252 const int __right_col) {
253 auto B_elems = __an - __left_row - 1;
254 ScalarType sum = 0;
255 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
256 #pragma unroll
257 #endif
258 for (int i = 0; i <= B_elems; i++) {
259 // sum += A[left_row, i+left_row] * B[i+left_row, right_col]
260 sum += __A[__left_row * __as0 + (i + __left_row) * __as1] * __B[(i + __left_row) * __bs0 + __bs1 * __right_col];
261 }
262 return sum;
263 };
264
265 if (bm <= 0 || bn <= 0 || am <= 0 || an <= 0) return 0;
266
267 if (alpha == zero)
268 KokkosBlas::Impl::SerialSetInternal::invoke(bm, bn, zero, B, bs0, bs1);
269 else {
270 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(bm, bn, alpha, B, bs0, bs1);
271
272 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
273 #pragma unroll
274 #endif
275 for (int m = 0; m < left_m; ++m) {
276 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
277 #pragma unroll
278 #endif
279 for (int n = 0; n < right_n; ++n) {
280 if (do_conj) {
281 B[m * bs0 + n * bs1] = dotUpperLeftConj(A, as0, as1, an, m, B, bs0, bs1, n);
282 } else {
283 B[m * bs0 + n * bs1] = dotUpperLeft(A, as0, as1, an, m, B, bs0, bs1, n);
284 }
285 }
286 }
287 }
288 return 0;
289 }
290
291 template <>
292 template <typename ScalarType, typename ValueType>
293 KOKKOS_INLINE_FUNCTION int SerialTrmmInternalRightUpper<Algo::Trmm::Unblocked>::invoke(
294 const bool /*use_unit_diag*/, const bool do_conj, const int am, const int an, const int bm, const int bn,
295 const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
296 /**/ ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1) {
297 const ScalarType one(1.0), zero(0.0);
298 typedef Kokkos::ArithTraits<ValueType> AT;
299 int left_m = bm;
300 int right_n = an;
301 // echo-TODO: See about coniditionally setting conjOp at compile time.
302 // auto conjOp = noop;
303 // if (do_conj) {
304 // conjOp = AT::conj;
305 //}
306
307 auto dotUpperRightConj = [&](const ValueType *KOKKOS_RESTRICT __A, const int __as0, const int __as1,
308 const int __left_row, ValueType *KOKKOS_RESTRICT __B, const int __bs0, const int __bs1,
309 const int __right_col) {
310 auto B_elems = __right_col;
311 ScalarType sum = 0;
312 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
313 #pragma unroll
314 #endif
315 for (int i = 0; i <= B_elems; i++) {
316 // sum += B[left_row, i] * A[i, right_col]
317 sum += __B[__left_row * __bs0 + i * __bs1] * AT::conj(__A[i * __as0 + __right_col * __as1]);
318 }
319 return sum;
320 };
321
322 auto dotUpperRight = [&](const ValueType *KOKKOS_RESTRICT __A, const int __as0, const int __as1, const int __left_row,
323 ValueType *KOKKOS_RESTRICT __B, const int __bs0, const int __bs1, const int __right_col) {
324 auto B_elems = __right_col;
325 ScalarType sum = 0;
326 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
327 #pragma unroll
328 #endif
329 for (int i = 0; i <= B_elems; i++) {
330 // sum += B[left_row, i] * A[i, right_col]
331 sum += __B[__left_row * __bs0 + i * __bs1] * __A[i * __as0 + __right_col * __as1];
332 }
333 return sum;
334 };
335
336 if (bm <= 0 || bn <= 0 || am <= 0 || an <= 0) return 0;
337
338 if (alpha == zero)
339 KokkosBlas::Impl::SerialSetInternal::invoke(bm, bn, zero, B, bs0, bs1);
340 else {
341 if (alpha != one) KokkosBlas::Impl::SerialScaleInternal::invoke(bm, bn, alpha, B, bs0, bs1);
342
343 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
344 #pragma unroll
345 #endif
346 for (int m = 0; m < left_m; ++m) {
347 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
348 #pragma unroll
349 #endif
350 for (int n = right_n - 1; n >= 0; --n) {
351 if (do_conj) {
352 B[m * bs0 + n * bs1] = dotUpperRightConj(A, as0, as1, m, B, bs0, bs1, n);
353 } else {
354 B[m * bs0 + n * bs1] = dotUpperRight(A, as0, as1, m, B, bs0, bs1, n);
355 }
356 }
357 }
358 }
359 return 0;
360 }
361 } // namespace KokkosBatched
362 #endif // KOKKOSBATCHED_TRMM_SERIAL_INTERNAL_HPP