Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Trsm_Serial_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_TRSM_SERIAL_IMPL_HPP
17 #define KOKKOSBATCHED_TRSM_SERIAL_IMPL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include "KokkosBatched_Trsm_Serial_Internal.hpp"
23
24 namespace KokkosBatched {
25
26 ///
27 /// L/L/NT
28 ///
29 /// B := inv(tril(A)) (alpha*B)
30 /// A(m x m), B(m x n)
31
32 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
33 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
34 template <typename ArgDiag>
35 struct SerialTrsm<Side::Left, Uplo::Lower, Trans::NoTranspose, ArgDiag, Algo::Trsm::CompactMKL> {
36 template <typename ScalarType, typename AViewType, typename BViewType>
37 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
38 typedef typename BViewType::value_type vector_type;
39 // typedef typename vector_type::value_type value_type;
40
41 const int m = B.extent(0), n = B.extent(1);
42
43 static_assert(is_vector<vector_type>::value, "value type is not vector type");
44 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
45 "AVX, AVX2 and AVX512 is supported");
46 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
47
48 // no error check
49 int r_val = 0;
50 if (A.stride_0() == 1 && B.stride_0() == 1) {
51 mkl_dtrsm_compact(MKL_COL_MAJOR, MKL_LEFT, MKL_LOWER, MKL_NOTRANS,
52 ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT, m, n, alpha, (const double *)A.data(),
53 A.stride_1(), (double *)B.data(), B.stride_1(), format, (MKL_INT)vector_type::vector_length);
54 } else if (A.stride_1() == 1 && B.stride_1() == 1) {
55 mkl_dtrsm_compact(MKL_ROW_MAJOR, MKL_LEFT, MKL_LOWER, MKL_NOTRANS,
56 ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT, m, n, alpha, (const double *)A.data(),
57 A.stride_0(), (double *)B.data(), B.stride_0(), format, (MKL_INT)vector_type::vector_length);
58 } else {
59 r_val = -1;
60 }
61 return r_val;
62 }
63 };
64 #endif
65
66 template <typename ArgDiag>
67 struct SerialTrsm<Side::Left, Uplo::Lower, Trans::NoTranspose, ArgDiag, Algo::Trsm::Unblocked> {
68 template <typename ScalarType, typename AViewType, typename BViewType>
69 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
70 return SerialTrsmInternalLeftLower<Algo::Trsm::Unblocked>::invoke(ArgDiag::use_unit_diag, B.extent(0), B.extent(1),
71 alpha, A.data(), A.stride_0(), A.stride_1(),
72 B.data(), B.stride_0(), B.stride_1());
73 }
74 };
75
76 template <typename ArgDiag>
77 struct SerialTrsm<Side::Left, Uplo::Lower, Trans::NoTranspose, ArgDiag, Algo::Trsm::Blocked> {
78 template <typename ScalarType, typename AViewType, typename BViewType>
79 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
80 return SerialTrsmInternalLeftLower<Algo::Trsm::Blocked>::invoke(ArgDiag::use_unit_diag, B.extent(0), B.extent(1),
81 alpha, A.data(), A.stride_0(), A.stride_1(),
82 B.data(), B.stride_0(), B.stride_1());
83 }
84 };
85
86 ///
87 /// R/U/NT
88 ///
89 /// B := (alpha*B) inv(triu(A))
90 /// A(n x n), B(m x n)
91 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
92 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
93 template <typename ArgDiag>
94 struct SerialTrsm<Side::Right, Uplo::Upper, Trans::NoTranspose, ArgDiag, Algo::Trsm::CompactMKL> {
95 template <typename ScalarType, typename AViewType, typename BViewType>
96 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
97 typedef typename BViewType::value_type vector_type;
98 // typedef typename vector_type::value_type value_type;
99
100 const int m = B.extent(0), n = B.extent(1);
101
102 static_assert(is_vector<vector_type>::value, "value type is not vector type");
103 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
104 "AVX, AVX2 and AVX512 is supported");
105 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
106
107 // no error check
108 int r_val = 0;
109 if (A.stride_0() == 1 && B.stride_0() == 1) {
110 mkl_dtrsm_compact(MKL_COL_MAJOR, MKL_RIGHT, MKL_UPPER, MKL_NOTRANS,
111 ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT, m, n, alpha, (const double *)A.data(),
112 A.stride_1(), (double *)B.data(), B.stride_1(), format, (MKL_INT)vector_type::vector_length);
113 } else if (A.stride_1() == 1 && B.stride_1() == 1) {
114 mkl_dtrsm_compact(MKL_ROW_MAJOR, MKL_RIGHT, MKL_UPPER, MKL_NOTRANS,
115 ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT, m, n, alpha, (const double *)A.data(),
116 A.stride_0(), (double *)B.data(), B.stride_0(), format, (MKL_INT)vector_type::vector_length);
117 } else {
118 r_val = -1;
119 }
120 return r_val;
121 }
122 };
123 #endif
124
125 template <typename ArgDiag>
126 struct SerialTrsm<Side::Right, Uplo::Upper, Trans::NoTranspose, ArgDiag, Algo::Trsm::Unblocked> {
127 template <typename ScalarType, typename AViewType, typename BViewType>
128 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
129 return SerialTrsmInternalLeftLower<Algo::Trsm::Unblocked>::invoke(ArgDiag::use_unit_diag, B.extent(1), B.extent(0),
130 alpha, A.data(), A.stride_1(), A.stride_0(),
131 B.data(), B.stride_1(), B.stride_0());
132 }
133 };
134
135 template <typename ArgDiag>
136 struct SerialTrsm<Side::Right, Uplo::Upper, Trans::NoTranspose, ArgDiag, Algo::Trsm::Blocked> {
137 template <typename ScalarType, typename AViewType, typename BViewType>
138 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
139 return SerialTrsmInternalLeftLower<Algo::Trsm::Blocked>::invoke(ArgDiag::use_unit_diag, B.extent(1), B.extent(0),
140 alpha, A.data(), A.stride_1(), A.stride_0(),
141 B.data(), B.stride_1(), B.stride_0());
142 }
143 };
144
145 template <typename ArgDiag>
146 struct SerialTrsm<Side::Right, Uplo::Upper, Trans::Transpose, ArgDiag, Algo::Trsm::Unblocked> {
147 template <typename ScalarType, typename AViewType, typename BViewType>
148 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
149 return SerialTrsmInternalLeftLower<Algo::Trsm::Unblocked>::invoke(ArgDiag::use_unit_diag, B.extent(1), B.extent(0),
150 alpha, A.data(), A.stride_0(), A.stride_1(),
151 B.data(), B.stride_1(), B.stride_0());
152 }
153 };
154
155 template <typename ArgDiag>
156 struct SerialTrsm<Side::Right, Uplo::Upper, Trans::Transpose, ArgDiag, Algo::Trsm::Blocked> {
157 template <typename ScalarType, typename AViewType, typename BViewType>
158 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
159 return SerialTrsmInternalLeftLower<Algo::Trsm::Blocked>::invoke(ArgDiag::use_unit_diag, B.extent(1), B.extent(0),
160 alpha, A.data(), A.stride_0(), A.stride_1(),
161 B.data(), B.stride_1(), B.stride_0());
162 }
163 };
164
165 ///
166 /// L/U/NT
167 ///
168 /// B := inv(triu(A)) (alpha*B)
169 /// A(m x m), B(m x n)
170 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
171 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
172 template <typename ArgDiag>
173 struct SerialTrsm<Side::Left, Uplo::Upper, Trans::NoTranspose, ArgDiag, Algo::Trsm::CompactMKL> {
174 template <typename ScalarType, typename AViewType, typename BViewType>
175 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
176 typedef typename BViewType::value_type vector_type;
177 // typedef typename vector_type::value_type value_type;
178
179 const int m = B.extent(0), n = B.extent(1);
180
181 static_assert(is_vector<vector_type>::value, "value type is not vector type");
182 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
183 "AVX, AVX2 and AVX512 is supported");
184 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
185
186 // no error check
187 int r_val = 0;
188 if (A.stride_0() == 1 && B.stride_0() == 1) {
189 mkl_dtrsm_compact(MKL_COL_MAJOR, MKL_LEFT, MKL_UPPER, MKL_NOTRANS,
190 ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT, m, n, alpha, (const double *)A.data(),
191 A.stride_1(), (double *)B.data(), B.stride_1(), format, (MKL_INT)vector_type::vector_length);
192 } else if (A.stride_1() == 1 && B.stride_1() == 1) {
193 mkl_dtrsm_compact(MKL_ROW_MAJOR, MKL_LEFT, MKL_UPPER, MKL_NOTRANS,
194 ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT, m, n, alpha, (const double *)A.data(),
195 A.stride_0(), (double *)B.data(), B.stride_0(), format, (MKL_INT)vector_type::vector_length);
196 } else {
197 r_val = -1;
198 }
199 return r_val;
200 }
201 };
202 #endif
203
204 template <typename ArgDiag>
205 struct SerialTrsm<Side::Left, Uplo::Upper, Trans::NoTranspose, ArgDiag, Algo::Trsm::Unblocked> {
206 template <typename ScalarType, typename AViewType, typename BViewType>
207 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
208 return SerialTrsmInternalLeftUpper<Algo::Trsm::Unblocked>::invoke(ArgDiag::use_unit_diag, B.extent(0), B.extent(1),
209 alpha, A.data(), A.stride_0(), A.stride_1(),
210 B.data(), B.stride_0(), B.stride_1());
211 }
212 };
213
214 template <typename ArgDiag>
215 struct SerialTrsm<Side::Left, Uplo::Upper, Trans::NoTranspose, ArgDiag, Algo::Trsm::Blocked> {
216 template <typename ScalarType, typename AViewType, typename BViewType>
217 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
218 return SerialTrsmInternalLeftUpper<Algo::Trsm::Blocked>::invoke(ArgDiag::use_unit_diag, B.extent(0), B.extent(1),
219 alpha, A.data(), A.stride_0(), A.stride_1(),
220 B.data(), B.stride_0(), B.stride_1());
221 }
222 };
223
224 ///
225 /// L/L/T
226 ///
227 /// B := inv(tril(AT)) (alpha*B)
228 /// A(m x m), B(m x n)
229
230 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
231 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
232 template <typename ArgDiag>
233 struct SerialTrsm<Side::Left, Uplo::Lower, Trans::Transpose, ArgDiag, Algo::Trsm::CompactMKL> {
234 template <typename ScalarType, typename AViewType, typename BViewType>
235 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
236 typedef typename BViewType::value_type vector_type;
237 // typedef typename vector_type::value_type value_type;
238
239 const int m = B.extent(0), n = B.extent(1);
240
241 static_assert(is_vector<vector_type>::value, "value type is not vector type");
242 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
243 "AVX, AVX2 and AVX512 is supported");
244 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
245
246 // no error check
247 int r_val = 0;
248 if (A.stride_0() == 1 && B.stride_0() == 1) {
249 mkl_dtrsm_compact(MKL_COL_MAJOR, MKL_LEFT, MKL_LOWER, MKL_TRANS, ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT,
250 m, n, alpha, (const double *)A.data(), A.stride_1(), (double *)B.data(), B.stride_1(), format,
251 (MKL_INT)vector_type::vector_length);
252 } else if (A.stride_1() == 1 && B.stride_1() == 1) {
253 mkl_dtrsm_compact(MKL_ROW_MAJOR, MKL_LEFT, MKL_LOWER, MKL_TRANS, ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT,
254 m, n, alpha, (const double *)A.data(), A.stride_0(), (double *)B.data(), B.stride_0(), format,
255 (MKL_INT)vector_type::vector_length);
256 } else {
257 r_val = -1;
258 }
259 return r_val;
260 }
261 };
262 #endif
263
264 template <typename ArgDiag>
265 struct SerialTrsm<Side::Left, Uplo::Lower, Trans::Transpose, ArgDiag, Algo::Trsm::Unblocked> {
266 template <typename ScalarType, typename AViewType, typename BViewType>
267 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
268 return SerialTrsmInternalLeftUpper<Algo::Trsm::Unblocked>::invoke(ArgDiag::use_unit_diag, B.extent(0), B.extent(1),
269 alpha, A.data(), A.stride_1(), A.stride_0(),
270 B.data(), B.stride_0(), B.stride_1());
271 }
272 };
273
274 template <typename ArgDiag>
275 struct SerialTrsm<Side::Left, Uplo::Lower, Trans::Transpose, ArgDiag, Algo::Trsm::Blocked> {
276 template <typename ScalarType, typename AViewType, typename BViewType>
277 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
278 return SerialTrsmInternalLeftUpper<Algo::Trsm::Blocked>::invoke(ArgDiag::use_unit_diag, B.extent(0), B.extent(1),
279 alpha, A.data(), A.stride_1(), A.stride_0(),
280 B.data(), B.stride_0(), B.stride_1());
281 }
282 };
283 ///
284 /// L/U/NT
285 ///
286 /// B := inv(triu(AT)) (alpha*B)
287 /// A(m x m), B(m x n)
288 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
289 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
290 template <typename ArgDiag>
291 struct SerialTrsm<Side::Left, Uplo::Upper, Trans::Transpose, ArgDiag, Algo::Trsm::CompactMKL> {
292 template <typename ScalarType, typename AViewType, typename BViewType>
293 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
294 typedef typename BViewType::value_type vector_type;
295 // typedef typename vector_type::value_type value_type;
296
297 const int m = B.extent(0), n = B.extent(1);
298
299 static_assert(is_vector<vector_type>::value, "value type is not vector type");
300 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
301 "AVX, AVX2 and AVX512 is supported");
302 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
303
304 // no error check
305 int r_val = 0;
306 if (A.stride_0() == 1 && B.stride_0() == 1) {
307 mkl_dtrsm_compact(MKL_COL_MAJOR, MKL_LEFT, MKL_UPPER, MKL_TRANS, ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT,
308 m, n, alpha, (const double *)A.data(), A.stride_1(), (double *)B.data(), B.stride_1(), format,
309 (MKL_INT)vector_type::vector_length);
310 } else if (A.stride_1() == 1 && B.stride_1() == 1) {
311 mkl_dtrsm_compact(MKL_ROW_MAJOR, MKL_LEFT, MKL_UPPER, MKL_TRANS, ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT,
312 m, n, alpha, (const double *)A.data(), A.stride_0(), (double *)B.data(), B.stride_0(), format,
313 (MKL_INT)vector_type::vector_length);
314 } else {
315 r_val = -1;
316 }
317 return r_val;
318 }
319 };
320 #endif
321
322 template <typename ArgDiag>
323 struct SerialTrsm<Side::Left, Uplo::Upper, Trans::Transpose, ArgDiag, Algo::Trsm::Unblocked> {
324 template <typename ScalarType, typename AViewType, typename BViewType>
325 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
326 return SerialTrsmInternalLeftLower<Algo::Trsm::Unblocked>::invoke(ArgDiag::use_unit_diag, B.extent(0), B.extent(1),
327 alpha, A.data(), A.stride_1(), A.stride_0(),
328 B.data(), B.stride_0(), B.stride_1());
329 }
330 };
331
332 template <typename ArgDiag>
333 struct SerialTrsm<Side::Left, Uplo::Upper, Trans::Transpose, ArgDiag, Algo::Trsm::Blocked> {
334 template <typename ScalarType, typename AViewType, typename BViewType>
335 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const BViewType &B) {
336 return SerialTrsmInternalLeftLower<Algo::Trsm::Blocked>::invoke(ArgDiag::use_unit_diag, B.extent(0), B.extent(1),
337 alpha, A.data(), A.stride_1(), A.stride_0(),
338 B.data(), B.stride_0(), B.stride_1());
339 }
340 };
341
342 } // namespace KokkosBatched
343
344 #endif