Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Trsv_Serial_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_TRSV_SERIAL_IMPL_HPP
17 #define KOKKOSBATCHED_TRSV_SERIAL_IMPL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include "KokkosBatched_Trsv_Serial_Internal.hpp"
23
24 namespace KokkosBatched {
25
26 ///
27 /// Serial Impl
28 /// ===========
29
30 ///
31 /// Implemented:
32 /// L/NT, U/NT, L/T, U/T
33 ///
34 /// Not yet implemented
35 /// L/CT, U/CT
36
37 ///
38 /// L/NT
39 ///
40
41 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
42 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
43 template <typename ArgDiag>
44 struct SerialTrsv<Uplo::Lower, Trans::NoTranspose, ArgDiag, Algo::Trsv::CompactMKL> {
45 template <typename ScalarType, typename AViewType, typename bViewType>
46 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
47 typedef typename bViewType::value_type vector_type;
48 // typedef typename vector_type::value_type value_type;
49
50 const int m = b.extent(0), n = 1;
51
52 static_assert(is_vector<vector_type>::value, "value type is not vector type");
53 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
54 "AVX, AVX2 and AVX512 is supported");
55 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
56
57 // no error check
58 int r_val = 0;
59 if (A.stride_0() == 1) {
60 mkl_dtrsm_compact(MKL_COL_MAJOR, MKL_LEFT, MKL_LOWER, MKL_NOTRANS,
61 ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT, m, n, alpha, (const double *)A.data(),
62 A.stride_0(), (double *)b.data(), b.stride_0(), format, (MKL_INT)vector_type::vector_length);
63 } else if (A.stride_1() == 1) {
64 mkl_dtrsm_compact(MKL_ROW_MAJOR, MKL_LEFT, MKL_LOWER, MKL_NOTRANS,
65 ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT, m, n, alpha, (const double *)A.data(),
66 A.stride_0(), (double *)b.data(), b.stride_0(), format, (MKL_INT)vector_type::vector_length);
67 } else {
68 r_val = -1;
69 }
70 return r_val;
71 }
72 };
73 #endif
74
75 template <typename ArgDiag>
76 struct SerialTrsv<Uplo::Lower, Trans::NoTranspose, ArgDiag, Algo::Trsv::Unblocked> {
77 template <typename ScalarType, typename AViewType, typename bViewType>
78 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
79 return SerialTrsvInternalLower<Algo::Trsv::Unblocked>::invoke(ArgDiag::use_unit_diag, A.extent(0), alpha, A.data(),
80 A.stride_0(), A.stride_1(), b.data(), b.stride_0());
81 }
82 };
83
84 template <typename ArgDiag>
85 struct SerialTrsv<Uplo::Lower, Trans::NoTranspose, ArgDiag, Algo::Trsv::Blocked> {
86 template <typename ScalarType, typename AViewType, typename bViewType>
87 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
88 return SerialTrsvInternalLower<Algo::Trsv::Blocked>::invoke(ArgDiag::use_unit_diag, A.extent(0), alpha, A.data(),
89 A.stride_0(), A.stride_1(), b.data(), b.stride_0());
90 }
91 };
92
93 ///
94 /// L/T
95 ///
96
97 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
98 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
99 template <typename ArgDiag>
100 struct SerialTrsv<Uplo::Lower, Trans::Transpose, ArgDiag, Algo::Trsv::CompactMKL> {
101 template <typename ScalarType, typename AViewType, typename bViewType>
102 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
103 typedef typename bViewType::value_type vector_type;
104 // typedef typename vector_type::value_type value_type;
105
106 const int m = b.extent(0), n = 1;
107
108 static_assert(is_vector<vector_type>::value, "value type is not vector type");
109 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
110 "AVX, AVX2 and AVX512 is supported");
111 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
112
113 // no error check
114 int r_val = 0;
115 if (A.stride_0() == 1) {
116 mkl_dtrsm_compact(MKL_COL_MAJOR, MKL_LEFT, MKL_LOWER, MKL_TRANS, ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT,
117 m, n, alpha, (const double *)A.data(), A.stride_0(), (double *)b.data(), b.stride_0(), format,
118 (MKL_INT)vector_type::vector_length);
119 } else if (A.stride_1() == 1) {
120 mkl_dtrsm_compact(MKL_ROW_MAJOR, MKL_LEFT, MKL_LOWER, MKL_TRANS, ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT,
121 m, n, alpha, (const double *)A.data(), A.stride_0(), (double *)b.data(), b.stride_0(), format,
122 (MKL_INT)vector_type::vector_length);
123 } else {
124 r_val = -1;
125 }
126 return r_val;
127 }
128 };
129 #endif
130
131 template <typename ArgDiag>
132 struct SerialTrsv<Uplo::Lower, Trans::Transpose, ArgDiag, Algo::Trsv::Unblocked> {
133 template <typename ScalarType, typename AViewType, typename bViewType>
134 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
135 return SerialTrsvInternalUpper<Algo::Trsv::Unblocked>::invoke(ArgDiag::use_unit_diag, A.extent(1), alpha, A.data(),
136 A.stride_1(), A.stride_0(), b.data(), b.stride_0());
137 }
138 };
139
140 template <typename ArgDiag>
141 struct SerialTrsv<Uplo::Lower, Trans::Transpose, ArgDiag, Algo::Trsv::Blocked> {
142 template <typename ScalarType, typename AViewType, typename bViewType>
143 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
144 return SerialTrsvInternalUpper<Algo::Trsv::Blocked>::invoke(ArgDiag::use_unit_diag, A.extent(1), alpha, A.data(),
145 A.stride_1(), A.stride_0(), b.data(), b.stride_0());
146 }
147 };
148
149 ///
150 /// U/NT
151 ///
152
153 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
154 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
155 template <typename ArgDiag>
156 struct SerialTrsv<Uplo::Upper, Trans::NoTranspose, ArgDiag, Algo::Trsv::CompactMKL> {
157 template <typename ScalarType, typename AViewType, typename bViewType>
158 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
159 typedef typename bViewType::value_type vector_type;
160 // typedef typename vector_type::value_type value_type;
161
162 const int m = b.extent(0), n = 1;
163
164 static_assert(is_vector<vector_type>::value, "value type is not vector type");
165 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
166 "AVX, AVX2 and AVX512 is supported");
167 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
168
169 // no error check
170 int r_val = 0;
171 if (A.stride_0() == 1) {
172 mkl_dtrsm_compact(MKL_COL_MAJOR, MKL_LEFT, MKL_UPPER, MKL_NOTRANS,
173 ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT, m, n, alpha, (const double *)A.data(),
174 A.stride_0(), (double *)b.data(), b.stride_0(), format, (MKL_INT)vector_type::vector_length);
175 } else if (A.stride_1() == 1) {
176 mkl_dtrsm_compact(MKL_ROW_MAJOR, MKL_LEFT, MKL_UPPER, MKL_NOTRANS,
177 ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT, m, n, alpha, (const double *)A.data(),
178 A.stride_0(), (double *)b.data(), b.stride_0(), format, (MKL_INT)vector_type::vector_length);
179 } else {
180 r_val = -1;
181 }
182 return r_val;
183 }
184 };
185 #endif
186
187 template <typename ArgDiag>
188 struct SerialTrsv<Uplo::Upper, Trans::NoTranspose, ArgDiag, Algo::Trsv::Unblocked> {
189 template <typename ScalarType, typename AViewType, typename bViewType>
190 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
191 return SerialTrsvInternalUpper<Algo::Trsv::Unblocked>::invoke(ArgDiag::use_unit_diag, A.extent(0), alpha, A.data(),
192 A.stride_0(), A.stride_1(), b.data(), b.stride_0());
193 }
194 };
195
196 template <typename ArgDiag>
197 struct SerialTrsv<Uplo::Upper, Trans::NoTranspose, ArgDiag, Algo::Trsv::Blocked> {
198 template <typename ScalarType, typename AViewType, typename bViewType>
199 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
200 return SerialTrsvInternalUpper<Algo::Trsv::Blocked>::invoke(ArgDiag::use_unit_diag, A.extent(0), alpha, A.data(),
201 A.stride_0(), A.stride_1(), b.data(), b.stride_0());
202 }
203 };
204
205 ///
206 /// U/T
207 ///
208
209 #if defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL) && defined(KOKKOSBATCHED_IMPL_ENABLE_INTEL_MKL_BATCHED) && \
210 defined(__KOKKOSBATCHED_ENABLE_INTEL_MKL_COMPACT_BATCHED__)
211 template <typename ArgDiag>
212 struct SerialTrsv<Uplo::Upper, Trans::Transpose, ArgDiag, Algo::Trsv::CompactMKL> {
213 template <typename ScalarType, typename AViewType, typename bViewType>
214 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
215 typedef typename bViewType::value_type vector_type;
216 // typedef typename vector_type::value_type value_type;
217
218 const int m = b.extent(0), n = 1;
219
220 static_assert(is_vector<vector_type>::value, "value type is not vector type");
221 static_assert(vector_type::vector_length == 4 || vector_type::vector_length == 8,
222 "AVX, AVX2 and AVX512 is supported");
223 const MKL_COMPACT_PACK format = vector_type::vector_length == 8 ? MKL_COMPACT_AVX512 : MKL_COMPACT_AVX;
224
225 // no error check
226 int r_val = 0;
227 if (A.stride_0() == 1) {
228 mkl_dtrsm_compact(MKL_COL_MAJOR, MKL_LEFT, MKL_UPPER, MKL_TRANS, ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT,
229 m, n, alpha, (const double *)A.data(), A.stride_0(), (double *)b.data(), b.stride_0(), format,
230 (MKL_INT)vector_type::vector_length);
231 } else if (A.stride_1() == 1) {
232 mkl_dtrsm_compact(MKL_ROW_MAJOR, MKL_LEFT, MKL_UPPER, MKL_TRANS, ArgDiag::use_unit_diag ? MKL_UNIT : MKL_NONUNIT,
233 m, n, alpha, (const double *)A.data(), A.stride_0(), (double *)b.data(), b.stride_0(), format,
234 (MKL_INT)vector_type::vector_length);
235 } else {
236 r_val = -1;
237 }
238 return r_val;
239 }
240 };
241 #endif
242
243 template <typename ArgDiag>
244 struct SerialTrsv<Uplo::Upper, Trans::Transpose, ArgDiag, Algo::Trsv::Unblocked> {
245 template <typename ScalarType, typename AViewType, typename bViewType>
246 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
247 return SerialTrsvInternalLower<Algo::Trsv::Unblocked>::invoke(ArgDiag::use_unit_diag, A.extent(1), alpha, A.data(),
248 A.stride_1(), A.stride_0(), b.data(), b.stride_0());
249 }
250 };
251
252 template <typename ArgDiag>
253 struct SerialTrsv<Uplo::Upper, Trans::Transpose, ArgDiag, Algo::Trsv::Blocked> {
254 template <typename ScalarType, typename AViewType, typename bViewType>
255 KOKKOS_INLINE_FUNCTION static int invoke(const ScalarType alpha, const AViewType &A, const bViewType &b) {
256 return SerialTrsvInternalLower<Algo::Trsv::Blocked>::invoke(ArgDiag::use_unit_diag, A.extent(1), alpha, A.data(),
257 A.stride_1(), A.stride_0(), b.data(), b.stride_0());
258 }
259 };
260
261 } // namespace KokkosBatched
262
263 #endif