Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_HostLevel_Gemm_Armpl_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_HOSTLEVEL_GEMM_ARMPL_IMPL_HPP
17 #define KOKKOSBATCHED_HOSTLEVEL_GEMM_ARMPL_IMPL_HPP
18 #if defined(KOKKOSKERNELS_ENABLE_TPL_ARMPL) && ARMPL_BUILD >= 1058
19 #include "KokkosBatched_Util.hpp"
20 #include "KokkosKernels_Error.hpp"
21
22 namespace KokkosBatched {
23 namespace Impl {
24 /********************* BEGIN non-functor-level routines *********************/
25
26 // clang-format off
27 /// \brief Blocking general matrix multiply on a batch of uniform matrices.
28 ///
29 ///
30 /// C = alpha * op(A) * op(B) + beta * C
31 ///
32 /// \tparam ArgTransA Specifies what op does to A:
33 /// Trans::NoTranspose for non-transpose
34 /// Trans::Transpose for transpose
35 /// Trans::ConjTranspose for conjugate transpose (unsupported)
36 /// \tparam ArgTransB Specifies what op does to B:
37 /// Trans::NoTranspose for non-transpose
38 /// Trans::Transpose for transpose
39 /// Trans::ConjTranspose for conjugate transpose (unsupported)
40 /// \tparam HandleType Specifies the handle type of the kernel handle
41 /// \tparam ScalarType Specifies the scalar type of alpha and beta
42 /// \tparam AViewType Input matrix, as a 3-rank Kokkos::View
43 /// \tparam BViewType Input matrix, as a 3-rank Kokkos::View
44 /// \tparam CViewType Input(RHS)/Output(LHS) matrix, as a 3-rank
45 /// Kokkos::View
46 ///
47 /// See struct BatchedGemmHandle for details
48 /// \param handle [in] A handle which specifies how to invoke the batched
49 /// gemm. handle->get_tpl_params() returns &ninter.
50 /// ninter: The number of matrices to interleave.
51 /// \param alpha [in] Input coefficient used for multiplication with A
52 /// \param A [in] Input matrix, as a 3-rank Kokkos::View
53 /// If ArgBatchSzDim == "BatchSzDim::Right", matrix A is MxKxB
54 /// If ArgBatchSzDim == "BatchSzDim::Left", matrix A is BxMxK
55 /// \param B [in] Input matrix, as a 3-rank Kokkos::View
56 /// If ArgBatchSzDim == "BatchSzDim::Right", matrix B is KxNxB
57 /// If ArgBatchSzDim == "BatchSzDim::Left", matrix B is BxKxN
58 /// \param beta [in] Input coefficient used for multiplication with C
59 /// \param C [in/out] Input/Output matrix, as a 3-rank Kokkos::View
60 /// If ArgBatchSzDim == "BatchSzDim::Right", matrix C is MxNxB
61 /// If ArgBatchSzDim == "BatchSzDim::Left", matrix C is BxMxN
62 /// \return 0 upon success, non-zero otherwise
63 ///
64
65 /// Usage Example:
66 /// BatchedArmplGemm<ArgTransA, ArgTransB, ArgBatchSzDim, HandleType,
67 /// ScalarType, AViewType, BViewType, CViewType>
68 /// (handle, alpha, A, B, beta, C).invoke();
69 // clang-format on
70 template <class ArgTransA, class ArgTransB, class ArgBatchSzDim, class HandleType, class ScalarType, class AViewType,
71 class BViewType, class CViewType>
72 class BatchedArmplGemm {
73 private:
74 HandleType *const __handle;
75 using avt = typename AViewType::value_type;
76 using bvt = typename BViewType::value_type;
77 using cvt = typename CViewType::value_type;
78 cvt tag;
79
80 AViewType __A;
81 avt *__Adp = nullptr;
82 armpl_int_t __Ajstrd, __Aistrd, __Abstrd;
83
84 BViewType __B;
85 bvt *__Bdp = nullptr;
86 armpl_int_t __Bjstrd, __Bistrd, __Bbstrd;
87
88 CViewType __C;
89 cvt *__Cdp = nullptr;
90 armpl_int_t __Cjstrd, __Cistrd, __Cbstrd;
91
92 ScalarType __alpha, __beta;
93 armpl_int_t __ninter, __nbatch;
94
95 char __transa, __transb;
96
97 ArgTransA __transa_tag;
98 ArgTransB __transb_tag;
99 Trans::NoTranspose __no_trans_tag;
100 ArgBatchSzDim __batch_layout_tag;
101
102 armpl_int_t __Am, __An, __Bm, __Bn, __Cm, __Cn;
103
104 template <class T>
105 std::enable_if_t<std::is_same<T, double>::value, void> __unpack_views(T &) {
106 for (int ib = 0; ib < __nbatch; ++ib) {
107 for (int i = 0; i < __ninter; ++i) {
108 auto svA =
109 subview_wrapper(__A, ib * __ninter + i, Kokkos::ALL(), Kokkos::ALL(), __batch_layout_tag, __no_trans_tag);
110 auto svB =
111 subview_wrapper(__B, ib * __ninter + i, Kokkos::ALL(), Kokkos::ALL(), __batch_layout_tag, __no_trans_tag);
112 auto svC =
113 subview_wrapper(__C, ib * __ninter + i, Kokkos::ALL(), Kokkos::ALL(), __batch_layout_tag, __no_trans_tag);
114
115 auto info = armpl_dge_interleave(__ninter, i, __Am, __An, svA.data(), svA.stride(0), svA.stride(1),
116 &__Adp[__Abstrd * ib], __Aistrd, __Ajstrd);
117 if (info != ARMPL_STATUS_SUCCESS) {
118 std::ostringstream os;
119 os << "armpl_dge_interleave(A) returned:" << info << std::endl;
120 KokkosKernels::Impl::throw_runtime_exception(os.str());
121 }
122
123 info = armpl_dge_interleave(__ninter, i, __Bm, __Bn, svB.data(), svB.stride(0), svB.stride(1),
124 &__Bdp[__Bbstrd * ib], __Bistrd, __Bjstrd);
125 if (info != ARMPL_STATUS_SUCCESS) {
126 std::ostringstream os;
127 os << "armpl_dge_interleave(B) returned:" << info << std::endl;
128 KokkosKernels::Impl::throw_runtime_exception(os.str());
129 }
130
131 info = armpl_dge_interleave(__ninter, i, __Cm, __Cn, svC.data(), svC.stride(0), svC.stride(1),
132 &__Cdp[__Cbstrd * ib], __Cistrd, __Cjstrd);
133 if (info != ARMPL_STATUS_SUCCESS) {
134 std::ostringstream os;
135 os << "armpl_dge_interleave(C) returned:" << info << std::endl;
136 KokkosKernels::Impl::throw_runtime_exception(os.str());
137 }
138 }
139 }
140 return;
141 }
142
143 template <class T>
144 std::enable_if_t<std::is_same<T, double>::value, void> __repack_view(T &) {
145 for (int ib = 0; ib < __nbatch; ++ib) {
146 for (int i = 0; i < __ninter; ++i) {
147 auto svC =
148 subview_wrapper(__C, ib * __ninter + i, Kokkos::ALL(), Kokkos::ALL(), __batch_layout_tag, __no_trans_tag);
149
150 auto info = armpl_dge_deinterleave(__ninter, i, __Cm, __Cn, svC.data(), svC.stride(0), svC.stride(1),
151 &__Cdp[__Cbstrd * ib], __Cistrd, __Cjstrd);
152 if (info != ARMPL_STATUS_SUCCESS) {
153 std::ostringstream os;
154 os << "armpl_dge_deinterleave returned:" << info << std::endl;
155 KokkosKernels::Impl::throw_runtime_exception(os.str());
156 }
157 }
158 }
159 return;
160 }
161
162 template <class T>
163 std::enable_if_t<std::is_same<T, double>::value, void> __run(T &) {
164 auto info = armpl_dgemm_interleave_batch(__ninter, __nbatch, __transa, __transb, __Cm, __Cn,
165 std::is_same<ArgTransA, Trans::NoTranspose>::value ? __An : __Am, __alpha,
166 __Adp, __Abstrd, __Aistrd, __Ajstrd, __Bdp, __Bbstrd, __Bistrd, __Bjstrd,
167 __beta, __Cdp, __Cbstrd, __Cistrd, __Cjstrd);
168 if (info != ARMPL_STATUS_SUCCESS) {
169 std::ostringstream os;
170 os << "armpl_dgemm_interleave_batch returned :" << info << std::endl;
171 KokkosKernels::Impl::throw_runtime_exception(os.str());
172 }
173 return;
174 }
175
176 // Fallback overloads for any type other than double
177 // These must be provided to allow compilation in and throw a runtime error
178 template <class T>
179 std::enable_if_t<!std::is_same<T, double>::value, void> __unpack_views(T &) {}
180 template <class T>
181 std::enable_if_t<!std::is_same<T, double>::value, void> __repack_view(T &) {}
182 template <class T>
183 std::enable_if_t<!std::is_same<T, double>::value, void> __run(T &) {}
184
185 public:
186 BatchedArmplGemm(HandleType *const handle, ScalarType alpha, AViewType A, BViewType B, ScalarType beta, CViewType C)
187 : __handle(handle), __A(A), __B(B), __C(C), __alpha(alpha), __beta(beta) {
188 __ninter = __handle->get_tpl_params()[0];
189
190 if (std::is_same<ArgBatchSzDim, BatchLayout::Left>::value) {
191 __Am = __A.extent(1);
192 __An = __A.extent(2);
193 __Bm = __B.extent(1);
194 __Bn = __B.extent(2);
195 __Cm = __C.extent(1);
196 __Cn = __C.extent(2);
197 __nbatch = __C.extent(0);
198 } else {
199 __Am = __A.extent(0);
200 __An = __A.extent(1);
201 __Bm = __B.extent(0);
202 __Bn = __B.extent(1);
203 __Cm = __C.extent(0);
204 __Cn = __C.extent(1);
205 __nbatch = __C.extent(2);
206 }
207
208 __Ajstrd = __ninter;
209 __Aistrd = __Ajstrd * __An;
210 __Abstrd = __Aistrd * __Am;
211
212 __Bjstrd = __ninter;
213 __Bistrd = __Bjstrd * __Bn;
214 __Bbstrd = __Bistrd * __Bm;
215
216 __Cjstrd = __ninter;
217 __Cistrd = __Cjstrd * __Cn;
218 __Cbstrd = __Cistrd * __Cm;
219
220 __transa = std::is_same<ArgTransA, Trans::NoTranspose>::value ? 'N' : 'T';
221 __transb = std::is_same<ArgTransB, Trans::NoTranspose>::value ? 'N' : 'T';
222 }
223
224 int invoke() {
225 if (__handle->enableDebug) {
226 std::cerr << "__nbatch:" << std::to_string(__nbatch) << ", __ninter:" << std::to_string(__ninter)
227 << ", __Am:" << std::to_string(__Am) << ", __An:" << std::to_string(__An) << std::endl;
228 }
229
230 if (!std::is_same<avt, double>::value || !std::is_same<bvt, double>::value || !std::is_same<cvt, double>::value ||
231 !std::is_same<ScalarType, double>::value) {
232 std::ostringstream os;
233 os << "KokkosBatched::Impl::BatchedArmplGemm only supports 'double' "
234 "scalar types."
235 << std::endl;
236 KokkosKernels::Impl::throw_runtime_exception(os.str());
237 }
238
239 if (__nbatch != 0) {
240 if (__ninter == 0 || __nbatch % __ninter) {
241 std::ostringstream os;
242 os << "batch size must be evenly divisible by ninter. __nbatch: " << std::to_string(__nbatch)
243 << ", __ninter: " << std::to_string(__ninter) << std::endl;
244 KokkosKernels::Impl::throw_runtime_exception(os.str());
245 }
246
247 // Calculate internal batch size for interleaving
248 __nbatch /= __ninter;
249
250 // Kokkos::Timer timer;
251
252 // Assume that matrices are interleaved properly if the ViewValueType is
253 // SIMD
254 using ViewValueType = typename CViewType::value_type;
255 if (is_vector<ViewValueType>::value || __ninter == 1) {
256 __Adp = __A.data();
257 __Bdp = __B.data();
258 __Cdp = __C.data();
259 } else {
260 // Allocate space for interleaving
261 __Adp = new avt[__Abstrd * __nbatch];
262 __Bdp = new bvt[__Bbstrd * __nbatch];
263 __Cdp = new cvt[__Cbstrd * __nbatch];
264
265 // timer.reset();
266 __unpack_views<cvt>(tag);
267 // std::cout << "TIME(s): __unpack_views: " << timer.seconds() <<
268 // std::endl;
269 }
270
271 // timer.reset();
272 __run<cvt>(tag);
273 // std::cout << "TIME(s): __run: " << timer.seconds() << std::endl;
274
275 if (!(is_vector<ViewValueType>::value || __ninter == 1)) {
276 delete __Adp;
277 delete __Bdp;
278 // timer.reset();
279 __repack_view<cvt>(tag);
280 // std::cout << "TIME(s): __repack_view: " << timer.seconds() <<
281 // std::endl;
282 delete __Cdp;
283 }
284 }
285 return 0;
286 }
287 };
288 /********************* END non-functor-level routines *********************/
289 } // namespace Impl
290 } // namespace KokkosBatched
291 #endif // KOKKOSKERNELS_ENABLE_TPL_ARMPL
292 #endif