| 1 |
//@HEADER |
| 2 |
// ************************************************************************ |
| 3 |
// |
| 4 |
// Kokkos v. 4.0 |
| 5 |
// Copyright (2022) National Technology & Engineering |
| 6 |
// Solutions of Sandia, LLC (NTESS). |
| 7 |
// |
| 8 |
// Under the terms of Contract DE-NA0003525 with NTESS, |
| 9 |
// the U.S. Government retains certain rights in this software. |
| 10 |
// |
| 11 |
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. |
| 12 |
// See https://kokkos.org/LICENSE for license information. |
| 13 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 14 |
// |
| 15 |
//@HEADER |
| 16 |
#ifndef KOKKOSBATCHED_HOSTLEVEL_GEMM_ARMPL_IMPL_HPP |
| 17 |
#define KOKKOSBATCHED_HOSTLEVEL_GEMM_ARMPL_IMPL_HPP |
| 18 |
#if defined(KOKKOSKERNELS_ENABLE_TPL_ARMPL) && ARMPL_BUILD >= 1058 |
| 19 |
#include "KokkosBatched_Util.hpp" |
| 20 |
#include "KokkosKernels_Error.hpp" |
| 21 |
|
| 22 |
namespace KokkosBatched { |
| 23 |
namespace Impl { |
| 24 |
/********************* BEGIN non-functor-level routines *********************/ |
| 25 |
|
| 26 |
// clang-format off |
| 27 |
/// \brief Blocking general matrix multiply on a batch of uniform matrices. |
| 28 |
/// |
| 29 |
/// |
| 30 |
/// C = alpha * op(A) * op(B) + beta * C |
| 31 |
/// |
| 32 |
/// \tparam ArgTransA Specifies what op does to A: |
| 33 |
/// Trans::NoTranspose for non-transpose |
| 34 |
/// Trans::Transpose for transpose |
| 35 |
/// Trans::ConjTranspose for conjugate transpose (unsupported) |
| 36 |
/// \tparam ArgTransB Specifies what op does to B: |
| 37 |
/// Trans::NoTranspose for non-transpose |
| 38 |
/// Trans::Transpose for transpose |
| 39 |
/// Trans::ConjTranspose for conjugate transpose (unsupported) |
| 40 |
/// \tparam HandleType Specifies the handle type of the kernel handle |
| 41 |
/// \tparam ScalarType Specifies the scalar type of alpha and beta |
| 42 |
/// \tparam AViewType Input matrix, as a 3-rank Kokkos::View |
| 43 |
/// \tparam BViewType Input matrix, as a 3-rank Kokkos::View |
| 44 |
/// \tparam CViewType Input(RHS)/Output(LHS) matrix, as a 3-rank |
| 45 |
/// Kokkos::View |
| 46 |
/// |
| 47 |
/// See struct BatchedGemmHandle for details |
| 48 |
/// \param handle [in] A handle which specifies how to invoke the batched |
| 49 |
/// gemm. handle->get_tpl_params() returns &ninter. |
| 50 |
/// ninter: The number of matrices to interleave. |
| 51 |
/// \param alpha [in] Input coefficient used for multiplication with A |
| 52 |
/// \param A [in] Input matrix, as a 3-rank Kokkos::View |
| 53 |
/// If ArgBatchSzDim == "BatchSzDim::Right", matrix A is MxKxB |
| 54 |
/// If ArgBatchSzDim == "BatchSzDim::Left", matrix A is BxMxK |
| 55 |
/// \param B [in] Input matrix, as a 3-rank Kokkos::View |
| 56 |
/// If ArgBatchSzDim == "BatchSzDim::Right", matrix B is KxNxB |
| 57 |
/// If ArgBatchSzDim == "BatchSzDim::Left", matrix B is BxKxN |
| 58 |
/// \param beta [in] Input coefficient used for multiplication with C |
| 59 |
/// \param C [in/out] Input/Output matrix, as a 3-rank Kokkos::View |
| 60 |
/// If ArgBatchSzDim == "BatchSzDim::Right", matrix C is MxNxB |
| 61 |
/// If ArgBatchSzDim == "BatchSzDim::Left", matrix C is BxMxN |
| 62 |
/// \return 0 upon success, non-zero otherwise |
| 63 |
/// |
| 64 |
|
| 65 |
/// Usage Example: |
| 66 |
/// BatchedArmplGemm<ArgTransA, ArgTransB, ArgBatchSzDim, HandleType, |
| 67 |
/// ScalarType, AViewType, BViewType, CViewType> |
| 68 |
/// (handle, alpha, A, B, beta, C).invoke(); |
| 69 |
// clang-format on |
| 70 |
template <class ArgTransA, class ArgTransB, class ArgBatchSzDim, class HandleType, class ScalarType, class AViewType, |
| 71 |
class BViewType, class CViewType> |
| 72 |
class BatchedArmplGemm { |
| 73 |
private: |
| 74 |
HandleType *const __handle; |
| 75 |
using avt = typename AViewType::value_type; |
| 76 |
using bvt = typename BViewType::value_type; |
| 77 |
using cvt = typename CViewType::value_type; |
| 78 |
cvt tag; |
| 79 |
|
| 80 |
AViewType __A; |
| 81 |
avt *__Adp = nullptr; |
| 82 |
armpl_int_t __Ajstrd, __Aistrd, __Abstrd; |
| 83 |
|
| 84 |
BViewType __B; |
| 85 |
bvt *__Bdp = nullptr; |
| 86 |
armpl_int_t __Bjstrd, __Bistrd, __Bbstrd; |
| 87 |
|
| 88 |
CViewType __C; |
| 89 |
cvt *__Cdp = nullptr; |
| 90 |
armpl_int_t __Cjstrd, __Cistrd, __Cbstrd; |
| 91 |
|
| 92 |
ScalarType __alpha, __beta; |
| 93 |
armpl_int_t __ninter, __nbatch; |
| 94 |
|
| 95 |
char __transa, __transb; |
| 96 |
|
| 97 |
ArgTransA __transa_tag; |
| 98 |
ArgTransB __transb_tag; |
| 99 |
Trans::NoTranspose __no_trans_tag; |
| 100 |
ArgBatchSzDim __batch_layout_tag; |
| 101 |
|
| 102 |
armpl_int_t __Am, __An, __Bm, __Bn, __Cm, __Cn; |
| 103 |
|
| 104 |
template <class T> |
| 105 |
std::enable_if_t<std::is_same<T, double>::value, void> __unpack_views(T &) { |
| 106 |
for (int ib = 0; ib < __nbatch; ++ib) { |
| 107 |
for (int i = 0; i < __ninter; ++i) { |
| 108 |
auto svA = |
| 109 |
subview_wrapper(__A, ib * __ninter + i, Kokkos::ALL(), Kokkos::ALL(), __batch_layout_tag, __no_trans_tag); |
| 110 |
auto svB = |
| 111 |
subview_wrapper(__B, ib * __ninter + i, Kokkos::ALL(), Kokkos::ALL(), __batch_layout_tag, __no_trans_tag); |
| 112 |
auto svC = |
| 113 |
subview_wrapper(__C, ib * __ninter + i, Kokkos::ALL(), Kokkos::ALL(), __batch_layout_tag, __no_trans_tag); |
| 114 |
|
| 115 |
auto info = armpl_dge_interleave(__ninter, i, __Am, __An, svA.data(), svA.stride(0), svA.stride(1), |
| 116 |
&__Adp[__Abstrd * ib], __Aistrd, __Ajstrd); |
| 117 |
if (info != ARMPL_STATUS_SUCCESS) { |
| 118 |
std::ostringstream os; |
| 119 |
os << "armpl_dge_interleave(A) returned:" << info << std::endl; |
| 120 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 121 |
} |
| 122 |
|
| 123 |
info = armpl_dge_interleave(__ninter, i, __Bm, __Bn, svB.data(), svB.stride(0), svB.stride(1), |
| 124 |
&__Bdp[__Bbstrd * ib], __Bistrd, __Bjstrd); |
| 125 |
if (info != ARMPL_STATUS_SUCCESS) { |
| 126 |
std::ostringstream os; |
| 127 |
os << "armpl_dge_interleave(B) returned:" << info << std::endl; |
| 128 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 129 |
} |
| 130 |
|
| 131 |
info = armpl_dge_interleave(__ninter, i, __Cm, __Cn, svC.data(), svC.stride(0), svC.stride(1), |
| 132 |
&__Cdp[__Cbstrd * ib], __Cistrd, __Cjstrd); |
| 133 |
if (info != ARMPL_STATUS_SUCCESS) { |
| 134 |
std::ostringstream os; |
| 135 |
os << "armpl_dge_interleave(C) returned:" << info << std::endl; |
| 136 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 137 |
} |
| 138 |
} |
| 139 |
} |
| 140 |
return; |
| 141 |
} |
| 142 |
|
| 143 |
template <class T> |
| 144 |
std::enable_if_t<std::is_same<T, double>::value, void> __repack_view(T &) { |
| 145 |
for (int ib = 0; ib < __nbatch; ++ib) { |
| 146 |
for (int i = 0; i < __ninter; ++i) { |
| 147 |
auto svC = |
| 148 |
subview_wrapper(__C, ib * __ninter + i, Kokkos::ALL(), Kokkos::ALL(), __batch_layout_tag, __no_trans_tag); |
| 149 |
|
| 150 |
auto info = armpl_dge_deinterleave(__ninter, i, __Cm, __Cn, svC.data(), svC.stride(0), svC.stride(1), |
| 151 |
&__Cdp[__Cbstrd * ib], __Cistrd, __Cjstrd); |
| 152 |
if (info != ARMPL_STATUS_SUCCESS) { |
| 153 |
std::ostringstream os; |
| 154 |
os << "armpl_dge_deinterleave returned:" << info << std::endl; |
| 155 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 156 |
} |
| 157 |
} |
| 158 |
} |
| 159 |
return; |
| 160 |
} |
| 161 |
|
| 162 |
template <class T> |
| 163 |
std::enable_if_t<std::is_same<T, double>::value, void> __run(T &) { |
| 164 |
auto info = armpl_dgemm_interleave_batch(__ninter, __nbatch, __transa, __transb, __Cm, __Cn, |
| 165 |
std::is_same<ArgTransA, Trans::NoTranspose>::value ? __An : __Am, __alpha, |
| 166 |
__Adp, __Abstrd, __Aistrd, __Ajstrd, __Bdp, __Bbstrd, __Bistrd, __Bjstrd, |
| 167 |
__beta, __Cdp, __Cbstrd, __Cistrd, __Cjstrd); |
| 168 |
if (info != ARMPL_STATUS_SUCCESS) { |
| 169 |
std::ostringstream os; |
| 170 |
os << "armpl_dgemm_interleave_batch returned :" << info << std::endl; |
| 171 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 172 |
} |
| 173 |
return; |
| 174 |
} |
| 175 |
|
| 176 |
// Fallback overloads for any type other than double |
| 177 |
// These must be provided to allow compilation in and throw a runtime error |
| 178 |
template <class T> |
| 179 |
std::enable_if_t<!std::is_same<T, double>::value, void> __unpack_views(T &) {} |
| 180 |
template <class T> |
| 181 |
std::enable_if_t<!std::is_same<T, double>::value, void> __repack_view(T &) {} |
| 182 |
template <class T> |
| 183 |
std::enable_if_t<!std::is_same<T, double>::value, void> __run(T &) {} |
| 184 |
|
| 185 |
public: |
| 186 |
BatchedArmplGemm(HandleType *const handle, ScalarType alpha, AViewType A, BViewType B, ScalarType beta, CViewType C) |
| 187 |
: __handle(handle), __A(A), __B(B), __C(C), __alpha(alpha), __beta(beta) { |
| 188 |
__ninter = __handle->get_tpl_params()[0]; |
| 189 |
|
| 190 |
if (std::is_same<ArgBatchSzDim, BatchLayout::Left>::value) { |
| 191 |
__Am = __A.extent(1); |
| 192 |
__An = __A.extent(2); |
| 193 |
__Bm = __B.extent(1); |
| 194 |
__Bn = __B.extent(2); |
| 195 |
__Cm = __C.extent(1); |
| 196 |
__Cn = __C.extent(2); |
| 197 |
__nbatch = __C.extent(0); |
| 198 |
} else { |
| 199 |
__Am = __A.extent(0); |
| 200 |
__An = __A.extent(1); |
| 201 |
__Bm = __B.extent(0); |
| 202 |
__Bn = __B.extent(1); |
| 203 |
__Cm = __C.extent(0); |
| 204 |
__Cn = __C.extent(1); |
| 205 |
__nbatch = __C.extent(2); |
| 206 |
} |
| 207 |
|
| 208 |
__Ajstrd = __ninter; |
| 209 |
__Aistrd = __Ajstrd * __An; |
| 210 |
__Abstrd = __Aistrd * __Am; |
| 211 |
|
| 212 |
__Bjstrd = __ninter; |
| 213 |
__Bistrd = __Bjstrd * __Bn; |
| 214 |
__Bbstrd = __Bistrd * __Bm; |
| 215 |
|
| 216 |
__Cjstrd = __ninter; |
| 217 |
__Cistrd = __Cjstrd * __Cn; |
| 218 |
__Cbstrd = __Cistrd * __Cm; |
| 219 |
|
| 220 |
__transa = std::is_same<ArgTransA, Trans::NoTranspose>::value ? 'N' : 'T'; |
| 221 |
__transb = std::is_same<ArgTransB, Trans::NoTranspose>::value ? 'N' : 'T'; |
| 222 |
} |
| 223 |
|
| 224 |
int invoke() { |
| 225 |
if (__handle->enableDebug) { |
| 226 |
std::cerr << "__nbatch:" << std::to_string(__nbatch) << ", __ninter:" << std::to_string(__ninter) |
| 227 |
<< ", __Am:" << std::to_string(__Am) << ", __An:" << std::to_string(__An) << std::endl; |
| 228 |
} |
| 229 |
|
| 230 |
if (!std::is_same<avt, double>::value || !std::is_same<bvt, double>::value || !std::is_same<cvt, double>::value || |
| 231 |
!std::is_same<ScalarType, double>::value) { |
| 232 |
std::ostringstream os; |
| 233 |
os << "KokkosBatched::Impl::BatchedArmplGemm only supports 'double' " |
| 234 |
"scalar types." |
| 235 |
<< std::endl; |
| 236 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 237 |
} |
| 238 |
|
| 239 |
if (__nbatch != 0) { |
| 240 |
if (__ninter == 0 || __nbatch % __ninter) { |
| 241 |
std::ostringstream os; |
| 242 |
os << "batch size must be evenly divisible by ninter. __nbatch: " << std::to_string(__nbatch) |
| 243 |
<< ", __ninter: " << std::to_string(__ninter) << std::endl; |
| 244 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 245 |
} |
| 246 |
|
| 247 |
// Calculate internal batch size for interleaving |
| 248 |
__nbatch /= __ninter; |
| 249 |
|
| 250 |
// Kokkos::Timer timer; |
| 251 |
|
| 252 |
// Assume that matrices are interleaved properly if the ViewValueType is |
| 253 |
// SIMD |
| 254 |
using ViewValueType = typename CViewType::value_type; |
| 255 |
if (is_vector<ViewValueType>::value || __ninter == 1) { |
| 256 |
__Adp = __A.data(); |
| 257 |
__Bdp = __B.data(); |
| 258 |
__Cdp = __C.data(); |
| 259 |
} else { |
| 260 |
// Allocate space for interleaving |
| 261 |
__Adp = new avt[__Abstrd * __nbatch]; |
| 262 |
__Bdp = new bvt[__Bbstrd * __nbatch]; |
| 263 |
__Cdp = new cvt[__Cbstrd * __nbatch]; |
| 264 |
|
| 265 |
// timer.reset(); |
| 266 |
__unpack_views<cvt>(tag); |
| 267 |
// std::cout << "TIME(s): __unpack_views: " << timer.seconds() << |
| 268 |
// std::endl; |
| 269 |
} |
| 270 |
|
| 271 |
// timer.reset(); |
| 272 |
__run<cvt>(tag); |
| 273 |
// std::cout << "TIME(s): __run: " << timer.seconds() << std::endl; |
| 274 |
|
| 275 |
if (!(is_vector<ViewValueType>::value || __ninter == 1)) { |
| 276 |
delete __Adp; |
| 277 |
delete __Bdp; |
| 278 |
// timer.reset(); |
| 279 |
__repack_view<cvt>(tag); |
| 280 |
// std::cout << "TIME(s): __repack_view: " << timer.seconds() << |
| 281 |
// std::endl; |
| 282 |
delete __Cdp; |
| 283 |
} |
| 284 |
} |
| 285 |
return 0; |
| 286 |
} |
| 287 |
}; |
| 288 |
/********************* END non-functor-level routines *********************/ |
| 289 |
} // namespace Impl |
| 290 |
} // namespace KokkosBatched |
| 291 |
#endif // KOKKOSKERNELS_ENABLE_TPL_ARMPL |
| 292 |
#endif |