| 1 |
//@HEADER |
| 2 |
// ************************************************************************ |
| 3 |
// |
| 4 |
// Kokkos v. 4.0 |
| 5 |
// Copyright (2022) National Technology & Engineering |
| 6 |
// Solutions of Sandia, LLC (NTESS). |
| 7 |
// |
| 8 |
// Under the terms of Contract DE-NA0003525 with NTESS, |
| 9 |
// the U.S. Government retains certain rights in this software. |
| 10 |
// |
| 11 |
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. |
| 12 |
// See https://kokkos.org/LICENSE for license information. |
| 13 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 14 |
// |
| 15 |
//@HEADER |
| 16 |
#ifndef KOKKOSBATCHED_HOSTLEVEL_GEMM_IMPL_HPP |
| 17 |
#define KOKKOSBATCHED_HOSTLEVEL_GEMM_IMPL_HPP |
| 18 |
#include <Kokkos_Core.hpp> |
| 19 |
#include <KokkosBatched_Util.hpp> // Trans, BatchLayout |
| 20 |
#include <KokkosKernels_ExecSpaceUtils.hpp> |
| 21 |
#include <KokkosKernels_Error.hpp> |
| 22 |
|
| 23 |
#include "KokkosBatched_HostLevel_Gemm_Handle.hpp" // BatchedGemmHandle |
| 24 |
#include "KokkosBatched_HostLevel_Gemm_Serial_Impl.hpp" |
| 25 |
#include "KokkosBatched_HostLevel_Gemm_DblBuf_Impl.hpp" |
| 26 |
#include "KokkosBatched_HostLevel_Gemm_Armpl_Impl.hpp" |
| 27 |
|
| 28 |
namespace KokkosBatched { |
| 29 |
namespace Impl { |
| 30 |
//////////////////////////////// tile_m ////////////////////////////////// |
| 31 |
template <typename ExecutionSpace> |
| 32 |
constexpr KOKKOS_INLINE_FUNCTION int kk_gemm_dbl_buf_tile_m() { |
| 33 |
return 32; |
| 34 |
} |
| 35 |
//////////////////////////////// tile_n ////////////////////////////////// |
| 36 |
template <typename ExecutionSpace> |
| 37 |
constexpr KOKKOS_INLINE_FUNCTION int kk_gemm_dbl_buf_tile_n() { |
| 38 |
return 32; |
| 39 |
} |
| 40 |
//////////////////////////////// tile_k ////////////////////////////////// |
| 41 |
template <typename ExecutionSpace> |
| 42 |
constexpr KOKKOS_INLINE_FUNCTION int kk_gemm_dbl_buf_tile_k() { |
| 43 |
return 8; |
| 44 |
} |
| 45 |
|
| 46 |
// On MI100, batched_scalar_batched_gemm_nt_nt_dcomplex_dcomplex_right fails |
| 47 |
// without this. See https://github.com/kokkos/kokkos-kernels/issues/1547. |
| 48 |
// This reduces the register allocations (REG_M and REG_N) in the double |
| 49 |
// buffering algorithm by a factor of 2. |
| 50 |
#if defined(KOKKOS_ENABLE_HIP) && defined(KOKKOS_ARCH_VEGA908) |
| 51 |
template <> |
| 52 |
constexpr KOKKOS_INLINE_FUNCTION int kk_gemm_dbl_buf_tile_k<Kokkos::HIP>() { |
| 53 |
return 16; |
| 54 |
} |
| 55 |
#endif |
| 56 |
////////////////////////// alpha_in_fma_thresh //////////////////////////// |
| 57 |
constexpr KOKKOS_INLINE_FUNCTION size_t kk_gemm_dbl_buf_alpha_in_fma_thresh() { |
| 58 |
#ifdef __CUDACC_RDC__ |
| 59 |
return 24; |
| 60 |
#else |
| 61 |
return 64; |
| 62 |
#endif // __CUDAACC_RDC__ |
| 63 |
} |
| 64 |
|
| 65 |
template <typename ArgTransA, typename ArgTransB, typename ArgBatchSzDim, typename BatchedGemmHandleType, |
| 66 |
typename ScalarType, typename AViewType, typename BViewType, typename CViewType> |
| 67 |
int BatchedGemmImpl(BatchedGemmHandleType *const handle, const ScalarType alpha, const AViewType &A, const BViewType &B, |
| 68 |
const ScalarType beta, const CViewType &C) { |
| 69 |
int ret = 0; |
| 70 |
size_t c_m, c_n; |
| 71 |
using ViewValueType = typename CViewType::value_type; |
| 72 |
// Check for valid input views |
| 73 |
static_assert(Kokkos::is_view<AViewType>::value, "AViewType must be a Kokkos::View."); |
| 74 |
static_assert(Kokkos::is_view<BViewType>::value, "BViewType must be a Kokkos::View."); |
| 75 |
static_assert(Kokkos::is_view<CViewType>::value, "CViewType must be a Kokkos::View."); |
| 76 |
static_assert(std::is_same<ArgTransA, Trans::NoTranspose>::value || std::is_same<ArgTransA, Trans::Transpose>::value, |
| 77 |
"ArgTransA must be either Trans::Transpose or Trans::NoTranspose."); |
| 78 |
static_assert(std::is_same<ArgTransB, Trans::NoTranspose>::value || std::is_same<ArgTransB, Trans::Transpose>::value, |
| 79 |
"ArgTransB must be either Trans::Transpose or Trans::NoTranspose."); |
| 80 |
if constexpr (is_vector<ViewValueType>::value) { |
| 81 |
// Check ranks of view with underlying SIMD value types |
| 82 |
// For SIMD views, we can have either 3-rank or 4-ranks inputs. |
| 83 |
switch (handle->get_kernel_algo_type()) { |
| 84 |
case BaseKokkosBatchedAlgos::KK_SERIAL: |
| 85 |
case BaseHeuristicAlgos::SQUARE: |
| 86 |
case BaseTplAlgos::ARMPL: |
| 87 |
assert(A.rank_dynamic() == 3 && "AViewType must have rank 3."); |
| 88 |
assert(B.rank_dynamic() == 3 && "BViewType must have rank 3."); |
| 89 |
assert(C.rank_dynamic() == 3 && "CViewType must have rank 3."); |
| 90 |
break; |
| 91 |
default: |
| 92 |
std::ostringstream os; |
| 93 |
os << "KokkosBatched::BatchedGemm does not support kernelAlgoType = " |
| 94 |
<< std::to_string(handle->get_kernel_algo_type()) << " with SIMD views." << std::endl; |
| 95 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 96 |
break; |
| 97 |
} |
| 98 |
} else { |
| 99 |
// Check ranks of views with underlying scalar value types |
| 100 |
static_assert(static_cast<int>(AViewType::rank) == 3, "AViewType must have rank 3."); |
| 101 |
static_assert(static_cast<int>(BViewType::rank) == 3, "BViewType must have rank 3."); |
| 102 |
static_assert(static_cast<int>(CViewType::rank) == 3, "CViewType must have rank 3."); |
| 103 |
} |
| 104 |
|
| 105 |
// Check for valid data access patterns |
| 106 |
// Skip checking a_layout == b_layout == c_layout |
| 107 |
// Skip checking for LayoutStride |
| 108 |
using c_layout = typename CViewType::array_layout; |
| 109 |
static_assert( |
| 110 |
!(std::is_same<c_layout, Kokkos::LayoutLeft>::value && !std::is_same<ArgBatchSzDim, BatchLayout::Right>::value), |
| 111 |
"LayoutLeft views require BatchLayout::Right"); |
| 112 |
static_assert( |
| 113 |
!(std::is_same<c_layout, Kokkos::LayoutRight>::value && !std::is_same<ArgBatchSzDim, BatchLayout::Left>::value), |
| 114 |
"LayoutRight views require BatchLayout::Left"); |
| 115 |
|
| 116 |
if constexpr (std::is_same<ArgBatchSzDim, BatchLayout::Left>::value) { |
| 117 |
// c_b = C.extent(0); |
| 118 |
c_m = C.extent(1); |
| 119 |
c_n = C.extent(2); |
| 120 |
} else { |
| 121 |
// c_b = C.extent(2); |
| 122 |
c_m = C.extent(0); |
| 123 |
c_n = C.extent(1); |
| 124 |
} |
| 125 |
|
| 126 |
// Begin checking conditions for optimal BatchedGemm invocation. |
| 127 |
using view_scalar_type = typename CViewType::value_type; |
| 128 |
using layout_type = typename CViewType::array_layout; |
| 129 |
using exec_space = typename CViewType::execution_space; |
| 130 |
constexpr bool is_vector = KokkosBatched::is_vector<view_scalar_type>::value; |
| 131 |
constexpr bool on_gpu = KokkosKernels::Impl::is_gpu_exec_space_v<exec_space>; |
| 132 |
constexpr bool on_x86_64 = KokkosKernels::Impl::kk_is_x86_64_mem_space<typename exec_space::memory_space>(); |
| 133 |
constexpr bool on_a64fx = KokkosKernels::Impl::kk_is_a64fx_mem_space<typename exec_space::memory_space>(); |
| 134 |
bool out_of_range = false; |
| 135 |
|
| 136 |
if (handle->enableDebug) { |
| 137 |
std::cout << "view_scalar_type:" << typeid(view_scalar_type).name() << std::endl |
| 138 |
<< "execution_space:" << typeid(exec_space).name() << std::endl |
| 139 |
<< std::endl |
| 140 |
<< "is_vector:" << is_vector << std::endl |
| 141 |
<< "on_gpu:" << on_gpu << std::endl |
| 142 |
<< "on_x86_64:" << on_x86_64 << std::endl |
| 143 |
<< "on_a64fx:" << on_a64fx << std::endl; |
| 144 |
} |
| 145 |
|
| 146 |
switch (handle->get_kernel_algo_type()) { |
| 147 |
////////////// HEURISTIC ALGOS ////////////// |
| 148 |
case BaseHeuristicAlgos::SQUARE: |
| 149 |
if (c_m != c_n) { |
| 150 |
std::ostringstream os; |
| 151 |
os << "KokkosBatched::BatchedGemm does not support kernelAlgoType = " |
| 152 |
<< std::to_string(handle->get_kernel_algo_type()) << " when c_m(" << std::to_string(c_m) << ") != c_n(" |
| 153 |
<< std::to_string(c_n) << ")" << std::endl; |
| 154 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 155 |
} |
| 156 |
|
| 157 |
// Select optimal resultsPerThread param for BatchedSerialGemm |
| 158 |
using bsgResultsPerThread = |
| 159 |
std::conditional_t<!is_vector && on_gpu, ResultsPerThread::Rank0, ResultsPerThread::Rank2>; |
| 160 |
|
| 161 |
// Select optimal mode param for SerialGemm. |
| 162 |
using bsgModeType = typename std::conditional< |
| 163 |
is_vector, typename std::conditional<on_gpu || on_x86_64, Algo::Gemm::Blocked, Algo::Gemm::Unblocked>::type, |
| 164 |
typename std::conditional< |
| 165 |
on_gpu, Algo::Gemm::Unblocked, |
| 166 |
typename std::conditional<on_a64fx, Algo::Gemm::Unblocked, Algo::Gemm::Blocked>::type>::type>::type; |
| 167 |
|
| 168 |
if (handle->enableDebug) { |
| 169 |
std::cout << "bsgResultsPerThread: " << typeid(bsgResultsPerThread).name() << std::endl |
| 170 |
<< "bsgModeType: " << typeid(bsgModeType).name() << std::endl; |
| 171 |
} |
| 172 |
|
| 173 |
if constexpr (on_gpu) { |
| 174 |
if (((std::is_same<layout_type, Kokkos::LayoutLeft>::value) ? (c_m >= 16) |
| 175 |
: (c_m >= 24 && c_m <= 32) || c_m >= 40)) { |
| 176 |
handle->teamSz = handle->vecLen = 8; |
| 177 |
constexpr int tile_m = Impl::kk_gemm_dbl_buf_tile_m<exec_space>(); |
| 178 |
constexpr int tile_n = Impl::kk_gemm_dbl_buf_tile_n<exec_space>(); |
| 179 |
constexpr int tile_k = Impl::kk_gemm_dbl_buf_tile_k<exec_space>(); |
| 180 |
constexpr size_t alpha_in_fma_thresh = Impl::kk_gemm_dbl_buf_alpha_in_fma_thresh(); |
| 181 |
|
| 182 |
if (c_m % 32 == 0) { // No bounds checking |
| 183 |
if (c_m >= alpha_in_fma_thresh) { // apply alpha in fma |
| 184 |
ret = Impl::BatchedDblBufGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType, |
| 185 |
AViewType, BViewType, CViewType, BoundsCheck::No, AlphaTag::Yes, tile_m, |
| 186 |
tile_n, tile_k>(handle, alpha, A, B, beta, C) |
| 187 |
.invoke(); |
| 188 |
} else { // apply alpha in mul |
| 189 |
ret = Impl::BatchedDblBufGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType, |
| 190 |
AViewType, BViewType, CViewType, BoundsCheck::No, AlphaTag::No, tile_m, |
| 191 |
tile_n, tile_k>(handle, alpha, A, B, beta, C) |
| 192 |
.invoke(); |
| 193 |
} |
| 194 |
} else { // bounds checking |
| 195 |
if (c_m >= alpha_in_fma_thresh) { // apply alpha in fma |
| 196 |
ret = Impl::BatchedDblBufGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType, |
| 197 |
AViewType, BViewType, CViewType, BoundsCheck::Yes, AlphaTag::Yes, tile_m, |
| 198 |
tile_n, tile_k>(handle, alpha, A, B, beta, C) |
| 199 |
.invoke(); |
| 200 |
} else { // apply alpha in mul |
| 201 |
ret = Impl::BatchedDblBufGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType, |
| 202 |
AViewType, BViewType, CViewType, BoundsCheck::Yes, AlphaTag::No, tile_m, |
| 203 |
tile_n, tile_k>(handle, alpha, A, B, beta, C) |
| 204 |
.invoke(); |
| 205 |
} |
| 206 |
} |
| 207 |
} else { |
| 208 |
out_of_range = true; |
| 209 |
} |
| 210 |
} |
| 211 |
if (!on_gpu || out_of_range) { |
| 212 |
ret = Impl::BatchedSerialGemm<ArgTransA, ArgTransB, bsgModeType, ArgBatchSzDim, bsgResultsPerThread, ScalarType, |
| 213 |
AViewType, BViewType, CViewType>(alpha, A, B, beta, C) |
| 214 |
.invoke(); |
| 215 |
} |
| 216 |
break; |
| 217 |
|
| 218 |
// case BaseHeuristicAlgos::TALL: |
| 219 |
// |
| 220 |
// case BaseHeuristicAlgos::WIDE: |
| 221 |
////////////// TPL ALGOS ////////////// |
| 222 |
#if defined(KOKKOSKERNELS_ENABLE_TPL_ARMPL) && ARMPL_BUILD >= 1058 |
| 223 |
case BaseTplAlgos::ARMPL: |
| 224 |
ret = Impl::BatchedArmplGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType, AViewType, |
| 225 |
BViewType, CViewType>(handle, alpha, A, B, beta, C) |
| 226 |
.invoke(); |
| 227 |
break; |
| 228 |
#endif // KOKKOSKERNELS_ENABLE_TPL_ARMPL |
| 229 |
// case BaseTplAlgos::MKL: |
| 230 |
// |
| 231 |
// case GemmTplAlgos::CUBLAS: |
| 232 |
// |
| 233 |
// case GemmTplAlgos::MAGMA: |
| 234 |
|
| 235 |
////////////// KokkosBatched ALGOS ////////////// |
| 236 |
case BaseKokkosBatchedAlgos::KK_SERIAL: |
| 237 |
ret = Impl::BatchedSerialGemm<ArgTransA, ArgTransB, Algo::Gemm::Unblocked, ArgBatchSzDim, ResultsPerThread::Rank2, |
| 238 |
ScalarType, AViewType, BViewType, CViewType>(alpha, A, B, beta, C) |
| 239 |
.invoke(); |
| 240 |
break; |
| 241 |
|
| 242 |
// case GemmKokkosBatchedAlgos::KK_SERIALSIMD: |
| 243 |
|
| 244 |
case GemmKokkosBatchedAlgos::KK_SERIAL_RANK0: |
| 245 |
ret = Impl::BatchedSerialGemm<ArgTransA, ArgTransB, Algo::Gemm::Unblocked, ArgBatchSzDim, ResultsPerThread::Rank0, |
| 246 |
ScalarType, AViewType, BViewType, CViewType>(alpha, A, B, beta, C) |
| 247 |
.invoke(); |
| 248 |
break; |
| 249 |
|
| 250 |
// case GemmKokkosBatchedAlgos::KK_SERIAL_SHMEM: |
| 251 |
// case GemmKokkosBatchedAlgos::KK_TEAM: |
| 252 |
// case GemmKokkosBatchedAlgos::KK_TEAMVECTOR: |
| 253 |
// case GemmKokkosBatchedAlgos::KK_TEAMSIMD: |
| 254 |
|
| 255 |
case GemmKokkosBatchedAlgos::KK_DBLBUF: |
| 256 |
// Note: The tile sizes of 1x1x1 here will not perform well but must be |
| 257 |
// selected in order to function on all devices since the serial |
| 258 |
// execution space has a max team size of 1. KokkosKernels API users |
| 259 |
// will need to follow an approach similar to KK_SQUARE above for best |
| 260 |
// performance. |
| 261 |
|
| 262 |
// TODO: Add auto-selection of tile size based on inputs and device type |
| 263 |
ret = Impl::BatchedDblBufGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType, AViewType, |
| 264 |
BViewType, CViewType, BoundsCheck::Yes, AlphaTag::No, 1, 1, 1>(handle, alpha, A, B, |
| 265 |
beta, C) |
| 266 |
.invoke(); |
| 267 |
break; |
| 268 |
|
| 269 |
default: |
| 270 |
std::ostringstream os; |
| 271 |
os << "KokkosBatched::BatchedGemm does not support kernelAlgoType = " |
| 272 |
<< std::to_string(handle->get_kernel_algo_type()) << "." << std::endl; |
| 273 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 274 |
break; |
| 275 |
} |
| 276 |
return ret; |
| 277 |
} |
| 278 |
} // namespace Impl |
| 279 |
} // namespace KokkosBatched |
| 280 |
#endif // KOKKOSBATCHED_HOSTLEVEL_GEMM_IMPL_HPP |