| 1 |
//@HEADER |
| 2 |
// ************************************************************************ |
| 3 |
// |
| 4 |
// Kokkos v. 4.0 |
| 5 |
// Copyright (2022) National Technology & Engineering |
| 6 |
// Solutions of Sandia, LLC (NTESS). |
| 7 |
// |
| 8 |
// Under the terms of Contract DE-NA0003525 with NTESS, |
| 9 |
// the U.S. Government retains certain rights in this software. |
| 10 |
// |
| 11 |
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. |
| 12 |
// See https://kokkos.org/LICENSE for license information. |
| 13 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 14 |
// |
| 15 |
//@HEADER |
| 16 |
#ifndef KOKKOSBATCHED_HOSTLEVEL_GEMM_DBLBUF_IMPL_HPP |
| 17 |
#define KOKKOSBATCHED_HOSTLEVEL_GEMM_DBLBUF_IMPL_HPP |
| 18 |
|
| 19 |
#include "KokkosBatched_Util.hpp" |
| 20 |
#include "KokkosKernels_Error.hpp" |
| 21 |
|
| 22 |
namespace KokkosBatched { |
| 23 |
/********************* BEGIN functor-level routines *********************/ |
| 24 |
/********************* END functor-level routines *********************/ |
| 25 |
|
| 26 |
namespace Impl { |
| 27 |
/********************* BEGIN non-functor-level routines *********************/ |
| 28 |
/// |
| 29 |
/// Implemented: |
| 30 |
/// NT/NT, T/NT, NT/T, T/T |
| 31 |
/// |
| 32 |
/// Not yet immplemented (ConjTranspose): |
| 33 |
/// CT/NT, NT/CT, CT/CT |
| 34 |
/// |
| 35 |
|
| 36 |
struct LayoutLeftTag {}; |
| 37 |
struct LayoutRightTag {}; |
| 38 |
template <class> |
| 39 |
struct TagFromLayoutHelper; |
| 40 |
template <> |
| 41 |
struct TagFromLayoutHelper<Kokkos::LayoutLeft> { |
| 42 |
using tag = LayoutLeftTag; |
| 43 |
}; |
| 44 |
template <> |
| 45 |
struct TagFromLayoutHelper<Kokkos::LayoutRight> { |
| 46 |
using tag = LayoutRightTag; |
| 47 |
}; |
| 48 |
template <class Layout> |
| 49 |
using TagFromLayout = typename TagFromLayoutHelper<Layout>::tag; |
| 50 |
|
| 51 |
// TODO - scaling between (32x32, 64x64) |
| 52 |
// Option 0: Increase number of tiles and figure out how to map kokkos teams |
| 53 |
// into cuda grid. Keep team size and vector lanes constant. |
| 54 |
// TODO: write up small example and ask Christian. [DONE, |
| 55 |
// MdRangePolicy not applicable here] |
| 56 |
// Option 1: Increase register sizes to handle rows/cols past tile size |
| 57 |
// Option 2: Fix league_size and have single team solve full tile followed |
| 58 |
// by same team solving extra rows/cols (without multiplying by the |
| 59 |
// zero rows/cols) |
| 60 |
|
| 61 |
// clang-format off |
| 62 |
/// \brief Non-blocking general matrix multiply on a batch of |
| 63 |
/// uniform matrices with an algorithm based on: |
| 64 |
/// B. P. D. J. Kunkel, Julian, “Performance, design, and autotuning of batched gemm for GPUs,” |
| 65 |
/// in Lecture Notes in Computer Science, ser. ISC High Performance Computing ’16, vol. 9697, 06 2016. |
| 66 |
/// |
| 67 |
/// |
| 68 |
/// C = alpha * op(A) * op(B) + beta * C |
| 69 |
/// |
| 70 |
/// \tparam ArgTransA Specifies what op does to A: |
| 71 |
/// Trans::NoTranspose for non-transpose |
| 72 |
/// Trans::Transpose for transpose |
| 73 |
/// Trans::ConjTranspose for conjugate transpose (unsupported) |
| 74 |
/// \tparam ArgTransB Specifies what op does to B: |
| 75 |
/// Trans::NoTranspose for non-transpose |
| 76 |
/// Trans::Transpose for transpose |
| 77 |
/// Trans::ConjTranspose for conjugate transpose (unsupported) |
| 78 |
/// \tparam ArgBatchSzDim Specifies where the batch dimension is allocated in |
| 79 |
/// AViewType, BViewType, and CViewType: |
| 80 |
/// BatchSzDim::Left Batch dimension is leftmost |
| 81 |
/// BatchSzDim::Right Batch dimension is rightmost |
| 82 |
/// \tparam ArgResultsPerThread Specifies how to divide work among threads. For |
| 83 |
/// this serial interface, each rank specifies how |
| 84 |
/// much work to assign a single thread. |
| 85 |
/// ResultsPerThread::Rank0 Each thread computes a scalar of C |
| 86 |
/// ResultsPerThread::Rank1 Each thread computes a 1-rank chunk of C |
| 87 |
/// ResultsPerThread::Rank2 Each thread computes a 2-rank chunk of C |
| 88 |
/// \tparam HandleType Specifies the handle type of the kernel handle |
| 89 |
/// \tparam ScalarType Specifies the scalar type of alpha and beta |
| 90 |
/// \tparam AViewType Input matrix, as either a 3-rank Kokkos::View or a |
| 91 |
/// 4-rank Kokkos::View for SIMD operations. |
| 92 |
/// \tparam BViewType Input matrix, as either a 3-rank Kokkos::View or a |
| 93 |
/// 4-rank Kokkos::View for SIMD operations. |
| 94 |
/// \tparam CViewType Input(RHS)/Output(LHS) matrix, as either a 3-rank |
| 95 |
/// Kokkos::View or a 4-rank Kokkos::View for SIMD |
| 96 |
/// operations. |
| 97 |
/// \tparam ArgBoundsCheck Specifies whether to perform global memory access |
| 98 |
/// bounds checks within the functor. Bounds checks |
| 99 |
/// are required when matrix sizes are not evenly divisible |
| 100 |
/// by tile sizes. |
| 101 |
/// BoundsCheck::Yes The functor will perform bound checks (recommended) |
| 102 |
/// BoundsCheck::No The functor will NOT perform bound checks |
| 103 |
/// \tparam ArgAlphaFmaTag Specifies whether to apply alpha during fmas. |
| 104 |
/// AlphaFmaTag::Yes alpha will be applied during fma (C = C * alpha + AB). |
| 105 |
/// AlphaFmaTag::No alpha will be applied during mul (A * B * alpha). |
| 106 |
/// \tparam TILE_M Specifies the number of rows in each tile. |
| 107 |
/// \tparam TILE_N Specifies the number of cols in each tile. |
| 108 |
/// \tparam TILE_K Specifies the number of cols or rows in a tile of A or tile of B, respectively. |
| 109 |
/// |
| 110 |
/// See struct BatchedGemmHandle for details. |
| 111 |
/// \param alpha [in] Input coefficient used for multiplication with A |
| 112 |
/// \param A [in] Input matrix, as a 3-rank Kokkos::View |
| 113 |
/// If ArgBatchSzDim == "BatchSzDim::Right", matrix A is MxKxB |
| 114 |
/// If ArgBatchSzDim == "BatchSzDim::Left", matrix A is BxMxK |
| 115 |
/// \param B [in] Input matrix, as a 3-rank Kokkos::View |
| 116 |
/// If ArgBatchSzDim == "BatchSzDim::Right", matrix B is KxNxB |
| 117 |
/// If ArgBatchSzDim == "BatchSzDim::Left", matrix B is BxKxN |
| 118 |
/// \param beta [in] Input coefficient used for multiplication with C |
| 119 |
/// \param C [in/out] Input/Output matrix, as a 3-rank Kokkos::View |
| 120 |
/// If ArgBatchSzDim == "BatchSzDim::Right", matrix C is MxNxB |
| 121 |
/// If ArgBatchSzDim == "BatchSzDim::Left", matrix C is BxMxN |
| 122 |
/// \return 0 upon success, non-zero otherwise |
| 123 |
/// |
| 124 |
/// Usage Example: |
| 125 |
/// BatchedSerialGemm<ArgTransA, ArgTransB, ArgMode, ArgBatchSzDim, |
| 126 |
/// ScalarType, AViewType, BViewType, CViewType |
| 127 |
/// ArgBoundsCheck, tile_m, tile_n, tile_k>(alpha, A, B, beta, C).invoke(); |
| 128 |
// clang-format on |
| 129 |
template <class ArgTransA, class ArgTransB, class ArgBatchSzDim, class HandleType, class ScalarType, class AViewType, |
| 130 |
class BViewType, class CViewType, class ArgBoundsCheck, class ArgAlphaFmaTag, int TILE_M, int TILE_N, |
| 131 |
int TILE_K> |
| 132 |
class BatchedDblBufGemm { |
| 133 |
private: |
| 134 |
using AlphaMulTag = |
| 135 |
std::conditional_t<std::is_same<ArgAlphaFmaTag, AlphaTag::Yes>::value, AlphaTag::No, AlphaTag::Yes>; |
| 136 |
|
| 137 |
HandleType *const __handle; |
| 138 |
AViewType __A; |
| 139 |
BViewType __B; |
| 140 |
CViewType __C; |
| 141 |
ScalarType __alpha, __beta; |
| 142 |
ArgTransA __transA_tag; |
| 143 |
ArgTransB __transB_tag; |
| 144 |
ArgBatchSzDim __batch_layout_tag; |
| 145 |
ArgBoundsCheck __bounds_check_tag; |
| 146 |
ArgAlphaFmaTag __alpha_fma_tag; |
| 147 |
AlphaMulTag __alpha_mul_tag; |
| 148 |
int __c_batch_size, __c_m, __c_n; |
| 149 |
|
| 150 |
using view_value_type = typename CViewType::value_type; |
| 151 |
using layout_type = typename CViewType::array_layout; |
| 152 |
using device_type = typename CViewType::device_type; |
| 153 |
using execution_space_type = typename device_type::execution_space; |
| 154 |
using scratch_space_type = typename execution_space_type::scratch_memory_space; |
| 155 |
using view_type_2d_scratch = Kokkos::View<view_value_type **, Kokkos::LayoutRight, scratch_space_type>; |
| 156 |
|
| 157 |
public: |
| 158 |
BatchedDblBufGemm(HandleType *const handle, ScalarType alpha, AViewType A, BViewType B, ScalarType beta, CViewType C) |
| 159 |
: __handle(handle), __A(A), __B(B), __C(C), __alpha(alpha), __beta(beta) {} |
| 160 |
|
| 161 |
int invoke() { |
| 162 |
__run(); |
| 163 |
return 0; |
| 164 |
} |
| 165 |
|
| 166 |
private: |
| 167 |
void __run() { |
| 168 |
using policy_type = Kokkos::TeamPolicy<TagFromLayout<layout_type>, execution_space_type>; |
| 169 |
using member_type = typename policy_type::member_type; |
| 170 |
|
| 171 |
// Compile-time expressions required for functor-level register allocations: |
| 172 |
// Each team uses a shmem buffer and statically allocated register buffer. |
| 173 |
// Below, we need a 1-1 mapping between GPU threads and register |
| 174 |
// allocations to ensure that each GPU thread does not step on another |
| 175 |
// GPU threads' registers. In short, we must map register allocations |
| 176 |
// to parallel_for loop bounds. |
| 177 |
// TODO: check these expressions for all tile_m, tile_n, tile_k in Z+. |
| 178 |
constexpr int reg_m = TILE_M / TILE_K; |
| 179 |
constexpr int reg_n = TILE_N / TILE_K + 2 * !!(TILE_N % TILE_K); |
| 180 |
constexpr int stride_m = TILE_K; |
| 181 |
constexpr int stride_n = TILE_N / reg_n; |
| 182 |
using functor_type = Functor<member_type, reg_m, reg_n, stride_m, stride_n>; |
| 183 |
|
| 184 |
functor_type functor(*this, __A, __B, __C); |
| 185 |
|
| 186 |
if (__handle->enableDebug) { |
| 187 |
std::cout << "algo_type:" << __handle->get_kernel_algo_type() << std::endl |
| 188 |
<< "__c_m:" << __c_m << std::endl |
| 189 |
<< "__c_n:" << __c_n << std::endl |
| 190 |
<< "reg_m:" << reg_m << std::endl |
| 191 |
<< "reg_n:" << reg_n << std::endl |
| 192 |
<< "stride_m:" << stride_m << std::endl |
| 193 |
<< "stride_n:" << stride_n << std::endl; |
| 194 |
} |
| 195 |
|
| 196 |
// Each team solves a single tile. Within each tile, the team solves |
| 197 |
// all __n_tile_k_tiles one at a time. |
| 198 |
size_t league_size = __c_batch_size * functor.get_n_sub_tiles(); |
| 199 |
int team_size = stride_m; |
| 200 |
int vector_len = stride_n; |
| 201 |
|
| 202 |
const int max_team_size = |
| 203 |
policy_type(league_size, Kokkos::AUTO, vector_len).team_size_max(functor, Kokkos::ParallelForTag()); |
| 204 |
if (team_size > max_team_size) { |
| 205 |
std::ostringstream os; |
| 206 |
os << "KokkosBatched::BatchedGemm with kernelAlgoType = " << std::to_string(__handle->get_kernel_algo_type()) |
| 207 |
<< " does not support team_size > " << std::to_string(max_team_size) << "." << std::endl |
| 208 |
<< " The tile dimensions must be adjusted." << std::endl; |
| 209 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 210 |
} |
| 211 |
|
| 212 |
const int max_vector_len = policy_type(league_size, team_size, Kokkos::AUTO).vector_length_max(); |
| 213 |
if (vector_len > max_vector_len) { |
| 214 |
std::ostringstream os; |
| 215 |
os << "KokkosBatched::BatchedGemm with kernelAlgoType = " << std::to_string(__handle->get_kernel_algo_type()) |
| 216 |
<< " does not support vector_len > " << std::to_string(max_vector_len) << "." << std::endl |
| 217 |
<< " The tile dimensions must be adjusted." << std::endl; |
| 218 |
KokkosKernels::Impl::throw_runtime_exception(os.str()); |
| 219 |
} |
| 220 |
|
| 221 |
if (__handle->enableDebug) { |
| 222 |
std::cout << "max_team_size:" << max_team_size << " team_size:" << team_size << std::endl |
| 223 |
<< "max_vector_len:" << max_vector_len << " vector_len:" << vector_len << std::endl |
| 224 |
<< "TILE_M:" << TILE_M << std::endl |
| 225 |
<< "TILE_N:" << TILE_N << std::endl |
| 226 |
<< "TILE_K:" << TILE_K << std::endl; |
| 227 |
} |
| 228 |
|
| 229 |
// TODO: Use statically allocated shmem |
| 230 |
int shmem_size = |
| 231 |
view_type_2d_scratch::shmem_size(TILE_M, TILE_K) + view_type_2d_scratch::shmem_size(TILE_K, TILE_N); |
| 232 |
|
| 233 |
// Each member solves a portion of TILE_K in parallel with other members |
| 234 |
policy_type team_policy(league_size, team_size, vector_len); |
| 235 |
team_policy.set_scratch_size(0, Kokkos::PerTeam(shmem_size)); |
| 236 |
|
| 237 |
Kokkos::parallel_for("BatchedDblBufGemm", team_policy, functor); |
| 238 |
} |
| 239 |
|
| 240 |
public: |
| 241 |
// Make Functor public for cuda 9. |
| 242 |
// See https://github.com/kokkos/kokkos-kernels/issues/1121. |
| 243 |
template <class MemberType, int REG_M, int REG_N, int STRIDE_M, int STRIDE_N> |
| 244 |
class Functor { |
| 245 |
private: |
| 246 |
BatchedDblBufGemm &__ei; |
| 247 |
AViewType __A; |
| 248 |
BViewType __B; |
| 249 |
CViewType __C; |
| 250 |
ScalarType __alpha, __beta; |
| 251 |
int __k; |
| 252 |
size_t __n_sub_tiles, __tiles_per_col, __tiles_per_row; |
| 253 |
|
| 254 |
public: |
| 255 |
size_t get_n_sub_tiles() { return __n_sub_tiles; } |
| 256 |
|
| 257 |
// NOTE: We cannot use __ei.{__A,__B,__C,__beta,__alpha,__k} in the operator |
| 258 |
// below. If those are used, we get an invalid memory error from cuda. I |
| 259 |
// suspect this is due the values not being copied to device and then |
| 260 |
// runtime resolution of the host address &__ei. |
| 261 |
Functor(BatchedDblBufGemm &ei, AViewType A, BViewType B, CViewType C) : __ei(ei), __A(A), __B(B), __C(C) { |
| 262 |
if (std::is_same<ArgBatchSzDim, BatchLayout::Left>::value) { |
| 263 |
ei.__c_batch_size = ei.__C.extent_int(0); |
| 264 |
ei.__c_m = ei.__C.extent_int(1); |
| 265 |
ei.__c_n = ei.__C.extent_int(2); |
| 266 |
if (std::is_same<ArgTransA, Trans::Transpose>::value) |
| 267 |
__k = ei.__A.extent_int(1); |
| 268 |
else |
| 269 |
__k = ei.__A.extent_int(2); |
| 270 |
} else { |
| 271 |
ei.__c_batch_size = ei.__C.extent_int(2); |
| 272 |
ei.__c_m = ei.__C.extent_int(0); |
| 273 |
ei.__c_n = ei.__C.extent_int(1); |
| 274 |
if (std::is_same<ArgTransA, Trans::Transpose>::value) |
| 275 |
__k = ei.__A.extent_int(0); |
| 276 |
else |
| 277 |
__k = ei.__A.extent_int(1); |
| 278 |
} |
| 279 |
__beta = ei.__beta; // Copy to device |
| 280 |
__alpha = ei.__alpha; // Copy to device |
| 281 |
// To handle truncation of tiles per row/col, round up to one extra tile |
| 282 |
// with '!!'. This extra tile will hang off the edge of the 2-rank matrix. |
| 283 |
// For cases where tiles hang off the edge, we over-compute 0s within |
| 284 |
// registers via a conditional bounds check selected at compile-time. |
| 285 |
__tiles_per_row = ei.__c_m / TILE_M + !!((size_t)ei.__c_m % TILE_M); |
| 286 |
__tiles_per_col = ei.__c_n / TILE_N + !!((size_t)ei.__c_n % TILE_N); |
| 287 |
|
| 288 |
__n_sub_tiles = __tiles_per_row * __tiles_per_col; |
| 289 |
} |
| 290 |
|
| 291 |
KOKKOS_INLINE_FUNCTION |
| 292 |
void __mul(view_value_type a, view_value_type b, view_value_type &c, const AlphaTag::No &) const { c += a * b; } |
| 293 |
|
| 294 |
KOKKOS_INLINE_FUNCTION |
| 295 |
void __mul(view_value_type a, view_value_type b, view_value_type &c, const AlphaTag::Yes &) const { |
| 296 |
c += a * b * __alpha; |
| 297 |
} |
| 298 |
|
| 299 |
KOKKOS_INLINE_FUNCTION |
| 300 |
void __rshmem_and_mul(const int &thread_id, const int &vlane_id, const unsigned &nk, view_value_type reg_a[REG_M], |
| 301 |
view_value_type reg_b[REG_N], view_value_type reg_c[REG_M][REG_N], |
| 302 |
view_type_2d_scratch &svA_scr, view_type_2d_scratch &svB_scr) const { |
| 303 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 304 |
#pragma unroll |
| 305 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 306 |
for (unsigned k = 0; k < nk; ++k) { |
| 307 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 308 |
#pragma unroll |
| 309 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 310 |
for (int m = 0; m < REG_M; ++m) reg_a[m] = svA_scr(thread_id + m * STRIDE_M, k); |
| 311 |
|
| 312 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 313 |
#pragma unroll |
| 314 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 315 |
for (int n = 0; n < REG_N; ++n) reg_b[n] = svB_scr(k, vlane_id + n * STRIDE_N); |
| 316 |
|
| 317 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 318 |
#pragma unroll |
| 319 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 320 |
for (int m = 0; m < REG_M; ++m) { |
| 321 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 322 |
#pragma unroll |
| 323 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 324 |
for (int n = 0; n < REG_N; ++n) __mul(reg_a[m], reg_b[n], reg_c[m][n], __ei.__alpha_mul_tag); |
| 325 |
} |
| 326 |
} |
| 327 |
} |
| 328 |
|
| 329 |
KOKKOS_INLINE_FUNCTION |
| 330 |
void __rshmem_and_mul_ll(const int &thread_id, const int &vlane_id, const unsigned &nk, |
| 331 |
view_value_type reg_a[REG_M], view_value_type reg_b[REG_N], |
| 332 |
view_value_type reg_c[REG_M][REG_N], view_type_2d_scratch &svA_scr, |
| 333 |
view_type_2d_scratch &svB_scr) const { |
| 334 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 335 |
#pragma unroll |
| 336 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 337 |
for (unsigned k = 0; k < nk; ++k) { |
| 338 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 339 |
#pragma unroll |
| 340 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 341 |
for (int m = 0; m < REG_M; ++m) reg_a[m] = svA_scr(k, vlane_id + m * STRIDE_M); |
| 342 |
|
| 343 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 344 |
#pragma unroll |
| 345 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 346 |
for (int n = 0; n < REG_N; ++n) reg_b[n] = svB_scr(thread_id + n * STRIDE_N, k); |
| 347 |
|
| 348 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 349 |
#pragma unroll |
| 350 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 351 |
for (int m = 0; m < REG_M; ++m) { |
| 352 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 353 |
#pragma unroll |
| 354 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 355 |
for (int n = 0; n < REG_N; ++n) __mul(reg_a[m], reg_b[n], reg_c[m][n], __ei.__alpha_mul_tag); |
| 356 |
} |
| 357 |
} |
| 358 |
} |
| 359 |
|
| 360 |
KOKKOS_INLINE_FUNCTION |
| 361 |
void operator()(LayoutRightTag, const MemberType &member) const { |
| 362 |
// TODO: use Kokkos view with compile-time size to allocating register?? |
| 363 |
// Then we can use local deep copy for prefetch_reg population. |
| 364 |
// Allocate registers used for prefetching |
| 365 |
view_value_type prefetch_reg_a[REG_M] = {0}, prefetch_reg_b[REG_N] = {0}; |
| 366 |
|
| 367 |
// Allocate registers used for FMAs |
| 368 |
view_value_type reg_a[REG_M] = {0}, reg_b[REG_N] = {0}, reg_c[REG_M][REG_N] = {{0}}; |
| 369 |
// TODO: look at local loads and stores via nvprof |
| 370 |
// TODO: look at GPU trace in nvprof to find out how many registers are |
| 371 |
// used. |
| 372 |
|
| 373 |
unsigned batch_idx = member.league_rank() / __n_sub_tiles; |
| 374 |
|
| 375 |
// Compute starting tile offsets for each team into svA, svB, svC |
| 376 |
unsigned local_team_idx = member.league_rank() % __n_sub_tiles; |
| 377 |
unsigned start_m = (local_team_idx / __tiles_per_col) * TILE_M; |
| 378 |
unsigned start_n = (local_team_idx % __tiles_per_col) * TILE_N; |
| 379 |
|
| 380 |
int kk; |
| 381 |
|
| 382 |
// Fetch entire 2-rank sub-matrix |
| 383 |
auto svA = |
| 384 |
subview_wrapper(__A, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag, __ei.__transA_tag); |
| 385 |
auto svB = |
| 386 |
subview_wrapper(__B, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag, __ei.__transB_tag); |
| 387 |
auto svC = subview_wrapper(__C, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag); |
| 388 |
|
| 389 |
// Allocate scratch memory buffers used for prefetching |
| 390 |
view_type_2d_scratch svA_scr(member.team_scratch(0), TILE_M, TILE_K); |
| 391 |
view_type_2d_scratch svB_scr(member.team_scratch(0), TILE_K, TILE_N); |
| 392 |
|
| 393 |
Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, STRIDE_M), [&](const int &thread_id) { |
| 394 |
int m_offset = thread_id + start_m; |
| 395 |
|
| 396 |
Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, 0, STRIDE_N), [&](const int &vlane_id) { |
| 397 |
int n_offset = vlane_id + start_n; |
| 398 |
|
| 399 |
// Here we populate scratch memory with one or more "k" tiles for |
| 400 |
// every thread of the team! |
| 401 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 402 |
#pragma unroll |
| 403 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 404 |
for (int i = 0; i < REG_N * STRIDE_N; i += STRIDE_N) |
| 405 |
svB_scr(thread_id, vlane_id + i) = |
| 406 |
access_view_bounds_check<view_value_type>(svB, thread_id, n_offset + i, __ei.__bounds_check_tag); |
| 407 |
|
| 408 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 409 |
#pragma unroll |
| 410 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 411 |
for (int i = 0; i < REG_M * STRIDE_M; i += STRIDE_M) |
| 412 |
svA_scr(thread_id + i, vlane_id) = |
| 413 |
access_view_bounds_check<view_value_type>(svA, m_offset + i, vlane_id, __ei.__bounds_check_tag); |
| 414 |
|
| 415 |
// Wait for A, B to reside in scratch memory |
| 416 |
member.team_barrier(); |
| 417 |
|
| 418 |
// Each thread calculates a single dot product in chunks of |
| 419 |
// size TILE_K |
| 420 |
for (kk = 0; kk < __k - TILE_K; kk += TILE_K) { |
| 421 |
int k_tile_offset = kk + TILE_K; |
| 422 |
|
| 423 |
// Get this threads next TILE_K entries from global memory |
| 424 |
// Each thread has its own copy of prefetch_reg_b. |
| 425 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 426 |
#pragma unroll |
| 427 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 428 |
for (int i = 0; i < REG_N; ++i) |
| 429 |
prefetch_reg_b[i] = access_view_bounds_check<view_value_type>( |
| 430 |
svB, thread_id + k_tile_offset, n_offset + i * STRIDE_N, __ei.__bounds_check_tag); |
| 431 |
|
| 432 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 433 |
#pragma unroll |
| 434 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 435 |
for (int i = 0; i < REG_M; ++i) |
| 436 |
prefetch_reg_a[i] = access_view_bounds_check<view_value_type>( |
| 437 |
svA, m_offset + i * STRIDE_M, vlane_id + k_tile_offset, __ei.__bounds_check_tag); |
| 438 |
|
| 439 |
__rshmem_and_mul(thread_id, vlane_id, TILE_K, reg_a, reg_b, reg_c, svA_scr, svB_scr); |
| 440 |
|
| 441 |
// Wait for: |
| 442 |
// 1. prefetch_regs to be populated |
| 443 |
// 2. for shmem to no longer be read from |
| 444 |
member.team_barrier(); |
| 445 |
|
| 446 |
// populate shmem from prefetch registers. Each thread has its own |
| 447 |
// copy of prefetch_reg_b. |
| 448 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 449 |
#pragma unroll |
| 450 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 451 |
for (int i = 0; i < REG_N; ++i) svB_scr(thread_id, vlane_id + i * STRIDE_N) = prefetch_reg_b[i]; |
| 452 |
|
| 453 |
// populate shmem from prefetch registers. Each thread has its own |
| 454 |
// copy of prefetch_reg_a. |
| 455 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 456 |
#pragma unroll |
| 457 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 458 |
for (int i = 0; i < REG_M; ++i) svA_scr(thread_id + i * STRIDE_M, vlane_id) = prefetch_reg_a[i]; |
| 459 |
|
| 460 |
// Wait for shmem stores to land before performing next |
| 461 |
// TILE_K multiply |
| 462 |
member.team_barrier(); |
| 463 |
} // end n_tile_k_tiles loop |
| 464 |
|
| 465 |
// Multiply last tile, may be a partial tile |
| 466 |
__rshmem_and_mul(thread_id, vlane_id, __k - kk, reg_a, reg_b, reg_c, svA_scr, svB_scr); |
| 467 |
|
| 468 |
// store results back to global memory |
| 469 |
if (__beta == 0.0F) { |
| 470 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 471 |
#pragma unroll |
| 472 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 473 |
for (int m = 0; m < REG_M; ++m) { |
| 474 |
int cm = m_offset + m * STRIDE_M; |
| 475 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 476 |
#pragma unroll |
| 477 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 478 |
for (int n = 0; n < REG_N; ++n) { |
| 479 |
int cn = n_offset + n * STRIDE_N; |
| 480 |
fma_bounds_check(svC, cm, cn, reg_c[m][n], __alpha, __ei.__alpha_fma_tag, __ei.__bounds_check_tag); |
| 481 |
} |
| 482 |
} |
| 483 |
} else { |
| 484 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 485 |
#pragma unroll |
| 486 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 487 |
for (int m = 0; m < REG_M; ++m) { |
| 488 |
int cm = m_offset + m * STRIDE_M; |
| 489 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 490 |
#pragma unroll |
| 491 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 492 |
for (int n = 0; n < REG_N; ++n) { |
| 493 |
int cn = n_offset + n * STRIDE_N; |
| 494 |
fma_bounds_check(svC, cm, cn, reg_c[m][n], __alpha, __beta, __ei.__alpha_fma_tag, |
| 495 |
__ei.__bounds_check_tag); |
| 496 |
} |
| 497 |
} |
| 498 |
} |
| 499 |
}); |
| 500 |
}); |
| 501 |
} |
| 502 |
|
| 503 |
KOKKOS_INLINE_FUNCTION |
| 504 |
void operator()(LayoutLeftTag, const MemberType &member) const { |
| 505 |
// TODO: use Kokkos view with compile-time size to allocating register?? |
| 506 |
// Then we can use local deep copy for prefetch_reg population. |
| 507 |
// Allocate registers used for prefetching |
| 508 |
view_value_type prefetch_reg_a[REG_M] = {0}, prefetch_reg_b[REG_N] = {0}; |
| 509 |
|
| 510 |
// Allocate registers used for FMAs |
| 511 |
view_value_type reg_a[REG_M] = {0}, reg_b[REG_N] = {0}, reg_c[REG_M][REG_N] = {{0}}; |
| 512 |
// TODO: look at local loads and stores via nvprof |
| 513 |
// TODO: look at GPU trace in nvprof to find out how many registers are |
| 514 |
// used. |
| 515 |
|
| 516 |
unsigned batch_idx = member.league_rank() / __n_sub_tiles; |
| 517 |
|
| 518 |
// Compute starting tile offsets for each team into svA, svB, svC |
| 519 |
unsigned local_team_idx = member.league_rank() % __n_sub_tiles; |
| 520 |
unsigned start_m = (local_team_idx % __tiles_per_row) * TILE_M; |
| 521 |
unsigned start_n = (local_team_idx / __tiles_per_row) * TILE_N; |
| 522 |
|
| 523 |
int kk; |
| 524 |
|
| 525 |
// Fetch entire 2-rank sub-matrix |
| 526 |
auto svA = |
| 527 |
subview_wrapper(__A, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag, __ei.__transA_tag); |
| 528 |
auto svB = |
| 529 |
subview_wrapper(__B, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag, __ei.__transB_tag); |
| 530 |
auto svC = subview_wrapper(__C, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag); |
| 531 |
|
| 532 |
// Allocate scratch memory buffers used for prefetching |
| 533 |
view_type_2d_scratch svA_scr(member.team_scratch(0), TILE_K, TILE_M); |
| 534 |
view_type_2d_scratch svB_scr(member.team_scratch(0), TILE_N, TILE_K); |
| 535 |
|
| 536 |
Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, STRIDE_N), [&](const int &thread_id) { |
| 537 |
int n_offset = thread_id + start_n; |
| 538 |
|
| 539 |
Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, 0, STRIDE_M), [&](const int &vlane_id) { |
| 540 |
int m_offset = vlane_id + start_m; |
| 541 |
|
| 542 |
// Here we populate scratch memory with one or more "k" tiles for |
| 543 |
// every thread of the team! |
| 544 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 545 |
#pragma unroll |
| 546 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 547 |
for (int i = 0; i < REG_N * STRIDE_N; i += STRIDE_N) |
| 548 |
svB_scr(thread_id + i, vlane_id) = |
| 549 |
access_view_bounds_check<view_value_type>(svB, vlane_id, n_offset + i, __ei.__bounds_check_tag); |
| 550 |
|
| 551 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 552 |
#pragma unroll |
| 553 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 554 |
for (int i = 0; i < REG_M * STRIDE_M; i += STRIDE_M) |
| 555 |
svA_scr(thread_id, vlane_id + i) = |
| 556 |
access_view_bounds_check<view_value_type>(svA, m_offset + i, thread_id, __ei.__bounds_check_tag); |
| 557 |
|
| 558 |
// Wait for A, B to reside in scratch memory |
| 559 |
member.team_barrier(); |
| 560 |
|
| 561 |
// Each thread calculates a single dot product in chunks of |
| 562 |
// size TILE_K |
| 563 |
for (kk = 0; kk < __k - TILE_K; kk += TILE_K) { |
| 564 |
int k_tile_offset = kk + TILE_K; |
| 565 |
|
| 566 |
// Get this threads next TILE_K entries from global memory |
| 567 |
// Each thread has its own copy of prefetch_reg_b. |
| 568 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 569 |
#pragma unroll |
| 570 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 571 |
for (int i = 0; i < REG_N; ++i) |
| 572 |
prefetch_reg_b[i] = access_view_bounds_check<view_value_type>( |
| 573 |
svB, vlane_id + k_tile_offset, n_offset + i * STRIDE_N, __ei.__bounds_check_tag); |
| 574 |
|
| 575 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 576 |
#pragma unroll |
| 577 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 578 |
for (int i = 0; i < REG_M; ++i) |
| 579 |
prefetch_reg_a[i] = access_view_bounds_check<view_value_type>( |
| 580 |
svA, m_offset + i * STRIDE_M, thread_id + k_tile_offset, __ei.__bounds_check_tag); |
| 581 |
|
| 582 |
__rshmem_and_mul_ll(thread_id, vlane_id, TILE_K, reg_a, reg_b, reg_c, svA_scr, svB_scr); |
| 583 |
|
| 584 |
// Wait for: |
| 585 |
// 1. prefetch_regs to be populated |
| 586 |
// 2. for shmem to no longer be read from |
| 587 |
member.team_barrier(); |
| 588 |
|
| 589 |
// populate shmem from prefetch registers. Each thread has its own |
| 590 |
// copy of prefetch_reg_b. |
| 591 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 592 |
#pragma unroll |
| 593 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 594 |
for (int i = 0; i < REG_N; ++i) svB_scr(thread_id + i * STRIDE_N, vlane_id) = prefetch_reg_b[i]; |
| 595 |
|
| 596 |
// populate shmem from prefetch registers. Each thread has its own |
| 597 |
// copy of prefetch_reg_a. |
| 598 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 599 |
#pragma unroll |
| 600 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 601 |
for (int i = 0; i < REG_M; ++i) svA_scr(thread_id, vlane_id + i * STRIDE_M) = prefetch_reg_a[i]; |
| 602 |
|
| 603 |
// Wait for shmem stores to land before performing next |
| 604 |
// TILE_K multiply |
| 605 |
member.team_barrier(); |
| 606 |
} // end n_tile_k_tiles loop |
| 607 |
|
| 608 |
// Multiply last tile, may be a partial tile |
| 609 |
__rshmem_and_mul_ll(thread_id, vlane_id, __k - kk, reg_a, reg_b, reg_c, svA_scr, svB_scr); |
| 610 |
|
| 611 |
// store results back to global memory |
| 612 |
if (__beta == 0.0F) { |
| 613 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 614 |
#pragma unroll |
| 615 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 616 |
for (int n = 0; n < REG_N; ++n) { |
| 617 |
int cn = n_offset + n * STRIDE_N; |
| 618 |
|
| 619 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 620 |
#pragma unroll |
| 621 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 622 |
for (int m = 0; m < REG_M; ++m) { |
| 623 |
int cm = m_offset + m * STRIDE_M; |
| 624 |
fma_bounds_check(svC, cm, cn, reg_c[m][n], __alpha, __ei.__alpha_fma_tag, __ei.__bounds_check_tag); |
| 625 |
} |
| 626 |
} |
| 627 |
} else { |
| 628 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 629 |
#pragma unroll |
| 630 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 631 |
for (int n = 0; n < REG_N; ++n) { |
| 632 |
int cn = n_offset + n * STRIDE_N; |
| 633 |
|
| 634 |
#if defined(KOKKOS_ENABLE_PRAGMA_UNROLL) |
| 635 |
#pragma unroll |
| 636 |
#endif // KOKKOS_ENABLE_PRAGMA_UNROLL |
| 637 |
for (int m = 0; m < REG_M; ++m) { |
| 638 |
int cm = m_offset + m * STRIDE_M; |
| 639 |
fma_bounds_check(svC, cm, cn, reg_c[m][n], __alpha, __beta, __ei.__alpha_fma_tag, |
| 640 |
__ei.__bounds_check_tag); |
| 641 |
} |
| 642 |
} |
| 643 |
} |
| 644 |
}); |
| 645 |
}); |
| 646 |
} |
| 647 |
}; |
| 648 |
}; |
| 649 |
/********************* END non-functor-level routines *********************/ |
| 650 |
} // namespace Impl |
| 651 |
|
| 652 |
} // namespace KokkosBatched |
| 653 |
|
| 654 |
#endif |