Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_HostLevel_Gemm_DblBuf_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_HOSTLEVEL_GEMM_DBLBUF_IMPL_HPP
17 #define KOKKOSBATCHED_HOSTLEVEL_GEMM_DBLBUF_IMPL_HPP
18
19 #include "KokkosBatched_Util.hpp"
20 #include "KokkosKernels_Error.hpp"
21
22 namespace KokkosBatched {
23 /********************* BEGIN functor-level routines *********************/
24 /********************* END functor-level routines *********************/
25
26 namespace Impl {
27 /********************* BEGIN non-functor-level routines *********************/
28 ///
29 /// Implemented:
30 /// NT/NT, T/NT, NT/T, T/T
31 ///
32 /// Not yet immplemented (ConjTranspose):
33 /// CT/NT, NT/CT, CT/CT
34 ///
35
36 struct LayoutLeftTag {};
37 struct LayoutRightTag {};
38 template <class>
39 struct TagFromLayoutHelper;
40 template <>
41 struct TagFromLayoutHelper<Kokkos::LayoutLeft> {
42 using tag = LayoutLeftTag;
43 };
44 template <>
45 struct TagFromLayoutHelper<Kokkos::LayoutRight> {
46 using tag = LayoutRightTag;
47 };
48 template <class Layout>
49 using TagFromLayout = typename TagFromLayoutHelper<Layout>::tag;
50
51 // TODO - scaling between (32x32, 64x64)
52 // Option 0: Increase number of tiles and figure out how to map kokkos teams
53 // into cuda grid. Keep team size and vector lanes constant.
54 // TODO: write up small example and ask Christian. [DONE,
55 // MdRangePolicy not applicable here]
56 // Option 1: Increase register sizes to handle rows/cols past tile size
57 // Option 2: Fix league_size and have single team solve full tile followed
58 // by same team solving extra rows/cols (without multiplying by the
59 // zero rows/cols)
60
61 // clang-format off
62 /// \brief Non-blocking general matrix multiply on a batch of
63 /// uniform matrices with an algorithm based on:
64 /// B. P. D. J. Kunkel, Julian, “Performance, design, and autotuning of batched gemm for GPUs,”
65 /// in Lecture Notes in Computer Science, ser. ISC High Performance Computing ’16, vol. 9697, 06 2016.
66 ///
67 ///
68 /// C = alpha * op(A) * op(B) + beta * C
69 ///
70 /// \tparam ArgTransA Specifies what op does to A:
71 /// Trans::NoTranspose for non-transpose
72 /// Trans::Transpose for transpose
73 /// Trans::ConjTranspose for conjugate transpose (unsupported)
74 /// \tparam ArgTransB Specifies what op does to B:
75 /// Trans::NoTranspose for non-transpose
76 /// Trans::Transpose for transpose
77 /// Trans::ConjTranspose for conjugate transpose (unsupported)
78 /// \tparam ArgBatchSzDim Specifies where the batch dimension is allocated in
79 /// AViewType, BViewType, and CViewType:
80 /// BatchSzDim::Left Batch dimension is leftmost
81 /// BatchSzDim::Right Batch dimension is rightmost
82 /// \tparam ArgResultsPerThread Specifies how to divide work among threads. For
83 /// this serial interface, each rank specifies how
84 /// much work to assign a single thread.
85 /// ResultsPerThread::Rank0 Each thread computes a scalar of C
86 /// ResultsPerThread::Rank1 Each thread computes a 1-rank chunk of C
87 /// ResultsPerThread::Rank2 Each thread computes a 2-rank chunk of C
88 /// \tparam HandleType Specifies the handle type of the kernel handle
89 /// \tparam ScalarType Specifies the scalar type of alpha and beta
90 /// \tparam AViewType Input matrix, as either a 3-rank Kokkos::View or a
91 /// 4-rank Kokkos::View for SIMD operations.
92 /// \tparam BViewType Input matrix, as either a 3-rank Kokkos::View or a
93 /// 4-rank Kokkos::View for SIMD operations.
94 /// \tparam CViewType Input(RHS)/Output(LHS) matrix, as either a 3-rank
95 /// Kokkos::View or a 4-rank Kokkos::View for SIMD
96 /// operations.
97 /// \tparam ArgBoundsCheck Specifies whether to perform global memory access
98 /// bounds checks within the functor. Bounds checks
99 /// are required when matrix sizes are not evenly divisible
100 /// by tile sizes.
101 /// BoundsCheck::Yes The functor will perform bound checks (recommended)
102 /// BoundsCheck::No The functor will NOT perform bound checks
103 /// \tparam ArgAlphaFmaTag Specifies whether to apply alpha during fmas.
104 /// AlphaFmaTag::Yes alpha will be applied during fma (C = C * alpha + AB).
105 /// AlphaFmaTag::No alpha will be applied during mul (A * B * alpha).
106 /// \tparam TILE_M Specifies the number of rows in each tile.
107 /// \tparam TILE_N Specifies the number of cols in each tile.
108 /// \tparam TILE_K Specifies the number of cols or rows in a tile of A or tile of B, respectively.
109 ///
110 /// See struct BatchedGemmHandle for details.
111 /// \param alpha [in] Input coefficient used for multiplication with A
112 /// \param A [in] Input matrix, as a 3-rank Kokkos::View
113 /// If ArgBatchSzDim == "BatchSzDim::Right", matrix A is MxKxB
114 /// If ArgBatchSzDim == "BatchSzDim::Left", matrix A is BxMxK
115 /// \param B [in] Input matrix, as a 3-rank Kokkos::View
116 /// If ArgBatchSzDim == "BatchSzDim::Right", matrix B is KxNxB
117 /// If ArgBatchSzDim == "BatchSzDim::Left", matrix B is BxKxN
118 /// \param beta [in] Input coefficient used for multiplication with C
119 /// \param C [in/out] Input/Output matrix, as a 3-rank Kokkos::View
120 /// If ArgBatchSzDim == "BatchSzDim::Right", matrix C is MxNxB
121 /// If ArgBatchSzDim == "BatchSzDim::Left", matrix C is BxMxN
122 /// \return 0 upon success, non-zero otherwise
123 ///
124 /// Usage Example:
125 /// BatchedSerialGemm<ArgTransA, ArgTransB, ArgMode, ArgBatchSzDim,
126 /// ScalarType, AViewType, BViewType, CViewType
127 /// ArgBoundsCheck, tile_m, tile_n, tile_k>(alpha, A, B, beta, C).invoke();
128 // clang-format on
129 template <class ArgTransA, class ArgTransB, class ArgBatchSzDim, class HandleType, class ScalarType, class AViewType,
130 class BViewType, class CViewType, class ArgBoundsCheck, class ArgAlphaFmaTag, int TILE_M, int TILE_N,
131 int TILE_K>
132 class BatchedDblBufGemm {
133 private:
134 using AlphaMulTag =
135 std::conditional_t<std::is_same<ArgAlphaFmaTag, AlphaTag::Yes>::value, AlphaTag::No, AlphaTag::Yes>;
136
137 HandleType *const __handle;
138 AViewType __A;
139 BViewType __B;
140 CViewType __C;
141 ScalarType __alpha, __beta;
142 ArgTransA __transA_tag;
143 ArgTransB __transB_tag;
144 ArgBatchSzDim __batch_layout_tag;
145 ArgBoundsCheck __bounds_check_tag;
146 ArgAlphaFmaTag __alpha_fma_tag;
147 AlphaMulTag __alpha_mul_tag;
148 int __c_batch_size, __c_m, __c_n;
149
150 using view_value_type = typename CViewType::value_type;
151 using layout_type = typename CViewType::array_layout;
152 using device_type = typename CViewType::device_type;
153 using execution_space_type = typename device_type::execution_space;
154 using scratch_space_type = typename execution_space_type::scratch_memory_space;
155 using view_type_2d_scratch = Kokkos::View<view_value_type **, Kokkos::LayoutRight, scratch_space_type>;
156
157 public:
158 BatchedDblBufGemm(HandleType *const handle, ScalarType alpha, AViewType A, BViewType B, ScalarType beta, CViewType C)
159 : __handle(handle), __A(A), __B(B), __C(C), __alpha(alpha), __beta(beta) {}
160
161 int invoke() {
162 __run();
163 return 0;
164 }
165
166 private:
167 void __run() {
168 using policy_type = Kokkos::TeamPolicy<TagFromLayout<layout_type>, execution_space_type>;
169 using member_type = typename policy_type::member_type;
170
171 // Compile-time expressions required for functor-level register allocations:
172 // Each team uses a shmem buffer and statically allocated register buffer.
173 // Below, we need a 1-1 mapping between GPU threads and register
174 // allocations to ensure that each GPU thread does not step on another
175 // GPU threads' registers. In short, we must map register allocations
176 // to parallel_for loop bounds.
177 // TODO: check these expressions for all tile_m, tile_n, tile_k in Z+.
178 constexpr int reg_m = TILE_M / TILE_K;
179 constexpr int reg_n = TILE_N / TILE_K + 2 * !!(TILE_N % TILE_K);
180 constexpr int stride_m = TILE_K;
181 constexpr int stride_n = TILE_N / reg_n;
182 using functor_type = Functor<member_type, reg_m, reg_n, stride_m, stride_n>;
183
184 functor_type functor(*this, __A, __B, __C);
185
186 if (__handle->enableDebug) {
187 std::cout << "algo_type:" << __handle->get_kernel_algo_type() << std::endl
188 << "__c_m:" << __c_m << std::endl
189 << "__c_n:" << __c_n << std::endl
190 << "reg_m:" << reg_m << std::endl
191 << "reg_n:" << reg_n << std::endl
192 << "stride_m:" << stride_m << std::endl
193 << "stride_n:" << stride_n << std::endl;
194 }
195
196 // Each team solves a single tile. Within each tile, the team solves
197 // all __n_tile_k_tiles one at a time.
198 size_t league_size = __c_batch_size * functor.get_n_sub_tiles();
199 int team_size = stride_m;
200 int vector_len = stride_n;
201
202 const int max_team_size =
203 policy_type(league_size, Kokkos::AUTO, vector_len).team_size_max(functor, Kokkos::ParallelForTag());
204 if (team_size > max_team_size) {
205 std::ostringstream os;
206 os << "KokkosBatched::BatchedGemm with kernelAlgoType = " << std::to_string(__handle->get_kernel_algo_type())
207 << " does not support team_size > " << std::to_string(max_team_size) << "." << std::endl
208 << " The tile dimensions must be adjusted." << std::endl;
209 KokkosKernels::Impl::throw_runtime_exception(os.str());
210 }
211
212 const int max_vector_len = policy_type(league_size, team_size, Kokkos::AUTO).vector_length_max();
213 if (vector_len > max_vector_len) {
214 std::ostringstream os;
215 os << "KokkosBatched::BatchedGemm with kernelAlgoType = " << std::to_string(__handle->get_kernel_algo_type())
216 << " does not support vector_len > " << std::to_string(max_vector_len) << "." << std::endl
217 << " The tile dimensions must be adjusted." << std::endl;
218 KokkosKernels::Impl::throw_runtime_exception(os.str());
219 }
220
221 if (__handle->enableDebug) {
222 std::cout << "max_team_size:" << max_team_size << " team_size:" << team_size << std::endl
223 << "max_vector_len:" << max_vector_len << " vector_len:" << vector_len << std::endl
224 << "TILE_M:" << TILE_M << std::endl
225 << "TILE_N:" << TILE_N << std::endl
226 << "TILE_K:" << TILE_K << std::endl;
227 }
228
229 // TODO: Use statically allocated shmem
230 int shmem_size =
231 view_type_2d_scratch::shmem_size(TILE_M, TILE_K) + view_type_2d_scratch::shmem_size(TILE_K, TILE_N);
232
233 // Each member solves a portion of TILE_K in parallel with other members
234 policy_type team_policy(league_size, team_size, vector_len);
235 team_policy.set_scratch_size(0, Kokkos::PerTeam(shmem_size));
236
237 Kokkos::parallel_for("BatchedDblBufGemm", team_policy, functor);
238 }
239
240 public:
241 // Make Functor public for cuda 9.
242 // See https://github.com/kokkos/kokkos-kernels/issues/1121.
243 template <class MemberType, int REG_M, int REG_N, int STRIDE_M, int STRIDE_N>
244 class Functor {
245 private:
246 BatchedDblBufGemm &__ei;
247 AViewType __A;
248 BViewType __B;
249 CViewType __C;
250 ScalarType __alpha, __beta;
251 int __k;
252 size_t __n_sub_tiles, __tiles_per_col, __tiles_per_row;
253
254 public:
255 size_t get_n_sub_tiles() { return __n_sub_tiles; }
256
257 // NOTE: We cannot use __ei.{__A,__B,__C,__beta,__alpha,__k} in the operator
258 // below. If those are used, we get an invalid memory error from cuda. I
259 // suspect this is due the values not being copied to device and then
260 // runtime resolution of the host address &__ei.
261 Functor(BatchedDblBufGemm &ei, AViewType A, BViewType B, CViewType C) : __ei(ei), __A(A), __B(B), __C(C) {
262 if (std::is_same<ArgBatchSzDim, BatchLayout::Left>::value) {
263 ei.__c_batch_size = ei.__C.extent_int(0);
264 ei.__c_m = ei.__C.extent_int(1);
265 ei.__c_n = ei.__C.extent_int(2);
266 if (std::is_same<ArgTransA, Trans::Transpose>::value)
267 __k = ei.__A.extent_int(1);
268 else
269 __k = ei.__A.extent_int(2);
270 } else {
271 ei.__c_batch_size = ei.__C.extent_int(2);
272 ei.__c_m = ei.__C.extent_int(0);
273 ei.__c_n = ei.__C.extent_int(1);
274 if (std::is_same<ArgTransA, Trans::Transpose>::value)
275 __k = ei.__A.extent_int(0);
276 else
277 __k = ei.__A.extent_int(1);
278 }
279 __beta = ei.__beta; // Copy to device
280 __alpha = ei.__alpha; // Copy to device
281 // To handle truncation of tiles per row/col, round up to one extra tile
282 // with '!!'. This extra tile will hang off the edge of the 2-rank matrix.
283 // For cases where tiles hang off the edge, we over-compute 0s within
284 // registers via a conditional bounds check selected at compile-time.
285 __tiles_per_row = ei.__c_m / TILE_M + !!((size_t)ei.__c_m % TILE_M);
286 __tiles_per_col = ei.__c_n / TILE_N + !!((size_t)ei.__c_n % TILE_N);
287
288 __n_sub_tiles = __tiles_per_row * __tiles_per_col;
289 }
290
291 KOKKOS_INLINE_FUNCTION
292 void __mul(view_value_type a, view_value_type b, view_value_type &c, const AlphaTag::No &) const { c += a * b; }
293
294 KOKKOS_INLINE_FUNCTION
295 void __mul(view_value_type a, view_value_type b, view_value_type &c, const AlphaTag::Yes &) const {
296 c += a * b * __alpha;
297 }
298
299 KOKKOS_INLINE_FUNCTION
300 void __rshmem_and_mul(const int &thread_id, const int &vlane_id, const unsigned &nk, view_value_type reg_a[REG_M],
301 view_value_type reg_b[REG_N], view_value_type reg_c[REG_M][REG_N],
302 view_type_2d_scratch &svA_scr, view_type_2d_scratch &svB_scr) const {
303 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
304 #pragma unroll
305 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
306 for (unsigned k = 0; k < nk; ++k) {
307 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
308 #pragma unroll
309 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
310 for (int m = 0; m < REG_M; ++m) reg_a[m] = svA_scr(thread_id + m * STRIDE_M, k);
311
312 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
313 #pragma unroll
314 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
315 for (int n = 0; n < REG_N; ++n) reg_b[n] = svB_scr(k, vlane_id + n * STRIDE_N);
316
317 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
318 #pragma unroll
319 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
320 for (int m = 0; m < REG_M; ++m) {
321 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
322 #pragma unroll
323 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
324 for (int n = 0; n < REG_N; ++n) __mul(reg_a[m], reg_b[n], reg_c[m][n], __ei.__alpha_mul_tag);
325 }
326 }
327 }
328
329 KOKKOS_INLINE_FUNCTION
330 void __rshmem_and_mul_ll(const int &thread_id, const int &vlane_id, const unsigned &nk,
331 view_value_type reg_a[REG_M], view_value_type reg_b[REG_N],
332 view_value_type reg_c[REG_M][REG_N], view_type_2d_scratch &svA_scr,
333 view_type_2d_scratch &svB_scr) const {
334 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
335 #pragma unroll
336 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
337 for (unsigned k = 0; k < nk; ++k) {
338 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
339 #pragma unroll
340 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
341 for (int m = 0; m < REG_M; ++m) reg_a[m] = svA_scr(k, vlane_id + m * STRIDE_M);
342
343 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
344 #pragma unroll
345 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
346 for (int n = 0; n < REG_N; ++n) reg_b[n] = svB_scr(thread_id + n * STRIDE_N, k);
347
348 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
349 #pragma unroll
350 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
351 for (int m = 0; m < REG_M; ++m) {
352 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
353 #pragma unroll
354 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
355 for (int n = 0; n < REG_N; ++n) __mul(reg_a[m], reg_b[n], reg_c[m][n], __ei.__alpha_mul_tag);
356 }
357 }
358 }
359
360 KOKKOS_INLINE_FUNCTION
361 void operator()(LayoutRightTag, const MemberType &member) const {
362 // TODO: use Kokkos view with compile-time size to allocating register??
363 // Then we can use local deep copy for prefetch_reg population.
364 // Allocate registers used for prefetching
365 view_value_type prefetch_reg_a[REG_M] = {0}, prefetch_reg_b[REG_N] = {0};
366
367 // Allocate registers used for FMAs
368 view_value_type reg_a[REG_M] = {0}, reg_b[REG_N] = {0}, reg_c[REG_M][REG_N] = {{0}};
369 // TODO: look at local loads and stores via nvprof
370 // TODO: look at GPU trace in nvprof to find out how many registers are
371 // used.
372
373 unsigned batch_idx = member.league_rank() / __n_sub_tiles;
374
375 // Compute starting tile offsets for each team into svA, svB, svC
376 unsigned local_team_idx = member.league_rank() % __n_sub_tiles;
377 unsigned start_m = (local_team_idx / __tiles_per_col) * TILE_M;
378 unsigned start_n = (local_team_idx % __tiles_per_col) * TILE_N;
379
380 int kk;
381
382 // Fetch entire 2-rank sub-matrix
383 auto svA =
384 subview_wrapper(__A, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag, __ei.__transA_tag);
385 auto svB =
386 subview_wrapper(__B, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag, __ei.__transB_tag);
387 auto svC = subview_wrapper(__C, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag);
388
389 // Allocate scratch memory buffers used for prefetching
390 view_type_2d_scratch svA_scr(member.team_scratch(0), TILE_M, TILE_K);
391 view_type_2d_scratch svB_scr(member.team_scratch(0), TILE_K, TILE_N);
392
393 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, STRIDE_M), [&](const int &thread_id) {
394 int m_offset = thread_id + start_m;
395
396 Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, 0, STRIDE_N), [&](const int &vlane_id) {
397 int n_offset = vlane_id + start_n;
398
399 // Here we populate scratch memory with one or more "k" tiles for
400 // every thread of the team!
401 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
402 #pragma unroll
403 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
404 for (int i = 0; i < REG_N * STRIDE_N; i += STRIDE_N)
405 svB_scr(thread_id, vlane_id + i) =
406 access_view_bounds_check<view_value_type>(svB, thread_id, n_offset + i, __ei.__bounds_check_tag);
407
408 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
409 #pragma unroll
410 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
411 for (int i = 0; i < REG_M * STRIDE_M; i += STRIDE_M)
412 svA_scr(thread_id + i, vlane_id) =
413 access_view_bounds_check<view_value_type>(svA, m_offset + i, vlane_id, __ei.__bounds_check_tag);
414
415 // Wait for A, B to reside in scratch memory
416 member.team_barrier();
417
418 // Each thread calculates a single dot product in chunks of
419 // size TILE_K
420 for (kk = 0; kk < __k - TILE_K; kk += TILE_K) {
421 int k_tile_offset = kk + TILE_K;
422
423 // Get this threads next TILE_K entries from global memory
424 // Each thread has its own copy of prefetch_reg_b.
425 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
426 #pragma unroll
427 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
428 for (int i = 0; i < REG_N; ++i)
429 prefetch_reg_b[i] = access_view_bounds_check<view_value_type>(
430 svB, thread_id + k_tile_offset, n_offset + i * STRIDE_N, __ei.__bounds_check_tag);
431
432 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
433 #pragma unroll
434 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
435 for (int i = 0; i < REG_M; ++i)
436 prefetch_reg_a[i] = access_view_bounds_check<view_value_type>(
437 svA, m_offset + i * STRIDE_M, vlane_id + k_tile_offset, __ei.__bounds_check_tag);
438
439 __rshmem_and_mul(thread_id, vlane_id, TILE_K, reg_a, reg_b, reg_c, svA_scr, svB_scr);
440
441 // Wait for:
442 // 1. prefetch_regs to be populated
443 // 2. for shmem to no longer be read from
444 member.team_barrier();
445
446 // populate shmem from prefetch registers. Each thread has its own
447 // copy of prefetch_reg_b.
448 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
449 #pragma unroll
450 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
451 for (int i = 0; i < REG_N; ++i) svB_scr(thread_id, vlane_id + i * STRIDE_N) = prefetch_reg_b[i];
452
453 // populate shmem from prefetch registers. Each thread has its own
454 // copy of prefetch_reg_a.
455 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
456 #pragma unroll
457 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
458 for (int i = 0; i < REG_M; ++i) svA_scr(thread_id + i * STRIDE_M, vlane_id) = prefetch_reg_a[i];
459
460 // Wait for shmem stores to land before performing next
461 // TILE_K multiply
462 member.team_barrier();
463 } // end n_tile_k_tiles loop
464
465 // Multiply last tile, may be a partial tile
466 __rshmem_and_mul(thread_id, vlane_id, __k - kk, reg_a, reg_b, reg_c, svA_scr, svB_scr);
467
468 // store results back to global memory
469 if (__beta == 0.0F) {
470 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
471 #pragma unroll
472 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
473 for (int m = 0; m < REG_M; ++m) {
474 int cm = m_offset + m * STRIDE_M;
475 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
476 #pragma unroll
477 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
478 for (int n = 0; n < REG_N; ++n) {
479 int cn = n_offset + n * STRIDE_N;
480 fma_bounds_check(svC, cm, cn, reg_c[m][n], __alpha, __ei.__alpha_fma_tag, __ei.__bounds_check_tag);
481 }
482 }
483 } else {
484 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
485 #pragma unroll
486 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
487 for (int m = 0; m < REG_M; ++m) {
488 int cm = m_offset + m * STRIDE_M;
489 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
490 #pragma unroll
491 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
492 for (int n = 0; n < REG_N; ++n) {
493 int cn = n_offset + n * STRIDE_N;
494 fma_bounds_check(svC, cm, cn, reg_c[m][n], __alpha, __beta, __ei.__alpha_fma_tag,
495 __ei.__bounds_check_tag);
496 }
497 }
498 }
499 });
500 });
501 }
502
503 KOKKOS_INLINE_FUNCTION
504 void operator()(LayoutLeftTag, const MemberType &member) const {
505 // TODO: use Kokkos view with compile-time size to allocating register??
506 // Then we can use local deep copy for prefetch_reg population.
507 // Allocate registers used for prefetching
508 view_value_type prefetch_reg_a[REG_M] = {0}, prefetch_reg_b[REG_N] = {0};
509
510 // Allocate registers used for FMAs
511 view_value_type reg_a[REG_M] = {0}, reg_b[REG_N] = {0}, reg_c[REG_M][REG_N] = {{0}};
512 // TODO: look at local loads and stores via nvprof
513 // TODO: look at GPU trace in nvprof to find out how many registers are
514 // used.
515
516 unsigned batch_idx = member.league_rank() / __n_sub_tiles;
517
518 // Compute starting tile offsets for each team into svA, svB, svC
519 unsigned local_team_idx = member.league_rank() % __n_sub_tiles;
520 unsigned start_m = (local_team_idx % __tiles_per_row) * TILE_M;
521 unsigned start_n = (local_team_idx / __tiles_per_row) * TILE_N;
522
523 int kk;
524
525 // Fetch entire 2-rank sub-matrix
526 auto svA =
527 subview_wrapper(__A, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag, __ei.__transA_tag);
528 auto svB =
529 subview_wrapper(__B, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag, __ei.__transB_tag);
530 auto svC = subview_wrapper(__C, batch_idx, Kokkos::ALL(), Kokkos::ALL(), __ei.__batch_layout_tag);
531
532 // Allocate scratch memory buffers used for prefetching
533 view_type_2d_scratch svA_scr(member.team_scratch(0), TILE_K, TILE_M);
534 view_type_2d_scratch svB_scr(member.team_scratch(0), TILE_N, TILE_K);
535
536 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, STRIDE_N), [&](const int &thread_id) {
537 int n_offset = thread_id + start_n;
538
539 Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, 0, STRIDE_M), [&](const int &vlane_id) {
540 int m_offset = vlane_id + start_m;
541
542 // Here we populate scratch memory with one or more "k" tiles for
543 // every thread of the team!
544 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
545 #pragma unroll
546 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
547 for (int i = 0; i < REG_N * STRIDE_N; i += STRIDE_N)
548 svB_scr(thread_id + i, vlane_id) =
549 access_view_bounds_check<view_value_type>(svB, vlane_id, n_offset + i, __ei.__bounds_check_tag);
550
551 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
552 #pragma unroll
553 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
554 for (int i = 0; i < REG_M * STRIDE_M; i += STRIDE_M)
555 svA_scr(thread_id, vlane_id + i) =
556 access_view_bounds_check<view_value_type>(svA, m_offset + i, thread_id, __ei.__bounds_check_tag);
557
558 // Wait for A, B to reside in scratch memory
559 member.team_barrier();
560
561 // Each thread calculates a single dot product in chunks of
562 // size TILE_K
563 for (kk = 0; kk < __k - TILE_K; kk += TILE_K) {
564 int k_tile_offset = kk + TILE_K;
565
566 // Get this threads next TILE_K entries from global memory
567 // Each thread has its own copy of prefetch_reg_b.
568 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
569 #pragma unroll
570 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
571 for (int i = 0; i < REG_N; ++i)
572 prefetch_reg_b[i] = access_view_bounds_check<view_value_type>(
573 svB, vlane_id + k_tile_offset, n_offset + i * STRIDE_N, __ei.__bounds_check_tag);
574
575 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
576 #pragma unroll
577 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
578 for (int i = 0; i < REG_M; ++i)
579 prefetch_reg_a[i] = access_view_bounds_check<view_value_type>(
580 svA, m_offset + i * STRIDE_M, thread_id + k_tile_offset, __ei.__bounds_check_tag);
581
582 __rshmem_and_mul_ll(thread_id, vlane_id, TILE_K, reg_a, reg_b, reg_c, svA_scr, svB_scr);
583
584 // Wait for:
585 // 1. prefetch_regs to be populated
586 // 2. for shmem to no longer be read from
587 member.team_barrier();
588
589 // populate shmem from prefetch registers. Each thread has its own
590 // copy of prefetch_reg_b.
591 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
592 #pragma unroll
593 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
594 for (int i = 0; i < REG_N; ++i) svB_scr(thread_id + i * STRIDE_N, vlane_id) = prefetch_reg_b[i];
595
596 // populate shmem from prefetch registers. Each thread has its own
597 // copy of prefetch_reg_a.
598 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
599 #pragma unroll
600 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
601 for (int i = 0; i < REG_M; ++i) svA_scr(thread_id, vlane_id + i * STRIDE_M) = prefetch_reg_a[i];
602
603 // Wait for shmem stores to land before performing next
604 // TILE_K multiply
605 member.team_barrier();
606 } // end n_tile_k_tiles loop
607
608 // Multiply last tile, may be a partial tile
609 __rshmem_and_mul_ll(thread_id, vlane_id, __k - kk, reg_a, reg_b, reg_c, svA_scr, svB_scr);
610
611 // store results back to global memory
612 if (__beta == 0.0F) {
613 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
614 #pragma unroll
615 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
616 for (int n = 0; n < REG_N; ++n) {
617 int cn = n_offset + n * STRIDE_N;
618
619 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
620 #pragma unroll
621 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
622 for (int m = 0; m < REG_M; ++m) {
623 int cm = m_offset + m * STRIDE_M;
624 fma_bounds_check(svC, cm, cn, reg_c[m][n], __alpha, __ei.__alpha_fma_tag, __ei.__bounds_check_tag);
625 }
626 }
627 } else {
628 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
629 #pragma unroll
630 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
631 for (int n = 0; n < REG_N; ++n) {
632 int cn = n_offset + n * STRIDE_N;
633
634 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
635 #pragma unroll
636 #endif // KOKKOS_ENABLE_PRAGMA_UNROLL
637 for (int m = 0; m < REG_M; ++m) {
638 int cm = m_offset + m * STRIDE_M;
639 fma_bounds_check(svC, cm, cn, reg_c[m][n], __alpha, __beta, __ei.__alpha_fma_tag,
640 __ei.__bounds_check_tag);
641 }
642 }
643 }
644 });
645 });
646 }
647 };
648 };
649 /********************* END non-functor-level routines *********************/
650 } // namespace Impl
651
652 } // namespace KokkosBatched
653
654 #endif