Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_HostLevel_Gemm_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_HOSTLEVEL_GEMM_IMPL_HPP
17 #define KOKKOSBATCHED_HOSTLEVEL_GEMM_IMPL_HPP
18 #include <Kokkos_Core.hpp>
19 #include <KokkosBatched_Util.hpp> // Trans, BatchLayout
20 #include <KokkosKernels_ExecSpaceUtils.hpp>
21 #include <KokkosKernels_Error.hpp>
22
23 #include "KokkosBatched_HostLevel_Gemm_Handle.hpp" // BatchedGemmHandle
24 #include "KokkosBatched_HostLevel_Gemm_Serial_Impl.hpp"
25 #include "KokkosBatched_HostLevel_Gemm_DblBuf_Impl.hpp"
26 #include "KokkosBatched_HostLevel_Gemm_Armpl_Impl.hpp"
27
28 namespace KokkosBatched {
29 namespace Impl {
30 //////////////////////////////// tile_m //////////////////////////////////
31 template <typename ExecutionSpace>
32 constexpr KOKKOS_INLINE_FUNCTION int kk_gemm_dbl_buf_tile_m() {
33 return 32;
34 }
35 //////////////////////////////// tile_n //////////////////////////////////
36 template <typename ExecutionSpace>
37 constexpr KOKKOS_INLINE_FUNCTION int kk_gemm_dbl_buf_tile_n() {
38 return 32;
39 }
40 //////////////////////////////// tile_k //////////////////////////////////
41 template <typename ExecutionSpace>
42 constexpr KOKKOS_INLINE_FUNCTION int kk_gemm_dbl_buf_tile_k() {
43 return 8;
44 }
45
46 // On MI100, batched_scalar_batched_gemm_nt_nt_dcomplex_dcomplex_right fails
47 // without this. See https://github.com/kokkos/kokkos-kernels/issues/1547.
48 // This reduces the register allocations (REG_M and REG_N) in the double
49 // buffering algorithm by a factor of 2.
50 #if defined(KOKKOS_ENABLE_HIP) && defined(KOKKOS_ARCH_VEGA908)
51 template <>
52 constexpr KOKKOS_INLINE_FUNCTION int kk_gemm_dbl_buf_tile_k<Kokkos::HIP>() {
53 return 16;
54 }
55 #endif
56 ////////////////////////// alpha_in_fma_thresh ////////////////////////////
57 constexpr KOKKOS_INLINE_FUNCTION size_t kk_gemm_dbl_buf_alpha_in_fma_thresh() {
58 #ifdef __CUDACC_RDC__
59 return 24;
60 #else
61 return 64;
62 #endif // __CUDAACC_RDC__
63 }
64
65 template <typename ArgTransA, typename ArgTransB, typename ArgBatchSzDim, typename BatchedGemmHandleType,
66 typename ScalarType, typename AViewType, typename BViewType, typename CViewType>
67 int BatchedGemmImpl(BatchedGemmHandleType *const handle, const ScalarType alpha, const AViewType &A, const BViewType &B,
68 const ScalarType beta, const CViewType &C) {
69 int ret = 0;
70 size_t c_m, c_n;
71 using ViewValueType = typename CViewType::value_type;
72 // Check for valid input views
73 static_assert(Kokkos::is_view<AViewType>::value, "AViewType must be a Kokkos::View.");
74 static_assert(Kokkos::is_view<BViewType>::value, "BViewType must be a Kokkos::View.");
75 static_assert(Kokkos::is_view<CViewType>::value, "CViewType must be a Kokkos::View.");
76 static_assert(std::is_same<ArgTransA, Trans::NoTranspose>::value || std::is_same<ArgTransA, Trans::Transpose>::value,
77 "ArgTransA must be either Trans::Transpose or Trans::NoTranspose.");
78 static_assert(std::is_same<ArgTransB, Trans::NoTranspose>::value || std::is_same<ArgTransB, Trans::Transpose>::value,
79 "ArgTransB must be either Trans::Transpose or Trans::NoTranspose.");
80 if constexpr (is_vector<ViewValueType>::value) {
81 // Check ranks of view with underlying SIMD value types
82 // For SIMD views, we can have either 3-rank or 4-ranks inputs.
83 switch (handle->get_kernel_algo_type()) {
84 case BaseKokkosBatchedAlgos::KK_SERIAL:
85 case BaseHeuristicAlgos::SQUARE:
86 case BaseTplAlgos::ARMPL:
87 assert(A.rank_dynamic() == 3 && "AViewType must have rank 3.");
88 assert(B.rank_dynamic() == 3 && "BViewType must have rank 3.");
89 assert(C.rank_dynamic() == 3 && "CViewType must have rank 3.");
90 break;
91 default:
92 std::ostringstream os;
93 os << "KokkosBatched::BatchedGemm does not support kernelAlgoType = "
94 << std::to_string(handle->get_kernel_algo_type()) << " with SIMD views." << std::endl;
95 KokkosKernels::Impl::throw_runtime_exception(os.str());
96 break;
97 }
98 } else {
99 // Check ranks of views with underlying scalar value types
100 static_assert(static_cast<int>(AViewType::rank) == 3, "AViewType must have rank 3.");
101 static_assert(static_cast<int>(BViewType::rank) == 3, "BViewType must have rank 3.");
102 static_assert(static_cast<int>(CViewType::rank) == 3, "CViewType must have rank 3.");
103 }
104
105 // Check for valid data access patterns
106 // Skip checking a_layout == b_layout == c_layout
107 // Skip checking for LayoutStride
108 using c_layout = typename CViewType::array_layout;
109 static_assert(
110 !(std::is_same<c_layout, Kokkos::LayoutLeft>::value && !std::is_same<ArgBatchSzDim, BatchLayout::Right>::value),
111 "LayoutLeft views require BatchLayout::Right");
112 static_assert(
113 !(std::is_same<c_layout, Kokkos::LayoutRight>::value && !std::is_same<ArgBatchSzDim, BatchLayout::Left>::value),
114 "LayoutRight views require BatchLayout::Left");
115
116 if constexpr (std::is_same<ArgBatchSzDim, BatchLayout::Left>::value) {
117 // c_b = C.extent(0);
118 c_m = C.extent(1);
119 c_n = C.extent(2);
120 } else {
121 // c_b = C.extent(2);
122 c_m = C.extent(0);
123 c_n = C.extent(1);
124 }
125
126 // Begin checking conditions for optimal BatchedGemm invocation.
127 using view_scalar_type = typename CViewType::value_type;
128 using layout_type = typename CViewType::array_layout;
129 using exec_space = typename CViewType::execution_space;
130 constexpr bool is_vector = KokkosBatched::is_vector<view_scalar_type>::value;
131 constexpr bool on_gpu = KokkosKernels::Impl::is_gpu_exec_space_v<exec_space>;
132 constexpr bool on_x86_64 = KokkosKernels::Impl::kk_is_x86_64_mem_space<typename exec_space::memory_space>();
133 constexpr bool on_a64fx = KokkosKernels::Impl::kk_is_a64fx_mem_space<typename exec_space::memory_space>();
134 bool out_of_range = false;
135
136 if (handle->enableDebug) {
137 std::cout << "view_scalar_type:" << typeid(view_scalar_type).name() << std::endl
138 << "execution_space:" << typeid(exec_space).name() << std::endl
139 << std::endl
140 << "is_vector:" << is_vector << std::endl
141 << "on_gpu:" << on_gpu << std::endl
142 << "on_x86_64:" << on_x86_64 << std::endl
143 << "on_a64fx:" << on_a64fx << std::endl;
144 }
145
146 switch (handle->get_kernel_algo_type()) {
147 ////////////// HEURISTIC ALGOS //////////////
148 case BaseHeuristicAlgos::SQUARE:
149 if (c_m != c_n) {
150 std::ostringstream os;
151 os << "KokkosBatched::BatchedGemm does not support kernelAlgoType = "
152 << std::to_string(handle->get_kernel_algo_type()) << " when c_m(" << std::to_string(c_m) << ") != c_n("
153 << std::to_string(c_n) << ")" << std::endl;
154 KokkosKernels::Impl::throw_runtime_exception(os.str());
155 }
156
157 // Select optimal resultsPerThread param for BatchedSerialGemm
158 using bsgResultsPerThread =
159 std::conditional_t<!is_vector && on_gpu, ResultsPerThread::Rank0, ResultsPerThread::Rank2>;
160
161 // Select optimal mode param for SerialGemm.
162 using bsgModeType = typename std::conditional<
163 is_vector, typename std::conditional<on_gpu || on_x86_64, Algo::Gemm::Blocked, Algo::Gemm::Unblocked>::type,
164 typename std::conditional<
165 on_gpu, Algo::Gemm::Unblocked,
166 typename std::conditional<on_a64fx, Algo::Gemm::Unblocked, Algo::Gemm::Blocked>::type>::type>::type;
167
168 if (handle->enableDebug) {
169 std::cout << "bsgResultsPerThread: " << typeid(bsgResultsPerThread).name() << std::endl
170 << "bsgModeType: " << typeid(bsgModeType).name() << std::endl;
171 }
172
173 if constexpr (on_gpu) {
174 if (((std::is_same<layout_type, Kokkos::LayoutLeft>::value) ? (c_m >= 16)
175 : (c_m >= 24 && c_m <= 32) || c_m >= 40)) {
176 handle->teamSz = handle->vecLen = 8;
177 constexpr int tile_m = Impl::kk_gemm_dbl_buf_tile_m<exec_space>();
178 constexpr int tile_n = Impl::kk_gemm_dbl_buf_tile_n<exec_space>();
179 constexpr int tile_k = Impl::kk_gemm_dbl_buf_tile_k<exec_space>();
180 constexpr size_t alpha_in_fma_thresh = Impl::kk_gemm_dbl_buf_alpha_in_fma_thresh();
181
182 if (c_m % 32 == 0) { // No bounds checking
183 if (c_m >= alpha_in_fma_thresh) { // apply alpha in fma
184 ret = Impl::BatchedDblBufGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType,
185 AViewType, BViewType, CViewType, BoundsCheck::No, AlphaTag::Yes, tile_m,
186 tile_n, tile_k>(handle, alpha, A, B, beta, C)
187 .invoke();
188 } else { // apply alpha in mul
189 ret = Impl::BatchedDblBufGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType,
190 AViewType, BViewType, CViewType, BoundsCheck::No, AlphaTag::No, tile_m,
191 tile_n, tile_k>(handle, alpha, A, B, beta, C)
192 .invoke();
193 }
194 } else { // bounds checking
195 if (c_m >= alpha_in_fma_thresh) { // apply alpha in fma
196 ret = Impl::BatchedDblBufGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType,
197 AViewType, BViewType, CViewType, BoundsCheck::Yes, AlphaTag::Yes, tile_m,
198 tile_n, tile_k>(handle, alpha, A, B, beta, C)
199 .invoke();
200 } else { // apply alpha in mul
201 ret = Impl::BatchedDblBufGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType,
202 AViewType, BViewType, CViewType, BoundsCheck::Yes, AlphaTag::No, tile_m,
203 tile_n, tile_k>(handle, alpha, A, B, beta, C)
204 .invoke();
205 }
206 }
207 } else {
208 out_of_range = true;
209 }
210 }
211 if (!on_gpu || out_of_range) {
212 ret = Impl::BatchedSerialGemm<ArgTransA, ArgTransB, bsgModeType, ArgBatchSzDim, bsgResultsPerThread, ScalarType,
213 AViewType, BViewType, CViewType>(alpha, A, B, beta, C)
214 .invoke();
215 }
216 break;
217
218 // case BaseHeuristicAlgos::TALL:
219 //
220 // case BaseHeuristicAlgos::WIDE:
221 ////////////// TPL ALGOS //////////////
222 #if defined(KOKKOSKERNELS_ENABLE_TPL_ARMPL) && ARMPL_BUILD >= 1058
223 case BaseTplAlgos::ARMPL:
224 ret = Impl::BatchedArmplGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType, AViewType,
225 BViewType, CViewType>(handle, alpha, A, B, beta, C)
226 .invoke();
227 break;
228 #endif // KOKKOSKERNELS_ENABLE_TPL_ARMPL
229 // case BaseTplAlgos::MKL:
230 //
231 // case GemmTplAlgos::CUBLAS:
232 //
233 // case GemmTplAlgos::MAGMA:
234
235 ////////////// KokkosBatched ALGOS //////////////
236 case BaseKokkosBatchedAlgos::KK_SERIAL:
237 ret = Impl::BatchedSerialGemm<ArgTransA, ArgTransB, Algo::Gemm::Unblocked, ArgBatchSzDim, ResultsPerThread::Rank2,
238 ScalarType, AViewType, BViewType, CViewType>(alpha, A, B, beta, C)
239 .invoke();
240 break;
241
242 // case GemmKokkosBatchedAlgos::KK_SERIALSIMD:
243
244 case GemmKokkosBatchedAlgos::KK_SERIAL_RANK0:
245 ret = Impl::BatchedSerialGemm<ArgTransA, ArgTransB, Algo::Gemm::Unblocked, ArgBatchSzDim, ResultsPerThread::Rank0,
246 ScalarType, AViewType, BViewType, CViewType>(alpha, A, B, beta, C)
247 .invoke();
248 break;
249
250 // case GemmKokkosBatchedAlgos::KK_SERIAL_SHMEM:
251 // case GemmKokkosBatchedAlgos::KK_TEAM:
252 // case GemmKokkosBatchedAlgos::KK_TEAMVECTOR:
253 // case GemmKokkosBatchedAlgos::KK_TEAMSIMD:
254
255 case GemmKokkosBatchedAlgos::KK_DBLBUF:
256 // Note: The tile sizes of 1x1x1 here will not perform well but must be
257 // selected in order to function on all devices since the serial
258 // execution space has a max team size of 1. KokkosKernels API users
259 // will need to follow an approach similar to KK_SQUARE above for best
260 // performance.
261
262 // TODO: Add auto-selection of tile size based on inputs and device type
263 ret = Impl::BatchedDblBufGemm<ArgTransA, ArgTransB, ArgBatchSzDim, BatchedGemmHandleType, ScalarType, AViewType,
264 BViewType, CViewType, BoundsCheck::Yes, AlphaTag::No, 1, 1, 1>(handle, alpha, A, B,
265 beta, C)
266 .invoke();
267 break;
268
269 default:
270 std::ostringstream os;
271 os << "KokkosBatched::BatchedGemm does not support kernelAlgoType = "
272 << std::to_string(handle->get_kernel_algo_type()) << "." << std::endl;
273 KokkosKernels::Impl::throw_runtime_exception(os.str());
274 break;
275 }
276 return ret;
277 }
278 } // namespace Impl
279 } // namespace KokkosBatched
280 #endif // KOKKOSBATCHED_HOSTLEVEL_GEMM_IMPL_HPP