Debian Codegraph @

packages/kokkos-kernels/batched/dense/unit_test/Test_Batched_BatchedGemm.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #include "gtest/gtest.h"
17 #include "Kokkos_Core.hpp"
18 #include "Kokkos_Random.hpp"
19
20 #include "KokkosBatched_HostLevel_Gemm.hpp"
21 #include "KokkosBatched_HostLevel_Gemm_DblBuf_Impl.hpp"
22
23 #include "KokkosKernels_TestUtils.hpp"
24
25 using namespace KokkosBatched;
26
27 namespace Test {
28 template <typename DeviceType, typename ViewType, typename ScalarType, typename ParamTagType>
29 void impl_test_batched_gemm_with_handle(BatchedGemmHandle* batchedGemmHandle, const int N, const int matAdim1,
30 const int matAdim2, const int matBdim1, const int matBdim2, const int matCdim1,
31 const int matCdim2, ScalarType alpha, ScalarType beta) {
32 using execution_space = typename DeviceType::execution_space;
33 using transA = typename ParamTagType::transA;
34 using transB = typename ParamTagType::transB;
35 using batchLayout = typename ParamTagType::batchLayout;
36 using ats = Kokkos::ArithTraits<ScalarType>;
37
38 int ret = 0;
39 auto algo_type = batchedGemmHandle->get_kernel_algo_type();
40 ViewType a_expected, a_actual, b_expected, b_actual, c_expected, c_actual;
41 std::string fmsg;
42 std::string fmsg_rhs = "algo_type:" + batchedGemmHandle->get_kernel_algo_type_str() + ", ";
43 fmsg_rhs += ("N:" + std::to_string(N) + ", ");
44 fmsg_rhs += ("A:" + std::to_string(matAdim1) + "x" + std::to_string(matAdim2) + ", ");
45 fmsg_rhs += ("B:" + std::to_string(matBdim1) + "x" + std::to_string(matBdim2) + ", ");
46 fmsg_rhs += ("C:" + std::to_string(matCdim1) + "x" + std::to_string(matCdim2) + "\n");
47
48 if (std::is_same<batchLayout, BatchLayout::Left>::value) {
49 a_expected = ViewType("a_expected", N, matAdim1, matAdim2);
50 a_actual = ViewType("a_actual", N, matAdim1, matAdim2);
51 b_expected = ViewType("b_expected", N, matBdim1, matBdim2);
52 b_actual = ViewType("b_actual", N, matBdim1, matBdim2);
53 c_expected = ViewType("c_expected", N, matCdim1, matCdim2);
54 c_actual = ViewType("c_actual", N, matCdim1, matCdim2);
55 } else if (std::is_same<batchLayout, BatchLayout::Right>::value) {
56 a_expected = ViewType("a_expected", matAdim1, matAdim2, N);
57 a_actual = ViewType("a_actual", matAdim1, matAdim2, N);
58 b_expected = ViewType("b_expected", matBdim1, matBdim2, N);
59 b_actual = ViewType("b_actual", matBdim1, matBdim2, N);
60 c_expected = ViewType("c_expected", matCdim1, matCdim2, N);
61 c_actual = ViewType("c_actual", matCdim1, matCdim2, N);
62 }
63
64 Kokkos::Random_XorShift64_Pool<execution_space> random(13718);
65
66 Kokkos::fill_random(a_expected, random, ScalarType(1.0));
67 Kokkos::fill_random(b_expected, random, ScalarType(1.0));
68 Kokkos::fill_random(c_expected, random, ScalarType(1.0));
69
70 Kokkos::fence();
71
72 Kokkos::deep_copy(a_actual, a_expected);
73 Kokkos::deep_copy(b_actual, b_expected);
74 Kokkos::deep_copy(c_actual, c_expected);
75
76 // Check for expected BatchedDblBufGemm runtime errors
77 if (algo_type == GemmKokkosBatchedAlgos::KK_DBLBUF) {
78 // Check for DblBuf runtime errors related to team_size
79 try {
80 fmsg = kk_failure_str(__FILE__, __FUNCTION__, __LINE__);
81 Impl::BatchedDblBufGemm<transA, transB, batchLayout, BatchedGemmHandle, ScalarType, decltype(a_actual),
82 decltype(b_actual), decltype(c_actual), BoundsCheck::Yes, AlphaTag::No, 65536, 1, 65536>(
83 batchedGemmHandle, alpha, a_actual, b_actual, beta, c_actual)
84 .invoke();
85 FAIL() << (fmsg + fmsg_rhs);
86 } catch (const std::runtime_error& error) {
87 ;
88 }
89
90 // Check for DblBuf runtime errors related to vector_len
91 try {
92 fmsg = kk_failure_str(__FILE__, __FUNCTION__, __LINE__);
93 Impl::BatchedDblBufGemm<transA, transB, batchLayout, BatchedGemmHandle, ScalarType, decltype(a_actual),
94 decltype(b_actual), decltype(c_actual), BoundsCheck::No, AlphaTag::No, 65536, 65536 * 2,
95 65536>(batchedGemmHandle, alpha, a_actual, b_actual, beta, c_actual)
96 .invoke();
97 FAIL() << (fmsg + fmsg_rhs);
98 } catch (const std::runtime_error& error) {
99 ;
100 }
101 }
102
103 // Check for expected BatchedGemm runtime errors
104 try {
105 #if defined(KOKKOSKERNELS_ENABLE_TPL_ARMPL)
106 if (algo_type == BaseTplAlgos::ARMPL && N % 2 == 0) {
107 auto ninter = batchedGemmHandle->get_tpl_params();
108 // Set ninter parameter for underlying armpl_dgemm_interleave_batch call
109 *ninter = N / 2;
110 }
111 #endif
112
113 fmsg = kk_failure_str(__FILE__, __FUNCTION__, __LINE__);
114 ret = BatchedGemm<transA, transB, batchLayout>(batchedGemmHandle, alpha, a_actual, b_actual, beta,
115 c_actual); // Compute c_actual
116 } catch (const std::runtime_error& error) {
117 std::string error_msg = error.what();
118 if (algo_type == BaseHeuristicAlgos::SQUARE && matCdim1 != matCdim2) {
119 ;
120 } else if (algo_type == BaseTplAlgos::ARMPL) {
121 #if defined(KOKKOSKERNELS_ENABLE_TPL_ARMPL) && ARMPL_BUILD >= 1058
122 auto ninter = batchedGemmHandle->get_tpl_params()[0];
123 // No runtime errors expected since layout is valid, double is a supported
124 // type, and ninter != 0
125 if (std::is_same<typename ViewType::value_type, double>::value && ninter != 0) {
126 FAIL() << (error_msg + fmsg + fmsg_rhs);
127 }
128 #else
129 ; // We expect a runtime error if the ARMPL TPL is not enabled
130 #endif
131 } else {
132 FAIL() << (error_msg + fmsg + fmsg_rhs);
133 }
134 return;
135 }
136 ASSERT_EQ(ret, 0) << (fmsg + fmsg_rhs);
137
138 Functor_BatchedVanillaGEMM<ViewType, ViewType, ViewType, execution_space> vgemm;
139 vgemm.A_t = std::is_same<transA, Trans::Transpose>::value;
140 vgemm.B_t = std::is_same<transB, Trans::Transpose>::value;
141 vgemm.batch_size_last_dim = std::is_same<batchLayout, BatchLayout::Right>::value;
142 vgemm.A_c = vgemm.B_c = false;
143 vgemm.A = a_expected;
144 vgemm.B = b_expected;
145 vgemm.C = c_expected;
146 vgemm.alpha = alpha;
147 vgemm.beta = beta;
148 vgemm.run(); // Compute c_expected
149
150 Kokkos::fence();
151
152 typename ViewType::HostMirror c_expected_host = Kokkos::create_mirror_view(c_expected);
153 typename ViewType::HostMirror c_actual_host = Kokkos::create_mirror_view(c_actual);
154
155 // Copy to host
156 Kokkos::deep_copy(c_expected_host, c_expected);
157 Kokkos::deep_copy(c_actual_host, c_actual);
158
159 Kokkos::fence();
160
161 // check c_expected = c_actual ; this eps is about 2^-9
162 // Set mag_type to host_value_type, we may not have half precision on host
163 using mag_type = float;
164 mag_type sum(1), diff(0);
165
166 auto eps = static_cast<mag_type>(ats::epsilon());
167
168 eps *= std::is_same<ScalarType, Kokkos::Experimental::half_t>::value ||
169 std::is_same<ScalarType, Kokkos::Experimental::bhalf_t>::value
170 ? 4
171 : 1e3;
172
173 for (int k = 0; k < N; ++k) {
174 for (int i = 0; i < matCdim1; ++i) {
175 for (int j = 0; j < matCdim2; ++j) {
176 if (std::is_same<batchLayout, BatchLayout::Right>::value) {
177 sum += ats::abs(c_expected_host(i, j, k));
178 diff += ats::abs(c_expected_host(i, j, k) - c_actual_host(i, j, k));
179 } else {
180 sum += ats::abs(c_expected_host(k, i, j));
181 diff += ats::abs(c_expected_host(k, i, j) - c_actual_host(k, i, j));
182 }
183 }
184 }
185 }
186
187 EXPECT_NEAR_KK(diff / sum, 0, eps, fmsg + fmsg_rhs);
188 }
189
190 template <typename DeviceType, typename ViewType, typename ScalarType, typename ParamTagType>
191 void impl_test_batched_gemm(const int N, const int matAdim1, const int matAdim2, const int matBdim1, const int matBdim2,
192 const int matCdim1, const int matCdim2) {
193 {
194 BatchedGemmHandle batchedGemmHandle;
195
196 ASSERT_EQ(batchedGemmHandle.get_kernel_algo_type(), BaseHeuristicAlgos::SQUARE);
197 ASSERT_EQ(batchedGemmHandle.teamSz, 0);
198 ASSERT_EQ(batchedGemmHandle.vecLen, 0);
199
200 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUBLAS)
201 cublasHandle_t cublas_handle;
202 BatchedGemmHandle batchedGemmHandleCublas(cublas_handle, GemmTplAlgos::CUBLAS, 0, 0);
203 ASSERT_EQ(&cublas_handle, batchedGemmHandleCublas.get_tpl_params());
204 ASSERT_EQ(batchedGemmHandleCublas.get_kernel_algo_type(), (int)GemmTplAlgos::CUBLAS);
205 ASSERT_EQ(batchedGemmHandleCublas.teamSz, 0);
206 ASSERT_EQ(batchedGemmHandleCublas.vecLen, 0);
207 #endif
208
209 // FIXME temporary workaround to run this magma test only if cublas is not
210 // enabled the design of the BatchedGemmHandle currently does not allow
211 // simultanous testing in this way. See issue #2177
212 #if defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA) && !defined(KOKKOSKERNELS_ENABLE_TPL_CUBLAS)
213 magma_queue_t magma_queue;
214 BatchedGemmHandle batchedGemmHandleMagma(magma_queue, GemmTplAlgos::MAGMA, 0, 0);
215 ASSERT_EQ(&magma_queue, batchedGemmHandleMagma.get_tpl_params());
216 ASSERT_EQ(batchedGemmHandleMagma.get_kernel_algo_type(), (int)GemmTplAlgos::MAGMA);
217 ASSERT_EQ(batchedGemmHandleMagma.teamSz, 0);
218 ASSERT_EQ(batchedGemmHandleMagma.vecLen, 0);
219 #endif
220 }
221
222 for (int algo_type = BaseHeuristicAlgos::SQUARE; algo_type < GemmKokkosBatchedAlgos::N; ++algo_type) {
223 {
224 try {
225 BatchedGemmHandle batchedGemmHandle(algo_type);
226
227 ASSERT_EQ(batchedGemmHandle.get_kernel_algo_type(), algo_type);
228
229 if (algo_type == BaseTplAlgos::ARMPL || algo_type == BaseKokkosBatchedAlgos::KK_SERIAL ||
230 algo_type == GemmKokkosBatchedAlgos::KK_SERIAL_RANK0 || algo_type == GemmKokkosBatchedAlgos::KK_DBLBUF) {
231 impl_test_batched_gemm_with_handle<DeviceType, ViewType, ScalarType, ParamTagType>(
232 &batchedGemmHandle, N, matAdim1, matAdim2, matBdim1, matBdim2, matCdim1, matCdim2, 1.5, 3.0);
233 } else if (algo_type == BaseHeuristicAlgos::SQUARE) {
234 // Invoke 4 times to ensure we cover all paths for alpha and beta
235 impl_test_batched_gemm_with_handle<DeviceType, ViewType, ScalarType, ParamTagType>(
236 &batchedGemmHandle, N, matAdim1, matAdim2, matBdim1, matBdim2, matCdim1, matCdim2, 0.0, 0.0);
237 impl_test_batched_gemm_with_handle<DeviceType, ViewType, ScalarType, ParamTagType>(
238 &batchedGemmHandle, N, matAdim1, matAdim2, matBdim1, matBdim2, matCdim1, matCdim2, 1.0, 0.0);
239 impl_test_batched_gemm_with_handle<DeviceType, ViewType, ScalarType, ParamTagType>(
240 &batchedGemmHandle, N, matAdim1, matAdim2, matBdim1, matBdim2, matCdim1, matCdim2, 0.0, 1.0);
241 impl_test_batched_gemm_with_handle<DeviceType, ViewType, ScalarType, ParamTagType>(
242 &batchedGemmHandle, N, matAdim1, matAdim2, matBdim1, matBdim2, matCdim1, matCdim2, 1.5, 3.0);
243 } else {
244 try {
245 // Allocate these views to invoke BatchedGemm with an unsupported
246 // algo type
247 ViewType a_actual("a_actual", N, matAdim1, matAdim2);
248 ViewType b_actual("b_actual", N, matBdim1, matBdim2);
249 ViewType c_actual("c_actual", N, matCdim1, matCdim2);
250 using ta = typename ParamTagType::transA;
251 using tb = typename ParamTagType::transB;
252 using bl = typename ParamTagType::batchLayout;
253 ScalarType alpha = 0.34;
254 ScalarType beta = 0.43;
255 BatchedGemm<ta, tb, bl>(&batchedGemmHandle, alpha, a_actual, b_actual, beta, c_actual);
256 std::string fmsg = kk_failure_str(__FILE__, __FUNCTION__, __LINE__);
257 FAIL() << fmsg;
258 } catch (const std::runtime_error& error) {
259 ;
260 }
261 }
262 } catch (const std::runtime_error& error) {
263 #if !defined(KOKKOSKERNELS_ENABLE_TPL_ARMPL) || (ARMPL_BUILD < 1058)
264 if (algo_type == BaseTplAlgos::ARMPL) {
265 ;
266 } else {
267 std::string fmsg = kk_failure_str(__FILE__, __FUNCTION__, __LINE__);
268 FAIL() << fmsg;
269 }
270 #endif // KOKKOSKERNELS_ENABLE_TPL_ARMPL
271 }
272 }
273 }
274 }
275 } // namespace Test
276
277 template <typename ViewType, typename DeviceType, typename ValueType, typename ScalarType, typename ParamTagType>
278 void test_batched_gemm_with_layout(int N) {
279 // Square cases
280 {
281 int i = 0;
282 Test::impl_test_batched_gemm<DeviceType, ViewType, ScalarType, ParamTagType>(N, i, i, i, i, i, i);
283
284 i = 10;
285 Test::impl_test_batched_gemm<DeviceType, ViewType, ScalarType, ParamTagType>(N, i, i, i, i, i, i);
286
287 i = 25;
288 Test::impl_test_batched_gemm<DeviceType, ViewType, ScalarType, ParamTagType>(N, i, i, i, i, i, i);
289
290 i = 32;
291 Test::impl_test_batched_gemm<DeviceType, ViewType, ScalarType, ParamTagType>(N, i, i, i, i, i, i);
292 }
293
294 // Non-square cases
295 for (int i = 1; i < 5; ++i) {
296 int dimM = 1 * i;
297 int dimN = 2 * i;
298 int dimK = 3 * i;
299 if ((std::is_same<typename ParamTagType::transA, KokkosBatched::Trans::NoTranspose>::value) &&
300 (std::is_same<typename ParamTagType::transB, KokkosBatched::Trans::NoTranspose>::value)) {
301 Test::impl_test_batched_gemm<DeviceType, ViewType, ScalarType, ParamTagType>(N, dimM, dimK, dimK, dimN, dimM,
302 dimN);
303 }
304 if ((std::is_same<typename ParamTagType::transA, KokkosBatched::Trans::NoTranspose>::value) &&
305 (std::is_same<typename ParamTagType::transB, KokkosBatched::Trans::Transpose>::value)) {
306 Test::impl_test_batched_gemm<DeviceType, ViewType, ScalarType, ParamTagType>(N, dimM, dimK, dimN, dimK, dimM,
307 dimN);
308 }
309 if ((std::is_same<typename ParamTagType::transA, KokkosBatched::Trans::Transpose>::value) &&
310 (std::is_same<typename ParamTagType::transB, KokkosBatched::Trans::NoTranspose>::value)) {
311 Test::impl_test_batched_gemm<DeviceType, ViewType, ScalarType, ParamTagType>(N, dimK, dimM, dimK, dimN, dimM,
312 dimN);
313 }
314 if ((std::is_same<typename ParamTagType::transA, KokkosBatched::Trans::Transpose>::value) &&
315 (std::is_same<typename ParamTagType::transB, KokkosBatched::Trans::Transpose>::value)) {
316 Test::impl_test_batched_gemm<DeviceType, ViewType, ScalarType, ParamTagType>(N, dimK, dimM, dimN, dimK, dimM,
317 dimN);
318 }
319 }
320 }
321
322 template <typename DeviceType, typename ValueType, typename ScalarType, typename ParamTagType>
323 int test_batched_gemm() {
324 #if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) || \
325 (!defined(KOKKOSKERNELS_ETI_ONLY) && !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS))
326 if constexpr (std::is_same_v<typename ParamTagType::batchLayout, typename BatchLayout::Right>) {
327 using param_tag_type =
328 ::Test::SharedParamTag<typename ParamTagType::transA, typename ParamTagType::transB, BatchLayout::Right>;
329 typedef Kokkos::View<ValueType***, Kokkos::LayoutLeft, DeviceType> llVt;
330 test_batched_gemm_with_layout<llVt, DeviceType, ValueType, ScalarType, param_tag_type>(0);
331 test_batched_gemm_with_layout<llVt, DeviceType, ValueType, ScalarType, param_tag_type>(1);
332 test_batched_gemm_with_layout<llVt, DeviceType, ValueType, ScalarType, param_tag_type>(4);
333 test_batched_gemm_with_layout<llVt, DeviceType, ValueType, ScalarType, param_tag_type>(8);
334 test_batched_gemm_with_layout<llVt, DeviceType, ValueType, ScalarType, param_tag_type>(16);
335 } else {
336 std::cerr << "TEST SKIPPED since BatchLayout is not Right." << std::endl;
337 }
338 #else
339 std::cerr << "TEST SKIPPED since LayoutLeft is not ETI'd." << std::endl;
340 #endif // KOKKOSKERNELS_INST_LAYOUTLEFT
341
342 #if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) || \
343 (!defined(KOKKOSKERNELS_ETI_ONLY) && !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS))
344 if constexpr (std::is_same_v<typename ParamTagType::batchLayout, typename BatchLayout::Left>) {
345 using param_tag_type =
346 ::Test::SharedParamTag<typename ParamTagType::transA, typename ParamTagType::transB, BatchLayout::Left>;
347 typedef Kokkos::View<ValueType***, Kokkos::LayoutRight, DeviceType> lrVt;
348 test_batched_gemm_with_layout<lrVt, DeviceType, ValueType, ScalarType, param_tag_type>(0);
349 test_batched_gemm_with_layout<lrVt, DeviceType, ValueType, ScalarType, param_tag_type>(1);
350 test_batched_gemm_with_layout<lrVt, DeviceType, ValueType, ScalarType, param_tag_type>(4);
351 test_batched_gemm_with_layout<lrVt, DeviceType, ValueType, ScalarType, param_tag_type>(8);
352 test_batched_gemm_with_layout<lrVt, DeviceType, ValueType, ScalarType, param_tag_type>(16);
353 } else {
354 std::cerr << "TEST SKIPPED since BatchLayout is not Left." << std::endl;
355 }
356 #else
357 std::cerr << "TEST SKIPPED since LayoutRight is not ETI'd." << std::endl;
358 #endif // KOKKOSKERNELS_INST_LAYOUTRIGHT
359 return 0;
360 }