//@HEADER // ************************************************************************ // // Kokkos v. 4.0 // Copyright (2022) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. // See https://kokkos.org/LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //@HEADER #include "gtest/gtest.h" #include "Kokkos_Core.hpp" #include "Kokkos_Random.hpp" #include "KokkosBatched_HostLevel_Gemm.hpp" #include "KokkosBatched_HostLevel_Gemm_DblBuf_Impl.hpp" #include "KokkosKernels_TestUtils.hpp" using namespace KokkosBatched; namespace Test { template void impl_test_batched_gemm_with_handle(BatchedGemmHandle* batchedGemmHandle, const int N, const int matAdim1, const int matAdim2, const int matBdim1, const int matBdim2, const int matCdim1, const int matCdim2, ScalarType alpha, ScalarType beta) { using execution_space = typename DeviceType::execution_space; using transA = typename ParamTagType::transA; using transB = typename ParamTagType::transB; using batchLayout = typename ParamTagType::batchLayout; using ats = Kokkos::ArithTraits; int ret = 0; auto algo_type = batchedGemmHandle->get_kernel_algo_type(); ViewType a_expected, a_actual, b_expected, b_actual, c_expected, c_actual; std::string fmsg; std::string fmsg_rhs = "algo_type:" + batchedGemmHandle->get_kernel_algo_type_str() + ", "; fmsg_rhs += ("N:" + std::to_string(N) + ", "); fmsg_rhs += ("A:" + std::to_string(matAdim1) + "x" + std::to_string(matAdim2) + ", "); fmsg_rhs += ("B:" + std::to_string(matBdim1) + "x" + std::to_string(matBdim2) + ", "); fmsg_rhs += ("C:" + std::to_string(matCdim1) + "x" + std::to_string(matCdim2) + "\n"); if (std::is_same::value) { a_expected = ViewType("a_expected", N, matAdim1, matAdim2); a_actual = ViewType("a_actual", N, matAdim1, matAdim2); b_expected = ViewType("b_expected", N, matBdim1, matBdim2); b_actual = ViewType("b_actual", N, matBdim1, matBdim2); c_expected = ViewType("c_expected", N, matCdim1, matCdim2); c_actual = ViewType("c_actual", N, matCdim1, matCdim2); } else if (std::is_same::value) { a_expected = ViewType("a_expected", matAdim1, matAdim2, N); a_actual = ViewType("a_actual", matAdim1, matAdim2, N); b_expected = ViewType("b_expected", matBdim1, matBdim2, N); b_actual = ViewType("b_actual", matBdim1, matBdim2, N); c_expected = ViewType("c_expected", matCdim1, matCdim2, N); c_actual = ViewType("c_actual", matCdim1, matCdim2, N); } Kokkos::Random_XorShift64_Pool random(13718); Kokkos::fill_random(a_expected, random, ScalarType(1.0)); Kokkos::fill_random(b_expected, random, ScalarType(1.0)); Kokkos::fill_random(c_expected, random, ScalarType(1.0)); Kokkos::fence(); Kokkos::deep_copy(a_actual, a_expected); Kokkos::deep_copy(b_actual, b_expected); Kokkos::deep_copy(c_actual, c_expected); // Check for expected BatchedDblBufGemm runtime errors if (algo_type == GemmKokkosBatchedAlgos::KK_DBLBUF) { // Check for DblBuf runtime errors related to team_size try { fmsg = kk_failure_str(__FILE__, __FUNCTION__, __LINE__); Impl::BatchedDblBufGemm( batchedGemmHandle, alpha, a_actual, b_actual, beta, c_actual) .invoke(); FAIL() << (fmsg + fmsg_rhs); } catch (const std::runtime_error& error) { ; } // Check for DblBuf runtime errors related to vector_len try { fmsg = kk_failure_str(__FILE__, __FUNCTION__, __LINE__); Impl::BatchedDblBufGemm(batchedGemmHandle, alpha, a_actual, b_actual, beta, c_actual) .invoke(); FAIL() << (fmsg + fmsg_rhs); } catch (const std::runtime_error& error) { ; } } // Check for expected BatchedGemm runtime errors try { #if defined(KOKKOSKERNELS_ENABLE_TPL_ARMPL) if (algo_type == BaseTplAlgos::ARMPL && N % 2 == 0) { auto ninter = batchedGemmHandle->get_tpl_params(); // Set ninter parameter for underlying armpl_dgemm_interleave_batch call *ninter = N / 2; } #endif fmsg = kk_failure_str(__FILE__, __FUNCTION__, __LINE__); ret = BatchedGemm(batchedGemmHandle, alpha, a_actual, b_actual, beta, c_actual); // Compute c_actual } catch (const std::runtime_error& error) { std::string error_msg = error.what(); if (algo_type == BaseHeuristicAlgos::SQUARE && matCdim1 != matCdim2) { ; } else if (algo_type == BaseTplAlgos::ARMPL) { #if defined(KOKKOSKERNELS_ENABLE_TPL_ARMPL) && ARMPL_BUILD >= 1058 auto ninter = batchedGemmHandle->get_tpl_params()[0]; // No runtime errors expected since layout is valid, double is a supported // type, and ninter != 0 if (std::is_same::value && ninter != 0) { FAIL() << (error_msg + fmsg + fmsg_rhs); } #else ; // We expect a runtime error if the ARMPL TPL is not enabled #endif } else { FAIL() << (error_msg + fmsg + fmsg_rhs); } return; } ASSERT_EQ(ret, 0) << (fmsg + fmsg_rhs); Functor_BatchedVanillaGEMM vgemm; vgemm.A_t = std::is_same::value; vgemm.B_t = std::is_same::value; vgemm.batch_size_last_dim = std::is_same::value; vgemm.A_c = vgemm.B_c = false; vgemm.A = a_expected; vgemm.B = b_expected; vgemm.C = c_expected; vgemm.alpha = alpha; vgemm.beta = beta; vgemm.run(); // Compute c_expected Kokkos::fence(); typename ViewType::HostMirror c_expected_host = Kokkos::create_mirror_view(c_expected); typename ViewType::HostMirror c_actual_host = Kokkos::create_mirror_view(c_actual); // Copy to host Kokkos::deep_copy(c_expected_host, c_expected); Kokkos::deep_copy(c_actual_host, c_actual); Kokkos::fence(); // check c_expected = c_actual ; this eps is about 2^-9 // Set mag_type to host_value_type, we may not have half precision on host using mag_type = float; mag_type sum(1), diff(0); auto eps = static_cast(ats::epsilon()); eps *= std::is_same::value || std::is_same::value ? 4 : 1e3; for (int k = 0; k < N; ++k) { for (int i = 0; i < matCdim1; ++i) { for (int j = 0; j < matCdim2; ++j) { if (std::is_same::value) { sum += ats::abs(c_expected_host(i, j, k)); diff += ats::abs(c_expected_host(i, j, k) - c_actual_host(i, j, k)); } else { sum += ats::abs(c_expected_host(k, i, j)); diff += ats::abs(c_expected_host(k, i, j) - c_actual_host(k, i, j)); } } } } EXPECT_NEAR_KK(diff / sum, 0, eps, fmsg + fmsg_rhs); } template void impl_test_batched_gemm(const int N, const int matAdim1, const int matAdim2, const int matBdim1, const int matBdim2, const int matCdim1, const int matCdim2) { { BatchedGemmHandle batchedGemmHandle; ASSERT_EQ(batchedGemmHandle.get_kernel_algo_type(), BaseHeuristicAlgos::SQUARE); ASSERT_EQ(batchedGemmHandle.teamSz, 0); ASSERT_EQ(batchedGemmHandle.vecLen, 0); #if defined(KOKKOSKERNELS_ENABLE_TPL_CUBLAS) cublasHandle_t cublas_handle; BatchedGemmHandle batchedGemmHandleCublas(cublas_handle, GemmTplAlgos::CUBLAS, 0, 0); ASSERT_EQ(&cublas_handle, batchedGemmHandleCublas.get_tpl_params()); ASSERT_EQ(batchedGemmHandleCublas.get_kernel_algo_type(), (int)GemmTplAlgos::CUBLAS); ASSERT_EQ(batchedGemmHandleCublas.teamSz, 0); ASSERT_EQ(batchedGemmHandleCublas.vecLen, 0); #endif // FIXME temporary workaround to run this magma test only if cublas is not // enabled the design of the BatchedGemmHandle currently does not allow // simultanous testing in this way. See issue #2177 #if defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA) && !defined(KOKKOSKERNELS_ENABLE_TPL_CUBLAS) magma_queue_t magma_queue; BatchedGemmHandle batchedGemmHandleMagma(magma_queue, GemmTplAlgos::MAGMA, 0, 0); ASSERT_EQ(&magma_queue, batchedGemmHandleMagma.get_tpl_params()); ASSERT_EQ(batchedGemmHandleMagma.get_kernel_algo_type(), (int)GemmTplAlgos::MAGMA); ASSERT_EQ(batchedGemmHandleMagma.teamSz, 0); ASSERT_EQ(batchedGemmHandleMagma.vecLen, 0); #endif } for (int algo_type = BaseHeuristicAlgos::SQUARE; algo_type < GemmKokkosBatchedAlgos::N; ++algo_type) { { try { BatchedGemmHandle batchedGemmHandle(algo_type); ASSERT_EQ(batchedGemmHandle.get_kernel_algo_type(), algo_type); if (algo_type == BaseTplAlgos::ARMPL || algo_type == BaseKokkosBatchedAlgos::KK_SERIAL || algo_type == GemmKokkosBatchedAlgos::KK_SERIAL_RANK0 || algo_type == GemmKokkosBatchedAlgos::KK_DBLBUF) { impl_test_batched_gemm_with_handle( &batchedGemmHandle, N, matAdim1, matAdim2, matBdim1, matBdim2, matCdim1, matCdim2, 1.5, 3.0); } else if (algo_type == BaseHeuristicAlgos::SQUARE) { // Invoke 4 times to ensure we cover all paths for alpha and beta impl_test_batched_gemm_with_handle( &batchedGemmHandle, N, matAdim1, matAdim2, matBdim1, matBdim2, matCdim1, matCdim2, 0.0, 0.0); impl_test_batched_gemm_with_handle( &batchedGemmHandle, N, matAdim1, matAdim2, matBdim1, matBdim2, matCdim1, matCdim2, 1.0, 0.0); impl_test_batched_gemm_with_handle( &batchedGemmHandle, N, matAdim1, matAdim2, matBdim1, matBdim2, matCdim1, matCdim2, 0.0, 1.0); impl_test_batched_gemm_with_handle( &batchedGemmHandle, N, matAdim1, matAdim2, matBdim1, matBdim2, matCdim1, matCdim2, 1.5, 3.0); } else { try { // Allocate these views to invoke BatchedGemm with an unsupported // algo type ViewType a_actual("a_actual", N, matAdim1, matAdim2); ViewType b_actual("b_actual", N, matBdim1, matBdim2); ViewType c_actual("c_actual", N, matCdim1, matCdim2); using ta = typename ParamTagType::transA; using tb = typename ParamTagType::transB; using bl = typename ParamTagType::batchLayout; ScalarType alpha = 0.34; ScalarType beta = 0.43; BatchedGemm(&batchedGemmHandle, alpha, a_actual, b_actual, beta, c_actual); std::string fmsg = kk_failure_str(__FILE__, __FUNCTION__, __LINE__); FAIL() << fmsg; } catch (const std::runtime_error& error) { ; } } } catch (const std::runtime_error& error) { #if !defined(KOKKOSKERNELS_ENABLE_TPL_ARMPL) || (ARMPL_BUILD < 1058) if (algo_type == BaseTplAlgos::ARMPL) { ; } else { std::string fmsg = kk_failure_str(__FILE__, __FUNCTION__, __LINE__); FAIL() << fmsg; } #endif // KOKKOSKERNELS_ENABLE_TPL_ARMPL } } } } } // namespace Test template void test_batched_gemm_with_layout(int N) { // Square cases { int i = 0; Test::impl_test_batched_gemm(N, i, i, i, i, i, i); i = 10; Test::impl_test_batched_gemm(N, i, i, i, i, i, i); i = 25; Test::impl_test_batched_gemm(N, i, i, i, i, i, i); i = 32; Test::impl_test_batched_gemm(N, i, i, i, i, i, i); } // Non-square cases for (int i = 1; i < 5; ++i) { int dimM = 1 * i; int dimN = 2 * i; int dimK = 3 * i; if ((std::is_same::value) && (std::is_same::value)) { Test::impl_test_batched_gemm(N, dimM, dimK, dimK, dimN, dimM, dimN); } if ((std::is_same::value) && (std::is_same::value)) { Test::impl_test_batched_gemm(N, dimM, dimK, dimN, dimK, dimM, dimN); } if ((std::is_same::value) && (std::is_same::value)) { Test::impl_test_batched_gemm(N, dimK, dimM, dimK, dimN, dimM, dimN); } if ((std::is_same::value) && (std::is_same::value)) { Test::impl_test_batched_gemm(N, dimK, dimM, dimN, dimK, dimM, dimN); } } } template int test_batched_gemm() { #if defined(KOKKOSKERNELS_INST_LAYOUTLEFT) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) if constexpr (std::is_same_v) { using param_tag_type = ::Test::SharedParamTag; typedef Kokkos::View llVt; test_batched_gemm_with_layout(0); test_batched_gemm_with_layout(1); test_batched_gemm_with_layout(4); test_batched_gemm_with_layout(8); test_batched_gemm_with_layout(16); } else { std::cerr << "TEST SKIPPED since BatchLayout is not Right." << std::endl; } #else std::cerr << "TEST SKIPPED since LayoutLeft is not ETI'd." << std::endl; #endif // KOKKOSKERNELS_INST_LAYOUTLEFT #if defined(KOKKOSKERNELS_INST_LAYOUTRIGHT) || \ (!defined(KOKKOSKERNELS_ETI_ONLY) && !defined(KOKKOSKERNELS_IMPL_CHECK_ETI_CALLS)) if constexpr (std::is_same_v) { using param_tag_type = ::Test::SharedParamTag; typedef Kokkos::View lrVt; test_batched_gemm_with_layout(0); test_batched_gemm_with_layout(1); test_batched_gemm_with_layout(4); test_batched_gemm_with_layout(8); test_batched_gemm_with_layout(16); } else { std::cerr << "TEST SKIPPED since BatchLayout is not Left." << std::endl; } #else std::cerr << "TEST SKIPPED since LayoutRight is not ETI'd." << std::endl; #endif // KOKKOSKERNELS_INST_LAYOUTRIGHT return 0; }