Debian Codegraph @

packages/kokkos-kernels/batched/dense/src/KokkosBatched_HostLevel_Gemm_Handle.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16
17 #ifndef KOKKOSBATCHED_HOSTLEVEL_GEMM_HANDLE_DECL_HPP
18 #define KOKKOSBATCHED_HOSTLEVEL_GEMM_HANDLE_DECL_HPP
19
20 #include "KokkosBatched_Kernel_Handle.hpp"
21
22 namespace KokkosBatched {
23
24 /// \brief Tpl algorithm types. See BatchedGemmHandle for details.
25 namespace GemmTplAlgos {
26 enum GEMM_TPL_ALGOS : int { CUBLAS = N_BASE_ALGOS, MAGMA, N };
27 }
28
29 /// \brief KokkosBatched algorithm types. See BatchedGemmHandle for details.
30 namespace GemmKokkosBatchedAlgos {
31 enum GEMM_KOKKOS_BATCHED_ALGOS : int {
32 KK_TEAM = GemmTplAlgos::N,
33 KK_TEAMVECTOR,
34 KK_SERIALSIMD,
35 KK_TEAMSIMD,
36 KK_SERIAL_RANK0,
37 KK_SERIAL_SHMEM,
38 KK_DBLBUF,
39 N
40 };
41 }
42
43 #define GEMM_ALGO_STRS \
44 "GemmTplAlgos::CUBLAS", "GemmTplAlgos::MAGMA", "GemmKokkosBatchedAlgos::KK_TEAM", \
45 "GemmKokkosBatchedAlgos::KK_TEAMVECTOR", "GemmKokkosBatchedAlgos::KK_SERIALSIMD", \
46 "GemmKokkosBatchedAlgos::KK_TEAMSIMD", "GemmKokkosBatchedAlgos::KK_SERIAL_RANK0", \
47 "GemmKokkosBatchedAlgos::KK_SERIAL_SHMEM", "GemmKokkosBatchedAlgos::KK_DBLBUF"
48 // clang-format off
49 /// \brief Handle for selecting runtime behavior of the BatchedGemm interface.
50 ///
51 /// \param kernelAlgoType Specifies which algorithm to use for invocation (default, SQUARE).
52 ///
53 /// Specifies whether to select optimal invocations based on inputs and
54 /// heuristics:
55 /// SQUARE select invocations based on square matrix heuristics where M=N
56 /// TALL select invocations based on tall matrix heuristics where M>N
57 /// WIDE select invocations based on wide matrix heuristics where M<N
58 ///
59 /// Specifies which cmake-enabled TPL algorithm to invoke:
60 /// ARMPL Invoke the ArmPL TPL interface (Currently UNSUPPORTED)
61 /// MKL Invoke the MKL TPL interface (Currently UNSUPPORTED)
62 /// CUBLAS Invoke the CuBLAS TPL interface (Currently UNSUPPORTED)
63 /// MAGMA Invoke the Magma TPL interface (Currently UNSUPPORTED)
64 /// Note: Requires that input views for A, B, and C reside on either host
65 /// or device depending on the TPL selected.
66 /// Note: If the user selects a TPL, an error will be thrown if:
67 /// 1. The TPL is not enabled via cmake
68 /// 2. The input views do not reside on the host/device as needed
69 ///
70 /// Specifies which kokkos-kernels (KK) algorithm to invoke:
71 /// KK_SERIAL Invoke SerialGemm via RangePolicy(BatchSz)
72 /// KK_TEAM Invoke TeamGemm via TeamPolicy(BatchSz)
73 /// KK_TEAMVECTOR Invoke TeamVectorGemm via TeamPolicy(BatchSz)
74 /// KK_SERIALSIMD Invoke SerialGemm via TeamPolicy(BatchSz)
75 /// KK_TEAMSIMD Invoke TeamGemm via TeamPolicy(BatchSz)
76 /// KK_SERIAL_RANK0 Invoke SerialGemm via RangePolicy(BatchSz*N*M)
77 /// Each thread computes one element of C.
78 /// KK_SERIAL_SHMEM Invoke SerialGemm via TeamPolicy(BatchSz)
79 /// Copies A and B to shared memory before GEMM.
80 /// Each vector lane solves one element of C via SerialGemm.
81 /// KK_DBLBUF Solve GEMM via TeamPolicy(BatchSz*TILES)
82 /// Uses a tuned functor with tiling and double buffering
83 /// via shared memory and register buffers.
84 /// KK_DBLBUF generally performs better on GPUs when M, N >= 24.
85 /// \param teamSz Specifies the team size that will affect any KK algorithm which uses
86 /// TeamPolicy (default, Kokkos::AUTO).
87 /// Note: Only applied if useAlgo_type == KK_*
88 /// \param vecLen Specifies the vector length that will affect any KK algorithm which
89 /// uses TeamPolicy and Kokkos::ThreadVectorRange or Kokkos::TeamVectorRange
90 /// (default, Kokkos::AUTO).
91 /// Note: Only applied if useAlgo_type == KK_*
92 // clang-format on
93 class BatchedGemmHandle : public BatchedKernelHandle {
94 public:
95 BatchedGemmHandle(int kernelAlgoType = BaseHeuristicAlgos::SQUARE, int teamSize = 0, int vecLength = 0)
96 : BatchedKernelHandle(kernelAlgoType, teamSize, vecLength) {
97 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUBLAS)
98 if (!_tplParamsSet && kernelAlgoType == GemmTplAlgos::CUBLAS) {
99 static cublasHandle_t cublas_handle;
100 _tplParamsSingleton.cublas_handle = &cublas_handle;
101 _tplParamsSet = true;
102 }
103 #endif // CUBLAS
104
105 #if defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA)
106 if (!_tplParamsSet && kernelAlgoType == GemmTplAlgos::MAGMA) {
107 static magma_queue_t magma_queue;
108 _tplParamsSingleton.magma_queue = &magma_queue;
109 _tplParamsSet = true;
110 }
111 #endif // MAGMA
112 };
113
114 BatchedGemmHandle(bool tplParamsSet, int kernelAlgoType = BaseHeuristicAlgos::SQUARE, int teamSize = 0,
115 int vecLength = 0)
116 : BatchedKernelHandle(kernelAlgoType, teamSize, vecLength) {
117 _tplParamsSet = tplParamsSet;
118 };
119
120 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUBLAS)
121 BatchedGemmHandle(cublasHandle_t &cublas_handle, int kernelAlgoType = BaseHeuristicAlgos::SQUARE, int teamSize = 0,
122 int vecLength = 0)
123 : BatchedGemmHandle(true, kernelAlgoType, teamSize, vecLength) {
124 _tplParamsSingleton.cublas_handle = &cublas_handle;
125 };
126 #endif // CUBLAS
127
128 #if defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA)
129 BatchedGemmHandle(magma_queue_t &magma_queue, int kernelAlgoType = BaseHeuristicAlgos::SQUARE, int teamSize = 0,
130 int vecLength = 0)
131 : BatchedGemmHandle(true, kernelAlgoType, teamSize, vecLength) {
132 _tplParamsSingleton.magma_queue = &magma_queue;
133 };
134 #endif // MAGMA
135
136 decltype(auto) get_tpl_params() {
137 #if _kernelAlgoType == CUBLAS && defined(KOKKOSKERNELS_ENABLE_TPL_CUBLAS)
138 return _tplParamsSingleton.cublas_handle;
139 #elif _kernelAlgoType == MAGMA && defined(KOKKOSKERNELS_ENABLE_TPL_MAGMA)
140 return _tplParamsSingleton.magma_queue;
141 #else
142 return this->BatchedKernelHandle::get_tpl_params();
143 #endif
144 }
145
146 std::string get_kernel_algo_type_str() const { return gemm_algo_type_strs[_kernelAlgoType]; }
147
148 private:
149 const char *gemm_algo_type_strs[GemmKokkosBatchedAlgos::N] = {BASE_ALGO_STRS, GEMM_ALGO_STRS};
150 };
151
152 } // namespace KokkosBatched
153
154 #endif // KOKKOSBATCHED_HOSTLEVEL_GEMM_HANDLE_DECL_HPP