Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_HostLevel_Gemm_Serial_Impl.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_HOSTLEVEL_GEMM_SERIAL_IMPL_HPP
17 #define KOKKOSBATCHED_HOSTLEVEL_GEMM_SERIAL_IMPL_HPP
18 #include "KokkosBatched_Gemm_Decl.hpp"
19
20 namespace KokkosBatched {
21 namespace Impl {
22 // clang-format off
23 /// \brief Non-blocking general matrix multiply on a batch of
24 /// uniform matrices.
25 ///
26 ///
27 /// C = alpha * op(A) * op(B) + beta * C
28 ///
29 /// \tparam ArgTransA Specifies what op does to A:
30 /// Trans::NoTranspose for non-transpose
31 /// Trans::Transpose for transpose
32 /// Trans::ConjTranspose for conjugate transpose
33 /// \tparam ArgTransB Specifies what op does to B:
34 /// Trans::NoTranspose for non-transpose
35 /// Trans::Transpose for transpose
36 /// Trans::ConjTranspose for conjugate transpose
37 /// \tparam ArgMode Specifies algorithm mode to use for serial work:
38 /// Algo::Gemm::Unblocked for no register blocking
39 /// Algo::Gemm::Blocked for register blocking
40 /// Algo::Gemm::CompactMKL for mkl compact tpl interface
41 /// \tparam ArgBatchSzDim Specifies where the batch dimension is allocated in
42 /// AViewType, BViewType, and CViewType:
43 /// BatchSzDim::Left Batch dimension is leftmost
44 /// BatchSzDim::Right Batch dimension is rightmost
45 /// \tparam ArgResultsPerThread Specifies how to divide work among threads. For
46 /// this serial interface, each rank specifies how
47 /// much work to assign a single thread.
48 /// ResultsPerThread::Rank0 Each thread computes a scalar of C
49 /// ResultsPerThread::Rank1 Each thread computes a 1-rank chunk of C
50 /// ResultsPerThread::Rank2 Each thread computes a 2-rank chunk of C
51 /// \tparam ScalarType Specifies the scalar type of alpha and beta
52 /// \tparam AViewType Input matrix, as either a 3-rank Kokkos::View or a
53 /// 4-rank Kokkos::View for SIMD operations.
54 /// \tparam BViewType Input matrix, as either a 3-rank Kokkos::View or a
55 /// 4-rank Kokkos::View for SIMD operations.
56 /// \tparam CViewType Input(RHS)/Output(LHS) matrix, as either a 3-rank
57 /// Kokkos::View or a 4-rank Kokkos::View for SIMD
58 /// operations.
59 ///
60 /// See struct BatchedGemmHandle for details.
61 /// \param alpha [in] Input coefficient used for multiplication with A
62 /// \param A [in] Input matrix, as a 3-rank Kokkos::View
63 /// If ArgBatchSzDim == "BatchSzDim::Right", matrix A is MxKxB
64 /// If ArgBatchSzDim == "BatchSzDim::Left", matrix A is BxMxK
65 /// \param B [in] Input matrix, as a 3-rank Kokkos::View
66 /// If ArgBatchSzDim == "BatchSzDim::Right", matrix B is KxNxB
67 /// If ArgBatchSzDim == "BatchSzDim::Left", matrix B is BxKxN
68 /// \param beta [in] Input coefficient used for multiplication with C
69 /// \param C [in/out] Input/Output matrix, as a 3-rank Kokkos::View
70 /// If ArgBatchSzDim == "BatchSzDim::Right", matrix C is MxNxB
71 /// If ArgBatchSzDim == "BatchSzDim::Left", matrix C is BxMxN
72 /// \return 0 upon success, non-zero otherwise
73 ///
74 /// Usage Example:
75 /// BatchedSerialGemm<ArgTransA, ArgTransB, ArgMode, ArgBatchSzDim,
76 /// ArgResultsPerThread, ScalarType, AViewType,
77 /// BViewType, CViewType>(alpha, A, B, beta, C).invoke();
78 // clang-format on
79 template <class ArgTransA, class ArgTransB, class ArgMode, class ArgBatchSzDim, class ArgResultsPerThread,
80 class ScalarType, class AViewType, class BViewType, class CViewType>
81 class BatchedSerialGemm {
82 private:
83 AViewType A;
84 BViewType B;
85 CViewType C;
86 ScalarType alpha, beta;
87 size_t divisor, c_cols, batch_size;
88 ArgBatchSzDim batch_layout_tag;
89 ArgTransA transA_tag;
90 ArgTransB transB_tag;
91
92 void run() {
93 using execution_space = typename CViewType::device_type::execution_space;
94 using policy_type = Kokkos::RangePolicy<ArgResultsPerThread, execution_space>;
95 Kokkos::parallel_for("BatchedSerialGemm", policy_type(0, batch_size), *this);
96 }
97
98 public:
99 int invoke() {
100 if (std::is_same<ArgResultsPerThread, ResultsPerThread::Rank0>::value) {
101 // Set members for ResultsPerThread::Rank0 operator; these members allow
102 // each thread to calculate its C output index
103 if (std::is_same<ArgBatchSzDim, BatchLayout::Left>::value) {
104 batch_size = C.extent(0);
105 divisor = C.extent(1) * C.extent(2);
106 c_cols = C.extent(2);
107 } else {
108 batch_size = C.extent(2);
109 divisor = C.extent(0) * C.extent(1);
110 c_cols = C.extent(1);
111 }
112
113 // Increase the number of threads by the divisor
114 batch_size *= divisor;
115
116 run();
117 } else if (std::is_same<ArgResultsPerThread, ResultsPerThread::Rank2>::value) {
118 if (std::is_same<ArgBatchSzDim, BatchLayout::Left>::value)
119 batch_size = C.extent(0);
120 else
121 batch_size = C.extent(2);
122
123 run();
124 } else {
125 std::cerr << "Error: ArgResultsPerThread not supported" << std::endl;
126 return -1;
127 }
128 return 0;
129 }
130
131 BatchedSerialGemm(ScalarType _alpha, AViewType _A, BViewType _B, ScalarType _beta, CViewType _C)
132 : A(_A), B(_B), C(_C), alpha(_alpha), beta(_beta) {}
133
134 KOKKOS_INLINE_FUNCTION
135 void operator()(const ResultsPerThread::Rank0 &, const int &i) const {
136 // Here, the batch_idx is strided by c_rows * c_cols
137 auto batch_idx = i / divisor;
138 // For every batch, we need mod in [0, c_rows*c_cols-1]
139 auto mod = i % divisor;
140 // For every mod, we need a column index in [0, c_cols-1]
141 auto col_idx = mod % c_cols;
142 // For every mod, we need a row index in [0, c_rows-1]
143 auto row_idx = mod / c_cols;
144
145 // Due to taking 1-rank subviews out, we must handle transpose here.
146 // Use overloads of subview_wrapper to handle transpose at compile time.
147 auto svA_row = subview_wrapper(A, batch_idx, row_idx, Kokkos::ALL(), batch_layout_tag, transA_tag);
148 auto svB_col = subview_wrapper(B, batch_idx, Kokkos::ALL(), col_idx, batch_layout_tag, transB_tag);
149 auto svC_ele = subview_wrapper(C, batch_idx, row_idx, col_idx, batch_layout_tag);
150
151 // Kokkos::subview(scalar, ALL) or Kokkos::subview(ALL, scalar) always
152 // returns a column vector. Since the subviews above handle the
153 // matrix transpositions, here we must perform the GEMM on:
154 // row_vec x col_vec, which is svA_row' x svB_col to compute the element
155 // of C.
156 KokkosBatched::SerialGemm<Trans::Transpose, Trans::NoTranspose, ArgMode>::invoke(alpha, svA_row, svB_col, beta,
157 svC_ele);
158 }
159
160 KOKKOS_INLINE_FUNCTION
161 void operator()(const ResultsPerThread::Rank2 &, const int &i) const {
162 auto svA = subview_wrapper(A, i, Kokkos::ALL(), Kokkos::ALL(), batch_layout_tag);
163 auto svB = subview_wrapper(B, i, Kokkos::ALL(), Kokkos::ALL(), batch_layout_tag);
164 auto svC = subview_wrapper(C, i, Kokkos::ALL(), Kokkos::ALL(), batch_layout_tag);
165
166 KokkosBatched::SerialGemm<ArgTransA, ArgTransB, ArgMode>::invoke(alpha, svA, svB, beta, svC);
167 }
168 };
169 } // namespace Impl
170 } // namespace KokkosBatched
171 #endif