| 1 |
//@HEADER |
| 2 |
// ************************************************************************ |
| 3 |
// |
| 4 |
// Kokkos v. 4.0 |
| 5 |
// Copyright (2022) National Technology & Engineering |
| 6 |
// Solutions of Sandia, LLC (NTESS). |
| 7 |
// |
| 8 |
// Under the terms of Contract DE-NA0003525 with NTESS, |
| 9 |
// the U.S. Government retains certain rights in this software. |
| 10 |
// |
| 11 |
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. |
| 12 |
// See https://kokkos.org/LICENSE for license information. |
| 13 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 14 |
// |
| 15 |
//@HEADER |
| 16 |
#ifndef KOKKOSBATCHED_HOSTLEVEL_GEMM_SERIAL_IMPL_HPP |
| 17 |
#define KOKKOSBATCHED_HOSTLEVEL_GEMM_SERIAL_IMPL_HPP |
| 18 |
#include "KokkosBatched_Gemm_Decl.hpp" |
| 19 |
|
| 20 |
namespace KokkosBatched { |
| 21 |
namespace Impl { |
| 22 |
// clang-format off |
| 23 |
/// \brief Non-blocking general matrix multiply on a batch of |
| 24 |
/// uniform matrices. |
| 25 |
/// |
| 26 |
/// |
| 27 |
/// C = alpha * op(A) * op(B) + beta * C |
| 28 |
/// |
| 29 |
/// \tparam ArgTransA Specifies what op does to A: |
| 30 |
/// Trans::NoTranspose for non-transpose |
| 31 |
/// Trans::Transpose for transpose |
| 32 |
/// Trans::ConjTranspose for conjugate transpose |
| 33 |
/// \tparam ArgTransB Specifies what op does to B: |
| 34 |
/// Trans::NoTranspose for non-transpose |
| 35 |
/// Trans::Transpose for transpose |
| 36 |
/// Trans::ConjTranspose for conjugate transpose |
| 37 |
/// \tparam ArgMode Specifies algorithm mode to use for serial work: |
| 38 |
/// Algo::Gemm::Unblocked for no register blocking |
| 39 |
/// Algo::Gemm::Blocked for register blocking |
| 40 |
/// Algo::Gemm::CompactMKL for mkl compact tpl interface |
| 41 |
/// \tparam ArgBatchSzDim Specifies where the batch dimension is allocated in |
| 42 |
/// AViewType, BViewType, and CViewType: |
| 43 |
/// BatchSzDim::Left Batch dimension is leftmost |
| 44 |
/// BatchSzDim::Right Batch dimension is rightmost |
| 45 |
/// \tparam ArgResultsPerThread Specifies how to divide work among threads. For |
| 46 |
/// this serial interface, each rank specifies how |
| 47 |
/// much work to assign a single thread. |
| 48 |
/// ResultsPerThread::Rank0 Each thread computes a scalar of C |
| 49 |
/// ResultsPerThread::Rank1 Each thread computes a 1-rank chunk of C |
| 50 |
/// ResultsPerThread::Rank2 Each thread computes a 2-rank chunk of C |
| 51 |
/// \tparam ScalarType Specifies the scalar type of alpha and beta |
| 52 |
/// \tparam AViewType Input matrix, as either a 3-rank Kokkos::View or a |
| 53 |
/// 4-rank Kokkos::View for SIMD operations. |
| 54 |
/// \tparam BViewType Input matrix, as either a 3-rank Kokkos::View or a |
| 55 |
/// 4-rank Kokkos::View for SIMD operations. |
| 56 |
/// \tparam CViewType Input(RHS)/Output(LHS) matrix, as either a 3-rank |
| 57 |
/// Kokkos::View or a 4-rank Kokkos::View for SIMD |
| 58 |
/// operations. |
| 59 |
/// |
| 60 |
/// See struct BatchedGemmHandle for details. |
| 61 |
/// \param alpha [in] Input coefficient used for multiplication with A |
| 62 |
/// \param A [in] Input matrix, as a 3-rank Kokkos::View |
| 63 |
/// If ArgBatchSzDim == "BatchSzDim::Right", matrix A is MxKxB |
| 64 |
/// If ArgBatchSzDim == "BatchSzDim::Left", matrix A is BxMxK |
| 65 |
/// \param B [in] Input matrix, as a 3-rank Kokkos::View |
| 66 |
/// If ArgBatchSzDim == "BatchSzDim::Right", matrix B is KxNxB |
| 67 |
/// If ArgBatchSzDim == "BatchSzDim::Left", matrix B is BxKxN |
| 68 |
/// \param beta [in] Input coefficient used for multiplication with C |
| 69 |
/// \param C [in/out] Input/Output matrix, as a 3-rank Kokkos::View |
| 70 |
/// If ArgBatchSzDim == "BatchSzDim::Right", matrix C is MxNxB |
| 71 |
/// If ArgBatchSzDim == "BatchSzDim::Left", matrix C is BxMxN |
| 72 |
/// \return 0 upon success, non-zero otherwise |
| 73 |
/// |
| 74 |
/// Usage Example: |
| 75 |
/// BatchedSerialGemm<ArgTransA, ArgTransB, ArgMode, ArgBatchSzDim, |
| 76 |
/// ArgResultsPerThread, ScalarType, AViewType, |
| 77 |
/// BViewType, CViewType>(alpha, A, B, beta, C).invoke(); |
| 78 |
// clang-format on |
| 79 |
template <class ArgTransA, class ArgTransB, class ArgMode, class ArgBatchSzDim, class ArgResultsPerThread, |
| 80 |
class ScalarType, class AViewType, class BViewType, class CViewType> |
| 81 |
class BatchedSerialGemm { |
| 82 |
private: |
| 83 |
AViewType A; |
| 84 |
BViewType B; |
| 85 |
CViewType C; |
| 86 |
ScalarType alpha, beta; |
| 87 |
size_t divisor, c_cols, batch_size; |
| 88 |
ArgBatchSzDim batch_layout_tag; |
| 89 |
ArgTransA transA_tag; |
| 90 |
ArgTransB transB_tag; |
| 91 |
|
| 92 |
void run() { |
| 93 |
using execution_space = typename CViewType::device_type::execution_space; |
| 94 |
using policy_type = Kokkos::RangePolicy<ArgResultsPerThread, execution_space>; |
| 95 |
Kokkos::parallel_for("BatchedSerialGemm", policy_type(0, batch_size), *this); |
| 96 |
} |
| 97 |
|
| 98 |
public: |
| 99 |
int invoke() { |
| 100 |
if (std::is_same<ArgResultsPerThread, ResultsPerThread::Rank0>::value) { |
| 101 |
// Set members for ResultsPerThread::Rank0 operator; these members allow |
| 102 |
// each thread to calculate its C output index |
| 103 |
if (std::is_same<ArgBatchSzDim, BatchLayout::Left>::value) { |
| 104 |
batch_size = C.extent(0); |
| 105 |
divisor = C.extent(1) * C.extent(2); |
| 106 |
c_cols = C.extent(2); |
| 107 |
} else { |
| 108 |
batch_size = C.extent(2); |
| 109 |
divisor = C.extent(0) * C.extent(1); |
| 110 |
c_cols = C.extent(1); |
| 111 |
} |
| 112 |
|
| 113 |
// Increase the number of threads by the divisor |
| 114 |
batch_size *= divisor; |
| 115 |
|
| 116 |
run(); |
| 117 |
} else if (std::is_same<ArgResultsPerThread, ResultsPerThread::Rank2>::value) { |
| 118 |
if (std::is_same<ArgBatchSzDim, BatchLayout::Left>::value) |
| 119 |
batch_size = C.extent(0); |
| 120 |
else |
| 121 |
batch_size = C.extent(2); |
| 122 |
|
| 123 |
run(); |
| 124 |
} else { |
| 125 |
std::cerr << "Error: ArgResultsPerThread not supported" << std::endl; |
| 126 |
return -1; |
| 127 |
} |
| 128 |
return 0; |
| 129 |
} |
| 130 |
|
| 131 |
BatchedSerialGemm(ScalarType _alpha, AViewType _A, BViewType _B, ScalarType _beta, CViewType _C) |
| 132 |
: A(_A), B(_B), C(_C), alpha(_alpha), beta(_beta) {} |
| 133 |
|
| 134 |
KOKKOS_INLINE_FUNCTION |
| 135 |
void operator()(const ResultsPerThread::Rank0 &, const int &i) const { |
| 136 |
// Here, the batch_idx is strided by c_rows * c_cols |
| 137 |
auto batch_idx = i / divisor; |
| 138 |
// For every batch, we need mod in [0, c_rows*c_cols-1] |
| 139 |
auto mod = i % divisor; |
| 140 |
// For every mod, we need a column index in [0, c_cols-1] |
| 141 |
auto col_idx = mod % c_cols; |
| 142 |
// For every mod, we need a row index in [0, c_rows-1] |
| 143 |
auto row_idx = mod / c_cols; |
| 144 |
|
| 145 |
// Due to taking 1-rank subviews out, we must handle transpose here. |
| 146 |
// Use overloads of subview_wrapper to handle transpose at compile time. |
| 147 |
auto svA_row = subview_wrapper(A, batch_idx, row_idx, Kokkos::ALL(), batch_layout_tag, transA_tag); |
| 148 |
auto svB_col = subview_wrapper(B, batch_idx, Kokkos::ALL(), col_idx, batch_layout_tag, transB_tag); |
| 149 |
auto svC_ele = subview_wrapper(C, batch_idx, row_idx, col_idx, batch_layout_tag); |
| 150 |
|
| 151 |
// Kokkos::subview(scalar, ALL) or Kokkos::subview(ALL, scalar) always |
| 152 |
// returns a column vector. Since the subviews above handle the |
| 153 |
// matrix transpositions, here we must perform the GEMM on: |
| 154 |
// row_vec x col_vec, which is svA_row' x svB_col to compute the element |
| 155 |
// of C. |
| 156 |
KokkosBatched::SerialGemm<Trans::Transpose, Trans::NoTranspose, ArgMode>::invoke(alpha, svA_row, svB_col, beta, |
| 157 |
svC_ele); |
| 158 |
} |
| 159 |
|
| 160 |
KOKKOS_INLINE_FUNCTION |
| 161 |
void operator()(const ResultsPerThread::Rank2 &, const int &i) const { |
| 162 |
auto svA = subview_wrapper(A, i, Kokkos::ALL(), Kokkos::ALL(), batch_layout_tag); |
| 163 |
auto svB = subview_wrapper(B, i, Kokkos::ALL(), Kokkos::ALL(), batch_layout_tag); |
| 164 |
auto svC = subview_wrapper(C, i, Kokkos::ALL(), Kokkos::ALL(), batch_layout_tag); |
| 165 |
|
| 166 |
KokkosBatched::SerialGemm<ArgTransA, ArgTransB, ArgMode>::invoke(alpha, svA, svB, beta, svC); |
| 167 |
} |
| 168 |
}; |
| 169 |
} // namespace Impl |
| 170 |
} // namespace KokkosBatched |
| 171 |
#endif |