Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Gemm_Team_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_GEMM_TEAM_INTERNAL_HPP
17 #define KOKKOSBATCHED_GEMM_TEAM_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22 #include "KokkosKernels_ExecSpaceUtils.hpp"
23
24 #include "KokkosBlas1_set_impl.hpp"
25 #include "KokkosBlas1_team_scal_impl.hpp"
26
27 #include "KokkosBatched_InnerGemmFixC_Serial_Impl.hpp"
28
29 namespace KokkosBatched {
30
31 ///
32 /// Team Internal Impl
33 /// ====================
34 template <typename ArgAlgo>
35 struct TeamGemmInternal {
36 template <typename MemberType, typename ScalarType, typename ValueType>
37 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const int m, const int n, const int k,
38 const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0,
39 const int as1, const ValueType *KOKKOS_RESTRICT B, const int bs0,
40 const int bs1, const ScalarType beta,
41 /**/ ValueType *KOKKOS_RESTRICT C, const int cs0, const int cs1);
42 };
43
44 template <>
45 template <typename MemberType, typename ScalarType, typename ValueType>
46 KOKKOS_INLINE_FUNCTION int TeamGemmInternal<Algo::Gemm::Unblocked>::invoke(
47 const MemberType &member, const int m, const int n, const int k, const ScalarType alpha,
48 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1, const ValueType *KOKKOS_RESTRICT B, const int bs0,
49 const int bs1, const ScalarType beta,
50 /**/ ValueType *KOKKOS_RESTRICT C, const int cs0, const int cs1) {
51 // C = beta C + alpha A B
52 // C (m x n), A(m x k), B(k x n)
53
54 const ScalarType one(1.0), zero(0.0);
55
56 if (beta == zero)
57 KokkosBlas::Impl::TeamSetInternal::invoke(member, m, n, zero, C, cs0, cs1);
58 else if (beta != one)
59 KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, beta, C, cs0, cs1);
60
61 if (alpha != ScalarType(0.0)) {
62 if (m <= 0 || n <= 0 || k <= 0) return 0;
63
64 if (beta != one) member.team_barrier();
65
66 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, 0, m * n), [&](const int &ij) {
67 // assume layout right for batched computation
68 const int i = ij / n, j = ij % n;
69 const ValueType *KOKKOS_RESTRICT pA = A + i * as0, *KOKKOS_RESTRICT pB = B + j * bs1;
70
71 ValueType c = ValueType(0);
72 for (int p = 0; p < k; ++p) c += pA[p * as1] * pB[p * bs0];
73 C[i * cs0 + j * cs1] += alpha * c;
74 });
75 }
76 return 0;
77 }
78
79 template <>
80 template <typename MemberType, typename ScalarType, typename ValueType>
81 KOKKOS_INLINE_FUNCTION int TeamGemmInternal<Algo::Gemm::Blocked>::invoke(
82 const MemberType &member, const int m, const int n, const int k, const ScalarType alpha,
83 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1, const ValueType *KOKKOS_RESTRICT B, const int bs0,
84 const int bs1, const ScalarType beta,
85 /**/ ValueType *KOKKOS_RESTRICT C, const int cs0, const int cs1) {
86 // C = beta C + alpha A B
87 // C (m x n), A(m x k), B(k x n)
88
89 constexpr int mbAlgo = Algo::Gemm::Blocked::mb();
90 constexpr int nbAlgo = Algo::Gemm::Blocked::mb();
91
92 const ScalarType one(1.0), zero(0.0);
93
94 if (beta == zero)
95 KokkosBlas::Impl::TeamSetInternal::invoke(member, m, n, zero, C, cs0, cs1);
96 else if (beta != one)
97 KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, beta, C, cs0, cs1);
98
99 if (alpha != ScalarType(0.0)) {
100 if (m <= 0 || n <= 0 || k <= 0) return 0;
101
102 if (beta != one) member.team_barrier();
103
104 ///
105 /// GPU case: team size is large and blocksize (mb,nb) is small
106 InnerGemmFixC<mbAlgo, nbAlgo> inner(as0, as1, bs0, bs1, cs0, cs1);
107 auto gemm = [&](const int ib, const int jb, const int pb, const ValueType *KOKKOS_RESTRICT AA,
108 const ValueType *KOKKOS_RESTRICT BB,
109 /**/ ValueType *KOKKOS_RESTRICT CC) {
110 // Made this non-const in order to WORKAROUND issue #349
111 int mb = mbAlgo, mp = (ib % mb), mq = (ib / mb) + (mp > 0), nb = nbAlgo, np = (jb % nb),
112 nq = (jb / nb) + (np > 0);
113
114 // square tiling
115 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, mq * nq), [&](const int &ij) {
116 int i, j;
117 // note: the condition is constexpr
118 if (KokkosKernels::Impl::is_gpu_exec_space_v<typename MemberType::execution_space>) {
119 i = ij % mq * mb;
120 j = ij / mq * nb;
121 } else {
122 i = ij / nq * mb;
123 j = ij % nq * nb;
124 }
125 inner.serial_invoke(alpha, AA + i * as0, BB + j * bs1, (i + mb) > ib ? mp : mb, (j + nb) > jb ? np : nb, pb,
126 CC + i * cs0 + j * cs1);
127 });
128 };
129
130 const bool is_small = true; //(m*n*k <= 64*64*64);
131 if (is_small) {
132 gemm(m, n, k, A, B, C);
133 } else {
134 // // cache blocking
135 // const int
136 // nc = nb*10, kc = mb*4, mc = mb*4;
137
138 // for (int jj=0;jj<n;jj+=nc) {
139 // const int tj = n-jj, jb = (tj < nc ? tj : nc);
140 // for (int pp=0;pp<k;pp+=kc) {
141 // const int tp = k-pp, pb = (tp < kc ? tp : kc);
142 // //const int pb = k, pp = 0;
143 // for (int ii=0;ii<m;ii+=mc) {
144 // const int ti = m-ii, ib = (ti < mc ? ti : mc);
145
146 // const ValueType *KOKKOS_RESTRICT AA = A+ii*as0+pp*as1;
147 // const ValueType *KOKKOS_RESTRICT BB = B+pp*bs0+jj*bs1;
148 // /**/ ValueType *KOKKOS_RESTRICT CC = C+ii*cs0+jj*cs1;
149
150 // gemm(ib, jb, pb, AA, BB, CC);
151 // } // for ii
152 // } // for pp
153 // } // for jj
154 }
155 }
156 return 0;
157 }
158
159 } // namespace KokkosBatched
160
161 #endif