Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Gemm_Serial_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_GEMM_SERIAL_INTERNAL_HPP
17 #define KOKKOSBATCHED_GEMM_SERIAL_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22
23 #include "KokkosBlas1_set_impl.hpp"
24 #include "KokkosBlas1_serial_scal_impl.hpp"
25
26 #include "KokkosBatched_InnerGemmFixC_Serial_Impl.hpp"
27
28 namespace KokkosBatched {
29
30 ///
31 /// Serial Internal Impl
32 /// ====================
33
34 template <typename ArgAlgo>
35 struct SerialGemmInternal {
36 template <typename ScalarType, typename ValueType>
37 KOKKOS_INLINE_FUNCTION static int invoke(const int m, const int n, const int k, const ScalarType alpha,
38 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1,
39 const ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1,
40 const ScalarType beta,
41 /**/ ValueType *KOKKOS_RESTRICT C, const int cs0, const int cs1);
42 };
43
44 template <>
45 template <typename ScalarType, typename ValueType>
46 KOKKOS_INLINE_FUNCTION int SerialGemmInternal<Algo::Gemm::Unblocked>::invoke(
47 const int m, const int n, const int k, const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0,
48 const int as1, const ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1, const ScalarType beta,
49 /**/ ValueType *KOKKOS_RESTRICT C, const int cs0, const int cs1) {
50 // C = beta C + alpha A B
51 // C (m x n), A(m x k), B(k x n)
52
53 const ScalarType one(1.0), zero(0.0);
54
55 if (beta == zero)
56 KokkosBlas::Impl::SerialSetInternal::invoke(m, n, zero, C, cs0, cs1);
57 else if (beta != one)
58 KokkosBlas::Impl::SerialScaleInternal::invoke(m, n, beta, C, cs0, cs1);
59
60 if (alpha != zero) {
61 if (m <= 0 || n <= 0 || k <= 0) return 0;
62
63 ValueType *KOKKOS_RESTRICT pC = C;
64 for (int p = 0; p < k; ++p) {
65 const ValueType *KOKKOS_RESTRICT pA = A + p * as1, *KOKKOS_RESTRICT pB = B + p * bs0;
66 for (int i = 0; i < m; ++i) {
67 const ValueType tA(alpha * pA[i * as0]);
68 #if defined(KOKKOS_ENABLE_PRAGMA_UNROLL)
69 #pragma unroll
70 #endif
71 for (int j = 0; j < n; ++j) pC[i * cs0 + j * cs1] += tA * pB[j * bs1];
72 }
73 }
74 }
75 return 0;
76 }
77
78 template <>
79 template <typename ScalarType, typename ValueType>
80 KOKKOS_INLINE_FUNCTION int SerialGemmInternal<Algo::Gemm::Blocked>::invoke(
81 const int m, const int n, const int k, const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0,
82 const int as1, const ValueType *KOKKOS_RESTRICT B, const int bs0, const int bs1, const ScalarType beta,
83 /**/ ValueType *KOKKOS_RESTRICT C, const int cs0, const int cs1) {
84 // C = beta C + alpha A B
85 // C (m x n), A(m x k), B(k x n)
86
87 constexpr int mbAlgo = Algo::Gemm::Blocked::mb();
88 constexpr int nbAlgo = Algo::Gemm::Blocked::mb();
89
90 const ScalarType one(1.0), zero(0.0);
91
92 if (beta == zero)
93 KokkosBlas::Impl::SerialSetInternal::invoke(m, n, zero, C, cs0, cs1);
94 else if (beta != one)
95 KokkosBlas::Impl::SerialScaleInternal::invoke(m, n, beta, C, cs0, cs1);
96
97 if (alpha != zero) {
98 if (m <= 0 || n <= 0 || k <= 0) return 0;
99 const ValueType alpha_value(alpha);
100
101 InnerGemmFixC<mbAlgo, nbAlgo> inner(as0, as1, bs0, bs1, cs0, cs1);
102 auto gemm = [&](const int ib, const int jb, const int pb, const ValueType *KOKKOS_RESTRICT AA,
103 const ValueType *KOKKOS_RESTRICT BB,
104 /**/ ValueType *KOKKOS_RESTRICT CC) {
105 const int mb = mbAlgo, nb = nbAlgo;
106 for (int i = 0; i < ib; i += mb)
107 for (int j = 0; j < jb; j += nb)
108 inner.serial_invoke(alpha_value, AA + i * as0, BB + j * bs1, (i + mb) > ib ? (ib - i) : mb,
109 (j + nb) > jb ? (jb - j) : nb, pb, CC + i * cs0 + j * cs1);
110 };
111
112 const bool is_small = true; //(m*n*k <= 64*64*64);
113 if (is_small) {
114 gemm(m, n, k, A, B, C);
115 } else {
116 // // cache blocking
117 // const int
118 // nc = nb*10, kc = mb*4, mc = mb*4;
119
120 // for (int jj=0;jj<n;jj+=nc) {
121 // const int tj = n-jj, jb = (tj < nc ? tj : nc);
122 // for (int pp=0;pp<k;pp+=kc) {
123 // const int tp = k-pp, pb = (tp < kc ? tp : kc);
124 // //const int pb = k, pp = 0;
125 // for (int ii=0;ii<m;ii+=mc) {
126 // const int ti = m-ii, ib = (ti < mc ? ti : mc);
127
128 // const ValueType *KOKKOS_RESTRICT AA = A+ii*as0+pp*as1;
129 // const ValueType *KOKKOS_RESTRICT BB = B+pp*bs0+jj*bs1;
130 // /**/ ValueType *KOKKOS_RESTRICT CC = C+ii*cs0+jj*cs1;
131
132 // gemm(ib, jb, pb, AA, BB, CC);
133 // } // for ii
134 // } // for pp
135 // } // for jj
136 }
137 }
138 return 0;
139 }
140
141 } // namespace KokkosBatched
142
143 #endif