Debian Codegraph @

packages/kokkos-kernels/batched/dense/impl/KokkosBatched_Gemm_TeamVector_Internal.hpp

raw
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 #ifndef KOKKOSBATCHED_GEMM_TEAMVECTOR_INTERNAL_HPP
17 #define KOKKOSBATCHED_GEMM_TEAMVECTOR_INTERNAL_HPP
18
19 /// \author Kyungjoo Kim ([email protected])
20
21 #include "KokkosBatched_Util.hpp"
22
23 #include "KokkosBlas1_set_impl.hpp"
24 #include "KokkosBlas1_team_scal_impl.hpp"
25
26 namespace KokkosBatched {
27
28 ///
29 /// TeamVector Internal Impl
30 /// ====================
31 template <typename ArgAlgo, bool useConjA = false>
32 struct TeamVectorGemmInternal {
33 template <typename MemberType, typename ScalarType, typename ValueType>
34 KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const int m, const int n, const int k,
35 const ScalarType alpha, const ValueType *KOKKOS_RESTRICT A, const int as0,
36 const int as1, const ValueType *KOKKOS_RESTRICT B, const int bs0,
37 const int bs1, const ScalarType beta,
38 /**/ ValueType *KOKKOS_RESTRICT C, const int cs0, const int cs1);
39 };
40
41 template <>
42 template <typename MemberType, typename ScalarType, typename ValueType>
43 KOKKOS_INLINE_FUNCTION int TeamVectorGemmInternal<Algo::Gemm::Unblocked, false>::invoke(
44 const MemberType &member, const int m, const int n, const int k, const ScalarType alpha,
45 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1, const ValueType *KOKKOS_RESTRICT B, const int bs0,
46 const int bs1, const ScalarType beta,
47 /**/ ValueType *KOKKOS_RESTRICT C, const int cs0, const int cs1) {
48 // C = beta C + alpha A B
49 // C (m x n), A(m x k), B(k x n)
50
51 const ScalarType one(1.0), zero(0.0);
52
53 if (beta == zero)
54 KokkosBlas::Impl::TeamVectorSetInternal::invoke(member, m, n, zero, C, cs0, cs1);
55 else if (beta != one)
56 KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, n, beta, C, cs0, cs1);
57
58 if (alpha != ScalarType(0.0)) {
59 if (m <= 0 || n <= 0 || k <= 0) return 0;
60
61 if (beta != one) member.team_barrier();
62
63 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, m), [&](const int &i) {
64 const ValueType *KOKKOS_RESTRICT pA = A + i * as0;
65 Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, n), [&](const int &j) {
66 const ValueType *KOKKOS_RESTRICT pB = B + j * bs1;
67
68 ValueType c = ValueType(0);
69 for (int p = 0; p < k; ++p) c += pA[p * as1] * pB[p * bs0];
70 C[i * cs0 + j * cs1] += alpha * c;
71 });
72 });
73 }
74 return 0;
75 }
76
77 template <>
78 template <typename MemberType, typename ScalarType, typename ValueType>
79 KOKKOS_INLINE_FUNCTION int TeamVectorGemmInternal<Algo::Gemm::Unblocked, true>::invoke(
80 const MemberType &member, const int m, const int n, const int k, const ScalarType alpha,
81 const ValueType *KOKKOS_RESTRICT A, const int as0, const int as1, const ValueType *KOKKOS_RESTRICT B, const int bs0,
82 const int bs1, const ScalarType beta,
83 /**/ ValueType *KOKKOS_RESTRICT C, const int cs0, const int cs1) {
84 // C = beta C + alpha A B
85 // C (m x n), A(m x k), B(k x n)
86
87 const ScalarType one(1.0), zero(0.0);
88
89 if (beta == zero)
90 KokkosBlas::Impl::TeamVectorSetInternal::invoke(member, m, n, zero, C, cs0, cs1);
91 else if (beta != one)
92 KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, n, beta, C, cs0, cs1);
93
94 if (alpha != ScalarType(0.0)) {
95 if (m <= 0 || n <= 0 || k <= 0) return 0;
96
97 if (beta != one) member.team_barrier();
98
99 Kokkos::parallel_for(Kokkos::TeamThreadRange(member, m), [&](const int &i) {
100 const ValueType *KOKKOS_RESTRICT pA = A + i * as0;
101 Kokkos::parallel_for(Kokkos::ThreadVectorRange(member, n), [&](const int &j) {
102 const ValueType *KOKKOS_RESTRICT pB = B + j * bs1;
103
104 ValueType c = ValueType(0);
105 for (int p = 0; p < k; ++p) c += Kokkos::ArithTraits<ValueType>::conj(pA[p * as1]) * pB[p * bs0];
106 C[i * cs0 + j * cs1] += alpha * c;
107 });
108 });
109 }
110 return 0;
111 }
112
113 } // namespace KokkosBatched
114
115 #endif