Debian Codegraph @

test/nextgen/operators/matmul/pack_rhs/matmul_pack_rhs_quant_wrapper.cpp

raw
1 //
2 // SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <[email protected]>
3 //
4 // SPDX-License-Identifier: Apache-2.0
5 //
6
7 #include "test/nextgen/operators/matmul/pack_rhs/matmul_pack_rhs_quant_wrapper.hpp"
8
9 #include <array>
10 #include <cstddef>
11 #include <cstdint>
12 #include <optional>
13 #include <string_view>
14 #include <vector>
15
16 #include "kai/kai_common.h"
17 #include "test/common/abi_checker.hpp"
18 #include "test/common/assert.hpp"
19 #include "test/common/span.hpp"
20 #include "test/nextgen/harness/tensor.hpp"
21 #include "test/nextgen/operators/matmul/matmul_bias_mode.hpp"
22 #include "test/nextgen/operators/matmul/matmul_config.hpp"
23 #include "test/nextgen/operators/matmul/matmul_pack_args.hpp"
24 #include "test/nextgen/operators/matmul/matmul_slots.hpp"
25
26 namespace kai::test {
27
28 namespace {
29
30 std::optional<size_t> determine_bias_tensor_id(Span<const Tensor> tensors) {
31 const MatMulConfig& config = tensors.at(MATMUL_SLOT_CONFIG).value<MatMulConfig>();
32
33 switch (config.bias_mode) {
34 case MatMulBiasMode::NO_BIAS:
35 return std::nullopt;
36
37 case MatMulBiasMode::PER_N:
38 return MATMUL_SLOT_BIAS_RAW;
39
40 default:
41 KAI_TEST_ERROR("Not supported.");
42 }
43 }
44
45 } // namespace
46
47 std::string_view MatMulPackRhsQuantWrapper::name() const {
48 return m_name;
49 }
50
51 std::vector<size_t> MatMulPackRhsQuantWrapper::run_inputs(Span<const Tensor> tensors) const {
52 std::vector<size_t> inputs = {MATMUL_SLOT_RHS_T_QDATA, MATMUL_SLOT_RHS_T_QSCALE};
53
54 const std::optional<size_t> bias_id = determine_bias_tensor_id(tensors);
55 if (bias_id.has_value()) {
56 inputs.emplace_back(bias_id.value());
57 }
58
59 return inputs;
60 }
61
62 std::vector<size_t> MatMulPackRhsQuantWrapper::ref_inputs(Span<const Tensor> tensors) const {
63 std::vector<size_t> inputs = {
64 MATMUL_SLOT_RHS_T_QDATA_SIGN, MATMUL_SLOT_RHS_T_QDATA_SIGN_SUM, MATMUL_SLOT_RHS_T_QSCALE};
65
66 const std::optional<size_t> bias_id = determine_bias_tensor_id(tensors);
67 if (bias_id.has_value()) {
68 inputs.emplace_back(bias_id.value());
69 }
70
71 return inputs;
72 }
73
74 std::vector<size_t> MatMulPackRhsQuantWrapper::steps(Span<const size_t> shape, Span<const Tensor> tensors) const {
75 KAI_TEST_ASSERT_MSG(shape.size() == 2, "Only N and K dimensions are expected.");
76
77 const auto& pack_args = tensors.at(MATMUL_SLOT_PACK_ARGS).value<MatMulPackArgs>();
78
79 const size_t n_step = m_kernel.get_n_step(pack_args.nr);
80 const size_t shape_k = shape.at(1);
81
82 return {n_step, shape_k};
83 }
84
85 void MatMulPackRhsQuantWrapper::populate_constant_info(Span<Tensor> tensors) const {
86 Tensor& rhs_t_qdata = tensors.at(MATMUL_SLOT_RHS_T_QDATA);
87 Tensor& rhs_t_qdata_sign_sum = tensors.at(MATMUL_SLOT_RHS_T_QDATA_SIGN_SUM);
88 Tensor& rhs_t_qscale = tensors.at(MATMUL_SLOT_RHS_T_QSCALE);
89 Tensor& packed_rhs = tensors.at(MATMUL_SLOT_IMP_RHS_PACKED);
90
91 rhs_t_qdata.set_format(m_src_data_format);
92 rhs_t_qdata_sign_sum.set_format(m_src_sum_format);
93 rhs_t_qscale.set_format(m_src_scale_format);
94 packed_rhs.set_format(m_dst_format);
95
96 const std::optional<size_t> bias_tensor_id = determine_bias_tensor_id(tensors);
97 if (bias_tensor_id.has_value()) {
98 Tensor& bias_raw = tensors.at(bias_tensor_id.value());
99 bias_raw.set_format(m_src_bias_format);
100 }
101 }
102
103 void MatMulPackRhsQuantWrapper::run(
104 Span<const size_t> full_shape, Span<const size_t> tile_coords, Span<const size_t> tile_shape,
105 Span<Tensor> tensors) const {
106 KAI_TEST_ASSERT_MSG(full_shape.size() == 2, "Only N and K dimensions are expected.");
107 KAI_TEST_ASSERT_MSG(tile_coords.size() == 2, "Only N and K dimensions are expected.");
108 KAI_TEST_ASSERT_MSG(tile_shape.size() == 2, "Only N and K dimensions are expected.");
109
110 const size_t full_n = full_shape.at(0);
111 const size_t full_k = full_shape.at(1);
112
113 const size_t start_n = tile_coords.at(0);
114 const size_t start_k = tile_coords.at(1);
115
116 const size_t size_n = tile_shape.at(0);
117 const size_t size_k = tile_shape.at(1);
118
119 KAI_TEST_ASSERT(start_k == 0);
120 KAI_TEST_ASSERT(size_k == full_k);
121
122 const std::optional<size_t> bias_tensor_id = determine_bias_tensor_id(tensors);
123 const bool has_bias = bias_tensor_id.has_value();
124
125 const Tensor& rhs_t_qdata = tensors.at(MATMUL_SLOT_RHS_T_QDATA);
126 const Tensor& rhs_t_qscale = tensors.at(MATMUL_SLOT_RHS_T_QSCALE);
127 const Tensor& bias_raw = tensors.at(bias_tensor_id.value_or(MATMUL_SLOT_BIAS_RAW));
128 Tensor& packed_rhs = tensors.at(MATMUL_SLOT_IMP_RHS_PACKED);
129
130 const auto& pack_args = tensors.at(MATMUL_SLOT_PACK_ARGS).value<MatMulPackArgs>();
131
132 packed_rhs.set_shape({full_n, full_k}).allocate();
133
134 const size_t rhs_stride = m_src_data_format->compute_size({1, full_k});
135
136 const size_t rhs_offset = m_src_data_format->compute_offset(full_shape, tile_coords);
137 const size_t imp_rhs_offset = m_kernel.get_rhs_offset(start_n, rhs_stride);
138 KAI_TEST_ASSERT(imp_rhs_offset == rhs_offset);
139
140 const size_t scale_offset = m_src_scale_format->compute_offset({full_n}, {start_n});
141 const size_t bias_offset = m_src_bias_format->compute_offset({full_n}, {start_n});
142
143 const size_t packed_rhs_offset = m_dst_format->compute_offset(full_shape, tile_coords);
144 const size_t imp_packed_rhs_offset =
145 m_kernel.get_rhs_packed_offset(start_n, full_k, pack_args.nr, pack_args.kr, pack_args.sr);
146 KAI_TEST_ASSERT(imp_packed_rhs_offset == packed_rhs_offset);
147
148 const size_t packed_rhs_size = packed_rhs.data().size();
149 const size_t imp_packed_rhs_size =
150 m_kernel.get_rhs_packed_size(full_n, full_k, pack_args.nr, pack_args.kr, pack_args.sr);
151 KAI_TEST_ASSERT(imp_packed_rhs_size == packed_rhs_size);
152
153 const Span<const std::byte> rhs_tile = rhs_t_qdata.data().subspan(rhs_offset);
154 const Span<const std::byte> scale_tile = rhs_t_qscale.data().subspan(scale_offset);
155 const Span<const std::byte> bias_tile = has_bias ? bias_raw.data().subspan(bias_offset) : Span<const std::byte>();
156 const Span<std::byte> packed_lhs_tile = packed_rhs.data().subspan(packed_rhs_offset);
157
158 const kai_rhs_pack_qs4cxs1s0_param params{1, 8};
159
160 abi_check([&] {
161 m_kernel.run(
162 1, size_n, size_k, pack_args.nr, pack_args.kr, pack_args.sr,
163 reinterpret_cast<const uint8_t*>(rhs_tile.data()), reinterpret_cast<const float*>(bias_tile.data()),
164 reinterpret_cast<const float*>(scale_tile.data()), packed_lhs_tile.data(), 0, &params);
165 });
166 }
167
168 void MatMulPackRhsQuantWrapper::compute_reference(Span<const size_t> shape, Span<Tensor> tensors) const {
169 KAI_TEST_ASSERT_MSG(shape.size() == 2, "Only N and K dimensions are expected.");
170 const size_t shape_n = shape.at(0);
171
172 const std::optional<size_t> bias_tensor_id = determine_bias_tensor_id(tensors);
173 const bool has_bias = bias_tensor_id.has_value();
174
175 const Tensor& rhs_t_qdata_sign = tensors.at(MATMUL_SLOT_RHS_T_QDATA_SIGN);
176 const Tensor& rhs_t_qdata_sign_sum = tensors.at(MATMUL_SLOT_RHS_T_QDATA_SIGN_SUM);
177 const Tensor& rhs_t_qscale = tensors.at(MATMUL_SLOT_RHS_T_QSCALE);
178 const Tensor& bias_raw = tensors.at(bias_tensor_id.value_or(MATMUL_SLOT_BIAS_RAW));
179 Tensor& ref_packed_rhs = tensors.at(MATMUL_SLOT_REF_RHS_PACKED);
180
181 Buffer empty_bias;
182 Span<const std::byte> bias_data;
183
184 if (has_bias) {
185 bias_data = bias_raw.data();
186 } else {
187 empty_bias = Buffer(m_src_bias_format->compute_size({shape_n}));
188 bias_data = empty_bias.view();
189 }
190
191 ref_packed_rhs.set_shape(shape)
192 .set_format(m_dst_format)
193 .set_data(m_dst_format->pack(
194 shape, std::array{rhs_t_qdata_sign.data(), rhs_t_qdata_sign_sum.data(), rhs_t_qscale.data(), bias_data}));
195 }
196
197 } // namespace kai::test