| 1 |
// |
| 2 |
// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <[email protected]> |
| 3 |
// |
| 4 |
// SPDX-License-Identifier: Apache-2.0 |
| 5 |
// |
| 6 |
|
| 7 |
#include "test/nextgen/operators/matmul/pack_rhs/matmul_pack_rhs_quant_wrapper.hpp" |
| 8 |
|
| 9 |
#include <array> |
| 10 |
#include <cstddef> |
| 11 |
#include <cstdint> |
| 12 |
#include <optional> |
| 13 |
#include <string_view> |
| 14 |
#include <vector> |
| 15 |
|
| 16 |
#include "kai/kai_common.h" |
| 17 |
#include "test/common/abi_checker.hpp" |
| 18 |
#include "test/common/assert.hpp" |
| 19 |
#include "test/common/span.hpp" |
| 20 |
#include "test/nextgen/harness/tensor.hpp" |
| 21 |
#include "test/nextgen/operators/matmul/matmul_bias_mode.hpp" |
| 22 |
#include "test/nextgen/operators/matmul/matmul_config.hpp" |
| 23 |
#include "test/nextgen/operators/matmul/matmul_pack_args.hpp" |
| 24 |
#include "test/nextgen/operators/matmul/matmul_slots.hpp" |
| 25 |
|
| 26 |
namespace kai::test { |
| 27 |
|
| 28 |
namespace { |
| 29 |
|
| 30 |
std::optional<size_t> determine_bias_tensor_id(Span<const Tensor> tensors) { |
| 31 |
const MatMulConfig& config = tensors.at(MATMUL_SLOT_CONFIG).value<MatMulConfig>(); |
| 32 |
|
| 33 |
switch (config.bias_mode) { |
| 34 |
case MatMulBiasMode::NO_BIAS: |
| 35 |
return std::nullopt; |
| 36 |
|
| 37 |
case MatMulBiasMode::PER_N: |
| 38 |
return MATMUL_SLOT_BIAS_RAW; |
| 39 |
|
| 40 |
default: |
| 41 |
KAI_TEST_ERROR("Not supported."); |
| 42 |
} |
| 43 |
} |
| 44 |
|
| 45 |
} // namespace |
| 46 |
|
| 47 |
std::string_view MatMulPackRhsQuantWrapper::name() const { |
| 48 |
return m_name; |
| 49 |
} |
| 50 |
|
| 51 |
std::vector<size_t> MatMulPackRhsQuantWrapper::run_inputs(Span<const Tensor> tensors) const { |
| 52 |
std::vector<size_t> inputs = {MATMUL_SLOT_RHS_T_QDATA, MATMUL_SLOT_RHS_T_QSCALE}; |
| 53 |
|
| 54 |
const std::optional<size_t> bias_id = determine_bias_tensor_id(tensors); |
| 55 |
if (bias_id.has_value()) { |
| 56 |
inputs.emplace_back(bias_id.value()); |
| 57 |
} |
| 58 |
|
| 59 |
return inputs; |
| 60 |
} |
| 61 |
|
| 62 |
std::vector<size_t> MatMulPackRhsQuantWrapper::ref_inputs(Span<const Tensor> tensors) const { |
| 63 |
std::vector<size_t> inputs = { |
| 64 |
MATMUL_SLOT_RHS_T_QDATA_SIGN, MATMUL_SLOT_RHS_T_QDATA_SIGN_SUM, MATMUL_SLOT_RHS_T_QSCALE}; |
| 65 |
|
| 66 |
const std::optional<size_t> bias_id = determine_bias_tensor_id(tensors); |
| 67 |
if (bias_id.has_value()) { |
| 68 |
inputs.emplace_back(bias_id.value()); |
| 69 |
} |
| 70 |
|
| 71 |
return inputs; |
| 72 |
} |
| 73 |
|
| 74 |
std::vector<size_t> MatMulPackRhsQuantWrapper::steps(Span<const size_t> shape, Span<const Tensor> tensors) const { |
| 75 |
KAI_TEST_ASSERT_MSG(shape.size() == 2, "Only N and K dimensions are expected."); |
| 76 |
|
| 77 |
const auto& pack_args = tensors.at(MATMUL_SLOT_PACK_ARGS).value<MatMulPackArgs>(); |
| 78 |
|
| 79 |
const size_t n_step = m_kernel.get_n_step(pack_args.nr); |
| 80 |
const size_t shape_k = shape.at(1); |
| 81 |
|
| 82 |
return {n_step, shape_k}; |
| 83 |
} |
| 84 |
|
| 85 |
void MatMulPackRhsQuantWrapper::populate_constant_info(Span<Tensor> tensors) const { |
| 86 |
Tensor& rhs_t_qdata = tensors.at(MATMUL_SLOT_RHS_T_QDATA); |
| 87 |
Tensor& rhs_t_qdata_sign_sum = tensors.at(MATMUL_SLOT_RHS_T_QDATA_SIGN_SUM); |
| 88 |
Tensor& rhs_t_qscale = tensors.at(MATMUL_SLOT_RHS_T_QSCALE); |
| 89 |
Tensor& packed_rhs = tensors.at(MATMUL_SLOT_IMP_RHS_PACKED); |
| 90 |
|
| 91 |
rhs_t_qdata.set_format(m_src_data_format); |
| 92 |
rhs_t_qdata_sign_sum.set_format(m_src_sum_format); |
| 93 |
rhs_t_qscale.set_format(m_src_scale_format); |
| 94 |
packed_rhs.set_format(m_dst_format); |
| 95 |
|
| 96 |
const std::optional<size_t> bias_tensor_id = determine_bias_tensor_id(tensors); |
| 97 |
if (bias_tensor_id.has_value()) { |
| 98 |
Tensor& bias_raw = tensors.at(bias_tensor_id.value()); |
| 99 |
bias_raw.set_format(m_src_bias_format); |
| 100 |
} |
| 101 |
} |
| 102 |
|
| 103 |
void MatMulPackRhsQuantWrapper::run( |
| 104 |
Span<const size_t> full_shape, Span<const size_t> tile_coords, Span<const size_t> tile_shape, |
| 105 |
Span<Tensor> tensors) const { |
| 106 |
KAI_TEST_ASSERT_MSG(full_shape.size() == 2, "Only N and K dimensions are expected."); |
| 107 |
KAI_TEST_ASSERT_MSG(tile_coords.size() == 2, "Only N and K dimensions are expected."); |
| 108 |
KAI_TEST_ASSERT_MSG(tile_shape.size() == 2, "Only N and K dimensions are expected."); |
| 109 |
|
| 110 |
const size_t full_n = full_shape.at(0); |
| 111 |
const size_t full_k = full_shape.at(1); |
| 112 |
|
| 113 |
const size_t start_n = tile_coords.at(0); |
| 114 |
const size_t start_k = tile_coords.at(1); |
| 115 |
|
| 116 |
const size_t size_n = tile_shape.at(0); |
| 117 |
const size_t size_k = tile_shape.at(1); |
| 118 |
|
| 119 |
KAI_TEST_ASSERT(start_k == 0); |
| 120 |
KAI_TEST_ASSERT(size_k == full_k); |
| 121 |
|
| 122 |
const std::optional<size_t> bias_tensor_id = determine_bias_tensor_id(tensors); |
| 123 |
const bool has_bias = bias_tensor_id.has_value(); |
| 124 |
|
| 125 |
const Tensor& rhs_t_qdata = tensors.at(MATMUL_SLOT_RHS_T_QDATA); |
| 126 |
const Tensor& rhs_t_qscale = tensors.at(MATMUL_SLOT_RHS_T_QSCALE); |
| 127 |
const Tensor& bias_raw = tensors.at(bias_tensor_id.value_or(MATMUL_SLOT_BIAS_RAW)); |
| 128 |
Tensor& packed_rhs = tensors.at(MATMUL_SLOT_IMP_RHS_PACKED); |
| 129 |
|
| 130 |
const auto& pack_args = tensors.at(MATMUL_SLOT_PACK_ARGS).value<MatMulPackArgs>(); |
| 131 |
|
| 132 |
packed_rhs.set_shape({full_n, full_k}).allocate(); |
| 133 |
|
| 134 |
const size_t rhs_stride = m_src_data_format->compute_size({1, full_k}); |
| 135 |
|
| 136 |
const size_t rhs_offset = m_src_data_format->compute_offset(full_shape, tile_coords); |
| 137 |
const size_t imp_rhs_offset = m_kernel.get_rhs_offset(start_n, rhs_stride); |
| 138 |
KAI_TEST_ASSERT(imp_rhs_offset == rhs_offset); |
| 139 |
|
| 140 |
const size_t scale_offset = m_src_scale_format->compute_offset({full_n}, {start_n}); |
| 141 |
const size_t bias_offset = m_src_bias_format->compute_offset({full_n}, {start_n}); |
| 142 |
|
| 143 |
const size_t packed_rhs_offset = m_dst_format->compute_offset(full_shape, tile_coords); |
| 144 |
const size_t imp_packed_rhs_offset = |
| 145 |
m_kernel.get_rhs_packed_offset(start_n, full_k, pack_args.nr, pack_args.kr, pack_args.sr); |
| 146 |
KAI_TEST_ASSERT(imp_packed_rhs_offset == packed_rhs_offset); |
| 147 |
|
| 148 |
const size_t packed_rhs_size = packed_rhs.data().size(); |
| 149 |
const size_t imp_packed_rhs_size = |
| 150 |
m_kernel.get_rhs_packed_size(full_n, full_k, pack_args.nr, pack_args.kr, pack_args.sr); |
| 151 |
KAI_TEST_ASSERT(imp_packed_rhs_size == packed_rhs_size); |
| 152 |
|
| 153 |
const Span<const std::byte> rhs_tile = rhs_t_qdata.data().subspan(rhs_offset); |
| 154 |
const Span<const std::byte> scale_tile = rhs_t_qscale.data().subspan(scale_offset); |
| 155 |
const Span<const std::byte> bias_tile = has_bias ? bias_raw.data().subspan(bias_offset) : Span<const std::byte>(); |
| 156 |
const Span<std::byte> packed_lhs_tile = packed_rhs.data().subspan(packed_rhs_offset); |
| 157 |
|
| 158 |
const kai_rhs_pack_qs4cxs1s0_param params{1, 8}; |
| 159 |
|
| 160 |
abi_check([&] { |
| 161 |
m_kernel.run( |
| 162 |
1, size_n, size_k, pack_args.nr, pack_args.kr, pack_args.sr, |
| 163 |
reinterpret_cast<const uint8_t*>(rhs_tile.data()), reinterpret_cast<const float*>(bias_tile.data()), |
| 164 |
reinterpret_cast<const float*>(scale_tile.data()), packed_lhs_tile.data(), 0, ¶ms); |
| 165 |
}); |
| 166 |
} |
| 167 |
|
| 168 |
void MatMulPackRhsQuantWrapper::compute_reference(Span<const size_t> shape, Span<Tensor> tensors) const { |
| 169 |
KAI_TEST_ASSERT_MSG(shape.size() == 2, "Only N and K dimensions are expected."); |
| 170 |
const size_t shape_n = shape.at(0); |
| 171 |
|
| 172 |
const std::optional<size_t> bias_tensor_id = determine_bias_tensor_id(tensors); |
| 173 |
const bool has_bias = bias_tensor_id.has_value(); |
| 174 |
|
| 175 |
const Tensor& rhs_t_qdata_sign = tensors.at(MATMUL_SLOT_RHS_T_QDATA_SIGN); |
| 176 |
const Tensor& rhs_t_qdata_sign_sum = tensors.at(MATMUL_SLOT_RHS_T_QDATA_SIGN_SUM); |
| 177 |
const Tensor& rhs_t_qscale = tensors.at(MATMUL_SLOT_RHS_T_QSCALE); |
| 178 |
const Tensor& bias_raw = tensors.at(bias_tensor_id.value_or(MATMUL_SLOT_BIAS_RAW)); |
| 179 |
Tensor& ref_packed_rhs = tensors.at(MATMUL_SLOT_REF_RHS_PACKED); |
| 180 |
|
| 181 |
Buffer empty_bias; |
| 182 |
Span<const std::byte> bias_data; |
| 183 |
|
| 184 |
if (has_bias) { |
| 185 |
bias_data = bias_raw.data(); |
| 186 |
} else { |
| 187 |
empty_bias = Buffer(m_src_bias_format->compute_size({shape_n})); |
| 188 |
bias_data = empty_bias.view(); |
| 189 |
} |
| 190 |
|
| 191 |
ref_packed_rhs.set_shape(shape) |
| 192 |
.set_format(m_dst_format) |
| 193 |
.set_data(m_dst_format->pack( |
| 194 |
shape, std::array{rhs_t_qdata_sign.data(), rhs_t_qdata_sign_sum.data(), rhs_t_qscale.data(), bias_data})); |
| 195 |
} |
| 196 |
|
| 197 |
} // namespace kai::test |