/* * (C) Copyright 2024 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "atlas/linalg/sparse/SparseMatrixMultiply_HicSparse.h" #include "atlas/parallel/omp/omp.h" #include "atlas/runtime/Exception.h" #include "hic/hic.h" #include "hic/hic_library_types.h" #include "hic/hicsparse.h" namespace { class HicSparseHandleRAIIWrapper { public: HicSparseHandleRAIIWrapper() { hicsparseCreate(&handle_); }; ~HicSparseHandleRAIIWrapper() { hicsparseDestroy(handle_); } hicsparseHandle_t value() { return handle_; } private: hicsparseHandle_t handle_; }; hicsparseHandle_t getDefaultHicSparseHandle() { static auto handle = HicSparseHandleRAIIWrapper(); return handle.value(); } template constexpr hicsparseIndexType_t getHicsparseIndexType() { using base_type = std::remove_const_t; if constexpr (std::is_same_v) { return HICSPARSE_INDEX_32I; } else { static_assert(std::is_same_v, "Unsupported index type"); return HICSPARSE_INDEX_64I; } } template constexpr auto getHicsparseValueType() { using base_type = std::remove_const_t; if constexpr (std::is_same_v) { return HIC_R_32F; } else { static_assert(std::is_same_v, "Unsupported value type");\ return HIC_R_64F; } } template hicsparseOrder_t getHicsparseOrder(const atlas::linalg::View& v) { constexpr int row_idx = (IndexLayout == atlas::linalg::Indexing::layout_left) ? 0 : 1; constexpr int col_idx = (IndexLayout == atlas::linalg::Indexing::layout_left) ? 1 : 0; if (v.stride(row_idx) == 1) { return HICSPARSE_ORDER_COL; } else if (v.stride(col_idx) == 1) { return HICSPARSE_ORDER_ROW; } else { atlas::throw_Exception("Unsupported dense matrix memory order", Here()); return HICSPARSE_ORDER_COL; } } template int64_t getLeadingDimension(const atlas::linalg::View& v) { if (v.stride(0) == 1) { return v.stride(1); } else if (v.stride(1) == 1) { return v.stride(0); } else { atlas::throw_Exception("Unsupported dense matrix memory order", Here()); return 0; } } } namespace atlas { namespace linalg { namespace sparse { template void hsSpMV(const SparseMatrixView& W, const View& src, Value beta, View& tgt) { // Assume that W, src, and tgt are all device views using Index [[maybe_unused]] = typename SparseMatrixView::index_type; ATLAS_ASSERT(src.shape(0) >= W.cols()); ATLAS_ASSERT(tgt.shape(0) >= W.rows()); [[maybe_unused]] auto handle = getDefaultHicSparseHandle(); // Create a sparse matrix descriptor [[maybe_unused]] hicsparseConstSpMatDescr_t matA; HICSPARSE_CALL(hicsparseCreateConstCsr( &matA, W.rows(), W.cols(), W.nnz(), W.outer(), // row_offsets W.inner(), // column_indices W.value(), // values getHicsparseIndexType(), getHicsparseIndexType(), HICSPARSE_INDEX_BASE_ZERO, getHicsparseValueType())); // Create dense matrix descriptors [[maybe_unused]] hicsparseConstDnVecDescr_t vecX; HICSPARSE_CALL(hicsparseCreateConstDnVec( &vecX, static_cast(W.cols()), src.data(), getHicsparseValueType())); [[maybe_unused]] hicsparseDnVecDescr_t vecY; HICSPARSE_CALL(hicsparseCreateDnVec( &vecY, W.rows(), tgt.data(), getHicsparseValueType())); [[maybe_unused]] const Value alpha = 1; // Determine buffer size [[maybe_unused]] size_t bufferSize = 0; HICSPARSE_CALL(hicsparseSpMV_bufferSize( handle, HICSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY, getHicsparseValueType(), HICSPARSE_SPMV_ALG_DEFAULT, &bufferSize)); // Allocate buffer [[maybe_unused]] char* buffer; HIC_CALL(hicMalloc(&buffer, bufferSize)); // Perform SpMV HICSPARSE_CALL(hicsparseSpMV( handle, HICSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY, getHicsparseValueType(), HICSPARSE_SPMV_ALG_DEFAULT, buffer)); HIC_CALL(hicFree(buffer)); HICSPARSE_CALL(hicsparseDestroyDnVec(vecX)); HICSPARSE_CALL(hicsparseDestroyDnVec(vecY)); HICSPARSE_CALL(hicsparseDestroySpMat(matA)); HIC_CALL(hicDeviceSynchronize()); } template void hsSpMM(const SparseMatrixView& W, const View& src, Value beta, View& tgt) { // Assume that W, src, and tgt are all device views using Index [[maybe_unused]] = typename SparseMatrixView::index_type; constexpr int row_idx = (IndexLayout == Indexing::layout_left) ? 0 : 1; constexpr int col_idx = (IndexLayout == Indexing::layout_left) ? 1 : 0; ATLAS_ASSERT(src.shape(row_idx) >= W.cols()); ATLAS_ASSERT(tgt.shape(row_idx) >= W.rows()); ATLAS_ASSERT(src.shape(col_idx) == tgt.shape(col_idx)); [[maybe_unused]] auto handle = getDefaultHicSparseHandle(); // Create a sparse matrix descriptor [[maybe_unused]] hicsparseConstSpMatDescr_t matA; HICSPARSE_CALL(hicsparseCreateConstCsr( &matA, W.rows(), W.cols(), W.nnz(), W.outer(), // row_offsets W.inner(), // column_indices W.value(), // values getHicsparseIndexType(), getHicsparseIndexType(), HICSPARSE_INDEX_BASE_ZERO, getHicsparseValueType())); // Create dense matrix descriptors [[maybe_unused]] hicsparseConstDnMatDescr_t matB; HICSPARSE_CALL(hicsparseCreateConstDnMat( &matB, W.cols(), src.shape(col_idx), getLeadingDimension(src), src.data(), getHicsparseValueType(), getHicsparseOrder(src))); [[maybe_unused]] hicsparseDnMatDescr_t matC; HICSPARSE_CALL(hicsparseCreateDnMat( &matC, W.rows(), tgt.shape(col_idx), getLeadingDimension(tgt), tgt.data(), getHicsparseValueType(), getHicsparseOrder(tgt))); [[maybe_unused]] const Value alpha = 1; // Determine buffer size [[maybe_unused]] size_t bufferSize = 0; HICSPARSE_CALL(hicsparseSpMM_bufferSize( handle, HICSPARSE_OPERATION_NON_TRANSPOSE, HICSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, matB, &beta, matC, getHicsparseValueType(), HICSPARSE_SPMM_ALG_DEFAULT, &bufferSize)); // Allocate buffer [[maybe_unused]] char* buffer; HIC_CALL(hicMalloc(&buffer, bufferSize)); // Perform SpMM HICSPARSE_CALL(hicsparseSpMM( handle, HICSPARSE_OPERATION_NON_TRANSPOSE, HICSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, matB, &beta, matC, getHicsparseValueType(), HICSPARSE_SPMM_ALG_DEFAULT, buffer)); HIC_CALL(hicFree(buffer)); HICSPARSE_CALL(hicsparseDestroyDnMat(matC)); HICSPARSE_CALL(hicsparseDestroyDnMat(matB)); HICSPARSE_CALL(hicsparseDestroySpMat(matA)); HIC_CALL(hicDeviceSynchronize()); } template void SparseMatrixMultiply::multiply( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { ScalarValue beta = 0; hsSpMV(W, src, beta, tgt); } template void SparseMatrixMultiply::multiply_add( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { ScalarValue beta = 1; hsSpMV(W, src, beta, tgt); } template void SparseMatrixMultiply::multiply( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { ScalarValue beta = 0; hsSpMM(W, src, beta, tgt); } template void SparseMatrixMultiply::multiply_add( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { ScalarValue beta = 1; hsSpMM(W, src, beta, tgt); } template void SparseMatrixMultiply::multiply( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { ScalarValue beta = 0; hsSpMV(W, src, beta, tgt); } template void SparseMatrixMultiply::multiply_add( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { ScalarValue beta = 1; hsSpMV(W, src, beta, tgt); } template void SparseMatrixMultiply::multiply( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { ScalarValue beta = 0; hsSpMM(W, src, beta, tgt); } template void SparseMatrixMultiply::multiply_add( const SparseMatrixView& W, const View& src, View& tgt, const Configuration&) { ScalarValue beta = 1; hsSpMM(W, src, beta, tgt); } #define EXPLICIT_TEMPLATE_INSTANTIATION(TYPE) \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; \ template struct SparseMatrixMultiply; EXPLICIT_TEMPLATE_INSTANTIATION(double); EXPLICIT_TEMPLATE_INSTANTIATION(float); } // namespace sparse } // namespace linalg } // namespace atlas