| 1 |
/* |
| 2 |
* (C) Copyright 2025- ECMWF. |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
* In applying this licence, ECMWF does not waive the privileges and immunities |
| 7 |
* granted to it by virtue of its status as an intergovernmental organisation nor |
| 8 |
* does it submit to any jurisdiction. |
| 9 |
*/ |
| 10 |
|
| 11 |
#include "Packer.h" |
| 12 |
#include "pack_index.h" |
| 13 |
#include "atlas/array/SVector.h" |
| 14 |
#include "atlas/runtime/Exception.h" |
| 15 |
|
| 16 |
#include "hic/hic.h" |
| 17 |
|
| 18 |
namespace atlas::parallel::detail { |
| 19 |
|
| 20 |
bool is_device_accessible(const void* ptr) { |
| 21 |
hicPointerAttributes attr; |
| 22 |
hicError_t code = hicPointerGetAttributes(&attr, ptr); |
| 23 |
if( code != hicSuccess ) { |
| 24 |
static_cast<void>(hicGetLastError()); |
| 25 |
return false; |
| 26 |
} |
| 27 |
return ptr == attr.devicePointer; |
| 28 |
} |
| 29 |
|
| 30 |
template<int ParallelDim, int RANK> |
| 31 |
struct get_buffer_index{ |
| 32 |
template<typename DATA_TYPE> |
| 33 |
ATLAS_HOST_DEVICE |
| 34 |
static idx_t apply(const array::ArrayView<DATA_TYPE, RANK>& field, |
| 35 |
const idx_t node_cnt, const idx_t var1_idx) { |
| 36 |
return field.shape(RANK-1) * field.shape(RANK-2) * node_cnt + |
| 37 |
field.shape(RANK-1) * var1_idx; |
| 38 |
} |
| 39 |
}; |
| 40 |
|
| 41 |
template<int ParallelDim> |
| 42 |
struct get_buffer_index<ParallelDim, 2>{ |
| 43 |
template<typename DATA_TYPE> |
| 44 |
ATLAS_HOST_DEVICE |
| 45 |
static idx_t apply(const array::ArrayView<DATA_TYPE, 2>& field, |
| 46 |
const idx_t node_cnt, const idx_t var1_idx) { |
| 47 |
return field.shape(1) * node_cnt + var1_idx; |
| 48 |
} |
| 49 |
}; |
| 50 |
|
| 51 |
template<int ParallelDim, typename DATA_TYPE, int RANK> |
| 52 |
__global__ void pack_kernel(const int sendcnt, const int* sendmap_ptr, const idx_t sendmap_size, |
| 53 |
const array::ArrayView<DATA_TYPE, RANK> field, DATA_TYPE* send_buffer, |
| 54 |
const idx_t send_buffer_size, const typename std::enable_if<RANK==1, int>::type = 0) { |
| 55 |
const array::SVector<int> sendmap(const_cast<int*>(sendmap_ptr), sendmap_size); |
| 56 |
|
| 57 |
const idx_t node_cnt = blockIdx.x*blockDim.x + threadIdx.x; |
| 58 |
|
| 59 |
if(node_cnt >= sendcnt) return; |
| 60 |
|
| 61 |
send_buffer[node_cnt] = field(sendmap[node_cnt]); |
| 62 |
} |
| 63 |
|
| 64 |
template<int ParallelDim, typename DATA_TYPE, int RANK> |
| 65 |
__global__ void pack_kernel(const int sendcnt, const int* sendmap_ptr, const idx_t sendmap_size, |
| 66 |
const array::ArrayView<DATA_TYPE, RANK> field, DATA_TYPE* send_buffer, |
| 67 |
const idx_t send_buffer_size, const typename std::enable_if<RANK>=2, int>::type = 0) { |
| 68 |
const array::SVector<int> sendmap(const_cast<int*>(sendmap_ptr), sendmap_size); |
| 69 |
|
| 70 |
const idx_t node_cnt = blockIdx.x*blockDim.x + threadIdx.x; |
| 71 |
const idx_t var1_idx = blockIdx.y*blockDim.y + threadIdx.y; |
| 72 |
|
| 73 |
if(node_cnt >= sendcnt || var1_idx >= field.shape(1) ) return; |
| 74 |
|
| 75 |
idx_t buff_idx = get_buffer_index<ParallelDim, RANK>::apply(field, node_cnt, var1_idx); |
| 76 |
const idx_t node_idx = sendmap[node_cnt]; |
| 77 |
|
| 78 |
pack_index<0, (RANK>=3) ? (RANK-2) : 0, 2>::apply(buff_idx, node_idx, field, send_buffer, node_idx,var1_idx); |
| 79 |
} |
| 80 |
|
| 81 |
template<int ParallelDim, typename DATA_TYPE, int RANK> |
| 82 |
__global__ void unpack_kernel(const int recvcnt, const int* recvmap_ptr, const idx_t recvmap_size, |
| 83 |
const DATA_TYPE* recv_buffer, const idx_t recv_buffer_size, array::ArrayView<DATA_TYPE, RANK> field, const typename std::enable_if<RANK==1, int>::type = 0) { |
| 84 |
|
| 85 |
const array::SVector<int> recvmap(const_cast<int*>(recvmap_ptr), recvmap_size); |
| 86 |
|
| 87 |
idx_t node_cnt = blockIdx.x*blockDim.x + threadIdx.x; |
| 88 |
|
| 89 |
if(node_cnt >= recvcnt) return; |
| 90 |
|
| 91 |
const idx_t node_idx = recvmap[node_cnt]; |
| 92 |
|
| 93 |
field(node_idx) = recv_buffer[node_cnt]; |
| 94 |
} |
| 95 |
|
| 96 |
template<int ParallelDim, typename DATA_TYPE, int RANK> |
| 97 |
__global__ void unpack_kernel(const int recvcnt, const int* recvmap_ptr, const idx_t recvmap_size, |
| 98 |
const DATA_TYPE* recv_buffer, const idx_t recv_buffer_size, array::ArrayView<DATA_TYPE, RANK> field, const typename std::enable_if<RANK>=2, int>::type = 0) { |
| 99 |
|
| 100 |
const array::SVector<int> recvmap(const_cast<int*>(recvmap_ptr), recvmap_size); |
| 101 |
|
| 102 |
const idx_t node_cnt = blockIdx.x*blockDim.x + threadIdx.x; |
| 103 |
const idx_t var1_idx = blockIdx.y*blockDim.y + threadIdx.y; |
| 104 |
|
| 105 |
if(node_cnt >= recvcnt || var1_idx >= field.shape(1) ) return; |
| 106 |
|
| 107 |
const idx_t node_idx = recvmap[node_cnt]; |
| 108 |
|
| 109 |
idx_t buff_idx = get_buffer_index<ParallelDim, RANK>::apply(field, node_cnt, var1_idx); |
| 110 |
|
| 111 |
unpack_index<0, (RANK>=3) ? (RANK-2) : 0, 2>::apply(buff_idx, node_idx, recv_buffer, field,node_idx,var1_idx); |
| 112 |
} |
| 113 |
|
| 114 |
template<int ParallelDim, int RANK, int DimCnt> |
| 115 |
struct get_first_non_parallel_dim |
| 116 |
{ |
| 117 |
static_assert((ParallelDim <= RANK), "Error: parallelDim larger than RANK"); |
| 118 |
constexpr static int apply() { |
| 119 |
return (DimCnt == ParallelDim) ? get_first_non_parallel_dim<ParallelDim, RANK, DimCnt+1>::apply() : DimCnt; |
| 120 |
} |
| 121 |
}; |
| 122 |
|
| 123 |
template<int ParallelDim, int RANK> |
| 124 |
struct get_first_non_parallel_dim<ParallelDim, RANK, RANK> |
| 125 |
{ |
| 126 |
static_assert((ParallelDim <= RANK), "Error: parallelDim larger than RANK"); |
| 127 |
constexpr static int apply() { |
| 128 |
return -1; |
| 129 |
} |
| 130 |
}; |
| 131 |
|
| 132 |
template<int ParallelDim, int RANK> |
| 133 |
struct get_n_hic_blocks |
| 134 |
{ |
| 135 |
template<typename DATA_TYPE> |
| 136 |
static unsigned int apply(const array::ArrayView<DATA_TYPE, RANK>& hfield, const unsigned int block_size_y) { |
| 137 |
constexpr auto dim = get_first_non_parallel_dim<ParallelDim, RANK, 0>::apply(); |
| 138 |
return (hfield.shape(dim)+block_size_y-1)/block_size_y; |
| 139 |
} |
| 140 |
}; |
| 141 |
|
| 142 |
template<> |
| 143 |
struct get_n_hic_blocks<0, 1> { |
| 144 |
template<typename DATA_TYPE> |
| 145 |
static unsigned int apply(const array::ArrayView<DATA_TYPE, 1>& hfield, const unsigned int block_size_y) { |
| 146 |
return 1; |
| 147 |
} |
| 148 |
}; |
| 149 |
|
| 150 |
template<int ParallelDim, typename DATA_TYPE, int RANK> |
| 151 |
void DevicePacker<ParallelDim, DATA_TYPE, RANK>::pack( const int sendcnt, array::SVector<int> const & sendmap, |
| 152 |
const array::ArrayView<DATA_TYPE, RANK>& array, |
| 153 |
DATA_TYPE* send_buffer, int send_buffer_size ) |
| 154 |
{ |
| 155 |
const unsigned int block_size_x = 32; |
| 156 |
const unsigned int block_size_y = (RANK==1) ? 1 : 4; |
| 157 |
|
| 158 |
const unsigned int nblocks_x = (sendcnt+block_size_x-1)/block_size_x; |
| 159 |
const unsigned int nblocks_y = get_n_hic_blocks<ParallelDim, RANK>::apply(array, block_size_y); |
| 160 |
|
| 161 |
dim3 threads(block_size_x, block_size_y); |
| 162 |
dim3 blocks(nblocks_x, nblocks_y); |
| 163 |
HIC_CALL(hicDeviceSynchronize()); |
| 164 |
hicError_t err = hicGetLastError(); |
| 165 |
if (err != hicSuccess) { |
| 166 |
std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err); |
| 167 |
throw_Exception(msg); |
| 168 |
} |
| 169 |
|
| 170 |
pack_kernel<ParallelDim, DATA_TYPE, RANK><<<blocks,threads>>>(sendcnt, sendmap.data(), sendmap.size(), array, send_buffer, send_buffer_size); |
| 171 |
err = hicGetLastError(); |
| 172 |
if (err != hicSuccess) |
| 173 |
throw_Exception("Error launching GPU packing kernel"); |
| 174 |
|
| 175 |
HIC_CALL(hicDeviceSynchronize()); |
| 176 |
err = hicGetLastError(); |
| 177 |
if (err != hicSuccess) { |
| 178 |
std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err); |
| 179 |
throw_Exception(msg); |
| 180 |
} |
| 181 |
|
| 182 |
} |
| 183 |
|
| 184 |
template<int ParallelDim, typename DATA_TYPE, int RANK> |
| 185 |
void DevicePacker<ParallelDim, DATA_TYPE, RANK>::unpack(const int recvcnt, array::SVector<int> const & recvmap, |
| 186 |
const DATA_TYPE* recv_buffer , int recv_buffer_size, |
| 187 |
array::ArrayView<DATA_TYPE, RANK>& array) |
| 188 |
{ |
| 189 |
const unsigned int block_size_x = 32; |
| 190 |
const unsigned int block_size_y = (RANK==1) ? 1 : 4; |
| 191 |
|
| 192 |
unsigned int nblocks_y = get_n_hic_blocks<ParallelDim, RANK>::apply(array, block_size_y); |
| 193 |
|
| 194 |
dim3 threads(block_size_x, block_size_y); |
| 195 |
dim3 blocks((recvcnt+block_size_x-1)/block_size_x, nblocks_y); |
| 196 |
|
| 197 |
HIC_CALL(hicDeviceSynchronize()); |
| 198 |
hicError_t err = hicGetLastError(); |
| 199 |
if (err != hicSuccess) { |
| 200 |
std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err); |
| 201 |
throw_Exception(msg); |
| 202 |
} |
| 203 |
|
| 204 |
unpack_kernel<ParallelDim, DATA_TYPE, RANK><<<blocks,threads>>>(recvcnt, recvmap.data(), recvmap.size(), recv_buffer, recv_buffer_size, array); |
| 205 |
|
| 206 |
err = hicGetLastError(); |
| 207 |
if (err != hicSuccess) { |
| 208 |
std::string msg = std::string("Error launching GPU packing kernel")+ hicGetErrorString(err); |
| 209 |
throw_Exception(msg); |
| 210 |
} |
| 211 |
|
| 212 |
HIC_CALL(hicDeviceSynchronize()); |
| 213 |
err = hicGetLastError(); |
| 214 |
if (err != hicSuccess) { |
| 215 |
std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err); |
| 216 |
throw_Exception(msg); |
| 217 |
} |
| 218 |
} |
| 219 |
|
| 220 |
|
| 221 |
#define ATLAS_REPEAT_MACRO(n, m, p) ATLAS_REPEAT_MACRO_ ## n(m, p) |
| 222 |
// expands to m(0,p) m(1,p) ... m(n-1,p) |
| 223 |
|
| 224 |
#define ATLAS_REPEAT_MACRO_0(m, p) |
| 225 |
#define ATLAS_REPEAT_MACRO_1(m, p) ATLAS_REPEAT_MACRO_0(m, p) m(0, p) |
| 226 |
#define ATLAS_REPEAT_MACRO_2(m, p) ATLAS_REPEAT_MACRO_1(m, p) m(1, p) |
| 227 |
#define ATLAS_REPEAT_MACRO_3(m, p) ATLAS_REPEAT_MACRO_2(m, p) m(2, p) |
| 228 |
#define ATLAS_REPEAT_MACRO_4(m, p) ATLAS_REPEAT_MACRO_3(m, p) m(3, p) |
| 229 |
#define ATLAS_REPEAT_MACRO_5(m, p) ATLAS_REPEAT_MACRO_4(m, p) m(4, p) |
| 230 |
#define ATLAS_REPEAT_MACRO_6(m, p) ATLAS_REPEAT_MACRO_5(m, p) m(5, p) |
| 231 |
|
| 232 |
|
| 233 |
#define EXPLICIT_TEMPLATE_INSTANTIATION( ParallelDim, RANK ) \ |
| 234 |
template struct DevicePacker<ParallelDim, int, RANK>; \ |
| 235 |
template struct DevicePacker<ParallelDim, long, RANK>; \ |
| 236 |
template struct DevicePacker<ParallelDim, long unsigned, RANK>; \ |
| 237 |
template struct DevicePacker<ParallelDim, float, RANK>; \ |
| 238 |
template struct DevicePacker<ParallelDim, double, RANK>; |
| 239 |
|
| 240 |
#define EXPLICIT_TEMPLATE_INSTANTIATION_RANK(RANK) \ |
| 241 |
ATLAS_REPEAT_MACRO(RANK, EXPLICIT_TEMPLATE_INSTANTIATION, RANK) |
| 242 |
|
| 243 |
EXPLICIT_TEMPLATE_INSTANTIATION_RANK(1) |
| 244 |
EXPLICIT_TEMPLATE_INSTANTIATION_RANK(2) |
| 245 |
EXPLICIT_TEMPLATE_INSTANTIATION_RANK(3) |
| 246 |
EXPLICIT_TEMPLATE_INSTANTIATION_RANK(4) |
| 247 |
EXPLICIT_TEMPLATE_INSTANTIATION_RANK(5) |
| 248 |
|
| 249 |
} //namespace atlas::parallel::detail |