Debian Codegraph @

src/atlas/parallel/detail/DevicePacker.hic

raw
1 /*
2 * (C) Copyright 2025- ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation nor
8 * does it submit to any jurisdiction.
9 */
10
11 #include "Packer.h"
12 #include "pack_index.h"
13 #include "atlas/array/SVector.h"
14 #include "atlas/runtime/Exception.h"
15
16 #include "hic/hic.h"
17
18 namespace atlas::parallel::detail {
19
20 bool is_device_accessible(const void* ptr) {
21 hicPointerAttributes attr;
22 hicError_t code = hicPointerGetAttributes(&attr, ptr);
23 if( code != hicSuccess ) {
24 static_cast<void>(hicGetLastError());
25 return false;
26 }
27 return ptr == attr.devicePointer;
28 }
29
30 template<int ParallelDim, int RANK>
31 struct get_buffer_index{
32 template<typename DATA_TYPE>
33 ATLAS_HOST_DEVICE
34 static idx_t apply(const array::ArrayView<DATA_TYPE, RANK>& field,
35 const idx_t node_cnt, const idx_t var1_idx) {
36 return field.shape(RANK-1) * field.shape(RANK-2) * node_cnt +
37 field.shape(RANK-1) * var1_idx;
38 }
39 };
40
41 template<int ParallelDim>
42 struct get_buffer_index<ParallelDim, 2>{
43 template<typename DATA_TYPE>
44 ATLAS_HOST_DEVICE
45 static idx_t apply(const array::ArrayView<DATA_TYPE, 2>& field,
46 const idx_t node_cnt, const idx_t var1_idx) {
47 return field.shape(1) * node_cnt + var1_idx;
48 }
49 };
50
51 template<int ParallelDim, typename DATA_TYPE, int RANK>
52 __global__ void pack_kernel(const int sendcnt, const int* sendmap_ptr, const idx_t sendmap_size,
53 const array::ArrayView<DATA_TYPE, RANK> field, DATA_TYPE* send_buffer,
54 const idx_t send_buffer_size, const typename std::enable_if<RANK==1, int>::type = 0) {
55 const array::SVector<int> sendmap(const_cast<int*>(sendmap_ptr), sendmap_size);
56
57 const idx_t node_cnt = blockIdx.x*blockDim.x + threadIdx.x;
58
59 if(node_cnt >= sendcnt) return;
60
61 send_buffer[node_cnt] = field(sendmap[node_cnt]);
62 }
63
64 template<int ParallelDim, typename DATA_TYPE, int RANK>
65 __global__ void pack_kernel(const int sendcnt, const int* sendmap_ptr, const idx_t sendmap_size,
66 const array::ArrayView<DATA_TYPE, RANK> field, DATA_TYPE* send_buffer,
67 const idx_t send_buffer_size, const typename std::enable_if<RANK>=2, int>::type = 0) {
68 const array::SVector<int> sendmap(const_cast<int*>(sendmap_ptr), sendmap_size);
69
70 const idx_t node_cnt = blockIdx.x*blockDim.x + threadIdx.x;
71 const idx_t var1_idx = blockIdx.y*blockDim.y + threadIdx.y;
72
73 if(node_cnt >= sendcnt || var1_idx >= field.shape(1) ) return;
74
75 idx_t buff_idx = get_buffer_index<ParallelDim, RANK>::apply(field, node_cnt, var1_idx);
76 const idx_t node_idx = sendmap[node_cnt];
77
78 pack_index<0, (RANK>=3) ? (RANK-2) : 0, 2>::apply(buff_idx, node_idx, field, send_buffer, node_idx,var1_idx);
79 }
80
81 template<int ParallelDim, typename DATA_TYPE, int RANK>
82 __global__ void unpack_kernel(const int recvcnt, const int* recvmap_ptr, const idx_t recvmap_size,
83 const DATA_TYPE* recv_buffer, const idx_t recv_buffer_size, array::ArrayView<DATA_TYPE, RANK> field, const typename std::enable_if<RANK==1, int>::type = 0) {
84
85 const array::SVector<int> recvmap(const_cast<int*>(recvmap_ptr), recvmap_size);
86
87 idx_t node_cnt = blockIdx.x*blockDim.x + threadIdx.x;
88
89 if(node_cnt >= recvcnt) return;
90
91 const idx_t node_idx = recvmap[node_cnt];
92
93 field(node_idx) = recv_buffer[node_cnt];
94 }
95
96 template<int ParallelDim, typename DATA_TYPE, int RANK>
97 __global__ void unpack_kernel(const int recvcnt, const int* recvmap_ptr, const idx_t recvmap_size,
98 const DATA_TYPE* recv_buffer, const idx_t recv_buffer_size, array::ArrayView<DATA_TYPE, RANK> field, const typename std::enable_if<RANK>=2, int>::type = 0) {
99
100 const array::SVector<int> recvmap(const_cast<int*>(recvmap_ptr), recvmap_size);
101
102 const idx_t node_cnt = blockIdx.x*blockDim.x + threadIdx.x;
103 const idx_t var1_idx = blockIdx.y*blockDim.y + threadIdx.y;
104
105 if(node_cnt >= recvcnt || var1_idx >= field.shape(1) ) return;
106
107 const idx_t node_idx = recvmap[node_cnt];
108
109 idx_t buff_idx = get_buffer_index<ParallelDim, RANK>::apply(field, node_cnt, var1_idx);
110
111 unpack_index<0, (RANK>=3) ? (RANK-2) : 0, 2>::apply(buff_idx, node_idx, recv_buffer, field,node_idx,var1_idx);
112 }
113
114 template<int ParallelDim, int RANK, int DimCnt>
115 struct get_first_non_parallel_dim
116 {
117 static_assert((ParallelDim <= RANK), "Error: parallelDim larger than RANK");
118 constexpr static int apply() {
119 return (DimCnt == ParallelDim) ? get_first_non_parallel_dim<ParallelDim, RANK, DimCnt+1>::apply() : DimCnt;
120 }
121 };
122
123 template<int ParallelDim, int RANK>
124 struct get_first_non_parallel_dim<ParallelDim, RANK, RANK>
125 {
126 static_assert((ParallelDim <= RANK), "Error: parallelDim larger than RANK");
127 constexpr static int apply() {
128 return -1;
129 }
130 };
131
132 template<int ParallelDim, int RANK>
133 struct get_n_hic_blocks
134 {
135 template<typename DATA_TYPE>
136 static unsigned int apply(const array::ArrayView<DATA_TYPE, RANK>& hfield, const unsigned int block_size_y) {
137 constexpr auto dim = get_first_non_parallel_dim<ParallelDim, RANK, 0>::apply();
138 return (hfield.shape(dim)+block_size_y-1)/block_size_y;
139 }
140 };
141
142 template<>
143 struct get_n_hic_blocks<0, 1> {
144 template<typename DATA_TYPE>
145 static unsigned int apply(const array::ArrayView<DATA_TYPE, 1>& hfield, const unsigned int block_size_y) {
146 return 1;
147 }
148 };
149
150 template<int ParallelDim, typename DATA_TYPE, int RANK>
151 void DevicePacker<ParallelDim, DATA_TYPE, RANK>::pack( const int sendcnt, array::SVector<int> const & sendmap,
152 const array::ArrayView<DATA_TYPE, RANK>& array,
153 DATA_TYPE* send_buffer, int send_buffer_size )
154 {
155 const unsigned int block_size_x = 32;
156 const unsigned int block_size_y = (RANK==1) ? 1 : 4;
157
158 const unsigned int nblocks_x = (sendcnt+block_size_x-1)/block_size_x;
159 const unsigned int nblocks_y = get_n_hic_blocks<ParallelDim, RANK>::apply(array, block_size_y);
160
161 dim3 threads(block_size_x, block_size_y);
162 dim3 blocks(nblocks_x, nblocks_y);
163 HIC_CALL(hicDeviceSynchronize());
164 hicError_t err = hicGetLastError();
165 if (err != hicSuccess) {
166 std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err);
167 throw_Exception(msg);
168 }
169
170 pack_kernel<ParallelDim, DATA_TYPE, RANK><<<blocks,threads>>>(sendcnt, sendmap.data(), sendmap.size(), array, send_buffer, send_buffer_size);
171 err = hicGetLastError();
172 if (err != hicSuccess)
173 throw_Exception("Error launching GPU packing kernel");
174
175 HIC_CALL(hicDeviceSynchronize());
176 err = hicGetLastError();
177 if (err != hicSuccess) {
178 std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err);
179 throw_Exception(msg);
180 }
181
182 }
183
184 template<int ParallelDim, typename DATA_TYPE, int RANK>
185 void DevicePacker<ParallelDim, DATA_TYPE, RANK>::unpack(const int recvcnt, array::SVector<int> const & recvmap,
186 const DATA_TYPE* recv_buffer , int recv_buffer_size,
187 array::ArrayView<DATA_TYPE, RANK>& array)
188 {
189 const unsigned int block_size_x = 32;
190 const unsigned int block_size_y = (RANK==1) ? 1 : 4;
191
192 unsigned int nblocks_y = get_n_hic_blocks<ParallelDim, RANK>::apply(array, block_size_y);
193
194 dim3 threads(block_size_x, block_size_y);
195 dim3 blocks((recvcnt+block_size_x-1)/block_size_x, nblocks_y);
196
197 HIC_CALL(hicDeviceSynchronize());
198 hicError_t err = hicGetLastError();
199 if (err != hicSuccess) {
200 std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err);
201 throw_Exception(msg);
202 }
203
204 unpack_kernel<ParallelDim, DATA_TYPE, RANK><<<blocks,threads>>>(recvcnt, recvmap.data(), recvmap.size(), recv_buffer, recv_buffer_size, array);
205
206 err = hicGetLastError();
207 if (err != hicSuccess) {
208 std::string msg = std::string("Error launching GPU packing kernel")+ hicGetErrorString(err);
209 throw_Exception(msg);
210 }
211
212 HIC_CALL(hicDeviceSynchronize());
213 err = hicGetLastError();
214 if (err != hicSuccess) {
215 std::string msg = std::string("Error synchronizing device")+ hicGetErrorString(err);
216 throw_Exception(msg);
217 }
218 }
219
220
221 #define ATLAS_REPEAT_MACRO(n, m, p) ATLAS_REPEAT_MACRO_ ## n(m, p)
222 // expands to m(0,p) m(1,p) ... m(n-1,p)
223
224 #define ATLAS_REPEAT_MACRO_0(m, p)
225 #define ATLAS_REPEAT_MACRO_1(m, p) ATLAS_REPEAT_MACRO_0(m, p) m(0, p)
226 #define ATLAS_REPEAT_MACRO_2(m, p) ATLAS_REPEAT_MACRO_1(m, p) m(1, p)
227 #define ATLAS_REPEAT_MACRO_3(m, p) ATLAS_REPEAT_MACRO_2(m, p) m(2, p)
228 #define ATLAS_REPEAT_MACRO_4(m, p) ATLAS_REPEAT_MACRO_3(m, p) m(3, p)
229 #define ATLAS_REPEAT_MACRO_5(m, p) ATLAS_REPEAT_MACRO_4(m, p) m(4, p)
230 #define ATLAS_REPEAT_MACRO_6(m, p) ATLAS_REPEAT_MACRO_5(m, p) m(5, p)
231
232
233 #define EXPLICIT_TEMPLATE_INSTANTIATION( ParallelDim, RANK ) \
234 template struct DevicePacker<ParallelDim, int, RANK>; \
235 template struct DevicePacker<ParallelDim, long, RANK>; \
236 template struct DevicePacker<ParallelDim, long unsigned, RANK>; \
237 template struct DevicePacker<ParallelDim, float, RANK>; \
238 template struct DevicePacker<ParallelDim, double, RANK>;
239
240 #define EXPLICIT_TEMPLATE_INSTANTIATION_RANK(RANK) \
241 ATLAS_REPEAT_MACRO(RANK, EXPLICIT_TEMPLATE_INSTANTIATION, RANK)
242
243 EXPLICIT_TEMPLATE_INSTANTIATION_RANK(1)
244 EXPLICIT_TEMPLATE_INSTANTIATION_RANK(2)
245 EXPLICIT_TEMPLATE_INSTANTIATION_RANK(3)
246 EXPLICIT_TEMPLATE_INSTANTIATION_RANK(4)
247 EXPLICIT_TEMPLATE_INSTANTIATION_RANK(5)
248
249 } //namespace atlas::parallel::detail