Debian Codegraph @

include/deal.II/base/mpi_large_count.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2022 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15 //
16 // The content of this file is a slightly modified version of the
17 // header-only library BigMPICompat - a tiny MPI 4.x compatibility
18 // library released under MIT license at
19 // https://github.com/tjhei/BigMPICompat and relicensed and included
20 // here with permission.
21 //
22
23 #ifndef BIG_MPI_COMPAT_H
24 #define BIG_MPI_COMPAT_H
25
26 #include <deal.II/base/config.h>
27
28 #ifdef DEAL_II_WITH_MPI
29
30 DEAL_II_DISABLE_EXTRA_DIAGNOSTICS
31 # include <mpi.h>
32 DEAL_II_ENABLE_EXTRA_DIAGNOSTICS
33
34 // required for std::numeric_limits used below.
35 # include <limits>
36
37 DEAL_II_NAMESPACE_OPEN
38
39 namespace Utilities
40 {
41 namespace MPI
42 {
43 /**
44 * This namespace contains symbols to support MPI routines with
45 * large "counts" on MPI implementations that implement version
46 * 3.x of the standard, where @p count is a signed integer.
47 */
48 namespace LargeCount
49 {
50 /**
51 * This is the largest @p count supported when it is represented
52 * with a signed integer (old MPI routines).
53 */
54 inline constexpr MPI_Count mpi_max_int_count =
55 std::numeric_limits<int>::max();
56
57 /**
58 * Create a contiguous type of (possibly large) @p count.
59 *
60 * See the MPI 4.x standard for details.
61 */
62 inline int
63 Type_contiguous_c(MPI_Count count,
64 MPI_Datatype oldtype,
65 MPI_Datatype *newtype)
66 {
67 # if MPI_VERSION >= 4
68 return MPI_Type_contiguous_c(count, oldtype, newtype);
69 # else
70 if (count <= LargeCount::mpi_max_int_count)
71 return MPI_Type_contiguous(count, oldtype, newtype);
72 else
73 {
74 int ierr;
75 const MPI_Count max_signed_int = LargeCount::mpi_max_int_count;
76
77 MPI_Count size_old;
78 ierr = MPI_Type_size_x(oldtype, &size_old);
79
80 MPI_Count n_chunks = count / max_signed_int;
81 MPI_Count n_remaining_elements = count % max_signed_int;
82
83 MPI_Datatype chunks;
84 ierr = MPI_Type_vector(
85 n_chunks, max_signed_int, max_signed_int, oldtype, &chunks);
86 if (ierr != MPI_SUCCESS)
87 return ierr;
88
89 MPI_Datatype remainder;
90 ierr =
91 MPI_Type_contiguous(n_remaining_elements, oldtype, &remainder);
92 if (ierr != MPI_SUCCESS)
93 return ierr;
94
95 int blocklengths[2] = {1, 1};
96 MPI_Aint displacements[2] = {0,
97 static_cast<MPI_Aint>(n_chunks) *
98 size_old * max_signed_int};
99 MPI_Datatype types[2] = {chunks, remainder};
100 ierr = MPI_Type_create_struct(
101 2, blocklengths, displacements, types, newtype);
102 if (ierr != MPI_SUCCESS)
103 return ierr;
104
105 ierr = MPI_Type_commit(newtype);
106 if (ierr != MPI_SUCCESS)
107 return ierr;
108
109 ierr = MPI_Type_free(&chunks);
110 if (ierr != MPI_SUCCESS)
111 return ierr;
112
113 ierr = MPI_Type_free(&remainder);
114 if (ierr != MPI_SUCCESS)
115 return ierr;
116
117 # ifndef MPI_COMPAT_SKIP_SIZE_CHECK
118 MPI_Count size_new;
119 ierr = MPI_Type_size_x(*newtype, &size_new);
120 if (ierr != MPI_SUCCESS)
121 return ierr;
122
123 if (size_old * count != size_new)
124 {
125 // This error can happen when you are using a very old and
126 // buggy MPI implementation. There is nothing we can do
127 // here, unfortunately. Please update your installation.
128 // std::cerr
129 //<< "MPI_Type_contiguous_c() produced an invalid result.
130 // Expected =
131 //"
132 //<< size_old << " * " << count << " = " << size_old * count
133 //<< " but received " << size_new << std::endl;
134 return MPI_ERR_INTERN;
135 }
136 # endif
137
138 return MPI_SUCCESS;
139 }
140 # endif
141 }
142
143
144 /**
145 * Send a package to rank @p dest with a (possibly large) @p count.
146 *
147 * See the MPI 4.x standard for details.
148 */
149 inline int
150 Send_c(const void *buf,
151 MPI_Count count,
152 MPI_Datatype datatype,
153 int dest,
154 int tag,
155 MPI_Comm comm)
156 {
157 # if MPI_VERSION >= 4
158 return MPI_Send_c(buf, count, datatype, dest, tag, comm);
159 # else
160 if (count <= LargeCount::mpi_max_int_count)
161 return MPI_Send(buf, count, datatype, dest, tag, comm);
162
163 MPI_Datatype bigtype;
164 int ierr;
165 ierr = Type_contiguous_c(count, datatype, &bigtype);
166 if (ierr != MPI_SUCCESS)
167 return ierr;
168 ierr = MPI_Type_commit(&bigtype);
169 if (ierr != MPI_SUCCESS)
170 return ierr;
171
172 ierr = MPI_Send(buf, 1, bigtype, dest, tag, comm);
173 if (ierr != MPI_SUCCESS)
174 return ierr;
175
176 ierr = MPI_Type_free(&bigtype);
177 if (ierr != MPI_SUCCESS)
178 return ierr;
179 return MPI_SUCCESS;
180 # endif
181 }
182
183 /**
184 * Receive a package from rank @p source with a (possibly large) @p count.
185 *
186 * See the MPI 4.x standard for details.
187 */
188 inline int
189 Recv_c(void *buf,
190 MPI_Count count,
191 MPI_Datatype datatype,
192 int source,
193 int tag,
194 MPI_Comm comm,
195 MPI_Status *status)
196 {
197 # if MPI_VERSION >= 4
198 return MPI_Recv_c(buf, count, datatype, source, tag, comm, status);
199 # else
200 if (count <= LargeCount::mpi_max_int_count)
201 return MPI_Recv(buf, count, datatype, source, tag, comm, status);
202
203 MPI_Datatype bigtype;
204 int ierr;
205 ierr = Type_contiguous_c(count, datatype, &bigtype);
206 if (ierr != MPI_SUCCESS)
207 return ierr;
208
209 ierr = MPI_Type_commit(&bigtype);
210 if (ierr != MPI_SUCCESS)
211 return ierr;
212
213 ierr = MPI_Recv(buf, 1, bigtype, source, tag, comm, status);
214 if (ierr != MPI_SUCCESS)
215 return ierr;
216
217 ierr = MPI_Type_free(&bigtype);
218 if (ierr != MPI_SUCCESS)
219 return ierr;
220 return MPI_SUCCESS;
221 # endif
222 }
223
224 /**
225 * Broadcast a message of possibly large @p count of data from
226 * the process with rank "root" to all other processes.
227 *
228 * See the MPI 4.x standard for details.
229 */
230 inline int
231 Bcast_c(void *buf,
232 MPI_Count count,
233 MPI_Datatype datatype,
234 unsigned int root_mpi_rank,
235 MPI_Comm comm)
236 {
237 # if MPI_VERSION >= 4
238 return MPI_Bcast_c(buf, count, datatype, root_mpi_rank, comm);
239 # else
240 if (count <= LargeCount::mpi_max_int_count)
241 return MPI_Bcast(buf, count, datatype, root_mpi_rank, comm);
242
243 MPI_Datatype bigtype;
244 int ierr;
245 ierr = Type_contiguous_c(count, datatype, &bigtype);
246 if (ierr != MPI_SUCCESS)
247 return ierr;
248 ierr = MPI_Type_commit(&bigtype);
249 if (ierr != MPI_SUCCESS)
250 return ierr;
251
252 ierr = MPI_Bcast(buf, 1, bigtype, root_mpi_rank, comm);
253 if (ierr != MPI_SUCCESS)
254 return ierr;
255
256 ierr = MPI_Type_free(&bigtype);
257 if (ierr != MPI_SUCCESS)
258 return ierr;
259 return MPI_SUCCESS;
260 # endif
261 }
262
263 /**
264 * Write a possibly large @p count of data at the location @p offset.
265 *
266 * See the MPI 4.x standard for details.
267 */
268 inline int
269 File_write_at_c(MPI_File fh,
270 MPI_Offset offset,
271 const void *buf,
272 MPI_Count count,
273 MPI_Datatype datatype,
274 MPI_Status *status)
275 {
276 if (count <= LargeCount::mpi_max_int_count)
277 return MPI_File_write_at(fh, offset, buf, count, datatype, status);
278
279 MPI_Datatype bigtype;
280 int ierr;
281 ierr = Type_contiguous_c(count, datatype, &bigtype);
282 if (ierr != MPI_SUCCESS)
283 return ierr;
284 ierr = MPI_Type_commit(&bigtype);
285 if (ierr != MPI_SUCCESS)
286 return ierr;
287
288 ierr = MPI_File_write_at(fh, offset, buf, 1, bigtype, status);
289 if (ierr != MPI_SUCCESS)
290 return ierr;
291
292 ierr = MPI_Type_free(&bigtype);
293 if (ierr != MPI_SUCCESS)
294 return ierr;
295 return MPI_SUCCESS;
296 }
297
298 /**
299 * Collectively write a possibly large @p count of data at the
300 * location @p offset.
301 *
302 * See the MPI 4.x standard for details.
303 */
304 inline int
305 File_write_at_all_c(MPI_File fh,
306 MPI_Offset offset,
307 const void *buf,
308 MPI_Count count,
309 MPI_Datatype datatype,
310 MPI_Status *status)
311 {
312 if (count <= LargeCount::mpi_max_int_count)
313 return MPI_File_write_at_all(
314 fh, offset, buf, count, datatype, status);
315
316 MPI_Datatype bigtype;
317 int ierr;
318 ierr = Type_contiguous_c(count, datatype, &bigtype);
319 if (ierr != MPI_SUCCESS)
320 return ierr;
321 ierr = MPI_Type_commit(&bigtype);
322 if (ierr != MPI_SUCCESS)
323 return ierr;
324
325 ierr = MPI_File_write_at_all(fh, offset, buf, 1, bigtype, status);
326 if (ierr != MPI_SUCCESS)
327 return ierr;
328
329 ierr = MPI_Type_free(&bigtype);
330 if (ierr != MPI_SUCCESS)
331 return ierr;
332 return MPI_SUCCESS;
333 }
334
335 /**
336 * Collectively write a possibly large @p count of data in order.
337 *
338 * See the MPI 4.x standard for details.
339 */
340 inline int
341 File_write_ordered_c(MPI_File fh,
342 const void *buf,
343 MPI_Count count,
344 MPI_Datatype datatype,
345 MPI_Status *status)
346 {
347 if (count <= LargeCount::mpi_max_int_count)
348 return MPI_File_write_ordered(fh, buf, count, datatype, status);
349
350 MPI_Datatype bigtype;
351 int ierr;
352 ierr = Type_contiguous_c(count, datatype, &bigtype);
353 if (ierr != MPI_SUCCESS)
354 return ierr;
355 ierr = MPI_Type_commit(&bigtype);
356 if (ierr != MPI_SUCCESS)
357 return ierr;
358
359 ierr = MPI_File_write_ordered(fh, buf, 1, bigtype, status);
360 if (ierr != MPI_SUCCESS)
361 return ierr;
362
363 ierr = MPI_Type_free(&bigtype);
364 if (ierr != MPI_SUCCESS)
365 return ierr;
366 return MPI_SUCCESS;
367 }
368
369 /**
370 * Read a possibly large @p count of data at the
371 * location @p offset.
372 *
373 * See the MPI 4.x standard for details.
374 */
375 inline int
376 File_read_at_c(MPI_File fh,
377 MPI_Offset offset,
378 void *buf,
379 MPI_Count count,
380 MPI_Datatype datatype,
381 MPI_Status *status)
382 {
383 if (count <= LargeCount::mpi_max_int_count)
384 return MPI_File_read_at(fh, offset, buf, count, datatype, status);
385
386 MPI_Datatype bigtype;
387 int ierr;
388 ierr = Type_contiguous_c(count, datatype, &bigtype);
389 if (ierr != MPI_SUCCESS)
390 return ierr;
391 ierr = MPI_Type_commit(&bigtype);
392 if (ierr != MPI_SUCCESS)
393 return ierr;
394
395 ierr = MPI_File_read_at(fh, offset, buf, 1, bigtype, status);
396 if (ierr != MPI_SUCCESS)
397 return ierr;
398
399 ierr = MPI_Type_free(&bigtype);
400 if (ierr != MPI_SUCCESS)
401 return ierr;
402 return MPI_SUCCESS;
403 }
404
405 /**
406 * Collectively read a possibly large @p count of data at the
407 * location @p offset.
408 *
409 * See the MPI 4.x standard for details.
410 */
411 inline int
412 File_read_at_all_c(MPI_File fh,
413 MPI_Offset offset,
414 void *buf,
415 MPI_Count count,
416 MPI_Datatype datatype,
417 MPI_Status *status)
418 {
419 if (count <= LargeCount::mpi_max_int_count)
420 return MPI_File_read_at_all(fh, offset, buf, count, datatype, status);
421
422 MPI_Datatype bigtype;
423 int ierr;
424 ierr = Type_contiguous_c(count, datatype, &bigtype);
425 if (ierr != MPI_SUCCESS)
426 return ierr;
427 ierr = MPI_Type_commit(&bigtype);
428 if (ierr != MPI_SUCCESS)
429 return ierr;
430
431 ierr = MPI_File_read_at_all(fh, offset, buf, 1, bigtype, status);
432 if (ierr != MPI_SUCCESS)
433 return ierr;
434
435 ierr = MPI_Type_free(&bigtype);
436 if (ierr != MPI_SUCCESS)
437 return ierr;
438 return MPI_SUCCESS;
439 }
440
441 } // namespace LargeCount
442 } // namespace MPI
443 } // namespace Utilities
444
445 DEAL_II_NAMESPACE_CLOSE
446
447 #else
448
449 // Make sure the scripts that create the C++20 module input files have
450 // something to latch on if the preprocessor #ifdef above would
451 // otherwise lead to an empty content of the file.
452 DEAL_II_NAMESPACE_OPEN
453 DEAL_II_NAMESPACE_CLOSE
454
455 #endif
456 #endif