| 1 |
# SPDX-FileCopyrightText: 2026 Oak Ridge National Laboratory and Contributors |
| 2 |
# |
| 3 |
# SPDX-License-Identifier: Apache-2.0 |
| 4 |
|
| 5 |
import numpy as np # pylint: disable=import-error |
| 6 |
|
| 7 |
|
| 8 |
def Locate(rank, nproc, datasize): |
| 9 |
extra = 0 |
| 10 |
if rank == nproc - 1: |
| 11 |
extra = datasize % nproc |
| 12 |
num = datasize // nproc |
| 13 |
start = num * rank |
| 14 |
size = num + extra |
| 15 |
return start, size |
| 16 |
|
| 17 |
|
| 18 |
class MPISetup(object): |
| 19 |
readargs = [] |
| 20 |
size = 1 |
| 21 |
rank = {"app": 0, "x": 0, "y": 0, "z": 0} |
| 22 |
|
| 23 |
def __init__(self, args, appID): |
| 24 |
|
| 25 |
self.nx = args.nx |
| 26 |
self.ny = args.ny |
| 27 |
self.nz = args.nz |
| 28 |
|
| 29 |
if not args.nompi: |
| 30 |
from mpi4py import MPI |
| 31 |
|
| 32 |
self.comm_app = MPI.COMM_WORLD.Split(appID, MPI.COMM_WORLD.Get_rank()) |
| 33 |
self.size = self.comm_app.Get_size() |
| 34 |
self.rank["app"] = self.comm_app.Get_rank() |
| 35 |
if self.nx * self.ny * self.nz == 1: |
| 36 |
self.nx = self.size |
| 37 |
if self.size != (self.nx * self.ny * self.nz): |
| 38 |
raise ValueError("nx * ny * nz != num processes") |
| 39 |
|
| 40 |
if (self.ny > 1) and (self.nx > 1) and (self.nz > 1): |
| 41 |
comm_x = self.comm_app.Split(self.rank["app"] % self.nx, self.rank["app"]) |
| 42 |
else: |
| 43 |
comm_x = self.comm_app.Split(self.rank["app"] / self.nx, self.rank["app"]) |
| 44 |
comm_y = self.comm_app.Split(self.rank["app"] / self.ny, self.rank["app"]) |
| 45 |
comm_z = self.comm_app.Split(self.rank["app"] / self.nz, self.rank["app"]) |
| 46 |
|
| 47 |
self.rank["x"] = comm_x.Get_rank() |
| 48 |
self.rank["y"] = comm_y.Get_rank() |
| 49 |
self.rank["z"] = comm_z.Get_rank() |
| 50 |
|
| 51 |
self.readargs.append(self.comm_app) |
| 52 |
|
| 53 |
else: |
| 54 |
if self.nx != 1: |
| 55 |
raise ValueError("nx must = 1 without MPI") |
| 56 |
if self.ny != 1: |
| 57 |
raise ValueError("ny must = 1 without MPI") |
| 58 |
self.comm_app = None |
| 59 |
|
| 60 |
# self.readargs.extend([args.xmlfile, "heat"]) |
| 61 |
|
| 62 |
def Partition_3D_3D(self, fp, args): |
| 63 |
datashape = np.zeros(3, dtype=np.int64) |
| 64 |
start = np.zeros(3, dtype=np.int64) |
| 65 |
size = np.zeros(3, dtype=np.int64) |
| 66 |
|
| 67 |
# print("dir fp {0}".format(dir(fp))) |
| 68 |
var = fp.available_variables() |
| 69 |
# print("var {0}".format(var)) |
| 70 |
# print('key') |
| 71 |
# print('type {0}'.format(type(var))) |
| 72 |
# print("args varname {0}".format(args.varname)) |
| 73 |
# print('var keys') |
| 74 |
# print(var.keys()) |
| 75 |
dshape = var[args.varname]["Shape"].split(",") |
| 76 |
for i in range(len(dshape)): |
| 77 |
datashape[i] = int(dshape[i]) |
| 78 |
|
| 79 |
start[0], size[0] = Locate(self.rank["y"], self.ny, datashape[0]) |
| 80 |
start[1], size[1] = Locate(self.rank["x"], self.nx, datashape[1]) |
| 81 |
start[0], size[0] = Locate(self.rank["y"], self.ny, datashape[0]) |
| 82 |
|
| 83 |
return start, size, datashape |
| 84 |
|
| 85 |
def Partition_3D_1D(self, fp, args): |
| 86 |
datashape = np.zeros(3, dtype=np.int64) |
| 87 |
start = np.zeros(3, dtype=np.int64) |
| 88 |
size = np.zeros(3, dtype=np.int64) |
| 89 |
|
| 90 |
# print("dir fp {0}".format(dir(fp))) |
| 91 |
var = fp.availablevariables() |
| 92 |
# print("var {0}".format(var)) |
| 93 |
# print('key') |
| 94 |
# print('type {0}'.format(type(var))) |
| 95 |
# print("args varname {0}".format(args.varname)) |
| 96 |
# print('var keys') |
| 97 |
# print(var.keys()) |
| 98 |
dshape = var[args.varname]["Shape"].split(",") |
| 99 |
for i in range(len(dshape)): |
| 100 |
datashape[i] = int(dshape[i]) |
| 101 |
|
| 102 |
start[0], size[0] = Locate(self.rank["app"], self.size, datashape[0]) |
| 103 |
start[1], size[1] = (0, datashape[1]) |
| 104 |
start[2], size[2] = (0, datashape[2]) |
| 105 |
|
| 106 |
return start, size, datashape |
| 107 |
|
| 108 |
def Partition_2D_1D(self, fp, args): |
| 109 |
datashape = np.zeros(2, dtype=np.int64) |
| 110 |
start = np.zeros(2, dtype=np.int64) |
| 111 |
size = np.zeros(2, dtype=np.int64) |
| 112 |
|
| 113 |
var = fp.availablevariables() |
| 114 |
dshape = var[args.varname]["Shape"].split(",") |
| 115 |
for i in range(len(dshape)): |
| 116 |
datashape[i] = int(dshape[i]) |
| 117 |
|
| 118 |
start[0], size[0] = Locate(self.rank["app"], self.size, datashape[0]) |
| 119 |
start[1], size[1] = (0, datashape[1]) |
| 120 |
|
| 121 |
return start, size, datashape |