Debian Codegraph @

kernel-boot/rdma_topo

raw
1 #!/usr/bin/env python3
2 # SPDX-License-Identifier: Linux-OpenIB
3 # Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES
4 # PYTHON_ARGCOMPLETE_OK
5 from __future__ import annotations
6 import argparse
7 import collections
8 import importlib
9 import inspect
10 import itertools
11 import os
12 import re
13 import subprocess
14 import sys
15 import tempfile
16
17 from typing import *
18
19 BDF_RE = re.compile(r"^([0-9a-f]+?):([0-9a-f]{2}?):([0-9a-f]{2}?)\.([0-9a-f])$")
20 KERNEL_ACS_ISOLATED = "xx111x1"
21 pci_vendors = {
22 "MELLANOX": 0x15B3,
23 "NVIDIA": 0x10DE,
24 }
25
26
27 class CommandError(Exception):
28 pass
29
30
31 def sysfs_read_str(sysdir: str, fn: str) -> str:
32 """Read the entire content of a sysfs file to a string"""
33 with open(os.path.join(sysdir, fn)) as F:
34 return F.read().strip()
35
36
37 def sysfs_read_link(sysdir: str, fn: str) -> str:
38 """Read a link in sysfs to an absolute path string"""
39 return os.readlink(os.path.join(sysdir, fn))
40
41
42 def PCI_VDEVICE(vendor: str, device_id: int) -> re.Pattern:
43 """Match a Vendor and device ID"""
44 vendor_id = pci_vendors[vendor]
45 return re.compile(rf"^pci:v{vendor_id:08X}d{device_id:08X}.*$")
46
47
48 def PCI_DEVICE_CLASS(cid: int) -> re.Pattern:
49 """Match by exact programming class using the int coding from the kernel"""
50 class_id = (cid >> 16) & 0xFF
51 subclass_id = (cid >> 8) & 0xFF
52 progif = cid & 0xFF
53 return re.compile(rf"^pci:.*bc{class_id:02X}sc{subclass_id:02X}i{progif:02X}.*$")
54
55
56 def PCI_NVGPU() -> re.Pattern:
57 """Match all NVIDIA GPUs"""
58 vendor_id = pci_vendors["NVIDIA"]
59 class_id = 0x03
60 return re.compile(rf"^pci:v{vendor_id:08X}.*bc{class_id:02X}.*$")
61
62
63 # Table of modalias matches to the device_type string
64 pci_device_types = {
65 PCI_VDEVICE("NVIDIA", 0x22B1): "grace_rp", # NVIDIA Grace PCI Root Port Bridge
66 PCI_VDEVICE("NVIDIA", 0x22B2): "grace_rp", # NVIDIA Grace PCI Root Port Bridge
67 PCI_VDEVICE("NVIDIA", 0x22B8): "grace_rp", # NVIDIA Grace PCI Root Port Bridge
68 PCI_VDEVICE("MELLANOX", 0x1021): "cx_nic", # ConnectX-7
69 PCI_VDEVICE("MELLANOX", 0x1023): "cx_nic", # ConnectX-8
70 PCI_VDEVICE("MELLANOX", 0xA2DC): "bf3_nic", # BlueField-3
71 PCI_VDEVICE("MELLANOX", 0x2100): "cx_dma", # ConnectX-8 DMA Controller
72 PCI_VDEVICE("MELLANOX", 0x197B): "bf3_switch", # USP/DSP of a BF3 switch
73 PCI_VDEVICE("MELLANOX", 0x197C): "cx_switch", # USP/DSP of a CX switch
74 PCI_DEVICE_CLASS(0x010802): "nvme",
75 PCI_NVGPU(): "nvgpu",
76 }
77
78
79 class PCIBDF(
80 collections.namedtuple("PCIBDF", ["segment", "bus", "device", "function"])
81 ):
82 """Bus Device Function for a PCI device"""
83
84 def as_pci(self):
85 return f"{self.segment}:{self.bus}:{self.device}.{self.function}"
86
87 def __str__(self):
88 return self.as_pci()
89
90 def __repr__(self):
91 return f"PCIBDF({self.segment}, {self.bus}, {self.device}, {self.function})"
92
93
94 def to_pcibdf(s: str) -> Optional[PCIBDF]:
95 g = BDF_RE.match(s)
96 if not g:
97 return None
98 return PCIBDF(*g.groups())
99
100
101 class PCIDevice(object):
102 device_type = ""
103 vpd_v3: str = None
104 parent: PCIDevice = None
105 lspci_data: str = None
106
107 def __init__(self, sysdir: str, bdf: PCIBDF):
108 self.sysdir = sysdir
109 self.bdf = bdf
110 try:
111 self.iommu_group = int(
112 os.path.split(sysfs_read_link(sysdir, "iommu_group"))[-1]
113 )
114 except FileNotFoundError:
115 self.iommu_group = None
116
117 try:
118 self.numa_node = int(sysfs_read_str(sysdir, "numa_node"))
119 except FileNotFoundError:
120 self.numa_node = None
121
122 self.modalias = sysfs_read_str(sysdir, "modalias")
123 for k, v in pci_device_types.items():
124 if k.match(self.modalias):
125 self.device_type = v
126 break
127
128 sysdir = os.path.realpath(sysdir)
129 parent = os.path.basename(os.path.dirname(sysdir))
130 self.parent_bdf = to_pcibdf(parent)
131 self.children: Set[PCIDevice] = set()
132
133 def finish_loading(self):
134 """Do more expensive parsing operations"""
135 if self.device_type == "cx_nic" or self.device_type == "cx_dma":
136 self.vpd_v3 = self.parse_vpd_v3()
137 if "switch" in self.device_type or self.device_type == "grace_rp":
138 self.has_acs = self.parse_has_acs()
139
140 def iterdownstream(self) -> Generator[PCIDevice, None, None]:
141 """Iterate over all downstream devices of this device recursively"""
142 for pdev in self.children:
143 yield pdev
144 yield from pdev.iterdownstream()
145
146 def iterfulltree(self):
147 for pdev in self.iterupstream_path():
148 if not pdev.parent:
149 yield from pdev.iterdownstream()
150
151 def iterupstream_path(self):
152 """Iterate over each step along the upstream path from the devices
153 parent to the root."""
154 pdev = self.parent
155 while pdev:
156 yield pdev
157 pdev = pdev.parent
158
159 def __repr__(self):
160 return f"PCIDevice({self.bdf})"
161
162 def lspci(self):
163 """Fetch the verbose output of lspci"""
164 vpdfn = os.path.join(self.sysdir, "vpd")
165 if os.path.exists(vpdfn) and not os.access(vpdfn, os.R_OK):
166 raise CommandError(
167 f"Need access to the PCI VPD information in {vpdfn}, are you root?"
168 )
169
170 if not self.lspci_data:
171 self.lspci_data = subprocess.check_output(
172 ["lspci", "-s", f"{self.bdf.as_pci()}", "-vv"]
173 ).decode()
174 return self.lspci_data
175
176 def parse_vpd_v3(self):
177 """Use lspci to parse the VPD and get the V3 UUID, this only works as
178 root on non-secure boot systems."""
179 g = re.search(
180 r"Capabilities: \[.+?\] Vital Product Data$.*Read-only fields:$.*\[V3\] Vendor specific: (.*?)$.*End$",
181 self.lspci(),
182 re.DOTALL | re.MULTILINE,
183 )
184 if not g:
185 return None
186 return g.group(1)
187
188 def parse_has_acs(self):
189 """True if the device has an ACS capability"""
190 return bool(
191 re.search(
192 r"Capabilities: \[.+?\] Access Control Services$",
193 self.lspci(),
194 re.DOTALL | re.MULTILINE,
195 )
196 )
197
198 def parse_vpd_name(self):
199 g = re.search(
200 r"Capabilities: \[.+?\] Vital Product Data$.*Product Name: (.*?)$.*End$",
201 self.lspci(),
202 re.DOTALL | re.MULTILINE,
203 )
204 if not g:
205 return None
206 return g.group(1).strip()
207
208 def read_config(self, regname: str):
209 """Use setpci to read a register"""
210 return int(
211 subprocess.check_output(
212 ["setpci", "-r", "-s", str(self.bdf), "ECAP_ACS+0x6.w"]
213 )
214 .decode()
215 .strip(),
216 16,
217 )
218
219 def get_subsystems(self):
220 """Return a list of subsystem the PCI device is connected to"""
221 res: Dict[str, Set[str]] = collections.defaultdict(set)
222 for fn in os.listdir(self.sysdir):
223 if fn in {"drm", "infiniband", "net", "nvme"}:
224 res[fn].update(os.listdir(os.path.join(self.sysdir, fn)))
225 return res
226
227
228 class NVCX_Complex(object):
229 """Hold the related PCI functions together. A complex includes a CX PF, a CX
230 DMA function, an GPU and related PCI switches in the DMA function
231 segment."""
232
233 def __init__(self, cx_pfs: Set[PCIDevice], cx_dma: PCIDevice, nvgpu: PCIDevice):
234 self.cx_pfs = cx_pfs - {cx_dma}
235 self.cx_pf = sorted(self.cx_pfs, key=lambda x: x.bdf)[0]
236 self.cx_dma = cx_dma
237 self.nvgpu = nvgpu
238
239 # Identify the switch ports that are part of the shared path that
240 # handles the P2P traffic
241 self.shared_usp = self.__find_shared_usp()
242 for pdev in self.cx_dma.iterupstream_path():
243 if pdev in self.shared_usp.children:
244 self.cx_dma_dsp = pdev
245 for pdev in self.nvgpu.iterupstream_path():
246 if pdev in self.shared_usp.children:
247 self.nvgpu_dsp = pdev
248
249 # There can be a NVMe device connected to the CX NIC as well. For NVMe
250 # it is best to match with GPUs on the same socket, so a NUMA aware
251 # approach would be fine, but also the GPU/NIC/NVMe could be
252 # consistently paired based on the physical layout.
253 self.nvmes: Set[PCIDevice] = set()
254 for pdev in self.cx_pf.iterfulltree():
255 if pdev.device_type == "nvme":
256 self.nvmes.add(pdev)
257
258 def __find_shared_usp(self) -> PCIDevice:
259 """Find the USP that is shared by both devices, the immediate downstream
260 bus is the point in the topology where P2P traffic will switch from an
261 upstream to downstream direction."""
262 common_path = set(self.cx_dma.iterupstream_path()).intersection(
263 set(self.nvgpu.iterupstream_path())
264 )
265 assert common_path
266
267 for pdev in self.cx_dma.iterupstream_path():
268 if pdev in common_path:
269 assert pdev.device_type == "cx_switch"
270 for i in pdev.children:
271 assert i.device_type == "cx_switch"
272 return pdev
273
274 def get_subsystems(self):
275 subsystems: Dict[str, Set[str]] = collections.defaultdict(set)
276 for pdev in itertools.chain(self.cx_pfs, [self.nvgpu, self.cx_dma], self.nvmes):
277 for k, v in pdev.get_subsystems().items():
278 subsystems[k].update(v)
279 return subsystems
280
281
282 def check_parent(pdev: PCIDevice, parent_type: str):
283 if not pdev or not pdev.parent:
284 return None
285 if pdev.parent.device_type != parent_type:
286 return None
287 return pdev.parent
288
289
290 class PCITopo(object):
291 """Load the PCI topology from sysfs and organize it"""
292
293 def __init__(self):
294 self.devices = self.__load_devices("/sys/bus/pci/devices/")
295 self.has_cx_dma = any(
296 pdev.device_type == "cx_dma" for pdev in self.devices.values()
297 )
298 if self.has_cx_dma:
299 for pdev in self.devices.values():
300 pdev.finish_loading()
301 self.__build_topo()
302
303 def __load_devices(self, sysdir: str):
304 res: Dict[PCIBDF, PCIDevice] = {}
305 for fn in os.listdir(sysdir):
306 bdf = to_pcibdf(fn)
307 if not bdf:
308 continue
309 assert bdf not in res
310 res[bdf] = PCIDevice(os.path.join(sysdir, fn), bdf)
311 return res
312
313 def __get_nvcx_complex(self, cx_dma: PCIDevice):
314 """Match the topology for the switch complex using a CX DMA function and a
315 single GPU. It has two nested switches:
316
317 RP --> SW -> CX_DMA
318 -> SW -> GPU
319 """
320 assert cx_dma.device_type == "cx_dma"
321 if not cx_dma.vpd_v3:
322 raise ValueError(f"CX DMA function {cx_dma} does not have a VPD V3 UUID")
323
324 # The DMA and PF are matched using the UUID from the VPD
325 cx_pfs = self.vpd_v3s.get(cx_dma.vpd_v3)
326 if cx_pfs is None:
327 raise ValueError(
328 f"CX DMA function {cx_dma} does not have a matching PF, V3 UUID matching failed"
329 )
330 return None
331
332 # Path from the DMA to the root port
333 cx_dma_dsp = check_parent(cx_dma, "cx_switch")
334 cx_usp = check_parent(cx_dma_dsp, "cx_switch")
335 grace_rp = check_parent(cx_usp, "grace_rp")
336 if not grace_rp:
337 raise ValueError(
338 f"CX DMA function {cx_dma} has an unrecognized upstream path"
339 )
340
341 # Path from the GPU to the root port
342 nvgpus = [
343 pdev for pdev in grace_rp.iterdownstream() if pdev.device_type == "nvgpu"
344 ]
345 if len(nvgpus) != 1:
346 raise ValueError(f"CX DMA function {cx_dma} does not have a nearby GPU")
347 nvgpu = nvgpus[0]
348 nvgpu_dsp2 = check_parent(nvgpu, "cx_switch")
349 nvgpu_usp2 = check_parent(nvgpu_dsp2, "cx_switch")
350 nvgpu_dsp1 = check_parent(nvgpu_usp2, "cx_switch")
351 if cx_usp != check_parent(nvgpu_dsp1, "cx_switch"):
352 raise ValueError(
353 f"CX DMA function {cx_dma} has an unrecognized upstream path from the GPU"
354 )
355
356 # Sanity check there is nothing unexpected in the topology
357 alldevs = {
358 cx_dma,
359 cx_dma_dsp,
360 cx_usp,
361 nvgpu,
362 nvgpu_dsp2,
363 nvgpu_usp2,
364 nvgpu_dsp1,
365 }
366 topodevs = set(grace_rp.iterdownstream())
367 if alldevs != topodevs:
368 raise ValueError(
369 f"CX DMA function {cx_dma} has unexpected PCI devices in the topology"
370 )
371 return NVCX_Complex(cx_pfs, cx_dma, nvgpu)
372
373 def __build_topo(self):
374 """Collect cross-device information together and build the NVCX_Complex
375 objects for the cx_dma functions"""
376 self.vpd_v3s: Dict[str, Set[PCIDevice]] = collections.defaultdict(set)
377 for pdev in self.devices.values():
378 if pdev.parent_bdf:
379 pdev.parent = self.devices[pdev.parent_bdf]
380 pdev.parent.children.add(pdev)
381
382 # Many PCI functions may share the same V3
383 if pdev.vpd_v3:
384 self.vpd_v3s[pdev.vpd_v3].add(pdev)
385
386 self.nvcxs: List[NVCX_Complex] = []
387 for pdev in self.devices.values():
388 if pdev.device_type == "cx_dma":
389 nvcx = self.__get_nvcx_complex(pdev)
390 self.nvcxs.append(nvcx)
391 self.nvcxs.sort(key=lambda x: x.cx_pf.bdf)
392
393 def compute_acs(self):
394 """Return a dictionary of PCI devices and the ACS mask the device should
395 have"""
396 acs: Dict[PCIDevice, str] = {}
397 for nvcx in self.nvcxs:
398 # For the DSP in the shared switch toward the CX8 DMA Direct interface:
399 # Enable these bits:
400 # bit-4 : ACS Upstream Forwarding
401 # bit-3 : ACS P2P Completion Redirect
402 # bit-0 : ACS Source Validation
403 # Disable these bits:
404 # bit-2 : ACS P2P Request Redirect
405 assert nvcx.cx_dma_dsp.has_acs
406 acs[nvcx.cx_dma_dsp] = "xx110x1"
407
408 # For the DSP in the shared switch toward the GPU:
409 # Enable the following bits:
410 # bit-4 : ACS Upstream Forwarding
411 # bit-2 : ACS P2P Request Redirect
412 # bit-0 : ACS Source Validation
413 # Disable the following bits:
414 # bit-3 : ACS P2P Completion Redirect
415 assert nvcx.nvgpu_dsp.has_acs
416 acs[nvcx.nvgpu_dsp] = "xx101x1"
417
418 # Disable ACS SV on the root port, this forces the entire segment
419 # into one iommu_group and avoids kernel bugs building groups for
420 # irregular ACS.
421 for pdev in nvcx.cx_dma_dsp.iterupstream_path():
422 if not pdev.parent:
423 assert pdev.has_acs
424 acs[pdev] = "xx111x0"
425
426 # For all other CX bridges set kernel's default ACS enable
427 # Enable these bits:
428 # bit-4 : ACS Upstream Forwarding
429 # bit-3 : ACS P2P Completion Redirect
430 # bit-2 : ACS P2P Request Redirect
431 # bit-0 : ACS Source Validation
432 # Which match the kernel default
433 for pdev in self.devices.values():
434 if (
435 pdev not in acs
436 and ("switch" in pdev.device_type or "grace_rp" in pdev.device_type)
437 and pdev.has_acs
438 ):
439 acs[pdev] = KERNEL_ACS_ISOLATED
440 return acs
441
442
443 # -------------------------------------------------------------------
444 def print_list(title: str, items: list[str]):
445 if not items:
446 return
447 if len(items) > 1:
448 title = title + "s"
449 list_str = ", ".join(sorted(items))
450 print(f"\t{title}: {list_str}")
451
452
453 def args_topology(parser):
454 parser.add_argument(
455 "-j",
456 "--json",
457 action="store_true",
458 dest="json",
459 help="Output in machine readable JSON format",
460 )
461
462
463 def topo_json(topo: PCITopo):
464 import json
465
466 jtop = []
467 for nvcx in topo.nvcxs:
468 jnvcx = {
469 "rdma_nic_pf_bdf": str(nvcx.cx_pf.bdf),
470 "rdma_dma_bdf": str(nvcx.cx_dma.bdf),
471 "gpu_bdf": str(nvcx.nvgpu.bdf),
472 "subsystems": {},
473 }
474 devname = nvcx.cx_pf.parse_vpd_name()
475 if devname:
476 jnvcx["rdma_nic_vpd_name"] = nvcx.cx_pf.parse_vpd_name()
477 if nvcx.cx_pf.numa_node is not None:
478 jnvcx["numa_node"] = nvcx.cx_pf.numa_node
479 if nvcx.nvmes:
480 jnvcx["nvme_bdf"] = str(next(iter(nvcx.nvmes)).bdf)
481
482 for pdev in sorted(
483 itertools.chain(nvcx.cx_pfs, [nvcx.nvgpu, nvcx.cx_dma], nvcx.nvmes),
484 key=lambda x: x.bdf,
485 ):
486 subsys = pdev.get_subsystems()
487 if subsys:
488 jnvcx["subsystems"][str(pdev.bdf)] = {
489 subsys: list(devs) for subsys, devs in subsys.items()
490 }
491 jtop.append(jnvcx)
492 print(json.dumps(jtop, indent=4))
493
494
495 def cmd_topology(args):
496 """List the ConnectX NICs in the system with the corresponding NIC
497 function, DMA Direct function and associated GPU."""
498 topo = PCITopo()
499 if not topo.has_cx_dma:
500 raise CommandError("No ConnectX DMA Direct functions detected")
501
502 if args.json:
503 return topo_json(topo)
504
505 for nvcx in topo.nvcxs:
506 print(
507 f"RDMA NIC={nvcx.cx_pf.bdf}, GPU={nvcx.nvgpu.bdf}, RDMA DMA Function={nvcx.cx_dma.bdf}"
508 )
509
510 devname = nvcx.cx_pf.parse_vpd_name()
511 if devname:
512 print(f"\t{devname}")
513
514 if nvcx.cx_pf.numa_node is not None:
515 print(f"\tNUMA Node: {nvcx.cx_pf.numa_node}")
516
517 if len(nvcx.cx_pfs):
518 print_list("NIC PCI device", [str(I.bdf) for I in nvcx.cx_pfs])
519
520 subsystems = nvcx.get_subsystems()
521 print_list("RDMA device", subsystems["infiniband"])
522 print_list("Net device", subsystems["net"])
523 print_list("DRM device", subsystems["drm"])
524 print_list("NVMe device", subsystems["nvme"])
525 cmd_topology.__aliases__ = ("topo",)
526
527 # -------------------------------------------------------------------
528 def update_file(fn: str, new_content: str):
529 """Make fn have new_content. If fn already has new_content nothing is
530 done."""
531 try:
532 with open(fn, "rt") as F:
533 old = F.read()
534 if old == new_content:
535 return False
536 except FileNotFoundError:
537 pass
538 with tempfile.NamedTemporaryFile(dir=os.path.dirname(fn), mode="wt") as F:
539 F.write(new_content)
540 F.flush()
541 os.chmod(F.name, 0o644)
542 try:
543 os.link(F.name, fn)
544 except FileExistsError:
545 os.unlink(fn)
546 os.link(F.name, fn)
547 return True
548
549
550 def args_write_grub_acs(parser):
551 parser.add_argument(
552 "-n",
553 "--dry-run",
554 action="store_true",
555 dest="dry_run",
556 help="Output the grub configuration to stdout and make no changes",
557 )
558 parser.add_argument(
559 "--output",
560 action="store",
561 default="/etc/default/grub.d/config-acs.cfg",
562 help="Grub dropin file to use for the kernel command line",
563 )
564
565
566 def cmd_write_grub_acs(args):
567 """Generate a grub dropin file to have the kernel commandline set the
568 required ACS flags during system boot. This is the recommended way to
569 configure ACS on systems but requires a compatible kernel.
570
571 If the system does not have any need of ACS flags the dropin file will be
572 removed. This command is intended for Debian style systems with a
573 /etc/default/grub.d and update-grub command."""
574 topo = PCITopo()
575 if not topo.has_cx_dma:
576 if args.dry_run:
577 raise CommandError("No ConnectX DMA Direct functions detected")
578 if os.path.exists(args.output):
579 os.unlink(args.output)
580 return
581
582 acs = topo.compute_acs()
583 config_acs = [
584 f"{acs}@{pdev.bdf}"
585 for pdev, acs in sorted(acs.items(), key=lambda x: x[0].bdf)
586 if acs != KERNEL_ACS_ISOLATED
587 ]
588 acs_arg = ";".join(config_acs)
589 grub_conf = [
590 f"# Generated by {sys.argv[0]} do not change. ACS settings for RDMA GPU Direct",
591 f'GRUB_CMDLINE_LINUX="$GRUB_CMDLINE_LINUX pci=config_acs=\\"{acs_arg}\\""',
592 ]
593 grub_conf = "\n".join(grub_conf)
594
595 if args.dry_run:
596 print(grub_conf)
597 return
598
599 try:
600 os.makedirs(os.path.dirname(args.output))
601 except FileExistsError:
602 pass
603 if update_file(args.output, grub_conf + "\n"):
604 subprocess.check_call(["update-grub"])
605
606
607 # -------------------------------------------------------------------
608 def combine_acs(cur_acs, new_acs):
609 for idx, val in enumerate(new_acs[::-1]):
610 if val == "1":
611 cur_acs = cur_acs | (1 << idx)
612 elif val == "0":
613 cur_acs = cur_acs & (0xFFFF ^ (1 << idx))
614 return cur_acs
615
616
617 def args_setpci_acs(parser):
618 parser.add_argument(
619 "-n",
620 "--dry-run",
621 action="store_true",
622 dest="dry_run",
623 help="Output the setpci commands to stdout and make no changes",
624 )
625
626
627 def cmd_setpci_acs(args):
628 """Execute a series of set_pci commands that will immediately change the ACS
629 settings to the required values. This is compatible with older kernels, but
630 is not recommended. The kernel must boot with ACS enabled and the GPU driver
631 must have the NVreg_GrdmaPciTopoCheckOverride=1 reg key set to disable
632 safety checks that old kernels cannot support.
633
634 NOTE: In this configuration unprivileged userspace can trigger platform RAS
635 failures, use with caution!
636 """
637 topo = PCITopo()
638 acs = topo.compute_acs()
639 cmds: List[List[str]] = []
640 for pdev, acs in sorted(acs.items(), key=lambda x: x[0].bdf):
641 cur_acs = pdev.read_config("ECAP_ACS+0x6.w")
642 new_acs = combine_acs(cur_acs, acs)
643 if new_acs == cur_acs:
644 continue
645
646 cmd = ["setpci", "-r", "-s", str(pdev.bdf), f"ECAP_ACS+0x6.w={new_acs:04x}"]
647 cmds.append(cmd)
648 if args.dry_run:
649 for cmd in cmds:
650 print(" ".join(cmd))
651 return
652 for cmd in cmds:
653 subprocess.check_call(cmd)
654
655
656 # -------------------------------------------------------------------
657 def args_check(parser):
658 pass
659
660
661 def check_ok(msg: str):
662 print(f"OK\t{msg}")
663
664
665 def check_fail(msg: str):
666 print(f"FAIL\t{msg}")
667 sys.exit(100)
668
669
670 def cmd_check(args):
671 """Check that the running kernel and PCI environment are setup correctly for
672 GPU Direct with ConnectX DMA Direct PCI functions."""
673 topo = PCITopo()
674 if not topo.has_cx_dma:
675 raise CommandError("No ConnectX DMA Direct functions detected")
676 check_ok("All ConnectX DMA functions have correct PCI topology")
677
678 acs = topo.compute_acs()
679 for pdev, acs in sorted(acs.items(), key=lambda x: x[0].bdf):
680 cur_acs = pdev.read_config("ECAP_ACS+0x6.w")
681 new_acs = combine_acs(cur_acs, acs)
682 if new_acs == cur_acs:
683 check_ok(
684 f"ACS for {pdev.device_type} {pdev.bdf} has correct values {cur_acs:07b} = {acs}"
685 )
686 else:
687 check_fail(
688 f"ACS for {pdev.device_type} {pdev.bdf} has incorrect values {cur_acs:07b} != {acs}, (0x{cur_acs:x} != 0x{new_acs:x})"
689 )
690
691 # Correct iommu_groups are required to avoid NVreg_GrdmaPciTopoCheckOverride
692 for nvcx in topo.nvcxs:
693 if (
694 nvcx.cx_dma.iommu_group == nvcx.nvgpu.iommu_group
695 and nvcx.cx_dma.iommu_group is not None
696 ):
697 check_ok(
698 f"Kernel iommu_group for DMA {nvcx.cx_dma.bdf} and GPU {nvcx.nvgpu.bdf} are both {nvcx.cx_dma.iommu_group}"
699 )
700 else:
701 check_fail(
702 f"Kernel iommu_group for DMA {nvcx.cx_dma.bdf} and GPU {nvcx.nvgpu.bdf} are not equal {nvcx.cx_dma.iommu_group} != {nvcx.nvgpu.iommu_group}"
703 )
704
705
706 # -------------------------------------------------------------------
707 def load_all_commands(name):
708 module = importlib.import_module(name)
709 for k in dir(module):
710 fn = getattr(module, k)
711 argsfn = getattr(module, "args_" + k[4:], None)
712 if argsfn is None or not k.startswith("cmd_") or not inspect.isfunction(fn):
713 continue
714 yield (k, fn, argsfn)
715
716
717 def get_cmd_aliases(fn):
718 if hasattr(fn, "__aliases__"):
719 return fn.__aliases__
720 return ()
721
722 def main():
723 parser = argparse.ArgumentParser(
724 formatter_class=argparse.RawDescriptionHelpFormatter,
725 description="""NVIDIA ConnectX GPU Direct ACS tool for Direct NIC platforms
726
727 This tool is used to view and control the PCI Access Control Flags (ACS) related
728 to the Direct NIC topology on supported NVIDIA platforms with ConnectX and
729 Blackwell family GPUs.
730
731 Direct NIC platforms have a unique multipath PCI topology where the ConnectX
732 has a main PCI function and a related DMA Direct function linked to the GPU.
733
734 This platform requires specific ACS flags in the PCI topology for reliable
735 operation, this tool helps users generate ACS settings for the local system.
736 """,
737 )
738 subparsers = parser.add_subparsers(title="Sub Commands", dest="command")
739 subparsers.required = True
740
741 commands = [I for I in load_all_commands(__name__)]
742 commands.sort()
743
744 # build sub parsers for all the loaded commands
745 for k, fn, argsfn in commands:
746 sparser = subparsers.add_parser(
747 k[4:].replace("_", "-"), aliases=get_cmd_aliases(fn), help=fn.__doc__
748 )
749 sparser.required = True
750 argsfn(sparser)
751 sparser.set_defaults(func=fn)
752
753 try:
754 import argcomplete
755
756 argcomplete.autocomplete(parser)
757 except ImportError:
758 pass
759
760 # argparse will set 'func' to the cmd_* that executes this command
761 args = parser.parse_args()
762 try:
763 args.func(args)
764 except CommandError as e:
765 print(f"E: {e}")
766 sys.exit(100)
767
768
769 main()