| 1 |
# Copyright (C) Advanced Micro Devices. All rights reserved. |
| 2 |
# |
| 3 |
# Permission is hereby granted, free of charge, to any person obtaining a copy of |
| 4 |
# this software and associated documentation files (the "Software"), to deal in |
| 5 |
# the Software without restriction, including without limitation the rights to |
| 6 |
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of |
| 7 |
# the Software, and to permit persons to whom the Software is furnished to do so, |
| 8 |
# subject to the following conditions: |
| 9 |
# |
| 10 |
# The above copyright notice and this permission notice shall be included in all |
| 11 |
# copies or substantial portions of the Software. |
| 12 |
# |
| 13 |
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 14 |
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS |
| 15 |
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR |
| 16 |
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER |
| 17 |
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 18 |
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 19 |
|
| 20 |
from amdsmi import * |
| 21 |
import os |
| 22 |
|
| 23 |
amdsmi_init() |
| 24 |
|
| 25 |
def get_severity_mask(severity): |
| 26 |
severity_mask = 0 |
| 27 |
if severity == "all": |
| 28 |
# Set bits for NON_FATAL_UNCORRECTED (0), FATAL (1), and NON_FATAL_CORRECTED (2) |
| 29 |
severity_mask |= ((1 << 0) | (1 << 1) | (1 << 2)) |
| 30 |
elif severity == "fatal": |
| 31 |
# Set bit corresponding to AMDSMI_CPER_SEV_FATAL (which is 1) |
| 32 |
severity_mask |= (1 << 1) |
| 33 |
elif severity in ("nonfatal", "nonfatal-uncorrected"): |
| 34 |
# Set bit corresponding to AMDSMI_CPER_SEV_NON_FATAL_UNCORRECTED (which is 0) |
| 35 |
severity_mask |= (1 << 0) |
| 36 |
elif severity in ("nonfatal-corrected", "corrected"): |
| 37 |
# Set bit corresponding to AMDSMI_CPER_SEV_NON_FATAL_CORRECTED (which is 2) |
| 38 |
severity_mask |= (1 << 2) |
| 39 |
return severity_mask |
| 40 |
|
| 41 |
def gpuid(device): |
| 42 |
for gpu_index, device_handle in enumerate(amdsmi_interface.amdsmi_get_processor_handles()): |
| 43 |
if device.value == device_handle.value: |
| 44 |
return gpu_index |
| 45 |
|
| 46 |
def dump_cper_entry(entry, cper_data, key): |
| 47 |
try: |
| 48 |
os.mkdir("/tmp/cper_dump", mode=0o777, dir_fd=None) |
| 49 |
except FileExistsError: |
| 50 |
pass |
| 51 |
cper_file = f"/tmp/cper_dump/cper_entry_{key}.bin" |
| 52 |
with open(cper_file, "wb") as file: |
| 53 |
size = cper_data[key]["size"] |
| 54 |
data = cper_data[key]["bytes"] |
| 55 |
data = bytes(x % 256 for x in data[:size]) |
| 56 |
file.write(data) |
| 57 |
print(f" Wrote cper data to file: {cper_file}") |
| 58 |
json_file = f"/tmp/cper_dump/cper_entry_{key}.json" |
| 59 |
with open(json_file, "wt") as file: |
| 60 |
file.write(str(entry)) |
| 61 |
|
| 62 |
def get_gpu_cper_entries(): |
| 63 |
try: |
| 64 |
devices = amdsmi_interface.amdsmi_get_processor_handles() |
| 65 |
buffer_size = 1024*100 |
| 66 |
initial_cursor = 0 |
| 67 |
severity = "all" |
| 68 |
for device in devices: |
| 69 |
while True: |
| 70 |
entries, new_cursor, cper_data, status_code = amdsmi_get_gpu_cper_entries( |
| 71 |
device, get_severity_mask(severity), buffer_size, initial_cursor) |
| 72 |
gpu_id = gpuid(device) |
| 73 |
print("#############################################################################") |
| 74 |
print(f"cper entries for severity: '{severity}', gpu #{gpu_id}, cursor: {initial_cursor}-{new_cursor - 1}") |
| 75 |
for key, entry in entries.items(): |
| 76 |
print("----------------") |
| 77 |
print("Entry", initial_cursor + key) |
| 78 |
print(" Error Severity:", entry.get("error_severity", "Unknown")) |
| 79 |
print(" Notify Type:", entry.get("notify_type", "Unknown")) |
| 80 |
print(" Timestamp:", entry.get("timestamp", "")) |
| 81 |
print(f" Cper entry metadata: {entry}") |
| 82 |
dump_cper_entry(entry, cper_data, key) |
| 83 |
if initial_cursor == new_cursor: |
| 84 |
break |
| 85 |
initial_cursor = new_cursor |
| 86 |
break |
| 87 |
except AmdSmiException as e: |
| 88 |
print(e) |
| 89 |
|
| 90 |
get_gpu_cper_entries() |
| 91 |
|
| 92 |
""" |
| 93 |
Sample output: |
| 94 |
|
| 95 |
cper entries for severity: 'all', gpu #0, cursor: 0-3 |
| 96 |
---------------- |
| 97 |
Entry 0 |
| 98 |
Error Severity: non_fatal_corrected |
| 99 |
Notify Type: CMC |
| 100 |
Timestamp: 2025/09/07 00:14:22 |
| 101 |
Cper entry metadata: {'error_severity': 'non_fatal_corrected', 'notify_type': 'CMC', 'timestamp': '2025/09/07 00:14:22', 'signature': b'CPER', 'revision': 256, 'signature_end': '0xffffffff', 'sec_cnt': 1, 'record_length': 472, 'platform_id': b'0x1002:0x74A2', 'creator_id': b'amdgpu', 'record_id': b'5:1', 'flags': 0, 'persistence_info': 0} |
| 102 |
Wrote cper data to file: /tmp/cper_dump/cper_entry_0.bin |
| 103 |
---------------- |
| 104 |
Entry 1 |
| 105 |
Error Severity: non_fatal_corrected |
| 106 |
Notify Type: CMC |
| 107 |
Timestamp: 2025/09/07 00:14:26 |
| 108 |
Cper entry metadata: {'error_severity': 'non_fatal_corrected', 'notify_type': 'CMC', 'timestamp': '2025/09/07 00:14:26', 'signature': b'CPER', 'revision': 256, 'signature_end': '0xffffffff', 'sec_cnt': 1, 'record_length': 472, 'platform_id': b'0x1002:0x74A2', 'creator_id': b'amdgpu', 'record_id': b'5:2', 'flags': 0, 'persistence_info': 0} |
| 109 |
Wrote cper data to file: /tmp/cper_dump/cper_entry_1.bin |
| 110 |
---------------- |
| 111 |
Entry 2 |
| 112 |
Error Severity: non_fatal_corrected |
| 113 |
Notify Type: CMC |
| 114 |
Timestamp: 2025/09/08 06:12:11 |
| 115 |
Cper entry metadata: {'error_severity': 'non_fatal_corrected', 'notify_type': 'CMC', 'timestamp': '2025/09/08 06:12:11', 'signature': b'CPER', 'revision': 256, 'signature_end': '0xffffffff', 'sec_cnt': 1, 'record_length': 472, 'platform_id': b'0x1002:0x74A2', 'creator_id': b'amdgpu', 'record_id': b'5:3', 'flags': 0, 'persistence_info': 0} |
| 116 |
Wrote cper data to file: /tmp/cper_dump/cper_entry_2.bin |
| 117 |
---------------- |
| 118 |
Entry 3 |
| 119 |
Error Severity: non_fatal_corrected |
| 120 |
Notify Type: CMC |
| 121 |
Timestamp: 2025/09/08 06:13:59 |
| 122 |
Cper entry metadata: {'error_severity': 'non_fatal_corrected', 'notify_type': 'CMC', 'timestamp': '2025/09/08 06:13:59', 'signature': b'CPER', 'revision': 256, 'signature_end': '0xffffffff', 'sec_cnt': 1, 'record_length': 472, 'platform_id': b'0x1002:0x74A2', 'creator_id': b'amdgpu', 'record_id': b'5:4', 'flags': 0, 'persistence_info': 0} |
| 123 |
Wrote cper data to file: /tmp/cper_dump/cper_entry_3.bin |
| 124 |
############################################################################# |
| 125 |
cper entries for severity: 'all', gpu #0, cursor: 4-3 |
| 126 |
""" |