/* * Copyright (c) Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "rocm_smi/rocm_smi_io_link.h" #include "rocm_smi/rocm_smi_kfd.h" #include "rocm_smi/rocm_smi.h" #include "rocm_smi/rocm_smi_exception.h" #include "rocm_smi/rocm_smi_utils.h" #include "rocm_smi/rocm_smi_main.h" #include "rocm_smi/rocm_smi_logger.h" namespace amd::smi { static const char *kKFDProcPathRoot = "/sys/class/kfd/kfd/proc"; static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes"; // KFD Node Property strings // static const char *kKFDNodePropCPU_CORES_COUNTStr = "cpu_cores_count"; // static const char *kKFDNodePropSIMD_COUNTStr = "simd_count"; // static const char *kKFDNodePropMEM_BANKS_COUNTStr = "mem_banks_count"; // static const char *kKFDNodePropCACHES_COUNTStr = "caches_count"; // static const char *kKFDNodePropIO_LINKS_COUNTStr = "io_links_count"; // static const char *kKFDNodePropCPU_CORE_ID_BASEStr = "cpu_core_id_base"; // static const char *kKFDNodePropSIMD_ID_BASEStr = "simd_id_base"; // static const char *kKFDNodePropMAX_WAVES_PER_SIMDStr = "max_waves_per_simd"; // static const char *kKFDNodePropLDS_SIZE_IN_KBStr = "lds_size_in_kb"; // static const char *kKFDNodePropGDS_SIZE_IN_KBStr = "gds_size_in_kb"; // static const char *kKFDNodePropNUM_GWSStr = "num_gws"; // static const char *kKFDNodePropWAVE_FRONT_SIZEStr = "wave_front_size"; static const char *kKFDNodePropARRAY_COUNTStr = "array_count"; static const char *kKFDNodePropSIMD_ARRAYS_PER_ENGINEStr = "simd_arrays_per_engine"; static const char *kKFDNodePropCU_PER_SIMD_ARRAYStr = "cu_per_simd_array"; // static const char *kKFDNodePropSIMD_PER_CUStr = "simd_per_cu"; // static const char *kKFDNodePropMAX_SLOTS_SCRATCH_CUStr = // "max_slots_scratch_cu"; // static const char *kKFDNodePropVENDOR_IDStr = "vendor_id"; // static const char *kKFDNodePropDEVICE_IDStr = "device_id"; static const char *kKFDNodePropLOCATION_IDStr = "location_id"; static const char *kKFDNodePropDOMAINStr = "domain"; // static const char *kKFDNodePropDRM_RENDER_MINORStr = "drm_render_minor"; static const char *kKFDNodePropHIVE_IDStr = "hive_id"; // static const char *kKFDNodePropNUM_SDMA_ENGINESStr = "num_sdma_engines"; // static const char *kKFDNodePropNUM_SDMA_XGMI_ENGINESStr = // "num_sdma_xgmi_engines"; // static const char *kKFDNodePropNUM_SDMA_QUEUES_PER_ENGINEStr = // "num_sdma_queues_per_engine"; // static const char *kKFDNodePropNUM_CP_QUEUESStr = "num_cp_queues"; // static const char *kKFDNodePropMAX_ENGINE_CLK_FCOMPUTEStr = // "max_engine_clk_fcompute"; // static const char *kKFDNodePropLOCAL_MEM_SIZEStr = "local_mem_size"; // static const char *kKFDNodePropFW_VERSIONStr = "fw_version"; // static const char *kKFDNodePropCAPABILITYStr = "capability"; // static const char *kKFDNodePropDEBUG_PROPStr = "debug_prop"; // static const char *kKFDNodePropSDMA_FW_VERSIOStr = "sdma_fw_versio"; // static const char *kKFDNodePropMAX_ENGINE_CLK_CCOMPUTEStr = // "max_engine_clk_ccompute"; static bool is_number(const std::string &s) { return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit); } static std::string KFDDevicePath(uint32_t dev_id) { std::string node_path = kKFDNodesPathRoot; node_path += '/'; node_path += std::to_string(dev_id); return node_path; } // A generic function to extract out a property from file. // return empty string if file or property not found // Assume the property_name is at the beginning of the line. static std::string get_properties_from_file(const std::string& file_name, const std::string& property_name) { std::ifstream infile(file_name); if (!infile) return ""; std::string line; while (std::getline(infile, line)) { std::istringstream iss(line); // the property name is at the beginning of the line if (line.rfind(property_name.c_str(), 0) == 0) { return line.substr(property_name.length()); } } return ""; } static int OpenKFDNodeFile(uint32_t dev_id, std::string node_file, std::ifstream *fs) { std::string line; int ret; std::string f_path; bool reg_file; assert(fs != nullptr); f_path = KFDDevicePath(dev_id); f_path += "/"; f_path += node_file; ret = isRegularFile(f_path, ®_file); if (ret != 0) { return ret; } if (!reg_file) { return ENOENT; } fs->open(f_path); if (!fs->is_open()) { return errno; } return 0; } bool KFDNodeSupported(uint32_t node_indx) { std::ifstream fs; bool ret = true; int err; err = OpenKFDNodeFile(node_indx, "properties", &fs); if (err == ENOENT) { return false; } if (fs.peek() == std::ifstream::traits_type::eof()) { ret = false; } fs.close(); return ret; } int ReadKFDDeviceProperties(uint32_t kfd_node_id, std::vector *retVec) { std::string line; int ret; std::ifstream fs; std::string properties_path; std::ostringstream ss; assert(retVec != nullptr); ret = OpenKFDNodeFile(kfd_node_id, "properties", &fs); if (ret) { return ret; } ss << __PRETTY_FUNCTION__ << " | properties file contains = {"; while (std::getline(fs, line)) { retVec->push_back(line); ss << line << ",\n"; } ss << "}"; // Leaving below to debug any future properties file changes // LOG_DEBUG(ss); if (retVec->empty()) { fs.close(); return ENOENT; } // Remove any *trailing* empty (whitespace) lines while (retVec->back().find_first_not_of(" \t\n\v\f\r") == std::string::npos) { retVec->pop_back(); } fs.close(); return 0; } static int ReadKFDGpuId(uint32_t kfd_node_id, uint64_t *gpu_id) { std::string line; int ret; std::ifstream fs; std::string gpu_id_str; assert(gpu_id != nullptr); ret = OpenKFDNodeFile(kfd_node_id, "gpu_id", &fs); if (ret) { fs.close(); return ret; } std::stringstream ss; ss << fs.rdbuf(); fs.close(); gpu_id_str = ss.str(); gpu_id_str.erase(std::remove(gpu_id_str.begin(), gpu_id_str.end(), '\n'), gpu_id_str.end()); if (!is_number(gpu_id_str)) { return ENXIO; } *gpu_id = static_cast(std::stoi(gpu_id_str)); return 0; } static int ReadKFDGpuName(uint32_t kfd_node_id, std::string *gpu_name) { std::string line; int ret; std::ifstream fs; assert(gpu_name != nullptr); ret = OpenKFDNodeFile(kfd_node_id, "name", &fs); if (ret) { fs.close(); return ret; } std::stringstream ss; ss << fs.rdbuf(); fs.close(); *gpu_name = ss.str(); gpu_name->erase(std::remove(gpu_name->begin(), gpu_name->end(), '\n'), gpu_name->end()); return 0; } int GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated, uint32_t *num_procs_found) { assert(num_procs_found != nullptr); *num_procs_found = 0; errno = 0; auto proc_dir = opendir(kKFDProcPathRoot); if (proc_dir == nullptr) { perror("Unable to open process directory"); return errno; } auto dentry = readdir(proc_dir); std::string proc_id_str; std::string tmp; while (dentry != nullptr) { if (dentry->d_name[0] == '.') { dentry = readdir(proc_dir); continue; } proc_id_str = dentry->d_name; assert(is_number(proc_id_str) && "Unexpected file name in kfd/proc dir"); if (!is_number(proc_id_str)) { dentry = readdir(proc_dir); continue; } if (procs && *num_procs_found < num_allocated) { procs[*num_procs_found].process_id = static_cast(std::stoi(proc_id_str)); } ++(*num_procs_found); dentry = readdir(proc_dir); } errno = 0; if (closedir(proc_dir)) { return errno; } return 0; } int GetKfdGpuIdsForPid(long pid, std::unordered_set* out){ if (!out) return EINVAL; out->clear(); std::string pdir = std::string(kKFDProcPathRoot) + "/" + std::to_string(pid); DIR* d = opendir(pdir.c_str()); if (!d) { perror(("Unable to open KFD process directory for process " + std::to_string(pid)).c_str()); return errno ? errno : ESRCH; } struct dirent* e; while ((e = readdir(d))) { if (e->d_name[0] == '.') continue; // skip "."/".." and hidden entries // Grab KFD GPU id from one of these fields if (!strncmp(e->d_name, "stats_", 6)) { out->insert(strtoull(e->d_name + 6, nullptr, 10)); } else if (!strncmp(e->d_name, "vram_", 5)) { out->insert(strtoull(e->d_name + 5, nullptr, 10)); } else if (!strncmp(e->d_name, "counters_", 9)) { out->insert(strtoull(e->d_name + 9, nullptr, 10)); } else if (!strncmp(e->d_name, "sdma_", 5)) { out->insert(strtoull(e->d_name + 5, nullptr, 10)); } } closedir(d); return 0; } // Read the gpuid files found in all the dirs and put them in // gpus_found. // Directory structure: // /sys/class/kfd/kfd/proc//queues//gpuid int GetProcessGPUs(uint32_t pid, std::unordered_set *gpu_set) { int err; assert(gpu_set != nullptr); if (gpu_set == nullptr) { return RSMI_STATUS_INVALID_ARGS; } // Skip amd-smi process itself if (pid == static_cast(getpid())) { return 0; } std::string queues_dir = kKFDProcPathRoot; queues_dir += "/"; queues_dir += std::to_string(pid); queues_dir += "/queues"; auto queues_dir_hd = opendir(queues_dir.c_str()); if (queues_dir_hd == nullptr) { std::string err_str = "Unable to open queues directory for process "; err_str += std::to_string(pid); perror(err_str.c_str()); return ESRCH; } auto q_dentry = readdir(queues_dir_hd); std::string q_gpu_id_str; std::string q_dir; std::string tmp; while (q_dentry != nullptr) { if (q_dentry->d_name[0] == '.') { q_dentry = readdir(queues_dir_hd); continue; } if (!is_number(q_dentry->d_name)) { q_dentry = readdir(queues_dir_hd); continue; } q_gpu_id_str = queues_dir + '/' + q_dentry->d_name + "/gpuid"; err = ReadSysfsStr(q_gpu_id_str, &tmp); if (err) { q_dentry = readdir(queues_dir_hd); continue; } uint64_t val; try { val = static_cast(std::stoi(tmp)); } catch (...) { std::cerr << "Error; read invalid data: " << tmp << " from " << q_gpu_id_str << std::endl; closedir(queues_dir_hd); return ENXIO; // Return "no such device" if we read an invalid gpu id } gpu_set->insert(val); q_dentry = readdir(queues_dir_hd); } // if no queues were present, fallback to grab KFD GPU IDs from parent dir names int kfd_ret = GetKfdGpuIdsForPid(pid, gpu_set); if (kfd_ret != 0) { return kfd_ret; } errno = 0; if (closedir(queues_dir_hd)) { return errno; } return 0; } static int CheckValidProcessInfoData(const std::string& s, int sysfs_ret){ if(sysfs_ret==0 && !is_number(s)){ return EINVAL; } return sysfs_ret; } static int GetProcessKFDStats(std::string path, uint32_t& val){ std::string tmp; int err = ReadSysfsStr(path, &tmp); auto sysfs_data_errcode = CheckValidProcessInfoData(tmp, err); if (!(sysfs_data_errcode == 0 || sysfs_data_errcode == ENOENT)){ return sysfs_data_errcode; } else if(sysfs_data_errcode==0){ // Update KFD stat by the process val = static_cast(std::stoul(tmp)); } else { // Some GFX revisions do not provide KFD stats debugfs method // which may cause ENOENT val = KFD_STATS_INVALID; } return 0; } int GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc, std::unordered_set *gpu_set) { assert(proc != nullptr); assert(gpu_set != nullptr); int err; std::string tmp; std::unordered_set::iterator itr; uint32_t kfd_stat; std::string proc_str_path = kKFDProcPathRoot; proc_str_path += "/"; proc_str_path += std::to_string(pid); if (!FileExists(proc_str_path.c_str())) { return ESRCH; } proc->process_id = pid; proc->vram_usage = 0; proc->sdma_usage = 0; proc->cu_occupancy = 0; proc->evicted_time = 0; for (itr = gpu_set->begin(); itr != gpu_set->end(); itr++) { uint64_t gpu_id = (*itr); std::string vram_str_path = proc_str_path; vram_str_path += "/vram_"; vram_str_path += std::to_string(gpu_id); err = ReadSysfsStr(vram_str_path, &tmp); auto sysfs_data_errcode = CheckValidProcessInfoData(tmp, err); // Report all errors, except ENOENT (2), which should be ignored // and the proc->vram_usage should be unmodified if (!(sysfs_data_errcode == 0 || sysfs_data_errcode == ENOENT)){ return sysfs_data_errcode; } // Do not store any invalid values else if (sysfs_data_errcode == 0) { proc->vram_usage += std::stoull(tmp); } std::string sdma_str_path = proc_str_path; sdma_str_path += "/sdma_"; sdma_str_path += std::to_string(gpu_id); err = ReadSysfsStr(sdma_str_path, &tmp); sysfs_data_errcode = CheckValidProcessInfoData(tmp, err); if (!(sysfs_data_errcode == 0 || sysfs_data_errcode == ENOENT)){ return sysfs_data_errcode; } else if (sysfs_data_errcode == 0) { proc->sdma_usage += std::stoull(tmp); } // Build the path and read from Sysfs file, info that // encodes Compute Unit usage by a process of interest std::string cu_occupancy_path = proc_str_path; cu_occupancy_path += "/stats_"; cu_occupancy_path += std::to_string(gpu_id); cu_occupancy_path += "/cu_occupancy"; err = GetProcessKFDStats(cu_occupancy_path, kfd_stat); if (err != 0){ return err; } proc->cu_occupancy = kfd_stat; std::string evicted_time_path = proc_str_path; evicted_time_path += "/stats_"; evicted_time_path += std::to_string(gpu_id); evicted_time_path += "/evicted_ms"; err = GetProcessKFDStats(evicted_time_path, kfd_stat); if (err != 0){ return err; } proc->evicted_time = kfd_stat; } return 0; } int DiscoverKFDNodes(std::map> *nodes) { assert(nodes != nullptr); if (nodes == nullptr) { return EINVAL; } assert(nodes->empty()); nodes->clear(); std::shared_ptr node; uint32_t node_indx; auto kfd_node_dir = opendir(kKFDNodesPathRoot); if (kfd_node_dir == nullptr) { return errno; } auto dentry = readdir(kfd_node_dir); while (dentry != nullptr) { if (dentry->d_name[0] == '.') { dentry = readdir(kfd_node_dir); continue; } if (!is_number(dentry->d_name)) { dentry = readdir(kfd_node_dir); continue; } node_indx = static_cast(std::stoi(dentry->d_name)); if (!KFDNodeSupported(node_indx)) { dentry = readdir(kfd_node_dir); continue; } node = std::make_shared(node_indx); node->Initialize(); if (node->gpu_id() == 0) { // Don't add; this is a cpu node. dentry = readdir(kfd_node_dir); continue; } uint64_t kfd_gpu_node_bus_fn; uint64_t kfd_gpu_node_domain; int ret; ret = node->get_property_value(kKFDNodePropLOCATION_IDStr, &kfd_gpu_node_bus_fn); if (ret != 0) { std:: cerr << "Failed to open properties file for kfd node " << node->node_index() << "." << std::endl; closedir(kfd_node_dir); return ret; } ret = node->get_property_value(kKFDNodePropDOMAINStr, &kfd_gpu_node_domain); if (ret != 0) { std::cerr << "Failed to get \"domain\" properity from properties " "files for kfd node " << node->node_index() << "." << std::endl; closedir(kfd_node_dir); return ret; } uint64_t kfd_bdfid = (kfd_gpu_node_domain << 32) | (kfd_gpu_node_bus_fn); (*nodes)[kfd_bdfid] = node; dentry = readdir(kfd_node_dir); } if (closedir(kfd_node_dir)) { std::string err_str = "Failed to close KFD node directory "; err_str += kKFDNodesPathRoot; err_str += "."; perror(err_str.c_str()); return 1; } return 0; } KFDNode::~KFDNode() = default; int KFDNode::ReadProperties(void) { int ret; std::vector propVec; assert(properties_.empty()); if (!properties_.empty()) { return 0; } ret = ReadKFDDeviceProperties(node_indx_, &propVec); if (ret) { return ret; } std::string key_str; std::string val_str; uint64_t val_int; // Assume all properties are unsigned integers for now std::istringstream fs; std::ostringstream ss; for (const auto & i : propVec) { fs.str(i); fs >> key_str; fs >> val_str; // Leaving below to debug any new properties file changes // ss << __PRETTY_FUNCTION__ << " | key = " << key_str // << "; val = " << val_str; // LOG_TRACE(ss); val_int = std::stoull(val_str); properties_[key_str] = val_int; fs.str(""); fs.clear(); } return 0; } int KFDNode::Initialize(void) { int ret = 0; ret = ReadProperties(); if (ret) {return ret;} ret = ReadKFDGpuId(node_indx_, &gpu_id_); if (ret || (gpu_id_ == 0)) {return ret;} ret = ReadKFDGpuName(node_indx_, &name_); ret = get_property_value(kKFDNodePropHIVE_IDStr, &xgmi_hive_id_); if (ret != 0) { throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR, "Failed to initialize rocm_smi library (get xgmi hive id)."); } std::map> io_link_map_tmp; ret = DiscoverIOLinksPerNode(node_indx_, &io_link_map_tmp); if (ret != 0) { throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR, "Failed to initialize rocm_smi library (IO Links discovery per node)."); } std::map>::iterator it; uint32_t node_to; uint64_t node_to_gpu_id; std::shared_ptr link; bool numa_node_found = false; for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) { io_link_map_[it->first] = it->second; node_to = it->first; link = it->second; ret = ReadKFDGpuId(node_to, &node_to_gpu_id); if (ret) {continue;} if (node_to_gpu_id == 0) { // CPU node if (numa_node_found) { if (numa_node_weight_ > link->weight()) { numa_node_number_ = node_to; numa_node_weight_ = link->weight(); numa_node_type_ = link->type(); } } else { numa_node_number_ = node_to; numa_node_weight_ = link->weight(); numa_node_type_ = link->type(); numa_node_found = true; } } else { io_link_type_[node_to] = link->type(); io_link_weight_[node_to] = link->weight(); io_link_max_bandwidth_[node_to] = link->max_bandwidth(); io_link_min_bandwidth_[node_to] = link->min_bandwidth(); } } // Pre-compute the total number of compute units a device has uint64_t tmp_val; ret = get_property_value(kKFDNodePropSIMD_ARRAYS_PER_ENGINEStr, &tmp_val); if (ret != 0) { throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR, "Failed to initialize rocm_smi library " "(get number of shader arrays per engine)."); } cu_count_ = uint32_t(tmp_val); ret = get_property_value(kKFDNodePropARRAY_COUNTStr, &tmp_val); if (ret != 0) { throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR, "Failed to initialize rocm_smi library (get number of shader arrays)."); } cu_count_ = cu_count_ * uint32_t(tmp_val); ret = get_property_value(kKFDNodePropCU_PER_SIMD_ARRAYStr, &tmp_val); if (ret != 0) { throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR, "Failed to initialize rocm_smi library (get number of CU's per array)."); } cu_count_ = cu_count_ * uint32_t(tmp_val); return ret; } int KFDNode::get_property_value(std::string property, uint64_t *value) { assert(value != nullptr); if (value == nullptr) { return EINVAL; } if (properties_.find(property) == properties_.end()) { return EINVAL; } *value = properties_[property]; return 0; } int KFDNode::get_io_link_type(uint32_t node_to, IO_LINK_TYPE *type) { assert(type != nullptr); if (type == nullptr) { return EINVAL; } if (io_link_type_.find(node_to) == io_link_type_.end()) { return EINVAL; } *type = io_link_type_[node_to]; return 0; } int KFDNode::get_io_link_weight(uint32_t node_to, uint64_t *weight) { assert(weight != nullptr); if (weight == nullptr) { return EINVAL; } if (io_link_weight_.find(node_to) == io_link_weight_.end()) { return EINVAL; } *weight = io_link_weight_[node_to]; return 0; } int KFDNode::get_io_link_bandwidth(uint32_t node_to, uint64_t *max_bandwidth, uint64_t *min_bandwidth){ assert (max_bandwidth != nullptr && min_bandwidth != nullptr); if (max_bandwidth == nullptr || min_bandwidth == nullptr ){ return EINVAL; } if (io_link_max_bandwidth_.find(node_to) == io_link_max_bandwidth_.end() || io_link_min_bandwidth_.find(node_to) == io_link_min_bandwidth_.end()){ return EINVAL; } *max_bandwidth = io_link_max_bandwidth_[node_to]; *min_bandwidth = io_link_min_bandwidth_[node_to]; return 0; } // /sys/class/kfd/kfd/topology/nodes/*/mem_banks/*/properties // size_in_bytes 68702699520 int KFDNode::get_total_memory(uint64_t* total) { std::ostringstream ss; if (total == nullptr) { return EINVAL; } *total = 0; std::string f_path = kKFDNodesPathRoot; f_path += "/"; f_path += std::to_string(node_indx_); f_path += "/mem_banks"; int subDirCount = subDirectoryCountInPath(f_path); ss << __PRETTY_FUNCTION__ << " | [before loop] Within " << f_path << " has subdirectory count = " << std::to_string(subDirCount); LOG_DEBUG(ss); auto kfd_node_dir = opendir(f_path.c_str()); if (kfd_node_dir == nullptr) { return errno; } auto dentry = readdir(kfd_node_dir); while (dentry != nullptr && subDirCount > 0) { ss << __PRETTY_FUNCTION__ << " | [inside loop] Within " << f_path << " has subdirectory count = " << std::to_string(subDirCount); LOG_DEBUG(ss); if (dentry->d_name[0] == '.') { dentry = readdir(kfd_node_dir); continue; } if (!is_number(dentry->d_name)) { dentry = readdir(kfd_node_dir); continue; } // read "size_in_bytes 68702699520" line const std::string size_in_bytes_property = "size_in_bytes "; std::string memory_bank_file = f_path + "/" + dentry->d_name + "/properties"; std::ifstream fs(memory_bank_file); if (!fs) { dentry = readdir(kfd_node_dir); continue; } std::string line; while (std::getline(fs, line)) { if (line.substr(0, size_in_bytes_property.length()) == size_in_bytes_property) { auto bytes = line.substr(size_in_bytes_property.length()); try { *total += std::stol(bytes); break; } catch(...) { dentry = readdir(kfd_node_dir); continue; } } } // end loop for lines in property file subDirCount--; } // end loop for mem_bank directory if (closedir(kfd_node_dir)) { std::string err_str = "Failed to close KFD node directory "; err_str += f_path; err_str += "."; perror(err_str.c_str()); return 1; } return 0; } // ioctl on kfd node device int KFDNode::get_used_memory(uint64_t* used) { if (used == nullptr) return EINVAL; static const char *kPathKFDIoctl = "/dev/kfd"; int kfd_fd = open(kPathKFDIoctl, O_RDWR | O_CLOEXEC); if (kfd_fd <= 0) { return 1; } struct kfd_ioctl_get_available_memory_args mem = {0, 0, 0}; mem.gpu_id = static_cast(gpu_id_); if (ioctl(kfd_fd, AMDKFD_IOC_AVAILABLE_MEMORY , &mem) != 0) { close(kfd_fd); return 1; } close(kfd_fd); // used = total - available uint64_t total = 0; int ret = get_total_memory(&total); if (ret == 0 && total > 0 && mem.available < total) { *used = total - mem.available; return 0; } return 1; } int KFDNode::get_cache_info(rsmi_gpu_cache_info_t *info) { if (info == nullptr) return EINVAL; uint64_t caches_count = 0; int ret = get_property_value("caches_count", &caches_count); if (ret != 0) return ret; // /sys/class/kfd/kfd/topology/nodes/1/caches/0/properties std::string f_path = kKFDNodesPathRoot; f_path += "/"; f_path += std::to_string(node_indx_); f_path += "/"; f_path += "caches/"; info->num_cache_types = 0; for (unsigned int cache_id = 0; cache_id < caches_count; cache_id++) { const auto prop_file = f_path + std::to_string(cache_id) + "/properties"; try { std::string level = get_properties_from_file(prop_file, "level "); int cache_level = std::stoi(level); if (cache_level < 0) continue; std::string type = get_properties_from_file(prop_file, "type "); int cache_type = std::stoi(type); if (cache_type <= 0) continue; std::string size = get_properties_from_file(prop_file, "size "); int cache_size = std::stoi(size); if (cache_size <= 0) continue; std::string sibling_map = get_properties_from_file(prop_file, "sibling_map "); uint32_t num_cu_shared = static_cast(std::count(sibling_map.begin(), sibling_map.end(), '1')); bool is_count_already = false; for (unsigned int i = 0; i < info->num_cache_types; i++) { if (info->cache[i].cache_level == static_cast(cache_level) && info->cache[i].flags == static_cast(cache_type) && info->cache[i].cache_size_kb == static_cast(cache_size) && info->cache[i].max_num_cu_shared == num_cu_shared) { is_count_already = true; info->cache[i].num_cache_instance++; break; } } if (is_count_already) continue; if (info->num_cache_types >= RSMI_MAX_CACHE_TYPES) return 1; info->cache[info->num_cache_types].cache_level = cache_level; info->cache[info->num_cache_types].cache_size_kb = cache_size; info->cache[info->num_cache_types].max_num_cu_shared = num_cu_shared; info->cache[info->num_cache_types].num_cache_instance = 1; info->cache[info->num_cache_types].flags = cache_type; info->num_cache_types++; } catch (...) { continue; } } return 0; } // /sys/class/kfd/kfd/topology/nodes/*/properties int read_node_properties(uint32_t node, std::string property_name, uint64_t *val) { std::ostringstream ss; std::string propertiesFullPath = "/sys/class/kfd/kfd/topology/nodes/" + std::to_string(node) + "/properties"; int retVal = EINVAL; if (property_name.empty() || val == nullptr) { ss << __PRETTY_FUNCTION__ << " | File: " << propertiesFullPath << " | Issue: Could not read node #" << std::to_string(node) << ", property_name is empty or *val is nullptr " << " | return = " << std::to_string(retVal) << " | "; LOG_DEBUG(ss); return retVal; } std::shared_ptr myNode = std::shared_ptr(new KFDNode(node)); myNode->Initialize(); if (KFDNodeSupported(node)) { retVal = myNode->get_property_value(property_name, val); ss << __PRETTY_FUNCTION__ << " | File: " << propertiesFullPath << " | Successfully read node #" << std::to_string(node) << " for property_name = " << property_name << " | Data (" << property_name << ") * val = " << std::to_string(*val) << " | return = " << std::to_string(retVal) << " | "; LOG_DEBUG(ss); } else { retVal = 1; ss << __PRETTY_FUNCTION__ << " | File: " << propertiesFullPath << " | Issue: Could not read node #" << std::to_string(node) << ", KFD node was an unsupported node." << " | return = " << std::to_string(retVal) << " | "; LOG_ERROR(ss); } return retVal; } // /sys/class/kfd/kfd/topology/nodes/*/gpu_id int get_gpu_id(uint32_t node, uint64_t *gpu_id) { std::ostringstream ss; std::string gpu_id_FullPath = "/sys/class/kfd/kfd/topology/nodes/" + std::to_string(node) + "/gpu_id"; int retVal = EINVAL; if (gpu_id == nullptr) { ss << __PRETTY_FUNCTION__ << " | File: " << gpu_id_FullPath << " | Issue: Could not read node #" << std::to_string(node) << ", gpu_id is a nullptr " << " | return = " << std::to_string(retVal) << " | "; LOG_DEBUG(ss); return retVal; } std::shared_ptr myNode = std::shared_ptr(new KFDNode(node)); myNode->Initialize(); if (KFDNodeSupported(node)) { retVal = ReadKFDGpuId(node, gpu_id); ss << __PRETTY_FUNCTION__ << " | File: " << gpu_id_FullPath << " | Successfully read node #" << std::to_string(node) << " for gpu_id" << " | Data (gpu_id) *gpu_id = " << std::to_string(*gpu_id) << " | return = " << std::to_string(retVal) << " | "; LOG_DEBUG(ss); } else { retVal = 1; ss << __PRETTY_FUNCTION__ << " | File: " << gpu_id_FullPath << " | Issue: Could not read node #" << std::to_string(node) << ", KFD node was an unsupported node." << " | return = " << std::to_string(retVal) << " | "; LOG_ERROR(ss); } return retVal; } // /sys/class/kfd/kfd/topology/nodes/*/properties | grep gfx_target_version int KFDNode::get_gfx_target_version(uint64_t *gfx_target_version) { std::ostringstream ss; std::string properties_path = "/sys/class/kfd/kfd/topology/nodes/" + std::to_string(this->node_indx_) + "/properties"; uint64_t gfx_version = 0; int ret = read_node_properties(this->node_indx_, "gfx_target_version", &gfx_version); *gfx_target_version = gfx_version; ss << __PRETTY_FUNCTION__ << " | File: " << properties_path << " | Read node: " << std::to_string(this->node_indx_) << " for gfx_target_version" << " | Data (*gfx_target_version): " << std::to_string(*gfx_target_version) << " | Return: " << getRSMIStatusString(amd::smi::ErrnoToRsmiStatus(ret), false) << " | "; LOG_DEBUG(ss); return ret; } int32_t KFDNode::get_simd_per_cu(uint64_t* simd_per_cu) const { const std::string properties_path("/sys/class/kfd/kfd/topology/nodes/" + std::to_string(this->node_indx_) + "/properties"); auto tmp_simd_per_cu = uint64_t(0); auto ret = read_node_properties(this->node_indx_, "simd_per_cu", &tmp_simd_per_cu); *simd_per_cu = tmp_simd_per_cu; return ret; } int32_t KFDNode::get_simd_count(uint64_t* simd_count) const { const std::string properties_path("/sys/class/kfd/kfd/topology/nodes/" + std::to_string(this->node_indx_) + "/properties"); auto tmp_simd_count = uint64_t(0); auto ret = read_node_properties(this->node_indx_, "simd_count", &tmp_simd_count); *simd_count = tmp_simd_count; return ret; } // Public interface for device // /sys/class/kfd/kfd/topology/nodes/*/gpu_id int KFDNode::get_gpu_id(uint64_t *gpu_id) { std::ostringstream ss; std::string gpuid_path = "/sys/class/kfd/kfd/topology/nodes/" + std::to_string(this->node_indx_) + "/gpu_id"; const uint64_t undefined_gpu_id = std::numeric_limits::max(); std::string gpu_id_string = ""; *gpu_id = undefined_gpu_id; int ret = ReadSysfsStr(gpuid_path, &gpu_id_string); if (ret != 0 || gpu_id_string.empty()) { ss << __PRETTY_FUNCTION__ << " | File: " << gpuid_path << " | Data (*gpu_id): empty or nullptr" << " | Issue: Could not read node #" << std::to_string(this->node_indx_) << ". KFD node was an unsupported node or value read was empty." << " | Return: " << getRSMIStatusString(amd::smi::ErrnoToRsmiStatus(ret), false) << " | "; LOG_ERROR(ss); return ret; } *gpu_id = std::stoull(gpu_id_string); if (*gpu_id == 0) { // CPU node - return not supported *gpu_id = undefined_gpu_id; ret = ENOENT; // map to RSMI_STATUS_NOT_SUPPORTED } ss << __PRETTY_FUNCTION__ << " | File: " << gpuid_path << " | Read node #: " << std::to_string(this->node_indx_) << " | Data (*gpu_id): " << std::to_string(*gpu_id) << " | Return: " << getRSMIStatusString(amd::smi::ErrnoToRsmiStatus(ret), false) << " | "; LOG_DEBUG(ss); return ret; } // Public interface for device // /sys/class/kfd/kfd/topology/nodes/ int KFDNode::get_node_id(uint32_t *node_id) { std::ostringstream ss; int ret = 0; std::string nodeid_path = "/sys/class/kfd/kfd/topology/nodes/" + std::to_string(this->node_indx_); *node_id = this->node_indx_; ss << __PRETTY_FUNCTION__ << " | File: " << nodeid_path << " | Read node #: " << std::to_string(this->node_indx_) << " | Data (*node_id): " << std::to_string(*node_id) << " | Return: " << getRSMIStatusString(amd::smi::ErrnoToRsmiStatus(ret), false) << " | "; LOG_DEBUG(ss); return ret; } } // namespace amd::smi