| 1 |
// SPDX-License-Identifier: GPL-2.0-only |
| 2 |
/* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. |
| 3 |
*/ |
| 4 |
#include <linux/dma-buf-mapping.h> |
| 5 |
#include <linux/pci-p2pdma.h> |
| 6 |
#include <linux/dma-resv.h> |
| 7 |
|
| 8 |
#include "vfio_pci_priv.h" |
| 9 |
|
| 10 |
MODULE_IMPORT_NS("DMA_BUF"); |
| 11 |
|
| 12 |
struct vfio_pci_dma_buf { |
| 13 |
struct dma_buf *dmabuf; |
| 14 |
struct vfio_pci_core_device *vdev; |
| 15 |
struct list_head dmabufs_elm; |
| 16 |
size_t size; |
| 17 |
struct phys_vec *phys_vec; |
| 18 |
struct p2pdma_provider *provider; |
| 19 |
u32 nr_ranges; |
| 20 |
u8 revoked : 1; |
| 21 |
}; |
| 22 |
|
| 23 |
static int vfio_pci_dma_buf_pin(struct dma_buf_attachment *attachment) |
| 24 |
{ |
| 25 |
return -EOPNOTSUPP; |
| 26 |
} |
| 27 |
|
| 28 |
static void vfio_pci_dma_buf_unpin(struct dma_buf_attachment *attachment) |
| 29 |
{ |
| 30 |
/* Do nothing */ |
| 31 |
} |
| 32 |
|
| 33 |
static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf, |
| 34 |
struct dma_buf_attachment *attachment) |
| 35 |
{ |
| 36 |
struct vfio_pci_dma_buf *priv = dmabuf->priv; |
| 37 |
|
| 38 |
if (!attachment->peer2peer) |
| 39 |
return -EOPNOTSUPP; |
| 40 |
|
| 41 |
if (priv->revoked) |
| 42 |
return -ENODEV; |
| 43 |
|
| 44 |
return 0; |
| 45 |
} |
| 46 |
|
| 47 |
static struct sg_table * |
| 48 |
vfio_pci_dma_buf_map(struct dma_buf_attachment *attachment, |
| 49 |
enum dma_data_direction dir) |
| 50 |
{ |
| 51 |
struct vfio_pci_dma_buf *priv = attachment->dmabuf->priv; |
| 52 |
|
| 53 |
dma_resv_assert_held(priv->dmabuf->resv); |
| 54 |
|
| 55 |
if (priv->revoked) |
| 56 |
return ERR_PTR(-ENODEV); |
| 57 |
|
| 58 |
return dma_buf_phys_vec_to_sgt(attachment, priv->provider, |
| 59 |
priv->phys_vec, priv->nr_ranges, |
| 60 |
priv->size, dir); |
| 61 |
} |
| 62 |
|
| 63 |
static void vfio_pci_dma_buf_unmap(struct dma_buf_attachment *attachment, |
| 64 |
struct sg_table *sgt, |
| 65 |
enum dma_data_direction dir) |
| 66 |
{ |
| 67 |
dma_buf_free_sgt(attachment, sgt, dir); |
| 68 |
} |
| 69 |
|
| 70 |
static void vfio_pci_dma_buf_release(struct dma_buf *dmabuf) |
| 71 |
{ |
| 72 |
struct vfio_pci_dma_buf *priv = dmabuf->priv; |
| 73 |
|
| 74 |
/* |
| 75 |
* Either this or vfio_pci_dma_buf_cleanup() will remove from the list. |
| 76 |
* The refcount prevents both. |
| 77 |
*/ |
| 78 |
if (priv->vdev) { |
| 79 |
down_write(&priv->vdev->memory_lock); |
| 80 |
list_del_init(&priv->dmabufs_elm); |
| 81 |
up_write(&priv->vdev->memory_lock); |
| 82 |
vfio_device_put_registration(&priv->vdev->vdev); |
| 83 |
} |
| 84 |
kfree(priv->phys_vec); |
| 85 |
kfree(priv); |
| 86 |
} |
| 87 |
|
| 88 |
static const struct dma_buf_ops vfio_pci_dmabuf_ops = { |
| 89 |
.pin = vfio_pci_dma_buf_pin, |
| 90 |
.unpin = vfio_pci_dma_buf_unpin, |
| 91 |
.attach = vfio_pci_dma_buf_attach, |
| 92 |
.map_dma_buf = vfio_pci_dma_buf_map, |
| 93 |
.unmap_dma_buf = vfio_pci_dma_buf_unmap, |
| 94 |
.release = vfio_pci_dma_buf_release, |
| 95 |
}; |
| 96 |
|
| 97 |
/* |
| 98 |
* This is a temporary "private interconnect" between VFIO DMABUF and iommufd. |
| 99 |
* It allows the two co-operating drivers to exchange the physical address of |
| 100 |
* the BAR. This is to be replaced with a formal DMABUF system for negotiated |
| 101 |
* interconnect types. |
| 102 |
* |
| 103 |
* If this function succeeds the following are true: |
| 104 |
* - There is one physical range and it is pointing to MMIO |
| 105 |
* - When move_notify is called it means revoke, not move, vfio_dma_buf_map |
| 106 |
* will fail if it is currently revoked |
| 107 |
*/ |
| 108 |
int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment, |
| 109 |
struct phys_vec *phys) |
| 110 |
{ |
| 111 |
struct vfio_pci_dma_buf *priv; |
| 112 |
|
| 113 |
dma_resv_assert_held(attachment->dmabuf->resv); |
| 114 |
|
| 115 |
if (attachment->dmabuf->ops != &vfio_pci_dmabuf_ops) |
| 116 |
return -EOPNOTSUPP; |
| 117 |
|
| 118 |
priv = attachment->dmabuf->priv; |
| 119 |
if (priv->revoked) |
| 120 |
return -ENODEV; |
| 121 |
|
| 122 |
/* More than one range to iommufd will require proper DMABUF support */ |
| 123 |
if (priv->nr_ranges != 1) |
| 124 |
return -EOPNOTSUPP; |
| 125 |
|
| 126 |
*phys = priv->phys_vec[0]; |
| 127 |
return 0; |
| 128 |
} |
| 129 |
EXPORT_SYMBOL_FOR_MODULES(vfio_pci_dma_buf_iommufd_map, "iommufd"); |
| 130 |
|
| 131 |
int vfio_pci_core_fill_phys_vec(struct phys_vec *phys_vec, |
| 132 |
struct vfio_region_dma_range *dma_ranges, |
| 133 |
size_t nr_ranges, phys_addr_t start, |
| 134 |
phys_addr_t len) |
| 135 |
{ |
| 136 |
phys_addr_t max_addr; |
| 137 |
unsigned int i; |
| 138 |
|
| 139 |
max_addr = start + len; |
| 140 |
for (i = 0; i < nr_ranges; i++) { |
| 141 |
phys_addr_t end; |
| 142 |
|
| 143 |
if (!dma_ranges[i].length) |
| 144 |
return -EINVAL; |
| 145 |
|
| 146 |
if (check_add_overflow(start, dma_ranges[i].offset, |
| 147 |
&phys_vec[i].paddr) || |
| 148 |
check_add_overflow(phys_vec[i].paddr, |
| 149 |
dma_ranges[i].length, &end)) |
| 150 |
return -EOVERFLOW; |
| 151 |
if (end > max_addr) |
| 152 |
return -EINVAL; |
| 153 |
|
| 154 |
phys_vec[i].len = dma_ranges[i].length; |
| 155 |
} |
| 156 |
return 0; |
| 157 |
} |
| 158 |
EXPORT_SYMBOL_GPL(vfio_pci_core_fill_phys_vec); |
| 159 |
|
| 160 |
int vfio_pci_core_get_dmabuf_phys(struct vfio_pci_core_device *vdev, |
| 161 |
struct p2pdma_provider **provider, |
| 162 |
unsigned int region_index, |
| 163 |
struct phys_vec *phys_vec, |
| 164 |
struct vfio_region_dma_range *dma_ranges, |
| 165 |
size_t nr_ranges) |
| 166 |
{ |
| 167 |
struct pci_dev *pdev = vdev->pdev; |
| 168 |
|
| 169 |
*provider = pcim_p2pdma_provider(pdev, region_index); |
| 170 |
if (!*provider) |
| 171 |
return -EINVAL; |
| 172 |
|
| 173 |
return vfio_pci_core_fill_phys_vec( |
| 174 |
phys_vec, dma_ranges, nr_ranges, |
| 175 |
pci_resource_start(pdev, region_index), |
| 176 |
pci_resource_len(pdev, region_index)); |
| 177 |
} |
| 178 |
EXPORT_SYMBOL_GPL(vfio_pci_core_get_dmabuf_phys); |
| 179 |
|
| 180 |
static int validate_dmabuf_input(struct vfio_device_feature_dma_buf *dma_buf, |
| 181 |
struct vfio_region_dma_range *dma_ranges, |
| 182 |
size_t *lengthp) |
| 183 |
{ |
| 184 |
size_t length = 0; |
| 185 |
u32 i; |
| 186 |
|
| 187 |
for (i = 0; i < dma_buf->nr_ranges; i++) { |
| 188 |
u64 offset = dma_ranges[i].offset; |
| 189 |
u64 len = dma_ranges[i].length; |
| 190 |
|
| 191 |
if (!len || !PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len)) |
| 192 |
return -EINVAL; |
| 193 |
|
| 194 |
if (check_add_overflow(length, len, &length)) |
| 195 |
return -EINVAL; |
| 196 |
} |
| 197 |
|
| 198 |
/* |
| 199 |
* dma_iova_try_alloc() will WARN on if userspace proposes a size that |
| 200 |
* is too big, eg with lots of ranges. |
| 201 |
*/ |
| 202 |
if ((u64)(length) & DMA_IOVA_USE_SWIOTLB) |
| 203 |
return -EINVAL; |
| 204 |
|
| 205 |
*lengthp = length; |
| 206 |
return 0; |
| 207 |
} |
| 208 |
|
| 209 |
int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, |
| 210 |
struct vfio_device_feature_dma_buf __user *arg, |
| 211 |
size_t argsz) |
| 212 |
{ |
| 213 |
struct vfio_device_feature_dma_buf get_dma_buf = {}; |
| 214 |
struct vfio_region_dma_range *dma_ranges; |
| 215 |
DEFINE_DMA_BUF_EXPORT_INFO(exp_info); |
| 216 |
struct vfio_pci_dma_buf *priv; |
| 217 |
size_t length; |
| 218 |
int ret; |
| 219 |
|
| 220 |
if (!vdev->pci_ops || !vdev->pci_ops->get_dmabuf_phys) |
| 221 |
return -EOPNOTSUPP; |
| 222 |
|
| 223 |
ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, |
| 224 |
sizeof(get_dma_buf)); |
| 225 |
if (ret != 1) |
| 226 |
return ret; |
| 227 |
|
| 228 |
if (copy_from_user(&get_dma_buf, arg, sizeof(get_dma_buf))) |
| 229 |
return -EFAULT; |
| 230 |
|
| 231 |
if (!get_dma_buf.nr_ranges || get_dma_buf.flags) |
| 232 |
return -EINVAL; |
| 233 |
|
| 234 |
/* |
| 235 |
* For PCI the region_index is the BAR number like everything else. |
| 236 |
*/ |
| 237 |
if (get_dma_buf.region_index >= VFIO_PCI_ROM_REGION_INDEX) |
| 238 |
return -ENODEV; |
| 239 |
|
| 240 |
dma_ranges = memdup_array_user(&arg->dma_ranges, get_dma_buf.nr_ranges, |
| 241 |
sizeof(*dma_ranges)); |
| 242 |
if (IS_ERR(dma_ranges)) |
| 243 |
return PTR_ERR(dma_ranges); |
| 244 |
|
| 245 |
ret = validate_dmabuf_input(&get_dma_buf, dma_ranges, &length); |
| 246 |
if (ret) |
| 247 |
goto err_free_ranges; |
| 248 |
|
| 249 |
priv = kzalloc_obj(*priv); |
| 250 |
if (!priv) { |
| 251 |
ret = -ENOMEM; |
| 252 |
goto err_free_ranges; |
| 253 |
} |
| 254 |
priv->phys_vec = kzalloc_objs(*priv->phys_vec, get_dma_buf.nr_ranges); |
| 255 |
if (!priv->phys_vec) { |
| 256 |
ret = -ENOMEM; |
| 257 |
goto err_free_priv; |
| 258 |
} |
| 259 |
|
| 260 |
priv->vdev = vdev; |
| 261 |
priv->nr_ranges = get_dma_buf.nr_ranges; |
| 262 |
priv->size = length; |
| 263 |
ret = vdev->pci_ops->get_dmabuf_phys(vdev, &priv->provider, |
| 264 |
get_dma_buf.region_index, |
| 265 |
priv->phys_vec, dma_ranges, |
| 266 |
priv->nr_ranges); |
| 267 |
if (ret) |
| 268 |
goto err_free_phys; |
| 269 |
|
| 270 |
kfree(dma_ranges); |
| 271 |
dma_ranges = NULL; |
| 272 |
|
| 273 |
if (!vfio_device_try_get_registration(&vdev->vdev)) { |
| 274 |
ret = -ENODEV; |
| 275 |
goto err_free_phys; |
| 276 |
} |
| 277 |
|
| 278 |
exp_info.ops = &vfio_pci_dmabuf_ops; |
| 279 |
exp_info.size = priv->size; |
| 280 |
exp_info.flags = get_dma_buf.open_flags; |
| 281 |
exp_info.priv = priv; |
| 282 |
|
| 283 |
priv->dmabuf = dma_buf_export(&exp_info); |
| 284 |
if (IS_ERR(priv->dmabuf)) { |
| 285 |
ret = PTR_ERR(priv->dmabuf); |
| 286 |
goto err_dev_put; |
| 287 |
} |
| 288 |
|
| 289 |
/* dma_buf_put() now frees priv */ |
| 290 |
INIT_LIST_HEAD(&priv->dmabufs_elm); |
| 291 |
down_write(&vdev->memory_lock); |
| 292 |
dma_resv_lock(priv->dmabuf->resv, NULL); |
| 293 |
priv->revoked = !__vfio_pci_memory_enabled(vdev); |
| 294 |
list_add_tail(&priv->dmabufs_elm, &vdev->dmabufs); |
| 295 |
dma_resv_unlock(priv->dmabuf->resv); |
| 296 |
up_write(&vdev->memory_lock); |
| 297 |
|
| 298 |
/* |
| 299 |
* dma_buf_fd() consumes the reference, when the file closes the dmabuf |
| 300 |
* will be released. |
| 301 |
*/ |
| 302 |
ret = dma_buf_fd(priv->dmabuf, get_dma_buf.open_flags); |
| 303 |
if (ret < 0) |
| 304 |
dma_buf_put(priv->dmabuf); |
| 305 |
|
| 306 |
return ret; |
| 307 |
|
| 308 |
err_dev_put: |
| 309 |
vfio_device_put_registration(&vdev->vdev); |
| 310 |
err_free_phys: |
| 311 |
kfree(priv->phys_vec); |
| 312 |
err_free_priv: |
| 313 |
kfree(priv); |
| 314 |
err_free_ranges: |
| 315 |
kfree(dma_ranges); |
| 316 |
return ret; |
| 317 |
} |
| 318 |
|
| 319 |
void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked) |
| 320 |
{ |
| 321 |
struct vfio_pci_dma_buf *priv; |
| 322 |
struct vfio_pci_dma_buf *tmp; |
| 323 |
|
| 324 |
lockdep_assert_held_write(&vdev->memory_lock); |
| 325 |
|
| 326 |
list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) { |
| 327 |
if (!get_file_active(&priv->dmabuf->file)) |
| 328 |
continue; |
| 329 |
|
| 330 |
if (priv->revoked != revoked) { |
| 331 |
dma_resv_lock(priv->dmabuf->resv, NULL); |
| 332 |
priv->revoked = revoked; |
| 333 |
dma_buf_move_notify(priv->dmabuf); |
| 334 |
dma_resv_unlock(priv->dmabuf->resv); |
| 335 |
} |
| 336 |
fput(priv->dmabuf->file); |
| 337 |
} |
| 338 |
} |
| 339 |
|
| 340 |
void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev) |
| 341 |
{ |
| 342 |
struct vfio_pci_dma_buf *priv; |
| 343 |
struct vfio_pci_dma_buf *tmp; |
| 344 |
|
| 345 |
down_write(&vdev->memory_lock); |
| 346 |
list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) { |
| 347 |
if (!get_file_active(&priv->dmabuf->file)) |
| 348 |
continue; |
| 349 |
|
| 350 |
dma_resv_lock(priv->dmabuf->resv, NULL); |
| 351 |
list_del_init(&priv->dmabufs_elm); |
| 352 |
priv->vdev = NULL; |
| 353 |
priv->revoked = true; |
| 354 |
dma_buf_move_notify(priv->dmabuf); |
| 355 |
dma_resv_unlock(priv->dmabuf->resv); |
| 356 |
vfio_device_put_registration(&vdev->vdev); |
| 357 |
fput(priv->dmabuf->file); |
| 358 |
} |
| 359 |
up_write(&vdev->memory_lock); |
| 360 |
} |