Debian Codegraph @

drivers/vfio/pci/vfio_pci_dmabuf.c

raw
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.
3 */
4 #include <linux/dma-buf-mapping.h>
5 #include <linux/pci-p2pdma.h>
6 #include <linux/dma-resv.h>
7
8 #include "vfio_pci_priv.h"
9
10 MODULE_IMPORT_NS("DMA_BUF");
11
12 struct vfio_pci_dma_buf {
13 struct dma_buf *dmabuf;
14 struct vfio_pci_core_device *vdev;
15 struct list_head dmabufs_elm;
16 size_t size;
17 struct phys_vec *phys_vec;
18 struct p2pdma_provider *provider;
19 u32 nr_ranges;
20 u8 revoked : 1;
21 };
22
23 static int vfio_pci_dma_buf_pin(struct dma_buf_attachment *attachment)
24 {
25 return -EOPNOTSUPP;
26 }
27
28 static void vfio_pci_dma_buf_unpin(struct dma_buf_attachment *attachment)
29 {
30 /* Do nothing */
31 }
32
33 static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf,
34 struct dma_buf_attachment *attachment)
35 {
36 struct vfio_pci_dma_buf *priv = dmabuf->priv;
37
38 if (!attachment->peer2peer)
39 return -EOPNOTSUPP;
40
41 if (priv->revoked)
42 return -ENODEV;
43
44 return 0;
45 }
46
47 static struct sg_table *
48 vfio_pci_dma_buf_map(struct dma_buf_attachment *attachment,
49 enum dma_data_direction dir)
50 {
51 struct vfio_pci_dma_buf *priv = attachment->dmabuf->priv;
52
53 dma_resv_assert_held(priv->dmabuf->resv);
54
55 if (priv->revoked)
56 return ERR_PTR(-ENODEV);
57
58 return dma_buf_phys_vec_to_sgt(attachment, priv->provider,
59 priv->phys_vec, priv->nr_ranges,
60 priv->size, dir);
61 }
62
63 static void vfio_pci_dma_buf_unmap(struct dma_buf_attachment *attachment,
64 struct sg_table *sgt,
65 enum dma_data_direction dir)
66 {
67 dma_buf_free_sgt(attachment, sgt, dir);
68 }
69
70 static void vfio_pci_dma_buf_release(struct dma_buf *dmabuf)
71 {
72 struct vfio_pci_dma_buf *priv = dmabuf->priv;
73
74 /*
75 * Either this or vfio_pci_dma_buf_cleanup() will remove from the list.
76 * The refcount prevents both.
77 */
78 if (priv->vdev) {
79 down_write(&priv->vdev->memory_lock);
80 list_del_init(&priv->dmabufs_elm);
81 up_write(&priv->vdev->memory_lock);
82 vfio_device_put_registration(&priv->vdev->vdev);
83 }
84 kfree(priv->phys_vec);
85 kfree(priv);
86 }
87
88 static const struct dma_buf_ops vfio_pci_dmabuf_ops = {
89 .pin = vfio_pci_dma_buf_pin,
90 .unpin = vfio_pci_dma_buf_unpin,
91 .attach = vfio_pci_dma_buf_attach,
92 .map_dma_buf = vfio_pci_dma_buf_map,
93 .unmap_dma_buf = vfio_pci_dma_buf_unmap,
94 .release = vfio_pci_dma_buf_release,
95 };
96
97 /*
98 * This is a temporary "private interconnect" between VFIO DMABUF and iommufd.
99 * It allows the two co-operating drivers to exchange the physical address of
100 * the BAR. This is to be replaced with a formal DMABUF system for negotiated
101 * interconnect types.
102 *
103 * If this function succeeds the following are true:
104 * - There is one physical range and it is pointing to MMIO
105 * - When move_notify is called it means revoke, not move, vfio_dma_buf_map
106 * will fail if it is currently revoked
107 */
108 int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment,
109 struct phys_vec *phys)
110 {
111 struct vfio_pci_dma_buf *priv;
112
113 dma_resv_assert_held(attachment->dmabuf->resv);
114
115 if (attachment->dmabuf->ops != &vfio_pci_dmabuf_ops)
116 return -EOPNOTSUPP;
117
118 priv = attachment->dmabuf->priv;
119 if (priv->revoked)
120 return -ENODEV;
121
122 /* More than one range to iommufd will require proper DMABUF support */
123 if (priv->nr_ranges != 1)
124 return -EOPNOTSUPP;
125
126 *phys = priv->phys_vec[0];
127 return 0;
128 }
129 EXPORT_SYMBOL_FOR_MODULES(vfio_pci_dma_buf_iommufd_map, "iommufd");
130
131 int vfio_pci_core_fill_phys_vec(struct phys_vec *phys_vec,
132 struct vfio_region_dma_range *dma_ranges,
133 size_t nr_ranges, phys_addr_t start,
134 phys_addr_t len)
135 {
136 phys_addr_t max_addr;
137 unsigned int i;
138
139 max_addr = start + len;
140 for (i = 0; i < nr_ranges; i++) {
141 phys_addr_t end;
142
143 if (!dma_ranges[i].length)
144 return -EINVAL;
145
146 if (check_add_overflow(start, dma_ranges[i].offset,
147 &phys_vec[i].paddr) ||
148 check_add_overflow(phys_vec[i].paddr,
149 dma_ranges[i].length, &end))
150 return -EOVERFLOW;
151 if (end > max_addr)
152 return -EINVAL;
153
154 phys_vec[i].len = dma_ranges[i].length;
155 }
156 return 0;
157 }
158 EXPORT_SYMBOL_GPL(vfio_pci_core_fill_phys_vec);
159
160 int vfio_pci_core_get_dmabuf_phys(struct vfio_pci_core_device *vdev,
161 struct p2pdma_provider **provider,
162 unsigned int region_index,
163 struct phys_vec *phys_vec,
164 struct vfio_region_dma_range *dma_ranges,
165 size_t nr_ranges)
166 {
167 struct pci_dev *pdev = vdev->pdev;
168
169 *provider = pcim_p2pdma_provider(pdev, region_index);
170 if (!*provider)
171 return -EINVAL;
172
173 return vfio_pci_core_fill_phys_vec(
174 phys_vec, dma_ranges, nr_ranges,
175 pci_resource_start(pdev, region_index),
176 pci_resource_len(pdev, region_index));
177 }
178 EXPORT_SYMBOL_GPL(vfio_pci_core_get_dmabuf_phys);
179
180 static int validate_dmabuf_input(struct vfio_device_feature_dma_buf *dma_buf,
181 struct vfio_region_dma_range *dma_ranges,
182 size_t *lengthp)
183 {
184 size_t length = 0;
185 u32 i;
186
187 for (i = 0; i < dma_buf->nr_ranges; i++) {
188 u64 offset = dma_ranges[i].offset;
189 u64 len = dma_ranges[i].length;
190
191 if (!len || !PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len))
192 return -EINVAL;
193
194 if (check_add_overflow(length, len, &length))
195 return -EINVAL;
196 }
197
198 /*
199 * dma_iova_try_alloc() will WARN on if userspace proposes a size that
200 * is too big, eg with lots of ranges.
201 */
202 if ((u64)(length) & DMA_IOVA_USE_SWIOTLB)
203 return -EINVAL;
204
205 *lengthp = length;
206 return 0;
207 }
208
209 int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
210 struct vfio_device_feature_dma_buf __user *arg,
211 size_t argsz)
212 {
213 struct vfio_device_feature_dma_buf get_dma_buf = {};
214 struct vfio_region_dma_range *dma_ranges;
215 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
216 struct vfio_pci_dma_buf *priv;
217 size_t length;
218 int ret;
219
220 if (!vdev->pci_ops || !vdev->pci_ops->get_dmabuf_phys)
221 return -EOPNOTSUPP;
222
223 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
224 sizeof(get_dma_buf));
225 if (ret != 1)
226 return ret;
227
228 if (copy_from_user(&get_dma_buf, arg, sizeof(get_dma_buf)))
229 return -EFAULT;
230
231 if (!get_dma_buf.nr_ranges || get_dma_buf.flags)
232 return -EINVAL;
233
234 /*
235 * For PCI the region_index is the BAR number like everything else.
236 */
237 if (get_dma_buf.region_index >= VFIO_PCI_ROM_REGION_INDEX)
238 return -ENODEV;
239
240 dma_ranges = memdup_array_user(&arg->dma_ranges, get_dma_buf.nr_ranges,
241 sizeof(*dma_ranges));
242 if (IS_ERR(dma_ranges))
243 return PTR_ERR(dma_ranges);
244
245 ret = validate_dmabuf_input(&get_dma_buf, dma_ranges, &length);
246 if (ret)
247 goto err_free_ranges;
248
249 priv = kzalloc_obj(*priv);
250 if (!priv) {
251 ret = -ENOMEM;
252 goto err_free_ranges;
253 }
254 priv->phys_vec = kzalloc_objs(*priv->phys_vec, get_dma_buf.nr_ranges);
255 if (!priv->phys_vec) {
256 ret = -ENOMEM;
257 goto err_free_priv;
258 }
259
260 priv->vdev = vdev;
261 priv->nr_ranges = get_dma_buf.nr_ranges;
262 priv->size = length;
263 ret = vdev->pci_ops->get_dmabuf_phys(vdev, &priv->provider,
264 get_dma_buf.region_index,
265 priv->phys_vec, dma_ranges,
266 priv->nr_ranges);
267 if (ret)
268 goto err_free_phys;
269
270 kfree(dma_ranges);
271 dma_ranges = NULL;
272
273 if (!vfio_device_try_get_registration(&vdev->vdev)) {
274 ret = -ENODEV;
275 goto err_free_phys;
276 }
277
278 exp_info.ops = &vfio_pci_dmabuf_ops;
279 exp_info.size = priv->size;
280 exp_info.flags = get_dma_buf.open_flags;
281 exp_info.priv = priv;
282
283 priv->dmabuf = dma_buf_export(&exp_info);
284 if (IS_ERR(priv->dmabuf)) {
285 ret = PTR_ERR(priv->dmabuf);
286 goto err_dev_put;
287 }
288
289 /* dma_buf_put() now frees priv */
290 INIT_LIST_HEAD(&priv->dmabufs_elm);
291 down_write(&vdev->memory_lock);
292 dma_resv_lock(priv->dmabuf->resv, NULL);
293 priv->revoked = !__vfio_pci_memory_enabled(vdev);
294 list_add_tail(&priv->dmabufs_elm, &vdev->dmabufs);
295 dma_resv_unlock(priv->dmabuf->resv);
296 up_write(&vdev->memory_lock);
297
298 /*
299 * dma_buf_fd() consumes the reference, when the file closes the dmabuf
300 * will be released.
301 */
302 ret = dma_buf_fd(priv->dmabuf, get_dma_buf.open_flags);
303 if (ret < 0)
304 dma_buf_put(priv->dmabuf);
305
306 return ret;
307
308 err_dev_put:
309 vfio_device_put_registration(&vdev->vdev);
310 err_free_phys:
311 kfree(priv->phys_vec);
312 err_free_priv:
313 kfree(priv);
314 err_free_ranges:
315 kfree(dma_ranges);
316 return ret;
317 }
318
319 void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked)
320 {
321 struct vfio_pci_dma_buf *priv;
322 struct vfio_pci_dma_buf *tmp;
323
324 lockdep_assert_held_write(&vdev->memory_lock);
325
326 list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) {
327 if (!get_file_active(&priv->dmabuf->file))
328 continue;
329
330 if (priv->revoked != revoked) {
331 dma_resv_lock(priv->dmabuf->resv, NULL);
332 priv->revoked = revoked;
333 dma_buf_move_notify(priv->dmabuf);
334 dma_resv_unlock(priv->dmabuf->resv);
335 }
336 fput(priv->dmabuf->file);
337 }
338 }
339
340 void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev)
341 {
342 struct vfio_pci_dma_buf *priv;
343 struct vfio_pci_dma_buf *tmp;
344
345 down_write(&vdev->memory_lock);
346 list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) {
347 if (!get_file_active(&priv->dmabuf->file))
348 continue;
349
350 dma_resv_lock(priv->dmabuf->resv, NULL);
351 list_del_init(&priv->dmabufs_elm);
352 priv->vdev = NULL;
353 priv->revoked = true;
354 dma_buf_move_notify(priv->dmabuf);
355 dma_resv_unlock(priv->dmabuf->resv);
356 vfio_device_put_registration(&vdev->vdev);
357 fput(priv->dmabuf->file);
358 }
359 up_write(&vdev->memory_lock);
360 }