| 1 |
// SPDX-License-Identifier: GPL-2.0-only |
| 2 |
/* |
| 3 |
* VFIO PCI config space virtualization |
| 4 |
* |
| 5 |
* Copyright (C) 2012 Red Hat, Inc. All rights reserved. |
| 6 |
* Author: Alex Williamson <[email protected]> |
| 7 |
* |
| 8 |
* Derived from original vfio: |
| 9 |
* Copyright 2010 Cisco Systems, Inc. All rights reserved. |
| 10 |
* Author: Tom Lyon, [email protected] |
| 11 |
*/ |
| 12 |
|
| 13 |
/* |
| 14 |
* This code handles reading and writing of PCI configuration registers. |
| 15 |
* This is hairy because we want to allow a lot of flexibility to the |
| 16 |
* user driver, but cannot trust it with all of the config fields. |
| 17 |
* Tables determine which fields can be read and written, as well as |
| 18 |
* which fields are 'virtualized' - special actions and translations to |
| 19 |
* make it appear to the user that he has control, when in fact things |
| 20 |
* must be negotiated with the underlying OS. |
| 21 |
*/ |
| 22 |
|
| 23 |
#include <linux/fs.h> |
| 24 |
#include <linux/pci.h> |
| 25 |
#include <linux/uaccess.h> |
| 26 |
#include <linux/vfio.h> |
| 27 |
#include <linux/slab.h> |
| 28 |
|
| 29 |
#include "vfio_pci_priv.h" |
| 30 |
|
| 31 |
/* Fake capability ID for standard config space */ |
| 32 |
#define PCI_CAP_ID_BASIC 0 |
| 33 |
|
| 34 |
#define is_bar(offset) \ |
| 35 |
((offset >= PCI_BASE_ADDRESS_0 && offset < PCI_BASE_ADDRESS_5 + 4) || \ |
| 36 |
(offset >= PCI_ROM_ADDRESS && offset < PCI_ROM_ADDRESS + 4)) |
| 37 |
|
| 38 |
/* |
| 39 |
* Lengths of PCI Config Capabilities |
| 40 |
* 0: Removed from the user visible capability list |
| 41 |
* FF: Variable length |
| 42 |
*/ |
| 43 |
static const u8 pci_cap_length[PCI_CAP_ID_MAX + 1] = { |
| 44 |
[PCI_CAP_ID_BASIC] = PCI_STD_HEADER_SIZEOF, /* pci config header */ |
| 45 |
[PCI_CAP_ID_PM] = PCI_PM_SIZEOF, |
| 46 |
[PCI_CAP_ID_AGP] = PCI_AGP_SIZEOF, |
| 47 |
[PCI_CAP_ID_VPD] = PCI_CAP_VPD_SIZEOF, |
| 48 |
[PCI_CAP_ID_SLOTID] = 0, /* bridge - don't care */ |
| 49 |
[PCI_CAP_ID_MSI] = 0xFF, /* 10, 14, 20, or 24 */ |
| 50 |
[PCI_CAP_ID_CHSWP] = 0, /* cpci - not yet */ |
| 51 |
[PCI_CAP_ID_PCIX] = 0xFF, /* 8 or 24 */ |
| 52 |
[PCI_CAP_ID_HT] = 0xFF, /* hypertransport */ |
| 53 |
[PCI_CAP_ID_VNDR] = 0xFF, /* variable */ |
| 54 |
[PCI_CAP_ID_DBG] = 0, /* debug - don't care */ |
| 55 |
[PCI_CAP_ID_CCRC] = 0, /* cpci - not yet */ |
| 56 |
[PCI_CAP_ID_SHPC] = 0, /* hotswap - not yet */ |
| 57 |
[PCI_CAP_ID_SSVID] = 0, /* bridge - don't care */ |
| 58 |
[PCI_CAP_ID_AGP3] = 0, /* AGP8x - not yet */ |
| 59 |
[PCI_CAP_ID_SECDEV] = 0, /* secure device not yet */ |
| 60 |
[PCI_CAP_ID_EXP] = 0xFF, /* 20 or 44 */ |
| 61 |
[PCI_CAP_ID_MSIX] = PCI_CAP_MSIX_SIZEOF, |
| 62 |
[PCI_CAP_ID_SATA] = 0xFF, |
| 63 |
[PCI_CAP_ID_AF] = PCI_CAP_AF_SIZEOF, |
| 64 |
}; |
| 65 |
|
| 66 |
/* |
| 67 |
* Lengths of PCIe/PCI-X Extended Config Capabilities |
| 68 |
* 0: Removed or masked from the user visible capability list |
| 69 |
* FF: Variable length |
| 70 |
*/ |
| 71 |
static const u16 pci_ext_cap_length[PCI_EXT_CAP_ID_MAX + 1] = { |
| 72 |
[PCI_EXT_CAP_ID_ERR] = PCI_ERR_ROOT_COMMAND, |
| 73 |
[PCI_EXT_CAP_ID_VC] = 0xFF, |
| 74 |
[PCI_EXT_CAP_ID_DSN] = PCI_EXT_CAP_DSN_SIZEOF, |
| 75 |
[PCI_EXT_CAP_ID_PWR] = PCI_EXT_CAP_PWR_SIZEOF, |
| 76 |
[PCI_EXT_CAP_ID_RCLD] = 0, /* root only - don't care */ |
| 77 |
[PCI_EXT_CAP_ID_RCILC] = 0, /* root only - don't care */ |
| 78 |
[PCI_EXT_CAP_ID_RCEC] = 0, /* root only - don't care */ |
| 79 |
[PCI_EXT_CAP_ID_MFVC] = 0xFF, |
| 80 |
[PCI_EXT_CAP_ID_VC9] = 0xFF, /* same as CAP_ID_VC */ |
| 81 |
[PCI_EXT_CAP_ID_RCRB] = 0, /* root only - don't care */ |
| 82 |
[PCI_EXT_CAP_ID_VNDR] = 0xFF, |
| 83 |
[PCI_EXT_CAP_ID_CAC] = 0, /* obsolete */ |
| 84 |
[PCI_EXT_CAP_ID_ACS] = 0xFF, |
| 85 |
[PCI_EXT_CAP_ID_ARI] = PCI_EXT_CAP_ARI_SIZEOF, |
| 86 |
[PCI_EXT_CAP_ID_ATS] = PCI_EXT_CAP_ATS_SIZEOF, |
| 87 |
[PCI_EXT_CAP_ID_SRIOV] = PCI_EXT_CAP_SRIOV_SIZEOF, |
| 88 |
[PCI_EXT_CAP_ID_MRIOV] = 0, /* not yet */ |
| 89 |
[PCI_EXT_CAP_ID_MCAST] = PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF, |
| 90 |
[PCI_EXT_CAP_ID_PRI] = PCI_EXT_CAP_PRI_SIZEOF, |
| 91 |
[PCI_EXT_CAP_ID_AMD_XXX] = 0, /* not yet */ |
| 92 |
[PCI_EXT_CAP_ID_REBAR] = 0xFF, |
| 93 |
[PCI_EXT_CAP_ID_DPA] = 0xFF, |
| 94 |
[PCI_EXT_CAP_ID_TPH] = 0xFF, |
| 95 |
[PCI_EXT_CAP_ID_LTR] = PCI_EXT_CAP_LTR_SIZEOF, |
| 96 |
[PCI_EXT_CAP_ID_SECPCI] = 0, /* not yet */ |
| 97 |
[PCI_EXT_CAP_ID_PMUX] = 0, /* not yet */ |
| 98 |
[PCI_EXT_CAP_ID_PASID] = 0, /* not yet */ |
| 99 |
[PCI_EXT_CAP_ID_DVSEC] = 0xFF, |
| 100 |
}; |
| 101 |
|
| 102 |
/* |
| 103 |
* Read/Write Permission Bits - one bit for each bit in capability |
| 104 |
* Any field can be read if it exists, but what is read depends on |
| 105 |
* whether the field is 'virtualized', or just pass through to the |
| 106 |
* hardware. Any virtualized field is also virtualized for writes. |
| 107 |
* Writes are only permitted if they have a 1 bit here. |
| 108 |
*/ |
| 109 |
struct perm_bits { |
| 110 |
u8 *virt; /* read/write virtual data, not hw */ |
| 111 |
u8 *write; /* writeable bits */ |
| 112 |
int (*readfn)(struct vfio_pci_core_device *vdev, int pos, int count, |
| 113 |
struct perm_bits *perm, int offset, __le32 *val); |
| 114 |
int (*writefn)(struct vfio_pci_core_device *vdev, int pos, int count, |
| 115 |
struct perm_bits *perm, int offset, __le32 val); |
| 116 |
}; |
| 117 |
|
| 118 |
#define NO_VIRT 0 |
| 119 |
#define ALL_VIRT 0xFFFFFFFFU |
| 120 |
#define NO_WRITE 0 |
| 121 |
#define ALL_WRITE 0xFFFFFFFFU |
| 122 |
|
| 123 |
static int vfio_user_config_read(struct pci_dev *pdev, int offset, |
| 124 |
__le32 *val, int count) |
| 125 |
{ |
| 126 |
int ret = -EINVAL; |
| 127 |
u32 tmp_val = 0; |
| 128 |
|
| 129 |
switch (count) { |
| 130 |
case 1: |
| 131 |
{ |
| 132 |
u8 tmp; |
| 133 |
ret = pci_user_read_config_byte(pdev, offset, &tmp); |
| 134 |
tmp_val = tmp; |
| 135 |
break; |
| 136 |
} |
| 137 |
case 2: |
| 138 |
{ |
| 139 |
u16 tmp; |
| 140 |
ret = pci_user_read_config_word(pdev, offset, &tmp); |
| 141 |
tmp_val = tmp; |
| 142 |
break; |
| 143 |
} |
| 144 |
case 4: |
| 145 |
ret = pci_user_read_config_dword(pdev, offset, &tmp_val); |
| 146 |
break; |
| 147 |
} |
| 148 |
|
| 149 |
*val = cpu_to_le32(tmp_val); |
| 150 |
|
| 151 |
return ret; |
| 152 |
} |
| 153 |
|
| 154 |
static int vfio_user_config_write(struct pci_dev *pdev, int offset, |
| 155 |
__le32 val, int count) |
| 156 |
{ |
| 157 |
int ret = -EINVAL; |
| 158 |
u32 tmp_val = le32_to_cpu(val); |
| 159 |
|
| 160 |
switch (count) { |
| 161 |
case 1: |
| 162 |
ret = pci_user_write_config_byte(pdev, offset, tmp_val); |
| 163 |
break; |
| 164 |
case 2: |
| 165 |
ret = pci_user_write_config_word(pdev, offset, tmp_val); |
| 166 |
break; |
| 167 |
case 4: |
| 168 |
ret = pci_user_write_config_dword(pdev, offset, tmp_val); |
| 169 |
break; |
| 170 |
} |
| 171 |
|
| 172 |
return ret; |
| 173 |
} |
| 174 |
|
| 175 |
static int vfio_default_config_read(struct vfio_pci_core_device *vdev, int pos, |
| 176 |
int count, struct perm_bits *perm, |
| 177 |
int offset, __le32 *val) |
| 178 |
{ |
| 179 |
__le32 virt = 0; |
| 180 |
|
| 181 |
memcpy(val, vdev->vconfig + pos, count); |
| 182 |
|
| 183 |
memcpy(&virt, perm->virt + offset, count); |
| 184 |
|
| 185 |
/* Any non-virtualized bits? */ |
| 186 |
if (cpu_to_le32(~0U >> (32 - (count * 8))) != virt) { |
| 187 |
struct pci_dev *pdev = vdev->pdev; |
| 188 |
__le32 phys_val = 0; |
| 189 |
int ret; |
| 190 |
|
| 191 |
ret = vfio_user_config_read(pdev, pos, &phys_val, count); |
| 192 |
if (ret) |
| 193 |
return ret; |
| 194 |
|
| 195 |
*val = (phys_val & ~virt) | (*val & virt); |
| 196 |
} |
| 197 |
|
| 198 |
return count; |
| 199 |
} |
| 200 |
|
| 201 |
static int vfio_default_config_write(struct vfio_pci_core_device *vdev, int pos, |
| 202 |
int count, struct perm_bits *perm, |
| 203 |
int offset, __le32 val) |
| 204 |
{ |
| 205 |
__le32 virt = 0, write = 0; |
| 206 |
|
| 207 |
memcpy(&write, perm->write + offset, count); |
| 208 |
|
| 209 |
if (!write) |
| 210 |
return count; /* drop, no writable bits */ |
| 211 |
|
| 212 |
memcpy(&virt, perm->virt + offset, count); |
| 213 |
|
| 214 |
/* Virtualized and writable bits go to vconfig */ |
| 215 |
if (write & virt) { |
| 216 |
__le32 virt_val = 0; |
| 217 |
|
| 218 |
memcpy(&virt_val, vdev->vconfig + pos, count); |
| 219 |
|
| 220 |
virt_val &= ~(write & virt); |
| 221 |
virt_val |= (val & (write & virt)); |
| 222 |
|
| 223 |
memcpy(vdev->vconfig + pos, &virt_val, count); |
| 224 |
} |
| 225 |
|
| 226 |
/* Non-virtualized and writable bits go to hardware */ |
| 227 |
if (write & ~virt) { |
| 228 |
struct pci_dev *pdev = vdev->pdev; |
| 229 |
__le32 phys_val = 0; |
| 230 |
int ret; |
| 231 |
|
| 232 |
ret = vfio_user_config_read(pdev, pos, &phys_val, count); |
| 233 |
if (ret) |
| 234 |
return ret; |
| 235 |
|
| 236 |
phys_val &= ~(write & ~virt); |
| 237 |
phys_val |= (val & (write & ~virt)); |
| 238 |
|
| 239 |
ret = vfio_user_config_write(pdev, pos, phys_val, count); |
| 240 |
if (ret) |
| 241 |
return ret; |
| 242 |
} |
| 243 |
|
| 244 |
return count; |
| 245 |
} |
| 246 |
|
| 247 |
/* Allow direct read from hardware, except for capability next pointer */ |
| 248 |
static int vfio_direct_config_read(struct vfio_pci_core_device *vdev, int pos, |
| 249 |
int count, struct perm_bits *perm, |
| 250 |
int offset, __le32 *val) |
| 251 |
{ |
| 252 |
int ret; |
| 253 |
|
| 254 |
ret = vfio_user_config_read(vdev->pdev, pos, val, count); |
| 255 |
if (ret) |
| 256 |
return ret; |
| 257 |
|
| 258 |
if (pos >= PCI_CFG_SPACE_SIZE) { /* Extended cap header mangling */ |
| 259 |
if (offset < 4) |
| 260 |
memcpy(val, vdev->vconfig + pos, count); |
| 261 |
} else if (pos >= PCI_STD_HEADER_SIZEOF) { /* Std cap mangling */ |
| 262 |
if (offset == PCI_CAP_LIST_ID && count > 1) |
| 263 |
memcpy(val, vdev->vconfig + pos, |
| 264 |
min(PCI_CAP_FLAGS, count)); |
| 265 |
else if (offset == PCI_CAP_LIST_NEXT) |
| 266 |
memcpy(val, vdev->vconfig + pos, 1); |
| 267 |
} |
| 268 |
|
| 269 |
return count; |
| 270 |
} |
| 271 |
|
| 272 |
/* Raw access skips any kind of virtualization */ |
| 273 |
static int vfio_raw_config_write(struct vfio_pci_core_device *vdev, int pos, |
| 274 |
int count, struct perm_bits *perm, |
| 275 |
int offset, __le32 val) |
| 276 |
{ |
| 277 |
int ret; |
| 278 |
|
| 279 |
ret = vfio_user_config_write(vdev->pdev, pos, val, count); |
| 280 |
if (ret) |
| 281 |
return ret; |
| 282 |
|
| 283 |
return count; |
| 284 |
} |
| 285 |
|
| 286 |
static int vfio_raw_config_read(struct vfio_pci_core_device *vdev, int pos, |
| 287 |
int count, struct perm_bits *perm, |
| 288 |
int offset, __le32 *val) |
| 289 |
{ |
| 290 |
int ret; |
| 291 |
|
| 292 |
ret = vfio_user_config_read(vdev->pdev, pos, val, count); |
| 293 |
if (ret) |
| 294 |
return ret; |
| 295 |
|
| 296 |
return count; |
| 297 |
} |
| 298 |
|
| 299 |
/* Virt access uses only virtualization */ |
| 300 |
static int vfio_virt_config_write(struct vfio_pci_core_device *vdev, int pos, |
| 301 |
int count, struct perm_bits *perm, |
| 302 |
int offset, __le32 val) |
| 303 |
{ |
| 304 |
memcpy(vdev->vconfig + pos, &val, count); |
| 305 |
return count; |
| 306 |
} |
| 307 |
|
| 308 |
static int vfio_virt_config_read(struct vfio_pci_core_device *vdev, int pos, |
| 309 |
int count, struct perm_bits *perm, |
| 310 |
int offset, __le32 *val) |
| 311 |
{ |
| 312 |
memcpy(val, vdev->vconfig + pos, count); |
| 313 |
return count; |
| 314 |
} |
| 315 |
|
| 316 |
static struct perm_bits direct_ro_perms = { |
| 317 |
.readfn = vfio_direct_config_read, |
| 318 |
}; |
| 319 |
|
| 320 |
/* Default capability regions to read-only, no-virtualization */ |
| 321 |
static struct perm_bits cap_perms[PCI_CAP_ID_MAX + 1] = { |
| 322 |
[0 ... PCI_CAP_ID_MAX] = { .readfn = vfio_direct_config_read } |
| 323 |
}; |
| 324 |
static struct perm_bits ecap_perms[PCI_EXT_CAP_ID_MAX + 1] = { |
| 325 |
[0 ... PCI_EXT_CAP_ID_MAX] = { .readfn = vfio_direct_config_read } |
| 326 |
}; |
| 327 |
/* |
| 328 |
* Default unassigned regions to raw read-write access. Some devices |
| 329 |
* require this to function as they hide registers between the gaps in |
| 330 |
* config space (be2net). Like MMIO and I/O port registers, we have |
| 331 |
* to trust the hardware isolation. |
| 332 |
*/ |
| 333 |
static struct perm_bits unassigned_perms = { |
| 334 |
.readfn = vfio_raw_config_read, |
| 335 |
.writefn = vfio_raw_config_write |
| 336 |
}; |
| 337 |
|
| 338 |
static struct perm_bits virt_perms = { |
| 339 |
.readfn = vfio_virt_config_read, |
| 340 |
.writefn = vfio_virt_config_write |
| 341 |
}; |
| 342 |
|
| 343 |
static void free_perm_bits(struct perm_bits *perm) |
| 344 |
{ |
| 345 |
kfree(perm->virt); |
| 346 |
kfree(perm->write); |
| 347 |
perm->virt = NULL; |
| 348 |
perm->write = NULL; |
| 349 |
} |
| 350 |
|
| 351 |
static int alloc_perm_bits(struct perm_bits *perm, int size) |
| 352 |
{ |
| 353 |
/* |
| 354 |
* Round up all permission bits to the next dword, this lets us |
| 355 |
* ignore whether a read/write exceeds the defined capability |
| 356 |
* structure. We can do this because: |
| 357 |
* - Standard config space is already dword aligned |
| 358 |
* - Capabilities are all dword aligned (bits 0:1 of next reserved) |
| 359 |
* - Express capabilities defined as dword aligned |
| 360 |
*/ |
| 361 |
size = round_up(size, 4); |
| 362 |
|
| 363 |
/* |
| 364 |
* Zero state is |
| 365 |
* - All Readable, None Writeable, None Virtualized |
| 366 |
*/ |
| 367 |
perm->virt = kzalloc(size, GFP_KERNEL); |
| 368 |
perm->write = kzalloc(size, GFP_KERNEL); |
| 369 |
if (!perm->virt || !perm->write) { |
| 370 |
free_perm_bits(perm); |
| 371 |
return -ENOMEM; |
| 372 |
} |
| 373 |
|
| 374 |
perm->readfn = vfio_default_config_read; |
| 375 |
perm->writefn = vfio_default_config_write; |
| 376 |
|
| 377 |
return 0; |
| 378 |
} |
| 379 |
|
| 380 |
/* |
| 381 |
* Helper functions for filling in permission tables |
| 382 |
*/ |
| 383 |
static inline void p_setb(struct perm_bits *p, int off, u8 virt, u8 write) |
| 384 |
{ |
| 385 |
p->virt[off] = virt; |
| 386 |
p->write[off] = write; |
| 387 |
} |
| 388 |
|
| 389 |
/* Handle endian-ness - pci and tables are little-endian */ |
| 390 |
static inline void p_setw(struct perm_bits *p, int off, u16 virt, u16 write) |
| 391 |
{ |
| 392 |
*(__le16 *)(&p->virt[off]) = cpu_to_le16(virt); |
| 393 |
*(__le16 *)(&p->write[off]) = cpu_to_le16(write); |
| 394 |
} |
| 395 |
|
| 396 |
/* Handle endian-ness - pci and tables are little-endian */ |
| 397 |
static inline void p_setd(struct perm_bits *p, int off, u32 virt, u32 write) |
| 398 |
{ |
| 399 |
*(__le32 *)(&p->virt[off]) = cpu_to_le32(virt); |
| 400 |
*(__le32 *)(&p->write[off]) = cpu_to_le32(write); |
| 401 |
} |
| 402 |
|
| 403 |
/* Caller should hold memory_lock semaphore */ |
| 404 |
bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev) |
| 405 |
{ |
| 406 |
struct pci_dev *pdev = vdev->pdev; |
| 407 |
u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]); |
| 408 |
|
| 409 |
/* |
| 410 |
* Memory region cannot be accessed if device power state is D3. |
| 411 |
* |
| 412 |
* SR-IOV VF memory enable is handled by the MSE bit in the |
| 413 |
* PF SR-IOV capability, there's therefore no need to trigger |
| 414 |
* faults based on the virtual value. |
| 415 |
*/ |
| 416 |
return pdev->current_state < PCI_D3hot && |
| 417 |
(pdev->no_command_memory || (cmd & PCI_COMMAND_MEMORY)); |
| 418 |
} |
| 419 |
EXPORT_SYMBOL_GPL(__vfio_pci_memory_enabled); |
| 420 |
|
| 421 |
/* |
| 422 |
* Restore the *real* BARs after we detect a FLR or backdoor reset. |
| 423 |
* (backdoor = some device specific technique that we didn't catch) |
| 424 |
*/ |
| 425 |
static void vfio_bar_restore(struct vfio_pci_core_device *vdev) |
| 426 |
{ |
| 427 |
struct pci_dev *pdev = vdev->pdev; |
| 428 |
u32 *rbar = vdev->rbar; |
| 429 |
u16 cmd; |
| 430 |
int i; |
| 431 |
|
| 432 |
if (pdev->is_virtfn) |
| 433 |
return; |
| 434 |
|
| 435 |
pci_info(pdev, "%s: reset recovery - restoring BARs\n", __func__); |
| 436 |
|
| 437 |
for (i = PCI_BASE_ADDRESS_0; i <= PCI_BASE_ADDRESS_5; i += 4, rbar++) |
| 438 |
pci_user_write_config_dword(pdev, i, *rbar); |
| 439 |
|
| 440 |
pci_user_write_config_dword(pdev, PCI_ROM_ADDRESS, *rbar); |
| 441 |
|
| 442 |
if (vdev->nointx) { |
| 443 |
pci_user_read_config_word(pdev, PCI_COMMAND, &cmd); |
| 444 |
cmd |= PCI_COMMAND_INTX_DISABLE; |
| 445 |
pci_user_write_config_word(pdev, PCI_COMMAND, cmd); |
| 446 |
} |
| 447 |
} |
| 448 |
|
| 449 |
static __le32 vfio_generate_bar_flags(struct pci_dev *pdev, int bar) |
| 450 |
{ |
| 451 |
unsigned long flags = pci_resource_flags(pdev, bar); |
| 452 |
u32 val; |
| 453 |
|
| 454 |
if (flags & IORESOURCE_IO) |
| 455 |
return cpu_to_le32(PCI_BASE_ADDRESS_SPACE_IO); |
| 456 |
|
| 457 |
val = PCI_BASE_ADDRESS_SPACE_MEMORY; |
| 458 |
|
| 459 |
if (flags & IORESOURCE_PREFETCH) |
| 460 |
val |= PCI_BASE_ADDRESS_MEM_PREFETCH; |
| 461 |
|
| 462 |
if (flags & IORESOURCE_MEM_64) |
| 463 |
val |= PCI_BASE_ADDRESS_MEM_TYPE_64; |
| 464 |
|
| 465 |
return cpu_to_le32(val); |
| 466 |
} |
| 467 |
|
| 468 |
/* |
| 469 |
* Pretend we're hardware and tweak the values of the *virtual* PCI BARs |
| 470 |
* to reflect the hardware capabilities. This implements BAR sizing. |
| 471 |
*/ |
| 472 |
static void vfio_bar_fixup(struct vfio_pci_core_device *vdev) |
| 473 |
{ |
| 474 |
struct pci_dev *pdev = vdev->pdev; |
| 475 |
int i; |
| 476 |
__le32 *vbar; |
| 477 |
u64 mask; |
| 478 |
|
| 479 |
if (!vdev->bardirty) |
| 480 |
return; |
| 481 |
|
| 482 |
vbar = (__le32 *)&vdev->vconfig[PCI_BASE_ADDRESS_0]; |
| 483 |
|
| 484 |
for (i = 0; i < PCI_STD_NUM_BARS; i++, vbar++) { |
| 485 |
int bar = i + PCI_STD_RESOURCES; |
| 486 |
|
| 487 |
if (!pci_resource_start(pdev, bar)) { |
| 488 |
*vbar = 0; /* Unmapped by host = unimplemented to user */ |
| 489 |
continue; |
| 490 |
} |
| 491 |
|
| 492 |
mask = ~(pci_resource_len(pdev, bar) - 1); |
| 493 |
|
| 494 |
*vbar &= cpu_to_le32((u32)mask); |
| 495 |
*vbar |= vfio_generate_bar_flags(pdev, bar); |
| 496 |
|
| 497 |
if (*vbar & cpu_to_le32(PCI_BASE_ADDRESS_MEM_TYPE_64)) { |
| 498 |
vbar++; |
| 499 |
*vbar &= cpu_to_le32((u32)(mask >> 32)); |
| 500 |
i++; |
| 501 |
} |
| 502 |
} |
| 503 |
|
| 504 |
vbar = (__le32 *)&vdev->vconfig[PCI_ROM_ADDRESS]; |
| 505 |
|
| 506 |
/* |
| 507 |
* NB. REGION_INFO will have reported zero size if we weren't able |
| 508 |
* to read the ROM, but we still return the actual BAR size here if |
| 509 |
* it exists (or the shadow ROM space). |
| 510 |
*/ |
| 511 |
if (pci_resource_start(pdev, PCI_ROM_RESOURCE)) { |
| 512 |
mask = ~(pci_resource_len(pdev, PCI_ROM_RESOURCE) - 1); |
| 513 |
mask |= PCI_ROM_ADDRESS_ENABLE; |
| 514 |
*vbar &= cpu_to_le32((u32)mask); |
| 515 |
} else if (pdev->rom && pdev->romlen) { |
| 516 |
mask = ~(roundup_pow_of_two(pdev->romlen) - 1); |
| 517 |
mask |= PCI_ROM_ADDRESS_ENABLE; |
| 518 |
*vbar &= cpu_to_le32((u32)mask); |
| 519 |
} else { |
| 520 |
*vbar = 0; |
| 521 |
} |
| 522 |
|
| 523 |
vdev->bardirty = false; |
| 524 |
} |
| 525 |
|
| 526 |
static int vfio_basic_config_read(struct vfio_pci_core_device *vdev, int pos, |
| 527 |
int count, struct perm_bits *perm, |
| 528 |
int offset, __le32 *val) |
| 529 |
{ |
| 530 |
if (is_bar(offset)) /* pos == offset for basic config */ |
| 531 |
vfio_bar_fixup(vdev); |
| 532 |
|
| 533 |
count = vfio_default_config_read(vdev, pos, count, perm, offset, val); |
| 534 |
|
| 535 |
/* Mask in virtual memory enable */ |
| 536 |
if (offset == PCI_COMMAND && vdev->pdev->no_command_memory) { |
| 537 |
u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]); |
| 538 |
u32 tmp_val = le32_to_cpu(*val); |
| 539 |
|
| 540 |
tmp_val |= cmd & PCI_COMMAND_MEMORY; |
| 541 |
*val = cpu_to_le32(tmp_val); |
| 542 |
} |
| 543 |
|
| 544 |
return count; |
| 545 |
} |
| 546 |
|
| 547 |
/* Test whether BARs match the value we think they should contain */ |
| 548 |
static bool vfio_need_bar_restore(struct vfio_pci_core_device *vdev) |
| 549 |
{ |
| 550 |
int i = 0, pos = PCI_BASE_ADDRESS_0, ret; |
| 551 |
u32 bar; |
| 552 |
|
| 553 |
for (; pos <= PCI_BASE_ADDRESS_5; i++, pos += 4) { |
| 554 |
if (vdev->rbar[i]) { |
| 555 |
ret = pci_user_read_config_dword(vdev->pdev, pos, &bar); |
| 556 |
if (ret || vdev->rbar[i] != bar) |
| 557 |
return true; |
| 558 |
} |
| 559 |
} |
| 560 |
|
| 561 |
return false; |
| 562 |
} |
| 563 |
|
| 564 |
static int vfio_basic_config_write(struct vfio_pci_core_device *vdev, int pos, |
| 565 |
int count, struct perm_bits *perm, |
| 566 |
int offset, __le32 val) |
| 567 |
{ |
| 568 |
struct pci_dev *pdev = vdev->pdev; |
| 569 |
__le16 *virt_cmd; |
| 570 |
u16 new_cmd = 0; |
| 571 |
int ret; |
| 572 |
|
| 573 |
virt_cmd = (__le16 *)&vdev->vconfig[PCI_COMMAND]; |
| 574 |
|
| 575 |
if (offset == PCI_COMMAND) { |
| 576 |
bool phys_mem, virt_mem, new_mem, phys_io, virt_io, new_io; |
| 577 |
u16 phys_cmd; |
| 578 |
|
| 579 |
ret = pci_user_read_config_word(pdev, PCI_COMMAND, &phys_cmd); |
| 580 |
if (ret) |
| 581 |
return ret; |
| 582 |
|
| 583 |
new_cmd = le32_to_cpu(val); |
| 584 |
|
| 585 |
phys_io = !!(phys_cmd & PCI_COMMAND_IO); |
| 586 |
virt_io = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_IO); |
| 587 |
new_io = !!(new_cmd & PCI_COMMAND_IO); |
| 588 |
|
| 589 |
phys_mem = !!(phys_cmd & PCI_COMMAND_MEMORY); |
| 590 |
virt_mem = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_MEMORY); |
| 591 |
new_mem = !!(new_cmd & PCI_COMMAND_MEMORY); |
| 592 |
|
| 593 |
if (!new_mem) { |
| 594 |
vfio_pci_zap_and_down_write_memory_lock(vdev); |
| 595 |
vfio_pci_dma_buf_move(vdev, true); |
| 596 |
} else { |
| 597 |
down_write(&vdev->memory_lock); |
| 598 |
} |
| 599 |
|
| 600 |
/* |
| 601 |
* If the user is writing mem/io enable (new_mem/io) and we |
| 602 |
* think it's already enabled (virt_mem/io), but the hardware |
| 603 |
* shows it disabled (phys_mem/io, then the device has |
| 604 |
* undergone some kind of backdoor reset and needs to be |
| 605 |
* restored before we allow it to enable the bars. |
| 606 |
* SR-IOV devices will trigger this - for mem enable let's |
| 607 |
* catch this now and for io enable it will be caught later |
| 608 |
*/ |
| 609 |
if ((new_mem && virt_mem && !phys_mem && |
| 610 |
!pdev->no_command_memory) || |
| 611 |
(new_io && virt_io && !phys_io) || |
| 612 |
vfio_need_bar_restore(vdev)) |
| 613 |
vfio_bar_restore(vdev); |
| 614 |
} |
| 615 |
|
| 616 |
count = vfio_default_config_write(vdev, pos, count, perm, offset, val); |
| 617 |
if (count < 0) { |
| 618 |
if (offset == PCI_COMMAND) |
| 619 |
up_write(&vdev->memory_lock); |
| 620 |
return count; |
| 621 |
} |
| 622 |
|
| 623 |
/* |
| 624 |
* Save current memory/io enable bits in vconfig to allow for |
| 625 |
* the test above next time. |
| 626 |
*/ |
| 627 |
if (offset == PCI_COMMAND) { |
| 628 |
u16 mask = PCI_COMMAND_MEMORY | PCI_COMMAND_IO; |
| 629 |
|
| 630 |
*virt_cmd &= cpu_to_le16(~mask); |
| 631 |
*virt_cmd |= cpu_to_le16(new_cmd & mask); |
| 632 |
|
| 633 |
if (__vfio_pci_memory_enabled(vdev)) |
| 634 |
vfio_pci_dma_buf_move(vdev, false); |
| 635 |
up_write(&vdev->memory_lock); |
| 636 |
} |
| 637 |
|
| 638 |
/* Emulate INTx disable */ |
| 639 |
if (offset >= PCI_COMMAND && offset <= PCI_COMMAND + 1) { |
| 640 |
bool virt_intx_disable; |
| 641 |
|
| 642 |
virt_intx_disable = !!(le16_to_cpu(*virt_cmd) & |
| 643 |
PCI_COMMAND_INTX_DISABLE); |
| 644 |
|
| 645 |
if (virt_intx_disable && !vdev->virq_disabled) { |
| 646 |
vdev->virq_disabled = true; |
| 647 |
vfio_pci_intx_mask(vdev); |
| 648 |
} else if (!virt_intx_disable && vdev->virq_disabled) { |
| 649 |
vdev->virq_disabled = false; |
| 650 |
vfio_pci_intx_unmask(vdev); |
| 651 |
} |
| 652 |
} |
| 653 |
|
| 654 |
if (is_bar(offset)) |
| 655 |
vdev->bardirty = true; |
| 656 |
|
| 657 |
return count; |
| 658 |
} |
| 659 |
|
| 660 |
/* Permissions for the Basic PCI Header */ |
| 661 |
static int __init init_pci_cap_basic_perm(struct perm_bits *perm) |
| 662 |
{ |
| 663 |
if (alloc_perm_bits(perm, PCI_STD_HEADER_SIZEOF)) |
| 664 |
return -ENOMEM; |
| 665 |
|
| 666 |
perm->readfn = vfio_basic_config_read; |
| 667 |
perm->writefn = vfio_basic_config_write; |
| 668 |
|
| 669 |
/* Virtualized for SR-IOV functions, which just have FFFF */ |
| 670 |
p_setw(perm, PCI_VENDOR_ID, (u16)ALL_VIRT, NO_WRITE); |
| 671 |
p_setw(perm, PCI_DEVICE_ID, (u16)ALL_VIRT, NO_WRITE); |
| 672 |
|
| 673 |
/* |
| 674 |
* Virtualize INTx disable, we use it internally for interrupt |
| 675 |
* control and can emulate it for non-PCI 2.3 devices. |
| 676 |
*/ |
| 677 |
p_setw(perm, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE, (u16)ALL_WRITE); |
| 678 |
|
| 679 |
/* Virtualize capability list, we might want to skip/disable */ |
| 680 |
p_setw(perm, PCI_STATUS, PCI_STATUS_CAP_LIST, NO_WRITE); |
| 681 |
|
| 682 |
/* No harm to write */ |
| 683 |
p_setb(perm, PCI_CACHE_LINE_SIZE, NO_VIRT, (u8)ALL_WRITE); |
| 684 |
p_setb(perm, PCI_LATENCY_TIMER, NO_VIRT, (u8)ALL_WRITE); |
| 685 |
p_setb(perm, PCI_BIST, NO_VIRT, (u8)ALL_WRITE); |
| 686 |
|
| 687 |
/* Virtualize all bars, can't touch the real ones */ |
| 688 |
p_setd(perm, PCI_BASE_ADDRESS_0, ALL_VIRT, ALL_WRITE); |
| 689 |
p_setd(perm, PCI_BASE_ADDRESS_1, ALL_VIRT, ALL_WRITE); |
| 690 |
p_setd(perm, PCI_BASE_ADDRESS_2, ALL_VIRT, ALL_WRITE); |
| 691 |
p_setd(perm, PCI_BASE_ADDRESS_3, ALL_VIRT, ALL_WRITE); |
| 692 |
p_setd(perm, PCI_BASE_ADDRESS_4, ALL_VIRT, ALL_WRITE); |
| 693 |
p_setd(perm, PCI_BASE_ADDRESS_5, ALL_VIRT, ALL_WRITE); |
| 694 |
p_setd(perm, PCI_ROM_ADDRESS, ALL_VIRT, ALL_WRITE); |
| 695 |
|
| 696 |
/* Allow us to adjust capability chain */ |
| 697 |
p_setb(perm, PCI_CAPABILITY_LIST, (u8)ALL_VIRT, NO_WRITE); |
| 698 |
|
| 699 |
/* Sometimes used by sw, just virtualize */ |
| 700 |
p_setb(perm, PCI_INTERRUPT_LINE, (u8)ALL_VIRT, (u8)ALL_WRITE); |
| 701 |
|
| 702 |
/* Virtualize interrupt pin to allow hiding INTx */ |
| 703 |
p_setb(perm, PCI_INTERRUPT_PIN, (u8)ALL_VIRT, (u8)NO_WRITE); |
| 704 |
|
| 705 |
return 0; |
| 706 |
} |
| 707 |
|
| 708 |
/* |
| 709 |
* It takes all the required locks to protect the access of power related |
| 710 |
* variables and then invokes vfio_pci_set_power_state(). |
| 711 |
*/ |
| 712 |
static void vfio_lock_and_set_power_state(struct vfio_pci_core_device *vdev, |
| 713 |
pci_power_t state) |
| 714 |
{ |
| 715 |
if (state >= PCI_D3hot) { |
| 716 |
vfio_pci_zap_and_down_write_memory_lock(vdev); |
| 717 |
vfio_pci_dma_buf_move(vdev, true); |
| 718 |
} else { |
| 719 |
down_write(&vdev->memory_lock); |
| 720 |
} |
| 721 |
|
| 722 |
vfio_pci_set_power_state(vdev, state); |
| 723 |
if (__vfio_pci_memory_enabled(vdev)) |
| 724 |
vfio_pci_dma_buf_move(vdev, false); |
| 725 |
up_write(&vdev->memory_lock); |
| 726 |
} |
| 727 |
|
| 728 |
static int vfio_pm_config_write(struct vfio_pci_core_device *vdev, int pos, |
| 729 |
int count, struct perm_bits *perm, |
| 730 |
int offset, __le32 val) |
| 731 |
{ |
| 732 |
count = vfio_default_config_write(vdev, pos, count, perm, offset, val); |
| 733 |
if (count < 0) |
| 734 |
return count; |
| 735 |
|
| 736 |
if (offset == PCI_PM_CTRL) { |
| 737 |
pci_power_t state; |
| 738 |
|
| 739 |
switch (le32_to_cpu(val) & PCI_PM_CTRL_STATE_MASK) { |
| 740 |
case 0: |
| 741 |
state = PCI_D0; |
| 742 |
break; |
| 743 |
case 1: |
| 744 |
state = PCI_D1; |
| 745 |
break; |
| 746 |
case 2: |
| 747 |
state = PCI_D2; |
| 748 |
break; |
| 749 |
case 3: |
| 750 |
state = PCI_D3hot; |
| 751 |
break; |
| 752 |
} |
| 753 |
|
| 754 |
vfio_lock_and_set_power_state(vdev, state); |
| 755 |
} |
| 756 |
|
| 757 |
return count; |
| 758 |
} |
| 759 |
|
| 760 |
/* Permissions for the Power Management capability */ |
| 761 |
static int __init init_pci_cap_pm_perm(struct perm_bits *perm) |
| 762 |
{ |
| 763 |
if (alloc_perm_bits(perm, pci_cap_length[PCI_CAP_ID_PM])) |
| 764 |
return -ENOMEM; |
| 765 |
|
| 766 |
perm->writefn = vfio_pm_config_write; |
| 767 |
|
| 768 |
/* |
| 769 |
* We always virtualize the next field so we can remove |
| 770 |
* capabilities from the chain if we want to. |
| 771 |
*/ |
| 772 |
p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); |
| 773 |
|
| 774 |
/* |
| 775 |
* The guests can't process PME events. If any PME event will be |
| 776 |
* generated, then it will be mostly handled in the host and the |
| 777 |
* host will clear the PME_STATUS. So virtualize PME_Support bits. |
| 778 |
* The vconfig bits will be cleared during device capability |
| 779 |
* initialization. |
| 780 |
*/ |
| 781 |
p_setw(perm, PCI_PM_PMC, PCI_PM_CAP_PME_MASK, NO_WRITE); |
| 782 |
|
| 783 |
/* |
| 784 |
* Power management is defined *per function*, so we can let |
| 785 |
* the user change power state, but we trap and initiate the |
| 786 |
* change ourselves, so the state bits are read-only. |
| 787 |
* |
| 788 |
* The guest can't process PME from D3cold so virtualize PME_Status |
| 789 |
* and PME_En bits. The vconfig bits will be cleared during device |
| 790 |
* capability initialization. |
| 791 |
*/ |
| 792 |
p_setd(perm, PCI_PM_CTRL, |
| 793 |
PCI_PM_CTRL_PME_ENABLE | PCI_PM_CTRL_PME_STATUS, |
| 794 |
~(PCI_PM_CTRL_PME_ENABLE | PCI_PM_CTRL_PME_STATUS | |
| 795 |
PCI_PM_CTRL_STATE_MASK)); |
| 796 |
|
| 797 |
return 0; |
| 798 |
} |
| 799 |
|
| 800 |
static int vfio_vpd_config_write(struct vfio_pci_core_device *vdev, int pos, |
| 801 |
int count, struct perm_bits *perm, |
| 802 |
int offset, __le32 val) |
| 803 |
{ |
| 804 |
struct pci_dev *pdev = vdev->pdev; |
| 805 |
__le16 *paddr = (__le16 *)(vdev->vconfig + pos - offset + PCI_VPD_ADDR); |
| 806 |
__le32 *pdata = (__le32 *)(vdev->vconfig + pos - offset + PCI_VPD_DATA); |
| 807 |
u16 addr; |
| 808 |
u32 data; |
| 809 |
|
| 810 |
/* |
| 811 |
* Write through to emulation. If the write includes the upper byte |
| 812 |
* of PCI_VPD_ADDR, then the PCI_VPD_ADDR_F bit is written and we |
| 813 |
* have work to do. |
| 814 |
*/ |
| 815 |
count = vfio_default_config_write(vdev, pos, count, perm, offset, val); |
| 816 |
if (count < 0 || offset > PCI_VPD_ADDR + 1 || |
| 817 |
offset + count <= PCI_VPD_ADDR + 1) |
| 818 |
return count; |
| 819 |
|
| 820 |
addr = le16_to_cpu(*paddr); |
| 821 |
|
| 822 |
if (addr & PCI_VPD_ADDR_F) { |
| 823 |
data = le32_to_cpu(*pdata); |
| 824 |
if (pci_write_vpd(pdev, addr & ~PCI_VPD_ADDR_F, 4, &data) != 4) |
| 825 |
return count; |
| 826 |
} else { |
| 827 |
data = 0; |
| 828 |
if (pci_read_vpd(pdev, addr, 4, &data) < 0) |
| 829 |
return count; |
| 830 |
*pdata = cpu_to_le32(data); |
| 831 |
} |
| 832 |
|
| 833 |
/* |
| 834 |
* Toggle PCI_VPD_ADDR_F in the emulated PCI_VPD_ADDR register to |
| 835 |
* signal completion. If an error occurs above, we assume that not |
| 836 |
* toggling this bit will induce a driver timeout. |
| 837 |
*/ |
| 838 |
addr ^= PCI_VPD_ADDR_F; |
| 839 |
*paddr = cpu_to_le16(addr); |
| 840 |
|
| 841 |
return count; |
| 842 |
} |
| 843 |
|
| 844 |
/* Permissions for Vital Product Data capability */ |
| 845 |
static int __init init_pci_cap_vpd_perm(struct perm_bits *perm) |
| 846 |
{ |
| 847 |
if (alloc_perm_bits(perm, pci_cap_length[PCI_CAP_ID_VPD])) |
| 848 |
return -ENOMEM; |
| 849 |
|
| 850 |
perm->writefn = vfio_vpd_config_write; |
| 851 |
|
| 852 |
/* |
| 853 |
* We always virtualize the next field so we can remove |
| 854 |
* capabilities from the chain if we want to. |
| 855 |
*/ |
| 856 |
p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); |
| 857 |
|
| 858 |
/* |
| 859 |
* Both the address and data registers are virtualized to |
| 860 |
* enable access through the pci_vpd_read/write functions |
| 861 |
*/ |
| 862 |
p_setw(perm, PCI_VPD_ADDR, (u16)ALL_VIRT, (u16)ALL_WRITE); |
| 863 |
p_setd(perm, PCI_VPD_DATA, ALL_VIRT, ALL_WRITE); |
| 864 |
|
| 865 |
return 0; |
| 866 |
} |
| 867 |
|
| 868 |
/* Permissions for PCI-X capability */ |
| 869 |
static int __init init_pci_cap_pcix_perm(struct perm_bits *perm) |
| 870 |
{ |
| 871 |
/* Alloc 24, but only 8 are used in v0 */ |
| 872 |
if (alloc_perm_bits(perm, PCI_CAP_PCIX_SIZEOF_V2)) |
| 873 |
return -ENOMEM; |
| 874 |
|
| 875 |
p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); |
| 876 |
|
| 877 |
p_setw(perm, PCI_X_CMD, NO_VIRT, (u16)ALL_WRITE); |
| 878 |
p_setd(perm, PCI_X_ECC_CSR, NO_VIRT, ALL_WRITE); |
| 879 |
return 0; |
| 880 |
} |
| 881 |
|
| 882 |
static int vfio_exp_config_write(struct vfio_pci_core_device *vdev, int pos, |
| 883 |
int count, struct perm_bits *perm, |
| 884 |
int offset, __le32 val) |
| 885 |
{ |
| 886 |
__le16 *ctrl = (__le16 *)(vdev->vconfig + pos - |
| 887 |
offset + PCI_EXP_DEVCTL); |
| 888 |
int readrq = le16_to_cpu(*ctrl) & PCI_EXP_DEVCTL_READRQ; |
| 889 |
|
| 890 |
count = vfio_default_config_write(vdev, pos, count, perm, offset, val); |
| 891 |
if (count < 0) |
| 892 |
return count; |
| 893 |
|
| 894 |
/* |
| 895 |
* The FLR bit is virtualized, if set and the device supports PCIe |
| 896 |
* FLR, issue a reset_function. Regardless, clear the bit, the spec |
| 897 |
* requires it to be always read as zero. NB, reset_function might |
| 898 |
* not use a PCIe FLR, we don't have that level of granularity. |
| 899 |
*/ |
| 900 |
if (*ctrl & cpu_to_le16(PCI_EXP_DEVCTL_BCR_FLR)) { |
| 901 |
u32 cap; |
| 902 |
int ret; |
| 903 |
|
| 904 |
*ctrl &= ~cpu_to_le16(PCI_EXP_DEVCTL_BCR_FLR); |
| 905 |
|
| 906 |
ret = pci_user_read_config_dword(vdev->pdev, |
| 907 |
pos - offset + PCI_EXP_DEVCAP, |
| 908 |
&cap); |
| 909 |
|
| 910 |
if (!ret && (cap & PCI_EXP_DEVCAP_FLR)) { |
| 911 |
vfio_pci_zap_and_down_write_memory_lock(vdev); |
| 912 |
vfio_pci_dma_buf_move(vdev, true); |
| 913 |
pci_try_reset_function(vdev->pdev); |
| 914 |
if (__vfio_pci_memory_enabled(vdev)) |
| 915 |
vfio_pci_dma_buf_move(vdev, false); |
| 916 |
up_write(&vdev->memory_lock); |
| 917 |
} |
| 918 |
} |
| 919 |
|
| 920 |
/* |
| 921 |
* MPS is virtualized to the user, writes do not change the physical |
| 922 |
* register since determining a proper MPS value requires a system wide |
| 923 |
* device view. The MRRS is largely independent of MPS, but since the |
| 924 |
* user does not have that system-wide view, they might set a safe, but |
| 925 |
* inefficiently low value. Here we allow writes through to hardware, |
| 926 |
* but we set the floor to the physical device MPS setting, so that |
| 927 |
* we can at least use full TLPs, as defined by the MPS value. |
| 928 |
* |
| 929 |
* NB, if any devices actually depend on an artificially low MRRS |
| 930 |
* setting, this will need to be revisited, perhaps with a quirk |
| 931 |
* though pcie_set_readrq(). |
| 932 |
*/ |
| 933 |
if (readrq != (le16_to_cpu(*ctrl) & PCI_EXP_DEVCTL_READRQ)) { |
| 934 |
readrq = 128 << |
| 935 |
((le16_to_cpu(*ctrl) & PCI_EXP_DEVCTL_READRQ) >> 12); |
| 936 |
readrq = max(readrq, pcie_get_mps(vdev->pdev)); |
| 937 |
|
| 938 |
pcie_set_readrq(vdev->pdev, readrq); |
| 939 |
} |
| 940 |
|
| 941 |
return count; |
| 942 |
} |
| 943 |
|
| 944 |
/* Permissions for PCI Express capability */ |
| 945 |
static int __init init_pci_cap_exp_perm(struct perm_bits *perm) |
| 946 |
{ |
| 947 |
/* Alloc largest of possible sizes */ |
| 948 |
if (alloc_perm_bits(perm, PCI_CAP_EXP_ENDPOINT_SIZEOF_V2)) |
| 949 |
return -ENOMEM; |
| 950 |
|
| 951 |
perm->writefn = vfio_exp_config_write; |
| 952 |
|
| 953 |
p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); |
| 954 |
|
| 955 |
/* |
| 956 |
* Allow writes to device control fields, except devctl_phantom, |
| 957 |
* which could confuse IOMMU, MPS, which can break communication |
| 958 |
* with other physical devices, and the ARI bit in devctl2, which |
| 959 |
* is set at probe time. FLR and MRRS get virtualized via our |
| 960 |
* writefn. |
| 961 |
*/ |
| 962 |
p_setw(perm, PCI_EXP_DEVCTL, |
| 963 |
PCI_EXP_DEVCTL_BCR_FLR | PCI_EXP_DEVCTL_PAYLOAD | |
| 964 |
PCI_EXP_DEVCTL_READRQ, ~PCI_EXP_DEVCTL_PHANTOM); |
| 965 |
p_setw(perm, PCI_EXP_DEVCTL2, NO_VIRT, ~PCI_EXP_DEVCTL2_ARI); |
| 966 |
return 0; |
| 967 |
} |
| 968 |
|
| 969 |
static int vfio_af_config_write(struct vfio_pci_core_device *vdev, int pos, |
| 970 |
int count, struct perm_bits *perm, |
| 971 |
int offset, __le32 val) |
| 972 |
{ |
| 973 |
u8 *ctrl = vdev->vconfig + pos - offset + PCI_AF_CTRL; |
| 974 |
|
| 975 |
count = vfio_default_config_write(vdev, pos, count, perm, offset, val); |
| 976 |
if (count < 0) |
| 977 |
return count; |
| 978 |
|
| 979 |
/* |
| 980 |
* The FLR bit is virtualized, if set and the device supports AF |
| 981 |
* FLR, issue a reset_function. Regardless, clear the bit, the spec |
| 982 |
* requires it to be always read as zero. NB, reset_function might |
| 983 |
* not use an AF FLR, we don't have that level of granularity. |
| 984 |
*/ |
| 985 |
if (*ctrl & PCI_AF_CTRL_FLR) { |
| 986 |
u8 cap; |
| 987 |
int ret; |
| 988 |
|
| 989 |
*ctrl &= ~PCI_AF_CTRL_FLR; |
| 990 |
|
| 991 |
ret = pci_user_read_config_byte(vdev->pdev, |
| 992 |
pos - offset + PCI_AF_CAP, |
| 993 |
&cap); |
| 994 |
|
| 995 |
if (!ret && (cap & PCI_AF_CAP_FLR) && (cap & PCI_AF_CAP_TP)) { |
| 996 |
vfio_pci_zap_and_down_write_memory_lock(vdev); |
| 997 |
vfio_pci_dma_buf_move(vdev, true); |
| 998 |
pci_try_reset_function(vdev->pdev); |
| 999 |
if (__vfio_pci_memory_enabled(vdev)) |
| 1000 |
vfio_pci_dma_buf_move(vdev, false); |
| 1001 |
up_write(&vdev->memory_lock); |
| 1002 |
} |
| 1003 |
} |
| 1004 |
|
| 1005 |
return count; |
| 1006 |
} |
| 1007 |
|
| 1008 |
/* Permissions for Advanced Function capability */ |
| 1009 |
static int __init init_pci_cap_af_perm(struct perm_bits *perm) |
| 1010 |
{ |
| 1011 |
if (alloc_perm_bits(perm, pci_cap_length[PCI_CAP_ID_AF])) |
| 1012 |
return -ENOMEM; |
| 1013 |
|
| 1014 |
perm->writefn = vfio_af_config_write; |
| 1015 |
|
| 1016 |
p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); |
| 1017 |
p_setb(perm, PCI_AF_CTRL, PCI_AF_CTRL_FLR, PCI_AF_CTRL_FLR); |
| 1018 |
return 0; |
| 1019 |
} |
| 1020 |
|
| 1021 |
/* Permissions for Advanced Error Reporting extended capability */ |
| 1022 |
static int __init init_pci_ext_cap_err_perm(struct perm_bits *perm) |
| 1023 |
{ |
| 1024 |
u32 mask; |
| 1025 |
|
| 1026 |
if (alloc_perm_bits(perm, pci_ext_cap_length[PCI_EXT_CAP_ID_ERR])) |
| 1027 |
return -ENOMEM; |
| 1028 |
|
| 1029 |
/* |
| 1030 |
* Virtualize the first dword of all express capabilities |
| 1031 |
* because it includes the next pointer. This lets us later |
| 1032 |
* remove capabilities from the chain if we need to. |
| 1033 |
*/ |
| 1034 |
p_setd(perm, 0, ALL_VIRT, NO_WRITE); |
| 1035 |
|
| 1036 |
/* Writable bits mask */ |
| 1037 |
mask = PCI_ERR_UNC_UND | /* Undefined */ |
| 1038 |
PCI_ERR_UNC_DLP | /* Data Link Protocol */ |
| 1039 |
PCI_ERR_UNC_SURPDN | /* Surprise Down */ |
| 1040 |
PCI_ERR_UNC_POISON_TLP | /* Poisoned TLP */ |
| 1041 |
PCI_ERR_UNC_FCP | /* Flow Control Protocol */ |
| 1042 |
PCI_ERR_UNC_COMP_TIME | /* Completion Timeout */ |
| 1043 |
PCI_ERR_UNC_COMP_ABORT | /* Completer Abort */ |
| 1044 |
PCI_ERR_UNC_UNX_COMP | /* Unexpected Completion */ |
| 1045 |
PCI_ERR_UNC_RX_OVER | /* Receiver Overflow */ |
| 1046 |
PCI_ERR_UNC_MALF_TLP | /* Malformed TLP */ |
| 1047 |
PCI_ERR_UNC_ECRC | /* ECRC Error Status */ |
| 1048 |
PCI_ERR_UNC_UNSUP | /* Unsupported Request */ |
| 1049 |
PCI_ERR_UNC_ACSV | /* ACS Violation */ |
| 1050 |
PCI_ERR_UNC_INTN | /* internal error */ |
| 1051 |
PCI_ERR_UNC_MCBTLP | /* MC blocked TLP */ |
| 1052 |
PCI_ERR_UNC_ATOMEG | /* Atomic egress blocked */ |
| 1053 |
PCI_ERR_UNC_TLPPRE; /* TLP prefix blocked */ |
| 1054 |
p_setd(perm, PCI_ERR_UNCOR_STATUS, NO_VIRT, mask); |
| 1055 |
p_setd(perm, PCI_ERR_UNCOR_MASK, NO_VIRT, mask); |
| 1056 |
p_setd(perm, PCI_ERR_UNCOR_SEVER, NO_VIRT, mask); |
| 1057 |
|
| 1058 |
mask = PCI_ERR_COR_RCVR | /* Receiver Error Status */ |
| 1059 |
PCI_ERR_COR_BAD_TLP | /* Bad TLP Status */ |
| 1060 |
PCI_ERR_COR_BAD_DLLP | /* Bad DLLP Status */ |
| 1061 |
PCI_ERR_COR_REP_ROLL | /* REPLAY_NUM Rollover */ |
| 1062 |
PCI_ERR_COR_REP_TIMER | /* Replay Timer Timeout */ |
| 1063 |
PCI_ERR_COR_ADV_NFAT | /* Advisory Non-Fatal */ |
| 1064 |
PCI_ERR_COR_INTERNAL | /* Corrected Internal */ |
| 1065 |
PCI_ERR_COR_LOG_OVER; /* Header Log Overflow */ |
| 1066 |
p_setd(perm, PCI_ERR_COR_STATUS, NO_VIRT, mask); |
| 1067 |
p_setd(perm, PCI_ERR_COR_MASK, NO_VIRT, mask); |
| 1068 |
|
| 1069 |
mask = PCI_ERR_CAP_ECRC_GENE | /* ECRC Generation Enable */ |
| 1070 |
PCI_ERR_CAP_ECRC_CHKE; /* ECRC Check Enable */ |
| 1071 |
p_setd(perm, PCI_ERR_CAP, NO_VIRT, mask); |
| 1072 |
return 0; |
| 1073 |
} |
| 1074 |
|
| 1075 |
/* Permissions for Power Budgeting extended capability */ |
| 1076 |
static int __init init_pci_ext_cap_pwr_perm(struct perm_bits *perm) |
| 1077 |
{ |
| 1078 |
if (alloc_perm_bits(perm, pci_ext_cap_length[PCI_EXT_CAP_ID_PWR])) |
| 1079 |
return -ENOMEM; |
| 1080 |
|
| 1081 |
p_setd(perm, 0, ALL_VIRT, NO_WRITE); |
| 1082 |
|
| 1083 |
/* Writing the data selector is OK, the info is still read-only */ |
| 1084 |
p_setb(perm, PCI_PWR_DATA, NO_VIRT, (u8)ALL_WRITE); |
| 1085 |
return 0; |
| 1086 |
} |
| 1087 |
|
| 1088 |
/* |
| 1089 |
* Initialize the shared permission tables |
| 1090 |
*/ |
| 1091 |
void vfio_pci_uninit_perm_bits(void) |
| 1092 |
{ |
| 1093 |
free_perm_bits(&cap_perms[PCI_CAP_ID_BASIC]); |
| 1094 |
|
| 1095 |
free_perm_bits(&cap_perms[PCI_CAP_ID_PM]); |
| 1096 |
free_perm_bits(&cap_perms[PCI_CAP_ID_VPD]); |
| 1097 |
free_perm_bits(&cap_perms[PCI_CAP_ID_PCIX]); |
| 1098 |
free_perm_bits(&cap_perms[PCI_CAP_ID_EXP]); |
| 1099 |
free_perm_bits(&cap_perms[PCI_CAP_ID_AF]); |
| 1100 |
|
| 1101 |
free_perm_bits(&ecap_perms[PCI_EXT_CAP_ID_ERR]); |
| 1102 |
free_perm_bits(&ecap_perms[PCI_EXT_CAP_ID_PWR]); |
| 1103 |
} |
| 1104 |
|
| 1105 |
int __init vfio_pci_init_perm_bits(void) |
| 1106 |
{ |
| 1107 |
int ret; |
| 1108 |
|
| 1109 |
/* Basic config space */ |
| 1110 |
ret = init_pci_cap_basic_perm(&cap_perms[PCI_CAP_ID_BASIC]); |
| 1111 |
|
| 1112 |
/* Capabilities */ |
| 1113 |
ret |= init_pci_cap_pm_perm(&cap_perms[PCI_CAP_ID_PM]); |
| 1114 |
ret |= init_pci_cap_vpd_perm(&cap_perms[PCI_CAP_ID_VPD]); |
| 1115 |
ret |= init_pci_cap_pcix_perm(&cap_perms[PCI_CAP_ID_PCIX]); |
| 1116 |
cap_perms[PCI_CAP_ID_VNDR].writefn = vfio_raw_config_write; |
| 1117 |
ret |= init_pci_cap_exp_perm(&cap_perms[PCI_CAP_ID_EXP]); |
| 1118 |
ret |= init_pci_cap_af_perm(&cap_perms[PCI_CAP_ID_AF]); |
| 1119 |
|
| 1120 |
/* Extended capabilities */ |
| 1121 |
ret |= init_pci_ext_cap_err_perm(&ecap_perms[PCI_EXT_CAP_ID_ERR]); |
| 1122 |
ret |= init_pci_ext_cap_pwr_perm(&ecap_perms[PCI_EXT_CAP_ID_PWR]); |
| 1123 |
ecap_perms[PCI_EXT_CAP_ID_VNDR].writefn = vfio_raw_config_write; |
| 1124 |
ecap_perms[PCI_EXT_CAP_ID_DVSEC].writefn = vfio_raw_config_write; |
| 1125 |
|
| 1126 |
if (ret) |
| 1127 |
vfio_pci_uninit_perm_bits(); |
| 1128 |
|
| 1129 |
return ret; |
| 1130 |
} |
| 1131 |
|
| 1132 |
static int vfio_find_cap_start(struct vfio_pci_core_device *vdev, int pos) |
| 1133 |
{ |
| 1134 |
u8 cap; |
| 1135 |
int base = (pos >= PCI_CFG_SPACE_SIZE) ? PCI_CFG_SPACE_SIZE : |
| 1136 |
PCI_STD_HEADER_SIZEOF; |
| 1137 |
cap = vdev->pci_config_map[pos]; |
| 1138 |
|
| 1139 |
if (cap == PCI_CAP_ID_BASIC) |
| 1140 |
return 0; |
| 1141 |
|
| 1142 |
/* XXX Can we have to abutting capabilities of the same type? */ |
| 1143 |
while (pos - 1 >= base && vdev->pci_config_map[pos - 1] == cap) |
| 1144 |
pos--; |
| 1145 |
|
| 1146 |
return pos; |
| 1147 |
} |
| 1148 |
|
| 1149 |
static int vfio_msi_config_read(struct vfio_pci_core_device *vdev, int pos, |
| 1150 |
int count, struct perm_bits *perm, |
| 1151 |
int offset, __le32 *val) |
| 1152 |
{ |
| 1153 |
/* Update max available queue size from msi_qmax */ |
| 1154 |
if (offset <= PCI_MSI_FLAGS && offset + count >= PCI_MSI_FLAGS) { |
| 1155 |
__le16 *flags; |
| 1156 |
int start; |
| 1157 |
|
| 1158 |
start = vfio_find_cap_start(vdev, pos); |
| 1159 |
|
| 1160 |
flags = (__le16 *)&vdev->vconfig[start]; |
| 1161 |
|
| 1162 |
*flags &= cpu_to_le16(~PCI_MSI_FLAGS_QMASK); |
| 1163 |
*flags |= cpu_to_le16(vdev->msi_qmax << 1); |
| 1164 |
} |
| 1165 |
|
| 1166 |
return vfio_default_config_read(vdev, pos, count, perm, offset, val); |
| 1167 |
} |
| 1168 |
|
| 1169 |
static int vfio_msi_config_write(struct vfio_pci_core_device *vdev, int pos, |
| 1170 |
int count, struct perm_bits *perm, |
| 1171 |
int offset, __le32 val) |
| 1172 |
{ |
| 1173 |
count = vfio_default_config_write(vdev, pos, count, perm, offset, val); |
| 1174 |
if (count < 0) |
| 1175 |
return count; |
| 1176 |
|
| 1177 |
/* Fixup and write configured queue size and enable to hardware */ |
| 1178 |
if (offset <= PCI_MSI_FLAGS && offset + count >= PCI_MSI_FLAGS) { |
| 1179 |
__le16 *pflags; |
| 1180 |
u16 flags; |
| 1181 |
int start, ret; |
| 1182 |
|
| 1183 |
start = vfio_find_cap_start(vdev, pos); |
| 1184 |
|
| 1185 |
pflags = (__le16 *)&vdev->vconfig[start + PCI_MSI_FLAGS]; |
| 1186 |
|
| 1187 |
flags = le16_to_cpu(*pflags); |
| 1188 |
|
| 1189 |
/* MSI is enabled via ioctl */ |
| 1190 |
if (vdev->irq_type != VFIO_PCI_MSI_IRQ_INDEX) |
| 1191 |
flags &= ~PCI_MSI_FLAGS_ENABLE; |
| 1192 |
|
| 1193 |
/* Check queue size */ |
| 1194 |
if ((flags & PCI_MSI_FLAGS_QSIZE) >> 4 > vdev->msi_qmax) { |
| 1195 |
flags &= ~PCI_MSI_FLAGS_QSIZE; |
| 1196 |
flags |= vdev->msi_qmax << 4; |
| 1197 |
} |
| 1198 |
|
| 1199 |
/* Write back to virt and to hardware */ |
| 1200 |
*pflags = cpu_to_le16(flags); |
| 1201 |
ret = pci_user_write_config_word(vdev->pdev, |
| 1202 |
start + PCI_MSI_FLAGS, |
| 1203 |
flags); |
| 1204 |
if (ret) |
| 1205 |
return ret; |
| 1206 |
} |
| 1207 |
|
| 1208 |
return count; |
| 1209 |
} |
| 1210 |
|
| 1211 |
/* |
| 1212 |
* MSI determination is per-device, so this routine gets used beyond |
| 1213 |
* initialization time. Don't add __init |
| 1214 |
*/ |
| 1215 |
static int init_pci_cap_msi_perm(struct perm_bits *perm, int len, u16 flags) |
| 1216 |
{ |
| 1217 |
if (alloc_perm_bits(perm, len)) |
| 1218 |
return -ENOMEM; |
| 1219 |
|
| 1220 |
perm->readfn = vfio_msi_config_read; |
| 1221 |
perm->writefn = vfio_msi_config_write; |
| 1222 |
|
| 1223 |
p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE); |
| 1224 |
|
| 1225 |
/* |
| 1226 |
* The upper byte of the control register is reserved, |
| 1227 |
* just setup the lower byte. |
| 1228 |
*/ |
| 1229 |
p_setb(perm, PCI_MSI_FLAGS, (u8)ALL_VIRT, (u8)ALL_WRITE); |
| 1230 |
p_setd(perm, PCI_MSI_ADDRESS_LO, ALL_VIRT, ALL_WRITE); |
| 1231 |
if (flags & PCI_MSI_FLAGS_64BIT) { |
| 1232 |
p_setd(perm, PCI_MSI_ADDRESS_HI, ALL_VIRT, ALL_WRITE); |
| 1233 |
p_setw(perm, PCI_MSI_DATA_64, (u16)ALL_VIRT, (u16)ALL_WRITE); |
| 1234 |
if (flags & PCI_MSI_FLAGS_MASKBIT) { |
| 1235 |
p_setd(perm, PCI_MSI_MASK_64, NO_VIRT, ALL_WRITE); |
| 1236 |
p_setd(perm, PCI_MSI_PENDING_64, NO_VIRT, ALL_WRITE); |
| 1237 |
} |
| 1238 |
} else { |
| 1239 |
p_setw(perm, PCI_MSI_DATA_32, (u16)ALL_VIRT, (u16)ALL_WRITE); |
| 1240 |
if (flags & PCI_MSI_FLAGS_MASKBIT) { |
| 1241 |
p_setd(perm, PCI_MSI_MASK_32, NO_VIRT, ALL_WRITE); |
| 1242 |
p_setd(perm, PCI_MSI_PENDING_32, NO_VIRT, ALL_WRITE); |
| 1243 |
} |
| 1244 |
} |
| 1245 |
return 0; |
| 1246 |
} |
| 1247 |
|
| 1248 |
/* Determine MSI CAP field length; initialize msi_perms on 1st call per vdev */ |
| 1249 |
static int vfio_msi_cap_len(struct vfio_pci_core_device *vdev, u8 pos) |
| 1250 |
{ |
| 1251 |
struct pci_dev *pdev = vdev->pdev; |
| 1252 |
int len, ret; |
| 1253 |
u16 flags; |
| 1254 |
|
| 1255 |
ret = pci_read_config_word(pdev, pos + PCI_MSI_FLAGS, &flags); |
| 1256 |
if (ret) |
| 1257 |
return pcibios_err_to_errno(ret); |
| 1258 |
|
| 1259 |
len = 10; /* Minimum size */ |
| 1260 |
if (flags & PCI_MSI_FLAGS_64BIT) |
| 1261 |
len += 4; |
| 1262 |
if (flags & PCI_MSI_FLAGS_MASKBIT) |
| 1263 |
len += 10; |
| 1264 |
|
| 1265 |
if (vdev->msi_perm) |
| 1266 |
return len; |
| 1267 |
|
| 1268 |
vdev->msi_perm = kmalloc_obj(struct perm_bits, GFP_KERNEL_ACCOUNT); |
| 1269 |
if (!vdev->msi_perm) |
| 1270 |
return -ENOMEM; |
| 1271 |
|
| 1272 |
ret = init_pci_cap_msi_perm(vdev->msi_perm, len, flags); |
| 1273 |
if (ret) { |
| 1274 |
kfree(vdev->msi_perm); |
| 1275 |
return ret; |
| 1276 |
} |
| 1277 |
|
| 1278 |
return len; |
| 1279 |
} |
| 1280 |
|
| 1281 |
/* Determine extended capability length for VC (2 & 9) and MFVC */ |
| 1282 |
static int vfio_vc_cap_len(struct vfio_pci_core_device *vdev, u16 pos) |
| 1283 |
{ |
| 1284 |
struct pci_dev *pdev = vdev->pdev; |
| 1285 |
u32 tmp; |
| 1286 |
int ret, evcc, phases, vc_arb; |
| 1287 |
int len = PCI_CAP_VC_BASE_SIZEOF; |
| 1288 |
|
| 1289 |
ret = pci_read_config_dword(pdev, pos + PCI_VC_PORT_CAP1, &tmp); |
| 1290 |
if (ret) |
| 1291 |
return pcibios_err_to_errno(ret); |
| 1292 |
|
| 1293 |
evcc = tmp & PCI_VC_CAP1_EVCC; /* extended vc count */ |
| 1294 |
ret = pci_read_config_dword(pdev, pos + PCI_VC_PORT_CAP2, &tmp); |
| 1295 |
if (ret) |
| 1296 |
return pcibios_err_to_errno(ret); |
| 1297 |
|
| 1298 |
if (tmp & PCI_VC_CAP2_128_PHASE) |
| 1299 |
phases = 128; |
| 1300 |
else if (tmp & PCI_VC_CAP2_64_PHASE) |
| 1301 |
phases = 64; |
| 1302 |
else if (tmp & PCI_VC_CAP2_32_PHASE) |
| 1303 |
phases = 32; |
| 1304 |
else |
| 1305 |
phases = 0; |
| 1306 |
|
| 1307 |
vc_arb = phases * 4; |
| 1308 |
|
| 1309 |
/* |
| 1310 |
* Port arbitration tables are root & switch only; |
| 1311 |
* function arbitration tables are function 0 only. |
| 1312 |
* In either case, we'll never let user write them so |
| 1313 |
* we don't care how big they are |
| 1314 |
*/ |
| 1315 |
len += (1 + evcc) * PCI_CAP_VC_PER_VC_SIZEOF; |
| 1316 |
if (vc_arb) { |
| 1317 |
len = round_up(len, 16); |
| 1318 |
len += vc_arb / 8; |
| 1319 |
} |
| 1320 |
return len; |
| 1321 |
} |
| 1322 |
|
| 1323 |
static int vfio_cap_len(struct vfio_pci_core_device *vdev, u8 cap, u8 pos) |
| 1324 |
{ |
| 1325 |
struct pci_dev *pdev = vdev->pdev; |
| 1326 |
u32 dword; |
| 1327 |
u16 word; |
| 1328 |
u8 byte; |
| 1329 |
int ret; |
| 1330 |
|
| 1331 |
switch (cap) { |
| 1332 |
case PCI_CAP_ID_MSI: |
| 1333 |
return vfio_msi_cap_len(vdev, pos); |
| 1334 |
case PCI_CAP_ID_PCIX: |
| 1335 |
ret = pci_read_config_word(pdev, pos + PCI_X_CMD, &word); |
| 1336 |
if (ret) |
| 1337 |
return pcibios_err_to_errno(ret); |
| 1338 |
|
| 1339 |
if (PCI_X_CMD_VERSION(word)) { |
| 1340 |
if (pdev->cfg_size > PCI_CFG_SPACE_SIZE) { |
| 1341 |
/* Test for extended capabilities */ |
| 1342 |
pci_read_config_dword(pdev, PCI_CFG_SPACE_SIZE, |
| 1343 |
&dword); |
| 1344 |
vdev->extended_caps = (dword != 0); |
| 1345 |
} |
| 1346 |
return PCI_CAP_PCIX_SIZEOF_V2; |
| 1347 |
} else |
| 1348 |
return PCI_CAP_PCIX_SIZEOF_V0; |
| 1349 |
case PCI_CAP_ID_VNDR: |
| 1350 |
/* length follows next field */ |
| 1351 |
ret = pci_read_config_byte(pdev, pos + PCI_CAP_FLAGS, &byte); |
| 1352 |
if (ret) |
| 1353 |
return pcibios_err_to_errno(ret); |
| 1354 |
|
| 1355 |
return byte; |
| 1356 |
case PCI_CAP_ID_EXP: |
| 1357 |
if (pdev->cfg_size > PCI_CFG_SPACE_SIZE) { |
| 1358 |
/* Test for extended capabilities */ |
| 1359 |
pci_read_config_dword(pdev, PCI_CFG_SPACE_SIZE, &dword); |
| 1360 |
vdev->extended_caps = (dword != 0); |
| 1361 |
} |
| 1362 |
|
| 1363 |
/* length based on version and type */ |
| 1364 |
if ((pcie_caps_reg(pdev) & PCI_EXP_FLAGS_VERS) == 1) { |
| 1365 |
if (pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END) |
| 1366 |
return 0xc; /* "All Devices" only, no link */ |
| 1367 |
return PCI_CAP_EXP_ENDPOINT_SIZEOF_V1; |
| 1368 |
} else { |
| 1369 |
if (pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END) |
| 1370 |
return 0x2c; /* No link */ |
| 1371 |
return PCI_CAP_EXP_ENDPOINT_SIZEOF_V2; |
| 1372 |
} |
| 1373 |
case PCI_CAP_ID_HT: |
| 1374 |
ret = pci_read_config_byte(pdev, pos + 3, &byte); |
| 1375 |
if (ret) |
| 1376 |
return pcibios_err_to_errno(ret); |
| 1377 |
|
| 1378 |
return (byte & HT_3BIT_CAP_MASK) ? |
| 1379 |
HT_CAP_SIZEOF_SHORT : HT_CAP_SIZEOF_LONG; |
| 1380 |
case PCI_CAP_ID_SATA: |
| 1381 |
ret = pci_read_config_byte(pdev, pos + PCI_SATA_REGS, &byte); |
| 1382 |
if (ret) |
| 1383 |
return pcibios_err_to_errno(ret); |
| 1384 |
|
| 1385 |
byte &= PCI_SATA_REGS_MASK; |
| 1386 |
if (byte == PCI_SATA_REGS_INLINE) |
| 1387 |
return PCI_SATA_SIZEOF_LONG; |
| 1388 |
else |
| 1389 |
return PCI_SATA_SIZEOF_SHORT; |
| 1390 |
default: |
| 1391 |
pci_warn(pdev, "%s: unknown length for PCI cap %#x@%#x\n", |
| 1392 |
__func__, cap, pos); |
| 1393 |
} |
| 1394 |
|
| 1395 |
return 0; |
| 1396 |
} |
| 1397 |
|
| 1398 |
static int vfio_ext_cap_len(struct vfio_pci_core_device *vdev, u16 ecap, u16 epos) |
| 1399 |
{ |
| 1400 |
struct pci_dev *pdev = vdev->pdev; |
| 1401 |
u8 byte; |
| 1402 |
u32 dword; |
| 1403 |
int ret; |
| 1404 |
|
| 1405 |
switch (ecap) { |
| 1406 |
case PCI_EXT_CAP_ID_VNDR: |
| 1407 |
ret = pci_read_config_dword(pdev, epos + PCI_VNDR_HEADER, |
| 1408 |
&dword); |
| 1409 |
if (ret) |
| 1410 |
return pcibios_err_to_errno(ret); |
| 1411 |
|
| 1412 |
return PCI_VNDR_HEADER_LEN(dword); |
| 1413 |
case PCI_EXT_CAP_ID_VC: |
| 1414 |
case PCI_EXT_CAP_ID_VC9: |
| 1415 |
case PCI_EXT_CAP_ID_MFVC: |
| 1416 |
return vfio_vc_cap_len(vdev, epos); |
| 1417 |
case PCI_EXT_CAP_ID_ACS: |
| 1418 |
ret = pci_read_config_byte(pdev, epos + PCI_ACS_CAP, &byte); |
| 1419 |
if (ret) |
| 1420 |
return pcibios_err_to_errno(ret); |
| 1421 |
|
| 1422 |
if (byte & PCI_ACS_EC) { |
| 1423 |
int bits; |
| 1424 |
|
| 1425 |
ret = pci_read_config_byte(pdev, |
| 1426 |
epos + PCI_ACS_EGRESS_BITS, |
| 1427 |
&byte); |
| 1428 |
if (ret) |
| 1429 |
return pcibios_err_to_errno(ret); |
| 1430 |
|
| 1431 |
bits = byte ? round_up(byte, 32) : 256; |
| 1432 |
return 8 + (bits / 8); |
| 1433 |
} |
| 1434 |
return 8; |
| 1435 |
|
| 1436 |
case PCI_EXT_CAP_ID_REBAR: |
| 1437 |
ret = pci_read_config_byte(pdev, epos + PCI_REBAR_CTRL, &byte); |
| 1438 |
if (ret) |
| 1439 |
return pcibios_err_to_errno(ret); |
| 1440 |
|
| 1441 |
byte &= PCI_REBAR_CTRL_NBAR_MASK; |
| 1442 |
byte >>= PCI_REBAR_CTRL_NBAR_SHIFT; |
| 1443 |
|
| 1444 |
return 4 + (byte * 8); |
| 1445 |
case PCI_EXT_CAP_ID_DPA: |
| 1446 |
ret = pci_read_config_byte(pdev, epos + PCI_DPA_CAP, &byte); |
| 1447 |
if (ret) |
| 1448 |
return pcibios_err_to_errno(ret); |
| 1449 |
|
| 1450 |
byte &= PCI_DPA_CAP_SUBSTATE_MASK; |
| 1451 |
return PCI_DPA_BASE_SIZEOF + byte + 1; |
| 1452 |
case PCI_EXT_CAP_ID_TPH: |
| 1453 |
ret = pci_read_config_dword(pdev, epos + PCI_TPH_CAP, &dword); |
| 1454 |
if (ret) |
| 1455 |
return pcibios_err_to_errno(ret); |
| 1456 |
|
| 1457 |
if ((dword & PCI_TPH_CAP_LOC_MASK) == PCI_TPH_LOC_CAP) { |
| 1458 |
int sts; |
| 1459 |
|
| 1460 |
sts = dword & PCI_TPH_CAP_ST_MASK; |
| 1461 |
sts >>= PCI_TPH_CAP_ST_SHIFT; |
| 1462 |
return PCI_TPH_BASE_SIZEOF + (sts * 2) + 2; |
| 1463 |
} |
| 1464 |
return PCI_TPH_BASE_SIZEOF; |
| 1465 |
case PCI_EXT_CAP_ID_DVSEC: |
| 1466 |
ret = pci_read_config_dword(pdev, epos + PCI_DVSEC_HEADER1, &dword); |
| 1467 |
if (ret) |
| 1468 |
return pcibios_err_to_errno(ret); |
| 1469 |
return PCI_DVSEC_HEADER1_LEN(dword); |
| 1470 |
default: |
| 1471 |
pci_warn(pdev, "%s: unknown length for PCI ecap %#x@%#x\n", |
| 1472 |
__func__, ecap, epos); |
| 1473 |
} |
| 1474 |
|
| 1475 |
return 0; |
| 1476 |
} |
| 1477 |
|
| 1478 |
static void vfio_update_pm_vconfig_bytes(struct vfio_pci_core_device *vdev, |
| 1479 |
int offset) |
| 1480 |
{ |
| 1481 |
__le16 *pmc = (__le16 *)&vdev->vconfig[offset + PCI_PM_PMC]; |
| 1482 |
__le16 *ctrl = (__le16 *)&vdev->vconfig[offset + PCI_PM_CTRL]; |
| 1483 |
|
| 1484 |
/* Clear vconfig PME_Support, PME_Status, and PME_En bits */ |
| 1485 |
*pmc &= ~cpu_to_le16(PCI_PM_CAP_PME_MASK); |
| 1486 |
*ctrl &= ~cpu_to_le16(PCI_PM_CTRL_PME_ENABLE | PCI_PM_CTRL_PME_STATUS); |
| 1487 |
} |
| 1488 |
|
| 1489 |
static int vfio_fill_vconfig_bytes(struct vfio_pci_core_device *vdev, |
| 1490 |
int offset, int size) |
| 1491 |
{ |
| 1492 |
struct pci_dev *pdev = vdev->pdev; |
| 1493 |
int ret = 0; |
| 1494 |
|
| 1495 |
/* |
| 1496 |
* We try to read physical config space in the largest chunks |
| 1497 |
* we can, assuming that all of the fields support dword access. |
| 1498 |
* pci_save_state() makes this same assumption and seems to do ok. |
| 1499 |
*/ |
| 1500 |
while (size) { |
| 1501 |
int filled; |
| 1502 |
|
| 1503 |
if (size >= 4 && !(offset % 4)) { |
| 1504 |
__le32 *dwordp = (__le32 *)&vdev->vconfig[offset]; |
| 1505 |
u32 dword; |
| 1506 |
|
| 1507 |
ret = pci_read_config_dword(pdev, offset, &dword); |
| 1508 |
if (ret) |
| 1509 |
return ret; |
| 1510 |
*dwordp = cpu_to_le32(dword); |
| 1511 |
filled = 4; |
| 1512 |
} else if (size >= 2 && !(offset % 2)) { |
| 1513 |
__le16 *wordp = (__le16 *)&vdev->vconfig[offset]; |
| 1514 |
u16 word; |
| 1515 |
|
| 1516 |
ret = pci_read_config_word(pdev, offset, &word); |
| 1517 |
if (ret) |
| 1518 |
return ret; |
| 1519 |
*wordp = cpu_to_le16(word); |
| 1520 |
filled = 2; |
| 1521 |
} else { |
| 1522 |
u8 *byte = &vdev->vconfig[offset]; |
| 1523 |
ret = pci_read_config_byte(pdev, offset, byte); |
| 1524 |
if (ret) |
| 1525 |
return ret; |
| 1526 |
filled = 1; |
| 1527 |
} |
| 1528 |
|
| 1529 |
offset += filled; |
| 1530 |
size -= filled; |
| 1531 |
} |
| 1532 |
|
| 1533 |
return ret; |
| 1534 |
} |
| 1535 |
|
| 1536 |
static int vfio_cap_init(struct vfio_pci_core_device *vdev) |
| 1537 |
{ |
| 1538 |
struct pci_dev *pdev = vdev->pdev; |
| 1539 |
u8 *map = vdev->pci_config_map; |
| 1540 |
u16 status; |
| 1541 |
u8 pos, *prev, cap; |
| 1542 |
int loops, ret, caps = 0; |
| 1543 |
|
| 1544 |
/* Any capabilities? */ |
| 1545 |
ret = pci_read_config_word(pdev, PCI_STATUS, &status); |
| 1546 |
if (ret) |
| 1547 |
return ret; |
| 1548 |
|
| 1549 |
if (!(status & PCI_STATUS_CAP_LIST)) |
| 1550 |
return 0; /* Done */ |
| 1551 |
|
| 1552 |
ret = pci_read_config_byte(pdev, PCI_CAPABILITY_LIST, &pos); |
| 1553 |
if (ret) |
| 1554 |
return ret; |
| 1555 |
|
| 1556 |
/* Mark the previous position in case we want to skip a capability */ |
| 1557 |
prev = &vdev->vconfig[PCI_CAPABILITY_LIST]; |
| 1558 |
|
| 1559 |
/* We can bound our loop, capabilities are dword aligned */ |
| 1560 |
loops = (PCI_CFG_SPACE_SIZE - PCI_STD_HEADER_SIZEOF) / PCI_CAP_SIZEOF; |
| 1561 |
while (pos && loops--) { |
| 1562 |
u8 next; |
| 1563 |
int i, len = 0; |
| 1564 |
|
| 1565 |
ret = pci_read_config_byte(pdev, pos, &cap); |
| 1566 |
if (ret) |
| 1567 |
return ret; |
| 1568 |
|
| 1569 |
ret = pci_read_config_byte(pdev, |
| 1570 |
pos + PCI_CAP_LIST_NEXT, &next); |
| 1571 |
if (ret) |
| 1572 |
return ret; |
| 1573 |
|
| 1574 |
/* |
| 1575 |
* ID 0 is a NULL capability, conflicting with our fake |
| 1576 |
* PCI_CAP_ID_BASIC. As it has no content, consider it |
| 1577 |
* hidden for now. |
| 1578 |
*/ |
| 1579 |
if (cap && cap <= PCI_CAP_ID_MAX) { |
| 1580 |
len = pci_cap_length[cap]; |
| 1581 |
if (len == 0xFF) { /* Variable length */ |
| 1582 |
len = vfio_cap_len(vdev, cap, pos); |
| 1583 |
if (len < 0) |
| 1584 |
return len; |
| 1585 |
} |
| 1586 |
} |
| 1587 |
|
| 1588 |
if (!len) { |
| 1589 |
pci_dbg(pdev, "%s: hiding cap %#x@%#x\n", __func__, |
| 1590 |
cap, pos); |
| 1591 |
*prev = next; |
| 1592 |
pos = next; |
| 1593 |
continue; |
| 1594 |
} |
| 1595 |
|
| 1596 |
/* Sanity check, do we overlap other capabilities? */ |
| 1597 |
for (i = 0; i < len; i++) { |
| 1598 |
if (likely(map[pos + i] == PCI_CAP_ID_INVALID)) |
| 1599 |
continue; |
| 1600 |
|
| 1601 |
pci_warn(pdev, "%s: PCI config conflict @%#x, was cap %#x now cap %#x\n", |
| 1602 |
__func__, pos + i, map[pos + i], cap); |
| 1603 |
} |
| 1604 |
|
| 1605 |
BUILD_BUG_ON(PCI_CAP_ID_MAX >= PCI_CAP_ID_INVALID_VIRT); |
| 1606 |
|
| 1607 |
memset(map + pos, cap, len); |
| 1608 |
ret = vfio_fill_vconfig_bytes(vdev, pos, len); |
| 1609 |
if (ret) |
| 1610 |
return ret; |
| 1611 |
|
| 1612 |
if (cap == PCI_CAP_ID_PM) |
| 1613 |
vfio_update_pm_vconfig_bytes(vdev, pos); |
| 1614 |
|
| 1615 |
prev = &vdev->vconfig[pos + PCI_CAP_LIST_NEXT]; |
| 1616 |
pos = next; |
| 1617 |
caps++; |
| 1618 |
} |
| 1619 |
|
| 1620 |
/* If we didn't fill any capabilities, clear the status flag */ |
| 1621 |
if (!caps) { |
| 1622 |
__le16 *vstatus = (__le16 *)&vdev->vconfig[PCI_STATUS]; |
| 1623 |
*vstatus &= ~cpu_to_le16(PCI_STATUS_CAP_LIST); |
| 1624 |
} |
| 1625 |
|
| 1626 |
return 0; |
| 1627 |
} |
| 1628 |
|
| 1629 |
static int vfio_ecap_init(struct vfio_pci_core_device *vdev) |
| 1630 |
{ |
| 1631 |
struct pci_dev *pdev = vdev->pdev; |
| 1632 |
u8 *map = vdev->pci_config_map; |
| 1633 |
u16 epos; |
| 1634 |
__le32 *prev = NULL; |
| 1635 |
int loops, ret, ecaps = 0; |
| 1636 |
|
| 1637 |
if (!vdev->extended_caps) |
| 1638 |
return 0; |
| 1639 |
|
| 1640 |
epos = PCI_CFG_SPACE_SIZE; |
| 1641 |
|
| 1642 |
loops = (pdev->cfg_size - PCI_CFG_SPACE_SIZE) / PCI_CAP_SIZEOF; |
| 1643 |
|
| 1644 |
while (loops-- && epos >= PCI_CFG_SPACE_SIZE) { |
| 1645 |
u32 header; |
| 1646 |
u16 ecap; |
| 1647 |
int i, len = 0; |
| 1648 |
bool hidden = false; |
| 1649 |
|
| 1650 |
ret = pci_read_config_dword(pdev, epos, &header); |
| 1651 |
if (ret) |
| 1652 |
return ret; |
| 1653 |
|
| 1654 |
ecap = PCI_EXT_CAP_ID(header); |
| 1655 |
|
| 1656 |
if (ecap <= PCI_EXT_CAP_ID_MAX) { |
| 1657 |
len = pci_ext_cap_length[ecap]; |
| 1658 |
if (len == 0xFF) { |
| 1659 |
len = vfio_ext_cap_len(vdev, ecap, epos); |
| 1660 |
if (len < 0) |
| 1661 |
return len; |
| 1662 |
} |
| 1663 |
} |
| 1664 |
|
| 1665 |
if (!len) { |
| 1666 |
pci_dbg(pdev, "%s: hiding ecap %#x@%#x\n", |
| 1667 |
__func__, ecap, epos); |
| 1668 |
|
| 1669 |
/* If not the first in the chain, we can skip over it */ |
| 1670 |
if (prev) { |
| 1671 |
u32 val = epos = PCI_EXT_CAP_NEXT(header); |
| 1672 |
*prev &= cpu_to_le32(~(0xffcU << 20)); |
| 1673 |
*prev |= cpu_to_le32(val << 20); |
| 1674 |
continue; |
| 1675 |
} |
| 1676 |
|
| 1677 |
/* |
| 1678 |
* Otherwise, fill in a placeholder, the direct |
| 1679 |
* readfn will virtualize this automatically |
| 1680 |
*/ |
| 1681 |
len = PCI_CAP_SIZEOF; |
| 1682 |
hidden = true; |
| 1683 |
} |
| 1684 |
|
| 1685 |
for (i = 0; i < len; i++) { |
| 1686 |
if (likely(map[epos + i] == PCI_CAP_ID_INVALID)) |
| 1687 |
continue; |
| 1688 |
|
| 1689 |
pci_warn(pdev, "%s: PCI config conflict @%#x, was ecap %#x now ecap %#x\n", |
| 1690 |
__func__, epos + i, map[epos + i], ecap); |
| 1691 |
} |
| 1692 |
|
| 1693 |
/* |
| 1694 |
* Even though ecap is 2 bytes, we're currently a long way |
| 1695 |
* from exceeding 1 byte capabilities. If we ever make it |
| 1696 |
* up to 0xFE we'll need to up this to a two-byte, byte map. |
| 1697 |
*/ |
| 1698 |
BUILD_BUG_ON(PCI_EXT_CAP_ID_MAX >= PCI_CAP_ID_INVALID_VIRT); |
| 1699 |
|
| 1700 |
memset(map + epos, ecap, len); |
| 1701 |
ret = vfio_fill_vconfig_bytes(vdev, epos, len); |
| 1702 |
if (ret) |
| 1703 |
return ret; |
| 1704 |
|
| 1705 |
/* |
| 1706 |
* If we're just using this capability to anchor the list, |
| 1707 |
* hide the real ID. Only count real ecaps. XXX PCI spec |
| 1708 |
* indicates to use cap id = 0, version = 0, next = 0 if |
| 1709 |
* ecaps are absent, hope users check all the way to next. |
| 1710 |
*/ |
| 1711 |
if (hidden) |
| 1712 |
*(__le32 *)&vdev->vconfig[epos] &= |
| 1713 |
cpu_to_le32((0xffcU << 20)); |
| 1714 |
else |
| 1715 |
ecaps++; |
| 1716 |
|
| 1717 |
prev = (__le32 *)&vdev->vconfig[epos]; |
| 1718 |
epos = PCI_EXT_CAP_NEXT(header); |
| 1719 |
} |
| 1720 |
|
| 1721 |
if (!ecaps) |
| 1722 |
*(u32 *)&vdev->vconfig[PCI_CFG_SPACE_SIZE] = 0; |
| 1723 |
|
| 1724 |
return 0; |
| 1725 |
} |
| 1726 |
|
| 1727 |
/* |
| 1728 |
* Nag about hardware bugs, hopefully to have vendors fix them, but at least |
| 1729 |
* to collect a list of dependencies for the VF INTx pin quirk below. |
| 1730 |
*/ |
| 1731 |
static const struct pci_device_id known_bogus_vf_intx_pin[] = { |
| 1732 |
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x270c) }, |
| 1733 |
{} |
| 1734 |
}; |
| 1735 |
|
| 1736 |
/* |
| 1737 |
* For each device we allocate a pci_config_map that indicates the |
| 1738 |
* capability occupying each dword and thus the struct perm_bits we |
| 1739 |
* use for read and write. We also allocate a virtualized config |
| 1740 |
* space which tracks reads and writes to bits that we emulate for |
| 1741 |
* the user. Initial values filled from device. |
| 1742 |
* |
| 1743 |
* Using shared struct perm_bits between all vfio-pci devices saves |
| 1744 |
* us from allocating cfg_size buffers for virt and write for every |
| 1745 |
* device. We could remove vconfig and allocate individual buffers |
| 1746 |
* for each area requiring emulated bits, but the array of pointers |
| 1747 |
* would be comparable in size (at least for standard config space). |
| 1748 |
*/ |
| 1749 |
int vfio_config_init(struct vfio_pci_core_device *vdev) |
| 1750 |
{ |
| 1751 |
struct pci_dev *pdev = vdev->pdev; |
| 1752 |
u8 *map, *vconfig; |
| 1753 |
int ret; |
| 1754 |
|
| 1755 |
/* |
| 1756 |
* Config space, caps and ecaps are all dword aligned, so we could |
| 1757 |
* use one byte per dword to record the type. However, there are |
| 1758 |
* no requirements on the length of a capability, so the gap between |
| 1759 |
* capabilities needs byte granularity. |
| 1760 |
*/ |
| 1761 |
map = kmalloc(pdev->cfg_size, GFP_KERNEL_ACCOUNT); |
| 1762 |
if (!map) |
| 1763 |
return -ENOMEM; |
| 1764 |
|
| 1765 |
vconfig = kmalloc(pdev->cfg_size, GFP_KERNEL_ACCOUNT); |
| 1766 |
if (!vconfig) { |
| 1767 |
kfree(map); |
| 1768 |
return -ENOMEM; |
| 1769 |
} |
| 1770 |
|
| 1771 |
vdev->pci_config_map = map; |
| 1772 |
vdev->vconfig = vconfig; |
| 1773 |
|
| 1774 |
memset(map, PCI_CAP_ID_BASIC, PCI_STD_HEADER_SIZEOF); |
| 1775 |
memset(map + PCI_STD_HEADER_SIZEOF, PCI_CAP_ID_INVALID, |
| 1776 |
pdev->cfg_size - PCI_STD_HEADER_SIZEOF); |
| 1777 |
|
| 1778 |
ret = vfio_fill_vconfig_bytes(vdev, 0, PCI_STD_HEADER_SIZEOF); |
| 1779 |
if (ret) |
| 1780 |
goto out; |
| 1781 |
|
| 1782 |
vdev->bardirty = true; |
| 1783 |
|
| 1784 |
/* |
| 1785 |
* XXX can we just pci_load_saved_state/pci_restore_state? |
| 1786 |
* may need to rebuild vconfig after that |
| 1787 |
*/ |
| 1788 |
|
| 1789 |
/* For restore after reset */ |
| 1790 |
vdev->rbar[0] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_0]); |
| 1791 |
vdev->rbar[1] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_1]); |
| 1792 |
vdev->rbar[2] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_2]); |
| 1793 |
vdev->rbar[3] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_3]); |
| 1794 |
vdev->rbar[4] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_4]); |
| 1795 |
vdev->rbar[5] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_5]); |
| 1796 |
vdev->rbar[6] = le32_to_cpu(*(__le32 *)&vconfig[PCI_ROM_ADDRESS]); |
| 1797 |
|
| 1798 |
if (pdev->is_virtfn) { |
| 1799 |
*(__le16 *)&vconfig[PCI_VENDOR_ID] = cpu_to_le16(pdev->vendor); |
| 1800 |
*(__le16 *)&vconfig[PCI_DEVICE_ID] = cpu_to_le16(pdev->device); |
| 1801 |
|
| 1802 |
/* |
| 1803 |
* Per SR-IOV spec rev 1.1, 3.4.1.18 the interrupt pin register |
| 1804 |
* does not apply to VFs and VFs must implement this register |
| 1805 |
* as read-only with value zero. Userspace is not readily able |
| 1806 |
* to identify whether a device is a VF and thus that the pin |
| 1807 |
* definition on the device is bogus should it violate this |
| 1808 |
* requirement. We already virtualize the pin register for |
| 1809 |
* other purposes, so we simply need to replace the bogus value |
| 1810 |
* and consider VFs when we determine INTx IRQ count. |
| 1811 |
*/ |
| 1812 |
if (vconfig[PCI_INTERRUPT_PIN] && |
| 1813 |
!pci_match_id(known_bogus_vf_intx_pin, pdev)) |
| 1814 |
pci_warn(pdev, |
| 1815 |
"Hardware bug: VF reports bogus INTx pin %d\n", |
| 1816 |
vconfig[PCI_INTERRUPT_PIN]); |
| 1817 |
|
| 1818 |
vconfig[PCI_INTERRUPT_PIN] = 0; /* Gratuitous for good VFs */ |
| 1819 |
} |
| 1820 |
if (pdev->no_command_memory) { |
| 1821 |
/* |
| 1822 |
* VFs and devices that set pdev->no_command_memory do not |
| 1823 |
* implement the memory enable bit of the COMMAND register |
| 1824 |
* therefore we'll not have it set in our initial copy of |
| 1825 |
* config space after pci_enable_device(). For consistency |
| 1826 |
* with PFs, set the virtual enable bit here. |
| 1827 |
*/ |
| 1828 |
*(__le16 *)&vconfig[PCI_COMMAND] |= |
| 1829 |
cpu_to_le16(PCI_COMMAND_MEMORY); |
| 1830 |
} |
| 1831 |
|
| 1832 |
if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || vdev->nointx || |
| 1833 |
!vdev->pdev->irq || vdev->pdev->irq == IRQ_NOTCONNECTED) |
| 1834 |
vconfig[PCI_INTERRUPT_PIN] = 0; |
| 1835 |
|
| 1836 |
ret = vfio_cap_init(vdev); |
| 1837 |
if (ret) |
| 1838 |
goto out; |
| 1839 |
|
| 1840 |
ret = vfio_ecap_init(vdev); |
| 1841 |
if (ret) |
| 1842 |
goto out; |
| 1843 |
|
| 1844 |
return 0; |
| 1845 |
|
| 1846 |
out: |
| 1847 |
kfree(map); |
| 1848 |
vdev->pci_config_map = NULL; |
| 1849 |
kfree(vconfig); |
| 1850 |
vdev->vconfig = NULL; |
| 1851 |
return pcibios_err_to_errno(ret); |
| 1852 |
} |
| 1853 |
|
| 1854 |
void vfio_config_free(struct vfio_pci_core_device *vdev) |
| 1855 |
{ |
| 1856 |
kfree(vdev->vconfig); |
| 1857 |
vdev->vconfig = NULL; |
| 1858 |
kfree(vdev->pci_config_map); |
| 1859 |
vdev->pci_config_map = NULL; |
| 1860 |
if (vdev->msi_perm) { |
| 1861 |
free_perm_bits(vdev->msi_perm); |
| 1862 |
kfree(vdev->msi_perm); |
| 1863 |
vdev->msi_perm = NULL; |
| 1864 |
} |
| 1865 |
} |
| 1866 |
|
| 1867 |
/* |
| 1868 |
* Find the remaining number of bytes in a dword that match the given |
| 1869 |
* position. Stop at either the end of the capability or the dword boundary. |
| 1870 |
*/ |
| 1871 |
static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_core_device *vdev, |
| 1872 |
loff_t pos) |
| 1873 |
{ |
| 1874 |
u8 cap = vdev->pci_config_map[pos]; |
| 1875 |
size_t i; |
| 1876 |
|
| 1877 |
for (i = 1; (pos + i) % 4 && vdev->pci_config_map[pos + i] == cap; i++) |
| 1878 |
/* nop */; |
| 1879 |
|
| 1880 |
return i; |
| 1881 |
} |
| 1882 |
|
| 1883 |
static ssize_t vfio_config_do_rw(struct vfio_pci_core_device *vdev, char __user *buf, |
| 1884 |
size_t count, loff_t *ppos, bool iswrite) |
| 1885 |
{ |
| 1886 |
struct pci_dev *pdev = vdev->pdev; |
| 1887 |
struct perm_bits *perm; |
| 1888 |
__le32 val = 0; |
| 1889 |
int cap_start = 0, offset; |
| 1890 |
u8 cap_id; |
| 1891 |
ssize_t ret; |
| 1892 |
|
| 1893 |
if (*ppos < 0 || *ppos >= pdev->cfg_size || |
| 1894 |
*ppos + count > pdev->cfg_size) |
| 1895 |
return -EFAULT; |
| 1896 |
|
| 1897 |
/* |
| 1898 |
* Chop accesses into aligned chunks containing no more than a |
| 1899 |
* single capability. Caller increments to the next chunk. |
| 1900 |
*/ |
| 1901 |
count = min(count, vfio_pci_cap_remaining_dword(vdev, *ppos)); |
| 1902 |
if (count >= 4 && !(*ppos % 4)) |
| 1903 |
count = 4; |
| 1904 |
else if (count >= 2 && !(*ppos % 2)) |
| 1905 |
count = 2; |
| 1906 |
else |
| 1907 |
count = 1; |
| 1908 |
|
| 1909 |
ret = count; |
| 1910 |
|
| 1911 |
cap_id = vdev->pci_config_map[*ppos]; |
| 1912 |
|
| 1913 |
if (cap_id == PCI_CAP_ID_INVALID) { |
| 1914 |
perm = &unassigned_perms; |
| 1915 |
cap_start = *ppos; |
| 1916 |
} else if (cap_id == PCI_CAP_ID_INVALID_VIRT) { |
| 1917 |
perm = &virt_perms; |
| 1918 |
cap_start = *ppos; |
| 1919 |
} else { |
| 1920 |
if (*ppos >= PCI_CFG_SPACE_SIZE) { |
| 1921 |
/* |
| 1922 |
* We can get a cap_id that exceeds PCI_EXT_CAP_ID_MAX |
| 1923 |
* if we're hiding an unknown capability at the start |
| 1924 |
* of the extended capability list. Use default, ro |
| 1925 |
* access, which will virtualize the id and next values. |
| 1926 |
*/ |
| 1927 |
if (cap_id > PCI_EXT_CAP_ID_MAX) |
| 1928 |
perm = &direct_ro_perms; |
| 1929 |
else |
| 1930 |
perm = &ecap_perms[cap_id]; |
| 1931 |
|
| 1932 |
cap_start = vfio_find_cap_start(vdev, *ppos); |
| 1933 |
} else { |
| 1934 |
WARN_ON(cap_id > PCI_CAP_ID_MAX); |
| 1935 |
|
| 1936 |
perm = &cap_perms[cap_id]; |
| 1937 |
|
| 1938 |
if (cap_id == PCI_CAP_ID_MSI) |
| 1939 |
perm = vdev->msi_perm; |
| 1940 |
|
| 1941 |
if (cap_id > PCI_CAP_ID_BASIC) |
| 1942 |
cap_start = vfio_find_cap_start(vdev, *ppos); |
| 1943 |
} |
| 1944 |
} |
| 1945 |
|
| 1946 |
WARN_ON(!cap_start && cap_id != PCI_CAP_ID_BASIC); |
| 1947 |
WARN_ON(cap_start > *ppos); |
| 1948 |
|
| 1949 |
offset = *ppos - cap_start; |
| 1950 |
|
| 1951 |
if (iswrite) { |
| 1952 |
if (!perm->writefn) |
| 1953 |
return ret; |
| 1954 |
|
| 1955 |
if (copy_from_user(&val, buf, count)) |
| 1956 |
return -EFAULT; |
| 1957 |
|
| 1958 |
ret = perm->writefn(vdev, *ppos, count, perm, offset, val); |
| 1959 |
} else { |
| 1960 |
if (perm->readfn) { |
| 1961 |
ret = perm->readfn(vdev, *ppos, count, |
| 1962 |
perm, offset, &val); |
| 1963 |
if (ret < 0) |
| 1964 |
return ret; |
| 1965 |
} |
| 1966 |
|
| 1967 |
if (copy_to_user(buf, &val, count)) |
| 1968 |
return -EFAULT; |
| 1969 |
} |
| 1970 |
|
| 1971 |
return ret; |
| 1972 |
} |
| 1973 |
|
| 1974 |
ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev, char __user *buf, |
| 1975 |
size_t count, loff_t *ppos, bool iswrite) |
| 1976 |
{ |
| 1977 |
size_t done = 0; |
| 1978 |
int ret = 0; |
| 1979 |
loff_t pos = *ppos; |
| 1980 |
|
| 1981 |
pos &= VFIO_PCI_OFFSET_MASK; |
| 1982 |
|
| 1983 |
while (count) { |
| 1984 |
ret = vfio_config_do_rw(vdev, buf, count, &pos, iswrite); |
| 1985 |
if (ret < 0) |
| 1986 |
return ret; |
| 1987 |
|
| 1988 |
count -= ret; |
| 1989 |
done += ret; |
| 1990 |
buf += ret; |
| 1991 |
pos += ret; |
| 1992 |
} |
| 1993 |
|
| 1994 |
*ppos += done; |
| 1995 |
|
| 1996 |
return done; |
| 1997 |
} |
| 1998 |
|
| 1999 |
/** |
| 2000 |
* vfio_pci_core_range_intersect_range() - Determine overlap between a buffer |
| 2001 |
* and register offset ranges. |
| 2002 |
* @buf_start: start offset of the buffer |
| 2003 |
* @buf_cnt: number of buffer bytes |
| 2004 |
* @reg_start: start register offset |
| 2005 |
* @reg_cnt: number of register bytes |
| 2006 |
* @buf_offset: start offset of overlap in the buffer |
| 2007 |
* @intersect_count: number of overlapping bytes |
| 2008 |
* @register_offset: start offset of overlap in register |
| 2009 |
* |
| 2010 |
* Returns: true if there is overlap, false if not. |
| 2011 |
* The overlap start and size is returned through function args. |
| 2012 |
*/ |
| 2013 |
bool vfio_pci_core_range_intersect_range(loff_t buf_start, size_t buf_cnt, |
| 2014 |
loff_t reg_start, size_t reg_cnt, |
| 2015 |
loff_t *buf_offset, |
| 2016 |
size_t *intersect_count, |
| 2017 |
size_t *register_offset) |
| 2018 |
{ |
| 2019 |
if (buf_start <= reg_start && |
| 2020 |
buf_start + buf_cnt > reg_start) { |
| 2021 |
*buf_offset = reg_start - buf_start; |
| 2022 |
*intersect_count = min_t(size_t, reg_cnt, |
| 2023 |
buf_start + buf_cnt - reg_start); |
| 2024 |
*register_offset = 0; |
| 2025 |
return true; |
| 2026 |
} |
| 2027 |
|
| 2028 |
if (buf_start > reg_start && |
| 2029 |
buf_start < reg_start + reg_cnt) { |
| 2030 |
*buf_offset = 0; |
| 2031 |
*intersect_count = min_t(size_t, buf_cnt, |
| 2032 |
reg_start + reg_cnt - buf_start); |
| 2033 |
*register_offset = buf_start - reg_start; |
| 2034 |
return true; |
| 2035 |
} |
| 2036 |
|
| 2037 |
return false; |
| 2038 |
} |
| 2039 |
EXPORT_SYMBOL_GPL(vfio_pci_core_range_intersect_range); |