| 1 |
// SPDX-License-Identifier: GPL-2.0-only |
| 2 |
/* |
| 3 |
* VFIO platform devices interrupt handling |
| 4 |
* |
| 5 |
* Copyright (C) 2013 - Virtual Open Systems |
| 6 |
* Author: Antonios Motakis <[email protected]> |
| 7 |
*/ |
| 8 |
|
| 9 |
#include <linux/eventfd.h> |
| 10 |
#include <linux/interrupt.h> |
| 11 |
#include <linux/slab.h> |
| 12 |
#include <linux/types.h> |
| 13 |
#include <linux/vfio.h> |
| 14 |
#include <linux/irq.h> |
| 15 |
|
| 16 |
#include "vfio_platform_private.h" |
| 17 |
|
| 18 |
static void vfio_platform_mask(struct vfio_platform_irq *irq_ctx) |
| 19 |
{ |
| 20 |
unsigned long flags; |
| 21 |
|
| 22 |
spin_lock_irqsave(&irq_ctx->lock, flags); |
| 23 |
|
| 24 |
if (!irq_ctx->masked) { |
| 25 |
disable_irq_nosync(irq_ctx->hwirq); |
| 26 |
irq_ctx->masked = true; |
| 27 |
} |
| 28 |
|
| 29 |
spin_unlock_irqrestore(&irq_ctx->lock, flags); |
| 30 |
} |
| 31 |
|
| 32 |
static int vfio_platform_mask_handler(void *opaque, void *unused) |
| 33 |
{ |
| 34 |
struct vfio_platform_irq *irq_ctx = opaque; |
| 35 |
|
| 36 |
vfio_platform_mask(irq_ctx); |
| 37 |
|
| 38 |
return 0; |
| 39 |
} |
| 40 |
|
| 41 |
static int vfio_platform_set_irq_mask(struct vfio_platform_device *vdev, |
| 42 |
unsigned index, unsigned start, |
| 43 |
unsigned count, uint32_t flags, |
| 44 |
void *data) |
| 45 |
{ |
| 46 |
if (start != 0 || count != 1) |
| 47 |
return -EINVAL; |
| 48 |
|
| 49 |
if (!(vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE)) |
| 50 |
return -EINVAL; |
| 51 |
|
| 52 |
if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
| 53 |
int32_t fd = *(int32_t *)data; |
| 54 |
|
| 55 |
if (fd >= 0) |
| 56 |
return vfio_virqfd_enable((void *) &vdev->irqs[index], |
| 57 |
vfio_platform_mask_handler, |
| 58 |
NULL, NULL, |
| 59 |
&vdev->irqs[index].mask, fd); |
| 60 |
|
| 61 |
vfio_virqfd_disable(&vdev->irqs[index].mask); |
| 62 |
return 0; |
| 63 |
} |
| 64 |
|
| 65 |
if (flags & VFIO_IRQ_SET_DATA_NONE) { |
| 66 |
vfio_platform_mask(&vdev->irqs[index]); |
| 67 |
|
| 68 |
} else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
| 69 |
uint8_t mask = *(uint8_t *)data; |
| 70 |
|
| 71 |
if (mask) |
| 72 |
vfio_platform_mask(&vdev->irqs[index]); |
| 73 |
} |
| 74 |
|
| 75 |
return 0; |
| 76 |
} |
| 77 |
|
| 78 |
static void vfio_platform_unmask(struct vfio_platform_irq *irq_ctx) |
| 79 |
{ |
| 80 |
unsigned long flags; |
| 81 |
|
| 82 |
spin_lock_irqsave(&irq_ctx->lock, flags); |
| 83 |
|
| 84 |
if (irq_ctx->masked) { |
| 85 |
enable_irq(irq_ctx->hwirq); |
| 86 |
irq_ctx->masked = false; |
| 87 |
} |
| 88 |
|
| 89 |
spin_unlock_irqrestore(&irq_ctx->lock, flags); |
| 90 |
} |
| 91 |
|
| 92 |
static int vfio_platform_unmask_handler(void *opaque, void *unused) |
| 93 |
{ |
| 94 |
struct vfio_platform_irq *irq_ctx = opaque; |
| 95 |
|
| 96 |
vfio_platform_unmask(irq_ctx); |
| 97 |
|
| 98 |
return 0; |
| 99 |
} |
| 100 |
|
| 101 |
static int vfio_platform_set_irq_unmask(struct vfio_platform_device *vdev, |
| 102 |
unsigned index, unsigned start, |
| 103 |
unsigned count, uint32_t flags, |
| 104 |
void *data) |
| 105 |
{ |
| 106 |
if (start != 0 || count != 1) |
| 107 |
return -EINVAL; |
| 108 |
|
| 109 |
if (!(vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE)) |
| 110 |
return -EINVAL; |
| 111 |
|
| 112 |
if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
| 113 |
int32_t fd = *(int32_t *)data; |
| 114 |
|
| 115 |
if (fd >= 0) |
| 116 |
return vfio_virqfd_enable((void *) &vdev->irqs[index], |
| 117 |
vfio_platform_unmask_handler, |
| 118 |
NULL, NULL, |
| 119 |
&vdev->irqs[index].unmask, |
| 120 |
fd); |
| 121 |
|
| 122 |
vfio_virqfd_disable(&vdev->irqs[index].unmask); |
| 123 |
return 0; |
| 124 |
} |
| 125 |
|
| 126 |
if (flags & VFIO_IRQ_SET_DATA_NONE) { |
| 127 |
vfio_platform_unmask(&vdev->irqs[index]); |
| 128 |
|
| 129 |
} else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
| 130 |
uint8_t unmask = *(uint8_t *)data; |
| 131 |
|
| 132 |
if (unmask) |
| 133 |
vfio_platform_unmask(&vdev->irqs[index]); |
| 134 |
} |
| 135 |
|
| 136 |
return 0; |
| 137 |
} |
| 138 |
|
| 139 |
/* |
| 140 |
* The trigger eventfd is guaranteed valid in the interrupt path |
| 141 |
* and protected by the igate mutex when triggered via ioctl. |
| 142 |
*/ |
| 143 |
static void vfio_send_eventfd(struct vfio_platform_irq *irq_ctx) |
| 144 |
{ |
| 145 |
if (likely(irq_ctx->trigger)) |
| 146 |
eventfd_signal(irq_ctx->trigger); |
| 147 |
} |
| 148 |
|
| 149 |
static irqreturn_t vfio_automasked_irq_handler(int irq, void *dev_id) |
| 150 |
{ |
| 151 |
struct vfio_platform_irq *irq_ctx = dev_id; |
| 152 |
unsigned long flags; |
| 153 |
int ret = IRQ_NONE; |
| 154 |
|
| 155 |
spin_lock_irqsave(&irq_ctx->lock, flags); |
| 156 |
|
| 157 |
if (!irq_ctx->masked) { |
| 158 |
ret = IRQ_HANDLED; |
| 159 |
|
| 160 |
/* automask maskable interrupts */ |
| 161 |
disable_irq_nosync(irq_ctx->hwirq); |
| 162 |
irq_ctx->masked = true; |
| 163 |
} |
| 164 |
|
| 165 |
spin_unlock_irqrestore(&irq_ctx->lock, flags); |
| 166 |
|
| 167 |
if (ret == IRQ_HANDLED) |
| 168 |
vfio_send_eventfd(irq_ctx); |
| 169 |
|
| 170 |
return ret; |
| 171 |
} |
| 172 |
|
| 173 |
static irqreturn_t vfio_irq_handler(int irq, void *dev_id) |
| 174 |
{ |
| 175 |
struct vfio_platform_irq *irq_ctx = dev_id; |
| 176 |
|
| 177 |
vfio_send_eventfd(irq_ctx); |
| 178 |
|
| 179 |
return IRQ_HANDLED; |
| 180 |
} |
| 181 |
|
| 182 |
static int vfio_set_trigger(struct vfio_platform_device *vdev, int index, |
| 183 |
int fd) |
| 184 |
{ |
| 185 |
struct vfio_platform_irq *irq = &vdev->irqs[index]; |
| 186 |
struct eventfd_ctx *trigger; |
| 187 |
|
| 188 |
if (irq->trigger) { |
| 189 |
disable_irq(irq->hwirq); |
| 190 |
eventfd_ctx_put(irq->trigger); |
| 191 |
irq->trigger = NULL; |
| 192 |
} |
| 193 |
|
| 194 |
if (fd < 0) /* Disable only */ |
| 195 |
return 0; |
| 196 |
|
| 197 |
trigger = eventfd_ctx_fdget(fd); |
| 198 |
if (IS_ERR(trigger)) |
| 199 |
return PTR_ERR(trigger); |
| 200 |
|
| 201 |
irq->trigger = trigger; |
| 202 |
|
| 203 |
/* |
| 204 |
* irq->masked effectively provides nested disables within the overall |
| 205 |
* enable relative to trigger. Specifically request_irq() is called |
| 206 |
* with NO_AUTOEN, therefore the IRQ is initially disabled. The user |
| 207 |
* may only further disable the IRQ with a MASK operations because |
| 208 |
* irq->masked is initially false. |
| 209 |
*/ |
| 210 |
enable_irq(irq->hwirq); |
| 211 |
|
| 212 |
return 0; |
| 213 |
} |
| 214 |
|
| 215 |
static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev, |
| 216 |
unsigned index, unsigned start, |
| 217 |
unsigned count, uint32_t flags, |
| 218 |
void *data) |
| 219 |
{ |
| 220 |
struct vfio_platform_irq *irq = &vdev->irqs[index]; |
| 221 |
irq_handler_t handler; |
| 222 |
|
| 223 |
if (vdev->irqs[index].flags & VFIO_IRQ_INFO_AUTOMASKED) |
| 224 |
handler = vfio_automasked_irq_handler; |
| 225 |
else |
| 226 |
handler = vfio_irq_handler; |
| 227 |
|
| 228 |
if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) |
| 229 |
return vfio_set_trigger(vdev, index, -1); |
| 230 |
|
| 231 |
if (start != 0 || count != 1) |
| 232 |
return -EINVAL; |
| 233 |
|
| 234 |
if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
| 235 |
int32_t fd = *(int32_t *)data; |
| 236 |
|
| 237 |
return vfio_set_trigger(vdev, index, fd); |
| 238 |
} |
| 239 |
|
| 240 |
if (flags & VFIO_IRQ_SET_DATA_NONE) { |
| 241 |
handler(irq->hwirq, irq); |
| 242 |
|
| 243 |
} else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
| 244 |
uint8_t trigger = *(uint8_t *)data; |
| 245 |
|
| 246 |
if (trigger) |
| 247 |
handler(irq->hwirq, irq); |
| 248 |
} |
| 249 |
|
| 250 |
return 0; |
| 251 |
} |
| 252 |
|
| 253 |
int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev, |
| 254 |
uint32_t flags, unsigned index, unsigned start, |
| 255 |
unsigned count, void *data) |
| 256 |
{ |
| 257 |
int (*func)(struct vfio_platform_device *vdev, unsigned index, |
| 258 |
unsigned start, unsigned count, uint32_t flags, |
| 259 |
void *data) = NULL; |
| 260 |
|
| 261 |
/* |
| 262 |
* For compatibility, errors from request_irq() are local to the |
| 263 |
* SET_IRQS path and reflected in the name pointer. This allows, |
| 264 |
* for example, polling mode fallback for an exclusive IRQ failure. |
| 265 |
*/ |
| 266 |
if (IS_ERR(vdev->irqs[index].name)) |
| 267 |
return PTR_ERR(vdev->irqs[index].name); |
| 268 |
|
| 269 |
switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { |
| 270 |
case VFIO_IRQ_SET_ACTION_MASK: |
| 271 |
func = vfio_platform_set_irq_mask; |
| 272 |
break; |
| 273 |
case VFIO_IRQ_SET_ACTION_UNMASK: |
| 274 |
func = vfio_platform_set_irq_unmask; |
| 275 |
break; |
| 276 |
case VFIO_IRQ_SET_ACTION_TRIGGER: |
| 277 |
func = vfio_platform_set_irq_trigger; |
| 278 |
break; |
| 279 |
} |
| 280 |
|
| 281 |
if (!func) |
| 282 |
return -ENOTTY; |
| 283 |
|
| 284 |
return func(vdev, index, start, count, flags, data); |
| 285 |
} |
| 286 |
|
| 287 |
int vfio_platform_irq_init(struct vfio_platform_device *vdev) |
| 288 |
{ |
| 289 |
int cnt = 0, i, ret = 0; |
| 290 |
|
| 291 |
while (vdev->get_irq(vdev, cnt) >= 0) |
| 292 |
cnt++; |
| 293 |
|
| 294 |
vdev->irqs = kzalloc_objs(struct vfio_platform_irq, cnt, |
| 295 |
GFP_KERNEL_ACCOUNT); |
| 296 |
if (!vdev->irqs) |
| 297 |
return -ENOMEM; |
| 298 |
|
| 299 |
for (i = 0; i < cnt; i++) { |
| 300 |
int hwirq = vdev->get_irq(vdev, i); |
| 301 |
irq_handler_t handler = vfio_irq_handler; |
| 302 |
|
| 303 |
if (hwirq < 0) { |
| 304 |
ret = -EINVAL; |
| 305 |
goto err; |
| 306 |
} |
| 307 |
|
| 308 |
spin_lock_init(&vdev->irqs[i].lock); |
| 309 |
|
| 310 |
vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD; |
| 311 |
|
| 312 |
if (irq_get_trigger_type(hwirq) & IRQ_TYPE_LEVEL_MASK) { |
| 313 |
vdev->irqs[i].flags |= VFIO_IRQ_INFO_MASKABLE |
| 314 |
| VFIO_IRQ_INFO_AUTOMASKED; |
| 315 |
handler = vfio_automasked_irq_handler; |
| 316 |
} |
| 317 |
|
| 318 |
vdev->irqs[i].count = 1; |
| 319 |
vdev->irqs[i].hwirq = hwirq; |
| 320 |
vdev->irqs[i].masked = false; |
| 321 |
vdev->irqs[i].name = kasprintf(GFP_KERNEL_ACCOUNT, |
| 322 |
"vfio-irq[%d](%s)", hwirq, |
| 323 |
vdev->name); |
| 324 |
if (!vdev->irqs[i].name) { |
| 325 |
ret = -ENOMEM; |
| 326 |
goto err; |
| 327 |
} |
| 328 |
|
| 329 |
ret = request_irq(hwirq, handler, IRQF_NO_AUTOEN, |
| 330 |
vdev->irqs[i].name, &vdev->irqs[i]); |
| 331 |
if (ret) { |
| 332 |
kfree(vdev->irqs[i].name); |
| 333 |
vdev->irqs[i].name = ERR_PTR(ret); |
| 334 |
} |
| 335 |
} |
| 336 |
|
| 337 |
vdev->num_irqs = cnt; |
| 338 |
|
| 339 |
return 0; |
| 340 |
err: |
| 341 |
for (--i; i >= 0; i--) { |
| 342 |
if (!IS_ERR(vdev->irqs[i].name)) { |
| 343 |
free_irq(vdev->irqs[i].hwirq, &vdev->irqs[i]); |
| 344 |
kfree(vdev->irqs[i].name); |
| 345 |
} |
| 346 |
} |
| 347 |
kfree(vdev->irqs); |
| 348 |
return ret; |
| 349 |
} |
| 350 |
|
| 351 |
void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev) |
| 352 |
{ |
| 353 |
int i; |
| 354 |
|
| 355 |
for (i = 0; i < vdev->num_irqs; i++) { |
| 356 |
vfio_virqfd_disable(&vdev->irqs[i].mask); |
| 357 |
vfio_virqfd_disable(&vdev->irqs[i].unmask); |
| 358 |
if (!IS_ERR(vdev->irqs[i].name)) { |
| 359 |
free_irq(vdev->irqs[i].hwirq, &vdev->irqs[i]); |
| 360 |
if (vdev->irqs[i].trigger) |
| 361 |
eventfd_ctx_put(vdev->irqs[i].trigger); |
| 362 |
kfree(vdev->irqs[i].name); |
| 363 |
} |
| 364 |
} |
| 365 |
|
| 366 |
vdev->num_irqs = 0; |
| 367 |
kfree(vdev->irqs); |
| 368 |
} |