| 1 |
// SPDX-License-Identifier: GPL-2.0-or-later |
| 2 |
/* |
| 3 |
* DMA Engine support for Tsi721 PCIExpress-to-SRIO bridge |
| 4 |
* |
| 5 |
* Copyright (c) 2011-2014 Integrated Device Technology, Inc. |
| 6 |
* Alexandre Bounine <[email protected]> |
| 7 |
*/ |
| 8 |
|
| 9 |
#include <linux/io.h> |
| 10 |
#include <linux/errno.h> |
| 11 |
#include <linux/init.h> |
| 12 |
#include <linux/ioport.h> |
| 13 |
#include <linux/kernel.h> |
| 14 |
#include <linux/module.h> |
| 15 |
#include <linux/pci.h> |
| 16 |
#include <linux/rio.h> |
| 17 |
#include <linux/rio_drv.h> |
| 18 |
#include <linux/dma-mapping.h> |
| 19 |
#include <linux/interrupt.h> |
| 20 |
#include <linux/kfifo.h> |
| 21 |
#include <linux/sched.h> |
| 22 |
#include <linux/delay.h> |
| 23 |
#include "../../dma/dmaengine.h" |
| 24 |
|
| 25 |
#include "tsi721.h" |
| 26 |
|
| 27 |
#ifdef CONFIG_PCI_MSI |
| 28 |
static irqreturn_t tsi721_bdma_msix(int irq, void *ptr); |
| 29 |
#endif |
| 30 |
static int tsi721_submit_sg(struct tsi721_tx_desc *desc); |
| 31 |
|
| 32 |
static unsigned int dma_desc_per_channel = 128; |
| 33 |
module_param(dma_desc_per_channel, uint, S_IRUGO); |
| 34 |
MODULE_PARM_DESC(dma_desc_per_channel, |
| 35 |
"Number of DMA descriptors per channel (default: 128)"); |
| 36 |
|
| 37 |
static unsigned int dma_txqueue_sz = 16; |
| 38 |
module_param(dma_txqueue_sz, uint, S_IRUGO); |
| 39 |
MODULE_PARM_DESC(dma_txqueue_sz, |
| 40 |
"DMA Transactions Queue Size (default: 16)"); |
| 41 |
|
| 42 |
static u8 dma_sel = 0x7f; |
| 43 |
module_param(dma_sel, byte, S_IRUGO); |
| 44 |
MODULE_PARM_DESC(dma_sel, |
| 45 |
"DMA Channel Selection Mask (default: 0x7f = all)"); |
| 46 |
|
| 47 |
static inline struct tsi721_bdma_chan *to_tsi721_chan(struct dma_chan *chan) |
| 48 |
{ |
| 49 |
return container_of(chan, struct tsi721_bdma_chan, dchan); |
| 50 |
} |
| 51 |
|
| 52 |
static inline struct tsi721_device *to_tsi721(struct dma_device *ddev) |
| 53 |
{ |
| 54 |
return container_of(ddev, struct rio_mport, dma)->priv; |
| 55 |
} |
| 56 |
|
| 57 |
static inline |
| 58 |
struct tsi721_tx_desc *to_tsi721_desc(struct dma_async_tx_descriptor *txd) |
| 59 |
{ |
| 60 |
return container_of(txd, struct tsi721_tx_desc, txd); |
| 61 |
} |
| 62 |
|
| 63 |
static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num) |
| 64 |
{ |
| 65 |
struct tsi721_dma_desc *bd_ptr; |
| 66 |
struct device *dev = bdma_chan->dchan.device->dev; |
| 67 |
u64 *sts_ptr; |
| 68 |
dma_addr_t bd_phys; |
| 69 |
dma_addr_t sts_phys; |
| 70 |
int sts_size; |
| 71 |
#ifdef CONFIG_PCI_MSI |
| 72 |
struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); |
| 73 |
#endif |
| 74 |
|
| 75 |
tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id); |
| 76 |
|
| 77 |
/* |
| 78 |
* Allocate space for DMA descriptors |
| 79 |
* (add an extra element for link descriptor) |
| 80 |
*/ |
| 81 |
bd_ptr = dma_alloc_coherent(dev, |
| 82 |
(bd_num + 1) * sizeof(struct tsi721_dma_desc), |
| 83 |
&bd_phys, GFP_ATOMIC); |
| 84 |
if (!bd_ptr) |
| 85 |
return -ENOMEM; |
| 86 |
|
| 87 |
bdma_chan->bd_num = bd_num; |
| 88 |
bdma_chan->bd_phys = bd_phys; |
| 89 |
bdma_chan->bd_base = bd_ptr; |
| 90 |
|
| 91 |
tsi_debug(DMA, &bdma_chan->dchan.dev->device, |
| 92 |
"DMAC%d descriptors @ %p (phys = %pad)", |
| 93 |
bdma_chan->id, bd_ptr, &bd_phys); |
| 94 |
|
| 95 |
/* Allocate space for descriptor status FIFO */ |
| 96 |
sts_size = ((bd_num + 1) >= TSI721_DMA_MINSTSSZ) ? |
| 97 |
(bd_num + 1) : TSI721_DMA_MINSTSSZ; |
| 98 |
sts_size = roundup_pow_of_two(sts_size); |
| 99 |
sts_ptr = dma_alloc_coherent(dev, |
| 100 |
sts_size * sizeof(struct tsi721_dma_sts), |
| 101 |
&sts_phys, GFP_ATOMIC); |
| 102 |
if (!sts_ptr) { |
| 103 |
/* Free space allocated for DMA descriptors */ |
| 104 |
dma_free_coherent(dev, |
| 105 |
(bd_num + 1) * sizeof(struct tsi721_dma_desc), |
| 106 |
bd_ptr, bd_phys); |
| 107 |
bdma_chan->bd_base = NULL; |
| 108 |
return -ENOMEM; |
| 109 |
} |
| 110 |
|
| 111 |
bdma_chan->sts_phys = sts_phys; |
| 112 |
bdma_chan->sts_base = sts_ptr; |
| 113 |
bdma_chan->sts_size = sts_size; |
| 114 |
|
| 115 |
tsi_debug(DMA, &bdma_chan->dchan.dev->device, |
| 116 |
"DMAC%d desc status FIFO @ %p (phys = %pad) size=0x%x", |
| 117 |
bdma_chan->id, sts_ptr, &sts_phys, sts_size); |
| 118 |
|
| 119 |
/* Initialize DMA descriptors ring using added link descriptor */ |
| 120 |
bd_ptr[bd_num].type_id = cpu_to_le32(DTYPE3 << 29); |
| 121 |
bd_ptr[bd_num].next_lo = cpu_to_le32((u64)bd_phys & |
| 122 |
TSI721_DMAC_DPTRL_MASK); |
| 123 |
bd_ptr[bd_num].next_hi = cpu_to_le32((u64)bd_phys >> 32); |
| 124 |
|
| 125 |
/* Setup DMA descriptor pointers */ |
| 126 |
iowrite32(((u64)bd_phys >> 32), |
| 127 |
bdma_chan->regs + TSI721_DMAC_DPTRH); |
| 128 |
iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK), |
| 129 |
bdma_chan->regs + TSI721_DMAC_DPTRL); |
| 130 |
|
| 131 |
/* Setup descriptor status FIFO */ |
| 132 |
iowrite32(((u64)sts_phys >> 32), |
| 133 |
bdma_chan->regs + TSI721_DMAC_DSBH); |
| 134 |
iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK), |
| 135 |
bdma_chan->regs + TSI721_DMAC_DSBL); |
| 136 |
iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size), |
| 137 |
bdma_chan->regs + TSI721_DMAC_DSSZ); |
| 138 |
|
| 139 |
/* Clear interrupt bits */ |
| 140 |
iowrite32(TSI721_DMAC_INT_ALL, |
| 141 |
bdma_chan->regs + TSI721_DMAC_INT); |
| 142 |
|
| 143 |
ioread32(bdma_chan->regs + TSI721_DMAC_INT); |
| 144 |
|
| 145 |
#ifdef CONFIG_PCI_MSI |
| 146 |
/* Request interrupt service if we are in MSI-X mode */ |
| 147 |
if (priv->flags & TSI721_USING_MSIX) { |
| 148 |
int rc, idx; |
| 149 |
|
| 150 |
idx = TSI721_VECT_DMA0_DONE + bdma_chan->id; |
| 151 |
|
| 152 |
rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0, |
| 153 |
priv->msix[idx].irq_name, (void *)bdma_chan); |
| 154 |
|
| 155 |
if (rc) { |
| 156 |
tsi_debug(DMA, &bdma_chan->dchan.dev->device, |
| 157 |
"Unable to get MSI-X for DMAC%d-DONE", |
| 158 |
bdma_chan->id); |
| 159 |
goto err_out; |
| 160 |
} |
| 161 |
|
| 162 |
idx = TSI721_VECT_DMA0_INT + bdma_chan->id; |
| 163 |
|
| 164 |
rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0, |
| 165 |
priv->msix[idx].irq_name, (void *)bdma_chan); |
| 166 |
|
| 167 |
if (rc) { |
| 168 |
tsi_debug(DMA, &bdma_chan->dchan.dev->device, |
| 169 |
"Unable to get MSI-X for DMAC%d-INT", |
| 170 |
bdma_chan->id); |
| 171 |
free_irq( |
| 172 |
priv->msix[TSI721_VECT_DMA0_DONE + |
| 173 |
bdma_chan->id].vector, |
| 174 |
(void *)bdma_chan); |
| 175 |
} |
| 176 |
|
| 177 |
err_out: |
| 178 |
if (rc) { |
| 179 |
/* Free space allocated for DMA descriptors */ |
| 180 |
dma_free_coherent(dev, |
| 181 |
(bd_num + 1) * sizeof(struct tsi721_dma_desc), |
| 182 |
bd_ptr, bd_phys); |
| 183 |
bdma_chan->bd_base = NULL; |
| 184 |
|
| 185 |
/* Free space allocated for status descriptors */ |
| 186 |
dma_free_coherent(dev, |
| 187 |
sts_size * sizeof(struct tsi721_dma_sts), |
| 188 |
sts_ptr, sts_phys); |
| 189 |
bdma_chan->sts_base = NULL; |
| 190 |
|
| 191 |
return -EIO; |
| 192 |
} |
| 193 |
} |
| 194 |
#endif /* CONFIG_PCI_MSI */ |
| 195 |
|
| 196 |
/* Toggle DMA channel initialization */ |
| 197 |
iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL); |
| 198 |
ioread32(bdma_chan->regs + TSI721_DMAC_CTL); |
| 199 |
bdma_chan->wr_count = bdma_chan->wr_count_next = 0; |
| 200 |
bdma_chan->sts_rdptr = 0; |
| 201 |
udelay(10); |
| 202 |
|
| 203 |
return 0; |
| 204 |
} |
| 205 |
|
| 206 |
static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan) |
| 207 |
{ |
| 208 |
u32 ch_stat; |
| 209 |
#ifdef CONFIG_PCI_MSI |
| 210 |
struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); |
| 211 |
#endif |
| 212 |
|
| 213 |
if (!bdma_chan->bd_base) |
| 214 |
return 0; |
| 215 |
|
| 216 |
/* Check if DMA channel still running */ |
| 217 |
ch_stat = ioread32(bdma_chan->regs + TSI721_DMAC_STS); |
| 218 |
if (ch_stat & TSI721_DMAC_STS_RUN) |
| 219 |
return -EFAULT; |
| 220 |
|
| 221 |
/* Put DMA channel into init state */ |
| 222 |
iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL); |
| 223 |
|
| 224 |
#ifdef CONFIG_PCI_MSI |
| 225 |
if (priv->flags & TSI721_USING_MSIX) { |
| 226 |
free_irq(priv->msix[TSI721_VECT_DMA0_DONE + |
| 227 |
bdma_chan->id].vector, (void *)bdma_chan); |
| 228 |
free_irq(priv->msix[TSI721_VECT_DMA0_INT + |
| 229 |
bdma_chan->id].vector, (void *)bdma_chan); |
| 230 |
} |
| 231 |
#endif /* CONFIG_PCI_MSI */ |
| 232 |
|
| 233 |
/* Free space allocated for DMA descriptors */ |
| 234 |
dma_free_coherent(bdma_chan->dchan.device->dev, |
| 235 |
(bdma_chan->bd_num + 1) * sizeof(struct tsi721_dma_desc), |
| 236 |
bdma_chan->bd_base, bdma_chan->bd_phys); |
| 237 |
bdma_chan->bd_base = NULL; |
| 238 |
|
| 239 |
/* Free space allocated for status FIFO */ |
| 240 |
dma_free_coherent(bdma_chan->dchan.device->dev, |
| 241 |
bdma_chan->sts_size * sizeof(struct tsi721_dma_sts), |
| 242 |
bdma_chan->sts_base, bdma_chan->sts_phys); |
| 243 |
bdma_chan->sts_base = NULL; |
| 244 |
return 0; |
| 245 |
} |
| 246 |
|
| 247 |
static void |
| 248 |
tsi721_bdma_interrupt_enable(struct tsi721_bdma_chan *bdma_chan, int enable) |
| 249 |
{ |
| 250 |
if (enable) { |
| 251 |
/* Clear pending BDMA channel interrupts */ |
| 252 |
iowrite32(TSI721_DMAC_INT_ALL, |
| 253 |
bdma_chan->regs + TSI721_DMAC_INT); |
| 254 |
ioread32(bdma_chan->regs + TSI721_DMAC_INT); |
| 255 |
/* Enable BDMA channel interrupts */ |
| 256 |
iowrite32(TSI721_DMAC_INT_ALL, |
| 257 |
bdma_chan->regs + TSI721_DMAC_INTE); |
| 258 |
} else { |
| 259 |
/* Disable BDMA channel interrupts */ |
| 260 |
iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE); |
| 261 |
/* Clear pending BDMA channel interrupts */ |
| 262 |
iowrite32(TSI721_DMAC_INT_ALL, |
| 263 |
bdma_chan->regs + TSI721_DMAC_INT); |
| 264 |
} |
| 265 |
|
| 266 |
} |
| 267 |
|
| 268 |
static bool tsi721_dma_is_idle(struct tsi721_bdma_chan *bdma_chan) |
| 269 |
{ |
| 270 |
u32 sts; |
| 271 |
|
| 272 |
sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); |
| 273 |
return ((sts & TSI721_DMAC_STS_RUN) == 0); |
| 274 |
} |
| 275 |
|
| 276 |
void tsi721_bdma_handler(struct tsi721_bdma_chan *bdma_chan) |
| 277 |
{ |
| 278 |
/* Disable BDMA channel interrupts */ |
| 279 |
iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE); |
| 280 |
if (bdma_chan->active) |
| 281 |
tasklet_hi_schedule(&bdma_chan->tasklet); |
| 282 |
} |
| 283 |
|
| 284 |
#ifdef CONFIG_PCI_MSI |
| 285 |
/** |
| 286 |
* tsi721_bdma_msix - MSI-X interrupt handler for BDMA channels |
| 287 |
* @irq: Linux interrupt number |
| 288 |
* @ptr: Pointer to interrupt-specific data (BDMA channel structure) |
| 289 |
* |
| 290 |
* Handles BDMA channel interrupts signaled using MSI-X. |
| 291 |
* |
| 292 |
* Returns: %IRQ_HANDLED |
| 293 |
*/ |
| 294 |
static irqreturn_t tsi721_bdma_msix(int irq, void *ptr) |
| 295 |
{ |
| 296 |
struct tsi721_bdma_chan *bdma_chan = ptr; |
| 297 |
|
| 298 |
if (bdma_chan->active) |
| 299 |
tasklet_hi_schedule(&bdma_chan->tasklet); |
| 300 |
return IRQ_HANDLED; |
| 301 |
} |
| 302 |
#endif /* CONFIG_PCI_MSI */ |
| 303 |
|
| 304 |
/* Must be called with the spinlock held */ |
| 305 |
static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan) |
| 306 |
{ |
| 307 |
if (!tsi721_dma_is_idle(bdma_chan)) { |
| 308 |
tsi_err(&bdma_chan->dchan.dev->device, |
| 309 |
"DMAC%d Attempt to start non-idle channel", |
| 310 |
bdma_chan->id); |
| 311 |
return; |
| 312 |
} |
| 313 |
|
| 314 |
if (bdma_chan->wr_count == bdma_chan->wr_count_next) { |
| 315 |
tsi_err(&bdma_chan->dchan.dev->device, |
| 316 |
"DMAC%d Attempt to start DMA with no BDs ready %d", |
| 317 |
bdma_chan->id, task_pid_nr(current)); |
| 318 |
return; |
| 319 |
} |
| 320 |
|
| 321 |
tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d (wrc=%d) %d", |
| 322 |
bdma_chan->id, bdma_chan->wr_count_next, |
| 323 |
task_pid_nr(current)); |
| 324 |
|
| 325 |
iowrite32(bdma_chan->wr_count_next, |
| 326 |
bdma_chan->regs + TSI721_DMAC_DWRCNT); |
| 327 |
ioread32(bdma_chan->regs + TSI721_DMAC_DWRCNT); |
| 328 |
|
| 329 |
bdma_chan->wr_count = bdma_chan->wr_count_next; |
| 330 |
} |
| 331 |
|
| 332 |
static int |
| 333 |
tsi721_desc_fill_init(struct tsi721_tx_desc *desc, |
| 334 |
struct tsi721_dma_desc *bd_ptr, |
| 335 |
struct scatterlist *sg, u32 sys_size) |
| 336 |
{ |
| 337 |
u64 rio_addr; |
| 338 |
|
| 339 |
if (!bd_ptr) |
| 340 |
return -EINVAL; |
| 341 |
|
| 342 |
/* Initialize DMA descriptor */ |
| 343 |
bd_ptr->type_id = cpu_to_le32((DTYPE1 << 29) | |
| 344 |
(desc->rtype << 19) | desc->destid); |
| 345 |
bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) | |
| 346 |
(sys_size << 26)); |
| 347 |
rio_addr = (desc->rio_addr >> 2) | |
| 348 |
((u64)(desc->rio_addr_u & 0x3) << 62); |
| 349 |
bd_ptr->raddr_lo = cpu_to_le32(rio_addr & 0xffffffff); |
| 350 |
bd_ptr->raddr_hi = cpu_to_le32(rio_addr >> 32); |
| 351 |
bd_ptr->t1.bufptr_lo = cpu_to_le32( |
| 352 |
(u64)sg_dma_address(sg) & 0xffffffff); |
| 353 |
bd_ptr->t1.bufptr_hi = cpu_to_le32((u64)sg_dma_address(sg) >> 32); |
| 354 |
bd_ptr->t1.s_dist = 0; |
| 355 |
bd_ptr->t1.s_size = 0; |
| 356 |
|
| 357 |
return 0; |
| 358 |
} |
| 359 |
|
| 360 |
static int |
| 361 |
tsi721_desc_fill_end(struct tsi721_dma_desc *bd_ptr, u32 bcount, bool interrupt) |
| 362 |
{ |
| 363 |
if (!bd_ptr) |
| 364 |
return -EINVAL; |
| 365 |
|
| 366 |
/* Update DMA descriptor */ |
| 367 |
if (interrupt) |
| 368 |
bd_ptr->type_id |= cpu_to_le32(TSI721_DMAD_IOF); |
| 369 |
bd_ptr->bcount |= cpu_to_le32(bcount & TSI721_DMAD_BCOUNT1); |
| 370 |
|
| 371 |
return 0; |
| 372 |
} |
| 373 |
|
| 374 |
static void tsi721_dma_tx_err(struct tsi721_bdma_chan *bdma_chan, |
| 375 |
struct tsi721_tx_desc *desc) |
| 376 |
{ |
| 377 |
struct dma_async_tx_descriptor *txd = &desc->txd; |
| 378 |
dma_async_tx_callback callback = txd->callback; |
| 379 |
void *param = txd->callback_param; |
| 380 |
|
| 381 |
list_move(&desc->desc_node, &bdma_chan->free_list); |
| 382 |
|
| 383 |
if (callback) |
| 384 |
callback(param); |
| 385 |
} |
| 386 |
|
| 387 |
static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan) |
| 388 |
{ |
| 389 |
u32 srd_ptr; |
| 390 |
u64 *sts_ptr; |
| 391 |
int i, j; |
| 392 |
|
| 393 |
/* Check and clear descriptor status FIFO entries */ |
| 394 |
srd_ptr = bdma_chan->sts_rdptr; |
| 395 |
sts_ptr = bdma_chan->sts_base; |
| 396 |
j = srd_ptr * 8; |
| 397 |
while (sts_ptr[j]) { |
| 398 |
for (i = 0; i < 8 && sts_ptr[j]; i++, j++) |
| 399 |
sts_ptr[j] = 0; |
| 400 |
|
| 401 |
++srd_ptr; |
| 402 |
srd_ptr %= bdma_chan->sts_size; |
| 403 |
j = srd_ptr * 8; |
| 404 |
} |
| 405 |
|
| 406 |
iowrite32(srd_ptr, bdma_chan->regs + TSI721_DMAC_DSRP); |
| 407 |
bdma_chan->sts_rdptr = srd_ptr; |
| 408 |
} |
| 409 |
|
| 410 |
/* Must be called with the channel spinlock held */ |
| 411 |
static int tsi721_submit_sg(struct tsi721_tx_desc *desc) |
| 412 |
{ |
| 413 |
struct dma_chan *dchan = desc->txd.chan; |
| 414 |
struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); |
| 415 |
u32 sys_size; |
| 416 |
u64 rio_addr; |
| 417 |
dma_addr_t next_addr; |
| 418 |
u32 bcount; |
| 419 |
struct scatterlist *sg; |
| 420 |
unsigned int i; |
| 421 |
int err = 0; |
| 422 |
struct tsi721_dma_desc *bd_ptr = NULL; |
| 423 |
u32 idx, rd_idx; |
| 424 |
u32 add_count = 0; |
| 425 |
struct device *ch_dev = &dchan->dev->device; |
| 426 |
|
| 427 |
if (!tsi721_dma_is_idle(bdma_chan)) { |
| 428 |
tsi_err(ch_dev, "DMAC%d ERR: Attempt to use non-idle channel", |
| 429 |
bdma_chan->id); |
| 430 |
return -EIO; |
| 431 |
} |
| 432 |
|
| 433 |
/* |
| 434 |
* Fill DMA channel's hardware buffer descriptors. |
| 435 |
* (NOTE: RapidIO destination address is limited to 64 bits for now) |
| 436 |
*/ |
| 437 |
rio_addr = desc->rio_addr; |
| 438 |
next_addr = -1; |
| 439 |
bcount = 0; |
| 440 |
sys_size = dma_to_mport(dchan->device)->sys_size; |
| 441 |
|
| 442 |
rd_idx = ioread32(bdma_chan->regs + TSI721_DMAC_DRDCNT); |
| 443 |
rd_idx %= (bdma_chan->bd_num + 1); |
| 444 |
|
| 445 |
idx = bdma_chan->wr_count_next % (bdma_chan->bd_num + 1); |
| 446 |
if (idx == bdma_chan->bd_num) { |
| 447 |
/* wrap around link descriptor */ |
| 448 |
idx = 0; |
| 449 |
add_count++; |
| 450 |
} |
| 451 |
|
| 452 |
tsi_debug(DMA, ch_dev, "DMAC%d BD ring status: rdi=%d wri=%d", |
| 453 |
bdma_chan->id, rd_idx, idx); |
| 454 |
|
| 455 |
for_each_sg(desc->sg, sg, desc->sg_len, i) { |
| 456 |
|
| 457 |
tsi_debug(DMAV, ch_dev, "DMAC%d sg%d/%d addr: 0x%llx len: %d", |
| 458 |
bdma_chan->id, i, desc->sg_len, |
| 459 |
(unsigned long long)sg_dma_address(sg), sg_dma_len(sg)); |
| 460 |
|
| 461 |
if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) { |
| 462 |
tsi_err(ch_dev, "DMAC%d SG entry %d is too large", |
| 463 |
bdma_chan->id, i); |
| 464 |
err = -EINVAL; |
| 465 |
break; |
| 466 |
} |
| 467 |
|
| 468 |
/* |
| 469 |
* If this sg entry forms contiguous block with previous one, |
| 470 |
* try to merge it into existing DMA descriptor |
| 471 |
*/ |
| 472 |
if (next_addr == sg_dma_address(sg) && |
| 473 |
bcount + sg_dma_len(sg) <= TSI721_BDMA_MAX_BCOUNT) { |
| 474 |
/* Adjust byte count of the descriptor */ |
| 475 |
bcount += sg_dma_len(sg); |
| 476 |
goto entry_done; |
| 477 |
} else if (next_addr != -1) { |
| 478 |
/* Finalize descriptor using total byte count value */ |
| 479 |
tsi721_desc_fill_end(bd_ptr, bcount, 0); |
| 480 |
tsi_debug(DMAV, ch_dev, "DMAC%d prev desc final len: %d", |
| 481 |
bdma_chan->id, bcount); |
| 482 |
} |
| 483 |
|
| 484 |
desc->rio_addr = rio_addr; |
| 485 |
|
| 486 |
if (i && idx == rd_idx) { |
| 487 |
tsi_debug(DMAV, ch_dev, |
| 488 |
"DMAC%d HW descriptor ring is full @ %d", |
| 489 |
bdma_chan->id, i); |
| 490 |
desc->sg = sg; |
| 491 |
desc->sg_len -= i; |
| 492 |
break; |
| 493 |
} |
| 494 |
|
| 495 |
bd_ptr = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[idx]; |
| 496 |
err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size); |
| 497 |
if (err) { |
| 498 |
tsi_err(ch_dev, "Failed to build desc: err=%d", err); |
| 499 |
break; |
| 500 |
} |
| 501 |
|
| 502 |
tsi_debug(DMAV, ch_dev, "DMAC%d bd_ptr = %p did=%d raddr=0x%llx", |
| 503 |
bdma_chan->id, bd_ptr, desc->destid, desc->rio_addr); |
| 504 |
|
| 505 |
next_addr = sg_dma_address(sg); |
| 506 |
bcount = sg_dma_len(sg); |
| 507 |
|
| 508 |
add_count++; |
| 509 |
if (++idx == bdma_chan->bd_num) { |
| 510 |
/* wrap around link descriptor */ |
| 511 |
idx = 0; |
| 512 |
add_count++; |
| 513 |
} |
| 514 |
|
| 515 |
entry_done: |
| 516 |
if (sg_is_last(sg)) { |
| 517 |
tsi721_desc_fill_end(bd_ptr, bcount, 0); |
| 518 |
tsi_debug(DMAV, ch_dev, |
| 519 |
"DMAC%d last desc final len: %d", |
| 520 |
bdma_chan->id, bcount); |
| 521 |
desc->sg_len = 0; |
| 522 |
} else { |
| 523 |
rio_addr += sg_dma_len(sg); |
| 524 |
next_addr += sg_dma_len(sg); |
| 525 |
} |
| 526 |
} |
| 527 |
|
| 528 |
if (!err) |
| 529 |
bdma_chan->wr_count_next += add_count; |
| 530 |
|
| 531 |
return err; |
| 532 |
} |
| 533 |
|
| 534 |
static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan, |
| 535 |
struct tsi721_tx_desc *desc) |
| 536 |
{ |
| 537 |
int err; |
| 538 |
|
| 539 |
tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id); |
| 540 |
|
| 541 |
if (!tsi721_dma_is_idle(bdma_chan)) |
| 542 |
return; |
| 543 |
|
| 544 |
/* |
| 545 |
* If there is no data transfer in progress, fetch new descriptor from |
| 546 |
* the pending queue. |
| 547 |
*/ |
| 548 |
if (!desc && !bdma_chan->active_tx && !list_empty(&bdma_chan->queue)) { |
| 549 |
desc = list_first_entry(&bdma_chan->queue, |
| 550 |
struct tsi721_tx_desc, desc_node); |
| 551 |
list_del_init((&desc->desc_node)); |
| 552 |
bdma_chan->active_tx = desc; |
| 553 |
} |
| 554 |
|
| 555 |
if (desc) { |
| 556 |
err = tsi721_submit_sg(desc); |
| 557 |
if (!err) |
| 558 |
tsi721_start_dma(bdma_chan); |
| 559 |
else { |
| 560 |
tsi721_dma_tx_err(bdma_chan, desc); |
| 561 |
tsi_debug(DMA, &bdma_chan->dchan.dev->device, |
| 562 |
"DMAC%d ERR: tsi721_submit_sg failed with err=%d", |
| 563 |
bdma_chan->id, err); |
| 564 |
} |
| 565 |
} |
| 566 |
|
| 567 |
tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d Exit", |
| 568 |
bdma_chan->id); |
| 569 |
} |
| 570 |
|
| 571 |
static void tsi721_dma_tasklet(unsigned long data) |
| 572 |
{ |
| 573 |
struct tsi721_bdma_chan *bdma_chan = (struct tsi721_bdma_chan *)data; |
| 574 |
u32 dmac_int, dmac_sts; |
| 575 |
|
| 576 |
dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT); |
| 577 |
tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d_INT = 0x%x", |
| 578 |
bdma_chan->id, dmac_int); |
| 579 |
/* Clear channel interrupts */ |
| 580 |
iowrite32(dmac_int, bdma_chan->regs + TSI721_DMAC_INT); |
| 581 |
|
| 582 |
if (dmac_int & TSI721_DMAC_INT_ERR) { |
| 583 |
int i = 10000; |
| 584 |
struct tsi721_tx_desc *desc; |
| 585 |
|
| 586 |
desc = bdma_chan->active_tx; |
| 587 |
dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); |
| 588 |
tsi_err(&bdma_chan->dchan.dev->device, |
| 589 |
"DMAC%d_STS = 0x%x did=%d raddr=0x%llx", |
| 590 |
bdma_chan->id, dmac_sts, desc->destid, desc->rio_addr); |
| 591 |
|
| 592 |
/* Re-initialize DMA channel if possible */ |
| 593 |
|
| 594 |
if ((dmac_sts & TSI721_DMAC_STS_ABORT) == 0) |
| 595 |
goto err_out; |
| 596 |
|
| 597 |
tsi721_clr_stat(bdma_chan); |
| 598 |
|
| 599 |
spin_lock(&bdma_chan->lock); |
| 600 |
|
| 601 |
/* Put DMA channel into init state */ |
| 602 |
iowrite32(TSI721_DMAC_CTL_INIT, |
| 603 |
bdma_chan->regs + TSI721_DMAC_CTL); |
| 604 |
do { |
| 605 |
udelay(1); |
| 606 |
dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); |
| 607 |
i--; |
| 608 |
} while ((dmac_sts & TSI721_DMAC_STS_ABORT) && i); |
| 609 |
|
| 610 |
if (dmac_sts & TSI721_DMAC_STS_ABORT) { |
| 611 |
tsi_err(&bdma_chan->dchan.dev->device, |
| 612 |
"Failed to re-initiate DMAC%d", bdma_chan->id); |
| 613 |
spin_unlock(&bdma_chan->lock); |
| 614 |
goto err_out; |
| 615 |
} |
| 616 |
|
| 617 |
/* Setup DMA descriptor pointers */ |
| 618 |
iowrite32(((u64)bdma_chan->bd_phys >> 32), |
| 619 |
bdma_chan->regs + TSI721_DMAC_DPTRH); |
| 620 |
iowrite32(((u64)bdma_chan->bd_phys & TSI721_DMAC_DPTRL_MASK), |
| 621 |
bdma_chan->regs + TSI721_DMAC_DPTRL); |
| 622 |
|
| 623 |
/* Setup descriptor status FIFO */ |
| 624 |
iowrite32(((u64)bdma_chan->sts_phys >> 32), |
| 625 |
bdma_chan->regs + TSI721_DMAC_DSBH); |
| 626 |
iowrite32(((u64)bdma_chan->sts_phys & TSI721_DMAC_DSBL_MASK), |
| 627 |
bdma_chan->regs + TSI721_DMAC_DSBL); |
| 628 |
iowrite32(TSI721_DMAC_DSSZ_SIZE(bdma_chan->sts_size), |
| 629 |
bdma_chan->regs + TSI721_DMAC_DSSZ); |
| 630 |
|
| 631 |
/* Clear interrupt bits */ |
| 632 |
iowrite32(TSI721_DMAC_INT_ALL, |
| 633 |
bdma_chan->regs + TSI721_DMAC_INT); |
| 634 |
|
| 635 |
ioread32(bdma_chan->regs + TSI721_DMAC_INT); |
| 636 |
|
| 637 |
bdma_chan->wr_count = bdma_chan->wr_count_next = 0; |
| 638 |
bdma_chan->sts_rdptr = 0; |
| 639 |
udelay(10); |
| 640 |
|
| 641 |
desc = bdma_chan->active_tx; |
| 642 |
desc->status = DMA_ERROR; |
| 643 |
dma_cookie_complete(&desc->txd); |
| 644 |
list_add(&desc->desc_node, &bdma_chan->free_list); |
| 645 |
bdma_chan->active_tx = NULL; |
| 646 |
if (bdma_chan->active) |
| 647 |
tsi721_advance_work(bdma_chan, NULL); |
| 648 |
spin_unlock(&bdma_chan->lock); |
| 649 |
} |
| 650 |
|
| 651 |
if (dmac_int & TSI721_DMAC_INT_STFULL) { |
| 652 |
tsi_err(&bdma_chan->dchan.dev->device, |
| 653 |
"DMAC%d descriptor status FIFO is full", |
| 654 |
bdma_chan->id); |
| 655 |
} |
| 656 |
|
| 657 |
if (dmac_int & (TSI721_DMAC_INT_DONE | TSI721_DMAC_INT_IOFDONE)) { |
| 658 |
struct tsi721_tx_desc *desc; |
| 659 |
|
| 660 |
tsi721_clr_stat(bdma_chan); |
| 661 |
spin_lock(&bdma_chan->lock); |
| 662 |
desc = bdma_chan->active_tx; |
| 663 |
|
| 664 |
if (desc->sg_len == 0) { |
| 665 |
dma_async_tx_callback callback = NULL; |
| 666 |
void *param = NULL; |
| 667 |
|
| 668 |
desc->status = DMA_COMPLETE; |
| 669 |
dma_cookie_complete(&desc->txd); |
| 670 |
if (desc->txd.flags & DMA_PREP_INTERRUPT) { |
| 671 |
callback = desc->txd.callback; |
| 672 |
param = desc->txd.callback_param; |
| 673 |
} |
| 674 |
list_add(&desc->desc_node, &bdma_chan->free_list); |
| 675 |
bdma_chan->active_tx = NULL; |
| 676 |
if (bdma_chan->active) |
| 677 |
tsi721_advance_work(bdma_chan, NULL); |
| 678 |
spin_unlock(&bdma_chan->lock); |
| 679 |
if (callback) |
| 680 |
callback(param); |
| 681 |
} else { |
| 682 |
if (bdma_chan->active) |
| 683 |
tsi721_advance_work(bdma_chan, |
| 684 |
bdma_chan->active_tx); |
| 685 |
spin_unlock(&bdma_chan->lock); |
| 686 |
} |
| 687 |
} |
| 688 |
err_out: |
| 689 |
/* Re-Enable BDMA channel interrupts */ |
| 690 |
iowrite32(TSI721_DMAC_INT_ALL, bdma_chan->regs + TSI721_DMAC_INTE); |
| 691 |
} |
| 692 |
|
| 693 |
static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd) |
| 694 |
{ |
| 695 |
struct tsi721_tx_desc *desc = to_tsi721_desc(txd); |
| 696 |
struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(txd->chan); |
| 697 |
dma_cookie_t cookie; |
| 698 |
|
| 699 |
/* Check if the descriptor is detached from any lists */ |
| 700 |
if (!list_empty(&desc->desc_node)) { |
| 701 |
tsi_err(&bdma_chan->dchan.dev->device, |
| 702 |
"DMAC%d wrong state of descriptor %p", |
| 703 |
bdma_chan->id, txd); |
| 704 |
return -EIO; |
| 705 |
} |
| 706 |
|
| 707 |
spin_lock_bh(&bdma_chan->lock); |
| 708 |
|
| 709 |
if (!bdma_chan->active) { |
| 710 |
spin_unlock_bh(&bdma_chan->lock); |
| 711 |
return -ENODEV; |
| 712 |
} |
| 713 |
|
| 714 |
cookie = dma_cookie_assign(txd); |
| 715 |
desc->status = DMA_IN_PROGRESS; |
| 716 |
list_add_tail(&desc->desc_node, &bdma_chan->queue); |
| 717 |
tsi721_advance_work(bdma_chan, NULL); |
| 718 |
|
| 719 |
spin_unlock_bh(&bdma_chan->lock); |
| 720 |
return cookie; |
| 721 |
} |
| 722 |
|
| 723 |
static int tsi721_alloc_chan_resources(struct dma_chan *dchan) |
| 724 |
{ |
| 725 |
struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); |
| 726 |
struct tsi721_tx_desc *desc; |
| 727 |
int i; |
| 728 |
|
| 729 |
tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); |
| 730 |
|
| 731 |
if (bdma_chan->bd_base) |
| 732 |
return dma_txqueue_sz; |
| 733 |
|
| 734 |
/* Initialize BDMA channel */ |
| 735 |
if (tsi721_bdma_ch_init(bdma_chan, dma_desc_per_channel)) { |
| 736 |
tsi_err(&dchan->dev->device, "Unable to initialize DMAC%d", |
| 737 |
bdma_chan->id); |
| 738 |
return -ENODEV; |
| 739 |
} |
| 740 |
|
| 741 |
/* Allocate queue of transaction descriptors */ |
| 742 |
desc = kzalloc_objs(struct tsi721_tx_desc, dma_txqueue_sz, GFP_ATOMIC); |
| 743 |
if (!desc) { |
| 744 |
tsi721_bdma_ch_free(bdma_chan); |
| 745 |
return -ENOMEM; |
| 746 |
} |
| 747 |
|
| 748 |
bdma_chan->tx_desc = desc; |
| 749 |
|
| 750 |
for (i = 0; i < dma_txqueue_sz; i++) { |
| 751 |
dma_async_tx_descriptor_init(&desc[i].txd, dchan); |
| 752 |
desc[i].txd.tx_submit = tsi721_tx_submit; |
| 753 |
desc[i].txd.flags = DMA_CTRL_ACK; |
| 754 |
list_add(&desc[i].desc_node, &bdma_chan->free_list); |
| 755 |
} |
| 756 |
|
| 757 |
dma_cookie_init(dchan); |
| 758 |
|
| 759 |
bdma_chan->active = true; |
| 760 |
tsi721_bdma_interrupt_enable(bdma_chan, 1); |
| 761 |
|
| 762 |
return dma_txqueue_sz; |
| 763 |
} |
| 764 |
|
| 765 |
static void tsi721_sync_dma_irq(struct tsi721_bdma_chan *bdma_chan) |
| 766 |
{ |
| 767 |
struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); |
| 768 |
|
| 769 |
#ifdef CONFIG_PCI_MSI |
| 770 |
if (priv->flags & TSI721_USING_MSIX) { |
| 771 |
synchronize_irq(priv->msix[TSI721_VECT_DMA0_DONE + |
| 772 |
bdma_chan->id].vector); |
| 773 |
synchronize_irq(priv->msix[TSI721_VECT_DMA0_INT + |
| 774 |
bdma_chan->id].vector); |
| 775 |
} else |
| 776 |
#endif |
| 777 |
synchronize_irq(priv->pdev->irq); |
| 778 |
} |
| 779 |
|
| 780 |
static void tsi721_free_chan_resources(struct dma_chan *dchan) |
| 781 |
{ |
| 782 |
struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); |
| 783 |
|
| 784 |
tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); |
| 785 |
|
| 786 |
if (!bdma_chan->bd_base) |
| 787 |
return; |
| 788 |
|
| 789 |
tsi721_bdma_interrupt_enable(bdma_chan, 0); |
| 790 |
bdma_chan->active = false; |
| 791 |
tsi721_sync_dma_irq(bdma_chan); |
| 792 |
tasklet_kill(&bdma_chan->tasklet); |
| 793 |
INIT_LIST_HEAD(&bdma_chan->free_list); |
| 794 |
kfree(bdma_chan->tx_desc); |
| 795 |
tsi721_bdma_ch_free(bdma_chan); |
| 796 |
} |
| 797 |
|
| 798 |
static |
| 799 |
enum dma_status tsi721_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, |
| 800 |
struct dma_tx_state *txstate) |
| 801 |
{ |
| 802 |
struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); |
| 803 |
enum dma_status status; |
| 804 |
|
| 805 |
spin_lock_bh(&bdma_chan->lock); |
| 806 |
status = dma_cookie_status(dchan, cookie, txstate); |
| 807 |
spin_unlock_bh(&bdma_chan->lock); |
| 808 |
return status; |
| 809 |
} |
| 810 |
|
| 811 |
static void tsi721_issue_pending(struct dma_chan *dchan) |
| 812 |
{ |
| 813 |
struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); |
| 814 |
|
| 815 |
tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); |
| 816 |
|
| 817 |
spin_lock_bh(&bdma_chan->lock); |
| 818 |
if (tsi721_dma_is_idle(bdma_chan) && bdma_chan->active) { |
| 819 |
tsi721_advance_work(bdma_chan, NULL); |
| 820 |
} |
| 821 |
spin_unlock_bh(&bdma_chan->lock); |
| 822 |
} |
| 823 |
|
| 824 |
static |
| 825 |
struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan, |
| 826 |
struct scatterlist *sgl, unsigned int sg_len, |
| 827 |
enum dma_transfer_direction dir, unsigned long flags, |
| 828 |
void *tinfo) |
| 829 |
{ |
| 830 |
struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); |
| 831 |
struct tsi721_tx_desc *desc; |
| 832 |
struct rio_dma_ext *rext = tinfo; |
| 833 |
enum dma_rtype rtype; |
| 834 |
struct dma_async_tx_descriptor *txd = NULL; |
| 835 |
|
| 836 |
if (!sgl || !sg_len) { |
| 837 |
tsi_err(&dchan->dev->device, "DMAC%d No SG list", |
| 838 |
bdma_chan->id); |
| 839 |
return ERR_PTR(-EINVAL); |
| 840 |
} |
| 841 |
|
| 842 |
tsi_debug(DMA, &dchan->dev->device, "DMAC%d %s", bdma_chan->id, |
| 843 |
(dir == DMA_DEV_TO_MEM)?"READ":"WRITE"); |
| 844 |
|
| 845 |
if (dir == DMA_DEV_TO_MEM) |
| 846 |
rtype = NREAD; |
| 847 |
else if (dir == DMA_MEM_TO_DEV) { |
| 848 |
switch (rext->wr_type) { |
| 849 |
case RDW_ALL_NWRITE: |
| 850 |
rtype = ALL_NWRITE; |
| 851 |
break; |
| 852 |
case RDW_ALL_NWRITE_R: |
| 853 |
rtype = ALL_NWRITE_R; |
| 854 |
break; |
| 855 |
case RDW_LAST_NWRITE_R: |
| 856 |
default: |
| 857 |
rtype = LAST_NWRITE_R; |
| 858 |
break; |
| 859 |
} |
| 860 |
} else { |
| 861 |
tsi_err(&dchan->dev->device, |
| 862 |
"DMAC%d Unsupported DMA direction option", |
| 863 |
bdma_chan->id); |
| 864 |
return ERR_PTR(-EINVAL); |
| 865 |
} |
| 866 |
|
| 867 |
spin_lock_bh(&bdma_chan->lock); |
| 868 |
|
| 869 |
if (!list_empty(&bdma_chan->free_list)) { |
| 870 |
desc = list_first_entry(&bdma_chan->free_list, |
| 871 |
struct tsi721_tx_desc, desc_node); |
| 872 |
list_del_init(&desc->desc_node); |
| 873 |
desc->destid = rext->destid; |
| 874 |
desc->rio_addr = rext->rio_addr; |
| 875 |
desc->rio_addr_u = 0; |
| 876 |
desc->rtype = rtype; |
| 877 |
desc->sg_len = sg_len; |
| 878 |
desc->sg = sgl; |
| 879 |
txd = &desc->txd; |
| 880 |
txd->flags = flags; |
| 881 |
} |
| 882 |
|
| 883 |
spin_unlock_bh(&bdma_chan->lock); |
| 884 |
|
| 885 |
if (!txd) { |
| 886 |
tsi_debug(DMA, &dchan->dev->device, |
| 887 |
"DMAC%d free TXD is not available", bdma_chan->id); |
| 888 |
return ERR_PTR(-EBUSY); |
| 889 |
} |
| 890 |
|
| 891 |
return txd; |
| 892 |
} |
| 893 |
|
| 894 |
static int tsi721_terminate_all(struct dma_chan *dchan) |
| 895 |
{ |
| 896 |
struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); |
| 897 |
struct tsi721_tx_desc *desc, *_d; |
| 898 |
LIST_HEAD(list); |
| 899 |
|
| 900 |
tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); |
| 901 |
|
| 902 |
spin_lock_bh(&bdma_chan->lock); |
| 903 |
|
| 904 |
bdma_chan->active = false; |
| 905 |
|
| 906 |
while (!tsi721_dma_is_idle(bdma_chan)) { |
| 907 |
|
| 908 |
udelay(5); |
| 909 |
#if (0) |
| 910 |
/* make sure to stop the transfer */ |
| 911 |
iowrite32(TSI721_DMAC_CTL_SUSP, |
| 912 |
bdma_chan->regs + TSI721_DMAC_CTL); |
| 913 |
|
| 914 |
/* Wait until DMA channel stops */ |
| 915 |
do { |
| 916 |
dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT); |
| 917 |
} while ((dmac_int & TSI721_DMAC_INT_SUSP) == 0); |
| 918 |
#endif |
| 919 |
} |
| 920 |
|
| 921 |
if (bdma_chan->active_tx) |
| 922 |
list_add(&bdma_chan->active_tx->desc_node, &list); |
| 923 |
list_splice_init(&bdma_chan->queue, &list); |
| 924 |
|
| 925 |
list_for_each_entry_safe(desc, _d, &list, desc_node) |
| 926 |
tsi721_dma_tx_err(bdma_chan, desc); |
| 927 |
|
| 928 |
spin_unlock_bh(&bdma_chan->lock); |
| 929 |
|
| 930 |
return 0; |
| 931 |
} |
| 932 |
|
| 933 |
static void tsi721_dma_stop(struct tsi721_bdma_chan *bdma_chan) |
| 934 |
{ |
| 935 |
if (!bdma_chan->active) |
| 936 |
return; |
| 937 |
spin_lock_bh(&bdma_chan->lock); |
| 938 |
if (!tsi721_dma_is_idle(bdma_chan)) { |
| 939 |
int timeout = 100000; |
| 940 |
|
| 941 |
/* stop the transfer in progress */ |
| 942 |
iowrite32(TSI721_DMAC_CTL_SUSP, |
| 943 |
bdma_chan->regs + TSI721_DMAC_CTL); |
| 944 |
|
| 945 |
/* Wait until DMA channel stops */ |
| 946 |
while (!tsi721_dma_is_idle(bdma_chan) && --timeout) |
| 947 |
udelay(1); |
| 948 |
} |
| 949 |
|
| 950 |
spin_unlock_bh(&bdma_chan->lock); |
| 951 |
} |
| 952 |
|
| 953 |
void tsi721_dma_stop_all(struct tsi721_device *priv) |
| 954 |
{ |
| 955 |
int i; |
| 956 |
|
| 957 |
for (i = 0; i < TSI721_DMA_MAXCH; i++) { |
| 958 |
if ((i != TSI721_DMACH_MAINT) && (dma_sel & (1 << i))) |
| 959 |
tsi721_dma_stop(&priv->bdma[i]); |
| 960 |
} |
| 961 |
} |
| 962 |
|
| 963 |
int tsi721_register_dma(struct tsi721_device *priv) |
| 964 |
{ |
| 965 |
int i; |
| 966 |
int nr_channels = 0; |
| 967 |
int err; |
| 968 |
struct rio_mport *mport = &priv->mport; |
| 969 |
|
| 970 |
INIT_LIST_HEAD(&mport->dma.channels); |
| 971 |
|
| 972 |
for (i = 0; i < TSI721_DMA_MAXCH; i++) { |
| 973 |
struct tsi721_bdma_chan *bdma_chan = &priv->bdma[i]; |
| 974 |
|
| 975 |
if ((i == TSI721_DMACH_MAINT) || (dma_sel & (1 << i)) == 0) |
| 976 |
continue; |
| 977 |
|
| 978 |
bdma_chan->regs = priv->regs + TSI721_DMAC_BASE(i); |
| 979 |
|
| 980 |
bdma_chan->dchan.device = &mport->dma; |
| 981 |
bdma_chan->dchan.cookie = 1; |
| 982 |
bdma_chan->dchan.chan_id = i; |
| 983 |
bdma_chan->id = i; |
| 984 |
bdma_chan->active = false; |
| 985 |
|
| 986 |
spin_lock_init(&bdma_chan->lock); |
| 987 |
|
| 988 |
bdma_chan->active_tx = NULL; |
| 989 |
INIT_LIST_HEAD(&bdma_chan->queue); |
| 990 |
INIT_LIST_HEAD(&bdma_chan->free_list); |
| 991 |
|
| 992 |
tasklet_init(&bdma_chan->tasklet, tsi721_dma_tasklet, |
| 993 |
(unsigned long)bdma_chan); |
| 994 |
list_add_tail(&bdma_chan->dchan.device_node, |
| 995 |
&mport->dma.channels); |
| 996 |
nr_channels++; |
| 997 |
} |
| 998 |
|
| 999 |
mport->dma.chancnt = nr_channels; |
| 1000 |
dma_cap_zero(mport->dma.cap_mask); |
| 1001 |
dma_cap_set(DMA_PRIVATE, mport->dma.cap_mask); |
| 1002 |
dma_cap_set(DMA_SLAVE, mport->dma.cap_mask); |
| 1003 |
|
| 1004 |
mport->dma.dev = &priv->pdev->dev; |
| 1005 |
mport->dma.device_alloc_chan_resources = tsi721_alloc_chan_resources; |
| 1006 |
mport->dma.device_free_chan_resources = tsi721_free_chan_resources; |
| 1007 |
mport->dma.device_tx_status = tsi721_tx_status; |
| 1008 |
mport->dma.device_issue_pending = tsi721_issue_pending; |
| 1009 |
mport->dma.device_prep_slave_sg = tsi721_prep_rio_sg; |
| 1010 |
mport->dma.device_terminate_all = tsi721_terminate_all; |
| 1011 |
|
| 1012 |
err = dma_async_device_register(&mport->dma); |
| 1013 |
if (err) |
| 1014 |
tsi_err(&priv->pdev->dev, "Failed to register DMA device"); |
| 1015 |
|
| 1016 |
return err; |
| 1017 |
} |
| 1018 |
|
| 1019 |
void tsi721_unregister_dma(struct tsi721_device *priv) |
| 1020 |
{ |
| 1021 |
struct rio_mport *mport = &priv->mport; |
| 1022 |
struct dma_chan *chan, *_c; |
| 1023 |
struct tsi721_bdma_chan *bdma_chan; |
| 1024 |
|
| 1025 |
tsi721_dma_stop_all(priv); |
| 1026 |
dma_async_device_unregister(&mport->dma); |
| 1027 |
|
| 1028 |
list_for_each_entry_safe(chan, _c, &mport->dma.channels, |
| 1029 |
device_node) { |
| 1030 |
bdma_chan = to_tsi721_chan(chan); |
| 1031 |
if (bdma_chan->active) { |
| 1032 |
tsi721_bdma_interrupt_enable(bdma_chan, 0); |
| 1033 |
bdma_chan->active = false; |
| 1034 |
tsi721_sync_dma_irq(bdma_chan); |
| 1035 |
tasklet_kill(&bdma_chan->tasklet); |
| 1036 |
INIT_LIST_HEAD(&bdma_chan->free_list); |
| 1037 |
kfree(bdma_chan->tx_desc); |
| 1038 |
tsi721_bdma_ch_free(bdma_chan); |
| 1039 |
} |
| 1040 |
|
| 1041 |
list_del(&chan->device_node); |
| 1042 |
} |
| 1043 |
} |