| 1 |
// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause |
| 2 |
/* |
| 3 |
* Copyright (C) 2003-2014, 2018-2024 Intel Corporation |
| 4 |
* Copyright (C) 2013-2015 Intel Mobile Communications GmbH |
| 5 |
* Copyright (C) 2016-2017 Intel Deutschland GmbH |
| 6 |
*/ |
| 7 |
#include <linux/sched.h> |
| 8 |
#include <linux/wait.h> |
| 9 |
#include <linux/gfp.h> |
| 10 |
|
| 11 |
#include "iwl-prph.h" |
| 12 |
#include "iwl-io.h" |
| 13 |
#include "internal.h" |
| 14 |
#include "iwl-op-mode.h" |
| 15 |
#include "pcie/iwl-context-info-v2.h" |
| 16 |
#include "fw/dbg.h" |
| 17 |
|
| 18 |
/****************************************************************************** |
| 19 |
* |
| 20 |
* RX path functions |
| 21 |
* |
| 22 |
******************************************************************************/ |
| 23 |
|
| 24 |
/* |
| 25 |
* Rx theory of operation |
| 26 |
* |
| 27 |
* Driver allocates a circular buffer of Receive Buffer Descriptors (RBDs), |
| 28 |
* each of which point to Receive Buffers to be filled by the NIC. These get |
| 29 |
* used not only for Rx frames, but for any command response or notification |
| 30 |
* from the NIC. The driver and NIC manage the Rx buffers by means |
| 31 |
* of indexes into the circular buffer. |
| 32 |
* |
| 33 |
* Rx Queue Indexes |
| 34 |
* The host/firmware share two index registers for managing the Rx buffers. |
| 35 |
* |
| 36 |
* The READ index maps to the first position that the firmware may be writing |
| 37 |
* to -- the driver can read up to (but not including) this position and get |
| 38 |
* good data. |
| 39 |
* The READ index is managed by the firmware once the card is enabled. |
| 40 |
* |
| 41 |
* The WRITE index maps to the last position the driver has read from -- the |
| 42 |
* position preceding WRITE is the last slot the firmware can place a packet. |
| 43 |
* |
| 44 |
* The queue is empty (no good data) if WRITE = READ - 1, and is full if |
| 45 |
* WRITE = READ. |
| 46 |
* |
| 47 |
* During initialization, the host sets up the READ queue position to the first |
| 48 |
* INDEX position, and WRITE to the last (READ - 1 wrapped) |
| 49 |
* |
| 50 |
* When the firmware places a packet in a buffer, it will advance the READ index |
| 51 |
* and fire the RX interrupt. The driver can then query the READ index and |
| 52 |
* process as many packets as possible, moving the WRITE index forward as it |
| 53 |
* resets the Rx queue buffers with new memory. |
| 54 |
* |
| 55 |
* The management in the driver is as follows: |
| 56 |
* + A list of pre-allocated RBDs is stored in iwl->rxq->rx_free. |
| 57 |
* When the interrupt handler is called, the request is processed. |
| 58 |
* The page is either stolen - transferred to the upper layer |
| 59 |
* or reused - added immediately to the iwl->rxq->rx_free list. |
| 60 |
* + When the page is stolen - the driver updates the matching queue's used |
| 61 |
* count, detaches the RBD and transfers it to the queue used list. |
| 62 |
* When there are two used RBDs - they are transferred to the allocator empty |
| 63 |
* list. Work is then scheduled for the allocator to start allocating |
| 64 |
* eight buffers. |
| 65 |
* When there are another 6 used RBDs - they are transferred to the allocator |
| 66 |
* empty list and the driver tries to claim the pre-allocated buffers and |
| 67 |
* add them to iwl->rxq->rx_free. If it fails - it continues to claim them |
| 68 |
* until ready. |
| 69 |
* When there are 8+ buffers in the free list - either from allocation or from |
| 70 |
* 8 reused unstolen pages - restock is called to update the FW and indexes. |
| 71 |
* + In order to make sure the allocator always has RBDs to use for allocation |
| 72 |
* the allocator has initial pool in the size of num_queues*(8-2) - the |
| 73 |
* maximum missing RBDs per allocation request (request posted with 2 |
| 74 |
* empty RBDs, there is no guarantee when the other 6 RBDs are supplied). |
| 75 |
* The queues supplies the recycle of the rest of the RBDs. |
| 76 |
* + A received packet is processed and handed to the kernel network stack, |
| 77 |
* detached from the iwl->rxq. The driver 'processed' index is updated. |
| 78 |
* + If there are no allocated buffers in iwl->rxq->rx_free, |
| 79 |
* the READ INDEX is not incremented and iwl->status(RX_STALLED) is set. |
| 80 |
* If there were enough free buffers and RX_STALLED is set it is cleared. |
| 81 |
* |
| 82 |
* |
| 83 |
* Driver sequence: |
| 84 |
* |
| 85 |
* iwl_rxq_alloc() Allocates rx_free |
| 86 |
* iwl_pcie_rx_replenish() Replenishes rx_free list from rx_used, and calls |
| 87 |
* iwl_pcie_rxq_restock. |
| 88 |
* Used only during initialization. |
| 89 |
* iwl_pcie_rxq_restock() Moves available buffers from rx_free into Rx |
| 90 |
* queue, updates firmware pointers, and updates |
| 91 |
* the WRITE index. |
| 92 |
* iwl_pcie_rx_allocator() Background work for allocating pages. |
| 93 |
* |
| 94 |
* -- enable interrupts -- |
| 95 |
* ISR - iwl_rx() Detach iwl_rx_mem_buffers from pool up to the |
| 96 |
* READ INDEX, detaching the SKB from the pool. |
| 97 |
* Moves the packet buffer from queue to rx_used. |
| 98 |
* Posts and claims requests to the allocator. |
| 99 |
* Calls iwl_pcie_rxq_restock to refill any empty |
| 100 |
* slots. |
| 101 |
* |
| 102 |
* RBD life-cycle: |
| 103 |
* |
| 104 |
* Init: |
| 105 |
* rxq.pool -> rxq.rx_used -> rxq.rx_free -> rxq.queue |
| 106 |
* |
| 107 |
* Regular Receive interrupt: |
| 108 |
* Page Stolen: |
| 109 |
* rxq.queue -> rxq.rx_used -> allocator.rbd_empty -> |
| 110 |
* allocator.rbd_allocated -> rxq.rx_free -> rxq.queue |
| 111 |
* Page not Stolen: |
| 112 |
* rxq.queue -> rxq.rx_free -> rxq.queue |
| 113 |
* ... |
| 114 |
* |
| 115 |
*/ |
| 116 |
|
| 117 |
/* |
| 118 |
* iwl_rxq_space - Return number of free slots available in queue. |
| 119 |
*/ |
| 120 |
static int iwl_rxq_space(const struct iwl_rxq *rxq) |
| 121 |
{ |
| 122 |
/* Make sure rx queue size is a power of 2 */ |
| 123 |
WARN_ON(rxq->queue_size & (rxq->queue_size - 1)); |
| 124 |
|
| 125 |
/* |
| 126 |
* There can be up to (RX_QUEUE_SIZE - 1) free slots, to avoid ambiguity |
| 127 |
* between empty and completely full queues. |
| 128 |
* The following is equivalent to modulo by RX_QUEUE_SIZE and is well |
| 129 |
* defined for negative dividends. |
| 130 |
*/ |
| 131 |
return (rxq->read - rxq->write - 1) & (rxq->queue_size - 1); |
| 132 |
} |
| 133 |
|
| 134 |
/* |
| 135 |
* iwl_dma_addr2rbd_ptr - convert a DMA address to a uCode read buffer ptr |
| 136 |
*/ |
| 137 |
static inline __le32 iwl_pcie_dma_addr2rbd_ptr(dma_addr_t dma_addr) |
| 138 |
{ |
| 139 |
return cpu_to_le32((u32)(dma_addr >> 8)); |
| 140 |
} |
| 141 |
|
| 142 |
/* |
| 143 |
* iwl_pcie_rx_stop - stops the Rx DMA |
| 144 |
*/ |
| 145 |
int iwl_pcie_rx_stop(struct iwl_trans *trans) |
| 146 |
{ |
| 147 |
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { |
| 148 |
/* TODO: remove this once fw does it */ |
| 149 |
iwl_write_umac_prph(trans, RFH_RXF_DMA_CFG_AX210, 0); |
| 150 |
return iwl_poll_umac_prph_bit(trans, RFH_GEN_STATUS_AX210, |
| 151 |
RXF_DMA_IDLE, RXF_DMA_IDLE, 1000); |
| 152 |
} else if (trans->mac_cfg->mq_rx_supported) { |
| 153 |
iwl_write_prph(trans, RFH_RXF_DMA_CFG, 0); |
| 154 |
return iwl_poll_prph_bit(trans, RFH_GEN_STATUS, |
| 155 |
RXF_DMA_IDLE, RXF_DMA_IDLE, 1000); |
| 156 |
} else { |
| 157 |
iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0); |
| 158 |
return iwl_poll_direct_bit(trans, FH_MEM_RSSR_RX_STATUS_REG, |
| 159 |
FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE, |
| 160 |
1000); |
| 161 |
} |
| 162 |
} |
| 163 |
|
| 164 |
/* |
| 165 |
* iwl_pcie_rxq_inc_wr_ptr - Update the write pointer for the RX queue |
| 166 |
*/ |
| 167 |
static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans, |
| 168 |
struct iwl_rxq *rxq) |
| 169 |
{ |
| 170 |
u32 reg; |
| 171 |
|
| 172 |
lockdep_assert_held(&rxq->lock); |
| 173 |
|
| 174 |
/* |
| 175 |
* explicitly wake up the NIC if: |
| 176 |
* 1. shadow registers aren't enabled |
| 177 |
* 2. there is a chance that the NIC is asleep |
| 178 |
*/ |
| 179 |
if (!trans->mac_cfg->base->shadow_reg_enable && |
| 180 |
test_bit(STATUS_TPOWER_PMI, &trans->status)) { |
| 181 |
reg = iwl_read32(trans, CSR_UCODE_DRV_GP1); |
| 182 |
|
| 183 |
if (reg & CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP) { |
| 184 |
IWL_DEBUG_INFO(trans, "Rx queue requesting wakeup, GP1 = 0x%x\n", |
| 185 |
reg); |
| 186 |
iwl_set_bit(trans, CSR_GP_CNTRL, |
| 187 |
CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); |
| 188 |
rxq->need_update = true; |
| 189 |
return; |
| 190 |
} |
| 191 |
} |
| 192 |
|
| 193 |
rxq->write_actual = round_down(rxq->write, 8); |
| 194 |
if (!trans->mac_cfg->mq_rx_supported) |
| 195 |
iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual); |
| 196 |
else if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) |
| 197 |
iwl_write32(trans, HBUS_TARG_WRPTR, rxq->write_actual | |
| 198 |
HBUS_TARG_WRPTR_RX_Q(rxq->id)); |
| 199 |
else |
| 200 |
iwl_write32(trans, RFH_Q_FRBDCB_WIDX_TRG(rxq->id), |
| 201 |
rxq->write_actual); |
| 202 |
} |
| 203 |
|
| 204 |
static void iwl_pcie_rxq_check_wrptr(struct iwl_trans *trans) |
| 205 |
{ |
| 206 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 207 |
int i; |
| 208 |
|
| 209 |
for (i = 0; i < trans->info.num_rxqs; i++) { |
| 210 |
struct iwl_rxq *rxq = &trans_pcie->rxq[i]; |
| 211 |
|
| 212 |
if (!rxq->need_update) |
| 213 |
continue; |
| 214 |
spin_lock_bh(&rxq->lock); |
| 215 |
iwl_pcie_rxq_inc_wr_ptr(trans, rxq); |
| 216 |
rxq->need_update = false; |
| 217 |
spin_unlock_bh(&rxq->lock); |
| 218 |
} |
| 219 |
} |
| 220 |
|
| 221 |
static void iwl_pcie_restock_bd(struct iwl_trans *trans, |
| 222 |
struct iwl_rxq *rxq, |
| 223 |
struct iwl_rx_mem_buffer *rxb) |
| 224 |
{ |
| 225 |
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { |
| 226 |
struct iwl_rx_transfer_desc *bd = rxq->bd; |
| 227 |
|
| 228 |
BUILD_BUG_ON(sizeof(*bd) != 2 * sizeof(u64)); |
| 229 |
|
| 230 |
bd[rxq->write].addr = cpu_to_le64(rxb->page_dma); |
| 231 |
bd[rxq->write].rbid = cpu_to_le16(rxb->vid); |
| 232 |
} else { |
| 233 |
__le64 *bd = rxq->bd; |
| 234 |
|
| 235 |
bd[rxq->write] = cpu_to_le64(rxb->page_dma | rxb->vid); |
| 236 |
} |
| 237 |
|
| 238 |
IWL_DEBUG_RX(trans, "Assigned virtual RB ID %u to queue %d index %d\n", |
| 239 |
(u32)rxb->vid, rxq->id, rxq->write); |
| 240 |
} |
| 241 |
|
| 242 |
/* |
| 243 |
* iwl_pcie_rxmq_restock - restock implementation for multi-queue rx |
| 244 |
*/ |
| 245 |
static void iwl_pcie_rxmq_restock(struct iwl_trans *trans, |
| 246 |
struct iwl_rxq *rxq) |
| 247 |
{ |
| 248 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 249 |
struct iwl_rx_mem_buffer *rxb; |
| 250 |
|
| 251 |
/* |
| 252 |
* If the device isn't enabled - no need to try to add buffers... |
| 253 |
* This can happen when we stop the device and still have an interrupt |
| 254 |
* pending. We stop the APM before we sync the interrupts because we |
| 255 |
* have to (see comment there). On the other hand, since the APM is |
| 256 |
* stopped, we cannot access the HW (in particular not prph). |
| 257 |
* So don't try to restock if the APM has been already stopped. |
| 258 |
*/ |
| 259 |
if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status)) |
| 260 |
return; |
| 261 |
|
| 262 |
spin_lock_bh(&rxq->lock); |
| 263 |
while (rxq->free_count) { |
| 264 |
/* Get next free Rx buffer, remove from free list */ |
| 265 |
rxb = list_first_entry(&rxq->rx_free, struct iwl_rx_mem_buffer, |
| 266 |
list); |
| 267 |
list_del(&rxb->list); |
| 268 |
rxb->invalid = false; |
| 269 |
/* some low bits are expected to be unset (depending on hw) */ |
| 270 |
WARN_ON(rxb->page_dma & trans_pcie->supported_dma_mask); |
| 271 |
/* Point to Rx buffer via next RBD in circular buffer */ |
| 272 |
iwl_pcie_restock_bd(trans, rxq, rxb); |
| 273 |
rxq->write = (rxq->write + 1) & (rxq->queue_size - 1); |
| 274 |
rxq->free_count--; |
| 275 |
} |
| 276 |
spin_unlock_bh(&rxq->lock); |
| 277 |
|
| 278 |
/* |
| 279 |
* If we've added more space for the firmware to place data, tell it. |
| 280 |
* Increment device's write pointer in multiples of 8. |
| 281 |
*/ |
| 282 |
if (rxq->write_actual != (rxq->write & ~0x7)) { |
| 283 |
spin_lock_bh(&rxq->lock); |
| 284 |
iwl_pcie_rxq_inc_wr_ptr(trans, rxq); |
| 285 |
spin_unlock_bh(&rxq->lock); |
| 286 |
} |
| 287 |
} |
| 288 |
|
| 289 |
/* |
| 290 |
* iwl_pcie_rxsq_restock - restock implementation for single queue rx |
| 291 |
*/ |
| 292 |
static void iwl_pcie_rxsq_restock(struct iwl_trans *trans, |
| 293 |
struct iwl_rxq *rxq) |
| 294 |
{ |
| 295 |
struct iwl_rx_mem_buffer *rxb; |
| 296 |
|
| 297 |
/* |
| 298 |
* If the device isn't enabled - not need to try to add buffers... |
| 299 |
* This can happen when we stop the device and still have an interrupt |
| 300 |
* pending. We stop the APM before we sync the interrupts because we |
| 301 |
* have to (see comment there). On the other hand, since the APM is |
| 302 |
* stopped, we cannot access the HW (in particular not prph). |
| 303 |
* So don't try to restock if the APM has been already stopped. |
| 304 |
*/ |
| 305 |
if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status)) |
| 306 |
return; |
| 307 |
|
| 308 |
spin_lock_bh(&rxq->lock); |
| 309 |
while ((iwl_rxq_space(rxq) > 0) && (rxq->free_count)) { |
| 310 |
__le32 *bd = (__le32 *)rxq->bd; |
| 311 |
/* The overwritten rxb must be a used one */ |
| 312 |
rxb = rxq->queue[rxq->write]; |
| 313 |
BUG_ON(rxb && rxb->page); |
| 314 |
|
| 315 |
/* Get next free Rx buffer, remove from free list */ |
| 316 |
rxb = list_first_entry(&rxq->rx_free, struct iwl_rx_mem_buffer, |
| 317 |
list); |
| 318 |
list_del(&rxb->list); |
| 319 |
rxb->invalid = false; |
| 320 |
|
| 321 |
/* Point to Rx buffer via next RBD in circular buffer */ |
| 322 |
bd[rxq->write] = iwl_pcie_dma_addr2rbd_ptr(rxb->page_dma); |
| 323 |
rxq->queue[rxq->write] = rxb; |
| 324 |
rxq->write = (rxq->write + 1) & RX_QUEUE_MASK; |
| 325 |
rxq->free_count--; |
| 326 |
} |
| 327 |
spin_unlock_bh(&rxq->lock); |
| 328 |
|
| 329 |
/* If we've added more space for the firmware to place data, tell it. |
| 330 |
* Increment device's write pointer in multiples of 8. */ |
| 331 |
if (rxq->write_actual != (rxq->write & ~0x7)) { |
| 332 |
spin_lock_bh(&rxq->lock); |
| 333 |
iwl_pcie_rxq_inc_wr_ptr(trans, rxq); |
| 334 |
spin_unlock_bh(&rxq->lock); |
| 335 |
} |
| 336 |
} |
| 337 |
|
| 338 |
/* |
| 339 |
* iwl_pcie_rxq_restock - refill RX queue from pre-allocated pool |
| 340 |
* |
| 341 |
* If there are slots in the RX queue that need to be restocked, |
| 342 |
* and we have free pre-allocated buffers, fill the ranks as much |
| 343 |
* as we can, pulling from rx_free. |
| 344 |
* |
| 345 |
* This moves the 'write' index forward to catch up with 'processed', and |
| 346 |
* also updates the memory address in the firmware to reference the new |
| 347 |
* target buffer. |
| 348 |
*/ |
| 349 |
static |
| 350 |
void iwl_pcie_rxq_restock(struct iwl_trans *trans, struct iwl_rxq *rxq) |
| 351 |
{ |
| 352 |
if (trans->mac_cfg->mq_rx_supported) |
| 353 |
iwl_pcie_rxmq_restock(trans, rxq); |
| 354 |
else |
| 355 |
iwl_pcie_rxsq_restock(trans, rxq); |
| 356 |
} |
| 357 |
|
| 358 |
/* |
| 359 |
* iwl_pcie_rx_alloc_page - allocates and returns a page. |
| 360 |
* |
| 361 |
*/ |
| 362 |
static struct page *iwl_pcie_rx_alloc_page(struct iwl_trans *trans, |
| 363 |
u32 *offset, gfp_t priority) |
| 364 |
{ |
| 365 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 366 |
unsigned int allocsize = PAGE_SIZE << trans_pcie->rx_page_order; |
| 367 |
unsigned int rbsize = trans_pcie->rx_buf_bytes; |
| 368 |
struct page *page; |
| 369 |
gfp_t gfp_mask = priority; |
| 370 |
|
| 371 |
if (trans_pcie->rx_page_order > 0) |
| 372 |
gfp_mask |= __GFP_COMP; |
| 373 |
|
| 374 |
if (trans_pcie->alloc_page) { |
| 375 |
spin_lock_bh(&trans_pcie->alloc_page_lock); |
| 376 |
/* recheck */ |
| 377 |
if (trans_pcie->alloc_page) { |
| 378 |
*offset = trans_pcie->alloc_page_used; |
| 379 |
page = trans_pcie->alloc_page; |
| 380 |
trans_pcie->alloc_page_used += rbsize; |
| 381 |
if (trans_pcie->alloc_page_used >= allocsize) |
| 382 |
trans_pcie->alloc_page = NULL; |
| 383 |
else |
| 384 |
get_page(page); |
| 385 |
spin_unlock_bh(&trans_pcie->alloc_page_lock); |
| 386 |
return page; |
| 387 |
} |
| 388 |
spin_unlock_bh(&trans_pcie->alloc_page_lock); |
| 389 |
} |
| 390 |
|
| 391 |
/* Alloc a new receive buffer */ |
| 392 |
page = alloc_pages(gfp_mask, trans_pcie->rx_page_order); |
| 393 |
if (!page) { |
| 394 |
if (net_ratelimit()) |
| 395 |
IWL_DEBUG_INFO(trans, "alloc_pages failed, order: %d\n", |
| 396 |
trans_pcie->rx_page_order); |
| 397 |
/* |
| 398 |
* Issue an error if we don't have enough pre-allocated |
| 399 |
* buffers. |
| 400 |
*/ |
| 401 |
if (!(gfp_mask & __GFP_NOWARN) && net_ratelimit()) |
| 402 |
IWL_CRIT(trans, |
| 403 |
"Failed to alloc_pages\n"); |
| 404 |
return NULL; |
| 405 |
} |
| 406 |
|
| 407 |
if (2 * rbsize <= allocsize) { |
| 408 |
spin_lock_bh(&trans_pcie->alloc_page_lock); |
| 409 |
if (!trans_pcie->alloc_page) { |
| 410 |
get_page(page); |
| 411 |
trans_pcie->alloc_page = page; |
| 412 |
trans_pcie->alloc_page_used = rbsize; |
| 413 |
} |
| 414 |
spin_unlock_bh(&trans_pcie->alloc_page_lock); |
| 415 |
} |
| 416 |
|
| 417 |
*offset = 0; |
| 418 |
return page; |
| 419 |
} |
| 420 |
|
| 421 |
/* |
| 422 |
* iwl_pcie_rxq_alloc_rbs - allocate a page for each used RBD |
| 423 |
* |
| 424 |
* A used RBD is an Rx buffer that has been given to the stack. To use it again |
| 425 |
* a page must be allocated and the RBD must point to the page. This function |
| 426 |
* doesn't change the HW pointer but handles the list of pages that is used by |
| 427 |
* iwl_pcie_rxq_restock. The latter function will update the HW to use the newly |
| 428 |
* allocated buffers. |
| 429 |
*/ |
| 430 |
void iwl_pcie_rxq_alloc_rbs(struct iwl_trans *trans, gfp_t priority, |
| 431 |
struct iwl_rxq *rxq) |
| 432 |
{ |
| 433 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 434 |
struct iwl_rx_mem_buffer *rxb; |
| 435 |
struct page *page; |
| 436 |
|
| 437 |
while (1) { |
| 438 |
unsigned int offset; |
| 439 |
|
| 440 |
spin_lock_bh(&rxq->lock); |
| 441 |
if (list_empty(&rxq->rx_used)) { |
| 442 |
spin_unlock_bh(&rxq->lock); |
| 443 |
return; |
| 444 |
} |
| 445 |
spin_unlock_bh(&rxq->lock); |
| 446 |
|
| 447 |
page = iwl_pcie_rx_alloc_page(trans, &offset, priority); |
| 448 |
if (!page) |
| 449 |
return; |
| 450 |
|
| 451 |
spin_lock_bh(&rxq->lock); |
| 452 |
|
| 453 |
if (list_empty(&rxq->rx_used)) { |
| 454 |
spin_unlock_bh(&rxq->lock); |
| 455 |
__free_pages(page, trans_pcie->rx_page_order); |
| 456 |
return; |
| 457 |
} |
| 458 |
rxb = list_first_entry(&rxq->rx_used, struct iwl_rx_mem_buffer, |
| 459 |
list); |
| 460 |
list_del(&rxb->list); |
| 461 |
spin_unlock_bh(&rxq->lock); |
| 462 |
|
| 463 |
BUG_ON(rxb->page); |
| 464 |
rxb->page = page; |
| 465 |
rxb->offset = offset; |
| 466 |
/* Get physical address of the RB */ |
| 467 |
rxb->page_dma = |
| 468 |
dma_map_page(trans->dev, page, rxb->offset, |
| 469 |
trans_pcie->rx_buf_bytes, |
| 470 |
DMA_FROM_DEVICE); |
| 471 |
if (dma_mapping_error(trans->dev, rxb->page_dma)) { |
| 472 |
rxb->page = NULL; |
| 473 |
spin_lock_bh(&rxq->lock); |
| 474 |
list_add(&rxb->list, &rxq->rx_used); |
| 475 |
spin_unlock_bh(&rxq->lock); |
| 476 |
__free_pages(page, trans_pcie->rx_page_order); |
| 477 |
return; |
| 478 |
} |
| 479 |
|
| 480 |
spin_lock_bh(&rxq->lock); |
| 481 |
|
| 482 |
list_add_tail(&rxb->list, &rxq->rx_free); |
| 483 |
rxq->free_count++; |
| 484 |
|
| 485 |
spin_unlock_bh(&rxq->lock); |
| 486 |
} |
| 487 |
} |
| 488 |
|
| 489 |
void iwl_pcie_free_rbs_pool(struct iwl_trans *trans) |
| 490 |
{ |
| 491 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 492 |
int i; |
| 493 |
|
| 494 |
if (!trans_pcie->rx_pool) |
| 495 |
return; |
| 496 |
|
| 497 |
for (i = 0; i < RX_POOL_SIZE(trans_pcie->num_rx_bufs); i++) { |
| 498 |
if (!trans_pcie->rx_pool[i].page) |
| 499 |
continue; |
| 500 |
dma_unmap_page(trans->dev, trans_pcie->rx_pool[i].page_dma, |
| 501 |
trans_pcie->rx_buf_bytes, DMA_FROM_DEVICE); |
| 502 |
__free_pages(trans_pcie->rx_pool[i].page, |
| 503 |
trans_pcie->rx_page_order); |
| 504 |
trans_pcie->rx_pool[i].page = NULL; |
| 505 |
} |
| 506 |
} |
| 507 |
|
| 508 |
/* |
| 509 |
* iwl_pcie_rx_allocator - Allocates pages in the background for RX queues |
| 510 |
* |
| 511 |
* Allocates for each received request 8 pages |
| 512 |
* Called as a scheduled work item. |
| 513 |
*/ |
| 514 |
static void iwl_pcie_rx_allocator(struct iwl_trans *trans) |
| 515 |
{ |
| 516 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 517 |
struct iwl_rb_allocator *rba = &trans_pcie->rba; |
| 518 |
struct list_head local_empty; |
| 519 |
int pending = atomic_read(&rba->req_pending); |
| 520 |
|
| 521 |
IWL_DEBUG_TPT(trans, "Pending allocation requests = %d\n", pending); |
| 522 |
|
| 523 |
/* If we were scheduled - there is at least one request */ |
| 524 |
spin_lock_bh(&rba->lock); |
| 525 |
/* swap out the rba->rbd_empty to a local list */ |
| 526 |
list_replace_init(&rba->rbd_empty, &local_empty); |
| 527 |
spin_unlock_bh(&rba->lock); |
| 528 |
|
| 529 |
while (pending) { |
| 530 |
int i; |
| 531 |
LIST_HEAD(local_allocated); |
| 532 |
gfp_t gfp_mask = GFP_KERNEL; |
| 533 |
|
| 534 |
/* Do not post a warning if there are only a few requests */ |
| 535 |
if (pending < RX_PENDING_WATERMARK) |
| 536 |
gfp_mask |= __GFP_NOWARN; |
| 537 |
|
| 538 |
for (i = 0; i < RX_CLAIM_REQ_ALLOC;) { |
| 539 |
struct iwl_rx_mem_buffer *rxb; |
| 540 |
struct page *page; |
| 541 |
|
| 542 |
/* List should never be empty - each reused RBD is |
| 543 |
* returned to the list, and initial pool covers any |
| 544 |
* possible gap between the time the page is allocated |
| 545 |
* to the time the RBD is added. |
| 546 |
*/ |
| 547 |
BUG_ON(list_empty(&local_empty)); |
| 548 |
/* Get the first rxb from the rbd list */ |
| 549 |
rxb = list_first_entry(&local_empty, |
| 550 |
struct iwl_rx_mem_buffer, list); |
| 551 |
BUG_ON(rxb->page); |
| 552 |
|
| 553 |
/* Alloc a new receive buffer */ |
| 554 |
page = iwl_pcie_rx_alloc_page(trans, &rxb->offset, |
| 555 |
gfp_mask); |
| 556 |
if (!page) |
| 557 |
continue; |
| 558 |
rxb->page = page; |
| 559 |
|
| 560 |
/* Get physical address of the RB */ |
| 561 |
rxb->page_dma = dma_map_page(trans->dev, page, |
| 562 |
rxb->offset, |
| 563 |
trans_pcie->rx_buf_bytes, |
| 564 |
DMA_FROM_DEVICE); |
| 565 |
if (dma_mapping_error(trans->dev, rxb->page_dma)) { |
| 566 |
rxb->page = NULL; |
| 567 |
__free_pages(page, trans_pcie->rx_page_order); |
| 568 |
continue; |
| 569 |
} |
| 570 |
|
| 571 |
/* move the allocated entry to the out list */ |
| 572 |
list_move(&rxb->list, &local_allocated); |
| 573 |
i++; |
| 574 |
} |
| 575 |
|
| 576 |
atomic_dec(&rba->req_pending); |
| 577 |
pending--; |
| 578 |
|
| 579 |
if (!pending) { |
| 580 |
pending = atomic_read(&rba->req_pending); |
| 581 |
if (pending) |
| 582 |
IWL_DEBUG_TPT(trans, |
| 583 |
"Got more pending allocation requests = %d\n", |
| 584 |
pending); |
| 585 |
} |
| 586 |
|
| 587 |
spin_lock_bh(&rba->lock); |
| 588 |
/* add the allocated rbds to the allocator allocated list */ |
| 589 |
list_splice_tail(&local_allocated, &rba->rbd_allocated); |
| 590 |
/* get more empty RBDs for current pending requests */ |
| 591 |
list_splice_tail_init(&rba->rbd_empty, &local_empty); |
| 592 |
spin_unlock_bh(&rba->lock); |
| 593 |
|
| 594 |
atomic_inc(&rba->req_ready); |
| 595 |
|
| 596 |
} |
| 597 |
|
| 598 |
spin_lock_bh(&rba->lock); |
| 599 |
/* return unused rbds to the allocator empty list */ |
| 600 |
list_splice_tail(&local_empty, &rba->rbd_empty); |
| 601 |
spin_unlock_bh(&rba->lock); |
| 602 |
|
| 603 |
IWL_DEBUG_TPT(trans, "%s, exit.\n", __func__); |
| 604 |
} |
| 605 |
|
| 606 |
/* |
| 607 |
* iwl_pcie_rx_allocator_get - returns the pre-allocated pages |
| 608 |
.* |
| 609 |
.* Called by queue when the queue posted allocation request and |
| 610 |
* has freed 8 RBDs in order to restock itself. |
| 611 |
* This function directly moves the allocated RBs to the queue's ownership |
| 612 |
* and updates the relevant counters. |
| 613 |
*/ |
| 614 |
static void iwl_pcie_rx_allocator_get(struct iwl_trans *trans, |
| 615 |
struct iwl_rxq *rxq) |
| 616 |
{ |
| 617 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 618 |
struct iwl_rb_allocator *rba = &trans_pcie->rba; |
| 619 |
int i; |
| 620 |
|
| 621 |
lockdep_assert_held(&rxq->lock); |
| 622 |
|
| 623 |
/* |
| 624 |
* atomic_dec_if_positive returns req_ready - 1 for any scenario. |
| 625 |
* If req_ready is 0 atomic_dec_if_positive will return -1 and this |
| 626 |
* function will return early, as there are no ready requests. |
| 627 |
* atomic_dec_if_positive will perofrm the *actual* decrement only if |
| 628 |
* req_ready > 0, i.e. - there are ready requests and the function |
| 629 |
* hands one request to the caller. |
| 630 |
*/ |
| 631 |
if (atomic_dec_if_positive(&rba->req_ready) < 0) |
| 632 |
return; |
| 633 |
|
| 634 |
spin_lock(&rba->lock); |
| 635 |
for (i = 0; i < RX_CLAIM_REQ_ALLOC; i++) { |
| 636 |
/* Get next free Rx buffer, remove it from free list */ |
| 637 |
struct iwl_rx_mem_buffer *rxb = |
| 638 |
list_first_entry(&rba->rbd_allocated, |
| 639 |
struct iwl_rx_mem_buffer, list); |
| 640 |
|
| 641 |
list_move(&rxb->list, &rxq->rx_free); |
| 642 |
} |
| 643 |
spin_unlock(&rba->lock); |
| 644 |
|
| 645 |
rxq->used_count -= RX_CLAIM_REQ_ALLOC; |
| 646 |
rxq->free_count += RX_CLAIM_REQ_ALLOC; |
| 647 |
} |
| 648 |
|
| 649 |
void iwl_pcie_rx_allocator_work(struct work_struct *data) |
| 650 |
{ |
| 651 |
struct iwl_rb_allocator *rba_p = |
| 652 |
container_of(data, struct iwl_rb_allocator, rx_alloc); |
| 653 |
struct iwl_trans_pcie *trans_pcie = |
| 654 |
container_of(rba_p, struct iwl_trans_pcie, rba); |
| 655 |
|
| 656 |
iwl_pcie_rx_allocator(trans_pcie->trans); |
| 657 |
} |
| 658 |
|
| 659 |
static int iwl_pcie_free_bd_size(struct iwl_trans *trans) |
| 660 |
{ |
| 661 |
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) |
| 662 |
return sizeof(struct iwl_rx_transfer_desc); |
| 663 |
|
| 664 |
return trans->mac_cfg->mq_rx_supported ? |
| 665 |
sizeof(__le64) : sizeof(__le32); |
| 666 |
} |
| 667 |
|
| 668 |
static int iwl_pcie_used_bd_size(struct iwl_trans *trans) |
| 669 |
{ |
| 670 |
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) |
| 671 |
return sizeof(struct iwl_rx_completion_desc_bz); |
| 672 |
|
| 673 |
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) |
| 674 |
return sizeof(struct iwl_rx_completion_desc); |
| 675 |
|
| 676 |
return sizeof(__le32); |
| 677 |
} |
| 678 |
|
| 679 |
static void iwl_pcie_free_rxq_dma(struct iwl_trans *trans, |
| 680 |
struct iwl_rxq *rxq) |
| 681 |
{ |
| 682 |
int free_size = iwl_pcie_free_bd_size(trans); |
| 683 |
|
| 684 |
if (rxq->bd) |
| 685 |
dma_free_coherent(trans->dev, |
| 686 |
free_size * rxq->queue_size, |
| 687 |
rxq->bd, rxq->bd_dma); |
| 688 |
rxq->bd_dma = 0; |
| 689 |
rxq->bd = NULL; |
| 690 |
|
| 691 |
rxq->rb_stts_dma = 0; |
| 692 |
rxq->rb_stts = NULL; |
| 693 |
|
| 694 |
if (rxq->used_bd) |
| 695 |
dma_free_coherent(trans->dev, |
| 696 |
iwl_pcie_used_bd_size(trans) * |
| 697 |
rxq->queue_size, |
| 698 |
rxq->used_bd, rxq->used_bd_dma); |
| 699 |
rxq->used_bd_dma = 0; |
| 700 |
rxq->used_bd = NULL; |
| 701 |
} |
| 702 |
|
| 703 |
static size_t iwl_pcie_rb_stts_size(struct iwl_trans *trans) |
| 704 |
{ |
| 705 |
bool use_rx_td = (trans->mac_cfg->device_family >= |
| 706 |
IWL_DEVICE_FAMILY_AX210); |
| 707 |
|
| 708 |
if (use_rx_td) |
| 709 |
return sizeof(__le16); |
| 710 |
|
| 711 |
return sizeof(struct iwl_rb_status); |
| 712 |
} |
| 713 |
|
| 714 |
static int iwl_pcie_alloc_rxq_dma(struct iwl_trans *trans, |
| 715 |
struct iwl_rxq *rxq) |
| 716 |
{ |
| 717 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 718 |
size_t rb_stts_size = iwl_pcie_rb_stts_size(trans); |
| 719 |
struct device *dev = trans->dev; |
| 720 |
int i; |
| 721 |
int free_size; |
| 722 |
|
| 723 |
spin_lock_init(&rxq->lock); |
| 724 |
if (trans->mac_cfg->mq_rx_supported) |
| 725 |
rxq->queue_size = iwl_trans_get_num_rbds(trans); |
| 726 |
else |
| 727 |
rxq->queue_size = RX_QUEUE_SIZE; |
| 728 |
|
| 729 |
free_size = iwl_pcie_free_bd_size(trans); |
| 730 |
|
| 731 |
/* |
| 732 |
* Allocate the circular buffer of Read Buffer Descriptors |
| 733 |
* (RBDs) |
| 734 |
*/ |
| 735 |
rxq->bd = dma_alloc_coherent(dev, free_size * rxq->queue_size, |
| 736 |
&rxq->bd_dma, GFP_KERNEL); |
| 737 |
if (!rxq->bd) |
| 738 |
goto err; |
| 739 |
|
| 740 |
if (trans->mac_cfg->mq_rx_supported) { |
| 741 |
rxq->used_bd = dma_alloc_coherent(dev, |
| 742 |
iwl_pcie_used_bd_size(trans) * |
| 743 |
rxq->queue_size, |
| 744 |
&rxq->used_bd_dma, |
| 745 |
GFP_KERNEL); |
| 746 |
if (!rxq->used_bd) |
| 747 |
goto err; |
| 748 |
} |
| 749 |
|
| 750 |
rxq->rb_stts = (u8 *)trans_pcie->base_rb_stts + rxq->id * rb_stts_size; |
| 751 |
rxq->rb_stts_dma = |
| 752 |
trans_pcie->base_rb_stts_dma + rxq->id * rb_stts_size; |
| 753 |
|
| 754 |
return 0; |
| 755 |
|
| 756 |
err: |
| 757 |
for (i = 0; i < trans->info.num_rxqs; i++) { |
| 758 |
struct iwl_rxq *rxq = &trans_pcie->rxq[i]; |
| 759 |
|
| 760 |
iwl_pcie_free_rxq_dma(trans, rxq); |
| 761 |
} |
| 762 |
|
| 763 |
return -ENOMEM; |
| 764 |
} |
| 765 |
|
| 766 |
static int iwl_pcie_rx_alloc(struct iwl_trans *trans) |
| 767 |
{ |
| 768 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 769 |
size_t rb_stts_size = iwl_pcie_rb_stts_size(trans); |
| 770 |
struct iwl_rb_allocator *rba = &trans_pcie->rba; |
| 771 |
int i, ret; |
| 772 |
|
| 773 |
if (WARN_ON(trans_pcie->rxq)) |
| 774 |
return -EINVAL; |
| 775 |
|
| 776 |
trans_pcie->rxq = kzalloc_objs(struct iwl_rxq, trans->info.num_rxqs); |
| 777 |
trans_pcie->rx_pool = kzalloc_objs(trans_pcie->rx_pool[0], |
| 778 |
RX_POOL_SIZE(trans_pcie->num_rx_bufs)); |
| 779 |
trans_pcie->global_table = |
| 780 |
kzalloc_objs(trans_pcie->global_table[0], |
| 781 |
RX_POOL_SIZE(trans_pcie->num_rx_bufs)); |
| 782 |
if (!trans_pcie->rxq || !trans_pcie->rx_pool || |
| 783 |
!trans_pcie->global_table) { |
| 784 |
ret = -ENOMEM; |
| 785 |
goto err; |
| 786 |
} |
| 787 |
|
| 788 |
spin_lock_init(&rba->lock); |
| 789 |
|
| 790 |
/* |
| 791 |
* Allocate the driver's pointer to receive buffer status. |
| 792 |
* Allocate for all queues continuously (HW requirement). |
| 793 |
*/ |
| 794 |
trans_pcie->base_rb_stts = |
| 795 |
dma_alloc_coherent(trans->dev, |
| 796 |
rb_stts_size * trans->info.num_rxqs, |
| 797 |
&trans_pcie->base_rb_stts_dma, |
| 798 |
GFP_KERNEL); |
| 799 |
if (!trans_pcie->base_rb_stts) { |
| 800 |
ret = -ENOMEM; |
| 801 |
goto err; |
| 802 |
} |
| 803 |
|
| 804 |
for (i = 0; i < trans->info.num_rxqs; i++) { |
| 805 |
struct iwl_rxq *rxq = &trans_pcie->rxq[i]; |
| 806 |
|
| 807 |
rxq->id = i; |
| 808 |
ret = iwl_pcie_alloc_rxq_dma(trans, rxq); |
| 809 |
if (ret) |
| 810 |
goto err; |
| 811 |
} |
| 812 |
return 0; |
| 813 |
|
| 814 |
err: |
| 815 |
if (trans_pcie->base_rb_stts) { |
| 816 |
dma_free_coherent(trans->dev, |
| 817 |
rb_stts_size * trans->info.num_rxqs, |
| 818 |
trans_pcie->base_rb_stts, |
| 819 |
trans_pcie->base_rb_stts_dma); |
| 820 |
trans_pcie->base_rb_stts = NULL; |
| 821 |
trans_pcie->base_rb_stts_dma = 0; |
| 822 |
} |
| 823 |
kfree(trans_pcie->rx_pool); |
| 824 |
trans_pcie->rx_pool = NULL; |
| 825 |
kfree(trans_pcie->global_table); |
| 826 |
trans_pcie->global_table = NULL; |
| 827 |
kfree(trans_pcie->rxq); |
| 828 |
trans_pcie->rxq = NULL; |
| 829 |
|
| 830 |
return ret; |
| 831 |
} |
| 832 |
|
| 833 |
static void iwl_pcie_rx_hw_init(struct iwl_trans *trans, struct iwl_rxq *rxq) |
| 834 |
{ |
| 835 |
u32 rb_size; |
| 836 |
const u32 rfdnlog = RX_QUEUE_SIZE_LOG; /* 256 RBDs */ |
| 837 |
|
| 838 |
switch (trans->conf.rx_buf_size) { |
| 839 |
case IWL_AMSDU_4K: |
| 840 |
rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K; |
| 841 |
break; |
| 842 |
case IWL_AMSDU_8K: |
| 843 |
rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_8K; |
| 844 |
break; |
| 845 |
case IWL_AMSDU_12K: |
| 846 |
rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_12K; |
| 847 |
break; |
| 848 |
default: |
| 849 |
WARN_ON(1); |
| 850 |
rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K; |
| 851 |
} |
| 852 |
|
| 853 |
if (!iwl_trans_grab_nic_access(trans)) |
| 854 |
return; |
| 855 |
|
| 856 |
/* Stop Rx DMA */ |
| 857 |
iwl_write32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0); |
| 858 |
/* reset and flush pointers */ |
| 859 |
iwl_write32(trans, FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0); |
| 860 |
iwl_write32(trans, FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0); |
| 861 |
iwl_write32(trans, FH_RSCSR_CHNL0_RDPTR, 0); |
| 862 |
|
| 863 |
/* Reset driver's Rx queue write index */ |
| 864 |
iwl_write32(trans, FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0); |
| 865 |
|
| 866 |
/* Tell device where to find RBD circular buffer in DRAM */ |
| 867 |
iwl_write32(trans, FH_RSCSR_CHNL0_RBDCB_BASE_REG, |
| 868 |
(u32)(rxq->bd_dma >> 8)); |
| 869 |
|
| 870 |
/* Tell device where in DRAM to update its Rx status */ |
| 871 |
iwl_write32(trans, FH_RSCSR_CHNL0_STTS_WPTR_REG, |
| 872 |
rxq->rb_stts_dma >> 4); |
| 873 |
|
| 874 |
/* Enable Rx DMA |
| 875 |
* FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in |
| 876 |
* the credit mechanism in 5000 HW RX FIFO |
| 877 |
* Direct rx interrupts to hosts |
| 878 |
* Rx buffer size 4 or 8k or 12k |
| 879 |
* RB timeout 0x10 |
| 880 |
* 256 RBDs |
| 881 |
*/ |
| 882 |
iwl_write32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, |
| 883 |
FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL | |
| 884 |
FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY | |
| 885 |
FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL | |
| 886 |
rb_size | |
| 887 |
(RX_RB_TIMEOUT << FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) | |
| 888 |
(rfdnlog << FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS)); |
| 889 |
|
| 890 |
iwl_trans_release_nic_access(trans); |
| 891 |
|
| 892 |
/* Set interrupt coalescing timer to default (2048 usecs) */ |
| 893 |
iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF); |
| 894 |
|
| 895 |
/* W/A for interrupt coalescing bug in 7260 and 3160 */ |
| 896 |
if (trans->cfg->host_interrupt_operation_mode) |
| 897 |
iwl_set_bit(trans, CSR_INT_COALESCING, IWL_HOST_INT_OPER_MODE); |
| 898 |
} |
| 899 |
|
| 900 |
static void iwl_pcie_rx_mq_hw_init(struct iwl_trans *trans) |
| 901 |
{ |
| 902 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 903 |
u32 rb_size, enabled = 0; |
| 904 |
int i; |
| 905 |
|
| 906 |
switch (trans->conf.rx_buf_size) { |
| 907 |
case IWL_AMSDU_2K: |
| 908 |
rb_size = RFH_RXF_DMA_RB_SIZE_2K; |
| 909 |
break; |
| 910 |
case IWL_AMSDU_4K: |
| 911 |
rb_size = RFH_RXF_DMA_RB_SIZE_4K; |
| 912 |
break; |
| 913 |
case IWL_AMSDU_8K: |
| 914 |
rb_size = RFH_RXF_DMA_RB_SIZE_8K; |
| 915 |
break; |
| 916 |
case IWL_AMSDU_12K: |
| 917 |
rb_size = RFH_RXF_DMA_RB_SIZE_12K; |
| 918 |
break; |
| 919 |
default: |
| 920 |
WARN_ON(1); |
| 921 |
rb_size = RFH_RXF_DMA_RB_SIZE_4K; |
| 922 |
} |
| 923 |
|
| 924 |
if (!iwl_trans_grab_nic_access(trans)) |
| 925 |
return; |
| 926 |
|
| 927 |
/* Stop Rx DMA */ |
| 928 |
iwl_write_prph_no_grab(trans, RFH_RXF_DMA_CFG, 0); |
| 929 |
/* disable free amd used rx queue operation */ |
| 930 |
iwl_write_prph_no_grab(trans, RFH_RXF_RXQ_ACTIVE, 0); |
| 931 |
|
| 932 |
for (i = 0; i < trans->info.num_rxqs; i++) { |
| 933 |
/* Tell device where to find RBD free table in DRAM */ |
| 934 |
iwl_write_prph64_no_grab(trans, |
| 935 |
RFH_Q_FRBDCB_BA_LSB(i), |
| 936 |
trans_pcie->rxq[i].bd_dma); |
| 937 |
/* Tell device where to find RBD used table in DRAM */ |
| 938 |
iwl_write_prph64_no_grab(trans, |
| 939 |
RFH_Q_URBDCB_BA_LSB(i), |
| 940 |
trans_pcie->rxq[i].used_bd_dma); |
| 941 |
/* Tell device where in DRAM to update its Rx status */ |
| 942 |
iwl_write_prph64_no_grab(trans, |
| 943 |
RFH_Q_URBD_STTS_WPTR_LSB(i), |
| 944 |
trans_pcie->rxq[i].rb_stts_dma); |
| 945 |
/* Reset device indice tables */ |
| 946 |
iwl_write_prph_no_grab(trans, RFH_Q_FRBDCB_WIDX(i), 0); |
| 947 |
iwl_write_prph_no_grab(trans, RFH_Q_FRBDCB_RIDX(i), 0); |
| 948 |
iwl_write_prph_no_grab(trans, RFH_Q_URBDCB_WIDX(i), 0); |
| 949 |
|
| 950 |
enabled |= BIT(i) | BIT(i + 16); |
| 951 |
} |
| 952 |
|
| 953 |
/* |
| 954 |
* Enable Rx DMA |
| 955 |
* Rx buffer size 4 or 8k or 12k |
| 956 |
* Min RB size 4 or 8 |
| 957 |
* Drop frames that exceed RB size |
| 958 |
* 512 RBDs |
| 959 |
*/ |
| 960 |
iwl_write_prph_no_grab(trans, RFH_RXF_DMA_CFG, |
| 961 |
RFH_DMA_EN_ENABLE_VAL | rb_size | |
| 962 |
RFH_RXF_DMA_MIN_RB_4_8 | |
| 963 |
RFH_RXF_DMA_DROP_TOO_LARGE_MASK | |
| 964 |
RFH_RXF_DMA_RBDCB_SIZE_512); |
| 965 |
|
| 966 |
/* |
| 967 |
* Activate DMA snooping. |
| 968 |
* Set RX DMA chunk size to 64B for IOSF and 128B for PCIe |
| 969 |
* Default queue is 0 |
| 970 |
*/ |
| 971 |
iwl_write_prph_no_grab(trans, RFH_GEN_CFG, |
| 972 |
RFH_GEN_CFG_RFH_DMA_SNOOP | |
| 973 |
RFH_GEN_CFG_VAL(DEFAULT_RXQ_NUM, 0) | |
| 974 |
RFH_GEN_CFG_SERVICE_DMA_SNOOP | |
| 975 |
RFH_GEN_CFG_VAL(RB_CHUNK_SIZE, |
| 976 |
trans->mac_cfg->integrated ? |
| 977 |
RFH_GEN_CFG_RB_CHUNK_SIZE_64 : |
| 978 |
RFH_GEN_CFG_RB_CHUNK_SIZE_128)); |
| 979 |
/* Enable the relevant rx queues */ |
| 980 |
iwl_write_prph_no_grab(trans, RFH_RXF_RXQ_ACTIVE, enabled); |
| 981 |
|
| 982 |
iwl_trans_release_nic_access(trans); |
| 983 |
|
| 984 |
/* Set interrupt coalescing timer to default (2048 usecs) */ |
| 985 |
iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF); |
| 986 |
} |
| 987 |
|
| 988 |
void iwl_pcie_rx_init_rxb_lists(struct iwl_rxq *rxq) |
| 989 |
{ |
| 990 |
lockdep_assert_held(&rxq->lock); |
| 991 |
|
| 992 |
INIT_LIST_HEAD(&rxq->rx_free); |
| 993 |
INIT_LIST_HEAD(&rxq->rx_used); |
| 994 |
rxq->free_count = 0; |
| 995 |
rxq->used_count = 0; |
| 996 |
} |
| 997 |
|
| 998 |
static int iwl_pcie_rx_handle(struct iwl_trans *trans, int queue, int budget); |
| 999 |
|
| 1000 |
static inline struct iwl_trans_pcie *iwl_netdev_to_trans_pcie(struct net_device *dev) |
| 1001 |
{ |
| 1002 |
return *(struct iwl_trans_pcie **)netdev_priv(dev); |
| 1003 |
} |
| 1004 |
|
| 1005 |
static int iwl_pcie_napi_poll(struct napi_struct *napi, int budget) |
| 1006 |
{ |
| 1007 |
struct iwl_rxq *rxq = container_of(napi, struct iwl_rxq, napi); |
| 1008 |
struct iwl_trans_pcie *trans_pcie; |
| 1009 |
struct iwl_trans *trans; |
| 1010 |
int ret; |
| 1011 |
|
| 1012 |
trans_pcie = iwl_netdev_to_trans_pcie(napi->dev); |
| 1013 |
trans = trans_pcie->trans; |
| 1014 |
|
| 1015 |
ret = iwl_pcie_rx_handle(trans, rxq->id, budget); |
| 1016 |
|
| 1017 |
IWL_DEBUG_ISR(trans, "[%d] handled %d, budget %d\n", |
| 1018 |
rxq->id, ret, budget); |
| 1019 |
|
| 1020 |
if (ret < budget) { |
| 1021 |
spin_lock(&trans_pcie->irq_lock); |
| 1022 |
if (test_bit(STATUS_INT_ENABLED, &trans->status)) |
| 1023 |
_iwl_enable_interrupts(trans); |
| 1024 |
spin_unlock(&trans_pcie->irq_lock); |
| 1025 |
|
| 1026 |
napi_complete_done(&rxq->napi, ret); |
| 1027 |
} |
| 1028 |
|
| 1029 |
return ret; |
| 1030 |
} |
| 1031 |
|
| 1032 |
static int iwl_pcie_napi_poll_msix(struct napi_struct *napi, int budget) |
| 1033 |
{ |
| 1034 |
struct iwl_rxq *rxq = container_of(napi, struct iwl_rxq, napi); |
| 1035 |
struct iwl_trans_pcie *trans_pcie; |
| 1036 |
struct iwl_trans *trans; |
| 1037 |
int ret; |
| 1038 |
|
| 1039 |
trans_pcie = iwl_netdev_to_trans_pcie(napi->dev); |
| 1040 |
trans = trans_pcie->trans; |
| 1041 |
|
| 1042 |
ret = iwl_pcie_rx_handle(trans, rxq->id, budget); |
| 1043 |
IWL_DEBUG_ISR(trans, "[%d] handled %d, budget %d\n", rxq->id, ret, |
| 1044 |
budget); |
| 1045 |
|
| 1046 |
if (ret < budget) { |
| 1047 |
int irq_line = rxq->id; |
| 1048 |
|
| 1049 |
/* FIRST_RSS is shared with line 0 */ |
| 1050 |
if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS && |
| 1051 |
rxq->id == 1) |
| 1052 |
irq_line = 0; |
| 1053 |
|
| 1054 |
spin_lock(&trans_pcie->irq_lock); |
| 1055 |
iwl_pcie_clear_irq(trans, irq_line); |
| 1056 |
spin_unlock(&trans_pcie->irq_lock); |
| 1057 |
|
| 1058 |
napi_complete_done(&rxq->napi, ret); |
| 1059 |
} |
| 1060 |
|
| 1061 |
return ret; |
| 1062 |
} |
| 1063 |
|
| 1064 |
void iwl_pcie_rx_napi_sync(struct iwl_trans *trans) |
| 1065 |
{ |
| 1066 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1067 |
int i; |
| 1068 |
|
| 1069 |
if (unlikely(!trans_pcie->rxq)) |
| 1070 |
return; |
| 1071 |
|
| 1072 |
for (i = 0; i < trans->info.num_rxqs; i++) { |
| 1073 |
struct iwl_rxq *rxq = &trans_pcie->rxq[i]; |
| 1074 |
|
| 1075 |
if (rxq && rxq->napi.poll) |
| 1076 |
napi_synchronize(&rxq->napi); |
| 1077 |
} |
| 1078 |
} |
| 1079 |
|
| 1080 |
static int _iwl_pcie_rx_init(struct iwl_trans *trans) |
| 1081 |
{ |
| 1082 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1083 |
struct iwl_rxq *def_rxq; |
| 1084 |
struct iwl_rb_allocator *rba = &trans_pcie->rba; |
| 1085 |
int i, err, queue_size, allocator_pool_size, num_alloc; |
| 1086 |
|
| 1087 |
if (!trans_pcie->rxq) { |
| 1088 |
err = iwl_pcie_rx_alloc(trans); |
| 1089 |
if (err) |
| 1090 |
return err; |
| 1091 |
} |
| 1092 |
def_rxq = trans_pcie->rxq; |
| 1093 |
|
| 1094 |
cancel_work_sync(&rba->rx_alloc); |
| 1095 |
|
| 1096 |
spin_lock_bh(&rba->lock); |
| 1097 |
atomic_set(&rba->req_pending, 0); |
| 1098 |
atomic_set(&rba->req_ready, 0); |
| 1099 |
INIT_LIST_HEAD(&rba->rbd_allocated); |
| 1100 |
INIT_LIST_HEAD(&rba->rbd_empty); |
| 1101 |
spin_unlock_bh(&rba->lock); |
| 1102 |
|
| 1103 |
/* free all first - we overwrite everything here */ |
| 1104 |
iwl_pcie_free_rbs_pool(trans); |
| 1105 |
|
| 1106 |
for (i = 0; i < RX_QUEUE_SIZE; i++) |
| 1107 |
def_rxq->queue[i] = NULL; |
| 1108 |
|
| 1109 |
for (i = 0; i < trans->info.num_rxqs; i++) { |
| 1110 |
struct iwl_rxq *rxq = &trans_pcie->rxq[i]; |
| 1111 |
|
| 1112 |
spin_lock_bh(&rxq->lock); |
| 1113 |
/* |
| 1114 |
* Set read write pointer to reflect that we have processed |
| 1115 |
* and used all buffers, but have not restocked the Rx queue |
| 1116 |
* with fresh buffers |
| 1117 |
*/ |
| 1118 |
rxq->read = 0; |
| 1119 |
rxq->write = 0; |
| 1120 |
rxq->write_actual = 0; |
| 1121 |
memset(rxq->rb_stts, 0, |
| 1122 |
(trans->mac_cfg->device_family >= |
| 1123 |
IWL_DEVICE_FAMILY_AX210) ? |
| 1124 |
sizeof(__le16) : sizeof(struct iwl_rb_status)); |
| 1125 |
|
| 1126 |
iwl_pcie_rx_init_rxb_lists(rxq); |
| 1127 |
|
| 1128 |
spin_unlock_bh(&rxq->lock); |
| 1129 |
|
| 1130 |
if (!rxq->napi.poll) { |
| 1131 |
int (*poll)(struct napi_struct *, int) = iwl_pcie_napi_poll; |
| 1132 |
|
| 1133 |
if (trans_pcie->msix_enabled) |
| 1134 |
poll = iwl_pcie_napi_poll_msix; |
| 1135 |
|
| 1136 |
netif_napi_add(trans_pcie->napi_dev, &rxq->napi, |
| 1137 |
poll); |
| 1138 |
napi_enable(&rxq->napi); |
| 1139 |
} |
| 1140 |
|
| 1141 |
} |
| 1142 |
|
| 1143 |
/* move the pool to the default queue and allocator ownerships */ |
| 1144 |
queue_size = trans->mac_cfg->mq_rx_supported ? |
| 1145 |
trans_pcie->num_rx_bufs - 1 : RX_QUEUE_SIZE; |
| 1146 |
allocator_pool_size = trans->info.num_rxqs * |
| 1147 |
(RX_CLAIM_REQ_ALLOC - RX_POST_REQ_ALLOC); |
| 1148 |
num_alloc = queue_size + allocator_pool_size; |
| 1149 |
|
| 1150 |
for (i = 0; i < num_alloc; i++) { |
| 1151 |
struct iwl_rx_mem_buffer *rxb = &trans_pcie->rx_pool[i]; |
| 1152 |
|
| 1153 |
if (i < allocator_pool_size) |
| 1154 |
list_add(&rxb->list, &rba->rbd_empty); |
| 1155 |
else |
| 1156 |
list_add(&rxb->list, &def_rxq->rx_used); |
| 1157 |
trans_pcie->global_table[i] = rxb; |
| 1158 |
rxb->vid = (u16)(i + 1); |
| 1159 |
rxb->invalid = true; |
| 1160 |
} |
| 1161 |
|
| 1162 |
iwl_pcie_rxq_alloc_rbs(trans, GFP_KERNEL, def_rxq); |
| 1163 |
|
| 1164 |
return 0; |
| 1165 |
} |
| 1166 |
|
| 1167 |
int iwl_pcie_rx_init(struct iwl_trans *trans) |
| 1168 |
{ |
| 1169 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1170 |
int ret = _iwl_pcie_rx_init(trans); |
| 1171 |
|
| 1172 |
if (ret) |
| 1173 |
return ret; |
| 1174 |
|
| 1175 |
if (trans->mac_cfg->mq_rx_supported) |
| 1176 |
iwl_pcie_rx_mq_hw_init(trans); |
| 1177 |
else |
| 1178 |
iwl_pcie_rx_hw_init(trans, trans_pcie->rxq); |
| 1179 |
|
| 1180 |
iwl_pcie_rxq_restock(trans, trans_pcie->rxq); |
| 1181 |
|
| 1182 |
spin_lock_bh(&trans_pcie->rxq->lock); |
| 1183 |
iwl_pcie_rxq_inc_wr_ptr(trans, trans_pcie->rxq); |
| 1184 |
spin_unlock_bh(&trans_pcie->rxq->lock); |
| 1185 |
|
| 1186 |
return 0; |
| 1187 |
} |
| 1188 |
|
| 1189 |
int iwl_pcie_gen2_rx_init(struct iwl_trans *trans) |
| 1190 |
{ |
| 1191 |
/* Set interrupt coalescing timer to default (2048 usecs) */ |
| 1192 |
iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF); |
| 1193 |
|
| 1194 |
/* |
| 1195 |
* We don't configure the RFH. |
| 1196 |
* Restock will be done at alive, after firmware configured the RFH. |
| 1197 |
*/ |
| 1198 |
return _iwl_pcie_rx_init(trans); |
| 1199 |
} |
| 1200 |
|
| 1201 |
void iwl_pcie_rx_free(struct iwl_trans *trans) |
| 1202 |
{ |
| 1203 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1204 |
size_t rb_stts_size = iwl_pcie_rb_stts_size(trans); |
| 1205 |
struct iwl_rb_allocator *rba = &trans_pcie->rba; |
| 1206 |
int i; |
| 1207 |
|
| 1208 |
/* |
| 1209 |
* if rxq is NULL, it means that nothing has been allocated, |
| 1210 |
* exit now |
| 1211 |
*/ |
| 1212 |
if (!trans_pcie->rxq) { |
| 1213 |
IWL_DEBUG_INFO(trans, "Free NULL rx context\n"); |
| 1214 |
return; |
| 1215 |
} |
| 1216 |
|
| 1217 |
cancel_work_sync(&rba->rx_alloc); |
| 1218 |
|
| 1219 |
iwl_pcie_free_rbs_pool(trans); |
| 1220 |
|
| 1221 |
if (trans_pcie->base_rb_stts) { |
| 1222 |
dma_free_coherent(trans->dev, |
| 1223 |
rb_stts_size * trans->info.num_rxqs, |
| 1224 |
trans_pcie->base_rb_stts, |
| 1225 |
trans_pcie->base_rb_stts_dma); |
| 1226 |
trans_pcie->base_rb_stts = NULL; |
| 1227 |
trans_pcie->base_rb_stts_dma = 0; |
| 1228 |
} |
| 1229 |
|
| 1230 |
for (i = 0; i < trans->info.num_rxqs; i++) { |
| 1231 |
struct iwl_rxq *rxq = &trans_pcie->rxq[i]; |
| 1232 |
|
| 1233 |
iwl_pcie_free_rxq_dma(trans, rxq); |
| 1234 |
|
| 1235 |
if (rxq->napi.poll) { |
| 1236 |
napi_disable(&rxq->napi); |
| 1237 |
netif_napi_del(&rxq->napi); |
| 1238 |
} |
| 1239 |
} |
| 1240 |
kfree(trans_pcie->rx_pool); |
| 1241 |
kfree(trans_pcie->global_table); |
| 1242 |
kfree(trans_pcie->rxq); |
| 1243 |
|
| 1244 |
if (trans_pcie->alloc_page) |
| 1245 |
__free_pages(trans_pcie->alloc_page, trans_pcie->rx_page_order); |
| 1246 |
} |
| 1247 |
|
| 1248 |
static void iwl_pcie_rx_move_to_allocator(struct iwl_rxq *rxq, |
| 1249 |
struct iwl_rb_allocator *rba) |
| 1250 |
{ |
| 1251 |
spin_lock(&rba->lock); |
| 1252 |
list_splice_tail_init(&rxq->rx_used, &rba->rbd_empty); |
| 1253 |
spin_unlock(&rba->lock); |
| 1254 |
} |
| 1255 |
|
| 1256 |
/* |
| 1257 |
* iwl_pcie_rx_reuse_rbd - Recycle used RBDs |
| 1258 |
* |
| 1259 |
* Called when a RBD can be reused. The RBD is transferred to the allocator. |
| 1260 |
* When there are 2 empty RBDs - a request for allocation is posted |
| 1261 |
*/ |
| 1262 |
static void iwl_pcie_rx_reuse_rbd(struct iwl_trans *trans, |
| 1263 |
struct iwl_rx_mem_buffer *rxb, |
| 1264 |
struct iwl_rxq *rxq, bool emergency) |
| 1265 |
{ |
| 1266 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1267 |
struct iwl_rb_allocator *rba = &trans_pcie->rba; |
| 1268 |
|
| 1269 |
/* Move the RBD to the used list, will be moved to allocator in batches |
| 1270 |
* before claiming or posting a request*/ |
| 1271 |
list_add_tail(&rxb->list, &rxq->rx_used); |
| 1272 |
|
| 1273 |
if (unlikely(emergency)) |
| 1274 |
return; |
| 1275 |
|
| 1276 |
/* Count the allocator owned RBDs */ |
| 1277 |
rxq->used_count++; |
| 1278 |
|
| 1279 |
/* If we have RX_POST_REQ_ALLOC new released rx buffers - |
| 1280 |
* issue a request for allocator. Modulo RX_CLAIM_REQ_ALLOC is |
| 1281 |
* used for the case we failed to claim RX_CLAIM_REQ_ALLOC, |
| 1282 |
* after but we still need to post another request. |
| 1283 |
*/ |
| 1284 |
if ((rxq->used_count % RX_CLAIM_REQ_ALLOC) == RX_POST_REQ_ALLOC) { |
| 1285 |
/* Move the 2 RBDs to the allocator ownership. |
| 1286 |
Allocator has another 6 from pool for the request completion*/ |
| 1287 |
iwl_pcie_rx_move_to_allocator(rxq, rba); |
| 1288 |
|
| 1289 |
atomic_inc(&rba->req_pending); |
| 1290 |
queue_work(rba->alloc_wq, &rba->rx_alloc); |
| 1291 |
} |
| 1292 |
} |
| 1293 |
|
| 1294 |
static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans, |
| 1295 |
struct iwl_rxq *rxq, |
| 1296 |
struct iwl_rx_mem_buffer *rxb, |
| 1297 |
bool emergency, |
| 1298 |
int i) |
| 1299 |
{ |
| 1300 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1301 |
struct iwl_txq *txq = trans_pcie->txqs.txq[trans->conf.cmd_queue]; |
| 1302 |
bool page_stolen = false; |
| 1303 |
int max_len = trans_pcie->rx_buf_bytes; |
| 1304 |
u32 offset = 0; |
| 1305 |
|
| 1306 |
if (WARN_ON(!rxb)) |
| 1307 |
return; |
| 1308 |
|
| 1309 |
dma_unmap_page(trans->dev, rxb->page_dma, max_len, DMA_FROM_DEVICE); |
| 1310 |
|
| 1311 |
while (offset + sizeof(u32) + sizeof(struct iwl_cmd_header) < max_len) { |
| 1312 |
struct iwl_rx_packet *pkt; |
| 1313 |
bool reclaim; |
| 1314 |
int len; |
| 1315 |
struct iwl_rx_cmd_buffer rxcb = { |
| 1316 |
._offset = rxb->offset + offset, |
| 1317 |
._rx_page_order = trans_pcie->rx_page_order, |
| 1318 |
._page = rxb->page, |
| 1319 |
._page_stolen = false, |
| 1320 |
.truesize = max_len, |
| 1321 |
}; |
| 1322 |
|
| 1323 |
pkt = rxb_addr(&rxcb); |
| 1324 |
|
| 1325 |
if (pkt->len_n_flags == cpu_to_le32(FH_RSCSR_FRAME_INVALID)) { |
| 1326 |
IWL_DEBUG_RX(trans, |
| 1327 |
"Q %d: RB end marker at offset %d\n", |
| 1328 |
rxq->id, offset); |
| 1329 |
break; |
| 1330 |
} |
| 1331 |
|
| 1332 |
WARN((le32_to_cpu(pkt->len_n_flags) & FH_RSCSR_RXQ_MASK) >> |
| 1333 |
FH_RSCSR_RXQ_POS != rxq->id, |
| 1334 |
"frame on invalid queue - is on %d and indicates %d\n", |
| 1335 |
rxq->id, |
| 1336 |
(le32_to_cpu(pkt->len_n_flags) & FH_RSCSR_RXQ_MASK) >> |
| 1337 |
FH_RSCSR_RXQ_POS); |
| 1338 |
|
| 1339 |
IWL_DEBUG_RX(trans, |
| 1340 |
"Q %d: cmd at offset %d: %s (%.2x.%2x, seq 0x%x)\n", |
| 1341 |
rxq->id, offset, |
| 1342 |
iwl_get_cmd_string(trans, |
| 1343 |
WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd)), |
| 1344 |
pkt->hdr.group_id, pkt->hdr.cmd, |
| 1345 |
le16_to_cpu(pkt->hdr.sequence)); |
| 1346 |
|
| 1347 |
len = iwl_rx_packet_len(pkt); |
| 1348 |
len += sizeof(u32); /* account for status word */ |
| 1349 |
|
| 1350 |
offset += ALIGN(len, FH_RSCSR_FRAME_ALIGN); |
| 1351 |
|
| 1352 |
/* check that what the device tells us made sense */ |
| 1353 |
if (len < sizeof(*pkt) || offset > max_len) |
| 1354 |
break; |
| 1355 |
|
| 1356 |
maybe_trace_iwlwifi_dev_rx(trans, pkt, len); |
| 1357 |
|
| 1358 |
/* Reclaim a command buffer only if this packet is a response |
| 1359 |
* to a (driver-originated) command. |
| 1360 |
* If the packet (e.g. Rx frame) originated from uCode, |
| 1361 |
* there is no command buffer to reclaim. |
| 1362 |
* Ucode should set SEQ_RX_FRAME bit if ucode-originated, |
| 1363 |
* but apparently a few don't get set; catch them here. */ |
| 1364 |
reclaim = !(pkt->hdr.sequence & SEQ_RX_FRAME); |
| 1365 |
if (reclaim && !pkt->hdr.group_id) { |
| 1366 |
int i; |
| 1367 |
|
| 1368 |
for (i = 0; i < trans->conf.n_no_reclaim_cmds; i++) { |
| 1369 |
if (trans->conf.no_reclaim_cmds[i] == |
| 1370 |
pkt->hdr.cmd) { |
| 1371 |
reclaim = false; |
| 1372 |
break; |
| 1373 |
} |
| 1374 |
} |
| 1375 |
} |
| 1376 |
|
| 1377 |
if (rxq->id == IWL_DEFAULT_RX_QUEUE) |
| 1378 |
iwl_op_mode_rx(trans->op_mode, &rxq->napi, |
| 1379 |
&rxcb); |
| 1380 |
else |
| 1381 |
iwl_op_mode_rx_rss(trans->op_mode, &rxq->napi, |
| 1382 |
&rxcb, rxq->id); |
| 1383 |
|
| 1384 |
/* |
| 1385 |
* After here, we should always check rxcb._page_stolen, |
| 1386 |
* if it is true then one of the handlers took the page. |
| 1387 |
*/ |
| 1388 |
|
| 1389 |
if (reclaim && txq) { |
| 1390 |
u16 sequence = le16_to_cpu(pkt->hdr.sequence); |
| 1391 |
int index = SEQ_TO_INDEX(sequence); |
| 1392 |
int cmd_index = iwl_txq_get_cmd_index(txq, index); |
| 1393 |
|
| 1394 |
kfree_sensitive(txq->entries[cmd_index].free_buf); |
| 1395 |
txq->entries[cmd_index].free_buf = NULL; |
| 1396 |
|
| 1397 |
/* Invoke any callbacks, transfer the buffer to caller, |
| 1398 |
* and fire off the (possibly) blocking |
| 1399 |
* iwl_trans_send_cmd() |
| 1400 |
* as we reclaim the driver command queue */ |
| 1401 |
if (!rxcb._page_stolen) |
| 1402 |
iwl_pcie_hcmd_complete(trans, &rxcb); |
| 1403 |
else |
| 1404 |
IWL_WARN(trans, "Claim null rxb?\n"); |
| 1405 |
} |
| 1406 |
|
| 1407 |
page_stolen |= rxcb._page_stolen; |
| 1408 |
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) |
| 1409 |
break; |
| 1410 |
} |
| 1411 |
|
| 1412 |
/* page was stolen from us -- free our reference */ |
| 1413 |
if (page_stolen) { |
| 1414 |
__free_pages(rxb->page, trans_pcie->rx_page_order); |
| 1415 |
rxb->page = NULL; |
| 1416 |
} |
| 1417 |
|
| 1418 |
/* Reuse the page if possible. For notification packets and |
| 1419 |
* SKBs that fail to Rx correctly, add them back into the |
| 1420 |
* rx_free list for reuse later. */ |
| 1421 |
if (rxb->page != NULL) { |
| 1422 |
rxb->page_dma = |
| 1423 |
dma_map_page(trans->dev, rxb->page, rxb->offset, |
| 1424 |
trans_pcie->rx_buf_bytes, |
| 1425 |
DMA_FROM_DEVICE); |
| 1426 |
if (dma_mapping_error(trans->dev, rxb->page_dma)) { |
| 1427 |
/* |
| 1428 |
* free the page(s) as well to not break |
| 1429 |
* the invariant that the items on the used |
| 1430 |
* list have no page(s) |
| 1431 |
*/ |
| 1432 |
__free_pages(rxb->page, trans_pcie->rx_page_order); |
| 1433 |
rxb->page = NULL; |
| 1434 |
iwl_pcie_rx_reuse_rbd(trans, rxb, rxq, emergency); |
| 1435 |
} else { |
| 1436 |
list_add_tail(&rxb->list, &rxq->rx_free); |
| 1437 |
rxq->free_count++; |
| 1438 |
} |
| 1439 |
} else |
| 1440 |
iwl_pcie_rx_reuse_rbd(trans, rxb, rxq, emergency); |
| 1441 |
} |
| 1442 |
|
| 1443 |
static struct iwl_rx_mem_buffer *iwl_pcie_get_rxb(struct iwl_trans *trans, |
| 1444 |
struct iwl_rxq *rxq, int i, |
| 1445 |
bool *join) |
| 1446 |
{ |
| 1447 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1448 |
struct iwl_rx_mem_buffer *rxb; |
| 1449 |
u16 vid; |
| 1450 |
|
| 1451 |
BUILD_BUG_ON(sizeof(struct iwl_rx_completion_desc) != 32); |
| 1452 |
BUILD_BUG_ON(sizeof(struct iwl_rx_completion_desc_bz) != 4); |
| 1453 |
|
| 1454 |
if (!trans->mac_cfg->mq_rx_supported) { |
| 1455 |
rxb = rxq->queue[i]; |
| 1456 |
rxq->queue[i] = NULL; |
| 1457 |
return rxb; |
| 1458 |
} |
| 1459 |
|
| 1460 |
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) { |
| 1461 |
struct iwl_rx_completion_desc_bz *cd = rxq->used_bd; |
| 1462 |
|
| 1463 |
vid = le16_to_cpu(cd[i].rbid); |
| 1464 |
*join = cd[i].flags & IWL_RX_CD_FLAGS_FRAGMENTED; |
| 1465 |
} else if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { |
| 1466 |
struct iwl_rx_completion_desc *cd = rxq->used_bd; |
| 1467 |
|
| 1468 |
vid = le16_to_cpu(cd[i].rbid); |
| 1469 |
*join = cd[i].flags & IWL_RX_CD_FLAGS_FRAGMENTED; |
| 1470 |
} else { |
| 1471 |
__le32 *cd = rxq->used_bd; |
| 1472 |
|
| 1473 |
vid = le32_to_cpu(cd[i]) & 0x0FFF; /* 12-bit VID */ |
| 1474 |
} |
| 1475 |
|
| 1476 |
if (!vid || vid > RX_POOL_SIZE(trans_pcie->num_rx_bufs)) |
| 1477 |
goto out_err; |
| 1478 |
|
| 1479 |
rxb = trans_pcie->global_table[vid - 1]; |
| 1480 |
if (rxb->invalid) |
| 1481 |
goto out_err; |
| 1482 |
|
| 1483 |
IWL_DEBUG_RX(trans, "Got virtual RB ID %u\n", (u32)rxb->vid); |
| 1484 |
|
| 1485 |
rxb->invalid = true; |
| 1486 |
|
| 1487 |
return rxb; |
| 1488 |
|
| 1489 |
out_err: |
| 1490 |
WARN(1, "Invalid rxb from HW %u\n", (u32)vid); |
| 1491 |
iwl_force_nmi(trans); |
| 1492 |
return NULL; |
| 1493 |
} |
| 1494 |
|
| 1495 |
/* |
| 1496 |
* iwl_pcie_rx_handle - Main entry function for receiving responses from fw |
| 1497 |
*/ |
| 1498 |
static int iwl_pcie_rx_handle(struct iwl_trans *trans, int queue, int budget) |
| 1499 |
{ |
| 1500 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1501 |
struct iwl_rxq *rxq; |
| 1502 |
u32 r, i, count = 0, handled = 0; |
| 1503 |
bool emergency = false; |
| 1504 |
|
| 1505 |
if (WARN_ON_ONCE(!trans_pcie->rxq || !trans_pcie->rxq[queue].bd)) |
| 1506 |
return budget; |
| 1507 |
|
| 1508 |
rxq = &trans_pcie->rxq[queue]; |
| 1509 |
|
| 1510 |
restart: |
| 1511 |
spin_lock(&rxq->lock); |
| 1512 |
/* uCode's read index (stored in shared DRAM) indicates the last Rx |
| 1513 |
* buffer that the driver may process (last buffer filled by ucode). */ |
| 1514 |
r = iwl_get_closed_rb_stts(trans, rxq); |
| 1515 |
i = rxq->read; |
| 1516 |
|
| 1517 |
/* W/A 9000 device step A0 wrap-around bug */ |
| 1518 |
r &= (rxq->queue_size - 1); |
| 1519 |
|
| 1520 |
/* Rx interrupt, but nothing sent from uCode */ |
| 1521 |
if (i == r) |
| 1522 |
IWL_DEBUG_RX(trans, "Q %d: HW = SW = %d\n", rxq->id, r); |
| 1523 |
|
| 1524 |
while (i != r && ++handled < budget) { |
| 1525 |
struct iwl_rb_allocator *rba = &trans_pcie->rba; |
| 1526 |
struct iwl_rx_mem_buffer *rxb; |
| 1527 |
/* number of RBDs still waiting for page allocation */ |
| 1528 |
u32 rb_pending_alloc = |
| 1529 |
atomic_read(&trans_pcie->rba.req_pending) * |
| 1530 |
RX_CLAIM_REQ_ALLOC; |
| 1531 |
bool join = false; |
| 1532 |
|
| 1533 |
if (unlikely(rb_pending_alloc >= rxq->queue_size / 2 && |
| 1534 |
!emergency)) { |
| 1535 |
iwl_pcie_rx_move_to_allocator(rxq, rba); |
| 1536 |
emergency = true; |
| 1537 |
IWL_DEBUG_TPT(trans, |
| 1538 |
"RX path is in emergency. Pending allocations %d\n", |
| 1539 |
rb_pending_alloc); |
| 1540 |
} |
| 1541 |
|
| 1542 |
IWL_DEBUG_RX(trans, "Q %d: HW = %d, SW = %d\n", rxq->id, r, i); |
| 1543 |
|
| 1544 |
rxb = iwl_pcie_get_rxb(trans, rxq, i, &join); |
| 1545 |
if (!rxb) |
| 1546 |
goto out; |
| 1547 |
|
| 1548 |
if (unlikely(join || rxq->next_rb_is_fragment)) { |
| 1549 |
rxq->next_rb_is_fragment = join; |
| 1550 |
/* |
| 1551 |
* We can only get a multi-RB in the following cases: |
| 1552 |
* - firmware issue, sending a too big notification |
| 1553 |
* - sniffer mode with a large A-MSDU |
| 1554 |
* - large MTU frames (>2k) |
| 1555 |
* since the multi-RB functionality is limited to newer |
| 1556 |
* hardware that cannot put multiple entries into a |
| 1557 |
* single RB. |
| 1558 |
* |
| 1559 |
* Right now, the higher layers aren't set up to deal |
| 1560 |
* with that, so discard all of these. |
| 1561 |
*/ |
| 1562 |
list_add_tail(&rxb->list, &rxq->rx_free); |
| 1563 |
rxq->free_count++; |
| 1564 |
} else { |
| 1565 |
iwl_pcie_rx_handle_rb(trans, rxq, rxb, emergency, i); |
| 1566 |
} |
| 1567 |
|
| 1568 |
i = (i + 1) & (rxq->queue_size - 1); |
| 1569 |
|
| 1570 |
/* |
| 1571 |
* If we have RX_CLAIM_REQ_ALLOC released rx buffers - |
| 1572 |
* try to claim the pre-allocated buffers from the allocator. |
| 1573 |
* If not ready - will try to reclaim next time. |
| 1574 |
* There is no need to reschedule work - allocator exits only |
| 1575 |
* on success |
| 1576 |
*/ |
| 1577 |
if (rxq->used_count >= RX_CLAIM_REQ_ALLOC) |
| 1578 |
iwl_pcie_rx_allocator_get(trans, rxq); |
| 1579 |
|
| 1580 |
if (rxq->used_count % RX_CLAIM_REQ_ALLOC == 0 && !emergency) { |
| 1581 |
/* Add the remaining empty RBDs for allocator use */ |
| 1582 |
iwl_pcie_rx_move_to_allocator(rxq, rba); |
| 1583 |
} else if (emergency) { |
| 1584 |
count++; |
| 1585 |
if (count == 8) { |
| 1586 |
count = 0; |
| 1587 |
if (rb_pending_alloc < rxq->queue_size / 3) { |
| 1588 |
IWL_DEBUG_TPT(trans, |
| 1589 |
"RX path exited emergency. Pending allocations %d\n", |
| 1590 |
rb_pending_alloc); |
| 1591 |
emergency = false; |
| 1592 |
} |
| 1593 |
|
| 1594 |
rxq->read = i; |
| 1595 |
spin_unlock(&rxq->lock); |
| 1596 |
iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC, rxq); |
| 1597 |
iwl_pcie_rxq_restock(trans, rxq); |
| 1598 |
goto restart; |
| 1599 |
} |
| 1600 |
} |
| 1601 |
} |
| 1602 |
out: |
| 1603 |
/* Backtrack one entry */ |
| 1604 |
rxq->read = i; |
| 1605 |
spin_unlock(&rxq->lock); |
| 1606 |
|
| 1607 |
/* |
| 1608 |
* handle a case where in emergency there are some unallocated RBDs. |
| 1609 |
* those RBDs are in the used list, but are not tracked by the queue's |
| 1610 |
* used_count which counts allocator owned RBDs. |
| 1611 |
* unallocated emergency RBDs must be allocated on exit, otherwise |
| 1612 |
* when called again the function may not be in emergency mode and |
| 1613 |
* they will be handed to the allocator with no tracking in the RBD |
| 1614 |
* allocator counters, which will lead to them never being claimed back |
| 1615 |
* by the queue. |
| 1616 |
* by allocating them here, they are now in the queue free list, and |
| 1617 |
* will be restocked by the next call of iwl_pcie_rxq_restock. |
| 1618 |
*/ |
| 1619 |
if (unlikely(emergency && count)) |
| 1620 |
iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC, rxq); |
| 1621 |
|
| 1622 |
iwl_pcie_rxq_restock(trans, rxq); |
| 1623 |
|
| 1624 |
return handled; |
| 1625 |
} |
| 1626 |
|
| 1627 |
static struct iwl_trans_pcie *iwl_pcie_get_trans_pcie(struct msix_entry *entry) |
| 1628 |
{ |
| 1629 |
u8 queue = entry->entry; |
| 1630 |
struct msix_entry *entries = entry - queue; |
| 1631 |
|
| 1632 |
return container_of(entries, struct iwl_trans_pcie, msix_entries[0]); |
| 1633 |
} |
| 1634 |
|
| 1635 |
/* |
| 1636 |
* iwl_pcie_rx_msix_handle - Main entry function for receiving responses from fw |
| 1637 |
* This interrupt handler should be used with RSS queue only. |
| 1638 |
*/ |
| 1639 |
irqreturn_t iwl_pcie_irq_rx_msix_handler(int irq, void *dev_id) |
| 1640 |
{ |
| 1641 |
struct msix_entry *entry = dev_id; |
| 1642 |
struct iwl_trans_pcie *trans_pcie = iwl_pcie_get_trans_pcie(entry); |
| 1643 |
struct iwl_trans *trans = trans_pcie->trans; |
| 1644 |
struct iwl_rxq *rxq; |
| 1645 |
|
| 1646 |
trace_iwlwifi_dev_irq_msix(trans->dev, entry, false, 0, 0); |
| 1647 |
|
| 1648 |
if (WARN_ON(entry->entry >= trans->info.num_rxqs)) |
| 1649 |
return IRQ_NONE; |
| 1650 |
|
| 1651 |
if (!trans_pcie->rxq) { |
| 1652 |
if (net_ratelimit()) |
| 1653 |
IWL_ERR(trans, |
| 1654 |
"[%d] Got MSI-X interrupt before we have Rx queues\n", |
| 1655 |
entry->entry); |
| 1656 |
return IRQ_NONE; |
| 1657 |
} |
| 1658 |
|
| 1659 |
rxq = &trans_pcie->rxq[entry->entry]; |
| 1660 |
lock_map_acquire(&trans->sync_cmd_lockdep_map); |
| 1661 |
IWL_DEBUG_ISR(trans, "[%d] Got interrupt\n", entry->entry); |
| 1662 |
|
| 1663 |
local_bh_disable(); |
| 1664 |
if (!napi_schedule(&rxq->napi)) |
| 1665 |
iwl_pcie_clear_irq(trans, entry->entry); |
| 1666 |
local_bh_enable(); |
| 1667 |
|
| 1668 |
lock_map_release(&trans->sync_cmd_lockdep_map); |
| 1669 |
|
| 1670 |
return IRQ_HANDLED; |
| 1671 |
} |
| 1672 |
|
| 1673 |
/* |
| 1674 |
* iwl_pcie_irq_handle_error - called for HW or SW error interrupt from card |
| 1675 |
*/ |
| 1676 |
static void iwl_pcie_irq_handle_error(struct iwl_trans *trans) |
| 1677 |
{ |
| 1678 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1679 |
int i; |
| 1680 |
|
| 1681 |
/* W/A for WiFi/WiMAX coex and WiMAX own the RF */ |
| 1682 |
if (trans->cfg->internal_wimax_coex && |
| 1683 |
!trans->mac_cfg->base->apmg_not_supported && |
| 1684 |
(!(iwl_read_prph(trans, APMG_CLK_CTRL_REG) & |
| 1685 |
APMS_CLK_VAL_MRB_FUNC_MODE) || |
| 1686 |
(iwl_read_prph(trans, APMG_PS_CTRL_REG) & |
| 1687 |
APMG_PS_CTRL_VAL_RESET_REQ))) { |
| 1688 |
clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); |
| 1689 |
iwl_op_mode_wimax_active(trans->op_mode); |
| 1690 |
wake_up(&trans_pcie->wait_command_queue); |
| 1691 |
return; |
| 1692 |
} |
| 1693 |
|
| 1694 |
for (i = 0; i < trans->mac_cfg->base->num_of_queues; i++) { |
| 1695 |
if (!trans_pcie->txqs.txq[i]) |
| 1696 |
continue; |
| 1697 |
timer_delete(&trans_pcie->txqs.txq[i]->stuck_timer); |
| 1698 |
} |
| 1699 |
|
| 1700 |
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_SC) { |
| 1701 |
u32 val = iwl_read32(trans, CSR_IPC_STATE); |
| 1702 |
|
| 1703 |
if (val & CSR_IPC_STATE_TOP_RESET_REQ) { |
| 1704 |
IWL_ERR(trans, "FW requested TOP reset for FSEQ\n"); |
| 1705 |
trans->do_top_reset = 1; |
| 1706 |
} |
| 1707 |
} |
| 1708 |
|
| 1709 |
/* The STATUS_FW_ERROR bit is set in this function. This must happen |
| 1710 |
* before we wake up the command caller, to ensure a proper cleanup. */ |
| 1711 |
iwl_trans_fw_error(trans, IWL_ERR_TYPE_IRQ); |
| 1712 |
|
| 1713 |
clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); |
| 1714 |
wake_up(&trans_pcie->wait_command_queue); |
| 1715 |
} |
| 1716 |
|
| 1717 |
static u32 iwl_pcie_int_cause_non_ict(struct iwl_trans *trans) |
| 1718 |
{ |
| 1719 |
u32 inta; |
| 1720 |
|
| 1721 |
lockdep_assert_held(&IWL_TRANS_GET_PCIE_TRANS(trans)->irq_lock); |
| 1722 |
|
| 1723 |
trace_iwlwifi_dev_irq(trans->dev); |
| 1724 |
|
| 1725 |
/* Discover which interrupts are active/pending */ |
| 1726 |
inta = iwl_read32(trans, CSR_INT); |
| 1727 |
|
| 1728 |
/* the thread will service interrupts and re-enable them */ |
| 1729 |
return inta; |
| 1730 |
} |
| 1731 |
|
| 1732 |
/* a device (PCI-E) page is 4096 bytes long */ |
| 1733 |
#define ICT_SHIFT 12 |
| 1734 |
#define ICT_SIZE (1 << ICT_SHIFT) |
| 1735 |
#define ICT_COUNT (ICT_SIZE / sizeof(u32)) |
| 1736 |
|
| 1737 |
/* interrupt handler using ict table, with this interrupt driver will |
| 1738 |
* stop using INTA register to get device's interrupt, reading this register |
| 1739 |
* is expensive, device will write interrupts in ICT dram table, increment |
| 1740 |
* index then will fire interrupt to driver, driver will OR all ICT table |
| 1741 |
* entries from current index up to table entry with 0 value. the result is |
| 1742 |
* the interrupt we need to service, driver will set the entries back to 0 and |
| 1743 |
* set index. |
| 1744 |
*/ |
| 1745 |
static u32 iwl_pcie_int_cause_ict(struct iwl_trans *trans) |
| 1746 |
{ |
| 1747 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1748 |
u32 inta; |
| 1749 |
u32 val = 0; |
| 1750 |
u32 read; |
| 1751 |
|
| 1752 |
trace_iwlwifi_dev_irq(trans->dev); |
| 1753 |
|
| 1754 |
/* Ignore interrupt if there's nothing in NIC to service. |
| 1755 |
* This may be due to IRQ shared with another device, |
| 1756 |
* or due to sporadic interrupts thrown from our NIC. */ |
| 1757 |
read = le32_to_cpu(trans_pcie->ict_tbl[trans_pcie->ict_index]); |
| 1758 |
trace_iwlwifi_dev_ict_read(trans->dev, trans_pcie->ict_index, read); |
| 1759 |
if (!read) |
| 1760 |
return 0; |
| 1761 |
|
| 1762 |
/* |
| 1763 |
* Collect all entries up to the first 0, starting from ict_index; |
| 1764 |
* note we already read at ict_index. |
| 1765 |
*/ |
| 1766 |
do { |
| 1767 |
val |= read; |
| 1768 |
IWL_DEBUG_ISR(trans, "ICT index %d value 0x%08X\n", |
| 1769 |
trans_pcie->ict_index, read); |
| 1770 |
trans_pcie->ict_tbl[trans_pcie->ict_index] = 0; |
| 1771 |
trans_pcie->ict_index = |
| 1772 |
((trans_pcie->ict_index + 1) & (ICT_COUNT - 1)); |
| 1773 |
|
| 1774 |
read = le32_to_cpu(trans_pcie->ict_tbl[trans_pcie->ict_index]); |
| 1775 |
trace_iwlwifi_dev_ict_read(trans->dev, trans_pcie->ict_index, |
| 1776 |
read); |
| 1777 |
} while (read); |
| 1778 |
|
| 1779 |
/* We should not get this value, just ignore it. */ |
| 1780 |
if (val == 0xffffffff) |
| 1781 |
val = 0; |
| 1782 |
|
| 1783 |
/* |
| 1784 |
* this is a w/a for a h/w bug. the h/w bug may cause the Rx bit |
| 1785 |
* (bit 15 before shifting it to 31) to clear when using interrupt |
| 1786 |
* coalescing. fortunately, bits 18 and 19 stay set when this happens |
| 1787 |
* so we use them to decide on the real state of the Rx bit. |
| 1788 |
* In order words, bit 15 is set if bit 18 or bit 19 are set. |
| 1789 |
*/ |
| 1790 |
if (val & 0xC0000) |
| 1791 |
val |= 0x8000; |
| 1792 |
|
| 1793 |
inta = (0xff & val) | ((0xff00 & val) << 16); |
| 1794 |
return inta; |
| 1795 |
} |
| 1796 |
|
| 1797 |
void iwl_pcie_handle_rfkill_irq(struct iwl_trans *trans, bool from_irq) |
| 1798 |
{ |
| 1799 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1800 |
struct isr_statistics *isr_stats = &trans_pcie->isr_stats; |
| 1801 |
bool hw_rfkill, prev, report; |
| 1802 |
|
| 1803 |
mutex_lock(&trans_pcie->mutex); |
| 1804 |
prev = test_bit(STATUS_RFKILL_OPMODE, &trans->status); |
| 1805 |
hw_rfkill = iwl_is_rfkill_set(trans); |
| 1806 |
if (hw_rfkill) { |
| 1807 |
set_bit(STATUS_RFKILL_OPMODE, &trans->status); |
| 1808 |
set_bit(STATUS_RFKILL_HW, &trans->status); |
| 1809 |
} |
| 1810 |
if (trans_pcie->opmode_down) |
| 1811 |
report = hw_rfkill; |
| 1812 |
else |
| 1813 |
report = test_bit(STATUS_RFKILL_OPMODE, &trans->status); |
| 1814 |
|
| 1815 |
IWL_WARN(trans, "RF_KILL bit toggled to %s.\n", |
| 1816 |
hw_rfkill ? "disable radio" : "enable radio"); |
| 1817 |
|
| 1818 |
isr_stats->rfkill++; |
| 1819 |
|
| 1820 |
if (prev != report) |
| 1821 |
iwl_trans_pcie_rf_kill(trans, report, from_irq); |
| 1822 |
mutex_unlock(&trans_pcie->mutex); |
| 1823 |
|
| 1824 |
if (hw_rfkill) { |
| 1825 |
if (test_and_clear_bit(STATUS_SYNC_HCMD_ACTIVE, |
| 1826 |
&trans->status)) |
| 1827 |
IWL_DEBUG_RF_KILL(trans, |
| 1828 |
"Rfkill while SYNC HCMD in flight\n"); |
| 1829 |
wake_up(&trans_pcie->wait_command_queue); |
| 1830 |
} else { |
| 1831 |
clear_bit(STATUS_RFKILL_HW, &trans->status); |
| 1832 |
if (trans_pcie->opmode_down) |
| 1833 |
clear_bit(STATUS_RFKILL_OPMODE, &trans->status); |
| 1834 |
} |
| 1835 |
} |
| 1836 |
|
| 1837 |
static void iwl_trans_pcie_handle_reset_interrupt(struct iwl_trans *trans) |
| 1838 |
{ |
| 1839 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1840 |
u32 state; |
| 1841 |
|
| 1842 |
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_SC) { |
| 1843 |
u32 val = iwl_read32(trans, CSR_IPC_STATE); |
| 1844 |
|
| 1845 |
state = u32_get_bits(val, CSR_IPC_STATE_RESET); |
| 1846 |
IWL_DEBUG_ISR(trans, "IPC state = 0x%x/%d\n", val, state); |
| 1847 |
} else { |
| 1848 |
state = CSR_IPC_STATE_RESET_SW_READY; |
| 1849 |
} |
| 1850 |
|
| 1851 |
switch (state) { |
| 1852 |
case CSR_IPC_STATE_RESET_SW_READY: |
| 1853 |
if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) { |
| 1854 |
IWL_DEBUG_ISR(trans, "Reset flow completed\n"); |
| 1855 |
trans_pcie->fw_reset_state = FW_RESET_OK; |
| 1856 |
wake_up(&trans_pcie->fw_reset_waitq); |
| 1857 |
break; |
| 1858 |
} |
| 1859 |
fallthrough; |
| 1860 |
case CSR_IPC_STATE_RESET_TOP_READY: |
| 1861 |
if (trans_pcie->fw_reset_state == FW_RESET_TOP_REQUESTED) { |
| 1862 |
IWL_DEBUG_ISR(trans, "TOP Reset continues\n"); |
| 1863 |
trans_pcie->fw_reset_state = FW_RESET_OK; |
| 1864 |
wake_up(&trans_pcie->fw_reset_waitq); |
| 1865 |
break; |
| 1866 |
} |
| 1867 |
fallthrough; |
| 1868 |
case CSR_IPC_STATE_RESET_NONE: |
| 1869 |
IWL_FW_CHECK_FAILED(trans, |
| 1870 |
"Invalid reset interrupt (state=%d)!\n", |
| 1871 |
state); |
| 1872 |
break; |
| 1873 |
case CSR_IPC_STATE_RESET_TOP_FOLLOWER: |
| 1874 |
if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) { |
| 1875 |
/* if we were in reset, wake that up */ |
| 1876 |
IWL_INFO(trans, |
| 1877 |
"TOP reset from BT while doing reset\n"); |
| 1878 |
trans_pcie->fw_reset_state = FW_RESET_OK; |
| 1879 |
wake_up(&trans_pcie->fw_reset_waitq); |
| 1880 |
} else { |
| 1881 |
IWL_INFO(trans, "TOP reset from BT\n"); |
| 1882 |
trans->state = IWL_TRANS_NO_FW; |
| 1883 |
iwl_trans_schedule_reset(trans, |
| 1884 |
IWL_ERR_TYPE_TOP_RESET_BY_BT); |
| 1885 |
} |
| 1886 |
break; |
| 1887 |
} |
| 1888 |
} |
| 1889 |
|
| 1890 |
irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id) |
| 1891 |
{ |
| 1892 |
struct iwl_trans *trans = dev_id; |
| 1893 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 1894 |
struct isr_statistics *isr_stats = &trans_pcie->isr_stats; |
| 1895 |
u32 inta = 0; |
| 1896 |
u32 handled = 0; |
| 1897 |
bool polling = false; |
| 1898 |
|
| 1899 |
lock_map_acquire(&trans->sync_cmd_lockdep_map); |
| 1900 |
|
| 1901 |
spin_lock_bh(&trans_pcie->irq_lock); |
| 1902 |
|
| 1903 |
/* dram interrupt table not set yet, |
| 1904 |
* use legacy interrupt. |
| 1905 |
*/ |
| 1906 |
if (likely(trans_pcie->use_ict)) |
| 1907 |
inta = iwl_pcie_int_cause_ict(trans); |
| 1908 |
else |
| 1909 |
inta = iwl_pcie_int_cause_non_ict(trans); |
| 1910 |
|
| 1911 |
if (iwl_have_debug_level(IWL_DL_ISR)) { |
| 1912 |
IWL_DEBUG_ISR(trans, |
| 1913 |
"ISR inta 0x%08x, enabled 0x%08x(sw), enabled(hw) 0x%08x, fh 0x%08x\n", |
| 1914 |
inta, trans_pcie->inta_mask, |
| 1915 |
iwl_read32(trans, CSR_INT_MASK), |
| 1916 |
iwl_read32(trans, CSR_FH_INT_STATUS)); |
| 1917 |
if (inta & (~trans_pcie->inta_mask)) |
| 1918 |
IWL_DEBUG_ISR(trans, |
| 1919 |
"We got a masked interrupt (0x%08x)\n", |
| 1920 |
inta & (~trans_pcie->inta_mask)); |
| 1921 |
} |
| 1922 |
|
| 1923 |
inta &= trans_pcie->inta_mask; |
| 1924 |
|
| 1925 |
/* |
| 1926 |
* Ignore interrupt if there's nothing in NIC to service. |
| 1927 |
* This may be due to IRQ shared with another device, |
| 1928 |
* or due to sporadic interrupts thrown from our NIC. |
| 1929 |
*/ |
| 1930 |
if (unlikely(!inta)) { |
| 1931 |
IWL_DEBUG_ISR(trans, "Ignore interrupt, inta == 0\n"); |
| 1932 |
/* |
| 1933 |
* Re-enable interrupts here since we don't |
| 1934 |
* have anything to service |
| 1935 |
*/ |
| 1936 |
if (test_bit(STATUS_INT_ENABLED, &trans->status)) |
| 1937 |
_iwl_enable_interrupts(trans); |
| 1938 |
spin_unlock_bh(&trans_pcie->irq_lock); |
| 1939 |
lock_map_release(&trans->sync_cmd_lockdep_map); |
| 1940 |
return IRQ_NONE; |
| 1941 |
} |
| 1942 |
|
| 1943 |
if (unlikely(inta == 0xFFFFFFFF || iwl_trans_is_hw_error_value(inta))) { |
| 1944 |
/* |
| 1945 |
* Hardware disappeared. It might have |
| 1946 |
* already raised an interrupt. |
| 1947 |
*/ |
| 1948 |
IWL_WARN(trans, "HARDWARE GONE?? INTA == 0x%08x\n", inta); |
| 1949 |
spin_unlock_bh(&trans_pcie->irq_lock); |
| 1950 |
goto out; |
| 1951 |
} |
| 1952 |
|
| 1953 |
/* Ack/clear/reset pending uCode interrupts. |
| 1954 |
* Note: Some bits in CSR_INT are "OR" of bits in CSR_FH_INT_STATUS, |
| 1955 |
*/ |
| 1956 |
/* There is a hardware bug in the interrupt mask function that some |
| 1957 |
* interrupts (i.e. CSR_INT_BIT_SCD) can still be generated even if |
| 1958 |
* they are disabled in the CSR_INT_MASK register. Furthermore the |
| 1959 |
* ICT interrupt handling mechanism has another bug that might cause |
| 1960 |
* these unmasked interrupts fail to be detected. We workaround the |
| 1961 |
* hardware bugs here by ACKing all the possible interrupts so that |
| 1962 |
* interrupt coalescing can still be achieved. |
| 1963 |
*/ |
| 1964 |
iwl_write32(trans, CSR_INT, inta | ~trans_pcie->inta_mask); |
| 1965 |
|
| 1966 |
if (iwl_have_debug_level(IWL_DL_ISR)) |
| 1967 |
IWL_DEBUG_ISR(trans, "inta 0x%08x, enabled 0x%08x\n", |
| 1968 |
inta, iwl_read32(trans, CSR_INT_MASK)); |
| 1969 |
|
| 1970 |
spin_unlock_bh(&trans_pcie->irq_lock); |
| 1971 |
|
| 1972 |
/* Now service all interrupt bits discovered above. */ |
| 1973 |
if (inta & CSR_INT_BIT_HW_ERR) { |
| 1974 |
IWL_ERR(trans, "Hardware error detected. Restarting.\n"); |
| 1975 |
|
| 1976 |
/* Tell the device to stop sending interrupts */ |
| 1977 |
iwl_disable_interrupts(trans); |
| 1978 |
|
| 1979 |
isr_stats->hw++; |
| 1980 |
iwl_pcie_irq_handle_error(trans); |
| 1981 |
|
| 1982 |
handled |= CSR_INT_BIT_HW_ERR; |
| 1983 |
|
| 1984 |
goto out; |
| 1985 |
} |
| 1986 |
|
| 1987 |
/* NIC fires this, but we don't use it, redundant with WAKEUP */ |
| 1988 |
if (inta & CSR_INT_BIT_SCD) { |
| 1989 |
IWL_DEBUG_ISR(trans, |
| 1990 |
"Scheduler finished to transmit the frame/frames.\n"); |
| 1991 |
isr_stats->sch++; |
| 1992 |
} |
| 1993 |
|
| 1994 |
/* Alive notification via Rx interrupt will do the real work */ |
| 1995 |
if (inta & CSR_INT_BIT_ALIVE) { |
| 1996 |
IWL_DEBUG_ISR(trans, "Alive interrupt\n"); |
| 1997 |
isr_stats->alive++; |
| 1998 |
if (trans->mac_cfg->gen2) { |
| 1999 |
/* |
| 2000 |
* We can restock, since firmware configured |
| 2001 |
* the RFH |
| 2002 |
*/ |
| 2003 |
iwl_pcie_rxmq_restock(trans, trans_pcie->rxq); |
| 2004 |
} |
| 2005 |
|
| 2006 |
handled |= CSR_INT_BIT_ALIVE; |
| 2007 |
} |
| 2008 |
|
| 2009 |
if (inta & CSR_INT_BIT_RESET_DONE) { |
| 2010 |
iwl_trans_pcie_handle_reset_interrupt(trans); |
| 2011 |
handled |= CSR_INT_BIT_RESET_DONE; |
| 2012 |
} |
| 2013 |
|
| 2014 |
/* Safely ignore these bits for debug checks below */ |
| 2015 |
inta &= ~(CSR_INT_BIT_SCD | CSR_INT_BIT_ALIVE); |
| 2016 |
|
| 2017 |
/* HW RF KILL switch toggled */ |
| 2018 |
if (inta & CSR_INT_BIT_RF_KILL) { |
| 2019 |
iwl_pcie_handle_rfkill_irq(trans, true); |
| 2020 |
handled |= CSR_INT_BIT_RF_KILL; |
| 2021 |
} |
| 2022 |
|
| 2023 |
/* Chip got too hot and stopped itself */ |
| 2024 |
if (inta & CSR_INT_BIT_CT_KILL) { |
| 2025 |
IWL_ERR(trans, "Microcode CT kill error detected.\n"); |
| 2026 |
isr_stats->ctkill++; |
| 2027 |
handled |= CSR_INT_BIT_CT_KILL; |
| 2028 |
} |
| 2029 |
|
| 2030 |
/* Error detected by uCode */ |
| 2031 |
if (inta & CSR_INT_BIT_SW_ERR) { |
| 2032 |
IWL_ERR(trans, "Microcode SW error detected. " |
| 2033 |
" Restarting 0x%X.\n", inta); |
| 2034 |
isr_stats->sw++; |
| 2035 |
if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) { |
| 2036 |
trans_pcie->fw_reset_state = FW_RESET_ERROR; |
| 2037 |
wake_up(&trans_pcie->fw_reset_waitq); |
| 2038 |
} else { |
| 2039 |
iwl_pcie_irq_handle_error(trans); |
| 2040 |
} |
| 2041 |
handled |= CSR_INT_BIT_SW_ERR; |
| 2042 |
} |
| 2043 |
|
| 2044 |
/* uCode wakes up after power-down sleep */ |
| 2045 |
if (inta & CSR_INT_BIT_WAKEUP) { |
| 2046 |
IWL_DEBUG_ISR(trans, "Wakeup interrupt\n"); |
| 2047 |
iwl_pcie_rxq_check_wrptr(trans); |
| 2048 |
iwl_pcie_txq_check_wrptrs(trans); |
| 2049 |
|
| 2050 |
isr_stats->wakeup++; |
| 2051 |
|
| 2052 |
handled |= CSR_INT_BIT_WAKEUP; |
| 2053 |
} |
| 2054 |
|
| 2055 |
/* All uCode command responses, including Tx command responses, |
| 2056 |
* Rx "responses" (frame-received notification), and other |
| 2057 |
* notifications from uCode come through here*/ |
| 2058 |
if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX | |
| 2059 |
CSR_INT_BIT_RX_PERIODIC)) { |
| 2060 |
IWL_DEBUG_ISR(trans, "Rx interrupt\n"); |
| 2061 |
if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX)) { |
| 2062 |
handled |= (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX); |
| 2063 |
iwl_write32(trans, CSR_FH_INT_STATUS, |
| 2064 |
CSR_FH_INT_RX_MASK); |
| 2065 |
} |
| 2066 |
if (inta & CSR_INT_BIT_RX_PERIODIC) { |
| 2067 |
handled |= CSR_INT_BIT_RX_PERIODIC; |
| 2068 |
iwl_write32(trans, |
| 2069 |
CSR_INT, CSR_INT_BIT_RX_PERIODIC); |
| 2070 |
} |
| 2071 |
/* Sending RX interrupt require many steps to be done in the |
| 2072 |
* device: |
| 2073 |
* 1- write interrupt to current index in ICT table. |
| 2074 |
* 2- dma RX frame. |
| 2075 |
* 3- update RX shared data to indicate last write index. |
| 2076 |
* 4- send interrupt. |
| 2077 |
* This could lead to RX race, driver could receive RX interrupt |
| 2078 |
* but the shared data changes does not reflect this; |
| 2079 |
* periodic interrupt will detect any dangling Rx activity. |
| 2080 |
*/ |
| 2081 |
|
| 2082 |
/* Disable periodic interrupt; we use it as just a one-shot. */ |
| 2083 |
iwl_write8(trans, CSR_INT_PERIODIC_REG, |
| 2084 |
CSR_INT_PERIODIC_DIS); |
| 2085 |
|
| 2086 |
/* |
| 2087 |
* Enable periodic interrupt in 8 msec only if we received |
| 2088 |
* real RX interrupt (instead of just periodic int), to catch |
| 2089 |
* any dangling Rx interrupt. If it was just the periodic |
| 2090 |
* interrupt, there was no dangling Rx activity, and no need |
| 2091 |
* to extend the periodic interrupt; one-shot is enough. |
| 2092 |
*/ |
| 2093 |
if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX)) |
| 2094 |
iwl_write8(trans, CSR_INT_PERIODIC_REG, |
| 2095 |
CSR_INT_PERIODIC_ENA); |
| 2096 |
|
| 2097 |
isr_stats->rx++; |
| 2098 |
|
| 2099 |
local_bh_disable(); |
| 2100 |
if (napi_schedule_prep(&trans_pcie->rxq[0].napi)) { |
| 2101 |
polling = true; |
| 2102 |
__napi_schedule(&trans_pcie->rxq[0].napi); |
| 2103 |
} |
| 2104 |
local_bh_enable(); |
| 2105 |
} |
| 2106 |
|
| 2107 |
/* This "Tx" DMA channel is used only for loading uCode */ |
| 2108 |
if (inta & CSR_INT_BIT_FH_TX) { |
| 2109 |
iwl_write32(trans, CSR_FH_INT_STATUS, CSR_FH_INT_TX_MASK); |
| 2110 |
IWL_DEBUG_ISR(trans, "uCode load interrupt\n"); |
| 2111 |
isr_stats->tx++; |
| 2112 |
handled |= CSR_INT_BIT_FH_TX; |
| 2113 |
/* Wake up uCode load routine, now that load is complete */ |
| 2114 |
trans_pcie->ucode_write_complete = true; |
| 2115 |
wake_up(&trans_pcie->ucode_write_waitq); |
| 2116 |
/* Wake up IMR write routine, now that write to SRAM is complete */ |
| 2117 |
if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { |
| 2118 |
trans_pcie->imr_status = IMR_D2S_COMPLETED; |
| 2119 |
wake_up(&trans_pcie->ucode_write_waitq); |
| 2120 |
} |
| 2121 |
} |
| 2122 |
|
| 2123 |
if (inta & ~handled) { |
| 2124 |
IWL_ERR(trans, "Unhandled INTA bits 0x%08x\n", inta & ~handled); |
| 2125 |
isr_stats->unhandled++; |
| 2126 |
} |
| 2127 |
|
| 2128 |
if (inta & ~(trans_pcie->inta_mask)) { |
| 2129 |
IWL_WARN(trans, "Disabled INTA bits 0x%08x were pending\n", |
| 2130 |
inta & ~trans_pcie->inta_mask); |
| 2131 |
} |
| 2132 |
|
| 2133 |
if (!polling) { |
| 2134 |
spin_lock_bh(&trans_pcie->irq_lock); |
| 2135 |
/* only Re-enable all interrupt if disabled by irq */ |
| 2136 |
if (test_bit(STATUS_INT_ENABLED, &trans->status)) |
| 2137 |
_iwl_enable_interrupts(trans); |
| 2138 |
/* we are loading the firmware, enable FH_TX interrupt only */ |
| 2139 |
else if (handled & CSR_INT_BIT_FH_TX) |
| 2140 |
iwl_enable_fw_load_int(trans); |
| 2141 |
/* Re-enable RF_KILL if it occurred */ |
| 2142 |
else if (handled & CSR_INT_BIT_RF_KILL) |
| 2143 |
iwl_enable_rfkill_int(trans); |
| 2144 |
/* Re-enable the ALIVE / Rx interrupt if it occurred */ |
| 2145 |
else if (handled & (CSR_INT_BIT_ALIVE | CSR_INT_BIT_FH_RX)) |
| 2146 |
iwl_enable_fw_load_int_ctx_info(trans, false); |
| 2147 |
spin_unlock_bh(&trans_pcie->irq_lock); |
| 2148 |
} |
| 2149 |
|
| 2150 |
out: |
| 2151 |
lock_map_release(&trans->sync_cmd_lockdep_map); |
| 2152 |
return IRQ_HANDLED; |
| 2153 |
} |
| 2154 |
|
| 2155 |
/****************************************************************************** |
| 2156 |
* |
| 2157 |
* ICT functions |
| 2158 |
* |
| 2159 |
******************************************************************************/ |
| 2160 |
|
| 2161 |
/* Free dram table */ |
| 2162 |
void iwl_pcie_free_ict(struct iwl_trans *trans) |
| 2163 |
{ |
| 2164 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 2165 |
|
| 2166 |
if (trans_pcie->ict_tbl) { |
| 2167 |
dma_free_coherent(trans->dev, ICT_SIZE, |
| 2168 |
trans_pcie->ict_tbl, |
| 2169 |
trans_pcie->ict_tbl_dma); |
| 2170 |
trans_pcie->ict_tbl = NULL; |
| 2171 |
trans_pcie->ict_tbl_dma = 0; |
| 2172 |
} |
| 2173 |
} |
| 2174 |
|
| 2175 |
/* |
| 2176 |
* allocate dram shared table, it is an aligned memory |
| 2177 |
* block of ICT_SIZE. |
| 2178 |
* also reset all data related to ICT table interrupt. |
| 2179 |
*/ |
| 2180 |
int iwl_pcie_alloc_ict(struct iwl_trans *trans) |
| 2181 |
{ |
| 2182 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 2183 |
|
| 2184 |
trans_pcie->ict_tbl = |
| 2185 |
dma_alloc_coherent(trans->dev, ICT_SIZE, |
| 2186 |
&trans_pcie->ict_tbl_dma, GFP_KERNEL); |
| 2187 |
if (!trans_pcie->ict_tbl) |
| 2188 |
return -ENOMEM; |
| 2189 |
|
| 2190 |
/* just an API sanity check ... it is guaranteed to be aligned */ |
| 2191 |
if (WARN_ON(trans_pcie->ict_tbl_dma & (ICT_SIZE - 1))) { |
| 2192 |
iwl_pcie_free_ict(trans); |
| 2193 |
return -EINVAL; |
| 2194 |
} |
| 2195 |
|
| 2196 |
return 0; |
| 2197 |
} |
| 2198 |
|
| 2199 |
/* Device is going up inform it about using ICT interrupt table, |
| 2200 |
* also we need to tell the driver to start using ICT interrupt. |
| 2201 |
*/ |
| 2202 |
void iwl_pcie_reset_ict(struct iwl_trans *trans) |
| 2203 |
{ |
| 2204 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 2205 |
u32 val; |
| 2206 |
|
| 2207 |
if (!trans_pcie->ict_tbl) |
| 2208 |
return; |
| 2209 |
|
| 2210 |
spin_lock_bh(&trans_pcie->irq_lock); |
| 2211 |
_iwl_disable_interrupts(trans); |
| 2212 |
|
| 2213 |
memset(trans_pcie->ict_tbl, 0, ICT_SIZE); |
| 2214 |
|
| 2215 |
val = trans_pcie->ict_tbl_dma >> ICT_SHIFT; |
| 2216 |
|
| 2217 |
val |= CSR_DRAM_INT_TBL_ENABLE | |
| 2218 |
CSR_DRAM_INIT_TBL_WRAP_CHECK | |
| 2219 |
CSR_DRAM_INIT_TBL_WRITE_POINTER; |
| 2220 |
|
| 2221 |
IWL_DEBUG_ISR(trans, "CSR_DRAM_INT_TBL_REG =0x%x\n", val); |
| 2222 |
|
| 2223 |
iwl_write32(trans, CSR_DRAM_INT_TBL_REG, val); |
| 2224 |
trans_pcie->use_ict = true; |
| 2225 |
trans_pcie->ict_index = 0; |
| 2226 |
iwl_write32(trans, CSR_INT, trans_pcie->inta_mask); |
| 2227 |
_iwl_enable_interrupts(trans); |
| 2228 |
spin_unlock_bh(&trans_pcie->irq_lock); |
| 2229 |
} |
| 2230 |
|
| 2231 |
/* Device is going down disable ict interrupt usage */ |
| 2232 |
void iwl_pcie_disable_ict(struct iwl_trans *trans) |
| 2233 |
{ |
| 2234 |
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); |
| 2235 |
|
| 2236 |
spin_lock_bh(&trans_pcie->irq_lock); |
| 2237 |
trans_pcie->use_ict = false; |
| 2238 |
spin_unlock_bh(&trans_pcie->irq_lock); |
| 2239 |
} |
| 2240 |
|
| 2241 |
irqreturn_t iwl_pcie_isr(int irq, void *data) |
| 2242 |
{ |
| 2243 |
struct iwl_trans *trans = data; |
| 2244 |
|
| 2245 |
if (!trans) |
| 2246 |
return IRQ_NONE; |
| 2247 |
|
| 2248 |
/* Disable (but don't clear!) interrupts here to avoid |
| 2249 |
* back-to-back ISRs and sporadic interrupts from our NIC. |
| 2250 |
* If we have something to service, the tasklet will re-enable ints. |
| 2251 |
* If we *don't* have something, we'll re-enable before leaving here. |
| 2252 |
*/ |
| 2253 |
iwl_write32(trans, CSR_INT_MASK, 0x00000000); |
| 2254 |
|
| 2255 |
return IRQ_WAKE_THREAD; |
| 2256 |
} |
| 2257 |
|
| 2258 |
irqreturn_t iwl_pcie_msix_isr(int irq, void *data) |
| 2259 |
{ |
| 2260 |
return IRQ_WAKE_THREAD; |
| 2261 |
} |
| 2262 |
|
| 2263 |
irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id) |
| 2264 |
{ |
| 2265 |
struct msix_entry *entry = dev_id; |
| 2266 |
struct iwl_trans_pcie *trans_pcie = iwl_pcie_get_trans_pcie(entry); |
| 2267 |
struct iwl_trans *trans = trans_pcie->trans; |
| 2268 |
struct isr_statistics *isr_stats = &trans_pcie->isr_stats; |
| 2269 |
u32 inta_fh_msk = ~MSIX_FH_INT_CAUSES_DATA_QUEUE; |
| 2270 |
u32 inta_fh, inta_hw; |
| 2271 |
bool polling = false; |
| 2272 |
bool sw_err; |
| 2273 |
|
| 2274 |
if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX) |
| 2275 |
inta_fh_msk |= MSIX_FH_INT_CAUSES_Q0; |
| 2276 |
|
| 2277 |
if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS) |
| 2278 |
inta_fh_msk |= MSIX_FH_INT_CAUSES_Q1; |
| 2279 |
|
| 2280 |
lock_map_acquire(&trans->sync_cmd_lockdep_map); |
| 2281 |
|
| 2282 |
spin_lock_bh(&trans_pcie->irq_lock); |
| 2283 |
inta_fh = iwl_read32(trans, CSR_MSIX_FH_INT_CAUSES_AD); |
| 2284 |
inta_hw = iwl_read32(trans, CSR_MSIX_HW_INT_CAUSES_AD); |
| 2285 |
/* |
| 2286 |
* Clear causes registers to avoid being handling the same cause. |
| 2287 |
*/ |
| 2288 |
iwl_write32(trans, CSR_MSIX_FH_INT_CAUSES_AD, inta_fh & inta_fh_msk); |
| 2289 |
iwl_write32(trans, CSR_MSIX_HW_INT_CAUSES_AD, inta_hw); |
| 2290 |
spin_unlock_bh(&trans_pcie->irq_lock); |
| 2291 |
|
| 2292 |
trace_iwlwifi_dev_irq_msix(trans->dev, entry, true, inta_fh, inta_hw); |
| 2293 |
|
| 2294 |
if (unlikely(!(inta_fh | inta_hw))) { |
| 2295 |
IWL_DEBUG_ISR(trans, "Ignore interrupt, inta == 0\n"); |
| 2296 |
lock_map_release(&trans->sync_cmd_lockdep_map); |
| 2297 |
return IRQ_NONE; |
| 2298 |
} |
| 2299 |
|
| 2300 |
if (iwl_have_debug_level(IWL_DL_ISR)) { |
| 2301 |
IWL_DEBUG_ISR(trans, |
| 2302 |
"ISR[%d] inta_fh 0x%08x, enabled (sw) 0x%08x (hw) 0x%08x\n", |
| 2303 |
entry->entry, inta_fh, trans_pcie->fh_mask, |
| 2304 |
iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD)); |
| 2305 |
if (inta_fh & ~trans_pcie->fh_mask) |
| 2306 |
IWL_DEBUG_ISR(trans, |
| 2307 |
"We got a masked interrupt (0x%08x)\n", |
| 2308 |
inta_fh & ~trans_pcie->fh_mask); |
| 2309 |
} |
| 2310 |
|
| 2311 |
inta_fh &= trans_pcie->fh_mask; |
| 2312 |
|
| 2313 |
if ((trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX) && |
| 2314 |
inta_fh & MSIX_FH_INT_CAUSES_Q0) { |
| 2315 |
local_bh_disable(); |
| 2316 |
if (napi_schedule_prep(&trans_pcie->rxq[0].napi)) { |
| 2317 |
polling = true; |
| 2318 |
__napi_schedule(&trans_pcie->rxq[0].napi); |
| 2319 |
} |
| 2320 |
local_bh_enable(); |
| 2321 |
} |
| 2322 |
|
| 2323 |
if ((trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS) && |
| 2324 |
inta_fh & MSIX_FH_INT_CAUSES_Q1) { |
| 2325 |
local_bh_disable(); |
| 2326 |
if (napi_schedule_prep(&trans_pcie->rxq[1].napi)) { |
| 2327 |
polling = true; |
| 2328 |
__napi_schedule(&trans_pcie->rxq[1].napi); |
| 2329 |
} |
| 2330 |
local_bh_enable(); |
| 2331 |
} |
| 2332 |
|
| 2333 |
/* This "Tx" DMA channel is used only for loading uCode */ |
| 2334 |
if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM && |
| 2335 |
trans_pcie->imr_status == IMR_D2S_REQUESTED) { |
| 2336 |
IWL_DEBUG_ISR(trans, "IMR Complete interrupt\n"); |
| 2337 |
isr_stats->tx++; |
| 2338 |
|
| 2339 |
/* Wake up IMR routine once write to SRAM is complete */ |
| 2340 |
if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { |
| 2341 |
trans_pcie->imr_status = IMR_D2S_COMPLETED; |
| 2342 |
wake_up(&trans_pcie->ucode_write_waitq); |
| 2343 |
} |
| 2344 |
} else if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM) { |
| 2345 |
IWL_DEBUG_ISR(trans, "uCode load interrupt\n"); |
| 2346 |
isr_stats->tx++; |
| 2347 |
/* |
| 2348 |
* Wake up uCode load routine, |
| 2349 |
* now that load is complete |
| 2350 |
*/ |
| 2351 |
trans_pcie->ucode_write_complete = true; |
| 2352 |
wake_up(&trans_pcie->ucode_write_waitq); |
| 2353 |
|
| 2354 |
/* Wake up IMR routine once write to SRAM is complete */ |
| 2355 |
if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { |
| 2356 |
trans_pcie->imr_status = IMR_D2S_COMPLETED; |
| 2357 |
wake_up(&trans_pcie->ucode_write_waitq); |
| 2358 |
} |
| 2359 |
} |
| 2360 |
|
| 2361 |
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) |
| 2362 |
sw_err = inta_hw & MSIX_HW_INT_CAUSES_REG_SW_ERR_BZ; |
| 2363 |
else |
| 2364 |
sw_err = inta_hw & MSIX_HW_INT_CAUSES_REG_SW_ERR; |
| 2365 |
|
| 2366 |
if (inta_hw & MSIX_HW_INT_CAUSES_REG_TOP_FATAL_ERR) { |
| 2367 |
IWL_ERR(trans, "TOP Fatal error detected, inta_hw=0x%x.\n", |
| 2368 |
inta_hw); |
| 2369 |
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) { |
| 2370 |
trans->request_top_reset = 1; |
| 2371 |
iwl_op_mode_nic_error(trans->op_mode, |
| 2372 |
IWL_ERR_TYPE_TOP_FATAL_ERROR); |
| 2373 |
iwl_trans_schedule_reset(trans, |
| 2374 |
IWL_ERR_TYPE_TOP_FATAL_ERROR); |
| 2375 |
} |
| 2376 |
} |
| 2377 |
|
| 2378 |
/* Error detected by uCode */ |
| 2379 |
if ((inta_fh & MSIX_FH_INT_CAUSES_FH_ERR) || sw_err) { |
| 2380 |
IWL_ERR(trans, |
| 2381 |
"Microcode SW error detected. Restarting 0x%X.\n", |
| 2382 |
inta_fh); |
| 2383 |
isr_stats->sw++; |
| 2384 |
/* during FW reset flow report errors from there */ |
| 2385 |
if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { |
| 2386 |
trans_pcie->imr_status = IMR_D2S_ERROR; |
| 2387 |
wake_up(&trans_pcie->imr_waitq); |
| 2388 |
} else if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) { |
| 2389 |
trans_pcie->fw_reset_state = FW_RESET_ERROR; |
| 2390 |
wake_up(&trans_pcie->fw_reset_waitq); |
| 2391 |
} else { |
| 2392 |
iwl_pcie_irq_handle_error(trans); |
| 2393 |
} |
| 2394 |
|
| 2395 |
if (trans_pcie->sx_state == IWL_SX_WAITING) { |
| 2396 |
trans_pcie->sx_state = IWL_SX_ERROR; |
| 2397 |
wake_up(&trans_pcie->sx_waitq); |
| 2398 |
} |
| 2399 |
} |
| 2400 |
|
| 2401 |
/* After checking FH register check HW register */ |
| 2402 |
if (iwl_have_debug_level(IWL_DL_ISR)) { |
| 2403 |
IWL_DEBUG_ISR(trans, |
| 2404 |
"ISR[%d] inta_hw 0x%08x, enabled (sw) 0x%08x (hw) 0x%08x\n", |
| 2405 |
entry->entry, inta_hw, trans_pcie->hw_mask, |
| 2406 |
iwl_read32(trans, CSR_MSIX_HW_INT_MASK_AD)); |
| 2407 |
if (inta_hw & ~trans_pcie->hw_mask) |
| 2408 |
IWL_DEBUG_ISR(trans, |
| 2409 |
"We got a masked interrupt 0x%08x\n", |
| 2410 |
inta_hw & ~trans_pcie->hw_mask); |
| 2411 |
} |
| 2412 |
|
| 2413 |
inta_hw &= trans_pcie->hw_mask; |
| 2414 |
|
| 2415 |
/* Alive notification via Rx interrupt will do the real work */ |
| 2416 |
if (inta_hw & MSIX_HW_INT_CAUSES_REG_ALIVE) { |
| 2417 |
IWL_DEBUG_ISR(trans, "Alive interrupt\n"); |
| 2418 |
isr_stats->alive++; |
| 2419 |
if (trans->mac_cfg->gen2) { |
| 2420 |
/* We can restock, since firmware configured the RFH */ |
| 2421 |
iwl_pcie_rxmq_restock(trans, trans_pcie->rxq); |
| 2422 |
} |
| 2423 |
} |
| 2424 |
|
| 2425 |
/* |
| 2426 |
* In some rare cases when the HW is in a bad state, we may |
| 2427 |
* get this interrupt too early, when prph_info is still NULL. |
| 2428 |
* So make sure that it's not NULL to prevent crashing. |
| 2429 |
*/ |
| 2430 |
if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP && trans_pcie->prph_info) { |
| 2431 |
u32 sleep_notif = |
| 2432 |
le32_to_cpu(trans_pcie->prph_info->sleep_notif); |
| 2433 |
|
| 2434 |
if (sleep_notif == IWL_D3_SLEEP_STATUS_SUSPEND || |
| 2435 |
sleep_notif == IWL_D3_SLEEP_STATUS_RESUME) { |
| 2436 |
IWL_DEBUG_ISR(trans, |
| 2437 |
"Sx interrupt: sleep notification = 0x%x\n", |
| 2438 |
sleep_notif); |
| 2439 |
if (trans_pcie->sx_state == IWL_SX_WAITING) { |
| 2440 |
trans_pcie->sx_state = IWL_SX_COMPLETE; |
| 2441 |
wake_up(&trans_pcie->sx_waitq); |
| 2442 |
} else { |
| 2443 |
IWL_ERR(trans, |
| 2444 |
"unexpected Sx interrupt (0x%x)\n", |
| 2445 |
sleep_notif); |
| 2446 |
} |
| 2447 |
} else { |
| 2448 |
/* uCode wakes up after power-down sleep */ |
| 2449 |
IWL_DEBUG_ISR(trans, "Wakeup interrupt\n"); |
| 2450 |
iwl_pcie_rxq_check_wrptr(trans); |
| 2451 |
iwl_pcie_txq_check_wrptrs(trans); |
| 2452 |
|
| 2453 |
isr_stats->wakeup++; |
| 2454 |
} |
| 2455 |
} |
| 2456 |
|
| 2457 |
/* Chip got too hot and stopped itself */ |
| 2458 |
if (inta_hw & MSIX_HW_INT_CAUSES_REG_CT_KILL) { |
| 2459 |
IWL_ERR(trans, "Microcode CT kill error detected.\n"); |
| 2460 |
isr_stats->ctkill++; |
| 2461 |
} |
| 2462 |
|
| 2463 |
/* HW RF KILL switch toggled */ |
| 2464 |
if (inta_hw & MSIX_HW_INT_CAUSES_REG_RF_KILL) |
| 2465 |
iwl_pcie_handle_rfkill_irq(trans, true); |
| 2466 |
|
| 2467 |
if (inta_hw & MSIX_HW_INT_CAUSES_REG_HW_ERR) { |
| 2468 |
IWL_ERR(trans, |
| 2469 |
"Hardware error detected. Restarting.\n"); |
| 2470 |
|
| 2471 |
isr_stats->hw++; |
| 2472 |
trans->dbg.hw_error = true; |
| 2473 |
iwl_pcie_irq_handle_error(trans); |
| 2474 |
} |
| 2475 |
|
| 2476 |
if (inta_hw & MSIX_HW_INT_CAUSES_REG_RESET_DONE) |
| 2477 |
iwl_trans_pcie_handle_reset_interrupt(trans); |
| 2478 |
|
| 2479 |
if (!polling) |
| 2480 |
iwl_pcie_clear_irq(trans, entry->entry); |
| 2481 |
|
| 2482 |
lock_map_release(&trans->sync_cmd_lockdep_map); |
| 2483 |
|
| 2484 |
return IRQ_HANDLED; |
| 2485 |
} |