| 1 |
// SPDX-License-Identifier: GPL-2.0-only |
| 2 |
/* |
| 3 |
* Copyright (c) 2025 AIROHA Inc |
| 4 |
* Author: Lorenzo Bianconi <[email protected]> |
| 5 |
*/ |
| 6 |
#include <linux/kernel.h> |
| 7 |
#include <net/flow_offload.h> |
| 8 |
#include <net/pkt_cls.h> |
| 9 |
|
| 10 |
#include "mt76.h" |
| 11 |
#include "dma.h" |
| 12 |
#include "mt76_connac.h" |
| 13 |
|
| 14 |
#define MT76_NPU_RX_BUF_SIZE (1800 + \ |
| 15 |
SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) |
| 16 |
|
| 17 |
int mt76_npu_fill_rx_queue(struct mt76_dev *dev, struct mt76_queue *q) |
| 18 |
{ |
| 19 |
int nframes = 0; |
| 20 |
|
| 21 |
while (q->queued < q->ndesc - 1) { |
| 22 |
struct airoha_npu_rx_dma_desc *desc = (void *)q->desc; |
| 23 |
struct mt76_queue_entry *e = &q->entry[q->head]; |
| 24 |
struct page *page; |
| 25 |
int offset; |
| 26 |
|
| 27 |
e->buf = mt76_get_page_pool_buf(q, &offset, q->buf_size); |
| 28 |
if (!e->buf) |
| 29 |
break; |
| 30 |
|
| 31 |
e->dma_len[0] = SKB_WITH_OVERHEAD(q->buf_size); |
| 32 |
page = virt_to_head_page(e->buf); |
| 33 |
e->dma_addr[0] = page_pool_get_dma_addr(page) + offset; |
| 34 |
|
| 35 |
memset(&desc[q->head], 0, sizeof(*desc)); |
| 36 |
desc[q->head].addr = e->dma_addr[0]; |
| 37 |
|
| 38 |
q->head = (q->head + 1) % q->ndesc; |
| 39 |
q->queued++; |
| 40 |
nframes++; |
| 41 |
} |
| 42 |
|
| 43 |
return nframes; |
| 44 |
} |
| 45 |
|
| 46 |
void mt76_npu_queue_cleanup(struct mt76_dev *dev, struct mt76_queue *q) |
| 47 |
{ |
| 48 |
spin_lock_bh(&q->lock); |
| 49 |
while (q->queued > 0) { |
| 50 |
struct mt76_queue_entry *e = &q->entry[q->tail]; |
| 51 |
|
| 52 |
dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], |
| 53 |
e->dma_len[0], |
| 54 |
page_pool_get_dma_dir(q->page_pool)); |
| 55 |
mt76_put_page_pool_buf(e->buf, false); |
| 56 |
q->tail = (q->tail + 1) % q->ndesc; |
| 57 |
q->queued--; |
| 58 |
} |
| 59 |
spin_unlock_bh(&q->lock); |
| 60 |
} |
| 61 |
|
| 62 |
static struct sk_buff *mt76_npu_dequeue(struct mt76_dev *dev, |
| 63 |
struct mt76_queue *q, |
| 64 |
u32 *info) |
| 65 |
{ |
| 66 |
struct airoha_npu_rx_dma_desc *desc = (void *)q->desc; |
| 67 |
int i, nframes, index = q->tail; |
| 68 |
struct sk_buff *skb = NULL; |
| 69 |
|
| 70 |
nframes = FIELD_GET(NPU_RX_DMA_PKT_COUNT_MASK, desc[index].info); |
| 71 |
nframes = max_t(int, nframes, 1); |
| 72 |
|
| 73 |
for (i = 0; i < nframes; i++) { |
| 74 |
struct mt76_queue_entry *e = &q->entry[index]; |
| 75 |
int len = FIELD_GET(NPU_RX_DMA_DESC_CUR_LEN_MASK, |
| 76 |
desc[index].ctrl); |
| 77 |
|
| 78 |
if (!FIELD_GET(NPU_RX_DMA_DESC_DONE_MASK, desc[index].ctrl)) { |
| 79 |
dev_kfree_skb(skb); |
| 80 |
return NULL; |
| 81 |
} |
| 82 |
|
| 83 |
dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], |
| 84 |
e->dma_len[0], |
| 85 |
page_pool_get_dma_dir(q->page_pool)); |
| 86 |
|
| 87 |
if (!skb) { |
| 88 |
skb = napi_build_skb(e->buf, q->buf_size); |
| 89 |
if (!skb) |
| 90 |
return NULL; |
| 91 |
|
| 92 |
__skb_put(skb, len); |
| 93 |
skb_reset_mac_header(skb); |
| 94 |
skb_mark_for_recycle(skb); |
| 95 |
} else { |
| 96 |
struct skb_shared_info *shinfo = skb_shinfo(skb); |
| 97 |
struct page *page = virt_to_head_page(e->buf); |
| 98 |
int nr_frags = shinfo->nr_frags; |
| 99 |
|
| 100 |
if (nr_frags < ARRAY_SIZE(shinfo->frags)) |
| 101 |
skb_add_rx_frag(skb, nr_frags, page, |
| 102 |
e->buf - page_address(page), |
| 103 |
len, q->buf_size); |
| 104 |
} |
| 105 |
|
| 106 |
*info = desc[index].info; |
| 107 |
index = (index + 1) % q->ndesc; |
| 108 |
} |
| 109 |
q->tail = index; |
| 110 |
q->queued -= i; |
| 111 |
Q_WRITE(q, dma_idx, q->tail); |
| 112 |
|
| 113 |
return skb; |
| 114 |
} |
| 115 |
|
| 116 |
void mt76_npu_check_ppe(struct mt76_dev *dev, struct sk_buff *skb, |
| 117 |
u32 info) |
| 118 |
{ |
| 119 |
struct airoha_ppe_dev *ppe_dev; |
| 120 |
u16 reason, hash; |
| 121 |
|
| 122 |
if (!mt76_npu_device_active(dev)) |
| 123 |
return; |
| 124 |
|
| 125 |
rcu_read_lock(); |
| 126 |
|
| 127 |
ppe_dev = rcu_dereference(dev->mmio.ppe_dev); |
| 128 |
if (!ppe_dev) |
| 129 |
goto out; |
| 130 |
|
| 131 |
hash = FIELD_GET(NPU_RX_DMA_FOE_ID_MASK, info); |
| 132 |
skb_set_hash(skb, hash, PKT_HASH_TYPE_L4); |
| 133 |
|
| 134 |
reason = FIELD_GET(NPU_RX_DMA_CRSN_MASK, info); |
| 135 |
if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) { |
| 136 |
skb_set_mac_header(skb, 0); |
| 137 |
airoha_ppe_dev_check_skb(ppe_dev, skb, hash, true); |
| 138 |
} |
| 139 |
out: |
| 140 |
rcu_read_unlock(); |
| 141 |
} |
| 142 |
EXPORT_SYMBOL_GPL(mt76_npu_check_ppe); |
| 143 |
|
| 144 |
static int mt76_npu_rx_poll(struct napi_struct *napi, int budget) |
| 145 |
{ |
| 146 |
struct mt76_dev *dev = mt76_priv(napi->dev); |
| 147 |
enum mt76_rxq_id qid = napi - dev->napi; |
| 148 |
struct airoha_npu *npu; |
| 149 |
int done = 0; |
| 150 |
|
| 151 |
rcu_read_lock(); |
| 152 |
|
| 153 |
npu = rcu_dereference(dev->mmio.npu); |
| 154 |
if (!npu) |
| 155 |
goto out; |
| 156 |
|
| 157 |
while (done < budget) { |
| 158 |
struct sk_buff *skb; |
| 159 |
u32 info = 0; |
| 160 |
|
| 161 |
skb = mt76_npu_dequeue(dev, &dev->q_rx[qid], &info); |
| 162 |
if (!skb) |
| 163 |
break; |
| 164 |
|
| 165 |
dev->drv->rx_skb(dev, qid, skb, &info); |
| 166 |
mt76_rx_poll_complete(dev, qid, napi); |
| 167 |
done++; |
| 168 |
} |
| 169 |
|
| 170 |
mt76_npu_fill_rx_queue(dev, &dev->q_rx[qid]); |
| 171 |
out: |
| 172 |
if (done < budget && napi_complete(napi)) |
| 173 |
dev->drv->rx_poll_complete(dev, qid); |
| 174 |
|
| 175 |
rcu_read_unlock(); |
| 176 |
|
| 177 |
return done; |
| 178 |
} |
| 179 |
|
| 180 |
static irqreturn_t mt76_npu_irq_handler(int irq, void *q_instance) |
| 181 |
{ |
| 182 |
struct mt76_queue *q = q_instance; |
| 183 |
struct mt76_dev *dev = q->dev; |
| 184 |
int qid = q - &dev->q_rx[0]; |
| 185 |
int index = qid - MT_RXQ_NPU0; |
| 186 |
struct airoha_npu *npu; |
| 187 |
u32 status; |
| 188 |
|
| 189 |
rcu_read_lock(); |
| 190 |
|
| 191 |
npu = rcu_dereference(dev->mmio.npu); |
| 192 |
if (!npu) |
| 193 |
goto out; |
| 194 |
|
| 195 |
status = airoha_npu_wlan_get_irq_status(npu, index); |
| 196 |
airoha_npu_wlan_set_irq_status(npu, status); |
| 197 |
|
| 198 |
airoha_npu_wlan_disable_irq(npu, index); |
| 199 |
napi_schedule(&dev->napi[qid]); |
| 200 |
out: |
| 201 |
rcu_read_unlock(); |
| 202 |
|
| 203 |
return IRQ_HANDLED; |
| 204 |
} |
| 205 |
|
| 206 |
int mt76_npu_dma_add_buf(struct mt76_phy *phy, struct mt76_queue *q, |
| 207 |
struct sk_buff *skb, struct mt76_queue_buf *buf, |
| 208 |
void *txwi_ptr) |
| 209 |
{ |
| 210 |
u16 txwi_len = min_t(u16, phy->dev->drv->txwi_size, NPU_TXWI_LEN); |
| 211 |
struct airoha_npu_tx_dma_desc *desc = (void *)q->desc; |
| 212 |
int ret; |
| 213 |
|
| 214 |
/* TODO: Take into account unlinear skbs */ |
| 215 |
memcpy(desc[q->head].txwi, txwi_ptr, txwi_len); |
| 216 |
desc[q->head].addr = buf->addr; |
| 217 |
desc[q->head].ctrl = FIELD_PREP(NPU_TX_DMA_DESC_VEND_LEN_MASK, txwi_len) | |
| 218 |
FIELD_PREP(NPU_TX_DMA_DESC_LEN_MASK, skb->len) | |
| 219 |
NPU_TX_DMA_DESC_DONE_MASK; |
| 220 |
|
| 221 |
ret = q->head; |
| 222 |
q->entry[q->head].skip_buf0 = true; |
| 223 |
q->entry[q->head].skip_buf1 = true; |
| 224 |
q->entry[q->head].txwi = NULL; |
| 225 |
q->entry[q->head].skb = NULL; |
| 226 |
q->entry[q->head].wcid = 0xffff; |
| 227 |
|
| 228 |
q->head = (q->head + 1) % q->ndesc; |
| 229 |
q->queued++; |
| 230 |
|
| 231 |
return ret; |
| 232 |
} |
| 233 |
|
| 234 |
void mt76_npu_txdesc_cleanup(struct mt76_queue *q, int index) |
| 235 |
{ |
| 236 |
struct airoha_npu_tx_dma_desc *desc = (void *)q->desc; |
| 237 |
|
| 238 |
if (!mt76_queue_is_npu_tx(q)) |
| 239 |
return; |
| 240 |
|
| 241 |
desc[index].ctrl &= ~NPU_TX_DMA_DESC_DONE_MASK; |
| 242 |
} |
| 243 |
|
| 244 |
void mt76_npu_queue_setup(struct mt76_dev *dev, struct mt76_queue *q) |
| 245 |
{ |
| 246 |
int qid = FIELD_GET(MT_QFLAG_WED_RING, q->flags); |
| 247 |
bool xmit = mt76_queue_is_npu_tx(q); |
| 248 |
struct airoha_npu *npu; |
| 249 |
|
| 250 |
if (!mt76_queue_is_npu(q)) |
| 251 |
return; |
| 252 |
|
| 253 |
npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex); |
| 254 |
if (npu) |
| 255 |
q->wed_regs = airoha_npu_wlan_get_queue_addr(npu, qid, xmit); |
| 256 |
} |
| 257 |
|
| 258 |
int mt76_npu_rx_queue_init(struct mt76_dev *dev, struct mt76_queue *q) |
| 259 |
{ |
| 260 |
int err, irq, qid = q - &dev->q_rx[0]; |
| 261 |
int size, index = qid - MT_RXQ_NPU0; |
| 262 |
struct airoha_npu *npu; |
| 263 |
const char *name; |
| 264 |
|
| 265 |
mutex_lock(&dev->mutex); |
| 266 |
|
| 267 |
npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex); |
| 268 |
irq = npu && index < ARRAY_SIZE(npu->irqs) ? npu->irqs[index] |
| 269 |
: -EINVAL; |
| 270 |
if (irq < 0) { |
| 271 |
err = irq; |
| 272 |
goto out; |
| 273 |
} |
| 274 |
|
| 275 |
q->flags = MT_NPU_Q_RX(index); |
| 276 |
size = qid == MT_RXQ_NPU1 ? NPU_RX1_DESC_NUM : NPU_RX0_DESC_NUM; |
| 277 |
err = dev->queue_ops->alloc(dev, q, 0, size, |
| 278 |
MT76_NPU_RX_BUF_SIZE, 0); |
| 279 |
if (err) |
| 280 |
goto out; |
| 281 |
|
| 282 |
name = devm_kasprintf(dev->dev, GFP_KERNEL, "mt76-npu.%d", index); |
| 283 |
if (!name) { |
| 284 |
err = -ENOMEM; |
| 285 |
goto out; |
| 286 |
} |
| 287 |
|
| 288 |
err = devm_request_irq(dev->dev, irq, mt76_npu_irq_handler, |
| 289 |
IRQF_SHARED, name, q); |
| 290 |
if (err) |
| 291 |
goto out; |
| 292 |
|
| 293 |
netif_napi_add(dev->napi_dev, &dev->napi[qid], mt76_npu_rx_poll); |
| 294 |
mt76_npu_fill_rx_queue(dev, q); |
| 295 |
napi_enable(&dev->napi[qid]); |
| 296 |
out: |
| 297 |
mutex_unlock(&dev->mutex); |
| 298 |
|
| 299 |
return err; |
| 300 |
} |
| 301 |
EXPORT_SYMBOL_GPL(mt76_npu_rx_queue_init); |
| 302 |
|
| 303 |
static int mt76_npu_setup_tc_block_cb(enum tc_setup_type type, |
| 304 |
void *type_data, void *cb_priv) |
| 305 |
{ |
| 306 |
struct mt76_phy *phy = cb_priv; |
| 307 |
struct mt76_dev *dev = phy->dev; |
| 308 |
struct airoha_ppe_dev *ppe_dev; |
| 309 |
int err = -EOPNOTSUPP; |
| 310 |
|
| 311 |
if (type != TC_SETUP_CLSFLOWER) |
| 312 |
return -EOPNOTSUPP; |
| 313 |
|
| 314 |
mutex_lock(&dev->mutex); |
| 315 |
|
| 316 |
ppe_dev = rcu_dereference_protected(dev->mmio.ppe_dev, &dev->mutex); |
| 317 |
if (ppe_dev) |
| 318 |
err = airoha_ppe_dev_setup_tc_block_cb(ppe_dev, type_data); |
| 319 |
|
| 320 |
mutex_unlock(&dev->mutex); |
| 321 |
|
| 322 |
return err; |
| 323 |
} |
| 324 |
|
| 325 |
static int mt76_npu_setup_tc_block(struct mt76_phy *phy, |
| 326 |
struct net_device *dev, |
| 327 |
struct flow_block_offload *f) |
| 328 |
{ |
| 329 |
flow_setup_cb_t *cb = mt76_npu_setup_tc_block_cb; |
| 330 |
static LIST_HEAD(block_cb_list); |
| 331 |
struct flow_block_cb *block_cb; |
| 332 |
|
| 333 |
if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) |
| 334 |
return -EOPNOTSUPP; |
| 335 |
|
| 336 |
if (!tc_can_offload(dev)) |
| 337 |
return -EOPNOTSUPP; |
| 338 |
|
| 339 |
f->driver_block_list = &block_cb_list; |
| 340 |
switch (f->command) { |
| 341 |
case FLOW_BLOCK_BIND: |
| 342 |
block_cb = flow_block_cb_lookup(f->block, cb, dev); |
| 343 |
if (block_cb) { |
| 344 |
flow_block_cb_incref(block_cb); |
| 345 |
return 0; |
| 346 |
} |
| 347 |
|
| 348 |
block_cb = flow_block_cb_alloc(cb, dev, phy, NULL); |
| 349 |
if (IS_ERR(block_cb)) |
| 350 |
return PTR_ERR(block_cb); |
| 351 |
|
| 352 |
flow_block_cb_incref(block_cb); |
| 353 |
flow_block_cb_add(block_cb, f); |
| 354 |
list_add_tail(&block_cb->driver_list, &block_cb_list); |
| 355 |
return 0; |
| 356 |
case FLOW_BLOCK_UNBIND: |
| 357 |
block_cb = flow_block_cb_lookup(f->block, cb, dev); |
| 358 |
if (!block_cb) |
| 359 |
return -ENOENT; |
| 360 |
|
| 361 |
if (!flow_block_cb_decref(block_cb)) { |
| 362 |
flow_block_cb_remove(block_cb, f); |
| 363 |
list_del(&block_cb->driver_list); |
| 364 |
} |
| 365 |
return 0; |
| 366 |
default: |
| 367 |
return -EOPNOTSUPP; |
| 368 |
} |
| 369 |
} |
| 370 |
|
| 371 |
int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif, |
| 372 |
struct net_device *dev, enum tc_setup_type type, |
| 373 |
void *type_data) |
| 374 |
{ |
| 375 |
struct mt76_phy *phy = hw->priv; |
| 376 |
|
| 377 |
if (!tc_can_offload(dev)) |
| 378 |
return -EOPNOTSUPP; |
| 379 |
|
| 380 |
if (!mt76_npu_device_active(phy->dev)) |
| 381 |
return -EOPNOTSUPP; |
| 382 |
|
| 383 |
switch (type) { |
| 384 |
case TC_SETUP_BLOCK: |
| 385 |
case TC_SETUP_FT: |
| 386 |
return mt76_npu_setup_tc_block(phy, dev, type_data); |
| 387 |
default: |
| 388 |
return -EOPNOTSUPP; |
| 389 |
} |
| 390 |
} |
| 391 |
EXPORT_SYMBOL_GPL(mt76_npu_net_setup_tc); |
| 392 |
|
| 393 |
void mt76_npu_disable_irqs(struct mt76_dev *dev) |
| 394 |
{ |
| 395 |
struct airoha_npu *npu; |
| 396 |
int i; |
| 397 |
|
| 398 |
rcu_read_lock(); |
| 399 |
|
| 400 |
npu = rcu_dereference(dev->mmio.npu); |
| 401 |
if (!npu) |
| 402 |
goto unlock; |
| 403 |
|
| 404 |
for (i = MT_RXQ_NPU0; i <= MT_RXQ_NPU1; i++) { |
| 405 |
int qid = i - MT_RXQ_NPU0; |
| 406 |
u32 status; |
| 407 |
|
| 408 |
status = airoha_npu_wlan_get_irq_status(npu, qid); |
| 409 |
airoha_npu_wlan_set_irq_status(npu, status); |
| 410 |
airoha_npu_wlan_disable_irq(npu, qid); |
| 411 |
} |
| 412 |
unlock: |
| 413 |
rcu_read_unlock(); |
| 414 |
} |
| 415 |
EXPORT_SYMBOL_GPL(mt76_npu_disable_irqs); |
| 416 |
|
| 417 |
int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, int type) |
| 418 |
{ |
| 419 |
struct airoha_ppe_dev *ppe_dev; |
| 420 |
struct airoha_npu *npu; |
| 421 |
int err = 0; |
| 422 |
|
| 423 |
/* NPU offloading is only supported by MT7992 */ |
| 424 |
if (!is_mt7992(dev)) |
| 425 |
return 0; |
| 426 |
|
| 427 |
mutex_lock(&dev->mutex); |
| 428 |
|
| 429 |
npu = airoha_npu_get(dev->dev); |
| 430 |
if (IS_ERR(npu)) { |
| 431 |
request_module("airoha-npu"); |
| 432 |
npu = airoha_npu_get(dev->dev); |
| 433 |
} |
| 434 |
|
| 435 |
if (IS_ERR(npu)) { |
| 436 |
err = PTR_ERR(npu); |
| 437 |
goto error_unlock; |
| 438 |
} |
| 439 |
|
| 440 |
ppe_dev = airoha_ppe_get_dev(dev->dev); |
| 441 |
if (IS_ERR(ppe_dev)) { |
| 442 |
request_module("airoha-eth"); |
| 443 |
ppe_dev = airoha_ppe_get_dev(dev->dev); |
| 444 |
} |
| 445 |
|
| 446 |
if (IS_ERR(ppe_dev)) { |
| 447 |
err = PTR_ERR(ppe_dev); |
| 448 |
goto error_npu_put; |
| 449 |
} |
| 450 |
|
| 451 |
err = airoha_npu_wlan_init_reserved_memory(npu); |
| 452 |
if (err) |
| 453 |
goto error_ppe_put; |
| 454 |
|
| 455 |
dev->dma_dev = npu->dev; |
| 456 |
dev->mmio.phy_addr = phy_addr; |
| 457 |
dev->mmio.npu_type = type; |
| 458 |
/* NPU offloading requires HW-RRO for RX packet reordering. */ |
| 459 |
dev->hwrro_mode = MT76_HWRRO_V3_1; |
| 460 |
dev->rx_token_size = 32768; |
| 461 |
|
| 462 |
rcu_assign_pointer(dev->mmio.npu, npu); |
| 463 |
rcu_assign_pointer(dev->mmio.ppe_dev, ppe_dev); |
| 464 |
synchronize_rcu(); |
| 465 |
|
| 466 |
mutex_unlock(&dev->mutex); |
| 467 |
|
| 468 |
return 0; |
| 469 |
|
| 470 |
error_ppe_put: |
| 471 |
airoha_ppe_put_dev(ppe_dev); |
| 472 |
error_npu_put: |
| 473 |
airoha_npu_put(npu); |
| 474 |
error_unlock: |
| 475 |
mutex_unlock(&dev->mutex); |
| 476 |
|
| 477 |
return err; |
| 478 |
} |
| 479 |
EXPORT_SYMBOL_GPL(mt76_npu_init); |
| 480 |
|
| 481 |
void mt76_npu_deinit(struct mt76_dev *dev) |
| 482 |
{ |
| 483 |
struct airoha_ppe_dev *ppe_dev; |
| 484 |
struct airoha_npu *npu; |
| 485 |
|
| 486 |
mutex_lock(&dev->mutex); |
| 487 |
|
| 488 |
npu = rcu_replace_pointer(dev->mmio.npu, NULL, |
| 489 |
lockdep_is_held(&dev->mutex)); |
| 490 |
if (npu) |
| 491 |
airoha_npu_put(npu); |
| 492 |
|
| 493 |
ppe_dev = rcu_replace_pointer(dev->mmio.ppe_dev, NULL, |
| 494 |
lockdep_is_held(&dev->mutex)); |
| 495 |
if (ppe_dev) |
| 496 |
airoha_ppe_put_dev(ppe_dev); |
| 497 |
|
| 498 |
mutex_unlock(&dev->mutex); |
| 499 |
|
| 500 |
mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU0]); |
| 501 |
mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU1]); |
| 502 |
} |