| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
/* Marvell RVU Virtual Function ethernet driver |
| 3 |
* |
| 4 |
* Copyright (C) 2020 Marvell. |
| 5 |
* |
| 6 |
*/ |
| 7 |
|
| 8 |
#include <linux/etherdevice.h> |
| 9 |
#include <linux/module.h> |
| 10 |
#include <linux/pci.h> |
| 11 |
#include <linux/net_tstamp.h> |
| 12 |
|
| 13 |
#include "otx2_common.h" |
| 14 |
#include "otx2_reg.h" |
| 15 |
#include "otx2_ptp.h" |
| 16 |
#include "cn10k.h" |
| 17 |
#include "cn10k_ipsec.h" |
| 18 |
|
| 19 |
#define DRV_NAME "rvu_nicvf" |
| 20 |
#define DRV_STRING "Marvell RVU NIC Virtual Function Driver" |
| 21 |
|
| 22 |
static const struct pci_device_id otx2_vf_id_table[] = { |
| 23 |
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_AFVF) }, |
| 24 |
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_VF) }, |
| 25 |
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_SDP_REP) }, |
| 26 |
{ } |
| 27 |
}; |
| 28 |
|
| 29 |
MODULE_AUTHOR("Sunil Goutham <[email protected]>"); |
| 30 |
MODULE_DESCRIPTION(DRV_STRING); |
| 31 |
MODULE_LICENSE("GPL v2"); |
| 32 |
MODULE_DEVICE_TABLE(pci, otx2_vf_id_table); |
| 33 |
|
| 34 |
/* RVU VF Interrupt Vector Enumeration */ |
| 35 |
enum { |
| 36 |
RVU_VF_INT_VEC_MBOX = 0x0, |
| 37 |
}; |
| 38 |
|
| 39 |
static void otx2vf_process_vfaf_mbox_msg(struct otx2_nic *vf, |
| 40 |
struct mbox_msghdr *msg) |
| 41 |
{ |
| 42 |
if (msg->id >= MBOX_MSG_MAX) { |
| 43 |
dev_err(vf->dev, |
| 44 |
"Mbox msg with unknown ID %d\n", msg->id); |
| 45 |
return; |
| 46 |
} |
| 47 |
|
| 48 |
if (msg->sig != OTX2_MBOX_RSP_SIG) { |
| 49 |
dev_err(vf->dev, |
| 50 |
"Mbox msg with wrong signature %x, ID %d\n", |
| 51 |
msg->sig, msg->id); |
| 52 |
return; |
| 53 |
} |
| 54 |
|
| 55 |
if (msg->rc == MBOX_MSG_INVALID) { |
| 56 |
dev_err(vf->dev, |
| 57 |
"PF/AF says the sent msg(s) %d were invalid\n", |
| 58 |
msg->id); |
| 59 |
return; |
| 60 |
} |
| 61 |
|
| 62 |
switch (msg->id) { |
| 63 |
case MBOX_MSG_READY: |
| 64 |
vf->pcifunc = msg->pcifunc; |
| 65 |
break; |
| 66 |
case MBOX_MSG_MSIX_OFFSET: |
| 67 |
mbox_handler_msix_offset(vf, (struct msix_offset_rsp *)msg); |
| 68 |
break; |
| 69 |
case MBOX_MSG_NPA_LF_ALLOC: |
| 70 |
mbox_handler_npa_lf_alloc(vf, (struct npa_lf_alloc_rsp *)msg); |
| 71 |
break; |
| 72 |
case MBOX_MSG_NIX_LF_ALLOC: |
| 73 |
mbox_handler_nix_lf_alloc(vf, (struct nix_lf_alloc_rsp *)msg); |
| 74 |
break; |
| 75 |
case MBOX_MSG_NIX_BP_ENABLE: |
| 76 |
mbox_handler_nix_bp_enable(vf, (struct nix_bp_cfg_rsp *)msg); |
| 77 |
break; |
| 78 |
default: |
| 79 |
if (msg->rc) |
| 80 |
dev_err(vf->dev, |
| 81 |
"Mbox msg response has err %d, ID %d\n", |
| 82 |
msg->rc, msg->id); |
| 83 |
} |
| 84 |
} |
| 85 |
|
| 86 |
static void otx2vf_vfaf_mbox_handler(struct work_struct *work) |
| 87 |
{ |
| 88 |
struct otx2_mbox_dev *mdev; |
| 89 |
struct mbox_hdr *rsp_hdr; |
| 90 |
struct mbox_msghdr *msg; |
| 91 |
struct otx2_mbox *mbox; |
| 92 |
struct mbox *af_mbox; |
| 93 |
int offset, id; |
| 94 |
u16 num_msgs; |
| 95 |
|
| 96 |
af_mbox = container_of(work, struct mbox, mbox_wrk); |
| 97 |
mbox = &af_mbox->mbox; |
| 98 |
mdev = &mbox->dev[0]; |
| 99 |
rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); |
| 100 |
num_msgs = rsp_hdr->num_msgs; |
| 101 |
|
| 102 |
if (num_msgs == 0) |
| 103 |
return; |
| 104 |
|
| 105 |
offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN); |
| 106 |
|
| 107 |
for (id = 0; id < num_msgs; id++) { |
| 108 |
msg = (struct mbox_msghdr *)(mdev->mbase + offset); |
| 109 |
otx2vf_process_vfaf_mbox_msg(af_mbox->pfvf, msg); |
| 110 |
offset = mbox->rx_start + msg->next_msgoff; |
| 111 |
if (mdev->msgs_acked == (af_mbox->num_msgs - 1)) |
| 112 |
__otx2_mbox_reset(mbox, 0); |
| 113 |
mdev->msgs_acked++; |
| 114 |
} |
| 115 |
} |
| 116 |
|
| 117 |
static int otx2vf_process_mbox_msg_up(struct otx2_nic *vf, |
| 118 |
struct mbox_msghdr *req) |
| 119 |
{ |
| 120 |
struct msg_rsp *rsp; |
| 121 |
int err; |
| 122 |
|
| 123 |
/* Check if valid, if not reply with a invalid msg */ |
| 124 |
if (req->sig != OTX2_MBOX_REQ_SIG) { |
| 125 |
otx2_reply_invalid_msg(&vf->mbox.mbox_up, 0, 0, req->id); |
| 126 |
return -ENODEV; |
| 127 |
} |
| 128 |
|
| 129 |
switch (req->id) { |
| 130 |
case MBOX_MSG_CGX_LINK_EVENT: |
| 131 |
rsp = (struct msg_rsp *)otx2_mbox_alloc_msg( |
| 132 |
&vf->mbox.mbox_up, 0, |
| 133 |
sizeof(struct msg_rsp)); |
| 134 |
if (!rsp) |
| 135 |
return -ENOMEM; |
| 136 |
|
| 137 |
rsp->hdr.id = MBOX_MSG_CGX_LINK_EVENT; |
| 138 |
rsp->hdr.sig = OTX2_MBOX_RSP_SIG; |
| 139 |
rsp->hdr.pcifunc = req->pcifunc; |
| 140 |
rsp->hdr.rc = 0; |
| 141 |
err = otx2_mbox_up_handler_cgx_link_event( |
| 142 |
vf, (struct cgx_link_info_msg *)req, rsp); |
| 143 |
return err; |
| 144 |
default: |
| 145 |
otx2_reply_invalid_msg(&vf->mbox.mbox_up, 0, 0, req->id); |
| 146 |
return -ENODEV; |
| 147 |
} |
| 148 |
return 0; |
| 149 |
} |
| 150 |
|
| 151 |
static void otx2vf_vfaf_mbox_up_handler(struct work_struct *work) |
| 152 |
{ |
| 153 |
struct otx2_mbox_dev *mdev; |
| 154 |
struct mbox_hdr *rsp_hdr; |
| 155 |
struct mbox_msghdr *msg; |
| 156 |
struct otx2_mbox *mbox; |
| 157 |
struct mbox *vf_mbox; |
| 158 |
struct otx2_nic *vf; |
| 159 |
int offset, id; |
| 160 |
u16 num_msgs; |
| 161 |
|
| 162 |
vf_mbox = container_of(work, struct mbox, mbox_up_wrk); |
| 163 |
vf = vf_mbox->pfvf; |
| 164 |
mbox = &vf_mbox->mbox_up; |
| 165 |
mdev = &mbox->dev[0]; |
| 166 |
|
| 167 |
rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); |
| 168 |
num_msgs = rsp_hdr->num_msgs; |
| 169 |
|
| 170 |
if (num_msgs == 0) |
| 171 |
return; |
| 172 |
|
| 173 |
offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN); |
| 174 |
|
| 175 |
for (id = 0; id < num_msgs; id++) { |
| 176 |
msg = (struct mbox_msghdr *)(mdev->mbase + offset); |
| 177 |
otx2vf_process_mbox_msg_up(vf, msg); |
| 178 |
offset = mbox->rx_start + msg->next_msgoff; |
| 179 |
} |
| 180 |
|
| 181 |
otx2_mbox_msg_send(mbox, 0); |
| 182 |
} |
| 183 |
|
| 184 |
static irqreturn_t otx2vf_vfaf_mbox_intr_handler(int irq, void *vf_irq) |
| 185 |
{ |
| 186 |
struct otx2_nic *vf = (struct otx2_nic *)vf_irq; |
| 187 |
struct otx2_mbox_dev *mdev; |
| 188 |
struct otx2_mbox *mbox; |
| 189 |
struct mbox_hdr *hdr; |
| 190 |
u64 mbox_data; |
| 191 |
|
| 192 |
/* Clear the IRQ */ |
| 193 |
otx2_write64(vf, RVU_VF_INT, BIT_ULL(0)); |
| 194 |
|
| 195 |
mbox_data = otx2_read64(vf, RVU_VF_VFPF_MBOX0); |
| 196 |
|
| 197 |
/* Read latest mbox data */ |
| 198 |
smp_rmb(); |
| 199 |
|
| 200 |
if (mbox_data & MBOX_DOWN_MSG) { |
| 201 |
mbox_data &= ~MBOX_DOWN_MSG; |
| 202 |
otx2_write64(vf, RVU_VF_VFPF_MBOX0, mbox_data); |
| 203 |
|
| 204 |
/* Check for PF => VF response messages */ |
| 205 |
mbox = &vf->mbox.mbox; |
| 206 |
mdev = &mbox->dev[0]; |
| 207 |
otx2_sync_mbox_bbuf(mbox, 0); |
| 208 |
|
| 209 |
hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); |
| 210 |
if (hdr->num_msgs) |
| 211 |
queue_work(vf->mbox_wq, &vf->mbox.mbox_wrk); |
| 212 |
|
| 213 |
trace_otx2_msg_interrupt(mbox->pdev, "DOWN reply from PF to VF", |
| 214 |
BIT_ULL(0)); |
| 215 |
} |
| 216 |
|
| 217 |
if (mbox_data & MBOX_UP_MSG) { |
| 218 |
mbox_data &= ~MBOX_UP_MSG; |
| 219 |
otx2_write64(vf, RVU_VF_VFPF_MBOX0, mbox_data); |
| 220 |
|
| 221 |
/* Check for PF => VF notification messages */ |
| 222 |
mbox = &vf->mbox.mbox_up; |
| 223 |
mdev = &mbox->dev[0]; |
| 224 |
otx2_sync_mbox_bbuf(mbox, 0); |
| 225 |
|
| 226 |
hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); |
| 227 |
if (hdr->num_msgs) |
| 228 |
queue_work(vf->mbox_wq, &vf->mbox.mbox_up_wrk); |
| 229 |
|
| 230 |
trace_otx2_msg_interrupt(mbox->pdev, "UP message from PF to VF", |
| 231 |
BIT_ULL(0)); |
| 232 |
} |
| 233 |
|
| 234 |
return IRQ_HANDLED; |
| 235 |
} |
| 236 |
|
| 237 |
static void otx2vf_disable_mbox_intr(struct otx2_nic *vf) |
| 238 |
{ |
| 239 |
int vector = pci_irq_vector(vf->pdev, RVU_VF_INT_VEC_MBOX); |
| 240 |
|
| 241 |
/* Disable VF => PF mailbox IRQ */ |
| 242 |
otx2_write64(vf, RVU_VF_INT_ENA_W1C, BIT_ULL(0)); |
| 243 |
|
| 244 |
if (is_cn20k(vf->pdev)) |
| 245 |
otx2_write64(vf, RVU_VF_INT_ENA_W1C, BIT_ULL(0) | BIT_ULL(1)); |
| 246 |
|
| 247 |
free_irq(vector, vf); |
| 248 |
} |
| 249 |
|
| 250 |
static int otx2vf_register_mbox_intr(struct otx2_nic *vf, bool probe_pf) |
| 251 |
{ |
| 252 |
struct otx2_hw *hw = &vf->hw; |
| 253 |
struct msg_req *req; |
| 254 |
char *irq_name; |
| 255 |
int err; |
| 256 |
|
| 257 |
/* Register mailbox interrupt handler */ |
| 258 |
irq_name = &hw->irq_name[RVU_VF_INT_VEC_MBOX * NAME_SIZE]; |
| 259 |
snprintf(irq_name, NAME_SIZE, "RVUVF%d AFVF Mbox", ((vf->pcifunc & |
| 260 |
RVU_PFVF_FUNC_MASK) - 1)); |
| 261 |
|
| 262 |
if (!is_cn20k(vf->pdev)) { |
| 263 |
err = request_irq(pci_irq_vector(vf->pdev, RVU_VF_INT_VEC_MBOX), |
| 264 |
otx2vf_vfaf_mbox_intr_handler, 0, irq_name, vf); |
| 265 |
} else { |
| 266 |
err = request_irq(pci_irq_vector(vf->pdev, RVU_VF_INT_VEC_MBOX), |
| 267 |
vf->hw_ops->vfaf_mbox_intr_handler, 0, irq_name, |
| 268 |
vf); |
| 269 |
} |
| 270 |
|
| 271 |
if (err) { |
| 272 |
dev_err(vf->dev, |
| 273 |
"RVUPF: IRQ registration failed for VFAF mbox irq\n"); |
| 274 |
return err; |
| 275 |
} |
| 276 |
|
| 277 |
/* Enable mailbox interrupt for msgs coming from PF. |
| 278 |
* First clear to avoid spurious interrupts, if any. |
| 279 |
*/ |
| 280 |
if (!is_cn20k(vf->pdev)) { |
| 281 |
otx2_write64(vf, RVU_VF_INT, BIT_ULL(0)); |
| 282 |
otx2_write64(vf, RVU_VF_INT_ENA_W1S, BIT_ULL(0)); |
| 283 |
} else { |
| 284 |
otx2_write64(vf, RVU_VF_INT, BIT_ULL(0) | BIT_ULL(1) | |
| 285 |
BIT_ULL(2) | BIT_ULL(3)); |
| 286 |
otx2_write64(vf, RVU_VF_INT_ENA_W1S, BIT_ULL(0) | |
| 287 |
BIT_ULL(1) | BIT_ULL(2) | BIT_ULL(3)); |
| 288 |
} |
| 289 |
|
| 290 |
if (!probe_pf) |
| 291 |
return 0; |
| 292 |
|
| 293 |
/* Check mailbox communication with PF */ |
| 294 |
req = otx2_mbox_alloc_msg_ready(&vf->mbox); |
| 295 |
if (!req) { |
| 296 |
otx2vf_disable_mbox_intr(vf); |
| 297 |
return -ENOMEM; |
| 298 |
} |
| 299 |
|
| 300 |
err = otx2_sync_mbox_msg(&vf->mbox); |
| 301 |
if (err) { |
| 302 |
dev_warn(vf->dev, |
| 303 |
"AF not responding to mailbox, deferring probe\n"); |
| 304 |
otx2vf_disable_mbox_intr(vf); |
| 305 |
return -EPROBE_DEFER; |
| 306 |
} |
| 307 |
return 0; |
| 308 |
} |
| 309 |
|
| 310 |
static void otx2vf_vfaf_mbox_destroy(struct otx2_nic *vf) |
| 311 |
{ |
| 312 |
struct mbox *mbox = &vf->mbox; |
| 313 |
|
| 314 |
if (vf->mbox_wq) { |
| 315 |
destroy_workqueue(vf->mbox_wq); |
| 316 |
vf->mbox_wq = NULL; |
| 317 |
} |
| 318 |
|
| 319 |
if (mbox->mbox.hwbase && !test_bit(CN10K_MBOX, &vf->hw.cap_flag)) |
| 320 |
iounmap((void __iomem *)mbox->mbox.hwbase); |
| 321 |
|
| 322 |
otx2_mbox_destroy(&mbox->mbox); |
| 323 |
otx2_mbox_destroy(&mbox->mbox_up); |
| 324 |
} |
| 325 |
|
| 326 |
static int otx2vf_vfaf_mbox_init(struct otx2_nic *vf) |
| 327 |
{ |
| 328 |
struct mbox *mbox = &vf->mbox; |
| 329 |
void __iomem *hwbase; |
| 330 |
int err; |
| 331 |
|
| 332 |
mbox->pfvf = vf; |
| 333 |
vf->mbox_wq = alloc_ordered_workqueue("otx2_vfaf_mailbox", |
| 334 |
WQ_HIGHPRI | WQ_MEM_RECLAIM); |
| 335 |
if (!vf->mbox_wq) |
| 336 |
return -ENOMEM; |
| 337 |
|
| 338 |
/* For cn20k platform, VF mailbox region is in dram aliased from AF |
| 339 |
* VF MBOX ADDR, MBOX is a separate RVU block. |
| 340 |
*/ |
| 341 |
if (is_cn20k(vf->pdev)) { |
| 342 |
hwbase = vf->reg_base + RVU_VF_MBOX_REGION + ((u64)BLKADDR_MBOX << |
| 343 |
RVU_FUNC_BLKADDR_SHIFT); |
| 344 |
} else if (test_bit(CN10K_MBOX, &vf->hw.cap_flag)) { |
| 345 |
/* For cn10k platform, VF mailbox region is in its BAR2 |
| 346 |
* register space |
| 347 |
*/ |
| 348 |
hwbase = vf->reg_base + RVU_VF_MBOX_REGION; |
| 349 |
} else { |
| 350 |
/* Mailbox is a reserved memory (in RAM) region shared between |
| 351 |
* admin function (i.e PF0) and this VF, shouldn't be mapped as |
| 352 |
* device memory to allow unaligned accesses. |
| 353 |
*/ |
| 354 |
hwbase = ioremap_wc(pci_resource_start(vf->pdev, |
| 355 |
PCI_MBOX_BAR_NUM), |
| 356 |
pci_resource_len(vf->pdev, |
| 357 |
PCI_MBOX_BAR_NUM)); |
| 358 |
if (!hwbase) { |
| 359 |
dev_err(vf->dev, "Unable to map VFAF mailbox region\n"); |
| 360 |
err = -ENOMEM; |
| 361 |
goto exit; |
| 362 |
} |
| 363 |
} |
| 364 |
|
| 365 |
err = otx2_mbox_init(&mbox->mbox, hwbase, vf->pdev, vf->reg_base, |
| 366 |
MBOX_DIR_VFPF, 1); |
| 367 |
if (err) |
| 368 |
goto exit; |
| 369 |
|
| 370 |
err = otx2_mbox_init(&mbox->mbox_up, hwbase, vf->pdev, vf->reg_base, |
| 371 |
MBOX_DIR_VFPF_UP, 1); |
| 372 |
if (err) |
| 373 |
goto exit; |
| 374 |
|
| 375 |
err = otx2_mbox_bbuf_init(mbox, vf->pdev); |
| 376 |
if (err) |
| 377 |
goto exit; |
| 378 |
|
| 379 |
INIT_WORK(&mbox->mbox_wrk, otx2vf_vfaf_mbox_handler); |
| 380 |
INIT_WORK(&mbox->mbox_up_wrk, otx2vf_vfaf_mbox_up_handler); |
| 381 |
mutex_init(&mbox->lock); |
| 382 |
|
| 383 |
return 0; |
| 384 |
exit: |
| 385 |
if (hwbase && !test_bit(CN10K_MBOX, &vf->hw.cap_flag)) |
| 386 |
iounmap(hwbase); |
| 387 |
destroy_workqueue(vf->mbox_wq); |
| 388 |
return err; |
| 389 |
} |
| 390 |
|
| 391 |
static int otx2vf_open(struct net_device *netdev) |
| 392 |
{ |
| 393 |
struct otx2_nic *vf; |
| 394 |
int err; |
| 395 |
|
| 396 |
err = otx2_open(netdev); |
| 397 |
if (err) |
| 398 |
return err; |
| 399 |
|
| 400 |
/* LBKs do not receive link events so tell everyone we are up here */ |
| 401 |
vf = netdev_priv(netdev); |
| 402 |
if (is_otx2_lbkvf(vf->pdev) || is_otx2_sdp_rep(vf->pdev)) { |
| 403 |
pr_info("%s NIC Link is UP\n", netdev->name); |
| 404 |
netif_carrier_on(netdev); |
| 405 |
netif_tx_start_all_queues(netdev); |
| 406 |
} |
| 407 |
|
| 408 |
return 0; |
| 409 |
} |
| 410 |
|
| 411 |
static int otx2vf_stop(struct net_device *netdev) |
| 412 |
{ |
| 413 |
return otx2_stop(netdev); |
| 414 |
} |
| 415 |
|
| 416 |
static netdev_tx_t otx2vf_xmit(struct sk_buff *skb, struct net_device *netdev) |
| 417 |
{ |
| 418 |
struct otx2_nic *vf = netdev_priv(netdev); |
| 419 |
int qidx = skb_get_queue_mapping(skb); |
| 420 |
struct otx2_dev_stats *dev_stats; |
| 421 |
struct otx2_snd_queue *sq; |
| 422 |
struct netdev_queue *txq; |
| 423 |
|
| 424 |
/* Check for minimum and maximum packet length */ |
| 425 |
if (skb->len <= ETH_HLEN || |
| 426 |
(!skb_shinfo(skb)->gso_size && skb->len > vf->tx_max_pktlen)) { |
| 427 |
dev_stats = &vf->hw.dev_stats; |
| 428 |
atomic_long_inc(&dev_stats->tx_discards); |
| 429 |
dev_kfree_skb(skb); |
| 430 |
return NETDEV_TX_OK; |
| 431 |
} |
| 432 |
|
| 433 |
sq = &vf->qset.sq[qidx]; |
| 434 |
txq = netdev_get_tx_queue(netdev, qidx); |
| 435 |
|
| 436 |
if (!otx2_sq_append_skb(vf, txq, sq, skb, qidx)) { |
| 437 |
netif_tx_stop_queue(txq); |
| 438 |
|
| 439 |
/* Check again, incase SQBs got freed up */ |
| 440 |
smp_mb(); |
| 441 |
if (((sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb) |
| 442 |
> sq->sqe_thresh) |
| 443 |
netif_tx_wake_queue(txq); |
| 444 |
|
| 445 |
return NETDEV_TX_BUSY; |
| 446 |
} |
| 447 |
|
| 448 |
return NETDEV_TX_OK; |
| 449 |
} |
| 450 |
|
| 451 |
static void otx2vf_set_rx_mode(struct net_device *netdev) |
| 452 |
{ |
| 453 |
struct otx2_nic *vf = netdev_priv(netdev); |
| 454 |
|
| 455 |
queue_work(vf->otx2_wq, &vf->rx_mode_work); |
| 456 |
} |
| 457 |
|
| 458 |
static void otx2vf_do_set_rx_mode(struct work_struct *work) |
| 459 |
{ |
| 460 |
struct otx2_nic *vf = container_of(work, struct otx2_nic, rx_mode_work); |
| 461 |
struct net_device *netdev = vf->netdev; |
| 462 |
unsigned int flags = netdev->flags; |
| 463 |
struct nix_rx_mode *req; |
| 464 |
|
| 465 |
mutex_lock(&vf->mbox.lock); |
| 466 |
|
| 467 |
req = otx2_mbox_alloc_msg_nix_set_rx_mode(&vf->mbox); |
| 468 |
if (!req) { |
| 469 |
mutex_unlock(&vf->mbox.lock); |
| 470 |
return; |
| 471 |
} |
| 472 |
|
| 473 |
req->mode = NIX_RX_MODE_UCAST; |
| 474 |
|
| 475 |
if (flags & IFF_PROMISC) |
| 476 |
req->mode |= NIX_RX_MODE_PROMISC; |
| 477 |
if (flags & (IFF_ALLMULTI | IFF_MULTICAST)) |
| 478 |
req->mode |= NIX_RX_MODE_ALLMULTI; |
| 479 |
|
| 480 |
req->mode |= NIX_RX_MODE_USE_MCE; |
| 481 |
|
| 482 |
otx2_sync_mbox_msg(&vf->mbox); |
| 483 |
|
| 484 |
mutex_unlock(&vf->mbox.lock); |
| 485 |
} |
| 486 |
|
| 487 |
static int otx2vf_change_mtu(struct net_device *netdev, int new_mtu) |
| 488 |
{ |
| 489 |
bool if_up = netif_running(netdev); |
| 490 |
int err = 0; |
| 491 |
|
| 492 |
if (if_up) |
| 493 |
otx2vf_stop(netdev); |
| 494 |
|
| 495 |
netdev_info(netdev, "Changing MTU from %d to %d\n", |
| 496 |
netdev->mtu, new_mtu); |
| 497 |
WRITE_ONCE(netdev->mtu, new_mtu); |
| 498 |
|
| 499 |
if (if_up) |
| 500 |
err = otx2vf_open(netdev); |
| 501 |
|
| 502 |
return err; |
| 503 |
} |
| 504 |
|
| 505 |
static void otx2vf_reset_task(struct work_struct *work) |
| 506 |
{ |
| 507 |
struct otx2_nic *vf = container_of(work, struct otx2_nic, reset_task); |
| 508 |
|
| 509 |
rtnl_lock(); |
| 510 |
|
| 511 |
if (netif_running(vf->netdev)) { |
| 512 |
otx2vf_stop(vf->netdev); |
| 513 |
vf->reset_count++; |
| 514 |
otx2vf_open(vf->netdev); |
| 515 |
} |
| 516 |
|
| 517 |
rtnl_unlock(); |
| 518 |
} |
| 519 |
|
| 520 |
static int otx2vf_set_features(struct net_device *netdev, |
| 521 |
netdev_features_t features) |
| 522 |
{ |
| 523 |
return otx2_handle_ntuple_tc_features(netdev, features); |
| 524 |
} |
| 525 |
|
| 526 |
static const struct net_device_ops otx2vf_netdev_ops = { |
| 527 |
.ndo_open = otx2vf_open, |
| 528 |
.ndo_stop = otx2vf_stop, |
| 529 |
.ndo_start_xmit = otx2vf_xmit, |
| 530 |
.ndo_select_queue = otx2_select_queue, |
| 531 |
.ndo_set_rx_mode = otx2vf_set_rx_mode, |
| 532 |
.ndo_set_mac_address = otx2_set_mac_address, |
| 533 |
.ndo_change_mtu = otx2vf_change_mtu, |
| 534 |
.ndo_set_features = otx2vf_set_features, |
| 535 |
.ndo_get_stats64 = otx2_get_stats64, |
| 536 |
.ndo_tx_timeout = otx2_tx_timeout, |
| 537 |
.ndo_setup_tc = otx2_setup_tc, |
| 538 |
.ndo_hwtstamp_get = otx2_config_hwtstamp_get, |
| 539 |
.ndo_hwtstamp_set = otx2_config_hwtstamp_set, |
| 540 |
}; |
| 541 |
|
| 542 |
static int otx2_vf_wq_init(struct otx2_nic *vf) |
| 543 |
{ |
| 544 |
vf->otx2_wq = create_singlethread_workqueue("otx2vf_wq"); |
| 545 |
if (!vf->otx2_wq) |
| 546 |
return -ENOMEM; |
| 547 |
|
| 548 |
INIT_WORK(&vf->rx_mode_work, otx2vf_do_set_rx_mode); |
| 549 |
INIT_WORK(&vf->reset_task, otx2vf_reset_task); |
| 550 |
return 0; |
| 551 |
} |
| 552 |
|
| 553 |
static int otx2vf_realloc_msix_vectors(struct otx2_nic *vf) |
| 554 |
{ |
| 555 |
struct otx2_hw *hw = &vf->hw; |
| 556 |
int num_vec, err; |
| 557 |
|
| 558 |
num_vec = hw->nix_msixoff; |
| 559 |
num_vec += NIX_LF_CINT_VEC_START + hw->max_queues; |
| 560 |
|
| 561 |
otx2vf_disable_mbox_intr(vf); |
| 562 |
pci_free_irq_vectors(hw->pdev); |
| 563 |
err = pci_alloc_irq_vectors(hw->pdev, num_vec, num_vec, PCI_IRQ_MSIX); |
| 564 |
if (err < 0) { |
| 565 |
dev_err(vf->dev, "%s: Failed to realloc %d IRQ vectors\n", |
| 566 |
__func__, num_vec); |
| 567 |
return err; |
| 568 |
} |
| 569 |
|
| 570 |
return otx2vf_register_mbox_intr(vf, false); |
| 571 |
} |
| 572 |
|
| 573 |
static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
| 574 |
{ |
| 575 |
int num_vec = pci_msix_vec_count(pdev); |
| 576 |
struct device *dev = &pdev->dev; |
| 577 |
int err, qcount, qos_txqs; |
| 578 |
struct net_device *netdev; |
| 579 |
struct otx2_nic *vf; |
| 580 |
struct otx2_hw *hw; |
| 581 |
|
| 582 |
err = pcim_enable_device(pdev); |
| 583 |
if (err) { |
| 584 |
dev_err(dev, "Failed to enable PCI device\n"); |
| 585 |
return err; |
| 586 |
} |
| 587 |
|
| 588 |
err = pcim_request_all_regions(pdev, DRV_NAME); |
| 589 |
if (err) { |
| 590 |
dev_err(dev, "PCI request regions failed 0x%x\n", err); |
| 591 |
return err; |
| 592 |
} |
| 593 |
|
| 594 |
err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); |
| 595 |
if (err) { |
| 596 |
dev_err(dev, "DMA mask config failed, abort\n"); |
| 597 |
return err; |
| 598 |
} |
| 599 |
|
| 600 |
pci_set_master(pdev); |
| 601 |
|
| 602 |
qcount = num_online_cpus(); |
| 603 |
qos_txqs = min_t(int, qcount, OTX2_QOS_MAX_LEAF_NODES); |
| 604 |
netdev = alloc_etherdev_mqs(sizeof(*vf), qcount + qos_txqs, qcount); |
| 605 |
if (!netdev) |
| 606 |
return -ENOMEM; |
| 607 |
|
| 608 |
pci_set_drvdata(pdev, netdev); |
| 609 |
SET_NETDEV_DEV(netdev, &pdev->dev); |
| 610 |
vf = netdev_priv(netdev); |
| 611 |
vf->netdev = netdev; |
| 612 |
vf->pdev = pdev; |
| 613 |
vf->dev = dev; |
| 614 |
vf->iommu_domain = iommu_get_domain_for_dev(dev); |
| 615 |
|
| 616 |
vf->flags |= OTX2_FLAG_INTF_DOWN; |
| 617 |
hw = &vf->hw; |
| 618 |
hw->pdev = vf->pdev; |
| 619 |
hw->rx_queues = qcount; |
| 620 |
hw->tx_queues = qcount; |
| 621 |
hw->max_queues = qcount; |
| 622 |
hw->non_qos_queues = qcount; |
| 623 |
hw->rbuf_len = OTX2_DEFAULT_RBUF_LEN; |
| 624 |
/* Use CQE of 128 byte descriptor size by default */ |
| 625 |
hw->xqe_size = 128; |
| 626 |
|
| 627 |
hw->irq_name = devm_kmalloc_array(&hw->pdev->dev, num_vec, NAME_SIZE, |
| 628 |
GFP_KERNEL); |
| 629 |
if (!hw->irq_name) { |
| 630 |
err = -ENOMEM; |
| 631 |
goto err_free_netdev; |
| 632 |
} |
| 633 |
|
| 634 |
hw->affinity_mask = devm_kcalloc(&hw->pdev->dev, num_vec, |
| 635 |
sizeof(cpumask_var_t), GFP_KERNEL); |
| 636 |
if (!hw->affinity_mask) { |
| 637 |
err = -ENOMEM; |
| 638 |
goto err_free_netdev; |
| 639 |
} |
| 640 |
|
| 641 |
err = pci_alloc_irq_vectors(hw->pdev, num_vec, num_vec, PCI_IRQ_MSIX); |
| 642 |
if (err < 0) { |
| 643 |
dev_err(dev, "%s: Failed to alloc %d IRQ vectors\n", |
| 644 |
__func__, num_vec); |
| 645 |
goto err_free_netdev; |
| 646 |
} |
| 647 |
|
| 648 |
vf->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); |
| 649 |
if (!vf->reg_base) { |
| 650 |
dev_err(dev, "Unable to map physical function CSRs, aborting\n"); |
| 651 |
err = -ENOMEM; |
| 652 |
goto err_free_irq_vectors; |
| 653 |
} |
| 654 |
|
| 655 |
otx2_setup_dev_hw_settings(vf); |
| 656 |
|
| 657 |
if (is_cn20k(vf->pdev)) |
| 658 |
cn20k_init(vf); |
| 659 |
else |
| 660 |
otx2_init_hw_ops(vf); |
| 661 |
|
| 662 |
/* Init VF <=> PF mailbox stuff */ |
| 663 |
err = otx2vf_vfaf_mbox_init(vf); |
| 664 |
if (err) |
| 665 |
goto err_free_irq_vectors; |
| 666 |
|
| 667 |
/* Register mailbox interrupt */ |
| 668 |
err = otx2vf_register_mbox_intr(vf, true); |
| 669 |
if (err) |
| 670 |
goto err_mbox_destroy; |
| 671 |
|
| 672 |
/* Request AF to attach NPA and LIX LFs to this AF */ |
| 673 |
err = otx2_attach_npa_nix(vf); |
| 674 |
if (err) |
| 675 |
goto err_disable_mbox_intr; |
| 676 |
|
| 677 |
err = otx2vf_realloc_msix_vectors(vf); |
| 678 |
if (err) |
| 679 |
goto err_detach_rsrc; |
| 680 |
|
| 681 |
err = otx2_set_real_num_queues(netdev, qcount, qcount); |
| 682 |
if (err) |
| 683 |
goto err_detach_rsrc; |
| 684 |
|
| 685 |
err = cn10k_lmtst_init(vf); |
| 686 |
if (err) |
| 687 |
goto err_detach_rsrc; |
| 688 |
|
| 689 |
/* Don't check for error. Proceed without ptp */ |
| 690 |
otx2_ptp_init(vf); |
| 691 |
|
| 692 |
/* Assign default mac address */ |
| 693 |
otx2_get_mac_from_af(netdev); |
| 694 |
|
| 695 |
netdev->hw_features = NETIF_F_RXCSUM | NETIF_F_IP_CSUM | |
| 696 |
NETIF_F_IPV6_CSUM | NETIF_F_RXHASH | |
| 697 |
NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | |
| 698 |
NETIF_F_GSO_UDP_L4; |
| 699 |
netdev->features = netdev->hw_features; |
| 700 |
/* Support TSO on tag interface */ |
| 701 |
netdev->vlan_features |= netdev->features; |
| 702 |
netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX | |
| 703 |
NETIF_F_HW_VLAN_STAG_TX; |
| 704 |
netdev->features |= netdev->hw_features; |
| 705 |
|
| 706 |
netdev->hw_features |= NETIF_F_NTUPLE; |
| 707 |
netdev->hw_features |= NETIF_F_RXALL; |
| 708 |
netdev->hw_features |= NETIF_F_HW_TC; |
| 709 |
|
| 710 |
netif_set_tso_max_segs(netdev, OTX2_MAX_GSO_SEGS); |
| 711 |
netdev->watchdog_timeo = OTX2_TX_TIMEOUT; |
| 712 |
|
| 713 |
netdev->netdev_ops = &otx2vf_netdev_ops; |
| 714 |
|
| 715 |
netdev->min_mtu = OTX2_MIN_MTU; |
| 716 |
netdev->max_mtu = otx2_get_max_mtu(vf); |
| 717 |
hw->max_mtu = netdev->max_mtu; |
| 718 |
|
| 719 |
/* To distinguish, for LBK VFs set netdev name explicitly */ |
| 720 |
if (is_otx2_lbkvf(vf->pdev)) { |
| 721 |
int n; |
| 722 |
|
| 723 |
n = (vf->pcifunc >> RVU_PFVF_FUNC_SHIFT) & RVU_PFVF_FUNC_MASK; |
| 724 |
/* Need to subtract 1 to get proper VF number */ |
| 725 |
n -= 1; |
| 726 |
snprintf(netdev->name, sizeof(netdev->name), "lbk%d", n); |
| 727 |
} |
| 728 |
|
| 729 |
if (is_otx2_sdp_rep(vf->pdev)) { |
| 730 |
int n; |
| 731 |
|
| 732 |
n = vf->pcifunc & RVU_PFVF_FUNC_MASK; |
| 733 |
n -= 1; |
| 734 |
snprintf(netdev->name, sizeof(netdev->name), "sdp%d-%d", |
| 735 |
pdev->bus->number, n); |
| 736 |
} |
| 737 |
|
| 738 |
err = cn10k_ipsec_init(netdev); |
| 739 |
if (err) |
| 740 |
goto err_ptp_destroy; |
| 741 |
|
| 742 |
err = register_netdev(netdev); |
| 743 |
if (err) { |
| 744 |
dev_err(dev, "Failed to register netdevice\n"); |
| 745 |
goto err_ipsec_clean; |
| 746 |
} |
| 747 |
|
| 748 |
err = otx2_vf_wq_init(vf); |
| 749 |
if (err) |
| 750 |
goto err_unreg_netdev; |
| 751 |
|
| 752 |
otx2vf_set_ethtool_ops(netdev); |
| 753 |
|
| 754 |
err = otx2vf_mcam_flow_init(vf); |
| 755 |
if (err) |
| 756 |
goto err_unreg_netdev; |
| 757 |
|
| 758 |
err = otx2_init_tc(vf); |
| 759 |
if (err) |
| 760 |
goto err_unreg_netdev; |
| 761 |
|
| 762 |
err = otx2_register_dl(vf); |
| 763 |
if (err) |
| 764 |
goto err_shutdown_tc; |
| 765 |
|
| 766 |
vf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL); |
| 767 |
if (!vf->af_xdp_zc_qidx) { |
| 768 |
err = -ENOMEM; |
| 769 |
goto err_unreg_devlink; |
| 770 |
} |
| 771 |
|
| 772 |
#ifdef CONFIG_DCB |
| 773 |
/* Priority flow control is not supported for LBK and SDP vf(s) */ |
| 774 |
if (!(is_otx2_lbkvf(vf->pdev) || is_otx2_sdp_rep(vf->pdev))) { |
| 775 |
err = otx2_dcbnl_set_ops(netdev); |
| 776 |
if (err) |
| 777 |
goto err_free_zc_bmap; |
| 778 |
} |
| 779 |
#endif |
| 780 |
otx2_qos_init(vf, qos_txqs); |
| 781 |
|
| 782 |
return 0; |
| 783 |
|
| 784 |
#ifdef CONFIG_DCB |
| 785 |
err_free_zc_bmap: |
| 786 |
bitmap_free(vf->af_xdp_zc_qidx); |
| 787 |
#endif |
| 788 |
err_unreg_devlink: |
| 789 |
otx2_unregister_dl(vf); |
| 790 |
err_shutdown_tc: |
| 791 |
otx2_shutdown_tc(vf); |
| 792 |
err_unreg_netdev: |
| 793 |
unregister_netdev(netdev); |
| 794 |
err_ipsec_clean: |
| 795 |
cn10k_ipsec_clean(vf); |
| 796 |
err_ptp_destroy: |
| 797 |
otx2_ptp_destroy(vf); |
| 798 |
err_detach_rsrc: |
| 799 |
free_percpu(vf->hw.lmt_info); |
| 800 |
if (test_bit(CN10K_LMTST, &vf->hw.cap_flag)) |
| 801 |
qmem_free(vf->dev, vf->dync_lmt); |
| 802 |
otx2_detach_resources(&vf->mbox); |
| 803 |
err_disable_mbox_intr: |
| 804 |
otx2vf_disable_mbox_intr(vf); |
| 805 |
err_mbox_destroy: |
| 806 |
otx2vf_vfaf_mbox_destroy(vf); |
| 807 |
err_free_irq_vectors: |
| 808 |
pci_free_irq_vectors(hw->pdev); |
| 809 |
err_free_netdev: |
| 810 |
pci_set_drvdata(pdev, NULL); |
| 811 |
free_netdev(netdev); |
| 812 |
return err; |
| 813 |
} |
| 814 |
|
| 815 |
static void otx2vf_remove(struct pci_dev *pdev) |
| 816 |
{ |
| 817 |
struct net_device *netdev = pci_get_drvdata(pdev); |
| 818 |
struct otx2_nic *vf; |
| 819 |
|
| 820 |
if (!netdev) |
| 821 |
return; |
| 822 |
|
| 823 |
vf = netdev_priv(netdev); |
| 824 |
|
| 825 |
/* Disable 802.3x pause frames */ |
| 826 |
if (vf->flags & OTX2_FLAG_RX_PAUSE_ENABLED || |
| 827 |
(vf->flags & OTX2_FLAG_TX_PAUSE_ENABLED)) { |
| 828 |
vf->flags &= ~OTX2_FLAG_RX_PAUSE_ENABLED; |
| 829 |
vf->flags &= ~OTX2_FLAG_TX_PAUSE_ENABLED; |
| 830 |
otx2_config_pause_frm(vf); |
| 831 |
} |
| 832 |
|
| 833 |
#ifdef CONFIG_DCB |
| 834 |
/* Disable PFC config */ |
| 835 |
if (vf->pfc_en) { |
| 836 |
vf->pfc_en = 0; |
| 837 |
otx2_config_priority_flow_ctrl(vf); |
| 838 |
} |
| 839 |
#endif |
| 840 |
|
| 841 |
cancel_work_sync(&vf->reset_task); |
| 842 |
otx2_unregister_dl(vf); |
| 843 |
unregister_netdev(netdev); |
| 844 |
if (vf->otx2_wq) |
| 845 |
destroy_workqueue(vf->otx2_wq); |
| 846 |
cn10k_ipsec_clean(vf); |
| 847 |
otx2_ptp_destroy(vf); |
| 848 |
otx2_mcam_flow_del(vf); |
| 849 |
otx2_shutdown_tc(vf); |
| 850 |
otx2_shutdown_qos(vf); |
| 851 |
otx2_detach_resources(&vf->mbox); |
| 852 |
otx2vf_disable_mbox_intr(vf); |
| 853 |
free_percpu(vf->hw.lmt_info); |
| 854 |
if (test_bit(CN10K_LMTST, &vf->hw.cap_flag)) |
| 855 |
qmem_free(vf->dev, vf->dync_lmt); |
| 856 |
otx2vf_vfaf_mbox_destroy(vf); |
| 857 |
pci_free_irq_vectors(vf->pdev); |
| 858 |
bitmap_free(vf->af_xdp_zc_qidx); |
| 859 |
pci_set_drvdata(pdev, NULL); |
| 860 |
free_netdev(netdev); |
| 861 |
} |
| 862 |
|
| 863 |
static struct pci_driver otx2vf_driver = { |
| 864 |
.name = DRV_NAME, |
| 865 |
.id_table = otx2_vf_id_table, |
| 866 |
.probe = otx2vf_probe, |
| 867 |
.remove = otx2vf_remove, |
| 868 |
.shutdown = otx2vf_remove, |
| 869 |
}; |
| 870 |
|
| 871 |
static int __init otx2vf_init_module(void) |
| 872 |
{ |
| 873 |
pr_info("%s: %s\n", DRV_NAME, DRV_STRING); |
| 874 |
|
| 875 |
return pci_register_driver(&otx2vf_driver); |
| 876 |
} |
| 877 |
|
| 878 |
static void __exit otx2vf_cleanup_module(void) |
| 879 |
{ |
| 880 |
pci_unregister_driver(&otx2vf_driver); |
| 881 |
} |
| 882 |
|
| 883 |
module_init(otx2vf_init_module); |
| 884 |
module_exit(otx2vf_cleanup_module); |