| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
/* Marvell RVU Ethernet driver |
| 3 |
* |
| 4 |
* Copyright (C) 2020 Marvell. |
| 5 |
* |
| 6 |
*/ |
| 7 |
|
| 8 |
#include <linux/interrupt.h> |
| 9 |
#include <linux/pci.h> |
| 10 |
#include <net/page_pool/helpers.h> |
| 11 |
#include <net/tso.h> |
| 12 |
#include <linux/bitfield.h> |
| 13 |
#include <linux/dcbnl.h> |
| 14 |
#include <net/xfrm.h> |
| 15 |
|
| 16 |
#include "otx2_reg.h" |
| 17 |
#include "otx2_common.h" |
| 18 |
#include "otx2_struct.h" |
| 19 |
#include "cn10k.h" |
| 20 |
#include "otx2_xsk.h" |
| 21 |
|
| 22 |
static bool otx2_is_pfc_enabled(struct otx2_nic *pfvf) |
| 23 |
{ |
| 24 |
return IS_ENABLED(CONFIG_DCB) && !!pfvf->pfc_en; |
| 25 |
} |
| 26 |
|
| 27 |
static void otx2_nix_rq_op_stats(struct queue_stats *stats, |
| 28 |
struct otx2_nic *pfvf, int qidx) |
| 29 |
{ |
| 30 |
u64 incr = (u64)qidx << 32; |
| 31 |
void __iomem *ptr; |
| 32 |
|
| 33 |
ptr = otx2_get_regaddr(pfvf, NIX_LF_RQ_OP_OCTS); |
| 34 |
stats->bytes = otx2_atomic64_add(incr, ptr); |
| 35 |
|
| 36 |
ptr = otx2_get_regaddr(pfvf, NIX_LF_RQ_OP_PKTS); |
| 37 |
stats->pkts = otx2_atomic64_add(incr, ptr); |
| 38 |
} |
| 39 |
|
| 40 |
static void otx2_nix_sq_op_stats(struct queue_stats *stats, |
| 41 |
struct otx2_nic *pfvf, int qidx) |
| 42 |
{ |
| 43 |
u64 incr = (u64)qidx << 32; |
| 44 |
void __iomem *ptr; |
| 45 |
|
| 46 |
ptr = otx2_get_regaddr(pfvf, NIX_LF_SQ_OP_OCTS); |
| 47 |
stats->bytes = otx2_atomic64_add(incr, ptr); |
| 48 |
|
| 49 |
ptr = otx2_get_regaddr(pfvf, NIX_LF_SQ_OP_PKTS); |
| 50 |
stats->pkts = otx2_atomic64_add(incr, ptr); |
| 51 |
} |
| 52 |
|
| 53 |
void otx2_update_lmac_stats(struct otx2_nic *pfvf) |
| 54 |
{ |
| 55 |
struct msg_req *req; |
| 56 |
|
| 57 |
if (!netif_running(pfvf->netdev)) |
| 58 |
return; |
| 59 |
|
| 60 |
mutex_lock(&pfvf->mbox.lock); |
| 61 |
req = otx2_mbox_alloc_msg_cgx_stats(&pfvf->mbox); |
| 62 |
if (!req) { |
| 63 |
mutex_unlock(&pfvf->mbox.lock); |
| 64 |
return; |
| 65 |
} |
| 66 |
|
| 67 |
otx2_sync_mbox_msg(&pfvf->mbox); |
| 68 |
mutex_unlock(&pfvf->mbox.lock); |
| 69 |
} |
| 70 |
|
| 71 |
void otx2_update_lmac_fec_stats(struct otx2_nic *pfvf) |
| 72 |
{ |
| 73 |
struct msg_req *req; |
| 74 |
|
| 75 |
if (!netif_running(pfvf->netdev)) |
| 76 |
return; |
| 77 |
mutex_lock(&pfvf->mbox.lock); |
| 78 |
req = otx2_mbox_alloc_msg_cgx_fec_stats(&pfvf->mbox); |
| 79 |
if (req) |
| 80 |
otx2_sync_mbox_msg(&pfvf->mbox); |
| 81 |
mutex_unlock(&pfvf->mbox.lock); |
| 82 |
} |
| 83 |
|
| 84 |
int otx2_update_rq_stats(struct otx2_nic *pfvf, int qidx) |
| 85 |
{ |
| 86 |
struct otx2_rcv_queue *rq = &pfvf->qset.rq[qidx]; |
| 87 |
|
| 88 |
if (!pfvf->qset.rq) |
| 89 |
return 0; |
| 90 |
|
| 91 |
otx2_nix_rq_op_stats(&rq->stats, pfvf, qidx); |
| 92 |
return 1; |
| 93 |
} |
| 94 |
EXPORT_SYMBOL(otx2_update_rq_stats); |
| 95 |
|
| 96 |
int otx2_update_sq_stats(struct otx2_nic *pfvf, int qidx) |
| 97 |
{ |
| 98 |
struct otx2_snd_queue *sq = &pfvf->qset.sq[qidx]; |
| 99 |
|
| 100 |
if (!pfvf->qset.sq) |
| 101 |
return 0; |
| 102 |
|
| 103 |
if (qidx >= pfvf->hw.non_qos_queues) { |
| 104 |
if (!test_bit(qidx - pfvf->hw.non_qos_queues, pfvf->qos.qos_sq_bmap)) |
| 105 |
return 0; |
| 106 |
} |
| 107 |
|
| 108 |
otx2_nix_sq_op_stats(&sq->stats, pfvf, qidx); |
| 109 |
return 1; |
| 110 |
} |
| 111 |
EXPORT_SYMBOL(otx2_update_sq_stats); |
| 112 |
|
| 113 |
void otx2_get_dev_stats(struct otx2_nic *pfvf) |
| 114 |
{ |
| 115 |
struct otx2_dev_stats *dev_stats = &pfvf->hw.dev_stats; |
| 116 |
|
| 117 |
dev_stats->rx_bytes = OTX2_GET_RX_STATS(RX_OCTS); |
| 118 |
dev_stats->rx_drops = OTX2_GET_RX_STATS(RX_DROP); |
| 119 |
dev_stats->rx_bcast_frames = OTX2_GET_RX_STATS(RX_BCAST); |
| 120 |
dev_stats->rx_mcast_frames = OTX2_GET_RX_STATS(RX_MCAST); |
| 121 |
dev_stats->rx_ucast_frames = OTX2_GET_RX_STATS(RX_UCAST); |
| 122 |
dev_stats->rx_frames = dev_stats->rx_bcast_frames + |
| 123 |
dev_stats->rx_mcast_frames + |
| 124 |
dev_stats->rx_ucast_frames; |
| 125 |
|
| 126 |
dev_stats->tx_bytes = OTX2_GET_TX_STATS(TX_OCTS); |
| 127 |
dev_stats->tx_drops = OTX2_GET_TX_STATS(TX_DROP) + |
| 128 |
(unsigned long)atomic_long_read(&dev_stats->tx_discards); |
| 129 |
|
| 130 |
dev_stats->tx_bcast_frames = OTX2_GET_TX_STATS(TX_BCAST); |
| 131 |
dev_stats->tx_mcast_frames = OTX2_GET_TX_STATS(TX_MCAST); |
| 132 |
dev_stats->tx_ucast_frames = OTX2_GET_TX_STATS(TX_UCAST); |
| 133 |
dev_stats->tx_frames = dev_stats->tx_bcast_frames + |
| 134 |
dev_stats->tx_mcast_frames + |
| 135 |
dev_stats->tx_ucast_frames; |
| 136 |
} |
| 137 |
|
| 138 |
void otx2_get_stats64(struct net_device *netdev, |
| 139 |
struct rtnl_link_stats64 *stats) |
| 140 |
{ |
| 141 |
struct otx2_nic *pfvf = netdev_priv(netdev); |
| 142 |
struct otx2_dev_stats *dev_stats; |
| 143 |
|
| 144 |
otx2_get_dev_stats(pfvf); |
| 145 |
|
| 146 |
dev_stats = &pfvf->hw.dev_stats; |
| 147 |
stats->rx_bytes = dev_stats->rx_bytes; |
| 148 |
stats->rx_packets = dev_stats->rx_frames; |
| 149 |
stats->rx_dropped = dev_stats->rx_drops; |
| 150 |
stats->multicast = dev_stats->rx_mcast_frames; |
| 151 |
|
| 152 |
stats->tx_bytes = dev_stats->tx_bytes; |
| 153 |
stats->tx_packets = dev_stats->tx_frames; |
| 154 |
stats->tx_dropped = dev_stats->tx_drops; |
| 155 |
} |
| 156 |
EXPORT_SYMBOL(otx2_get_stats64); |
| 157 |
|
| 158 |
/* Sync MAC address with RVU AF */ |
| 159 |
static int otx2_hw_set_mac_addr(struct otx2_nic *pfvf, u8 *mac) |
| 160 |
{ |
| 161 |
struct nix_set_mac_addr *req; |
| 162 |
int err; |
| 163 |
|
| 164 |
mutex_lock(&pfvf->mbox.lock); |
| 165 |
req = otx2_mbox_alloc_msg_nix_set_mac_addr(&pfvf->mbox); |
| 166 |
if (!req) { |
| 167 |
mutex_unlock(&pfvf->mbox.lock); |
| 168 |
return -ENOMEM; |
| 169 |
} |
| 170 |
|
| 171 |
ether_addr_copy(req->mac_addr, mac); |
| 172 |
|
| 173 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 174 |
mutex_unlock(&pfvf->mbox.lock); |
| 175 |
return err; |
| 176 |
} |
| 177 |
|
| 178 |
static int otx2_hw_get_mac_addr(struct otx2_nic *pfvf, |
| 179 |
struct net_device *netdev) |
| 180 |
{ |
| 181 |
struct nix_get_mac_addr_rsp *rsp; |
| 182 |
struct mbox_msghdr *msghdr; |
| 183 |
struct msg_req *req; |
| 184 |
int err; |
| 185 |
|
| 186 |
mutex_lock(&pfvf->mbox.lock); |
| 187 |
req = otx2_mbox_alloc_msg_nix_get_mac_addr(&pfvf->mbox); |
| 188 |
if (!req) { |
| 189 |
mutex_unlock(&pfvf->mbox.lock); |
| 190 |
return -ENOMEM; |
| 191 |
} |
| 192 |
|
| 193 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 194 |
if (err) { |
| 195 |
mutex_unlock(&pfvf->mbox.lock); |
| 196 |
return err; |
| 197 |
} |
| 198 |
|
| 199 |
msghdr = otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); |
| 200 |
if (IS_ERR(msghdr)) { |
| 201 |
mutex_unlock(&pfvf->mbox.lock); |
| 202 |
return PTR_ERR(msghdr); |
| 203 |
} |
| 204 |
rsp = (struct nix_get_mac_addr_rsp *)msghdr; |
| 205 |
eth_hw_addr_set(netdev, rsp->mac_addr); |
| 206 |
mutex_unlock(&pfvf->mbox.lock); |
| 207 |
|
| 208 |
return 0; |
| 209 |
} |
| 210 |
|
| 211 |
int otx2_set_mac_address(struct net_device *netdev, void *p) |
| 212 |
{ |
| 213 |
struct otx2_nic *pfvf = netdev_priv(netdev); |
| 214 |
struct sockaddr *addr = p; |
| 215 |
|
| 216 |
if (!is_valid_ether_addr(addr->sa_data)) |
| 217 |
return -EADDRNOTAVAIL; |
| 218 |
|
| 219 |
if (!otx2_hw_set_mac_addr(pfvf, addr->sa_data)) { |
| 220 |
eth_hw_addr_set(netdev, addr->sa_data); |
| 221 |
/* update dmac field in vlan offload rule */ |
| 222 |
if (netif_running(netdev) && |
| 223 |
pfvf->flags & OTX2_FLAG_RX_VLAN_SUPPORT) |
| 224 |
otx2_install_rxvlan_offload_flow(pfvf); |
| 225 |
/* update dmac address in ntuple and DMAC filter list */ |
| 226 |
if (pfvf->flags & OTX2_FLAG_DMACFLTR_SUPPORT) |
| 227 |
otx2_dmacflt_update_pfmac_flow(pfvf); |
| 228 |
} else { |
| 229 |
return -EPERM; |
| 230 |
} |
| 231 |
|
| 232 |
return 0; |
| 233 |
} |
| 234 |
EXPORT_SYMBOL(otx2_set_mac_address); |
| 235 |
|
| 236 |
int otx2_hw_set_mtu(struct otx2_nic *pfvf, int mtu) |
| 237 |
{ |
| 238 |
struct nix_frs_cfg *req; |
| 239 |
u16 maxlen; |
| 240 |
int err; |
| 241 |
|
| 242 |
maxlen = pfvf->hw.max_mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN; |
| 243 |
|
| 244 |
mutex_lock(&pfvf->mbox.lock); |
| 245 |
req = otx2_mbox_alloc_msg_nix_set_hw_frs(&pfvf->mbox); |
| 246 |
if (!req) { |
| 247 |
mutex_unlock(&pfvf->mbox.lock); |
| 248 |
return -ENOMEM; |
| 249 |
} |
| 250 |
|
| 251 |
req->maxlen = mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN; |
| 252 |
|
| 253 |
/* Use max receive length supported by hardware for loopback devices */ |
| 254 |
if (is_otx2_lbkvf(pfvf->pdev)) |
| 255 |
req->maxlen = maxlen; |
| 256 |
|
| 257 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 258 |
mutex_unlock(&pfvf->mbox.lock); |
| 259 |
return err; |
| 260 |
} |
| 261 |
EXPORT_SYMBOL(otx2_hw_set_mtu); |
| 262 |
|
| 263 |
int otx2_config_pause_frm(struct otx2_nic *pfvf) |
| 264 |
{ |
| 265 |
struct cgx_pause_frm_cfg *req; |
| 266 |
int err; |
| 267 |
|
| 268 |
if (is_otx2_lbkvf(pfvf->pdev) || is_otx2_sdp_rep(pfvf->pdev)) |
| 269 |
return 0; |
| 270 |
|
| 271 |
mutex_lock(&pfvf->mbox.lock); |
| 272 |
req = otx2_mbox_alloc_msg_cgx_cfg_pause_frm(&pfvf->mbox); |
| 273 |
if (!req) { |
| 274 |
err = -ENOMEM; |
| 275 |
goto unlock; |
| 276 |
} |
| 277 |
|
| 278 |
req->rx_pause = !!(pfvf->flags & OTX2_FLAG_RX_PAUSE_ENABLED); |
| 279 |
req->tx_pause = !!(pfvf->flags & OTX2_FLAG_TX_PAUSE_ENABLED); |
| 280 |
req->set = 1; |
| 281 |
|
| 282 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 283 |
unlock: |
| 284 |
mutex_unlock(&pfvf->mbox.lock); |
| 285 |
return err; |
| 286 |
} |
| 287 |
EXPORT_SYMBOL(otx2_config_pause_frm); |
| 288 |
|
| 289 |
int otx2_set_flowkey_cfg(struct otx2_nic *pfvf) |
| 290 |
{ |
| 291 |
struct otx2_rss_info *rss = &pfvf->hw.rss_info; |
| 292 |
struct nix_rss_flowkey_cfg_rsp *rsp; |
| 293 |
struct nix_rss_flowkey_cfg *req; |
| 294 |
int err; |
| 295 |
|
| 296 |
mutex_lock(&pfvf->mbox.lock); |
| 297 |
req = otx2_mbox_alloc_msg_nix_rss_flowkey_cfg(&pfvf->mbox); |
| 298 |
if (!req) { |
| 299 |
mutex_unlock(&pfvf->mbox.lock); |
| 300 |
return -ENOMEM; |
| 301 |
} |
| 302 |
req->mcam_index = -1; /* Default or reserved index */ |
| 303 |
req->flowkey_cfg = rss->flowkey_cfg; |
| 304 |
req->group = DEFAULT_RSS_CONTEXT_GROUP; |
| 305 |
|
| 306 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 307 |
if (err) |
| 308 |
goto fail; |
| 309 |
|
| 310 |
rsp = (struct nix_rss_flowkey_cfg_rsp *) |
| 311 |
otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); |
| 312 |
if (IS_ERR(rsp)) { |
| 313 |
err = PTR_ERR(rsp); |
| 314 |
goto fail; |
| 315 |
} |
| 316 |
|
| 317 |
pfvf->hw.flowkey_alg_idx = rsp->alg_idx; |
| 318 |
fail: |
| 319 |
mutex_unlock(&pfvf->mbox.lock); |
| 320 |
return err; |
| 321 |
} |
| 322 |
|
| 323 |
int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id, const u32 *ind_tbl) |
| 324 |
{ |
| 325 |
struct otx2_rss_info *rss = &pfvf->hw.rss_info; |
| 326 |
const int index = rss->rss_size * ctx_id; |
| 327 |
struct mbox *mbox = &pfvf->mbox; |
| 328 |
struct nix_aq_enq_req *aq; |
| 329 |
int idx, err; |
| 330 |
|
| 331 |
mutex_lock(&mbox->lock); |
| 332 |
ind_tbl = ind_tbl ?: rss->ind_tbl; |
| 333 |
/* Get memory to put this msg */ |
| 334 |
for (idx = 0; idx < rss->rss_size; idx++) { |
| 335 |
/* Ignore the queue if AF_XDP zero copy is enabled */ |
| 336 |
if (test_bit(ind_tbl[idx], pfvf->af_xdp_zc_qidx)) |
| 337 |
continue; |
| 338 |
|
| 339 |
aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); |
| 340 |
if (!aq) { |
| 341 |
/* The shared memory buffer can be full. |
| 342 |
* Flush it and retry |
| 343 |
*/ |
| 344 |
err = otx2_sync_mbox_msg(mbox); |
| 345 |
if (err) { |
| 346 |
mutex_unlock(&mbox->lock); |
| 347 |
return err; |
| 348 |
} |
| 349 |
aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); |
| 350 |
if (!aq) { |
| 351 |
mutex_unlock(&mbox->lock); |
| 352 |
return -ENOMEM; |
| 353 |
} |
| 354 |
} |
| 355 |
|
| 356 |
aq->rss.rq = ind_tbl[idx]; |
| 357 |
|
| 358 |
/* Fill AQ info */ |
| 359 |
aq->qidx = index + idx; |
| 360 |
aq->ctype = NIX_AQ_CTYPE_RSS; |
| 361 |
aq->op = NIX_AQ_INSTOP_INIT; |
| 362 |
} |
| 363 |
err = otx2_sync_mbox_msg(mbox); |
| 364 |
mutex_unlock(&mbox->lock); |
| 365 |
return err; |
| 366 |
} |
| 367 |
|
| 368 |
void otx2_set_rss_key(struct otx2_nic *pfvf) |
| 369 |
{ |
| 370 |
struct otx2_rss_info *rss = &pfvf->hw.rss_info; |
| 371 |
u64 *key = (u64 *)&rss->key[4]; |
| 372 |
int idx; |
| 373 |
|
| 374 |
/* 352bit or 44byte key needs to be configured as below |
| 375 |
* NIX_LF_RX_SECRETX0 = key<351:288> |
| 376 |
* NIX_LF_RX_SECRETX1 = key<287:224> |
| 377 |
* NIX_LF_RX_SECRETX2 = key<223:160> |
| 378 |
* NIX_LF_RX_SECRETX3 = key<159:96> |
| 379 |
* NIX_LF_RX_SECRETX4 = key<95:32> |
| 380 |
* NIX_LF_RX_SECRETX5<63:32> = key<31:0> |
| 381 |
*/ |
| 382 |
otx2_write64(pfvf, NIX_LF_RX_SECRETX(5), |
| 383 |
(u64)(*((u32 *)&rss->key)) << 32); |
| 384 |
idx = sizeof(rss->key) / sizeof(u64); |
| 385 |
while (idx > 0) { |
| 386 |
idx--; |
| 387 |
otx2_write64(pfvf, NIX_LF_RX_SECRETX(idx), *key++); |
| 388 |
} |
| 389 |
} |
| 390 |
|
| 391 |
int otx2_rss_init(struct otx2_nic *pfvf) |
| 392 |
{ |
| 393 |
struct otx2_rss_info *rss = &pfvf->hw.rss_info; |
| 394 |
int idx, ret = 0; |
| 395 |
|
| 396 |
rss->rss_size = sizeof(*rss->ind_tbl); |
| 397 |
|
| 398 |
/* Init RSS key if it is not setup already */ |
| 399 |
if (!rss->enable) |
| 400 |
netdev_rss_key_fill(rss->key, sizeof(rss->key)); |
| 401 |
otx2_set_rss_key(pfvf); |
| 402 |
|
| 403 |
if (!netif_is_rxfh_configured(pfvf->netdev)) |
| 404 |
for (idx = 0; idx < rss->rss_size; idx++) |
| 405 |
rss->ind_tbl[idx] = |
| 406 |
ethtool_rxfh_indir_default(idx, |
| 407 |
pfvf->hw.rx_queues); |
| 408 |
|
| 409 |
ret = otx2_set_rss_table(pfvf, DEFAULT_RSS_CONTEXT_GROUP, NULL); |
| 410 |
if (ret) |
| 411 |
return ret; |
| 412 |
|
| 413 |
/* Flowkey or hash config to be used for generating flow tag */ |
| 414 |
rss->flowkey_cfg = rss->enable ? rss->flowkey_cfg : |
| 415 |
NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6 | |
| 416 |
NIX_FLOW_KEY_TYPE_TCP | NIX_FLOW_KEY_TYPE_UDP | |
| 417 |
NIX_FLOW_KEY_TYPE_SCTP | NIX_FLOW_KEY_TYPE_VLAN | |
| 418 |
NIX_FLOW_KEY_TYPE_IPV4_PROTO; |
| 419 |
|
| 420 |
ret = otx2_set_flowkey_cfg(pfvf); |
| 421 |
if (ret) |
| 422 |
return ret; |
| 423 |
|
| 424 |
rss->enable = true; |
| 425 |
return 0; |
| 426 |
} |
| 427 |
|
| 428 |
/* Setup UDP segmentation algorithm in HW */ |
| 429 |
static void otx2_setup_udp_segmentation(struct nix_lso_format_cfg *lso, bool v4) |
| 430 |
{ |
| 431 |
struct nix_lso_format *field; |
| 432 |
|
| 433 |
field = (struct nix_lso_format *)&lso->fields[0]; |
| 434 |
lso->field_mask = GENMASK(18, 0); |
| 435 |
|
| 436 |
/* IP's Length field */ |
| 437 |
field->layer = NIX_TXLAYER_OL3; |
| 438 |
/* In ipv4, length field is at offset 2 bytes, for ipv6 it's 4 */ |
| 439 |
field->offset = v4 ? 2 : 4; |
| 440 |
field->sizem1 = 1; /* i.e 2 bytes */ |
| 441 |
field->alg = NIX_LSOALG_ADD_PAYLEN; |
| 442 |
field++; |
| 443 |
|
| 444 |
/* No ID field in IPv6 header */ |
| 445 |
if (v4) { |
| 446 |
/* Increment IPID */ |
| 447 |
field->layer = NIX_TXLAYER_OL3; |
| 448 |
field->offset = 4; |
| 449 |
field->sizem1 = 1; /* i.e 2 bytes */ |
| 450 |
field->alg = NIX_LSOALG_ADD_SEGNUM; |
| 451 |
field++; |
| 452 |
} |
| 453 |
|
| 454 |
/* Update length in UDP header */ |
| 455 |
field->layer = NIX_TXLAYER_OL4; |
| 456 |
field->offset = 4; |
| 457 |
field->sizem1 = 1; |
| 458 |
field->alg = NIX_LSOALG_ADD_PAYLEN; |
| 459 |
} |
| 460 |
|
| 461 |
/* Setup segmentation algorithms in HW and retrieve algorithm index */ |
| 462 |
void otx2_setup_segmentation(struct otx2_nic *pfvf) |
| 463 |
{ |
| 464 |
struct nix_lso_format_cfg_rsp *rsp; |
| 465 |
struct nix_lso_format_cfg *lso; |
| 466 |
struct otx2_hw *hw = &pfvf->hw; |
| 467 |
int err; |
| 468 |
|
| 469 |
mutex_lock(&pfvf->mbox.lock); |
| 470 |
|
| 471 |
/* UDPv4 segmentation */ |
| 472 |
lso = otx2_mbox_alloc_msg_nix_lso_format_cfg(&pfvf->mbox); |
| 473 |
if (!lso) |
| 474 |
goto fail; |
| 475 |
|
| 476 |
/* Setup UDP/IP header fields that HW should update per segment */ |
| 477 |
otx2_setup_udp_segmentation(lso, true); |
| 478 |
|
| 479 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 480 |
if (err) |
| 481 |
goto fail; |
| 482 |
|
| 483 |
rsp = (struct nix_lso_format_cfg_rsp *) |
| 484 |
otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &lso->hdr); |
| 485 |
if (IS_ERR(rsp)) |
| 486 |
goto fail; |
| 487 |
|
| 488 |
hw->lso_udpv4_idx = rsp->lso_format_idx; |
| 489 |
|
| 490 |
/* UDPv6 segmentation */ |
| 491 |
lso = otx2_mbox_alloc_msg_nix_lso_format_cfg(&pfvf->mbox); |
| 492 |
if (!lso) |
| 493 |
goto fail; |
| 494 |
|
| 495 |
/* Setup UDP/IP header fields that HW should update per segment */ |
| 496 |
otx2_setup_udp_segmentation(lso, false); |
| 497 |
|
| 498 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 499 |
if (err) |
| 500 |
goto fail; |
| 501 |
|
| 502 |
rsp = (struct nix_lso_format_cfg_rsp *) |
| 503 |
otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &lso->hdr); |
| 504 |
if (IS_ERR(rsp)) |
| 505 |
goto fail; |
| 506 |
|
| 507 |
hw->lso_udpv6_idx = rsp->lso_format_idx; |
| 508 |
mutex_unlock(&pfvf->mbox.lock); |
| 509 |
return; |
| 510 |
fail: |
| 511 |
mutex_unlock(&pfvf->mbox.lock); |
| 512 |
netdev_info(pfvf->netdev, |
| 513 |
"Failed to get LSO index for UDP GSO offload, disabling\n"); |
| 514 |
pfvf->netdev->hw_features &= ~NETIF_F_GSO_UDP_L4; |
| 515 |
} |
| 516 |
|
| 517 |
void otx2_config_irq_coalescing(struct otx2_nic *pfvf, int qidx) |
| 518 |
{ |
| 519 |
/* Configure CQE interrupt coalescing parameters |
| 520 |
* |
| 521 |
* HW triggers an irq when ECOUNT > cq_ecount_wait, hence |
| 522 |
* set 1 less than cq_ecount_wait. And cq_time_wait is in |
| 523 |
* usecs, convert that to 100ns count. |
| 524 |
*/ |
| 525 |
otx2_write64(pfvf, NIX_LF_CINTX_WAIT(qidx), |
| 526 |
((u64)(pfvf->hw.cq_time_wait * 10) << 48) | |
| 527 |
((u64)pfvf->hw.cq_qcount_wait << 32) | |
| 528 |
(pfvf->hw.cq_ecount_wait - 1)); |
| 529 |
} |
| 530 |
|
| 531 |
static int otx2_alloc_pool_buf(struct otx2_nic *pfvf, struct otx2_pool *pool, |
| 532 |
dma_addr_t *dma) |
| 533 |
{ |
| 534 |
unsigned int offset = 0; |
| 535 |
struct page *page; |
| 536 |
size_t sz; |
| 537 |
|
| 538 |
sz = SKB_DATA_ALIGN(pool->rbsize); |
| 539 |
sz = ALIGN(sz, OTX2_ALIGN); |
| 540 |
|
| 541 |
page = page_pool_alloc_frag(pool->page_pool, &offset, sz, GFP_ATOMIC); |
| 542 |
if (unlikely(!page)) |
| 543 |
return -ENOMEM; |
| 544 |
|
| 545 |
*dma = page_pool_get_dma_addr(page) + offset; |
| 546 |
return 0; |
| 547 |
} |
| 548 |
|
| 549 |
static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, |
| 550 |
dma_addr_t *dma, int qidx, int idx) |
| 551 |
{ |
| 552 |
u8 *buf; |
| 553 |
|
| 554 |
if (pool->xsk_pool) |
| 555 |
return otx2_xsk_pool_alloc_buf(pfvf, pool, dma, idx); |
| 556 |
|
| 557 |
if (pool->page_pool) |
| 558 |
return otx2_alloc_pool_buf(pfvf, pool, dma); |
| 559 |
|
| 560 |
buf = napi_alloc_frag_align(pool->rbsize, OTX2_ALIGN); |
| 561 |
if (unlikely(!buf)) |
| 562 |
return -ENOMEM; |
| 563 |
|
| 564 |
*dma = dma_map_single_attrs(pfvf->dev, buf, pool->rbsize, |
| 565 |
DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); |
| 566 |
if (unlikely(dma_mapping_error(pfvf->dev, *dma))) { |
| 567 |
page_frag_free(buf); |
| 568 |
return -ENOMEM; |
| 569 |
} |
| 570 |
|
| 571 |
return 0; |
| 572 |
} |
| 573 |
|
| 574 |
int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, |
| 575 |
dma_addr_t *dma, int qidx, int idx) |
| 576 |
{ |
| 577 |
int ret; |
| 578 |
|
| 579 |
local_bh_disable(); |
| 580 |
ret = __otx2_alloc_rbuf(pfvf, pool, dma, qidx, idx); |
| 581 |
local_bh_enable(); |
| 582 |
return ret; |
| 583 |
} |
| 584 |
|
| 585 |
int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, |
| 586 |
dma_addr_t *dma) |
| 587 |
{ |
| 588 |
if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma, |
| 589 |
cq->cq_idx, cq->pool_ptrs - 1))) |
| 590 |
return -ENOMEM; |
| 591 |
return 0; |
| 592 |
} |
| 593 |
|
| 594 |
void otx2_tx_timeout(struct net_device *netdev, unsigned int txq) |
| 595 |
{ |
| 596 |
struct otx2_nic *pfvf = netdev_priv(netdev); |
| 597 |
|
| 598 |
schedule_work(&pfvf->reset_task); |
| 599 |
} |
| 600 |
EXPORT_SYMBOL(otx2_tx_timeout); |
| 601 |
|
| 602 |
void otx2_get_mac_from_af(struct net_device *netdev) |
| 603 |
{ |
| 604 |
struct otx2_nic *pfvf = netdev_priv(netdev); |
| 605 |
int err; |
| 606 |
|
| 607 |
err = otx2_hw_get_mac_addr(pfvf, netdev); |
| 608 |
if (err) |
| 609 |
dev_warn(pfvf->dev, "Failed to read mac from hardware\n"); |
| 610 |
|
| 611 |
/* If AF doesn't provide a valid MAC, generate a random one */ |
| 612 |
if (!is_valid_ether_addr(netdev->dev_addr)) |
| 613 |
eth_hw_addr_random(netdev); |
| 614 |
} |
| 615 |
EXPORT_SYMBOL(otx2_get_mac_from_af); |
| 616 |
|
| 617 |
int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool txschq_for_pfc) |
| 618 |
{ |
| 619 |
u16 (*schq_list)[MAX_TXSCHQ_PER_FUNC]; |
| 620 |
struct otx2_hw *hw = &pfvf->hw; |
| 621 |
struct nix_txschq_config *req; |
| 622 |
u64 schq, parent; |
| 623 |
u64 dwrr_val; |
| 624 |
|
| 625 |
dwrr_val = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); |
| 626 |
|
| 627 |
req = otx2_mbox_alloc_msg_nix_txschq_cfg(&pfvf->mbox); |
| 628 |
if (!req) |
| 629 |
return -ENOMEM; |
| 630 |
|
| 631 |
req->lvl = lvl; |
| 632 |
req->num_regs = 1; |
| 633 |
|
| 634 |
schq_list = hw->txschq_list; |
| 635 |
#ifdef CONFIG_DCB |
| 636 |
if (txschq_for_pfc) |
| 637 |
schq_list = pfvf->pfc_schq_list; |
| 638 |
#endif |
| 639 |
|
| 640 |
schq = schq_list[lvl][prio]; |
| 641 |
/* Set topology e.t.c configuration */ |
| 642 |
if (lvl == NIX_TXSCH_LVL_SMQ) { |
| 643 |
req->reg[0] = NIX_AF_SMQX_CFG(schq); |
| 644 |
req->regval[0] = ((u64)pfvf->tx_max_pktlen << 8) | OTX2_MIN_MTU; |
| 645 |
req->regval[0] |= (0x20ULL << 51) | (0x80ULL << 39) | |
| 646 |
(0x2ULL << 36); |
| 647 |
/* Set link type for DWRR MTU selection on CN10K silicons */ |
| 648 |
if (!is_dev_otx2(pfvf->pdev)) |
| 649 |
req->regval[0] |= FIELD_PREP(GENMASK_ULL(58, 57), |
| 650 |
(u64)hw->smq_link_type); |
| 651 |
req->num_regs++; |
| 652 |
/* MDQ config */ |
| 653 |
parent = schq_list[NIX_TXSCH_LVL_TL4][prio]; |
| 654 |
req->reg[1] = NIX_AF_MDQX_PARENT(schq); |
| 655 |
req->regval[1] = parent << 16; |
| 656 |
req->num_regs++; |
| 657 |
/* Set DWRR quantum */ |
| 658 |
req->reg[2] = NIX_AF_MDQX_SCHEDULE(schq); |
| 659 |
req->regval[2] = dwrr_val; |
| 660 |
} else if (lvl == NIX_TXSCH_LVL_TL4) { |
| 661 |
int sdp_chan = hw->tx_chan_base + prio; |
| 662 |
|
| 663 |
if (is_otx2_sdp_rep(pfvf->pdev)) |
| 664 |
prio = 0; |
| 665 |
parent = schq_list[NIX_TXSCH_LVL_TL3][prio]; |
| 666 |
req->reg[0] = NIX_AF_TL4X_PARENT(schq); |
| 667 |
req->regval[0] = (u64)parent << 16; |
| 668 |
req->num_regs++; |
| 669 |
req->reg[1] = NIX_AF_TL4X_SCHEDULE(schq); |
| 670 |
req->regval[1] = dwrr_val; |
| 671 |
if (is_otx2_sdp_rep(pfvf->pdev)) { |
| 672 |
req->num_regs++; |
| 673 |
req->reg[2] = NIX_AF_TL4X_SDP_LINK_CFG(schq); |
| 674 |
req->regval[2] = BIT_ULL(12) | BIT_ULL(13) | |
| 675 |
(sdp_chan & 0xff); |
| 676 |
} |
| 677 |
} else if (lvl == NIX_TXSCH_LVL_TL3) { |
| 678 |
parent = schq_list[NIX_TXSCH_LVL_TL2][prio]; |
| 679 |
req->reg[0] = NIX_AF_TL3X_PARENT(schq); |
| 680 |
req->regval[0] = (u64)parent << 16; |
| 681 |
req->num_regs++; |
| 682 |
req->reg[1] = NIX_AF_TL3X_SCHEDULE(schq); |
| 683 |
req->regval[1] = dwrr_val; |
| 684 |
if (lvl == hw->txschq_link_cfg_lvl && |
| 685 |
!is_otx2_sdp_rep(pfvf->pdev)) { |
| 686 |
req->num_regs++; |
| 687 |
req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); |
| 688 |
/* Enable this queue and backpressure |
| 689 |
* and set relative channel |
| 690 |
*/ |
| 691 |
req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; |
| 692 |
} |
| 693 |
} else if (lvl == NIX_TXSCH_LVL_TL2) { |
| 694 |
parent = schq_list[NIX_TXSCH_LVL_TL1][prio]; |
| 695 |
req->reg[0] = NIX_AF_TL2X_PARENT(schq); |
| 696 |
req->regval[0] = (u64)parent << 16; |
| 697 |
|
| 698 |
req->num_regs++; |
| 699 |
req->reg[1] = NIX_AF_TL2X_SCHEDULE(schq); |
| 700 |
req->regval[1] = (u64)hw->txschq_aggr_lvl_rr_prio << 24 | dwrr_val; |
| 701 |
|
| 702 |
if (lvl == hw->txschq_link_cfg_lvl && |
| 703 |
!is_otx2_sdp_rep(pfvf->pdev)) { |
| 704 |
req->num_regs++; |
| 705 |
req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); |
| 706 |
/* Enable this queue and backpressure |
| 707 |
* and set relative channel |
| 708 |
*/ |
| 709 |
req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; |
| 710 |
} |
| 711 |
} else if (lvl == NIX_TXSCH_LVL_TL1) { |
| 712 |
/* Default config for TL1. |
| 713 |
* For VF this is always ignored. |
| 714 |
*/ |
| 715 |
|
| 716 |
/* On CN10K, if RR_WEIGHT is greater than 16384, HW will |
| 717 |
* clip it to 16384, so configuring a 24bit max value |
| 718 |
* will work on both OTx2 and CN10K. |
| 719 |
*/ |
| 720 |
req->reg[0] = NIX_AF_TL1X_SCHEDULE(schq); |
| 721 |
req->regval[0] = TXSCH_TL1_DFLT_RR_QTM; |
| 722 |
|
| 723 |
req->num_regs++; |
| 724 |
req->reg[1] = NIX_AF_TL1X_TOPOLOGY(schq); |
| 725 |
req->regval[1] = hw->txschq_aggr_lvl_rr_prio << 1; |
| 726 |
|
| 727 |
req->num_regs++; |
| 728 |
req->reg[2] = NIX_AF_TL1X_CIR(schq); |
| 729 |
req->regval[2] = 0; |
| 730 |
} |
| 731 |
|
| 732 |
return otx2_sync_mbox_msg(&pfvf->mbox); |
| 733 |
} |
| 734 |
EXPORT_SYMBOL(otx2_txschq_config); |
| 735 |
|
| 736 |
int otx2_smq_flush(struct otx2_nic *pfvf, int smq) |
| 737 |
{ |
| 738 |
struct nix_txschq_config *req; |
| 739 |
int rc; |
| 740 |
|
| 741 |
mutex_lock(&pfvf->mbox.lock); |
| 742 |
|
| 743 |
req = otx2_mbox_alloc_msg_nix_txschq_cfg(&pfvf->mbox); |
| 744 |
if (!req) { |
| 745 |
mutex_unlock(&pfvf->mbox.lock); |
| 746 |
return -ENOMEM; |
| 747 |
} |
| 748 |
|
| 749 |
req->lvl = NIX_TXSCH_LVL_SMQ; |
| 750 |
req->reg[0] = NIX_AF_SMQX_CFG(smq); |
| 751 |
req->regval[0] |= BIT_ULL(49); |
| 752 |
req->num_regs++; |
| 753 |
|
| 754 |
rc = otx2_sync_mbox_msg(&pfvf->mbox); |
| 755 |
mutex_unlock(&pfvf->mbox.lock); |
| 756 |
return rc; |
| 757 |
} |
| 758 |
EXPORT_SYMBOL(otx2_smq_flush); |
| 759 |
|
| 760 |
int otx2_txsch_alloc(struct otx2_nic *pfvf) |
| 761 |
{ |
| 762 |
int chan_cnt = pfvf->hw.tx_chan_cnt; |
| 763 |
struct nix_txsch_alloc_req *req; |
| 764 |
struct nix_txsch_alloc_rsp *rsp; |
| 765 |
int lvl, schq, rc; |
| 766 |
|
| 767 |
/* Get memory to put this msg */ |
| 768 |
req = otx2_mbox_alloc_msg_nix_txsch_alloc(&pfvf->mbox); |
| 769 |
if (!req) |
| 770 |
return -ENOMEM; |
| 771 |
|
| 772 |
/* Request one schq per level */ |
| 773 |
for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) |
| 774 |
req->schq[lvl] = 1; |
| 775 |
|
| 776 |
if (is_otx2_sdp_rep(pfvf->pdev) && chan_cnt > 1) { |
| 777 |
req->schq[NIX_TXSCH_LVL_SMQ] = chan_cnt; |
| 778 |
req->schq[NIX_TXSCH_LVL_TL4] = chan_cnt; |
| 779 |
} |
| 780 |
|
| 781 |
rc = otx2_sync_mbox_msg(&pfvf->mbox); |
| 782 |
if (rc) |
| 783 |
return rc; |
| 784 |
|
| 785 |
rsp = (struct nix_txsch_alloc_rsp *) |
| 786 |
otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); |
| 787 |
if (IS_ERR(rsp)) |
| 788 |
return PTR_ERR(rsp); |
| 789 |
|
| 790 |
/* Setup transmit scheduler list */ |
| 791 |
for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { |
| 792 |
pfvf->hw.txschq_cnt[lvl] = rsp->schq[lvl]; |
| 793 |
for (schq = 0; schq < rsp->schq[lvl]; schq++) |
| 794 |
pfvf->hw.txschq_list[lvl][schq] = |
| 795 |
rsp->schq_list[lvl][schq]; |
| 796 |
} |
| 797 |
|
| 798 |
pfvf->hw.txschq_link_cfg_lvl = rsp->link_cfg_lvl; |
| 799 |
pfvf->hw.txschq_aggr_lvl_rr_prio = rsp->aggr_lvl_rr_prio; |
| 800 |
|
| 801 |
return 0; |
| 802 |
} |
| 803 |
|
| 804 |
void otx2_txschq_free_one(struct otx2_nic *pfvf, u16 lvl, u16 schq) |
| 805 |
{ |
| 806 |
struct nix_txsch_free_req *free_req; |
| 807 |
int err; |
| 808 |
|
| 809 |
mutex_lock(&pfvf->mbox.lock); |
| 810 |
|
| 811 |
free_req = otx2_mbox_alloc_msg_nix_txsch_free(&pfvf->mbox); |
| 812 |
if (!free_req) { |
| 813 |
mutex_unlock(&pfvf->mbox.lock); |
| 814 |
netdev_err(pfvf->netdev, |
| 815 |
"Failed alloc txschq free req\n"); |
| 816 |
return; |
| 817 |
} |
| 818 |
|
| 819 |
free_req->schq_lvl = lvl; |
| 820 |
free_req->schq = schq; |
| 821 |
|
| 822 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 823 |
if (err) { |
| 824 |
netdev_err(pfvf->netdev, |
| 825 |
"Failed stop txschq %d at level %d\n", schq, lvl); |
| 826 |
} |
| 827 |
|
| 828 |
mutex_unlock(&pfvf->mbox.lock); |
| 829 |
} |
| 830 |
EXPORT_SYMBOL(otx2_txschq_free_one); |
| 831 |
|
| 832 |
void otx2_txschq_stop(struct otx2_nic *pfvf) |
| 833 |
{ |
| 834 |
int lvl, schq, idx; |
| 835 |
|
| 836 |
/* free non QOS TLx nodes */ |
| 837 |
for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { |
| 838 |
for (idx = 0; idx < pfvf->hw.txschq_cnt[lvl]; idx++) { |
| 839 |
otx2_txschq_free_one(pfvf, lvl, |
| 840 |
pfvf->hw.txschq_list[lvl][idx]); |
| 841 |
} |
| 842 |
} |
| 843 |
|
| 844 |
/* Clear the txschq list */ |
| 845 |
for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { |
| 846 |
for (schq = 0; schq < MAX_TXSCHQ_PER_FUNC; schq++) |
| 847 |
pfvf->hw.txschq_list[lvl][schq] = 0; |
| 848 |
} |
| 849 |
|
| 850 |
} |
| 851 |
|
| 852 |
void otx2_sqb_flush(struct otx2_nic *pfvf) |
| 853 |
{ |
| 854 |
int qidx, sqe_tail, sqe_head; |
| 855 |
struct otx2_snd_queue *sq; |
| 856 |
void __iomem *ptr; |
| 857 |
u64 incr, val; |
| 858 |
|
| 859 |
ptr = otx2_get_regaddr(pfvf, NIX_LF_SQ_OP_STATUS); |
| 860 |
for (qidx = 0; qidx < otx2_get_total_tx_queues(pfvf); qidx++) { |
| 861 |
sq = &pfvf->qset.sq[qidx]; |
| 862 |
if (!sq->sqb_ptrs) |
| 863 |
continue; |
| 864 |
|
| 865 |
incr = (u64)qidx << 32; |
| 866 |
val = otx2_atomic64_add(incr, ptr); |
| 867 |
sqe_head = (val >> 20) & 0x3F; |
| 868 |
sqe_tail = (val >> 28) & 0x3F; |
| 869 |
if (sqe_head != sqe_tail) |
| 870 |
usleep_range(50, 60); |
| 871 |
} |
| 872 |
} |
| 873 |
|
| 874 |
/* RED and drop levels of CQ on packet reception. |
| 875 |
* For CQ level is measure of emptiness ( 0x0 = full, 255 = empty). |
| 876 |
*/ |
| 877 |
#define RQ_PASS_LVL_CQ(skid, qsize) ((((skid) + 16) * 256) / (qsize)) |
| 878 |
#define RQ_DROP_LVL_CQ(skid, qsize) (((skid) * 256) / (qsize)) |
| 879 |
|
| 880 |
/* RED and drop levels of AURA for packet reception. |
| 881 |
* For AURA level is measure of fullness (0x0 = empty, 255 = full). |
| 882 |
* Eg: For RQ length 1K, for pass/drop level 204/230. |
| 883 |
* RED accepts pkts if free pointers > 102 & <= 205. |
| 884 |
* Drops pkts if free pointers < 102. |
| 885 |
*/ |
| 886 |
#define RQ_BP_LVL_AURA (255 - ((85 * 256) / 100)) /* BP when 85% is full */ |
| 887 |
#define RQ_PASS_LVL_AURA (255 - ((95 * 256) / 100)) /* RED when 95% is full */ |
| 888 |
#define RQ_DROP_LVL_AURA (255 - ((99 * 256) / 100)) /* Drop when 99% is full */ |
| 889 |
|
| 890 |
int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura) |
| 891 |
{ |
| 892 |
struct otx2_qset *qset = &pfvf->qset; |
| 893 |
struct nix_aq_enq_req *aq; |
| 894 |
|
| 895 |
/* Get memory to put this msg */ |
| 896 |
aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); |
| 897 |
if (!aq) |
| 898 |
return -ENOMEM; |
| 899 |
|
| 900 |
aq->rq.cq = qidx; |
| 901 |
aq->rq.ena = 1; |
| 902 |
aq->rq.pb_caching = 1; |
| 903 |
aq->rq.lpb_aura = lpb_aura; /* Use large packet buffer aura */ |
| 904 |
aq->rq.lpb_sizem1 = (DMA_BUFFER_LEN(pfvf->rbsize) / 8) - 1; |
| 905 |
aq->rq.xqe_imm_size = 0; /* Copying of packet to CQE not needed */ |
| 906 |
aq->rq.flow_tagw = 32; /* Copy full 32bit flow_tag to CQE header */ |
| 907 |
aq->rq.qint_idx = 0; |
| 908 |
aq->rq.lpb_drop_ena = 1; /* Enable RED dropping for AURA */ |
| 909 |
aq->rq.xqe_drop_ena = 1; /* Enable RED dropping for CQ/SSO */ |
| 910 |
aq->rq.xqe_pass = RQ_PASS_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); |
| 911 |
aq->rq.xqe_drop = RQ_DROP_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); |
| 912 |
aq->rq.lpb_aura_pass = RQ_PASS_LVL_AURA; |
| 913 |
aq->rq.lpb_aura_drop = RQ_DROP_LVL_AURA; |
| 914 |
|
| 915 |
/* Fill AQ info */ |
| 916 |
aq->qidx = qidx; |
| 917 |
aq->ctype = NIX_AQ_CTYPE_RQ; |
| 918 |
aq->op = NIX_AQ_INSTOP_INIT; |
| 919 |
|
| 920 |
return otx2_sync_mbox_msg(&pfvf->mbox); |
| 921 |
} |
| 922 |
|
| 923 |
int otx2_sq_aq_init(void *dev, u16 qidx, u8 chan_offset, u16 sqb_aura) |
| 924 |
{ |
| 925 |
struct otx2_nic *pfvf = dev; |
| 926 |
struct otx2_snd_queue *sq; |
| 927 |
struct nix_aq_enq_req *aq; |
| 928 |
|
| 929 |
sq = &pfvf->qset.sq[qidx]; |
| 930 |
sq->lmt_addr = (__force u64 *)(pfvf->reg_base + LMT_LF_LMTLINEX(qidx)); |
| 931 |
/* Get memory to put this msg */ |
| 932 |
aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); |
| 933 |
if (!aq) |
| 934 |
return -ENOMEM; |
| 935 |
|
| 936 |
aq->sq.cq = pfvf->hw.rx_queues + qidx; |
| 937 |
aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ |
| 938 |
aq->sq.cq_ena = 1; |
| 939 |
aq->sq.ena = 1; |
| 940 |
aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); |
| 941 |
aq->sq.smq_rr_quantum = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); |
| 942 |
aq->sq.default_chan = pfvf->hw.tx_chan_base + chan_offset; |
| 943 |
aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ |
| 944 |
aq->sq.sqb_aura = sqb_aura; |
| 945 |
aq->sq.sq_int_ena = NIX_SQINT_BITS; |
| 946 |
aq->sq.qint_idx = 0; |
| 947 |
/* Due pipelining impact minimum 2000 unused SQ CQE's |
| 948 |
* need to maintain to avoid CQ overflow. |
| 949 |
*/ |
| 950 |
aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); |
| 951 |
|
| 952 |
/* Fill AQ info */ |
| 953 |
aq->qidx = qidx; |
| 954 |
aq->ctype = NIX_AQ_CTYPE_SQ; |
| 955 |
aq->op = NIX_AQ_INSTOP_INIT; |
| 956 |
|
| 957 |
return otx2_sync_mbox_msg(&pfvf->mbox); |
| 958 |
} |
| 959 |
|
| 960 |
int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) |
| 961 |
{ |
| 962 |
struct otx2_qset *qset = &pfvf->qset; |
| 963 |
struct otx2_snd_queue *sq; |
| 964 |
struct otx2_pool *pool; |
| 965 |
u8 chan_offset; |
| 966 |
int err; |
| 967 |
|
| 968 |
pool = &pfvf->qset.pool[sqb_aura]; |
| 969 |
sq = &qset->sq[qidx]; |
| 970 |
sq->sqe_size = NIX_SQESZ_W16 ? 64 : 128; |
| 971 |
sq->sqe_cnt = qset->sqe_cnt; |
| 972 |
|
| 973 |
err = qmem_alloc(pfvf->dev, &sq->sqe, 1, sq->sqe_size); |
| 974 |
if (err) |
| 975 |
return err; |
| 976 |
|
| 977 |
/* Allocate memory for NIX SQE (which includes NIX SG) and CPT SG. |
| 978 |
* SG of NIX and CPT are same in size. Allocate memory for CPT SG |
| 979 |
* same as NIX SQE for base address alignment. |
| 980 |
* Layout of a NIX SQE and CPT SG entry: |
| 981 |
* ----------------------------- |
| 982 |
* | CPT Scatter Gather | |
| 983 |
* | (SQE SIZE) | |
| 984 |
* | | |
| 985 |
* ----------------------------- |
| 986 |
* | NIX SQE | |
| 987 |
* | (SQE SIZE) | |
| 988 |
* | | |
| 989 |
* ----------------------------- |
| 990 |
*/ |
| 991 |
err = qmem_alloc(pfvf->dev, &sq->sqe_ring, qset->sqe_cnt, |
| 992 |
sq->sqe_size * 2); |
| 993 |
if (err) |
| 994 |
return err; |
| 995 |
|
| 996 |
err = qmem_alloc(pfvf->dev, &sq->cpt_resp, qset->sqe_cnt, 64); |
| 997 |
if (err) |
| 998 |
return err; |
| 999 |
|
| 1000 |
if (qidx < pfvf->hw.tx_queues) { |
| 1001 |
err = qmem_alloc(pfvf->dev, &sq->tso_hdrs, qset->sqe_cnt, |
| 1002 |
TSO_HEADER_SIZE); |
| 1003 |
if (err) |
| 1004 |
return err; |
| 1005 |
} |
| 1006 |
|
| 1007 |
sq->sqe_base = sq->sqe->base; |
| 1008 |
sq->sg = kzalloc_objs(struct sg_list, qset->sqe_cnt); |
| 1009 |
if (!sq->sg) |
| 1010 |
return -ENOMEM; |
| 1011 |
|
| 1012 |
if (pfvf->ptp && qidx < pfvf->hw.tx_queues) { |
| 1013 |
err = qmem_alloc(pfvf->dev, &sq->timestamps, qset->sqe_cnt, |
| 1014 |
sizeof(*sq->timestamps)); |
| 1015 |
if (err) { |
| 1016 |
kfree(sq->sg); |
| 1017 |
sq->sg = NULL; |
| 1018 |
return err; |
| 1019 |
} |
| 1020 |
} |
| 1021 |
|
| 1022 |
sq->head = 0; |
| 1023 |
sq->cons_head = 0; |
| 1024 |
sq->sqe_per_sqb = (pfvf->hw.sqb_size / sq->sqe_size) - 1; |
| 1025 |
sq->num_sqbs = (qset->sqe_cnt + sq->sqe_per_sqb) / sq->sqe_per_sqb; |
| 1026 |
/* Set SQE threshold to 10% of total SQEs */ |
| 1027 |
sq->sqe_thresh = ((sq->num_sqbs * sq->sqe_per_sqb) * 10) / 100; |
| 1028 |
sq->aura_id = sqb_aura; |
| 1029 |
sq->aura_fc_addr = pool->fc_addr->base; |
| 1030 |
sq->io_addr = (__force u64)otx2_get_regaddr(pfvf, NIX_LF_OP_SENDX(0)); |
| 1031 |
|
| 1032 |
sq->stats.bytes = 0; |
| 1033 |
sq->stats.pkts = 0; |
| 1034 |
/* Attach XSK_BUFF_POOL to XDP queue */ |
| 1035 |
if (qidx > pfvf->hw.xdp_queues) |
| 1036 |
otx2_attach_xsk_buff(pfvf, sq, (qidx - pfvf->hw.xdp_queues)); |
| 1037 |
|
| 1038 |
|
| 1039 |
chan_offset = qidx % pfvf->hw.tx_chan_cnt; |
| 1040 |
err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, chan_offset, sqb_aura); |
| 1041 |
if (err) { |
| 1042 |
kfree(sq->sg); |
| 1043 |
sq->sg = NULL; |
| 1044 |
return err; |
| 1045 |
} |
| 1046 |
|
| 1047 |
return 0; |
| 1048 |
|
| 1049 |
} |
| 1050 |
|
| 1051 |
int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) |
| 1052 |
{ |
| 1053 |
struct otx2_qset *qset = &pfvf->qset; |
| 1054 |
int err, pool_id, non_xdp_queues; |
| 1055 |
struct nix_aq_enq_req *aq; |
| 1056 |
struct otx2_cq_queue *cq; |
| 1057 |
struct otx2_pool *pool; |
| 1058 |
|
| 1059 |
cq = &qset->cq[qidx]; |
| 1060 |
cq->cq_idx = qidx; |
| 1061 |
non_xdp_queues = pfvf->hw.rx_queues + pfvf->hw.tx_queues; |
| 1062 |
if (qidx < pfvf->hw.rx_queues) { |
| 1063 |
cq->cq_type = CQ_RX; |
| 1064 |
cq->cint_idx = qidx; |
| 1065 |
cq->cqe_cnt = qset->rqe_cnt; |
| 1066 |
if (pfvf->xdp_prog) { |
| 1067 |
xdp_rxq_info_reg(&cq->xdp_rxq, pfvf->netdev, qidx, 0); |
| 1068 |
pool = &qset->pool[qidx]; |
| 1069 |
if (pool->xsk_pool) { |
| 1070 |
xdp_rxq_info_reg_mem_model(&cq->xdp_rxq, |
| 1071 |
MEM_TYPE_XSK_BUFF_POOL, |
| 1072 |
NULL); |
| 1073 |
xsk_pool_set_rxq_info(pool->xsk_pool, &cq->xdp_rxq); |
| 1074 |
} else if (pool->page_pool) { |
| 1075 |
xdp_rxq_info_reg_mem_model(&cq->xdp_rxq, |
| 1076 |
MEM_TYPE_PAGE_POOL, |
| 1077 |
pool->page_pool); |
| 1078 |
} |
| 1079 |
} |
| 1080 |
} else if (qidx < non_xdp_queues) { |
| 1081 |
cq->cq_type = CQ_TX; |
| 1082 |
cq->cint_idx = qidx - pfvf->hw.rx_queues; |
| 1083 |
cq->cqe_cnt = qset->sqe_cnt; |
| 1084 |
} else { |
| 1085 |
if (pfvf->hw.xdp_queues && |
| 1086 |
qidx < non_xdp_queues + pfvf->hw.xdp_queues) { |
| 1087 |
cq->cq_type = CQ_XDP; |
| 1088 |
cq->cint_idx = qidx - non_xdp_queues; |
| 1089 |
cq->cqe_cnt = qset->sqe_cnt; |
| 1090 |
} else { |
| 1091 |
cq->cq_type = CQ_QOS; |
| 1092 |
cq->cint_idx = qidx - non_xdp_queues - |
| 1093 |
pfvf->hw.xdp_queues; |
| 1094 |
cq->cqe_cnt = qset->sqe_cnt; |
| 1095 |
} |
| 1096 |
} |
| 1097 |
cq->cqe_size = pfvf->qset.xqe_size; |
| 1098 |
|
| 1099 |
/* Allocate memory for CQEs */ |
| 1100 |
err = qmem_alloc(pfvf->dev, &cq->cqe, cq->cqe_cnt, cq->cqe_size); |
| 1101 |
if (err) |
| 1102 |
return err; |
| 1103 |
|
| 1104 |
/* Save CQE CPU base for faster reference */ |
| 1105 |
cq->cqe_base = cq->cqe->base; |
| 1106 |
/* In case where all RQs auras point to single pool, |
| 1107 |
* all CQs receive buffer pool also point to same pool. |
| 1108 |
*/ |
| 1109 |
pool_id = ((cq->cq_type == CQ_RX) && |
| 1110 |
(pfvf->hw.rqpool_cnt != pfvf->hw.rx_queues)) ? 0 : qidx; |
| 1111 |
cq->rbpool = &qset->pool[pool_id]; |
| 1112 |
cq->refill_task_sched = false; |
| 1113 |
|
| 1114 |
/* Get memory to put this msg */ |
| 1115 |
aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); |
| 1116 |
if (!aq) |
| 1117 |
return -ENOMEM; |
| 1118 |
|
| 1119 |
aq->cq.ena = 1; |
| 1120 |
aq->cq.qsize = Q_SIZE(cq->cqe_cnt, 4); |
| 1121 |
aq->cq.caching = 1; |
| 1122 |
aq->cq.base = cq->cqe->iova; |
| 1123 |
aq->cq.cint_idx = cq->cint_idx; |
| 1124 |
aq->cq.cq_err_int_ena = NIX_CQERRINT_BITS; |
| 1125 |
aq->cq.qint_idx = 0; |
| 1126 |
aq->cq.avg_level = 255; |
| 1127 |
|
| 1128 |
if (qidx < pfvf->hw.rx_queues) { |
| 1129 |
aq->cq.drop = RQ_DROP_LVL_CQ(pfvf->hw.rq_skid, cq->cqe_cnt); |
| 1130 |
aq->cq.drop_ena = 1; |
| 1131 |
|
| 1132 |
if (!is_otx2_lbkvf(pfvf->pdev)) { |
| 1133 |
/* Enable receive CQ backpressure */ |
| 1134 |
aq->cq.bp_ena = 1; |
| 1135 |
#ifdef CONFIG_DCB |
| 1136 |
aq->cq.bpid = pfvf->bpid[pfvf->queue_to_pfc_map[qidx]]; |
| 1137 |
#else |
| 1138 |
aq->cq.bpid = pfvf->bpid[0]; |
| 1139 |
#endif |
| 1140 |
|
| 1141 |
/* Set backpressure level is same as cq pass level */ |
| 1142 |
aq->cq.bp = RQ_PASS_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); |
| 1143 |
} |
| 1144 |
} |
| 1145 |
|
| 1146 |
/* Fill AQ info */ |
| 1147 |
aq->qidx = qidx; |
| 1148 |
aq->ctype = NIX_AQ_CTYPE_CQ; |
| 1149 |
aq->op = NIX_AQ_INSTOP_INIT; |
| 1150 |
|
| 1151 |
return otx2_sync_mbox_msg(&pfvf->mbox); |
| 1152 |
} |
| 1153 |
|
| 1154 |
static void otx2_pool_refill_task(struct work_struct *work) |
| 1155 |
{ |
| 1156 |
struct otx2_cq_queue *cq; |
| 1157 |
struct refill_work *wrk; |
| 1158 |
struct otx2_nic *pfvf; |
| 1159 |
int qidx; |
| 1160 |
|
| 1161 |
wrk = container_of(work, struct refill_work, pool_refill_work.work); |
| 1162 |
pfvf = wrk->pf; |
| 1163 |
qidx = wrk - pfvf->refill_wrk; |
| 1164 |
cq = &pfvf->qset.cq[qidx]; |
| 1165 |
|
| 1166 |
cq->refill_task_sched = false; |
| 1167 |
|
| 1168 |
local_bh_disable(); |
| 1169 |
napi_schedule(wrk->napi); |
| 1170 |
local_bh_enable(); |
| 1171 |
} |
| 1172 |
|
| 1173 |
int otx2_config_nix_queues(struct otx2_nic *pfvf) |
| 1174 |
{ |
| 1175 |
int qidx, err; |
| 1176 |
|
| 1177 |
/* Initialize RX queues */ |
| 1178 |
for (qidx = 0; qidx < pfvf->hw.rx_queues; qidx++) { |
| 1179 |
u16 lpb_aura = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx); |
| 1180 |
|
| 1181 |
err = otx2_rq_init(pfvf, qidx, lpb_aura); |
| 1182 |
if (err) |
| 1183 |
return err; |
| 1184 |
} |
| 1185 |
|
| 1186 |
/* Initialize TX queues */ |
| 1187 |
for (qidx = 0; qidx < pfvf->hw.non_qos_queues; qidx++) { |
| 1188 |
u16 sqb_aura = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, qidx); |
| 1189 |
|
| 1190 |
err = otx2_sq_init(pfvf, qidx, sqb_aura); |
| 1191 |
if (err) |
| 1192 |
return err; |
| 1193 |
} |
| 1194 |
|
| 1195 |
/* Initialize completion queues */ |
| 1196 |
for (qidx = 0; qidx < pfvf->qset.cq_cnt; qidx++) { |
| 1197 |
err = otx2_cq_init(pfvf, qidx); |
| 1198 |
if (err) |
| 1199 |
return err; |
| 1200 |
} |
| 1201 |
|
| 1202 |
pfvf->cq_op_addr = (__force u64 *)otx2_get_regaddr(pfvf, |
| 1203 |
NIX_LF_CQ_OP_STATUS); |
| 1204 |
|
| 1205 |
/* Initialize work queue for receive buffer refill */ |
| 1206 |
pfvf->refill_wrk = devm_kcalloc(pfvf->dev, pfvf->qset.cq_cnt, |
| 1207 |
sizeof(struct refill_work), GFP_KERNEL); |
| 1208 |
if (!pfvf->refill_wrk) |
| 1209 |
return -ENOMEM; |
| 1210 |
|
| 1211 |
for (qidx = 0; qidx < pfvf->qset.cq_cnt; qidx++) { |
| 1212 |
pfvf->refill_wrk[qidx].pf = pfvf; |
| 1213 |
INIT_DELAYED_WORK(&pfvf->refill_wrk[qidx].pool_refill_work, |
| 1214 |
otx2_pool_refill_task); |
| 1215 |
} |
| 1216 |
return 0; |
| 1217 |
} |
| 1218 |
|
| 1219 |
int otx2_config_nix(struct otx2_nic *pfvf) |
| 1220 |
{ |
| 1221 |
struct nix_lf_alloc_req *nixlf; |
| 1222 |
struct nix_lf_alloc_rsp *rsp; |
| 1223 |
int err; |
| 1224 |
|
| 1225 |
pfvf->qset.xqe_size = pfvf->hw.xqe_size; |
| 1226 |
|
| 1227 |
/* Get memory to put this msg */ |
| 1228 |
nixlf = otx2_mbox_alloc_msg_nix_lf_alloc(&pfvf->mbox); |
| 1229 |
if (!nixlf) |
| 1230 |
return -ENOMEM; |
| 1231 |
|
| 1232 |
/* Set RQ/SQ/CQ counts */ |
| 1233 |
nixlf->rq_cnt = pfvf->hw.rx_queues; |
| 1234 |
nixlf->sq_cnt = otx2_get_total_tx_queues(pfvf); |
| 1235 |
nixlf->cq_cnt = pfvf->qset.cq_cnt; |
| 1236 |
nixlf->rss_sz = MAX_RSS_INDIR_TBL_SIZE; |
| 1237 |
nixlf->rss_grps = MAX_RSS_GROUPS; |
| 1238 |
nixlf->xqe_sz = pfvf->hw.xqe_size == 128 ? NIX_XQESZ_W16 : NIX_XQESZ_W64; |
| 1239 |
/* We don't know absolute NPA LF idx attached. |
| 1240 |
* AF will replace 'RVU_DEFAULT_PF_FUNC' with |
| 1241 |
* NPA LF attached to this RVU PF/VF. |
| 1242 |
*/ |
| 1243 |
nixlf->npa_func = RVU_DEFAULT_PF_FUNC; |
| 1244 |
/* Disable alignment pad, enable L2 length check, |
| 1245 |
* enable L4 TCP/UDP checksum verification. |
| 1246 |
*/ |
| 1247 |
nixlf->rx_cfg = BIT_ULL(33) | BIT_ULL(35) | BIT_ULL(37); |
| 1248 |
|
| 1249 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 1250 |
if (err) |
| 1251 |
return err; |
| 1252 |
|
| 1253 |
rsp = (struct nix_lf_alloc_rsp *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, |
| 1254 |
&nixlf->hdr); |
| 1255 |
if (IS_ERR(rsp)) |
| 1256 |
return PTR_ERR(rsp); |
| 1257 |
|
| 1258 |
if (rsp->qints < 1) |
| 1259 |
return -ENXIO; |
| 1260 |
|
| 1261 |
return rsp->hdr.rc; |
| 1262 |
} |
| 1263 |
|
| 1264 |
void otx2_sq_free_sqbs(struct otx2_nic *pfvf) |
| 1265 |
{ |
| 1266 |
struct otx2_qset *qset = &pfvf->qset; |
| 1267 |
struct otx2_hw *hw = &pfvf->hw; |
| 1268 |
struct otx2_snd_queue *sq; |
| 1269 |
int sqb, qidx; |
| 1270 |
u64 iova, pa; |
| 1271 |
|
| 1272 |
for (qidx = 0; qidx < otx2_get_total_tx_queues(pfvf); qidx++) { |
| 1273 |
sq = &qset->sq[qidx]; |
| 1274 |
if (!sq->sqb_ptrs) |
| 1275 |
continue; |
| 1276 |
for (sqb = 0; sqb < sq->sqb_count; sqb++) { |
| 1277 |
if (!sq->sqb_ptrs[sqb]) |
| 1278 |
continue; |
| 1279 |
iova = sq->sqb_ptrs[sqb]; |
| 1280 |
pa = otx2_iova_to_phys(pfvf->iommu_domain, iova); |
| 1281 |
dma_unmap_page_attrs(pfvf->dev, iova, hw->sqb_size, |
| 1282 |
DMA_FROM_DEVICE, |
| 1283 |
DMA_ATTR_SKIP_CPU_SYNC); |
| 1284 |
put_page(virt_to_page(phys_to_virt(pa))); |
| 1285 |
} |
| 1286 |
sq->sqb_count = 0; |
| 1287 |
} |
| 1288 |
} |
| 1289 |
|
| 1290 |
void otx2_free_bufs(struct otx2_nic *pfvf, struct otx2_pool *pool, |
| 1291 |
u64 iova, int size) |
| 1292 |
{ |
| 1293 |
struct page *page; |
| 1294 |
u64 pa; |
| 1295 |
|
| 1296 |
pa = otx2_iova_to_phys(pfvf->iommu_domain, iova); |
| 1297 |
page = virt_to_head_page(phys_to_virt(pa)); |
| 1298 |
if (pool->page_pool) { |
| 1299 |
page_pool_put_full_page(pool->page_pool, page, true); |
| 1300 |
} else if (pool->xsk_pool) { |
| 1301 |
/* Note: No way of identifying xdp_buff */ |
| 1302 |
} else { |
| 1303 |
dma_unmap_page_attrs(pfvf->dev, iova, size, |
| 1304 |
DMA_FROM_DEVICE, |
| 1305 |
DMA_ATTR_SKIP_CPU_SYNC); |
| 1306 |
|
| 1307 |
put_page(page); |
| 1308 |
} |
| 1309 |
} |
| 1310 |
|
| 1311 |
void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type) |
| 1312 |
{ |
| 1313 |
int pool_id, pool_start = 0, pool_end = 0, size = 0; |
| 1314 |
struct otx2_pool *pool; |
| 1315 |
u64 iova; |
| 1316 |
int idx; |
| 1317 |
|
| 1318 |
if (type == AURA_NIX_SQ) { |
| 1319 |
pool_start = otx2_get_pool_idx(pfvf, type, 0); |
| 1320 |
pool_end = pool_start + pfvf->hw.sqpool_cnt; |
| 1321 |
size = pfvf->hw.sqb_size; |
| 1322 |
} |
| 1323 |
if (type == AURA_NIX_RQ) { |
| 1324 |
pool_start = otx2_get_pool_idx(pfvf, type, 0); |
| 1325 |
pool_end = pfvf->hw.rqpool_cnt; |
| 1326 |
size = pfvf->rbsize; |
| 1327 |
} |
| 1328 |
|
| 1329 |
/* Free SQB and RQB pointers from the aura pool */ |
| 1330 |
for (pool_id = pool_start; pool_id < pool_end; pool_id++) { |
| 1331 |
pool = &pfvf->qset.pool[pool_id]; |
| 1332 |
iova = otx2_aura_allocptr(pfvf, pool_id); |
| 1333 |
while (iova) { |
| 1334 |
if (type == AURA_NIX_RQ) |
| 1335 |
iova -= OTX2_HEAD_ROOM; |
| 1336 |
otx2_free_bufs(pfvf, pool, iova, size); |
| 1337 |
iova = otx2_aura_allocptr(pfvf, pool_id); |
| 1338 |
} |
| 1339 |
|
| 1340 |
for (idx = 0 ; idx < pool->xdp_cnt; idx++) { |
| 1341 |
if (!pool->xdp[idx]) |
| 1342 |
continue; |
| 1343 |
|
| 1344 |
xsk_buff_free(pool->xdp[idx]); |
| 1345 |
} |
| 1346 |
} |
| 1347 |
} |
| 1348 |
|
| 1349 |
void otx2_aura_pool_free(struct otx2_nic *pfvf) |
| 1350 |
{ |
| 1351 |
struct otx2_pool *pool; |
| 1352 |
int pool_id; |
| 1353 |
|
| 1354 |
if (!pfvf->qset.pool) |
| 1355 |
return; |
| 1356 |
|
| 1357 |
for (pool_id = 0; pool_id < pfvf->hw.pool_cnt; pool_id++) { |
| 1358 |
pool = &pfvf->qset.pool[pool_id]; |
| 1359 |
qmem_free(pfvf->dev, pool->stack); |
| 1360 |
qmem_free(pfvf->dev, pool->fc_addr); |
| 1361 |
page_pool_destroy(pool->page_pool); |
| 1362 |
devm_kfree(pfvf->dev, pool->xdp); |
| 1363 |
pool->xsk_pool = NULL; |
| 1364 |
} |
| 1365 |
devm_kfree(pfvf->dev, pfvf->qset.pool); |
| 1366 |
pfvf->qset.pool = NULL; |
| 1367 |
} |
| 1368 |
|
| 1369 |
int otx2_aura_init(struct otx2_nic *pfvf, int aura_id, |
| 1370 |
int pool_id, int numptrs) |
| 1371 |
{ |
| 1372 |
return pfvf->hw_ops->aura_aq_init(pfvf, aura_id, pool_id, |
| 1373 |
numptrs); |
| 1374 |
} |
| 1375 |
|
| 1376 |
int otx2_aura_aq_init(struct otx2_nic *pfvf, int aura_id, |
| 1377 |
int pool_id, int numptrs) |
| 1378 |
{ |
| 1379 |
struct npa_aq_enq_req *aq; |
| 1380 |
struct otx2_pool *pool; |
| 1381 |
int err; |
| 1382 |
|
| 1383 |
pool = &pfvf->qset.pool[pool_id]; |
| 1384 |
|
| 1385 |
/* Allocate memory for HW to update Aura count. |
| 1386 |
* Alloc one cache line, so that it fits all FC_STYPE modes. |
| 1387 |
*/ |
| 1388 |
if (!pool->fc_addr) { |
| 1389 |
err = qmem_alloc(pfvf->dev, &pool->fc_addr, 1, OTX2_ALIGN); |
| 1390 |
if (err) |
| 1391 |
return err; |
| 1392 |
} |
| 1393 |
|
| 1394 |
/* Initialize this aura's context via AF */ |
| 1395 |
aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); |
| 1396 |
if (!aq) { |
| 1397 |
/* Shared mbox memory buffer is full, flush it and retry */ |
| 1398 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 1399 |
if (err) |
| 1400 |
return err; |
| 1401 |
aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); |
| 1402 |
if (!aq) |
| 1403 |
return -ENOMEM; |
| 1404 |
} |
| 1405 |
|
| 1406 |
aq->aura_id = aura_id; |
| 1407 |
/* Will be filled by AF with correct pool context address */ |
| 1408 |
aq->aura.pool_addr = pool_id; |
| 1409 |
aq->aura.pool_caching = 1; |
| 1410 |
aq->aura.shift = ilog2(numptrs) - 8; |
| 1411 |
aq->aura.count = numptrs; |
| 1412 |
aq->aura.limit = numptrs; |
| 1413 |
aq->aura.avg_level = 255; |
| 1414 |
aq->aura.ena = 1; |
| 1415 |
aq->aura.fc_ena = 1; |
| 1416 |
aq->aura.fc_addr = pool->fc_addr->iova; |
| 1417 |
aq->aura.fc_hyst_bits = 0; /* Store count on all updates */ |
| 1418 |
|
| 1419 |
/* Enable backpressure for RQ aura */ |
| 1420 |
if (aura_id < pfvf->hw.rqpool_cnt && !is_otx2_lbkvf(pfvf->pdev)) { |
| 1421 |
aq->aura.bp_ena = 0; |
| 1422 |
/* If NIX1 LF is attached then specify NIX1_RX. |
| 1423 |
* |
| 1424 |
* Below NPA_AURA_S[BP_ENA] is set according to the |
| 1425 |
* NPA_BPINTF_E enumeration given as: |
| 1426 |
* 0x0 + a*0x1 where 'a' is 0 for NIX0_RX and 1 for NIX1_RX so |
| 1427 |
* NIX0_RX is 0x0 + 0*0x1 = 0 |
| 1428 |
* NIX1_RX is 0x0 + 1*0x1 = 1 |
| 1429 |
* But in HRM it is given that |
| 1430 |
* "NPA_AURA_S[BP_ENA](w1[33:32]) - Enable aura backpressure to |
| 1431 |
* NIX-RX based on [BP] level. One bit per NIX-RX; index |
| 1432 |
* enumerated by NPA_BPINTF_E." |
| 1433 |
*/ |
| 1434 |
if (pfvf->nix_blkaddr == BLKADDR_NIX1) |
| 1435 |
aq->aura.bp_ena = 1; |
| 1436 |
#ifdef CONFIG_DCB |
| 1437 |
aq->aura.nix0_bpid = pfvf->bpid[pfvf->queue_to_pfc_map[aura_id]]; |
| 1438 |
#else |
| 1439 |
aq->aura.nix0_bpid = pfvf->bpid[0]; |
| 1440 |
#endif |
| 1441 |
|
| 1442 |
/* Set backpressure level for RQ's Aura */ |
| 1443 |
aq->aura.bp = RQ_BP_LVL_AURA; |
| 1444 |
} |
| 1445 |
|
| 1446 |
/* Fill AQ info */ |
| 1447 |
aq->ctype = NPA_AQ_CTYPE_AURA; |
| 1448 |
aq->op = NPA_AQ_INSTOP_INIT; |
| 1449 |
|
| 1450 |
return 0; |
| 1451 |
} |
| 1452 |
|
| 1453 |
int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id, |
| 1454 |
int stack_pages, int numptrs, int buf_size, int type) |
| 1455 |
{ |
| 1456 |
return pfvf->hw_ops->pool_aq_init(pfvf, pool_id, stack_pages, numptrs, |
| 1457 |
buf_size, type); |
| 1458 |
} |
| 1459 |
|
| 1460 |
int otx2_pool_aq_init(struct otx2_nic *pfvf, u16 pool_id, |
| 1461 |
int stack_pages, int numptrs, int buf_size, int type) |
| 1462 |
{ |
| 1463 |
struct page_pool_params pp_params = { 0 }; |
| 1464 |
struct xsk_buff_pool *xsk_pool; |
| 1465 |
struct npa_aq_enq_req *aq; |
| 1466 |
struct otx2_pool *pool; |
| 1467 |
int err; |
| 1468 |
|
| 1469 |
pool = &pfvf->qset.pool[pool_id]; |
| 1470 |
/* Alloc memory for stack which is used to store buffer pointers */ |
| 1471 |
err = qmem_alloc(pfvf->dev, &pool->stack, |
| 1472 |
stack_pages, pfvf->hw.stack_pg_bytes); |
| 1473 |
if (err) |
| 1474 |
return err; |
| 1475 |
|
| 1476 |
pool->rbsize = buf_size; |
| 1477 |
|
| 1478 |
/* Initialize this pool's context via AF */ |
| 1479 |
aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); |
| 1480 |
if (!aq) { |
| 1481 |
/* Shared mbox memory buffer is full, flush it and retry */ |
| 1482 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 1483 |
if (err) { |
| 1484 |
qmem_free(pfvf->dev, pool->stack); |
| 1485 |
pool->stack = NULL; |
| 1486 |
return err; |
| 1487 |
} |
| 1488 |
aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); |
| 1489 |
if (!aq) { |
| 1490 |
qmem_free(pfvf->dev, pool->stack); |
| 1491 |
pool->stack = NULL; |
| 1492 |
return -ENOMEM; |
| 1493 |
} |
| 1494 |
} |
| 1495 |
|
| 1496 |
aq->aura_id = pool_id; |
| 1497 |
aq->pool.stack_base = pool->stack->iova; |
| 1498 |
aq->pool.stack_caching = 1; |
| 1499 |
aq->pool.ena = 1; |
| 1500 |
aq->pool.buf_size = buf_size / 128; |
| 1501 |
aq->pool.stack_max_pages = stack_pages; |
| 1502 |
aq->pool.shift = ilog2(numptrs) - 8; |
| 1503 |
aq->pool.ptr_start = 0; |
| 1504 |
aq->pool.ptr_end = ~0ULL; |
| 1505 |
|
| 1506 |
/* Fill AQ info */ |
| 1507 |
aq->ctype = NPA_AQ_CTYPE_POOL; |
| 1508 |
aq->op = NPA_AQ_INSTOP_INIT; |
| 1509 |
|
| 1510 |
if (type != AURA_NIX_RQ) |
| 1511 |
return 0; |
| 1512 |
|
| 1513 |
if (!test_bit(pool_id, pfvf->af_xdp_zc_qidx)) { |
| 1514 |
pp_params.order = get_order(buf_size); |
| 1515 |
pp_params.flags = PP_FLAG_DMA_MAP; |
| 1516 |
pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs); |
| 1517 |
pp_params.nid = NUMA_NO_NODE; |
| 1518 |
pp_params.dev = pfvf->dev; |
| 1519 |
pp_params.dma_dir = DMA_FROM_DEVICE; |
| 1520 |
pool->page_pool = page_pool_create(&pp_params); |
| 1521 |
if (IS_ERR(pool->page_pool)) { |
| 1522 |
netdev_err(pfvf->netdev, "Creation of page pool failed\n"); |
| 1523 |
return PTR_ERR(pool->page_pool); |
| 1524 |
} |
| 1525 |
return 0; |
| 1526 |
} |
| 1527 |
|
| 1528 |
/* Set XSK pool to support AF_XDP zero-copy */ |
| 1529 |
xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, pool_id); |
| 1530 |
if (xsk_pool) { |
| 1531 |
pool->xsk_pool = xsk_pool; |
| 1532 |
pool->xdp_cnt = numptrs; |
| 1533 |
pool->xdp = devm_kcalloc(pfvf->dev, |
| 1534 |
numptrs, sizeof(struct xdp_buff *), GFP_KERNEL); |
| 1535 |
if (!pool->xdp) |
| 1536 |
return -ENOMEM; |
| 1537 |
} |
| 1538 |
|
| 1539 |
return 0; |
| 1540 |
} |
| 1541 |
|
| 1542 |
int otx2_sq_aura_pool_init(struct otx2_nic *pfvf) |
| 1543 |
{ |
| 1544 |
int qidx, pool_id, stack_pages, num_sqbs; |
| 1545 |
struct otx2_qset *qset = &pfvf->qset; |
| 1546 |
struct otx2_hw *hw = &pfvf->hw; |
| 1547 |
struct otx2_snd_queue *sq; |
| 1548 |
struct otx2_pool *pool; |
| 1549 |
dma_addr_t bufptr; |
| 1550 |
int err, ptr; |
| 1551 |
|
| 1552 |
/* Calculate number of SQBs needed. |
| 1553 |
* |
| 1554 |
* For a 128byte SQE, and 4K size SQB, 31 SQEs will fit in one SQB. |
| 1555 |
* Last SQE is used for pointing to next SQB. |
| 1556 |
*/ |
| 1557 |
num_sqbs = (hw->sqb_size / 128) - 1; |
| 1558 |
num_sqbs = (qset->sqe_cnt + num_sqbs) / num_sqbs; |
| 1559 |
|
| 1560 |
/* Get no of stack pages needed */ |
| 1561 |
stack_pages = |
| 1562 |
(num_sqbs + hw->stack_pg_ptrs - 1) / hw->stack_pg_ptrs; |
| 1563 |
|
| 1564 |
for (qidx = 0; qidx < hw->non_qos_queues; qidx++) { |
| 1565 |
pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, qidx); |
| 1566 |
/* Initialize aura context */ |
| 1567 |
err = otx2_aura_init(pfvf, pool_id, pool_id, num_sqbs); |
| 1568 |
if (err) |
| 1569 |
goto fail; |
| 1570 |
|
| 1571 |
/* Initialize pool context */ |
| 1572 |
err = otx2_pool_init(pfvf, pool_id, stack_pages, |
| 1573 |
num_sqbs, hw->sqb_size, AURA_NIX_SQ); |
| 1574 |
if (err) |
| 1575 |
goto fail; |
| 1576 |
} |
| 1577 |
|
| 1578 |
/* Flush accumulated messages */ |
| 1579 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 1580 |
if (err) |
| 1581 |
goto fail; |
| 1582 |
|
| 1583 |
/* Allocate pointers and free them to aura/pool */ |
| 1584 |
for (qidx = 0; qidx < hw->non_qos_queues; qidx++) { |
| 1585 |
pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, qidx); |
| 1586 |
pool = &pfvf->qset.pool[pool_id]; |
| 1587 |
|
| 1588 |
sq = &qset->sq[qidx]; |
| 1589 |
sq->sqb_count = 0; |
| 1590 |
sq->sqb_ptrs = kzalloc_objs(*sq->sqb_ptrs, num_sqbs); |
| 1591 |
if (!sq->sqb_ptrs) { |
| 1592 |
err = -ENOMEM; |
| 1593 |
goto err_mem; |
| 1594 |
} |
| 1595 |
|
| 1596 |
for (ptr = 0; ptr < num_sqbs; ptr++) { |
| 1597 |
err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr); |
| 1598 |
if (err) { |
| 1599 |
if (pool->xsk_pool) { |
| 1600 |
ptr--; |
| 1601 |
while (ptr >= 0) { |
| 1602 |
xsk_buff_free(pool->xdp[ptr]); |
| 1603 |
ptr--; |
| 1604 |
} |
| 1605 |
} |
| 1606 |
goto err_mem; |
| 1607 |
} |
| 1608 |
|
| 1609 |
pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr); |
| 1610 |
sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr; |
| 1611 |
} |
| 1612 |
} |
| 1613 |
|
| 1614 |
err_mem: |
| 1615 |
return err ? -ENOMEM : 0; |
| 1616 |
|
| 1617 |
fail: |
| 1618 |
otx2_mbox_reset(&pfvf->mbox.mbox, 0); |
| 1619 |
otx2_aura_pool_free(pfvf); |
| 1620 |
return err; |
| 1621 |
} |
| 1622 |
|
| 1623 |
int otx2_rq_aura_pool_init(struct otx2_nic *pfvf) |
| 1624 |
{ |
| 1625 |
struct otx2_hw *hw = &pfvf->hw; |
| 1626 |
int stack_pages, pool_id, rq; |
| 1627 |
struct otx2_pool *pool; |
| 1628 |
int err, ptr, num_ptrs; |
| 1629 |
dma_addr_t bufptr; |
| 1630 |
|
| 1631 |
num_ptrs = pfvf->qset.rqe_cnt; |
| 1632 |
|
| 1633 |
stack_pages = |
| 1634 |
(num_ptrs + hw->stack_pg_ptrs - 1) / hw->stack_pg_ptrs; |
| 1635 |
|
| 1636 |
for (rq = 0; rq < hw->rx_queues; rq++) { |
| 1637 |
pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, rq); |
| 1638 |
/* Initialize aura context */ |
| 1639 |
err = otx2_aura_init(pfvf, pool_id, pool_id, num_ptrs); |
| 1640 |
if (err) |
| 1641 |
goto fail; |
| 1642 |
} |
| 1643 |
for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) { |
| 1644 |
err = otx2_pool_init(pfvf, pool_id, stack_pages, |
| 1645 |
num_ptrs, pfvf->rbsize, AURA_NIX_RQ); |
| 1646 |
if (err) |
| 1647 |
goto fail; |
| 1648 |
} |
| 1649 |
|
| 1650 |
/* Flush accumulated messages */ |
| 1651 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 1652 |
if (err) |
| 1653 |
goto fail; |
| 1654 |
|
| 1655 |
/* Allocate pointers and free them to aura/pool */ |
| 1656 |
for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) { |
| 1657 |
pool = &pfvf->qset.pool[pool_id]; |
| 1658 |
|
| 1659 |
for (ptr = 0; ptr < num_ptrs; ptr++) { |
| 1660 |
err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr); |
| 1661 |
if (err) { |
| 1662 |
if (pool->xsk_pool) { |
| 1663 |
while (ptr) |
| 1664 |
xsk_buff_free(pool->xdp[--ptr]); |
| 1665 |
} |
| 1666 |
return -ENOMEM; |
| 1667 |
} |
| 1668 |
|
| 1669 |
pfvf->hw_ops->aura_freeptr(pfvf, pool_id, |
| 1670 |
pool->xsk_pool ? bufptr : |
| 1671 |
bufptr + OTX2_HEAD_ROOM); |
| 1672 |
} |
| 1673 |
} |
| 1674 |
return 0; |
| 1675 |
fail: |
| 1676 |
otx2_mbox_reset(&pfvf->mbox.mbox, 0); |
| 1677 |
otx2_aura_pool_free(pfvf); |
| 1678 |
return err; |
| 1679 |
} |
| 1680 |
|
| 1681 |
int otx2_config_npa(struct otx2_nic *pfvf) |
| 1682 |
{ |
| 1683 |
struct otx2_qset *qset = &pfvf->qset; |
| 1684 |
struct npa_lf_alloc_req *npalf; |
| 1685 |
struct otx2_hw *hw = &pfvf->hw; |
| 1686 |
int aura_cnt; |
| 1687 |
|
| 1688 |
/* Pool - Stack of free buffer pointers |
| 1689 |
* Aura - Alloc/frees pointers from/to pool for NIX DMA. |
| 1690 |
*/ |
| 1691 |
|
| 1692 |
if (!hw->pool_cnt) |
| 1693 |
return -EINVAL; |
| 1694 |
|
| 1695 |
qset->pool = devm_kcalloc(pfvf->dev, hw->pool_cnt, |
| 1696 |
sizeof(struct otx2_pool), GFP_KERNEL); |
| 1697 |
if (!qset->pool) |
| 1698 |
return -ENOMEM; |
| 1699 |
|
| 1700 |
/* Get memory to put this msg */ |
| 1701 |
npalf = otx2_mbox_alloc_msg_npa_lf_alloc(&pfvf->mbox); |
| 1702 |
if (!npalf) |
| 1703 |
return -ENOMEM; |
| 1704 |
|
| 1705 |
/* Set aura and pool counts */ |
| 1706 |
npalf->nr_pools = hw->pool_cnt; |
| 1707 |
aura_cnt = ilog2(roundup_pow_of_two(hw->pool_cnt)); |
| 1708 |
npalf->aura_sz = (aura_cnt >= ilog2(128)) ? (aura_cnt - 6) : 1; |
| 1709 |
|
| 1710 |
return otx2_sync_mbox_msg(&pfvf->mbox); |
| 1711 |
} |
| 1712 |
|
| 1713 |
int otx2_detach_resources(struct mbox *mbox) |
| 1714 |
{ |
| 1715 |
struct rsrc_detach *detach; |
| 1716 |
|
| 1717 |
mutex_lock(&mbox->lock); |
| 1718 |
detach = otx2_mbox_alloc_msg_detach_resources(mbox); |
| 1719 |
if (!detach) { |
| 1720 |
mutex_unlock(&mbox->lock); |
| 1721 |
return -ENOMEM; |
| 1722 |
} |
| 1723 |
|
| 1724 |
/* detach all */ |
| 1725 |
detach->partial = false; |
| 1726 |
|
| 1727 |
/* Send detach request to AF */ |
| 1728 |
otx2_sync_mbox_msg(mbox); |
| 1729 |
mutex_unlock(&mbox->lock); |
| 1730 |
return 0; |
| 1731 |
} |
| 1732 |
EXPORT_SYMBOL(otx2_detach_resources); |
| 1733 |
|
| 1734 |
int otx2_attach_npa_nix(struct otx2_nic *pfvf) |
| 1735 |
{ |
| 1736 |
struct rsrc_attach *attach; |
| 1737 |
struct msg_req *msix; |
| 1738 |
int err; |
| 1739 |
|
| 1740 |
mutex_lock(&pfvf->mbox.lock); |
| 1741 |
/* Get memory to put this msg */ |
| 1742 |
attach = otx2_mbox_alloc_msg_attach_resources(&pfvf->mbox); |
| 1743 |
if (!attach) { |
| 1744 |
mutex_unlock(&pfvf->mbox.lock); |
| 1745 |
return -ENOMEM; |
| 1746 |
} |
| 1747 |
|
| 1748 |
attach->npalf = true; |
| 1749 |
attach->nixlf = true; |
| 1750 |
|
| 1751 |
/* Send attach request to AF */ |
| 1752 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 1753 |
if (err) { |
| 1754 |
mutex_unlock(&pfvf->mbox.lock); |
| 1755 |
return err; |
| 1756 |
} |
| 1757 |
|
| 1758 |
pfvf->nix_blkaddr = BLKADDR_NIX0; |
| 1759 |
|
| 1760 |
/* If the platform has two NIX blocks then LF may be |
| 1761 |
* allocated from NIX1. |
| 1762 |
*/ |
| 1763 |
if (otx2_read64(pfvf, RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_NIX1)) & 0x1FFULL) |
| 1764 |
pfvf->nix_blkaddr = BLKADDR_NIX1; |
| 1765 |
|
| 1766 |
/* Get NPA and NIX MSIX vector offsets */ |
| 1767 |
msix = otx2_mbox_alloc_msg_msix_offset(&pfvf->mbox); |
| 1768 |
if (!msix) { |
| 1769 |
mutex_unlock(&pfvf->mbox.lock); |
| 1770 |
return -ENOMEM; |
| 1771 |
} |
| 1772 |
|
| 1773 |
err = otx2_sync_mbox_msg(&pfvf->mbox); |
| 1774 |
if (err) { |
| 1775 |
mutex_unlock(&pfvf->mbox.lock); |
| 1776 |
return err; |
| 1777 |
} |
| 1778 |
mutex_unlock(&pfvf->mbox.lock); |
| 1779 |
|
| 1780 |
if (pfvf->hw.npa_msixoff == MSIX_VECTOR_INVALID || |
| 1781 |
pfvf->hw.nix_msixoff == MSIX_VECTOR_INVALID) { |
| 1782 |
dev_err(pfvf->dev, |
| 1783 |
"RVUPF: Invalid MSIX vector offset for NPA/NIX\n"); |
| 1784 |
return -EINVAL; |
| 1785 |
} |
| 1786 |
|
| 1787 |
return 0; |
| 1788 |
} |
| 1789 |
EXPORT_SYMBOL(otx2_attach_npa_nix); |
| 1790 |
|
| 1791 |
void otx2_ctx_disable(struct mbox *mbox, int type, bool npa) |
| 1792 |
{ |
| 1793 |
struct hwctx_disable_req *req; |
| 1794 |
|
| 1795 |
mutex_lock(&mbox->lock); |
| 1796 |
/* Request AQ to disable this context */ |
| 1797 |
if (npa) |
| 1798 |
req = otx2_mbox_alloc_msg_npa_hwctx_disable(mbox); |
| 1799 |
else |
| 1800 |
req = otx2_mbox_alloc_msg_nix_hwctx_disable(mbox); |
| 1801 |
|
| 1802 |
if (!req) { |
| 1803 |
mutex_unlock(&mbox->lock); |
| 1804 |
return; |
| 1805 |
} |
| 1806 |
|
| 1807 |
req->ctype = type; |
| 1808 |
|
| 1809 |
if (otx2_sync_mbox_msg(mbox)) |
| 1810 |
dev_err(mbox->pfvf->dev, "%s failed to disable context\n", |
| 1811 |
__func__); |
| 1812 |
|
| 1813 |
mutex_unlock(&mbox->lock); |
| 1814 |
} |
| 1815 |
|
| 1816 |
int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable) |
| 1817 |
{ |
| 1818 |
struct nix_bp_cfg_req *req; |
| 1819 |
|
| 1820 |
if (enable) |
| 1821 |
req = otx2_mbox_alloc_msg_nix_bp_enable(&pfvf->mbox); |
| 1822 |
else |
| 1823 |
req = otx2_mbox_alloc_msg_nix_bp_disable(&pfvf->mbox); |
| 1824 |
|
| 1825 |
if (!req) |
| 1826 |
return -ENOMEM; |
| 1827 |
|
| 1828 |
req->chan_base = 0; |
| 1829 |
if (otx2_is_pfc_enabled(pfvf)) { |
| 1830 |
req->chan_cnt = IEEE_8021QAZ_MAX_TCS; |
| 1831 |
req->bpid_per_chan = 1; |
| 1832 |
} else { |
| 1833 |
req->chan_cnt = pfvf->hw.rx_chan_cnt; |
| 1834 |
req->bpid_per_chan = 0; |
| 1835 |
} |
| 1836 |
|
| 1837 |
return otx2_sync_mbox_msg(&pfvf->mbox); |
| 1838 |
} |
| 1839 |
EXPORT_SYMBOL(otx2_nix_config_bp); |
| 1840 |
|
| 1841 |
int otx2_nix_cpt_config_bp(struct otx2_nic *pfvf, bool enable) |
| 1842 |
{ |
| 1843 |
struct nix_bp_cfg_req *req; |
| 1844 |
|
| 1845 |
if (enable) |
| 1846 |
req = otx2_mbox_alloc_msg_nix_cpt_bp_enable(&pfvf->mbox); |
| 1847 |
else |
| 1848 |
req = otx2_mbox_alloc_msg_nix_cpt_bp_disable(&pfvf->mbox); |
| 1849 |
|
| 1850 |
if (!req) |
| 1851 |
return -ENOMEM; |
| 1852 |
|
| 1853 |
req->chan_base = 0; |
| 1854 |
if (otx2_is_pfc_enabled(pfvf)) { |
| 1855 |
req->chan_cnt = IEEE_8021QAZ_MAX_TCS; |
| 1856 |
req->bpid_per_chan = 1; |
| 1857 |
} else { |
| 1858 |
req->chan_cnt = pfvf->hw.rx_chan_cnt; |
| 1859 |
req->bpid_per_chan = 0; |
| 1860 |
} |
| 1861 |
|
| 1862 |
return otx2_sync_mbox_msg(&pfvf->mbox); |
| 1863 |
} |
| 1864 |
EXPORT_SYMBOL(otx2_nix_cpt_config_bp); |
| 1865 |
|
| 1866 |
/* Mbox message handlers */ |
| 1867 |
void mbox_handler_cgx_stats(struct otx2_nic *pfvf, |
| 1868 |
struct cgx_stats_rsp *rsp) |
| 1869 |
{ |
| 1870 |
int id; |
| 1871 |
|
| 1872 |
for (id = 0; id < CGX_RX_STATS_COUNT; id++) |
| 1873 |
pfvf->hw.cgx_rx_stats[id] = rsp->rx_stats[id]; |
| 1874 |
for (id = 0; id < CGX_TX_STATS_COUNT; id++) |
| 1875 |
pfvf->hw.cgx_tx_stats[id] = rsp->tx_stats[id]; |
| 1876 |
} |
| 1877 |
|
| 1878 |
void mbox_handler_cgx_fec_stats(struct otx2_nic *pfvf, |
| 1879 |
struct cgx_fec_stats_rsp *rsp) |
| 1880 |
{ |
| 1881 |
pfvf->hw.cgx_fec_corr_blks += rsp->fec_corr_blks; |
| 1882 |
pfvf->hw.cgx_fec_uncorr_blks += rsp->fec_uncorr_blks; |
| 1883 |
} |
| 1884 |
|
| 1885 |
void mbox_handler_npa_lf_alloc(struct otx2_nic *pfvf, |
| 1886 |
struct npa_lf_alloc_rsp *rsp) |
| 1887 |
{ |
| 1888 |
pfvf->hw.stack_pg_ptrs = rsp->stack_pg_ptrs; |
| 1889 |
pfvf->hw.stack_pg_bytes = rsp->stack_pg_bytes; |
| 1890 |
} |
| 1891 |
EXPORT_SYMBOL(mbox_handler_npa_lf_alloc); |
| 1892 |
|
| 1893 |
void mbox_handler_nix_lf_alloc(struct otx2_nic *pfvf, |
| 1894 |
struct nix_lf_alloc_rsp *rsp) |
| 1895 |
{ |
| 1896 |
pfvf->hw.sqb_size = rsp->sqb_size; |
| 1897 |
pfvf->hw.rx_chan_base = rsp->rx_chan_base; |
| 1898 |
pfvf->hw.tx_chan_base = rsp->tx_chan_base; |
| 1899 |
pfvf->hw.rx_chan_cnt = rsp->rx_chan_cnt; |
| 1900 |
pfvf->hw.tx_chan_cnt = rsp->tx_chan_cnt; |
| 1901 |
pfvf->hw.lso_tsov4_idx = rsp->lso_tsov4_idx; |
| 1902 |
pfvf->hw.lso_tsov6_idx = rsp->lso_tsov6_idx; |
| 1903 |
pfvf->hw.cgx_links = rsp->cgx_links; |
| 1904 |
pfvf->hw.lbk_links = rsp->lbk_links; |
| 1905 |
pfvf->hw.tx_link = rsp->tx_link; |
| 1906 |
} |
| 1907 |
EXPORT_SYMBOL(mbox_handler_nix_lf_alloc); |
| 1908 |
|
| 1909 |
void mbox_handler_msix_offset(struct otx2_nic *pfvf, |
| 1910 |
struct msix_offset_rsp *rsp) |
| 1911 |
{ |
| 1912 |
pfvf->hw.npa_msixoff = rsp->npa_msixoff; |
| 1913 |
pfvf->hw.nix_msixoff = rsp->nix_msixoff; |
| 1914 |
} |
| 1915 |
EXPORT_SYMBOL(mbox_handler_msix_offset); |
| 1916 |
|
| 1917 |
void mbox_handler_nix_bp_enable(struct otx2_nic *pfvf, |
| 1918 |
struct nix_bp_cfg_rsp *rsp) |
| 1919 |
{ |
| 1920 |
int chan, chan_id; |
| 1921 |
|
| 1922 |
for (chan = 0; chan < rsp->chan_cnt; chan++) { |
| 1923 |
chan_id = ((rsp->chan_bpid[chan] >> 10) & 0x7F); |
| 1924 |
pfvf->bpid[chan_id] = rsp->chan_bpid[chan] & 0x3FF; |
| 1925 |
} |
| 1926 |
} |
| 1927 |
EXPORT_SYMBOL(mbox_handler_nix_bp_enable); |
| 1928 |
|
| 1929 |
void otx2_free_cints(struct otx2_nic *pfvf, int n) |
| 1930 |
{ |
| 1931 |
struct otx2_qset *qset = &pfvf->qset; |
| 1932 |
struct otx2_hw *hw = &pfvf->hw; |
| 1933 |
int irq, qidx; |
| 1934 |
|
| 1935 |
for (qidx = 0, irq = hw->nix_msixoff + NIX_LF_CINT_VEC_START; |
| 1936 |
qidx < n; |
| 1937 |
qidx++, irq++) { |
| 1938 |
int vector = pci_irq_vector(pfvf->pdev, irq); |
| 1939 |
|
| 1940 |
irq_set_affinity_hint(vector, NULL); |
| 1941 |
free_cpumask_var(hw->affinity_mask[irq]); |
| 1942 |
free_irq(vector, &qset->napi[qidx]); |
| 1943 |
} |
| 1944 |
} |
| 1945 |
EXPORT_SYMBOL(otx2_free_cints); |
| 1946 |
|
| 1947 |
void otx2_set_cints_affinity(struct otx2_nic *pfvf) |
| 1948 |
{ |
| 1949 |
struct otx2_hw *hw = &pfvf->hw; |
| 1950 |
int vec, cpu, irq, cint; |
| 1951 |
|
| 1952 |
vec = hw->nix_msixoff + NIX_LF_CINT_VEC_START; |
| 1953 |
cpu = cpumask_first(cpu_online_mask); |
| 1954 |
|
| 1955 |
/* CQ interrupts */ |
| 1956 |
for (cint = 0; cint < pfvf->hw.cint_cnt; cint++, vec++) { |
| 1957 |
if (!alloc_cpumask_var(&hw->affinity_mask[vec], GFP_KERNEL)) |
| 1958 |
return; |
| 1959 |
|
| 1960 |
cpumask_set_cpu(cpu, hw->affinity_mask[vec]); |
| 1961 |
|
| 1962 |
irq = pci_irq_vector(pfvf->pdev, vec); |
| 1963 |
irq_set_affinity_hint(irq, hw->affinity_mask[vec]); |
| 1964 |
|
| 1965 |
cpu = cpumask_next(cpu, cpu_online_mask); |
| 1966 |
if (unlikely(cpu >= nr_cpu_ids)) |
| 1967 |
cpu = 0; |
| 1968 |
} |
| 1969 |
} |
| 1970 |
|
| 1971 |
static u32 get_dwrr_mtu(struct otx2_nic *pfvf, struct nix_hw_info *hw) |
| 1972 |
{ |
| 1973 |
if (is_otx2_lbkvf(pfvf->pdev)) { |
| 1974 |
pfvf->hw.smq_link_type = SMQ_LINK_TYPE_LBK; |
| 1975 |
return hw->lbk_dwrr_mtu; |
| 1976 |
} |
| 1977 |
|
| 1978 |
pfvf->hw.smq_link_type = SMQ_LINK_TYPE_RPM; |
| 1979 |
return hw->rpm_dwrr_mtu; |
| 1980 |
} |
| 1981 |
|
| 1982 |
u16 otx2_get_max_mtu(struct otx2_nic *pfvf) |
| 1983 |
{ |
| 1984 |
struct nix_hw_info *rsp; |
| 1985 |
struct msg_req *req; |
| 1986 |
u16 max_mtu; |
| 1987 |
int rc; |
| 1988 |
|
| 1989 |
mutex_lock(&pfvf->mbox.lock); |
| 1990 |
|
| 1991 |
req = otx2_mbox_alloc_msg_nix_get_hw_info(&pfvf->mbox); |
| 1992 |
if (!req) { |
| 1993 |
rc = -ENOMEM; |
| 1994 |
goto out; |
| 1995 |
} |
| 1996 |
|
| 1997 |
rc = otx2_sync_mbox_msg(&pfvf->mbox); |
| 1998 |
if (!rc) { |
| 1999 |
rsp = (struct nix_hw_info *) |
| 2000 |
otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); |
| 2001 |
if (IS_ERR(rsp)) { |
| 2002 |
rc = PTR_ERR(rsp); |
| 2003 |
goto out; |
| 2004 |
} |
| 2005 |
|
| 2006 |
/* HW counts VLAN insertion bytes (8 for double tag) |
| 2007 |
* irrespective of whether SQE is requesting to insert VLAN |
| 2008 |
* in the packet or not. Hence these 8 bytes have to be |
| 2009 |
* discounted from max packet size otherwise HW will throw |
| 2010 |
* SMQ errors |
| 2011 |
*/ |
| 2012 |
max_mtu = rsp->max_mtu - 8 - OTX2_ETH_HLEN; |
| 2013 |
|
| 2014 |
/* Also save DWRR MTU, needed for DWRR weight calculation */ |
| 2015 |
pfvf->hw.dwrr_mtu = get_dwrr_mtu(pfvf, rsp); |
| 2016 |
if (!pfvf->hw.dwrr_mtu) |
| 2017 |
pfvf->hw.dwrr_mtu = 1; |
| 2018 |
} |
| 2019 |
|
| 2020 |
out: |
| 2021 |
mutex_unlock(&pfvf->mbox.lock); |
| 2022 |
if (rc) { |
| 2023 |
dev_warn(pfvf->dev, |
| 2024 |
"Failed to get MTU from hardware setting default value(1500)\n"); |
| 2025 |
max_mtu = 1500; |
| 2026 |
} |
| 2027 |
return max_mtu; |
| 2028 |
} |
| 2029 |
EXPORT_SYMBOL(otx2_get_max_mtu); |
| 2030 |
|
| 2031 |
int otx2_handle_ntuple_tc_features(struct net_device *netdev, netdev_features_t features) |
| 2032 |
{ |
| 2033 |
netdev_features_t changed = features ^ netdev->features; |
| 2034 |
struct otx2_nic *pfvf = netdev_priv(netdev); |
| 2035 |
bool ntuple = !!(features & NETIF_F_NTUPLE); |
| 2036 |
bool tc = !!(features & NETIF_F_HW_TC); |
| 2037 |
|
| 2038 |
if ((changed & NETIF_F_NTUPLE) && !ntuple) |
| 2039 |
otx2_destroy_ntuple_flows(pfvf); |
| 2040 |
|
| 2041 |
if ((changed & NETIF_F_NTUPLE) && ntuple) { |
| 2042 |
if (!pfvf->flow_cfg->max_flows) { |
| 2043 |
netdev_err(netdev, |
| 2044 |
"Can't enable NTUPLE, MCAM entries not allocated\n"); |
| 2045 |
return -EINVAL; |
| 2046 |
} |
| 2047 |
} |
| 2048 |
|
| 2049 |
if ((changed & NETIF_F_HW_TC) && !tc && |
| 2050 |
otx2_tc_flower_rule_cnt(pfvf)) { |
| 2051 |
netdev_err(netdev, "Can't disable TC hardware offload while flows are active\n"); |
| 2052 |
return -EBUSY; |
| 2053 |
} |
| 2054 |
|
| 2055 |
if ((changed & NETIF_F_NTUPLE) && ntuple && |
| 2056 |
otx2_tc_flower_rule_cnt(pfvf) && !(changed & NETIF_F_HW_TC)) { |
| 2057 |
netdev_err(netdev, |
| 2058 |
"Can't enable NTUPLE when TC flower offload is active, disable TC rules and retry\n"); |
| 2059 |
return -EINVAL; |
| 2060 |
} |
| 2061 |
|
| 2062 |
return 0; |
| 2063 |
} |
| 2064 |
EXPORT_SYMBOL(otx2_handle_ntuple_tc_features); |
| 2065 |
|
| 2066 |
int otx2_set_hw_capabilities(struct otx2_nic *pfvf) |
| 2067 |
{ |
| 2068 |
struct mbox *mbox = &pfvf->mbox; |
| 2069 |
struct otx2_hw *hw = &pfvf->hw; |
| 2070 |
struct get_hw_cap_rsp *rsp; |
| 2071 |
struct msg_req *req; |
| 2072 |
int ret = -ENOMEM; |
| 2073 |
|
| 2074 |
mutex_lock(&mbox->lock); |
| 2075 |
|
| 2076 |
req = otx2_mbox_alloc_msg_get_hw_cap(mbox); |
| 2077 |
if (!req) |
| 2078 |
goto fail; |
| 2079 |
|
| 2080 |
ret = otx2_sync_mbox_msg(mbox); |
| 2081 |
if (ret) |
| 2082 |
goto fail; |
| 2083 |
|
| 2084 |
rsp = (struct get_hw_cap_rsp *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, |
| 2085 |
0, &req->hdr); |
| 2086 |
if (IS_ERR(rsp)) { |
| 2087 |
ret = -EINVAL; |
| 2088 |
goto fail; |
| 2089 |
} |
| 2090 |
|
| 2091 |
if (rsp->hw_caps & HW_CAP_MACSEC) |
| 2092 |
__set_bit(CN10K_HW_MACSEC, &hw->cap_flag); |
| 2093 |
|
| 2094 |
mutex_unlock(&mbox->lock); |
| 2095 |
|
| 2096 |
return 0; |
| 2097 |
fail: |
| 2098 |
dev_err(pfvf->dev, "Cannot get MACSEC capability from AF\n"); |
| 2099 |
mutex_unlock(&mbox->lock); |
| 2100 |
return ret; |
| 2101 |
} |
| 2102 |
|
| 2103 |
#define M(_name, _id, _fn_name, _req_type, _rsp_type) \ |
| 2104 |
int __weak \ |
| 2105 |
otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \ |
| 2106 |
struct _req_type *req, \ |
| 2107 |
struct _rsp_type *rsp) \ |
| 2108 |
{ \ |
| 2109 |
/* Nothing to do here */ \ |
| 2110 |
return 0; \ |
| 2111 |
} \ |
| 2112 |
EXPORT_SYMBOL(otx2_mbox_up_handler_ ## _fn_name); |
| 2113 |
MBOX_UP_CGX_MESSAGES |
| 2114 |
MBOX_UP_MCS_MESSAGES |
| 2115 |
#undef M |
| 2116 |
|
| 2117 |
dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf, |
| 2118 |
struct sk_buff *skb, int seg, int *len) |
| 2119 |
{ |
| 2120 |
enum dma_data_direction dir = DMA_TO_DEVICE; |
| 2121 |
const skb_frag_t *frag; |
| 2122 |
struct page *page; |
| 2123 |
int offset; |
| 2124 |
|
| 2125 |
/* Crypto hardware need write permission for ipsec crypto offload */ |
| 2126 |
if (unlikely(xfrm_offload(skb))) { |
| 2127 |
dir = DMA_BIDIRECTIONAL; |
| 2128 |
skb = skb_unshare(skb, GFP_ATOMIC); |
| 2129 |
} |
| 2130 |
|
| 2131 |
/* First segment is always skb->data */ |
| 2132 |
if (!seg) { |
| 2133 |
page = virt_to_page(skb->data); |
| 2134 |
offset = offset_in_page(skb->data); |
| 2135 |
*len = skb_headlen(skb); |
| 2136 |
} else { |
| 2137 |
frag = &skb_shinfo(skb)->frags[seg - 1]; |
| 2138 |
page = skb_frag_page(frag); |
| 2139 |
offset = skb_frag_off(frag); |
| 2140 |
*len = skb_frag_size(frag); |
| 2141 |
} |
| 2142 |
return otx2_dma_map_page(pfvf, page, offset, *len, dir); |
| 2143 |
} |
| 2144 |
|
| 2145 |
void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg) |
| 2146 |
{ |
| 2147 |
enum dma_data_direction dir = DMA_TO_DEVICE; |
| 2148 |
struct sk_buff *skb = NULL; |
| 2149 |
int seg; |
| 2150 |
|
| 2151 |
skb = (struct sk_buff *)sg->skb; |
| 2152 |
if (unlikely(xfrm_offload(skb))) |
| 2153 |
dir = DMA_BIDIRECTIONAL; |
| 2154 |
|
| 2155 |
for (seg = 0; seg < sg->num_segs; seg++) { |
| 2156 |
otx2_dma_unmap_page(pfvf, sg->dma_addr[seg], |
| 2157 |
sg->size[seg], dir); |
| 2158 |
} |
| 2159 |
sg->num_segs = 0; |
| 2160 |
} |