Debian Codegraph @

drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c

raw
1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell IPSEC offload driver
3 *
4 * Copyright (C) 2024 Marvell.
5 */
6
7 #include <net/xfrm.h>
8 #include <linux/netdevice.h>
9 #include <linux/bitfield.h>
10 #include <crypto/aead.h>
11 #include <crypto/gcm.h>
12
13 #include "otx2_common.h"
14 #include "otx2_struct.h"
15 #include "cn10k_ipsec.h"
16
17 static bool is_dev_support_ipsec_offload(struct pci_dev *pdev)
18 {
19 return is_dev_cn10ka_b0(pdev) || is_dev_cn10kb(pdev);
20 }
21
22 static bool cn10k_cpt_device_set_inuse(struct otx2_nic *pf)
23 {
24 enum cn10k_cpt_hw_state_e state;
25
26 while (true) {
27 state = atomic_cmpxchg(&pf->ipsec.cpt_state,
28 CN10K_CPT_HW_AVAILABLE,
29 CN10K_CPT_HW_IN_USE);
30 if (state == CN10K_CPT_HW_AVAILABLE)
31 return true;
32 if (state == CN10K_CPT_HW_UNAVAILABLE)
33 return false;
34
35 mdelay(1);
36 }
37 }
38
39 static void cn10k_cpt_device_set_available(struct otx2_nic *pf)
40 {
41 atomic_set(&pf->ipsec.cpt_state, CN10K_CPT_HW_AVAILABLE);
42 }
43
44 static void cn10k_cpt_device_set_unavailable(struct otx2_nic *pf)
45 {
46 atomic_set(&pf->ipsec.cpt_state, CN10K_CPT_HW_UNAVAILABLE);
47 }
48
49 static int cn10k_outb_cptlf_attach(struct otx2_nic *pf)
50 {
51 struct rsrc_attach *attach;
52 int ret = -ENOMEM;
53
54 mutex_lock(&pf->mbox.lock);
55 /* Get memory to put this msg */
56 attach = otx2_mbox_alloc_msg_attach_resources(&pf->mbox);
57 if (!attach)
58 goto unlock;
59
60 attach->cptlfs = true;
61 attach->modify = true;
62
63 /* Send attach request to AF */
64 ret = otx2_sync_mbox_msg(&pf->mbox);
65
66 unlock:
67 mutex_unlock(&pf->mbox.lock);
68 return ret;
69 }
70
71 static int cn10k_outb_cptlf_detach(struct otx2_nic *pf)
72 {
73 struct rsrc_detach *detach;
74 int ret = -ENOMEM;
75
76 mutex_lock(&pf->mbox.lock);
77 detach = otx2_mbox_alloc_msg_detach_resources(&pf->mbox);
78 if (!detach)
79 goto unlock;
80
81 detach->partial = true;
82 detach->cptlfs = true;
83
84 /* Send detach request to AF */
85 ret = otx2_sync_mbox_msg(&pf->mbox);
86
87 unlock:
88 mutex_unlock(&pf->mbox.lock);
89 return ret;
90 }
91
92 static int cn10k_outb_cptlf_alloc(struct otx2_nic *pf)
93 {
94 struct cpt_lf_alloc_req_msg *req;
95 int ret = -ENOMEM;
96
97 mutex_lock(&pf->mbox.lock);
98 req = otx2_mbox_alloc_msg_cpt_lf_alloc(&pf->mbox);
99 if (!req)
100 goto unlock;
101
102 /* PF function */
103 req->nix_pf_func = pf->pcifunc;
104 /* Enable SE-IE Engine Group */
105 req->eng_grpmsk = 1 << CN10K_DEF_CPT_IPSEC_EGRP;
106
107 ret = otx2_sync_mbox_msg(&pf->mbox);
108
109 unlock:
110 mutex_unlock(&pf->mbox.lock);
111 return ret;
112 }
113
114 static void cn10k_outb_cptlf_free(struct otx2_nic *pf)
115 {
116 mutex_lock(&pf->mbox.lock);
117 otx2_mbox_alloc_msg_cpt_lf_free(&pf->mbox);
118 otx2_sync_mbox_msg(&pf->mbox);
119 mutex_unlock(&pf->mbox.lock);
120 }
121
122 static int cn10k_outb_cptlf_config(struct otx2_nic *pf)
123 {
124 struct cpt_inline_ipsec_cfg_msg *req;
125 int ret = -ENOMEM;
126
127 mutex_lock(&pf->mbox.lock);
128 req = otx2_mbox_alloc_msg_cpt_inline_ipsec_cfg(&pf->mbox);
129 if (!req)
130 goto unlock;
131
132 req->dir = CPT_INLINE_OUTBOUND;
133 req->enable = 1;
134 req->nix_pf_func = pf->pcifunc;
135 ret = otx2_sync_mbox_msg(&pf->mbox);
136 unlock:
137 mutex_unlock(&pf->mbox.lock);
138 return ret;
139 }
140
141 static void cn10k_outb_cptlf_iq_enable(struct otx2_nic *pf)
142 {
143 u64 reg_val;
144
145 /* Set Execution Enable of instruction queue */
146 reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
147 reg_val |= BIT_ULL(16);
148 otx2_write64(pf, CN10K_CPT_LF_INPROG, reg_val);
149
150 /* Set iqueue's enqueuing */
151 reg_val = otx2_read64(pf, CN10K_CPT_LF_CTL);
152 reg_val |= BIT_ULL(0);
153 otx2_write64(pf, CN10K_CPT_LF_CTL, reg_val);
154 }
155
156 static void cn10k_outb_cptlf_iq_disable(struct otx2_nic *pf)
157 {
158 u32 inflight, grb_cnt, gwb_cnt;
159 u32 nq_ptr, dq_ptr;
160 int timeout = 20;
161 u64 reg_val;
162 int cnt;
163
164 /* Disable instructions enqueuing */
165 otx2_write64(pf, CN10K_CPT_LF_CTL, 0ull);
166
167 /* Wait for instruction queue to become empty.
168 * CPT_LF_INPROG.INFLIGHT count is zero
169 */
170 do {
171 reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
172 inflight = FIELD_GET(CPT_LF_INPROG_INFLIGHT, reg_val);
173 if (!inflight)
174 break;
175
176 usleep_range(10000, 20000);
177 if (timeout-- < 0) {
178 netdev_err(pf->netdev, "Timeout to cleanup CPT IQ\n");
179 break;
180 }
181 } while (1);
182
183 /* Disable executions in the LF's queue,
184 * the queue should be empty at this point
185 */
186 reg_val &= ~BIT_ULL(16);
187 otx2_write64(pf, CN10K_CPT_LF_INPROG, reg_val);
188
189 /* Wait for instruction queue to become empty */
190 cnt = 0;
191 do {
192 reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
193 if (reg_val & BIT_ULL(31))
194 cnt = 0;
195 else
196 cnt++;
197 reg_val = otx2_read64(pf, CN10K_CPT_LF_Q_GRP_PTR);
198 nq_ptr = FIELD_GET(CPT_LF_Q_GRP_PTR_DQ_PTR, reg_val);
199 dq_ptr = FIELD_GET(CPT_LF_Q_GRP_PTR_DQ_PTR, reg_val);
200 } while ((cnt < 10) && (nq_ptr != dq_ptr));
201
202 cnt = 0;
203 do {
204 reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
205 inflight = FIELD_GET(CPT_LF_INPROG_INFLIGHT, reg_val);
206 grb_cnt = FIELD_GET(CPT_LF_INPROG_GRB_CNT, reg_val);
207 gwb_cnt = FIELD_GET(CPT_LF_INPROG_GWB_CNT, reg_val);
208 if (inflight == 0 && gwb_cnt < 40 &&
209 (grb_cnt == 0 || grb_cnt == 40))
210 cnt++;
211 else
212 cnt = 0;
213 } while (cnt < 10);
214 }
215
216 /* Allocate memory for CPT outbound Instruction queue.
217 * Instruction queue memory format is:
218 * -----------------------------
219 * | Instruction Group memory |
220 * | (CPT_LF_Q_SIZE[SIZE_DIV40] |
221 * | x 16 Bytes) |
222 * | |
223 * ----------------------------- <-- CPT_LF_Q_BASE[ADDR]
224 * | Flow Control (128 Bytes) |
225 * | |
226 * -----------------------------
227 * | Instruction Memory |
228 * | (CPT_LF_Q_SIZE[SIZE_DIV40] |
229 * | × 40 × 64 bytes) |
230 * | |
231 * -----------------------------
232 */
233 static int cn10k_outb_cptlf_iq_alloc(struct otx2_nic *pf)
234 {
235 struct cn10k_cpt_inst_queue *iq = &pf->ipsec.iq;
236
237 iq->size = CN10K_CPT_INST_QLEN_BYTES + CN10K_CPT_Q_FC_LEN +
238 CN10K_CPT_INST_GRP_QLEN_BYTES + OTX2_ALIGN;
239
240 iq->real_vaddr = dma_alloc_coherent(pf->dev, iq->size,
241 &iq->real_dma_addr, GFP_KERNEL);
242 if (!iq->real_vaddr)
243 return -ENOMEM;
244
245 /* iq->vaddr/dma_addr points to Flow Control location */
246 iq->vaddr = iq->real_vaddr + CN10K_CPT_INST_GRP_QLEN_BYTES;
247 iq->dma_addr = iq->real_dma_addr + CN10K_CPT_INST_GRP_QLEN_BYTES;
248
249 /* Align pointers */
250 iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_ALIGN);
251 iq->dma_addr = PTR_ALIGN(iq->dma_addr, OTX2_ALIGN);
252 return 0;
253 }
254
255 static void cn10k_outb_cptlf_iq_free(struct otx2_nic *pf)
256 {
257 struct cn10k_cpt_inst_queue *iq = &pf->ipsec.iq;
258
259 if (iq->real_vaddr)
260 dma_free_coherent(pf->dev, iq->size, iq->real_vaddr,
261 iq->real_dma_addr);
262
263 iq->real_vaddr = NULL;
264 iq->vaddr = NULL;
265 }
266
267 static int cn10k_outb_cptlf_iq_init(struct otx2_nic *pf)
268 {
269 u64 reg_val;
270 int ret;
271
272 /* Allocate Memory for CPT IQ */
273 ret = cn10k_outb_cptlf_iq_alloc(pf);
274 if (ret)
275 return ret;
276
277 /* Disable IQ */
278 cn10k_outb_cptlf_iq_disable(pf);
279
280 /* Set IQ base address */
281 otx2_write64(pf, CN10K_CPT_LF_Q_BASE, pf->ipsec.iq.dma_addr);
282
283 /* Set IQ size */
284 reg_val = FIELD_PREP(CPT_LF_Q_SIZE_DIV40, CN10K_CPT_SIZE_DIV40 +
285 CN10K_CPT_EXTRA_SIZE_DIV40);
286 otx2_write64(pf, CN10K_CPT_LF_Q_SIZE, reg_val);
287
288 return 0;
289 }
290
291 static int cn10k_outb_cptlf_init(struct otx2_nic *pf)
292 {
293 int ret;
294
295 /* Initialize CPTLF Instruction Queue (IQ) */
296 ret = cn10k_outb_cptlf_iq_init(pf);
297 if (ret)
298 return ret;
299
300 /* Configure CPTLF for outbound ipsec offload */
301 ret = cn10k_outb_cptlf_config(pf);
302 if (ret)
303 goto iq_clean;
304
305 /* Enable CPTLF IQ */
306 cn10k_outb_cptlf_iq_enable(pf);
307 return 0;
308 iq_clean:
309 cn10k_outb_cptlf_iq_free(pf);
310 return ret;
311 }
312
313 static int cn10k_outb_cpt_init(struct net_device *netdev)
314 {
315 struct otx2_nic *pf = netdev_priv(netdev);
316 int ret;
317
318 /* Attach a CPT LF for outbound ipsec offload */
319 ret = cn10k_outb_cptlf_attach(pf);
320 if (ret)
321 return ret;
322
323 /* Allocate a CPT LF for outbound ipsec offload */
324 ret = cn10k_outb_cptlf_alloc(pf);
325 if (ret)
326 goto detach;
327
328 /* Initialize the CPTLF for outbound ipsec offload */
329 ret = cn10k_outb_cptlf_init(pf);
330 if (ret)
331 goto lf_free;
332
333 pf->ipsec.io_addr = (__force u64)otx2_get_regaddr(pf,
334 CN10K_CPT_LF_NQX(0));
335
336 /* Set ipsec offload enabled for this device */
337 pf->flags |= OTX2_FLAG_IPSEC_OFFLOAD_ENABLED;
338
339 cn10k_cpt_device_set_available(pf);
340 return 0;
341
342 lf_free:
343 cn10k_outb_cptlf_free(pf);
344 detach:
345 cn10k_outb_cptlf_detach(pf);
346 return ret;
347 }
348
349 static int cn10k_outb_cpt_clean(struct otx2_nic *pf)
350 {
351 int ret;
352
353 if (!cn10k_cpt_device_set_inuse(pf)) {
354 netdev_err(pf->netdev, "CPT LF device unavailable\n");
355 return -ENODEV;
356 }
357
358 /* Set ipsec offload disabled for this device */
359 pf->flags &= ~OTX2_FLAG_IPSEC_OFFLOAD_ENABLED;
360
361 /* Disable CPTLF Instruction Queue (IQ) */
362 cn10k_outb_cptlf_iq_disable(pf);
363
364 /* Set IQ base address and size to 0 */
365 otx2_write64(pf, CN10K_CPT_LF_Q_BASE, 0);
366 otx2_write64(pf, CN10K_CPT_LF_Q_SIZE, 0);
367
368 /* Free CPTLF IQ */
369 cn10k_outb_cptlf_iq_free(pf);
370
371 /* Free and detach CPT LF */
372 cn10k_outb_cptlf_free(pf);
373 ret = cn10k_outb_cptlf_detach(pf);
374 if (ret)
375 netdev_err(pf->netdev, "Failed to detach CPT LF\n");
376
377 cn10k_cpt_device_set_unavailable(pf);
378 return ret;
379 }
380
381 static void cn10k_cpt_inst_flush(struct otx2_nic *pf, struct cpt_inst_s *inst,
382 u64 size)
383 {
384 struct otx2_lmt_info *lmt_info;
385 u64 val = 0, tar_addr = 0;
386
387 lmt_info = per_cpu_ptr(pf->hw.lmt_info, smp_processor_id());
388 /* FIXME: val[0:10] LMT_ID.
389 * [12:15] no of LMTST - 1 in the burst.
390 * [19:63] data size of each LMTST in the burst except first.
391 */
392 val = (lmt_info->lmt_id & 0x7FF);
393 /* Target address for LMTST flush tells HW how many 128bit
394 * words are present.
395 * tar_addr[6:4] size of first LMTST - 1 in units of 128b.
396 */
397 tar_addr |= pf->ipsec.io_addr | (((size / 16) - 1) & 0x7) << 4;
398 dma_wmb();
399 memcpy((u64 *)lmt_info->lmt_addr, inst, size);
400 cn10k_lmt_flush(val, tar_addr);
401 }
402
403 static int cn10k_wait_for_cpt_respose(struct otx2_nic *pf,
404 struct cpt_res_s *res)
405 {
406 unsigned long timeout = jiffies + msecs_to_jiffies(100);
407 u64 *completion_ptr = (u64 *)res;
408
409 do {
410 if (time_after(jiffies, timeout)) {
411 netdev_err(pf->netdev, "CPT response timeout\n");
412 return -EBUSY;
413 }
414 } while ((READ_ONCE(*completion_ptr) & CN10K_CPT_COMP_E_MASK) ==
415 CN10K_CPT_COMP_E_NOTDONE);
416
417 if (!(res->compcode == CN10K_CPT_COMP_E_GOOD ||
418 res->compcode == CN10K_CPT_COMP_E_WARN) || res->uc_compcode) {
419 netdev_err(pf->netdev, "compcode=%x doneint=%x\n",
420 res->compcode, res->doneint);
421 netdev_err(pf->netdev, "uc_compcode=%x uc_info=%llx esn=%llx\n",
422 res->uc_compcode, (u64)res->uc_info, res->esn);
423 }
424 return 0;
425 }
426
427 static int cn10k_outb_write_sa(struct otx2_nic *pf, struct qmem *sa_info)
428 {
429 dma_addr_t res_iova, dptr_iova, sa_iova;
430 struct cn10k_tx_sa_s *sa_dptr;
431 struct cpt_inst_s inst = {};
432 struct cpt_res_s *res;
433 u32 sa_size, off;
434 u64 *sptr, *dptr;
435 u64 reg_val;
436 int ret;
437
438 sa_iova = sa_info->iova;
439 if (!sa_iova)
440 return -EINVAL;
441
442 res = dma_alloc_coherent(pf->dev, sizeof(struct cpt_res_s),
443 &res_iova, GFP_ATOMIC);
444 if (!res)
445 return -ENOMEM;
446
447 sa_size = sizeof(struct cn10k_tx_sa_s);
448 sa_dptr = dma_alloc_coherent(pf->dev, sa_size, &dptr_iova, GFP_ATOMIC);
449 if (!sa_dptr) {
450 dma_free_coherent(pf->dev, sizeof(struct cpt_res_s), res,
451 res_iova);
452 return -ENOMEM;
453 }
454
455 sptr = (__force u64 *)sa_info->base;
456 dptr = (__force u64 *)sa_dptr;
457 for (off = 0; off < (sa_size / 8); off++)
458 *(dptr + off) = (__force u64)cpu_to_be64(*(sptr + off));
459
460 res->compcode = CN10K_CPT_COMP_E_NOTDONE;
461 inst.res_addr = res_iova;
462 inst.dptr = (u64)dptr_iova;
463 inst.param2 = sa_size >> 3;
464 inst.dlen = sa_size;
465 inst.opcode_major = CN10K_IPSEC_MAJOR_OP_WRITE_SA;
466 inst.opcode_minor = CN10K_IPSEC_MINOR_OP_WRITE_SA;
467 inst.cptr = sa_iova;
468 inst.ctx_val = 1;
469 inst.egrp = CN10K_DEF_CPT_IPSEC_EGRP;
470
471 /* Check if CPT-LF available */
472 if (!cn10k_cpt_device_set_inuse(pf)) {
473 ret = -ENODEV;
474 goto free_mem;
475 }
476
477 cn10k_cpt_inst_flush(pf, &inst, sizeof(struct cpt_inst_s));
478 dma_wmb();
479 ret = cn10k_wait_for_cpt_respose(pf, res);
480 if (ret)
481 goto set_available;
482
483 /* Trigger CTX flush to write dirty data back to DRAM */
484 reg_val = FIELD_PREP(CPT_LF_CTX_FLUSH_CPTR, sa_iova >> 7);
485 otx2_write64(pf, CN10K_CPT_LF_CTX_FLUSH, reg_val);
486
487 set_available:
488 cn10k_cpt_device_set_available(pf);
489 free_mem:
490 dma_free_coherent(pf->dev, sa_size, sa_dptr, dptr_iova);
491 dma_free_coherent(pf->dev, sizeof(struct cpt_res_s), res, res_iova);
492 return ret;
493 }
494
495 static int cn10k_ipsec_get_hw_ctx_offset(void)
496 {
497 /* Offset on Hardware-context offset in word */
498 return (offsetof(struct cn10k_tx_sa_s, hw_ctx) / sizeof(u64)) & 0x7F;
499 }
500
501 static int cn10k_ipsec_get_ctx_push_size(void)
502 {
503 /* Context push size is round up and in multiple of 8 Byte */
504 return (roundup(offsetof(struct cn10k_tx_sa_s, hw_ctx), 8) / 8) & 0x7F;
505 }
506
507 static int cn10k_ipsec_get_aes_key_len(int key_len)
508 {
509 /* key_len is aes key length in bytes */
510 switch (key_len) {
511 case 16:
512 return CN10K_IPSEC_SA_AES_KEY_LEN_128;
513 case 24:
514 return CN10K_IPSEC_SA_AES_KEY_LEN_192;
515 default:
516 return CN10K_IPSEC_SA_AES_KEY_LEN_256;
517 }
518 }
519
520 static void cn10k_outb_prepare_sa(struct xfrm_state *x,
521 struct cn10k_tx_sa_s *sa_entry)
522 {
523 int key_len = (x->aead->alg_key_len + 7) / 8;
524 struct net_device *netdev = x->xso.dev;
525 u8 *key = x->aead->alg_key;
526 struct otx2_nic *pf;
527 u32 *tmp_salt;
528 u64 *tmp_key;
529 int idx;
530
531 memset(sa_entry, 0, sizeof(struct cn10k_tx_sa_s));
532
533 /* context size, 128 Byte aligned up */
534 pf = netdev_priv(netdev);
535 sa_entry->ctx_size = (pf->ipsec.sa_size / OTX2_ALIGN) & 0xF;
536 sa_entry->hw_ctx_off = cn10k_ipsec_get_hw_ctx_offset();
537 sa_entry->ctx_push_size = cn10k_ipsec_get_ctx_push_size();
538
539 /* Ucode to skip two words of CPT_CTX_HW_S */
540 sa_entry->ctx_hdr_size = 1;
541
542 /* Allow Atomic operation (AOP) */
543 sa_entry->aop_valid = 1;
544
545 /* Outbound, ESP TRANSPORT/TUNNEL Mode, AES-GCM with */
546 sa_entry->sa_dir = CN10K_IPSEC_SA_DIR_OUTB;
547 sa_entry->ipsec_protocol = CN10K_IPSEC_SA_IPSEC_PROTO_ESP;
548 sa_entry->enc_type = CN10K_IPSEC_SA_ENCAP_TYPE_AES_GCM;
549 sa_entry->iv_src = CN10K_IPSEC_SA_IV_SRC_PACKET;
550 if (x->props.mode == XFRM_MODE_TUNNEL)
551 sa_entry->ipsec_mode = CN10K_IPSEC_SA_IPSEC_MODE_TUNNEL;
552 else
553 sa_entry->ipsec_mode = CN10K_IPSEC_SA_IPSEC_MODE_TRANSPORT;
554
555 /* Last 4 bytes are salt */
556 key_len -= 4;
557 sa_entry->aes_key_len = cn10k_ipsec_get_aes_key_len(key_len);
558 memcpy(sa_entry->cipher_key, key, key_len);
559 tmp_key = (u64 *)sa_entry->cipher_key;
560
561 for (idx = 0; idx < key_len / 8; idx++)
562 tmp_key[idx] = (__force u64)cpu_to_be64(tmp_key[idx]);
563
564 memcpy(&sa_entry->iv_gcm_salt, key + key_len, 4);
565 tmp_salt = (u32 *)&sa_entry->iv_gcm_salt;
566 *tmp_salt = (__force u32)cpu_to_be32(*tmp_salt);
567
568 /* Write SA context data to memory before enabling */
569 wmb();
570
571 /* Enable SA */
572 sa_entry->sa_valid = 1;
573 }
574
575 static int cn10k_ipsec_validate_state(struct xfrm_state *x,
576 struct netlink_ext_ack *extack)
577 {
578 if (x->props.aalgo != SADB_AALG_NONE) {
579 NL_SET_ERR_MSG_MOD(extack,
580 "Cannot offload authenticated xfrm states");
581 return -EINVAL;
582 }
583 if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) {
584 NL_SET_ERR_MSG_MOD(extack,
585 "Only AES-GCM-ICV16 xfrm state may be offloaded");
586 return -EINVAL;
587 }
588 if (x->props.calgo != SADB_X_CALG_NONE) {
589 NL_SET_ERR_MSG_MOD(extack,
590 "Cannot offload compressed xfrm states");
591 return -EINVAL;
592 }
593 if (x->props.flags & XFRM_STATE_ESN) {
594 NL_SET_ERR_MSG_MOD(extack, "Cannot offload ESN xfrm states");
595 return -EINVAL;
596 }
597 if (x->props.family != AF_INET && x->props.family != AF_INET6) {
598 NL_SET_ERR_MSG_MOD(extack,
599 "Only IPv4/v6 xfrm states may be offloaded");
600 return -EINVAL;
601 }
602 if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) {
603 NL_SET_ERR_MSG_MOD(extack,
604 "Cannot offload other than crypto-mode");
605 return -EINVAL;
606 }
607 if (x->props.mode != XFRM_MODE_TRANSPORT &&
608 x->props.mode != XFRM_MODE_TUNNEL) {
609 NL_SET_ERR_MSG_MOD(extack,
610 "Only tunnel/transport xfrm states may be offloaded");
611 return -EINVAL;
612 }
613 if (x->id.proto != IPPROTO_ESP) {
614 NL_SET_ERR_MSG_MOD(extack,
615 "Only ESP xfrm state may be offloaded");
616 return -EINVAL;
617 }
618 if (x->encap) {
619 NL_SET_ERR_MSG_MOD(extack,
620 "Encapsulated xfrm state may not be offloaded");
621 return -EINVAL;
622 }
623 if (!x->aead) {
624 NL_SET_ERR_MSG_MOD(extack,
625 "Cannot offload xfrm states without aead");
626 return -EINVAL;
627 }
628
629 if (x->aead->alg_icv_len != 128) {
630 NL_SET_ERR_MSG_MOD(extack,
631 "Cannot offload xfrm states with AEAD ICV length other than 128bit");
632 return -EINVAL;
633 }
634 if (x->aead->alg_key_len != 128 + 32 &&
635 x->aead->alg_key_len != 192 + 32 &&
636 x->aead->alg_key_len != 256 + 32) {
637 NL_SET_ERR_MSG_MOD(extack,
638 "Cannot offload xfrm states with AEAD key length other than 128/192/256bit");
639 return -EINVAL;
640 }
641 if (x->tfcpad) {
642 NL_SET_ERR_MSG_MOD(extack,
643 "Cannot offload xfrm states with tfc padding");
644 return -EINVAL;
645 }
646 if (!x->geniv) {
647 NL_SET_ERR_MSG_MOD(extack,
648 "Cannot offload xfrm states without geniv");
649 return -EINVAL;
650 }
651 if (strcmp(x->geniv, "seqiv")) {
652 NL_SET_ERR_MSG_MOD(extack,
653 "Cannot offload xfrm states with geniv other than seqiv");
654 return -EINVAL;
655 }
656 return 0;
657 }
658
659 static int cn10k_ipsec_inb_add_state(struct xfrm_state *x,
660 struct netlink_ext_ack *extack)
661 {
662 NL_SET_ERR_MSG_MOD(extack, "xfrm inbound offload not supported");
663 return -EOPNOTSUPP;
664 }
665
666 static int cn10k_ipsec_outb_add_state(struct net_device *dev,
667 struct xfrm_state *x,
668 struct netlink_ext_ack *extack)
669 {
670 struct cn10k_tx_sa_s *sa_entry;
671 struct qmem *sa_info;
672 struct otx2_nic *pf;
673 int err;
674
675 err = cn10k_ipsec_validate_state(x, extack);
676 if (err)
677 return err;
678
679 pf = netdev_priv(dev);
680
681 err = qmem_alloc(pf->dev, &sa_info, pf->ipsec.sa_size, OTX2_ALIGN);
682 if (err)
683 return err;
684
685 sa_entry = (struct cn10k_tx_sa_s *)sa_info->base;
686 cn10k_outb_prepare_sa(x, sa_entry);
687
688 err = cn10k_outb_write_sa(pf, sa_info);
689 if (err) {
690 NL_SET_ERR_MSG_MOD(extack, "Error writing outbound SA");
691 qmem_free(pf->dev, sa_info);
692 return err;
693 }
694
695 x->xso.offload_handle = (unsigned long)sa_info;
696 /* Enable static branch when first SA setup */
697 if (!pf->ipsec.outb_sa_count)
698 static_branch_enable(&cn10k_ipsec_sa_enabled);
699 pf->ipsec.outb_sa_count++;
700 return 0;
701 }
702
703 static int cn10k_ipsec_add_state(struct net_device *dev,
704 struct xfrm_state *x,
705 struct netlink_ext_ack *extack)
706 {
707 if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
708 return cn10k_ipsec_inb_add_state(x, extack);
709 else
710 return cn10k_ipsec_outb_add_state(dev, x, extack);
711 }
712
713 static void cn10k_ipsec_del_state(struct net_device *dev, struct xfrm_state *x)
714 {
715 struct cn10k_tx_sa_s *sa_entry;
716 struct qmem *sa_info;
717 struct otx2_nic *pf;
718 int err;
719
720 if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
721 return;
722
723 pf = netdev_priv(dev);
724
725 sa_info = (struct qmem *)x->xso.offload_handle;
726 sa_entry = (struct cn10k_tx_sa_s *)sa_info->base;
727 memset(sa_entry, 0, sizeof(struct cn10k_tx_sa_s));
728 /* Disable SA in CPT h/w */
729 sa_entry->ctx_push_size = cn10k_ipsec_get_ctx_push_size();
730 sa_entry->ctx_size = (pf->ipsec.sa_size / OTX2_ALIGN) & 0xF;
731 sa_entry->aop_valid = 1;
732
733 err = cn10k_outb_write_sa(pf, sa_info);
734 if (err)
735 netdev_err(dev, "Error (%d) deleting SA\n", err);
736
737 x->xso.offload_handle = 0;
738 qmem_free(pf->dev, sa_info);
739
740 /* If no more SA's then update netdev feature for potential change
741 * in NETIF_F_HW_ESP.
742 */
743 if (!--pf->ipsec.outb_sa_count)
744 queue_work(pf->ipsec.sa_workq, &pf->ipsec.sa_work);
745 }
746
747 static const struct xfrmdev_ops cn10k_ipsec_xfrmdev_ops = {
748 .xdo_dev_state_add = cn10k_ipsec_add_state,
749 .xdo_dev_state_delete = cn10k_ipsec_del_state,
750 };
751
752 static void cn10k_ipsec_sa_wq_handler(struct work_struct *work)
753 {
754 struct cn10k_ipsec *ipsec = container_of(work, struct cn10k_ipsec,
755 sa_work);
756 struct otx2_nic *pf = container_of(ipsec, struct otx2_nic, ipsec);
757
758 /* Disable static branch when no more SA enabled */
759 static_branch_disable(&cn10k_ipsec_sa_enabled);
760 rtnl_lock();
761 netdev_update_features(pf->netdev);
762 rtnl_unlock();
763 }
764
765 int cn10k_ipsec_ethtool_init(struct net_device *netdev, bool enable)
766 {
767 struct otx2_nic *pf = netdev_priv(netdev);
768
769 /* IPsec offload supported on cn10k */
770 if (!is_dev_support_ipsec_offload(pf->pdev))
771 return -EOPNOTSUPP;
772
773 /* Initialize CPT for outbound ipsec offload */
774 if (enable)
775 return cn10k_outb_cpt_init(netdev);
776
777 /* Don't do CPT cleanup if SA installed */
778 if (pf->ipsec.outb_sa_count) {
779 netdev_err(pf->netdev, "SA installed on this device\n");
780 return -EBUSY;
781 }
782
783 return cn10k_outb_cpt_clean(pf);
784 }
785
786 int cn10k_ipsec_init(struct net_device *netdev)
787 {
788 struct otx2_nic *pf = netdev_priv(netdev);
789 u32 sa_size;
790
791 if (!is_dev_support_ipsec_offload(pf->pdev))
792 return 0;
793
794 /* Each SA entry size is 128 Byte round up in size */
795 sa_size = sizeof(struct cn10k_tx_sa_s) % OTX2_ALIGN ?
796 (sizeof(struct cn10k_tx_sa_s) / OTX2_ALIGN + 1) *
797 OTX2_ALIGN : sizeof(struct cn10k_tx_sa_s);
798 pf->ipsec.sa_size = sa_size;
799
800 INIT_WORK(&pf->ipsec.sa_work, cn10k_ipsec_sa_wq_handler);
801 pf->ipsec.sa_workq = alloc_workqueue("cn10k_ipsec_sa_workq",
802 WQ_PERCPU, 0);
803 if (!pf->ipsec.sa_workq) {
804 netdev_err(pf->netdev, "SA alloc workqueue failed\n");
805 return -ENOMEM;
806 }
807
808 /* Set xfrm device ops */
809 netdev->xfrmdev_ops = &cn10k_ipsec_xfrmdev_ops;
810 netdev->hw_features |= NETIF_F_HW_ESP;
811 netdev->hw_enc_features |= NETIF_F_HW_ESP;
812
813 cn10k_cpt_device_set_unavailable(pf);
814 return 0;
815 }
816 EXPORT_SYMBOL(cn10k_ipsec_init);
817
818 void cn10k_ipsec_clean(struct otx2_nic *pf)
819 {
820 if (!is_dev_support_ipsec_offload(pf->pdev))
821 return;
822
823 if (!(pf->flags & OTX2_FLAG_IPSEC_OFFLOAD_ENABLED))
824 return;
825
826 if (pf->ipsec.sa_workq) {
827 destroy_workqueue(pf->ipsec.sa_workq);
828 pf->ipsec.sa_workq = NULL;
829 }
830
831 cn10k_outb_cpt_clean(pf);
832 }
833 EXPORT_SYMBOL(cn10k_ipsec_clean);
834
835 static u16 cn10k_ipsec_get_ip_data_len(struct xfrm_state *x,
836 struct sk_buff *skb)
837 {
838 struct ipv6hdr *ipv6h;
839 struct iphdr *iph;
840 u8 *src;
841
842 src = (u8 *)skb->data + ETH_HLEN;
843
844 if (x->props.family == AF_INET) {
845 iph = (struct iphdr *)src;
846 return ntohs(iph->tot_len);
847 }
848
849 ipv6h = (struct ipv6hdr *)src;
850 return ntohs(ipv6h->payload_len) + sizeof(struct ipv6hdr);
851 }
852
853 /* Prepare CPT and NIX SQE scatter/gather subdescriptor structure.
854 * SG of NIX and CPT are same in size.
855 * Layout of a NIX SQE and CPT SG entry:
856 * -----------------------------
857 * | CPT Scatter Gather |
858 * | (SQE SIZE) |
859 * | |
860 * -----------------------------
861 * | NIX SQE |
862 * | (SQE SIZE) |
863 * | |
864 * -----------------------------
865 */
866 bool otx2_sqe_add_sg_ipsec(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
867 struct sk_buff *skb, int num_segs, int *offset)
868 {
869 struct cpt_sg_s *cpt_sg = NULL;
870 struct nix_sqe_sg_s *sg = NULL;
871 u64 dma_addr, *iova = NULL;
872 u64 *cpt_iova = NULL;
873 u16 *sg_lens = NULL;
874 int seg, len;
875
876 sq->sg[sq->head].num_segs = 0;
877 cpt_sg = (struct cpt_sg_s *)(sq->sqe_base - sq->sqe_size);
878
879 for (seg = 0; seg < num_segs; seg++) {
880 if ((seg % MAX_SEGS_PER_SG) == 0) {
881 sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
882 sg->ld_type = NIX_SEND_LDTYPE_LDD;
883 sg->subdc = NIX_SUBDC_SG;
884 sg->segs = 0;
885 sg_lens = (void *)sg;
886 iova = (void *)sg + sizeof(*sg);
887 /* Next subdc always starts at a 16byte boundary.
888 * So if sg->segs is whether 2 or 3, offset += 16bytes.
889 */
890 if ((num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
891 *offset += sizeof(*sg) + (3 * sizeof(u64));
892 else
893 *offset += sizeof(*sg) + sizeof(u64);
894
895 cpt_sg += (seg / MAX_SEGS_PER_SG) * 4;
896 cpt_iova = (void *)cpt_sg + sizeof(*cpt_sg);
897 }
898 dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
899 if (dma_mapping_error(pfvf->dev, dma_addr))
900 return false;
901
902 sg_lens[seg % MAX_SEGS_PER_SG] = len;
903 sg->segs++;
904 *iova++ = dma_addr;
905 *cpt_iova++ = dma_addr;
906
907 /* Save DMA mapping info for later unmapping */
908 sq->sg[sq->head].dma_addr[seg] = dma_addr;
909 sq->sg[sq->head].size[seg] = len;
910 sq->sg[sq->head].num_segs++;
911
912 *cpt_sg = *(struct cpt_sg_s *)sg;
913 cpt_sg->rsvd_63_50 = 0;
914 }
915
916 sq->sg[sq->head].skb = (u64)skb;
917 return true;
918 }
919
920 static u16 cn10k_ipsec_get_param1(u8 iv_offset)
921 {
922 u16 param1_val;
923
924 /* Set Crypto mode, disable L3/L4 checksum */
925 param1_val = CN10K_IPSEC_INST_PARAM1_DIS_L4_CSUM |
926 CN10K_IPSEC_INST_PARAM1_DIS_L3_CSUM;
927 param1_val |= (u16)iv_offset << CN10K_IPSEC_INST_PARAM1_IV_OFFSET_SHIFT;
928 return param1_val;
929 }
930
931 bool cn10k_ipsec_transmit(struct otx2_nic *pf, struct netdev_queue *txq,
932 struct otx2_snd_queue *sq, struct sk_buff *skb,
933 int num_segs, int size)
934 {
935 struct cpt_inst_s inst;
936 struct cpt_res_s *res;
937 struct xfrm_state *x;
938 struct qmem *sa_info;
939 dma_addr_t dptr_iova;
940 struct sec_path *sp;
941 u8 encap_offset;
942 u8 auth_offset;
943 u8 gthr_size;
944 u8 iv_offset;
945 u16 dlen;
946
947 /* Check for IPSEC offload enabled */
948 if (!(pf->flags & OTX2_FLAG_IPSEC_OFFLOAD_ENABLED))
949 goto drop;
950
951 sp = skb_sec_path(skb);
952 if (unlikely(!sp->len))
953 goto drop;
954
955 x = xfrm_input_state(skb);
956 if (unlikely(!x))
957 goto drop;
958
959 if (x->props.mode != XFRM_MODE_TRANSPORT &&
960 x->props.mode != XFRM_MODE_TUNNEL)
961 goto drop;
962
963 dlen = cn10k_ipsec_get_ip_data_len(x, skb);
964 if (dlen == 0 && netif_msg_tx_err(pf)) {
965 netdev_err(pf->netdev, "Invalid IP header, ip-length zero\n");
966 goto drop;
967 }
968
969 /* Check for valid SA context */
970 sa_info = (struct qmem *)x->xso.offload_handle;
971 if (!sa_info)
972 goto drop;
973
974 memset(&inst, 0, sizeof(struct cpt_inst_s));
975
976 /* Get authentication offset */
977 if (x->props.family == AF_INET)
978 auth_offset = sizeof(struct iphdr);
979 else
980 auth_offset = sizeof(struct ipv6hdr);
981
982 /* IV offset is after ESP header */
983 iv_offset = auth_offset + sizeof(struct ip_esp_hdr);
984 /* Encap will start after IV */
985 encap_offset = iv_offset + GCM_RFC4106_IV_SIZE;
986
987 /* CPT Instruction word-1 */
988 res = (struct cpt_res_s *)(sq->cpt_resp->base + (64 * sq->head));
989 res->compcode = 0;
990 inst.res_addr = sq->cpt_resp->iova + (64 * sq->head);
991
992 /* CPT Instruction word-2 */
993 inst.rvu_pf_func = pf->pcifunc;
994
995 /* CPT Instruction word-3:
996 * Set QORD to force CPT_RES_S write completion
997 */
998 inst.qord = 1;
999
1000 /* CPT Instruction word-4 */
1001 /* inst.dlen should not include ICV length */
1002 inst.dlen = dlen + ETH_HLEN - (x->aead->alg_icv_len / 8);
1003 inst.opcode_major = CN10K_IPSEC_MAJOR_OP_OUTB_IPSEC;
1004 inst.param1 = cn10k_ipsec_get_param1(iv_offset);
1005
1006 inst.param2 = encap_offset <<
1007 CN10K_IPSEC_INST_PARAM2_ENC_DATA_OFFSET_SHIFT;
1008 inst.param2 |= (u16)auth_offset <<
1009 CN10K_IPSEC_INST_PARAM2_AUTH_DATA_OFFSET_SHIFT;
1010
1011 /* CPT Instruction word-5 */
1012 gthr_size = num_segs / MAX_SEGS_PER_SG;
1013 gthr_size = (num_segs % MAX_SEGS_PER_SG) ? gthr_size + 1 : gthr_size;
1014
1015 gthr_size &= 0xF;
1016 dptr_iova = (sq->sqe_ring->iova + (sq->head * (sq->sqe_size * 2)));
1017 inst.dptr = dptr_iova | ((u64)gthr_size << 60);
1018
1019 /* CPT Instruction word-6 */
1020 inst.rptr = inst.dptr;
1021
1022 /* CPT Instruction word-7 */
1023 inst.cptr = sa_info->iova;
1024 inst.ctx_val = 1;
1025 inst.egrp = CN10K_DEF_CPT_IPSEC_EGRP;
1026
1027 /* CPT Instruction word-0 */
1028 inst.nixtxl = (size / 16) - 1;
1029 inst.dat_offset = ETH_HLEN;
1030 inst.nixtx_offset = sq->sqe_size;
1031
1032 netdev_tx_sent_queue(txq, skb->len);
1033
1034 /* Finally Flush the CPT instruction */
1035 sq->head++;
1036 sq->head &= (sq->sqe_cnt - 1);
1037 cn10k_cpt_inst_flush(pf, &inst, sizeof(struct cpt_inst_s));
1038 return true;
1039 drop:
1040 dev_kfree_skb_any(skb);
1041 return false;
1042 }