Fixes and improvements in cnxk crypto PMDs

index dca8742be3..8a0e4dea4c 100644
@@ -58,6 +58,7 @@ ot_ipsec_sa_common_param_fill(union roc_ot_ipsec_sa_word2 *w2,
 {
 	struct rte_crypto_sym_xform *auth_xfrm, *cipher_xfrm;
 	const uint8_t *key = NULL;
+	uint8_t ccm_flag = 0;
 	uint32_t *tmp_salt;
 	uint64_t *tmp_key;
 	int i, length = 0;

@@ -113,6 +114,15 @@ ot_ipsec_sa_common_param_fill(union roc_ot_ipsec_sa_word2 *w2,
 			tmp_salt = (uint32_t *)salt_key;
 			*tmp_salt = rte_be_to_cpu_32(*tmp_salt);
 			break;
+		case RTE_CRYPTO_AEAD_AES_CCM:
+			w2->s.enc_type = ROC_IE_OT_SA_ENC_AES_CCM;
+			w2->s.auth_type = ROC_IE_OT_SA_AUTH_NULL;
+			ccm_flag = 0x07 & ~ROC_CPT_AES_CCM_CTR_LEN;
+			*salt_key = ccm_flag;
+			memcpy(PLT_PTR_ADD(salt_key, 1), &ipsec_xfrm->salt, 3);
+			tmp_salt = (uint32_t *)salt_key;
+			*tmp_salt = rte_be_to_cpu_32(*tmp_salt);
+			break;
 		default:
 			return -ENOTSUP;
 		}

@@ -204,6 +214,7 @@ ot_ipsec_sa_common_param_fill(union roc_ot_ipsec_sa_word2 *w2,
 	    w2->s.enc_type == ROC_IE_OT_SA_ENC_AES_CCM ||
 	    w2->s.enc_type == ROC_IE_OT_SA_ENC_AES_CTR ||
 	    w2->s.enc_type == ROC_IE_OT_SA_ENC_AES_GCM ||
+	    w2->s.enc_type == ROC_IE_OT_SA_ENC_AES_CCM ||
 	    w2->s.auth_type == ROC_IE_OT_SA_AUTH_AES_GMAC) {
 		switch (length) {
 		case ROC_CPT_AES128_KEY_LEN:

@@ -612,6 +623,7 @@ onf_ipsec_sa_common_param_fill(struct roc_ie_onf_sa_ctl *ctl, uint8_t *salt,
 	struct rte_crypto_sym_xform *auth_xfrm, *cipher_xfrm;
 	int rc, length, auth_key_len;
 	const uint8_t *key = NULL;
+	uint8_t ccm_flag = 0;
 
 	/* Set direction */
 	switch (ipsec_xfrm->direction) {

@@ -663,6 +675,14 @@ onf_ipsec_sa_common_param_fill(struct roc_ie_onf_sa_ctl *ctl, uint8_t *salt,
 			memcpy(salt, &ipsec_xfrm->salt, 4);
 			key = crypto_xfrm->aead.key.data;
 			break;
+		case RTE_CRYPTO_AEAD_AES_CCM:
+			ctl->enc_type = ROC_IE_ON_SA_ENC_AES_CCM;
+			ctl->auth_type = ROC_IE_ON_SA_AUTH_NULL;
+			ccm_flag = 0x07 & ~ROC_CPT_AES_CCM_CTR_LEN;
+			*salt = ccm_flag;
+			memcpy(PLT_PTR_ADD(salt, 1), &ipsec_xfrm->salt, 3);
+			key = crypto_xfrm->aead.key.data;
+			break;
 		default:
 			return -ENOTSUP;
 		}

@@ -810,7 +830,7 @@ cnxk_ipsec_ivlen_get(enum rte_crypto_cipher_algorithm c_algo,
 {
 	uint8_t ivlen = 0;
 
-	if (aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+	if ((aead_algo == RTE_CRYPTO_AEAD_AES_GCM) || (aead_algo == RTE_CRYPTO_AEAD_AES_CCM))
 		ivlen = 8;
 
 	switch (c_algo) {

@@ -873,6 +893,7 @@ cnxk_ipsec_icvlen_get(enum rte_crypto_cipher_algorithm c_algo,
 
 	switch (aead_algo) {
 	case RTE_CRYPTO_AEAD_AES_GCM:
+	case RTE_CRYPTO_AEAD_AES_CCM:
 		icv = 16;
 		break;
 	default:

@@ -888,7 +909,7 @@ cnxk_ipsec_outb_roundup_byte(enum rte_crypto_cipher_algorithm c_algo,
 {
 	uint8_t roundup_byte = 4;
 
-	if (aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+	if ((aead_algo == RTE_CRYPTO_AEAD_AES_GCM) || (aead_algo == RTE_CRYPTO_AEAD_AES_CCM))
 		return roundup_byte;
 
 	switch (c_algo) {

@@ -1023,6 +1044,10 @@ on_ipsec_sa_ctl_set(struct rte_security_ipsec_xform *ipsec,
 			ctl->enc_type = ROC_IE_ON_SA_ENC_AES_GCM;
 			aes_key_len = crypto_xform->aead.key.length;
 			break;
+		case RTE_CRYPTO_AEAD_AES_CCM:
+			ctl->enc_type = ROC_IE_ON_SA_ENC_AES_CCM;
+			aes_key_len = crypto_xform->aead.key.length;
+			break;
 		default:
 			plt_err("Unsupported AEAD algorithm");
 			return -ENOTSUP;

@@ -1087,6 +1112,7 @@ on_ipsec_sa_ctl_set(struct rte_security_ipsec_xform *ipsec,
 	    ctl->enc_type == ROC_IE_ON_SA_ENC_AES_CCM ||
 	    ctl->enc_type == ROC_IE_ON_SA_ENC_AES_CTR ||
 	    ctl->enc_type == ROC_IE_ON_SA_ENC_AES_GCM ||
+	    ctl->enc_type == ROC_IE_ON_SA_ENC_AES_CCM ||
 	    ctl->auth_type == ROC_IE_ON_SA_AUTH_AES_GMAC) {
 		switch (aes_key_len) {
 		case 16:

@@ -1129,6 +1155,7 @@ on_fill_ipsec_common_sa(struct rte_security_ipsec_xform *ipsec,
 	struct rte_crypto_sym_xform *cipher_xform, *auth_xform;
 	const uint8_t *cipher_key;
 	int cipher_key_len = 0;
+	uint8_t ccm_flag = 0;
 	int ret;
 
 	ret = on_ipsec_sa_ctl_set(ipsec, crypto_xform, &common_sa->ctl);

@@ -1146,6 +1173,11 @@ on_fill_ipsec_common_sa(struct rte_security_ipsec_xform *ipsec,
 	if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 		if (crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
 			memcpy(common_sa->iv.gcm.nonce, &ipsec->salt, 4);
+		else if (crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
+			ccm_flag = 0x07 & ~ROC_CPT_AES_CCM_CTR_LEN;
+			*common_sa->iv.gcm.nonce = ccm_flag;
+			memcpy(PLT_PTR_ADD(common_sa->iv.gcm.nonce, 1), &ipsec->salt, 3);
+		}
 		cipher_key = crypto_xform->aead.key.data;
 		cipher_key_len = crypto_xform->aead.key.length;
 	} else {

@@ -1194,7 +1226,7 @@ cnxk_on_ipsec_outb_sa_create(struct rte_security_ipsec_xform *ipsec,
 		return ret;
 
 	if (ctl->enc_type == ROC_IE_ON_SA_ENC_AES_GCM ||
-	    ctl->auth_type == ROC_IE_ON_SA_AUTH_NULL ||
+	    ctl->enc_type == ROC_IE_ON_SA_ENC_AES_CCM || ctl->auth_type == ROC_IE_ON_SA_AUTH_NULL ||
 	    ctl->auth_type == ROC_IE_ON_SA_AUTH_AES_GMAC) {
 		template = &out_sa->aes_gcm.template;
 		ctx_len = offsetof(struct roc_ie_on_outb_sa, aes_gcm.template);

index a3a65f1e94..0cebc05c74 100644
@@ -43,12 +43,13 @@
 	 ROC_CN10K_CPT_INST_DW_M1 << (19 + 3 * 14))
 
 /* CPT helper macros */
-#define ROC_CPT_AH_HDR_LEN	 12
-#define ROC_CPT_AES_GCM_IV_LEN	 8
-#define ROC_CPT_AES_GCM_MAC_LEN	 16
-#define ROC_CPT_AES_CBC_IV_LEN	 16
-#define ROC_CPT_SHA1_HMAC_LEN	 12
-#define ROC_CPT_SHA2_HMAC_LEN	 16
+#define ROC_CPT_AH_HDR_LEN	12
+#define ROC_CPT_AES_GCM_IV_LEN	8
+#define ROC_CPT_AES_GCM_MAC_LEN 16
+#define ROC_CPT_AES_CCM_CTR_LEN 4
+#define ROC_CPT_AES_CBC_IV_LEN	16
+#define ROC_CPT_SHA1_HMAC_LEN	12
+#define ROC_CPT_SHA2_HMAC_LEN	16
 
 #define ROC_CPT_DES3_KEY_LEN	    24
 #define ROC_CPT_AES128_KEY_LEN	    16

index 66cfe6ca98..e220863799 100644
@@ -66,6 +66,7 @@ process_outb_sa(struct roc_cpt_lf *lf, struct rte_crypto_op *cop,
 #ifdef LA_IPSEC_DEBUG
 	if (sess->out_sa.w2.s.iv_src == ROC_IE_OT_SA_IV_SRC_FROM_SA) {
 		if (sess->out_sa.w2.s.enc_type == ROC_IE_OT_SA_ENC_AES_GCM ||
+		    sess->out_sa.w2.s.enc_type == ROC_IE_OT_SA_ENC_AES_CCM ||
 		    sess->out_sa.w2.s.auth_type == ROC_IE_OT_SA_AUTH_AES_GMAC)
 			ipsec_po_sa_aes_gcm_iv_set(sess, cop);
 		else

index 8870021725..a3dcfbfa6d 100644
@@ -11,7 +11,7 @@
 #include "roc_cpt.h"
 
 #define CNXK_CPT_MAX_CAPS	 35
-#define CNXK_SEC_CRYPTO_MAX_CAPS 13
+#define CNXK_SEC_CRYPTO_MAX_CAPS 14
 #define CNXK_SEC_MAX_CAPS	 9
 #define CNXK_AE_EC_ID_MAX	 8
 /**

73 lines not on this page · raw message

index 07ab2cf4ee..00873ca6ac 100644
@@ -87,7 +87,8 @@ ipsec_xform_aead_verify(struct rte_security_ipsec_xform *ipsec_xform,
 	    crypto_xform->aead.op != RTE_CRYPTO_AEAD_OP_DECRYPT)
 		return -EINVAL;
 
-	if (crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
+	if (crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM ||
+	    crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
 		switch (crypto_xform->aead.key.length) {
 		case 16:
 		case 24:

index f761ba36e2..bfa6374005 100644
@@ -9,11 +9,12 @@
 
 #include "cn10k_cryptodev.h"
 #include "cn10k_cryptodev_ops.h"
-#include "cn10k_ipsec_la_ops.h"
 #include "cn10k_ipsec.h"
+#include "cn10k_ipsec_la_ops.h"
 #include "cnxk_ae.h"
 #include "cnxk_cryptodev.h"
 #include "cnxk_cryptodev_ops.h"
+#include "cnxk_eventdev.h"
 #include "cnxk_se.h"
 
 #include "roc_api.h"

@@ -391,79 +392,135 @@ cn10k_ca_meta_info_extract(struct rte_crypto_op *op,
 	return 0;
 }
 
-uint16_t
-cn10k_cpt_crypto_adapter_enqueue(uintptr_t base, struct rte_crypto_op *op)
+static inline uint16_t
+ca_lmtst_burst_submit(struct cn10k_sso_hws *ws, uint64_t w2[], struct cnxk_cpt_qp *qp,
+		      struct rte_crypto_op *op[], uint16_t nb_ops)
 {
+	struct cpt_inflight_req *infl_reqs[PKTS_PER_LOOP];
+	uint64_t lmt_base, lmt_arg, io_addr;
+	struct cpt_inst_s *inst, *inst_base;
 	struct cpt_inflight_req *infl_req;
-	uint64_t lmt_base, lmt_arg, w2;
-	struct cpt_inst_s *inst;
 	union cpt_fc_write_s fc;
-	struct cnxk_cpt_qp *qp;
 	uint64_t *fc_addr;
 	uint16_t lmt_id;
-	int ret;
+	int ret, i;
 
-	ret = cn10k_ca_meta_info_extract(op, &qp, &w2);
-	if (unlikely(ret)) {
-		rte_errno = EINVAL;
-		return 0;
-	}
+	lmt_base = qp->lmtline.lmt_base;
+	io_addr = qp->lmtline.io_addr;
+	fc_addr = qp->lmtline.fc_addr;
+
+	const uint32_t fc_thresh = qp->lmtline.fc_thresh;
+
+	ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
+	inst_base = (struct cpt_inst_s *)lmt_base;
 
 	if (unlikely(!qp->ca.enabled)) {
 		rte_errno = EINVAL;
 		return 0;
 	}
 
-	if (unlikely(rte_mempool_get(qp->ca.req_mp, (void **)&infl_req))) {
+	if (unlikely(rte_mempool_get_bulk(qp->ca.req_mp, (void **)infl_reqs, nb_ops))) {
 		rte_errno = ENOMEM;
 		return 0;
 	}
-	infl_req->op_flags = 0;
-
-	lmt_base = qp->lmtline.lmt_base;
-	fc_addr = qp->lmtline.fc_addr;
 
-	const uint32_t fc_thresh = qp->lmtline.fc_thresh;
+	for (i = 0; i < nb_ops; i++) {
+		inst = &inst_base[2 * i];
+		infl_req = infl_reqs[i];
+		infl_req->op_flags = 0;
 
-	ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
-	inst = (struct cpt_inst_s *)lmt_base;
+		ret = cn10k_cpt_fill_inst(qp, &op[i], inst, infl_req);
+		if (unlikely(ret != 1)) {
+			plt_dp_err("Could not process op: %p", op[i]);
+			if (i != 0)
+				goto submit;
+			else
+				goto put;
+		}
 
-	ret = cn10k_cpt_fill_inst(qp, &op, inst, infl_req);
-	if (unlikely(ret != 1)) {
-		plt_dp_err("Could not process op: %p", op);
-		rte_mempool_put(qp->ca.req_mp, infl_req);
-		return 0;
+		infl_req->res.cn10k.compcode = CPT_COMP_NOT_DONE;
+		infl_req->qp = qp;
+		inst->w0.u64 = 0;
+		inst->res_addr = (uint64_t)&infl_req->res;
+		inst->w2.u64 = w2[i];
+		inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
 	}
 
-	infl_req->cop = op;
-	infl_req->res.cn10k.compcode = CPT_COMP_NOT_DONE;
-	infl_req->qp = qp;
-	inst->w0.u64 = 0;
-	inst->res_addr = (uint64_t)&infl_req->res;
-	inst->w2.u64 = w2;
-	inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
-
 	fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED);
 	if (unlikely(fc.s.qsize > fc_thresh)) {
-		rte_mempool_put(qp->ca.req_mp, infl_req);
 		rte_errno = EAGAIN;
-		return 0;
+		i = 0;
+		goto put;
 	}
 
-	if (inst->w2.s.tt == RTE_SCHED_TYPE_ORDERED)
-		roc_sso_hws_head_wait(base);
+submit:
+	if (CNXK_TT_FROM_TAG(ws->gw_rdata) == SSO_TT_ORDERED)
+		roc_sso_hws_head_wait(ws->base);
 
-	lmt_arg = ROC_CN10K_CPT_LMT_ARG | (uint64_t)lmt_id;
-	roc_lmt_submit_steorl(lmt_arg, qp->lmtline.io_addr);
+	if (i > PKTS_PER_STEORL) {
+		lmt_arg = ROC_CN10K_CPT_LMT_ARG | (PKTS_PER_STEORL - 1) << 12 | (uint64_t)lmt_id;
+		roc_lmt_submit_steorl(lmt_arg, io_addr);
+		lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - PKTS_PER_STEORL - 1) << 12 |
+			  (uint64_t)(lmt_id + PKTS_PER_STEORL);
+		roc_lmt_submit_steorl(lmt_arg, io_addr);
+	} else {
+		lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - 1) << 12 | (uint64_t)lmt_id;
+		roc_lmt_submit_steorl(lmt_arg, io_addr);
+	}
 
 	rte_io_wmb();
 
-	return 1;
+put:
+	if (unlikely(i != nb_ops))
+		rte_mempool_put_bulk(qp->ca.req_mp, (void *)&infl_reqs[i], nb_ops - i);
+
+	return i;
+}
+
+uint16_t __rte_hot
+cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events)
+{
+	struct rte_crypto_op *ops[PKTS_PER_LOOP], *op;
+	struct cnxk_cpt_qp *qp, *curr_qp = NULL;
+	uint64_t w2s[PKTS_PER_LOOP], w2;
+	uint16_t submitted, count = 0;
+	int ret, i, ops_len = 0;
+
+	for (i = 0; i < nb_events; i++) {
+		op = ev[i].event_ptr;
+		ret = cn10k_ca_meta_info_extract(op, &qp, &w2);
+		if (unlikely(ret)) {
+			rte_errno = EINVAL;
+			return count;
+		}
+
+		if (qp != curr_qp) {
+			if (ops_len) {
+				submitted = ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
+				count += submitted;
+				if (unlikely(submitted != ops_len))
+					return count;
+				ops_len = 0;
+			}
+			curr_qp = qp;
+		}
+		w2s[ops_len] = w2;
+		ops[ops_len] = op;
+		if (++ops_len == PKTS_PER_LOOP) {
+			submitted = ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
+			count += submitted;
+			if (unlikely(submitted != ops_len))
+				return count;
+			ops_len = 0;
+		}
+	}
+	if (ops_len)
+		count += ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
+	return count;
 }
 
 static inline void
-cn10k_cpt_sec_post_process(struct rte_crypto_op *cop,
-			   struct cpt_cn10k_res_s *res)
+cn10k_cpt_sec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *res)
 {
 	struct rte_mbuf *mbuf = cop->sym->m_src;
 	const uint16_t m_len = res->rlen;

index 1ad4c16873..628d6a567c 100644
@@ -5,16 +5,17 @@
 #ifndef _CN10K_CRYPTODEV_OPS_H_
 #define _CN10K_CRYPTODEV_OPS_H_
 
-#include <rte_cryptodev.h>
 #include <cryptodev_pmd.h>
+#include <rte_cryptodev.h>
+#include <rte_eventdev.h>
 
 extern struct rte_cryptodev_ops cn10k_cpt_ops;
 
 void cn10k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev);
 
 __rte_internal
-uint16_t cn10k_cpt_crypto_adapter_enqueue(uintptr_t base,
-					  struct rte_crypto_op *op);
+uint16_t __rte_hot cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[],
+		uint16_t nb_events);
 __rte_internal
 uintptr_t cn10k_cpt_crypto_adapter_dequeue(uintptr_t get_work1);
 

index 5a0cab40a9..25c85902d6 100644
@@ -454,7 +454,7 @@ cn10k_sso_fp_fns_set(struct rte_eventdev *event_dev)
 					       sso_hws_deq_tmo_ca_burst);
 		}
 	}
-	event_dev->ca_enqueue = cn10k_sso_hws_ca_enq;
+	event_dev->ca_enqueue = cn10k_cpt_crypto_adapter_enqueue;
 
 	if (dev->tx_offloads & NIX_TX_MULTI_SEG_F)
 		CN10K_SET_EVDEV_ENQ_OP(dev, event_dev->txa_enqueue,

index f953e19dd0..4581c41233 100644
@@ -64,13 +64,3 @@ cn10k_sso_hws_enq_fwd_burst(void *port, const struct rte_event ev[],
 
 	return 1;
 }
-
-uint16_t __rte_hot
-cn10k_sso_hws_ca_enq(void *port, struct rte_event ev[], uint16_t nb_events)
-{
-	struct cn10k_sso_hws *ws = port;
-
-	RTE_SET_USED(nb_events);
-
-	return cn10k_cpt_crypto_adapter_enqueue(ws->base, ev->event_ptr);
-}

index 0915f404e0..65bb08c0a1 100644
@@ -353,8 +353,6 @@ uint16_t __rte_hot cn10k_sso_hws_enq_new_burst(void *port,
 uint16_t __rte_hot cn10k_sso_hws_enq_fwd_burst(void *port,
 					       const struct rte_event ev[],
 					       uint16_t nb_events);
-uint16_t __rte_hot cn10k_sso_hws_ca_enq(void *port, struct rte_event ev[],
-					uint16_t nb_events);
 
 #define R(name, flags)                                                         \
 	uint16_t __rte_hot cn10k_sso_hws_deq_##name(                           \

index a75003f2c6..5574dbfa04 100644
@@ -91,10 +91,6 @@ pdcp_iv_copy(uint8_t *iv_d, uint8_t *iv_s, const uint8_t pdcp_alg_type,
 			memcpy(iv_d + 6, iv_s + 8, 17);
 		} else
 			memcpy(iv_d, iv_s, 16);
-	} else {
-		/* AES-CMAC EIA2, microcode expects 16B zeroized IV */
-		for (j = 0; j < 16; j++)
-			iv_d[j] = 0;
 	}
 }
 

index 80071872f1..a73c156d01 100644
@@ -10,12 +10,13 @@
 
 #include "cnxk_ae.h"
 #include "cnxk_cryptodev.h"
-#include "cnxk_cryptodev_ops.h"
 #include "cnxk_cryptodev_capabilities.h"
+#include "cnxk_cryptodev_ops.h"
 #include "cnxk_se.h"
 
-#define CNXK_CPT_MAX_ASYM_OP_NUM_PARAMS 5
-#define CNXK_CPT_MAX_ASYM_OP_MOD_LEN	1024
+#define CNXK_CPT_MAX_ASYM_OP_NUM_PARAMS	 5
+#define CNXK_CPT_MAX_ASYM_OP_MOD_LEN	 1024
+#define CNXK_CPT_META_BUF_MAX_CACHE_SIZE 128
 
 static int
 cnxk_cpt_get_mlen(void)

@@ -200,7 +201,7 @@ cnxk_cpt_metabuf_mempool_create(const struct rte_cryptodev *dev,
 	}
 
 	mb_pool_sz = nb_elements;
-	cache_sz = RTE_MIN(RTE_MEMPOOL_CACHE_MAX_SIZE, nb_elements / 1.5);
+	cache_sz = RTE_MIN(CNXK_CPT_META_BUF_MAX_CACHE_SIZE, nb_elements / 1.5);
 
 	/* For poll mode, core that enqueues and core that dequeues can be
 	 * different. For event mode, all cores are allowed to use same crypto

index 3c4cc41bdb..2182c1bd2f 100644
@@ -28,11 +28,12 @@ cn9k_cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 
 	cpt_op = sess->cpt_op;
 
-	if (cpt_op & ROC_SE_OP_CIPHER_MASK)
+	if (sess->roc_se_ctx.fc_type == ROC_SE_PDCP_CHAIN)
+		ret = fill_pdcp_chain_params(op, sess, &qp->meta_info, infl_req, inst);
+	else if (cpt_op & ROC_SE_OP_CIPHER_MASK)
 		ret = fill_fc_params(op, sess, &qp->meta_info, infl_req, inst);
 	else
-		ret = fill_digest_params(op, sess, &qp->meta_info, infl_req,
-					 inst);
+		ret = fill_digest_params(op, sess, &qp->meta_info, infl_req, inst);
 
 	return ret;
 }

297 lines not on this page · raw message

index bfa6374005..1b70d02e2a 100644
@@ -77,25 +77,6 @@ cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 	return ret;
 }
 
-static __rte_always_inline int __rte_hot
-cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
-		  struct cnxk_se_sess *sess, struct cpt_inflight_req *infl_req,
-		  struct cpt_inst_s *inst)
-{
-	uint64_t cpt_op;
-	int ret;
-
-	cpt_op = sess->cpt_op;
-
-	if (cpt_op & ROC_SE_OP_CIPHER_MASK)
-		ret = fill_fc_params(op, sess, &qp->meta_info, infl_req, inst);
-	else
-		ret = fill_digest_params(op, sess, &qp->meta_info, infl_req,
-					 inst);
-
-	return ret;
-}
-
 static inline int
 cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[],
 		    struct cpt_inst_s inst[], struct cpt_inflight_req *infl_req)

index 2182c1bd2f..3d69723809 100644
@@ -17,27 +17,6 @@
 #include "cnxk_cryptodev_ops.h"
 #include "cnxk_se.h"
 
-static __rte_always_inline int __rte_hot
-cn9k_cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
-		       struct cnxk_se_sess *sess,
-		       struct cpt_inflight_req *infl_req,
-		       struct cpt_inst_s *inst)
-{
-	uint64_t cpt_op;
-	int ret;
-
-	cpt_op = sess->cpt_op;
-
-	if (sess->roc_se_ctx.fc_type == ROC_SE_PDCP_CHAIN)
-		ret = fill_pdcp_chain_params(op, sess, &qp->meta_info, infl_req, inst);
-	else if (cpt_op & ROC_SE_OP_CIPHER_MASK)
-		ret = fill_fc_params(op, sess, &qp->meta_info, infl_req, inst);
-	else
-		ret = fill_digest_params(op, sess, &qp->meta_info, infl_req, inst);
-
-	return ret;
-}
-
 static __rte_always_inline int __rte_hot
 cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,
 		       struct cpt_inflight_req *infl_req,

@@ -118,8 +97,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 			sym_op = op->sym;
 			sess = get_sym_session_private_data(
 				sym_op->session, cn9k_cryptodev_driver_id);
-			ret = cn9k_cpt_sym_inst_fill(qp, op, sess, infl_req,
-						     inst);
+			ret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst);
 			inst->w7.u64 = sess->cpt_inst_w7;
 		} else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
 			ret = cn9k_cpt_sec_inst_fill(op, infl_req, inst);

@@ -130,8 +108,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 				return -1;
 			}
 
-			ret = cn9k_cpt_sym_inst_fill(qp, op, sess, infl_req,
-						     inst);
+			ret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst);
 			if (unlikely(ret)) {
 				sym_session_clear(cn9k_cryptodev_driver_id,
 						  op->sym->session);

233 lines not on this page · raw message

index 35d074ea34..2c44dba9a0 100644
@@ -2453,6 +2453,7 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 	m_dst = sym_op->m_dst;
 
 	if (sess->aes_gcm || sess->chacha_poly) {
+		struct rte_mbuf *m;
 		uint8_t *salt;
 		uint8_t *aad_data;
 		uint16_t aad_len;

@@ -2483,20 +2484,19 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 			cpt_fc_salt_update(&sess->roc_se_ctx, salt);
 			sess->salt = *(uint32_t *)salt;
 		}
-		fc_params.iv_buf = salt + 4;
-		if (likely(sess->mac_len)) {
-			struct rte_mbuf *m = cpt_m_dst_get(cpt_op, m_src, m_dst);
 
-			/* hmac immediately following data is best case */
-			if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
-					     mc_hash_off !=
-				     (uint8_t *)sym_op->aead.digest.data)) {
-				flags |= ROC_SE_VALID_MAC_BUF;
-				fc_params.mac_buf.size = sess->mac_len;
-				fc_params.mac_buf.vaddr =
-					sym_op->aead.digest.data;
-				inplace = 0;
-			}
+		fc_params.iv_buf = PLT_PTR_ADD(salt, 4);
+		fc_params.mac_buf.size = 0;
+		fc_params.mac_buf.vaddr = NULL;
+		m = cpt_m_dst_get(cpt_op, m_src, m_dst);
+
+		/* Digest immediately following data is best case */
+		if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + mc_hash_off !=
+			     (uint8_t *)sym_op->aead.digest.data)) {
+			flags |= ROC_SE_VALID_MAC_BUF;
+			fc_params.mac_buf.size = sess->mac_len;
+			fc_params.mac_buf.vaddr = sym_op->aead.digest.data;
+			inplace = 0;
 		}
 	} else {
 		uint32_t ci_data_length = sym_op->cipher.data.length;

index 3d69723809..b753c1cb4b 100644
@@ -487,22 +487,19 @@ cn9k_cpt_crypto_adapter_enqueue(uintptr_t base, struct rte_crypto_op *op)
 }
 
 static inline int
-ipsec_antireplay_check(struct cn9k_ipsec_sa *sa, uint32_t win_sz,
-		       struct roc_ie_on_inb_hdr *data)
+ipsec_antireplay_check(struct cn9k_ipsec_sa *sa, uint32_t win_sz, struct roc_ie_on_inb_hdr *data)
 {
+	uint32_t esn_low, esn_hi, seql, seqh = 0;
 	struct roc_ie_on_common_sa *common_sa;
 	struct roc_ie_on_inb_sa *in_sa;
-	struct roc_ie_on_sa_ctl *ctl;
-	uint32_t seql, seqh = 0;
-	uint64_t seq;
+	uint64_t seq, seq_in_sa;
 	uint8_t esn;
 	int ret;
 
 	in_sa = &sa->in_sa;
 	common_sa = &in_sa->common_sa;
-	ctl = &common_sa->ctl;
 
-	esn = ctl->esn_en;
+	esn = common_sa->ctl.esn_en;
 	seql = rte_be_to_cpu_32(data->seql);
 
 	if (!esn) {

@@ -517,9 +514,13 @@ ipsec_antireplay_check(struct cn9k_ipsec_sa *sa, uint32_t win_sz,
 
 	ret = cnxk_on_anti_replay_check(seq, &sa->ar, win_sz);
 	if (esn && !ret) {
-		common_sa = &sa->in_sa.common_sa;
-		if (seq > common_sa->seq_t.u64)
-			common_sa->seq_t.u64 = seq;
+		esn_low = rte_be_to_cpu_32(common_sa->seq_t.tl);
+		esn_hi = rte_be_to_cpu_32(common_sa->seq_t.th);
+		seq_in_sa = ((uint64_t)esn_hi << 32) | esn_low;
+		if (seq > seq_in_sa) {
+			common_sa->seq_t.tl = rte_cpu_to_be_32(seql);
+			common_sa->seq_t.th = rte_cpu_to_be_32(seqh);
+		}
 	}
 
 	return ret;

index 2c44dba9a0..c176cda0b1 100644
@@ -1852,36 +1852,9 @@ cpt_kasumi_dec_prep(uint64_t d_offs, uint64_t d_lens,
 	return 0;
 }
 
-static __rte_always_inline int
-cpt_fc_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
-		     struct roc_se_fc_params *fc_params,
-		     struct cpt_inst_s *inst)
-{
-	struct roc_se_ctx *ctx = fc_params->ctx_buf.vaddr;
-	uint8_t fc_type;
-	int ret = -1;
-
-	fc_type = ctx->fc_type;
-
-	if (likely(fc_type == ROC_SE_FC_GEN)) {
-		ret = cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, inst);
-	} else if (fc_type == ROC_SE_KASUMI) {
-		ret = cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, inst);
-	}
-
-	/*
-	 * For AUTH_ONLY case,
-	 * MC only supports digest generation and verification
-	 * should be done in software by memcmp()
-	 */
-
-	return ret;
-}
-
 static __rte_always_inline int
 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
-		     struct roc_se_fc_params *fc_params,
-		     struct cpt_inst_s *inst)
+		     struct roc_se_fc_params *fc_params, struct cpt_inst_s *inst)
 {
 	struct roc_se_ctx *ctx = fc_params->ctx_buf.vaddr;
 	uint8_t fc_type;

@@ -2600,8 +2573,8 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 		}
 	}
 
-	if (unlikely(!((flags & ROC_SE_SINGLE_BUF_INPLACE) &&
-		       (flags & ROC_SE_SINGLE_BUF_HEADROOM) && !is_kasumi))) {
+	if (unlikely(is_kasumi || !((flags & ROC_SE_SINGLE_BUF_INPLACE) &&
+				    (flags & ROC_SE_SINGLE_BUF_HEADROOM)))) {
 		mdata = alloc_op_meta(&fc_params.meta_buf, m_info->mlen, m_info->pool, infl_req);
 		if (mdata == NULL) {
 			plt_dp_err("Error allocating meta buffer for request");

@@ -2610,12 +2583,18 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 	}
 
 	/* Finally prepare the instruction */
-	if (cpt_op & ROC_SE_OP_ENCODE)
-		ret = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, &fc_params,
-					   inst);
-	else
-		ret = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, &fc_params,
-					   inst);
+
+	if (is_kasumi) {
+		if (cpt_op & ROC_SE_OP_ENCODE)
+			ret = cpt_kasumi_enc_prep(flags, d_offs, d_lens, &fc_params, inst);
+		else
+			ret = cpt_kasumi_dec_prep(d_offs, d_lens, &fc_params, inst);
+	} else {
+		if (cpt_op & ROC_SE_OP_ENCODE)
+			ret = cpt_enc_hmac_prep(flags, d_offs, d_lens, &fc_params, inst);
+		else
+			ret = cpt_dec_hmac_prep(flags, d_offs, d_lens, &fc_params, inst);
+	}
 
 	if (unlikely(ret)) {
 		plt_dp_err("Preparing request failed due to bad input arg");

index c176cda0b1..64a66ef911 100644
@@ -680,11 +680,9 @@ cpt_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
 					return -1;
 				}
 			}
-			/* mac_data */
-			if (mac_len) {
-				i = fill_sg_comp_from_buf(scatter_comp, i,
-							  &fc_params->mac_buf);
-			}
+
+			/* Digest buffer */
+			i = fill_sg_comp_from_buf(scatter_comp, i, &fc_params->mac_buf);
 		} else {
 			/* Output including mac */
 			size = outputlen - iv_len;

index 64a66ef911..55f411d50c 100644
@@ -2372,8 +2372,6 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 	       struct cpt_qp_meta_info *m_info, struct cpt_inflight_req *infl_req,
 	       struct cpt_inst_s *inst, const bool is_kasumi)
 {
-	struct roc_se_ctx *ctx = &sess->roc_se_ctx;
-	uint8_t op_minor = ctx->template_w4.s.opcode_minor;
 	struct rte_crypto_sym_op *sym_op = cop->sym;
 	void *mdata = NULL;
 	uint32_t mc_hash_off;

@@ -2474,6 +2472,9 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 		uint32_t ci_data_offset = sym_op->cipher.data.offset;
 		uint32_t a_data_length = sym_op->auth.data.length;
 		uint32_t a_data_offset = sym_op->auth.data.offset;
+		struct roc_se_ctx *ctx = &sess->roc_se_ctx;
+
+		const uint8_t op_minor = ctx->template_w4.s.opcode_minor;
 
 		d_offs = ci_data_offset;
 		d_offs = (d_offs << 16) | a_data_offset;

index 55f411d50c..3de9718f8b 100644
@@ -2409,13 +2409,11 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 		}
 	}
 
-	/* Kasumi would need auth IV */
-	if (is_kasumi && sess->zsk_flag) {
-		fc_params.auth_iv_len = sess->auth_iv_length;
-		if (sess->auth_iv_length)
-			fc_params.auth_iv_buf =
-				rte_crypto_op_ctod_offset(cop, uint8_t *, sess->auth_iv_offset);
+	/* Kasumi would need SG mode */
+	if (is_kasumi) {
 		inplace = 0;
+		fc_params.mac_buf.vaddr = NULL;
+		fc_params.mac_buf.size = 0;
 	}
 
 	m_src = sym_op->m_src;

index a73c156d01..85bb1d27a1 100644
@@ -611,7 +611,7 @@ cnxk_cpt_inst_w7_get(struct cnxk_se_sess *sess, struct roc_cpt *roc_cpt)
 	inst_w7.s.cptr = (uint64_t)&sess->roc_se_ctx.se_ctx;
 
 	/* Set the engine group */
-	if (sess->zsk_flag || sess->chacha_poly || sess->aes_ctr_eea2)
+	if (sess->zsk_flag || sess->aes_ctr_eea2)
 		inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_SE];
 	else
 		inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];

index 85bb1d27a1..cf91b92c2c 100644
@@ -625,6 +625,7 @@ sym_session_configure(struct roc_cpt *roc_cpt, int driver_id,
 		      struct rte_cryptodev_sym_session *sess,
 		      struct rte_mempool *pool)
 {
+	enum cpt_dp_thread_type thr_type;
 	struct cnxk_se_sess *sess_priv;
 	void *priv;
 	int ret;

@@ -642,6 +643,34 @@ sym_session_configure(struct roc_cpt *roc_cpt, int driver_id,
 	if (ret)
 		goto priv_put;
 
+	if (sess_priv->cpt_op & ROC_SE_OP_CIPHER_MASK) {
+		switch (sess_priv->roc_se_ctx.fc_type) {
+		case ROC_SE_FC_GEN:
+			if (sess_priv->aes_gcm || sess_priv->chacha_poly)
+				thr_type = CPT_DP_THREAD_TYPE_FC_AEAD;
+			else
+				thr_type = CPT_DP_THREAD_TYPE_FC_CHAIN;
+			break;
+		case ROC_SE_PDCP:
+			thr_type = CPT_DP_THREAD_TYPE_PDCP;
+			break;
+		case ROC_SE_KASUMI:
+			thr_type = CPT_DP_THREAD_TYPE_KASUMI;
+			break;
+		case ROC_SE_PDCP_CHAIN:
+			thr_type = CPT_DP_THREAD_TYPE_PDCP_CHAIN;
+			break;
+		default:
+			plt_err("Invalid op type");
+			ret = -ENOTSUP;
+			goto priv_put;
+		}
+	} else {
+		thr_type = CPT_DP_THREAD_AUTH_ONLY;
+	}
+
+	sess_priv->dp_thr_type = thr_type;
+
 	if ((sess_priv->roc_se_ctx.fc_type == ROC_SE_HASH_HMAC) &&
 	    cpt_mac_len_verify(&xform->auth)) {
 		plt_dp_err("MAC length is not supported");

85 lines not on this page · raw message

index 86bb3aa79d..d1a87a96da 100644
@@ -287,32 +287,6 @@ struct roc_se_iov_ptr {
 	struct roc_se_buf_ptr bufs[];
 };
 
-struct roc_se_fc_params {
-	/* 0th cache line */
-	union {
-		struct roc_se_buf_ptr bufs[1];
-		struct {
-			struct roc_se_iov_ptr *src_iov;
-			struct roc_se_iov_ptr *dst_iov;
-		};
-	};
-	void *iv_buf;
-	void *auth_iv_buf;
-	struct roc_se_buf_ptr meta_buf;
-	struct roc_se_buf_ptr ctx_buf;
-	uint32_t rsvd2;
-	uint8_t rsvd3;
-	uint8_t iv_ovr;
-	uint8_t cipher_iv_len;
-	uint8_t auth_iv_len;
-
-	/* 1st cache line */
-	struct roc_se_buf_ptr aad_buf __plt_cache_aligned;
-	struct roc_se_buf_ptr mac_buf;
-};
-
-PLT_STATIC_ASSERT((offsetof(struct roc_se_fc_params, aad_buf) % 128) == 0);
-
 #define ROC_SE_PDCP_ALG_TYPE_ZUC	  0
 #define ROC_SE_PDCP_ALG_TYPE_SNOW3G	  1
 #define ROC_SE_PDCP_ALG_TYPE_AES_CTR	  2

@@ -349,6 +323,25 @@ struct roc_se_ctx {
 	uint8_t *auth_key;
 };
 
+struct roc_se_fc_params {
+	union {
+		struct roc_se_buf_ptr bufs[1];
+		struct {
+			struct roc_se_iov_ptr *src_iov;
+			struct roc_se_iov_ptr *dst_iov;
+		};
+	};
+	void *iv_buf;
+	void *auth_iv_buf;
+	struct roc_se_ctx *ctx;
+	struct roc_se_buf_ptr meta_buf;
+	uint8_t cipher_iv_len;
+	uint8_t auth_iv_len;
+
+	struct roc_se_buf_ptr aad_buf;
+	struct roc_se_buf_ptr mac_buf;
+};
+
 static inline void
 roc_se_zuc_bytes_swap(uint8_t *arr, int len)
 {

index dc9621fb84..44b31bfe97 100644
@@ -334,7 +334,7 @@ cpt_digest_gen_prep(uint32_t flags, uint64_t d_lens,
 	uint32_t g_size_bytes, s_size_bytes;
 	union cpt_inst_w4 cpt_inst_w4;
 
-	ctx = params->ctx_buf.vaddr;
+	ctx = params->ctx;
 
 	hash_type = ctx->hash_type;
 	mac_len = ctx->mac_len;

@@ -478,7 +478,8 @@ cpt_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
 		aad_len = fc_params->aad_buf.size;
 		aad_buf = &fc_params->aad_buf;
 	}
-	se_ctx = fc_params->ctx_buf.vaddr;
+
+	se_ctx = fc_params->ctx;
 	cipher_type = se_ctx->enc_cipher;
 	hash_type = se_ctx->hash_type;
 	mac_len = se_ctx->mac_len;

@@ -777,7 +778,7 @@ cpt_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
 		aad_buf = &fc_params->aad_buf;
 	}
 
-	se_ctx = fc_params->ctx_buf.vaddr;
+	se_ctx = fc_params->ctx;
 	hash_type = se_ctx->hash_type;
 	mac_len = se_ctx->mac_len;
 	op_minor = se_ctx->template_w4.s.opcode_minor;

@@ -1087,7 +1088,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 		return -1;
 	}
 
-	se_ctx = params->ctx_buf.vaddr;
+	se_ctx = params->ctx;
 	mac_len = se_ctx->mac_len;
 	pdcp_ci_alg = se_ctx->pdcp_ci_alg;
 	pdcp_auth_alg = se_ctx->pdcp_auth_alg;

@@ -1281,7 +1282,7 @@ cpt_pdcp_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 	uint8_t pack_iv = 0;
 	union cpt_inst_w4 cpt_inst_w4;
 
-	se_ctx = params->ctx_buf.vaddr;
+	se_ctx = params->ctx;
 	flags = se_ctx->zsk_flags;
 	mac_len = se_ctx->mac_len;
 

@@ -1562,7 +1563,7 @@ cpt_kasumi_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 	encr_data_len = ROC_SE_ENCR_DLEN(d_lens);
 	auth_data_len = ROC_SE_AUTH_DLEN(d_lens);
 
-	se_ctx = params->ctx_buf.vaddr;
+	se_ctx = params->ctx;
 	flags = se_ctx->zsk_flags;
 	mac_len = se_ctx->mac_len;
 

@@ -1751,7 +1752,7 @@ cpt_kasumi_dec_prep(uint64_t d_offs, uint64_t d_lens,
 	encr_offset = ROC_SE_ENCR_OFFSET(d_offs) / 8;
 	encr_data_len = ROC_SE_ENCR_DLEN(d_lens);
 
-	se_ctx = params->ctx_buf.vaddr;
+	se_ctx = params->ctx;
 	flags = se_ctx->zsk_flags;
 
 	cpt_inst_w4.u64 = 0;

@@ -1863,7 +1864,7 @@ static __rte_always_inline int
 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
 		     struct roc_se_fc_params *fc_params, struct cpt_inst_s *inst)
 {
-	struct roc_se_ctx *ctx = fc_params->ctx_buf.vaddr;
+	struct roc_se_ctx *ctx = fc_params->ctx;
 	uint8_t fc_type;
 	int ret = -1;
 

@@ -2408,7 +2409,7 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 		flags |= ROC_SE_VALID_IV_BUF;
 		fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, uint8_t *,
 							     sess->iv_offset);
-		if (sess->aes_ctr && unlikely(sess->iv_length != 16)) {
+		if (!is_aead && sess->aes_ctr && unlikely(sess->iv_length != 16)) {
 			memcpy((uint8_t *)iv_buf,
 			       rte_crypto_op_ctod_offset(cop, uint8_t *,
 							 sess->iv_offset),

@@ -2523,7 +2524,7 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 			}
 		}
 	}
-	fc_params.ctx_buf.vaddr = &sess->roc_se_ctx;
+	fc_params.ctx = &sess->roc_se_ctx;
 
 	if (!(sess->auth_first) && unlikely(sess->is_null || sess->cpt_op == ROC_SE_OP_DECODE))
 		inplace = 0;

@@ -2651,7 +2652,7 @@ fill_pdcp_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 	d_offs = (uint64_t)c_data_off << 16;
 	d_lens = (uint64_t)c_data_len << 32;
 
-	fc_params.ctx_buf.vaddr = &sess->roc_se_ctx;
+	fc_params.ctx = &sess->roc_se_ctx;
 
 	if (likely(m_dst == NULL || m_src == m_dst)) {
 		fc_params.dst_iov = fc_params.src_iov = (void *)src;

@@ -2794,7 +2795,7 @@ fill_pdcp_chain_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 					ci_data_length, true);
 	}
 
-	fc_params.ctx_buf.vaddr = &sess->roc_se_ctx;
+	fc_params.ctx = &sess->roc_se_ctx;
 
 	if (likely((m_dst == NULL || m_dst == m_src)) && inplace) {
 		fc_params.dst_iov = fc_params.src_iov = (void *)src;

@@ -2993,7 +2994,7 @@ fill_digest_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 
 	d_lens = sym_op->auth.data.length;
 
-	params.ctx_buf.vaddr = &sess->roc_se_ctx;
+	params.ctx = &sess->roc_se_ctx;
 
 	if (auth_op == ROC_SE_OP_AUTH_GENERATE) {
 		if (sym_op->auth.digest.data) {

index 502f243a81..0cde3e0cc3 100644
@@ -269,6 +269,8 @@ extern int cnxk_logtype_ree;
 	RTE_LOG_DP(ERR, PMD, "%s():%u " fmt "\n", __func__, __LINE__, ##args)
 #define plt_dp_info(fmt, args...)                                              \
 	RTE_LOG_DP(INFO, PMD, "%s():%u " fmt "\n", __func__, __LINE__, ##args)
+#define plt_dp_dbg(fmt, args...)                                              \
+	RTE_LOG_DP(DEBUG, PMD, "%s():%u " fmt "\n", __func__, __LINE__, ##args)
 
 #ifdef __cplusplus
 #define CNXK_PCI_ID(subsystem_dev, dev)                                        \

index 1b70d02e2a..586941cd70 100644
@@ -412,7 +412,7 @@ ca_lmtst_burst_submit(struct cn10k_sso_hws *ws, uint64_t w2[], struct cnxk_cpt_q
 
 		ret = cn10k_cpt_fill_inst(qp, &op[i], inst, infl_req);
 		if (unlikely(ret != 1)) {
-			plt_dp_err("Could not process op: %p", op[i]);
+			plt_dp_dbg("Could not process op: %p", op[i]);
 			if (i != 0)
 				goto submit;
 			else

index fdc646a6fc..ffb0c289a0 100644
@@ -1294,6 +1294,7 @@ cn10k_sec_caps_update(struct rte_security_capability *sec_cap)
 	sec_cap->ipsec.options.l4_csum_enable = 1;
 	sec_cap->ipsec.options.stats = 1;
 	sec_cap->ipsec.options.esn = 1;
+	sec_cap->ipsec.options.copy_flabel = 1;
 	sec_cap->ipsec.replay_win_sz_max = ROC_AR_WIN_SIZE_MAX;
 }
 

index 8d6446c3a0..2663480099 100644
@@ -63,6 +63,7 @@ cpt_ciph_type_set(roc_se_cipher_type type, struct roc_se_ctx *ctx,
 		break;
 	case ROC_SE_DES3_CBC:
 	case ROC_SE_DES3_ECB:
+	case ROC_SE_DES_DOCSISBPI:
 		fc_type = ROC_SE_FC_GEN;
 		break;
 	case ROC_SE_AES_CBC:

@@ -70,6 +71,7 @@ cpt_ciph_type_set(roc_se_cipher_type type, struct roc_se_ctx *ctx,
 	case ROC_SE_AES_CFB:
 	case ROC_SE_AES_CTR:
 	case ROC_SE_AES_GCM:
+	case ROC_SE_AES_DOCSISBPI:
 		if (unlikely(cpt_ciph_aes_key_validate(key_len) != 0))
 			return -1;
 		fc_type = ROC_SE_FC_GEN;

@@ -451,7 +453,7 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
 	uint8_t *zuc_const;
 	uint32_t keyx[4];
 	uint8_t *ci_key;
-	int ret;
+	int i, ret;
 
 	zs_ch_ctx = &se_ctx->se_ctx.zs_ch_ctx;
 

@@ -531,6 +533,27 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
 		memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
 		memcpy(fctx->hmac.ipad, &key[key_len], key_len);
 		break;
+	case ROC_SE_AES_DOCSISBPI:
+		/*
+		 * DOCSIS uses the combination of AES-CBC and residual termination blocks that are
+		 * less than 128. Pass it as regular AES-CBC cipher to CPT, but keep type in
+		 * se_ctx as AES_DOCSISBPI to skip block size checks in instruction preparation.
+		 */
+		cpt_ciph_aes_key_type_set(fctx, key_len);
+		fctx->enc.enc_cipher = ROC_SE_AES_CBC;
+		memcpy(fctx->enc.encr_key, key, key_len);
+		goto success;
+	case ROC_SE_DES_DOCSISBPI:
+		/* See case ROC_SE_DES3_CBC: for explanation */
+		for (i = 0; i < 3; i++)
+			memcpy(fctx->enc.encr_key + key_len * i, key, key_len);
+		/*
+		 * DOCSIS uses DES-CBC mode with special handling of residual termination blocks
+		 * that are less than 64 bits. Pass it as regular DES-CBC, but keep type in
+		 * se_ctx as DES_DOCSISBPI to skip block size checks in instruction preparation.
+		 */
+		fctx->enc.enc_cipher = ROC_SE_DES3_CBC;
+		goto success;
 	case ROC_SE_SNOW3G_UEA2:
 		if (chained_op == true) {
 			struct roc_se_onk_zuc_chain_ctx *ctx =

index d1a87a96da..e70a197d4f 100644
@@ -10,6 +10,7 @@
 #define ROC_SE_FC_MINOR_OP_ENCRYPT    0x0
 #define ROC_SE_FC_MINOR_OP_DECRYPT    0x1
 #define ROC_SE_FC_MINOR_OP_HMAC_FIRST 0x10
+#define ROC_SE_FC_MINOR_OP_DOCSIS     0x40
 
 #define ROC_SE_MAJOR_OP_HASH	   0x34
 #define ROC_SE_MAJOR_OP_HMAC	   0x35

@@ -17,10 +18,10 @@
 #define ROC_SE_MAJOR_OP_KASUMI	   0x38
 #define ROC_SE_MAJOR_OP_PDCP_CHAIN 0x3C
 
-#define ROC_SE_MAJOR_OP_MISC		 0x01
-#define ROC_SE_MISC_MINOR_OP_PASSTHROUGH 0x03
-#define ROC_SE_MISC_MINOR_OP_DUMMY	 0x04
-#define ROC_SE_MISC_MINOR_OP_HW_SUPPORT	 0x08
+#define ROC_SE_MAJOR_OP_MISC		 0x01ULL
+#define ROC_SE_MISC_MINOR_OP_PASSTHROUGH 0x03ULL
+#define ROC_SE_MISC_MINOR_OP_DUMMY	 0x04ULL
+#define ROC_SE_MISC_MINOR_OP_HW_SUPPORT	 0x08ULL
 
 #define ROC_SE_MAX_AAD_SIZE 64
 #define ROC_SE_MAX_MAC_LEN  64

@@ -125,6 +126,8 @@ typedef enum {
 	ROC_SE_AES_CTR_EEA2 = 0x92,
 	ROC_SE_KASUMI_F8_CBC = 0x93,
 	ROC_SE_KASUMI_F8_ECB = 0x94,
+	ROC_SE_AES_DOCSISBPI = 0x95,
+	ROC_SE_DES_DOCSISBPI = 0x96,
 } roc_se_cipher_type;
 
 typedef enum {

index a3dcfbfa6d..588760cfb0 100644
@@ -10,7 +10,7 @@
 
 #include "roc_cpt.h"
 
-#define CNXK_CPT_MAX_CAPS	 35
+#define CNXK_CPT_MAX_CAPS	 37
 #define CNXK_SEC_CRYPTO_MAX_CAPS 14
 #define CNXK_SEC_MAX_CAPS	 9
 #define CNXK_AE_EC_ID_MAX	 8

70 lines not on this page · raw message

index 44b31bfe97..b7d4b19d01 100644
@@ -2032,6 +2032,18 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
 		enc_type = ROC_SE_AES_ECB;
 		cipher_key_len = 16;
 		break;
+	case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
+		/* Set DOCSIS flag */
+		sess->roc_se_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DOCSIS;
+		enc_type = ROC_SE_AES_DOCSISBPI;
+		cipher_key_len = 16;
+		break;
+	case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
+		/* Set DOCSIS flag */
+		sess->roc_se_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DOCSIS;
+		enc_type = ROC_SE_DES_DOCSISBPI;
+		cipher_key_len = 8;
+		break;
 	case RTE_CRYPTO_CIPHER_3DES_CTR:
 	case RTE_CRYPTO_CIPHER_AES_F8:
 	case RTE_CRYPTO_CIPHER_ARC4:

index dca8742be3..8a0e4dea4c 100644
@@ -58,6 +58,7 @@ ot_ipsec_sa_common_param_fill(union roc_ot_ipsec_sa_word2 *w2,
 {
 	struct rte_crypto_sym_xform *auth_xfrm, *cipher_xfrm;
 	const uint8_t *key = NULL;
+	uint8_t ccm_flag = 0;
 	uint32_t *tmp_salt;
 	uint64_t *tmp_key;
 	int i, length = 0;

@@ -113,6 +114,15 @@ ot_ipsec_sa_common_param_fill(union roc_ot_ipsec_sa_word2 *w2,
 			tmp_salt = (uint32_t *)salt_key;
 			*tmp_salt = rte_be_to_cpu_32(*tmp_salt);
 			break;
+		case RTE_CRYPTO_AEAD_AES_CCM:
+			w2->s.enc_type = ROC_IE_OT_SA_ENC_AES_CCM;
+			w2->s.auth_type = ROC_IE_OT_SA_AUTH_NULL;
+			ccm_flag = 0x07 & ~ROC_CPT_AES_CCM_CTR_LEN;
+			*salt_key = ccm_flag;
+			memcpy(PLT_PTR_ADD(salt_key, 1), &ipsec_xfrm->salt, 3);
+			tmp_salt = (uint32_t *)salt_key;
+			*tmp_salt = rte_be_to_cpu_32(*tmp_salt);
+			break;
 		default:
 			return -ENOTSUP;
 		}

@@ -204,6 +214,7 @@ ot_ipsec_sa_common_param_fill(union roc_ot_ipsec_sa_word2 *w2,
 	    w2->s.enc_type == ROC_IE_OT_SA_ENC_AES_CCM ||
 	    w2->s.enc_type == ROC_IE_OT_SA_ENC_AES_CTR ||
 	    w2->s.enc_type == ROC_IE_OT_SA_ENC_AES_GCM ||
+	    w2->s.enc_type == ROC_IE_OT_SA_ENC_AES_CCM ||
 	    w2->s.auth_type == ROC_IE_OT_SA_AUTH_AES_GMAC) {
 		switch (length) {
 		case ROC_CPT_AES128_KEY_LEN:

@@ -612,6 +623,7 @@ onf_ipsec_sa_common_param_fill(struct roc_ie_onf_sa_ctl *ctl, uint8_t *salt,
 	struct rte_crypto_sym_xform *auth_xfrm, *cipher_xfrm;
 	int rc, length, auth_key_len;
 	const uint8_t *key = NULL;
+	uint8_t ccm_flag = 0;
 
 	/* Set direction */
 	switch (ipsec_xfrm->direction) {

@@ -663,6 +675,14 @@ onf_ipsec_sa_common_param_fill(struct roc_ie_onf_sa_ctl *ctl, uint8_t *salt,
 			memcpy(salt, &ipsec_xfrm->salt, 4);
 			key = crypto_xfrm->aead.key.data;
 			break;
+		case RTE_CRYPTO_AEAD_AES_CCM:
+			ctl->enc_type = ROC_IE_ON_SA_ENC_AES_CCM;
+			ctl->auth_type = ROC_IE_ON_SA_AUTH_NULL;
+			ccm_flag = 0x07 & ~ROC_CPT_AES_CCM_CTR_LEN;
+			*salt = ccm_flag;
+			memcpy(PLT_PTR_ADD(salt, 1), &ipsec_xfrm->salt, 3);
+			key = crypto_xfrm->aead.key.data;
+			break;
 		default:
 			return -ENOTSUP;
 		}

@@ -810,7 +830,7 @@ cnxk_ipsec_ivlen_get(enum rte_crypto_cipher_algorithm c_algo,
 {
 	uint8_t ivlen = 0;
 
-	if (aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+	if ((aead_algo == RTE_CRYPTO_AEAD_AES_GCM) || (aead_algo == RTE_CRYPTO_AEAD_AES_CCM))
 		ivlen = 8;
 
 	switch (c_algo) {

@@ -873,6 +893,7 @@ cnxk_ipsec_icvlen_get(enum rte_crypto_cipher_algorithm c_algo,
 
 	switch (aead_algo) {
 	case RTE_CRYPTO_AEAD_AES_GCM:
+	case RTE_CRYPTO_AEAD_AES_CCM:
 		icv = 16;
 		break;
 	default:

@@ -888,7 +909,7 @@ cnxk_ipsec_outb_roundup_byte(enum rte_crypto_cipher_algorithm c_algo,
 {
 	uint8_t roundup_byte = 4;
 
-	if (aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+	if ((aead_algo == RTE_CRYPTO_AEAD_AES_GCM) || (aead_algo == RTE_CRYPTO_AEAD_AES_CCM))
 		return roundup_byte;
 
 	switch (c_algo) {

@@ -1023,6 +1044,10 @@ on_ipsec_sa_ctl_set(struct rte_security_ipsec_xform *ipsec,
 			ctl->enc_type = ROC_IE_ON_SA_ENC_AES_GCM;
 			aes_key_len = crypto_xform->aead.key.length;
 			break;
+		case RTE_CRYPTO_AEAD_AES_CCM:
+			ctl->enc_type = ROC_IE_ON_SA_ENC_AES_CCM;
+			aes_key_len = crypto_xform->aead.key.length;
+			break;
 		default:
 			plt_err("Unsupported AEAD algorithm");
 			return -ENOTSUP;

@@ -1087,6 +1112,7 @@ on_ipsec_sa_ctl_set(struct rte_security_ipsec_xform *ipsec,
 	    ctl->enc_type == ROC_IE_ON_SA_ENC_AES_CCM ||
 	    ctl->enc_type == ROC_IE_ON_SA_ENC_AES_CTR ||
 	    ctl->enc_type == ROC_IE_ON_SA_ENC_AES_GCM ||
+	    ctl->enc_type == ROC_IE_ON_SA_ENC_AES_CCM ||
 	    ctl->auth_type == ROC_IE_ON_SA_AUTH_AES_GMAC) {
 		switch (aes_key_len) {
 		case 16:

@@ -1129,6 +1155,7 @@ on_fill_ipsec_common_sa(struct rte_security_ipsec_xform *ipsec,
 	struct rte_crypto_sym_xform *cipher_xform, *auth_xform;
 	const uint8_t *cipher_key;
 	int cipher_key_len = 0;
+	uint8_t ccm_flag = 0;
 	int ret;
 
 	ret = on_ipsec_sa_ctl_set(ipsec, crypto_xform, &common_sa->ctl);

@@ -1146,6 +1173,11 @@ on_fill_ipsec_common_sa(struct rte_security_ipsec_xform *ipsec,
 	if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 		if (crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
 			memcpy(common_sa->iv.gcm.nonce, &ipsec->salt, 4);
+		else if (crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
+			ccm_flag = 0x07 & ~ROC_CPT_AES_CCM_CTR_LEN;
+			*common_sa->iv.gcm.nonce = ccm_flag;
+			memcpy(PLT_PTR_ADD(common_sa->iv.gcm.nonce, 1), &ipsec->salt, 3);
+		}
 		cipher_key = crypto_xform->aead.key.data;
 		cipher_key_len = crypto_xform->aead.key.length;
 	} else {

@@ -1194,7 +1226,7 @@ cnxk_on_ipsec_outb_sa_create(struct rte_security_ipsec_xform *ipsec,
 		return ret;
 
 	if (ctl->enc_type == ROC_IE_ON_SA_ENC_AES_GCM ||
-	    ctl->auth_type == ROC_IE_ON_SA_AUTH_NULL ||
+	    ctl->enc_type == ROC_IE_ON_SA_ENC_AES_CCM || ctl->auth_type == ROC_IE_ON_SA_AUTH_NULL ||
 	    ctl->auth_type == ROC_IE_ON_SA_AUTH_AES_GMAC) {
 		template = &out_sa->aes_gcm.template;
 		ctx_len = offsetof(struct roc_ie_on_outb_sa, aes_gcm.template);

index a3a65f1e94..0cebc05c74 100644
@@ -43,12 +43,13 @@
 	 ROC_CN10K_CPT_INST_DW_M1 << (19 + 3 * 14))
 
 /* CPT helper macros */
-#define ROC_CPT_AH_HDR_LEN	 12
-#define ROC_CPT_AES_GCM_IV_LEN	 8
-#define ROC_CPT_AES_GCM_MAC_LEN	 16
-#define ROC_CPT_AES_CBC_IV_LEN	 16
-#define ROC_CPT_SHA1_HMAC_LEN	 12
-#define ROC_CPT_SHA2_HMAC_LEN	 16
+#define ROC_CPT_AH_HDR_LEN	12
+#define ROC_CPT_AES_GCM_IV_LEN	8
+#define ROC_CPT_AES_GCM_MAC_LEN 16
+#define ROC_CPT_AES_CCM_CTR_LEN 4
+#define ROC_CPT_AES_CBC_IV_LEN	16
+#define ROC_CPT_SHA1_HMAC_LEN	12
+#define ROC_CPT_SHA2_HMAC_LEN	16
 
 #define ROC_CPT_DES3_KEY_LEN	    24
 #define ROC_CPT_AES128_KEY_LEN	    16

index 66cfe6ca98..e220863799 100644
@@ -66,6 +66,7 @@ process_outb_sa(struct roc_cpt_lf *lf, struct rte_crypto_op *cop,
 #ifdef LA_IPSEC_DEBUG
 	if (sess->out_sa.w2.s.iv_src == ROC_IE_OT_SA_IV_SRC_FROM_SA) {
 		if (sess->out_sa.w2.s.enc_type == ROC_IE_OT_SA_ENC_AES_GCM ||
+		    sess->out_sa.w2.s.enc_type == ROC_IE_OT_SA_ENC_AES_CCM ||
 		    sess->out_sa.w2.s.auth_type == ROC_IE_OT_SA_AUTH_AES_GMAC)
 			ipsec_po_sa_aes_gcm_iv_set(sess, cop);
 		else

index 8870021725..a3dcfbfa6d 100644
@@ -11,7 +11,7 @@
 #include "roc_cpt.h"
 
 #define CNXK_CPT_MAX_CAPS	 35
-#define CNXK_SEC_CRYPTO_MAX_CAPS 13
+#define CNXK_SEC_CRYPTO_MAX_CAPS 14
 #define CNXK_SEC_MAX_CAPS	 9
 #define CNXK_AE_EC_ID_MAX	 8
 /**

73 lines not on this page · raw message

index 07ab2cf4ee..00873ca6ac 100644
@@ -87,7 +87,8 @@ ipsec_xform_aead_verify(struct rte_security_ipsec_xform *ipsec_xform,
 	    crypto_xform->aead.op != RTE_CRYPTO_AEAD_OP_DECRYPT)
 		return -EINVAL;
 
-	if (crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
+	if (crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM ||
+	    crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
 		switch (crypto_xform->aead.key.length) {
 		case 16:
 		case 24:

index f761ba36e2..bfa6374005 100644
@@ -9,11 +9,12 @@
 
 #include "cn10k_cryptodev.h"
 #include "cn10k_cryptodev_ops.h"
-#include "cn10k_ipsec_la_ops.h"
 #include "cn10k_ipsec.h"
+#include "cn10k_ipsec_la_ops.h"
 #include "cnxk_ae.h"
 #include "cnxk_cryptodev.h"
 #include "cnxk_cryptodev_ops.h"
+#include "cnxk_eventdev.h"
 #include "cnxk_se.h"
 
 #include "roc_api.h"

@@ -391,79 +392,135 @@ cn10k_ca_meta_info_extract(struct rte_crypto_op *op,
 	return 0;
 }
 
-uint16_t
-cn10k_cpt_crypto_adapter_enqueue(uintptr_t base, struct rte_crypto_op *op)
+static inline uint16_t
+ca_lmtst_burst_submit(struct cn10k_sso_hws *ws, uint64_t w2[], struct cnxk_cpt_qp *qp,
+		      struct rte_crypto_op *op[], uint16_t nb_ops)
 {
+	struct cpt_inflight_req *infl_reqs[PKTS_PER_LOOP];
+	uint64_t lmt_base, lmt_arg, io_addr;
+	struct cpt_inst_s *inst, *inst_base;
 	struct cpt_inflight_req *infl_req;
-	uint64_t lmt_base, lmt_arg, w2;
-	struct cpt_inst_s *inst;
 	union cpt_fc_write_s fc;
-	struct cnxk_cpt_qp *qp;
 	uint64_t *fc_addr;
 	uint16_t lmt_id;
-	int ret;
+	int ret, i;
 
-	ret = cn10k_ca_meta_info_extract(op, &qp, &w2);
-	if (unlikely(ret)) {
-		rte_errno = EINVAL;
-		return 0;
-	}
+	lmt_base = qp->lmtline.lmt_base;
+	io_addr = qp->lmtline.io_addr;
+	fc_addr = qp->lmtline.fc_addr;
+
+	const uint32_t fc_thresh = qp->lmtline.fc_thresh;
+
+	ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
+	inst_base = (struct cpt_inst_s *)lmt_base;
 
 	if (unlikely(!qp->ca.enabled)) {
 		rte_errno = EINVAL;
 		return 0;
 	}
 
-	if (unlikely(rte_mempool_get(qp->ca.req_mp, (void **)&infl_req))) {
+	if (unlikely(rte_mempool_get_bulk(qp->ca.req_mp, (void **)infl_reqs, nb_ops))) {
 		rte_errno = ENOMEM;
 		return 0;
 	}
-	infl_req->op_flags = 0;
-
-	lmt_base = qp->lmtline.lmt_base;
-	fc_addr = qp->lmtline.fc_addr;
 
-	const uint32_t fc_thresh = qp->lmtline.fc_thresh;
+	for (i = 0; i < nb_ops; i++) {
+		inst = &inst_base[2 * i];
+		infl_req = infl_reqs[i];
+		infl_req->op_flags = 0;
 
-	ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
-	inst = (struct cpt_inst_s *)lmt_base;
+		ret = cn10k_cpt_fill_inst(qp, &op[i], inst, infl_req);
+		if (unlikely(ret != 1)) {
+			plt_dp_err("Could not process op: %p", op[i]);
+			if (i != 0)
+				goto submit;
+			else
+				goto put;
+		}
 
-	ret = cn10k_cpt_fill_inst(qp, &op, inst, infl_req);
-	if (unlikely(ret != 1)) {
-		plt_dp_err("Could not process op: %p", op);
-		rte_mempool_put(qp->ca.req_mp, infl_req);
-		return 0;
+		infl_req->res.cn10k.compcode = CPT_COMP_NOT_DONE;
+		infl_req->qp = qp;
+		inst->w0.u64 = 0;
+		inst->res_addr = (uint64_t)&infl_req->res;
+		inst->w2.u64 = w2[i];
+		inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
 	}
 
-	infl_req->cop = op;
-	infl_req->res.cn10k.compcode = CPT_COMP_NOT_DONE;
-	infl_req->qp = qp;
-	inst->w0.u64 = 0;
-	inst->res_addr = (uint64_t)&infl_req->res;
-	inst->w2.u64 = w2;
-	inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
-
 	fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED);
 	if (unlikely(fc.s.qsize > fc_thresh)) {
-		rte_mempool_put(qp->ca.req_mp, infl_req);
 		rte_errno = EAGAIN;
-		return 0;
+		i = 0;
+		goto put;
 	}
 
-	if (inst->w2.s.tt == RTE_SCHED_TYPE_ORDERED)
-		roc_sso_hws_head_wait(base);
+submit:
+	if (CNXK_TT_FROM_TAG(ws->gw_rdata) == SSO_TT_ORDERED)
+		roc_sso_hws_head_wait(ws->base);
 
-	lmt_arg = ROC_CN10K_CPT_LMT_ARG | (uint64_t)lmt_id;
-	roc_lmt_submit_steorl(lmt_arg, qp->lmtline.io_addr);
+	if (i > PKTS_PER_STEORL) {
+		lmt_arg = ROC_CN10K_CPT_LMT_ARG | (PKTS_PER_STEORL - 1) << 12 | (uint64_t)lmt_id;
+		roc_lmt_submit_steorl(lmt_arg, io_addr);
+		lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - PKTS_PER_STEORL - 1) << 12 |
+			  (uint64_t)(lmt_id + PKTS_PER_STEORL);
+		roc_lmt_submit_steorl(lmt_arg, io_addr);
+	} else {
+		lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - 1) << 12 | (uint64_t)lmt_id;
+		roc_lmt_submit_steorl(lmt_arg, io_addr);
+	}
 
 	rte_io_wmb();
 
-	return 1;
+put:
+	if (unlikely(i != nb_ops))
+		rte_mempool_put_bulk(qp->ca.req_mp, (void *)&infl_reqs[i], nb_ops - i);
+
+	return i;
+}
+
+uint16_t __rte_hot
+cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events)
+{
+	struct rte_crypto_op *ops[PKTS_PER_LOOP], *op;
+	struct cnxk_cpt_qp *qp, *curr_qp = NULL;
+	uint64_t w2s[PKTS_PER_LOOP], w2;
+	uint16_t submitted, count = 0;
+	int ret, i, ops_len = 0;
+
+	for (i = 0; i < nb_events; i++) {
+		op = ev[i].event_ptr;
+		ret = cn10k_ca_meta_info_extract(op, &qp, &w2);
+		if (unlikely(ret)) {
+			rte_errno = EINVAL;
+			return count;
+		}
+
+		if (qp != curr_qp) {
+			if (ops_len) {
+				submitted = ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
+				count += submitted;
+				if (unlikely(submitted != ops_len))
+					return count;
+				ops_len = 0;
+			}
+			curr_qp = qp;
+		}
+		w2s[ops_len] = w2;
+		ops[ops_len] = op;
+		if (++ops_len == PKTS_PER_LOOP) {
+			submitted = ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
+			count += submitted;
+			if (unlikely(submitted != ops_len))
+				return count;
+			ops_len = 0;
+		}
+	}
+	if (ops_len)
+		count += ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
+	return count;
 }
 
 static inline void
-cn10k_cpt_sec_post_process(struct rte_crypto_op *cop,
-			   struct cpt_cn10k_res_s *res)
+cn10k_cpt_sec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *res)
 {
 	struct rte_mbuf *mbuf = cop->sym->m_src;
 	const uint16_t m_len = res->rlen;

index 1ad4c16873..628d6a567c 100644
@@ -5,16 +5,17 @@
 #ifndef _CN10K_CRYPTODEV_OPS_H_
 #define _CN10K_CRYPTODEV_OPS_H_
 
-#include <rte_cryptodev.h>
 #include <cryptodev_pmd.h>
+#include <rte_cryptodev.h>
+#include <rte_eventdev.h>
 
 extern struct rte_cryptodev_ops cn10k_cpt_ops;
 
 void cn10k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev);
 
 __rte_internal
-uint16_t cn10k_cpt_crypto_adapter_enqueue(uintptr_t base,
-					  struct rte_crypto_op *op);
+uint16_t __rte_hot cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[],
+		uint16_t nb_events);
 __rte_internal
 uintptr_t cn10k_cpt_crypto_adapter_dequeue(uintptr_t get_work1);
 

index 5a0cab40a9..25c85902d6 100644
@@ -454,7 +454,7 @@ cn10k_sso_fp_fns_set(struct rte_eventdev *event_dev)
 					       sso_hws_deq_tmo_ca_burst);
 		}
 	}
-	event_dev->ca_enqueue = cn10k_sso_hws_ca_enq;
+	event_dev->ca_enqueue = cn10k_cpt_crypto_adapter_enqueue;
 
 	if (dev->tx_offloads & NIX_TX_MULTI_SEG_F)
 		CN10K_SET_EVDEV_ENQ_OP(dev, event_dev->txa_enqueue,

index f953e19dd0..4581c41233 100644
@@ -64,13 +64,3 @@ cn10k_sso_hws_enq_fwd_burst(void *port, const struct rte_event ev[],
 
 	return 1;
 }
-
-uint16_t __rte_hot
-cn10k_sso_hws_ca_enq(void *port, struct rte_event ev[], uint16_t nb_events)
-{
-	struct cn10k_sso_hws *ws = port;
-
-	RTE_SET_USED(nb_events);
-
-	return cn10k_cpt_crypto_adapter_enqueue(ws->base, ev->event_ptr);
-}

index 0915f404e0..65bb08c0a1 100644
@@ -353,8 +353,6 @@ uint16_t __rte_hot cn10k_sso_hws_enq_new_burst(void *port,
 uint16_t __rte_hot cn10k_sso_hws_enq_fwd_burst(void *port,
 					       const struct rte_event ev[],
 					       uint16_t nb_events);
-uint16_t __rte_hot cn10k_sso_hws_ca_enq(void *port, struct rte_event ev[],
-					uint16_t nb_events);
 
 #define R(name, flags)                                                         \
 	uint16_t __rte_hot cn10k_sso_hws_deq_##name(                           \

index a75003f2c6..5574dbfa04 100644
@@ -91,10 +91,6 @@ pdcp_iv_copy(uint8_t *iv_d, uint8_t *iv_s, const uint8_t pdcp_alg_type,
 			memcpy(iv_d + 6, iv_s + 8, 17);
 		} else
 			memcpy(iv_d, iv_s, 16);
-	} else {
-		/* AES-CMAC EIA2, microcode expects 16B zeroized IV */
-		for (j = 0; j < 16; j++)
-			iv_d[j] = 0;
 	}
 }
 

index 80071872f1..a73c156d01 100644
@@ -10,12 +10,13 @@
 
 #include "cnxk_ae.h"
 #include "cnxk_cryptodev.h"
-#include "cnxk_cryptodev_ops.h"
 #include "cnxk_cryptodev_capabilities.h"
+#include "cnxk_cryptodev_ops.h"
 #include "cnxk_se.h"
 
-#define CNXK_CPT_MAX_ASYM_OP_NUM_PARAMS 5
-#define CNXK_CPT_MAX_ASYM_OP_MOD_LEN	1024
+#define CNXK_CPT_MAX_ASYM_OP_NUM_PARAMS	 5
+#define CNXK_CPT_MAX_ASYM_OP_MOD_LEN	 1024
+#define CNXK_CPT_META_BUF_MAX_CACHE_SIZE 128
 
 static int
 cnxk_cpt_get_mlen(void)

@@ -200,7 +201,7 @@ cnxk_cpt_metabuf_mempool_create(const struct rte_cryptodev *dev,
 	}
 
 	mb_pool_sz = nb_elements;
-	cache_sz = RTE_MIN(RTE_MEMPOOL_CACHE_MAX_SIZE, nb_elements / 1.5);
+	cache_sz = RTE_MIN(CNXK_CPT_META_BUF_MAX_CACHE_SIZE, nb_elements / 1.5);
 
 	/* For poll mode, core that enqueues and core that dequeues can be
 	 * different. For event mode, all cores are allowed to use same crypto

index 3c4cc41bdb..2182c1bd2f 100644
@@ -28,11 +28,12 @@ cn9k_cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 
 	cpt_op = sess->cpt_op;
 
-	if (cpt_op & ROC_SE_OP_CIPHER_MASK)
+	if (sess->roc_se_ctx.fc_type == ROC_SE_PDCP_CHAIN)
+		ret = fill_pdcp_chain_params(op, sess, &qp->meta_info, infl_req, inst);
+	else if (cpt_op & ROC_SE_OP_CIPHER_MASK)
 		ret = fill_fc_params(op, sess, &qp->meta_info, infl_req, inst);
 	else
-		ret = fill_digest_params(op, sess, &qp->meta_info, infl_req,
-					 inst);
+		ret = fill_digest_params(op, sess, &qp->meta_info, infl_req, inst);
 
 	return ret;
 }

297 lines not on this page · raw message

index bfa6374005..1b70d02e2a 100644
@@ -77,25 +77,6 @@ cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 	return ret;
 }
 
-static __rte_always_inline int __rte_hot
-cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
-		  struct cnxk_se_sess *sess, struct cpt_inflight_req *infl_req,
-		  struct cpt_inst_s *inst)
-{
-	uint64_t cpt_op;
-	int ret;
-
-	cpt_op = sess->cpt_op;
-
-	if (cpt_op & ROC_SE_OP_CIPHER_MASK)
-		ret = fill_fc_params(op, sess, &qp->meta_info, infl_req, inst);
-	else
-		ret = fill_digest_params(op, sess, &qp->meta_info, infl_req,
-					 inst);
-
-	return ret;
-}
-
 static inline int
 cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[],
 		    struct cpt_inst_s inst[], struct cpt_inflight_req *infl_req)

index 2182c1bd2f..3d69723809 100644
@@ -17,27 +17,6 @@
 #include "cnxk_cryptodev_ops.h"
 #include "cnxk_se.h"
 
-static __rte_always_inline int __rte_hot
-cn9k_cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
-		       struct cnxk_se_sess *sess,
-		       struct cpt_inflight_req *infl_req,
-		       struct cpt_inst_s *inst)
-{
-	uint64_t cpt_op;
-	int ret;
-
-	cpt_op = sess->cpt_op;
-
-	if (sess->roc_se_ctx.fc_type == ROC_SE_PDCP_CHAIN)
-		ret = fill_pdcp_chain_params(op, sess, &qp->meta_info, infl_req, inst);
-	else if (cpt_op & ROC_SE_OP_CIPHER_MASK)
-		ret = fill_fc_params(op, sess, &qp->meta_info, infl_req, inst);
-	else
-		ret = fill_digest_params(op, sess, &qp->meta_info, infl_req, inst);
-
-	return ret;
-}
-
 static __rte_always_inline int __rte_hot
 cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,
 		       struct cpt_inflight_req *infl_req,

@@ -118,8 +97,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 			sym_op = op->sym;
 			sess = get_sym_session_private_data(
 				sym_op->session, cn9k_cryptodev_driver_id);
-			ret = cn9k_cpt_sym_inst_fill(qp, op, sess, infl_req,
-						     inst);
+			ret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst);
 			inst->w7.u64 = sess->cpt_inst_w7;
 		} else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
 			ret = cn9k_cpt_sec_inst_fill(op, infl_req, inst);

@@ -130,8 +108,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 				return -1;
 			}
 
-			ret = cn9k_cpt_sym_inst_fill(qp, op, sess, infl_req,
-						     inst);
+			ret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst);
 			if (unlikely(ret)) {
 				sym_session_clear(cn9k_cryptodev_driver_id,
 						  op->sym->session);

233 lines not on this page · raw message

index 35d074ea34..2c44dba9a0 100644
@@ -2453,6 +2453,7 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 	m_dst = sym_op->m_dst;
 
 	if (sess->aes_gcm || sess->chacha_poly) {
+		struct rte_mbuf *m;
 		uint8_t *salt;
 		uint8_t *aad_data;
 		uint16_t aad_len;

@@ -2483,20 +2484,19 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 			cpt_fc_salt_update(&sess->roc_se_ctx, salt);
 			sess->salt = *(uint32_t *)salt;
 		}
-		fc_params.iv_buf = salt + 4;
-		if (likely(sess->mac_len)) {
-			struct rte_mbuf *m = cpt_m_dst_get(cpt_op, m_src, m_dst);
 
-			/* hmac immediately following data is best case */
-			if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
-					     mc_hash_off !=
-				     (uint8_t *)sym_op->aead.digest.data)) {
-				flags |= ROC_SE_VALID_MAC_BUF;
-				fc_params.mac_buf.size = sess->mac_len;
-				fc_params.mac_buf.vaddr =
-					sym_op->aead.digest.data;
-				inplace = 0;
-			}
+		fc_params.iv_buf = PLT_PTR_ADD(salt, 4);
+		fc_params.mac_buf.size = 0;
+		fc_params.mac_buf.vaddr = NULL;
+		m = cpt_m_dst_get(cpt_op, m_src, m_dst);
+
+		/* Digest immediately following data is best case */
+		if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + mc_hash_off !=
+			     (uint8_t *)sym_op->aead.digest.data)) {
+			flags |= ROC_SE_VALID_MAC_BUF;
+			fc_params.mac_buf.size = sess->mac_len;
+			fc_params.mac_buf.vaddr = sym_op->aead.digest.data;
+			inplace = 0;
 		}
 	} else {
 		uint32_t ci_data_length = sym_op->cipher.data.length;

index 3d69723809..b753c1cb4b 100644
@@ -487,22 +487,19 @@ cn9k_cpt_crypto_adapter_enqueue(uintptr_t base, struct rte_crypto_op *op)
 }
 
 static inline int
-ipsec_antireplay_check(struct cn9k_ipsec_sa *sa, uint32_t win_sz,
-		       struct roc_ie_on_inb_hdr *data)
+ipsec_antireplay_check(struct cn9k_ipsec_sa *sa, uint32_t win_sz, struct roc_ie_on_inb_hdr *data)
 {
+	uint32_t esn_low, esn_hi, seql, seqh = 0;
 	struct roc_ie_on_common_sa *common_sa;
 	struct roc_ie_on_inb_sa *in_sa;
-	struct roc_ie_on_sa_ctl *ctl;
-	uint32_t seql, seqh = 0;
-	uint64_t seq;
+	uint64_t seq, seq_in_sa;
 	uint8_t esn;
 	int ret;
 
 	in_sa = &sa->in_sa;
 	common_sa = &in_sa->common_sa;
-	ctl = &common_sa->ctl;
 
-	esn = ctl->esn_en;
+	esn = common_sa->ctl.esn_en;
 	seql = rte_be_to_cpu_32(data->seql);
 
 	if (!esn) {

@@ -517,9 +514,13 @@ ipsec_antireplay_check(struct cn9k_ipsec_sa *sa, uint32_t win_sz,
 
 	ret = cnxk_on_anti_replay_check(seq, &sa->ar, win_sz);
 	if (esn && !ret) {
-		common_sa = &sa->in_sa.common_sa;
-		if (seq > common_sa->seq_t.u64)
-			common_sa->seq_t.u64 = seq;
+		esn_low = rte_be_to_cpu_32(common_sa->seq_t.tl);
+		esn_hi = rte_be_to_cpu_32(common_sa->seq_t.th);
+		seq_in_sa = ((uint64_t)esn_hi << 32) | esn_low;
+		if (seq > seq_in_sa) {
+			common_sa->seq_t.tl = rte_cpu_to_be_32(seql);
+			common_sa->seq_t.th = rte_cpu_to_be_32(seqh);
+		}
 	}
 
 	return ret;

index 2c44dba9a0..64100e8376 100644
@@ -1852,36 +1852,9 @@ cpt_kasumi_dec_prep(uint64_t d_offs, uint64_t d_lens,
 	return 0;
 }
 
-static __rte_always_inline int
-cpt_fc_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
-		     struct roc_se_fc_params *fc_params,
-		     struct cpt_inst_s *inst)
-{
-	struct roc_se_ctx *ctx = fc_params->ctx_buf.vaddr;
-	uint8_t fc_type;
-	int ret = -1;
-
-	fc_type = ctx->fc_type;
-
-	if (likely(fc_type == ROC_SE_FC_GEN)) {
-		ret = cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, inst);
-	} else if (fc_type == ROC_SE_KASUMI) {
-		ret = cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, inst);
-	}
-
-	/*
-	 * For AUTH_ONLY case,
-	 * MC only supports digest generation and verification
-	 * should be done in software by memcmp()
-	 */
-
-	return ret;
-}
-
 static __rte_always_inline int
 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
-		     struct roc_se_fc_params *fc_params,
-		     struct cpt_inst_s *inst)
+		     struct roc_se_fc_params *fc_params, struct cpt_inst_s *inst)
 {
 	struct roc_se_ctx *ctx = fc_params->ctx_buf.vaddr;
 	uint8_t fc_type;

@@ -2425,6 +2398,8 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 	fc_params.auth_iv_len = 0;
 	fc_params.auth_iv_buf = NULL;
 	fc_params.iv_buf = NULL;
+	fc_params.mac_buf.size = 0;
+	fc_params.mac_buf.vaddr = 0;
 
 	if (likely(sess->iv_length)) {
 		flags |= ROC_SE_VALID_IV_BUF;

@@ -2486,8 +2461,6 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 		}
 
 		fc_params.iv_buf = PLT_PTR_ADD(salt, 4);
-		fc_params.mac_buf.size = 0;
-		fc_params.mac_buf.vaddr = NULL;
 		m = cpt_m_dst_get(cpt_op, m_src, m_dst);
 
 		/* Digest immediately following data is best case */

@@ -2600,8 +2573,8 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 		}
 	}
 
-	if (unlikely(!((flags & ROC_SE_SINGLE_BUF_INPLACE) &&
-		       (flags & ROC_SE_SINGLE_BUF_HEADROOM) && !is_kasumi))) {
+	if (unlikely(is_kasumi || !((flags & ROC_SE_SINGLE_BUF_INPLACE) &&
+				    (flags & ROC_SE_SINGLE_BUF_HEADROOM)))) {
 		mdata = alloc_op_meta(&fc_params.meta_buf, m_info->mlen, m_info->pool, infl_req);
 		if (mdata == NULL) {
 			plt_dp_err("Error allocating meta buffer for request");

@@ -2610,12 +2583,18 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 	}
 
 	/* Finally prepare the instruction */
-	if (cpt_op & ROC_SE_OP_ENCODE)
-		ret = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, &fc_params,
-					   inst);
-	else
-		ret = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, &fc_params,
-					   inst);
+
+	if (is_kasumi) {
+		if (cpt_op & ROC_SE_OP_ENCODE)
+			ret = cpt_kasumi_enc_prep(flags, d_offs, d_lens, &fc_params, inst);
+		else
+			ret = cpt_kasumi_dec_prep(d_offs, d_lens, &fc_params, inst);
+	} else {
+		if (cpt_op & ROC_SE_OP_ENCODE)
+			ret = cpt_enc_hmac_prep(flags, d_offs, d_lens, &fc_params, inst);
+		else
+			ret = cpt_dec_hmac_prep(flags, d_offs, d_lens, &fc_params, inst);
+	}
 
 	if (unlikely(ret)) {
 		plt_dp_err("Preparing request failed due to bad input arg");

index 64100e8376..f382194c27 100644
@@ -680,11 +680,9 @@ cpt_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
 					return -1;
 				}
 			}
-			/* mac_data */
-			if (mac_len) {
-				i = fill_sg_comp_from_buf(scatter_comp, i,
-							  &fc_params->mac_buf);
-			}
+
+			/* Digest buffer */
+			i = fill_sg_comp_from_buf(scatter_comp, i, &fc_params->mac_buf);
 		} else {
 			/* Output including mac */
 			size = outputlen - iv_len;

index f382194c27..a75e98362d 100644
@@ -2372,8 +2372,6 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 	       struct cpt_qp_meta_info *m_info, struct cpt_inflight_req *infl_req,
 	       struct cpt_inst_s *inst, const bool is_kasumi)
 {
-	struct roc_se_ctx *ctx = &sess->roc_se_ctx;
-	uint8_t op_minor = ctx->template_w4.s.opcode_minor;
 	struct rte_crypto_sym_op *sym_op = cop->sym;
 	void *mdata = NULL;
 	uint32_t mc_hash_off;

@@ -2474,6 +2472,9 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 		uint32_t ci_data_offset = sym_op->cipher.data.offset;
 		uint32_t a_data_length = sym_op->auth.data.length;
 		uint32_t a_data_offset = sym_op->auth.data.offset;
+		struct roc_se_ctx *ctx = &sess->roc_se_ctx;
+
+		const uint8_t op_minor = ctx->template_w4.s.opcode_minor;
 
 		d_offs = ci_data_offset;
 		d_offs = (d_offs << 16) | a_data_offset;

index a75e98362d..d8126567f9 100644
@@ -2411,14 +2411,9 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 		}
 	}
 
-	/* Kasumi would need auth IV */
-	if (is_kasumi && sess->zsk_flag) {
-		fc_params.auth_iv_len = sess->auth_iv_length;
-		if (sess->auth_iv_length)
-			fc_params.auth_iv_buf =
-				rte_crypto_op_ctod_offset(cop, uint8_t *, sess->auth_iv_offset);
+	/* Kasumi would need SG mode */
+	if (is_kasumi)
 		inplace = 0;
-	}
 
 	m_src = sym_op->m_src;
 	m_dst = sym_op->m_dst;

index a73c156d01..85bb1d27a1 100644
@@ -611,7 +611,7 @@ cnxk_cpt_inst_w7_get(struct cnxk_se_sess *sess, struct roc_cpt *roc_cpt)
 	inst_w7.s.cptr = (uint64_t)&sess->roc_se_ctx.se_ctx;
 
 	/* Set the engine group */
-	if (sess->zsk_flag || sess->chacha_poly || sess->aes_ctr_eea2)
+	if (sess->zsk_flag || sess->aes_ctr_eea2)
 		inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_SE];
 	else
 		inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];

index 85bb1d27a1..cf91b92c2c 100644
@@ -625,6 +625,7 @@ sym_session_configure(struct roc_cpt *roc_cpt, int driver_id,
 		      struct rte_cryptodev_sym_session *sess,
 		      struct rte_mempool *pool)
 {
+	enum cpt_dp_thread_type thr_type;
 	struct cnxk_se_sess *sess_priv;
 	void *priv;
 	int ret;

@@ -642,6 +643,34 @@ sym_session_configure(struct roc_cpt *roc_cpt, int driver_id,
 	if (ret)
 		goto priv_put;
 
+	if (sess_priv->cpt_op & ROC_SE_OP_CIPHER_MASK) {
+		switch (sess_priv->roc_se_ctx.fc_type) {
+		case ROC_SE_FC_GEN:
+			if (sess_priv->aes_gcm || sess_priv->chacha_poly)
+				thr_type = CPT_DP_THREAD_TYPE_FC_AEAD;
+			else
+				thr_type = CPT_DP_THREAD_TYPE_FC_CHAIN;
+			break;
+		case ROC_SE_PDCP:
+			thr_type = CPT_DP_THREAD_TYPE_PDCP;
+			break;
+		case ROC_SE_KASUMI:
+			thr_type = CPT_DP_THREAD_TYPE_KASUMI;
+			break;
+		case ROC_SE_PDCP_CHAIN:
+			thr_type = CPT_DP_THREAD_TYPE_PDCP_CHAIN;
+			break;
+		default:
+			plt_err("Invalid op type");
+			ret = -ENOTSUP;
+			goto priv_put;
+		}
+	} else {
+		thr_type = CPT_DP_THREAD_AUTH_ONLY;
+	}
+
+	sess_priv->dp_thr_type = thr_type;
+
 	if ((sess_priv->roc_se_ctx.fc_type == ROC_SE_HASH_HMAC) &&
 	    cpt_mac_len_verify(&xform->auth)) {
 		plt_dp_err("MAC length is not supported");

92 lines not on this page · raw message

index 86bb3aa79d..d1a87a96da 100644
@@ -287,32 +287,6 @@ struct roc_se_iov_ptr {
 	struct roc_se_buf_ptr bufs[];
 };
 
-struct roc_se_fc_params {
-	/* 0th cache line */
-	union {
-		struct roc_se_buf_ptr bufs[1];
-		struct {
-			struct roc_se_iov_ptr *src_iov;
-			struct roc_se_iov_ptr *dst_iov;
-		};
-	};
-	void *iv_buf;
-	void *auth_iv_buf;
-	struct roc_se_buf_ptr meta_buf;
-	struct roc_se_buf_ptr ctx_buf;
-	uint32_t rsvd2;
-	uint8_t rsvd3;
-	uint8_t iv_ovr;
-	uint8_t cipher_iv_len;
-	uint8_t auth_iv_len;
-
-	/* 1st cache line */
-	struct roc_se_buf_ptr aad_buf __plt_cache_aligned;
-	struct roc_se_buf_ptr mac_buf;
-};
-
-PLT_STATIC_ASSERT((offsetof(struct roc_se_fc_params, aad_buf) % 128) == 0);
-
 #define ROC_SE_PDCP_ALG_TYPE_ZUC	  0
 #define ROC_SE_PDCP_ALG_TYPE_SNOW3G	  1
 #define ROC_SE_PDCP_ALG_TYPE_AES_CTR	  2

@@ -349,6 +323,25 @@ struct roc_se_ctx {
 	uint8_t *auth_key;
 };
 
+struct roc_se_fc_params {
+	union {
+		struct roc_se_buf_ptr bufs[1];
+		struct {
+			struct roc_se_iov_ptr *src_iov;
+			struct roc_se_iov_ptr *dst_iov;
+		};
+	};
+	void *iv_buf;
+	void *auth_iv_buf;
+	struct roc_se_ctx *ctx;
+	struct roc_se_buf_ptr meta_buf;
+	uint8_t cipher_iv_len;
+	uint8_t auth_iv_len;
+
+	struct roc_se_buf_ptr aad_buf;
+	struct roc_se_buf_ptr mac_buf;
+};
+
 static inline void
 roc_se_zuc_bytes_swap(uint8_t *arr, int len)
 {

index 99fb3b899e..33e65eb4e3 100644
@@ -334,7 +334,7 @@ cpt_digest_gen_prep(uint32_t flags, uint64_t d_lens,
 	uint32_t g_size_bytes, s_size_bytes;
 	union cpt_inst_w4 cpt_inst_w4;
 
-	ctx = params->ctx_buf.vaddr;
+	ctx = params->ctx;
 
 	hash_type = ctx->hash_type;
 	mac_len = ctx->mac_len;

@@ -478,7 +478,8 @@ cpt_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
 		aad_len = fc_params->aad_buf.size;
 		aad_buf = &fc_params->aad_buf;
 	}
-	se_ctx = fc_params->ctx_buf.vaddr;
+
+	se_ctx = fc_params->ctx;
 	cipher_type = se_ctx->enc_cipher;
 	hash_type = se_ctx->hash_type;
 	mac_len = se_ctx->mac_len;

@@ -777,7 +778,7 @@ cpt_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
 		aad_buf = &fc_params->aad_buf;
 	}
 
-	se_ctx = fc_params->ctx_buf.vaddr;
+	se_ctx = fc_params->ctx;
 	hash_type = se_ctx->hash_type;
 	mac_len = se_ctx->mac_len;
 	op_minor = se_ctx->template_w4.s.opcode_minor;

@@ -1087,7 +1088,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 		return -1;
 	}
 
-	se_ctx = params->ctx_buf.vaddr;
+	se_ctx = params->ctx;
 	mac_len = se_ctx->mac_len;
 	pdcp_ci_alg = se_ctx->pdcp_ci_alg;
 	pdcp_auth_alg = se_ctx->pdcp_auth_alg;

@@ -1281,7 +1282,7 @@ cpt_pdcp_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 	uint8_t pack_iv = 0;
 	union cpt_inst_w4 cpt_inst_w4;
 
-	se_ctx = params->ctx_buf.vaddr;
+	se_ctx = params->ctx;
 	flags = se_ctx->zsk_flags;
 	mac_len = se_ctx->mac_len;
 

@@ -1562,7 +1563,7 @@ cpt_kasumi_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 	encr_data_len = ROC_SE_ENCR_DLEN(d_lens);
 	auth_data_len = ROC_SE_AUTH_DLEN(d_lens);
 
-	se_ctx = params->ctx_buf.vaddr;
+	se_ctx = params->ctx;
 	flags = se_ctx->zsk_flags;
 	mac_len = se_ctx->mac_len;
 

@@ -1751,7 +1752,7 @@ cpt_kasumi_dec_prep(uint64_t d_offs, uint64_t d_lens,
 	encr_offset = ROC_SE_ENCR_OFFSET(d_offs) / 8;
 	encr_data_len = ROC_SE_ENCR_DLEN(d_lens);
 
-	se_ctx = params->ctx_buf.vaddr;
+	se_ctx = params->ctx;
 	flags = se_ctx->zsk_flags;
 
 	cpt_inst_w4.u64 = 0;

@@ -1863,7 +1864,7 @@ static __rte_always_inline int
 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
 		     struct roc_se_fc_params *fc_params, struct cpt_inst_s *inst)
 {
-	struct roc_se_ctx *ctx = fc_params->ctx_buf.vaddr;
+	struct roc_se_ctx *ctx = fc_params->ctx;
 	uint8_t fc_type;
 	int ret = -1;
 

@@ -2410,7 +2411,7 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 		flags |= ROC_SE_VALID_IV_BUF;
 		fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, uint8_t *,
 							     sess->iv_offset);
-		if (sess->aes_ctr && unlikely(sess->iv_length != 16)) {
+		if (!is_aead && sess->aes_ctr && unlikely(sess->iv_length != 16)) {
 			memcpy((uint8_t *)iv_buf,
 			       rte_crypto_op_ctod_offset(cop, uint8_t *,
 							 sess->iv_offset),

@@ -2520,7 +2521,7 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 			}
 		}
 	}
-	fc_params.ctx_buf.vaddr = &sess->roc_se_ctx;
+	fc_params.ctx = &sess->roc_se_ctx;
 
 	if (!(sess->auth_first) && unlikely(sess->is_null || sess->cpt_op == ROC_SE_OP_DECODE))
 		inplace = 0;

@@ -2649,7 +2650,7 @@ fill_pdcp_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 	d_offs = (uint64_t)c_data_off << 16;
 	d_lens = (uint64_t)c_data_len << 32;
 
-	fc_params.ctx_buf.vaddr = &sess->roc_se_ctx;
+	fc_params.ctx = &sess->roc_se_ctx;
 
 	if (likely(m_dst == NULL || m_src == m_dst)) {
 		fc_params.dst_iov = fc_params.src_iov = (void *)src;

@@ -2792,7 +2793,7 @@ fill_pdcp_chain_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 					ci_data_length, true);
 	}
 
-	fc_params.ctx_buf.vaddr = &sess->roc_se_ctx;
+	fc_params.ctx = &sess->roc_se_ctx;
 
 	if (likely((m_dst == NULL || m_dst == m_src)) && inplace) {
 		fc_params.dst_iov = fc_params.src_iov = (void *)src;

@@ -2991,7 +2992,7 @@ fill_digest_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 
 	d_lens = sym_op->auth.data.length;
 
-	params.ctx_buf.vaddr = &sess->roc_se_ctx;
+	params.ctx = &sess->roc_se_ctx;
 
 	if (auth_op == ROC_SE_OP_AUTH_GENERATE) {
 		if (sym_op->auth.digest.data) {

index 502f243a81..0cde3e0cc3 100644
@@ -269,6 +269,8 @@ extern int cnxk_logtype_ree;
 	RTE_LOG_DP(ERR, PMD, "%s():%u " fmt "\n", __func__, __LINE__, ##args)
 #define plt_dp_info(fmt, args...)                                              \
 	RTE_LOG_DP(INFO, PMD, "%s():%u " fmt "\n", __func__, __LINE__, ##args)
+#define plt_dp_dbg(fmt, args...)                                              \
+	RTE_LOG_DP(DEBUG, PMD, "%s():%u " fmt "\n", __func__, __LINE__, ##args)
 
 #ifdef __cplusplus
 #define CNXK_PCI_ID(subsystem_dev, dev)                                        \

index 1b70d02e2a..586941cd70 100644
@@ -412,7 +412,7 @@ ca_lmtst_burst_submit(struct cn10k_sso_hws *ws, uint64_t w2[], struct cnxk_cpt_q
 
 		ret = cn10k_cpt_fill_inst(qp, &op[i], inst, infl_req);
 		if (unlikely(ret != 1)) {
-			plt_dp_err("Could not process op: %p", op[i]);
+			plt_dp_dbg("Could not process op: %p", op[i]);
 			if (i != 0)
 				goto submit;
 			else

index fdc646a6fc..ffb0c289a0 100644
@@ -1294,6 +1294,7 @@ cn10k_sec_caps_update(struct rte_security_capability *sec_cap)
 	sec_cap->ipsec.options.l4_csum_enable = 1;
 	sec_cap->ipsec.options.stats = 1;
 	sec_cap->ipsec.options.esn = 1;
+	sec_cap->ipsec.options.copy_flabel = 1;
 	sec_cap->ipsec.replay_win_sz_max = ROC_AR_WIN_SIZE_MAX;
 }
 

index 8d6446c3a0..2663480099 100644
@@ -63,6 +63,7 @@ cpt_ciph_type_set(roc_se_cipher_type type, struct roc_se_ctx *ctx,
 		break;
 	case ROC_SE_DES3_CBC:
 	case ROC_SE_DES3_ECB:
+	case ROC_SE_DES_DOCSISBPI:
 		fc_type = ROC_SE_FC_GEN;
 		break;
 	case ROC_SE_AES_CBC:

@@ -70,6 +71,7 @@ cpt_ciph_type_set(roc_se_cipher_type type, struct roc_se_ctx *ctx,
 	case ROC_SE_AES_CFB:
 	case ROC_SE_AES_CTR:
 	case ROC_SE_AES_GCM:
+	case ROC_SE_AES_DOCSISBPI:
 		if (unlikely(cpt_ciph_aes_key_validate(key_len) != 0))
 			return -1;
 		fc_type = ROC_SE_FC_GEN;

@@ -451,7 +453,7 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
 	uint8_t *zuc_const;
 	uint32_t keyx[4];
 	uint8_t *ci_key;
-	int ret;
+	int i, ret;
 
 	zs_ch_ctx = &se_ctx->se_ctx.zs_ch_ctx;
 

@@ -531,6 +533,27 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
 		memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
 		memcpy(fctx->hmac.ipad, &key[key_len], key_len);
 		break;
+	case ROC_SE_AES_DOCSISBPI:
+		/*
+		 * DOCSIS uses the combination of AES-CBC and residual termination blocks that are
+		 * less than 128. Pass it as regular AES-CBC cipher to CPT, but keep type in
+		 * se_ctx as AES_DOCSISBPI to skip block size checks in instruction preparation.
+		 */
+		cpt_ciph_aes_key_type_set(fctx, key_len);
+		fctx->enc.enc_cipher = ROC_SE_AES_CBC;
+		memcpy(fctx->enc.encr_key, key, key_len);
+		goto success;
+	case ROC_SE_DES_DOCSISBPI:
+		/* See case ROC_SE_DES3_CBC: for explanation */
+		for (i = 0; i < 3; i++)
+			memcpy(fctx->enc.encr_key + key_len * i, key, key_len);
+		/*
+		 * DOCSIS uses DES-CBC mode with special handling of residual termination blocks
+		 * that are less than 64 bits. Pass it as regular DES-CBC, but keep type in
+		 * se_ctx as DES_DOCSISBPI to skip block size checks in instruction preparation.
+		 */
+		fctx->enc.enc_cipher = ROC_SE_DES3_CBC;
+		goto success;
 	case ROC_SE_SNOW3G_UEA2:
 		if (chained_op == true) {
 			struct roc_se_onk_zuc_chain_ctx *ctx =

index d1a87a96da..e70a197d4f 100644
@@ -10,6 +10,7 @@
 #define ROC_SE_FC_MINOR_OP_ENCRYPT    0x0
 #define ROC_SE_FC_MINOR_OP_DECRYPT    0x1
 #define ROC_SE_FC_MINOR_OP_HMAC_FIRST 0x10
+#define ROC_SE_FC_MINOR_OP_DOCSIS     0x40
 
 #define ROC_SE_MAJOR_OP_HASH	   0x34
 #define ROC_SE_MAJOR_OP_HMAC	   0x35

@@ -17,10 +18,10 @@
 #define ROC_SE_MAJOR_OP_KASUMI	   0x38
 #define ROC_SE_MAJOR_OP_PDCP_CHAIN 0x3C
 
-#define ROC_SE_MAJOR_OP_MISC		 0x01
-#define ROC_SE_MISC_MINOR_OP_PASSTHROUGH 0x03
-#define ROC_SE_MISC_MINOR_OP_DUMMY	 0x04
-#define ROC_SE_MISC_MINOR_OP_HW_SUPPORT	 0x08
+#define ROC_SE_MAJOR_OP_MISC		 0x01ULL
+#define ROC_SE_MISC_MINOR_OP_PASSTHROUGH 0x03ULL
+#define ROC_SE_MISC_MINOR_OP_DUMMY	 0x04ULL
+#define ROC_SE_MISC_MINOR_OP_HW_SUPPORT	 0x08ULL
 
 #define ROC_SE_MAX_AAD_SIZE 64
 #define ROC_SE_MAX_MAC_LEN  64

@@ -125,6 +126,8 @@ typedef enum {
 	ROC_SE_AES_CTR_EEA2 = 0x92,
 	ROC_SE_KASUMI_F8_CBC = 0x93,
 	ROC_SE_KASUMI_F8_ECB = 0x94,
+	ROC_SE_AES_DOCSISBPI = 0x95,
+	ROC_SE_DES_DOCSISBPI = 0x96,
 } roc_se_cipher_type;
 
 typedef enum {

index a3dcfbfa6d..588760cfb0 100644
@@ -10,7 +10,7 @@
 
 #include "roc_cpt.h"
 
-#define CNXK_CPT_MAX_CAPS	 35
+#define CNXK_CPT_MAX_CAPS	 37
 #define CNXK_SEC_CRYPTO_MAX_CAPS 14
 #define CNXK_SEC_MAX_CAPS	 9
 #define CNXK_AE_EC_ID_MAX	 8

70 lines not on this page · raw message

index 33e65eb4e3..54a78d0a5a 100644
@@ -2032,6 +2032,18 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
 		enc_type = ROC_SE_AES_ECB;
 		cipher_key_len = 16;
 		break;
+	case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
+		/* Set DOCSIS flag */
+		sess->roc_se_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DOCSIS;
+		enc_type = ROC_SE_AES_DOCSISBPI;
+		cipher_key_len = 16;
+		break;
+	case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
+		/* Set DOCSIS flag */
+		sess->roc_se_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DOCSIS;
+		enc_type = ROC_SE_DES_DOCSISBPI;
+		cipher_key_len = 8;
+		break;
 	case RTE_CRYPTO_CIPHER_3DES_CTR:
 	case RTE_CRYPTO_CIPHER_AES_F8:
 	case RTE_CRYPTO_CIPHER_ARC4: