Fixes and improvements to cnxk crypto PMDs

94 lines not on this page · raw message

index 15ebd57..0c9e244 100644
@@ -74,7 +74,7 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
 		if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 			sa->iv_offset = crypto_xfrm->aead.iv.offset;
 			sa->iv_length = crypto_xfrm->aead.iv.length;
-		} else {
+		} else if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 			sa->iv_offset = crypto_xfrm->cipher.iv.offset;
 			sa->iv_length = crypto_xfrm->cipher.iv.length;
 		}

151 lines not on this page · raw message

index 16e7572..542c93b 100644
@@ -12,7 +12,7 @@
 
 #define CNXK_CPT_MAX_CAPS	 34
 #define CNXK_SEC_CRYPTO_MAX_CAPS 11
-#define CNXK_SEC_MAX_CAPS	 5
+#define CNXK_SEC_MAX_CAPS	 9
 #define CNXK_AE_EC_ID_MAX	 8
 /**
  * Device private data

50 lines not on this page · raw message

82 lines not on this page · raw message

index afefbd2..69a962d 100644
@@ -155,6 +155,14 @@ ot_ipsec_sa_common_param_fill(union roc_ot_ipsec_sa_word2 *w2,
 		case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 			w2->s.auth_type = ROC_IE_OT_SA_AUTH_AES_XCBC_128;
 			break;
+		case RTE_CRYPTO_AUTH_AES_GMAC:
+			w2->s.auth_type = ROC_IE_OT_SA_AUTH_AES_GMAC;
+			key = auth_xfrm->auth.key.data;
+			length = auth_xfrm->auth.key.length;
+			memcpy(salt_key, &ipsec_xfrm->salt, 4);
+			tmp_salt = (uint32_t *)salt_key;
+			*tmp_salt = rte_be_to_cpu_32(*tmp_salt);
+			break;
 		default:
 			return -ENOTSUP;
 		}

index 0c9e244..3a2bf0f 100644
@@ -77,6 +77,9 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
 		} else if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 			sa->iv_offset = crypto_xfrm->cipher.iv.offset;
 			sa->iv_length = crypto_xfrm->cipher.iv.length;
+		} else {
+			sa->iv_offset = crypto_xfrm->auth.iv.offset;
+			sa->iv_length = crypto_xfrm->auth.iv.length;
 		}
 	}
 #else

index f2d8122..66cfe6c 100644
@@ -65,7 +65,8 @@ process_outb_sa(struct roc_cpt_lf *lf, struct rte_crypto_op *cop,
 
 #ifdef LA_IPSEC_DEBUG
 	if (sess->out_sa.w2.s.iv_src == ROC_IE_OT_SA_IV_SRC_FROM_SA) {
-		if (sess->out_sa.w2.s.enc_type == ROC_IE_OT_SA_ENC_AES_GCM)
+		if (sess->out_sa.w2.s.enc_type == ROC_IE_OT_SA_ENC_AES_GCM ||
+		    sess->out_sa.w2.s.auth_type == ROC_IE_OT_SA_AUTH_AES_GMAC)
 			ipsec_po_sa_aes_gcm_iv_set(sess, cop);
 		else
 			ipsec_po_sa_iv_set(sess, cop);

index eaa3698..82b8dae 100644
@@ -211,6 +211,7 @@ ipsec_sa_ctl_set(struct rte_security_ipsec_xform *ipsec,
 			break;
 		case RTE_CRYPTO_AUTH_AES_GMAC:
 			ctl->auth_type = ROC_IE_ON_SA_AUTH_AES_GMAC;
+			aes_key_len = auth_xform->auth.key.length;
 			break;
 		case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 			ctl->auth_type = ROC_IE_ON_SA_AUTH_AES_XCBC_128;

@@ -265,7 +266,7 @@ fill_ipsec_common_sa(struct rte_security_ipsec_xform *ipsec,
 		     struct rte_crypto_sym_xform *crypto_xform,
 		     struct roc_ie_on_common_sa *common_sa)
 {
-	struct rte_crypto_sym_xform *cipher_xform;
+	struct rte_crypto_sym_xform *cipher_xform, *auth_xform;
 	const uint8_t *cipher_key;
 	int cipher_key_len = 0;
 	int ret;

@@ -279,13 +280,13 @@ fill_ipsec_common_sa(struct rte_security_ipsec_xform *ipsec,
 		common_sa->esn_hi = ipsec->esn.hi;
 	}
 
-	if (ipsec->proto == RTE_SECURITY_IPSEC_SA_PROTO_AH)
-		return 0;
-
-	if (ipsec->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
+	if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
+		auth_xform = crypto_xform;
 		cipher_xform = crypto_xform->next;
-	else
+	} else {
 		cipher_xform = crypto_xform;
+		auth_xform = crypto_xform->next;
+	}
 
 	if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 		if (crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)

@@ -293,8 +294,16 @@ fill_ipsec_common_sa(struct rte_security_ipsec_xform *ipsec,
 		cipher_key = crypto_xform->aead.key.data;
 		cipher_key_len = crypto_xform->aead.key.length;
 	} else {
-		cipher_key = cipher_xform->cipher.key.data;
-		cipher_key_len = cipher_xform->cipher.key.length;
+		if (cipher_xform) {
+			cipher_key = cipher_xform->cipher.key.data;
+			cipher_key_len = cipher_xform->cipher.key.length;
+		}
+
+		if (auth_xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
+			memcpy(common_sa->iv.gcm.nonce, &ipsec->salt, 4);
+			cipher_key = auth_xform->auth.key.data;
+			cipher_key_len = auth_xform->auth.key.length;
+		}
 	}
 
 	if (cipher_key_len != 0)

@@ -358,7 +367,8 @@ cn9k_ipsec_outb_sa_create(struct cnxk_cpt_qp *qp,
 		return ret;
 
 	if (ctl->enc_type == ROC_IE_ON_SA_ENC_AES_GCM ||
-	    ctl->auth_type == ROC_IE_ON_SA_AUTH_NULL) {
+	    ctl->auth_type == ROC_IE_ON_SA_AUTH_NULL ||
+	    ctl->auth_type == ROC_IE_ON_SA_AUTH_AES_GMAC) {
 		template = &out_sa->aes_gcm.template;
 		ctx_len = offsetof(struct roc_ie_on_outb_sa, aes_gcm.template);
 	} else {

@@ -453,6 +463,7 @@ cn9k_ipsec_outb_sa_create(struct cnxk_cpt_qp *qp,
 		auth_key_len = auth_xform->auth.key.length;
 
 		switch (auth_xform->auth.algo) {
+		case RTE_CRYPTO_AUTH_AES_GMAC:
 		case RTE_CRYPTO_AUTH_NULL:
 			break;
 		case RTE_CRYPTO_AUTH_SHA1_HMAC:

@@ -497,6 +508,9 @@ cn9k_ipsec_outb_sa_create(struct cnxk_cpt_qp *qp,
 		} else if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 			sa->cipher_iv_off = crypto_xform->cipher.iv.offset;
 			sa->cipher_iv_len = crypto_xform->cipher.iv.length;
+		} else {
+			sa->cipher_iv_off = crypto_xform->auth.iv.offset;
+			sa->cipher_iv_len = crypto_xform->auth.iv.length;
 		}
 	}
 #else

@@ -553,7 +567,8 @@ cn9k_ipsec_inb_sa_create(struct cnxk_cpt_qp *qp,
 		return ret;
 
 	if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD ||
-	    auth_xform->auth.algo == RTE_CRYPTO_AUTH_NULL) {
+	    auth_xform->auth.algo == RTE_CRYPTO_AUTH_NULL ||
+	    auth_xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 		ctx_len = offsetof(struct roc_ie_on_inb_sa,
 				   sha1_or_gcm.hmac_key[0]);
 	} else {

index 542c93b..fe2904b 100644
@@ -11,7 +11,7 @@
 #include "roc_cpt.h"
 
 #define CNXK_CPT_MAX_CAPS	 34
-#define CNXK_SEC_CRYPTO_MAX_CAPS 11
+#define CNXK_SEC_CRYPTO_MAX_CAPS 12
 #define CNXK_SEC_MAX_CAPS	 9
 #define CNXK_AE_EC_ID_MAX	 8
 /**

index efd53db..98b002d 100644
@@ -835,6 +835,31 @@ static const struct rte_cryptodev_capabilities sec_caps_aes[] = {
 			}, }
 		}, }
 	},
+	{	/* AES GMAC (AUTH) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_AES_GMAC,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 8
+				},
+				.digest_size = {
+					.min = 8,
+					.max = 16,
+					.increment = 4
+				},
+				.iv_size = {
+					.min = 12,
+					.max = 12,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
 };
 
 static const struct rte_cryptodev_capabilities sec_caps_sha1_sha2[] = {

index 1524217..171ea27 100644
@@ -59,6 +59,9 @@ ipsec_xform_auth_verify(struct rte_crypto_sym_xform *crypto_xform)
 	} else if (crypto_xform->auth.algo == RTE_CRYPTO_AUTH_SHA512_HMAC) {
 		if (keylen == 64)
 			return 0;
+	} else if (crypto_xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
+		if (keylen >= 16 && keylen <= 32)
+			return 0;
 	}
 
 	if (crypto_xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC &&

index e988d57..ce7ca2e 100644
@@ -2047,7 +2047,7 @@ prepare_iov_from_pkt(struct rte_mbuf *pkt, struct roc_se_iov_ptr *iovec,
 	return 0;
 }
 
-static __rte_always_inline uint32_t
+static __rte_always_inline void
 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt,
 			     struct roc_se_fc_params *param, uint32_t *flags)
 {

@@ -2070,7 +2070,7 @@ prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt,
 
 		param->bufs[0].vaddr = seg_data;
 		param->bufs[0].size = seg_size;
-		return 0;
+		return;
 	}
 	iovec = param->src_iov;
 	iovec->bufs[index].vaddr = seg_data;

@@ -2094,7 +2094,7 @@ prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt,
 	}
 
 	iovec->buf_cnt = index;
-	return 0;
+	return;
 }
 
 static __rte_always_inline int

@@ -2254,12 +2254,7 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
 		 */
 		fc_params.dst_iov = fc_params.src_iov = (void *)src;
 
-		if (unlikely(prepare_iov_from_pkt_inplace(m_src, &fc_params,
-							  &flags))) {
-			plt_dp_err("Prepare inplace src iov failed");
-			ret = -EINVAL;
-			goto err_exit;
-		}
+		prepare_iov_from_pkt_inplace(m_src, &fc_params, &flags);
 
 	} else {
 		/* Out of place processing */

index b3a3649..742723a 100644
@@ -915,9 +915,9 @@ roc_cpt_ctx_write(struct roc_cpt_lf *lf, void *sa_dptr, void *sa_cptr,
 		  uint16_t sa_len)
 {
 	uintptr_t lmt_base = lf->lmt_base;
+	union cpt_res_s res, *hw_res;
 	uint64_t lmt_arg, io_addr;
 	struct cpt_inst_s *inst;
-	union cpt_res_s *res;
 	uint16_t lmt_id;
 	uint64_t *dptr;
 	int i;

@@ -927,8 +927,8 @@ roc_cpt_ctx_write(struct roc_cpt_lf *lf, void *sa_dptr, void *sa_cptr,
 
 	memset(inst, 0, sizeof(struct cpt_inst_s));
 
-	res = plt_zmalloc(sizeof(*res), ROC_CPT_RES_ALIGN);
-	if (res == NULL) {
+	hw_res = plt_zmalloc(sizeof(*hw_res), ROC_CPT_RES_ALIGN);
+	if (hw_res == NULL) {
 		plt_err("Couldn't allocate memory for result address");
 		return -ENOMEM;
 	}

@@ -936,7 +936,7 @@ roc_cpt_ctx_write(struct roc_cpt_lf *lf, void *sa_dptr, void *sa_cptr,
 	dptr = plt_zmalloc(sa_len, 8);
 	if (dptr == NULL) {
 		plt_err("Couldn't allocate memory for SA dptr");
-		plt_free(res);
+		plt_free(hw_res);
 		return -ENOMEM;
 	}
 

@@ -944,8 +944,8 @@ roc_cpt_ctx_write(struct roc_cpt_lf *lf, void *sa_dptr, void *sa_cptr,
 		dptr[i] = plt_cpu_to_be_64(((uint64_t *)sa_dptr)[i]);
 
 	/* Fill CPT_INST_S for WRITE_SA microcode op */
-	res->cn10k.compcode = CPT_COMP_NOT_DONE;
-	inst->res_addr = (uint64_t)res;
+	hw_res->cn10k.compcode = CPT_COMP_NOT_DONE;
+	inst->res_addr = (uint64_t)hw_res;
 	inst->dptr = (uint64_t)dptr;
 	inst->w4.s.param2 = sa_len >> 3;
 	inst->w4.s.dlen = sa_len;

@@ -959,14 +959,25 @@ roc_cpt_ctx_write(struct roc_cpt_lf *lf, void *sa_dptr, void *sa_cptr,
 	io_addr = lf->io_addr | ROC_CN10K_CPT_INST_DW_M1 << 4;
 
 	roc_lmt_submit_steorl(lmt_arg, io_addr);
-	plt_wmb();
+	plt_io_wmb();
+
+	/* Use 1 min timeout for the poll */
+	const uint64_t timeout = plt_tsc_cycles() + 60 * plt_tsc_hz();
 
 	/* Wait until CPT instruction completes */
-	while (res->cn10k.compcode == CPT_COMP_NOT_DONE)
-		plt_delay_ms(1);
+	do {
+		res.u64[0] = __atomic_load_n(&hw_res->u64[0], __ATOMIC_RELAXED);
+		if (unlikely(plt_tsc_cycles() > timeout))
+			break;
+	} while (res.cn10k.compcode == CPT_COMP_NOT_DONE);
 
-	plt_free(res);
 	plt_free(dptr);
+	plt_free(hw_res);
+
+	if (res.cn10k.compcode != CPT_COMP_WARN) {
+		plt_err("Write SA operation timed out");
+		return -ETIMEDOUT;
+	}
 
 	return 0;
 }

index 28004b1..86987ae 100644
@@ -184,9 +184,10 @@
 #define plt_memzone_reserve_aligned(name, len, flags, align)                   \
 	rte_memzone_reserve_aligned((name), (len), 0, (flags), (align))
 
-#define plt_tsc_hz   rte_get_tsc_hz
-#define plt_delay_ms rte_delay_ms
-#define plt_delay_us rte_delay_us
+#define plt_tsc_hz     rte_get_tsc_hz
+#define plt_tsc_cycles rte_get_tsc_cycles
+#define plt_delay_ms   rte_delay_ms
+#define plt_delay_us   rte_delay_us
 
 #define plt_lcore_id rte_lcore_id
 

index 3a2bf0f..d6ff134 100644
@@ -333,6 +333,8 @@ cn10k_sec_session_destroy(void *dev, struct rte_security_session *sec_sess)
 	struct cn10k_ipsec_sa *sa;
 	struct cnxk_cpt_qp *qp;
 	struct roc_cpt_lf *lf;
+	void *sa_dptr = NULL;
+	int ret;
 
 	sess = get_sec_session_private_data(sec_sess);
 	if (sess == NULL)

@@ -349,16 +351,44 @@ cn10k_sec_session_destroy(void *dev, struct rte_security_session *sec_sess)
 	/* Trigger CTX flush to write dirty data back to DRAM */
 	roc_cpt_lf_ctx_flush(lf, &sa->in_sa, false);
 
-	/* Wait for 1 ms so that flush is complete */
-	rte_delay_ms(1);
+	ret = -1;
 
-	w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
-	w2->s.valid = 0;
+	if (sa->is_outbound) {
+		sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_outb_sa), 8);
+		if (sa_dptr != NULL) {
+			roc_ot_ipsec_outb_sa_init(sa_dptr);
 
-	plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+			ret = roc_cpt_ctx_write(
+				lf, sa_dptr, &sa->out_sa,
+				sizeof(struct roc_ot_ipsec_outb_sa));
+		}
+	} else {
+		sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_inb_sa), 8);
+		if (sa_dptr != NULL) {
+			roc_ot_ipsec_inb_sa_init(sa_dptr, false);
+
+			ret = roc_cpt_ctx_write(
+				lf, sa_dptr, &sa->in_sa,
+				sizeof(struct roc_ot_ipsec_inb_sa));
+		}
+	}
 
-	/* Trigger CTX reload to fetch new data from DRAM */
-	roc_cpt_lf_ctx_reload(lf, &sa->in_sa);
+	plt_free(sa_dptr);
+
+	if (ret) {
+		/* MC write_ctx failed. Attempt reload of CTX */
+
+		/* Wait for 1 ms so that flush is complete */
+		rte_delay_ms(1);
+
+		w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
+		w2->s.valid = 0;
+
+		plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
+		/* Trigger CTX reload to fetch new data from DRAM */
+		roc_cpt_lf_ctx_reload(lf, &sa->in_sa);
+	}
 
 	sess_mp = rte_mempool_from_obj(sess);