support new full context firmware
cover support new full context firmware
This series adds support for new full context
microcode in cn9k IPsec PMD.
Anoob Joseph (1):
crypto/cnxk: improvements to fastpath handling
Tejasree Kondoj (1):
crypto/cnxk: add anti-replay as per new firmware
Vidya Sagar Velumuri (1):
crypto/cnxk: move IPsec SA creation to common
drivers/common/cnxk/cnxk_security.c | 398 ++++++++++++ drivers/common/cnxk/cnxk_security.h | 11 + drivers/common/cnxk/roc_cpt.c | 93 +++ drivers/common/cnxk/roc_cpt.h | 3 + drivers/common/cnxk/roc_ie_on.h | 24 +- drivers/common/cnxk/version.map | 3 + drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 96 ++- drivers/crypto/cnxk/cn9k_ipsec.c | 606 +++--------------- drivers/crypto/cnxk/cn9k_ipsec.h | 7 +- drivers/crypto/cnxk/cn9k_ipsec_la_ops.h | 125 +--- .../crypto/cnxk/cnxk_cryptodev_capabilities.c | 1 + drivers/crypto/cnxk/cnxk_cryptodev_ops.h | 13 +- 12 files changed, 736 insertions(+), 644 deletions(-)
Signature · 1 line
2.25.1
PATCH 1/3 crypto/cnxk: move IPsec SA creation to common
From: Vidya Sagar Velumuri <[email protected]>
Move the IPsec SA creation to common.
The code can be used by fastpath also to create the SAs Add changes to support new full context microcode
--- drivers/common/cnxk/cnxk_security.c | 398 +++++++++++++++ drivers/common/cnxk/cnxk_security.h | 11 + drivers/common/cnxk/roc_cpt.c | 93 ++++ drivers/common/cnxk/roc_cpt.h | 3 + drivers/common/cnxk/roc_ie_on.h | 21 +- drivers/common/cnxk/version.map | 3 + drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 24 + drivers/crypto/cnxk/cn9k_ipsec.c | 594 +++-------------------- drivers/crypto/cnxk/cn9k_ipsec_la_ops.h | 16 +- drivers/crypto/cnxk/cnxk_cryptodev_ops.h | 1 + 10 files changed, 631 insertions(+), 533 deletions(-)
drivers/common/cnxk/cnxk_security.c +398 -0
@@ -971,3 +971,401 @@ cnxk_ipsec_outb_rlens_get(struct cnxk_ipsec_outb_rlens *rlens, rlens->max_extended_len = partial_len + roundup_len + roundup_byte;return 0 ; } + +static inline int +on_ipsec_sa_ctl_set (struct rte_security_ipsec_xform *ipsec, +struct rte_crypto_sym_xform *crypto_xform, +struct roc_ie_on_sa_ctl *ctl) +{ +struct rte_crypto_sym_xform *cipher_xform, *auth_xform; +int aes_key_len =0 ; + +if (crypto_xform->type ==RTE_CRYPTO_SYM_XFORM_AUTH ) { + auth_xform = crypto_xform; + cipher_xform = crypto_xform->next; + }else { + cipher_xform = crypto_xform; + auth_xform = crypto_xform->next; + } + +if (ipsec->direction ==RTE_SECURITY_IPSEC_SA_DIR_EGRESS ) + ctl->direction =ROC_IE_SA_DIR_OUTBOUND ; +else + ctl->direction =ROC_IE_SA_DIR_INBOUND ; + +if (ipsec->mode ==RTE_SECURITY_IPSEC_SA_MODE_TUNNEL ) { +if (ipsec->tunnel.type ==RTE_SECURITY_IPSEC_TUNNEL_IPV4 ) + ctl->outer_ip_ver =ROC_IE_SA_IP_VERSION_4 ; +else if (ipsec->tunnel.type ==RTE_SECURITY_IPSEC_TUNNEL_IPV6 ) + ctl->outer_ip_ver =ROC_IE_SA_IP_VERSION_6 ; +else +return -EINVAL ; + } + +if (ipsec->mode ==RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT ) { + ctl->ipsec_mode =ROC_IE_SA_MODE_TRANSPORT ; + ctl->outer_ip_ver =ROC_IE_SA_IP_VERSION_4 ; + }else if (ipsec->mode ==RTE_SECURITY_IPSEC_SA_MODE_TUNNEL ) + ctl->ipsec_mode =ROC_IE_SA_MODE_TUNNEL ; +else +return -EINVAL ; + +if (ipsec->proto ==RTE_SECURITY_IPSEC_SA_PROTO_AH ) + ctl->ipsec_proto =ROC_IE_SA_PROTOCOL_AH ; +else if (ipsec->proto ==RTE_SECURITY_IPSEC_SA_PROTO_ESP ) + ctl->ipsec_proto =ROC_IE_SA_PROTOCOL_ESP ; +else +return -EINVAL ; + +if (crypto_xform->type ==RTE_CRYPTO_SYM_XFORM_AEAD ) { +switch (crypto_xform->aead.algo) { +case RTE_CRYPTO_AEAD_AES_GCM : + ctl->enc_type =ROC_IE_ON_SA_ENC_AES_GCM ; + aes_key_len = crypto_xform->aead.key.length; +break ; +default : +plt_err ("Unsupported AEAD algorithm" ); +return -ENOTSUP ; + } + }else { +if (cipher_xform !=NULL ) { +switch (cipher_xform->cipher.algo) { +case RTE_CRYPTO_CIPHER_NULL : + ctl->enc_type =ROC_IE_ON_SA_ENC_NULL ; +break ; +case RTE_CRYPTO_CIPHER_AES_CBC : + ctl->enc_type =ROC_IE_ON_SA_ENC_AES_CBC ; + aes_key_len = cipher_xform->cipher.key.length; +break ; +case RTE_CRYPTO_CIPHER_AES_CTR : + ctl->enc_type =ROC_IE_ON_SA_ENC_AES_CTR ; + aes_key_len = cipher_xform->cipher.key.length; +break ; +default : +plt_err ("Unsupported cipher algorithm" ); +return -ENOTSUP ; + } + } + +switch (auth_xform->auth.algo) { +case RTE_CRYPTO_AUTH_NULL : + ctl->auth_type =ROC_IE_ON_SA_AUTH_NULL ; +break ; +case RTE_CRYPTO_AUTH_MD5_HMAC : + ctl->auth_type =ROC_IE_ON_SA_AUTH_MD5 ; +break ; +case RTE_CRYPTO_AUTH_SHA1_HMAC : + ctl->auth_type =ROC_IE_ON_SA_AUTH_SHA1 ; +break ; +case RTE_CRYPTO_AUTH_SHA224_HMAC : + ctl->auth_type =ROC_IE_ON_SA_AUTH_SHA2_224 ; +break ; +case RTE_CRYPTO_AUTH_SHA256_HMAC : + ctl->auth_type =ROC_IE_ON_SA_AUTH_SHA2_256 ; +break ; +case RTE_CRYPTO_AUTH_SHA384_HMAC : + ctl->auth_type =ROC_IE_ON_SA_AUTH_SHA2_384 ; +break ; +case RTE_CRYPTO_AUTH_SHA512_HMAC : + ctl->auth_type =ROC_IE_ON_SA_AUTH_SHA2_512 ; +break ; +case RTE_CRYPTO_AUTH_AES_GMAC : + ctl->auth_type =ROC_IE_ON_SA_AUTH_AES_GMAC ; + aes_key_len = auth_xform->auth.key.length; +break ; +case RTE_CRYPTO_AUTH_AES_XCBC_MAC : + ctl->auth_type =ROC_IE_ON_SA_AUTH_AES_XCBC_128 ; +break ; +default : +plt_err ("Unsupported auth algorithm" ); +return -ENOTSUP ; + } + } + +/* Set AES key length */ +if (ctl->enc_type ==ROC_IE_ON_SA_ENC_AES_CBC || + ctl->enc_type ==ROC_IE_ON_SA_ENC_AES_CCM || + ctl->enc_type ==ROC_IE_ON_SA_ENC_AES_CTR || + ctl->enc_type ==ROC_IE_ON_SA_ENC_AES_GCM || + ctl->auth_type ==ROC_IE_ON_SA_AUTH_AES_GMAC ) { +switch (aes_key_len) { +case 16 : + ctl->aes_key_len =ROC_IE_SA_AES_KEY_LEN_128 ; +break ; +case 24 : + ctl->aes_key_len =ROC_IE_SA_AES_KEY_LEN_192 ; +break ; +case 32 : + ctl->aes_key_len =ROC_IE_SA_AES_KEY_LEN_256 ; +break ; +default : +plt_err ("Invalid AES key length" ); +return -EINVAL ; + } + } + +if (ipsec->options.esn) + ctl->esn_en =1 ; + +if (ipsec->options.udp_encap ==1 ) + ctl->encap_type =ROC_IE_ON_SA_ENCAP_UDP ; + + ctl->copy_df = ipsec->options.copy_df; + + ctl->spi =rte_cpu_to_be_32 (ipsec->spi); + +rte_io_wmb (); + + ctl->valid =1 ; + +return 0 ; +} + +static inline int +on_fill_ipsec_common_sa (struct rte_security_ipsec_xform *ipsec, +struct rte_crypto_sym_xform *crypto_xform, +struct roc_ie_on_common_sa *common_sa) +{ +struct rte_crypto_sym_xform *cipher_xform, *auth_xform; +const uint8_t *cipher_key; +int cipher_key_len =0 ; +int ret; + + ret =on_ipsec_sa_ctl_set (ipsec, crypto_xform, &common_sa->ctl); +if (ret) +return ret; + +if (crypto_xform->type ==RTE_CRYPTO_SYM_XFORM_AUTH ) { + auth_xform = crypto_xform; + cipher_xform = crypto_xform->next; + }else { + cipher_xform = crypto_xform; + auth_xform = crypto_xform->next; + } + +if (crypto_xform->type ==RTE_CRYPTO_SYM_XFORM_AEAD ) { +if (crypto_xform->aead.algo ==RTE_CRYPTO_AEAD_AES_GCM ) +memcpy (common_sa->iv.gcm.nonce, &ipsec->salt,4 ); + cipher_key = crypto_xform->aead.key.data; + cipher_key_len = crypto_xform->aead.key.length; + }else { +if (cipher_xform) { + cipher_key = cipher_xform->cipher.key.data; + cipher_key_len = cipher_xform->cipher.key.length; + } + +if (auth_xform->auth.algo ==RTE_CRYPTO_AUTH_AES_GMAC ) { +memcpy (common_sa->iv.gcm.nonce, &ipsec->salt,4 ); + cipher_key = auth_xform->auth.key.data; + cipher_key_len = auth_xform->auth.key.length; + } + } + +if (cipher_key_len !=0 ) +memcpy (common_sa->cipher_key, cipher_key, cipher_key_len); + +return 0 ; +} + +int +cnxk_on_ipsec_outb_sa_create (struct rte_security_ipsec_xform *ipsec, +struct rte_crypto_sym_xform *crypto_xform, +struct roc_ie_on_outb_sa *out_sa) +{ +struct roc_ie_on_ip_template *template =NULL ; +struct rte_crypto_sym_xform *auth_xform; +struct roc_ie_on_sa_ctl *ctl; +struct rte_ipv6_hdr *ip6; +struct rte_ipv4_hdr *ip4; +const uint8_t *auth_key; +int auth_key_len =0 ; +size_t ctx_len; +int ret; + + ctl = &out_sa->common_sa.ctl; + +if (crypto_xform->type ==RTE_CRYPTO_SYM_XFORM_AUTH ) + auth_xform = crypto_xform; +else + auth_xform = crypto_xform->next; + + ret =on_fill_ipsec_common_sa (ipsec, crypto_xform, &out_sa->common_sa); +if (ret) +return ret; + +if (ctl->enc_type ==ROC_IE_ON_SA_ENC_AES_GCM || + ctl->auth_type ==ROC_IE_ON_SA_AUTH_NULL || + ctl->auth_type ==ROC_IE_ON_SA_AUTH_AES_GMAC ) { + template = &out_sa->aes_gcm.template; + ctx_len = offsetof(struct roc_ie_on_outb_sa, aes_gcm.template); + }else { +switch (ctl->auth_type) { +case ROC_IE_ON_SA_AUTH_SHA1 : + template = &out_sa->sha1.template; + ctx_len = offsetof(struct roc_ie_on_outb_sa , + sha1.template); +break ; +case ROC_IE_ON_SA_AUTH_SHA2_256 : +case ROC_IE_ON_SA_AUTH_SHA2_384 : +case ROC_IE_ON_SA_AUTH_SHA2_512 : + template = &out_sa->sha2.template; + ctx_len = offsetof(struct roc_ie_on_outb_sa , + sha2.template); +break ; +case ROC_IE_ON_SA_AUTH_AES_XCBC_128 : + template = &out_sa->aes_xcbc.template; + ctx_len = offsetof(struct roc_ie_on_outb_sa , + aes_xcbc.template); +break ; +default : +plt_err ("Unsupported auth algorithm" ); +return -EINVAL ; + } + } + + ip4 = (struct rte_ipv4_hdr *)&template->ip4.ipv4_hdr; +if (ipsec->options.udp_encap) { + ip4->next_proto_id =IPPROTO_UDP ; + template->ip4.udp_src =rte_be_to_cpu_16 (4500 ); + template->ip4.udp_dst =rte_be_to_cpu_16 (4500 ); + }else { +if (ipsec->proto ==RTE_SECURITY_IPSEC_SA_PROTO_AH ) + ip4->next_proto_id =IPPROTO_AH ; +else + ip4->next_proto_id =IPPROTO_ESP ; + } + +if (ipsec->mode ==RTE_SECURITY_IPSEC_SA_MODE_TUNNEL ) { +if (ipsec->tunnel.type ==RTE_SECURITY_IPSEC_TUNNEL_IPV4 ) { +uint16_t frag_off =0 ; + + ctx_len +=sizeof (template->ip4); + + ip4->version_ihl =RTE_IPV4_VHL_DEF ; + ip4->time_to_live = ipsec->tunnel.ipv4.ttl; + ip4->type_of_service |= (ipsec->tunnel.ipv4.dscp <<2 ); +if (ipsec->tunnel.ipv4.df) + frag_off |=RTE_IPV4_HDR_DF_FLAG ; + ip4->fragment_offset =rte_cpu_to_be_16 (frag_off); + +memcpy (&ip4->src_addr, &ipsec->tunnel.ipv4.src_ip, +sizeof (struct in_addr )); +memcpy (&ip4->dst_addr, &ipsec->tunnel.ipv4.dst_ip, +sizeof (struct in_addr )); + }else if (ipsec->tunnel.type == +RTE_SECURITY_IPSEC_TUNNEL_IPV6 ) { + ctx_len +=sizeof (template->ip6); + + ip6 = (struct rte_ipv6_hdr *)&template->ip6.ipv6_hdr; +if (ipsec->options.udp_encap) { + ip6->proto =IPPROTO_UDP ; + template->ip6.udp_src =rte_be_to_cpu_16 (4500 ); + template->ip6.udp_dst =rte_be_to_cpu_16 (4500 ); + }else { + ip6->proto = (ipsec->proto == +RTE_SECURITY_IPSEC_SA_PROTO_ESP ) ? +IPPROTO_ESP : +IPPROTO_AH ; + } + ip6->vtc_flow = +rte_cpu_to_be_32 (0x60000000 | + ((ipsec->tunnel.ipv6.dscp + <<RTE_IPV6_HDR_TC_SHIFT ) & +RTE_IPV6_HDR_TC_MASK ) | + ((ipsec->tunnel.ipv6.flabel + <<RTE_IPV6_HDR_FL_SHIFT ) & +RTE_IPV6_HDR_FL_MASK )); + ip6->hop_limits = ipsec->tunnel.ipv6.hlimit; +memcpy (&ip6->src_addr, &ipsec->tunnel.ipv6.src_addr, +sizeof (struct in6_addr )); +memcpy (&ip6->dst_addr, &ipsec->tunnel.ipv6.dst_addr, +sizeof (struct in6_addr )); + } + }else + ctx_len +=sizeof (template->ip4); + + ctx_len +=RTE_ALIGN_CEIL (ctx_len,8 ); + +if (crypto_xform->type !=RTE_CRYPTO_SYM_XFORM_AEAD ) { + auth_key = auth_xform->auth.key.data; + auth_key_len = auth_xform->auth.key.length; + +switch (auth_xform->auth.algo) { +case RTE_CRYPTO_AUTH_AES_GMAC : +case RTE_CRYPTO_AUTH_NULL : +break ; +case RTE_CRYPTO_AUTH_SHA1_HMAC : +memcpy (out_sa->sha1.hmac_key, auth_key, auth_key_len); +break ; +case RTE_CRYPTO_AUTH_SHA256_HMAC : +case RTE_CRYPTO_AUTH_SHA384_HMAC : +case RTE_CRYPTO_AUTH_SHA512_HMAC : +memcpy (out_sa->sha2.hmac_key, auth_key, auth_key_len); +break ; +case RTE_CRYPTO_AUTH_AES_XCBC_MAC : +memcpy (out_sa->aes_xcbc.key, auth_key, auth_key_len); +break ; +default : +plt_err ("Unsupported auth algorithm %u" , + auth_xform->auth.algo); +return -ENOTSUP ; + } + } + +return ctx_len; +} + +int +cnxk_on_ipsec_inb_sa_create (struct rte_security_ipsec_xform *ipsec, +struct rte_crypto_sym_xform *crypto_xform, +struct roc_ie_on_inb_sa *in_sa) +{ +struct rte_crypto_sym_xform *auth_xform = crypto_xform; +const uint8_t *auth_key; +int auth_key_len =0 ; +size_t ctx_len =0 ; +int ret; + + ret =on_fill_ipsec_common_sa (ipsec, crypto_xform, &in_sa->common_sa); +if (ret) +return ret; + +if (crypto_xform->type ==RTE_CRYPTO_SYM_XFORM_AEAD || + auth_xform->auth.algo ==RTE_CRYPTO_AUTH_NULL || + auth_xform->auth.algo ==RTE_CRYPTO_AUTH_AES_GMAC ) { + ctx_len = offsetof(struct roc_ie_on_inb_sa , + sha1_or_gcm.hmac_key[0]); + }else { + auth_key = auth_xform->auth.key.data; + auth_key_len = auth_xform->auth.key.length; + +switch (auth_xform->auth.algo) { +case RTE_CRYPTO_AUTH_NULL : +break ; +case RTE_CRYPTO_AUTH_SHA1_HMAC : +memcpy (in_sa->sha1_or_gcm.hmac_key, auth_key, + auth_key_len); + ctx_len = offsetof(struct roc_ie_on_inb_sa , + sha1_or_gcm.selector); +break ; +case RTE_CRYPTO_AUTH_SHA256_HMAC : +case RTE_CRYPTO_AUTH_SHA384_HMAC : +case RTE_CRYPTO_AUTH_SHA512_HMAC : +memcpy (in_sa->sha2.hmac_key, auth_key, auth_key_len); + ctx_len = offsetof(struct roc_ie_on_inb_sa , + sha2.selector); +break ; +case RTE_CRYPTO_AUTH_AES_XCBC_MAC : +memcpy (in_sa->aes_xcbc.key, auth_key, auth_key_len); + ctx_len = offsetof(struct roc_ie_on_inb_sa , + aes_xcbc.selector); +break ; +default : +plt_err ("Unsupported auth algorithm %u" , + auth_xform->auth.algo); +return -ENOTSUP ; + } + } + +return ctx_len; +}
drivers/common/cnxk/cnxk_security.h +11 -0
@@ -59,4 +59,15 @@ cnxk_onf_ipsec_outb_sa_fill(struct roc_onf_ipsec_outb_sa *sa, bool __roc_api cnxk_onf_ipsec_inb_sa_valid(struct roc_onf_ipsec_inb_sa *sa); bool __roc_api cnxk_onf_ipsec_outb_sa_valid(struct roc_onf_ipsec_outb_sa *sa); +/* [CN9K] */ +int __roc_api +cnxk_on_ipsec_inb_sa_create (struct rte_security_ipsec_xform *ipsec, +struct rte_crypto_sym_xform *crypto_xform, +struct roc_ie_on_inb_sa *in_sa); + +int __roc_api +cnxk_on_ipsec_outb_sa_create (struct rte_security_ipsec_xform *ipsec, +struct rte_crypto_sym_xform *crypto_xform, +struct roc_ie_on_outb_sa *out_sa); +#endif /* _CNXK_SECURITY_H__ */
drivers/common/cnxk/roc_cpt.c +93 -0
96 lines not on this page · raw message
drivers/common/cnxk/roc_cpt.h +3 -0
@@ -181,4 +181,7 @@ void __roc_api roc_cpt_parse_hdr_dump(const struct cpt_parse_hdr_s *cpth); int __roc_api roc_cpt_ctx_write(struct roc_cpt_lf *lf, void *sa_dptr,void *sa_cptr,uint16_t sa_len); +int __roc_api roc_on_cpt_ctx_write(struct roc_cpt_lf *lf, void *sa, +uint8_t opcode,uint16_t ctx_len, +uint8_t egrp);#endif /* _ROC_CPT_H_ */
drivers/common/cnxk/roc_ie_on.h +17 -4
@@ -23,7 +23,7 @@ enum roc_ie_on_ucc_ipsec { };/* Helper macros */ -#define ROC_IE_ON_INB_RPTR_HDR 0x8 +#define ROC_IE_ON_INB_RPTR_HDR 16#define ROC_IE_ON_MAX_IV_LEN 16#define ROC_IE_ON_PER_PKT_IV BIT(43)
@@ -67,9 +67,17 @@ enum {struct roc_ie_on_outb_hdr {uint32_t ip_id;uint32_t seq; +uint32_t esn; +uint32_t df_tos;uint8_t iv[16 ]; }; +struct roc_ie_on_inb_hdr { +uint32_t sa_index; +uint64_t seq; +uint32_t pad; +}; +union roc_ie_on_bit_perfect_iv {uint8_t aes_iv[16 ];uint8_t des_iv[8 ];
@@ -113,7 +121,7 @@ struct roc_ie_on_ip_template {union roc_on_ipsec_outb_param1 {uint16_t u16;struct { -uint16_t frag_num :4 ; +uint16_t l2hdr_len :4 ;uint16_t rsvd_4_6 :3 ;uint16_t gre_select :1 ;uint16_t dsiv :1 ;
@@ -171,8 +179,13 @@ struct roc_ie_on_common_sa {union roc_ie_on_bit_perfect_iv iv;/* w7 */ -uint32_t esn_hi; -uint32_t esn_low; +union { +uint64_t u64; +struct { +uint32_t th; +uint32_t tl; + }; + } seq_t; };struct roc_ie_on_outb_sa {
drivers/common/cnxk/version.map +3 -0
@@ -23,6 +23,8 @@ INTERNAL { cnxk_ot_ipsec_outb_sa_fill; cnxk_ot_ipsec_inb_sa_valid; cnxk_ot_ipsec_outb_sa_valid; + cnxk_on_ipsec_inb_sa_create; + cnxk_on_ipsec_outb_sa_create; roc_ae_ec_grp_get; roc_ae_ec_grp_put; roc_ae_fpm_get;
@@ -72,6 +74,7 @@ INTERNAL { roc_cpt_parse_hdr_dump; roc_cpt_rxc_time_cfg; roc_cpt_ctx_write; + roc_on_cpt_ctx_write; roc_dpi_configure; roc_dpi_dev_fini; roc_dpi_dev_init;
drivers/crypto/cnxk/cn9k_cryptodev_ops.c +24 -0
@@ -43,7 +43,9 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op, struct cpt_inst_s *inst) {struct rte_crypto_sym_op *sym_op = op->sym; +struct roc_ie_on_common_sa *common_sa;struct cn9k_sec_session *priv; +struct roc_ie_on_sa_ctl *ctl;struct cn9k_ipsec_sa *sa;if (unlikely (sym_op->m_dst && sym_op->m_dst != sym_op->m_src)) {
@@ -64,6 +66,12 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op, infl_req->op_flags |=CPT_OP_FLAGS_IPSEC_DIR_INBOUND ; + common_sa = &sa->in_sa.common_sa; + ctl = &common_sa->ctl; + +if (ctl->esn_en) + infl_req->op_flags |=CPT_OP_FLAGS_IPSEC_INB_ESN ; +return process_inb_sa (op, sa, inst); }
@@ -491,14 +499,28 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop, {struct rte_crypto_sym_op *sym_op = cop->sym;struct rte_mbuf *m = sym_op->m_src; +struct cn9k_sec_session *priv; +struct cn9k_ipsec_sa *sa;struct rte_ipv6_hdr *ip6;struct rte_ipv4_hdr *ip;uint16_t m_len =0 ;char *data; + priv =get_sec_session_private_data (cop->sym->sec_session); + sa = &priv->sa; +if (infl_req->op_flags &CPT_OP_FLAGS_IPSEC_DIR_INBOUND ) { +struct roc_ie_on_common_sa *common_sa = &sa->in_sa.common_sa; + data = rte_pktmbuf_mtod(m, char *); +if (infl_req->op_flags ==CPT_OP_FLAGS_IPSEC_INB_ESN ) { +struct roc_ie_on_inb_hdr *inb_hdr = + (struct roc_ie_on_inb_hdr *)data; +uint64_t seq =rte_be_to_cpu_64 (inb_hdr->seq); +if (seq > common_sa->seq_t.u64) + common_sa->seq_t.u64 = seq; + } ip = (struct rte_ipv4_hdr *)(data +ROC_IE_ON_INB_RPTR_HDR ); if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) ==
@@ -515,6 +537,8 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop, m->data_len = m_len; m->pkt_len = m_len; m->data_off +=ROC_IE_ON_INB_RPTR_HDR ; + }else { +rte_pktmbuf_adj (m, sa->custom_hdr_len); } }
drivers/crypto/cnxk/cn9k_ipsec.c +75 -519
678 lines not on this page · raw message
drivers/crypto/cnxk/cn9k_ipsec_la_ops.h +6 -10
@@ -20,7 +20,7 @@ ipsec_po_out_rlen_get(struct cn9k_ipsec_sa *sa, uint32_t plen) enc_payload_len =RTE_ALIGN_CEIL (plen + sa->rlens.roundup_len, sa->rlens.roundup_byte); -return sa->rlens.partial_len + enc_payload_len; +return sa->custom_hdr_len + sa->rlens.partial_len + enc_payload_len; }static __rte_always_inline int
@@ -41,8 +41,8 @@ ipsec_antireplay_check(struct cn9k_ipsec_sa *sa, uint32_t win_sz, ctl = &common_sa->ctl; esn = ctl->esn_en; - esn_low =rte_be_to_cpu_32 (common_sa->esn_low); - esn_hi =rte_be_to_cpu_32 (common_sa->esn_hi); + esn_low =rte_be_to_cpu_32 (common_sa->seq_t.tl); + esn_hi =rte_be_to_cpu_32 (common_sa->seq_t.th); esp = rte_pktmbuf_mtod_offset(m, void *, sizeof(struct rte_ipv4_hdr)); seql =rte_be_to_cpu_32 (esp->seq);
@@ -62,8 +62,8 @@ ipsec_antireplay_check(struct cn9k_ipsec_sa *sa, uint32_t win_sz,if (esn && !ret) { seq_in_sa = ((uint64_t )esn_hi <<32 ) | esn_low;if (seq > seq_in_sa) { - common_sa->esn_low =rte_cpu_to_be_32 (seql); - common_sa->esn_hi =rte_cpu_to_be_32 (seqh); + common_sa->seq_t.tl =rte_cpu_to_be_32 (seql); + common_sa->seq_t.th =rte_cpu_to_be_32 (seqh); } }
@@ -77,13 +77,10 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_ipsec_sa *sa,const unsigned int hdr_len = sa->custom_hdr_len;struct rte_crypto_sym_op *sym_op = cop->sym;struct rte_mbuf *m_src = sym_op->m_src; -struct roc_ie_on_outb_sa *out_sa;struct roc_ie_on_outb_hdr *hdr;uint32_t dlen, rlen;int32_t extend_tail; - out_sa = &sa->out_sa; - dlen =rte_pktmbuf_pkt_len (m_src) + hdr_len; rlen =ipsec_po_out_rlen_get (sa, dlen - hdr_len);
@@ -114,8 +111,7 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_ipsec_sa *sa, hdr->seq =rte_cpu_to_be_32 (sa->seq_lo); hdr->ip_id =rte_cpu_to_be_32 (sa->ip_id); - - out_sa->common_sa.esn_hi = sa->seq_hi; + hdr->esn =rte_cpu_to_be_32 (sa->seq_hi); sa->ip_id++; sa->esn++;
drivers/crypto/cnxk/cnxk_cryptodev_ops.h +1 -0
@@ -33,6 +33,7 @@ struct cpt_qp_meta_info {#define CPT_OP_FLAGS_METABUF (1 << 1)#define CPT_OP_FLAGS_AUTH_VERIFY (1 << 0)#define CPT_OP_FLAGS_IPSEC_DIR_INBOUND (1 << 2) +#define CPT_OP_FLAGS_IPSEC_INB_ESN (1 << 3)struct cpt_inflight_req {union cpt_res_s res;
Signature · 1 line
2.25.1
PATCH 2/3 crypto/cnxk: improvements to fastpath handling
In reply to Tejasree KondojFrom: Anoob Joseph <[email protected]>
Remove SA & packet accesses in dequeue path by adjusting the headers in the enqueue path for outbound packets. For inbound packets, add extra esn_en flag in the SA to minimize cache line accesses in the datapath.
Also, use seq_lo for IPID. IPID just need to be unique. Instead of incrementing per packet, use ESN low bits.
--- drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 69 ++++++++++++++---------- drivers/crypto/cnxk/cn9k_ipsec.c | 11 ++-- drivers/crypto/cnxk/cn9k_ipsec.h | 7 ++- drivers/crypto/cnxk/cn9k_ipsec_la_ops.h | 55 +++++++++++-------- drivers/crypto/cnxk/cnxk_cryptodev_ops.h | 12 ++--- 5 files changed, 87 insertions(+), 67 deletions(-)
drivers/crypto/cnxk/cn9k_cryptodev_ops.c +40 -29
@@ -43,10 +43,12 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op, struct cpt_inst_s *inst) {struct rte_crypto_sym_op *sym_op = op->sym; -struct roc_ie_on_common_sa *common_sa;struct cn9k_sec_session *priv; -struct roc_ie_on_sa_ctl *ctl;struct cn9k_ipsec_sa *sa; +int ret; + + priv =get_sec_session_private_data (op->sym->sec_session); + sa = &priv->sa;if (unlikely (sym_op->m_dst && sym_op->m_dst != sym_op->m_src)) {plt_dp_err ("Out of place is not supported" );
@@ -58,21 +60,17 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,return -ENOTSUP ; } - priv =get_sec_session_private_data (op->sym->sec_session); - sa = &priv->sa; -if (sa->dir ==RTE_SECURITY_IPSEC_SA_DIR_EGRESS ) -return process_outb_sa (op, sa, inst); - - infl_req->op_flags |=CPT_OP_FLAGS_IPSEC_DIR_INBOUND ; - - common_sa = &sa->in_sa.common_sa; - ctl = &common_sa->ctl; - -if (ctl->esn_en) - infl_req->op_flags |=CPT_OP_FLAGS_IPSEC_INB_ESN ; + ret =process_outb_sa (op, sa, inst); +else { + infl_req->op_flags |=CPT_OP_FLAGS_IPSEC_DIR_INBOUND ; +process_inb_sa (op, sa, inst); +if (unlikely (sa->esn_en)) + infl_req->op_flags |=CPT_OP_FLAGS_IPSEC_INB_ESN ; + ret =0 ; + } -return process_inb_sa (op, sa, inst); +return ret; } static inline struct cnxk_se_sess *
@@ -234,19 +232,29 @@ cn9k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops) }; pend_q = &qp->pend_q; - -const uint64_t lmt_base = qp->lf.lmt_base; -const uint64_t io_addr = qp->lf.io_addr; -const uint64_t pq_mask = pend_q->pq_mask; +rte_prefetch2 (pend_q);/* Clear w0, w2, w3 of both inst */ +#if defined(RTE_ARCH_ARM64 ) +uint64x2_t zero =vdupq_n_u64 (0 ); + +vst1q_u64 (&inst[0 ].w0.u64, zero); +vst1q_u64 (&inst[1 ].w0.u64, zero); +vst1q_u64 (&inst[0 ].w2.u64, zero); +vst1q_u64 (&inst[1 ].w2.u64, zero); +#else inst[0 ].w0.u64 =0 ; inst[0 ].w2.u64 =0 ; inst[0 ].w3.u64 =0 ; inst[1 ].w0.u64 =0 ; inst[1 ].w2.u64 =0 ; inst[1 ].w3.u64 =0 ; +#endif + +const uint64_t lmt_base = qp->lf.lmt_base; +const uint64_t io_addr = qp->lf.io_addr; +const uint64_t pq_mask = pend_q->pq_mask; head = pend_q->head; nb_allowed =pending_queue_free_cnt (head, pend_q->tail, pq_mask);
@@ -506,21 +514,26 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop,uint16_t m_len =0 ;char *data; - priv =get_sec_session_private_data (cop->sym->sec_session); - sa = &priv->sa; -if (infl_req->op_flags &CPT_OP_FLAGS_IPSEC_DIR_INBOUND ) { -struct roc_ie_on_common_sa *common_sa = &sa->in_sa.common_sa; +struct roc_ie_on_common_sa *common_sa; data = rte_pktmbuf_mtod(m, char *); -if (infl_req->op_flags ==CPT_OP_FLAGS_IPSEC_INB_ESN ) { -struct roc_ie_on_inb_hdr *inb_hdr = - (struct roc_ie_on_inb_hdr *)data; -uint64_t seq =rte_be_to_cpu_64 (inb_hdr->seq); +if (unlikely (infl_req->op_flags &CPT_OP_FLAGS_IPSEC_INB_ESN )) { +struct roc_ie_on_inb_hdr *inb_hdr; +uint64_t seq; + + priv =get_sec_session_private_data ( + sym_op->sec_session); + sa = &priv->sa; + common_sa = &sa->in_sa.common_sa; + + inb_hdr = (struct roc_ie_on_inb_hdr *)data; + seq =rte_be_to_cpu_64 (inb_hdr->seq);if (seq > common_sa->seq_t.u64) common_sa->seq_t.u64 = seq; } + ip = (struct rte_ipv4_hdr *)(data +ROC_IE_ON_INB_RPTR_HDR ); if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) ==
@@ -537,8 +550,6 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop, m->data_len = m_len; m->pkt_len = m_len; m->data_off +=ROC_IE_ON_INB_RPTR_HDR ; - }else { -rte_pktmbuf_adj (m, sa->custom_hdr_len); } }
drivers/crypto/cnxk/cn9k_ipsec.c +4 -7
@@ -40,13 +40,8 @@ cn9k_ipsec_outb_sa_create(struct cnxk_cpt_qp *qp,/* Initialize lookaside IPsec private data */ sa->dir =RTE_SECURITY_IPSEC_SA_DIR_EGRESS ; -/* Start ip id from 1 */ - sa->ip_id =1 ; - sa->seq_lo =1 ; - sa->seq_hi =0 ; -if (ipsec->esn.value) - sa->esn = ipsec->esn.value; + sa->esn = ipsec->esn.value; ret =cnxk_ipsec_outb_rlens_get (&sa->rlens, ipsec, crypto_xform);if (ret)
@@ -166,10 +161,12 @@ cn9k_ipsec_inb_sa_create(struct cnxk_cpt_qp *qp, } ret =cnxk_on_ipsec_inb_sa_create (ipsec, crypto_xform, &sa->in_sa); -if (ret <0 )return ret; +if (sa->in_sa.common_sa.ctl.esn_en) + sa->esn_en =1 ; + ctx_len = ret; opcode =ROC_IE_ON_MAJOR_OP_WRITE_IPSEC_INBOUND ; egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE ];
drivers/crypto/cnxk/cn9k_ipsec.h +5 -2
@@ -28,8 +28,6 @@ struct cn9k_ipsec_sa {uint8_t custom_hdr_len;/** Response length calculation data */ struct cnxk_ipsec_outb_rlens rlens; -/** Outbound IP-ID */ -uint16_t ip_id;/** ESN */ union {uint64_t esn;
@@ -42,6 +40,11 @@ struct cn9k_ipsec_sa {struct cnxk_on_ipsec_ar ar;/** Anti replay window size */ uint32_t replay_win_sz; +/* +* ESN enable flag. Copy of in_sa ctl.esn_en to have single cache line +* access in the non-esn fastpath. +*/ +uint8_t esn_en;/** Queue pair */ struct cnxk_cpt_qp *qp; };
drivers/crypto/cnxk/cn9k_ipsec_la_ops.h +32 -23
100 lines not on this page · raw message
drivers/crypto/cnxk/cnxk_cryptodev_ops.h +6 -6
@@ -70,16 +70,16 @@ struct cnxk_cpt_qp {/**< Crypto LF */ struct pending_queue pend_q;/**< Pending queue */ -struct rte_mempool *sess_mp; -/**< Session mempool */ -struct rte_mempool *sess_mp_priv; -/**< Session private data mempool */ -struct cpt_qp_meta_info meta_info; -/**< Metabuf info required to support operations on the queue pair */ struct roc_cpt_lmtline lmtline;/**< Lmtline information */ +struct cpt_qp_meta_info meta_info; +/**< Metabuf info required to support operations on the queue pair */ struct crypto_adpter_info ca;/**< Crypto adapter related info */ +struct rte_mempool *sess_mp; +/**< Session mempool */ +struct rte_mempool *sess_mp_priv; +/**< Session private data mempool */ }; int cnxk_cpt_dev_config(struct rte_cryptodev *dev,
Signature · 1 line
2.25.1
PATCH 3/3 crypto/cnxk: add anti-replay as per new firmware
In reply to Tejasree KondojAdding anti-replay changes as per new FP-FC microcode.
--- drivers/common/cnxk/roc_ie_on.h | 5 +- drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 63 +++++++++++++---- drivers/crypto/cnxk/cn9k_ipsec.c | 3 + drivers/crypto/cnxk/cn9k_ipsec_la_ops.h | 68 ------------------- .../crypto/cnxk/cnxk_cryptodev_capabilities.c | 1 + drivers/crypto/cnxk/cnxk_cryptodev_ops.h | 2 +- 6 files changed, 58 insertions(+), 84 deletions(-)
drivers/common/cnxk/roc_ie_on.h +2 -3
@@ -18,8 +18,6 @@ enum roc_ie_on_ucc_ipsec {ROC_IE_ON_UCC_SUCCESS =0 ,ROC_IE_ON_AUTH_UNSUPPORTED =0xB0 , ROC_IE_ON_ENCRYPT_UNSUPPORTED = 0xB1, -/* Software defined completion code for anti-replay failed packets */ - ROC_IE_ON_SWCC_ANTI_REPLAY = 0xE7, };/* Helper macros */
@@ -74,7 +72,8 @@ struct roc_ie_on_outb_hdr {struct roc_ie_on_inb_hdr {uint32_t sa_index; -uint64_t seq; +uint32_t seql; +uint32_t seqh;uint32_t pad; };
drivers/crypto/cnxk/cn9k_cryptodev_ops.c +51 -12
88 lines not on this page · raw message
drivers/crypto/cnxk/cn9k_ipsec.c +3 -0
@@ -156,6 +156,9 @@ cn9k_ipsec_inb_sa_create(struct cnxk_cpt_qp *qp, sa->ar.wint = sa->replay_win_sz; sa->ar.base = sa->replay_win_sz; + sa->seq_lo = ipsec->esn.low; + sa->seq_hi = ipsec->esn.hi; + sa->in_sa.common_sa.seq_t.tl = sa->seq_lo; sa->in_sa.common_sa.seq_t.th = sa->seq_hi; }
drivers/crypto/cnxk/cn9k_ipsec_la_ops.h +0 -68
80 lines not on this page · raw message
drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c +1 -0
@@ -1269,6 +1269,7 @@ cn9k_sec_caps_update(struct rte_security_capability *sec_cap)#endif } sec_cap->ipsec.replay_win_sz_max =CNXK_ON_AR_WIN_SIZE_MAX ; + sec_cap->ipsec.options.esn =1 ; }void
drivers/crypto/cnxk/cnxk_cryptodev_ops.h +1 -1
@@ -33,7 +33,7 @@ struct cpt_qp_meta_info {#define CPT_OP_FLAGS_METABUF (1 << 1)#define CPT_OP_FLAGS_AUTH_VERIFY (1 << 0)#define CPT_OP_FLAGS_IPSEC_DIR_INBOUND (1 << 2) -#define CPT_OP_FLAGS_IPSEC_INB_ESN (1 << 3) +#define CPT_OP_FLAGS_IPSEC_INB_REPLAY (1 << 3)struct cpt_inflight_req {union cpt_res_s res;
Signature · 1 line
2.25.1
Quoted 12 lines
This series adds support for new full context
microcode in cn9k IPsec PMD.Anoob Joseph (1):
crypto/cnxk: improvements to fastpath handlingTejasree Kondoj (1):
crypto/cnxk: add anti-replay as per new firmwareVidya Sagar Velumuri (1):
crypto/cnxk: move IPsec SA creation to common
Quoted 13 lines
drivers/common/cnxk/cnxk_security.c | 398 ++++++++++++ drivers/common/cnxk/cnxk_security.h | 11 + drivers/common/cnxk/roc_cpt.c | 93 +++ drivers/common/cnxk/roc_cpt.h | 3 + drivers/common/cnxk/roc_ie_on.h | 24 +- drivers/common/cnxk/version.map | 3 + drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 96 ++- drivers/crypto/cnxk/cn9k_ipsec.c | 606 +++--------------- drivers/crypto/cnxk/cn9k_ipsec.h | 7 +- drivers/crypto/cnxk/cn9k_ipsec_la_ops.h | 125 +--- .../crypto/cnxk/cnxk_cryptodev_capabilities.c | 1 + drivers/crypto/cnxk/cnxk_cryptodev_ops.h | 13 +- 12 files changed, 736 insertions(+), 644 deletions(-)
Series
Applied to dpdk-next-crypto
Thanks.