Skip to contentSkip to conversation

crypto/mlx5: support AES-GCM

RFC PATCH Superseded Suanming Mou 4 participants 54 messages Apr 18, 2023 – Jun 23, 2023

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover crypto/mlx5: support AES-GCM

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE. The UMR WQE and crypto WQE are handled in two different QPs.

The QP for UMR operation contains two types of WQE, UMR and SEND_EN WQE. The WQEs are built dynamically according to the crypto operation buffer address. Crypto operation with non-contiguous buffers will have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP. The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.

Suanming Mou (5):
crypto/mlx5: add AES-GCM capability
crypto/mlx5: add AES-GCM encryption key
crypto/mlx5: add AES-GCM session configure
crypto/mlx5: add queue pair setup
crypto/mlx5: add enqueue and dequeue operations

 doc/guides/nics/mlx5.rst              |   8 +
 drivers/common/mlx5/mlx5_devx_cmds.c  |  29 +-
 drivers/common/mlx5/mlx5_devx_cmds.h  |  18 +
 drivers/common/mlx5/mlx5_prm.h        |  62 +-
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  64 +-
 drivers/crypto/mlx5/mlx5_crypto.h     |  57 +-
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 157 +++--
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 803 ++++++++++++++++++++++++++
 9 files changed, 1139 insertions(+), 60 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_gcm.c
Signature · 1 line
2.25.1
#1 lore raw

RFC PATCH 1/5 crypto/mlx5: add AES-GCM capability

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

This commit adds the AES-GCM capability query and check. An new devarg "algo" is added to identify if the crypto PMD will be initialized as AES-GCM(algo=1) or AES-XTS(algo=0, default).

---
 doc/guides/nics/mlx5.rst              |   8 +++
 drivers/common/mlx5/mlx5_devx_cmds.c  |  17 +++++
 drivers/common/mlx5/mlx5_devx_cmds.h  |  14 ++++
 drivers/common/mlx5/mlx5_prm.h        |  19 ++++-
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  30 +++++++-
 drivers/crypto/mlx5/mlx5_crypto.h     |   5 ++
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 100 ++++++++++++++++++++++++++
 8 files changed, 189 insertions(+), 5 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_gcm.c
doc/guides/nics/mlx5.rst +8 -0
index 9d111ed436..5eb2150613 100644
@@ -1270,6 +1270,14 @@ for an additional list of options shared with other mlx5 drivers.
 
   Set to zero by default.
 
+- ``algo`` parameter [int]
+
+  - 0. AES-XTS crypto.
+
+  - 1. AES-GCM crypto.
+
+  Set to zero(AES-XTS) by default.
+
 Supported NICs
 --------------
 
drivers/common/mlx5/mlx5_devx_cmds.c +17 -0
index 96d3e3e373..592a7cffdb 100644
@@ -1197,6 +1197,23 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		attr->crypto_wrapped_import_method = !!(MLX5_GET(crypto_caps,
 						hcattr, wrapped_import_method)
 						& 1 << 2);
+		attr->sw_wrapped_dek = MLX5_GET(crypto_caps, hcattr, sw_wrapped_dek_key_purpose) ?
+			MLX5_GET(crypto_caps, hcattr, sw_wrapped_dek_new) : 0;
+		attr->crypto_mmo.crypto_mmo_qp = MLX5_GET(crypto_caps, hcattr, crypto_mmo_qp);
+		attr->crypto_mmo.gcm_256_encrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_256_encrypt);
+		attr->crypto_mmo.gcm_128_encrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_128_encrypt);
+		attr->crypto_mmo.gcm_256_decrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_256_decrypt);
+		attr->crypto_mmo.gcm_128_decrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_128_decrypt);
+		attr->crypto_mmo.gcm_auth_tag_128 =
+			MLX5_GET(crypto_caps, hcattr, gcm_auth_tag_128);
+		attr->crypto_mmo.gcm_auth_tag_96 =
+			MLX5_GET(crypto_caps, hcattr, gcm_auth_tag_96);
+		attr->crypto_mmo.log_crypto_mmo_max_size =
+			MLX5_GET(crypto_caps, hcattr, log_crypto_mmo_max_size);
 	}
 	if (hca_cap_2_sup) {
 		hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
drivers/common/mlx5/mlx5_devx_cmds.h +14 -0
index e006a04d68..d640482346 100644
@@ -153,6 +153,18 @@ struct mlx5_hca_ipsec_attr {
 	struct mlx5_hca_ipsec_reformat_attr reformat_fdb;
 };
 
+__extension__
+struct mlx5_hca_crypto_mmo_attr {
+	uint32_t crypto_mmo_qp:1;
+	uint32_t gcm_256_encrypt:1;
+	uint32_t gcm_128_encrypt:1;
+	uint32_t gcm_256_decrypt:1;
+	uint32_t gcm_128_decrypt:1;
+	uint32_t gcm_auth_tag_128:1;
+	uint32_t gcm_auth_tag_96:1;
+	uint32_t log_crypto_mmo_max_size:6;
+};
+
 /* ISO C restricts enumerator values to range of 'int' */
 __extension__
 enum {
@@ -266,6 +278,7 @@ struct mlx5_hca_attr {
 	uint32_t import_kek:1; /* General obj type IMPORT_KEK supported. */
 	uint32_t credential:1; /* General obj type CREDENTIAL supported. */
 	uint32_t crypto_login:1; /* General obj type CRYPTO_LOGIN supported. */
+	uint32_t sw_wrapped_dek:16; /* DEKs wrapped by SW are supported */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t inner_ipv4_ihl:1;
@@ -281,6 +294,7 @@ struct mlx5_hca_attr {
 	struct mlx5_hca_flow_attr flow;
 	struct mlx5_hca_flex_attr flex;
 	struct mlx5_hca_ipsec_attr ipsec;
+	struct mlx5_hca_crypto_mmo_attr crypto_mmo;
 	int log_max_qp_sz;
 	int log_max_cq_sz;
 	int log_max_qp;
drivers/common/mlx5/mlx5_prm.h +16 -3
index 31db082c50..a3b85f514e 100644
@@ -4654,7 +4654,9 @@ struct mlx5_ifc_crypto_caps_bits {
 	u8 synchronize_dek[0x1];
 	u8 int_kek_manual[0x1];
 	u8 int_kek_auto[0x1];
-	u8 reserved_at_6[0x12];
+	u8 reserved_at_6[0xd];
+	u8 sw_wrapped_dek_key_purpose[0x1];
+	u8 reserved_at_14[0x4];
 	u8 wrapped_import_method[0x8];
 	u8 reserved_at_20[0x3];
 	u8 log_dek_max_alloc[0x5];
@@ -4671,8 +4673,19 @@ struct mlx5_ifc_crypto_caps_bits {
 	u8 log_dek_granularity[0x5];
 	u8 reserved_at_68[0x3];
 	u8 log_max_num_int_kek[0x5];
-	u8 reserved_at_70[0x10];
-	u8 reserved_at_80[0x780];
+	u8 sw_wrapped_dek_new[0x10];
+	u8 reserved_at_80[0x80];
+	u8 crypto_mmo_qp[0x1];
+	u8 crypto_aes_gcm_256_encrypt[0x1];
+	u8 crypto_aes_gcm_128_encrypt[0x1];
+	u8 crypto_aes_gcm_256_decrypt[0x1];
+	u8 crypto_aes_gcm_128_decrypt[0x1];
+	u8 gcm_auth_tag_128[0x1];
+	u8 gcm_auth_tag_96[0x1];
+	u8 reserved_at_107[0x3];
+	u8 log_crypto_mmo_max_size[0x6];
+	u8 reserved_at_110[0x10];
+	u8 reserved_at_120[0x6d0];
 };
 
 struct mlx5_ifc_crypto_commissioning_register_bits {
drivers/crypto/mlx5/meson.build +1 -0
index a830a4c7b9..930a31c795 100644
@@ -15,6 +15,7 @@ endif
 
 sources = files(
         'mlx5_crypto.c',
+	'mlx5_crypto_gcm.c',
         'mlx5_crypto_dek.c',
 )
 
drivers/crypto/mlx5/mlx5_crypto.c +28 -2
index 9dec1cfbe0..6963d8a9c9 100644
@@ -23,6 +23,13 @@
 #define MLX5_CRYPTO_MAX_QPS 128
 #define MLX5_CRYPTO_MAX_SEGS 56
 
+enum mlx5_crypto_pmd_support_algo {
+	MLX5_CRYPTO_PMD_SUPPORT_ALGO_NULL,
+	MLX5_CRYPTO_PMD_SUPPORT_ALGO_AES_XTS,
+	MLX5_CRYPTO_PMD_SUPPORT_ALGO_AES_GCM,
+	MLX5_CRYPTO_PMD_SUPPORT_ALGO_MAX,
+};
+
 #define MLX5_CRYPTO_FEATURE_FLAGS(wrapped_mode) \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
 	 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
@@ -102,7 +109,7 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 		dev_info->driver_id = mlx5_crypto_driver_id;
 		dev_info->feature_flags =
 			MLX5_CRYPTO_FEATURE_FLAGS(priv->is_wrapped_mode);
-		dev_info->capabilities = mlx5_crypto_caps;
+		dev_info->capabilities = priv->caps;
 		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 		dev_info->min_mbuf_headroom_req = 0;
 		dev_info->min_mbuf_tailroom_req = 0;
@@ -749,6 +756,14 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->credential_pointer = (uint32_t)tmp;
 	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
+	} else if (strcmp(key, "algo") == 0) {
+		if (tmp == 1) {
+			devarg_prms->is_aes_gcm = 1;
+		} else if (tmp > 1) {
+			DRV_LOG(ERR, "Invalid algo.");
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
 	}
 	return 0;
 }
@@ -765,6 +780,7 @@ mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl *mkvlist,
 		"keytag",
 		"max_segs_num",
 		"wcs_file",
+		"algo",
 		NULL,
 	};
 
@@ -895,7 +911,9 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
+	if (!cdev->config.hca_attr.crypto ||
+	   (!cdev->config.hca_attr.aes_xts &&
+	    !cdev->config.hca_attr.crypto_mmo.crypto_mmo_qp)) {
 		DRV_LOG(ERR, "Not enough capabilities to support crypto "
 			"operations, maybe old FW/OFED version?");
 		rte_errno = ENOTSUP;
@@ -924,6 +942,14 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
 	priv->cdev = cdev;
 	priv->crypto_dev = crypto_dev;
 	priv->is_wrapped_mode = wrapped_mode;
+	priv->caps = mlx5_crypto_caps;
+	/* Init and override AES-GCM configuration. */
+	if (devarg_prms.is_aes_gcm) {
+		ret = mlx5_crypto_gcm_init(priv);
+		if (ret) {
+			DRV_LOG(ERR, "Failed to init AES-GCM crypto.");
+		}
+	}
 	if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) {
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		return -1;
drivers/crypto/mlx5/mlx5_crypto.h +5 -0
index a2771b3dab..80c2cab0dd 100644
@@ -31,6 +31,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_uar uar; /* User Access Region. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	const struct rte_cryptodev_capabilities *caps;
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
@@ -68,6 +69,7 @@ struct mlx5_crypto_devarg_params {
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
 	uint32_t max_segs_num;
+	uint32_t is_aes_gcm:1;
 };
 
 int
@@ -84,4 +86,7 @@ mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
 void
 mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
 
+int
+mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv);
+
 #endif /* MLX5_CRYPTO_H_ */
drivers/crypto/mlx5/mlx5_crypto_gcm.c new file +100 -0

100 lines not on this page · raw message

Signature · 1 line
2.25.1
#2 lore raw

RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 9 lines

Subject: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

This commit adds the AES-GCM capability query and check. An new devarg "algo" is added to identify if the crypto PMD will be initialized as AES-GCM(algo=1) or AES-XTS(algo=0, default).

Why do you need a devarg for identifying the algorithm? Is it not sufficient to use enums rte_crypto_aead_algorithm and rte_crypto_cipher_algorithm?

Devargs are normally added for things which are specific to a particular PMD And which is not exposed via public APIs.
For identification of algo, it is not needed to use devargs.

Quoted 3 lines

---

#3 lore raw

RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 22 lines

-----Original Message-----
From: Akhil Goyal <[email protected]>
Sent: Wednesday, May 17, 2023 3:37 PM
To: Suanming Mou <[email protected]>; Matan Azrad <[email protected]>
Cc: Raslan Darawsheh <[email protected]>; Maayan Kashani <[email protected]>; [email protected]; NBU-Contact-Thomas Monjalon (EXTERNAL) <[email protected]>
Subject: RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 9 lines

Subject: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

This commit adds the AES-GCM capability query and check. An new devarg "algo" is added to identify if the crypto PMD will be initialized as AES-GCM(algo=1) or AES-XTS(algo=0, default).

Quoted 7 lines

Why do you need a devarg for identifying the algorithm? Is it not sufficient to use enums rte_crypto_aead_algorithm and rte_crypto_cipher_algorithm?

Devargs are normally added for things which are specific to a particular PMD And which is not exposed via public APIs.
For identification of algo, it is not needed to use devargs.

Due to current HW limitation, the NIC can only be initialized as GCM or XTS working mode during probe. It's not able to provide both in running time. That's the main reason for the devarg. Session configure with algo is too late.

Quoted 4 lines
Quoted 4 lines

---

#4 lore raw

RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 16 lines
Quoted 16 lines

Subject: RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 9 lines

Subject: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

This commit adds the AES-GCM capability query and check. An new devarg "algo" is added to identify if the crypto PMD will be initialized as AES-GCM(algo=1) or AES-XTS(algo=0, default).

Why do you need a devarg for identifying the algorithm? Is it not sufficient to use enums rte_crypto_aead_algorithm and rte_crypto_cipher_algorithm?

Quoted 8 lines
Quoted 1 line

Devargs are normally added for things which are specific to a particular PMD

And

Quoted 2 lines

which is not exposed via public APIs.
For identification of algo, it is not needed to use devargs.

Due to current HW limitation, the NIC can only be initialized as GCM or XTS working mode during probe. It's not able to provide both in running time. That's the main reason for the devarg.
Session configure with algo is too late.

Is it not possible to reconfigure the NIC when GCM is detected in session create?

#5 lore raw

RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 33 lines

-----Original Message-----
From: Akhil Goyal <[email protected]>
Sent: Wednesday, May 17, 2023 3:47 PM
To: Suanming Mou <[email protected]>; Matan Azrad <[email protected]>
Cc: Raslan Darawsheh <[email protected]>; Maayan Kashani <[email protected]>; [email protected]; NBU-Contact-Thomas Monjalon (EXTERNAL) <[email protected]>
Subject: RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 21 lines
Quoted 14 lines

Subject: RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 10 lines

Subject: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

This commit adds the AES-GCM capability query and check. An new devarg "algo" is added to identify if the crypto PMD will be initialized as
AES-GCM(algo=1) or AES-XTS(algo=0, default).

Quoted 6 lines

Why do you need a devarg for identifying the algorithm? Is it not sufficient to use enums rte_crypto_aead_algorithm and rte_crypto_cipher_algorithm?

Devargs are normally added for things which are specific to a particular PMD

And

Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

which is not exposed via public APIs.
For identification of algo, it is not needed to use devargs.

Due to current HW limitation, the NIC can only be initialized as GCM or XTS working mode during probe. It's not able to provide both in running time. That's the main reason for the devarg. Session configure with algo is too late.

Is it not possible to reconfigure the NIC when GCM is detected in session create?

That means in dev info, we need to put both XTS and GCM in the capability. But the fact is if we reconfigure the NIC to GCM, XTS will not be supported. If user wants to create both XTS and GCM session, one of them will fail.

Quoted 1 line
#6 lore raw

RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 30 lines
Quoted 30 lines

Subject: RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 27 lines
Quoted 14 lines

Subject: RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 10 lines

Subject: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

This commit adds the AES-GCM capability query and check. An new devarg "algo" is added to identify if the crypto PMD will be initialized as
AES-GCM(algo=1) or AES-XTS(algo=0, default).

Quoted 6 lines

Why do you need a devarg for identifying the algorithm? Is it not sufficient to use enums rte_crypto_aead_algorithm and rte_crypto_cipher_algorithm?

Devargs are normally added for things which are specific to a particular PMD

And

Quoted 2 lines

which is not exposed via public APIs.
For identification of algo, it is not needed to use devargs.

Due to current HW limitation, the NIC can only be initialized as GCM or XTS working mode during probe. It's not able to provide both in running time. That's the main reason for the devarg. Session configure with algo is too late.

Quoted 5 lines
Quoted 1 line

Is it not possible to reconfigure the NIC when GCM is detected in session

create?
That means in dev info, we need to put both XTS and GCM in the capability. But the fact is if we reconfigure the NIC to GCM, XTS will not be supported. If user wants to create both XTS and GCM session, one of them will fail.

That would fail even in current patchset.
On another thought, is it not good to create 2 separate instances of drivers in same folder, like ipsec_mb and cnxk drivers are organized.
You can change the function pointers based on the driver instance(mlx5_gcm, mlx5_xts)

#7 lore raw

RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 50 lines

-----Original Message-----
From: Akhil Goyal <[email protected]>
Sent: Wednesday, May 17, 2023 4:03 PM
To: Suanming Mou <[email protected]>; Matan Azrad <[email protected]>
Cc: Raslan Darawsheh <[email protected]>; Maayan Kashani <[email protected]>; [email protected]; NBU-Contact-Thomas Monjalon (EXTERNAL) <[email protected]>
Subject: RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 32 lines
Quoted 32 lines

Subject: RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 28 lines
Quoted 15 lines

Subject: RE: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

Quoted 11 lines

Subject: [EXT] [RFC PATCH 1/5] crypto/mlx5: add AES-GCM capability

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

This commit adds the AES-GCM capability query and check. An new devarg "algo" is added to identify if the crypto PMD will be initialized as
AES-GCM(algo=1) or AES-XTS(algo=0, default).

Quoted 6 lines

Why do you need a devarg for identifying the algorithm? Is it not sufficient to use enums rte_crypto_aead_algorithm and rte_crypto_cipher_algorithm?

Devargs are normally added for things which are specific to a particular PMD

And

Quoted 2 lines

which is not exposed via public APIs.
For identification of algo, it is not needed to use devargs.

Due to current HW limitation, the NIC can only be initialized as GCM or XTS working mode during probe. It's not able to provide both in running time. That's the main reason for the devarg. Session configure with algo is too late.

Quoted 6 lines
Quoted 2 lines

Is it not possible to reconfigure the NIC when GCM is detected in session

create?
That means in dev info, we need to put both XTS and GCM in the capability. But the fact is if we reconfigure the NIC to GCM, XTS will not be supported. If user wants to create both XTS and GCM session, one of

Quoted 7 lines

them will fail.

That would fail even in current patchset.
On another thought, is it not good to create 2 separate instances of drivers in same folder, like ipsec_mb and cnxk drivers are organized.
You can change the function pointers based on the driver instance(mlx5_gcm, mlx5_xts)

Current, we are initial the capability based on the algo, so it will not fail.
Regarding separate the instance, yes, we will do that in the next version. We will reuse the most of the common code in mlx5_crypto.c, mlx5_crypto_gcm.c for GCM and mlx5_crypto_xts.c for XTS.

#8 lore raw

RFC PATCH 2/5 crypto/mlx5: add AES-GCM encryption key

In reply to Suanming Mou

The crypto device requires the DEK(data encryption key) object for data encryption/decryption operation.

This commit adds the AES-GCM DEK object management support.

---
 drivers/common/mlx5/mlx5_devx_cmds.c  |   6 +-
 drivers/common/mlx5/mlx5_devx_cmds.h  |   1 +
 drivers/common/mlx5/mlx5_prm.h        |   6 +-
 drivers/crypto/mlx5/mlx5_crypto.c     |   2 +-
 drivers/crypto/mlx5/mlx5_crypto.h     |   3 +-
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 157 ++++++++++++++++++++------
 drivers/crypto/mlx5/mlx5_crypto_gcm.c |   2 +
 7 files changed, 137 insertions(+), 40 deletions(-)
drivers/common/mlx5/mlx5_devx_cmds.c +5 -1
index 592a7cffdb..8b51a75cc8 100644
@@ -3166,10 +3166,14 @@ mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr)
 	ptr = MLX5_ADDR_OF(create_dek_in, in, dek);
 	MLX5_SET(dek, ptr, key_size, attr->key_size);
 	MLX5_SET(dek, ptr, has_keytag, attr->has_keytag);
+	MLX5_SET(dek, ptr, sw_wrapped, attr->sw_wrapped);
 	MLX5_SET(dek, ptr, key_purpose, attr->key_purpose);
 	MLX5_SET(dek, ptr, pd, attr->pd);
 	MLX5_SET64(dek, ptr, opaque, attr->opaque);
-	key_addr = MLX5_ADDR_OF(dek, ptr, key);
+	if (attr->sw_wrapped)
+		key_addr = MLX5_ADDR_OF(dek, ptr, sw_wrapped_dek);
+	else
+		key_addr = MLX5_ADDR_OF(dek, ptr, key);
 	memcpy(key_addr, (void *)(attr->key), MLX5_CRYPTO_KEY_MAX_SIZE);
 	dek_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
 						  out, sizeof(out));
drivers/common/mlx5/mlx5_devx_cmds.h +1 -0
index d640482346..79502cda08 100644
@@ -664,6 +664,7 @@ struct mlx5_devx_dek_attr {
 	uint32_t key_size:4;
 	uint32_t has_keytag:1;
 	uint32_t key_purpose:4;
+	uint32_t sw_wrapped:1;
 	uint32_t pd:24;
 	uint64_t opaque;
 	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
drivers/common/mlx5/mlx5_prm.h +4 -2
index a3b85f514e..9728be24dd 100644
@@ -3736,7 +3736,8 @@ enum {
 struct mlx5_ifc_dek_bits {
 	u8 modify_field_select[0x40];
 	u8 state[0x8];
-	u8 reserved_at_48[0xc];
+	u8 sw_wrapped[0x1];
+	u8 reserved_at_49[0xb];
 	u8 key_size[0x4];
 	u8 has_keytag[0x1];
 	u8 reserved_at_59[0x3];
@@ -3747,7 +3748,8 @@ struct mlx5_ifc_dek_bits {
 	u8 opaque[0x40];
 	u8 reserved_at_1c0[0x40];
 	u8 key[0x400];
-	u8 reserved_at_600[0x200];
+	u8 sw_wrapped_dek[0x400];
+	u8 reserved_at_a00[0x300];
 };
 
 struct mlx5_ifc_create_dek_in_bits {
drivers/crypto/mlx5/mlx5_crypto.c +1 -1
index 6963d8a9c9..66c9f94346 100644
@@ -196,7 +196,7 @@ mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 		return -ENOTSUP;
 	}
 	cipher = &xform->cipher;
-	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
+	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, xform);
 	if (sess_private_data->dek == NULL) {
 		DRV_LOG(ERR, "Failed to prepare dek.");
 		return -ENOMEM;
drivers/crypto/mlx5/mlx5_crypto.h +2 -1
index 80c2cab0dd..11352f9409 100644
@@ -40,6 +40,7 @@ struct mlx5_crypto_priv {
 	uint16_t umr_wqe_stride;
 	uint16_t max_rdmar_ds;
 	uint32_t is_wrapped_mode:1;
+	uint32_t is_gcm_dek_wrap:1;
 };
 
 struct mlx5_crypto_qp {
@@ -78,7 +79,7 @@ mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 
 struct mlx5_crypto_dek *
 mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
-			struct rte_crypto_cipher_xform *cipher);
+			struct rte_crypto_sym_xform *xform);
 
 int
 mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
drivers/crypto/mlx5/mlx5_crypto_dek.c +122 -35

227 lines not on this page · raw message

drivers/crypto/mlx5/mlx5_crypto_gcm.c +2 -0
index d60ac379cf..c7fd86d7b9 100644
@@ -95,6 +95,8 @@ mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv)
 		return -1;
 	}
 	priv->caps = mlx5_crypto_gcm_caps;
+	priv->is_gcm_dek_wrap = !!(cdev->config.hca_attr.sw_wrapped_dek &
+				(1 << MLX5_CRYPTO_KEY_PURPOSE_GCM));
 	return 0;
 }
 
Signature · 1 line
2.25.1
#9 lore raw

RFC PATCH 3/5 crypto/mlx5: add AES-GCM session configure

In reply to Suanming Mou

Sessions are used in symmetric transformations in order to prepare objects and data for packet processing stage.

The AES-GCM session includes IV, AAD, digest(tag), DEK, operation mode information.

---
 drivers/common/mlx5/mlx5_prm.h        | 12 +++++++
 drivers/crypto/mlx5/mlx5_crypto.c     | 15 ---------
 drivers/crypto/mlx5/mlx5_crypto.h     | 35 ++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 46 +++++++++++++++++++++++++++
 4 files changed, 93 insertions(+), 15 deletions(-)
drivers/common/mlx5/mlx5_prm.h +12 -0
index 9728be24dd..25ff66ee7e 100644
@@ -528,11 +528,23 @@ enum {
 	MLX5_BLOCK_SIZE_4048B	= 0x6,
 };
 
+enum {
+	MLX5_ENCRYPTION_TYPE_AES_GCM = 0x3,
+};
+
+enum {
+	MLX5_CRYPTO_OP_TYPE_ENCRYPTION = 0x0,
+	MLX5_CRYPTO_OP_TYPE_DECRYPTION = 0x1,
+};
+
 #define MLX5_BSF_SIZE_OFFSET		30
 #define MLX5_BSF_P_TYPE_OFFSET		24
 #define MLX5_ENCRYPTION_ORDER_OFFSET	16
 #define MLX5_BLOCK_SIZE_OFFSET		24
 
+#define MLX5_CRYPTO_MMO_TYPE_OFFSET 24
+#define MLX5_CRYPTO_MMO_OP_OFFSET 20
+
 struct mlx5_wqe_umr_bsf_seg {
 	/*
 	 * bs_bpt_eo_es contains:
drivers/crypto/mlx5/mlx5_crypto.c +0 -15
index 66c9f94346..8946f13e5e 100644
@@ -83,21 +83,6 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
-struct mlx5_crypto_session {
-	uint32_t bs_bpt_eo_es;
-	/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
-	 * saved in big endian format.
-	 */
-	uint32_t bsp_res;
-	/**< crypto_block_size_pointer and reserved 24 bits saved in big
-	 * endian format.
-	 */
-	uint32_t iv_offset:16;
-	/**< Starting point for Initialisation Vector. */
-	struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
-	uint32_t dek_id; /**< DEK ID */
-} __rte_packed;
-
 static void
 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 			  struct rte_cryptodev_info *dev_info)
drivers/crypto/mlx5/mlx5_crypto.h +35 -0
index 11352f9409..c34a860404 100644
@@ -73,6 +73,41 @@ struct mlx5_crypto_devarg_params {
 	uint32_t is_aes_gcm:1;
 };
 
+struct mlx5_crypto_session {
+	union {
+		/**< AES-XTS configuration. */
+		struct {
+			uint32_t bs_bpt_eo_es;
+			/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
+			 * saved in big endian format.
+			 */
+			uint32_t bsp_res;
+			/**< crypto_block_size_pointer and reserved 24 bits saved in big
+			 * endian format.
+			 */
+		};
+		/**< AES-GCM configuration. */
+		struct {
+			uint32_t mmo_ctrl;
+			/**< Crypto control fields with algo type and op type in big
+			 * endian format.
+			 */
+			uint16_t tag_len;
+			/**< AES-GCM crypto digest size in bytes. */
+			uint16_t aad_len;
+			/**< The length of the additional authenticated data (AAD) in bytes. */
+			uint32_t op_type;
+			/**< Operation type. */
+		};
+	};
+	uint32_t iv_offset:16;
+	/**< Starting point for Initialisation Vector. */
+	uint32_t iv_len;
+	/**< Initialisation Vector length. */
+	struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
+	uint32_t dek_id; /**< DEK ID */
+} __rte_packed;
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
drivers/crypto/mlx5/mlx5_crypto_gcm.c +46 -0

58 lines not on this page · raw message

Signature · 1 line
2.25.1
#10 lore raw

RFC PATCH 4/5 crypto/mlx5: add queue pair setup

In reply to Suanming Mou

Crypto queue pair is for handling the encryption/decryption operations.

As AES-GCM AEAD API provides AAD, mbuf, digest separately, low-level FW only accepts the data in a single contiguous memory region, two internal QPs are created for AES-GCM queue pair. One for organizing the memory to be contiguous if they are not. The other is for crypto.

If the buffers are checked as implicitly contiguous, the buffer will be sent to the crypto QP directly for encryption/decryption. If not, the buffers will be handled by the first UMR QP. The UMR QP will convert the buffers to be contiguous one. Then the well organized "new" buffer can be handled by crypto QP.

The crypto QP is initialized as follower, and UMR as leader. Once crypto operation input buffer requires memory address space converting by UMR QP, the crypto QP processing will be triggered by UMR QP. Otherwise, the ring crypto QP doorbell directly.

The existing max_segs_num devarg is used for define how many segments the chained mbuf contains same as AES-XTS before.

---
 drivers/common/mlx5/mlx5_devx_cmds.c  |   6 +
 drivers/common/mlx5/mlx5_devx_cmds.h  |   3 +
 drivers/common/mlx5/mlx5_prm.h        |  24 +++
 drivers/crypto/mlx5/mlx5_crypto.c     |  17 ++
 drivers/crypto/mlx5/mlx5_crypto.h     |  12 ++
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 254 ++++++++++++++++++++++++++
 6 files changed, 316 insertions(+)
drivers/common/mlx5/mlx5_devx_cmds.c +6 -0
index 8b51a75cc8..6be02c0a65 100644
@@ -2563,6 +2563,12 @@ mlx5_devx_cmd_create_qp(void *ctx,
 				 attr->dbr_umem_valid);
 			MLX5_SET(qpc, qpc, dbr_umem_id, attr->dbr_umem_id);
 		}
+		if (attr->cd_master)
+			MLX5_SET(qpc, qpc, cd_master, attr->cd_master);
+		if (attr->cd_slave_send)
+			MLX5_SET(qpc, qpc, cd_slave_send, attr->cd_slave_send);
+		if (attr->cd_slave_recv)
+			MLX5_SET(qpc, qpc, cd_slave_receive, attr->cd_slave_recv);
 		MLX5_SET64(qpc, qpc, dbr_addr, attr->dbr_address);
 		MLX5_SET64(create_qp_in, in, wq_umem_offset,
 			   attr->wq_umem_offset);
drivers/common/mlx5/mlx5_devx_cmds.h +3 -0
index 79502cda08..e68aa077d7 100644
@@ -590,6 +590,9 @@ struct mlx5_devx_qp_attr {
 	uint64_t wq_umem_offset;
 	uint32_t user_index:24;
 	uint32_t mmo:1;
+	uint32_t cd_master:1;
+	uint32_t cd_slave_send:1;
+	uint32_t cd_slave_recv:1;
 };
 
 struct mlx5_devx_virtio_q_couners_attr {
drivers/common/mlx5/mlx5_prm.h +24 -0
index 25ff66ee7e..c8d73a8456 100644
@@ -594,6 +594,17 @@ struct mlx5_rdma_write_wqe {
 	struct mlx5_wqe_dseg dseg[];
 } __rte_packed;
 
+struct mlx5_wqe_send_en_seg {
+	uint32_t reserve[2];
+	uint32_t sqnpc;
+	uint32_t qpn;
+} __rte_packed;
+
+struct mlx5_wqe_send_en_wqe {
+	struct mlx5_wqe_cseg ctr;
+	struct mlx5_wqe_send_en_seg sseg;
+} __rte_packed;
+
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -668,6 +679,19 @@ union mlx5_gga_compress_opaque {
 	uint32_t data[64];
 };
 
+union mlx5_gga_crypto_opaque {
+	struct {
+		uint32_t syndrome;
+		uint32_t reserved0[2];
+		struct {
+			uint32_t iv[3];
+			uint32_t tag_size;
+			uint32_t aad_size;
+		} cp __rte_packed;
+	} __rte_packed;
+	uint8_t data[64];
+};
+
 struct mlx5_ifc_regexp_mmo_control_bits {
 	uint8_t reserved_at_31[0x2];
 	uint8_t le[0x1];
drivers/crypto/mlx5/mlx5_crypto.c +17 -0
index 8946f13e5e..f2e5b25c15 100644
@@ -849,12 +849,27 @@ mlx5_crypto_max_segs_num(uint16_t max_wqe_size)
 	return max_segs_cap;
 }
 
+static __rte_always_inline int
+mlx5_crypto_configure_gcm_wqe_size(struct mlx5_crypto_priv *priv)
+{
+	uint32_t send_en_wqe_size;
+
+	priv->umr_wqe_size = RTE_ALIGN(sizeof(struct mlx5_umr_wqe) + sizeof(struct mlx5_wqe_dseg),
+		MLX5_SEND_WQE_BB);
+	send_en_wqe_size = RTE_ALIGN(sizeof(struct mlx5_wqe_send_en_wqe), MLX5_SEND_WQE_BB);
+	priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
+	priv->wqe_set_size = priv->umr_wqe_size + send_en_wqe_size;
+	return 0;
+}
+
 static int
 mlx5_crypto_configure_wqe_size(struct mlx5_crypto_priv *priv,
 				uint16_t max_wqe_size, uint32_t max_segs_num)
 {
 	uint32_t rdmw_wqe_size, umr_wqe_size;
 
+	if (priv->is_gcm_dek_wrap)
+		return mlx5_crypto_configure_gcm_wqe_size(priv);
 	mlx5_crypto_get_wqe_sizes(max_segs_num, &umr_wqe_size,
 					&rdmw_wqe_size);
 	priv->wqe_set_size = rdmw_wqe_size + umr_wqe_size;
@@ -927,12 +942,14 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
 	priv->cdev = cdev;
 	priv->crypto_dev = crypto_dev;
 	priv->is_wrapped_mode = wrapped_mode;
+	priv->max_segs_num = devarg_prms.max_segs_num;
 	priv->caps = mlx5_crypto_caps;
 	/* Init and override AES-GCM configuration. */
 	if (devarg_prms.is_aes_gcm) {
 		ret = mlx5_crypto_gcm_init(priv);
 		if (ret) {
 			DRV_LOG(ERR, "Failed to init AES-GCM crypto.");
+			return -ENOTSUP;
 		}
 	}
 	if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) {
drivers/crypto/mlx5/mlx5_crypto.h +12 -0
index c34a860404..9945891ea8 100644
@@ -47,15 +47,27 @@ struct mlx5_crypto_qp {
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_qp qp_obj;
+	struct mlx5_devx_cq umr_cq_obj;
+	struct mlx5_devx_qp umr_qp_obj;
 	struct rte_cryptodev_stats stats;
 	struct rte_crypto_op **ops;
 	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
+	struct mlx5_klm *klm_array;
 	struct mlx5_mr_ctrl mr_ctrl;
+	struct mlx5_pmd_mr opaque_mr;
+	struct mlx5_pmd_mr klm_mr;
+	/* Crypto QP. */
 	uint8_t *wqe;
 	uint16_t entries_n;
 	uint16_t pi;
 	uint16_t ci;
 	uint16_t db_pi;
+	/* UMR QP. */
+	uint8_t *umr_wqe;
+	uint16_t umr_wqbbs;
+	uint16_t umr_pi;
+	uint16_t umr_ci;
+	uint32_t umr_errors;
 };
 
 struct mlx5_crypto_dek {
drivers/crypto/mlx5/mlx5_crypto_gcm.c +254 -0

272 lines not on this page · raw message

Signature · 1 line
2.25.1
#11 lore raw

RFC PATCH 5/5 crypto/mlx5: add enqueue and dequeue operations

In reply to Suanming Mou

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE. The UMR WQE and crypto WQE are handled in two different QPs.

The QP for UMR operation contains two types of WQE, UMR and SEND_EN WQE. The WQEs are built dynamically according to the crypto operation buffer address. Crypto operation with non-contiguous buffers will have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP. The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.

---
 drivers/common/mlx5/mlx5_prm.h        |   1 +
 drivers/crypto/mlx5/mlx5_crypto.h     |   2 +
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 401 ++++++++++++++++++++++++++
 3 files changed, 404 insertions(+)
drivers/common/mlx5/mlx5_prm.h +1 -0
index c8d73a8456..71000ebf02 100644
@@ -613,6 +613,7 @@ struct mlx5_wqe_send_en_wqe {
 /* MMO metadata segment */
 
 #define	MLX5_OPCODE_MMO	0x2fu
+#define	MLX5_OPC_MOD_MMO_CRYPTO 0x6u
 #define	MLX5_OPC_MOD_MMO_REGEX 0x4u
 #define	MLX5_OPC_MOD_MMO_COMP 0x2u
 #define	MLX5_OPC_MOD_MMO_DECOMP 0x3u
drivers/crypto/mlx5/mlx5_crypto.h +2 -0
index 9945891ea8..0b0ef1a84d 100644
@@ -66,8 +66,10 @@ struct mlx5_crypto_qp {
 	uint8_t *umr_wqe;
 	uint16_t umr_wqbbs;
 	uint16_t umr_pi;
+	uint16_t umr_last_pi;
 	uint16_t umr_ci;
 	uint32_t umr_errors;
+	bool has_umr;
 };
 
 struct mlx5_crypto_dek {
drivers/crypto/mlx5/mlx5_crypto_gcm.c +401 -0

431 lines not on this page · raw message

Signature · 1 line
2.25.1
#12 lore raw

v2 cover crypto/mlx5: support AES-GCM

In reply to Suanming Mou

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous and there is no enough headroom or tailroom for AAD or digest, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE. The UMR WQE and crypto WQE are handled in two different QPs.

The QP for UMR operation contains two types of WQE, UMR and SEND_EN WQE. The WQEs are built dynamically according to the crypto operation buffer address. Crypto operation with non-contiguous buffers will have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP. The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.

Suanming Mou (9):
common/mlx5: export memory region lookup by address crypto/mlx5: split AES-XTS
crypto/mlx5: add AES-GCM query and initialization crypto/mlx5: add AES-GCM encryption key crypto/mlx5: add AES-GCM session configure common/mlx5: add WQE-based QP synchronous basics crypto/mlx5: add queue pair setup for GCM crypto/mlx5: add enqueue and dequeue operations crypto/mlx5: enable AES-GCM capability

 doc/guides/cryptodevs/mlx5.rst         |  48 +-
 doc/guides/rel_notes/release_23_07.rst |   1 +
 drivers/common/mlx5/mlx5_common_mr.c   |   2 +-
 drivers/common/mlx5/mlx5_common_mr.h   |   5 +
 drivers/common/mlx5/mlx5_devx_cmds.c   |  21 +
 drivers/common/mlx5/mlx5_devx_cmds.h   |  16 +
 drivers/common/mlx5/mlx5_prm.h         |  65 +-
 drivers/common/mlx5/version.map        |   3 +
 drivers/crypto/mlx5/meson.build        |   2 +
 drivers/crypto/mlx5/mlx5_crypto.c      | 673 ++---------------
 drivers/crypto/mlx5/mlx5_crypto.h      | 101 ++-
 drivers/crypto/mlx5/mlx5_crypto_dek.c  | 102 ++-
 drivers/crypto/mlx5/mlx5_crypto_gcm.c  | 995 +++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_xts.c  | 645 ++++++++++++++++
 14 files changed, 2014 insertions(+), 665 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_gcm.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_xts.c
Signature · 1 line
2.25.1
#13 lore raw

PATCH v2 1/9 common/mlx5: export memory region lookup by address

In case user provides the address without mempool. Export the function to lookup the address without mempool is required.

---
 drivers/common/mlx5/mlx5_common_mr.c | 2 +-
 drivers/common/mlx5/mlx5_common_mr.h | 4 ++++
 drivers/common/mlx5/version.map      | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)
drivers/common/mlx5/mlx5_common_mr.c +1 -1
index 7b14b0c7bf..40ff9153bd 100644
@@ -1059,7 +1059,7 @@ mr_lookup_caches(struct mlx5_mr_ctrl *mr_ctrl,
  * @return
  *   Searched LKey on success, UINT32_MAX on no match.
  */
-static uint32_t
+uint32_t
 mlx5_mr_addr2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, uintptr_t addr)
 {
 	uint32_t lkey;
drivers/common/mlx5/mlx5_common_mr.h +4 -0
index 12def1585f..66623868a2 100644
@@ -240,6 +240,10 @@ mlx5_mr_create(struct mlx5_common_device *cdev,
 	       struct mlx5_mr_share_cache *share_cache,
 	       struct mr_cache_entry *entry, uintptr_t addr);
 
+__rte_internal
+uint32_t
+mlx5_mr_addr2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, uintptr_t addr);
+
 /* mlx5_common_verbs.c */
 
 __rte_internal
drivers/common/mlx5/version.map +1 -0
index e05e1aa8c5..f860b069de 100644
@@ -122,6 +122,7 @@ INTERNAL {
 	mlx5_mr_ctrl_init;
 	mlx5_mr_flush_local_cache;
 	mlx5_mr_mb2mr_bh;
+	mlx5_mr_addr2mr_bh;
 
 	mlx5_nl_allmulti; # WINDOWS_NO_EXPORT
 	mlx5_nl_ifindex; # WINDOWS_NO_EXPORT
Signature · 1 line
2.25.1
#14 lore raw

PATCH v2 2/9 crypto/mlx5: split AES-XTS

In reply to Suanming Mou

As there will be other crypto algo be supported. This commit splits AES-XTS code to another *_xts.c file. The mlx5_crypto.c file will just contain the common code.

---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     | 642 ++------------------------
 drivers/crypto/mlx5/mlx5_crypto.h     |  33 ++
 drivers/crypto/mlx5/mlx5_crypto_xts.c | 594 ++++++++++++++++++++++++
 4 files changed, 667 insertions(+), 603 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_xts.c
drivers/crypto/mlx5/meson.build +1 -0
index a2691ec0f0..045e8ce81d 100644
@@ -15,6 +15,7 @@ endif
 
 sources = files(
         'mlx5_crypto.c',
+	'mlx5_crypto_xts.c',
         'mlx5_crypto_dek.c',
 )
 
drivers/crypto/mlx5/mlx5_crypto.c +39 -603

709 lines not on this page · raw message

drivers/crypto/mlx5/mlx5_crypto.h +33 -0
index a2771b3dab..05d8fe97fe 100644
@@ -31,6 +31,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_uar uar; /* User Access Region. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	const struct rte_cryptodev_capabilities *caps;
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
@@ -70,6 +71,35 @@ struct mlx5_crypto_devarg_params {
 	uint32_t max_segs_num;
 };
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t bsp_res;
+	/**< crypto_block_size_pointer and reserved 24 bits saved in big
+	 * endian format.
+	 */
+	uint32_t iv_offset:16;
+	/**< Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
+	uint32_t dek_id; /**< DEK ID */
+} __rte_packed;
+
+typedef void *(*mlx5_crypto_mkey_update_t)(struct mlx5_crypto_priv *priv,
+					   struct mlx5_crypto_qp *qp,
+					   uint32_t idx);
+
+void
+mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp,
+				   uint16_t n);
+
+int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+				   struct mlx5_crypto_qp *qp,
+				   struct mlx5_devx_mkey_attr *attr,
+				   mlx5_crypto_mkey_update_t update_cb);
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
@@ -84,4 +114,7 @@ mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
 void
 mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
 
+int
+mlx5_crypto_xts_init(struct mlx5_crypto_priv *priv);
+
 #endif /* MLX5_CRYPTO_H_ */
drivers/crypto/mlx5/mlx5_crypto_xts.c new file +594 -0

594 lines not on this page · raw message

Signature · 1 line
2.25.1
#15 lore raw

PATCH v2 3/9 crypto/mlx5: add AES-GCM query and initialization

In reply to Suanming Mou

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

This commit adds the AES-GCM attributes query and initialization function.

---
 drivers/common/mlx5/mlx5_devx_cmds.c  | 15 +++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h  | 13 ++++++++++
 drivers/common/mlx5/mlx5_prm.h        | 19 +++++++++++---
 drivers/crypto/mlx5/meson.build       |  1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  4 ++-
 drivers/crypto/mlx5/mlx5_crypto.h     |  3 +++
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 36 +++++++++++++++++++++++++++
 7 files changed, 87 insertions(+), 4 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_gcm.c
drivers/common/mlx5/mlx5_devx_cmds.c +15 -0
index 1e418a0353..4332081165 100644
@@ -1117,6 +1117,21 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		attr->crypto_wrapped_import_method = !!(MLX5_GET(crypto_caps,
 						hcattr, wrapped_import_method)
 						& 1 << 2);
+		attr->crypto_mmo.crypto_mmo_qp = MLX5_GET(crypto_caps, hcattr, crypto_mmo_qp);
+		attr->crypto_mmo.gcm_256_encrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_256_encrypt);
+		attr->crypto_mmo.gcm_128_encrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_128_encrypt);
+		attr->crypto_mmo.gcm_256_decrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_256_decrypt);
+		attr->crypto_mmo.gcm_128_decrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_128_decrypt);
+		attr->crypto_mmo.gcm_auth_tag_128 =
+			MLX5_GET(crypto_caps, hcattr, gcm_auth_tag_128);
+		attr->crypto_mmo.gcm_auth_tag_96 =
+			MLX5_GET(crypto_caps, hcattr, gcm_auth_tag_96);
+		attr->crypto_mmo.log_crypto_mmo_max_size =
+			MLX5_GET(crypto_caps, hcattr, log_crypto_mmo_max_size);
 	}
 	if (hca_cap_2_sup) {
 		hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
drivers/common/mlx5/mlx5_devx_cmds.h +13 -0
index dc3359268d..cb3f3a211b 100644
@@ -125,6 +125,18 @@ struct mlx5_hca_flex_attr {
 	uint8_t  header_length_mask_width;
 };
 
+__extension__
+struct mlx5_hca_crypto_mmo_attr {
+	uint32_t crypto_mmo_qp:1;
+	uint32_t gcm_256_encrypt:1;
+	uint32_t gcm_128_encrypt:1;
+	uint32_t gcm_256_decrypt:1;
+	uint32_t gcm_128_decrypt:1;
+	uint32_t gcm_auth_tag_128:1;
+	uint32_t gcm_auth_tag_96:1;
+	uint32_t log_crypto_mmo_max_size:6;
+};
+
 /* ISO C restricts enumerator values to range of 'int' */
 __extension__
 enum {
@@ -250,6 +262,7 @@ struct mlx5_hca_attr {
 	struct mlx5_hca_vdpa_attr vdpa;
 	struct mlx5_hca_flow_attr flow;
 	struct mlx5_hca_flex_attr flex;
+	struct mlx5_hca_crypto_mmo_attr crypto_mmo;
 	int log_max_qp_sz;
 	int log_max_cq_sz;
 	int log_max_qp;
drivers/common/mlx5/mlx5_prm.h +16 -3
index 9f749a2dcc..b4446f56b9 100644
@@ -4577,7 +4577,9 @@ struct mlx5_ifc_crypto_caps_bits {
 	u8 synchronize_dek[0x1];
 	u8 int_kek_manual[0x1];
 	u8 int_kek_auto[0x1];
-	u8 reserved_at_6[0x12];
+	u8 reserved_at_6[0xd];
+	u8 sw_wrapped_dek_key_purpose[0x1];
+	u8 reserved_at_14[0x4];
 	u8 wrapped_import_method[0x8];
 	u8 reserved_at_20[0x3];
 	u8 log_dek_max_alloc[0x5];
@@ -4594,8 +4596,19 @@ struct mlx5_ifc_crypto_caps_bits {
 	u8 log_dek_granularity[0x5];
 	u8 reserved_at_68[0x3];
 	u8 log_max_num_int_kek[0x5];
-	u8 reserved_at_70[0x10];
-	u8 reserved_at_80[0x780];
+	u8 sw_wrapped_dek_new[0x10];
+	u8 reserved_at_80[0x80];
+	u8 crypto_mmo_qp[0x1];
+	u8 crypto_aes_gcm_256_encrypt[0x1];
+	u8 crypto_aes_gcm_128_encrypt[0x1];
+	u8 crypto_aes_gcm_256_decrypt[0x1];
+	u8 crypto_aes_gcm_128_decrypt[0x1];
+	u8 gcm_auth_tag_128[0x1];
+	u8 gcm_auth_tag_96[0x1];
+	u8 reserved_at_107[0x3];
+	u8 log_crypto_mmo_max_size[0x6];
+	u8 reserved_at_110[0x10];
+	u8 reserved_at_120[0x6e0];
 };
 
 struct mlx5_ifc_crypto_commissioning_register_bits {
drivers/crypto/mlx5/meson.build +1 -0
index 045e8ce81d..17ffce89f0 100644
@@ -16,6 +16,7 @@ endif
 sources = files(
         'mlx5_crypto.c',
 	'mlx5_crypto_xts.c',
+	'mlx5_crypto_gcm.c',
         'mlx5_crypto_dek.c',
 )
 
drivers/crypto/mlx5/mlx5_crypto.c +3 -1
index 2e6bcc6ddc..ff632cd69a 100644
@@ -335,7 +335,9 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
+	if (!cdev->config.hca_attr.crypto ||
+	   (!cdev->config.hca_attr.aes_xts &&
+	    !cdev->config.hca_attr.crypto_mmo.crypto_mmo_qp)) {
 		DRV_LOG(ERR, "Not enough capabilities to support crypto "
 			"operations, maybe old FW/OFED version?");
 		rte_errno = ENOTSUP;
drivers/crypto/mlx5/mlx5_crypto.h +3 -0
index 05d8fe97fe..76f368ee91 100644
@@ -117,4 +117,7 @@ mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
 int
 mlx5_crypto_xts_init(struct mlx5_crypto_priv *priv);
 
+int
+mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv);
+
 #endif /* MLX5_CRYPTO_H_ */
drivers/crypto/mlx5/mlx5_crypto_gcm.c new file +36 -0
new file mode 100644
index 0000000000..bd78c6d66b
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_eal_paging.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+#include <bus_pci_driver.h>
+#include <rte_memory.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+static struct rte_cryptodev_capabilities mlx5_crypto_gcm_caps[] = {
+	{
+		.op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
+	},
+	{
+		.op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
+	}
+};
+
+int
+mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv)
+{
+	priv->caps = mlx5_crypto_gcm_caps;
+	return 0;
+}
+
Signature · 1 line
2.25.1
#16 lore raw

PATCH v2 4/9 crypto/mlx5: add AES-GCM encryption key

In reply to Suanming Mou

The crypto device requires the DEK(data encryption key) object for data encryption/decryption operation.

This commit adds the AES-GCM DEK object management support.

---
 drivers/crypto/mlx5/mlx5_crypto.h     |  17 ++++-
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 102 +++++++++++++-------------
 drivers/crypto/mlx5/mlx5_crypto_gcm.c |  31 ++++++++
 drivers/crypto/mlx5/mlx5_crypto_xts.c |  53 ++++++++++++-
 4 files changed, 148 insertions(+), 55 deletions(-)
drivers/crypto/mlx5/mlx5_crypto.h +16 -1
index 76f368ee91..bb5a557a38 100644
@@ -86,6 +86,11 @@ struct mlx5_crypto_session {
 	uint32_t dek_id; /**< DEK ID */
 } __rte_packed;
 
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_sym_xform *xform;
+	struct mlx5_crypto_priv *priv;
+};
+
 typedef void *(*mlx5_crypto_mkey_update_t)(struct mlx5_crypto_priv *priv,
 					   struct mlx5_crypto_qp *qp,
 					   uint32_t idx);
@@ -106,7 +111,7 @@ mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 
 struct mlx5_crypto_dek *
 mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
-			struct rte_crypto_cipher_xform *cipher);
+			struct rte_crypto_sym_xform *xform);
 
 int
 mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
@@ -120,4 +125,14 @@ mlx5_crypto_xts_init(struct mlx5_crypto_priv *priv);
 int
 mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv);
 
+int
+mlx5_crypto_dek_fill_xts_attr(struct mlx5_crypto_dek *dek,
+			      struct mlx5_devx_dek_attr *dek_attr,
+			      void *cb_ctx);
+
+int
+mlx5_crypto_dek_fill_gcm_attr(struct mlx5_crypto_dek *dek,
+			      struct mlx5_devx_dek_attr *dek_attr,
+			      void *cb_ctx);
+
 #endif /* MLX5_CRYPTO_H_ */
drivers/crypto/mlx5/mlx5_crypto_dek.c +49 -53

154 lines not on this page · raw message

drivers/crypto/mlx5/mlx5_crypto_gcm.c +31 -0
index bd78c6d66b..676bec6b18 100644
@@ -27,6 +27,37 @@ static struct rte_cryptodev_capabilities mlx5_crypto_gcm_caps[] = {
 	}
 };
 
+int
+mlx5_crypto_dek_fill_gcm_attr(struct mlx5_crypto_dek *dek,
+			      struct mlx5_devx_dek_attr *dek_attr,
+			      void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_aead_xform *aead_ctx = &ctx->xform->aead;
+
+	if (aead_ctx->algo != RTE_CRYPTO_AEAD_AES_GCM) {
+		DRV_LOG(ERR, "Only AES-GCM algo supported.");
+		return -EINVAL;
+	}
+	dek_attr->key_purpose = MLX5_CRYPTO_KEY_PURPOSE_GCM;
+	switch (aead_ctx->key.length) {
+	case 16:
+		dek->size = 16;
+		dek_attr->key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 32:
+		dek->size = 32;
+		dek_attr->key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Wrapped key size not supported.");
+		return -EINVAL;
+	}
+	memcpy(&dek_attr->key, aead_ctx->key.data, aead_ctx->key.length);
+	memcpy(&dek->data, aead_ctx->key.data, aead_ctx->key.length);
+	return 0;
+}
+
 int
 mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv)
 {
drivers/crypto/mlx5/mlx5_crypto_xts.c +52 -1

65 lines not on this page · raw message

Signature · 1 line
2.25.1
#17 lore raw

PATCH v2 5/9 crypto/mlx5: add AES-GCM session configure

In reply to Suanming Mou

Sessions are used in symmetric transformations in order to prepare objects and data for packet processing stage.

The AES-GCM session includes IV, AAD, digest(tag), DEK, operation mode information.

---
 drivers/common/mlx5/mlx5_prm.h        | 12 +++++++
 drivers/crypto/mlx5/mlx5_crypto.h     | 40 ++++++++++++++++++-----
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 47 +++++++++++++++++++++++++++
 3 files changed, 91 insertions(+), 8 deletions(-)
drivers/common/mlx5/mlx5_prm.h +12 -0
index b4446f56b9..3b26499a47 100644
@@ -523,11 +523,23 @@ enum {
 	MLX5_BLOCK_SIZE_4048B	= 0x6,
 };
 
+enum {
+	MLX5_ENCRYPTION_TYPE_AES_GCM = 0x3,
+};
+
+enum {
+	MLX5_CRYPTO_OP_TYPE_ENCRYPTION = 0x0,
+	MLX5_CRYPTO_OP_TYPE_DECRYPTION = 0x1,
+};
+
 #define MLX5_BSF_SIZE_OFFSET		30
 #define MLX5_BSF_P_TYPE_OFFSET		24
 #define MLX5_ENCRYPTION_ORDER_OFFSET	16
 #define MLX5_BLOCK_SIZE_OFFSET		24
 
+#define MLX5_CRYPTO_MMO_TYPE_OFFSET 24
+#define MLX5_CRYPTO_MMO_OP_OFFSET 20
+
 struct mlx5_wqe_umr_bsf_seg {
 	/*
 	 * bs_bpt_eo_es contains:
drivers/crypto/mlx5/mlx5_crypto.h +32 -8
index bb5a557a38..6cb4d4ddec 100644
@@ -72,16 +72,40 @@ struct mlx5_crypto_devarg_params {
 };
 
 struct mlx5_crypto_session {
-	uint32_t bs_bpt_eo_es;
-	/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
-	 * saved in big endian format.
-	 */
-	uint32_t bsp_res;
-	/**< crypto_block_size_pointer and reserved 24 bits saved in big
-	 * endian format.
-	 */
+	union {
+		/**< AES-XTS configuration. */
+		struct {
+			uint32_t bs_bpt_eo_es;
+			/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
+			 * saved in big endian format.
+			 */
+			uint32_t bsp_res;
+			/**< crypto_block_size_pointer and reserved 24 bits saved in big
+			 * endian format.
+			 */
+		};
+		/**< AES-GCM configuration. */
+		struct {
+			uint32_t mmo_ctrl;
+			/**< Crypto control fields with algo type and op type in big
+			 * endian format.
+			 */
+			uint32_t wqe_aad_len;
+			/**< Crypto AAD length field in big endian format. */
+			uint32_t wqe_tag_len;
+			/**< Crypto tag length field in big endian format. */
+			uint16_t tag_len;
+			/**< AES-GCM crypto digest size in bytes. */
+			uint16_t aad_len;
+			/**< The length of the additional authenticated data (AAD) in bytes. */
+			uint32_t op_type;
+			/**< Operation type. */
+		};
+	};
 	uint32_t iv_offset:16;
 	/**< Starting point for Initialisation Vector. */
+	uint32_t iv_len;
+	/**< Initialisation Vector length. */
 	struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
 	uint32_t dek_id; /**< DEK ID */
 } __rte_packed;
drivers/crypto/mlx5/mlx5_crypto_gcm.c +47 -0

56 lines not on this page · raw message

Signature · 1 line
2.25.1
#18 lore raw

PATCH v2 6/9 common/mlx5: add WQE-based QP synchronous basics

In reply to Suanming Mou

Nvidia HW provides a synchronous mechanism between QPs. When creating the QPs, user can set one as primary and another as follower. The follower QP's WQE execution can be controlled by primary QP via SEND_EN WQE.

This commit introduces the SEND_EN WQE to improve the WQE execution sync-up between primary and follower QPs.

---
 drivers/common/mlx5/mlx5_devx_cmds.c |  6 ++++++
 drivers/common/mlx5/mlx5_devx_cmds.h |  3 +++
 drivers/common/mlx5/mlx5_prm.h       | 11 +++++++++++
 3 files changed, 20 insertions(+)
drivers/common/mlx5/mlx5_devx_cmds.c +6 -0
index 4332081165..ef87862a6d 100644
@@ -2475,6 +2475,12 @@ mlx5_devx_cmd_create_qp(void *ctx,
 				 attr->dbr_umem_valid);
 			MLX5_SET(qpc, qpc, dbr_umem_id, attr->dbr_umem_id);
 		}
+		if (attr->cd_master)
+			MLX5_SET(qpc, qpc, cd_master, attr->cd_master);
+		if (attr->cd_slave_send)
+			MLX5_SET(qpc, qpc, cd_slave_send, attr->cd_slave_send);
+		if (attr->cd_slave_recv)
+			MLX5_SET(qpc, qpc, cd_slave_receive, attr->cd_slave_recv);
 		MLX5_SET64(qpc, qpc, dbr_addr, attr->dbr_address);
 		MLX5_SET64(create_qp_in, in, wq_umem_offset,
 			   attr->wq_umem_offset);
drivers/common/mlx5/mlx5_devx_cmds.h +3 -0
index cb3f3a211b..e071cd841f 100644
@@ -559,6 +559,9 @@ struct mlx5_devx_qp_attr {
 	uint64_t wq_umem_offset;
 	uint32_t user_index:24;
 	uint32_t mmo:1;
+	uint32_t cd_master:1;
+	uint32_t cd_slave_send:1;
+	uint32_t cd_slave_recv:1;
 };
 
 struct mlx5_devx_virtio_q_couners_attr {
drivers/common/mlx5/mlx5_prm.h +11 -0
index 3b26499a47..96d5eb8de3 100644
@@ -589,6 +589,17 @@ struct mlx5_rdma_write_wqe {
 	struct mlx5_wqe_dseg dseg[];
 } __rte_packed;
 
+struct mlx5_wqe_send_en_seg {
+	uint32_t reserve[2];
+	uint32_t sqnpc;
+	uint32_t qpn;
+} __rte_packed;
+
+struct mlx5_wqe_send_en_wqe {
+	struct mlx5_wqe_cseg ctr;
+	struct mlx5_wqe_send_en_seg sseg;
+} __rte_packed;
+
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
Signature · 1 line
2.25.1
#19 lore raw

PATCH v2 7/9 crypto/mlx5: add queue pair setup for GCM

In reply to Suanming Mou

Crypto queue pair is for handling the encryption/decryption operations.

As AES-GCM AEAD API provides AAD, mbuf, digest separately, low-level FW only accepts the data in a single contiguous memory region, two internal QPs are created for AES-GCM queue pair. One for organizing the memory to be contiguous if they are not. The other is for crypto.

If the buffers are checked as implicitly contiguous, the buffer will be sent to the crypto QP directly for encryption/decryption. If not, the buffers will be handled by the first UMR QP. The UMR QP will convert the buffers to be contiguous one. Then the well organized "new" buffer can be handled by crypto QP.

The crypto QP is initialized as follower, and UMR as leader. Once crypto operation input buffer requires memory address space converting by UMR QP, the crypto QP processing will be triggered by UMR QP. Otherwise, the ring crypto QP doorbell directly.

The existing max_segs_num devarg is used for define how many segments the chained mbuf contains same as AES-XTS before.

---
 drivers/common/mlx5/mlx5_common_mr.h  |   1 +
 drivers/common/mlx5/mlx5_prm.h        |  22 +++
 drivers/common/mlx5/version.map       |   2 +
 drivers/crypto/mlx5/mlx5_crypto.h     |  15 ++
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 230 ++++++++++++++++++++++++++
 5 files changed, 270 insertions(+)
drivers/common/mlx5/mlx5_common_mr.h +1 -0
index 66623868a2..8789d403b1 100644
@@ -254,6 +254,7 @@ __rte_internal
 void
 mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
 
+__rte_internal
 void
 mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
 
drivers/common/mlx5/mlx5_prm.h +22 -0
index 96d5eb8de3..a502e29bd8 100644
@@ -470,6 +470,15 @@ struct mlx5_wqe_rseg {
 #define MLX5_UMRC_KO_OFFSET 16u
 #define MLX5_UMRC_TO_BS_OFFSET 0u
 
+/*
+ * As PRM describes, the address of the UMR pointer must be
+ * aligned to 2KB.
+ */
+#define MLX5_UMR_KLM_PTR_ALIGN (1 << 11)
+
+#define MLX5_UMR_KLM_NUM_ALIGN \
+	(MLX5_UMR_KLM_PTR_ALIGN / sizeof(struct mlx5_klm))
+
 struct mlx5_wqe_umr_cseg {
 	uint32_t if_cf_toe_cq_res;
 	uint32_t ko_to_bs;
@@ -674,6 +683,19 @@ union mlx5_gga_compress_opaque {
 	uint32_t data[64];
 };
 
+union mlx5_gga_crypto_opaque {
+	struct {
+		uint32_t syndrome;
+		uint32_t reserved0[2];
+		struct {
+			uint32_t iv[3];
+			uint32_t tag_size;
+			uint32_t aad_size;
+		} cp __rte_packed;
+	} __rte_packed;
+	uint8_t data[64];
+};
+
 struct mlx5_ifc_regexp_mmo_control_bits {
 	uint8_t reserved_at_31[0x2];
 	uint8_t le[0x1];
drivers/common/mlx5/version.map +2 -0
index f860b069de..0758ba76de 100644
@@ -159,5 +159,7 @@ INTERNAL {
 
 	mlx5_os_interrupt_handler_create; # WINDOWS_NO_EXPORT
 	mlx5_os_interrupt_handler_destroy; # WINDOWS_NO_EXPORT
+
+	mlx5_os_set_reg_mr_cb;
 	local: *;
 };
drivers/crypto/mlx5/mlx5_crypto.h +15 -0
index 6cb4d4ddec..88a09a6b1c 100644
@@ -28,8 +28,11 @@ struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
 	struct mlx5_common_device *cdev; /* Backend mlx5 device. */
 	struct rte_cryptodev *crypto_dev;
+	mlx5_reg_mr_t reg_mr_cb; /* Callback to reg_mr func */
+	mlx5_dereg_mr_t dereg_mr_cb; /* Callback to dereg_mr func */
 	struct mlx5_uar uar; /* User Access Region. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
+	uint32_t max_klm_num; /* Maximum supported klm. */
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	const struct rte_cryptodev_capabilities *caps;
 	struct rte_cryptodev_config dev_config;
@@ -46,15 +49,27 @@ struct mlx5_crypto_qp {
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_qp qp_obj;
+	struct mlx5_devx_qp umr_qp_obj;
 	struct rte_cryptodev_stats stats;
 	struct rte_crypto_op **ops;
 	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
+	struct mlx5_klm *klm_array;
+	union mlx5_gga_crypto_opaque *opaque_addr;
 	struct mlx5_mr_ctrl mr_ctrl;
+	struct mlx5_pmd_mr mr;
+	/* Crypto QP. */
 	uint8_t *wqe;
 	uint16_t entries_n;
+	uint16_t cq_entries_n;
 	uint16_t pi;
 	uint16_t ci;
 	uint16_t db_pi;
+	/* UMR QP. */
+	uint8_t *umr_wqe;
+	uint16_t umr_wqbbs;
+	uint16_t umr_pi;
+	uint16_t umr_ci;
+	uint32_t umr_errors;
 };
 
 struct mlx5_crypto_dek {
drivers/crypto/mlx5/mlx5_crypto_gcm.c +230 -0

254 lines not on this page · raw message

Signature · 1 line
2.25.1
#20 lore raw

PATCH v2 8/9 crypto/mlx5: add enqueue and dequeue operations

In reply to Suanming Mou

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous and there is no enough headroom/tailroom for copying AAD/digest, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE. The UMR WQE and crypto WQE are handled in two different QPs.

Crypto operation with non-contiguous buffers will have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, an additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP. The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.

---
 drivers/common/mlx5/mlx5_prm.h        |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |   9 +-
 drivers/crypto/mlx5/mlx5_crypto.h     |   8 +
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 588 ++++++++++++++++++++++++++
 4 files changed, 604 insertions(+), 2 deletions(-)
drivers/common/mlx5/mlx5_prm.h +1 -0
index a502e29bd8..98b71a4031 100644
@@ -617,6 +617,7 @@ struct mlx5_wqe_send_en_wqe {
 /* MMO metadata segment */
 
 #define	MLX5_OPCODE_MMO	0x2fu
+#define	MLX5_OPC_MOD_MMO_CRYPTO 0x6u
 #define	MLX5_OPC_MOD_MMO_REGEX 0x4u
 #define	MLX5_OPC_MOD_MMO_COMP 0x2u
 #define	MLX5_OPC_MOD_MMO_DECOMP 0x3u
drivers/crypto/mlx5/mlx5_crypto.c +7 -2
index ff632cd69a..4d7d3ef2a3 100644
@@ -62,8 +62,13 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 			MLX5_CRYPTO_FEATURE_FLAGS(priv->is_wrapped_mode);
 		dev_info->capabilities = priv->caps;
 		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
-		dev_info->min_mbuf_headroom_req = 0;
-		dev_info->min_mbuf_tailroom_req = 0;
+		if (priv->caps->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) {
+			dev_info->min_mbuf_headroom_req = MLX5_CRYPTO_GCM_MAX_AAD;
+			dev_info->min_mbuf_tailroom_req = MLX5_CRYPTO_GCM_MAX_DIGEST;
+		} else {
+			dev_info->min_mbuf_headroom_req = 0;
+			dev_info->min_mbuf_tailroom_req = 0;
+		}
 		dev_info->sym.max_nb_sessions = 0;
 		/*
 		 * If 0, the device does not have any limitation in number of
drivers/crypto/mlx5/mlx5_crypto.h +8 -0
index 88a09a6b1c..6dcb41b27c 100644
@@ -23,6 +23,8 @@
 #define MLX5_CRYPTO_KLM_SEGS_NUM(umr_wqe_sz) ((umr_wqe_sz -\
 					MLX5_CRYPTO_UMR_WQE_STATIC_SIZE) /\
 					MLX5_WSEG_SIZE)
+#define MLX5_CRYPTO_GCM_MAX_AAD 64
+#define MLX5_CRYPTO_GCM_MAX_DIGEST 16
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -61,6 +63,9 @@ struct mlx5_crypto_qp {
 	uint8_t *wqe;
 	uint16_t entries_n;
 	uint16_t cq_entries_n;
+	uint16_t reported_ci;
+	uint16_t qp_ci;
+	uint16_t cq_ci;
 	uint16_t pi;
 	uint16_t ci;
 	uint16_t db_pi;
@@ -70,6 +75,9 @@ struct mlx5_crypto_qp {
 	uint16_t umr_pi;
 	uint16_t umr_ci;
 	uint32_t umr_errors;
+	uint16_t last_gga_pi;
+	bool has_umr;
+	uint16_t cpy_tag_op;
 };
 
 struct mlx5_crypto_dek {
drivers/crypto/mlx5/mlx5_crypto_gcm.c +588 -0

612 lines not on this page · raw message

Signature · 1 line
2.25.1
#21 lore raw

PATCH v2 9/9 crypto/mlx5: enable AES-GCM capability

In reply to Suanming Mou

This commit generates AES-GCM capability based on the NIC attributes and enables AES-GCM algo.

An new devarg "algo" is added to identify if the crypto PMD will be initialized as AES-GCM(algo=1) or AES-XTS(algo=0, default).

---
 doc/guides/cryptodevs/mlx5.rst         | 48 +++++++++++++++++++-
 doc/guides/rel_notes/release_23_07.rst |  1 +
 drivers/crypto/mlx5/mlx5_crypto.c      | 26 +++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h      |  1 +
 drivers/crypto/mlx5/mlx5_crypto_gcm.c  | 63 ++++++++++++++++++++++++++
 5 files changed, 134 insertions(+), 5 deletions(-)
doc/guides/cryptodevs/mlx5.rst +47 -1
index b35ac5f5f2..9a0ae8b0d2 100644
@@ -21,6 +21,11 @@ and **NVIDIA BlueField-3** family adapters.
 Overview
 --------
 
+Nvidia MLX5 crypto driver supports AES-XTs and AES-GCM cryption.
+
+AES-XTS
+-------
+
 The device can provide disk encryption services,
 allowing data encryption and decryption towards a disk.
 Having all encryption/decryption operations done in a single device
@@ -38,13 +43,19 @@ The encryption does not require text to be aligned to the AES block size (128b).
 
 See :doc:`../../platform/mlx5` guide for more design details.
 
+AES-GCM
+-------
+The encryption and decryption processes the traffic as standard RTE crypto
+API defines. The supported AAD/digest/key size can be read from dev_info.
+
+
 Configuration
 -------------
 
 See the :ref:`mlx5 common configuration <mlx5_common_env>`.
 
 A device comes out of NVIDIA factory with pre-defined import methods.
-There are two possible import methods: wrapped or plaintext.
+There are two possible import methods: wrapped or plaintext(valid to AES-XTS only).
 
 In case the device is in wrapped mode, it needs to be moved to crypto operational mode.
 In order to move the device to crypto operational mode, credential and KEK
@@ -120,24 +131,36 @@ Driver options
 Please refer to :ref:`mlx5 common options <mlx5_common_driver_options>`
 for an additional list of options shared with other mlx5 drivers.
 
+- ``algo`` parameter [int]
+
+  - 0. AES-XTS crypto.
+
+  - 1. AES-GCM crypto.
+
+  Set to zero(AES-XTS) by default.
+
 - ``wcs_file`` parameter [string] - mandatory in wrapped mode
 
   File path including only the wrapped credential in string format of hexadecimal
   numbers, represent 48 bytes (8 bytes IV added by the AES key wrap algorithm).
+  This option is valid only to AES-XTS.
 
 - ``import_kek_id`` parameter [int]
 
   The identifier of the KEK, default value is 0 represents the operational
   register import_kek..
+  This option is valid only to AES-XTS.
 
 - ``credential_id`` parameter [int]
 
   The identifier of the credential, default value is 0 represents the operational
   register credential.
+  This option is valid only to AES-XTS.
 
 - ``keytag`` parameter [int]
 
   The plaintext of the keytag appended to the AES-XTS keys, default value is 0.
+  This option is valid only to AES-XTS.
 
 - ``max_segs_num`` parameter [int]
 
@@ -161,6 +184,8 @@ Limitations
 - The supported data-unit lengths are 512B and 4KB and 1MB. In case the `dataunit_len`
   is not provided in the cipher xform, the OP length is limited to the above
   values.
+- AES-GCM is only supported on BlueField-3.
+- AES-GCM only supported key import plaintext mode.
 
 
 Prerequisites
@@ -172,6 +197,7 @@ FW Prerequisites
 - xx.31.0328 for ConnectX-6.
 - xx.32.0108 for ConnectX-6 Dx and BlueField-2.
 - xx.36.xxxx for ConnectX-7 and BlueField-3.
+- xx.37.3010 for BlueField-3 and newer for AES-GCM.
 
 Linux Prerequisites
 ~~~~~~~~~~~~~~~~~~~
@@ -186,3 +212,23 @@ Windows Prerequisites
 
 - NVIDIA WINOF-2 version: **2.60** or higher.
   See :ref:`mlx5 common prerequisites <mlx5_windows_prerequisites>` for more details.
+
+
+Notes for rte_crypto AES-GCM
+----------------------------
+
+In AES-GCM mode, the HW requires continuous input and output of Additional
+Authenticated Data (AAD), payload, and digest (if needed). However, the RTE
+API only provides a single AAD input, which means that in the out-of-place
+mode, the AAD will be used in both input and output. This reuse of AAD in the
+out-of-place mode breaks the continuous output, which degrades the performance
+and introduces extra UMR WQE. If digest is not continuous after payload will
+also lead to that extra UMR WQE.
+
+To address this issue, current RTE API provides min_mbuf_headroom_req and
+min_mbuf_tailroom_req in rte_cryptodev_info as a hint to the PMD. It
+indicates the PMD can use the buffer before and after the mbuf payload as AAD
+and digest space. With this hint, the PMD will use the buffer before and
+after the mbuf payload directly via copying AAD and digest. However, the
+application must ensure that there is enough headroom and tailroom reserved
+for the mbuf. Or, for non-continuous operations, extra UMR WQE will be used.
doc/guides/rel_notes/release_23_07.rst +1 -0
index 946f89e83b..fbbdceab0b 100644
@@ -29,6 +29,7 @@ New Features
   * Added support for multi-packet RQ on Windows.
   * Added support for CQE compression on Windows.
   * Added support for enhanced multi-packet write on Windows.
+  * Added support for AES-GCM crypto.
 
 * **Added flow matching of tx queue.**
 
drivers/crypto/mlx5/mlx5_crypto.c +22 -4
index 4d7d3ef2a3..081e96ad4d 100644
@@ -269,6 +269,14 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->credential_pointer = (uint32_t)tmp;
 	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
+	} else if (strcmp(key, "algo") == 0) {
+		if (tmp == 1) {
+			devarg_prms->is_aes_gcm = 1;
+		} else if (tmp > 1) {
+			DRV_LOG(ERR, "Invalid algo.");
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
 	}
 	return 0;
 }
@@ -285,6 +293,7 @@ mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl *mkvlist,
 		"keytag",
 		"max_segs_num",
 		"wcs_file",
+		"algo",
 		NULL,
 	};
 
@@ -370,10 +379,19 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
 	priv->crypto_dev = crypto_dev;
 	priv->is_wrapped_mode = wrapped_mode;
 	priv->max_segs_num = devarg_prms.max_segs_num;
-	ret = mlx5_crypto_xts_init(priv);
-	if (ret) {
-		DRV_LOG(ERR, "Failed to init AES-XTS crypto.");
-		return -ENOTSUP;
+	/* Init and override AES-GCM configuration. */
+	if (devarg_prms.is_aes_gcm) {
+		ret = mlx5_crypto_gcm_init(priv);
+		if (ret) {
+			DRV_LOG(ERR, "Failed to init AES-GCM crypto.");
+			return -ENOTSUP;
+		}
+	} else {
+		ret = mlx5_crypto_xts_init(priv);
+		if (ret) {
+			DRV_LOG(ERR, "Failed to init AES-XTS crypto.");
+			return -ENOTSUP;
+		}
 	}
 	if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) {
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
drivers/crypto/mlx5/mlx5_crypto.h +1 -0
index 6dcb41b27c..36dacdcda4 100644
@@ -92,6 +92,7 @@ struct mlx5_crypto_devarg_params {
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
 	uint32_t max_segs_num;
+	uint32_t is_aes_gcm:1;
 };
 
 struct mlx5_crypto_session {
drivers/crypto/mlx5/mlx5_crypto_gcm.c +63 -0

83 lines not on this page · raw message

Signature · 1 line
2.25.1
#22 lore raw
In reply to Suanming Mou
Quoted 20 lines

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous and there is no enough headroom or tailroom for AAD or digest, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE. The UMR WQE and crypto WQE are handled in two different QPs.

The QP for UMR operation contains two types of WQE, UMR and SEND_EN WQE. The WQEs are built dynamically according to the crypto operation buffer address. Crypto operation with non-contiguous buffers will have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP. The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

Quoted 4 lines

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.
\

Change log missing.

Please get it reviewed from PMD maintainer. Also rebase the patchset, and fix checkpatch issues.

Quoted 31 lines

Suanming Mou (9):
common/mlx5: export memory region lookup by address crypto/mlx5: split AES-XTS
crypto/mlx5: add AES-GCM query and initialization crypto/mlx5: add AES-GCM encryption key crypto/mlx5: add AES-GCM session configure common/mlx5: add WQE-based QP synchronous basics crypto/mlx5: add queue pair setup for GCM crypto/mlx5: add enqueue and dequeue operations crypto/mlx5: enable AES-GCM capability

 doc/guides/cryptodevs/mlx5.rst         |  48 +-
 doc/guides/rel_notes/release_23_07.rst |   1 +
 drivers/common/mlx5/mlx5_common_mr.c   |   2 +-
 drivers/common/mlx5/mlx5_common_mr.h   |   5 +
 drivers/common/mlx5/mlx5_devx_cmds.c   |  21 +
 drivers/common/mlx5/mlx5_devx_cmds.h   |  16 +
 drivers/common/mlx5/mlx5_prm.h         |  65 +-
 drivers/common/mlx5/version.map        |   3 +
 drivers/crypto/mlx5/meson.build        |   2 +
 drivers/crypto/mlx5/mlx5_crypto.c      | 673 ++---------------
 drivers/crypto/mlx5/mlx5_crypto.h      | 101 ++-
 drivers/crypto/mlx5/mlx5_crypto_dek.c  | 102 ++-
 drivers/crypto/mlx5/mlx5_crypto_gcm.c  | 995 +++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_xts.c  | 645 ++++++++++++++++
 14 files changed, 2014 insertions(+), 665 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_gcm.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_xts.c

--
2.25.1

#23 lore raw

Hi,

Sorry for the late response.

Quoted 27 lines

-----Original Message-----
From: Akhil Goyal <[email protected]>
Sent: Thursday, June 15, 2023 2:12 AM
To: Suanming Mou <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: RE: [EXT] [PATCH v2 0/9] crypto/mlx5: support AES-GCM

Quoted 20 lines

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous and there is no enough headroom or tailroom for AAD or digest, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE. The UMR WQE and crypto WQE are handled in two different QPs.

The QP for UMR operation contains two types of WQE, UMR and SEND_EN WQE. The WQEs are built dynamically according to the crypto operation buffer address. Crypto operation with non-contiguous buffers will have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP.
The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

Quoted 5 lines
Quoted 4 lines

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.
\

Change log missing.

Wii updated in V3.

Quoted 3 lines

Please get it reviewed from PMD maintainer. Also rebase the patchset, and fix checkpatch issues.

The checkpatch issues are mainly due to the struct names false positive.

Quoted 33 lines
Quoted 33 lines

Suanming Mou (9):
common/mlx5: export memory region lookup by address crypto/mlx5: split AES-XTS
crypto/mlx5: add AES-GCM query and initialization crypto/mlx5: add AES-GCM encryption key crypto/mlx5: add AES-GCM session configure common/mlx5: add WQE-based QP synchronous basics crypto/mlx5: add queue pair setup for GCM crypto/mlx5: add enqueue and dequeue operations crypto/mlx5: enable AES-GCM capability

 doc/guides/cryptodevs/mlx5.rst         |  48 +-
 doc/guides/rel_notes/release_23_07.rst |   1 +
 drivers/common/mlx5/mlx5_common_mr.c   |   2 +-
 drivers/common/mlx5/mlx5_common_mr.h   |   5 +
 drivers/common/mlx5/mlx5_devx_cmds.c   |  21 +
 drivers/common/mlx5/mlx5_devx_cmds.h   |  16 +
 drivers/common/mlx5/mlx5_prm.h         |  65 +-
 drivers/common/mlx5/version.map        |   3 +
 drivers/crypto/mlx5/meson.build        |   2 +
 drivers/crypto/mlx5/mlx5_crypto.c      | 673 ++---------------
 drivers/crypto/mlx5/mlx5_crypto.h      | 101 ++-
 drivers/crypto/mlx5/mlx5_crypto_dek.c  | 102 ++-
drivers/crypto/mlx5/mlx5_crypto_gcm.c  | 995 +++++++++++++++++++++++++
drivers/crypto/mlx5/mlx5_crypto_xts.c  | 645 ++++++++++++++++
 14 files changed, 2014 insertions(+), 665 deletions(-)  create mode
100644 drivers/crypto/mlx5/mlx5_crypto_gcm.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_xts.c

--
2.25.1

#24 lore raw

v2 cover crypto/mlx5: support AES-GCM

In reply to Suanming Mou

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous and there is no enough headroom or tailroom for AAD or digest, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE. The UMR WQE and crypto WQE are handled in two different QPs.

The QP for UMR operation contains two types of WQE, UMR and SEND_EN WQE. The WQEs are built dynamically according to the crypto operation buffer address. Crypto operation with non-contiguous buffers will have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP. The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.

v2:
- split XTS and GCM code to different file.
- add headroom and tailroom optimize.

v3:
- fix AES-GCM 128b key creation.

Suanming Mou (9):
common/mlx5: export memory region lookup by address crypto/mlx5: split AES-XTS
crypto/mlx5: add AES-GCM query and initialization crypto/mlx5: add AES-GCM encryption key crypto/mlx5: add AES-GCM session configure common/mlx5: add WQE-based QP synchronous basics crypto/mlx5: add queue pair setup for GCM crypto/mlx5: add enqueue and dequeue operations crypto/mlx5: enable AES-GCM capability

 doc/guides/cryptodevs/mlx5.rst         |  48 +-
 doc/guides/rel_notes/release_23_07.rst |   1 +
 drivers/common/mlx5/mlx5_common_mr.c   |   2 +-
 drivers/common/mlx5/mlx5_common_mr.h   |   5 +
 drivers/common/mlx5/mlx5_devx_cmds.c   |  21 +
 drivers/common/mlx5/mlx5_devx_cmds.h   |  16 +
 drivers/common/mlx5/mlx5_prm.h         |  65 +-
 drivers/common/mlx5/version.map        |   3 +
 drivers/crypto/mlx5/meson.build        |   2 +
 drivers/crypto/mlx5/mlx5_crypto.c      | 673 ++---------------
 drivers/crypto/mlx5/mlx5_crypto.h      | 101 ++-
 drivers/crypto/mlx5/mlx5_crypto_dek.c  | 102 ++-
 drivers/crypto/mlx5/mlx5_crypto_gcm.c  | 997 +++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_xts.c  | 645 ++++++++++++++++
 14 files changed, 2016 insertions(+), 665 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_gcm.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_xts.c
Signature · 1 line
2.25.1
#25 lore raw

PATCH v3 1/9 common/mlx5: export memory region lookup by address

In case user provides the address without mempool. Export the function to lookup the address without mempool is required.

---
 drivers/common/mlx5/mlx5_common_mr.c | 2 +-
 drivers/common/mlx5/mlx5_common_mr.h | 4 ++++
 drivers/common/mlx5/version.map      | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)
drivers/common/mlx5/mlx5_common_mr.c +1 -1
index 7b14b0c7bf..40ff9153bd 100644
@@ -1059,7 +1059,7 @@ mr_lookup_caches(struct mlx5_mr_ctrl *mr_ctrl,
  * @return
  *   Searched LKey on success, UINT32_MAX on no match.
  */
-static uint32_t
+uint32_t
 mlx5_mr_addr2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, uintptr_t addr)
 {
 	uint32_t lkey;
drivers/common/mlx5/mlx5_common_mr.h +4 -0
index 12def1585f..66623868a2 100644
@@ -240,6 +240,10 @@ mlx5_mr_create(struct mlx5_common_device *cdev,
 	       struct mlx5_mr_share_cache *share_cache,
 	       struct mr_cache_entry *entry, uintptr_t addr);
 
+__rte_internal
+uint32_t
+mlx5_mr_addr2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, uintptr_t addr);
+
 /* mlx5_common_verbs.c */
 
 __rte_internal
drivers/common/mlx5/version.map +1 -0
index e05e1aa8c5..f860b069de 100644
@@ -122,6 +122,7 @@ INTERNAL {
 	mlx5_mr_ctrl_init;
 	mlx5_mr_flush_local_cache;
 	mlx5_mr_mb2mr_bh;
+	mlx5_mr_addr2mr_bh;
 
 	mlx5_nl_allmulti; # WINDOWS_NO_EXPORT
 	mlx5_nl_ifindex; # WINDOWS_NO_EXPORT
Signature · 1 line
2.25.1
#26 lore raw

PATCH v3 2/9 crypto/mlx5: split AES-XTS

In reply to Suanming Mou

As there will be other crypto algo be supported. This commit splits AES-XTS code to another *_xts.c file. The mlx5_crypto.c file will just contain the common code.

---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     | 642 ++------------------------
 drivers/crypto/mlx5/mlx5_crypto.h     |  33 ++
 drivers/crypto/mlx5/mlx5_crypto_xts.c | 594 ++++++++++++++++++++++++
 4 files changed, 667 insertions(+), 603 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_xts.c
drivers/crypto/mlx5/meson.build +1 -0
index a2691ec0f0..045e8ce81d 100644
@@ -15,6 +15,7 @@ endif
 
 sources = files(
         'mlx5_crypto.c',
+	'mlx5_crypto_xts.c',
         'mlx5_crypto_dek.c',
 )
 
drivers/crypto/mlx5/mlx5_crypto.c +39 -603

709 lines not on this page · raw message

drivers/crypto/mlx5/mlx5_crypto.h +33 -0
index a2771b3dab..05d8fe97fe 100644
@@ -31,6 +31,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_uar uar; /* User Access Region. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	const struct rte_cryptodev_capabilities *caps;
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
@@ -70,6 +71,35 @@ struct mlx5_crypto_devarg_params {
 	uint32_t max_segs_num;
 };
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t bsp_res;
+	/**< crypto_block_size_pointer and reserved 24 bits saved in big
+	 * endian format.
+	 */
+	uint32_t iv_offset:16;
+	/**< Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
+	uint32_t dek_id; /**< DEK ID */
+} __rte_packed;
+
+typedef void *(*mlx5_crypto_mkey_update_t)(struct mlx5_crypto_priv *priv,
+					   struct mlx5_crypto_qp *qp,
+					   uint32_t idx);
+
+void
+mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp,
+				   uint16_t n);
+
+int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+				   struct mlx5_crypto_qp *qp,
+				   struct mlx5_devx_mkey_attr *attr,
+				   mlx5_crypto_mkey_update_t update_cb);
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
@@ -84,4 +114,7 @@ mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
 void
 mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
 
+int
+mlx5_crypto_xts_init(struct mlx5_crypto_priv *priv);
+
 #endif /* MLX5_CRYPTO_H_ */
drivers/crypto/mlx5/mlx5_crypto_xts.c new file +594 -0

594 lines not on this page · raw message

Signature · 1 line
2.25.1
#27 lore raw

PATCH v3 3/9 crypto/mlx5: add AES-GCM query and initialization

In reply to Suanming Mou

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

This commit adds the AES-GCM attributes query and initialization function.

---
 drivers/common/mlx5/mlx5_devx_cmds.c  | 15 +++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h  | 13 ++++++++++
 drivers/common/mlx5/mlx5_prm.h        | 19 +++++++++++---
 drivers/crypto/mlx5/meson.build       |  1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  4 ++-
 drivers/crypto/mlx5/mlx5_crypto.h     |  3 +++
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 36 +++++++++++++++++++++++++++
 7 files changed, 87 insertions(+), 4 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_gcm.c
drivers/common/mlx5/mlx5_devx_cmds.c +15 -0
index 1e418a0353..4332081165 100644
@@ -1117,6 +1117,21 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		attr->crypto_wrapped_import_method = !!(MLX5_GET(crypto_caps,
 						hcattr, wrapped_import_method)
 						& 1 << 2);
+		attr->crypto_mmo.crypto_mmo_qp = MLX5_GET(crypto_caps, hcattr, crypto_mmo_qp);
+		attr->crypto_mmo.gcm_256_encrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_256_encrypt);
+		attr->crypto_mmo.gcm_128_encrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_128_encrypt);
+		attr->crypto_mmo.gcm_256_decrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_256_decrypt);
+		attr->crypto_mmo.gcm_128_decrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_128_decrypt);
+		attr->crypto_mmo.gcm_auth_tag_128 =
+			MLX5_GET(crypto_caps, hcattr, gcm_auth_tag_128);
+		attr->crypto_mmo.gcm_auth_tag_96 =
+			MLX5_GET(crypto_caps, hcattr, gcm_auth_tag_96);
+		attr->crypto_mmo.log_crypto_mmo_max_size =
+			MLX5_GET(crypto_caps, hcattr, log_crypto_mmo_max_size);
 	}
 	if (hca_cap_2_sup) {
 		hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
drivers/common/mlx5/mlx5_devx_cmds.h +13 -0
index dc3359268d..cb3f3a211b 100644
@@ -125,6 +125,18 @@ struct mlx5_hca_flex_attr {
 	uint8_t  header_length_mask_width;
 };
 
+__extension__
+struct mlx5_hca_crypto_mmo_attr {
+	uint32_t crypto_mmo_qp:1;
+	uint32_t gcm_256_encrypt:1;
+	uint32_t gcm_128_encrypt:1;
+	uint32_t gcm_256_decrypt:1;
+	uint32_t gcm_128_decrypt:1;
+	uint32_t gcm_auth_tag_128:1;
+	uint32_t gcm_auth_tag_96:1;
+	uint32_t log_crypto_mmo_max_size:6;
+};
+
 /* ISO C restricts enumerator values to range of 'int' */
 __extension__
 enum {
@@ -250,6 +262,7 @@ struct mlx5_hca_attr {
 	struct mlx5_hca_vdpa_attr vdpa;
 	struct mlx5_hca_flow_attr flow;
 	struct mlx5_hca_flex_attr flex;
+	struct mlx5_hca_crypto_mmo_attr crypto_mmo;
 	int log_max_qp_sz;
 	int log_max_cq_sz;
 	int log_max_qp;
drivers/common/mlx5/mlx5_prm.h +16 -3
index d67c4336e6..755bd73275 100644
@@ -4581,7 +4581,9 @@ struct mlx5_ifc_crypto_caps_bits {
 	u8 synchronize_dek[0x1];
 	u8 int_kek_manual[0x1];
 	u8 int_kek_auto[0x1];
-	u8 reserved_at_6[0x12];
+	u8 reserved_at_6[0xd];
+	u8 sw_wrapped_dek_key_purpose[0x1];
+	u8 reserved_at_14[0x4];
 	u8 wrapped_import_method[0x8];
 	u8 reserved_at_20[0x3];
 	u8 log_dek_max_alloc[0x5];
@@ -4598,8 +4600,19 @@ struct mlx5_ifc_crypto_caps_bits {
 	u8 log_dek_granularity[0x5];
 	u8 reserved_at_68[0x3];
 	u8 log_max_num_int_kek[0x5];
-	u8 reserved_at_70[0x10];
-	u8 reserved_at_80[0x780];
+	u8 sw_wrapped_dek_new[0x10];
+	u8 reserved_at_80[0x80];
+	u8 crypto_mmo_qp[0x1];
+	u8 crypto_aes_gcm_256_encrypt[0x1];
+	u8 crypto_aes_gcm_128_encrypt[0x1];
+	u8 crypto_aes_gcm_256_decrypt[0x1];
+	u8 crypto_aes_gcm_128_decrypt[0x1];
+	u8 gcm_auth_tag_128[0x1];
+	u8 gcm_auth_tag_96[0x1];
+	u8 reserved_at_107[0x3];
+	u8 log_crypto_mmo_max_size[0x6];
+	u8 reserved_at_110[0x10];
+	u8 reserved_at_120[0x6e0];
 };
 
 struct mlx5_ifc_crypto_commissioning_register_bits {
drivers/crypto/mlx5/meson.build +1 -0
index 045e8ce81d..17ffce89f0 100644
@@ -16,6 +16,7 @@ endif
 sources = files(
         'mlx5_crypto.c',
 	'mlx5_crypto_xts.c',
+	'mlx5_crypto_gcm.c',
         'mlx5_crypto_dek.c',
 )
 
drivers/crypto/mlx5/mlx5_crypto.c +3 -1
index 2e6bcc6ddc..ff632cd69a 100644
@@ -335,7 +335,9 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
+	if (!cdev->config.hca_attr.crypto ||
+	   (!cdev->config.hca_attr.aes_xts &&
+	    !cdev->config.hca_attr.crypto_mmo.crypto_mmo_qp)) {
 		DRV_LOG(ERR, "Not enough capabilities to support crypto "
 			"operations, maybe old FW/OFED version?");
 		rte_errno = ENOTSUP;
drivers/crypto/mlx5/mlx5_crypto.h +3 -0
index 05d8fe97fe..76f368ee91 100644
@@ -117,4 +117,7 @@ mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
 int
 mlx5_crypto_xts_init(struct mlx5_crypto_priv *priv);
 
+int
+mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv);
+
 #endif /* MLX5_CRYPTO_H_ */
drivers/crypto/mlx5/mlx5_crypto_gcm.c new file +36 -0
new file mode 100644
index 0000000000..bd78c6d66b
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_eal_paging.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+#include <bus_pci_driver.h>
+#include <rte_memory.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+static struct rte_cryptodev_capabilities mlx5_crypto_gcm_caps[] = {
+	{
+		.op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
+	},
+	{
+		.op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
+	}
+};
+
+int
+mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv)
+{
+	priv->caps = mlx5_crypto_gcm_caps;
+	return 0;
+}
+
Signature · 1 line
2.25.1
#28 lore raw

PATCH v3 4/9 crypto/mlx5: add AES-GCM encryption key

In reply to Suanming Mou

The crypto device requires the DEK(data encryption key) object for data encryption/decryption operation.

This commit adds the AES-GCM DEK object management support.

---
 drivers/crypto/mlx5/mlx5_crypto.h     |  17 ++++-
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 102 +++++++++++++-------------
 drivers/crypto/mlx5/mlx5_crypto_gcm.c |  33 +++++++++
 drivers/crypto/mlx5/mlx5_crypto_xts.c |  53 ++++++++++++-
 4 files changed, 150 insertions(+), 55 deletions(-)
drivers/crypto/mlx5/mlx5_crypto.h +16 -1
index 76f368ee91..bb5a557a38 100644
@@ -86,6 +86,11 @@ struct mlx5_crypto_session {
 	uint32_t dek_id; /**< DEK ID */
 } __rte_packed;
 
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_sym_xform *xform;
+	struct mlx5_crypto_priv *priv;
+};
+
 typedef void *(*mlx5_crypto_mkey_update_t)(struct mlx5_crypto_priv *priv,
 					   struct mlx5_crypto_qp *qp,
 					   uint32_t idx);
@@ -106,7 +111,7 @@ mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 
 struct mlx5_crypto_dek *
 mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
-			struct rte_crypto_cipher_xform *cipher);
+			struct rte_crypto_sym_xform *xform);
 
 int
 mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
@@ -120,4 +125,14 @@ mlx5_crypto_xts_init(struct mlx5_crypto_priv *priv);
 int
 mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv);
 
+int
+mlx5_crypto_dek_fill_xts_attr(struct mlx5_crypto_dek *dek,
+			      struct mlx5_devx_dek_attr *dek_attr,
+			      void *cb_ctx);
+
+int
+mlx5_crypto_dek_fill_gcm_attr(struct mlx5_crypto_dek *dek,
+			      struct mlx5_devx_dek_attr *dek_attr,
+			      void *cb_ctx);
+
 #endif /* MLX5_CRYPTO_H_ */
drivers/crypto/mlx5/mlx5_crypto_dek.c +49 -53

154 lines not on this page · raw message

drivers/crypto/mlx5/mlx5_crypto_gcm.c +33 -0
index bd78c6d66b..5b315ef42c 100644
@@ -27,6 +27,39 @@ static struct rte_cryptodev_capabilities mlx5_crypto_gcm_caps[] = {
 	}
 };
 
+int
+mlx5_crypto_dek_fill_gcm_attr(struct mlx5_crypto_dek *dek,
+			      struct mlx5_devx_dek_attr *dek_attr,
+			      void *cb_ctx)
+{
+	uint32_t offset = 0;
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_aead_xform *aead_ctx = &ctx->xform->aead;
+
+	if (aead_ctx->algo != RTE_CRYPTO_AEAD_AES_GCM) {
+		DRV_LOG(ERR, "Only AES-GCM algo supported.");
+		return -EINVAL;
+	}
+	dek_attr->key_purpose = MLX5_CRYPTO_KEY_PURPOSE_GCM;
+	switch (aead_ctx->key.length) {
+	case 16:
+		offset = 16;
+		dek->size = 16;
+		dek_attr->key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 32:
+		dek->size = 32;
+		dek_attr->key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Wrapped key size not supported.");
+		return -EINVAL;
+	}
+	memcpy(&dek_attr->key[offset], aead_ctx->key.data, aead_ctx->key.length);
+	memcpy(&dek->data, aead_ctx->key.data, aead_ctx->key.length);
+	return 0;
+}
+
 int
 mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv)
 {
drivers/crypto/mlx5/mlx5_crypto_xts.c +52 -1

65 lines not on this page · raw message

Signature · 1 line
2.25.1
#29 lore raw

PATCH v3 5/9 crypto/mlx5: add AES-GCM session configure

In reply to Suanming Mou

Sessions are used in symmetric transformations in order to prepare objects and data for packet processing stage.

The AES-GCM session includes IV, AAD, digest(tag), DEK, operation mode information.

---
 drivers/common/mlx5/mlx5_prm.h        | 12 +++++++
 drivers/crypto/mlx5/mlx5_crypto.h     | 40 ++++++++++++++++++-----
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 47 +++++++++++++++++++++++++++
 3 files changed, 91 insertions(+), 8 deletions(-)
drivers/common/mlx5/mlx5_prm.h +12 -0
index 755bd73275..6b48c6ca32 100644
@@ -523,11 +523,23 @@ enum {
 	MLX5_BLOCK_SIZE_4048B	= 0x6,
 };
 
+enum {
+	MLX5_ENCRYPTION_TYPE_AES_GCM = 0x3,
+};
+
+enum {
+	MLX5_CRYPTO_OP_TYPE_ENCRYPTION = 0x0,
+	MLX5_CRYPTO_OP_TYPE_DECRYPTION = 0x1,
+};
+
 #define MLX5_BSF_SIZE_OFFSET		30
 #define MLX5_BSF_P_TYPE_OFFSET		24
 #define MLX5_ENCRYPTION_ORDER_OFFSET	16
 #define MLX5_BLOCK_SIZE_OFFSET		24
 
+#define MLX5_CRYPTO_MMO_TYPE_OFFSET 24
+#define MLX5_CRYPTO_MMO_OP_OFFSET 20
+
 struct mlx5_wqe_umr_bsf_seg {
 	/*
 	 * bs_bpt_eo_es contains:
drivers/crypto/mlx5/mlx5_crypto.h +32 -8
index bb5a557a38..6cb4d4ddec 100644
@@ -72,16 +72,40 @@ struct mlx5_crypto_devarg_params {
 };
 
 struct mlx5_crypto_session {
-	uint32_t bs_bpt_eo_es;
-	/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
-	 * saved in big endian format.
-	 */
-	uint32_t bsp_res;
-	/**< crypto_block_size_pointer and reserved 24 bits saved in big
-	 * endian format.
-	 */
+	union {
+		/**< AES-XTS configuration. */
+		struct {
+			uint32_t bs_bpt_eo_es;
+			/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
+			 * saved in big endian format.
+			 */
+			uint32_t bsp_res;
+			/**< crypto_block_size_pointer and reserved 24 bits saved in big
+			 * endian format.
+			 */
+		};
+		/**< AES-GCM configuration. */
+		struct {
+			uint32_t mmo_ctrl;
+			/**< Crypto control fields with algo type and op type in big
+			 * endian format.
+			 */
+			uint32_t wqe_aad_len;
+			/**< Crypto AAD length field in big endian format. */
+			uint32_t wqe_tag_len;
+			/**< Crypto tag length field in big endian format. */
+			uint16_t tag_len;
+			/**< AES-GCM crypto digest size in bytes. */
+			uint16_t aad_len;
+			/**< The length of the additional authenticated data (AAD) in bytes. */
+			uint32_t op_type;
+			/**< Operation type. */
+		};
+	};
 	uint32_t iv_offset:16;
 	/**< Starting point for Initialisation Vector. */
+	uint32_t iv_len;
+	/**< Initialisation Vector length. */
 	struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
 	uint32_t dek_id; /**< DEK ID */
 } __rte_packed;
drivers/crypto/mlx5/mlx5_crypto_gcm.c +47 -0

56 lines not on this page · raw message

Signature · 1 line
2.25.1
#30 lore raw

PATCH v3 6/9 common/mlx5: add WQE-based QP synchronous basics

In reply to Suanming Mou

Nvidia HW provides a synchronous mechanism between QPs. When creating the QPs, user can set one as primary and another as follower. The follower QP's WQE execution can be controlled by primary QP via SEND_EN WQE.

This commit introduces the SEND_EN WQE to improve the WQE execution sync-up between primary and follower QPs.

---
 drivers/common/mlx5/mlx5_devx_cmds.c |  6 ++++++
 drivers/common/mlx5/mlx5_devx_cmds.h |  3 +++
 drivers/common/mlx5/mlx5_prm.h       | 11 +++++++++++
 3 files changed, 20 insertions(+)
drivers/common/mlx5/mlx5_devx_cmds.c +6 -0
index 4332081165..ef87862a6d 100644
@@ -2475,6 +2475,12 @@ mlx5_devx_cmd_create_qp(void *ctx,
 				 attr->dbr_umem_valid);
 			MLX5_SET(qpc, qpc, dbr_umem_id, attr->dbr_umem_id);
 		}
+		if (attr->cd_master)
+			MLX5_SET(qpc, qpc, cd_master, attr->cd_master);
+		if (attr->cd_slave_send)
+			MLX5_SET(qpc, qpc, cd_slave_send, attr->cd_slave_send);
+		if (attr->cd_slave_recv)
+			MLX5_SET(qpc, qpc, cd_slave_receive, attr->cd_slave_recv);
 		MLX5_SET64(qpc, qpc, dbr_addr, attr->dbr_address);
 		MLX5_SET64(create_qp_in, in, wq_umem_offset,
 			   attr->wq_umem_offset);
drivers/common/mlx5/mlx5_devx_cmds.h +3 -0
index cb3f3a211b..e071cd841f 100644
@@ -559,6 +559,9 @@ struct mlx5_devx_qp_attr {
 	uint64_t wq_umem_offset;
 	uint32_t user_index:24;
 	uint32_t mmo:1;
+	uint32_t cd_master:1;
+	uint32_t cd_slave_send:1;
+	uint32_t cd_slave_recv:1;
 };
 
 struct mlx5_devx_virtio_q_couners_attr {
drivers/common/mlx5/mlx5_prm.h +11 -0
index 6b48c6ca32..2d0a34ffbc 100644
@@ -589,6 +589,17 @@ struct mlx5_rdma_write_wqe {
 	struct mlx5_wqe_dseg dseg[];
 } __rte_packed;
 
+struct mlx5_wqe_send_en_seg {
+	uint32_t reserve[2];
+	uint32_t sqnpc;
+	uint32_t qpn;
+} __rte_packed;
+
+struct mlx5_wqe_send_en_wqe {
+	struct mlx5_wqe_cseg ctr;
+	struct mlx5_wqe_send_en_seg sseg;
+} __rte_packed;
+
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
Signature · 1 line
2.25.1
#31 lore raw

PATCH v3 7/9 crypto/mlx5: add queue pair setup for GCM

In reply to Suanming Mou

Crypto queue pair is for handling the encryption/decryption operations.

As AES-GCM AEAD API provides AAD, mbuf, digest separately, low-level FW only accepts the data in a single contiguous memory region, two internal QPs are created for AES-GCM queue pair. One for organizing the memory to be contiguous if they are not. The other is for crypto.

If the buffers are checked as implicitly contiguous, the buffer will be sent to the crypto QP directly for encryption/decryption. If not, the buffers will be handled by the first UMR QP. The UMR QP will convert the buffers to be contiguous one. Then the well organized "new" buffer can be handled by crypto QP.

The crypto QP is initialized as follower, and UMR as leader. Once crypto operation input buffer requires memory address space converting by UMR QP, the crypto QP processing will be triggered by UMR QP. Otherwise, the ring crypto QP doorbell directly.

The existing max_segs_num devarg is used for define how many segments the chained mbuf contains same as AES-XTS before.

---
 drivers/common/mlx5/mlx5_common_mr.h  |   1 +
 drivers/common/mlx5/mlx5_prm.h        |  22 +++
 drivers/common/mlx5/version.map       |   2 +
 drivers/crypto/mlx5/mlx5_crypto.h     |  15 ++
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 230 ++++++++++++++++++++++++++
 5 files changed, 270 insertions(+)
drivers/common/mlx5/mlx5_common_mr.h +1 -0
index 66623868a2..8789d403b1 100644
@@ -254,6 +254,7 @@ __rte_internal
 void
 mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
 
+__rte_internal
 void
 mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
 
drivers/common/mlx5/mlx5_prm.h +22 -0
index 2d0a34ffbc..60dff9dcda 100644
@@ -470,6 +470,15 @@ struct mlx5_wqe_rseg {
 #define MLX5_UMRC_KO_OFFSET 16u
 #define MLX5_UMRC_TO_BS_OFFSET 0u
 
+/*
+ * As PRM describes, the address of the UMR pointer must be
+ * aligned to 2KB.
+ */
+#define MLX5_UMR_KLM_PTR_ALIGN (1 << 11)
+
+#define MLX5_UMR_KLM_NUM_ALIGN \
+	(MLX5_UMR_KLM_PTR_ALIGN / sizeof(struct mlx5_klm))
+
 struct mlx5_wqe_umr_cseg {
 	uint32_t if_cf_toe_cq_res;
 	uint32_t ko_to_bs;
@@ -674,6 +683,19 @@ union mlx5_gga_compress_opaque {
 	uint32_t data[64];
 };
 
+union mlx5_gga_crypto_opaque {
+	struct {
+		uint32_t syndrome;
+		uint32_t reserved0[2];
+		struct {
+			uint32_t iv[3];
+			uint32_t tag_size;
+			uint32_t aad_size;
+		} cp __rte_packed;
+	} __rte_packed;
+	uint8_t data[64];
+};
+
 struct mlx5_ifc_regexp_mmo_control_bits {
 	uint8_t reserved_at_31[0x2];
 	uint8_t le[0x1];
drivers/common/mlx5/version.map +2 -0
index f860b069de..0758ba76de 100644
@@ -159,5 +159,7 @@ INTERNAL {
 
 	mlx5_os_interrupt_handler_create; # WINDOWS_NO_EXPORT
 	mlx5_os_interrupt_handler_destroy; # WINDOWS_NO_EXPORT
+
+	mlx5_os_set_reg_mr_cb;
 	local: *;
 };
drivers/crypto/mlx5/mlx5_crypto.h +15 -0
index 6cb4d4ddec..88a09a6b1c 100644
@@ -28,8 +28,11 @@ struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
 	struct mlx5_common_device *cdev; /* Backend mlx5 device. */
 	struct rte_cryptodev *crypto_dev;
+	mlx5_reg_mr_t reg_mr_cb; /* Callback to reg_mr func */
+	mlx5_dereg_mr_t dereg_mr_cb; /* Callback to dereg_mr func */
 	struct mlx5_uar uar; /* User Access Region. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
+	uint32_t max_klm_num; /* Maximum supported klm. */
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	const struct rte_cryptodev_capabilities *caps;
 	struct rte_cryptodev_config dev_config;
@@ -46,15 +49,27 @@ struct mlx5_crypto_qp {
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_qp qp_obj;
+	struct mlx5_devx_qp umr_qp_obj;
 	struct rte_cryptodev_stats stats;
 	struct rte_crypto_op **ops;
 	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
+	struct mlx5_klm *klm_array;
+	union mlx5_gga_crypto_opaque *opaque_addr;
 	struct mlx5_mr_ctrl mr_ctrl;
+	struct mlx5_pmd_mr mr;
+	/* Crypto QP. */
 	uint8_t *wqe;
 	uint16_t entries_n;
+	uint16_t cq_entries_n;
 	uint16_t pi;
 	uint16_t ci;
 	uint16_t db_pi;
+	/* UMR QP. */
+	uint8_t *umr_wqe;
+	uint16_t umr_wqbbs;
+	uint16_t umr_pi;
+	uint16_t umr_ci;
+	uint32_t umr_errors;
 };
 
 struct mlx5_crypto_dek {
drivers/crypto/mlx5/mlx5_crypto_gcm.c +230 -0

254 lines not on this page · raw message

Signature · 1 line
2.25.1
#32 lore raw

PATCH v3 8/9 crypto/mlx5: add enqueue and dequeue operations

In reply to Suanming Mou

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous and there is no enough headroom/tailroom for copying AAD/digest, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE. The UMR WQE and crypto WQE are handled in two different QPs.

Crypto operation with non-contiguous buffers will have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, an additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP. The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.

---
 drivers/common/mlx5/mlx5_prm.h        |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |   9 +-
 drivers/crypto/mlx5/mlx5_crypto.h     |   8 +
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 588 ++++++++++++++++++++++++++
 4 files changed, 604 insertions(+), 2 deletions(-)
drivers/common/mlx5/mlx5_prm.h +1 -0
index 60dff9dcda..0d84d92af5 100644
@@ -617,6 +617,7 @@ struct mlx5_wqe_send_en_wqe {
 /* MMO metadata segment */
 
 #define	MLX5_OPCODE_MMO	0x2fu
+#define	MLX5_OPC_MOD_MMO_CRYPTO 0x6u
 #define	MLX5_OPC_MOD_MMO_REGEX 0x4u
 #define	MLX5_OPC_MOD_MMO_COMP 0x2u
 #define	MLX5_OPC_MOD_MMO_DECOMP 0x3u
drivers/crypto/mlx5/mlx5_crypto.c +7 -2
index ff632cd69a..4d7d3ef2a3 100644
@@ -62,8 +62,13 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 			MLX5_CRYPTO_FEATURE_FLAGS(priv->is_wrapped_mode);
 		dev_info->capabilities = priv->caps;
 		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
-		dev_info->min_mbuf_headroom_req = 0;
-		dev_info->min_mbuf_tailroom_req = 0;
+		if (priv->caps->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) {
+			dev_info->min_mbuf_headroom_req = MLX5_CRYPTO_GCM_MAX_AAD;
+			dev_info->min_mbuf_tailroom_req = MLX5_CRYPTO_GCM_MAX_DIGEST;
+		} else {
+			dev_info->min_mbuf_headroom_req = 0;
+			dev_info->min_mbuf_tailroom_req = 0;
+		}
 		dev_info->sym.max_nb_sessions = 0;
 		/*
 		 * If 0, the device does not have any limitation in number of
drivers/crypto/mlx5/mlx5_crypto.h +8 -0
index 88a09a6b1c..6dcb41b27c 100644
@@ -23,6 +23,8 @@
 #define MLX5_CRYPTO_KLM_SEGS_NUM(umr_wqe_sz) ((umr_wqe_sz -\
 					MLX5_CRYPTO_UMR_WQE_STATIC_SIZE) /\
 					MLX5_WSEG_SIZE)
+#define MLX5_CRYPTO_GCM_MAX_AAD 64
+#define MLX5_CRYPTO_GCM_MAX_DIGEST 16
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -61,6 +63,9 @@ struct mlx5_crypto_qp {
 	uint8_t *wqe;
 	uint16_t entries_n;
 	uint16_t cq_entries_n;
+	uint16_t reported_ci;
+	uint16_t qp_ci;
+	uint16_t cq_ci;
 	uint16_t pi;
 	uint16_t ci;
 	uint16_t db_pi;
@@ -70,6 +75,9 @@ struct mlx5_crypto_qp {
 	uint16_t umr_pi;
 	uint16_t umr_ci;
 	uint32_t umr_errors;
+	uint16_t last_gga_pi;
+	bool has_umr;
+	uint16_t cpy_tag_op;
 };
 
 struct mlx5_crypto_dek {
drivers/crypto/mlx5/mlx5_crypto_gcm.c +588 -0

612 lines not on this page · raw message

Signature · 1 line
2.25.1
#33 lore raw

PATCH v3 9/9 crypto/mlx5: enable AES-GCM capability

In reply to Suanming Mou

This commit generates AES-GCM capability based on the NIC attributes and enables AES-GCM algo.

An new devarg "algo" is added to identify if the crypto PMD will be initialized as AES-GCM(algo=1) or AES-XTS(algo=0, default).

---
 doc/guides/cryptodevs/mlx5.rst         | 48 +++++++++++++++++++-
 doc/guides/rel_notes/release_23_07.rst |  1 +
 drivers/crypto/mlx5/mlx5_crypto.c      | 26 +++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h      |  1 +
 drivers/crypto/mlx5/mlx5_crypto_gcm.c  | 63 ++++++++++++++++++++++++++
 5 files changed, 134 insertions(+), 5 deletions(-)
doc/guides/cryptodevs/mlx5.rst +47 -1
index b35ac5f5f2..9a0ae8b0d2 100644
@@ -21,6 +21,11 @@ and **NVIDIA BlueField-3** family adapters.
 Overview
 --------
 
+Nvidia MLX5 crypto driver supports AES-XTs and AES-GCM cryption.
+
+AES-XTS
+-------
+
 The device can provide disk encryption services,
 allowing data encryption and decryption towards a disk.
 Having all encryption/decryption operations done in a single device
@@ -38,13 +43,19 @@ The encryption does not require text to be aligned to the AES block size (128b).
 
 See :doc:`../../platform/mlx5` guide for more design details.
 
+AES-GCM
+-------
+The encryption and decryption processes the traffic as standard RTE crypto
+API defines. The supported AAD/digest/key size can be read from dev_info.
+
+
 Configuration
 -------------
 
 See the :ref:`mlx5 common configuration <mlx5_common_env>`.
 
 A device comes out of NVIDIA factory with pre-defined import methods.
-There are two possible import methods: wrapped or plaintext.
+There are two possible import methods: wrapped or plaintext(valid to AES-XTS only).
 
 In case the device is in wrapped mode, it needs to be moved to crypto operational mode.
 In order to move the device to crypto operational mode, credential and KEK
@@ -120,24 +131,36 @@ Driver options
 Please refer to :ref:`mlx5 common options <mlx5_common_driver_options>`
 for an additional list of options shared with other mlx5 drivers.
 
+- ``algo`` parameter [int]
+
+  - 0. AES-XTS crypto.
+
+  - 1. AES-GCM crypto.
+
+  Set to zero(AES-XTS) by default.
+
 - ``wcs_file`` parameter [string] - mandatory in wrapped mode
 
   File path including only the wrapped credential in string format of hexadecimal
   numbers, represent 48 bytes (8 bytes IV added by the AES key wrap algorithm).
+  This option is valid only to AES-XTS.
 
 - ``import_kek_id`` parameter [int]
 
   The identifier of the KEK, default value is 0 represents the operational
   register import_kek..
+  This option is valid only to AES-XTS.
 
 - ``credential_id`` parameter [int]
 
   The identifier of the credential, default value is 0 represents the operational
   register credential.
+  This option is valid only to AES-XTS.
 
 - ``keytag`` parameter [int]
 
   The plaintext of the keytag appended to the AES-XTS keys, default value is 0.
+  This option is valid only to AES-XTS.
 
 - ``max_segs_num`` parameter [int]
 
@@ -161,6 +184,8 @@ Limitations
 - The supported data-unit lengths are 512B and 4KB and 1MB. In case the `dataunit_len`
   is not provided in the cipher xform, the OP length is limited to the above
   values.
+- AES-GCM is only supported on BlueField-3.
+- AES-GCM only supported key import plaintext mode.
 
 
 Prerequisites
@@ -172,6 +197,7 @@ FW Prerequisites
 - xx.31.0328 for ConnectX-6.
 - xx.32.0108 for ConnectX-6 Dx and BlueField-2.
 - xx.36.xxxx for ConnectX-7 and BlueField-3.
+- xx.37.3010 for BlueField-3 and newer for AES-GCM.
 
 Linux Prerequisites
 ~~~~~~~~~~~~~~~~~~~
@@ -186,3 +212,23 @@ Windows Prerequisites
 
 - NVIDIA WINOF-2 version: **2.60** or higher.
   See :ref:`mlx5 common prerequisites <mlx5_windows_prerequisites>` for more details.
+
+
+Notes for rte_crypto AES-GCM
+----------------------------
+
+In AES-GCM mode, the HW requires continuous input and output of Additional
+Authenticated Data (AAD), payload, and digest (if needed). However, the RTE
+API only provides a single AAD input, which means that in the out-of-place
+mode, the AAD will be used in both input and output. This reuse of AAD in the
+out-of-place mode breaks the continuous output, which degrades the performance
+and introduces extra UMR WQE. If digest is not continuous after payload will
+also lead to that extra UMR WQE.
+
+To address this issue, current RTE API provides min_mbuf_headroom_req and
+min_mbuf_tailroom_req in rte_cryptodev_info as a hint to the PMD. It
+indicates the PMD can use the buffer before and after the mbuf payload as AAD
+and digest space. With this hint, the PMD will use the buffer before and
+after the mbuf payload directly via copying AAD and digest. However, the
+application must ensure that there is enough headroom and tailroom reserved
+for the mbuf. Or, for non-continuous operations, extra UMR WQE will be used.
doc/guides/rel_notes/release_23_07.rst +1 -0
index 027ae7bd2d..bbb8eddbca 100644
@@ -131,6 +131,7 @@ New Features
   * Added support for CQE compression on Windows.
   * Added support for enhanced multi-packet write on Windows.
   * Added support for quota flow action and item.
+  * Added support for AES-GCM crypto.
 
 * **Added vmxnet3 version 7 support.**
 
drivers/crypto/mlx5/mlx5_crypto.c +22 -4
index 4d7d3ef2a3..081e96ad4d 100644
@@ -269,6 +269,14 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->credential_pointer = (uint32_t)tmp;
 	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
+	} else if (strcmp(key, "algo") == 0) {
+		if (tmp == 1) {
+			devarg_prms->is_aes_gcm = 1;
+		} else if (tmp > 1) {
+			DRV_LOG(ERR, "Invalid algo.");
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
 	}
 	return 0;
 }
@@ -285,6 +293,7 @@ mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl *mkvlist,
 		"keytag",
 		"max_segs_num",
 		"wcs_file",
+		"algo",
 		NULL,
 	};
 
@@ -370,10 +379,19 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
 	priv->crypto_dev = crypto_dev;
 	priv->is_wrapped_mode = wrapped_mode;
 	priv->max_segs_num = devarg_prms.max_segs_num;
-	ret = mlx5_crypto_xts_init(priv);
-	if (ret) {
-		DRV_LOG(ERR, "Failed to init AES-XTS crypto.");
-		return -ENOTSUP;
+	/* Init and override AES-GCM configuration. */
+	if (devarg_prms.is_aes_gcm) {
+		ret = mlx5_crypto_gcm_init(priv);
+		if (ret) {
+			DRV_LOG(ERR, "Failed to init AES-GCM crypto.");
+			return -ENOTSUP;
+		}
+	} else {
+		ret = mlx5_crypto_xts_init(priv);
+		if (ret) {
+			DRV_LOG(ERR, "Failed to init AES-XTS crypto.");
+			return -ENOTSUP;
+		}
 	}
 	if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) {
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
drivers/crypto/mlx5/mlx5_crypto.h +1 -0
index 6dcb41b27c..36dacdcda4 100644
@@ -92,6 +92,7 @@ struct mlx5_crypto_devarg_params {
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
 	uint32_t max_segs_num;
+	uint32_t is_aes_gcm:1;
 };
 
 struct mlx5_crypto_session {
drivers/crypto/mlx5/mlx5_crypto_gcm.c +63 -0

83 lines not on this page · raw message

Signature · 1 line
2.25.1
#34 lore raw

RE: [EXT] [PATCH v3 9/9] crypto/mlx5: enable AES-GCM capability

Quoted 7 lines

This commit generates AES-GCM capability based on the NIC attributes and enables AES-GCM algo.

An new devarg "algo" is added to identify if the crypto PMD will be initialized as AES-GCM(algo=1) or AES-XTS(algo=0, default).

You should mention changelog for the changes done in the current patchset over previous one. It helps in review.

Also update to "doc/guides/cryptodevs/features/mlx5.ini" is missing in this patch.

Also get an ack from mlx5 crypto maintainer.

Quoted 7 lines
---
 doc/guides/cryptodevs/mlx5.rst         | 48 +++++++++++++++++++-
 doc/guides/rel_notes/release_23_07.rst |  1 +
 drivers/crypto/mlx5/mlx5_crypto.c      | 26 +++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h      |  1 +
 drivers/crypto/mlx5/mlx5_crypto_gcm.c  | 63 ++++++++++++++++++++++++++
 5 files changed, 134 insertions(+), 5 deletions(-)
#35 lore raw

RE: [EXT] [PATCH v3 9/9] crypto/mlx5: enable AES-GCM capability

Quoted 13 lines

-----Original Message-----
From: Akhil Goyal <[email protected]>
Sent: Tuesday, June 20, 2023 5:25 PM
To: Suanming Mou <[email protected]>; Matan Azrad <[email protected]>
Cc: Raslan Darawsheh <[email protected]>; [email protected]
Subject: RE: [EXT] [PATCH v3 9/9] crypto/mlx5: enable AES-GCM capability

Quoted 3 lines

This commit generates AES-GCM capability based on the NIC attributes and enables AES-GCM algo.

Quoted 8 lines
Quoted 4 lines

An new devarg "algo" is added to identify if the crypto PMD will be initialized as AES-GCM(algo=1) or AES-XTS(algo=0, default).

You should mention changelog for the changes done in the current patchset over previous one.
It helps in review.

V1 is RFC, V1 compared to V2 was xts and gcm file split. V3 just did a minor fix in aes-gcm 256 key creation. I put the change log in the cover-letter, but thanks for the suggestion, will update the change log in single patch as well.

Quoted 2 lines

Also update to "doc/guides/cryptodevs/features/mlx5.ini" is missing in this patch.

You are right, it is missing here.

Quoted 12 lines

Also get an ack from mlx5 crypto maintainer.

Quoted 8 lines
---
 doc/guides/cryptodevs/mlx5.rst         | 48 +++++++++++++++++++-
 doc/guides/rel_notes/release_23_07.rst |  1 +
 drivers/crypto/mlx5/mlx5_crypto.c      | 26 +++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h      |  1 +
 drivers/crypto/mlx5/mlx5_crypto_gcm.c  | 63
++++++++++++++++++++++++++
 5 files changed, 134 insertions(+), 5 deletions(-)
#36 lore raw

RE: [EXT] [PATCH v3 9/9] crypto/mlx5: enable AES-GCM capability

Quoted 10 lines
Quoted 1 line

You should mention changelog for the changes done in the current patchset

over

Quoted 2 lines

previous one.
It helps in review.

V1 is RFC, V1 compared to V2 was xts and gcm file split. V3 just did a minor fix in aes-gcm 256 key creation.
I put the change log in the cover-letter, but thanks for the suggestion, will update the change log in single patch as well.

I do not see the cover-letter for v3 in mailing list.

#37 lore raw

RE: [EXT] [PATCH v3 9/9] crypto/mlx5: enable AES-GCM capability

Quoted 16 lines

-----Original Message-----
From: Akhil Goyal <[email protected]>
Sent: Tuesday, June 20, 2023 5:48 PM
To: Suanming Mou <[email protected]>; Matan Azrad <[email protected]>
Cc: Raslan Darawsheh <[email protected]>; [email protected]
Subject: RE: [EXT] [PATCH v3 9/9] crypto/mlx5: enable AES-GCM capability

Quoted 6 lines
Quoted 2 lines

You should mention changelog for the changes done in the current patchset

over

Quoted 2 lines

previous one.
It helps in review.

Quoted 6 lines
Quoted 5 lines

V1 is RFC, V1 compared to V2 was xts and gcm file split. V3 just did a minor fix in aes-gcm 256 key creation.
I put the change log in the cover-letter, but thanks for the suggestion, will update the change log in single patch as well.

I do not see the cover-letter for v3 in mailing list.

Please see my reply in the v3 cover-letter.

#38 lore raw
In reply to Suanming Mou

Hi Akhil,

Maybe due to "To" is empty, it was not collected correctly to the ML. But it was in my inbox, and you were cced.

Thanks,
Suanming

Quoted 69 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Tuesday, June 20, 2023 9:23 AM
Cc: Raslan Darawsheh <[email protected]>; [email protected]; [email protected]
Subject: [PATCH v2 0/9] crypto/mlx5: support AES-GCM

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous and there is no enough headroom or tailroom for AAD or digest, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE.
The UMR WQE and crypto WQE are handled in two different QPs.

The QP for UMR operation contains two types of WQE, UMR and SEND_EN WQE. The WQEs are built dynamically according to the crypto operation buffer address. Crypto operation with non-contiguous buffers will
have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP.
The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.

v2:
- split XTS and GCM code to different file.
- add headroom and tailroom optimize.

v3:
- fix AES-GCM 128b key creation.

Suanming Mou (9):
common/mlx5: export memory region lookup by address crypto/mlx5: split AES-XTS
crypto/mlx5: add AES-GCM query and initialization crypto/mlx5: add AES-GCM encryption key crypto/mlx5: add AES-GCM session configure common/mlx5: add WQE-based QP synchronous basics crypto/mlx5: add queue pair setup for GCM crypto/mlx5: add enqueue and dequeue operations crypto/mlx5: enable AES-GCM capability

 doc/guides/cryptodevs/mlx5.rst         |  48 +-
 doc/guides/rel_notes/release_23_07.rst |   1 +
 drivers/common/mlx5/mlx5_common_mr.c   |   2 +-
 drivers/common/mlx5/mlx5_common_mr.h   |   5 +
 drivers/common/mlx5/mlx5_devx_cmds.c   |  21 +
 drivers/common/mlx5/mlx5_devx_cmds.h   |  16 +
 drivers/common/mlx5/mlx5_prm.h         |  65 +-
 drivers/common/mlx5/version.map        |   3 +
 drivers/crypto/mlx5/meson.build        |   2 +
 drivers/crypto/mlx5/mlx5_crypto.c      | 673 ++---------------
 drivers/crypto/mlx5/mlx5_crypto.h      | 101 ++-
 drivers/crypto/mlx5/mlx5_crypto_dek.c  | 102 ++-
drivers/crypto/mlx5/mlx5_crypto_gcm.c  | 997 +++++++++++++++++++++++++
drivers/crypto/mlx5/mlx5_crypto_xts.c  | 645 ++++++++++++++++
 14 files changed, 2016 insertions(+), 665 deletions(-)  create mode 100644
drivers/crypto/mlx5/mlx5_crypto_gcm.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_xts.c

--
2.25.1

#39 lore raw

Hi Suanming,

Quoted 5 lines

Hi Akhil,

Maybe due to "To" is empty, it was not collected correctly to the ML. But it was in my inbox, and you were cced.

This is a v2 cover-letter as per the title.

#40 lore raw
Quoted 9 lines

-----Original Message-----
From: Akhil Goyal <[email protected]>
Sent: Tuesday, June 20, 2023 5:59 PM
To: Suanming Mou <[email protected]>
Cc: Raslan Darawsheh <[email protected]>; [email protected]
Subject: RE: [PATCH v2 0/9] crypto/mlx5: support AES-GCM

Quoted 7 lines

Hi Suanming,

Quoted 5 lines

Hi Akhil,

Maybe due to "To" is empty, it was not collected correctly to the ML. But it was in my inbox, and you were cced.

This is a v2 cover-letter as per the title.

Sorry, v2 is typo here, it is v3 in fact. So I understand why the cover-letter was "missing".

#41 lore raw

From: Suanming Mou

Quoted 10 lines
Quoted 10 lines

-----Original Message-----
From: Akhil Goyal <[email protected]>
Sent: Tuesday, June 20, 2023 5:59 PM
To: Suanming Mou <[email protected]>
Cc: Raslan Darawsheh <[email protected]>; [email protected]
Subject: RE: [PATCH v2 0/9] crypto/mlx5: support AES-GCM

Hi Suanming,

Quoted 2 lines

Hi Akhil,

Quoted 7 lines
Quoted 4 lines
Quoted 3 lines

Maybe due to "To" is empty, it was not collected correctly to the ML. But it was in my inbox, and you were cced.

This is a v2 cover-letter as per the title.

Sorry, v2 is typo here, it is v3 in fact. So I understand why the cover-letter was "missing".

For v3 series:

#42 lore raw

v4 cover crypto/mlx5: support AES-GCM

In reply to Suanming Mou

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous and there is no enough headroom or tailroom for AAD or digest, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE. The UMR WQE and crypto WQE are handled in two different QPs.

The QP for UMR operation contains two types of WQE, UMR and SEND_EN WQE. The WQEs are built dynamically according to the crypto operation buffer address. Crypto operation with non-contiguous buffers will have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP. The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.

v2:
- split XTS and GCM code to different file.
- add headroom and tailroom optimize.

v3:
- fix AES-GCM 128b key creation.

v4:
- add missing feature cap in mlx5.ini

Suanming Mou (9):
common/mlx5: export memory region lookup by address crypto/mlx5: split AES-XTS
crypto/mlx5: add AES-GCM query and initialization crypto/mlx5: add AES-GCM encryption key crypto/mlx5: add AES-GCM session configure common/mlx5: add WQE-based QP synchronous basics crypto/mlx5: add queue pair setup for GCM crypto/mlx5: add enqueue and dequeue operations crypto/mlx5: enable AES-GCM capability

 doc/guides/cryptodevs/features/mlx5.ini |   2 +
 doc/guides/cryptodevs/mlx5.rst          |  48 +-
 doc/guides/rel_notes/release_23_07.rst  |   1 +
 drivers/common/mlx5/mlx5_common_mr.c    |   2 +-
 drivers/common/mlx5/mlx5_common_mr.h    |   5 +
 drivers/common/mlx5/mlx5_devx_cmds.c    |  21 +
 drivers/common/mlx5/mlx5_devx_cmds.h    |  16 +
 drivers/common/mlx5/mlx5_prm.h          |  65 +-
 drivers/common/mlx5/version.map         |   3 +
 drivers/crypto/mlx5/meson.build         |   2 +
 drivers/crypto/mlx5/mlx5_crypto.c       | 673 ++--------------
 drivers/crypto/mlx5/mlx5_crypto.h       | 101 ++-
 drivers/crypto/mlx5/mlx5_crypto_dek.c   | 102 ++-
 drivers/crypto/mlx5/mlx5_crypto_gcm.c   | 997 ++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_xts.c   | 645 +++++++++++++++
 15 files changed, 2018 insertions(+), 665 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_gcm.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_xts.c
Signature · 1 line
2.25.1
#43 lore raw

PATCH v4 1/9 common/mlx5: export memory region lookup by address

In case user provides the address without mempool. Export the function to lookup the address without mempool is required.

---
 drivers/common/mlx5/mlx5_common_mr.c | 2 +-
 drivers/common/mlx5/mlx5_common_mr.h | 4 ++++
 drivers/common/mlx5/version.map      | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)
drivers/common/mlx5/mlx5_common_mr.c +1 -1
index 7b14b0c7bf..40ff9153bd 100644
@@ -1059,7 +1059,7 @@ mr_lookup_caches(struct mlx5_mr_ctrl *mr_ctrl,
  * @return
  *   Searched LKey on success, UINT32_MAX on no match.
  */
-static uint32_t
+uint32_t
 mlx5_mr_addr2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, uintptr_t addr)
 {
 	uint32_t lkey;
drivers/common/mlx5/mlx5_common_mr.h +4 -0
index 12def1585f..66623868a2 100644
@@ -240,6 +240,10 @@ mlx5_mr_create(struct mlx5_common_device *cdev,
 	       struct mlx5_mr_share_cache *share_cache,
 	       struct mr_cache_entry *entry, uintptr_t addr);
 
+__rte_internal
+uint32_t
+mlx5_mr_addr2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, uintptr_t addr);
+
 /* mlx5_common_verbs.c */
 
 __rte_internal
drivers/common/mlx5/version.map +1 -0
index e05e1aa8c5..f860b069de 100644
@@ -122,6 +122,7 @@ INTERNAL {
 	mlx5_mr_ctrl_init;
 	mlx5_mr_flush_local_cache;
 	mlx5_mr_mb2mr_bh;
+	mlx5_mr_addr2mr_bh;
 
 	mlx5_nl_allmulti; # WINDOWS_NO_EXPORT
 	mlx5_nl_ifindex; # WINDOWS_NO_EXPORT
Signature · 1 line
2.25.1
#44 lore raw

PATCH v4 2/9 crypto/mlx5: split AES-XTS

In reply to Suanming Mou

As there will be other crypto algo be supported. This commit splits AES-XTS code to another *_xts.c file. The mlx5_crypto.c file will just contain the common code.

---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     | 642 ++------------------------
 drivers/crypto/mlx5/mlx5_crypto.h     |  33 ++
 drivers/crypto/mlx5/mlx5_crypto_xts.c | 594 ++++++++++++++++++++++++
 4 files changed, 667 insertions(+), 603 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_xts.c
drivers/crypto/mlx5/meson.build +1 -0
index a2691ec0f0..045e8ce81d 100644
@@ -15,6 +15,7 @@ endif
 
 sources = files(
         'mlx5_crypto.c',
+	'mlx5_crypto_xts.c',
         'mlx5_crypto_dek.c',
 )
 
drivers/crypto/mlx5/mlx5_crypto.c +39 -603

709 lines not on this page · raw message

drivers/crypto/mlx5/mlx5_crypto.h +33 -0
index a2771b3dab..05d8fe97fe 100644
@@ -31,6 +31,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_uar uar; /* User Access Region. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	const struct rte_cryptodev_capabilities *caps;
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
@@ -70,6 +71,35 @@ struct mlx5_crypto_devarg_params {
 	uint32_t max_segs_num;
 };
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t bsp_res;
+	/**< crypto_block_size_pointer and reserved 24 bits saved in big
+	 * endian format.
+	 */
+	uint32_t iv_offset:16;
+	/**< Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
+	uint32_t dek_id; /**< DEK ID */
+} __rte_packed;
+
+typedef void *(*mlx5_crypto_mkey_update_t)(struct mlx5_crypto_priv *priv,
+					   struct mlx5_crypto_qp *qp,
+					   uint32_t idx);
+
+void
+mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp,
+				   uint16_t n);
+
+int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+				   struct mlx5_crypto_qp *qp,
+				   struct mlx5_devx_mkey_attr *attr,
+				   mlx5_crypto_mkey_update_t update_cb);
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
@@ -84,4 +114,7 @@ mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
 void
 mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
 
+int
+mlx5_crypto_xts_init(struct mlx5_crypto_priv *priv);
+
 #endif /* MLX5_CRYPTO_H_ */
drivers/crypto/mlx5/mlx5_crypto_xts.c new file +594 -0

594 lines not on this page · raw message

Signature · 1 line
2.25.1
#45 lore raw

PATCH v4 3/9 crypto/mlx5: add AES-GCM query and initialization

In reply to Suanming Mou

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

This commit adds the AES-GCM attributes query and initialization function.

---
 drivers/common/mlx5/mlx5_devx_cmds.c  | 15 +++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h  | 13 ++++++++++
 drivers/common/mlx5/mlx5_prm.h        | 19 +++++++++++---
 drivers/crypto/mlx5/meson.build       |  1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  4 ++-
 drivers/crypto/mlx5/mlx5_crypto.h     |  3 +++
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 36 +++++++++++++++++++++++++++
 7 files changed, 87 insertions(+), 4 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_gcm.c
drivers/common/mlx5/mlx5_devx_cmds.c +15 -0
index 1e418a0353..4332081165 100644
@@ -1117,6 +1117,21 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		attr->crypto_wrapped_import_method = !!(MLX5_GET(crypto_caps,
 						hcattr, wrapped_import_method)
 						& 1 << 2);
+		attr->crypto_mmo.crypto_mmo_qp = MLX5_GET(crypto_caps, hcattr, crypto_mmo_qp);
+		attr->crypto_mmo.gcm_256_encrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_256_encrypt);
+		attr->crypto_mmo.gcm_128_encrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_128_encrypt);
+		attr->crypto_mmo.gcm_256_decrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_256_decrypt);
+		attr->crypto_mmo.gcm_128_decrypt =
+			MLX5_GET(crypto_caps, hcattr, crypto_aes_gcm_128_decrypt);
+		attr->crypto_mmo.gcm_auth_tag_128 =
+			MLX5_GET(crypto_caps, hcattr, gcm_auth_tag_128);
+		attr->crypto_mmo.gcm_auth_tag_96 =
+			MLX5_GET(crypto_caps, hcattr, gcm_auth_tag_96);
+		attr->crypto_mmo.log_crypto_mmo_max_size =
+			MLX5_GET(crypto_caps, hcattr, log_crypto_mmo_max_size);
 	}
 	if (hca_cap_2_sup) {
 		hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
drivers/common/mlx5/mlx5_devx_cmds.h +13 -0
index dc3359268d..cb3f3a211b 100644
@@ -125,6 +125,18 @@ struct mlx5_hca_flex_attr {
 	uint8_t  header_length_mask_width;
 };
 
+__extension__
+struct mlx5_hca_crypto_mmo_attr {
+	uint32_t crypto_mmo_qp:1;
+	uint32_t gcm_256_encrypt:1;
+	uint32_t gcm_128_encrypt:1;
+	uint32_t gcm_256_decrypt:1;
+	uint32_t gcm_128_decrypt:1;
+	uint32_t gcm_auth_tag_128:1;
+	uint32_t gcm_auth_tag_96:1;
+	uint32_t log_crypto_mmo_max_size:6;
+};
+
 /* ISO C restricts enumerator values to range of 'int' */
 __extension__
 enum {
@@ -250,6 +262,7 @@ struct mlx5_hca_attr {
 	struct mlx5_hca_vdpa_attr vdpa;
 	struct mlx5_hca_flow_attr flow;
 	struct mlx5_hca_flex_attr flex;
+	struct mlx5_hca_crypto_mmo_attr crypto_mmo;
 	int log_max_qp_sz;
 	int log_max_cq_sz;
 	int log_max_qp;
drivers/common/mlx5/mlx5_prm.h +16 -3
index 3d5b6b9ba5..e41b5b3528 100644
@@ -4582,7 +4582,9 @@ struct mlx5_ifc_crypto_caps_bits {
 	u8 synchronize_dek[0x1];
 	u8 int_kek_manual[0x1];
 	u8 int_kek_auto[0x1];
-	u8 reserved_at_6[0x12];
+	u8 reserved_at_6[0xd];
+	u8 sw_wrapped_dek_key_purpose[0x1];
+	u8 reserved_at_14[0x4];
 	u8 wrapped_import_method[0x8];
 	u8 reserved_at_20[0x3];
 	u8 log_dek_max_alloc[0x5];
@@ -4599,8 +4601,19 @@ struct mlx5_ifc_crypto_caps_bits {
 	u8 log_dek_granularity[0x5];
 	u8 reserved_at_68[0x3];
 	u8 log_max_num_int_kek[0x5];
-	u8 reserved_at_70[0x10];
-	u8 reserved_at_80[0x780];
+	u8 sw_wrapped_dek_new[0x10];
+	u8 reserved_at_80[0x80];
+	u8 crypto_mmo_qp[0x1];
+	u8 crypto_aes_gcm_256_encrypt[0x1];
+	u8 crypto_aes_gcm_128_encrypt[0x1];
+	u8 crypto_aes_gcm_256_decrypt[0x1];
+	u8 crypto_aes_gcm_128_decrypt[0x1];
+	u8 gcm_auth_tag_128[0x1];
+	u8 gcm_auth_tag_96[0x1];
+	u8 reserved_at_107[0x3];
+	u8 log_crypto_mmo_max_size[0x6];
+	u8 reserved_at_110[0x10];
+	u8 reserved_at_120[0x6e0];
 };
 
 struct mlx5_ifc_crypto_commissioning_register_bits {
drivers/crypto/mlx5/meson.build +1 -0
index 045e8ce81d..17ffce89f0 100644
@@ -16,6 +16,7 @@ endif
 sources = files(
         'mlx5_crypto.c',
 	'mlx5_crypto_xts.c',
+	'mlx5_crypto_gcm.c',
         'mlx5_crypto_dek.c',
 )
 
drivers/crypto/mlx5/mlx5_crypto.c +3 -1
index 2e6bcc6ddc..ff632cd69a 100644
@@ -335,7 +335,9 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
+	if (!cdev->config.hca_attr.crypto ||
+	   (!cdev->config.hca_attr.aes_xts &&
+	    !cdev->config.hca_attr.crypto_mmo.crypto_mmo_qp)) {
 		DRV_LOG(ERR, "Not enough capabilities to support crypto "
 			"operations, maybe old FW/OFED version?");
 		rte_errno = ENOTSUP;
drivers/crypto/mlx5/mlx5_crypto.h +3 -0
index 05d8fe97fe..76f368ee91 100644
@@ -117,4 +117,7 @@ mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
 int
 mlx5_crypto_xts_init(struct mlx5_crypto_priv *priv);
 
+int
+mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv);
+
 #endif /* MLX5_CRYPTO_H_ */
drivers/crypto/mlx5/mlx5_crypto_gcm.c new file +36 -0
new file mode 100644
index 0000000000..bd78c6d66b
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_eal_paging.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+#include <bus_pci_driver.h>
+#include <rte_memory.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+static struct rte_cryptodev_capabilities mlx5_crypto_gcm_caps[] = {
+	{
+		.op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
+	},
+	{
+		.op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
+	}
+};
+
+int
+mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv)
+{
+	priv->caps = mlx5_crypto_gcm_caps;
+	return 0;
+}
+
Signature · 1 line
2.25.1
#46 lore raw

PATCH v4 4/9 crypto/mlx5: add AES-GCM encryption key

In reply to Suanming Mou

The crypto device requires the DEK(data encryption key) object for data encryption/decryption operation.

This commit adds the AES-GCM DEK object management support.

---
 drivers/crypto/mlx5/mlx5_crypto.h     |  17 ++++-
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 102 +++++++++++++-------------
 drivers/crypto/mlx5/mlx5_crypto_gcm.c |  33 +++++++++
 drivers/crypto/mlx5/mlx5_crypto_xts.c |  53 ++++++++++++-
 4 files changed, 150 insertions(+), 55 deletions(-)
drivers/crypto/mlx5/mlx5_crypto.h +16 -1
index 76f368ee91..bb5a557a38 100644
@@ -86,6 +86,11 @@ struct mlx5_crypto_session {
 	uint32_t dek_id; /**< DEK ID */
 } __rte_packed;
 
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_sym_xform *xform;
+	struct mlx5_crypto_priv *priv;
+};
+
 typedef void *(*mlx5_crypto_mkey_update_t)(struct mlx5_crypto_priv *priv,
 					   struct mlx5_crypto_qp *qp,
 					   uint32_t idx);
@@ -106,7 +111,7 @@ mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 
 struct mlx5_crypto_dek *
 mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
-			struct rte_crypto_cipher_xform *cipher);
+			struct rte_crypto_sym_xform *xform);
 
 int
 mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
@@ -120,4 +125,14 @@ mlx5_crypto_xts_init(struct mlx5_crypto_priv *priv);
 int
 mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv);
 
+int
+mlx5_crypto_dek_fill_xts_attr(struct mlx5_crypto_dek *dek,
+			      struct mlx5_devx_dek_attr *dek_attr,
+			      void *cb_ctx);
+
+int
+mlx5_crypto_dek_fill_gcm_attr(struct mlx5_crypto_dek *dek,
+			      struct mlx5_devx_dek_attr *dek_attr,
+			      void *cb_ctx);
+
 #endif /* MLX5_CRYPTO_H_ */
drivers/crypto/mlx5/mlx5_crypto_dek.c +49 -53

154 lines not on this page · raw message

drivers/crypto/mlx5/mlx5_crypto_gcm.c +33 -0
index bd78c6d66b..5b315ef42c 100644
@@ -27,6 +27,39 @@ static struct rte_cryptodev_capabilities mlx5_crypto_gcm_caps[] = {
 	}
 };
 
+int
+mlx5_crypto_dek_fill_gcm_attr(struct mlx5_crypto_dek *dek,
+			      struct mlx5_devx_dek_attr *dek_attr,
+			      void *cb_ctx)
+{
+	uint32_t offset = 0;
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_aead_xform *aead_ctx = &ctx->xform->aead;
+
+	if (aead_ctx->algo != RTE_CRYPTO_AEAD_AES_GCM) {
+		DRV_LOG(ERR, "Only AES-GCM algo supported.");
+		return -EINVAL;
+	}
+	dek_attr->key_purpose = MLX5_CRYPTO_KEY_PURPOSE_GCM;
+	switch (aead_ctx->key.length) {
+	case 16:
+		offset = 16;
+		dek->size = 16;
+		dek_attr->key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 32:
+		dek->size = 32;
+		dek_attr->key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Wrapped key size not supported.");
+		return -EINVAL;
+	}
+	memcpy(&dek_attr->key[offset], aead_ctx->key.data, aead_ctx->key.length);
+	memcpy(&dek->data, aead_ctx->key.data, aead_ctx->key.length);
+	return 0;
+}
+
 int
 mlx5_crypto_gcm_init(struct mlx5_crypto_priv *priv)
 {
drivers/crypto/mlx5/mlx5_crypto_xts.c +52 -1

65 lines not on this page · raw message

Signature · 1 line
2.25.1
#47 lore raw

PATCH v4 5/9 crypto/mlx5: add AES-GCM session configure

In reply to Suanming Mou

Sessions are used in symmetric transformations in order to prepare objects and data for packet processing stage.

The AES-GCM session includes IV, AAD, digest(tag), DEK, operation mode information.

---
 drivers/common/mlx5/mlx5_prm.h        | 12 +++++++
 drivers/crypto/mlx5/mlx5_crypto.h     | 40 ++++++++++++++++++-----
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 47 +++++++++++++++++++++++++++
 3 files changed, 91 insertions(+), 8 deletions(-)
drivers/common/mlx5/mlx5_prm.h +12 -0
index e41b5b3528..96ce342d29 100644
@@ -523,11 +523,23 @@ enum {
 	MLX5_BLOCK_SIZE_4048B	= 0x6,
 };
 
+enum {
+	MLX5_ENCRYPTION_TYPE_AES_GCM = 0x3,
+};
+
+enum {
+	MLX5_CRYPTO_OP_TYPE_ENCRYPTION = 0x0,
+	MLX5_CRYPTO_OP_TYPE_DECRYPTION = 0x1,
+};
+
 #define MLX5_BSF_SIZE_OFFSET		30
 #define MLX5_BSF_P_TYPE_OFFSET		24
 #define MLX5_ENCRYPTION_ORDER_OFFSET	16
 #define MLX5_BLOCK_SIZE_OFFSET		24
 
+#define MLX5_CRYPTO_MMO_TYPE_OFFSET 24
+#define MLX5_CRYPTO_MMO_OP_OFFSET 20
+
 struct mlx5_wqe_umr_bsf_seg {
 	/*
 	 * bs_bpt_eo_es contains:
drivers/crypto/mlx5/mlx5_crypto.h +32 -8
index bb5a557a38..6cb4d4ddec 100644
@@ -72,16 +72,40 @@ struct mlx5_crypto_devarg_params {
 };
 
 struct mlx5_crypto_session {
-	uint32_t bs_bpt_eo_es;
-	/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
-	 * saved in big endian format.
-	 */
-	uint32_t bsp_res;
-	/**< crypto_block_size_pointer and reserved 24 bits saved in big
-	 * endian format.
-	 */
+	union {
+		/**< AES-XTS configuration. */
+		struct {
+			uint32_t bs_bpt_eo_es;
+			/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
+			 * saved in big endian format.
+			 */
+			uint32_t bsp_res;
+			/**< crypto_block_size_pointer and reserved 24 bits saved in big
+			 * endian format.
+			 */
+		};
+		/**< AES-GCM configuration. */
+		struct {
+			uint32_t mmo_ctrl;
+			/**< Crypto control fields with algo type and op type in big
+			 * endian format.
+			 */
+			uint32_t wqe_aad_len;
+			/**< Crypto AAD length field in big endian format. */
+			uint32_t wqe_tag_len;
+			/**< Crypto tag length field in big endian format. */
+			uint16_t tag_len;
+			/**< AES-GCM crypto digest size in bytes. */
+			uint16_t aad_len;
+			/**< The length of the additional authenticated data (AAD) in bytes. */
+			uint32_t op_type;
+			/**< Operation type. */
+		};
+	};
 	uint32_t iv_offset:16;
 	/**< Starting point for Initialisation Vector. */
+	uint32_t iv_len;
+	/**< Initialisation Vector length. */
 	struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
 	uint32_t dek_id; /**< DEK ID */
 } __rte_packed;
drivers/crypto/mlx5/mlx5_crypto_gcm.c +47 -0

56 lines not on this page · raw message

Signature · 1 line
2.25.1
#48 lore raw

PATCH v4 6/9 common/mlx5: add WQE-based QP synchronous basics

In reply to Suanming Mou

Nvidia HW provides a synchronous mechanism between QPs. When creating the QPs, user can set one as primary and another as follower. The follower QP's WQE execution can be controlled by primary QP via SEND_EN WQE.

This commit introduces the SEND_EN WQE to improve the WQE execution sync-up between primary and follower QPs.

---
 drivers/common/mlx5/mlx5_devx_cmds.c |  6 ++++++
 drivers/common/mlx5/mlx5_devx_cmds.h |  3 +++
 drivers/common/mlx5/mlx5_prm.h       | 11 +++++++++++
 3 files changed, 20 insertions(+)
drivers/common/mlx5/mlx5_devx_cmds.c +6 -0
index 4332081165..ef87862a6d 100644
@@ -2475,6 +2475,12 @@ mlx5_devx_cmd_create_qp(void *ctx,
 				 attr->dbr_umem_valid);
 			MLX5_SET(qpc, qpc, dbr_umem_id, attr->dbr_umem_id);
 		}
+		if (attr->cd_master)
+			MLX5_SET(qpc, qpc, cd_master, attr->cd_master);
+		if (attr->cd_slave_send)
+			MLX5_SET(qpc, qpc, cd_slave_send, attr->cd_slave_send);
+		if (attr->cd_slave_recv)
+			MLX5_SET(qpc, qpc, cd_slave_receive, attr->cd_slave_recv);
 		MLX5_SET64(qpc, qpc, dbr_addr, attr->dbr_address);
 		MLX5_SET64(create_qp_in, in, wq_umem_offset,
 			   attr->wq_umem_offset);
drivers/common/mlx5/mlx5_devx_cmds.h +3 -0
index cb3f3a211b..e071cd841f 100644
@@ -559,6 +559,9 @@ struct mlx5_devx_qp_attr {
 	uint64_t wq_umem_offset;
 	uint32_t user_index:24;
 	uint32_t mmo:1;
+	uint32_t cd_master:1;
+	uint32_t cd_slave_send:1;
+	uint32_t cd_slave_recv:1;
 };
 
 struct mlx5_devx_virtio_q_couners_attr {
drivers/common/mlx5/mlx5_prm.h +11 -0
index 96ce342d29..4990bcaacd 100644
@@ -589,6 +589,17 @@ struct mlx5_rdma_write_wqe {
 	struct mlx5_wqe_dseg dseg[];
 } __rte_packed;
 
+struct mlx5_wqe_send_en_seg {
+	uint32_t reserve[2];
+	uint32_t sqnpc;
+	uint32_t qpn;
+} __rte_packed;
+
+struct mlx5_wqe_send_en_wqe {
+	struct mlx5_wqe_cseg ctr;
+	struct mlx5_wqe_send_en_seg sseg;
+} __rte_packed;
+
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
Signature · 1 line
2.25.1
#49 lore raw

PATCH v4 7/9 crypto/mlx5: add queue pair setup for GCM

In reply to Suanming Mou

Crypto queue pair is for handling the encryption/decryption operations.

As AES-GCM AEAD API provides AAD, mbuf, digest separately, low-level FW only accepts the data in a single contiguous memory region, two internal QPs are created for AES-GCM queue pair. One for organizing the memory to be contiguous if they are not. The other is for crypto.

If the buffers are checked as implicitly contiguous, the buffer will be sent to the crypto QP directly for encryption/decryption. If not, the buffers will be handled by the first UMR QP. The UMR QP will convert the buffers to be contiguous one. Then the well organized "new" buffer can be handled by crypto QP.

The crypto QP is initialized as follower, and UMR as leader. Once crypto operation input buffer requires memory address space converting by UMR QP, the crypto QP processing will be triggered by UMR QP. Otherwise, the ring crypto QP doorbell directly.

The existing max_segs_num devarg is used for define how many segments the chained mbuf contains same as AES-XTS before.

---
 drivers/common/mlx5/mlx5_common_mr.h  |   1 +
 drivers/common/mlx5/mlx5_prm.h        |  22 +++
 drivers/common/mlx5/version.map       |   2 +
 drivers/crypto/mlx5/mlx5_crypto.h     |  15 ++
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 230 ++++++++++++++++++++++++++
 5 files changed, 270 insertions(+)
drivers/common/mlx5/mlx5_common_mr.h +1 -0
index 66623868a2..8789d403b1 100644
@@ -254,6 +254,7 @@ __rte_internal
 void
 mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
 
+__rte_internal
 void
 mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
 
drivers/common/mlx5/mlx5_prm.h +22 -0
index 4990bcaacd..4f6925733a 100644
@@ -470,6 +470,15 @@ struct mlx5_wqe_rseg {
 #define MLX5_UMRC_KO_OFFSET 16u
 #define MLX5_UMRC_TO_BS_OFFSET 0u
 
+/*
+ * As PRM describes, the address of the UMR pointer must be
+ * aligned to 2KB.
+ */
+#define MLX5_UMR_KLM_PTR_ALIGN (1 << 11)
+
+#define MLX5_UMR_KLM_NUM_ALIGN \
+	(MLX5_UMR_KLM_PTR_ALIGN / sizeof(struct mlx5_klm))
+
 struct mlx5_wqe_umr_cseg {
 	uint32_t if_cf_toe_cq_res;
 	uint32_t ko_to_bs;
@@ -674,6 +683,19 @@ union mlx5_gga_compress_opaque {
 	uint32_t data[64];
 };
 
+union mlx5_gga_crypto_opaque {
+	struct {
+		uint32_t syndrome;
+		uint32_t reserved0[2];
+		struct {
+			uint32_t iv[3];
+			uint32_t tag_size;
+			uint32_t aad_size;
+		} cp __rte_packed;
+	} __rte_packed;
+	uint8_t data[64];
+};
+
 struct mlx5_ifc_regexp_mmo_control_bits {
 	uint8_t reserved_at_31[0x2];
 	uint8_t le[0x1];
drivers/common/mlx5/version.map +2 -0
index f860b069de..0758ba76de 100644
@@ -159,5 +159,7 @@ INTERNAL {
 
 	mlx5_os_interrupt_handler_create; # WINDOWS_NO_EXPORT
 	mlx5_os_interrupt_handler_destroy; # WINDOWS_NO_EXPORT
+
+	mlx5_os_set_reg_mr_cb;
 	local: *;
 };
drivers/crypto/mlx5/mlx5_crypto.h +15 -0
index 6cb4d4ddec..88a09a6b1c 100644
@@ -28,8 +28,11 @@ struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
 	struct mlx5_common_device *cdev; /* Backend mlx5 device. */
 	struct rte_cryptodev *crypto_dev;
+	mlx5_reg_mr_t reg_mr_cb; /* Callback to reg_mr func */
+	mlx5_dereg_mr_t dereg_mr_cb; /* Callback to dereg_mr func */
 	struct mlx5_uar uar; /* User Access Region. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
+	uint32_t max_klm_num; /* Maximum supported klm. */
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	const struct rte_cryptodev_capabilities *caps;
 	struct rte_cryptodev_config dev_config;
@@ -46,15 +49,27 @@ struct mlx5_crypto_qp {
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_qp qp_obj;
+	struct mlx5_devx_qp umr_qp_obj;
 	struct rte_cryptodev_stats stats;
 	struct rte_crypto_op **ops;
 	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
+	struct mlx5_klm *klm_array;
+	union mlx5_gga_crypto_opaque *opaque_addr;
 	struct mlx5_mr_ctrl mr_ctrl;
+	struct mlx5_pmd_mr mr;
+	/* Crypto QP. */
 	uint8_t *wqe;
 	uint16_t entries_n;
+	uint16_t cq_entries_n;
 	uint16_t pi;
 	uint16_t ci;
 	uint16_t db_pi;
+	/* UMR QP. */
+	uint8_t *umr_wqe;
+	uint16_t umr_wqbbs;
+	uint16_t umr_pi;
+	uint16_t umr_ci;
+	uint32_t umr_errors;
 };
 
 struct mlx5_crypto_dek {
drivers/crypto/mlx5/mlx5_crypto_gcm.c +230 -0

254 lines not on this page · raw message

Signature · 1 line
2.25.1
#50 lore raw

PATCH v4 8/9 crypto/mlx5: add enqueue and dequeue operations

In reply to Suanming Mou

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous and there is no enough headroom/tailroom for copying AAD/digest, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE. The UMR WQE and crypto WQE are handled in two different QPs.

Crypto operation with non-contiguous buffers will have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, an additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP. The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.

---
 drivers/common/mlx5/mlx5_prm.h        |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |   9 +-
 drivers/crypto/mlx5/mlx5_crypto.h     |   8 +
 drivers/crypto/mlx5/mlx5_crypto_gcm.c | 588 ++++++++++++++++++++++++++
 4 files changed, 604 insertions(+), 2 deletions(-)
drivers/common/mlx5/mlx5_prm.h +1 -0
index 4f6925733a..d33d05238c 100644
@@ -617,6 +617,7 @@ struct mlx5_wqe_send_en_wqe {
 /* MMO metadata segment */
 
 #define	MLX5_OPCODE_MMO	0x2fu
+#define	MLX5_OPC_MOD_MMO_CRYPTO 0x6u
 #define	MLX5_OPC_MOD_MMO_REGEX 0x4u
 #define	MLX5_OPC_MOD_MMO_COMP 0x2u
 #define	MLX5_OPC_MOD_MMO_DECOMP 0x3u
drivers/crypto/mlx5/mlx5_crypto.c +7 -2
index ff632cd69a..4d7d3ef2a3 100644
@@ -62,8 +62,13 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 			MLX5_CRYPTO_FEATURE_FLAGS(priv->is_wrapped_mode);
 		dev_info->capabilities = priv->caps;
 		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
-		dev_info->min_mbuf_headroom_req = 0;
-		dev_info->min_mbuf_tailroom_req = 0;
+		if (priv->caps->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) {
+			dev_info->min_mbuf_headroom_req = MLX5_CRYPTO_GCM_MAX_AAD;
+			dev_info->min_mbuf_tailroom_req = MLX5_CRYPTO_GCM_MAX_DIGEST;
+		} else {
+			dev_info->min_mbuf_headroom_req = 0;
+			dev_info->min_mbuf_tailroom_req = 0;
+		}
 		dev_info->sym.max_nb_sessions = 0;
 		/*
 		 * If 0, the device does not have any limitation in number of
drivers/crypto/mlx5/mlx5_crypto.h +8 -0
index 88a09a6b1c..6dcb41b27c 100644
@@ -23,6 +23,8 @@
 #define MLX5_CRYPTO_KLM_SEGS_NUM(umr_wqe_sz) ((umr_wqe_sz -\
 					MLX5_CRYPTO_UMR_WQE_STATIC_SIZE) /\
 					MLX5_WSEG_SIZE)
+#define MLX5_CRYPTO_GCM_MAX_AAD 64
+#define MLX5_CRYPTO_GCM_MAX_DIGEST 16
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -61,6 +63,9 @@ struct mlx5_crypto_qp {
 	uint8_t *wqe;
 	uint16_t entries_n;
 	uint16_t cq_entries_n;
+	uint16_t reported_ci;
+	uint16_t qp_ci;
+	uint16_t cq_ci;
 	uint16_t pi;
 	uint16_t ci;
 	uint16_t db_pi;
@@ -70,6 +75,9 @@ struct mlx5_crypto_qp {
 	uint16_t umr_pi;
 	uint16_t umr_ci;
 	uint32_t umr_errors;
+	uint16_t last_gga_pi;
+	bool has_umr;
+	uint16_t cpy_tag_op;
 };
 
 struct mlx5_crypto_dek {
drivers/crypto/mlx5/mlx5_crypto_gcm.c +588 -0

612 lines not on this page · raw message

Signature · 1 line
2.25.1
#51 lore raw

PATCH v4 9/9 crypto/mlx5: enable AES-GCM capability

In reply to Suanming Mou

This commit generates AES-GCM capability based on the NIC attributes and enables AES-GCM algo.

An new devarg "algo" is added to identify if the crypto PMD will be initialized as AES-GCM(algo=1) or AES-XTS(algo=0, default).

---
 doc/guides/cryptodevs/features/mlx5.ini |  2 +
 doc/guides/cryptodevs/mlx5.rst          | 48 ++++++++++++++++++-
 doc/guides/rel_notes/release_23_07.rst  |  1 +
 drivers/crypto/mlx5/mlx5_crypto.c       | 26 ++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h       |  1 +
 drivers/crypto/mlx5/mlx5_crypto_gcm.c   | 63 +++++++++++++++++++++++++
 6 files changed, 136 insertions(+), 5 deletions(-)
doc/guides/cryptodevs/features/mlx5.ini +2 -0
index 0d210b2114..9bf1defac8 100644
@@ -30,6 +30,8 @@ AES XTS (256)  = Y
 ; Supported AEAD algorithms of a mlx5 crypto driver.
 ;
 [AEAD]
+AES GCM (128)  = Y
+AES GCM (256)  = Y
 
 ;
 ; Supported Asymmetric algorithms of a mlx5 crypto driver.
doc/guides/cryptodevs/mlx5.rst +47 -1

105 lines not on this page · raw message

doc/guides/rel_notes/release_23_07.rst +1 -0
index 027ae7bd2d..bbb8eddbca 100644
@@ -131,6 +131,7 @@ New Features
   * Added support for CQE compression on Windows.
   * Added support for enhanced multi-packet write on Windows.
   * Added support for quota flow action and item.
+  * Added support for AES-GCM crypto.
 
 * **Added vmxnet3 version 7 support.**
 
drivers/crypto/mlx5/mlx5_crypto.c +22 -4
index 4d7d3ef2a3..081e96ad4d 100644
@@ -269,6 +269,14 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->credential_pointer = (uint32_t)tmp;
 	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
+	} else if (strcmp(key, "algo") == 0) {
+		if (tmp == 1) {
+			devarg_prms->is_aes_gcm = 1;
+		} else if (tmp > 1) {
+			DRV_LOG(ERR, "Invalid algo.");
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
 	}
 	return 0;
 }
@@ -285,6 +293,7 @@ mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl *mkvlist,
 		"keytag",
 		"max_segs_num",
 		"wcs_file",
+		"algo",
 		NULL,
 	};
 
@@ -370,10 +379,19 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
 	priv->crypto_dev = crypto_dev;
 	priv->is_wrapped_mode = wrapped_mode;
 	priv->max_segs_num = devarg_prms.max_segs_num;
-	ret = mlx5_crypto_xts_init(priv);
-	if (ret) {
-		DRV_LOG(ERR, "Failed to init AES-XTS crypto.");
-		return -ENOTSUP;
+	/* Init and override AES-GCM configuration. */
+	if (devarg_prms.is_aes_gcm) {
+		ret = mlx5_crypto_gcm_init(priv);
+		if (ret) {
+			DRV_LOG(ERR, "Failed to init AES-GCM crypto.");
+			return -ENOTSUP;
+		}
+	} else {
+		ret = mlx5_crypto_xts_init(priv);
+		if (ret) {
+			DRV_LOG(ERR, "Failed to init AES-XTS crypto.");
+			return -ENOTSUP;
+		}
 	}
 	if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) {
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
drivers/crypto/mlx5/mlx5_crypto.h +1 -0
index 6dcb41b27c..36dacdcda4 100644
@@ -92,6 +92,7 @@ struct mlx5_crypto_devarg_params {
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
 	uint32_t max_segs_num;
+	uint32_t is_aes_gcm:1;
 };
 
 struct mlx5_crypto_session {
drivers/crypto/mlx5/mlx5_crypto_gcm.c +63 -0

83 lines not on this page · raw message

Signature · 1 line
2.25.1
#52 lore raw
In reply to Suanming Mou
Quoted 28 lines

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

The crypto operations are performed with crypto WQE. If the input buffers(AAD, mbuf, digest) are not contiguous and there is no enough headroom or tailroom for AAD or digest, as the requirement from FW, an UMR WQE is needed to generate contiguous address space for crypto WQE. The UMR WQE and crypto WQE are handled in two different QPs.

The QP for UMR operation contains two types of WQE, UMR and SEND_EN WQE. The WQEs are built dynamically according to the crypto operation buffer address. Crypto operation with non-contiguous buffers will have its own UMR WQE, while the operation with contiguous buffers doesn't need the UMR WQE. Once the all the operations WQE in the enqueue burst built finishes, if any UMR WQEs are built, additional SEND_EN WQE will be as the final WQE of the burst in the UMR QP. The purpose of that SEND_EN WQE is to trigger the crypto QP processing with the UMR ready input memory address space buffers.

The QP for crypto operations contains only the crypto WQE and the QP WQEs are built as fixed in QP setup. The QP processing is triggered by doorbell ring or the SEND_EN WQE from UMR QP.

v2:
- split XTS and GCM code to different file.
- add headroom and tailroom optimize.

Quoted 5 lines

v3:
- fix AES-GCM 128b key creation.

v4:
- add missing feature cap in mlx5.ini

Series applied to dpdk-next-crypto
Thanks.

#53 lore raw

20/06/2023 20:49, Akhil Goyal:

Quoted 3 lines
Quoted 3 lines

AES-GCM provides both authenticated encryption and the ability to check the integrity and authentication of additional authenticated data (AAD) that is sent in the clear.

[...]

Quoted 1 line

Series applied to dpdk-next-crypto

This has to be added to avoid a compilation failure with MinGW:

@@ -25,6 +25,8 @@ mlx5_crypto_dek_get_key(struct rte_crypto_sym_xform *xform,

                *key = xform->aead.key.data;
                *key_len = xform->aead.key.length;
        } else {
+               *key = NULL;
+               *key_len = 0;
                DRV_LOG(ERR, "Xform dek type not supported.");
                rte_errno = -EINVAL;
                return -1;

I will squash it where appropriate.