drivers: update relaxed ordering policy for mlx5 mkeys
Subsystems (3)
PATCH 1/1 drivers: update relaxed ordering policy for mlx5 mkeys
New adapters expose additional ordering capabilities.
Query the new caps and apply them when creating DevX mkeys via mlx5_devx_mkey_attr_set_ordering(), which sets PCI relaxed ordering and RAW=RO when relaxed order is supported.
Use this helper on Windows (still gated by Haswell/Broadwell) and for Linux wrapped mkeys and crypto/regex/vdpa indirect mkeys when relaxed order only flag is set.
Linux wrapped mkeys continue to use the legacy Haswell/Broadwell rule for IBV_ACCESS_RELAXED_ORDERING on the verbs MR.
Upcoming FW will requires setting the correct ordering attributes, otherwise it fails to create the memory key.
--- drivers/common/mlx5/linux/mlx5_common_os.c | 6 ++++ drivers/common/mlx5/mlx5_devx_cmds.c | 31 ++++++++++++++++++++ drivers/common/mlx5/mlx5_devx_cmds.h | 9 ++++++ drivers/common/mlx5/mlx5_prm.h | 18 ++++++++++-- drivers/common/mlx5/windows/mlx5_common_os.c | 8 ++--- drivers/crypto/mlx5/mlx5_crypto.c | 4 +++ drivers/regex/mlx5/mlx5_regex_fastpath.c | 5 ++++ drivers/regex/mlx5/mlx5_rxp.c | 4 +++ drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 4 +++ 9 files changed, 81 insertions(+), 8 deletions(-)
drivers/common/mlx5/linux/mlx5_common_os.c +6 -0
@@ -997,6 +997,7 @@ int mlx5_os_wrapped_mkey_create(void *ctx, void *pd, uint32_t pdn, void *addr, size_t length, struct mlx5_pmd_wrapped_mr *pmd_mr) { +struct mlx5_hca_attr hca_attr = {0 };struct mlx5_klm klm = { .byte_count = length, .address = (uintptr_t )addr,
@@ -1019,6 +1020,11 @@ mlx5_os_wrapped_mkey_create(void *ctx, void *pd, uint32_t pdn, void *addr, klm.mkey = ibv_mr->lkey; mkey_attr.addr = (uintptr_t )addr; mkey_attr.size = length; +if (mlx5_devx_cmd_query_hca_attr (ctx, &hca_attr)) +return -1 ; +/* If only relaxed order is allowed. */ +if (hca_attr.mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, &hca_attr); mkey =mlx5_devx_cmd_mkey_create (ctx, &mkey_attr);if (!mkey) {claim_zero (mlx5_glue->dereg_mr (ibv_mr));
drivers/common/mlx5/mlx5_devx_cmds.c +31 -0
@@ -331,6 +331,29 @@ mlx5_devx_cmd_flow_counter_query(struct mlx5_devx_obj *dcs,return 0 ; } +/** +* Apply PCI relaxed-ordering and read-after-write ordering to mkey attributes. +* +* @param[in, out] mkey_attr +* Mkey attributes to update. +* @param[in] hca_attr +* HCA capabilities from mlx5_devx_cmd_query_hca_attr(). +*/ +RTE_EXPORT_INTERNAL_SYMBOL (mlx5_devx_mkey_attr_set_ordering) +void +mlx5_devx_mkey_attr_set_ordering (struct mlx5_devx_mkey_attr *mkey_attr, +const struct mlx5_hca_attr *hca_attr) +{ +if (!mkey_attr || !hca_attr) +return ; + + mkey_attr->relaxed_ordering_write = hca_attr->relaxed_ordering_write; + mkey_attr->relaxed_ordering_read = + hca_attr->relaxed_ordering_read || hca_attr->pci_relaxed_ordered_read; +if (hca_attr->mkc_order_read_after_write) + mkey_attr->read_after_write_ordering =MLX5_MKC_RAW_ORDERING_RO ; +} +/** * Create a new mkey. *
@@ -417,6 +440,8 @@ mlx5_devx_cmd_mkey_create(void *ctx,MLX5_SET (mkc, mkc, relaxed_ordering_write, attr->relaxed_ordering_write);MLX5_SET (mkc, mkc, relaxed_ordering_read, attr->relaxed_ordering_read); +MLX5_SET (mkc, mkc, order_read_after_write, + attr->read_after_write_ordering);MLX5_SET64 (mkc, mkc, start_addr, attr->addr);MLX5_SET64 (mkc, mkc, len, attr->size);MLX5_SET (mkc, mkc, crypto_en, attr->crypto_en);
@@ -1003,6 +1028,12 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, relaxed_ordering_write); attr->relaxed_ordering_read =MLX5_GET (cmd_hca_cap, hcattr, relaxed_ordering_read); + attr->pci_relaxed_ordered_read =MLX5_GET (cmd_hca_cap, hcattr, + pci_relaxed_ordered_read); + attr->mkc_order_read_after_write =MLX5_GET (cmd_hca_cap, hcattr, + mkc_order_read_after_write); + attr->mkc_order_write_after_write_ro_only =MLX5_GET (cmd_hca_cap, hcattr, + mkc_order_write_after_write_ro_only); attr->access_register_user =MLX5_GET (cmd_hca_cap, hcattr, access_register_user); attr->eth_net_offloads = MLX5_GET(cmd_hca_cap, hcattr,
drivers/common/mlx5/mlx5_devx_cmds.h +9 -0
@@ -34,6 +34,7 @@ struct mlx5_devx_mkey_attr { uint32_t pg_access:1; uint32_t relaxed_ordering_write:1; uint32_t relaxed_ordering_read:1; + uint32_t read_after_write_ordering:2; uint32_t umr_en:1; uint32_t crypto_en:2; uint32_t set_remote_rw:1;
@@ -237,6 +238,9 @@ struct mlx5_hca_attr { uint32_t vhca_id:16; uint32_t relaxed_ordering_write:1; uint32_t relaxed_ordering_read:1; + uint32_t pci_relaxed_ordered_read:1; + uint32_t mkc_order_read_after_write:1; + uint32_t mkc_order_write_after_write_ro_only:1; uint32_t access_register_user:1; uint32_t wqe_index_ignore:1; uint32_t cross_channel:1;
@@ -748,6 +752,11 @@ int mlx5_devx_cmd_query_hca_attr(void *ctx,__rte_internal struct mlx5_devx_obj *mlx5_devx_cmd_mkey_create(void *ctx,struct mlx5_devx_mkey_attr *attr); + +__rte_internal +void +mlx5_devx_mkey_attr_set_ordering (struct mlx5_devx_mkey_attr *mkey_attr, +const struct mlx5_hca_attr *hca_attr);__rte_internal int mlx5_devx_get_out_command_status(void *out); __rte_internal
drivers/common/mlx5/mlx5_prm.h +15 -3
@@ -1463,7 +1463,9 @@ struct mlx5_ifc_mkc_bits {u8 bsf_octword_size[0x20 ];u8 reserved_at_120[0x80 ];u8 translations_octword_size[0x20 ]; -u8 reserved_at_1c0[0x19 ]; +u8 reserved_at_1c0[0x16 ]; +u8 order_read_after_write[0x2 ]; +u8 reserved_at_1d8[0x1 ];u8 relaxed_ordering_read[0x1 ];u8 reserved_at_1da[0x1 ];u8 log_page_size[0x5 ];
@@ -1478,6 +1480,13 @@ enum { MLX5_MKEY_CRYPTO_ENABLED = 0x1, }; +/* MKC read_after_write_ordering field (2-bit, dword 0x38 bits 9:8). */ +enum mlx5_mkc_raw_ordering { +MLX5_MKC_RAW_ORDERING_SO =0x0 , +MLX5_MKC_RAW_ORDERING_SAO =0x1 , +MLX5_MKC_RAW_ORDERING_RO =0x2 , +}; +struct mlx5_ifc_create_mkey_out_bits {u8 status[0x8 ];u8 reserved_at_8[0x18 ];
@@ -1827,7 +1836,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {u8 log_max_mcg[0x8 ];u8 reserved_at_320[0x3 ];u8 log_max_transport_domain[0x5 ]; -u8 reserved_at_328[0x3 ]; +u8 reserved_at_328[0x2 ]; +u8 pci_relaxed_ordered_read[0x1 ];u8 log_max_pd[0x5 ];u8 reserved_at_330[0xb ];u8 log_max_xrcd[0x5 ];
@@ -1860,7 +1870,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {u8 ext_stride_num_range[0x1 ];u8 reserved_at_3a1[0x2 ];u8 log_max_stride_sz_rq[0x5 ]; -u8 reserved_at_3a8[0x3 ]; +u8 mkc_order_read_after_write[0x1 ]; +u8 mkc_order_write_after_write_ro_only[0x1 ]; +u8 reserved_at_3aa[0x1 ];u8 log_min_stride_sz_rq[0x5 ];u8 reserved_at_3b0[0x3 ];u8 log_max_stride_sz_sq[0x5 ];
drivers/common/mlx5/windows/mlx5_common_os.c +3 -5
@@ -384,7 +384,7 @@ mlx5_os_reg_mr(void *pd, {struct mlx5_devx_mkey_attr mkey_attr;struct mlx5_pd *mlx5_pd = (struct mlx5_pd *)pd; -struct mlx5_hca_attr attr; +struct mlx5_hca_attr attr = {0 };struct mlx5_devx_obj *mkey;void *obj;
@@ -403,10 +403,8 @@ mlx5_os_reg_mr(void *pd, mkey_attr.size = length; mkey_attr.umem_id = ((struct mlx5_devx_umem *)(obj))->umem_id; mkey_attr.pd = mlx5_pd->pdn; -if (!mlx5_haswell_broadwell_cpu) { - mkey_attr.relaxed_ordering_write = attr.relaxed_ordering_write; - mkey_attr.relaxed_ordering_read = attr.relaxed_ordering_read; - } +if (!mlx5_haswell_broadwell_cpu) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, &attr); mkey =mlx5_devx_cmd_mkey_create (mlx5_pd->devx_ctx, &mkey_attr);if (!mkey) {claim_zero (mlx5_os_umem_dereg (obj));
drivers/crypto/mlx5/mlx5_crypto.c +4 -0
@@ -97,7 +97,11 @@ mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv, mlx5_crypto_mkey_update_t update_cb) {uint32_t i; +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr; +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (attr, hca_attr);for (i =0 ; i < qp->entries_n; i++) { attr->klm_array =update_cb (priv, qp, i); qp->mkey[i] =mlx5_devx_cmd_mkey_create (priv->cdev->ctx, attr);
drivers/regex/mlx5/mlx5_regex_fastpath.c +5 -0
@@ -755,9 +755,14 @@ mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id)setup_qps (priv, qp);if (priv->has_umr) { +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr; +#ifdef HAVE_IBV_FLOW_DV_SUPPORT attr.pd = priv->cdev->pdn;#endif +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&attr, hca_attr);for (i =0 ; i < qp->nb_desc; i++) { attr.klm_num =MLX5_REGEX_MAX_KLM_NUM ; attr.klm_array = qp->jobs[i].imkey_array;
drivers/regex/mlx5/mlx5_rxp.c +4 -0
@@ -54,6 +54,7 @@ rxp_create_mkey(struct mlx5_regex_priv *priv, void *ptr, size_t size, uint32_t access, struct mlx5_regex_mkey *mkey) { struct mlx5_devx_mkey_attr mkey_attr; +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr;/* Register the memory. */ mkey->umem = mlx5_glue->devx_umem_reg (priv->cdev->ctx, ptr, size, access);
@@ -72,6 +73,9 @@ rxp_create_mkey(struct mlx5_regex_priv *priv, void *ptr, size_t size,#ifdef HAVE_IBV_FLOW_DV_SUPPORT mkey_attr.pd = priv->cdev->pdn;#endif +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, hca_attr); mkey->mkey =mlx5_devx_cmd_mkey_create (priv->cdev->ctx, &mkey_attr);if (!mkey->mkey) {DRV_LOG (ERR ,"Failed to create direct mkey!" );
drivers/vdpa/mlx5/mlx5_vdpa_mem.c +4 -0
@@ -179,6 +179,7 @@ static int mlx5_vdpa_create_indirect_mkey(struct mlx5_vdpa_priv *priv) {struct mlx5_devx_mkey_attr mkey_attr; +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr;struct mlx5_vdpa_query_mr *mrs = (struct mlx5_vdpa_query_mr *)priv->mrs;struct mlx5_vdpa_query_mr *entry;
@@ -242,6 +243,9 @@ mlx5_vdpa_create_indirect_mkey(struct mlx5_vdpa_priv *priv) mkey_attr.pg_access =0 ; mkey_attr.klm_array = klm_array; mkey_attr.klm_num = klm_index; +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, hca_attr); entry = &mrs[mem->nregions]; entry->mkey =mlx5_devx_cmd_mkey_create (priv->cdev->ctx, &mkey_attr);if (!entry->mkey) {
Signature · 1 line
2.21.0
PATCH v2 1/1 drivers: update relaxed ordering policy for mlx5 mkeys
New adapters expose additional ordering capabilities.
Query the new caps and apply them when creating DevX mkeys via mlx5_devx_mkey_attr_set_ordering(), which sets PCI relaxed ordering and RAW=RO when relaxed order is supported.
Use this helper on Windows (still gated by Haswell/Broadwell) and for Linux wrapped mkeys and crypto/regex/vdpa indirect mkeys when relaxed order only flag is set.
Linux wrapped mkeys continue to use the legacy Haswell/Broadwell rule for IBV_ACCESS_RELAXED_ORDERING on the verbs MR.
Upcoming FW will requires setting the correct ordering attributes, otherwise it fails to create the memory key.
--- drivers/common/mlx5/linux/mlx5_common_os.c | 8 +++++ drivers/common/mlx5/mlx5_devx_cmds.c | 31 ++++++++++++++++++++ drivers/common/mlx5/mlx5_devx_cmds.h | 9 ++++++ drivers/common/mlx5/mlx5_prm.h | 18 ++++++++++-- drivers/common/mlx5/windows/mlx5_common_os.c | 8 ++--- drivers/crypto/mlx5/mlx5_crypto.c | 4 +++ drivers/regex/mlx5/mlx5_regex_fastpath.c | 5 ++++ drivers/regex/mlx5/mlx5_rxp.c | 4 +++ drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 4 +++ 9 files changed, 83 insertions(+), 8 deletions(-)
drivers/common/mlx5/linux/mlx5_common_os.c +8 -0
@@ -997,6 +997,7 @@ int mlx5_os_wrapped_mkey_create(void *ctx, void *pd, uint32_t pdn, void *addr, size_t length, struct mlx5_pmd_wrapped_mr *pmd_mr) { +struct mlx5_hca_attr hca_attr = {0 };struct mlx5_klm klm = { .byte_count = length, .address = (uintptr_t )addr,
@@ -1019,6 +1020,13 @@ mlx5_os_wrapped_mkey_create(void *ctx, void *pd, uint32_t pdn, void *addr, klm.mkey = ibv_mr->lkey; mkey_attr.addr = (uintptr_t )addr; mkey_attr.size = length; +if (mlx5_devx_cmd_query_hca_attr (ctx, &hca_attr)) { +claim_zero (mlx5_glue->dereg_mr (ibv_mr)); +return -1 ; + } +/* If only relaxed order is allowed. */ +if (hca_attr.mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, &hca_attr); mkey =mlx5_devx_cmd_mkey_create (ctx, &mkey_attr);if (!mkey) {claim_zero (mlx5_glue->dereg_mr (ibv_mr));
drivers/common/mlx5/mlx5_devx_cmds.c +31 -0
@@ -331,6 +331,29 @@ mlx5_devx_cmd_flow_counter_query(struct mlx5_devx_obj *dcs,return 0 ; } +/** +* Apply PCI relaxed-ordering and read-after-write ordering to mkey attributes. +* +* @param[in, out] mkey_attr +* Mkey attributes to update. +* @param[in] hca_attr +* HCA capabilities from mlx5_devx_cmd_query_hca_attr(). +*/ +RTE_EXPORT_INTERNAL_SYMBOL (mlx5_devx_mkey_attr_set_ordering) +void +mlx5_devx_mkey_attr_set_ordering (struct mlx5_devx_mkey_attr *mkey_attr, +const struct mlx5_hca_attr *hca_attr) +{ +if (!mkey_attr || !hca_attr) +return ; + + mkey_attr->relaxed_ordering_write = hca_attr->relaxed_ordering_write; + mkey_attr->relaxed_ordering_read = + hca_attr->relaxed_ordering_read || hca_attr->pci_relaxed_ordered_read; +if (hca_attr->mkc_order_read_after_write) + mkey_attr->read_after_write_ordering =MLX5_MKC_RAW_ORDERING_RO ; +} +/** * Create a new mkey. *
@@ -417,6 +440,8 @@ mlx5_devx_cmd_mkey_create(void *ctx,MLX5_SET (mkc, mkc, relaxed_ordering_write, attr->relaxed_ordering_write);MLX5_SET (mkc, mkc, relaxed_ordering_read, attr->relaxed_ordering_read); +MLX5_SET (mkc, mkc, order_read_after_write, + attr->read_after_write_ordering);MLX5_SET64 (mkc, mkc, start_addr, attr->addr);MLX5_SET64 (mkc, mkc, len, attr->size);MLX5_SET (mkc, mkc, crypto_en, attr->crypto_en);
@@ -1003,6 +1028,12 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, relaxed_ordering_write); attr->relaxed_ordering_read =MLX5_GET (cmd_hca_cap, hcattr, relaxed_ordering_read); + attr->pci_relaxed_ordered_read =MLX5_GET (cmd_hca_cap, hcattr, + pci_relaxed_ordered_read); + attr->mkc_order_read_after_write =MLX5_GET (cmd_hca_cap, hcattr, + mkc_order_read_after_write); + attr->mkc_order_write_after_write_ro_only =MLX5_GET (cmd_hca_cap, hcattr, + mkc_order_write_after_write_ro_only); attr->access_register_user =MLX5_GET (cmd_hca_cap, hcattr, access_register_user); attr->eth_net_offloads = MLX5_GET(cmd_hca_cap, hcattr,
drivers/common/mlx5/mlx5_devx_cmds.h +9 -0
@@ -34,6 +34,7 @@ struct mlx5_devx_mkey_attr { uint32_t pg_access:1; uint32_t relaxed_ordering_write:1; uint32_t relaxed_ordering_read:1; + uint32_t read_after_write_ordering:2; uint32_t umr_en:1; uint32_t crypto_en:2; uint32_t set_remote_rw:1;
@@ -237,6 +238,9 @@ struct mlx5_hca_attr { uint32_t vhca_id:16; uint32_t relaxed_ordering_write:1; uint32_t relaxed_ordering_read:1; + uint32_t pci_relaxed_ordered_read:1; + uint32_t mkc_order_read_after_write:1; + uint32_t mkc_order_write_after_write_ro_only:1; uint32_t access_register_user:1; uint32_t wqe_index_ignore:1; uint32_t cross_channel:1;
@@ -748,6 +752,11 @@ int mlx5_devx_cmd_query_hca_attr(void *ctx,__rte_internal struct mlx5_devx_obj *mlx5_devx_cmd_mkey_create(void *ctx,struct mlx5_devx_mkey_attr *attr); + +__rte_internal +void +mlx5_devx_mkey_attr_set_ordering (struct mlx5_devx_mkey_attr *mkey_attr, +const struct mlx5_hca_attr *hca_attr);__rte_internal int mlx5_devx_get_out_command_status(void *out); __rte_internal
drivers/common/mlx5/mlx5_prm.h +15 -3
@@ -1463,7 +1463,9 @@ struct mlx5_ifc_mkc_bits {u8 bsf_octword_size[0x20 ];u8 reserved_at_120[0x80 ];u8 translations_octword_size[0x20 ]; -u8 reserved_at_1c0[0x19 ]; +u8 reserved_at_1c0[0x16 ]; +u8 order_read_after_write[0x2 ]; +u8 reserved_at_1d8[0x1 ];u8 relaxed_ordering_read[0x1 ];u8 reserved_at_1da[0x1 ];u8 log_page_size[0x5 ];
@@ -1478,6 +1480,13 @@ enum { MLX5_MKEY_CRYPTO_ENABLED = 0x1, }; +/* MKC read_after_write_ordering field (2-bit, dword 0x38 bits 9:8). */ +enum mlx5_mkc_raw_ordering { +MLX5_MKC_RAW_ORDERING_SO =0x0 , +MLX5_MKC_RAW_ORDERING_SAO =0x1 , +MLX5_MKC_RAW_ORDERING_RO =0x2 , +}; +struct mlx5_ifc_create_mkey_out_bits {u8 status[0x8 ];u8 reserved_at_8[0x18 ];
@@ -1827,7 +1836,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {u8 log_max_mcg[0x8 ];u8 reserved_at_320[0x3 ];u8 log_max_transport_domain[0x5 ]; -u8 reserved_at_328[0x3 ]; +u8 reserved_at_328[0x2 ]; +u8 pci_relaxed_ordered_read[0x1 ];u8 log_max_pd[0x5 ];u8 reserved_at_330[0xb ];u8 log_max_xrcd[0x5 ];
@@ -1860,7 +1870,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {u8 ext_stride_num_range[0x1 ];u8 reserved_at_3a1[0x2 ];u8 log_max_stride_sz_rq[0x5 ]; -u8 reserved_at_3a8[0x3 ]; +u8 mkc_order_read_after_write[0x1 ]; +u8 mkc_order_write_after_write_ro_only[0x1 ]; +u8 reserved_at_3aa[0x1 ];u8 log_min_stride_sz_rq[0x5 ];u8 reserved_at_3b0[0x3 ];u8 log_max_stride_sz_sq[0x5 ];
drivers/common/mlx5/windows/mlx5_common_os.c +3 -5
@@ -384,7 +384,7 @@ mlx5_os_reg_mr(void *pd, {struct mlx5_devx_mkey_attr mkey_attr;struct mlx5_pd *mlx5_pd = (struct mlx5_pd *)pd; -struct mlx5_hca_attr attr; +struct mlx5_hca_attr attr = {0 };struct mlx5_devx_obj *mkey;void *obj;
@@ -403,10 +403,8 @@ mlx5_os_reg_mr(void *pd, mkey_attr.size = length; mkey_attr.umem_id = ((struct mlx5_devx_umem *)(obj))->umem_id; mkey_attr.pd = mlx5_pd->pdn; -if (!mlx5_haswell_broadwell_cpu) { - mkey_attr.relaxed_ordering_write = attr.relaxed_ordering_write; - mkey_attr.relaxed_ordering_read = attr.relaxed_ordering_read; - } +if (!mlx5_haswell_broadwell_cpu) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, &attr); mkey =mlx5_devx_cmd_mkey_create (mlx5_pd->devx_ctx, &mkey_attr);if (!mkey) {claim_zero (mlx5_os_umem_dereg (obj));
drivers/crypto/mlx5/mlx5_crypto.c +4 -0
@@ -97,7 +97,11 @@ mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv, mlx5_crypto_mkey_update_t update_cb) {uint32_t i; +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr; +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (attr, hca_attr);for (i =0 ; i < qp->entries_n; i++) { attr->klm_array =update_cb (priv, qp, i); qp->mkey[i] =mlx5_devx_cmd_mkey_create (priv->cdev->ctx, attr);
drivers/regex/mlx5/mlx5_regex_fastpath.c +5 -0
@@ -755,9 +755,14 @@ mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id)setup_qps (priv, qp);if (priv->has_umr) { +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr; +#ifdef HAVE_IBV_FLOW_DV_SUPPORT attr.pd = priv->cdev->pdn;#endif +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&attr, hca_attr);for (i =0 ; i < qp->nb_desc; i++) { attr.klm_num =MLX5_REGEX_MAX_KLM_NUM ; attr.klm_array = qp->jobs[i].imkey_array;
drivers/regex/mlx5/mlx5_rxp.c +4 -0
@@ -54,6 +54,7 @@ rxp_create_mkey(struct mlx5_regex_priv *priv, void *ptr, size_t size, uint32_t access, struct mlx5_regex_mkey *mkey) { struct mlx5_devx_mkey_attr mkey_attr; +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr;/* Register the memory. */ mkey->umem = mlx5_glue->devx_umem_reg (priv->cdev->ctx, ptr, size, access);
@@ -72,6 +73,9 @@ rxp_create_mkey(struct mlx5_regex_priv *priv, void *ptr, size_t size,#ifdef HAVE_IBV_FLOW_DV_SUPPORT mkey_attr.pd = priv->cdev->pdn;#endif +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, hca_attr); mkey->mkey =mlx5_devx_cmd_mkey_create (priv->cdev->ctx, &mkey_attr);if (!mkey->mkey) {DRV_LOG (ERR ,"Failed to create direct mkey!" );
drivers/vdpa/mlx5/mlx5_vdpa_mem.c +4 -0
@@ -179,6 +179,7 @@ static int mlx5_vdpa_create_indirect_mkey(struct mlx5_vdpa_priv *priv) {struct mlx5_devx_mkey_attr mkey_attr; +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr;struct mlx5_vdpa_query_mr *mrs = (struct mlx5_vdpa_query_mr *)priv->mrs;struct mlx5_vdpa_query_mr *entry;
@@ -242,6 +243,9 @@ mlx5_vdpa_create_indirect_mkey(struct mlx5_vdpa_priv *priv) mkey_attr.pg_access =0 ; mkey_attr.klm_array = klm_array; mkey_attr.klm_num = klm_index; +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, hca_attr); entry = &mrs[mem->nregions]; entry->mkey =mlx5_devx_cmd_mkey_create (priv->cdev->ctx, &mkey_attr);if (!entry->mkey) {
Signature · 1 line
2.21.0
PATCH v3 1/1 drivers: update relaxed ordering policy for mlx5 mkeys
In reply to Maayan KashaniNew adapters expose additional ordering capabilities.
Query the new caps and apply them when creating DevX mkeys via mlx5_devx_mkey_attr_set_ordering(), which sets PCI relaxed ordering and RAW=RO when relaxed order is supported.
Use this helper on Windows (still gated by Haswell/Broadwell) and for Linux wrapped mkeys and crypto/regex/vdpa indirect mkeys when relaxed order only flag is set.
Linux wrapped mkeys continue to use the legacy Haswell/Broadwell rule for IBV_ACCESS_RELAXED_ORDERING on the verbs MR.
Upcoming FW will requires setting the correct ordering attributes, otherwise it fails to create the memory key.
--- drivers/common/mlx5/linux/mlx5_common_os.c | 10 +++++++ drivers/common/mlx5/mlx5_devx_cmds.c | 31 ++++++++++++++++++++ drivers/common/mlx5/mlx5_devx_cmds.h | 9 ++++++ drivers/common/mlx5/mlx5_prm.h | 18 ++++++++++-- drivers/common/mlx5/windows/mlx5_common_os.c | 8 ++--- drivers/crypto/mlx5/mlx5_crypto.c | 4 +++ drivers/regex/mlx5/mlx5_regex_fastpath.c | 5 ++++ drivers/regex/mlx5/mlx5_rxp.c | 4 +++ drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 4 +++ 9 files changed, 85 insertions(+), 8 deletions(-)
drivers/common/mlx5/linux/mlx5_common_os.c +10 -0
@@ -997,6 +997,7 @@ int mlx5_os_wrapped_mkey_create(void *ctx, void *pd, uint32_t pdn, void *addr, size_t length, struct mlx5_pmd_wrapped_mr *pmd_mr) { +struct mlx5_hca_attr hca_attr = {0 };struct mlx5_klm klm = { .byte_count = length, .address = (uintptr_t )addr,
@@ -1007,6 +1008,7 @@ mlx5_os_wrapped_mkey_create(void *ctx, void *pd, uint32_t pdn, void *addr, .klm_num = 1, };struct mlx5_devx_obj *mkey; +int ret; struct ibv_mr *ibv_mr = mlx5_glue->reg_mr(pd, addr, length, IBV_ACCESS_LOCAL_WRITE | (mlx5_haswell_broadwell_cpu ? 0 :
@@ -1019,6 +1021,14 @@ mlx5_os_wrapped_mkey_create(void *ctx, void *pd, uint32_t pdn, void *addr, klm.mkey = ibv_mr->lkey; mkey_attr.addr = (uintptr_t )addr; mkey_attr.size = length; + ret =mlx5_devx_cmd_query_hca_attr (ctx, &hca_attr); +if (ret !=0 ) { +claim_zero (mlx5_glue->dereg_mr (ibv_mr)); +return ret; + } +/* Apply MKC ordering when FW requires relaxed-order-only mode. */ +if (hca_attr.mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, &hca_attr); mkey =mlx5_devx_cmd_mkey_create (ctx, &mkey_attr);if (!mkey) {claim_zero (mlx5_glue->dereg_mr (ibv_mr));
drivers/common/mlx5/mlx5_devx_cmds.c +31 -0
@@ -331,6 +331,29 @@ mlx5_devx_cmd_flow_counter_query(struct mlx5_devx_obj *dcs,return 0 ; } +/** +* Apply PCI relaxed-ordering and read-after-write ordering to mkey attributes. +* +* @param[in, out] mkey_attr +* Mkey attributes to update. +* @param[in] hca_attr +* HCA capabilities from mlx5_devx_cmd_query_hca_attr(). +*/ +RTE_EXPORT_INTERNAL_SYMBOL (mlx5_devx_mkey_attr_set_ordering) +void +mlx5_devx_mkey_attr_set_ordering (struct mlx5_devx_mkey_attr *mkey_attr, +const struct mlx5_hca_attr *hca_attr) +{ +if (!mkey_attr || !hca_attr) +return ; + + mkey_attr->relaxed_ordering_write = hca_attr->relaxed_ordering_write; + mkey_attr->relaxed_ordering_read = + hca_attr->relaxed_ordering_read || hca_attr->pci_relaxed_ordered_read; +if (hca_attr->mkc_order_read_after_write) + mkey_attr->read_after_write_ordering =MLX5_MKC_RAW_ORDERING_RO ; +} +/** * Create a new mkey. *
@@ -417,6 +440,8 @@ mlx5_devx_cmd_mkey_create(void *ctx,MLX5_SET (mkc, mkc, relaxed_ordering_write, attr->relaxed_ordering_write);MLX5_SET (mkc, mkc, relaxed_ordering_read, attr->relaxed_ordering_read); +MLX5_SET (mkc, mkc, order_read_after_write, + attr->read_after_write_ordering);MLX5_SET64 (mkc, mkc, start_addr, attr->addr);MLX5_SET64 (mkc, mkc, len, attr->size);MLX5_SET (mkc, mkc, crypto_en, attr->crypto_en);
@@ -1003,6 +1028,12 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, relaxed_ordering_write); attr->relaxed_ordering_read =MLX5_GET (cmd_hca_cap, hcattr, relaxed_ordering_read); + attr->pci_relaxed_ordered_read =MLX5_GET (cmd_hca_cap, hcattr, + pci_relaxed_ordered_read); + attr->mkc_order_read_after_write =MLX5_GET (cmd_hca_cap, hcattr, + mkc_order_read_after_write); + attr->mkc_order_write_after_write_ro_only =MLX5_GET (cmd_hca_cap, hcattr, + mkc_order_write_after_write_ro_only); attr->access_register_user =MLX5_GET (cmd_hca_cap, hcattr, access_register_user); attr->eth_net_offloads = MLX5_GET(cmd_hca_cap, hcattr,
drivers/common/mlx5/mlx5_devx_cmds.h +9 -0
@@ -34,6 +34,7 @@ struct mlx5_devx_mkey_attr { uint32_t pg_access:1; uint32_t relaxed_ordering_write:1; uint32_t relaxed_ordering_read:1; + uint32_t read_after_write_ordering:2; uint32_t umr_en:1; uint32_t crypto_en:2; uint32_t set_remote_rw:1;
@@ -237,6 +238,9 @@ struct mlx5_hca_attr { uint32_t vhca_id:16; uint32_t relaxed_ordering_write:1; uint32_t relaxed_ordering_read:1; + uint32_t pci_relaxed_ordered_read:1; + uint32_t mkc_order_read_after_write:1; + uint32_t mkc_order_write_after_write_ro_only:1; uint32_t access_register_user:1; uint32_t wqe_index_ignore:1; uint32_t cross_channel:1;
@@ -748,6 +752,11 @@ int mlx5_devx_cmd_query_hca_attr(void *ctx,__rte_internal struct mlx5_devx_obj *mlx5_devx_cmd_mkey_create(void *ctx,struct mlx5_devx_mkey_attr *attr); + +__rte_internal +void +mlx5_devx_mkey_attr_set_ordering (struct mlx5_devx_mkey_attr *mkey_attr, +const struct mlx5_hca_attr *hca_attr);__rte_internal int mlx5_devx_get_out_command_status(void *out); __rte_internal
drivers/common/mlx5/mlx5_prm.h +15 -3
@@ -1463,7 +1463,9 @@ struct mlx5_ifc_mkc_bits {u8 bsf_octword_size[0x20 ];u8 reserved_at_120[0x80 ];u8 translations_octword_size[0x20 ]; -u8 reserved_at_1c0[0x19 ]; +u8 reserved_at_1c0[0x16 ]; +u8 order_read_after_write[0x2 ]; +u8 reserved_at_1d8[0x1 ];u8 relaxed_ordering_read[0x1 ];u8 reserved_at_1da[0x1 ];u8 log_page_size[0x5 ];
@@ -1478,6 +1480,13 @@ enum { MLX5_MKEY_CRYPTO_ENABLED = 0x1, }; +/* MKC read_after_write_ordering field (2-bit, dword 0x38 bits 9:8). */ +enum mlx5_mkc_raw_ordering { +MLX5_MKC_RAW_ORDERING_SO =0x0 , +MLX5_MKC_RAW_ORDERING_SAO =0x1 , +MLX5_MKC_RAW_ORDERING_RO =0x2 , +}; +struct mlx5_ifc_create_mkey_out_bits {u8 status[0x8 ];u8 reserved_at_8[0x18 ];
@@ -1827,7 +1836,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {u8 log_max_mcg[0x8 ];u8 reserved_at_320[0x3 ];u8 log_max_transport_domain[0x5 ]; -u8 reserved_at_328[0x3 ]; +u8 reserved_at_328[0x2 ]; +u8 pci_relaxed_ordered_read[0x1 ];u8 log_max_pd[0x5 ];u8 reserved_at_330[0xb ];u8 log_max_xrcd[0x5 ];
@@ -1860,7 +1870,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {u8 ext_stride_num_range[0x1 ];u8 reserved_at_3a1[0x2 ];u8 log_max_stride_sz_rq[0x5 ]; -u8 reserved_at_3a8[0x3 ]; +u8 mkc_order_read_after_write[0x1 ]; +u8 mkc_order_write_after_write_ro_only[0x1 ]; +u8 reserved_at_3aa[0x1 ];u8 log_min_stride_sz_rq[0x5 ];u8 reserved_at_3b0[0x3 ];u8 log_max_stride_sz_sq[0x5 ];
drivers/common/mlx5/windows/mlx5_common_os.c +3 -5
@@ -384,7 +384,7 @@ mlx5_os_reg_mr(void *pd, {struct mlx5_devx_mkey_attr mkey_attr;struct mlx5_pd *mlx5_pd = (struct mlx5_pd *)pd; -struct mlx5_hca_attr attr; +struct mlx5_hca_attr attr = {0 };struct mlx5_devx_obj *mkey;void *obj;
@@ -403,10 +403,8 @@ mlx5_os_reg_mr(void *pd, mkey_attr.size = length; mkey_attr.umem_id = ((struct mlx5_devx_umem *)(obj))->umem_id; mkey_attr.pd = mlx5_pd->pdn; -if (!mlx5_haswell_broadwell_cpu) { - mkey_attr.relaxed_ordering_write = attr.relaxed_ordering_write; - mkey_attr.relaxed_ordering_read = attr.relaxed_ordering_read; - } +if (!mlx5_haswell_broadwell_cpu) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, &attr); mkey =mlx5_devx_cmd_mkey_create (mlx5_pd->devx_ctx, &mkey_attr);if (!mkey) {claim_zero (mlx5_os_umem_dereg (obj));
drivers/crypto/mlx5/mlx5_crypto.c +4 -0
@@ -97,7 +97,11 @@ mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv, mlx5_crypto_mkey_update_t update_cb) {uint32_t i; +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr; +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (attr, hca_attr);for (i =0 ; i < qp->entries_n; i++) { attr->klm_array =update_cb (priv, qp, i); qp->mkey[i] =mlx5_devx_cmd_mkey_create (priv->cdev->ctx, attr);
drivers/regex/mlx5/mlx5_regex_fastpath.c +5 -0
@@ -755,9 +755,14 @@ mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id)setup_qps (priv, qp);if (priv->has_umr) { +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr; +#ifdef HAVE_IBV_FLOW_DV_SUPPORT attr.pd = priv->cdev->pdn;#endif +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&attr, hca_attr);for (i =0 ; i < qp->nb_desc; i++) { attr.klm_num =MLX5_REGEX_MAX_KLM_NUM ; attr.klm_array = qp->jobs[i].imkey_array;
drivers/regex/mlx5/mlx5_rxp.c +4 -0
@@ -54,6 +54,7 @@ rxp_create_mkey(struct mlx5_regex_priv *priv, void *ptr, size_t size, uint32_t access, struct mlx5_regex_mkey *mkey) { struct mlx5_devx_mkey_attr mkey_attr; +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr;/* Register the memory. */ mkey->umem = mlx5_glue->devx_umem_reg (priv->cdev->ctx, ptr, size, access);
@@ -72,6 +73,9 @@ rxp_create_mkey(struct mlx5_regex_priv *priv, void *ptr, size_t size,#ifdef HAVE_IBV_FLOW_DV_SUPPORT mkey_attr.pd = priv->cdev->pdn;#endif +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, hca_attr); mkey->mkey =mlx5_devx_cmd_mkey_create (priv->cdev->ctx, &mkey_attr);if (!mkey->mkey) {DRV_LOG (ERR ,"Failed to create direct mkey!" );
drivers/vdpa/mlx5/mlx5_vdpa_mem.c +4 -0
@@ -179,6 +179,7 @@ static int mlx5_vdpa_create_indirect_mkey(struct mlx5_vdpa_priv *priv) {struct mlx5_devx_mkey_attr mkey_attr; +struct mlx5_hca_attr *hca_attr = &priv->cdev->config.hca_attr;struct mlx5_vdpa_query_mr *mrs = (struct mlx5_vdpa_query_mr *)priv->mrs;struct mlx5_vdpa_query_mr *entry;
@@ -242,6 +243,9 @@ mlx5_vdpa_create_indirect_mkey(struct mlx5_vdpa_priv *priv) mkey_attr.pg_access =0 ; mkey_attr.klm_array = klm_array; mkey_attr.klm_num = klm_index; +/* If only relaxed order is allowed. */ +if (hca_attr->mkc_order_write_after_write_ro_only) +mlx5_devx_mkey_attr_set_ordering (&mkey_attr, hca_attr); entry = &mrs[mem->nregions]; entry->mkey =mlx5_devx_cmd_mkey_create (priv->cdev->ctx, &mkey_attr);if (!entry->mkey) {
Signature · 1 line
2.21.0
Hi,
Quoted 12 lines · Maayan Kashani
New adapters expose additional ordering capabilities.
Query the new caps and apply them when creating DevX mkeys viamlx5_devx_mkey_attr_set_ordering(), which sets PCI relaxed ordering and RAW=RO when relaxed order is supported.
Use this helper on Windows (still gated by Haswell/Broadwell) and for Linux wrapped mkeys andcrypto/regex/vdpaindirect mkeys when relaxed order only flag is set.
Linux wrapped mkeys continue to use the legacy Haswell/Broadwell rule forIBV_ACCESS_RELAXED_ORDERINGon the verbs MR.
Upcoming FW will requires setting the correct ordering attributes, otherwise it fails to create the memory key.
Quoted 2 lines
Signed-off-by Maayan Kashani <[email protected]>
Patch applied to next-net-mlx,
Kindest regards
Raslan Darawsheh