common/mlx5: get number of ports that can be bonded
Subsystems (2)
PATCH 1/2 common/mlx5: get number of ports that can be bonded
Get HCA capability: number of physical ports that can be bonded.
--- drivers/common/mlx5/mlx5_devx_cmds.c | 5 +++-- drivers/common/mlx5/mlx5_devx_cmds.h | 1 + 2 files changed, 4 insertions(+), 2 deletions(-)
drivers/common/mlx5/mlx5_devx_cmds.c +3 -2
@@ -711,6 +711,7 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, attr->non_wire_sq =MLX5_GET (cmd_hca_cap, hcattr, non_wire_sq); attr->log_max_static_sq_wq =MLX5_GET (cmd_hca_cap, hcattr, log_max_static_sq_wq); + attr->num_lag_ports =MLX5_GET (cmd_hca_cap, hcattr, num_lag_ports); attr->dev_freq_khz =MLX5_GET (cmd_hca_cap, hcattr, device_frequency_khz); attr->scatter_fcs_w_decap_disable =
@@ -1429,8 +1430,8 @@ mlx5_devx_cmd_create_tis(void *ctx, tis_ctx =MLX5_ADDR_OF (create_tis_in, in, ctx);MLX5_SET (tisc, tis_ctx, strict_lag_tx_port_affinity, tis_attr->strict_lag_tx_port_affinity); -MLX5_SET (tisc, tis_ctx, strict_lag_tx_port_affinity, - tis_attr->strict_lag_tx_port_affinity); +MLX5_SET (tisc, tis_ctx, lag_tx_port_affinity, + tis_attr->lag_tx_port_affinity);MLX5_SET (tisc, tis_ctx, prio, tis_attr->prio);MLX5_SET (tisc, tis_ctx, transport_domain, tis_attr->transport_domain);
drivers/common/mlx5/mlx5_devx_cmds.h +1 -0
@@ -99,6 +99,7 @@ struct mlx5_hca_attr { uint32_t cross_channel:1; uint32_t non_wire_sq:1; /* SQ with non-wire ops is supported. */ uint32_t log_max_static_sq_wq:5; /* Static WQE size SQ. */ + uint32_t num_lag_ports:4; /* Number of ports can be bonded. */uint32_t dev_freq_khz;/* Timestamp counter frequency, kHz. */ uint32_t scatter_fcs_w_decap_disable:1; uint32_t regex:1;
Signature · 1 line
2.25.1
PATCH 1/2 common/mlx5: get number of ports that can be bonded
Get HCA capability: number of physical ports that can be bonded.
--- drivers/common/mlx5/mlx5_devx_cmds.c | 5 +++-- drivers/common/mlx5/mlx5_devx_cmds.h | 1 + 2 files changed, 4 insertions(+), 2 deletions(-)
drivers/common/mlx5/mlx5_devx_cmds.c +3 -2
@@ -711,6 +711,7 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, attr->non_wire_sq =MLX5_GET (cmd_hca_cap, hcattr, non_wire_sq); attr->log_max_static_sq_wq =MLX5_GET (cmd_hca_cap, hcattr, log_max_static_sq_wq); + attr->num_lag_ports =MLX5_GET (cmd_hca_cap, hcattr, num_lag_ports); attr->dev_freq_khz =MLX5_GET (cmd_hca_cap, hcattr, device_frequency_khz); attr->scatter_fcs_w_decap_disable =
@@ -1429,8 +1430,8 @@ mlx5_devx_cmd_create_tis(void *ctx, tis_ctx =MLX5_ADDR_OF (create_tis_in, in, ctx);MLX5_SET (tisc, tis_ctx, strict_lag_tx_port_affinity, tis_attr->strict_lag_tx_port_affinity); -MLX5_SET (tisc, tis_ctx, strict_lag_tx_port_affinity, - tis_attr->strict_lag_tx_port_affinity); +MLX5_SET (tisc, tis_ctx, lag_tx_port_affinity, + tis_attr->lag_tx_port_affinity);MLX5_SET (tisc, tis_ctx, prio, tis_attr->prio);MLX5_SET (tisc, tis_ctx, transport_domain, tis_attr->transport_domain);
drivers/common/mlx5/mlx5_devx_cmds.h +1 -0
@@ -99,6 +99,7 @@ struct mlx5_hca_attr { uint32_t cross_channel:1; uint32_t non_wire_sq:1; /* SQ with non-wire ops is supported. */ uint32_t log_max_static_sq_wq:5; /* Static WQE size SQ. */ + uint32_t num_lag_ports:4; /* Number of ports can be bonded. */uint32_t dev_freq_khz;/* Timestamp counter frequency, kHz. */ uint32_t scatter_fcs_w_decap_disable:1; uint32_t regex:1;
Signature · 1 line
2.25.1
PATCH 2/2 vdpa/mlx5: specify lag port affinity
If set TIS lag port affinity to auto, firmware assign port affinity on each creation with Round Robin. In case of 2 PFs, if create virtq, destroy and create again, then each virtq will get same port affinity.
To resolve this fw limitation, this patch sets create TIS with specified affinity for each PF.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 3 +++ drivers/vdpa/mlx5/mlx5_vdpa.h | 3 ++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 23 ++++++++++++++--------- 3 files changed, 19 insertions(+), 10 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +3 -0
@@ -730,6 +730,9 @@ mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, } priv->caps = attr.vdpa; priv->log_max_rqt_size = attr.log_max_rqt_size; + priv->num_lag_ports = attr.num_lag_ports; +if (attr.num_lag_ports ==0 ) + priv->num_lag_ports =1 ; priv->ctx = ctx; priv->pci_dev = pci_dev; priv->var = mlx5_glue->dv_alloc_var (ctx,0 );
drivers/vdpa/mlx5/mlx5_vdpa.h +2 -1
@@ -146,8 +146,9 @@ struct mlx5_vdpa_priv {struct mlx5dv_devx_uar *uar;struct rte_intr_handle intr_handle;struct mlx5_devx_obj *td; -struct mlx5_devx_obj *tis; +struct mlx5_devx_obj *tiss[16 ];/* TIS list for each LAG port. */ uint16_t nr_virtqs; +uint8_t num_lag_ports;uint64_t features;/* Negotiated features. */ uint16_t log_max_rqt_size;struct mlx5_vdpa_steer steer;
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +14 -9
@@ -103,12 +103,13 @@ void mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv) {int i; -for (i =0 ; i < priv->nr_virtqs; i++)mlx5_vdpa_virtq_unset (&priv->virtqs[i]); -if (priv->tis) { -claim_zero (mlx5_devx_cmd_destroy (priv->tis)); - priv->tis =NULL ; +for (i =0 ; i < priv->num_lag_ports; i++) { +if (priv->tiss[i]) { +claim_zero (mlx5_devx_cmd_destroy (priv->tiss[i])); + priv->tiss[i] =NULL ; + } }if (priv->td) {claim_zero (mlx5_devx_cmd_destroy (priv->td));
@@ -302,7 +303,7 @@ mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index) attr.hw_used_index = last_used_idx; attr.q_size = vq.size; attr.mkey = priv->gpa_mkey_index; - attr.tis_id = priv->tis->id; + attr.tis_id = priv->tiss[(index /2 ) % priv->num_lag_ports]->id; attr.queue_index = index; attr.pd = priv->pdn; virtq->virtq =mlx5_devx_cmd_create_virtq (priv->ctx, &attr);
@@ -432,10 +433,14 @@ mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv)return -rte_errno; } tis_attr.transport_domain = priv->td->id; - priv->tis =mlx5_devx_cmd_create_tis (priv->ctx, &tis_attr); -if (!priv->tis) { -DRV_LOG (ERR ,"Failed to create TIS." ); - goto error; +for (i =0 ; i < priv->num_lag_ports; i++) { +/* 0 is auto affinity, non-zero value to propose port. */ + tis_attr.lag_tx_port_affinity = i +1 ; + priv->tiss[i] =mlx5_devx_cmd_create_tis (priv->ctx, &tis_attr); +if (!priv->tiss[i]) { +DRV_LOG (ERR ,"Failed to create TIS %u." , i); + goto error; + } } priv->nr_virtqs = nr_vring;for (i =0 ; i < nr_vring; i++)
Signature · 1 line
2.25.1
Maxime Coquelin
AppliedRe: [dpdk-dev] [PATCH v1 2/2] vdpa/mlx5: specify lag port affinity
Quoted 11 lines · Xueming Li
If set TIS lag port affinity to auto, firmware assign port affinity on each creation with Round Robin. In case of 2 PFs, if create virtq, destroy and create again, then each virtq will get same port affinity.
To resolve this fw limitation, this patch sets create TIS with specified affinity for each PF.
Quoted 8 lines
Signed-off-by Xueming Li <[email protected]>Acked-by Matan Azrad <[email protected]>Reviewed-by Maxime Coquelin <[email protected]>--- drivers/vdpa/mlx5/mlx5_vdpa.c | 3 +++ drivers/vdpa/mlx5/mlx5_vdpa.h | 3 ++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 23 ++++++++++++++--------- 3 files changed, 19 insertions(+), 10 deletions(-)
Applied to dpdk-next-virtio/main.
Thanks!
Maxime
Maxime Coquelin
AppliedQuoted 11 lines · Xueming Li
Get HCA capability: number of physical ports that can be bonded.
Signed-off-by Xueming Li <[email protected]>Acked-by Matan Azrad <[email protected]>Reviewed-by Maxime Coquelin <[email protected]>--- drivers/common/mlx5/mlx5_devx_cmds.c | 5 +++-- drivers/common/mlx5/mlx5_devx_cmds.h | 1 + 2 files changed, 4 insertions(+), 2 deletions(-)
Applied to dpdk-next-virtio/main.
Thanks!
Maxime
Hi,
Quoted 3 lines · Xueming Li
Get HCA capability: number of physical ports that can be bonded.
Is that a fix or a new feature? The commit title and description seem to indicate the former. But stable ML being cc'ed seems to indicate the latter.
If this is a fix, please describe what it is fixing, and add Fixes tag pointing on the commit it is fixing.
Thanks,
Maxime
Quoted 43 lines
Signed-off-by Xueming Li <[email protected]>Acked-by Matan Azrad <[email protected]>--- drivers/common/mlx5/mlx5_devx_cmds.c | 5 +++-- drivers/common/mlx5/mlx5_devx_cmds.h | 1 + 2 files changed, 4 insertions(+), 2 deletions(-)drivers/common/mlx5/mlx5_devx_cmds.c +3
-2@@ -711,6 +711,7 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, attr->non_wire_sq =MLX5_GET (cmd_hca_cap, hcattr, non_wire_sq); attr->log_max_static_sq_wq =MLX5_GET (cmd_hca_cap, hcattr, log_max_static_sq_wq); + attr->num_lag_ports =MLX5_GET (cmd_hca_cap, hcattr, num_lag_ports); attr->dev_freq_khz =MLX5_GET (cmd_hca_cap, hcattr, device_frequency_khz); attr->scatter_fcs_w_decap_disable =@@ -1429,8 +1430,8 @@ mlx5_devx_cmd_create_tis(void *ctx, tis_ctx =MLX5_ADDR_OF (create_tis_in, in, ctx);MLX5_SET (tisc, tis_ctx, strict_lag_tx_port_affinity, tis_attr->strict_lag_tx_port_affinity); -MLX5_SET (tisc, tis_ctx, strict_lag_tx_port_affinity, - tis_attr->strict_lag_tx_port_affinity); +MLX5_SET (tisc, tis_ctx, lag_tx_port_affinity, + tis_attr->lag_tx_port_affinity);MLX5_SET (tisc, tis_ctx, prio, tis_attr->prio);MLX5_SET (tisc, tis_ctx, transport_domain, tis_attr->transport_domain);drivers/common/mlx5/mlx5_devx_cmds.h +1
-0@@ -99,6 +99,7 @@ struct mlx5_hca_attr { uint32_t cross_channel:1; uint32_t non_wire_sq:1; /* SQ with non-wire ops is supported. */ uint32_t log_max_static_sq_wq:5; /* Static WQE size SQ. */ + uint32_t num_lag_ports:4; /* Number of ports can be bonded. */uint32_t dev_freq_khz;/* Timestamp counter frequency, kHz. */ uint32_t scatter_fcs_w_decap_disable:1; uint32_t regex:1;
PATCH 2/2 vdpa/mlx5: specify lag port affinity
In reply to Xueming LiIf set TIS lag port affinity to auto, firmware assign port affinity on each creation with Round Robin. In case of 2 PFs, if create virtq, destroy and create again, then each virtq will get same port affinity.
To resolve this fw limitation, this patch sets create TIS with specified affinity for each PF.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 3 +++ drivers/vdpa/mlx5/mlx5_vdpa.h | 3 ++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 23 ++++++++++++++--------- 3 files changed, 19 insertions(+), 10 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +3 -0
@@ -730,6 +730,9 @@ mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, } priv->caps = attr.vdpa; priv->log_max_rqt_size = attr.log_max_rqt_size; + priv->num_lag_ports = attr.num_lag_ports; +if (attr.num_lag_ports ==0 ) + priv->num_lag_ports =1 ; priv->ctx = ctx; priv->pci_dev = pci_dev; priv->var = mlx5_glue->dv_alloc_var (ctx,0 );
drivers/vdpa/mlx5/mlx5_vdpa.h +2 -1
@@ -146,8 +146,9 @@ struct mlx5_vdpa_priv {struct mlx5dv_devx_uar *uar;struct rte_intr_handle intr_handle;struct mlx5_devx_obj *td; -struct mlx5_devx_obj *tis; +struct mlx5_devx_obj *tiss[16 ];/* TIS list for each LAG port. */ uint16_t nr_virtqs; +uint8_t num_lag_ports;uint64_t features;/* Negotiated features. */ uint16_t log_max_rqt_size;struct mlx5_vdpa_steer steer;
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +14 -9
@@ -103,12 +103,13 @@ void mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv) {int i; -for (i =0 ; i < priv->nr_virtqs; i++)mlx5_vdpa_virtq_unset (&priv->virtqs[i]); -if (priv->tis) { -claim_zero (mlx5_devx_cmd_destroy (priv->tis)); - priv->tis =NULL ; +for (i =0 ; i < priv->num_lag_ports; i++) { +if (priv->tiss[i]) { +claim_zero (mlx5_devx_cmd_destroy (priv->tiss[i])); + priv->tiss[i] =NULL ; + } }if (priv->td) {claim_zero (mlx5_devx_cmd_destroy (priv->td));
@@ -302,7 +303,7 @@ mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index) attr.hw_used_index = last_used_idx; attr.q_size = vq.size; attr.mkey = priv->gpa_mkey_index; - attr.tis_id = priv->tis->id; + attr.tis_id = priv->tiss[(index /2 ) % priv->num_lag_ports]->id; attr.queue_index = index; attr.pd = priv->pdn; virtq->virtq =mlx5_devx_cmd_create_virtq (priv->ctx, &attr);
@@ -432,10 +433,14 @@ mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv)return -rte_errno; } tis_attr.transport_domain = priv->td->id; - priv->tis =mlx5_devx_cmd_create_tis (priv->ctx, &tis_attr); -if (!priv->tis) { -DRV_LOG (ERR ,"Failed to create TIS." ); - goto error; +for (i =0 ; i < priv->num_lag_ports; i++) { +/* 0 is auto affinity, non-zero value to propose port. */ + tis_attr.lag_tx_port_affinity = i +1 ; + priv->tiss[i] =mlx5_devx_cmd_create_tis (priv->ctx, &tis_attr); +if (!priv->tiss[i]) { +DRV_LOG (ERR ,"Failed to create TIS %u." , i); + goto error; + } } priv->nr_virtqs = nr_vring;for (i =0 ; i < nr_vring; i++)
Signature · 1 line
2.25.1
Re: [dpdk-dev] [PATCH 2/2] vdpa/mlx5: specify lag port affinity
Quoted 6 lines · Xueming Li
If set TIS lag port affinity to auto, firmware assign port affinity on each creation with Round Robin. In case of 2 PFs, if create virtq, destroy and create again, then each virtq will get same port affinity.
To resolve this fw limitation, this patch sets create TIS with specified affinity for each PF.
OK, this patch describes better the issue the series is addressing. Could you add a Fixes tag, so that it helps maintainers to backport it?
Other than that:
Thanks,
Maxime
Quoted 10 lines
Signed-off-by Xueming Li <[email protected]>Acked-by Matan Azrad <[email protected]>--- drivers/vdpa/mlx5/mlx5_vdpa.c | 3 +++ drivers/vdpa/mlx5/mlx5_vdpa.h | 3 ++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 23 ++++++++++++++--------- 3 files changed, 19 insertions(+), 10 deletions(-)
Re: [dpdk-dev] [PATCH 2/2] vdpa/mlx5: specify lag port affinity
Quoted 17 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Wednesday, October 28, 2020 5:35 PM
To: Xueming(Steven) Li <[email protected]>; Matan Azrad <[email protected]>; Slava Ovsiienko <[email protected]>
Cc: [email protected]; Asaf Penso <[email protected]>; [email protected]
Subject: Re: [dpdk-dev] [PATCH 2/2] vdpa/mlx5: specify lag port affinityQuoted 4 lines · Xueming Li
If set TIS lag port affinity to auto, firmware assign port affinity on each creation with Round Robin. In case of 2 PFs, if create virtq, destroy and create again, then each virtq will get same port affinity.
Quoted 6 lines
Quoted 2 lines
To resolve this fw limitation, this patch sets create TIS with specified affinity for each PF.
OK, this patch describes better the issue the series is addressing. Could you add a Fixes tag, so that it helps maintainers to backport it?
Sure, I'll come up with a new version, thanks!
Quoted 16 lines
Other than that:
Reviewed-by Maxime Coquelin <[email protected]>Thanks,
MaximeQuoted 10 lines
Signed-off-by Xueming Li <[email protected]>Acked-by Matan Azrad <[email protected]>--- drivers/vdpa/mlx5/mlx5_vdpa.c | 3 +++ drivers/vdpa/mlx5/mlx5_vdpa.h | 3 ++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 23 ++++++++++++++--------- 3 files changed, 19 insertions(+), 10 deletions(-)