common/mlx5: support vDPA completion queue moderation
Subsystems (2)
PATCH 1/2 common/mlx5: support vDPA completion queue moderation
This patch introduces new parameters for VirtQ CQ moderation, used for performance tuning.
--- drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++ drivers/common/mlx5/mlx5_devx_cmds.h | 3 +++ drivers/common/mlx5/mlx5_prm.h | 6 +++++- 3 files changed, 11 insertions(+), 1 deletion(-)
drivers/common/mlx5/mlx5_devx_cmds.c +3 -0
@@ -1662,6 +1662,9 @@ mlx5_devx_cmd_create_virtq(void *ctx,MLX5_SET64 (virtio_q, virtctx, umem_3_offset, attr->umems[2 ].offset);MLX5_SET (virtio_q, virtctx, counter_set_id, attr->counters_obj_id);MLX5_SET (virtio_q, virtctx, pd, attr->pd); +MLX5_SET (virtio_q, virtctx, queue_period_mode, attr->hw_latency_mode); +MLX5_SET (virtio_q, virtctx, queue_period_us, attr->hw_max_latency_us); +MLX5_SET (virtio_q, virtctx, queue_max_count, attr->hw_max_pending_comp);MLX5_SET (virtio_net_q, virtq, tisn_or_qpn, attr->tis_id); virtq_obj->obj = mlx5_glue->devx_obj_create (ctx, in,sizeof (in), out,sizeof (out));
drivers/common/mlx5/mlx5_devx_cmds.h +3 -0
@@ -302,6 +302,9 @@ struct mlx5_devx_virtq_attr { uint32_t rx_csum:1; uint32_t event_mode:3; uint32_t state:4; + uint32_t hw_latency_mode:2; + uint32_t hw_max_latency_us:12; + uint32_t hw_max_pending_comp:16; uint32_t dirty_bitmap_dump_enable:1;uint32_t dirty_bitmap_mkey;uint32_t dirty_bitmap_size;
drivers/common/mlx5/mlx5_prm.h +5 -1
@@ -2379,7 +2379,11 @@ struct mlx5_ifc_virtio_q_bits {u8 counter_set_id[0x20 ];u8 reserved_at_320[0x8 ];u8 pd[0x18 ]; -u8 reserved_at_340[0xc0 ]; +u8 reserved_at_340[0x2 ]; +u8 queue_period_mode[0x2 ]; +u8 queue_period_us[0xc ]; +u8 queue_max_count[0x10 ]; +u8 reserved_at_360[0xa0 ]; };struct mlx5_ifc_virtio_net_q_bits {
Signature · 1 line
2.25.1
PATCH 2/2 vdpa/mlx5: hardware queue moderation
The next parameters control the HW queue moderation feature. This feature helps to control the traffic performance and latency tradeoff.
Each packet completion report from HW to SW requires CQ processing by SW and triggers interrupt for the guest driver. Interrupt report and handling cost CPU cycles and time and the amount of this affects directly on packet performance and latency.
hw_latency_mode parameters [int]
0, HW default.
1, Latency is counted from the first packet completion report. 2, Latency is counted from the last packet completion.hw_max_latency_us parameters [int]
0 - 4095, The maximum time in microseconds that packet completion
report can be delayed.hw_max_pending_comp parameter [int]
0 - 65535, The maximum number of pending packets completions in an HW
queue.
--- doc/guides/vdpadevs/mlx5.rst | 24 ++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa.c | 6 ++++++ drivers/vdpa/mlx5/mlx5_vdpa.h | 3 +++ drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 3 +++ 4 files changed, 36 insertions(+)
doc/guides/vdpadevs/mlx5.rst +24 -0
@@ -135,6 +135,30 @@ Driver options interrupts are configured to the device in order to notify traffic for the driver. Default value is 2s. +- ``hw_latency_mode`` parameter [int] + + The completion queue moderation mode: + + - 0, HW default. + + - 1, Latency is counted from the first packet completion report. + + - 2, Latency is counted from the last packet completion. + +- ``hw_max_latency_us`` parameter [int] + + - 1 - 4095, The maximum time in microseconds that packet completion report + can be delayed. + + - 0, HW default. + +- ``hw_max_pending_comp`` parameter [int] + + - 1 - 65535, The maximum number of pending packets completions in an HW queue. + + - 0, HW default. + + Error handling ^^^^^^^^^^^^^^
drivers/vdpa/mlx5/mlx5_vdpa.c +6 -0
@@ -630,6 +630,12 @@ mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque) priv->event_us = (uint32_t )tmp; }else if (strcmp (key,"no_traffic_time" ) ==0 ) { priv->no_traffic_time_s = (uint32_t )tmp; + }else if (strcmp (key,"hw_latency_mode" ) ==0 ) { + priv->hw_latency_mode = (uint32_t )tmp; + }else if (strcmp (key,"hw_max_latency_us" ) ==0 ) { + priv->hw_max_latency_us = (uint32_t )tmp; + }else if (strcmp (key,"hw_max_pending_comp" ) ==0 ) { + priv->hw_max_pending_comp = (uint32_t )tmp; }else {DRV_LOG (WARNING ,"Invalid key %s." , key); }
drivers/vdpa/mlx5/mlx5_vdpa.h +3 -0
@@ -134,6 +134,9 @@ struct mlx5_vdpa_priv {uint32_t event_us;uint32_t timer_delay_us;uint32_t no_traffic_time_s; +uint8_t hw_latency_mode;/* Hardware CQ moderation mode. */ +uint16_t hw_max_latency_us;/* Hardware CQ moderation period in usec. */ +uint16_t hw_max_pending_comp;/* Hardware CQ moderation counter. */ struct rte_vdpa_device *vdev;/* vDPA device. */ int vid;/* vhost device id. */ struct ibv_context *ctx;/* Device context. */
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +3 -0
@@ -327,6 +327,9 @@ mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index) attr.tis_id = priv->tiss[(index /2 ) % priv->num_lag_ports]->id; attr.queue_index = index; attr.pd = priv->pdn; + attr.hw_latency_mode = priv->hw_latency_mode; + attr.hw_max_latency_us = priv->hw_max_latency_us; + attr.hw_max_pending_comp = priv->hw_max_pending_comp; virtq->virtq =mlx5_devx_cmd_create_virtq (priv->ctx, &attr); virtq->priv = priv;if (!virtq->virtq)
Signature · 1 line
2.25.1
Quoted 10 lines · Xueming Li
This patch introduces new parameters for VirtQ CQ moderation, used for performance tuning.
Signed-off-by Xueming Li <[email protected]>--- drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++ drivers/common/mlx5/mlx5_devx_cmds.h | 3 +++ drivers/common/mlx5/mlx5_prm.h | 6 +++++- 3 files changed, 11 insertions(+), 1 deletion(-)
Thanks,
Maxime
Re: [dpdk-dev] [PATCH 2/2] vdpa/mlx5: hardware queue moderation
In reply to Xueming LiQuoted 20 lines · Xueming Li
The next parameters control the HW queue moderation feature. This feature helps to control the traffic performance and latency tradeoff.
Each packet completion report from HW to SW requires CQ processing by SW and triggers interrupt for the guest driver. Interrupt report and handling cost CPU cycles and time and the amount of this affects directly on packet performance and latency.
hw_latency_modeparameters [int]
0, HW default.
1, Latency is counted from the first packet completion report. 2, Latency is counted from the last packet completion.hw_max_latency_usparameters [int]
0 - 4095, The maximum time in microseconds that packet completion
report can be delayed.hw_max_pending_compparameter [int]
0 - 65535, The maximum number of pending packets completions in an HW
queue.
Quoted 7 lines
Signed-off-by Xueming Li <[email protected]>--- doc/guides/vdpadevs/mlx5.rst | 24 ++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa.c | 6 ++++++ drivers/vdpa/mlx5/mlx5_vdpa.h | 3 +++ drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 3 +++ 4 files changed, 36 insertions(+)
Thanks,
Maxime
Quoted 9 lines · Xueming Li
This patch introduces new parameters for VirtQ CQ moderation, used for performance tuning.
Signed-off-by Xueming Li <[email protected]>--- drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++ drivers/common/mlx5/mlx5_devx_cmds.h | 3 +++ drivers/common/mlx5/mlx5_prm.h | 6 +++++- 3 files changed, 11 insertions(+), 1 deletion(-)
Series applied to dpdk-next-virtio/main.
Thanks,
Maxime
Re: [dpdk-dev] [PATCH 2/2] vdpa/mlx5: hardware queue moderation
In reply to Xueming LiQuoted 20 lines · Xueming Li
The next parameters control the HW queue moderation feature. This feature helps to control the traffic performance and latency tradeoff.
Each packet completion report from HW to SW requires CQ processing by SW and triggers interrupt for the guest driver. Interrupt report and handling cost CPU cycles and time and the amount of this affects directly on packet performance and latency.
hw_latency_modeparameters [int]
0, HW default.
1, Latency is counted from the first packet completion report. 2, Latency is counted from the last packet completion.hw_max_latency_usparameters [int]
0 - 4095, The maximum time in microseconds that packet completion
report can be delayed.hw_max_pending_compparameter [int]
0 - 65535, The maximum number of pending packets completions in an HW
queue.
Quoted 7 lines
Signed-off-by Xueming Li <[email protected]>--- doc/guides/vdpadevs/mlx5.rst | 24 ++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa.c | 6 ++++++ drivers/vdpa/mlx5/mlx5_vdpa.h | 3 +++ drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 3 +++ 4 files changed, 36 insertions(+)
Series applied to dpdk-next-virtio/main.
Thanks,
Maxime