common/mlx5: add virtq attributes error fields
Subsystems (2)
PATCH 1/2 common/mlx5: add virtq attributes error fields
Add the needed fields for virtq DevX object to read the error state.
--- drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++ drivers/common/mlx5/mlx5_devx_cmds.h | 1 + drivers/common/mlx5/mlx5_prm.h | 9 +++++++-- 3 files changed, 11 insertions(+), 2 deletions(-)
drivers/common/mlx5/mlx5_devx_cmds.c +3 -0
@@ -1754,6 +1754,9 @@ mlx5_devx_cmd_query_virtq(struct mlx5_devx_obj *virtq_obj, attr->hw_available_index =MLX5_GET16 (virtio_net_q, virtq, hw_available_index); attr->hw_used_index =MLX5_GET16 (virtio_net_q, virtq, hw_used_index); + attr->state =MLX5_GET16 (virtio_net_q, virtq, state); + attr->error_type =MLX5_GET16 (virtio_net_q, virtq, + virtio_q_context.error_type);return ret; }
drivers/common/mlx5/mlx5_devx_cmds.h +1 -0
@@ -298,6 +298,7 @@ struct mlx5_devx_virtq_attr {uint32_t size;uint64_t offset; } umems[3 ]; +uint8_t error_type; };
drivers/common/mlx5/mlx5_prm.h +7 -2
@@ -2280,7 +2280,8 @@ struct mlx5_ifc_virtio_q_bits {u8 used_addr[0x40 ];u8 available_addr[0x40 ];u8 virtio_q_mkey[0x20 ]; -u8 reserved_at_160[0x20 ]; +u8 reserved_at_160[0x18 ]; +u8 error_type[0x8 ];u8 umem_1_id[0x20 ];u8 umem_1_size[0x20 ];u8 umem_1_offset[0x40 ];
@@ -2308,7 +2309,7 @@ struct mlx5_ifc_virtio_net_q_bits {u8 vhost_log_page[0x5 ];u8 reserved_at_90[0xc ];u8 state[0x4 ]; -u8 error_type[0x8 ]; +u8 reserved_at_a0[0x8 ];u8 tisn_or_qpn[0x18 ];u8 dirty_bitmap_mkey[0x20 ];u8 dirty_bitmap_size[0x20 ];
@@ -2329,6 +2330,10 @@ struct mlx5_ifc_query_virtq_out_bits {struct mlx5_ifc_virtio_net_q_bits virtq; }; +enum { +MLX5_EVENT_TYPE_OBJECT_CHANGE =0x27 , +}; +enum {MLX5_QP_ST_RC =0x0 , };
Signature · 1 line
2.25.1
PATCH 1/2 common/mlx5: add virtq attributes error fields
Add the needed fields for virtq DevX object to read the error state.
--- drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++ drivers/common/mlx5/mlx5_devx_cmds.h | 1 + drivers/common/mlx5/mlx5_prm.h | 9 +++++++-- 3 files changed, 11 insertions(+), 2 deletions(-)
drivers/common/mlx5/mlx5_devx_cmds.c +3 -0
@@ -1754,6 +1754,9 @@ mlx5_devx_cmd_query_virtq(struct mlx5_devx_obj *virtq_obj, attr->hw_available_index =MLX5_GET16 (virtio_net_q, virtq, hw_available_index); attr->hw_used_index =MLX5_GET16 (virtio_net_q, virtq, hw_used_index); + attr->state =MLX5_GET16 (virtio_net_q, virtq, state); + attr->error_type =MLX5_GET16 (virtio_net_q, virtq, + virtio_q_context.error_type);return ret; }
drivers/common/mlx5/mlx5_devx_cmds.h +1 -0
@@ -298,6 +298,7 @@ struct mlx5_devx_virtq_attr {uint32_t size;uint64_t offset; } umems[3 ]; +uint8_t error_type; };
drivers/common/mlx5/mlx5_prm.h +7 -2
@@ -2280,7 +2280,8 @@ struct mlx5_ifc_virtio_q_bits {u8 used_addr[0x40 ];u8 available_addr[0x40 ];u8 virtio_q_mkey[0x20 ]; -u8 reserved_at_160[0x20 ]; +u8 reserved_at_160[0x18 ]; +u8 error_type[0x8 ];u8 umem_1_id[0x20 ];u8 umem_1_size[0x20 ];u8 umem_1_offset[0x40 ];
@@ -2308,7 +2309,7 @@ struct mlx5_ifc_virtio_net_q_bits {u8 vhost_log_page[0x5 ];u8 reserved_at_90[0xc ];u8 state[0x4 ]; -u8 error_type[0x8 ]; +u8 reserved_at_a0[0x8 ];u8 tisn_or_qpn[0x18 ];u8 dirty_bitmap_mkey[0x20 ];u8 dirty_bitmap_size[0x20 ];
@@ -2329,6 +2330,10 @@ struct mlx5_ifc_query_virtq_out_bits {struct mlx5_ifc_virtio_net_q_bits virtq; }; +enum { +MLX5_EVENT_TYPE_OBJECT_CHANGE =0x27 , +}; +enum {MLX5_QP_ST_RC =0x0 , };
Signature · 1 line
2.25.1
PATCH 2/2 vdpa/mlx5: hardware error handling
When hardware error happens, vdpa didn't get such information and leave driver in silent: working state but no response.
This patch subscribes firmware virtq error event and try to recover max 3 times in 10 seconds, stop virtq if max retry number reached.
When error happens, PMD log in warning level. If failed to recover, outputs error log. Query virtq statistics to get error counters report.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 2 + drivers/vdpa/mlx5/mlx5_vdpa.h | 37 ++++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 140 ++++++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 61 +++++++++--- 4 files changed, 225 insertions(+), 15 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +2 -0
@@ -283,6 +283,7 @@ mlx5_vdpa_dev_close(int vid) }if (priv->configured) ret |=mlx5_vdpa_lm_log (priv); +mlx5_vdpa_err_event_unset (priv);mlx5_vdpa_cqe_event_unset (priv);mlx5_vdpa_steer_unset (priv);mlx5_vdpa_virtqs_release (priv);
@@ -318,6 +319,7 @@ mlx5_vdpa_dev_config(int vid)DRV_LOG (WARNING ,"MTU cannot be set on device %s." , vdev->device->name);if (mlx5_vdpa_pd_create (priv) ||mlx5_vdpa_mem_register (priv) || +mlx5_vdpa_err_event_setup (priv) ||mlx5_vdpa_virtqs_prepare (priv) ||mlx5_vdpa_steer_setup (priv) ||mlx5_vdpa_cqe_event_setup (priv)) {mlx5_vdpa_dev_close (vid);
drivers/vdpa/mlx5/mlx5_vdpa.h +37 -0
@@ -87,6 +87,7 @@ struct mlx5_vdpa_virtq {uint16_t vq_size;uint8_t notifier_state;bool stopped; +uint32_t version;struct mlx5_vdpa_priv *priv;struct mlx5_devx_obj *virtq;struct mlx5_devx_obj *counters;
@@ -97,6 +98,8 @@ struct mlx5_vdpa_virtq {uint32_t size; } umems[3 ];struct rte_intr_handle intr_handle; +uint64_t err_time[3 ];/* RDTSC time of recent errors. */ +uint32_t n_retry;struct mlx5_devx_virtio_q_couners_attr reset; };
@@ -143,8 +146,10 @@ struct mlx5_vdpa_priv {struct rte_vhost_memory *vmem;uint32_t eqn;struct mlx5dv_devx_event_channel *eventc; +struct mlx5dv_devx_event_channel *err_chnl;struct mlx5dv_devx_uar *uar;struct rte_intr_handle intr_handle; +struct rte_intr_handle err_intr_handle;struct mlx5_devx_obj *td;struct mlx5_devx_obj *tis;uint16_t nr_virtqs;
@@ -259,6 +264,25 @@ int mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv);*/ void mlx5_vdpa_cqe_event_unset (struct mlx5_vdpa_priv *priv); +/** +* Setup error interrupt handler. +* +* @param[in] priv +* The vdpa driver private structure. +* +* @return +* 0 on success, a negative errno value otherwise and rte_errno is set. +*/ +int mlx5_vdpa_err_event_setup (struct mlx5_vdpa_priv *priv); + +/** +* Unset error event handler. +* +* @param[in] priv +* The vdpa driver private structure. +*/ +void mlx5_vdpa_err_event_unset (struct mlx5_vdpa_priv *priv); +/** * Release a virtq and all its related resources. *
@@ -392,6 +416,19 @@ int mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state);*/ int mlx5_vdpa_virtq_stop (struct mlx5_vdpa_priv *priv,int index); +/** +* Query virtq information. +* +* @param[in] priv +* The vdpa driver private structure. +* @param[in] index +* The virtq index. +* +* @return +* 0 on success, a negative value otherwise. +*/ +int mlx5_vdpa_virtq_query (struct mlx5_vdpa_priv *priv,int index); +/** * Get virtq statistics. *
drivers/vdpa/mlx5/mlx5_vdpa_event.c +140 -0
157 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +46 -15
130 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH 2/2 vdpa/mlx5: hardware error handling
In reply to Xueming LiWhen hardware error happens, vdpa didn't get such information and leave driver in silent: working state but no response.
This patch subscribes firmware virtq error event and try to recover max 3 times in 10 seconds, stop virtq if max retry number reached.
When error happens, PMD log in warning level. If failed to recover, outputs error log. Query virtq statitics to get error counters report.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 2 + drivers/vdpa/mlx5/mlx5_vdpa.h | 37 ++++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 140 ++++++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 61 +++++++++--- 4 files changed, 225 insertions(+), 15 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +2 -0
@@ -283,6 +283,7 @@ mlx5_vdpa_dev_close(int vid) }if (priv->configured) ret |=mlx5_vdpa_lm_log (priv); +mlx5_vdpa_err_event_unset (priv);mlx5_vdpa_cqe_event_unset (priv);mlx5_vdpa_steer_unset (priv);mlx5_vdpa_virtqs_release (priv);
@@ -318,6 +319,7 @@ mlx5_vdpa_dev_config(int vid)DRV_LOG (WARNING ,"MTU cannot be set on device %s." , vdev->device->name);if (mlx5_vdpa_pd_create (priv) ||mlx5_vdpa_mem_register (priv) || +mlx5_vdpa_err_event_setup (priv) ||mlx5_vdpa_virtqs_prepare (priv) ||mlx5_vdpa_steer_setup (priv) ||mlx5_vdpa_cqe_event_setup (priv)) {mlx5_vdpa_dev_close (vid);
drivers/vdpa/mlx5/mlx5_vdpa.h +37 -0
@@ -87,6 +87,7 @@ struct mlx5_vdpa_virtq {uint16_t vq_size;uint8_t notifier_state;bool stopped; +uint32_t version;struct mlx5_vdpa_priv *priv;struct mlx5_devx_obj *virtq;struct mlx5_devx_obj *counters;
@@ -97,6 +98,8 @@ struct mlx5_vdpa_virtq {uint32_t size; } umems[3 ];struct rte_intr_handle intr_handle; +uint64_t err_time[3 ];/* RDTSC time of recent errors. */ +uint32_t n_retry;struct mlx5_devx_virtio_q_couners_attr reset; };
@@ -143,8 +146,10 @@ struct mlx5_vdpa_priv {struct rte_vhost_memory *vmem;uint32_t eqn;struct mlx5dv_devx_event_channel *eventc; +struct mlx5dv_devx_event_channel *err_chnl;struct mlx5dv_devx_uar *uar;struct rte_intr_handle intr_handle; +struct rte_intr_handle err_intr_handle;struct mlx5_devx_obj *td;struct mlx5_devx_obj *tis;uint16_t nr_virtqs;
@@ -259,6 +264,25 @@ int mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv);*/ void mlx5_vdpa_cqe_event_unset (struct mlx5_vdpa_priv *priv); +/** +* Setup error interrupt handler. +* +* @param[in] priv +* The vdpa driver private structure. +* +* @return +* 0 on success, a negative errno value otherwise and rte_errno is set. +*/ +int mlx5_vdpa_err_event_setup (struct mlx5_vdpa_priv *priv); + +/** +* Unset error event handler. +* +* @param[in] priv +* The vdpa driver private structure. +*/ +void mlx5_vdpa_err_event_unset (struct mlx5_vdpa_priv *priv); +/** * Release a virtq and all its related resources. *
@@ -392,6 +416,19 @@ int mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state);*/ int mlx5_vdpa_virtq_stop (struct mlx5_vdpa_priv *priv,int index); +/** +* Query virtq information. +* +* @param[in] priv +* The vdpa driver private structure. +* @param[in] index +* The virtq index. +* +* @return +* 0 on success, a negative value otherwise. +*/ +int mlx5_vdpa_virtq_query (struct mlx5_vdpa_priv *priv,int index); +/** * Get virtq statistics. *
drivers/vdpa/mlx5/mlx5_vdpa_event.c +140 -0
157 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +46 -15
130 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH v2 1/2 common/mlx5: add virtq attributes error fields
In reply to Xueming LiAdd the needed fields for virtq DevX object to read the error state.
--- drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++ drivers/common/mlx5/mlx5_devx_cmds.h | 1 + drivers/common/mlx5/mlx5_prm.h | 9 +++++++-- 3 files changed, 11 insertions(+), 2 deletions(-)
drivers/common/mlx5/mlx5_devx_cmds.c +3 -0
@@ -1754,6 +1754,9 @@ mlx5_devx_cmd_query_virtq(struct mlx5_devx_obj *virtq_obj, attr->hw_available_index =MLX5_GET16 (virtio_net_q, virtq, hw_available_index); attr->hw_used_index =MLX5_GET16 (virtio_net_q, virtq, hw_used_index); + attr->state =MLX5_GET16 (virtio_net_q, virtq, state); + attr->error_type =MLX5_GET16 (virtio_net_q, virtq, + virtio_q_context.error_type);return ret; }
drivers/common/mlx5/mlx5_devx_cmds.h +1 -0
@@ -298,6 +298,7 @@ struct mlx5_devx_virtq_attr {uint32_t size;uint64_t offset; } umems[3 ]; +uint8_t error_type; };
drivers/common/mlx5/mlx5_prm.h +7 -2
@@ -2280,7 +2280,8 @@ struct mlx5_ifc_virtio_q_bits {u8 used_addr[0x40 ];u8 available_addr[0x40 ];u8 virtio_q_mkey[0x20 ]; -u8 reserved_at_160[0x20 ]; +u8 reserved_at_160[0x18 ]; +u8 error_type[0x8 ];u8 umem_1_id[0x20 ];u8 umem_1_size[0x20 ];u8 umem_1_offset[0x40 ];
@@ -2308,7 +2309,7 @@ struct mlx5_ifc_virtio_net_q_bits {u8 vhost_log_page[0x5 ];u8 reserved_at_90[0xc ];u8 state[0x4 ]; -u8 error_type[0x8 ]; +u8 reserved_at_a0[0x8 ];u8 tisn_or_qpn[0x18 ];u8 dirty_bitmap_mkey[0x20 ];u8 dirty_bitmap_size[0x20 ];
@@ -2329,6 +2330,10 @@ struct mlx5_ifc_query_virtq_out_bits {struct mlx5_ifc_virtio_net_q_bits virtq; }; +enum { +MLX5_EVENT_TYPE_OBJECT_CHANGE =0x27 , +}; +enum {MLX5_QP_ST_RC =0x0 , };
Signature · 1 line
2.25.1
PATCH v2 2/2 vdpa/mlx5: hardware error handling
When hardware error happens, vdpa didn't get such information and leave driver in silent: working state but no response.
This patch subscribes firmware virtq error event and try to recover max 3 times in 10 seconds, stop virtq if max retry number reached.
When error happens, PMD log in warning level. If failed to recover, outputs error log. Query virtq statistics to get error counters report.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 2 + drivers/vdpa/mlx5/mlx5_vdpa.h | 37 ++++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 142 ++++++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 61 +++++++++--- 4 files changed, 227 insertions(+), 15 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +2 -0
@@ -283,6 +283,7 @@ mlx5_vdpa_dev_close(int vid) }if (priv->configured) ret |=mlx5_vdpa_lm_log (priv); +mlx5_vdpa_err_event_unset (priv);mlx5_vdpa_cqe_event_unset (priv);mlx5_vdpa_steer_unset (priv);mlx5_vdpa_virtqs_release (priv);
@@ -318,6 +319,7 @@ mlx5_vdpa_dev_config(int vid)DRV_LOG (WARNING ,"MTU cannot be set on device %s." , vdev->device->name);if (mlx5_vdpa_pd_create (priv) ||mlx5_vdpa_mem_register (priv) || +mlx5_vdpa_err_event_setup (priv) ||mlx5_vdpa_virtqs_prepare (priv) ||mlx5_vdpa_steer_setup (priv) ||mlx5_vdpa_cqe_event_setup (priv)) {mlx5_vdpa_dev_close (vid);
drivers/vdpa/mlx5/mlx5_vdpa.h +37 -0
@@ -87,6 +87,7 @@ struct mlx5_vdpa_virtq {uint16_t vq_size;uint8_t notifier_state;bool stopped; +uint32_t version;struct mlx5_vdpa_priv *priv;struct mlx5_devx_obj *virtq;struct mlx5_devx_obj *counters;
@@ -97,6 +98,8 @@ struct mlx5_vdpa_virtq {uint32_t size; } umems[3 ];struct rte_intr_handle intr_handle; +uint64_t err_time[3 ];/* RDTSC time of recent errors. */ +uint32_t n_retry;struct mlx5_devx_virtio_q_couners_attr reset; };
@@ -143,8 +146,10 @@ struct mlx5_vdpa_priv {struct rte_vhost_memory *vmem;uint32_t eqn;struct mlx5dv_devx_event_channel *eventc; +struct mlx5dv_devx_event_channel *err_chnl;struct mlx5dv_devx_uar *uar;struct rte_intr_handle intr_handle; +struct rte_intr_handle err_intr_handle;struct mlx5_devx_obj *td;struct mlx5_devx_obj *tis;uint16_t nr_virtqs;
@@ -259,6 +264,25 @@ int mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv);*/ void mlx5_vdpa_cqe_event_unset (struct mlx5_vdpa_priv *priv); +/** +* Setup error interrupt handler. +* +* @param[in] priv +* The vdpa driver private structure. +* +* @return +* 0 on success, a negative errno value otherwise and rte_errno is set. +*/ +int mlx5_vdpa_err_event_setup (struct mlx5_vdpa_priv *priv); + +/** +* Unset error event handler. +* +* @param[in] priv +* The vdpa driver private structure. +*/ +void mlx5_vdpa_err_event_unset (struct mlx5_vdpa_priv *priv); +/** * Release a virtq and all its related resources. *
@@ -392,6 +416,19 @@ int mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state);*/ int mlx5_vdpa_virtq_stop (struct mlx5_vdpa_priv *priv,int index); +/** +* Query virtq information. +* +* @param[in] priv +* The vdpa driver private structure. +* @param[in] index +* The virtq index. +* +* @return +* 0 on success, a negative value otherwise. +*/ +int mlx5_vdpa_virtq_query (struct mlx5_vdpa_priv *priv,int index); +/** * Get virtq statistics. *
drivers/vdpa/mlx5/mlx5_vdpa_event.c +142 -0
159 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +46 -15
130 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH v3 1/2 common/mlx5: add virtq attributes error fields
In reply to Xueming LiAdd the needed fields for virtq DevX object to read the error state.
--- drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++ drivers/common/mlx5/mlx5_devx_cmds.h | 1 + drivers/common/mlx5/mlx5_prm.h | 9 +++++++-- 3 files changed, 11 insertions(+), 2 deletions(-)
drivers/common/mlx5/mlx5_devx_cmds.c +3 -0
@@ -1754,6 +1754,9 @@ mlx5_devx_cmd_query_virtq(struct mlx5_devx_obj *virtq_obj, attr->hw_available_index =MLX5_GET16 (virtio_net_q, virtq, hw_available_index); attr->hw_used_index =MLX5_GET16 (virtio_net_q, virtq, hw_used_index); + attr->state =MLX5_GET16 (virtio_net_q, virtq, state); + attr->error_type =MLX5_GET16 (virtio_net_q, virtq, + virtio_q_context.error_type);return ret; }
drivers/common/mlx5/mlx5_devx_cmds.h +1 -0
@@ -298,6 +298,7 @@ struct mlx5_devx_virtq_attr {uint32_t size;uint64_t offset; } umems[3 ]; +uint8_t error_type; };
drivers/common/mlx5/mlx5_prm.h +7 -2
@@ -2280,7 +2280,8 @@ struct mlx5_ifc_virtio_q_bits {u8 used_addr[0x40 ];u8 available_addr[0x40 ];u8 virtio_q_mkey[0x20 ]; -u8 reserved_at_160[0x20 ]; +u8 reserved_at_160[0x18 ]; +u8 error_type[0x8 ];u8 umem_1_id[0x20 ];u8 umem_1_size[0x20 ];u8 umem_1_offset[0x40 ];
@@ -2308,7 +2309,7 @@ struct mlx5_ifc_virtio_net_q_bits {u8 vhost_log_page[0x5 ];u8 reserved_at_90[0xc ];u8 state[0x4 ]; -u8 error_type[0x8 ]; +u8 reserved_at_a0[0x8 ];u8 tisn_or_qpn[0x18 ];u8 dirty_bitmap_mkey[0x20 ];u8 dirty_bitmap_size[0x20 ];
@@ -2329,6 +2330,10 @@ struct mlx5_ifc_query_virtq_out_bits {struct mlx5_ifc_virtio_net_q_bits virtq; }; +enum { +MLX5_EVENT_TYPE_OBJECT_CHANGE =0x27 , +}; +enum {MLX5_QP_ST_RC =0x0 , };
Signature · 1 line
2.25.1
PATCH v3 2/2 vdpa/mlx5: hardware error handling
When hardware error happens, vdpa didn't get such information and leave driver in silent: working state but no response.
This patch subscribes firmware virtq error event and try to recover max 3 times in 10 seconds, stop virtq if max retry number reached.
When error happens, PMD log in warning level. If failed to recover, outputs error log. Query virtq statistics to get error counters report.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 2 + drivers/vdpa/mlx5/mlx5_vdpa.h | 37 +++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 143 ++++++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 61 +++++++++--- 4 files changed, 228 insertions(+), 15 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +2 -0
@@ -283,6 +283,7 @@ mlx5_vdpa_dev_close(int vid) }if (priv->configured) ret |=mlx5_vdpa_lm_log (priv); +mlx5_vdpa_err_event_unset (priv);mlx5_vdpa_cqe_event_unset (priv);mlx5_vdpa_steer_unset (priv);mlx5_vdpa_virtqs_release (priv);
@@ -318,6 +319,7 @@ mlx5_vdpa_dev_config(int vid)DRV_LOG (WARNING ,"MTU cannot be set on device %s." , vdev->device->name);if (mlx5_vdpa_pd_create (priv) ||mlx5_vdpa_mem_register (priv) || +mlx5_vdpa_err_event_setup (priv) ||mlx5_vdpa_virtqs_prepare (priv) ||mlx5_vdpa_steer_setup (priv) ||mlx5_vdpa_cqe_event_setup (priv)) {mlx5_vdpa_dev_close (vid);
drivers/vdpa/mlx5/mlx5_vdpa.h +37 -0
@@ -87,6 +87,7 @@ struct mlx5_vdpa_virtq {uint16_t vq_size;uint8_t notifier_state;bool stopped; +uint32_t version;struct mlx5_vdpa_priv *priv;struct mlx5_devx_obj *virtq;struct mlx5_devx_obj *counters;
@@ -97,6 +98,8 @@ struct mlx5_vdpa_virtq {uint32_t size; } umems[3 ];struct rte_intr_handle intr_handle; +uint64_t err_time[3 ];/* RDTSC time of recent errors. */ +uint32_t n_retry;struct mlx5_devx_virtio_q_couners_attr reset; };
@@ -143,8 +146,10 @@ struct mlx5_vdpa_priv {struct rte_vhost_memory *vmem;uint32_t eqn;struct mlx5dv_devx_event_channel *eventc; +struct mlx5dv_devx_event_channel *err_chnl;struct mlx5dv_devx_uar *uar;struct rte_intr_handle intr_handle; +struct rte_intr_handle err_intr_handle;struct mlx5_devx_obj *td;struct mlx5_devx_obj *tis;uint16_t nr_virtqs;
@@ -259,6 +264,25 @@ int mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv);*/ void mlx5_vdpa_cqe_event_unset (struct mlx5_vdpa_priv *priv); +/** +* Setup error interrupt handler. +* +* @param[in] priv +* The vdpa driver private structure. +* +* @return +* 0 on success, a negative errno value otherwise and rte_errno is set. +*/ +int mlx5_vdpa_err_event_setup (struct mlx5_vdpa_priv *priv); + +/** +* Unset error event handler. +* +* @param[in] priv +* The vdpa driver private structure. +*/ +void mlx5_vdpa_err_event_unset (struct mlx5_vdpa_priv *priv); +/** * Release a virtq and all its related resources. *
@@ -392,6 +416,19 @@ int mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state);*/ int mlx5_vdpa_virtq_stop (struct mlx5_vdpa_priv *priv,int index); +/** +* Query virtq information. +* +* @param[in] priv +* The vdpa driver private structure. +* @param[in] index +* The virtq index. +* +* @return +* 0 on success, a negative value otherwise. +*/ +int mlx5_vdpa_virtq_query (struct mlx5_vdpa_priv *priv,int index); +/** * Get virtq statistics. *
drivers/vdpa/mlx5/mlx5_vdpa_event.c +143 -0
160 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +46 -15
130 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH v4 1/2 common/mlx5: add virtq attributes error fields
In reply to Xueming LiAdd the needed fields for virtq DevX object to read the error state.
--- drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++ drivers/common/mlx5/mlx5_devx_cmds.h | 1 + drivers/common/mlx5/mlx5_prm.h | 9 +++++++-- 3 files changed, 11 insertions(+), 2 deletions(-)
drivers/common/mlx5/mlx5_devx_cmds.c +3 -0
@@ -1754,6 +1754,9 @@ mlx5_devx_cmd_query_virtq(struct mlx5_devx_obj *virtq_obj, attr->hw_available_index =MLX5_GET16 (virtio_net_q, virtq, hw_available_index); attr->hw_used_index =MLX5_GET16 (virtio_net_q, virtq, hw_used_index); + attr->state =MLX5_GET16 (virtio_net_q, virtq, state); + attr->error_type =MLX5_GET16 (virtio_net_q, virtq, + virtio_q_context.error_type);return ret; }
drivers/common/mlx5/mlx5_devx_cmds.h +1 -0
@@ -298,6 +298,7 @@ struct mlx5_devx_virtq_attr {uint32_t size;uint64_t offset; } umems[3 ]; +uint8_t error_type; };
drivers/common/mlx5/mlx5_prm.h +7 -2
@@ -2280,7 +2280,8 @@ struct mlx5_ifc_virtio_q_bits {u8 used_addr[0x40 ];u8 available_addr[0x40 ];u8 virtio_q_mkey[0x20 ]; -u8 reserved_at_160[0x20 ]; +u8 reserved_at_160[0x18 ]; +u8 error_type[0x8 ];u8 umem_1_id[0x20 ];u8 umem_1_size[0x20 ];u8 umem_1_offset[0x40 ];
@@ -2308,7 +2309,7 @@ struct mlx5_ifc_virtio_net_q_bits {u8 vhost_log_page[0x5 ];u8 reserved_at_90[0xc ];u8 state[0x4 ]; -u8 error_type[0x8 ]; +u8 reserved_at_a0[0x8 ];u8 tisn_or_qpn[0x18 ];u8 dirty_bitmap_mkey[0x20 ];u8 dirty_bitmap_size[0x20 ];
@@ -2329,6 +2330,10 @@ struct mlx5_ifc_query_virtq_out_bits {struct mlx5_ifc_virtio_net_q_bits virtq; }; +enum { +MLX5_EVENT_TYPE_OBJECT_CHANGE =0x27 , +}; +enum {MLX5_QP_ST_RC =0x0 , };
Signature · 1 line
2.25.1
PATCH v4 2/2 vdpa/mlx5: hardware error handling
When hardware error happens, vdpa didn't get such information and leave driver in silent: working state but no response.
This patch subscribes firmware virtq error event and try to recover max 3 times in 3 seconds, stop virtq if max retry number reached.
When error happens, PMD log in warning level. If failed to recover, outputs error log. Query virtq statistics to get error counters report.
--- doc/guides/rel_notes/release_20_11.rst | 6 +- doc/guides/vdpadevs/mlx5.rst | 8 ++ drivers/vdpa/mlx5/mlx5_vdpa.c | 2 + drivers/vdpa/mlx5/mlx5_vdpa.h | 37 +++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 144 +++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 61 ++++++++--- 6 files changed, 242 insertions(+), 16 deletions(-)
doc/guides/rel_notes/release_20_11.rst +5 -1
@@ -348,7 +348,7 @@ New Features * **Updated Mellanox mlx5 driver.** - Updated Mellanox mlx5 driver with new features and improvements, including: + Updated Mellanox mlx5 net driver with new features and improvements, including: * Added support for matching on fragmented/non-fragmented IPv4/IPv6 packets. * Updated the supported timeout for Age action to the maximal value supported
@@ -359,6 +359,10 @@ New Features * Added support for the new vlan fields ``has_vlan`` in the eth item and ``has_more_vlan`` in the vlan item. + Updated Mellanox mlx5 vDPA driver: + + * Added support of vDPA VirtQ error handling. + * **Updated vhost sample application.** Added vhost asynchronous APIs support, which demonstrated how the application
doc/guides/vdpadevs/mlx5.rst +8 -0
@@ -134,3 +134,11 @@ Driver options driver to no-traffic mode. In this mode the timer events are stopped and interrupts are configured to the device in order to notify traffic for the driver. Default value is 2s. + +Error handling +^^^^^^^^^^^^^^ + +Upon potential hardware errors, mlx5 PMD try to recover, give up if failed 3 +times in 3 seconds, virtq will be put in disable state. User should check log +to get error information, or query vdpa statistics counter to know error type +and count report.
drivers/vdpa/mlx5/mlx5_vdpa.c +2 -0
@@ -283,6 +283,7 @@ mlx5_vdpa_dev_close(int vid) }if (priv->configured) ret |=mlx5_vdpa_lm_log (priv); +mlx5_vdpa_err_event_unset (priv);mlx5_vdpa_cqe_event_unset (priv);mlx5_vdpa_steer_unset (priv);mlx5_vdpa_virtqs_release (priv);
@@ -318,6 +319,7 @@ mlx5_vdpa_dev_config(int vid)DRV_LOG (WARNING ,"MTU cannot be set on device %s." , vdev->device->name);if (mlx5_vdpa_pd_create (priv) ||mlx5_vdpa_mem_register (priv) || +mlx5_vdpa_err_event_setup (priv) ||mlx5_vdpa_virtqs_prepare (priv) ||mlx5_vdpa_steer_setup (priv) ||mlx5_vdpa_cqe_event_setup (priv)) {mlx5_vdpa_dev_close (vid);
drivers/vdpa/mlx5/mlx5_vdpa.h +37 -0
@@ -87,6 +87,7 @@ struct mlx5_vdpa_virtq {uint16_t vq_size;uint8_t notifier_state;bool stopped; +uint32_t version;struct mlx5_vdpa_priv *priv;struct mlx5_devx_obj *virtq;struct mlx5_devx_obj *counters;
@@ -97,6 +98,8 @@ struct mlx5_vdpa_virtq {uint32_t size; } umems[3 ];struct rte_intr_handle intr_handle; +uint64_t err_time[3 ];/* RDTSC time of recent errors. */ +uint32_t n_retry;struct mlx5_devx_virtio_q_couners_attr reset; };
@@ -143,8 +146,10 @@ struct mlx5_vdpa_priv {struct rte_vhost_memory *vmem;uint32_t eqn;struct mlx5dv_devx_event_channel *eventc; +struct mlx5dv_devx_event_channel *err_chnl;struct mlx5dv_devx_uar *uar;struct rte_intr_handle intr_handle; +struct rte_intr_handle err_intr_handle;struct mlx5_devx_obj *td;struct mlx5_devx_obj *tis;uint16_t nr_virtqs;
@@ -259,6 +264,25 @@ int mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv);*/ void mlx5_vdpa_cqe_event_unset (struct mlx5_vdpa_priv *priv); +/** +* Setup error interrupt handler. +* +* @param[in] priv +* The vdpa driver private structure. +* +* @return +* 0 on success, a negative errno value otherwise and rte_errno is set. +*/ +int mlx5_vdpa_err_event_setup (struct mlx5_vdpa_priv *priv); + +/** +* Unset error event handler. +* +* @param[in] priv +* The vdpa driver private structure. +*/ +void mlx5_vdpa_err_event_unset (struct mlx5_vdpa_priv *priv); +/** * Release a virtq and all its related resources. *
@@ -392,6 +416,19 @@ int mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state);*/ int mlx5_vdpa_virtq_stop (struct mlx5_vdpa_priv *priv,int index); +/** +* Query virtq information. +* +* @param[in] priv +* The vdpa driver private structure. +* @param[in] index +* The virtq index. +* +* @return +* 0 on success, a negative value otherwise. +*/ +int mlx5_vdpa_virtq_query (struct mlx5_vdpa_priv *priv,int index); +/** * Get virtq statistics. *
drivers/vdpa/mlx5/mlx5_vdpa_event.c +144 -0
161 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +46 -15
130 lines not on this page · raw message
Signature · 1 line
2.25.1
Re: [dpdk-dev] [PATCH v4 2/2] vdpa/mlx5: hardware error handling
Quoted 11 lines · Xueming Li
When hardware error happens, vdpa didn't get such information and leave driver in silent: working state but no response.
This patch subscribes firmware virtq error event and try to recover max 3 times in 3 seconds, stop virtq if max retry number reached.
When error happens, PMD log in warning level. If failed to recover, outputs error log. Query virtq statistics to get error counters report.
Acked-by Matan Azrad <[email protected]>Signed-off-by Xueming Li <[email protected]>
Quoted 9 lines
--- doc/guides/rel_notes/release_20_11.rst | 6 +- doc/guides/vdpadevs/mlx5.rst | 8 ++ drivers/vdpa/mlx5/mlx5_vdpa.c | 2 + drivers/vdpa/mlx5/mlx5_vdpa.h | 37 +++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 144 +++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 61 ++++++++--- 6 files changed, 242 insertions(+), 16 deletions(-)
Thanks,
Maxime
Maxime Coquelin
AppliedRe: [dpdk-dev] [PATCH v4 2/2] vdpa/mlx5: hardware error handling
In reply to Xueming LiQuoted 11 lines · Xueming Li
When hardware error happens, vdpa didn't get such information and leave driver in silent: working state but no response.
This patch subscribes firmware virtq error event and try to recover max 3 times in 3 seconds, stop virtq if max retry number reached.
When error happens, PMD log in warning level. If failed to recover, outputs error log. Query virtq statistics to get error counters report.
Acked-by Matan Azrad <[email protected]>Signed-off-by Xueming Li <[email protected]>
Quoted 9 lines
--- doc/guides/rel_notes/release_20_11.rst | 6 +- doc/guides/vdpadevs/mlx5.rst | 8 ++ drivers/vdpa/mlx5/mlx5_vdpa.c | 2 + drivers/vdpa/mlx5/mlx5_vdpa.h | 37 +++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 144 +++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 61 ++++++++--- 6 files changed, 242 insertions(+), 16 deletions(-)
Applied to dpdk-next-virtio/main.
Thanks!
Maxime
Quoted 10 lines · Xueming Li
Add the needed fields for virtq DevX object to read the error state.
Acked-by Matan Azrad <[email protected]>Signed-off-by Xueming Li <[email protected]>--- drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++ drivers/common/mlx5/mlx5_devx_cmds.h | 1 + drivers/common/mlx5/mlx5_prm.h | 9 +++++++-- 3 files changed, 11 insertions(+), 2 deletions(-)
Thanks,
Maxime
Maxime Coquelin
AppliedQuoted 9 lines · Xueming Li
Add the needed fields for virtq DevX object to read the error state.
Acked-by Matan Azrad <[email protected]>Signed-off-by Xueming Li <[email protected]>--- drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++ drivers/common/mlx5/mlx5_devx_cmds.h | 1 + drivers/common/mlx5/mlx5_prm.h | 9 +++++++-- 3 files changed, 11 insertions(+), 2 deletions(-)
Applied to dpdk-next-virtio/main.
Thanks!
Maxime