Add vDPA multi-threads optiomization
PATCH v3 6/15 vdpa/mlx5: pre-create virtq at probe time
In reply to Li Zhangdev_config operation is called in LM progress. LM time is very critical because all
the VM packets are dropped directly at that time.
Move the virtq creation to probe time and only modify the configuration later in the dev_config stage using the new ability to modify virtq.
This optimization accelerates the LM process and reduces its time by 70%.
--- doc/guides/rel_notes/release_22_07.rst | 4 + drivers/vdpa/mlx5/mlx5_vdpa.h | 4 + drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 19 +- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 257 +++++++++++++++---------- 4 files changed, 176 insertions(+), 108 deletions(-)
doc/guides/rel_notes/release_22_07.rst +4 -0
@@ -175,6 +175,10 @@ New Features This is a fall-back implementation for platforms that don't support vector operations. +* **Updated Nvidia mlx5 vDPA driver.** + + * Added new devargs ``queue_size`` and ``queues`` to allow prior creation of virtq resources. + Removed Items -------------
drivers/vdpa/mlx5/mlx5_vdpa.h +4 -0
@@ -80,6 +80,7 @@ struct mlx5_vdpa_virtq {uint16_t vq_size;uint8_t notifier_state;bool stopped; + uint32_t configured:1;uint32_t version;struct mlx5_vdpa_priv *priv;struct mlx5_devx_obj *virtq;
@@ -489,4 +490,7 @@ mlx5_vdpa_virtq_stats_reset(struct mlx5_vdpa_priv *priv, int qid);*/ void mlx5_vdpa_drain_cq (struct mlx5_vdpa_priv *priv); + +bool +mlx5_vdpa_is_modify_virtq_supported (struct mlx5_vdpa_priv *priv);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_lm.c +11 -8
@@ -12,20 +12,21 @@ int mlx5_vdpa_logging_enable(struct mlx5_vdpa_priv *priv, int enable) {struct mlx5_devx_virtq_attr attr = { - .type =MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_DUMP_ENABLE , + .mod_fields_bitmap = +MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_DUMP_ENABLE , .dirty_bitmap_dump_enable = enable, }; +struct mlx5_vdpa_virtq *virtq;int i;for (i =0 ; i < priv->nr_virtqs; ++i) { attr.queue_index = i; -if (!priv->virtqs[i].virtq) { -DRV_LOG (DEBUG ,"virtq %d is invalid for dirty bitmap " -"enabling." , i); + virtq = &priv->virtqs[i]; +if (!virtq->configured) { +DRV_LOG (DEBUG ,"virtq %d is invalid for dirty bitmap enabling." , i); }else if (mlx5_devx_cmd_modify_virtq (priv->virtqs[i].virtq, &attr)) { -DRV_LOG (ERR ,"Failed to modify virtq %d for dirty " -"bitmap enabling." , i); +DRV_LOG (ERR ,"Failed to modify virtq %d for dirty bitmap enabling." , i);return -1 ; } }
@@ -37,10 +38,11 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base, uint64_t log_size) {struct mlx5_devx_virtq_attr attr = { - .type =MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS , + .mod_fields_bitmap =MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS , .dirty_bitmap_addr = log_base, .dirty_bitmap_size = log_size, }; +struct mlx5_vdpa_virtq *virtq;int i; int ret = mlx5_os_wrapped_mkey_create(priv->cdev->ctx, priv->cdev->pd, priv->cdev->pdn,
@@ -54,7 +56,8 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base, attr.dirty_bitmap_mkey = priv->lm_mr.lkey;for (i =0 ; i < priv->nr_virtqs; ++i) { attr.queue_index = i; -if (!priv->virtqs[i].virtq) { + virtq = &priv->virtqs[i]; +if (!virtq->configured) {DRV_LOG (DEBUG ,"virtq %d is invalid for LM." , i); }else if (mlx5_devx_cmd_modify_virtq (priv->virtqs[i].virtq, &attr)) {
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +157 -100
347 lines not on this page · raw message
Signature · 1 line
2.31.1
PATCH v3 7/15 vdpa/mlx5: optimize datapath-control synchronization
In reply to Li ZhangThe driver used a single global lock for any synchronization needed for the datapath and control path.
It is better to group the critical sections with the other ones that should be synchronized.
Replace the global lock with the following locks:
1.virtq locks(per virtq) synchronize datapath polling and
parallel configurations on the same virtq.
2.A doorbell lock synchronizes doorbell update,
which is shared for all the virtqs in the device.
3.A steering lock for the shared steering objects updates.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 24 ++++--- drivers/vdpa/mlx5/mlx5_vdpa.h | 13 ++-- drivers/vdpa/mlx5/mlx5_vdpa_event.c | 97 ++++++++++++++++++----------- drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 34 +++++++--- drivers/vdpa/mlx5/mlx5_vdpa_steer.c | 7 ++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 88 +++++++++++++++++++------- 6 files changed, 184 insertions(+), 79 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +16 -8
@@ -135,6 +135,7 @@ mlx5_vdpa_set_vring_state(int vid, int vring, int state)struct rte_vdpa_device *vdev =rte_vhost_get_vdpa_device (vid);struct mlx5_vdpa_priv *priv =mlx5_vdpa_find_priv_resource_by_vdev (vdev); +struct mlx5_vdpa_virtq *virtq;int ret;if (priv ==NULL ) {
@@ -145,9 +146,10 @@ mlx5_vdpa_set_vring_state(int vid, int vring, int state)DRV_LOG (ERR ,"Too big vring id: %d." , vring);return -E2BIG ; } -pthread_mutex_lock (&priv->vq_config_lock); + virtq = &priv->virtqs[vring]; +pthread_mutex_lock (&virtq->virtq_lock); ret =mlx5_vdpa_virtq_enable (priv, vring, state); -pthread_mutex_unlock (&priv->vq_config_lock); +pthread_mutex_unlock (&virtq->virtq_lock);return ret; }
@@ -267,7 +269,9 @@ mlx5_vdpa_dev_close(int vid) ret |=mlx5_vdpa_lm_log (priv); priv->state =MLX5_VDPA_STATE_IN_PROGRESS ; } +pthread_mutex_lock (&priv->steer_update_lock);mlx5_vdpa_steer_unset (priv); +pthread_mutex_unlock (&priv->steer_update_lock);mlx5_vdpa_virtqs_release (priv);mlx5_vdpa_drain_cq (priv);if (priv->lm_mr.addr)
@@ -276,8 +280,6 @@ mlx5_vdpa_dev_close(int vid)if (!priv->connected)mlx5_vdpa_dev_cache_clean (priv); priv->vid =0 ; -/* The mutex may stay locked after event thread cancel - initiate it. */ -pthread_mutex_init (&priv->vq_config_lock,NULL );DRV_LOG (INFO ,"vDPA device %d was closed." , vid);return ret; }
@@ -549,15 +551,21 @@ mlx5_vdpa_config_get(struct mlx5_kvargs_ctrl *mkvlist,static int mlx5_vdpa_virtq_resource_prepare (struct mlx5_vdpa_priv *priv) { +struct mlx5_vdpa_virtq *virtq;uint32_t index;uint32_t i; +for (index =0 ; index < priv->caps.max_num_virtio_queues *2 ; + index++) { + virtq = &priv->virtqs[index]; +pthread_mutex_init (&virtq->virtq_lock,NULL ); + }if (!priv->queues)return 0 ;for (index =0 ; index < (priv->queues *2 ); ++index) { -struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index]; + virtq = &priv->virtqs[index];int ret =mlx5_vdpa_event_qp_prepare (priv, priv->queue_size, --1 , &virtq->eqp); +-1 , virtq);if (ret) { DRV_LOG(ERR, "Failed to create event QPs for virtq %d.",
@@ -713,7 +721,8 @@ mlx5_vdpa_dev_probe(struct mlx5_common_device *cdev, priv->num_lag_ports = attr->num_lag_ports;if (attr->num_lag_ports ==0 ) priv->num_lag_ports =1 ; -pthread_mutex_init (&priv->vq_config_lock,NULL ); +rte_spinlock_init (&priv->db_lock); +pthread_mutex_init (&priv->steer_update_lock,NULL ); priv->cdev = cdev;mlx5_vdpa_config_get (mkvlist, priv);if (mlx5_vdpa_create_dev_resources (priv))
@@ -797,7 +806,6 @@ mlx5_vdpa_dev_release(struct mlx5_vdpa_priv *priv)mlx5_vdpa_release_dev_resources (priv);if (priv->vdev)rte_vdpa_unregister_device (priv->vdev); -pthread_mutex_destroy (&priv->vq_config_lock);rte_free (priv); }
drivers/vdpa/mlx5/mlx5_vdpa.h +8 -5
@@ -82,6 +82,7 @@ struct mlx5_vdpa_virtq {bool stopped; uint32_t configured:1;uint32_t version; +pthread_mutex_t virtq_lock;struct mlx5_vdpa_priv *priv;struct mlx5_devx_obj *virtq;struct mlx5_devx_obj *counters;
@@ -126,7 +127,8 @@ struct mlx5_vdpa_priv {TAILQ_ENTRY (mlx5_vdpa_priv) next;bool connected;enum mlx5_dev_state state; -pthread_mutex_t vq_config_lock; +rte_spinlock_t db_lock; +pthread_mutex_t steer_update_lock;uint64_t no_traffic_counter;pthread_t timer_tid;int event_mode;
@@ -222,14 +224,15 @@ int mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv);* Number of descriptors. * @param[in] callfd * The guest notification file descriptor. -* @param[in/out] eqp -* Pointer to the event QP structure. +* @param[in/out] virtq +* Pointer to the virt-queue structure. * * @return * 0 on success, -1 otherwise and rte_errno is set. */ -int mlx5_vdpa_event_qp_prepare (struct mlx5_vdpa_priv *priv,uint16_t desc_n, -int callfd,struct mlx5_vdpa_event_qp *eqp); +int +mlx5_vdpa_event_qp_prepare (struct mlx5_vdpa_priv *priv,uint16_t desc_n, +int callfd,struct mlx5_vdpa_virtq *virtq);/** * Destroy an event QP and all its related resources.
drivers/vdpa/mlx5/mlx5_vdpa_event.c +62 -35
232 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_lm.c +27 -7
@@ -24,10 +24,17 @@ mlx5_vdpa_logging_enable(struct mlx5_vdpa_priv *priv, int enable) virtq = &priv->virtqs[i];if (!virtq->configured) {DRV_LOG (DEBUG ,"virtq %d is invalid for dirty bitmap enabling." , i); - }else if (mlx5_devx_cmd_modify_virtq (priv->virtqs[i].virtq, + }else { +struct mlx5_vdpa_virtq *virtq = &priv->virtqs[i]; + +pthread_mutex_lock (&virtq->virtq_lock); +if (mlx5_devx_cmd_modify_virtq (priv->virtqs[i].virtq, &attr)) { -DRV_LOG (ERR ,"Failed to modify virtq %d for dirty bitmap enabling." , i); -return -1 ; +pthread_mutex_unlock (&virtq->virtq_lock); +DRV_LOG (ERR ,"Failed to modify virtq %d for dirty bitmap enabling." , i); +return -1 ; + } +pthread_mutex_unlock (&virtq->virtq_lock); } }return 0 ;
@@ -59,10 +66,19 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base, virtq = &priv->virtqs[i];if (!virtq->configured) {DRV_LOG (DEBUG ,"virtq %d is invalid for LM." , i); - }else if (mlx5_devx_cmd_modify_virtq (priv->virtqs[i].virtq, - &attr)) { -DRV_LOG (ERR ,"Failed to modify virtq %d for LM." , i); - goto err; + }else { +struct mlx5_vdpa_virtq *virtq = &priv->virtqs[i]; + +pthread_mutex_lock (&virtq->virtq_lock); +if (mlx5_devx_cmd_modify_virtq ( + priv->virtqs[i].virtq, + &attr)) { +pthread_mutex_unlock (&virtq->virtq_lock); +DRV_LOG (ERR , +"Failed to modify virtq %d for LM." , i); + goto err; + } +pthread_mutex_unlock (&virtq->virtq_lock); } }return 0 ;
@@ -77,6 +93,7 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,int mlx5_vdpa_lm_log (struct mlx5_vdpa_priv *priv) { +struct mlx5_vdpa_virtq *virtq;uint64_t features;int ret =rte_vhost_get_negotiated_features (priv->vid, &features);int i;
@@ -88,10 +105,13 @@ mlx5_vdpa_lm_log(struct mlx5_vdpa_priv *priv)if (!RTE_VHOST_NEED_LOG (features))return 0 ;for (i =0 ; i < priv->nr_virtqs; ++i) { + virtq = &priv->virtqs[i];if (!priv->virtqs[i].virtq) {DRV_LOG (DEBUG ,"virtq %d is invalid for LM log." , i); }else { +pthread_mutex_lock (&virtq->virtq_lock); ret =mlx5_vdpa_virtq_stop (priv, i); +pthread_mutex_unlock (&virtq->virtq_lock);if (ret) {DRV_LOG (ERR ,"Failed to stop virtq %d for LM " "log." , i);
drivers/vdpa/mlx5/mlx5_vdpa_steer.c +6 -1
@@ -237,19 +237,24 @@ mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv *priv)int mlx5_vdpa_steer_update (struct mlx5_vdpa_priv *priv) { -int ret =mlx5_vdpa_rqt_prepare (priv); +int ret; +pthread_mutex_lock (&priv->steer_update_lock); + ret =mlx5_vdpa_rqt_prepare (priv);if (ret ==0 ) {mlx5_vdpa_steer_unset (priv); }else if (ret <0 ) { +pthread_mutex_unlock (&priv->steer_update_lock);return ret; }else if (!priv->steer.rss[0 ].flow) { ret =mlx5_vdpa_rss_flows_create (priv);if (ret) {DRV_LOG (ERR ,"Cannot create RSS flows." ); +pthread_mutex_unlock (&priv->steer_update_lock);return -1 ; } } +pthread_mutex_unlock (&priv->steer_update_lock);return 0 ; }
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +65 -23
173 lines not on this page · raw message
Signature · 1 line
2.31.1
PATCH v3 8/15 vdpa/mlx5: add multi-thread management for configuration
In reply to Li ZhangThe LM process includes a lot of objects creations and destructions in the source and the destination servers. As much as LM time increases, the packet drop of the VM increases. To improve LM time need to parallel the configurations for mlx5 FW. Add internal multi-thread management in the driver for it.
A new devarg defines the number of threads and their CPU. The management is shared between all the devices of the driver. Since the event_core also affects the datapath events thread, reduce the priority of the datapath event thread to allow fast configuration of the devices doing the LM.
--- doc/guides/vdpadevs/mlx5.rst | 11 +++ drivers/vdpa/mlx5/meson.build | 1 + drivers/vdpa/mlx5/mlx5_vdpa.c | 41 ++++++++ drivers/vdpa/mlx5/mlx5_vdpa.h | 36 +++++++ drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 129 ++++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 2 +- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 8 +- 7 files changed, 223 insertions(+), 5 deletions(-) create mode 100644 drivers/vdpa/mlx5/mlx5_vdpa_cthread.c
doc/guides/vdpadevs/mlx5.rst +11 -0
@@ -78,6 +78,17 @@ for an additional list of options shared with other mlx5 drivers. CPU core number to set polling thread affinity to, default to control plane cpu. +- ``max_conf_threads`` parameter [int] + + Allow the driver to use internal threads to obtain fast configuration. + All the threads will be open on the same core of the event completion queue scheduling thread. + + - 0, default, don't use internal threads for configuration. + + - 1 - 256, number of internal threads in addition to the caller thread (8 is suggested). + This value, if not 0, should be the same for all the devices; + the first prob will take it with the event_core for all the multi-thread configurations in the driver. + - ``hw_latency_mode`` parameter [int] The completion queue moderation mode:
drivers/vdpa/mlx5/meson.build +1 -0
@@ -15,6 +15,7 @@ sources = files( 'mlx5_vdpa_virtq.c', 'mlx5_vdpa_steer.c', 'mlx5_vdpa_lm.c', + 'mlx5_vdpa_cthread.c', ) cflags_options = [ '-std=c11',
drivers/vdpa/mlx5/mlx5_vdpa.c +41 -0
77 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa.h +36 -0
@@ -73,6 +73,22 @@ enum {MLX5_VDPA_NOTIFIER_STATE_ERR }; +#define MLX5_VDPA_MAX_C_THRD 256 + +/* Generic mlx5_vdpa_c_thread information. */ +struct mlx5_vdpa_c_thread { +pthread_t tid; +}; + +struct mlx5_vdpa_conf_thread_mng { +void *initializer_priv; +uint32_t refcnt; +uint32_t max_thrds; +pthread_mutex_t cthrd_lock; +struct mlx5_vdpa_c_thread cthrd[MLX5_VDPA_MAX_C_THRD ]; +}; +extern struct mlx5_vdpa_conf_thread_mng conf_thread_mng; +struct mlx5_vdpa_virtq {SLIST_ENTRY (mlx5_vdpa_virtq) next;uint8_t enable;
@@ -126,6 +142,7 @@ enum mlx5_dev_state {struct mlx5_vdpa_priv {TAILQ_ENTRY (mlx5_vdpa_priv) next;bool connected; +bool use_c_thread;enum mlx5_dev_state state;rte_spinlock_t db_lock;pthread_mutex_t steer_update_lock;
@@ -496,4 +513,23 @@ mlx5_vdpa_drain_cq(struct mlx5_vdpa_priv *priv);bool mlx5_vdpa_is_modify_virtq_supported (struct mlx5_vdpa_priv *priv); + +/** +* Create configuration multi-threads resource +* +* @param[in] cpu_core +* CPU core number to set configuration threads affinity to. +* +* @return +* 0 on success, a negative value otherwise. +*/ +int +mlx5_vdpa_mult_threads_create (int cpu_core); + +/** +* Destroy configuration multi-threads resource +* +*/ +void +mlx5_vdpa_mult_threads_destroy (bool need_unlock);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c new file +129 -0
129 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_event.c +1 -1
@@ -507,7 +507,7 @@ mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv)pthread_attr_t attr;char name[16 ];const struct sched_param sp = { - .sched_priority =sched_get_priority_max (SCHED_RR ), + .sched_priority =sched_get_priority_max (SCHED_RR ) -1 , };if (!priv->eventc)
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +4 -4
@@ -43,7 +43,7 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg) errno ==EWOULDBLOCK || errno == EAGAIN) continue; -DRV_LOG (ERR ,"Failed to read kickfd of virtq %d: %s" , +DRV_LOG (ERR ,"Failed to read kickfd of virtq %d: %s." , virtq->index,strerror (errno)); }break ;
@@ -57,7 +57,7 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg)rte_spinlock_unlock (&priv->db_lock);pthread_mutex_unlock (&virtq->virtq_lock);if (priv->state !=MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) { -DRV_LOG (ERR ,"device %d queue %d down, skip kick handling" , +DRV_LOG (ERR ,"device %d queue %d down, skip kick handling." , priv->vid, virtq->index);return ; }
@@ -218,7 +218,7 @@ mlx5_vdpa_virtq_query(struct mlx5_vdpa_priv *priv, int index)return -1 ; }if (attr.state ==MLX5_VIRTQ_STATE_ERROR ) -DRV_LOG (WARNING ,"vid %d vring %d hw error=%hhu" , +DRV_LOG (WARNING ,"vid %d vring %d hw error=%hhu." , priv->vid, index, attr.error_type);return 0 ; }
@@ -380,7 +380,7 @@ mlx5_vdpa_virtq_sub_objs_prepare(struct mlx5_vdpa_priv *priv,if (ret) { last_avail_idx =0 ; last_used_idx =0 ; -DRV_LOG (WARNING ,"Couldn't get vring base, idx are set to 0" ); +DRV_LOG (WARNING ,"Couldn't get vring base, idx are set to 0." ); }else { DRV_LOG(INFO, "vid %d: Init last_avail_idx=%d, last_used_idx=%d for " "virtq %d.", priv->vid, last_avail_idx,
Signature · 1 line
2.31.1
PATCH v3 9/15 vdpa/mlx5: add task ring for MT management
In reply to Li ZhangThe configuration threads tasks need a container to support multiple tasks assigned to a thread in parallel. Use rte_ring container per thread to manage
the thread tasks without locks.
The caller thread from the user context opens a task to a thread and enqueue it to the thread ring.
The thread polls its ring and dequeue tasks. That’s why the ring should be in multi-producer and single consumer mode.
Anatomic counter manages the tasks completion notification. The threads report errors to the caller by
a dedicated error counter per task.
--- drivers/vdpa/mlx5/mlx5_vdpa.h | 17 ++++ drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 115 +++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 2 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.h +17 -0
@@ -74,10 +74,22 @@ enum { };#define MLX5_VDPA_MAX_C_THRD 256 +#define MLX5_VDPA_MAX_TASKS_PER_THRD 4096 +#define MLX5_VDPA_TASKS_PER_DEV 64 + +/* Generic task information and size must be multiple of 4B. */ +struct mlx5_vdpa_task { +struct mlx5_vdpa_priv *priv; +uint32_t *remaining_cnt; +uint32_t *err_cnt; +uint32_t idx; +} __rte_packed __rte_aligned(4);/* Generic mlx5_vdpa_c_thread information. */ struct mlx5_vdpa_c_thread {pthread_t tid; +struct rte_ring *rng; +pthread_cond_t c_cond; };struct mlx5_vdpa_conf_thread_mng {
@@ -532,4 +544,9 @@ mlx5_vdpa_mult_threads_create(int cpu_core);*/ void mlx5_vdpa_mult_threads_destroy (bool need_unlock); + +bool +mlx5_vdpa_task_add (struct mlx5_vdpa_priv *priv, +uint32_t thrd_idx, +uint32_t num);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +113 -2
156 lines not on this page · raw message
Signature · 1 line
2.31.1
PATCH v3 10/15 vdpa/mlx5: add MT task for VM memory registration
In reply to Li ZhangThe driver creates a direct MR object of
the HW for each VM memory region,
which maps the VM physical address to
the actual physical address.
Later, after all the MRs are ready,
the driver creates an indirect MR to group all the direct MRs into one virtual space from the HW perspective.
Create direct MRs in parallel using the MT mechanism. After completion, the primary thread creates the indirect MR needed for the following virtqs configurations.
This optimization accelerrate the LM process and
reduce its time by 5%.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 1 - drivers/vdpa/mlx5/mlx5_vdpa.h | 31 ++- drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 47 ++++- drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 270 ++++++++++++++++++-------- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 6 +- 5 files changed, 258 insertions(+), 97 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +0 -1
@@ -768,7 +768,6 @@ mlx5_vdpa_dev_probe(struct mlx5_common_device *cdev, rte_errno = rte_errno ? rte_errno :EINVAL ; goto error; } -SLIST_INIT (&priv->mr_list);pthread_mutex_lock (&priv_list_lock);TAILQ_INSERT_TAIL (&priv_list, priv, next);pthread_mutex_unlock (&priv_list_lock);
drivers/vdpa/mlx5/mlx5_vdpa.h +27 -4
@@ -59,7 +59,6 @@ struct mlx5_vdpa_event_qp { };struct mlx5_vdpa_query_mr { -SLIST_ENTRY (mlx5_vdpa_query_mr) next;union {struct ibv_mr *mr;struct mlx5_devx_obj *mkey;
@@ -76,10 +75,17 @@ enum {#define MLX5_VDPA_MAX_C_THRD 256#define MLX5_VDPA_MAX_TASKS_PER_THRD 4096#define MLX5_VDPA_TASKS_PER_DEV 64 +#define MLX5_VDPA_MAX_MRS 0xFFFF + +/* Vdpa task types. */ +enum mlx5_vdpa_task_type { +MLX5_VDPA_TASK_REG_MR =1 , +};/* Generic task information and size must be multiple of 4B. */ struct mlx5_vdpa_task {struct mlx5_vdpa_priv *priv; +enum mlx5_vdpa_task_type type;uint32_t *remaining_cnt;uint32_t *err_cnt;uint32_t idx;
@@ -101,6 +107,14 @@ struct mlx5_vdpa_conf_thread_mng { };extern struct mlx5_vdpa_conf_thread_mng conf_thread_mng; +struct mlx5_vdpa_vmem_info { +struct rte_vhost_memory *vmem; +uint32_t entries_num; +uint64_t gcd; +uint64_t size; +uint8_t mode; +}; +struct mlx5_vdpa_virtq {SLIST_ENTRY (mlx5_vdpa_virtq) next;uint8_t enable;
@@ -176,7 +190,7 @@ struct mlx5_vdpa_priv {struct mlx5_hca_vdpa_attr caps;uint32_t gpa_mkey_index;struct ibv_mr *null_mr; -struct rte_vhost_memory *vmem; +struct mlx5_vdpa_vmem_info vmem_info;struct mlx5dv_devx_event_channel *eventc;struct mlx5dv_devx_event_channel *err_chnl;struct mlx5_uar uar;
@@ -187,11 +201,13 @@ struct mlx5_vdpa_priv {uint8_t num_lag_ports;uint64_t features;/* Negotiated features. */ uint16_t log_max_rqt_size; +uint16_t last_c_thrd_idx; +uint16_t num_mrs;/* Number of memory regions. */ struct mlx5_vdpa_steer steer;struct mlx5dv_var *var;void *virtq_db_addr;struct mlx5_pmd_wrapped_mr lm_mr; - SLIST_HEAD(mr_list, mlx5_vdpa_query_mr) mr_list; +struct mlx5_vdpa_query_mr **mrs;struct mlx5_vdpa_virtq virtqs[]; };
@@ -548,5 +564,12 @@ mlx5_vdpa_mult_threads_destroy(bool need_unlock);bool mlx5_vdpa_task_add (struct mlx5_vdpa_priv *priv,uint32_t thrd_idx, -uint32_t num); +enum mlx5_vdpa_task_type task_type, +uint32_t *bulk_refcnt,uint32_t *bulk_err_cnt, +void **task_data,uint32_t num); +int +mlx5_vdpa_register_mr (struct mlx5_vdpa_priv *priv,uint32_t idx); +bool +mlx5_vdpa_c_thread_wait_bulk_tasks_done (uint32_t *remaining_cnt, +uint32_t *err_cnt,uint32_t sleep_time);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +44 -3
85 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_mem.c +184 -86
317 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +3 -3
@@ -353,21 +353,21 @@ mlx5_vdpa_virtq_sub_objs_prepare(struct mlx5_vdpa_priv *priv, } }if (attr->q_type ==MLX5_VIRTQ_TYPE_SPLIT ) { - gpa =mlx5_vdpa_hva_to_gpa (priv->vmem, + gpa =mlx5_vdpa_hva_to_gpa (priv->vmem_info.vmem, (uint64_t )(uintptr_t )vq->desc);if (!gpa) {DRV_LOG (ERR ,"Failed to get descriptor ring GPA." );return -1 ; } attr->desc_addr = gpa; - gpa =mlx5_vdpa_hva_to_gpa (priv->vmem, + gpa =mlx5_vdpa_hva_to_gpa (priv->vmem_info.vmem, (uint64_t )(uintptr_t )vq->used);if (!gpa) {DRV_LOG (ERR ,"Failed to get GPA for used ring." );return -1 ; } attr->used_addr = gpa; - gpa =mlx5_vdpa_hva_to_gpa (priv->vmem, + gpa =mlx5_vdpa_hva_to_gpa (priv->vmem_info.vmem, (uint64_t )(uintptr_t )vq->avail);if (!gpa) {DRV_LOG (ERR ,"Failed to get GPA for available ring." );
Signature · 1 line
2.31.1
PATCH v3 11/15 vdpa/mlx5: add virtq creation task for MT management
In reply to Li ZhangThe virtq object and all its sub-resources use a lot of FW commands and can be accelerated by the MT management. Split the virtqs creation between the configuration threads. This accelerates the LM process and reduces its time by 20%.
--- drivers/vdpa/mlx5/mlx5_vdpa.h | 9 +- drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 14 +++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 2 +- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 149 +++++++++++++++++++------- 4 files changed, 134 insertions(+), 40 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.h +6 -3
@@ -80,6 +80,7 @@ enum {/* Vdpa task types. */ enum mlx5_vdpa_task_type {MLX5_VDPA_TASK_REG_MR =1 , +MLX5_VDPA_TASK_SETUP_VIRTQ , };/* Generic task information and size must be multiple of 4B. */
@@ -117,12 +118,12 @@ struct mlx5_vdpa_vmem_info {struct mlx5_vdpa_virtq {SLIST_ENTRY (mlx5_vdpa_virtq) next; -uint8_t enable;uint16_t index;uint16_t vq_size;uint8_t notifier_state; -bool stopped;uint32_t configured:1 ; +uint32_t enable:1 ; +uint32_t stopped:1 ;uint32_t version;pthread_mutex_t virtq_lock;struct mlx5_vdpa_priv *priv;
@@ -565,11 +566,13 @@ bool mlx5_vdpa_task_add(struct mlx5_vdpa_priv *priv, uint32_t thrd_idx, enum mlx5_vdpa_task_type task_type, - uint32_t *bulk_refcnt, uint32_t *bulk_err_cnt, + uint32_t *remaining_cnt, uint32_t *err_cnt, void **task_data, uint32_t num);int mlx5_vdpa_register_mr (struct mlx5_vdpa_priv *priv,uint32_t idx);bool mlx5_vdpa_c_thread_wait_bulk_tasks_done (uint32_t *remaining_cnt,uint32_t *err_cnt,uint32_t sleep_time); +int +mlx5_vdpa_virtq_setup (struct mlx5_vdpa_priv *priv,int index,bool reg_kick);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +14 -0
@@ -100,6 +100,7 @@ mlx5_vdpa_c_thread_handle(void *arg) {struct mlx5_vdpa_conf_thread_mng *multhrd = arg;pthread_t thread_id =pthread_self (); +struct mlx5_vdpa_virtq *virtq;struct mlx5_vdpa_priv *priv;struct mlx5_vdpa_task task;struct rte_ring *rng;
@@ -139,6 +140,19 @@ mlx5_vdpa_c_thread_handle(void *arg) __ATOMIC_RELAXED); }break ; +case MLX5_VDPA_TASK_SETUP_VIRTQ : + virtq = &priv->virtqs[task.idx]; +pthread_mutex_lock (&virtq->virtq_lock); + ret =mlx5_vdpa_virtq_setup (priv, + task.idx, false); +if (ret) { +DRV_LOG (ERR , +"Failed to setup virtq %d." , task.idx); +__atomic_fetch_add ( + task.err_cnt,1 , __ATOMIC_RELAXED); + } +pthread_mutex_unlock (&virtq->virtq_lock); +break ;default :DRV_LOG (ERR ,"Invalid vdpa task type %d." , task.type);
drivers/vdpa/mlx5/mlx5_vdpa_event.c +1 -1
@@ -371,7 +371,7 @@ mlx5_vdpa_err_interrupt_handler(void *cb_arg __rte_unused) goto unlock;if (rte_rdtsc () /rte_get_tsc_hz () <MLX5_VDPA_ERROR_TIME_SEC ) goto unlock; - virtq->stopped = true; + virtq->stopped =1 ;/* Query error info. */ if (mlx5_vdpa_virtq_query (priv, vq_index)) goto log;
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +113 -36
215 lines not on this page · raw message
Signature · 1 line
2.31.1
PATCH v3 12/15 vdpa/mlx5: add virtq LM log task
In reply to Li ZhangSplit the virtqs LM log between the configuration threads. This accelerates the LM process and reduces its time by 20%.
--- drivers/vdpa/mlx5/mlx5_vdpa.h | 3 + drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 34 +++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 85 +++++++++++++++++++++------ 3 files changed, 105 insertions(+), 17 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.h +3 -0
@@ -72,6 +72,8 @@ enum { MLX5_VDPA_NOTIFIER_STATE_ERR }; +#define MLX5_VDPA_USED_RING_LEN (size) \ + ((size) * sizeof(struct vring_used_elem) + sizeof(uint16_t) * 3)#define MLX5_VDPA_MAX_C_THRD 256#define MLX5_VDPA_MAX_TASKS_PER_THRD 4096#define MLX5_VDPA_TASKS_PER_DEV 64
@@ -81,6 +83,7 @@ enum {enum mlx5_vdpa_task_type {MLX5_VDPA_TASK_REG_MR =1 ,MLX5_VDPA_TASK_SETUP_VIRTQ , +MLX5_VDPA_TASK_STOP_VIRTQ , };/* Generic task information and size must be multiple of 4B. */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +34 -0
@@ -104,6 +104,7 @@ mlx5_vdpa_c_thread_handle(void *arg)struct mlx5_vdpa_priv *priv;struct mlx5_vdpa_task task;struct rte_ring *rng; +uint64_t features;uint32_t thrd_idx;uint32_t task_num;int ret;
@@ -153,6 +154,39 @@ mlx5_vdpa_c_thread_handle(void *arg) }pthread_mutex_unlock (&virtq->virtq_lock);break ; +case MLX5_VDPA_TASK_STOP_VIRTQ : + virtq = &priv->virtqs[task.idx]; +pthread_mutex_lock (&virtq->virtq_lock); + ret =mlx5_vdpa_virtq_stop (priv, + task.idx); +if (ret) { +DRV_LOG (ERR , +"Failed to stop virtq %d." , + task.idx); +__atomic_fetch_add ( + task.err_cnt,1 , + __ATOMIC_RELAXED); +pthread_mutex_unlock (&virtq->virtq_lock); +break ; + } + ret =rte_vhost_get_negotiated_features ( + priv->vid, &features); +if (ret) { +DRV_LOG (ERR , +"Failed to get negotiated features virtq %d." , + task.idx); +__atomic_fetch_add ( + task.err_cnt,1 , + __ATOMIC_RELAXED); +pthread_mutex_unlock (&virtq->virtq_lock); +break ; + } +if (RTE_VHOST_NEED_LOG (features)) +rte_vhost_log_used_vring ( + priv->vid, task.idx,0 , +MLX5_VDPA_USED_RING_LEN (virtq->vq_size)); +pthread_mutex_unlock (&virtq->virtq_lock); +break ;default :DRV_LOG (ERR ,"Invalid vdpa task type %d." , task.type);
drivers/vdpa/mlx5/mlx5_vdpa_lm.c +68 -17
107 lines not on this page · raw message
Signature · 1 line
2.31.1
PATCH v3 13/15 vdpa/mlx5: add device close task
In reply to Li ZhangSplit the virtqs device close tasks after stopping virt-queue between the configuration threads. This accelerates the LM process and
reduces its time by 50%.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 56 +++++++++++++++++++++++++-- drivers/vdpa/mlx5/mlx5_vdpa.h | 8 ++++ drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 20 +++++++++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 14 +++++++ 4 files changed, 94 insertions(+), 4 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +53 -3
@@ -245,7 +245,7 @@ mlx5_vdpa_mtu_set(struct mlx5_vdpa_priv *priv)return kern_mtu == vhost_mtu ?0 :-1 ; } -static void +void mlx5_vdpa_dev_cache_clean (struct mlx5_vdpa_priv *priv) {/* Clean pre-created resource in dev removal only. */
@@ -254,6 +254,26 @@ mlx5_vdpa_dev_cache_clean(struct mlx5_vdpa_priv *priv)mlx5_vdpa_mem_dereg (priv); } +static bool +mlx5_vdpa_wait_dev_close_tasks_done (struct mlx5_vdpa_priv *priv) +{ +uint32_t timeout =0 ; + +/* Check and wait all close tasks done. */ +while (__atomic_load_n (&priv->dev_close_progress, + __ATOMIC_RELAXED) !=0 && timeout <1000 ) { +rte_delay_us_sleep (10000 ); + timeout++; + } +if (priv->dev_close_progress) { +DRV_LOG (ERR , +"Failed to wait close device tasks done vid %d." , + priv->vid); +return true; + } +return false; +} +static int mlx5_vdpa_dev_close (int vid) {
@@ -271,6 +291,27 @@ mlx5_vdpa_dev_close(int vid) ret |=mlx5_vdpa_lm_log (priv); priv->state =MLX5_VDPA_STATE_IN_PROGRESS ; } +if (priv->use_c_thread) { +if (priv->last_c_thrd_idx >= + (conf_thread_mng.max_thrds -1 )) + priv->last_c_thrd_idx =0 ; +else + priv->last_c_thrd_idx++; +__atomic_store_n (&priv->dev_close_progress, +1 , __ATOMIC_RELAXED); +if (mlx5_vdpa_task_add (priv, + priv->last_c_thrd_idx, +MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT , +NULL ,NULL ,NULL ,1 )) { +DRV_LOG (ERR , +"Fail to add dev close task. " ); + goto single_thrd; + } + priv->state =MLX5_VDPA_STATE_PROBED ; +DRV_LOG (INFO ,"vDPA device %d was closed." , vid); +return ret; + } +single_thrd:pthread_mutex_lock (&priv->steer_update_lock);mlx5_vdpa_steer_unset (priv);pthread_mutex_unlock (&priv->steer_update_lock);
@@ -278,10 +319,12 @@ mlx5_vdpa_dev_close(int vid)mlx5_vdpa_drain_cq (priv);if (priv->lm_mr.addr)mlx5_os_wrapped_mkey_destroy (&priv->lm_mr); - priv->state =MLX5_VDPA_STATE_PROBED ;if (!priv->connected)mlx5_vdpa_dev_cache_clean (priv); priv->vid =0 ; +__atomic_store_n (&priv->dev_close_progress,0 , + __ATOMIC_RELAXED); + priv->state =MLX5_VDPA_STATE_PROBED ;DRV_LOG (INFO ,"vDPA device %d was closed." , vid);return ret; }
@@ -302,6 +345,8 @@ mlx5_vdpa_dev_config(int vid)DRV_LOG (ERR ,"Failed to reconfigure vid %d." , vid);return -1 ; } +if (mlx5_vdpa_wait_dev_close_tasks_done (priv)) +return -1 ; priv->vid = vid; priv->connected = true;if (mlx5_vdpa_mtu_set (priv))
@@ -444,8 +489,11 @@ mlx5_vdpa_dev_cleanup(int vid)DRV_LOG (ERR ,"Invalid vDPA device: %s." , vdev->device->name);return -1 ; } -if (priv->state ==MLX5_VDPA_STATE_PROBED ) +if (priv->state ==MLX5_VDPA_STATE_PROBED ) { +if (priv->use_c_thread) +mlx5_vdpa_wait_dev_close_tasks_done (priv);mlx5_vdpa_dev_cache_clean (priv); + } priv->connected = false;return 0 ; }
@@ -839,6 +887,8 @@ mlx5_vdpa_dev_release(struct mlx5_vdpa_priv *priv) {if (priv->state ==MLX5_VDPA_STATE_CONFIGURED )mlx5_vdpa_dev_close (priv->vid); +if (priv->use_c_thread) +mlx5_vdpa_wait_dev_close_tasks_done (priv);mlx5_vdpa_release_dev_resources (priv);if (priv->vdev)rte_vdpa_unregister_device (priv->vdev);
drivers/vdpa/mlx5/mlx5_vdpa.h +8 -0
@@ -84,6 +84,7 @@ enum mlx5_vdpa_task_type {MLX5_VDPA_TASK_REG_MR =1 ,MLX5_VDPA_TASK_SETUP_VIRTQ ,MLX5_VDPA_TASK_STOP_VIRTQ , + MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT, };/* Generic task information and size must be multiple of 4B. */
@@ -206,6 +207,7 @@ struct mlx5_vdpa_priv {uint64_t features;/* Negotiated features. */ uint16_t log_max_rqt_size;uint16_t last_c_thrd_idx; +uint16_t dev_close_progress;uint16_t num_mrs;/* Number of memory regions. */ struct mlx5_vdpa_steer steer;struct mlx5dv_var *var;
@@ -578,4 +580,10 @@ mlx5_vdpa_c_thread_wait_bulk_tasks_done(uint32_t *remaining_cnt, uint32_t *err_cnt, uint32_t sleep_time);int mlx5_vdpa_virtq_setup (struct mlx5_vdpa_priv *priv,int index,bool reg_kick); +void +mlx5_vdpa_vq_destroy (struct mlx5_vdpa_virtq *virtq); +void +mlx5_vdpa_dev_cache_clean (struct mlx5_vdpa_priv *priv); +void +mlx5_vdpa_virtq_unreg_intr_handle_all (struct mlx5_vdpa_priv *priv);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +19 -1
@@ -63,7 +63,8 @@ mlx5_vdpa_task_add(struct mlx5_vdpa_priv *priv, task[i].type = task_type; task[i].remaining_cnt = remaining_cnt; task[i].err_cnt = err_cnt; - task[i].idx = data[i]; +if (data) + task[i].idx = data[i]; }if (!mlx5_vdpa_c_thrd_ring_enqueue_bulk (rng, (void **)&task, num,NULL ))return -1 ;
@@ -187,6 +188,23 @@ mlx5_vdpa_c_thread_handle(void *arg) MLX5_VDPA_USED_RING_LEN(virtq->vq_size));pthread_mutex_unlock (&virtq->virtq_lock);break ; +case MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT : +mlx5_vdpa_virtq_unreg_intr_handle_all (priv); +pthread_mutex_lock (&priv->steer_update_lock); +mlx5_vdpa_steer_unset (priv); +pthread_mutex_unlock (&priv->steer_update_lock); +mlx5_vdpa_virtqs_release (priv); +mlx5_vdpa_drain_cq (priv); +if (priv->lm_mr.addr) +mlx5_os_wrapped_mkey_destroy ( + &priv->lm_mr); +if (!priv->connected) +mlx5_vdpa_dev_cache_clean (priv); + priv->vid =0 ; +__atomic_store_n ( + &priv->dev_close_progress,0 , + __ATOMIC_RELAXED); +break ;default :DRV_LOG (ERR ,"Invalid vdpa task type %d." , task.type);
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +14 -0
@@ -102,6 +102,20 @@ mlx5_vdpa_virtq_unregister_intr_handle(struct mlx5_vdpa_virtq *virtq) virtq->intr_handle =NULL ; } +void +mlx5_vdpa_virtq_unreg_intr_handle_all (struct mlx5_vdpa_priv *priv) +{ +uint32_t i; +struct mlx5_vdpa_virtq *virtq; + +for (i =0 ; i < priv->nr_virtqs; i++) { + virtq = &priv->virtqs[i]; +pthread_mutex_lock (&virtq->virtq_lock); +mlx5_vdpa_virtq_unregister_intr_handle (virtq); +pthread_mutex_unlock (&virtq->virtq_lock); + } +} +/* Release cached VQ resources. */ void mlx5_vdpa_virtqs_cleanup (struct mlx5_vdpa_priv *priv)
Signature · 1 line
2.31.1
PATCH v3 14/15 vdpa/mlx5: add virtq sub-resources creation
In reply to Li Zhangpre-created virt-queue sub-resource in device probe stage and then modify virtqueue in device config stage. Steer table also need to support dummy virt-queue. This accelerates the LM process and reduces its time by 40%.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 72 +++++++-------------- drivers/vdpa/mlx5/mlx5_vdpa.h | 17 +++-- drivers/vdpa/mlx5/mlx5_vdpa_event.c | 11 ++-- drivers/vdpa/mlx5/mlx5_vdpa_steer.c | 17 +++-- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 99 +++++++++++++++++++++-------- 5 files changed, 123 insertions(+), 93 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +23 -49
@@ -627,65 +627,39 @@ mlx5_vdpa_config_get(struct mlx5_kvargs_ctrl *mkvlist,static int mlx5_vdpa_virtq_resource_prepare (struct mlx5_vdpa_priv *priv) { -struct mlx5_vdpa_virtq *virtq; +uint32_t max_queues;uint32_t index; -uint32_t i; +struct mlx5_vdpa_virtq *virtq; -for (index =0 ; index < priv->caps.max_num_virtio_queues *2 ; +for (index =0 ; index < priv->caps.max_num_virtio_queues; index++) { virtq = &priv->virtqs[index];pthread_mutex_init (&virtq->virtq_lock,NULL ); } -if (!priv->queues) +if (!priv->queues || !priv->queue_size)return 0 ; -for (index =0 ; index < (priv->queues *2 ); ++index) { + max_queues = (priv->queues < priv->caps.max_num_virtio_queues) ? + (priv->queues *2 ) : (priv->caps.max_num_virtio_queues); +for (index =0 ; index < max_queues; ++index) +if (mlx5_vdpa_virtq_single_resource_prepare (priv, + index)) + goto error; +if (mlx5_vdpa_is_modify_virtq_supported (priv)) +if (mlx5_vdpa_steer_update (priv, true)) + goto error; +return 0 ; +error: +for (index =0 ; index < max_queues; ++index) { virtq = &priv->virtqs[index]; -int ret =mlx5_vdpa_event_qp_prepare (priv, priv->queue_size, --1 , virtq); - -if (ret) { -DRV_LOG (ERR ,"Failed to create event QPs for virtq %d." , - index); -return -1 ; - } -if (priv->caps.queue_counters_valid) { -if (!virtq->counters) - virtq->counters = -mlx5_devx_cmd_create_virtio_q_counters - (priv->cdev->ctx); -if (!virtq->counters) { -DRV_LOG (ERR ,"Failed to create virtq couners for virtq" -" %d." , index); -return -1 ; - } - } -for (i =0 ; i <RTE_DIM (virtq->umems); ++i) { -uint32_t size; -void *buf; -struct mlx5dv_devx_umem *obj; - - size = priv->caps.umems[i].a * priv->queue_size + - priv->caps.umems[i].b; - buf =rte_zmalloc (__func__, size,4096 ); -if (buf ==NULL ) { -DRV_LOG (ERR ,"Cannot allocate umem %d memory for virtq" -" %u." , i, index); -return -1 ; - } - obj = mlx5_glue->devx_umem_reg (priv->cdev->ctx, buf, - size,IBV_ACCESS_LOCAL_WRITE ); -if (obj ==NULL ) { -rte_free (buf); -DRV_LOG (ERR ,"Failed to register umem %d for virtq %u." , - i, index); -return -1 ; - } - virtq->umems[i].size = size; - virtq->umems[i].buf = buf; - virtq->umems[i].obj = obj; +if (virtq->virtq) { +pthread_mutex_lock (&virtq->virtq_lock); +mlx5_vdpa_virtq_unset (virtq); +pthread_mutex_unlock (&virtq->virtq_lock); } } -return 0 ; +if (mlx5_vdpa_is_modify_virtq_supported (priv)) +mlx5_vdpa_steer_unset (priv); +return -1 ; } static int
drivers/vdpa/mlx5/mlx5_vdpa.h +13 -4
@@ -277,13 +277,15 @@ int mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv);* The guest notification file descriptor. * @param[in/out] virtq * Pointer to the virt-queue structure. +* @param[in] reset +* If true, it will reset event qp. * * @return * 0 on success, -1 otherwise and rte_errno is set. */ int mlx5_vdpa_event_qp_prepare (struct mlx5_vdpa_priv *priv,uint16_t desc_n, -int callfd,struct mlx5_vdpa_virtq *virtq); +int callfd,struct mlx5_vdpa_virtq *virtq,bool reset);/** * Destroy an event QP and all its related resources.
@@ -403,11 +405,13 @@ void mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv);* * @param[in] priv * The vdpa driver private structure. +* @param[in] is_dummy +* If set, it is updated with dummy queue for prepare resource. * * @return * 0 on success, a negative value otherwise. */ -int mlx5_vdpa_steer_update (struct mlx5_vdpa_priv *priv); +int mlx5_vdpa_steer_update (struct mlx5_vdpa_priv *priv,bool is_dummy);/** * Setup steering and all its related resources to enable RSS traffic from the
@@ -581,9 +585,14 @@ mlx5_vdpa_c_thread_wait_bulk_tasks_done(uint32_t *remaining_cnt,int mlx5_vdpa_virtq_setup (struct mlx5_vdpa_priv *priv,int index,bool reg_kick);void -mlx5_vdpa_vq_destroy (struct mlx5_vdpa_virtq *virtq); -void mlx5_vdpa_dev_cache_clean (struct mlx5_vdpa_priv *priv);void mlx5_vdpa_virtq_unreg_intr_handle_all (struct mlx5_vdpa_priv *priv); +bool +mlx5_vdpa_virtq_single_resource_prepare (struct mlx5_vdpa_priv *priv, +int index); +int +mlx5_vdpa_qps2rst2rts (struct mlx5_vdpa_event_qp *eqp); +void +mlx5_vdpa_virtq_unset (struct mlx5_vdpa_virtq *virtq);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_event.c +5 -6
@@ -249,7 +249,7 @@ mlx5_vdpa_drain_cq(struct mlx5_vdpa_priv *priv) {unsigned int i; -for (i =0 ; i < priv->caps.max_num_virtio_queues *2 ; i++) { +for (i =0 ; i < priv->caps.max_num_virtio_queues; i++) {struct mlx5_vdpa_cq *cq = &priv->virtqs[i].eqp.cq;mlx5_vdpa_queue_complete (cq);
@@ -618,7 +618,7 @@ mlx5_vdpa_qps2rts(struct mlx5_vdpa_event_qp *eqp)return 0 ; } -static int +int mlx5_vdpa_qps2rst2rts (struct mlx5_vdpa_event_qp *eqp) { if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_QP_2RST,
@@ -638,7 +638,7 @@ mlx5_vdpa_qps2rst2rts(struct mlx5_vdpa_event_qp *eqp)int mlx5_vdpa_event_qp_prepare (struct mlx5_vdpa_priv *priv,uint16_t desc_n, -int callfd,struct mlx5_vdpa_virtq *virtq) +int callfd,struct mlx5_vdpa_virtq *virtq,bool reset) {struct mlx5_vdpa_event_qp *eqp = &virtq->eqp;struct mlx5_devx_qp_attr attr = {0 };
@@ -649,11 +649,10 @@ mlx5_vdpa_event_qp_prepare(struct mlx5_vdpa_priv *priv, uint16_t desc_n,/* Reuse existing resources. */ eqp->cq.callfd = callfd;/* FW will set event qp to error state in q destroy. */ -if (!mlx5_vdpa_qps2rst2rts (eqp)) { +if (reset && !mlx5_vdpa_qps2rst2rts (eqp))rte_write32 (rte_cpu_to_be_32 (RTE_BIT32 (log_desc_n)), &eqp->sw_qp.db_rec[0 ]); -return 0 ; - } +return 0 ; }if (eqp->fw_qp)mlx5_vdpa_event_qp_destroy (eqp);
drivers/vdpa/mlx5/mlx5_vdpa_steer.c +11 -6
@@ -57,7 +57,7 @@ mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv)* -1 on error. */ static int -mlx5_vdpa_rqt_prepare (struct mlx5_vdpa_priv *priv) +mlx5_vdpa_rqt_prepare (struct mlx5_vdpa_priv *priv,bool is_dummy) {int i; uint32_t rqt_n = RTE_MIN(MLX5_VDPA_DEFAULT_RQT_SIZE,
@@ -67,15 +67,20 @@ mlx5_vdpa_rqt_prepare(struct mlx5_vdpa_priv *priv) sizeof(uint32_t), 0);uint32_t k =0 , j;int ret =0 , num; +uint16_t nr_vring = is_dummy ? + (((priv->queues *2 ) < priv->caps.max_num_virtio_queues) ? + (priv->queues *2 ) : priv->caps.max_num_virtio_queues) : priv->nr_virtqs;if (!attr) {DRV_LOG (ERR ,"Failed to allocate RQT attributes memory." ); rte_errno =ENOMEM ;return -ENOMEM ; } -for (i =0 ; i < priv->nr_virtqs; i++) { +for (i =0 ; i < nr_vring; i++) {if (is_virtq_recvq (i, priv->nr_virtqs) && - priv->virtqs[i].enable && priv->virtqs[i].virtq) { + (is_dummy || (priv->virtqs[i].enable && + priv->virtqs[i].configured)) && + priv->virtqs[i].virtq) { attr->rq_list[k] = priv->virtqs[i].virtq->id; k++; }
@@ -235,12 +240,12 @@ mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv *priv) }int -mlx5_vdpa_steer_update (struct mlx5_vdpa_priv *priv) +mlx5_vdpa_steer_update (struct mlx5_vdpa_priv *priv,bool is_dummy) {int ret;pthread_mutex_lock (&priv->steer_update_lock); - ret =mlx5_vdpa_rqt_prepare (priv); + ret =mlx5_vdpa_rqt_prepare (priv, is_dummy);if (ret ==0 ) {mlx5_vdpa_steer_unset (priv); }else if (ret <0 ) {
@@ -261,7 +266,7 @@ mlx5_vdpa_steer_update(struct mlx5_vdpa_priv *priv)int mlx5_vdpa_steer_setup (struct mlx5_vdpa_priv *priv) { -if (mlx5_vdpa_steer_update (priv)) +if (mlx5_vdpa_steer_update (priv, false)) goto error;return 0 ; error:
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +71 -28
198 lines not on this page · raw message
Signature · 1 line
2.31.1
PATCH v3 15/15 vdpa/mlx5: prepare virtqueue resource creation
In reply to Li ZhangSplit the virtqs virt-queue resource between
the configuration threads.
Also need pre-created virt-queue resource
after virtq destruction.
This accelerates the LM process and reduces its time by 30%.
--- doc/guides/rel_notes/release_22_07.rst | 1 + drivers/vdpa/mlx5/mlx5_vdpa.c | 115 +++++++++++++++++++------ drivers/vdpa/mlx5/mlx5_vdpa.h | 12 ++- drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 15 +++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 111 ++++++++++++++++++++---- 5 files changed, 209 insertions(+), 45 deletions(-)
doc/guides/rel_notes/release_22_07.rst +1 -0
@@ -178,6 +178,7 @@ New Features * **Updated Nvidia mlx5 vDPA driver.** * Added new devargs ``queue_size`` and ``queues`` to allow prior creation of virtq resources. + * Added new devarg ``max_conf_threads`` defines the number of multi-thread management to parallel the configurations. Removed Items
drivers/vdpa/mlx5/mlx5_vdpa.c +89 -26
168 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa.h +11 -1
@@ -85,6 +85,7 @@ enum mlx5_vdpa_task_type {MLX5_VDPA_TASK_SETUP_VIRTQ ,MLX5_VDPA_TASK_STOP_VIRTQ ,MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT , + MLX5_VDPA_TASK_PREPARE_VIRTQ, };/* Generic task information and size must be multiple of 4B. */
@@ -128,6 +129,9 @@ struct mlx5_vdpa_virtq { uint32_t configured:1; uint32_t enable:1; uint32_t stopped:1; + uint32_t rx_csum:1; + uint32_t virtio_version_1_0:1; + uint32_t event_mode:3;uint32_t version;pthread_mutex_t virtq_lock;struct mlx5_vdpa_priv *priv;
@@ -355,8 +359,12 @@ void mlx5_vdpa_err_event_unset(struct mlx5_vdpa_priv *priv);* * @param[in] priv * The vdpa driver private structure. +* @param[in] release_resource +* The vdpa driver release resource without prepare resource. */ -void mlx5_vdpa_virtqs_release (struct mlx5_vdpa_priv *priv); +void +mlx5_vdpa_virtqs_release (struct mlx5_vdpa_priv *priv, +bool release_resource);/** * Cleanup cached resources of all virtqs.
@@ -595,4 +603,6 @@ int mlx5_vdpa_qps2rst2rts(struct mlx5_vdpa_event_qp *eqp);void mlx5_vdpa_virtq_unset (struct mlx5_vdpa_virtq *virtq); +void +mlx5_vdpa_prepare_virtq_destroy (struct mlx5_vdpa_priv *priv);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +14 -1
@@ -153,6 +153,7 @@ mlx5_vdpa_c_thread_handle(void *arg)__atomic_fetch_add ( task.err_cnt,1 , __ATOMIC_RELAXED); } + virtq->enable =1 ;pthread_mutex_unlock (&virtq->virtq_lock);break ;case MLX5_VDPA_TASK_STOP_VIRTQ :
@@ -193,7 +194,7 @@ mlx5_vdpa_c_thread_handle(void *arg)pthread_mutex_lock (&priv->steer_update_lock);mlx5_vdpa_steer_unset (priv);pthread_mutex_unlock (&priv->steer_update_lock); -mlx5_vdpa_virtqs_release (priv); +mlx5_vdpa_virtqs_release (priv, false);mlx5_vdpa_drain_cq (priv);if (priv->lm_mr.addr) mlx5_os_wrapped_mkey_destroy(
@@ -205,6 +206,18 @@ mlx5_vdpa_c_thread_handle(void *arg) &priv->dev_close_progress,0 , __ATOMIC_RELAXED);break ; +case MLX5_VDPA_TASK_PREPARE_VIRTQ : + ret =mlx5_vdpa_virtq_single_resource_prepare ( + priv, task.idx); +if (ret) { +DRV_LOG (ERR , +"Failed to prepare virtq %d." , + task.idx); +__atomic_fetch_add ( + task.err_cnt,1 , + __ATOMIC_RELAXED); + } +break ;default :DRV_LOG (ERR ,"Invalid vdpa task type %d." , task.type);
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +94 -17
197 lines not on this page · raw message
Signature · 1 line
2.31.1
v4 cover mlx5/vdpa: optimize live migration time
In reply to Li ZhangAllow the driver to use internal threads to obtain fast configuration.
All the threads will be open on the same core of the event completion queue scheduling thread.
Add max_conf_threads parameter to configure
the maximum number of internal threads in addition to the caller thread (8 is suggested).
These internal threads to pipeline handle VDPA tasks in system and shared with all VDPA devices. Default is 0, don't use internal threads for configuration.
Depends-on: series=21868 ("vdpa/mlx5: improve device shutdown time")
http://patchwork.dpdk.org/project/dpdk/list/?series=21868
RFC ("Add vDPA multi-threads optiomization")
https://patchwork.dpdk.org/project/dpdk/cover/[email protected]/
V2:
* Drop eal device removal patch in series.
* Add release note in release_22_07.rst.
V3:
* Fix comments about commit log issue.
* Avoid cutting logs.
V4:
* Fix coding style issue
Li Zhang (12):
vdpa/mlx5: fix usage of capability for max number of virtqs common/mlx5: extend virtq modifiable fields vdpa/mlx5: pre-create virtq at probe time vdpa/mlx5: optimize datapath-control synchronization vdpa/mlx5: add multi-thread management for configuration vdpa/mlx5: add task ring for MT management vdpa/mlx5: add MT task for VM memory registration vdpa/mlx5: add virtq creation task for MT management vdpa/mlx5: add virtq LM log task
vdpa/mlx5: add device close task
vdpa/mlx5: add virtq sub-resources creation vdpa/mlx5: prepare virtqueue resource creation
Yajun Wu (3):
vdpa/mlx5: support pre create virtq resource common/mlx5: add DevX API to move QP to reset state vdpa/mlx5: support event qp reuse
doc/guides/rel_notes/release_22_07.rst | 5 + doc/guides/vdpadevs/mlx5.rst | 25 + drivers/common/mlx5/mlx5_devx_cmds.c | 77 ++- drivers/common/mlx5/mlx5_devx_cmds.h | 6 +- drivers/common/mlx5/mlx5_prm.h | 30 +- drivers/vdpa/mlx5/meson.build | 1 + drivers/vdpa/mlx5/mlx5_vdpa.c | 270 ++++++++-- drivers/vdpa/mlx5/mlx5_vdpa.h | 152 +++++- drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 360 ++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 160 ++++-- drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 134 +++-- drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 270 ++++++---- drivers/vdpa/mlx5/mlx5_vdpa_steer.c | 22 +- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 654 ++++++++++++++++++------- 14 files changed, 1779 insertions(+), 387 deletions(-) create mode 100644 drivers/vdpa/mlx5/mlx5_vdpa_cthread.c
Signature · 1 line
2.31.1
PATCH v4 1/15 vdpa/mlx5: fix usage of capability for max number of virtqs
The driver wrongly takes the capability value for
the number of virtq pairs instead of just the number of virtqs.
Adjust all the usages of it to be the number of virtqs.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 12 ++++++------ drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +6 -6
@@ -84,7 +84,7 @@ mlx5_vdpa_get_queue_num(struct rte_vdpa_device *vdev, uint32_t *queue_num)DRV_LOG (ERR ,"Invalid vDPA device: %s." , vdev->device->name);return -1 ; } - *queue_num = priv->caps.max_num_virtio_queues; + *queue_num = priv->caps.max_num_virtio_queues /2 ;return 0 ; }
@@ -141,7 +141,7 @@ mlx5_vdpa_set_vring_state(int vid, int vring, int state)DRV_LOG (ERR ,"Invalid vDPA device: %s." , vdev->device->name);return -EINVAL ; } -if (vring >= (int )priv->caps.max_num_virtio_queues *2 ) { +if (vring >= (int )priv->caps.max_num_virtio_queues) {DRV_LOG (ERR ,"Too big vring id: %d." , vring);return -E2BIG ; }
@@ -388,7 +388,7 @@ mlx5_vdpa_get_stats(struct rte_vdpa_device *vdev, int qid,DRV_LOG (ERR ,"Invalid device: %s." , vdev->device->name);return -ENODEV ; } -if (qid >= (int )priv->caps.max_num_virtio_queues *2 ) { +if (qid >= (int )priv->caps.max_num_virtio_queues) {DRV_LOG (ERR ,"Too big vring id: %d for device %s." , qid, vdev->device->name);return -E2BIG ;
@@ -411,7 +411,7 @@ mlx5_vdpa_reset_stats(struct rte_vdpa_device *vdev, int qid)DRV_LOG (ERR ,"Invalid device: %s." , vdev->device->name);return -ENODEV ; } -if (qid >= (int )priv->caps.max_num_virtio_queues *2 ) { +if (qid >= (int )priv->caps.max_num_virtio_queues) {DRV_LOG (ERR ,"Too big vring id: %d for device %s." , qid, vdev->device->name);return -E2BIG ;
@@ -624,7 +624,7 @@ mlx5_vdpa_dev_probe(struct mlx5_common_device *cdev,DRV_LOG (DEBUG ,"No capability to support virtq statistics." ); priv =rte_zmalloc ("mlx5 vDPA device private" ,sizeof (*priv) +sizeof (struct mlx5_vdpa_virtq ) * - attr->vdpa.max_num_virtio_queues *2 , + attr->vdpa.max_num_virtio_queues,RTE_CACHE_LINE_SIZE );if (!priv) {DRV_LOG (ERR ,"Failed to allocate private memory." );
@@ -685,7 +685,7 @@ mlx5_vdpa_release_dev_resources(struct mlx5_vdpa_priv *priv)uint32_t i;mlx5_vdpa_dev_cache_clean (priv); -for (i =0 ; i < priv->caps.max_num_virtio_queues *2 ; i++) { +for (i =0 ; i < priv->caps.max_num_virtio_queues; i++) {if (!priv->virtqs[i].counters)continue ;claim_zero (mlx5_devx_cmd_destroy (priv->virtqs[i].counters));
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +3 -3
@@ -72,7 +72,7 @@ mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv) {unsigned int i, j; -for (i =0 ; i < priv->caps.max_num_virtio_queues *2 ; i++) { +for (i =0 ; i < priv->caps.max_num_virtio_queues; i++) {struct mlx5_vdpa_virtq *virtq = &priv->virtqs[i];for (j =0 ; j <RTE_DIM (virtq->umems); ++j) {
@@ -492,9 +492,9 @@ mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv)DRV_LOG (INFO ,"TSO is enabled without CSUM, force CSUM." ); priv->features |= (1ULL <<VIRTIO_NET_F_CSUM ); } -if (nr_vring > priv->caps.max_num_virtio_queues *2 ) { +if (nr_vring > priv->caps.max_num_virtio_queues) {DRV_LOG (ERR ,"Do not support more than %d virtqs(%d)." , - (int )priv->caps.max_num_virtio_queues *2 , + (int )priv->caps.max_num_virtio_queues, (int )nr_vring);return -1 ; }
Signature · 1 line
2.31.1
PATCH v4 2/15 vdpa/mlx5: support pre create virtq resource
In reply to Li ZhangFrom: Yajun Wu <[email protected]>
The motivation of this change is to reduce vDPA device queue creation time by creating some queue resource in vDPA device probe stage.
In VM live migration scenario, this can reduce 0.8ms for each queue creation, thus reduce LM network downtime.
To create queue resource(umem/counter) in advance, we need to know virtio queue depth and max number of queue VM will use.
Introduce two new devargs: queues(max queue pair number) and queue_size (queue depth). Two args must be both provided, if only one argument provided, the argument will be ignored and no pre-creation.
The queues and queue_size must also be identical to vhost configuration driver later receive. Otherwise either the pre-create resource is wasted or missing or the resource need destroy and recreate(in case queue_size mismatch).
Pre-create umem/counter will keep alive until vDPA device removal.
--- doc/guides/vdpadevs/mlx5.rst | 14 +++++++ drivers/vdpa/mlx5/mlx5_vdpa.c | 75 ++++++++++++++++++++++++++++++++++- drivers/vdpa/mlx5/mlx5_vdpa.h | 2 + 3 files changed, 89 insertions(+), 2 deletions(-)
doc/guides/vdpadevs/mlx5.rst +14 -0
@@ -101,6 +101,20 @@ for an additional list of options shared with other mlx5 drivers. - 0, HW default. +- ``queue_size`` parameter [int] + + - 1 - 1024, Virio Queue depth for pre-creating queue resource to speed up + first time queue creation. Set it together with queues devarg. + + - 0, default value, no pre-create virtq resource. + +- ``queues`` parameter [int] + + - 1 - 128, Max number of virio queue pair(including 1 rx queue and 1 tx queue) + for pre-create queue resource to speed up first time queue creation. Set it + together with queue_size devarg. + + - 0, default value, no pre-create virtq resource. Error handling ^^^^^^^^^^^^^^
drivers/vdpa/mlx5/mlx5_vdpa.c +73 -2
120 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa.h +2 -0
@@ -135,6 +135,8 @@ struct mlx5_vdpa_priv {uint8_t hw_latency_mode;/* Hardware CQ moderation mode. */ uint16_t hw_max_latency_us;/* Hardware CQ moderation period in usec. */ uint16_t hw_max_pending_comp;/* Hardware CQ moderation counter. */ +uint16_t queue_size;/* virtq depth for pre-creating virtq resource */ +uint16_t queues;/* Max virtq pair for pre-creating virtq resource */ struct rte_vdpa_device *vdev;/* vDPA device. */ struct mlx5_common_device *cdev;/* Backend mlx5 device. */ int vid;/* vhost device id. */
Signature · 1 line
2.31.1
PATCH v4 3/15 common/mlx5: add DevX API to move QP to reset state
In reply to Li ZhangFrom: Yajun Wu <[email protected]>
Support set QP to RESET state.
--- drivers/common/mlx5/mlx5_devx_cmds.c | 7 +++++++ drivers/common/mlx5/mlx5_prm.h | 17 +++++++++++++++++ 2 files changed, 24 insertions(+)
drivers/common/mlx5/mlx5_devx_cmds.c +7 -0
@@ -2264,11 +2264,13 @@ mlx5_devx_cmd_modify_qp_state(struct mlx5_devx_obj *qp, uint32_t qp_st_mod_op,uint32_t rst2init[MLX5_ST_SZ_DW (rst2init_qp_in)];uint32_t init2rtr[MLX5_ST_SZ_DW (init2rtr_qp_in)];uint32_t rtr2rts[MLX5_ST_SZ_DW (rtr2rts_qp_in)]; + uint32_t qp2rst[MLX5_ST_SZ_DW(2rst_qp_in)]; } in;union {uint32_t rst2init[MLX5_ST_SZ_DW (rst2init_qp_out)];uint32_t init2rtr[MLX5_ST_SZ_DW (init2rtr_qp_out)];uint32_t rtr2rts[MLX5_ST_SZ_DW (rtr2rts_qp_out)]; + uint32_t qp2rst[MLX5_ST_SZ_DW(2rst_qp_out)]; } out;void *qpc;int ret;
@@ -2311,6 +2313,11 @@ mlx5_devx_cmd_modify_qp_state(struct mlx5_devx_obj *qp, uint32_t qp_st_mod_op, inlen =sizeof (in.rtr2rts); outlen =sizeof (out.rtr2rts);break ; +case MLX5_CMD_OP_QP_2RST : + MLX5_SET(2rst_qp_in, &in, qpn, qp->id); + inlen =sizeof (in.qp2rst); + outlen =sizeof (out.qp2rst); +break ;default :DRV_LOG (ERR ,"Invalid or unsupported QP modify op %u." , qp_st_mod_op);
drivers/common/mlx5/mlx5_prm.h +17 -0
@@ -3657,6 +3657,23 @@ struct mlx5_ifc_init2init_qp_in_bits {u8 reserved_at_800[0x80 ]; }; +struct mlx5_ifc_2rst_qp_out_bits { +u8 status[0x8 ]; +u8 reserved_at_8[0x18 ]; +u8 syndrome[0x20 ]; +u8 reserved_at_40[0x40 ]; +}; + +struct mlx5_ifc_2rst_qp_in_bits { +u8 opcode[0x10 ]; +u8 uid[0x10 ]; +u8 vhca_tunnel_id[0x10 ]; +u8 op_mod[0x10 ]; +u8 reserved_at_80[0x8 ]; +u8 qpn[0x18 ]; +u8 reserved_at_a0[0x20 ]; +}; +struct mlx5_ifc_dealloc_pd_out_bits {u8 status[0x8 ];u8 reserved_0[0x18 ];
Signature · 1 line
2.31.1
PATCH v4 4/15 vdpa/mlx5: support event qp reuse
In reply to Li ZhangFrom: Yajun Wu <[email protected]>
To speed up queue create time, event qp and cq will create only once. Each virtq creation will reuse same event qp and cq.
Because FW will set event qp to error state during virtq destroy, need modify event qp to RESET state, then modify qp to RTS state as usual. This can save about 1.5ms for each virtq creation.
After SW qp reset, qp pi/ci all become 0 while cq pi/ci keep as previous. Add new variable qp_ci to save SW qp ci. Move qp pi independently with cq ci.
Add new function mlx5_vdpa_drain_cq to drain cq CQE after virtq release.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 8 ++++ drivers/vdpa/mlx5/mlx5_vdpa.h | 12 +++++- drivers/vdpa/mlx5/mlx5_vdpa_event.c | 60 +++++++++++++++++++++++++++-- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 6 +-- 4 files changed, 78 insertions(+), 8 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +8 -0
@@ -269,6 +269,7 @@ mlx5_vdpa_dev_close(int vid) }mlx5_vdpa_steer_unset (priv);mlx5_vdpa_virtqs_release (priv); +mlx5_vdpa_drain_cq (priv);if (priv->lm_mr.addr)mlx5_os_wrapped_mkey_destroy (&priv->lm_mr); priv->state =MLX5_VDPA_STATE_PROBED ;
@@ -555,7 +556,14 @@ mlx5_vdpa_virtq_resource_prepare(struct mlx5_vdpa_priv *priv)return 0 ;for (index =0 ; index < (priv->queues *2 ); ++index) {struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index]; +int ret =mlx5_vdpa_event_qp_prepare (priv, priv->queue_size, +-1 , &virtq->eqp); +if (ret) { +DRV_LOG (ERR ,"Failed to create event QPs for virtq %d." , + index); +return -1 ; + }if (priv->caps.queue_counters_valid) {if (!virtq->counters) virtq->counters =
drivers/vdpa/mlx5/mlx5_vdpa.h +11 -1
@@ -55,6 +55,7 @@ struct mlx5_vdpa_event_qp {struct mlx5_vdpa_cq cq;struct mlx5_devx_obj *fw_qp;struct mlx5_devx_qp sw_qp; +uint16_t qp_pi; };struct mlx5_vdpa_query_mr {
@@ -226,7 +227,7 @@ int mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv);* @return * 0 on success, -1 otherwise and rte_errno is set. */ -int mlx5_vdpa_event_qp_create (struct mlx5_vdpa_priv *priv,uint16_t desc_n, +int mlx5_vdpa_event_qp_prepare (struct mlx5_vdpa_priv *priv,uint16_t desc_n,int callfd,struct mlx5_vdpa_event_qp *eqp);/**
@@ -479,4 +480,13 @@ mlx5_vdpa_virtq_stats_get(struct mlx5_vdpa_priv *priv, int qid,*/ int mlx5_vdpa_virtq_stats_reset (struct mlx5_vdpa_priv *priv,int qid); + +/** +* Drain virtq CQ CQE. +* +* @param[in] priv +* The vdpa driver private structure. +*/ +void +mlx5_vdpa_drain_cq (struct mlx5_vdpa_priv *priv);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_event.c +56 -4
100 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +3 -3
@@ -87,6 +87,8 @@ mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv) } virtq->umems[j].size =0 ; } +if (virtq->eqp.fw_qp) +mlx5_vdpa_event_qp_destroy (&virtq->eqp); } }
@@ -117,8 +119,6 @@ mlx5_vdpa_virtq_unset(struct mlx5_vdpa_virtq *virtq)claim_zero (mlx5_devx_cmd_destroy (virtq->virtq)); } virtq->virtq =NULL ; -if (virtq->eqp.fw_qp) -mlx5_vdpa_event_qp_destroy (&virtq->eqp); virtq->notifier_state =MLX5_VDPA_NOTIFIER_STATE_DISABLED ;return 0 ; }
@@ -246,7 +246,7 @@ mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index) MLX5_VIRTQ_EVENT_MODE_QP :MLX5_VIRTQ_EVENT_MODE_NO_MSIX ;if (attr.event_mode ==MLX5_VIRTQ_EVENT_MODE_QP ) { - ret =mlx5_vdpa_event_qp_create (priv, vq.size, vq.callfd, + ret =mlx5_vdpa_event_qp_prepare (priv, vq.size, vq.callfd, &virtq->eqp);if (ret) { DRV_LOG(ERR, "Failed to create event QPs for virtq %d.",
Signature · 1 line
2.31.1
Re: [PATCH v4 04/15] vdpa/mlx5: support event qp reuse
Quoted 16 lines · Li Zhang
From: Yajun Wu <[email protected]>
To speed up queue create time, event qp and cq will create only once. Each virtq creation will reuse same event qp and cq.
Because FW will set event qp to error state during virtq destroy, need modify event qp to RESET state, then modify qp to RTS state as usual. This can save about 1.5ms for each virtq creation.
After SW qp reset, qp pi/ci all become 0 while cq pi/ci keep as previous. Add new variable
qp_cito save SW qp ci. Move qp pi independently with cq ci.Add new function
mlx5_vdpa_drain_cqto drain cq CQE after virtq release.
Quoted 9 lines
Signed-off-by Yajun Wu <[email protected]>Acked-by Matan Azrad <[email protected]>--- drivers/vdpa/mlx5/mlx5_vdpa.c | 8 ++++ drivers/vdpa/mlx5/mlx5_vdpa.h | 12 +++++- drivers/vdpa/mlx5/mlx5_vdpa_event.c | 60 +++++++++++++++++++++++++++-- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 6 +-- 4 files changed, 78 insertions(+), 8 deletions(-)
Thanks,
Maxime
PATCH v4 5/15 common/mlx5: extend virtq modifiable fields
In reply to Li ZhangA virtq configuration can be modified after the virtq creation. Added the following modifiable fields:
1.address fields: desc_addr/used_addr/available_addr 2.hw_available_index
3.hw_used_index
4.virtio_q_type
5.version type
6.queue mkey
7.feature bit mask: tso_ipv4/tso_ipv6/tx_csum/rx_csum 8.event mode: event_mode/event_qpn_or_msix
--- drivers/common/mlx5/mlx5_devx_cmds.c | 70 +++++++++++++++++++++++----- drivers/common/mlx5/mlx5_devx_cmds.h | 6 ++- drivers/common/mlx5/mlx5_prm.h | 13 +++++- 3 files changed, 76 insertions(+), 13 deletions(-)
drivers/common/mlx5/mlx5_devx_cmds.c +59 -11
@@ -545,6 +545,15 @@ mlx5_devx_cmd_query_hca_vdpa_attr(void *ctx, vdpa_attr->log_doorbell_stride =MLX5_GET (virtio_emulation_cap, hcattr, log_doorbell_stride); + vdpa_attr->vnet_modify_ext = +MLX5_GET (virtio_emulation_cap, hcattr, + vnet_modify_ext); + vdpa_attr->virtio_net_q_addr_modify = +MLX5_GET (virtio_emulation_cap, hcattr, + virtio_net_q_addr_modify); + vdpa_attr->virtio_q_index_modify = +MLX5_GET (virtio_emulation_cap, hcattr, + virtio_q_index_modify); vdpa_attr->log_doorbell_bar_size =MLX5_GET (virtio_emulation_cap, hcattr, log_doorbell_bar_size);
@@ -2074,27 +2083,66 @@ mlx5_devx_cmd_modify_virtq(struct mlx5_devx_obj *virtq_obj,MLX5_SET (general_obj_in_cmd_hdr, hdr, obj_type,MLX5_GENERAL_OBJ_TYPE_VIRTQ );MLX5_SET (general_obj_in_cmd_hdr, hdr, obj_id, virtq_obj->id); -MLX5_SET64 (virtio_net_q, virtq, modify_field_select, attr->type); +MLX5_SET64 (virtio_net_q, virtq, modify_field_select, + attr->mod_fields_bitmap);MLX5_SET16 (virtio_q, virtctx, queue_index, attr->queue_index); -switch (attr->type) { -case MLX5_VIRTQ_MODIFY_TYPE_STATE : +if (!attr->mod_fields_bitmap) { +DRV_LOG (ERR ,"Failed to modify VIRTQ for no type set." ); + rte_errno =EINVAL ; +return -rte_errno; + } +if (attr->mod_fields_bitmap &MLX5_VIRTQ_MODIFY_TYPE_STATE )MLX5_SET16 (virtio_net_q, virtq, state, attr->state); -break ; -case MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS : +if (attr->mod_fields_bitmap & +MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS ) {MLX5_SET (virtio_net_q, virtq, dirty_bitmap_mkey, attr->dirty_bitmap_mkey);MLX5_SET64 (virtio_net_q, virtq, dirty_bitmap_addr, attr->dirty_bitmap_addr);MLX5_SET (virtio_net_q, virtq, dirty_bitmap_size, attr->dirty_bitmap_size); -break ; -case MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_DUMP_ENABLE : + } +if (attr->mod_fields_bitmap & +MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_DUMP_ENABLE )MLX5_SET (virtio_net_q, virtq, dirty_bitmap_dump_enable, attr->dirty_bitmap_dump_enable); -break ; -default : - rte_errno =EINVAL ; -return -rte_errno; +if (attr->mod_fields_bitmap &MLX5_VIRTQ_MODIFY_TYPE_QUEUE_PERIOD ) { +MLX5_SET (virtio_q, virtctx, queue_period_mode, + attr->hw_latency_mode); +MLX5_SET (virtio_q, virtctx, queue_period_us, + attr->hw_max_latency_us); +MLX5_SET (virtio_q, virtctx, queue_max_count, + attr->hw_max_pending_comp); + } +if (attr->mod_fields_bitmap &MLX5_VIRTQ_MODIFY_TYPE_ADDR ) { +MLX5_SET64 (virtio_q, virtctx, desc_addr, attr->desc_addr); +MLX5_SET64 (virtio_q, virtctx, used_addr, attr->used_addr); +MLX5_SET64 (virtio_q, virtctx, available_addr, + attr->available_addr); + } +if (attr->mod_fields_bitmap &MLX5_VIRTQ_MODIFY_TYPE_HW_AVAILABLE_INDEX ) +MLX5_SET16 (virtio_net_q, virtq, hw_available_index, + attr->hw_available_index); +if (attr->mod_fields_bitmap &MLX5_VIRTQ_MODIFY_TYPE_HW_USED_INDEX ) +MLX5_SET16 (virtio_net_q, virtq, hw_used_index, + attr->hw_used_index); +if (attr->mod_fields_bitmap &MLX5_VIRTQ_MODIFY_TYPE_Q_TYPE ) +MLX5_SET16 (virtio_q, virtctx, virtio_q_type, attr->q_type); +if (attr->mod_fields_bitmap &MLX5_VIRTQ_MODIFY_TYPE_VERSION_1_0 ) +MLX5_SET16 (virtio_q, virtctx, virtio_version_1_0, + attr->virtio_version_1_0); +if (attr->mod_fields_bitmap &MLX5_VIRTQ_MODIFY_TYPE_Q_MKEY ) +MLX5_SET (virtio_q, virtctx, virtio_q_mkey, attr->mkey); +if (attr->mod_fields_bitmap & +MLX5_VIRTQ_MODIFY_TYPE_QUEUE_FEATURE_BIT_MASK ) { +MLX5_SET16 (virtio_net_q, virtq, tso_ipv4, attr->tso_ipv4); +MLX5_SET16 (virtio_net_q, virtq, tso_ipv6, attr->tso_ipv6); +MLX5_SET16 (virtio_net_q, virtq, tx_csum, attr->tx_csum); +MLX5_SET16 (virtio_net_q, virtq, rx_csum, attr->rx_csum); + } +if (attr->mod_fields_bitmap &MLX5_VIRTQ_MODIFY_TYPE_EVENT_MODE ) { +MLX5_SET16 (virtio_q, virtctx, event_mode, attr->event_mode); +MLX5_SET (virtio_q, virtctx, event_qpn_or_msix, attr->qp_id); } ret = mlx5_glue->devx_obj_modify (virtq_obj->obj, in,sizeof (in), out,sizeof (out));
drivers/common/mlx5/mlx5_devx_cmds.h +5 -1
@@ -74,6 +74,9 @@ struct mlx5_hca_vdpa_attr { uint32_t log_doorbell_stride:5; uint32_t log_doorbell_bar_size:5; uint32_t queue_counters_valid:1; + uint32_t vnet_modify_ext:1; + uint32_t virtio_net_q_addr_modify:1; + uint32_t virtio_q_index_modify:1;uint32_t max_num_virtio_queues;struct {uint32_t a;
@@ -465,7 +468,7 @@ struct mlx5_devx_virtq_attr {uint32_t tis_id;uint32_t counters_obj_id;uint64_t dirty_bitmap_addr; -uint64_t type; +uint64_t mod_fields_bitmap;uint64_t desc_addr;uint64_t used_addr;uint64_t available_addr;
@@ -475,6 +478,7 @@ struct mlx5_devx_virtq_attr {uint64_t offset; } umems[3 ];uint8_t error_type; +uint8_t q_type; };
drivers/common/mlx5/mlx5_prm.h +12 -1
@@ -1802,7 +1802,9 @@ struct mlx5_ifc_virtio_emulation_cap_bits {u8 virtio_queue_type[0x8 ];u8 reserved_at_20[0x13 ];u8 log_doorbell_stride[0x5 ]; -u8 reserved_at_3b[0x3 ]; +u8 vnet_modify_ext[0x1 ]; +u8 virtio_net_q_addr_modify[0x1 ]; +u8 virtio_q_index_modify[0x1 ];u8 log_doorbell_bar_size[0x5 ];u8 doorbell_bar_offset[0x40 ];u8 reserved_at_80[0x8 ];
@@ -3024,6 +3026,15 @@ enum {MLX5_VIRTQ_MODIFY_TYPE_STATE = (1UL <<0 ),MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS = (1UL <<3 ),MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_DUMP_ENABLE = (1UL <<4 ), +MLX5_VIRTQ_MODIFY_TYPE_QUEUE_PERIOD = (1UL <<5 ), +MLX5_VIRTQ_MODIFY_TYPE_ADDR = (1UL <<6 ), +MLX5_VIRTQ_MODIFY_TYPE_HW_AVAILABLE_INDEX = (1UL <<7 ), +MLX5_VIRTQ_MODIFY_TYPE_HW_USED_INDEX = (1UL <<8 ), +MLX5_VIRTQ_MODIFY_TYPE_Q_TYPE = (1UL <<9 ), +MLX5_VIRTQ_MODIFY_TYPE_VERSION_1_0 = (1UL <<10 ), +MLX5_VIRTQ_MODIFY_TYPE_Q_MKEY = (1UL <<11 ), +MLX5_VIRTQ_MODIFY_TYPE_QUEUE_FEATURE_BIT_MASK = (1UL <<12 ), + MLX5_VIRTQ_MODIFY_TYPE_EVENT_MODE = (1UL << 13), }; struct mlx5_ifc_virtio_q_bits {
Signature · 1 line
2.31.1
Re: [PATCH v4 05/15] common/mlx5: extend virtq modifiable fields
Quoted 11 lines · Li Zhang
A virtq configuration can be modified after the virtq creation. Added the following modifiable fields:
1.address fields: desc_addr/used_addr/available_addr 2.hw_available_index
3.hw_used_index
4.virtio_q_type
5.version type
6.queue mkey
7.feature bit mask: tso_ipv4/tso_ipv6/tx_csum/rx_csum 8.event mode: event_mode/event_qpn_or_msix
Quoted 8 lines
Signed-off-by Li Zhang <[email protected]>Acked-by Matan Azrad <[email protected]>--- drivers/common/mlx5/mlx5_devx_cmds.c | 70 +++++++++++++++++++++++----- drivers/common/mlx5/mlx5_devx_cmds.h | 6 ++- drivers/common/mlx5/mlx5_prm.h | 13 +++++- 3 files changed, 76 insertions(+), 13 deletions(-)
Thanks,
Maxime
PATCH v4 6/15 vdpa/mlx5: pre-create virtq at probe time
In reply to Li Zhangdev_config operation is called in LM progress. LM time is very critical because all
the VM packets are dropped directly at that time.
Move the virtq creation to probe time and only modify the configuration later in the dev_config stage using the new ability to modify virtq.
This optimization accelerates the LM process and reduces its time by 70%.
--- doc/guides/rel_notes/release_22_07.rst | 4 + drivers/vdpa/mlx5/mlx5_vdpa.h | 4 + drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 19 +- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 257 +++++++++++++++---------- 4 files changed, 176 insertions(+), 108 deletions(-)
doc/guides/rel_notes/release_22_07.rst +4 -0
@@ -175,6 +175,10 @@ New Features This is a fall-back implementation for platforms that don't support vector operations. +* **Updated Nvidia mlx5 vDPA driver.** + + * Added new devargs ``queue_size`` and ``queues`` to allow prior creation of virtq resources. + Removed Items -------------
drivers/vdpa/mlx5/mlx5_vdpa.h +4 -0
@@ -80,6 +80,7 @@ struct mlx5_vdpa_virtq {uint16_t vq_size;uint8_t notifier_state;bool stopped; + uint32_t configured:1;uint32_t version;struct mlx5_vdpa_priv *priv;struct mlx5_devx_obj *virtq;
@@ -489,4 +490,7 @@ mlx5_vdpa_virtq_stats_reset(struct mlx5_vdpa_priv *priv, int qid);*/ void mlx5_vdpa_drain_cq (struct mlx5_vdpa_priv *priv); + +bool +mlx5_vdpa_is_modify_virtq_supported (struct mlx5_vdpa_priv *priv);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_lm.c +11 -8
@@ -12,20 +12,21 @@ int mlx5_vdpa_logging_enable(struct mlx5_vdpa_priv *priv, int enable) {struct mlx5_devx_virtq_attr attr = { - .type =MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_DUMP_ENABLE , + .mod_fields_bitmap = +MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_DUMP_ENABLE , .dirty_bitmap_dump_enable = enable, }; +struct mlx5_vdpa_virtq *virtq;int i;for (i =0 ; i < priv->nr_virtqs; ++i) { attr.queue_index = i; -if (!priv->virtqs[i].virtq) { -DRV_LOG (DEBUG ,"virtq %d is invalid for dirty bitmap " -"enabling." , i); + virtq = &priv->virtqs[i]; +if (!virtq->configured) { +DRV_LOG (DEBUG ,"virtq %d is invalid for dirty bitmap enabling." , i); }else if (mlx5_devx_cmd_modify_virtq (priv->virtqs[i].virtq, &attr)) { -DRV_LOG (ERR ,"Failed to modify virtq %d for dirty " -"bitmap enabling." , i); +DRV_LOG (ERR ,"Failed to modify virtq %d for dirty bitmap enabling." , i);return -1 ; } }
@@ -37,10 +38,11 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base, uint64_t log_size) {struct mlx5_devx_virtq_attr attr = { - .type =MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS , + .mod_fields_bitmap =MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS , .dirty_bitmap_addr = log_base, .dirty_bitmap_size = log_size, }; +struct mlx5_vdpa_virtq *virtq;int i; int ret = mlx5_os_wrapped_mkey_create(priv->cdev->ctx, priv->cdev->pd, priv->cdev->pdn,
@@ -54,7 +56,8 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base, attr.dirty_bitmap_mkey = priv->lm_mr.lkey;for (i =0 ; i < priv->nr_virtqs; ++i) { attr.queue_index = i; -if (!priv->virtqs[i].virtq) { + virtq = &priv->virtqs[i]; +if (!virtq->configured) {DRV_LOG (DEBUG ,"virtq %d is invalid for LM." , i); }else if (mlx5_devx_cmd_modify_virtq (priv->virtqs[i].virtq, &attr)) {
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +157 -100
347 lines not on this page · raw message
Signature · 1 line
2.31.1
PATCH v4 7/15 vdpa/mlx5: optimize datapath-control synchronization
In reply to Li ZhangThe driver used a single global lock for any synchronization needed for the datapath and control path.
It is better to group the critical sections with the other ones that should be synchronized.
Replace the global lock with the following locks:
1.virtq locks(per virtq) synchronize datapath polling and
parallel configurations on the same virtq.
2.A doorbell lock synchronizes doorbell update,
which is shared for all the virtqs in the device.
3.A steering lock for the shared steering objects updates.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 24 ++++--- drivers/vdpa/mlx5/mlx5_vdpa.h | 13 ++-- drivers/vdpa/mlx5/mlx5_vdpa_event.c | 97 ++++++++++++++++++----------- drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 36 ++++++++--- drivers/vdpa/mlx5/mlx5_vdpa_steer.c | 7 ++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 88 +++++++++++++++++++------- 6 files changed, 186 insertions(+), 79 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +16 -8
@@ -135,6 +135,7 @@ mlx5_vdpa_set_vring_state(int vid, int vring, int state)struct rte_vdpa_device *vdev =rte_vhost_get_vdpa_device (vid);struct mlx5_vdpa_priv *priv =mlx5_vdpa_find_priv_resource_by_vdev (vdev); +struct mlx5_vdpa_virtq *virtq;int ret;if (priv ==NULL ) {
@@ -145,9 +146,10 @@ mlx5_vdpa_set_vring_state(int vid, int vring, int state)DRV_LOG (ERR ,"Too big vring id: %d." , vring);return -E2BIG ; } -pthread_mutex_lock (&priv->vq_config_lock); + virtq = &priv->virtqs[vring]; +pthread_mutex_lock (&virtq->virtq_lock); ret =mlx5_vdpa_virtq_enable (priv, vring, state); -pthread_mutex_unlock (&priv->vq_config_lock); +pthread_mutex_unlock (&virtq->virtq_lock);return ret; }
@@ -267,7 +269,9 @@ mlx5_vdpa_dev_close(int vid) ret |=mlx5_vdpa_lm_log (priv); priv->state =MLX5_VDPA_STATE_IN_PROGRESS ; } +pthread_mutex_lock (&priv->steer_update_lock);mlx5_vdpa_steer_unset (priv); +pthread_mutex_unlock (&priv->steer_update_lock);mlx5_vdpa_virtqs_release (priv);mlx5_vdpa_drain_cq (priv);if (priv->lm_mr.addr)
@@ -276,8 +280,6 @@ mlx5_vdpa_dev_close(int vid)if (!priv->connected)mlx5_vdpa_dev_cache_clean (priv); priv->vid =0 ; -/* The mutex may stay locked after event thread cancel - initiate it. */ -pthread_mutex_init (&priv->vq_config_lock,NULL );DRV_LOG (INFO ,"vDPA device %d was closed." , vid);return ret; }
@@ -549,15 +551,21 @@ mlx5_vdpa_config_get(struct mlx5_kvargs_ctrl *mkvlist,static int mlx5_vdpa_virtq_resource_prepare (struct mlx5_vdpa_priv *priv) { +struct mlx5_vdpa_virtq *virtq;uint32_t index;uint32_t i; +for (index =0 ; index < priv->caps.max_num_virtio_queues *2 ; + index++) { + virtq = &priv->virtqs[index]; +pthread_mutex_init (&virtq->virtq_lock,NULL ); + }if (!priv->queues)return 0 ;for (index =0 ; index < (priv->queues *2 ); ++index) { -struct mlx5_vdpa_virtq *virtq = &priv->virtqs[index]; + virtq = &priv->virtqs[index];int ret =mlx5_vdpa_event_qp_prepare (priv, priv->queue_size, --1 , &virtq->eqp); +-1 , virtq);if (ret) { DRV_LOG(ERR, "Failed to create event QPs for virtq %d.",
@@ -713,7 +721,8 @@ mlx5_vdpa_dev_probe(struct mlx5_common_device *cdev, priv->num_lag_ports = attr->num_lag_ports;if (attr->num_lag_ports ==0 ) priv->num_lag_ports =1 ; -pthread_mutex_init (&priv->vq_config_lock,NULL ); +rte_spinlock_init (&priv->db_lock); +pthread_mutex_init (&priv->steer_update_lock,NULL ); priv->cdev = cdev;mlx5_vdpa_config_get (mkvlist, priv);if (mlx5_vdpa_create_dev_resources (priv))
@@ -797,7 +806,6 @@ mlx5_vdpa_dev_release(struct mlx5_vdpa_priv *priv)mlx5_vdpa_release_dev_resources (priv);if (priv->vdev)rte_vdpa_unregister_device (priv->vdev); -pthread_mutex_destroy (&priv->vq_config_lock);rte_free (priv); }
drivers/vdpa/mlx5/mlx5_vdpa.h +8 -5
@@ -82,6 +82,7 @@ struct mlx5_vdpa_virtq {bool stopped; uint32_t configured:1;uint32_t version; +pthread_mutex_t virtq_lock;struct mlx5_vdpa_priv *priv;struct mlx5_devx_obj *virtq;struct mlx5_devx_obj *counters;
@@ -126,7 +127,8 @@ struct mlx5_vdpa_priv {TAILQ_ENTRY (mlx5_vdpa_priv) next;bool connected;enum mlx5_dev_state state; -pthread_mutex_t vq_config_lock; +rte_spinlock_t db_lock; +pthread_mutex_t steer_update_lock;uint64_t no_traffic_counter;pthread_t timer_tid;int event_mode;
@@ -222,14 +224,15 @@ int mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv);* Number of descriptors. * @param[in] callfd * The guest notification file descriptor. -* @param[in/out] eqp -* Pointer to the event QP structure. +* @param[in/out] virtq +* Pointer to the virt-queue structure. * * @return * 0 on success, -1 otherwise and rte_errno is set. */ -int mlx5_vdpa_event_qp_prepare (struct mlx5_vdpa_priv *priv,uint16_t desc_n, -int callfd,struct mlx5_vdpa_event_qp *eqp); +int +mlx5_vdpa_event_qp_prepare (struct mlx5_vdpa_priv *priv,uint16_t desc_n, +int callfd,struct mlx5_vdpa_virtq *virtq);/** * Destroy an event QP and all its related resources.
drivers/vdpa/mlx5/mlx5_vdpa_event.c +62 -35
232 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_lm.c +29 -7
@@ -24,10 +24,19 @@ mlx5_vdpa_logging_enable(struct mlx5_vdpa_priv *priv, int enable) virtq = &priv->virtqs[i];if (!virtq->configured) {DRV_LOG (DEBUG ,"virtq %d is invalid for dirty bitmap enabling." , i); - }else if (mlx5_devx_cmd_modify_virtq (priv->virtqs[i].virtq, + }else { +struct mlx5_vdpa_virtq *virtq = &priv->virtqs[i]; + +pthread_mutex_lock (&virtq->virtq_lock); +if (mlx5_devx_cmd_modify_virtq (priv->virtqs[i].virtq, &attr)) { -DRV_LOG (ERR ,"Failed to modify virtq %d for dirty bitmap enabling." , i); -return -1 ; +pthread_mutex_unlock (&virtq->virtq_lock); +DRV_LOG (ERR , +"Failed to modify virtq %d for dirty bitmap enabling." , + i); +return -1 ; + } +pthread_mutex_unlock (&virtq->virtq_lock); } }return 0 ;
@@ -59,10 +68,19 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base, virtq = &priv->virtqs[i];if (!virtq->configured) {DRV_LOG (DEBUG ,"virtq %d is invalid for LM." , i); - }else if (mlx5_devx_cmd_modify_virtq (priv->virtqs[i].virtq, - &attr)) { -DRV_LOG (ERR ,"Failed to modify virtq %d for LM." , i); - goto err; + }else { +struct mlx5_vdpa_virtq *virtq = &priv->virtqs[i]; + +pthread_mutex_lock (&virtq->virtq_lock); +if (mlx5_devx_cmd_modify_virtq ( + priv->virtqs[i].virtq, + &attr)) { +pthread_mutex_unlock (&virtq->virtq_lock); +DRV_LOG (ERR , +"Failed to modify virtq %d for LM." , i); + goto err; + } +pthread_mutex_unlock (&virtq->virtq_lock); } }return 0 ;
@@ -77,6 +95,7 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,int mlx5_vdpa_lm_log (struct mlx5_vdpa_priv *priv) { +struct mlx5_vdpa_virtq *virtq;uint64_t features;int ret =rte_vhost_get_negotiated_features (priv->vid, &features);int i;
@@ -88,10 +107,13 @@ mlx5_vdpa_lm_log(struct mlx5_vdpa_priv *priv)if (!RTE_VHOST_NEED_LOG (features))return 0 ;for (i =0 ; i < priv->nr_virtqs; ++i) { + virtq = &priv->virtqs[i];if (!priv->virtqs[i].virtq) {DRV_LOG (DEBUG ,"virtq %d is invalid for LM log." , i); }else { +pthread_mutex_lock (&virtq->virtq_lock); ret =mlx5_vdpa_virtq_stop (priv, i); +pthread_mutex_unlock (&virtq->virtq_lock);if (ret) {DRV_LOG (ERR ,"Failed to stop virtq %d for LM " "log." , i);
drivers/vdpa/mlx5/mlx5_vdpa_steer.c +6 -1
@@ -237,19 +237,24 @@ mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv *priv)int mlx5_vdpa_steer_update (struct mlx5_vdpa_priv *priv) { -int ret =mlx5_vdpa_rqt_prepare (priv); +int ret; +pthread_mutex_lock (&priv->steer_update_lock); + ret =mlx5_vdpa_rqt_prepare (priv);if (ret ==0 ) {mlx5_vdpa_steer_unset (priv); }else if (ret <0 ) { +pthread_mutex_unlock (&priv->steer_update_lock);return ret; }else if (!priv->steer.rss[0 ].flow) { ret =mlx5_vdpa_rss_flows_create (priv);if (ret) {DRV_LOG (ERR ,"Cannot create RSS flows." ); +pthread_mutex_unlock (&priv->steer_update_lock);return -1 ; } } +pthread_mutex_unlock (&priv->steer_update_lock);return 0 ; }
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +65 -23
173 lines not on this page · raw message
Signature · 1 line
2.31.1
Re: [PATCH v4 07/15] vdpa/mlx5: optimize datapath-control synchronization
Quoted 15 lines · Li Zhang
The driver used a single global lock for any synchronization needed for the datapath and control path.
It is better to group the critical sections with the other ones that should be synchronized.Replace the global lock with the following locks:
1.virtq locks(per virtq) synchronize datapath polling and
parallel configurations on the same virtq.
2.A doorbell lock synchronizes doorbell update,
which is shared for all the virtqs in the device.
3.A steering lock for the shared steering objects updates.Signed-off-by Li Zhang <[email protected]>Acked-by Matan Azrad <[email protected]>
Quoted 9 lines
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 24 ++++--- drivers/vdpa/mlx5/mlx5_vdpa.h | 13 ++-- drivers/vdpa/mlx5/mlx5_vdpa_event.c | 97 ++++++++++++++++++----------- drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 36 ++++++++--- drivers/vdpa/mlx5/mlx5_vdpa_steer.c | 7 ++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 88 +++++++++++++++++++------- 6 files changed, 186 insertions(+), 79 deletions(-)
Thanks,
Maxime
PATCH v4 8/15 vdpa/mlx5: add multi-thread management for configuration
In reply to Li ZhangThe LM process includes a lot of objects creations and destructions in the source and the destination servers. As much as LM time increases, the packet drop of the VM increases. To improve LM time need to parallel the configurations for mlx5 FW. Add internal multi-thread management in the driver for it.
A new devarg defines the number of threads and their CPU. The management is shared between all the devices of the driver. Since the event_core also affects the datapath events thread, reduce the priority of the datapath event thread to allow fast configuration of the devices doing the LM.
--- doc/guides/vdpadevs/mlx5.rst | 11 +++ drivers/vdpa/mlx5/meson.build | 1 + drivers/vdpa/mlx5/mlx5_vdpa.c | 41 ++++++++ drivers/vdpa/mlx5/mlx5_vdpa.h | 36 +++++++ drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 129 ++++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 2 +- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 8 +- 7 files changed, 223 insertions(+), 5 deletions(-) create mode 100644 drivers/vdpa/mlx5/mlx5_vdpa_cthread.c
doc/guides/vdpadevs/mlx5.rst +11 -0
@@ -78,6 +78,17 @@ for an additional list of options shared with other mlx5 drivers. CPU core number to set polling thread affinity to, default to control plane cpu. +- ``max_conf_threads`` parameter [int] + + Allow the driver to use internal threads to obtain fast configuration. + All the threads will be open on the same core of the event completion queue scheduling thread. + + - 0, default, don't use internal threads for configuration. + + - 1 - 256, number of internal threads in addition to the caller thread (8 is suggested). + This value, if not 0, should be the same for all the devices; + the first prob will take it with the event_core for all the multi-thread configurations in the driver. + - ``hw_latency_mode`` parameter [int] The completion queue moderation mode:
drivers/vdpa/mlx5/meson.build +1 -0
@@ -15,6 +15,7 @@ sources = files( 'mlx5_vdpa_virtq.c', 'mlx5_vdpa_steer.c', 'mlx5_vdpa_lm.c', + 'mlx5_vdpa_cthread.c', ) cflags_options = [ '-std=c11',
drivers/vdpa/mlx5/mlx5_vdpa.c +41 -0
77 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa.h +36 -0
@@ -73,6 +73,22 @@ enum {MLX5_VDPA_NOTIFIER_STATE_ERR }; +#define MLX5_VDPA_MAX_C_THRD 256 + +/* Generic mlx5_vdpa_c_thread information. */ +struct mlx5_vdpa_c_thread { +pthread_t tid; +}; + +struct mlx5_vdpa_conf_thread_mng { +void *initializer_priv; +uint32_t refcnt; +uint32_t max_thrds; +pthread_mutex_t cthrd_lock; +struct mlx5_vdpa_c_thread cthrd[MLX5_VDPA_MAX_C_THRD ]; +}; +extern struct mlx5_vdpa_conf_thread_mng conf_thread_mng; +struct mlx5_vdpa_virtq {SLIST_ENTRY (mlx5_vdpa_virtq) next;uint8_t enable;
@@ -126,6 +142,7 @@ enum mlx5_dev_state {struct mlx5_vdpa_priv {TAILQ_ENTRY (mlx5_vdpa_priv) next;bool connected; +bool use_c_thread;enum mlx5_dev_state state;rte_spinlock_t db_lock;pthread_mutex_t steer_update_lock;
@@ -496,4 +513,23 @@ mlx5_vdpa_drain_cq(struct mlx5_vdpa_priv *priv);bool mlx5_vdpa_is_modify_virtq_supported (struct mlx5_vdpa_priv *priv); + +/** +* Create configuration multi-threads resource +* +* @param[in] cpu_core +* CPU core number to set configuration threads affinity to. +* +* @return +* 0 on success, a negative value otherwise. +*/ +int +mlx5_vdpa_mult_threads_create (int cpu_core); + +/** +* Destroy configuration multi-threads resource +* +*/ +void +mlx5_vdpa_mult_threads_destroy (bool need_unlock);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c new file +129 -0
129 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_event.c +1 -1
@@ -507,7 +507,7 @@ mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv)pthread_attr_t attr;char name[16 ];const struct sched_param sp = { - .sched_priority =sched_get_priority_max (SCHED_RR ), + .sched_priority =sched_get_priority_max (SCHED_RR ) -1 , };if (!priv->eventc)
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +4 -4
@@ -43,7 +43,7 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg) errno ==EWOULDBLOCK || errno == EAGAIN) continue; -DRV_LOG (ERR ,"Failed to read kickfd of virtq %d: %s" , +DRV_LOG (ERR ,"Failed to read kickfd of virtq %d: %s." , virtq->index,strerror (errno)); }break ;
@@ -57,7 +57,7 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg)rte_spinlock_unlock (&priv->db_lock);pthread_mutex_unlock (&virtq->virtq_lock);if (priv->state !=MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) { -DRV_LOG (ERR ,"device %d queue %d down, skip kick handling" , +DRV_LOG (ERR ,"device %d queue %d down, skip kick handling." , priv->vid, virtq->index);return ; }
@@ -218,7 +218,7 @@ mlx5_vdpa_virtq_query(struct mlx5_vdpa_priv *priv, int index)return -1 ; }if (attr.state ==MLX5_VIRTQ_STATE_ERROR ) -DRV_LOG (WARNING ,"vid %d vring %d hw error=%hhu" , +DRV_LOG (WARNING ,"vid %d vring %d hw error=%hhu." , priv->vid, index, attr.error_type);return 0 ; }
@@ -380,7 +380,7 @@ mlx5_vdpa_virtq_sub_objs_prepare(struct mlx5_vdpa_priv *priv,if (ret) { last_avail_idx =0 ; last_used_idx =0 ; -DRV_LOG (WARNING ,"Couldn't get vring base, idx are set to 0" ); +DRV_LOG (WARNING ,"Couldn't get vring base, idx are set to 0." ); }else { DRV_LOG(INFO, "vid %d: Init last_avail_idx=%d, last_used_idx=%d for " "virtq %d.", priv->vid, last_avail_idx,
Signature · 1 line
2.31.1
Re: [PATCH v4 08/15] vdpa/mlx5: add multi-thread management for configuration
Quoted 23 lines · Li Zhang
The LM process includes a lot of objects creations and destructions in the source and the destination servers. As much as LM time increases, the packet drop of the VM increases. To improve LM time need to parallel the configurations for mlx5 FW. Add internal multi-thread management in the driver for it.
A new devarg defines the number of threads and their CPU. The management is shared between all the devices of the driver. Since the
event_corealso affects the datapath events thread, reduce the priority of the datapath event thread to allow fast configuration of the devices doing the LM.Signed-off-by Li Zhang <[email protected]>Acked-by Matan Azrad <[email protected]>--- doc/guides/vdpadevs/mlx5.rst | 11 +++ drivers/vdpa/mlx5/meson.build | 1 + drivers/vdpa/mlx5/mlx5_vdpa.c | 41 ++++++++ drivers/vdpa/mlx5/mlx5_vdpa.h | 36 +++++++ drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 129 ++++++++++++++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 2 +- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 8 +- 7 files changed, 223 insertions(+), 5 deletions(-)
Quoted 2 lines
create mode 100644
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c
Thanks,
Maxime
PATCH v4 9/15 vdpa/mlx5: add task ring for MT management
In reply to Li ZhangThe configuration threads tasks need a container to support multiple tasks assigned to a thread in parallel. Use rte_ring container per thread to manage
the thread tasks without locks.
The caller thread from the user context opens a task to a thread and enqueue it to the thread ring.
The thread polls its ring and dequeue tasks. That’s why the ring should be in multi-producer and single consumer mode.
Anatomic counter manages the tasks completion notification. The threads report errors to the caller by
a dedicated error counter per task.
--- drivers/vdpa/mlx5/mlx5_vdpa.h | 17 ++++ drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 115 +++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 2 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.h +17 -0
@@ -74,10 +74,22 @@ enum { };#define MLX5_VDPA_MAX_C_THRD 256 +#define MLX5_VDPA_MAX_TASKS_PER_THRD 4096 +#define MLX5_VDPA_TASKS_PER_DEV 64 + +/* Generic task information and size must be multiple of 4B. */ +struct mlx5_vdpa_task { +struct mlx5_vdpa_priv *priv; +uint32_t *remaining_cnt; +uint32_t *err_cnt; +uint32_t idx; +} __rte_packed __rte_aligned(4);/* Generic mlx5_vdpa_c_thread information. */ struct mlx5_vdpa_c_thread {pthread_t tid; +struct rte_ring *rng; +pthread_cond_t c_cond; };struct mlx5_vdpa_conf_thread_mng {
@@ -532,4 +544,9 @@ mlx5_vdpa_mult_threads_create(int cpu_core);*/ void mlx5_vdpa_mult_threads_destroy (bool need_unlock); + +bool +mlx5_vdpa_task_add (struct mlx5_vdpa_priv *priv, +uint32_t thrd_idx, +uint32_t num);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +113 -2
156 lines not on this page · raw message
Signature · 1 line
2.31.1
Re: [PATCH v4 09/15] vdpa/mlx5: add task ring for MT management
Quoted 13 lines · Li Zhang
The configuration threads tasks need a container to support multiple tasks assigned to a thread in parallel. Use
rte_ringcontainer per thread to manage
the thread tasks without locks.
The caller thread from the user context opens a task to a thread and enqueue it to the thread ring.
The thread polls its ring and dequeue tasks. That’s why the ring should be in multi-producer and single consumer mode.
Anatomic counter manages the tasks completion notification. The threads report errors to the caller by
a dedicated error counter per task.
Quoted 7 lines
Signed-off-by Li Zhang <[email protected]>Acked-by Matan Azrad <[email protected]>--- drivers/vdpa/mlx5/mlx5_vdpa.h | 17 ++++ drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 115 +++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 2 deletions(-)
Thanks,
Maxime
PATCH v4 10/15 vdpa/mlx5: add MT task for VM memory registration
In reply to Li ZhangThe driver creates a direct MR object of
the HW for each VM memory region,
which maps the VM physical address to
the actual physical address.
Later, after all the MRs are ready,
the driver creates an indirect MR to group all the direct MRs into one virtual space from the HW perspective.
Create direct MRs in parallel using the MT mechanism. After completion, the primary thread creates the indirect MR needed for the following virtqs configurations.
This optimization accelerrate the LM process and
reduce its time by 5%.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 1 - drivers/vdpa/mlx5/mlx5_vdpa.h | 31 ++- drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 47 ++++- drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 270 ++++++++++++++++++-------- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 6 +- 5 files changed, 258 insertions(+), 97 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +0 -1
@@ -768,7 +768,6 @@ mlx5_vdpa_dev_probe(struct mlx5_common_device *cdev, rte_errno = rte_errno ? rte_errno :EINVAL ; goto error; } -SLIST_INIT (&priv->mr_list);pthread_mutex_lock (&priv_list_lock);TAILQ_INSERT_TAIL (&priv_list, priv, next);pthread_mutex_unlock (&priv_list_lock);
drivers/vdpa/mlx5/mlx5_vdpa.h +27 -4
@@ -59,7 +59,6 @@ struct mlx5_vdpa_event_qp { };struct mlx5_vdpa_query_mr { -SLIST_ENTRY (mlx5_vdpa_query_mr) next;union {struct ibv_mr *mr;struct mlx5_devx_obj *mkey;
@@ -76,10 +75,17 @@ enum {#define MLX5_VDPA_MAX_C_THRD 256#define MLX5_VDPA_MAX_TASKS_PER_THRD 4096#define MLX5_VDPA_TASKS_PER_DEV 64 +#define MLX5_VDPA_MAX_MRS 0xFFFF + +/* Vdpa task types. */ +enum mlx5_vdpa_task_type { +MLX5_VDPA_TASK_REG_MR =1 , +};/* Generic task information and size must be multiple of 4B. */ struct mlx5_vdpa_task {struct mlx5_vdpa_priv *priv; +enum mlx5_vdpa_task_type type;uint32_t *remaining_cnt;uint32_t *err_cnt;uint32_t idx;
@@ -101,6 +107,14 @@ struct mlx5_vdpa_conf_thread_mng { };extern struct mlx5_vdpa_conf_thread_mng conf_thread_mng; +struct mlx5_vdpa_vmem_info { +struct rte_vhost_memory *vmem; +uint32_t entries_num; +uint64_t gcd; +uint64_t size; +uint8_t mode; +}; +struct mlx5_vdpa_virtq {SLIST_ENTRY (mlx5_vdpa_virtq) next;uint8_t enable;
@@ -176,7 +190,7 @@ struct mlx5_vdpa_priv {struct mlx5_hca_vdpa_attr caps;uint32_t gpa_mkey_index;struct ibv_mr *null_mr; -struct rte_vhost_memory *vmem; +struct mlx5_vdpa_vmem_info vmem_info;struct mlx5dv_devx_event_channel *eventc;struct mlx5dv_devx_event_channel *err_chnl;struct mlx5_uar uar;
@@ -187,11 +201,13 @@ struct mlx5_vdpa_priv {uint8_t num_lag_ports;uint64_t features;/* Negotiated features. */ uint16_t log_max_rqt_size; +uint16_t last_c_thrd_idx; +uint16_t num_mrs;/* Number of memory regions. */ struct mlx5_vdpa_steer steer;struct mlx5dv_var *var;void *virtq_db_addr;struct mlx5_pmd_wrapped_mr lm_mr; - SLIST_HEAD(mr_list, mlx5_vdpa_query_mr) mr_list; +struct mlx5_vdpa_query_mr **mrs;struct mlx5_vdpa_virtq virtqs[]; };
@@ -548,5 +564,12 @@ mlx5_vdpa_mult_threads_destroy(bool need_unlock);bool mlx5_vdpa_task_add (struct mlx5_vdpa_priv *priv,uint32_t thrd_idx, -uint32_t num); +enum mlx5_vdpa_task_type task_type, +uint32_t *bulk_refcnt,uint32_t *bulk_err_cnt, +void **task_data,uint32_t num); +int +mlx5_vdpa_register_mr (struct mlx5_vdpa_priv *priv,uint32_t idx); +bool +mlx5_vdpa_c_thread_wait_bulk_tasks_done (uint32_t *remaining_cnt, +uint32_t *err_cnt,uint32_t sleep_time);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +44 -3
85 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_mem.c +184 -86
317 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +3 -3
@@ -353,21 +353,21 @@ mlx5_vdpa_virtq_sub_objs_prepare(struct mlx5_vdpa_priv *priv, } }if (attr->q_type ==MLX5_VIRTQ_TYPE_SPLIT ) { - gpa =mlx5_vdpa_hva_to_gpa (priv->vmem, + gpa =mlx5_vdpa_hva_to_gpa (priv->vmem_info.vmem, (uint64_t )(uintptr_t )vq->desc);if (!gpa) {DRV_LOG (ERR ,"Failed to get descriptor ring GPA." );return -1 ; } attr->desc_addr = gpa; - gpa =mlx5_vdpa_hva_to_gpa (priv->vmem, + gpa =mlx5_vdpa_hva_to_gpa (priv->vmem_info.vmem, (uint64_t )(uintptr_t )vq->used);if (!gpa) {DRV_LOG (ERR ,"Failed to get GPA for used ring." );return -1 ; } attr->used_addr = gpa; - gpa =mlx5_vdpa_hva_to_gpa (priv->vmem, + gpa =mlx5_vdpa_hva_to_gpa (priv->vmem_info.vmem, (uint64_t )(uintptr_t )vq->avail);if (!gpa) {DRV_LOG (ERR ,"Failed to get GPA for available ring." );
Signature · 1 line
2.31.1
Re: [PATCH v4 10/15] vdpa/mlx5: add MT task for VM memory registration
Quoted 18 lines · Li Zhang
The driver creates a direct MR object of
the HW for each VM memory region,
which maps the VM physical address to
the actual physical address.Later, after all the MRs are ready,
the driver creates an indirect MR to group all the direct MRs into one virtual space from the HW perspective.Create direct MRs in parallel using the MT mechanism. After completion, the primary thread creates the indirect MR needed for the following virtqs configurations.
This optimization accelerrate the LM process and
reduce its time by 5%.Signed-off-by Li Zhang <[email protected]>Acked-by Matan Azrad <[email protected]>
Quoted 8 lines
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 1 - drivers/vdpa/mlx5/mlx5_vdpa.h | 31 ++- drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 47 ++++- drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 270 ++++++++++++++++++-------- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 6 +- 5 files changed, 258 insertions(+), 97 deletions(-)
Thanks,
Maxime
PATCH v4 11/15 vdpa/mlx5: add virtq creation task for MT management
In reply to Li ZhangThe virtq object and all its sub-resources use a lot of FW commands and can be accelerated by the MT management. Split the virtqs creation between the configuration threads. This accelerates the LM process and reduces its time by 20%.
--- drivers/vdpa/mlx5/mlx5_vdpa.h | 9 +- drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 14 +++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 2 +- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 149 +++++++++++++++++++------- 4 files changed, 134 insertions(+), 40 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.h +6 -3
@@ -80,6 +80,7 @@ enum {/* Vdpa task types. */ enum mlx5_vdpa_task_type {MLX5_VDPA_TASK_REG_MR =1 , +MLX5_VDPA_TASK_SETUP_VIRTQ , };/* Generic task information and size must be multiple of 4B. */
@@ -117,12 +118,12 @@ struct mlx5_vdpa_vmem_info {struct mlx5_vdpa_virtq {SLIST_ENTRY (mlx5_vdpa_virtq) next; -uint8_t enable;uint16_t index;uint16_t vq_size;uint8_t notifier_state; -bool stopped;uint32_t configured:1 ; +uint32_t enable:1 ; +uint32_t stopped:1 ;uint32_t version;pthread_mutex_t virtq_lock;struct mlx5_vdpa_priv *priv;
@@ -565,11 +566,13 @@ bool mlx5_vdpa_task_add(struct mlx5_vdpa_priv *priv, uint32_t thrd_idx, enum mlx5_vdpa_task_type task_type, - uint32_t *bulk_refcnt, uint32_t *bulk_err_cnt, + uint32_t *remaining_cnt, uint32_t *err_cnt, void **task_data, uint32_t num);int mlx5_vdpa_register_mr (struct mlx5_vdpa_priv *priv,uint32_t idx);bool mlx5_vdpa_c_thread_wait_bulk_tasks_done (uint32_t *remaining_cnt,uint32_t *err_cnt,uint32_t sleep_time); +int +mlx5_vdpa_virtq_setup (struct mlx5_vdpa_priv *priv,int index,bool reg_kick);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +14 -0
@@ -100,6 +100,7 @@ mlx5_vdpa_c_thread_handle(void *arg) {struct mlx5_vdpa_conf_thread_mng *multhrd = arg;pthread_t thread_id =pthread_self (); +struct mlx5_vdpa_virtq *virtq;struct mlx5_vdpa_priv *priv;struct mlx5_vdpa_task task;struct rte_ring *rng;
@@ -139,6 +140,19 @@ mlx5_vdpa_c_thread_handle(void *arg) __ATOMIC_RELAXED); }break ; +case MLX5_VDPA_TASK_SETUP_VIRTQ : + virtq = &priv->virtqs[task.idx]; +pthread_mutex_lock (&virtq->virtq_lock); + ret =mlx5_vdpa_virtq_setup (priv, + task.idx, false); +if (ret) { +DRV_LOG (ERR , +"Failed to setup virtq %d." , task.idx); +__atomic_fetch_add ( + task.err_cnt,1 , __ATOMIC_RELAXED); + } +pthread_mutex_unlock (&virtq->virtq_lock); +break ;default :DRV_LOG (ERR ,"Invalid vdpa task type %d." , task.type);
drivers/vdpa/mlx5/mlx5_vdpa_event.c +1 -1
@@ -371,7 +371,7 @@ mlx5_vdpa_err_interrupt_handler(void *cb_arg __rte_unused) goto unlock;if (rte_rdtsc () /rte_get_tsc_hz () <MLX5_VDPA_ERROR_TIME_SEC ) goto unlock; - virtq->stopped = true; + virtq->stopped =1 ;/* Query error info. */ if (mlx5_vdpa_virtq_query (priv, vq_index)) goto log;
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +113 -36
215 lines not on this page · raw message
Signature · 1 line
2.31.1
Re: [PATCH v4 11/15] vdpa/mlx5: add virtq creation task for MT management
Quoted 5 lines · Li Zhang
The virtq object and all its sub-resources use a lot of FW commands and can be accelerated by the MT management. Split the virtqs creation between the configuration threads. This accelerates the LM process and reduces its time by 20%.
Quoted 9 lines
Signed-off-by Li Zhang <[email protected]>Acked-by Matan Azrad <[email protected]>--- drivers/vdpa/mlx5/mlx5_vdpa.h | 9 +- drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 14 +++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 2 +- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 149 +++++++++++++++++++------- 4 files changed, 134 insertions(+), 40 deletions(-)
Thanks,
Maxime
PATCH v4 12/15 vdpa/mlx5: add virtq LM log task
In reply to Li ZhangSplit the virtqs LM log between the configuration threads. This accelerates the LM process and reduces its time by 20%.
--- drivers/vdpa/mlx5/mlx5_vdpa.h | 3 + drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 34 +++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 85 +++++++++++++++++++++------ 3 files changed, 105 insertions(+), 17 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.h +3 -0
@@ -72,6 +72,8 @@ enum { MLX5_VDPA_NOTIFIER_STATE_ERR }; +#define MLX5_VDPA_USED_RING_LEN (size) \ + ((size) * sizeof(struct vring_used_elem) + sizeof(uint16_t) * 3)#define MLX5_VDPA_MAX_C_THRD 256#define MLX5_VDPA_MAX_TASKS_PER_THRD 4096#define MLX5_VDPA_TASKS_PER_DEV 64
@@ -81,6 +83,7 @@ enum {enum mlx5_vdpa_task_type {MLX5_VDPA_TASK_REG_MR =1 ,MLX5_VDPA_TASK_SETUP_VIRTQ , +MLX5_VDPA_TASK_STOP_VIRTQ , };/* Generic task information and size must be multiple of 4B. */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +34 -0
@@ -104,6 +104,7 @@ mlx5_vdpa_c_thread_handle(void *arg)struct mlx5_vdpa_priv *priv;struct mlx5_vdpa_task task;struct rte_ring *rng; +uint64_t features;uint32_t thrd_idx;uint32_t task_num;int ret;
@@ -153,6 +154,39 @@ mlx5_vdpa_c_thread_handle(void *arg) }pthread_mutex_unlock (&virtq->virtq_lock);break ; +case MLX5_VDPA_TASK_STOP_VIRTQ : + virtq = &priv->virtqs[task.idx]; +pthread_mutex_lock (&virtq->virtq_lock); + ret =mlx5_vdpa_virtq_stop (priv, + task.idx); +if (ret) { +DRV_LOG (ERR , +"Failed to stop virtq %d." , + task.idx); +__atomic_fetch_add ( + task.err_cnt,1 , + __ATOMIC_RELAXED); +pthread_mutex_unlock (&virtq->virtq_lock); +break ; + } + ret =rte_vhost_get_negotiated_features ( + priv->vid, &features); +if (ret) { +DRV_LOG (ERR , +"Failed to get negotiated features virtq %d." , + task.idx); +__atomic_fetch_add ( + task.err_cnt,1 , + __ATOMIC_RELAXED); +pthread_mutex_unlock (&virtq->virtq_lock); +break ; + } +if (RTE_VHOST_NEED_LOG (features)) +rte_vhost_log_used_vring ( + priv->vid, task.idx,0 , +MLX5_VDPA_USED_RING_LEN (virtq->vq_size)); +pthread_mutex_unlock (&virtq->virtq_lock); +break ;default :DRV_LOG (ERR ,"Invalid vdpa task type %d." , task.type);
drivers/vdpa/mlx5/mlx5_vdpa_lm.c +68 -17
107 lines not on this page · raw message
Signature · 1 line
2.31.1
Re: [PATCH v4 12/15] vdpa/mlx5: add virtq LM log task
Quoted 11 lines · Li Zhang
Split the virtqs LM log between the configuration threads. This accelerates the LM process and reduces its time by 20%.
Signed-off-by Li Zhang <[email protected]>Acked-by Matan Azrad <[email protected]>--- drivers/vdpa/mlx5/mlx5_vdpa.h | 3 + drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 34 +++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 85 +++++++++++++++++++++------ 3 files changed, 105 insertions(+), 17 deletions(-)
Thanks,
Maxime
PATCH v4 13/15 vdpa/mlx5: add device close task
In reply to Li ZhangSplit the virtqs device close tasks after stopping virt-queue between the configuration threads. This accelerates the LM process and
reduces its time by 50%.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 56 +++++++++++++++++++++++++-- drivers/vdpa/mlx5/mlx5_vdpa.h | 8 ++++ drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 20 +++++++++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 14 +++++++ 4 files changed, 94 insertions(+), 4 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +53 -3
@@ -245,7 +245,7 @@ mlx5_vdpa_mtu_set(struct mlx5_vdpa_priv *priv)return kern_mtu == vhost_mtu ?0 :-1 ; } -static void +void mlx5_vdpa_dev_cache_clean (struct mlx5_vdpa_priv *priv) {/* Clean pre-created resource in dev removal only. */
@@ -254,6 +254,26 @@ mlx5_vdpa_dev_cache_clean(struct mlx5_vdpa_priv *priv)mlx5_vdpa_mem_dereg (priv); } +static bool +mlx5_vdpa_wait_dev_close_tasks_done (struct mlx5_vdpa_priv *priv) +{ +uint32_t timeout =0 ; + +/* Check and wait all close tasks done. */ +while (__atomic_load_n (&priv->dev_close_progress, + __ATOMIC_RELAXED) !=0 && timeout <1000 ) { +rte_delay_us_sleep (10000 ); + timeout++; + } +if (priv->dev_close_progress) { +DRV_LOG (ERR , +"Failed to wait close device tasks done vid %d." , + priv->vid); +return true; + } +return false; +} +static int mlx5_vdpa_dev_close (int vid) {
@@ -271,6 +291,27 @@ mlx5_vdpa_dev_close(int vid) ret |=mlx5_vdpa_lm_log (priv); priv->state =MLX5_VDPA_STATE_IN_PROGRESS ; } +if (priv->use_c_thread) { +if (priv->last_c_thrd_idx >= + (conf_thread_mng.max_thrds -1 )) + priv->last_c_thrd_idx =0 ; +else + priv->last_c_thrd_idx++; +__atomic_store_n (&priv->dev_close_progress, +1 , __ATOMIC_RELAXED); +if (mlx5_vdpa_task_add (priv, + priv->last_c_thrd_idx, +MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT , +NULL ,NULL ,NULL ,1 )) { +DRV_LOG (ERR , +"Fail to add dev close task. " ); + goto single_thrd; + } + priv->state =MLX5_VDPA_STATE_PROBED ; +DRV_LOG (INFO ,"vDPA device %d was closed." , vid); +return ret; + } +single_thrd:pthread_mutex_lock (&priv->steer_update_lock);mlx5_vdpa_steer_unset (priv);pthread_mutex_unlock (&priv->steer_update_lock);
@@ -278,10 +319,12 @@ mlx5_vdpa_dev_close(int vid)mlx5_vdpa_drain_cq (priv);if (priv->lm_mr.addr)mlx5_os_wrapped_mkey_destroy (&priv->lm_mr); - priv->state =MLX5_VDPA_STATE_PROBED ;if (!priv->connected)mlx5_vdpa_dev_cache_clean (priv); priv->vid =0 ; +__atomic_store_n (&priv->dev_close_progress,0 , + __ATOMIC_RELAXED); + priv->state =MLX5_VDPA_STATE_PROBED ;DRV_LOG (INFO ,"vDPA device %d was closed." , vid);return ret; }
@@ -302,6 +345,8 @@ mlx5_vdpa_dev_config(int vid)DRV_LOG (ERR ,"Failed to reconfigure vid %d." , vid);return -1 ; } +if (mlx5_vdpa_wait_dev_close_tasks_done (priv)) +return -1 ; priv->vid = vid; priv->connected = true;if (mlx5_vdpa_mtu_set (priv))
@@ -444,8 +489,11 @@ mlx5_vdpa_dev_cleanup(int vid)DRV_LOG (ERR ,"Invalid vDPA device: %s." , vdev->device->name);return -1 ; } -if (priv->state ==MLX5_VDPA_STATE_PROBED ) +if (priv->state ==MLX5_VDPA_STATE_PROBED ) { +if (priv->use_c_thread) +mlx5_vdpa_wait_dev_close_tasks_done (priv);mlx5_vdpa_dev_cache_clean (priv); + } priv->connected = false;return 0 ; }
@@ -839,6 +887,8 @@ mlx5_vdpa_dev_release(struct mlx5_vdpa_priv *priv) {if (priv->state ==MLX5_VDPA_STATE_CONFIGURED )mlx5_vdpa_dev_close (priv->vid); +if (priv->use_c_thread) +mlx5_vdpa_wait_dev_close_tasks_done (priv);mlx5_vdpa_release_dev_resources (priv);if (priv->vdev)rte_vdpa_unregister_device (priv->vdev);
drivers/vdpa/mlx5/mlx5_vdpa.h +8 -0
@@ -84,6 +84,7 @@ enum mlx5_vdpa_task_type {MLX5_VDPA_TASK_REG_MR =1 ,MLX5_VDPA_TASK_SETUP_VIRTQ ,MLX5_VDPA_TASK_STOP_VIRTQ , + MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT, };/* Generic task information and size must be multiple of 4B. */
@@ -206,6 +207,7 @@ struct mlx5_vdpa_priv {uint64_t features;/* Negotiated features. */ uint16_t log_max_rqt_size;uint16_t last_c_thrd_idx; +uint16_t dev_close_progress;uint16_t num_mrs;/* Number of memory regions. */ struct mlx5_vdpa_steer steer;struct mlx5dv_var *var;
@@ -578,4 +580,10 @@ mlx5_vdpa_c_thread_wait_bulk_tasks_done(uint32_t *remaining_cnt, uint32_t *err_cnt, uint32_t sleep_time);int mlx5_vdpa_virtq_setup (struct mlx5_vdpa_priv *priv,int index,bool reg_kick); +void +mlx5_vdpa_vq_destroy (struct mlx5_vdpa_virtq *virtq); +void +mlx5_vdpa_dev_cache_clean (struct mlx5_vdpa_priv *priv); +void +mlx5_vdpa_virtq_unreg_intr_handle_all (struct mlx5_vdpa_priv *priv);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +19 -1
@@ -63,7 +63,8 @@ mlx5_vdpa_task_add(struct mlx5_vdpa_priv *priv, task[i].type = task_type; task[i].remaining_cnt = remaining_cnt; task[i].err_cnt = err_cnt; - task[i].idx = data[i]; +if (data) + task[i].idx = data[i]; }if (!mlx5_vdpa_c_thrd_ring_enqueue_bulk (rng, (void **)&task, num,NULL ))return -1 ;
@@ -187,6 +188,23 @@ mlx5_vdpa_c_thread_handle(void *arg) MLX5_VDPA_USED_RING_LEN(virtq->vq_size));pthread_mutex_unlock (&virtq->virtq_lock);break ; +case MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT : +mlx5_vdpa_virtq_unreg_intr_handle_all (priv); +pthread_mutex_lock (&priv->steer_update_lock); +mlx5_vdpa_steer_unset (priv); +pthread_mutex_unlock (&priv->steer_update_lock); +mlx5_vdpa_virtqs_release (priv); +mlx5_vdpa_drain_cq (priv); +if (priv->lm_mr.addr) +mlx5_os_wrapped_mkey_destroy ( + &priv->lm_mr); +if (!priv->connected) +mlx5_vdpa_dev_cache_clean (priv); + priv->vid =0 ; +__atomic_store_n ( + &priv->dev_close_progress,0 , + __ATOMIC_RELAXED); +break ;default :DRV_LOG (ERR ,"Invalid vdpa task type %d." , task.type);
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +14 -0
@@ -102,6 +102,20 @@ mlx5_vdpa_virtq_unregister_intr_handle(struct mlx5_vdpa_virtq *virtq) virtq->intr_handle =NULL ; } +void +mlx5_vdpa_virtq_unreg_intr_handle_all (struct mlx5_vdpa_priv *priv) +{ +uint32_t i; +struct mlx5_vdpa_virtq *virtq; + +for (i =0 ; i < priv->nr_virtqs; i++) { + virtq = &priv->virtqs[i]; +pthread_mutex_lock (&virtq->virtq_lock); +mlx5_vdpa_virtq_unregister_intr_handle (virtq); +pthread_mutex_unlock (&virtq->virtq_lock); + } +} +/* Release cached VQ resources. */ void mlx5_vdpa_virtqs_cleanup (struct mlx5_vdpa_priv *priv)
Signature · 1 line
2.31.1
Re: [PATCH v4 13/15] vdpa/mlx5: add device close task
Quoted 5 lines · Li Zhang
Split the virtqs device close tasks after stopping virt-queue between the configuration threads. This accelerates the LM process and
reduces its time by 50%.
Quoted 9 lines
Signed-off-by Li Zhang <[email protected]>Acked-by Matan Azrad <[email protected]>--- drivers/vdpa/mlx5/mlx5_vdpa.c | 56 +++++++++++++++++++++++++-- drivers/vdpa/mlx5/mlx5_vdpa.h | 8 ++++ drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 20 +++++++++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 14 +++++++ 4 files changed, 94 insertions(+), 4 deletions(-)
Thanks,
Maxime
PATCH v4 14/15 vdpa/mlx5: add virtq sub-resources creation
In reply to Li Zhangpre-created virt-queue sub-resource in device probe stage and then modify virtqueue in device config stage. Steer table also need to support dummy virt-queue. This accelerates the LM process and reduces its time by 40%.
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 72 +++++++-------------- drivers/vdpa/mlx5/mlx5_vdpa.h | 17 +++-- drivers/vdpa/mlx5/mlx5_vdpa_event.c | 11 ++-- drivers/vdpa/mlx5/mlx5_vdpa_steer.c | 17 +++-- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 99 +++++++++++++++++++++-------- 5 files changed, 123 insertions(+), 93 deletions(-)
drivers/vdpa/mlx5/mlx5_vdpa.c +23 -49
@@ -627,65 +627,39 @@ mlx5_vdpa_config_get(struct mlx5_kvargs_ctrl *mkvlist,static int mlx5_vdpa_virtq_resource_prepare (struct mlx5_vdpa_priv *priv) { -struct mlx5_vdpa_virtq *virtq; +uint32_t max_queues;uint32_t index; -uint32_t i; +struct mlx5_vdpa_virtq *virtq; -for (index =0 ; index < priv->caps.max_num_virtio_queues *2 ; +for (index =0 ; index < priv->caps.max_num_virtio_queues; index++) { virtq = &priv->virtqs[index];pthread_mutex_init (&virtq->virtq_lock,NULL ); } -if (!priv->queues) +if (!priv->queues || !priv->queue_size)return 0 ; -for (index =0 ; index < (priv->queues *2 ); ++index) { + max_queues = (priv->queues < priv->caps.max_num_virtio_queues) ? + (priv->queues *2 ) : (priv->caps.max_num_virtio_queues); +for (index =0 ; index < max_queues; ++index) +if (mlx5_vdpa_virtq_single_resource_prepare (priv, + index)) + goto error; +if (mlx5_vdpa_is_modify_virtq_supported (priv)) +if (mlx5_vdpa_steer_update (priv, true)) + goto error; +return 0 ; +error: +for (index =0 ; index < max_queues; ++index) { virtq = &priv->virtqs[index]; -int ret =mlx5_vdpa_event_qp_prepare (priv, priv->queue_size, --1 , virtq); - -if (ret) { -DRV_LOG (ERR ,"Failed to create event QPs for virtq %d." , - index); -return -1 ; - } -if (priv->caps.queue_counters_valid) { -if (!virtq->counters) - virtq->counters = -mlx5_devx_cmd_create_virtio_q_counters - (priv->cdev->ctx); -if (!virtq->counters) { -DRV_LOG (ERR ,"Failed to create virtq couners for virtq" -" %d." , index); -return -1 ; - } - } -for (i =0 ; i <RTE_DIM (virtq->umems); ++i) { -uint32_t size; -void *buf; -struct mlx5dv_devx_umem *obj; - - size = priv->caps.umems[i].a * priv->queue_size + - priv->caps.umems[i].b; - buf =rte_zmalloc (__func__, size,4096 ); -if (buf ==NULL ) { -DRV_LOG (ERR ,"Cannot allocate umem %d memory for virtq" -" %u." , i, index); -return -1 ; - } - obj = mlx5_glue->devx_umem_reg (priv->cdev->ctx, buf, - size,IBV_ACCESS_LOCAL_WRITE ); -if (obj ==NULL ) { -rte_free (buf); -DRV_LOG (ERR ,"Failed to register umem %d for virtq %u." , - i, index); -return -1 ; - } - virtq->umems[i].size = size; - virtq->umems[i].buf = buf; - virtq->umems[i].obj = obj; +if (virtq->virtq) { +pthread_mutex_lock (&virtq->virtq_lock); +mlx5_vdpa_virtq_unset (virtq); +pthread_mutex_unlock (&virtq->virtq_lock); } } -return 0 ; +if (mlx5_vdpa_is_modify_virtq_supported (priv)) +mlx5_vdpa_steer_unset (priv); +return -1 ; } static int
drivers/vdpa/mlx5/mlx5_vdpa.h +13 -4
@@ -277,13 +277,15 @@ int mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv);* The guest notification file descriptor. * @param[in/out] virtq * Pointer to the virt-queue structure. +* @param[in] reset +* If true, it will reset event qp. * * @return * 0 on success, -1 otherwise and rte_errno is set. */ int mlx5_vdpa_event_qp_prepare (struct mlx5_vdpa_priv *priv,uint16_t desc_n, -int callfd,struct mlx5_vdpa_virtq *virtq); +int callfd,struct mlx5_vdpa_virtq *virtq,bool reset);/** * Destroy an event QP and all its related resources.
@@ -403,11 +405,13 @@ void mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv);* * @param[in] priv * The vdpa driver private structure. +* @param[in] is_dummy +* If set, it is updated with dummy queue for prepare resource. * * @return * 0 on success, a negative value otherwise. */ -int mlx5_vdpa_steer_update (struct mlx5_vdpa_priv *priv); +int mlx5_vdpa_steer_update (struct mlx5_vdpa_priv *priv,bool is_dummy);/** * Setup steering and all its related resources to enable RSS traffic from the
@@ -581,9 +585,14 @@ mlx5_vdpa_c_thread_wait_bulk_tasks_done(uint32_t *remaining_cnt,int mlx5_vdpa_virtq_setup (struct mlx5_vdpa_priv *priv,int index,bool reg_kick);void -mlx5_vdpa_vq_destroy (struct mlx5_vdpa_virtq *virtq); -void mlx5_vdpa_dev_cache_clean (struct mlx5_vdpa_priv *priv);void mlx5_vdpa_virtq_unreg_intr_handle_all (struct mlx5_vdpa_priv *priv); +bool +mlx5_vdpa_virtq_single_resource_prepare (struct mlx5_vdpa_priv *priv, +int index); +int +mlx5_vdpa_qps2rst2rts (struct mlx5_vdpa_event_qp *eqp); +void +mlx5_vdpa_virtq_unset (struct mlx5_vdpa_virtq *virtq);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_event.c +5 -6
@@ -249,7 +249,7 @@ mlx5_vdpa_drain_cq(struct mlx5_vdpa_priv *priv) {unsigned int i; -for (i =0 ; i < priv->caps.max_num_virtio_queues *2 ; i++) { +for (i =0 ; i < priv->caps.max_num_virtio_queues; i++) {struct mlx5_vdpa_cq *cq = &priv->virtqs[i].eqp.cq;mlx5_vdpa_queue_complete (cq);
@@ -618,7 +618,7 @@ mlx5_vdpa_qps2rts(struct mlx5_vdpa_event_qp *eqp)return 0 ; } -static int +int mlx5_vdpa_qps2rst2rts (struct mlx5_vdpa_event_qp *eqp) { if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_QP_2RST,
@@ -638,7 +638,7 @@ mlx5_vdpa_qps2rst2rts(struct mlx5_vdpa_event_qp *eqp)int mlx5_vdpa_event_qp_prepare (struct mlx5_vdpa_priv *priv,uint16_t desc_n, -int callfd,struct mlx5_vdpa_virtq *virtq) +int callfd,struct mlx5_vdpa_virtq *virtq,bool reset) {struct mlx5_vdpa_event_qp *eqp = &virtq->eqp;struct mlx5_devx_qp_attr attr = {0 };
@@ -649,11 +649,10 @@ mlx5_vdpa_event_qp_prepare(struct mlx5_vdpa_priv *priv, uint16_t desc_n,/* Reuse existing resources. */ eqp->cq.callfd = callfd;/* FW will set event qp to error state in q destroy. */ -if (!mlx5_vdpa_qps2rst2rts (eqp)) { +if (reset && !mlx5_vdpa_qps2rst2rts (eqp))rte_write32 (rte_cpu_to_be_32 (RTE_BIT32 (log_desc_n)), &eqp->sw_qp.db_rec[0 ]); -return 0 ; - } +return 0 ; }if (eqp->fw_qp)mlx5_vdpa_event_qp_destroy (eqp);
drivers/vdpa/mlx5/mlx5_vdpa_steer.c +11 -6
@@ -57,7 +57,7 @@ mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv)* -1 on error. */ static int -mlx5_vdpa_rqt_prepare (struct mlx5_vdpa_priv *priv) +mlx5_vdpa_rqt_prepare (struct mlx5_vdpa_priv *priv,bool is_dummy) {int i; uint32_t rqt_n = RTE_MIN(MLX5_VDPA_DEFAULT_RQT_SIZE,
@@ -67,15 +67,20 @@ mlx5_vdpa_rqt_prepare(struct mlx5_vdpa_priv *priv) sizeof(uint32_t), 0);uint32_t k =0 , j;int ret =0 , num; +uint16_t nr_vring = is_dummy ? + (((priv->queues *2 ) < priv->caps.max_num_virtio_queues) ? + (priv->queues *2 ) : priv->caps.max_num_virtio_queues) : priv->nr_virtqs;if (!attr) {DRV_LOG (ERR ,"Failed to allocate RQT attributes memory." ); rte_errno =ENOMEM ;return -ENOMEM ; } -for (i =0 ; i < priv->nr_virtqs; i++) { +for (i =0 ; i < nr_vring; i++) {if (is_virtq_recvq (i, priv->nr_virtqs) && - priv->virtqs[i].enable && priv->virtqs[i].virtq) { + (is_dummy || (priv->virtqs[i].enable && + priv->virtqs[i].configured)) && + priv->virtqs[i].virtq) { attr->rq_list[k] = priv->virtqs[i].virtq->id; k++; }
@@ -235,12 +240,12 @@ mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv *priv) }int -mlx5_vdpa_steer_update (struct mlx5_vdpa_priv *priv) +mlx5_vdpa_steer_update (struct mlx5_vdpa_priv *priv,bool is_dummy) {int ret;pthread_mutex_lock (&priv->steer_update_lock); - ret =mlx5_vdpa_rqt_prepare (priv); + ret =mlx5_vdpa_rqt_prepare (priv, is_dummy);if (ret ==0 ) {mlx5_vdpa_steer_unset (priv); }else if (ret <0 ) {
@@ -261,7 +266,7 @@ mlx5_vdpa_steer_update(struct mlx5_vdpa_priv *priv)int mlx5_vdpa_steer_setup (struct mlx5_vdpa_priv *priv) { -if (mlx5_vdpa_steer_update (priv)) +if (mlx5_vdpa_steer_update (priv, false)) goto error;return 0 ; error:
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +71 -28
198 lines not on this page · raw message
Signature · 1 line
2.31.1
Re: [PATCH v4 14/15] vdpa/mlx5: add virtq sub-resources creation
Quoted 8 lines · Li Zhang
pre-created virt-queue sub-resource in device probe stage and then modify virtqueue in device config stage. Steer table also need to support dummy virt-queue. This accelerates the LM process and reduces its time by 40%.
Signed-off-by Li Zhang <[email protected]>Signed-off-by Yajun Wu <[email protected]>Acked-by Matan Azrad <[email protected]>
Quoted 8 lines
--- drivers/vdpa/mlx5/mlx5_vdpa.c | 72 +++++++-------------- drivers/vdpa/mlx5/mlx5_vdpa.h | 17 +++-- drivers/vdpa/mlx5/mlx5_vdpa_event.c | 11 ++-- drivers/vdpa/mlx5/mlx5_vdpa_steer.c | 17 +++-- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 99 +++++++++++++++++++++-------- 5 files changed, 123 insertions(+), 93 deletions(-)
Thanks,
Maxime
PATCH v4 15/15 vdpa/mlx5: prepare virtqueue resource creation
In reply to Li ZhangSplit the virtqs virt-queue resource between
the configuration threads.
Also need pre-created virt-queue resource
after virtq destruction.
This accelerates the LM process and reduces its time by 30%.
--- doc/guides/rel_notes/release_22_07.rst | 1 + drivers/vdpa/mlx5/mlx5_vdpa.c | 115 +++++++++++++++++++------ drivers/vdpa/mlx5/mlx5_vdpa.h | 12 ++- drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 15 +++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 111 ++++++++++++++++++++---- 5 files changed, 209 insertions(+), 45 deletions(-)
doc/guides/rel_notes/release_22_07.rst +1 -0
@@ -178,6 +178,7 @@ New Features * **Updated Nvidia mlx5 vDPA driver.** * Added new devargs ``queue_size`` and ``queues`` to allow prior creation of virtq resources. + * Added new devarg ``max_conf_threads`` defines the number of multi-thread management to parallel the configurations. Removed Items
drivers/vdpa/mlx5/mlx5_vdpa.c +89 -26
168 lines not on this page · raw message
drivers/vdpa/mlx5/mlx5_vdpa.h +11 -1
@@ -85,6 +85,7 @@ enum mlx5_vdpa_task_type {MLX5_VDPA_TASK_SETUP_VIRTQ ,MLX5_VDPA_TASK_STOP_VIRTQ ,MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT , + MLX5_VDPA_TASK_PREPARE_VIRTQ, };/* Generic task information and size must be multiple of 4B. */
@@ -128,6 +129,9 @@ struct mlx5_vdpa_virtq { uint32_t configured:1; uint32_t enable:1; uint32_t stopped:1; + uint32_t rx_csum:1; + uint32_t virtio_version_1_0:1; + uint32_t event_mode:3;uint32_t version;pthread_mutex_t virtq_lock;struct mlx5_vdpa_priv *priv;
@@ -355,8 +359,12 @@ void mlx5_vdpa_err_event_unset(struct mlx5_vdpa_priv *priv);* * @param[in] priv * The vdpa driver private structure. +* @param[in] release_resource +* The vdpa driver release resource without prepare resource. */ -void mlx5_vdpa_virtqs_release (struct mlx5_vdpa_priv *priv); +void +mlx5_vdpa_virtqs_release (struct mlx5_vdpa_priv *priv, +bool release_resource);/** * Cleanup cached resources of all virtqs.
@@ -595,4 +603,6 @@ int mlx5_vdpa_qps2rst2rts(struct mlx5_vdpa_event_qp *eqp);void mlx5_vdpa_virtq_unset (struct mlx5_vdpa_virtq *virtq); +void +mlx5_vdpa_prepare_virtq_destroy (struct mlx5_vdpa_priv *priv);#endif /* RTE_PMD_MLX5_VDPA_H_ */
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c +14 -1
@@ -153,6 +153,7 @@ mlx5_vdpa_c_thread_handle(void *arg)__atomic_fetch_add ( task.err_cnt,1 , __ATOMIC_RELAXED); } + virtq->enable =1 ;pthread_mutex_unlock (&virtq->virtq_lock);break ;case MLX5_VDPA_TASK_STOP_VIRTQ :
@@ -193,7 +194,7 @@ mlx5_vdpa_c_thread_handle(void *arg)pthread_mutex_lock (&priv->steer_update_lock);mlx5_vdpa_steer_unset (priv);pthread_mutex_unlock (&priv->steer_update_lock); -mlx5_vdpa_virtqs_release (priv); +mlx5_vdpa_virtqs_release (priv, false);mlx5_vdpa_drain_cq (priv);if (priv->lm_mr.addr) mlx5_os_wrapped_mkey_destroy(
@@ -205,6 +206,18 @@ mlx5_vdpa_c_thread_handle(void *arg) &priv->dev_close_progress,0 , __ATOMIC_RELAXED);break ; +case MLX5_VDPA_TASK_PREPARE_VIRTQ : + ret =mlx5_vdpa_virtq_single_resource_prepare ( + priv, task.idx); +if (ret) { +DRV_LOG (ERR , +"Failed to prepare virtq %d." , + task.idx); +__atomic_fetch_add ( + task.err_cnt,1 , + __ATOMIC_RELAXED); + } +break ;default :DRV_LOG (ERR ,"Invalid vdpa task type %d." , task.type);
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +94 -17
197 lines not on this page · raw message
Signature · 1 line
2.31.1
Re: [PATCH v4 15/15] vdpa/mlx5: prepare virtqueue resource creation
Quoted 8 lines · Li Zhang
Split the virtqs virt-queue resource between
the configuration threads.
Also need pre-created virt-queue resource
after virtq destruction.
This accelerates the LM process and reduces its time by 30%.Signed-off-by Li Zhang <[email protected]>Acked-by Matan Azrad <[email protected]>
Quoted 8 lines
--- doc/guides/rel_notes/release_22_07.rst | 1 + drivers/vdpa/mlx5/mlx5_vdpa.c | 115 +++++++++++++++++++------ drivers/vdpa/mlx5/mlx5_vdpa.h | 12 ++- drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 15 +++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 111 ++++++++++++++++++++---- 5 files changed, 209 insertions(+), 45 deletions(-)
Thanks,
Maxime
Maxime Coquelin
AppliedRe: [PATCH v4 00/15] mlx5/vdpa: optimize live migration time
In reply to Li ZhangQuoted 63 lines · Li Zhang
Allow the driver to use internal threads to obtain fast configuration.
All the threads will be open on the same core of the event completion queue scheduling thread.Add
max_conf_threadsparameter to configure
the maximum number of internal threads in addition to the caller thread (8 is suggested).
These internal threads to pipeline handle VDPA tasks in system and shared with all VDPA devices. Default is 0, don't use internal threads for configuration.Depends-on: series=21868 ("vdpa/mlx5: improve device shutdown time")
http://patchwork.dpdk.org/project/dpdk/list/?series=21868RFC ("Add vDPA multi-threads optiomization")
https://patchwork.dpdk.org/project/dpdk/cover/[email protected]/V2:
* Drop eal device removal patch in series.
* Add release note inrelease_22_07.rst.V3:
* Fix comments about commit log issue.
* Avoid cutting logs.V4:
* Fix coding style issueLi Zhang (12):
vdpa/mlx5: fix usage of capability for max number of virtqs common/mlx5: extend virtq modifiable fields vdpa/mlx5: pre-create virtq at probe time vdpa/mlx5: optimize datapath-control synchronization vdpa/mlx5: add multi-thread management for configuration vdpa/mlx5: add task ring for MT management vdpa/mlx5: add MT task for VM memory registration vdpa/mlx5: add virtq creation task for MT management vdpa/mlx5: add virtq LM log task
vdpa/mlx5: add device close task
vdpa/mlx5: add virtq sub-resources creation vdpa/mlx5: prepare virtqueue resource creationYajun Wu (3):
vdpa/mlx5: support pre create virtq resource common/mlx5: add DevX API to move QP to reset state vdpa/mlx5: support event qp reusedoc/guides/rel_notes/release_22_07.rst | 5 + doc/guides/vdpadevs/mlx5.rst | 25 + drivers/common/mlx5/mlx5_devx_cmds.c | 77 ++- drivers/common/mlx5/mlx5_devx_cmds.h | 6 +- drivers/common/mlx5/mlx5_prm.h | 30 +- drivers/vdpa/mlx5/meson.build | 1 + drivers/vdpa/mlx5/mlx5_vdpa.c | 270 ++++++++-- drivers/vdpa/mlx5/mlx5_vdpa.h | 152 +++++- drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 360 ++++++++++++++ drivers/vdpa/mlx5/mlx5_vdpa_event.c | 160 ++++-- drivers/vdpa/mlx5/mlx5_vdpa_lm.c | 134 +++-- drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 270 ++++++---- drivers/vdpa/mlx5/mlx5_vdpa_steer.c | 22 +- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 654 ++++++++++++++++++------- 14 files changed, 1779 insertions(+), 387 deletions(-)
Quoted 2 lines
create mode 100644
drivers/vdpa/mlx5/mlx5_vdpa_cthread.c
Applied to dpdk-next-virtio/main.
Thanks,
Maxime