vhost: introduce new API to get the active vring number
PATCH 1/4 vhost: introduce new API to get the active vring number
It's useful for hardware vhost backend (like vDPA devices) to set multiqueue configuration accordingly.
--- lib/librte_vhost/rte_vhost.h | 12 ++++++++++++ lib/librte_vhost/vhost.c | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+)
lib/librte_vhost/rte_vhost.h +12 -0
@@ -525,6 +525,18 @@ int rte_vhost_driver_callback_register(const char *path,uint16_t rte_vhost_get_vring_num (int vid);/** +* Get the number of active vrings of the device. +* +* @param vid +* vhost device ID +* +* @return +* The number of active vrings, 0 on failure +*/ +uint16_t +rte_vhost_get_active_vring_num (int vid); + +/** * Get the virtio net device's ifname, which is the vhost-user socket * file path. *
lib/librte_vhost/vhost.c +19 -0
@@ -674,6 +674,25 @@return dev->nr_vring; } +uint16_t +rte_vhost_get_active_vring_num (int vid) +{ +struct virtio_net *dev =get_device (vid); +struct vhost_virtqueue *vq; +uint16_t qid; + +if (dev ==NULL ) +return 0 ; + +for (qid =0 ; qid < dev->nr_vring; qid++) { + vq = dev->virtqueue[qid]; +if (!vq->enabled) +break ; + } + +return qid; +} +int rte_vhost_get_ifname (int vid,char *buf,size_t len) {
Signature · 1 line
1.8.3.1
PATCH 2/4 net/ifcvf: add multiqueue configuration
This is in preparation for multiqueue enabling for vDPA devices.
--- drivers/net/ifc/base/ifcvf.c | 1 + drivers/net/ifc/base/ifcvf.h | 2 ++ 2 files changed, 3 insertions(+)
drivers/net/ifc/base/ifcvf.c +1 -0
@@ -82,6 +82,7 @@ } hw->lm_cfg = hw->mem_resource[4 ].addr; + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ;if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {
drivers/net/ifc/base/ifcvf.h +2 -0
@@ -38,6 +38,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0
@@ -127,6 +128,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;struct ifcvf_pci_mem_resource mem_resource[IFCVF_PCI_MAX_RESOURCE ];
Signature · 1 line
1.8.3.1
PATCH 3/4 vhost: call vDPA callback at the end of vring enable handler
In reply to Andy PeivDPA's set_vring_state callback would need to know the virtqueues' enable status to configure the hardware.
--- lib/librte_vhost/rte_vdpa.h | 4 ++-- lib/librte_vhost/vhost_user.c | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-)
lib/librte_vhost/rte_vdpa.h +2 -2
@@ -54,8 +54,8 @@ struct rte_vdpa_dev_ops {int (*dev_conf)(int vid);int (*dev_close)(int vid); -/** Enable/disable this vring */ -int (*set_vring_state)(int vid,int vring,int state); +/** Enable/disable vring queue pairs */ +int (*set_vring_state)(int vid);/** Set features when changed */ int (*set_features)(int vid);
lib/librte_vhost/vhost_user.c +3 -2
@@ -1346,8 +1346,6 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused, did = dev->vdpa_dev_id; vdpa_dev =rte_vdpa_get_device (did); -if (vdpa_dev && vdpa_dev->ops->set_vring_state) - vdpa_dev->ops->set_vring_state (dev->vid, index, enable);if (dev->notify_ops->vring_state_changed) dev->notify_ops->vring_state_changed(dev->vid,
@@ -1359,6 +1357,9 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused, dev->virtqueue[index]->enabled = enable; +if (vdpa_dev && vdpa_dev->ops->set_vring_state) + vdpa_dev->ops->set_vring_state (dev->vid); +return RTE_VHOST_MSG_RESULT_OK ; }
Signature · 1 line
1.8.3.1
Re: [dpdk-dev] [PATCH 3/4] vhost: call vDPA callback at the end of vring enable handler
Hi Andy,
Quoted 24 lines
-----Original Message-----
From: Pei, Andy
Sent: Friday, September 6, 2019 11:21 AM
To: [email protected]
Cc: Xu, Rosen <[email protected]>; Ye, Xiaolong <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Xiao W <[email protected]>
Subject: [PATCH 3/4] vhost: call vDPA callback at the end of vring enable handlervDPA's
set_vring_statecallback would need to know the virtqueues' enable status to configure the hardware.Signed-off-by Xiaolong Ye <[email protected]>Signed-off-by Andy Pei <[email protected]>--- lib/librte_vhost/rte_vdpa.h | 4 ++-- lib/librte_vhost/vhost_user.c | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-)lib/librte_vhost/rte_vdpa.h +0
-0
lib/librte_vhost/rte_vdpa.h struct rte_vdpa_dev_ops { · line 58
Quoted 7 lines
lib/librte_vhost/rte_vdpa.h +2
-2@@ -54,8 +54,8 @@ struct rte_vdpa_dev_ops {int (*dev_conf)(int vid);int (*dev_close)(int vid); -/** Enable/disable this vring */ -int (*set_vring_state)(int vid,int vring,int state); +/** Enable/disable vring queue pairs */ +int (*set_vring_state)(int vid);
If rte_vhost_get_active_vring_num() always needs be called inside set_vring_state callback, inorder to get the active queue pair number, then why not pass the active queue pair number directly as a parameter to set_vring_state() ? This could help to avoid exposing new API.
BRs,
Xiao
Quoted 30 lines
lib/librte_vhost/rte_vdpa.h +0
-0/** Set features when changed */ int (*set_features)(int vid);lib/librte_vhost/vhost_user.c +0
-2@@ -1346,8 +1346,6 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused, did = dev->vdpa_dev_id; vdpa_dev =rte_vdpa_get_device (did); -if (vdpa_dev && vdpa_dev->ops->set_vring_state) - vdpa_dev->ops->set_vring_state (dev->vid, index, enable);if (dev->notify_ops->vring_state_changed)lib/librte_vhost/vhost_user.c +3
-1dev->notify_ops->vring_state_changed(dev->vid,@@ -1359,6 +1357,9 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused, dev->virtqueue[index]->enabled = enable; +if (vdpa_dev && vdpa_dev->ops->set_vring_state) + vdpa_dev->ops->set_vring_state (dev->vid); +return RTE_VHOST_MSG_RESULT_OK ; } --1.8.3.1
Re: [dpdk-dev] [PATCH 3/4] vhost: call vDPA callback at the end of vring enable handler
Yes, I will do this way in v2.
-----Original Message-----
From: Wang, Xiao W
Sent: Thursday, September 12, 2019 3:42 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Ye, Xiaolong <[email protected]>; Bie, Tiwei <[email protected]>; Liang, Cunming <[email protected]>
Subject: RE: [PATCH 3/4] vhost: call vDPA callback at the end of vring enable handler
Hi Andy,
Quoted 25 lines
-----Original Message-----
From: Pei, Andy
Sent: Friday, September 6, 2019 11:21 AM
To: [email protected]
Cc: Xu, Rosen <[email protected]>; Ye, Xiaolong <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Xiao W <[email protected]>
Subject: [PATCH 3/4] vhost: call vDPA callback at the end of vring enable handlervDPA's
set_vring_statecallback would need to know the virtqueues' enable status to configure the hardware.Signed-off-by Xiaolong Ye <[email protected]>Signed-off-by Andy Pei <[email protected]>--- lib/librte_vhost/rte_vdpa.h | 4 ++-- lib/librte_vhost/vhost_user.c | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-)lib/librte_vhost/rte_vdpa.h +0
-0
lib/librte_vhost/rte_vdpa.h struct rte_vdpa_dev_ops { · line 58
Quoted 7 lines
lib/librte_vhost/rte_vdpa.h +2
-2@@ -54,8 +54,8 @@ struct rte_vdpa_dev_ops {int (*dev_conf)(int vid);int (*dev_close)(int vid); -/** Enable/disable this vring */ -int (*set_vring_state)(int vid,int vring,int state); +/** Enable/disable vring queue pairs */ +int (*set_vring_state)(int vid);
If rte_vhost_get_active_vring_num() always needs be called inside set_vring_state callback, inorder to get the active queue pair number, then why not pass the active queue pair number directly as a parameter to set_vring_state() ? This could help to avoid exposing new API.
BRs,
Xiao
Quoted 30 lines
lib/librte_vhost/rte_vdpa.h +0
-0/** Set features when changed */ int (*set_features)(int vid);(quoted diff) +0
-0
b/lib/librte_vhost/vhost_user.cindex 0b72648..21028cc 100644lib/librte_vhost/vhost_user.c +0
-2@@ -1346,8 +1346,6 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused, did = dev->vdpa_dev_id; vdpa_dev =rte_vdpa_get_device (did); -if (vdpa_dev && vdpa_dev->ops->set_vring_state) - vdpa_dev->ops->set_vring_state (dev->vid, index, enable);if (dev->notify_ops->vring_state_changed)lib/librte_vhost/vhost_user.c +3
-1dev->notify_ops->vring_state_changed(dev->vid,@@ -1359,6 +1357,9 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused, dev->virtqueue[index]->enabled = enable; +if (vdpa_dev && vdpa_dev->ops->set_vring_state) + vdpa_dev->ops->set_vring_state (dev->vid); +return RTE_VHOST_MSG_RESULT_OK ; } --1.8.3.1
PATCH 4/4 net/ifcvf: enable mutliqueue support
In reply to Andy PeiEnable mutliqueue support for ifcvf devices by setting the mutliqueue configuration space.
--- drivers/net/ifc/base/ifcvf.c | 8 ++++++++ drivers/net/ifc/base/ifcvf.h | 5 ++++- drivers/net/ifc/ifcvf_vdpa.c | 31 +++++++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 3 deletions(-)
drivers/net/ifc/base/ifcvf.c +8 -0
@@ -310,6 +310,14 @@ *(u32 *)(lm_cfg +IFCVF_LM_LOGGING_CTRL ) =IFCVF_LM_DISABLE ; } +void ifcvf_enable_multiqueue (struct ifcvf_hw *hw,u16 nr_queue_pair) +{ +u8 *mq_cfg; + + mq_cfg = hw->mq_cfg; + *(u32 *)mq_cfg = nr_queue_pair; +} +void ifcvf_notify_queue (struct ifcvf_hw *hw,u16 qid) {
drivers/net/ifc/base/ifcvf.h +4 -1
@@ -12,7 +12,7 @@#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#define VIRTIO_F_IOMMU_PLATFORM 33/* Common configuration */
@@ -153,6 +153,9 @@ struct ifcvf_hw { ifcvf_disable_logging(struct ifcvf_hw *hw);void +ifcvf_enable_multiqueue (struct ifcvf_hw *hw,u16 nr_queue_pair); + +void ifcvf_notify_queue (struct ifcvf_hw *hw,u16 qid); u8
drivers/net/ifc/ifcvf_vdpa.c +29 -2
@@ -994,6 +994,31 @@ struct internal_list { }static int +ifcvf_set_vring_state (int vid) +{ +int did, nr_active_vring, nr_queue_pair; +struct internal_list *list; + + nr_active_vring =rte_vhost_get_active_vring_num (vid); +if (nr_active_vring ==0 ) { +DRV_LOG (ERR ,"No enabled vring" ); +return -1 ; + } + nr_queue_pair = (nr_active_vring +1 ) /2 ; + + did =rte_vhost_get_vdpa_device_id (vid); + list =find_internal_resource_by_did (did); +if (list ==NULL ) { +DRV_LOG (ERR ,"Invalid device id: %d" , did); +return -1 ; + } + +ifcvf_enable_multiqueue (&list->internal->hw, nr_queue_pair); + +return 0 ; +} + +static int ifcvf_get_notify_area (int vid,int qid,uint64_t *offset,uint64_t *size) {int did;
@@ -1062,7 +1087,9 @@ struct internal_list { 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ | \ 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \ 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \ - 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) + 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ + 1ULL << VHOST_USER_PROTOCOL_F_MQ) +static int ifcvf_get_protocol_features(int did __rte_unused, uint64_t *features) {
@@ -1076,7 +1103,7 @@ struct internal_list { .get_protocol_features = ifcvf_get_protocol_features, .dev_conf = ifcvf_dev_config, .dev_close = ifcvf_dev_close, - .set_vring_state = NULL, + .set_vring_state = ifcvf_set_vring_state, .set_features = ifcvf_set_features, .migration_done = NULL, .get_vfio_group_fd = ifcvf_get_vfio_group_fd,
Signature · 1 line
1.8.3.1
Re: [dpdk-dev] [PATCH 4/4] net/ifcvf: enable mutliqueue support
Hi Andy,
Quoted 43 lines
-----Original Message-----
From: Pei, Andy
Sent: Friday, September 6, 2019 11:21 AM
To: [email protected]
Cc: Xu, Rosen <[email protected]>; Ye, Xiaolong <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Xiao W <[email protected]>
Subject: [PATCH 4/4] net/ifcvf: enable mutliqueue supportEnable mutliqueue support for ifcvf devices by setting the mutliqueue configuration space.
Signed-off-by Xiaolong Ye <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/net/ifc/base/ifcvf.c | 8 ++++++++ drivers/net/ifc/base/ifcvf.h | 5 ++++- drivers/net/ifc/ifcvf_vdpa.c | 31 +++++++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 3 deletions(-)drivers/net/ifc/base/ifcvf.c +8
-0@@ -310,6 +310,14 @@ *(u32 *)(lm_cfg +IFCVF_LM_LOGGING_CTRL ) =IFCVF_LM_DISABLE ; } +void ifcvf_enable_multiqueue (struct ifcvf_hw *hw,u16 nr_queue_pair) +{ +u8 *mq_cfg; + + mq_cfg = hw->mq_cfg; + *(u32 *)mq_cfg = nr_queue_pair; +} +void ifcvf_notify_queue (struct ifcvf_hw *hw,u16 qid) {drivers/net/ifc/base/ifcvf.h +0
-0
drivers/net/ifc/base/ifcvf.h line 15
Quoted 5 lines
drivers/net/ifc/base/ifcvf.h +1
-1@@ -12,7 +12,7 @@#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32
The IFC supports flexible queue resource allocation among VFs, a VF may have at most 32 queue pair allocated, but the actual number depends on user's allocation policy. You can just get this number from a register in virtio device config capability.
BRs,
Xiao
Quoted 71 lines
drivers/net/ifc/base/ifcvf.h +3
-0#define VIRTIO_F_IOMMU_PLATFORM 33/* Common configuration */ @@ -153,6 +153,9 @@ struct ifcvf_hw { ifcvf_disable_logging(struct ifcvf_hw *hw);void +ifcvf_enable_multiqueue (struct ifcvf_hw *hw,u16 nr_queue_pair); + +void ifcvf_notify_queue (struct ifcvf_hw *hw,u16 qid); u8drivers/net/ifc/ifcvf_vdpa.c +29
-3@@ -994,6 +994,31 @@ struct internal_list { }static int +ifcvf_set_vring_state (int vid) +{ +int did, nr_active_vring, nr_queue_pair; +struct internal_list *list; + + nr_active_vring =rte_vhost_get_active_vring_num (vid); +if (nr_active_vring ==0 ) { +DRV_LOG (ERR ,"No enabled vring" ); +return -1 ; + } + nr_queue_pair = (nr_active_vring +1 ) /2 ; + + did =rte_vhost_get_vdpa_device_id (vid); + list =find_internal_resource_by_did (did); +if (list ==NULL ) { +DRV_LOG (ERR ,"Invalid device id: %d" , did); +return -1 ; + } + +ifcvf_enable_multiqueue (&list->internal->hw, nr_queue_pair); + +return 0 ; +} + +static int ifcvf_get_notify_area (int vid,int qid,uint64_t *offset,uint64_t *size) {int did;@@ -1062,7 +1087,9 @@ struct internal_list { 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ | \ 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \ 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \ - 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) + 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ + 1ULL << VHOST_USER_PROTOCOL_F_MQ) +static int ifcvf_get_protocol_features(int did __rte_unused, uint64_t *features) {@@ -1076,7 +1103,7 @@ struct internal_list { .get_protocol_features = ifcvf_get_protocol_features, .dev_conf = ifcvf_dev_config, .dev_close = ifcvf_dev_close, - .set_vring_state = NULL, + .set_vring_state = ifcvf_set_vring_state, .set_features = ifcvf_set_features, .migration_done = NULL, .get_vfio_group_fd = ifcvf_get_vfio_group_fd, --1.8.3.1
Quoted 6 lines · Andy Pei
It's useful for hardware vhost backend (like vDPA devices) to set multiqueue configuration accordingly.
Signed-off-by Xiaolong Ye <[email protected]>Signed-off-by Andy Pei <[email protected]>---
I think there's something wrong with this patch - I see the following error after all the patches are applied:
drivers/a715181@@tmp_rte_pmd_ifc@sta/net_ifc_ifcvf_vdpa.c.o: In function `ifcvf_set_vring_state':
ifcvf_vdpa.c:(.text+0x1157): undefined reference to `rte_vhost_get_active_vring_num'
Looking at the linker line, I see that librte_vhost is being included.
Is it possible that you need to make an export for this?
Travis build:
https://travis-ci.com/ovsrobot/dpdk/jobs/231687745
Possibly, this is because the robot needs to call the right script to apply to a -next tree. I'd be a little surprised, though.
Quoted 57 lines
lib/librte_vhost/rte_vhost.h | 12 ++++++++++++ lib/librte_vhost/vhost.c | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+)lib/librte_vhost/rte_vhost.h +12
-0@@ -525,6 +525,18 @@ int rte_vhost_driver_callback_register(const char *path,uint16_t rte_vhost_get_vring_num (int vid);/** +* Get the number of active vrings of the device. +* +* @param vid +* vhost device ID +* +* @return +* The number of active vrings, 0 on failure +*/ +uint16_t +rte_vhost_get_active_vring_num (int vid); + +/** * Get the virtio net device's ifname, which is the vhost-user socket * file path. * lib/librte_vhost/vhost.c +19
-0@@ -674,6 +674,25 @@return dev->nr_vring; } +uint16_t +rte_vhost_get_active_vring_num (int vid) +{ +struct virtio_net *dev =get_device (vid); +struct vhost_virtqueue *vq; +uint16_t qid; + +if (dev ==NULL ) +return 0 ; + +for (qid =0 ; qid < dev->nr_vring; qid++) { + vq = dev->virtqueue[qid]; +if (!vq->enabled) +break ; + } + +return qid; +} +int rte_vhost_get_ifname (int vid,char *buf,size_t len) {
Hi Aaron,
I try to solve this by avoiding exposing this new API. Hope this works.
BRs,
-----Original Message-----
From: Aaron Conole [mailto:[email protected]]
Sent: Friday, September 6, 2019 9:26 PM
To: Pei, Andy <[email protected]>
Cc: [email protected]; Xu, Rosen <[email protected]>; Ye, Xiaolong <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Xiao W <[email protected]>
Subject: Re: [dpdk-dev] [PATCH 1/4] vhost: introduce new API to get the active vring number
Quoted 6 lines · Andy Pei
It's useful for hardware vhost backend (like vDPA devices) to set multiqueue configuration accordingly.
Signed-off-by Xiaolong Ye <[email protected]>Signed-off-by Andy Pei <[email protected]>---
I think there's something wrong with this patch - I see the following error after all the patches are applied:
drivers/a715181@@tmp_rte_pmd_ifc@sta/net_ifc_ifcvf_vdpa.c.o: In function `ifcvf_set_vring_state':
ifcvf_vdpa.c:(.text+0x1157): undefined reference to `rte_vhost_get_active_vring_num'
Looking at the linker line, I see that librte_vhost is being included.
Is it possible that you need to make an export for this?
Travis build:
https://travis-ci.com/ovsrobot/dpdk/jobs/231687745
Possibly, this is because the robot needs to call the right script to apply to a -next tree. I'd be a little surprised, though.
Quoted 53 lines
lib/librte_vhost/rte_vhost.h | 12 ++++++++++++ lib/librte_vhost/vhost.c | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+)(quoted diff) +0
-0
b/lib/librte_vhost/rte_vhost.hindex 7fb1729..28811b0 100644lib/librte_vhost/rte_vhost.h +10
-0@@ -525,6 +525,18 @@ int rte_vhost_driver_callback_register(const char *path, uint16_t rte_vhost_get_vring_num(int vid);/** +* Get the number of active vrings of the device. +* +* @param vid +* vhost device ID +* +* @return +* The number of active vrings, 0 on failure */ uint16_t +rte_vhost_get_active_vring_num (int vid); + +/** * Get the virtio net device's ifname, which is the vhost-user socket * file path. * lib/librte_vhost/vhost.c index renamed from lib/librte_vhost/vhost.c +0
-0981837b..c714818 100644
lib/librte_vhost/vhost.c +18
-0@@ -674,6 +674,25 @@return dev->nr_vring; } +uint16_t +rte_vhost_get_active_vring_num (int vid) { +struct virtio_net *dev =get_device (vid); +struct vhost_virtqueue *vq; +uint16_t qid; + +if (dev ==NULL ) +return 0 ; + +for (qid =0 ; qid < dev->nr_vring; qid++) { + vq = dev->virtqueue[qid]; +if (!vq->enabled) +break ; + } + +return qid; +} +int rte_vhost_get_ifname (int vid,char *buf,size_t len) {