add multi queue support to vDPA ifc driver
RE: [PATCH v8 07/12] vdpa/ifc: change internal function name
In reply to Maxime CoquelinHi Maxime,
Thanks for your efforts.
Quoted 46 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Tuesday, October 18, 2022 9:45 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>
Subject: Re: [PATCH v8 07/12] vdpa/ifc: change internal function nameQuoted 22 lines · Andy Pei
Change internal function name "
find_internal_resource_by_dev" to "find_internal_resource_by_pci_dev".Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 9c49f9c..73d04ed 100644drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1@@ -121,7 +121,7 @@ struct rte_vdpa_dev_info { }static struct internal_list * -find_internal_resource_by_dev (struct rte_pci_device *pdev) +find_internal_resource_by_pci_dev (struct rte_pci_device *pdev) {int found =0 ;struct internal_list *list;drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1752
Quoted 8 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1@@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ; - list =find_internal_resource_by_dev (pci_dev); + list =find_internal_resource_by_pci_dev (pci_dev);if (list ==NULL ) {DRV_LOG (ERR ,"Invalid device: %s" , pci_dev->name);return -1 ;Reviewed-by Maxime Coquelin <[email protected]>Thanks,
Maxime
RE: [PATCH v8 07/12] vdpa/ifc: change internal function name
In reply to Andy PeiQuoted 40 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 8:08 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v8 07/12] vdpa/ifc: change internal function nameChange internal function name "
find_internal_resource_by_dev" to "find_internal_resource_by_pci_dev".Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)drivers/vdpa/ifc/ifcvf_vdpa.c +2
-3@@ -121,7 +121,7 @@ struct rte_vdpa_dev_info { }static struct internal_list * -find_internal_resource_by_dev (struct rte_pci_device *pdev) +find_internal_resource_by_pci_dev (struct rte_pci_device *pdev) {int found =0 ;struct internal_list *list;@@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ; - list =find_internal_resource_by_dev (pci_dev); + list =find_internal_resource_by_pci_dev (pci_dev);if (list ==NULL ) {DRV_LOG (ERR ,"Invalid device: %s" , pci_dev->name);return -1 ; --
Quoted 1 line
1.8.3.1
RE: [PATCH v8 07/12] vdpa/ifc: change internal function name
HI Chenbo,
Thanks for your review.
Quoted 50 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 19, 2022 2:59 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v8 07/12] vdpa/ifc: change internal function nameQuoted 39 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 8:08 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v8 07/12] vdpa/ifc: change internal function nameChange internal function name "
find_internal_resource_by_dev" to "find_internal_resource_by_pci_dev".Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 9c49f9c..73d04ed 100644drivers/vdpa/ifc/ifcvf_vdpa.c +2
-3@@ -121,7 +121,7 @@ struct rte_vdpa_dev_info { }static struct internal_list * -find_internal_resource_by_dev (struct rte_pci_device *pdev) +find_internal_resource_by_pci_dev (struct rte_pci_device *pdev) {int found =0 ;struct internal_list *list;@@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ; - list =find_internal_resource_by_dev (pci_dev); + list =find_internal_resource_by_pci_dev (pci_dev);if (list ==NULL ) {DRV_LOG (ERR ,"Invalid device: %s" , pci_dev->name);return -1 ; --Quoted 1 line
1.8.3.1
Reviewed-by Chenbo Xia <[email protected]>
PATCH v8 8/12 vdpa/ifc: add internal API to get device
In reply to Andy PeiAdd new internal API "find_internal_resource_by_rte_dev" to get device.
--- drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/ifcvf_vdpa.c +26 -2
@@ -144,6 +144,29 @@ struct rte_vdpa_dev_info {return list; } +static struct internal_list * +find_internal_resource_by_rte_dev (struct rte_device *rte_dev) +{ +int found =0 ; +struct internal_list *list; + +pthread_mutex_lock (&internal_list_lock); + + TAILQ_FOREACH(list, &internal_list, next) { +if (rte_dev == &list->internal->pdev->device) { + found =1 ; +break ; + } + } + +pthread_mutex_unlock (&internal_list_lock); + +if (!found) +return NULL ; + +return list; +} +static int ifcvf_vfio_setup (struct ifcvf_internal *internal) {
@@ -1398,10 +1421,11 @@ struct rte_vdpa_dev_info { {struct ifcvf_internal *internal;struct internal_list *list; +struct rte_device *rte_dev = vdev->device; - list =find_internal_resource_by_vdev (vdev); + list =find_internal_resource_by_rte_dev (rte_dev);if (list ==NULL ) { -DRV_LOG (ERR ,"Invalid vDPA device: %p" , vdev); +DRV_LOG (ERR ,"Invalid rte device: %p" , rte_dev);return -1 ; }
Signature · 1 line
1.8.3.1
Re: [PATCH v8 08/12] vdpa/ifc: add internal API to get device
Quoted 8 lines · Andy Pei
Add new internal API "
find_internal_resource_by_rte_dev" to get device.Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)
Thanks,
Maxime
RE: [PATCH v8 08/12] vdpa/ifc: add internal API to get device
Hi Maxime,
Thanks for your efforts.
Quoted 24 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Tuesday, October 18, 2022 9:48 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>
Subject: Re: [PATCH v8 08/12] vdpa/ifc: add internal API to get deviceQuoted 8 lines · Andy Pei
Add new internal API "
find_internal_resource_by_rte_dev" to get device.Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)Reviewed-by Maxime Coquelin <[email protected]>Thanks,
Maxime
RE: [PATCH v8 08/12] vdpa/ifc: add internal API to get device
In reply to Andy PeiQuoted 66 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 8:08 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v8 08/12] vdpa/ifc: add internal API to get deviceAdd new internal API "
find_internal_resource_by_rte_dev" to get device.Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)drivers/vdpa/ifc/ifcvf_vdpa.c +26
-3@@ -144,6 +144,29 @@ struct rte_vdpa_dev_info {return list; } +static struct internal_list * +find_internal_resource_by_rte_dev (struct rte_device *rte_dev) +{ +int found =0 ; +struct internal_list *list; + +pthread_mutex_lock (&internal_list_lock); + + TAILQ_FOREACH(list, &internal_list, next) { +if (rte_dev == &list->internal->pdev->device) { + found =1 ; +break ; + } + } + +pthread_mutex_unlock (&internal_list_lock); + +if (!found) +return NULL ; + +return list; +} +static int ifcvf_vfio_setup (struct ifcvf_internal *internal) {@@ -1398,10 +1421,11 @@ struct rte_vdpa_dev_info { {struct ifcvf_internal *internal;struct internal_list *list; +struct rte_device *rte_dev = vdev->device; - list =find_internal_resource_by_vdev (vdev); + list =find_internal_resource_by_rte_dev (rte_dev);if (list ==NULL ) { -DRV_LOG (ERR ,"Invalid vDPA device: %p" , vdev); +DRV_LOG (ERR ,"Invalid rte device: %p" , rte_dev);return -1 ; } --
Quoted 1 line
1.8.3.1
RE: [PATCH v8 08/12] vdpa/ifc: add internal API to get device
Hi Chenbo,
Thanks for your review.
Quoted 74 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 19, 2022 3:04 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v8 08/12] vdpa/ifc: add internal API to get deviceQuoted 63 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 8:08 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v8 08/12] vdpa/ifc: add internal API to get deviceAdd new internal API "
find_internal_resource_by_rte_dev" to get device.Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 73d04ed..c16e263 100644drivers/vdpa/ifc/ifcvf_vdpa.c +23
-0@@ -144,6 +144,29 @@ struct rte_vdpa_dev_info {return list; } +static struct internal_list * +find_internal_resource_by_rte_dev (struct rte_device *rte_dev) { +int found =0 ; +struct internal_list *list; + +pthread_mutex_lock (&internal_list_lock); + + TAILQ_FOREACH(list, &internal_list, next) { +if (rte_dev == &list->internal->pdev->device) { + found =1 ; +break ; + } + } + +pthread_mutex_unlock (&internal_list_lock); + +if (!found) +return NULL ; + +return list; +} +static int ifcvf_vfio_setup(struct ifcvf_internal *internal) { @@ -1398,10 +1421,11 @@ struct rte_vdpa_dev_info { { struct ifcvf_internal *internal;drivers/vdpa/ifc/ifcvf_vdpa.c +3
-3struct internal_list *list; +struct rte_device *rte_dev = vdev->device; - list =find_internal_resource_by_vdev (vdev); + list =find_internal_resource_by_rte_dev (rte_dev);if (list ==NULL ) { -DRV_LOG (ERR ,"Invalid vDPA device: %p" , vdev); +DRV_LOG (ERR ,"Invalid rte device: %p" , rte_dev);return -1 ; } --Quoted 1 line
1.8.3.1
Reviewed-by Chenbo Xia <[email protected]>
PATCH v8 9/12 vdpa/ifc: change some driver logic
In reply to Andy PeiInsert internal list element to internal list before register vdpa device, in order to call vdpa ops during vdpa device registration.
--- drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
drivers/vdpa/ifc/ifcvf_vdpa.c +7 -4
@@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->sw_lm = sw_fallback_lm; +pthread_mutex_lock (&internal_list_lock); +TAILQ_INSERT_TAIL (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); + internal->vdev =rte_vdpa_register_device (&pci_dev->device, dev_info[internal->hw.device_type].ops);if (internal->vdev ==NULL ) {DRV_LOG (ERR ,"failed to register device %s" , pci_dev->name); +pthread_mutex_lock (&internal_list_lock); +TAILQ_REMOVE (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); goto error; } -pthread_mutex_lock (&internal_list_lock); -TAILQ_INSERT_TAIL (&internal_list, list, next); -pthread_mutex_unlock (&internal_list_lock); -rte_atomic32_set (&internal->started,1 );update_datapath (internal);
Signature · 1 line
1.8.3.1
Re: [PATCH v8 09/12] vdpa/ifc: change some driver logic
Quoted 9 lines · Andy Pei
Insert internal list element to internal list before register vdpa device, in order to call vdpa ops during vdpa device registration.
Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
Thanks,
Maxime
RE: [PATCH v8 09/12] vdpa/ifc: change some driver logic
Hi Maxime,
Thanks.
Quoted 24 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Tuesday, October 18, 2022 9:57 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>
Subject: Re: [PATCH v8 09/12] vdpa/ifc: change some driver logicQuoted 8 lines · Andy Pei
Insert internal list element to internal list before register vdpa device, in order to call vdpa ops during vdpa device registration.
Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)Reviewed-by Maxime Coquelin <[email protected]>Thanks,
Maxime
RE: [PATCH v8 09/12] vdpa/ifc: change some driver logic
In reply to Andy PeiQuoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 8:08 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v8 09/12] vdpa/ifc: change some driver logic
Quoted 3 lines
Insert internal list element to internal list before register vdpa device, in order to call vdpa ops during vdpa device registration.
Patch seems good but title may be too general..
Maybe: improve internal list logic ?
Thanks,
Chenbo
Quoted 37 lines
Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)drivers/vdpa/ifc/ifcvf_vdpa.c +7
-5@@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->sw_lm = sw_fallback_lm; +pthread_mutex_lock (&internal_list_lock); +TAILQ_INSERT_TAIL (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); + internal->vdev =rte_vdpa_register_device (&pci_dev->device, dev_info[internal->hw.device_type].ops);if (internal->vdev ==NULL ) {DRV_LOG (ERR ,"failed to register device %s" , pci_dev->name); +pthread_mutex_lock (&internal_list_lock); +TAILQ_REMOVE (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); goto error; } -pthread_mutex_lock (&internal_list_lock); -TAILQ_INSERT_TAIL (&internal_list, list, next); -pthread_mutex_unlock (&internal_list_lock); -rte_atomic32_set (&internal->started,1 );update_datapath (internal); --1.8.3.1
RE: [PATCH v8 09/12] vdpa/ifc: change some driver logic
Hi Chenbo,
Thanks for your review, I will send v9 to do this.
Quoted 64 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 19, 2022 5:13 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v8 09/12] vdpa/ifc: change some driver logicQuoted 11 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 8:08 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v8 09/12] vdpa/ifc: change some driver logicInsert internal list element to internal list before register vdpa device, in order to call vdpa ops during vdpa device registration.
Patch seems good but title may be too general..
Maybe: improve internal list logic ?
Thanks,
ChenboQuoted 37 lines
Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex c16e263..8dfd493 100644drivers/vdpa/ifc/ifcvf_vdpa.c +7
-5@@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->sw_lm = sw_fallback_lm; +pthread_mutex_lock (&internal_list_lock); +TAILQ_INSERT_TAIL (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); + internal->vdev =rte_vdpa_register_device (&pci_dev->device, dev_info[internal->hw.device_type].ops);if (internal->vdev ==NULL ) {DRV_LOG (ERR ,"failed to register device %s" , pci_dev->name); +pthread_mutex_lock (&internal_list_lock); +TAILQ_REMOVE (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); goto error; } -pthread_mutex_lock (&internal_list_lock); -TAILQ_INSERT_TAIL (&internal_list, list, next); -pthread_mutex_unlock (&internal_list_lock); -rte_atomic32_set (&internal->started,1 );update_datapath (internal); --1.8.3.1
PATCH v8 10/12 vhost: add type to rte vdpa device
In reply to Andy PeiAdd type to rte_vdpa_device to store device type. Call vdpa ops get_dev_type to fill type when register vdpa device.
--- lib/vhost/socket.c | 15 +-------------- lib/vhost/vdpa.c | 15 +++++++++++++++ lib/vhost/vdpa_driver.h | 2 ++ 3 files changed, 18 insertions(+), 14 deletions(-)
lib/vhost/socket.c +1 -14
@@ -627,7 +627,6 @@ struct rte_vdpa_device * {struct vhost_user_socket *vsocket;struct rte_vdpa_device *vdpa_dev; -uint32_t vdpa_type =0 ;int ret =0 ;pthread_mutex_lock (&vhost_user.mutex);
@@ -644,19 +643,7 @@ struct rte_vdpa_device * goto unlock_exit; } -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (path,ERR , -"failed to get vdpa dev type for socket file.\n" ); - ret =-1 ; - goto unlock_exit; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } - - *type = vdpa_type; + *type = vdpa_dev->type; unlock_exit:pthread_mutex_unlock (&vhost_user.mutex);
lib/vhost/vdpa.c +15 -0
@@ -73,6 +73,7 @@ struct rte_vdpa_device * struct rte_vdpa_dev_ops *ops) {struct rte_vdpa_device *dev; +int ret =0 ;if (ops ==NULL )return NULL ;
@@ -101,6 +102,20 @@ struct rte_vdpa_device * dev->device = rte_dev; dev->ops = ops; + +if (ops->get_dev_type) { + ret = ops->get_dev_type (dev, &dev->type); +if (ret) { +VHOST_LOG_CONFIG (rte_dev->name,ERR , +"Failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out_unlock; + } + }else { +/** by default, we assume vdpa device is a net device */ + dev->type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } +TAILQ_INSERT_TAIL (&vdpa_device_list, dev, next); out_unlock:rte_spinlock_unlock (&vdpa_device_list_lock);
lib/vhost/vdpa_driver.h +2 -0
@@ -92,6 +92,8 @@ struct rte_vdpa_device {struct rte_device *device;/** vdpa device operations */ struct rte_vdpa_dev_ops *ops; +/** vdpa device type: net, blk... */ +uint32_t type; };/**
Signature · 1 line
1.8.3.1
Re: [PATCH v8 10/12] vhost: add type to rte vdpa device
Quoted 11 lines · Andy Pei
Add type to
rte_vdpa_deviceto store device type. Call vdpa opsget_dev_typeto fill type when register vdpa device.Signed-off-by Andy Pei <[email protected]>--- lib/vhost/socket.c | 15 +-------------- lib/vhost/vdpa.c | 15 +++++++++++++++ lib/vhost/vdpa_driver.h | 2 ++ 3 files changed, 18 insertions(+), 14 deletions(-)
Thanks for doing the change!
Maxime
RE: [PATCH v8 10/12] vhost: add type to rte vdpa device
Hi Maxime,
Thanks for your efforts.
Quoted 26 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Tuesday, October 18, 2022 9:15 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>
Subject: Re: [PATCH v8 10/12] vhost: add type to rte vdpa deviceQuoted 10 lines · Andy Pei
Add type to
rte_vdpa_deviceto store device type.
Call vdpa opsget_dev_typeto fill type when register vdpa device.Signed-off-by Andy Pei <[email protected]>--- lib/vhost/socket.c | 15 +-------------- lib/vhost/vdpa.c | 15 +++++++++++++++ lib/vhost/vdpa_driver.h | 2 ++ 3 files changed, 18 insertions(+), 14 deletions(-)Reviewed-by Maxime Coquelin <[email protected]>Thanks for doing the change!
Maxime
PATCH v8 11/12 vhost: vDPA blk device gets ready when the first queue is ready
In reply to Andy PeiWhen boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when the first queue is ready.
--- lib/vhost/vhost_user.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-)
lib/vhost/vhost_user.c +17 -16
@@ -1441,11 +1441,14 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int virtio_is_ready (struct virtio_net *dev) { +struct rte_vdpa_device *vdpa_dev;struct vhost_virtqueue *vq; +uint32_t vdpa_type;uint32_t i, nr_vring = dev->nr_vring;if (dev->flags &VIRTIO_DEV_READY )
@@ -1454,13 +1457,22 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) + vdpa_type = vdpa_dev->type; +else + vdpa_type =-1 ; -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];
@@ -2958,7 +2970,6 @@ static int is_vring_iotlb(struct virtio_net *dev,int ret;int unlock_required =0 ;bool handled; -uint32_t vdpa_type =0 ;uint32_t request;uint32_t i;
@@ -3170,17 +3181,7 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (dev->ifname,ERR ,"failed to get vdpa dev type.\n" ); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK +if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL ) goto out;
Signature · 1 line
1.8.3.1
Re: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is ready
Quoted 10 lines · Andy Pei
When boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when the first queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-)
Thanks,
Maxime
RE: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is ready
HI Maxime,
Thanks for your review.
Quoted 27 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Tuesday, October 18, 2022 10:09 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>
Subject: Re: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is readyQuoted 10 lines · Andy Pei
When boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when the first queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-)Reviewed-by Maxime Coquelin <[email protected]>Thanks,
Maxime
RE: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is ready
In reply to Andy PeiQuoted 95 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 8:08 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is readyWhen boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when the first queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-)lib/vhost/vhost_user.c +17
-17@@ -1441,11 +1441,14 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int virtio_is_ready (struct virtio_net *dev) { +struct rte_vdpa_device *vdpa_dev;struct vhost_virtqueue *vq; +uint32_t vdpa_type;uint32_t i, nr_vring = dev->nr_vring;if (dev->flags &VIRTIO_DEV_READY )@@ -1454,13 +1457,22 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) + vdpa_type = vdpa_dev->type; +else + vdpa_type =-1 ; -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];@@ -2958,7 +2970,6 @@ static int is_vring_iotlb(struct virtio_net *dev,int ret;int unlock_required =0 ;bool handled; -uint32_t vdpa_type =0 ;uint32_t request;uint32_t i;@@ -3170,17 +3181,7 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n"); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK +if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL ) goto out; --
Quoted 1 line
1.8.3.1
RE: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is ready
Hi Chenbo,
Thanks for your review.
Quoted 108 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 19, 2022 5:15 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is readyQuoted 47 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 8:08 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is readyWhen boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when the first queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-0cd65257..e0ff79d 100644
lib/vhost/vhost_user.c +6
-2@@ -1441,11 +1441,14 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int virtio_is_ready (struct virtio_net *dev) { +struct rte_vdpa_device *vdpa_dev;struct vhost_virtqueue *vq; +uint32_t vdpa_type;uint32_t i, nr_vring = dev->nr_vring;if (dev->flags &VIRTIO_DEV_READY )@@ -1454,13 +1457,22 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) + vdpa_type = vdpa_dev->type;lib/vhost/vhost_user.c line 1470
Quoted 10 lines
lib/vhost/vhost_user.c +7
-2+else + vdpa_type =-1 ; -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =
VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;Quoted 17 lines
lib/vhost/vhost_user.c +3
-1} +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];@@ -2958,7 +2970,6 @@ static int is_vring_iotlb(struct virtio_net *dev,int ret;int unlock_required =0 ;bool handled; -uint32_t vdpa_type =0 ;uint32_t request;uint32_t i;lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev, · line 3176
Quoted 7 lines
lib/vhost/vhost_user.c +0
-4@@ -3170,17 +3181,7 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed toget vdpa
Quoted 13 lines
lib/vhost/vhost_user.c +1
-8dev type.\n"); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK +if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL ) goto out; --Quoted 1 line
1.8.3.1
Reviewed-by Chenbo Xia <[email protected]>
PATCH v8 12/12 vhost: improve vDPA blk device configure condition
In reply to Andy PeiTo support multi-queue, configure device
after call fd of all queues are set.
--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
lib/vhost/vhost_user.c +10 -3
@@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,bool handled;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )
@@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (vdpa_dev->ops->dev_conf (dev->vid))
Signature · 1 line
1.8.3.1
Re: [PATCH v8 12/12] vhost: improve vDPA blk device configure condition
Quoted 30 lines · Andy Pei
To support multi-queue, configure device
after call fd of all queues are set.Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)lib/vhost/vhost_user.c +3
-3@@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,bool handled;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )@@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) {
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev, · line 3196
Quoted 10 lines
lib/vhost/vhost_user.c +7
-0+ blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (vdpa_dev->ops->dev_conf (dev->vid))
Thanks,
Maxime
RE: [PATCH v8 12/12] vhost: improve vDPA blk device configure condition
Hi Maxime,
Thanks for your review.
Quoted 58 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Tuesday, October 18, 2022 10:15 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>
Subject: Re: [PATCH v8 12/12] vhost: improve vDPA blk device configure conditionQuoted 22 lines · Andy Pei
To support multi-queue, configure device after call fd of all queues are set.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-0e0ff79d..9902ae9 100644
lib/vhost/vhost_user.c +1
-0@@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,bool handled;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev, · line 3187
Quoted 9 lines
lib/vhost/vhost_user.c +3
-3@@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &
VHOST_USER_VRING_IDX_MASK;lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev, · line 3196
Quoted 9 lines
lib/vhost/vhost_user.c +6
-0+if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (vdpa_dev->ops->dev_conf (dev->vid))Reviewed-by Maxime Coquelin <[email protected]>Thanks,
Maxime
RE: [PATCH v8 12/12] vhost: improve vDPA blk device configure condition
In reply to Andy PeiQuoted 50 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 8:08 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v8 12/12] vhost: improve vDPA blk device configure conditionTo support multi-queue, configure device
after call fd of all queues are set.Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)lib/vhost/vhost_user.c +10
-3@@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,bool handled;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )@@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {
Quoted 3 lines
if (
vdpa_dev->ops->dev_conf(dev->vid))
--
1.8.3.1
RE: [PATCH v8 12/12] vhost: improve vDPA blk device configure condition
Hi Chenbo,
Thanks for your review.
Quoted 64 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 19, 2022 5:15 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v8 12/12] vhost: improve vDPA blk device configure conditionQuoted 50 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 8:08 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v8 12/12] vhost: improve vDPA blk device configure conditionTo support multi-queue, configure device after call fd of all queues are set.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-0e0ff79d..9902ae9 100644
lib/vhost/vhost_user.c +10
-3@@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,bool handled;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )@@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {Quoted 3 lines
if (
vdpa_dev->ops->dev_conf(dev->vid))
--
1.8.3.1Reviewed-by Chenbo Xia <[email protected]>
v9 cover vdpa/ifc: add multi queue support
In reply to Andy Peiv9:
fix some commit message.
v8:
change "vdpa_device_type" in "rte_vdpa_device" to "type".
v7:
Fill vdpa_device_type in vdpa device registration.
v6:
Add vdpa_device_type to rte_vdpa_device to store vDPA device type.
v5:
fix some commit message.
rework some code logic.
v4:
fix some commit message.
add some commets to code.
fix some code to reduce confusion.
v3:
rename device ID macro name.
fix some patch title and commit message.
delete some used marco.
rework some code logic.
v2:
fix some coding style issue.
support dynamic enable/disable queue at run time.
Andy Pei (10):
vdpa/ifc: add multi-queue support
vdpa/ifc: set max queues based on virtio spec vdpa/ifc: write queue count to MQ register
vdpa/ifc: only configure enabled queue
vdpa/ifc: change internal function name
vdpa/ifc: add internal API to get device
vdpa/ifc: improve internal list logic
vhost: add type to rte vdpa device
vhost: vDPA blk device gets ready when the first queue is ready
vhost: improve vDPA blk device configure condition
Huang Wei (2):
vdpa/ifc: add new device ID for legacy network device vdpa/ifc: support dynamic enable/disable queue
drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 16 +++- drivers/vdpa/ifc/ifcvf_vdpa.c | 185 +++++++++++++++++++++++++++++++++++------- lib/vhost/socket.c | 15 +--- lib/vhost/vdpa.c | 15 ++++ lib/vhost/vdpa_driver.h | 2 + lib/vhost/vhost_user.c | 38 +++++---- 7 files changed, 354 insertions(+), 61 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v9 1/12 vdpa/ifc: add new device ID for legacy network device
From: Huang Wei <[email protected]>
Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000). Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".
--- drivers/vdpa/ifc/base/ifcvf.h | 6 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +4 -2
@@ -12,12 +12,14 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_BLK_DEVICE_ID 0x0002 +#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001 +#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1
drivers/vdpa/ifc/ifcvf_vdpa.c +10 -3
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_SUBSYS_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .vendor_id =0 ,/* sentinel */
Signature · 1 line
1.8.3.1
PATCH v9 2/12 vdpa/ifc: add multi-queue support
In reply to Andy PeiEnable VHOST_USER_PROTOCOL_F_MQ feature. Expose IFCVF_MQ_OFFSET register to enable multi-queue.
--- drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 12 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +9 -0
@@ -90,6 +90,15 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +/* For some hardware implementation, for example: +* the BAR 4 of PF is NULL, while BAR 4 of VF is not. +* This code makes sure hw->mq_cfg is a valid address. +*/ +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ; +if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );
drivers/vdpa/ifc/base/ifcvf.h +2 -0
@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0
@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;
drivers/vdpa/ifc/ifcvf_vdpa.c +1 -0
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \
Signature · 1 line
1.8.3.1
PATCH v9 3/12 vdpa/ifc: set max queues based on virtio spec
In reply to Andy PeiSet max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues in struct virtio_blk_config.
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg, get the queue pair number using num_queues and set max_queues to it.
--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +1 -1
@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33
drivers/vdpa/ifc/ifcvf_vdpa.c +18 -1
@@ -26,6 +26,12 @@#include "base/ifcvf.h" +/* +* RTE_MIN() cannot be used since braced-group within expression allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc device always has CTRL_VQ, +* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ; +DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues =MIN (IFCVF_MAX_QUEUES , queue_pairs); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal;
Signature · 1 line
1.8.3.1
PATCH v9 4/12 vdpa/ifc: write queue count to MQ register
In reply to Andy PeiWrite queue count to IFCVF_MQ_OFFSET register to enable multi-queue feature.
--- drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +32 -0
@@ -202,6 +202,37 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_mq (struct ifcvf_hw *hw) +{ +u8 *mq_cfg; +u8 qid; +int nr_queue =0 ; + +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue++; + } + +if (nr_queue ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + + mq_cfg = hw->mq_cfg; +if (mq_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + *(u32 *)mq_cfg = nr_queue; +RTE_LOG (INFO ,PMD ,"%d queues are enabled\n" , nr_queue); + }else { + *(u32 *)mq_cfg = nr_queue /2 ; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue /2 ); + } + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
@@ -219,6 +250,7 @@return -1 ; } +ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
Signature · 1 line
1.8.3.1
PATCH v9 5/12 vdpa/ifc: only configure enabled queue
In reply to Andy PeiWhen configuring the hardware queue, we only configure queues which have been enabled by vhost.
--- drivers/vdpa/ifc/base/ifcvf.c | 3 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +3 -0
@@ -252,6 +252,9 @@ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);
drivers/vdpa/ifc/ifcvf_vdpa.c +14 -2
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) !=0 ) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; }
Signature · 1 line
1.8.3.1
PATCH v9 6/12 vdpa/ifc: support dynamic enable/disable queue
In reply to Andy PeiFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
--- drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 93 ++++++++++++++++++++++++++++++++------- 3 files changed, 184 insertions(+), 15 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +100 -0
@@ -233,6 +233,106 @@ } } +int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (i >= (int )hw->nr_vring) +return -1 ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return -1 ; + } + +ifcvf_enable_mq (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +RTE_LOG (ERR ,PMD ,"queue %d, msix vec alloc failed\n" , + i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + } + + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (i >= (int )hw->nr_vring) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); + }else { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ); + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 ); + } + hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
drivers/vdpa/ifc/base/ifcvf.h +6 -0
@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void
drivers/vdpa/ifc/ifcvf_vdpa.c +78 -15
124 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v9 7/12 vdpa/ifc: change internal function name
In reply to Andy PeiChange internal function name "find_internal_resource_by_dev" to "find_internal_resource_by_pci_dev".
--- drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/ifcvf_vdpa.c +2 -2
@@ -121,7 +121,7 @@ struct rte_vdpa_dev_info { }static struct internal_list * -find_internal_resource_by_dev (struct rte_pci_device *pdev) +find_internal_resource_by_pci_dev (struct rte_pci_device *pdev) {int found =0 ;struct internal_list *list;
@@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ; - list =find_internal_resource_by_dev (pci_dev); + list =find_internal_resource_by_pci_dev (pci_dev);if (list ==NULL ) {DRV_LOG (ERR ,"Invalid device: %s" , pci_dev->name);return -1 ;
Signature · 1 line
1.8.3.1
PATCH v9 8/12 vdpa/ifc: add internal API to get device
In reply to Andy PeiAdd new internal API "find_internal_resource_by_rte_dev" to get device.
--- drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/ifcvf_vdpa.c +26 -2
@@ -144,6 +144,29 @@ struct rte_vdpa_dev_info {return list; } +static struct internal_list * +find_internal_resource_by_rte_dev (struct rte_device *rte_dev) +{ +int found =0 ; +struct internal_list *list; + +pthread_mutex_lock (&internal_list_lock); + + TAILQ_FOREACH(list, &internal_list, next) { +if (rte_dev == &list->internal->pdev->device) { + found =1 ; +break ; + } + } + +pthread_mutex_unlock (&internal_list_lock); + +if (!found) +return NULL ; + +return list; +} +static int ifcvf_vfio_setup (struct ifcvf_internal *internal) {
@@ -1398,10 +1421,11 @@ struct rte_vdpa_dev_info { {struct ifcvf_internal *internal;struct internal_list *list; +struct rte_device *rte_dev = vdev->device; - list =find_internal_resource_by_vdev (vdev); + list =find_internal_resource_by_rte_dev (rte_dev);if (list ==NULL ) { -DRV_LOG (ERR ,"Invalid vDPA device: %p" , vdev); +DRV_LOG (ERR ,"Invalid rte device: %p" , rte_dev);return -1 ; }
Signature · 1 line
1.8.3.1
PATCH v9 9/12 vdpa/ifc: improve internal list logic
In reply to Andy PeiInsert internal list element to internal list before register vdpa device, in order to call vdpa ops during vdpa device registration.
--- drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
drivers/vdpa/ifc/ifcvf_vdpa.c +7 -4
@@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->sw_lm = sw_fallback_lm; +pthread_mutex_lock (&internal_list_lock); +TAILQ_INSERT_TAIL (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); + internal->vdev =rte_vdpa_register_device (&pci_dev->device, dev_info[internal->hw.device_type].ops);if (internal->vdev ==NULL ) {DRV_LOG (ERR ,"failed to register device %s" , pci_dev->name); +pthread_mutex_lock (&internal_list_lock); +TAILQ_REMOVE (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); goto error; } -pthread_mutex_lock (&internal_list_lock); -TAILQ_INSERT_TAIL (&internal_list, list, next); -pthread_mutex_unlock (&internal_list_lock); -rte_atomic32_set (&internal->started,1 );update_datapath (internal);
Signature · 1 line
1.8.3.1
RE: [PATCH v9 09/12] vdpa/ifc: improve internal list logic
Quoted 49 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Wednesday, October 19, 2022 4:41 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v9 09/12] vdpa/ifc: improve internal list logicInsert internal list element to internal list before register vdpa device, in order to call vdpa ops during vdpa device registration.
Signed-off-by Andy Pei <[email protected]>Reviewed-by Maxime Coquelin <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)drivers/vdpa/ifc/ifcvf_vdpa.c +7
-5@@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->sw_lm = sw_fallback_lm; +pthread_mutex_lock (&internal_list_lock); +TAILQ_INSERT_TAIL (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); + internal->vdev =rte_vdpa_register_device (&pci_dev->device, dev_info[internal->hw.device_type].ops);if (internal->vdev ==NULL ) {DRV_LOG (ERR ,"failed to register device %s" , pci_dev->name); +pthread_mutex_lock (&internal_list_lock); +TAILQ_REMOVE (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); goto error; } -pthread_mutex_lock (&internal_list_lock); -TAILQ_INSERT_TAIL (&internal_list, list, next); -pthread_mutex_unlock (&internal_list_lock); -rte_atomic32_set (&internal->started,1 );update_datapath (internal); --
Quoted 1 line
1.8.3.1
RE: [PATCH v9 09/12] vdpa/ifc: improve internal list logic
Hi Chenbo,
Thanks.
Quoted 59 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Thursday, October 20, 2022 11:22 AM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v9 09/12] vdpa/ifc: improve internal list logicQuoted 48 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Wednesday, October 19, 2022 4:41 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v9 09/12] vdpa/ifc: improve internal list logicInsert internal list element to internal list before register vdpa device, in order to call vdpa ops during vdpa device registration.
Signed-off-by Andy Pei <[email protected]>Reviewed-by Maxime Coquelin <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex c16e263..8dfd493 100644drivers/vdpa/ifc/ifcvf_vdpa.c +7
-5@@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->sw_lm = sw_fallback_lm; +pthread_mutex_lock (&internal_list_lock); +TAILQ_INSERT_TAIL (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); + internal->vdev =rte_vdpa_register_device (&pci_dev->device, dev_info[internal->hw.device_type].ops);if (internal->vdev ==NULL ) {DRV_LOG (ERR ,"failed to register device %s" , pci_dev->name); +pthread_mutex_lock (&internal_list_lock); +TAILQ_REMOVE (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); goto error; } -pthread_mutex_lock (&internal_list_lock); -TAILQ_INSERT_TAIL (&internal_list, list, next); -pthread_mutex_unlock (&internal_list_lock); -rte_atomic32_set (&internal->started,1 );update_datapath (internal); --Quoted 1 line
1.8.3.1
Reviewed-by Chenbo Xia <[email protected]>
PATCH v9 10/12 vhost: add type to rte vdpa device
In reply to Andy PeiAdd type to rte_vdpa_device to store device type. Call vdpa ops get_dev_type to fill type when register vdpa device.
--- lib/vhost/socket.c | 15 +-------------- lib/vhost/vdpa.c | 15 +++++++++++++++ lib/vhost/vdpa_driver.h | 2 ++ 3 files changed, 18 insertions(+), 14 deletions(-)
lib/vhost/socket.c +1 -14
@@ -627,7 +627,6 @@ struct rte_vdpa_device * {struct vhost_user_socket *vsocket;struct rte_vdpa_device *vdpa_dev; -uint32_t vdpa_type =0 ;int ret =0 ;pthread_mutex_lock (&vhost_user.mutex);
@@ -644,19 +643,7 @@ struct rte_vdpa_device * goto unlock_exit; } -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (path,ERR , -"failed to get vdpa dev type for socket file.\n" ); - ret =-1 ; - goto unlock_exit; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } - - *type = vdpa_type; + *type = vdpa_dev->type; unlock_exit:pthread_mutex_unlock (&vhost_user.mutex);
lib/vhost/vdpa.c +15 -0
@@ -73,6 +73,7 @@ struct rte_vdpa_device * struct rte_vdpa_dev_ops *ops) {struct rte_vdpa_device *dev; +int ret =0 ;if (ops ==NULL )return NULL ;
@@ -101,6 +102,20 @@ struct rte_vdpa_device * dev->device = rte_dev; dev->ops = ops; + +if (ops->get_dev_type) { + ret = ops->get_dev_type (dev, &dev->type); +if (ret) { +VHOST_LOG_CONFIG (rte_dev->name,ERR , +"Failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out_unlock; + } + }else { +/** by default, we assume vdpa device is a net device */ + dev->type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } +TAILQ_INSERT_TAIL (&vdpa_device_list, dev, next); out_unlock:rte_spinlock_unlock (&vdpa_device_list_lock);
lib/vhost/vdpa_driver.h +2 -0
@@ -92,6 +92,8 @@ struct rte_vdpa_device {struct rte_device *device;/** vdpa device operations */ struct rte_vdpa_dev_ops *ops; +/** vdpa device type: net, blk... */ +uint32_t type; };/**
Signature · 1 line
1.8.3.1
PATCH v9 11/12 vhost: vDPA blk device gets ready when the first queue is ready
In reply to Andy PeiWhen boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when the first queue is ready.
--- lib/vhost/vhost_user.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-)
lib/vhost/vhost_user.c +17 -16
@@ -1441,11 +1441,14 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int virtio_is_ready (struct virtio_net *dev) { +struct rte_vdpa_device *vdpa_dev;struct vhost_virtqueue *vq; +uint32_t vdpa_type;uint32_t i, nr_vring = dev->nr_vring;if (dev->flags &VIRTIO_DEV_READY )
@@ -1454,13 +1457,22 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) + vdpa_type = vdpa_dev->type; +else + vdpa_type =-1 ; -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];
@@ -2958,7 +2970,6 @@ static int is_vring_iotlb(struct virtio_net *dev,int ret;int unlock_required =0 ;bool handled; -uint32_t vdpa_type =0 ;uint32_t request;uint32_t i;
@@ -3170,17 +3181,7 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (dev->ifname,ERR ,"failed to get vdpa dev type.\n" ); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK +if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL ) goto out;
Signature · 1 line
1.8.3.1
PATCH v9 12/12 vhost: improve vDPA blk device configure condition
In reply to Andy PeiTo support multi-queue, configure device after call fd of all queues are set.
--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
lib/vhost/vhost_user.c +10 -3
@@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,bool handled;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )
@@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_dev->type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (vdpa_dev->ops->dev_conf (dev->vid))
Signature · 1 line
1.8.3.1
RE: [PATCH v9 00/12] vdpa/ifc: add multi queue support
In reply to Andy PeiQuoted 66 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Wednesday, October 19, 2022 4:41 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v9 00/12] vdpa/ifc: add multi queue supportv9:
fix some commit message.v8:
change "vdpa_device_type" in "rte_vdpa_device" to "type".v7:
Fillvdpa_device_typein vdpa device registration.v6:
Addvdpa_device_typetorte_vdpa_deviceto store vDPA device type.v5:
fix some commit message.
rework some code logic.v4:
fix some commit message.
add some commets to code.
fix some code to reduce confusion.v3:
rename device ID macro name.fix some patch title and commit message.
delete some used marco.
rework some code logic.v2:
fix some coding style issue.
support dynamic enable/disable queue at run time.Andy Pei (10):
vdpa/ifc: add multi-queue support
vdpa/ifc: set max queues based on virtio spec vdpa/ifc: write queue count to MQ register
vdpa/ifc: only configure enabled queue
vdpa/ifc: change internal function name
vdpa/ifc: add internal API to get device
vdpa/ifc: improve internal list logic
vhost: add type to rte vdpa device
vhost: vDPA blk device gets ready when the first queue is ready
vhost: improve vDPA blk device configure conditionHuang Wei (2):
vdpa/ifc: add new device ID for legacy network device vdpa/ifc: support dynamic enable/disable queuedrivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 16 +++- drivers/vdpa/ifc/ifcvf_vdpa.c | 185 +++++++++++++++++++++++++++++++++++-- ----- lib/vhost/socket.c | 15 +--- lib/vhost/vdpa.c | 15 ++++ lib/vhost/vdpa_driver.h | 2 + lib/vhost/vhost_user.c | 38 +++++---- 7 files changed, 354 insertions(+), 61 deletions(-)
Quoted 2 lines
--
1.8.3.1
Change title ' vhost: add type to rte vdpa device ' to ' vhost: add type to vDPA device '
Series applied to next-virtio/main, Thanks
RE: [PATCH v9 00/12] vdpa/ifc: add multi queue support
Quoted 78 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 26, 2022 5:00 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v9 00/12] vdpa/ifc: add multi queue supportQuoted 52 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Wednesday, October 19, 2022 4:41 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v9 00/12] vdpa/ifc: add multi queue supportv9:
fix some commit message.v8:
change "vdpa_device_type" in "rte_vdpa_device" to "type".v7:
Fillvdpa_device_typein vdpa device registration.v6:
Addvdpa_device_typetorte_vdpa_deviceto store vDPA device type.v5:
fix some commit message.
rework some code logic.v4:
fix some commit message.
add some commets to code.
fix some code to reduce confusion.v3:
rename device ID macro name.fix some patch title and commit message.
delete some used marco.
rework some code logic.v2:
fix some coding style issue.
support dynamic enable/disable queue at run time.Andy Pei (10):
vdpa/ifc: add multi-queue support
vdpa/ifc: set max queues based on virtio spec vdpa/ifc: write queue count to MQ register
vdpa/ifc: only configure enabled queue
vdpa/ifc: change internal function name
vdpa/ifc: add internal API to get device
vdpa/ifc: improve internal list logic
vhost: add type to rte vdpa device
vhost: vDPA blk device gets ready when the first queue is ready
vhost: improve vDPA blk device configure conditionQuoted 5 lines
Huang Wei (2):
vdpa/ifc: add new device ID for legacy network device vdpa/ifc: support dynamic enable/disable queuedrivers/vdpa/ifc/base/ifcvf.c | 144++++++++++++++++++++++++++++++++
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.h| 16 +++-drivers/vdpa/ifc/ifcvf_vdpa.c| 185
+++++++++++++++++++++++++++++++++++--
-----lib/vhost/socket.c | 15 +--- lib/vhost/vdpa.c | 15 ++++ lib/vhost/vdpa_driver.h | 2 + lib/vhost/vhost_user.c | 38 +++++---- 7 files changed, 354 insertions(+), 61 deletions(-)
Quoted 6 lines
Quoted 2 lines
--
1.8.3.1Change title ' vhost: add type to rte vdpa device ' to ' vhost: add type to vDPA device '
OK. Thanks Chenbo.
Quoted 1 line
Series applied to next-virtio/main, Thanks
PATCH 2/8 vdpa/ifc: add multi queue suppoort
In reply to Andy PeiEnable VHOST_USER_PROTOCOL_F_MQ feature. ExposeIFCVF_MQ_OFFSET register to enable multi queue.
--- drivers/vdpa/ifc/base/ifcvf.c | 5 +++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 8 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +5 -0
@@ -90,6 +90,11 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ; +if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );
drivers/vdpa/ifc/base/ifcvf.h +2 -0
@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0
@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;
drivers/vdpa/ifc/ifcvf_vdpa.c +1 -0
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \
Signature · 1 line
1.8.3.1
PATCH 3/8 vdpa/ifc: set max queues according to HW spec
In reply to Andy PeiSet max_queues according to virtio HW spec.
For virtio BLK device, set max_queues to the value of "num_queues". "num_queues" is element of struct virtio_blk_config.
--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +1 -1
@@ -21,7 +21,7 @@#define IFCVF_NET_DEVICE_ID 0x0001#define IFCVF_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33
drivers/vdpa/ifc/ifcvf_vdpa.c +17 -1
@@ -26,6 +26,18 @@#include "base/ifcvf.h" +/** +** RTE_MAX() and RTE_MIN() cannot be used since braced-group within +** expression allowed only inside a function, but MAX() is used as +** a number of elements in array. +**/ +#ifndef MAX +#define MAX (v1, v2) ((v1) > (v2) ? (v1) : (v2)) +#endif +#ifndef MIN +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +#endif +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1559,7 +1571,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);
@@ -1570,6 +1581,8 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + (internal->hw.common_cfg->num_queues -1 )/2 ); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;
@@ -1609,6 +1622,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal;
Signature · 1 line
1.8.3.1
PATCH 4/8 vdpa/ifc: write queue count to MQ register
In reply to Andy PeiWrite queue count to IFCVF_MQ_OFFSET register
to enable multi queue feature.
--- drivers/vdpa/ifc/base/ifcvf.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +14 -0
@@ -198,6 +198,19 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_multiqueue (struct ifcvf_hw *hw,u16 nr_queue_pair) +{ +u8 *mq_cfg; + +if (hw->device_type ==IFCVF_NET ) + nr_queue_pair = (nr_queue_pair +1 ) /2 ; + + mq_cfg = hw->mq_cfg; +if (mq_cfg) + *(u32 *)mq_cfg = nr_queue_pair; +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
@@ -215,6 +228,7 @@return -1 ; } +ifcvf_enable_multiqueue (hw, hw->nr_vring);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
Signature · 1 line
1.8.3.1
PATCH 5/8 vdpa/ifc: only configure enabled queue
In reply to Andy Peiwhen configure the hardware queue, we only configure queues which have been enabled by vhost.
--- drivers/vdpa/ifc/base/ifcvf.c | 5 ++++- drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-)
drivers/vdpa/ifc/base/ifcvf.c +4 -1
@@ -230,6 +230,8 @@ifcvf_enable_multiqueue (hw, hw->nr_vring);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);
@@ -264,7 +266,8 @@ notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + notify_off * hw->notify_off_multiplier); -IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); +if (hw->vring[i].enable) +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); }return 0 ;
drivers/vdpa/ifc/ifcvf_vdpa.c +4 -0
@@ -290,6 +290,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {
@@ -505,6 +507,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;
Signature · 1 line
1.8.3.1
PATCH 6/8 vdpa/ifc: set vring state callback update data path
In reply to Andy PeiTo support multi queue, in the case that first queue is ready and device is configured, when more queues need to be configured, we just close and restart data path.
This also fix the situation that using set_vring_state callback to disable one queue will cause all vfio interrupts being disabled.
--- drivers/vdpa/ifc/ifcvf_vdpa.c | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-)
drivers/vdpa/ifc/ifcvf_vdpa.c +9 -18
@@ -1286,8 +1286,6 @@ struct rte_vdpa_dev_info {struct internal_list *list;struct ifcvf_internal *internal;struct ifcvf_hw *hw; -struct ifcvf_pci_common_cfg *cfg; -int ret =0 ; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);
@@ -1303,27 +1301,20 @@ struct rte_vdpa_dev_info { } hw = &internal->hw; -if (!internal->configured) - goto exit; - cfg = hw->common_cfg; -IFCVF_WRITE_REG16 (vring, &cfg->queue_select); -IFCVF_WRITE_REG16 (!!state, &cfg->queue_enable); + hw->vring[vring].enable = !!state; -if (!state && hw->vring[vring].enable) { - ret =vdpa_disable_vfio_intr (internal); -if (ret) -return ret; - } +if (!internal->configured) + goto exit; -if (state && !hw->vring[vring].enable) { - ret =vdpa_enable_vfio_intr (internal, false); -if (ret) -return ret; - } +/* close data path */ +rte_atomic32_set (&internal->dev_attached,0 ); +update_datapath (internal); +/* restart data path */ +rte_atomic32_set (&internal->dev_attached,1 ); +update_datapath (internal); exit: - hw->vring[vring].enable = !!state;return 0 ; }
Signature · 1 line
1.8.3.1
PATCH 7/8 vhost: configure device when any queue is ready for BLK device
In reply to Andy PeiWhen boot from virtio blk device, seabois in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end conf_dev when any queue is ready.
--- lib/vhost/vhost_user.c | 56 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 12 deletions(-)
lib/vhost/vhost_user.c +44 -12
@@ -1451,6 +1451,25 @@#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2ustatic int +virtio_has_queue_ready (struct virtio_net *dev) +{ +struct vhost_virtqueue *vq; +uint32_t i, nr_vring = dev->nr_vring; + +if (!dev->nr_vring) +return 0 ; + +for (i =0 ; i < nr_vring; i++) { + vq = dev->virtqueue[i]; + +if (vq_is_ready (dev, vq)) +return 1 ; + } + +return 0 ; +} + +static int virtio_is_ready (struct virtio_net *dev) {struct vhost_virtqueue *vq;
@@ -3167,9 +3186,33 @@ static int is_vring_iotlb(struct virtio_net *dev,if (unlock_required)vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) goto out; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev)) { +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (!virtio_has_queue_ready (dev)) + goto out; + }else { + goto out; + } + } +/* * Virtio is now ready. If not done already, it is time * to notify the application it can process the rings and
@@ -3181,20 +3224,9 @@ static int is_vring_iotlb(struct virtio_net *dev, dev->flags |=VIRTIO_DEV_RUNNING ; } - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (dev->ifname,ERR ,"failed to get vdpa dev type.\n" ); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL ) goto out;
Signature · 1 line
1.8.3.1
PATCH 8/8 vhost: vDPA BLK devices configure device when all queue callfds are set
In reply to Andy PeiIn the virtio blk vDPA live migration use case, before the live migration process, QEMU will set call fd to vDPA back-end. QEMU and vDPA back-end stand by until live migration starts. During live migration process, QEMU sets kick fd and a new call fd. However, after the kick fd is set to the vDPA back-end, the vDPA back-end configures device and data path starts. The new call fd will cause some kind of "re-configuration", this kind of "re-configuration" cause IO drop.
After this patch, vDPA back-end configures device after kick fd and call fd are well set and make sure no IO drops. This patch only impact virtio blk vDPA device and does not impact net device.
--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
lib/vhost/vhost_user.c +10 -3
@@ -2994,6 +2994,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type =0 ;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )
@@ -3227,9 +3228,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (vdpa_dev->ops->dev_conf (dev->vid))
Signature · 1 line
1.8.3.1