add multi queue support to vDPA ifc driver
cover add multi queue support to vDPA ifc driver
Add multi queue support to vDPA ifc driver.
Multi queue support for virtio-net device and virtio-blk device.
Andy Pei (7):
vdpa/ifc: add multi queue suppoort
vdpa/ifc: set max queues according to HW spec
vdpa/ifc: write queue count to MQ register
vdpa/ifc: only configure enabled queue
vdpa/ifc: set vring state callback update data path
vhost: configure device when any queue is ready for BLK device
vhost: vDPA BLK devices configure device when all queue callfds are
set
Huang Wei (1):
vdpa/ifc: add new device ID
drivers/vdpa/ifc/base/ifcvf.c | 24 ++++++++++++++++- drivers/vdpa/ifc/base/ifcvf.h | 8 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 59 ++++++++++++++++++++++++++-------------- lib/vhost/vhost_user.c | 63 ++++++++++++++++++++++++++++++++++--------- 4 files changed, 119 insertions(+), 35 deletions(-)
Signature · 1 line
1.8.3.1
PATCH 1/8 vdpa/ifc: add new device ID
From: Huang Wei <[email protected]>
Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
--- drivers/vdpa/ifc/base/ifcvf.h | 4 +++- drivers/vdpa/ifc/ifcvf_vdpa.c | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +3 -1
@@ -12,11 +12,13 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A +#define IFCVF_NET_DEVICE_ID 0x0001#define IFCVF_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1
drivers/vdpa/ifc/ifcvf_vdpa.c +8 -1
@@ -1684,13 +1684,20 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_BLK_DEVICE_ID ,
Signature · 1 line
1.8.3.1
v2 cover vdpa/ifc: add multi queue support
v2:
fix some coding style issue.
support dynamic enable/disable queue at run time.
Andy Pei (6):
vdpa/ifc: add multi queue support
vdpa/ifc: set max queues according to HW spec
vdpa/ifc: write queue count to MQ register
vdpa/ifc: only configure enabled queue
vhost: configure device when any queue is ready for BLK device
vhost: vDPA BLK devices configure device when all queue callfds are
set
Huang Wei (2):
vdpa/ifc: add new device ID
vdpa/ifc: support dynamic enable/disable queue
drivers/vdpa/ifc/base/ifcvf.c | 145 +++++++++++++++++++++++++++++++++++- drivers/vdpa/ifc/base/ifcvf.h | 14 +++- drivers/vdpa/ifc/base/ifcvf_osdep.h | 1 + drivers/vdpa/ifc/ifcvf_vdpa.c | 140 +++++++++++++++++++++++++++++----- lib/vhost/vhost_user.c | 63 +++++++++++++--- 5 files changed, 329 insertions(+), 34 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v2 1/8 vdpa/ifc: add new device ID
From: Huang Wei <[email protected]>
Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
--- drivers/vdpa/ifc/base/ifcvf.h | 4 +++- drivers/vdpa/ifc/ifcvf_vdpa.c | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +3 -1
@@ -12,11 +12,13 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A +#define IFCVF_NET_DEVICE_ID 0x0001#define IFCVF_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1
drivers/vdpa/ifc/ifcvf_vdpa.c +8 -1
@@ -1684,13 +1684,20 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_BLK_DEVICE_ID ,
Signature · 1 line
1.8.3.1
RE: [PATCH v2 1/8] vdpa/ifc: add new device ID
Hi Andy,
Quoted 8 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 1/8] vdpa/ifc: add new device ID
Title could be: add new device ID for legacy network device
Quoted 18 lines
From: Huang Wei <[email protected]>
Add new device id to support
IFCVF_NET_TRANSITIONAL_DEVICE_ID(0x1000).Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.h | 4 +++- drivers/vdpa/ifc/ifcvf_vdpa.c | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-)drivers/vdpa/ifc/base/ifcvf.h +0
-0@@ -12,11 +12,13 @@#define IFCVF_BLK 1
drivers/vdpa/ifc/base/ifcvf.h line 21
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.h +3
-1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A +#define IFCVF_NET_DEVICE_ID 0x0001
For subsystem device ID, I suggest to add _SUBSYS_, please check all Subsystem device ID and make all the names well-defined.
Thanks,
Chenbo
Quoted 31 lines
drivers/vdpa/ifc/base/ifcvf.h +0
-0#define IFCVF_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1drivers/vdpa/ifc/ifcvf_vdpa.c +8
-2@@ -1684,13 +1684,20 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_BLK_DEVICE_ID , --1.8.3.1
RE: [PATCH v2 1/8] vdpa/ifc: add new device ID
Hi Chenbo,
Thanks for your reply.
I will send V3 patch to address your 2 comments.
Quoted 89 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Friday, September 9, 2022 8:29 AM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: RE: [PATCH v2 1/8] vdpa/ifc: add new device IDHi Andy,
Quoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 1/8] vdpa/ifc: add new device IDTitle could be: add new device ID for legacy network device
Quoted 18 lines
From: Huang Wei <[email protected]>
Add new device id to support
IFCVF_NET_TRANSITIONAL_DEVICE_ID(0x1000).Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>---
drivers/vdpa/ifc/base/ifcvf.h| 4 +++-drivers/vdpa/ifc/ifcvf_vdpa.c
| 9 ++++++++-2 files changed, 11 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.hindex 9d95aac..7ede738 100644drivers/vdpa/ifc/base/ifcvf.h +0
-0@@ -12,11 +12,13 @@#define IFCVF_BLK 1drivers/vdpa/ifc/base/ifcvf.h line 21
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.h +3
-1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A +#define IFCVF_NET_DEVICE_ID 0x0001For subsystem device ID, I suggest to add _SUBSYS_, please check all Subsystem device ID and make all the names well-defined.
Thanks,
ChenboQuoted 31 lines
drivers/vdpa/ifc/base/ifcvf.h +0
-0#define IFCVF_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex ac42de9..61d0250 100644drivers/vdpa/ifc/ifcvf_vdpa.c +8
-2@@ -1684,13 +1684,20 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_BLK_DEVICE_ID , --1.8.3.1
PATCH v2 2/8 vdpa/ifc: add multi queue support
In reply to Andy PeiEnable VHOST_USER_PROTOCOL_F_MQ feature. ExposeIFCVF_MQ_OFFSET register to enable multi queue.
--- drivers/vdpa/ifc/base/ifcvf.c | 5 +++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 8 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +5 -0
@@ -90,6 +90,11 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ; +if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );
drivers/vdpa/ifc/base/ifcvf.h +2 -0
@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0
@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;
drivers/vdpa/ifc/ifcvf_vdpa.c +1 -0
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \
Signature · 1 line
1.8.3.1
RE: [PATCH v2 2/8] vdpa/ifc: add multi queue support
Quoted 8 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 2/8] vdpa/ifc: add multi queue support
multi-queue
Quoted 3 lines
Enable
VHOST_USER_PROTOCOL_F_MQfeature.ExposeIFCVF_MQ_OFFSETregister to enable multi queue.
Please rephase it in a better way, at least add a space before IFCVF_MP_OFFSET...
Quoted 14 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 5 +++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 8 insertions(+)drivers/vdpa/ifc/base/ifcvf.c +0
-0
drivers/vdpa/ifc/base/ifcvf.c line 96
Quoted 7 lines
drivers/vdpa/ifc/base/ifcvf.c +4
-0@@ -90,6 +90,11 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ;
Could you help me understand the logic here? There are two cases that BAR 4 mmap-able and not mmap-able?
Thanks,
Chenbo
Quoted 38 lines
drivers/vdpa/ifc/base/ifcvf.c +1
-0+if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );drivers/vdpa/ifc/base/ifcvf.h +2
-0@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \ --1.8.3.1
RE: [PATCH v2 2/8] vdpa/ifc: add multi queue support
Hi Chenbo,
Thanks for your reply.
My reply is inline.
Quoted 19 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Friday, September 9, 2022 8:29 AM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: RE: [PATCH v2 2/8] vdpa/ifc: add multi queue supportQuoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 2/8] vdpa/ifc: add multi queue support
Quoted 2 lines
multi-queue
Sure.
Quoted 7 lines
Quoted 3 lines
Enable
VHOST_USER_PROTOCOL_F_MQfeature.ExposeIFCVF_MQ_OFFSETregister to enable multi queue.Please rephase it in a better way, at least add a space before
IFCVF_MP_OFFSET...
I will make it "Expose IFCVF_MQ_OFFSET register to enable multi queue."
Quoted 17 lines
Quoted 17 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 5 +++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 8 insertions(+)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex f1e1474..34c8226 100644drivers/vdpa/ifc/base/ifcvf.c +0
-0@@ -90,6 +90,11 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" );
Quoted 9 lines
drivers/vdpa/ifc/base/ifcvf.c line 94
Quoted 2 lines
drivers/vdpa/ifc/base/ifcvf.c +2
-0+if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4].addr +
IFCVF_MQ_OFFSET;drivers/vdpa/ifc/base/ifcvf.c line 96
Quoted 2 lines
drivers/vdpa/ifc/base/ifcvf.c +2
-0+else + hw->mq_cfg =NULL ;Could you help me understand the logic here? There are two cases that BAR 4 mmap-able and not mmap-able?
For some hardware implementation, for example the BAR 4 of PF is NULL, while BAR 4 of VF is not. This code makes sure hw->mq_cfg is a valid address.
I am planning a optimization to check parameter consistent at init phase.
For example, if hardware does not support HW-assist live migration, we cannot use parameter like "sw-live-migration=0".
Quoted 41 lines
Thanks,
ChenboQuoted 38 lines
drivers/vdpa/ifc/base/ifcvf.c +1
-0+if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) { DEBUGOUT("capability incomplete\n"); diff --git
a/drivers/vdpa/ifc/base/ifcvf.hb/drivers/vdpa/ifc/base/ifcvf.hindex 7ede738..ad505f1 100644drivers/vdpa/ifc/base/ifcvf.h +2
-0@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 61d0250..2d165c0 100644drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \ --1.8.3.1
PATCH v2 3/8 vdpa/ifc: set max queues according to HW spec
In reply to Andy PeiSet max_queues according to virtio HW spec.
For virtio BLK device, set max_queues to the value of "num_queues". "num_queues" is element of struct virtio_blk_config.
--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +1 -1
@@ -21,7 +21,7 @@#define IFCVF_NET_DEVICE_ID 0x0001#define IFCVF_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33
drivers/vdpa/ifc/ifcvf_vdpa.c +20 -1
@@ -26,6 +26,18 @@#include "base/ifcvf.h" +/* +* RTE_MAX() and RTE_MIN() cannot be used since braced-group within +* expression allowed only inside a function, but MAX() is used as +* a number of elements in array. +*/ +#ifndef MAX +#define MAX (v1, v2) ((v1) > (v2) ? (v1) : (v2)) +#endif +#ifndef MIN +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +#endif +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1524,7 @@ struct rte_vdpa_dev_info dev_info[] = {uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;
@@ -1559,7 +1572,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);
@@ -1570,6 +1582,10 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ; +DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues =MIN (IFCVF_MAX_QUEUES , queue_pairs); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;
@@ -1609,6 +1625,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal;
Signature · 1 line
1.8.3.1
RE: [PATCH v2 3/8] vdpa/ifc: set max queues according to HW spec
Quoted 8 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 3/8] vdpa/ifc: set max queues according to HW spec
vdpa/ifc: set max queues based on virtio spec
Quoted 4 lines
Set
max_queuesaccording to virtio HW spec.
For virtio BLK device, setmax_queuesto the value of "num_queues". "num_queues" is element of structvirtio_blk_config.
Both virtio-net/blk should be described.
Quoted 3 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>
Email is wrong, please fix all in next version
Quoted 27 lines
--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-)drivers/vdpa/ifc/base/ifcvf.h +1
-1@@ -21,7 +21,7 @@#define IFCVF_NET_DEVICE_ID 0x0001#define IFCVF_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33drivers/vdpa/ifc/ifcvf_vdpa.c +1
-0@@ -26,6 +26,18 @@#include "base/ifcvf.h" +/*
drivers/vdpa/ifc/ifcvf_vdpa.c line 39
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +10
-0+* RTE_MAX() and RTE_MIN() cannot be used since braced-group within +* expression allowed only inside a function, but MAX() is used as +* a number of elements in array. +*/ +#ifndef MAX +#define MAX (v1, v2) ((v1) > (v2) ? (v1) : (v2)) +#endif +#ifndef MIN +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +#endif
Above ifndef is not needed?
Seems MAX is not used, so remove it
Quoted 21 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +2
-1+RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \@@ -1512,6 +1524,7 @@ struct rte_vdpa_dev_info dev_info[] = {uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;@@ -1559,7 +1572,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1585
Quoted 4 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +1
-0@@ -1570,6 +1582,10 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ;
Please note this logic assumes CTRL_VQ is always there, if for the read hardware, that is the case, then it's fine. You can decide yourself to check CTRL_VQ feature is there or not.
Thanks,
Chenbo
Quoted 18 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +6
-1+DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues =MIN (IFCVF_MAX_QUEUES , queue_pairs); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;@@ -1609,6 +1625,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal; --1.8.3.1
RE: [PATCH v2 3/8] vdpa/ifc: set max queues according to HW spec
Hi Chenbo,
Thanks for your reply.
My reply is inline.
Quoted 18 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Friday, September 9, 2022 1:56 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v2 3/8] vdpa/ifc: set max queues according to HW specQuoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 3/8] vdpa/ifc: set max queues according to HW spec
Quoted 2 lines
vdpa/ifc: set max queues based on virtio spec
OK
Quoted 7 lines
Quoted 4 lines
Set
max_queuesaccording to virtio HW spec.
For virtio BLK device, setmax_queuesto the value of "num_queues". "num_queues" is element of structvirtio_blk_config.Both virtio-net/blk should be described.
OK
Quoted 6 lines
Quoted 3 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>Email is wrong, please fix all in next version
OK
Quoted 40 lines
Quoted 27 lines
---
drivers/vdpa/ifc/base/ifcvf.h| 2 +-drivers/vdpa/ifc/ifcvf_vdpa.c
| 21 ++++++++++++++++++++-2 files changed, 21 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.hindex ad505f1..c17bf2a 100644drivers/vdpa/ifc/base/ifcvf.h +1
-1@@ -21,7 +21,7 @@#define IFCVF_NET_DEVICE_ID 0x0001#define IFCVF_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 2d165c0..2b42850 100644drivers/vdpa/ifc/ifcvf_vdpa.c +1
-0@@ -26,6 +26,18 @@#include "base/ifcvf.h" +/* drivers/vdpa/ifc/ifcvf_vdpa.c line 39
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +10
-0+* RTE_MAX() and RTE_MIN() cannot be used since braced-group within +* expression allowed only inside a function, but MAX() is used as +* a number of elements in array. +*/ +#ifndef MAX +#define MAX (v1, v2) ((v1) > (v2) ? (v1) : (v2)) +#endif +#ifndef MIN +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +#endif Above ifndef is not needed?
Quoted 2 lines
Seems MAX is not used, so remove it
OK
Quoted 22 lines
Quoted 22 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +1
-0+RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \ @@ -1512,6 +1524,7@@ struct
rte_vdpa_dev_infodev_info[] = {drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;@@ -1559,7 +1572,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);@@ -1570,6 +1582,10 @@ struct rte_vdpa_dev_info dev_info[] = {
Quoted 8 lines
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1585
Quoted 3 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +1
-0if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ;Please note this logic assumes
CTRL_VQis always there, if for the read hardware, that is the case, then it's fine. You can decide yourself to checkCTRL_VQfeature is there or not.
I will keep the code as it is now, and add some comments to explain ifc driver assumes CTRL_VQ is always there.
Quoted 24 lines
Thanks,
Chenbodrivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = {
Quoted 3 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +3
-0+DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues = MIN(IFCVF_MAX_QUEUES,
queue_pairs);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1590
Quoted 2 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +0
-0internal->features = features & ~(1ULL <<
VIRTIO_F_IOMMU_PLATFORM);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1591
Quoted 1 line
drivers/vdpa/ifc/ifcvf_vdpa.c +0
-0internal->features |= dev_info[IFCVF_NET].features; @@ -1609,6
Quoted 12 lines
+1625,9 @@ struct
rte_vdpa_dev_infodev_info[] = {drivers/vdpa/ifc/ifcvf_vdpa.c +3
-1internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal; --1.8.3.1
PATCH v2 4/8 vdpa/ifc: write queue count to MQ register
In reply to Andy PeiWrite queue count to IFCVF_MQ_OFFSET register
to enable multi queue feature.
--- drivers/vdpa/ifc/base/ifcvf.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +33 -0
@@ -198,6 +198,38 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_multiqueue (struct ifcvf_hw *hw) +{ +u8 *mq_cfg; +int qid; +int nr_queue_pair =0 ; + +if (hw->device_type ==IFCVF_NET ) { +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue_pair++; + } + +if (nr_queue_pair ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + + nr_queue_pair = (nr_queue_pair +1 ) /2 ; + }else if (hw->device_type ==IFCVF_BLK ) { + nr_queue_pair = hw->nr_vring; + } + + mq_cfg = hw->mq_cfg; +if (mq_cfg) { + *(u32 *)mq_cfg = nr_queue_pair; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue_pair); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
@@ -215,6 +247,7 @@return -1 ; } +ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
Signature · 1 line
1.8.3.1
RE: [PATCH v2 4/8] vdpa/ifc: write queue count to MQ register
Quoted 39 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 4/8] vdpa/ifc: write queue count to MQ registerWrite queue count to
IFCVF_MQ_OFFSETregister
to enable multi queue feature.Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+)drivers/vdpa/ifc/base/ifcvf.c +13
-0@@ -198,6 +198,38 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_multiqueue (struct ifcvf_hw *hw) +{ +u8 *mq_cfg; +int qid; +int nr_queue_pair =0 ; + +if (hw->device_type ==IFCVF_NET ) { +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue_pair++; + }
drivers/vdpa/ifc/base/ifcvf.c line 223
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-0+ +if (nr_queue_pair ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + + nr_queue_pair = (nr_queue_pair +1 ) /2 ; + }else if (hw->device_type ==IFCVF_BLK ) { + nr_queue_pair = hw->nr_vring; + }
Why above logic will be different for NET/BLK, could you help me understand it?
Thanks,
Chenbo
Quoted 22 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-1+ + mq_cfg = hw->mq_cfg; +if (mq_cfg) { + *(u32 *)mq_cfg = nr_queue_pair; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue_pair); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {@@ -215,6 +247,7 @@return -1 ; } +ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, --1.8.3.1
RE: [PATCH v2 4/8] vdpa/ifc: write queue count to MQ register
Hi Chenbo,
For BLK devices, ifc driver does not have VHOST_USER_SET_VRING_ENABLE msg. So it does not need to dynamic configuration.
But now I think it is fine to use the same logic.
Quoted 86 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Friday, September 9, 2022 3:35 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: RE: [PATCH v2 4/8] vdpa/ifc: write queue count to MQ registerQuoted 39 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 4/8] vdpa/ifc: write queue count to MQ registerWrite queue count to
IFCVF_MQ_OFFSETregister to enable multi queue feature.Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex 34c8226..0444d74 100644drivers/vdpa/ifc/base/ifcvf.c +12
-0@@ -198,6 +198,38 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_multiqueue (struct ifcvf_hw *hw) { +u8 *mq_cfg; +int qid; +int nr_queue_pair =0 ; + +if (hw->device_type ==IFCVF_NET ) { +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue_pair++; + }drivers/vdpa/ifc/base/ifcvf.c line 222
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-0+ +if (nr_queue_pair ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + + nr_queue_pair = (nr_queue_pair +1 ) /2 ; + }else if (hw->device_type ==IFCVF_BLK ) { + nr_queue_pair = hw->nr_vring; + }Why above logic will be different for NET/BLK, could you help me understand it?
Thanks,
ChenboQuoted 22 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-1+ + mq_cfg = hw->mq_cfg; +if (mq_cfg) { + *(u32 *)mq_cfg = nr_queue_pair; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue_pair); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {@@ -215,6 +247,7 @@return -1 ; } +ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, --1.8.3.1
PATCH v2 5/8 vdpa/ifc: only configure enabled queue
In reply to Andy Peiwhen configure the hardware queue, we only configure queues which have been enabled by vhost.
--- drivers/vdpa/ifc/base/ifcvf.c | 6 +++++- drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +5 -1
@@ -249,6 +249,9 @@ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);
@@ -283,7 +286,8 @@ notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + notify_off * hw->notify_off_multiplier); -IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); +if (hw->vring[i].enable) +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); }return 0 ;
drivers/vdpa/ifc/ifcvf_vdpa.c +14 -2
@@ -290,6 +290,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {
@@ -505,6 +507,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;
@@ -1064,6 +1068,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);
@@ -1077,11 +1083,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) !=0 ) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; }
Signature · 1 line
1.8.3.1
RE: [PATCH v2 5/8] vdpa/ifc: only configure enabled queue
Quoted 34 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 5/8] vdpa/ifc: only configure enabled queuewhen configure the hardware queue, we only configure queues which have been enabled by vhost.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 6 +++++- drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-)drivers/vdpa/ifc/base/ifcvf.c +3
-0@@ -249,6 +249,9 @@ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);
drivers/vdpa/ifc/base/ifcvf.c line 289
Quoted 5 lines
drivers/vdpa/ifc/base/ifcvf.c +1
-1@@ -283,7 +286,8 @@ notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + notify_off * hw->notify_off_multiplier); -IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); +if (hw->vring[i].enable)
Seems useless check as it already checked when the for loop starts
Thanks,
Chenbo
Quoted 58 lines
drivers/vdpa/ifc/base/ifcvf.c +1
-0+IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); }return 0 ;drivers/vdpa/ifc/ifcvf_vdpa.c +14
-3@@ -290,6 +290,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {@@ -505,6 +507,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;@@ -1064,6 +1068,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);@@ -1077,11 +1083,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) != 0) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; } --1.8.3.1
RE: [PATCH v2 5/8] vdpa/ifc: only configure enabled queue
Quoted 37 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, September 14, 2022 09:59
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: RE: [PATCH v2 5/8] vdpa/ifc: only configure enabled queueQuoted 26 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 5/8] vdpa/ifc: only configure enabled queuewhen configure the hardware queue, we only configure queues which have been enabled by vhost.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 6 +++++- drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex 0444d74..4875ea1 100644drivers/vdpa/ifc/base/ifcvf.c +0
-0@@ -249,6 +249,9 @@
Quoted 16 lines
Quoted 9 lines
drivers/vdpa/ifc/base/ifcvf.c +3
-0ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);drivers/vdpa/ifc/base/ifcvf.c line 289
Quoted 5 lines
drivers/vdpa/ifc/base/ifcvf.c +1
-1@@ -283,7 +286,8 @@ notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + notify_off * hw->notify_off_multiplier); -IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); +if (hw->vring[i].enable)Seems useless check as it already checked when the for loop starts
Yes, it should be removed.
Quoted 62 lines
Thanks,
ChenboQuoted 58 lines
drivers/vdpa/ifc/base/ifcvf.c +1
-0+IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); }return 0 ;(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 2b42850..48f1a89 100644drivers/vdpa/ifc/ifcvf_vdpa.c +15
-3@@ -290,6 +290,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {@@ -505,6 +507,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t)vring.kickfd << 32; @@ -1064,6 +1068,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev = rte_vhost_get_vdpa_device(vid); list = find_internal_resource_by_vdev(vdev);@@ -1077,11 +1083,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) != 0) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; } --1.8.3.1
PATCH v2 6/8 vdpa/ifc: support dynamic enable/disable queue
In reply to Andy PeiFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configurate queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
--- drivers/vdpa/ifc/base/ifcvf.c | 101 ++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/base/ifcvf_osdep.h | 1 + drivers/vdpa/ifc/ifcvf_vdpa.c | 93 +++++++++++++++++++++++++++------ 4 files changed, 186 insertions(+), 15 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +101 -0
@@ -230,6 +230,107 @@ } } +int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (!hw || (i >= (int )hw->nr_vring)) +return -1 ; + + cfg = hw->common_cfg; +if (!cfg) { +ERROUT ("common_cfg in HW is NULL.\n" ); +return -1 ; + } + +ifcvf_enable_multiqueue (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +ERROUT ("queue %u, msix vec alloc failed\n" , i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + } + + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (!hw || (i >= (int )hw->nr_vring)) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +ERROUT ("common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); +else + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ); + +if (hw->device_type ==IFCVF_BLK ) + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); +else + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 ); + hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
drivers/vdpa/ifc/base/ifcvf.h +6 -0
@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void
drivers/vdpa/ifc/base/ifcvf_osdep.h +1 -0
@@ -16,6 +16,7 @@ #define WARNINGOUT(S, args...) RTE_LOG(WARNING, PMD, S, ##args) #define DEBUGOUT(S, args...) RTE_LOG(DEBUG, PMD, S, ##args) +#define ERROUT(S, args...) RTE_LOG(ERR, PMD, S, ##args)#define STATIC static#define msec_delay (x) rte_delay_us_sleep(1000 * (x))
drivers/vdpa/ifc/ifcvf_vdpa.c +78 -15
124 lines not on this page · raw message
Signature · 1 line
1.8.3.1
RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue
Quoted 34 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queueFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configurate queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 101 ++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/base/ifcvf_osdep.h | 1 + drivers/vdpa/ifc/ifcvf_vdpa.c | 93 +++++++++++++++++++++++++++---- -- 4 files changed, 186 insertions(+), 15 deletions(-)drivers/vdpa/ifc/base/ifcvf.c +0
-0@@ -230,6 +230,107 @@ } }
drivers/vdpa/ifc/base/ifcvf.c line 242
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-0+int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (!hw || (i >= (int )hw->nr_vring)) +return -1 ;
Seems HW will always be not NULL
drivers/vdpa/ifc/base/ifcvf.c line 246
Quoted 4 lines
drivers/vdpa/ifc/base/ifcvf.c +4
-0+ + cfg = hw->common_cfg; +if (!cfg) { +ERROUT ("common_cfg in HW is NULL.\n" );
I am thinking why you introduce this new log? Why not just use DRV_LOG that is already defined?
Quoted 28 lines
drivers/vdpa/ifc/base/ifcvf.c +28
-0+return -1 ; + } + +ifcvf_enable_multiqueue (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +ERROUT ("queue %u, msix vec alloc failed\n" , i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
drivers/vdpa/ifc/base/ifcvf.c line 284
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-0+ i * IFCVF_LM_CFG_SIZE) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + else + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + }
So the register layout is different for blk and net?
Quoted 35 lines
drivers/vdpa/ifc/base/ifcvf.c +35
-0+ + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (!hw || (i >= (int )hw->nr_vring)) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +ERROUT ("common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); +else
drivers/vdpa/ifc/base/ifcvf.c line 329
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-0+ ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ); + +if (hw->device_type ==IFCVF_BLK ) + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); +else + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 );
Above two if-else should be combined.
Thanks,
Chenbo
Quoted 170 lines
drivers/vdpa/ifc/base/ifcvf.c +4
-0+ hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {drivers/vdpa/ifc/base/ifcvf.h +6
-0@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void drivers/vdpa/ifc/base/ifcvf_osdep.h +1
-0@@ -16,6 +16,7 @@ #define WARNINGOUT(S, args...) RTE_LOG(WARNING, PMD, S, ##args) #define DEBUGOUT(S, args...) RTE_LOG(DEBUG, PMD, S, ##args) +#define ERROUT(S, args...) RTE_LOG(ERR, PMD, S, ##args)#define STATIC static#define msec_delay (x) rte_delay_us_sleep(1000 * (x))drivers/vdpa/ifc/ifcvf_vdpa.c +78
-16@@ -1288,13 +1288,59 @@ struct rte_vdpa_dev_info { }static int +ifcvf_config_vring (struct ifcvf_internal *internal,int vring) +{ +struct ifcvf_hw *hw = &internal->hw; +int vid = internal->vid; +struct rte_vhost_vring vq; +uint64_t gpa; + +if (hw->vring[vring].enable) { +rte_vhost_get_vhost_vring (vid, vring, &vq); + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for descriptor ring." ); +return -1 ; + } + hw->vring[vring].desc = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.avail); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for available ring." ); +return -1 ; + } + hw->vring[vring].avail = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.used); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for used ring." ); +return -1 ; + } + hw->vring[vring].used = gpa; + + hw->vring[vring].size = vq.size; +rte_vhost_get_vring_base (vid, vring, + &hw->vring[vring].last_avail_idx, + &hw->vring[vring].last_used_idx); +ifcvf_enable_vring_hw (&internal->hw, vring); + }else { +ifcvf_disable_vring_hw (&internal->hw, vring); +rte_vhost_set_vring_base (vid, vring, + hw->vring[vring].last_avail_idx, + hw->vring[vring].last_used_idx); + } + +return 0 ; +} + +static int ifcvf_set_vring_state (int vid,int vring,int state) {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal;struct ifcvf_hw *hw; -struct ifcvf_pci_common_cfg *cfg; +bool enable = !!state;int ret =0 ; vdev =rte_vhost_get_vdpa_device (vid);@@ -1304,6 +1350,9 @@ struct rte_vdpa_dev_info {return -1 ; } +DRV_LOG (INFO ,"%s queue %d of vDPA device %s" , + enable ?"enable" :"disable" , vring, vdev->device->name); + internal = list->internal;if (vring <0 || vring >= internal->max_queues *2 ) {DRV_LOG (ERR ,"Vring index %d not correct" , vring);@@ -1311,27 +1360,41 @@ struct rte_vdpa_dev_info { } hw = &internal->hw; + hw->vring[vring].enable = enable; +if (!internal->configured) - goto exit; +return 0 ; - cfg = hw->common_cfg; -IFCVF_WRITE_REG16 (vring, &cfg->queue_select); -IFCVF_WRITE_REG16 (!!state, &cfg->queue_enable); +unset_notify_relay (internal); -if (!state && hw->vring[vring].enable) { - ret =vdpa_disable_vfio_intr (internal); -if (ret) -return ret; + ret =vdpa_enable_vfio_intr (internal, false); +if (ret) { +DRV_LOG (ERR ,"failed to set vfio interrupt of vDPA device %s" , + vdev->device->name); +return ret; } -if (state && !hw->vring[vring].enable) { - ret =vdpa_enable_vfio_intr (internal, false); -if (ret) -return ret; + ret =ifcvf_config_vring (internal, vring); +if (ret) { +DRV_LOG (ERR ,"failed to configure queue %d of vDPA device %s" , + vring, vdev->device->name); +return ret; + } + + ret =setup_notify_relay (internal); +if (ret) { +DRV_LOG (ERR ,"failed to setup notify relay of vDPA device %s" , + vdev->device->name); +return ret; + } + + ret =rte_vhost_host_notifier_ctrl (vid, vring, enable); +if (ret) { +DRV_LOG (ERR ,"vDPA device %s queue %d host notifier ctrl fail" , + vdev->device->name, vring); +return ret; } -exit: - hw->vring[vring].enable = !!state;return 0 ; } --1.8.3.1
RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue
Quoted 46 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, September 14, 2022 10:23
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queueQuoted 35 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queueFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configurate queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 101 ++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/base/ifcvf_osdep.h | 1 + drivers/vdpa/ifc/ifcvf_vdpa.c | 93 +++++++++++++++++++++++++++---- -- 4 files changed, 186 insertions(+), 15 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex 4875ea1..34f32f8 100644drivers/vdpa/ifc/base/ifcvf.c +0
-0@@ -230,6 +230,107 @@ } }
Quoted 11 lines
drivers/vdpa/ifc/base/ifcvf.c line 242
Quoted 9 lines
drivers/vdpa/ifc/base/ifcvf.c +9
-0+int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) { +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (!hw || (i >= (int )hw->nr_vring)) +return -1 ;Seems HW will always be not NULL
As a external function, we should not assume the input argument is always valid.
Quoted 8 lines
drivers/vdpa/ifc/base/ifcvf.c line 246
Quoted 5 lines
drivers/vdpa/ifc/base/ifcvf.c +4
-0+ + cfg = hw->common_cfg; +if (!cfg) { +ERROUT ("common_cfg in HW is NULL.\n" );I am thinking why you introduce this new log? Why not just use
DRV_LOGthat is already defined?
Because below type of log macros are already defined and used in original code, I just follow its tradition.
#define WARNINGOUT(S, args...) RTE_LOG(WARNING, PMD, S, ##args) #define DEBUGOUT(S, args...) RTE_LOG(DEBUG, PMD, S, ##args) #define ERROUT(S, args...) RTE_LOG(ERR, PMD, S, ##args)
Quoted 33 lines
Quoted 33 lines
drivers/vdpa/ifc/base/ifcvf.c +32
-0+return -1 ; + } + +ifcvf_enable_multiqueue (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +ERROUT ("queue %u, msix vec alloc failed\n" , i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else
Quoted 8 lines
drivers/vdpa/ifc/base/ifcvf.c line 284
Quoted 6 lines
drivers/vdpa/ifc/base/ifcvf.c +6
-0+ *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + }So the register layout is different for blk and net?
That's sure.
Quoted 39 lines
Quoted 39 lines
drivers/vdpa/ifc/base/ifcvf.c +38
-0+ + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) { +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (!hw || (i >= (int )hw->nr_vring)) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +ERROUT ("common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); +else + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 );
Quoted 8 lines
drivers/vdpa/ifc/base/ifcvf.c line 329
Quoted 6 lines
drivers/vdpa/ifc/base/ifcvf.c +6
-0+ +if (hw->device_type ==IFCVF_BLK ) + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); +else + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 );Above two if-else should be combined.
It's a good suggestion.
Quoted 174 lines
Thanks,
ChenboQuoted 89 lines
drivers/vdpa/ifc/base/ifcvf.c +4
-0+ hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.hindex c17bf2a..e67d4e8 100644drivers/vdpa/ifc/base/ifcvf.h +6
-0@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void drivers/vdpa/ifc/base/ifcvf_osdep.h +1
-0@@ -16,6 +16,7 @@ #define WARNINGOUT(S, args...) RTE_LOG(WARNING, PMD, S, ##args) #define DEBUGOUT(S, args...) RTE_LOG(DEBUG, PMD, S, ##args) +#define ERROUT(S, args...) RTE_LOG(ERR, PMD, S, ##args)#define STATIC static#define msec_delay (x) rte_delay_us_sleep(1000 * (x))(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 48f1a89..16fd0fd 100644drivers/vdpa/ifc/ifcvf_vdpa.c +45
-0@@ -1288,13 +1288,59 @@ struct rte_vdpa_dev_info { }static int +ifcvf_config_vring (struct ifcvf_internal *internal,int vring) { +struct ifcvf_hw *hw = &internal->hw; +int vid = internal->vid; +struct rte_vhost_vring vq; +uint64_t gpa; + +if (hw->vring[vring].enable) { +rte_vhost_get_vhost_vring (vid, vring, &vq); + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for descriptor ring." ); +return -1 ; + } + hw->vring[vring].desc = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.avail); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for available ring." ); +return -1 ; + } + hw->vring[vring].avail = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.used); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for used ring." ); +return -1 ; + } + hw->vring[vring].used = gpa; + + hw->vring[vring].size = vq.size; +rte_vhost_get_vring_base (vid, vring, + &hw->vring[vring].last_avail_idx, + &hw->vring[vring].last_used_idx); +ifcvf_enable_vring_hw (&internal->hw, vring); + }else { +ifcvf_disable_vring_hw (&internal->hw, vring); +rte_vhost_set_vring_base (vid, vring, + hw->vring[vring].last_avail_idx, + hw->vring[vring].last_used_idx); + } + +return 0 ; +} + +static intdrivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info { } · line 1343
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1ifcvf_set_vring_state(int vid, int vring, int state) { struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal;struct ifcvf_hw *hw; -struct ifcvf_pci_common_cfg *cfg; +bool enable = !!state;int ret =0 ; vdev = rte_vhost_get_vdpa_device(vid); @@ -1304,6 +1350,9 @@struct
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info {
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +3
-0rte_vdpa_dev_info {return -1 ; } +DRV_LOG (INFO ,"%s queue %d of vDPA device %s" , + enable ?"enable" :"disable" , vring, vdev->device->name); + internal = list->internal;if (vring <0 || vring >= internal->max_queues *2 ) { DRV_LOG(ERR, "Vring index %d not correct", vring); @@ -1311,27
Quoted 23 lines
+1360,41 @@ struct
rte_vdpa_dev_info{drivers/vdpa/ifc/ifcvf_vdpa.c +8
-8} hw = &internal->hw; + hw->vring[vring].enable = enable; +if (!internal->configured) - goto exit; +return 0 ; - cfg = hw->common_cfg; -IFCVF_WRITE_REG16 (vring, &cfg->queue_select); -IFCVF_WRITE_REG16 (!!state, &cfg->queue_enable); +unset_notify_relay (internal); -if (!state && hw->vring[vring].enable) { - ret =vdpa_disable_vfio_intr (internal); -if (ret) -return ret; + ret =vdpa_enable_vfio_intr (internal, false); +if (ret) { +DRV_LOG (ERR ,"failed to set vfio interrupt of vDPA device %s" , + vdev->device->name);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info {
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +4
-4+return ret; } -if (state && !hw->vring[vring].enable) { - ret =vdpa_enable_vfio_intr (internal, false); -if (ret) -return ret; + ret =ifcvf_config_vring (internal, vring); +if (ret) { + DRV_LOG(ERR, "failed to configure queue %d of vDPAdevice %s",
Quoted 25 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +16
-3+ vring, vdev->device->name); +return ret; + } + + ret =setup_notify_relay (internal); +if (ret) { +DRV_LOG (ERR ,"failed to setup notify relay of vDPA device %s" , + vdev->device->name); +return ret; + } + + ret =rte_vhost_host_notifier_ctrl (vid, vring, enable); +if (ret) { +DRV_LOG (ERR ,"vDPA device %s queue %d host notifier ctrl fail" , + vdev->device->name, vring); +return ret; } -exit: - hw->vring[vring].enable = !!state;return 0 ; } --1.8.3.1
RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue
Quoted 69 lines
-----Original Message-----
From: Huang, Wei <[email protected]>
Sent: Wednesday, September 14, 2022 11:04 AM
To: Xia, Chenbo <[email protected]>; Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queueQuoted 5 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, September 14, 2022 10:23
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>;Cao,
Quoted 3 lines
Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disablequeue
Quoted 17 lines
Quoted 17 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queueFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configurate queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.Quoted 8 lines
Quoted 8 lines
Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 101 ++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/base/ifcvf_osdep.h | 1 + drivers/vdpa/ifc/ifcvf_vdpa.c | 93+++++++++++++++++++++++++++----
Quoted 11 lines
Quoted 11 lines
--
4 files changed, 186 insertions(+), 15 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex 4875ea1..34f32f8 100644drivers/vdpa/ifc/base/ifcvf.c +0
-0@@ -230,6 +230,107 @@ } }Quoted 11 lines
drivers/vdpa/ifc/base/ifcvf.c line 242
Quoted 9 lines
drivers/vdpa/ifc/base/ifcvf.c +9
-0+int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) { +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (!hw || (i >= (int )hw->nr_vring)) +return -1 ;Seems HW will always be not NULL
Quoted 2 lines
As a external function, we should not assume the input argument is always valid.
It's actually internal: all logic is inside a your driver. It's not like a library API.
Quoted 7 lines
Quoted 7 lines
drivers/vdpa/ifc/base/ifcvf.c line 246
Quoted 5 lines
drivers/vdpa/ifc/base/ifcvf.c +4
-0+ + cfg = hw->common_cfg; +if (!cfg) { +ERROUT ("common_cfg in HW is NULL.\n" );I am thinking why you introduce this new log? Why not just use
DRV_LOG
Quoted 7 lines
that is
Quoted 1 line
already defined?
Because below type of log macros are already defined and used in original code, I just follow its tradition.
#define WARNINGOUT(S, args...) RTE_LOG(WARNING, PMD, S, ##args) #define DEBUGOUT(S, args...) RTE_LOG(DEBUG, PMD, S, ##args) #define ERROUT(S, args...) RTE_LOG(ERR, PMD, S, ##args)
I think all above seems not needed. DRV_LOG could cover all.
Thanks,
Chenbo
Quoted 264 lines
Quoted 33 lines
Quoted 33 lines
drivers/vdpa/ifc/base/ifcvf.c +32
-0+return -1 ; + } + +ifcvf_enable_multiqueue (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +ERROUT ("queue %u, msix vec alloc failed\n" , i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else Quoted 8 lines
drivers/vdpa/ifc/base/ifcvf.c line 284
Quoted 6 lines
drivers/vdpa/ifc/base/ifcvf.c +6
-0+ *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + }So the register layout is different for blk and net?
That's sure.
Quoted 39 lines
Quoted 39 lines
drivers/vdpa/ifc/base/ifcvf.c +38
-0+ + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) { +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (!hw || (i >= (int )hw->nr_vring)) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +ERROUT ("common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); +else + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 );Quoted 8 lines
drivers/vdpa/ifc/base/ifcvf.c line 329
Quoted 6 lines
drivers/vdpa/ifc/base/ifcvf.c +6
-0+ +if (hw->device_type ==IFCVF_BLK ) + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); +else + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 );Above two if-else should be combined.
It's a good suggestion.
Quoted 174 lines
Thanks,
ChenboQuoted 89 lines
drivers/vdpa/ifc/base/ifcvf.c +4
-0+ hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.hindex c17bf2a..e67d4e8 100644drivers/vdpa/ifc/base/ifcvf.h +6
-0@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void drivers/vdpa/ifc/base/ifcvf_osdep.h +1
-0@@ -16,6 +16,7 @@ #define WARNINGOUT(S, args...) RTE_LOG(WARNING, PMD, S, ##args) #define DEBUGOUT(S, args...) RTE_LOG(DEBUG, PMD, S, ##args) +#define ERROUT(S, args...) RTE_LOG(ERR, PMD, S, ##args)#define STATIC static#define msec_delay (x) rte_delay_us_sleep(1000 * (x))(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 48f1a89..16fd0fd 100644drivers/vdpa/ifc/ifcvf_vdpa.c +45
-0@@ -1288,13 +1288,59 @@ struct rte_vdpa_dev_info { }static int +ifcvf_config_vring (struct ifcvf_internal *internal,int vring) { +struct ifcvf_hw *hw = &internal->hw; +int vid = internal->vid; +struct rte_vhost_vring vq; +uint64_t gpa; + +if (hw->vring[vring].enable) { +rte_vhost_get_vhost_vring (vid, vring, &vq); + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for descriptor ring." ); +return -1 ; + } + hw->vring[vring].desc = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.avail); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for available ring." ); +return -1 ; + } + hw->vring[vring].avail = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.used); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for used ring." ); +return -1 ; + } + hw->vring[vring].used = gpa; + + hw->vring[vring].size = vq.size; +rte_vhost_get_vring_base (vid, vring, + &hw->vring[vring].last_avail_idx, + &hw->vring[vring].last_used_idx); +ifcvf_enable_vring_hw (&internal->hw, vring); + }else { +ifcvf_disable_vring_hw (&internal->hw, vring); +rte_vhost_set_vring_base (vid, vring, + hw->vring[vring].last_avail_idx, + hw->vring[vring].last_used_idx); + } + +return 0 ; +} + +static intdrivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info { } · line 1343
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1ifcvf_set_vring_state(int vid, int vring, int state) { struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal;struct ifcvf_hw *hw; -struct ifcvf_pci_common_cfg *cfg; +bool enable = !!state;int ret =0 ; vdev = rte_vhost_get_vdpa_device(vid); @@ -1304,6 +1350,9 @@struct
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info {
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +3
-0rte_vdpa_dev_info {return -1 ; } +DRV_LOG (INFO ,"%s queue %d of vDPA device %s" , + enable ?"enable" :"disable" , vring, vdev->device->name); + internal = list->internal;if (vring <0 || vring >= internal->max_queues *2 ) { DRV_LOG(ERR, "Vring index %d not correct", vring); @@ -1311,27
Quoted 23 lines
+1360,41 @@ struct
rte_vdpa_dev_info{drivers/vdpa/ifc/ifcvf_vdpa.c +8
-8} hw = &internal->hw; + hw->vring[vring].enable = enable; +if (!internal->configured) - goto exit; +return 0 ; - cfg = hw->common_cfg; -IFCVF_WRITE_REG16 (vring, &cfg->queue_select); -IFCVF_WRITE_REG16 (!!state, &cfg->queue_enable); +unset_notify_relay (internal); -if (!state && hw->vring[vring].enable) { - ret =vdpa_disable_vfio_intr (internal); -if (ret) -return ret; + ret =vdpa_enable_vfio_intr (internal, false); +if (ret) { +DRV_LOG (ERR ,"failed to set vfio interrupt of vDPA device %s" , + vdev->device->name);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info {
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +4
-4+return ret; } -if (state && !hw->vring[vring].enable) { - ret =vdpa_enable_vfio_intr (internal, false); -if (ret) -return ret; + ret =ifcvf_config_vring (internal, vring); +if (ret) { + DRV_LOG(ERR, "failed to configure queue %d of vDPAdevice %s",
Quoted 25 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +16
-3+ vring, vdev->device->name); +return ret; + } + + ret =setup_notify_relay (internal); +if (ret) { +DRV_LOG (ERR ,"failed to setup notify relay of vDPA device %s" , + vdev->device->name); +return ret; + } + + ret =rte_vhost_host_notifier_ctrl (vid, vring, enable); +if (ret) { +DRV_LOG (ERR ,"vDPA device %s queue %d host notifier ctrl fail" , + vdev->device->name, vring); +return ret; } -exit: - hw->vring[vring].enable = !!state;return 0 ; } --1.8.3.1
RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue
In reply to Huang, WeiHi Chenbo,
See my reply inline.
Quoted 123 lines
-----Original Message-----
From: Huang, Wei <[email protected]>
Sent: Wednesday, September 14, 2022 11:04 AM
To: Xia, Chenbo <[email protected]>; Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queueQuoted 46 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, September 14, 2022 10:23
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queueQuoted 36 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queueFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configurate queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 101 ++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/base/ifcvf_osdep.h | 1 + drivers/vdpa/ifc/ifcvf_vdpa.c | 93 +++++++++++++++++++++++++++---- -- 4 files changed, 186 insertions(+), 15 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex 4875ea1..34f32f8 100644drivers/vdpa/ifc/base/ifcvf.c +0
-0@@ -230,6 +230,107 @@ } }Quoted 11 lines
drivers/vdpa/ifc/base/ifcvf.c line 242
Quoted 9 lines
drivers/vdpa/ifc/base/ifcvf.c +9
-0+int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) { +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (!hw || (i >= (int )hw->nr_vring)) +return -1 ;Seems HW will always be not NULL
As a external function, we should not assume the input argument is always valid.
Quoted 8 lines
drivers/vdpa/ifc/base/ifcvf.c line 246
Quoted 5 lines
drivers/vdpa/ifc/base/ifcvf.c +4
-0+ + cfg = hw->common_cfg; +if (!cfg) { +ERROUT ("common_cfg in HW is NULL.\n" );I am thinking why you introduce this new log? Why not just use
DRV_LOGthat is already defined?Because below type of log macros are already defined and used in original code, I just follow its tradition.
#define WARNINGOUT(S, args...) RTE_LOG(WARNING, PMD, S, ##args) #define DEBUGOUT(S, args...) RTE_LOG(DEBUG, PMD, S, ##args) #define ERROUT(S, args...) RTE_LOG(ERR, PMD, S, ##args)Quoted 33 lines
Quoted 33 lines
drivers/vdpa/ifc/base/ifcvf.c +32
-0+return -1 ; + } + +ifcvf_enable_multiqueue (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +ERROUT ("queue %u, msix vec alloc failed\n" , i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else Quoted 8 lines
drivers/vdpa/ifc/base/ifcvf.c line 284
Quoted 6 lines
drivers/vdpa/ifc/base/ifcvf.c +6
-0+ *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + }So the register layout is different for blk and net?
Quoted 1 line
That's sure.
For the register layout differences, the story is as follow:
When I add support for blk device, I tried to re-use the existing code. However, the BAR4 layout of blk HW is different.
I keep code for net device unchanged, and add code for blk device.
Quoted 221 lines
Quoted 39 lines
Quoted 39 lines
drivers/vdpa/ifc/base/ifcvf.c +38
-0+ + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) { +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (!hw || (i >= (int )hw->nr_vring)) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +ERROUT ("common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); +else + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 );Quoted 8 lines
drivers/vdpa/ifc/base/ifcvf.c line 329
Quoted 6 lines
drivers/vdpa/ifc/base/ifcvf.c +6
-0+ +if (hw->device_type ==IFCVF_BLK ) + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); +else + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 );Above two if-else should be combined.
It's a good suggestion.
Quoted 173 lines
Thanks,
ChenboQuoted 88 lines
drivers/vdpa/ifc/base/ifcvf.c +4
-0+ hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC int ifcvf_hw_enable(struct ifcvf_hw *hw) { diff --git
a/drivers/vdpa/ifc/base/ifcvf.hb/drivers/vdpa/ifc/base/ifcvf.hindex c17bf2a..e67d4e8 100644drivers/vdpa/ifc/base/ifcvf.h +6
-0@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void drivers/vdpa/ifc/base/ifcvf_osdep.h +1
-0@@ -16,6 +16,7 @@ #define WARNINGOUT(S, args...) RTE_LOG(WARNING, PMD, S, ##args) #define DEBUGOUT(S, args...) RTE_LOG(DEBUG, PMD, S, ##args) +#define ERROUT(S, args...) RTE_LOG(ERR, PMD, S, ##args)#define STATIC static#define msec_delay (x) rte_delay_us_sleep(1000 * (x))(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 48f1a89..16fd0fd 100644drivers/vdpa/ifc/ifcvf_vdpa.c +45
-0@@ -1288,13 +1288,59 @@ struct rte_vdpa_dev_info { }static int +ifcvf_config_vring (struct ifcvf_internal *internal,int vring) { +struct ifcvf_hw *hw = &internal->hw; +int vid = internal->vid; +struct rte_vhost_vring vq; +uint64_t gpa; + +if (hw->vring[vring].enable) { +rte_vhost_get_vhost_vring (vid, vring, &vq); + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for descriptor ring." ); +return -1 ; + } + hw->vring[vring].desc = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.avail); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for available ring." ); +return -1 ; + } + hw->vring[vring].avail = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.used); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for used ring." ); +return -1 ; + } + hw->vring[vring].used = gpa; + + hw->vring[vring].size = vq.size; +rte_vhost_get_vring_base (vid, vring, + &hw->vring[vring].last_avail_idx, + &hw->vring[vring].last_used_idx); +ifcvf_enable_vring_hw (&internal->hw, vring); + }else { +ifcvf_disable_vring_hw (&internal->hw, vring); +rte_vhost_set_vring_base (vid, vring, + hw->vring[vring].last_avail_idx, + hw->vring[vring].last_used_idx); + } + +return 0 ; +} + +static intdrivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info { } · line 1343
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1ifcvf_set_vring_state(int vid, int vring, int state) { struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal;struct ifcvf_hw *hw; -struct ifcvf_pci_common_cfg *cfg; +bool enable = !!state;int ret =0 ; vdev = rte_vhost_get_vdpa_device(vid); @@ -1304,6 +1350,9 @@struct
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info {
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +3
-0rte_vdpa_dev_info {return -1 ; } +DRV_LOG (INFO ,"%s queue %d of vDPA device %s" , + enable ?"enable" :"disable" , vring, vdev->device->name); + internal = list->internal;if (vring <0 || vring >= internal->max_queues *2 ) { DRV_LOG(ERR, "Vring index %d not correct", vring); @@ -1311,27
Quoted 23 lines
+1360,41 @@ struct
rte_vdpa_dev_info{drivers/vdpa/ifc/ifcvf_vdpa.c +8
-8} hw = &internal->hw; + hw->vring[vring].enable = enable; +if (!internal->configured) - goto exit; +return 0 ; - cfg = hw->common_cfg; -IFCVF_WRITE_REG16 (vring, &cfg->queue_select); -IFCVF_WRITE_REG16 (!!state, &cfg->queue_enable); +unset_notify_relay (internal); -if (!state && hw->vring[vring].enable) { - ret =vdpa_disable_vfio_intr (internal); -if (ret) -return ret; + ret =vdpa_enable_vfio_intr (internal, false); +if (ret) { +DRV_LOG (ERR ,"failed to set vfio interrupt of vDPA device %s" , + vdev->device->name);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info {
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +4
-4+return ret; } -if (state && !hw->vring[vring].enable) { - ret =vdpa_enable_vfio_intr (internal, false); -if (ret) -return ret; + ret =ifcvf_config_vring (internal, vring); +if (ret) { + DRV_LOG(ERR, "failed to configure queue %d of vDPAdevice %s",
Quoted 25 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +16
-3+ vring, vdev->device->name); +return ret; + } + + ret =setup_notify_relay (internal); +if (ret) { +DRV_LOG (ERR ,"failed to setup notify relay of vDPA device %s" , + vdev->device->name); +return ret; + } + + ret =rte_vhost_host_notifier_ctrl (vid, vring, enable); +if (ret) { +DRV_LOG (ERR ,"vDPA device %s queue %d host notifier ctrl fail" , + vdev->device->name, vring); +return ret; } -exit: - hw->vring[vring].enable = !!state;return 0 ; } --1.8.3.1
PATCH v2 7/8 vhost: configure device when any queue is ready for BLK device
In reply to Andy PeiWhen boot from virtio blk device, seabois in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end conf_dev when any queue is ready.
--- lib/vhost/vhost_user.c | 56 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 12 deletions(-)
lib/vhost/vhost_user.c +44 -12
@@ -1451,6 +1451,25 @@#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2ustatic int +virtio_has_queue_ready (struct virtio_net *dev) +{ +struct vhost_virtqueue *vq; +uint32_t i, nr_vring = dev->nr_vring; + +if (!dev->nr_vring) +return 0 ; + +for (i =0 ; i < nr_vring; i++) { + vq = dev->virtqueue[i]; + +if (vq_is_ready (dev, vq)) +return 1 ; + } + +return 0 ; +} + +static int virtio_is_ready (struct virtio_net *dev) {struct vhost_virtqueue *vq;
@@ -3167,9 +3186,33 @@ static int is_vring_iotlb(struct virtio_net *dev,if (unlock_required)vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) goto out; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev)) { +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (!virtio_has_queue_ready (dev)) + goto out; + }else { + goto out; + } + } +/* * Virtio is now ready. If not done already, it is time * to notify the application it can process the rings and
@@ -3181,20 +3224,9 @@ static int is_vring_iotlb(struct virtio_net *dev, dev->flags |=VIRTIO_DEV_RUNNING ; } - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (dev->ifname,ERR ,"failed to get vdpa dev type.\n" ); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL ) goto out;
Signature · 1 line
1.8.3.1
RE: [PATCH v2 7/8] vhost: configure device when any queue is ready for BLK device
Quoted 10 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 7/8] vhost: configure device when any queue is ready for BLK device
Quoted 3 lines
When boot from virtio blk device, seabois in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end
conf_devwhen any queue is ready.
So this is the case only for vdpa blk, for SW vhost-blk, all queues need to be ready?
Quoted 60 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 56 +++++++++++++++++++++++++++++++++++++++------ ----- 1 file changed, 44 insertions(+), 12 deletions(-)lib/vhost/vhost_user.c +34
-1@@ -1451,6 +1451,25 @@#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2ustatic int +virtio_has_queue_ready (struct virtio_net *dev) +{ +struct vhost_virtqueue *vq; +uint32_t i, nr_vring = dev->nr_vring; + +if (!dev->nr_vring) +return 0 ; + +for (i =0 ; i < nr_vring; i++) { + vq = dev->virtqueue[i]; + +if (vq_is_ready (dev, vq)) +return 1 ; + } + +return 0 ; +} + +static int virtio_is_ready (struct virtio_net *dev) {struct vhost_virtqueue *vq;@@ -3167,9 +3186,33 @@ static int is_vring_iotlb(struct virtio_net *dev,if (unlock_required)vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) goto out; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + }
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev, · line 3215
Quoted 10 lines
lib/vhost/vhost_user.c +10
-0+ +if (!virtio_is_ready (dev)) { +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (!virtio_has_queue_ready (dev)) + goto out; + }else { + goto out; + } + } +
I feel like if possible, above logic should all be in virtio_is_ready.
Thanks,
Chenbo
Quoted 27 lines
lib/vhost/vhost_user.c +0
-11/* * Virtio is now ready. If not done already, it is time * to notify the application it can process the rings and @@ -3181,20 +3224,9 @@ static int is_vring_iotlb(struct virtio_net *dev, dev->flags |=VIRTIO_DEV_RUNNING ; } - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n"); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL )goto out;
--
1.8.3.1
RE: [PATCH v2 7/8] vhost: configure device when any queue is ready for BLK device
HI Chenbo,
Thanks for your reply, my reply is inline.
Quoted 20 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, September 14, 2022 10:51 AM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v2 7/8] vhost: configure device when any queue is ready for BLK deviceQuoted 11 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, September 8, 2022 1:54 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]; Huang Wei <[email protected]>
Subject: [PATCH v2 7/8] vhost: configure device when any queue is ready for BLK device
Quoted 7 lines
Quoted 3 lines
When boot from virtio blk device, seabois in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end
conf_devwhen any queue is ready.So this is the case only for vdpa blk, for SW vhost-blk, all queues need to be ready?
The case I mention is that the OS image is in the vdpa device.
In this case, QEMU bios (seabios) will take charge. There is a virtio driver in seabios. This driver only use one queue.
So in this case vdpa driver need to conf_dev when one queue is ready.
for SW vhost-blk, I am not sure you mean OS image is in your back-end vhost-blk. I have not test that case.
Quoted 64 lines
Quoted 39 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 56 +++++++++++++++++++++++++++++++++++++++------ ----- 1 file changed, 44 insertions(+), 12 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-04ad28ba..b65fba3 100644
lib/vhost/vhost_user.c +18
-0@@ -1451,6 +1451,25 @@#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2ustatic int +virtio_has_queue_ready (struct virtio_net *dev) { +struct vhost_virtqueue *vq; +uint32_t i, nr_vring = dev->nr_vring; + +if (!dev->nr_vring) +return 0 ; + +for (i =0 ; i < nr_vring; i++) { + vq = dev->virtqueue[i]; + +if (vq_is_ready (dev, vq)) +return 1 ; + } + +return 0 ; +} + +static int virtio_is_ready (struct virtio_net *dev) {struct vhost_virtqueue *vq;@@ -3167,9 +3186,33 @@ static int is_vring_iotlb(struct virtio_net *dev,if (unlock_required)lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev, · line 3195
Quoted 10 lines
lib/vhost/vhost_user.c +5
-1vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) goto out; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
&vdpa_type);Quoted 14 lines
lib/vhost/vhost_user.c +14
-0+if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev)) { +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (!virtio_has_queue_ready (dev))
Quoted 9 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev, · line 3215
Quoted 6 lines
lib/vhost/vhost_user.c +6
-0+ goto out; + }else { + goto out; + } + } +I feel like if possible, above logic should all be in
virtio_is_ready.
Maybe I can introduce something like VIRTIO_DEV_BUILTIN_VIRTIO_NET, What do you think?
Quoted 31 lines
Thanks,
ChenboQuoted 5 lines
lib/vhost/vhost_user.c +0
-1/* * Virtio is now ready. If not done already, it is time * to notify the application it can process the rings and @@ -3181,20 +3224,9 @@ static int is_vring_iotlb(struct virtio_net *dev, dev->flags |= VIRTIO_DEV_RUNNING; lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev,
Quoted 10 lines
lib/vhost/vhost_user.c +0
-5} - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to getvdpa
Quoted 12 lines
dev type.\n");
lib/vhost/vhost_user.c +0
-7- ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL ) goto out; --1.8.3.1
PATCH v2 8/8 vhost: vDPA BLK devices configure device when all queue callfds are set
In reply to Andy PeiIn the virtio blk vDPA live migration use case, before the live migration process, QEMU will set call fd to vDPA back-end. QEMU and vDPA back-end stand by until live migration starts. During live migration process, QEMU sets kick fd and a new call fd. However, after the kick fd is set to the vDPA back-end, the vDPA back-end configures device and data path starts. The new call fd will cause some kind of "re-configuration", this kind of "re-configuration" cause IO drop.
After this patch, vDPA back-end configures device after kick fd and call fd are well set and make sure no IO drops. This patch only impact virtio blk vDPA device and does not impact net device.
--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
lib/vhost/vhost_user.c +10 -3
@@ -2994,6 +2994,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type =0 ;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )
@@ -3227,9 +3228,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (vdpa_dev->ops->dev_conf (dev->vid))
Signature · 1 line
1.8.3.1
v3 cover vdpa/ifc: add multi queue support
In reply to Andy Peiv3:
rename device ID macro name.
fix some patch title and commit message.
delete some used marco.
rework some code logic.
v2:
fix some coding style issue.
support dynamic enable/disable queue at run time.
Andy Pei (6):
vdpa/ifc: add multi-queue support
vdpa/ifc: set max queues based on virtio spec vdpa/ifc: write queue count to MQ register vdpa/ifc: only configure enabled queue
vhost: vDPA blk device gets ready when any queue is ready
vhost: improve vDPA blk device readiness condition
Huang Wei (2):
vdpa/ifc: add new device ID for legacy network device vdpa/ifc: support dynamic enable/disable queue
drivers/vdpa/ifc/base/ifcvf.c | 137 ++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 16 ++++- drivers/vdpa/ifc/ifcvf_vdpa.c | 142 +++++++++++++++++++++++++++++++++++------- lib/vhost/vhost_user.c | 58 +++++++++++------ 4 files changed, 311 insertions(+), 42 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v3 1/8 vdpa/ifc: add new device ID for legacy network device
From: Huang Wei <[email protected]>
Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000). Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".
--- drivers/vdpa/ifc/base/ifcvf.h | 6 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +4 -2
@@ -12,12 +12,14 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_BLK_DEVICE_ID 0x0002 +#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001 +#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1
drivers/vdpa/ifc/ifcvf_vdpa.c +10 -3
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_SUBSYS_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .vendor_id =0 ,/* sentinel */
Signature · 1 line
1.8.3.1
Re: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network device
Quoted 61 lines · Andy Pei
From: Huang Wei <[email protected]>
Add new device id to support
IFCVF_NET_TRANSITIONAL_DEVICE_ID(0x1000). Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.h | 6 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-)drivers/vdpa/ifc/base/ifcvf.h +4
-2@@ -12,12 +12,14 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_BLK_DEVICE_ID 0x0002 +#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001 +#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1drivers/vdpa/ifc/ifcvf_vdpa.c +9
-2@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_SUBSYS_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , },
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1713
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1{ .class_id = RTE_CLASS_ANY_ID, .vendor_id = IFCVF_VENDOR_ID, .device_id = IFCVF_BLK_MODERN_DEVICE_ID, .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID, - .subsystem_device_id = IFCVF_BLK_DEVICE_ID, + .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID, }, { .vendor_id = 0, /* sentinel */
Thanks,
Maxime
RE: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network device
Hi Maxime,
Thanks for your effort.
Quoted 90 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Wednesday, October 12, 2022 12:49 AM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>
Subject: Re: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network deviceQuoted 3 lines · Andy Pei
From: Huang Wei <[email protected]>
Add new device id to support
IFCVF_NET_TRANSITIONAL_DEVICE_ID(
0x1000).Quoted 1 line
Rename macro from "
IFCVF_BLK_DEVICE_ID" to"
IFCVF_SUBSYS_BLK_DEVICE_ID".Quoted 57 lines
Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.h | 6 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.hindex 9d95aac..ef7697a 100644drivers/vdpa/ifc/base/ifcvf.h +4
-2@@ -12,12 +12,14 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_BLK_DEVICE_ID 0x0002 +#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001 +#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex ac42de9..07cc63d 100644drivers/vdpa/ifc/ifcvf_vdpa.c +9
-2@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_SUBSYS_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , },drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1713
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1{ .class_id = RTE_CLASS_ANY_ID, .vendor_id = IFCVF_VENDOR_ID, .device_id = IFCVF_BLK_MODERN_DEVICE_ID, .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID, - .subsystem_device_id = IFCVF_BLK_DEVICE_ID, + .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID, }, { .vendor_id = 0, /* sentinel */Reviewed-by Maxime Coquelin <[email protected]>Thanks,
Maxime
RE: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network device
In reply to Andy PeiQuoted 82 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network deviceFrom: Huang Wei <[email protected]>
Add new device id to support
IFCVF_NET_TRANSITIONAL_DEVICE_ID(0x1000). Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.h | 6 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-)drivers/vdpa/ifc/base/ifcvf.h +4
-2@@ -12,12 +12,14 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_BLK_DEVICE_ID 0x0002 +#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001 +#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1drivers/vdpa/ifc/ifcvf_vdpa.c +10
-4@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_SUBSYS_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .vendor_id =0 ,/* sentinel */ --
Quoted 1 line
1.8.3.1
RE: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network device
Hi Chenbo,
Thanks for your effort.
Quoted 96 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 11:00 AM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network deviceQuoted 10 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network deviceQuoted 3 lines
From: Huang Wei <[email protected]>
Add new device id to support
IFCVF_NET_TRANSITIONAL_DEVICE_ID(
0x1000).Quoted 1 line
Rename macro from "
IFCVF_BLK_DEVICE_ID" to"
IFCVF_SUBSYS_BLK_DEVICE_ID".Quoted 68 lines
Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.h | 6 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.hindex 9d95aac..ef7697a 100644drivers/vdpa/ifc/base/ifcvf.h +4
-2@@ -12,12 +12,14 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_BLK_DEVICE_ID 0x0002 +#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001 +#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex ac42de9..07cc63d 100644drivers/vdpa/ifc/ifcvf_vdpa.c +10
-4@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_SUBSYS_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .vendor_id =0 ,/* sentinel */ --Quoted 1 line
1.8.3.1
Reviewed-by Chenbo Xia <[email protected]>
PATCH v3 2/8 vdpa/ifc: add multi-queue support
In reply to Andy PeiEnable VHOST_USER_PROTOCOL_F_MQ feature. Expose IFCVF_MQ_OFFSET register to enable multi queue.
--- drivers/vdpa/ifc/base/ifcvf.c | 5 +++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 8 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +5 -0
@@ -90,6 +90,11 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ; +if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );
drivers/vdpa/ifc/base/ifcvf.h +2 -0
@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0
@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;
drivers/vdpa/ifc/ifcvf_vdpa.c +1 -0
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \
Signature · 1 line
1.8.3.1
RE: [PATCH v3 2/8] vdpa/ifc: add multi-queue support
Quoted 11 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 2/8] vdpa/ifc: add multi-queue supportEnable
VHOST_USER_PROTOCOL_F_MQfeature. ExposeIFCVF_MQ_OFFSETregister to enable multi queue.
Multi-queue
Quoted 14 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 5 +++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 8 insertions(+)drivers/vdpa/ifc/base/ifcvf.c +0
-0
drivers/vdpa/ifc/base/ifcvf.c line 97
Quoted 8 lines
drivers/vdpa/ifc/base/ifcvf.c +5
-0@@ -90,6 +90,11 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ; +
It will be better to add comments about the explanation that you sent before.
Thanks,
Chenbo
Quoted 37 lines
drivers/vdpa/ifc/base/ifcvf.c +0
-0if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );drivers/vdpa/ifc/base/ifcvf.h +2
-0@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \ --1.8.3.1
RE: [PATCH v3 2/8] vdpa/ifc: add multi-queue support
Hi Chenbo,
Sure. I will address in next version.
Quoted 88 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 11:02 AM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 2/8] vdpa/ifc: add multi-queue supportQuoted 11 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 2/8] vdpa/ifc: add multi-queue supportEnable
VHOST_USER_PROTOCOL_F_MQfeature. ExposeIFCVF_MQ_OFFSETregister to enable multi queue.Multi-queue
Quoted 14 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 5 +++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 8 insertions(+)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex f1e1474..34c8226 100644drivers/vdpa/ifc/base/ifcvf.c line 94
Quoted 5 lines
drivers/vdpa/ifc/base/ifcvf.c +2
-0@@ -90,6 +90,11 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4].addr +
IFCVF_MQ_OFFSET;drivers/vdpa/ifc/base/ifcvf.c line 97
Quoted 3 lines
drivers/vdpa/ifc/base/ifcvf.c +3
-0+else + hw->mq_cfg =NULL ; +It will be better to add comments about the explanation that you sent before.
Thanks,
ChenboQuoted 37 lines
drivers/vdpa/ifc/base/ifcvf.c +0
-0if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) { DEBUGOUT("capability incomplete\n"); diff --git
a/drivers/vdpa/ifc/base/ifcvf.hb/drivers/vdpa/ifc/base/ifcvf.hindex ef7697a..d16d9ab 100644drivers/vdpa/ifc/base/ifcvf.h +2
-0@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 07cc63d..3e5ffba 100644drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \ --1.8.3.1
PATCH v3 3/8 vdpa/ifc: set max queues based on virtio spec
In reply to Andy PeiSet max_queues according to virtio HW spec.
For virtio BLK device, set max_queues to the value of "num_queues". "num_queues" is element of struct virtio_blk_config.
For virtio NET device, read num_queues from truct ifcvf_pci_common_cfg, calculate "queue_pairs = (num_queues - 1) / 2" and get queue_pairs. Set max_queues to the value of "queue_pairs".
--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +1 -1
@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33
drivers/vdpa/ifc/ifcvf_vdpa.c +18 -1
@@ -26,6 +26,12 @@#include "base/ifcvf.h" +/* +* RTE_MIN() cannot be used since braced-group within expression allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc driver always has CTRL_VQ, +* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ; +DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues =MIN (IFCVF_MAX_QUEUES , queue_pairs); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal;
Signature · 1 line
1.8.3.1
RE: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec
Quoted 10 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio specSet
max_queuesaccording to virtio HW spec.
Virtio spec
Quoted 2 lines
For virtio BLK device, set
max_queuesto the value of "num_queues". "num_queues" is element of structvirtio_blk_config.
Set max_queues to the value of num_queues in struct virtio_blk_config
Quoted 1 line
For virtio NET device, read
num_queuesfrom tructifcvf_pci_common_cfg,
truct -> struct
Quoted 2 lines
calculate "
queue_pairs= (num_queues- 1) / 2" and getqueue_pairs. Setmax_queuesto the value of "queue_pairs".
And say 'get the queue pair number using num_queues and set max_queues to it'.
Quoted 55 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-)drivers/vdpa/ifc/base/ifcvf.h +1
-1@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33drivers/vdpa/ifc/ifcvf_vdpa.c +7
-1@@ -26,6 +26,12 @@#include "base/ifcvf.h" +/* +* RTE_MIN() cannot be used since braced-group within expression allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1580
Quoted 5 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +2
-0@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc driver always has CTRL_VQ,
I think it's ifc _device_ has CTRL_VQ
Thanks,
Chenbo
Quoted 21 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +9
-1+* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ; +DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues =MIN (IFCVF_MAX_QUEUES , queue_pairs); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal; --1.8.3.1
RE: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec
Hi Chenbo,
Sure. I will address in next version.
Quoted 131 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 2:09 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio specQuoted 10 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio specSet
max_queuesaccording to virtio HW spec.Virtio spec
Quoted 2 lines
For virtio BLK device, set
max_queuesto the value of "num_queues". "num_queues" is element of structvirtio_blk_config.Set
max_queuesto the value ofnum_queuesin structvirtio_blk_configQuoted 2 lines
For virtio NET device, read
num_queuesfrom tructifcvf_pci_common_cfg,truct -> struct
Quoted 2 lines
calculate "
queue_pairs= (num_queues- 1) / 2" and getqueue_pairs. Setmax_queuesto the value of "queue_pairs".And say 'get the queue pair number using
num_queuesand setmax_queuesto it'.Quoted 29 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>---
drivers/vdpa/ifc/base/ifcvf.h| 2 +-drivers/vdpa/ifc/ifcvf_vdpa.c
| 19 ++++++++++++++++++-2 files changed, 19 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.hindex d16d9ab..1e133c0 100644drivers/vdpa/ifc/base/ifcvf.h +1
-1@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 3e5ffba..376239a 100644drivers/vdpa/ifc/ifcvf_vdpa.c +0
-0@@ -26,6 +26,12 @@#include "base/ifcvf.h" drivers/vdpa/ifc/ifcvf_vdpa.c
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +7
-0+/* +* RTE_MIN() cannot be used since braced-group within expression +allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \ @@ -1512,6+1518,7
Quoted 17 lines
@@ struct
rte_vdpa_dev_infodev_info[] = {drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1580
Quoted 5 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +2
-0@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc driver always has CTRL_VQ, I think it's ifc _device_ has
CTRL_VQThanks,
Chenbodrivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = {
Quoted 3 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +3
-0+* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 )/ 2;
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1587
Quoted 4 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +3
-0+DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev- name, + queue_pairs); + internal->max_queues = MIN(IFCVF_MAX_QUEUES,
queue_pairs);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1589
Quoted 2 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +0
-0internal->features = features & ~(1ULL <<
VIRTIO_F_IOMMU_PLATFORM);Quoted 1 line
internal->features|= dev_info[IFCVF_NET].features; @@ -1609,6
Quoted 12 lines
+1623,9 @@ struct
rte_vdpa_dev_infodev_info[] = {drivers/vdpa/ifc/ifcvf_vdpa.c +3
-1internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal; --1.8.3.1
RE: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec
In reply to Xia, ChenboHi Chenbo,
Sure. I will address in next version.
Quoted 131 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 2:09 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio specQuoted 10 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio specSet
max_queuesaccording to virtio HW spec.Virtio spec
Quoted 2 lines
For virtio BLK device, set
max_queuesto the value of "num_queues". "num_queues" is element of structvirtio_blk_config.Set
max_queuesto the value ofnum_queuesin structvirtio_blk_configQuoted 2 lines
For virtio NET device, read
num_queuesfrom tructifcvf_pci_common_cfg,truct -> struct
Quoted 2 lines
calculate "
queue_pairs= (num_queues- 1) / 2" and getqueue_pairs. Setmax_queuesto the value of "queue_pairs".And say 'get the queue pair number using
num_queuesand setmax_queuesto it'.Quoted 29 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>---
drivers/vdpa/ifc/base/ifcvf.h| 2 +-drivers/vdpa/ifc/ifcvf_vdpa.c
| 19 ++++++++++++++++++-2 files changed, 19 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.hindex d16d9ab..1e133c0 100644drivers/vdpa/ifc/base/ifcvf.h +1
-1@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 3e5ffba..376239a 100644drivers/vdpa/ifc/ifcvf_vdpa.c +0
-0@@ -26,6 +26,12 @@#include "base/ifcvf.h" drivers/vdpa/ifc/ifcvf_vdpa.c
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +7
-0+/* +* RTE_MIN() cannot be used since braced-group within expression +allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \ @@ -1512,6+1518,7
Quoted 17 lines
@@ struct
rte_vdpa_dev_infodev_info[] = {drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1580
Quoted 5 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +2
-0@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc driver always has CTRL_VQ, I think it's ifc _device_ has
CTRL_VQThanks,
Chenbodrivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = {
Quoted 3 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +3
-0+* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 )/ 2;
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1587
Quoted 4 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +3
-0+DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev- name, + queue_pairs); + internal->max_queues = MIN(IFCVF_MAX_QUEUES,
queue_pairs);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1589
Quoted 2 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +0
-0internal->features = features & ~(1ULL <<
VIRTIO_F_IOMMU_PLATFORM);Quoted 1 line
internal->features|= dev_info[IFCVF_NET].features; @@ -1609,6
Quoted 12 lines
+1623,9 @@ struct
rte_vdpa_dev_infodev_info[] = {drivers/vdpa/ifc/ifcvf_vdpa.c +3
-1internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal; --1.8.3.1
PATCH v3 4/8 vdpa/ifc: write queue count to MQ register
In reply to Andy PeiWrite queue count to IFCVF_MQ_OFFSET register
to enable multi queue feature.
--- drivers/vdpa/ifc/base/ifcvf.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +30 -0
@@ -198,6 +198,35 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_multiqueue (struct ifcvf_hw *hw) +{ +u8 *mq_cfg; +int qid; +int nr_queue_pair =0 ; + +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue_pair++; + } + +if (nr_queue_pair ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + +if (hw->device_type ==IFCVF_NET ) + nr_queue_pair = (nr_queue_pair +1 ) /2 ; + + mq_cfg = hw->mq_cfg; +if (mq_cfg) { + *(u32 *)mq_cfg = nr_queue_pair; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue_pair); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
@@ -215,6 +244,7 @@return -1 ; } +ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
Signature · 1 line
1.8.3.1
RE: [PATCH v3 4/8] vdpa/ifc: write queue count to MQ register
Quoted 11 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 4/8] vdpa/ifc: write queue count to MQ registerWrite queue count to
IFCVF_MQ_OFFSETregister
to enable multi queue feature.
Multi-queue
Quoted 12 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+)drivers/vdpa/ifc/base/ifcvf.c +0
-0
drivers/vdpa/ifc/base/ifcvf.c line 202
Quoted 5 lines
drivers/vdpa/ifc/base/ifcvf.c +2
-0@@ -198,6 +198,35 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_multiqueue (struct ifcvf_hw *hw)
Maybe ifcvf_enable_mq?
drivers/vdpa/ifc/base/ifcvf.c line 205
Quoted 3 lines
drivers/vdpa/ifc/base/ifcvf.c +3
-0+{ +u8 *mq_cfg; +int qid;
Qid should be the same type as nr_vring, so u8
Quoted 5 lines
drivers/vdpa/ifc/base/ifcvf.c +5
-0+int nr_queue_pair =0 ; + +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ;
drivers/vdpa/ifc/base/ifcvf.c line 220
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-0+ nr_queue_pair++; + } + +if (nr_queue_pair ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + +if (hw->device_type ==IFCVF_NET ) + nr_queue_pair = (nr_queue_pair +1 ) /2 ;
Why +1?
drivers/vdpa/ifc/base/ifcvf.c line 227
Quoted 7 lines
drivers/vdpa/ifc/base/ifcvf.c +7
-0+ + mq_cfg = hw->mq_cfg; +if (mq_cfg) { + *(u32 *)mq_cfg = nr_queue_pair; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue_pair); + }
As blk does not have queue pair, the log should be different for net/blk.
The variable nr_queue_pair should be nr_queue as it's the number of queues. When device type is net, you can make *(u32 *)mq_cfg equal to nr_queue / 2
Thanks,
Chenbo
Quoted 15 lines
drivers/vdpa/ifc/base/ifcvf.c +3
-1+} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {@@ -215,6 +244,7 @@return -1 ; } +ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, --1.8.3.1
RE: [PATCH v3 4/8] vdpa/ifc: write queue count to MQ register
Hi Chenbo
Thanks for your reply, my reply inline.
Quoted 17 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 4:08 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 4/8] vdpa/ifc: write queue count to MQ registerQuoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 4/8] vdpa/ifc: write queue count to MQ register
Quoted 5 lines
Quoted 2 lines
Write queue count to
IFCVF_MQ_OFFSETregister to enable multi queue feature.Multi-queue
Sure.
Quoted 15 lines
Quoted 15 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex 34c8226..7efb408 100644drivers/vdpa/ifc/base/ifcvf.c +0
-0@@ -198,6 +198,35 @@IFCVF_WRITE_REG32 (val >>32 , hi); }
Quoted 5 lines
drivers/vdpa/ifc/base/ifcvf.c line 202
Quoted 2 lines
drivers/vdpa/ifc/base/ifcvf.c +2
-0+STATIC void +ifcvf_enable_multiqueue (struct ifcvf_hw *hw)Maybe
ifcvf_enable_mq?
OK.
Quoted 34 lines
drivers/vdpa/ifc/base/ifcvf.c line 205
Quoted 3 lines
drivers/vdpa/ifc/base/ifcvf.c +3
-0+{ +u8 *mq_cfg; +int qid;Qid should be the same type as
nr_vring, so u8Quoted 5 lines
drivers/vdpa/ifc/base/ifcvf.c +5
-0+int nr_queue_pair =0 ; + +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ;drivers/vdpa/ifc/base/ifcvf.c line 220
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-0+ nr_queue_pair++; + } + +if (nr_queue_pair ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + +if (hw->device_type ==IFCVF_NET ) + nr_queue_pair = (nr_queue_pair +1 ) /2 ;Why +1?
drivers/vdpa/ifc/base/ifcvf.c line 227
Quoted 7 lines
drivers/vdpa/ifc/base/ifcvf.c +7
-0+ + mq_cfg = hw->mq_cfg; +if (mq_cfg) { + *(u32 *)mq_cfg = nr_queue_pair; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue_pair); + }As blk does not have queue pair, the log should be different for net/blk.
Quoted 4 lines
The variable
nr_queue_pairshould benr_queueas it's the number of queues.
When device type is net, you can make *(u32 *)mq_cfg equal tonr_queue/ 2
OK. I will sent a new version to address it.
Quoted 18 lines
Thanks,
ChenboQuoted 15 lines
drivers/vdpa/ifc/base/ifcvf.c +3
-1+} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {@@ -215,6 +244,7 @@return -1 ; } +ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, --1.8.3.1
PATCH v3 5/8 vdpa/ifc: only configure enabled queue
In reply to Andy Peiwhen configure the hardware queue, we only configure queues which have been enabled by vhost.
--- drivers/vdpa/ifc/base/ifcvf.c | 3 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +3 -0
@@ -246,6 +246,9 @@ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);
drivers/vdpa/ifc/ifcvf_vdpa.c +14 -2
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) !=0 ) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; }
Signature · 1 line
1.8.3.1
RE: [PATCH v3 5/8] vdpa/ifc: only configure enabled queue
Quoted 11 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 5/8] vdpa/ifc: only configure enabled queuewhen configure the hardware queue, we only configure queues which have been enabled by vhost.
Should be capital 'W' and 'When configuring'
With this fixed:
Quoted 76 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 3 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-)drivers/vdpa/ifc/base/ifcvf.c +3
-0@@ -246,6 +246,9 @@ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);drivers/vdpa/ifc/ifcvf_vdpa.c +14
-3@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) != 0) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; } --1.8.3.1
RE: [PATCH v3 5/8] vdpa/ifc: only configure enabled queue
Hi Chenbo,
Thanks for your effort.
Quoted 104 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 4:12 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 5/8] vdpa/ifc: only configure enabled queueQuoted 11 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 5/8] vdpa/ifc: only configure enabled queuewhen configure the hardware queue, we only configure queues which have been enabled by vhost.
Should be capital 'W' and 'When configuring'
With this fixed:
Reviewed-by Chenbo Xia <[email protected]>Quoted 56 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>---
drivers/vdpa/ifc/base/ifcvf.c| 3 +++drivers/vdpa/ifc/ifcvf_vdpa.c
| 16 ++++++++++++++--2 files changed, 17 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex 7efb408..619b034 100644drivers/vdpa/ifc/base/ifcvf.c +3
-0@@ -246,6 +246,9 @@ifcvf_enable_multiqueue (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 376239a..b00afdb 100644drivers/vdpa/ifc/ifcvf_vdpa.c +7
-0@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t)vring.kickfd << 32; @@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev = rte_vhost_get_vdpa_device(vid); list = find_internal_resource_by_vdev(vdev);@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info { · line 1086
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +6
-2-if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) != 0) - DRV_LOG(NOTICE, "vDPA (%s): software relay is used.", + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) + DRV_LOG(NOTICE, "vDPA (%s): software relay isused.",
Quoted 11 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +2
-0vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device- name);return 0 ;}
--
1.8.3.1
PATCH v3 6/8 vdpa/ifc: support dynamic enable/disable queue
In reply to Andy PeiFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configurate queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
--- drivers/vdpa/ifc/base/ifcvf.c | 99 +++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 93 +++++++++++++++++++++++++++++++++------- 3 files changed, 183 insertions(+), 15 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +99 -0
@@ -227,6 +227,105 @@ } } +int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (i >= (int )hw->nr_vring) +return -1 ; + + cfg = hw->common_cfg; +if (!cfg) { +WARNINGOUT ("common_cfg in HW is NULL.\n" ); +return -1 ; + } + +ifcvf_enable_multiqueue (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +WARNINGOUT ("queue %u, msix vec alloc failed\n" , i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + } + + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (i >= (int )hw->nr_vring) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +WARNINGOUT ("common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); + }else { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ); + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 ); + } + hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
drivers/vdpa/ifc/base/ifcvf.h +6 -0
@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void
drivers/vdpa/ifc/ifcvf_vdpa.c +78 -15
124 lines not on this page · raw message
Signature · 1 line
1.8.3.1
RE: [PATCH v3 6/8] vdpa/ifc: support dynamic enable/disable queue
Quoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 6/8] vdpa/ifc: support dynamic enable/disable queue
Quoted 4 lines
From: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configurate queue.
configure
Quoted 24 lines
For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 99 +++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 93 +++++++++++++++++++++++++++++++++----- -- 3 files changed, 183 insertions(+), 15 deletions(-)drivers/vdpa/ifc/base/ifcvf.c +4
-0@@ -227,6 +227,105 @@ } } +int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg;
drivers/vdpa/ifc/base/ifcvf.c line 243
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-0+u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (i >= (int )hw->nr_vring) +return -1 ; + + cfg = hw->common_cfg; +if (!cfg) { +WARNINGOUT ("common_cfg in HW is NULL.\n" );
This should be error log
drivers/vdpa/ifc/base/ifcvf.c line 255
Quoted 12 lines
drivers/vdpa/ifc/base/ifcvf.c +12
-0+return -1 ; + } + +ifcvf_enable_multiqueue (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +WARNINGOUT ("queue %u, msix vec alloc failed\n" , i);
Ditto. And %u -> %d
Same for the function ifcvf_disable_vring_hw.
Thanks,
Chenbo
Quoted 226 lines
drivers/vdpa/ifc/base/ifcvf.c +73
-0+return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + } + + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (i >= (int )hw->nr_vring) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +WARNINGOUT ("common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); + }else { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ); + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 ); + } + hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {drivers/vdpa/ifc/base/ifcvf.h +6
-0@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void drivers/vdpa/ifc/ifcvf_vdpa.c +78
-16@@ -1282,13 +1282,59 @@ struct rte_vdpa_dev_info { }static int +ifcvf_config_vring (struct ifcvf_internal *internal,int vring) +{ +struct ifcvf_hw *hw = &internal->hw; +int vid = internal->vid; +struct rte_vhost_vring vq; +uint64_t gpa; + +if (hw->vring[vring].enable) { +rte_vhost_get_vhost_vring (vid, vring, &vq); + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for descriptor ring." ); +return -1 ; + } + hw->vring[vring].desc = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.avail); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for available ring." ); +return -1 ; + } + hw->vring[vring].avail = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.used); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for used ring." ); +return -1 ; + } + hw->vring[vring].used = gpa; + + hw->vring[vring].size = vq.size; +rte_vhost_get_vring_base (vid, vring, + &hw->vring[vring].last_avail_idx, + &hw->vring[vring].last_used_idx); +ifcvf_enable_vring_hw (&internal->hw, vring); + }else { +ifcvf_disable_vring_hw (&internal->hw, vring); +rte_vhost_set_vring_base (vid, vring, + hw->vring[vring].last_avail_idx, + hw->vring[vring].last_used_idx); + } + +return 0 ; +} + +static int ifcvf_set_vring_state (int vid,int vring,int state) {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal;struct ifcvf_hw *hw; -struct ifcvf_pci_common_cfg *cfg; +bool enable = !!state;int ret =0 ; vdev =rte_vhost_get_vdpa_device (vid);@@ -1298,6 +1344,9 @@ struct rte_vdpa_dev_info {return -1 ; } +DRV_LOG (INFO ,"%s queue %d of vDPA device %s" , + enable ?"enable" :"disable" , vring, vdev->device->name); + internal = list->internal;if (vring <0 || vring >= internal->max_queues *2 ) {DRV_LOG (ERR ,"Vring index %d not correct" , vring);@@ -1305,27 +1354,41 @@ struct rte_vdpa_dev_info { } hw = &internal->hw; + hw->vring[vring].enable = enable; +if (!internal->configured) - goto exit; +return 0 ; - cfg = hw->common_cfg; -IFCVF_WRITE_REG16 (vring, &cfg->queue_select); -IFCVF_WRITE_REG16 (!!state, &cfg->queue_enable); +unset_notify_relay (internal); -if (!state && hw->vring[vring].enable) { - ret =vdpa_disable_vfio_intr (internal); -if (ret) -return ret; + ret =vdpa_enable_vfio_intr (internal, false); +if (ret) { +DRV_LOG (ERR ,"failed to set vfio interrupt of vDPA device %s" , + vdev->device->name); +return ret; } -if (state && !hw->vring[vring].enable) { - ret =vdpa_enable_vfio_intr (internal, false); -if (ret) -return ret; + ret =ifcvf_config_vring (internal, vring); +if (ret) { +DRV_LOG (ERR ,"failed to configure queue %d of vDPA device %s" , + vring, vdev->device->name); +return ret; + } + + ret =setup_notify_relay (internal); +if (ret) { +DRV_LOG (ERR ,"failed to setup notify relay of vDPA device %s" , + vdev->device->name); +return ret; + } + + ret =rte_vhost_host_notifier_ctrl (vid, vring, enable); +if (ret) { +DRV_LOG (ERR ,"vDPA device %s queue %d host notifier ctrl fail" , + vdev->device->name, vring); +return ret; } -exit: - hw->vring[vring].enable = !!state;return 0 ; } --1.8.3.1
RE: [PATCH v3 6/8] vdpa/ifc: support dynamic enable/disable queue
Hi Chenbo,
Thanks for your reply.
My reply is inline.
Quoted 17 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 4:20 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 6/8] vdpa/ifc: support dynamic enable/disable queueQuoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 6/8] vdpa/ifc: support dynamic enable/disable queue
Quoted 7 lines
Quoted 4 lines
From: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configurate queue.configure
Fix in next version.
Quoted 52 lines
Quoted 23 lines
For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 99+++++++++++++++++++++++++++++++++++++++++++
drivers/vdpa/ifc/base/ifcvf.h| 6 +++drivers/vdpa/ifc/ifcvf_vdpa.c
| 93 +++++++++++++++++++++++++++++++++-----
--3 files changed, 183 insertions(+), 15 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex 619b034..792d258 100644drivers/vdpa/ifc/base/ifcvf.c +3
-0@@ -227,6 +227,105 @@ } } +int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) { +struct ifcvf_pci_common_cfg *cfg;drivers/vdpa/ifc/base/ifcvf.c line 242
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-0+u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (i >= (int )hw->nr_vring) +return -1 ; + + cfg = hw->common_cfg; +if (!cfg) { +WARNINGOUT ("common_cfg in HW is NULL.\n" );This should be error log
drivers/vdpa/ifc/base/ifcvf.c line 255
Quoted 13 lines
drivers/vdpa/ifc/base/ifcvf.c +12
-0+return -1 ; + } + +ifcvf_enable_multiqueue (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg- queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +WARNINGOUT ("queue %u, msix vec alloc failed\n" , i);Ditto. And %u -> %d
Quoted 2 lines
Same for the function
ifcvf_disable_vring_hw.
I will useRTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
Quoted 231 lines
Thanks,
ChenboQuoted 144 lines
drivers/vdpa/ifc/base/ifcvf.c +72
-0+return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + } + + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) { +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (i >= (int )hw->nr_vring) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +WARNINGOUT ("common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); + }else { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ); + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 ); + } + hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.hindex 1e133c0..3726da7 100644drivers/vdpa/ifc/base/ifcvf.h +6
-0@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void (quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex b00afdb..32bc1c9 100644drivers/vdpa/ifc/ifcvf_vdpa.c +45
-0@@ -1282,13 +1282,59 @@ struct rte_vdpa_dev_info { }static int +ifcvf_config_vring (struct ifcvf_internal *internal,int vring) { +struct ifcvf_hw *hw = &internal->hw; +int vid = internal->vid; +struct rte_vhost_vring vq; +uint64_t gpa; + +if (hw->vring[vring].enable) { +rte_vhost_get_vhost_vring (vid, vring, &vq); + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for descriptor ring." ); +return -1 ; + } + hw->vring[vring].desc = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.avail); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for available ring." ); +return -1 ; + } + hw->vring[vring].avail = gpa; + + gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.used); +if (gpa ==0 ) { +DRV_LOG (ERR ,"Fail to get GPA for used ring." ); +return -1 ; + } + hw->vring[vring].used = gpa; + + hw->vring[vring].size = vq.size; +rte_vhost_get_vring_base (vid, vring, + &hw->vring[vring].last_avail_idx, + &hw->vring[vring].last_used_idx); +ifcvf_enable_vring_hw (&internal->hw, vring); + }else { +ifcvf_disable_vring_hw (&internal->hw, vring); +rte_vhost_set_vring_base (vid, vring, + hw->vring[vring].last_avail_idx, + hw->vring[vring].last_used_idx); + } + +return 0 ; +} + +static intdrivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info { } · line 1337
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1ifcvf_set_vring_state(int vid, int vring, int state) { struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal;struct ifcvf_hw *hw; -struct ifcvf_pci_common_cfg *cfg; +bool enable = !!state;int ret =0 ; vdev = rte_vhost_get_vdpa_device(vid); @@ -1298,6 +1344,9 @@struct
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info {
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +3
-0rte_vdpa_dev_info {return -1 ; } +DRV_LOG (INFO ,"%s queue %d of vDPA device %s" , + enable ?"enable" :"disable" , vring, vdev->device->name); + internal = list->internal;if (vring <0 || vring >= internal->max_queues *2 ) { DRV_LOG(ERR, "Vring index %d not correct", vring); @@ -1305,27
Quoted 12 lines
+1354,41 @@ struct
rte_vdpa_dev_info{drivers/vdpa/ifc/ifcvf_vdpa.c +3
-3} hw = &internal->hw; + hw->vring[vring].enable = enable; +if (!internal->configured) - goto exit; +return 0 ; - cfg = hw->common_cfg; -IFCVF_WRITE_REG16 (vring, &cfg->queue_select);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info {
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +4
-5-IFCVF_WRITE_REG16 (!!state, &cfg->queue_enable); +unset_notify_relay (internal); -if (!state && hw->vring[vring].enable) { - ret =vdpa_disable_vfio_intr (internal); -if (ret) -return ret; + ret =vdpa_enable_vfio_intr (internal, false); +if (ret) { + DRV_LOG(ERR, "failed to set vfio interrupt of vDPAdevice %s",
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info {
Quoted 11 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +5
-4+ vdev->device->name); +return ret; } -if (state && !hw->vring[vring].enable) { - ret =vdpa_enable_vfio_intr (internal, false); -if (ret) -return ret; + ret =ifcvf_config_vring (internal, vring); +if (ret) { + DRV_LOG(ERR, "failed to configure queue %d of vDPAdevice %s",
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info {
Quoted 7 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +7
-0+ vring, vdev->device->name); +return ret; + } + + ret =setup_notify_relay (internal); +if (ret) { + DRV_LOG(ERR, "failed to setup notify relay of vDPAdevice %s",
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info {
Quoted 7 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +7
-0+ vdev->device->name); +return ret; + } + + ret =rte_vhost_host_notifier_ctrl (vid, vring, enable); +if (ret) { + DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrlfail",
Quoted 11 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +2
-3+ vdev->device->name, vring); +return ret; } -exit: - hw->vring[vring].enable = !!state;return 0 ; } --1.8.3.1
PATCH v3 7/8 vhost: vDPA blk device gets ready when any queue is ready
In reply to Andy PeiWhen boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end conf_dev when any queue is ready.
--- lib/vhost/vhost_user.c | 51 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-)
lib/vhost/vhost_user.c +33 -18
@@ -1449,9 +1449,10 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int -virtio_is_ready (struct virtio_net *dev) +virtio_is_ready (struct virtio_net *dev,uint32_t vdpa_type) {struct vhost_virtqueue *vq;uint32_t i, nr_vring = dev->nr_vring;
@@ -1462,13 +1463,20 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; - -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_NET ) { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + }else { +/* +* vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK +* is the only case currently +*/ + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];
@@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net *dev,if (unlock_required)vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) + goto out; + + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev, vdpa_type)) goto out;/*
@@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net *dev, dev->flags |=VIRTIO_DEV_RUNNING ; } - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (dev->ifname,ERR ,"failed to get vdpa dev type.\n" ); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL ) goto out;
Signature · 1 line
1.8.3.1
RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready
Quoted 10 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready
Quoted 2 lines
When boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end
conf_devwhen any
What is conf_dev?
Quoted 33 lines
queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 51 ++++++++++++++++++++++++++++++++------------- ----- 1 file changed, 33 insertions(+), 18 deletions(-)lib/vhost/vhost_user.c +2
-5@@ -1449,9 +1449,10 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int -virtio_is_ready (struct virtio_net *dev) +virtio_is_ready (struct virtio_net *dev,uint32_t vdpa_type) {struct vhost_virtqueue *vq;uint32_t i, nr_vring = dev->nr_vring;@@ -1462,13 +1463,20 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; - -if (dev->nr_vring < nr_vring)
lib/vhost/vhost_user.c line 1474
Quoted 10 lines
lib/vhost/vhost_user.c +9
-1-return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_NET ) { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + }else { +/* +* vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK +* is the only case currently +*/ + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ;
You should consider the case when vdpa device is not there. Maybe you can use int for vdpa_type, -1 for non-vdpa.
Also note that below check is only needed for some cases.
Thanks,
Chenbo
Quoted 59 lines
lib/vhost/vhost_user.c +22
-12} +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];@@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net *dev,if (unlock_required)vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) + goto out; + + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev, vdpa_type)) goto out;/* @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net *dev, dev->flags |=VIRTIO_DEV_RUNNING ; } - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n"); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL )goto out;
--
1.8.3.1
RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready
Hi Chenbo,
Thanks for your reply.
My reply is inline.
Quoted 19 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 5:09 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyQuoted 10 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready
Quoted 5 lines
Quoted 2 lines
When boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end
conf_devwhen anyWhat is
conf_dev?
I refer to
/** Driver configure the device (Mandatory) */ int (*dev_conf)(int vid);
So do you think I should use "configure device"?
Quoted 38 lines
Quoted 27 lines
queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 51 ++++++++++++++++++++++++++++++++------------- ----- 1 file changed, 33 insertions(+), 18 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-04ad28ba..9169cf5 100644
lib/vhost/vhost_user.c +2
-1@@ -1449,9 +1449,10 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int -virtio_is_ready (struct virtio_net *dev) +virtio_is_ready (struct virtio_net *dev,uint32_t vdpa_type) {struct vhost_virtqueue *vq; uint32_t i, nr_vring = dev->nr_vring; @@ -1462,13 +1463,20 @@lib/vhost/vhost_user.c +0
-0if (!dev->nr_vring)lib/vhost/vhost_user.c
Quoted 10 lines
lib/vhost/vhost_user.c +3
-5return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; - -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_NET ) { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =
VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
Quoted 10 lines
lib/vhost/vhost_user.c line 1474
Quoted 6 lines
lib/vhost/vhost_user.c +6
-0+ }else { +/* +* vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK +* is the only case currently +*/ + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ;You should consider the case when vdpa device is not there. Maybe you can use int for
vdpa_type, -1 for non-vdpa.
I init vdpa_type to 0;
#define RTE_VHOST_VDPA_DEVICE_TYPE_NET 0
#define RTE_VHOST_VDPA_DEVICE_TYPE_BLK 1
And get_dev_type only return RTE_VHOST_VDPA_DEVICE_TYPE_BLK or RTE_VHOST_VDPA_DEVICE_TYPE_NET. I think if when vdpa device is not there, this code runs in the original way. Do you think use init vdpa_type to -1 is better?
Quoted 2 lines
Also note that below check is only needed for some cases.
Yes, I got it. I will fix it in next version.
Quoted 64 lines
Thanks,
ChenboQuoted 10 lines
lib/vhost/vhost_user.c +3
-0} +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];@@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net *dev,if (unlock_required)lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev, · line 3184
Quoted 10 lines
lib/vhost/vhost_user.c +7
-1vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) + goto out; + + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
&vdpa_type);Quoted 17 lines
lib/vhost/vhost_user.c +12
-0+if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev, vdpa_type)) goto out;/* @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net *dev, dev->flags |=VIRTIO_DEV_RUNNING ;lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev, · line 3191
Quoted 10 lines
lib/vhost/vhost_user.c +0
-5} - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed toget vdpa
Quoted 12 lines
lib/vhost/vhost_user.c +0
-6dev type.\n"); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL )goto out;
--
1.8.3.1
RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready
Quoted 35 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Wednesday, October 12, 2022 8:13 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyHi Chenbo,
Thanks for your reply.
My reply is inline.Quoted 20 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 5:09 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyQuoted 11 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyWhen boot from virtio blk device, seabios in QEMU only enables one
queue.
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
To work in this scenario, vDPA BLK device back-end
conf_devwhen anyWhat is
conf_dev?I refer to
/** Driver configure the device (Mandatory) */ int (*dev_conf)(int vid);So do you think I should use "configure device"?
Yes. It will be better
Quoted 50 lines
Quoted 39 lines
Quoted 28 lines
queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 51 ++++++++++++++++++++++++++++++++------------- ----- 1 file changed, 33 insertions(+), 18 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-04ad28ba..9169cf5 100644
lib/vhost/vhost_user.c +2
-1@@ -1449,9 +1449,10 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int -virtio_is_ready (struct virtio_net *dev) +virtio_is_ready (struct virtio_net *dev,uint32_t vdpa_type) {struct vhost_virtqueue *vq; uint32_t i, nr_vring = dev->nr_vring; @@ -1462,13 +1463,20 @@lib/vhost/vhost_user.c +0
-0if (!dev->nr_vring)lib/vhost/vhost_user.c
Quoted 10 lines
lib/vhost/vhost_user.c +3
-5return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; - -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_NET ) { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =
VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;Quoted 8 lines
lib/vhost/vhost_user.c line 1474
Quoted 6 lines
lib/vhost/vhost_user.c +6
-0+ }else { +/* +* vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK +* is the only case currently +*/ + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ;You should consider the case when vdpa device is not there. Maybe you
can
Quoted 2 lines
use int for
vdpa_type, -1 for non-vdpa.
Quoted 8 lines
I init
vdpa_typeto 0;
#defineRTE_VHOST_VDPA_DEVICE_TYPE_NET0
#defineRTE_VHOST_VDPA_DEVICE_TYPE_BLK1
Andget_dev_typeonly returnRTE_VHOST_VDPA_DEVICE_TYPE_BLKorRTE_VHOST_VDPA_DEVICE_TYPE_NET.
I think if when vdpa device is not there, this code runs in the original way.
Do you think use initvdpa_typeto -1 is better?
I was talking about readability, current way will be confusing. So adding -1 will be better. The check could be if (type == blk) ... else ... as Type 0/-1 has the same handling.
Thanks,
Chenbo
Quoted 71 lines
Quoted 4 lines
Also note that below check is only needed for some cases.
Yes, I got it. I will fix it in next version.
Quoted 5 lines
Thanks,
ChenboQuoted 2 lines
}
Quoted 7 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net
Quoted 7 lines
lib/vhost/vhost_user.c +3
-0+if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];@@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net*dev,
Quoted 26 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev,
Quoted 11 lines
lib/vhost/vhost_user.c +7
-1if (unlock_required)vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) + goto out; + + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
&vdpa_type);Quoted 14 lines
lib/vhost/vhost_user.c +12
-0+if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev, vdpa_type)) goto out;Quoted 2 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net
Quoted 2 lines
lib/vhost/vhost_user.c +0
-0/* @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net*dev,
Quoted 24 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev,
Quoted 11 lines
lib/vhost/vhost_user.c +0
-5dev->flags |=VIRTIO_DEV_RUNNING ; } - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed toget vdpa
Quoted 12 lines
lib/vhost/vhost_user.c +0
-6dev type.\n"); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL )goto out;
--
1.8.3.1
RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready
Hi Chenbo,
Thanks for your reply.
My reply is inline.
Quoted 48 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Thursday, October 13, 2022 9:00 AM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyQuoted 39 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Wednesday, October 12, 2022 8:13 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyHi Chenbo,
Thanks for your reply.
My reply is inline.Quoted 21 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 5:09 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyQuoted 11 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyWhen boot from virtio blk device, seabios in QEMU only enables one
queue.
Quoted 3 lines
Quoted 2 lines
To work in this scenario, vDPA BLK device back-end
conf_devwhen any
Quoted 9 lines
Quoted 6 lines
Quoted 2 lines
What is
conf_dev?I refer to
/** Driver configure the device (Mandatory) */ int (*dev_conf)(int vid);So do you think I should use "configure device"?
Yes. It will be better
OK. I will make it this way in next version.
Quoted 55 lines
Quoted 48 lines
Quoted 39 lines
Quoted 28 lines
queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 51 ++++++++++++++++++++++++++++++++------------- ----- 1 file changed, 33 insertions(+), 18 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-04ad28ba..9169cf5 100644
lib/vhost/vhost_user.c +2
-1@@ -1449,9 +1449,10 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int -virtio_is_ready (struct virtio_net *dev) +virtio_is_ready (struct virtio_net *dev,uint32_t vdpa_type) {struct vhost_virtqueue *vq; uint32_t i, nr_vring = dev->nr_vring; @@ -1462,13 +1463,20 @@lib/vhost/vhost_user.c +0
-0if (!dev->nr_vring)lib/vhost/vhost_user.c
Quoted 10 lines
lib/vhost/vhost_user.c +3
-5return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; - -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_NET ) { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =
VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;Quoted 9 lines
lib/vhost/vhost_user.c line 1474
Quoted 6 lines
lib/vhost/vhost_user.c +6
-0+ }else { +/* +* vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK +* is the only case currently +*/ + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ;You should consider the case when vdpa device is not there. Maybe you
Quoted 7 lines
can
Quoted 2 lines
use int for
vdpa_type, -1 for non-vdpa.I init
vdpa_typeto 0;
#defineRTE_VHOST_VDPA_DEVICE_TYPE_NET0 #defineRTE_VHOST_VDPA_DEVICE_TYPE_BLK1 Andget_dev_typeonly returnRTE_VHOST_VDPA_DEVICE_TYPE_BLKor
Quoted 9 lines
RTE_VHOST_VDPA_DEVICE_TYPE_NET.Quoted 3 lines
I think if when vdpa device is not there, this code runs in the original way.
Do you think use initvdpa_typeto -1 is better?I was talking about readability, current way will be confusing. So adding -1 will be better. The check could be if (type == blk) ... else ... as Type 0/-1 has the same handling.
I will add
#define RTE_VHOST_NO_VDPA_DEVICE -1
And init vdpa_type to RTE_VHOST_NO_VDPA_DEVICE and follow your suggestion.
Quoted 11 lines
Thanks,
ChenboQuoted 8 lines
Quoted 4 lines
Also note that below check is only needed for some cases.
Yes, I got it. I will fix it in next version.
Quoted 3 lines
Thanks,
Chenbo
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
lib/vhost/vhost_user.c line 1481
Quoted 8 lines
lib/vhost/vhost_user.c +3
-0} +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];
To make sure queue index is not out of range, I will keep above check for all the cases.
Quoted 55 lines
Quoted 55 lines
Quoted 1 line
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net
Quoted 1 line
lib/vhost/vhost_user.c +0
-0@@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net*dev,
Quoted 26 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev,
Quoted 11 lines
lib/vhost/vhost_user.c +7
-1if (unlock_required)vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) + goto out; + + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
&vdpa_type);Quoted 14 lines
lib/vhost/vhost_user.c +12
-0+if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev, vdpa_type)) goto out;Quoted 2 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net
Quoted 2 lines
lib/vhost/vhost_user.c +0
-0/* @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net*dev,
Quoted 24 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev,
Quoted 11 lines
lib/vhost/vhost_user.c +0
-5dev->flags |=VIRTIO_DEV_RUNNING ; } - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed toget vdpa
Quoted 12 lines
lib/vhost/vhost_user.c +0
-6dev type.\n"); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL )goto out;
--
1.8.3.1
Re: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready
In reply to Xia, ChenboQuoted 105 lines · Chenbo
Quoted 35 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Wednesday, October 12, 2022 8:13 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyHi Chenbo,
Thanks for your reply.
My reply is inline.Quoted 20 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 5:09 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyQuoted 11 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyWhen boot from virtio blk device, seabios in QEMU only enables one
queue.
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
To work in this scenario, vDPA BLK device back-end
conf_devwhen anyWhat is
conf_dev?I refer to
/** Driver configure the device (Mandatory) */ int (*dev_conf)(int vid);So do you think I should use "configure device"?
Yes. It will be better
Quoted 50 lines
Quoted 39 lines
Quoted 30 lines
queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 51 ++++++++++++++++++++++++++++++++------------- ----- 1 file changed, 33 insertions(+), 18 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-04ad28ba..9169cf5 100644
lib/vhost/vhost_user.c +2
-1@@ -1449,9 +1449,10 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int -virtio_is_ready (struct virtio_net *dev) +virtio_is_ready (struct virtio_net *dev,uint32_t vdpa_type) {struct vhost_virtqueue *vq; uint32_t i, nr_vring = dev->nr_vring; @@ -1462,13 +1463,20 @@if (!dev->nr_vring)
return 0;lib/vhost/vhost_user.c
Quoted 8 lines
lib/vhost/vhost_user.c +3
-5-if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; - -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_NET ) { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =
VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;Quoted 8 lines
lib/vhost/vhost_user.c line 1474
Quoted 6 lines
lib/vhost/vhost_user.c +6
-0+ }else { +/* +* vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK +* is the only case currently +*/ + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ;You should consider the case when vdpa device is not there. Maybe you
can
Quoted 2 lines
use int for
vdpa_type, -1 for non-vdpa.Quoted 8 lines
I init
vdpa_typeto 0;
#defineRTE_VHOST_VDPA_DEVICE_TYPE_NET0
#defineRTE_VHOST_VDPA_DEVICE_TYPE_BLK1
Andget_dev_typeonly returnRTE_VHOST_VDPA_DEVICE_TYPE_BLKorRTE_VHOST_VDPA_DEVICE_TYPE_NET.
I think if when vdpa device is not there, this code runs in the original way.
Do you think use initvdpa_typeto -1 is better?
Quoted 3 lines
I was talking about readability, current way will be confusing. So adding -1 will be better. The check could be if (type == blk) ... else ... as Type 0/-1 has the same handling.
Also, the vdpa_type can be obtained from dev, so instead of passing the vdpa_type as argument for the function, it should get fetched directly in virtio_is_ready().
Maxime
Quoted 75 lines
Thanks,
ChenboQuoted 71 lines
Quoted 4 lines
Also note that below check is only needed for some cases.
Yes, I got it. I will fix it in next version.
Quoted 5 lines
Thanks,
ChenboQuoted 2 lines
}Quoted 7 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net
Quoted 7 lines
lib/vhost/vhost_user.c +3
-0+if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];@@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net*dev,
Quoted 26 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev,
Quoted 11 lines
lib/vhost/vhost_user.c +7
-1if (unlock_required)vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) + goto out; + + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
&vdpa_type);Quoted 14 lines
lib/vhost/vhost_user.c +12
-0+if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev, vdpa_type)) goto out;Quoted 2 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net
Quoted 2 lines
lib/vhost/vhost_user.c +0
-0/* @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net*dev,
Quoted 24 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev,
Quoted 11 lines
lib/vhost/vhost_user.c +0
-5dev->flags |=VIRTIO_DEV_RUNNING ; } - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed toget vdpa
Quoted 12 lines
lib/vhost/vhost_user.c +0
-6dev type.\n"); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL )goto out;
--
1.8.3.1
RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready
Hi Maxime,
Sorry, I did not notice your email before I send out V4 version. My reply is inline.
Quoted 121 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Thursday, October 13, 2022 3:17 PM
To: Xia, Chenbo <[email protected]>; Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>
Subject: Re: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyQuoted 39 lines · Chenbo
Quoted 39 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Wednesday, October 12, 2022 8:13 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyHi Chenbo,
Thanks for your reply.
My reply is inline.Quoted 21 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 5:09 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyQuoted 11 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:16 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is readyWhen boot from virtio blk device, seabios in QEMU only enables one
queue.
Quoted 3 lines
Quoted 2 lines
To work in this scenario, vDPA BLK device back-end
conf_devwhen anyQuoted 64 lines
Quoted 6 lines
Quoted 2 lines
What is
conf_dev?I refer to
/** Driver configure the device (Mandatory) */ int (*dev_conf)(int vid);So do you think I should use "configure device"?
Yes. It will be better
Quoted 55 lines
Quoted 39 lines
Quoted 30 lines
queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 51 ++++++++++++++++++++++++++++++++------------- ----- 1 file changed, 33 insertions(+), 18 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-04ad28ba..9169cf5 100644
lib/vhost/vhost_user.c +2
-1@@ -1449,9 +1449,10 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int -virtio_is_ready (struct virtio_net *dev) +virtio_is_ready (struct virtio_net *dev,uint32_t vdpa_type) {struct vhost_virtqueue *vq; uint32_t i, nr_vring = dev->nr_vring; @@ -1462,13 +1463,20 @@if (!dev->nr_vring)
return 0;lib/vhost/vhost_user.c
Quoted 8 lines
lib/vhost/vhost_user.c +3
-5-if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; - -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_NET ) { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =
VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;Quoted 9 lines
lib/vhost/vhost_user.c line 1474
Quoted 6 lines
lib/vhost/vhost_user.c +6
-0+ }else { +/* +* vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK +* is the only case currently +*/ + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ;You should consider the case when vdpa device is not there. Maybe you
can
Quoted 2 lines
use int for
vdpa_type, -1 for non-vdpa.I init
vdpa_typeto 0;
#defineRTE_VHOST_VDPA_DEVICE_TYPE_NET0 #defineRTE_VHOST_VDPA_DEVICE_TYPE_BLK1 Andget_dev_typeonly returnRTE_VHOST_VDPA_DEVICE_TYPE_BLKor
RTE_VHOST_VDPA_DEVICE_TYPE_NET.Quoted 4 lines
Quoted 3 lines
I think if when vdpa device is not there, this code runs in the original way.
Do you think use initvdpa_typeto -1 is better?
Quoted 9 lines
Quoted 4 lines
I was talking about readability, current way will be confusing. So adding
-1 will be better. The check could be if (type == blk) ... else ... as Type 0/-1 has the same handling.Also, the
vdpa_typecan be obtained from dev, so instead of passing thevdpa_typeas argument for the function, it should get fetched directly invirtio_is_ready().
As you mention, vdpa_type can be fetch directly in virtio_is_ready(). virtio_is_ready() is an internal function, and virtio_is_ready() is used only once. vdpa_type is also used vhost_user_msg_handler(), just need virtio_is_ready(). I just think I can reduce one function call, at the cost of passing a parameter to virtio_is_ready().
I do not know which way is better, what is your opinion?
Quoted 77 lines
Maxime
Quoted 75 lines
Thanks,
ChenboQuoted 71 lines
Quoted 4 lines
Also note that below check is only needed for some cases.
Yes, I got it. I will fix it in next version.
Quoted 5 lines
Thanks,
ChenboQuoted 2 lines
}Quoted 7 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net
Quoted 7 lines
lib/vhost/vhost_user.c +3
-0+if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];@@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net*dev,
Quoted 26 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev,
Quoted 11 lines
lib/vhost/vhost_user.c +7
-1if (unlock_required)vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) + goto out; + + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
&vdpa_type);Quoted 14 lines
lib/vhost/vhost_user.c +12
-0+if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev, vdpa_type)) goto out;Quoted 2 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net
Quoted 2 lines
lib/vhost/vhost_user.c +0
-0/* @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net*dev,
Quoted 24 lines
lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev,
Quoted 11 lines
lib/vhost/vhost_user.c +0
-5dev->flags |=VIRTIO_DEV_RUNNING ; } - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed toget vdpa
Quoted 12 lines
lib/vhost/vhost_user.c +0
-6dev type.\n"); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL )goto out;
--
1.8.3.1
PATCH v3 8/8 vhost: improve vDPA blk device readiness condition
In reply to Andy PeiIn the virtio blk vDPA live migration use case, for the target VM, before the live migration process, QEMU will set call fd to all queues of vDPA back-end. QEMU and vDPA back-end stand by until live migration starts. During live migration process, QEMU sets kick fd and new call fd. However, after the kick fd is set to the vDPA back-end, the vDPA back-end configures device and data path starts. The new call fd will cause some kind of "re-configuration", this kind of "re-configuration" cause IO drop. After this patch, vDPA back-end configures device after kick fd and call fd are well set and make sure no IO drops. This patch only impact virtio blk vDPA device and does not impact net device.
--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
lib/vhost/vhost_user.c +10 -3
@@ -2983,6 +2983,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type =0 ;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )
@@ -3210,9 +3211,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (vdpa_dev->ops->dev_conf (dev->vid))
Signature · 1 line
1.8.3.1
RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness condition
Quoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:17 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 8/8] vhost: improve vDPA blk device readiness condition
Quoted 12 lines
In the virtio blk vDPA live migration use case, for the target VM, before the live migration process, QEMU will set call fd to all queues of vDPA back-end. QEMU and vDPA back-end stand by until live migration starts. During live migration process, QEMU sets kick fd and new call fd. However, after the kick fd is set to the vDPA back-end, the vDPA back-end configures device and data path starts. The new call fd will cause some kind of "re-configuration", this kind of "re-configuration" cause IO drop. After this patch, vDPA back-end configures device after kick fd and call fd are well set and make sure no IO drops. This patch only impact virtio blk vDPA device and does not impact net device.
IIUC, this is an improvement for MQ to make sure all call fds are well-set, but previously it only makes sure one is well-set. If I am right, the title and commit message should describe it in better way.
Thanks,
Chenbo
Quoted 41 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)lib/vhost/vhost_user.c +10
-3@@ -2983,6 +2983,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type =0 ;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )@@ -3210,9 +3211,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (
vdpa_dev->ops->dev_conf(dev->vid))
--
1.8.3.1
RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness condition
Hi Chenbo,
Thanks for your reply, my reply is inline.
Quoted 32 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 5:36 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness conditionQuoted 10 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:17 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 8/8] vhost: improve vDPA blk device readiness conditionQuoted 12 lines
In the virtio blk vDPA live migration use case, for the target VM, before the live migration process, QEMU will set call fd to all queues of vDPA back-end. QEMU and vDPA back-end stand by until live migration starts. During live migration process, QEMU sets kick fd and new call fd. However, after the kick fd is set to the vDPA back-end, the vDPA back-end configures device and data path starts. The new call fd will cause some kind of "re-configuration", this kind of "re-configuration" cause IO drop.
After this patch, vDPA back-end configures device after kick fd and call fd are well set and make sure no IO drops.
This patch only impact virtio blk vDPA device and does not impact net device.
Quoted 7 lines
IIUC, this is an improvement for MQ to make sure all call fds are well-set, but previously it only makes sure one is well-set. If I am right, the title and commit message should describe it in better way.
Thanks,
Chenbo
In the virtio blk vDPA live migration use case, for the target VM,
before the live migration process, QEMU will set call fd to all queues of vDPA back-end. QEMU and vDPA back-end stand by until live migration starts.
During live migration process, QEMU sets kick fd to all queues one by one, and new call fd all queues one by one.
However, after the kick fd is set to the vDPA back-end,
with the original call fd, the queue is ready.
Then the vDPA back-end configures device and data path starts.
The new call fd will cause some kind of "re-configuration",
this kind of "re-configuration" cause IO drop.
After this patch, vDPA back-end configures device after kick fd and
call fd are well set and make sure no IO drops.
This patch only impact virtio blk vDPA device and does not impact net device.
Is this new commit message good?
Quoted 41 lines
Quoted 41 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-09169cf5..14ff266 100644
lib/vhost/vhost_user.c +10
-3@@ -2983,6 +2983,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type =0 ;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )@@ -3210,9 +3211,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (
vdpa_dev->ops->dev_conf(dev->vid))
--
1.8.3.1
RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness condition
Quoted 69 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 3:56 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness conditionHi Chenbo,
Thanks for your reply, my reply is inline.
Quoted 26 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 12, 2022 5:36 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness conditionQuoted 17 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Friday, September 16, 2022 2:17 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v3 8/8] vhost: improve vDPA blk device readiness conditionIn the virtio blk vDPA live migration use case, for the target VM, before the live migration process, QEMU will set call fd to all queues of vDPA back-end. QEMU and vDPA back-end stand by until live migration starts. During live migration process, QEMU sets kick fd and new call fd. However, after the kick fd is set to the vDPA back-end, the vDPA back-end configures device and data path starts. The new call fd will cause some kind of "re-configuration", this kind of
"re-configuration"
Quoted 6 lines
Quoted 5 lines
cause IO drop.
After this patch, vDPA back-end configures device after kick fd and call fd are well set and make sure no IO drops.
This patch only impact virtio blk vDPA device and does not impact net device.Quoted 7 lines
IIUC, this is an improvement for MQ to make sure all call fds are well-set, but previously it only makes sure one is well-set. If I am right, the title and commit message should describe it in better way.
Thanks,
ChenboIn the virtio blk vDPA live migration use case, for the target VM, before the live migration process, QEMU will set call fd to all queues of vDPA back-end. QEMU and vDPA back-end stand by until live migration starts.
During live migration process, QEMU sets kick fd to all queues one by one, and new call fd all queues one by one.
However, after the kick fd is set to the vDPA back-end, with the original call fd, the queue is ready.
Then the vDPA back-end configures device and data path starts.
The new call fd will cause some kind of "re-configuration", this kind of "re- configuration" cause IO drop.
After this patch, vDPA back-end configures device after kick fd and call fd are well set and make sure no IO drops.
This patch only impact virtio blk vDPA device and does not impact net device.
Quoted 1 line
Is this new commit message good?
Explanation seems duplicate with former commit
" vhost: fix virtio block vDPA live migration IO drop".
So I will make it simple here,
"To support multi-queue, configure device after call fd of all queues are set."
I will use " vhost: improve vDPA blk device configure condition" for title To reduce confusion.
Quoted 41 lines
Quoted 41 lines
Quoted 41 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-09169cf5..14ff266 100644
lib/vhost/vhost_user.c +10
-3@@ -2983,6 +2983,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type =0 ;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )@@ -3210,9 +3211,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (
vdpa_dev->ops->dev_conf(dev->vid))
--
1.8.3.1
v4 cover vdpa/ifc: add multi queue support
In reply to Andy Peiv4:
fix some commit message.
add some commets to code.
fix some code to reduce confusion.
v3:
rename device ID macro name.
fix some patch title and commit message.
delete some used marco.
rework some code logic.
v2:
fix some coding style issue.
support dynamic enable/disable queue at run time.
Andy Pei (6):
vdpa/ifc: add multi-queue support
vdpa/ifc: set max queues based on virtio spec vdpa/ifc: write queue count to MQ register vdpa/ifc: only configure enabled queue
vhost: vDPA blk device gets ready when any queue is ready
vhost: improve vDPA blk device readiness condition
Huang Wei (2):
vdpa/ifc: add new device ID for legacy network device vdpa/ifc: support dynamic enable/disable queue
drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 16 ++++- drivers/vdpa/ifc/ifcvf_vdpa.c | 142 +++++++++++++++++++++++++++++++++++------ lib/vhost/vhost_user.c | 56 ++++++++++------ 4 files changed, 315 insertions(+), 43 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v4 1/8 vdpa/ifc: add new device ID for legacy network device
From: Huang Wei <[email protected]>
Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000). Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".
--- drivers/vdpa/ifc/base/ifcvf.h | 6 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +4 -2
@@ -12,12 +12,14 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_BLK_DEVICE_ID 0x0002 +#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001 +#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1
drivers/vdpa/ifc/ifcvf_vdpa.c +10 -3
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_SUBSYS_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .vendor_id =0 ,/* sentinel */
Signature · 1 line
1.8.3.1
PATCH v4 2/8 vdpa/ifc: add multi-queue support
In reply to Andy PeiEnable VHOST_USER_PROTOCOL_F_MQ feature. Expose IFCVF_MQ_OFFSET register to enable multi-queue.
--- drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 12 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +9 -0
@@ -90,6 +90,15 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +/* For some hardware implementation, for example: +* the BAR 4 of PF is NULL, while BAR 4 of VF is not. +* This code makes sure hw->mq_cfg is a valid address. +*/ +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ; +if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );
drivers/vdpa/ifc/base/ifcvf.h +2 -0
@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0
@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;
drivers/vdpa/ifc/ifcvf_vdpa.c +1 -0
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \
Signature · 1 line
1.8.3.1
RE: [PATCH v4 2/8] vdpa/ifc: add multi-queue support
Quoted 73 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:44 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 2/8] vdpa/ifc: add multi-queue supportEnable
VHOST_USER_PROTOCOL_F_MQfeature. ExposeIFCVF_MQ_OFFSETregister to enable multi-queue.Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 12 insertions(+)drivers/vdpa/ifc/base/ifcvf.c +9
-0@@ -90,6 +90,15 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +/* For some hardware implementation, for example: +* the BAR 4 of PF is NULL, while BAR 4 of VF is not. +* This code makes sure hw->mq_cfg is a valid address. +*/ +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ; +if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );drivers/vdpa/ifc/base/ifcvf.h +2
-0@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \ --
Quoted 1 line
1.8.3.1
RE: [PATCH v4 2/8] vdpa/ifc: add multi-queue support
Hi Chenbo,
Thanks for your effort.
Quoted 85 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, October 17, 2022 2:21 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v4 2/8] vdpa/ifc: add multi-queue supportQuoted 25 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:44 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 2/8] vdpa/ifc: add multi-queue supportEnable
VHOST_USER_PROTOCOL_F_MQfeature. ExposeIFCVF_MQ_OFFSETregister to enable multi-queue.Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 12 insertions(+)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex f1e1474..81c68c0 100644drivers/vdpa/ifc/base/ifcvf.c line 98
Quoted 9 lines
drivers/vdpa/ifc/base/ifcvf.c +6
-0@@ -90,6 +90,15 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +/* For some hardware implementation, for example: +* the BAR 4 of PF is NULL, while BAR 4 of VF is not. +* This code makes sure hw->mq_cfg is a valid address. +*/ +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4].addr +
IFCVF_MQ_OFFSET;Quoted 39 lines
drivers/vdpa/ifc/base/ifcvf.c +3
-0+else + hw->mq_cfg =NULL ; +if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) { DEBUGOUT("capability incomplete\n"); diff --git
a/drivers/vdpa/ifc/base/ifcvf.hb/drivers/vdpa/ifc/base/ifcvf.hindex ef7697a..d16d9ab 100644drivers/vdpa/ifc/base/ifcvf.h +2
-0@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex b4389a0..008cf89 100644drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \ --Quoted 1 line
1.8.3.1
Reviewed-by Chenbo Xia <[email protected]>
PATCH v4 3/8 vdpa/ifc: set max queues based on virtio spec
In reply to Andy PeiSet max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues in struct virtio_blk_config
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg, get the queue pair number using num_queues and set max_queues to it.
--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +1 -1
@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33
drivers/vdpa/ifc/ifcvf_vdpa.c +18 -1
@@ -26,6 +26,12 @@#include "base/ifcvf.h" +/* +* RTE_MIN() cannot be used since braced-group within expression allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc device always has CTRL_VQ, +* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ; +DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues =MIN (IFCVF_MAX_QUEUES , queue_pairs); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal;
Signature · 1 line
1.8.3.1
RE: [PATCH v4 3/8] vdpa/ifc: set max queues based on virtio spec
Quoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:44 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 3/8] vdpa/ifc: set max queues based on virtio spec
Quoted 3 lines
Set
max_queuesaccording to virtio spec.
For virtio BLK device, setmax_queuesto the value ofnum_queuesin structvirtio_blk_config
Missing '.'
With this fixed:
Quoted 83 lines
For virtio NET device, read
num_queuesfrom structifcvf_pci_common_cfg, get the queue pair number usingnum_queuesand setmax_queuesto it.Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-)drivers/vdpa/ifc/base/ifcvf.h +1
-1@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33drivers/vdpa/ifc/ifcvf_vdpa.c +18
-2@@ -26,6 +26,12 @@#include "base/ifcvf.h" +/* +* RTE_MIN() cannot be used since braced-group within expression allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc device always has CTRL_VQ, +* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ; +DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues =MIN (IFCVF_MAX_QUEUES , queue_pairs); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal; --1.8.3.1
RE: [PATCH v4 3/8] vdpa/ifc: set max queues based on virtio spec
Hi Chenbo,
Thanks for your effort.
Quoted 17 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, October 17, 2022 2:22 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v4 3/8] vdpa/ifc: set max queues based on virtio specQuoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:44 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 3/8] vdpa/ifc: set max queues based on virtio spec
Quoted 6 lines
Quoted 3 lines
Set
max_queuesaccording to virtio spec.
For virtio BLK device, setmax_queuesto the value ofnum_queuesin structvirtio_blk_configMissing '.'
I will fix in next version.
Quoted 95 lines
With this fixed:
Reviewed-by Chenbo Xia <[email protected]>Quoted 2 lines
For virtio NET device, read
num_queuesfrom structifcvf_pci_common_cfg, get the queue pair number usingnum_queuesandset
max_queuesto it.Quoted 29 lines
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>---
drivers/vdpa/ifc/base/ifcvf.h| 2 +-drivers/vdpa/ifc/ifcvf_vdpa.c
| 19 ++++++++++++++++++-2 files changed, 19 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.hindex d16d9ab..1e133c0 100644drivers/vdpa/ifc/base/ifcvf.h +1
-1@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 008cf89..5a24204 100644drivers/vdpa/ifc/ifcvf_vdpa.c +0
-0@@ -26,6 +26,12 @@#include "base/ifcvf.h" drivers/vdpa/ifc/ifcvf_vdpa.c
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +7
-0+/* +* RTE_MIN() cannot be used since braced-group within expression +allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \ @@ -1512,6+1518,7
Quoted 17 lines
@@ struct
rte_vdpa_dev_infodev_info[] = {drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1583
Quoted 8 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +5
-0@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc device always has CTRL_VQ, +* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 )/ 2;
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1587
Quoted 4 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +3
-0+DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev- name, + queue_pairs); + internal->max_queues = MIN(IFCVF_MAX_QUEUES,
queue_pairs);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1589
Quoted 2 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +0
-0internal->features = features & ~(1ULL <<
VIRTIO_F_IOMMU_PLATFORM);Quoted 1 line
internal->features|= dev_info[IFCVF_NET].features; @@ -1609,6
Quoted 12 lines
+1623,9 @@ struct
rte_vdpa_dev_infodev_info[] = {drivers/vdpa/ifc/ifcvf_vdpa.c +3
-1internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal; --1.8.3.1
PATCH v4 4/8 vdpa/ifc: write queue count to MQ register
In reply to Andy PeiWrite queue count to IFCVF_MQ_OFFSET register
to enable multi-queue feature.
--- drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +32 -0
@@ -202,6 +202,37 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_mq (struct ifcvf_hw *hw) +{ +u8 *mq_cfg; +u8 qid; +int nr_queue =0 ; + +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue++; + } + +if (nr_queue ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + + mq_cfg = hw->mq_cfg; +if (mq_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + *(u32 *)mq_cfg = nr_queue; +RTE_LOG (INFO ,PMD ,"%d queue are enabled\n" , nr_queue); + }else { + *(u32 *)mq_cfg = nr_queue /2 ; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue /2 ); + } + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
@@ -219,6 +250,7 @@return -1 ; } +ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
Signature · 1 line
1.8.3.1
RE: [PATCH v4 4/8] vdpa/ifc: write queue count to MQ register
Quoted 39 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:45 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 4/8] vdpa/ifc: write queue count to MQ registerWrite queue count to
IFCVF_MQ_OFFSETregister
to enable multi-queue feature.Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)drivers/vdpa/ifc/base/ifcvf.c +13
-0@@ -202,6 +202,37 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_mq (struct ifcvf_hw *hw) +{ +u8 *mq_cfg; +u8 qid; +int nr_queue =0 ; + +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue++; + } +
drivers/vdpa/ifc/base/ifcvf.c line 227
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-0+if (nr_queue ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + + mq_cfg = hw->mq_cfg; +if (mq_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + *(u32 *)mq_cfg = nr_queue; +RTE_LOG (INFO ,PMD ,"%d queue are enabled\n" , nr_queue);
queue -> queues
With this fixed:
Quoted 21 lines
drivers/vdpa/ifc/base/ifcvf.c +9
-1+ }else { + *(u32 *)mq_cfg = nr_queue /2 ; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue /2 ); + } + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {@@ -219,6 +250,7 @@return -1 ; } +ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, --1.8.3.1
RE: [PATCH v4 4/8] vdpa/ifc: write queue count to MQ register
Hi Chenbo,
Thanks for your effort.
I will send a new version to fix this.
Quoted 87 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, October 17, 2022 2:24 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v4 4/8] vdpa/ifc: write queue count to MQ registerQuoted 39 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:45 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 4/8] vdpa/ifc: write queue count to MQ registerWrite queue count to
IFCVF_MQ_OFFSETregister to enable multi-queue feature.Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex 81c68c0..60c7017 100644drivers/vdpa/ifc/base/ifcvf.c +13
-0@@ -202,6 +202,37 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_mq (struct ifcvf_hw *hw) +{ +u8 *mq_cfg; +u8 qid; +int nr_queue =0 ; + +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue++; + } +drivers/vdpa/ifc/base/ifcvf.c line 227
Quoted 10 lines
drivers/vdpa/ifc/base/ifcvf.c +10
-0+if (nr_queue ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + + mq_cfg = hw->mq_cfg; +if (mq_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + *(u32 *)mq_cfg = nr_queue; + RTE_LOG(INFO, PMD, "%d queue are enabled\n",
nr_queue);queue -> queues
With this fixed:
Reviewed-by Chenbo Xia <[email protected]>drivers/vdpa/ifc/base/ifcvf.c
Quoted 3 lines
drivers/vdpa/ifc/base/ifcvf.c +3
-0+ }else { + *(u32 *)mq_cfg = nr_queue /2 ; + RTE_LOG(INFO, PMD, "%d queue pairs areenabled\n",
Quoted 18 lines
drivers/vdpa/ifc/base/ifcvf.c +6
-1+ nr_queue / 2); + } + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {@@ -219,6 +250,7 @@return -1 ; } +ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, --1.8.3.1
PATCH v4 5/8 vdpa/ifc: only configure enabled queue
In reply to Andy PeiWhen configuring the hardware queue, we only configure queues which have been enabled by vhost.
--- drivers/vdpa/ifc/base/ifcvf.c | 3 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +3 -0
@@ -252,6 +252,9 @@ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);
drivers/vdpa/ifc/ifcvf_vdpa.c +14 -2
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) !=0 ) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; }
Signature · 1 line
1.8.3.1
RE: [PATCH v4 5/8] vdpa/ifc: only configure enabled queue
Quoted 87 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:45 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 5/8] vdpa/ifc: only configure enabled queueWhen configuring the hardware queue, we only configure queues which have been enabled by vhost.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>Reviewed-by Chenbo Xia <[email protected]>--- drivers/vdpa/ifc/base/ifcvf.c | 3 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-)drivers/vdpa/ifc/base/ifcvf.c +3
-0@@ -252,6 +252,9 @@ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);drivers/vdpa/ifc/ifcvf_vdpa.c +14
-3@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) != 0) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; } --
Quoted 1 line
1.8.3.1
RE: [PATCH v4 5/8] vdpa/ifc: only configure enabled queue
Hi Chenbo,
Thanks for your efforts.
Quoted 100 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, October 17, 2022 2:24 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v4 5/8] vdpa/ifc: only configure enabled queueQuoted 68 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:45 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 5/8] vdpa/ifc: only configure enabled queueWhen configuring the hardware queue, we only configure queues which have been enabled by vhost.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>Reviewed-by Chenbo Xia <[email protected]>---
drivers/vdpa/ifc/base/ifcvf.c| 3 +++drivers/vdpa/ifc/ifcvf_vdpa.c
| 16 ++++++++++++++--2 files changed, 17 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/drivers/vdpa/ifc/base/ifcvf.cindex 60c7017..4d85911 100644drivers/vdpa/ifc/base/ifcvf.c +3
-0@@ -252,6 +252,9 @@ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);(quoted diff) +0
-0
b/drivers/vdpa/ifc/ifcvf_vdpa.cindex 5a24204..0c3407a 100644drivers/vdpa/ifc/ifcvf_vdpa.c +7
-0@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t)vring.kickfd << 32; @@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev = rte_vhost_get_vdpa_device(vid); list = find_internal_resource_by_vdev(vdev);@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal);drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info { · line 1086
Quoted 10 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +6
-2-if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) != 0) - DRV_LOG(NOTICE, "vDPA (%s): software relay is used.", + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) + DRV_LOG(NOTICE, "vDPA (%s): software relay isused.",
Quoted 11 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +2
-0vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device- name);return 0 ;}
--
1.8.3.1Reviewed-by Chenbo Xia <[email protected]>
PATCH v4 6/8 vdpa/ifc: support dynamic enable/disable queue
In reply to Andy PeiFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
--- drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 93 ++++++++++++++++++++++++++++++++------- 3 files changed, 184 insertions(+), 15 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +100 -0
@@ -233,6 +233,106 @@ } } +int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (i >= (int )hw->nr_vring) +return -1 ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return -1 ; + } + +ifcvf_enable_mq (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +RTE_LOG (ERR ,PMD ,"queue %d, msix vec alloc failed\n" , + i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + } + + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (i >= (int )hw->nr_vring) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); + }else { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ); + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 ); + } + hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
drivers/vdpa/ifc/base/ifcvf.h +6 -0
@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void
drivers/vdpa/ifc/ifcvf_vdpa.c +78 -15
124 lines not on this page · raw message
Signature · 1 line
1.8.3.1
RE: [PATCH v4 6/8] vdpa/ifc: support dynamic enable/disable queue
Quoted 17 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:45 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 6/8] vdpa/ifc: support dynamic enable/disable queueFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>
Quoted 8 lines
--- drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 93 ++++++++++++++++++++++++++++++++----- -- 3 files changed, 184 insertions(+), 15 deletions(-)
RE: [PATCH v4 6/8] vdpa/ifc: support dynamic enable/disable queue
Hi Chenbo,
Thanks for your efforts.
Quoted 36 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, October 17, 2022 2:26 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v4 6/8] vdpa/ifc: support dynamic enable/disable queueQuoted 17 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:45 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 6/8] vdpa/ifc: support dynamic enable/disable queueFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.Signed-off-by Huang Wei <[email protected]>Signed-off-by Andy Pei <[email protected]>Quoted 9 lines
--- drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 93 ++++++++++++++++++++++++++++++++----- -- 3 files changed, 184 insertions(+), 15 deletions(-)Reviewed-by Chenbo Xia <[email protected]>
PATCH v4 7/8 vhost: vDPA blk device gets ready when any queue is ready
In reply to Andy PeiWhen boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when any queue is ready.
--- lib/vhost/vhost_user.c | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-)
lib/vhost/vhost_user.c +30 -19
@@ -1441,9 +1441,10 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int -virtio_is_ready (struct virtio_net *dev) +virtio_is_ready (struct virtio_net *dev,uint32_t vdpa_type) {struct vhost_virtqueue *vq;uint32_t i, nr_vring = dev->nr_vring;
@@ -1454,13 +1455,16 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; - -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];
@@ -2958,7 +2962,7 @@ static int is_vring_iotlb(struct virtio_net *dev,int ret;int unlock_required =0 ;bool handled; -uint32_t vdpa_type =0 ; +uint32_t vdpa_type =-1 ;uint32_t request;uint32_t i;
@@ -3152,7 +3156,25 @@ static int is_vring_iotlb(struct virtio_net *dev,if (unlock_required)vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) + goto out; + + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev, vdpa_type)) goto out;/*
@@ -3166,20 +3188,9 @@ static int is_vring_iotlb(struct virtio_net *dev, dev->flags |=VIRTIO_DEV_RUNNING ; } - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (dev->ifname,ERR ,"failed to get vdpa dev type.\n" ); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL ) goto out;
Signature · 1 line
1.8.3.1
RE: [PATCH v4 7/8] vhost: vDPA blk device gets ready when any queue is ready
Quoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:45 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 7/8] vhost: vDPA blk device gets ready when any queue is ready
This title does not match to the code. You mean first queue?
Quoted 17 lines
When boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when any queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 49 ++++++++++++++++++++++++++++++--------------- ---- 1 file changed, 30 insertions(+), 19 deletions(-)lib/vhost/vhost_user.c +0
-0
lib/vhost/vhost_user.c line 1447
Quoted 8 lines
lib/vhost/vhost_user.c +2
-1@@ -1441,9 +1441,10 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int -virtio_is_ready (struct virtio_net *dev) +virtio_is_ready (struct virtio_net *dev,uint32_t vdpa_type)
I agree with Maxime's v3 comment. We don't need a new parameter.
Thanks,
Chenbo
Quoted 85 lines
lib/vhost/vhost_user.c +28
-18{struct vhost_virtqueue *vq;uint32_t i, nr_vring = dev->nr_vring;@@ -1454,13 +1455,16 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; - -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];@@ -2958,7 +2962,7 @@ static int is_vring_iotlb(struct virtio_net *dev,int ret;int unlock_required =0 ;bool handled; -uint32_t vdpa_type =0 ; +uint32_t vdpa_type =-1 ;uint32_t request;uint32_t i;@@ -3152,7 +3156,25 @@ static int is_vring_iotlb(struct virtio_net *dev,if (unlock_required)vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) + goto out; + + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev, vdpa_type)) goto out;/* @@ -3166,20 +3188,9 @@ static int is_vring_iotlb(struct virtio_net *dev, dev->flags |=VIRTIO_DEV_RUNNING ; } - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n"); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL )goto out;
--
1.8.3.1
RE: [PATCH v4 7/8] vhost: vDPA blk device gets ready when any queue is ready
Hi Chenbo,
Thanks for your efforts, my reply is inline.
Quoted 19 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, October 17, 2022 2:34 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v4 7/8] vhost: vDPA blk device gets ready when any queue is readyQuoted 9 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:45 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 7/8] vhost: vDPA blk device gets ready when any queue is ready
Quoted 2 lines
This title does not match to the code. You mean first queue?
Yes, I think I can make the title the first queue.
Quoted 20 lines
Quoted 20 lines
When boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when any queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 49 ++++++++++++++++++++++++++++++--------------- ---- 1 file changed, 30 insertions(+), 19 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-0cd65257..0509025 100644
lib/vhost/vhost_user.c +0
-0@@ -1441,9 +1441,10 @@ }
Quoted 9 lines
lib/vhost/vhost_user.c line 1447
Quoted 6 lines
lib/vhost/vhost_user.c +2
-1#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int -virtio_is_ready (struct virtio_net *dev) +virtio_is_ready (struct virtio_net *dev,uint32_t vdpa_type)I agree with Maxime's v3 comment. We don't need a new parameter.
OK, I will send a new version to fix this.
Quoted 90 lines
Thanks,
ChenboQuoted 6 lines
lib/vhost/vhost_user.c +0
-0{struct vhost_virtqueue *vq; uint32_t i, nr_vring = dev->nr_vring; @@ -1454,13 +1455,16 @@lib/vhost/vhost_user.c +0
-0if (!dev->nr_vring)return 0 ;lib/vhost/vhost_user.c
Quoted 10 lines
lib/vhost/vhost_user.c +5
-5-if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; - -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =
VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;Quoted 19 lines
lib/vhost/vhost_user.c +4
-1} +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];@@ -2958,7 +2962,7 @@ static int is_vring_iotlb(struct virtio_net *dev,int ret;int unlock_required =0 ;bool handled; -uint32_t vdpa_type =0 ; +uint32_t vdpa_type =-1 ;uint32_t request;uint32_t i;@@ -3152,7 +3156,25 @@ static int is_vring_iotlb(struct virtio_net *dev,if (unlock_required)lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev, · line 3165
Quoted 10 lines
lib/vhost/vhost_user.c +7
-1vhost_user_unlock_all_queue_pairs (dev); -if (ret !=0 || !virtio_is_ready (dev)) +if (ret !=0 ) + goto out; + + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
&vdpa_type);Quoted 17 lines
lib/vhost/vhost_user.c +12
-0+if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + } + +if (!virtio_is_ready (dev, vdpa_type)) goto out;/* @@ -3166,20 +3188,9 @@ static int is_vring_iotlb(struct virtio_net *dev, dev->flags |=VIRTIO_DEV_RUNNING ;lib/vhost/vhost_user.c static int is_vring_iotlb(struct virtio_net *dev, · line 3176
Quoted 10 lines
lib/vhost/vhost_user.c +0
-5} - vdpa_dev = dev->vdpa_dev;if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(dev->ifname, ERR, "failed toget vdpa
Quoted 12 lines
lib/vhost/vhost_user.c +0
-6dev type.\n"); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - }if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL )goto out;
--
1.8.3.1
PATCH v4 8/8 vhost: improve vDPA blk device configure condition
In reply to Andy PeiTo support multi-queue, configure device
after call fd of all queues are set.
--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
lib/vhost/vhost_user.c +10 -3
@@ -2965,6 +2965,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type =-1 ;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )
@@ -3191,9 +3192,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (vdpa_dev->ops->dev_conf (dev->vid))
Signature · 1 line
1.8.3.1
RE: [PATCH v4 8/8] vhost: improve vDPA blk device configure condition
Quoted 49 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:45 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 8/8] vhost: improve vDPA blk device configure conditionTo support multi-queue, configure device
after call fd of all queues are set.Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)lib/vhost/vhost_user.c +10
-3@@ -2965,6 +2965,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type =-1 ;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )@@ -3191,9 +3192,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {
Quoted 3 lines
if (
vdpa_dev->ops->dev_conf(dev->vid))
--
1.8.3.1
RE: [PATCH v4 8/8] vhost: improve vDPA blk device configure condition
Hi Chenbo,
Thanks for your efforts.
Quoted 64 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, October 17, 2022 2:35 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v4 8/8] vhost: improve vDPA blk device configure conditionQuoted 50 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Thursday, October 13, 2022 4:45 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v4 8/8] vhost: improve vDPA blk device configure conditionTo support multi-queue, configure device after call fd of all queues are set.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)lib/vhost/vhost_user.c index renamed from lib/vhost/vhost_user.c +0
-00509025..3ede83f 100644
lib/vhost/vhost_user.c +10
-3@@ -2965,6 +2965,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type =-1 ;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )@@ -3191,9 +3192,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {Quoted 3 lines
if (
vdpa_dev->ops->dev_conf(dev->vid))
--
1.8.3.1Reviewed-by Chenbo Xia <[email protected]>
v5 cover vdpa/ifc: add multi queue support
In reply to Andy Peiv5:
fix some commit message.
rework some code logic.
v4:
fix some commit message.
add some commets to code.
fix some code to reduce confusion.
v3:
rename device ID macro name.
fix some patch title and commit message.
delete some used marco.
rework some code logic.
v2:
fix some coding style issue.
support dynamic enable/disable queue at run time.
Andy Pei (6):
vdpa/ifc: add multi-queue support
vdpa/ifc: set max queues based on virtio spec vdpa/ifc: write queue count to MQ register
vdpa/ifc: only configure enabled queue
vhost: vDPA blk device gets ready when the first queue is ready
vhost: improve vDPA blk device configure condition
Huang Wei (2):
vdpa/ifc: add new device ID for legacy network device vdpa/ifc: support dynamic enable/disable queue
drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 16 ++++- drivers/vdpa/ifc/ifcvf_vdpa.c | 142 +++++++++++++++++++++++++++++++++++------ lib/vhost/vhost_user.c | 44 +++++++++++-- 4 files changed, 315 insertions(+), 31 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v5 1/8 vdpa/ifc: add new device ID for legacy network device
From: Huang Wei <[email protected]>
Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000). Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".
--- drivers/vdpa/ifc/base/ifcvf.h | 6 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +4 -2
@@ -12,12 +12,14 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_BLK_DEVICE_ID 0x0002 +#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001 +#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1
drivers/vdpa/ifc/ifcvf_vdpa.c +10 -3
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_SUBSYS_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .vendor_id =0 ,/* sentinel */
Signature · 1 line
1.8.3.1
PATCH v5 2/8 vdpa/ifc: add multi-queue support
In reply to Andy PeiEnable VHOST_USER_PROTOCOL_F_MQ feature. Expose IFCVF_MQ_OFFSET register to enable multi-queue.
--- drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 12 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +9 -0
@@ -90,6 +90,15 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +/* For some hardware implementation, for example: +* the BAR 4 of PF is NULL, while BAR 4 of VF is not. +* This code makes sure hw->mq_cfg is a valid address. +*/ +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ; +if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );
drivers/vdpa/ifc/base/ifcvf.h +2 -0
@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0
@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;
drivers/vdpa/ifc/ifcvf_vdpa.c +1 -0
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \
Signature · 1 line
1.8.3.1
PATCH v5 3/8 vdpa/ifc: set max queues based on virtio spec
In reply to Andy PeiSet max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues in struct virtio_blk_config.
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg, get the queue pair number using num_queues and set max_queues to it.
--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +1 -1
@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33
drivers/vdpa/ifc/ifcvf_vdpa.c +18 -1
@@ -26,6 +26,12 @@#include "base/ifcvf.h" +/* +* RTE_MIN() cannot be used since braced-group within expression allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc device always has CTRL_VQ, +* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ; +DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues =MIN (IFCVF_MAX_QUEUES , queue_pairs); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal;
Signature · 1 line
1.8.3.1
PATCH v5 4/8 vdpa/ifc: write queue count to MQ register
In reply to Andy PeiWrite queue count to IFCVF_MQ_OFFSET register to enable multi-queue feature.
--- drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +32 -0
@@ -202,6 +202,37 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_mq (struct ifcvf_hw *hw) +{ +u8 *mq_cfg; +u8 qid; +int nr_queue =0 ; + +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue++; + } + +if (nr_queue ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + + mq_cfg = hw->mq_cfg; +if (mq_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + *(u32 *)mq_cfg = nr_queue; +RTE_LOG (INFO ,PMD ,"%d queues are enabled\n" , nr_queue); + }else { + *(u32 *)mq_cfg = nr_queue /2 ; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue /2 ); + } + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
@@ -219,6 +250,7 @@return -1 ; } +ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
Signature · 1 line
1.8.3.1
PATCH v5 5/8 vdpa/ifc: only configure enabled queue
In reply to Andy PeiWhen configuring the hardware queue, we only configure queues which have been enabled by vhost.
--- drivers/vdpa/ifc/base/ifcvf.c | 3 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +3 -0
@@ -252,6 +252,9 @@ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);
drivers/vdpa/ifc/ifcvf_vdpa.c +14 -2
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) !=0 ) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; }
Signature · 1 line
1.8.3.1
PATCH v5 6/8 vdpa/ifc: support dynamic enable/disable queue
In reply to Andy PeiFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
--- drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 93 ++++++++++++++++++++++++++++++++------- 3 files changed, 184 insertions(+), 15 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +100 -0
@@ -233,6 +233,106 @@ } } +int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (i >= (int )hw->nr_vring) +return -1 ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return -1 ; + } + +ifcvf_enable_mq (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +RTE_LOG (ERR ,PMD ,"queue %d, msix vec alloc failed\n" , + i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + } + + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (i >= (int )hw->nr_vring) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); + }else { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ); + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 ); + } + hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
drivers/vdpa/ifc/base/ifcvf.h +6 -0
@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void
drivers/vdpa/ifc/ifcvf_vdpa.c +78 -15
124 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v5 7/8 vhost: vDPA blk device gets ready when the first queue is ready
In reply to Andy PeiWhen boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when any queue is ready.
--- lib/vhost/vhost_user.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-)
lib/vhost/vhost_user.c +27 -4
@@ -1441,11 +1441,15 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int virtio_is_ready (struct virtio_net *dev) { +struct rte_vdpa_device *vdpa_dev;struct vhost_virtqueue *vq; +uint32_t vdpa_type; +int ret =0 ;uint32_t i, nr_vring = dev->nr_vring;if (dev->flags &VIRTIO_DEV_READY )
@@ -1454,13 +1458,32 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); +return -1 ; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + }else { + vdpa_type =-1 ; + } -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];
Signature · 1 line
1.8.3.1
RE: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is ready
Quoted 51 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Monday, October 17, 2022 3:14 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is readyWhen boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when any queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-)lib/vhost/vhost_user.c +9
-2@@ -1441,11 +1441,15 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int virtio_is_ready (struct virtio_net *dev) { +struct rte_vdpa_device *vdpa_dev;struct vhost_virtqueue *vq; +uint32_t vdpa_type; +int ret =0 ;uint32_t i, nr_vring = dev->nr_vring;if (dev->flags &VIRTIO_DEV_READY )@@ -1454,13 +1458,32 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) {
lib/vhost/vhost_user.c line 1475
Quoted 10 lines
lib/vhost/vhost_user.c +10
-0+VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); +return -1 ; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + }else { + vdpa_type =-1 ; + }
Looking at it again, instead of getting the device type again and again, Should we just get and set the type into vdpa_dev when registering a vdpa device?
What do you think?
Thanks,
Chenbo
Quoted 18 lines
lib/vhost/vhost_user.c +8
-3-if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i]; --1.8.3.1
Re: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is ready
Quoted 65 lines · Chenbo
Quoted 51 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Monday, October 17, 2022 3:14 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is readyWhen boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when any queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-)lib/vhost/vhost_user.c +9
-2@@ -1441,11 +1441,15 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int virtio_is_ready (struct virtio_net *dev) { +struct rte_vdpa_device *vdpa_dev;struct vhost_virtqueue *vq; +uint32_t vdpa_type; +int ret =0 ;uint32_t i, nr_vring = dev->nr_vring;if (dev->flags &VIRTIO_DEV_READY )@@ -1454,13 +1458,32 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) {lib/vhost/vhost_user.c line 1475
Quoted 10 lines
lib/vhost/vhost_user.c +10
-0+VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); +return -1 ; + } + }else { + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + }else { + vdpa_type =-1 ; + }Looking at it again, instead of getting the device type again and again, Should we just get and set the type into
vdpa_devwhen registering a vdpa device?
Quoted 1 line
What do you think?
I think this would be preferable.
If it is to be in struct virtio_net, please take care to place it properly not to create new holes. Also, it does not have to be placed in hot cachelines.
Thanks,
Maxime
Quoted 22 lines
Thanks,
ChenboQuoted 18 lines
lib/vhost/vhost_user.c +8
-3-if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i]; --1.8.3.1
RE: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is ready
Quoted 81 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Monday, October 17, 2022 4:37 PM
To: Xia, Chenbo <[email protected]>; Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>
Subject: Re: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is readyQuoted 8 lines · Chenbo
Quoted 8 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Monday, October 17, 2022 3:14 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v5 7/8] vhost: vDPA blk device gets ready when thefirst
Quoted 3 lines
Quoted 3 lines
queue is ready
When boot from virtio blk device, seabios in QEMU only enables one
queue.
Quoted 45 lines
Quoted 45 lines
To work in this scenario, vDPA BLK device back-end configure device when any queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-)lib/vhost/vhost_user.c +14
-2@@ -1441,11 +1441,15 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int virtio_is_ready (struct virtio_net *dev) { +struct rte_vdpa_device *vdpa_dev;struct vhost_virtqueue *vq; +uint32_t vdpa_type; +int ret =0 ;uint32_t i, nr_vring = dev->nr_vring;if (dev->flags &VIRTIO_DEV_READY )@@ -1454,13 +1458,32 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); +return -1 ; + } + }else {Quoted 8 lines
lib/vhost/vhost_user.c line 1475
Quoted 5 lines
lib/vhost/vhost_user.c +5
-0+ vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + }else { + vdpa_type =-1 ; + }Looking at it again, instead of getting the device type again and again, Should we just get and set the type into
vdpa_devwhen registering avdpa device?
Quoted 1 line
Quoted 7 lines
Quoted 1 line
What do you think?
I think this would be preferable.
If it is to be in struct
virtio_net, please take care to place it properly not to create new holes. Also, it does not have to be placed in hot cachelines.
I am thinking struct rte_vdpa_device. Because if no vdpa device, there will be no vdpa device type. What do you think?
Thanks,
Chenbo
Quoted 25 lines
Thanks,
MaximeQuoted 22 lines
Thanks,
ChenboQuoted 18 lines
lib/vhost/vhost_user.c +8
-3-if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i]; --1.8.3.1
Re: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is ready
Quoted 83 lines · Chenbo
Quoted 83 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Monday, October 17, 2022 4:37 PM
To: Xia, Chenbo <[email protected]>; Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>
Subject: Re: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is readyQuoted 8 lines · Chenbo
Quoted 8 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Monday, October 17, 2022 3:14 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v5 7/8] vhost: vDPA blk device gets ready when thefirst
Quoted 3 lines
Quoted 3 lines
queue is ready
When boot from virtio blk device, seabios in QEMU only enables one
queue.
Quoted 45 lines
Quoted 45 lines
To work in this scenario, vDPA BLK device back-end configure device when any queue is ready.
Signed-off-by Andy Pei <[email protected]>Signed-off-by Huang Wei <[email protected]>--- lib/vhost/vhost_user.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-)lib/vhost/vhost_user.c +14
-2@@ -1441,11 +1441,15 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int virtio_is_ready (struct virtio_net *dev) { +struct rte_vdpa_device *vdpa_dev;struct vhost_virtqueue *vq; +uint32_t vdpa_type; +int ret =0 ;uint32_t i, nr_vring = dev->nr_vring;if (dev->flags &VIRTIO_DEV_READY )@@ -1454,13 +1458,32 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) { +if (vdpa_dev->ops->get_dev_type) { + ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (dev->ifname,ERR , +"failed to get vdpa dev type.\n" ); +return -1 ; + } + }else {Quoted 8 lines
lib/vhost/vhost_user.c line 1475
Quoted 5 lines
lib/vhost/vhost_user.c +5
-0+ vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + }else { + vdpa_type =-1 ; + }Looking at it again, instead of getting the device type again and again, Should we just get and set the type into
vdpa_devwhen registering avdpa device?
Quoted 2 lines
What do you think?
Quoted 8 lines
Quoted 5 lines
I think this would be preferable.
If it is to be in struct
virtio_net, please take care to place it properly not to create new holes. Also, it does not have to be placed in hot cachelines.I am thinking struct
rte_vdpa_device. Because if no vdpa device, there will be no vdpa device type. What do you think?
Just looked at the code and indeed, struct rte_vdpa_device is the right place.
Thanks,
Maxime
Quoted 29 lines
Thanks,
ChenboQuoted 25 lines
Thanks,
MaximeQuoted 22 lines
Thanks,
ChenboQuoted 18 lines
lib/vhost/vhost_user.c +8
-3-if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i]; --1.8.3.1
PATCH v5 8/8 vhost: improve vDPA blk device configure condition
In reply to Andy PeiTo support multi-queue, configure device after call fd of all queues are set.
--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
lib/vhost/vhost_user.c +10 -3
@@ -2984,6 +2984,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type =0 ;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )
@@ -3203,9 +3204,15 @@ static int is_vring_iotlb(struct virtio_net *dev, }else { vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; } -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (vdpa_dev->ops->dev_conf (dev->vid))
Signature · 1 line
1.8.3.1
v6 cover vdpa/ifc: add multi queue support
In reply to Andy Peiv6:
Add vdpa_device_type to rte_vdpa_device to store vDPA device type.
v5:
fix some commit message.
rework some code logic.
v4:
fix some commit message.
add some commets to code.
fix some code to reduce confusion.
v3:
rename device ID macro name.
fix some patch title and commit message.
delete some used marco.
rework some code logic.
v2:
fix some coding style issue.
support dynamic enable/disable queue at run time.
Andy Pei (6):
vdpa/ifc: add multi-queue support
vdpa/ifc: set max queues based on virtio spec vdpa/ifc: write queue count to MQ register
vdpa/ifc: only configure enabled queue
vhost: vDPA blk device gets ready when the first queue is ready
vhost: improve vDPA blk device configure condition
Huang Wei (2):
vdpa/ifc: add new device ID for legacy network device vdpa/ifc: support dynamic enable/disable queue
drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 16 ++++- drivers/vdpa/ifc/ifcvf_vdpa.c | 142 +++++++++++++++++++++++++++++++++++------ lib/vhost/vdpa_driver.h | 2 + lib/vhost/vhost_user.c | 62 +++++++++++++----- 5 files changed, 325 insertions(+), 41 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v6 1/8 vdpa/ifc: add new device ID for legacy network device
From: Huang Wei <[email protected]>
Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000). Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".
--- drivers/vdpa/ifc/base/ifcvf.h | 6 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +4 -2
@@ -12,12 +12,14 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_BLK_DEVICE_ID 0x0002 +#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001 +#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1
drivers/vdpa/ifc/ifcvf_vdpa.c +10 -3
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_SUBSYS_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .vendor_id =0 ,/* sentinel */
Signature · 1 line
1.8.3.1
PATCH v6 2/8 vdpa/ifc: add multi-queue support
In reply to Andy PeiEnable VHOST_USER_PROTOCOL_F_MQ feature. Expose IFCVF_MQ_OFFSET register to enable multi-queue.
--- drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 12 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +9 -0
@@ -90,6 +90,15 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +/* For some hardware implementation, for example: +* the BAR 4 of PF is NULL, while BAR 4 of VF is not. +* This code makes sure hw->mq_cfg is a valid address. +*/ +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ; +if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );
drivers/vdpa/ifc/base/ifcvf.h +2 -0
@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0
@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;
drivers/vdpa/ifc/ifcvf_vdpa.c +1 -0
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \
Signature · 1 line
1.8.3.1
PATCH v6 3/8 vdpa/ifc: set max queues based on virtio spec
In reply to Andy PeiSet max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues in struct virtio_blk_config.
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg, get the queue pair number using num_queues and set max_queues to it.
--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +1 -1
@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33
drivers/vdpa/ifc/ifcvf_vdpa.c +18 -1
@@ -26,6 +26,12 @@#include "base/ifcvf.h" +/* +* RTE_MIN() cannot be used since braced-group within expression allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc device always has CTRL_VQ, +* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ; +DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues =MIN (IFCVF_MAX_QUEUES , queue_pairs); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal;
Signature · 1 line
1.8.3.1
PATCH v6 4/8 vdpa/ifc: write queue count to MQ register
In reply to Andy PeiWrite queue count to IFCVF_MQ_OFFSET register to enable multi-queue feature.
--- drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +32 -0
@@ -202,6 +202,37 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_mq (struct ifcvf_hw *hw) +{ +u8 *mq_cfg; +u8 qid; +int nr_queue =0 ; + +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue++; + } + +if (nr_queue ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + + mq_cfg = hw->mq_cfg; +if (mq_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + *(u32 *)mq_cfg = nr_queue; +RTE_LOG (INFO ,PMD ,"%d queues are enabled\n" , nr_queue); + }else { + *(u32 *)mq_cfg = nr_queue /2 ; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue /2 ); + } + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
@@ -219,6 +250,7 @@return -1 ; } +ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
Signature · 1 line
1.8.3.1
PATCH v6 5/8 vdpa/ifc: only configure enabled queue
In reply to Andy PeiWhen configuring the hardware queue, we only configure queues which have been enabled by vhost.
--- drivers/vdpa/ifc/base/ifcvf.c | 3 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +3 -0
@@ -252,6 +252,9 @@ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);
drivers/vdpa/ifc/ifcvf_vdpa.c +14 -2
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) !=0 ) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; }
Signature · 1 line
1.8.3.1
PATCH v6 6/8 vdpa/ifc: support dynamic enable/disable queue
In reply to Andy PeiFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
--- drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 93 ++++++++++++++++++++++++++++++++------- 3 files changed, 184 insertions(+), 15 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +100 -0
@@ -233,6 +233,106 @@ } } +int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (i >= (int )hw->nr_vring) +return -1 ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return -1 ; + } + +ifcvf_enable_mq (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +RTE_LOG (ERR ,PMD ,"queue %d, msix vec alloc failed\n" , + i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + } + + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (i >= (int )hw->nr_vring) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); + }else { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ); + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 ); + } + hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
drivers/vdpa/ifc/base/ifcvf.h +6 -0
@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void
drivers/vdpa/ifc/ifcvf_vdpa.c +78 -15
124 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v6 7/8 vhost: vDPA blk device gets ready when the first queue is ready
In reply to Andy PeiWhen boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when any queue is ready.
Add vdpa_device_type to rte_vdpa_device to store vDPA device type.
--- lib/vhost/vdpa_driver.h | 2 ++ lib/vhost/vhost_user.c | 55 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 17 deletions(-)
lib/vhost/vdpa_driver.h +2 -0
@@ -92,6 +92,8 @@ struct rte_vdpa_device {struct rte_device *device;/** vdpa device operations */ struct rte_vdpa_dev_ops *ops; +/** vdpa device type: net, blk... */ +uint32_t vdpa_device_type; };/**
lib/vhost/vhost_user.c +38 -17
96 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v6 8/8 vhost: improve vDPA blk device configure condition
In reply to Andy PeiTo support multi-queue, configure device after call fd of all queues are set.
--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
lib/vhost/vhost_user.c +10 -3
@@ -2973,6 +2973,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type =-1 ;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )
@@ -3201,9 +3202,15 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (vdpa_dev->ops->dev_conf (dev->vid))
Signature · 1 line
1.8.3.1
v7 cover vdpa/ifc: add multi queue support
In reply to Andy Peiv7:
Fill vdpa_device_type in vdpa device registration.
v6:
Add vdpa_device_type to rte_vdpa_device to store vDPA device type.
v5:
fix some commit message.
rework some code logic.
v4:
fix some commit message.
add some commets to code.
fix some code to reduce confusion.
v3:
rename device ID macro name.
fix some patch title and commit message.
delete some used marco.
rework some code logic.
v2:
fix some coding style issue.
support dynamic enable/disable queue at run time.
Andy Pei (10):
vdpa/ifc: add multi-queue support
vdpa/ifc: set max queues based on virtio spec vdpa/ifc: write queue count to MQ register
vdpa/ifc: only configure enabled queue
vdpa/ifc: change internal function name
vdpa/ifc: add internal API to get device.
vdpa/ifc: change some driver logic
vhost: add vdpa device type to rte vdpa device
vhost: vDPA blk device gets ready when the first queue is ready
vhost: improve vDPA blk device configure condition
Huang Wei (2):
vdpa/ifc: add new device ID for legacy network device vdpa/ifc: support dynamic enable/disable queue
drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 16 +++- drivers/vdpa/ifc/ifcvf_vdpa.c | 185 +++++++++++++++++++++++++++++++++++------- lib/vhost/socket.c | 15 +--- lib/vhost/vdpa.c | 17 ++++ lib/vhost/vdpa_driver.h | 2 + lib/vhost/vhost_user.c | 40 +++++---- 7 files changed, 358 insertions(+), 61 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v7 1/12 vdpa/ifc: add new device ID for legacy network device
From: Huang Wei <[email protected]>
Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000). Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".
--- drivers/vdpa/ifc/base/ifcvf.h | 6 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +4 -2
@@ -12,12 +12,14 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_BLK_DEVICE_ID 0x0002 +#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001 +#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1
drivers/vdpa/ifc/ifcvf_vdpa.c +10 -3
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_SUBSYS_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .vendor_id =0 ,/* sentinel */
Signature · 1 line
1.8.3.1
PATCH v7 2/12 vdpa/ifc: add multi-queue support
In reply to Andy PeiEnable VHOST_USER_PROTOCOL_F_MQ feature. Expose IFCVF_MQ_OFFSET register to enable multi-queue.
--- drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 12 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +9 -0
@@ -90,6 +90,15 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +/* For some hardware implementation, for example: +* the BAR 4 of PF is NULL, while BAR 4 of VF is not. +* This code makes sure hw->mq_cfg is a valid address. +*/ +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ; +if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );
drivers/vdpa/ifc/base/ifcvf.h +2 -0
@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0
@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;
drivers/vdpa/ifc/ifcvf_vdpa.c +1 -0
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \
Signature · 1 line
1.8.3.1
PATCH v7 3/12 vdpa/ifc: set max queues based on virtio spec
In reply to Andy PeiSet max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues in struct virtio_blk_config.
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg, get the queue pair number using num_queues and set max_queues to it.
--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +1 -1
@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33
drivers/vdpa/ifc/ifcvf_vdpa.c +18 -1
@@ -26,6 +26,12 @@#include "base/ifcvf.h" +/* +* RTE_MIN() cannot be used since braced-group within expression allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc device always has CTRL_VQ, +* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ; +DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues =MIN (IFCVF_MAX_QUEUES , queue_pairs); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal;
Signature · 1 line
1.8.3.1
PATCH v7 4/12 vdpa/ifc: write queue count to MQ register
In reply to Andy PeiWrite queue count to IFCVF_MQ_OFFSET register to enable multi-queue feature.
--- drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +32 -0
@@ -202,6 +202,37 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_mq (struct ifcvf_hw *hw) +{ +u8 *mq_cfg; +u8 qid; +int nr_queue =0 ; + +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue++; + } + +if (nr_queue ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + + mq_cfg = hw->mq_cfg; +if (mq_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + *(u32 *)mq_cfg = nr_queue; +RTE_LOG (INFO ,PMD ,"%d queues are enabled\n" , nr_queue); + }else { + *(u32 *)mq_cfg = nr_queue /2 ; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue /2 ); + } + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
@@ -219,6 +250,7 @@return -1 ; } +ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
Signature · 1 line
1.8.3.1
PATCH v7 5/12 vdpa/ifc: only configure enabled queue
In reply to Andy PeiWhen configuring the hardware queue, we only configure queues which have been enabled by vhost.
--- drivers/vdpa/ifc/base/ifcvf.c | 3 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +3 -0
@@ -252,6 +252,9 @@ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);
drivers/vdpa/ifc/ifcvf_vdpa.c +14 -2
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) !=0 ) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; }
Signature · 1 line
1.8.3.1
PATCH v7 6/12 vdpa/ifc: support dynamic enable/disable queue
In reply to Andy PeiFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
--- drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 93 ++++++++++++++++++++++++++++++++------- 3 files changed, 184 insertions(+), 15 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +100 -0
@@ -233,6 +233,106 @@ } } +int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (i >= (int )hw->nr_vring) +return -1 ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return -1 ; + } + +ifcvf_enable_mq (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +RTE_LOG (ERR ,PMD ,"queue %d, msix vec alloc failed\n" , + i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + } + + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (i >= (int )hw->nr_vring) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); + }else { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ); + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 ); + } + hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
drivers/vdpa/ifc/base/ifcvf.h +6 -0
@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void
drivers/vdpa/ifc/ifcvf_vdpa.c +78 -15
124 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v7 7/12 vdpa/ifc: change internal function name
In reply to Andy PeiChange internal function name "find_internal_resource_by_dev" to "find_internal_resource_by_pci_dev".
--- drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/ifcvf_vdpa.c +2 -2
@@ -121,7 +121,7 @@ struct rte_vdpa_dev_info { }static struct internal_list * -find_internal_resource_by_dev (struct rte_pci_device *pdev) +find_internal_resource_by_pci_dev (struct rte_pci_device *pdev) {int found =0 ;struct internal_list *list;
@@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ; - list =find_internal_resource_by_dev (pci_dev); + list =find_internal_resource_by_pci_dev (pci_dev);if (list ==NULL ) {DRV_LOG (ERR ,"Invalid device: %s" , pci_dev->name);return -1 ;
Signature · 1 line
1.8.3.1
PATCH v7 8/12 vdpa/ifc: add internal API to get device
In reply to Andy PeiAdd new internal API "find_internal_resource_by_rte_dev" to get device.
--- drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/ifcvf_vdpa.c +26 -2
@@ -144,6 +144,29 @@ struct rte_vdpa_dev_info {return list; } +static struct internal_list * +find_internal_resource_by_rte_dev (struct rte_device *rte_dev) +{ +int found =0 ; +struct internal_list *list; + +pthread_mutex_lock (&internal_list_lock); + + TAILQ_FOREACH(list, &internal_list, next) { +if (rte_dev == &list->internal->pdev->device) { + found =1 ; +break ; + } + } + +pthread_mutex_unlock (&internal_list_lock); + +if (!found) +return NULL ; + +return list; +} +static int ifcvf_vfio_setup (struct ifcvf_internal *internal) {
@@ -1398,10 +1421,11 @@ struct rte_vdpa_dev_info { {struct ifcvf_internal *internal;struct internal_list *list; +struct rte_device *rte_dev = vdev->device; - list =find_internal_resource_by_vdev (vdev); + list =find_internal_resource_by_rte_dev (rte_dev);if (list ==NULL ) { -DRV_LOG (ERR ,"Invalid vDPA device: %p" , vdev); +DRV_LOG (ERR ,"Invalid rte device: %p" , rte_dev);return -1 ; }
Signature · 1 line
1.8.3.1
PATCH v7 9/12 vdpa/ifc: change some driver logic
In reply to Andy PeiInsert internal list element to internal list before register vdpa device, in order to call vdpa ops during vdpa device registration.
--- drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
drivers/vdpa/ifc/ifcvf_vdpa.c +7 -4
@@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->sw_lm = sw_fallback_lm; +pthread_mutex_lock (&internal_list_lock); +TAILQ_INSERT_TAIL (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); + internal->vdev =rte_vdpa_register_device (&pci_dev->device, dev_info[internal->hw.device_type].ops);if (internal->vdev ==NULL ) {DRV_LOG (ERR ,"failed to register device %s" , pci_dev->name); +pthread_mutex_lock (&internal_list_lock); +TAILQ_REMOVE (&internal_list, list, next); +pthread_mutex_unlock (&internal_list_lock); goto error; } -pthread_mutex_lock (&internal_list_lock); -TAILQ_INSERT_TAIL (&internal_list, list, next); -pthread_mutex_unlock (&internal_list_lock); -rte_atomic32_set (&internal->started,1 );update_datapath (internal);
Signature · 1 line
1.8.3.1
PATCH v7 10/12 vhost: add vdpa device type to rte vdpa device
In reply to Andy PeiAdd vdpa_device_type to rte_vdpa_device to store device type. Call vdpa ops get_dev_type to fill vdpa_device_type when register vdpa device.
--- lib/vhost/socket.c | 15 +-------------- lib/vhost/vdpa.c | 17 +++++++++++++++++ lib/vhost/vdpa_driver.h | 2 ++ 3 files changed, 20 insertions(+), 14 deletions(-)
lib/vhost/socket.c +1 -14
@@ -627,7 +627,6 @@ struct rte_vdpa_device * {struct vhost_user_socket *vsocket;struct rte_vdpa_device *vdpa_dev; -uint32_t vdpa_type =0 ;int ret =0 ;pthread_mutex_lock (&vhost_user.mutex);
@@ -644,19 +643,7 @@ struct rte_vdpa_device * goto unlock_exit; } -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (path,ERR , -"failed to get vdpa dev type for socket file.\n" ); - ret =-1 ; - goto unlock_exit; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } - - *type = vdpa_type; + *type = vdpa_dev->vdpa_device_type; unlock_exit:pthread_mutex_unlock (&vhost_user.mutex);
lib/vhost/vdpa.c +17 -0
@@ -73,6 +73,8 @@ struct rte_vdpa_device * struct rte_vdpa_dev_ops *ops) {struct rte_vdpa_device *dev; +uint32_t vdpa_type =-1 ; +int ret =0 ;if (ops ==NULL )return NULL ;
@@ -101,6 +103,21 @@ struct rte_vdpa_device * dev->device = rte_dev; dev->ops = ops; + +if (ops->get_dev_type) { + ret = ops->get_dev_type (dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (rte_dev->name,ERR , +"Failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out_unlock; + } + }else { +/** by default, we assume vdpa device is a net device */ + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + dev->vdpa_device_type = vdpa_type; +TAILQ_INSERT_TAIL (&vdpa_device_list, dev, next); out_unlock:rte_spinlock_unlock (&vdpa_device_list_lock);
lib/vhost/vdpa_driver.h +2 -0
@@ -92,6 +92,8 @@ struct rte_vdpa_device {struct rte_device *device;/** vdpa device operations */ struct rte_vdpa_dev_ops *ops; +/** vdpa device type: net, blk... */ +uint32_t vdpa_device_type; };/**
Signature · 1 line
1.8.3.1
Re: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device
Quoted 84 lines · Andy Pei
Add
vdpa_device_typetorte_vdpa_deviceto store device type. Call vdpa opsget_dev_typeto fillvdpa_device_typewhen register vdpa device.Signed-off-by Andy Pei <[email protected]>--- lib/vhost/socket.c | 15 +-------------- lib/vhost/vdpa.c | 17 +++++++++++++++++ lib/vhost/vdpa_driver.h | 2 ++ 3 files changed, 20 insertions(+), 14 deletions(-)lib/vhost/socket.c +1
-14@@ -627,7 +627,6 @@ struct rte_vdpa_device * {struct vhost_user_socket *vsocket;struct rte_vdpa_device *vdpa_dev; -uint32_t vdpa_type =0 ;int ret =0 ;pthread_mutex_lock (&vhost_user.mutex);@@ -644,19 +643,7 @@ struct rte_vdpa_device * goto unlock_exit; } -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (path,ERR , -"failed to get vdpa dev type for socket file.\n" ); - ret =-1 ; - goto unlock_exit; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } - - *type = vdpa_type; + *type = vdpa_dev->vdpa_device_type; unlock_exit:pthread_mutex_unlock (&vhost_user.mutex);lib/vhost/vdpa.c +17
-0@@ -73,6 +73,8 @@ struct rte_vdpa_device * struct rte_vdpa_dev_ops *ops) {struct rte_vdpa_device *dev; +uint32_t vdpa_type =-1 ; +int ret =0 ;if (ops ==NULL )return NULL ;@@ -101,6 +103,21 @@ struct rte_vdpa_device * dev->device = rte_dev; dev->ops = ops; + +if (ops->get_dev_type) { + ret = ops->get_dev_type (dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (rte_dev->name,ERR , +"Failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out_unlock; + } + }else { +/** by default, we assume vdpa device is a net device */ + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + dev->vdpa_device_type = vdpa_type; +TAILQ_INSERT_TAIL (&vdpa_device_list, dev, next); out_unlock:rte_spinlock_unlock (&vdpa_device_list_lock);lib/vhost/vdpa_driver.h +0
-0
lib/vhost/vdpa_driver.h struct rte_vdpa_device { · line 96
Quoted 5 lines
lib/vhost/vdpa_driver.h +2
-0@@ -92,6 +92,8 @@ struct rte_vdpa_device {struct rte_device *device;/** vdpa device operations */ struct rte_vdpa_dev_ops *ops; +/** vdpa device type: net, blk... */ +uint32_t vdpa_device_type;
I would name it simply "type".
It is part of rte_vdpa_device struct, no need to duplicate information.
Doing that, you can use it directly in later patches using 'vdpa_dev->type'.
Maxime
Quoted 3 lines
lib/vhost/vdpa_driver.h +0
-0};/**
RE: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device
Hi Maxime,
Thanks for your reply.
I think your suggestion is good, I will send a new version to do it.
Quoted 112 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Tuesday, October 18, 2022 3:15 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>
Subject: Re: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa deviceQuoted 24 lines · Andy Pei
Add
vdpa_device_typetorte_vdpa_deviceto store device type. Call vdpa opsget_dev_typeto fillvdpa_device_typewhen register vdpa device.Signed-off-by Andy Pei <[email protected]>--- lib/vhost/socket.c | 15 +-------------- lib/vhost/vdpa.c | 17 +++++++++++++++++ lib/vhost/vdpa_driver.h | 2 ++ 3 files changed, 20 insertions(+), 14 deletions(-)lib/vhost/socket.c index renamed from lib/vhost/socket.c +0
-0608ae57..f768114 100644
lib/vhost/socket.c +0
-1@@ -627,7 +627,6 @@ struct rte_vdpa_device * {struct vhost_user_socket *vsocket;struct rte_vdpa_device *vdpa_dev; -uint32_t vdpa_type =0 ;int ret =0 ;pthread_mutex_lock (&vhost_user.mutex);lib/vhost/socket.c struct rte_vdpa_device * · line 651
Quoted 8 lines
lib/vhost/socket.c +0
-5@@ -644,19 +643,7 @@ struct rte_vdpa_device * goto unlock_exit; } -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(path, ERR, - "failed to get vdpa dev type for socketfile.\n");
Quoted 52 lines
lib/vhost/socket.c +1
-8- ret =-1 ; - goto unlock_exit; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } - - *type = vdpa_type; + *type = vdpa_dev->vdpa_device_type; unlock_exit:pthread_mutex_unlock (&vhost_user.mutex);lib/vhost/vdpa.c index renamed from lib/vhost/vdpa.c +0
-0bb82857..b487f4d 100644
lib/vhost/vdpa.c +17
-0@@ -73,6 +73,8 @@ struct rte_vdpa_device * struct rte_vdpa_dev_ops *ops) {struct rte_vdpa_device *dev; +uint32_t vdpa_type =-1 ; +int ret =0 ;if (ops ==NULL )return NULL ;@@ -101,6 +103,21 @@ struct rte_vdpa_device * dev->device = rte_dev; dev->ops = ops; + +if (ops->get_dev_type) { + ret = ops->get_dev_type (dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (rte_dev->name,ERR , +"Failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out_unlock; + } + }else { +/** by default, we assume vdpa device is a net device */ + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + dev->vdpa_device_type = vdpa_type; +TAILQ_INSERT_TAIL (&vdpa_device_list, dev, next); out_unlock:rte_spinlock_unlock (&vdpa_device_list_lock);lib/vhost/vdpa_driver.h index renamed from lib/vhost/vdpa_driver.h +0
-08b88a53..c4ec222 100644
lib/vhost/vdpa_driver.h struct rte_vdpa_device { · line 96
Quoted 5 lines
lib/vhost/vdpa_driver.h +2
-0@@ -92,6 +92,8 @@ struct rte_vdpa_device {struct rte_device *device;/** vdpa device operations */ struct rte_vdpa_dev_ops *ops; +/** vdpa device type: net, blk... */ +uint32_t vdpa_device_type;I would name it simply "type".
It is part ofrte_vdpa_devicestruct, no need to duplicate information.Doing that, you can use it directly in later patches using '
vdpa_dev->type'.Maxime
Quoted 3 lines
lib/vhost/vdpa_driver.h +0
-0};/**
RE: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device
In reply to Andy PeiQuoted 102 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 2:20 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa deviceAdd
vdpa_device_typetorte_vdpa_deviceto store device type. Call vdpa opsget_dev_typeto fillvdpa_device_typewhen register vdpa device.Signed-off-by Andy Pei <[email protected]>--- lib/vhost/socket.c | 15 +-------------- lib/vhost/vdpa.c | 17 +++++++++++++++++ lib/vhost/vdpa_driver.h | 2 ++ 3 files changed, 20 insertions(+), 14 deletions(-)lib/vhost/socket.c +1
-14@@ -627,7 +627,6 @@ struct rte_vdpa_device * {struct vhost_user_socket *vsocket;struct rte_vdpa_device *vdpa_dev; -uint32_t vdpa_type =0 ;int ret =0 ;pthread_mutex_lock (&vhost_user.mutex);@@ -644,19 +643,7 @@ struct rte_vdpa_device * goto unlock_exit; } -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (path,ERR , -"failed to get vdpa dev type for socket file.\n" ); - ret =-1 ; - goto unlock_exit; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } - - *type = vdpa_type; + *type = vdpa_dev->vdpa_device_type; unlock_exit:pthread_mutex_unlock (&vhost_user.mutex);lib/vhost/vdpa.c +17
-0@@ -73,6 +73,8 @@ struct rte_vdpa_device * struct rte_vdpa_dev_ops *ops) {struct rte_vdpa_device *dev; +uint32_t vdpa_type =-1 ; +int ret =0 ;if (ops ==NULL )return NULL ;@@ -101,6 +103,21 @@ struct rte_vdpa_device * dev->device = rte_dev; dev->ops = ops; + +if (ops->get_dev_type) { + ret = ops->get_dev_type (dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (rte_dev->name,ERR , +"Failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out_unlock; + } + }else { +/** by default, we assume vdpa device is a net device */ + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + dev->vdpa_device_type = vdpa_type; +TAILQ_INSERT_TAIL (&vdpa_device_list, dev, next); out_unlock:rte_spinlock_unlock (&vdpa_device_list_lock);lib/vhost/vdpa_driver.h +2
-1@@ -92,6 +92,8 @@ struct rte_vdpa_device {struct rte_device *device;/** vdpa device operations */ struct rte_vdpa_dev_ops *ops; +/** vdpa device type: net, blk... */ +uint32_t vdpa_device_type; };/** --
Quoted 1 line
1.8.3.1
RE: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device
Hi Chebo,
Thanks for your review.
Quoted 114 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Wednesday, October 19, 2022 5:14 PM
To: Pei, Andy <[email protected]>; [email protected]
Cc: Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: RE: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa deviceQuoted 33 lines
-----Original Message-----
From: Pei, Andy <[email protected]>
Sent: Tuesday, October 18, 2022 2:20 PM
To: [email protected]
Cc: Xia, Chenbo <[email protected]>; Xu, Rosen <[email protected]>; Huang, Wei <[email protected]>; Cao, Gang <[email protected]>; [email protected]
Subject: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa deviceAdd
vdpa_device_typetorte_vdpa_deviceto store device type. Call vdpa opsget_dev_typeto fillvdpa_device_typewhen register vdpa device.Signed-off-by Andy Pei <[email protected]>--- lib/vhost/socket.c | 15 +-------------- lib/vhost/vdpa.c | 17 +++++++++++++++++ lib/vhost/vdpa_driver.h | 2 ++ 3 files changed, 20 insertions(+), 14 deletions(-)lib/vhost/socket.c index renamed from lib/vhost/socket.c +0
-0608ae57..f768114 100644
lib/vhost/socket.c +0
-1@@ -627,7 +627,6 @@ struct rte_vdpa_device * {struct vhost_user_socket *vsocket;struct rte_vdpa_device *vdpa_dev; -uint32_t vdpa_type =0 ;int ret =0 ;pthread_mutex_lock (&vhost_user.mutex);lib/vhost/socket.c struct rte_vdpa_device * · line 651
Quoted 8 lines
lib/vhost/socket.c +0
-5@@ -644,19 +643,7 @@ struct rte_vdpa_device * goto unlock_exit; } -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { - VHOST_LOG_CONFIG(path, ERR, - "failed to get vdpa dev type for socketfile.\n");
Quoted 61 lines
lib/vhost/socket.c +1
-8- ret =-1 ; - goto unlock_exit; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } - - *type = vdpa_type; + *type = vdpa_dev->vdpa_device_type; unlock_exit:pthread_mutex_unlock (&vhost_user.mutex);lib/vhost/vdpa.c index renamed from lib/vhost/vdpa.c +0
-0bb82857..b487f4d 100644
lib/vhost/vdpa.c +17
-0@@ -73,6 +73,8 @@ struct rte_vdpa_device * struct rte_vdpa_dev_ops *ops) {struct rte_vdpa_device *dev; +uint32_t vdpa_type =-1 ; +int ret =0 ;if (ops ==NULL )return NULL ;@@ -101,6 +103,21 @@ struct rte_vdpa_device * dev->device = rte_dev; dev->ops = ops; + +if (ops->get_dev_type) { + ret = ops->get_dev_type (dev, &vdpa_type); +if (ret) { +VHOST_LOG_CONFIG (rte_dev->name,ERR , +"Failed to get vdpa dev type.\n" ); + ret =-1 ; + goto out_unlock; + } + }else { +/** by default, we assume vdpa device is a net device */ + vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; + } + dev->vdpa_device_type = vdpa_type; +TAILQ_INSERT_TAIL (&vdpa_device_list, dev, next); out_unlock:rte_spinlock_unlock (&vdpa_device_list_lock);lib/vhost/vdpa_driver.h index renamed from lib/vhost/vdpa_driver.h +0
-08b88a53..c4ec222 100644
lib/vhost/vdpa_driver.h +2
-1@@ -92,6 +92,8 @@ struct rte_vdpa_device {struct rte_device *device;/** vdpa device operations */ struct rte_vdpa_dev_ops *ops; +/** vdpa device type: net, blk... */ +uint32_t vdpa_device_type; };/** -- Quoted 1 line
1.8.3.1
Reviewed-by Chenbo Xia <[email protected]>
PATCH v7 11/12 vhost: vDPA blk device gets ready when the first queue is ready
In reply to Andy PeiWhen boot from virtio blk device, seabios in QEMU only enables one queue. To work in this scenario, vDPA BLK device back-end configure device when the first queue is ready.
--- lib/vhost/vhost_user.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-)
lib/vhost/vhost_user.c +18 -15
@@ -1441,11 +1441,14 @@ }#define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1ustatic int virtio_is_ready (struct virtio_net *dev) { +struct rte_vdpa_device *vdpa_dev;struct vhost_virtqueue *vq; +uint32_t vdpa_type;uint32_t i, nr_vring = dev->nr_vring;if (dev->flags &VIRTIO_DEV_READY )
@@ -1454,13 +1457,22 @@if (!dev->nr_vring)return 0 ; -if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) { - nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; + vdpa_dev = dev->vdpa_dev; +if (vdpa_dev) + vdpa_type = vdpa_dev->vdpa_device_type; +else + vdpa_type =-1 ; -if (dev->nr_vring < nr_vring) -return 0 ; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { + nr_vring =VIRTIO_BLK_NUM_VQS_TO_BE_READY ; + }else { +if (dev->flags &VIRTIO_DEV_BUILTIN_VIRTIO_NET ) + nr_vring =VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY ; } +if (dev->nr_vring < nr_vring) +return 0 ; +for (i =0 ; i < nr_vring; i++) { vq = dev->virtqueue[i];
@@ -2958,7 +2970,7 @@ static int is_vring_iotlb(struct virtio_net *dev,int ret;int unlock_required =0 ;bool handled; -uint32_t vdpa_type =0 ; +uint32_t vdpa_type;uint32_t request;uint32_t i;
@@ -3170,16 +3182,7 @@ static int is_vring_iotlb(struct virtio_net *dev,if (!vdpa_dev) goto out; -if (vdpa_dev->ops->get_dev_type) { - ret = vdpa_dev->ops->get_dev_type (vdpa_dev, &vdpa_type); -if (ret) { -VHOST_LOG_CONFIG (dev->ifname,ERR ,"failed to get vdpa dev type.\n" ); - ret =-1 ; - goto out; - } - }else { - vdpa_type =RTE_VHOST_VDPA_DEVICE_TYPE_NET ; - } + vdpa_type = vdpa_dev->vdpa_device_type;if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK && request !=VHOST_USER_SET_VRING_CALL ) goto out;
Signature · 1 line
1.8.3.1
PATCH v7 12/12 vhost: improve vDPA blk device configure condition
In reply to Andy PeiTo support multi-queue, configure device after call fd of all queues are set.
--- lib/vhost/vhost_user.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
lib/vhost/vhost_user.c +10 -3
@@ -2973,6 +2973,7 @@ static int is_vring_iotlb(struct virtio_net *dev,uint32_t vdpa_type;uint32_t request;uint32_t i; +uint16_t blk_call_fd; dev =get_device (vid);if (dev ==NULL )
@@ -3183,9 +3184,15 @@ static int is_vring_iotlb(struct virtio_net *dev, goto out; vdpa_type = vdpa_dev->vdpa_device_type; -if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK - && request !=VHOST_USER_SET_VRING_CALL ) - goto out; +if (vdpa_type ==RTE_VHOST_VDPA_DEVICE_TYPE_BLK ) { +if (request ==VHOST_USER_SET_VRING_CALL ) { + blk_call_fd = ctx.msg.payload.u64 &VHOST_USER_VRING_IDX_MASK ; +if (blk_call_fd != dev->nr_vring -1 ) + goto out; + }else { + goto out; + } + }if (!(dev->flags &VIRTIO_DEV_VDPA_CONFIGURED )) {if (vdpa_dev->ops->dev_conf (dev->vid))
Signature · 1 line
1.8.3.1
v8 cover vdpa/ifc: add multi queue support
In reply to Andy Peiv8:
change "vdpa_device_type" in "rte_vdpa_device" to "type".
v7:
Fill vdpa_device_type in vdpa device registration.
v6:
Add vdpa_device_type to rte_vdpa_device to store vDPA device type.
v5:
fix some commit message.
rework some code logic.
v4:
fix some commit message.
add some commets to code.
fix some code to reduce confusion.
v3:
rename device ID macro name.
fix some patch title and commit message.
delete some used marco.
rework some code logic.
v2:
fix some coding style issue.
support dynamic enable/disable queue at run time.
Andy Pei (10):
vdpa/ifc: add multi-queue support
vdpa/ifc: set max queues based on virtio spec vdpa/ifc: write queue count to MQ register
vdpa/ifc: only configure enabled queue
vdpa/ifc: change internal function name
vdpa/ifc: add internal API to get device
vdpa/ifc: change some driver logic
vhost: add type to rte vdpa device
vhost: vDPA blk device gets ready when the first queue is ready
vhost: improve vDPA blk device configure condition
Huang Wei (2):
vdpa/ifc: add new device ID for legacy network device vdpa/ifc: support dynamic enable/disable queue
drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 16 +++- drivers/vdpa/ifc/ifcvf_vdpa.c | 185 +++++++++++++++++++++++++++++++++++------- lib/vhost/socket.c | 15 +--- lib/vhost/vdpa.c | 15 ++++ lib/vhost/vdpa_driver.h | 2 + lib/vhost/vhost_user.c | 38 +++++---- 7 files changed, 354 insertions(+), 61 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v8 1/12 vdpa/ifc: add new device ID for legacy network device
From: Huang Wei <[email protected]>
Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000). Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".
--- drivers/vdpa/ifc/base/ifcvf.h | 6 ++++-- drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +4 -2
@@ -12,12 +12,14 @@#define IFCVF_BLK 1#define IFCVF_VENDOR_ID 0x1AF4 -#define IFCVF_NET_DEVICE_ID 0x1041 +#define IFCVF_NET_MODERN_DEVICE_ID 0x1041#define IFCVF_BLK_MODERN_DEVICE_ID 0x1042 +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID 0x1000#define IFCVF_BLK_TRANSITIONAL_DEVICE_ID 0x1001#define IFCVF_SUBSYS_VENDOR_ID 0x8086#define IFCVF_SUBSYS_DEVICE_ID 0x001A -#define IFCVF_BLK_DEVICE_ID 0x0002 +#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001 +#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002#define IFCVF_MAX_QUEUES 1
drivers/vdpa/ifc/ifcvf_vdpa.c +10 -3
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {static const struct rte_pci_id pci_id_ifcvf_map[] = { { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , - .device_id =IFCVF_NET_DEVICE_ID , + .device_id =IFCVF_NET_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , .subsystem_device_id =IFCVF_SUBSYS_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , + .device_id =IFCVF_NET_TRANSITIONAL_DEVICE_ID , + .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , + .subsystem_device_id =IFCVF_SUBSYS_NET_DEVICE_ID , + }, + + { .class_id =RTE_CLASS_ANY_ID , + .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_TRANSITIONAL_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .class_id =RTE_CLASS_ANY_ID , .vendor_id =IFCVF_VENDOR_ID , .device_id =IFCVF_BLK_MODERN_DEVICE_ID , .subsystem_vendor_id =IFCVF_SUBSYS_VENDOR_ID , - .subsystem_device_id =IFCVF_BLK_DEVICE_ID , + .subsystem_device_id =IFCVF_SUBSYS_BLK_DEVICE_ID , }, { .vendor_id =0 ,/* sentinel */
Signature · 1 line
1.8.3.1
PATCH v8 2/12 vdpa/ifc: add multi-queue support
In reply to Andy PeiEnable VHOST_USER_PROTOCOL_F_MQ feature. Expose IFCVF_MQ_OFFSET register to enable multi-queue.
--- drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++ drivers/vdpa/ifc/base/ifcvf.h | 2 ++ drivers/vdpa/ifc/ifcvf_vdpa.c | 1 + 3 files changed, 12 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +9 -0
@@ -90,6 +90,15 @@if (!hw->lm_cfg)WARNINGOUT ("HW support live migration not support!\n" ); +/* For some hardware implementation, for example: +* the BAR 4 of PF is NULL, while BAR 4 of VF is not. +* This code makes sure hw->mq_cfg is a valid address. +*/ +if (hw->mem_resource[4 ].addr) + hw->mq_cfg = hw->mem_resource[4 ].addr +IFCVF_MQ_OFFSET ; +else + hw->mq_cfg =NULL ; +if (hw->common_cfg ==NULL || hw->notify_base ==NULL || hw->isr ==NULL || hw->dev_cfg ==NULL ) {DEBUGOUT ("capability incomplete\n" );
drivers/vdpa/ifc/base/ifcvf.h +2 -0
@@ -50,6 +50,7 @@#define IFCVF_LM_CFG_SIZE 0x40#define IFCVF_LM_RING_STATE_OFFSET 0x20 +#define IFCVF_MQ_OFFSET 0x28#define IFCVF_LM_LOGGING_CTRL 0x0
@@ -149,6 +150,7 @@ struct ifcvf_hw {u16 *notify_base;u16 *notify_addr[IFCVF_MAX_QUEUES *2 ];u8 *lm_cfg; +u8 *mq_cfg;struct vring_info vring[IFCVF_MAX_QUEUES *2 ];u8 nr_vring;int device_type;
drivers/vdpa/ifc/ifcvf_vdpa.c +1 -0
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | \ +1ULL <<VHOST_USER_PROTOCOL_F_MQ | \ 1ULL << VHOST_USER_PROTOCOL_F_STATUS) #define VDPA_BLK_PROTOCOL_FEATURES \
Signature · 1 line
1.8.3.1
PATCH v8 3/12 vdpa/ifc: set max queues based on virtio spec
In reply to Andy PeiSet max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues in struct virtio_blk_config.
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg, get the queue pair number using num_queues and set max_queues to it.
--- drivers/vdpa/ifc/base/ifcvf.h | 2 +- drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.h +1 -1
@@ -21,7 +21,7 @@#define IFCVF_SUBSYS_NET_DEVICE_ID 0x0001#define IFCVF_SUBSYS_BLK_DEVICE_ID 0x0002 -#define IFCVF_MAX_QUEUES 1 +#define IFCVF_MAX_QUEUES 32#ifndef VIRTIO_F_IOMMU_PLATFORM #define VIRTIO_F_IOMMU_PLATFORM 33
drivers/vdpa/ifc/ifcvf_vdpa.c +18 -1
@@ -26,6 +26,12 @@#include "base/ifcvf.h" +/* +* RTE_MIN() cannot be used since braced-group within expression allowed +* only inside a function. +*/ +#define MIN (v1, v2) ((v1) < (v2) ? (v1) : (v2)) +RTE_LOG_REGISTER (ifcvf_vdpa_logtype, pmd.vdpa.ifcvf,NOTICE ); #define DRV_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {uint64_t capacity =0 ;uint8_t *byte;uint32_t i; +uint16_t queue_pairs;if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = { } internal->configured =0 ; - internal->max_queues =IFCVF_MAX_QUEUES ; features =ifcvf_get_features (&internal->hw); device_id =ifcvf_pci_get_device_type (pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {if (device_id ==VIRTIO_ID_NET ) { internal->hw.device_type =IFCVF_NET ; +/* +* ifc device always has CTRL_VQ, +* and supports VIRTIO_NET_F_CTRL_VQ feature. +*/ + queue_pairs = (internal->hw.common_cfg->num_queues -1 ) /2 ; +DRV_LOG (INFO ,"%s support %u queue pairs" , pci_dev->name, + queue_pairs); + internal->max_queues =MIN (IFCVF_MAX_QUEUES , queue_pairs); internal->features = features & ~(1ULL <<VIRTIO_F_IOMMU_PLATFORM ); internal->features |= dev_info[IFCVF_NET ].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = { internal->hw.blk_cfg->geometry.sectors);DRV_LOG (DEBUG ,"num_queues: 0x%08x" , internal->hw.blk_cfg->num_queues); + + internal->max_queues =MIN (IFCVF_MAX_QUEUES , + internal->hw.blk_cfg->num_queues); } list->internal = internal;
Signature · 1 line
1.8.3.1
PATCH v8 4/12 vdpa/ifc: write queue count to MQ register
In reply to Andy PeiWrite queue count to IFCVF_MQ_OFFSET register to enable multi-queue feature.
--- drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)
drivers/vdpa/ifc/base/ifcvf.c +32 -0
@@ -202,6 +202,37 @@IFCVF_WRITE_REG32 (val >>32 , hi); } +STATIC void +ifcvf_enable_mq (struct ifcvf_hw *hw) +{ +u8 *mq_cfg; +u8 qid; +int nr_queue =0 ; + +for (qid =0 ; qid < hw->nr_vring; qid++) { +if (!hw->vring[qid].enable) +continue ; + nr_queue++; + } + +if (nr_queue ==0 ) { +WARNINGOUT ("no enabled vring\n" ); +return ; + } + + mq_cfg = hw->mq_cfg; +if (mq_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + *(u32 *)mq_cfg = nr_queue; +RTE_LOG (INFO ,PMD ,"%d queues are enabled\n" , nr_queue); + }else { + *(u32 *)mq_cfg = nr_queue /2 ; +RTE_LOG (INFO ,PMD ,"%d queue pairs are enabled\n" , + nr_queue /2 ); + } + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
@@ -219,6 +250,7 @@return -1 ; } +ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) {IFCVF_WRITE_REG16 (i, &cfg->queue_select); io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
Signature · 1 line
1.8.3.1
PATCH v8 5/12 vdpa/ifc: only configure enabled queue
In reply to Andy PeiWhen configuring the hardware queue, we only configure queues which have been enabled by vhost.
--- drivers/vdpa/ifc/base/ifcvf.c | 3 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +3 -0
@@ -252,6 +252,9 @@ifcvf_enable_mq (hw);for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +IFCVF_WRITE_REG16 (i, &cfg->queue_select);io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi);
drivers/vdpa/ifc/ifcvf_vdpa.c +14 -2
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {rte_vhost_get_negotiated_features (vid, &hw->req_features);for (i =0 ; i < nr_vring; i++) { +if (!hw->vring[i].enable) +continue ;rte_vhost_get_vhost_vring (vid, i, &vq); gpa =hva_to_gpa (vid, (uint64_t )(uintptr_t )vq.desc);if (gpa ==0 ) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info { vring.kickfd =-1 ;for (qid =0 ; qid < q_num; qid++) { +if (!hw->vring[qid].enable) +continue ; ev.events =EPOLLIN |EPOLLPRI ;rte_vhost_get_vhost_vring (internal->vid, qid, &vring); ev.data.u64 = qid | (uint64_t )vring.kickfd <<32 ;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {struct rte_vdpa_device *vdev;struct internal_list *list;struct ifcvf_internal *internal; +struct ifcvf_hw *hw; +uint16_t i; vdev =rte_vhost_get_vdpa_device (vid); list =find_internal_resource_by_vdev (vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {rte_atomic32_set (&internal->dev_attached,1 );update_datapath (internal); -if (rte_vhost_host_notifier_ctrl (vid,RTE_VHOST_QUEUE_ALL , true) !=0 ) -DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , + hw = &internal->hw; +for (i =0 ; i < hw->nr_vring; i++) { +if (!hw->vring[i].enable) +continue ; +if (rte_vhost_host_notifier_ctrl (vid, i, true) !=0 ) +DRV_LOG (NOTICE ,"vDPA (%s): software relay is used." , vdev->device->name); + } internal->configured =1 ; +DRV_LOG (INFO ,"vDPA device %s is configured" , vdev->device->name);return 0 ; }
Signature · 1 line
1.8.3.1
PATCH v8 6/12 vdpa/ifc: support dynamic enable/disable queue
In reply to Andy PeiFrom: Huang Wei <[email protected]>
Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue. For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
--- drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++ drivers/vdpa/ifc/base/ifcvf.h | 6 +++ drivers/vdpa/ifc/ifcvf_vdpa.c | 93 ++++++++++++++++++++++++++++++++------- 3 files changed, 184 insertions(+), 15 deletions(-)
drivers/vdpa/ifc/base/ifcvf.c +100 -0
@@ -233,6 +233,106 @@ } } +int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u8 *lm_cfg; +u16 notify_off; +int msix_vector; + +if (i >= (int )hw->nr_vring) +return -1 ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return -1 ; + } + +ifcvf_enable_mq (hw); + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector != (i +1 )) { +IFCVF_WRITE_REG16 (i +1 , &cfg->queue_msix_vector); + msix_vector =IFCVF_READ_REG16 (&cfg->queue_msix_vector); +if (msix_vector ==IFCVF_MSI_NO_VECTOR ) { +RTE_LOG (ERR ,PMD ,"queue %d, msix vec alloc failed\n" , + i); +return -1 ; + } + } + +io_write64_twopart (hw->vring[i].desc, &cfg->queue_desc_lo, + &cfg->queue_desc_hi); +io_write64_twopart (hw->vring[i].avail, &cfg->queue_avail_lo, + &cfg->queue_avail_hi); +io_write64_twopart (hw->vring[i].used, &cfg->queue_used_lo, + &cfg->queue_used_hi); +IFCVF_WRITE_REG16 (hw->vring[i].size, &cfg->queue_size); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); +else + *(u32 *)(lm_cfg +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ) = + (u32 )hw->vring[i].last_avail_idx | + ((u32 )hw->vring[i].last_used_idx <<16 ); + } + + notify_off =IFCVF_READ_REG16 (&cfg->queue_notify_off); + hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + + notify_off * hw->notify_off_multiplier); +IFCVF_WRITE_REG16 (1 , &cfg->queue_enable); + +return 0 ; +} + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i) +{ +struct ifcvf_pci_common_cfg *cfg; +u32 ring_state; +u8 *lm_cfg; + +if (i >= (int )hw->nr_vring) +return ; + + cfg = hw->common_cfg; +if (!cfg) { +RTE_LOG (ERR ,PMD ,"common_cfg in HW is NULL.\n" ); +return ; + } + +IFCVF_WRITE_REG16 (i, &cfg->queue_select); +IFCVF_WRITE_REG16 (0 , &cfg->queue_enable); + + lm_cfg = hw->lm_cfg; +if (lm_cfg) { +if (hw->device_type ==IFCVF_BLK ) { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + i *IFCVF_LM_CFG_SIZE ); + hw->vring[i].last_avail_idx = + (u16)(ring_state &IFCVF_16_BIT_MASK ); + }else { + ring_state = *(u32 *)(lm_cfg + +IFCVF_LM_RING_STATE_OFFSET + + (i /2 ) *IFCVF_LM_CFG_SIZE + + (i %2 ) *4 ); + hw->vring[i].last_avail_idx = (u16)(ring_state >>16 ); + } + hw->vring[i].last_used_idx = (u16)(ring_state >>16 ); + } +} + STATIC intifcvf_hw_enable (struct ifcvf_hw *hw) {
drivers/vdpa/ifc/base/ifcvf.h +6 -0
@@ -164,6 +164,12 @@ struct ifcvf_hw { ifcvf_get_features(struct ifcvf_hw *hw);int +ifcvf_enable_vring_hw (struct ifcvf_hw *hw,int i); + +void +ifcvf_disable_vring_hw (struct ifcvf_hw *hw,int i); + +int ifcvf_start_hw (struct ifcvf_hw *hw);void
drivers/vdpa/ifc/ifcvf_vdpa.c +78 -15
124 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v8 7/12 vdpa/ifc: change internal function name
In reply to Andy PeiChange internal function name "find_internal_resource_by_dev" to "find_internal_resource_by_pci_dev".
--- drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/vdpa/ifc/ifcvf_vdpa.c +2 -2
@@ -121,7 +121,7 @@ struct rte_vdpa_dev_info { }static struct internal_list * -find_internal_resource_by_dev (struct rte_pci_device *pdev) +find_internal_resource_by_pci_dev (struct rte_pci_device *pdev) {int found =0 ;struct internal_list *list;
@@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ; - list =find_internal_resource_by_dev (pci_dev); + list =find_internal_resource_by_pci_dev (pci_dev);if (list ==NULL ) {DRV_LOG (ERR ,"Invalid device: %s" , pci_dev->name);return -1 ;
Signature · 1 line
1.8.3.1
Re: [PATCH v8 07/12] vdpa/ifc: change internal function name
Quoted 22 lines · Andy Pei
Change internal function name "
find_internal_resource_by_dev" to "find_internal_resource_by_pci_dev".Signed-off-by Andy Pei <[email protected]>--- drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1@@ -121,7 +121,7 @@ struct rte_vdpa_dev_info { }static struct internal_list * -find_internal_resource_by_dev (struct rte_pci_device *pdev) +find_internal_resource_by_pci_dev (struct rte_pci_device *pdev) {int found =0 ;struct internal_list *list;
drivers/vdpa/ifc/ifcvf_vdpa.c struct rte_vdpa_dev_info dev_info[] = { · line 1752
Quoted 8 lines
drivers/vdpa/ifc/ifcvf_vdpa.c +1
-1@@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {if (rte_eal_process_type () !=RTE_PROC_PRIMARY )return 0 ; - list =find_internal_resource_by_dev (pci_dev); + list =find_internal_resource_by_pci_dev (pci_dev);if (list ==NULL ) {DRV_LOG (ERR ,"Invalid device: %s" , pci_dev->name);return -1 ;
Thanks,
Maxime