Skip to contentSkip to conversation

net/ifcvf: add multiqueue configuration

PATCH In review Andy Pei 3 participants 5 messages Sep 17, 2019 – Oct 18, 2019 Network drivers

lore
Share & tools
lore

Andy Pei

author
#0 lore raw

PATCH v2 1/3 net/ifcvf: add multiqueue configuration

This is in preparation for multiqueue enabling for vDPA devices.

---
 drivers/net/ifc/base/ifcvf.c | 1 +
 drivers/net/ifc/base/ifcvf.h | 2 ++
 2 files changed, 3 insertions(+)
drivers/net/ifc/base/ifcvf.c +1 -0
index 3c0b2df..6462281 100644
@@ -82,6 +82,7 @@
 	}
 
 	hw->lm_cfg = hw->mem_resource[4].addr;
+	hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
 
 	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
 			hw->isr == NULL || hw->dev_cfg == NULL) {
drivers/net/ifc/base/ifcvf.h +2 -0
index 9be2770..a4cb1a4 100644
@@ -38,6 +38,7 @@
 
 #define IFCVF_LM_CFG_SIZE		0x40
 #define IFCVF_LM_RING_STATE_OFFSET	0x20
+#define IFCVF_MQ_OFFSET			0x28
 
 #define IFCVF_LM_LOGGING_CTRL		0x0
 
@@ -127,6 +128,7 @@ struct ifcvf_hw {
 	u16    *notify_base;
 	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
 	u8     *lm_cfg;
+	u8     *mq_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
 	u8 nr_vring;
 	struct ifcvf_pci_mem_resource mem_resource[IFCVF_PCI_MAX_RESOURCE];
Signature · 1 line
1.8.3.1
author
#1 lore raw

PATCH v2 2/3 vhost: call vDPA callback at the end of vring enable handler

vDPA's set_vring_state callback would need to know the virtqueues' enable status to configure the hardware.

---
v2:
add nr_active_vring as a parameter to ops function set_vring_state in case of callback in set_vring_state() and avoid exposing new API.

 lib/librte_vhost/rte_vdpa.h   |  4 ++--
 lib/librte_vhost/vhost_user.c | 27 +++++++++++++++++++++++++--
 2 files changed, 27 insertions(+), 4 deletions(-)
lib/librte_vhost/rte_vdpa.h +2 -2
index 9a3deb3..6e55d4d 100644
@@ -54,8 +54,8 @@ struct rte_vdpa_dev_ops {
 	int (*dev_conf)(int vid);
 	int (*dev_close)(int vid);
 
-	/** Enable/disable this vring */
-	int (*set_vring_state)(int vid, int vring, int state);
+	/** Enable/disable vring queue pairs */
+	int (*set_vring_state)(int vid, int nr_active_vring);
 
 	/** Set features when changed */
 	int (*set_features)(int vid);
lib/librte_vhost/vhost_user.c +25 -2
index 0b72648..4d7de44 100644
@@ -1325,6 +1325,25 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
 	return RTE_VHOST_MSG_RESULT_REPLY;
 }
 
+static uint16_t
+vhost_get_active_vring_num(int vid)
+{
+	struct virtio_net *dev = get_device(vid);
+	struct vhost_virtqueue *vq;
+	uint16_t qid;
+
+	if (dev == NULL)
+		return 0;
+
+	for (qid = 0; qid < dev->nr_vring; qid++) {
+		vq = dev->virtqueue[qid];
+		if (!vq->enabled)
+			break;
+	}
+
+	return qid;
+}
+
 /*
  * when virtio queues are ready to work, qemu will send us to
  * enable the virtio queue pair.
@@ -1339,6 +1358,7 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
 	int index = (int)msg->payload.state.index;
 	struct rte_vdpa_device *vdpa_dev;
 	int did = -1;
+	int nr_active_vring;
 
 	RTE_LOG(INFO, VHOST_CONFIG,
 		"set queue enable: %d to qp idx: %d\n",
@@ -1346,8 +1366,6 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
 
 	did = dev->vdpa_dev_id;
 	vdpa_dev = rte_vdpa_get_device(did);
-	if (vdpa_dev && vdpa_dev->ops->set_vring_state)
-		vdpa_dev->ops->set_vring_state(dev->vid, index, enable);
 
 	if (dev->notify_ops->vring_state_changed)
 		dev->notify_ops->vring_state_changed(dev->vid,
@@ -1359,6 +1377,11 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
 
 	dev->virtqueue[index]->enabled = enable;
 
+	if (vdpa_dev && vdpa_dev->ops->set_vring_state) {
+		nr_active_vring = vhost_get_active_vring_num(dev->vid);
+		vdpa_dev->ops->set_vring_state(dev->vid, nr_active_vring);
+	}
+
 	return RTE_VHOST_MSG_RESULT_OK;
 }
 
Signature · 1 line
1.8.3.1
#2 lore raw

Re: [dpdk-dev] [PATCH v2 2/3] vhost: call vDPA callback at the end of vring enable handler

Quoted 19 lines · Andy Pei

vDPA's set_vring_state callback would need to know the virtqueues' enable status to configure the hardware.

---
v2:
add nr_active_vring as a parameter to ops function set_vring_state in case of callback in set_vring_state() and avoid exposing new API.

 lib/librte_vhost/rte_vdpa.h   |  4 ++--
 lib/librte_vhost/vhost_user.c | 27 +++++++++++++++++++++++++--
 2 files changed, 27 insertions(+), 4 deletions(-)
lib/librte_vhost/rte_vdpa.h +0 -0
index 9a3deb3..6e55d4d 100644

lib/librte_vhost/rte_vdpa.h struct rte_vdpa_dev_ops { · line 58

Quoted 7 lines
lib/librte_vhost/rte_vdpa.h +2 -2
@@ -54,8 +54,8 @@ struct rte_vdpa_dev_ops {
 	int (*dev_conf)(int vid);
 	int (*dev_close)(int vid);
 
-	/** Enable/disable this vring */
-	int (*set_vring_state)(int vid, int vring, int state);
+	/** Enable/disable vring queue pairs */
+	int (*set_vring_state)(int vid, int nr_active_vring);

We should avoid changing the API/ABI unless we have a very good justification.

With the existing API, it should be easy to get the number of active rings by maintaining a bitmap or something similar in ifc driver.

Besides, please keep other maintainers got from get-maintainer.sh in the Cc list as well.

Thanks,
Tiwei

Quoted 65 lines
lib/librte_vhost/rte_vdpa.h +0 -0
 	/** Set features when changed */
 	int (*set_features)(int vid);
lib/librte_vhost/vhost_user.c +25 -2
index 0b72648..4d7de44 100644
@@ -1325,6 +1325,25 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
 	return RTE_VHOST_MSG_RESULT_REPLY;
 }
 
+static uint16_t
+vhost_get_active_vring_num(int vid)
+{
+	struct virtio_net *dev = get_device(vid);
+	struct vhost_virtqueue *vq;
+	uint16_t qid;
+
+	if (dev == NULL)
+		return 0;
+
+	for (qid = 0; qid < dev->nr_vring; qid++) {
+		vq = dev->virtqueue[qid];
+		if (!vq->enabled)
+			break;
+	}
+
+	return qid;
+}
+
 /*
  * when virtio queues are ready to work, qemu will send us to
  * enable the virtio queue pair.
@@ -1339,6 +1358,7 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
 	int index = (int)msg->payload.state.index;
 	struct rte_vdpa_device *vdpa_dev;
 	int did = -1;
+	int nr_active_vring;
 
 	RTE_LOG(INFO, VHOST_CONFIG,
 		"set queue enable: %d to qp idx: %d\n",
@@ -1346,8 +1366,6 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
 
 	did = dev->vdpa_dev_id;
 	vdpa_dev = rte_vdpa_get_device(did);
-	if (vdpa_dev && vdpa_dev->ops->set_vring_state)
-		vdpa_dev->ops->set_vring_state(dev->vid, index, enable);
 
 	if (dev->notify_ops->vring_state_changed)
 		dev->notify_ops->vring_state_changed(dev->vid,
@@ -1359,6 +1377,11 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
 
 	dev->virtqueue[index]->enabled = enable;
 
+	if (vdpa_dev && vdpa_dev->ops->set_vring_state) {
+		nr_active_vring = vhost_get_active_vring_num(dev->vid);
+		vdpa_dev->ops->set_vring_state(dev->vid, nr_active_vring);
+	}
+
 	return RTE_VHOST_MSG_RESULT_OK;
 }
 
Signature · 1 line
1.8.3.1
#3 lore raw

Re: [dpdk-dev] [PATCH v2 2/3] vhost: call vDPA callback at the end of vring enable handler

Quoted 31 lines · Tiwei Bie
Quoted 19 lines · Andy Pei

vDPA's set_vring_state callback would need to know the virtqueues' enable status to configure the hardware.

---
v2:
add nr_active_vring as a parameter to ops function set_vring_state in case of callback in set_vring_state() and avoid exposing new API.

 lib/librte_vhost/rte_vdpa.h   |  4 ++--
 lib/librte_vhost/vhost_user.c | 27 +++++++++++++++++++++++++--
 2 files changed, 27 insertions(+), 4 deletions(-)
lib/librte_vhost/rte_vdpa.h +0 -0
index 9a3deb3..6e55d4d 100644

lib/librte_vhost/rte_vdpa.h struct rte_vdpa_dev_ops { · line 58

Quoted 7 lines
lib/librte_vhost/rte_vdpa.h +2 -2
@@ -54,8 +54,8 @@ struct rte_vdpa_dev_ops {
 	int (*dev_conf)(int vid);
 	int (*dev_close)(int vid);
 
-	/** Enable/disable this vring */
-	int (*set_vring_state)(int vid, int vring, int state);
+	/** Enable/disable vring queue pairs */
+	int (*set_vring_state)(int vid, int nr_active_vring);

We should avoid changing the API/ABI unless we have a very good justification.

Quoted 7 lines

With the existing API, it should be easy to get the number of active rings by maintaining a bitmap or something similar in ifc driver.

Besides, please keep other maintainers got from get-maintainer.sh in the Cc list as well.

updating patchset [1] as "Change Requested" based on above comment.

[1]
https://patches.dpdk.org/user/todo/dpdk/?series=6424&delegate=319&state=*

author
#4 lore raw

PATCH v2 3/3 net/ifcvf: enable mutliqueue support

In reply to Andy Pei

Enable mutliqueue support for ifcvf devices by setting the mutliqueue configuration space.

---
 drivers/net/ifc/base/ifcvf.c |  8 ++++++++
 drivers/net/ifc/base/ifcvf.h |  5 ++++-
 drivers/net/ifc/ifcvf_vdpa.c | 30 ++++++++++++++++++++++++++++--
 3 files changed, 40 insertions(+), 3 deletions(-)
drivers/net/ifc/base/ifcvf.c +8 -0
index 6462281..ac09537 100644
@@ -310,6 +310,14 @@
 	*(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_DISABLE;
 }
 
+void ifcvf_enable_multiqueue(struct ifcvf_hw *hw, u16 nr_queue_pair)
+{
+	u8 *mq_cfg;
+
+	mq_cfg = hw->mq_cfg;
+	*(u32 *)mq_cfg = nr_queue_pair;
+}
+
 void
 ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
 {
drivers/net/ifc/base/ifcvf.h +4 -1
index a4cb1a4..3eb0bf3 100644
@@ -12,7 +12,7 @@
 #define IFCVF_SUBSYS_VENDOR_ID	0x8086
 #define IFCVF_SUBSYS_DEVICE_ID	0x001A
 
-#define IFCVF_MAX_QUEUES		1
+#define IFCVF_MAX_QUEUES		32
 #define VIRTIO_F_IOMMU_PLATFORM		33
 
 /* Common configuration */
@@ -153,6 +153,9 @@ struct ifcvf_hw {
 ifcvf_disable_logging(struct ifcvf_hw *hw);
 
 void
+ifcvf_enable_multiqueue(struct ifcvf_hw *hw, u16 nr_queue_pair);
+
+void
 ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid);
 
 u8
drivers/net/ifc/ifcvf_vdpa.c +28 -2
index 8de9ef1..b3f71c4 100644
@@ -994,6 +994,30 @@ struct internal_list {
 }
 
 static int
+ifcvf_set_vring_state(int vid, int  nr_active_vring)
+{
+	int did, nr_queue_pair;
+	struct internal_list *list;
+
+	if (nr_active_vring == 0) {
+		DRV_LOG(ERR, "No enabled vring");
+		return -1;
+	}
+	nr_queue_pair = (nr_active_vring + 1) / 2;
+
+	did = rte_vhost_get_vdpa_device_id(vid);
+	list = find_internal_resource_by_did(did);
+	if (list == NULL) {
+		DRV_LOG(ERR, "Invalid device id: %d", did);
+		return -1;
+	}
+
+	ifcvf_enable_multiqueue(&list->internal->hw, nr_queue_pair);
+
+	return 0;
+}
+
+static int
 ifcvf_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 {
 	int did;
@@ -1062,7 +1086,9 @@ struct internal_list {
 		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ | \
 		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
 		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
-		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD)
+		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
+		 1ULL << VHOST_USER_PROTOCOL_F_MQ)
+
 static int
 ifcvf_get_protocol_features(int did __rte_unused, uint64_t *features)
 {
@@ -1076,7 +1102,7 @@ struct internal_list {
 	.get_protocol_features = ifcvf_get_protocol_features,
 	.dev_conf = ifcvf_dev_config,
 	.dev_close = ifcvf_dev_close,
-	.set_vring_state = NULL,
+	.set_vring_state = ifcvf_set_vring_state,
 	.set_features = ifcvf_set_features,
 	.migration_done = NULL,
 	.get_vfio_group_fd = ifcvf_get_vfio_group_fd,
Signature · 1 line
1.8.3.1