Skip to contentSkip to conversation

vDPA/ifcvf: implement doorbell mapping feature

PATCH RESEND Superseded Zhu Lingshan 2 participants 6 messages May 31, 2021 – Jun 1, 2021

Subsystems (5)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v2 cover vDPA/ifcvf: implement doorbell mapping feature

This series implements doorbell mapping feature for ifcvf.

Please help review

Thanks!

Chagnes from V1:
calculate the doorbell address per vq than per device(Jason) let upper layer driver decide how to use the non page_size aligned doorbell(Jason)

Zhu Lingshan (2):
vDPA/ifcvf: record virtio notify base vDPA/ifcvf: implement doorbell mapping for ifcvf

 drivers/vdpa/ifcvf/ifcvf_base.c |  4 ++++
 drivers/vdpa/ifcvf/ifcvf_base.h |  2 ++
 drivers/vdpa/ifcvf/ifcvf_main.c | 17 +++++++++++++++++
 3 files changed, 23 insertions(+)
Signature · 1 line
2.27.0
#1 lore raw

PATCH RESEND v2 1/2 vDPA/ifcvf: record virtio notify base

This commit records virtio notify base physical addr and calculate doorbell physical address for vqs.

---
 drivers/vdpa/ifcvf/ifcvf_base.c | 4 ++++
 drivers/vdpa/ifcvf/ifcvf_base.h | 2 ++
 2 files changed, 6 insertions(+)
drivers/vdpa/ifcvf/ifcvf_base.c +4 -0
index 1a661ab45af5..6e197fe0fcf9 100644
@@ -133,6 +133,8 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
 					      &hw->notify_off_multiplier);
 			hw->notify_bar = cap.bar;
 			hw->notify_base = get_cap_addr(hw, &cap);
+			hw->notify_base_pa = pci_resource_start(pdev, cap.bar) +
+					le32_to_cpu(cap.offset);
 			IFCVF_DBG(pdev, "hw->notify_base = %p\n",
 				  hw->notify_base);
 			break;
@@ -161,6 +163,8 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
 		notify_off = ifc_ioread16(&hw->common_cfg->queue_notify_off);
 		hw->vring[i].notify_addr = hw->notify_base +
 			notify_off * hw->notify_off_multiplier;
+		hw->vring[i].notify_pa = hw->notify_base_pa +
+			notify_off * hw->notify_off_multiplier;
 	}
 
 	hw->lm_cfg = hw->base[IFCVF_LM_BAR];
drivers/vdpa/ifcvf/ifcvf_base.h +2 -0
index 0111bfdeb342..447f4ad9c0bf 100644
@@ -73,6 +73,7 @@ struct vring_info {
 	u16 last_avail_idx;
 	bool ready;
 	void __iomem *notify_addr;
+	phys_addr_t notify_pa;
 	u32 irq;
 	struct vdpa_callback cb;
 	char msix_name[256];
@@ -87,6 +88,7 @@ struct ifcvf_hw {
 	u8 notify_bar;
 	/* Notificaiton bar address */
 	void __iomem *notify_base;
+	phys_addr_t notify_base_pa;
 	u32 notify_off_multiplier;
 	u64 req_features;
 	u64 hw_features;
Signature · 1 line
2.27.0
#2 lore raw

PATCH RESEND v2 2/2 vDPA/ifcvf: implement doorbell mapping for ifcvf

In reply to Zhu Lingshan

This commit implements doorbell mapping feature for ifcvf. This feature maps the notify page to userspace, to eliminate vmexit when kick a vq.

---
 drivers/vdpa/ifcvf/ifcvf_main.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
drivers/vdpa/ifcvf/ifcvf_main.c +17 -0
index ab0ab5cf0f6e..effb0e549135 100644
@@ -413,6 +413,22 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,
 	return vf->vring[qid].irq;
 }
 
+static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
+							       u16 idx)
+{
+	struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
+	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+	struct pci_dev *pdev = adapter->pdev;
+	struct vdpa_notification_area area;
+
+	area.addr = vf->vring[idx].notify_pa;
+	area.size = PAGE_SIZE;
+	if (area.addr % PAGE_SIZE)
+		IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE aligned\n", idx);
+
+	return area;
+}
+
 /*
  * IFCVF currently does't have on-chip IOMMU, so not
  * implemented set_map()/dma_map()/dma_unmap()
@@ -440,6 +456,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
 	.get_config	= ifcvf_vdpa_get_config,
 	.set_config	= ifcvf_vdpa_set_config,
 	.set_config_cb  = ifcvf_vdpa_set_config_cb,
+	.get_vq_notification = ifcvf_get_vq_notification,
 };
 
 static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
Signature · 1 line
2.27.0
#3 lore raw

Re: [PATCH V2 RESEND 2/2] vDPA/ifcvf: implement doorbell mapping for ifcvf

Quoted 19 lines · Zhu Lingshan

This commit implements doorbell mapping feature for ifcvf. This feature maps the notify page to userspace, to eliminate vmexit when kick a vq.

---
  drivers/vdpa/ifcvf/ifcvf_main.c | 17 +++++++++++++++++
  1 file changed, 17 insertions(+)
drivers/vdpa/ifcvf/ifcvf_main.c +2 -0
index ab0ab5cf0f6e..effb0e549135 100644
@@ -413,6 +413,22 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,
  	return vf->vring[qid].irq;
  }
 
+static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
+							       u16 idx)

drivers/vdpa/ifcvf/ifcvf_main.c static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, · line 427

Quoted 10 lines
drivers/vdpa/ifcvf/ifcvf_main.c +10 -0
+{
+	struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
+	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+	struct pci_dev *pdev = adapter->pdev;
+	struct vdpa_notification_area area;
+
+	area.addr = vf->vring[idx].notify_pa;
+	area.size = PAGE_SIZE;
+	if (area.addr % PAGE_SIZE)
+		IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE aligned\n", idx);

Let's leave the decision to upper layer by: (see vp_vdpa_get_vq_notification)

area.addr = notify_pa;
area.size = notify_offset_multiplier;

Thanks

Quoted 8 lines
drivers/vdpa/ifcvf/ifcvf_main.c +4 -0
+
+	return area;
+}
+
  /*
   * IFCVF currently does't have on-chip IOMMU, so not
   * implemented set_map()/dma_map()/dma_unmap()

drivers/vdpa/ifcvf/ifcvf_main.c static const struct vdpa_config_ops ifc_vdpa_ops = { · line 462

Quoted 7 lines
drivers/vdpa/ifcvf/ifcvf_main.c +1 -0
@@ -440,6 +456,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
  	.get_config	= ifcvf_vdpa_get_config,
  	.set_config	= ifcvf_vdpa_set_config,
  	.set_config_cb  = ifcvf_vdpa_set_config_cb,
+	.get_vq_notification = ifcvf_get_vq_notification,
  };
 
  static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#4 lore raw

Re: [PATCH V2 RESEND 2/2] vDPA/ifcvf: implement doorbell mapping for ifcvf

Quoted 36 lines · Jason Wang
Quoted 20 lines · Zhu Lingshan

This commit implements doorbell mapping feature for ifcvf. This feature maps the notify page to userspace, to eliminate vmexit when kick a vq.

---
  drivers/vdpa/ifcvf/ifcvf_main.c | 17 +++++++++++++++++

  1 file changed, 17 insertions(+)
drivers/vdpa/ifcvf/ifcvf_main.c +2 -0
index ab0ab5cf0f6e..effb0e549135 100644
@@ -413,6 +413,22 @@ static int ifcvf_vdpa_get_vq_irq(struct 
 vdpa_device *vdpa_dev,
       return vf->vring[qid].irq;
   }
   +static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
+                                   u16 idx)
+{

drivers/vdpa/ifcvf/ifcvf_main.c static int ifcvf_vdpa_get_vq_irq(struct · line 428

Quoted 10 lines
drivers/vdpa/ifcvf/ifcvf_main.c +9 -0
+    struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
+    struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+    struct pci_dev *pdev = adapter->pdev;
+    struct vdpa_notification_area area;
+
+    area.addr = vf->vring[idx].notify_pa;
+    area.size = PAGE_SIZE;
+    if (area.addr % PAGE_SIZE)
+        IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE 
 aligned\n", idx);

Let's leave the decision to upper layer by: (see vp_vdpa_get_vq_notification)

Quoted 4 lines

area.addr = notify_pa;
area.size = notify_offset_multiplier;

Thanks

Hi Jason,

notify_offset_multiplier can be zero, means vqs share the same doorbell address, distinguished by qid. and in vdpa.c:

        if (vma->vm_end - vma->vm_start != notify.size) return -ENOTSUPP;

so a zero size would cause this feature failure.
mmap should work on at least a page, so if we really want "area.size = notify_offset_multiplier;" I think we should add some code in vdpa.c, like:

if(!notify.size)
    notify.size = PAGE_SIZE;

sounds good?

Thanks
Zhu Lingshan

Quoted 17 lines
Quoted 16 lines
drivers/vdpa/ifcvf/ifcvf_main.c +4 -0
+
+    return area;
+}
+
   /*

   * IFCVF currently does't have on-chip IOMMU, so not
   * implemented set_map()/dma_map()/dma_unmap()

drivers/vdpa/ifcvf/ifcvf_main.c +1 -0
@@ -440,6 +456,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
       .get_config    = ifcvf_vdpa_get_config,
       .set_config    = ifcvf_vdpa_set_config,
       .set_config_cb  = ifcvf_vdpa_set_config_cb,
+    .get_vq_notification = ifcvf_get_vq_notification,
   };
     static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
#5 lore raw

Re: [PATCH V2 RESEND 2/2] vDPA/ifcvf: implement doorbell mapping for ifcvf

Quoted 52 lines · Lingshan
Quoted 36 lines · Jason Wang
Quoted 20 lines · Zhu Lingshan

This commit implements doorbell mapping feature for ifcvf. This feature maps the notify page to userspace, to eliminate vmexit when kick a vq.

---
  drivers/vdpa/ifcvf/ifcvf_main.c | 17 +++++++++++++++++

  1 file changed, 17 insertions(+)
drivers/vdpa/ifcvf/ifcvf_main.c +2 -0
index ab0ab5cf0f6e..effb0e549135 100644
@@ -413,6 +413,22 @@ static int ifcvf_vdpa_get_vq_irq(struct 
 vdpa_device *vdpa_dev,
       return vf->vring[qid].irq;
   }
   +static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
+                                   u16 idx)
+{

drivers/vdpa/ifcvf/ifcvf_main.c static int ifcvf_vdpa_get_vq_irq(struct · line 428

Quoted 10 lines
drivers/vdpa/ifcvf/ifcvf_main.c +9 -0
+    struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
+    struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+    struct pci_dev *pdev = adapter->pdev;
+    struct vdpa_notification_area area;
+
+    area.addr = vf->vring[idx].notify_pa;
+    area.size = PAGE_SIZE;
+    if (area.addr % PAGE_SIZE)
+        IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE 
 aligned\n", idx);

Let's leave the decision to upper layer by: (see vp_vdpa_get_vq_notification)

Quoted 4 lines

area.addr = notify_pa;
area.size = notify_offset_multiplier;

Thanks

Hi Jason,

notify_offset_multiplier can be zero, means vqs share the same doorbell address, distinguished by qid. and in vdpa.c:

        if (vma->vm_end - vma->vm_start != notify.size) return -ENOTSUPP;

Quoted 8 lines

so a zero size would cause this feature failure.
mmap should work on at least a page, so if we really want "area.size = notify_offset_multiplier;" I think we should add some code in vdpa.c, like:

if(!notify.size)
    notify.size = PAGE_SIZE;

sounds good?

It's the responsibility of the driver to report a correct one. So I think it's better to tweak it as:

area.size = notify_offset_multiplier ?: PAGE_SIZE;

Thanks

Quoted 22 lines

Thanks
Zhu Lingshan

Quoted 18 lines
Quoted 17 lines
drivers/vdpa/ifcvf/ifcvf_main.c +4 -0
+
+    return area;
+}
+
   /*

   * IFCVF currently does't have on-chip IOMMU, so not
   * implemented set_map()/dma_map()/dma_unmap()

drivers/vdpa/ifcvf/ifcvf_main.c +1 -0
@@ -440,6 +456,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops 
 = {
       .get_config    = ifcvf_vdpa_get_config,
       .set_config    = ifcvf_vdpa_set_config,
       .set_config_cb  = ifcvf_vdpa_set_config_cb,
+    .get_vq_notification = ifcvf_get_vq_notification,
   };
     static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)