Skip to contentSkip to conversation

net: liquidio: resolve VF pci_dev on demand for FLR requests

PATCH In review Yuho Choi 2 participants 3 messages Apr 20 – Apr 21

Subsystems (3)
lore
Share & tools
lore

Yuho Choi

author
#0 lore raw

PATCH 1/1 net: liquidio: resolve VF pci_dev on demand for FLR requests

The PF SR-IOV enable path caches VF pci_dev pointers in dpiring_to_vfpcidev_lut[] by iterating with pci_get_device(). Those entries do not own a reference, because the iterator drops the previous device reference on each step. The cached pointer is then dereferenced later when handling OCTEON_VF_FLR_REQUEST.

This can leave stale VF pci_dev pointers in the lookup table and makes the FLR path rely on a PCI device object whose lifetime is not pinned.

Drop the long-lived lookup table and resolve the VF pci_dev only when an FLR request arrives. Use the PF's SR-IOV metadata to derive the VF's bus/devfn, get a referenced pci_dev for immediate use, issue the FLR, and then drop the reference.

Fixes ca6139ffc67ee ("liquidio CN23XX: sysfs VF config support")
Fixes 8c978d059224 ("liquidio CN23XX: Mailbox support")
---
 .../net/ethernet/cavium/liquidio/lio_main.c   | 27 ----------------
 .../ethernet/cavium/liquidio/octeon_device.h  |  3 --
 .../ethernet/cavium/liquidio/octeon_mailbox.c | 31 ++++++++++++++++++-
 3 files changed, 30 insertions(+), 31 deletions(-)
drivers/net/ethernet/cavium/liquidio/lio_main.c +0 -27
index c1a3df2252549..32dd9b25760e3 100644
@@ -3781,9 +3781,7 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 static int octeon_enable_sriov(struct octeon_device *oct)
 {
 	unsigned int num_vfs_alloced = oct->sriov_info.num_vfs_alloced;
-	struct pci_dev *vfdev;
 	int err;
-	u32 u;
 
 	if (OCTEON_CN23XX_PF(oct) && num_vfs_alloced) {
 		err = pci_enable_sriov(oct->pci_dev,
@@ -3796,23 +3794,6 @@ static int octeon_enable_sriov(struct octeon_device *oct)
 			return err;
 		}
 		oct->sriov_info.sriov_enabled = 1;
-
-		/* init lookup table that maps DPI ring number to VF pci_dev
-		 * struct pointer
-		 */
-		u = 0;
-		vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
-				       OCTEON_CN23XX_VF_VID, NULL);
-		while (vfdev) {
-			if (vfdev->is_virtfn &&
-			    (vfdev->physfn == oct->pci_dev)) {
-				oct->sriov_info.dpiring_to_vfpcidev_lut[u] =
-					vfdev;
-				u += oct->sriov_info.rings_per_vf;
-			}
-			vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
-					       OCTEON_CN23XX_VF_VID, vfdev);
-		}
 	}
 
 	return num_vfs_alloced;
@@ -3820,8 +3801,6 @@ static int octeon_enable_sriov(struct octeon_device *oct)
 
 static int lio_pci_sriov_disable(struct octeon_device *oct)
 {
-	int u;
-
 	if (pci_vfs_assigned(oct->pci_dev)) {
 		dev_err(&oct->pci_dev->dev, "VFs are still assigned to VMs.\n");
 		return -EPERM;
@@ -3829,12 +3808,6 @@ static int lio_pci_sriov_disable(struct octeon_device *oct)
 
 	pci_disable_sriov(oct->pci_dev);
 
-	u = 0;
-	while (u < MAX_POSSIBLE_VFS) {
-		oct->sriov_info.dpiring_to_vfpcidev_lut[u] = NULL;
-		u += oct->sriov_info.rings_per_vf;
-	}
-
 	oct->sriov_info.num_vfs_alloced = 0;
 	dev_info(&oct->pci_dev->dev, "oct->pf_num:%d disabled VFs\n",
 		 oct->pf_num);
drivers/net/ethernet/cavium/liquidio/octeon_device.h +0 -3
index 19344b21f8fb9..858a0fff2cc0b 100644
@@ -390,9 +390,6 @@ struct octeon_sriov_info {
 
 	struct lio_trusted_vf	trusted_vf;
 
-	/*lookup table that maps DPI ring number to VF pci_dev struct pointer*/
-	struct pci_dev *dpiring_to_vfpcidev_lut[MAX_POSSIBLE_VFS];
-
 	u64	vf_macaddr[MAX_POSSIBLE_VFS];
 
 	u16	vf_vlantci[MAX_POSSIBLE_VFS];
drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c +30 -1
index ad685f5d0a136..b967c7928b4a7 100644
@@ -26,6 +26,29 @@
 #include "octeon_mailbox.h"
 #include "cn23xx_pf_device.h"
 
+static struct pci_dev *lio_vf_pci_dev_by_qno(struct octeon_device *oct, u32 q_no)
+{
+	int vfidx, bus, devfn;
+
+	if (!oct->sriov_info.rings_per_vf)
+		return NULL;
+
+	if (q_no % oct->sriov_info.rings_per_vf)
+		return NULL;
+
+	vfidx = q_no / oct->sriov_info.rings_per_vf;
+	if (vfidx >= oct->sriov_info.num_vfs_alloced)
+		return NULL;
+
+	bus = pci_iov_virtfn_bus(oct->pci_dev, vfidx);
+	devfn = pci_iov_virtfn_devfn(oct->pci_dev, vfidx);
+	if (bus < 0 || devfn < 0)
+		return NULL;
+
+	return pci_get_domain_bus_and_slot(pci_domain_nr(oct->pci_dev->bus),
+					   bus, devfn);
+}
+
 /**
  * octeon_mbox_read:
  * @mbox: Pointer mailbox
@@ -237,6 +260,7 @@ static int octeon_mbox_process_cmd(struct octeon_mbox *mbox,
 				   struct octeon_mbox_cmd *mbox_cmd)
 {
 	struct octeon_device *oct = mbox->oct_dev;
+	struct pci_dev *vfdev;
 
 	switch (mbox_cmd->msg.s.cmd) {
 	case OCTEON_VF_ACTIVE:
@@ -260,7 +284,12 @@ static int octeon_mbox_process_cmd(struct octeon_mbox *mbox,
 		dev_info(&oct->pci_dev->dev,
 			 "got a request for FLR from VF that owns DPI ring %u\n",
 			 mbox->q_no);
-		pcie_flr(oct->sriov_info.dpiring_to_vfpcidev_lut[mbox->q_no]);
+		vfdev = lio_vf_pci_dev_by_qno(oct, mbox->q_no);
+		if (!vfdev)
+			break;
+
+		pcie_flr(vfdev);
+		pci_dev_put(vfdev);
 		break;
 
 	case OCTEON_PF_CHANGED_VF_MACADDR:
Signature · 1 line
2.50.1 (Apple Git-155)
reviewer
#1 lore raw
Quoted 14 lines · Yuho Choi

The PF SR-IOV enable path caches VF pci_dev pointers in dpiring_to_vfpcidev_lut[] by iterating with pci_get_device(). Those entries do not own a reference, because the iterator drops the previous device reference on each step. The cached pointer is then dereferenced later when handling OCTEON_VF_FLR_REQUEST.

This can leave stale VF pci_dev pointers in the lookup table and makes the FLR path rely on a PCI device object whose lifetime is not pinned.

Drop the long-lived lookup table and resolve the VF pci_dev only when an FLR request arrives. Use the PF's SR-IOV metadata to derive the VF's bus/devfn, get a referenced pci_dev for immediate use, issue the FLR, and then drop the reference.

Quoted 9 lines
Fixes ca6139ffc67ee ("liquidio CN23XX: sysfs VF config support")
Fixes 8c978d059224 ("liquidio CN23XX: Mailbox support")

As this fixes code present in the net tree, it should be targeted at that tree, like this:

Subject: [PATCH net] ...

In this case the CI defaulted to the net-next tree.
Which might be harmless. But please keep this in mind for next time.

...

Quoted 13 lines
drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c +5 -0
index ad685f5d0a136..b967c7928b4a7 100644
@@ -26,6 +26,29 @@
 #include "octeon_mailbox.h"
 #include "cn23xx_pf_device.h"
 
+static struct pci_dev *lio_vf_pci_dev_by_qno(struct octeon_device *oct, u32 q_no)
+{
+	int vfidx, bus, devfn;
+
+	if (!oct->sriov_info.rings_per_vf)

drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c line 43

Quoted 10 lines
drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c +10 -0
+		return NULL;
+
+	if (q_no % oct->sriov_info.rings_per_vf)
+		return NULL;
+
+	vfidx = q_no / oct->sriov_info.rings_per_vf;
+	if (vfidx >= oct->sriov_info.num_vfs_alloced)
+		return NULL;
+
+	bus = pci_iov_virtfn_bus(oct->pci_dev, vfidx);

When applied against net-next this causes a linker error with x86_64 allmodconfig (at least) because pci_iov_virtfn_bus is not defined.

Quoted 11 lines
drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c +8 -0
+	devfn = pci_iov_virtfn_devfn(oct->pci_dev, vfidx);
+	if (bus < 0 || devfn < 0)
+		return NULL;
+
+	return pci_get_domain_bus_and_slot(pci_domain_nr(oct->pci_dev->bus),
+					   bus, devfn);
+}
+
 /**
  * octeon_mbox_read:
  * @mbox: Pointer mailbox
Signature · 1 line
pw-bot: changes-requested
author
#2 lore raw

Dear Simon,

Thanks for pointing out the correct tree target and the linker error on net-next.
I will rework this, targeting 'net' and ensuring the necessary functions are handled correctly.

Best regards,

Quoted 77 lines · Simon Horman
Quoted 14 lines · Yuho Choi

The PF SR-IOV enable path caches VF pci_dev pointers in dpiring_to_vfpcidev_lut[] by iterating with pci_get_device(). Those entries do not own a reference, because the iterator drops the previous device reference on each step. The cached pointer is then dereferenced later when handling OCTEON_VF_FLR_REQUEST.

This can leave stale VF pci_dev pointers in the lookup table and makes the FLR path rely on a PCI device object whose lifetime is not pinned.

Drop the long-lived lookup table and resolve the VF pci_dev only when an FLR request arrives. Use the PF's SR-IOV metadata to derive the VF's bus/devfn, get a referenced pci_dev for immediate use, issue the FLR, and then drop the reference.

Quoted 9 lines
Fixes ca6139ffc67ee ("liquidio CN23XX: sysfs VF config support")
Fixes 8c978d059224 ("liquidio CN23XX: Mailbox support")

As this fixes code present in the net tree, it should be targeted at that tree, like this:

Subject: [PATCH net] ...

In this case the CI defaulted to the net-next tree.
Which might be harmless. But please keep this in mind for next time.

...

Quoted 13 lines
drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c +5 -0
index ad685f5d0a136..b967c7928b4a7 100644
@@ -26,6 +26,29 @@
 #include "octeon_mailbox.h"
 #include "cn23xx_pf_device.h"
 
+static struct pci_dev *lio_vf_pci_dev_by_qno(struct octeon_device *oct, u32 q_no)
+{
+     int vfidx, bus, devfn;
+
+     if (!oct->sriov_info.rings_per_vf)

drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c line 43

Quoted 10 lines
drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c +10 -0
+             return NULL;
+
+     if (q_no % oct->sriov_info.rings_per_vf)
+             return NULL;
+
+     vfidx = q_no / oct->sriov_info.rings_per_vf;
+     if (vfidx >= oct->sriov_info.num_vfs_alloced)
+             return NULL;
+
+     bus = pci_iov_virtfn_bus(oct->pci_dev, vfidx);

When applied against net-next this causes a linker error with x86_64 allmodconfig (at least) because pci_iov_virtfn_bus is not defined.

drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c line 54

Quoted 11 lines
drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c +8 -0
+     devfn = pci_iov_virtfn_devfn(oct->pci_dev, vfidx);
+     if (bus < 0 || devfn < 0)
+             return NULL;
+
+     return pci_get_domain_bus_and_slot(pci_domain_nr(oct->pci_dev->bus),
+                                        bus, devfn);
+}
+
 /**
  * octeon_mbox_read:
  * @mbox: Pointer mailbox

--
pw-bot: changes-requested