fw_devlink overlay fix

Overlays don't work correctly with fw_devlink. This patch series fixes it. This series is now ready for review and merging once Geert and Herve give they Tested-by.

Geert and Herve,

This patch series should hopefully fix both of your use cases [1][2][3]. Can you please check to make sure the device links created to/from the overlay devices are to/from the right ones?

Thanks,
Saravana

Saravana Kannan (2):
Revert "treewide: Fix probing of devices in DT overlays"
of: dynamic: Fix overlayed devices not probing because of fw_devlink

 drivers/base/core.c       | 76 ++++++++++++++++++++++++++++++++++-----
 drivers/bus/imx-weim.c    |  6 ----
 drivers/i2c/i2c-core-of.c |  5 ---
 drivers/of/dynamic.c      |  1 -
 drivers/of/overlay.c      | 15 ++++++++
 drivers/of/platform.c     |  5 ---
 drivers/spi/spi.c         |  5 ---
 include/linux/fwnode.h    |  1 +
 8 files changed, 83 insertions(+), 31 deletions(-)

2.44.0.683.g7961c838ac-goog

This reverts commit 1a50d9403fb90cbe4dea0ec9fd0351d2ecbd8924.

While the commit fixed fw_devlink overlay handling for one case, it broke it for another case. So revert it and redo the fix in a separate patch.

---
 drivers/bus/imx-weim.c    | 6 ------
 drivers/i2c/i2c-core-of.c | 5 -----
 drivers/of/dynamic.c      | 1 -
 drivers/of/platform.c     | 5 -----
 drivers/spi/spi.c         | 5 -----
 5 files changed, 22 deletions(-)

index 837bf9d51c6e..caaf887e0ccc 100644
@@ -331,12 +331,6 @@ static int of_weim_notify(struct notifier_block *nb, unsigned long action,
 				 "Failed to setup timing for '%pOF'\n", rd->dn);
 
 		if (!of_node_check_flag(rd->dn, OF_POPULATED)) {
-			/*
-			 * Clear the flag before adding the device so that
-			 * fw_devlink doesn't skip adding consumers to this
-			 * device.
-			 */
-			rd->dn->fwnode.flags &= ~FWNODE_FLAG_NOT_DEVICE;
 			if (!of_platform_device_create(rd->dn, NULL, &pdev->dev)) {
 				dev_err(&pdev->dev,
 					"Failed to create child device '%pOF'\n",

index a6c407d36800..a250921bbce0 100644
@@ -178,11 +178,6 @@ static int of_i2c_notify(struct notifier_block *nb, unsigned long action,
 			return NOTIFY_OK;
 		}
 
-		/*
-		 * Clear the flag before adding the device so that fw_devlink
-		 * doesn't skip adding consumers to this device.
-		 */
-		rd->dn->fwnode.flags &= ~FWNODE_FLAG_NOT_DEVICE;
 		client = of_i2c_register_device(adap, rd->dn);
 		if (IS_ERR(client)) {
 			dev_err(&adap->dev, "failed to create client for '%pOF'\n",

index 4d57a4e34105..19a1a38554f2 100644
@@ -224,7 +224,6 @@ static void __of_attach_node(struct device_node *np)
 	np->sibling = np->parent->child;
 	np->parent->child = np;
 	of_node_clear_flag(np, OF_DETACHED);
-	np->fwnode.flags |= FWNODE_FLAG_NOT_DEVICE;
 
 	raw_spin_unlock_irqrestore(&devtree_lock, flags);
 

index 389d4ea6bfc1..efd861fa254f 100644
@@ -743,11 +743,6 @@ static int of_platform_notify(struct notifier_block *nb,
 		if (of_node_check_flag(rd->dn, OF_POPULATED))
 			return NOTIFY_OK;
 
-		/*
-		 * Clear the flag before adding the device so that fw_devlink
-		 * doesn't skip adding consumers to this device.
-		 */
-		rd->dn->fwnode.flags &= ~FWNODE_FLAG_NOT_DEVICE;
 		/* pdev_parent may be NULL when no bus platform device */
 		pdev_parent = of_find_device_by_node(rd->dn->parent);
 		pdev = of_platform_device_create(rd->dn, NULL,

index ff75838c1b5d..17cd417f7681 100644
@@ -4761,11 +4761,6 @@ static int of_spi_notify(struct notifier_block *nb, unsigned long action,
 			return NOTIFY_OK;
 		}
 
-		/*
-		 * Clear the flag before adding the device so that fw_devlink
-		 * doesn't skip adding consumers to this device.
-		 */
-		rd->dn->fwnode.flags &= ~FWNODE_FLAG_NOT_DEVICE;
 		spi = of_register_spi_device(ctlr, rd->dn);
 		put_device(&ctlr->dev);
 

2.44.0.683.g7961c838ac-goog

When an overlay is applied, if the target device has already probed successfully and bound to a device, then some of the fw_devlink logic that ran when the device was probed needs to be rerun. This allows newly created dangling consumers of the overlayed device tree nodes to be moved to become consumers of the target device.

---
 drivers/base/core.c    | 76 +++++++++++++++++++++++++++++++++++++-----
 drivers/of/overlay.c   | 15 +++++++++
 include/linux/fwnode.h |  1 +
 3 files changed, 83 insertions(+), 9 deletions(-)

index 5f4e03336e68..1a646f393dd7 100644
@@ -46,6 +46,8 @@ static bool fw_devlink_drv_reg_done;
 static bool fw_devlink_best_effort;
 static struct workqueue_struct *device_link_wq;
 
+#define get_dev_from_fwnode(fwnode)	get_device((fwnode)->dev)
+
 /**
  * __fwnode_link_add - Create a link between two fwnode_handles.
  * @con: Consumer end of the link.

@@ -237,6 +239,70 @@ static void __fw_devlink_pickup_dangling_consumers(struct fwnode_handle *fwnode,
 		__fw_devlink_pickup_dangling_consumers(child, new_sup);
 }
 
+static void fw_devlink_pickup_dangling_consumers(struct device *dev)
+{
+	struct fwnode_handle *child;
+
+	mutex_lock(&fwnode_link_lock);
+	fwnode_for_each_available_child_node(dev->fwnode, child)
+		__fw_devlink_pickup_dangling_consumers(child, dev->fwnode);
+	__fw_devlink_link_to_consumers(dev);
+	mutex_unlock(&fwnode_link_lock);
+}
+
+/**
+ * fw_devlink_refresh_fwnode - Recheck the tree under this firmware node
+ * @fwnode: The fwnode under which the fwnode tree has changed
+ *
+ * This function is mainly meant to adjust the supplier/consumer dependencies
+ * after a fwnode tree overlay has occurred.
+ */
+void fw_devlink_refresh_fwnode(struct fwnode_handle *fwnode)
+{
+	struct device *dev;
+
+	/*
+	 * Find the closest ancestor fwnode that has been converted to a device
+	 * that can bind to a driver (bus device).
+	 */
+	fwnode_handle_get(fwnode);
+	do {
+		if (fwnode->flags & FWNODE_FLAG_NOT_DEVICE)
+			continue;
+
+		dev = get_dev_from_fwnode(fwnode);
+		if (!dev)
+			continue;
+
+		if (dev->bus)
+			break;
+
+		put_device(dev);
+	} while ((fwnode = fwnode_get_next_parent(fwnode)));
+
+	/*
+	 * If none of the ancestor fwnodes have (yet) been converted to a device
+	 * that can bind to a driver, there's nothing to fix up.
+	 */
+	if (!fwnode)
+		return;
+
+	WARN(device_is_bound(dev) && dev->links.status != DL_DEV_DRIVER_BOUND,
+	     "Don't multithread overlaying and probing the same device!\n");
+
+	/*
+	 * If the device has already bound to a driver, then we need to redo
+	 * some of the work that was done after the device was bound to a
+	 * driver. If the device hasn't bound to a driver, running thing too
+	 * soon would incorrectly pick up consumers that it shouldn't.
+	 */
+	if (dev->links.status == DL_DEV_DRIVER_BOUND)
+		fw_devlink_pickup_dangling_consumers(dev);
+
+	put_device(dev);
+	fwnode_handle_put(fwnode);
+}
+
 static DEFINE_MUTEX(device_links_lock);
 DEFINE_STATIC_SRCU(device_links_srcu);
 

@@ -1322,14 +1388,8 @@ void device_links_driver_bound(struct device *dev)
 	 * child firmware node.
 	 */
 	if (dev->fwnode && dev->fwnode->dev == dev) {
-		struct fwnode_handle *child;
 		fwnode_links_purge_suppliers(dev->fwnode);
-		mutex_lock(&fwnode_link_lock);
-		fwnode_for_each_available_child_node(dev->fwnode, child)
-			__fw_devlink_pickup_dangling_consumers(child,
-							       dev->fwnode);
-		__fw_devlink_link_to_consumers(dev);
-		mutex_unlock(&fwnode_link_lock);
+		fw_devlink_pickup_dangling_consumers(dev);
 	}
 	device_remove_file(dev, &dev_attr_waiting_for_supplier);
 

@@ -1888,8 +1948,6 @@ static void fw_devlink_unblock_consumers(struct device *dev)
 	device_links_write_unlock();
 }
 
-#define get_dev_from_fwnode(fwnode)	get_device((fwnode)->dev)
-
 static bool fwnode_init_without_drv(struct fwnode_handle *fwnode)
 {
 	struct device *dev;

index 2ae7e9d24a64..7b2396c53127 100644
@@ -179,6 +179,15 @@ static int overlay_notify(struct overlay_changeset *ovcs,
 	return 0;
 }
 
+static void overlay_fw_devlink_refresh(struct overlay_changeset *ovcs)
+{
+	for (int i = 0; i < ovcs->count; i++) {
+		struct device_node *np = ovcs->fragments[i].target;
+
+		fw_devlink_refresh_fwnode(of_fwnode_handle(np));
+	}
+}
+
 /*
  * The values of properties in the "/__symbols__" node are paths in
  * the ovcs->overlay_root.  When duplicating the properties, the paths

@@ -953,6 +962,12 @@ static int of_overlay_apply(struct overlay_changeset *ovcs,
 		pr_err("overlay apply changeset entry notify error %d\n", ret);
 	/* notify failure is not fatal, continue */
 
+	/*
+	 * Needs to happen after changset notify to give the listeners a chance
+	 * to finish creating all the devices they need to create.
+	 */
+	overlay_fw_devlink_refresh(ovcs);
+
 	ret_tmp = overlay_notify(ovcs, OF_OVERLAY_POST_APPLY);
 	if (ret_tmp)
 		if (!ret)

index 0d79070c5a70..95a78b87777a 100644
@@ -220,6 +220,7 @@ int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup,
 		    u8 flags);
 void fwnode_links_purge(struct fwnode_handle *fwnode);
 void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode);
+void fw_devlink_refresh_fwnode(struct fwnode_handle *fwnode);
 bool fw_devlink_is_strict(void);
 
 #endif

2.44.0.683.g7961c838ac-goog

I don't like the idea because we already have a big design issue with fwnode that is used in struct device. Ideally, fwnode has to be a node in the linked list, head of which is provided by the user (struct device, for example). When it's done, it will be easy to handle.

Have you read the comment in the struct fwnode_handle definition?

With Best Regards,
Andy Shevchenko

Or perhaps calling it get_device_from_fwnode() would make it more obvious that it is a get_device() variant?

Ack, I'll do this in my next version. Right now I'm waiting for Herve and Geert to confirm this series fixes their issues.

-Saravana

Hi Saravana,

I tested the series.

On my Microchip use case (i.e. DT overlay on a PCIe device), I observed that some driver removal were done in a wrong order. For instance, the onboard PCIe device interrupt controller (oic@e00c0120) was removed before its consumers.

I enabled debug traces in core.c and observed that many links were dropped. These links are related to pinctrl, clock, reset, interrupts, ... I have the feeling that these links should not be dropped.

Here are extracted traces:
--- 8< ---

  [    9.225355] mchp_lan966x 0000:01:00.0: enabling device (0000 -> 0002)
  [    9.279024] device: 'd0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0': device_add
  [    9.286555] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/switch@e0000000 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/oic@e00c0120
  [    9.302168] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/switch@e0000000 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/reset@e200400c
  [    9.317489] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/switch@e0000000 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064/tod_pins
  ...
  [    9.403387] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/mdio@e200413c Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/reset@e200400c
  [    9.418466] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/sfp-eth3 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064
  [    9.433263] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/sfp-eth2 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064
  ...
  [    9.495849] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/flexcom@e0040000 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/clock-controller@e00c00a8
  [    9.512111] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/flexcom@e0040000/i2c@600 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/oic@e00c0120
  [    9.527931] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/flexcom@e0040000/i2c@600 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/clock-controller@e00c00a8
  [    9.544878] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/flexcom@e0040000/i2c@600 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064/fcb4-i2c-pins
  ...
  [    9.562283] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/reset@e200400c
  [    9.577563] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/oic@e00c0120
  ...
  [    9.592813] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/clock-controller@e00c00a8 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/cpu_clk
  [    9.608258] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/clock-controller@e00c00a8 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/ddr_clk
  [    9.623731] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/clock-controller@e00c00a8 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/sys_clk
  [    9.639269] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/oic@e00c0120 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0
  [    9.651930] device: 'pci:0000:01:00.0--platform:d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0': device_add
  [    9.661716] platform d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0: Linked as a sync state only consumer to 0000:01:00.0
  ...
  [    9.803560] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/sfp-eth2 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [    9.816827] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/sfp-eth2 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064
  [    9.831309] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/sfp-eth3 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [    9.844561] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/sfp-eth3 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064
  ...
  [    9.919384] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/flexcom@e0040000/i2c@600 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [    9.934028] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/flexcom@e0040000/i2c@600 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064/fcb4-i2c-pins
  [    9.951138] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/switch@e0000000 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [    9.965012] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/switch@e0000000 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064/tod_pins
  [    9.980886] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [    9.994831] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/reset@e200400c
  [   10.009834] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/mdio@e200413c Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [   10.023520] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/mdio@e200413c Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/reset@e200400c
  [   10.038248] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/switch@e0000000 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/reset@e200400c
  [   10.053160] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/flexcom@e0040000/i2c@600 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/clock-controller@e00c00a8
  [   10.069820] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/flexcom@e0040000 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [   10.083772] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/flexcom@e0040000 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/clock-controller@e00c00a8
  [   10.099739] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/clock-controller@e00c00a8 Linked as a fwnode consumer to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [   10.114471] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/clock-controller@e00c00a8 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/sys_clk
  [   10.129648] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/clock-controller@e00c00a8 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/ddr_clk
  [   10.144818] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/clock-controller@e00c00a8 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/cpu_clk
  [   10.159986] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/pinctrl@e2004064 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/oic@e00c0120
  [   10.174802] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/flexcom@e0040000/i2c@600 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/oic@e00c0120
  [   10.190323] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/switch@e0000000 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/oic@e00c0120
  [   10.205098] simple-pm-bus d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0: Dropping the link to 0000:01:00.0
  [   10.214501] device: 'pci:0000:01:00.0--platform:d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0': device_unregister
  [   10.225187] device: 'ea000000.switch': device_add
  [   10.230321] device: 'platform:d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0--platform:ea000000.switch': device_add
  [   10.240806] devices_kset: Moving ea000000.switch to end of list
  [   10.246794] platform ea000000.switch: Linked as a consumer to d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0
  [   10.256399] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/switch@e0000000 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [   10.270803] device: 'e800413c.mdio': device_add
  [   10.275775] device: 'platform:d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0--platform:e800413c.mdio': device_add
  [   10.286036] devices_kset: Moving e800413c.mdio to end of list
  [   10.291863] platform e800413c.mdio: Linked as a consumer to d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0
  [   10.301213] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/mdio@e200413c Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [   10.315260] mscc-miim e800413c.mdio: error -EPROBE_DEFER: Failed to get reset
  ...
  [   10.387676] device: 'd0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0:sfp-eth3': device_add
  [   10.388124] devices_kset: Moving ea000000.switch to end of list
  [   10.402096] device: 'platform:d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0--platform:d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0:sfp-eth3': device_add
  [   10.416004] devices_kset: Moving d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0:sfp-eth3 to end of list
  [   10.425281] platform d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0:sfp-eth3: Linked as a consumer to d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0
  [   10.437942] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/sfp-eth3 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [   10.451352] devices_kset: Moving e800413c.mdio to end of list
  [   10.451912] device: 'd0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0:sfp-eth2': device_add
  [   10.458161] mscc-miim e800413c.mdio: error -EPROBE_DEFER: Failed to get reset
  [   10.466501] device: 'platform:d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0--platform:d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0:sfp-eth2': device_add
  [   10.485906] devices_kset: Moving d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0:sfp-eth2 to end of list
  [   10.495096] platform d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0:sfp-eth2: Linked as a consumer to d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0
  [   10.507855] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/sfp-eth2 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  ...
  [   10.580486] device: 'ea040000.flexcom': device_add
  [   10.585753] device: 'platform:d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0--platform:ea040000.flexcom': device_add
  [   10.596336] devices_kset: Moving ea040000.flexcom to end of list
  [   10.602429] platform ea040000.flexcom: Linked as a consumer to d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0
  [   10.612031] /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0/flexcom@e0040000 Dropping the fwnode link to /soc/pcie@d0070000/pci@0,0/dev@0,0/pci-ep-bus@0
  [   10.626863] device: 'e8004064.pinctrl': device_add
  [   10.632045] device: 'platform:d0070000.pcie:pci@0,0:dev@0,0:pci-ep-bus@0--platform:e8004064.pinctrl': device_add
  ...

--- 8< ---

Here is the related dtso applied at the PCIe device DT node:
--- 8< ---

/ {
	fragment@0 {
		target-path="";
		__overlay__ {
			#address-cells = <3>;
			#size-cells = <2>;

			pci-ep-bus@0 {
				compatible = "simple-bus";
				#address-cells = <1>;
				#size-cells = <1>;

				/*
				 * map @0xe2000000 (32MB) to BAR0 (CPU)
				 * map @0xe0000000 (16MB) to BAR1 (AMBA)
				 */
				ranges = <0xe2000000 0x00 0x00 0x00 0x2000000
				          0xe0000000 0x01 0x00 0x00 0x1000000>;

				oic: oic@e00c0120 {
					compatible = "microchip,lan966x-oic";
					#interrupt-cells = <2>;
					interrupt-controller;
					interrupts = <0>; /* PCI INTx assigned interrupt */
					reg = <0xe00c0120 0x190>;
				};

				cpu_clk: cpu_clk {
					compatible = "fixed-clock";
					#clock-cells = <0>;
					clock-frequency = <600000000>;  // CPU clock = 600MHz
				};

				ddr_clk: ddr_clk {
					compatible = "fixed-clock";
					#clock-cells = <0>;
					clock-frequency = <30000000>;  // Fabric clock = 30MHz
				};

				sys_clk: sys_clk {
					compatible = "fixed-clock";
					#clock-cells = <0>;
					clock-frequency = <15625000>;  // System clock = 15.625MHz
				};

				clks: clock-controller@e00c00a8 {
					compatible = "microchip,lan966x-gck";
					#clock-cells = <1>;
					clocks = <&cpu_clk>, <&ddr_clk>, <&sys_clk>;
					clock-names = "cpu", "ddr", "sys";
					reg = <0xe00c00a8 0x38>, <0xe00c02cc 0x4>;
				};

				cpu_ctrl: syscon@e00c0000 {
					compatible = "microchip,lan966x-cpu-syscon", "syscon";
					reg = <0xe00c0000 0xa8>;
				};

				reset: reset@e200400c {
					compatible = "microchip,lan966x-switch-reset";
					reg = <0xe200400c 0x4>;
					reg-names = "gcb";
					#reset-cells = <1>;
					cpu-syscon = <&cpu_ctrl>;
				};

				gpio: pinctrl@e2004064 {
					compatible = "microchip,lan966x-pinctrl";
					reg = <0xe2004064 0xb4>,
					      <0xe2010024 0x138>;
					resets = <&reset 0>;
					reset-names = "switch";
					gpio-controller;
					#gpio-cells = <2>;
					gpio-ranges = <&gpio 0 0 78>;
					interrupt-parent = <&oic>;
					interrupt-controller;
					interrupts = <17 IRQ_TYPE_LEVEL_HIGH>;
					#interrupt-cells = <2>;

					tod_pins: tod_pins {
						pins = "GPIO_36";
						function = "ptpsync_1";
					};

					fc0_a_pins: fcb4-i2c-pins {
						/* RXD, TXD */
						pins = "GPIO_9", "GPIO_10";
						function = "fc0_a";
					};

					i2cmux_pins: i2cmux-pins {
						pins = "GPIO_76", "GPIO_77";
						function = "twi_slc_gate";
						output-low;
					};

					i2cmux_0: i2cmux-0 {
						pins = "GPIO_76";
						function = "twi_slc_gate";
						output-high;
					};

					i2cmux_1: i2cmux-1 {
						pins = "GPIO_77";
						function = "twi_slc_gate";
						output-high;
					};
				};

				flx0: flexcom@e0040000 {
					compatible = "atmel,sama5d2-flexcom";
					reg = <0xe0040000 0x100>;
					clocks = <&clks GCK_ID_FLEXCOM0>;
					#address-cells = <1>;
					#size-cells = <1>;
					ranges = <0x0 0xe0040000 0x800>;

					atmel,flexcom-mode = <ATMEL_FLEXCOM_MODE_TWI>;

					i2c_lan966x: i2c@600 {
						compatible = "microchip,lan966x-i2c";
						reg = <0x600 0x200>;
						interrupt-parent = <&oic>;
						interrupts = <48 IRQ_TYPE_LEVEL_HIGH>;
						#address-cells = <1>;
						#size-cells = <0>;
						clocks = <&clks GCK_ID_FLEXCOM0>;
						assigned-clocks = <&clks GCK_ID_FLEXCOM0>;
						assigned-clock-rates = <20000000>;
						pinctrl-0 = <&fc0_a_pins>;
						pinctrl-names = "default";
						i2c-analog-filter;
						i2c-digital-filter;
						i2c-digital-filter-width-ns = <35>;
					};
				};

				i2c0_emux: i2c0-emux@0 {
					compatible = "i2c-mux-pinctrl";
					#address-cells = <1>;
					#size-cells = <0>;
					i2c-parent = <&i2c_lan966x>;
					pinctrl-names = "i2c102", "i2c103", "idle";
					pinctrl-0 = <&i2cmux_0>;
					pinctrl-1 = <&i2cmux_1>;
					pinctrl-2 = <&i2cmux_pins>;

					i2c102: i2c_sfp1 {
						reg = <0>;
						#address-cells = <1>;
						#size-cells = <0>;
					};

					i2c103: i2c_sfp2 {
						reg = <1>;
						#address-cells = <1>;
						#size-cells = <0>;
					};
				};

				sfp_eth2: sfp-eth2 {
					compatible       = "sff,sfp";
					i2c-bus          = <&i2c102>;
					tx-disable-gpios = <&gpio  0 GPIO_ACTIVE_HIGH>;
					los-gpios        = <&gpio 25 GPIO_ACTIVE_HIGH>;
					mod-def0-gpios   = <&gpio 18 GPIO_ACTIVE_LOW>;
					tx-fault-gpios   = <&gpio  2 GPIO_ACTIVE_HIGH>;
				};

				sfp_eth3: sfp-eth3 {
					compatible       = "sff,sfp";
					i2c-bus          = <&i2c103>;
					tx-disable-gpios = <&gpio  1 GPIO_ACTIVE_HIGH>;
					los-gpios        = <&gpio 26 GPIO_ACTIVE_HIGH>;
					mod-def0-gpios   = <&gpio 19 GPIO_ACTIVE_LOW>;
					tx-fault-gpios   = <&gpio  3 GPIO_ACTIVE_HIGH>;
				};

				serdes: serdes@e202c000 {
					compatible = "microchip,lan966x-serdes";
					reg = <0xe202c000 0x9c>,
					      <0xe2004010 0x4>;
					#phy-cells = <2>;
				};

				mdio1: mdio@e200413c {
					#address-cells = <1>;
					#size-cells = <0>;
					compatible = "microchip,lan966x-miim";
					reg = <0xe200413c 0x24>,
					      <0xe2010020 0x4>;

					resets = <&reset 0>;
					reset-names = "switch";

					lan966x_phy0: ethernet-lan966x_phy@1 {
						reg = <1>;
					};

					lan966x_phy1: ethernet-lan966x_phy@2 {
						reg = <2>;
					};
				};

				switch: switch@e0000000 {
					compatible = "microchip,lan966x-switch";
					reg = <0xe0000000 0x0100000>,
					      <0xe2000000 0x0800000>;
					reg-names = "cpu", "gcb";

					interrupt-parent = <&oic>;
					interrupts = <12 IRQ_TYPE_LEVEL_HIGH>,
						     <9 IRQ_TYPE_LEVEL_HIGH>;
					interrupt-names = "xtr", "ana";

					resets = <&reset 0>;
					reset-names = "switch";

					pinctrl-names = "default";
					pinctrl-0 = <&tod_pins>;

					ethernet-ports {
						#address-cells = <1>;
						#size-cells = <0>;

						port0: port@0 {
							phy-handle = <&lan966x_phy0>;

							reg = <0>;
							phy-mode = "gmii";
							phys = <&serdes 0 CU(0)>;
						};

						port1: port@1 {
							phy-handle = <&lan966x_phy1>;

							reg = <1>;
							phy-mode = "gmii";
							phys = <&serdes 1 CU(1)>;
						};

						port2: port@2 {
							reg = <2>;
							phy-mode = "sgmii";
							phys = <&serdes 2 SERDES6G(0)>;
							sfp = <&sfp_eth2>;
							managed = "in-band-status";
						};

						port3: port@3 {
							reg = <3>;
							phy-mode = "sgmii";
							phys = <&serdes 3 SERDES6G(1)>;
							sfp = <&sfp_eth3>;
							managed = "in-band-status";
						};
					};
				};
			};
		};
	};
};

--- 8< ---

Best regards,
Hervé

Hi Saravana,

Have you made any progress on this topic ?

I haven't seen any updates.
Maybe I missed something.

Best regards,
Hervé

Hi Saravana,

Unfortunately it doesn't, and the result is worse than v2.

After applying the first patch (the revert), the issue reported in
[1] is back, as expected.

After applying both patches, that issue is not fixed, i.e. I still need an add/rm/add cycle to instantiate the devices from the overlay.

/sys/class/devlink shows one extra link after the first add:
platform:e6060000.pinctrl--platform:e6e90000.spi -> ../../devices/virtual/devlink/platform:e6060000.pinctrl--platform:e6e90000.spi

Gr{oetje,eeting}s,

Geert

Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]

In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
                                -- Linus Torvalds