PCI: vmd: Issue secondary bus reset and vmd domain window reset
Subsystems (3)
Nirmal Patel
maintainerPATCH v3 1/1 PCI: vmd: Issue secondary bus reset and vmd domain window reset
In order to properly re-initialize the VMD domain during repetitive driver attachment or reboot tests, ensure that the VMD root ports are re-initialized to a blank state that can be re-enumerated appropriately by the PCI core. This is performed by re-initializing all of the bridge windows to ensure that PCI core enumeration does not detect potentially invalid bridge windows and misinterpret them as firmware-assigned windows, when they simply may be invalid bridge window information from a previous boot.
During VT-d passthrough repetitive reboot tests, it was determined that the VMD domain needed to be reset in order to allow downstream devices to reinitialize properly. This is done using setting secondary bus reset bit of each of the VMD root port and will propagate reset through downstream bridges.
v2->v3: Combining two functions into one, Remove redundant definations
and Formatting fixes
--- drivers/pci/controller/vmd.c | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+)
drivers/pci/controller/vmd.c +63 -0
@@ -15,6 +15,9 @@#include <linux/srcu.h> #include <linux/rculist.h> #include <linux/rcupdate.h> +#include <linux/delay.h> +#include <linux/pci_regs.h> +#include <linux/pci_ids.h> #include <asm/irqdomain.h> #include <asm/device.h>
@@ -447,6 +450,64 @@ static struct pci_ops vmd_ops = { .write = vmd_pci_write, }; +static void vmd_domain_reset (struct vmd_dev *vmd) +{ +char __iomem *base; +char __iomem *addr; +u16 ctl; +int dev_seq; +int max_devs =32 ; +int max_buses =resource_size (&vmd->resources[0 ]); +int bus_seq; +u8 functions; +u8 fn_seq; +u8 hdr_type; + +for (bus_seq =0 ; bus_seq < max_buses; bus_seq++) { +for (dev_seq =0 ; dev_seq < max_devs; dev_seq++) { + base = vmd->cfgbar + +PCIE_ECAM_OFFSET (bus_seq, +PCI_DEVFN (dev_seq,0 ),PCI_VENDOR_ID ); + +if (readw (base) !=PCI_VENDOR_ID_INTEL ) +continue ; + + hdr_type =readb (base +PCI_HEADER_TYPE ) &PCI_HEADER_TYPE_MASK ; +if (hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +continue ; + + functions = !!(hdr_type &0x80 ) ?8 :1 ; +for (fn_seq =0 ; fn_seq < functions; fn_seq++) + { + addr = vmd->cfgbar + +PCIE_ECAM_OFFSET (0x0 , +PCI_DEVFN (dev_seq, fn_seq),PCI_VENDOR_ID ); +if (readw (addr) !=PCI_VENDOR_ID_INTEL ) +continue ; + +memset_io ((vmd->cfgbar + +PCIE_ECAM_OFFSET (0x0 ,PCI_DEVFN (dev_seq, fn_seq),PCI_IO_BASE )), +0 ,PCI_ROM_ADDRESS1 -PCI_IO_BASE ); + } + +if (readw (base +PCI_CLASS_DEVICE ) !=PCI_CLASS_BRIDGE_PCI ) +continue ; + +/* pci_reset_secondary_bus() */ + ctl =readw (base +PCI_BRIDGE_CONTROL ); + ctl |=PCI_BRIDGE_CTL_BUS_RESET ; +writew (ctl, base +PCI_BRIDGE_CONTROL ); +readw (base +PCI_BRIDGE_CONTROL ); +msleep (2 ); + + ctl &= ~PCI_BRIDGE_CTL_BUS_RESET ; +writew (ctl, base +PCI_BRIDGE_CONTROL ); +readw (base +PCI_BRIDGE_CONTROL ); + } + } +ssleep (1 ); +} +static void vmd_attach_resources (struct vmd_dev *vmd) { vmd->dev->resource[VMD_MEMBAR1 ].child = &vmd->resources[1 ];
@@ -747,6 +808,8 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)if (vmd->irq_domain)dev_set_msi_domain (&vmd->bus->dev, vmd->irq_domain); +vmd_domain_reset (vmd); +pci_scan_child_bus (vmd->bus);pci_assign_unassigned_bus_resources (vmd->bus);
Signature · 1 line
2.27.0
Hey Nirmal
Quoted 9 lines · Nirmal Patel
In order to properly re-initialize the VMD domain during repetitive driver attachment or reboot tests, ensure that the VMD root ports are re-initialized to a blank state that can be re-enumerated appropriately by the PCI core. This is performed by re-initializing all of the bridge windows to ensure that PCI core enumeration does not detect potentially invalid bridge windows and misinterpret them as firmware-assigned windows, when they simply may be invalid bridge window information from a previous boot.
Quoted 5 lines
During VT-d passthrough repetitive reboot tests, it was determined that the VMD domain needed to be reset in order to allow downstream devices to reinitialize properly. This is done using setting secondary bus reset bit of each of the VMD root port and will propagate reset through downstream bridges.
Can we better combine these two paragraphs?
Quoted 3 lines
v2->v3: Combining two functions into one, Remove redundant definations
and Formatting fixes
Below the dashes please
Quoted 3 lines
Signed-off-by Nirmal Patel <[email protected]>Reviewed-by Jon Derrick <[email protected]>
Not yet :)
Quoted 9 lines
--- drivers/pci/controller/vmd.c | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+)drivers/pci/controller/vmd.c +0
-0
drivers/pci/controller/vmd.c line 20
Quoted 6 lines
drivers/pci/controller/vmd.c +3
-0@@ -15,6 +15,9 @@#include <linux/srcu.h> #include <linux/rculist.h> #include <linux/rcupdate.h> +#include <linux/delay.h> +#include <linux/pci_regs.h> +#include <linux/pci_ids.h>
Do you need to include pci_regs.h and pci_ids.h?
Quoted 12 lines
drivers/pci/controller/vmd.c +5
-0#include <asm/irqdomain.h> #include <asm/device.h> @@ -447,6 +450,64 @@ static struct pci_ops vmd_ops = { .write = vmd_pci_write, }; +static void vmd_domain_reset (struct vmd_dev *vmd) +{ +char __iomem *base; +char __iomem *addr; +u16 ctl;
drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 467
Quoted 10 lines
drivers/pci/controller/vmd.c +10
-0+int dev_seq; +int max_devs =32 ; +int max_buses =resource_size (&vmd->resources[0 ]); +int bus_seq; +u8 functions; +u8 fn_seq; +u8 hdr_type; + +for (bus_seq =0 ; bus_seq < max_buses; bus_seq++) { +for (dev_seq =0 ; dev_seq < max_devs; dev_seq++) {
No need for max_devs - just open-code '32'
drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 470
Quoted 3 lines
drivers/pci/controller/vmd.c +3
-0+ base = vmd->cfgbar + +PCIE_ECAM_OFFSET (bus_seq, +PCI_DEVFN (dev_seq,0 ),PCI_VENDOR_ID );
How about:
base = vmd->cfgbar + PCIE_ECAM_OFFSET(bus_seq, PCI_DEVFN(dev_seq, 0), PCI_VENDOR_ID);
drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 473
Quoted 3 lines
drivers/pci/controller/vmd.c +3
-0+ +if (readw (base) !=PCI_VENDOR_ID_INTEL ) +continue ;
Now that it's iterating all of the bridges in all of the buses, should it be limited to Intel devices?
drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 484
Quoted 11 lines
drivers/pci/controller/vmd.c +11
-0+ + hdr_type =readb (base +PCI_HEADER_TYPE ) &PCI_HEADER_TYPE_MASK ; +if (hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +continue ; + + functions = !!(hdr_type &0x80 ) ?8 :1 ; +for (fn_seq =0 ; fn_seq < functions; fn_seq++) + { + addr = vmd->cfgbar + +PCIE_ECAM_OFFSET (0x0 , +PCI_DEVFN (dev_seq, fn_seq),PCI_VENDOR_ID );
Can you do the same as above here? Putting PCIE_ECAM_OFFSET on the same line as vmd->cfgbar? Also could you change bus from 0x0 to 0?
drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 486
Quoted 2 lines
drivers/pci/controller/vmd.c +2
-0+if (readw (addr) !=PCI_VENDOR_ID_INTEL ) +continue ;
Is this necessary?
drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 489
Quoted 3 lines
drivers/pci/controller/vmd.c +3
-0+ + memset_io((vmd->cfgbar + + PCIE_ECAM_OFFSET(0x0,PCI_DEVFN(dev_seq, fn_seq),PCI_IO_BASE)),
Needs a space after the commas, and please use 0 instead of 0x0.
Quoted 25 lines
drivers/pci/controller/vmd.c +21
-0+ 0, PCI_ROM_ADDRESS1 - PCI_IO_BASE); + } + +if (readw (base +PCI_CLASS_DEVICE ) !=PCI_CLASS_BRIDGE_PCI ) +continue ; + +/* pci_reset_secondary_bus() */ + ctl =readw (base +PCI_BRIDGE_CONTROL ); + ctl |=PCI_BRIDGE_CTL_BUS_RESET ; +writew (ctl, base +PCI_BRIDGE_CONTROL ); +readw (base +PCI_BRIDGE_CONTROL ); +msleep (2 ); + + ctl &= ~PCI_BRIDGE_CTL_BUS_RESET ; +writew (ctl, base +PCI_BRIDGE_CONTROL ); +readw (base +PCI_BRIDGE_CONTROL ); + } + } +ssleep (1 ); +} +static void vmd_attach_resources (struct vmd_dev *vmd) { vmd->dev->resource[VMD_MEMBAR1 ].child = &vmd->resources[1 ];
drivers/pci/controller/vmd.c static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) · line 812
Quoted 5 lines
drivers/pci/controller/vmd.c +2
-0@@ -747,6 +808,8 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)if (vmd->irq_domain)dev_set_msi_domain (&vmd->bus->dev, vmd->irq_domain); +vmd_domain_reset (vmd); +
I'd remove this blank line
Quoted 4 lines
drivers/pci/controller/vmd.c +0
-0pci_scan_child_bus (vmd->bus);pci_assign_unassigned_bus_resources (vmd->bus);
Patel, Nirmal
maintainerQuoted 12 lines · Jonathan
Hey Nirmal
Quoted 9 lines · Nirmal Patel
In order to properly re-initialize the VMD domain during repetitive driver attachment or reboot tests, ensure that the VMD root ports are re-initialized to a blank state that can be re-enumerated appropriately by the PCI core. This is performed by re-initializing all of the bridge windows to ensure that PCI core enumeration does not detect potentially invalid bridge windows and misinterpret them as firmware-assigned windows, when they simply may be invalid bridge window information from a previous boot.
Quoted 6 lines
Quoted 5 lines
During VT-d passthrough repetitive reboot tests, it was determined that the VMD domain needed to be reset in order to allow downstream devices to reinitialize properly. This is done using setting secondary bus reset bit of each of the VMD root port and will propagate reset through downstream bridges.
Can we better combine these two paragraphs?
I will try to create better summary.
Quoted 5 lines
Quoted 4 lines
v2->v3: Combining two functions into one, Remove redundant definations
and Formatting fixesBelow the dashes please
Ack
Quoted 4 lines
Quoted 3 lines
Signed-off-by Nirmal Patel <[email protected]>Reviewed-by Jon Derrick <[email protected]>Not yet :)
Sorry about that. will fix it.
Quoted 5 lines
Quoted 5 lines
--- drivers/pci/controller/vmd.c | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+)
Quoted 12 lines
drivers/pci/controller/vmd.c line 20
Quoted 11 lines
drivers/pci/controller/vmd.c +3
-0@@ -15,6 +15,9 @@#include <linux/srcu.h> #include <linux/rculist.h> #include <linux/rcupdate.h> +#include <linux/delay.h> +#include <linux/pci_regs.h> +#include <linux/pci_ids.h> Do you need to include
pci_regs.handpci_ids.h?
Works without including header files too.
Quoted 9 lines
Quoted 9 lines
drivers/pci/controller/vmd.c +0
-0#include <asm/irqdomain.h> #include <asm/device.h> @@ -447,6 +450,64 @@ static struct pci_ops vmd_ops = { .write = vmd_pci_write, };
Quoted 16 lines
Quoted 5 lines
drivers/pci/controller/vmd.c +5
-0+static void vmd_domain_reset (struct vmd_dev *vmd) +{ +char __iomem *base; +char __iomem *addr; +u16 ctl;drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 467
Quoted 10 lines
drivers/pci/controller/vmd.c +10
-0+int dev_seq; +int max_devs =32 ; +int max_buses =resource_size (&vmd->resources[0 ]); +int bus_seq; +u8 functions; +u8 fn_seq; +u8 hdr_type; + +for (bus_seq =0 ; bus_seq < max_buses; bus_seq++) { +for (dev_seq =0 ; dev_seq < max_devs; dev_seq++) {No need for
max_devs- just open-code '32'
Ack.
Quoted 8 lines
drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 470
Quoted 5 lines
drivers/pci/controller/vmd.c +3
-0+ base = vmd->cfgbar + +PCIE_ECAM_OFFSET (bus_seq, +PCI_DEVFN (dev_seq,0 ),PCI_VENDOR_ID );How about:
base = vmd->cfgbar + PCIE_ECAM_OFFSET(bus_seq, PCI_DEVFN(dev_seq, 0), PCI_VENDOR_ID);
Ack.
Quoted 7 lines
drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 473
Quoted 5 lines
drivers/pci/controller/vmd.c +3
-0+ +if (readw (base) !=PCI_VENDOR_ID_INTEL ) +continue ;Now that it's iterating all of the bridges in all of the buses, should it be limited to Intel devices?
Ack. I will remove it. It shouldn't have significant performance hit.
Quoted 15 lines
drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 484
Quoted 13 lines
drivers/pci/controller/vmd.c +11
-0+ + hdr_type =readb (base +PCI_HEADER_TYPE ) &PCI_HEADER_TYPE_MASK ; +if (hdr_type !=PCI_HEADER_TYPE_BRIDGE ) +continue ; + + functions = !!(hdr_type &0x80 ) ?8 :1 ; +for (fn_seq =0 ; fn_seq < functions; fn_seq++) + { + addr = vmd->cfgbar + +PCIE_ECAM_OFFSET (0x0 , +PCI_DEVFN (dev_seq, fn_seq),PCI_VENDOR_ID );Can you do the same as above here? Putting
PCIE_ECAM_OFFSETon the same line asvmd->cfgbar? Also could you change bus from0x0to 0?
Yes.
Quoted 5 lines
drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 486
Quoted 4 lines
drivers/pci/controller/vmd.c +2
-0+if (readw (addr) !=PCI_VENDOR_ID_INTEL ) +continue ;Is this necessary?
Ack.
Quoted 6 lines
drivers/pci/controller/vmd.c static struct pci_ops vmd_ops = { · line 489
Quoted 5 lines
drivers/pci/controller/vmd.c +3
-0+ + memset_io((vmd->cfgbar + + PCIE_ECAM_OFFSET(0x0,PCI_DEVFN(dev_seq, fn_seq),PCI_IO_BASE)),Needs a space after the commas, and please use 0 instead of
0x0.
Ack.
Quoted 30 lines
Quoted 30 lines
drivers/pci/controller/vmd.c +21
-0+ 0, PCI_ROM_ADDRESS1 - PCI_IO_BASE); + } + +if (readw (base +PCI_CLASS_DEVICE ) !=PCI_CLASS_BRIDGE_PCI ) +continue ; + +/* pci_reset_secondary_bus() */ + ctl =readw (base +PCI_BRIDGE_CONTROL ); + ctl |=PCI_BRIDGE_CTL_BUS_RESET ; +writew (ctl, base +PCI_BRIDGE_CONTROL ); +readw (base +PCI_BRIDGE_CONTROL ); +msleep (2 ); + + ctl &= ~PCI_BRIDGE_CTL_BUS_RESET ; +writew (ctl, base +PCI_BRIDGE_CONTROL ); +readw (base +PCI_BRIDGE_CONTROL ); + } + } +ssleep (1 ); +} +static void vmd_attach_resources (struct vmd_dev *vmd) { vmd->dev->resource[VMD_MEMBAR1 ].child = &vmd->resources[1 ];@@ -747,6 +808,8 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)if (vmd->irq_domain)dev_set_msi_domain (&vmd->bus->dev, vmd->irq_domain);
Quoted 3 lines
drivers/pci/controller/vmd.c static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) · line 812
Quoted 2 lines
drivers/pci/controller/vmd.c +2
-0+vmd_domain_reset (vmd); +I'd remove this blank line
Ack.
Quoted 5 lines
Quoted 5 lines
drivers/pci/controller/vmd.c +0
-0pci_scan_child_bus (vmd->bus);pci_assign_unassigned_bus_resources (vmd->bus);