PCI/MSI: Cleanup init and improve 32-bit MSI checking
v3 cover PCI/MSI: Cleanup init and improve 32-bit MSI checking
From: Bjorn Helgaas <[email protected]>
MSI/MSI-X init was a little unconventional. We had pci_msi_setup_pci_dev() to disable MSI and MSI-X, in probe.c instead of msi.c so we could do it even without CONFIG_PCI_MSI. Move that to msi.c and fix the config issue with an #ifdef.
Then add Vidya's patch on top. Previous postings at
https://lore.kernel.org/linux-pci/[email protected]/
https://lore.kernel.org/linux-pci/[email protected]/
Bjorn Helgaas (2):
PCI/MSI: Move MSI/MSI-X init to msi.c
PCI/MSI: Move MSI/MSI-X flags updaters to msi.c
Vidya Sagar (1):
PCI/MSI: Set device flag indicating only 32-bit MSI support
drivers/pci/Makefile | 3 +- drivers/pci/msi.c | 70 ++++++++++++++++++++++++++++++++++++++++---- drivers/pci/pci.h | 23 ++------------- drivers/pci/probe.c | 21 ++----------- 4 files changed, 70 insertions(+), 47 deletions(-)
Signature · 1 line
2.25.1
PATCH v3 1/3 PCI/MSI: Move MSI/MSI-X init to msi.c
From: Bjorn Helgaas <[email protected]>
Move pci_msi_setup_pci_dev(), which disables MSI and MSI-X interrupts, from probe.c to msi.c so it's with all the other MSI code and more consistent with other capability initialization. This means we must compile msi.c always, even without CONFIG_PCI_MSI, so wrap the rest of msi.c in an #ifdef and adjust the Makefile accordingly. No functional change intended.
--- drivers/pci/Makefile | 3 +-- drivers/pci/msi.c | 36 ++++++++++++++++++++++++++++++++++++ drivers/pci/pci.h | 2 ++ drivers/pci/probe.c | 21 ++------------------- 4 files changed, 41 insertions(+), 21 deletions(-)
drivers/pci/Makefile +1 -2
@@ -5,7 +5,7 @@ obj-$(CONFIG_PCI) += access.o bus.o probe.o host-bridge.o \ remove.o pci.o pci-driver.o search.o \ pci-sysfs.o rom.o setup-res.o irq.o vpd.o \ - setup-bus.o vc.o mmap.o setup-irq.o + setup-bus.o vc.o mmap.o setup-irq.o msi.o obj-$(CONFIG_PCI) += pcie/
@@ -18,7 +18,6 @@ endif obj-$(CONFIG_OF) += of.o obj-$(CONFIG_PCI_QUIRKS) += quirks.o obj-$(CONFIG_HOTPLUG_PCI) += hotplug/ -obj-$(CONFIG_PCI_MSI) += msi.o obj-$(CONFIG_PCI_ATS) += ats.o obj-$(CONFIG_PCI_IOV) += iov.o obj-$(CONFIG_PCI_BRIDGE_EMUL) += pci-bridge-emul.o
drivers/pci/msi.c +36 -0
@@ -26,6 +26,8 @@#include "pci.h" +#ifdef CONFIG_MSI +static int pci_msi_enable =1 ;int pci_msi_ignore_mask;
@@ -1577,3 +1579,37 @@ bool pci_dev_has_special_msi_domain(struct pci_dev *pdev) }#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */ +#endif /* CONFIG_PCI_MSI */ + +void pci_msi_init (struct pci_dev *dev) +{ +u16 ctrl; + +/* +* Disable the MSI hardware to avoid screaming interrupts +* during boot. This is the power on reset default so +* usually this should be a noop. +*/ + dev->msi_cap =pci_find_capability (dev,PCI_CAP_ID_MSI ); +if (!dev->msi_cap) +return ; + +pci_read_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , &ctrl); +if (ctrl &PCI_MSI_FLAGS_ENABLE ) +pci_write_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , + ctrl & ~PCI_MSI_FLAGS_ENABLE ); +} + +void pci_msix_init (struct pci_dev *dev) +{ +u16 ctrl; + + dev->msix_cap =pci_find_capability (dev,PCI_CAP_ID_MSIX ); +if (!dev->msix_cap) +return ; + +pci_read_config_word (dev, dev->msix_cap +PCI_MSIX_FLAGS , &ctrl); +if (ctrl &PCI_MSIX_FLAGS_ENABLE ) +pci_write_config_word (dev, dev->msix_cap +PCI_MSIX_FLAGS , + ctrl & ~PCI_MSIX_FLAGS_ENABLE ); +}
drivers/pci/pci.h +2 -0
@@ -104,6 +104,8 @@ void pci_config_pm_runtime_get(struct pci_dev *dev);void pci_config_pm_runtime_put (struct pci_dev *dev);void pci_pm_init (struct pci_dev *dev);void pci_ea_init (struct pci_dev *dev); +void pci_msi_init (struct pci_dev *dev); +void pci_msix_init (struct pci_dev *dev);void pci_allocate_cap_save_buffers (struct pci_dev *dev);void pci_free_cap_save_buffers (struct pci_dev *dev);bool pci_bridge_d3_possible (struct pci_dev *dev);
drivers/pci/probe.c +2 -19
@@ -1716,22 +1716,6 @@ static u8 pci_hdr_type(struct pci_dev *dev)#define LEGACY_IO_RESOURCE (IORESOURCE_IO | IORESOURCE_PCI_FIXED) -static void pci_msi_setup_pci_dev (struct pci_dev *dev) -{ -/* -* Disable the MSI hardware to avoid screaming interrupts -* during boot. This is the power on reset default so -* usually this should be a noop. -*/ - dev->msi_cap =pci_find_capability (dev,PCI_CAP_ID_MSI ); -if (dev->msi_cap) -pci_msi_set_enable (dev,0 ); - - dev->msix_cap =pci_find_capability (dev,PCI_CAP_ID_MSIX ); -if (dev->msix_cap) -pci_msix_clear_and_set_ctrl (dev,PCI_MSIX_FLAGS_ENABLE ,0 ); -} -/** * pci_intx_mask_broken - Test PCI_COMMAND_INTX_DISABLE writability * @dev: PCI device
@@ -2397,9 +2381,8 @@ void pcie_report_downtraining(struct pci_dev *dev)static void pci_init_capabilities (struct pci_dev *dev) {pci_ea_init (dev);/* Enhanced Allocation */ - -/* Setup MSI caps & disable MSI/MSI-X interrupts */ -pci_msi_setup_pci_dev (dev); +pci_msi_init (dev);/* Disable MSI */ +pci_msix_init (dev);/* Disable MSI-X */ /* Buffers for saving PCIe and PCI-X capabilities */ pci_allocate_cap_save_buffers (dev);
Signature · 1 line
2.25.1
Re: [PATCH v3 1/3] PCI/MSI: Move MSI/MSI-X init to msi.c
Quoted 64 lines · Bjorn Helgaas
From: Bjorn Helgaas <[email protected]>
Move
pci_msi_setup_pci_dev(), which disables MSI and MSI-X interrupts, fromprobe.ctomsi.cso it's with all the other MSI code and more consistent with other capability initialization. This means we must compilemsi.calways, even withoutCONFIG_PCI_MSI, so wrap the rest ofmsi.cin an #ifdef and adjust theMakefileaccordingly. No functional change intended.Signed-off-by Bjorn Helgaas <[email protected]>--- drivers/pci/Makefile | 3 +-- drivers/pci/msi.c | 36 ++++++++++++++++++++++++++++++++++++ drivers/pci/pci.h | 2 ++ drivers/pci/probe.c | 21 ++------------------- 4 files changed, 41 insertions(+), 21 deletions(-)drivers/pci/Makefile +1
-2@@ -5,7 +5,7 @@ obj-$(CONFIG_PCI) += access.o bus.o probe.o host-bridge.o \ remove.o pci.o pci-driver.o search.o \ pci-sysfs.o rom.o setup-res.o irq.o vpd.o \ - setup-bus.o vc.o mmap.o setup-irq.o + setup-bus.o vc.o mmap.o setup-irq.o msi.o obj-$(CONFIG_PCI) += pcie/@@ -18,7 +18,6 @@ endif obj-$(CONFIG_OF) += of.o obj-$(CONFIG_PCI_QUIRKS) += quirks.o obj-$(CONFIG_HOTPLUG_PCI) += hotplug/ -obj-$(CONFIG_PCI_MSI) += msi.o obj-$(CONFIG_PCI_ATS) += ats.o obj-$(CONFIG_PCI_IOV) += iov.o obj-$(CONFIG_PCI_BRIDGE_EMUL) += pci-bridge-emul.odrivers/pci/msi.c +12
-0@@ -26,6 +26,8 @@#include "pci.h" +#ifdef CONFIG_MSI +static int pci_msi_enable =1 ;int pci_msi_ignore_mask;@@ -1577,3 +1579,37 @@ bool pci_dev_has_special_msi_domain(struct pci_dev *pdev) }#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */ +#endif /* CONFIG_PCI_MSI */ + +void pci_msi_init (struct pci_dev *dev) +{ +u16 ctrl; + +/* +* Disable the MSI hardware to avoid screaming interrupts +* during boot. This is the power on reset default so +* usually this should be a noop.
drivers/pci/msi.c bool pci_dev_has_special_msi_domain(struct pci_dev *pdev) · line 1601
Quoted 10 lines
drivers/pci/msi.c +10
-0+*/ + dev->msi_cap =pci_find_capability (dev,PCI_CAP_ID_MSI ); +if (!dev->msi_cap) +return ; + +pci_read_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , &ctrl); +if (ctrl &PCI_MSI_FLAGS_ENABLE ) +pci_write_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , + ctrl & ~PCI_MSI_FLAGS_ENABLE ); +}
The old code used the pci_msi_set_enable() helper here...
Quoted 4 lines
drivers/pci/msi.c +4
-0+ +void pci_msix_init (struct pci_dev *dev) +{ +u16 ctrl;
drivers/pci/msi.c bool pci_dev_has_special_msi_domain(struct pci_dev *pdev) · line 1615
Quoted 10 lines
drivers/pci/msi.c +10
-0+ + dev->msix_cap =pci_find_capability (dev,PCI_CAP_ID_MSIX ); +if (!dev->msix_cap) +return ; + +pci_read_config_word (dev, dev->msix_cap +PCI_MSIX_FLAGS , &ctrl); +if (ctrl &PCI_MSIX_FLAGS_ENABLE ) +pci_write_config_word (dev, dev->msix_cap +PCI_MSIX_FLAGS , + ctrl & ~PCI_MSIX_FLAGS_ENABLE ); +}
... and pci_msix_clear_and_set_ctrl() here. I like your version here better because it avoids the unnecessary write in case the flag isn't set. But it got me thinking if perhaps the helpers aren't very useful and perhaps should be dropped in favour of open-coded variants. Especially since there seem to be only 4 and 6 occurrences of them after this patch.
Anyway, this patch looks correct to me and is a nice improvement, so:
Re: [PATCH v3 1/3] PCI/MSI: Move MSI/MSI-X init to msi.c
Quoted 91 lines · Thierry Reding
Quoted 64 lines · Bjorn Helgaas
From: Bjorn Helgaas <[email protected]>
Move
pci_msi_setup_pci_dev(), which disables MSI and MSI-X interrupts, fromprobe.ctomsi.cso it's with all the other MSI code and more consistent with other capability initialization. This means we must compilemsi.calways, even withoutCONFIG_PCI_MSI, so wrap the rest ofmsi.cin an #ifdef and adjust theMakefileaccordingly. No functional change intended.Signed-off-by Bjorn Helgaas <[email protected]>--- drivers/pci/Makefile | 3 +-- drivers/pci/msi.c | 36 ++++++++++++++++++++++++++++++++++++ drivers/pci/pci.h | 2 ++ drivers/pci/probe.c | 21 ++------------------- 4 files changed, 41 insertions(+), 21 deletions(-)drivers/pci/Makefile +1
-2@@ -5,7 +5,7 @@ obj-$(CONFIG_PCI) += access.o bus.o probe.o host-bridge.o \ remove.o pci.o pci-driver.o search.o \ pci-sysfs.o rom.o setup-res.o irq.o vpd.o \ - setup-bus.o vc.o mmap.o setup-irq.o + setup-bus.o vc.o mmap.o setup-irq.o msi.o obj-$(CONFIG_PCI) += pcie/@@ -18,7 +18,6 @@ endif obj-$(CONFIG_OF) += of.o obj-$(CONFIG_PCI_QUIRKS) += quirks.o obj-$(CONFIG_HOTPLUG_PCI) += hotplug/ -obj-$(CONFIG_PCI_MSI) += msi.o obj-$(CONFIG_PCI_ATS) += ats.o obj-$(CONFIG_PCI_IOV) += iov.o obj-$(CONFIG_PCI_BRIDGE_EMUL) += pci-bridge-emul.odrivers/pci/msi.c +12
-0@@ -26,6 +26,8 @@#include "pci.h" +#ifdef CONFIG_MSI +static int pci_msi_enable =1 ;int pci_msi_ignore_mask;@@ -1577,3 +1579,37 @@ bool pci_dev_has_special_msi_domain(struct pci_dev *pdev) }#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */ +#endif /* CONFIG_PCI_MSI */ + +void pci_msi_init (struct pci_dev *dev) +{ +u16 ctrl; + +/* +* Disable the MSI hardware to avoid screaming interrupts +* during boot. This is the power on reset default so +* usually this should be a noop. drivers/pci/msi.c bool pci_dev_has_special_msi_domain(struct pci_dev *pdev) · line 1601
Quoted 10 lines
drivers/pci/msi.c +10
-0+*/ + dev->msi_cap =pci_find_capability (dev,PCI_CAP_ID_MSI ); +if (!dev->msi_cap) +return ; + +pci_read_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , &ctrl); +if (ctrl &PCI_MSI_FLAGS_ENABLE ) +pci_write_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , + ctrl & ~PCI_MSI_FLAGS_ENABLE ); +}The old code used the
pci_msi_set_enable()helper here...Quoted 13 lines
drivers/pci/msi.c +13
-0+ +void pci_msix_init (struct pci_dev *dev) +{ +u16 ctrl; + + dev->msix_cap =pci_find_capability (dev,PCI_CAP_ID_MSIX ); +if (!dev->msix_cap) +return ; + +pci_read_config_word (dev, dev->msix_cap +PCI_MSIX_FLAGS , &ctrl); +if (ctrl &PCI_MSIX_FLAGS_ENABLE ) +pci_write_config_word (dev, dev->msix_cap +PCI_MSIX_FLAGS , + ctrl & ~PCI_MSIX_FLAGS_ENABLE );
Quoted 8 lines
drivers/pci/msi.c bool pci_dev_has_special_msi_domain(struct pci_dev *pdev) · line 1615
Quoted 1 line
drivers/pci/msi.c +1
-0+}... and
pci_msix_clear_and_set_ctrl()here. I like your version here better because it avoids the unnecessary write in case the flag isn't set. But it got me thinking if perhaps the helpers aren't very useful and perhaps should be dropped in favour of open-coded variants. Especially since there seem to be only 4 and 6 occurrences of them after this patch.
I agree, they might be overkill. I didn't want to spend that much time on it, so I just left them for now. Thanks for your review!
Quoted 3 lines
Anyway, this patch looks correct to me and is a nice improvement, so:
Reviewed-by Thierry Reding <[email protected]>
PATCH v3 2/3 PCI/MSI: Move MSI/MSI-X flags updaters to msi.c
In reply to Bjorn HelgaasFrom: Bjorn Helgaas <[email protected]>
pci_msi_set_enable() and pci_msix_clear_and_set_ctrl() are only used from msi.c, so move them from drivers/pci/pci.h to msi.c. No functional change intended.
--- drivers/pci/msi.c | 21 +++++++++++++++++++++ drivers/pci/pci.h | 21 --------------------- 2 files changed, 21 insertions(+), 21 deletions(-)
drivers/pci/msi.c +21 -0
@@ -412,6 +412,17 @@ static void pci_intx_for_msi(struct pci_dev *dev, int enable)pci_intx (dev, enable); } +static void pci_msi_set_enable (struct pci_dev *dev,int enable) +{ +u16 control; + +pci_read_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , &control); + control &= ~PCI_MSI_FLAGS_ENABLE ; +if (enable) + control |=PCI_MSI_FLAGS_ENABLE ; +pci_write_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , control); +} +static void __pci_restore_msi_state (struct pci_dev *dev) {u16 control;
@@ -434,6 +445,16 @@ static void __pci_restore_msi_state(struct pci_dev *dev)pci_write_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , control); } +static void pci_msix_clear_and_set_ctrl (struct pci_dev *dev,u16 clear,u16 set) +{ +u16 ctrl; + +pci_read_config_word (dev, dev->msix_cap +PCI_MSIX_FLAGS , &ctrl); + ctrl &= ~clear; + ctrl |= set; +pci_write_config_word (dev, dev->msix_cap +PCI_MSIX_FLAGS , ctrl); +} +static void __pci_restore_msix_state (struct pci_dev *dev) {struct msi_desc *entry;
drivers/pci/pci.h +0 -21
@@ -187,27 +187,6 @@ void pci_no_msi(void);static inline void pci_no_msi (void ) { }#endif -static inline void pci_msi_set_enable (struct pci_dev *dev,int enable) -{ -u16 control; - -pci_read_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , &control); - control &= ~PCI_MSI_FLAGS_ENABLE ; -if (enable) - control |=PCI_MSI_FLAGS_ENABLE ; -pci_write_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , control); -} - -static inline void pci_msix_clear_and_set_ctrl (struct pci_dev *dev,u16 clear,u16 set) -{ -u16 ctrl; - -pci_read_config_word (dev, dev->msix_cap +PCI_MSIX_FLAGS , &ctrl); - ctrl &= ~clear; - ctrl |= set; -pci_write_config_word (dev, dev->msix_cap +PCI_MSIX_FLAGS , ctrl); -} -void pci_realloc_get_opt (char *);static inline int pci_no_d1d2 (struct pci_dev *dev)
Signature · 1 line
2.25.1
Re: [PATCH v3 2/3] PCI/MSI: Move MSI/MSI-X flags updaters to msi.c
Quoted 11 lines · Bjorn Helgaas
From: Bjorn Helgaas <[email protected]>
pci_msi_set_enable()andpci_msix_clear_and_set_ctrl()are only used frommsi.c, so move them fromdrivers/pci/pci.htomsi.c. No functional change intended.Signed-off-by Bjorn Helgaas <[email protected]>--- drivers/pci/msi.c | 21 +++++++++++++++++++++ drivers/pci/pci.h | 21 --------------------- 2 files changed, 21 insertions(+), 21 deletions(-)
Ah... I suppose this cleans this up a little more. I have no objection to these helpers, though I still think they are a bit unnecessary:
PATCH v3 3/3 PCI/MSI: Set device flag indicating only 32-bit MSI support
In reply to Bjorn HelgaasFrom: Vidya Sagar <[email protected]>
The MSI-X Capability requires devices to support 64-bit Message Addresses, but the MSI Capability can support either 32- or 64-bit addresses.
Previously, we set dev->no_64bit_msi for a few broken devices that advertise 64-bit MSI support but don't correctly support it.
In addition, check the MSI "64-bit Address Capable" bit for all devices and set dev->no_64bit_msi for devices that don't advertise 64-bit support. This allows msi_verify_entries() to catch arch code defects that assign 64-bit addresses when they're not supported.
--- drivers/pci/msi.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
drivers/pci/msi.c +8 -5
@@ -623,11 +623,11 @@ static int msi_verify_entries(struct pci_dev *dev)struct msi_desc *entry;for_each_pci_msi_entry (entry, dev) { -if (!dev->no_64bit_msi || !entry->msg.address_hi) -continue ; -pci_err (dev,"Device has broken 64-bit MSI but arch" -" tried to assign one above 4G\n" ); -return -EIO ; +if (entry->msg.address_hi && dev->no_64bit_msi) { +pci_err (dev,"arch assigned 64-bit MSI address %#x%08x but device only supports 32 bits\n" , + entry->msg.address_hi, entry->msg.address_lo); +return -EIO ; + } }return 0 ; }
@@ -1619,6 +1619,9 @@ void pci_msi_init(struct pci_dev *dev)if (ctrl &PCI_MSI_FLAGS_ENABLE )pci_write_config_word (dev, dev->msi_cap +PCI_MSI_FLAGS , ctrl & ~PCI_MSI_FLAGS_ENABLE ); + +if (!(ctrl &PCI_MSI_FLAGS_64BIT )) + dev->no_64bit_msi =1 ; }void pci_msix_init (struct pci_dev *dev)
Signature · 1 line
2.25.1
Re: [PATCH v3 3/3] PCI/MSI: Set device flag indicating only 32-bit MSI support
Quoted 13 lines · Bjorn Helgaas
From: Vidya Sagar <[email protected]>
The MSI-X Capability requires devices to support 64-bit Message Addresses, but the MSI Capability can support either 32- or 64-bit addresses.
Previously, we set
dev->no_64bit_msifor a few broken devices that advertise 64-bit MSI support but don't correctly support it.In addition, check the MSI "64-bit Address Capable" bit for all devices and set
dev->no_64bit_msifor devices that don't advertise 64-bit support. This allowsmsi_verify_entries()to catch arch code defects that assign 64-bit addresses when they're not supported.
Quoted 7 lines
[bhelgaas: set no_64bit_msi in pci_msi_init(), commit log]Signed-off-by Vidya Sagar <[email protected]>Signed-off-by Bjorn Helgaas <[email protected]>--- drivers/pci/msi.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
Quoted 16 lines · Bjorn Helgaas
From: Bjorn Helgaas <[email protected]>
MSI/MSI-X init was a little unconventional. We had
pci_msi_setup_pci_dev()to disable MSI and MSI-X, inprobe.cinstead ofmsi.cso we could do it even withoutCONFIG_PCI_MSI. Move that tomsi.cand fix the config issue with an #ifdef.Then add Vidya's patch on top. Previous postings at
https://lore.kernel.org/linux-pci/[email protected]/
https://lore.kernel.org/linux-pci/[email protected]/Bjorn Helgaas (2):
PCI/MSI: Move MSI/MSI-X init tomsi.c
PCI/MSI: Move MSI/MSI-X flags updaters tomsi.c
Quoted 8 lines
Vidya Sagar (1):
PCI/MSI: Set device flag indicating only 32-bit MSI supportdrivers/pci/Makefile | 3 +- drivers/pci/msi.c | 70 ++++++++++++++++++++++++++++++++++++++++---- drivers/pci/pci.h | 23 ++------------- drivers/pci/probe.c | 21 ++----------- 4 files changed, 70 insertions(+), 47 deletions(-)
I fixed my typo ("#ifdef CONFIG_MSI" when it should have been "#ifdef CONFIG_PCI_MSI"), added the reference from Vidya, added Thierry's Reviewed-by, and put these on pci/msi for v5.11.