PCI/MSI: Export all remapped MSIs to sysfs attributes
PATCH 1/1 PCI/MSI: Export all remapped MSIs to sysfs attributes
irqbalance uses these attributes to populate its internal database, which is then used to bind the irq on the appropriate NUMA node.
On a device accepting multiple MSIs and with interrupt remapping enabled, only the first irq entry is exported to msi_irqs directory.
This results in irqbalance having no clue of the NUMA affinity for the extra irqs and starting to bind them on random nodes.
This patch exports all MSI interrupts as sysfs attributes when relevant.
--- drivers/pci/msi.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-)
drivers/pci/msi.c +17 -14
@@ -475,10 +475,11 @@ static int populate_msi_sysfs(struct pci_dev *pdev)int ret = -ENOMEM ;int num_msi =0 ;int count =0 ; +int i;/* Determine how many msi entries we have */ for_each_pci_msi_entry (entry, pdev) - ++num_msi; + num_msi += entry->nvec_used;if (!num_msi)return 0 ;
@@ -487,19 +488,21 @@ static int populate_msi_sysfs(struct pci_dev *pdev)if (!msi_attrs)return -ENOMEM ;for_each_pci_msi_entry (entry, pdev) { - msi_dev_attr =kzalloc (sizeof (*msi_dev_attr),GFP_KERNEL ); -if (!msi_dev_attr) - goto error_attrs; - msi_attrs[count] = &msi_dev_attr->attr; - -sysfs_attr_init (&msi_dev_attr->attr); - msi_dev_attr->attr.name =kasprintf (GFP_KERNEL ,"%d" , - entry->irq); -if (!msi_dev_attr->attr.name) - goto error_attrs; - msi_dev_attr->attr.mode =S_IRUGO ; - msi_dev_attr->show = msi_mode_show; - ++count; +for (i =0 ; i < entry->nvec_used; i++) { + msi_dev_attr =kzalloc (sizeof (*msi_dev_attr),GFP_KERNEL ); +if (!msi_dev_attr) + goto error_attrs; + msi_attrs[count] = &msi_dev_attr->attr; + +sysfs_attr_init (&msi_dev_attr->attr); + msi_dev_attr->attr.name =kasprintf (GFP_KERNEL ,"%d" , + entry->irq + i); +if (!msi_dev_attr->attr.name) + goto error_attrs; + msi_dev_attr->attr.mode =S_IRUGO ; + msi_dev_attr->show = msi_mode_show; + ++count; + } } msi_irq_group =kzalloc (sizeof (*msi_irq_group),GFP_KERNEL );
Signature · 1 line
2.4.9
[+cc Neil, Thomas, linux-kernel]
Hi Romain,
Quoted 11 lines · Romain Bezut
irqbalance uses these attributes to populate its internal database, which is then used to bind the irq on the appropriate NUMA node.
On a device accepting multiple MSIs and with interrupt remapping enabled, only the first irq entry is exported to
msi_irqsdirectory.
This results in irqbalance having no clue of the NUMA affinity for the extra irqs and starting to bind them on random nodes.This patch exports all MSI interrupts as sysfs attributes when relevant.
Signed-off-by Romain Bezut <[email protected]>
This seems like it makes sense to me. Do you have any bug reports related to this problem? If so, I'll mention them in the changelog.
Added Neil, Thomas, and linux-kernel since this is of interest to folks beyond the usual linux-pci audience.
Quoted 63 lines
--- drivers/pci/msi.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-)drivers/pci/msi.c +17
-14@@ -475,10 +475,11 @@ static int populate_msi_sysfs(struct pci_dev *pdev)int ret = -ENOMEM ;int num_msi =0 ;int count =0 ; +int i;/* Determine how many msi entries we have */ for_each_pci_msi_entry (entry, pdev) - ++num_msi; + num_msi += entry->nvec_used;if (!num_msi)return 0 ;@@ -487,19 +488,21 @@ static int populate_msi_sysfs(struct pci_dev *pdev)if (!msi_attrs)return -ENOMEM ;for_each_pci_msi_entry (entry, pdev) { - msi_dev_attr =kzalloc (sizeof (*msi_dev_attr),GFP_KERNEL ); -if (!msi_dev_attr) - goto error_attrs; - msi_attrs[count] = &msi_dev_attr->attr; - -sysfs_attr_init (&msi_dev_attr->attr); - msi_dev_attr->attr.name =kasprintf (GFP_KERNEL ,"%d" , - entry->irq); -if (!msi_dev_attr->attr.name) - goto error_attrs; - msi_dev_attr->attr.mode =S_IRUGO ; - msi_dev_attr->show = msi_mode_show; - ++count; +for (i =0 ; i < entry->nvec_used; i++) { + msi_dev_attr =kzalloc (sizeof (*msi_dev_attr),GFP_KERNEL ); +if (!msi_dev_attr) + goto error_attrs; + msi_attrs[count] = &msi_dev_attr->attr; + +sysfs_attr_init (&msi_dev_attr->attr); + msi_dev_attr->attr.name =kasprintf (GFP_KERNEL ,"%d" , + entry->irq + i); +if (!msi_dev_attr->attr.name) + goto error_attrs; + msi_dev_attr->attr.mode =S_IRUGO ; + msi_dev_attr->show = msi_mode_show; + ++count; + } } msi_irq_group =kzalloc (sizeof (*msi_irq_group),GFP_KERNEL );Signature · 6 lines
2.4.9 -- To unsubscribe from this list: send the line "unsubscribe linux-pci" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Thomas Gleixner
Acked-byQuoted 11 lines · Bjorn Helgaas
Quoted 10 lines · Romain Bezut
irqbalance uses these attributes to populate its internal database, which is then used to bind the irq on the appropriate NUMA node.
On a device accepting multiple MSIs and with interrupt remapping enabled, only the first irq entry is exported to
msi_irqsdirectory.
This results in irqbalance having no clue of the NUMA affinity for the extra irqs and starting to bind them on random nodes.This patch exports all MSI interrupts as sysfs attributes when relevant.
Quoted 7 lines
Quoted 1 line
Signed-off-by Romain Bezut <[email protected]>This seems like it makes sense to me. Do you have any bug reports related to this problem? If so, I'll mention them in the changelog.
Added Neil, Thomas, and linux-kernel since this is of interest to folks beyond the usual linux-pci audience.
Quoted 11 lines · Romain Bezut
irqbalance uses these attributes to populate its internal database, which is then used to bind the irq on the appropriate NUMA node.
On a device accepting multiple MSIs and with interrupt remapping enabled, only the first irq entry is exported to
msi_irqsdirectory.
This results in irqbalance having no clue of the NUMA affinity for the extra irqs and starting to bind them on random nodes.This patch exports all MSI interrupts as sysfs attributes when relevant.
Signed-off-by Romain Bezut <[email protected]>
Applied with Thomas' ack to pci/msi for v4.4, thanks, Romain!
Quoted 63 lines
--- drivers/pci/msi.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-)drivers/pci/msi.c +17
-14@@ -475,10 +475,11 @@ static int populate_msi_sysfs(struct pci_dev *pdev)int ret = -ENOMEM ;int num_msi =0 ;int count =0 ; +int i;/* Determine how many msi entries we have */ for_each_pci_msi_entry (entry, pdev) - ++num_msi; + num_msi += entry->nvec_used;if (!num_msi)return 0 ;@@ -487,19 +488,21 @@ static int populate_msi_sysfs(struct pci_dev *pdev)if (!msi_attrs)return -ENOMEM ;for_each_pci_msi_entry (entry, pdev) { - msi_dev_attr =kzalloc (sizeof (*msi_dev_attr),GFP_KERNEL ); -if (!msi_dev_attr) - goto error_attrs; - msi_attrs[count] = &msi_dev_attr->attr; - -sysfs_attr_init (&msi_dev_attr->attr); - msi_dev_attr->attr.name =kasprintf (GFP_KERNEL ,"%d" , - entry->irq); -if (!msi_dev_attr->attr.name) - goto error_attrs; - msi_dev_attr->attr.mode =S_IRUGO ; - msi_dev_attr->show = msi_mode_show; - ++count; +for (i =0 ; i < entry->nvec_used; i++) { + msi_dev_attr =kzalloc (sizeof (*msi_dev_attr),GFP_KERNEL ); +if (!msi_dev_attr) + goto error_attrs; + msi_attrs[count] = &msi_dev_attr->attr; + +sysfs_attr_init (&msi_dev_attr->attr); + msi_dev_attr->attr.name =kasprintf (GFP_KERNEL ,"%d" , + entry->irq + i); +if (!msi_dev_attr->attr.name) + goto error_attrs; + msi_dev_attr->attr.mode =S_IRUGO ; + msi_dev_attr->show = msi_mode_show; + ++count; + } } msi_irq_group =kzalloc (sizeof (*msi_irq_group),GFP_KERNEL );Signature · 6 lines
2.4.9 -- To unsubscribe from this list: send the line "unsubscribe linux-pci" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 9 lines · Bjorn Helgaas
Quoted 8 lines · Romain Bezut
irqbalance uses these attributes to populate its internal database, which is then used to bind the irq on the appropriate NUMA node.
On a device accepting multiple MSIs and with interrupt remapping enabled, only the first irq entry is exported to
msi_irqsdirectory.
This results in irqbalance having no clue of the NUMA affinity for the extra irqs and starting to bind them on random nodes.
Quoted 5 lines
Quoted 3 lines
This patch exports all MSI interrupts as sysfs attributes when relevant.
Signed-off-by Romain Bezut <[email protected]>Applied with Thomas' ack to pci/msi for v4.4, thanks, Romain!
Internal testing with netperf - network performance between two machines with 10Gb Intel NICs running 24 instances of the netperf tool in parallel (to utilize all CPU cores) - shows a roughly 20% performance degradation. Bi-section showed the offending commit to be this patch: commit a86760664f4 ("PCI/MSI: Export all remapped MSIs to sysfs attributes").
Prior: 9.62 +-0.00 gbits/sec
After: 7.77 +-0.17 gbits/sec
Quoted 67 lines
Quoted 59 lines
--- drivers/pci/msi.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-)drivers/pci/msi.c +17
-15@@ -475,10 +475,11 @@ static int populate_msi_sysfs(struct pci_dev *pdev)int ret = -ENOMEM ;int num_msi =0 ;int count =0 ; +int i;/* Determine how many msi entries we have */ for_each_pci_msi_entry (entry, pdev) - ++num_msi; + num_msi += entry->nvec_used;if (!num_msi)return 0 ;@@ -487,19 +488,21 @@ static int populate_msi_sysfs(struct pci_dev *pdev)if (!msi_attrs)return -ENOMEM ;for_each_pci_msi_entry (entry, pdev) { - msi_dev_attr =kzalloc (sizeof (*msi_dev_attr),GFP_KERNEL ); -if (!msi_dev_attr) - goto error_attrs; - msi_attrs[count] = &msi_dev_attr->attr; - -sysfs_attr_init (&msi_dev_attr->attr); - msi_dev_attr->attr.name =kasprintf (GFP_KERNEL ,"%d" , - entry->irq); -if (!msi_dev_attr->attr.name) - goto error_attrs; - msi_dev_attr->attr.mode =S_IRUGO ; - msi_dev_attr->show = msi_mode_show; - ++count; +for (i =0 ; i < entry->nvec_used; i++) { + msi_dev_attr =kzalloc (sizeof (*msi_dev_attr),GFP_KERNEL ); +if (!msi_dev_attr) + goto error_attrs; + msi_attrs[count] = &msi_dev_attr->attr; + +sysfs_attr_init (&msi_dev_attr->attr); + msi_dev_attr->attr.name =kasprintf (GFP_KERNEL ,"%d" , + entry->irq + i); +if (!msi_dev_attr->attr.name) + goto error_attrs; + msi_dev_attr->attr.mode =S_IRUGO ; + msi_dev_attr->show = msi_mode_show; + ++count; + } } msi_irq_group =kzalloc (sizeof (*msi_irq_group),GFP_KERNEL ); --2.4.9
Quoted 4 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 23 lines · Myron Stowe
Quoted 9 lines · Bjorn Helgaas
Quoted 8 lines · Romain Bezut
irqbalance uses these attributes to populate its internal database, which is then used to bind the irq on the appropriate NUMA node.
On a device accepting multiple MSIs and with interrupt remapping enabled, only the first irq entry is exported to
msi_irqsdirectory.
This results in irqbalance having no clue of the NUMA affinity for the extra irqs and starting to bind them on random nodes.Quoted 5 lines
Quoted 3 lines
This patch exports all MSI interrupts as sysfs attributes when relevant.
Signed-off-by Romain Bezut <[email protected]>Applied with Thomas' ack to pci/msi for v4.4, thanks, Romain!
Internal testing with netperf - network performance between two machines with 10Gb Intel NICs running 24 instances of the netperf tool in parallel (to utilize all CPU cores) - shows a roughly 20% performance degradation. Bi-section showed the offending commit to be this patch: commit
a86760664f4("PCI/MSI: Export all remapped MSIs to sysfs attributes").
Quoted 3 lines
Prior: 9.62 +-0.00 gbits/sec
After: 7.77 +-0.17 gbits/sec
Hmm, any idea whats leading to the performance degradation? Seems odd that exposing that info should lead to reduced performance. is irqbalance placing the interrupts on a node that is inappropriate (i.e. making the wrong decision based on the info exposed)?
Neil
Quoted 68 lines
Quoted 67 lines
Quoted 59 lines
--- drivers/pci/msi.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-)drivers/pci/msi.c +17
-15@@ -475,10 +475,11 @@ static int populate_msi_sysfs(struct pci_dev *pdev)int ret = -ENOMEM ;int num_msi =0 ;int count =0 ; +int i;/* Determine how many msi entries we have */ for_each_pci_msi_entry (entry, pdev) - ++num_msi; + num_msi += entry->nvec_used;if (!num_msi)return 0 ;@@ -487,19 +488,21 @@ static int populate_msi_sysfs(struct pci_dev *pdev)if (!msi_attrs)return -ENOMEM ;for_each_pci_msi_entry (entry, pdev) { - msi_dev_attr =kzalloc (sizeof (*msi_dev_attr),GFP_KERNEL ); -if (!msi_dev_attr) - goto error_attrs; - msi_attrs[count] = &msi_dev_attr->attr; - -sysfs_attr_init (&msi_dev_attr->attr); - msi_dev_attr->attr.name =kasprintf (GFP_KERNEL ,"%d" , - entry->irq); -if (!msi_dev_attr->attr.name) - goto error_attrs; - msi_dev_attr->attr.mode =S_IRUGO ; - msi_dev_attr->show = msi_mode_show; - ++count; +for (i =0 ; i < entry->nvec_used; i++) { + msi_dev_attr =kzalloc (sizeof (*msi_dev_attr),GFP_KERNEL ); +if (!msi_dev_attr) + goto error_attrs; + msi_attrs[count] = &msi_dev_attr->attr; + +sysfs_attr_init (&msi_dev_attr->attr); + msi_dev_attr->attr.name =kasprintf (GFP_KERNEL ,"%d" , + entry->irq + i); +if (!msi_dev_attr->attr.name) + goto error_attrs; + msi_dev_attr->attr.mode =S_IRUGO ; + msi_dev_attr->show = msi_mode_show; + ++count; + } } msi_irq_group =kzalloc (sizeof (*msi_irq_group),GFP_KERNEL ); --2.4.9
Quoted 4 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html