Skip to contentSkip to conversation

PCI: export MSI mode using attributes, not kobjects

PATCH Superseded Greg Kroah-Hartman 6 participants 17 messages Nov 27, 2013 – Dec 13, 2013 PCI

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 PCI: export MSI mode using attributes, not kobjects

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

   └── msi_irqs
       └── 40
           └── mode

new MSI attributes:
pci_device

   └── msi_irqs
       └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

---

Neil, can you test this patch to see if irqbalance works as-is or not?

thanks,

greg k-h

 drivers/pci/msi.c   |  152 +++++++++++++++++++++++++---------------------------
 include/linux/pci.h |    2 
 2 files changed, 76 insertions(+), 78 deletions(-)
drivers/pci/msi.c +75 -77
@@ -363,6 +363,9 @@ void write_msi_msg(unsigned int irq, str
 static void free_msi_irqs(struct pci_dev *dev)
 {
 	struct msi_desc *entry, *tmp;
+	struct attribute **msi_attrs;
+	struct device_attribute *dev_attr;
+	int count = 0;
 
 	list_for_each_entry(entry, &dev->msi_list, list) {
 		int i, nvec;
@@ -398,6 +401,22 @@ static void free_msi_irqs(struct pci_dev
 		list_del(&entry->list);
 		kfree(entry);
 	}
+
+	if (dev->msi_irq_groups) {
+		sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups);
+		msi_attrs = dev->msi_irq_groups[0]->attrs;
+		list_for_each_entry(entry, &dev->msi_list, list) {
+			dev_attr = container_of(msi_attrs[count],
+						struct device_attribute, attr);
+			kfree(dev_attr->attr.name);
+			kfree(dev_attr);
+			++count;
+		}
+		kfree(msi_attrs);
+		kfree(dev->msi_irq_groups[0]);
+		kfree(dev->msi_irq_groups);
+		dev->msi_irq_groups = NULL;
+	}
 }
 
 static struct msi_desc *alloc_msi_entry(struct pci_dev *dev)
@@ -471,96 +490,79 @@ void pci_restore_msi_state(struct pci_de
 }
 EXPORT_SYMBOL_GPL(pci_restore_msi_state);
 
-
-#define to_msi_attr(obj) container_of(obj, struct msi_attribute, attr)
-#define to_msi_desc(obj) container_of(obj, struct msi_desc, kobj)
-
-struct msi_attribute {
-	struct attribute        attr;
-	ssize_t (*show)(struct msi_desc *entry, struct msi_attribute *attr,
-			char *buf);
-	ssize_t (*store)(struct msi_desc *entry, struct msi_attribute *attr,
-			 const char *buf, size_t count);
-};
-
-static ssize_t show_msi_mode(struct msi_desc *entry, struct msi_attribute *atr,
+static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
 			     char *buf)
 {
-	return sprintf(buf, "%s\n", entry->msi_attrib.is_msix ? "msix" : "msi");
-}
-
-static ssize_t msi_irq_attr_show(struct kobject *kobj,
-				 struct attribute *attr, char *buf)
-{
-	struct msi_attribute *attribute = to_msi_attr(attr);
-	struct msi_desc *entry = to_msi_desc(kobj);
-
-	if (!attribute->show)
-		return -EIO;
-
-	return attribute->show(entry, attribute, buf);
-}
-
-static const struct sysfs_ops msi_irq_sysfs_ops = {
-	.show = msi_irq_attr_show,
-};
-
-static struct msi_attribute mode_attribute =
-	__ATTR(mode, S_IRUGO, show_msi_mode, NULL);
-
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct msi_desc *entry;
+	unsigned long irq;
+	int retval;
 
-static struct attribute *msi_irq_default_attrs[] = {
-	&mode_attribute.attr,
-	NULL
-};
+	retval = kstrtoul(attr->attr.name, 10, &irq);
+	if (retval)
+		return retval;
 
-static void msi_kobj_release(struct kobject *kobj)
-{
-	struct msi_desc *entry = to_msi_desc(kobj);
-
-	pci_dev_put(entry->dev);
+	list_for_each_entry(entry, &pdev->msi_list, list) {
+		if (entry->irq == irq) {
+			return sprintf(buf, "%s\n",
+				       entry->msi_attrib.is_msix ? "msix" : "msi");
+		}
+	}
+	return -ENODEV;
 }
 
-static struct kobj_type msi_irq_ktype = {
-	.release = msi_kobj_release,
-	.sysfs_ops = &msi_irq_sysfs_ops,
-	.default_attrs = msi_irq_default_attrs,
-};
-
 static int populate_msi_sysfs(struct pci_dev *pdev)
 {
+	struct attribute **msi_attrs;
+	struct device_attribute *msi_dev_attr;
+	struct attribute_group *msi_irq_group;
+	const struct attribute_group **msi_irq_groups;
 	struct msi_desc *entry;
-	struct kobject *kobj;
 	int ret;
+	int num_msi = 0;
 	int count = 0;
 
-	pdev->msi_kset = kset_create_and_add("msi_irqs", NULL, &pdev->dev.kobj);
-	if (!pdev->msi_kset)
-		return -ENOMEM;
-
+	/* Determine how many msi entries we have */
 	list_for_each_entry(entry, &pdev->msi_list, list) {
-		kobj = &entry->kobj;
-		kobj->kset = pdev->msi_kset;
-		pci_dev_get(pdev);
-		ret = kobject_init_and_add(kobj, &msi_irq_ktype, NULL,
-				     "%u", entry->irq);
-		if (ret)
-			goto out_unroll;
-
-		count++;
+		++num_msi;
 	}
+	if (!num_msi)
+		return 0;
 
-	return 0;
-
-out_unroll:
+	/* Dynamically create the MSI attributes for the PCI device */
+	msi_attrs = kzalloc(sizeof(void *) * (num_msi + 1), GFP_KERNEL);
+	if (!msi_attrs)
+		return -ENOMEM;
 	list_for_each_entry(entry, &pdev->msi_list, list) {
-		if (!count)
-			break;
-		kobject_del(&entry->kobj);
-		kobject_put(&entry->kobj);
-		count--;
+		char *name = kmalloc(20, GFP_KERNEL);
+		msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
+		if (!msi_dev_attr)
+			return -ENOMEM;
+		sprintf(name, "%d", entry->irq);
+		msi_dev_attr->attr.name = name;
+		msi_dev_attr->attr.mode = S_IRUGO;
+		msi_dev_attr->show = msi_mode_show;
+		msi_attrs[count] = &msi_dev_attr->attr;
+		++count;
 	}
-	return ret;
+
+	msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
+	if (!msi_irq_group)
+		return -ENOMEM;
+	msi_irq_group->name = "msi_irqs";
+	msi_irq_group->attrs = msi_attrs;
+
+	msi_irq_groups = kzalloc(sizeof(void *) * 2, GFP_KERNEL);
+	if (!msi_irq_groups)
+		return -ENOMEM;
+	msi_irq_groups[0] = msi_irq_group;
+
+	ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups);
+	if (ret)
+		return ret;
+	pdev->msi_irq_groups = msi_irq_groups;
+
+	return 0;
 }
 
 /**
@@ -925,8 +927,6 @@ void pci_disable_msi(struct pci_dev *dev
 
 	pci_msi_shutdown(dev);
 	free_msi_irqs(dev);
-	kset_unregister(dev->msi_kset);
-	dev->msi_kset = NULL;
 }
 EXPORT_SYMBOL(pci_disable_msi);
 
@@ -1023,8 +1023,6 @@ void pci_disable_msix(struct pci_dev *de
 
 	pci_msix_shutdown(dev);
 	free_msi_irqs(dev);
-	kset_unregister(dev->msi_kset);
-	dev->msi_kset = NULL;
 }
 EXPORT_SYMBOL(pci_disable_msix);
 
include/linux/pci.h +1 -1
@@ -351,7 +351,7 @@ struct pci_dev {
 	struct bin_attribute *res_attr_wc[DEVICE_COUNT_RESOURCE]; /* sysfs file for WC mapping of resources */
 #ifdef CONFIG_PCI_MSI
 	struct list_head msi_list;
-	struct kset *msi_kset;
+	const struct attribute_group **msi_irq_groups;
 #endif
 	struct pci_vpd *vpd;
 #ifdef CONFIG_PCI_ATS
Acked-by on v1 1/1
#1 lore raw
Quoted 24 lines · Greg Kroah-Hartman

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

   └── msi_irqs
       └── 40
           └── mode

new MSI attributes:
pci_device

   └── msi_irqs
       └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

Quoted 6 lines

---

Neil, can you test this patch to see if irqbalance works as-is or not?

ACK, Testing with git-head irqbalance on F19 with this patch and it works fine.

Thanks!

Reviewed-by on v1 1/1
#8 lore raw
In reply to Greg Kroah-Hartman
Quoted 17 lines · Greg Kroah-Hartman

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

  └── msi_irqs
      └── 40
          └── mode

new MSI attributes:
pci_device

Quoted 8 lines
  └── msi_irqs
      └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

Works like a charm for me.

FWIW,

Quoted 230 lines

---

Neil, can you test this patch to see if irqbalance works as-is or not?

thanks,

greg k-h

drivers/pci/msi.c | 152 +++++++++++++++++++++++++---------------------------
include/linux/pci.h | 2

2 files changed, 76 insertions(+), 78 deletions(-)
drivers/pci/msi.c +75 -77
@@ -363,6 +363,9 @@ void write_msi_msg(unsigned int irq, str
 static void free_msi_irqs(struct pci_dev *dev)
 {
 	struct msi_desc *entry, *tmp;
+	struct attribute **msi_attrs;
+	struct device_attribute *dev_attr;
+	int count = 0;
 
 	list_for_each_entry(entry, &dev->msi_list, list) {
 		int i, nvec;
@@ -398,6 +401,22 @@ static void free_msi_irqs(struct pci_dev
 		list_del(&entry->list);
 		kfree(entry);
 	}
+
+	if (dev->msi_irq_groups) {
+		sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups);
+		msi_attrs = dev->msi_irq_groups[0]->attrs;
+		list_for_each_entry(entry, &dev->msi_list, list) {
+			dev_attr = container_of(msi_attrs[count],
+						struct device_attribute, attr);
+			kfree(dev_attr->attr.name);
+			kfree(dev_attr);
+			++count;
+		}
+		kfree(msi_attrs);
+		kfree(dev->msi_irq_groups[0]);
+		kfree(dev->msi_irq_groups);
+		dev->msi_irq_groups = NULL;
+	}
 }
 
 static struct msi_desc *alloc_msi_entry(struct pci_dev *dev)
@@ -471,96 +490,79 @@ void pci_restore_msi_state(struct pci_de
 }
 EXPORT_SYMBOL_GPL(pci_restore_msi_state);
 
-
-#define to_msi_attr(obj) container_of(obj, struct msi_attribute, attr)
-#define to_msi_desc(obj) container_of(obj, struct msi_desc, kobj)
-
-struct msi_attribute {
-	struct attribute        attr;
-	ssize_t (*show)(struct msi_desc *entry, struct msi_attribute *attr,
-			char *buf);
-	ssize_t (*store)(struct msi_desc *entry, struct msi_attribute *attr,
-			 const char *buf, size_t count);
-};
-
-static ssize_t show_msi_mode(struct msi_desc *entry, struct msi_attribute *atr,
+static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
 			     char *buf)
 {
-	return sprintf(buf, "%s\n", entry->msi_attrib.is_msix ? "msix" : "msi");
-}
-
-static ssize_t msi_irq_attr_show(struct kobject *kobj,
-				 struct attribute *attr, char *buf)
-{
-	struct msi_attribute *attribute = to_msi_attr(attr);
-	struct msi_desc *entry = to_msi_desc(kobj);
-
-	if (!attribute->show)
-		return -EIO;
-
-	return attribute->show(entry, attribute, buf);
-}
-
-static const struct sysfs_ops msi_irq_sysfs_ops = {
-	.show = msi_irq_attr_show,
-};
-
-static struct msi_attribute mode_attribute =
-	__ATTR(mode, S_IRUGO, show_msi_mode, NULL);
-
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct msi_desc *entry;
+	unsigned long irq;
+	int retval;
 
-static struct attribute *msi_irq_default_attrs[] = {
-	&mode_attribute.attr,
-	NULL
-};
+	retval = kstrtoul(attr->attr.name, 10, &irq);
+	if (retval)
+		return retval;
 
-static void msi_kobj_release(struct kobject *kobj)
-{
-	struct msi_desc *entry = to_msi_desc(kobj);
-
-	pci_dev_put(entry->dev);
+	list_for_each_entry(entry, &pdev->msi_list, list) {
+		if (entry->irq == irq) {
+			return sprintf(buf, "%s\n",
+				       entry->msi_attrib.is_msix ? "msix" : "msi");
+		}
+	}
+	return -ENODEV;
 }
 
-static struct kobj_type msi_irq_ktype = {
-	.release = msi_kobj_release,
-	.sysfs_ops = &msi_irq_sysfs_ops,
-	.default_attrs = msi_irq_default_attrs,
-};
-
 static int populate_msi_sysfs(struct pci_dev *pdev)
 {
+	struct attribute **msi_attrs;
+	struct device_attribute *msi_dev_attr;
+	struct attribute_group *msi_irq_group;
+	const struct attribute_group **msi_irq_groups;
 	struct msi_desc *entry;
-	struct kobject *kobj;
 	int ret;
+	int num_msi = 0;
 	int count = 0;
 
-	pdev->msi_kset = kset_create_and_add("msi_irqs", NULL, &pdev->dev.kobj);
-	if (!pdev->msi_kset)
-		return -ENOMEM;
-
+	/* Determine how many msi entries we have */
 	list_for_each_entry(entry, &pdev->msi_list, list) {
-		kobj = &entry->kobj;
-		kobj->kset = pdev->msi_kset;
-		pci_dev_get(pdev);
-		ret = kobject_init_and_add(kobj, &msi_irq_ktype, NULL,
-				     "%u", entry->irq);
-		if (ret)
-			goto out_unroll;
-
-		count++;
+		++num_msi;
 	}
+	if (!num_msi)
+		return 0;
 
-	return 0;
-
-out_unroll:
+	/* Dynamically create the MSI attributes for the PCI device */
+	msi_attrs = kzalloc(sizeof(void *) * (num_msi + 1), GFP_KERNEL);
+	if (!msi_attrs)
+		return -ENOMEM;
 	list_for_each_entry(entry, &pdev->msi_list, list) {
-		if (!count)
-			break;
-		kobject_del(&entry->kobj);
-		kobject_put(&entry->kobj);
-		count--;
+		char *name = kmalloc(20, GFP_KERNEL);
+		msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
+		if (!msi_dev_attr)
+			return -ENOMEM;
+		sprintf(name, "%d", entry->irq);
+		msi_dev_attr->attr.name = name;
+		msi_dev_attr->attr.mode = S_IRUGO;
+		msi_dev_attr->show = msi_mode_show;
+		msi_attrs[count] = &msi_dev_attr->attr;
+		++count;
 	}
-	return ret;
+
+	msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
+	if (!msi_irq_group)
+		return -ENOMEM;
+	msi_irq_group->name = "msi_irqs";
+	msi_irq_group->attrs = msi_attrs;
+
+	msi_irq_groups = kzalloc(sizeof(void *) * 2, GFP_KERNEL);
+	if (!msi_irq_groups)
+		return -ENOMEM;
+	msi_irq_groups[0] = msi_irq_group;
+
+	ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups);
+	if (ret)
+		return ret;
+	pdev->msi_irq_groups = msi_irq_groups;
+
+	return 0;
 }
 
 /**
@@ -925,8 +927,6 @@ void pci_disable_msi(struct pci_dev *dev
 
 	pci_msi_shutdown(dev);
 	free_msi_irqs(dev);
-	kset_unregister(dev->msi_kset);
-	dev->msi_kset = NULL;
 }
 EXPORT_SYMBOL(pci_disable_msi);
 
@@ -1023,8 +1023,6 @@ void pci_disable_msix(struct pci_dev *de
 
 	pci_msix_shutdown(dev);
 	free_msi_irqs(dev);
-	kset_unregister(dev->msi_kset);
-	dev->msi_kset = NULL;
 }
 EXPORT_SYMBOL(pci_disable_msix);
 
include/linux/pci.h +1 -1
@@ -351,7 +351,7 @@ struct pci_dev {
 	struct bin_attribute *res_attr_wc[DEVICE_COUNT_RESOURCE]; /* sysfs file for WC mapping of resources */
 #ifdef CONFIG_PCI_MSI
 	struct list_head msi_list;
-	struct kset *msi_kset;
+	const struct attribute_group **msi_irq_groups;
 #endif
 	struct pci_vpd *vpd;
 #ifdef CONFIG_PCI_ATS
#2 lore raw
In reply to Neil Horman
Quoted 32 lines · Veaceslav Falico
Quoted 17 lines · Greg Kroah-Hartman

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

  └── msi_irqs
      └── 40
          └── mode

new MSI attributes:
pci_device

Quoted 8 lines
  └── msi_irqs
      └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

Works like a charm for me.

FWIW,

Quoted 4 lines · Neil Horman

ACK, Testing with git-head irqbalance on F19 with this patch and it works fine.

Thanks!

Great, thanks for both of you testing this.

Bjorn, any objection to take this through your tree for 3.14-rc1?

thanks,

gre k-h

maintainer
#3 lore raw
Quoted 35 lines · Greg KH
Quoted 25 lines · Veaceslav Falico
Quoted 24 lines · Greg Kroah-Hartman

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

  └── msi_irqs
      └── 40
          └── mode

new MSI attributes:
pci_device

  └── msi_irqs
      └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

Quoted 7 lines
Quoted 1 line

Works like a charm for me.

FWIW,

On Thu, Nov 28, 2013 at 10:14:19PM -0500, Neil Horman wrote:

Quoted 8 lines
Quoted 4 lines

ACK, Testing with git-head irqbalance on F19 with this patch and it works fine.

Thanks!

Great, thanks for both of you testing this.

Bjorn, any objection to take this through your tree for 3.14-rc1?

Nope, I don't object at all. We probably should update Documentation/ABI/testing/sysfs-bus-pci at the same time, though.

Bjorn

#4 lore raw

PATCH 1/1 PCI: update MSI attribute file documentation

From: Greg Kroah-Hartman <[email protected]>

Now that the MSI modes are now attribute files in sysfs, not subdirectories, update the documentation to reflect this.

Documentation/ABI/testing/sysfs-bus-pci +4 -7
index 5210a51c90fd..a3c5a6685036 100644
@@ -70,18 +70,15 @@ Date:		September, 2011
 Contact:	Neil Horman <[email protected]>
 Description:
 		The /sys/devices/.../msi_irqs directory contains a variable set
-		of sub-directories, with each sub-directory being named after a
-		corresponding msi irq vector allocated to that device.  Each
-		numbered sub-directory N contains attributes of that irq.
-		Note that this directory is not created for device drivers which
-		do not support msi irqs
+		of files, with each file being named after a corresponding msi
+		irq vector allocated to that device.
 
-What:		/sys/bus/pci/devices/.../msi_irqs/<N>/mode
+What:		/sys/bus/pci/devices/.../msi_irqs/<N>
 Date:		September 2011
 Contact:	Neil Horman <[email protected]>
 Description:
 		This attribute indicates the mode that the irq vector named by
-		the parent directory is in (msi vs. msix)
+		the file is in (msi vs. msix)
 
 What:		/sys/bus/pci/devices/.../remove
 Date:		January 2009
#7 lore raw
In reply to Bjorn Helgaas
Quoted 41 lines · Bjorn Helgaas
Quoted 40 lines · Greg KH
Quoted 32 lines · Veaceslav Falico
Quoted 17 lines · Greg Kroah-Hartman

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

  └── msi_irqs
      └── 40
          └── mode

new MSI attributes:
pci_device

Quoted 8 lines
  └── msi_irqs
      └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

Works like a charm for me.

FWIW,

Quoted 4 lines · Neil Horman

ACK, Testing with git-head irqbalance on F19 with this patch and it works fine.

Thanks!

Quoted 6 lines
Quoted 3 lines

Great, thanks for both of you testing this.

Bjorn, any objection to take this through your tree for 3.14-rc1?

Nope, I don't object at all. We probably should update Documentation/ABI/testing/sysfs-bus-pci at the same time, though.

Now sent.

thanks,

greg k-h

maintainer
#5 lore raw

Re: [PATCH] PCI: update MSI attribute file documentation

In reply to Greg KH
Quoted 6 lines · Greg KH

From: Greg Kroah-Hartman <[email protected]>

Now that the MSI modes are now attribute files in sysfs, not subdirectories, update the documentation to reflect this.

Thanks, I'll fold this into the patch that makes the code change, if you don't mind.

Bjorn

Quoted 27 lines
Documentation/ABI/testing/sysfs-bus-pci +4 -7
index 5210a51c90fd..a3c5a6685036 100644
@@ -70,18 +70,15 @@ Date:               September, 2011
 Contact:       Neil Horman <[email protected]>
 Description:
                The /sys/devices/.../msi_irqs directory contains a variable set
-               of sub-directories, with each sub-directory being named after a
-               corresponding msi irq vector allocated to that device.  Each
-               numbered sub-directory N contains attributes of that irq.
-               Note that this directory is not created for device drivers which
-               do not support msi irqs
+               of files, with each file being named after a corresponding msi
+               irq vector allocated to that device.
 
-What:          /sys/bus/pci/devices/.../msi_irqs/<N>/mode
+What:          /sys/bus/pci/devices/.../msi_irqs/<N>
 Date:          September 2011
 Contact:       Neil Horman <[email protected]>
 Description:
                This attribute indicates the mode that the irq vector named by
-               the parent directory is in (msi vs. msix)
+               the file is in (msi vs. msix)
 
 What:          /sys/bus/pci/devices/.../remove
 Date:          January 2009
#6 lore raw

Re: [PATCH] PCI: update MSI attribute file documentation

Quoted 10 lines · Bjorn Helgaas
Quoted 6 lines · Greg KH

From: Greg Kroah-Hartman <[email protected]>

Now that the MSI modes are now attribute files in sysfs, not subdirectories, update the documentation to reflect this.

Thanks, I'll fold this into the patch that makes the code change, if you don't mind.

I don't mind at all, thanks.

maintainer
#9 lore raw
In reply to Greg Kroah-Hartman
Quoted 17 lines · Greg Kroah-Hartman

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

   └── msi_irqs
       └── 40
           └── mode

new MSI attributes:
pci_device

Quoted 8 lines
   └── msi_irqs
       └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

I added the acks from Neil and Veaceslav, folded in the Documentation/ABI update, and applied the whole thing to my pci/misc branch for v3.14. Thanks!

Bjorn

Quoted 229 lines

---

Neil, can you test this patch to see if irqbalance works as-is or not?

thanks,

greg k-h

 drivers/pci/msi.c   |  152 +++++++++++++++++++++++++---------------------------
 include/linux/pci.h |    2
 2 files changed, 76 insertions(+), 78 deletions(-)
drivers/pci/msi.c +75 -77
@@ -363,6 +363,9 @@ void write_msi_msg(unsigned int irq, str
 static void free_msi_irqs(struct pci_dev *dev)
 {
        struct msi_desc *entry, *tmp;
+       struct attribute **msi_attrs;
+       struct device_attribute *dev_attr;
+       int count = 0;
 
        list_for_each_entry(entry, &dev->msi_list, list) {
                int i, nvec;
@@ -398,6 +401,22 @@ static void free_msi_irqs(struct pci_dev
                list_del(&entry->list);
                kfree(entry);
        }
+
+       if (dev->msi_irq_groups) {
+               sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups);
+               msi_attrs = dev->msi_irq_groups[0]->attrs;
+               list_for_each_entry(entry, &dev->msi_list, list) {
+                       dev_attr = container_of(msi_attrs[count],
+                                               struct device_attribute, attr);
+                       kfree(dev_attr->attr.name);
+                       kfree(dev_attr);
+                       ++count;
+               }
+               kfree(msi_attrs);
+               kfree(dev->msi_irq_groups[0]);
+               kfree(dev->msi_irq_groups);
+               dev->msi_irq_groups = NULL;
+       }
 }
 
 static struct msi_desc *alloc_msi_entry(struct pci_dev *dev)
@@ -471,96 +490,79 @@ void pci_restore_msi_state(struct pci_de
 }
 EXPORT_SYMBOL_GPL(pci_restore_msi_state);
 
-
-#define to_msi_attr(obj) container_of(obj, struct msi_attribute, attr)
-#define to_msi_desc(obj) container_of(obj, struct msi_desc, kobj)
-
-struct msi_attribute {
-       struct attribute        attr;
-       ssize_t (*show)(struct msi_desc *entry, struct msi_attribute *attr,
-                       char *buf);
-       ssize_t (*store)(struct msi_desc *entry, struct msi_attribute *attr,
-                        const char *buf, size_t count);
-};
-
-static ssize_t show_msi_mode(struct msi_desc *entry, struct msi_attribute *atr,
+static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
                             char *buf)
 {
-       return sprintf(buf, "%s\n", entry->msi_attrib.is_msix ? "msix" : "msi");
-}
-
-static ssize_t msi_irq_attr_show(struct kobject *kobj,
-                                struct attribute *attr, char *buf)
-{
-       struct msi_attribute *attribute = to_msi_attr(attr);
-       struct msi_desc *entry = to_msi_desc(kobj);
-
-       if (!attribute->show)
-               return -EIO;
-
-       return attribute->show(entry, attribute, buf);
-}
-
-static const struct sysfs_ops msi_irq_sysfs_ops = {
-       .show = msi_irq_attr_show,
-};
-
-static struct msi_attribute mode_attribute =
-       __ATTR(mode, S_IRUGO, show_msi_mode, NULL);
-
+       struct pci_dev *pdev = to_pci_dev(dev);
+       struct msi_desc *entry;
+       unsigned long irq;
+       int retval;
 
-static struct attribute *msi_irq_default_attrs[] = {
-       &mode_attribute.attr,
-       NULL
-};
+       retval = kstrtoul(attr->attr.name, 10, &irq);
+       if (retval)
+               return retval;
 
-static void msi_kobj_release(struct kobject *kobj)
-{
-       struct msi_desc *entry = to_msi_desc(kobj);
-
-       pci_dev_put(entry->dev);
+       list_for_each_entry(entry, &pdev->msi_list, list) {
+               if (entry->irq == irq) {
+                       return sprintf(buf, "%s\n",
+                                      entry->msi_attrib.is_msix ? "msix" : "msi");
+               }
+       }
+       return -ENODEV;
 }
 
-static struct kobj_type msi_irq_ktype = {
-       .release = msi_kobj_release,
-       .sysfs_ops = &msi_irq_sysfs_ops,
-       .default_attrs = msi_irq_default_attrs,
-};
-
 static int populate_msi_sysfs(struct pci_dev *pdev)
 {
+       struct attribute **msi_attrs;
+       struct device_attribute *msi_dev_attr;
+       struct attribute_group *msi_irq_group;
+       const struct attribute_group **msi_irq_groups;
        struct msi_desc *entry;
-       struct kobject *kobj;
        int ret;
+       int num_msi = 0;
        int count = 0;
 
-       pdev->msi_kset = kset_create_and_add("msi_irqs", NULL, &pdev->dev.kobj);
-       if (!pdev->msi_kset)
-               return -ENOMEM;
-
+       /* Determine how many msi entries we have */
        list_for_each_entry(entry, &pdev->msi_list, list) {
-               kobj = &entry->kobj;
-               kobj->kset = pdev->msi_kset;
-               pci_dev_get(pdev);
-               ret = kobject_init_and_add(kobj, &msi_irq_ktype, NULL,
-                                    "%u", entry->irq);
-               if (ret)
-                       goto out_unroll;
-
-               count++;
+               ++num_msi;
        }
+       if (!num_msi)
+               return 0;
 
-       return 0;
-
-out_unroll:
+       /* Dynamically create the MSI attributes for the PCI device */
+       msi_attrs = kzalloc(sizeof(void *) * (num_msi + 1), GFP_KERNEL);
+       if (!msi_attrs)
+               return -ENOMEM;
        list_for_each_entry(entry, &pdev->msi_list, list) {
-               if (!count)
-                       break;
-               kobject_del(&entry->kobj);
-               kobject_put(&entry->kobj);
-               count--;
+               char *name = kmalloc(20, GFP_KERNEL);
+               msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
+               if (!msi_dev_attr)
+                       return -ENOMEM;
+               sprintf(name, "%d", entry->irq);
+               msi_dev_attr->attr.name = name;
+               msi_dev_attr->attr.mode = S_IRUGO;
+               msi_dev_attr->show = msi_mode_show;
+               msi_attrs[count] = &msi_dev_attr->attr;
+               ++count;
        }
-       return ret;
+
+       msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
+       if (!msi_irq_group)
+               return -ENOMEM;
+       msi_irq_group->name = "msi_irqs";
+       msi_irq_group->attrs = msi_attrs;
+
+       msi_irq_groups = kzalloc(sizeof(void *) * 2, GFP_KERNEL);
+       if (!msi_irq_groups)
+               return -ENOMEM;
+       msi_irq_groups[0] = msi_irq_group;
+
+       ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups);
+       if (ret)
+               return ret;
+       pdev->msi_irq_groups = msi_irq_groups;
+
+       return 0;
 }
 
 /**
@@ -925,8 +927,6 @@ void pci_disable_msi(struct pci_dev *dev
 
        pci_msi_shutdown(dev);
        free_msi_irqs(dev);
-       kset_unregister(dev->msi_kset);
-       dev->msi_kset = NULL;
 }
 EXPORT_SYMBOL(pci_disable_msi);
 
@@ -1023,8 +1023,6 @@ void pci_disable_msix(struct pci_dev *de
 
        pci_msix_shutdown(dev);
        free_msi_irqs(dev);
-       kset_unregister(dev->msi_kset);
-       dev->msi_kset = NULL;
 }
 EXPORT_SYMBOL(pci_disable_msix);
 
include/linux/pci.h +1 -1
@@ -351,7 +351,7 @@ struct pci_dev {
        struct bin_attribute *res_attr_wc[DEVICE_COUNT_RESOURCE]; /* sysfs file for WC mapping of resources */
 #ifdef CONFIG_PCI_MSI
        struct list_head msi_list;
-       struct kset *msi_kset;
+       const struct attribute_group **msi_irq_groups;
 #endif
        struct pci_vpd *vpd;
 #ifdef CONFIG_PCI_ATS
#10 lore raw
Quoted 26 lines · Bjorn Helgaas
Quoted 24 lines · Greg Kroah-Hartman

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

   └── msi_irqs
       └── 40
           └── mode

new MSI attributes:
pci_device

   └── msi_irqs
       └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

Quoted 5 lines
Quoted 1 line

I added the acks from Neil and Veaceslav, folded in the Documentation/ABI update, and applied the whole thing to my pci/misc branch for v3.14. Thanks!

got:

[   71.429735] BUG: key ffff887fcf082a58 not in .data!
[   71.429737] ------------[ cut here ]------------
[   71.429742] WARNING: CPU: 0 PID: 4 at kernel/locking/lockdep.c:2987 lockdep_i
nit_map+0x127/0x5b0()
[   71.429743] DEBUG_LOCKS_WARN_ON(1)
[   71.429744] Modules linked in:
[   71.429747] CPU: 0 PID: 4 Comm: kworker/0:0 Tainted: G          I  3.13.0-rc3
-yh-01187-ge0e4e4e-dirty #45
[   71.429757] Workqueue: events work_for_cpu_fn
[   71.429792]  0000000000000009 ffff881fcf95da58 ffffffff82020475 ffff881fcf95d
aa0
[   71.429800]  ffff881fcf95da90 ffffffff81097c3d ffff887fcf082a58 ffff88dfcec1a
d28
[   71.429809]  0000000000000000 0000000000000000 ffff889fcf2eecd0 ffff881fcf95d
af0
[   71.429809] Call Trace:
[   71.429814]  [<ffffffff82020475>] dump_stack+0x45/0x56
[   71.429818]  [<ffffffff81097c3d>] warn_slowpath_common+0x7d/0xa0
[   71.429822]  [<ffffffff81097cac>] warn_slowpath_fmt+0x4c/0x50
[   71.429826]  [<ffffffff810e3d97>] lockdep_init_map+0x127/0x5b0
[   71.429835]  [<ffffffff81242fe8>] ? sysfs_new_dirent+0x98/0x140
[   71.429839]  [<ffffffff81242033>] sysfs_add_file_mode_ns+0x63/0xc0
[   71.429843]  [<ffffffff8124491d>] internal_create_group+0x18d/0x260
[   71.429853]  [<ffffffff81556345>] ? populate_msi_sysfs+0x185/0x1d0
[   71.429857]  [<ffffffff81244a52>] sysfs_create_groups+0x42/0xa0
[   71.429861]  [<ffffffff81556367>] populate_msi_sysfs+0x1a7/0x1d0
[   71.429865]  [<ffffffff81556b67>] pci_enable_msi_block+0x1f7/0x2a0
[   71.429870]  [<ffffffff81549795>] pcie_port_device_register+0x335/0x520
[   71.429874]  [<ffffffff81549f08>] pcie_portdrv_probe+0x68/0xa0
[   71.429883]  [<ffffffff8153cf75>] local_pci_probe+0x45/0xa0
[   71.429887]  [<ffffffff810b0af4>] work_for_cpu_fn+0x14/0x20
[   71.429891]  [<ffffffff810b431b>] process_one_work+0x28b/0x4a0
[   71.429895]  [<ffffffff810b4292>] ? process_one_work+0x202/0x4a0
[   71.429899]  [<ffffffff810b55db>] worker_thread+0x26b/0x3a0
[   71.429903]  [<ffffffff810e262d>] ? trace_hardirqs_on+0xd/0x10
[   71.429906]  [<ffffffff810b5370>] ? manage_workers.isra.17+0x340/0x340
[   71.429912]  [<ffffffff810bc031>] kthread+0x111/0x120
[   71.429919]  [<ffffffff810cddab>] ? local_clock+0x2b/0x40
[   71.429923]  [<ffffffff810bbf20>] ? kthread_stop+0xf0/0xf0
[   71.429927]  [<ffffffff8203842c>] ret_from_fork+0x7c/0xb0
[   71.429931]  [<ffffffff810bbf20>] ? kthread_stop+0xf0/0xf0
[   71.429933] ---[ end trace c511e3d74efea94e ]---

looks like Greg forgot adding attr init.

Can you fold attached patch into Greg's patch ?

drivers/pci/msi.c +1 -0
Index: linux-2.6/drivers/pci/msi.c
===================================================================
@@ -547,6 +547,7 @@ static int populate_msi_sysfs(struct pci
                msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
                if (!msi_dev_attr)
                        return -ENOMEM;
+               sysfs_attr_init(&msi_dev_attr->attr);
                sprintf(name, "%d", entry->irq);
                msi_dev_attr->attr.name = name;
                msi_dev_attr->attr.mode = S_IRUGO;
---
 drivers/pci/msi.c |    1 +
 1 file changed, 1 insertion(+)
drivers/pci/msi.c +1 -0
Index: linux-2.6/drivers/pci/msi.c
===================================================================
@@ -547,6 +547,7 @@ static int populate_msi_sysfs(struct pci
 		msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
 		if (!msi_dev_attr)
 			return -ENOMEM;
+		sysfs_attr_init(&msi_dev_attr->attr);
 		sprintf(name, "%d", entry->irq);
 		msi_dev_attr->attr.name = name;
 		msi_dev_attr->attr.mode = S_IRUGO;
author Acked-by
#11 lore raw
Quoted 103 lines · Yinghai Lu
Quoted 26 lines · Bjorn Helgaas
Quoted 24 lines · Greg Kroah-Hartman

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

   └── msi_irqs
       └── 40
           └── mode

new MSI attributes:
pci_device

   └── msi_irqs
       └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

Quoted 5 lines
Quoted 1 line

I added the acks from Neil and Veaceslav, folded in the Documentation/ABI update, and applied the whole thing to my pci/misc branch for v3.14. Thanks!

got:

[   71.429735] BUG: key ffff887fcf082a58 not in .data!
[   71.429737] ------------[ cut here ]------------
[   71.429742] WARNING: CPU: 0 PID: 4 at kernel/locking/lockdep.c:2987 lockdep_i
nit_map+0x127/0x5b0()
[   71.429743] DEBUG_LOCKS_WARN_ON(1)
[   71.429744] Modules linked in:
[   71.429747] CPU: 0 PID: 4 Comm: kworker/0:0 Tainted: G          I  3.13.0-rc3
-yh-01187-ge0e4e4e-dirty #45
[   71.429757] Workqueue: events work_for_cpu_fn
[   71.429792]  0000000000000009 ffff881fcf95da58 ffffffff82020475 ffff881fcf95d
aa0
[   71.429800]  ffff881fcf95da90 ffffffff81097c3d ffff887fcf082a58 ffff88dfcec1a
d28
[   71.429809]  0000000000000000 0000000000000000 ffff889fcf2eecd0 ffff881fcf95d
af0
[   71.429809] Call Trace:
[   71.429814]  [<ffffffff82020475>] dump_stack+0x45/0x56
[   71.429818]  [<ffffffff81097c3d>] warn_slowpath_common+0x7d/0xa0
[   71.429822]  [<ffffffff81097cac>] warn_slowpath_fmt+0x4c/0x50
[   71.429826]  [<ffffffff810e3d97>] lockdep_init_map+0x127/0x5b0
[   71.429835]  [<ffffffff81242fe8>] ? sysfs_new_dirent+0x98/0x140
[   71.429839]  [<ffffffff81242033>] sysfs_add_file_mode_ns+0x63/0xc0
[   71.429843]  [<ffffffff8124491d>] internal_create_group+0x18d/0x260
[   71.429853]  [<ffffffff81556345>] ? populate_msi_sysfs+0x185/0x1d0
[   71.429857]  [<ffffffff81244a52>] sysfs_create_groups+0x42/0xa0
[   71.429861]  [<ffffffff81556367>] populate_msi_sysfs+0x1a7/0x1d0
[   71.429865]  [<ffffffff81556b67>] pci_enable_msi_block+0x1f7/0x2a0
[   71.429870]  [<ffffffff81549795>] pcie_port_device_register+0x335/0x520
[   71.429874]  [<ffffffff81549f08>] pcie_portdrv_probe+0x68/0xa0
[   71.429883]  [<ffffffff8153cf75>] local_pci_probe+0x45/0xa0
[   71.429887]  [<ffffffff810b0af4>] work_for_cpu_fn+0x14/0x20
[   71.429891]  [<ffffffff810b431b>] process_one_work+0x28b/0x4a0
[   71.429895]  [<ffffffff810b4292>] ? process_one_work+0x202/0x4a0
[   71.429899]  [<ffffffff810b55db>] worker_thread+0x26b/0x3a0
[   71.429903]  [<ffffffff810e262d>] ? trace_hardirqs_on+0xd/0x10
[   71.429906]  [<ffffffff810b5370>] ? manage_workers.isra.17+0x340/0x340
[   71.429912]  [<ffffffff810bc031>] kthread+0x111/0x120
[   71.429919]  [<ffffffff810cddab>] ? local_clock+0x2b/0x40
[   71.429923]  [<ffffffff810bbf20>] ? kthread_stop+0xf0/0xf0
[   71.429927]  [<ffffffff8203842c>] ret_from_fork+0x7c/0xb0
[   71.429931]  [<ffffffff810bbf20>] ? kthread_stop+0xf0/0xf0
[   71.429933] ---[ end trace c511e3d74efea94e ]---

looks like Greg forgot adding attr init.

Can you fold attached patch into Greg's patch ?

Index: linux-2.6/drivers/pci/msi.c
===================================================================

drivers/pci/msi.c +1 -1
@@ -547,6 +547,7 @@ static int populate_msi_sysfs(struct pci
                msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
                if (!msi_dev_attr)
                        return -ENOMEM;
+               sysfs_attr_init(&msi_dev_attr->attr);
                sprintf(name, "%d", entry->irq);
                msi_dev_attr->attr.name = name;
                msi_dev_attr->attr.mode = S_IRUGO;
 
---
 drivers/pci/msi.c |    1 +
 1 file changed, 1 insertion(+)

Index: linux-2.6/drivers/pci/msi.c
===================================================================

drivers/pci/msi.c static int populate_msi_sysfs(struct pci · line 553

Quoted 7 lines
drivers/pci/msi.c +1 -0
@@ -547,6 +547,7 @@ static int populate_msi_sysfs(struct pci
 		msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
 		if (!msi_dev_attr)
 			return -ENOMEM;
+		sysfs_attr_init(&msi_dev_attr->attr);
 		sprintf(name, "%d", entry->irq);
 		msi_dev_attr->attr.name = name;
 		msi_dev_attr->attr.mode = S_IRUGO;

Ah, crap, yes, you are right, my fault, very nice catch.

greg k-h

maintainer
#12 lore raw
In reply to Yinghai Lu
Quoted 77 lines · Yinghai Lu
Quoted 26 lines · Bjorn Helgaas
Quoted 24 lines · Greg Kroah-Hartman

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

   └── msi_irqs
       └── 40
           └── mode

new MSI attributes:
pci_device

   └── msi_irqs
       └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

Quoted 5 lines
Quoted 1 line

I added the acks from Neil and Veaceslav, folded in the Documentation/ABI update, and applied the whole thing to my pci/misc branch for v3.14. Thanks!

got:

[   71.429735] BUG: key ffff887fcf082a58 not in .data!
[   71.429737] ------------[ cut here ]------------
[   71.429742] WARNING: CPU: 0 PID: 4 at kernel/locking/lockdep.c:2987 lockdep_i
nit_map+0x127/0x5b0()
[   71.429743] DEBUG_LOCKS_WARN_ON(1)
[   71.429744] Modules linked in:
[   71.429747] CPU: 0 PID: 4 Comm: kworker/0:0 Tainted: G          I  3.13.0-rc3
-yh-01187-ge0e4e4e-dirty #45
[   71.429757] Workqueue: events work_for_cpu_fn
[   71.429792]  0000000000000009 ffff881fcf95da58 ffffffff82020475 ffff881fcf95d
aa0
[   71.429800]  ffff881fcf95da90 ffffffff81097c3d ffff887fcf082a58 ffff88dfcec1a
d28
[   71.429809]  0000000000000000 0000000000000000 ffff889fcf2eecd0 ffff881fcf95d
af0
[   71.429809] Call Trace:
[   71.429814]  [<ffffffff82020475>] dump_stack+0x45/0x56
[   71.429818]  [<ffffffff81097c3d>] warn_slowpath_common+0x7d/0xa0
[   71.429822]  [<ffffffff81097cac>] warn_slowpath_fmt+0x4c/0x50
[   71.429826]  [<ffffffff810e3d97>] lockdep_init_map+0x127/0x5b0
[   71.429835]  [<ffffffff81242fe8>] ? sysfs_new_dirent+0x98/0x140
[   71.429839]  [<ffffffff81242033>] sysfs_add_file_mode_ns+0x63/0xc0
[   71.429843]  [<ffffffff8124491d>] internal_create_group+0x18d/0x260
[   71.429853]  [<ffffffff81556345>] ? populate_msi_sysfs+0x185/0x1d0
[   71.429857]  [<ffffffff81244a52>] sysfs_create_groups+0x42/0xa0
[   71.429861]  [<ffffffff81556367>] populate_msi_sysfs+0x1a7/0x1d0
[   71.429865]  [<ffffffff81556b67>] pci_enable_msi_block+0x1f7/0x2a0
[   71.429870]  [<ffffffff81549795>] pcie_port_device_register+0x335/0x520
[   71.429874]  [<ffffffff81549f08>] pcie_portdrv_probe+0x68/0xa0
[   71.429883]  [<ffffffff8153cf75>] local_pci_probe+0x45/0xa0
[   71.429887]  [<ffffffff810b0af4>] work_for_cpu_fn+0x14/0x20
[   71.429891]  [<ffffffff810b431b>] process_one_work+0x28b/0x4a0
[   71.429895]  [<ffffffff810b4292>] ? process_one_work+0x202/0x4a0
[   71.429899]  [<ffffffff810b55db>] worker_thread+0x26b/0x3a0
[   71.429903]  [<ffffffff810e262d>] ? trace_hardirqs_on+0xd/0x10
[   71.429906]  [<ffffffff810b5370>] ? manage_workers.isra.17+0x340/0x340
[   71.429912]  [<ffffffff810bc031>] kthread+0x111/0x120
[   71.429919]  [<ffffffff810cddab>] ? local_clock+0x2b/0x40
[   71.429923]  [<ffffffff810bbf20>] ? kthread_stop+0xf0/0xf0
[   71.429927]  [<ffffffff8203842c>] ret_from_fork+0x7c/0xb0
[   71.429931]  [<ffffffff810bbf20>] ? kthread_stop+0xf0/0xf0
[   71.429933] ---[ end trace c511e3d74efea94e ]---
Quoted 3 lines

looks like Greg forgot adding attr init.

Can you fold attached patch into Greg's patch ?

Sure. I don't know how to reproduce it, so I couldn't verify the fix, but I added it to Greg's patch. Thanks!

Quoted 12 lines

Index: linux-2.6/drivers/pci/msi.c
===================================================================

drivers/pci/msi.c +1 -0
@@ -547,6 +547,7 @@ static int populate_msi_sysfs(struct pci
                msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
                if (!msi_dev_attr)
                        return -ENOMEM;
+               sysfs_attr_init(&msi_dev_attr->attr);
                sprintf(name, "%d", entry->irq);
                msi_dev_attr->attr.name = name;
                msi_dev_attr->attr.mode = S_IRUGO;
#13 lore raw
Quoted 78 lines · Bjorn Helgaas
Quoted 77 lines · Yinghai Lu
Quoted 26 lines · Bjorn Helgaas
Quoted 24 lines · Greg Kroah-Hartman

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

   └── msi_irqs
       └── 40
           └── mode

new MSI attributes:
pci_device

   └── msi_irqs
       └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

Quoted 5 lines
Quoted 1 line

I added the acks from Neil and Veaceslav, folded in the Documentation/ABI update, and applied the whole thing to my pci/misc branch for v3.14. Thanks!

got:

[   71.429735] BUG: key ffff887fcf082a58 not in .data!
[   71.429737] ------------[ cut here ]------------
[   71.429742] WARNING: CPU: 0 PID: 4 at kernel/locking/lockdep.c:2987 lockdep_i
nit_map+0x127/0x5b0()
[   71.429743] DEBUG_LOCKS_WARN_ON(1)
[   71.429744] Modules linked in:
[   71.429747] CPU: 0 PID: 4 Comm: kworker/0:0 Tainted: G          I  3.13.0-rc3
-yh-01187-ge0e4e4e-dirty #45
[   71.429757] Workqueue: events work_for_cpu_fn
[   71.429792]  0000000000000009 ffff881fcf95da58 ffffffff82020475 ffff881fcf95d
aa0
[   71.429800]  ffff881fcf95da90 ffffffff81097c3d ffff887fcf082a58 ffff88dfcec1a
d28
[   71.429809]  0000000000000000 0000000000000000 ffff889fcf2eecd0 ffff881fcf95d
af0
[   71.429809] Call Trace:
[   71.429814]  [<ffffffff82020475>] dump_stack+0x45/0x56
[   71.429818]  [<ffffffff81097c3d>] warn_slowpath_common+0x7d/0xa0
[   71.429822]  [<ffffffff81097cac>] warn_slowpath_fmt+0x4c/0x50
[   71.429826]  [<ffffffff810e3d97>] lockdep_init_map+0x127/0x5b0
[   71.429835]  [<ffffffff81242fe8>] ? sysfs_new_dirent+0x98/0x140
[   71.429839]  [<ffffffff81242033>] sysfs_add_file_mode_ns+0x63/0xc0
[   71.429843]  [<ffffffff8124491d>] internal_create_group+0x18d/0x260
[   71.429853]  [<ffffffff81556345>] ? populate_msi_sysfs+0x185/0x1d0
[   71.429857]  [<ffffffff81244a52>] sysfs_create_groups+0x42/0xa0
[   71.429861]  [<ffffffff81556367>] populate_msi_sysfs+0x1a7/0x1d0
[   71.429865]  [<ffffffff81556b67>] pci_enable_msi_block+0x1f7/0x2a0
[   71.429870]  [<ffffffff81549795>] pcie_port_device_register+0x335/0x520
[   71.429874]  [<ffffffff81549f08>] pcie_portdrv_probe+0x68/0xa0
[   71.429883]  [<ffffffff8153cf75>] local_pci_probe+0x45/0xa0
[   71.429887]  [<ffffffff810b0af4>] work_for_cpu_fn+0x14/0x20
[   71.429891]  [<ffffffff810b431b>] process_one_work+0x28b/0x4a0
[   71.429895]  [<ffffffff810b4292>] ? process_one_work+0x202/0x4a0
[   71.429899]  [<ffffffff810b55db>] worker_thread+0x26b/0x3a0
[   71.429903]  [<ffffffff810e262d>] ? trace_hardirqs_on+0xd/0x10
[   71.429906]  [<ffffffff810b5370>] ? manage_workers.isra.17+0x340/0x340
[   71.429912]  [<ffffffff810bc031>] kthread+0x111/0x120
[   71.429919]  [<ffffffff810cddab>] ? local_clock+0x2b/0x40
[   71.429923]  [<ffffffff810bbf20>] ? kthread_stop+0xf0/0xf0
[   71.429927]  [<ffffffff8203842c>] ret_from_fork+0x7c/0xb0
[   71.429931]  [<ffffffff810bbf20>] ? kthread_stop+0xf0/0xf0
[   71.429933] ---[ end trace c511e3d74efea94e ]---
Quoted 6 lines
Quoted 3 lines

looks like Greg forgot adding attr init.

Can you fold attached patch into Greg's patch ?

Sure. I don't know how to reproduce it, so I couldn't verify the fix, but I added it to Greg's patch. Thanks!

I think lockdep has to be enabled for this to show up, I wasn't running with that enabled when I tested the code, my fault, sorry.

greg k-h

maintainer
#14 lore raw
Quoted 81 lines · Greg Kroah-Hartman
Quoted 80 lines · Bjorn Helgaas
Quoted 79 lines · Yinghai Lu
Quoted 26 lines · Bjorn Helgaas
Quoted 24 lines · Greg Kroah-Hartman

From: Greg Kroah-Hartman <[email protected]>

The PCI MSI sysfs code is a mess with kobjects for things that don't really need to be kobjects. This patch creates attributes dynamically for the MSI interrupts instead of using kobjects.

Note, this removes a directory from the current MSI interrupt sysfs code:

old MSI kobjects:
pci_device

   └── msi_irqs
       └── 40
           └── mode

new MSI attributes:
pci_device

   └── msi_irqs
       └── 40

As there was only one file "mode" with the kobject model, the interrupt number is now a file that returns the "mode" of the interrupt (msi vs. msix).

Quoted 5 lines
Quoted 1 line

I added the acks from Neil and Veaceslav, folded in the Documentation/ABI update, and applied the whole thing to my pci/misc branch for v3.14. Thanks!

got:

[   71.429735] BUG: key ffff887fcf082a58 not in .data!
[   71.429737] ------------[ cut here ]------------
[   71.429742] WARNING: CPU: 0 PID: 4 at kernel/locking/lockdep.c:2987 lockdep_i
nit_map+0x127/0x5b0()
[   71.429743] DEBUG_LOCKS_WARN_ON(1)
[   71.429744] Modules linked in:
[   71.429747] CPU: 0 PID: 4 Comm: kworker/0:0 Tainted: G          I  3.13.0-rc3
-yh-01187-ge0e4e4e-dirty #45
[   71.429757] Workqueue: events work_for_cpu_fn
[   71.429792]  0000000000000009 ffff881fcf95da58 ffffffff82020475 ffff881fcf95d
aa0
[   71.429800]  ffff881fcf95da90 ffffffff81097c3d ffff887fcf082a58 ffff88dfcec1a
d28
[   71.429809]  0000000000000000 0000000000000000 ffff889fcf2eecd0 ffff881fcf95d
af0
[   71.429809] Call Trace:
[   71.429814]  [<ffffffff82020475>] dump_stack+0x45/0x56
[   71.429818]  [<ffffffff81097c3d>] warn_slowpath_common+0x7d/0xa0
[   71.429822]  [<ffffffff81097cac>] warn_slowpath_fmt+0x4c/0x50
[   71.429826]  [<ffffffff810e3d97>] lockdep_init_map+0x127/0x5b0
[   71.429835]  [<ffffffff81242fe8>] ? sysfs_new_dirent+0x98/0x140
[   71.429839]  [<ffffffff81242033>] sysfs_add_file_mode_ns+0x63/0xc0
[   71.429843]  [<ffffffff8124491d>] internal_create_group+0x18d/0x260
[   71.429853]  [<ffffffff81556345>] ? populate_msi_sysfs+0x185/0x1d0
[   71.429857]  [<ffffffff81244a52>] sysfs_create_groups+0x42/0xa0
[   71.429861]  [<ffffffff81556367>] populate_msi_sysfs+0x1a7/0x1d0
[   71.429865]  [<ffffffff81556b67>] pci_enable_msi_block+0x1f7/0x2a0
[   71.429870]  [<ffffffff81549795>] pcie_port_device_register+0x335/0x520
[   71.429874]  [<ffffffff81549f08>] pcie_portdrv_probe+0x68/0xa0
[   71.429883]  [<ffffffff8153cf75>] local_pci_probe+0x45/0xa0
[   71.429887]  [<ffffffff810b0af4>] work_for_cpu_fn+0x14/0x20
[   71.429891]  [<ffffffff810b431b>] process_one_work+0x28b/0x4a0
[   71.429895]  [<ffffffff810b4292>] ? process_one_work+0x202/0x4a0
[   71.429899]  [<ffffffff810b55db>] worker_thread+0x26b/0x3a0
[   71.429903]  [<ffffffff810e262d>] ? trace_hardirqs_on+0xd/0x10
[   71.429906]  [<ffffffff810b5370>] ? manage_workers.isra.17+0x340/0x340
[   71.429912]  [<ffffffff810bc031>] kthread+0x111/0x120
[   71.429919]  [<ffffffff810cddab>] ? local_clock+0x2b/0x40
[   71.429923]  [<ffffffff810bbf20>] ? kthread_stop+0xf0/0xf0
[   71.429927]  [<ffffffff8203842c>] ret_from_fork+0x7c/0xb0
[   71.429931]  [<ffffffff810bbf20>] ? kthread_stop+0xf0/0xf0
[   71.429933] ---[ end trace c511e3d74efea94e ]---

looks like Greg forgot adding attr init.

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

Can you fold attached patch into Greg's patch ?

Sure. I don't know how to reproduce it, so I couldn't verify the fix, but I added it to Greg's patch. Thanks!

I think lockdep has to be enabled for this to show up, I wasn't running with that enabled when I tested the code, my fault, sorry.

No problem. I have

CONFIG_LOCKDEP_SUPPORT=y
CONFIG_LOCKDEP=y
CONFIG_DEBUG_LOCKDEP=y

and I still don't see the problem booting the qemu q35 model. Whatever, I folded it in anyway.

Bjorn

#15 lore raw
Quoted 8 lines · Bjorn Helgaas

No problem. I have

CONFIG_LOCKDEP_SUPPORT=y
CONFIG_LOCKDEP=y
CONFIG_DEBUG_LOCKDEP=y

and I still don't see the problem booting the qemu q35 model. Whatever, I folded it in anyway.

Does qemu q35 use MSI?

Yinghai

maintainer
#16 lore raw
Quoted 11 lines · Yinghai Lu
Quoted 8 lines · Bjorn Helgaas

No problem. I have

CONFIG_LOCKDEP_SUPPORT=y
CONFIG_LOCKDEP=y
CONFIG_DEBUG_LOCKDEP=y

and I still don't see the problem booting the qemu q35 model. Whatever, I folded it in anyway.

Does qemu q35 use MSI?

Oops, sorry, I *do* see the problem on qemu q35. I must have messed something up when I tried it yesterday. And I verified that your change fixes it. Sorry for the noise.

Bjorn