Skip to contentSkip to conversation

Add PCIe enclosure management support

RFC PATCH Superseded Stuart Hayes 3 participants 7 messages Jan 18, 2022 – Feb 2, 2022

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover Add PCIe enclosure management support

This patch set adds support for two PCIe enclosure management methods, "Native PCIe Enclosure Management" (defined in the PCI Express Base Spec) and "_DSM Definitions for PCIe SSD Status LED" (defined in the PCI Firmware Spec). The latter is very similar to the former, but uses a _DSM interface (which can be provided by firmware) rather than a PCIe extended capability. Each provides a way to control up to ten indicator states (such as locate, fault, etc.)

There are three patches in the set:

(1) Expands the existing enclosure driver to support more indicators.

(2) Adds an auxiliary driver "pcie_em" that can attach to auxiliary devices created by the drivers of any devices that can support PCIe enclosure management (nvme, pcieport, and cxl). It will register an enclosure device with one component for each device with an enclosure management interface.

(3) Modifies the nvme driver to create an auxiliary device to which the pcie_em driver can attach.

These patches do not modify the cxl or pcieport drivers to add support, though the driver was designed to make it easy to do so.
---
Sorry for the long delay in getting this out!
---

Stuart Hayes (3):
Add support for seven more indicators in enclosure driver Add PCIe enclosure management auxiliary driver Register auxiliary device for PCIe enclosure management

 drivers/misc/enclosure.c      | 189 ++++++++------
 drivers/nvme/host/pci.c       |  11 +
 drivers/pci/pcie/Kconfig      |   8 +
 drivers/pci/pcie/Makefile     |   1 +
 drivers/pci/pcie/pcie_em.c    | 473 ++++++++++++++++++++++++++++++++++
 include/linux/enclosure.h     |  22 ++
 include/linux/pcie_em.h       |  97 +++++++
 include/uapi/linux/pci_regs.h |  11 +-
 8 files changed, 733 insertions(+), 79 deletions(-)
 create mode 100644 drivers/pci/pcie/pcie_em.c
 create mode 100644 include/linux/pcie_em.h
Signature · 1 line
2.31.1
#1 lore raw

RFC PATCH 1/3 Add support for seven more indicators in enclosure driver

This patch adds support for seven additional indicators (ok, rebuild, prdfail, hotspare, ica, ifa, disabled) to the enclosure driver, which currently only supports three (fault, active, locate). It also reduces duplicated code for the set and show functions for the sysfs attributes for all of the indicators, and allows users of the driver to provide common get_led and set_led callbacks to control all indicators (though it continues to support the existing callbacks for the three currently supported indicators, so it does not require any changes to code that already uses the enclosure driver).

This will be used by the pcie_em driver.

---
 drivers/misc/enclosure.c  | 189 ++++++++++++++++++++++----------------
 include/linux/enclosure.h |  22 +++++
 2 files changed, 133 insertions(+), 78 deletions(-)
drivers/misc/enclosure.c +111 -78
index 1b010d9267c9..e5a0bb9a719f 100644
@@ -473,30 +473,6 @@ static const char *const enclosure_type[] = {
 	[ENCLOSURE_COMPONENT_ARRAY_DEVICE] = "array device",
 };
 
-static ssize_t get_component_fault(struct device *cdev,
-				   struct device_attribute *attr, char *buf)
-{
-	struct enclosure_device *edev = to_enclosure_device(cdev->parent);
-	struct enclosure_component *ecomp = to_enclosure_component(cdev);
-
-	if (edev->cb->get_fault)
-		edev->cb->get_fault(edev, ecomp);
-	return sysfs_emit(buf, "%d\n", ecomp->fault);
-}
-
-static ssize_t set_component_fault(struct device *cdev,
-				   struct device_attribute *attr,
-				   const char *buf, size_t count)
-{
-	struct enclosure_device *edev = to_enclosure_device(cdev->parent);
-	struct enclosure_component *ecomp = to_enclosure_component(cdev);
-	int val = simple_strtoul(buf, NULL, 0);
-
-	if (edev->cb->set_fault)
-		edev->cb->set_fault(edev, ecomp, val);
-	return count;
-}
-
 static ssize_t get_component_status(struct device *cdev,
 				    struct device_attribute *attr,char *buf)
 {
@@ -531,54 +507,6 @@ static ssize_t set_component_status(struct device *cdev,
 		return -EINVAL;
 }
 
-static ssize_t get_component_active(struct device *cdev,
-				    struct device_attribute *attr, char *buf)
-{
-	struct enclosure_device *edev = to_enclosure_device(cdev->parent);
-	struct enclosure_component *ecomp = to_enclosure_component(cdev);
-
-	if (edev->cb->get_active)
-		edev->cb->get_active(edev, ecomp);
-	return sysfs_emit(buf, "%d\n", ecomp->active);
-}
-
-static ssize_t set_component_active(struct device *cdev,
-				    struct device_attribute *attr,
-				    const char *buf, size_t count)
-{
-	struct enclosure_device *edev = to_enclosure_device(cdev->parent);
-	struct enclosure_component *ecomp = to_enclosure_component(cdev);
-	int val = simple_strtoul(buf, NULL, 0);
-
-	if (edev->cb->set_active)
-		edev->cb->set_active(edev, ecomp, val);
-	return count;
-}
-
-static ssize_t get_component_locate(struct device *cdev,
-				    struct device_attribute *attr, char *buf)
-{
-	struct enclosure_device *edev = to_enclosure_device(cdev->parent);
-	struct enclosure_component *ecomp = to_enclosure_component(cdev);
-
-	if (edev->cb->get_locate)
-		edev->cb->get_locate(edev, ecomp);
-	return sysfs_emit(buf, "%d\n", ecomp->locate);
-}
-
-static ssize_t set_component_locate(struct device *cdev,
-				    struct device_attribute *attr,
-				    const char *buf, size_t count)
-{
-	struct enclosure_device *edev = to_enclosure_device(cdev->parent);
-	struct enclosure_component *ecomp = to_enclosure_component(cdev);
-	int val = simple_strtoul(buf, NULL, 0);
-
-	if (edev->cb->set_locate)
-		edev->cb->set_locate(edev, ecomp, val);
-	return count;
-}
-
 static ssize_t get_component_power_status(struct device *cdev,
 					  struct device_attribute *attr,
 					  char *buf)
@@ -641,29 +569,134 @@ static ssize_t get_component_slot(struct device *cdev,
 	return sysfs_emit(buf, "%d\n", slot);
 }
 
-static DEVICE_ATTR(fault, S_IRUGO | S_IWUSR, get_component_fault,
-		    set_component_fault);
+/*
+ * callbacks for attrs using enum enclosure_component_setting (LEDs)
+ */
+static ssize_t led_show(struct device *cdev,
+			enum enclosure_component_led led,
+			char *buf)
+{
+	struct enclosure_device *edev = to_enclosure_device(cdev->parent);
+	struct enclosure_component *ecomp = to_enclosure_component(cdev);
+
+	if (edev->cb->get_led)
+		edev->cb->get_led(edev, ecomp, led);
+	else
+		/*
+		 * support old callbacks for fault/active/locate
+		 */
+		switch (led) {
+		case ENCLOSURE_LED_FAULT:
+			if (edev->cb->get_fault) {
+				edev->cb->get_fault(edev, ecomp);
+				ecomp->led[led] = ecomp->fault;
+			}
+			break;
+		case ENCLOSURE_LED_ACTIVE:
+			if (edev->cb->get_active) {
+				edev->cb->get_active(edev, ecomp);
+				ecomp->led[led] = ecomp->active;
+			}
+			break;
+		case ENCLOSURE_LED_LOCATE:
+			if (edev->cb->get_locate) {
+				edev->cb->get_locate(edev, ecomp);
+				ecomp->led[led] = ecomp->locate;
+			}
+			break;
+		default:
+		}
+
+	return sysfs_emit(buf, "%d\n", ecomp->led[led]);
+}
+
+static ssize_t led_set(struct device *cdev,
+		       enum enclosure_component_led led,
+		       const char *buf, size_t count)
+{
+	struct enclosure_device *edev = to_enclosure_device(cdev->parent);
+	struct enclosure_component *ecomp = to_enclosure_component(cdev);
+	int err, val;
+
+	err = kstrtoint(buf, 0, &val);
+	if (err)
+		return err;
+
+	if (edev->cb->set_led)
+		edev->cb->set_led(edev, ecomp, led, val);
+	else
+		/*
+		 * support old callbacks for fault/active/locate
+		 */
+		switch (led) {
+		case ENCLOSURE_LED_FAULT:
+			if (edev->cb->set_fault)
+				edev->cb->set_fault(edev, ecomp, val);
+			break;
+		case ENCLOSURE_LED_ACTIVE:
+			if (edev->cb->set_active)
+				edev->cb->set_active(edev, ecomp, val);
+			break;
+		case ENCLOSURE_LED_LOCATE:
+			if (edev->cb->set_locate)
+				edev->cb->set_locate(edev, ecomp, val);
+			break;
+		default:
+		}
+
+	return count;
+}
+
+#define LED_ATTR_RW(led_attr, led)					\
+static ssize_t led_attr##_show(struct device *cdev,			\
+			       struct device_attribute *attr,		\
+			       char *buf)				\
+{									\
+	return led_show(cdev, led, buf);				\
+}									\
+static ssize_t led_attr##_store(struct device *cdev,			\
+				struct device_attribute *attr,		\
+				const char *buf, size_t count)		\
+{									\
+	return led_set(cdev, led, buf, count);				\
+}									\
+static DEVICE_ATTR_RW(led_attr)
+
 static DEVICE_ATTR(status, S_IRUGO | S_IWUSR, get_component_status,
 		   set_component_status);
-static DEVICE_ATTR(active, S_IRUGO | S_IWUSR, get_component_active,
-		   set_component_active);
-static DEVICE_ATTR(locate, S_IRUGO | S_IWUSR, get_component_locate,
-		   set_component_locate);
 static DEVICE_ATTR(power_status, S_IRUGO | S_IWUSR, get_component_power_status,
 		   set_component_power_status);
 static DEVICE_ATTR(type, S_IRUGO, get_component_type, NULL);
 static DEVICE_ATTR(slot, S_IRUGO, get_component_slot, NULL);
+LED_ATTR_RW(fault, ENCLOSURE_LED_FAULT);
+LED_ATTR_RW(active, ENCLOSURE_LED_ACTIVE);
+LED_ATTR_RW(locate, ENCLOSURE_LED_LOCATE);
+LED_ATTR_RW(ok, ENCLOSURE_LED_OK);
+LED_ATTR_RW(rebuild, ENCLOSURE_LED_REBUILD);
+LED_ATTR_RW(prdfail, ENCLOSURE_LED_PRDFAIL);
+LED_ATTR_RW(hotspare, ENCLOSURE_LED_HOTSPARE);
+LED_ATTR_RW(ica, ENCLOSURE_LED_ICA);
+LED_ATTR_RW(ifa, ENCLOSURE_LED_IFA);
+LED_ATTR_RW(disabled, ENCLOSURE_LED_DISABLED);
 
 static struct attribute *enclosure_component_attrs[] = {
 	&dev_attr_fault.attr,
 	&dev_attr_status.attr,
 	&dev_attr_active.attr,
 	&dev_attr_locate.attr,
+	&dev_attr_ok.attr,
+	&dev_attr_rebuild.attr,
+	&dev_attr_prdfail.attr,
+	&dev_attr_hotspare.attr,
+	&dev_attr_ica.attr,
+	&dev_attr_ifa.attr,
+	&dev_attr_disabled.attr,
 	&dev_attr_power_status.attr,
 	&dev_attr_type.attr,
 	&dev_attr_slot.attr,
 	NULL
 };
+
 ATTRIBUTE_GROUPS(enclosure_component);
 
 static int __init enclosure_init(void)
include/linux/enclosure.h +22 -0
index 1c630e2c2756..98dd0f05efb7 100644
@@ -49,6 +49,20 @@ enum enclosure_component_setting {
 	ENCLOSURE_SETTING_BLINK_B_OFF_ON = 7,
 };
 
+enum enclosure_component_led {
+	ENCLOSURE_LED_FAULT,
+	ENCLOSURE_LED_ACTIVE,
+	ENCLOSURE_LED_LOCATE,
+	ENCLOSURE_LED_OK,
+	ENCLOSURE_LED_REBUILD,
+	ENCLOSURE_LED_PRDFAIL,
+	ENCLOSURE_LED_HOTSPARE,
+	ENCLOSURE_LED_ICA,
+	ENCLOSURE_LED_IFA,
+	ENCLOSURE_LED_DISABLED,
+	ENCLOSURE_LED_MAX,
+};
+
 struct enclosure_device;
 struct enclosure_component;
 struct enclosure_component_callbacks {
@@ -72,6 +86,13 @@ struct enclosure_component_callbacks {
 	int (*set_locate)(struct enclosure_device *,
 			  struct enclosure_component *,
 			  enum enclosure_component_setting);
+	void (*get_led)(struct enclosure_device *edev,
+			struct enclosure_component *ecomp,
+			enum enclosure_component_led);
+	int (*set_led)(struct enclosure_device *edev,
+		       struct enclosure_component *ecomp,
+		       enum enclosure_component_led,
+		       enum enclosure_component_setting);
 	void (*get_power_status)(struct enclosure_device *,
 				 struct enclosure_component *);
 	int (*set_power_status)(struct enclosure_device *,
@@ -90,6 +111,7 @@ struct enclosure_component {
 	int fault;
 	int active;
 	int locate;
+	int led[ENCLOSURE_LED_MAX];
 	int slot;
 	enum enclosure_status status;
 	int power_status;
Signature · 1 line
2.31.1
#2 lore raw

Re: [RFC PATCH 1/3] Add support for seven more indicators in enclosure driver

Quoted 10 lines · Stuart Hayes

This patch adds support for seven additional indicators (ok, rebuild, prdfail, hotspare, ica, ifa, disabled) to the enclosure driver, which currently only supports three (fault, active, locate). It also reduces duplicated code for the set and show functions for the sysfs attributes for all of the indicators, and allows users of the driver to provide common get_led and set_led callbacks to control all indicators (though it continues to support the existing callbacks for the three currently supported indicators, so it does not require any changes to code that already uses the enclosure driver).

Quoted 7 lines

This will be used by the pcie_em driver.

---
 drivers/misc/enclosure.c  | 189 ++++++++++++++++++++++----------------
 include/linux/enclosure.h |  22 +++++
 2 files changed, 133 insertions(+), 78 deletions(-)

You did not document your new sysfs files in Documentation/ABI/ anywhere :(

#3 lore raw

RFC PATCH 2/3 Add PCIe enclosure management auxiliary driver

In reply to Stuart Hayes

This patch adds a new auxiliary driver to support PCIe enclosure management (NPEM and _DSM). Drivers whose PCIe devices could have NPEM and/or _DSM enclosure management support could register an auxiliary device, and this driver will attach to that and register an enclosure to allow user space control of the associated state indicators.

---
 drivers/pci/pcie/Kconfig      |   8 +
 drivers/pci/pcie/Makefile     |   1 +
 drivers/pci/pcie/pcie_em.c    | 473 ++++++++++++++++++++++++++++++++++
 include/linux/pcie_em.h       |  97 +++++++
 include/uapi/linux/pci_regs.h |  11 +-
 5 files changed, 589 insertions(+), 1 deletion(-)
 create mode 100644 drivers/pci/pcie/pcie_em.c
 create mode 100644 include/linux/pcie_em.h
drivers/pci/pcie/Kconfig +8 -0
index 45a2ef702b45..deaaf00c0d38 100644
@@ -142,3 +142,11 @@ config PCIE_EDR
 	  the PCI Firmware Specification r3.2.  Enable this if you want to
 	  support hybrid DPC model which uses both firmware and OS to
 	  implement DPC.
+
+config PCIE_EM
+	tristate "PCIe Enclosure Management support"
+	depends on ENCLOSURE_SERVICES
+	help
+	  Auxiliary driver for PCI Express enclosure management (support
+	  for Native PCIe Enclosure Management (NPEM) and the related _DSM
+	  interface for status LED management).
drivers/pci/pcie/Makefile +1 -0
index 5783a2f79e6a..3b2b160b6f13 100644
@@ -13,3 +13,4 @@ obj-$(CONFIG_PCIE_PME)		+= pme.o
 obj-$(CONFIG_PCIE_DPC)		+= dpc.o
 obj-$(CONFIG_PCIE_PTM)		+= ptm.o
 obj-$(CONFIG_PCIE_EDR)		+= edr.o
+obj-$(CONFIG_PCIE_EM)		+= pcie_em.o
drivers/pci/pcie/pcie_em.c new file +473 -0

473 lines not on this page · raw message

include/linux/pcie_em.h new file +97 -0

97 lines not on this page · raw message

include/uapi/linux/pci_regs.h +10 -1
index ff6ccbc6efe9..375540065c29 100644
@@ -736,7 +736,8 @@
 #define PCI_EXT_CAP_ID_DVSEC	0x23	/* Designated Vendor-Specific */
 #define PCI_EXT_CAP_ID_DLF	0x25	/* Data Link Feature */
 #define PCI_EXT_CAP_ID_PL_16GT	0x26	/* Physical Layer 16.0 GT/s */
-#define PCI_EXT_CAP_ID_MAX	PCI_EXT_CAP_ID_PL_16GT
+#define PCI_EXT_CAP_ID_NPEM	0x29	/* Native PCIe Enclosure Management */
+#define PCI_EXT_CAP_ID_MAX	PCI_EXT_CAP_ID_NPEM
 
 #define PCI_EXT_CAP_DSN_SIZEOF	12
 #define PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF 40
@@ -1098,4 +1099,12 @@
 #define  PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK		0x000000F0
 #define  PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT	4
 
+/* NPEM */
+#define PCI_NPEM_CAP		0x04
+#define  PCI_NPEM_CAP_NPEM_CAP		0x00000001  /* dev is NPEM capable */
+#define PCI_NPEM_CTRL		0x08
+#define  PCI_NPEM_CTRL_EN		0x00000001  /* Enable NPEM */
+#define PCI_NPEM_STATUS		0x0c
+#define  PCI_NPEM_STATUS_CC		0x00000001  /* NPEM command completed */
+
 #endif /* LINUX_PCI_REGS_H */
Signature · 1 line
2.31.1
#4 lore raw

Re: [RFC PATCH 2/3] Add PCIe enclosure management auxiliary driver

Hi Stuart,

Quoted 15 lines · On Mon, 17 Jan 2022 22:17:57 -0600 Stuart Hayes <[email protected]>…
drivers/pci/pcie/pcie_em.c new file +15 -0
+	switch (output->status) {
+	case 0:
+		break;
+	case 1:
+		pci_dbg(pdev, "_DSM not supported\n");
+		break;
+	case 2:
+		pci_dbg(pdev, "_DSM invalid input parameters\n");
+		break;
+	case 3:
+		pci_dbg(pdev, "_DSM communication error\n");
+		break;
+	case 4:
+		pci_dbg(pdev, "_DSM function-specific error 0x%x\n",
+			output->function_specific_err);

drivers/pci/pcie/pcie_em.c line 122

Quoted 10 lines
drivers/pci/pcie/pcie_em.c new file +10 -0
+		break;
+	case 5:
+		pci_dbg(pdev, "_DSM vendor-specific error 0x%x\n",
+			output->vendor_specific_err);
+		break;
+	default:
+		pci_dbg(pdev, "_DSM returned unknown status 0x%x\n",
+			output->status);
+	}
+}

I tired to compile it and I failed:
drivers/misc/enclosure.c: In function ‘led_show’:
drivers/misc/enclosure.c:607:3: error: label at end of compound statement default:
^~~~~~~
drivers/misc/enclosure.c: In function ‘led_set’:
drivers/misc/enclosure.c:644:3: error: label at end of compound statement default:
^~~~~~~
make[2]: *** [scripts/Makefile.build:288: drivers/misc/enclosure.o] Error 1 make[1]: *** [scripts/Makefile.build:550: drivers/misc] Error 2 make[1]: *** Waiting for unfinished jobs....

My gcc version:
gcc (SUSE Linux) 7.5.0

Could you please, resolve the issue?

Thanks,
Mariusz

#5 lore raw

Re: [RFC PATCH 2/3] Add PCIe enclosure management auxiliary driver

Quoted 43 lines · Mariusz Tkaczyk

Hi Stuart,

Quoted 15 lines · On Mon, 17 Jan 2022 22:17:57 -0600 Stuart Hayes <[email protected]>…
drivers/pci/pcie/pcie_em.c new file +15 -0
+	switch (output->status) {
+	case 0:
+		break;
+	case 1:
+		pci_dbg(pdev, "_DSM not supported\n");
+		break;
+	case 2:
+		pci_dbg(pdev, "_DSM invalid input parameters\n");
+		break;
+	case 3:
+		pci_dbg(pdev, "_DSM communication error\n");
+		break;
+	case 4:
+		pci_dbg(pdev, "_DSM function-specific error 0x%x\n",
+			output->function_specific_err);

drivers/pci/pcie/pcie_em.c line 122

Quoted 10 lines
drivers/pci/pcie/pcie_em.c new file +10 -0
+		break;
+	case 5:
+		pci_dbg(pdev, "_DSM vendor-specific error 0x%x\n",
+			output->vendor_specific_err);
+		break;
+	default:
+		pci_dbg(pdev, "_DSM returned unknown status 0x%x\n",
+			output->status);
+	}
+}

I tired to compile it and I failed:
drivers/misc/enclosure.c: In function ‘led_show’:
drivers/misc/enclosure.c:607:3: error: label at end of compound statement default:

    ^~~~~~~
drivers/misc/enclosure.c: In function ‘led_set’:
drivers/misc/enclosure.c:644:3: error: label at end of compound
statement default:
    ^~~~~~~

make[2]: *** [scripts/Makefile.build:288: drivers/misc/enclosure.o] Error 1 make[1]: *** [scripts/Makefile.build:550: drivers/misc] Error 2 make[1]: *** Waiting for unfinished jobs....

Quoted 8 lines

My gcc version:
gcc (SUSE Linux) 7.5.0

Could you please, resolve the issue?

Thanks,
Mariusz

No problem. I'll send a v2 now.
Thanks,
Stuart

#6 lore raw

RFC PATCH 3/3 Register auxiliary device for PCIe enclosure management

In reply to Stuart Hayes

This patch will register an auxiliary devivce for the pcie_em driver, allowing it to attach and provide access to PCIe enclosure management (LEDs), if present.

---
 drivers/nvme/host/pci.c | 11 +++++++++++
 1 file changed, 11 insertions(+)
drivers/nvme/host/pci.c +11 -0
index ca2ee806d74b..22ff10798333 100644
@@ -27,6 +27,8 @@
 #include <linux/io-64-nonatomic-hi-lo.h>
 #include <linux/sed-opal.h>
 #include <linux/pci-p2pdma.h>
+#include <linux/auxiliary_bus.h>
+#include <linux/pcie_em.h>
 
 #include "trace.h"
 #include "nvme.h"
@@ -159,6 +161,9 @@ struct nvme_dev {
 	unsigned int nr_poll_queues;
 
 	bool attrs_added;
+
+	/* auxiliary_device for pcie_em driver (LEDs) */
+	struct auxiliary_device *pcie_em_auxdev;
 };
 
 static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
@@ -2838,6 +2843,11 @@ static void nvme_reset_work(struct work_struct *work)
 		nvme_unfreeze(&dev->ctrl);
 	}
 
+	/*
+	 * register auxiliary device if this supports PCIe enclosure management
+	 */
+	dev->pcie_em_auxdev = register_pcie_em_auxdev(dev->dev, dev->ctrl.instance);
+
 	/*
 	 * If only admin queue live, keep it to do further investigation or
 	 * recovery.
@@ -3135,6 +3145,7 @@ static void nvme_remove(struct pci_dev *pdev)
 	nvme_free_queues(dev, 0);
 	nvme_release_prp_pools(dev);
 	nvme_dev_unmap(dev);
+	unregister_pcie_em_auxdev(dev->pcie_em_auxdev);
 	nvme_uninit_ctrl(&dev->ctrl);
 }
 
Signature · 1 line
2.31.1