Skip to contentSkip to conversation

cciss updates, PCI ID changes, kdump reset fixes

PATCH In review Stephen M. Cameron 4 participants 9 messages Oct 22, 2010 Block layer

lore
Share & tools
lore
#0 lore raw

cover cciss updates, PCI ID changes, kdump reset fixes

The following series implements fixes in the board resetting code invoked by the reset_devices kernel parameter, typically used by kdump kernels.

Also removes support for some controllers which are also supported by hpsa. We do not wish to have any overlap between the two drivers.

Also change one small msleep to use usleep_range.

---

Stephen M. Cameron (5):
cciss: remove controllers supported by hpsa
cciss: fix board status waiting code
cciss: Use kernel provided PCI state save and restore functions
cciss: limit commands allocated on reset_devices
cciss: use usleep_range not msleep for small sleeps

 drivers/block/cciss.c |  169 +++++++++++++++++--------------------------------
 drivers/block/cciss.h |    4 +
 2 files changed, 64 insertions(+), 109 deletions(-)
Signature · 1 line
-- steve
#1 lore raw

PATCH 1/5 cciss: remove controllers supported by hpsa

From: Stephen M. Cameron <[email protected]>

We would prefer not to have any overlap between the two drivers. Remove the cciss_allow_hpsa option, as it it is no longer needed.

---
 drivers/block/cciss.c |   46 ++++------------------------------------------
 1 files changed, 4 insertions(+), 42 deletions(-)
drivers/block/cciss.c +4 -42
index 41f9acb..dea1f2a 100644
@@ -66,12 +66,6 @@ MODULE_SUPPORTED_DEVICE("HP Smart Array Controllers");
 MODULE_VERSION("3.6.26");
 MODULE_LICENSE("GPL");
 
-static int cciss_allow_hpsa;
-module_param(cciss_allow_hpsa, int, S_IRUGO|S_IWUSR);
-MODULE_PARM_DESC(cciss_allow_hpsa,
-	"Prevent cciss driver from accessing hardware known to be "
-	" supported by the hpsa driver");
-
 #include "cciss_cmd.h"
 #include "cciss.h"
 #include <linux/cciss_ioctl.h>
@@ -98,19 +92,6 @@ static const struct pci_device_id cciss_pci_device_id[] = {
 	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSD,     0x103C, 0x3215},
 	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSC,     0x103C, 0x3237},
 	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSC,     0x103C, 0x323D},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSE,     0x103C, 0x3241},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSE,     0x103C, 0x3243},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSE,     0x103C, 0x3245},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSE,     0x103C, 0x3247},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSE,     0x103C, 0x3249},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSE,     0x103C, 0x324A},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSE,     0x103C, 0x324B},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSF,     0x103C, 0x3350},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSF,     0x103C, 0x3351},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSF,     0x103C, 0x3352},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSF,     0x103C, 0x3353},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSF,     0x103C, 0x3354},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSF,     0x103C, 0x3355},
 	{0,}
 };
 
@@ -131,6 +112,8 @@ static struct board_type products[] = {
 	{0x409D0E11, "Smart Array 6400 EM", &SA5_access},
 	{0x40910E11, "Smart Array 6i", &SA5_access},
 	{0x3225103C, "Smart Array P600", &SA5_access},
+	{0x3223103C, "Smart Array P800", &SA5_access},
+	{0x3234103C, "Smart Array P400", &SA5_access},
 	{0x3235103C, "Smart Array P400i", &SA5_access},
 	{0x3211103C, "Smart Array E200i", &SA5_access},
 	{0x3212103C, "Smart Array E200", &SA5_access},
@@ -138,24 +121,7 @@ static struct board_type products[] = {
 	{0x3214103C, "Smart Array E200i", &SA5_access},
 	{0x3215103C, "Smart Array E200i", &SA5_access},
 	{0x3237103C, "Smart Array E500", &SA5_access},
-/* controllers below this line are also supported by the hpsa driver. */
-#define HPSA_BOUNDARY 0x3223103C
-	{0x3223103C, "Smart Array P800", &SA5_access},
-	{0x3234103C, "Smart Array P400", &SA5_access},
-	{0x323D103C, "Smart Array P700m", &SA5_access},
-	{0x3241103C, "Smart Array P212", &SA5_access},
-	{0x3243103C, "Smart Array P410", &SA5_access},
-	{0x3245103C, "Smart Array P410i", &SA5_access},
-	{0x3247103C, "Smart Array P411", &SA5_access},
-	{0x3249103C, "Smart Array P812", &SA5_access},
-	{0x324A103C, "Smart Array P712m", &SA5_access},
-	{0x324B103C, "Smart Array P711m", &SA5_access},
-	{0x3350103C, "Smart Array", &SA5_access},
-	{0x3351103C, "Smart Array", &SA5_access},
-	{0x3352103C, "Smart Array", &SA5_access},
-	{0x3353103C, "Smart Array", &SA5_access},
-	{0x3354103C, "Smart Array", &SA5_access},
-	{0x3355103C, "Smart Array", &SA5_access},
+	{0x323d103c, "Smart Array P700M", &SA5_access},
 };
 
 /* How long to wait (in milliseconds) for board to go into simple mode */
@@ -3967,13 +3933,9 @@ static int __devinit cciss_lookup_board_id(struct pci_dev *pdev, u32 *board_id)
 	*board_id = ((subsystem_device_id << 16) & 0xffff0000) |
 			subsystem_vendor_id;
 
-	for (i = 0; i < ARRAY_SIZE(products); i++) {
-		/* Stand aside for hpsa driver on request */
-		if (cciss_allow_hpsa && products[i].board_id == HPSA_BOUNDARY)
-			return -ENODEV;
+	for (i = 0; i < ARRAY_SIZE(products); i++)
 		if (*board_id == products[i].board_id)
 			return i;
-	}
 	dev_warn(&pdev->dev, "unrecognized board ID: 0x%08x, ignoring.\n",
 		*board_id);
 	return -ENODEV;
#2 lore raw

Re: [PATCH 1/5] cciss: remove controllers supported by hpsa

Quoted 6 lines · On Fri, 22 October 2010 "Stephen M. Cameron" wrote:

From: Stephen M. Cameron <[email protected]>

We would prefer not to have any overlap between the two drivers. Remove the cciss_allow_hpsa option, as it it is no longer needed.

I think it would be helpful to provide clear indication in cciss' Kconfig description about which controllers are the last ones to be supported by cciss and starting from which ones hpsa is needed.

In addition the dev_warn before returning -ENODEV could include a hint that newish controllers need hpsa driver instead of just saying "unrecognized board ...", this at least when hpsa driver was not included in kernel build.

Regards,
Bruno

#3 lore raw

PATCH 2/5 cciss: fix board status waiting code

In reply to Stephen M. Cameron

From: Stephen M. Cameron <[email protected]>

After a reset, we should first wait for the board to become "not ready", and then wait for it to become "ready", instead of immediately waiting for it to become "ready", and do this waiting *after* restoring PCI config space registers.

---
 drivers/block/cciss.c |   43 +++++++++++++++++++++++++++++++++++--------
 drivers/block/cciss.h |    4 ++++
 2 files changed, 39 insertions(+), 8 deletions(-)
drivers/block/cciss.c +35 -8
index dea1f2a..a9e9db4 100644
@@ -3966,18 +3966,31 @@ static int __devinit cciss_pci_find_memory_BAR(struct pci_dev *pdev,
 	return -ENODEV;
 }
 
-static int __devinit cciss_wait_for_board_ready(ctlr_info_t *h)
+static int __devinit cciss_wait_for_board_state(struct pci_dev *pdev,
+	void __iomem *vaddr, int wait_for_ready)
+#define BOARD_READY 1
+#define BOARD_NOT_READY 0
 {
-	int i;
+	int i, iterations;
 	u32 scratchpad;
 
-	for (i = 0; i < CCISS_BOARD_READY_ITERATIONS; i++) {
-		scratchpad = readl(h->vaddr + SA5_SCRATCHPAD_OFFSET);
-		if (scratchpad == CCISS_FIRMWARE_READY)
-			return 0;
+	if (wait_for_ready)
+		iterations = CCISS_BOARD_READY_ITERATIONS;
+	else
+		iterations = CCISS_BOARD_NOT_READY_ITERATIONS;
+
+	for (i = 0; i < iterations; i++) {
+		scratchpad = readl(vaddr + SA5_SCRATCHPAD_OFFSET);
+		if (wait_for_ready) {
+			if (scratchpad == CCISS_FIRMWARE_READY)
+				return 0;
+		} else {
+			if (scratchpad != CCISS_FIRMWARE_READY)
+				return 0;
+		}
 		msleep(CCISS_BOARD_READY_POLL_INTERVAL_MSECS);
 	}
-	dev_warn(&h->pdev->dev, "board not ready, timed out.\n");
+	dev_warn(&pdev->dev, "board not ready, timed out.\n");
 	return -ENODEV;
 }
 
@@ -4143,7 +4156,7 @@ static int __devinit cciss_pci_init(ctlr_info_t *h)
 		err = -ENOMEM;
 		goto err_out_free_res;
 	}
-	err = cciss_wait_for_board_ready(h);
+	err = cciss_wait_for_board_state(h->pdev, h->vaddr, BOARD_READY);
 	if (err)
 		goto err_out_free_res;
 	err = cciss_find_cfgtables(h);
@@ -4488,6 +4501,20 @@ static __devinit int cciss_kdump_hard_reset_controller(struct pci_dev *pdev)
 	   need a little pause here */
 	msleep(CCISS_POST_RESET_PAUSE_MSECS);
 
+	/* Wait for board to become not ready, then ready. */
+	dev_info(&pdev->dev, "Waiting for board to become ready.\n");
+	rc = cciss_wait_for_board_state(pdev, vaddr, BOARD_NOT_READY);
+	if (rc) /* Don't bail, might be E500, etc. which can't be reset */
+		dev_warn(&pdev->dev,
+			"failed waiting for board to become not ready\n");
+	rc = cciss_wait_for_board_state(pdev, vaddr, BOARD_READY);
+	if (rc) {
+		dev_warn(&pdev->dev,
+			"failed waiting for board to become ready\n");
+		goto unmap_cfgtable;
+	}
+	dev_info(&pdev->dev, "board ready.\n");
+
 	/* Controller should be in simple mode at this point.  If it's not,
 	 * It means we're on one of those controllers which doesn't support
 	 * the doorbell reset method and on which the PCI power management reset
drivers/block/cciss.h +4 -0
index ae340ff..4b8933d 100644
@@ -200,10 +200,14 @@ struct ctlr_info
  * the above.
  */
 #define CCISS_BOARD_READY_WAIT_SECS (120)
+#define CCISS_BOARD_NOT_READY_WAIT_SECS (10)
 #define CCISS_BOARD_READY_POLL_INTERVAL_MSECS (100)
 #define CCISS_BOARD_READY_ITERATIONS \
 	((CCISS_BOARD_READY_WAIT_SECS * 1000) / \
 		CCISS_BOARD_READY_POLL_INTERVAL_MSECS)
+#define CCISS_BOARD_NOT_READY_ITERATIONS \
+	((CCISS_BOARD_NOT_READY_WAIT_SECS * 1000) / \
+		CCISS_BOARD_READY_POLL_INTERVAL_MSECS)
 #define CCISS_POST_RESET_PAUSE_MSECS (3000)
 #define CCISS_POST_RESET_NOOP_INTERVAL_MSECS (1000)
 #define CCISS_POST_RESET_NOOP_RETRIES (12)
#4 lore raw

PATCH 3/5 cciss: Use kernel provided PCI state save and restore functions

In reply to Stephen M. Cameron

From: Stephen M. Cameron <[email protected]>

and use the doorbell reset method if available (which doesn't lock up the controller if you properly save and restore all the PCI registers that you're supposed to.)

---
 drivers/block/cciss.c |   73 ++++++++++---------------------------------------
 1 files changed, 15 insertions(+), 58 deletions(-)
drivers/block/cciss.c +15 -58
index a9e9db4..03afef8 100644
@@ -4321,36 +4321,6 @@ static __devinit int cciss_message(struct pci_dev *pdev, unsigned char opcode, u
 #define cciss_soft_reset_controller(p) cciss_message(p, 1, 0)
 #define cciss_noop(p) cciss_message(p, 3, 0)
 
-static __devinit int cciss_reset_msi(struct pci_dev *pdev)
-{
-/* the #defines are stolen from drivers/pci/msi.h. */
-#define msi_control_reg(base)		(base + PCI_MSI_FLAGS)
-#define PCI_MSIX_FLAGS_ENABLE		(1 << 15)
-
-	int pos;
-	u16 control = 0;
-
-	pos = pci_find_capability(pdev, PCI_CAP_ID_MSI);
-	if (pos) {
-		pci_read_config_word(pdev, msi_control_reg(pos), &control);
-		if (control & PCI_MSI_FLAGS_ENABLE) {
-			dev_info(&pdev->dev, "resetting MSI\n");
-			pci_write_config_word(pdev, msi_control_reg(pos), control & ~PCI_MSI_FLAGS_ENABLE);
-		}
-	}
-
-	pos = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
-	if (pos) {
-		pci_read_config_word(pdev, msi_control_reg(pos), &control);
-		if (control & PCI_MSIX_FLAGS_ENABLE) {
-			dev_info(&pdev->dev, "resetting MSI-X\n");
-			pci_write_config_word(pdev, msi_control_reg(pos), control & ~PCI_MSIX_FLAGS_ENABLE);
-		}
-	}
-
-	return 0;
-}
-
 static int cciss_controller_hard_reset(struct pci_dev *pdev,
 	void * __iomem vaddr, bool use_doorbell)
 {
@@ -4405,17 +4375,17 @@ static int cciss_controller_hard_reset(struct pci_dev *pdev,
  * states or using the doorbell register. */
 static __devinit int cciss_kdump_hard_reset_controller(struct pci_dev *pdev)
 {
-	u16 saved_config_space[32];
 	u64 cfg_offset;
 	u32 cfg_base_addr;
 	u64 cfg_base_addr_index;
 	void __iomem *vaddr;
 	unsigned long paddr;
 	u32 misc_fw_support, active_transport;
-	int rc, i;
+	int rc;
 	CfgTable_struct __iomem *cfgtable;
 	bool use_doorbell;
 	u32 board_id;
+	u16 command_register;
 
 	/* For controllers as old a the p600, this is very nearly
 	 * the same thing as
@@ -4425,14 +4395,6 @@ static __devinit int cciss_kdump_hard_reset_controller(struct pci_dev *pdev)
 	 * pci_set_power_state(pci_dev, PCI_D0);
 	 * pci_restore_state(pci_dev);
 	 *
-	 * but we can't use these nice canned kernel routines on
-	 * kexec, because they also check the MSI/MSI-X state in PCI
-	 * configuration space and do the wrong thing when it is
-	 * set/cleared.  Also, the pci_save/restore_state functions
-	 * violate the ordering requirements for restoring the
-	 * configuration space from the CCISS document (see the
-	 * comment below).  So we roll our own ....
-	 *
 	 * For controllers newer than the P600, the pci power state
 	 * method of resetting doesn't work so we have another way
 	 * using the doorbell register.
@@ -4451,8 +4413,13 @@ static __devinit int cciss_kdump_hard_reset_controller(struct pci_dev *pdev)
 		return -ENODEV;
 	}
 
-	for (i = 0; i < 32; i++)
-		pci_read_config_word(pdev, 2*i, &saved_config_space[i]);
+	/* Save the PCI command register */
+	pci_read_config_word(pdev, 4, &command_register);
+	/* Turn the board off.  This is so that later pci_restore_state()
+	 * won't turn the board on before the rest of config space is ready.
+	 */
+	pci_disable_device(pdev);
+	pci_save_state(pdev);
 
 	/* find the first memory BAR, so we can find the cfg table */
 	rc = cciss_pci_find_memory_BAR(pdev, &paddr);
@@ -4481,21 +4448,13 @@ static __devinit int cciss_kdump_hard_reset_controller(struct pci_dev *pdev)
 	rc = cciss_controller_hard_reset(pdev, vaddr, use_doorbell);
 	if (rc)
 		goto unmap_cfgtable;
-
-	/* Restore the PCI configuration space.  The Open CISS
-	 * Specification says, "Restore the PCI Configuration
-	 * Registers, offsets 00h through 60h. It is important to
-	 * restore the command register, 16-bits at offset 04h,
-	 * last. Do not restore the configuration status register,
-	 * 16-bits at offset 06h."  Note that the offset is 2*i.
-	 */
-	for (i = 0; i < 32; i++) {
-		if (i == 2 || i == 3)
-			continue;
-		pci_write_config_word(pdev, 2*i, saved_config_space[i]);
+	pci_restore_state(pdev);
+	rc = pci_enable_device(pdev);
+	if (rc) {
+		dev_warn(&pdev->dev, "failed to enable device.\n");
+		goto unmap_cfgtable;
 	}
-	wmb();
-	pci_write_config_word(pdev, 4, saved_config_space[2]);
+	pci_write_config_word(pdev, 4, command_register);
 
 	/* Some devices (notably the HP Smart Array 5i Controller)
 	   need a little pause here */
@@ -4555,8 +4514,6 @@ static __devinit int cciss_init_reset_devices(struct pci_dev *pdev)
 		return 0; /* just try to do the kdump anyhow. */
 	if (rc)
 		return -ENODEV;
-	if (cciss_reset_msi(pdev))
-		return -ENODEV;
 
 	/* Now try to get the controller to respond to a no-op */
 	for (i = 0; i < CCISS_POST_RESET_NOOP_RETRIES; i++) {
#5 lore raw

Re: [PATCH 3/5] cciss: Use kernel provided PCI state save and restore functions

Quoted 10 lines · Stephen M. Cameron

From: Stephen M. Cameron <[email protected]>
Subject: [PATCH 3/5] cciss: Use kernel provided PCI state save and restore functions
To: [email protected]
Cc: [email protected], [email protected], [email protected], [email protected], [email protected]
Date: Friday, October 22, 2010, 2:21 PM
From: Stephen M. Cameron <[email protected]>

and use the doorbell reset method if available (which doesn't lock up the controller if you properly save and restore all the PCI registers that you're supposed to.)

This patch, and all these cciss patches in this series, are vs. the jens/for-2.6.37/drivers branch.

One difference I noticed between linux-2.6.36 and jens/for-2.6.37/drivers is that linux-2.6.36 contains this in drivers/block/cciss.c in the function cciss_kdump_hard_reset_controller:

        /* The doorbell reset seems to cause lockups on some Smart
         * Arrays (e.g. P410, P410i, maybe others).  Until this is
         * fixed or at least isolated, avoid the doorbell reset.
         */
        use_doorbell = 0;

This is not present in jens/for-2.6.37/drivers.

That was added to linus tree by me, because I'd noticed we were getting controller lockups and subsequent data corruption when attempting kdumps using the doorbell reset when (unbeknownst to me at the time) the pci state saving and restoring code wasn't quite right. So these patches fix the pci state saving and restoring code, so the above "use_doorbell = 0" needs to come out when these patches go into Linus's tree. But such a patch cannot be made vs. Jens's tree, because that code isn't in Jens's tree.

-- steve

maintainer
#6 lore raw

Re: [PATCH 3/5] cciss: Use kernel provided PCI state save and restore functions

Quoted 27 lines · Stephen Cameron
Quoted 10 lines · Stephen M. Cameron

From: Stephen M. Cameron <[email protected]>
Subject: [PATCH 3/5] cciss: Use kernel provided PCI state save and restore functions
To: [email protected]
Cc: [email protected], [email protected], [email protected], [email protected], [email protected]
Date: Friday, October 22, 2010, 2:21 PM
From: Stephen M. Cameron <[email protected]>

and use the doorbell reset method if available (which doesn't lock up the controller if you properly save and restore all the PCI registers that you're supposed to.)

This patch, and all these cciss patches in this series, are vs. the jens/for-2.6.37/drivers branch.

One difference I noticed between linux-2.6.36 and jens/for-2.6.37/drivers is that linux-2.6.36 contains this in drivers/block/cciss.c in the function cciss_kdump_hard_reset_controller:

        /* The doorbell reset seems to cause lockups on some Smart
         * Arrays (e.g. P410, P410i, maybe others).  Until this is
         * fixed or at least isolated, avoid the doorbell reset.
         */
        use_doorbell = 0;

This is not present in jens/for-2.6.37/drivers.

Quoted 8 lines

That was added to linus tree by me, because I'd noticed we were getting controller lockups and subsequent data corruption when attempting kdumps using the doorbell reset when (unbeknownst to me at the time) the pci state saving and restoring code wasn't quite right. So these patches fix the pci state saving and restoring code, so the above "use_doorbell = 0" needs to come out when these patches go into Linus's tree. But such a patch cannot be made vs. Jens's tree, because that code isn't in Jens's tree.

I sent a pull request to Linus earlier today and I know the drivers branch merges cleanly into -git, so we should be fine.

I'll get this series queued up as well, but it'll be for a later round early next week. JFYI.

Signature · 1 line
Jens Axboe
#7 lore raw

PATCH 4/5 cciss: limit commands allocated on reset_devices

In reply to Stephen M. Cameron

From: Stephen M. Cameron <[email protected]>

This is to conserve memory in a memory-limited kdump scenario

---
 drivers/block/cciss.c |    5 +++++
 1 files changed, 5 insertions(+), 0 deletions(-)
drivers/block/cciss.c +5 -0
index 03afef8..55fed21 100644
@@ -4039,6 +4039,11 @@ static int __devinit cciss_find_cfgtables(ctlr_info_t *h)
 static void __devinit cciss_get_max_perf_mode_cmds(struct ctlr_info *h)
 {
 	h->max_commands = readl(&(h->cfgtable->MaxPerformantModeCommands));
+
+	/* Limit commands in memory limited kdump scenario. */
+	if (reset_devices && h->max_commands > 32)
+		h->max_commands = 32;
+
 	if (h->max_commands < 16) {
 		dev_warn(&h->pdev->dev, "Controller reports "
 			"max supported commands of %d, an obvious lie. "
#8 lore raw

PATCH 5/5 cciss: use usleep_range not msleep for small sleeps

In reply to Stephen M. Cameron

From: Stephen M. Cameron <[email protected]>

---
 drivers/block/cciss.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
drivers/block/cciss.c +1 -1
index 55fed21..95bd799 100644
@@ -3749,7 +3749,7 @@ static void __devinit cciss_wait_for_mode_change_ack(ctlr_info_t *h)
 	for (i = 0; i < MAX_CONFIG_WAIT; i++) {
 		if (!(readl(h->vaddr + SA5_DOORBELL) & CFGTBL_ChangeReq))
 			break;
-		msleep(10);
+		usleep_range(10000, 20000);
 	}
 }