Skip to contentSkip to conversation

ioat fixes for 2.6.32

PATCH New Dan Williams 1 participant 9 messages Nov 20, 2009

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

cover ioat fixes for 2.6.32

This primarily addresses driver stability issues around pq/xor-validate operations. The complete fix is staged for 2.6.33. Commit: "async_tx:
build-time toggling of async_{syndrome,xor}_val dma support" simply disables this operation type for 2.6.32.

The rest are 1-line or few-line fixes.

---

Dan Williams (8):
ioat3: fix pq completion versus channel deallocation race
async_tx: build-time toggling of async_{syndrome,xor}_val dma support
dmaengine: include xor/pq validate in device_has_all_tx_types() ioat2,3: report all uncorrectable errors
ioat3: specify valid address for disabled-Q or disabled-P ioat2,3: disable asynchronous error notifications
ioat3: dca and raid operations are incompatible
ioat: silence "dca disabled" messages

 crypto/async_tx/Kconfig      |    5 +++++
 crypto/async_tx/async_pq.c   |   14 +++++++++++---
 crypto/async_tx/async_xor.c  |   15 ++++++++++++---
 drivers/dma/Kconfig          |    2 ++
 drivers/dma/dmaengine.c      |   10 ++++++++++
 drivers/dma/ioat/dca.c       |    6 +++---
 drivers/dma/ioat/dma.h       |    4 +---
 drivers/dma/ioat/dma_v2.c    |    2 ++
 drivers/dma/ioat/dma_v3.c    |   42 ++++++++++++++++++++++++++++++++++++------
 drivers/dma/ioat/hw.h        |    2 ++
 drivers/dma/ioat/registers.h |    4 +---
 11 files changed, 85 insertions(+), 21 deletions(-)
Signature · 1 line
Dan
#1 lore raw

PATCH 1/8 ioat: silence "dca disabled" messages

Turning off dca is not an "error", and the dca-enabled state can be viewed from sysfs.

---
 drivers/dma/ioat/dca.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)
drivers/dma/ioat/dca.c +2 -2
index 69d0261..14f0a78 100644
@@ -98,7 +98,7 @@ static int dca_enabled_in_bios(struct pci_dev *pdev)
 	cpuid_level_9 = cpuid_eax(9);
 	res = test_bit(0, &cpuid_level_9);
 	if (!res)
-		dev_err(&pdev->dev, "DCA is disabled in BIOS\n");
+		dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n");
 
 	return res;
 }
@@ -108,7 +108,7 @@ static int system_has_dca_enabled(struct pci_dev *pdev)
 	if (boot_cpu_has(X86_FEATURE_DCA))
 		return dca_enabled_in_bios(pdev);
 
-	dev_err(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n");
+	dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n");
 	return 0;
 }
 
#2 lore raw

PATCH 2/8 ioat3: dca and raid operations are incompatible

In reply to Dan Williams

RAID operations cause a system hang on platforms with DCA (Direct-Cache-Access) enabled. So turn off RAID capabilities in this case.

---
 drivers/dma/ioat/dca.c    |    2 +-
 drivers/dma/ioat/dma_v3.c |    6 ++++++
 drivers/dma/ioat/hw.h     |    2 ++
 3 files changed, 9 insertions(+), 1 deletions(-)
drivers/dma/ioat/dca.c +1 -1
index 14f0a78..abd9038 100644
@@ -103,7 +103,7 @@ static int dca_enabled_in_bios(struct pci_dev *pdev)
 	return res;
 }
 
-static int system_has_dca_enabled(struct pci_dev *pdev)
+int system_has_dca_enabled(struct pci_dev *pdev)
 {
 	if (boot_cpu_has(X86_FEATURE_DCA))
 		return dca_enabled_in_bios(pdev);
drivers/dma/ioat/dma_v3.c +6 -0
index 35d1e33..4f305f6 100644
@@ -1117,6 +1117,7 @@ static int __devinit ioat3_dma_self_test(struct ioatdma_device *device)
 int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
 {
 	struct pci_dev *pdev = device->pdev;
+	int dca_en = system_has_dca_enabled(pdev);
 	struct dma_device *dma;
 	struct dma_chan *c;
 	struct ioat_chan_common *chan;
@@ -1137,6 +1138,11 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
 	dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock;
 
 	cap = readl(device->reg_base + IOAT_DMA_CAP_OFFSET);
+
+	/* dca is incompatible with raid operations */
+	if (dca_en && (cap & (IOAT_CAP_XOR|IOAT_CAP_PQ)))
+		cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ);
+
 	if (cap & IOAT_CAP_XOR) {
 		is_raid_device = true;
 		dma->max_xor = 8;
drivers/dma/ioat/hw.h +2 -0
index 99afb12..60e6754 100644
@@ -39,6 +39,8 @@
 #define IOAT_VER_3_0            0x30    /* Version 3.0 */
 #define IOAT_VER_3_2            0x32    /* Version 3.2 */
 
+int system_has_dca_enabled(struct pci_dev *pdev);
+
 struct ioat_dma_descriptor {
 	uint32_t	size;
 	union {
#3 lore raw

PATCH 3/8 ioat2,3: disable asynchronous error notifications

In reply to Dan Williams

Error interrupts and error completions may cause channel hangs, so poll the channel status register after a timeout.

---
 drivers/dma/ioat/registers.h |    4 +---
 1 files changed, 1 insertions(+), 3 deletions(-)
drivers/dma/ioat/registers.h +1 -3
index 63038e1..f015ec1 100644
@@ -92,9 +92,7 @@
 #define IOAT_CHANCTRL_ERR_COMPLETION_EN		0x0004
 #define IOAT_CHANCTRL_INT_REARM			0x0001
 #define IOAT_CHANCTRL_RUN			(IOAT_CHANCTRL_INT_REARM |\
-						 IOAT_CHANCTRL_ERR_COMPLETION_EN |\
-						 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |\
-						 IOAT_CHANCTRL_ERR_INT_EN)
+						 IOAT_CHANCTRL_ANY_ERR_ABORT_EN)
 
 #define IOAT_DMA_COMP_OFFSET			0x02	/* 16-bit DMA channel compatibility */
 #define IOAT_DMA_COMP_V1			0x0001	/* Compatibility with DMA version 1 */
#4 lore raw

PATCH 4/8 ioat3: specify valid address for disabled-Q or disabled-P

In reply to Dan Williams

Although disabled, hardware still checks address validity, so duplicate the known address.

---
 drivers/dma/ioat/dma_v3.c |   22 +++++++++++++++++-----
 1 files changed, 17 insertions(+), 5 deletions(-)
drivers/dma/ioat/dma_v3.c +17 -5
index 4f305f6..fad2e9f 100644
@@ -736,10 +736,16 @@ ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
 	      unsigned int src_cnt, const unsigned char *scf, size_t len,
 	      unsigned long flags)
 {
+	/* specify valid address for disabled result */
+	if (flags & DMA_PREP_PQ_DISABLE_P)
+		dst[0] = dst[1];
+	if (flags & DMA_PREP_PQ_DISABLE_Q)
+		dst[1] = dst[0];
+
 	/* handle the single source multiply case from the raid6
 	 * recovery path
 	 */
-	if (unlikely((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1)) {
+	if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) {
 		dma_addr_t single_source[2];
 		unsigned char single_source_coef[2];
 
@@ -761,6 +767,12 @@ ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
 		  unsigned int src_cnt, const unsigned char *scf, size_t len,
 		  enum sum_check_flags *pqres, unsigned long flags)
 {
+	/* specify valid address for disabled result */
+	if (flags & DMA_PREP_PQ_DISABLE_P)
+		pq[0] = pq[1];
+	if (flags & DMA_PREP_PQ_DISABLE_Q)
+		pq[1] = pq[0];
+
 	/* the cleanup routine only sets bits on validate failure, it
 	 * does not clear bits on validate success... so clear it here
 	 */
@@ -778,9 +790,9 @@ ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
 	dma_addr_t pq[2];
 
 	memset(scf, 0, src_cnt);
-	flags |= DMA_PREP_PQ_DISABLE_Q;
 	pq[0] = dst;
-	pq[1] = ~0;
+	flags |= DMA_PREP_PQ_DISABLE_Q;
+	pq[1] = dst; /* specify valid address for disabled result */
 
 	return __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
 				    flags);
@@ -800,9 +812,9 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
 	*result = 0;
 
 	memset(scf, 0, src_cnt);
-	flags |= DMA_PREP_PQ_DISABLE_Q;
 	pq[0] = src[0];
-	pq[1] = ~0;
+	flags |= DMA_PREP_PQ_DISABLE_Q;
+	pq[1] = pq[0]; /* specify valid address for disabled result */
 
 	return __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, scf,
 				    len, flags);
#5 lore raw

PATCH 5/8 ioat2,3: report all uncorrectable errors

In reply to Dan Williams

Modify is_ioat_bug() to catch all errors that are uncorrectable, or not currently handled.

---
 drivers/dma/ioat/dma.h    |    4 +---
 drivers/dma/ioat/dma_v2.c |    2 ++
 drivers/dma/ioat/dma_v3.c |    2 ++
 3 files changed, 5 insertions(+), 3 deletions(-)
drivers/dma/ioat/dma.h +1 -3
index c14fdfe..45edde9 100644
@@ -297,9 +297,7 @@ static inline bool is_ioat_suspended(unsigned long status)
 /* channel was fatally programmed */
 static inline bool is_ioat_bug(unsigned long err)
 {
-	return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR|IOAT_CHANERR_DEST_ADDR_ERR|
-			 IOAT_CHANERR_NEXT_ADDR_ERR|IOAT_CHANERR_CONTROL_ERR|
-			 IOAT_CHANERR_LENGTH_ERR));
+	return !!err;
 }
 
 static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
drivers/dma/ioat/dma_v2.c +2 -0
index 96ffab7..8f1f7f0 100644
@@ -279,6 +279,8 @@ void ioat2_timer_event(unsigned long data)
 			u32 chanerr;
 
 			chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+			dev_err(to_dev(chan), "%s: Channel halted (%x)\n",
+				__func__, chanerr);
 			BUG_ON(is_ioat_bug(chanerr));
 		}
 
drivers/dma/ioat/dma_v3.c +2 -0
index fad2e9f..252cf2d 100644
@@ -378,6 +378,8 @@ static void ioat3_timer_event(unsigned long data)
 			u32 chanerr;
 
 			chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+			dev_err(to_dev(chan), "%s: Channel halted (%x)\n",
+				__func__, chanerr);
 			BUG_ON(is_ioat_bug(chanerr));
 		}
 
#6 lore raw

PATCH 6/8 dmaengine: include xor/pq validate in device_has_all_tx_types()

In reply to Dan Williams

A channel must include these capabilities to satisfy ASYNC_TX_DISABLE_CHANNEL_SWITCH.

---
 drivers/dma/dmaengine.c |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)
drivers/dma/dmaengine.c +4 -0
index bd0b248..b6442f0 100644
@@ -632,11 +632,15 @@ static bool device_has_all_tx_types(struct dma_device *device)
 	#if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE)
 	if (!dma_has_cap(DMA_XOR, device->cap_mask))
 		return false;
+	if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask))
+		return false;
 	#endif
 
 	#if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE)
 	if (!dma_has_cap(DMA_PQ, device->cap_mask))
 		return false;
+	if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask))
+		return false;
 	#endif
 
 	return true;
#7 lore raw

PATCH 7/8 async_tx: build-time toggling of async_{syndrome, xor}_val dma support

In reply to Dan Williams

ioat3.2 does not support asynchronous error notifications which makes the driver experience latencies when non-zero pq validate results are expected. Provide a mechanism for turning off async_xor_val and async_syndrome_val via Kconfig. This approach is generally useful for any driver that specifies ASYNC_TX_DISABLE_CHANNEL_SWITCH and would like to force the async_tx api to fall back to the synchronous path for certain operations.

---
 crypto/async_tx/Kconfig     |    5 +++++
 crypto/async_tx/async_pq.c  |   14 +++++++++++---
 crypto/async_tx/async_xor.c |   15 ++++++++++++---
 drivers/dma/Kconfig         |    2 ++
 drivers/dma/dmaengine.c     |    6 ++++++
 drivers/dma/ioat/dma_v3.c   |   10 ++++++++++
 6 files changed, 46 insertions(+), 6 deletions(-)
crypto/async_tx/Kconfig +5 -0
index e5aeb2b..e28e276 100644
@@ -23,3 +23,8 @@ config ASYNC_RAID6_RECOV
 	select ASYNC_CORE
 	select ASYNC_PQ
 
+config ASYNC_TX_DISABLE_PQ_VAL_DMA
+	bool
+
+config ASYNC_TX_DISABLE_XOR_VAL_DMA
+	bool
crypto/async_tx/async_pq.c +11 -3
index 6b5cc4f..ec87f53 100644
@@ -240,6 +240,16 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
 }
 EXPORT_SYMBOL_GPL(async_gen_syndrome);
 
+static inline struct dma_chan *
+pq_val_chan(struct async_submit_ctl *submit, struct page **blocks, int disks, size_t len)
+{
+	#ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
+	return NULL;
+	#endif
+	return async_tx_find_channel(submit, DMA_PQ_VAL, NULL, 0,  blocks,
+				     disks, len);
+}
+
 /**
  * async_syndrome_val - asynchronously validate a raid6 syndrome
  * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1
@@ -260,9 +270,7 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks,
 		   size_t len, enum sum_check_flags *pqres, struct page *spare,
 		   struct async_submit_ctl *submit)
 {
-	struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ_VAL,
-						      NULL, 0,  blocks, disks,
-						      len);
+	struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len);
 	struct dma_device *device = chan ? chan->device : NULL;
 	struct dma_async_tx_descriptor *tx;
 	unsigned char coefs[disks-2];
crypto/async_tx/async_xor.c +12 -3
index 79182dc..079ae8c 100644
@@ -234,6 +234,17 @@ static int page_is_zero(struct page *p, unsigned int offset, size_t len)
 		memcmp(a, a + 4, len - 4) == 0);
 }
 
+static inline struct dma_chan *
+xor_val_chan(struct async_submit_ctl *submit, struct page *dest,
+		 struct page **src_list, int src_cnt, size_t len)
+{
+	#ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
+	return NULL;
+	#endif
+	return async_tx_find_channel(submit, DMA_XOR_VAL, &dest, 1, src_list,
+				     src_cnt, len);
+}
+
 /**
  * async_xor_val - attempt a xor parity check with a dma engine.
  * @dest: destination page used if the xor is performed synchronously
@@ -255,9 +266,7 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
 	      int src_cnt, size_t len, enum sum_check_flags *result,
 	      struct async_submit_ctl *submit)
 {
-	struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR_VAL,
-						      &dest, 1, src_list,
-						      src_cnt, len);
+	struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len);
 	struct dma_device *device = chan ? chan->device : NULL;
 	struct dma_async_tx_descriptor *tx = NULL;
 	dma_addr_t *dma_src = NULL;
drivers/dma/Kconfig +2 -0
index 5903a88..b401dad 100644
@@ -26,6 +26,8 @@ config INTEL_IOATDMA
 	select DMA_ENGINE
 	select DCA
 	select ASYNC_TX_DISABLE_CHANNEL_SWITCH
+	select ASYNC_TX_DISABLE_PQ_VAL_DMA
+	select ASYNC_TX_DISABLE_XOR_VAL_DMA
 	help
 	  Enable support for the Intel(R) I/OAT DMA engine present
 	  in recent Intel Xeon chipsets.
drivers/dma/dmaengine.c +6 -0
index b6442f0..8f99354 100644
@@ -632,16 +632,22 @@ static bool device_has_all_tx_types(struct dma_device *device)
 	#if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE)
 	if (!dma_has_cap(DMA_XOR, device->cap_mask))
 		return false;
+
+	#ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
 	if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask))
 		return false;
 	#endif
+	#endif
 
 	#if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE)
 	if (!dma_has_cap(DMA_PQ, device->cap_mask))
 		return false;
+
+	#ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
 	if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask))
 		return false;
 	#endif
+	#endif
 
 	return true;
 }
drivers/dma/ioat/dma_v3.c +10 -0
index 252cf2d..189788f 100644
@@ -1206,6 +1206,16 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
 		device->timer_fn = ioat2_timer_event;
 	}
 
+	#ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
+	dma_cap_clear(DMA_PQ_VAL, dma->cap_mask);
+	dma->device_prep_dma_pq_val = NULL;
+	#endif
+
+	#ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
+	dma_cap_clear(DMA_XOR_VAL, dma->cap_mask);
+	dma->device_prep_dma_xor_val = NULL;
+	#endif
+
 	/* -= IOAT ver.3 workarounds =- */
 	/* Write CHANERRMSK_INT with 3E07h to mask out the errors
 	 * that can cause stability issues for IOAT ver.3
#8 lore raw

PATCH 8/8 ioat3: fix pq completion versus channel deallocation race

In reply to Dan Williams

The completion of a pq operation is notified with a null descriptor appended to the end of the chain. This descriptor needs to be visible to dma clients otherwise the client is precluded from ensuring all operations are quiesced before freeing channel resources, i.e. due to descriptor polling it may get the completion notification ahead of the interrupt delivered by the null descriptor.

---
 drivers/dma/ioat/dma_v3.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
drivers/dma/ioat/dma_v3.c +1 -1
index 189788f..e30e0ac 100644
@@ -730,7 +730,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
 	dump_desc_dbg(ioat, compl_desc);
 
 	/* we leave the channel locked to ensure in order submission */
-	return &desc->txd;
+	return &compl_desc->txd;
 }
 
 static struct dma_async_tx_descriptor *