ioat fixes for 2.6.32
cover ioat fixes for 2.6.32
This primarily addresses driver stability issues around pq/xor-validate operations. The complete fix is staged for 2.6.33. Commit: "async_tx:
build-time toggling of async_{syndrome,xor}_val dma support" simply disables this operation type for 2.6.32.
The rest are 1-line or few-line fixes.
---
Dan Williams (8):
ioat3: fix pq completion versus channel deallocation race
async_tx: build-time toggling of async_{syndrome,xor}_val dma support
dmaengine: include xor/pq validate in device_has_all_tx_types() ioat2,3: report all uncorrectable errors
ioat3: specify valid address for disabled-Q or disabled-P ioat2,3: disable asynchronous error notifications
ioat3: dca and raid operations are incompatible
ioat: silence "dca disabled" messages
crypto/async_tx/Kconfig | 5 +++++ crypto/async_tx/async_pq.c | 14 +++++++++++--- crypto/async_tx/async_xor.c | 15 ++++++++++++--- drivers/dma/Kconfig | 2 ++ drivers/dma/dmaengine.c | 10 ++++++++++ drivers/dma/ioat/dca.c | 6 +++--- drivers/dma/ioat/dma.h | 4 +--- drivers/dma/ioat/dma_v2.c | 2 ++ drivers/dma/ioat/dma_v3.c | 42 ++++++++++++++++++++++++++++++++++++------ drivers/dma/ioat/hw.h | 2 ++ drivers/dma/ioat/registers.h | 4 +--- 11 files changed, 85 insertions(+), 21 deletions(-)
Signature · 1 line
Dan
PATCH 1/8 ioat: silence "dca disabled" messages
Turning off dca is not an "error", and the dca-enabled state can be viewed from sysfs.
--- drivers/dma/ioat/dca.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
drivers/dma/ioat/dca.c +2 -2
@@ -98,7 +98,7 @@ static int dca_enabled_in_bios(struct pci_dev *pdev) cpuid_level_9 =cpuid_eax (9 ); res =test_bit (0 , &cpuid_level_9);if (!res) -dev_err (&pdev->dev,"DCA is disabled in BIOS\n" ); +dev_dbg (&pdev->dev,"DCA is disabled in BIOS\n" );return res; }
@@ -108,7 +108,7 @@ static int system_has_dca_enabled(struct pci_dev *pdev)if (boot_cpu_has (X86_FEATURE_DCA ))return dca_enabled_in_bios (pdev); -dev_err (&pdev->dev,"boot cpu doesn't have X86_FEATURE_DCA\n" ); +dev_dbg (&pdev->dev,"boot cpu doesn't have X86_FEATURE_DCA\n" );return 0 ; }
PATCH 2/8 ioat3: dca and raid operations are incompatible
In reply to Dan WilliamsRAID operations cause a system hang on platforms with DCA (Direct-Cache-Access) enabled. So turn off RAID capabilities in this case.
--- drivers/dma/ioat/dca.c | 2 +- drivers/dma/ioat/dma_v3.c | 6 ++++++ drivers/dma/ioat/hw.h | 2 ++ 3 files changed, 9 insertions(+), 1 deletions(-)
drivers/dma/ioat/dca.c +1 -1
@@ -103,7 +103,7 @@ static int dca_enabled_in_bios(struct pci_dev *pdev)return res; } -static int system_has_dca_enabled (struct pci_dev *pdev) +int system_has_dca_enabled (struct pci_dev *pdev) {if (boot_cpu_has (X86_FEATURE_DCA ))return dca_enabled_in_bios (pdev);
drivers/dma/ioat/dma_v3.c +6 -0
@@ -1117,6 +1117,7 @@ static int __devinit ioat3_dma_self_test(struct ioatdma_device *device)int __devinit ioat3_dma_probe (struct ioatdma_device *device,int dca) {struct pci_dev *pdev = device->pdev; +int dca_en =system_has_dca_enabled (pdev);struct dma_device *dma;struct dma_chan *c;struct ioat_chan_common *chan;
@@ -1137,6 +1138,11 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock; cap =readl (device->reg_base +IOAT_DMA_CAP_OFFSET ); + +/* dca is incompatible with raid operations */ +if (dca_en && (cap & (IOAT_CAP_XOR |IOAT_CAP_PQ ))) + cap &= ~(IOAT_CAP_XOR |IOAT_CAP_PQ ); +if (cap &IOAT_CAP_XOR ) { is_raid_device = true; dma->max_xor =8 ;
drivers/dma/ioat/hw.h +2 -0
@@ -39,6 +39,8 @@#define IOAT_VER_3_0 0x30/* Version 3.0 */ #define IOAT_VER_3_2 0x32/* Version 3.2 */ +int system_has_dca_enabled (struct pci_dev *pdev); +struct ioat_dma_descriptor {uint32_t size;union {
PATCH 3/8 ioat2,3: disable asynchronous error notifications
In reply to Dan WilliamsError interrupts and error completions may cause channel hangs, so poll the channel status register after a timeout.
--- drivers/dma/ioat/registers.h | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-)
drivers/dma/ioat/registers.h +1 -3
@@ -92,9 +92,7 @@#define IOAT_CHANCTRL_ERR_COMPLETION_EN 0x0004#define IOAT_CHANCTRL_INT_REARM 0x0001#define IOAT_CHANCTRL_RUN (IOAT_CHANCTRL_INT_REARM |\ - IOAT_CHANCTRL_ERR_COMPLETION_EN |\ - IOAT_CHANCTRL_ANY_ERR_ABORT_EN |\ - IOAT_CHANCTRL_ERR_INT_EN) + IOAT_CHANCTRL_ANY_ERR_ABORT_EN)#define IOAT_DMA_COMP_OFFSET 0x02/* 16-bit DMA channel compatibility */ #define IOAT_DMA_COMP_V1 0x0001/* Compatibility with DMA version 1 */
PATCH 4/8 ioat3: specify valid address for disabled-Q or disabled-P
In reply to Dan WilliamsAlthough disabled, hardware still checks address validity, so duplicate the known address.
--- drivers/dma/ioat/dma_v3.c | 22 +++++++++++++++++----- 1 files changed, 17 insertions(+), 5 deletions(-)
drivers/dma/ioat/dma_v3.c +17 -5
@@ -736,10 +736,16 @@ ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, unsigned int src_cnt, const unsigned char *scf, size_t len, unsigned long flags) { + /* specify valid address for disabled result */ + if (flags & DMA_PREP_PQ_DISABLE_P) + dst[0 ] = dst[1 ]; +if (flags &DMA_PREP_PQ_DISABLE_Q ) + dst[1 ] = dst[0 ]; +/* handle the single source multiply case from the raid6 * recovery path */ - if (unlikely((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1)) { +if ((flags &DMA_PREP_PQ_DISABLE_P ) && src_cnt ==1 ) {dma_addr_t single_source[2 ];unsigned char single_source_coef[2 ];
@@ -761,6 +767,12 @@ ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, unsigned int src_cnt, const unsigned char *scf, size_t len, enum sum_check_flags *pqres, unsigned long flags) { + /* specify valid address for disabled result */ + if (flags & DMA_PREP_PQ_DISABLE_P) + pq[0 ] = pq[1 ]; +if (flags &DMA_PREP_PQ_DISABLE_Q ) + pq[1 ] = pq[0 ]; +/* the cleanup routine only sets bits on validate failure, it * does not clear bits on validate success... so clear it here */
@@ -778,9 +790,9 @@ ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,dma_addr_t pq[2 ];memset (scf,0 , src_cnt); - flags |=DMA_PREP_PQ_DISABLE_Q ; pq[0 ] = dst; - pq[1 ] = ~0 ; + flags |=DMA_PREP_PQ_DISABLE_Q ; + pq[1 ] = dst;/* specify valid address for disabled result */ return __ioat3_prep_pq_lock (chan,NULL , pq, src, src_cnt, scf, len, flags);
@@ -800,9 +812,9 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, *result =0 ;memset (scf,0 , src_cnt); - flags |=DMA_PREP_PQ_DISABLE_Q ; pq[0 ] = src[0 ]; - pq[1 ] = ~0 ; + flags |=DMA_PREP_PQ_DISABLE_Q ; + pq[1 ] = pq[0 ];/* specify valid address for disabled result */ return __ioat3_prep_pq_lock (chan, result, pq, &src[1 ], src_cnt -1 , scf, len, flags);
PATCH 5/8 ioat2,3: report all uncorrectable errors
In reply to Dan WilliamsModify is_ioat_bug() to catch all errors that are uncorrectable, or not currently handled.
--- drivers/dma/ioat/dma.h | 4 +--- drivers/dma/ioat/dma_v2.c | 2 ++ drivers/dma/ioat/dma_v3.c | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-)
drivers/dma/ioat/dma.h +1 -3
@@ -297,9 +297,7 @@ static inline bool is_ioat_suspended(unsigned long status)/* channel was fatally programmed */ static inline bool is_ioat_bug (unsigned long err) { -return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR |IOAT_CHANERR_DEST_ADDR_ERR | -IOAT_CHANERR_NEXT_ADDR_ERR |IOAT_CHANERR_CONTROL_ERR | -IOAT_CHANERR_LENGTH_ERR )); +return !!err; } static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
drivers/dma/ioat/dma_v2.c +2 -0
@@ -279,6 +279,8 @@ void ioat2_timer_event(unsigned long data)u32 chanerr; chanerr =readl (chan->reg_base +IOAT_CHANERR_OFFSET ); +dev_err (to_dev (chan),"%s: Channel halted (%x)\n" , + __func__, chanerr);BUG_ON (is_ioat_bug (chanerr)); }
drivers/dma/ioat/dma_v3.c +2 -0
@@ -378,6 +378,8 @@ static void ioat3_timer_event(unsigned long data)u32 chanerr; chanerr =readl (chan->reg_base +IOAT_CHANERR_OFFSET ); +dev_err (to_dev (chan),"%s: Channel halted (%x)\n" , + __func__, chanerr);BUG_ON (is_ioat_bug (chanerr)); }
PATCH 6/8 dmaengine: include xor/pq validate in device_has_all_tx_types()
In reply to Dan WilliamsA channel must include these capabilities to satisfy ASYNC_TX_DISABLE_CHANNEL_SWITCH.
--- drivers/dma/dmaengine.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
drivers/dma/dmaengine.c +4 -0
@@ -632,11 +632,15 @@ static bool device_has_all_tx_types(struct dma_device *device)#if defined(CONFIG_ASYNC_XOR ) || defined(CONFIG_ASYNC_XOR_MODULE )if (!dma_has_cap (DMA_XOR , device->cap_mask))return false; +if (!dma_has_cap (DMA_XOR_VAL , device->cap_mask)) +return false;#endif #if defined(CONFIG_ASYNC_PQ ) || defined(CONFIG_ASYNC_PQ_MODULE )if (!dma_has_cap (DMA_PQ , device->cap_mask))return false; +if (!dma_has_cap (DMA_PQ_VAL , device->cap_mask)) +return false;#endif return true;
PATCH 7/8 async_tx: build-time toggling of async_{syndrome, xor}_val dma support
In reply to Dan Williamsioat3.2 does not support asynchronous error notifications which makes the driver experience latencies when non-zero pq validate results are expected. Provide a mechanism for turning off async_xor_val and async_syndrome_val via Kconfig. This approach is generally useful for any driver that specifies ASYNC_TX_DISABLE_CHANNEL_SWITCH and would like to force the async_tx api to fall back to the synchronous path for certain operations.
--- crypto/async_tx/Kconfig | 5 +++++ crypto/async_tx/async_pq.c | 14 +++++++++++--- crypto/async_tx/async_xor.c | 15 ++++++++++++--- drivers/dma/Kconfig | 2 ++ drivers/dma/dmaengine.c | 6 ++++++ drivers/dma/ioat/dma_v3.c | 10 ++++++++++ 6 files changed, 46 insertions(+), 6 deletions(-)
crypto/async_tx/Kconfig +5 -0
@@ -23,3 +23,8 @@ config ASYNC_RAID6_RECOV select ASYNC_CORE select ASYNC_PQ +config ASYNC_TX_DISABLE_PQ_VAL_DMA + bool + +config ASYNC_TX_DISABLE_XOR_VAL_DMA + bool
crypto/async_tx/async_pq.c +11 -3
@@ -240,6 +240,16 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, }EXPORT_SYMBOL_GPL (async_gen_syndrome); +static inline struct dma_chan * +pq_val_chan (struct async_submit_ctl *submit,struct page **blocks,int disks,size_t len) +{ +#ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA +return NULL ; +#endif +return async_tx_find_channel (submit,DMA_PQ_VAL ,NULL ,0 , blocks, + disks, len); +} +/** * async_syndrome_val - asynchronously validate a raid6 syndrome * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1
@@ -260,9 +270,7 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks, size_t len, enum sum_check_flags *pqres, struct page *spare, struct async_submit_ctl *submit) { - struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ_VAL, -NULL ,0 , blocks, disks, - len); + struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len);struct dma_device *device = chan ? chan->device :NULL ;struct dma_async_tx_descriptor *tx;unsigned char coefs[disks-2 ];
crypto/async_tx/async_xor.c +12 -3
@@ -234,6 +234,17 @@ static int page_is_zero(struct page *p, unsigned int offset, size_t len) memcmp(a, a + 4, len - 4) == 0); } +static inline struct dma_chan * +xor_val_chan (struct async_submit_ctl *submit,struct page *dest, +struct page **src_list,int src_cnt,size_t len) +{ +#ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA +return NULL ; +#endif +return async_tx_find_channel (submit,DMA_XOR_VAL , &dest,1 , src_list, + src_cnt, len); +} +/** * async_xor_val - attempt a xor parity check with a dma engine. * @dest: destination page used if the xor is performed synchronously
@@ -255,9 +266,7 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, int src_cnt, size_t len, enum sum_check_flags *result, struct async_submit_ctl *submit) { - struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR_VAL, - &dest,1 , src_list, - src_cnt, len); + struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len);struct dma_device *device = chan ? chan->device :NULL ;struct dma_async_tx_descriptor *tx =NULL ;dma_addr_t *dma_src =NULL ;
drivers/dma/Kconfig +2 -0
@@ -26,6 +26,8 @@ config INTEL_IOATDMA select DMA_ENGINE select DCA select ASYNC_TX_DISABLE_CHANNEL_SWITCH + select ASYNC_TX_DISABLE_PQ_VAL_DMA + select ASYNC_TX_DISABLE_XOR_VAL_DMA help Enable support for the Intel(R) I/OAT DMA engine present in recent Intel Xeon chipsets.
drivers/dma/dmaengine.c +6 -0
@@ -632,16 +632,22 @@ static bool device_has_all_tx_types(struct dma_device *device)#if defined(CONFIG_ASYNC_XOR ) || defined(CONFIG_ASYNC_XOR_MODULE )if (!dma_has_cap (DMA_XOR , device->cap_mask))return false; + +#ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA if (!dma_has_cap (DMA_XOR_VAL , device->cap_mask))return false;#endif +#endif #if defined(CONFIG_ASYNC_PQ ) || defined(CONFIG_ASYNC_PQ_MODULE )if (!dma_has_cap (DMA_PQ , device->cap_mask))return false; + +#ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA if (!dma_has_cap (DMA_PQ_VAL , device->cap_mask))return false;#endif +#endif return true; }
drivers/dma/ioat/dma_v3.c +10 -0
@@ -1206,6 +1206,16 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) device->timer_fn = ioat2_timer_event; } +#ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA +dma_cap_clear (DMA_PQ_VAL , dma->cap_mask); + dma->device_prep_dma_pq_val =NULL ; +#endif + +#ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA +dma_cap_clear (DMA_XOR_VAL , dma->cap_mask); + dma->device_prep_dma_xor_val =NULL ; +#endif +/* -= IOAT ver.3 workarounds =- */ /* Write CHANERRMSK_INT with 3E07h to mask out the errors * that can cause stability issues for IOAT ver.3
PATCH 8/8 ioat3: fix pq completion versus channel deallocation race
In reply to Dan WilliamsThe completion of a pq operation is notified with a null descriptor appended to the end of the chain. This descriptor needs to be visible to dma clients otherwise the client is precluded from ensuring all operations are quiesced before freeing channel resources, i.e. due to descriptor polling it may get the completion notification ahead of the interrupt delivered by the null descriptor.
--- drivers/dma/ioat/dma_v3.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
drivers/dma/ioat/dma_v3.c +1 -1
@@ -730,7 +730,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,dump_desc_dbg (ioat, compl_desc);/* we leave the channel locked to ensure in order submission */ -return &desc->txd; +return &compl_desc->txd; } static struct dma_async_tx_descriptor *