Asynchronous raid6 acceleration (part 2 of 3)
Subsystems (4)
cover Asynchronous raid6 acceleration (part 2 of 3)
This series adds P+Q operations to the iop-adma driver. Part1 [1] added the async_tx api extensions for raid6. Part3 includes the changes to md/raid6 to utilize the api extensions. All three topics are merged into the 'raid6' branch of async_tx.git:
git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx.git raid6
This implementation has been verified with the raid6test module [2].
Please review.
Thanks,
Dan
[1] http://marc.info/?l=linux-raid&m=124269482227272&w=2
[2] http://marc.info/?l=linux-raid&m=124269519527629&w=2
---
Dan Williams (3):
iop-adma: P+Q self test
iop-adma: P+Q support for iop13xx adma engines
iop-adma: cleanup iop_adma_run_tx_complete_actions
arch/arm/include/asm/hardware/iop3xx-adma.h | 76 +++++ arch/arm/include/asm/hardware/iop_adma.h | 1 arch/arm/mach-iop13xx/include/mach/adma.h | 107 +++++++ drivers/dma/iop-adma.c | 427 ++++++++++++++++++++++++--- 4 files changed, 568 insertions(+), 43 deletions(-)
PATCH 1/3 iop-adma: cleanup iop_adma_run_tx_complete_actions
Replace 'desc->async_tx.' with 'tx->'
[ Impact: pure cleanup ]
--- drivers/dma/iop-adma.c | 19 ++++++++++--------- 1 files changed, 10 insertions(+), 9 deletions(-)
drivers/dma/iop-adma.c +10 -9
@@ -61,17 +61,18 @@ static dma_cookie_t iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, struct iop_adma_chan *iop_chan, dma_cookie_t cookie) { - BUG_ON(desc->async_tx.cookie < 0); -if (desc->async_tx.cookie >0 ) { - cookie = desc->async_tx.cookie; - desc->async_tx.cookie =0 ; + struct dma_async_tx_descriptor *tx = &desc->async_tx; + +BUG_ON (tx->cookie <0 ); +if (tx->cookie >0 ) { + cookie = tx->cookie; + tx->cookie =0 ;/* call the callback (must not sleep or submit new * operations to this channel) */ -if (desc->async_tx.callback) - desc->async_tx.callback ( - desc->async_tx.callback_param); +if (tx->callback) + tx->callback (tx->callback_param);/* unmap dma addresses * (unmap_single vs unmap_page?)
@@ -81,7 +82,7 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,struct device *dev = &iop_chan->device->pdev->dev;u32 len = unmap->unmap_len; -enum dma_ctrl_flags flags = desc->async_tx.flags; +enum dma_ctrl_flags flags = tx->flags;u32 src_cnt;dma_addr_t addr;dma_addr_t dest;
@@ -115,7 +116,7 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, }/* run dependent operations */ -dma_run_dependencies (&desc->async_tx); +dma_run_dependencies (tx);return cookie; }
PATCH 2/3 iop-adma: P+Q support for iop13xx adma engines
In reply to Dan Williamsiop33x support is not included because that engine is a bit more awkward to handle in that it can either be in xor mode or pq mode. The dmaengine/async_tx layers currently only comprehend static capabilities.
Note iop13xx does not support hardware PQ continuation so the driver must handle the DMA_PREP_CONTINUE flag for operations across > 16 sources. From the comment for dma_maxpq:
/* When an engine does not support native continuation we need 3 extra
* source slots to reuse P and Q with the following coefficients:
* 1/ {00} * P : remove P from Q', but use it as a source for P'
* 2/ {01} * Q : use Q to continue Q' calculation
* 3/ {00} * Q : subtract Q from P' to cancel (2)
*/
--- arch/arm/include/asm/hardware/iop3xx-adma.h | 76 +++++++++ arch/arm/include/asm/hardware/iop_adma.h | 1 arch/arm/mach-iop13xx/include/mach/adma.h | 107 +++++++++++++ drivers/dma/iop-adma.c | 228 +++++++++++++++++++++++---- 4 files changed, 378 insertions(+), 34 deletions(-)
arch/arm/include/asm/hardware/iop3xx-adma.h +76 -0
@@ -187,11 +187,74 @@ union iop3xx_desc {void *ptr; }; +/* No support for p+q operations */ +static inline int +iop_chan_pq_slot_count (size_t len,int src_cnt,int *slots_per_op) +{ +BUG (); +return 0 ; +} + +static inline void +iop_desc_init_pq (struct iop_adma_desc_slot *desc,int src_cnt, +unsigned long flags) +{ +BUG (); +} + +static inline void +iop_desc_set_pq_addr (struct iop_adma_desc_slot *desc,dma_addr_t *addr) +{ +BUG (); +} + +static inline void +iop_desc_set_pq_src_addr (struct iop_adma_desc_slot *desc,int src_idx, +dma_addr_t addr,unsigned char coef) +{ +BUG (); +} + +static inline int +iop_chan_pq_zero_sum_slot_count (size_t len,int src_cnt,int *slots_per_op) +{ +BUG (); +return 0 ; +} + +static inline void +iop_desc_init_pq_zero_sum (struct iop_adma_desc_slot *desc,int src_cnt, +unsigned long flags) +{ +BUG (); +} + +static inline void +iop_desc_set_pq_zero_sum_byte_count (struct iop_adma_desc_slot *desc,u32 len) +{ +BUG (); +} + +#define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr + +static inline void +iop_desc_set_pq_zero_sum_addr (struct iop_adma_desc_slot *desc,int pq_idx, +dma_addr_t *src) +{ +BUG (); +} +static inline int iop_adma_get_max_xor (void ) {return 32 ; } +static inline int iop_adma_get_max_pq (void ) +{ +BUG (); +return 0 ; +} +static inline u32 iop_chan_get_current_descriptor (struct iop_adma_chan *chan) {int id = chan->device->id;
@@ -332,6 +395,11 @@ static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt,return slot_cnt; } +static inline int iop_desc_is_pq (struct iop_adma_desc_slot *desc) +{ +return 0 ; +} +static inline u32 iop_desc_get_dest_addr (struct iop_adma_desc_slot *desc,struct iop_adma_chan *chan) {
@@ -349,6 +417,14 @@ static inline u32 iop_desc_get_dest_addr(struct iop_adma_desc_slot *desc,return 0 ; } + +static inline u32 iop_desc_get_qdest_addr (struct iop_adma_desc_slot *desc, +struct iop_adma_chan *chan) +{ +BUG (); +return 0 ; +} +static inline u32 iop_desc_get_byte_count (struct iop_adma_desc_slot *desc,struct iop_adma_chan *chan) {
arch/arm/include/asm/hardware/iop_adma.h +1 -0
@@ -106,6 +106,7 @@ struct iop_adma_desc_slot {union {u32 *xor_check_result;u32 *crc32_result; +u32 *pq_check_result; }; };
arch/arm/mach-iop13xx/include/mach/adma.h +107 -0
156 lines not on this page · raw message
drivers/dma/iop-adma.c +194 -34
252 lines not on this page · raw message
PATCH 3/3 iop-adma: P+Q self test
In reply to Dan WilliamsEven though the intent is to extend dmatest with P+Q tests there is still value in having an always-on sanity check to prevent an unintentionally broken driver from registering.
This depends on raid6_pq.ko for verification, the side effect being that PQ capable channels will fail to register when raid6 is disabled.
--- drivers/dma/iop-adma.c | 182 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 181 insertions(+), 1 deletions(-)
drivers/dma/iop-adma.c +181 -1
@@ -31,6 +31,7 @@#include <linux/platform_device.h> #include <linux/memory.h> #include <linux/ioport.h> +#include <linux/raid/pq.h> #include <mach/adma.h>
@@ -1259,6 +1260,170 @@ out:return err; } +#ifdef CONFIG_MD_RAID6_PQ +static int __devinit +iop_adma_pq_zero_sum_self_test (struct iop_adma_device *device) +{ +/* combined sources, software pq results, and extra hw pq results */ +struct page *pq[IOP_ADMA_NUM_SRC_TEST +2 +2 ]; +/* ptr to the extra hw pq buffers defined above */ +struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST +2 ]; +/* address conversion buffers (dma_map / page_address) */ +void *pq_sw[IOP_ADMA_NUM_SRC_TEST +2 ]; +dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST ]; +dma_addr_t pq_dest[2 ]; + +int i; +struct dma_async_tx_descriptor *tx; +struct dma_chan *dma_chan; +dma_cookie_t cookie; +u32 zero_sum_result; +int err =0 ; +struct device *dev; + +dev_dbg (device->common.dev,"%s\n" , __func__); + +for (i =0 ; i <ARRAY_SIZE (pq); i++) { + pq[i] =alloc_page (GFP_KERNEL ); +if (!pq[i]) { +while (i--) +__free_page (pq[i]); +return -ENOMEM ; + } + } + +/* Fill in src buffers */ +for (i =0 ; i <IOP_ADMA_NUM_SRC_TEST ; i++) { + pq_sw[i] =page_address (pq[i]); +memset (pq_sw[i],0x11111111 * (1 <<i),PAGE_SIZE ); + } + pq_sw[i] =page_address (pq[i]); + pq_sw[i+1 ] =page_address (pq[i+1 ]); + + dma_chan =container_of (device->common.channels.next, + struct dma_chan, + device_node); +if (iop_adma_alloc_chan_resources (dma_chan) <1 ) { + err = -ENODEV ; + goto out; + } + + dev = dma_chan->device->dev; + +/* initialize the dests */ +memset (page_address (pq_hw[0 ]),0 ,PAGE_SIZE ); +memset (page_address (pq_hw[1 ]),0 ,PAGE_SIZE ); + +/* test pq */ + pq_dest[0 ] =dma_map_page (dev, pq_hw[0 ],0 ,PAGE_SIZE ,DMA_FROM_DEVICE ); + pq_dest[1 ] =dma_map_page (dev, pq_hw[1 ],0 ,PAGE_SIZE ,DMA_FROM_DEVICE ); +for (i =0 ; i <IOP_ADMA_NUM_SRC_TEST ; i++) + pq_src[i] =dma_map_page (dev, pq[i],0 ,PAGE_SIZE , +DMA_TO_DEVICE ); + + tx =iop_adma_prep_dma_pq (dma_chan, pq_dest, pq_src, +IOP_ADMA_NUM_SRC_TEST , (u8 *)raid6_gfexp, +PAGE_SIZE , +DMA_PREP_INTERRUPT | +DMA_CTRL_ACK ); + + cookie =iop_adma_tx_submit (tx); +iop_adma_issue_pending (dma_chan); +msleep (8 ); + +if (iop_adma_is_complete (dma_chan, cookie,NULL ,NULL ) != +DMA_SUCCESS ) { +dev_err (dev,"Self-test pq timed out, disabling\n" ); + err = -ENODEV ; + goto free_resources; + } + + raid6_call.gen_syndrome (IOP_ADMA_NUM_SRC_TEST +2 ,PAGE_SIZE , pq_sw); + +if (memcmp (pq_sw[IOP_ADMA_NUM_SRC_TEST ], +page_address (pq_hw[0 ]),PAGE_SIZE ) !=0 ) { +dev_err (dev,"Self-test p failed compare, disabling\n" ); + err = -ENODEV ; + goto free_resources; + } +if (memcmp (pq_sw[IOP_ADMA_NUM_SRC_TEST +1 ], +page_address (pq_hw[1 ]),PAGE_SIZE ) !=0 ) { +dev_err (dev,"Self-test q failed compare, disabling\n" ); + err = -ENODEV ; + goto free_resources; + } + +/* test correct zero sum using the software generated pq values */ +for (i =0 ; i <IOP_ADMA_NUM_SRC_TEST +2 ; i++) + pq_src[i] =dma_map_page (dev, pq[i],0 ,PAGE_SIZE , +DMA_TO_DEVICE ); + + zero_sum_result = ~0 ; + tx =iop_adma_prep_dma_pq_val (dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST ], + pq_src,IOP_ADMA_NUM_SRC_TEST , + raid6_gfexp,PAGE_SIZE , &zero_sum_result, +DMA_PREP_INTERRUPT |DMA_CTRL_ACK ); + + cookie =iop_adma_tx_submit (tx); +iop_adma_issue_pending (dma_chan); +msleep (8 ); + +if (iop_adma_is_complete (dma_chan, cookie,NULL ,NULL ) != +DMA_SUCCESS ) { +dev_err (dev,"Self-test pq-zero-sum timed out, disabling\n" ); + err = -ENODEV ; + goto free_resources; + } + +if (zero_sum_result !=0 ) { +dev_err (dev,"Self-test pq-zero-sum failed to validate: %x\n" , + zero_sum_result); + err = -ENODEV ; + goto free_resources; + } + +/* test incorrect zero sum */ + i =IOP_ADMA_NUM_SRC_TEST ; +memset (pq_sw[i] +100 ,0 ,100 ); +memset (pq_sw[i+1 ] +200 ,0 ,200 ); +for (i =0 ; i <IOP_ADMA_NUM_SRC_TEST +2 ; i++) + pq_src[i] =dma_map_page (dev, pq[i],0 ,PAGE_SIZE , +DMA_TO_DEVICE ); + + zero_sum_result =0 ; + tx =iop_adma_prep_dma_pq_val (dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST ], + pq_src,IOP_ADMA_NUM_SRC_TEST , + raid6_gfexp,PAGE_SIZE , &zero_sum_result, +DMA_PREP_INTERRUPT |DMA_CTRL_ACK ); + + cookie =iop_adma_tx_submit (tx); +iop_adma_issue_pending (dma_chan); +msleep (8 ); + +if (iop_adma_is_complete (dma_chan, cookie,NULL ,NULL ) != +DMA_SUCCESS ) { +dev_err (dev,"Self-test !pq-zero-sum timed out, disabling\n" ); + err = -ENODEV ; + goto free_resources; + } + +if (zero_sum_result != (SUM_CHECK_P_RESULT |SUM_CHECK_Q_RESULT )) { +dev_err (dev,"Self-test !pq-zero-sum failed to validate: %x\n" , + zero_sum_result); + err = -ENODEV ; + goto free_resources; + } + +free_resources: +iop_adma_free_chan_resources (dma_chan); +out: + i =ARRAY_SIZE (pq); +while (i--) +__free_page (pq[i]); +return err; +} +#endif + static int __devexit iop_adma_remove(struct platform_device *dev) {struct iop_adma_device *device =platform_get_drvdata (dev);
@@ -1409,13 +1574,28 @@ static int __devinit iop_adma_probe(struct platform_device *pdev) }if (dma_has_cap (DMA_XOR , dma_dev->cap_mask) || -dma_has_cap (DMA_MEMSET , dma_dev->cap_mask)) { +dma_has_cap (DMA_MEMSET , dma_dev->cap_mask)) { ret =iop_adma_xor_val_self_test (adev);dev_dbg (&pdev->dev,"xor self test returned %d\n" , ret);if (ret) goto err_free_iop_chan; } +if (dma_has_cap (DMA_PQ , dma_dev->cap_mask) && +dma_has_cap (DMA_PQ_VAL , dma_dev->cap_mask)) { +#ifdef CONFIG_MD_RAID6_PQ + ret =iop_adma_pq_zero_sum_self_test (adev); +dev_dbg (&pdev->dev,"pq self test returned %d\n" , ret); +#else +/* can not test raid6, so do not publish capability */ +dma_cap_clear (DMA_PQ , dma_dev->cap_mask); +dma_cap_clear (DMA_PQ_VAL , dma_dev->cap_mask); + ret =0 ; +#endif +if (ret) + goto err_free_iop_chan; + } + dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: " "( %s%s%s%s%s%s%s%s%s%s)\n", dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",