async_tx: add support for asynchronous GF multiplication
PATCH v3 2/11 async_tx: add support for asynchronous GF multiplication
This adds support for doing asynchronous GF multiplication by adding four additional functions to async_tx API:
async_pq() does simultaneous XOR of sources and XOR of sources
GF-multiplied by given coefficients.
async_pq_zero_sum() checks if results of calculations match given
ones.
async_gen_syndrome() does sumultaneous XOR and R/S syndrome of sources.
async_syndrome_zerosum() checks if results of XOR/syndrome calculation
matches given ones.
Latter two functions just use async_pq() with the approprite coefficients in asynchronous case but have significant optimizations if synchronous case.
To support this API dmaengine driver should set DMA_PQ and DMA_PQ_ZERO_SUM capabilities and provide device_prep_dma_pq and device_prep_dma_pqzero_sum methods in dma_device structure.
--- crypto/async_tx/Kconfig | 4 + crypto/async_tx/Makefile | 1 + crypto/async_tx/async_pq.c | 615 +++++++++++++++++++++++++++++++++++++++++++ crypto/async_tx/async_xor.c | 2 +- include/linux/async_tx.h | 46 +++- include/linux/dmaengine.h | 30 ++- 6 files changed, 693 insertions(+), 5 deletions(-) create mode 100644 crypto/async_tx/async_pq.c
crypto/async_tx/Kconfig +4 -0
@@ -14,3 +14,7 @@ config ASYNC_MEMSET tristate select ASYNC_CORE +config ASYNC_PQ + tristate + select ASYNC_CORE +
crypto/async_tx/Makefile +1 -0
@@ -2,3 +2,4 @@ obj-$(CONFIG_ASYNC_CORE) += async_tx.o obj-$(CONFIG_ASYNC_MEMCPY) += async_memcpy.o obj-$(CONFIG_ASYNC_MEMSET) += async_memset.o obj-$(CONFIG_ASYNC_XOR) += async_xor.o +obj-$(CONFIG_ASYNC_PQ) += async_pq.o
crypto/async_tx/async_pq.c new file +615 -0
615 lines not on this page · raw message
crypto/async_tx/async_xor.c +1 -1
@@ -68,7 +68,7 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,while (src_cnt) { async_flags = flags; dma_flags =0 ; - xor_src_cnt =min (src_cnt, dma->max_xor); + xor_src_cnt =min (src_cnt, (int )dma->max_xor);/* if we are submitting additional xors, leave the chain open, * clear the callback parameters, and leave the destination * buffer mapped
include/linux/async_tx.h +44 -2
64 lines not on this page · raw message
include/linux/dmaengine.h +28 -2
@@ -55,7 +55,7 @@ enum dma_status {enum dma_transaction_type {DMA_MEMCPY ,DMA_XOR , -DMA_PQ_XOR , +DMA_PQ ,DMA_DUAL_XOR ,DMA_PQ_UPDATE ,DMA_ZERO_SUM ,
@@ -81,14 +81,28 @@ enum dma_transaction_type {* dependency chains * @DMA_COMPL_SKIP_SRC_UNMAP - set to disable dma-unmapping the source buffer(s) * @DMA_COMPL_SKIP_DEST_UNMAP - set to disable dma-unmapping the destination(s) +* @DMA_PREP_HAVE_P - set if the destination list includes the correct +* address of P (P-parity should be handled) +* @DMA_PREP_HAVE_Q - set if the destination list includes the correct +* address of Q (Q-parity should be handled) +* @DMA_PREP_ZERO_P - set if P has to be zeroed before proceeding +* @DMA_PREP_ZERO_Q - set if Q has to be zeroed before proceeding */ enum dma_ctrl_flags {DMA_PREP_INTERRUPT = (1 <<0 ),DMA_CTRL_ACK = (1 <<1 ),DMA_COMPL_SKIP_SRC_UNMAP = (1 <<2 ),DMA_COMPL_SKIP_DEST_UNMAP = (1 <<3 ), + +DMA_PREP_HAVE_P = (1 <<4 ), +DMA_PREP_HAVE_Q = (1 <<5 ), +DMA_PREP_ZERO_P = (1 <<6 ), +DMA_PREP_ZERO_Q = (1 <<7 ), }; +#define DMA_PCHECK_FAILED (1 << 0) +#define DMA_QCHECK_FAILED (1 << 1) +/** * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t. * See linux/cpumask.h
@@ -211,6 +225,7 @@ struct dma_async_tx_descriptor {* @global_node: list_head for global dma_device_list * @cap_mask: one or more dma_capability flags * @max_xor: maximum number of xor sources, 0 if no capability +* @max_pq: maximum number of PQ sources, 0 if no capability * @refcount: reference count * @done: IO completion struct * @dev_id: unique device ID
@@ -220,7 +235,9 @@ struct dma_async_tx_descriptor {* @device_free_chan_resources: release DMA channel's resources * @device_prep_dma_memcpy: prepares a memcpy operation * @device_prep_dma_xor: prepares a xor operation +* @device_prep_dma_pq: prepares a pq operation * @device_prep_dma_zero_sum: prepares a zero_sum operation +* @device_prep_dma_pqzero_sum: prepares a pqzero_sum operation * @device_prep_dma_memset: prepares a memset operation * @device_prep_dma_interrupt: prepares an end of chain interrupt operation * @device_prep_slave_sg: prepares a slave dma operation
@@ -233,7 +250,8 @@ struct dma_device {struct list_head channels;struct list_head global_node;dma_cap_mask_t cap_mask; -int max_xor; +unsigned short max_xor; +unsigned short max_pq;int dev_id;struct device *dev;
@@ -247,9 +265,17 @@ struct dma_device {struct dma_async_tx_descriptor *(*device_prep_dma_xor)(struct dma_chan *chan,dma_addr_t dest,dma_addr_t *src,unsigned int src_cnt,size_t len,unsigned long flags); +struct dma_async_tx_descriptor *(*device_prep_dma_pq)( +struct dma_chan *chan,dma_addr_t *dst,dma_addr_t *src, +unsigned int src_cnt,unsigned char *scf, +size_t len,unsigned long flags);struct dma_async_tx_descriptor *(*device_prep_dma_zero_sum)(struct dma_chan *chan,dma_addr_t *src,unsigned int src_cnt,size_t len,u32 *result,unsigned long flags); +struct dma_async_tx_descriptor *(*device_prep_dma_pqzero_sum)( +struct dma_chan *chan,dma_addr_t *src,unsigned int src_cnt, +unsigned char *scf,size_t len,u32 *pqres, +unsigned long flags);struct dma_async_tx_descriptor *(*device_prep_dma_memset)(struct dma_chan *chan,dma_addr_t dest,int value,size_t len,unsigned long flags);
Signature · 1 line
1.6.0.6
Quoted 194 lines · Yuri Tikhonov
This adds support for doing asynchronous GF multiplication by adding four additional functions to
async_txAPI:
async_pq()does simultaneous XOR of sources and XOR of sources GF-multiplied by given coefficients.
async_pq_zero_sum()checks if results of calculations match given ones.
async_gen_syndrome()does sumultaneous XOR and R/S syndrome of sources.
async_syndrome_zerosum()checks if results of XOR/syndrome calculation matches given ones.Latter two functions just use
async_pq()with the approprite coefficients in asynchronous case but have significant optimizations if synchronous case.To support this API dmaengine driver should set
DMA_PQandDMA_PQ_ZERO_SUMcapabilities and providedevice_prep_dma_pqanddevice_prep_dma_pqzero_summethods indma_devicestructure.Signed-off-by Yuri Tikhonov <[email protected]>Signed-off-by Ilya Yanok <[email protected]>--- crypto/async_tx/Kconfig | 4 + crypto/async_tx/Makefile | 1 + crypto/async_tx/async_pq.c | 615 +++++++++++++++++++++++++++++++++++++++++++ crypto/async_tx/async_xor.c | 2 +- include/linux/async_tx.h | 46 +++- include/linux/dmaengine.h | 30 ++- 6 files changed, 693 insertions(+), 5 deletions(-) create mode 100644 crypto/async_tx/async_pq.ccrypto/async_tx/Kconfig +4
-0@@ -14,3 +14,7 @@ config ASYNC_MEMSET tristate select ASYNC_CORE +config ASYNC_PQ + tristate + select ASYNC_CORE +crypto/async_tx/Makefile +1
-0@@ -2,3 +2,4 @@ obj-$(CONFIG_ASYNC_CORE) += async_tx.o obj-$(CONFIG_ASYNC_MEMCPY) += async_memcpy.o obj-$(CONFIG_ASYNC_MEMSET) += async_memset.o obj-$(CONFIG_ASYNC_XOR) += async_xor.o +obj-$(CONFIG_ASYNC_PQ) += async_pq.ocrypto/async_tx/async_pq.c new file +133
-0@@ -0,0 +1,615 @@ +/* +* Copyright(c) 2007 Yuri Tikhonov <[email protected]> +* +* Developed for DENX Software Engineering GmbH +* +* Asynchronous GF-XOR calculations ASYNC_TX API. +* +* based on async_xor.c code written by: +* Dan Williams <[email protected]> +* +* This program is free software; you can redistribute it and/or modify it +* under the terms of the GNU General Public License as published by the Free +* Software Foundation; either version 2 of the License, or (at your option) +* any later version. +* +* This program is distributed in the hope that it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +* more details. +* +* You should have received a copy of the GNU General Public License along with +* this program; if not, write to the Free Software Foundation, Inc., 59 +* Temple Place - Suite 330, Boston, MA 02111-1307, USA. +* +* The full GNU General Public License is included in this distribution in the +* file called COPYING. +*/ +#include <linux/kernel.h> +#include <linux/interrupt.h> +#include <linux/dma-mapping.h> +#include <linux/raid/xor.h> +#include <linux/async_tx.h> + +#include "../drivers/md/raid6.h" + +/** +* The following static variables are used in cases of synchronous +* zero sum to save the values to check. Two pages used for zero sum and +* the third one is for dumb P destination when calling gen_syndrome() +*/ +static spinlock_t spare_lock; +static struct page *spare_pages[3 ]; + +/** +* do_async_pq - asynchronously calculate P and/or Q +*/ +static struct dma_async_tx_descriptor * +do_async_pq (struct dma_chan *chan,struct page **blocks,unsigned char *scfs, +unsigned int offset,int src_cnt,size_t len,enum async_tx_flags flags, +struct dma_async_tx_descriptor *depend_tx, +dma_async_tx_callback cb_fn,void *cb_param) +{ +struct dma_device *dma = chan->device; +dma_addr_t dma_dest[2 ], dma_src[src_cnt]; +struct dma_async_tx_descriptor *tx =NULL ; +dma_async_tx_callback _cb_fn; +void *_cb_param; +unsigned char *scf =NULL ; +int i, src_off =0 ; +unsigned short pq_src_cnt; +enum async_tx_flags async_flags; +enum dma_ctrl_flags dma_flags =0 ; + +/* If we won't handle src_cnt in one shot, then the following +* flag(s) will be set only on the first pass of prep_dma +*/ +if (flags &ASYNC_TX_PQ_ZERO_P ) + dma_flags |=DMA_PREP_ZERO_P ; +if (flags &ASYNC_TX_PQ_ZERO_Q ) + dma_flags |=DMA_PREP_ZERO_Q ; + +/* DMAs use destinations as sources, so use BIDIRECTIONAL mapping */ +if (blocks[src_cnt]) { + dma_dest[0 ] =dma_map_page (dma->dev, blocks[src_cnt], + offset, len,DMA_BIDIRECTIONAL ); + dma_flags |=DMA_PREP_HAVE_P ; + } +if (blocks[src_cnt+1 ]) { + dma_dest[1 ] =dma_map_page (dma->dev, blocks[src_cnt+1 ], + offset, len,DMA_BIDIRECTIONAL ); + dma_flags |=DMA_PREP_HAVE_Q ; + } + +for (i =0 ; i < src_cnt; i++) + dma_src[i] =dma_map_page (dma->dev, blocks[i], + offset, len,DMA_TO_DEVICE ); + +while (src_cnt) { + async_flags = flags; + pq_src_cnt =min (src_cnt, (int )dma->max_pq); +/* if we are submitting additional pqs, leave the chain open, +* clear the callback parameters, and leave the destination +* buffers mapped +*/ +if (src_cnt > pq_src_cnt) { + async_flags &= ~ASYNC_TX_ACK ; + dma_flags |=DMA_COMPL_SKIP_DEST_UNMAP ; + _cb_fn =NULL ; + _cb_param =NULL ; + }else { + _cb_fn = cb_fn; + _cb_param = cb_param; + } +if (_cb_fn) + dma_flags |=DMA_PREP_INTERRUPT ; +if (scfs) + scf = &scfs[src_off]; + +/* Since we have clobbered the src_list we are committed +* to doing this asynchronously. Drivers force forward +* progress in case they can not provide a descriptor +*/ + tx = dma->device_prep_dma_pq (chan, dma_dest, + &dma_src[src_off], pq_src_cnt, + scf, len, dma_flags); +if (unlikely (!tx)) +async_tx_quiesce (&depend_tx); + +/* spin wait for the preceeding transactions to complete */ +while (unlikely (!tx)) { +dma_async_issue_pending (chan); + tx = dma->device_prep_dma_pq (chan, dma_dest, + &dma_src[src_off], pq_src_cnt, + scf, len, dma_flags); + } + +async_tx_submit (chan, tx, async_flags, depend_tx, + _cb_fn, _cb_param); + + depend_tx = tx; + flags |=ASYNC_TX_DEP_ACK ; + +if (src_cnt > pq_src_cnt) {
crypto/async_tx/async_pq.c line 143
Quoted 10 lines
crypto/async_tx/async_pq.c new file +10
-0+/* drop completed sources */ + src_cnt -= pq_src_cnt; + src_off += pq_src_cnt; + +/* use the intermediate result as a source; we +* clear DMA_PREP_ZERO, so prep_dma_pq will +* include destination(s) into calculations. Thus +* keep DMA_PREP_HAVE_x in dma_flags only +*/ + dma_flags &= (DMA_PREP_HAVE_P |DMA_PREP_HAVE_Q );
I don't think this will work as we will be mixing Q into the new P and P into the new Q. In order to support (src_cnt > device->max_pq) we need to explicitly tell the driver that the operation is being continued (DMA_PREP_CONTINUE) and to apply different coeffeicients to P and Q to cancel the effect of including them as sources. Here is an example of supporting a 5 source pq operation where max_pq == 4 (the minimum).
p, q = PQ(src0, src1, src2, src3, COEF({01}, {02}, {04}, {08}))
p', q' = PQ(p, q, q, src4, COEF({00}, {01}, {00}, {10}))
p' = p + q + q + src4 = p + src4 = P
q' = {00}*p + {01}*q + {00}*q + {10}*src4 = q + {10)*src4 = Q...at no point do we need to zero P or Q. Yes, this requires a lot of extra work for incremental sources, but at this point I do not see a cleaner alternatve for engines like iop13xx.
Quoted 36 lines
crypto/async_tx/async_pq.c new file +36
-0+ }else +break ; + } + +return tx; +} + +/** +* do_sync_pq - synchronously calculate P and Q +*/ +static void +do_sync_pq (struct page **blocks,unsigned char *scfs,unsigned int offset, +int src_cnt,size_t len,enum async_tx_flags flags, +struct dma_async_tx_descriptor *depend_tx, +dma_async_tx_callback cb_fn,void *cb_param) +{ +int i, pos; +uint8_t *p =NULL , *q =NULL , *src; + +/* set destination addresses */ +if (blocks[src_cnt]) + p = (uint8_t *)(page_address (blocks[src_cnt]) + offset); +if (blocks[src_cnt+1 ]) + q = (uint8_t *)(page_address (blocks[src_cnt+1 ]) + offset); + +if (flags &ASYNC_TX_PQ_ZERO_P ) { +BUG_ON (!p); +memset (p,0 , len); + } + +if (flags &ASYNC_TX_PQ_ZERO_Q ) { +BUG_ON (!q); +memset (q,0 , len); + } + +for (i =0 ; i < src_cnt; i++) {
crypto/async_tx/async_pq.c line 189
Quoted 10 lines
crypto/async_tx/async_pq.c new file +10
-0+ src = (uint8_t *)(page_address (blocks[i]) + offset); +for (pos =0 ; pos < len; pos++) { +if (p) + p[pos] ^= src[pos]; +if (q) + q[pos] ^= raid6_gfmul[scfs[i]][src[pos]]; + } + } +async_tx_sync_epilog (cb_fn, cb_param); +}
sync_pq like sync_gensyndrome should not care about the current contents of p and q, just regenerate from the current sources. This kills another site where ASYNC_TX_PQ_ZERO_{P,Q} is used.
Quoted 68 lines
crypto/async_tx/async_pq.c new file +68
-0+ +/** +* async_pq - attempt to do XOR and Galois calculations in parallel using +* a dma engine. +* @blocks: source block array from 0 to (src_cnt-1) with the p destination +* at blocks[src_cnt] and q at blocks[src_cnt + 1]. Only one of two +* destinations may be present (another then has to be set to NULL). +* By default, the result of calculations is XOR-ed with the initial +* content of the destinationa buffers. Use ASYNC_TX_PQ_ZERO_x flags +* to avoid this. +* NOTE: client code must assume the contents of this array are destroyed +* @scfs: array of source coefficients used in GF-multiplication +* @offset: offset in pages to start transaction +* @src_cnt: number of source pages +* @len: length in bytes +* @flags: ASYNC_TX_PQ_ZERO_P, ASYNC_TX_PQ_ZERO_Q, ASYNC_TX_ASSUME_COHERENT, +* ASYNC_TX_ACK, ASYNC_TX_DEP_ACK, ASYNC_TX_ASYNC_ONLY +* @depend_tx: depends on the result of this transaction. +* @cb_fn: function to call when the operation completes +* @cb_param: parameter to pass to the callback routine +*/ +struct dma_async_tx_descriptor * +async_pq (struct page **blocks,unsigned char *scfs,unsigned int offset, +int src_cnt,size_t len,enum async_tx_flags flags, +struct dma_async_tx_descriptor *depend_tx, +dma_async_tx_callback cb_fn,void *cb_param) +{ +struct dma_chan *chan =async_tx_find_channel (depend_tx,DMA_PQ , + &blocks[src_cnt],2 , + blocks, src_cnt, len); +struct dma_device *device = chan ? chan->device :NULL ; +struct dma_async_tx_descriptor *tx =NULL ; + +if (!device && (flags &ASYNC_TX_ASYNC_ONLY )) +return NULL ; + +if (device) { +/* run pq asynchronously */ + tx =do_async_pq (chan, blocks, scfs, offset, src_cnt, + len, flags, depend_tx, cb_fn,cb_param); + }else { +/* run pq synchronously */ +if (!blocks[src_cnt+1 ]) { +struct page *pdst = blocks[src_cnt]; +int i; + +/* Calculate P-parity only. +* As opposite to async_xor(), async_pq() assumes +* that destinations are included into calculations, +* so we should re-arrange the xor src list to +* achieve the similar behavior. +*/ +if (!(flags &ASYNC_TX_PQ_ZERO_P )) { +/* If async_pq() user doesn't set ZERO flag, +* it's assumed that destination has some +* reasonable data to include in calculations. +* The destination must be at position 0, so +* shift the sources and put pdst at the +* beginning of the list. +*/ +for (i = src_cnt -1 ; i >=0 ; i--) + blocks[i+1 ] = blocks[i]; + blocks[0 ] = pdst; + src_cnt++; + flags |=ASYNC_TX_XOR_DROP_DST ; + }else { +/* If async_pq() user want to clear P, then +* this will be done automatically in async
crypto/async_tx/async_pq.c line 267
Quoted 10 lines
crypto/async_tx/async_pq.c new file +10
-0+* case, and with the help of ZERO_DST in +* the sync one. +*/ + flags &= ~ASYNC_TX_PQ_ZERO_P ; + flags |=ASYNC_TX_XOR_ZERO_DST ; + } + +return async_xor (pdst, blocks, offset, + src_cnt, len, flags, depend_tx, + cb_fn, cb_param);
If we assume that async_pq always regenerates parity and never reuses the old value then we can get gid of the !(flags & ASYNC_TX_PQ_ZERO_P) path. In the case where code does need to reuse the old P, async_r6recov.c, it should call async_xor directly since that routine provides this semantic.
Quoted 34 lines
crypto/async_tx/async_pq.c new file +34
-0+ } + +/* wait for any prerequisite operations */ +async_tx_quiesce (&depend_tx); + +do_sync_pq (blocks, scfs, offset, src_cnt, len, flags, + depend_tx, cb_fn, cb_param); + } + +return tx; +} +EXPORT_SYMBOL_GPL (async_pq); + +/** +* do_sync_gen_syndrome - synchronously calculate P (xor) and Q (Reed-Solomon +* code) +*/ +static void +do_sync_gen_syndrome (struct page **blocks,unsigned int offset,int src_cnt, +size_t len,enum async_tx_flags flags, +struct dma_async_tx_descriptor *depend_tx, +dma_async_tx_callback cb_fn,void *cb_param) +{ +int i; +void *tsrc[src_cnt+2 ]; + +for (i =0 ; i < src_cnt +2 ; i++) + tsrc[i] =page_address (blocks[i]) + offset; + + raid6_call.gen_syndrome (i, len, tsrc); + +async_tx_sync_epilog (cb_fn, cb_param); +} +
[..]
Quoted 28 lines
include/linux/dmaengine.h +7
-1@@ -55,7 +55,7 @@ enum dma_status {enum dma_transaction_type {DMA_MEMCPY ,DMA_XOR , -DMA_PQ_XOR , +DMA_PQ ,DMA_DUAL_XOR ,DMA_PQ_UPDATE ,DMA_ZERO_SUM ,@@ -81,14 +81,28 @@ enum dma_transaction_type {* dependency chains * @DMA_COMPL_SKIP_SRC_UNMAP - set to disable dma-unmapping the source buffer(s) * @DMA_COMPL_SKIP_DEST_UNMAP - set to disable dma-unmapping the destination(s) +* @DMA_PREP_HAVE_P - set if the destination list includes the correct +* address of P (P-parity should be handled) +* @DMA_PREP_HAVE_Q - set if the destination list includes the correct +* address of Q (Q-parity should be handled) +* @DMA_PREP_ZERO_P - set if P has to be zeroed before proceeding +* @DMA_PREP_ZERO_Q - set if Q has to be zeroed before proceeding */ enum dma_ctrl_flags {DMA_PREP_INTERRUPT = (1 <<0 ),DMA_CTRL_ACK = (1 <<1 ),DMA_COMPL_SKIP_SRC_UNMAP = (1 <<2 ),
include/linux/dmaengine.h enum dma_transaction_type { · line 104
Quoted 10 lines
include/linux/dmaengine.h +7
-0DMA_COMPL_SKIP_DEST_UNMAP = (1 <<3 ), + +DMA_PREP_HAVE_P = (1 <<4 ), +DMA_PREP_HAVE_Q = (1 <<5 ), +DMA_PREP_ZERO_P = (1 <<6 ), + DMA_PREP_ZERO_Q = (1 << 7), }; +#define DMA_PCHECK_FAILED (1 << 0) +#define DMA_QCHECK_FAILED (1 << 1)
Perhaps turn these into an enum such that we can pass around a enum pq_check_flags pointer rather than a non-descript u32 *.
Quoted 51 lines
include/linux/dmaengine.h +14
-2+/** * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t. * See linux/cpumask.h @@ -211,6 +225,7 @@ struct dma_async_tx_descriptor {* @global_node: list_head for global dma_device_list * @cap_mask: one or more dma_capability flags * @max_xor: maximum number of xor sources, 0 if no capability +* @max_pq: maximum number of PQ sources, 0 if no capability * @refcount: reference count * @done: IO completion struct * @dev_id: unique device ID @@ -220,7 +235,9 @@ struct dma_async_tx_descriptor {* @device_free_chan_resources: release DMA channel's resources * @device_prep_dma_memcpy: prepares a memcpy operation * @device_prep_dma_xor: prepares a xor operation +* @device_prep_dma_pq: prepares a pq operation * @device_prep_dma_zero_sum: prepares a zero_sum operation +* @device_prep_dma_pqzero_sum: prepares a pqzero_sum operation * @device_prep_dma_memset: prepares a memset operation * @device_prep_dma_interrupt: prepares an end of chain interrupt operation * @device_prep_slave_sg: prepares a slave dma operation @@ -233,7 +250,8 @@ struct dma_device {struct list_head channels;struct list_head global_node;dma_cap_mask_t cap_mask; -int max_xor; +unsigned short max_xor; +unsigned short max_pq;int dev_id;struct device *dev;@@ -247,9 +265,17 @@ struct dma_device {struct dma_async_tx_descriptor *(*device_prep_dma_xor)(struct dma_chan *chan,dma_addr_t dest,dma_addr_t *src,unsigned int src_cnt,size_t len,unsigned long flags); +struct dma_async_tx_descriptor *(*device_prep_dma_pq)( +struct dma_chan *chan,dma_addr_t *dst,dma_addr_t *src, +unsigned int src_cnt,unsigned char *scf, +size_t len,unsigned long flags);struct dma_async_tx_descriptor *(*device_prep_dma_zero_sum)(struct dma_chan *chan,dma_addr_t *src,unsigned int src_cnt,size_t len,u32 *result,unsigned long flags); +struct dma_async_tx_descriptor *(*device_prep_dma_pqzero_sum)( +struct dma_chan *chan,dma_addr_t *src,unsigned int src_cnt, +unsigned char *scf,size_t len,u32 *pqres, +unsigned long flags);struct dma_async_tx_descriptor *(*device_prep_dma_memset)(struct dma_chan *chan,dma_addr_t dest,int value,size_t len,unsigned long flags); --
Quoted 2 lines
1.6.0.6
Regards,
Dan
Hello Dan,
Thanks for review. Some comments below.
On Thursday, January 15, 2009 you wrote:
[..]
Quoted 98 lines
Quoted 98 lines
crypto/async_tx/async_pq.c new file +98
-0+/** +* do_async_pq - asynchronously calculate P and/or Q +*/ +static struct dma_async_tx_descriptor * +do_async_pq (struct dma_chan *chan,struct page **blocks,unsigned char *scfs, +unsigned int offset,int src_cnt,size_t len,enum async_tx_flags flags, +struct dma_async_tx_descriptor *depend_tx, +dma_async_tx_callback cb_fn,void *cb_param) +{ +struct dma_device *dma = chan->device; +dma_addr_t dma_dest[2 ], dma_src[src_cnt]; +struct dma_async_tx_descriptor *tx =NULL ; +dma_async_tx_callback _cb_fn; +void *_cb_param; +unsigned char *scf =NULL ; +int i, src_off =0 ; +unsigned short pq_src_cnt; +enum async_tx_flags async_flags; +enum dma_ctrl_flags dma_flags =0 ; + +/* If we won't handle src_cnt in one shot, then the following +* flag(s) will be set only on the first pass of prep_dma +*/ +if (flags &ASYNC_TX_PQ_ZERO_P ) + dma_flags |=DMA_PREP_ZERO_P ; +if (flags &ASYNC_TX_PQ_ZERO_Q ) + dma_flags |=DMA_PREP_ZERO_Q ; + +/* DMAs use destinations as sources, so use BIDIRECTIONAL mapping */ +if (blocks[src_cnt]) { + dma_dest[0 ] =dma_map_page (dma->dev, blocks[src_cnt], + offset, len,DMA_BIDIRECTIONAL ); + dma_flags |=DMA_PREP_HAVE_P ; + } +if (blocks[src_cnt+1 ]) { + dma_dest[1 ] =dma_map_page (dma->dev, blocks[src_cnt+1 ], + offset, len,DMA_BIDIRECTIONAL ); + dma_flags |=DMA_PREP_HAVE_Q ; + } + +for (i =0 ; i < src_cnt; i++) + dma_src[i] =dma_map_page (dma->dev, blocks[i], + offset, len,DMA_TO_DEVICE ); + +while (src_cnt) { + async_flags = flags; + pq_src_cnt =min (src_cnt, (int )dma->max_pq); +/* if we are submitting additional pqs, leave the chain open, +* clear the callback parameters, and leave the destination +* buffers mapped +*/ +if (src_cnt > pq_src_cnt) { + async_flags &= ~ASYNC_TX_ACK ; + dma_flags |=DMA_COMPL_SKIP_DEST_UNMAP ; + _cb_fn =NULL ; + _cb_param =NULL ; + }else { + _cb_fn = cb_fn; + _cb_param = cb_param; + } +if (_cb_fn) + dma_flags |=DMA_PREP_INTERRUPT ; +if (scfs) + scf = &scfs[src_off]; + +/* Since we have clobbered the src_list we are committed +* to doing this asynchronously. Drivers force forward +* progress in case they can not provide a descriptor +*/ + tx = dma->device_prep_dma_pq (chan, dma_dest, + &dma_src[src_off], pq_src_cnt, + scf, len, dma_flags); +if (unlikely (!tx)) +async_tx_quiesce (&depend_tx); + +/* spin wait for the preceeding transactions to complete */ +while (unlikely (!tx)) { +dma_async_issue_pending (chan); + tx = dma->device_prep_dma_pq (chan, dma_dest, + &dma_src[src_off], pq_src_cnt, + scf, len, dma_flags); + } + +async_tx_submit (chan, tx, async_flags, depend_tx, + _cb_fn, _cb_param); + + depend_tx = tx; + flags |=ASYNC_TX_DEP_ACK ; + +if (src_cnt > pq_src_cnt) { +/* drop completed sources */ + src_cnt -= pq_src_cnt; + src_off += pq_src_cnt; + +/* use the intermediate result as a source; we +* clear DMA_PREP_ZERO, so prep_dma_pq will +* include destination(s) into calculations. Thus +* keep DMA_PREP_HAVE_x in dma_flags only
Quoted 8 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+ */ + dma_flags &= (DMA_PREP_HAVE_P | DMA_PREP_HAVE_Q);I don't think this will work as we will be mixing Q into the new P and P into the new Q. In order to support (
src_cnt>device->max_pq) we need to explicitly tell the driver that the operation is being continued (DMA_PREP_CONTINUE) and to apply different coeffeicients to P and Q to cancel the effect of including them as sources.
With DMA_PREP_ZERO_P/Q approach, the Q isn't mixed into new P, and P
isn't mixed into new Q. For your example of max_pq=4:
p, q = PQ(src0, src1, src2, src3, src4, COEF({01}, {02}, {04}, {08}, {10}))
with the current implementation will be split into:
p, q = PQ(src0, src1, src2, src3, COEF({01}, {02}, {04}, {08}) p,q = PQ(src4, COEF({10}))
which will result to the following:
p = ((dma_flags & DMA_PREP_ZERO_P) ? 0 : old_p) + src0 + src1 + src2 + src3
q = ((dma_flags & DMA_PREP_ZERO_Q) ? 0 : old_q) + {01}*src0 + {02}*src1 + {04}*src2 + {08}*src3
p` = p + src4
q` = q + {10}*src4
But, if we get rid of DMA_PREP_ZERO_P/Q, then the mess with P/Q will
have a place indeed.
Quoted 4 lines
Here is an
example of supporting a 5 source pq operation wheremax_pq== 4 (the minimum).
Quoted 8 lines
p, q = PQ(src0, src1, src2, src3, COEF({01}, {02}, {04}, {08})) p', q' = PQ(p, q, q, src4, COEF({00}, {01}, {00}, {10})) p' = p + q + q + src4 = p + src4 = P q' = {00}*p + {01}*q + {00}*q + {10}*src4 = q + {10)*src4 = Q...at no point do we need to zero P or Q. Yes, this requires a lot of extra work for incremental sources,
I would say, that 'very very lot'. In general this means that for
the cases of N sources > max_pq we'll have to do:
C = 1 + ceil((N-max_pq)/(max_pq - 3)) number of calls to ADMA.
E.g., for max_pq = 4:
N = 5 => C = 2,
N = 6 => C = 3,
..
N = 15 => C = 12,
N = 16 => C = 13,
..
N = 128 => C = 125.
If we stay with the current approach of using DMA_PREP_ZERO_P/Q, then
C = 1 + ceil((N-max_pq)/max_pq)) number of calls to ADMA.
And the same series will result to:
N = 5 => C = 2,
N = 6 => C = 2,
..
N = 15 => C = 4,
N = 16 => C = 4,
..
N = 128 => C = 32.
I'm afraid that the difference (13/4, 125/32) is very significant, so
getting rid of DMA_PREP_ZERO_P/Q will eat most of the improvement which could be achieved with the current approach.
Quoted 1 line
but at this point I do not see a cleaner alternatve for engines like iop13xx.
I can't find any description of iop13xx processors at Intel's
web-site, only 3xx:
http://www.intel.com/design/iio/index.htm?iid=ipp_embed+embed_io
So, it's hard for me to do any suggestions. I just wonder - doesn't
iop13xx allow users to program destination addresses into the sources fields of descriptors?
Quoted 42 lines
Quoted 42 lines
crypto/async_tx/async_pq.c new file +42
-0+ }else +break ; + } + +return tx; +} + +/** +* do_sync_pq - synchronously calculate P and Q +*/ +static void +do_sync_pq (struct page **blocks,unsigned char *scfs,unsigned int offset, +int src_cnt,size_t len,enum async_tx_flags flags, +struct dma_async_tx_descriptor *depend_tx, +dma_async_tx_callback cb_fn,void *cb_param) +{ +int i, pos; +uint8_t *p =NULL , *q =NULL , *src; + +/* set destination addresses */ +if (blocks[src_cnt]) + p = (uint8_t *)(page_address (blocks[src_cnt]) + offset); +if (blocks[src_cnt+1 ]) + q = (uint8_t *)(page_address (blocks[src_cnt+1 ]) + offset); + +if (flags &ASYNC_TX_PQ_ZERO_P ) { +BUG_ON (!p); +memset (p,0 , len); + } + +if (flags &ASYNC_TX_PQ_ZERO_Q ) { +BUG_ON (!q); +memset (q,0 , len); + } + +for (i =0 ; i < src_cnt; i++) { + src = (uint8_t *)(page_address (blocks[i]) + offset); +for (pos =0 ; pos < len; pos++) { +if (p) + p[pos] ^= src[pos]; +if (q) + q[pos] ^= raid6_gfmul[scfs[i]][src[pos]];
Quoted 8 lines
crypto/async_tx/async_pq.c line 189
Quoted 4 lines
crypto/async_tx/async_pq.c new file +4
-0+ } + } +async_tx_sync_epilog (cb_fn, cb_param); +}
sync_pqlikesync_gensyndromeshould not care about the current contents of p and q, just regenerate from the current sources. This kills another site where ASYNC_TX_PQ_ZERO_{P,Q} is used.
Well, perhaps you are right. The ASYNC_TX_PQ_ZERO_{P,Q} is set for
the most common cases of using async_pq, i.e. the parity generating. The wrap-around async_gen_syndrome() function always set these flags before calling async_pq().
The cases where ASYNC_TX_PQ_ZERO_{P,Q} isn't set are:
(a) async_pq can't process the sources in one short because of src_cnt > max_pq, so it should re-use the intermediate results (destination) as the sources;
(b) async_r6_dd_recov() does XOR with async_pq() assuming re-using the destination as the source.
So, I would say that ASYNC_TX_PQ_ZERO_{P,Q} should definitely go
away, if there were no significant overheads in (a) implemented without these flags (see above).
Quoted 76 lines
Quoted 76 lines
crypto/async_tx/async_pq.c new file +76
-0+ +/** +* async_pq - attempt to do XOR and Galois calculations in parallel using +* a dma engine. +* @blocks: source block array from 0 to (src_cnt-1) with the p destination +* at blocks[src_cnt] and q at blocks[src_cnt + 1]. Only one of two +* destinations may be present (another then has to be set to NULL). +* By default, the result of calculations is XOR-ed with the initial +* content of the destinationa buffers. Use ASYNC_TX_PQ_ZERO_x flags +* to avoid this. +* NOTE: client code must assume the contents of this array are destroyed +* @scfs: array of source coefficients used in GF-multiplication +* @offset: offset in pages to start transaction +* @src_cnt: number of source pages +* @len: length in bytes +* @flags: ASYNC_TX_PQ_ZERO_P, ASYNC_TX_PQ_ZERO_Q, ASYNC_TX_ASSUME_COHERENT, +* ASYNC_TX_ACK, ASYNC_TX_DEP_ACK, ASYNC_TX_ASYNC_ONLY +* @depend_tx: depends on the result of this transaction. +* @cb_fn: function to call when the operation completes +* @cb_param: parameter to pass to the callback routine +*/ +struct dma_async_tx_descriptor * +async_pq (struct page **blocks,unsigned char *scfs,unsigned int offset, +int src_cnt,size_t len,enum async_tx_flags flags, +struct dma_async_tx_descriptor *depend_tx, +dma_async_tx_callback cb_fn,void *cb_param) +{ +struct dma_chan *chan =async_tx_find_channel (depend_tx,DMA_PQ , + &blocks[src_cnt],2 , + blocks, src_cnt, len); +struct dma_device *device = chan ? chan->device :NULL ; +struct dma_async_tx_descriptor *tx =NULL ; + +if (!device && (flags &ASYNC_TX_ASYNC_ONLY )) +return NULL ; + +if (device) { +/* run pq asynchronously */ + tx =do_async_pq (chan, blocks, scfs, offset, src_cnt, + len, flags, depend_tx, cb_fn,cb_param); + }else { +/* run pq synchronously */ +if (!blocks[src_cnt+1 ]) { +struct page *pdst = blocks[src_cnt]; +int i; + +/* Calculate P-parity only. +* As opposite to async_xor(), async_pq() assumes +* that destinations are included into calculations, +* so we should re-arrange the xor src list to +* achieve the similar behavior. +*/ +if (!(flags &ASYNC_TX_PQ_ZERO_P )) { +/* If async_pq() user doesn't set ZERO flag, +* it's assumed that destination has some +* reasonable data to include in calculations. +* The destination must be at position 0, so +* shift the sources and put pdst at the +* beginning of the list. +*/ +for (i = src_cnt -1 ; i >=0 ; i--) + blocks[i+1 ] = blocks[i]; + blocks[0 ] = pdst; + src_cnt++; + flags |=ASYNC_TX_XOR_DROP_DST ; + }else { +/* If async_pq() user want to clear P, then +* this will be done automatically in async +* case, and with the help of ZERO_DST in +* the sync one. +*/ + flags &= ~ASYNC_TX_PQ_ZERO_P ; + flags |=ASYNC_TX_XOR_ZERO_DST ; + } + + return async_xor(pdst, blocks, offset,
Quoted 8 lines
crypto/async_tx/async_pq.c line 267
Quoted 2 lines
crypto/async_tx/async_pq.c new file +2
-0+ src_cnt, len, flags, depend_tx, + cb_fn, cb_param);If we assume that
async_pqalways regenerates parity and never reuses the old value then we can get gid of the !(flags &ASYNC_TX_PQ_ZERO_P) path. In the case where code does need to reuse the old P,async_r6recov.c, it should callasync_xordirectly since that routine provides this semantic.
Right. The question is - will we get rid of ZERO_P/Q or not.
[..]
Quoted 23 lines
Quoted 23 lines
include/linux/dmaengine.h +11
-0@@ -81,14 +81,28 @@ enum dma_transaction_type {* dependency chains * @DMA_COMPL_SKIP_SRC_UNMAP - set to disable dma-unmapping the source buffer(s) * @DMA_COMPL_SKIP_DEST_UNMAP - set to disable dma-unmapping the destination(s) +* @DMA_PREP_HAVE_P - set if the destination list includes the correct +* address of P (P-parity should be handled) +* @DMA_PREP_HAVE_Q - set if the destination list includes the correct +* address of Q (Q-parity should be handled) +* @DMA_PREP_ZERO_P - set if P has to be zeroed before proceeding +* @DMA_PREP_ZERO_Q - set if Q has to be zeroed before proceeding */ enum dma_ctrl_flags {DMA_PREP_INTERRUPT = (1 <<0 ),DMA_CTRL_ACK = (1 <<1 ),DMA_COMPL_SKIP_SRC_UNMAP = (1 <<2 ),DMA_COMPL_SKIP_DEST_UNMAP = (1 <<3 ), + +DMA_PREP_HAVE_P = (1 <<4 ), +DMA_PREP_HAVE_Q = (1 <<5 ), +DMA_PREP_ZERO_P = (1 <<6 ), +DMA_PREP_ZERO_Q = (1 <<7 ), };
Quoted 5 lines
include/linux/dmaengine.h enum dma_transaction_type { · line 104
Quoted 2 lines
include/linux/dmaengine.h +2
-0+#define DMA_PCHECK_FAILED (1 << 0) +#define DMA_QCHECK_FAILED (1 << 1)Perhaps turn these into an enum such that we can pass around a enum
pq_check_flagspointer rather than a non-descript u32 *.
Agree.
Regards, Yuri
--
Yuri Tikhonov, Senior Software Engineer
Emcraft Systems, www.emcraft.com
Quoted 16 lines · Yuri Tikhonov
Quoted 5 lines
I don't think this will work as we will be mixing Q into the new P and P into the new Q. In order to support (
src_cnt>device->max_pq) we need to explicitly tell the driver that the operation is being continued (DMA_PREP_CONTINUE) and to apply different coeffeicients to P and Q to cancel the effect of including them as sources.With DMA_PREP_ZERO_P/Q approach, the Q isn't mixed into new P, and P
isn't mixed into new Q. For your example of max_pq=4:p, q =
PQ(src0, src1, src2, src3, src4, COEF({01}, {02}, {04}, {08}, {10}))with the current implementation will be split into:
p, q = PQ(src0, src1, src2, src3,
COEF({01}, {02}, {04}, {08})p,q=PQ(src4, COEF({10}))
Quoted 8 lines
which will result to the following:
p = ((
dma_flags&DMA_PREP_ZERO_P) ? 0 :old_p) + src0 + src1 + src2 + src3
q = ((dma_flags&DMA_PREP_ZERO_Q) ? 0 :old_q) + {01}*src0 + {02}*src1 + {04}*src2 + {08}*src3p` = p + src4
q` = q + {10}*src4
Huh? Does the ppc440spe engine have some notion of flagging a source as old_p/old_q? Otherwise I do not see how the engine will not turn this into:
p` = p + src4 + q
q` = q + {10}*src4 + {x}*p
I think you missed the fact that we have passed p and q back in as sources. Unless we have multiple p destinations and multiple q destinations, or hardware support for continuations I do not see how you can guarantee this split.
Quoted 3 lines
I'm afraid that the difference (13/4, 125/32) is very significant, so
getting rid of DMA_PREP_ZERO_P/Q will eat most of the improvement which could be achieved with the current approach.
Data corruption is a slightly higher cost :-).
Quoted 11 lines
Quoted 2 lines
but at this point I do not see a cleaner alternatve for engines like iop13xx.
I can't find any description of iop13xx processors at Intel's
web-site, only 3xx:http://www.intel.com/design/iio/index.htm?iid=ipp_embed+embed_io
So, it's hard for me to do any suggestions. I just wonder - doesn't
iop13xx allow users to program destination addresses into the sources fields of descriptors?
Yes it does, but the engine does not know it is a destination.
Take a look at page 496 of the following and tell me if you come to a different conclusion.
http://download.intel.com/design/iio/docs/31503602.pdf
Thanks,
Dan
Hello Dan,
Quoted 30 lines · On Friday, January 16, 2009 you wrote:
Quoted 16 lines · Yuri Tikhonov
Quoted 5 lines
I don't think this will work as we will be mixing Q into the new P and P into the new Q. In order to support (
src_cnt>device->max_pq) we need to explicitly tell the driver that the operation is being continued (DMA_PREP_CONTINUE) and to apply different coeffeicients to P and Q to cancel the effect of including them as sources.With DMA_PREP_ZERO_P/Q approach, the Q isn't mixed into new P, and P
isn't mixed into new Q. For your example of max_pq=4:p, q =
PQ(src0, src1, src2, src3, src4, COEF({01}, {02}, {04}, {08}, {10}))with the current implementation will be split into:
p, q = PQ(src0, src1, src2, src3,
COEF({01}, {02}, {04}, {08})p,q=PQ(src4, COEF({10}))Quoted 8 lines
which will result to the following:
p = ((
dma_flags&DMA_PREP_ZERO_P) ? 0 :old_p) + src0 + src1 + src2 + src3
q = ((dma_flags&DMA_PREP_ZERO_Q) ? 0 :old_q) + {01}*src0 + {02}*src1 + {04}*src2 + {08}*src3p` = p + src4
q` = q + {10}*src4Huh? Does the ppc440spe engine have some notion of flagging a source as old_p/old_q? Otherwise I do not see how the engine will not turn this into:
Quoted 7 lines
p` = p + src4 + q
q` = q + {10}*src4 + {x}*pI think you missed the fact that we have passed p and q back in as sources. Unless we have multiple p destinations and multiple q destinations, or hardware support for continuations I do not see how you can guarantee this split.
I guess, I've got your point. You are missing the fact that
destinations for 'p' and 'q' are passed in device_prep_dma_pq() method separately from sources. Speaking your words: we do not have multiple destinations through the while() cycles, the destinations are the same in each pass.
Please look at do_async_pq() implementation more carefully: 'blocks'
is a pointer to 'src_cnt' sources _plus_ two destination pages (as it's stated in async_pq() description). Before coming into the while() cycle we save destinations in the dma_dest[] array, and then pass this to device_prep_dma_pq() in each (src_cnt/max_pq) cycle. That is, we do not passes destinations as the sources explicitly: we just clear DMA_PREP_ZERO_P/Q flags to notify ADMA level that this have to XOR the current content of destination(s) with the result of new operation.
Quoted 18 lines
Quoted 3 lines
I'm afraid that the difference (13/4, 125/32) is very significant, so
getting rid of DMA_PREP_ZERO_P/Q will eat most of the improvement which could be achieved with the current approach.Data corruption is a slightly higher cost :-).
Quoted 11 lines
Quoted 2 lines
but at this point I do not see a cleaner alternatve for engines like iop13xx.
I can't find any description of iop13xx processors at Intel's
web-site, only 3xx:http://www.intel.com/design/iio/index.htm?iid=ipp_embed+embed_io
So, it's hard for me to do any suggestions. I just wonder - doesn't
iop13xx allow users to program destination addresses into the sources fields of descriptors?
Quoted 5 lines
Yes it does, but the engine does not know it is a destination.
Take a look at page 496 of the following and tell me if you come to a different conclusion.
http://download.intel.com/design/iio/docs/31503602.pdf
I see. The major difference in the implementation of support for P+Q
in ppc440spe DMA engines is that ppc440spe allows to include (xor) the previous content of P_Result and/or Q_Result just by setting a corresponding indication in the destination (P_Result and/or Q_Result) address(es)
The "5.7.5 P+Q Update Operation" case won't help here, since, if
I understand it right, it doesn't allow to set up different multipliers for Old and New Data.
So, it looks like your approach:
p', q' = PQ(p, q, q, src4, COEF({00}, {01}, {00}, {10}))
is the only possible way of including the previous P/Q content into
the calculation.
But I still think, that this p'/q' hack should have a place on the
ADMA level, not ASYNC_TX. It looks more generic if ASYNC_TX will assume that ADMA is capable of p'=p+src / q'=q+{}*src. Otherwise, we'll have an overhead for the DMAs which could work without this overhead.
In your case, the IOP ADMA driver should handle the situation when it
receives 4 sources to be P+Qed with the previous contents of destinations, for example, by generating the sequence of 4 descriptors to process such a request.
Regards, Yuri
--
Yuri Tikhonov, Senior Software Engineer
Emcraft Systems, www.emcraft.com
Hello Dan,
Quoted 30 lines · On Friday, January 16, 2009 you wrote:
Quoted 16 lines · Yuri Tikhonov
Quoted 5 lines
I don't think this will work as we will be mixing Q into the new P and P into the new Q. In order to support (
src_cnt>device->max_pq) we need to explicitly tell the driver that the operation is being continued (DMA_PREP_CONTINUE) and to apply different coeffeicients to P and Q to cancel the effect of including them as sources.With DMA_PREP_ZERO_P/Q approach, the Q isn't mixed into new P, and P
isn't mixed into new Q. For your example of max_pq=4:p, q =
PQ(src0, src1, src2, src3, src4, COEF({01}, {02}, {04}, {08}, {10}))with the current implementation will be split into:
p, q = PQ(src0, src1, src2, src3,
COEF({01}, {02}, {04}, {08})p,q=PQ(src4, COEF({10}))Quoted 8 lines
which will result to the following:
p = ((
dma_flags&DMA_PREP_ZERO_P) ? 0 :old_p) + src0 + src1 + src2 + src3
q = ((dma_flags&DMA_PREP_ZERO_Q) ? 0 :old_q) + {01}*src0 + {02}*src1 + {04}*src2 + {08}*src3p` = p + src4
q` = q + {10}*src4Huh? Does the ppc440spe engine have some notion of flagging a source as old_p/old_q? Otherwise I do not see how the engine will not turn this into:
Quoted 7 lines
p` = p + src4 + q
q` = q + {10}*src4 + {x}*pI think you missed the fact that we have passed p and q back in as sources. Unless we have multiple p destinations and multiple q destinations, or hardware support for continuations I do not see how you can guarantee this split.
I guess, I've got your point. You are missing the fact that
destinations for 'p' and 'q' are passed in device_prep_dma_pq() method separately from sources. Speaking your words: we do not have multiple destinations through the while() cycles, the destinations are the same in each pass.
Please look at do_async_pq() implementation more carefully: 'blocks'
is a pointer to 'src_cnt' sources _plus_ two destination pages (as it's stated in async_pq() description). Before coming into the while() cycle we save destinations in the dma_dest[] array, and then pass this to device_prep_dma_pq() in each (src_cnt/max_pq) cycle. That is, we do not passes destinations as the sources explicitly: we just clear DMA_PREP_ZERO_P/Q flags to notify ADMA level that this have to XOR the current content of destination(s) with the result of new operation.
Quoted 18 lines
Quoted 3 lines
I'm afraid that the difference (13/4, 125/32) is very significant, so
getting rid of DMA_PREP_ZERO_P/Q will eat most of the improvement which could be achieved with the current approach.Data corruption is a slightly higher cost :-).
Quoted 11 lines
Quoted 2 lines
but at this point I do not see a cleaner alternatve for engines like iop13xx.
I can't find any description of iop13xx processors at Intel's
web-site, only 3xx:http://www.intel.com/design/iio/index.htm?iid=ipp_embed+embed_io
So, it's hard for me to do any suggestions. I just wonder - doesn't
iop13xx allow users to program destination addresses into the sources fields of descriptors?
Quoted 5 lines
Yes it does, but the engine does not know it is a destination.
Take a look at page 496 of the following and tell me if you come to a different conclusion.
http://download.intel.com/design/iio/docs/31503602.pdf
I see. The major difference in the implementation of support for P+Q
in ppc440spe DMA engines is that ppc440spe allows to include (xor) the previous content of P_Result and/or Q_Result just by setting a corresponding indication in the destination (P_Result and/or Q_Result) address(es)
The "5.7.5 P+Q Update Operation" case won't help here, since, if
I understand it right, it doesn't allow to set up different multipliers for Old and New Data.
So, it looks like your approach:
p', q' = PQ(p, q, q, src4, COEF({00}, {01}, {00}, {10}))
is the only possible way of including the previous P/Q content into
the calculation.
But I still think, that this p'/q' hack should have a place on the
ADMA level, not ASYNC_TX. It looks more generic if ASYNC_TX will assume that ADMA is capable of p'=p+src / q'=q+{}*src. Otherwise, we'll have an overhead for the DMAs which could work without this overhead.
In your case, the IOP ADMA driver should handle the situation when it
receives 4 sources to be P+Qed with the previous contents of destinations, for example, by generating the sequence of 4 descriptors to process such a request.
Regards, Yuri
--
Yuri Tikhonov, Senior Software Engineer
Emcraft Systems, www.emcraft.com
Hello Dan,
Quoted 11 lines · On Friday, January 16, 2009 you wrote:
Quoted 10 lines · Yuri Tikhonov
Quoted 5 lines
I don't think this will work as we will be mixing Q into the new P and P into the new Q. In order to support (
src_cnt>device->max_pq) we need to explicitly tell the driver that the operation is being continued (DMA_PREP_CONTINUE) and to apply different coeffeicients to P and Q to cancel the effect of including them as sources.With DMA_PREP_ZERO_P/Q approach, the Q isn't mixed into new P, and P
isn't mixed into new Q. For your example of max_pq=3D4:p, q =3D PQ(src0, src1, src2, src3, src4, COEF({01}, {02}, {04}, {08}, =
{10}))
Quoted 9 lines
Quoted 9 lines
with the current implementation will be split into:
p, q =3D PQ(src0, src1, src2, src3,
COEF({01}, {02}, {04}, {08})p,q=3DPQ(src4, COEF({10}))which will result to the following:
p =3D ((
dma_flags&DMA_PREP_ZERO_P) ? 0 :old_p) + src0 + src1 + src2 =
+ src3
Quoted 1 line
Quoted 1 line
q =3D ((
dma_flags&DMA_PREP_ZERO_Q) ? 0 :old_q) + {01}*src0 + {02}*sr=
c1 + {04}*src2 + {08}*src3
Quoted 9 lines
Quoted 4 lines
p` =3D p + src4
q` =3D q + {10}*src4Huh? Does the ppc440spe engine have some notion of flagging a source as old_p/old_q? Otherwise I do not see how the engine will not turn this into:
Quoted 7 lines
p` =3D p + src4 + q
q` =3D q + {10}*src4 + {x}*pI think you missed the fact that we have passed p and q back in as sources. Unless we have multiple p destinations and multiple q destinations, or hardware support for continuations I do not see how you can guarantee this split.
I guess, I've got your point. You are missing the fact that=20
destinations for 'p' and 'q' are passed in device_prep_dma_pq() method=20 separately from sources. Speaking your words: we do not have multiple=20 destinations through the while() cycles, the destinations are the same=20 in each pass.
Please look at do_async_pq() implementation more carefully: 'blocks'=20
is a pointer to 'src_cnt' sources _plus_ two destination pages (as=20 it's stated in async_pq() description). Before coming into the while()=20 cycle we save destinations in the dma_dest[] array, and then pass this=20 to device_prep_dma_pq() in each (src_cnt/max_pq) cycle. That is, we do=20 not passes destinations as the sources explicitly: we just clear=20 DMA_PREP_ZERO_P/Q flags to notify ADMA level that this have to XOR the=20 current content of destination(s) with the result of new operation.
Quoted 8 lines
Quoted 3 lines
I'm afraid that the difference (13/4, 125/32) is very significant, so
getting rid of DMA_PREP_ZERO_P/Q will eat most of the improvement which could be achieved with the current approach.Data corruption is a slightly higher cost :-).
Quoted 2 lines
Quoted 2 lines
but at this point I do not see a cleaner alternatve for engines like i=
op13xx.
Quoted 10 lines
Quoted 9 lines
I can't find any description of iop13xx processors at Intel's
web-site, only 3xx:http://www.intel.com/design/iio/index.htm?iid=3Dipp_embed+embed_io
So, it's hard for me to do any suggestions. I just wonder - doesn't
iop13xx allow users to program destination addresses into the sources fields of descriptors?
Quoted 5 lines
Yes it does, but the engine does not know it is a destination.
Take a look at page 496 of the following and tell me if you come to a different conclusion.
http://download.intel.com/design/iio/docs/31503602.pdf
I see. The major difference in the implementation of support for P+Q=20
in ppc440spe DMA engines is that ppc440spe allows to include (xor) the=20 previous content of P_Result and/or Q_Result just by setting a=20 corresponding indication in the destination (P_Result and/or Q_Result)=20 address(es)=20
The "5.7.5 P+Q Update Operation" case won't help here, since, if=20
I understand it right, it doesn't allow to set up different=20 multipliers for Old and New Data.
So, it looks like your approach:
p', q' =3D PQ(p, q, q, src4, COEF({00}, {01}, {00}, {10}))
is the only possible way of including the previous P/Q content into=20
the calculation.
But I still think, that this p'/q' hack should have a place on the=20
ADMA level, not ASYNC_TX. It looks more generic if ASYNC_TX will=20 assume that ADMA is capable of p'=3Dp+src / q'=3Dq+{}*src. Otherwise,=20 we'll have an overhead for the DMAs which could work without this=20 overhead.
In your case, the IOP ADMA driver should handle the situation when it=20
receives 4 sources to be P+Qed with the previous contents of=20 destinations, for example, by generating the sequence of 4 descriptors=20 to process such a request.
Regards, Yuri
--
Yuri Tikhonov, Senior Software Engineer
Emcraft Systems, www.emcraft.com