scatterlist: add I/O variant of sg_pcopy & sg_copy and use them
Subsystems (3)
cover scatterlist: add I/O variant of sg_pcopy & sg_copy and use them
A local variant of sg_copy_buffer has been introduced in the meson-gx mmc driver [1] after a mempcy optimization, fixing the iomem buffer manipulation and the reported system crash [2].
But, the fix is suboptimal in terms of performace/ugliness [3] and a proper I/O variant of sg_copy_buffer should be added and used instead.
[1] https://lore.kernel.org/r/[email protected]
[2] https://lore.kernel.org/r/[email protected]
[3] https://lore.kernel.org/r/CAPDyKFrLSMpPJOgd5e4B1x3Vwfg4q23zgy4ESc8EmFL2MnyK7g@mail.gmail.com
Neil Armstrong (2):
scatterlist: add I/O variant of sg_pcopy & sg_copy
mmc: meson-gx: use sg_copy_to/from_io instead of local version
drivers/mmc/host/meson-gx-mmc.c | 53 +++----------- include/linux/scatterlist.h | 14 ++++ lib/scatterlist.c | 119 ++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 43 deletions(-)
Signature · 1 line
2.25.1
RFC PATCH 1/2 scatterlist: add I/O variant of sg_pcopy & sg_copy
When copying from/to an iomem mapped memory, the current sg_copy & sg_pcopy can't be used and lead to local variants in drivers like in [1] & [2].
This introduces an I/O variant to be used instead of the local variants.
[1] mv_cesa_sg_copy in drivers/crypto/marvell/cesa/tdma.c
[2] meson_mmc_copy_buffer in drivers/mmc/host/meson-gx-mmc.c
--- include/linux/scatterlist.h | 14 +++++ lib/scatterlist.c | 119 ++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+)
include/linux/scatterlist.h +14 -0
@@ -308,15 +308,29 @@ void sgl_free(struct scatterlist *sgl);size_t sg_copy_buffer (struct scatterlist *sgl,unsigned int nents,void *buf,size_t buflen,off_t skip,bool to_buffer); +size_t sg_copy_io (struct scatterlist *sgl,unsigned int nents,void __iomem *buf, +size_t buflen,off_t skip,bool to_buffer); +size_t sg_copy_from_buffer (struct scatterlist *sgl,unsigned int nents,const void *buf,size_t buflen);size_t sg_copy_to_buffer (struct scatterlist *sgl,unsigned int nents,void *buf,size_t buflen); +size_t sg_copy_from_io (struct scatterlist *sgl,unsigned int nents, +const void __iomem *buf,size_t buflen); +size_t sg_copy_to_io (struct scatterlist *sgl,unsigned int nents, +void __iomem *buf,size_t buflen); +size_t sg_pcopy_from_buffer (struct scatterlist *sgl,unsigned int nents,const void *buf,size_t buflen,off_t skip);size_t sg_pcopy_to_buffer (struct scatterlist *sgl,unsigned int nents,void *buf,size_t buflen,off_t skip); + +size_t sg_pcopy_from_io (struct scatterlist *sgl,unsigned int nents, +const void __iomem *buf,size_t buflen,off_t skip); +size_t sg_pcopy_to_io (struct scatterlist *sgl,unsigned int nents, +void __iomem *buf,size_t buflen,off_t skip); +size_t sg_zero_buffer (struct scatterlist *sgl,unsigned int nents,size_t buflen,off_t skip);
lib/scatterlist.c +119 -0
137 lines not on this page · raw message
Signature · 1 line
2.25.1
Re: [PATCH RFC 1/2] scatterlist: add I/O variant of sg_pcopy & sg_copy
Quoted 7 lines · Neil Armstrong
When copying from/to an iomem mapped memory, the current
sg_copy&sg_pcopycan't be used and lead to local variants in drivers like in [1] & [2].This introduces an I/O variant to be used instead of the local variants.
[1]
mv_cesa_sg_copyindrivers/crypto/marvell/cesa/tdma.c
[2]meson_mmc_copy_bufferindrivers/mmc/host/meson-gx-mmc.c
Other than one apparent typo below, this looks like what I imagined, thanks for putting it together!
Quoted 107 lines
Cc Robin Murphy <[email protected]>Signed-off-by Neil Armstrong <[email protected]>--- include/linux/scatterlist.h | 14 +++++ lib/scatterlist.c | 119 ++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+)include/linux/scatterlist.h +14
-0@@ -308,15 +308,29 @@ void sgl_free(struct scatterlist *sgl);size_t sg_copy_buffer (struct scatterlist *sgl,unsigned int nents,void *buf,size_t buflen,off_t skip,bool to_buffer); +size_t sg_copy_io (struct scatterlist *sgl,unsigned int nents,void __iomem *buf, +size_t buflen,off_t skip,bool to_buffer); +size_t sg_copy_from_buffer (struct scatterlist *sgl,unsigned int nents,const void *buf,size_t buflen);size_t sg_copy_to_buffer (struct scatterlist *sgl,unsigned int nents,void *buf,size_t buflen); +size_t sg_copy_from_io (struct scatterlist *sgl,unsigned int nents, +const void __iomem *buf,size_t buflen); +size_t sg_copy_to_io (struct scatterlist *sgl,unsigned int nents, +void __iomem *buf,size_t buflen); +size_t sg_pcopy_from_buffer (struct scatterlist *sgl,unsigned int nents,const void *buf,size_t buflen,off_t skip);size_t sg_pcopy_to_buffer (struct scatterlist *sgl,unsigned int nents,void *buf,size_t buflen,off_t skip); + +size_t sg_pcopy_from_io (struct scatterlist *sgl,unsigned int nents, +const void __iomem *buf,size_t buflen,off_t skip); +size_t sg_pcopy_to_io (struct scatterlist *sgl,unsigned int nents, +void __iomem *buf,size_t buflen,off_t skip); +size_t sg_zero_buffer (struct scatterlist *sgl,unsigned int nents,size_t buflen,off_t skip);lib/scatterlist.c +51
-0@@ -954,6 +954,55 @@ size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf, }EXPORT_SYMBOL (sg_copy_buffer); +/** +* sg_copy_io - Copy data between an I/O mapped buffer and an SG list +* @sgl: The SG list +* @nents: Number of SG entries +* @buf: Where to copy from +* @buflen: The number of bytes to copy +* @skip: Number of bytes to skip before copying +* @to_buffer: transfer direction (true == from an sg list to a +* buffer, false == from a buffer to an sg list) +* +* Returns the number of copied bytes. +* +**/ +size_t sg_copy_io (struct scatterlist *sgl,unsigned int nents,void __iomem *buf, +size_t buflen,off_t skip,bool to_buffer) +{ +unsigned int offset =0 ; +struct sg_mapping_iter miter; +unsigned int sg_flags =SG_MITER_ATOMIC ; + +if (to_buffer) + sg_flags |=SG_MITER_FROM_SG ; +else + sg_flags |=SG_MITER_TO_SG ; + +sg_miter_start (&miter, sgl, nents, sg_flags); + +if (!sg_miter_skip (&miter, skip)) +return 0 ; + +while ((offset < buflen) &&sg_miter_next (&miter)) { +unsigned int len; + + len =min (miter.length, buflen - offset); + +if (to_buffer) +memcpy_toio (buf + offset, miter.addr, len); +else +memcpy_fromio (miter.addr, buf + offset, len); + + offset += len; + } + +sg_miter_stop (&miter); + +return offset; +} +EXPORT_SYMBOL (sg_copy_io); +/** * sg_copy_from_buffer - Copy from a linear buffer to an SG list * @sgl: The SG list @@ -988,6 +1037,40 @@ size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents, }EXPORT_SYMBOL (sg_copy_to_buffer); +/** +* sg_copy_from_io - Copy from an I/O mapped buffer to an SG list
lib/scatterlist.c size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents, · line 1051
Quoted 10 lines
lib/scatterlist.c +10
-0+* @sgl: The SG list +* @nents: Number of SG entries +* @buf: Where to copy from +* @buflen: The number of bytes to copy +* +* Returns the number of copied bytes. +* +**/ +size_t sg_copy_from_io (struct scatterlist *sgl,unsigned int nents, +const void *buf,size_t buflen)
The __iomem annotation wants to be on buf here raher than cast in below, to match the prototype (and everything else).
Cheers,
Robin.
Quoted 69 lines
lib/scatterlist.c +58
-0+{ +return sg_copy_io (sgl, nents, (void __iomem *)buf, buflen,0 , false); +} +EXPORT_SYMBOL (sg_copy_from_io); + +/** +* sg_copy_to_io - Copy from an SG list to an I/O mapped buffer +* @sgl: The SG list +* @nents: Number of SG entries +* @buf: Where to copy to +* @buflen: The number of bytes to copy +* +* Returns the number of copied bytes. +* +**/ +size_t sg_copy_to_io (struct scatterlist *sgl,unsigned int nents, +void __iomem *buf,size_t buflen) +{ +return sg_copy_io (sgl, nents, buf, buflen,0 , true); +} +EXPORT_SYMBOL (sg_copy_to_io); +/** * sg_pcopy_from_buffer - Copy from a linear buffer to an SG list * @sgl: The SG list @@ -1024,6 +1107,42 @@ size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents, }EXPORT_SYMBOL (sg_pcopy_to_buffer); +/** +* sg_pcopy_from_io - Copy from an I/O mapped buffer to an SG list +* @sgl: The SG list +* @nents: Number of SG entries +* @buf: Where to copy from +* @buflen: The number of bytes to copy +* @skip: Number of bytes to skip before copying +* +* Returns the number of copied bytes. +* +**/ +size_t sg_pcopy_from_io (struct scatterlist *sgl,unsigned int nents, +const void __iomem *buf,size_t buflen,off_t skip) +{ +return sg_copy_io (sgl, nents, (void __iomem *)buf, buflen, skip, false); +} +EXPORT_SYMBOL (sg_pcopy_from_io); + +/** +* sg_pcopy_to_io - Copy from an SG list to an I/O mapped buffer +* @sgl: The SG list +* @nents: Number of SG entries +* @buf: Where to copy to +* @buflen: The number of bytes to copy +* @skip: Number of bytes to skip before copying +* +* Returns the number of copied bytes. +* +**/ +size_t sg_pcopy_to_io (struct scatterlist *sgl,unsigned int nents, +void __iomem *buf,size_t buflen,off_t skip) +{ +return sg_copy_io (sgl, nents, buf, buflen, skip, true); +} +EXPORT_SYMBOL (sg_pcopy_to_io); +/** * sg_zero_buffer - Zero-out a part of a SG list * @sgl: The SG list
RFC PATCH 2/2 mmc: meson-gx: use sg_copy_to/from_io instead of local version
In reply to Neil ArmstrongUse the proper sg_copy_to_io & sg_copy_from_io instead of having a local sg_copy_buffer variant to handle the I/O mapped buffer case.
--- drivers/mmc/host/meson-gx-mmc.c | 53 +++++++-------------------------- 1 file changed, 10 insertions(+), 43 deletions(-)
drivers/mmc/host/meson-gx-mmc.c +10 -43
@@ -746,47 +746,6 @@ static void meson_mmc_desc_chain_transfer(struct mmc_host *mmc, u32 cmd_cfg)writel (start, host->regs +SD_EMMC_START ); } -/* local sg copy to buffer version with _to/fromio usage for dram_access_quirk */ -static void meson_mmc_copy_buffer (struct meson_host *host,struct mmc_data *data, -size_t buflen,bool to_buffer) -{ -unsigned int sg_flags =SG_MITER_ATOMIC ; -struct scatterlist *sgl = data->sg; -unsigned int nents = data->sg_len; -struct sg_mapping_iter miter; -unsigned int offset =0 ; - -if (to_buffer) - sg_flags |=SG_MITER_FROM_SG ; -else - sg_flags |=SG_MITER_TO_SG ; - -sg_miter_start (&miter, sgl, nents, sg_flags); - -while ((offset < buflen) &&sg_miter_next (&miter)) { -unsigned int len; - - len =min (miter.length, buflen - offset); - -/* When dram_access_quirk, the bounce buffer is a iomem mapping */ -if (host->dram_access_quirk) { -if (to_buffer) -memcpy_toio (host->bounce_iomem_buf + offset, miter.addr, len); -else -memcpy_fromio (miter.addr, host->bounce_iomem_buf + offset, len); - }else { -if (to_buffer) -memcpy (host->bounce_buf + offset, miter.addr, len); -else -memcpy (miter.addr, host->bounce_buf + offset, len); - } - - offset += len; - } - -sg_miter_stop (&miter); -} -static void meson_mmc_start_cmd (struct mmc_host *mmc,struct mmc_command *cmd) {struct meson_host *host =mmc_priv (mmc);
@@ -830,7 +789,12 @@ static void meson_mmc_start_cmd(struct mmc_host *mmc, struct mmc_command *cmd)if (data->flags &MMC_DATA_WRITE ) { cmd_cfg |=CMD_CFG_DATA_WR ;WARN_ON (xfer_bytes > host->bounce_buf_size); -meson_mmc_copy_buffer (host, data, xfer_bytes, true); +if (host->dram_access_quirk) +sg_copy_to_io (data->sg, data->sg_len, + host->bounce_iomem_buf, xfer_bytes); +else +sg_copy_to_buffer (data->sg, data->sg_len, + host->bounce_buf, xfer_bytes);dma_wmb (); }
@@ -999,7 +963,10 @@ static irqreturn_t meson_mmc_irq_thread(int irq, void *dev_id)if (meson_mmc_bounce_buf_read (data)) { xfer_bytes = data->blksz * data->blocks;WARN_ON (xfer_bytes > host->bounce_buf_size); -meson_mmc_copy_buffer (host, data, xfer_bytes, false); +if (host->dram_access_quirk) +sg_copy_from_io (data->sg, data->sg_len, host->bounce_iomem_buf, xfer_bytes); +else +sg_copy_from_buffer (data->sg, data->sg_len, host->bounce_buf, xfer_bytes); } next_cmd =meson_mmc_get_next_command (cmd);
Signature · 1 line
2.25.1
Re: [PATCH RFC 2/2] mmc: meson-gx: use sg_copy_to/from_io instead of local version
Quoted 62 lines · Neil Armstrong
Use the proper
sg_copy_to_io&sg_copy_from_ioinstead of having a localsg_copy_buffervariant to handle the I/O mapped buffer case.Cc Robin Murphy <[email protected]>Signed-off-by Neil Armstrong <[email protected]>--- drivers/mmc/host/meson-gx-mmc.c | 53 +++++++-------------------------- 1 file changed, 10 insertions(+), 43 deletions(-)drivers/mmc/host/meson-gx-mmc.c +0
-41@@ -746,47 +746,6 @@ static void meson_mmc_desc_chain_transfer(struct mmc_host *mmc, u32 cmd_cfg)writel (start, host->regs +SD_EMMC_START ); } -/* local sg copy to buffer version with _to/fromio usage for dram_access_quirk */ -static void meson_mmc_copy_buffer (struct meson_host *host,struct mmc_data *data, -size_t buflen,bool to_buffer) -{ -unsigned int sg_flags =SG_MITER_ATOMIC ; -struct scatterlist *sgl = data->sg; -unsigned int nents = data->sg_len; -struct sg_mapping_iter miter; -unsigned int offset =0 ; - -if (to_buffer) - sg_flags |=SG_MITER_FROM_SG ; -else - sg_flags |=SG_MITER_TO_SG ; - -sg_miter_start (&miter, sgl, nents, sg_flags); - -while ((offset < buflen) &&sg_miter_next (&miter)) { -unsigned int len; - - len =min (miter.length, buflen - offset); - -/* When dram_access_quirk, the bounce buffer is a iomem mapping */ -if (host->dram_access_quirk) { -if (to_buffer) -memcpy_toio (host->bounce_iomem_buf + offset, miter.addr, len); -else -memcpy_fromio (miter.addr, host->bounce_iomem_buf + offset, len); - }else { -if (to_buffer) -memcpy (host->bounce_buf + offset, miter.addr, len); -else -memcpy (miter.addr, host->bounce_buf + offset, len); - } - - offset += len; - } - -sg_miter_stop (&miter); -} -static void meson_mmc_start_cmd (struct mmc_host *mmc,struct mmc_command *cmd) {struct meson_host *host =mmc_priv (mmc);
drivers/mmc/host/meson-gx-mmc.c static void meson_mmc_start_cmd(struct mmc_host *mmc, struct mmc_command *cmd) · line 794
Quoted 7 lines
drivers/mmc/host/meson-gx-mmc.c +3
-1@@ -830,7 +789,12 @@ static void meson_mmc_start_cmd(struct mmc_host *mmc, struct mmc_command *cmd)if (data->flags &MMC_DATA_WRITE ) { cmd_cfg |=CMD_CFG_DATA_WR ;WARN_ON (xfer_bytes > host->bounce_buf_size); -meson_mmc_copy_buffer (host, data, xfer_bytes, true); +if (host->dram_access_quirk) +sg_copy_to_io (data->sg, data->sg_len, + host->bounce_iomem_buf, xfer_bytes);
Maybe you could just use host->regs + SD_EMMC_SRAM_DATA_BUF_OFF directly here (and below) and save carrying host->bounce_iomem_buf around?
Robin.
Quoted 19 lines
drivers/mmc/host/meson-gx-mmc.c +7
-1+else +sg_copy_to_buffer (data->sg, data->sg_len, + host->bounce_buf, xfer_bytes);dma_wmb (); }@@ -999,7 +963,10 @@ static irqreturn_t meson_mmc_irq_thread(int irq, void *dev_id)if (meson_mmc_bounce_buf_read (data)) { xfer_bytes = data->blksz * data->blocks;WARN_ON (xfer_bytes > host->bounce_buf_size); -meson_mmc_copy_buffer (host, data, xfer_bytes, false); +if (host->dram_access_quirk) +sg_copy_from_io (data->sg, data->sg_len, host->bounce_iomem_buf, xfer_bytes); +else +sg_copy_from_buffer (data->sg, data->sg_len, host->bounce_buf, xfer_bytes); } next_cmd =meson_mmc_get_next_command (cmd);