Skip to contentSkip to conversation

mm/zswap & crypto/acompress: remove a couple of memcpy

PATCH Superseded Barry Song 3 participants 9 messages Feb 16, 2024 – Feb 17, 2024

Subsystems (3)

v2 of 6 Superseded Latest is v6 →

lore
Share & tools
lore

Barry Song

author
#0 lore raw

v2 cover mm/zswap & crypto/acompress: remove a couple of memcpy

From: Barry Song <[email protected]>

The patchset removes a couple of memcpy in zswap and crypto to improve zswap's performance.

Thanks for Chengming Zhou's test and perf data.
Quote from Chengming,
I just tested these three patches on my server, found improvement in the kernel build testcase on a tmpfs with zswap (lz4 + zsmalloc) enabled.

         mm-stable 501a06fe8e4c  patched
 real    1m38.028s               1m32.317s
 user    19m11.482s              18m39.439s
 sys     19m26.445s              17m5.646s

As zswap is the direct use of this patchset and zswap benefits from this series, It is probably better for this patchset to go through Andrew's mm tree than Herbert's crypto tree if there is no objection from Herbert.

-v2:
* add flush_dcache_page() in scomp_acomp_comp_decomp() according to Herbert's suggestion, thanks!
* collect Reviewed-by of Nhat, thanks!
* rename is_async to is_sleepable according to Yosry's suggestion, thanks!

Barry Song (3):
crypto: introduce acomp_is_sleepable to expose if a acomp has a scomp
backend
mm/zswap: remove the memcpy if acomp is not sleepable
crypto: scompress: remove memcpy if sg_nents is 1

 crypto/acompress.c         |  8 ++++++++
 crypto/scompress.c         | 36 +++++++++++++++++++++++++++++-------
 include/crypto/acompress.h |  9 +++++++++
 mm/zswap.c                 |  6 ++++--
 4 files changed, 50 insertions(+), 9 deletions(-)
Signature · 1 line
2.34.1
author
#1 lore raw

PATCH v2 1/3 crypto: introduce acomp_is_sleepable to expose if a acomp has a scomp backend

From: Barry Song <[email protected]>

Almost all CPU-based compressors/decompressors are actually synchronous though they support acomp APIs. While some hardware has hardware-based accelerators to offload CPU's work such as hisilicon and intel/qat/, their drivers are working in async mode.
Letting acomp's users know exactly if the acomp is really async will help users know if the compression and decompression procedure can sleep.

---
 crypto/acompress.c         | 8 ++++++++
 include/crypto/acompress.h | 9 +++++++++
 2 files changed, 17 insertions(+)
crypto/acompress.c +8 -0
index 1c682810a484..fa15df394a4c 100644
@@ -152,6 +152,14 @@ struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type,
 }
 EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node);
 
+bool acomp_is_sleepable(struct crypto_acomp *acomp)
+{
+	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp);
+
+	return tfm->__crt_alg->cra_type == &crypto_acomp_type;
+}
+EXPORT_SYMBOL_GPL(acomp_is_sleepable);
+
 struct acomp_req *acomp_request_alloc(struct crypto_acomp *acomp)
 {
 	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp);
include/crypto/acompress.h +9 -0
index 574cffc90730..88ca33532313 100644
@@ -204,6 +204,15 @@ struct acomp_req *acomp_request_alloc(struct crypto_acomp *tfm);
  */
 void acomp_request_free(struct acomp_req *req);
 
+/**
+ * acomp_is_sleepable() -- check if an acomp is sleepable
+ *
+ * @tfm:	ACOMPRESS tfm handle allocated with crypto_alloc_acomp()
+ *
+ * Return:	true if the acomp is sleepable, otherwise, false
+ */
+bool acomp_is_sleepable(struct crypto_acomp *tfm);
+
 /**
  * acomp_request_set_callback() -- Sets an asynchronous callback
  *
Signature · 1 line
2.34.1
author
#2 lore raw

PATCH v2 2/3 mm/zswap: remove the memcpy if acomp is not sleepable

In reply to Barry Song

From: Barry Song <[email protected]>

Most compressors are actually CPU-based and won't sleep during compression and decompression. We should remove the redundant memcpy for them.

---
 mm/zswap.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
mm/zswap.c +4 -2
index 350dd2fc8159..6319d2281020 100644
@@ -168,6 +168,7 @@ struct crypto_acomp_ctx {
 	struct crypto_wait wait;
 	u8 *buffer;
 	struct mutex mutex;
+	bool is_sleepable;
 };
 
 /*
@@ -716,6 +717,7 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
 		goto acomp_fail;
 	}
 	acomp_ctx->acomp = acomp;
+	acomp_ctx->is_sleepable = acomp_is_sleepable(acomp);
 
 	req = acomp_request_alloc(acomp_ctx->acomp);
 	if (!req) {
@@ -1368,7 +1370,7 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
 	mutex_lock(&acomp_ctx->mutex);
 
 	src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
-	if (!zpool_can_sleep_mapped(zpool)) {
+	if (acomp_ctx->is_sleepable && !zpool_can_sleep_mapped(zpool)) {
 		memcpy(acomp_ctx->buffer, src, entry->length);
 		src = acomp_ctx->buffer;
 		zpool_unmap_handle(zpool, entry->handle);
@@ -1382,7 +1384,7 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
 	BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
 	mutex_unlock(&acomp_ctx->mutex);
 
-	if (zpool_can_sleep_mapped(zpool))
+	if (!acomp_ctx->is_sleepable || zpool_can_sleep_mapped(zpool))
 		zpool_unmap_handle(zpool, entry->handle);
 }
 
Signature · 1 line
2.34.1
Acked-by on v2 2/3
#3 lore raw

Re: [PATCH v2 2/3] mm/zswap: remove the memcpy if acomp is not sleepable

Quoted 26 lines · Barry Song

From: Barry Song <[email protected]>

Most compressors are actually CPU-based and won't sleep during compression and decompression. We should remove the redundant memcpy for them.

---
 mm/zswap.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
mm/zswap.c +1 -0
index 350dd2fc8159..6319d2281020 100644
@@ -168,6 +168,7 @@ struct crypto_acomp_ctx {
 	struct crypto_wait wait;
 	u8 *buffer;
 	struct mutex mutex;
+	bool is_sleepable;
 };
 
 /*

mm/zswap.c static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) · line 720

Quoted 4 lines
mm/zswap.c +1 -0
@@ -716,6 +717,7 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
 		goto acomp_fail;
 	}
 	acomp_ctx->acomp = acomp;
+	acomp_ctx->is_sleepable = acomp_is_sleepable(acomp);

Just one question here. In patch 1, sleepable seems to mean "not async". IIUC, even a synchronous algorithm may sleep (e.g. if there is a cond_resched or waiting for a mutex). Does sleepable in acomp terms the same as "atomic" in scheduling/preemption terms?

Also, was this tested with debug options to catch any possible sleeps in atomic context?

If the answer to both questions is yes, the change otherwise LGTM. Feel free to add:

Thanks!

Quoted 24 lines
mm/zswap.c +2 -2
 	req = acomp_request_alloc(acomp_ctx->acomp);
 	if (!req) {
@@ -1368,7 +1370,7 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
 	mutex_lock(&acomp_ctx->mutex);
 
 	src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
-	if (!zpool_can_sleep_mapped(zpool)) {
+	if (acomp_ctx->is_sleepable && !zpool_can_sleep_mapped(zpool)) {
 		memcpy(acomp_ctx->buffer, src, entry->length);
 		src = acomp_ctx->buffer;
 		zpool_unmap_handle(zpool, entry->handle);
@@ -1382,7 +1384,7 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
 	BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
 	mutex_unlock(&acomp_ctx->mutex);
 
-	if (zpool_can_sleep_mapped(zpool))
+	if (!acomp_ctx->is_sleepable || zpool_can_sleep_mapped(zpool))
 		zpool_unmap_handle(zpool, entry->handle);
 }
 
Signature · 1 line
2.34.1
author
#4 lore raw

Re: [PATCH v2 2/3] mm/zswap: remove the memcpy if acomp is not sleepable

Quoted 26 lines · Yosry Ahmed
Quoted 24 lines · Barry Song

From: Barry Song <[email protected]>

Most compressors are actually CPU-based and won't sleep during compression and decompression. We should remove the redundant memcpy for them.

---
 mm/zswap.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
mm/zswap.c +1 -0
index 350dd2fc8159..6319d2281020 100644
@@ -168,6 +168,7 @@ struct crypto_acomp_ctx {
      struct crypto_wait wait;
      u8 *buffer;
      struct mutex mutex;
+     bool is_sleepable;
 };
 
Quoted 11 lines

mm/zswap.c static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) · line 720

Quoted 6 lines
mm/zswap.c +1 -0
 /*
@@ -716,6 +717,7 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
              goto acomp_fail;
      }
      acomp_ctx->acomp = acomp;
+     acomp_ctx->is_sleepable = acomp_is_sleepable(acomp);

Just one question here. In patch 1, sleepable seems to mean "not async". IIUC, even a synchronous algorithm may sleep (e.g. if there is a cond_resched or waiting for a mutex). Does sleepable in acomp terms the same as "atomic" in scheduling/preemption terms?

I think the answer is yes though async and sleepable are slightly different semantically
generally speaking. but for comp cases, they are equal.

We have two backends for compression/ decompression - scomp and acomp. if comp is using scomp backend, we can safely think they are not sleepable at least from the
below three facts.

1. in zRAM, we are using scomp APIs only - crypto_comp_decompress()/ crypto_comp_compress(), which are definitely scomp, we have never considered sleeping problem in zram drivers:
static int zram_read_from_zspool(struct zram *zram, struct page *page,

                                 u32 index)
{
        struct zcomp_strm *zstrm;
        unsigned long handle;
        unsigned int size;
        void *src, *dst;
        u32 prio;
        int ret;

        handle = zram_get_handle(zram, index);
        ...
        src = zs_map_object(zram->mem_pool, handle, ZS_MM_RO);
        if (size == PAGE_SIZE) {
                dst = kmap_local_page(page);
                memcpy(dst, src, PAGE_SIZE);
                kunmap_local(dst);
                ret = 0;
        } else {
                dst = kmap_local_page(page);
                ret = zcomp_decompress(zstrm, src, size, dst);
                kunmap_local(dst);
                zcomp_stream_put(zram->comps[prio]);
        }
        zs_unmap_object(zram->mem_pool, handle);
        return ret;
}

2. zswap used to only support scomp before we moved to use crypto_acomp_compress()
and crypto_acomp_decompress() APIs whose backends can be either scomp or acomp, thus new hardware-based compression drivers can be used in zswap.

But before we moved to these new APIs in commit 1ec3b5fe6eec782 ("mm/zswap:
move to use crypto_acomp API for hardware acceleration") , zswap had never considered
sleeping problems just like zRAM.

3. There is no sleeping in drivers using scomp backend.

$ git grep crypto_register_scomp
crypto/842.c:   ret = crypto_register_scomp(&scomp);
crypto/deflate.c:       ret = crypto_register_scomp(&scomp);
crypto/lz4.c:   ret = crypto_register_scomp(&scomp);
crypto/lz4hc.c: ret = crypto_register_scomp(&scomp);
crypto/lzo-rle.c:       ret = crypto_register_scomp(&scomp);
crypto/lzo.c:   ret = crypto_register_scomp(&scomp);
crypto/zstd.c:  ret = crypto_register_scomp(&scomp);
drivers/crypto/cavium/zip/zip_main.c:   ret =
crypto_register_scomp(&zip_scomp_deflate);
drivers/crypto/cavium/zip/zip_main.c:   ret =
crypto_register_scomp(&zip_scomp_lzs);

which are the most common cases.

Quoted 3 lines

Also, was this tested with debug options to catch any possible sleeps in atomic context?

yes. i have enabled CONFIG_DEBUG_ATOMIC_SLEEP=y.

Quoted 4 lines

If the answer to both questions is yes, the change otherwise LGTM. Feel free to add:

Thanks!

Quoted 19 lines

Thanks!

Quoted 16 lines
mm/zswap.c +1 -1
      req = acomp_request_alloc(acomp_ctx->acomp);
      if (!req) {
@@ -1368,7 +1370,7 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
      mutex_lock(&acomp_ctx->mutex);
 
      src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
-     if (!zpool_can_sleep_mapped(zpool)) {
+     if (acomp_ctx->is_sleepable && !zpool_can_sleep_mapped(zpool)) {
              memcpy(acomp_ctx->buffer, src, entry->length);
              src = acomp_ctx->buffer;
              zpool_unmap_handle(zpool, entry->handle);
@@ -1382,7 +1384,7 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
      BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
      mutex_unlock(&acomp_ctx->mutex);
 
Quoted 8 lines
Quoted 8 lines
mm/zswap.c +1 -2
-     if (zpool_can_sleep_mapped(zpool))
+     if (!acomp_ctx->is_sleepable || zpool_can_sleep_mapped(zpool))
              zpool_unmap_handle(zpool, entry->handle);
 }
 
--

2.34.1

Thanks
Barry

#5 lore raw

Re: [PATCH v2 2/3] mm/zswap: remove the memcpy if acomp is not sleepable

Quoted 104 lines · Barry Song
Quoted 26 lines · Yosry Ahmed
Quoted 24 lines · Barry Song

From: Barry Song <[email protected]>

Most compressors are actually CPU-based and won't sleep during compression and decompression. We should remove the redundant memcpy for them.

---
 mm/zswap.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
mm/zswap.c +1 -0
index 350dd2fc8159..6319d2281020 100644
@@ -168,6 +168,7 @@ struct crypto_acomp_ctx {
      struct crypto_wait wait;
      u8 *buffer;
      struct mutex mutex;
+     bool is_sleepable;
 };
 
Quoted 11 lines

mm/zswap.c static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) · line 720

Quoted 6 lines
mm/zswap.c +1 -0
 /*
@@ -716,6 +717,7 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
              goto acomp_fail;
      }
      acomp_ctx->acomp = acomp;
+     acomp_ctx->is_sleepable = acomp_is_sleepable(acomp);

Just one question here. In patch 1, sleepable seems to mean "not async". IIUC, even a synchronous algorithm may sleep (e.g. if there is a cond_resched or waiting for a mutex). Does sleepable in acomp terms the same as "atomic" in scheduling/preemption terms?

I think the answer is yes though async and sleepable are slightly different semantically
generally speaking. but for comp cases, they are equal.

We have two backends for compression/ decompression - scomp and acomp. if comp is using scomp backend, we can safely think they are not sleepable at least from the
below three facts.

1. in zRAM, we are using scomp APIs only - crypto_comp_decompress()/ crypto_comp_compress(), which are definitely scomp, we have never considered sleeping problem in zram drivers:
static int zram_read_from_zspool(struct zram *zram, struct page *page,

                                 u32 index)
{
        struct zcomp_strm *zstrm;
        unsigned long handle;
        unsigned int size;
        void *src, *dst;
        u32 prio;
        int ret;

        handle = zram_get_handle(zram, index);
        ...
        src = zs_map_object(zram->mem_pool, handle, ZS_MM_RO);
        if (size == PAGE_SIZE) {
                dst = kmap_local_page(page);
                memcpy(dst, src, PAGE_SIZE);
                kunmap_local(dst);
                ret = 0;
        } else {
                dst = kmap_local_page(page);
                ret = zcomp_decompress(zstrm, src, size, dst);
                kunmap_local(dst);
                zcomp_stream_put(zram->comps[prio]);
        }
        zs_unmap_object(zram->mem_pool, handle);
        return ret;
}

2. zswap used to only support scomp before we moved to use crypto_acomp_compress()
and crypto_acomp_decompress() APIs whose backends can be either scomp or acomp, thus new hardware-based compression drivers can be used in zswap.

But before we moved to these new APIs in commit 1ec3b5fe6eec782 ("mm/zswap:
move to use crypto_acomp API for hardware acceleration") , zswap had never considered
sleeping problems just like zRAM.

3. There is no sleeping in drivers using scomp backend.

$ git grep crypto_register_scomp
crypto/842.c:   ret = crypto_register_scomp(&scomp);
crypto/deflate.c:       ret = crypto_register_scomp(&scomp);
crypto/lz4.c:   ret = crypto_register_scomp(&scomp);
crypto/lz4hc.c: ret = crypto_register_scomp(&scomp);
crypto/lzo-rle.c:       ret = crypto_register_scomp(&scomp);
crypto/lzo.c:   ret = crypto_register_scomp(&scomp);
crypto/zstd.c:  ret = crypto_register_scomp(&scomp);
drivers/crypto/cavium/zip/zip_main.c:   ret =
crypto_register_scomp(&zip_scomp_deflate);
drivers/crypto/cavium/zip/zip_main.c:   ret =
crypto_register_scomp(&zip_scomp_lzs);
Quoted 1 line

which are the most common cases.

Thanks for explaining. Ideally we should be able to catch any violations with proper debug options as you mentioned. Please include more info the commit message about sleepability, a summarized version of what you described above.

author
#6 lore raw

Re: [PATCH v2 2/3] mm/zswap: remove the memcpy if acomp is not sleepable

Quoted 106 lines · Yosry Ahmed
Quoted 104 lines · Barry Song
Quoted 26 lines · Yosry Ahmed
Quoted 24 lines · Barry Song

From: Barry Song <[email protected]>

Most compressors are actually CPU-based and won't sleep during compression and decompression. We should remove the redundant memcpy for them.

---
 mm/zswap.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
mm/zswap.c +1 -0
index 350dd2fc8159..6319d2281020 100644
@@ -168,6 +168,7 @@ struct crypto_acomp_ctx {
      struct crypto_wait wait;
      u8 *buffer;
      struct mutex mutex;
+     bool is_sleepable;
 };
 
Quoted 11 lines

mm/zswap.c static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) · line 720

Quoted 6 lines
mm/zswap.c +1 -0
 /*
@@ -716,6 +717,7 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
              goto acomp_fail;
      }
      acomp_ctx->acomp = acomp;
+     acomp_ctx->is_sleepable = acomp_is_sleepable(acomp);

Just one question here. In patch 1, sleepable seems to mean "not async". IIUC, even a synchronous algorithm may sleep (e.g. if there is a cond_resched or waiting for a mutex). Does sleepable in acomp terms the same as "atomic" in scheduling/preemption terms?

I think the answer is yes though async and sleepable are slightly different semantically
generally speaking. but for comp cases, they are equal.

We have two backends for compression/ decompression - scomp and acomp. if comp is using scomp backend, we can safely think they are not sleepable at least from the
below three facts.

1. in zRAM, we are using scomp APIs only - crypto_comp_decompress()/ crypto_comp_compress(), which are definitely scomp, we have never considered sleeping problem in zram drivers:
static int zram_read_from_zspool(struct zram *zram, struct page *page,

                                 u32 index)
{
        struct zcomp_strm *zstrm;
        unsigned long handle;
        unsigned int size;
        void *src, *dst;
        u32 prio;
        int ret;

        handle = zram_get_handle(zram, index);
        ...
        src = zs_map_object(zram->mem_pool, handle, ZS_MM_RO);
        if (size == PAGE_SIZE) {
                dst = kmap_local_page(page);
                memcpy(dst, src, PAGE_SIZE);
                kunmap_local(dst);
                ret = 0;
        } else {
                dst = kmap_local_page(page);
                ret = zcomp_decompress(zstrm, src, size, dst);
                kunmap_local(dst);
                zcomp_stream_put(zram->comps[prio]);
        }
        zs_unmap_object(zram->mem_pool, handle);
        return ret;
}

2. zswap used to only support scomp before we moved to use crypto_acomp_compress()
and crypto_acomp_decompress() APIs whose backends can be either scomp or acomp, thus new hardware-based compression drivers can be used in zswap.

But before we moved to these new APIs in commit 1ec3b5fe6eec782 ("mm/zswap:
move to use crypto_acomp API for hardware acceleration") , zswap had never considered
sleeping problems just like zRAM.

3. There is no sleeping in drivers using scomp backend.

$ git grep crypto_register_scomp
crypto/842.c:   ret = crypto_register_scomp(&scomp);
crypto/deflate.c:       ret = crypto_register_scomp(&scomp);
crypto/lz4.c:   ret = crypto_register_scomp(&scomp);
crypto/lz4hc.c: ret = crypto_register_scomp(&scomp);
crypto/lzo-rle.c:       ret = crypto_register_scomp(&scomp);
crypto/lzo.c:   ret = crypto_register_scomp(&scomp);
crypto/zstd.c:  ret = crypto_register_scomp(&scomp);
drivers/crypto/cavium/zip/zip_main.c:   ret =
crypto_register_scomp(&zip_scomp_deflate);
drivers/crypto/cavium/zip/zip_main.c:   ret =
crypto_register_scomp(&zip_scomp_lzs);
Quoted 6 lines
Quoted 1 line

which are the most common cases.

Thanks for explaining. Ideally we should be able to catch any violations with proper debug options as you mentioned. Please include more info the commit message about sleepability, a summarized version of what you described above.

ok. I will enhance the commit message of patch 1/3 with the summary.

Thanks
Barry

Reviewed-by on v2 2/3
#7 lore raw

Re: [PATCH v2 2/3] mm/zswap: remove the memcpy if acomp is not sleepable

In reply to Barry Song
Quoted 9 lines · Barry Song

From: Barry Song <[email protected]>

Most compressors are actually CPU-based and won't sleep during compression and decompression. We should remove the redundant memcpy for them.

LGTM, thanks!

Quoted 42 lines
---
 mm/zswap.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
mm/zswap.c +4 -2
index 350dd2fc8159..6319d2281020 100644
@@ -168,6 +168,7 @@ struct crypto_acomp_ctx {
 	struct crypto_wait wait;
 	u8 *buffer;
 	struct mutex mutex;
+	bool is_sleepable;
 };
 
 /*
@@ -716,6 +717,7 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
 		goto acomp_fail;
 	}
 	acomp_ctx->acomp = acomp;
+	acomp_ctx->is_sleepable = acomp_is_sleepable(acomp);
 
 	req = acomp_request_alloc(acomp_ctx->acomp);
 	if (!req) {
@@ -1368,7 +1370,7 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
 	mutex_lock(&acomp_ctx->mutex);
 
 	src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
-	if (!zpool_can_sleep_mapped(zpool)) {
+	if (acomp_ctx->is_sleepable && !zpool_can_sleep_mapped(zpool)) {
 		memcpy(acomp_ctx->buffer, src, entry->length);
 		src = acomp_ctx->buffer;
 		zpool_unmap_handle(zpool, entry->handle);
@@ -1382,7 +1384,7 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
 	BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
 	mutex_unlock(&acomp_ctx->mutex);
 
-	if (zpool_can_sleep_mapped(zpool))
+	if (!acomp_ctx->is_sleepable || zpool_can_sleep_mapped(zpool))
 		zpool_unmap_handle(zpool, entry->handle);
 }
 
author
#8 lore raw

PATCH v2 3/3 crypto: scompress: remove memcpy if sg_nents is 1

In reply to Barry Song

From: Barry Song <[email protected]>

while sg_nents is 1 which is always true for the current kernel as the only user - zswap is the case, we should remove two big memcpy.

---
 crypto/scompress.c | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)
crypto/scompress.c +29 -7
index b108a30a7600..50a487eac792 100644
@@ -117,6 +117,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
 	struct crypto_scomp *scomp = *tfm_ctx;
 	void **ctx = acomp_request_ctx(req);
 	struct scomp_scratch *scratch;
+	void *src, *dst;
 	unsigned int dlen;
 	int ret;
 
@@ -134,13 +135,25 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
 	scratch = raw_cpu_ptr(&scomp_scratch);
 	spin_lock(&scratch->lock);
 
-	scatterwalk_map_and_copy(scratch->src, req->src, 0, req->slen, 0);
+	if (sg_nents(req->src) == 1) {
+		src = kmap_local_page(sg_page(req->src)) + req->src->offset;
+	} else {
+		scatterwalk_map_and_copy(scratch->src, req->src, 0,
+					 req->slen, 0);
+		src = scratch->src;
+	}
+
+	if (req->dst && sg_nents(req->dst) == 1)
+		dst = kmap_local_page(sg_page(req->dst)) + req->dst->offset;
+	else
+		dst = scratch->dst;
+
 	if (dir)
-		ret = crypto_scomp_compress(scomp, scratch->src, req->slen,
-					    scratch->dst, &req->dlen, *ctx);
+		ret = crypto_scomp_compress(scomp, src, req->slen,
+					    dst, &req->dlen, *ctx);
 	else
-		ret = crypto_scomp_decompress(scomp, scratch->src, req->slen,
-					      scratch->dst, &req->dlen, *ctx);
+		ret = crypto_scomp_decompress(scomp, src, req->slen,
+					      dst, &req->dlen, *ctx);
 	if (!ret) {
 		if (!req->dst) {
 			req->dst = sgl_alloc(req->dlen, GFP_ATOMIC, NULL);
@@ -152,10 +165,19 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
 			ret = -ENOSPC;
 			goto out;
 		}
-		scatterwalk_map_and_copy(scratch->dst, req->dst, 0, req->dlen,
-					 1);
+		if (dst == scratch->dst) {
+			scatterwalk_map_and_copy(scratch->dst, req->dst, 0,
+						 req->dlen, 1);
+		} else {
+			flush_dcache_page(sg_page(req->dst));
+		}
 	}
 out:
+	if (src != scratch->src)
+		kunmap_local(src);
+	if (dst != scratch->dst)
+		kunmap_local(dst);
+
 	spin_unlock(&scratch->lock);
 	return ret;
 }
Signature · 1 line
2.34.1