Batched zswap_store() with compression batching

When storing a folio in zswap, save the folio's node id in the zswap entry instead of relying on the zswap_entry being allocated on the same node. This enables optimizations such as batch allocation of zswap entries for large folios using kmem_cache_alloc_bulk() that does not support specifying a node id.

The size of the zswap_entry remains unchanged as 'referenced' is lumped in with the 'length' (as it doesn't need a full unsigned int anyway).

Suggested-by Nhat Pham <[email protected]>

Suggested-by Yosry Ahmed <[email protected]>

Signed-off-by Kanchana P Sridhar <[email protected]>

Signed-off-by Kanchana P. Sridhar <[email protected]>

---
 mm/zswap.c | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

mm/zswap.c +1 -0

index ca513a9f89e3..c2731d3acb2c 100644
@@ -194,6 +194,7 @@ static struct shrinker *zswap_shrinker;
  *              writeback logic. The entry is only reclaimed by the writeback
  *              logic if referenced is unset. See comments in the shrinker
  *              section for context.
+ * nid - NUMA node id of the page for which this is the zswap entry.
  * pool_idx - id of the zswap_pool that the entry's data is in.
  * handle - zsmalloc allocation handle that stores the compressed page data
  * objcg - the obj_cgroup that the compressed memory is charged to

mm/zswap.c +2 -9

 	u8 pool_idx;
 	unsigned long handle;
 	struct obj_cgroup *objcg;

@@ -672,14 +676,8 @@ static int zswap_enabled_param_set(const char *val,
 * lru functions
 **********************************/
 
-static inline int entry_to_nid(struct zswap_entry *entry)
-{
-	return page_to_nid(virt_to_page(entry));
-}
-
 static void zswap_lru_add(struct zswap_entry *entry)
 {
-	int nid = entry_to_nid(entry);
 	struct mem_cgroup *memcg;
 
 	/*

@@ -696,19 +694,18 @@ static void zswap_lru_add(struct zswap_entry *entry)
 	rcu_read_lock();
 	memcg = obj_cgroup_memcg(entry->objcg);
 	/* will always succeed */
-	list_lru_add(&zswap_list_lru, &entry->lru, nid, memcg);
+	list_lru_add(&zswap_list_lru, &entry->lru, entry->nid, memcg);
 	rcu_read_unlock();
 }
 
 static void zswap_lru_del(struct zswap_entry *entry)
 {
-	int nid = entry_to_nid(entry);
 	struct mem_cgroup *memcg;
 
 	rcu_read_lock();
 	memcg = obj_cgroup_memcg(entry->objcg);
 	/* will always succeed */
-	list_lru_del(&zswap_list_lru, &entry->lru, nid, memcg);
+	list_lru_del(&zswap_list_lru, &entry->lru, entry->nid, memcg);
 	rcu_read_unlock();
 }
 

mm/zswap.c +1 -0

 	if (!entry) {
 		zswap_reject_kmemcache_fail++;
 		return false;

@@ -1466,6 +1464,7 @@ static bool zswap_store_page(struct folio *folio, long index,
 	 * to an unrelated live pool.
 	 */
 	entry->pool_idx = pool->idx;
+	entry->nid = nid;
 
 	old = xa_store(swap_zswap_tree(page_swpentry),
 		       swp_offset(page_swpentry),

Signature · 1 line

2.39.5

Quoted 34 lines · Kanchana P. Sridhar

When storing a folio in zswap, save the folio's node id in the zswap entry instead of relying on the zswap_entry being allocated on the same node. This enables optimizations such as batch allocation of zswap entries for large folios using kmem_cache_alloc_bulk() that does not support specifying a node id.

The size of the zswap_entry remains unchanged as 'referenced' is lumped in with the 'length' (as it doesn't need a full unsigned int anyway).

Suggested-by Nhat Pham <[email protected]>

Suggested-by Yosry Ahmed <[email protected]>

Signed-off-by Kanchana P Sridhar <[email protected]>

Signed-off-by Kanchana P. Sridhar <[email protected]>

---
 mm/zswap.c | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

mm/zswap.c +1 -1

index ca513a9f89e3..c2731d3acb2c 100644
@@ -194,6 +194,7 @@ static struct shrinker *zswap_shrinker;
  *              writeback logic. The entry is only reclaimed by the writeback
  *              logic if referenced is unset. See comments in the shrinker
  *              section for context.
+ * nid - NUMA node id of the page for which this is the zswap entry.
  * pool_idx - id of the zswap_pool that the entry's data is in.
  * handle - zsmalloc allocation handle that stores the compressed page data
  * objcg - the obj_cgroup that the compressed memory is charged to

@@ -201,8 +202,11 @@ static struct shrinker *zswap_shrinker;
  */
 struct zswap_entry {
      swp_entry_t swpentry;
-     unsigned int length;

mm/zswap.c static struct shrinker *zswap_shrinker; · line 209

Quoted 6 lines

mm/zswap.c +5 -1

-     bool referenced;
+     struct {
+             unsigned int length:31;
+             bool referenced:1;
+     };
+     int nid;

This will increase the size of the struct now with pool_idx, right?

Before you patch on x86_64:

struct zswap_entry {

        swp_entry_t                swpentry;             /*     0     8 */
        unsigned int               length;               /*     8     4 */
        bool                       referenced;           /*    12     1 */
        u8                         pool_idx;             /*    13     1 */

        /* XXX 2 bytes hole, try to pack */

        unsigned long              handle;               /*    16     8 */
        struct obj_cgroup *        objcg;                /*    24     8 */
        struct list_head           lru;                  /*    32    16 */

        /* size: 48, cachelines: 1, members: 7 */
        /* sum members: 46, holes: 1, sum holes: 2 */
        /* last cacheline: 48 bytes */
};

After:

struct zswap_entry {
        swp_entry_t                swpentry;             /*     0     8 */
        struct {
                unsigned int       length:31;            /*     8: 0  4 */

                /* Bitfield combined with next fields */

                bool               referenced:1;         /*    11: 7  1 */
        };                                               /*     8     4 */
        struct {
                unsigned int               length:31;            /*     0: 0  4 */

                /* Bitfield combined with next fields */

                bool                       referenced:1;         /*     3: 7  1 */

                /* size: 4, cachelines: 1, members: 2 */
                /* last cacheline: 4 bytes */
        };

        int                        nid;                  /*    12     4 */
        u8                         pool_idx;             /*    16     1 */

        /* XXX 7 bytes hole, try to pack */

        unsigned long              handle;               /*    24     8 */
        struct obj_cgroup *        objcg;                /*    32     8 */
        struct list_head           lru;                  /*    40    16 */

        /* size: 56, cachelines: 1, members: 7 */
        /* sum members: 49, holes: 1, sum holes: 7 */
        /* last cacheline: 56 bytes */
};

I think we want to reorder the fields. We can probably also avoid the bitfield if the maximum page size is 64K (is it?)

struct zswap_entry {

        swp_entry_t                swpentry;             /*     0     8 */
        u16                        length;               /*     8     2 */
        u8                         pool_idx;             /*    10     1 */
        bool                       referenced;           /*    11     1 */
        int                        nid;                  /*    12     4 */
        unsigned long              handle;               /*    16     8 */
        struct obj_cgroup *        objcg;                /*    24     8 */
        struct list_head           lru;                  /*    32    16 */

        /* size: 48, cachelines: 1, members: 8 */
        /* last cacheline: 48 bytes */
};

Quoted 46 lines

mm/zswap.c +3 -9

      u8 pool_idx;
      unsigned long handle;
      struct obj_cgroup *objcg;

@@ -672,14 +676,8 @@ static int zswap_enabled_param_set(const char *val,
 * lru functions
 **********************************/
 
-static inline int entry_to_nid(struct zswap_entry *entry)
-{
-     return page_to_nid(virt_to_page(entry));
-}
-
 static void zswap_lru_add(struct zswap_entry *entry)
 {
-     int nid = entry_to_nid(entry);
      struct mem_cgroup *memcg;
 
      /*

@@ -696,19 +694,18 @@ static void zswap_lru_add(struct zswap_entry *entry)
      rcu_read_lock();
      memcg = obj_cgroup_memcg(entry->objcg);
      /* will always succeed */
-     list_lru_add(&zswap_list_lru, &entry->lru, nid, memcg);
+     list_lru_add(&zswap_list_lru, &entry->lru, entry->nid, memcg);
      rcu_read_unlock();
 }
 
 static void zswap_lru_del(struct zswap_entry *entry)
 {
-     int nid = entry_to_nid(entry);
      struct mem_cgroup *memcg;
 
      rcu_read_lock();
      memcg = obj_cgroup_memcg(entry->objcg);
      /* will always succeed */
-     list_lru_del(&zswap_list_lru, &entry->lru, nid, memcg);
+     list_lru_del(&zswap_list_lru, &entry->lru, entry->nid, memcg);
      rcu_read_unlock();
 }
 

@@ -1449,9 +1446,10 @@ static bool zswap_store_page(struct folio *folio, long index,
 {
      swp_entry_t page_swpentry = folio_swap_entry(folio, index);
      struct zswap_entry *entry, *old;
+     int nid = folio_nid(folio);
 

mm/zswap.c static bool zswap_store_page(struct folio *folio, long index, · line 1452

Quoted 3 lines

mm/zswap.c +1 -1

      /* allocate entry */
-     entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio));
+     entry = zswap_entry_cache_alloc(GFP_KERNEL, nid);

We don't need to allocate the entry on the same node anymore, should we drop this and let the slab allocator handle it as it sees fit?

Quoted 15 lines

mm/zswap.c +1 -1

      if (!entry) {
              zswap_reject_kmemcache_fail++;
              return false;

@@ -1466,6 +1464,7 @@ static bool zswap_store_page(struct folio *folio, long index,
       * to an unrelated live pool.
       */
      entry->pool_idx = pool->idx;
+     entry->nid = nid;
 
      old = xa_store(swap_zswap_tree(page_swpentry),
                     swp_offset(page_swpentry),
--

2.39.5

Quoted 36 lines · Yosry Ahmed

Quoted 34 lines · Kanchana P. Sridhar

When storing a folio in zswap, save the folio's node id in the zswap entry instead of relying on the zswap_entry being allocated on the same node. This enables optimizations such as batch allocation of zswap entries for large folios using kmem_cache_alloc_bulk() that does not support specifying a node id.

The size of the zswap_entry remains unchanged as 'referenced' is lumped in with the 'length' (as it doesn't need a full unsigned int anyway).

Suggested-by Nhat Pham <[email protected]>

Suggested-by Yosry Ahmed <[email protected]>

Signed-off-by Kanchana P Sridhar <[email protected]>

Signed-off-by Kanchana P. Sridhar <[email protected]>

---
 mm/zswap.c | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

mm/zswap.c +1 -1

index ca513a9f89e3..c2731d3acb2c 100644
@@ -194,6 +194,7 @@ static struct shrinker *zswap_shrinker;
  *              writeback logic. The entry is only reclaimed by the writeback
  *              logic if referenced is unset. See comments in the shrinker
  *              section for context.
+ * nid - NUMA node id of the page for which this is the zswap entry.
  * pool_idx - id of the zswap_pool that the entry's data is in.
  * handle - zsmalloc allocation handle that stores the compressed page data
  * objcg - the obj_cgroup that the compressed memory is charged to

@@ -201,8 +202,11 @@ static struct shrinker *zswap_shrinker;
  */
 struct zswap_entry {
      swp_entry_t swpentry;
-     unsigned int length;

Quoted 8 lines

mm/zswap.c static struct shrinker *zswap_shrinker; · line 209

Quoted 6 lines

mm/zswap.c +5 -1

-     bool referenced;
+     struct {
+             unsigned int length:31;
+             bool referenced:1;
+     };
+     int nid;

This will increase the size of the struct now with pool_idx, right?

I was gathering the struct size with the struct {unsigned int length:31; bool referenced:1} as suggested in v14, for my before/after comparisons:

before:
struct zswap_entry {
swp_entry_t swpentry; /* 0 8 */
struct {
unsigned int length:31; /* 8: 0 4 */

/* Bitfield combined with next fields */

bool referenced:1; /* 11: 7 1 */
}; /* 8 4 */
int nid; /* 12 4 */
u8 pool_idx; /* 16 1 */

/* XXX 7 bytes hole, try to pack */

long unsigned int handle; /* 24 8 */
struct obj_cgroup * objcg; /* 32 8 */
struct list_head lru; /* 40 16 */

/* size: 56, cachelines: 1, members: 7 */ /* sum members: 49, holes: 1, sum holes: 7 */ /* last cacheline: 56 bytes */
};

after:
struct zswap_entry {
swp_entry_t swpentry; /* 0 8 */
struct {
unsigned int length:31; /* 8: 0 4 */

/* Bitfield combined with next fields */

bool referenced:1; /* 11: 7 1 */
}; /* 8 4 */
int nid; /* 12 4 */
long unsigned int handle; /* 16 8 */
struct obj_cgroup * objcg; /* 24 8 */
struct list_head lru; /* 32 16 */
u8 pool_idx; /* 48 1 */

/* size: 56, cachelines: 1, members: 7 */
/* padding: 7 */
/* last cacheline: 56 bytes */
};

And decided to go with the placement in patch 7, since it also separates out members by compress/decompress path accesses. pool_idx is set once, and never changes. Further, moving pool_idx to the end gets rid of holes.

Quoted 57 lines

Before you patch on x86_64:

struct zswap_entry {

        swp_entry_t                swpentry;             /*     0     8 */
        unsigned int               length;               /*     8     4 */
        bool                       referenced;           /*    12     1 */
        u8                         pool_idx;             /*    13     1 */

        /* XXX 2 bytes hole, try to pack */

        unsigned long              handle;               /*    16     8 */
        struct obj_cgroup *        objcg;                /*    24     8 */
        struct list_head           lru;                  /*    32    16 */

        /* size: 48, cachelines: 1, members: 7 */
        /* sum members: 46, holes: 1, sum holes: 2 */
        /* last cacheline: 48 bytes */
};

After:

struct zswap_entry {
        swp_entry_t                swpentry;             /*     0     8 */
        struct {
                unsigned int       length:31;            /*     8: 0  4 */

                /* Bitfield combined with next fields */

                bool               referenced:1;         /*    11: 7  1 */
        };                                               /*     8     4 */
        struct {
                unsigned int               length:31;            /*     0: 0  4 */

                /* Bitfield combined with next fields */

                bool                       referenced:1;         /*     3: 7  1 */

                /* size: 4, cachelines: 1, members: 2 */
                /* last cacheline: 4 bytes */
        };

        int                        nid;                  /*    12     4 */
        u8                         pool_idx;             /*    16     1 */

        /* XXX 7 bytes hole, try to pack */

        unsigned long              handle;               /*    24     8 */
        struct obj_cgroup *        objcg;                /*    32     8 */
        struct list_head           lru;                  /*    40    16 */

        /* size: 56, cachelines: 1, members: 7 */
        /* sum members: 49, holes: 1, sum holes: 7 */
        /* last cacheline: 56 bytes */
};

Quoted 2 lines

I think we want to reorder the fields. We can probably also avoid the bitfield if the maximum page size is 64K (is it?)

It seems on ARM64, the max page size is 64K, however on PPC, it can be 256K. So it appears we need at least 18 bits on currently supported architectures.

If you agree with the pool_idx placement to be moved to the end as proposed in patch 7, I can try and optimize this further, by limiting the "unsigned int length:18", if you think this makes sense.

Quoted 63 lines

struct zswap_entry {

        swp_entry_t                swpentry;             /*     0     8 */
        u16                        length;               /*     8     2 */
        u8                         pool_idx;             /*    10     1 */
        bool                       referenced;           /*    11     1 */
        int                        nid;                  /*    12     4 */
        unsigned long              handle;               /*    16     8 */
        struct obj_cgroup *        objcg;                /*    24     8 */
        struct list_head           lru;                  /*    32    16 */

        /* size: 48, cachelines: 1, members: 8 */
        /* last cacheline: 48 bytes */
};

Quoted 46 lines

mm/zswap.c +3 -9

      u8 pool_idx;
      unsigned long handle;
      struct obj_cgroup *objcg;

@@ -672,14 +676,8 @@ static int zswap_enabled_param_set(const char *val,
 * lru functions
 **********************************/
 
-static inline int entry_to_nid(struct zswap_entry *entry)
-{
-     return page_to_nid(virt_to_page(entry));
-}
-
 static void zswap_lru_add(struct zswap_entry *entry)
 {
-     int nid = entry_to_nid(entry);
      struct mem_cgroup *memcg;
 
      /*

@@ -696,19 +694,18 @@ static void zswap_lru_add(struct zswap_entry *entry)
      rcu_read_lock();
      memcg = obj_cgroup_memcg(entry->objcg);
      /* will always succeed */
-     list_lru_add(&zswap_list_lru, &entry->lru, nid, memcg);
+     list_lru_add(&zswap_list_lru, &entry->lru, entry->nid, memcg);
      rcu_read_unlock();
 }
 
 static void zswap_lru_del(struct zswap_entry *entry)
 {
-     int nid = entry_to_nid(entry);
      struct mem_cgroup *memcg;
 
      rcu_read_lock();
      memcg = obj_cgroup_memcg(entry->objcg);
      /* will always succeed */
-     list_lru_del(&zswap_list_lru, &entry->lru, nid, memcg);
+     list_lru_del(&zswap_list_lru, &entry->lru, entry->nid, memcg);
      rcu_read_unlock();
 }
 

@@ -1449,9 +1446,10 @@ static bool zswap_store_page(struct folio *folio, long index,
 {
      swp_entry_t page_swpentry = folio_swap_entry(folio, index);
      struct zswap_entry *entry, *old;
+     int nid = folio_nid(folio);
 

Quoted 6 lines

mm/zswap.c static bool zswap_store_page(struct folio *folio, long index, · line 1452

Quoted 3 lines

mm/zswap.c +1 -1

      /* allocate entry */
-     entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio));
+     entry = zswap_entry_cache_alloc(GFP_KERNEL, nid);

We don't need to allocate the entry on the same node anymore, should we drop this and let the slab allocator handle it as it sees fit?

Was this a recent change? I see the folio_nid(folio) on my version of mm-unstable as of 10-3-2026. Let me check, and yes, allowing the slab allocated handle this is probably a good idea.

Along the same lines, I have been working on replacing zsmalloc with direct SLAB allocations for the zswap compressed data storage and see very promising results: memory savings and latency improvements. I will submit an RFC shortly.

Quoted 16 lines

Quoted 16 lines

mm/zswap.c +1 -1

      if (!entry) {
              zswap_reject_kmemcache_fail++;
              return false;

@@ -1466,6 +1464,7 @@ static bool zswap_store_page(struct folio *folio, long index,
       * to an unrelated live pool.
       */
      entry->pool_idx = pool->idx;
+     entry->nid = nid;
 
      old = xa_store(swap_zswap_tree(page_swpentry),
                     swp_offset(page_swpentry),
--

2.39.5

Quoted 96 lines · Kanchana P. Sridhar

Quoted 36 lines · Yosry Ahmed

Quoted 34 lines · Kanchana P. Sridhar

When storing a folio in zswap, save the folio's node id in the zswap entry instead of relying on the zswap_entry being allocated on the same node. This enables optimizations such as batch allocation of zswap entries for large folios using kmem_cache_alloc_bulk() that does not support specifying a node id.

The size of the zswap_entry remains unchanged as 'referenced' is lumped in with the 'length' (as it doesn't need a full unsigned int anyway).

Suggested-by Nhat Pham <[email protected]>

Suggested-by Yosry Ahmed <[email protected]>

Signed-off-by Kanchana P Sridhar <[email protected]>

Signed-off-by Kanchana P. Sridhar <[email protected]>

---
 mm/zswap.c | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

mm/zswap.c +1 -1

index ca513a9f89e3..c2731d3acb2c 100644
@@ -194,6 +194,7 @@ static struct shrinker *zswap_shrinker;
  *              writeback logic. The entry is only reclaimed by the writeback
  *              logic if referenced is unset. See comments in the shrinker
  *              section for context.
+ * nid - NUMA node id of the page for which this is the zswap entry.
  * pool_idx - id of the zswap_pool that the entry's data is in.
  * handle - zsmalloc allocation handle that stores the compressed page data
  * objcg - the obj_cgroup that the compressed memory is charged to

@@ -201,8 +202,11 @@ static struct shrinker *zswap_shrinker;
  */
 struct zswap_entry {
      swp_entry_t swpentry;
-     unsigned int length;

Quoted 8 lines

mm/zswap.c static struct shrinker *zswap_shrinker; · line 209

Quoted 6 lines

mm/zswap.c +5 -1

-     bool referenced;
+     struct {
+             unsigned int length:31;
+             bool referenced:1;
+     };
+     int nid;

This will increase the size of the struct now with pool_idx, right?

I was gathering the struct size with the struct {unsigned int length:31; bool referenced:1} as suggested in v14, for my before/after comparisons:

before:
struct zswap_entry {
swp_entry_t swpentry; /* 0 8 */
struct {
unsigned int length:31; /* 8: 0 4 */

/* Bitfield combined with next fields */

bool referenced:1; /* 11: 7 1 */
}; /* 8 4 */
int nid; /* 12 4 */
u8 pool_idx; /* 16 1 */

/* XXX 7 bytes hole, try to pack */

long unsigned int handle; /* 24 8 */
struct obj_cgroup * objcg; /* 32 8 */
struct list_head lru; /* 40 16 */

/* size: 56, cachelines: 1, members: 7 */ /* sum members: 49, holes: 1, sum holes: 7 */ /* last cacheline: 56 bytes */
};

after:
struct zswap_entry {
swp_entry_t swpentry; /* 0 8 */
struct {
unsigned int length:31; /* 8: 0 4 */

/* Bitfield combined with next fields */

bool referenced:1; /* 11: 7 1 */
}; /* 8 4 */
int nid; /* 12 4 */
long unsigned int handle; /* 16 8 */
struct obj_cgroup * objcg; /* 24 8 */
struct list_head lru; /* 32 16 */
u8 pool_idx; /* 48 1 */

/* size: 56, cachelines: 1, members: 7 */
/* padding: 7 */
/* last cacheline: 56 bytes */
};

Quoted 4 lines

And decided to go with the placement in patch 7, since it also separates out members by compress/decompress path accesses. pool_idx is set once, and never changes. Further, moving pool_idx to the end gets rid of holes.

Please look at the layout I pasted (quoted below). Your patch increases the struct size from 48 bytes to 56 bytes, undoing the optimization introduced by pool_idx. We need to pack length, pool_idx, and referenced in a single u32 to keep the size constant.

FWIW, moving pool_idx to the end just trades holes for padding, I am not sure if that's necessarily an improvement.

Quoted 57 lines

Quoted 57 lines

Before you patch on x86_64:

struct zswap_entry {

        swp_entry_t                swpentry;             /*     0     8 */
        unsigned int               length;               /*     8     4 */
        bool                       referenced;           /*    12     1 */
        u8                         pool_idx;             /*    13     1 */

        /* XXX 2 bytes hole, try to pack */

        unsigned long              handle;               /*    16     8 */
        struct obj_cgroup *        objcg;                /*    24     8 */
        struct list_head           lru;                  /*    32    16 */

        /* size: 48, cachelines: 1, members: 7 */
        /* sum members: 46, holes: 1, sum holes: 2 */
        /* last cacheline: 48 bytes */
};

After:

struct zswap_entry {
        swp_entry_t                swpentry;             /*     0     8 */
        struct {
                unsigned int       length:31;            /*     8: 0  4 */

                /* Bitfield combined with next fields */

                bool               referenced:1;         /*    11: 7  1 */
        };                                               /*     8     4 */
        struct {
                unsigned int               length:31;            /*     0: 0  4 */

                /* Bitfield combined with next fields */

                bool                       referenced:1;         /*     3: 7  1 */

                /* size: 4, cachelines: 1, members: 2 */
                /* last cacheline: 4 bytes */
        };

        int                        nid;                  /*    12     4 */
        u8                         pool_idx;             /*    16     1 */

        /* XXX 7 bytes hole, try to pack */

        unsigned long              handle;               /*    24     8 */
        struct obj_cgroup *        objcg;                /*    32     8 */
        struct list_head           lru;                  /*    40    16 */

        /* size: 56, cachelines: 1, members: 7 */
        /* sum members: 49, holes: 1, sum holes: 7 */
        /* last cacheline: 56 bytes */
};

Quoted 6 lines

Quoted 2 lines

I think we want to reorder the fields. We can probably also avoid the bitfield if the maximum page size is 64K (is it?)

It seems on ARM64, the max page size is 64K, however on PPC, it can be 256K. So it appears we need at least 18 bits on currently supported architectures.

In this case unfortunately we need to stick with bitfields. We should probably add a build-time assertion that the width of length is enough to cover PAGE_SIZE.

Quoted 3 lines

If you agree with the pool_idx placement to be moved to the end as proposed in patch 7, I can try and optimize this further, by limiting the "unsigned int length:18", if you think this makes sense.

I don't think we can put pool_idx to the end and keep the struct size constant. For length, yeah let's use the minimum number of bits needed and add an assertion, so that the free to use bits are obvious.

Quoted 71 lines

Quoted 70 lines

struct zswap_entry {

        swp_entry_t                swpentry;             /*     0     8 */
        u16                        length;               /*     8     2 */
        u8                         pool_idx;             /*    10     1 */
        bool                       referenced;           /*    11     1 */
        int                        nid;                  /*    12     4 */
        unsigned long              handle;               /*    16     8 */
        struct obj_cgroup *        objcg;                /*    24     8 */
        struct list_head           lru;                  /*    32    16 */

        /* size: 48, cachelines: 1, members: 8 */
        /* last cacheline: 48 bytes */
};

Quoted 41 lines

mm/zswap.c +2 -9

      u8 pool_idx;
      unsigned long handle;
      struct obj_cgroup *objcg;

@@ -672,14 +676,8 @@ static int zswap_enabled_param_set(const char *val,
 * lru functions
 **********************************/
 
-static inline int entry_to_nid(struct zswap_entry *entry)
-{
-     return page_to_nid(virt_to_page(entry));
-}
-
 static void zswap_lru_add(struct zswap_entry *entry)
 {
-     int nid = entry_to_nid(entry);
      struct mem_cgroup *memcg;
 
      /*

@@ -696,19 +694,18 @@ static void zswap_lru_add(struct zswap_entry *entry)
      rcu_read_lock();
      memcg = obj_cgroup_memcg(entry->objcg);
      /* will always succeed */
-     list_lru_add(&zswap_list_lru, &entry->lru, nid, memcg);
+     list_lru_add(&zswap_list_lru, &entry->lru, entry->nid, memcg);
      rcu_read_unlock();
 }
 
 static void zswap_lru_del(struct zswap_entry *entry)
 {
-     int nid = entry_to_nid(entry);
      struct mem_cgroup *memcg;
 
      rcu_read_lock();
      memcg = obj_cgroup_memcg(entry->objcg);
      /* will always succeed */
-     list_lru_del(&zswap_list_lru, &entry->lru, nid, memcg);
+     list_lru_del(&zswap_list_lru, &entry->lru, entry->nid, memcg);
      rcu_read_unlock();
 }
 

mm/zswap.c static bool zswap_store_page(struct folio *folio, long index, · line 1452

Quoted 8 lines

mm/zswap.c +2 -1

@@ -1449,9 +1446,10 @@ static bool zswap_store_page(struct folio *folio, long index,
 {
      swp_entry_t page_swpentry = folio_swap_entry(folio, index);
      struct zswap_entry *entry, *old;
+     int nid = folio_nid(folio);
 
      /* allocate entry */
-     entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio));
+     entry = zswap_entry_cache_alloc(GFP_KERNEL, nid);

We don't need to allocate the entry on the same node anymore, should we drop this and let the slab allocator handle it as it sees fit?

Quoted 3 lines

Was this a recent change? I see the folio_nid(folio) on my version of mm-unstable as of 10-3-2026. Let me check, and yes, allowing the slab allocated handle this is probably a good idea.

We allocate the entry on the same node to avoid having to store nid in the zswap entry (see entry_to_nid()). If we store it, then there's no need to allocate on the node.

Quoted 6 lines

Along the same lines, I have been working on replacing zsmalloc with direct SLAB allocations for the zswap compressed data storage and see very promising results: memory savings and latency improvements. I will submit an RFC shortly.

Keep in mind that zsmalloc is movable memory, while slab memory isn't. Zsmalloc may also be less likely to make a new allocation in the store path. There are other considerations beyond memory overhead and latency here.

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

mm/zswap.c +1 -1

      if (!entry) {
              zswap_reject_kmemcache_fail++;
              return false;

@@ -1466,6 +1464,7 @@ static bool zswap_store_page(struct folio *folio, long index,
       * to an unrelated live pool.
       */
      entry->pool_idx = pool->idx;
+     entry->nid = nid;
 
      old = xa_store(swap_zswap_tree(page_swpentry),
                     swp_offset(page_swpentry),
--

2.39.5

Quoted 17 lines

Quoted 16 lines

Quoted 6 lines

Quoted 6 lines

mm/zswap.c +1 -0

@@ -1449,9 +1446,10 @@ static bool zswap_store_page(struct folio *folio, long index,
 {
      swp_entry_t page_swpentry = folio_swap_entry(folio, index);
      struct zswap_entry *entry, *old;
+     int nid = folio_nid(folio);
 

Quoted 6 lines

mm/zswap.c static bool zswap_store_page(struct folio *folio, long index, · line 1452

Quoted 3 lines

mm/zswap.c +1 -1

      /* allocate entry */
-     entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio));
+     entry = zswap_entry_cache_alloc(GFP_KERNEL, nid);

We don't need to allocate the entry on the same node anymore, should we drop this and let the slab allocator handle it as it sees fit?

Was this a recent change? I see the folio_nid(folio) on my version of mm-unstable as of 10-3-2026. Let me check, and yes, allowing the slab allocated handle this is probably a good idea.

Quoted 3 lines

We allocate the entry on the same node to avoid having to store nid in the zswap entry (see entry_to_nid()). If we store it, then there's no need to allocate on the node.

With patch 0006, we try to bulk allocate all entries for the batch. Only if this fails, we call zswap_entry_cache_alloc(gfp, nid). So patch 0006 addresses this already, and the nid is used on in the fallback case.

Quoted 8 lines

Quoted 7 lines

Along the same lines, I have been working on replacing zsmalloc with direct SLAB allocations for the zswap compressed data storage and see very promising results: memory savings and latency improvements. I will submit an RFC shortly.

Quoted 4 lines

Keep in mind that zsmalloc is movable memory, while slab memory isn't. Zsmalloc may also be less likely to make a new allocation in the store path. There are other considerations beyond memory overhead and latency here.

I realize this means we trade off page migration for writeback for the pages representing zswap compressed data. This might not necessarily be a bad thing for 2 main reasons, the way I see it in this specific context:

1) Page migration would only be transferring a memory pressure problem elsewhere. Not sure how well this would work with memcgs or containers. At some point someone would need to do writeback.
2) I don't believe there is a zswap specific page migration flow in the kernel other than zsmalloc compaction. IOW, memory tiering components like CXL are handled directly in the reclaim path. From what I understand, things wouldn't break with slab memory for zswap compressed storage, but I could be wrong. I hope to include the architecture level considerations/trade-offs in the RFC.

Quoted 22 lines

Quoted 21 lines

Quoted 14 lines

Quoted 7 lines

Quoted 7 lines

mm/zswap.c +1 -0

@@ -1449,9 +1446,10 @@ static bool zswap_store_page(struct folio *folio, long index,
 {
      swp_entry_t page_swpentry = folio_swap_entry(folio, index);
      struct zswap_entry *entry, *old;
+     int nid = folio_nid(folio);
 

Quoted 6 lines

mm/zswap.c static bool zswap_store_page(struct folio *folio, long index, · line 1452

Quoted 3 lines

mm/zswap.c +1 -1

      /* allocate entry */
-     entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio));
+     entry = zswap_entry_cache_alloc(GFP_KERNEL, nid);

We don't need to allocate the entry on the same node anymore, should we drop this and let the slab allocator handle it as it sees fit?

Quoted 3 lines

Was this a recent change? I see the folio_nid(folio) on my version of mm-unstable as of 10-3-2026. Let me check, and yes, allowing the slab allocated handle this is probably a good idea.

We allocate the entry on the same node to avoid having to store nid in the zswap entry (see entry_to_nid()). If we store it, then there's no need to allocate on the node.

Quoted 4 lines

With patch 0006, we try to bulk allocate all entries for the batch. Only if this fails, we call zswap_entry_cache_alloc(gfp, nid). So patch 0006 addresses this already, and the nid is used on in the fallback case.

If we are storing the nid in the zswap entry, there is no need for zswap_entry_cache_alloc() to use the nid anymore.

Quoted 19 lines

Quoted 9 lines

Quoted 8 lines

Along the same lines, I have been working on replacing zsmalloc with direct SLAB allocations for the zswap compressed data storage and see very promising results: memory savings and latency improvements. I will submit an RFC shortly.

Quoted 4 lines

Keep in mind that zsmalloc is movable memory, while slab memory isn't. Zsmalloc may also be less likely to make a new allocation in the store path. There are other considerations beyond memory overhead and latency here.

I realize this means we trade off page migration for writeback for the pages representing zswap compressed data. This might not necessarily be a bad thing for 2 main reasons, the way I see it in this specific context:

Quoted 9 lines

1) Page migration would only be transferring a memory pressure problem elsewhere. Not sure how well this would work with memcgs or containers. At some point someone would need to do writeback.
2) I don't believe there is a zswap specific page migration flow in the kernel other than zsmalloc compaction. IOW, memory tiering components like CXL are handled directly in the reclaim path. From what I understand, things wouldn't break with slab memory for zswap compressed storage, but I could be wrong. I hope to include the architecture level considerations/trade-offs in the RFC.

Zsmalloc memory is currently movable, so compaction can move it around to free contiguous blocks (e.g. for THP allocations). Slab memory is not movable. It can be reclaimable, but writing back memory from zswap when we are not under memory pressure is also bad.

Note that compressed memory can be very large compared to slab allocations (or kernel allocations in general), so it can increase unmovable (or reclaimable but not movable) memory significantly, causing fragmentation.

Quoted 27 lines · Yosry Ahmed

Quoted 26 lines

Quoted 14 lines

Quoted 14 lines

Quoted 13 lines

Review static bool zswap_store_page(struct folio *folio, long index, · line 1452

Quoted 10 lines

(quoted diff) +2 -1

@@ -1449,9 +1446,10 @@ static bool zswap_store_page(struct folio *folio, long index,
 {
      swp_entry_t page_swpentry = folio_swap_entry(folio, index);
      struct zswap_entry *entry, *old;
+     int nid = folio_nid(folio);
 
      /* allocate entry */
-     entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio));
+     entry = zswap_entry_cache_alloc(GFP_KERNEL, nid);

We don't need to allocate the entry on the same node anymore, should we drop this and let the slab allocator handle it as it sees fit?

Quoted 7 lines

Quoted 3 lines

Was this a recent change? I see the folio_nid(folio) on my version of mm-unstable as of 10-3-2026. Let me check, and yes, allowing the slab allocated handle this is probably a good idea.

We allocate the entry on the same node to avoid having to store nid in the zswap entry (see entry_to_nid()). If we store it, then there's no need to allocate on the node.

With patch 0006, we try to bulk allocate all entries for the batch. Only if this fails, we call zswap_entry_cache_alloc(gfp, nid). So patch 0006 addresses this already, and the nid is used on in the fallback case.

Quoted 2 lines

If we are storing the nid in the zswap entry, there is no need for zswap_entry_cache_alloc() to use the nid anymore.

As of now, zswap_entry_cache_alloc() uses the nid for kmem_cache_alloc_node(). Is the suggestion to change this to be kmem_cache_alloc()?

Quoted 30 lines

Quoted 20 lines

Quoted 10 lines

Quoted 9 lines

Along the same lines, I have been working on replacing zsmalloc with direct SLAB allocations for the zswap compressed data storage and see very promising results: memory savings and latency improvements. I will submit an RFC shortly.

Quoted 4 lines

Keep in mind that zsmalloc is movable memory, while slab memory isn't. Zsmalloc may also be less likely to make a new allocation in the store path. There are other considerations beyond memory overhead and latency here.

I realize this means we trade off page migration for writeback for the pages representing zswap compressed data. This might not necessarily be a bad thing for 2 main reasons, the way I see it in this specific context:

Quoted 9 lines

1) Page migration would only be transferring a memory pressure problem elsewhere. Not sure how well this would work with memcgs or containers. At some point someone would need to do writeback.
2) I don't believe there is a zswap specific page migration flow in the kernel other than zsmalloc compaction. IOW, memory tiering components like CXL are handled directly in the reclaim path. From what I understand, things wouldn't break with slab memory for zswap compressed storage, but I could be wrong. I hope to include the architecture level considerations/trade-offs in the RFC.

Quoted 4 lines

Zsmalloc memory is currently movable, so compaction can move it around to free contiguous blocks (e.g. for THP allocations). Slab memory is not movable. It can be reclaimable, but writing back memory from zswap when we are not under memory pressure is also bad.

All good points, thanks. I agree, writing back memory from zswap when not under memory pressure is bad, however I have not seen any writebacks in my experiments with usemem at 60% over-commit and with 64kB mTHP.

I am running these experiments in a memory constrained environment:

QEMU VM: 7G RAM
Swap: 3G
cgroup memory.high: 3G
4 usemem processes want to allocate 1200M each. To make the experiment even more stringent and representative of actual compression datasets, each usemem process reads the Silesia dataset from a pre-fork-allocated buffer into the 1200M. This causes realistic compression payloads in usemem.

Quoted 5 lines

Note that compressed memory can be very large compared to slab allocations (or kernel allocations in general), so it can increase unmovable (or reclaimable but not movable) memory significantly, causing fragmentation.

It took me a while to experiment with compressed size bins, until I settled on a compromise between memory wasted per slab and zswap latency in finding the correct size bin to minimize fragmentation. I have been gathering extensive SLUB stats which indicate:

1) Even under peak zswap compressed quota ("pool") utilization across 10 consecutive iterations, SLUB comes out ahead in less overall memory overhead.

2) At steady state (when 10 iterations of the workload are done), most of the slabs have been returned to the Buddy allocator, except for 3M worth of slab setup per compressed size bin (0 objects in use): this is as compared to 32K of zsmalloc slub metadata that remains at steady state.

I learned some really interesting things from these experiments, which pose questions about the memory penalty due to latency that zsmalloc adds to zswap compress/decompress paths vs. "internal fragmentation". I expect to quantify this with data from my experimental setup.

Quoted 32 lines · Kanchana P. Sridhar

Quoted 29 lines · Yosry Ahmed

Quoted 18 lines

Quoted 18 lines

Quoted 17 lines

Quoted 7 lines

Quoted 7 lines

(quoted diff) +1 -0

@@ -1449,9 +1446,10 @@ static bool zswap_store_page(struct folio *folio, long index,
 {
      swp_entry_t page_swpentry = folio_swap_entry(folio, index);
      struct zswap_entry *entry, *old;
+     int nid = folio_nid(folio);
 

Quoted 6 lines

Review static bool zswap_store_page(struct folio *folio, long index, · line 1452

Quoted 3 lines

(quoted diff) +1 -1

      /* allocate entry */
-     entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio));
+     entry = zswap_entry_cache_alloc(GFP_KERNEL, nid);

We don't need to allocate the entry on the same node anymore, should we drop this and let the slab allocator handle it as it sees fit?

Was this a recent change? I see the folio_nid(folio) on my version of mm-unstable as of 10-3-2026. Let me check, and yes, allowing the slab allocated handle this is probably a good idea.

Quoted 8 lines

Quoted 3 lines

We allocate the entry on the same node to avoid having to store nid in the zswap entry (see entry_to_nid()). If we store it, then there's no need to allocate on the node.

With patch 0006, we try to bulk allocate all entries for the batch. Only if this fails, we call zswap_entry_cache_alloc(gfp, nid). So patch 0006 addresses this already, and the nid is used on in the fallback case.

If we are storing the nid in the zswap entry, there is no need for zswap_entry_cache_alloc() to use the nid anymore.

Quoted 3 lines

As of now, zswap_entry_cache_alloc() uses the nid for kmem_cache_alloc_node(). Is the suggestion to change this to be kmem_cache_alloc()?

Yes. I don't think we need to use kmem_cache_alloc_node() if we store nid in zswap_entry.

Quoted 36 lines

Quoted 35 lines

Quoted 21 lines

Quoted 11 lines

Quoted 10 lines

Along the same lines, I have been working on replacing zsmalloc with direct SLAB allocations for the zswap compressed data storage and see very promising results: memory savings and latency improvements. I will submit an RFC shortly.

Quoted 4 lines

Keep in mind that zsmalloc is movable memory, while slab memory isn't. Zsmalloc may also be less likely to make a new allocation in the store path. There are other considerations beyond memory overhead and latency here.

I realize this means we trade off page migration for writeback for the pages representing zswap compressed data. This might not necessarily be a bad thing for 2 main reasons, the way I see it in this specific context:

Quoted 9 lines

1) Page migration would only be transferring a memory pressure problem elsewhere. Not sure how well this would work with memcgs or containers. At some point someone would need to do writeback.
2) I don't believe there is a zswap specific page migration flow in the kernel other than zsmalloc compaction. IOW, memory tiering components like CXL are handled directly in the reclaim path. From what I understand, things wouldn't break with slab memory for zswap compressed storage, but I could be wrong. I hope to include the architecture level considerations/trade-offs in the RFC.

Zsmalloc memory is currently movable, so compaction can move it around to free contiguous blocks (e.g. for THP allocations). Slab memory is not movable. It can be reclaimable, but writing back memory from zswap when we are not under memory pressure is also bad.

Quoted 4 lines

All good points, thanks. I agree, writing back memory from zswap when not under memory pressure is bad, however I have not seen any writebacks in my experiments with usemem at 60% over-commit and with 64kB mTHP.

We can discuss further when you make your posting :)

Quoted 38 lines

I am running these experiments in a memory constrained environment:

QEMU VM: 7G RAM
Swap: 3G
cgroup memory.high: 3G
4 usemem processes want to allocate 1200M each. To make the experiment even more stringent and representative of actual compression datasets, each usemem process reads the Silesia dataset from a pre-fork-allocated buffer into the 1200M. This causes realistic compression payloads in usemem.

Quoted 5 lines

Note that compressed memory can be very large compared to slab allocations (or kernel allocations in general), so it can increase unmovable (or reclaimable but not movable) memory significantly, causing fragmentation.

It took me a while to experiment with compressed size bins, until I settled on a compromise between memory wasted per slab and zswap latency in finding the correct size bin to minimize fragmentation. I have been gathering extensive SLUB stats which indicate:

1) Even under peak zswap compressed quota ("pool") utilization across 10 consecutive iterations, SLUB comes out ahead in less overall memory overhead.

2) At steady state (when 10 iterations of the workload are done), most of the slabs have been returned to the Buddy allocator, except for 3M worth of slab setup per compressed size bin (0 objects in use): this is as compared to 32K of zsmalloc slub metadata that remains at steady state.

I learned some really interesting things from these experiments, which pose questions about the memory penalty due to latency that zsmalloc adds to zswap compress/decompress paths vs. "internal fragmentation". I expect to quantify this with data from my experimental setup.

Place and align the members of struct zswap_pool and struct zswap_entry such that members used together in compress/decompress hot paths are able to share the same cacheline. Minimize holes within these structs while doing so.

Signed-off-by Kanchana P. Sridhar <[email protected]>

---
 mm/zswap.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

mm/zswap.c +0 -0

index 8ae67fd10522..ade84ac4076e 100644
@@ -160,11 +160,11 @@ struct crypto_acomp_ctx {
 struct zswap_pool {

mm/zswap.c +1 -1

 	char tfm_name[CRYPTO_MAX_ALG_NAME];
 };
 

@@ -202,10 +202,10 @@ static struct shrinker *zswap_shrinker;
  *              logic if referenced is unset. See comments in the shrinker
  *              section for context.
  * nid - NUMA node id of the page for which this is the zswap entry.
- * pool_idx - id of the zswap_pool that the entry's data is in.
  * handle - zsmalloc allocation handle that stores the compressed page data
  * objcg - the obj_cgroup that the compressed memory is charged to
  * lru - handle to the pool's lru used to evict pages.
+ * pool_idx - id of the zswap_pool that the entry's data is in.
  */
 struct zswap_entry {
 	swp_entry_t swpentry;

mm/zswap.c +0 -0

 };
 
 /*

Signature · 1 line

2.39.5

Quoted 21 lines · Kanchana P. Sridhar

Place and align the members of struct zswap_pool and struct zswap_entry such that members used together in compress/decompress hot paths are able to share the same cacheline. Minimize holes within these structs while doing so.

Signed-off-by Kanchana P. Sridhar <[email protected]>

---
 mm/zswap.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

mm/zswap.c +0 -2

index 8ae67fd10522..ade84ac4076e 100644
@@ -160,11 +160,11 @@ struct crypto_acomp_ctx {
 struct zswap_pool {
      struct zs_pool *zs_pool;
      struct crypto_acomp_ctx __percpu *acomp_ctx;
-     u8 compr_batch_size;
      struct percpu_ref ref;
-     struct rcu_work release_rwork;

mm/zswap.c struct crypto_acomp_ctx { · line 167

Quoted 5 lines

mm/zswap.c +3 -1

-     struct hlist_node node;
      u8 idx;
+     u8 compr_batch_size;
+     struct hlist_node node;
+     struct rcu_work release_rwork;

compr_batch_size is added by this series, why not just add it before idx to begin with?

Quoted 19 lines

mm/zswap.c +1 -1

      char tfm_name[CRYPTO_MAX_ALG_NAME];
 };
 

@@ -202,10 +202,10 @@ static struct shrinker *zswap_shrinker;
  *              logic if referenced is unset. See comments in the shrinker
  *              section for context.
  * nid - NUMA node id of the page for which this is the zswap entry.
- * pool_idx - id of the zswap_pool that the entry's data is in.
  * handle - zsmalloc allocation handle that stores the compressed page data
  * objcg - the obj_cgroup that the compressed memory is charged to
  * lru - handle to the pool's lru used to evict pages.
+ * pool_idx - id of the zswap_pool that the entry's data is in.
  */
 struct zswap_entry {
      swp_entry_t swpentry;

@@ -214,10 +214,10 @@ struct zswap_entry {
              bool referenced:1;
      };

mm/zswap.c struct zswap_entry { · line 220

Quoted 6 lines

mm/zswap.c +1 -1

      int nid;
-     u8 pool_idx;
      unsigned long handle;
      struct obj_cgroup *objcg;
      struct list_head lru;
+     u8 pool_idx;

As mentioned previously, this needs to be packed with the length.

Quoted 7 lines

mm/zswap.c +0 -1

 };
 
 /*
--

2.39.5

Quoted 23 lines · Yosry Ahmed

Quoted 21 lines · Kanchana P. Sridhar

Place and align the members of struct zswap_pool and struct zswap_entry such that members used together in compress/decompress hot paths are able to share the same cacheline. Minimize holes within these structs while doing so.

Signed-off-by Kanchana P. Sridhar <[email protected]>

---
 mm/zswap.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

mm/zswap.c +0 -2

index 8ae67fd10522..ade84ac4076e 100644
@@ -160,11 +160,11 @@ struct crypto_acomp_ctx {
 struct zswap_pool {
      struct zs_pool *zs_pool;
      struct crypto_acomp_ctx __percpu *acomp_ctx;
-     u8 compr_batch_size;
      struct percpu_ref ref;
-     struct rcu_work release_rwork;

Quoted 8 lines

mm/zswap.c struct crypto_acomp_ctx { · line 167

Quoted 5 lines

mm/zswap.c +3 -1

-     struct hlist_node node;
      u8 idx;
+     u8 compr_batch_size;
+     struct hlist_node node;
+     struct rcu_work release_rwork;

compr_batch_size is added by this series, why not just add it before idx to begin with?

In patch 0006, where compr_batch_size is introduced, I was attempting to place it in the same cacheline as the acomp_ctx. This increased the size of struct zswap_pool to 248 bytes. I later optimized this in patch 0007, bringing back the size to 240 with the addition of compr_batch_size.

Quoted 34 lines · Kanchana P. Sridhar

Quoted 23 lines · Yosry Ahmed

Quoted 21 lines · Kanchana P. Sridhar

Place and align the members of struct zswap_pool and struct zswap_entry such that members used together in compress/decompress hot paths are able to share the same cacheline. Minimize holes within these structs while doing so.

Signed-off-by Kanchana P. Sridhar <[email protected]>

---
 mm/zswap.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

mm/zswap.c +0 -2

index 8ae67fd10522..ade84ac4076e 100644
@@ -160,11 +160,11 @@ struct crypto_acomp_ctx {
 struct zswap_pool {
      struct zs_pool *zs_pool;
      struct crypto_acomp_ctx __percpu *acomp_ctx;
-     u8 compr_batch_size;
      struct percpu_ref ref;
-     struct rcu_work release_rwork;

Quoted 8 lines

mm/zswap.c struct crypto_acomp_ctx { · line 167

Quoted 5 lines

mm/zswap.c +3 -1

-     struct hlist_node node;
      u8 idx;
+     u8 compr_batch_size;
+     struct hlist_node node;
+     struct rcu_work release_rwork;

compr_batch_size is added by this series, why not just add it before idx to begin with?

Quoted 5 lines

In patch 0006, where compr_batch_size is introduced, I was attempting to place it in the same cacheline as the acomp_ctx. This increased the size of struct zswap_pool to 248 bytes. I later optimized this in patch 0007, bringing back the size to 240 with the addition of compr_batch_size.

This doesn't make much sense, and the size of zswap_pool is not that important (usually only one globally). I would drop the patch and add compr_batch_size directly where it needs to end up.

v1: Batched zswap_store() with compression batching
===================================================

This series is a spin-off of "v14: zswap compression batching with optimized iaa_crypto driver" [1], and focuses on the generic crypto_acomp and zswap compression batching enablers. Resuming work on this series following a gap since v14 (January 2026). Please note that my email address and affiliation have changed from [email protected] to [email protected]. I have retained the original Intel sign-offs on prior patches for historical attribution while signing off v1 of this series with my current address.

To summarize the main contribution of this patch series: Batching is introduced in zswap_store() of large folios: we carve out a large folio into batches of ZSWAP_MAX_BATCH_SIZE (8 pages) for software compressors, and store each batch. Moving the fundamental operations that need to be happen to store a page to now work at the granularity of a batch offers opportunities to optimize them from these perspectives:

1) Utilize existing kernel API that minimize obtaining locks for bulk operations, e.g. kmem_cache_alloc_bulk()/kmem_cache_free_bulk().

2) We keep the working set for each of the core store functionalities (alloc entries, compress pages, store pages in zsmalloc) to be focused on a batch, thereby minimizing cache evictions and benefiting from cache locality.

3) Along the lines of (1), the main zswap_compress() batching routine obtains the acomp_ctx mutex lock once per batch, and releases it after it has processed all pages in that batch.

4) Any zswap store functionality that works on a batch can potentially be parallelized as a next step, to further improve performance.

Performance data: usemem with 60% over-commit:
==============================================

Validation was done with mm-unstable as of 10-3-2026 (Baseline) and with this patch series.

Testing was done on a Chromebook with a QEMU "vng" VM running in Crostini.

Chromebook Diagnostics reports:
Intel(R) Core(TM) Ultra 5 115U (10 CPUs, 4.200GHz)
RAM: 16GB

vng setup:

         Memory: 7G
           Swap: 3G
    Pinned CPUs: 4 (E-Cores: 4,5,6,7)

zswap compressor:
ZSTD

mTHP enabled:
64kB

vm-scalability/usemem setup:

cgroup memory.high: 3G
4 usemem processes, each allocates 1200M anonymous memory, reads the Silesia dataset [2] from a pre-fork-allocated buffer into the 1200M, writes to each unsigned long in the 1200M, sleeps for 10 sec, then frees the memory and exits.

    usemem --read-silesia -w -O -b 1 -s 10 -n 4 1200M

usemem metrics averaged across 10 iterations*:
================================================

--------------------------------------------------------------------------
Baseline Patch Change
mm-unstable series with
10-3-2026 v1 v1
--------------------------------------------------------------------------

 Average Throughput (KB/s)                  11,794      17,810    +51.00%
 Total Throughput (KB/s)                    47,179      71,242    +51.00%
 Average Time to Free Memory (usecs)       490,973     294,920    -39.93%
 Total Time to Free Memory (usecs)       1,963,892   1,179,682    -39.93%

--------------------------------------------------------------------------
* vmstats are as reported at the end of the 10th iteration:

 64kB-alloc                                834,567     828,199     -0.76%
 64kB-zswpout                              700,138     743,697      6.22%

64kB-swpout-fallback 35,342 17,978 -49.13%
64kB-swpout 0 0
--------------------------------------------------------------------------

The speedup with zswap store batching of large folios has a propelling effect on other parts of mm, especially, by enabling large folios to be swapped out without splitting, as the data shows. This enables other mm components that are optimized for large folios, such as munmap(), folio refcounts/mapcounts updates, PTE invalidation upon freeing memory, zswap_invalidate(); which benefit from xarray offsets/range continuity. As a result, subsequent large folio swapouts are more likely to find contiguous swap slots, there are fewer large folio splits during reclaim, more mTHP zswpouts and the cycle continues. As the data demonstrates, this directly contributes to more memory over-commit opportunity.

Detailed usemem metrics/cgroup memory utilization/vmstats:
==========================================================

A) MAX zswap pool_total_size vmstats:
=====================================
This is a snapshot at the 10th iteration's peak zswap pool usage:

--------------------------------------------------------------------------
Baseline Patch Change
mm-unstable series with
10-3-2026 v1 v1
--------------------------------------------------------------------------

 pswpin                                    0                 0
 pswpout                                   0                 0
 thp_swpout                                0                 0
 thp_swpout_fallback                       0                 0
 swpin_zero                              594               548
 swpout_zero                           1,062               864
 zswpin                           10,603,882        11,070,673
 zswpout                          16,594,356        16,509,201     -0.51%
 zswpwb                                    0                 0           
 nrswpin                                   0                 0           
 nrswpout                                  0                 0           
 pgmajfault                        1,460,360         1,470,092      0.67%
 nr_free_pages                       897,249           908,935      1.30%
 nr_free_pages_blocks                892,928           903,680      1.20%
 64kB-alloc                          831,788           825,556     -0.75%
 64kB-zswpout                        690,961           735,863      6.50%
 64kB-swpout                               0                 0           
 64kB-swpout-fallback                 32,982            16,545    -49.84%
 64kB-swpin                                0                 0           
 zswap_pool_total_size         1,420,120,064     1,377,210,368     -3.02%
 memory.current                3,390,189,568     3,299,569,664     -2.67%
 memory.swap.current           3,221,221,376     3,221,221,376

--------------------------------------------------------------------------

B) STEADY STATE vmstats after usemem finishes:
==============================================
This is a snapshot after the 10th iteration finishes:

--------------------------------------------------------------------------
Baseline Patch Change
mm-unstable series with
10-3-2026 v1 v1
--------------------------------------------------------------------------

 pswpin                                    0                 0 
 pswpout                                   0                 0
 thp_swpout                                0                 0
 thp_swpout_fallback                       0                 0
 swpin_zero                              594               548
 swpout_zero                           1,062               949
 zswpin                           11,526,695        11,963,729
 zswpout                          17,293,120        17,219,911     -0.42%
 zswpwb                                    0                 0           
 nrswpin                                   0                 0           
 nrswpout                                  0                 0           
 pgmajfault                        1,591,298         1,596,095      0.30%
 nr_free_pages                     1,752,789         1,753,264      0.03%
 nr_free_pages_blocks              1,258,496         1,240,064     -1.46%
 64kB-alloc                          834,567           828,199     -0.76%
 64kB-zswpout                        700,138           743,697      6.22%
 64kB-swpout                               0                 0           
 64kB-swpout-fallback                 35,342            17,978    -49.13%
 64kB-swpin                                0                 0           
 zswap_pool_total_size                     0                 0           

memory.current 757,760 802,816 5.95%
memory.swap.current 0 0
--------------------------------------------------------------------------
zswap debugfs counters:

  pool_limit_hit                           0                 0
  written_back_pages                       0                 0
  reject_alloc_fail                        0                 0
  reject_kmemcache_fail                    0                 0
  reject_reclaim_fail                      0                 0
  reject_compress_fail                     0                 0
  reject_compress_poor                     0                 0
  decompress_fail                          0                 0
  stored_incompressible_pages              0                 0
  stored_pages                             0                 0

--------------------------------------------------------------------------

[1]: https://patchwork.kernel.org/project/linux-mm/cover/[email protected]/
[2]: https://wanos.co/assets/silesia.tar

Changes since [1]:
==================
1) Better cacheline placement of struct zswap_pool and struct zswap_entry members for temporal memory access efficiency in zswap store/load hot paths, while minimizing holes.

2) Refactored the core compression batching functionality to a separate __zswap_compress() function, which is called by zswap_compress(), as per Nhat's and Yosry's comments. Hopefully this helps explain the outer "store batching" zswap_compress() vis-a-vis the inner "compress batching" __zswap_compress() distinction.

3) The acomp_ctx is locked once and is passed to the routine that compresses a batch, per the algorithm's batch-size. Similarly, the folio's node id and writeback_enabled status are computed once and passed to the routines that implement the different store functionalities, to save latency by avoid localized recomputes of information that doesn't change.

4) Moved the folio node id being stored in zswap_entry to a separate patch, per Yosry's comment.

5) Addressed the kunmap_local() issue flagged by Jixin He in the IS_ERR_VALUE(handle) error path
(https://patchwork.kernel.org/comment/27095925) - thanks!

6) Moved comments inline, per Yosry's suggestions.

Thanks,
Kanchana

Kanchana P. Sridhar (8):
crypto: acomp - Define a unit_size in struct acomp_req to enable
batching. mm: zswap: Set the unit size for zswap to PAGE_SIZE.
crypto: acomp - Add bit to indicate segmentation support.
crypto: acomp - Add trivial segmentation wrapper.
crypto: acomp - Add API to get a compression algorithm's batch-size.
mm: zswap: Store the folio's node id in the zswap_entry.
mm: zswap: Store large folios in batches.
mm: zswap: Optimally place members of struct zswap_pool, struct
zswap_entry.
mm: zswap: Batched zswap_compress() for storing large folios in
batches.

 crypto/acompress.c                  |  47 ++-
 include/crypto/acompress.h          |  62 +++
 include/crypto/algapi.h             |   5 +
 include/crypto/internal/acompress.h |   8 +
 include/linux/crypto.h              |   3 +
 mm/zswap.c                          | 571 ++++++++++++++++++++--------
 6 files changed, 528 insertions(+), 168 deletions(-)

Signature · 1 line

2.39.5

Quoted 30 lines · Kanchana P. Sridhar

v1: Batched zswap_store() with compression batching
===================================================

This series is a spin-off of "v14: zswap compression batching with optimized iaa_crypto driver" [1], and focuses on the generic crypto_acomp and zswap compression batching enablers. Resuming work on this series following a gap since v14 (January 2026). Please note that my email address and affiliation have changed from [email protected] to [email protected]. I have retained the original Intel sign-offs on prior patches for historical attribution while signing off v1 of this series with my current address.

To summarize the main contribution of this patch series: Batching is introduced in zswap_store() of large folios: we carve out a large folio into batches of ZSWAP_MAX_BATCH_SIZE (8 pages) for software compressors, and store each batch. Moving the fundamental operations that need to be happen to store a page to now work at the granularity of a batch offers opportunities to optimize them from these perspectives:

1) Utilize existing kernel API that minimize obtaining locks for bulk operations, e.g. kmem_cache_alloc_bulk()/kmem_cache_free_bulk().

2) We keep the working set for each of the core store functionalities (alloc entries, compress pages, store pages in zsmalloc) to be focused on a batch, thereby minimizing cache evictions and benefiting from cache locality.

3) Along the lines of (1), the main zswap_compress() batching routine obtains the acomp_ctx mutex lock once per batch, and releases it after it has processed all pages in that batch.

Quoted 5 lines

4) Any zswap store functionality that works on a batch can potentially be parallelized as a next step, to further improve performance.

Performance data: usemem with 60% over-commit:
==============================================

How much of the performance benefits come from patch 6 vs patch 8?

Quoted 205 lines

Validation was done with mm-unstable as of 10-3-2026 (Baseline) and with this patch series.

Testing was done on a Chromebook with a QEMU "vng" VM running in Crostini.

Chromebook Diagnostics reports:
Intel(R) Core(TM) Ultra 5 115U (10 CPUs, 4.200GHz)
RAM: 16GB

vng setup:

         Memory: 7G
           Swap: 3G
    Pinned CPUs: 4 (E-Cores: 4,5,6,7)

zswap compressor:
ZSTD

mTHP enabled:
64kB

vm-scalability/usemem setup:

cgroup memory.high: 3G
4 usemem processes, each allocates 1200M anonymous memory, reads the Silesia dataset [2] from a pre-fork-allocated buffer into the 1200M, writes to each unsigned long in the 1200M, sleeps for 10 sec, then frees the memory and exits.

    usemem --read-silesia -w -O -b 1 -s 10 -n 4 1200M

usemem metrics averaged across 10 iterations*:
================================================

--------------------------------------------------------------------------
Baseline Patch Change
mm-unstable series with
10-3-2026 v1 v1
--------------------------------------------------------------------------

 Average Throughput (KB/s)                  11,794      17,810    +51.00%
 Total Throughput (KB/s)                    47,179      71,242    +51.00%
 Average Time to Free Memory (usecs)       490,973     294,920    -39.93%
 Total Time to Free Memory (usecs)       1,963,892   1,179,682    -39.93%

--------------------------------------------------------------------------
* vmstats are as reported at the end of the 10th iteration:

 64kB-alloc                                834,567     828,199     -0.76%
 64kB-zswpout                              700,138     743,697      6.22%

64kB-swpout-fallback 35,342 17,978 -49.13%
64kB-swpout 0 0
--------------------------------------------------------------------------

The speedup with zswap store batching of large folios has a propelling effect on other parts of mm, especially, by enabling large folios to be swapped out without splitting, as the data shows. This enables other mm components that are optimized for large folios, such as munmap(), folio refcounts/mapcounts updates, PTE invalidation upon freeing memory, zswap_invalidate(); which benefit from xarray offsets/range continuity. As a result, subsequent large folio swapouts are more likely to find contiguous swap slots, there are fewer large folio splits during reclaim, more mTHP zswpouts and the cycle continues. As the data demonstrates, this directly contributes to more memory over-commit opportunity.

Detailed usemem metrics/cgroup memory utilization/vmstats:
==========================================================

A) MAX zswap pool_total_size vmstats:
=====================================
This is a snapshot at the 10th iteration's peak zswap pool usage:

--------------------------------------------------------------------------
Baseline Patch Change
mm-unstable series with
10-3-2026 v1 v1
--------------------------------------------------------------------------

 pswpin                                    0                 0
 pswpout                                   0                 0
 thp_swpout                                0                 0
 thp_swpout_fallback                       0                 0
 swpin_zero                              594               548
 swpout_zero                           1,062               864
 zswpin                           10,603,882        11,070,673
 zswpout                          16,594,356        16,509,201     -0.51%
 zswpwb                                    0                 0
 nrswpin                                   0                 0
 nrswpout                                  0                 0
 pgmajfault                        1,460,360         1,470,092      0.67%
 nr_free_pages                       897,249           908,935      1.30%
 nr_free_pages_blocks                892,928           903,680      1.20%
 64kB-alloc                          831,788           825,556     -0.75%
 64kB-zswpout                        690,961           735,863      6.50%
 64kB-swpout                               0                 0
 64kB-swpout-fallback                 32,982            16,545    -49.84%
 64kB-swpin                                0                 0
 zswap_pool_total_size         1,420,120,064     1,377,210,368     -3.02%
 memory.current                3,390,189,568     3,299,569,664     -2.67%
 memory.swap.current           3,221,221,376     3,221,221,376

--------------------------------------------------------------------------

B) STEADY STATE vmstats after usemem finishes:
==============================================
This is a snapshot after the 10th iteration finishes:

--------------------------------------------------------------------------
Baseline Patch Change
mm-unstable series with
10-3-2026 v1 v1
--------------------------------------------------------------------------

 pswpin                                    0                 0
 pswpout                                   0                 0
 thp_swpout                                0                 0
 thp_swpout_fallback                       0                 0
 swpin_zero                              594               548
 swpout_zero                           1,062               949
 zswpin                           11,526,695        11,963,729
 zswpout                          17,293,120        17,219,911     -0.42%
 zswpwb                                    0                 0
 nrswpin                                   0                 0
 nrswpout                                  0                 0
 pgmajfault                        1,591,298         1,596,095      0.30%
 nr_free_pages                     1,752,789         1,753,264      0.03%
 nr_free_pages_blocks              1,258,496         1,240,064     -1.46%
 64kB-alloc                          834,567           828,199     -0.76%
 64kB-zswpout                        700,138           743,697      6.22%
 64kB-swpout                               0                 0
 64kB-swpout-fallback                 35,342            17,978    -49.13%
 64kB-swpin                                0                 0
 zswap_pool_total_size                     0                 0

memory.current 757,760 802,816 5.95%
memory.swap.current 0 0
--------------------------------------------------------------------------
zswap debugfs counters:

  pool_limit_hit                           0                 0
  written_back_pages                       0                 0
  reject_alloc_fail                        0                 0
  reject_kmemcache_fail                    0                 0
  reject_reclaim_fail                      0                 0
  reject_compress_fail                     0                 0
  reject_compress_poor                     0                 0
  decompress_fail                          0                 0
  stored_incompressible_pages              0                 0
  stored_pages                             0                 0

--------------------------------------------------------------------------

[1]: https://patchwork.kernel.org/project/linux-mm/cover/[email protected]/
[2]: https://wanos.co/assets/silesia.tar

Changes since [1]:
==================
1) Better cacheline placement of struct zswap_pool and struct zswap_entry members for temporal memory access efficiency in zswap store/load hot paths, while minimizing holes.

2) Refactored the core compression batching functionality to a separate __zswap_compress() function, which is called by zswap_compress(), as per Nhat's and Yosry's comments. Hopefully this helps explain the outer "store batching" zswap_compress() vis-a-vis the inner "compress batching" __zswap_compress() distinction.

3) The acomp_ctx is locked once and is passed to the routine that compresses a batch, per the algorithm's batch-size. Similarly, the folio's node id and writeback_enabled status are computed once and passed to the routines that implement the different store functionalities, to save latency by avoid localized recomputes of information that doesn't change.

4) Moved the folio node id being stored in zswap_entry to a separate patch, per Yosry's comment.

5) Addressed the kunmap_local() issue flagged by Jixin He in the IS_ERR_VALUE(handle) error path
(https://patchwork.kernel.org/comment/27095925) - thanks!

6) Moved comments inline, per Yosry's suggestions.

Thanks,
Kanchana

Kanchana P. Sridhar (8):
crypto: acomp - Define a unit_size in struct acomp_req to enable
batching. mm: zswap: Set the unit size for zswap to PAGE_SIZE.
crypto: acomp - Add bit to indicate segmentation support.
crypto: acomp - Add trivial segmentation wrapper.
crypto: acomp - Add API to get a compression algorithm's batch-size.
mm: zswap: Store the folio's node id in the zswap_entry.
mm: zswap: Store large folios in batches.
mm: zswap: Optimally place members of struct zswap_pool, struct
zswap_entry.
mm: zswap: Batched zswap_compress() for storing large folios in
batches.

 crypto/acompress.c                  |  47 ++-
 include/crypto/acompress.h          |  62 +++
 include/crypto/algapi.h             |   5 +
 include/crypto/internal/acompress.h |   8 +
 include/linux/crypto.h              |   3 +
 mm/zswap.c                          | 571 ++++++++++++++++++++--------
 6 files changed, 528 insertions(+), 168 deletions(-)

--
2.39.5

Quoted 32 lines · Yosry Ahmed

Quoted 30 lines · Kanchana P. Sridhar

v1: Batched zswap_store() with compression batching
===================================================

This series is a spin-off of "v14: zswap compression batching with optimized iaa_crypto driver" [1], and focuses on the generic crypto_acomp and zswap compression batching enablers. Resuming work on this series following a gap since v14 (January 2026). Please note that my email address and affiliation have changed from [email protected] to [email protected]. I have retained the original Intel sign-offs on prior patches for historical attribution while signing off v1 of this series with my current address.

To summarize the main contribution of this patch series: Batching is introduced in zswap_store() of large folios: we carve out a large folio into batches of ZSWAP_MAX_BATCH_SIZE (8 pages) for software compressors, and store each batch. Moving the fundamental operations that need to be happen to store a page to now work at the granularity of a batch offers opportunities to optimize them from these perspectives:

1) Utilize existing kernel API that minimize obtaining locks for bulk operations, e.g. kmem_cache_alloc_bulk()/kmem_cache_free_bulk().

2) We keep the working set for each of the core store functionalities (alloc entries, compress pages, store pages in zsmalloc) to be focused on a batch, thereby minimizing cache evictions and benefiting from cache locality.

3) Along the lines of (1), the main zswap_compress() batching routine obtains the acomp_ctx mutex lock once per batch, and releases it after it has processed all pages in that batch.

Quoted 7 lines

Quoted 5 lines

4) Any zswap store functionality that works on a batch can potentially be parallelized as a next step, to further improve performance.

Performance data: usemem with 60% over-commit:
==============================================

How much of the performance benefits come from patch 6 vs patch 8?

This is how the performance changes: Average throughput is averaged from 10 iterations of usemem.

patch 6: 19,119 KB/s
patch 8: 17,593 KB/s

So the acomp_do_req_chain() path that ZSTD goes through with patch 8 causes a regression, even though it provides a generic batching framework in acomp.

I was able to retrieve performance with the following change in acompress.c:

patch 3 with this change in acompress.c: 19,578 KB/s

crypto/acompress.c +6 -2

index 192ea996e779..dc609bd7d927 100644
@@ -293,8 +293,12 @@ int crypto_acomp_compress(struct acomp_req *req)
                return -EAGAIN;
 
        if (likely(req->unit_size && acomp_request_issg(req))) {
-               if (!crypto_acomp_req_seg(tfm))
-                       return acomp_do_req_chain(req, true);
+               if (!crypto_acomp_req_seg(tfm)) {
+                       int err = tfm->compress(req);
+
+                       req->dst->length = unlikely(err) ? err : req->dlen;
+                       return err;
+               }
 
                return tfm->compress(req);
        }

Herbert, would you be Ok with this change in patch 3?

Quoted 48 lines · Kanchana P. Sridhar

Quoted 32 lines · Yosry Ahmed

Quoted 30 lines · Kanchana P. Sridhar

v1: Batched zswap_store() with compression batching
===================================================

This series is a spin-off of "v14: zswap compression batching with optimized iaa_crypto driver" [1], and focuses on the generic crypto_acomp and zswap compression batching enablers. Resuming work on this series following a gap since v14 (January 2026). Please note that my email address and affiliation have changed from [email protected] to [email protected]. I have retained the original Intel sign-offs on prior patches for historical attribution while signing off v1 of this series with my current address.

To summarize the main contribution of this patch series: Batching is introduced in zswap_store() of large folios: we carve out a large folio into batches of ZSWAP_MAX_BATCH_SIZE (8 pages) for software compressors, and store each batch. Moving the fundamental operations that need to be happen to store a page to now work at the granularity of a batch offers opportunities to optimize them from these perspectives:

1) Utilize existing kernel API that minimize obtaining locks for bulk operations, e.g. kmem_cache_alloc_bulk()/kmem_cache_free_bulk().

2) We keep the working set for each of the core store functionalities (alloc entries, compress pages, store pages in zsmalloc) to be focused on a batch, thereby minimizing cache evictions and benefiting from cache locality.

3) Along the lines of (1), the main zswap_compress() batching routine obtains the acomp_ctx mutex lock once per batch, and releases it after it has processed all pages in that batch.

Quoted 7 lines

Quoted 5 lines

4) Any zswap store functionality that works on a batch can potentially be parallelized as a next step, to further improve performance.

Performance data: usemem with 60% over-commit:
==============================================

How much of the performance benefits come from patch 6 vs patch 8?

This is how the performance changes: Average throughput is averaged from 10 iterations of usemem.

patch 6: 19,119 KB/s
patch 8: 17,593 KB/s

Quoted 5 lines

So the acomp_do_req_chain() path that ZSTD goes through with patch 8 causes a regression, even though it provides a generic batching framework in acomp.

I was able to retrieve performance with the following change in acompress.c:

Why add it to begin with in this case?

Quoted 25 lines

patch 3 with this change in acompress.c: 19,578 KB/s

crypto/acompress.c +6 -2

index 192ea996e779..dc609bd7d927 100644
@@ -293,8 +293,12 @@ int crypto_acomp_compress(struct acomp_req *req)
                return -EAGAIN;
 
        if (likely(req->unit_size && acomp_request_issg(req))) {
-               if (!crypto_acomp_req_seg(tfm))
-                       return acomp_do_req_chain(req, true);
+               if (!crypto_acomp_req_seg(tfm)) {
+                       int err = tfm->compress(req);
+
+                       req->dst->length = unlikely(err) ? err : req->dlen;
+                       return err;
+               }
 
                return tfm->compress(req);
        }

Herbert, would you be Ok with this change in patch 3?

Quoted 57 lines · Yosry Ahmed

Quoted 53 lines · Kanchana P. Sridhar

Quoted 32 lines · Yosry Ahmed

Quoted 30 lines · Kanchana P. Sridhar

v1: Batched zswap_store() with compression batching
===================================================

This series is a spin-off of "v14: zswap compression batching with optimized iaa_crypto driver" [1], and focuses on the generic crypto_acomp and zswap compression batching enablers. Resuming work on this series following a gap since v14 (January 2026). Please note that my email address and affiliation have changed from [email protected] to [email protected]. I have retained the original Intel sign-offs on prior patches for historical attribution while signing off v1 of this series with my current address.

To summarize the main contribution of this patch series: Batching is introduced in zswap_store() of large folios: we carve out a large folio into batches of ZSWAP_MAX_BATCH_SIZE (8 pages) for software compressors, and store each batch. Moving the fundamental operations that need to be happen to store a page to now work at the granularity of a batch offers opportunities to optimize them from these perspectives:

1) Utilize existing kernel API that minimize obtaining locks for bulk operations, e.g. kmem_cache_alloc_bulk()/kmem_cache_free_bulk().

2) We keep the working set for each of the core store functionalities (alloc entries, compress pages, store pages in zsmalloc) to be focused on a batch, thereby minimizing cache evictions and benefiting from cache locality.

3) Along the lines of (1), the main zswap_compress() batching routine obtains the acomp_ctx mutex lock once per batch, and releases it after it has processed all pages in that batch.

Quoted 7 lines

Quoted 5 lines

4) Any zswap store functionality that works on a batch can potentially be parallelized as a next step, to further improve performance.

Performance data: usemem with 60% over-commit:
==============================================

How much of the performance benefits come from patch 6 vs patch 8?

This is how the performance changes: Average throughput is averaged from 10 iterations of usemem.

patch 6: 19,119 KB/s
patch 8: 17,593 KB/s

So the acomp_do_req_chain() path that ZSTD goes through with patch 8 causes a regression, even though it provides a generic batching framework in acomp.

I was able to retrieve performance with the following change in acompress.c:

Quoted 1 line

Why add it to begin with in this case?

Patch 3 ("crypto: acomp - Add trivial segmentation wrapper.") was co-developed with Herbert. The request chaining for non-segmentation algorithms was as in Herbert's original implementation. Since this causes a regression, I would like to get Herbert's Ok on replacing the request chaining in this part of crypto_acomp_compress().

Quoted 59 lines · Kanchana P. Sridhar

Quoted 57 lines · Yosry Ahmed

Quoted 53 lines · Kanchana P. Sridhar

Quoted 32 lines · Yosry Ahmed

Quoted 30 lines · Kanchana P. Sridhar

v1: Batched zswap_store() with compression batching
===================================================

This series is a spin-off of "v14: zswap compression batching with optimized iaa_crypto driver" [1], and focuses on the generic crypto_acomp and zswap compression batching enablers. Resuming work on this series following a gap since v14 (January 2026). Please note that my email address and affiliation have changed from [email protected] to [email protected]. I have retained the original Intel sign-offs on prior patches for historical attribution while signing off v1 of this series with my current address.

To summarize the main contribution of this patch series: Batching is introduced in zswap_store() of large folios: we carve out a large folio into batches of ZSWAP_MAX_BATCH_SIZE (8 pages) for software compressors, and store each batch. Moving the fundamental operations that need to be happen to store a page to now work at the granularity of a batch offers opportunities to optimize them from these perspectives:

1) Utilize existing kernel API that minimize obtaining locks for bulk operations, e.g. kmem_cache_alloc_bulk()/kmem_cache_free_bulk().

2) We keep the working set for each of the core store functionalities (alloc entries, compress pages, store pages in zsmalloc) to be focused on a batch, thereby minimizing cache evictions and benefiting from cache locality.

3) Along the lines of (1), the main zswap_compress() batching routine obtains the acomp_ctx mutex lock once per batch, and releases it after it has processed all pages in that batch.

Quoted 7 lines

Quoted 5 lines

4) Any zswap store functionality that works on a batch can potentially be parallelized as a next step, to further improve performance.

Performance data: usemem with 60% over-commit:
==============================================

How much of the performance benefits come from patch 6 vs patch 8?

This is how the performance changes: Average throughput is averaged from 10 iterations of usemem.

patch 6: 19,119 KB/s
patch 8: 17,593 KB/s

So the acomp_do_req_chain() path that ZSTD goes through with patch 8 causes a regression, even though it provides a generic batching framework in acomp.

I was able to retrieve performance with the following change in acompress.c:

Quoted 7 lines

Quoted 1 line

Why add it to begin with in this case?

Patch 3 ("crypto: acomp - Add trivial segmentation wrapper.") was co-developed with Herbert. The request chaining for non-segmentation algorithms was as in Herbert's original implementation. Since this causes a regression, I would like to get Herbert's Ok on replacing the request chaining in this part of crypto_acomp_compress().

I mean, why add patch 8 if it doesn't improve performance? Seems like even with your diff, we only recover performance, but we don't improve it after patch 6?

Quoted 22 lines

Quoted 21 lines

Quoted 8 lines

Quoted 8 lines

Quoted 1 line

How much of the performance benefits come from patch 6 vs patch 8?

This is how the performance changes: Average throughput is averaged from 10 iterations of usemem.

patch 6: 19,119 KB/s
patch 8: 17,593 KB/s

Quoted 7 lines

Quoted 5 lines

So the acomp_do_req_chain() path that ZSTD goes through with patch 8 causes a regression, even though it provides a generic batching framework in acomp.

I was able to retrieve performance with the following change in acompress.c:

Why add it to begin with in this case?

Patch 3 ("crypto: acomp - Add trivial segmentation wrapper.") was co-developed with Herbert. The request chaining for non-segmentation algorithms was as in Herbert's original implementation. Since this causes a regression, I would like to get Herbert's Ok on replacing the request chaining in this part of crypto_acomp_compress().

Quoted 3 lines

I mean, why add patch 8 if it doesn't improve performance? Seems like even with your diff, we only recover performance, but we don't improve it after patch 6?

Patch 8 does improve performance over patch 6 (19,578 KB/s vs. 19,119 KB/s) by 2.4%. Besides, patch 8 provides a compress batching framework in zswap, which is the purpose of this patch series.

zswap is the
first user of the crypto_acomp compress batching API provided earlier in this series, which I think, is a great starting point for us to evolve this further, as mentioned in the cover letter.

Quoted 23 lines

Quoted 23 lines

Quoted 22 lines

Quoted 9 lines

Quoted 9 lines

Quoted 2 lines

How much of the performance benefits come from patch 6 vs patch 8?

This is how the performance changes: Average throughput is averaged from 10 iterations of usemem.

patch 6: 19,119 KB/s
patch 8: 17,593 KB/s

Quoted 7 lines

Quoted 5 lines

So the acomp_do_req_chain() path that ZSTD goes through with patch 8 causes a regression, even though it provides a generic batching framework in acomp.

I was able to retrieve performance with the following change in acompress.c:

Why add it to begin with in this case?

Patch 3 ("crypto: acomp - Add trivial segmentation wrapper.") was co-developed with Herbert. The request chaining for non-segmentation algorithms was as in Herbert's original implementation. Since this causes a regression, I would like to get Herbert's Ok on replacing the request chaining in this part of crypto_acomp_compress().

Quoted 7 lines

Quoted 3 lines

I mean, why add patch 8 if it doesn't improve performance? Seems like even with your diff, we only recover performance, but we don't improve it after patch 6?

Patch 8 does improve performance over patch 6 (19,578 KB/s vs. 19,119 KB/s) by 2.4%. Besides, patch 8 provides a compress batching framework in zswap, which is the purpose of this patch series.

No, the purpose of this patch series is to improve performance, be it through the compression batching framework or otherwise. The compression batching framework made sense as it provided greater performance gains with HW batching/concurrency. Without that, I am honestly skeptical. 2.4% can be within the noise.

Quoted 5 lines

zswap is the
first user of the crypto_acomp compress batching API provided earlier in this series, which I think, is a great starting point for us to evolve this further, as mentioned in the cover letter.

Without a non-trivial performance benefit (e.g. with HW batching), I don't see enough justification. My advice is to try to push 6 (and its dependencies) separately.

If you are Ok with all this, we will need Herbert's Ok for this change in patch 3:

crypto/acompress.c +0 -0

index 192ea996e779..db8e4c82b900 100644
@@ -293,8 +293,12 @@ int crypto_acomp_compress(struct acomp_req *req)
                return -EAGAIN;

Quoted 11 lines · Kanchana P. Sridhar

If you are Ok with all this, we will need Herbert's Ok for this change in patch 3:

crypto/acompress.c +0 -0

index 192ea996e779..db8e4c82b900 100644
@@ -293,8 +293,12 @@ int crypto_acomp_compress(struct acomp_req *req)
                return -EAGAIN;
 

crypto/acompress.c int crypto_acomp_compress(struct acomp_req *req) · line 301

Quoted 9 lines

crypto/acompress.c +6 -2

        if (likely(req->unit_size && acomp_request_issg(req))) {
-               if (!crypto_acomp_req_seg(tfm))
-                       return acomp_do_req_chain(req, true);
+               if (likely(!crypto_acomp_req_seg(tfm))) {
+                       int err = tfm->compress(req);
+
+                       req->dst->length = unlikely(err) ? err : req->dlen;
+                       return err;
+               }

What if you put this short-circuit into acomp_do_req_chain instead, does it still give you the same performance boost?

Cheers,
--
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Quoted 11 lines · Kanchana P. Sridhar

If you are Ok with all this, we will need Herbert's Ok for this change in patch 3:

crypto/acompress.c +0 -0

index 192ea996e779..db8e4c82b900 100644
@@ -293,8 +293,12 @@ int crypto_acomp_compress(struct acomp_req *req)
                return -EAGAIN;
 

crypto/acompress.c int crypto_acomp_compress(struct acomp_req *req) · line 301

Quoted 9 lines

crypto/acompress.c +6 -2

        if (likely(req->unit_size && acomp_request_issg(req))) {
-               if (!crypto_acomp_req_seg(tfm))
-                       return acomp_do_req_chain(req, true);
+               if (likely(!crypto_acomp_req_seg(tfm))) {
+                       int err = tfm->compress(req);
+
+                       req->dst->length = unlikely(err) ? err : req->dlen;
+                       return err;
+               }

What if you put this short-circuit into acomp_do_req_chain instead, does it still give you the same performance boost?

Cheers,
--
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

The Average throughput of 10 iterations usemem is 19,058 KB/s. IOW, I don't get the same performance boost as making the localized change in crypto_acomp_compress(), likely because of added computes per compress call (one function call, two conditionals).

Quoted 5 lines · Kanchana P. Sridhar

The Average throughput of 10 iterations usemem is 19,058 KB/s. IOW, I don't get the same performance boost as making the localized change in crypto_acomp_compress(), likely because of added computes per compress call (one function call, two conditionals).

What are we comparing against?

The thing is that the conditional will have to be there at some point as otherwise this becomes pointless.

What if you just do the conditional and short-circuit in crypto_acomp_compress?

Thanks,
--
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Quoted 3 lines

What if you just do the conditional and short-circuit in crypto_acomp_compress?

Yes, this is the first option I tried (20,946 KB/s).

[1]:

int crypto_acomp_compress(struct acomp_req *req) {
struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);

if (unlikely(acomp_req_on_stack(req) && acomp_is_async(tfm))) return -EAGAIN;

if (likely(req->unit_size && acomp_request_issg(req))) { if (likely(!crypto_acomp_req_seg(tfm))) {
int err = tfm->compress(req);

req->dst->length = unlikely(err) ? err : req->dlen; return err;
}

return tfm->compress(req);
}

if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req)) return tfm->compress(req);

return acomp_do_req_chain(req, true);
}

Quoted 23 lines · Kanchana P. Sridhar

Quoted 3 lines

What if you just do the conditional and short-circuit in crypto_acomp_compress?

Yes, this is the first option I tried (20,946 KB/s).

[1]:

int crypto_acomp_compress(struct acomp_req *req) {
struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);

if (unlikely(acomp_req_on_stack(req) && acomp_is_async(tfm))) return -EAGAIN;

if (likely(req->unit_size && acomp_request_issg(req))) { if (likely(!crypto_acomp_req_seg(tfm))) {
int err = tfm->compress(req);

req->dst->length = unlikely(err) ? err : req->dlen; return err;
}

Quoted 8 lines

return tfm->compress(req);
}

if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req)) return tfm->compress(req);

return acomp_do_req_chain(req, true);
}

That isn't quite right though. We need to ensure that it really is a singleton before taking the short-cut. So you need at least one conditional:

        if (likely(req->unit_size == req->slen))
                take short-cut

Cheers,
--
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

On Fri, Oct 16, 2026 at 10:02:13PM -0700, Kanchana P. Sridhar wrote:
[ .. ]

That isn't quite right though. We need to ensure that it really is a singleton before taking the short-cut. So you need at least one conditional:

        if (likely(req->unit_size == req->slen))
                take short-cut

Cheers,
--
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Please let me know if [2] looks good and I can include this in v2, thanks!

[2]:

int crypto_acomp_compress(struct acomp_req *req) {
struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);

if (unlikely(acomp_req_on_stack(req) && acomp_is_async(tfm))) return -EAGAIN;

if (likely(req->unit_size && acomp_request_issg(req))) { if (likely(!crypto_acomp_req_seg(tfm))) {
if (likely(req->unit_size == req->slen)) {
int err = tfm->compress(req);

req->dst->length = unlikely(err) ? err : req->dlen; return err;
} else {
return acomp_do_req_chain(req, true);
}
}

return tfm->compress(req);
}

if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req)) return tfm->compress(req);

return acomp_do_req_chain(req, true);
}

Quoted 27 lines · Kanchana P. Sridhar

Please let me know if [2] looks good and I can include this in v2, thanks!

[2]:

int crypto_acomp_compress(struct acomp_req *req) {
struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);

if (unlikely(acomp_req_on_stack(req) && acomp_is_async(tfm))) return -EAGAIN;

if (likely(req->unit_size && acomp_request_issg(req))) { if (likely(!crypto_acomp_req_seg(tfm))) {
if (likely(req->unit_size == req->slen)) {
int err = tfm->compress(req);

req->dst->length = unlikely(err) ? err : req->dlen; return err;
} else {
return acomp_do_req_chain(req, true);
}
}

return tfm->compress(req);
}

Quoted 5 lines

if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req)) return tfm->compress(req);

return acomp_do_req_chain(req, true);
}

This looks good to me.

Thanks,
--
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Quoted 10 lines · Herbert Xu

On Fri, Oct 16, 2026 at 10:02:13PM -0700, Kanchana P. Sridhar wrote:
[ .. ]

That isn't quite right though. We need to ensure that it really is a singleton before taking the short-cut. So you need at least one conditional:

        if (likely(req->unit_size == req->slen))
                take short-cut

I added the check for the singleton req [2]. I hope this looks Ok.

With the latest crypto_acomp_compress(), the batch compress latency improves considerably:

usemem 10 iterations average throughput:

   patch 6 (store batching only): 19,119 KB/s

patch 8 (+ compress batching): 24,076 KB/s (+25.93%)

bpftrace measurement of zswap compress latency per-page:

Sequential_zswap_compress nr_samples p25 p50 p75
p95 p99
=================================================================================
zswap_compress_ns_per_page 1,722,489 43,947 57,791 88,489
130,165 323,961

Batched_zswap_compress nr_samples p25 p50 p75
p95 p99
=================================================================================
zswap_compress_ns_per_page 665,267 41,511 54,915 77,707
124,648 213,962
=================================================================================
Latency saving (ns) -2,435 -2,876 -10,782
-5,517 -109,999
=================================================================================

Quoted 14 lines · Kanchana P. Sridhar

Quoted 10 lines · Herbert Xu

On Fri, Oct 16, 2026 at 10:02:13PM -0700, Kanchana P. Sridhar wrote:
[ .. ]

That isn't quite right though. We need to ensure that it really is a singleton before taking the short-cut. So you need at least one conditional:

        if (likely(req->unit_size == req->slen))
                take short-cut

I added the check for the singleton req [2]. I hope this looks Ok.

Quoted 7 lines

With the latest crypto_acomp_compress(), the batch compress latency improves considerably:

usemem 10 iterations average throughput:

   patch 6 (store batching only): 19,119 KB/s

patch 8 (+ compress batching): 24,076 KB/s (+25.93%)

This is exactly what I mean by the numbers varying too much to draw any conclusions. I am not sure if it's the VM setup or the usemem test itself, but previously it was argued that the short-circuit in crypto_acomp_compress() was ~10% faster than the one in acomp_do_req_chain() (20,946 vs 19,058 KB/s) due to avoiding a function call and two conditionals, and now *adding* a conditional improves performance by 15% (20,946 -> 24,076 KB/s)?

Moreover, for zswap with ZSTD the added check is always true: the batch size is 1, so slen == unit_size == PAGE_SIZE for every request. The previous version and this one execute the same code, so the 15% difference can only be noise. With the stddev you reported (~5.4k KB/s over 10 iterations), both differences are well within the noise.

I think something is wrong with the performance testing methodology here. The results are self-contradicting and do not make much sense to me. Did I miss something?

Quoted 10 lines

bpftrace measurement of zswap compress latency per-page:

Sequential_zswap_compress nr_samples p25 p50 p75
p95 p99
=================================================================================
zswap_compress_ns_per_page 1,722,489 43,947 57,791 88,489
130,165 323,961

Quoted 9 lines

Batched_zswap_compress nr_samples p25 p50 p75
p95 p99
=================================================================================
zswap_compress_ns_per_page 665,267 41,511 54,915 77,707
124,648 213,962
=================================================================================
Latency saving (ns) -2,435 -2,876 -10,782
-5,517 -109,999
=================================================================================

I measured zswap_store() (not zswap_compress()), to get the per-folio latency, and it didn't seem to change much beyond noise in my testing. I also notice that the number of samples is very different between cases.

v1: Batched zswap_store() with compression batching
===================================================

This series is a spin-off of "v14: zswap compression batching with optimized iaa_crypto driver" [1], and focuses on the generic crypto_acomp and zswap compression batching enablers. Resuming work on this series following a gap since v14 (January 2026). Please note that my email address and affiliation have changed from [email protected] to [email protected]. I have retained the original Intel sign-offs on prior patches for historical attribution while signing off v1 of this series with my current address.

To summarize the main contribution of this patch series: Batching is introduced in zswap_store() of large folios: we carve out a large folio into batches of ZSWAP_MAX_BATCH_SIZE (8 pages) for software compressors, and store each batch. Moving the fundamental operations that need to be happen to store a page to now work at the granularity of a batch offers opportunities to optimize them from these perspectives:

1) Utilize existing kernel API that minimize obtaining locks for bulk operations, e.g. kmem_cache_alloc_bulk()/kmem_cache_free_bulk().

2) We keep the working set for each of the core store functionalities (alloc entries, compress pages, store pages in zsmalloc) to be focused on a batch, thereby minimizing cache evictions and benefiting from cache locality.

3) Along the lines of (1), the main zswap_compress() batching routine obtains the acomp_ctx mutex lock once per batch, and releases it after it has processed all pages in that batch.

4) Any zswap store functionality that works on a batch can potentially be parallelized as a next step, to further improve performance.

Performance data: usemem with 60% over-commit:
==============================================
Validation was done with mm-unstable as of 10-3-2026 (Baseline) and with this patch series.

Testing was done on a Chromebook with a QEMU "vng" VM running in Crostini.

Chromebook Diagnostics reports:
Intel(R) Core(TM) Ultra 5 115U (10 CPUs, 4.200GHz)
RAM: 16GB

vng setup:

         Memory: 7G
           Swap: 3G
    Pinned CPUs: 4 (E-Cores: 4,5,6,7)

zswap compressor:
ZSTD

mTHP enabled:
64kB

vm-scalability/usemem setup:

cgroup memory.high: 3G
4 usemem processes, each allocates 1200M anonymous memory, reads the Silesia dataset [2] from a pre-fork-allocated buffer into the 1200M, writes to each unsigned long in the 1200M, sleeps for 10 sec, then frees the memory and exits.

    usemem --read-silesia -w -O -b 1 -s 10 -n 4 1200M

Quoted 64 lines · Kanchana P. Sridhar

v1: Batched zswap_store() with compression batching
===================================================

This series is a spin-off of "v14: zswap compression batching with optimized iaa_crypto driver" [1], and focuses on the generic crypto_acomp and zswap compression batching enablers. Resuming work on this series following a gap since v14 (January 2026). Please note that my email address and affiliation have changed from [email protected] to [email protected]. I have retained the original Intel sign-offs on prior patches for historical attribution while signing off v1 of this series with my current address.

To summarize the main contribution of this patch series: Batching is introduced in zswap_store() of large folios: we carve out a large folio into batches of ZSWAP_MAX_BATCH_SIZE (8 pages) for software compressors, and store each batch. Moving the fundamental operations that need to be happen to store a page to now work at the granularity of a batch offers opportunities to optimize them from these perspectives:

1) Utilize existing kernel API that minimize obtaining locks for bulk operations, e.g. kmem_cache_alloc_bulk()/kmem_cache_free_bulk().

2) We keep the working set for each of the core store functionalities (alloc entries, compress pages, store pages in zsmalloc) to be focused on a batch, thereby minimizing cache evictions and benefiting from cache locality.

3) Along the lines of (1), the main zswap_compress() batching routine obtains the acomp_ctx mutex lock once per batch, and releases it after it has processed all pages in that batch.

4) Any zswap store functionality that works on a batch can potentially be parallelized as a next step, to further improve performance.

Performance data: usemem with 60% over-commit:
==============================================
Validation was done with mm-unstable as of 10-3-2026 (Baseline) and with this patch series.

Testing was done on a Chromebook with a QEMU "vng" VM running in Crostini.

Chromebook Diagnostics reports:
Intel(R) Core(TM) Ultra 5 115U (10 CPUs, 4.200GHz)
RAM: 16GB

vng setup:

         Memory: 7G
           Swap: 3G
    Pinned CPUs: 4 (E-Cores: 4,5,6,7)

zswap compressor:
ZSTD

mTHP enabled:
64kB

vm-scalability/usemem setup:

cgroup memory.high: 3G
4 usemem processes, each allocates 1200M anonymous memory, reads the Silesia dataset [2] from a pre-fork-allocated buffer into the 1200M, writes to each unsigned long in the 1200M, sleeps for 10 sec, then frees the memory and exits.

Quoted 1 line

    usemem --read-silesia -w -O -b 1 -s 10 -n 4 1200M

What is --read-silesia? Is this a local change you made to usemem?