mm/swap: ghost swapfile with backend switching via Redirect entries

Allocate additional physical pages and map them at the end of the already-reserved sparse vmalloc area via vm_area_map_pages(). The flat cluster_info pointer never moves — new clusters appear contiguously at higher indices. New clusters are fully initialized (spinlocks, list heads, trailing BAD mask) before si->max is published via WRITE_ONCE.

Page-pointer tracking in cluster_vm->pages is reallocated to accommodate additional entries. Returns -E2BIG if the requested size exceeds the reserved VA range. Extension is serialized by swapon_mutex.

Signed-off-by Baoquan He <[email protected]>

---
 mm/swapfile.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)

mm/swapfile.c +101 -0

index da085a75d7b5..8bf336678887 100644
@@ -58,6 +58,8 @@ static void move_cluster(struct swap_info_struct *si,
                         enum swap_cluster_flags new_flags);
 #ifdef CONFIG_ZSWAP
 #define GHOST_MAX_CLUSTER_BYTES (64UL * 1024 * 1024)
+static int swap_ghost_extend_max(struct swap_info_struct *si,
+                                unsigned int new_maxpages);
 #endif
 
 /*

@@ -4247,6 +4249,114 @@ swp_entry_t ghost_redirect_to_phys(struct swap_info_struct *si, pgoff_t offset)
 
        return (swp_entry_t){};
 }
+
+/*
+ * swap_ghost_extend_max — Extend an active ghost swap device.
+ * @si:  ghost device (SWP_GHOST set)
+ * @new: new max pages (> si->max)
+ *
+ * Allocates physical pages and maps them at the end of the reserved
+ * sparse vmalloc area.  si->cluster_info never moves.  New clusters
+ * are fully initialized before si->max is published via WRITE_ONCE.
+ *
+ * Returns 0, -ENOMEM, -EINVAL, or -E2BIG.  Caller holds swapon_mutex.
+ */
+static int swap_ghost_extend_max(struct swap_info_struct *si,
+                                unsigned int new)
+{
+       unsigned long old_max = si->max, old_nr, new_nr;
+       unsigned long old_bytes, new_bytes, nr_new_pg;
+       struct page **new_pglist;
+       unsigned long i;
+
+       if (!(si->flags & SWP_GHOST) ||
+           !(si->flags & (SWP_USED | SWP_WRITEOK)) ||
+           new <= si->max)
+               return -EINVAL;
+
+       old_nr = DIV_ROUND_UP(si->max, SWAPFILE_CLUSTER);
+       new_nr = DIV_ROUND_UP(new, SWAPFILE_CLUSTER);
+       old_bytes = ALIGN(old_nr * sizeof(struct swap_cluster_info), PAGE_SIZE);
+       new_bytes = ALIGN(new_nr * sizeof(struct swap_cluster_info), PAGE_SIZE);
+       nr_new_pg = (new_bytes - old_bytes) / PAGE_SIZE;
+
+       if (old_bytes + nr_new_pg * PAGE_SIZE > si->cluster_vm->size) {
+               pr_warn("ghost: extend to %u exceeds reserved VA (%lu bytes)\n",
+                       new, si->cluster_vm->size);
+               return -E2BIG;
+       }
+
+       new_pglist = kvmalloc_array(si->cluster_vm->nr_pages + nr_new_pg,
+                                   sizeof(struct page *), GFP_KERNEL);
+       if (!new_pglist)
+               return -ENOMEM;
+       if (si->cluster_vm->pages) {
+               memcpy(new_pglist, si->cluster_vm->pages,
+                      si->cluster_vm->nr_pages * sizeof(struct page *));
+               kvfree(si->cluster_vm->pages);
+       }
+       si->cluster_vm->pages = (void *)new_pglist;
+
+       for (i = 0; i < nr_new_pg; i++) {
+               struct page *page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+               if (!page) {
+                       while (i-- > 0)
+                               __free_page(new_pglist[si->cluster_vm->nr_pages + i]);
+                       return -ENOMEM;
+               }
+               new_pglist[si->cluster_vm->nr_pages + i] = page;
+       }
+
+       if (vm_area_map_pages(si->cluster_vm,
+                             (unsigned long)si->cluster_vm->addr + old_bytes,
+                             (unsigned long)si->cluster_vm->addr + new_bytes,
+                             &new_pglist[si->cluster_vm->nr_pages])) {
+               for (i = 0; i < nr_new_pg; i++)
+                       __free_page(new_pglist[si->cluster_vm->nr_pages + i]);
+               return -ENOMEM;
+       }
+       si->cluster_vm->nr_pages += nr_new_pg;
+
+       for (i = old_nr; i < new_nr; i++) {
+               struct swap_cluster_info *ci = &si->cluster_info[i];
+               spin_lock_init(&ci->lock);
+               INIT_LIST_HEAD(&ci->list);
+       }
+
+       if (new % SWAPFILE_CLUSTER) {
+               unsigned long base = (new - 1) / SWAPFILE_CLUSTER *
+                                   SWAPFILE_CLUSTER;
+               unsigned long idx = base / SWAPFILE_CLUSTER;
+               struct swap_cluster_info *ci = &si->cluster_info[idx];
+
+               if (!ci->count && swap_cluster_alloc_table(ci, GFP_KERNEL)) {
+                       for (i = 0; i < nr_new_pg; i++)
+                               __free_page(new_pglist[si->cluster_vm->nr_pages + i]);
+                       return -ENOMEM;
+               }
+               spin_lock(&ci->lock);
+               for (i = new % SWAPFILE_CLUSTER; i < SWAPFILE_CLUSTER; i++)
+                       __swap_table_set(ci, i, SWP_TB_BAD);
+               spin_unlock(&ci->lock);
+       }
+
+       WRITE_ONCE(si->max, new);
+
+       for (i = old_nr; i < new_nr; i++) {
+               struct swap_cluster_info *ci = &si->cluster_info[i];
+               ci->flags = CLUSTER_FLAG_FREE;
+               spin_lock(&si->lock);
+               list_add_tail(&ci->list, &si->free_clusters);
+               spin_unlock(&si->lock);

Quoted 138 lines · Baoquan He

Allocate additional physical pages and map them at the end of the already-reserved sparse vmalloc area via vm_area_map_pages(). The flat cluster_info pointer never moves — new clusters appear contiguously at higher indices. New clusters are fully initialized (spinlocks, list heads, trailing BAD mask) before si->max is published via WRITE_ONCE.

Page-pointer tracking in cluster_vm->pages is reallocated to accommodate additional entries. Returns -E2BIG if the requested size exceeds the reserved VA range. Extension is serialized by swapon_mutex.

Signed-off-by Baoquan He <[email protected]>

---
 mm/swapfile.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)

mm/swapfile.c +107 -0

index da085a75d7b5..8bf336678887 100644
@@ -58,6 +58,8 @@ static void move_cluster(struct swap_info_struct *si,
                         enum swap_cluster_flags new_flags);
 #ifdef CONFIG_ZSWAP
 #define GHOST_MAX_CLUSTER_BYTES (64UL * 1024 * 1024)
+static int swap_ghost_extend_max(struct swap_info_struct *si,
+                                unsigned int new_maxpages);
 #endif
 
 /*

@@ -4247,6 +4249,114 @@ swp_entry_t ghost_redirect_to_phys(struct swap_info_struct *si, pgoff_t offset)
 
        return (swp_entry_t){};
 }
+
+/*
+ * swap_ghost_extend_max — Extend an active ghost swap device.
+ * @si:  ghost device (SWP_GHOST set)
+ * @new: new max pages (> si->max)
+ *
+ * Allocates physical pages and maps them at the end of the reserved
+ * sparse vmalloc area.  si->cluster_info never moves.  New clusters
+ * are fully initialized before si->max is published via WRITE_ONCE.
+ *
+ * Returns 0, -ENOMEM, -EINVAL, or -E2BIG.  Caller holds swapon_mutex.
+ */
+static int swap_ghost_extend_max(struct swap_info_struct *si,
+                                unsigned int new)
+{
+       unsigned long old_max = si->max, old_nr, new_nr;
+       unsigned long old_bytes, new_bytes, nr_new_pg;
+       struct page **new_pglist;
+       unsigned long i;
+
+       if (!(si->flags & SWP_GHOST) ||
+           !(si->flags & (SWP_USED | SWP_WRITEOK)) ||
+           new <= si->max)
+               return -EINVAL;
+
+       old_nr = DIV_ROUND_UP(si->max, SWAPFILE_CLUSTER);
+       new_nr = DIV_ROUND_UP(new, SWAPFILE_CLUSTER);
+       old_bytes = ALIGN(old_nr * sizeof(struct swap_cluster_info), PAGE_SIZE);
+       new_bytes = ALIGN(new_nr * sizeof(struct swap_cluster_info), PAGE_SIZE);
+       nr_new_pg = (new_bytes - old_bytes) / PAGE_SIZE;
+
+       if (old_bytes + nr_new_pg * PAGE_SIZE > si->cluster_vm->size) {
+               pr_warn("ghost: extend to %u exceeds reserved VA (%lu bytes)\n",
+                       new, si->cluster_vm->size);
+               return -E2BIG;
+       }
+
+       new_pglist = kvmalloc_array(si->cluster_vm->nr_pages + nr_new_pg,
+                                   sizeof(struct page *), GFP_KERNEL);
+       if (!new_pglist)
+               return -ENOMEM;
+       if (si->cluster_vm->pages) {
+               memcpy(new_pglist, si->cluster_vm->pages,
+                      si->cluster_vm->nr_pages * sizeof(struct page *));
+               kvfree(si->cluster_vm->pages);
+       }
+       si->cluster_vm->pages = (void *)new_pglist;
+
+       for (i = 0; i < nr_new_pg; i++) {
+               struct page *page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+               if (!page) {
+                       while (i-- > 0)
+                               __free_page(new_pglist[si->cluster_vm->nr_pages + i]);
+                       return -ENOMEM;
+               }
+               new_pglist[si->cluster_vm->nr_pages + i] = page;
+       }
+
+       if (vm_area_map_pages(si->cluster_vm,
+                             (unsigned long)si->cluster_vm->addr + old_bytes,
+                             (unsigned long)si->cluster_vm->addr + new_bytes,
+                             &new_pglist[si->cluster_vm->nr_pages])) {
+               for (i = 0; i < nr_new_pg; i++)
+                       __free_page(new_pglist[si->cluster_vm->nr_pages + i]);
+               return -ENOMEM;
+       }
+       si->cluster_vm->nr_pages += nr_new_pg;
+
+       for (i = old_nr; i < new_nr; i++) {
+               struct swap_cluster_info *ci = &si->cluster_info[i];
+               spin_lock_init(&ci->lock);
+               INIT_LIST_HEAD(&ci->list);
+       }
+
+       if (new % SWAPFILE_CLUSTER) {
+               unsigned long base = (new - 1) / SWAPFILE_CLUSTER *
+                                   SWAPFILE_CLUSTER;
+               unsigned long idx = base / SWAPFILE_CLUSTER;
+               struct swap_cluster_info *ci = &si->cluster_info[idx];
+
+               if (!ci->count && swap_cluster_alloc_table(ci, GFP_KERNEL)) {
+                       for (i = 0; i < nr_new_pg; i++)
+                               __free_page(new_pglist[si->cluster_vm->nr_pages + i]);
+                       return -ENOMEM;
+               }
+               spin_lock(&ci->lock);
+               for (i = new % SWAPFILE_CLUSTER; i < SWAPFILE_CLUSTER; i++)
+                       __swap_table_set(ci, i, SWP_TB_BAD);
+               spin_unlock(&ci->lock);
+       }
+
+       WRITE_ONCE(si->max, new);
+
+       for (i = old_nr; i < new_nr; i++) {
+               struct swap_cluster_info *ci = &si->cluster_info[i];
+               ci->flags = CLUSTER_FLAG_FREE;
+               spin_lock(&si->lock);
+               list_add_tail(&ci->list, &si->free_clusters);
+               spin_unlock(&si->lock);
+       }
+
+       si->pages += new - old_max;
+       atomic_long_add(new - old_max, &nr_swap_pages);
+       spin_lock(&swap_lock);
+       total_swap_pages += new - old_max;

mm/swapfile.c swp_entry_t ghost_redirect_to_phys(struct swap_info_struct *si, pgoff_t offset) · line 4360

Quoted 4 lines

mm/swapfile.c +3 -0

+       spin_unlock(&swap_lock);
+       return 0;
+}
 #endif /* CONFIG_ZSWAP */

Does this only grow? Say I have a huge but temporary spike in swap usage, which triggers the extension of the the ghost swapfile address space. Do these metadata space live forever?

If that's the case, might as well just swapon...?

On 07/07/26 at 03:36pm, Nhat Pham wrote:

Yes, it is grow-only for now. The metadata overhead of an extended ghost device is very small: cluster_info[] costs ~32 MB per TB of capacity. For a 64 GB machine, 1 TB of ghost swap space costs about 0.05% of RAM in metadata. This is not a resource that needs urgent reclamation after a spike subsides.

If it does need to be reclaimed — for instance, after a one-time spike pushed capacity far beyond normal needs — ghost swap supports swapoff. The admin can swapoff the ghost device and swapon a new one at a smaller size. No swap type in the kernel today supports online shrinking (not block devices, not swapfiles, not zram), so this is the standard mechanism.

Allocate additional physical pages and map them at the end of the already-reserved sparse vmalloc area via vm_area_map_pages(). The flat cluster_info pointer never moves — new clusters appear contiguously at higher indices. New clusters are fully initialized (spinlocks, list heads, trailing BAD mask) before si->max is published via WRITE_ONCE.

Page-pointer tracking in cluster_vm->pages is reallocated to accommodate additional entries. Returns -E2BIG if the requested size exceeds the reserved VA range. Extension is serialized by swapon_mutex.

Signed-off-by Baoquan He <[email protected]>

---
 mm/swapfile.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)

mm/swapfile.c +37 -0

index da085a75d7b5..8bf336678887 100644
@@ -58,6 +58,8 @@ static void move_cluster(struct swap_info_struct *si,
                         enum swap_cluster_flags new_flags);
 #ifdef CONFIG_ZSWAP
 #define GHOST_MAX_CLUSTER_BYTES (64UL * 1024 * 1024)
+static int swap_ghost_extend_max(struct swap_info_struct *si,
+                                unsigned int new_maxpages);
 #endif
 
 /*

@@ -4247,6 +4249,114 @@ swp_entry_t ghost_redirect_to_phys(struct swap_info_struct *si, pgoff_t offset)
 
        return (swp_entry_t){};
 }
+
+/*
+ * swap_ghost_extend_max — Extend an active ghost swap device.
+ * @si:  ghost device (SWP_GHOST set)
+ * @new: new max pages (> si->max)
+ *
+ * Allocates physical pages and maps them at the end of the reserved
+ * sparse vmalloc area.  si->cluster_info never moves.  New clusters
+ * are fully initialized before si->max is published via WRITE_ONCE.
+ *
+ * Returns 0, -ENOMEM, -EINVAL, or -E2BIG.  Caller holds swapon_mutex.
+ */
+static int swap_ghost_extend_max(struct swap_info_struct *si,
+                                unsigned int new)
+{
+       unsigned long old_max = si->max, old_nr, new_nr;
+       unsigned long old_bytes, new_bytes, nr_new_pg;
+       struct page **new_pglist;
+       unsigned long i;
+
+       if (!(si->flags & SWP_GHOST) ||
+           !(si->flags & (SWP_USED | SWP_WRITEOK)) ||
+           new <= si->max)
+               return -EINVAL;
+
+       old_nr = DIV_ROUND_UP(si->max, SWAPFILE_CLUSTER);
+       new_nr = DIV_ROUND_UP(new, SWAPFILE_CLUSTER);
+       old_bytes = ALIGN(old_nr * sizeof(struct swap_cluster_info), PAGE_SIZE);
+       new_bytes = ALIGN(new_nr * sizeof(struct swap_cluster_info), PAGE_SIZE);
+       nr_new_pg = (new_bytes - old_bytes) / PAGE_SIZE;
+
+       if (old_bytes + nr_new_pg * PAGE_SIZE > si->cluster_vm->size) {
+               pr_warn("ghost: extend to %u exceeds reserved VA (%lu bytes)\n",
+                       new, si->cluster_vm->size);
+               return -E2BIG;

mm/swapfile.c +63 -1

+       }
+       si->cluster_vm->pages = (void *)new_pglist;
+
+       for (i = 0; i < nr_new_pg; i++) {
+               struct page *page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+               if (!page) {
+                       while (i-- > 0)
+                               __free_page(new_pglist[si->cluster_vm->nr_pages + i]);
+                       return -ENOMEM;
+               }
+               new_pglist[si->cluster_vm->nr_pages + i] = page;
+       }
+
+       if (vm_area_map_pages(si->cluster_vm,
+                             (unsigned long)si->cluster_vm->addr + old_bytes,
+                             (unsigned long)si->cluster_vm->addr + new_bytes,
+                             &new_pglist[si->cluster_vm->nr_pages])) {
+               for (i = 0; i < nr_new_pg; i++)
+                       __free_page(new_pglist[si->cluster_vm->nr_pages + i]);
+               return -ENOMEM;
+       }
+       si->cluster_vm->nr_pages += nr_new_pg;
+
+       for (i = old_nr; i < new_nr; i++) {
+               struct swap_cluster_info *ci = &si->cluster_info[i];
+               spin_lock_init(&ci->lock);
+               INIT_LIST_HEAD(&ci->list);
+       }
+
+       if (new % SWAPFILE_CLUSTER) {
+               unsigned long base = (new - 1) / SWAPFILE_CLUSTER *
+                                   SWAPFILE_CLUSTER;
+               unsigned long idx = base / SWAPFILE_CLUSTER;
+               struct swap_cluster_info *ci = &si->cluster_info[idx];
+
+               if (!ci->count && swap_cluster_alloc_table(ci, GFP_KERNEL)) {
+                       for (i = 0; i < nr_new_pg; i++)
+                               __free_page(new_pglist[si->cluster_vm->nr_pages + i]);
+                       return -ENOMEM;
+               }
+               spin_lock(&ci->lock);
+               for (i = new % SWAPFILE_CLUSTER; i < SWAPFILE_CLUSTER; i++)
+                       __swap_table_set(ci, i, SWP_TB_BAD);
+               spin_unlock(&ci->lock);
+       }
+
+       WRITE_ONCE(si->max, new);
+
+       for (i = old_nr; i < new_nr; i++) {
+               struct swap_cluster_info *ci = &si->cluster_info[i];
+               ci->flags = CLUSTER_FLAG_FREE;
+               spin_lock(&si->lock);
+               list_add_tail(&ci->list, &si->free_clusters);
+               spin_unlock(&si->lock);
+       }
+
+       si->pages += new - old_max;
+       atomic_long_add(new - old_max, &nr_swap_pages);
+       spin_lock(&swap_lock);
+       total_swap_pages += new - old_max;
+       spin_unlock(&swap_lock);
+       return 0;
+}
 #endif /* CONFIG_ZSWAP */
 
 static int __init swapfile_init(void)
--

2.54.0

This series introduces a virtual "ghost" swapfile that can be backed by multiple physical swap devices, switching between them at runtime using a new "Redirect" swap table entry type.

Motivation
----------
Physical swap devices have fixed, large sizes but a machine may only need a fraction at any given time. Ghost swap decouples the logical swap capacity from physical swap allocation: it presents a large virtual swap device to userspace while lazily mapping slots to physical devices only when data must leave zswap (writeback). This enables overcommit of swap capacity without wasting physical disk space, and allows the ghost device to grow dynamically at runtime.

Ghost swapfile characteristics
--------------------------------
Ghost swap decouples logical swap capacity from physical swap allocation. Key properties:

- No backing store required. A ghost swap device can operate purely as a zswap container ??? compressed pages stay in the zswap pool without ever touching a physical swap device. No physical swap device needs to be attached.

- Optional physical backing. One or more physical swap devices can be swapon'd alongside the ghost device, acting as writeback targets when the zswap pool needs to evict cold pages. The Redirect entry in the ghost swap table links each evicted slot to its physical destination.

- Dynamic size growth. The ghost device can be extended at runtime via sysfs, up to a current maximum of 1TB. This is implemented through lazy vmalloc: a sparse 64MB virtual address space is reserved at swapon time to cover the full 1TB range, but physical pages for cluster_info[] structures are allocated only on demand as the device grows. The cluster_info pointer never changes, so all existing accessors (__swap_offset_to_cluster, cluster_index) work unchanged.

- sysfs control. Each ghost device exposes attributes under /sys/kernel/mm/ghost_swap/ghost_<N>/:
path (RO) ??? backing device name
max (RW) ??? current max pages; writing a larger value triggers
swap_ghost_extend_max()

Design overview
---------------
A swap table slot can now hold a Redirect entry:

Redirect: |----- physical swp_entry_t -----|101|

This stores a plain swp_entry_t pointing to a slot on a real (physical) swap device. The low 3-bit mark (0b101) distinguishes it from Pointer (0b100), PFN (0b10), Shadow (0b1), and NULL (0) entries.

The lifecycle is:

1. swapon a ghost device ??? reserves sparse vmalloc area (64MB virtual) and physically backs only the initial range. The device starts with no physical backing store: pages swapped out go to zswap, and zswap pool evictions go to any concurrently attached physical device.

2. zswap writeback from ghost: when the zswap pool needs to evict, zswap_writeback_ghost() allocates a physical slot from any real swap device, writes the data there, and plants a Redirect entry in the ghost's swap table pointing to the physical slot.

3. Swap-in from ghost: swap_read_folio() detects the ghost device (SWP_GHOST), resolves the Redirect to the physical swp_entry_t, and forwards the read I/O to the physical device.

4. When the swap cache folio is removed, the Redirect entry is restored from an xarray (redirect_xa) that persists the mapping, so subsequent reads still find the physical backing.

5. When the physical slot is freed, the ghost slot is also freed, cascading through swap_range_free().

Note:
------
Ghost swapfile is a proof of concept. Compared with the other patchset:
[RFC PATCH v2 0/7] mm, swap: Virtual Swap Space (Swap Table Edition), it explores a different approach to implementing virtual swap on top of the existing swap table infrastructure, with a simpler implementation. It builds on Chris Li's earlier ghost swapfile work and is currently tentatively named "ghost swapfile" ??? alternative naming suggestions are welcome. Ghost swapfile setup can also be integrated through modifications to the swapon API and the swapon command in util-linux, for example:
swapon --ghost ghost_1:2G -p 5.

Baoquan He (9):
mm/swap_table: add Redirect entry encoding for ghost swap backend
switching
mm/swap: add redirect_xa field and ghost redirect helper declarations mm/swapfile: implement ghost redirect helpers and free-path cascade mm/swap_state: restore Redirect entry when swap cache folio is removed mm/zswap: implement ghost-to-physical writeback for backend switching mm/page_io: forward ghost swap reads to physical device via Redirect mm/swapfile: manage ghost cluster_info via lazy vmalloc mm/swapfile: implement swap_ghost_extend_max() for dynamic growth mm/swapfile: add sysfs interface for ghost swap extension

Chris Li (1):
mm: ghost swapfile support for zswap

 include/linux/swap.h |  10 +
 mm/page_io.c         |  51 +++-
 mm/swap.h            |  16 +-
 mm/swap_state.c      |  28 +++
 mm/swap_table.h      |  38 +++
 mm/swapfile.c        | 579 ++++++++++++++++++++++++++++++++++++++++---
 mm/zswap.c           |  98 +++++++-
 7 files changed, 772 insertions(+), 48 deletions(-)

Signature · 1 line

2.54.0

This series introduces a virtual "ghost" swapfile that can be backed by multiple physical swap devices, switching between them at runtime using a new "Redirect" swap table entry type.

Motivation
----------
Physical swap devices have fixed, large sizes but a machine may only need a fraction at any given time. Ghost swap decouples the logical swap capacity from physical swap allocation: it presents a large virtual swap device to userspace while lazily mapping slots to physical devices only when data must leave zswap (writeback). This enables overcommit of swap capacity without wasting physical disk space, and allows the ghost device to grow dynamically at runtime.

Ghost swapfile characteristics
--------------------------------
Ghost swap decouples logical swap capacity from physical swap allocation. Key properties:

- No backing store required. A ghost swap device can operate purely as a zswap container — compressed pages stay in the zswap pool without ever touching a physical swap device. No physical swap device needs to be attached.

- Optional physical backing. One or more physical swap devices can be swapon'd alongside the ghost device, acting as writeback targets when the zswap pool needs to evict cold pages. The Redirect entry in the ghost swap table links each evicted slot to its physical destination.

- Dynamic size growth. The ghost device can be extended at runtime via

sysfs, up to a current maximum of 1TB. This is implemented through lazy vmalloc: a sparse 64MB virtual address space is reserved at swapon time to cover the full 1TB range, but physical pages for

cluster_info[] structures are allocated only on demand as the device grows. The cluster_info pointer never changes, so all existing accessors (__swap_offset_to_cluster, cluster_index) work unchanged.

- sysfs control. Each ghost device exposes attributes under /sys/kernel/mm/ghost_swap/ghost_<N>/:
path (RO) — backing device name
max (RW) — current max pages; writing a larger value triggers
swap_ghost_extend_max()

Design overview
---------------
A swap table slot can now hold a Redirect entry:

Redirect: |----- physical swp_entry_t -----|101|

This stores a plain swp_entry_t pointing to a slot on a real (physical) swap device. The low 3-bit mark (0b101) distinguishes it from Pointer (0b100), PFN (0b10), Shadow (0b1), and NULL (0) entries.

The lifecycle is:

1. swapon a ghost device — reserves sparse vmalloc area (64MB virtual) and physically backs only the initial range. The device starts with no physical backing store: pages swapped out go to zswap, and zswap pool evictions go to any concurrently attached physical device.

2. zswap writeback from ghost: when the zswap pool needs to evict, zswap_writeback_ghost() allocates a physical slot from any real swap device, writes the data there, and plants a Redirect entry in the ghost's swap table pointing to the physical slot.

3. Swap-in from ghost: swap_read_folio() detects the ghost device (SWP_GHOST), resolves the Redirect to the physical swp_entry_t, and forwards the read I/O to the physical device.

4. When the swap cache folio is removed, the Redirect entry is restored from an xarray (redirect_xa) that persists the mapping, so subsequent reads still find the physical backing.

5. When the physical slot is freed, the ghost slot is also freed, cascading through swap_range_free().

Note:
------
Ghost swapfile is a proof of concept. Compared with the other patchset:
[RFC PATCH v2 0/7] mm, swap: Virtual Swap Space (Swap Table Edition), it explores a different approach to implementing virtual swap on top of the existing swap table infrastructure, with a simpler implementation.

On Tue, Jul 7, 2026 at 1:26 AM Baoquan He <[email protected]> wrote:

So it's more complicated, and achieve less - this is specific to zswap, whereas what I provide is something generic to swap infrastructure, which can be extended for future backends. I would understand if it resulted in less overhead (which was the argument for my pre-swap-table vswap design), but this seems like it's just virtual swap, with similar space overhead (technically a bit worse since you added redirect_xa xarray to physical swap cluster too), and with more complications?

Quoted 25 lines · Baoquan He

This series introduces a virtual "ghost" swapfile that can be backed by multiple physical swap devices, switching between them at runtime using a new "Redirect" swap table entry type.

Motivation
----------
Physical swap devices have fixed, large sizes but a machine may only need a fraction at any given time. Ghost swap decouples the logical swap capacity from physical swap allocation: it presents a large virtual swap device to userspace while lazily mapping slots to physical devices only when data must leave zswap (writeback). This enables overcommit of swap capacity without wasting physical disk space, and allows the ghost device to grow dynamically at runtime.

Ghost swapfile characteristics
--------------------------------
Ghost swap decouples logical swap capacity from physical swap allocation. Key properties:

- No backing store required. A ghost swap device can operate purely as a zswap container — compressed pages stay in the zswap pool without ever touching a physical swap device. No physical swap device needs to be attached.

Quoted 6 lines

- Optional physical backing. One or more physical swap devices can be swapon'd alongside the ghost device, acting as writeback targets when the zswap pool needs to evict cold pages. The Redirect entry in the ghost swap table links each evicted slot to its physical destination.

- Dynamic size growth. The ghost device can be extended at runtime via

We don't want to put additional operational burden on users. Besides, expecting userspace to react to swap space being filled, especially under memory pressure (at which point userspace processes might be stalling already), is tricky.

This is especially bad when redirection entry still eats up space in the upper level (ghost zswap) swapfile. You will run out of zswap space really fast under concurrent zswap store and zswap writeback.

Quoted 4 lines

sysfs, up to a current maximum of 1TB. This is implemented through lazy vmalloc: a sparse 64MB virtual address space is reserved at swapon time to cover the full 1TB range, but physical pages for

This range seems a bit small, especially from something essentially fixed.

Quoted 23 lines

cluster_info[] structures are allocated only on demand as the device grows. The cluster_info pointer never changes, so all existing accessors (__swap_offset_to_cluster, cluster_index) work unchanged.

- sysfs control. Each ghost device exposes attributes under /sys/kernel/mm/ghost_swap/ghost_<N>/:
path (RO) — backing device name
max (RW) — current max pages; writing a larger value triggers
swap_ghost_extend_max()

Design overview
---------------
A swap table slot can now hold a Redirect entry:

Redirect: |----- physical swp_entry_t -----|101|

This stores a plain swp_entry_t pointing to a slot on a real (physical) swap device. The low 3-bit mark (0b101) distinguishes it from Pointer (0b100), PFN (0b10), Shadow (0b1), and NULL (0) entries.

The lifecycle is:

Quoted 4 lines

1. swapon a ghost device — reserves sparse vmalloc area (64MB virtual) and physically backs only the initial range. The device starts with no physical backing store: pages swapped out go to zswap, and zswap pool evictions go to any concurrently attached physical device.

If you think this data structure is superior to the xarray for dynamic swap address, it's a fairly simple change on top of what I had in the virtual swap RFC, no?

It can be grafted on top of what I had in a subsequent patch, or squashed into my first patch. If you have concerns with me implementing your idea here, I'd take your code if it proves to be superior.

Quoted 1 line

Quoted 4 lines

2. zswap writeback from ghost: when the zswap pool needs to evict, zswap_writeback_ghost() allocates a physical slot from any real swap device, writes the data there, and plants a Redirect entry in the ghost's swap table pointing to the physical slot.

What if user swapoff the physical swapfile, whose slot is backing a redirection entry of a ghost swapfile, rather than a PTE. Has this been handled yet?

Similarly, say the physical swapfile is full, and you attempt to reclaim a physical swap slot that is swap-cache-only (swap count = 0). Say this physical swap slot was allocated from a previous zswap writeback attempt, i.e it has a redirection entry in the ghost zswap swapfile associated with it. Did we handle it somehow?

My apologies if I missed it somehow.

Quoted 9 lines

3. Swap-in from ghost: swap_read_folio() detects the ghost device (SWP_GHOST), resolves the Redirect to the physical swp_entry_t, and forwards the read I/O to the physical device.

4. When the swap cache folio is removed, the Redirect entry is restored from an xarray (redirect_xa) that persists the mapping, so subsequent reads still find the physical backing.

Quoted 2 lines

5. When the physical slot is freed, the ghost slot is also freed, cascading through swap_range_free().

I don't get how *any* of this is simpler from what I had. If anything, it's more complicated. With a vswap device, you just allocate a physical swap slot (which you also did), change the backend, submit the IO (which you also did), and that's it. No need to juggle this redirect_xa entry. What does this redirect_xa buy us?

Not to mention now it entagles generic swap logic into backend-specific code, unnecessarily. And we have more state in the state machine to consider: With a generic vswap device, vswap -> disk swap directly is the same state as vswap -> written back zswap entry. With this zswap ghost swapfile design, you have to treat the latter separately, as there is a redirection entry involved, which means concurrent swap operations interact with it.

So it's more complicated, and achieve less - this is specific to zswap, whereas what I provide is something generic to swap infrastructure, which can be extended for future backends. I would understand if it resulted in less overhead (which was the argument for my pre-swap-table vswap design), but this seems like it's just virtual swap, with similar space overhead (technically a bit worse since you added redirect_xa xarray to physical swap cluster too), and with more complications?

Quoted 7 lines

Note:
------
Ghost swapfile is a proof of concept. Compared with the other patchset:
[RFC PATCH v2 0/7] mm, swap: Virtual Swap Space (Swap Table Edition), it explores a different approach to implementing virtual swap on top of the existing swap table infrastructure, with a simpler implementation.

You neglected to mention that this "simpler" implementation lacks a lot of design choices. Just on top of my head:

1. Certain optimizations for cases when vswap points to disk swap.

2. No reverse mapping, or at least some mechanisms to handle the case where you have to go from physical swap slots to the upper layer.

3. Charging behavior difference, and deduplication of metadata when we have multiple swap levels. (In fact, you seem to be double charging a writtenback entry here towards the swap counter, not just duplicating the metadata space)

4. Infrastructure to make sure allocation from this swap device and allocation from the backing swap device does not invalidate each other's per-cpu allocation cache. You seem to just go for the slow path with alloc_phys_swap_slot() for zswap writeback?

It's fine to disagree with any or all of these design choices of mine
- I'm receptive to discussion. But it's misleading to say you have a simpler design here, IMHO. My design choices resulted from multiple thoughtful iterations (a lot of them could be found in my pre-swap-table version of vswap) and real production pains.

Thanks for the thorough review, Nhat.

Quoted 5 lines

We don't want to put additional operational burden on users. Besides, expecting userspace to react to swap space being filled, especially under memory pressure (at which point userspace processes might be stalling already), is tricky.

The manual sysfs extension is the minimal viable interface for the RFC, not the final choice. But even in its current form, it is a dramatic improvement over the status quo.

Today, to use zswap you MUST provision a physical swap device of the same size as the swap capacity wanted — even though most of that disk space will never be touched. With ghost swap, you swapon a ghost device at whatever size you expect to need (e.g. 64 GB), and if your workload grows, you grow it via sysfs. That is already a step change in operational simplicity: no disk pre-provisioning, no wasted storage, and a one-line resize when needed.

Automatic growth can be considered later if it's really demanded.

Quoted 5 lines

This is especially bad when redirection entry still eats up space in the upper level (ghost zswap) swapfile. You will run out of zswap space really fast under concurrent zswap store and zswap writeback.

I think there is a misunderstanding here. A Redirect entry does not consume an *additional* ghost slot. Before writeback, the ghost slot held a Pointer entry (zswap). After writeback, that same slot holds a Redirect entry (pointing to the physical device). It's a state transition — one slot, one entry — not an extra allocation. The ghost device's total capacity is unchanged by writeback activity.

Quoted 6 lines

Quoted 5 lines

sysfs, up to a current maximum of 1TB. This is implemented through lazy vmalloc: a sparse 64MB virtual address space is reserved at swapon time to cover the full 1TB range, but physical pages for

Quoted 1 line

This range seems a bit small, especially from something essentially fixed.

The 1TB limit comes from a conservative 64 MB sparse vmalloc reservation in RFC. It certainly can be redefined as a bigger one which is more reasonable in reality. For most real-world deployments today, 1 TB per ghost device — with support for multiple ghost devices — is more than sufficient.

Quoted 24 lines

Quoted 24 lines

cluster_info[] structures are allocated only on demand as the device grows. The cluster_info pointer never changes, so all existing accessors (__swap_offset_to_cluster, cluster_index) work unchanged.

- sysfs control. Each ghost device exposes attributes under /sys/kernel/mm/ghost_swap/ghost_<N>/:
path (RO) — backing device name
max (RW) — current max pages; writing a larger value triggers
swap_ghost_extend_max()

Design overview
---------------
A swap table slot can now hold a Redirect entry:

Redirect: |----- physical swp_entry_t -----|101|

This stores a plain swp_entry_t pointing to a slot on a real (physical) swap device. The low 3-bit mark (0b101) distinguishes it from Pointer (0b100), PFN (0b10), Shadow (0b1), and NULL (0) entries.

The lifecycle is:

Quoted 8 lines

Quoted 4 lines

1. swapon a ghost device — reserves sparse vmalloc area (64MB virtual) and physically backs only the initial range. The device starts with no physical backing store: pages swapped out go to zswap, and zswap pool evictions go to any concurrently attached physical device.

If you think this data structure is superior to the xarray for dynamic swap address, it's a fairly simple change on top of what I had in the virtual swap RFC, no?

I appreciate the suggestion. However, the difference between the two approaches goes beyond the cluster allocation strategy — it's the overall architecture:

- Virtual swap introduces a mandatory, always-on vswap device (CONFIG_VSWAP). with per-cluster virtual_table arrays, xarray-backed dynamic clusters. per-CPU allocation caching.

- Ghost swap extends the existing swap device model with one new entry type (Redirect, 0b101) and one new flag (SWP_GHOST). It reuses swapon/swapoff, the existing allocator, and the swap cache essentially unchanged. No Kconfig option is needed.

These are two explorations of the same problem, while virtual swap goes far beyond the problem itself, the decoupling of physical swap and virtal zswap. We can discuss and choose one approach.

Quoted 13 lines

It can be grafted on top of what I had in a subsequent patch, or squashed into my first patch. If you have concerns with me implementing your idea here, I'd take your code if it proves to be superior.

Quoted 5 lines

2. zswap writeback from ghost: when the zswap pool needs to evict, zswap_writeback_ghost() allocates a physical slot from any real swap device, writes the data there, and plants a Redirect entry in the ghost's swap table pointing to the physical slot.

Quoted 3 lines

What if user swapoff the physical swapfile, whose slot is backing a redirection entry of a ghost swapfile, rather than a PTE. Has this been handled yet?

This is a gap in the current RFC. Both swapoff and physical-slot reclaim would need a reverse mappingfrom the physical slot back to the ghost entry to be handled robustly. This can be added in a future version

if there is consensus that writeback-to-physical is a necessary part of the design.

I think the primary motivation for ghost swap is to enable zswap usage *without* a backing physical swap device at all — the "No backing
store required" mode described in the cover letter. In that mode,
writeback never happens (there is no physical device to write to), so the swapoff and reclaim questions do not arise. Ghost writeback to a physical device is an optional extension for users who want the safety net of disk-backed swap. Surely we can discuss and decide if we need add it or not.

Quoted 18 lines

Similarly, say the physical swapfile is full, and you attempt to reclaim a physical swap slot that is swap-cache-only (swap count = 0). Say this physical swap slot was allocated from a previous zswap writeback attempt, i.e it has a redirection entry in the ghost zswap swapfile associated with it. Did we handle it somehow?

My apologies if I missed it somehow.

Quoted 9 lines

3. Swap-in from ghost: swap_read_folio() detects the ghost device (SWP_GHOST), resolves the Redirect to the physical swp_entry_t, and forwards the read I/O to the physical device.

4. When the swap cache folio is removed, the Redirect entry is restored from an xarray (redirect_xa) that persists the mapping, so subsequent reads still find the physical backing.

Quoted 8 lines

Quoted 2 lines

5. When the physical slot is freed, the ghost slot is also freed, cascading through swap_range_free().

I don't get how *any* of this is simpler from what I had. If anything, it's more complicated. With a vswap device, you just allocate a physical swap slot (which you also did), change the backend, submit the IO (which you also did), and that's it. No need to juggle this redirect_xa entry. What does this redirect_xa buy us?

The redirect_xa persists the ghost→physical mapping across swap-cache folio teardown.

Ghost: redirect_xa[offset] = phys (only allocated when Redirect
is active) + swap_table[slot] = Redirect(phys) once the swap cache folio is removed

Quoted 9 lines

Not to mention now it entagles generic swap logic into backend-specific code, unnecessarily. And we have more state in the state machine to consider: With a generic vswap device, vswap -> disk swap directly is the same state as vswap -> written back zswap entry. With this zswap ghost swapfile design, you have to treat the latter separately, as there is a redirection entry involved, which means concurrent swap operations interact with it.

I'd argue the opposite. Ghost swap touches the generic swap paths in very few places:

- swap_read_folio: one SWP_GHOST branch → resolve Redirect
- __swap_cluster_free_entries: one Redirect check → cascade free
- __swap_cache_do_del_folio: one redirect_xa lookup → restore
- zswap_writeback_entry: one SWP_GHOST branch → ghost writeback

Ghost swap only affects behavior when a ghost device has been explicitly swapon'd, and even then only on the paths listed above.

This is different than virtual swap which is more ambitious. As I said, ghost swapfile focuses on the problem itself, and based on the swap table.

Quoted 4 lines

2. No reverse mapping, or at least some mechanisms to handle the case where you have to go from physical swap slots to the upper layer.

Acknowledged — a reverse mapping can be added in a future version if writeback-to-physical becomes a priority.

Quoted 11 lines

3. Charging behavior difference, and deduplication of metadata when we have multiple swap levels. (In fact, you seem to be double charging a writtenback entry here towards the swap counter, not just duplicating the metadata space)

4. Infrastructure to make sure allocation from this swap device and allocation from the backing swap device does not invalidate each other's per-cpu allocation cache. You seem to just go for the slow path with alloc_phys_swap_slot() for zswap writeback?

Yes, alloc_phys_swap_slot() uses the slow-path allocator (cluster_alloc_swap_entry), which is correct for this low-frequency operation. zswap writeback is not a hot allocation path — it runs from the zswap shrinker under memory pressure, typically hundreds to thousands of operations per second at most. The fast-path per-CPU caches remain dedicated to folio_alloc_swap(), which handles the high-frequency swapout path. There is no cache invalidation problem because the two paths use different allocation mechanisms on different devices.

Quoted 6 lines

It's fine to disagree with any or all of these design choices of mine
- I'm receptive to discussion. But it's misleading to say you have a simpler design here, IMHO. My design choices resulted from multiple thoughtful iterations (a lot of them could be found in my pre-swap-table version of vswap) and real production pains.

You're right that "simpler" needs qualification, and I'll update the cover letter accordingly. What I meant — and what I should have stated explicitly — is that ghost swap is less invasive to the existing swap infrastructure:

- It does not require a Kconfig option.
- It does not create a boot-time device that lives for the kernel's lifetime.

- It does not introduce a new per-cluster data structure (swap_cluster_info_dynamic / virtual_table).

- It does not add a separate per-CPU allocation cache.

- It does not split the swap entry namespace into "vswap" vs "physical" — a ghost device is just another swap device with one extra flag and one extra swap-table entry type.

I think VSS goes beyond solving the "zswap without backing store" problem — it is fundamentally an architectural restructuring of the swap subsystem, with virtual swap as a first-class abstraction layered above physical devices. That is both its strength (comprehensive, future-proof) and its trade-off (more invasive to existing paths). I respect that.

My goal with ghost swap is different: address the immediate pain point with the smallest possible change to the existing infrastructure. Both approaches have value, and I hope the community discussion will help identify the right balance.

Thank you again for the detailed review — it has already clarified which parts of the design need more thought and documentation, and will make the next version stronger.

Quoted 13 lines · Baoquan He

Thanks for the thorough review, Nhat.

Quoted 5 lines

We don't want to put additional operational burden on users. Besides, expecting userspace to react to swap space being filled, especially under memory pressure (at which point userspace processes might be stalling already), is tricky.

The manual sysfs extension is the minimal viable interface for the RFC, not the final choice. But even in its current form, it is a dramatic improvement over the status quo.

Quoted 7 lines

Today, to use zswap you MUST provision a physical swap device of the same size as the swap capacity wanted — even though most of that disk space will never be touched. With ghost swap, you swapon a ghost device at whatever size you expect to need (e.g. 64 GB), and if your workload grows, you grow it via sysfs. That is already a step change in operational simplicity: no disk pre-provisioning, no wasted storage, and a one-line resize when needed.

If these are the only three requirements you have, in fact, you can already achieve something equivalent to a ghost swapfile: just put zswap on top of any of the in-memory swapfile (and perhaps disable writeback, although with careful limit setting it's almost impossible to trigger writeback with zswap in newer kernels anyway). I know this works, because I even run tests with zswap-on-top of zram :) You can swapon new swapfiles to support your "userspace-triggered" growth strategy.

Quoted 2 lines

Automatic growth can be considered later if it's really demanded.

It is really, really necessary. Please read my older virtual swap space patch series for why it's necessary (see [5]).

Quoted 6 lines

Quoted 5 lines

This is especially bad when redirection entry still eats up space in the upper level (ghost zswap) swapfile. You will run out of zswap space really fast under concurrent zswap store and zswap writeback.

Quoted 6 lines

I think there is a misunderstanding here. A Redirect entry does not consume an *additional* ghost slot. Before writeback, the ghost slot held a Pointer entry (zswap). After writeback, that same slot holds a Redirect entry (pointing to the physical device). It's a state transition — one slot, one entry — not an extra allocation. The ghost device's total capacity is unchanged by writeback activity.

No, what I meant is, after writeback, you still hold on to the old slot on the ghost device, AND a new slot on the physical device, correct?

Quoted 6 lines

Quoted 6 lines

Quoted 5 lines

sysfs, up to a current maximum of 1TB. This is implemented through lazy vmalloc: a sparse 64MB virtual address space is reserved at swapon time to cover the full 1TB range, but physical pages for

Quoted 7 lines

Quoted 1 line

This range seems a bit small, especially from something essentially fixed.

The 1TB limit comes from a conservative 64 MB sparse vmalloc reservation in RFC. It certainly can be redefined as a bigger one which is more reasonable in reality. For most real-world deployments today, 1 TB per ghost device — with support for multiple ghost devices — is more than sufficient.

I beg to differ. We already have multi-terrabyte scale machines in production in multiple places, and more and more overcommitting. At least make this configurable.

Quoted 41 lines

Quoted 24 lines

Quoted 24 lines

cluster_info[] structures are allocated only on demand as the device grows. The cluster_info pointer never changes, so all existing accessors (__swap_offset_to_cluster, cluster_index) work unchanged.

- sysfs control. Each ghost device exposes attributes under /sys/kernel/mm/ghost_swap/ghost_<N>/:
path (RO) — backing device name
max (RW) — current max pages; writing a larger value triggers
swap_ghost_extend_max()

Design overview
---------------
A swap table slot can now hold a Redirect entry:

Redirect: |----- physical swp_entry_t -----|101|

This stores a plain swp_entry_t pointing to a slot on a real (physical) swap device. The low 3-bit mark (0b101) distinguishes it from Pointer (0b100), PFN (0b10), Shadow (0b1), and NULL (0) entries.

The lifecycle is:

Quoted 8 lines

Quoted 4 lines

1. swapon a ghost device — reserves sparse vmalloc area (64MB virtual) and physically backs only the initial range. The device starts with no physical backing store: pages swapped out go to zswap, and zswap pool evictions go to any concurrently attached physical device.

If you think this data structure is superior to the xarray for dynamic swap address, it's a fairly simple change on top of what I had in the virtual swap RFC, no?

I appreciate the suggestion. However, the difference between the two approaches goes beyond the cluster allocation strategy — it's the overall architecture:

- Virtual swap introduces a mandatory, always-on vswap device (CONFIG_VSWAP). with per-cluster virtual_table arrays, xarray-backed dynamic clusters. per-CPU allocation caching.

Quoted 8 lines

- Ghost swap extends the existing swap device model with one new entry type (Redirect, 0b101) and one new flag (SWP_GHOST). It reuses swapon/swapoff, the existing allocator, and the swap cache essentially unchanged. No Kconfig option is needed.

These are two explorations of the same problem, while virtual swap goes far beyond the problem itself, the decoupling of physical swap and virtal zswap. We can discuss and choose one approach.

If my problems are real, why do we have to choose? If we go with my approach, all of our problems are solved. If we go with your approach, then your problem is solved, with a promise that my problems will be tackled in the future.

Quoted 13 lines

Quoted 13 lines

It can be grafted on top of what I had in a subsequent patch, or squashed into my first patch. If you have concerns with me implementing your idea here, I'd take your code if it proves to be superior.

Quoted 5 lines

2. zswap writeback from ghost: when the zswap pool needs to evict, zswap_writeback_ghost() allocates a physical slot from any real swap device, writes the data there, and plants a Redirect entry in the ghost's swap table pointing to the physical slot.

Quoted 8 lines

Quoted 3 lines

What if user swapoff the physical swapfile, whose slot is backing a redirection entry of a ghost swapfile, rather than a PTE. Has this been handled yet?

This is a gap in the current RFC. Both swapoff and physical-slot reclaim would need a reverse mappingfrom the physical slot back to the ghost entry to be handled robustly. This can be added in a future version

To be clear, this is not a feature gap. In the current form of your RFC, it is an actual bug: it breaks writeback *completely*. Swapoff (and reclaim of swap-cache-only physical slots) leaks ghost zswap swapfile slots that are still backed by these reclaimed slots.

I understand that this is just an RFC, or rather a proof-of-concept of what you want to propose. But you can't claim it both way. It is only simpler if you're dropping writeback altogether.

Quoted 2 lines

if there is consensus that writeback-to-physical is a necessary part of the design.

I'll not spend my time argue for the necessity of writeback. I've already written essays about it, but the fact that we're implementing new mechanism to trigger writeback (see [4]), optimization on the writeback path (see [1]), and the fact that even ZRAM has some writeback mechanism, mean multiple parties are interested. We need to make it better, not digging ourselves into a hole that prevent it from moving forwards.

Quoted 36 lines

I think the primary motivation for ghost swap is to enable zswap usage *without* a backing physical swap device at all — the "No backing
store required" mode described in the cover letter. In that mode,
writeback never happens (there is no physical device to write to), so the swapoff and reclaim questions do not arise. Ghost writeback to a physical device is an optional extension for users who want the safety net of disk-backed swap. Surely we can discuss and decide if we need add it or not.

Quoted 18 lines

Similarly, say the physical swapfile is full, and you attempt to reclaim a physical swap slot that is swap-cache-only (swap count = 0). Say this physical swap slot was allocated from a previous zswap writeback attempt, i.e it has a redirection entry in the ghost zswap swapfile associated with it. Did we handle it somehow?

My apologies if I missed it somehow.

Quoted 9 lines

3. Swap-in from ghost: swap_read_folio() detects the ghost device (SWP_GHOST), resolves the Redirect to the physical swp_entry_t, and forwards the read I/O to the physical device.

4. When the swap cache folio is removed, the Redirect entry is restored from an xarray (redirect_xa) that persists the mapping, so subsequent reads still find the physical backing.

Quoted 8 lines

Quoted 2 lines

5. When the physical slot is freed, the ghost slot is also freed, cascading through swap_range_free().

I don't get how *any* of this is simpler from what I had. If anything, it's more complicated. With a vswap device, you just allocate a physical swap slot (which you also did), change the backend, submit the IO (which you also did), and that's it. No need to juggle this redirect_xa entry. What does this redirect_xa buy us?

Quoted 6 lines

The redirect_xa persists the ghost→physical mapping across swap-cache folio teardown.

Ghost: redirect_xa[offset] = phys (only allocated when Redirect
is active) + swap_table[slot] = Redirect(phys) once the swap cache folio is removed

So it's basically the same thing as vswap's vtable, which houses the redirection, except:

1. It's not cluster-partitioned, so more lock contention.

2. It only holds the redirection temporarily. Which complicates the logic, because now you have to add more logic to handle the transition of the redirection entry to the ghost swapfile's swap table?

So it's potentially less efficient, and logic-wise, just as if not more complicated?

Quoted 10 lines

Quoted 9 lines

Not to mention now it entagles generic swap logic into backend-specific code, unnecessarily. And we have more state in the state machine to consider: With a generic vswap device, vswap -> disk swap directly is the same state as vswap -> written back zswap entry. With this zswap ghost swapfile design, you have to treat the latter separately, as there is a redirection entry involved, which means concurrent swap operations interact with it.

Quoted 7 lines

I'd argue the opposite. Ghost swap touches the generic swap paths in very few places:

- swap_read_folio: one SWP_GHOST branch → resolve Redirect
- __swap_cluster_free_entries: one Redirect check → cascade free
- __swap_cache_do_del_folio: one redirect_xa lookup → restore
- zswap_writeback_entry: one SWP_GHOST branch → ghost writeback

That's because you just don't handle all the cases required to make it correct (rmap) and efficient (per-device per-CPU allocation caching) :)

Quoted 7 lines

Ghost swap only affects behavior when a ghost device has been explicitly swapon'd, and even then only on the paths listed above.

This is different than virtual swap which is more ambitious. As I said, ghost swapfile focuses on the problem itself, and based on the swap table.

I'm considering adding a sysctl knob to enable/disable vswap at runtime in the next version of my patch series. This is trivial, as even with CONFIG_VSWAP, both the new and the old code paths already co-exist (as I still keep the old code path as a potential fallback for the cases when vswap slot allocation fails for whatever reason).

That means that you can get functionally the same thing here if you want - the old behavior, even with CONFIG_VSWAP.

Quoted 7 lines

Quoted 4 lines

2. No reverse mapping, or at least some mechanisms to handle the case where you have to go from physical swap slots to the upper layer.

Acknowledged — a reverse mapping can be added in a future version if writeback-to-physical becomes a priority.

More like, if you want writeback-to-physical *correct* at all.

Quoted 12 lines

Quoted 11 lines

3. Charging behavior difference, and deduplication of metadata when we have multiple swap levels. (In fact, you seem to be double charging a writtenback entry here towards the swap counter, not just duplicating the metadata space)

4. Infrastructure to make sure allocation from this swap device and allocation from the backing swap device does not invalidate each other's per-cpu allocation cache. You seem to just go for the slow path with alloc_phys_swap_slot() for zswap writeback?

Quoted 9 lines

Yes, alloc_phys_swap_slot() uses the slow-path allocator (cluster_alloc_swap_entry), which is correct for this low-frequency operation. zswap writeback is not a hot allocation path — it runs from the zswap shrinker under memory pressure, typically hundreds to thousands of operations per second at most. The fast-path per-CPU caches remain dedicated to folio_alloc_swap(), which handles the high-frequency swapout path. There is no cache invalidation problem because the two paths use different allocation mechanisms on different devices.

I'm nooot entirely sure about this. Zswap writeback can be quite a hot path - there is work on optimizing it (see [1] for a recent attempt to limit the expensive rstat flushing here - and this flushing is *per shrinker iteration*, whereas the physical device allocation will be *per writeback*, which is much, much more frequent).

Quoted 7 lines

Quoted 6 lines

It's fine to disagree with any or all of these design choices of mine
- I'm receptive to discussion. But it's misleading to say you have a simpler design here, IMHO. My design choices resulted from multiple thoughtful iterations (a lot of them could be found in my pre-swap-table version of vswap) and real production pains.

Quoted 8 lines

You're right that "simpler" needs qualification, and I'll update the cover letter accordingly. What I meant — and what I should have stated explicitly — is that ghost swap is less invasive to the existing swap infrastructure:

- It does not require a Kconfig option.
- It does not create a boot-time device that lives for the kernel's lifetime.

I don't think these 2 are significant.

Quoted 2 lines

- It does not introduce a new per-cluster data structure (swap_cluster_info_dynamic / virtual_table).

Instead, you add a redirect_xa xarray, which, as I noted above, plays a *very* similar role (store the redirection), except a bit worse performance-wise and more complicated.

Again, unless you're willing to drop writeback :)

Quoted 1 line

- It does not add a separate per-CPU allocation cache.

This is *by design*, because I want to avoid writeback invalidating the main allocation path.

If the argument is that it is not a problem, then I'm happy to drop it. But I don't think that's the case - in fact, as Kairui have pointed out, what we're moving towards is *every* swap device having its own per-CPU allocation cache (see [2]). It's really necessary, not just for vswap, but also for swap tiering (see [3]). So i don't think the argument (that we don't need separate caching) here is correct, and it's also immaterial soon enough (once we have generic per-device per-CPU allocation cache I'll drop the vswap-specific one).

Quoted 3 lines

- It does not split the swap entry namespace into "vswap" vs "physical" — a ghost device is just another swap device with one extra flag and one extra swap-table entry type.

Actually, if we're to take in your full design, it's *two* entry types. It's the redirection entry (if you want to support writeback) + the Pointer entry (for your zswap "optimization").

Quoted 8 lines

I think VSS goes beyond solving the "zswap without backing store" problem — it is fundamentally an architectural restructuring of the swap subsystem, with virtual swap as a first-class abstraction layered above physical devices. That is both its strength (comprehensive, future-proof) and its trade-off (more invasive to existing paths). I respect that.

Quoted 4 lines

My goal with ghost swap is different: address the immediate pain point with the smallest possible change to the existing infrastructure. Both approaches have value, and I hope the community discussion will help identify the right balance.

That's because I believe "zswap without backing store" just by itself is a bit narrow of a problem. It's certainly nice to fix, but it's more of a nuisance - you already have userspace hacks for it (as I mentioned above).

The problem I'm trying to solve is to support:

1. Writeback.

2. Dynamicity.

3. Decoupled backends.

all of which are motivated by real production issues, not some theoretical problems. I'm concerned that if we only focus on the third goal, we'll dig ourself into a hole that prevents us from solving 1 and 2 efficiently down the line.

Quoted 4 lines

Thank you again for the detailed review — it has already clarified which parts of the design need more thought and documentation, and will make the next version stronger.

I really don't want to be antagonistic, but I hope you'd take the real production pains that we've had (and have tried to communicate in multiple mailing threads, across a timeline of almost 2 years at this point) seriously.

Collaborations should go both ways. I've gone out of my way to try to address the concerns of various parties, from spending *multiple weeks* testing and investigating performance regression on zram backend (which my company does not use), to a rewrite/re-design of virtual swap to accommodate parties who wished to opt out of virtual swap for now.

I hope you can extend the same good will to our needs :) I've included you (and other swap folks such as Chris and Kairui) in my cc-lists. If you have concerns, you could have commented. Instead, you decided to send a patch series, which is basically just the ghost swapfile, with a bit of afterthought to handle writeback and dynamicity, rather inefficiently (no dedicated per-CPU caching), and not even correctly (the lack of rmap means swapoff / swap-cache-only physical swap slot reclaim is broken).

I respect you and Chris very much to assume bad faith, but please work with me rather than against me.

On Thu, Jul 9, 2026 at 4:39 AM Baoquan He <[email protected]> wrote:

Quoted 11 lines

I think VSS goes beyond solving the "zswap without backing store" problem — it is fundamentally an architectural restructuring of the swap subsystem, with virtual swap as a first-class abstraction layered above physical devices. That is both its strength (comprehensive, future-proof) and its trade-off (more invasive to existing paths). I respect that.

My goal with ghost swap is different: address the immediate pain point with the smallest possible change to the existing infrastructure. Both approaches have value, and I hope the community discussion will help identify the right balance.

That's because I believe "zswap without backing store" just by itself is a bit narrow of a problem. It's certainly nice to fix, but it's more of a nuisance - you already have userspace hacks for it (as I mentioned above).

The problem I'm trying to solve is to support:

1. Writeback.

2. Dynamicity.

3. Decoupled backends.

all of which are motivated by real production issues, not some theoretical problems. I'm concerned that if we only focus on the third goal, we'll dig ourself into a hole that prevents us from solving 1 and 2 efficiently down the line.

Quoted 4 lines

Thank you again for the detailed review — it has already clarified which parts of the design need more thought and documentation, and will make the next version stronger.

I really don't want to be antagonistic, but I hope you'd take the real production pains that we've had (and have tried to communicate in multiple mailing threads, across a timeline of almost 2 years at this point) seriously.

Collaborations should go both ways. I've gone out of my way to try to address the concerns of various parties, from spending *multiple weeks* testing and investigating performance regression on zram backend (which my company does not use), to a rewrite/re-design of virtual swap to accommodate parties who wished to opt out of virtual swap for now.

I hope you can extend the same good will to our needs :) I've included you (and other swap folks such as Chris and Kairui) in my cc-lists. If you have concerns, you could have commented. Instead, you decided to send a patch series, which is basically just the ghost swapfile, with a bit of afterthought to handle writeback and dynamicity, rather inefficiently (no dedicated per-CPU caching), and not even correctly (the lack of rmap means swapoff / swap-cache-only physical swap slot reclaim is broken).

I respect you and Chris very much to assume bad faith, but please work with me rather than against me.

Hi Nhat,

Quoted 1 line · Nhat Pham

On Thu, Jul 9, 2026 at 4:39 AM Baoquan He <[email protected]> wrote:

...snip...

Sorry, I won't reply to each comment individually. I'll just leave a summary reply at the end.

Quoted 56 lines

Quoted 11 lines

I think VSS goes beyond solving the "zswap without backing store" problem — it is fundamentally an architectural restructuring of the swap subsystem, with virtual swap as a first-class abstraction layered above physical devices. That is both its strength (comprehensive, future-proof) and its trade-off (more invasive to existing paths). I respect that.

My goal with ghost swap is different: address the immediate pain point with the smallest possible change to the existing infrastructure. Both approaches have value, and I hope the community discussion will help identify the right balance.

That's because I believe "zswap without backing store" just by itself is a bit narrow of a problem. It's certainly nice to fix, but it's more of a nuisance - you already have userspace hacks for it (as I mentioned above).

The problem I'm trying to solve is to support:

1. Writeback.

2. Dynamicity.

3. Decoupled backends.

all of which are motivated by real production issues, not some theoretical problems. I'm concerned that if we only focus on the third goal, we'll dig ourself into a hole that prevents us from solving 1 and 2 efficiently down the line.

Quoted 4 lines

Thank you again for the detailed review — it has already clarified which parts of the design need more thought and documentation, and will make the next version stronger.

I really don't want to be antagonistic, but I hope you'd take the real production pains that we've had (and have tried to communicate in multiple mailing threads, across a timeline of almost 2 years at this point) seriously.

Collaborations should go both ways. I've gone out of my way to try to address the concerns of various parties, from spending *multiple weeks* testing and investigating performance regression on zram backend (which my company does not use), to a rewrite/re-design of virtual swap to accommodate parties who wished to opt out of virtual swap for now.

I hope you can extend the same good will to our needs :) I've included you (and other swap folks such as Chris and Kairui) in my cc-lists. If you have concerns, you could have commented. Instead, you decided to send a patch series, which is basically just the ghost swapfile, with a bit of afterthought to handle writeback and dynamicity, rather inefficiently (no dedicated per-CPU caching), and not even correctly (the lack of rmap means swapoff / swap-cache-only physical swap slot reclaim is broken).

Quoted 2 lines

I respect you and Chris very much to assume bad faith, but please work with me rather than against me.

Before I get into the technical discussion, I want to address something you raised at the end of your reply — the concern that I'm working against you rather than with you. Our paths in MM simply haven't crossed much until now. I can't think of reason why I would want to target you or your work.

I respect the work you've put into VSS. I did check it. Honestly I am not fan of it. In your v2: 2244 insertions, 250 deletions, 15 files touched, a new 455-line header (vswap.h). While swap subsystem has just absorbed the swap table series. Then another 2200 lines of architectural restructuring on top is added. Not sure if anybody raise concern about your solution, or anybody suggests other directions.

I posted ghost swap because I saw a concrete, narrow problem that I believed could be solved with a small change. It makes decoupling and dynamic growth with the minimum possible mechanism. Writeback is explicitly deferred
— not because it's unimportant, but because it's a separate problem that
shouldn't block the common case. With my shallow knowledge, it's optimal solution to the encountered problem. If you agree, you can change to

Otherwise, please continue to push virtual swap forward. I'll stop and wait.

Thanks
Baoquan

Quoted 74 lines · Baoquan He

Hi Nhat,

Quoted 1 line · Nhat Pham

On Thu, Jul 9, 2026 at 4:39 AM Baoquan He <[email protected]> wrote:

...snip...

Sorry, I won't reply to each comment individually. I'll just leave a summary reply at the end.

Quoted 56 lines

Quoted 11 lines

I think VSS goes beyond solving the "zswap without backing store" problem — it is fundamentally an architectural restructuring of the swap subsystem, with virtual swap as a first-class abstraction layered above physical devices. That is both its strength (comprehensive, future-proof) and its trade-off (more invasive to existing paths). I respect that.

My goal with ghost swap is different: address the immediate pain point with the smallest possible change to the existing infrastructure. Both approaches have value, and I hope the community discussion will help identify the right balance.

That's because I believe "zswap without backing store" just by itself is a bit narrow of a problem. It's certainly nice to fix, but it's more of a nuisance - you already have userspace hacks for it (as I mentioned above).

The problem I'm trying to solve is to support:

1. Writeback.

2. Dynamicity.

3. Decoupled backends.

all of which are motivated by real production issues, not some theoretical problems. I'm concerned that if we only focus on the third goal, we'll dig ourself into a hole that prevents us from solving 1 and 2 efficiently down the line.

Quoted 4 lines

Thank you again for the detailed review — it has already clarified which parts of the design need more thought and documentation, and will make the next version stronger.

I really don't want to be antagonistic, but I hope you'd take the real production pains that we've had (and have tried to communicate in multiple mailing threads, across a timeline of almost 2 years at this point) seriously.

Collaborations should go both ways. I've gone out of my way to try to address the concerns of various parties, from spending *multiple weeks* testing and investigating performance regression on zram backend (which my company does not use), to a rewrite/re-design of virtual swap to accommodate parties who wished to opt out of virtual swap for now.

I hope you can extend the same good will to our needs :) I've included you (and other swap folks such as Chris and Kairui) in my cc-lists. If you have concerns, you could have commented. Instead, you decided to send a patch series, which is basically just the ghost swapfile, with a bit of afterthought to handle writeback and dynamicity, rather inefficiently (no dedicated per-CPU caching), and not even correctly (the lack of rmap means swapoff / swap-cache-only physical swap slot reclaim is broken).

Quoted 2 lines

I respect you and Chris very much to assume bad faith, but please work with me rather than against me.

Before I get into the technical discussion, I want to address something you raised at the end of your reply — the concern that I'm working against you rather than with you. Our paths in MM simply haven't crossed much until now. I can't think of reason why I would want to target you or your work.

Quoted 6 lines

I respect the work you've put into VSS. I did check it. Honestly I am not fan of it. In your v2: 2244 insertions, 250 deletions, 15 files touched, a new 455-line header (vswap.h). While swap subsystem has just absorbed the swap table series. Then another 2200 lines of architectural restructuring on top is added. Not sure if anybody raise concern about your solution, or anybody suggests other directions.

Quoted 7 lines

I posted ghost swap because I saw a concrete, narrow problem that I believed could be solved with a small change. It makes decoupling and dynamic growth with the minimum possible mechanism. Writeback is explicitly deferred
— not because it's unimportant, but because it's a separate problem that
shouldn't block the common case. With my shallow knowledge, it's optimal solution to the encountered problem. If you agree, you can change to

Part of above sentence is missing. I meant if you agree on the ghost solution, you can change to take its way and continue, just add me to CC. Or we can work together.

Quoted 3 lines

Otherwise, please continue to push virtual swap forward. I'll stop and wait.

Quoted 76 lines · Baoquan He

Quoted 74 lines · Baoquan He

Hi Nhat,

Quoted 1 line · Nhat Pham

On Thu, Jul 9, 2026 at 4:39 AM Baoquan He <[email protected]> wrote:

...snip...

Sorry, I won't reply to each comment individually. I'll just leave a summary reply at the end.

Quoted 56 lines

Quoted 11 lines

I think VSS goes beyond solving the "zswap without backing store" problem — it is fundamentally an architectural restructuring of the swap subsystem, with virtual swap as a first-class abstraction layered above physical devices. That is both its strength (comprehensive, future-proof) and its trade-off (more invasive to existing paths). I respect that.

My goal with ghost swap is different: address the immediate pain point with the smallest possible change to the existing infrastructure. Both approaches have value, and I hope the community discussion will help identify the right balance.

That's because I believe "zswap without backing store" just by itself is a bit narrow of a problem. It's certainly nice to fix, but it's more of a nuisance - you already have userspace hacks for it (as I mentioned above).

The problem I'm trying to solve is to support:

1. Writeback.

2. Dynamicity.

3. Decoupled backends.

all of which are motivated by real production issues, not some theoretical problems. I'm concerned that if we only focus on the third goal, we'll dig ourself into a hole that prevents us from solving 1 and 2 efficiently down the line.

Quoted 4 lines

Thank you again for the detailed review — it has already clarified which parts of the design need more thought and documentation, and will make the next version stronger.

I really don't want to be antagonistic, but I hope you'd take the real production pains that we've had (and have tried to communicate in multiple mailing threads, across a timeline of almost 2 years at this point) seriously.

Collaborations should go both ways. I've gone out of my way to try to address the concerns of various parties, from spending *multiple weeks* testing and investigating performance regression on zram backend (which my company does not use), to a rewrite/re-design of virtual swap to accommodate parties who wished to opt out of virtual swap for now.

I hope you can extend the same good will to our needs :) I've included you (and other swap folks such as Chris and Kairui) in my cc-lists. If you have concerns, you could have commented. Instead, you decided to send a patch series, which is basically just the ghost swapfile, with a bit of afterthought to handle writeback and dynamicity, rather inefficiently (no dedicated per-CPU caching), and not even correctly (the lack of rmap means swapoff / swap-cache-only physical swap slot reclaim is broken).

Quoted 2 lines

I respect you and Chris very much to assume bad faith, but please work with me rather than against me.

Before I get into the technical discussion, I want to address something you raised at the end of your reply — the concern that I'm working against you rather than with you. Our paths in MM simply haven't crossed much until now. I can't think of reason why I would want to target you or your work.

Quoted 6 lines

Quoted 6 lines

I respect the work you've put into VSS. I did check it. Honestly I am not fan of it. In your v2: 2244 insertions, 250 deletions, 15 files touched, a new 455-line header (vswap.h). While swap subsystem has just absorbed the swap table series. Then another 2200 lines of architectural restructuring on top is added. Not sure if anybody raise concern about your solution, or anybody suggests other directions.

Yes, it's a big patch series. But as you pointed out earlier, it solves a larger problem with a single coherent design :)

Quoted 7 lines

Quoted 7 lines

I posted ghost swap because I saw a concrete, narrow problem that I believed could be solved with a small change. It makes decoupling and dynamic growth with the minimum possible mechanism. Writeback is explicitly deferred
— not because it's unimportant, but because it's a separate problem that
shouldn't block the common case. With my shallow knowledge, it's optimal solution to the encountered problem. If you agree, you can change to

The problem is not the lack of writeback support per se. I understand that sometimes pieces of a design can be deferred to a follow-up patch series.

I'm more concerned that the ghost direction so far does not seriously take it into account in the design at all, which might make it either impossible or very hard to redesign around down the line. I don't want to rip a huge chunk of it out down the line later - we just avoided it with the swap table with Kairui's proposal.

Quoted 4 lines

Part of above sentence is missing. I meant if you agree on the ghost solution, you can change to take its way and continue, just add me to CC. Or we can work together.

How about this - let's solve one piece of the puzzle together, one at a time.

First, your optimization patch series. Have you taken a look at my proposal ([1])? I believe that design should be cleaner, while still allowing you to put it all you want to get (elimination of xarray being the chief benefit), without requiring you to modify swap cache and swap count operations (since you don't have to move the metadata in and out of struct zswap_entry). It does require a bit more code to dynamically allocated the backend array, but should be trivial IMHO.

[1]: https://lore.kernel.org/all/CAKEwX=OvR7GbU_9f2h_MtU4m0g6s-esHmNQKYNhJz610M0P3Sw@mail.gmail.com/

Next, if you just want pure ghost swapfile without writeback support, maybe something similar to the first 2 patches in my RFC? i.e up to this patch:

https://lore.kernel.org/all/[email protected]/

I'm sure you could find issues/bugs in it too with some extensive testing (a lot of my performance testing is done on the entire series as a whole), but from a design point of view it should achieve what us both want, no? It's literally just a ghost swap device - most of the other design elements of vswap (the new charging behavior, rmap, support for writeback) comes in latter patches. I think the difference is:

1. It's transparently dynamic, whereas in your RFC it's driven by a userspace knob. Do you think there are inherent values in a userspace knob?

2. The address space can also shrink when swap usage drops at runtime.

3. The dynamic address space is backed by an xarray. Note that it's structured differently than zswap's one (it's an xarray pointing to clusters). If you have rationales or numbers to show that vmalloc array is superior (while still support the kernel-driven expansion and shrink), please let me know.

If you have no faith in me and do not like my code, you can take the code as a starting point, or just the idea, and drive it yourself. Or let me know if you're unhappy with any piece of it - we can discuss further.

Quoted 3 lines

Quoted 3 lines

Otherwise, please continue to push virtual swap forward. I'll stop and wait.

Look man, I care about the problems being solved and I'm a sucker for good ideas, but I'm not married to anything, be it my code or my ideas.

I have literally dropped my old code and adopted Kairui's suggestions after playing with it and liking it, to make sure every party's concern is met.

If you have concerns, please let me know, or keep sending code if you

Quoted 121 lines · Nhat Pham

Quoted 76 lines · Baoquan He

Quoted 74 lines · Baoquan He

Hi Nhat,

Quoted 1 line · Nhat Pham

On Thu, Jul 9, 2026 at 4:39 AM Baoquan He <[email protected]> wrote:

...snip...

Sorry, I won't reply to each comment individually. I'll just leave a summary reply at the end.

Quoted 56 lines

Quoted 11 lines

I think VSS goes beyond solving the "zswap without backing store" problem — it is fundamentally an architectural restructuring of the swap subsystem, with virtual swap as a first-class abstraction layered above physical devices. That is both its strength (comprehensive, future-proof) and its trade-off (more invasive to existing paths). I respect that.

My goal with ghost swap is different: address the immediate pain point with the smallest possible change to the existing infrastructure. Both approaches have value, and I hope the community discussion will help identify the right balance.

That's because I believe "zswap without backing store" just by itself is a bit narrow of a problem. It's certainly nice to fix, but it's more of a nuisance - you already have userspace hacks for it (as I mentioned above).

The problem I'm trying to solve is to support:

1. Writeback.

2. Dynamicity.

3. Decoupled backends.

all of which are motivated by real production issues, not some theoretical problems. I'm concerned that if we only focus on the third goal, we'll dig ourself into a hole that prevents us from solving 1 and 2 efficiently down the line.

Quoted 4 lines

Thank you again for the detailed review — it has already clarified which parts of the design need more thought and documentation, and will make the next version stronger.

I really don't want to be antagonistic, but I hope you'd take the real production pains that we've had (and have tried to communicate in multiple mailing threads, across a timeline of almost 2 years at this point) seriously.

Collaborations should go both ways. I've gone out of my way to try to address the concerns of various parties, from spending *multiple weeks* testing and investigating performance regression on zram backend (which my company does not use), to a rewrite/re-design of virtual swap to accommodate parties who wished to opt out of virtual swap for now.

I hope you can extend the same good will to our needs :) I've included you (and other swap folks such as Chris and Kairui) in my cc-lists. If you have concerns, you could have commented. Instead, you decided to send a patch series, which is basically just the ghost swapfile, with a bit of afterthought to handle writeback and dynamicity, rather inefficiently (no dedicated per-CPU caching), and not even correctly (the lack of rmap means swapoff / swap-cache-only physical swap slot reclaim is broken).

Quoted 2 lines

I respect you and Chris very much to assume bad faith, but please work with me rather than against me.

Before I get into the technical discussion, I want to address something you raised at the end of your reply — the concern that I'm working against you rather than with you. Our paths in MM simply haven't crossed much until now. I can't think of reason why I would want to target you or your work.

Quoted 6 lines

Quoted 6 lines

I respect the work you've put into VSS. I did check it. Honestly I am not fan of it. In your v2: 2244 insertions, 250 deletions, 15 files touched, a new 455-line header (vswap.h). While swap subsystem has just absorbed the swap table series. Then another 2200 lines of architectural restructuring on top is added. Not sure if anybody raise concern about your solution, or anybody suggests other directions.

Yes, it's a big patch series. But as you pointed out earlier, it solves a larger problem with a single coherent design :)

Quoted 7 lines

Quoted 7 lines

I posted ghost swap because I saw a concrete, narrow problem that I believed could be solved with a small change. It makes decoupling and dynamic growth with the minimum possible mechanism. Writeback is explicitly deferred
— not because it's unimportant, but because it's a separate problem that
shouldn't block the common case. With my shallow knowledge, it's optimal solution to the encountered problem. If you agree, you can change to

The problem is not the lack of writeback support per se. I understand that sometimes pieces of a design can be deferred to a follow-up patch series.

I'm more concerned that the ghost direction so far does not seriously take it into account in the design at all, which might make it either impossible or very hard to redesign around down the line. I don't want to rip a huge chunk of it out down the line later - we just avoided it with the swap table with Kairui's proposal.

Quoted 4 lines

Part of above sentence is missing. I meant if you agree on the ghost solution, you can change to take its way and continue, just add me to CC. Or we can work together.

How about this - let's solve one piece of the puzzle together, one at a time.

First, your optimization patch series. Have you taken a look at my proposal ([1])? I believe that design should be cleaner, while still allowing you to put it all you want to get (elimination of xarray being the chief benefit), without requiring you to modify swap cache and swap count operations (since you don't have to move the metadata in and out of struct zswap_entry). It does require a bit more code to dynamically allocated the backend array, but should be trivial IMHO.

Quoted 1 line

[1]: https://lore.kernel.org/all/CAKEwX=OvR7GbU_9f2h_MtU4m0g6s-esHmNQKYNhJz610M0P3Sw@mail.gmail.com/

Hi Nhat,

Sorry for the late reply. I've been busy and just saw this. First impression looks promising, it is better than any previous version of VS. Thanks for incorporating Kairui's feedback. Please give me some time to sleep on it. I will get back to you on that.

Quoted 40 lines

Next, if you just want pure ghost swapfile without writeback support, maybe something similar to the first 2 patches in my RFC? i.e up to this patch:

https://lore.kernel.org/all/[email protected]/

I'm sure you could find issues/bugs in it too with some extensive testing (a lot of my performance testing is done on the entire series as a whole), but from a design point of view it should achieve what us both want, no? It's literally just a ghost swap device - most of the other design elements of vswap (the new charging behavior, rmap, support for writeback) comes in latter patches. I think the difference is:

1. It's transparently dynamic, whereas in your RFC it's driven by a userspace knob. Do you think there are inherent values in a userspace knob?

2. The address space can also shrink when swap usage drops at runtime.

3. The dynamic address space is backed by an xarray. Note that it's structured differently than zswap's one (it's an xarray pointing to clusters). If you have rationales or numbers to show that vmalloc array is superior (while still support the kernel-driven expansion and shrink), please let me know.

If you have no faith in me and do not like my code, you can take the code as a starting point, or just the idea, and drive it yourself. Or let me know if you're unhappy with any piece of it - we can discuss further.

Quoted 3 lines

Quoted 3 lines

Otherwise, please continue to push virtual swap forward. I'll stop and wait.

Look man, I care about the problems being solved and I'm a sucker for good ideas, but I'm not married to anything, be it my code or my ideas.

Quoted 5 lines

I have literally dropped my old code and adopted Kairui's suggestions after playing with it and liking it, to make sure every party's concern is met.

If you have concerns, please let me know, or keep sending code if you

May I add one wishlist item related to ghost swapfile here? I will reply to your VS thread for other feedback.

For the ghost swapfile usage case (no writeback). Can VS support this usage case while maintaining per-swap-slot metadata usage comparable to the ghost swapfile implementation? If we have to put a number to it, let's say the increase in metadata is less than 1 byte per swap slot compared to the ghost swapfile implementation.

We have large deployments of ghost swapfile in the fleet. I want an upstream aligned solution that does not increase the existing memory overhead compared to the current ghost swapfile deployment, for that usage. As you know, memory pricing and optimization pressure are high. The ghost swapfile patch itself is relatively simple. If VS can address this usage case, it will remove a lot of incentive to explore ghost swapfile-like solutions.

Thanks

Chris

On Wed, Jul 15, 2026 at 4:04 PM Chris Li <[email protected]> wrote:

Hi Chris. I have some thoughts on this - let me know what you think.

I understand the memory overhead discussion here. In the case where vswap slot is backed by a real swapfile, I have not figured out a way to make the overhead disappear yet. The damn rmap is very annoying - I have some strategies to get around it for certain operations, but none of it can be done without a full fledged patch series of its own.

However, with the ghost zswap swapfile backend specifically, I think

Quoted 123 lines · Chris Li

Quoted 121 lines · Nhat Pham

Quoted 76 lines · Baoquan He

Quoted 74 lines · Baoquan He

Hi Nhat,

Quoted 1 line · Nhat Pham

On Thu, Jul 9, 2026 at 4:39 AM Baoquan He <[email protected]> wrote:

...snip...

Sorry, I won't reply to each comment individually. I'll just leave a summary reply at the end.

Quoted 56 lines

Quoted 11 lines

I think VSS goes beyond solving the "zswap without backing store" problem — it is fundamentally an architectural restructuring of the swap subsystem, with virtual swap as a first-class abstraction layered above physical devices. That is both its strength (comprehensive, future-proof) and its trade-off (more invasive to existing paths). I respect that.

My goal with ghost swap is different: address the immediate pain point with the smallest possible change to the existing infrastructure. Both approaches have value, and I hope the community discussion will help identify the right balance.

That's because I believe "zswap without backing store" just by itself is a bit narrow of a problem. It's certainly nice to fix, but it's more of a nuisance - you already have userspace hacks for it (as I mentioned above).

The problem I'm trying to solve is to support:

1. Writeback.

2. Dynamicity.

3. Decoupled backends.

all of which are motivated by real production issues, not some theoretical problems. I'm concerned that if we only focus on the third goal, we'll dig ourself into a hole that prevents us from solving 1 and 2 efficiently down the line.

Quoted 4 lines

Thank you again for the detailed review — it has already clarified which parts of the design need more thought and documentation, and will make the next version stronger.

I really don't want to be antagonistic, but I hope you'd take the real production pains that we've had (and have tried to communicate in multiple mailing threads, across a timeline of almost 2 years at this point) seriously.

Collaborations should go both ways. I've gone out of my way to try to address the concerns of various parties, from spending *multiple weeks* testing and investigating performance regression on zram backend (which my company does not use), to a rewrite/re-design of virtual swap to accommodate parties who wished to opt out of virtual swap for now.

I hope you can extend the same good will to our needs :) I've included you (and other swap folks such as Chris and Kairui) in my cc-lists. If you have concerns, you could have commented. Instead, you decided to send a patch series, which is basically just the ghost swapfile, with a bit of afterthought to handle writeback and dynamicity, rather inefficiently (no dedicated per-CPU caching), and not even correctly (the lack of rmap means swapoff / swap-cache-only physical swap slot reclaim is broken).

Quoted 2 lines

I respect you and Chris very much to assume bad faith, but please work with me rather than against me.

Before I get into the technical discussion, I want to address something you raised at the end of your reply — the concern that I'm working against you rather than with you. Our paths in MM simply haven't crossed much until now. I can't think of reason why I would want to target you or your work.

Quoted 6 lines

Quoted 6 lines

I respect the work you've put into VSS. I did check it. Honestly I am not fan of it. In your v2: 2244 insertions, 250 deletions, 15 files touched, a new 455-line header (vswap.h). While swap subsystem has just absorbed the swap table series. Then another 2200 lines of architectural restructuring on top is added. Not sure if anybody raise concern about your solution, or anybody suggests other directions.

Yes, it's a big patch series. But as you pointed out earlier, it solves a larger problem with a single coherent design :)

Quoted 7 lines

Quoted 7 lines

I posted ghost swap because I saw a concrete, narrow problem that I believed could be solved with a small change. It makes decoupling and dynamic growth with the minimum possible mechanism. Writeback is explicitly deferred
— not because it's unimportant, but because it's a separate problem that
shouldn't block the common case. With my shallow knowledge, it's optimal solution to the encountered problem. If you agree, you can change to

The problem is not the lack of writeback support per se. I understand that sometimes pieces of a design can be deferred to a follow-up patch series.

I'm more concerned that the ghost direction so far does not seriously take it into account in the design at all, which might make it either impossible or very hard to redesign around down the line. I don't want to rip a huge chunk of it out down the line later - we just avoided it with the swap table with Kairui's proposal.

Quoted 4 lines

Part of above sentence is missing. I meant if you agree on the ghost solution, you can change to take its way and continue, just add me to CC. Or we can work together.

How about this - let's solve one piece of the puzzle together, one at a time.

First, your optimization patch series. Have you taken a look at my proposal ([1])? I believe that design should be cleaner, while still allowing you to put it all you want to get (elimination of xarray being the chief benefit), without requiring you to modify swap cache and swap count operations (since you don't have to move the metadata in and out of struct zswap_entry). It does require a bit more code to dynamically allocated the backend array, but should be trivial IMHO.

Quoted 8 lines

Quoted 1 line

[1]: https://lore.kernel.org/all/CAKEwX=OvR7GbU_9f2h_MtU4m0g6s-esHmNQKYNhJz610M0P3Sw@mail.gmail.com/

Hi Nhat,

Sorry for the late reply. I've been busy and just saw this. First impression looks promising, it is better than any previous version of VS. Thanks for incorporating Kairui's feedback. Please give me some time to sleep on it. I will get back to you on that.

Please take a look. I was waiting for your response as well :)

The patch series is large, yes, but it's an RFC meant to show case the final end design.

Quoted 55 lines

Quoted 40 lines

Next, if you just want pure ghost swapfile without writeback support, maybe something similar to the first 2 patches in my RFC? i.e up to this patch:

https://lore.kernel.org/all/[email protected]/

I'm sure you could find issues/bugs in it too with some extensive testing (a lot of my performance testing is done on the entire series as a whole), but from a design point of view it should achieve what us both want, no? It's literally just a ghost swap device - most of the other design elements of vswap (the new charging behavior, rmap, support for writeback) comes in latter patches. I think the difference is:

1. It's transparently dynamic, whereas in your RFC it's driven by a userspace knob. Do you think there are inherent values in a userspace knob?

2. The address space can also shrink when swap usage drops at runtime.

3. The dynamic address space is backed by an xarray. Note that it's structured differently than zswap's one (it's an xarray pointing to clusters). If you have rationales or numbers to show that vmalloc array is superior (while still support the kernel-driven expansion and shrink), please let me know.

If you have no faith in me and do not like my code, you can take the code as a starting point, or just the idea, and drive it yourself. Or let me know if you're unhappy with any piece of it - we can discuss further.

Quoted 3 lines

Quoted 3 lines

Otherwise, please continue to push virtual swap forward. I'll stop and wait.

Look man, I care about the problems being solved and I'm a sucker for good ideas, but I'm not married to anything, be it my code or my ideas.

Quoted 5 lines

I have literally dropped my old code and adopted Kairui's suggestions after playing with it and liking it, to make sure every party's concern is met.

If you have concerns, please let me know, or keep sending code if you

May I add one wishlist item related to ghost swapfile here? I will reply to your VS thread for other feedback.

For the ghost swapfile usage case (no writeback). Can VS support this usage case while maintaining per-swap-slot metadata usage comparable to the ghost swapfile implementation? If we have to put a number to it, let's say the increase in metadata is less than 1 byte per swap slot compared to the ghost swapfile implementation.

Quoted 7 lines

We have large deployments of ghost swapfile in the fleet. I want an upstream aligned solution that does not increase the existing memory overhead compared to the current ghost swapfile deployment, for that usage. As you know, memory pricing and optimization pressure are high. The ghost swapfile patch itself is relatively simple. If VS can address this usage case, it will remove a lot of incentive to explore ghost swapfile-like solutions.

Hi Chris. I have some thoughts on this - let me know what you think.

I understand the memory overhead discussion here. In the case where vswap slot is backed by a real swapfile, I have not figured out a way to make the overhead disappear yet. The damn rmap is very annoying - I have some strategies to get around it for certain operations, but none of it can be done without a full fledged patch series of its own.

However, with the ghost zswap swapfile backend specifically, I think the overhead is going to be negligible compared to the status quo upstream (things might be different internally at Google - please let me know!). This is because the main remaining per-slot overhead comes in the form of one per-cluster array to store the "backend", here it's going to store struct zswap_entry pointers. This is not extra overhead though, because it basically replaces the xarray (that you added all those years ago)!

The rest is small stuff here and there that is not O(slot) but rather O(cluster). For example one struct rcu_head per cluster, xarray overhead for the dynamicization - note that this xarray manages cluster, not slot. I think that's all, unless I miss something.

So if you don't writeback, i.e no physical swapfile, then I think there is negligible overhead (no rmap and all that jazz).

Does this sound acceptable to you?

Baoquan, what do you think?

On 07/16/26 at 09:49am, Nhat Pham wrote:

Thanks for the thoughtful reply and the very constructive proposal. Your suggestion about moving forward in small, incremental steps in last mail is really appealing — I think that's the right way to make progress on this. I need a little time to think it through more carefully before giving you a proper response. Will get back to you soon.

Thanks
Baoquan

Quoted 9 lines · Baoquan He

On 07/16/26 at 09:49am, Nhat Pham wrote:

Thanks for the thoughtful reply and the very constructive proposal. Your suggestion about moving forward in small, incremental steps in last mail is really appealing — I think that's the right way to make progress on this. I need a little time to think it through more carefully before giving you a proper response. Will get back to you soon.

Yeah let me know what you think. I'm planning to send out v3 with some more cleanups and fixes, but I'm happy to hear what you and Chris wants to make it as useful as possible for everyone.

Quoted 11 lines · Nhat Pham

Quoted 9 lines · Baoquan He

On 07/16/26 at 09:49am, Nhat Pham wrote:

Thanks for the thoughtful reply and the very constructive proposal. Your suggestion about moving forward in small, incremental steps in last mail is really appealing — I think that's the right way to make progress on this. I need a little time to think it through more carefully before giving you a proper response. Will get back to you soon.

Quoted 3 lines

Yeah let me know what you think. I'm planning to send out v3 with some more cleanups and fixes, but I'm happy to hear what you and Chris wants to make it as useful as possible for everyone.

Hi Nhat,

I've thought about it more carefully. Here's what I think the path forward could look like. I'll try to lay it out from the bottom up.

1. Data structure

Instead of introducing a separate swap_cluster_info_dynamic wrapper, I'd prefer to add a per-slot backend pointer array directly into struct swap_cluster_info:

   struct swap_cluster_info {
       ...
   #ifdef CONFIG_ZSWAP
       atomic_long_t *vs_pointer;   /* or xs_pointer — per-slot backend pointers */
   #endif
       struct list_head list;
   };

This is allocated together with or alongside the swap table when the cluster is first used. No wrapper struct, no new ->cluster_info pointer type change — just one extra field that is NULL for clusters that don't need it.

The name is bike-sheddable: vs_pointer (virtual-space pointer) or xs_pointer (cross-space pointer), xs_table (xswap extension table), aux_table (auxiliary table), priv_table (private data table), slot_priv (per-slot private data) or s_ext (slot extension). The point is that it stores a pointer to the backend owner of each slot — initially just struct zswap_entry *, eventually extensible to other backend types. I prefer to name it ext_table, but we have had extend_table. My second favorite option is aux_table. What's your choice?

2. Eliminate the zswap xarray

Once vs_pointer is in place, zswap can store and load entries through vs_pointer[ci_off] instead of xa_store/xa_load on the per-device zswap xarray. The cluster lock protects the access. This is exactly the same idea as your virtual_table, just placed directly in swap_cluster_info rather than in a wrapper.

The benefit: zswap_swapon and zswap_swapoff become no-ops (no global xarray to allocate or tear down). The per-cluster allocation is lazy — only clusters that actually see a zswap store allocate the array.

3. First step for ghost swap / virtual swap: dynamic growth and shrink,

no writeback yet

I think we agree that the first landing should be narrow in scope:
- A virtual swap device that decouples PTE swap entries from physical backing.
- Backed only by zswap (and zero pages) initially.
- No writeback to physical swap.
- Dynamic growth and shrink of the cluster space.

For the dynamic cluster management, I'd like to try a lazy vmalloc
approach: allocate the cluster_info array as a vmalloc region, and use vm_area_map_pages() to map newly populated clusters on demand. Growth is automatic via page faults; shrink means reclaiming the physical pages backing freed clusters.

The vmalloc approach has O(1) lookup (simple array index), which matters on the swapout hot path. The trade-off is that shrink is coarser than xarray-based management (we can't easily unmap an individual cluster from the middle of the vmalloc region). But for the initial landing where swap usage tends to be ratchet-like, I think this is a reasonable trade-off. We can always switch to xarray later if shrink granularity becomes a real issue.

4. Writeback and rmap as follow-up

Once the basic virtual swap device is upstream, we add writeback support:
- When zswap store fails, fall back to allocating a physical swap slot and writing out.
- The vs_pointer entry is updated to point to the physical slot (tagged to distinguish from a zswap_entry pointer).
- rmap for physical -> virtual reverse lookup, needed for swapoff and physical slot reclaim.

This is a natural extension: the vs_pointer is already per-slot, so adding a new backend type is a tag-bit change, not a rewrite.

5. Longer-term: swap tiering

Looking further ahead, the per-slot pointer can also serve as the basis for a swap tiering model. Imagine a memcg with multiple
tiers: ghost/virtual swap at the top, then pmem, then SSD, then HDD. Swapout goes to the top tier first; when that fills, data cascades down to the next tier. Each swap device only needs to know about the device directly below it — a simple "push down" interface.

This is a longer-term vision, but having a uniform per-slot backend pointer from the start keeps the door open without painting us into a corner.

6. Naming

We have three names floating around: "ghost swap", "virtual swap", and Chris suggested "xswap". We're going to be talking about this thing a lot, and having a single agreed-upon name would help avoid confusion in discussions, commit messages, and documentation. I don't have strong feelings about which one — but I'd like us to settle on something everyone can live with.

7. How to swapon a ghost / virtual swap device

There are several approaches on the table right now:

a) A sparse file created via truncate (Chris's approach): a swap file that looks large (e.g. 1GiB) but only has one real backing page. Works with the existing swapon syscall, but the sparse file has to live somewhere and the filesystem interaction is a bit of a hack.

b) A boot-time, always-present device (your VSS approach): the vswap device is created unconditionally at late_initcall when CONFIG_VSWAP=y. No userspace action needed, but no runtime control either — you can't create or destroy it, and you can't have per-memcg or per-workload virtual swap spaces.

c) A swapon flag: something like SWAP_FLAG_VIRTUAL or SWAP_FLAG_GHOST passed to swapon(2), which tells the kernel to treat the swap device as a ghost device with no backing storage.

d) A dedicated sysfs / debugfs knob: create and size the ghost swap device at runtime via a file under /sys/kernel/mm/swap/.

e) Something I'm missing?

Each has trade-offs between flexibility, simplicity, and the existing swap management model. I'd like to hear what you and Chris think — especially whether the boot-time-only nature of the current VSS design is a deliberate simplification or something you'd be open to revisiting.

Curious to hear what you think — especially if you see any structural issues with putting vs_pointer directly in swap_cluster_info rather than in a wrapper. If the wrapper is important for your v3 design, I'd like to understand the rationale better.

Thanks,
Baoquan

Quoted 13 lines · Nhat Pham

Quoted 11 lines · Baoquan He

Quoted 9 lines · Nhat Pham

Thanks for the thoughtful reply and the very constructive proposal. Your suggestion about moving forward in small, incremental steps in last mail is really appealing — I think that's the right way to make progress on this. I need a little time to think it through more carefully before giving you a proper response. Will get back to you soon.

Quoted 4 lines

Quoted 4 lines

Yeah let me know what you think. I'm planning to send out v3 with some more cleanups and fixes, but I'm happy to hear what you and Chris wants to make it as useful as possible for everyone.
I agree with that. The current VS patch is a great improvement over

the previous version of VS. However it places the cluster info and swap table in different locations when accessing swap table-related information. The code path requires different locks as well, the xarray needs to take the xa lock. This kind of dual-personality issue will make reasoning harder. Keeping the swap table in the same place is simpler in my mind.

The other issue I want to add is that, using xarray as a swap allocator is unverified. It has no size limit which is one of its claimed advantages, but also one of its drawbacks. Because it does not have swap_full() concept. It will never do the normal if swap_full() fail to allocate, let's do swap cache related. That code path cannot trigger because the xarray has no size limit.

There are other considerations as well. It works fine if VS only wants to allocate 4K swap entries. If VS later wants to go into the mthp swap entry allocation. I don't think xarray has similar considerations as the current swap allocator regarding fragment lists, etc. The xarray isn't designed to be used that way, at least not in the common usage situation. You are likely running into your xarray allocation specific issue.

The xarray allocator also has a fragmentation issue where not all memory released from the xarray returns to the system as free pages. Using kvmalloc grow it will wait some unused cluster info. But let's put it into perspective: cluster info is shared by 512 swap entries. Each swap entry is 4K. So the memory used by swap cluster is pretty small. Much smaller than 1 byte per swap entry. We can also modify the cluster info array, making it an array of pointers. Then the waste will be even smaller (if unused). At the cost of one more pointer per swap_cluster_info. We can consider all this later, not at this current stage.

Quoted 2 lines

4. Writeback and rmap as follow-up

I agree that adding cluster info growth as the first step. I recall Kairui has an RFC patch series to implement virtual entry lookup (still using the xarray). I think the combination of growth and indirection lookup should provide most of what VS depends on from the core swap point of view.

1. If the page is reclaimable, reclaim path will take care of swap cache too.

2. For non-physical swap users, we actually eagerly attempt to get rid of the swap space (and swap cache) once the final reference is dropped. Zswap load invalidates the compressed copy, so there's not a lot of value in caching. I'm not sure if you missed this or not.

On Tue, Jul 21, 2026 at 11:03 AM Chris Li <[email protected]> wrote:

I'm writing a full response to Baoquan, but there are a couple points I'd like to point out first, to save everyone's from the confusion going forward :)

Quoted 25 lines · Baoquan He

Quoted 13 lines · Nhat Pham

Quoted 11 lines · Baoquan He

Quoted 9 lines · Nhat Pham

Thanks for the thoughtful reply and the very constructive proposal. Your suggestion about moving forward in small, incremental steps in last mail is really appealing — I think that's the right way to make progress on this. I need a little time to think it through more carefully before giving you a proper response. Will get back to you soon.

Quoted 4 lines

Quoted 4 lines

Yeah let me know what you think. I'm planning to send out v3 with some more cleanups and fixes, but I'm happy to hear what you and Chris wants to make it as useful as possible for everyone.
I agree with that. The current VS patch is a great improvement over

the previous version of VS. However it places the cluster info and swap table in different locations when accessing swap table-related information. The code path requires different locks as well, the xarray needs to take the xa lock. This kind of dual-personality issue will make reasoning harder. Keeping the swap table in the same place is simpler in my mind.

Quoted 6 lines

The other issue I want to add is that, using xarray as a swap allocator is unverified. It has no size limit which is one of its claimed advantages, but also one of its drawbacks. Because it does not have swap_full() concept. It will never do the normal if swap_full() fail to allocate, let's do swap cache related. That code path cannot trigger because the xarray has no size limit.

1. If the page is reclaimable, reclaim path will take care of swap cache too.

2. For non-physical swap users, we actually eagerly attempt to get rid of the swap space (and swap cache) once the final reference is dropped. Zswap load invalidates the compressed copy, so there's not a lot of value in caching. I'm not sure if you missed this or not.

Quoted 7 lines

There are other considerations as well. It works fine if VS only wants to allocate 4K swap entries. If VS later wants to go into the mthp swap entry allocation. I don't think xarray has similar considerations as the current swap allocator regarding fragment lists, etc. The xarray isn't designed to be used that way, at least not in the common usage situation. You are likely running into your xarray allocation specific issue.

I don't understand what you're asking here. mTHP swap allocation already works with my RFC.

The xarray maps to 512 entry cluster, not individual swap slots. Once you get a cluster (either from a partial cluster or a new cluster from the xarray), allocation of mTHP works exactly the same as physical swap allocator.

Quoted 10 lines

The xarray allocator also has a fragmentation issue where not all memory released from the xarray returns to the system as free pages. Using kvmalloc grow it will wait some unused cluster info. But let's put it into perspective: cluster info is shared by 512 swap entries. Each swap entry is 4K. So the memory used by swap cluster is pretty small. Much smaller than 1 byte per swap entry. We can also modify the cluster info array, making it an array of pointers. Then the waste will be even smaller (if unused). At the cost of one more pointer per swap_cluster_info. We can consider all this later, not at this current stage.

The xarray also maps each index to a cluster too.

Quoted 7 lines

Quoted 1 line

4. Writeback and rmap as follow-up

I agree that adding cluster info growth as the first step. I recall Kairui has an RFC patch series to implement virtual entry lookup (still using the xarray). I think the combination of growth and indirection lookup should provide most of what VS depends on from the core swap point of view.

That is *exactly* what my RFC is using. Did you see the first patch? I put Co-developed-by Kairui for a reason.

Quoted 8 lines

Quoted 1 line

[1]: https://lore.kernel.org/all/CAKEwX=OvR7GbU_9f2h_MtU4m0g6s-esHmNQKYNhJz610M0P3Sw@mail.gmail.com/

Hi Nhat,

Sorry for the late reply. I've been busy and just saw this. First impression looks promising, it is better than any previous version of VS. Thanks for incorporating Kairui's feedback. Please give me some time to sleep on it. I will get back to you on that.

Please take a look. I was waiting for your response as well :)

The patch series is large, yes, but it's an RFC meant to show case the final end design.

Quoted 9 lines

May I add one wishlist item related to ghost swapfile here? I will reply to your VS thread for other feedback.

For the ghost swapfile usage case (no writeback). Can VS support this usage case while maintaining per-swap-slot metadata usage comparable to the ghost swapfile implementation? If we have to put a number to it, let's say the increase in metadata is less than 1 byte per swap slot compared to the ghost swapfile implementation.

Quoted 7 lines

We have large deployments of ghost swapfile in the fleet. I want an upstream aligned solution that does not increase the existing memory overhead compared to the current ghost swapfile deployment, for that usage. As you know, memory pricing and optimization pressure are high. The ghost swapfile patch itself is relatively simple. If VS can address this usage case, it will remove a lot of incentive to explore ghost swapfile-like solutions.

Hi Chris. I have some thoughts on this - let me know what you think.

I understand the memory overhead discussion here. In the case where vswap slot is backed by a real swapfile, I have not figured out a way to make the overhead disappear yet. The damn rmap is very annoying - I have some strategies to get around it for certain operations, but none of it can be done without a full fledged patch series of its own.

However, with the ghost zswap swapfile backend specifically, I think the overhead is going to be negligible compared to the status quo

upstream (things might be different internally at Google - please let me know!). This is because the main remaining per-slot overhead comes in the form of one per-cluster array to store the "backend", here it's going to store struct zswap_entry pointers. This is not extra overhead though, because it basically replaces the xarray (that you added all those years ago)!

The rest is small stuff here and there that is not O(slot) but rather O(cluster). For example one struct rcu_head per cluster, xarray overhead for the dynamicization - note that this xarray manages cluster, not slot. I think that's all, unless I miss something.

So if you don't writeback, i.e no physical swapfile, then I think there is negligible overhead (no rmap and all that jazz).

Does this sound acceptable to you?

Quoted 9 lines · Nhat Pham

Quoted 8 lines

Quoted 1 line

[1]: https://lore.kernel.org/all/CAKEwX=OvR7GbU_9f2h_MtU4m0g6s-esHmNQKYNhJz610M0P3Sw@mail.gmail.com/

Hi Nhat,

Sorry for the late reply. I've been busy and just saw this. First impression looks promising, it is better than any previous version of VS. Thanks for incorporating Kairui's feedback. Please give me some time to sleep on it. I will get back to you on that.

Quoted 4 lines

Please take a look. I was waiting for your response as well :)

The patch series is large, yes, but it's an RFC meant to show case the final end design.

Understand.

Quoted 25 lines

Quoted 9 lines

May I add one wishlist item related to ghost swapfile here? I will reply to your VS thread for other feedback.

For the ghost swapfile usage case (no writeback). Can VS support this usage case while maintaining per-swap-slot metadata usage comparable to the ghost swapfile implementation? If we have to put a number to it, let's say the increase in metadata is less than 1 byte per swap slot compared to the ghost swapfile implementation.

Quoted 7 lines

We have large deployments of ghost swapfile in the fleet. I want an upstream aligned solution that does not increase the existing memory overhead compared to the current ghost swapfile deployment, for that usage. As you know, memory pricing and optimization pressure are high. The ghost swapfile patch itself is relatively simple. If VS can address this usage case, it will remove a lot of incentive to explore ghost swapfile-like solutions.

Hi Chris. I have some thoughts on this - let me know what you think.

I understand the memory overhead discussion here. In the case where vswap slot is backed by a real swapfile, I have not figured out a way to make the overhead disappear yet. The damn rmap is very annoying - I have some strategies to get around it for certain operations, but none of it can be done without a full fledged patch series of its own.

Quoted 2 lines

However, with the ghost zswap swapfile backend specifically, I think the overhead is going to be negligible compared to the status quo

We might need a clearer definition of negligible. e.g. in my mind less than 1 byte per 4K size swap slot.

Quoted 7 lines

upstream (things might be different internally at Google - please let me know!). This is because the main remaining per-slot overhead comes in the form of one per-cluster array to store the "backend", here it's going to store struct zswap_entry pointers. This is not extra overhead though, because it basically replaces the xarray (that you added all those years ago)!

I hope you are not clinging to that xarray because I want to get rid of that as well.

Quoted 5 lines

The rest is small stuff here and there that is not O(slot) but rather O(cluster). For example one struct rcu_head per cluster, xarray overhead for the dynamicization - note that this xarray manages cluster, not slot. I think that's all, unless I miss something.

I also mean the normal path that accesses the swap table will take an xarray lookup with different locking requirements.
I prefer to keep the swap table and cluster info where they are now to avoid that extra complexity in the mental model.

It seems allowing the cluster array's virtual entry to grow should mostly answer your VS need?

Quoted 5 lines

So if you don't writeback, i.e no physical swapfile, then I think there is negligible overhead (no rmap and all that jazz).

Does this sound acceptable to you?

Regarding the memory size, yes, if the overhead is less than 1 byte per swap slot compared to the ghost swapfile. The location and lookup

path for the swap table and cluster info are not ideal because VS will force a different location and lookup code path. I prefer growing the cluster info array to gain more swap entries and keep the current swap cluster and swap table data structures where they are.

Quoted 2 lines

I hope you are not clinging to that xarray because I want to get rid of that as well.

That's what I meant though. The virtual table replaces it for vswap users.

Quoted 12 lines

Quoted 6 lines

The rest is small stuff here and there that is not O(slot) but rather O(cluster). For example one struct rcu_head per cluster, xarray overhead for the dynamicization - note that this xarray manages cluster, not slot. I think that's all, unless I miss something.

I also mean the normal path that accesses the swap table will take an xarray lookup with different locking requirements.
I prefer to keep the swap table and cluster info where they are now to avoid that extra complexity in the mental model.

Quoted 2 lines

It seems allowing the cluster array's virtual entry to grow should mostly answer your VS need?

I don't get it. What do you mean here?

Quoted 7 lines

Quoted 4 lines

So if you don't writeback, i.e no physical swapfile, then I think there is negligible overhead (no rmap and all that jazz).

Does this sound acceptable to you?

Regarding the memory size, yes, if the overhead is less than 1 byte per swap slot compared to the ghost swapfile. The location and lookup

If you see where I miss, point out. Otherwise I'll take that as a yes? ;)

Quoted 4 lines

path for the swap table and cluster info are not ideal because VS will force a different location and lookup code path. I prefer growing the cluster info array to gain more swap entries and keep the current swap cluster and swap table data structures where they are.

The current swap cluster and swap table data structure are *exactly* where they are. In fact my proposal keep it closer to the status quo -

once you lock the cluster, swap cache and swap count operations are *exactly* the same, whereas Baoquan's approach requires taking swap table information and shove it into the struct zswap entry.

Can you clarify what you meant here? :)

On Wed, Jul 15, 2026 at 7:19 AM Baoquan He <[email protected]> wrote:

Quoted 3 lines

Part of above sentence is missing. I meant if you agree on the ghost solution, you can change to take its way and continue, just add me to CC. Or we can work together.

How about this - let's solve one piece of the puzzle together, one at a time.

First, your optimization patch series. Have you taken a look at my proposal ([1])? I believe that design should be cleaner, while still allowing you to put it all you want to get (elimination of xarray being the chief benefit), without requiring you to modify swap cache and swap count operations (since you don't have to move the metadata in and out of struct zswap_entry). It does require a bit more code to dynamically allocated the backend array, but should be trivial IMHO.

[1]: https://lore.kernel.org/all/CAKEwX=OvR7GbU_9f2h_MtU4m0g6s-esHmNQKYNhJz610M0P3Sw@mail.gmail.com/

Next, if you just want pure ghost swapfile without writeback support, maybe something similar to the first 2 patches in my RFC? i.e up to this patch:

https://lore.kernel.org/all/[email protected]/

I'm sure you could find issues/bugs in it too with some extensive testing (a lot of my performance testing is done on the entire series as a whole), but from a design point of view it should achieve what us both want, no? It's literally just a ghost swap device - most of the other design elements of vswap (the new charging behavior, rmap, support for writeback) comes in latter patches. I think the difference is:

1. It's transparently dynamic, whereas in your RFC it's driven by a userspace knob. Do you think there are inherent values in a userspace knob?

2. The address space can also shrink when swap usage drops at runtime.

3. The dynamic address space is backed by an xarray. Note that it's structured differently than zswap's one (it's an xarray pointing to clusters). If you have rationales or numbers to show that vmalloc array is superior (while still support the kernel-driven expansion and shrink), please let me know.

If you have no faith in me and do not like my code, you can take the code as a starting point, or just the idea, and drive it yourself. Or let me know if you're unhappy with any piece of it - we can discuss further.

Quoted 3 lines

Quoted 3 lines

Otherwise, please continue to push virtual swap forward. I'll stop and wait.

Look man, I care about the problems being solved and I'm a sucker for good ideas, but I'm not married to anything, be it my code or my ideas.

I have literally dropped my old code and adopted Kairui's suggestions after playing with it and liking it, to make sure every party's concern is met.

If you have concerns, please let me know, or keep sending code if you believe you can get it done better than I do :) All I'm asking is you take my concerns into account in your design.

This series introduces a virtual "ghost" swapfile that can be backed by multiple physical swap devices, switching between them at runtime using a new "Redirect" swap table entry type.

Motivation
----------
Physical swap devices have fixed, large sizes but a machine may only need a fraction at any given time. Ghost swap decouples the logical swap capacity from physical swap allocation: it presents a large virtual swap device to userspace while lazily mapping slots to physical devices only when data must leave zswap (writeback). This enables overcommit of swap capacity without wasting physical disk space, and allows the ghost device to grow dynamically at runtime.

Ghost swapfile characteristics
--------------------------------
Ghost swap decouples logical swap capacity from physical swap allocation. Key properties:

- No backing store required. A ghost swap device can operate purely as a zswap container — compressed pages stay in the zswap pool without ever touching a physical swap device. No physical swap device needs to be attached.

- Optional physical backing. One or more physical swap devices can be swapon'd alongside the ghost device, acting as writeback targets when the zswap pool needs to evict cold pages. The Redirect entry in the ghost swap table links each evicted slot to its physical destination.

- Dynamic size growth. The ghost device can be extended at runtime via sysfs, up to a current maximum of 1TB. This is implemented through lazy vmalloc: a sparse 64MB virtual address space is reserved at swapon time to cover the full 1TB range, but physical pages for cluster_info[] structures are allocated only on demand as the device grows. The cluster_info pointer never changes, so all existing accessors (__swap_offset_to_cluster, cluster_index) work unchanged.

- sysfs control. Each ghost device exposes attributes under /sys/kernel/mm/ghost_swap/ghost_<N>/:

path (RO) — backing device name

max (RW) — current max pages; writing a larger value triggers
swap_ghost_extend_max()

Design overview
---------------
A swap table slot can now hold a Redirect entry:

Redirect: |----- physical swp_entry_t -----|101|

This stores a plain swp_entry_t pointing to a slot on a real (physical) swap device. The low 3-bit mark (0b101) distinguishes it from Pointer (0b100), PFN (0b10), Shadow (0b1), and NULL (0) entries.

The lifecycle is:

1. swapon a ghost device — reserves sparse vmalloc area (64MB virtual) and physically backs only the initial range. The device starts with no physical backing store: pages swapped out go to zswap, and zswap pool evictions go to any concurrently attached physical device.

2. zswap writeback from ghost: when the zswap pool needs to evict, zswap_writeback_ghost() allocates a physical slot from any real swap device, writes the data there, and plants a Redirect entry in the ghost's swap table pointing to the physical slot.

3. Swap-in from ghost: swap_read_folio() detects the ghost device (SWP_GHOST), resolves the Redirect to the physical swp_entry_t, and forwards the read I/O to the physical device.

4. When the swap cache folio is removed, the Redirect entry is restored from an xarray (redirect_xa) that persists the mapping, so subsequent reads still find the physical backing.

5. When the physical slot is freed, the ghost slot is also freed, cascading through swap_range_free().

Note:
------
Ghost swapfile is a proof of concept. Compared with the other patchset:
[RFC PATCH v2 0/7] mm, swap: Virtual Swap Space (Swap Table Edition), it explores a different approach to implementing virtual swap on top of the existing swap table infrastructure, with a simpler implementation. It builds on Chris Li's earlier ghost swapfile work and is currently tentatively named "ghost swapfile" — alternative naming suggestions are welcome.

Ghost swapfile setup can also be integrated through modifications to the swapon API and the swapon command in util-linux, for example:
swapon --ghost ghost_1:2G -p 5.

Baoquan He (9):
mm/swap_table: add Redirect entry encoding for ghost swap backend
switching
mm/swap: add redirect_xa field and ghost redirect helper declarations mm/swapfile: implement ghost redirect helpers and free-path cascade mm/swap_state: restore Redirect entry when swap cache folio is removed mm/zswap: implement ghost-to-physical writeback for backend switching mm/page_io: forward ghost swap reads to physical device via Redirect mm/swapfile: manage ghost cluster_info via lazy vmalloc mm/swapfile: implement swap_ghost_extend_max() for dynamic growth mm/swapfile: add sysfs interface for ghost swap extension

Chris Li (1):
mm: ghost swapfile support for zswap

 include/linux/swap.h |  10 +
 mm/page_io.c         |  51 +++-
 mm/swap.h            |  16 +-
 mm/swap_state.c      |  28 +++
 mm/swap_table.h      |  38 +++
 mm/swapfile.c        | 579 ++++++++++++++++++++++++++++++++++++++++---
 mm/zswap.c           |  98 +++++++-
 7 files changed, 772 insertions(+), 48 deletions(-)

--
2.54.0