mm, swap: dynamic cluster management for xswap devices

Implement dynamic cluster_info array growth for xswap devices using a VM_SPARSE vmalloc area:

1. xswap_map_clusters(): Allocate physical pages and map them into the pre-reserved VM_SPARSE KVA region via vm_area_map_pages().

2. xswap_unmap_clusters(): Unmap pages from the VM_SPARSE area via vm_area_unmap_pages() (used by the error/teardown paths, shrink comes later).

3. setup_swap_clusters_info() xswap path: Use get_vm_area(VM_SPARSE) for the cluster_info array, lazily mapping only the initial chunk.

4. free_swap_cluster_info(): Refactor to take swap_info_struct*. For xswap, unmap all clusters and free_vm_area().

5. swapoff: Remove snapshot locals; move p->max/p->cluster_info clearing after free_swap_cluster_info().

Signed-off-by Baoquan He <[email protected]>

---
 mm/swapfile.c | 228 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 216 insertions(+), 12 deletions(-)

mm/swapfile.c +91 -10

index 143088dae07d..6d9c95ed09bd 100644
@@ -49,6 +49,24 @@
 #include "internal.h"
 #include "swap.h"
 
+#ifdef CONFIG_XSWAP
+/*
+ * xswap: dynamically grow the cluster_info array via a VM_SPARSE area.
+ *
+ * XSWAP_GROW_CLUSTERS is the number of clusters to map in one grow
+ * operation.  It is set to the number of cluster_info structs that
+ * fit in a single page (at least 16), so that the vmalloc page table
+ * overhead is proportional to the number of clusters mapped.
+ */
+#define XSWAP_GROW_CLUSTERS \
+       max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
+
+static int xswap_map_clusters(struct swap_info_struct *si,
+                             unsigned long start_idx, unsigned long nr);
+static void xswap_unmap_clusters(struct swap_info_struct *si,
+                                unsigned long start_idx, unsigned long nr);
+#endif
+
 static void swap_range_alloc(struct swap_info_struct *si,
                             unsigned int nr_entries);
 static bool folio_swapcache_freeable(struct folio *folio);

@@ -3041,20 +3059,47 @@ static void wait_for_allocation(struct swap_info_struct *si)
 
        BUG_ON(si->flags & SWP_WRITEOK);
 
+#ifdef CONFIG_XSWAP
+       /*
+        * xswap clusters beyond nr_clusters_mapped have been unmapped
+        * by the shrinker and their vmalloc pages are no longer
+        * accessible.  Only iterate over currently mapped clusters.
+        */
+       if (si->flags & SWP_XSWAP)
+               end = min(end, READ_ONCE(si->nr_clusters_mapped) *
+                         SWAPFILE_CLUSTER);
+#endif
+
        for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
                ci = swap_cluster_lock(si, offset);
                swap_cluster_unlock(ci);
        }
 }
 
-static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
-                                  unsigned long maxpages)
+static void free_swap_cluster_info(struct swap_info_struct *si)
 {
+       struct swap_cluster_info *cluster_info = si->cluster_info;
+       unsigned long maxpages = si->max;
        struct swap_cluster_info *ci;
-       int i, nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
+       int i, nr_clusters;
 
        if (!cluster_info)
                return;
+
+#ifdef CONFIG_XSWAP
+       if (si->flags & SWP_XSWAP) {
+               /* Unmap all mapped clusters and free the VM_SPARSE area */
+               if (si->nr_clusters_mapped > 0)
+                       xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+               free_vm_area(si->cluster_vm);
+               si->cluster_vm = NULL;
+               si->nr_clusters = 0;
+               si->nr_clusters_mapped = 0;
+               return;
+       }
+#endif
+
+       nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
        for (i = 0; i < nr_clusters; i++) {
                ci = cluster_info + i;
                /* Cluster with bad marks count will have a remaining table */

@@ -3093,11 +3138,9 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si)
 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 {
        struct swap_info_struct *p = NULL;
-       struct swap_cluster_info *cluster_info;
        struct file *swap_file, *victim;
        struct address_space *mapping;
        struct inode *inode;
-       unsigned int maxpages;
        int err, found = 0;
 
        if (!capable(CAP_SYS_ADMIN))

@@ -3189,10 +3232,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 
        swap_file = p->swap_file;
        p->swap_file = NULL;
-       maxpages = p->max;
-       cluster_info = p->cluster_info;
-       p->max = 0;
-       p->cluster_info = NULL;
        spin_unlock(&p->lock);
        spin_unlock(&swap_lock);
        arch_swap_invalidate_area(p->type);

@@ -3200,7 +3239,9 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
        mutex_unlock(&swapon_mutex);
        kfree(p->global_cluster);
        p->global_cluster = NULL;
-       free_swap_cluster_info(cluster_info, maxpages);
+       free_swap_cluster_info(p);
+       p->max = 0;
+       p->cluster_info = NULL;
 
        inode = mapping->host;
 

@@ -3564,6 +3605,112 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
        return maxpages;
 }
 
+#ifdef CONFIG_XSWAP
+static int xswap_map_clusters(struct swap_info_struct *si,
+                             unsigned long start_idx, unsigned long nr)
+{
+       unsigned long start_addr = (unsigned long)si->cluster_info +
+                                  (size_t)start_idx * sizeof(struct swap_cluster_info);
+       unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
+       /*
+        * vm_area_map_pages() requires that start and end be page-aligned.
+        * If start_addr falls within a page that was already mapped by a
+        * previous batch (grow path), round it up to skip the already-mapped
+        * partial page.  Always round end_addr up so the vmap page table walk
+        * terminates correctly (the walk loop exits when addr == end, and addr
+        * advances by PAGE_SIZE each iteration).
+        */
+       unsigned long vm_start = PAGE_ALIGN(start_addr);
+       unsigned long vm_end = PAGE_ALIGN(end_addr);
+       unsigned long npages;
+       struct page **pages;
+       unsigned long i;
+
+       if (vm_start >= vm_end) {
+               /* All requested clusters fall within already-mapped pages. */
+               for (i = start_idx; i < start_idx + nr; i++)
+                       spin_lock_init(&si->cluster_info[i].lock);
+               WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+               return 0;
+       }
+
+       npages = (vm_end - vm_start) >> PAGE_SHIFT;
+
+       pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
+       if (!pages)
+               return -ENOMEM;
+
+       for (i = 0; i < npages; i++) {
+               /*
+                * __GFP_ZERO is critical: cluster_info structs contain pointer
+                * fields (extend_table, zero_bitmap, memcg_table, table) that
+                * must start as NULL.  Without zeroing, stale data from a

Quoted 14 lines · Baoquan He

Implement dynamic cluster_info array growth for xswap devices using a VM_SPARSE vmalloc area:

1. xswap_map_clusters(): Allocate physical pages and map them into the pre-reserved VM_SPARSE KVA region via vm_area_map_pages().

2. xswap_unmap_clusters(): Unmap pages from the VM_SPARSE area via vm_area_unmap_pages() (used by the error/teardown paths, shrink comes later).

3. setup_swap_clusters_info() xswap path: Use get_vm_area(VM_SPARSE) for the cluster_info array, lazily mapping only the initial chunk.

Quoted 7 lines

4. free_swap_cluster_info(): Refactor to take swap_info_struct*. For xswap, unmap all clusters and free_vm_area().

5. swapoff: Remove snapshot locals; move p->max/p->cluster_info clearing after free_swap_cluster_info().

Signed-off-by Baoquan He <[email protected]>

I'll give my 2 cents on the direction we're pursuing later. I've done a lot of thinking over the past weeks, and I have some new concerns now regarding how we plan to land this joint venture :)

That said, this is a new piece of extension, done per my request (kernel-driven dynamic swap address space growth - thanks for taking my concerns seriously), and I've also evaluated this vmalloc-based data structure in the past week, so I'll put in my inquiries/concerns.

Quoted 162 lines

---
 mm/swapfile.c | 228 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 216 insertions(+), 12 deletions(-)

mm/swapfile.c +91 -10

index 143088dae07d..6d9c95ed09bd 100644
@@ -49,6 +49,24 @@
 #include "internal.h"
 #include "swap.h"
 
+#ifdef CONFIG_XSWAP
+/*
+ * xswap: dynamically grow the cluster_info array via a VM_SPARSE area.
+ *
+ * XSWAP_GROW_CLUSTERS is the number of clusters to map in one grow
+ * operation.  It is set to the number of cluster_info structs that
+ * fit in a single page (at least 16), so that the vmalloc page table
+ * overhead is proportional to the number of clusters mapped.
+ */
+#define XSWAP_GROW_CLUSTERS \
+       max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
+
+static int xswap_map_clusters(struct swap_info_struct *si,
+                             unsigned long start_idx, unsigned long nr);
+static void xswap_unmap_clusters(struct swap_info_struct *si,
+                                unsigned long start_idx, unsigned long nr);
+#endif
+
 static void swap_range_alloc(struct swap_info_struct *si,
                             unsigned int nr_entries);
 static bool folio_swapcache_freeable(struct folio *folio);

@@ -3041,20 +3059,47 @@ static void wait_for_allocation(struct swap_info_struct *si)
 
        BUG_ON(si->flags & SWP_WRITEOK);
 
+#ifdef CONFIG_XSWAP
+       /*
+        * xswap clusters beyond nr_clusters_mapped have been unmapped
+        * by the shrinker and their vmalloc pages are no longer
+        * accessible.  Only iterate over currently mapped clusters.
+        */
+       if (si->flags & SWP_XSWAP)
+               end = min(end, READ_ONCE(si->nr_clusters_mapped) *
+                         SWAPFILE_CLUSTER);
+#endif
+
        for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
                ci = swap_cluster_lock(si, offset);
                swap_cluster_unlock(ci);
        }
 }
 
-static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
-                                  unsigned long maxpages)
+static void free_swap_cluster_info(struct swap_info_struct *si)
 {
+       struct swap_cluster_info *cluster_info = si->cluster_info;
+       unsigned long maxpages = si->max;
        struct swap_cluster_info *ci;
-       int i, nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
+       int i, nr_clusters;
 
        if (!cluster_info)
                return;
+
+#ifdef CONFIG_XSWAP
+       if (si->flags & SWP_XSWAP) {
+               /* Unmap all mapped clusters and free the VM_SPARSE area */
+               if (si->nr_clusters_mapped > 0)
+                       xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+               free_vm_area(si->cluster_vm);
+               si->cluster_vm = NULL;
+               si->nr_clusters = 0;
+               si->nr_clusters_mapped = 0;
+               return;
+       }
+#endif
+
+       nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
        for (i = 0; i < nr_clusters; i++) {
                ci = cluster_info + i;
                /* Cluster with bad marks count will have a remaining table */

@@ -3093,11 +3138,9 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si)
 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 {
        struct swap_info_struct *p = NULL;
-       struct swap_cluster_info *cluster_info;
        struct file *swap_file, *victim;
        struct address_space *mapping;
        struct inode *inode;
-       unsigned int maxpages;
        int err, found = 0;
 
        if (!capable(CAP_SYS_ADMIN))

@@ -3189,10 +3232,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 
        swap_file = p->swap_file;
        p->swap_file = NULL;
-       maxpages = p->max;
-       cluster_info = p->cluster_info;
-       p->max = 0;
-       p->cluster_info = NULL;
        spin_unlock(&p->lock);
        spin_unlock(&swap_lock);
        arch_swap_invalidate_area(p->type);

@@ -3200,7 +3239,9 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
        mutex_unlock(&swapon_mutex);
        kfree(p->global_cluster);
        p->global_cluster = NULL;
-       free_swap_cluster_info(cluster_info, maxpages);
+       free_swap_cluster_info(p);
+       p->max = 0;
+       p->cluster_info = NULL;
 
        inode = mapping->host;
 

@@ -3564,6 +3605,112 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
        return maxpages;
 }
 
+#ifdef CONFIG_XSWAP
+static int xswap_map_clusters(struct swap_info_struct *si,
+                             unsigned long start_idx, unsigned long nr)
+{
+       unsigned long start_addr = (unsigned long)si->cluster_info +
+                                  (size_t)start_idx * sizeof(struct swap_cluster_info);
+       unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
+       /*
+        * vm_area_map_pages() requires that start and end be page-aligned.
+        * If start_addr falls within a page that was already mapped by a
+        * previous batch (grow path), round it up to skip the already-mapped
+        * partial page.  Always round end_addr up so the vmap page table walk
+        * terminates correctly (the walk loop exits when addr == end, and addr
+        * advances by PAGE_SIZE each iteration).
+        */
+       unsigned long vm_start = PAGE_ALIGN(start_addr);
+       unsigned long vm_end = PAGE_ALIGN(end_addr);
+       unsigned long npages;
+       struct page **pages;
+       unsigned long i;
+
+       if (vm_start >= vm_end) {
+               /* All requested clusters fall within already-mapped pages. */
+               for (i = start_idx; i < start_idx + nr; i++)
+                       spin_lock_init(&si->cluster_info[i].lock);
+               WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+               return 0;
+       }
+
+       npages = (vm_end - vm_start) >> PAGE_SHIFT;
+
+       pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
+       if (!pages)
+               return -ENOMEM;
+
+       for (i = 0; i < npages; i++) {
+               /*
+                * __GFP_ZERO is critical: cluster_info structs contain pointer
+                * fields (extend_table, zero_bitmap, memcg_table, table) that
+                * must start as NULL.  Without zeroing, stale data from a

mm/swapfile.c static unsigned long read_swap_header(struct swap_info_struct *si, · line 3657

Quoted 10 lines

mm/swapfile.c +10 -0

+                * previous user of the page would look like valid pointers.
+                */
+               pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
+               if (!pages[i])
+                       goto fail;
+       }
+
+       if (vm_area_map_pages(si->cluster_vm, vm_start, vm_end, pages)) {
+               i = npages; /* free all pages on failure */
+               goto fail;

I was evaluating whether your vmalloc array can be extended to support kernel-driven dynamic growth at least (either as a slot-in replacement for xarray, or as a follow-up optimization if it's too complicated), and I stumble this mapping action.

Seems like it does not take any GFP flag argument, and under the hood it calls GFP_KERNEL. Would this be safe in the swap allocation path (where you slotted it in in patch 4)? Kairui used a more precise set of flags in this path, for e.g for swap table allocation:

ret = swap_cluster_alloc_table(ci, __GFP_HIGH | __GFP_NOMEMALLOC |
GFP_KERNEL);

I'm guessing this has to do with the fact that we often entered swapping out paths with PF_MEMALLOC... Is there any risk of deadlock etc.?

Quoted 1 line · Nhat Pham

On Mon, Jul 27, 2026 at 7:05 AM Baoquan He <[email protected]> wrote:

Quoted 61 lines

Quoted 40 lines

mm/swapfile.c +40 -0

+#ifdef CONFIG_XSWAP
+static int xswap_map_clusters(struct swap_info_struct *si,
+                             unsigned long start_idx, unsigned long nr)
+{
+       unsigned long start_addr = (unsigned long)si->cluster_info +
+                                  (size_t)start_idx * sizeof(struct swap_cluster_info);
+       unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
+       /*
+        * vm_area_map_pages() requires that start and end be page-aligned.
+        * If start_addr falls within a page that was already mapped by a
+        * previous batch (grow path), round it up to skip the already-mapped
+        * partial page.  Always round end_addr up so the vmap page table walk
+        * terminates correctly (the walk loop exits when addr == end, and addr
+        * advances by PAGE_SIZE each iteration).
+        */
+       unsigned long vm_start = PAGE_ALIGN(start_addr);
+       unsigned long vm_end = PAGE_ALIGN(end_addr);
+       unsigned long npages;
+       struct page **pages;
+       unsigned long i;
+
+       if (vm_start >= vm_end) {
+               /* All requested clusters fall within already-mapped pages. */
+               for (i = start_idx; i < start_idx + nr; i++)
+                       spin_lock_init(&si->cluster_info[i].lock);
+               WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+               return 0;
+       }
+
+       npages = (vm_end - vm_start) >> PAGE_SHIFT;
+
+       pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
+       if (!pages)
+               return -ENOMEM;
+
+       for (i = 0; i < npages; i++) {
+               /*
+                * __GFP_ZERO is critical: cluster_info structs contain pointer
+                * fields (extend_table, zero_bitmap, memcg_table, table) that
+                * must start as NULL.  Without zeroing, stale data from a

mm/swapfile.c static unsigned long read_swap_header(struct swap_info_struct *si, · line 3657

Quoted 10 lines

mm/swapfile.c +10 -0

+                * previous user of the page would look like valid pointers.
+                */
+               pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
+               if (!pages[i])
+                       goto fail;
+       }
+
+       if (vm_area_map_pages(si->cluster_vm, vm_start, vm_end, pages)) {
+               i = npages; /* free all pages on failure */
+               goto fail;

I was evaluating whether your vmalloc array can be extended to support kernel-driven dynamic growth at least (either as a slot-in replacement for xarray, or as a follow-up optimization if it's too complicated), and I stumble this mapping action.

Seems like it does not take any GFP flag argument, and under the hood it calls GFP_KERNEL. Would this be safe in the swap allocation path (where you slotted it in in patch 4)? Kairui used a more precise set of flags in this path, for e.g for swap table allocation:

Quoted 7 lines

ret = swap_cluster_alloc_table(ci, __GFP_HIGH | __GFP_NOMEMALLOC |
GFP_KERNEL);

I'm guessing this has to do with the fact that we often entered swapping out paths with PF_MEMALLOC... Is there any risk of deadlock etc.?

Thanks for catching this. You're right — there are actually three layers of hardcoded GFP_KERNEL in the VM_SPARSE path:

1. alloc_page(GFP_KERNEL | __GFP_ZERO) in xswap_map_clusters() for the backing pages
2. vmap_pages_range() hardcodes GFP_KERNEL (passed only to KMSAN, but still semantically wrong)
3. The page-table allocations inside vmap_pages_range_noflush_walk() (pte_alloc_kernel_track etc.) use GFP_PGTABLE_KERNEL, which is GFP_KERNEL | __GFP_ZERO — also lacking __GFP_NOMEMALLOC, and completely invisible to the caller

When PF_MEMALLOC is set (as it is during reclaim), __gfp_pfmemalloc_flags() returns ALLOC_NO_WATERMARKS for any allocation without __GFP_NOMEMALLOC, meaning those allocations can consume all emergency reserves — including the reserves that the reclaim machinery itself needs to make forward progress. That is a real deadlock risk.

One way is to fix them one by one. Another way is to drop the VM_SPARSE approach entirely and implemented Chris's suggestion instead: cluster_info is now a struct swap_cluster_info ** pointer array, with each cluster individually kmem_cache_zalloc()'d. The grow path (xswap_grow_clusters) now uses:

kmem_cache_zalloc(swap_cluster_cachep,
__GFP_HIGH | __GFP_NOMEMALLOC | GFP_KERNEL)
matching the pattern Kairui established in swap_cluster_alloc_table(). All allocations are fully GFP-controllable — no hidden vmap page-table allocations, no hardcoded GFP_PGTABLE_KERNEL.

With kmem_cache, shrink is just kmem_cache_free() — each cluster is independently allocated and freed. Seems the logic is simpler. If no objection, I will give it a shot soon.

Thanks
Baoquan

Implement dynamic cluster_info array growth for xswap devices using a VM_SPARSE vmalloc area:

1. xswap_map_clusters(): Allocate physical pages and map them into the pre-reserved VM_SPARSE KVA region via vm_area_map_pages().

2. xswap_unmap_clusters(): Unmap pages from the VM_SPARSE area via vm_area_unmap_pages() (used by the error/teardown paths, shrink comes later).

3. setup_swap_clusters_info() xswap path: Use get_vm_area(VM_SPARSE) for the cluster_info array, lazily mapping only the initial chunk.

4. free_swap_cluster_info(): Refactor to take swap_info_struct*. For xswap, unmap all clusters and free_vm_area().

5. swapoff: Remove snapshot locals; move p->max/p->cluster_info clearing after free_swap_cluster_info().

Signed-off-by Baoquan He <[email protected]>

---
 mm/swapfile.c | 228 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 216 insertions(+), 12 deletions(-)

mm/swapfile.c +216 -13

index 143088dae07d..6d9c95ed09bd 100644
@@ -49,6 +49,24 @@
 #include "internal.h"
 #include "swap.h"
 
+#ifdef CONFIG_XSWAP
+/*
+ * xswap: dynamically grow the cluster_info array via a VM_SPARSE area.
+ *
+ * XSWAP_GROW_CLUSTERS is the number of clusters to map in one grow
+ * operation.  It is set to the number of cluster_info structs that
+ * fit in a single page (at least 16), so that the vmalloc page table
+ * overhead is proportional to the number of clusters mapped.
+ */
+#define XSWAP_GROW_CLUSTERS \
+       max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
+
+static int xswap_map_clusters(struct swap_info_struct *si,
+                             unsigned long start_idx, unsigned long nr);
+static void xswap_unmap_clusters(struct swap_info_struct *si,
+                                unsigned long start_idx, unsigned long nr);
+#endif
+
 static void swap_range_alloc(struct swap_info_struct *si,
                             unsigned int nr_entries);
 static bool folio_swapcache_freeable(struct folio *folio);

@@ -3041,20 +3059,47 @@ static void wait_for_allocation(struct swap_info_struct *si)
 
        BUG_ON(si->flags & SWP_WRITEOK);
 
+#ifdef CONFIG_XSWAP
+       /*
+        * xswap clusters beyond nr_clusters_mapped have been unmapped
+        * by the shrinker and their vmalloc pages are no longer
+        * accessible.  Only iterate over currently mapped clusters.
+        */
+       if (si->flags & SWP_XSWAP)
+               end = min(end, READ_ONCE(si->nr_clusters_mapped) *
+                         SWAPFILE_CLUSTER);
+#endif
+
        for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
                ci = swap_cluster_lock(si, offset);
                swap_cluster_unlock(ci);
        }
 }
 
-static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
-                                  unsigned long maxpages)
+static void free_swap_cluster_info(struct swap_info_struct *si)
 {
+       struct swap_cluster_info *cluster_info = si->cluster_info;
+       unsigned long maxpages = si->max;
        struct swap_cluster_info *ci;
-       int i, nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
+       int i, nr_clusters;
 
        if (!cluster_info)
                return;
+
+#ifdef CONFIG_XSWAP
+       if (si->flags & SWP_XSWAP) {
+               /* Unmap all mapped clusters and free the VM_SPARSE area */
+               if (si->nr_clusters_mapped > 0)
+                       xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+               free_vm_area(si->cluster_vm);
+               si->cluster_vm = NULL;
+               si->nr_clusters = 0;
+               si->nr_clusters_mapped = 0;
+               return;
+       }
+#endif
+
+       nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
        for (i = 0; i < nr_clusters; i++) {
                ci = cluster_info + i;
                /* Cluster with bad marks count will have a remaining table */

@@ -3093,11 +3138,9 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si)
 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 {
        struct swap_info_struct *p = NULL;
-       struct swap_cluster_info *cluster_info;
        struct file *swap_file, *victim;
        struct address_space *mapping;
        struct inode *inode;
-       unsigned int maxpages;
        int err, found = 0;
 
        if (!capable(CAP_SYS_ADMIN))

@@ -3189,10 +3232,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 
        swap_file = p->swap_file;
        p->swap_file = NULL;
-       maxpages = p->max;
-       cluster_info = p->cluster_info;
-       p->max = 0;
-       p->cluster_info = NULL;
        spin_unlock(&p->lock);
        spin_unlock(&swap_lock);
        arch_swap_invalidate_area(p->type);

@@ -3200,7 +3239,9 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
        mutex_unlock(&swapon_mutex);
        kfree(p->global_cluster);
        p->global_cluster = NULL;
-       free_swap_cluster_info(cluster_info, maxpages);
+       free_swap_cluster_info(p);
+       p->max = 0;
+       p->cluster_info = NULL;
 
        inode = mapping->host;
 

@@ -3564,6 +3605,112 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
        return maxpages;
 }
 
+#ifdef CONFIG_XSWAP
+static int xswap_map_clusters(struct swap_info_struct *si,
+                             unsigned long start_idx, unsigned long nr)
+{
+       unsigned long start_addr = (unsigned long)si->cluster_info +
+                                  (size_t)start_idx * sizeof(struct swap_cluster_info);
+       unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
+       /*
+        * vm_area_map_pages() requires that start and end be page-aligned.
+        * If start_addr falls within a page that was already mapped by a
+        * previous batch (grow path), round it up to skip the already-mapped
+        * partial page.  Always round end_addr up so the vmap page table walk
+        * terminates correctly (the walk loop exits when addr == end, and addr
+        * advances by PAGE_SIZE each iteration).
+        */
+       unsigned long vm_start = PAGE_ALIGN(start_addr);
+       unsigned long vm_end = PAGE_ALIGN(end_addr);
+       unsigned long npages;
+       struct page **pages;
+       unsigned long i;
+
+       if (vm_start >= vm_end) {
+               /* All requested clusters fall within already-mapped pages. */
+               for (i = start_idx; i < start_idx + nr; i++)
+                       spin_lock_init(&si->cluster_info[i].lock);
+               WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+               return 0;
+       }
+
+       npages = (vm_end - vm_start) >> PAGE_SHIFT;
+
+       pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
+       if (!pages)
+               return -ENOMEM;
+
+       for (i = 0; i < npages; i++) {
+               /*
+                * __GFP_ZERO is critical: cluster_info structs contain pointer
+                * fields (extend_table, zero_bitmap, memcg_table, table) that
+                * must start as NULL.  Without zeroing, stale data from a
+                * previous user of the page would look like valid pointers.
+                */
+               pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
+               if (!pages[i])
+                       goto fail;
+       }
+
+       if (vm_area_map_pages(si->cluster_vm, vm_start, vm_end, pages)) {
+               i = npages; /* free all pages on failure */
+               goto fail;
+       }
+
+       kfree(pages);
+
+       /* Initialize spinlocks for newly mapped clusters */
+       for (i = start_idx; i < start_idx + nr; i++)
+               spin_lock_init(&si->cluster_info[i].lock);
+
+       /*
+        * Pairs with READ_ONCE() in shrink/grow paths.
+        */
+       WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+       return 0;
+
+fail:
+       while (i > 0) {
+               i--;
+               if (pages[i])
+                       __free_page(pages[i]);
+       }
+       kfree(pages);
+       return -ENOMEM;
+}
+
+static void xswap_unmap_clusters(struct swap_info_struct *si,
+                                unsigned long start_idx, unsigned long nr)
+{
+       unsigned long start_addr = (unsigned long)si->cluster_info +
+                                  (size_t)start_idx * sizeof(struct swap_cluster_info);
+       unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
+       /*
+        * Round to page boundaries: start up (skip partial page that may
+        * contain clusters still in use before start_idx), end up so the
+        * entire range is covered.  vm_area_unmap_pages() operates on
+        * whole pages.
+        */
+       unsigned long vm_start = PAGE_ALIGN(start_addr);
+       unsigned long vm_end = PAGE_ALIGN(end_addr);
+
+       if (vm_start >= vm_end)
+               goto out;
+
+       vm_area_unmap_pages(si->cluster_vm, vm_start, vm_end);
+       /*
+        * vm_area_unmap_pages() only clears PTEs; it does not free the
+        * physical pages.  Walk the page table to find and free them.
+        */
+       /* TODO: free backing pages via page table walk or tracking bitmap */
+       /*
+        * Pairs with READ_ONCE() in shrink/grow paths.
+        */
+       out:
+       WRITE_ONCE(si->nr_clusters_mapped, start_idx);
+}
+#endif /* CONFIG_XSWAP */
+
 static int setup_swap_clusters_info(struct swap_info_struct *si,
                                    union swap_header *swap_header,
                                    unsigned long maxpages)

@@ -3573,6 +3720,63 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
        int err = -ENOMEM;
        unsigned long i;
 
+#ifdef CONFIG_XSWAP
+       if (si->flags & SWP_XSWAP) {
+               unsigned long size = PAGE_ALIGN(nr_clusters * sizeof(*cluster_info));
+               struct vm_struct *vm;
+
+               vm = get_vm_area(size, VM_SPARSE);
+               if (!vm)
+                       goto err;
+
+               cluster_info = vm->addr;
+               si->cluster_vm = vm;
+               si->nr_clusters = nr_clusters;
+               si->cluster_info = cluster_info;
+
+               /* Map the initial chunk (at least cluster 0) */
+               if (xswap_map_clusters(si, 0, min_t(unsigned long,
+                                       XSWAP_GROW_CLUSTERS, nr_clusters)))
+                       goto err_free_vm;
+
+               /* xswap: only cluster 0 slot 0 is bad */
+               err = swap_cluster_setup_bad_slot(si, cluster_info, 0, false);
+               if (err)
+                       goto err_unmap;
+
+               INIT_LIST_HEAD(&si->free_clusters);
+               INIT_LIST_HEAD(&si->full_clusters);
+               INIT_LIST_HEAD(&si->discard_clusters);
+               for (i = 0; i < SWAP_NR_ORDERS; i++) {
+                       INIT_LIST_HEAD(&si->nonfull_clusters[i]);
+                       INIT_LIST_HEAD(&si->frag_clusters[i]);
+               }
+
+               /* Mark mapped clusters: cluster 0 has 1 bad slot, rest free */
+               for (i = 0; i < si->nr_clusters_mapped; i++) {
+                       struct swap_cluster_info *ci = &cluster_info[i];
+
+                       if (i == 0) {
+                               ci->flags = CLUSTER_FLAG_NONFULL;
+                               list_add_tail(&ci->list, &si->nonfull_clusters[0]);
+                       } else {
+                               ci->flags = CLUSTER_FLAG_FREE;
+                               list_add_tail(&ci->list, &si->free_clusters);
+                       }
+               }
+
+               return 0;
+
+err_unmap:
+               xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+err_free_vm:
+               free_vm_area(si->cluster_vm);
+               si->cluster_vm = NULL;
+               si->cluster_info = NULL;
+               return err;
+       }
+#endif /* CONFIG_XSWAP */
+
        cluster_info = kvzalloc_objs(*cluster_info, nr_clusters);
        if (!cluster_info)
                goto err;

@@ -3640,7 +3844,7 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
        si->cluster_info = cluster_info;
        return 0;
 err:
-       free_swap_cluster_info(cluster_info, maxpages);
+       free_swap_cluster_info(si);
        return err;
 }
 

@@ -3852,7 +4056,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
        si->global_cluster = NULL;
        inode = NULL;
        destroy_swap_extents(si, swap_file);
-       free_swap_cluster_info(si->cluster_info, si->max);
+       free_swap_cluster_info(si);
        si->cluster_info = NULL;
        /*
         * Clear the SWP_USED flag after all resources are freed so
--

2.54.0

Quoted 11 lines · Baoquan He

Implement dynamic cluster_info array growth for xswap devices using a VM_SPARSE vmalloc area:

1. xswap_map_clusters(): Allocate physical pages and map them into the pre-reserved VM_SPARSE KVA region via vm_area_map_pages().

2. xswap_unmap_clusters(): Unmap pages from the VM_SPARSE area via vm_area_unmap_pages() (used by the error/teardown paths, shrink comes later).

Quoted 8 lines

3. setup_swap_clusters_info() xswap path: Use get_vm_area(VM_SPARSE) for the cluster_info array, lazily mapping only the initial chunk.

4. free_swap_cluster_info(): Refactor to take swap_info_struct*. For xswap, unmap all clusters and free_vm_area().

5. swapoff: Remove snapshot locals; move p->max/p->cluster_info clearing after free_swap_cluster_info().

I feel that we should provide more generic support for dynamically growing an array of element size A to size N. Cluster array is just a user of such an API. There might be more users. This patch hardcodes the map/umpa of clusters. What about other users of such a dynamic array?
Another consideration is that the map/unmap logic, while more generic, adds quite a bit of complexity to an approach intended only for growth. Any reason we want map/unmap rathern than simple grow from the current si->max?

I read a bit more about the patch. It seems you need the unmap for the error cleanup path. In that case I suspect just make cluster a kmem_cache allocation, cluster array just a pointer. The cluster pointer array should just grow, not shrink. That might be overall simpler.
I'm worried someone might abuse the cluster unmap causing unwanted side effects.

I see the other reason might be you want the cluster code to be exactly the same when XSWAP is off. For that I think it is acceptiale to turn cluster into kmem_cache allocated and cluster array as pointers array as seperate patch as the base even when XSWAP is not configed.

Quoted 319 lines

Signed-off-by Baoquan He <[email protected]>

---
 mm/swapfile.c | 228 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 216 insertions(+), 12 deletions(-)

mm/swapfile.c +216 -13

index 143088dae07d..6d9c95ed09bd 100644
@@ -49,6 +49,24 @@
 #include "internal.h"
 #include "swap.h"
 
+#ifdef CONFIG_XSWAP
+/*
+ * xswap: dynamically grow the cluster_info array via a VM_SPARSE area.
+ *
+ * XSWAP_GROW_CLUSTERS is the number of clusters to map in one grow
+ * operation.  It is set to the number of cluster_info structs that
+ * fit in a single page (at least 16), so that the vmalloc page table
+ * overhead is proportional to the number of clusters mapped.
+ */
+#define XSWAP_GROW_CLUSTERS \
+       max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
+
+static int xswap_map_clusters(struct swap_info_struct *si,
+                             unsigned long start_idx, unsigned long nr);
+static void xswap_unmap_clusters(struct swap_info_struct *si,
+                                unsigned long start_idx, unsigned long nr);
+#endif
+
 static void swap_range_alloc(struct swap_info_struct *si,
                             unsigned int nr_entries);
 static bool folio_swapcache_freeable(struct folio *folio);

@@ -3041,20 +3059,47 @@ static void wait_for_allocation(struct swap_info_struct *si)
 
        BUG_ON(si->flags & SWP_WRITEOK);
 
+#ifdef CONFIG_XSWAP
+       /*
+        * xswap clusters beyond nr_clusters_mapped have been unmapped
+        * by the shrinker and their vmalloc pages are no longer
+        * accessible.  Only iterate over currently mapped clusters.
+        */
+       if (si->flags & SWP_XSWAP)
+               end = min(end, READ_ONCE(si->nr_clusters_mapped) *
+                         SWAPFILE_CLUSTER);
+#endif
+
        for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
                ci = swap_cluster_lock(si, offset);
                swap_cluster_unlock(ci);
        }
 }
 
-static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
-                                  unsigned long maxpages)
+static void free_swap_cluster_info(struct swap_info_struct *si)
 {
+       struct swap_cluster_info *cluster_info = si->cluster_info;
+       unsigned long maxpages = si->max;
        struct swap_cluster_info *ci;
-       int i, nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
+       int i, nr_clusters;
 
        if (!cluster_info)
                return;
+
+#ifdef CONFIG_XSWAP
+       if (si->flags & SWP_XSWAP) {
+               /* Unmap all mapped clusters and free the VM_SPARSE area */
+               if (si->nr_clusters_mapped > 0)
+                       xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+               free_vm_area(si->cluster_vm);
+               si->cluster_vm = NULL;
+               si->nr_clusters = 0;
+               si->nr_clusters_mapped = 0;
+               return;
+       }
+#endif
+
+       nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
        for (i = 0; i < nr_clusters; i++) {
                ci = cluster_info + i;
                /* Cluster with bad marks count will have a remaining table */

@@ -3093,11 +3138,9 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si)
 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 {
        struct swap_info_struct *p = NULL;
-       struct swap_cluster_info *cluster_info;
        struct file *swap_file, *victim;
        struct address_space *mapping;
        struct inode *inode;
-       unsigned int maxpages;
        int err, found = 0;
 
        if (!capable(CAP_SYS_ADMIN))

@@ -3189,10 +3232,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 
        swap_file = p->swap_file;
        p->swap_file = NULL;
-       maxpages = p->max;
-       cluster_info = p->cluster_info;
-       p->max = 0;
-       p->cluster_info = NULL;
        spin_unlock(&p->lock);
        spin_unlock(&swap_lock);
        arch_swap_invalidate_area(p->type);

@@ -3200,7 +3239,9 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
        mutex_unlock(&swapon_mutex);
        kfree(p->global_cluster);
        p->global_cluster = NULL;
-       free_swap_cluster_info(cluster_info, maxpages);
+       free_swap_cluster_info(p);
+       p->max = 0;
+       p->cluster_info = NULL;
 
        inode = mapping->host;
 

@@ -3564,6 +3605,112 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
        return maxpages;
 }
 
+#ifdef CONFIG_XSWAP
+static int xswap_map_clusters(struct swap_info_struct *si,
+                             unsigned long start_idx, unsigned long nr)
+{
+       unsigned long start_addr = (unsigned long)si->cluster_info +
+                                  (size_t)start_idx * sizeof(struct swap_cluster_info);
+       unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
+       /*
+        * vm_area_map_pages() requires that start and end be page-aligned.
+        * If start_addr falls within a page that was already mapped by a
+        * previous batch (grow path), round it up to skip the already-mapped
+        * partial page.  Always round end_addr up so the vmap page table walk
+        * terminates correctly (the walk loop exits when addr == end, and addr
+        * advances by PAGE_SIZE each iteration).
+        */
+       unsigned long vm_start = PAGE_ALIGN(start_addr);
+       unsigned long vm_end = PAGE_ALIGN(end_addr);
+       unsigned long npages;
+       struct page **pages;
+       unsigned long i;
+
+       if (vm_start >= vm_end) {
+               /* All requested clusters fall within already-mapped pages. */
+               for (i = start_idx; i < start_idx + nr; i++)
+                       spin_lock_init(&si->cluster_info[i].lock);
+               WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+               return 0;
+       }
+
+       npages = (vm_end - vm_start) >> PAGE_SHIFT;
+
+       pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
+       if (!pages)
+               return -ENOMEM;
+
+       for (i = 0; i < npages; i++) {
+               /*
+                * __GFP_ZERO is critical: cluster_info structs contain pointer
+                * fields (extend_table, zero_bitmap, memcg_table, table) that
+                * must start as NULL.  Without zeroing, stale data from a
+                * previous user of the page would look like valid pointers.
+                */
+               pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
+               if (!pages[i])
+                       goto fail;
+       }
+
+       if (vm_area_map_pages(si->cluster_vm, vm_start, vm_end, pages)) {
+               i = npages; /* free all pages on failure */
+               goto fail;
+       }
+
+       kfree(pages);
+
+       /* Initialize spinlocks for newly mapped clusters */
+       for (i = start_idx; i < start_idx + nr; i++)
+               spin_lock_init(&si->cluster_info[i].lock);
+
+       /*
+        * Pairs with READ_ONCE() in shrink/grow paths.
+        */
+       WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+       return 0;
+
+fail:
+       while (i > 0) {
+               i--;
+               if (pages[i])
+                       __free_page(pages[i]);
+       }
+       kfree(pages);
+       return -ENOMEM;
+}
+
+static void xswap_unmap_clusters(struct swap_info_struct *si,
+                                unsigned long start_idx, unsigned long nr)
+{
+       unsigned long start_addr = (unsigned long)si->cluster_info +
+                                  (size_t)start_idx * sizeof(struct swap_cluster_info);
+       unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
+       /*
+        * Round to page boundaries: start up (skip partial page that may
+        * contain clusters still in use before start_idx), end up so the
+        * entire range is covered.  vm_area_unmap_pages() operates on
+        * whole pages.
+        */
+       unsigned long vm_start = PAGE_ALIGN(start_addr);
+       unsigned long vm_end = PAGE_ALIGN(end_addr);
+
+       if (vm_start >= vm_end)
+               goto out;
+
+       vm_area_unmap_pages(si->cluster_vm, vm_start, vm_end);
+       /*
+        * vm_area_unmap_pages() only clears PTEs; it does not free the
+        * physical pages.  Walk the page table to find and free them.
+        */
+       /* TODO: free backing pages via page table walk or tracking bitmap */
+       /*
+        * Pairs with READ_ONCE() in shrink/grow paths.
+        */
+       out:
+       WRITE_ONCE(si->nr_clusters_mapped, start_idx);
+}
+#endif /* CONFIG_XSWAP */
+
 static int setup_swap_clusters_info(struct swap_info_struct *si,
                                    union swap_header *swap_header,
                                    unsigned long maxpages)

@@ -3573,6 +3720,63 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
        int err = -ENOMEM;
        unsigned long i;
 
+#ifdef CONFIG_XSWAP
+       if (si->flags & SWP_XSWAP) {
+               unsigned long size = PAGE_ALIGN(nr_clusters * sizeof(*cluster_info));
+               struct vm_struct *vm;
+
+               vm = get_vm_area(size, VM_SPARSE);
+               if (!vm)
+                       goto err;
+
+               cluster_info = vm->addr;
+               si->cluster_vm = vm;
+               si->nr_clusters = nr_clusters;
+               si->cluster_info = cluster_info;
+
+               /* Map the initial chunk (at least cluster 0) */
+               if (xswap_map_clusters(si, 0, min_t(unsigned long,
+                                       XSWAP_GROW_CLUSTERS, nr_clusters)))
+                       goto err_free_vm;
+
+               /* xswap: only cluster 0 slot 0 is bad */
+               err = swap_cluster_setup_bad_slot(si, cluster_info, 0, false);
+               if (err)
+                       goto err_unmap;
+
+               INIT_LIST_HEAD(&si->free_clusters);
+               INIT_LIST_HEAD(&si->full_clusters);
+               INIT_LIST_HEAD(&si->discard_clusters);
+               for (i = 0; i < SWAP_NR_ORDERS; i++) {
+                       INIT_LIST_HEAD(&si->nonfull_clusters[i]);
+                       INIT_LIST_HEAD(&si->frag_clusters[i]);
+               }
+
+               /* Mark mapped clusters: cluster 0 has 1 bad slot, rest free */
+               for (i = 0; i < si->nr_clusters_mapped; i++) {
+                       struct swap_cluster_info *ci = &cluster_info[i];
+
+                       if (i == 0) {
+                               ci->flags = CLUSTER_FLAG_NONFULL;
+                               list_add_tail(&ci->list, &si->nonfull_clusters[0]);
+                       } else {
+                               ci->flags = CLUSTER_FLAG_FREE;
+                               list_add_tail(&ci->list, &si->free_clusters);
+                       }
+               }
+
+               return 0;
+
+err_unmap:
+               xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+err_free_vm:
+               free_vm_area(si->cluster_vm);
+               si->cluster_vm = NULL;
+               si->cluster_info = NULL;
+               return err;
+       }
+#endif /* CONFIG_XSWAP */
+
        cluster_info = kvzalloc_objs(*cluster_info, nr_clusters);
        if (!cluster_info)
                goto err;

@@ -3640,7 +3844,7 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
        si->cluster_info = cluster_info;
        return 0;
 err:
-       free_swap_cluster_info(cluster_info, maxpages);
+       free_swap_cluster_info(si);
        return err;
 }
 

@@ -3852,7 +4056,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
        si->global_cluster = NULL;
        inode = NULL;
        destroy_swap_extents(si, swap_file);
-       free_swap_cluster_info(si->cluster_info, si->max);
+       free_swap_cluster_info(si);
        si->cluster_info = NULL;
        /*
         * Clear the SWP_USED flag after all resources are freed so
--

2.54.0

Quoted 1 line · Chris Li

On Mon, Jul 27, 2026 at 7:05 AM Baoquan He <[email protected]> wrote:

Quoted 7 lines

I read a bit more about the patch. It seems you need the unmap for the error cleanup path. In that case I suspect just make cluster a kmem_cache allocation, cluster array just a pointer. The cluster pointer array should just grow, not shrink. That might be overall simpler.
I'm worried someone might abuse the cluster unmap causing unwanted side effects.

Quoted 5 lines

I see the other reason might be you want the cluster code to be exactly the same when XSWAP is off. For that I think it is acceptiale to turn cluster into kmem_cache allocated and cluster array as pointers array as seperate patch as the base even when XSWAP is not configed.

I think about your suggestion carefully. Instead of VM_SPARSE, cluster_info is now struct swap_cluster_info ** — a pointer array that only grows, never shrinks. Individual clusters are kmem_cache_zalloc()'d on demand and kmem_cache_free()'d during shrink. This completely eliminates the VM_SPARSE vmap/unmap complexity and, more importantly, gives us full control over GFP flags (__GFP_NOMEMALLOC) to avoid dipping into emergency reserves from reclaim context. The data structure would look like below:

include/linux/swap.h +5 -1

index 80c83dd8804f..dbb94686ca7c 100644
@@ -247,7 +247,7 @@ struct swap_info_struct {
        struct plist_node list;         /* entry in swap_active_head */
        signed char     type;           /* strange name for an index */
        unsigned int    max;            /* size of this swap device */
-       struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
+       struct swap_cluster_info **cluster_info; /* cluster pointer array */
        struct list_head free_clusters; /* free clusters list */
        struct list_head full_clusters; /* full clusters list */
        struct list_head nonfull_clusters[SWAP_NR_ORDERS];

@@ -277,6 +277,11 @@ struct swap_info_struct {
        struct list_head discard_clusters; /* discard clusters list */
        struct plist_node avail_list;   /* entry in swap_avail_head */
        const struct swap_ops *ops;
+#ifdef CONFIG_XSWAP
+       unsigned long nr_clusters;      /* total cluster count for xswap */
+       unsigned long nr_clusters_mapped; /* currently mapped cluster count */
+#endif
 };
 
 

mm/swap.h +2 -1

index abd26588abd2..f32dea150f76 100644
@@ -62,6 +62,7 @@ struct swap_cluster_info {
        unsigned long *zero_bitmap;
 #endif
        struct list_head list;
+       unsigned int idx;               /* index in si->cluster_info pointer array */
 };
 

@@ -402,7 +411,7 @@ static inline bool cluster_is_usable(struct swap_cluster_info *ci, int order)
 static inline unsigned int cluster_index(struct swap_info_struct *si,
                                         struct swap_cluster_info *ci)
 {
-       return ci - si->cluster_info;
+       return ci->idx;
 }
 
 The pattern itself is very simple — allocate pointer array via kvmalloc_array,

allocate elements via kmem_cache_zalloc, free elements via kmem_cache_free, never shrink the pointer array — so extraction should be straightforward. Seems this is doable and code change will be simpler than VM_SPARSE way. If this is agreed on, I can try to make a v2 in that way.

Thanks
Baoquan

Quoted 319 lines

Quoted 319 lines

Signed-off-by Baoquan He <[email protected]>

---
 mm/swapfile.c | 228 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 216 insertions(+), 12 deletions(-)

mm/swapfile.c +216 -13

index 143088dae07d..6d9c95ed09bd 100644
@@ -49,6 +49,24 @@
 #include "internal.h"
 #include "swap.h"
 
+#ifdef CONFIG_XSWAP
+/*
+ * xswap: dynamically grow the cluster_info array via a VM_SPARSE area.
+ *
+ * XSWAP_GROW_CLUSTERS is the number of clusters to map in one grow
+ * operation.  It is set to the number of cluster_info structs that
+ * fit in a single page (at least 16), so that the vmalloc page table
+ * overhead is proportional to the number of clusters mapped.
+ */
+#define XSWAP_GROW_CLUSTERS \
+       max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
+
+static int xswap_map_clusters(struct swap_info_struct *si,
+                             unsigned long start_idx, unsigned long nr);
+static void xswap_unmap_clusters(struct swap_info_struct *si,
+                                unsigned long start_idx, unsigned long nr);
+#endif
+
 static void swap_range_alloc(struct swap_info_struct *si,
                             unsigned int nr_entries);
 static bool folio_swapcache_freeable(struct folio *folio);

@@ -3041,20 +3059,47 @@ static void wait_for_allocation(struct swap_info_struct *si)
 
        BUG_ON(si->flags & SWP_WRITEOK);
 
+#ifdef CONFIG_XSWAP
+       /*
+        * xswap clusters beyond nr_clusters_mapped have been unmapped
+        * by the shrinker and their vmalloc pages are no longer
+        * accessible.  Only iterate over currently mapped clusters.
+        */
+       if (si->flags & SWP_XSWAP)
+               end = min(end, READ_ONCE(si->nr_clusters_mapped) *
+                         SWAPFILE_CLUSTER);
+#endif
+
        for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
                ci = swap_cluster_lock(si, offset);
                swap_cluster_unlock(ci);
        }
 }
 
-static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
-                                  unsigned long maxpages)
+static void free_swap_cluster_info(struct swap_info_struct *si)
 {
+       struct swap_cluster_info *cluster_info = si->cluster_info;
+       unsigned long maxpages = si->max;
        struct swap_cluster_info *ci;
-       int i, nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
+       int i, nr_clusters;
 
        if (!cluster_info)
                return;
+
+#ifdef CONFIG_XSWAP
+       if (si->flags & SWP_XSWAP) {
+               /* Unmap all mapped clusters and free the VM_SPARSE area */
+               if (si->nr_clusters_mapped > 0)
+                       xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+               free_vm_area(si->cluster_vm);
+               si->cluster_vm = NULL;
+               si->nr_clusters = 0;
+               si->nr_clusters_mapped = 0;
+               return;
+       }
+#endif
+
+       nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
        for (i = 0; i < nr_clusters; i++) {
                ci = cluster_info + i;
                /* Cluster with bad marks count will have a remaining table */

@@ -3093,11 +3138,9 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si)
 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 {
        struct swap_info_struct *p = NULL;
-       struct swap_cluster_info *cluster_info;
        struct file *swap_file, *victim;
        struct address_space *mapping;
        struct inode *inode;
-       unsigned int maxpages;
        int err, found = 0;
 
        if (!capable(CAP_SYS_ADMIN))

@@ -3189,10 +3232,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
 
        swap_file = p->swap_file;
        p->swap_file = NULL;
-       maxpages = p->max;
-       cluster_info = p->cluster_info;
-       p->max = 0;
-       p->cluster_info = NULL;
        spin_unlock(&p->lock);
        spin_unlock(&swap_lock);
        arch_swap_invalidate_area(p->type);

@@ -3200,7 +3239,9 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
        mutex_unlock(&swapon_mutex);
        kfree(p->global_cluster);
        p->global_cluster = NULL;
-       free_swap_cluster_info(cluster_info, maxpages);
+       free_swap_cluster_info(p);
+       p->max = 0;
+       p->cluster_info = NULL;
 
        inode = mapping->host;
 

@@ -3564,6 +3605,112 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
        return maxpages;
 }
 
+#ifdef CONFIG_XSWAP
+static int xswap_map_clusters(struct swap_info_struct *si,
+                             unsigned long start_idx, unsigned long nr)
+{
+       unsigned long start_addr = (unsigned long)si->cluster_info +
+                                  (size_t)start_idx * sizeof(struct swap_cluster_info);
+       unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
+       /*
+        * vm_area_map_pages() requires that start and end be page-aligned.
+        * If start_addr falls within a page that was already mapped by a
+        * previous batch (grow path), round it up to skip the already-mapped
+        * partial page.  Always round end_addr up so the vmap page table walk
+        * terminates correctly (the walk loop exits when addr == end, and addr
+        * advances by PAGE_SIZE each iteration).
+        */
+       unsigned long vm_start = PAGE_ALIGN(start_addr);
+       unsigned long vm_end = PAGE_ALIGN(end_addr);
+       unsigned long npages;
+       struct page **pages;
+       unsigned long i;
+
+       if (vm_start >= vm_end) {
+               /* All requested clusters fall within already-mapped pages. */
+               for (i = start_idx; i < start_idx + nr; i++)
+                       spin_lock_init(&si->cluster_info[i].lock);
+               WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+               return 0;
+       }
+
+       npages = (vm_end - vm_start) >> PAGE_SHIFT;
+
+       pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
+       if (!pages)
+               return -ENOMEM;
+
+       for (i = 0; i < npages; i++) {
+               /*
+                * __GFP_ZERO is critical: cluster_info structs contain pointer
+                * fields (extend_table, zero_bitmap, memcg_table, table) that
+                * must start as NULL.  Without zeroing, stale data from a
+                * previous user of the page would look like valid pointers.
+                */
+               pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
+               if (!pages[i])
+                       goto fail;
+       }
+
+       if (vm_area_map_pages(si->cluster_vm, vm_start, vm_end, pages)) {
+               i = npages; /* free all pages on failure */
+               goto fail;
+       }
+
+       kfree(pages);
+
+       /* Initialize spinlocks for newly mapped clusters */
+       for (i = start_idx; i < start_idx + nr; i++)
+               spin_lock_init(&si->cluster_info[i].lock);
+
+       /*
+        * Pairs with READ_ONCE() in shrink/grow paths.
+        */
+       WRITE_ONCE(si->nr_clusters_mapped, start_idx + nr);
+       return 0;
+
+fail:
+       while (i > 0) {
+               i--;
+               if (pages[i])
+                       __free_page(pages[i]);
+       }
+       kfree(pages);
+       return -ENOMEM;
+}
+
+static void xswap_unmap_clusters(struct swap_info_struct *si,
+                                unsigned long start_idx, unsigned long nr)
+{
+       unsigned long start_addr = (unsigned long)si->cluster_info +
+                                  (size_t)start_idx * sizeof(struct swap_cluster_info);
+       unsigned long end_addr = start_addr + (size_t)nr * sizeof(struct swap_cluster_info);
+       /*
+        * Round to page boundaries: start up (skip partial page that may
+        * contain clusters still in use before start_idx), end up so the
+        * entire range is covered.  vm_area_unmap_pages() operates on
+        * whole pages.
+        */
+       unsigned long vm_start = PAGE_ALIGN(start_addr);
+       unsigned long vm_end = PAGE_ALIGN(end_addr);
+
+       if (vm_start >= vm_end)
+               goto out;
+
+       vm_area_unmap_pages(si->cluster_vm, vm_start, vm_end);
+       /*
+        * vm_area_unmap_pages() only clears PTEs; it does not free the
+        * physical pages.  Walk the page table to find and free them.
+        */
+       /* TODO: free backing pages via page table walk or tracking bitmap */
+       /*
+        * Pairs with READ_ONCE() in shrink/grow paths.
+        */
+       out:
+       WRITE_ONCE(si->nr_clusters_mapped, start_idx);
+}
+#endif /* CONFIG_XSWAP */
+
 static int setup_swap_clusters_info(struct swap_info_struct *si,
                                    union swap_header *swap_header,
                                    unsigned long maxpages)

@@ -3573,6 +3720,63 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
        int err = -ENOMEM;
        unsigned long i;
 
+#ifdef CONFIG_XSWAP
+       if (si->flags & SWP_XSWAP) {
+               unsigned long size = PAGE_ALIGN(nr_clusters * sizeof(*cluster_info));
+               struct vm_struct *vm;
+
+               vm = get_vm_area(size, VM_SPARSE);
+               if (!vm)
+                       goto err;
+
+               cluster_info = vm->addr;
+               si->cluster_vm = vm;
+               si->nr_clusters = nr_clusters;
+               si->cluster_info = cluster_info;
+
+               /* Map the initial chunk (at least cluster 0) */
+               if (xswap_map_clusters(si, 0, min_t(unsigned long,
+                                       XSWAP_GROW_CLUSTERS, nr_clusters)))
+                       goto err_free_vm;
+
+               /* xswap: only cluster 0 slot 0 is bad */
+               err = swap_cluster_setup_bad_slot(si, cluster_info, 0, false);
+               if (err)
+                       goto err_unmap;
+
+               INIT_LIST_HEAD(&si->free_clusters);
+               INIT_LIST_HEAD(&si->full_clusters);
+               INIT_LIST_HEAD(&si->discard_clusters);
+               for (i = 0; i < SWAP_NR_ORDERS; i++) {
+                       INIT_LIST_HEAD(&si->nonfull_clusters[i]);
+                       INIT_LIST_HEAD(&si->frag_clusters[i]);
+               }
+
+               /* Mark mapped clusters: cluster 0 has 1 bad slot, rest free */
+               for (i = 0; i < si->nr_clusters_mapped; i++) {
+                       struct swap_cluster_info *ci = &cluster_info[i];
+
+                       if (i == 0) {
+                               ci->flags = CLUSTER_FLAG_NONFULL;
+                               list_add_tail(&ci->list, &si->nonfull_clusters[0]);
+                       } else {
+                               ci->flags = CLUSTER_FLAG_FREE;
+                               list_add_tail(&ci->list, &si->free_clusters);
+                       }
+               }
+
+               return 0;
+
+err_unmap:
+               xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+err_free_vm:
+               free_vm_area(si->cluster_vm);
+               si->cluster_vm = NULL;
+               si->cluster_info = NULL;
+               return err;
+       }
+#endif /* CONFIG_XSWAP */
+
        cluster_info = kvzalloc_objs(*cluster_info, nr_clusters);
        if (!cluster_info)
                goto err;

@@ -3640,7 +3844,7 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
        si->cluster_info = cluster_info;
        return 0;
 err:
-       free_swap_cluster_info(cluster_info, maxpages);
+       free_swap_cluster_info(si);
        return err;
 }
 

@@ -3852,7 +4056,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
        si->global_cluster = NULL;
        inode = NULL;
        destroy_swap_extents(si, swap_file);
-       free_swap_cluster_info(si->cluster_info, si->max);
+       free_swap_cluster_info(si);
        si->cluster_info = NULL;
        /*
         * Clear the SWP_USED flag after all resources are freed so
--

2.54.0

Concurrent xswap grow operations race on vm_area_map_pages(), triggering WARN_ON(!pte_none) in the vmap page table walk when one thread sees a PTE already set by another. The -EBUSY safety net catches it but the warning is noisy and unavoidable without serialization.

Add a per-device mutex (xswap_lock) held across xswap_map_clusters and xswap_unmap_clusters so that only one thread can modify the VM_SPARSE page table at a time. The shrink path is already deferred to a workqueue so it does not contend with itself.

Signed-off-by Baoquan He <[email protected]>

Quoted 7 lines · Baoquan He

Concurrent xswap grow operations race on vm_area_map_pages(), triggering WARN_ON(!pte_none) in the vmap page table walk when one thread sees a PTE already set by another. The -EBUSY safety net catches it but the warning is noisy and unavoidable without serialization.

Quoted 6 lines

Add a per-device mutex (xswap_lock) held across xswap_map_clusters and xswap_unmap_clusters so that only one thread can modify the VM_SPARSE page table at a time. The shrink path is already deferred to a workqueue so it does not contend with itself.

Signed-off-by Baoquan He <[email protected]>

Seems like this should be squashed into one of the older patches, no?

Add CONFIG_XSWAP Kconfig option (depends on SWAP && 64BIT) for extendable (virtual) swap device support.

Add three fields to struct swap_info_struct under CONFIG_XSWAP:
- cluster_vm: the VM_SPARSE vm_struct backing the cluster_info array
- nr_clusters: total number of clusters in the xswap address space
- nr_clusters_mapped: number of clusters currently mapped (lazy grow)

These fields enable lazy vmalloc-based dynamic cluster management where the cluster_info array is backed by a sparse vmalloc area that grows on demand and shrinks when clusters are freed.

Signed-off-by Baoquan He <[email protected]>

---
 include/linux/swap.h | 5 +++++
 mm/Kconfig           | 9 +++++++++
 2 files changed, 14 insertions(+)

include/linux/swap.h +0 -0

index 80c83dd8804f..5f6b14d30758 100644

Quoted 30 lines · Baoquan He

Add CONFIG_XSWAP Kconfig option (depends on SWAP && 64BIT) for extendable (virtual) swap device support.

Add three fields to struct swap_info_struct under CONFIG_XSWAP:
- cluster_vm: the VM_SPARSE vm_struct backing the cluster_info array
- nr_clusters: total number of clusters in the xswap address space
- nr_clusters_mapped: number of clusters currently mapped (lazy grow)

These fields enable lazy vmalloc-based dynamic cluster management where the cluster_info array is backed by a sparse vmalloc area that grows on demand and shrinks when clusters are freed.

Signed-off-by Baoquan He <[email protected]>

---
 include/linux/swap.h | 5 +++++
 mm/Kconfig           | 9 +++++++++
 2 files changed, 14 insertions(+)

include/linux/swap.h +3 -0

index 80c83dd8804f..5f6b14d30758 100644
@@ -248,6 +248,11 @@ struct swap_info_struct {
        signed char     type;           /* strange name for an index */
        unsigned int    max;            /* size of this swap device */
        struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
+#ifdef CONFIG_XSWAP
+       struct vm_struct        *cluster_vm;    /* VM_SPARSE area for xswap dynamic cluster_info */
+       unsigned long           nr_clusters;    /* total cluster count for xswap */

include/linux/swap.h struct swap_info_struct { · line 255

Quoted 2 lines

include/linux/swap.h +2 -0

+       unsigned long           nr_clusters_mapped; /* currently mapped cluster count */
+#endif

We might need more than one dynamic kvmalloc grow array. Maybe abstract this into a struct so it can be reused for arrays other than clusters as well. The abstraction can also include the element size so it can have a generic function to calculate how many pages it needs for N elements.

This series implements step 3 of the incremental path proposed in [1]:
a virtual swap device with dynamic cluster growth and shrink, backed only by zswap, no writeback yet.

During the discussion in [2], Chris and Nhat debated xarray-based vs array-based cluster lookup for virtual swap. Chris argued for keeping the swap table and cluster_info in the same place with O(1) array indexing, and suggested that the kvmalloc grow approach here should provide the cluster management VS needs. He further proposed using the same grow technique for the future per-slot backend pointer array (vs_table), keeping a single coherent array layout throughout.

This series therefore serves as the foundation: once the dynamic cluster management lands, Nhat's VS series can build the per-slot backend pointer, writeback, and rmap on top of it — without an extra xarray lookup in the cluster access path.

Quoted 13 lines · Baoquan He

This series implements step 3 of the incremental path proposed in [1]:
a virtual swap device with dynamic cluster growth and shrink, backed only by zswap, no writeback yet.

During the discussion in [2], Chris and Nhat debated xarray-based vs array-based cluster lookup for virtual swap. Chris argued for keeping the swap table and cluster_info in the same place with O(1) array indexing, and suggested that the kvmalloc grow approach here should provide the cluster management VS needs. He further proposed using the same grow technique for the future per-slot backend pointer array (vs_table), keeping a single coherent array layout throughout.

Quoted 4 lines

This series therefore serves as the foundation: once the dynamic cluster management lands, Nhat's VS series can build the per-slot backend pointer, writeback, and rmap on top of it — without an extra xarray lookup in the cluster access path.

Thanks Baoquan. I've given it some more thoughts over the past week or so, to figure out a way to so that everyone would be happy with the final results.

To be perfectly clear, I'm not married with the xarray data structure. I only care about use case aspect (as you have seen, I switch over to swap table the moment I figure out a way to realize Kairui's design proposal - it's way cleaner) - and I've seen that you have addressed a lot of the design concerns I had that would hindered the use cases I presented. If the vmalloc data structure outperforms the xarray, it should *definitely* be the data structure we use when everything settles.

That just leaves the sequencing of our efforts :)

My only fear with landing one piece at a time is that that we land something that ended up not working for the entire big series in some unforeseen cases, and we have to rip it out or rewrite a huge chunks of it. Besides, without an actual use case, we can't really test its correctness, and purported performance improvement benefits.

That's why so far I have bitten the bullet and sent out the whole series (even if it's really large) with the core use cases implemented. I understand that it is big, and is burdensome for reviewers - but at least you know that the core use cases works - and if you don't take my work for it, you can test with it and benchmark it, etc. And with Kairui's proposal, we still maintain the old machinery, which allows us to even merge with something a bit suboptimal, then optimize later. FWIW, so far in my testing, the numbers have been quite close, at least for zswap backend, and the core concern (memory overhead) has been resolved. So it might actually already be usable out of the box? :)

This patch series has reached 700 LoCs, and I still have some correctness concerns - I have commented one in another patch. It has gotten more complicated now, which is no fault of you - we just need more logic to handle the things xarray provided us out of the box, with the hope that it would be more optimal. And that's fine. I just think it would be better to land the xarray version first to unblock users, then just slot your patch series on top as a follow-up optimization. It would show the actual win of your data structure over xarray data structure with concrete numbers :) Win-win?

But thanks again for taking a stab at it. Seriously, you're awesome. I want to re-iterate again - this is NOT a Nacked-by Nhat. Not with this patch series, nor with the direction. I'm just trying to figure out the sequencing of our efforts that would make everything work. I'll spend sometimes on my own trying to synthesize this data structure with vswap design too, to see if I can fish out more gaps, and to see if I can clean things up in a way that would make the switch from xarray to this new data structure as painless as possible. As well as the other design points we have touched on in our last conversation - it makes sense to me at the time, but I need to code some prototype out and stare at it a bit more.