zram: support asynchronous writeback
Subsystems (3)
v2 cover zram: support asynchronous writeback
This patch series introduces asynchronous writeback to the zram module. By moving to an asynchronous model, we can perform writeback operations more efficiently.
The series is structured as follows:
The first patch is a preparatory refactoring that moves all writeback-related code into its own files (zram_wb.[ch]) without any functional changes.
The second patch introduces the core infrastructure for asynchronicity, including a dedicated kthread, a request queue, and helper functions to manage writeback requests.
The final patch enables asynchronous writeback by switching from submit_bio_wait() to the non-blocking submit_bio(). This patch also includes performance benchmarks demonstrating a 27% improvement in idle writeback speed on an Android platform.
Changes in v2:
- Rebase and spilt to a series of patchset
- Add test results
- Link to v1: https://lore.kernel.org/all/[email protected]/
Richard Chang (3):
zram: refactor writeback helpers
zram: add async writeback infrastructure
zram: enable asynchronous writeback
drivers/block/zram/Makefile | 1 + drivers/block/zram/zram_drv.c | 157 +++++++-------------- drivers/block/zram/zram_drv.h | 30 ++++ drivers/block/zram/zram_wb.c | 248 ++++++++++++++++++++++++++++++++++ drivers/block/zram/zram_wb.h | 42 ++++++ 5 files changed, 371 insertions(+), 107 deletions(-) create mode 100644 drivers/block/zram/zram_wb.c create mode 100644 drivers/block/zram/zram_wb.h
Signature · 1 line
2.50.1.565.gc32cd1483b-goog
PATCH v2 1/3 zram: refactor writeback helpers
Move writeback-related functions and data structures from zram_drv.[ch] to a new zram_wb.[ch] file. This is a pure refactoring patch with no functional changes, preparing the ground for the upcoming asynchronous writeback implementation.
--- drivers/block/zram/Makefile | 1 + drivers/block/zram/zram_drv.c | 44 +---------------------------------- drivers/block/zram/zram_drv.h | 19 +++++++++++++++ drivers/block/zram/zram_wb.c | 35 ++++++++++++++++++++++++++++ drivers/block/zram/zram_wb.h | 18 ++++++++++++++ 5 files changed, 74 insertions(+), 43 deletions(-) create mode 100644 drivers/block/zram/zram_wb.c create mode 100644 drivers/block/zram/zram_wb.h
drivers/block/zram/Makefile +1 -0
@@ -8,5 +8,6 @@ zram-$(CONFIG_ZRAM_BACKEND_LZ4HC) += backend_lz4hc.o zram-$(CONFIG_ZRAM_BACKEND_ZSTD) += backend_zstd.o zram-$(CONFIG_ZRAM_BACKEND_DEFLATE) += backend_deflate.o zram-$(CONFIG_ZRAM_BACKEND_842) += backend_842.o +zram-$(CONFIG_ZRAM_WRITEBACK) += zram_wb.o obj-$(CONFIG_ZRAM) += zram.o
drivers/block/zram/zram_drv.c +1 -43
68 lines not on this page · raw message
drivers/block/zram/zram_drv.h +19 -0
@@ -139,4 +139,23 @@ struct zram {#endif atomic_t pp_in_progress; }; + +#if definedCONFIG_ZRAM_WRITEBACK || definedCONFIG_ZRAM_MULTI_COMP +struct zram_pp_slot { +unsigned long index; +struct list_head entry; +}; + +/* +* A post-processing bucket is, essentially, a size class, this defines +* the range (in bytes) of pp-slots sizes in particular bucket. +*/ +#define PP_BUCKET_SIZE_RANGE 64 +#define NUM_PP_BUCKETS ((PAGE_SIZE / PP_BUCKET_SIZE_RANGE) + 1) + +struct zram_pp_ctl { +struct list_head pp_buckets[NUM_PP_BUCKETS ]; +}; +#endif +#endif
drivers/block/zram/zram_wb.c new file +35 -0
@@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#define KMSG_COMPONENT "zram_wb" +#define pr_fmt (fmt) KMSG_COMPONENT ": " fmt + +#include <linux/module.h> +#include <linux/kernel.h> + +#include "zram_wb.h" + +unsigned long alloc_block_bdev (struct zram *zram) +{ +unsigned long blk_idx =1 ; +retry: +/* skip 0 bit to confuse zram.handle = 0 */ + blk_idx =find_next_zero_bit (zram->bitmap, zram->nr_pages, blk_idx); +if (blk_idx == zram->nr_pages) +return 0 ; + +if (test_and_set_bit (blk_idx, zram->bitmap)) + goto retry; + +atomic64_inc (&zram->stats.bd_count); +return blk_idx; +} + +void free_block_bdev (struct zram *zram,unsigned long blk_idx) +{ +int was_set; + + was_set =test_and_clear_bit (blk_idx, zram->bitmap); +WARN_ON_ONCE (!was_set); +atomic64_dec (&zram->stats.bd_count); +} +
drivers/block/zram/zram_wb.h new file +18 -0
@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef _ZRAM_WRITEBACK_H_ +#define _ZRAM_WRITEBACK_H_ + +#include <linux/bio.h> +#include "zram_drv.h" + +#if IS_ENABLED (CONFIG_ZRAM_WRITEBACK ) +unsigned long alloc_block_bdev (struct zram *zram); +void free_block_bdev (struct zram *zram,unsigned long blk_idx); +#else +inline unsigned long alloc_block_bdev (struct zram *zram) {return 0 ; } +inline void free_block_bdev (struct zram *zram,unsigned long blk_idx) {}; +#endif + +#endif /* _ZRAM_WRITEBACK_H_ */ +
Signature · 1 line
2.50.1.565.gc32cd1483b-goog
Minchan Kim
maintainerRe: [PATCH v2 1/3] zram: refactor writeback helpers
Quoted 4 lines · Richard Chang
Move writeback-related functions and data structures from zram_drv.[ch] to a new zram_wb.[ch] file. This is a pure refactoring patch with no functional changes, preparing the ground for the upcoming asynchronous writeback implementation.
Can we also move zram_writeback_slots and scan_slots_for_writeback(iow, all the writeback related to sutff) into zram_wb.ch?
PATCH v2 2/3 zram: add async writeback infrastructure
In reply to Richard ChangIntroduce the necessary infrastructure for performing writeback operations asynchronously.
It adds a dedicated kernel thread (zram_wb_thread), a request queue for managing pending writebacks, and helper functions to deal with the writeback requests.
This patch lays the foundation for enabling asynchronous writeback in a subsequent change.
--- drivers/block/zram/zram_drv.c | 4 ++ drivers/block/zram/zram_wb.c | 114 ++++++++++++++++++++++++++++++++++ drivers/block/zram/zram_wb.h | 22 +++++++ 3 files changed, 140 insertions(+)
drivers/block/zram/zram_drv.c +4 -0
@@ -2871,6 +2871,9 @@ static int __init zram_init(void) num_devices--; } +if (setup_zram_writeback ()) + goto out_error; +return 0 ; out_error:
@@ -2880,6 +2883,7 @@ static int __init zram_init(void) static void __exit zram_exit(void) { +destroy_zram_writeback ();destroy_devices (); }
drivers/block/zram/zram_wb.c +114 -0
126 lines not on this page · raw message
drivers/block/zram/zram_wb.h +22 -0
@@ -6,12 +6,34 @@#include <linux/bio.h> #include "zram_drv.h" +struct zram_wb_request { +struct zram *zram; +unsigned long blk_idx; +struct zram_pp_slot *pps; +struct zram_pp_ctl *ppctl; +struct bio *bio; +struct list_head node; +}; + +struct zram_wb_request_list { +struct list_head head; +int count; +spinlock_t lock; +}; +#if IS_ENABLED (CONFIG_ZRAM_WRITEBACK )unsigned long alloc_block_bdev (struct zram *zram);void free_block_bdev (struct zram *zram,unsigned long blk_idx); +int setup_zram_writeback (void ); +void destroy_zram_writeback (void ); +void free_wb_request (struct zram_wb_request *req); +void enqueue_wb_request (struct zram_wb_request_list *req_list, +struct zram_wb_request *req);#else inline unsigned long alloc_block_bdev (struct zram *zram) {return 0 ; }inline void free_block_bdev (struct zram *zram,unsigned long blk_idx) {}; +inline int setup_zram_writeback (void ) {return 0 ; } +inline void destroy_zram_writeback (void ) {}#endif #endif /* _ZRAM_WRITEBACK_H_ */
Signature · 1 line
2.50.1.565.gc32cd1483b-goog
Re: [PATCH v2 2/3] zram: add async writeback infrastructure
Quoted 5 lines · Richard Chang
Introduce the necessary infrastructure for performing writeback operations asynchronously.
It adds a dedicated kernel thread (zram_wb_thread), a request queue for managing pending writebacks, and helper functions to deal with the writeback requests.
Why a new kernel thread instead of a workqueue? More memory (e.g. for a stack) is required when a new kernel thread is created compared to using the workqueue mechanism.
Bart.
PATCH v2 3/3 zram: enable asynchronous writeback
In reply to Richard ChangReplace the synchronous submit_bio_wait() with the non-blocking submit_bio() to perform writeback operations asynchronously. This leverages the infrastructure added in the previous commit to handle the completion of I/O operations in a dedicated kthread.
This change shows 27% speed improvement of idle writeback on Android platform.
Test Flow:
- mkfs on the zram device, mount it
- cp three linux-6.16-rc2.tar.gz tarball files as the data
- do idle writeback
- check bd_stat writes 185072 pages
Test Result:
idle writeback for 185072 4k-pages (~723 MiB)
$ echo all > /sys/block/zram0/idle $ time echo idle > /sys/block/zram0/writeback
Async writeback:
0m02.49s real 0m00.00s user 0m01.19s system 0m02.32s real 0m00.00s user 0m00.89s system 0m02.35s real 0m00.00s user 0m00.93s system 0m02.29s real 0m00.00s user 0m00.88s system
Sync writeback:
0m03.09s real 0m00.00s user 0m01.07s system 0m03.18s real 0m00.00s user 0m01.12s system 0m03.47s real 0m00.00s user 0m01.16s system 0m03.36s real 0m00.00s user 0m01.27s system
--- drivers/block/zram/zram_drv.c | 109 ++++++++++++++-------------------- drivers/block/zram/zram_drv.h | 11 ++++ drivers/block/zram/zram_wb.c | 103 +++++++++++++++++++++++++++++++- drivers/block/zram/zram_wb.h | 6 +- 4 files changed, 161 insertions(+), 68 deletions(-)
drivers/block/zram/zram_drv.c +45 -64
@@ -57,7 +57,6 @@ static size_t huge_class_size;static const struct block_device_operations zram_devops; -static void zram_free_page (struct zram *zram,size_t index);static int zram_read_from_zspool (struct zram *zram,struct page *page,u32 index);
@@ -96,7 +95,7 @@ static __must_check bool zram_slot_trylock(struct zram *zram, u32 index)return false; } -static void zram_slot_lock (struct zram *zram,u32 index) +void zram_slot_lock (struct zram *zram,u32 index) {unsigned long *lock = &zram->table[index].flags;
@@ -105,7 +104,7 @@ static void zram_slot_lock(struct zram *zram, u32 index)lock_acquired (slot_dep_map (zram, index), _RET_IP_); } -static void zram_slot_unlock (struct zram *zram,u32 index) +void zram_slot_unlock (struct zram *zram,u32 index) {unsigned long *lock = &zram->table[index].flags;
@@ -128,19 +127,17 @@ static unsigned long zram_get_handle(struct zram *zram, u32 index)return zram->table[index].handle; } -static void zram_set_handle (struct zram *zram,u32 index,unsigned long handle) +void zram_set_handle (struct zram *zram,u32 index,unsigned long handle) { zram->table[index].handle = handle; } -static bool zram_test_flag (struct zram *zram,u32 index, -enum zram_pageflags flag) +bool zram_test_flag (struct zram *zram,u32 index,enum zram_pageflags flag) {return zram->table[index].flags &BIT (flag); } -static void zram_set_flag (struct zram *zram,u32 index, -enum zram_pageflags flag) +void zram_set_flag (struct zram *zram,u32 index,enum zram_pageflags flag) { zram->table[index].flags |=BIT (flag); }
@@ -243,15 +240,20 @@ static struct zram_pp_ctl *init_pp_ctl(void)if (!ctl)return NULL ; +init_completion (&ctl->all_done); +atomic_set (&ctl->num_pp_slots,0 );for (idx =0 ; idx <NUM_PP_BUCKETS ; idx++)INIT_LIST_HEAD (&ctl->pp_buckets[idx]);return ctl; } -static void release_pp_slot (struct zram *zram,struct zram_pp_slot *pps) +static void remove_pp_slot_from_ctl (struct zram_pp_slot *pps) {list_del_init (&pps->entry); +} +void free_pp_slot (struct zram *zram,struct zram_pp_slot *pps) +{zram_slot_lock (zram, pps->index);zram_clear_flag (zram, pps->index,ZRAM_PP_SLOT );zram_slot_unlock (zram, pps->index);
@@ -259,6 +261,12 @@ static void release_pp_slot(struct zram *zram, struct zram_pp_slot *pps)kfree (pps); } +static void release_pp_slot (struct zram *zram,struct zram_pp_slot *pps) +{ +remove_pp_slot_from_ctl (pps); +free_pp_slot (zram, pps); +} +static void release_pp_ctl (struct zram *zram,struct zram_pp_ctl *ctl) {u32 idx;
@@ -297,6 +305,7 @@ static bool place_pp_slot(struct zram *zram, struct zram_pp_ctl *ctl,list_add (&pps->entry, &ctl->pp_buckets[bid]);zram_set_flag (zram, pps->index,ZRAM_PP_SLOT ); +atomic_inc (&ctl->num_pp_slots);return true; }
@@ -697,18 +706,18 @@ static void read_from_bdev_async(struct zram *zram, struct page *page,static int zram_writeback_slots (struct zram *zram,struct zram_pp_ctl *ctl) {unsigned long blk_idx =0 ; -struct page *page =NULL ;struct zram_pp_slot *pps; -struct bio_vec bio_vec; -struct bio bio; -int ret =0 , err; +int ret =0 ;u32 index; +int nr_pps =atomic_read (&ctl->num_pp_slots); - page =alloc_page (GFP_KERNEL ); -if (!page) -return -ENOMEM ; +if (!nr_pps) +return 0 ;while ((pps =select_pp_slot (ctl))) { +struct zram_wb_request *req; +struct page *page; +spin_lock (&zram->wb_limit_lock);if (zram->wb_limit_enable && !zram->bd_wb_limit) {spin_unlock (&zram->wb_limit_lock);
@@ -725,6 +734,13 @@ static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl) } } + req =alloc_wb_request (zram, pps, ctl, blk_idx); +if (IS_ERR (req)) { + ret =PTR_ERR (req); +break ; + } + page =bio_first_page_all (req->bio); + index = pps->index;zram_slot_lock (zram, index);/*
@@ -739,63 +755,28 @@ static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl) goto next;zram_slot_unlock (zram, index); -bio_init (&bio, zram->bdev, &bio_vec,1 , -REQ_OP_WRITE |REQ_SYNC ); - bio.bi_iter.bi_sector = blk_idx * (PAGE_SIZE >>9 ); -__bio_add_page (&bio, page,PAGE_SIZE ,0 ); - -/* -* XXX: A single page IO would be inefficient for write -* but it would be not bad as starter. -*/ - err =submit_bio_wait (&bio); -if (err) { -release_pp_slot (zram, pps); -/* -* BIO errors are not fatal, we continue and simply -* attempt to writeback the remaining objects (pages). -* At the same time we need to signal user-space that -* some writes (at least one, but also could be all of -* them) were not successful and we do so by returning -* the most recent BIO error. -*/ - ret = err; -continue ; - } - -atomic64_inc (&zram->stats.bd_writes); -zram_slot_lock (zram, index); -/* -* Same as above, we release slot lock during writeback so -* slot can change under us: slot_free() or slot_free() and -* reallocation (zram_write_page()). In both cases slot loses -* ZRAM_PP_SLOT flag. No concurrent post-processing can set -* ZRAM_PP_SLOT on such slots until current post-processing -* finishes. -*/ -if (!zram_test_flag (zram, index,ZRAM_PP_SLOT )) - goto next; - -zram_free_page (zram, index); -zram_set_flag (zram, index,ZRAM_WB ); -zram_set_handle (zram, index, blk_idx); + nr_pps--; +remove_pp_slot_from_ctl (pps); blk_idx =0 ; -atomic64_inc (&zram->stats.pages_stored); -spin_lock (&zram->wb_limit_lock); -if (zram->wb_limit_enable && zram->bd_wb_limit >0 ) - zram->bd_wb_limit -=1UL << (PAGE_SHIFT -12 ); -spin_unlock (&zram->wb_limit_lock); +submit_bio (req->bio); +continue ; + next:zram_slot_unlock (zram, index);release_pp_slot (zram, pps); +free_wb_request (req);cond_resched (); }if (blk_idx)free_block_bdev (zram, blk_idx); -if (page) -__free_page (page); + +if (nr_pps &&atomic_sub_and_test (nr_pps, &ctl->num_pp_slots)) +complete (&ctl->all_done); + +/* wait until all async bios completed */ +wait_for_completion (&ctl->all_done);return ret; }
@@ -1579,7 +1560,7 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize)return true; } -static void zram_free_page (struct zram *zram,size_t index) +void zram_free_page (struct zram *zram,size_t index) {unsigned long handle;
drivers/block/zram/zram_drv.h +11 -0
@@ -140,6 +140,13 @@ struct zram {atomic_t pp_in_progress; }; +void zram_slot_lock (struct zram *zram,u32 index); +void zram_slot_unlock (struct zram *zram,u32 index); +void zram_set_handle (struct zram *zram,u32 index,unsigned long handle); +bool zram_test_flag (struct zram *zram,u32 index,enum zram_pageflags flag); +void zram_set_flag (struct zram *zram,u32 index,enum zram_pageflags flag); +void zram_free_page (struct zram *zram,size_t index); + #if defined CONFIG_ZRAM_WRITEBACK || defined CONFIG_ZRAM_MULTI_COMP struct zram_pp_slot { unsigned long index;
@@ -155,7 +162,11 @@ struct zram_pp_slot {struct zram_pp_ctl {struct list_head pp_buckets[NUM_PP_BUCKETS ]; +struct completion all_done; +atomic_t num_pp_slots; }; + +void free_pp_slot (struct zram *zram,struct zram_pp_slot *pps);#endif #endif
drivers/block/zram/zram_wb.c +101 -2
128 lines not on this page · raw message
drivers/block/zram/zram_wb.h +4 -2
@@ -26,9 +26,11 @@ unsigned long alloc_block_bdev(struct zram *zram);void free_block_bdev (struct zram *zram,unsigned long blk_idx);int setup_zram_writeback (void );void destroy_zram_writeback (void ); +struct zram_wb_request *alloc_wb_request (struct zram *zram, +struct zram_pp_slot *pps, +struct zram_pp_ctl *ppctl, +unsigned long blk_idx);void free_wb_request (struct zram_wb_request *req); -void enqueue_wb_request (struct zram_wb_request_list *req_list, -struct zram_wb_request *req);#else inline unsigned long alloc_block_bdev (struct zram *zram) {return 0 ; }inline void free_block_bdev (struct zram *zram,unsigned long blk_idx) {};
Signature · 1 line
2.50.1.565.gc32cd1483b-goog
Re: [PATCH v2 3/3] zram: enable asynchronous writeback
drivers/block/zram/zram_wb.c static int wb_thread_func(void *data) · line 199
Quoted 11 lines · Richard Chang
drivers/block/zram/zram_wb.c +11
-0+ bio =bio_alloc (zram->bdev,1 ,REQ_OP_WRITE ,GFP_NOIO | __GFP_NOWARN); +if (!bio) { + err = -ENOMEM ; + goto out_free_page; + } + + req =kmalloc (sizeof (struct zram_wb_request ),GFP_NOIO | __GFP_NOWARN); +if (!req) { + err = -ENOMEM ; + goto out_free_bio; + }
Why are 'req' and 'bio' allocated separately instead of creating a bio_set with front padding? See also bioset_init().
Thanks,
Bart.