Skip to contentSkip to conversation

bcache: make writeback inflight configurable in sysfs

PATCH In review [email protected] 2 participants 4 messages Feb 2, 2023 – Mar 15, 2023 bcache (block layer cache)

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/3 bcache: make writeback inflight configurable in sysfs

From: Dongsheng Yang <[email protected]>

This commit introduce a new sysfs file:
/sys/block/bcache0/bcache/writeback_inflight (read only) /sys/block/bcache0/bcache/writeback_inflight_max (read write)

(1) read the writeback_inflight will output the current inflight writeback op. (2)read the writeback_inflight_max will output the max number of writeback inflight.
(3) write the writeback_inflight_max can set the max number of writeback inflight, valid range is [1, INT_MAX).

E.g:

 $ ll /sys/block/bcache0/bcache/writeback_inflight*
-r--r--r-- 1 root root 4096 Oct 27 08:45 /sys/block/bcache0/bcache/writeback_inflight
-rw-r--r-- 1 root root 4096 Oct 27 08:45 /sys/block/bcache0/bcache/writeback_inflight_max
 $ cat /sys/block/bcache0/bcache/writeback_inflight
0
 $ cat /sys/block/bcache0/bcache/writeback_inflight_max
64
 $ echo 1024 > /sys/block/bcache0/bcache/writeback_inflight_max
 $ cat /sys/block/bcache0/bcache/writeback_inflight_max
1024
---
 drivers/md/bcache/bcache.h    |  6 ++++-
 drivers/md/bcache/sysfs.c     | 21 +++++++++++++++++
 drivers/md/bcache/writeback.c | 43 ++++++++++++++++++++++++++++++++---
 3 files changed, 66 insertions(+), 4 deletions(-)
drivers/md/bcache/bcache.h +5 -1
index aebb7ef10e63..74434a7730bb 100644
@@ -337,7 +337,11 @@ struct cached_dev {
 	struct delayed_work	writeback_rate_update;
 
 	/* Limit number of writeback bios in flight */
-	struct semaphore	in_flight;
+	atomic_t		wb_inflight;
+	unsigned long		wb_inflight_max;
+	spinlock_t		wb_inflight_lock;
+	wait_queue_head_t	wb_inflight_wait;
+
 	struct task_struct	*writeback_thread;
 	struct workqueue_struct	*writeback_write_wq;
 
drivers/md/bcache/sysfs.c +21 -0
index c6f677059214..0382b70c29d5 100644
@@ -119,6 +119,9 @@ rw_attribute(writeback_delay);
 rw_attribute(writeback_rate);
 rw_attribute(writeback_consider_fragment);
 
+read_attribute(writeback_inflight);
+rw_attribute(writeback_inflight_max);
+
 rw_attribute(writeback_rate_update_seconds);
 rw_attribute(writeback_rate_i_term_inverse);
 rw_attribute(writeback_rate_p_term_inverse);
@@ -201,6 +204,8 @@ SHOW(__bch_cached_dev)
 	var_printf(writeback_consider_fragment,	"%i");
 	var_print(writeback_delay);
 	var_print(writeback_percent);
+	sysfs_printf(writeback_inflight, "%i", atomic_read(&dc->wb_inflight));
+	sysfs_printf(writeback_inflight_max, "%li", dc->wb_inflight_max);
 	sysfs_hprint(writeback_rate,
 		     wb ? atomic_long_read(&dc->writeback_rate.rate) << 9 : 0);
 	sysfs_printf(io_errors,		"%i", atomic_read(&dc->io_errors));
@@ -448,6 +453,20 @@ STORE(__cached_dev)
 	if (attr == &sysfs_detach && dc->disk.c)
 		bch_cached_dev_detach(dc);
 
+	if (attr == &sysfs_writeback_inflight_max) {
+		ssize_t ret;
+		unsigned long v;
+
+		ret = strtoul_safe_clamp(buf, v, 1, INT_MAX);
+		if (ret)
+			return ret;
+
+		spin_lock(&dc->wb_inflight_lock);
+		dc->wb_inflight_max = v;
+		spin_unlock(&dc->wb_inflight_lock);
+		wake_up(&dc->wb_inflight_wait);
+	}
+
 	if (attr == &sysfs_stop)
 		bcache_device_stop(&dc->disk);
 
@@ -514,6 +533,8 @@ static struct attribute *bch_cached_dev_attrs[] = {
 	&sysfs_writeback_running,
 	&sysfs_writeback_delay,
 	&sysfs_writeback_percent,
+	&sysfs_writeback_inflight,
+	&sysfs_writeback_inflight_max,
 	&sysfs_writeback_rate,
 	&sysfs_writeback_consider_fragment,
 	&sysfs_writeback_rate_update_seconds,
drivers/md/bcache/writeback.c +40 -3

73 lines not on this page · raw message

Signature · 1 line
2.17.1
#1 lore raw

PATCH v2 2/3 bcache: submit writeback inflight dirty writes in batch

From: Dongsheng Yang <[email protected]>

If we have a backing device of log-structured block device (such as bcache flash dev), there is a possibility to merge the writes in writeback, as the all writes into bcache flash_dev are stored in bucket as log-structured.

That means, if we have a cached_dev as below:

        ----------------------------
        | bcache2 (cached_dev)     |
        | ------------------------ |
        | |   sdb (cache_dev)    | |
        | ------------------------ |
        | ------------------------ |
        | |   bcache1 (flash_dev)| |
        | ------------------------ |
        ----------------------------

we can merge the dirty writes in writeback, if we can submit the dirty writes in batch and around start_plug/finish_plug.

So this commit change the dirty_write to add the a dirty_io into a rb_tree, and queue a worker to submit all dirty_io, This provide a timing to merge these writes, which can improve the writeback bandwidth.

---
 drivers/md/bcache/bcache.h    |   4 ++
 drivers/md/bcache/writeback.c | 102 ++++++++++++++++++++++------------
 2 files changed, 72 insertions(+), 34 deletions(-)
drivers/md/bcache/bcache.h +4 -0
index 74434a7730bb..a82974aefc90 100644
@@ -356,6 +356,10 @@ struct cached_dev {
 	struct closure_waitlist writeback_ordering_wait;
 	atomic_t		writeback_sequence_next;
 
+	struct rb_root		writeback_ios;
+	spinlock_t		writeback_ios_lock;
+	struct work_struct	write_dirty_work;
+
 	/* For tracking sequential IO */
 #define RECENT_IO_BITS	7
 #define RECENT_IO	(1 << RECENT_IO_BITS)
drivers/md/bcache/writeback.c +68 -34

146 lines not on this page · raw message

Signature · 1 line
2.17.1
#2 lore raw

PATCH v2 3/3 bcache: support overlay bcache

In reply to [email protected]

From: Dongsheng Yang <[email protected]>

If we want to build a bcache device with backing device of a bcache flash device, we will fail with creating a duplicated sysfs filename.

E.g:
(1) we create bcache0 with vdc, then there is "/sys/block/bcache0/bcache" as a link to "/sys/block/vdc/bcache"

 $ readlink /sys/block/bcache0/bcache
../../../pci0000:00/0000:00:0b.0/virtio4/block/vdc/bcache

(2) if we continue to create bcache1 with bcache0:

 $ make-bcache -B /dev/bcache0

We will fail to register bdev with "sysfs: cannot create duplicate filename '/devices/virtual/block/bcache0/bcache'"

How this commit solving this problem?
E.g:
we have vdf as cache disk, vdc as backing disk.

 $ make-bcache -C /dev/vdf -B /dev/vdc --wipe-bcache
 $ echo 100G > /sys/block/vdf/bcache_cache/set/flash_vol_create
 $ lsblk
NAME                       MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
vdf                        252:80   0   50G  0 disk
├─bcache0                  251:0    0  100G  0 disk
└─bcache1                  251:128  0  100G  0 disk
vdc                        252:32   0  100G  0 disk
└─bcache0                  251:0    0  100G  0 disk

(a) rename sysfs file to more meaningful name:
(a.2) bcahce_cache -> sysfs filename under cache disk (/sys/block/vdf/bcache_cache) (a.1) bcache_fdev -> flash bcache device (/sys/block/bcache1/bcache_fdev) (a.4) bcache_bdev -> sysfs filename for backing disk (/sys/block/vdc/bcache_bdev) (a.3) bcache_cdev -> link to /sys/block/vdc/bcache_bdev (/sys/block/bcache0/bcache_cdev)

(b) create ->bcache lagacy link file for backward compatability

$ ll /sys/block/vdc/bcache
lrwxrwxrwx 1 root root 0 Oct 26 11:21 /sys/block/vdc/bcache -> bcache_bdev
$ ll /sys/block/bcache0/bcache
lrwxrwxrwx 1 root root 0 Oct 26 11:21 /sys/block/bcache0/bcache -> ../../../pci0000:00/0000:00:0b.0/virtio4/block/vdc/bcache_bdev
$ ll /sys/block/bcache1/bcache
lrwxrwxrwx 1 root root 0 Oct 26 11:19 /sys/block/bcache1/bcache -> bcache_fdev
$ ll /sys/block/vdf/bcache
lrwxrwxrwx 1 root root 0 Oct 26 11:17 /sys/block/vdf/bcache -> bcache_cache

These link are created with sysfs_create_link_nowarn(), that means, we dont care about the failure when creating if these links are already created. Because these lagacy sysfile are only for backwards compatability in no-overlay usecase of bcache, in the no-overlay use, bcache will never create duplicated link.

In overlay usecase after this commit, please dont use bcache link any more, instead use bcache_cdev, bcache_fdev, bcache_bdev or bcache_cache.

Then we can create a cached_dev with bcache1 (flash dev) as backing dev.

$ make-bcache -B /dev/bcache1
$ lsblk
NAME                       MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
vdf                        252:80   0   50G  0 disk
├─bcache0                  251:0    0  100G  0 disk
└─bcache1                  251:128  0  100G  0 disk
  └─bcache2                251:256  0  100G  0 disk

As a result there is a cached device bcache2 with backing device of a flash device bcache1.

        ----------------------------
        | bcache2 (cached_dev)     |
        | ------------------------ |
        | |   sdb (cache_dev)    | |
        | ------------------------ |
        | ------------------------ |
        | |   bcache1 (flash_dev)| |
        | ------------------------ |
        ----------------------------
---
 drivers/md/bcache/super.c | 38 ++++++++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)
drivers/md/bcache/super.c +34 -4
index ba3909bb6bea..8a922ce91d1e 100644
@@ -1087,12 +1087,19 @@ int bch_cached_dev_run(struct cached_dev *dc)
 
 	if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") ||
 	    sysfs_create_link(&disk_to_dev(d->disk)->kobj,
-			      &d->kobj, "bcache")) {
+			      &d->kobj, "bcache_cdev")) {
 		pr_err("Couldn't create bcache dev <-> disk sysfs symlinks\n");
 		ret = -ENOMEM;
 		goto out;
 	}
 
+	ret = sysfs_create_link_nowarn(&disk_to_dev(d->disk)->kobj,
+				       &d->kobj, "bcache");
+	if (ret && ret != -EEXIST) {
+		pr_err("Couldn't create lagacy disk sysfs ->bcache symlinks\n");
+		goto out;
+	}
+
 	dc->status_update_thread = kthread_run(cached_dev_status_update,
 					       dc, "bcache_status_update");
 	if (IS_ERR(dc->status_update_thread)) {
@@ -1461,8 +1468,16 @@ static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
 		goto err;
 
 	err = "error creating kobject";
-	if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache"))
+	if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache_bdev"))
 		goto err;
+
+	err = "error creating lagacy sysfs link";
+	ret = sysfs_create_link_nowarn(bdev_kobj(bdev), &dc->disk.kobj, "bcache");
+	if (ret && ret != -EEXIST) {
+		pr_err("Couldn't create lagacy disk sysfs ->bcache");
+		goto err;
+	}
+
 	if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
 		goto err;
 
@@ -1524,6 +1539,7 @@ static void flash_dev_flush(struct closure *cl)
 
 static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
 {
+	int ret;
 	int err = -ENOMEM;
 	struct bcache_device *d = kzalloc(sizeof(struct bcache_device),
 					  GFP_KERNEL);
@@ -1546,10 +1562,17 @@ static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
 	if (err)
 		goto err;
 
-	err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache");
+	err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache_fdev");
 	if (err)
 		goto err;
 
+	ret = sysfs_create_link_nowarn(&disk_to_dev(d->disk)->kobj,
+				       &d->kobj, "bcache");
+	if (ret && ret != -EEXIST) {
+		pr_err("Couldn't create lagacy flash dev ->bcache");
+		goto err;
+	}
+
 	bcache_device_link(d, c, "volume");
 
 	if (bch_has_feature_obso_large_bucket(&c->cache->sb)) {
@@ -2370,12 +2393,19 @@ static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
 		goto err;
 	}
 
-	if (kobject_add(&ca->kobj, bdev_kobj(bdev), "bcache")) {
+	if (kobject_add(&ca->kobj, bdev_kobj(bdev), "bcache_cache")) {
 		err = "error calling kobject_add";
 		ret = -ENOMEM;
 		goto out;
 	}
 
+	ret = sysfs_create_link_nowarn(bdev_kobj(bdev), &ca->kobj, "bcache");
+	if (ret && ret != -EEXIST) {
+		pr_err("Couldn't create lagacy disk sysfs ->cache symlinks\n");
+		goto out;
+	} else
+		ret = 0;
+
 	mutex_lock(&bch_register_lock);
 	err = register_cache_set(ca);
 	mutex_unlock(&bch_register_lock);
Signature · 1 line
2.17.1
#3 lore raw

Re: [PATCH v2 3/3] bcache: support overlay bcache

Hi,

https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/mingzhe-zou-easystack-cn/bcache-submit-writeback-inflight-dirty-writes-in-batch/20230202-110624
patch link:    https://lore.kernel.org/r/20230202030221.14397-3-mingzhe.zou%40easystack.cn
patch subject: [PATCH v2 3/3] bcache: support overlay bcache
config: ia64-randconfig-m031-20230312 (https://download.01.org/0day-ci/archive/20230315/[email protected]/config)
compiler: ia64-linux-gcc (GCC) 12.1.0

If you fix the issue, kindly add following tag where applicable

| Reported-by: kernel test robot <[email protected]>
| Reported-by: Dan Carpenter <[email protected]>
| Link: https://lore.kernel.org/r/[email protected]/

smatch warnings:
drivers/md/bcache/super.c:1573 flash_dev_run() warn: missing error code 'err'

vim +/err +1573 drivers/md/bcache/super.c

cafe563591446c Kent Overstreet   2013-03-23  1540  static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
cafe563591446c Kent Overstreet   2013-03-23  1541  {
bbce89267a538f Dongsheng Yang    2023-02-02  1542  	int ret;
2961c3bbcaec0e Luis Chamberlain  2021-10-15  1543  	int err = -ENOMEM;
cafe563591446c Kent Overstreet   2013-03-23  1544  	struct bcache_device *d = kzalloc(sizeof(struct bcache_device),
cafe563591446c Kent Overstreet   2013-03-23  1545  					  GFP_KERNEL);
cafe563591446c Kent Overstreet   2013-03-23  1546  	if (!d)
2961c3bbcaec0e Luis Chamberlain  2021-10-15  1547  		goto err_ret;
cafe563591446c Kent Overstreet   2013-03-23  1548  
cafe563591446c Kent Overstreet   2013-03-23  1549  	closure_init(&d->cl, NULL);
cafe563591446c Kent Overstreet   2013-03-23  1550  	set_closure_fn(&d->cl, flash_dev_flush, system_wq);
cafe563591446c Kent Overstreet   2013-03-23  1551  
cafe563591446c Kent Overstreet   2013-03-23  1552  	kobject_init(&d->kobj, &bch_flash_dev_ktype);
cafe563591446c Kent Overstreet   2013-03-23  1553  
4e1ebae3ee4e0c Coly Li           2020-10-01  1554  	if (bcache_device_init(d, block_bytes(c->cache), u->sectors,
c62b37d96b6eb3 Christoph Hellwig 2020-07-01  1555  			NULL, &bcache_flash_ops))
cafe563591446c Kent Overstreet   2013-03-23  1556  		goto err;
cafe563591446c Kent Overstreet   2013-03-23  1557  
cafe563591446c Kent Overstreet   2013-03-23  1558  	bcache_device_attach(d, c, u - c->uuids);
175206cf9ab631 Tang Junhui       2017-09-07  1559  	bch_sectors_dirty_init(d);
cafe563591446c Kent Overstreet   2013-03-23  1560  	bch_flash_dev_request_init(d);
2961c3bbcaec0e Luis Chamberlain  2021-10-15  1561  	err = add_disk(d->disk);
2961c3bbcaec0e Luis Chamberlain  2021-10-15  1562  	if (err)
2961c3bbcaec0e Luis Chamberlain  2021-10-15  1563  		goto err;
cafe563591446c Kent Overstreet   2013-03-23  1564  
bbce89267a538f Dongsheng Yang    2023-02-02  1565  	err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache_fdev");
2961c3bbcaec0e Luis Chamberlain  2021-10-15  1566  	if (err)
cafe563591446c Kent Overstreet   2013-03-23  1567  		goto err;
cafe563591446c Kent Overstreet   2013-03-23  1568  
bbce89267a538f Dongsheng Yang    2023-02-02  1569  	ret = sysfs_create_link_nowarn(&disk_to_dev(d->disk)->kobj,
bbce89267a538f Dongsheng Yang    2023-02-02  1570  				       &d->kobj, "bcache");
bbce89267a538f Dongsheng Yang    2023-02-02  1571  	if (ret && ret != -EEXIST) {
bbce89267a538f Dongsheng Yang    2023-02-02  1572  		pr_err("Couldn't create lagacy flash dev ->bcache");

bbce89267a538f Dongsheng Yang 2023-02-02 @1573 goto err;

Get rid of the "ret" variable and use "err" instead.

bbce89267a538f Dongsheng Yang    2023-02-02  1574  	}
bbce89267a538f Dongsheng Yang    2023-02-02  1575  
cafe563591446c Kent Overstreet   2013-03-23  1576  	bcache_device_link(d, c, "volume");
cafe563591446c Kent Overstreet   2013-03-23  1577  
5342fd4255021e Coly Li           2021-01-04  1578  	if (bch_has_feature_obso_large_bucket(&c->cache->sb)) {
5342fd4255021e Coly Li           2021-01-04  1579  		pr_err("The obsoleted large bucket layout is unsupported, set the bcache device into read-only\n");
5342fd4255021e Coly Li           2021-01-04  1580  		pr_err("Please update to the latest bcache-tools to create the cache device\n");
5342fd4255021e Coly Li           2021-01-04  1581  		set_disk_ro(d->disk, 1);
5342fd4255021e Coly Li           2021-01-04  1582  	}
5342fd4255021e Coly Li           2021-01-04  1583  
cafe563591446c Kent Overstreet   2013-03-23  1584  	return 0;
cafe563591446c Kent Overstreet   2013-03-23  1585  err:
cafe563591446c Kent Overstreet   2013-03-23  1586  	kobject_put(&d->kobj);
2961c3bbcaec0e Luis Chamberlain  2021-10-15  1587  err_ret:
2961c3bbcaec0e Luis Chamberlain  2021-10-15  1588  	return err;
cafe563591446c Kent Overstreet   2013-03-23  1589  }
Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests