bcache: add the writeback_rate of cache_set in sysfs
PATCH v2 1/2 bcache: add the writeback_rate of cache_set in sysfs
From: mingzhe <[email protected]>
The PI controller of the writeback thread takes backing as the control object. When multiple backings share a cache, it is difficult for us to get the real-time writeback_rate of cache_set.
This patch counts the writeback_rate of cache_set, and add it in sysfs.
Changelog:
v2: Fix up the missing "cancel_delayed_work_sync"
v1: Original verison
--- drivers/md/bcache/bcache.h | 6 ++++++ drivers/md/bcache/super.c | 10 ++++++++++ drivers/md/bcache/sysfs.c | 4 ++++ drivers/md/bcache/writeback.c | 30 ++++++++++++++++++++++++++++++ drivers/md/bcache/writeback.h | 2 ++ 5 files changed, 52 insertions(+)
drivers/md/bcache/bcache.h +6 -0
@@ -713,6 +713,12 @@ struct cache_set {atomic_long_t writeback_keys_done;atomic_long_t writeback_keys_failed; +uint64_t writeback_rate; +uint64_t writeback_rate_time; +uint64_t writeback_sectors; +atomic_long_t writeback_counter; +struct delayed_work writeback_rate_update; +atomic_long_t reclaim;atomic_long_t reclaimed_journal_buckets;atomic_long_t flush_write;
drivers/md/bcache/super.c +10 -0
@@ -1700,6 +1700,8 @@ static void cache_set_free(struct closure *cl)mempool_exit (&c->search);kfree (c->devices); +cancel_delayed_work_sync (&c->writeback_rate_update); +list_del (&c->list);mutex_unlock (&bch_register_lock);
@@ -1904,6 +1906,14 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)spin_lock_init (&c->btree_split_time.lock);spin_lock_init (&c->btree_read_time.lock); + c->writeback_rate =0 ; + c->writeback_sectors =0 ; + c->writeback_rate_time =local_clock (); +atomic_long_set (&c->writeback_counter,0 ); +INIT_DELAYED_WORK (&c->writeback_rate_update, + cache_set_update_writeback_rate); +schedule_delayed_work (&c->writeback_rate_update,HZ ); +bch_moving_init_cache_set (c);INIT_LIST_HEAD (&c->list);
drivers/md/bcache/sysfs.c +4 -0
@@ -100,6 +100,7 @@ read_attribute(reclaimed_journal_buckets);read_attribute (flush_write);read_attribute (writeback_keys_done);read_attribute (writeback_keys_failed); +read_attribute (cache_writeback_rate);read_attribute (io_errors);read_attribute (congested);read_attribute (cutoff_writeback);
@@ -768,6 +769,7 @@ SHOW(__bch_cache_set) atomic_long_read(&c->writeback_keys_done));sysfs_print (writeback_keys_failed,atomic_long_read (&c->writeback_keys_failed)); +sysfs_hprint (cache_writeback_rate, c->writeback_rate);if (attr == &sysfs_errors) return bch_snprint_string_list(buf, PAGE_SIZE, error_actions,
@@ -980,6 +982,8 @@ static struct attribute *bch_cache_set_attrs[] = { &sysfs_congested_read_threshold_us, &sysfs_congested_write_threshold_us, &sysfs_clear_stats, + + &sysfs_cache_writeback_rate, NULL };ATTRIBUTE_GROUPS (bch_cache_set);
drivers/md/bcache/writeback.c +30 -0
@@ -309,6 +309,33 @@ static void update_writeback_rate(struct work_struct *work)smp_mb__after_atomic (); } +void cache_set_update_writeback_rate (struct work_struct *work) +{ +struct cache_set *c =container_of (to_delayed_work (work), + struct cache_set, + writeback_rate_update); +uint64_t bytes, clock, delta; + + clock =local_clock (); + delta =div64_u64 (clock - c->writeback_rate_time,NSEC_PER_MSEC ); +if (delta >MSEC_PER_SEC ) { + c->writeback_rate_time = clock; +/* +* The c->writeback_sectors records the value of +* c->writeback_counter at the last update. +* So, the increment of write after the last update +* is c->writeback_counter - c->writeback_sectors. +*/ + c->writeback_sectors =atomic_long_sub_return ( + c->writeback_sectors, + &c->writeback_counter); + bytes = c->writeback_sectors <<9 ; + c->writeback_rate =div_u64 (bytes *MSEC_PER_SEC , delta); + } + +schedule_delayed_work (&c->writeback_rate_update,HZ ); +} +static unsigned int writeback_delay (struct cached_dev *dc,unsigned int sectors) {
@@ -379,6 +406,9 @@ static void write_dirty_finish(struct closure *cl)atomic_long_inc (ret ? &dc->disk.c->writeback_keys_failed : &dc->disk.c->writeback_keys_done); + +atomic_long_add (KEY_SIZE (&w->key), + &dc->disk.c->writeback_counter); }bch_keybuf_del (&dc->writeback_keys, w);
drivers/md/bcache/writeback.h +2 -0
@@ -152,4 +152,6 @@ void bch_sectors_dirty_init(struct bcache_device *d);void bch_cached_dev_writeback_init (struct cached_dev *dc);int bch_cached_dev_writeback_start (struct cached_dev *dc); +void cache_set_update_writeback_rate (struct work_struct *work); +#endif
Signature · 1 line
2.17.1
PATCH v2 2/2 bcache: support QoS for the writeback_rate of cache_set
From: mingzhe <[email protected]>
The PI controller of the writeback thread takes backing as the control object, but it cannot specify the upper limit.
If the backing device is not an independent disk, too fast writeback will also affect other business IO. For example, when rbd is used as the backing device, the writeback_rate is not limited, the ceph cluster will appear slow_request.
This patch supports QoS for the writeback of cache_set.
Changelog:
v2: Fix up the missing "cancel_delayed_work_sync"
v1: Original verison
--- drivers/md/bcache/bcache.h | 9 ++++ drivers/md/bcache/super.c | 12 +++++ drivers/md/bcache/sysfs.c | 18 +++++++ drivers/md/bcache/writeback.c | 99 +++++++++++++++++++++++++++++++++++ drivers/md/bcache/writeback.h | 9 ++++ 5 files changed, 147 insertions(+)
drivers/md/bcache/bcache.h +9 -0
@@ -719,6 +719,15 @@ struct cache_set {atomic_long_t writeback_counter;struct delayed_work writeback_rate_update; +uint64_t writeback_qos_bw; +uint64_t writeback_qos_io; +uint64_t writeback_qos_time; +atomic_long_t writeback_token_bw; +atomic_long_t writeback_token_io; +spinlock_t writeback_qos_lock; +wait_queue_head_t writeback_qos_wait; +struct delayed_work writeback_qos_update; +atomic_long_t reclaim;atomic_long_t reclaimed_journal_buckets;atomic_long_t flush_write;
drivers/md/bcache/super.c +12 -0
@@ -1700,6 +1700,7 @@ static void cache_set_free(struct closure *cl)mempool_exit (&c->search);kfree (c->devices); +cancel_delayed_work_sync (&c->writeback_qos_update);cancel_delayed_work_sync (&c->writeback_rate_update);list_del (&c->list);
@@ -1914,6 +1915,17 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb) cache_set_update_writeback_rate);schedule_delayed_work (&c->writeback_rate_update,HZ ); + c->writeback_qos_time =local_clock (); + c->writeback_qos_bw =WRITEBACK_QOS_BW_DEFAULT ; + c->writeback_qos_io =WRITEBACK_QOS_IOPS_DEFAULT ; +atomic_long_set (&c->writeback_token_bw,WRITEBACK_QOS_BW_DEFAULT ); +atomic_long_set (&c->writeback_token_io,WRITEBACK_QOS_IOPS_DEFAULT ); +spin_lock_init (&c->writeback_qos_lock); +init_waitqueue_head (&c->writeback_qos_wait); +INIT_DELAYED_WORK (&c->writeback_qos_update, + cache_set_update_writeback_qos); +schedule_delayed_work (&c->writeback_qos_update,HZ ); +bch_moving_init_cache_set (c);INIT_LIST_HEAD (&c->list);
drivers/md/bcache/sysfs.c +18 -0
@@ -101,6 +101,8 @@ read_attribute(flush_write);read_attribute (writeback_keys_done);read_attribute (writeback_keys_failed);read_attribute (cache_writeback_rate); +rw_attribute (cache_writeback_qos_bw); +rw_attribute (cache_writeback_qos_iops);read_attribute (io_errors);read_attribute (congested);read_attribute (cutoff_writeback);
@@ -770,6 +772,8 @@ SHOW(__bch_cache_set)sysfs_print (writeback_keys_failed,atomic_long_read (&c->writeback_keys_failed));sysfs_hprint (cache_writeback_rate, c->writeback_rate); +sysfs_hprint (cache_writeback_qos_bw, c->writeback_qos_bw); +sysfs_print (cache_writeback_qos_iops, c->writeback_qos_io);if (attr == &sysfs_errors) return bch_snprint_string_list(buf, PAGE_SIZE, error_actions,
@@ -933,6 +937,18 @@ STORE(__bch_cache_set)*/ sysfs_strtoul_clamp (gc_after_writeback, c->gc_after_writeback,0 ,1 ); +sysfs_strtoul_clamp (cache_writeback_qos_iops, + c->writeback_qos_io, +WRITEBACK_QOS_IOPS_MIN , +WRITEBACK_QOS_IOPS_MAX ); +if (attr == &sysfs_cache_writeback_qos_bw) { +uint64_t v; +strtoi_h_or_return (buf, v); + c->writeback_qos_bw =clamp_t (uint64_t, v, +WRITEBACK_QOS_BW_MIN , +WRITEBACK_QOS_BW_MAX ); + } +return size; }STORE_LOCKED (bch_cache_set)
@@ -984,6 +1000,8 @@ static struct attribute *bch_cache_set_attrs[] = { &sysfs_clear_stats, &sysfs_cache_writeback_rate, + &sysfs_cache_writeback_qos_bw, + &sysfs_cache_writeback_qos_iops, NULL };ATTRIBUTE_GROUPS (bch_cache_set);
drivers/md/bcache/writeback.c +99 -0
117 lines not on this page · raw message
drivers/md/bcache/writeback.h +9 -0
@@ -14,6 +14,14 @@#define WRITEBACK_RATE_UPDATE_SECS_MAX 60#define WRITEBACK_RATE_UPDATE_SECS_DEFAULT 5 +#define WRITEBACK_QOS_UPDATE_MSECS_MIN 50 +#define WRITEBACK_QOS_IOPS_DEFAULT 100000LLU +#define WRITEBACK_QOS_IOPS_MAX 10000000LLU +#define WRITEBACK_QOS_IOPS_MIN 1000LLU +#define WRITEBACK_QOS_BW_DEFAULT 52428800LLU +#define WRITEBACK_QOS_BW_MAX 107374182400LLU +#define WRITEBACK_QOS_BW_MIN 1048576LLU +#define BCH_AUTO_GC_DIRTY_THRESHOLD 50#define BCH_WRITEBACK_FRAGMENT_THRESHOLD_LOW 50
@@ -153,5 +161,6 @@ void bch_cached_dev_writeback_init(struct cached_dev *dc);int bch_cached_dev_writeback_start (struct cached_dev *dc);void cache_set_update_writeback_rate (struct work_struct *work); +void cache_set_update_writeback_qos (struct work_struct *work);#endif
Signature · 1 line
2.17.1