loop: cleanup charging io to mem/blkcg
Subsystems (2)
v3 cover loop: cleanup charging io to mem/blkcg
Hello Guys,
Cleanup charging io to mem/blkcg a bit:
- avoid to store blkcg_css/memcg_css in loop_cmd, and store blkcg_css in loop_worker instead
- avoid to acquire ->lo_work_lock in IO path
- simplify blkcg_css query via xarray
- other misc cleanup
V3:
- one patch style change in 7/7
- rebase patch 4/7 against for-5.15/block
- add acked-by tag
V2:
- add helper of memcg_get_e_css
- cleanup #ifdef
- improve the last patch, as discussed with Dan Schatzberg
Ming Lei (7):
mm: memcontrol: add helper of memcg_get_e_css
loop: clean up blkcg association
loop: conver timer for monitoring idle worker into dwork
loop: add __loop_free_idle_workers() for covering freeing workers in
clearing FD
loop: improve loop_process_work
loop: use xarray to store workers
loop: don't add worker into idle list
drivers/block/loop.c | 320 ++++++++++++++++++++----------------- drivers/block/loop.h | 7 +- include/linux/memcontrol.h | 8 + 3 files changed, 188 insertions(+), 147 deletions(-)
Signature · 1 line
2.31.1
PATCH v3 1/7 mm: memcontrol: add helper of memcg_get_e_css
And helper of memcg_get_e_css() so that the consumer needn't to call cgroup_get_e_css(cgroup, &memory_cgrp_subsys) directly, since &memory_cgrp_subsys has to be used in case that MEMCG is enabled.
--- include/linux/memcontrol.h | 8 ++++++++ 1 file changed, 8 insertions(+)
include/linux/memcontrol.h +8 -0
@@ -1101,6 +1101,10 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,gfp_t gfp_mask, unsigned long *total_scanned); +static inline struct cgroup_subsys_state *memcg_get_e_css (struct cgroup *cgrp) +{ +return cgroup_get_e_css (cgrp, &memory_cgrp_subsys); +}#else /* CONFIG_MEMCG */ #define MEM_CGROUP_ID_SHIFT 0
@@ -1456,6 +1460,10 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, {return 0 ; } +static inline struct cgroup_subsys_state *memcg_get_e_css (struct cgroup *cgrp) +{ +return NULL ; +}#endif /* CONFIG_MEMCG */ static inline void __inc_lruvec_kmem_state (void *p,enum node_stat_item idx)
Signature · 1 line
2.31.1
PATCH v3 2/7 loop: clean up blkcg association
In reply to Ming LeiEach loop_worker is responsible for running requests originated from same blkcg, so:
1) associate with kthread in the entry of loop_process_work(), and disassociate in the end of this function, then we can avoid to do both for each request.
2) remove ->blkcg_css and ->memcg_css from 'loop_cmd' since both are per loop_worker.
Also kill #ifdef in the related functions.
--- drivers/block/loop.c | 81 +++++++++++++++++++++++--------------------- drivers/block/loop.h | 2 -- 2 files changed, 43 insertions(+), 40 deletions(-)
drivers/block/loop.c +43 -38
@@ -989,23 +989,46 @@ static inline int queue_on_root_worker(struct cgroup_subsys_state *css) {return !css || css == blkcg_root_css; } +static struct cgroup_subsys_state *loop_rq_blkcg_css (struct loop_cmd *cmd) +{ +struct request *rq =blk_mq_rq_from_pdu (cmd); +/* always use the first bio's css */ +struct blkcg *blkcg =bio_blkcg (rq->bio); + +if (blkcg) +return &blkcg->css; +return NULL ; +}#else static inline int queue_on_root_worker (struct cgroup_subsys_state *css) {return !css; } +static struct cgroup_subsys_state *loop_rq_blkcg_css (struct loop_cmd *cmd) +{ +return NULL ; +}#endif +static struct cgroup_subsys_state *loop_rq_get_memcg_css ( +struct cgroup_subsys_state *blkcg_css) +{ +if (blkcg_css) +return memcg_get_e_css (blkcg_css->cgroup); +return NULL ; +} +static void loop_queue_work (struct loop_device *lo,struct loop_cmd *cmd) {struct rb_node **node = &(lo->worker_tree.rb_node), *parent =NULL ;struct loop_worker *cur_worker, *worker =NULL ;struct work_struct *work;struct list_head *cmd_list; +struct cgroup_subsys_state *blkcg_css =loop_rq_blkcg_css (cmd);spin_lock_irq (&lo->lo_work_lock); -if (queue_on_root_worker (cmd->blkcg_css)) +if (queue_on_root_worker (blkcg_css)) goto queue_work; node = &lo->worker_tree.rb_node;
@@ -1013,10 +1036,10 @@ static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd)while (*node) { parent = *node; cur_worker = container_of(*node, struct loop_worker, rb_node); -if (cur_worker->blkcg_css == cmd->blkcg_css) { +if (cur_worker->blkcg_css == blkcg_css) { worker = cur_worker;break ; - }else if ((long )cur_worker->blkcg_css < (long )cmd->blkcg_css) { + }else if ((long )cur_worker->blkcg_css < (long )blkcg_css) { node = &(*node)->rb_left; }else { node = &(*node)->rb_right;
@@ -1030,15 +1053,10 @@ static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd)* In the event we cannot allocate a worker, just queue on the * rootcg worker and issue the I/O as the rootcg */ -if (!worker) { - cmd->blkcg_css =NULL ; -if (cmd->memcg_css) -css_put (cmd->memcg_css); - cmd->memcg_css =NULL ; +if (!worker) goto queue_work; - } - worker->blkcg_css = cmd->blkcg_css; + worker->blkcg_css = blkcg_css;css_get (worker->blkcg_css);INIT_WORK (&worker->work, loop_workfn);INIT_LIST_HEAD (&worker->cmd_list);
@@ -2162,19 +2180,6 @@ static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx,break ; } -/* always use the first bio's css */ - cmd->blkcg_css =NULL ; - cmd->memcg_css =NULL ; -#ifdef CONFIG_BLK_CGROUP -if (rq->bio && rq->bio->bi_blkg) { - cmd->blkcg_css = &bio_blkcg (rq->bio)->css; -#ifdef CONFIG_MEMCG - cmd->memcg_css = -cgroup_get_e_css (cmd->blkcg_css->cgroup, - &memory_cgrp_subsys); -#endif - } -#endif loop_queue_work (lo, cmd);return BLK_STS_OK ;
@@ -2186,28 +2191,14 @@ static void loop_handle_cmd(struct loop_cmd *cmd)const bool write =op_is_write (req_op (rq));struct loop_device *lo = rq->q->queuedata;int ret =0 ; -struct mem_cgroup *old_memcg =NULL ;if (write && (lo->lo_flags &LO_FLAGS_READ_ONLY )) { ret = -EIO ; goto failed; } -if (cmd->blkcg_css) -kthread_associate_blkcg (cmd->blkcg_css); -if (cmd->memcg_css) - old_memcg =set_active_memcg ( -mem_cgroup_from_css (cmd->memcg_css)); - ret =do_req_filebacked (lo, rq); -if (cmd->blkcg_css) -kthread_associate_blkcg (NULL ); - -if (cmd->memcg_css) { -set_active_memcg (old_memcg); -css_put (cmd->memcg_css); - } failed:/* complete non-aio request */ if (!cmd->use_aio || ret) {
@@ -2263,7 +2254,21 @@ static void loop_workfn(struct work_struct *work) {struct loop_worker *worker = container_of(work, struct loop_worker, work); -loop_process_work (worker, &worker->cmd_list, worker->lo); +struct mem_cgroup *old_memcg =NULL ; +struct cgroup_subsys_state *memcg_css =NULL ; + +kthread_associate_blkcg (worker->blkcg_css); + memcg_css =loop_rq_get_memcg_css (worker->blkcg_css); +if (memcg_css) { + old_memcg =set_active_memcg ( +mem_cgroup_from_css (memcg_css)); +loop_process_work (worker, &worker->cmd_list, worker->lo); +set_active_memcg (old_memcg); +css_put (memcg_css); + }else { +loop_process_work (worker, &worker->cmd_list, worker->lo); + } +kthread_associate_blkcg (NULL ); }static void loop_rootcg_workfn (struct work_struct *work)
drivers/block/loop.h +0 -2
@@ -77,8 +77,6 @@ struct loop_cmd {long ret;struct kiocb iocb;struct bio_vec *bvec; -struct cgroup_subsys_state *blkcg_css; -struct cgroup_subsys_state *memcg_css; };/* Support for loadable transfer modules */
Signature · 1 line
2.31.1
kernel test robot
build botRe: [PATCH V3 2/7] loop: clean up blkcg association
Thank you for the patch! Yet something to improve:
Hi Ming,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on block/for-next]
[also build test ERROR on hch-configfs/for-next linus/master v5.14-rc4 next-20210805] [cannot apply to linux/master]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Ming-Lei/loop-cleanup-charging-io-to-mem-blkcg/20210806-103713 base: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-block.git for-next config: nios2-randconfig-r032-20210804 (attached as .config) compiler: nios2-linux-gcc (GCC) 10.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/6688155f28596e93bdb7ced3578e7ada41c09552
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Ming-Lei/loop-cleanup-charging-io-to-mem-blkcg/20210806-103713 git checkout 6688155f28596e93bdb7ced3578e7ada41c09552
# save the attached .config to linux build tree
mkdir build_dir
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-10.3.0 make.cross O=build_dir ARCH=nios2 SHELL=/bin/bash drivers/block/
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
drivers/block/loop.c: In function 'loop_rq_get_memcg_css':
>> drivers/block/loop.c:1017:35: error: invalid use of undefined type 'struct cgroup_subsys_state'
1017 | return memcg_get_e_css(blkcg_css->cgroup);
| ^~vim +1017 drivers/block/loop.c
1012
1013 static struct cgroup_subsys_state *loop_rq_get_memcg_css(
1014 struct cgroup_subsys_state *blkcg_css)
1015 {
1016 if (blkcg_css)
> 1017 return memcg_get_e_css(blkcg_css->cgroup);
1018 return NULL;
1019 }
1020
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
PATCH v3 3/7 loop: conver timer for monitoring idle worker into dwork
In reply to Ming LeiNot necessary to use a timer to do that, and dwork is just fine, then we don't need to always disable interrupt when acquiring ->loop_work_lock.
--- drivers/block/loop.c | 75 ++++++++++++++++++++++---------------------- drivers/block/loop.h | 2 +- 2 files changed, 38 insertions(+), 39 deletions(-)
drivers/block/loop.c +37 -38
@@ -982,7 +982,6 @@ struct loop_worker {static void loop_workfn (struct work_struct *work);static void loop_rootcg_workfn (struct work_struct *work); -static void loop_free_idle_workers (struct timer_list *timer);#ifdef CONFIG_BLK_CGROUP static inline int queue_on_root_worker (struct cgroup_subsys_state *css)
@@ -1026,7 +1025,7 @@ static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd)struct list_head *cmd_list;struct cgroup_subsys_state *blkcg_css =loop_rq_blkcg_css (cmd); -spin_lock_irq (&lo->lo_work_lock); +spin_lock (&lo->lo_work_lock);if (queue_on_root_worker (blkcg_css)) goto queue_work;
@@ -1081,7 +1080,7 @@ static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd) }list_add_tail (&cmd->list_entry, cmd_list);queue_work (lo->workqueue, work); -spin_unlock_irq (&lo->lo_work_lock); +spin_unlock (&lo->lo_work_lock); }static void loop_update_rotational (struct loop_device *lo)
@@ -1203,6 +1202,33 @@ loop_set_status_from_info(struct loop_device *lo,return 0 ; } +static void loop_set_timer (struct loop_device *lo) +{ +schedule_delayed_work (&lo->idle_work,LOOP_IDLE_WORKER_TIMEOUT ); +} + +static void loop_free_idle_workers (struct work_struct *work) +{ + struct loop_device *lo = container_of(work, struct loop_device, + idle_work.work); +struct loop_worker *pos, *worker; + +spin_lock (&lo->lo_work_lock); +list_for_each_entry_safe (worker, pos, &lo->idle_worker_list, + idle_list) { +if (time_is_after_jiffies (worker->last_ran_at + +LOOP_IDLE_WORKER_TIMEOUT )) +break ; +list_del (&worker->idle_list); +rb_erase (&worker->rb_node, &lo->worker_tree); +css_put (worker->blkcg_css); +kfree (worker); + } +if (!list_empty (&lo->idle_worker_list)) +loop_set_timer (lo); +spin_unlock (&lo->lo_work_lock); +} +static int loop_configure (struct loop_device *lo,fmode_t mode,struct block_device *bdev,const struct loop_config *config)
@@ -1283,8 +1309,7 @@ static int loop_configure(struct loop_device *lo, fmode_t mode,INIT_LIST_HEAD (&lo->rootcg_cmd_list);INIT_LIST_HEAD (&lo->idle_worker_list); lo->worker_tree =RB_ROOT ; -timer_setup (&lo->timer, loop_free_idle_workers, -TIMER_DEFERRABLE ); +INIT_DELAYED_WORK (&lo->idle_work, loop_free_idle_workers); lo->use_dio = lo->lo_flags &LO_FLAGS_DIRECT_IO ; lo->lo_device = bdev; lo->lo_backing_file = file;
@@ -1384,7 +1409,7 @@ static int __loop_clr_fd(struct loop_device *lo, bool release)blk_mq_freeze_queue (lo->lo_queue);destroy_workqueue (lo->workqueue); -spin_lock_irq (&lo->lo_work_lock); +spin_lock (&lo->lo_work_lock);list_for_each_entry_safe (worker, pos, &lo->idle_worker_list, idle_list) {list_del (&worker->idle_list);
@@ -1392,8 +1417,8 @@ static int __loop_clr_fd(struct loop_device *lo, bool release)css_put (worker->blkcg_css);kfree (worker); } -spin_unlock_irq (&lo->lo_work_lock); -del_timer_sync (&lo->timer); +spin_unlock (&lo->lo_work_lock); +cancel_delayed_work_sync (&lo->idle_work);spin_lock_irq (&lo->lo_lock); lo->lo_backing_file =NULL ;
@@ -2211,11 +2236,6 @@ static void loop_handle_cmd(struct loop_cmd *cmd) } } -static void loop_set_timer (struct loop_device *lo) -{ -timer_reduce (&lo->timer, jiffies +LOOP_IDLE_WORKER_TIMEOUT ); -} -static void loop_process_work (struct loop_worker *worker,struct list_head *cmd_list,struct loop_device *lo) {
@@ -2223,17 +2243,17 @@ static void loop_process_work(struct loop_worker *worker,struct loop_cmd *cmd; current->flags |=PF_LOCAL_THROTTLE |PF_MEMALLOC_NOIO ; -spin_lock_irq (&lo->lo_work_lock); +spin_lock (&lo->lo_work_lock);while (!list_empty (cmd_list)) { cmd =container_of ( cmd_list->next, struct loop_cmd, list_entry);list_del (cmd_list->next); -spin_unlock_irq (&lo->lo_work_lock); +spin_unlock (&lo->lo_work_lock);loop_handle_cmd (cmd);cond_resched (); -spin_lock_irq (&lo->lo_work_lock); +spin_lock (&lo->lo_work_lock); }/*
@@ -2246,7 +2266,7 @@ static void loop_process_work(struct loop_worker *worker,list_add_tail (&worker->idle_list, &lo->idle_worker_list);loop_set_timer (lo); } -spin_unlock_irq (&lo->lo_work_lock); +spin_unlock (&lo->lo_work_lock); current->flags = orig_flags; }
@@ -2278,27 +2298,6 @@ static void loop_rootcg_workfn(struct work_struct *work)loop_process_work (NULL , &lo->rootcg_cmd_list, lo); } -static void loop_free_idle_workers (struct timer_list *timer) -{ - struct loop_device *lo = container_of(timer, struct loop_device, timer); -struct loop_worker *pos, *worker; - -spin_lock_irq (&lo->lo_work_lock); -list_for_each_entry_safe (worker, pos, &lo->idle_worker_list, - idle_list) { -if (time_is_after_jiffies (worker->last_ran_at + -LOOP_IDLE_WORKER_TIMEOUT )) -break ; -list_del (&worker->idle_list); -rb_erase (&worker->rb_node, &lo->worker_tree); -css_put (worker->blkcg_css); -kfree (worker); - } -if (!list_empty (&lo->idle_worker_list)) -loop_set_timer (lo); -spin_unlock_irq (&lo->lo_work_lock); -} -static const struct blk_mq_ops loop_mq_ops = { .queue_rq = loop_queue_rq, .complete = lo_complete_rq,
drivers/block/loop.h +1 -1
@@ -60,7 +60,7 @@ struct loop_device {struct list_head rootcg_cmd_list;struct list_head idle_worker_list;struct rb_root worker_tree; -struct timer_list timer; +struct delayed_work idle_work;bool use_dio;bool sysfs_inited;
Signature · 1 line
2.31.1
PATCH v3 4/7 loop: add __loop_free_idle_workers() for covering freeing workers in clearing FD
In reply to Ming LeiAdd the helper, so we can remove the duplicated code for freeing all workers in clearing FD.
--- drivers/block/loop.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-)
drivers/block/loop.c +11 -14
@@ -1207,16 +1207,14 @@ static void loop_set_timer(struct loop_device *lo)schedule_delayed_work (&lo->idle_work,LOOP_IDLE_WORKER_TIMEOUT ); } -static void loop_free_idle_workers (struct work_struct *work) +static void __loop_free_idle_workers (struct loop_device *lo,bool force) { - struct loop_device *lo = container_of(work, struct loop_device, - idle_work.work);struct loop_worker *pos, *worker;spin_lock (&lo->lo_work_lock);list_for_each_entry_safe (worker, pos, &lo->idle_worker_list, idle_list) { -if (time_is_after_jiffies (worker->last_ran_at + +if (!force &&time_is_after_jiffies (worker->last_ran_at +LOOP_IDLE_WORKER_TIMEOUT ))break ;list_del (&worker->idle_list);
@@ -1229,6 +1227,14 @@ static void loop_free_idle_workers(struct work_struct *work)spin_unlock (&lo->lo_work_lock); } +static void loop_free_idle_workers (struct work_struct *work) +{ + struct loop_device *lo = container_of(work, struct loop_device, + idle_work.work); + +__loop_free_idle_workers (lo, false); +} +static int loop_configure (struct loop_device *lo,fmode_t mode,struct block_device *bdev,const struct loop_config *config)
@@ -1376,7 +1382,6 @@ static int __loop_clr_fd(struct loop_device *lo, bool release)int err =0 ;bool partscan = false;int lo_number; -struct loop_worker *pos, *worker;/* * Flush loop_configure() and loop_change_fd(). It is acceptable for
@@ -1409,15 +1414,7 @@ static int __loop_clr_fd(struct loop_device *lo, bool release)blk_mq_freeze_queue (lo->lo_queue);destroy_workqueue (lo->workqueue); -spin_lock (&lo->lo_work_lock); -list_for_each_entry_safe (worker, pos, &lo->idle_worker_list, - idle_list) { -list_del (&worker->idle_list); -rb_erase (&worker->rb_node, &lo->worker_tree); -css_put (worker->blkcg_css); -kfree (worker); - } -spin_unlock (&lo->lo_work_lock); +__loop_free_idle_workers (lo, true);cancel_delayed_work_sync (&lo->idle_work);spin_lock_irq (&lo->lo_lock);
Signature · 1 line
2.31.1
PATCH v3 5/7 loop: improve loop_process_work
In reply to Ming LeiAvoid to acquire the spinlock for handling every loop command, and hold lock once for taking all commands.
--- drivers/block/loop.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-)
drivers/block/loop.c +13 -8
@@ -2238,21 +2238,26 @@ static void loop_process_work(struct loop_worker *worker, {int orig_flags = current->flags;struct loop_cmd *cmd; +LIST_HEAD (list); current->flags |=PF_LOCAL_THROTTLE |PF_MEMALLOC_NOIO ; +spin_lock (&lo->lo_work_lock); -while (!list_empty (cmd_list)) { - cmd =container_of ( - cmd_list->next, struct loop_cmd, list_entry); -list_del (cmd_list->next); -spin_unlock (&lo->lo_work_lock); + again: +list_splice_init (cmd_list, &list); +spin_unlock (&lo->lo_work_lock); -loop_handle_cmd (cmd); -cond_resched (); +while (!list_empty (&list)) { + cmd = list_first_entry(&list, struct loop_cmd, list_entry); +list_del_init (&cmd->list_entry); -spin_lock (&lo->lo_work_lock); +loop_handle_cmd (cmd); } +spin_lock (&lo->lo_work_lock); +if (!list_empty (cmd_list)) + goto again; +/* * We only add to the idle list if there are no pending cmds * *and* the worker will not run again which ensures that it
Signature · 1 line
2.31.1
PATCH v3 6/7 loop: use xarray to store workers
In reply to Ming Leicss->id is unique in io controller wide, so replace rbtree with xarray for querying/storing 'blkcg_css' by using css->id as key, then code is simplified a lot.
--- drivers/block/loop.c | 36 ++++++++++++++---------------------- drivers/block/loop.h | 3 ++- 2 files changed, 16 insertions(+), 23 deletions(-)
drivers/block/loop.c +14 -22
@@ -971,7 +971,6 @@ static void loop_config_discard(struct loop_device *lo) }struct loop_worker { -struct rb_node rb_node;struct work_struct work;struct list_head cmd_list;struct list_head idle_list;
@@ -1019,35 +1018,23 @@ static struct cgroup_subsys_state *loop_rq_get_memcg_css(static void loop_queue_work (struct loop_device *lo,struct loop_cmd *cmd) { -struct rb_node **node = &(lo->worker_tree.rb_node), *parent =NULL ; -struct loop_worker *cur_worker, *worker =NULL ; +struct loop_worker *worker =NULL ;struct work_struct *work;struct list_head *cmd_list;struct cgroup_subsys_state *blkcg_css =loop_rq_blkcg_css (cmd); +gfp_t gfp =GFP_NOWAIT | __GFP_NOWARN;spin_lock (&lo->lo_work_lock);if (queue_on_root_worker (blkcg_css)) goto queue_work; - node = &lo->worker_tree.rb_node; - -while (*node) { - parent = *node; - cur_worker = container_of(*node, struct loop_worker, rb_node); -if (cur_worker->blkcg_css == blkcg_css) { - worker = cur_worker; -break ; - }else if ((long )cur_worker->blkcg_css < (long )blkcg_css) { - node = &(*node)->rb_left; - }else { - node = &(*node)->rb_right; - } - } +/* css->id is unique in each cgroup subsystem */ + worker =xa_load (&lo->workers, blkcg_css->id);if (worker) goto queue_work; - worker =kzalloc (sizeof (struct loop_worker ),GFP_NOWAIT | __GFP_NOWARN); + worker =kzalloc (sizeof (*worker), gfp);/* * In the event we cannot allocate a worker, just queue on the * rootcg worker and issue the I/O as the rootcg
@@ -1061,8 +1048,12 @@ static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd)INIT_LIST_HEAD (&worker->cmd_list);INIT_LIST_HEAD (&worker->idle_list); worker->lo = lo; -rb_link_node (&worker->rb_node, parent, node); -rb_insert_color (&worker->rb_node, &lo->worker_tree); + +if (xa_err (xa_store (&lo->workers, blkcg_css->id, worker, gfp))) { +kfree (worker); + worker =NULL ; + } + queue_work:if (worker) {/*
@@ -1218,7 +1209,7 @@ static void __loop_free_idle_workers(struct loop_device *lo, bool force) LOOP_IDLE_WORKER_TIMEOUT))break ;list_del (&worker->idle_list); -rb_erase (&worker->rb_node, &lo->worker_tree); +xa_erase (&lo->workers, worker->blkcg_css->id);css_put (worker->blkcg_css);kfree (worker); }
@@ -1314,7 +1305,7 @@ static int loop_configure(struct loop_device *lo, fmode_t mode,INIT_WORK (&lo->rootcg_work, loop_rootcg_workfn);INIT_LIST_HEAD (&lo->rootcg_cmd_list);INIT_LIST_HEAD (&lo->idle_worker_list); - lo->worker_tree =RB_ROOT ; +xa_init (&lo->workers);INIT_DELAYED_WORK (&lo->idle_work, loop_free_idle_workers); lo->use_dio = lo->lo_flags &LO_FLAGS_DIRECT_IO ; lo->lo_device = bdev;
@@ -1416,6 +1407,7 @@ static int __loop_clr_fd(struct loop_device *lo, bool release)destroy_workqueue (lo->workqueue);__loop_free_idle_workers (lo, true);cancel_delayed_work_sync (&lo->idle_work); +xa_destroy (&lo->workers);spin_lock_irq (&lo->lo_lock); lo->lo_backing_file =NULL ;
drivers/block/loop.h +2 -1
@@ -14,6 +14,7 @@#include <linux/blk-mq.h> #include <linux/spinlock.h> #include <linux/mutex.h> +#include <linux/xarray.h> #include <uapi/linux/loop.h> /* Possible states of device */
@@ -59,7 +60,7 @@ struct loop_device {struct work_struct rootcg_work;struct list_head rootcg_cmd_list;struct list_head idle_worker_list; -struct rb_root worker_tree; +struct xarray workers;struct delayed_work idle_work;bool use_dio;bool sysfs_inited;
Signature · 1 line
2.31.1
PATCH v3 7/7 loop: don't add worker into idle list
In reply to Ming LeiWe can retrieve any workers via xarray, so not add it into idle list. Meantime reduce .lo_work_lock coverage, especially we don't need that in IO path except for adding/deleting worker into xarray.
Also replace .last_ran_at with .reclaim_time, which is set when adding loop command into worker->cmd_list. Meantime reclaim the worker when the worker is expired and no any pending commands.
--- drivers/block/loop.c | 172 ++++++++++++++++++++++++++----------------- 1 file changed, 104 insertions(+), 68 deletions(-)
drivers/block/loop.c +104 -68
@@ -973,10 +973,11 @@ static void loop_config_discard(struct loop_device *lo)struct loop_worker {struct work_struct work;struct list_head cmd_list; -struct list_head idle_list;struct loop_device *lo;struct cgroup_subsys_state *blkcg_css; -unsigned long last_ran_at; +unsigned long reclaim_time; +spinlock_t lock; +refcount_t refcnt; };static void loop_workfn (struct work_struct *work);
@@ -1016,62 +1017,93 @@ static struct cgroup_subsys_state *loop_rq_get_memcg_css(return NULL ; } -static void loop_queue_work (struct loop_device *lo,struct loop_cmd *cmd) +static struct loop_worker *loop_alloc_or_get_worker (struct loop_device *lo, +struct cgroup_subsys_state *blkcg_css) { -struct loop_worker *worker =NULL ; -struct work_struct *work; -struct list_head *cmd_list; -struct cgroup_subsys_state *blkcg_css =loop_rq_blkcg_css (cmd);gfp_t gfp =GFP_NOWAIT | __GFP_NOWARN; +struct loop_worker *worker =kzalloc (sizeof (*worker), gfp); +struct loop_worker *worker_old; -spin_lock (&lo->lo_work_lock); - -if (queue_on_root_worker (blkcg_css)) - goto queue_work; - -/* css->id is unique in each cgroup subsystem */ - worker =xa_load (&lo->workers, blkcg_css->id); -if (worker) - goto queue_work; - - worker =kzalloc (sizeof (*worker), gfp); -/* -* In the event we cannot allocate a worker, just queue on the -* rootcg worker and issue the I/O as the rootcg -*/ if (!worker) - goto queue_work; +return NULL ; worker->blkcg_css = blkcg_css; -css_get (worker->blkcg_css);INIT_WORK (&worker->work, loop_workfn);INIT_LIST_HEAD (&worker->cmd_list); -INIT_LIST_HEAD (&worker->idle_list); worker->lo = lo; +spin_lock_init (&worker->lock); +refcount_set (&worker->refcnt,2 );/* INIT + INC */ -if (xa_err (xa_store (&lo->workers, blkcg_css->id, worker, gfp))) { +spin_lock (&lo->lo_work_lock); +/* maybe someone is storing a new worker */ + worker_old =xa_load (&lo->workers, blkcg_css->id); +if (!worker_old || !refcount_inc_not_zero (&worker_old->refcnt)) { +if (xa_err (xa_store (&lo->workers, blkcg_css->id, worker, gfp))) { +kfree (worker); + worker =NULL ; + }else { +if (!work_pending (&lo->idle_work.work)) +schedule_delayed_work (&lo->idle_work, +LOOP_IDLE_WORKER_TIMEOUT ); +css_get (worker->blkcg_css); + } + }else {kfree (worker); - worker =NULL ; + worker = worker_old; } +spin_unlock (&lo->lo_work_lock); -queue_work: -if (worker) { +return worker; +} + +static void loop_release_worker (struct loop_worker *worker) +{ +css_put (worker->blkcg_css); +kfree_rcu (worker); +} + +static void loop_queue_work (struct loop_device *lo,struct loop_cmd *cmd) +{ +struct loop_worker *worker =NULL ; +struct work_struct *work; +struct list_head *cmd_list; +struct cgroup_subsys_state *blkcg_css =loop_rq_blkcg_css (cmd); +spinlock_t *lock; + +if (!queue_on_root_worker (blkcg_css)) { +int ret =0 ; + +rcu_read_lock (); +/* css->id is unique in each cgroup subsystem */ + worker =xa_load (&lo->workers, blkcg_css->id); +if (worker) + ret =refcount_inc_not_zero (&worker->refcnt); +rcu_read_unlock (); + +if (!worker || !ret) + worker =loop_alloc_or_get_worker (lo, blkcg_css);/* -* We need to remove from the idle list here while -* holding the lock so that the idle timer doesn't -* free the worker +* In the event we cannot allocate a worker, just queue on the +* rootcg worker and issue the I/O as the rootcg */ -if (!list_empty (&worker->idle_list)) -list_del_init (&worker->idle_list); + } + +if (worker) { work = &worker->work; cmd_list = &worker->cmd_list; + lock = &worker->lock; }else { work = &lo->rootcg_work; cmd_list = &lo->rootcg_cmd_list; + lock = &lo->lo_work_lock; } + +spin_lock (lock);list_add_tail (&cmd->list_entry, cmd_list); +if (worker) + worker->reclaim_time = jiffies +LOOP_IDLE_WORKER_TIMEOUT ; +spin_unlock (lock);queue_work (lo->workqueue, work); -spin_unlock (&lo->lo_work_lock); }static void loop_update_rotational (struct loop_device *lo)
@@ -1193,28 +1225,38 @@ loop_set_status_from_info(struct loop_device *lo,return 0 ; } -static void loop_set_timer (struct loop_device *lo) +static bool loop_need_reclaim_worker (struct loop_worker *worker) { -schedule_delayed_work (&lo->idle_work,LOOP_IDLE_WORKER_TIMEOUT ); +bool reclaim; + +spin_lock (&worker->lock); +if (list_empty (&worker->cmd_list) && +time_is_before_jiffies (worker->reclaim_time)) + reclaim = true; +else + reclaim = false; +spin_unlock (&worker->lock); + +return reclaim; }static void __loop_free_idle_workers (struct loop_device *lo,bool force) { -struct loop_worker *pos, *worker; +struct loop_worker *worker; +unsigned long id;spin_lock (&lo->lo_work_lock); -list_for_each_entry_safe (worker, pos, &lo->idle_worker_list, - idle_list) { -if (!force &&time_is_after_jiffies (worker->last_ran_at + -LOOP_IDLE_WORKER_TIMEOUT )) -break ; -list_del (&worker->idle_list); -xa_erase (&lo->workers, worker->blkcg_css->id); -css_put (worker->blkcg_css); -kfree (worker); +xa_for_each (&lo->workers, id, worker) { +if (!force && !loop_need_reclaim_worker (worker)) +continue ; + +xa_erase (&worker->lo->workers, worker->blkcg_css->id); +if (refcount_dec_and_test (&worker->refcnt)) +loop_release_worker (worker); } -if (!list_empty (&lo->idle_worker_list)) -loop_set_timer (lo); +if (!xa_empty (&lo->workers)) +schedule_delayed_work (&lo->idle_work, +LOOP_IDLE_WORKER_TIMEOUT );spin_unlock (&lo->lo_work_lock); }
@@ -2226,42 +2268,36 @@ static void loop_handle_cmd(struct loop_cmd *cmd) }static void loop_process_work (struct loop_worker *worker, -struct list_head *cmd_list,struct loop_device *lo) +struct list_head *cmd_list,spinlock_t *lock) {int orig_flags = current->flags;struct loop_cmd *cmd;LIST_HEAD (list); +int cnt =0 ; current->flags |=PF_LOCAL_THROTTLE |PF_MEMALLOC_NOIO ; -spin_lock (&lo->lo_work_lock); +spin_lock (lock); again:list_splice_init (cmd_list, &list); -spin_unlock (&lo->lo_work_lock); +spin_unlock (lock);while (!list_empty (&list)) { cmd = list_first_entry(&list, struct loop_cmd, list_entry);list_del_init (&cmd->list_entry);loop_handle_cmd (cmd); + cnt++; } -spin_lock (&lo->lo_work_lock); +spin_lock (lock);if (!list_empty (cmd_list)) goto again; - -/* -* We only add to the idle list if there are no pending cmds -* *and* the worker will not run again which ensures that it -* is safe to free any worker on the idle list -*/ -if (worker && !work_pending (&worker->work)) { - worker->last_ran_at = jiffies; -list_add_tail (&worker->idle_list, &lo->idle_worker_list); -loop_set_timer (lo); - } -spin_unlock (&lo->lo_work_lock); +spin_unlock (lock); current->flags = orig_flags; + +if (worker &&refcount_sub_and_test (cnt, &worker->refcnt)) +loop_release_worker (worker); }static void loop_workfn (struct work_struct *work)
@@ -2276,11 +2312,11 @@ static void loop_workfn(struct work_struct *work)if (memcg_css) { old_memcg =set_active_memcg (mem_cgroup_from_css (memcg_css)); -loop_process_work (worker, &worker->cmd_list, worker->lo); +loop_process_work (worker, &worker->cmd_list, &worker->lock);set_active_memcg (old_memcg);css_put (memcg_css); }else { -loop_process_work (worker, &worker->cmd_list, worker->lo); +loop_process_work (worker, &worker->cmd_list, &worker->lock); }kthread_associate_blkcg (NULL ); }
@@ -2289,7 +2325,7 @@ static void loop_rootcg_workfn(struct work_struct *work) {struct loop_device *lo = container_of(work, struct loop_device, rootcg_work); -loop_process_work (NULL , &lo->rootcg_cmd_list, lo); +loop_process_work (NULL , &lo->rootcg_cmd_list, &lo->lo_work_lock); }static const struct blk_mq_ops loop_mq_ops = {
Signature · 1 line
2.31.1