Use lockless lists for smp_call_function_single()
cover Use lockless lists for smp_call_function_single()
Hello,
these three patches make smp_call_function_single() use lockless lists and
cleanup abuse of csd.list in the block layer. I have some use for lockless smp_call_function_single() in printk code and that's why I've picked up Christoph's patch. Jens, can you have a look at the series please?
Honza
PATCH 1/3 kernel: use lockless list for smp_call_function_single()
From: Christoph Hellwig <[email protected]>
Make smp_call_function_single and friends more efficient by using a lockless list.
--- include/linux/elevator.h | 2 +- include/linux/smp.h | 7 ++++++- kernel/smp.c | 51 ++++++++++-------------------------------------- 3 files changed, 17 insertions(+), 43 deletions(-)
include/linux/elevator.h +1 -1
@@ -211,7 +211,7 @@ enum {#define rq_entry_fifo (ptr) list_entry((ptr), struct request, queuelist)#define rq_fifo_clear (rq) do { \ list_del_init(&(rq)->queuelist); \ - INIT_LIST_HEAD(&(rq)->csd.list); \ + (rq)->csd.llist.next = NULL; \ } while (0)#else /* CONFIG_BLOCK */
include/linux/smp.h +6 -1
@@ -11,12 +11,17 @@#include <linux/list.h> #include <linux/cpumask.h> #include <linux/init.h> +#include <linux/llist.h> extern void cpu_idle (void );typedef void (*smp_call_func_t )(void *info);struct call_single_data { -struct list_head list; +/* We still need list_head because of abuse by a block layer */ +union { +struct list_head list; +struct llist_node llist; + };smp_call_func_t func;void *info;u16 flags;
kernel/smp.c +10 -41
102 lines not on this page · raw message
Signature · 1 line
1.8.1.4
PATCH 2/3 block: Stop abusing csd.list for fifo_time
In reply to Jan KaraBlock layer currently abuses rq->csd.list.next for storing fifo_time. That is a terrible hack and completely unnecessary as well. Union achieves the same space saving in a cleaner way.
--- block/cfq-iosched.c | 8 ++++---- block/deadline-iosched.c | 8 ++++---- include/linux/blkdev.h | 1 + include/linux/elevator.h | 6 ------ 4 files changed, 9 insertions(+), 14 deletions(-)
block/cfq-iosched.c +4 -4
@@ -2382,10 +2382,10 @@ cfq_merged_requests(struct request_queue *q, struct request *rq,* reposition in fifo if next is older than rq */ if (!list_empty (&rq->queuelist) && !list_empty (&next->queuelist) && -time_before (rq_fifo_time (next),rq_fifo_time (rq)) && +time_before (next->fifo_time, rq->fifo_time) && cfqq ==RQ_CFQQ (next)) {list_move (&rq->queuelist, &next->queuelist); -rq_set_fifo_time (rq,rq_fifo_time (next)); + rq->fifo_time = next->fifo_time; }if (cfqq->next_rq == next)
@@ -2829,7 +2829,7 @@ static struct request *cfq_check_fifo(struct cfq_queue *cfqq)return NULL ; rq =rq_entry_fifo (cfqq->fifo.next); -if (time_before (jiffies,rq_fifo_time (rq))) +if (time_before (jiffies, rq->fifo_time)) rq =NULL ;cfq_log_cfqq (cfqq->cfqd, cfqq,"fifo=%p" , rq);
@@ -3942,7 +3942,7 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq)cfq_log_cfqq (cfqd, cfqq,"insert_request" );cfq_init_prio_data (cfqq,RQ_CIC (rq)); -rq_set_fifo_time (rq, jiffies + cfqd->cfq_fifo_expire[rq_is_sync (rq)]); + rq->fifo_time = jiffies + cfqd->cfq_fifo_expire[rq_is_sync (rq)];list_add_tail (&rq->queuelist, &cfqq->fifo);cfq_add_rq_rb (rq); cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group,
block/deadline-iosched.c +4 -4
@@ -106,7 +106,7 @@ deadline_add_request(struct request_queue *q, struct request *rq)/* * set expire time and add to fifo list */ -rq_set_fifo_time (rq, jiffies + dd->fifo_expire[data_dir]); + rq->fifo_time = jiffies + dd->fifo_expire[data_dir];list_add_tail (&rq->queuelist, &dd->fifo_list[data_dir]); }
@@ -174,9 +174,9 @@ deadline_merged_requests(struct request_queue *q, struct request *req,* and move into next position (next will be deleted) in fifo */ if (!list_empty (&req->queuelist) && !list_empty (&next->queuelist)) { -if (time_before (rq_fifo_time (next),rq_fifo_time (req))) { +if (time_before (next->fifo_time, req->fifo_time)) {list_move (&req->queuelist, &next->queuelist); -rq_set_fifo_time (req,rq_fifo_time (next)); + req->fifo_time = next->fifo_time; } }
@@ -230,7 +230,7 @@ static inline int deadline_check_fifo(struct deadline_data *dd, int ddir)/* * rq is expired! */ -if (time_after_eq (jiffies,rq_fifo_time (rq))) +if (time_after_eq (jiffies, rq->fifo_time))return 1 ;return 0 ;
include/linux/blkdev.h +1 -0
@@ -102,6 +102,7 @@ struct request {union {struct call_single_data csd;struct work_struct mq_flush_data; +unsigned long fifo_time; };struct request_queue *q;
include/linux/elevator.h +0 -6
@@ -202,12 +202,6 @@ enum {#define rq_end_sector (rq) (blk_rq_pos(rq) + blk_rq_sectors(rq))#define rb_entry_rq (node) rb_entry((node), struct request, rb_node) -/* -* Hack to reuse the csd.list list_head as the fifo time holder while -* the request is in the io scheduler. Saves an unsigned long in rq. -*/ -#define rq_fifo_time (rq) ((unsigned long) (rq)->csd.list.next) -#define rq_set_fifo_time (rq,exp) ((rq)->csd.list.next = (void *) (exp))#define rq_entry_fifo (ptr) list_entry((ptr), struct request, queuelist)#define rq_fifo_clear (rq) do { \ list_del_init(&(rq)->queuelist); \
Signature · 1 line
1.8.1.4
PATCH 3/3 block: Stop abusing rq->csd.list in blk-softirq
In reply to Jan KaraAbusing rq->csd.list for a list of requests to complete is rather ugly. Especially since using queuelist should be safe and much cleaner.
--- block/blk-softirq.c | 12 ++++++------ include/linux/smp.h | 6 +----- 2 files changed, 7 insertions(+), 11 deletions(-)
block/blk-softirq.c +6 -6
@@ -30,8 +30,8 @@ static void blk_done_softirq(struct softirq_action *h)while (!list_empty (&local_list)) {struct request *rq; - rq = list_entry(local_list.next, struct request, csd.list); -list_del_init (&rq->csd.list); + rq = list_entry(local_list.next, struct request, queuelist); +list_del_init (&rq->queuelist); rq->q->softirq_done_fn (rq); } }
@@ -45,9 +45,9 @@ static void trigger_softirq(void *data)local_irq_save (flags); list =this_cpu_ptr (&blk_cpu_done); -list_add_tail (&rq->csd.list, list); +list_add_tail (&rq->queuelist, list); -if (list->next == &rq->csd.list) +if (list->next == &rq->queuelist)raise_softirq_irqoff (BLOCK_SOFTIRQ );local_irq_restore (flags);
@@ -136,7 +136,7 @@ void __blk_complete_request(struct request *req)struct list_head *list; do_local: list =this_cpu_ptr (&blk_cpu_done); -list_add_tail (&req->csd.list, list); +list_add_tail (&req->queuelist, list);/* * if the list only contains our just added request,
@@ -144,7 +144,7 @@ do_local:* entries there, someone already raised the irq but it * hasn't run yet. */ -if (list->next == &req->csd.list) +if (list->next == &req->queuelist)raise_softirq_irqoff (BLOCK_SOFTIRQ ); }else if (raise_blk_irq (ccpu, req)) goto do_local;
include/linux/smp.h +1 -5
@@ -17,11 +17,7 @@ extern void cpu_idle(void);typedef void (*smp_call_func_t )(void *info);struct call_single_data { -/* We still need list_head because of abuse by a block layer */ -union { -struct list_head list; -struct llist_node llist; - }; +struct llist_node llist;smp_call_func_t func;void *info;u16 flags;
Signature · 1 line
1.8.1.4
Quoted 6 lines · Jan Kara
Hello,
these three patches make
smp_call_function_single()use lockless lists and
cleanup abuse of csd.list in the block layer. I have some use for locklesssmp_call_function_single()in printk code and that's why I've picked up Christoph's patch. Jens, can you have a look at the series please?
seconded. Also I think your two cleanups should go before my patch. I'll also dust up my other patch to use __smp_call_function_single directly in the block layer once we make progress on these.
Quoted 9 lines · Christoph Hellwig
Quoted 6 lines · Jan Kara
Hello,
these three patches make
smp_call_function_single()use lockless lists and
cleanup abuse of csd.list in the block layer. I have some use for locklesssmp_call_function_single()in printk code and that's why I've picked up Christoph's patch. Jens, can you have a look at the series please?seconded. Also I think your two cleanups should go before my patch.
Fair enough, I'll reorder the patches in the series and resend.
Quoted 2 lines
I'll also dust up my other patch to use
__smp_call_function_singledirectly in the block layer once we make progress on these.
Honza
Signature · 2 lines
Jan Kara <[email protected]> SUSE Labs, CR