SUNRPC: remainder of srv queueing work
Subsystems (5)
cover SUNRPC: remainder of srv queueing work
This series is on top of topic-sunrpc-thread-scheduling (4fa42add1a7)
It includes a fixup for an earlier patch in that topic, revised versions of patches that have been sent but are yet to land, and the remainder of what I have been working on.
With the full set enqueueing and dequeueing of the list of idle threads is completely lockless following a LIFO discipline. Threads are woken gradually without the possibility of a thundering herd. If lots of work becomes ready only one thread is woken and once it dequeues the first work item, it will dequeue another thread if there is more work to do.
Queueing and dequeueing for work-items for these threads is partially lockless following a FIFO discipline. Enqueueing, which can happen in bh context, is lockless. Dequeueing in the server threads still requires a spinlock, but does not need to disable 'bh'.
NeilBrown
[PATCH 01/10] SQUASH: SUNRPC: rename and refactor svc_get_next_xprt() [PATCH 02/10] SUNRPC: add list of idle threads
[PATCH 03/10] SUNRPC: discard SP_CONGESTED
[PATCH 04/10] SUNRPC: change service idle list to be an llist [PATCH 05/10] SUNRPC: only have one thread waking up at a time [PATCH 06/10] SUNRPC/svc: add light-weight queuing mechanism. [PATCH 07/10] SUNRPC: use lwq for sp_sockets - renamed to sp_xprts [PATCH 08/10] SUNRPC: change sp_nrthreads to atomic_t [PATCH 09/10] SUNRPC: discard sp_lock
[PATCH 10/10] SUNRPC: change the back-channel queue to lwq
PATCH 1/10 SQUASH: SUNRPC: rename and refactor svc_get_next_xprt()
Previous patch dropped the "!" when moving this test
--- net/sunrpc/svc_xprt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
net/sunrpc/svc_xprt.c +1 -1
@@ -867,7 +867,7 @@ void svc_recv(struct svc_rqst *rqstp)/* Normally we will wait up to 5 seconds for any required * cache information to be provided. */ -if (test_bit (SP_CONGESTED , &pool->sp_flags)) +if (!test_bit (SP_CONGESTED , &pool->sp_flags)) rqstp->rq_chandle.thread_wait =5 *HZ ;else rqstp->rq_chandle.thread_wait =1 *HZ ;
Signature · 1 line
2.40.1
PATCH 2/10 SUNRPC: add list of idle threads
In reply to NeilBrownRather than searching a list of threads to find an idle one, having a list of idle threads allows an idle thread to be found immediately.
This adds some spin_lock calls which is not ideal, but as the hold-time is tiny it is still faster than searching a list. A future patch will remove them using llist.h. This involves some subtlety and so is left to a separate patch.
This removes the need for the RQ_BUSY flag. The rqst is "busy" precisely when it is not on the "idle" list.
--- include/linux/sunrpc/svc.h | 25 ++++++++++++++++++++++++- include/trace/events/sunrpc.h | 1 - net/sunrpc/svc.c | 14 +++++++++----- net/sunrpc/svc_xprt.c | 15 +++++++++++---- 4 files changed, 44 insertions(+), 11 deletions(-)
include/linux/sunrpc/svc.h +24 -1
@@ -37,6 +37,7 @@ struct svc_pool {struct list_head sp_sockets;/* pending sockets */ unsigned int sp_nrthreads;/* # of threads in pool */ struct list_head sp_all_threads;/* all server threads */ +struct list_head sp_idle_threads;/* idle server threads */ /* statistics on pool operation */ struct percpu_counter sp_messages_arrived;
@@ -186,6 +187,7 @@ extern u32 svc_max_payload(const struct svc_rqst *rqstp);*/ struct svc_rqst {struct list_head rq_all;/* all threads list */ +struct list_head rq_idle;/* On the idle list */ struct rcu_head rq_rcu_head;/* for RCU deferred kfree */ struct svc_xprt * rq_xprt;/* transport ptr */
@@ -262,10 +264,31 @@ enum {RQ_SPLICE_OK ,/* turned off in gss privacy to prevent * encrypting page cache pages */ RQ_VICTIM ,/* Have agreed to shut down */ -RQ_BUSY ,/* request is busy */ RQ_DATA, /* request has data */ }; +/** +* svc_thread_set_busy - mark a thread as busy +* @rqstp: the thread which is now busy +* +* If rq_idle is "empty", the thread must be busy. +*/ +static inline void svc_thread_set_busy (struct svc_rqst *rqstp) +{ +INIT_LIST_HEAD (&rqstp->rq_idle); +} + +/** +* svc_thread_busy - check if a thread as busy +* @rqstp: the thread which might be busy +* +* If rq_idle is "empty", the thread must be busy. +*/ +static inline bool svc_thread_busy (struct svc_rqst *rqstp) +{ +return list_empty (&rqstp->rq_idle); +} +#define SVC_NET (rqst) (rqst->rq_xprt ? rqst->rq_xprt->xpt_net : rqst->rq_bc_net)/*
include/trace/events/sunrpc.h +0 -1
@@ -1677,7 +1677,6 @@ DEFINE_SVCXDRBUF_EVENT(sendto); svc_rqst_flag(DROPME) \ svc_rqst_flag(SPLICE_OK) \ svc_rqst_flag(VICTIM) \ - svc_rqst_flag(BUSY) \ svc_rqst_flag_end(DATA)#undef svc_rqst_flag
net/sunrpc/svc.c +9 -5
@@ -510,6 +510,7 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools, pool->sp_id = i;INIT_LIST_HEAD (&pool->sp_sockets);INIT_LIST_HEAD (&pool->sp_all_threads); +INIT_LIST_HEAD (&pool->sp_idle_threads);spin_lock_init (&pool->sp_lock);percpu_counter_init (&pool->sp_messages_arrived,0 ,GFP_KERNEL );
@@ -641,7 +642,7 @@ svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node)folio_batch_init (&rqstp->rq_fbatch); -__set_bit (RQ_BUSY , &rqstp->rq_flags); +svc_thread_set_busy (rqstp); rqstp->rq_server = serv; rqstp->rq_pool = pool;
@@ -702,10 +703,13 @@ void svc_pool_wake_idle_thread(struct svc_pool *pool)struct svc_rqst *rqstp;rcu_read_lock (); -list_for_each_entry_rcu (rqstp, &pool->sp_all_threads, rq_all) { -if (test_and_set_bit (RQ_BUSY , &rqstp->rq_flags)) -continue ; - +spin_lock_bh (&pool->sp_lock); + rqstp =list_first_entry_or_null (&pool->sp_idle_threads, + struct svc_rqst, rq_idle); +if (rqstp) +list_del_init (&rqstp->rq_idle); +spin_unlock_bh (&pool->sp_lock); +if (rqstp) {WRITE_ONCE (rqstp->rq_qtime,ktime_get ());wake_up_process (rqstp->rq_task);rcu_read_unlock ();
net/sunrpc/svc_xprt.c +11 -4
@@ -737,8 +737,9 @@ static void svc_rqst_wait_for_work(struct svc_rqst *rqstp)set_current_state (TASK_IDLE );smp_mb__before_atomic ();clear_bit (SP_CONGESTED , &pool->sp_flags); -clear_bit (RQ_BUSY , &rqstp->rq_flags); -smp_mb__after_atomic (); +spin_lock_bh (&pool->sp_lock); +list_add (&rqstp->rq_idle, &pool->sp_idle_threads); +spin_unlock_bh (&pool->sp_lock);/* Need to check should_sleep() again after * setting task state in case a wakeup happened
@@ -751,8 +752,14 @@ static void svc_rqst_wait_for_work(struct svc_rqst *rqstp)cond_resched (); } -set_bit (RQ_BUSY , &rqstp->rq_flags); -smp_mb__after_atomic (); +/* We *must* be removed from the list before we can continue. +* If we were woken, this is already done +*/ +if (!svc_thread_busy (rqstp)) { +spin_lock_bh (&pool->sp_lock); +list_del_init (&rqstp->rq_idle); +spin_unlock_bh (&pool->sp_lock); + } }else {cond_resched (); }
Signature · 1 line
2.40.1
PATCH 3/10 SUNRPC: discard SP_CONGESTED
In reply to NeilBrownWe can tell if a pool is congested by checking if the idle list is empty. We don't need a separate flag.
--- include/linux/sunrpc/svc.h | 1 - net/sunrpc/svc.c | 1 - net/sunrpc/svc_xprt.c | 4 +--- 3 files changed, 1 insertion(+), 5 deletions(-)
include/linux/sunrpc/svc.h +0 -1
@@ -50,7 +50,6 @@ struct svc_pool {/* bits for sp_flags */ enum {SP_TASK_PENDING ,/* still work to do even if no xprt is queued */ -SP_CONGESTED ,/* all threads are busy, none idle */ SP_NEED_VICTIM ,/* One thread needs to agree to exit */ SP_VICTIM_REMAINS ,/* One thread needs to actually exit */ };
net/sunrpc/svc.c +0 -1
@@ -719,7 +719,6 @@ void svc_pool_wake_idle_thread(struct svc_pool *pool) }rcu_read_unlock (); -set_bit (SP_CONGESTED , &pool->sp_flags); }EXPORT_SYMBOL_GPL (svc_pool_wake_idle_thread);
net/sunrpc/svc_xprt.c +1 -3
@@ -735,8 +735,6 @@ static void svc_rqst_wait_for_work(struct svc_rqst *rqstp)if (rqst_should_sleep (rqstp)) {set_current_state (TASK_IDLE ); -smp_mb__before_atomic (); -clear_bit (SP_CONGESTED , &pool->sp_flags);spin_lock_bh (&pool->sp_lock);list_add (&rqstp->rq_idle, &pool->sp_idle_threads);spin_unlock_bh (&pool->sp_lock);
@@ -874,7 +872,7 @@ void svc_recv(struct svc_rqst *rqstp)/* Normally we will wait up to 5 seconds for any required * cache information to be provided. */ -if (!test_bit (SP_CONGESTED , &pool->sp_flags)) +if (!list_empty (&pool->sp_idle_threads)) rqstp->rq_chandle.thread_wait =5 *HZ ;else rqstp->rq_chandle.thread_wait =1 *HZ ;
Signature · 1 line
2.40.1
PATCH 4/10 SUNRPC: change service idle list to be an llist
In reply to NeilBrownWith an llist we don't need to take a lock to add a thread to the list, though we still need a lock to remove it. That will go in the next patch.
Unlike double-linked lists, a thread cannot reliably remove itself from the list. Only the first thread can be removed, and that can change asynchronously. So some care is needed.
We already check if there is pending work to do, so we are unlikely to add ourselves to the idle list and then want to remove ourselves again.
If we DO find something needs to be done after adding ourselves to the list, we simply wake up the first thread on the list. If that was us, we successfully removed ourselves and can continue. If it was some other thread, they will do the work that needs to be done. We can safely sleep until woken.
We also remove the test on freezing() from rqst_should_sleep(). Instead we always try_to_freeze() before scheduling, which is needed as we now schedule() in a loop waiting to be removed from the idle queue.
--- include/linux/sunrpc/svc.h | 14 ++++++----- net/sunrpc/svc.c | 13 +++++----- net/sunrpc/svc_xprt.c | 50 +++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 37 deletions(-)
include/linux/sunrpc/svc.h +8 -6
@@ -37,7 +37,7 @@ struct svc_pool {struct list_head sp_sockets;/* pending sockets */ unsigned int sp_nrthreads;/* # of threads in pool */ struct list_head sp_all_threads;/* all server threads */ -struct list_head sp_idle_threads;/* idle server threads */ +struct llist_head sp_idle_threads;/* idle server threads */ /* statistics on pool operation */ struct percpu_counter sp_messages_arrived;
@@ -186,7 +186,7 @@ extern u32 svc_max_payload(const struct svc_rqst *rqstp);*/ struct svc_rqst {struct list_head rq_all;/* all threads list */ -struct list_head rq_idle;/* On the idle list */ +struct llist_node rq_idle;/* On the idle list */ struct rcu_head rq_rcu_head;/* for RCU deferred kfree */ struct svc_xprt * rq_xprt;/* transport ptr */
@@ -270,22 +270,24 @@ enum {* svc_thread_set_busy - mark a thread as busy * @rqstp: the thread which is now busy * -* If rq_idle is "empty", the thread must be busy. +* By convention a thread is busy if rq_idle.next points to rq_idle. +* This ensures it is not on the idle list. */ static inline void svc_thread_set_busy (struct svc_rqst *rqstp) { -INIT_LIST_HEAD (&rqstp->rq_idle); + rqstp->rq_idle.next = &rqstp->rq_idle; }/** * svc_thread_busy - check if a thread as busy * @rqstp: the thread which might be busy * -* If rq_idle is "empty", the thread must be busy. +* By convention a thread is busy if rq_idle.next points to rq_idle. +* This ensures it is not on the idle list. */ static inline bool svc_thread_busy (struct svc_rqst *rqstp) { -return list_empty (&rqstp->rq_idle); +return rqstp->rq_idle.next == &rqstp->rq_idle; }#define SVC_NET (rqst) (rqst->rq_xprt ? rqst->rq_xprt->xpt_net : rqst->rq_bc_net)
net/sunrpc/svc.c +7 -6
@@ -510,7 +510,7 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools, pool->sp_id = i;INIT_LIST_HEAD (&pool->sp_sockets);INIT_LIST_HEAD (&pool->sp_all_threads); -INIT_LIST_HEAD (&pool->sp_idle_threads); +init_llist_head (&pool->sp_idle_threads);spin_lock_init (&pool->sp_lock);percpu_counter_init (&pool->sp_messages_arrived,0 ,GFP_KERNEL );
@@ -701,15 +701,16 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)void svc_pool_wake_idle_thread (struct svc_pool *pool) {struct svc_rqst *rqstp; +struct llist_node *ln;rcu_read_lock ();spin_lock_bh (&pool->sp_lock); - rqstp =list_first_entry_or_null (&pool->sp_idle_threads, - struct svc_rqst, rq_idle); -if (rqstp) -list_del_init (&rqstp->rq_idle); + ln =llist_del_first (&pool->sp_idle_threads);spin_unlock_bh (&pool->sp_lock); -if (rqstp) { +if (ln) { + rqstp = llist_entry(ln, struct svc_rqst, rq_idle); +svc_thread_set_busy (rqstp); +WRITE_ONCE (rqstp->rq_qtime,ktime_get ());wake_up_process (rqstp->rq_task);rcu_read_unlock ();
net/sunrpc/svc_xprt.c +25 -25
73 lines not on this page · raw message
Signature · 1 line
2.40.1
Re: [PATCH 04/10] SUNRPC: change service idle list to be an llist
Quoted 152 lines · NeilBrown
With an llist we don't need to take a lock to add a thread to the list, though we still need a lock to remove it. That will go in the next patch.
Unlike double-linked lists, a thread cannot reliably remove itself from the list. Only the first thread can be removed, and that can change asynchronously. So some care is needed.
We already check if there is pending work to do, so we are unlikely to add ourselves to the idle list and then want to remove ourselves again.
If we DO find something needs to be done after adding ourselves to the list, we simply wake up the first thread on the list. If that was us, we successfully removed ourselves and can continue. If it was some other thread, they will do the work that needs to be done. We can safely sleep until woken.
We also remove the test on
freezing()fromrqst_should_sleep(). Instead we alwaystry_to_freeze()before scheduling, which is needed as we nowschedule()in a loop waiting to be removed from the idle queue.Signed-off-by NeilBrown <[email protected]>--- include/linux/sunrpc/svc.h | 14 ++++++----- net/sunrpc/svc.c | 13 +++++----- net/sunrpc/svc_xprt.c | 50 +++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 37 deletions(-)include/linux/sunrpc/svc.h +8
-6@@ -37,7 +37,7 @@ struct svc_pool {struct list_head sp_sockets;/* pending sockets */ unsigned int sp_nrthreads;/* # of threads in pool */ struct list_head sp_all_threads;/* all server threads */ -struct list_head sp_idle_threads;/* idle server threads */ +struct llist_head sp_idle_threads;/* idle server threads */ /* statistics on pool operation */ struct percpu_counter sp_messages_arrived;@@ -186,7 +186,7 @@ extern u32 svc_max_payload(const struct svc_rqst *rqstp);*/ struct svc_rqst {struct list_head rq_all;/* all threads list */ -struct list_head rq_idle;/* On the idle list */ +struct llist_node rq_idle;/* On the idle list */ struct rcu_head rq_rcu_head;/* for RCU deferred kfree */ struct svc_xprt * rq_xprt;/* transport ptr */ @@ -270,22 +270,24 @@ enum {* svc_thread_set_busy - mark a thread as busy * @rqstp: the thread which is now busy * -* If rq_idle is "empty", the thread must be busy. +* By convention a thread is busy if rq_idle.next points to rq_idle. +* This ensures it is not on the idle list. */ static inline void svc_thread_set_busy (struct svc_rqst *rqstp) { -INIT_LIST_HEAD (&rqstp->rq_idle); + rqstp->rq_idle.next = &rqstp->rq_idle; }/** * svc_thread_busy - check if a thread as busy * @rqstp: the thread which might be busy * -* If rq_idle is "empty", the thread must be busy. +* By convention a thread is busy if rq_idle.next points to rq_idle. +* This ensures it is not on the idle list. */ static inline bool svc_thread_busy (struct svc_rqst *rqstp) { -return list_empty (&rqstp->rq_idle); +return rqstp->rq_idle.next == &rqstp->rq_idle; }#define SVC_NET (rqst) (rqst->rq_xprt ? rqst->rq_xprt->xpt_net : rqst->rq_bc_net)net/sunrpc/svc.c +7
-6@@ -510,7 +510,7 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools, pool->sp_id = i;INIT_LIST_HEAD (&pool->sp_sockets);INIT_LIST_HEAD (&pool->sp_all_threads); -INIT_LIST_HEAD (&pool->sp_idle_threads); +init_llist_head (&pool->sp_idle_threads);spin_lock_init (&pool->sp_lock);percpu_counter_init (&pool->sp_messages_arrived,0 ,GFP_KERNEL );@@ -701,15 +701,16 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)void svc_pool_wake_idle_thread (struct svc_pool *pool) {struct svc_rqst *rqstp; +struct llist_node *ln;rcu_read_lock ();spin_lock_bh (&pool->sp_lock); - rqstp =list_first_entry_or_null (&pool->sp_idle_threads, - struct svc_rqst, rq_idle); -if (rqstp) -list_del_init (&rqstp->rq_idle); + ln =llist_del_first (&pool->sp_idle_threads);spin_unlock_bh (&pool->sp_lock); -if (rqstp) { +if (ln) { + rqstp = llist_entry(ln, struct svc_rqst, rq_idle); +svc_thread_set_busy (rqstp); +WRITE_ONCE (rqstp->rq_qtime,ktime_get ());wake_up_process (rqstp->rq_task);rcu_read_unlock ();net/sunrpc/svc_xprt.c +14
-8@@ -715,10 +715,6 @@ rqst_should_sleep(struct svc_rqst *rqstp)if (svc_thread_should_stop (rqstp))return false; -/* are we freezing? */ -if (freezing (current)) -return false; -#if defined(CONFIG_SUNRPC_BACKCHANNEL )if (svc_is_backchannel (rqstp)) {if (!list_empty (&rqstp->rq_server->sv_cb_list))@@ -735,29 +731,32 @@ static void svc_rqst_wait_for_work(struct svc_rqst *rqstp)if (rqst_should_sleep (rqstp)) {set_current_state (TASK_IDLE ); -spin_lock_bh (&pool->sp_lock); -list_add (&rqstp->rq_idle, &pool->sp_idle_threads); -spin_unlock_bh (&pool->sp_lock); +llist_add (&rqstp->rq_idle, &pool->sp_idle_threads); + +if (unlikely (!rqst_should_sleep (rqstp))) +/* maybe there were no idle threads when some work +* became ready and so nothing was woken. We've just +* become idle so someone can to the work - maybe us. +* But we cannot reliably remove ourselves from the +* idle list - we can only remove the first task which +* might be us, and might not. +* So remove and wake it, then schedule(). If it was +* us, we won't sleep. If it is some other thread, they +* will do the work. +*/ +svc_pool_wake_idle_thread (pool); -/* Need to check should_sleep() again after
net/sunrpc/svc_xprt.c static void svc_rqst_wait_for_work(struct svc_rqst *rqstp) · line 755
Quoted 10 lines
net/sunrpc/svc_xprt.c +6
-3-* setting task state in case a wakeup happened -* between testing and setting. +/* We mustn't continue while on the idle list, and we +* cannot remove outselves reliably. The only "work" +* we can do while on the idle list is to freeze. +* So loop until someone removes us */ -if (rqst_should_sleep (rqstp)) { +while (!svc_thread_busy (rqstp)) { +try_to_freeze ();
For testing, I've applied up to this patch. When nfsd is started I now get this splat:
do not call blocking ops when !TASK_RUNNING; state=402 set at [<000000001e3d6995>] svc_recv+0x40/0x252 [sunrpc]
WARNING: CPU: 3 PID: 1228 at kernel/sched/core.c:10112 __might_sleep+0x52/0x6a Modules linked in: 8021q garp stp mrp llc rfkill rpcrdma rdma_ucm ib_srpt ib_isert iscsi_target_mod snd_hda_codec_realtek target_core_mod intel> CPU: 3 PID: 1228 Comm: lockd Not tainted 6.5.0-rc6-00060-gd10a6b1ad2a1 #1 Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0b 06/12/2017 RIP: 0010:__might_sleep+0x52/0x6a Code: 00 00 74 28 80 3d 45 40 d5 01 00 75 1f 48 8b 90 f0 1a 00 00 48 c7 c7 b3 d6 49 9b c6 05 2e 40 d5 01 01 48 89 d1 e8 e6 a2 fc ff <0f> 0b 44 > RSP: 0018:ffffb3e3836abe68 EFLAGS: 00010286 RAX: 000000000000006f RBX: ffffffffc0c20599 RCX: 0000000000000027 RDX: 0000000000000027 RSI: 00000000ffffdfff RDI: 0000000000000001 RBP: ffffb3e3836abe78 R08: 0000000000000000 R09: ffffb3e380e70020 R10: 0000000000000000 R11: ffffa0ae94aa4c00 R12: 0000000000000035 R13: ffffa0ae94bfa040 R14: ffffa0ae9e85c010 R15: ffffa0ae94bfa040 FS: 0000000000000000(0000) GS:ffffa0bdbfd80000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f59c2611760 CR3: 000000069ec34006 CR4: 00000000003706e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: <TASK> ? show_regs+0x5d/0x64 ? __might_sleep+0x52/0x6a ? __warn+0xab/0x132 ? report_bug+0xd0/0x144 ? __might_sleep+0x52/0x6a ? handle_bug+0x45/0x74 ? exc_invalid_op+0x18/0x68 ? asm_exc_invalid_op+0x1b/0x20 ? __might_sleep+0x52/0x6a ? __might_sleep+0x52/0x6a try_to_freeze.isra.0+0x15/0x3d [sunrpc] svc_recv+0x97/0x252 [sunrpc] ? __pfx_lockd+0x10/0x10 [lockd] lockd+0x6b/0xda [lockd] kthread+0x10d/0x115 ? __pfx_kthread+0x10/0x10 ret_from_fork+0x2a/0x43 ? __pfx_kthread+0x10/0x10 ret_from_fork_asm+0x1b/0x30 </TASK>
Quoted 35 lines
net/sunrpc/svc_xprt.c +5
-14schedule (); - }else { -__set_current_state (TASK_RUNNING ); -cond_resched (); - } - -/* We *must* be removed from the list before we can continue. -* If we were woken, this is already done -*/ -if (!svc_thread_busy (rqstp)) { -spin_lock_bh (&pool->sp_lock); -list_del_init (&rqstp->rq_idle); -spin_unlock_bh (&pool->sp_lock); +set_current_state (TASK_IDLE ); } +__set_current_state (TASK_RUNNING ); }else {cond_resched (); }@@ -870,9 +869,10 @@ void svc_recv(struct svc_rqst *rqstp)struct svc_xprt *xprt = rqstp->rq_xprt;/* Normally we will wait up to 5 seconds for any required -* cache information to be provided. +* cache information to be provided. When there are no +* idle threads, we reduce the wait time. */ -if (!list_empty (&pool->sp_idle_threads)) +if (pool->sp_idle_threads.first) rqstp->rq_chandle.thread_wait =5 *HZ ;else rqstp->rq_chandle.thread_wait =1 *HZ ;Signature · 1 line
2.40.1
Signature · 1 line
Chuck Lever
Re: [PATCH 04/10] SUNRPC: change service idle list to be an llist
Quoted 167 lines · Chuck Lever
Quoted 152 lines · NeilBrown
With an llist we don't need to take a lock to add a thread to the list, though we still need a lock to remove it. That will go in the next patch.
Unlike double-linked lists, a thread cannot reliably remove itself from the list. Only the first thread can be removed, and that can change asynchronously. So some care is needed.
We already check if there is pending work to do, so we are unlikely to add ourselves to the idle list and then want to remove ourselves again.
If we DO find something needs to be done after adding ourselves to the list, we simply wake up the first thread on the list. If that was us, we successfully removed ourselves and can continue. If it was some other thread, they will do the work that needs to be done. We can safely sleep until woken.
We also remove the test on
freezing()fromrqst_should_sleep(). Instead we alwaystry_to_freeze()before scheduling, which is needed as we nowschedule()in a loop waiting to be removed from the idle queue.Signed-off-by NeilBrown <[email protected]>--- include/linux/sunrpc/svc.h | 14 ++++++----- net/sunrpc/svc.c | 13 +++++----- net/sunrpc/svc_xprt.c | 50 +++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 37 deletions(-)include/linux/sunrpc/svc.h +8
-6@@ -37,7 +37,7 @@ struct svc_pool {struct list_head sp_sockets;/* pending sockets */ unsigned int sp_nrthreads;/* # of threads in pool */ struct list_head sp_all_threads;/* all server threads */ -struct list_head sp_idle_threads;/* idle server threads */ +struct llist_head sp_idle_threads;/* idle server threads */ /* statistics on pool operation */ struct percpu_counter sp_messages_arrived;@@ -186,7 +186,7 @@ extern u32 svc_max_payload(const struct svc_rqst *rqstp);*/ struct svc_rqst {struct list_head rq_all;/* all threads list */ -struct list_head rq_idle;/* On the idle list */ +struct llist_node rq_idle;/* On the idle list */ struct rcu_head rq_rcu_head;/* for RCU deferred kfree */ struct svc_xprt * rq_xprt;/* transport ptr */ @@ -270,22 +270,24 @@ enum {* svc_thread_set_busy - mark a thread as busy * @rqstp: the thread which is now busy * -* If rq_idle is "empty", the thread must be busy. +* By convention a thread is busy if rq_idle.next points to rq_idle. +* This ensures it is not on the idle list. */ static inline void svc_thread_set_busy (struct svc_rqst *rqstp) { -INIT_LIST_HEAD (&rqstp->rq_idle); + rqstp->rq_idle.next = &rqstp->rq_idle; }/** * svc_thread_busy - check if a thread as busy * @rqstp: the thread which might be busy * -* If rq_idle is "empty", the thread must be busy. +* By convention a thread is busy if rq_idle.next points to rq_idle. +* This ensures it is not on the idle list. */ static inline bool svc_thread_busy (struct svc_rqst *rqstp) { -return list_empty (&rqstp->rq_idle); +return rqstp->rq_idle.next == &rqstp->rq_idle; }#define SVC_NET (rqst) (rqst->rq_xprt ? rqst->rq_xprt->xpt_net : rqst->rq_bc_net)net/sunrpc/svc.c +7
-6@@ -510,7 +510,7 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools, pool->sp_id = i;INIT_LIST_HEAD (&pool->sp_sockets);INIT_LIST_HEAD (&pool->sp_all_threads); -INIT_LIST_HEAD (&pool->sp_idle_threads); +init_llist_head (&pool->sp_idle_threads);spin_lock_init (&pool->sp_lock);percpu_counter_init (&pool->sp_messages_arrived,0 ,GFP_KERNEL );@@ -701,15 +701,16 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)void svc_pool_wake_idle_thread (struct svc_pool *pool) {struct svc_rqst *rqstp; +struct llist_node *ln;rcu_read_lock ();spin_lock_bh (&pool->sp_lock); - rqstp =list_first_entry_or_null (&pool->sp_idle_threads, - struct svc_rqst, rq_idle); -if (rqstp) -list_del_init (&rqstp->rq_idle); + ln =llist_del_first (&pool->sp_idle_threads);spin_unlock_bh (&pool->sp_lock); -if (rqstp) { +if (ln) { + rqstp = llist_entry(ln, struct svc_rqst, rq_idle); +svc_thread_set_busy (rqstp); +WRITE_ONCE (rqstp->rq_qtime,ktime_get ());wake_up_process (rqstp->rq_task);rcu_read_unlock ();net/sunrpc/svc_xprt.c +14
-8@@ -715,10 +715,6 @@ rqst_should_sleep(struct svc_rqst *rqstp)if (svc_thread_should_stop (rqstp))return false; -/* are we freezing? */ -if (freezing (current)) -return false; -#if defined(CONFIG_SUNRPC_BACKCHANNEL )if (svc_is_backchannel (rqstp)) {if (!list_empty (&rqstp->rq_server->sv_cb_list))@@ -735,29 +731,32 @@ static void svc_rqst_wait_for_work(struct svc_rqst *rqstp)if (rqst_should_sleep (rqstp)) {set_current_state (TASK_IDLE ); -spin_lock_bh (&pool->sp_lock); -list_add (&rqstp->rq_idle, &pool->sp_idle_threads); -spin_unlock_bh (&pool->sp_lock); +llist_add (&rqstp->rq_idle, &pool->sp_idle_threads); + +if (unlikely (!rqst_should_sleep (rqstp))) +/* maybe there were no idle threads when some work +* became ready and so nothing was woken. We've just +* become idle so someone can to the work - maybe us. +* But we cannot reliably remove ourselves from the +* idle list - we can only remove the first task which +* might be us, and might not. +* So remove and wake it, then schedule(). If it was +* us, we won't sleep. If it is some other thread, they +* will do the work. +*/ +svc_pool_wake_idle_thread (pool); -/* Need to check should_sleep() again after net/sunrpc/svc_xprt.c static void svc_rqst_wait_for_work(struct svc_rqst *rqstp) · line 755
Quoted 10 lines
net/sunrpc/svc_xprt.c +6
-3-* setting task state in case a wakeup happened -* between testing and setting. +/* We mustn't continue while on the idle list, and we +* cannot remove outselves reliably. The only "work" +* we can do while on the idle list is to freeze. +* So loop until someone removes us */ -if (rqst_should_sleep (rqstp)) { +while (!svc_thread_busy (rqstp)) { +try_to_freeze ();For testing, I've applied up to this patch. When nfsd is started I now get this splat:
Quoted 1 line
do not call blocking ops when !TASK_RUNNING; state=402 set at [<
000000001e3d6995>]svc_recv+0x40/0x252[sunrpc]
Thanks. I didn't have the right CONFIG options to trigger that warning. I do now.
I'm not surprised I got something wrong with freezing. I did some research and now I see the part of freezing that I was missing. This incremental patch
net/sunrpc/svc_xprt.c +2 -3
@@ -730,7 +730,7 @@ static void svc_rqst_wait_for_work(struct svc_rqst *rqstp)struct svc_pool *pool = rqstp->rq_pool;if (rqst_should_sleep (rqstp)) { -set_current_state (TASK_IDLE ); +set_current_state (TASK_IDLE |TASK_FREEZABLE );llist_add (&rqstp->rq_idle, &pool->sp_idle_threads);if (unlikely (!rqst_should_sleep (rqstp)))
@@ -752,9 +752,8 @@ static void svc_rqst_wait_for_work(struct svc_rqst *rqstp)* So loop until someone removes us */ while (!svc_thread_busy (rqstp)) { -try_to_freeze ();schedule (); -set_current_state (TASK_IDLE ); +set_current_state (TASK_IDLE |TASK_FREEZABLE ); }__set_current_state (TASK_RUNNING ); }else {
fixes that issue. "TASK_FREEZABLE" was the missing bit. I'll need to look back at the other patches and probably introduce this earlier.
Thanks,
NeilBrown
Quoted 80 lines
WARNING: CPU: 3 PID: 1228 at kernel/sched/core.c:10112 __might_sleep+0x52/0x6a Modules linked in: 8021q garp stp mrp llc rfkill rpcrdma rdma_ucm ib_srpt ib_isert iscsi_target_mod snd_hda_codec_realtek target_core_mod intel> CPU: 3 PID: 1228 Comm: lockd Not tainted 6.5.0-rc6-00060-gd10a6b1ad2a1 #1 Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0b 06/12/2017 RIP: 0010:__might_sleep+0x52/0x6a Code: 00 00 74 28 80 3d 45 40 d5 01 00 75 1f 48 8b 90 f0 1a 00 00 48 c7 c7 b3 d6 49 9b c6 05 2e 40 d5 01 01 48 89 d1 e8 e6 a2 fc ff <0f> 0b 44 > RSP: 0018:ffffb3e3836abe68 EFLAGS: 00010286 RAX: 000000000000006f RBX: ffffffffc0c20599 RCX: 0000000000000027 RDX: 0000000000000027 RSI: 00000000ffffdfff RDI: 0000000000000001 RBP: ffffb3e3836abe78 R08: 0000000000000000 R09: ffffb3e380e70020 R10: 0000000000000000 R11: ffffa0ae94aa4c00 R12: 0000000000000035 R13: ffffa0ae94bfa040 R14: ffffa0ae9e85c010 R15: ffffa0ae94bfa040 FS: 0000000000000000(0000) GS:ffffa0bdbfd80000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f59c2611760 CR3: 000000069ec34006 CR4: 00000000003706e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: <TASK> ? show_regs+0x5d/0x64 ? __might_sleep+0x52/0x6a ? __warn+0xab/0x132 ? report_bug+0xd0/0x144 ? __might_sleep+0x52/0x6a ? handle_bug+0x45/0x74 ? exc_invalid_op+0x18/0x68 ? asm_exc_invalid_op+0x1b/0x20 ? __might_sleep+0x52/0x6a ? __might_sleep+0x52/0x6a try_to_freeze.isra.0+0x15/0x3d [sunrpc] svc_recv+0x97/0x252 [sunrpc] ? __pfx_lockd+0x10/0x10 [lockd] lockd+0x6b/0xda [lockd] kthread+0x10d/0x115 ? __pfx_kthread+0x10/0x10 ret_from_fork+0x2a/0x43 ? __pfx_kthread+0x10/0x10 ret_from_fork_asm+0x1b/0x30 </TASK>Quoted 35 lines
net/sunrpc/svc_xprt.c +5
-14schedule (); - }else { -__set_current_state (TASK_RUNNING ); -cond_resched (); - } - -/* We *must* be removed from the list before we can continue. -* If we were woken, this is already done -*/ -if (!svc_thread_busy (rqstp)) { -spin_lock_bh (&pool->sp_lock); -list_del_init (&rqstp->rq_idle); -spin_unlock_bh (&pool->sp_lock); +set_current_state (TASK_IDLE ); } +__set_current_state (TASK_RUNNING ); }else {cond_resched (); }@@ -870,9 +869,10 @@ void svc_recv(struct svc_rqst *rqstp)struct svc_xprt *xprt = rqstp->rq_xprt;/* Normally we will wait up to 5 seconds for any required -* cache information to be provided. +* cache information to be provided. When there are no +* idle threads, we reduce the wait time. */ -if (!list_empty (&pool->sp_idle_threads)) +if (pool->sp_idle_threads.first) rqstp->rq_chandle.thread_wait =5 *HZ ;else rqstp->rq_chandle.thread_wait =1 *HZ ;Signature · 1 line
2.40.1Signature · 1 line
Chuck Lever
Re: [PATCH 04/10] SUNRPC: change service idle list to be an llist
Quoted 202 lines · NeilBrown
Quoted 167 lines · Chuck Lever
Quoted 152 lines · NeilBrown
With an llist we don't need to take a lock to add a thread to the list, though we still need a lock to remove it. That will go in the next patch.
Unlike double-linked lists, a thread cannot reliably remove itself from the list. Only the first thread can be removed, and that can change asynchronously. So some care is needed.
We already check if there is pending work to do, so we are unlikely to add ourselves to the idle list and then want to remove ourselves again.
If we DO find something needs to be done after adding ourselves to the list, we simply wake up the first thread on the list. If that was us, we successfully removed ourselves and can continue. If it was some other thread, they will do the work that needs to be done. We can safely sleep until woken.
We also remove the test on
freezing()fromrqst_should_sleep(). Instead we alwaystry_to_freeze()before scheduling, which is needed as we nowschedule()in a loop waiting to be removed from the idle queue.Signed-off-by NeilBrown <[email protected]>--- include/linux/sunrpc/svc.h | 14 ++++++----- net/sunrpc/svc.c | 13 +++++----- net/sunrpc/svc_xprt.c | 50 +++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 37 deletions(-)include/linux/sunrpc/svc.h +8
-6@@ -37,7 +37,7 @@ struct svc_pool {struct list_head sp_sockets;/* pending sockets */ unsigned int sp_nrthreads;/* # of threads in pool */ struct list_head sp_all_threads;/* all server threads */ -struct list_head sp_idle_threads;/* idle server threads */ +struct llist_head sp_idle_threads;/* idle server threads */ /* statistics on pool operation */ struct percpu_counter sp_messages_arrived;@@ -186,7 +186,7 @@ extern u32 svc_max_payload(const struct svc_rqst *rqstp);*/ struct svc_rqst {struct list_head rq_all;/* all threads list */ -struct list_head rq_idle;/* On the idle list */ +struct llist_node rq_idle;/* On the idle list */ struct rcu_head rq_rcu_head;/* for RCU deferred kfree */ struct svc_xprt * rq_xprt;/* transport ptr */ @@ -270,22 +270,24 @@ enum {* svc_thread_set_busy - mark a thread as busy * @rqstp: the thread which is now busy * -* If rq_idle is "empty", the thread must be busy. +* By convention a thread is busy if rq_idle.next points to rq_idle. +* This ensures it is not on the idle list. */ static inline void svc_thread_set_busy (struct svc_rqst *rqstp) { -INIT_LIST_HEAD (&rqstp->rq_idle); + rqstp->rq_idle.next = &rqstp->rq_idle; }/** * svc_thread_busy - check if a thread as busy * @rqstp: the thread which might be busy * -* If rq_idle is "empty", the thread must be busy. +* By convention a thread is busy if rq_idle.next points to rq_idle. +* This ensures it is not on the idle list. */ static inline bool svc_thread_busy (struct svc_rqst *rqstp) { -return list_empty (&rqstp->rq_idle); +return rqstp->rq_idle.next == &rqstp->rq_idle; }#define SVC_NET (rqst) (rqst->rq_xprt ? rqst->rq_xprt->xpt_net : rqst->rq_bc_net)net/sunrpc/svc.c +7
-6@@ -510,7 +510,7 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools, pool->sp_id = i;INIT_LIST_HEAD (&pool->sp_sockets);INIT_LIST_HEAD (&pool->sp_all_threads); -INIT_LIST_HEAD (&pool->sp_idle_threads); +init_llist_head (&pool->sp_idle_threads);spin_lock_init (&pool->sp_lock);percpu_counter_init (&pool->sp_messages_arrived,0 ,GFP_KERNEL );@@ -701,15 +701,16 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)void svc_pool_wake_idle_thread (struct svc_pool *pool) {struct svc_rqst *rqstp; +struct llist_node *ln;rcu_read_lock ();spin_lock_bh (&pool->sp_lock); - rqstp =list_first_entry_or_null (&pool->sp_idle_threads, - struct svc_rqst, rq_idle); -if (rqstp) -list_del_init (&rqstp->rq_idle); + ln =llist_del_first (&pool->sp_idle_threads);spin_unlock_bh (&pool->sp_lock); -if (rqstp) { +if (ln) { + rqstp = llist_entry(ln, struct svc_rqst, rq_idle); +svc_thread_set_busy (rqstp); +WRITE_ONCE (rqstp->rq_qtime,ktime_get ());wake_up_process (rqstp->rq_task);rcu_read_unlock ();net/sunrpc/svc_xprt.c +14
-8@@ -715,10 +715,6 @@ rqst_should_sleep(struct svc_rqst *rqstp)if (svc_thread_should_stop (rqstp))return false; -/* are we freezing? */ -if (freezing (current)) -return false; -#if defined(CONFIG_SUNRPC_BACKCHANNEL )if (svc_is_backchannel (rqstp)) {if (!list_empty (&rqstp->rq_server->sv_cb_list))@@ -735,29 +731,32 @@ static void svc_rqst_wait_for_work(struct svc_rqst *rqstp)if (rqst_should_sleep (rqstp)) {set_current_state (TASK_IDLE ); -spin_lock_bh (&pool->sp_lock); -list_add (&rqstp->rq_idle, &pool->sp_idle_threads); -spin_unlock_bh (&pool->sp_lock); +llist_add (&rqstp->rq_idle, &pool->sp_idle_threads); + +if (unlikely (!rqst_should_sleep (rqstp))) +/* maybe there were no idle threads when some work +* became ready and so nothing was woken. We've just +* become idle so someone can to the work - maybe us. +* But we cannot reliably remove ourselves from the +* idle list - we can only remove the first task which +* might be us, and might not. +* So remove and wake it, then schedule(). If it was +* us, we won't sleep. If it is some other thread, they +* will do the work. +*/ +svc_pool_wake_idle_thread (pool); -/* Need to check should_sleep() again after net/sunrpc/svc_xprt.c static void svc_rqst_wait_for_work(struct svc_rqst *rqstp) · line 755
Quoted 10 lines
net/sunrpc/svc_xprt.c +6
-3-* setting task state in case a wakeup happened -* between testing and setting. +/* We mustn't continue while on the idle list, and we +* cannot remove outselves reliably. The only "work" +* we can do while on the idle list is to freeze. +* So loop until someone removes us */ -if (rqst_should_sleep (rqstp)) { +while (!svc_thread_busy (rqstp)) { +try_to_freeze ();For testing, I've applied up to this patch. When nfsd is started I now get this splat:
Quoted 1 line
do not call blocking ops when !TASK_RUNNING; state=402 set at [<
000000001e3d6995>]svc_recv+0x40/0x252[sunrpc]Thanks. I didn't have the right CONFIG options to trigger that warning. I do now.
I'm not surprised I got something wrong with freezing. I did some research and now I see the part of freezing that I was missing. This incremental patchnet/sunrpc/svc_xprt.c +2
-3@@ -730,7 +730,7 @@ static void svc_rqst_wait_for_work(struct svc_rqst *rqstp)struct svc_pool *pool = rqstp->rq_pool;if (rqst_should_sleep (rqstp)) { -set_current_state (TASK_IDLE ); +set_current_state (TASK_IDLE |TASK_FREEZABLE );llist_add (&rqstp->rq_idle, &pool->sp_idle_threads);if (unlikely (!rqst_should_sleep (rqstp)))@@ -752,9 +752,8 @@ static void svc_rqst_wait_for_work(struct svc_rqst *rqstp)* So loop until someone removes us */ while (!svc_thread_busy (rqstp)) { -try_to_freeze ();schedule (); -set_current_state (TASK_IDLE ); +set_current_state (TASK_IDLE |TASK_FREEZABLE ); }__set_current_state (TASK_RUNNING ); }else {
Quoted 1 line
fixes that issue.
Confirmed.
Quoted 2 lines
"
TASK_FREEZABLE" was the missing bit. I'll need to
look back at the other patches and probably introduce this earlier.
Also, it makes subsequent patches fail to apply. Can you send an updated series?
Quoted 81 lines
Quoted 80 lines
WARNING: CPU: 3 PID: 1228 at kernel/sched/core.c:10112 __might_sleep+0x52/0x6a Modules linked in: 8021q garp stp mrp llc rfkill rpcrdma rdma_ucm ib_srpt ib_isert iscsi_target_mod snd_hda_codec_realtek target_core_mod intel> CPU: 3 PID: 1228 Comm: lockd Not tainted 6.5.0-rc6-00060-gd10a6b1ad2a1 #1 Hardware name: Supermicro X10SRA-F/X10SRA-F, BIOS 2.0b 06/12/2017 RIP: 0010:__might_sleep+0x52/0x6a Code: 00 00 74 28 80 3d 45 40 d5 01 00 75 1f 48 8b 90 f0 1a 00 00 48 c7 c7 b3 d6 49 9b c6 05 2e 40 d5 01 01 48 89 d1 e8 e6 a2 fc ff <0f> 0b 44 > RSP: 0018:ffffb3e3836abe68 EFLAGS: 00010286 RAX: 000000000000006f RBX: ffffffffc0c20599 RCX: 0000000000000027 RDX: 0000000000000027 RSI: 00000000ffffdfff RDI: 0000000000000001 RBP: ffffb3e3836abe78 R08: 0000000000000000 R09: ffffb3e380e70020 R10: 0000000000000000 R11: ffffa0ae94aa4c00 R12: 0000000000000035 R13: ffffa0ae94bfa040 R14: ffffa0ae9e85c010 R15: ffffa0ae94bfa040 FS: 0000000000000000(0000) GS:ffffa0bdbfd80000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f59c2611760 CR3: 000000069ec34006 CR4: 00000000003706e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: <TASK> ? show_regs+0x5d/0x64 ? __might_sleep+0x52/0x6a ? __warn+0xab/0x132 ? report_bug+0xd0/0x144 ? __might_sleep+0x52/0x6a ? handle_bug+0x45/0x74 ? exc_invalid_op+0x18/0x68 ? asm_exc_invalid_op+0x1b/0x20 ? __might_sleep+0x52/0x6a ? __might_sleep+0x52/0x6a try_to_freeze.isra.0+0x15/0x3d [sunrpc] svc_recv+0x97/0x252 [sunrpc] ? __pfx_lockd+0x10/0x10 [lockd] lockd+0x6b/0xda [lockd] kthread+0x10d/0x115 ? __pfx_kthread+0x10/0x10 ret_from_fork+0x2a/0x43 ? __pfx_kthread+0x10/0x10 ret_from_fork_asm+0x1b/0x30 </TASK>Quoted 35 lines
net/sunrpc/svc_xprt.c +5
-14schedule (); - }else { -__set_current_state (TASK_RUNNING ); -cond_resched (); - } - -/* We *must* be removed from the list before we can continue. -* If we were woken, this is already done -*/ -if (!svc_thread_busy (rqstp)) { -spin_lock_bh (&pool->sp_lock); -list_del_init (&rqstp->rq_idle); -spin_unlock_bh (&pool->sp_lock); +set_current_state (TASK_IDLE ); } +__set_current_state (TASK_RUNNING ); }else {cond_resched (); }@@ -870,9 +869,10 @@ void svc_recv(struct svc_rqst *rqstp)struct svc_xprt *xprt = rqstp->rq_xprt;/* Normally we will wait up to 5 seconds for any required -* cache information to be provided. +* cache information to be provided. When there are no +* idle threads, we reduce the wait time. */ -if (!list_empty (&pool->sp_idle_threads)) +if (pool->sp_idle_threads.first) rqstp->rq_chandle.thread_wait =5 *HZ ;else rqstp->rq_chandle.thread_wait =1 *HZ ;Signature · 1 line
2.40.1Signature · 1 line
Chuck Lever
Signature · 1 line
Chuck Lever
PATCH 5/10 SUNRPC: only have one thread waking up at a time
In reply to NeilBrownCurrently if several items of work become available in quick succession, that number of threads (if available) will be woken. By the time some of them wake up another thread that was already cache-warm might have come along and completed the work. Anecdotal evidence suggests as many as 15% of wakes find nothing to do once they get to the point of looking.
This patch changes svc_pool_wake_idle_thread() to wake the first thread on the queue but NOT remove it. Subsequent calls will wake the same thread. Once that thread starts it will dequeue itself and after dequeueing some work to do, it will wake the next thread if there is more work ready. This results in a more orderly increase in the number of busy threads.
As a bonus, this allows us to reduce locking around the idle queue. svc_pool_wake_idle_thread() no longer needs to take a lock (beyond rcu_read_lock()) as it doesn't manipulate the queue, it just looks at the first item.
The thread itself can avoid locking by using the new llist_del_first_this() interface. This will safely remove the thread itself if it is the head. If it isn't the head, it will do nothing. If multiple threads call this concurrently only one will succeed. The others will do nothing, so no corruption can result.
If a thread wakes up and finds that it cannot dequeue itself that means either
- that it wasn't woken because it was the head of the queue. Maybe the freezer woke it. In that case it can go back to sleep (after trying to freeze of course).
- some other thread found there was nothing to do very recently, and placed itself on the head of the queue in front of this thread. It must check again after placing itself there, so it can be deemed to be responsible for any pending work, and this thread can go back to sleep until woken.
No code ever tests for busy threads any more. Only each thread itself cares if it is busy. So functions to set and test the busy status are no longer needed.
--- include/linux/llist.h | 2 ++ include/linux/sunrpc/svc.h | 24 ------------------- lib/llist.c | 27 +++++++++++++++++++++ net/sunrpc/svc.c | 15 +++++------- net/sunrpc/svc_xprt.c | 48 +++++++++++++++++++++++--------------- 5 files changed, 64 insertions(+), 52 deletions(-)
include/linux/llist.h +2 -0
@@ -248,6 +248,8 @@ static inline struct llist_node *__llist_del_all(struct llist_head *head) }extern struct llist_node *llist_del_first (struct llist_head *head); +extern struct llist_node *llist_del_first_this (struct llist_head *head, +struct llist_node *this);struct llist_node *llist_reverse_order (struct llist_node *head);
include/linux/sunrpc/svc.h +0 -24
@@ -266,30 +266,6 @@ enum { RQ_DATA, /* request has data */ }; -/** -* svc_thread_set_busy - mark a thread as busy -* @rqstp: the thread which is now busy -* -* By convention a thread is busy if rq_idle.next points to rq_idle. -* This ensures it is not on the idle list. -*/ -static inline void svc_thread_set_busy (struct svc_rqst *rqstp) -{ - rqstp->rq_idle.next = &rqstp->rq_idle; -} - -/** -* svc_thread_busy - check if a thread as busy -* @rqstp: the thread which might be busy -* -* By convention a thread is busy if rq_idle.next points to rq_idle. -* This ensures it is not on the idle list. -*/ -static inline bool svc_thread_busy (struct svc_rqst *rqstp) -{ -return rqstp->rq_idle.next == &rqstp->rq_idle; -} -#define SVC_NET (rqst) (rqst->rq_xprt ? rqst->rq_xprt->xpt_net : rqst->rq_bc_net)/*
lib/llist.c +27 -0
@@ -65,6 +65,33 @@ struct llist_node *llist_del_first(struct llist_head *head) }EXPORT_SYMBOL_GPL (llist_del_first); +/** +* llist_del_first_this - delete given entry of lock-less list if it is first +* @head: the head for your lock-less list +* @this: a list entry. +* +* If head of the list is given entry, delete and return it, else +* return %NULL. +* +* Providing the caller has exclusive access to @this, multiple callers can +* safely call this concurrently with multiple llist_add() callers. +*/ +struct llist_node *llist_del_first_this (struct llist_head *head, +struct llist_node *this) +{ +struct llist_node *entry, *next; + + entry =smp_load_acquire (&head->first); +do { +if (entry != this) +return NULL ; + next =READ_ONCE (entry->next); + }while (!try_cmpxchg (&head->first, &entry, next)); + +return entry; +} +EXPORT_SYMBOL_GPL (llist_del_first_this); +/** * llist_reverse_order - reverse order of a llist chain * @head: first item of the list to be reversed
net/sunrpc/svc.c +6 -9
@@ -642,7 +642,6 @@ svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node)folio_batch_init (&rqstp->rq_fbatch); -svc_thread_set_busy (rqstp); rqstp->rq_server = serv; rqstp->rq_pool = pool;
@@ -704,18 +703,16 @@ void svc_pool_wake_idle_thread(struct svc_pool *pool)struct llist_node *ln;rcu_read_lock (); -spin_lock_bh (&pool->sp_lock); - ln =llist_del_first (&pool->sp_idle_threads); -spin_unlock_bh (&pool->sp_lock); + ln =READ_ONCE (pool->sp_idle_threads.first);if (ln) { rqstp = llist_entry(ln, struct svc_rqst, rq_idle); -svc_thread_set_busy (rqstp); -WRITE_ONCE (rqstp->rq_qtime,ktime_get ()); -wake_up_process (rqstp->rq_task); +if (!task_is_running (rqstp->rq_task)) { +wake_up_process (rqstp->rq_task); +trace_svc_wake_up (rqstp->rq_task->pid); +percpu_counter_inc (&pool->sp_threads_woken); + }rcu_read_unlock (); -percpu_counter_inc (&pool->sp_threads_woken); -trace_svc_wake_up (rqstp->rq_task->pid);return ; }rcu_read_unlock ();
net/sunrpc/svc_xprt.c +29 -19
80 lines not on this page · raw message
Signature · 1 line
2.40.1
PATCH 6/10 SUNRPC/svc: add light-weight queuing mechanism.
In reply to NeilBrownlwq is a FIFO single-linked queue that only requires a spinlock for dequeueing, which happens in process context. Enqueueing is atomic with no spinlock and can happen in any context.
Include a unit test for basic functionality - runs a boot/module-load time. Does not use kunit framework.
--- include/linux/sunrpc/svc_lwq.h | 79 +++++++++++++++++++ net/sunrpc/Kconfig | 6 ++ net/sunrpc/Makefile | 2 +- net/sunrpc/svc_lwq.c | 135 +++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 include/linux/sunrpc/svc_lwq.h create mode 100644 net/sunrpc/svc_lwq.c
include/linux/sunrpc/svc_lwq.h new file +79 -0
@@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef SUNRPC_SVC_LWQ_H +#define SUNRPC_SVC_LWQ_H + +/* +* light-weight single linked queue +* +* Entries can be enqueued from any context with no locking. +* Entries can be dequeued from process context with integrated locking. +* +*/ +#include <linux/container_of.h> +#include <linux/spinlock.h> +#include <linux/llist.h> + +struct lwq_node { +struct llist_node node; +}; + +struct lwq { +spinlock_t lock; +struct llist_node *ready;/* entries to be dequeued */ +struct llist_head new;/* entries being enqueued */ +}; + +static inline void lwq_init (struct lwq *q) +{ +spin_lock_init (&q->lock); + q->ready =NULL ; +init_llist_head (&q->new); +} + +static inline bool lwq_empty (struct lwq *q) +{ +return READ_ONCE (q->ready) ==NULL &&llist_empty (&q->new); +} + +struct llist_node *__lwq_dequeue (struct lwq *q); +#define lwq_dequeue (_q, _type, _member) \ + ({ struct llist_node *_n = __lwq_dequeue(_q); \ + _n ? container_of(_n, _type, _member.node) : NULL; }) + +struct llist_node *lwq_dequeue_all (struct lwq *q); + +/** +* lwq_for_each_safe: iterate over detached queue allowing deletion +* @_n: iterator variable +* @_t1: temporary struct llist_node ** +* @_t2: temporary struct llist_node * +* @_l: address of llist_node pointer from lwq_dequeue_all() +* @_member: member in _n where lwq_node is found. +* +* Iterate over members in a dequeued list. If the iterator variable +* is set to NULL, the iterator removes that entry from the queue. +*/ +#define lwq_for_each_safe (_n, _t1, _t2, _l, _member) \ + for (_t1 = (_l); \ + *(_t1) ? (_n = container_of(*(_t1), typeof(*(_n)), _member.node),\ + _t2 = ((*_t1)->next), \ + true) \ + : false; \ + (_n) ? (_t1 = &(_n)->_member.node.next, 0) \ + : ((*(_t1) = (_t2)), 0)) + +static inline bool lwq_enqueue (struct lwq_node *n,struct lwq *q) +{ +return llist_add (&n->node, &q->new) &&READ_ONCE (q->ready) ==NULL ; +} + +static inline bool lwq_enqueue_batch (struct llist_node *n,struct lwq *q) +{ +struct llist_node *e = n; + +return llist_add_batch (llist_reverse_order (n), e, &q->new) && +READ_ONCE (q->ready) ==NULL ; +} + +#endif /* SUNRPC_SVC_LWQ_H */
net/sunrpc/Kconfig +6 -0
@@ -115,3 +115,9 @@ config SUNRPC_XPRT_RDMA If unsure, or you know there is no RDMA capability on your hardware platform, say N. + +config SUNRPC_LWQ_TEST + bool "RPC: enable boot-time test for lwq queuing" + depends on SUNRPC + help + Enable boot-time test of lwq functionality.
net/sunrpc/Makefile +1 -1
@@ -10,7 +10,7 @@ obj-$(CONFIG_SUNRPC_XPRT_RDMA) += xprtrdma/ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \ auth.o auth_null.o auth_tls.o auth_unix.o \ - svc.o svcsock.o svcauth.o svcauth_unix.o \ + svc.o svc_lwq.o svcsock.o svcauth.o svcauth_unix.o \ addr.o rpcb_clnt.o timer.o xdr.o \ sunrpc_syms.o cache.o rpc_pipe.o sysfs.o \ svc_xprt.o \
net/sunrpc/svc_lwq.c new file +135 -0
135 lines not on this page · raw message
Signature · 1 line
2.40.1
Re: [PATCH 06/10] SUNRPC/svc: add light-weight queuing mechanism.
Quoted 10 lines · NeilBrown
lwq is a FIFO single-linked queue that only requires a spinlock for dequeueing, which happens in process context. Enqueueing is atomic with no spinlock and can happen in any context.
Include a unit test for basic functionality - runs a boot/module-load time. Does not use kunit framework.
Signed-off-by NeilBrown <[email protected]>--- include/linux/sunrpc/svc_lwq.h | 79 +++++++++++++++++++
I'm wondering what your longer-term intentions are for this new mechanism. If it is only useful for SunRPC, then perhaps this header belongs under net/sunrpc instead.
Quoted 263 lines
net/sunrpc/Kconfig | 6 ++ net/sunrpc/Makefile | 2 +- net/sunrpc/svc_lwq.c | 135 +++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 include/linux/sunrpc/svc_lwq.h create mode 100644 net/sunrpc/svc_lwq.cinclude/linux/sunrpc/svc_lwq.h new file +79
-0@@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef SUNRPC_SVC_LWQ_H +#define SUNRPC_SVC_LWQ_H + +/* +* light-weight single linked queue +* +* Entries can be enqueued from any context with no locking. +* Entries can be dequeued from process context with integrated locking. +* +*/ +#include <linux/container_of.h> +#include <linux/spinlock.h> +#include <linux/llist.h> + +struct lwq_node { +struct llist_node node; +}; + +struct lwq { +spinlock_t lock; +struct llist_node *ready;/* entries to be dequeued */ +struct llist_head new;/* entries being enqueued */ +}; + +static inline void lwq_init (struct lwq *q) +{ +spin_lock_init (&q->lock); + q->ready =NULL ; +init_llist_head (&q->new); +} + +static inline bool lwq_empty (struct lwq *q) +{ +return READ_ONCE (q->ready) ==NULL &&llist_empty (&q->new); +} + +struct llist_node *__lwq_dequeue (struct lwq *q); +#define lwq_dequeue (_q, _type, _member) \ + ({ struct llist_node *_n = __lwq_dequeue(_q); \ + _n ? container_of(_n, _type, _member.node) : NULL; }) + +struct llist_node *lwq_dequeue_all (struct lwq *q); + +/** +* lwq_for_each_safe: iterate over detached queue allowing deletion +* @_n: iterator variable +* @_t1: temporary struct llist_node ** +* @_t2: temporary struct llist_node * +* @_l: address of llist_node pointer from lwq_dequeue_all() +* @_member: member in _n where lwq_node is found. +* +* Iterate over members in a dequeued list. If the iterator variable +* is set to NULL, the iterator removes that entry from the queue. +*/ +#define lwq_for_each_safe (_n, _t1, _t2, _l, _member) \ + for (_t1 = (_l); \ + *(_t1) ? (_n = container_of(*(_t1), typeof(*(_n)), _member.node),\ + _t2 = ((*_t1)->next), \ + true) \ + : false; \ + (_n) ? (_t1 = &(_n)->_member.node.next, 0) \ + : ((*(_t1) = (_t2)), 0)) + +static inline bool lwq_enqueue (struct lwq_node *n,struct lwq *q) +{ +return llist_add (&n->node, &q->new) &&READ_ONCE (q->ready) ==NULL ; +} + +static inline bool lwq_enqueue_batch (struct llist_node *n,struct lwq *q) +{ +struct llist_node *e = n; + +return llist_add_batch (llist_reverse_order (n), e, &q->new) && +READ_ONCE (q->ready) ==NULL ; +} + +#endif /* SUNRPC_SVC_LWQ_H */ net/sunrpc/Kconfig +6
-0@@ -115,3 +115,9 @@ config SUNRPC_XPRT_RDMA If unsure, or you know there is no RDMA capability on your hardware platform, say N. + +config SUNRPC_LWQ_TEST + bool "RPC: enable boot-time test for lwq queuing" + depends on SUNRPC + help + Enable boot-time test of lwq functionality.net/sunrpc/Makefile +1
-1@@ -10,7 +10,7 @@ obj-$(CONFIG_SUNRPC_XPRT_RDMA) += xprtrdma/ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \ auth.o auth_null.o auth_tls.o auth_unix.o \ - svc.o svcsock.o svcauth.o svcauth_unix.o \ + svc.o svc_lwq.o svcsock.o svcauth.o svcauth_unix.o \ addr.o rpcb_clnt.o timer.o xdr.o \ sunrpc_syms.o cache.o rpc_pipe.o sysfs.o \ svc_xprt.o \net/sunrpc/svc_lwq.c new file +135
-0@@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +* Light weight single-linked queue. +* +* Entries are enqueued to the head of an llist, with no blocking. +* This can happen in any context. +* +* Entries are dequeued using a spinlock to protect against +* multiple access. The llist is staged in reverse order, and refreshed +* from the llist when it exhausts. +*/ +#include <linux/rcupdate.h> +#include <linux/sunrpc/svc_lwq.h> + +struct llist_node *__lwq_dequeue (struct lwq *q) +{ +struct llist_node *this; + +if (lwq_empty (q)) +return NULL ; +spin_lock (&q->lock); + this = q->ready; +if (!this) + this =llist_reverse_order (llist_del_all (&q->new)); +if (this) + q->ready =llist_next (this); +spin_unlock (&q->lock); +return this; +} + +struct llist_node *lwq_dequeue_all (struct lwq *q) +{ +struct llist_node *r, *t, **ep; + +if (lwq_empty (q)) +return NULL ; + +spin_lock (&q->lock); + r = q->ready; + q->ready =NULL ; + t =llist_del_all (&q->new); +spin_unlock (&q->lock); + ep = &r; +while (*ep) + ep = &(*ep)->next; + *ep =llist_reverse_order (t); +return r; +} + +#if IS_ENABLED (CONFIG_SUNRPC_LWQ_TEST ) + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/wait_bit.h> +#include <linux/kthread.h> +#include <linux/delay.h> +struct tnode { +struct lwq_node n; +int i; +int c; +}; + +static int lwq_exercise (void *qv) +{ +struct lwq *q = qv; +int cnt; +struct tnode *t; + +for (cnt =0 ; cnt <10000 ; cnt++) { + wait_var_event(q, (t = lwq_dequeue(q, struct tnode, n)) != NULL); + t->c++; +if (lwq_enqueue (&t->n, q)) +wake_up_var (q); + } +wait_var_event (q,kthread_should_stop ()); +return 0 ; +} + +static int lwq_test (void ) +{ +int i; +struct lwq q; +struct llist_node *l, **t1, *t2; +struct tnode *t; +struct task_struct *threads[8 ]; + +printk (KERN_INFO "testing lwq....\n" ); +lwq_init (&q); +printk (KERN_INFO " lwq: run some threads\n" ); +for (i =0 ; i <ARRAY_SIZE (threads); i++) + threads[i] =kthread_run (lwq_exercise, &q,"lwq_test-%d" , i); +for (i =0 ; i <100 ; i++) { + t =kmalloc (sizeof (*t),GFP_KERNEL ); + t->i = i; + t->c =0 ; +if (lwq_enqueue (&t->n, &q)) +wake_up_var (&q); + }; +/* wait for threads to exit */ +for (i =0 ; i <ARRAY_SIZE (threads); i++) +if (!IS_ERR_OR_NULL (threads[i])) +kthread_stop (threads[i]); +printk (KERN_INFO " lwq: dequeue first 50:" ); +for (i =0 ; i <50 ; i++) { +if (i && (i %10 ) ==0 ) { +printk (KERN_CONT "\n" ); +printk (KERN_INFO " lwq: ... " ); + } + t = lwq_dequeue(&q, struct tnode, n); +printk (KERN_CONT " %d(%d)" , t->i, t->c); +kfree (t); + } +printk (KERN_CONT "\n" ); + l =lwq_dequeue_all (&q); +printk (KERN_INFO " lwq: delete the multiples of 3 (test lwq_for_each_safe())\n" ); +lwq_for_each_safe (t, t1, t2, &l, n) { +if ((t->i %3 ) ==0 ) { + t->i =-1 ; +kfree (t); + t =NULL ; + } + } +if (l) +lwq_enqueue_batch (l, &q); +printk (KERN_INFO " lwq: dequeue remaining:" ); + while ((t = lwq_dequeue(&q, struct tnode, n)) != NULL) { +printk (KERN_CONT " %d" , t->i); +kfree (t); + } +printk (KERN_CONT "\n" ); +return 0 ; +} + +module_init (lwq_test); +#endif /* CONFIG_SUNRPC_LWQ_TEST*/ Signature · 1 line
2.40.1
Signature · 1 line
Chuck Lever
Re: [PATCH 06/10] SUNRPC/svc: add light-weight queuing mechanism.
Quoted 8 lines · Chuck Lever
Quoted 7 lines · NeilBrown
lwq is a FIFO single-linked queue that only requires a spinlock for dequeueing, which happens in process context. Enqueueing is atomic with no spinlock and can happen in any context.
Include a unit test for basic functionality - runs a boot/module-load time. Does not use kunit framework.
Quoted 7 lines
Quoted 3 lines
Signed-off-by NeilBrown <[email protected]>--- include/linux/sunrpc/svc_lwq.h | 79 +++++++++++++++++++I'm wondering what your longer-term intentions are for this new mechanism. If it is only useful for SunRPC, then perhaps this header belongs under net/sunrpc instead.
I try to avoid long-term intentions, they rarely work out :-)
I did want to put it under net/sunrpc. But that requires moving structure definitions for svc_pool, svc_serv, and svc_xprt into net/sunrpc - which I would like to do.
But there are a few places where svc_xprt (at least) is accessed from fs/nfsd/ either directly (xpt_flags, xpt_cred, xpt_local ...) or through inlines. (svc_xprt_get(), svc_xpt_set_local() ...).
We we would need to create APIs to replace the direct accesses, and turn the inlines into EXPORT_SYMBOL function.
So I don't think it is practical.
I did contemplate putting lwq.c in lib/ but thought that could come later if someone else found it useful.
NeilBrown
Re: [PATCH 06/10] SUNRPC/svc: add light-weight queuing mechanism.
Quoted 26 lines · NeilBrown
Quoted 8 lines · Chuck Lever
Quoted 7 lines · NeilBrown
lwq is a FIFO single-linked queue that only requires a spinlock for dequeueing, which happens in process context. Enqueueing is atomic with no spinlock and can happen in any context.
Include a unit test for basic functionality - runs a boot/module-load time. Does not use kunit framework.
Quoted 7 lines
Quoted 3 lines
Signed-off-by NeilBrown <[email protected]>--- include/linux/sunrpc/svc_lwq.h | 79 +++++++++++++++++++I'm wondering what your longer-term intentions are for this new mechanism. If it is only useful for SunRPC, then perhaps this header belongs under net/sunrpc instead.
I try to avoid long-term intentions, they rarely work out :-)
I did want to put it under net/sunrpc. But that requires moving structure definitions for
svc_pool,svc_serv, andsvc_xprtinto net/sunrpc - which I would like to do.
But there are a few places wheresvc_xprt(at least) is accessed fromfs/nfsd/either directly (xpt_flags,xpt_cred,xpt_local...) or through inlines. (svc_xprt_get(),svc_xpt_set_local()...).
Quoted 4 lines
We we would need to create APIs to replace the direct accesses, and turn the inlines into
EXPORT_SYMBOLfunction.So I don't think it is practical.
Fair enough. It doesn't look difficult to fix those issues (which are effectively layering violations) but perhaps that's for another day.
Meanwhile, do these new files need names that begin with "svc_" ? I don't see anything server-specific about them. I'd prefer just "lwq.[ch]".
Signature · 1 line
Chuck Lever
PATCH 7/10 SUNRPC: use lwq for sp_sockets - renamed to sp_xprts
In reply to NeilBrownlwq avoids using back pointers in lists, and uses less locking. This introduces a new spinlock, but the other one will be removed in a future patch.
For svc_clean_up_xprts(), we now dequeue the entire queue, walk it to remove and process the xprts that need cleaning up, then re-enqueue the remaining queue.
--- include/linux/sunrpc/svc.h | 3 +- include/linux/sunrpc/svc_xprt.h | 2 +- net/sunrpc/svc.c | 2 +- net/sunrpc/svc_xprt.c | 57 ++++++++++----------------------- 4 files changed, 21 insertions(+), 43 deletions(-)
include/linux/sunrpc/svc.h +2 -1
@@ -17,6 +17,7 @@#include <linux/sunrpc/xdr.h> #include <linux/sunrpc/auth.h> #include <linux/sunrpc/svcauth.h> +#include <linux/sunrpc/svc_lwq.h> #include <linux/wait.h> #include <linux/mm.h> #include <linux/pagevec.h>
@@ -34,7 +35,7 @@struct svc_pool {unsigned int sp_id;/* pool id; also node id on NUMA */ spinlock_t sp_lock;/* protects all fields */ -struct list_head sp_sockets;/* pending sockets */ +struct lwq sp_xprts;/* pending transports */ unsigned int sp_nrthreads;/* # of threads in pool */ struct list_head sp_all_threads;/* all server threads */ struct llist_head sp_idle_threads;/* idle server threads */
include/linux/sunrpc/svc_xprt.h +1 -1
@@ -54,7 +54,7 @@ struct svc_xprt {const struct svc_xprt_ops *xpt_ops;struct kref xpt_ref;struct list_head xpt_list; -struct list_head xpt_ready; +struct lwq_node xpt_ready;unsigned long xpt_flags;struct svc_serv *xpt_server;/* service for transport */
net/sunrpc/svc.c +1 -1
@@ -508,7 +508,7 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools, i, serv->sv_name); pool->sp_id = i; -INIT_LIST_HEAD (&pool->sp_sockets); +lwq_init (&pool->sp_xprts);INIT_LIST_HEAD (&pool->sp_all_threads);init_llist_head (&pool->sp_idle_threads);spin_lock_init (&pool->sp_lock);
net/sunrpc/svc_xprt.c +17 -40
101 lines not on this page · raw message
Signature · 1 line
2.40.1
PATCH 8/10 SUNRPC: change sp_nrthreads to atomic_t
In reply to NeilBrownUsing an atomic_t avoids the need to take a spinlock (which can soon be removed).
Choosing a thread to kill needs to be careful as we cannot set the "die now" bit atomically with the test on the count. Instead we temporarily increase the count.
--- fs/nfsd/nfssvc.c | 3 ++- include/linux/sunrpc/svc.h | 2 +- net/sunrpc/svc.c | 37 ++++++++++++++++++++----------------- 3 files changed, 23 insertions(+), 19 deletions(-)
fs/nfsd/nfssvc.c +2 -1
@@ -718,7 +718,8 @@ int nfsd_get_nrthreads(int n, int *nthreads, struct net *net)if (nn->nfsd_serv !=NULL ) {for (i =0 ; i < nn->nfsd_serv->sv_nrpools && i < n; i++) - nthreads[i] = nn->nfsd_serv->sv_pools[i].sp_nrthreads; + nthreads[i] = +atomic_read (&nn->nfsd_serv->sv_pools[i].sp_nrthreads); }return 0 ;
include/linux/sunrpc/svc.h +1 -1
@@ -36,7 +36,7 @@ struct svc_pool {unsigned int sp_id;/* pool id; also node id on NUMA */ spinlock_t sp_lock;/* protects all fields */ struct lwq sp_xprts;/* pending transports */ -unsigned int sp_nrthreads;/* # of threads in pool */ +atomic_t sp_nrthreads;/* # of threads in pool */ struct list_head sp_all_threads;/* all server threads */ struct llist_head sp_idle_threads;/* idle server threads */
net/sunrpc/svc.c +20 -17
@@ -681,8 +681,8 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) serv->sv_nrthreads +=1 ;spin_unlock_bh (&serv->sv_lock); +atomic_inc (&pool->sp_nrthreads);spin_lock_bh (&pool->sp_lock); - pool->sp_nrthreads++;list_add_rcu (&rqstp->rq_all, &pool->sp_all_threads);spin_unlock_bh (&pool->sp_lock);return rqstp;
@@ -727,23 +727,24 @@ svc_pool_next(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state) }static struct svc_pool * -svc_pool_victim (struct svc_serv *serv,struct svc_pool *pool,unsigned int *state) +svc_pool_victim (struct svc_serv *serv,struct svc_pool *target_pool, +unsigned int *state) { +struct svc_pool *pool;unsigned int i; +retry: + pool = target_pool; +if (pool !=NULL ) { -spin_lock_bh (&pool->sp_lock); -if (pool->sp_nrthreads) +if (atomic_inc_not_zero (&pool->sp_nrthreads)) goto found_pool; -spin_unlock_bh (&pool->sp_lock);return NULL ; }else {for (i =0 ; i < serv->sv_nrpools; i++) { pool = &serv->sv_pools[--(*state) % serv->sv_nrpools]; -spin_lock_bh (&pool->sp_lock); -if (pool->sp_nrthreads) +if (atomic_inc_not_zero (&pool->sp_nrthreads)) goto found_pool; -spin_unlock_bh (&pool->sp_lock); }return NULL ; }
@@ -751,8 +752,12 @@ svc_pool_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *stat found_pool:set_bit (SP_VICTIM_REMAINS , &pool->sp_flags);set_bit (SP_NEED_VICTIM , &pool->sp_flags); -spin_unlock_bh (&pool->sp_lock); -return pool; +if (!atomic_dec_and_test (&pool->sp_nrthreads)) +return pool; +/* Nothing left in this pool any more */ +clear_bit (SP_NEED_VICTIM , &pool->sp_flags); +clear_bit (SP_VICTIM_REMAINS , &pool->sp_flags); + goto retry; } static int
@@ -828,13 +833,10 @@ svc_stop_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)int svc_set_num_threads (struct svc_serv *serv,struct svc_pool *pool,int nrservs) { -if (pool ==NULL ) { +if (!pool) nrservs -= serv->sv_nrthreads; - }else { -spin_lock_bh (&pool->sp_lock); - nrservs -= pool->sp_nrthreads; -spin_unlock_bh (&pool->sp_lock); - } +else + nrservs -=atomic_read (&pool->sp_nrthreads);if (nrservs >0 )return svc_start_kthreads (serv, pool, nrservs);
@@ -921,10 +923,11 @@ svc_exit_thread(struct svc_rqst *rqstp)struct svc_pool *pool = rqstp->rq_pool;spin_lock_bh (&pool->sp_lock); - pool->sp_nrthreads--;list_del_rcu (&rqstp->rq_all);spin_unlock_bh (&pool->sp_lock); +atomic_dec (&pool->sp_nrthreads); +spin_lock_bh (&serv->sv_lock); serv->sv_nrthreads -=1 ;spin_unlock_bh (&serv->sv_lock);
Signature · 1 line
2.40.1
PATCH 9/10 SUNRPC: discard sp_lock
In reply to NeilBrownsp_lock is now only used to protect sp_all_threads. This isn't needed as sp_all_threads is only manipulated through svc_set_num_threads(), which must be locked. Read-acccess only requires rcu_read_lock(). So no more locking is needed.
--- include/linux/sunrpc/svc.h | 1 - net/sunrpc/svc.c | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-)
include/linux/sunrpc/svc.h +0 -1
@@ -34,7 +34,6 @@*/ struct svc_pool {unsigned int sp_id;/* pool id; also node id on NUMA */ -spinlock_t sp_lock;/* protects all fields */ struct lwq sp_xprts;/* pending transports */ atomic_t sp_nrthreads;/* # of threads in pool */ struct list_head sp_all_threads;/* all server threads */
net/sunrpc/svc.c +5 -5
@@ -511,7 +511,6 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools,lwq_init (&pool->sp_xprts);INIT_LIST_HEAD (&pool->sp_all_threads);init_llist_head (&pool->sp_idle_threads); -spin_lock_init (&pool->sp_lock);percpu_counter_init (&pool->sp_messages_arrived,0 ,GFP_KERNEL );percpu_counter_init (&pool->sp_sockets_queued,0 ,GFP_KERNEL );
@@ -682,9 +681,12 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)spin_unlock_bh (&serv->sv_lock);atomic_inc (&pool->sp_nrthreads); -spin_lock_bh (&pool->sp_lock); + +/* Protected by whatever lock the service uses when calling +* svc_set_num_threads() +*/ list_add_rcu (&rqstp->rq_all, &pool->sp_all_threads); -spin_unlock_bh (&pool->sp_lock); +return rqstp; }
@@ -922,9 +924,7 @@ svc_exit_thread(struct svc_rqst *rqstp)struct svc_serv *serv = rqstp->rq_server;struct svc_pool *pool = rqstp->rq_pool; -spin_lock_bh (&pool->sp_lock);list_del_rcu (&rqstp->rq_all); -spin_unlock_bh (&pool->sp_lock);atomic_dec (&pool->sp_nrthreads);
Signature · 1 line
2.40.1
PATCH 10/10 SUNRPC: change the back-channel queue to lwq
In reply to NeilBrownThis removes the need to store and update back-links in the list. It also remove the need for the _bh version of spin_lock().
--- include/linux/sunrpc/svc.h | 3 +-- include/linux/sunrpc/xprt.h | 3 ++- net/sunrpc/backchannel_rqst.c | 5 +---- net/sunrpc/svc.c | 3 +-- net/sunrpc/svc_xprt.c | 12 +++--------- net/sunrpc/xprtrdma/backchannel.c | 4 +--- 6 files changed, 9 insertions(+), 21 deletions(-)
include/linux/sunrpc/svc.h +1 -2
@@ -90,10 +90,9 @@ struct svc_serv {int (*sv_threadfn)(void *data);#if defined(CONFIG_SUNRPC_BACKCHANNEL ) -struct list_head sv_cb_list;/* queue for callback requests +struct lwq sv_cb_list;/* queue for callback requests * that arrive over the same * connection */ -spinlock_t sv_cb_lock;/* protects the svc_cb_list */ bool sv_bc_enabled;/* service uses backchannel */ #endif /* CONFIG_SUNRPC_BACKCHANNEL */ };
include/linux/sunrpc/xprt.h +2 -1
@@ -57,6 +57,7 @@ struct xprt_class;struct seq_file ;struct svc_serv ;struct net ; +#include <linux/sunrpc/svc_lwq.h> /* * This describes a complete RPC request
@@ -121,7 +122,7 @@ struct rpc_rqst {int rq_ntrans;#if defined(CONFIG_SUNRPC_BACKCHANNEL ) -struct list_head rq_bc_list;/* Callback service list */ +struct lwq_node rq_bc_list;/* Callback service list */ unsigned long rq_bc_pa_state;/* Backchannel prealloc state */ struct list_head rq_bc_pa_list;/* Backchannel prealloc list */ #endif /* CONFIG_SUNRPC_BACKCHANEL */
net/sunrpc/backchannel_rqst.c +1 -4
@@ -83,7 +83,6 @@ static struct rpc_rqst *xprt_alloc_bc_req(struct rpc_xprt *xprt)return NULL ; req->rq_xprt = xprt; -INIT_LIST_HEAD (&req->rq_bc_list);/* Preallocate one XDR receive buffer */ if (xprt_alloc_xdr_buf (&req->rq_rcv_buf, gfp_flags) <0 ) {
@@ -367,8 +366,6 @@ void xprt_complete_bc_request(struct rpc_rqst *req, uint32_t copied)dprintk ("RPC: add callback request to list\n" );xprt_get (xprt); -spin_lock (&bc_serv->sv_cb_lock); -list_add (&req->rq_bc_list, &bc_serv->sv_cb_list); -spin_unlock (&bc_serv->sv_cb_lock); +lwq_enqueue (&req->rq_bc_list, &bc_serv->sv_cb_list);svc_pool_wake_idle_thread (&bc_serv->sv_pools[0 ]); }
net/sunrpc/svc.c +1 -2
@@ -438,8 +438,7 @@ EXPORT_SYMBOL_GPL(svc_bind);static void __svc_init_bc (struct svc_serv *serv) { -INIT_LIST_HEAD (&serv->sv_cb_list); -spin_lock_init (&serv->sv_cb_lock); +lwq_init (&serv->sv_cb_list); }#else static void
net/sunrpc/svc_xprt.c +3 -9
@@ -705,7 +705,7 @@ rqst_should_sleep(struct svc_rqst *rqstp)#if defined(CONFIG_SUNRPC_BACKCHANNEL )if (svc_is_backchannel (rqstp)) { -if (!list_empty (&rqstp->rq_server->sv_cb_list)) +if (!lwq_empty (&rqstp->rq_server->sv_cb_list))return false; }#endif
@@ -883,18 +883,12 @@ void svc_recv(struct svc_rqst *rqstp)struct svc_serv *serv = rqstp->rq_server;struct rpc_rqst *req; -spin_lock_bh (&serv->sv_cb_lock); - req =list_first_entry_or_null (&serv->sv_cb_list, - struct rpc_rqst, rq_bc_list); + req =lwq_dequeue (&serv->sv_cb_list, + struct rpc_rqst, rq_bc_list);if (req) { -list_del (&req->rq_bc_list); -spin_unlock_bh (&serv->sv_cb_lock);wake_next (rqstp); -svc_process_bc (req, rqstp); -return ; } - spin_unlock_bh(&serv->sv_cb_lock); }#endif }
net/sunrpc/xprtrdma/backchannel.c +1 -3
@@ -263,9 +263,7 @@ void rpcrdma_bc_receive_call(struct rpcrdma_xprt *r_xprt,/* Queue rqst for ULP's callback service */ bc_serv = xprt->bc_serv;xprt_get (xprt); -spin_lock (&bc_serv->sv_cb_lock); -list_add (&rqst->rq_bc_list, &bc_serv->sv_cb_list); -spin_unlock (&bc_serv->sv_cb_lock); +lwq_enqueue (&rqst->rq_bc_list, &bc_serv->sv_cb_list);svc_pool_wake_idle_thread (&bc_serv->sv_pools[0 ]);
Signature · 1 line
2.40.1