rcu: uninline __rcu_pending()
PATCH 1/3 rcu: uninline __rcu_pending()
__rcu_pending() is rather fat and called twice from rcu_pending().
rcu_pending() has multiple callers, and not that small too.
This patch uninlines both of them.
Signed-off-by Oleg Nesterov <[email protected]>
include/linux/rcupdate.h renamed from include/linux/rcupdate.h~1_OUTL +1 -30
@@ -125,36 +125,7 @@ static inline void rcu_bh_qsctr_inc(int rdp->passed_quiesc =1 ; } -static inline int __rcu_pending (struct rcu_ctrlblk *rcp, -struct rcu_data *rdp) -{ -/* This cpu has pending rcu entries and the grace period -* for them has completed. -*/ -if (rdp->curlist && !rcu_batch_before (rcp->completed, rdp->batch)) -return 1 ; - -/* This cpu has no pending entries, but there are new entries */ -if (!rdp->curlist && rdp->nxtlist) -return 1 ; - -/* This cpu has finished callbacks to invoke */ -if (rdp->donelist) -return 1 ; - -/* The rcu core waits for a quiescent state from the cpu */ -if (rdp->quiescbatch != rcp->cur || rdp->qs_pending) -return 1 ; - -/* nothing to do */ -return 0 ; -} - -static inline int rcu_pending (int cpu) -{ -return __rcu_pending (&rcu_ctrlblk, &per_cpu (rcu_data, cpu)) || -__rcu_pending (&rcu_bh_ctrlblk, &per_cpu (rcu_bh_data, cpu)); -} +extern int rcu_pending (int cpu);/** * rcu_read_lock - mark the beginning of an RCU read-side critical section.
kernel/rcupdate.c renamed from kernel/rcupdate.c~1_OUTL +30 -0
@@ -442,6 +442,36 @@ static void rcu_process_callbacks(unsign &__get_cpu_var(rcu_bh_data)); } +static int __rcu_pending (struct rcu_ctrlblk *rcp,struct rcu_data *rdp) +{ +/* This cpu has pending rcu entries and the grace period +* for them has completed. +*/ +if (rdp->curlist && !rcu_batch_before (rcp->completed, rdp->batch)) +return 1 ; + +/* This cpu has no pending entries, but there are new entries */ +if (!rdp->curlist && rdp->nxtlist) +return 1 ; + +/* This cpu has finished callbacks to invoke */ +if (rdp->donelist) +return 1 ; + +/* The rcu core waits for a quiescent state from the cpu */ +if (rdp->quiescbatch != rcp->cur || rdp->qs_pending) +return 1 ; + +/* nothing to do */ +return 0 ; +} + +int rcu_pending (int cpu) +{ +return __rcu_pending (&rcu_ctrlblk, &per_cpu (rcu_data, cpu)) || +__rcu_pending (&rcu_bh_ctrlblk, &per_cpu (rcu_bh_data, cpu)); +} +void rcu_check_callbacks (int cpu,int user) { if (user ||
Quoted 5 lines · Oleg Nesterov
__rcu_pending()is rather fat and called twice fromrcu_pending().
rcu_pending()has multiple callers, and not that small too.This patch uninlines both of them.
Looks good to me! Passes a quick rcutorture test on x86. Running more extensive tests on both x86 and ppc64.
Thanx, Paul
Acked-by Paul E. McKenney <[email protected]>
Quoted 82 lines
Signed-off-by Oleg Nesterov <[email protected]>include/linux/rcupdate.h renamed from include/linux/rcupdate.h~1_OUTL +1
-30@@ -125,36 +125,7 @@ static inline void rcu_bh_qsctr_inc(int rdp->passed_quiesc =1 ; } -static inline int __rcu_pending (struct rcu_ctrlblk *rcp, -struct rcu_data *rdp) -{ -/* This cpu has pending rcu entries and the grace period -* for them has completed. -*/ -if (rdp->curlist && !rcu_batch_before (rcp->completed, rdp->batch)) -return 1 ; - -/* This cpu has no pending entries, but there are new entries */ -if (!rdp->curlist && rdp->nxtlist) -return 1 ; - -/* This cpu has finished callbacks to invoke */ -if (rdp->donelist) -return 1 ; - -/* The rcu core waits for a quiescent state from the cpu */ -if (rdp->quiescbatch != rcp->cur || rdp->qs_pending) -return 1 ; - -/* nothing to do */ -return 0 ; -} - -static inline int rcu_pending (int cpu) -{ -return __rcu_pending (&rcu_ctrlblk, &per_cpu (rcu_data, cpu)) || -__rcu_pending (&rcu_bh_ctrlblk, &per_cpu (rcu_bh_data, cpu)); -} +extern int rcu_pending (int cpu);/** * rcu_read_lock - mark the beginning of an RCU read-side critical section. kernel/rcupdate.c renamed from kernel/rcupdate.c~1_OUTL +30
-0@@ -442,6 +442,36 @@ static void rcu_process_callbacks(unsign &__get_cpu_var(rcu_bh_data)); } +static int __rcu_pending (struct rcu_ctrlblk *rcp,struct rcu_data *rdp) +{ +/* This cpu has pending rcu entries and the grace period +* for them has completed. +*/ +if (rdp->curlist && !rcu_batch_before (rcp->completed, rdp->batch)) +return 1 ; + +/* This cpu has no pending entries, but there are new entries */ +if (!rdp->curlist && rdp->nxtlist) +return 1 ; + +/* This cpu has finished callbacks to invoke */ +if (rdp->donelist) +return 1 ; + +/* The rcu core waits for a quiescent state from the cpu */ +if (rdp->quiescbatch != rcp->cur || rdp->qs_pending) +return 1 ; + +/* nothing to do */ +return 0 ; +} + +int rcu_pending (int cpu) +{ +return __rcu_pending (&rcu_ctrlblk, &per_cpu (rcu_data, cpu)) || +__rcu_pending (&rcu_bh_ctrlblk, &per_cpu (rcu_bh_data, cpu)); +} +void rcu_check_callbacks (int cpu,int user) { if (user ||