kfree_rcu() updates for v5.6
Subsystems (4)
Paul E. McKenney
maintainercover kfree_rcu() updates for v5.6
Hello!
The following series disentangles kfree_rcu() from core RCU and also provides batching, all courtesy of Joel Fernandes:
1. Add basic support for kfree_rcu() batching.
2. Add kfree_rcu() performance tests.
3. Make kfree_rcu() use a non-atomic ->monitor_todo.
4. Add multiple in-flight batches of kfree_rcu() work.
5. Add support for debug_objects debugging for kfree_rcu().
6. Remove kfree_rcu() special casing and lazy-callback handling.
7. Remove kfree_call_rcu_nobatch().
Thanx, Paul
------------------------------------------------------------------------
Documentation/RCU/stallwarn.txt | 11 Documentation/admin-guide/kernel-parameters.txt | 21 + include/linux/rcu_segcblist.h | 2 include/linux/rcutiny.h | 11 include/linux/rcutree.h | 3 include/trace/events/rcu.h | 32 -- kernel/rcu/rcu.h | 27 - kernel/rcu/rcu_segcblist.c | 25 - kernel/rcu/rcu_segcblist.h | 25 - kernel/rcu/rcuperf.c | 191 ++++++++++++-- kernel/rcu/srcutree.c | 4 kernel/rcu/tiny.c | 28 +- kernel/rcu/tree.c | 327 ++++++++++++++++++++---- kernel/rcu/tree.h | 1 kernel/rcu/tree_plugin.h | 48 --- kernel/rcu/tree_stall.h | 6 kernel/rcu/update.c | 10 17 files changed, 548 insertions(+), 224 deletions(-)
[email protected]
maintainerPATCH 1/7 rcu: Add basic support for kfree_rcu() batching
From: "Joel Fernandes (Google)" <[email protected]>
Recently a discussion about stability and performance of a system involving a high rate of kfree_rcu() calls surfaced on the list [1] which led to another discussion how to prepare for this situation.
This patch adds basic batching support for kfree_rcu(). It is "basic" because we do none of the slab management, dynamic allocation, code moving or any of the other things, some of which previous attempts did [2]. These fancier improvements can be follow-up patches and there are different ideas being discussed in those regards. This is an effort to start simple, and build up from there. In the future, an extension to use kfree_bulk and possibly per-slab batching could be done to further improve performance due to cache-locality and slab-specific bulk free optimizations. By using an array of pointers, the worker thread processing the work would need to read lesser data since it does not need to deal with large rcu_head(s) any longer.
Torture tests follow in the next patch and show improvements of around 5x reduction in number of grace periods on a 16 CPU system. More details and test data are in that patch.
There is an implication with rcu_barrier() with this patch. Since the kfree_rcu() calls can be batched, and may not be handed yet to the RCU machinery in fact, the monitor may not have even run yet to do the queue_rcu_work(), there seems no easy way of implementing rcu_barrier() to wait for those kfree_rcu()s that are already made. So this means a kfree_rcu() followed by an rcu_barrier() does not imply that memory will be freed once rcu_barrier() returns.
Another implication is higher active memory usage (although not run-away..) until the kfree_rcu() flooding ends, in comparison to without batching. More details about this are in the second patch which adds an rcuperf test.
Finally, in the near future we will get rid of kfree_rcu() special casing within RCU such as in rcu_do_batch and switch everything to just batching. Currently we don't do that since timer subsystem is not yet up and we cannot schedule the kfree_rcu() monitor as the timer subsystem's lock are not initialized. That would also mean getting rid of kfree_call_rcu_nobatch() entirely.
[1] http://lore.kernel.org/lkml/[email protected]
[2] https://lkml.org/lkml/2017/12/19/824
--- include/linux/rcutiny.h | 6 ++ include/linux/rcutree.h | 2 + kernel/rcu/tree.c | 194 ++++++++++++++++++++++++++++++++++++++++++++++-- kernel/rcu/update.c | 10 +++ 4 files changed, 206 insertions(+), 6 deletions(-)
include/linux/rcutiny.h +6 -0
@@ -39,6 +39,11 @@ static inline void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)call_rcu (head, func); } +static inline void kfree_call_rcu_nobatch (struct rcu_head *head,rcu_callback_t func) +{ +call_rcu (head, func); +} +void rcu_qs (void );static inline void rcu_softirq_qs (void )
@@ -85,6 +90,7 @@ static inline void rcu_scheduler_starting(void) { }static inline void rcu_end_inkernel_boot (void ) { }static inline bool rcu_is_watching (void ) {return true; }static inline void rcu_momentary_dyntick_idle (void ) { } +static inline void kfree_rcu_scheduler_running (void ) { }/* Avoid RCU read-side critical sections leaking across. */ static inline void rcu_all_qs (void ) {barrier (); }
include/linux/rcutree.h +2 -0
@@ -34,10 +34,12 @@ static inline void rcu_virt_note_context_switch(int cpu)void synchronize_rcu_expedited (void );void kfree_call_rcu (struct rcu_head *head,rcu_callback_t func); +void kfree_call_rcu_nobatch (struct rcu_head *head,rcu_callback_t func);void rcu_barrier (void );bool rcu_eqs_special_set (int cpu);void rcu_momentary_dyntick_idle (void ); +void kfree_rcu_scheduler_running (void );unsigned long get_state_synchronize_rcu (void );void cond_synchronize_rcu (unsigned long oldstate);
kernel/rcu/tree.c +188 -6
219 lines not on this page · raw message
kernel/rcu/update.c +10 -0
@@ -40,6 +40,7 @@#include <linux/rcupdate_wait.h> #include <linux/sched/isolation.h> #include <linux/kprobes.h> +#include <linux/slab.h> #define CREATE_TRACE_POINTS
@@ -218,6 +219,7 @@ static int __init rcu_set_runtime_mode(void) {rcu_test_sync_prims (); rcu_scheduler_active =RCU_SCHEDULER_RUNNING ; +kfree_rcu_scheduler_running ();rcu_test_sync_prims ();return 0 ; }
@@ -853,14 +855,22 @@ static void test_callback(struct rcu_head *r)DEFINE_STATIC_SRCU (early_srcu); +struct early_boot_kfree_rcu { +struct rcu_head rh; +}; +static void early_boot_test_call_rcu (void ) {static struct rcu_head head;static struct rcu_head shead; +struct early_boot_kfree_rcu *rhp;call_rcu (&head, test_callback);if (IS_ENABLED (CONFIG_SRCU ))call_srcu (&early_srcu, &shead, test_callback); + rhp =kmalloc (sizeof (*rhp),GFP_KERNEL ); +if (!WARN_ON_ONCE (!rhp)) +kfree_rcu (rhp, rh); }void rcu_early_boot_tests (void )
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 2/7 rcuperf: Add kfree_rcu() performance Tests
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
This test runs kfree_rcu() in a loop to measure performance of the new kfree_rcu() batching functionality.
The following table shows results when booting with arguments:
rcuperf.kfree_loops=20000 rcuperf.kfree_alloc_num=8000 rcuperf.kfree_rcu_test=1 rcuperf.kfree_no_batch=X
rcuperf.kfree_no_batch=X # Grace Periods Test Duration (s)
X=1 (old behavior) 9133 11.5 X=0 (new behavior) 1732 12.5
On a 16 CPU system with the above boot parameters, we see that the total number of grace periods that elapse during the test drops from 9133 when not batching to 1732 when batching (a 5X improvement). The kfree_rcu() flood itself slows down a bit when batching, though, as shown.
Note that the active memory consumption during the kfree_rcu() flood does increase to around 200-250MB due to the batching (from around 50MB without batching). However, this memory consumption is relatively constant. In other words, the system is able to keep up with the kfree_rcu() load. The memory consumption comes down considerably if KFREE_DRAIN_JIFFIES is increased from HZ/50 to HZ/80. A later patch will reduce memory consumption further by using multiple lists.
Also, when running the test, please disable CONFIG_DEBUG_PREEMPT and CONFIG_PROVE_RCU for realistic comparisons with/without batching.
--- Documentation/admin-guide/kernel-parameters.txt | 17 +++ kernel/rcu/rcuperf.c | 181 ++++++++++++++++++++++-- 2 files changed, 190 insertions(+), 8 deletions(-)
Documentation/admin-guide/kernel-parameters.txt +17 -0
@@ -3978,6 +3978,23 @@ test until boot completes in order to avoid interference. + rcuperf.kfree_rcu_test= [KNL] + Set to measure performance of kfree_rcu() flooding. + + rcuperf.kfree_nthreads= [KNL] + The number of threads running loops of kfree_rcu(). + + rcuperf.kfree_alloc_num= [KNL] + Number of allocations and frees done in an iteration. + + rcuperf.kfree_loops= [KNL] + Number of loops doing rcuperf.kfree_alloc_num number + of allocations and frees. + + rcuperf.kfree_no_batch= [KNL] + Use the non-batching (less efficient) version of kfree_rcu(). + This is useful for comparing with the batched version. + rcuperf.nreaders= [KNL] Set number of RCU readers. The value -1 selects N, where N is the number of CPUs. A value
kernel/rcu/rcuperf.c +173 -8
227 lines not on this page · raw message
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 3/7 rcu: Make kfree_rcu() use a non-atomic ->monitor_todo
In reply to Paul E. McKenneyFrom: Joel Fernandes <[email protected]>
Because the ->monitor_todo field is always protected by krcp->lock, this commit downgrades from xchg() to non-atomic unmarked assignment statements.
--- kernel/rcu/tree.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-)
kernel/rcu/tree.c +10 -6
@@ -2708,7 +2708,7 @@ struct kfree_rcu_cpu {struct rcu_head *head_free;spinlock_t lock;struct delayed_work monitor_work; -int monitor_todo; +bool monitor_todo;bool initialized; };
@@ -2765,6 +2765,7 @@ static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp, unsigned long flags) {// Attempt to start a new batch. + krcp->monitor_todo = false;if (queue_kfree_rcu_work (krcp)) {// Success! Our job is done here. spin_unlock_irqrestore (&krcp->lock, flags);
@@ -2772,8 +2773,8 @@ static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp, }// Previous RCU batch still in progress, try again later. -if (!xchg (&krcp->monitor_todo, true)) -schedule_delayed_work (&krcp->monitor_work,KFREE_DRAIN_JIFFIES ); + krcp->monitor_todo = true; +schedule_delayed_work (&krcp->monitor_work,KFREE_DRAIN_JIFFIES );spin_unlock_irqrestore (&krcp->lock, flags); }
@@ -2788,7 +2789,7 @@ static void kfree_rcu_monitor(struct work_struct *work) monitor_work.work);spin_lock_irqsave (&krcp->lock, flags); -if (xchg (&krcp->monitor_todo, false)) +if (krcp->monitor_todo)kfree_rcu_drain_unlock (krcp, flags);else spin_unlock_irqrestore (&krcp->lock, flags);
@@ -2837,8 +2838,10 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)// Set timer to drain after KFREE_DRAIN_JIFFIES. if (rcu_scheduler_active ==RCU_SCHEDULER_RUNNING && - !xchg (&krcp->monitor_todo, true)) + !krcp->monitor_todo) { + krcp->monitor_todo = true;schedule_delayed_work (&krcp->monitor_work,KFREE_DRAIN_JIFFIES ); + }if (krcp->initialized)spin_unlock (&krcp->lock);
@@ -2855,10 +2858,11 @@ void __init kfree_rcu_scheduler_running(void)struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu);spin_lock_irqsave (&krcp->lock, flags); -if (!krcp->head ||xchg (&krcp->monitor_todo, true)) { +if (!krcp->head || krcp->monitor_todo) {spin_unlock_irqrestore (&krcp->lock, flags);continue ; } + krcp->monitor_todo = true;schedule_delayed_work (&krcp->monitor_work,KFREE_DRAIN_JIFFIES );spin_unlock_irqrestore (&krcp->lock, flags); }
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 4/7 rcu: Add multiple in-flight batches of kfree_rcu() work
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
During testing, it was observed that amount of memory consumed due kfree_rcu() batching is 300-400MB. Previously we had only a single head_free pointer pointing to the list of rcu_head(s) that are to be freed after a grace period. Until this list is drained, we cannot queue any more objects on it since such objects may not be ready to be reclaimed when the worker thread eventually gets to drainin g the head_free list.
We can do better by maintaining multiple lists as done by this patch. Testing shows that memory consumption came down by around 100-150MB with just adding another list. Adding more than 1 additional list did not show any improvement.
--- kernel/rcu/tree.c | 51 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 12 deletions(-)
kernel/rcu/tree.c +39 -12
@@ -2686,12 +2686,25 @@ EXPORT_SYMBOL_GPL(call_rcu);/* Maximum number of jiffies to wait before draining a batch. */ #define KFREE_DRAIN_JIFFIES (HZ / 50) +#define KFREE_N_BATCHES 2/** -* struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period +* @head_free: List of kfree_rcu() objects waiting for a grace period +* @krcp: Pointer to @kfree_rcu_cpu structure +*/ + +struct kfree_rcu_cpu_work { +struct rcu_work rcu_work; +struct rcu_head *head_free; +struct kfree_rcu_cpu *krcp; +}; + +/** +* struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period * @head: List of kfree_rcu() objects not yet waiting for a grace period -* @head_free: List of kfree_rcu() objects already waiting for a grace period +* @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period * @lock: Synchronize access to this structure * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES * @monitor_todo: Tracks whether a @monitor_work delayed work is pending
@@ -2703,9 +2716,8 @@ EXPORT_SYMBOL_GPL(call_rcu);* the interactions with the slab allocators. */ struct kfree_rcu_cpu { -struct rcu_work rcu_work;struct rcu_head *head; -struct rcu_head *head_free; +struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];spinlock_t lock;struct delayed_work monitor_work;bool monitor_todo;
@@ -2723,11 +2735,14 @@ static void kfree_rcu_work(struct work_struct *work)unsigned long flags;struct rcu_head *head, *next;struct kfree_rcu_cpu *krcp; +struct kfree_rcu_cpu_work *krwp; - krcp = container_of(to_rcu_work(work), struct kfree_rcu_cpu, rcu_work); + krwp =container_of (to_rcu_work (work), + struct kfree_rcu_cpu_work, rcu_work); + krcp = krwp->krcp;spin_lock_irqsave (&krcp->lock, flags); - head = krcp->head_free; - krcp->head_free =NULL ; + head = krwp->head_free; + krwp->head_free =NULL ;spin_unlock_irqrestore (&krcp->lock, flags);// List "head" is now private, so traverse locklessly.
@@ -2747,17 +2762,25 @@ static void kfree_rcu_work(struct work_struct *work)*/ static inline bool queue_kfree_rcu_work (struct kfree_rcu_cpu *krcp) { +int i; +struct kfree_rcu_cpu_work *krwp =NULL ; +lockdep_assert_held (&krcp->lock); +for (i =0 ; i <KFREE_N_BATCHES ; i++) +if (!krcp->krw_arr[i].head_free) { + krwp = &(krcp->krw_arr[i]); +break ; + }// If a previous RCU batch is in progress, we cannot immediately // queue another one, so return false to tell caller to retry. -if (krcp->head_free) +if (!krwp)return false; - krcp->head_free = krcp->head; + krwp->head_free = krcp->head; krcp->head =NULL ; -INIT_RCU_WORK (&krcp->rcu_work, kfree_rcu_work); -queue_rcu_work (system_wq, &krcp->rcu_work); +INIT_RCU_WORK (&krwp->rcu_work, kfree_rcu_work); +queue_rcu_work (system_wq, &krwp->rcu_work);return true; }
@@ -2863,7 +2886,8 @@ void __init kfree_rcu_scheduler_running(void)continue ; } krcp->monitor_todo = true; -schedule_delayed_work (&krcp->monitor_work,KFREE_DRAIN_JIFFIES ); +schedule_delayed_work_on (cpu, &krcp->monitor_work, +KFREE_DRAIN_JIFFIES );spin_unlock_irqrestore (&krcp->lock, flags); } }
@@ -3732,11 +3756,14 @@ struct workqueue_struct *rcu_par_gp_wq;static void __init kfree_rcu_batch_init (void ) {int cpu; +int i;for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu);spin_lock_init (&krcp->lock); +for (i =0 ; i <KFREE_N_BATCHES ; i++) + krcp->krw_arr[i].krcp = krcp;INIT_DELAYED_WORK (&krcp->monitor_work, kfree_rcu_monitor); krcp->initialized = true; }
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 5/7 rcu: Add support for debug_objects debugging for kfree_rcu()
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
This commit applies RCU's debug_objects debugging to the new batched kfree_rcu() implementations. The object is queued at the kfree_rcu() call and dequeued during reclaim.
Tested that enabling CONFIG_DEBUG_OBJECTS_RCU_HEAD successfully detects double kfree_rcu() calls.
--- kernel/rcu/tree.c | 8 ++++++++ 1 file changed, 8 insertions(+)
kernel/rcu/tree.c +8 -0
@@ -2749,6 +2749,7 @@ static void kfree_rcu_work(struct work_struct *work)for (; head; head = next) { next = head->next;// Potentially optimize with kfree_bulk in future. +debug_rcu_head_unqueue (head);__rcu_reclaim (rcu_state.name, head);cond_resched_tasks_rcu_qs (); }
@@ -2855,6 +2856,12 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)spin_lock (&krcp->lock);// Queue the object but don't yet schedule the batch. +if (debug_rcu_head_queue (head)) { +// Probable double kfree_rcu(), just leak. +WARN_ONCE (1 ,"%s(): Double-freed call. rcu_head %p\n" , + __func__, head); + goto unlock_return; + } head->func = func; head->next = krcp->head; krcp->head = head;
@@ -2866,6 +2873,7 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)schedule_delayed_work (&krcp->monitor_work,KFREE_DRAIN_JIFFIES ); } +unlock_return:if (krcp->initialized)spin_unlock (&krcp->lock);local_irq_restore (flags);
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 6/7 rcu: Remove kfree_rcu() special casing and lazy-callback handling
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
This commit removes kfree_rcu() special-casing and the lazy-callback handling from Tree RCU. It moves some of this special casing to Tiny RCU, the removal of which will be the subject of later commits.
This results in a nice negative delta.
--- Documentation/RCU/stallwarn.txt | 11 +++------- include/linux/rcu_segcblist.h | 2 -- include/trace/events/rcu.h | 32 +++++++++++---------------- kernel/rcu/rcu.h | 27 ----------------------- kernel/rcu/rcu_segcblist.c | 25 +++------------------ kernel/rcu/rcu_segcblist.h | 25 ++------------------- kernel/rcu/srcutree.c | 4 ++-- kernel/rcu/tiny.c | 28 +++++++++++++++++++++++- kernel/rcu/tree.c | 40 +++++++++++++++++++++++----------- kernel/rcu/tree.h | 1 - kernel/rcu/tree_plugin.h | 48 ++++++++++------------------------------- kernel/rcu/tree_stall.h | 6 ++---- 12 files changed, 90 insertions(+), 159 deletions(-)
Documentation/RCU/stallwarn.txt +3 -8
@@ -225,18 +225,13 @@ an estimate of the total number of RCU callbacks queued across all CPUs In kernels with CONFIG_RCU_FAST_NO_HZ, more information is printed for each CPU: - 0: (64628 ticks this GP) idle=dd5/3fffffffffffffff/0 softirq=82/543 last_accelerate: a345/d342 Nonlazy posted: ..D + 0: (64628 ticks this GP) idle=dd5/3fffffffffffffff/0 softirq=82/543 last_accelerate: a345/d342 dyntick_enabled: 1 The "last_accelerate:" prints the low-order 16 bits (in hex) of the jiffies counter when this CPU last invoked rcu_try_advance_all_cbs() from rcu_needs_cpu() or last invoked rcu_accelerate_cbs() from -rcu_prepare_for_idle(). The "Nonlazy posted:" indicates lazy-callback -status, so that an "l" indicates that all callbacks were lazy at the start -of the last idle period and an "L" indicates that there are currently -no non-lazy callbacks (in both cases, "." is printed otherwise, as -shown above) and "D" indicates that dyntick-idle processing is enabled -("." is printed otherwise, for example, if disabled via the "nohz=" -kernel boot parameter). +rcu_prepare_for_idle(). "dyntick_enabled: 1" indicates that dyntick-idle +processing is enabled. If the grace period ends just as the stall warning starts printing, there will be a spurious stall-warning message, which will include
include/linux/rcu_segcblist.h +0 -2
@@ -22,7 +22,6 @@ struct rcu_cblist {struct rcu_head *head;struct rcu_head **tail;long len; -long len_lazy; };#define RCU_CBLIST_INITIALIZER (n) { .head = NULL, .tail = &n.head }
@@ -73,7 +72,6 @@ struct rcu_segcblist {#else long len;#endif -long len_lazy;u8 enabled;u8 offloaded; };
include/trace/events/rcu.h +12 -20
@@ -481,16 +481,14 @@ TRACE_EVENT_RCU(rcu_dyntick,*/ TRACE_EVENT_RCU(rcu_callback, - TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen_lazy, - long qlen), + TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen), - TP_ARGS(rcuname, rhp, qlen_lazy, qlen), + TP_ARGS(rcuname, rhp, qlen), TP_STRUCT__entry( __field(const char *, rcuname) __field(void *, rhp) __field(void *, func) - __field(long, qlen_lazy) __field(long, qlen) ),
@@ -498,13 +496,12 @@ TRACE_EVENT_RCU(rcu_callback, __entry->rcuname = rcuname; __entry->rhp = rhp; __entry->func = rhp->func; - __entry->qlen_lazy = qlen_lazy; __entry->qlen = qlen; ), -TP_printk ("%s rhp=%p func=%ps %ld/%ld" , +TP_printk ("%s rhp=%p func=%ps %ld" , __entry->rcuname, __entry->rhp, __entry->func, - __entry->qlen_lazy, __entry->qlen) + __entry->qlen) );/*
@@ -518,15 +515,14 @@ TRACE_EVENT_RCU(rcu_callback, TRACE_EVENT_RCU(rcu_kfree_callback, TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset, - long qlen_lazy, long qlen), + long qlen), - TP_ARGS(rcuname, rhp, offset, qlen_lazy, qlen), + TP_ARGS(rcuname, rhp, offset, qlen), TP_STRUCT__entry( __field(const char *, rcuname) __field(void *, rhp) __field(unsigned long, offset) - __field(long, qlen_lazy) __field(long, qlen) ),
@@ -534,13 +530,12 @@ TRACE_EVENT_RCU(rcu_kfree_callback, __entry->rcuname = rcuname; __entry->rhp = rhp; __entry->offset = offset; - __entry->qlen_lazy = qlen_lazy; __entry->qlen = qlen; ), -TP_printk ("%s rhp=%p func=%ld %ld/%ld" , +TP_printk ("%s rhp=%p func=%ld %ld" , __entry->rcuname, __entry->rhp, __entry->offset, - __entry->qlen_lazy, __entry->qlen) + __entry->qlen) );/*
@@ -552,27 +547,24 @@ TRACE_EVENT_RCU(rcu_kfree_callback,*/ TRACE_EVENT_RCU (rcu_batch_start, - TP_PROTO(const char *rcuname, long qlen_lazy, long qlen, long blimit), + TP_PROTO(const char *rcuname, long qlen, long blimit), -TP_ARGS (rcuname, qlen_lazy, qlen, blimit), +TP_ARGS (rcuname, qlen, blimit),TP_STRUCT__entry ( __field(const char *, rcuname) - __field(long, qlen_lazy) __field(long, qlen)__field (long, blimit) ),TP_fast_assign ( __entry->rcuname = rcuname; - __entry->qlen_lazy = qlen_lazy; __entry->qlen = qlen; __entry->blimit = blimit; ), -TP_printk ("%s CBs=%ld/%ld bl=%ld" , - __entry->rcuname, __entry->qlen_lazy, __entry->qlen, - __entry->blimit) +TP_printk ("%s CBs=%ld bl=%ld" , + __entry->rcuname, __entry->qlen, __entry->blimit) );/*
kernel/rcu/rcu.h +0 -27
@@ -198,33 +198,6 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head) }#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */ -void kfree (const void *); - -/* -* Reclaim the specified callback, either by invoking it (non-lazy case) -* or freeing it directly (lazy case). Return true if lazy, false otherwise. -*/ -static inline bool __rcu_reclaim (const char *rn,struct rcu_head *head) -{ -rcu_callback_t f; -unsigned long offset = (unsigned long )head->func; - -rcu_lock_acquire (&rcu_callback_map); -if (__is_kfree_rcu_offset (offset)) { -trace_rcu_invoke_kfree_callback (rn, head, offset); -kfree ((void *)head - offset); -rcu_lock_release (&rcu_callback_map); -return true; - }else { -trace_rcu_invoke_callback (rn, head); - f = head->func; -WRITE_ONCE (head->func, (rcu_callback_t )0L ); -f (head); -rcu_lock_release (&rcu_callback_map); -return false; - } -} -#ifdef CONFIG_RCU_STALL_COMMON extern int rcu_cpu_stall_ftrace_dump;
kernel/rcu/rcu_segcblist.c +3 -22
@@ -20,14 +20,10 @@ void rcu_cblist_init(struct rcu_cblist *rclp) rclp->head =NULL ; rclp->tail = &rclp->head; rclp->len =0 ; - rclp->len_lazy =0 ; }/* * Enqueue an rcu_head structure onto the specified callback list. -* This function assumes that the callback is non-lazy because it -* is intended for use by no-CBs CPUs, which do not distinguish -* between lazy and non-lazy RCU callbacks. */ void rcu_cblist_enqueue (struct rcu_cblist *rclp,struct rcu_head *rhp) {
@@ -54,7 +50,6 @@ void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,else drclp->tail = &drclp->head; drclp->len = srclp->len; - drclp->len_lazy = srclp->len_lazy;if (!rhp) {rcu_cblist_init (srclp); }else {
@@ -62,16 +57,12 @@ void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, srclp->head = rhp; srclp->tail = &rhp->next;WRITE_ONCE (srclp->len,1 ); - srclp->len_lazy =0 ; } }/* * Dequeue the oldest rcu_head structure from the specified callback -* list. This function assumes that the callback is non-lazy, but -* the caller can later invoke rcu_cblist_dequeued_lazy() if it -* finds otherwise (and if it cares about laziness). This allows -* different users to have different ways of determining laziness. +* list. */ struct rcu_head *rcu_cblist_dequeue (struct rcu_cblist *rclp) {
@@ -161,7 +152,6 @@ void rcu_segcblist_init(struct rcu_segcblist *rsclp)for (i =0 ; i <RCU_CBLIST_NSEGS ; i++) rsclp->tails[i] = &rsclp->head;rcu_segcblist_set_len (rsclp,0 ); - rsclp->len_lazy =0 ; rsclp->enabled =1 ; }
@@ -173,7 +163,6 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp) {WARN_ON_ONCE (!rcu_segcblist_empty (rsclp));WARN_ON_ONCE (rcu_segcblist_n_cbs (rsclp)); -WARN_ON_ONCE (rcu_segcblist_n_lazy_cbs (rsclp)); rsclp->enabled =0 ; }
@@ -253,11 +242,9 @@ bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp)* absolutely not OK for it to ever miss posting a callback. */ void rcu_segcblist_enqueue (struct rcu_segcblist *rsclp, -struct rcu_head *rhp,bool lazy) +struct rcu_head *rhp) {rcu_segcblist_inc_len (rsclp); -if (lazy) - rsclp->len_lazy++;smp_mb ();/* Ensure counts are updated before callback is enqueued. */ rhp->next =NULL ;WRITE_ONCE (*rsclp->tails[RCU_NEXT_TAIL ], rhp);
@@ -275,15 +262,13 @@ void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,* period. You have been warned. */ bool rcu_segcblist_entrain (struct rcu_segcblist *rsclp, -struct rcu_head *rhp,bool lazy) +struct rcu_head *rhp) {int i;if (rcu_segcblist_n_cbs (rsclp) ==0 )return false;rcu_segcblist_inc_len (rsclp); -if (lazy) - rsclp->len_lazy++;smp_mb ();/* Ensure counts are updated before callback is entrained. */ rhp->next =NULL ;for (i =RCU_NEXT_TAIL ; i >RCU_DONE_TAIL ; i--)
@@ -307,8 +292,6 @@ bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,void rcu_segcblist_extract_count (struct rcu_segcblist *rsclp,struct rcu_cblist *rclp) { - rclp->len_lazy += rsclp->len_lazy; - rsclp->len_lazy =0 ; rclp->len =rcu_segcblist_xchg_len (rsclp,0 ); }
@@ -361,9 +344,7 @@ void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,void rcu_segcblist_insert_count (struct rcu_segcblist *rsclp,struct rcu_cblist *rclp) { - rsclp->len_lazy += rclp->len_lazy;rcu_segcblist_add_len (rsclp, rclp->len); - rclp->len_lazy =0 ; rclp->len =0 ; }
kernel/rcu/rcu_segcblist.h +2 -23
@@ -15,15 +15,6 @@ static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)return READ_ONCE (rclp->len); } -/* -* Account for the fact that a previously dequeued callback turned out -* to be marked as lazy. -*/ -static inline void rcu_cblist_dequeued_lazy (struct rcu_cblist *rclp) -{ - rclp->len_lazy--; -} -void rcu_cblist_init (struct rcu_cblist *rclp);void rcu_cblist_enqueue (struct rcu_cblist *rclp,struct rcu_head *rhp); void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
@@ -59,18 +50,6 @@ static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)#endif } -/* Return number of lazy callbacks in segmented callback list. */ -static inline long rcu_segcblist_n_lazy_cbs (struct rcu_segcblist *rsclp) -{ -return rsclp->len_lazy; -} - -/* Return number of lazy callbacks in segmented callback list. */ -static inline long rcu_segcblist_n_nonlazy_cbs (struct rcu_segcblist *rsclp) -{ -return rcu_segcblist_n_cbs (rsclp) - rsclp->len_lazy; -} -/* * Is the specified rcu_segcblist enabled, for example, not corresponding * to an offline CPU?
@@ -106,9 +85,9 @@ struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);struct rcu_head *rcu_segcblist_first_pend_cb (struct rcu_segcblist *rsclp);bool rcu_segcblist_nextgp (struct rcu_segcblist *rsclp,unsigned long *lp);void rcu_segcblist_enqueue (struct rcu_segcblist *rsclp, -struct rcu_head *rhp,bool lazy); +struct rcu_head *rhp);bool rcu_segcblist_entrain (struct rcu_segcblist *rsclp, -struct rcu_head *rhp,bool lazy); +struct rcu_head *rhp);void rcu_segcblist_extract_count (struct rcu_segcblist *rsclp,struct rcu_cblist *rclp); void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
kernel/rcu/srcutree.c +2 -2
@@ -853,7 +853,7 @@ static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,local_irq_save (flags); sdp =this_cpu_ptr (ssp->sda);spin_lock_rcu_node (sdp); -rcu_segcblist_enqueue (&sdp->srcu_cblist, rhp, false); +rcu_segcblist_enqueue (&sdp->srcu_cblist, rhp);rcu_segcblist_advance (&sdp->srcu_cblist,rcu_seq_current (&ssp->srcu_gp_seq)); s =rcu_seq_snap (&ssp->srcu_gp_seq);
@@ -1052,7 +1052,7 @@ void srcu_barrier(struct srcu_struct *ssp) sdp->srcu_barrier_head.func = srcu_barrier_cb;debug_rcu_head_queue (&sdp->srcu_barrier_head);if (!rcu_segcblist_entrain (&sdp->srcu_cblist, - &sdp->srcu_barrier_head,0 )) { + &sdp->srcu_barrier_head)) {debug_rcu_head_unqueue (&sdp->srcu_barrier_head);atomic_dec (&ssp->srcu_barrier_cpu_cnt); }
kernel/rcu/tiny.c +27 -1
@@ -22,6 +22,7 @@#include <linux/time.h> #include <linux/cpu.h> #include <linux/prefetch.h> +#include <linux/slab.h> #include "rcu.h"
@@ -73,6 +74,31 @@ void rcu_sched_clock_irq(int user) } } +/* +* Reclaim the specified callback, either by invoking it for non-kfree cases or +* freeing it directly (for kfree). Return true if kfreeing, false otherwise. +*/ +static inline bool rcu_reclaim_tiny (struct rcu_head *head) +{ +rcu_callback_t f; +unsigned long offset = (unsigned long )head->func; + +rcu_lock_acquire (&rcu_callback_map); +if (__is_kfree_rcu_offset (offset)) { +trace_rcu_invoke_kfree_callback ("" , head, offset); +kfree ((void *)head - offset); +rcu_lock_release (&rcu_callback_map); +return true; + } + +trace_rcu_invoke_callback ("" , head); + f = head->func; +WRITE_ONCE (head->func, (rcu_callback_t )0L ); +f (head); +rcu_lock_release (&rcu_callback_map); +return false; +} +/* Invoke the RCU callbacks whose grace period has elapsed. */ static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused) {
@@ -100,7 +126,7 @@ static __latent_entropy void rcu_process_callbacks(struct softirq_action *unusedprefetch (next);debug_rcu_head_unqueue (list);local_bh_disable (); -__rcu_reclaim ("" , list); +rcu_reclaim_tiny (list);local_bh_enable (); list = next; }
kernel/rcu/tree.c +28 -12
@@ -55,6 +55,7 @@#include <linux/oom.h> #include <linux/smpboot.h> #include <linux/jiffies.h> +#include <linux/slab.h> #include <linux/sched/isolation.h> #include <linux/sched/clock.h> #include "../time/tick-internal.h"
@@ -2146,7 +2147,6 @@ static void rcu_do_batch(struct rcu_data *rdp)/* If no callbacks are ready, just return. */ if (!rcu_segcblist_ready_cbs (&rdp->cblist)) {trace_rcu_batch_start (rcu_state.name, -rcu_segcblist_n_lazy_cbs (&rdp->cblist),rcu_segcblist_n_cbs (&rdp->cblist),0 ); trace_rcu_batch_end(rcu_state.name, 0, !rcu_segcblist_empty(&rdp->cblist),
@@ -2168,7 +2168,6 @@ static void rcu_do_batch(struct rcu_data *rdp)if (unlikely (bl >100 )) tlimit =local_clock () + rcu_resched_ns;trace_rcu_batch_start (rcu_state.name, -rcu_segcblist_n_lazy_cbs (&rdp->cblist),rcu_segcblist_n_cbs (&rdp->cblist), bl);rcu_segcblist_extract_done_cbs (&rdp->cblist, &rcl);if (offloaded)
@@ -2179,9 +2178,19 @@ static void rcu_do_batch(struct rcu_data *rdp)tick_dep_set_task (current,TICK_DEP_BIT_RCU ); rhp =rcu_cblist_dequeue (&rcl);for (; rhp; rhp =rcu_cblist_dequeue (&rcl)) { +rcu_callback_t f; +debug_rcu_head_unqueue (rhp); -if (__rcu_reclaim (rcu_state.name, rhp)) -rcu_cblist_dequeued_lazy (&rcl); + +rcu_lock_acquire (&rcu_callback_map); +trace_rcu_invoke_callback (rcu_state.name, rhp); + + f = rhp->func; +WRITE_ONCE (rhp->func, (rcu_callback_t )0L ); +f (rhp); + +rcu_lock_release (&rcu_callback_map); +/* * Stop only if limit reached and CPU has something to do. * Note: The rcl structure counts down from zero.
@@ -2583,7 +2592,7 @@ static void rcu_leak_callback(struct rcu_head *rhp)* is expected to specify a CPU. */ static void -__call_rcu (struct rcu_head *head,rcu_callback_t func,bool lazy) +__call_rcu (struct rcu_head *head,rcu_callback_t func) {unsigned long flags;struct rcu_data *rdp;
@@ -2618,18 +2627,17 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, bool lazy)if (rcu_segcblist_empty (&rdp->cblist))rcu_segcblist_init (&rdp->cblist); } +if (rcu_nocb_try_bypass (rdp, head, &was_alldone, flags))return ;// Enqueued onto ->nocb_bypass, so just leave. /* If we get here, rcu_nocb_try_bypass() acquired ->nocb_lock. */ -rcu_segcblist_enqueue (&rdp->cblist, head, lazy); +rcu_segcblist_enqueue (&rdp->cblist, head);if (__is_kfree_rcu_offset ((unsigned long )func))trace_rcu_kfree_callback (rcu_state.name, head, (unsigned long )func, -rcu_segcblist_n_lazy_cbs (&rdp->cblist),rcu_segcblist_n_cbs (&rdp->cblist));else trace_rcu_callback (rcu_state.name, head, -rcu_segcblist_n_lazy_cbs (&rdp->cblist),rcu_segcblist_n_cbs (&rdp->cblist));/* Go handle any RCU core processing required. */
@@ -2679,7 +2687,7 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, bool lazy)*/ void call_rcu (struct rcu_head *head,rcu_callback_t func) { -__call_rcu (head, func,0 ); +__call_rcu (head, func); }EXPORT_SYMBOL_GPL (call_rcu);
@@ -2747,10 +2755,18 @@ static void kfree_rcu_work(struct work_struct *work)// List "head" is now private, so traverse locklessly. for (; head; head = next) { +unsigned long offset = (unsigned long )head->func; + next = head->next;// Potentially optimize with kfree_bulk in future. debug_rcu_head_unqueue (head); -__rcu_reclaim (rcu_state.name, head); +rcu_lock_acquire (&rcu_callback_map); +trace_rcu_invoke_kfree_callback (rcu_state.name, head, offset); + +/* Could be possible to optimize with kfree_bulk in future */ +kfree ((void *)head - offset); + +rcu_lock_release (&rcu_callback_map);cond_resched_tasks_rcu_qs (); } }
@@ -2825,7 +2841,7 @@ static void kfree_rcu_monitor(struct work_struct *work)*/ void kfree_call_rcu_nobatch (struct rcu_head *head,rcu_callback_t func) { -__call_rcu (head, func,1 ); +__call_rcu (head, func); }EXPORT_SYMBOL_GPL (kfree_call_rcu_nobatch);
@@ -3100,7 +3116,7 @@ static void rcu_barrier_func(void *unused)debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp);WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); -if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head,0 )) { +if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count); }else {debug_rcu_head_unqueue (&rdp->barrier_head);
kernel/rcu/tree.h +0 -1
@@ -183,7 +183,6 @@ struct rcu_data {bool rcu_urgent_qs;/* GP old need light quiescent state. */ bool rcu_forced_tick;/* Forced tick to provide QS. */ #ifdef CONFIG_RCU_FAST_NO_HZ -bool all_lazy;/* All CPU's CBs lazy at idle start? */ unsigned long last_accelerate;/* Last jiffy CBs were accelerated. */ unsigned long last_advance_all;/* Last jiffy CBs were all advanced. */ int tick_nohz_enabled_snap;/* Previously seen value from sysfs. */
kernel/rcu/tree_plugin.h +11 -37
92 lines not on this page · raw message
kernel/rcu/tree_stall.h +2 -4
@@ -263,11 +263,9 @@ static void print_cpu_stall_fast_no_hz(char *cp, int cpu) {struct rcu_data *rdp = &per_cpu (rcu_data, cpu); -sprintf (cp,"last_accelerate: %04lx/%04lx, Nonlazy posted: %c%c%c" , +sprintf (cp,"last_accelerate: %04lx/%04lx dyntick_enabled: %d" , rdp->last_accelerate &0xffff , jiffies &0xffff , -".l" [rdp->all_lazy], -".L" [!rcu_segcblist_n_nonlazy_cbs (&rdp->cblist)], -".D" [!!rdp->tick_nohz_enabled_snap]); + !!rdp->tick_nohz_enabled_snap); }#else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 7/7 rcu: Remove kfree_call_rcu_nobatch()
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
Now that the kfree_rcu() special-casing has been removed from tree RCU, this commit removes kfree_call_rcu_nobatch() since it is no longer needed.
--- Documentation/admin-guide/kernel-parameters.txt | 4 ---- include/linux/rcutiny.h | 5 ----- include/linux/rcutree.h | 1 - kernel/rcu/rcuperf.c | 10 +--------- kernel/rcu/tree.c | 18 ++++-------------- 5 files changed, 5 insertions(+), 33 deletions(-)
Documentation/admin-guide/kernel-parameters.txt +0 -4
@@ -3991,10 +3991,6 @@ Number of loops doing rcuperf.kfree_alloc_num number of allocations and frees. - rcuperf.kfree_no_batch= [KNL] - Use the non-batching (less efficient) version of kfree_rcu(). - This is useful for comparing with the batched version. - rcuperf.nreaders= [KNL] Set number of RCU readers. The value -1 selects N, where N is the number of CPUs. A value
include/linux/rcutiny.h +0 -5
@@ -39,11 +39,6 @@ static inline void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)call_rcu (head, func); } -static inline void kfree_call_rcu_nobatch (struct rcu_head *head,rcu_callback_t func) -{ -call_rcu (head, func); -} -void rcu_qs (void );static inline void rcu_softirq_qs (void )
include/linux/rcutree.h +0 -1
@@ -34,7 +34,6 @@ static inline void rcu_virt_note_context_switch(int cpu)void synchronize_rcu_expedited (void );void kfree_call_rcu (struct rcu_head *head,rcu_callback_t func); -void kfree_call_rcu_nobatch (struct rcu_head *head,rcu_callback_t func);void rcu_barrier (void );bool rcu_eqs_special_set (int cpu);
kernel/rcu/rcuperf.c +1 -9
@@ -593,7 +593,6 @@ rcu_perf_shutdown(void *arg)torture_param (int, kfree_nthreads,-1 ,"Number of threads running loops of kfree_rcu()." );torture_param (int, kfree_alloc_num,8000 ,"Number of allocations and frees done in an iteration." );torture_param (int, kfree_loops,10 ,"Number of loops doing kfree_alloc_num allocations and frees." ); -torture_param (int, kfree_no_batch,0 ,"Use the non-batching (slower) version of kfree_rcu()." );static struct task_struct **kfree_reader_tasks;static int kfree_nrealthreads;
@@ -632,14 +631,7 @@ kfree_perf_thread(void *arg)if (!alloc_ptr)return -ENOMEM ; -if (!kfree_no_batch) { -kfree_rcu (alloc_ptr, rh); - }else { -rcu_callback_t cb; - - cb = (rcu_callback_t )(unsigned long )offsetof(struct kfree_obj , rh); -kfree_call_rcu_nobatch (&(alloc_ptr->rh), cb); - } +kfree_rcu (alloc_ptr, rh); }cond_resched ();
kernel/rcu/tree.c +4 -14
@@ -2763,8 +2763,10 @@ static void kfree_rcu_work(struct work_struct *work)rcu_lock_acquire (&rcu_callback_map);trace_rcu_invoke_kfree_callback (rcu_state.name, head, offset); -/* Could be possible to optimize with kfree_bulk in future */ -kfree ((void *)head - offset); +if (!WARN_ON_ONCE (!__is_kfree_rcu_offset (offset))) { +/* Could be optimized with kfree_bulk() in future. */ +kfree ((void *)head - offset); + }rcu_lock_release (&rcu_callback_map);cond_resched_tasks_rcu_qs ();
@@ -2836,16 +2838,6 @@ static void kfree_rcu_monitor(struct work_struct *work) }/* -* This version of kfree_call_rcu does not do batching of kfree_rcu() requests. -* Used only by rcuperf torture test for comparison with kfree_rcu_batch(). -*/ -void kfree_call_rcu_nobatch (struct rcu_head *head,rcu_callback_t func) -{ -__call_rcu (head, func); -} -EXPORT_SYMBOL_GPL (kfree_call_rcu_nobatch); - -/* * Queue a request for lazy invocation of kfree() after a grace period. * * Each kfree_call_rcu() request is added to a batch. The batch will be drained
@@ -2864,8 +2856,6 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)unsigned long flags;struct kfree_rcu_cpu *krcp; - head->func = func; -local_irq_save (flags);// For safely calling this_cpu_ptr(). krcp =this_cpu_ptr (&krc);if (krcp->initialized)
Signature · 1 line
2.9.5