lockdep: Add lightweight IRQs disabled/enabled check v2
Subsystems (12)
Frederic Weisbecker
maintainerPATCH 1/14 lockdep: Add IRQs disabled/enabled assertion
In reply to Frederic WeisbeckerChecking whether IRQs are enabled or disabled is a very common sanity check, however not free of overhead especially on fastpath where such assertion is very common.
Lockdep is a good host for such concurrency correctness check and it even already tracks down IRQs disablement state. Just reuse its machinery. This will allow us to get rid of the flags pop and check overhead from fast path when kernel is built for production.
--- include/linux/lockdep.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+)
include/linux/lockdep.h +15 -0
@@ -724,9 +724,24 @@ do { \lock_acquire (&(lock)->dep_map,0 ,0 ,1 ,1 ,NULL , _THIS_IP_); \lock_release (&(lock)->dep_map,0 , _THIS_IP_); \ } while (0) + +#define lockdep_assert_irqs_enabled () do { \ + WARN_ONCE(debug_locks && !current->lockdep_recursion && \ + !current->hardirqs_enabled, \ + "IRQs not enabled as expected\n"); \ + } while (0) + +#define lockdep_assert_irqs_disabled () do { \ + WARN_ONCE(debug_locks && !current->lockdep_recursion && \ + current->hardirqs_enabled, \ + "IRQs not disabled as expected\n"); \ + } while (0) +#else # define might_lock (lock) do { } while (0)# define might_lock_read (lock) do { } while (0) +# define lockdep_assert_irqs_enabled () do { } while (0) +# define lockdep_assert_irqs_disabled () do { } while (0)#endif #ifdef CONFIG_LOCKDEP
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 2/14 irq: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/softirq.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
kernel/softirq.c +5 -5
@@ -137,7 +137,7 @@ EXPORT_SYMBOL(__local_bh_disable_ip);static void __local_bh_enable (unsigned int cnt) { -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();if (softirq_count () == (cnt &SOFTIRQ_MASK ))trace_softirqs_on (_RET_IP_);
@@ -158,7 +158,8 @@ EXPORT_SYMBOL(_local_bh_enable);void __local_bh_enable_ip (unsigned long ip,unsigned int cnt) { -WARN_ON_ONCE (in_irq () ||irqs_disabled ()); +WARN_ON_ONCE (in_irq ()); +lockdep_assert_irqs_enabled ();#ifdef CONFIG_TRACE_IRQFLAGS local_irq_disable ();#endif
@@ -396,9 +397,8 @@ void irq_exit(void)#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLEDlocal_irq_disable ();#else -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();#endif -account_irq_exit_time (current);preempt_count_sub (HARDIRQ_OFFSET );if (!in_interrupt () &&local_softirq_pending ())
@@ -488,7 +488,7 @@ EXPORT_SYMBOL(__tasklet_hi_schedule);void __tasklet_hi_schedule_first (struct tasklet_struct *t) { -BUG_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); t->next =__this_cpu_read (tasklet_hi_vec.head);__this_cpu_write (tasklet_hi_vec.head, t);
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 5/14 hrtimer: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/time/hrtimer.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
kernel/time/hrtimer.c +1 -3
@@ -758,9 +758,7 @@ void clock_was_set(void)*/ void hrtimers_resume (void ) { -WARN_ONCE (!irqs_disabled (), -KERN_INFO "hrtimers_resume() called with IRQs enabled!" ); - +lockdep_assert_irqs_disabled ();/* Retrigger on the local CPU */ retrigger_next_event (NULL );/* And schedule a retrigger for all others */
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 4/14 nohz: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/time/tick-sched.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
kernel/time/tick-sched.c +2 -3
@@ -198,7 +198,7 @@ static bool check_tick_dependency(atomic_t *dep)static bool can_stop_full_tick (int cpu,struct tick_sched *ts) { -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();if (unlikely (!cpu_online (cpu)))return false;
@@ -960,8 +960,7 @@ void tick_nohz_idle_enter(void) {struct tick_sched *ts; -WARN_ON_ONCE (irqs_disabled ()); - +lockdep_assert_irqs_enabled ();/* * Update the idle state in the scheduler domain hierarchy * when tick_nohz_stop_sched_tick() is called from the idle loop.
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 7/14 x86: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
It also makes no more sense to fix the IRQ flags when a bug is detected as the assertion is now pure config-dependent debugging. And to quote Peter Zijlstra:
The whole if !disabled, disable logic is uber paranoid programming, but I don't think we've ever seen that WARN trigger, and if it does (and then burns the kernel) we at least know what happend.
--- arch/x86/entry/common.c | 4 +--- arch/x86/kernel/smpboot.c | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-)
arch/x86/entry/common.c +1 -3
@@ -186,9 +186,7 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)addr_limit_user_check (); -if (IS_ENABLED (CONFIG_PROVE_LOCKING ) &&WARN_ON (!irqs_disabled ())) -local_irq_disable (); - +lockdep_assert_irqs_disabled ();lockdep_sys_exit (); cached_flags =READ_ONCE (ti->flags);
arch/x86/kernel/smpboot.c +1 -1
@@ -1095,7 +1095,7 @@ int native_cpu_up(unsigned int cpu, struct task_struct *tidle)unsigned long flags;int err, ret =0 ; -WARN_ON (irqs_disabled ()); +lockdep_assert_irqs_enabled ();pr_debug ("++++++++++++++++++++=_---CPU UP %u\n" , cpu);
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 9/14 irq/timings: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/irq/timings.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/irq/timings.c +1 -1
@@ -264,7 +264,7 @@ u64 irq_timings_next_event(u64 now)* order to prevent the timings circular buffer to be updated * while we are reading it. */ -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();/* * Number of elements in the circular buffer: If it happens it
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 10/14 irq_work: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/irq_work.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/irq_work.c +1 -1
@@ -188,7 +188,7 @@ void irq_work_tick(void)*/ void irq_work_sync (struct irq_work *work) { -WARN_ON_ONCE (irqs_disabled ()); +lockdep_assert_irqs_enabled ();while (work->flags &IRQ_WORK_BUSY )cpu_relax ();
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 11/14 sched: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/sched/clock.c | 2 +- kernel/sched/cputime.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-)
kernel/sched/clock.c +1 -1
@@ -388,7 +388,7 @@ void sched_clock_tick(void)if (unlikely (!sched_clock_running))return ; -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); scd =this_scd ();__scd_stamp (scd);
kernel/sched/cputime.c +1 -2
@@ -259,8 +259,7 @@ static inline u64 account_other_time(u64 max) {u64 accounted; -/* Shall be converted to a lockdep-enabled lightweight check */ -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); accounted =steal_account_process_time (max);
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 13/14 netpoll: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- net/core/netpoll.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
net/core/netpoll.c +1 -1
@@ -334,7 +334,7 @@ void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb,/* It is up to the caller to keep npinfo alive. */ struct netpoll_info *npinfo; -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); npinfo =rcu_dereference_bh (np->dev->npinfo);if (!npinfo || !netif_running (dev) || !netif_device_present (dev)) {
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 14/14 rcu: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerLockdep now has an integrated IRQs disabled/enabled sanity check. Just use it instead of the ad-hoc RCU version.
--- kernel/rcu/tree.c | 16 ++++++++-------- kernel/rcu/tree_plugin.h | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-)
kernel/rcu/tree.c +8 -8
@@ -734,7 +734,7 @@ static int rcu_future_needs_gp(struct rcu_state *rsp)int idx = (READ_ONCE (rnp->completed) +1 ) &0x1 ;int *fp = &rnp->need_future_gp[idx]; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_future_needs_gp() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();return READ_ONCE (*fp); }
@@ -746,7 +746,7 @@ static int rcu_future_needs_gp(struct rcu_state *rsp)static bool cpu_needs_another_gp (struct rcu_state *rsp,struct rcu_data *rdp) { -RCU_LOCKDEP_WARN (!irqs_disabled (),"cpu_needs_another_gp() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();if (rcu_gp_in_progress (rsp))return false;/* No, a grace period is already in progress. */ if (rcu_future_needs_gp (rsp))
@@ -773,7 +773,7 @@ static void rcu_eqs_enter_common(bool user)struct rcu_data *rdp;struct rcu_dynticks *rdtp =this_cpu_ptr (&rcu_dynticks); -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_eqs_enter_common() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();trace_rcu_dyntick (TPS ("Start" ), rdtp->dynticks_nesting,0 );if (IS_ENABLED (CONFIG_RCU_EQS_DEBUG ) && !user && !is_idle_task (current)) {
@@ -840,7 +840,7 @@ static void rcu_eqs_enter(bool user)*/ void rcu_idle_enter (void ) { -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_idle_enter() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();rcu_eqs_enter (false); }
@@ -855,7 +855,7 @@ void rcu_idle_enter(void)*/ void rcu_user_enter (void ) { -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_user_enter() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();rcu_eqs_enter (true); }#endif /* CONFIG_NO_HZ_FULL */
@@ -880,7 +880,7 @@ void rcu_irq_exit(void) {struct rcu_dynticks *rdtp; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_irq_exit() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled (); rdtp =this_cpu_ptr (&rcu_dynticks);/* Page faults can happen in NMI handlers, so check... */
@@ -947,7 +947,7 @@ static void rcu_eqs_exit(bool user)struct rcu_dynticks *rdtp;long long oldval; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_eqs_exit() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled (); rdtp =this_cpu_ptr (&rcu_dynticks); oldval = rdtp->dynticks_nesting;WARN_ON_ONCE (IS_ENABLED (CONFIG_RCU_EQS_DEBUG ) && oldval <0 );
@@ -1018,7 +1018,7 @@ void rcu_irq_enter(void)struct rcu_dynticks *rdtp;long long oldval; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_irq_enter() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled (); rdtp =this_cpu_ptr (&rcu_dynticks);/* Page faults can happen in NMI handlers, so check... */
kernel/rcu/tree_plugin.h +5 -5
@@ -325,7 +325,7 @@ static void rcu_preempt_note_context_switch(bool preempt)struct rcu_data *rdp;struct rcu_node *rnp; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_preempt_note_context_switch() invoked with interrupts enabled!!!\n" ); +lockdep_assert_irqs_disabled ();WARN_ON_ONCE (!preempt && t->rcu_read_lock_nesting >0 );if (t->rcu_read_lock_nesting >0 && !t->rcu_read_unlock_special.b.blocked) {
@@ -1421,7 +1421,7 @@ int rcu_needs_cpu(u64 basemono, u64 *nextevt)struct rcu_dynticks *rdtp =this_cpu_ptr (&rcu_dynticks);unsigned long dj; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_needs_cpu() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();/* Snapshot to detect later posting of non-lazy callback. */ rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
@@ -1470,7 +1470,7 @@ static void rcu_prepare_for_idle(void)struct rcu_state *rsp;int tne; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_prepare_for_idle() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();if (rcu_is_nocb_cpu (smp_processor_id ()))return ;
@@ -1525,7 +1525,7 @@ static void rcu_prepare_for_idle(void)*/ static void rcu_cleanup_after_idle (void ) { -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_cleanup_after_idle() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();if (rcu_is_nocb_cpu (smp_processor_id ()))return ;if (rcu_try_advance_all_cbs ())
@@ -2012,7 +2012,7 @@ static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp,struct rcu_data *rdp, unsigned long flags) { - RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_nocb_adopt_orphan_cbs() invoked with irqs enabled!!!"); +lockdep_assert_irqs_disabled ();if (!rcu_is_nocb_cpu (smp_processor_id ()))return false;/* Not NOCBs CPU, caller must migrate CBs. */ __call_rcu_nocb_enqueue(my_rdp, rcu_segcblist_head(&rdp->cblist),
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 12/14 posix-cpu-timers: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/time/posix-cpu-timers.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
kernel/time/posix-cpu-timers.c +3 -3
@@ -602,7 +602,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,/* * Disarm any old timer after extracting its expiry time. */ -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); ret =0 ; old_incr = timer->it.cpu.incr;
@@ -1033,7 +1033,7 @@ static void posix_cpu_timer_rearm(struct k_itimer *timer)/* * Now re-arm for the new expiry time. */ -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();arm_timer (timer); unlock:unlock_task_sighand (p, &flags);
@@ -1124,7 +1124,7 @@ void run_posix_cpu_timers(struct task_struct *tsk)struct k_itimer *timer, *next;unsigned long flags; -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();/* * The fast path checks that there are no expired thread or thread
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 8/14 perf: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/events/core.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)
kernel/events/core.c +6 -6
@@ -209,7 +209,7 @@ static int event_function(void *info)struct perf_event_context *task_ctx = cpuctx->task_ctx;int ret =0 ; -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();perf_ctx_lock (cpuctx, task_ctx);/*
@@ -306,7 +306,7 @@ static void event_function_local(struct perf_event *event, event_f func, void *dstruct task_struct *task =READ_ONCE (ctx->task);struct perf_event_context *task_ctx =NULL ; -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();if (task) {if (task ==TASK_TOMBSTONE )
@@ -1004,7 +1004,7 @@ static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)struct perf_cpu_context *cpuctx;int rotations =0 ; -WARN_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); cpuctx = container_of(hr, struct perf_cpu_context, hrtimer); rotations =perf_rotate_context (cpuctx);
@@ -1091,7 +1091,7 @@ static void perf_event_ctx_activate(struct perf_event_context *ctx) {struct list_head *head =this_cpu_ptr (&active_ctx_list); -WARN_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();WARN_ON (!list_empty (&ctx->active_ctx_list));
@@ -1100,7 +1100,7 @@ static void perf_event_ctx_activate(struct perf_event_context *ctx)static void perf_event_ctx_deactivate (struct perf_event_context *ctx) { -WARN_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();WARN_ON (list_empty (&ctx->active_ctx_list));
@@ -3521,7 +3521,7 @@ void perf_event_task_tick(void)struct perf_event_context *ctx, *tmp;int throttled; -WARN_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();__this_cpu_inc (perf_throttled_seq); throttled =__this_cpu_xchg (perf_throttled_count,0 );
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 6/14 smp: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/smp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/smp.c +1 -1
@@ -213,7 +213,7 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline)call_single_data_t *csd, *csd_next;static bool warned; -WARN_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); head =this_cpu_ptr (&call_single_queue); entry =llist_del_all (head);
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerPATCH 3/14 workqueue: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic WeisbeckerUse lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/workqueue.c +1 -1
@@ -1375,7 +1375,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,* queued or lose PENDING. Grabbing PENDING and queueing should * happen with IRQ disabled. */ -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();debug_work_activate (work);
Signature · 1 line
2.7.4
Frederic Weisbecker
maintainerIngo,
Please pull the lockdep/irq-v2 branch that can be found at:
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
lockdep/irq-v2
HEAD: c741fbe70afb0e488f4356456e6397d205f115ce
--
Changes on v2:
* Fix spurious warnings when lockdep is temporarily disabled, like NTFS does while mounting superblock (reported by Wu Fengguang).
* Remove the IRQ flags auto-correction which doesn't make anymore sense now that the assertion is config-dependent. (Suggested by Peter Zijlstra).
--
Summary:
Checking whether IRQs are disabled or enabled is often an important sanity check. Now we can be sometimes reluctant to have it on fastpath because reading the flags can be somewhat costly.
As suggested by Peter Zijlstra some time ago, this would be a nice lockdep extension: it's the natural concurrency sanity checker and it already tracks down the irqflags state.
Thanks,
Frederic
---
Frederic Weisbecker (14):
lockdep: Add IRQs disabled/enabled assertion
irq: Use lockdep to assert IRQs are disabled/enabled
workqueue: Use lockdep to assert IRQs are disabled/enabled
nohz: Use lockdep to assert IRQs are disabled/enabled
hrtimer: Use lockdep to assert IRQs are disabled/enabled
smp: Use lockdep to assert IRQs are disabled/enabled
x86: Use lockdep to assert IRQs are disabled/enabled
perf: Use lockdep to assert IRQs are disabled/enabled irq/timings: Use lockdep to assert IRQs are disabled/enabled
irq_work: Use lockdep to assert IRQs are disabled/enabled
sched: Use lockdep to assert IRQs are disabled/enabled
posix-cpu-timers: Use lockdep to assert IRQs are disabled/enabled
netpoll: Use lockdep to assert IRQs are disabled/enabled
rcu: Use lockdep to assert IRQs are disabled/enabled
arch/x86/entry/common.c | 4 +--- arch/x86/kernel/smpboot.c | 2 +- include/linux/lockdep.h | 15 +++++++++++++++ kernel/events/core.c | 12 ++++++------ kernel/irq/timings.c | 2 +- kernel/irq_work.c | 2 +- kernel/rcu/tree.c | 16 ++++++++-------- kernel/rcu/tree_plugin.h | 10 +++++----- kernel/sched/clock.c | 2 +- kernel/sched/cputime.c | 3 +-- kernel/smp.c | 2 +- kernel/softirq.c | 10 +++++----- kernel/time/hrtimer.c | 4 +--- kernel/time/posix-cpu-timers.c | 6 +++--- kernel/time/tick-sched.c | 5 ++--- kernel/workqueue.c | 2 +- net/core/netpoll.c | 2 +- 17 files changed, 54 insertions(+), 45 deletions(-)
Thomas Gleixner
Acked-byQuoted 6 lines · Frederic Weisbecker
Ingo,
Please pull the lockdep/irq-v2 branch that can be found at:
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
lockdep/irq-v2
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] locking/lockdep: Add IRQs disabled/enabled assertion APIs: lockdep_assert_irqs_enabled()/disabled()
In reply to Frederic Weisbeckerlocking/lockdep: Add IRQs disabled/enabled assertion APIs: lockdep_assert_irqs_enabled()/disabled()
Commit-ID: f54bb2ec02c839f6bfe3e8d438cd93d30b4809dd Gitweb: https://git.kernel.org/tip/f54bb2ec02c839f6bfe3e8d438cd93d30b4809dd Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:17 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:47 +0100
locking/lockdep: Add IRQs disabled/enabled assertion APIs: lockdep_assert_irqs_enabled()/disabled()
Checking whether IRQs are enabled or disabled is a very common sanity check, however not free of overhead especially on fastpath where such assertion is very common.
Lockdep is a good host for such concurrency correctness check and it even already tracks down IRQs disablement state. Just reuse its machinery. This will allow us to get rid of the flags pop and check overhead from fast path when kernel is built for production.
--- include/linux/lockdep.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+)
include/linux/lockdep.h +15 -0
@@ -725,9 +725,24 @@ do { \lock_acquire (&(lock)->dep_map,0 ,0 ,1 ,1 ,NULL , _THIS_IP_); \lock_release (&(lock)->dep_map,0 , _THIS_IP_); \ } while (0) + +#define lockdep_assert_irqs_enabled () do { \ + WARN_ONCE(debug_locks && !current->lockdep_recursion && \ + !current->hardirqs_enabled, \ + "IRQs not enabled as expected\n"); \ + } while (0) + +#define lockdep_assert_irqs_disabled () do { \ + WARN_ONCE(debug_locks && !current->lockdep_recursion && \ + current->hardirqs_enabled, \ + "IRQs not disabled as expected\n"); \ + } while (0) +#else # define might_lock (lock) do { } while (0)# define might_lock_read (lock) do { } while (0) +# define lockdep_assert_irqs_enabled () do { } while (0) +# define lockdep_assert_irqs_disabled () do { } while (0)#endif #ifdef CONFIG_LOCKDEP
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] irq/softirqs: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckerirq/softirqs: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: f71b74bca637fca7de78f1d44eaefde5bc900f9f Gitweb: https://git.kernel.org/tip/f71b74bca637fca7de78f1d44eaefde5bc900f9f Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:18 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:48 +0100
irq/softirqs: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/softirq.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
kernel/softirq.c +5 -5
@@ -137,7 +137,7 @@ EXPORT_SYMBOL(__local_bh_disable_ip);static void __local_bh_enable (unsigned int cnt) { -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();if (softirq_count () == (cnt &SOFTIRQ_MASK ))trace_softirqs_on (_RET_IP_);
@@ -158,7 +158,8 @@ EXPORT_SYMBOL(_local_bh_enable);void __local_bh_enable_ip (unsigned long ip,unsigned int cnt) { -WARN_ON_ONCE (in_irq () ||irqs_disabled ()); +WARN_ON_ONCE (in_irq ()); +lockdep_assert_irqs_enabled ();#ifdef CONFIG_TRACE_IRQFLAGS local_irq_disable ();#endif
@@ -396,9 +397,8 @@ void irq_exit(void)#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLEDlocal_irq_disable ();#else -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();#endif -account_irq_exit_time (current);preempt_count_sub (HARDIRQ_OFFSET );if (!in_interrupt () &&local_softirq_pending ())
@@ -488,7 +488,7 @@ EXPORT_SYMBOL(__tasklet_hi_schedule);void __tasklet_hi_schedule_first (struct tasklet_struct *t) { -BUG_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); t->next =__this_cpu_read (tasklet_hi_vec.head);__this_cpu_write (tasklet_hi_vec.head, t);
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] workqueue: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckerworkqueue: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: 8e8eb730759f9cfd7a761b0b4ee41d714e720993 Gitweb: https://git.kernel.org/tip/8e8eb730759f9cfd7a761b0b4ee41d714e720993 Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:19 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:48 +0100
workqueue: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/workqueue.c +1 -1
@@ -1376,7 +1376,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,* queued or lose PENDING. Grabbing PENDING and queueing should * happen with IRQ disabled. */ -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();debug_work_activate (work);
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] timers/nohz: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckertimers/nohz: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: ebf3adbad012b89c4a51a3beae718a587d988a3a Gitweb: https://git.kernel.org/tip/ebf3adbad012b89c4a51a3beae718a587d988a3a Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:20 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:49 +0100
timers/nohz: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/time/tick-sched.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
kernel/time/tick-sched.c +2 -3
@@ -198,7 +198,7 @@ static bool check_tick_dependency(atomic_t *dep)static bool can_stop_full_tick (int cpu,struct tick_sched *ts) { -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();if (unlikely (!cpu_online (cpu)))return false;
@@ -960,8 +960,7 @@ void tick_nohz_idle_enter(void) {struct tick_sched *ts; -WARN_ON_ONCE (irqs_disabled ()); - +lockdep_assert_irqs_enabled ();/* * Update the idle state in the scheduler domain hierarchy * when tick_nohz_stop_sched_tick() is called from the idle loop.
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] timers/hrtimer: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckertimers/hrtimer: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: 53bef3fd47f69e40b52c9f9acd3551dfff9f8702 Gitweb: https://git.kernel.org/tip/53bef3fd47f69e40b52c9f9acd3551dfff9f8702 Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:21 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:49 +0100
timers/hrtimer: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/time/hrtimer.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
kernel/time/hrtimer.c +1 -3
@@ -758,9 +758,7 @@ void clock_was_set(void)*/ void hrtimers_resume (void ) { -WARN_ONCE (!irqs_disabled (), -KERN_INFO "hrtimers_resume() called with IRQs enabled!" ); - +lockdep_assert_irqs_disabled ();/* Retrigger on the local CPU */ retrigger_next_event (NULL );/* And schedule a retrigger for all others */
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] smp/core: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckersmp/core: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: 83efcbd028ad3aec36b5a3882cfa32490c135df7 Gitweb: https://git.kernel.org/tip/83efcbd028ad3aec36b5a3882cfa32490c135df7 Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:22 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:50 +0100
smp/core: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/smp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/smp.c +1 -1
@@ -213,7 +213,7 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline)call_single_data_t *csd, *csd_next;static bool warned; -WARN_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); head =this_cpu_ptr (&call_single_queue); entry =llist_del_all (head);
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] x86: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckerx86: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: 7a10e2a9190628a4024ea394ce7bd641ae40ffd1 Gitweb: https://git.kernel.org/tip/7a10e2a9190628a4024ea394ce7bd641ae40ffd1 Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:23 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:50 +0100
x86: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
It also makes no more sense to fix the IRQ flags when a bug is detected as the assertion is now pure config-dependent debugging. And to quote Peter Zijlstra:
The whole if !disabled, disable logic is uber paranoid programming, but I don't think we've ever seen that WARN trigger, and if it does (and then burns the kernel) we at least know what happend.
--- arch/x86/entry/common.c | 4 +--- arch/x86/kernel/smpboot.c | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-)
arch/x86/entry/common.c +1 -3
@@ -186,9 +186,7 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)addr_limit_user_check (); -if (IS_ENABLED (CONFIG_PROVE_LOCKING ) &&WARN_ON (!irqs_disabled ())) -local_irq_disable (); - +lockdep_assert_irqs_disabled ();lockdep_sys_exit (); cached_flags =READ_ONCE (ti->flags);
arch/x86/kernel/smpboot.c +1 -1
@@ -1095,7 +1095,7 @@ int native_cpu_up(unsigned int cpu, struct task_struct *tidle)unsigned long flags;int err, ret =0 ; -WARN_ON (irqs_disabled ()); +lockdep_assert_irqs_enabled ();pr_debug ("++++++++++++++++++++=_---CPU UP %u\n" , cpu);
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] perf/core: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckerperf/core: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: 164446455a5d3f1402b5a0ea42acce33fd576ed7 Gitweb: https://git.kernel.org/tip/164446455a5d3f1402b5a0ea42acce33fd576ed7 Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:24 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:51 +0100
perf/core: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/events/core.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)
kernel/events/core.c +6 -6
@@ -209,7 +209,7 @@ static int event_function(void *info)struct perf_event_context *task_ctx = cpuctx->task_ctx;int ret =0 ; -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();perf_ctx_lock (cpuctx, task_ctx);/*
@@ -306,7 +306,7 @@ static void event_function_local(struct perf_event *event, event_f func, void *dstruct task_struct *task =READ_ONCE (ctx->task);struct perf_event_context *task_ctx =NULL ; -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();if (task) {if (task ==TASK_TOMBSTONE )
@@ -1006,7 +1006,7 @@ static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)struct perf_cpu_context *cpuctx;int rotations =0 ; -WARN_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); cpuctx = container_of(hr, struct perf_cpu_context, hrtimer); rotations =perf_rotate_context (cpuctx);
@@ -1093,7 +1093,7 @@ static void perf_event_ctx_activate(struct perf_event_context *ctx) {struct list_head *head =this_cpu_ptr (&active_ctx_list); -WARN_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();WARN_ON (!list_empty (&ctx->active_ctx_list));
@@ -1102,7 +1102,7 @@ static void perf_event_ctx_activate(struct perf_event_context *ctx)static void perf_event_ctx_deactivate (struct perf_event_context *ctx) { -WARN_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();WARN_ON (list_empty (&ctx->active_ctx_list));
@@ -3523,7 +3523,7 @@ void perf_event_task_tick(void)struct perf_event_context *ctx, *tmp;int throttled; -WARN_ON (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();__this_cpu_inc (perf_throttled_seq); throttled =__this_cpu_xchg (perf_throttled_count,0 );
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] irq/timings: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckerirq/timings: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: a934d4d15f040cdd254350e7270b35cc7521e12e Gitweb: https://git.kernel.org/tip/a934d4d15f040cdd254350e7270b35cc7521e12e Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:25 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:52 +0100
irq/timings: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/irq/timings.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/irq/timings.c +1 -1
@@ -264,7 +264,7 @@ u64 irq_timings_next_event(u64 now)* order to prevent the timings circular buffer to be updated * while we are reading it. */ -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();/* * Number of elements in the circular buffer: If it happens it
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] irq_work: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckerirq_work: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: 3c7169a3bf8216a56761a8edf775072dd36a00a0 Gitweb: https://git.kernel.org/tip/3c7169a3bf8216a56761a8edf775072dd36a00a0 Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:26 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:52 +0100
irq_work: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/irq_work.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/irq_work.c +1 -1
@@ -188,7 +188,7 @@ void irq_work_tick(void)*/ void irq_work_sync (struct irq_work *work) { -WARN_ON_ONCE (irqs_disabled ()); +lockdep_assert_irqs_enabled ();while (work->flags &IRQ_WORK_BUSY )cpu_relax ();
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] sched/clock, sched/cputime: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckersched/clock, sched/cputime: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: 2c11dba00a39007b457c7607c1b1a4db95ca04bc Gitweb: https://git.kernel.org/tip/2c11dba00a39007b457c7607c1b1a4db95ca04bc Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:27 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:53 +0100
sched/clock, sched/cputime: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/sched/clock.c | 2 +- kernel/sched/cputime.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-)
kernel/sched/clock.c +1 -1
@@ -388,7 +388,7 @@ void sched_clock_tick(void)if (unlikely (!sched_clock_running))return ; -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); scd =this_scd ();__scd_stamp (scd);
kernel/sched/cputime.c +1 -2
@@ -259,8 +259,7 @@ static inline u64 account_other_time(u64 max) {u64 accounted; -/* Shall be converted to a lockdep-enabled lightweight check */ -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); accounted =steal_account_process_time (max);
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] timers/posix-cpu-timers: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckertimers/posix-cpu-timers: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: a69682200db9c2c26594188f81dd2df560af4683 Gitweb: https://git.kernel.org/tip/a69682200db9c2c26594188f81dd2df560af4683 Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:28 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:54 +0100
timers/posix-cpu-timers: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- kernel/time/posix-cpu-timers.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
kernel/time/posix-cpu-timers.c +3 -3
@@ -603,7 +603,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,/* * Disarm any old timer after extracting its expiry time. */ -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); ret =0 ; old_incr = timer->it.cpu.incr;
@@ -1034,7 +1034,7 @@ static void posix_cpu_timer_rearm(struct k_itimer *timer)/* * Now re-arm for the new expiry time. */ -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();arm_timer (timer); unlock:unlock_task_sighand (p, &flags);
@@ -1125,7 +1125,7 @@ void run_posix_cpu_timers(struct task_struct *tsk)struct k_itimer *timer, *next;unsigned long flags; -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled ();/* * The fast path checks that there are no expired thread or thread
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] netpoll: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckernetpoll: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: af0733937317e1e03b60f3af8cf9cd59d665593c Gitweb: https://git.kernel.org/tip/af0733937317e1e03b60f3af8cf9cd59d665593c Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:29 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:54 +0100
netpoll: Use lockdep to assert IRQs are disabled/enabled
Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled.
--- net/core/netpoll.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
net/core/netpoll.c +1 -1
@@ -334,7 +334,7 @@ void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb,/* It is up to the caller to keep npinfo alive. */ struct netpoll_info *npinfo; -WARN_ON_ONCE (!irqs_disabled ()); +lockdep_assert_irqs_disabled (); npinfo =rcu_dereference_bh (np->dev->npinfo);if (!npinfo || !netif_running (dev) || !netif_device_present (dev)) {
tip-bot for Frederic Weisbecker
merge bot[tip:locking/core] rcu: Use lockdep to assert IRQs are disabled/enabled
In reply to Frederic Weisbeckerrcu: Use lockdep to assert IRQs are disabled/enabled
Commit-ID: b04db8e19fc2e9131524dec43057c1b96d5ba3ba Gitweb: https://git.kernel.org/tip/b04db8e19fc2e9131524dec43057c1b96d5ba3ba Author: Frederic Weisbecker <[email protected]> AuthorDate: Mon, 6 Nov 2017 16:01:30 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 8 Nov 2017 11:13:55 +0100
rcu: Use lockdep to assert IRQs are disabled/enabled
Lockdep now has an integrated IRQs disabled/enabled sanity check. Just use it instead of the ad-hoc RCU version.
--- kernel/rcu/tree.c | 16 ++++++++-------- kernel/rcu/tree_plugin.h | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-)
kernel/rcu/tree.c +8 -8
@@ -734,7 +734,7 @@ static int rcu_future_needs_gp(struct rcu_state *rsp)int idx = (READ_ONCE (rnp->completed) +1 ) &0x1 ;int *fp = &rnp->need_future_gp[idx]; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_future_needs_gp() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();return READ_ONCE (*fp); }
@@ -746,7 +746,7 @@ static int rcu_future_needs_gp(struct rcu_state *rsp)static bool cpu_needs_another_gp (struct rcu_state *rsp,struct rcu_data *rdp) { -RCU_LOCKDEP_WARN (!irqs_disabled (),"cpu_needs_another_gp() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();if (rcu_gp_in_progress (rsp))return false;/* No, a grace period is already in progress. */ if (rcu_future_needs_gp (rsp))
@@ -773,7 +773,7 @@ static void rcu_eqs_enter_common(bool user)struct rcu_data *rdp;struct rcu_dynticks *rdtp =this_cpu_ptr (&rcu_dynticks); -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_eqs_enter_common() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();trace_rcu_dyntick (TPS ("Start" ), rdtp->dynticks_nesting,0 );if (IS_ENABLED (CONFIG_RCU_EQS_DEBUG ) && !user && !is_idle_task (current)) {
@@ -840,7 +840,7 @@ static void rcu_eqs_enter(bool user)*/ void rcu_idle_enter (void ) { -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_idle_enter() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();rcu_eqs_enter (false); }
@@ -855,7 +855,7 @@ void rcu_idle_enter(void)*/ void rcu_user_enter (void ) { -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_user_enter() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();rcu_eqs_enter (true); }#endif /* CONFIG_NO_HZ_FULL */
@@ -880,7 +880,7 @@ void rcu_irq_exit(void) {struct rcu_dynticks *rdtp; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_irq_exit() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled (); rdtp =this_cpu_ptr (&rcu_dynticks);/* Page faults can happen in NMI handlers, so check... */
@@ -947,7 +947,7 @@ static void rcu_eqs_exit(bool user)struct rcu_dynticks *rdtp;long long oldval; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_eqs_exit() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled (); rdtp =this_cpu_ptr (&rcu_dynticks); oldval = rdtp->dynticks_nesting;WARN_ON_ONCE (IS_ENABLED (CONFIG_RCU_EQS_DEBUG ) && oldval <0 );
@@ -1018,7 +1018,7 @@ void rcu_irq_enter(void)struct rcu_dynticks *rdtp;long long oldval; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_irq_enter() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled (); rdtp =this_cpu_ptr (&rcu_dynticks);/* Page faults can happen in NMI handlers, so check... */
kernel/rcu/tree_plugin.h +5 -5
@@ -325,7 +325,7 @@ static void rcu_preempt_note_context_switch(bool preempt)struct rcu_data *rdp;struct rcu_node *rnp; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_preempt_note_context_switch() invoked with interrupts enabled!!!\n" ); +lockdep_assert_irqs_disabled ();WARN_ON_ONCE (!preempt && t->rcu_read_lock_nesting >0 );if (t->rcu_read_lock_nesting >0 && !t->rcu_read_unlock_special.b.blocked) {
@@ -1421,7 +1421,7 @@ int rcu_needs_cpu(u64 basemono, u64 *nextevt)struct rcu_dynticks *rdtp =this_cpu_ptr (&rcu_dynticks);unsigned long dj; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_needs_cpu() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();/* Snapshot to detect later posting of non-lazy callback. */ rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
@@ -1470,7 +1470,7 @@ static void rcu_prepare_for_idle(void)struct rcu_state *rsp;int tne; -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_prepare_for_idle() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();if (rcu_is_nocb_cpu (smp_processor_id ()))return ;
@@ -1525,7 +1525,7 @@ static void rcu_prepare_for_idle(void)*/ static void rcu_cleanup_after_idle (void ) { -RCU_LOCKDEP_WARN (!irqs_disabled (),"rcu_cleanup_after_idle() invoked with irqs enabled!!!" ); +lockdep_assert_irqs_disabled ();if (rcu_is_nocb_cpu (smp_processor_id ()))return ;if (rcu_try_advance_all_cbs ())
@@ -2012,7 +2012,7 @@ static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp,struct rcu_data *rdp, unsigned long flags) { - RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_nocb_adopt_orphan_cbs() invoked with irqs enabled!!!"); +lockdep_assert_irqs_disabled ();if (!rcu_is_nocb_cpu (smp_processor_id ()))return false;/* Not NOCBs CPU, caller must migrate CBs. */ __call_rcu_nocb_enqueue(my_rdp, rcu_segcblist_head(&rdp->cblist),