RCU list-traversal lockdep updates for v5.4
Subsystems (15)
cover RCU list-traversal lockdep updates for v5.4
Hello!
This series provides enhancements to RCU pointer and list macros, all courtesy of Joel Fernandes:
1. Rename rcu_dereference_raw_notrace() to _check().
2. Remove redundant debug_locks check in rcu_read_lock_sched_held().
3. Add support for consolidated-RCU reader checking.
4. Remove custom check for RCU readers.
5. Add lockdep condition to fix for_each_entry().
6. Convert to use built-in RCU list checking.
7. Pass lockdep condition to pcm_mmcfg_list iterator.
8. Use built-in RCU list checking for acpi_ioremaps list.
Thanx, Paul
------------------------------------------------------------------------
Documentation/RCU/Design/Requirements/Requirements.html | 2 arch/powerpc/include/asm/kvm_book3s_64.h | 2 arch/x86/pci/mmconfig-shared.c | 5 - drivers/acpi/osl.c | 6 - drivers/base/base.h | 1 drivers/base/core.c | 10 ++ drivers/base/power/runtime.c | 15 ++- include/linux/rcu_sync.h | 4 include/linux/rculist.h | 38 +++++++-- include/linux/rcupdate.h | 9 +- kernel/rcu/Kconfig.debug | 11 ++ kernel/rcu/update.c | 67 ++++++++++------ kernel/trace/ftrace_internal.h | 8 - kernel/trace/trace.c | 4 net/ipv4/fib_frontend.c | 3 15 files changed, 132 insertions(+), 53 deletions(-)
PATCH 1/8 treewide: Rename rcu_dereference_raw_notrace() to _check()
From: "Joel Fernandes (Google)" <[email protected]>
The rcu_dereference_raw_notrace() API name is confusing. It is equivalent to rcu_dereference_raw() except that it also does sparse pointer checking.
There are only a few users of rcu_dereference_raw_notrace(). This patches renames all of them to be rcu_dereference_raw_check() with the "_check()" indicating sparse checking.
--- Documentation/RCU/Design/Requirements/Requirements.html | 2 +- arch/powerpc/include/asm/kvm_book3s_64.h | 2 +- include/linux/rculist.h | 6 +++--- include/linux/rcupdate.h | 2 +- kernel/trace/ftrace_internal.h | 8 ++++---- kernel/trace/trace.c | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-)
Documentation/RCU/Design/Requirements/Requirements.html +1 -1
@@ -2512,7 +2512,7 @@ disabled across the entire RCU read-side critical section. <p> It is possible to use tracing on RCU code, but tracing itself uses RCU. -For this reason, <tt>rcu_dereference_raw_notrace()</tt> +For this reason, <tt>rcu_dereference_raw_check()</tt> is provided for use by tracing, which avoids the destructive recursion that could otherwise ensue. This API is also used by virtualization in some architectures,
arch/powerpc/include/asm/kvm_book3s_64.h +1 -1
@@ -535,7 +535,7 @@ static inline void note_hpte_modification(struct kvm *kvm,*/ static inline struct kvm_memslots *kvm_memslots_raw (struct kvm *kvm) { -return rcu_dereference_raw_notrace (kvm->memslots[0 ]); +return rcu_dereference_raw_check (kvm->memslots[0 ]); }extern void kvmppc_mmu_debugfs_init (struct kvm *kvm);
include/linux/rculist.h +3 -3
@@ -622,7 +622,7 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,* as long as the traversal is guarded by rcu_read_lock(). */ #define hlist_for_each_entry_rcu (pos, head, member) \ - for (pos = hlist_entry_safe (rcu_dereference_raw(hlist_first_rcu(head)),\ + for (pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\ typeof(*(pos)), member); \ pos; \ pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
@@ -642,10 +642,10 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,* not do any RCU debugging or tracing. */ #define hlist_for_each_entry_rcu_notrace (pos, head, member) \ - for (pos = hlist_entry_safe (rcu_dereference_raw_notrace(hlist_first_rcu(head)),\ + for (pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_first_rcu(head)),\ typeof(*(pos)), member); \ pos; \ - pos = hlist_entry_safe(rcu_dereference_raw_notrace(hlist_next_rcu(\ + pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_next_rcu(\ &(pos)->member)), typeof(*(pos)), member))/**
include/linux/rcupdate.h +1 -1
@@ -476,7 +476,7 @@ do { \* The no-tracing version of rcu_dereference_raw() must not call * rcu_read_lock_held(). */ -#define rcu_dereference_raw_notrace (p) __rcu_dereference_check((p), 1, __rcu) +#define rcu_dereference_raw_check (p) __rcu_dereference_check((p), 1, __rcu)/** * rcu_dereference_protected() - fetch RCU pointer when updates prevented
kernel/trace/ftrace_internal.h +4 -4
@@ -6,22 +6,22 @@/* * Traverse the ftrace_global_list, invoking all entries. The reason that we -* can use rcu_dereference_raw_notrace() is that elements removed from this list +* can use rcu_dereference_raw_check() is that elements removed from this list * are simply leaked, so there is no need to interact with a grace-period -* mechanism. The rcu_dereference_raw_notrace() calls are needed to handle +* mechanism. The rcu_dereference_raw_check() calls are needed to handle * concurrent insertions into the ftrace_global_list. * * Silly Alpha and silly pointer-speculation compiler optimizations! */ #define do_for_each_ftrace_op (op, list) \ - op = rcu_dereference_raw_notrace(list); \ + op = rcu_dereference_raw_check(list); \ do/* * Optimized for just a single item in the list (as that is the normal case). */ #define while_for_each_ftrace_op (op) \ - while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \ + while (likely(op = rcu_dereference_raw_check((op)->next)) && \ unlikely((op) != &ftrace_list_end))extern struct ftrace_ops __rcu *ftrace_ops_list;
kernel/trace/trace.c +2 -2
@@ -2642,10 +2642,10 @@ static void ftrace_exports(struct ring_buffer_event *event)preempt_disable_notrace (); - export =rcu_dereference_raw_notrace (ftrace_exports_list); + export =rcu_dereference_raw_check (ftrace_exports_list);while (export) {trace_process_export (export, event); - export =rcu_dereference_raw_notrace (export->next); + export =rcu_dereference_raw_check (export->next); }preempt_enable_notrace ();
Signature · 1 line
2.17.1
PATCH 2/8 rcu: Remove redundant debug_locks check in rcu_read_lock_sched_held()
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
The debug_locks flag can never be true at the end of rcu_read_lock_sched_held() because it is already checked by the earlier
call todebug_lockdep_rcu_enabled(). This commit therefore removes this
redundant check.
--- kernel/rcu/update.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-)
kernel/rcu/update.c +1 -5
@@ -93,17 +93,13 @@ module_param(rcu_normal_after_boot, int, 0);*/ int rcu_read_lock_sched_held (void ) { -int lockdep_opinion =0 ; -if (!debug_lockdep_rcu_enabled ())return 1 ;if (!rcu_is_watching ())return 0 ;if (!rcu_lockdep_current_cpu_online ())return 0 ; -if (debug_locks) - lockdep_opinion =lock_is_held (&rcu_sched_lock_map); -return lockdep_opinion || !preemptible (); +return lock_is_held (&rcu_sched_lock_map) || !preemptible (); }EXPORT_SYMBOL (rcu_read_lock_sched_held);#endif
Signature · 1 line
2.17.1
PATCH 3/8 rcu: Add support for consolidated-RCU reader checking
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
This commit adds RCU-reader checks to list_for_each_entry_rcu() and hlist_for_each_entry_rcu(). These checks are optional, and are indicated by a lockdep expression passed to a new optional argument to these two macros. If this optional lockdep expression is omitted, these two macros act as before, checking for an RCU read-side critical section.
--- include/linux/rculist.h | 32 +++++++++++++++++---- include/linux/rcupdate.h | 7 +++++ kernel/rcu/Kconfig.debug | 11 ++++++++ kernel/rcu/update.c | 61 ++++++++++++++++++++++++++++------------ 4 files changed, 88 insertions(+), 23 deletions(-)
include/linux/rculist.h +27 -5
@@ -40,6 +40,24 @@ static inline void INIT_LIST_HEAD_RCU(struct list_head *list)*/ #define list_next_rcu (list) (*((struct list_head __rcu **)(&(list)->next))) +/* +* Check during list traversal that we are within an RCU reader +*/ + +#define check_arg_count_one (dummy) + +#ifdef CONFIG_PROVE_RCU_LIST +#define __list_check_rcu(dummy, cond, extra...) \ + ({ \ + check_arg_count_one(extra); \ + RCU_LOCKDEP_WARN(!cond && !rcu_read_lock_any_held(), \ + "RCU-list traversed in non-reader section!"); \ + }) +#else +#define __list_check_rcu(dummy, cond, extra...) \ + ({ check_arg_count_one(extra); }) +#endif +/* * Insert a new entry between two known consecutive entries. *
@@ -343,14 +361,16 @@ static inline void list_splice_tail_init_rcu(struct list_head *list,* @pos: the type * to use as a loop cursor. * @head: the head for your list. * @member: the name of the list_head within the struct. +* @cond: optional lockdep expression if called from non-RCU protection. * * This list-traversal primitive may safely run concurrently with * the _rcu list-mutation primitives such as list_add_rcu() * as long as the traversal is guarded by rcu_read_lock(). */ -#define list_for_each_entry_rcu (pos, head, member) \ - for (pos = list_entry_rcu((head)->next, typeof(*pos), member); \ - &pos->member != (head); \ +#define list_for_each_entry_rcu(pos, head, member, cond...) \ + for (__list_check_rcu(dummy, ## cond, 0), \ + pos = list_entry_rcu((head)->next, typeof(*pos), member); \ + &pos->member != (head); \ pos = list_entry_rcu(pos->member.next, typeof(*pos), member))/**
@@ -616,13 +636,15 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,* @pos: the type * to use as a loop cursor. * @head: the head for your list. * @member: the name of the hlist_node within the struct. +* @cond: optional lockdep expression if called from non-RCU protection. * * This list-traversal primitive may safely run concurrently with * the _rcu list-mutation primitives such as hlist_add_head_rcu() * as long as the traversal is guarded by rcu_read_lock(). */ -#define hlist_for_each_entry_rcu (pos, head, member) \ - for (pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\ +#define hlist_for_each_entry_rcu(pos, head, member, cond...) \ + for (__list_check_rcu(dummy, ## cond, 0), \ + pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\ typeof(*(pos)), member); \ pos; \ pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
include/linux/rcupdate.h +7 -0
@@ -221,6 +221,7 @@ int debug_lockdep_rcu_enabled(void);int rcu_read_lock_held (void );int rcu_read_lock_bh_held (void );int rcu_read_lock_sched_held (void ); +int rcu_read_lock_any_held (void );#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
@@ -241,6 +242,12 @@ static inline int rcu_read_lock_sched_held(void) {return !preemptible (); } + +static inline int rcu_read_lock_any_held (void ) +{ +return !preemptible (); +} +#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ #ifdef CONFIG_PROVE_RCU
kernel/rcu/Kconfig.debug +11 -0
@@ -8,6 +8,17 @@ menu "RCU Debugging" config PROVE_RCU def_bool PROVE_LOCKING +config PROVE_RCU_LIST + bool "RCU list lockdep debugging" + depends on PROVE_RCU && RCU_EXPERT + default n + help + Enable RCU lockdep checking for list usages. By default it is + turned off since there are several list RCU users that still + need to be converted to pass a lockdep expression. To prevent + false-positive splats, we keep it default disabled but once all + users are converted, we can remove this config option. + config TORTURE_TEST tristate default n
kernel/rcu/update.c +43 -18
85 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 4/8 rcu/sync: Remove custom check for RCU readers
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
The rcu/sync code currently does a special check for being in an RCU read-side critical section. With RCU consolidating flavors and the generic helper added earlier in this series, this check is no longer need. This commit switches to the generic helper, saving a couple of lines of code.
--- include/linux/rcu_sync.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
include/linux/rcu_sync.h +1 -3
@@ -31,9 +31,7 @@ struct rcu_sync {*/ static inline bool rcu_sync_is_idle (struct rcu_sync *rsp) { -RCU_LOCKDEP_WARN (!rcu_read_lock_held () && - !rcu_read_lock_bh_held () && - !rcu_read_lock_sched_held (), +RCU_LOCKDEP_WARN (!rcu_read_lock_any_held (),"suspicious rcu_sync_is_idle() usage" );return !READ_ONCE (rsp->gp_state);/* GP_IDLE */ }
Signature · 1 line
2.17.1
PATCH 5/8 ipv4: Add lockdep condition to fix for_each_entry()
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
This commit applies the consolidated list_for_each_entry_rcu() support for lockdep conditions.
--- net/ipv4/fib_frontend.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
net/ipv4/fib_frontend.c +2 -1
@@ -124,7 +124,8 @@ struct fib_table *fib_get_table(struct net *net, u32 id) h = id & (FIB_TABLE_HASHSZ -1 ); head = &net->ipv4.fib_table_hash[h]; -hlist_for_each_entry_rcu (tb, head, tb_hlist) { +hlist_for_each_entry_rcu (tb, head, tb_hlist, +lockdep_rtnl_is_held ()) {if (tb->tb_id == id)return tb; }
Signature · 1 line
2.17.1
PATCH 6/8 driver/core: Convert to use built-in RCU list checking
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
This commit applies the consolidated hlist_for_each_entry_rcu() support for lockdep conditions.
--- drivers/base/base.h | 1 + drivers/base/core.c | 10 ++++++++++ drivers/base/power/runtime.c | 15 ++++++++++----- 3 files changed, 21 insertions(+), 5 deletions(-)
drivers/base/base.h +1 -0
@@ -165,6 +165,7 @@ static inline int devtmpfs_init(void) { return 0; }/* Device links support */ extern int device_links_read_lock (void );extern void device_links_read_unlock (int idx); +extern int device_links_read_lock_held (void );extern int device_links_check_suppliers (struct device *dev);extern void device_links_driver_bound (struct device *dev);extern void device_links_driver_cleanup (struct device *dev);
drivers/base/core.c +10 -0
@@ -68,6 +68,11 @@ void device_links_read_unlock(int idx) {srcu_read_unlock (&device_links_srcu, idx); } + +int device_links_read_lock_held (void ) +{ +return srcu_read_lock_held (&device_links_srcu); +}#else /* !CONFIG_SRCU */ static DECLARE_RWSEM (device_links_lock);
@@ -91,6 +96,11 @@ void device_links_read_unlock(int not_used) {up_read (&device_links_lock); } + +int device_links_read_lock_held (void ) +{ +return lock_is_held (&device_links_lock); +}#endif /* !CONFIG_SRCU */ /**
drivers/base/power/runtime.c +10 -5
@@ -287,7 +287,8 @@ static int rpm_get_suppliers(struct device *dev) {struct device_link *link; -list_for_each_entry_rcu (link, &dev->links.suppliers, c_node) { +list_for_each_entry_rcu (link, &dev->links.suppliers, c_node, +device_links_read_lock_held ()) {int retval; if (!(link->flags & DL_FLAG_PM_RUNTIME) ||
@@ -309,7 +310,8 @@ static void rpm_put_suppliers(struct device *dev) {struct device_link *link; -list_for_each_entry_rcu (link, &dev->links.suppliers, c_node) { +list_for_each_entry_rcu (link, &dev->links.suppliers, c_node, +device_links_read_lock_held ()) {if (READ_ONCE (link->status) ==DL_STATE_SUPPLIER_UNBIND )continue ;
@@ -1640,7 +1642,8 @@ void pm_runtime_clean_up_links(struct device *dev) idx =device_links_read_lock (); -list_for_each_entry_rcu (link, &dev->links.consumers, s_node) { +list_for_each_entry_rcu (link, &dev->links.consumers, s_node, +device_links_read_lock_held ()) {if (link->flags &DL_FLAG_STATELESS )continue ;
@@ -1662,7 +1665,8 @@ void pm_runtime_get_suppliers(struct device *dev) idx =device_links_read_lock (); -list_for_each_entry_rcu (link, &dev->links.suppliers, c_node) +list_for_each_entry_rcu (link, &dev->links.suppliers, c_node, +device_links_read_lock_held ())if (link->flags &DL_FLAG_PM_RUNTIME ) { link->supplier_preactivated = true;refcount_inc (&link->rpm_active);
@@ -1683,7 +1687,8 @@ void pm_runtime_put_suppliers(struct device *dev) idx =device_links_read_lock (); -list_for_each_entry_rcu (link, &dev->links.suppliers, c_node) +list_for_each_entry_rcu (link, &dev->links.suppliers, c_node, +device_links_read_lock_held ())if (link->supplier_preactivated) { link->supplier_preactivated = false;if (refcount_dec_not_one (&link->rpm_active))
Signature · 1 line
2.17.1
PATCH 7/8 x86/pci: Pass lockdep condition to pcm_mmcfg_list iterator
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
The pcm_mmcfg_list is traversed by list_for_each_entry_rcu() outside of an RCU read-side critical section, which is safe because the pci_mmcfg_lock is held. This commit therefore adds a lockdep expression to list_for_each_entry_rcu() in order t avoid lockdep warnings.
--- arch/x86/pci/mmconfig-shared.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
arch/x86/pci/mmconfig-shared.c +3 -2
@@ -29,6 +29,7 @@static bool pci_mmcfg_running_state;static bool pci_mmcfg_arch_init_failed;static DEFINE_MUTEX (pci_mmcfg_lock); +#define pci_mmcfg_lock_held () lock_is_held(&(pci_mmcfg_lock).dep_map)LIST_HEAD (pci_mmcfg_list);
@@ -54,7 +55,7 @@ static void list_add_sorted(struct pci_mmcfg_region *new)struct pci_mmcfg_region *cfg;/* keep list sorted by segment and starting bus number */ -list_for_each_entry_rcu (cfg, &pci_mmcfg_list, list) { +list_for_each_entry_rcu (cfg, &pci_mmcfg_list, list,pci_mmcfg_lock_held ()) {if (cfg->segment > new->segment || (cfg->segment == new->segment && cfg->start_bus >= new->start_bus)) {
@@ -118,7 +119,7 @@ struct pci_mmcfg_region *pci_mmconfig_lookup(int segment, int bus) {struct pci_mmcfg_region *cfg; -list_for_each_entry_rcu (cfg, &pci_mmcfg_list, list) +list_for_each_entry_rcu (cfg, &pci_mmcfg_list, list,pci_mmcfg_lock_held ())if (cfg->segment == segment && cfg->start_bus <= bus && bus <= cfg->end_bus)return cfg;
Signature · 1 line
2.17.1
PATCH 8/8 acpi: Use built-in RCU list checking for acpi_ioremaps list
In reply to Paul E. McKenneyFrom: "Joel Fernandes (Google)" <[email protected]>
This commit applies the consolidated list_for_each_entry_rcu() support for lockdep conditions.
--- drivers/acpi/osl.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
drivers/acpi/osl.c +4 -2
@@ -14,6 +14,7 @@#include <linux/slab.h> #include <linux/mm.h> #include <linux/highmem.h> +#include <linux/lockdep.h> #include <linux/pci.h> #include <linux/interrupt.h> #include <linux/kmod.h>
@@ -80,6 +81,7 @@ struct acpi_ioremap {static LIST_HEAD (acpi_ioremaps);static DEFINE_MUTEX (acpi_ioremap_lock); +#define acpi_ioremap_lock_held () lock_is_held(&acpi_ioremap_lock.dep_map)static void __init acpi_request_region (struct acpi_generic_address *gas,unsigned int length,char *desc)
@@ -206,7 +208,7 @@ acpi_map_lookup(acpi_physical_address phys, acpi_size size) {struct acpi_ioremap *map; -list_for_each_entry_rcu (map, &acpi_ioremaps, list) +list_for_each_entry_rcu (map, &acpi_ioremaps, list,acpi_ioremap_lock_held ())if (map->phys <= phys && phys + size <= map->phys + map->size)return map;
@@ -249,7 +251,7 @@ acpi_map_lookup_virt(void __iomem *virt, acpi_size size) {struct acpi_ioremap *map; -list_for_each_entry_rcu (map, &acpi_ioremaps, list) +list_for_each_entry_rcu (map, &acpi_ioremaps, list,acpi_ioremap_lock_held ())if (map->virt <= virt && virt + size <= map->virt + map->size)return map;
Signature · 1 line
2.17.1