RCU list updates for v5.6
Subsystems (2)
Paul E. McKenney
maintainercover RCU list updates for v5.6
Hello!
This series provides RCU-list updates:
1. Describe variadic macro argument in a Sphinx-compatible way,
courtesy of Jonathan Neuschäfer.
2. Add hlist_unhashed_lockless(), courtesy of Eric Dumazet.
3. Use hlist_unhashed_lockless() in timer_pending(), courtesy
of Eric Dumazet.
4. Use WRITE_ONCE() for assignments to ->pprev for hlist_nulls.
5. Add and update docbook header comments in list.h.
6. Add a hlist_nulls_unhashed_lockless() function.
7. Add rculist_nulls docbook comments, courtesy of Madhuparna Bhowmik.
8. Change rculist_nulls docbook comment headers, courtesy of
Madhuparna Bhowmik.
9. Add list_tail_rcu(), courtesy of Madhuparna Bhowmik.
Thanx, Paul
------------------------------------------------------------------------
list.h | 144 +++++++++++++++++++++++++++++++++++++++++++++----------- list_nulls.h | 30 ++++++++++- rculist.h | 38 +++++++++----- rculist_nulls.h | 20 +++++-- timer.h | 2 5 files changed, 181 insertions(+), 53 deletions(-)
[email protected]
maintainerPATCH 1/9 rculist: Describe variadic macro argument in a Sphinx-compatible way
From: Jonathan Neuschäfer <[email protected]>
Without this patch, Sphinx shows "variable arguments" as the description of the cond argument, rather than the intended description, and prints the following warnings:
./include/linux/rculist.h:374: warning: Excess function parameter 'cond' description in 'list_for_each_entry_rcu' ./include/linux/rculist.h:651: warning: Excess function parameter 'cond' description in 'hlist_for_each_entry_rcu'
--- include/linux/rculist.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
include/linux/rculist.h +2 -2
@@ -361,7 +361,7 @@ static inline void list_splice_tail_init_rcu(struct list_head *list,* @pos: the type * to use as a loop cursor. * @head: the head for your list. * @member: the name of the list_head within the struct. -* @cond: optional lockdep expression if called from non-RCU protection. +* @cond...: optional lockdep expression if called from non-RCU protection. * * This list-traversal primitive may safely run concurrently with * the _rcu list-mutation primitives such as list_add_rcu()
@@ -636,7 +636,7 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,* @pos: the type * to use as a loop cursor. * @head: the head for your list. * @member: the name of the hlist_node within the struct. -* @cond: optional lockdep expression if called from non-RCU protection. +* @cond...: optional lockdep expression if called from non-RCU protection. * * This list-traversal primitive may safely run concurrently with * the _rcu list-mutation primitives such as hlist_add_head_rcu()
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 2/9 list: Add hlist_unhashed_lockless()
In reply to Paul E. McKenneyFrom: Eric Dumazet <[email protected]>
We would like to use hlist_unhashed() from timer_pending(), which runs without protection of a lock.
Note that other callers might also want to use this variant.
Instead of forcing a READ_ONCE() for all hlist_unhashed() callers, add a new helper with an explicit _lockless suffix in the name to better document what is going on.
Also add various WRITE_ONCE() in __hlist_del(), hlist_add_head() and hlist_add_before()/hlist_add_behind() to pair with the READ_ONCE().
--- include/linux/list.h | 32 +++++++++++++++++++++----------- include/linux/rculist.h | 24 ++++++++++++------------ 2 files changed, 33 insertions(+), 23 deletions(-)
include/linux/list.h +21 -11
@@ -749,6 +749,16 @@ static inline int hlist_unhashed(const struct hlist_node *h)return !h->pprev; } +/* This variant of hlist_unhashed() must be used in lockless contexts +* to avoid potential load-tearing. +* The READ_ONCE() is paired with the various WRITE_ONCE() in hlist +* helpers that are defined below. +*/ +static inline int hlist_unhashed_lockless (const struct hlist_node *h) +{ +return !READ_ONCE (h->pprev); +} +static inline int hlist_empty (const struct hlist_head *h) {return !READ_ONCE (h->first);
@@ -761,7 +771,7 @@ static inline void __hlist_del(struct hlist_node *n)WRITE_ONCE (*pprev, next);if (next) - next->pprev = pprev; +WRITE_ONCE (next->pprev, pprev); }static inline void hlist_del (struct hlist_node *n)
@@ -782,32 +792,32 @@ static inline void hlist_del_init(struct hlist_node *n)static inline void hlist_add_head (struct hlist_node *n,struct hlist_head *h) {struct hlist_node *first = h->first; - n->next = first; +WRITE_ONCE (n->next, first);if (first) - first->pprev = &n->next; +WRITE_ONCE (first->pprev, &n->next);WRITE_ONCE (h->first, n); - n->pprev = &h->first; +WRITE_ONCE (n->pprev, &h->first); }/* next must be != NULL */ static inline void hlist_add_before (struct hlist_node *n,struct hlist_node *next) { - n->pprev = next->pprev; - n->next = next; - next->pprev = &n->next; +WRITE_ONCE (n->pprev, next->pprev); +WRITE_ONCE (n->next, next); +WRITE_ONCE (next->pprev, &n->next);WRITE_ONCE (*(n->pprev), n); }static inline void hlist_add_behind (struct hlist_node *n,struct hlist_node *prev) { - n->next = prev->next; - prev->next = n; - n->pprev = &prev->next; +WRITE_ONCE (n->next, prev->next); +WRITE_ONCE (prev->next, n); +WRITE_ONCE (n->pprev, &prev->next);if (n->next) - n->next->pprev = &n->next; +WRITE_ONCE (n->next->pprev, &n->next); }/* after that we'll appear to be on some hlist and hlist_del will work */
include/linux/rculist.h +12 -12
@@ -173,7 +173,7 @@ static inline void hlist_del_init_rcu(struct hlist_node *n) {if (!hlist_unhashed (n)) {__hlist_del (n); - n->pprev =NULL ; +WRITE_ONCE (n->pprev,NULL ); } }
@@ -473,7 +473,7 @@ static inline void list_splice_tail_init_rcu(struct list_head *list,static inline void hlist_del_rcu (struct hlist_node *n) {__hlist_del (n); - n->pprev =LIST_POISON2 ; +WRITE_ONCE (n->pprev,LIST_POISON2 ); }/**
@@ -489,11 +489,11 @@ static inline void hlist_replace_rcu(struct hlist_node *old,struct hlist_node *next = old->next; new->next = next; - new->pprev = old->pprev; +WRITE_ONCE (new->pprev, old->pprev);rcu_assign_pointer (*(struct hlist_node __rcu **)new->pprev, new);if (next) - new->next->pprev = &new->next; - old->pprev =LIST_POISON2 ; +WRITE_ONCE (new->next->pprev, &new->next); +WRITE_ONCE (old->pprev,LIST_POISON2 ); }/*
@@ -528,10 +528,10 @@ static inline void hlist_add_head_rcu(struct hlist_node *n,struct hlist_node *first = h->first; n->next = first; - n->pprev = &h->first; +WRITE_ONCE (n->pprev, &h->first);rcu_assign_pointer (hlist_first_rcu (h), n);if (first) - first->pprev = &n->next; +WRITE_ONCE (first->pprev, &n->next); }/**
@@ -564,7 +564,7 @@ static inline void hlist_add_tail_rcu(struct hlist_node *n,if (last) { n->next = last->next; - n->pprev = &last->next; +WRITE_ONCE (n->pprev, &last->next);rcu_assign_pointer (hlist_next_rcu (last), n); }else {hlist_add_head_rcu (n, h);
@@ -592,10 +592,10 @@ static inline void hlist_add_tail_rcu(struct hlist_node *n,static inline void hlist_add_before_rcu (struct hlist_node *n,struct hlist_node *next) { - n->pprev = next->pprev; +WRITE_ONCE (n->pprev, next->pprev); n->next = next;rcu_assign_pointer (hlist_pprev_rcu (n), n); - next->pprev = &n->next; +WRITE_ONCE (next->pprev, &n->next); }/**
@@ -620,10 +620,10 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n, struct hlist_node *prev) { n->next = prev->next; - n->pprev = &prev->next; +WRITE_ONCE (n->pprev, &prev->next);rcu_assign_pointer (hlist_next_rcu (prev), n);if (n->next) - n->next->pprev = &n->next; +WRITE_ONCE (n->next->pprev, &n->next); }#define __hlist_for_each_rcu (pos, head) \
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 3/9 timer: Use hlist_unhashed_lockless() in timer_pending()
In reply to Paul E. McKenneyFrom: Eric Dumazet <[email protected]>
The timer_pending() function is mostly used in lockless contexts, so Without proper annotations, KCSAN might detect a data-race [1].
Using hlist_unhashed_lockless() instead of hand-coding it seems appropriate (as suggested by Paul E. McKenney).
[1]
BUG: KCSAN: data-race in del_timer / detach_if_pending
write to 0xffff88808697d870 of 8 bytes by task 10 on cpu 0:
__hlist_del include/linux/list.h:764 [inline] detach_timer kernel/time/timer.c:815 [inline] detach_if_pending+0xcd/0x2d0 kernel/time/timer.c:832 try_to_del_timer_sync+0x60/0xb0 kernel/time/timer.c:1226 del_timer_sync+0x6b/0xa0 kernel/time/timer.c:1365 schedule_timeout+0x2d2/0x6e0 kernel/time/timer.c:1896 rcu_gp_fqs_loop+0x37c/0x580 kernel/rcu/tree.c:1639 rcu_gp_kthread+0x143/0x230 kernel/rcu/tree.c:1799 kthread+0x1d4/0x200 drivers/block/aoe/aoecmd.c:1253 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:352
read to 0xffff88808697d870 of 8 bytes by task 12060 on cpu 1:
del_timer+0x3b/0xb0 kernel/time/timer.c:1198 sk_stop_timer+0x25/0x60 net/core/sock.c:2845 inet_csk_clear_xmit_timers+0x69/0xa0 net/ipv4/inet_connection_sock.c:523 tcp_clear_xmit_timers include/net/tcp.h:606 [inline] tcp_v4_destroy_sock+0xa3/0x3f0 net/ipv4/tcp_ipv4.c:2096 inet_csk_destroy_sock+0xf4/0x250 net/ipv4/inet_connection_sock.c:836 tcp_close+0x6f3/0x970 net/ipv4/tcp.c:2497 inet_release+0x86/0x100 net/ipv4/af_inet.c:427 __sock_release+0x85/0x160 net/socket.c:590 sock_close+0x24/0x30 net/socket.c:1268 __fput+0x1e1/0x520 fs/file_table.c:280 ____fput+0x1f/0x30 fs/file_table.c:313 task_work_run+0xf6/0x130 kernel/task_work.c:113 tracehook_notify_resume include/linux/tracehook.h:188 [inline] exit_to_usermode_loop+0x2b4/0x2c0 arch/x86/entry/common.c:163
Reported by Kernel Concurrency Sanitizer on:
CPU: 1 PID: 12060 Comm: syz-executor.5 Not tainted 5.4.0-rc3+ #0 Hardware name: Google Google Compute Engine/Google Compute Engine,
--- include/linux/timer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/timer.h +1 -1
@@ -164,7 +164,7 @@ static inline void destroy_timer_on_stack(struct timer_list *timer) { }*/ static inline int timer_pending (const struct timer_list * timer) { -return timer->entry.pprev !=NULL ; +return !hlist_unhashed_lockless (&timer->entry); }extern void add_timer_on (struct timer_list *timer,int cpu);
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 4/9 rcu: Use WRITE_ONCE() for assignments to ->pprev for hlist_nulls
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
Eric Dumazet supplied a KCSAN report of a bug that forces use of hlist_unhashed_lockless() from sk_unhashed():
------------------------------------------------------------------------
BUG: KCSAN: data-race in inet_unhash / inet_unhash
write to 0xffff8880a69a0170 of 8 bytes by interrupt on cpu 1:
__hlist_nulls_del include/linux/list_nulls.h:88 [inline] hlist_nulls_del_init_rcu include/linux/rculist_nulls.h:36 [inline] __sk_nulls_del_node_init_rcu include/net/sock.h:676 [inline] inet_unhash+0x38f/0x4a0 net/ipv4/inet_hashtables.c:612 tcp_set_state+0xfa/0x3e0 net/ipv4/tcp.c:2249 tcp_done+0x93/0x1e0 net/ipv4/tcp.c:3854 tcp_write_err+0x7e/0xc0 net/ipv4/tcp_timer.c:56 tcp_retransmit_timer+0x9b8/0x16d0 net/ipv4/tcp_timer.c:479 tcp_write_timer_handler+0x42d/0x510 net/ipv4/tcp_timer.c:599 tcp_write_timer+0xd1/0xf0 net/ipv4/tcp_timer.c:619 call_timer_fn+0x5f/0x2f0 kernel/time/timer.c:1404 expire_timers kernel/time/timer.c:1449 [inline] __run_timers kernel/time/timer.c:1773 [inline] __run_timers kernel/time/timer.c:1740 [inline] run_timer_softirq+0xc0c/0xcd0 kernel/time/timer.c:1786 __do_softirq+0x115/0x33f kernel/softirq.c:292 invoke_softirq kernel/softirq.c:373 [inline] irq_exit+0xbb/0xe0 kernel/softirq.c:413 exiting_irq arch/x86/include/asm/apic.h:536 [inline] smp_apic_timer_interrupt+0xe6/0x280 arch/x86/kernel/apic/apic.c:1137 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:830 native_safe_halt+0xe/0x10 arch/x86/kernel/paravirt.c:71 arch_cpu_idle+0x1f/0x30 arch/x86/kernel/process.c:571 default_idle_call+0x1e/0x40 kernel/sched/idle.c:94 cpuidle_idle_call kernel/sched/idle.c:154 [inline] do_idle+0x1af/0x280 kernel/sched/idle.c:263 cpu_startup_entry+0x1b/0x20 kernel/sched/idle.c:355 start_secondary+0x208/0x260 arch/x86/kernel/smpboot.c:264 secondary_startup_64+0xa4/0xb0 arch/x86/kernel/head_64.S:241
read to 0xffff8880a69a0170 of 8 bytes by interrupt on cpu 0:
sk_unhashed include/net/sock.h:607 [inline] inet_unhash+0x3d/0x4a0 net/ipv4/inet_hashtables.c:592 tcp_set_state+0xfa/0x3e0 net/ipv4/tcp.c:2249 tcp_done+0x93/0x1e0 net/ipv4/tcp.c:3854 tcp_write_err+0x7e/0xc0 net/ipv4/tcp_timer.c:56 tcp_retransmit_timer+0x9b8/0x16d0 net/ipv4/tcp_timer.c:479 tcp_write_timer_handler+0x42d/0x510 net/ipv4/tcp_timer.c:599 tcp_write_timer+0xd1/0xf0 net/ipv4/tcp_timer.c:619 call_timer_fn+0x5f/0x2f0 kernel/time/timer.c:1404 expire_timers kernel/time/timer.c:1449 [inline] __run_timers kernel/time/timer.c:1773 [inline] __run_timers kernel/time/timer.c:1740 [inline] run_timer_softirq+0xc0c/0xcd0 kernel/time/timer.c:1786 __do_softirq+0x115/0x33f kernel/softirq.c:292 invoke_softirq kernel/softirq.c:373 [inline] irq_exit+0xbb/0xe0 kernel/softirq.c:413 exiting_irq arch/x86/include/asm/apic.h:536 [inline] smp_apic_timer_interrupt+0xe6/0x280 arch/x86/kernel/apic/apic.c:1137 apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:830 native_safe_halt+0xe/0x10 arch/x86/kernel/paravirt.c:71 arch_cpu_idle+0x1f/0x30 arch/x86/kernel/process.c:571 default_idle_call+0x1e/0x40 kernel/sched/idle.c:94 cpuidle_idle_call kernel/sched/idle.c:154 [inline] do_idle+0x1af/0x280 kernel/sched/idle.c:263 cpu_startup_entry+0x1b/0x20 kernel/sched/idle.c:355 rest_init+0xec/0xf6 init/main.c:452 arch_call_rest_init+0x17/0x37 start_kernel+0x838/0x85e init/main.c:786 x86_64_start_reservations+0x29/0x2b arch/x86/kernel/head64.c:490 x86_64_start_kernel+0x72/0x76 arch/x86/kernel/head64.c:471 secondary_startup_64+0xa4/0xb0 arch/x86/kernel/head_64.S:241
Reported by Kernel Concurrency Sanitizer on:
CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.4.0-rc6+ #0 Hardware name: Google Google Compute Engine/Google Compute Engine,
BIOS Google 01/01/2011
------------------------------------------------------------------------
This commit therefore replaces C-language assignments with WRITE_ONCE() in include/linux/list_nulls.h and include/linux/rculist_nulls.h.
--- include/linux/list_nulls.h | 8 ++++---- include/linux/rculist_nulls.h | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-)
include/linux/list_nulls.h +4 -4
@@ -72,10 +72,10 @@ static inline void hlist_nulls_add_head(struct hlist_nulls_node *n,struct hlist_nulls_node *first = h->first; n->next = first; - n->pprev = &h->first; +WRITE_ONCE (n->pprev, &h->first); h->first = n;if (!is_a_nulls (first)) - first->pprev = &n->next; +WRITE_ONCE (first->pprev, &n->next); }static inline void __hlist_nulls_del (struct hlist_nulls_node *n)
@@ -85,13 +85,13 @@ static inline void __hlist_nulls_del(struct hlist_nulls_node *n)WRITE_ONCE (*pprev, next);if (!is_a_nulls (next)) - next->pprev = pprev; +WRITE_ONCE (next->pprev, pprev); }static inline void hlist_nulls_del (struct hlist_nulls_node *n) {__hlist_nulls_del (n); - n->pprev =LIST_POISON2 ; +WRITE_ONCE (n->pprev,LIST_POISON2 ); }/**
include/linux/rculist_nulls.h +4 -4
@@ -34,7 +34,7 @@ static inline void hlist_nulls_del_init_rcu(struct hlist_nulls_node *n) {if (!hlist_nulls_unhashed (n)) {__hlist_nulls_del (n); - n->pprev =NULL ; +WRITE_ONCE (n->pprev,NULL ); } }
@@ -66,7 +66,7 @@ static inline void hlist_nulls_del_init_rcu(struct hlist_nulls_node *n)static inline void hlist_nulls_del_rcu (struct hlist_nulls_node *n) {__hlist_nulls_del (n); - n->pprev =LIST_POISON2 ; +WRITE_ONCE (n->pprev,LIST_POISON2 ); }/**
@@ -94,10 +94,10 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n,struct hlist_nulls_node *first = h->first; n->next = first; - n->pprev = &h->first; +WRITE_ONCE (n->pprev, &h->first);rcu_assign_pointer (hlist_nulls_first_rcu (h), n);if (!is_a_nulls (first)) - first->pprev = &n->next; +WRITE_ONCE (first->pprev, &n->next); }/**
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 5/9 rcu: Add and update docbook header comments in list.h
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
--- include/linux/list.h | 112 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 95 insertions(+), 17 deletions(-)
include/linux/list.h +95 -17
@@ -23,6 +23,13 @@#define LIST_HEAD (name) \ struct list_head name = LIST_HEAD_INIT(name) +/** +* INIT_LIST_HEAD - Initialize a list_head structure +* @list: list_head structure to be initialized. +* +* Initializes the list_head to point to itself. If it is a list header, +* the result is an empty list. +*/ static inline void INIT_LIST_HEAD (struct list_head *list) {WRITE_ONCE (list->next, list);
@@ -120,12 +127,6 @@ static inline void __list_del_clearprev(struct list_head *entry) entry->prev =NULL ; } -/** -* list_del - deletes entry from list. -* @entry: the element to delete from the list. -* Note: list_empty() on entry does not return true after this, the entry is -* in an undefined state. -*/ static inline void __list_del_entry (struct list_head *entry) {if (!__list_del_entry_valid (entry))
@@ -134,6 +135,12 @@ static inline void __list_del_entry(struct list_head *entry)__list_del (entry->prev, entry->next); } +/** +* list_del - deletes entry from list. +* @entry: the element to delete from the list. +* Note: list_empty() on entry does not return true after this, the entry is +* in an undefined state. +*/ static inline void list_del (struct list_head *entry) {__list_del_entry (entry);
@@ -157,8 +164,15 @@ static inline void list_replace(struct list_head *old, new->prev->next = new; } +/** +* list_replace_init - replace old entry by new one and initialize the old one +* @old : the element to be replaced +* @new : the new element to insert +* +* If @old was empty, it will be overwritten. +*/ static inline void list_replace_init (struct list_head *old, -struct list_head *new) +struct list_head *new) {list_replace (old, new);INIT_LIST_HEAD (old);
@@ -744,21 +758,36 @@ static inline void INIT_HLIST_NODE(struct hlist_node *h) h->pprev =NULL ; } +/** +* hlist_unhashed - Has node been removed from list and reinitialized? +* @h: Node to be checked +* +* Not that not all removal functions will leave a node in unhashed +* state. For example, hlist_nulls_del_init_rcu() does leave the +* node in unhashed state, but hlist_nulls_del() does not. +*/ static inline int hlist_unhashed (const struct hlist_node *h) {return !h->pprev; } -/* This variant of hlist_unhashed() must be used in lockless contexts -* to avoid potential load-tearing. -* The READ_ONCE() is paired with the various WRITE_ONCE() in hlist -* helpers that are defined below. +/** +* hlist_unhashed_lockless - Version of hlist_unhashed for lockless use +* @h: Node to be checked +* +* This variant of hlist_unhashed() must be used in lockless contexts +* to avoid potential load-tearing. The READ_ONCE() is paired with the +* various WRITE_ONCE() in hlist helpers that are defined below. */ static inline int hlist_unhashed_lockless (const struct hlist_node *h) {return !READ_ONCE (h->pprev); } +/** +* hlist_empty - Is the specified hlist_head structure an empty hlist? +* @h: Structure to check. +*/ static inline int hlist_empty (const struct hlist_head *h) {return !READ_ONCE (h->first);
@@ -774,6 +803,13 @@ static inline void __hlist_del(struct hlist_node *n)WRITE_ONCE (next->pprev, pprev); } +/** +* hlist_del - Delete the specified hlist_node from its list +* @n: Node to delete. +* +* Note that this function leaves the node in hashed state. Use +* hlist_del_init() or similar instead to unhash @n. +*/ static inline void hlist_del (struct hlist_node *n) {__hlist_del (n);
@@ -781,6 +817,12 @@ static inline void hlist_del(struct hlist_node *n) n->pprev =LIST_POISON2 ; } +/** +* hlist_del_init - Delete the specified hlist_node from its list and initialize +* @n: Node to delete. +* +* Note that this function leaves the node in unhashed state. +*/ static inline void hlist_del_init (struct hlist_node *n) {if (!hlist_unhashed (n)) {
@@ -789,6 +831,14 @@ static inline void hlist_del_init(struct hlist_node *n) } } +/** +* hlist_add_head - add a new entry at the beginning of the hlist +* @n: new entry to be added +* @h: hlist head to add it after +* +* Insert a new entry after the specified head. +* This is good for implementing stacks. +*/ static inline void hlist_add_head (struct hlist_node *n,struct hlist_head *h) {struct hlist_node *first = h->first;
@@ -799,9 +849,13 @@ static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)WRITE_ONCE (n->pprev, &h->first); } -/* next must be != NULL */ +/** +* hlist_add_before - add a new entry before the one specified +* @n: new entry to be added +* @next: hlist node to add it before, which must be non-NULL +*/ static inline void hlist_add_before (struct hlist_node *n, -struct hlist_node *next) +struct hlist_node *next) {WRITE_ONCE (n->pprev, next->pprev);WRITE_ONCE (n->next, next);
@@ -809,6 +863,11 @@ static inline void hlist_add_before(struct hlist_node *n,WRITE_ONCE (*(n->pprev), n); } +/** +* hlist_add_behing - add a new entry after the one specified +* @n: new entry to be added +* @prev: hlist node to add it after, which must be non-NULL +*/ static inline void hlist_add_behind (struct hlist_node *n,struct hlist_node *prev) {
@@ -820,20 +879,35 @@ static inline void hlist_add_behind(struct hlist_node *n,WRITE_ONCE (n->next->pprev, &n->next); } -/* after that we'll appear to be on some hlist and hlist_del will work */ +/** +* hlist_add_fake - create a fake hlist consisting of a single headless node +* @n: Node to make a fake list out of +* +* This makes @n appear to be its own predecessor on a headless hlist. +* The point of this is to allow things like hlist_del() to work correctly +* in cases where there is no list. +*/ static inline void hlist_add_fake (struct hlist_node *n) { n->pprev = &n->next; } +/** +* hlist_fake: Is this node a fake hlist? +* @h: Node to check for being a self-referential fake hlist. +*/ static inline bool hlist_fake (struct hlist_node *h) {return h->pprev == &h->next; } -/* +/** +* hlist_is_singular_node - is node the only element of the specified hlist? +* @n: Node to check for singularity. +* @h: Header for potentially singular list. +* * Check whether the node is the only node of the head without -* accessing head: +* accessing head, thus avoiding unnecessary cache misses. */ static inline bool hlist_is_singular_node (struct hlist_node *n,struct hlist_head *h)
@@ -841,7 +915,11 @@ hlist_is_singular_node(struct hlist_node *n, struct hlist_head *h)return !n->next && n->pprev == &h->first; } -/* +/** +* hlist_move_list - Move an hlist +* @old: hlist_head for old list. +* @new: hlist_head for new list. +* * Move a list from one list head to another. Fixup the pprev * reference of the first entry if it exists. */
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 6/9 rcu: Add a hlist_nulls_unhashed_lockless() function
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
This commit adds an hlist_nulls_unhashed_lockless() to allow lockless checking for whether or note an hlist_nulls_node is hashed or not. While in the area, this commit also adds a docbook comment to the existing hlist_nulls_unhashed() function.
--- include/linux/list_nulls.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)
include/linux/list_nulls.h +22 -0
@@ -56,11 +56,33 @@ static inline unsigned long get_nulls_value(const struct hlist_nulls_node *ptr)return ((unsigned long )ptr) >>1 ; } +/** +* hlist_nulls_unhashed - Has node been removed and reinitialized? +* @h: Node to be checked +* +* Not that not all removal functions will leave a node in unhashed state. +* For example, hlist_del_init_rcu() leaves the node in unhashed state, +* but hlist_nulls_del() does not. +*/ static inline int hlist_nulls_unhashed (const struct hlist_nulls_node *h) {return !h->pprev; } +/** +* hlist_nulls_unhashed_lockless - Has node been removed and reinitialized? +* @h: Node to be checked +* +* Not that not all removal functions will leave a node in unhashed state. +* For example, hlist_del_init_rcu() leaves the node in unhashed state, +* but hlist_nulls_del() does not. Unlike hlist_nulls_unhashed(), this +* function may be used locklessly. +*/ +static inline int hlist_nulls_unhashed_lockless (const struct hlist_nulls_node *h) +{ +return !READ_ONCE (h->pprev); +} +static inline int hlist_nulls_empty (const struct hlist_nulls_head *h) {return is_a_nulls (READ_ONCE (h->first));
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 7/9 rculist_nulls: Add docbook comments
In reply to Paul E. McKenneyFrom: Madhuparna Bhowmik <[email protected]>
This patch adds docbook comment headers for hlist_nulls_first_rcu() and hlist_nulls_next_rcu() in rculist_nulls.h.
--- include/linux/rculist_nulls.h | 8 ++++++++ 1 file changed, 8 insertions(+)
include/linux/rculist_nulls.h +8 -0
@@ -38,9 +38,17 @@ static inline void hlist_nulls_del_init_rcu(struct hlist_nulls_node *n) } } +/** +* hlist_nulls_first_rcu - returns the first element of the hash list. +* @head: the head of the list. +*/ #define hlist_nulls_first_rcu (head) \ (*((struct hlist_nulls_node __rcu __force **)&(head)->first)) +/** +* hlist_nulls_next_rcu - returns the element of the list after @node. +* @node: element of the list. +*/ #define hlist_nulls_next_rcu (node) \ (*((struct hlist_nulls_node __rcu __force **)&(node)->next))
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 8/9 rculist_nulls: Change docbook comment headers
In reply to Paul E. McKenneyFrom: Madhuparna Bhowmik <[email protected]>
This patch changes the docbook comment "head for your list" to "head of the list".
--- include/linux/rculist_nulls.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
include/linux/rculist_nulls.h +2 -2
@@ -112,7 +112,7 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n,* hlist_nulls_for_each_entry_rcu - iterate over rcu list of given type * @tpos: the type * to use as a loop cursor. * @pos: the &struct hlist_nulls_node to use as a loop cursor. -* @head: the head for your list. +* @head: the head of the list. * @member: the name of the hlist_nulls_node within the struct. * * The barrier() is needed to make sure compiler doesn't cache first element [1],
@@ -132,7 +132,7 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n,* iterate over list of given type safe against removal of list entry * @tpos: the type * to use as a loop cursor. * @pos: the &struct hlist_nulls_node to use as a loop cursor. -* @head: the head for your list. +* @head: the head of the list. * @member: the name of the hlist_nulls_node within the struct. */ #define hlist_nulls_for_each_entry_safe (tpos, pos, head, member) \
Signature · 1 line
2.9.5
[email protected]
maintainerPATCH 9/9 rculist.h: Add list_tail_rcu()
In reply to Paul E. McKenneyFrom: Madhuparna Bhowmik <[email protected]>
This patch adds the macro list_tail_rcu() and documents it.
--- include/linux/rculist.h | 10 ++++++++++ 1 file changed, 10 insertions(+)
include/linux/rculist.h +10 -0
@@ -40,6 +40,16 @@ static inline void INIT_LIST_HEAD_RCU(struct list_head *list)*/ #define list_next_rcu (list) (*((struct list_head __rcu **)(&(list)->next))) +/** +* list_tail_rcu - returns the prev pointer of the head of the list +* @head: the head of the list +* +* Note: This should only be used with the list header, and even then +* only if list_del() and similar primitives are not also used on the +* list header. +*/ +#define list_tail_rcu (head) (*((struct list_head __rcu **)(&(head)->prev))) +/* * Check during list traversal that we are within an RCU reader */
Signature · 1 line
2.9.5