Convert packet and key sockets to RCU
Subsystems (7)
cover Convert packet and key sockets to RCU
This set converts AF_PACKET and AF_KEY socket familys to use RCU rather than reader/writer lock.
PATCH 1/3 seq_file: add RCU versions of new hlist/list iterators (v3)
Many usages of seq_file use RCU protected lists, so non RCU iterators will not work safely.
---
This version drops the list_rcu variants because not needed yet
fs/seq_file.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/rculist.h | 5 ++++ include/linux/seq_file.h | 12 +++++++--- 3 files changed, 70 insertions(+), 3 deletions(-)
fs/seq_file.c +56 -0
@@ -750,3 +750,59 @@ struct hlist_node *seq_hlist_next(void *return node->next; }EXPORT_SYMBOL (seq_hlist_next); + +/** +* seq_hlist_start_rcu - start an iteration of a hlist protected by RCU +* @head: the head of the hlist +* @pos: the start position of the sequence +* +* Called at seq_file->op->start(). +*/ +struct hlist_node *seq_hlist_start_rcu (struct hlist_head *head,loff_t pos) +{ +struct hlist_node *node; + +__hlist_for_each_rcu (node, head) +if (pos-- ==0 ) +return node; +return NULL ; +} +EXPORT_SYMBOL (seq_hlist_start_rcu); + +/** +* seq_hlist_start_head_rcu - start an iteration of a hlist protected by RCU +* @head: the head of the hlist +* @pos: the start position of the sequence +* +* Called at seq_file->op->start(). Call this function if you want to +* print a header at the top of the output. +*/ +struct hlist_node *seq_hlist_start_head_rcu (struct hlist_head *head,loff_t pos) +{ +if (!pos) +return SEQ_START_TOKEN ; + +return seq_hlist_start_rcu (head, pos -1 ); +} +EXPORT_SYMBOL (seq_hlist_start_head_rcu); + +/** +* seq_hlist_next_rcu - move to the next position of the hlist protected by RCU +* @v: the current iterator +* @head: the head of the hlist +* @pos: the current posision +* +* Called at seq_file->op->next(). +*/ +struct hlist_node *seq_hlist_next_rcu (void *v,struct hlist_head *head, +loff_t *ppos) +{ +struct hlist_node *node = v; + + ++*ppos; +if (v ==SEQ_START_TOKEN ) +return rcu_dereference (head->first); +else +return rcu_dereference (node->next); +} +EXPORT_SYMBOL (seq_hlist_next_rcu);
include/linux/seq_file.h +9 -3
@@ -140,10 +140,16 @@ extern struct list_head *seq_list_next(v*/ extern struct hlist_node *seq_hlist_start (struct hlist_head *head, -loff_t pos); +loff_t pos);extern struct hlist_node *seq_hlist_start_head (struct hlist_head *head, -loff_t pos); +loff_t pos);extern struct hlist_node *seq_hlist_next (void *v,struct hlist_head *head, -loff_t *ppos); +loff_t *ppos); +extern struct hlist_node *seq_hlist_start_rcu (struct hlist_head *head, +loff_t pos); +extern struct hlist_node *seq_hlist_start_head_rcu (struct hlist_head *head, +loff_t pos); +extern struct hlist_node *seq_hlist_next_rcu (void *v,struct hlist_head *head, +loff_t *ppos);#endif
include/linux/rculist.h +5 -0
@@ -406,6 +406,11 @@ static inline void hlist_add_after_rcu(s n->next->pprev = &n->next; } +#define __hlist_for_each_rcu (pos, head) \ + for (pos = rcu_dereference((head)->first); \ + pos && ({ prefetch(pos->next); 1; }); \ + pos = rcu_dereference(pos->next)) +/** * hlist_for_each_entry_rcu - iterate over rcu list of given type * @tpos: the type * to use as a loop cursor.
Al Viro
maintainerRe: [PATCH 1/3] seq_file: add RCU versions of new hlist/list iterators (v3)
Quoted 2 lines · Stephen Hemminger
Many usages of
seq_fileuse RCU protected lists, so non RCU iterators will not work safely.
Looks sane, provided that you give strong warnings about access to contents of list elements...
PATCH 2/3 packet: convert socket list to RCU (v2)
In reply to Stephen HemmingerConvert AF_PACKET to use RCU, eliminating one more reader/writer lock.
There is no need for a real sk_del_node_init_rcu(), because sk_del_node_init is doing the equivalent thing to hlst_del_init_rcu already; but added some comments to try and make that obvious.
---
Incorporates Eric fix for race between notify and destroy.
include/net/netns/packet.h | 4 ++-- include/net/sock.h | 10 ++++++++++ net/packet/af_packet.c | 42 ++++++++++++++++++++---------------------- 3 files changed, 32 insertions(+), 24 deletions(-)
include/net/netns/packet.h +2 -2
@@ -4,11 +4,11 @@#ifndef __NETNS_PACKET_H__#define __NETNS_PACKET_H__ -#include <linux/list.h> +#include <linux/rculist.h> #include <linux/spinlock.h> struct netns_packet { -rwlock_t sklist_lock; +spinlock_t sklist_lock;struct hlist_head sklist; };
net/packet/af_packet.c +30 -29
121 lines not on this page · raw message
include/net/sock.h +10 -0
@@ -381,6 +381,7 @@ static __inline__ void __sk_del_node(str__hlist_del (&sk->sk_node); } +/* NB: equivalent to hlist_del_init_rcu */ static __inline__int __sk_del_node_init (struct sock *sk) {if (sk_hashed (sk)) {
@@ -421,6 +422,7 @@ static __inline__ int sk_del_node_init(s }return rc; } +#define sk_del_node_init_rcu (sk) sk_del_node_init(sk)static __inline__int __sk_nulls_del_node_init_rcu (struct sock *sk) {
@@ -454,6 +456,12 @@ static __inline__ void sk_add_node(struc__sk_add_node (sk, list); } +static __inline__void sk_add_node_rcu (struct sock *sk,struct hlist_head *list) +{ +sock_hold (sk); +hlist_add_head_rcu (&sk->sk_node, list); +} +static __inline__void __sk_nulls_add_node_rcu (struct sock *sk,struct hlist_nulls_head *list) {hlist_nulls_add_head_rcu (&sk->sk_nulls_node, list);
@@ -478,6 +486,8 @@ static __inline__ void sk_add_bind_node(#define sk_for_each (__sk, node, list) \ hlist_for_each_entry(__sk, node, list, sk_node) +#define sk_for_each_rcu (__sk, node, list) \ + hlist_for_each_entry_rcu(__sk, node, list, sk_node)#define sk_nulls_for_each (__sk, node, list) \ hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)#define sk_nulls_for_each_rcu (__sk, node, list) \
PATCH 1/1 packet: convert socket list to RCU (v3)
Convert AF_PACKET to use RCU, eliminating one more reader/writer lock.
There is no need for a real sk_del_node_init_rcu(), because sk_del_node_init is doing the equivalent thing to hlst_del_init_rcu already; but added some comments to try and make that obvious.
---
v2 - > v3: Need to not call remove_packet with lock held.
include/net/netns/packet.h | 4 ++-- include/net/sock.h | 10 ++++++++++ net/packet/af_packet.c | 42 ++++++++++++++++++++---------------------- 3 files changed, 32 insertions(+), 24 deletions(-)
include/net/netns/packet.h +2 -2
@@ -4,11 +4,11 @@#ifndef __NETNS_PACKET_H__#define __NETNS_PACKET_H__ -#include <linux/list.h> +#include <linux/rculist.h> #include <linux/spinlock.h> struct netns_packet { -rwlock_t sklist_lock; +spinlock_t sklist_lock;struct hlist_head sklist; };
net/packet/af_packet.c +31 -31
132 lines not on this page · raw message
include/net/sock.h +10 -0
@@ -381,6 +381,7 @@ static __inline__ void __sk_del_node(str__hlist_del (&sk->sk_node); } +/* NB: equivalent to hlist_del_init_rcu */ static __inline__int __sk_del_node_init (struct sock *sk) {if (sk_hashed (sk)) {
@@ -421,6 +422,7 @@ static __inline__ int sk_del_node_init(s }return rc; } +#define sk_del_node_init_rcu (sk) sk_del_node_init(sk)static __inline__int __sk_nulls_del_node_init_rcu (struct sock *sk) {
@@ -454,6 +456,12 @@ static __inline__ void sk_add_node(struc__sk_add_node (sk, list); } +static __inline__void sk_add_node_rcu (struct sock *sk,struct hlist_head *list) +{ +sock_hold (sk); +hlist_add_head_rcu (&sk->sk_node, list); +} +static __inline__void __sk_nulls_add_node_rcu (struct sock *sk,struct hlist_nulls_head *list) {hlist_nulls_add_head_rcu (&sk->sk_nulls_node, list);
@@ -478,6 +486,8 @@ static __inline__ void sk_add_bind_node(#define sk_for_each (__sk, node, list) \ hlist_for_each_entry(__sk, node, list, sk_node) +#define sk_for_each_rcu (__sk, node, list) \ + hlist_for_each_entry_rcu(__sk, node, list, sk_node)#define sk_nulls_for_each (__sk, node, list) \ hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)#define sk_nulls_for_each_rcu (__sk, node, list) \
Re: [PATCH] packet: convert socket list to RCU (v3)
Quoted 8 lines · Stephen Hemminger
Convert
AF_PACKETto use RCU, eliminating one more reader/writer lock.There is no need for a real
sk_del_node_init_rcu(), becausesk_del_node_initis doing the equivalent thing tohlst_del_init_rcualready; but added some comments to try and make that obvious.Signed-off-by Stephen Hemminger <[email protected]>
Quoted 7 lines
---
v2 - > v3: Need to not callremove_packetwith lock held.include/net/netns/packet.h | 4 ++-- include/net/sock.h | 10 ++++++++++ net/packet/af_packet.c | 42 ++++++++++++++++++++---------------------- 3 files changed, 32 insertions(+), 24 deletions(-)
Thanks Stephen
PATCH 3/3 af_key: locking change
In reply to Stephen HemmingerGet rid of custom locking that was using wait queue, lock, and atomic to basically build a queued mutex. Use RCU for read side.
net/key/af_key.c +15 -60
@@ -41,9 +41,7 @@ struct netns_pfkey {struct hlist_head table;atomic_t socks_nr; }; -static DECLARE_WAIT_QUEUE_HEAD (pfkey_table_wait); -static DEFINE_RWLOCK (pfkey_table_lock); -static atomic_t pfkey_table_users =ATOMIC_INIT (0 ); +static DEFINE_MUTEX (pfkey_mutex);struct pfkey_sock {/* struct sock must be the first member of struct pfkey_sock */
@@ -108,50 +106,6 @@ static void pfkey_sock_destruct(struct satomic_dec (&net_pfkey->socks_nr); } -static void pfkey_table_grab (void ) -{ -write_lock_bh (&pfkey_table_lock); - -if (atomic_read (&pfkey_table_users)) { -DECLARE_WAITQUEUE (wait, current); - -add_wait_queue_exclusive (&pfkey_table_wait, &wait); -for (;;) { -set_current_state (TASK_UNINTERRUPTIBLE ); -if (atomic_read (&pfkey_table_users) ==0 ) -break ; -write_unlock_bh (&pfkey_table_lock); -schedule (); -write_lock_bh (&pfkey_table_lock); - } - -__set_current_state (TASK_RUNNING ); -remove_wait_queue (&pfkey_table_wait, &wait); - } -} - -static __inline__void pfkey_table_ungrab (void ) -{ -write_unlock_bh (&pfkey_table_lock); -wake_up (&pfkey_table_wait); -} - -static __inline__void pfkey_lock_table (void ) -{ -/* read_lock() synchronizes us to pfkey_table_grab */ - -read_lock (&pfkey_table_lock); -atomic_inc (&pfkey_table_users); -read_unlock (&pfkey_table_lock); -} - -static __inline__void pfkey_unlock_table (void ) -{ -if (atomic_dec_and_test (&pfkey_table_users)) -wake_up (&pfkey_table_wait); -} - -static const struct proto_ops pfkey_ops;static void pfkey_insert (struct sock *sk)
@@ -159,16 +113,16 @@ static void pfkey_insert(struct sock *skstruct net *net =sock_net (sk);struct netns_pfkey *net_pfkey =net_generic (net, pfkey_net_id); -pfkey_table_grab (); -sk_add_node (sk, &net_pfkey->table); -pfkey_table_ungrab (); +mutex_lock (&pfkey_mutex); +sk_add_node_rcu (sk, &net_pfkey->table); +mutex_unlock (&pfkey_mutex); }static void pfkey_remove (struct sock *sk) { -pfkey_table_grab (); -sk_del_node_init (sk); -pfkey_table_ungrab (); +mutex_lock (&pfkey_mutex); +sk_del_node_init_rcu (sk); +mutex_unlock (&pfkey_mutex); }static struct proto key_proto = {
@@ -223,6 +177,7 @@ static int pfkey_release(struct socket *sock_orphan (sk); sock->sk =NULL ;skb_queue_purge (&sk->sk_write_queue); +synchronize_sched ();sock_put (sk);return 0 ;
@@ -277,8 +232,8 @@ static int pfkey_broadcast(struct sk_bufif (!skb)return -ENOMEM ; -pfkey_lock_table (); -sk_for_each (sk, node, &net_pfkey->table) { +rcu_read_lock (); +sk_for_each_rcu (sk, node, &net_pfkey->table) {struct pfkey_sock *pfk =pfkey_sk (sk);int err2;
@@ -309,7 +264,7 @@ static int pfkey_broadcast(struct sk_bufif ((broadcast_flags &BROADCAST_REGISTERED ) && err) err = err2; } -pfkey_unlock_table (); +rcu_read_unlock ();if (one_sk !=NULL ) err =pfkey_broadcast_one (skb, &skb2, allocation, one_sk);
@@ -3702,8 +3657,8 @@ static void *pfkey_seq_start(struct seq_struct net *net =seq_file_net (f);struct netns_pfkey *net_pfkey =net_generic (net, pfkey_net_id); -read_lock (&pfkey_table_lock); -return seq_hlist_start_head (&net_pfkey->table, *ppos); +rcu_read_lock (); +return seq_hlist_start_head_rcu (&net_pfkey->table, *ppos); }static void *pfkey_seq_next (struct seq_file *f,void *v,loff_t *ppos)
@@ -3711,12 +3666,12 @@ static void *pfkey_seq_next(struct seq_fstruct net *net =seq_file_net (f);struct netns_pfkey *net_pfkey =net_generic (net, pfkey_net_id); -return seq_hlist_next (v, &net_pfkey->table, ppos); +return seq_hlist_next_rcu (v, &net_pfkey->table, ppos); }static void pfkey_seq_stop (struct seq_file *f,void *v) { -read_unlock (&pfkey_table_lock); +rcu_read_unlock (); }static const struct seq_operations pfkey_seq_ops = {
Re: [PATCH 3/3] af_key: locking change
Quoted 5 lines · Stephen Hemminger
pièce jointe document texte brut (pfkey-rcu.patch)
Get rid of custom locking that was using wait queue, lock, and atomic to basically build a queued mutex. Use RCU for read side.Signed-off-by Stephen Hemminger <[email protected]>
Excellent
I dont know why you used synchronize_sched() instead of synchronize_rcu().
I prefer _rcu() variant because its all about rcu after all, and fact it is mapped to synchronize_sched() is an implementation detail.
(I am not saying implementation doesnt matter, Paul :) )
Quoted 11 lines
}
static struct proto
key_proto= {net/key/af_key.c +1
-0@@ -223,6 +177,7 @@ static int pfkey_release(struct socket *sock_orphan (sk); sock->sk =NULL ;skb_queue_purge (&sk->sk_write_queue); +synchronize_sched ();sock_put (sk);return 0 ;
Re: [PATCH 3/3] af_key: locking change
Quoted 5 lines · On Sat, 20 Feb 2010 05:07:30 +0100 Eric Dumazet <[email protected]> wrote:
Quoted 4 lines · Stephen Hemminger
pièce jointe document texte brut (pfkey-rcu.patch)
Get rid of custom locking that was using wait queue, lock, and atomic to basically build a queued mutex. Use RCU for read side.
Quoted 8 lines
Quoted 1 line
Signed-off-by Stephen Hemminger <[email protected]>Excellent
Acked-by Eric Dumazet <[email protected]>I dont know why you used
synchronize_sched()instead ofsynchronize_rcu().
I'll go back to synchronize_rcu.
David Miller
maintainerRe: [PATCH 3/3] af_key: locking change
From: Stephen Hemminger <[email protected]>
Date: Fri, 19 Feb 2010 21:53:14 -0800
Quoted 10 lines
Quoted 7 lines · On Sat, 20 Feb 2010 05:07:30 +0100 Eric Dumazet <[email protected]> wrote:
Quoted 5 lines · Stephen Hemminger
pièce jointe document texte brut (pfkey-rcu.patch)
Get rid of custom locking that was using wait queue, lock, and atomic to basically build a queued mutex. Use RCU for read side.Signed-off-by Stephen Hemminger <[email protected]>
Quoted 8 lines
Quoted 6 lines
Excellent
Acked-by Eric Dumazet <[email protected]>I dont know why you used
synchronize_sched()instead ofsynchronize_rcu().I'll go back to
synchronize_rcu.
Please also add some comments for patch #1 as requested by Al Viro.
When you post the new series with these changes I'll apply it to net-next-2.6, thanks!