net: annotate data-races around sk->sk_{data_ready,write_space}
PATCH 1/1 net: annotate data-races around sk->sk_{data_ready,write_space}
From: Eric Dumazet <[email protected]>
[ Upstream commit 2ef2b20cf4e04ac8a6ba68493f8780776ff84300 ]
skmsg (and probably other layers) are changing these pointers while other cpus might read them concurrently.
Add corresponding READ_ONCE()/WRITE_ONCE() annotations for UDP, TCP and AF_UNIX.
Fixes 604326b41a6f ("bpf, sockmap: convert to generic sk_msg interface")
Reported-by [email protected]
Signed-off-by Eric Dumazet <[email protected]>
Cc Daniel Borkmann <[email protected]>
Cc John Fastabend <[email protected]>
Cc Jakub Sitnicki <[email protected]>
Cc Willem de Bruijn <[email protected]>
Reviewed-by Kuniyuki Iwashima <[email protected]>
Signed-off-by Jakub Kicinski <[email protected]>
Signed-off-by Leon Chen <[email protected]>
--- net/core/skmsg.c | 14 +++++++------- net/ipv4/tcp.c | 4 ++-- net/ipv4/tcp_bpf.c | 2 +- net/ipv4/tcp_input.c | 14 ++++++++------ net/ipv4/tcp_minisocks.c | 2 +- net/ipv4/udp.c | 3 ++- net/ipv4/udp_bpf.c | 2 +- net/unix/af_unix.c | 8 ++++---- 8 files changed, 26 insertions(+), 23 deletions(-)
net/core/skmsg.c +7 -7
@@ -1204,8 +1204,8 @@ void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)return ; psock->saved_data_ready = sk->sk_data_ready; - sk->sk_data_ready = sk_psock_strp_data_ready; - sk->sk_write_space = sk_psock_write_space; +WRITE_ONCE (sk->sk_data_ready, sk_psock_strp_data_ready); +WRITE_ONCE (sk->sk_write_space, sk_psock_write_space); }void sk_psock_stop_strp (struct sock *sk,struct sk_psock *psock)
@@ -1215,8 +1215,8 @@ void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)if (!psock->saved_data_ready)return ; - sk->sk_data_ready = psock->saved_data_ready; - psock->saved_data_ready =NULL ; +WRITE_ONCE (sk->sk_data_ready, psock->saved_data_ready); +WRITE_ONCE (psock->saved_data_ready,NULL );strp_stop (&psock->strp); }
@@ -1298,8 +1298,8 @@ void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)return ; psock->saved_data_ready = sk->sk_data_ready; - sk->sk_data_ready = sk_psock_verdict_data_ready; - sk->sk_write_space = sk_psock_write_space; +WRITE_ONCE (sk->sk_data_ready, sk_psock_verdict_data_ready); +WRITE_ONCE (sk->sk_write_space, sk_psock_write_space); }void sk_psock_stop_verdict (struct sock *sk,struct sk_psock *psock)
@@ -1310,6 +1310,6 @@ void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)if (!psock->saved_data_ready)return ; - sk->sk_data_ready = psock->saved_data_ready; +WRITE_ONCE (sk->sk_data_ready, psock->saved_data_ready); psock->saved_data_ready =NULL ; }
net/ipv4/tcp.c +2 -2
@@ -1346,7 +1346,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) err =sk_stream_error (sk, flags, err);/* make sure we wake any epoll edge trigger waiter */ if (unlikely (tcp_rtx_and_write_queues_empty (sk) && err == -EAGAIN )) { - sk->sk_write_space (sk); +READ_ONCE (sk->sk_write_space)(sk);tcp_chrono_stop (sk,TCP_CHRONO_SNDBUF_LIMITED ); }return err;
@@ -4023,7 +4023,7 @@ int do_tcp_setsockopt(struct sock *sk, int level, int optname,break ;case TCP_NOTSENT_LOWAT :WRITE_ONCE (tp->notsent_lowat, val); - sk->sk_write_space (sk); +READ_ONCE (sk->sk_write_space)(sk);break ;case TCP_INQ :if (val >1 || val <0 )
net/ipv4/tcp_bpf.c +1 -1
@@ -725,7 +725,7 @@ int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore)WRITE_ONCE (sk->sk_prot->unhash, psock->saved_unhash);tcp_update_ulp (sk, psock->sk_proto, psock->saved_write_space); }else { - sk->sk_write_space = psock->saved_write_space; +WRITE_ONCE (sk->sk_write_space, psock->saved_write_space);/* Pairs with lockless read in sk_clone_lock() */ sock_replace_proto (sk, psock->sk_proto); }
net/ipv4/tcp_input.c +8 -6
@@ -5034,7 +5034,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb)if (unlikely (tcp_try_rmem_schedule (sk, skb, skb->truesize))) {NET_INC_STATS (sock_net (sk),LINUX_MIB_TCPOFODROP ); - sk->sk_data_ready (sk); +READ_ONCE (sk->sk_data_ready)(sk);tcp_drop_reason (sk, skb,SKB_DROP_REASON_PROTO_MEM );return ; }
@@ -5241,7 +5241,7 @@ int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size)void tcp_data_ready (struct sock *sk) {if (tcp_epollin_ready (sk, sk->sk_rcvlowat) ||sock_flag (sk,SOCK_DONE )) - sk->sk_data_ready (sk); +READ_ONCE (sk->sk_data_ready)(sk); }static void tcp_data_queue (struct sock *sk,struct sk_buff *skb)
@@ -5297,7 +5297,7 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)inet_csk (sk)->icsk_ack.pending |= (ICSK_ACK_NOMEM |ICSK_ACK_NOW );inet_csk_schedule_ack (sk); - sk->sk_data_ready (sk); +READ_ONCE (sk->sk_data_ready)(sk);if (skb_queue_len (&sk->sk_receive_queue) && skb->len) { reason =SKB_DROP_REASON_PROTO_MEM ;
@@ -5735,7 +5735,9 @@ static void tcp_new_space(struct sock *sk) tp->snd_cwnd_stamp = tcp_jiffies32; } -INDIRECT_CALL_1 (sk->sk_write_space, sk_stream_write_space, sk); +INDIRECT_CALL_1 (READ_ONCE (sk->sk_write_space), + sk_stream_write_space, + sk); }/* Caller made space either from:
@@ -5941,7 +5943,7 @@ static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *tBUG ();WRITE_ONCE (tp->urg_data,TCP_URG_VALID | tmp);if (!sock_flag (sk,SOCK_DEAD )) - sk->sk_data_ready (sk); +READ_ONCE (sk->sk_data_ready)(sk); } } }
@@ -7341,7 +7343,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops,sock_put (fastopen_sk); goto drop_and_free; } - sk->sk_data_ready (sk); +READ_ONCE (sk->sk_data_ready)(sk);bh_unlock_sock (fastopen_sk);sock_put (fastopen_sk); }else {
net/ipv4/tcp_minisocks.c +1 -1
@@ -928,7 +928,7 @@ enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child, reason =tcp_rcv_state_process (child, skb);/* Wakeup parent, send SIGIO */ if (state ==TCP_SYN_RECV && child->sk_state != state) - parent->sk_data_ready (parent); +READ_ONCE (parent->sk_data_ready)(parent); }else {/* Alas, it is possible again, because we do lookup * in main socket hash table and lock on listening
net/ipv4/udp.c +2 -1
@@ -1622,7 +1622,8 @@ int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)spin_unlock (&list->lock);if (!sock_flag (sk,SOCK_DEAD )) -INDIRECT_CALL_1 (sk->sk_data_ready, sock_def_readable, sk); +INDIRECT_CALL_1 (READ_ONCE (sk->sk_data_ready), + sock_def_readable, sk);busylock_release (busy);return 0 ;
net/ipv4/udp_bpf.c +1 -1
@@ -158,7 +158,7 @@ int udp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore)int family = sk->sk_family ==AF_INET ?UDP_BPF_IPV4 :UDP_BPF_IPV6 ;if (restore) { - sk->sk_write_space = psock->saved_write_space; +WRITE_ONCE (sk->sk_write_space, psock->saved_write_space);sock_replace_proto (sk, psock->sk_proto);return 0 ; }
net/unix/af_unix.c +4 -4
@@ -1707,7 +1707,7 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr,__skb_queue_tail (&other->sk_receive_queue, skb);spin_unlock (&other->sk_receive_queue.lock);unix_state_unlock (other); - other->sk_data_ready (other); +READ_ONCE (other->sk_data_ready)(other);sock_put (other);return 0 ;
@@ -2175,7 +2175,7 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg,scm_stat_add (other, skb);skb_queue_tail (&other->sk_receive_queue, skb);unix_state_unlock (other); - other->sk_data_ready (other); +READ_ONCE (other->sk_data_ready)(other);sock_put (other);scm_destroy (&scm);return len;
@@ -2243,7 +2243,7 @@ static int queue_oob(struct socket *sock, struct msghdr *msg, struct sock *othersk_send_sigurg (other);unix_state_unlock (other); - other->sk_data_ready (other); +READ_ONCE (other->sk_data_ready)(other);return err; }
@@ -2354,7 +2354,7 @@ static int unix_stream_sendmsg(struct socket *sock, struct msghdr *msg,scm_stat_add (other, skb);skb_queue_tail (&other->sk_receive_queue, skb);unix_state_unlock (other); - other->sk_data_ready (other); +READ_ONCE (other->sk_data_ready)(other); sent += size; }
Signature · 1 line
2.35.3