BUG: soft lockup in do_sock_setsockopt
Subsystems (4)
syzbot
syzbotHello,
syzbot found the following issue on:
HEAD commit: 707081b61156 Merge branch 'for-next/core', remote-tracking.. git tree: git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git for-kernelci console output: https://syzkaller.appspot.com/x/log.txt?x=16d23231180000 kernel config: https://syzkaller.appspot.com/x/.config?x=caeac3f3565b057a dashboard link: https://syzkaller.appspot.com/bug?extid=10a41dc44eef71aa9450 compiler: Debian clang version 15.0.6, GNU ld (GNU Binutils for Debian) 2.40 userspace arch: arm64 syz repro: https://syzkaller.appspot.com/x/repro.syz?x=126fea6e180000 C reproducer: https://syzkaller.appspot.com/x/repro.c?x=114e4c81180000
Downloadable assets:
disk image: https://storage.googleapis.com/syzbot-assets/6cad68bf7532/disk-707081b6.raw.xz
vmlinux: https://storage.googleapis.com/syzbot-assets/1a27e5400778/vmlinux-707081b6.xz
kernel image: https://storage.googleapis.com/syzbot-assets/67dfc53755d0/Image-707081b6.gz.xz
IMPORTANT: if you fix the issue, please add the following tag to the commit:
watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [syz-executor178:13146] Modules linked in: irq event stamp: 29265101 hardirqs last enabled at (29265100): [<ffff8000801dae10>] __local_bh_enable_ip+0x224/0x44c kernel/softirq.c:386 hardirqs last disabled at (29265101): [<ffff80008ad66a78>] __el1_irq arch/arm64/kernel/entry-common.c:533 [inline] hardirqs last disabled at (29265101): [<ffff80008ad66a78>] el1_interrupt+0x24/0x68 arch/arm64/kernel/entry-common.c:551 softirqs last enabled at (638): [<ffff80008a7fe730>] spin_unlock_bh include/linux/spinlock.h:396 [inline] softirqs last enabled at (638): [<ffff80008a7fe730>] tipc_skb_peek_port net/tipc/msg.h:1235 [inline] softirqs last enabled at (638): [<ffff80008a7fe730>] tipc_sk_rcv+0x34c/0x1888 net/tipc/socket.c:2494 softirqs last disabled at (640): [<ffff80008a7fe750>] spin_trylock_bh include/linux/spinlock.h:411 [inline] softirqs last disabled at (640): [<ffff80008a7fe750>] tipc_sk_rcv+0x36c/0x1888 net/tipc/socket.c:2499 CPU: 0 PID: 13146 Comm: syz-executor178 Not tainted 6.8.0-rc7-syzkaller-g707081b61156 #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 02/29/2024 pstate: 60400005 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : lock_acquire+0x278/0x71c lr : lockdep_recursion_finish kernel/locking/lockdep.c:467 [inline] lr : lock_acquire+0x248/0x71c kernel/locking/lockdep.c:5756
sp : ffff80009ab86080
x29: ffff80009ab86190 x28: dfff800000000000 x27: ffff700013570c1c
x26: ffff0001b3fffdc0 x25: ffff80008ee74ac0 x24: 0000000000000000
x23: 0000000000000000 x22: 0000000000000000 x21: 0000000000000000
x20: ffff80008ebebdc0 x19: ffff0001b3fffdc0 x18: ffff80009ab862e0
x17: 000000000000c55c x16: ffff80008ad6b1c0 x15: 0000000000000001
x14: ffff80008eca0458 x13: dfff800000000000 x12: 00000000af1c601b
x11: 00000000a4e03a31 x10: 0000000000000003 x9 : 0000000000000000 x8 : 00000000000000c0 x7 : ffff80008a80b3f8 x6 : 0000000000000000 x5 : 0000000000000000 x4 : 0000000000000000 x3 : 0000000000000002 x2 : 0000000000000008 x1 : ffff80008aedfba0 x0 : 0000000000000000
Call trace: __daif_local_irq_restore arch/arm64/include/asm/irqflags.h:176 [inline] arch_local_irq_restore arch/arm64/include/asm/irqflags.h:196 [inline] lock_acquire+0x278/0x71c kernel/locking/lockdep.c:5757 rcu_lock_acquire+0x40/0x4c include/linux/rcupdate.h:298 rcu_read_lock include/linux/rcupdate.h:750 [inline] tipc_sk_lookup+0xc8/0x8b4 net/tipc/socket.c:3003 tipc_sk_rcv+0x358/0x1888 net/tipc/socket.c:2495 tipc_node_xmit+0x1b0/0xdb0 net/tipc/node.c:1703 tipc_node_xmit_skb net/tipc/node.c:1768 [inline] tipc_node_distr_xmit+0x28c/0x3a4 net/tipc/node.c:1783 tipc_sk_rcv+0x1280/0x1888 net/tipc/socket.c:2504 tipc_node_xmit+0x1b0/0xdb0 net/tipc/node.c:1703 tipc_sk_push_backlog net/tipc/socket.c:1317 [inline] tipc_sk_filter_connect net/tipc/socket.c:2258 [inline] tipc_sk_filter_rcv+0x13f8/0x2cac net/tipc/socket.c:2367 tipc_sk_enqueue net/tipc/socket.c:2448 [inline] tipc_sk_rcv+0x6d0/0x1888 net/tipc/socket.c:2500 tipc_node_xmit+0x1b0/0xdb0 net/tipc/node.c:1703 tipc_node_xmit_skb net/tipc/node.c:1768 [inline] tipc_node_distr_xmit+0x28c/0x3a4 net/tipc/node.c:1783 tipc_sk_backlog_rcv+0x164/0x214 net/tipc/socket.c:2415 sk_backlog_rcv include/net/sock.h:1092 [inline] __release_sock+0x1a8/0x408 net/core/sock.c:2972 release_sock+0x68/0x1b8 net/core/sock.c:3538 sk_setsockopt+0xbdc/0x306c sock_setsockopt+0x68/0x80 net/core/sock.c:1548 do_sock_setsockopt+0x238/0x4e0 net/socket.c:2307 __sys_setsockopt+0x128/0x1a8 net/socket.c:2334 __do_sys_setsockopt net/socket.c:2343 [inline] __se_sys_setsockopt net/socket.c:2340 [inline] __arm64_sys_setsockopt+0xb8/0xd4 net/socket.c:2340 __invoke_syscall arch/arm64/kernel/syscall.c:34 [inline] invoke_syscall+0x98/0x2b8 arch/arm64/kernel/syscall.c:48 el0_svc_common+0x130/0x23c arch/arm64/kernel/syscall.c:133 do_el0_svc+0x48/0x58 arch/arm64/kernel/syscall.c:152 el0_svc+0x54/0x168 arch/arm64/kernel/entry-common.c:712 el0t_64_sync_handler+0x84/0xfc arch/arm64/kernel/entry-common.c:730 el0t_64_sync+0x190/0x194 arch/arm64/kernel/entry.S:598
---
This report is generated by a bot. It may contain errors. See https://goo.gl/tpsmEJ for more information about syzbot. syzbot engineers can be reached at [email protected].
syzbot will keep track of this issue. See:
https://goo.gl/tpsmEJ#status for how to communicate with syzbot.
If the report is already addressed, let syzbot know by replying with:
#syz fix: exact-commit-title
If you want syzbot to run the reproducer, reply with:
#syz test: git://repo/address.git branch-or-commit-hash
If you attach or paste a git patch, syzbot will apply it before testing.
If you want to overwrite report's subsystems, reply with:
#syz set subsystems: new-subsystem
(See the list of subsystem names on the web dashboard)
If the report is a duplicate of another one, reply with:
#syz dup: exact-subject-of-another-report
If you want to undo deduplication, reply with:
#syz undup
On Fri, 22 Mar 2024 07:23:18 -0700
Quoted 5 lines
syzbot found the following issue on:
HEAD commit: 707081b61156 Merge branch 'for-next/core', remote-tracking.. git tree: git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git for-kernelci C reproducer: https://syzkaller.appspot.com/x/repro.c?x=114e4c81180000
#syz test https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git for-kernelci
net/tipc/socket.c +6 -0
@@ -2488,6 +2488,11 @@ void tipc_sk_rcv(struct net *net, structstruct tipc_sock *tsk;struct sock *sk;struct sk_buff *skb; +static int reent =0 ; + +if (reent) +return ; + reent++;__skb_queue_head_init (&xmitq);while (skb_queue_len (inputq)) {
@@ -2524,6 +2529,7 @@ xmit: dnode =msg_destnode (buf_msg (skb));tipc_node_xmit_skb (net, skb, dnode, dport); } + --reent; }static int tipc_wait_for_connect (struct socket *sock,long *timeo_p)
--
syzbot
syzbotsyzbot has tested the proposed patch and the reproducer did not trigger any issue:
Hello,
syzbot has tested the proposed patch and the reproducer did not trigger any issue:
Tested on:
commit: 707081b6 Merge branch 'for-next/core', remote-tracking.. git tree: https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git for-kernelci console output: https://syzkaller.appspot.com/x/log.txt?x=17e6b481180000 kernel config: https://syzkaller.appspot.com/x/.config?x=caeac3f3565b057a dashboard link: https://syzkaller.appspot.com/bug?extid=10a41dc44eef71aa9450 compiler: Debian clang version 15.0.6, GNU ld (GNU Binutils for Debian) 2.40 userspace arch: arm64 patch: https://syzkaller.appspot.com/x/patch.diff?x=1222a94e180000
Note: testing is done by a robot and is best-effort only.
syzbot
syzbotsyzbot has bisected this issue to:
syzbot has bisected this issue to:
commit 5f6bd380c7bdbe10f7b4e8ddcceed60ce0714c6d
Author: Peter Zijlstra <[email protected]> Date: Mon May 27 12:06:55 2024 +0000
sched/rt: Remove default bandwidth control
bisection log: https://syzkaller.appspot.com/x/bisect.txt?x=104b93d0580000 start commit: 3efc57369a0c Merge tag 'for-linus' of git://git.kernel.org.. git tree: upstream final oops: https://syzkaller.appspot.com/x/report.txt?x=124b93d0580000 console output: https://syzkaller.appspot.com/x/log.txt?x=144b93d0580000 kernel config: https://syzkaller.appspot.com/x/.config?x=a4fcb065287cdb84 dashboard link: https://syzkaller.appspot.com/bug?extid=10a41dc44eef71aa9450 syz repro: https://syzkaller.appspot.com/x/repro.syz?x=1636fe27980000 C reproducer: https://syzkaller.appspot.com/x/repro.c?x=136b5e80580000
For information about bisection process see: https://goo.gl/tpsmEJ#bisection
syzbot
syzbotForwarded: Re: [syzbot] [tipc?] BUG: soft lockup in do_sock_setsockopt
In reply to syzbotFor archival purposes, forwarding an incoming command email to [email protected], [email protected].
For archival purposes, forwarding an incoming command email to [email protected], [email protected].
***
Subject: Re: [syzbot] [tipc?] BUG: soft lockup in do_sock_setsockopt
Author: [email protected]
#syz test: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git a13307e97d5c54b65720bb71fa379960ded1e51a
net/tipc/socket.c +13 -6
@@ -156,7 +156,8 @@ static int tipc_sk_insert(struct tipc_sock *tsk);static void tipc_sk_remove (struct tipc_sock *tsk);static int __tipc_sendstream (struct socket *sock,struct msghdr *m,size_t dsz);static int __tipc_sendmsg (struct socket *sock,struct msghdr *m,size_t dsz); -static void tipc_sk_push_backlog (struct tipc_sock *tsk,bool nagle_ack); +static void tipc_sk_push_backlog (struct tipc_sock *tsk,bool nagle_ack, +struct sk_buff_head *xmitq);static int tipc_wait_for_connect (struct socket *sock,long *timeo_p);static const struct proto_ops packet_ops;
@@ -561,7 +562,7 @@ static void __tipc_shutdown(struct socket *sock, int error) !tsk_conn_cong(tsk)));/* Push out delayed messages if in Nagle mode */ -tipc_sk_push_backlog (tsk, false); +tipc_sk_push_backlog (tsk, false,NULL );/* Remove pending SYN */ __skb_queue_purge (&sk->sk_write_queue);
@@ -1268,8 +1269,10 @@ void tipc_sk_mcast_rcv(struct net *net, struct sk_buff_head *arrvq,/* tipc_sk_push_backlog(): send accumulated buffers in socket write queue * when socket is in Nagle mode +* @xmitq: receive output queue, or NULL outside receive context */ -static void tipc_sk_push_backlog (struct tipc_sock *tsk,bool nagle_ack) +static void tipc_sk_push_backlog (struct tipc_sock *tsk,bool nagle_ack, +struct sk_buff_head *xmitq) {struct sk_buff_head *txq = &tsk->sk.sk_write_queue;struct sk_buff *skb =skb_peek_tail (txq);
@@ -1311,6 +1314,10 @@ static void tipc_sk_push_backlog(struct tipc_sock *tsk, bool nagle_ack) tsk->pkt_cnt +=skb_queue_len (txq); tsk->snt_unacked += tsk->snd_backlog; tsk->snd_backlog =0 ; +if (xmitq &&in_own_node (net, dnode)) { +skb_queue_splice_tail_init (txq, xmitq); +return ; + } rc =tipc_node_xmit (net, txq, dnode, tsk->portid);if (rc == -ELINKCONG ) tsk->cong_link_cnt =1 ;
@@ -1368,7 +1375,7 @@ static void tipc_sk_conn_proto_rcv(struct tipc_sock *tsk, struct sk_buff *skb, goto exit; was_cong =tsk_conn_cong (tsk); -tipc_sk_push_backlog (tsk,msg_nagle_ack (hdr)); +tipc_sk_push_backlog (tsk,msg_nagle_ack (hdr), xmitq); tsk->snt_unacked -=msg_conn_ack (hdr);if (tsk->peer_caps &TIPC_BLOCK_FLOWCTL ) tsk->snd_win =msg_adv_win (hdr);
@@ -2166,7 +2173,7 @@ static void tipc_sk_proto_rcv(struct sock *sk,smp_wmb (); tsk->cong_link_cnt--; wakeup = true; -tipc_sk_push_backlog (tsk, false); +tipc_sk_push_backlog (tsk, false, xmitq);break ;case GROUP_PROTOCOL :tipc_group_proto_rcv (grp, &wakeup, hdr, inputq, xmitq);
@@ -2257,7 +2264,7 @@ static bool tipc_sk_filter_connect(struct tipc_sock *tsk, struct sk_buff *skb,return false;case TIPC_ESTABLISHED :if (!skb_queue_empty (&sk->sk_write_queue)) -tipc_sk_push_backlog (tsk, false); +tipc_sk_push_backlog (tsk, false, xmitq);/* Accept only connection-based messages sent by peer */ if (likely (con_msg && !err && pport == oport && pnode == onode)) {
syzbot
syzbotForwarded: Re: [syzbot] [tipc?] BUG: soft lockup in do_sock_setsockopt
In reply to syzbotFor archival purposes, forwarding an incoming command email to [email protected].
For archival purposes, forwarding an incoming command email to [email protected].
***
Subject: Re: [syzbot] [tipc?] BUG: soft lockup in do_sock_setsockopt
Author: [email protected]
#syz test: https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git 6430fb7f1cc4e329e386521b7b208df443de92f2
net/tipc/socket.c +38 -4
@@ -94,6 +94,7 @@ struct sockaddr_pair {* @peer: 'connected' peer for dgram/rdm * @node: hash table node * @mc_method: cookie for use between socket and broadcast layer +* @backlog_work: processes receive backlog under socket ownership * @rcu: rcu struct for tipc_sock * @group: TIPC communications group * @oneway: message count in one direction (FIXME)
@@ -128,6 +129,7 @@ struct tipc_sock {struct sockaddr_tipc peer;struct rhash_head node;struct tipc_mc_method mc_method; +struct work_struct backlog_work;struct rcu_head rcu;struct tipc_group *group;u32 oneway;
@@ -143,6 +145,7 @@ struct tipc_sock { };static int tipc_sk_backlog_rcv (struct sock *sk,struct sk_buff *skb); +static void tipc_sk_backlog_work (struct work_struct *work);static void tipc_data_ready (struct sock *sk);static void tipc_write_space (struct sock *sk);static void tipc_sock_destruct (struct sock *sk);
@@ -499,6 +502,7 @@ static int tipc_sk_create(struct net *net, struct socket *sock,/* Finish initializing socket data structures */ sock->ops = ops;sock_init_data (sock, sk); +INIT_WORK (&tsk->backlog_work, tipc_sk_backlog_work);tipc_set_sk_state (sk,TIPC_OPEN );if (tipc_sk_insert (tsk)) {sk_free (sk);
@@ -2428,6 +2432,17 @@ static int tipc_sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)return 0 ; } +static void tipc_sk_backlog_work (struct work_struct *work) +{ + struct tipc_sock *tsk = container_of(work, struct tipc_sock, + backlog_work); +struct sock *sk = &tsk->sk; + +lock_sock (sk); +release_sock (sk); +sock_put (sk); +} +/** * tipc_sk_enqueue - extract all buffers with destination 'dport' from * inputq and try adding them to socket or backlog queue
@@ -2445,18 +2460,30 @@ static void tipc_sk_enqueue(struct sk_buff_head *inputq, struct sock *sk,struct sk_buff *skb;unsigned int lim;atomic_t *dcnt; +bool deferred = false; +bool owned; +bool local_backlog;u32 onode;while (skb_queue_len (inputq)) {if (unlikely (time_after_eq (jiffies, time_limit))) -return ; +break ; skb =tipc_skb_dequeue (inputq, dport);if (unlikely (!skb)) -return ; +break ; -/* Add message directly to receive queue if possible */ -if (!sock_owned_by_user (sk)) { +/* +* A local Nagle backlog may loop back into this socket while its +* spinlock is held. Queue the triggering input for processing under +* socket ownership, which keeps a concurrent sender from overtaking +* the already queued output. +*/ + owned =sock_owned_by_user (sk); + local_backlog = !owned && !skb_queue_empty (&sk->sk_write_queue) && +in_own_node (sock_net (sk), +tsk_peer_node (tipc_sk (sk))); +if (!owned && !local_backlog) {tipc_sk_filter_rcv (sk, skb, xmitq);continue ; }
@@ -2467,6 +2494,7 @@ static void tipc_sk_enqueue(struct sk_buff_head *inputq, struct sock *sk,atomic_set (dcnt,0 ); lim =rcvbuf_limit (sk, skb) +atomic_read (dcnt);if (likely (!sk_add_backlog (sk, skb, lim))) { + deferred |= !owned;trace_tipc_sk_overlimit1 (sk, skb,TIPC_DUMP_SK_BKLGQ ,"bklg & rcvq >90% allocated!" );continue ;
@@ -2483,6 +2511,12 @@ static void tipc_sk_enqueue(struct sk_buff_head *inputq, struct sock *sk, }break ; } + +if (deferred) { +sock_hold (sk); +if (!schedule_work (&tipc_sk (sk)->backlog_work)) +sock_put (sk); + } }/**