net/smc: Avoid setting clcsock options after clcsock released
Wen Gu
reviewerPATCH 1/1 net/smc: Avoid setting clcsock options after clcsock released
We encountered a crash in smc_setsockopt() and it is caused by accessing smc->clcsock after clcsock was released.
BUG: kernel NULL pointer dereference, address: 0000000000000020 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 0 P4D 0 Oops: 0000 [#1] PREEMPT SMP PTI CPU: 1 PID: 50309 Comm: nginx Kdump: loaded Tainted: G E 5.16.0-rc4+ #53 RIP: 0010:smc_setsockopt+0x59/0x280 [smc] Call Trace: <TASK> __sys_setsockopt+0xfc/0x190 __x64_sys_setsockopt+0x20/0x30 do_syscall_64+0x34/0x90 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7f16ba83918e </TASK>
This patch tries to fix it by holding clcsock_release_lock and checking whether clcsock has already been released. In case that a crash of the same reason happens in smc_getsockopt(), this patch also checkes smc->clcsock in smc_getsockopt().
--- net/smc/af_smc.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-)
net/smc/af_smc.c +15 -1
@@ -2441,6 +2441,11 @@ static int smc_setsockopt(struct socket *sock, int level, int optname,/* generic setsockopts reaching us here always apply to the * CLC socket */ +mutex_lock (&smc->clcsock_release_lock); +if (!smc->clcsock) { +mutex_unlock (&smc->clcsock_release_lock); +return -EBADF ; + }if (unlikely (!smc->clcsock->ops->setsockopt)) rc = -EOPNOTSUPP ;else
@@ -2450,6 +2455,7 @@ static int smc_setsockopt(struct socket *sock, int level, int optname, sk->sk_err = smc->clcsock->sk->sk_err;sk_error_report (sk); } +mutex_unlock (&smc->clcsock_release_lock);if (optlen <sizeof (int ))return -EINVAL ;
@@ -2509,13 +2515,21 @@ static int smc_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen) { struct smc_sock *smc; +int rc; smc =smc_sk (sock->sk); +mutex_lock (&smc->clcsock_release_lock); +if (!smc->clcsock) { +mutex_unlock (&smc->clcsock_release_lock); +return -EBADF ; + }/* socket options apply to the CLC socket */ if (unlikely (!smc->clcsock->ops->getsockopt))return -EOPNOTSUPP ; -return smc->clcsock->ops->getsockopt (smc->clcsock, level, optname, + rc = smc->clcsock->ops->getsockopt (smc->clcsock, level, optname, optval, optlen); +mutex_unlock (&smc->clcsock_release_lock); +return rc; } static int smc_ioctl(struct socket *sock, unsigned int cmd,
Signature · 1 line
1.8.3.1
Quoted 46 lines · Wen Gu
We encountered a crash in
smc_setsockopt()and it is caused by accessingsmc->clcsockafter clcsock was released.BUG: kernel NULL pointer dereference, address: 0000000000000020 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 0 P4D 0 Oops: 0000 [#1] PREEMPT SMP PTI CPU: 1 PID: 50309 Comm: nginx Kdump: loaded Tainted: G E 5.16.0-rc4+ #53 RIP: 0010:smc_setsockopt+0x59/0x280 [smc] Call Trace: <TASK> __sys_setsockopt+0xfc/0x190 __x64_sys_setsockopt+0x20/0x30 do_syscall_64+0x34/0x90 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7f16ba83918e </TASK>This patch tries to fix it by holding
clcsock_release_lockand checking whether clcsock has already been released. In case that a crash of the same reason happens insmc_getsockopt(), this patch also checkessmc->clcsockinsmc_getsockopt().Signed-off-by Wen Gu <[email protected]>--- net/smc/af_smc.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-)net/smc/af_smc.c +5
-0@@ -2441,6 +2441,11 @@ static int smc_setsockopt(struct socket *sock, int level, int optname,/* generic setsockopts reaching us here always apply to the * CLC socket */ +mutex_lock (&smc->clcsock_release_lock); +if (!smc->clcsock) { +mutex_unlock (&smc->clcsock_release_lock); +return -EBADF ; + }if (unlikely (!smc->clcsock->ops->setsockopt)) rc = -EOPNOTSUPP ;else
net/smc/af_smc.c static int smc_setsockopt(struct socket *sock, int level, int optname, · line 2458
Quoted 4 lines
net/smc/af_smc.c +1
-0@@ -2450,6 +2455,7 @@ static int smc_setsockopt(struct socket *sock, int level, int optname, sk->sk_err = smc->clcsock->sk->sk_err;sk_error_report (sk); } +mutex_unlock (&smc->clcsock_release_lock);
In the switch() the function smc_switch_to_fallback() might be called which also accesses smc->clcsock without further checking. This should also be protected then? Also from all callers of smc_switch_to_fallback() ?
There are more uses of smc->clcsock (e.g. smc_bind(), ...), so why does this problem happen in setsockopt() for you only? I suspect it depends on the test case.
I wonder if it makes sense to check and protect smc->clcsock at all places in the code where it is used... as of now we had no such races like you encountered. But I see that in theory this problem could also happen in other code areas.
Quoted 26 lines
net/smc/af_smc.c +9
-1if (optlen <sizeof (int ))return -EINVAL ;@@ -2509,13 +2515,21 @@ static int smc_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen) { struct smc_sock *smc; +int rc; smc =smc_sk (sock->sk); +mutex_lock (&smc->clcsock_release_lock); +if (!smc->clcsock) { +mutex_unlock (&smc->clcsock_release_lock); +return -EBADF ; + }/* socket options apply to the CLC socket */ if (unlikely (!smc->clcsock->ops->getsockopt))return -EOPNOTSUPP ; -return smc->clcsock->ops->getsockopt (smc->clcsock, level, optname, + rc = smc->clcsock->ops->getsockopt (smc->clcsock, level, optname, optval, optlen); +mutex_unlock (&smc->clcsock_release_lock); +return rc; } static int smc_ioctl(struct socket *sock, unsigned int cmd,
Signature · 1 line
Karsten
Wen Gu
reviewerThanks for your reply.
Quoted 5 lines · Karsten Graul
Quoted 3 lines · Wen Gu
We encountered a crash in
smc_setsockopt()and it is caused by accessingsmc->clcsockafter clcsock was released.
Quoted 7 lines
In the
switch()the functionsmc_switch_to_fallback()might be called which also accessessmc->clcsockwithout further checking. This should also be protected then? Also from all callers ofsmc_switch_to_fallback()?There are more uses of
smc->clcsock(e.g.smc_bind(), ...), so why does this problem happen insetsockopt()for you only? I suspect it depends on the test case.
Yes, it depends on the test case. The crash described here only happens one time when I run a stress test of nginx/wrk, accompanied with frequent RNIC up/down operations.
Considering accessing smc->clcsock after its release is an uncommon, low probability issue and only happens in setsockopt() in my test, I choce an simple way to fix it, using the existing clcsock_release_lock, and only check in smc_setsockopt() and smc_getsockopt().
Quoted 4 lines
I wonder if it makes sense to check and protect
smc->clcsockat all places in the code where it is used... as of now we had no such races like you encountered. But I see that in theory this problem could also happen in other code areas.
But inspired by your questions, I think maybe we should treat the race as a general problem? Do you think it is necessary to find all the potential race related to the clcsock release and fix them in a unified approach? like define smc->clcsock as RCU pointer, hold rcu read lock before accessing smc->clcsock and call synchronize_rcu() before resetting smc->clcsock? just a rough idea :)
Or we should decide it later, do some more tests to see the probability of recurrence of this problem?
Thanks,
Wen Gu
Quoted 28 lines · Wen Gu
Thanks for your reply.
Quoted 5 lines · Karsten Graul
Quoted 3 lines · Wen Gu
We encountered a crash in
smc_setsockopt()and it is caused by accessingsmc->clcsockafter clcsock was released.Quoted 7 lines
In the
switch()the functionsmc_switch_to_fallback()might be called which also accessessmc->clcsockwithout further checking. This should also be protected then? Also from all callers ofsmc_switch_to_fallback()?There are more uses of
smc->clcsock(e.g.smc_bind(), ...), so why does this problem happen insetsockopt()for you only? I suspect it depends on the test case.Yes, it depends on the test case. The crash described here only happens one time when I run a stress test of nginx/wrk, accompanied with frequent RNIC up/down operations.
Considering accessing
smc->clcsockafter its release is an uncommon, low probability issue and only happens insetsockopt()in my test, I choce an simple way to fix it, using the existingclcsock_release_lock, and only check insmc_setsockopt()andsmc_getsockopt().Quoted 4 lines
I wonder if it makes sense to check and protect
smc->clcsockat all places in the code where it is used... as of now we had no such races like you encountered. But I see that in theory this problem could also happen in other code areas.
Quoted 6 lines
But inspired by your questions, I think maybe we should treat the race as a general problem? Do you think it is necessary to find all the potential race related to the clcsock release and fix them in a unified approach? like define
smc->clcsockas RCU pointer, hold rcu read lock before accessingsmc->clcsockand callsynchronize_rcu()before resettingsmc->clcsock? just a rough idea :)Or we should decide it later, do some more tests to see the probability of recurrence of this problem?
I like the idea to use RCU with rcu_assign_pointer() to protect that pointer!
Lets go with your initial patch (improved to address the access in smc_switch_to_fallback()) for now because it solves your current problem.
I put that RCU thing on our list, but if either of us here starts working on that please let the others know so we don't end up doing parallel work on this. But I doubt that we will be able to start working on that soon.
Thanks for the good idea!
Wen Gu
reviewerThanks for your reply.
Quoted 4 lines · Karsten Graul
Quoted 3 lines · Wen Gu
Thanks for your reply.
On 2022/1/11 6:03 pm, Karsten Graul wrote:
Quoted 9 lines
Quoted 3 lines
Quoted 3 lines · Wen Gu
Quoted 2 lines
We encountered a crash in
smc_setsockopt()and it is caused by accessingsmc->clcsockafter clcsock was released.I like the idea to use RCU with
rcu_assign_pointer()to protect that pointer!Lets go with your initial patch (improved to address the access in
smc_switch_to_fallback()) for now because it solves your current problem.
OK, I will improve the patch, adding check before clcsock access in smc_switch_to_fallback() and return an error (-EBADF) if smc->clcsock is NULL. The caller of smc_switch_to_fallback() will check the return value to identify whether fallback is possible.
Quoted 5 lines
I put that RCU thing on our list, but if either of us here starts working on that please let the others know so we don't end up doing parallel work on this. But I doubt that we will be able to start working on that soon.
Thanks for the good idea!
Thank you! If I start working on the RCU things, I will send a RFC to let you know.
Thanks,
Wen Gu
Wen Gu
reviewerQuoted 9 lines · Karsten Graul
Quoted 8 lines · Wen Gu
Thanks for your reply.
Quoted 5 lines · Karsten Graul
Quoted 3 lines · Wen Gu
We encountered a crash in
smc_setsockopt()and it is caused by accessingsmc->clcsockafter clcsock was released.
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
In the
switch()the functionsmc_switch_to_fallback()might be called which also accessessmc->clcsockwithout further checking. This should also be protected then? Also from all callers ofsmc_switch_to_fallback()?Lets go with your initial patch (improved to address the access in
smc_switch_to_fallback()) for now because it solves your current problem.
Since the upcoming v2 patch added clcsock check in smc_switch_to_fallback(), the original subject is inappropriate.
So I sent a new patch named 'net/smc: Transitional solution for clcsock race issue' instead of a v2.
Thanks,
Wen Gu
Jakub Kicinski
maintainernet/smc/af_smc.c static int smc_getsockopt(struct socket *sock, int level, int optname, · line 2530
Quoted 3 lines · On Mon, 10 Jan 2022 17:38:25 +0800 Wen Gu wrote:
net/smc/af_smc.c +1
-1-return smc->clcsock->ops->getsockopt (smc->clcsock, level, optname, + rc = smc->clcsock->ops->getsockopt (smc->clcsock, level, optname, optval, optlen);
Please do realign the continuation line when moving the opening bracket.
Wen Gu
reviewerQuoted 6 lines · Jakub Kicinski
net/smc/af_smc.c static int smc_getsockopt(struct socket *sock, int level, int optname, · line 2530
Quoted 3 lines · On Mon, 10 Jan 2022 17:38:25 +0800 Wen Gu wrote:
net/smc/af_smc.c +1
-1-return smc->clcsock->ops->getsockopt (smc->clcsock, level, optname, + rc = smc->clcsock->ops->getsockopt (smc->clcsock, level, optname, optval, optlen);Please do realign the continuation line when moving the opening bracket.
Thanks for pointing this out. Will fix it.
Thanks,
Wen Gu
dust.li
maintainerQuoted 57 lines · Wen Gu
We encountered a crash in
smc_setsockopt()and it is caused by accessingsmc->clcsockafter clcsock was released.BUG: kernel NULL pointer dereference, address: 0000000000000020 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 0 P4D 0 Oops: 0000 [#1] PREEMPT SMP PTI CPU: 1 PID: 50309 Comm: nginx Kdump: loaded Tainted: G E 5.16.0-rc4+ #53 RIP: 0010:smc_setsockopt+0x59/0x280 [smc] Call Trace: <TASK> __sys_setsockopt+0xfc/0x190 __x64_sys_setsockopt+0x20/0x30 do_syscall_64+0x34/0x90 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7f16ba83918e </TASK>This patch tries to fix it by holding
clcsock_release_lockand checking whether clcsock has already been released. In case that a crash of the same reason happens insmc_getsockopt(), this patch also checkessmc->clcsockinsmc_getsockopt().Signed-off-by Wen Gu <[email protected]>---
net/smc/af_smc.c| 16 +++++++++++++++-1 file changed, 15 insertions(+), 1 deletion(-)net/smc/af_smc.c +6
-0@@ -2441,6 +2441,11 @@ static int smc_setsockopt(struct socket *sock, int level, int optname,/* generic setsockopts reaching us here always apply to the * CLC socket */ +mutex_lock (&smc->clcsock_release_lock); +if (!smc->clcsock) { +mutex_unlock (&smc->clcsock_release_lock); +return -EBADF ; + }if (unlikely (!smc->clcsock->ops->setsockopt)) rc = -EOPNOTSUPP ;else @@ -2450,6 +2455,7 @@ static int smc_setsockopt(struct socket *sock, int level, int optname, sk->sk_err = smc->clcsock->sk->sk_err;sk_error_report (sk); } +mutex_unlock (&smc->clcsock_release_lock);if (optlen <sizeof (int ))return -EINVAL ;@@ -2509,13 +2515,21 @@ static int smc_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen) { struct smc_sock *smc;
net/smc/af_smc.c static int smc_getsockopt(struct socket *sock, int level, int optname, · line 2527
Quoted 10 lines
net/smc/af_smc.c +6
-0+int rc; smc =smc_sk (sock->sk); +mutex_lock (&smc->clcsock_release_lock); +if (!smc->clcsock) { +mutex_unlock (&smc->clcsock_release_lock); +return -EBADF ; + }/* socket options apply to the CLC socket */ if (unlikely (!smc->clcsock->ops->getsockopt))
Missed a mutex_unlock() here ?
Quoted 12 lines
net/smc/af_smc.c +3
-1return -EOPNOTSUPP ; -return smc->clcsock->ops->getsockopt (smc->clcsock, level, optname, + rc = smc->clcsock->ops->getsockopt (smc->clcsock, level, optname, optval, optlen); +mutex_unlock (&smc->clcsock_release_lock); +return rc; }static int smc_ioctl(struct socket *sock, unsigned int cmd,
Signature · 1 line
1.8.3.1
Wen Gu
reviewerQuoted 13 lines · dust.li
Quoted 12 lines · Wen Gu
This patch tries to fix it by holding
clcsock_release_lockand checking whether clcsock has already been released. In case that a crash of the same reason happens insmc_getsockopt(), this patch also checkessmc->clcsockinsmc_getsockopt().net/smc/af_smc.c +1
-0@@ -2509,13 +2515,21 @@ static int smc_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen) { struct smc_sock *smc; +int rc;
Quoted 11 lines
net/smc/af_smc.c static int smc_getsockopt(struct socket *sock, int level, int optname, · line 2527
Quoted 8 lines
net/smc/af_smc.c +5
-0smc =smc_sk (sock->sk); +mutex_lock (&smc->clcsock_release_lock); +if (!smc->clcsock) { +mutex_unlock (&smc->clcsock_release_lock); +return -EBADF ; + }/* socket options apply to the CLC socket */ if (unlikely (!smc->clcsock->ops->getsockopt))Missed a
mutex_unlock()here ?Quoted 1 line
net/smc/af_smc.c +0
-0return -EOPNOTSUPP ;
Thanks for pointing it out. Will add an additional mutex_unlock().
Thanks,
Wen Gu