misc: bcm_vk: Fix potential deadlock on &vk->ctx_lock
Subsystems (2)
PATCH v3 1/1 misc: bcm_vk: Fix potential deadlock on &vk->ctx_lock
As &vk->ctx_lock is acquired by timer bcm_vk_hb_poll() under softirq context, other process context code should disable irq or bottom-half before acquire the same lock, otherwise deadlock could happen if the timer preempt the execution while the lock is held in process context on the same CPU.
Possible deadlock scenariobcm_vk_open()
-> bcm_vk_get_ctx()
-> spin_lock(&vk->ctx_lock)
<timer iterrupt>
-> bcm_vk_hb_poll()
-> bcm_vk_blk_drv_access()
-> spin_lock_irqsave(&vk->ctx_lock, flags) (deadlock here)This flaw was found using an experimental static analysis tool we are developing for irq-related deadlock, which reported the following warning when analyzing the linux kernel 6.4-rc7 release.
[Deadlock]: &vk->ctx_lock
[Interrupt]: bcm_vk_hb_poll
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512
[Locking Unit]: bcm_vk_ioctl
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1181
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512[Deadlock]: &vk->ctx_lock
[Interrupt]: bcm_vk_hb_poll
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512
[Locking Unit]: bcm_vk_ioctl
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1169[Deadlock]: &vk->ctx_lock
[Interrupt]: bcm_vk_hb_poll
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512
[Locking Unit]: bcm_vk_open
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:216[Deadlock]: &vk->ctx_lock
[Interrupt]: bcm_vk_hb_poll
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512
[Locking Unit]: bcm_vk_release
-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:306As suggested by Arnd, the tentative patch fix the potential deadlocks by replacing the timer with delay workqueue. x86_64 allyesconfig using GCC shows no new warning. Note that no runtime testing was performed due to no device on hand.
--- drivers/misc/bcm-vk/bcm_vk.h | 2 +- drivers/misc/bcm-vk/bcm_vk_msg.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-)
drivers/misc/bcm-vk/bcm_vk.h +1 -1
@@ -340,7 +340,7 @@ struct bcm_vk_proc_mon_info { };struct bcm_vk_hb_ctrl { -struct timer_list timer; +struct delayed_work work;u32 last_uptime;u32 lost_cnt; };
drivers/misc/bcm-vk/bcm_vk_msg.c +7 -7
@@ -137,11 +137,11 @@ void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask)#define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ)#define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S) -static void bcm_vk_hb_poll (struct timer_list *t) +static void bcm_vk_hb_poll (struct work_struct *work) {u32 uptime_s; - struct bcm_vk_hb_ctrl *hb = container_of(t, struct bcm_vk_hb_ctrl, - timer); + struct bcm_vk_hb_ctrl *hb = container_of(to_delayed_work(work), struct bcm_vk_hb_ctrl, + work); struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl);if (bcm_vk_drv_access_ok (vk) &&hb_mon_is_on ()) {
@@ -177,22 +177,22 @@ static void bcm_vk_hb_poll(struct timer_list *t)bcm_vk_set_host_alert (vk,ERR_LOG_HOST_HB_FAIL ); }/* re-arm timer */ -mod_timer (&hb->timer, jiffies +BCM_VK_HB_TIMER_VALUE ); +schedule_delayed_work (&hb->work,BCM_VK_HB_TIMER_VALUE ); }void bcm_vk_hb_init (struct bcm_vk *vk) {struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; -timer_setup (&hb->timer, bcm_vk_hb_poll,0 ); -mod_timer (&hb->timer, jiffies +BCM_VK_HB_TIMER_VALUE ); +INIT_DELAYED_WORK (&hb->work, bcm_vk_hb_poll); +schedule_delayed_work (&hb->work,BCM_VK_HB_TIMER_VALUE ); }void bcm_vk_hb_deinit (struct bcm_vk *vk) {struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; -del_timer (&hb->timer); +cancel_delayed_work_sync (&hb->work); } static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk,
Signature · 1 line
2.17.1
Scott Branden
maintainerHi Chengfeng,
Thanks for the work on this. We need some time to review and test.
Quoted 118 lines · Chengfeng Ye
As
&vk->ctx_lockis acquired by timerbcm_vk_hb_poll()under softirq context, other process context code should disable irq or bottom-half before acquire the same lock, otherwise deadlock could happen if the timer preempt the execution while the lock is held in process context on the same CPU.Possible deadlock scenario
bcm_vk_open()-> bcm_vk_get_ctx() -> spin_lock(&vk->ctx_lock) <timer iterrupt> -> bcm_vk_hb_poll() -> bcm_vk_blk_drv_access() -> spin_lock_irqsave(&vk->ctx_lock, flags) (deadlock here)This flaw was found using an experimental static analysis tool we are developing for irq-related deadlock, which reported the following warning when analyzing the linux kernel 6.4-rc7 release.
[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_ioctl -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1181 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_ioctl -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1169[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_open -->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:216[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_release -->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:306As suggested by Arnd, the tentative patch fix the potential deadlocks by replacing the timer with delay workqueue.
x86_64allyesconfig using GCC shows no new warning. Note that no runtime testing was performed due to no device on hand.Signed-off-by Chengfeng Ye <[email protected]>--- drivers/misc/bcm-vk/bcm_vk.h | 2 +- drivers/misc/bcm-vk/bcm_vk_msg.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-)drivers/misc/bcm-vk/bcm_vk.h +1
-1@@ -340,7 +340,7 @@ struct bcm_vk_proc_mon_info { };struct bcm_vk_hb_ctrl { -struct timer_list timer; +struct delayed_work work;u32 last_uptime;u32 lost_cnt; };drivers/misc/bcm-vk/bcm_vk_msg.c +7
-7@@ -137,11 +137,11 @@ void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask)#define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ)#define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S) -static void bcm_vk_hb_poll (struct timer_list *t) +static void bcm_vk_hb_poll (struct work_struct *work) {u32 uptime_s; - struct bcm_vk_hb_ctrl *hb = container_of(t, struct bcm_vk_hb_ctrl, - timer); + struct bcm_vk_hb_ctrl *hb = container_of(to_delayed_work(work), struct bcm_vk_hb_ctrl, + work); struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl);if (bcm_vk_drv_access_ok (vk) &&hb_mon_is_on ()) {@@ -177,22 +177,22 @@ static void bcm_vk_hb_poll(struct timer_list *t)bcm_vk_set_host_alert (vk,ERR_LOG_HOST_HB_FAIL ); }/* re-arm timer */ -mod_timer (&hb->timer, jiffies +BCM_VK_HB_TIMER_VALUE ); +schedule_delayed_work (&hb->work,BCM_VK_HB_TIMER_VALUE ); }void bcm_vk_hb_init (struct bcm_vk *vk) {struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; -timer_setup (&hb->timer, bcm_vk_hb_poll,0 ); -mod_timer (&hb->timer, jiffies +BCM_VK_HB_TIMER_VALUE ); +INIT_DELAYED_WORK (&hb->work, bcm_vk_hb_poll); +schedule_delayed_work (&hb->work,BCM_VK_HB_TIMER_VALUE ); }void bcm_vk_hb_deinit (struct bcm_vk *vk) {struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; -del_timer (&hb->timer); +cancel_delayed_work_sync (&hb->work); } static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk,
Works fine - thanks.
Quoted 48 lines · Chengfeng Ye
As
&vk->ctx_lockis acquired by timerbcm_vk_hb_poll()under softirq context, other process context code should disable irq or bottom-half before acquire the same lock, otherwise deadlock could happen if the timer preempt the execution while the lock is held in process context on the same CPU.Possible deadlock scenario
bcm_vk_open()-> bcm_vk_get_ctx() -> spin_lock(&vk->ctx_lock) <timer iterrupt> -> bcm_vk_hb_poll() -> bcm_vk_blk_drv_access() -> spin_lock_irqsave(&vk->ctx_lock, flags) (deadlock here)This flaw was found using an experimental static analysis tool we are developing for irq-related deadlock, which reported the following warning when analyzing the linux kernel 6.4-rc7 release.
[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_ioctl -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1181 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_ioctl -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1169[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_open -->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:216[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_release -->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:306
Quoted 6 lines
As suggested by Arnd, the tentative patch fix the potential deadlocks by replacing the timer with delay workqueue.
x86_64allyesconfig using GCC shows no new warning. Note that no runtime testing was performed due to no device on hand.Signed-off-by Chengfeng Ye <[email protected]>
Quoted 64 lines
--- drivers/misc/bcm-vk/bcm_vk.h | 2 +- drivers/misc/bcm-vk/bcm_vk_msg.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-)drivers/misc/bcm-vk/bcm_vk.h +1
-1@@ -340,7 +340,7 @@ struct bcm_vk_proc_mon_info { };struct bcm_vk_hb_ctrl { -struct timer_list timer; +struct delayed_work work;u32 last_uptime;u32 lost_cnt; };drivers/misc/bcm-vk/bcm_vk_msg.c +7
-7@@ -137,11 +137,11 @@ void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask)#define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ)#define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S) -static void bcm_vk_hb_poll (struct timer_list *t) +static void bcm_vk_hb_poll (struct work_struct *work) {u32 uptime_s; - struct bcm_vk_hb_ctrl *hb = container_of(t, struct bcm_vk_hb_ctrl, - timer); + struct bcm_vk_hb_ctrl *hb = container_of(to_delayed_work(work), struct bcm_vk_hb_ctrl, + work); struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl);if (bcm_vk_drv_access_ok (vk) &&hb_mon_is_on ()) {@@ -177,22 +177,22 @@ static void bcm_vk_hb_poll(struct timer_list *t)bcm_vk_set_host_alert (vk,ERR_LOG_HOST_HB_FAIL ); }/* re-arm timer */ -mod_timer (&hb->timer, jiffies +BCM_VK_HB_TIMER_VALUE ); +schedule_delayed_work (&hb->work,BCM_VK_HB_TIMER_VALUE ); }void bcm_vk_hb_init (struct bcm_vk *vk) {struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; -timer_setup (&hb->timer, bcm_vk_hb_poll,0 ); -mod_timer (&hb->timer, jiffies +BCM_VK_HB_TIMER_VALUE ); +INIT_DELAYED_WORK (&hb->work, bcm_vk_hb_poll); +schedule_delayed_work (&hb->work,BCM_VK_HB_TIMER_VALUE ); }void bcm_vk_hb_deinit (struct bcm_vk *vk) {struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; -del_timer (&hb->timer); +cancel_delayed_work_sync (&hb->work); } static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk,
Quoted 1 line · Scott Branden
Works fine - thanks.
So apparently the choice of using the timer previously was not due to performance reasons?
If performance is a concern by converting to use workqueue (now it runs in process/thread context than softirq), I assume you are aware of another easy way to fix this potential deadlock issue? :)
Quoted 118 lines · Chengfeng Ye
Quoted 48 lines
As
&vk->ctx_lockis acquired by timerbcm_vk_hb_poll()under softirq context, other process context code should disable irq or bottom-half before acquire the same lock, otherwise deadlock could happen if the timer preempt the execution while the lock is held in process context on the same CPU.Possible deadlock scenario
bcm_vk_open()
->bcm_vk_get_ctx()
->spin_lock(&vk->ctx_lock)
<timer iterrupt>
->bcm_vk_hb_poll()
->bcm_vk_blk_drv_access()
->spin_lock_irqsave(&vk->ctx_lock, flags)(deadlock here)This flaw was found using an experimental static analysis tool we are developing for irq-related deadlock, which reported the following warning when analyzing the linux kernel 6.4-rc7 release.
[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]:bcm_vk_ioctl-->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1181 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]:bcm_vk_ioctl-->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1169[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]:bcm_vk_open-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:216[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]:bcm_vk_release-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:306Quoted 6 lines
As suggested by Arnd, the tentative patch fix the potential deadlocks by replacing the timer with delay workqueue.
x86_64allyesconfig using GCC shows no new warning. Note that no runtime testing was performed due to no device on hand.Signed-off-by Chengfeng Ye <[email protected]>Acked-by Scott Branden <[email protected]>Tested-by Desmond Yan <[email protected]>Quoted 59 lines
---
drivers/misc/bcm-vk/bcm_vk.h | 2 +-drivers/misc/bcm-vk/bcm_vk_msg.c| 14 +++++++-------2 files changed, 8 insertions(+), 8 deletions(-)drivers/misc/bcm-vk/bcm_vk.h +1
-1@@ -340,7 +340,7 @@ struct bcm_vk_proc_mon_info { }; struct bcm_vk_hb_ctrl { - struct timer_list timer; + struct delayed_work work; u32 last_uptime; u32 lost_cnt; };drivers/misc/bcm-vk/bcm_vk_msg.c +3
-2@@ -137,11 +137,11 @@ void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask) #define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ) #define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S) -static void bcm_vk_hb_poll(struct timer_list *t) +static void bcm_vk_hb_poll(struct work_struct *work) { u32 uptime_s; - struct bcm_vk_hb_ctrl *hb = container_of(t, struct bcm_vk_hb_ctrl, - timer); + struct bcm_vk_hb_ctrl *hb = container_of(to_delayed_work(work), struct bcm_vk_hb_ctrl, + work); struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl);if (
bcm_vk_drv_access_ok(vk)&&hb_mon_is_on()) {drivers/misc/bcm-vk/bcm_vk_msg.c +4
-2@@ -177,22 +177,22 @@ static void bcm_vk_hb_poll(struct timer_list *t) bcm_vk_set_host_alert(vk, ERR_LOG_HOST_HB_FAIL); } /* re-arm timer */ - mod_timer(&hb->timer, jiffies + BCM_VK_HB_TIMER_VALUE); + schedule_delayed_work(&hb->work, BCM_VK_HB_TIMER_VALUE); } void bcm_vk_hb_init(struct bcm_vk *vk) { struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; - timer_setup(&hb->timer, bcm_vk_hb_poll, 0); - mod_timer(&hb->timer, jiffies + BCM_VK_HB_TIMER_VALUE); + INIT_DELAYED_WORK(&hb->work, bcm_vk_hb_poll); + schedule_delayed_work(&hb->work, BCM_VK_HB_TIMER_VALUE); } void bcm_vk_hb_deinit(struct bcm_vk *vk) { struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; - del_timer(&hb->timer); + cancel_delayed_work_sync(&hb->work); } static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk,
Scott Branden
maintainerQuoted 7 lines · Ray Jui
Quoted 1 line · Scott Branden
Works fine - thanks.
So apparently the choice of using the timer previously was not due to performance reasons?
Correct - the heartbeat does not need high performance.
Quoted 4 lines
If performance is a concern by converting to use workqueue (now it runs in process/thread context than softirq), I assume you are aware of another easy way to fix this potential deadlock issue? :)
Performance not a concern.
Quoted 118 lines
Quoted 118 lines · Chengfeng Ye
Quoted 48 lines
As
&vk->ctx_lockis acquired by timerbcm_vk_hb_poll()under softirq context, other process context code should disable irq or bottom-half before acquire the same lock, otherwise deadlock could happen if the timer preempt the execution while the lock is held in process context on the same CPU.Possible deadlock scenario
bcm_vk_open()
->bcm_vk_get_ctx()
->spin_lock(&vk->ctx_lock)
<timer iterrupt>
->bcm_vk_hb_poll()
->bcm_vk_blk_drv_access()
->spin_lock_irqsave(&vk->ctx_lock, flags)(deadlock here)This flaw was found using an experimental static analysis tool we are developing for irq-related deadlock, which reported the following warning when analyzing the linux kernel 6.4-rc7 release.
[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]:bcm_vk_ioctl-->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1181 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]:bcm_vk_ioctl-->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1169[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]:bcm_vk_open-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:216[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]:bcm_vk_release-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:306Quoted 6 lines
As suggested by Arnd, the tentative patch fix the potential deadlocks by replacing the timer with delay workqueue.
x86_64allyesconfig using GCC shows no new warning. Note that no runtime testing was performed due to no device on hand.Signed-off-by Chengfeng Ye <[email protected]>Acked-by Scott Branden <[email protected]>Tested-by Desmond Yan <[email protected]>Quoted 59 lines
--- drivers/misc/bcm-vk/bcm_vk.h | 2 +- drivers/misc/bcm-vk/bcm_vk_msg.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-)drivers/misc/bcm-vk/bcm_vk.h +1
-1@@ -340,7 +340,7 @@ struct bcm_vk_proc_mon_info { }; struct bcm_vk_hb_ctrl { - struct timer_list timer; + struct delayed_work work; u32 last_uptime; u32 lost_cnt; };drivers/misc/bcm-vk/bcm_vk_msg.c +7
-4@@ -137,11 +137,11 @@ void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask) #define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ) #define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S) -static void bcm_vk_hb_poll(struct timer_list *t) +static void bcm_vk_hb_poll(struct work_struct *work) { u32 uptime_s; - struct bcm_vk_hb_ctrl *hb = container_of(t, struct bcm_vk_hb_ctrl, - timer); + struct bcm_vk_hb_ctrl *hb = container_of(to_delayed_work(work), struct bcm_vk_hb_ctrl, + work); struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl); if (bcm_vk_drv_access_ok(vk) && hb_mon_is_on()) {@@ -177,22 +177,22 @@ static void bcm_vk_hb_poll(struct timer_list *t) bcm_vk_set_host_alert(vk, ERR_LOG_HOST_HB_FAIL); } /* re-arm timer */ - mod_timer(&hb->timer, jiffies + BCM_VK_HB_TIMER_VALUE); + schedule_delayed_work(&hb->work, BCM_VK_HB_TIMER_VALUE); } void bcm_vk_hb_init(struct bcm_vk *vk) { struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; - timer_setup(&hb->timer, bcm_vk_hb_poll, 0); - mod_timer(&hb->timer, jiffies + BCM_VK_HB_TIMER_VALUE); + INIT_DELAYED_WORK(&hb->work, bcm_vk_hb_poll); + schedule_delayed_work(&hb->work, BCM_VK_HB_TIMER_VALUE); } void bcm_vk_hb_deinit(struct bcm_vk *vk) { struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; - del_timer(&hb->timer); + cancel_delayed_work_sync(&hb->work); } static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk,
Quoted 3 lines
Works fine. Corrected Acked-by and Tested-by below.
Tested-by Desmond Yan <[email protected]>Acked-by Scott Branden <[email protected]>
Thanks much for your effort in reviewing and testing the patch.
Best Regards,
Chengfeng
Works fine. Corrected Acked-by and Tested-by below.
Quoted 48 lines · Chengfeng Ye
As
&vk->ctx_lockis acquired by timerbcm_vk_hb_poll()under softirq context, other process context code should disable irq or bottom-half before acquire the same lock, otherwise deadlock could happen if the timer preempt the execution while the lock is held in process context on the same CPU.Possible deadlock scenario
bcm_vk_open()-> bcm_vk_get_ctx() -> spin_lock(&vk->ctx_lock) <timer iterrupt> -> bcm_vk_hb_poll() -> bcm_vk_blk_drv_access() -> spin_lock_irqsave(&vk->ctx_lock, flags) (deadlock here)This flaw was found using an experimental static analysis tool we are developing for irq-related deadlock, which reported the following warning when analyzing the linux kernel 6.4-rc7 release.
[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_ioctl -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1181 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_ioctl -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:1169[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_open -->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:216[Deadlock]:
&vk->ctx_lock
[Interrupt]:bcm_vk_hb_poll-->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:176 -->/root/linux/drivers/misc/bcm-vk/bcm_vk_dev.c:512 [Locking Unit]: bcm_vk_release -->/root/linux/drivers/misc/bcm-vk/bcm_vk_msg.c:306
Quoted 6 lines
As suggested by Arnd, the tentative patch fix the potential deadlocks by replacing the timer with delay workqueue.
x86_64allyesconfig using GCC shows no new warning. Note that no runtime testing was performed due to no device on hand.Signed-off-by Chengfeng Ye <[email protected]>
Quoted 64 lines
--- drivers/misc/bcm-vk/bcm_vk.h | 2 +- drivers/misc/bcm-vk/bcm_vk_msg.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-)drivers/misc/bcm-vk/bcm_vk.h +1
-1@@ -340,7 +340,7 @@ struct bcm_vk_proc_mon_info { };struct bcm_vk_hb_ctrl { -struct timer_list timer; +struct delayed_work work;u32 last_uptime;u32 lost_cnt; };drivers/misc/bcm-vk/bcm_vk_msg.c +7
-7@@ -137,11 +137,11 @@ void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask)#define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ)#define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S) -static void bcm_vk_hb_poll (struct timer_list *t) +static void bcm_vk_hb_poll (struct work_struct *work) {u32 uptime_s; - struct bcm_vk_hb_ctrl *hb = container_of(t, struct bcm_vk_hb_ctrl, - timer); + struct bcm_vk_hb_ctrl *hb = container_of(to_delayed_work(work), struct bcm_vk_hb_ctrl, + work); struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl);if (bcm_vk_drv_access_ok (vk) &&hb_mon_is_on ()) {@@ -177,22 +177,22 @@ static void bcm_vk_hb_poll(struct timer_list *t)bcm_vk_set_host_alert (vk,ERR_LOG_HOST_HB_FAIL ); }/* re-arm timer */ -mod_timer (&hb->timer, jiffies +BCM_VK_HB_TIMER_VALUE ); +schedule_delayed_work (&hb->work,BCM_VK_HB_TIMER_VALUE ); }void bcm_vk_hb_init (struct bcm_vk *vk) {struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; -timer_setup (&hb->timer, bcm_vk_hb_poll,0 ); -mod_timer (&hb->timer, jiffies +BCM_VK_HB_TIMER_VALUE ); +INIT_DELAYED_WORK (&hb->work, bcm_vk_hb_poll); +schedule_delayed_work (&hb->work,BCM_VK_HB_TIMER_VALUE ); }void bcm_vk_hb_deinit (struct bcm_vk *vk) {struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; -del_timer (&hb->timer); +cancel_delayed_work_sync (&hb->work); } static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk,