watchdog: Fix the race between the release of watchdog_core_data and cdev
PATCH 1/1 watchdog: Fix the race between the release of watchdog_core_data and cdev
The struct cdev is embedded in the struct watchdog_core_data. In the
current code, we manage the watchdog_core_data with a kref, but the
cdev is manged by a kobject. There is no any relationship between
this kref and kobject. So it is possible that the watchdog_core_data is
freed before the cdev is entirely released. We can easily get the
following call trace with CONFIG_DEBUG_KOBJECT_RELEASE and CONFIG_DEBUG_OBJECTS_TIMERS enabled.
ODEBUG: free active (active state 0) object type: timer_list hint: delayed_work_timer_fn+0x0/0x38
WARNING: CPU: 23 PID: 1028 at lib/debugobjects.c:481 debug_print_object+0xb0/0xf0 Modules linked in: softdog(-) deflate ctr twofish_generic twofish_common camellia_generic serpent_generic blowfish_generic blowfish_common cast5_generic cast_common cmac xcbc af_key sch_fq_codel openvswitch nsh nf_conncount nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 CPU: 23 PID: 1028 Comm: modprobe Not tainted 5.3.0-next-20190924-yoctodev-standard+ #180 Hardware name: Marvell OcteonTX CN96XX board (DT) pstate: 00400009 (nzcv daif +PAN -UAO) pc : debug_print_object+0xb0/0xf0 lr : debug_print_object+0xb0/0xf0 sp : ffff80001cbcfc70 x29: ffff80001cbcfc70 x28: ffff800010ea2128 x27: ffff800010bad000 x26: 0000000000000000 x25: ffff80001103c640 x24: ffff80001107b268 x23: ffff800010bad9e8 x22: ffff800010ea2128 x21: ffff000bc2c62af8 x20: ffff80001103c600 x19: ffff800010e867d8 x18: 0000000000000060 x17: 0000000000000000 x16: 0000000000000000 x15: ffff000bd7240470 x14: 6e6968207473696c x13: 5f72656d6974203a x12: 6570797420746365 x11: 6a626f2029302065 x10: 7461747320657669 x9 : 7463612820657669 x8 : 3378302f3078302b x7 : 0000000000001d7a x6 : ffff800010fd5889 x5 : 0000000000000000 x4 : 0000000000000000 x3 : 0000000000000000 x2 : ffff000bff948548 x1 : 276a1c9e1edc2300 x0 : 0000000000000000 Call trace: debug_print_object+0xb0/0xf0 debug_check_no_obj_freed+0x1e8/0x210 kfree+0x1b8/0x368 watchdog_cdev_unregister+0x88/0xc8 watchdog_dev_unregister+0x38/0x48 watchdog_unregister_device+0xa8/0x100 softdog_exit+0x18/0xfec4 [softdog] __arm64_sys_delete_module+0x174/0x200 el0_svc_handler+0xd0/0x1c8 el0_svc+0x8/0xc
This is a common issue when using cdev embedded in a struct. Fortunately, we already have a mechanism to solve this kind of issue. Please see commit 233ed09d7fda ("chardev: add helper function to register char devs with a struct device") for more detail.
In this patch, we choose to embed the struct device into the watchdog_core_data, and use the API provided by the commit 233ed09d7fda to make sure that the release of watchdog_core_data and cdev are in sequence.
--- drivers/watchdog/watchdog_dev.c | 70 +++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 38 deletions(-)
drivers/watchdog/watchdog_dev.c +32 -38
@@ -34,7 +34,6 @@#include <linux/init.h> /* For __init/__exit/... */ #include <linux/hrtimer.h> /* For hrtimers */ #include <linux/kernel.h> /* For printk/panic/... */ -#include <linux/kref.h> /* For data references */ #include <linux/kthread.h> /* For kthread_work */ #include <linux/miscdevice.h> /* For handling misc devices */ #include <linux/module.h> /* For module stuff/... */
@@ -52,14 +51,14 @@/* * struct watchdog_core_data - watchdog core internal data -* @kref: Reference count. +* @dev: The watchdog's internal device * @cdev: The watchdog's Character device. * @wdd: Pointer to watchdog device. * @lock: Lock for watchdog core. * @status: Watchdog core internal status bits. */ struct watchdog_core_data { -struct kref kref; +struct device dev;struct cdev cdev;struct watchdog_device *wdd;struct mutex lock;
@@ -839,7 +838,7 @@ static int watchdog_open(struct inode *inode, struct file *file) file->private_data = wd_data;if (!hw_running) -kref_get (&wd_data->kref); +get_device (&wd_data->dev);/* * open_timeout only applies for the first open from
@@ -860,11 +859,11 @@ static int watchdog_open(struct inode *inode, struct file *file)return err; } -static void watchdog_core_data_release (struct kref *kref) +static void watchdog_core_data_release (struct device *dev) {struct watchdog_core_data *wd_data; - wd_data = container_of(kref, struct watchdog_core_data, kref); + wd_data = container_of(dev, struct watchdog_core_data, dev);kfree (wd_data); }
@@ -924,7 +923,7 @@ static int watchdog_release(struct inode *inode, struct file *file)*/ if (!running) {module_put (wd_data->cdev.owner); -kref_put (&wd_data->kref, watchdog_core_data_release); +put_device (&wd_data->dev); }return 0 ; }
@@ -943,17 +942,22 @@ static struct miscdevice watchdog_miscdev = { .fops = &watchdog_fops, }; +static struct class watchdog_class = { + .name ="watchdog" , + .owner =THIS_MODULE , + .dev_groups = wdt_groups, +}; +/* * watchdog_cdev_register: register watchdog character device * @wdd: watchdog device -* @devno: character device number * * Register a watchdog character device including handling the legacy * /dev/watchdog node. /dev/watchdog is actually a miscdevice and * thus we set it up like that. */ -static int watchdog_cdev_register (struct watchdog_device *wdd,dev_t devno) +static int watchdog_cdev_register (struct watchdog_device *wdd) {struct watchdog_core_data *wd_data;int err;
@@ -961,7 +965,6 @@ static int watchdog_cdev_register(struct watchdog_device *wdd, dev_t devno) wd_data =kzalloc (sizeof (struct watchdog_core_data ),GFP_KERNEL );if (!wd_data)return -ENOMEM ; -kref_init (&wd_data->kref);mutex_init (&wd_data->lock); wd_data->wdd = wdd;
@@ -990,23 +993,33 @@ static int watchdog_cdev_register(struct watchdog_device *wdd, dev_t devno) } } +device_initialize (&wd_data->dev); + wd_data->dev.devt =MKDEV (MAJOR (watchdog_devt), wdd->id); + wd_data->dev.class = &watchdog_class; + wd_data->dev.parent = wdd->parent; + wd_data->dev.groups = wdd->groups; + wd_data->dev.release = watchdog_core_data_release; +dev_set_drvdata (&wd_data->dev, wdd); +dev_set_name (&wd_data->dev,"watchdog%d" , wdd->id); +/* Fill in the data structures */ cdev_init (&wd_data->cdev, &watchdog_fops); - wd_data->cdev.owner = wdd->ops->owner;/* Add the device */ - err =cdev_add (&wd_data->cdev, devno,1 ); + err =cdev_device_add (&wd_data->cdev, &wd_data->dev);if (err) {pr_err ("watchdog%d unable to add device %d:%d\n" , wdd->id,MAJOR (watchdog_devt), wdd->id);if (wdd->id ==0 ) {misc_deregister (&watchdog_miscdev); old_wd_data =NULL ; -kref_put (&wd_data->kref, watchdog_core_data_release); +put_device (&wd_data->dev); }return err; } + wd_data->cdev.owner = wdd->ops->owner; +/* Record time of most recent heartbeat as 'just before now'. */ wd_data->last_hw_keepalive =ktime_sub (ktime_get (),1 );watchdog_set_open_deadline (wd_data);
@@ -1017,7 +1030,7 @@ static int watchdog_cdev_register(struct watchdog_device *wdd, dev_t devno)*/ if (watchdog_hw_running (wdd)) {__module_get (wdd->ops->owner); -kref_get (&wd_data->kref); +get_device (&wd_data->dev);if (handle_boot_enabled)hrtimer_start (&wd_data->timer,0 ,HRTIMER_MODE_REL );else
@@ -1040,7 +1053,7 @@ static void watchdog_cdev_unregister(struct watchdog_device *wdd) {struct watchdog_core_data *wd_data = wdd->wd_data; -cdev_del (&wd_data->cdev); +cdev_device_del (&wd_data->cdev, &wd_data->dev);if (wdd->id ==0 ) {misc_deregister (&watchdog_miscdev); old_wd_data =NULL ;
@@ -1059,15 +1072,9 @@ static void watchdog_cdev_unregister(struct watchdog_device *wdd)hrtimer_cancel (&wd_data->timer);kthread_cancel_work_sync (&wd_data->work); -kref_put (&wd_data->kref, watchdog_core_data_release); +put_device (&wd_data->dev); } -static struct class watchdog_class = { - .name ="watchdog" , - .owner =THIS_MODULE , - .dev_groups = wdt_groups, -}; -static int watchdog_reboot_notifier (struct notifier_block *nb,unsigned long code,void *data) {
@@ -1098,27 +1105,14 @@ static int watchdog_reboot_notifier(struct notifier_block *nb,int watchdog_dev_register (struct watchdog_device *wdd) { -struct device *dev; -dev_t devno;int ret; - devno =MKDEV (MAJOR (watchdog_devt), wdd->id); - - ret =watchdog_cdev_register (wdd, devno); + ret =watchdog_cdev_register (wdd);if (ret)return ret; - dev =device_create_with_groups (&watchdog_class, wdd->parent, - devno, wdd, wdd->groups, -"watchdog%d" , wdd->id); -if (IS_ERR (dev)) { -watchdog_cdev_unregister (wdd); -return PTR_ERR (dev); - } - ret =watchdog_register_pretimeout (wdd);if (ret) { -device_destroy (&watchdog_class, devno);watchdog_cdev_unregister (wdd);return ret; }
@@ -1126,7 +1120,8 @@ int watchdog_dev_register(struct watchdog_device *wdd)if (test_bit (WDOG_STOP_ON_REBOOT , &wdd->status)) { wdd->reboot_nb.notifier_call = watchdog_reboot_notifier; - ret =devm_register_reboot_notifier (dev, &wdd->reboot_nb); + ret =devm_register_reboot_notifier (&wdd->wd_data->dev, + &wdd->reboot_nb);if (ret) {pr_err ("watchdog%d: Cannot register reboot notifier (%d)\n" , wdd->id, ret);
@@ -1148,7 +1143,6 @@ int watchdog_dev_register(struct watchdog_device *wdd)void watchdog_dev_unregister (struct watchdog_device *wdd) {watchdog_unregister_pretimeout (wdd); -device_destroy (&watchdog_class, wdd->wd_data->cdev.dev);watchdog_cdev_unregister (wdd); }
Signature · 1 line
2.14.4
Quoted 48 lines · Kevin Hao
The struct cdev is embedded in the struct
watchdog_core_data. In the
current code, we manage thewatchdog_core_datawith a kref, but the
cdev is manged by a kobject. There is no any relationship between
this kref and kobject. So it is possible that thewatchdog_core_datais
freed before the cdev is entirely released. We can easily get the
following call trace withCONFIG_DEBUG_KOBJECT_RELEASEandCONFIG_DEBUG_OBJECTS_TIMERSenabled.
ODEBUG: free active (active state 0) object type:timer_listhint:delayed_work_timer_fn+0x0/0x38WARNING: CPU: 23 PID: 1028 at lib/debugobjects.c:481 debug_print_object+0xb0/0xf0 Modules linked in: softdog(-) deflate ctr twofish_generic twofish_common camellia_generic serpent_generic blowfish_generic blowfish_common cast5_generic cast_common cmac xcbc af_key sch_fq_codel openvswitch nsh nf_conncount nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 CPU: 23 PID: 1028 Comm: modprobe Not tainted 5.3.0-next-20190924-yoctodev-standard+ #180 Hardware name: Marvell OcteonTX CN96XX board (DT) pstate: 00400009 (nzcv daif +PAN -UAO) pc : debug_print_object+0xb0/0xf0 lr : debug_print_object+0xb0/0xf0 sp : ffff80001cbcfc70 x29: ffff80001cbcfc70 x28: ffff800010ea2128 x27: ffff800010bad000 x26: 0000000000000000 x25: ffff80001103c640 x24: ffff80001107b268 x23: ffff800010bad9e8 x22: ffff800010ea2128 x21: ffff000bc2c62af8 x20: ffff80001103c600 x19: ffff800010e867d8 x18: 0000000000000060 x17: 0000000000000000 x16: 0000000000000000 x15: ffff000bd7240470 x14: 6e6968207473696c x13: 5f72656d6974203a x12: 6570797420746365 x11: 6a626f2029302065 x10: 7461747320657669 x9 : 7463612820657669 x8 : 3378302f3078302b x7 : 0000000000001d7a x6 : ffff800010fd5889 x5 : 0000000000000000 x4 : 0000000000000000 x3 : 0000000000000000 x2 : ffff000bff948548 x1 : 276a1c9e1edc2300 x0 : 0000000000000000 Call trace: debug_print_object+0xb0/0xf0 debug_check_no_obj_freed+0x1e8/0x210 kfree+0x1b8/0x368 watchdog_cdev_unregister+0x88/0xc8 watchdog_dev_unregister+0x38/0x48 watchdog_unregister_device+0xa8/0x100 softdog_exit+0x18/0xfec4 [softdog] __arm64_sys_delete_module+0x174/0x200 el0_svc_handler+0xd0/0x1c8 el0_svc+0x8/0xcThis is a common issue when using cdev embedded in a struct. Fortunately, we already have a mechanism to solve this kind of issue. Please see commit
233ed09d7fda("chardev: add helper function to register char devs with a struct device") for more detail.
Quoted 6 lines
In this patch, we choose to embed the struct device into the
watchdog_core_data, and use the API provided by the commit233ed09d7fdato make sure that the release ofwatchdog_core_dataand cdev are in sequence.Signed-off-by Kevin Hao <[email protected]>
Nice job. Looks good to me.
Wim, any chance you can add this patch to your -next branch sometime soon ? It should get some bake time before being pushed upstream.
Thanks,
Guenter
Quoted 218 lines
--- drivers/watchdog/watchdog_dev.c | 70 +++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 38 deletions(-)drivers/watchdog/watchdog_dev.c +32
-38@@ -34,7 +34,6 @@#include <linux/init.h> /* For __init/__exit/... */ #include <linux/hrtimer.h> /* For hrtimers */ #include <linux/kernel.h> /* For printk/panic/... */ -#include <linux/kref.h> /* For data references */ #include <linux/kthread.h> /* For kthread_work */ #include <linux/miscdevice.h> /* For handling misc devices */ #include <linux/module.h> /* For module stuff/... */ @@ -52,14 +51,14 @@/* * struct watchdog_core_data - watchdog core internal data -* @kref: Reference count. +* @dev: The watchdog's internal device * @cdev: The watchdog's Character device. * @wdd: Pointer to watchdog device. * @lock: Lock for watchdog core. * @status: Watchdog core internal status bits. */ struct watchdog_core_data { -struct kref kref; +struct device dev;struct cdev cdev;struct watchdog_device *wdd;struct mutex lock;@@ -839,7 +838,7 @@ static int watchdog_open(struct inode *inode, struct file *file) file->private_data = wd_data;if (!hw_running) -kref_get (&wd_data->kref); +get_device (&wd_data->dev);/* * open_timeout only applies for the first open from @@ -860,11 +859,11 @@ static int watchdog_open(struct inode *inode, struct file *file)return err; } -static void watchdog_core_data_release (struct kref *kref) +static void watchdog_core_data_release (struct device *dev) {struct watchdog_core_data *wd_data; - wd_data = container_of(kref, struct watchdog_core_data, kref); + wd_data = container_of(dev, struct watchdog_core_data, dev);kfree (wd_data); }@@ -924,7 +923,7 @@ static int watchdog_release(struct inode *inode, struct file *file)*/ if (!running) {module_put (wd_data->cdev.owner); -kref_put (&wd_data->kref, watchdog_core_data_release); +put_device (&wd_data->dev); }return 0 ; }@@ -943,17 +942,22 @@ static struct miscdevice watchdog_miscdev = { .fops = &watchdog_fops, }; +static struct class watchdog_class = { + .name ="watchdog" , + .owner =THIS_MODULE , + .dev_groups = wdt_groups, +}; +/* * watchdog_cdev_register: register watchdog character device * @wdd: watchdog device -* @devno: character device number * * Register a watchdog character device including handling the legacy * /dev/watchdog node. /dev/watchdog is actually a miscdevice and * thus we set it up like that. */ -static int watchdog_cdev_register (struct watchdog_device *wdd,dev_t devno) +static int watchdog_cdev_register (struct watchdog_device *wdd) {struct watchdog_core_data *wd_data;int err;@@ -961,7 +965,6 @@ static int watchdog_cdev_register(struct watchdog_device *wdd, dev_t devno) wd_data =kzalloc (sizeof (struct watchdog_core_data ),GFP_KERNEL );if (!wd_data)return -ENOMEM ; -kref_init (&wd_data->kref);mutex_init (&wd_data->lock); wd_data->wdd = wdd;@@ -990,23 +993,33 @@ static int watchdog_cdev_register(struct watchdog_device *wdd, dev_t devno) } } +device_initialize (&wd_data->dev); + wd_data->dev.devt =MKDEV (MAJOR (watchdog_devt), wdd->id); + wd_data->dev.class = &watchdog_class; + wd_data->dev.parent = wdd->parent; + wd_data->dev.groups = wdd->groups; + wd_data->dev.release = watchdog_core_data_release; +dev_set_drvdata (&wd_data->dev, wdd); +dev_set_name (&wd_data->dev,"watchdog%d" , wdd->id); +/* Fill in the data structures */ cdev_init (&wd_data->cdev, &watchdog_fops); - wd_data->cdev.owner = wdd->ops->owner;/* Add the device */ - err =cdev_add (&wd_data->cdev, devno,1 ); + err =cdev_device_add (&wd_data->cdev, &wd_data->dev);if (err) {pr_err ("watchdog%d unable to add device %d:%d\n" , wdd->id,MAJOR (watchdog_devt), wdd->id);if (wdd->id ==0 ) {misc_deregister (&watchdog_miscdev); old_wd_data =NULL ; -kref_put (&wd_data->kref, watchdog_core_data_release); +put_device (&wd_data->dev); }return err; } + wd_data->cdev.owner = wdd->ops->owner; +/* Record time of most recent heartbeat as 'just before now'. */ wd_data->last_hw_keepalive =ktime_sub (ktime_get (),1 );watchdog_set_open_deadline (wd_data);@@ -1017,7 +1030,7 @@ static int watchdog_cdev_register(struct watchdog_device *wdd, dev_t devno)*/ if (watchdog_hw_running (wdd)) {__module_get (wdd->ops->owner); -kref_get (&wd_data->kref); +get_device (&wd_data->dev);if (handle_boot_enabled)hrtimer_start (&wd_data->timer,0 ,HRTIMER_MODE_REL );else @@ -1040,7 +1053,7 @@ static void watchdog_cdev_unregister(struct watchdog_device *wdd) {struct watchdog_core_data *wd_data = wdd->wd_data; -cdev_del (&wd_data->cdev); +cdev_device_del (&wd_data->cdev, &wd_data->dev);if (wdd->id ==0 ) {misc_deregister (&watchdog_miscdev); old_wd_data =NULL ;@@ -1059,15 +1072,9 @@ static void watchdog_cdev_unregister(struct watchdog_device *wdd)hrtimer_cancel (&wd_data->timer);kthread_cancel_work_sync (&wd_data->work); -kref_put (&wd_data->kref, watchdog_core_data_release); +put_device (&wd_data->dev); } -static struct class watchdog_class = { - .name ="watchdog" , - .owner =THIS_MODULE , - .dev_groups = wdt_groups, -}; -static int watchdog_reboot_notifier (struct notifier_block *nb,unsigned long code,void *data) {@@ -1098,27 +1105,14 @@ static int watchdog_reboot_notifier(struct notifier_block *nb,int watchdog_dev_register (struct watchdog_device *wdd) { -struct device *dev; -dev_t devno;int ret; - devno =MKDEV (MAJOR (watchdog_devt), wdd->id); - - ret =watchdog_cdev_register (wdd, devno); + ret =watchdog_cdev_register (wdd);if (ret)return ret; - dev =device_create_with_groups (&watchdog_class, wdd->parent, - devno, wdd, wdd->groups, -"watchdog%d" , wdd->id); -if (IS_ERR (dev)) { -watchdog_cdev_unregister (wdd); -return PTR_ERR (dev); - } - ret =watchdog_register_pretimeout (wdd);if (ret) { -device_destroy (&watchdog_class, devno);watchdog_cdev_unregister (wdd);return ret; }@@ -1126,7 +1120,8 @@ int watchdog_dev_register(struct watchdog_device *wdd)if (test_bit (WDOG_STOP_ON_REBOOT , &wdd->status)) { wdd->reboot_nb.notifier_call = watchdog_reboot_notifier; - ret =devm_register_reboot_notifier (dev, &wdd->reboot_nb); + ret =devm_register_reboot_notifier (&wdd->wd_data->dev, + &wdd->reboot_nb);if (ret) {pr_err ("watchdog%d: Cannot register reboot notifier (%d)\n" , wdd->id, ret);@@ -1148,7 +1143,6 @@ int watchdog_dev_register(struct watchdog_device *wdd)void watchdog_dev_unregister (struct watchdog_device *wdd) {watchdog_unregister_pretimeout (wdd); -device_destroy (&watchdog_class, wdd->wd_data->cdev.dev);watchdog_cdev_unregister (wdd); }