ocfs2/heartbeat: serialize concurrent writes to a region's dev attribute
Subsystems (2)
PATCH 1/1 ocfs2/heartbeat: serialize concurrent writes to a region's dev attribute
o2hb_region_dev_store() checks reg->hr_bdev_file to see if the region is already started, but nothing stops two writers from doing that at the same time. configfs only serializes writes through the same open file, so two processes that each open the dev attribute can both get past the check:
A: hr_bdev_file is NULL, opens file A, sets hr_bdev_file = A
B: hr_bdev_file is NULL, opens file B, sets hr_bdev_file = B
A: start fails, fput(hr_bdev_file) drops B, sets it to NULL
B: start fails, fput(NULL)
Both calls also use the same hr_slots, hr_slot_data and hr_tmp_block, so the error paths can free the slot data twice.
syzbot hit the fput(NULL) with an unbound loop device, where every start fails with -EIO. With four threads doing the same thing on a KASAN kernel I get both crashes within a second:
BUG: KASAN: double-free in o2hb_unmap_slot_data+0x14d/0x2f0 BUG: kernel NULL pointer dereference, address: 0000000000000158
Add a mutex to the region and take it in dev_store(), in the block_bytes, start_block and blocks stores, and in dev_show(), since they all look at hr_bdev_file. dev_store() sleeps with it held while it waits for the heartbeat thread. That is fine because neither the thread nor the teardown paths take it.
With the patch the same test ran for 10 minutes with lockdep on and nothing was reported.
--- fs/ocfs2/cluster/heartbeat.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-)
fs/ocfs2/cluster/heartbeat.c +11 -2
@@ -269,6 +269,8 @@ struct o2hb_region {struct list_head hr_handler_list;/* Serializes timeout arming against failed-start and teardown. */ struct mutex hr_arming_mutex; +/* Serializes the region's configfs attributes against dev_store(). */ +struct mutex hr_dev_mutex;bool hr_stopping;/* last hb status, 0 for success, other value for error. */
@@ -1676,6 +1678,7 @@ static ssize_t o2hb_region_block_bytes_store(struct config_item *item,unsigned long block_bytes;unsigned int block_bits; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;
@@ -1705,6 +1708,7 @@ static ssize_t o2hb_region_start_block_store(struct config_item *item,char *p = (char *)page;ssize_t ret; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;
@@ -1731,6 +1735,7 @@ static ssize_t o2hb_region_blocks_store(struct config_item *item,char *p = (char *)page;int ret; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;
@@ -1748,10 +1753,12 @@ static ssize_t o2hb_region_blocks_store(struct config_item *item,static ssize_t o2hb_region_dev_show (struct config_item *item,char *page) { +struct o2hb_region *reg =to_o2hb_region (item);unsigned int ret =0 ; -if (to_o2hb_region (item)->hr_bdev_file) - ret =sprintf (page,"%pg\n" ,reg_bdev (to_o2hb_region (item))); +guard (mutex)(®->hr_dev_mutex); +if (reg->hr_bdev_file) + ret =sprintf (page,"%pg\n" ,reg_bdev (reg));return ret; }
@@ -1879,6 +1886,7 @@ static ssize_t o2hb_region_dev_store(struct config_item *item,ssize_t ret = -EINVAL ;int live_threshold; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;
@@ -2156,6 +2164,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g reg->hr_key =crc32_le (reg->hr_region_num +O2NM_MAX_REGIONS , name,strlen (name));mutex_init (®->hr_arming_mutex); +mutex_init (®->hr_dev_mutex); reg->hr_stopping = true;INIT_DELAYED_WORK (®->hr_write_timeout_work, o2hb_write_timeout);INIT_DELAYED_WORK (®->hr_nego_timeout_work, o2hb_nego_timeout);
Signature · 1 line
2.55.0
PATCH v2 1/1 ocfs2/heartbeat: serialize concurrent writes to a region's dev attribute
o2hb_region_dev_store() checks reg->hr_bdev_file to see if the region is already started, but nothing stops two writers from doing that at the same time. configfs only serializes writes through the same open file, so two processes that each open the dev attribute can both get past the check:
A: hr_bdev_file is NULL, opens file A, sets hr_bdev_file = A
B: hr_bdev_file is NULL, opens file B, sets hr_bdev_file = B
A: start fails, fput(hr_bdev_file) drops B, sets it to NULL
B: start fails, fput(NULL)
Both calls also use the same hr_slots, hr_slot_data and hr_tmp_block, so the error paths can free the slot data twice.
syzbot hit the fput(NULL) with an unbound loop device, where every start fails with -EIO. With four threads doing the same thing on a KASAN kernel I get both crashes within a second:
BUG: KASAN: double-free in o2hb_unmap_slot_data+0x14d/0x2f0 BUG: kernel NULL pointer dereference, address: 0000000000000158
Add a mutex to the region and hold it in dev_store(), in the block_bytes, start_block and blocks stores, and in dev_show(), since they all look at hr_bdev_file. dev_store() cleans up with goto, so it takes the mutex in a small wrapper rather than with guard(). It sleeps with the mutex held while it waits for the heartbeat thread. That is fine because neither the thread nor the teardown paths take it.
With the patch the same test ran for 10 minutes with lockdep on and nothing was reported.
---
v2: take the mutex in a wrapper around dev_store() instead of with guard(), since dev_store() uses goto for its cleanup. No other changes.
v1: https://lore.kernel.org/all/[email protected]/
fs/ocfs2/cluster/heartbeat.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-)
fs/ocfs2/cluster/heartbeat.c +26 -6
@@ -269,6 +269,8 @@ struct o2hb_region {struct list_head hr_handler_list;/* Serializes timeout arming against failed-start and teardown. */ struct mutex hr_arming_mutex; +/* Serializes the region's configfs attributes against dev_store(). */ +struct mutex hr_dev_mutex;bool hr_stopping;/* last hb status, 0 for success, other value for error. */
@@ -1676,6 +1678,7 @@ static ssize_t o2hb_region_block_bytes_store(struct config_item *item,unsigned long block_bytes;unsigned int block_bits; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;
@@ -1705,6 +1708,7 @@ static ssize_t o2hb_region_start_block_store(struct config_item *item,char *p = (char *)page;ssize_t ret; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;
@@ -1731,6 +1735,7 @@ static ssize_t o2hb_region_blocks_store(struct config_item *item,char *p = (char *)page;int ret; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;
@@ -1748,10 +1753,12 @@ static ssize_t o2hb_region_blocks_store(struct config_item *item,static ssize_t o2hb_region_dev_show (struct config_item *item,char *page) { +struct o2hb_region *reg =to_o2hb_region (item);unsigned int ret =0 ; -if (to_o2hb_region (item)->hr_bdev_file) - ret =sprintf (page,"%pg\n" ,reg_bdev (to_o2hb_region (item))); +guard (mutex)(®->hr_dev_mutex); +if (reg->hr_bdev_file) + ret =sprintf (page,"%pg\n" ,reg_bdev (reg));return ret; }
@@ -1867,11 +1874,9 @@ static int o2hb_populate_slot_data(struct o2hb_region *reg)* this is acting as commit; we set up all of hr_bdev_file and hr_task or * nothing */ -static ssize_t o2hb_region_dev_store (struct config_item *item, -const char *page, -size_t count) +static ssize_t __o2hb_region_dev_store (struct o2hb_region *reg, +const char *page,size_t count) { -struct o2hb_region *reg =to_o2hb_region (item);struct task_struct *hb_task;long fd;int sectsize;
@@ -2032,6 +2037,20 @@ static ssize_t o2hb_region_dev_store(struct config_item *item,return ret; } +static ssize_t o2hb_region_dev_store (struct config_item *item, +const char *page, +size_t count) +{ +struct o2hb_region *reg =to_o2hb_region (item); +ssize_t ret; + +mutex_lock (®->hr_dev_mutex); + ret =__o2hb_region_dev_store (reg, page, count); +mutex_unlock (®->hr_dev_mutex); + +return ret; +} +static ssize_t o2hb_region_pid_show (struct config_item *item,char *page) {struct o2hb_region *reg =to_o2hb_region (item);
@@ -2156,6 +2175,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g reg->hr_key =crc32_le (reg->hr_region_num +O2NM_MAX_REGIONS , name,strlen (name));mutex_init (®->hr_arming_mutex); +mutex_init (®->hr_dev_mutex); reg->hr_stopping = true;INIT_DELAYED_WORK (®->hr_write_timeout_work, o2hb_write_timeout);INIT_DELAYED_WORK (®->hr_nego_timeout_work, o2hb_nego_timeout);
Signature · 1 line
2.55.0
Joseph Qi
maintainerQuoted 50 lines · Eklavya Singh
o2hb_region_dev_store()checksreg->hr_bdev_fileto see if the region is already started, but nothing stops two writers from doing that at the same time. configfs only serializes writes through the same open file, so two processes that each open the dev attribute can both get past the check:A:
hr_bdev_fileis NULL, opens file A, setshr_bdev_file= A
B:hr_bdev_fileis NULL, opens file B, setshr_bdev_file= B
A: start fails,fput(hr_bdev_file)drops B, sets it to NULL
B: start fails, fput(NULL)Both calls also use the same
hr_slots,hr_slot_dataandhr_tmp_block, so the error paths can free the slot data twice.syzbot hit the fput(NULL) with an unbound loop device, where every start fails with
-EIO. With four threads doing the same thing on a KASAN kernel I get both crashes within a second:BUG: KASAN: double-free in o2hb_unmap_slot_data+0x14d/0x2f0 BUG: kernel NULL pointer dereference, address: 0000000000000158Add a mutex to the region and hold it in
dev_store(), in theblock_bytes,start_blockand blocks stores, and indev_show(), since they all look athr_bdev_file.dev_store()cleans up with goto, so it takes the mutex in a small wrapper rather than withguard(). It sleeps with the mutex held while it waits for the heartbeat thread. That is fine because neither the thread nor the teardown paths take it.With the patch the same test ran for 10 minutes with lockdep on and nothing was reported.
Fixes a7f6a5fb4bde ("[PATCH] OCFS2: The Second Oracle Cluster Filesystem")Reported-by [email protected]Signed-off-by Eklavya Singh <[email protected]>---
v2: take the mutex in a wrapper arounddev_store()instead of withguard(), sincedev_store()uses goto for its cleanup. No other changes.v1: https://lore.kernel.org/all/[email protected]/
fs/ocfs2/cluster/heartbeat.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-)fs/ocfs2/cluster/heartbeat.c +0
-0
fs/ocfs2/cluster/heartbeat.c struct o2hb_region { · line 272
Quoted 4 lines
fs/ocfs2/cluster/heartbeat.c +1
-0@@ -269,6 +269,8 @@ struct o2hb_region {struct list_head hr_handler_list;/* Serializes timeout arming against failed-start and teardown. */ struct mutex hr_arming_mutex; +/* Serializes the region's configfs attributes against dev_store(). */
Naming the covered 4 region's attributes would be clearer.
Quoted 44 lines
fs/ocfs2/cluster/heartbeat.c +8
-2+struct mutex hr_dev_mutex;bool hr_stopping;/* last hb status, 0 for success, other value for error. */ @@ -1676,6 +1678,7 @@ static ssize_t o2hb_region_block_bytes_store(struct config_item *item,unsigned long block_bytes;unsigned int block_bits; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;@@ -1705,6 +1708,7 @@ static ssize_t o2hb_region_start_block_store(struct config_item *item,char *p = (char *)page;ssize_t ret; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;@@ -1731,6 +1735,7 @@ static ssize_t o2hb_region_blocks_store(struct config_item *item,char *p = (char *)page;int ret; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;@@ -1748,10 +1753,12 @@ static ssize_t o2hb_region_blocks_store(struct config_item *item,static ssize_t o2hb_region_dev_show (struct config_item *item,char *page) { +struct o2hb_region *reg =to_o2hb_region (item);unsigned int ret =0 ; -if (to_o2hb_region (item)->hr_bdev_file) - ret =sprintf (page,"%pg\n" ,reg_bdev (to_o2hb_region (item))); +guard (mutex)(®->hr_dev_mutex); +if (reg->hr_bdev_file) + ret =sprintf (page,"%pg\n" ,reg_bdev (reg));return ret; }
fs/ocfs2/cluster/heartbeat.c static int o2hb_populate_slot_data(struct o2hb_region *reg) · line 1874
Quoted 10 lines
fs/ocfs2/cluster/heartbeat.c +2
-4@@ -1867,11 +1874,9 @@ static int o2hb_populate_slot_data(struct o2hb_region *reg)* this is acting as commit; we set up all of hr_bdev_file and hr_task or * nothing */ -static ssize_t o2hb_region_dev_store (struct config_item *item, -const char *page, -size_t count) +static ssize_t __o2hb_region_dev_store (struct o2hb_region *reg, +const char *page,size_t count) { -struct o2hb_region *reg =to_o2hb_region (item);
Would it be worth recording that hr_dev_mutex must be held now?
Quoted 11 lines
fs/ocfs2/cluster/heartbeat.c +4
-0struct task_struct *hb_task;long fd;int sectsize;@@ -2032,6 +2037,20 @@ static ssize_t o2hb_region_dev_store(struct config_item *item,return ret; } +static ssize_t o2hb_region_dev_store (struct config_item *item, +const char *page, +size_t count) +{
fs/ocfs2/cluster/heartbeat.c static ssize_t o2hb_region_dev_store(struct config_item *item, · line 2053
Quoted 10 lines
fs/ocfs2/cluster/heartbeat.c +10
-0+struct o2hb_region *reg =to_o2hb_region (item); +ssize_t ret; + +mutex_lock (®->hr_dev_mutex); + ret =__o2hb_region_dev_store (reg, page, count); +mutex_unlock (®->hr_dev_mutex); + +return ret; +} +
The wrapper keeps hr_dev_mutex held across the whole start, including the wait for the thread:
ret = wait_event_interruptible(o2hb_steady_queue,
atomic_read(®->hr_steady_iterations) == 0 ||
reg->hr_node_deleted);hr_timeout_ms is O2HB_REGION_TIMEOUT_MS, 2000, and hr_steady_iterations starts at O2HB_LIVE_THRESHOLD + 1, or one more doubling again when this is the only region and global heartbeat is active. So a read of the dev attribute that races a start blocks for roughly six seconds, or ten for that first region, where it used to return at once.
Thanks,
Joseph
Quoted 11 lines
fs/ocfs2/cluster/heartbeat.c +1
-0static ssize_t o2hb_region_pid_show (struct config_item *item,char *page) {struct o2hb_region *reg =to_o2hb_region (item);@@ -2156,6 +2175,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g reg->hr_key =crc32_le (reg->hr_region_num +O2NM_MAX_REGIONS , name,strlen (name));mutex_init (®->hr_arming_mutex); +mutex_init (®->hr_dev_mutex); reg->hr_stopping = true;INIT_DELAYED_WORK (®->hr_write_timeout_work, o2hb_write_timeout);INIT_DELAYED_WORK (®->hr_nego_timeout_work, o2hb_nego_timeout);
PATCH v3 1/1 ocfs2/heartbeat: serialize concurrent writes to a region's dev attribute
In reply to Eklavya Singho2hb_region_dev_store() checks reg->hr_bdev_file to see if the region is already started, but nothing stops two writers from doing that at the same time. configfs only serializes writes through the same open file, so two processes that each open the dev attribute can both get past the check:
A: hr_bdev_file is NULL, opens file A, sets hr_bdev_file = A
B: hr_bdev_file is NULL, opens file B, sets hr_bdev_file = B
A: start fails, fput(hr_bdev_file) drops B, sets it to NULL
B: start fails, fput(NULL)
Both calls also use the same hr_slots, hr_slot_data and hr_tmp_block, so the error paths can free the slot data twice.
syzbot hit the fput(NULL) with an unbound loop device, where every start fails with -EIO. With four threads doing the same thing on a KASAN kernel I get both crashes within a second:
BUG: KASAN: double-free in o2hb_unmap_slot_data+0x14d/0x2f0 BUG: kernel NULL pointer dereference, address: 0000000000000158
Add a mutex to the region and hold it in dev_store(), in the block_bytes, start_block and blocks stores, and in dev_show(), since they all look at hr_bdev_file. dev_store() cleans up with goto, so it takes the mutex in a small wrapper rather than with guard(). The mutex is dropped while dev_store() waits for the heartbeat to become steady, which takes seconds, so a read of dev doesn't stall behind a start. hr_bdev_file is already set at that point, so other writers still get -EINVAL, and the cleanup after the wait retakes the mutex.
With the patch the same test ran for 10 minutes with lockdep on and nothing was reported. So did a second one, with a working loop device, that interrupts starts during the wait while other threads read and write the attributes.
---
Changes in v3:
- rebase onto linux-next
- drop hr_dev_mutex while waiting for the heartbeat to become steady, so a read of dev that races a start no longer blocks (Joseph)
- name the four attributes in the hr_dev_mutex comment and assert the mutex is held in __o2hb_region_dev_store() (Joseph)
Changes in v2:
- take the mutex in a wrapper around dev_store() instead of with guard(), since dev_store() uses goto for its cleanup
Deepanshu Kartikey [1] and Cen Zhang [2] have posted patches for the same race, and they conflict with this one. This version also takes the mutex in dev_show(), which otherwise can read hr_bdev_file while a failed start is dropping it.
[1] https://lore.kernel.org/all/[email protected]/
[2] https://lore.kernel.org/all/pm-ocfs2-objects-candidate-0077-v7-34980443ec254b2a1ffc@gmail.com/
v2: https://lore.kernel.org/all/[email protected]/
v1: https://lore.kernel.org/all/[email protected]/
fs/ocfs2/cluster/heartbeat.c | 40 ++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-)
fs/ocfs2/cluster/heartbeat.c +34 -6
@@ -269,6 +269,8 @@ struct o2hb_region {struct list_head hr_handler_list;/* Serializes timeout arming against failed-start and teardown. */ struct mutex hr_arming_mutex; +/* Serializes the dev, block_bytes, start_block and blocks attributes. */ +struct mutex hr_dev_mutex;bool hr_stopping;/* last hb status, 0 for success, other value for error. */
@@ -1681,6 +1683,7 @@ static ssize_t o2hb_region_block_bytes_store(struct config_item *item,unsigned long block_bytes;unsigned int block_bits; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;
@@ -1710,6 +1713,7 @@ static ssize_t o2hb_region_start_block_store(struct config_item *item,char *p = (char *)page;ssize_t ret; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;
@@ -1736,6 +1740,7 @@ static ssize_t o2hb_region_blocks_store(struct config_item *item,char *p = (char *)page;int ret; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;
@@ -1753,10 +1758,12 @@ static ssize_t o2hb_region_blocks_store(struct config_item *item,static ssize_t o2hb_region_dev_show (struct config_item *item,char *page) { +struct o2hb_region *reg =to_o2hb_region (item);unsigned int ret =0 ; -if (to_o2hb_region (item)->hr_bdev_file) - ret =sprintf (page,"%pg\n" ,reg_bdev (to_o2hb_region (item))); +guard (mutex)(®->hr_dev_mutex); +if (reg->hr_bdev_file) + ret =sprintf (page,"%pg\n" ,reg_bdev (reg));return ret; }
@@ -1872,11 +1879,9 @@ out:* this is acting as commit; we set up all of hr_bdev_file and hr_task or * nothing */ -static ssize_t o2hb_region_dev_store (struct config_item *item, -const char *page, -size_t count) +static ssize_t __o2hb_region_dev_store (struct o2hb_region *reg, +const char *page,size_t count) { -struct o2hb_region *reg =to_o2hb_region (item);struct task_struct *hb_task;long fd;int sectsize;
@@ -1884,6 +1889,8 @@ static ssize_t o2hb_region_dev_store(struct config_item *item,ssize_t ret = -EINVAL ;int live_threshold; +lockdep_assert_held (®->hr_dev_mutex); +if (reg->hr_bdev_file)return -EINVAL ;
@@ -1987,9 +1994,15 @@ static ssize_t o2hb_region_dev_store(struct config_item *item,wake_up_process (hb_task); +/* +* Don't hold hr_dev_mutex across the wait, which can take seconds. +* hr_bdev_file is already set, so other writers still get -EINVAL. +*/ +mutex_unlock (®->hr_dev_mutex); ret =wait_event_interruptible (o2hb_steady_queue,atomic_read (®->hr_steady_iterations) ==0 || reg->hr_node_deleted); +mutex_lock (®->hr_dev_mutex);if (ret) {atomic_set (®->hr_steady_iterations,0 ); reg->hr_aborted_start =1 ;
@@ -2041,6 +2054,20 @@ out:return ret; } +static ssize_t o2hb_region_dev_store (struct config_item *item, +const char *page, +size_t count) +{ +struct o2hb_region *reg =to_o2hb_region (item); +ssize_t ret; + +mutex_lock (®->hr_dev_mutex); + ret =__o2hb_region_dev_store (reg, page, count); +mutex_unlock (®->hr_dev_mutex); + +return ret; +} +static ssize_t o2hb_region_pid_show (struct config_item *item,char *page) {struct o2hb_region *reg =to_o2hb_region (item);
@@ -2143,6 +2170,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *gINIT_LIST_HEAD (®->hr_all_item);mutex_init (®->hr_arming_mutex); +mutex_init (®->hr_dev_mutex); reg->hr_stopping = true;INIT_DELAYED_WORK (®->hr_write_timeout_work, o2hb_write_timeout);INIT_DELAYED_WORK (®->hr_nego_timeout_work, o2hb_nego_timeout);
Signature · 1 line
2.52.0
Quoted 35 lines · Eklavya Singh
o2hb_region_dev_store()checksreg->hr_bdev_fileto see if the region is already started, but nothing stops two writers from doing that at the same time. configfs only serializes writes through the same open file, so two processes that each open the dev attribute can both get past the check:A:
hr_bdev_fileis NULL, opens file A, setshr_bdev_file= A
B:hr_bdev_fileis NULL, opens file B, setshr_bdev_file= B
A: start fails,fput(hr_bdev_file)drops B, sets it to NULL
B: start fails, fput(NULL)Both calls also use the same
hr_slots,hr_slot_dataandhr_tmp_block, so the error paths can free the slot data twice.syzbot hit the fput(NULL) with an unbound loop device, where every start fails with
-EIO. With four threads doing the same thing on a KASAN kernel I get both crashes within a second:BUG: KASAN: double-free in o2hb_unmap_slot_data+0x14d/0x2f0 BUG: kernel NULL pointer dereference, address: 0000000000000158Add a mutex to the region and hold it in
dev_store(), in theblock_bytes,start_blockand blocks stores, and indev_show(), since they all look athr_bdev_file.dev_store()cleans up with goto, so it takes the mutex in a small wrapper rather than withguard(). The mutex is dropped whiledev_store()waits for the heartbeat to become steady, which takes seconds, so a read of dev doesn't stall behind a start.hr_bdev_fileis already set at that point, so other writers still get-EINVAL, and the cleanup after the wait retakes the mutex.With the patch the same test ran for 10 minutes with lockdep on and nothing was reported. So did a second one, with a working loop device, that interrupts starts during the wait while other threads read and write the attributes.
Quoted 5 lines
Fixes a7f6a5fb4bde ("[PATCH] OCFS2: The Second Oracle Cluster Filesystem")Reported-by [email protected]Signed-off-by Eklavya Singh <[email protected]>
Looks fine.
Quoted 146 lines
---
Changes in v3:
- rebase onto linux-next
- drophr_dev_mutexwhile waiting for the heartbeat to become steady, so a read of dev that races a start no longer blocks (Joseph)
- name the four attributes in thehr_dev_mutexcomment and assert the mutex is held in__o2hb_region_dev_store()(Joseph)Changes in v2:
- take the mutex in a wrapper arounddev_store()instead of withguard(), sincedev_store()uses goto for its cleanupDeepanshu Kartikey [1] and Cen Zhang [2] have posted patches for the same race, and they conflict with this one. This version also takes the mutex in
dev_show(), which otherwise can readhr_bdev_filewhile a failed start is dropping it.[1] https://lore.kernel.org/all/[email protected]/
[2] https://lore.kernel.org/all/pm-ocfs2-objects-candidate-0077-v7-34980443ec254b2a1ffc@gmail.com/v2: https://lore.kernel.org/all/[email protected]/
v1: https://lore.kernel.org/all/[email protected]/fs/ocfs2/cluster/heartbeat.c | 40 ++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-)fs/ocfs2/cluster/heartbeat.c +34
-6@@ -269,6 +269,8 @@ struct o2hb_region {struct list_head hr_handler_list;/* Serializes timeout arming against failed-start and teardown. */ struct mutex hr_arming_mutex; +/* Serializes the dev, block_bytes, start_block and blocks attributes. */ +struct mutex hr_dev_mutex;bool hr_stopping;/* last hb status, 0 for success, other value for error. */ @@ -1681,6 +1683,7 @@ static ssize_t o2hb_region_block_bytes_store(struct config_item *item,unsigned long block_bytes;unsigned int block_bits; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;@@ -1710,6 +1713,7 @@ static ssize_t o2hb_region_start_block_store(struct config_item *item,char *p = (char *)page;ssize_t ret; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;@@ -1736,6 +1740,7 @@ static ssize_t o2hb_region_blocks_store(struct config_item *item,char *p = (char *)page;int ret; +guard (mutex)(®->hr_dev_mutex);if (reg->hr_bdev_file)return -EINVAL ;@@ -1753,10 +1758,12 @@ static ssize_t o2hb_region_blocks_store(struct config_item *item,static ssize_t o2hb_region_dev_show (struct config_item *item,char *page) { +struct o2hb_region *reg =to_o2hb_region (item);unsigned int ret =0 ; -if (to_o2hb_region (item)->hr_bdev_file) - ret =sprintf (page,"%pg\n" ,reg_bdev (to_o2hb_region (item))); +guard (mutex)(®->hr_dev_mutex); +if (reg->hr_bdev_file) + ret =sprintf (page,"%pg\n" ,reg_bdev (reg));return ret; }@@ -1872,11 +1879,9 @@ out:* this is acting as commit; we set up all of hr_bdev_file and hr_task or * nothing */ -static ssize_t o2hb_region_dev_store (struct config_item *item, -const char *page, -size_t count) +static ssize_t __o2hb_region_dev_store (struct o2hb_region *reg, +const char *page,size_t count) { -struct o2hb_region *reg =to_o2hb_region (item);struct task_struct *hb_task;long fd;int sectsize;@@ -1884,6 +1889,8 @@ static ssize_t o2hb_region_dev_store(struct config_item *item,ssize_t ret = -EINVAL ;int live_threshold; +lockdep_assert_held (®->hr_dev_mutex); +if (reg->hr_bdev_file)return -EINVAL ;@@ -1987,9 +1994,15 @@ static ssize_t o2hb_region_dev_store(struct config_item *item,wake_up_process (hb_task); +/* +* Don't hold hr_dev_mutex across the wait, which can take seconds. +* hr_bdev_file is already set, so other writers still get -EINVAL. +*/ +mutex_unlock (®->hr_dev_mutex); ret =wait_event_interruptible (o2hb_steady_queue,atomic_read (®->hr_steady_iterations) ==0 || reg->hr_node_deleted); +mutex_lock (®->hr_dev_mutex);if (ret) {atomic_set (®->hr_steady_iterations,0 ); reg->hr_aborted_start =1 ;@@ -2041,6 +2054,20 @@ out:return ret; } +static ssize_t o2hb_region_dev_store (struct config_item *item, +const char *page, +size_t count) +{ +struct o2hb_region *reg =to_o2hb_region (item); +ssize_t ret; + +mutex_lock (®->hr_dev_mutex); + ret =__o2hb_region_dev_store (reg, page, count); +mutex_unlock (®->hr_dev_mutex); + +return ret; +} +static ssize_t o2hb_region_pid_show (struct config_item *item,char *page) {struct o2hb_region *reg =to_o2hb_region (item);@@ -2143,6 +2170,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *gINIT_LIST_HEAD (®->hr_all_item);mutex_init (®->hr_arming_mutex); +mutex_init (®->hr_dev_mutex); reg->hr_stopping = true;INIT_DELAYED_WORK (®->hr_write_timeout_work, o2hb_write_timeout);INIT_DELAYED_WORK (®->hr_nego_timeout_work, o2hb_nego_timeout);