bcachefs: fix deadlock in journal_entry_open()
PATCH v2 1/1 bcachefs: fix deadlock in journal_entry_open()
In the previous commit b3d82c2f2761, code was added to prevent journal sequence overflow. Among them, the code added to journal_entry_open() uses the bch2_fs_fatal_err_on() function to handle errors.
However, __journal_res_get() , which calls journal_entry_open() , calls journal_entry_open() while holding journal->lock , but bch2_fs_fatal_err_on() internally tries to acquire journal->lock , which results in a deadlock.
Therefore, we need to add bch2_fs_fatal_err_on_locked() to handle fatal errors even when journal->lock is held.
--- fs/bcachefs/error.c | 6 ++++++ fs/bcachefs/error.h | 16 ++++++++++++++++ fs/bcachefs/journal.c | 10 +++++++++- fs/bcachefs/journal.h | 1 + fs/bcachefs/super.c | 11 +++++++++++ fs/bcachefs/super.h | 1 + 6 files changed, 44 insertions(+), 1 deletion(-)
fs/bcachefs/error.c +6 -0
@@ -50,6 +50,12 @@ void bch2_fatal_error(struct bch_fs *c)bch_err (c,"fatal error - emergency read only" ); } +void bch2_fatal_error_locked (struct bch_fs *c) +{ +if (bch2_fs_emergency_read_only_locked (c)) +bch_err (c,"fatal error - emergency read only" ); +} +void bch2_io_error_work (struct work_struct *work) { struct bch_dev *ca = container_of(work, struct bch_dev, io_error_work);
fs/bcachefs/error.h +16 -0
@@ -189,6 +189,7 @@ do { \*/ void bch2_fatal_error (struct bch_fs *); +void bch2_fatal_error_locked (struct bch_fs *);#define bch2_fs_fatal_error (c, _msg, ...) \ do { \
@@ -205,6 +206,21 @@ do { \ _ret; \ }) +#define bch2_fs_fatal_error_locked (c, _msg, ...) \ +do { \ + bch_err(c, "%s(): fatal error " _msg, __func__, ##__VA_ARGS__); \ + bch2_fatal_error_locked(c); \ +} while (0) + +#define bch2_fs_fatal_err_on_locked (cond, c, ...) \ +({ \ + bool _ret = unlikely(!!(cond)); \ + \ + if (_ret) \ + bch2_fs_fatal_error_locked(c, __VA_ARGS__); \ + _ret; \ +}) +/* * IO errors: either recoverable metadata IO (because we have replicas), or data * IO - we need to log it and print out a message, but we don't (necessarily)
fs/bcachefs/journal.c +9 -1
@@ -320,6 +320,14 @@ void bch2_journal_halt(struct journal *j)spin_unlock (&j->lock); } +void bch2_journal_halt_locked (struct journal *j) +{ +__journal_entry_close (j,JOURNAL_ENTRY_ERROR_VAL , true); +if (!j->err_seq) + j->err_seq =journal_cur_seq (j); +journal_wake (j); +} +static bool journal_entry_want_write (struct journal *j) { bool ret = !journal_entry_is_open(j) ||
@@ -382,7 +390,7 @@ static int journal_entry_open(struct journal *j)if (nr_unwritten_journal_entries (j) ==ARRAY_SIZE (j->buf))return JOURNAL_ERR_max_in_flight; -if (bch2_fs_fatal_err_on (journal_cur_seq (j) >=JOURNAL_SEQ_MAX , +if (bch2_fs_fatal_err_on_locked (journal_cur_seq (j) >=JOURNAL_SEQ_MAX , c,"cannot start: journal seq overflow" ))return JOURNAL_ERR_insufficient_devices;/* -EROFS */
fs/bcachefs/journal.h +1 -0
@@ -408,6 +408,7 @@ bool bch2_journal_noflush_seq(struct journal *, u64, u64);int bch2_journal_meta (struct journal *);void bch2_journal_halt (struct journal *); +void bch2_journal_halt_locked (struct journal *);static inline int bch2_journal_error (struct journal *j) {
fs/bcachefs/super.c +11 -0
@@ -411,6 +411,17 @@ bool bch2_fs_emergency_read_only(struct bch_fs *c)return ret; } +bool bch2_fs_emergency_read_only_locked (struct bch_fs *c) +{ +bool ret = !test_and_set_bit (BCH_FS_emergency_ro, &c->flags); + +bch2_journal_halt_locked (&c->journal); +bch2_fs_read_only_async (c); + +wake_up (&bch2_read_only_wait); +return ret; +} +static int bch2_fs_read_write_late (struct bch_fs *c) {int ret;
fs/bcachefs/super.h +1 -0
@@ -29,6 +29,7 @@ int bch2_dev_resize(struct bch_fs *, struct bch_dev *, u64);struct bch_dev *bch2_dev_lookup (struct bch_fs *,const char *);bool bch2_fs_emergency_read_only (struct bch_fs *); +bool bch2_fs_emergency_read_only_locked (struct bch_fs *);void bch2_fs_read_only (struct bch_fs *);int bch2_fs_read_write (struct bch_fs *);
--
Quoted 4 lines · Jeongjun Park
In the previous commit
b3d82c2f2761, code was added to prevent journal sequence overflow. Among them, the code added tojournal_entry_open()uses thebch2_fs_fatal_err_on()function to handle errors.
At least, bch2_journal_halt could invoke bch2_journal_halt_locked.
Is it possible to release the lock before bch2_fs_fatal_err_on() ? Not familiar with journal code yet.
Quoted 146 lines
However,
__journal_res_get(), which callsjournal_entry_open(), callsjournal_entry_open()while holdingjournal->lock, butbch2_fs_fatal_err_on()internally tries to acquirejournal->lock, which results in a deadlock.Therefore, we need to add
bch2_fs_fatal_err_on_locked()to handle fatal errors even whenjournal->lockis held.Fixes b3d82c2f2761 ("bcachefs: Guard against journal seq overflow")Signed-off-by Jeongjun Park <[email protected]>---
fs/bcachefs/error.c | 6 ++++++ fs/bcachefs/error.h | 16 ++++++++++++++++ fs/bcachefs/journal.c | 10 +++++++++- fs/bcachefs/journal.h | 1 + fs/bcachefs/super.c | 11 +++++++++++ fs/bcachefs/super.h | 1 + 6 files changed, 44 insertions(+), 1 deletion(-)fs/bcachefs/error.c +6
-0@@ -50,6 +50,12 @@ void bch2_fatal_error(struct bch_fs *c)bch_err (c,"fatal error - emergency read only" ); } +void bch2_fatal_error_locked (struct bch_fs *c) +{ +if (bch2_fs_emergency_read_only_locked (c)) +bch_err (c,"fatal error - emergency read only" ); +} +void bch2_io_error_work (struct work_struct *work) { struct bch_dev *ca = container_of(work, struct bch_dev, io_error_work);fs/bcachefs/error.h +16
-0@@ -189,6 +189,7 @@ do { \*/ void bch2_fatal_error (struct bch_fs *); +void bch2_fatal_error_locked (struct bch_fs *);#define bch2_fs_fatal_error (c, _msg, ...) \ do { \@@ -205,6 +206,21 @@ do { \ _ret; \ }) +#define bch2_fs_fatal_error_locked (c, _msg, ...) \ +do { \ + bch_err(c, "%s(): fatal error " _msg, __func__, ##__VA_ARGS__); \ + bch2_fatal_error_locked(c); \ +} while (0) + +#define bch2_fs_fatal_err_on_locked (cond, c, ...) \ +({ \ + bool _ret = unlikely(!!(cond)); \ + \ + if (_ret) \ + bch2_fs_fatal_error_locked(c, __VA_ARGS__); \ + _ret; \ +}) +/* * IO errors: either recoverable metadata IO (because we have replicas), or data * IO - we need to log it and print out a message, but we don't (necessarily) fs/bcachefs/journal.c +8
-0@@ -320,6 +320,14 @@ void bch2_journal_halt(struct journal *j)spin_unlock (&j->lock); } +void bch2_journal_halt_locked (struct journal *j) +{ +__journal_entry_close (j,JOURNAL_ENTRY_ERROR_VAL , true); +if (!j->err_seq) + j->err_seq =journal_cur_seq (j); +journal_wake (j); +} +static bool journal_entry_want_write (struct journal *j) {bool ret = !journal_entry_is_open(j) ||
fs/bcachefs/journal.c +1
-1@@ -382,7 +390,7 @@ static int journal_entry_open(struct journal *j)if (nr_unwritten_journal_entries (j) ==ARRAY_SIZE (j->buf))return JOURNAL_ERR_max_in_flight; -if (bch2_fs_fatal_err_on (journal_cur_seq (j) >=JOURNAL_SEQ_MAX , +if (bch2_fs_fatal_err_on_locked (journal_cur_seq (j) >=JOURNAL_SEQ_MAX , c,"cannot start: journal seq overflow" ))return
JOURNAL_ERR_insufficient_devices; /*-EROFS*/fs/bcachefs/journal.h +1
-0@@ -408,6 +408,7 @@ bool bch2_journal_noflush_seq(struct journal *, u64, u64);int bch2_journal_meta (struct journal *);void bch2_journal_halt (struct journal *); +void bch2_journal_halt_locked (struct journal *);static inline int bch2_journal_error (struct journal *j) {fs/bcachefs/super.c +11
-0@@ -411,6 +411,17 @@ bool bch2_fs_emergency_read_only(struct bch_fs *c)return ret; } +bool bch2_fs_emergency_read_only_locked (struct bch_fs *c) +{ +bool ret = !test_and_set_bit (BCH_FS_emergency_ro, &c->flags); + +bch2_journal_halt_locked (&c->journal); +bch2_fs_read_only_async (c); + +wake_up (&bch2_read_only_wait); +return ret; +} +static int bch2_fs_read_write_late (struct bch_fs *c) {int ret;fs/bcachefs/super.h +1
-1@@ -29,6 +29,7 @@ int bch2_dev_resize(struct bch_fs *, struct bch_dev *, u64);struct bch_dev *bch2_dev_lookup (struct bch_fs *,const char *);bool bch2_fs_emergency_read_only (struct bch_fs *); +bool bch2_fs_emergency_read_only_locked (struct bch_fs *);void bch2_fs_read_only (struct bch_fs *);int bch2_fs_read_write (struct bch_fs *); --
Quoted 7 lines · Alan Huang
Quoted 4 lines · Jeongjun Park
In the previous commit
b3d82c2f2761, code was added to prevent journal sequence overflow. Among them, the code added tojournal_entry_open()uses thebch2_fs_fatal_err_on()function to handle errors.At least,
bch2_journal_haltcould invokebch2_journal_halt_locked.
yep
Quoted 1 line
Is it possible to release the lock before
bch2_fs_fatal_err_on()? Not familiar with journal code yet.
The control flow makes it awkward, halt_locked() looks like the cleanest solution.