(More complex) logged operations
cover (More complex) logged operations
Hi all,
this patch series introduces a logged operations btree, for operations that are too big to be done in a single btree transaction, and then changes truncate and finsert/fcollapse to use it.
This fixes a recently noticed bug where the deleted inodes btree broke truncate atomicity after unclean shutdown, because that was relying on scanning the inodes btree for BCH_INODE_I_SIZE_DIRTY inodes...
Kent Overstreet (3):
bcachefs: BTREE_ID_logged_ops
bcachefs: Log truncate operations
bcachefs: Log finsert/fcollapse operations
fs/bcachefs/Makefile | 1 + fs/bcachefs/bcachefs.h | 1 + fs/bcachefs/bcachefs_format.h | 40 ++++- fs/bcachefs/bkey_methods.c | 1 + fs/bcachefs/btree_update.c | 18 +++ fs/bcachefs/btree_update.h | 1 + fs/bcachefs/io_misc.c | 275 ++++++++++++++++++++-------------- fs/bcachefs/io_misc.h | 19 +++ fs/bcachefs/logged_ops.c | 110 ++++++++++++++ fs/bcachefs/logged_ops.h | 20 +++ fs/bcachefs/recovery.c | 1 + fs/bcachefs/recovery_types.h | 1 + fs/bcachefs/super.c | 1 + 13 files changed, 376 insertions(+), 113 deletions(-) create mode 100644 fs/bcachefs/logged_ops.c create mode 100644 fs/bcachefs/logged_ops.h
Signature · 1 line
2.40.1
PATCH 1/3 bcachefs: BTREE_ID_logged_ops
Add a new btree for long running logged operations - i.e. for logging operations that we can't do within a single btree transaction, so that they can be resumed if we crash.
Keys in the logged operations btree will represent operations in progress, with the state of the operation stored in the value.
--- fs/bcachefs/Makefile | 1 + fs/bcachefs/bcachefs.h | 1 + fs/bcachefs/bcachefs_format.h | 4 +- fs/bcachefs/btree_update.c | 18 ++++++ fs/bcachefs/btree_update.h | 1 + fs/bcachefs/logged_ops.c | 109 ++++++++++++++++++++++++++++++++++ fs/bcachefs/logged_ops.h | 18 ++++++ fs/bcachefs/recovery.c | 1 + fs/bcachefs/recovery_types.h | 1 + fs/bcachefs/super.c | 1 + 10 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 fs/bcachefs/logged_ops.c create mode 100644 fs/bcachefs/logged_ops.h
fs/bcachefs/Makefile +1 -0
@@ -55,6 +55,7 @@ bcachefs-y := \ journal_sb.o \ journal_seq_blacklist.o \ keylist.o \ + logged_ops.o \ lru.o \ migrate.o \ move.o \
fs/bcachefs/bcachefs.h +1 -0
@@ -454,6 +454,7 @@ enum gc_phase { GC_PHASE_BTREE_bucket_gens, GC_PHASE_BTREE_snapshot_trees, GC_PHASE_BTREE_deleted_inodes, + GC_PHASE_BTREE_logged_ops, GC_PHASE_PENDING_DELETE, };
fs/bcachefs/bcachefs_format.h +3 -1
@@ -2249,7 +2249,9 @@ enum btree_id_flags {x (snapshot_trees,15 ,0 , \ BIT_ULL(KEY_TYPE_snapshot_tree)) \x (deleted_inodes,16 ,BTREE_ID_SNAPSHOTS , \ - BIT_ULL(KEY_TYPE_set)) + BIT_ULL(KEY_TYPE_set)) \ +x (logged_ops,17 ,0 , \ + 0)enum btree_id {#define x(name, nr, ...) BTREE_ID_##name = nr,
fs/bcachefs/btree_update.c +18 -0
@@ -653,6 +653,7 @@ int bch2_btree_insert_nonextent(struct btree_trans *trans,int ret;bch2_trans_iter_init (trans, &iter, btree, k->k.p, +BTREE_ITER_CACHED |BTREE_ITER_NOT_EXTENTS |BTREE_ITER_INTENT ); ret = bch2_btree_iter_traverse(&iter) ?:
@@ -727,6 +728,23 @@ int bch2_btree_delete_at_buffered(struct btree_trans *trans,return bch2_trans_update_buffered (trans, btree, k); } +int bch2_btree_delete (struct btree_trans *trans, +enum btree_id btree,struct bpos pos, +unsigned update_flags) +{ +struct btree_iter iter; +int ret; + +bch2_trans_iter_init (trans, &iter, btree, pos, +BTREE_ITER_CACHED | +BTREE_ITER_INTENT ); + ret =bch2_btree_iter_traverse (&iter) ?: +bch2_btree_delete_at (trans, &iter, update_flags); +bch2_trans_iter_exit (trans, &iter); + +return ret; +} + int bch2_btree_delete_range_trans(struct btree_trans *trans, enum btree_id id, struct bpos start, struct bpos end, unsigned update_flags,
fs/bcachefs/btree_update.h +1 -0
@@ -58,6 +58,7 @@ int bch2_btree_delete_extent_at(struct btree_trans *, struct btree_iter *, unsigned, unsigned);int bch2_btree_delete_at (struct btree_trans *,struct btree_iter *,unsigned );int bch2_btree_delete_at_buffered (struct btree_trans *,enum btree_id ,struct bpos ); +int bch2_btree_delete (struct btree_trans *,enum btree_id ,struct bpos ,unsigned );int bch2_btree_insert_nonextent (struct btree_trans *,enum btree_id ,struct bkey_i *,enum btree_update_flags );
fs/bcachefs/logged_ops.c new file +109 -0
109 lines not on this page · raw message
fs/bcachefs/logged_ops.h new file +18 -0
@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BCACHEFS_LOGGED_OPS_H +#define _BCACHEFS_LOGGED_OPS_H + +#include "bkey.h" + +#define BCH_LOGGED_OPS () + +static inline int bch2_logged_op_update (struct btree_trans *trans,struct bkey_i *op) +{ +return bch2_btree_insert_nonextent (trans, BTREE_ID_logged_ops, op,0 ); +} + +int bch2_resume_logged_ops (struct bch_fs *); +int bch2_logged_op_start (struct btree_trans *,struct bkey_i *); +void bch2_logged_op_finish (struct btree_trans *,struct bkey_i *); + +#endif /* _BCACHEFS_LOGGED_OPS_H */
fs/bcachefs/recovery.c +1 -0
@@ -20,6 +20,7 @@#include "journal_reclaim.h" #include "journal_seq_blacklist.h" #include "lru.h" +#include "logged_ops.h" #include "move.h" #include "quota.h" #include "recovery.h"
fs/bcachefs/recovery_types.h +1 -0
@@ -24,6 +24,7 @@ x(check_alloc_to_lru_refs, PASS_FSCK) \ x(fs_freespace_init, PASS_ALWAYS|PASS_SILENT) \ x(bucket_gens_init, 0) \ + x(resume_logged_ops, PASS_ALWAYS) \ x(check_snapshot_trees, PASS_FSCK) \ x(check_snapshots, PASS_FSCK) \x (check_subvols,PASS_FSCK ) \
fs/bcachefs/super.c +1 -0
@@ -791,6 +791,7 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) c->btree_key_cache_btrees |=1U << BTREE_ID_alloc;if (c->opts.inodes_use_key_cache) c->btree_key_cache_btrees |=1U << BTREE_ID_inodes; + c->btree_key_cache_btrees |=1U << BTREE_ID_logged_ops; c->block_bits =ilog2 (block_sectors (c)); c->btree_foreground_merge_threshold =BTREE_FOREGROUND_MERGE_THRESHOLD (c);
Signature · 1 line
2.40.1
PATCH 2/3 bcachefs: Log truncate operations
In reply to Kent OverstreetPreviously, we guaranteed atomicity of truncate after unclean shutdown with the BCH_INODE_I_SIZE_DIRTY flag - which required a full scan of the inodes btree.
Recently the deleted inodes btree was added so that we no longer have to scan for deleted inodes, but truncate was unfinished and that change left it broken.
This patch uses the new logged operations btree to fix truncate atomicity; we now log an operation that can be replayed at the start of a truncate.
--- fs/bcachefs/bcachefs_format.h | 19 ++++++++--- fs/bcachefs/bkey_methods.c | 1 + fs/bcachefs/io_misc.c | 64 +++++++++++++++++++++++++---------- fs/bcachefs/io_misc.h | 9 +++++ fs/bcachefs/logged_ops.c | 1 + fs/bcachefs/logged_ops.h | 3 +- 6 files changed, 75 insertions(+), 22 deletions(-)
fs/bcachefs/bcachefs_format.h +15 -4
@@ -370,7 +370,8 @@ static inline void bkey_init(struct bkey *k) x(backpointer, 28) \ x(inode_v3, 29) \ x(bucket_gens, 30) \ - x(snapshot_tree, 31) + x(snapshot_tree, 31) \ + x(logged_op_truncate, 32)enum bch_bkey_type {#define x(name, nr) KEY_TYPE_##name = nr,
@@ -847,8 +848,8 @@ enum { __BCH_INODE_NODUMP =3 , __BCH_INODE_NOATIME =4 , - __BCH_INODE_I_SIZE_DIRTY =5 , - __BCH_INODE_I_SECTORS_DIRTY =6 , + __BCH_INODE_I_SIZE_DIRTY =5 ,/* obsolete */ + __BCH_INODE_I_SECTORS_DIRTY =6 ,/* obsolete */ __BCH_INODE_UNLINKED =7 , __BCH_INODE_BACKPTR_UNTRUSTED = 8,
@@ -1183,6 +1184,16 @@ struct bch_lru {#define LRU_ID_STRIPES (1U << 16) +/* Logged operations btree: */ + +struct bch_logged_op_truncate { +struct bch_val v; +__le32 subvol; +__le32 pad; +__le64 inum; +__le64 new_i_size; +}; +/* Optional/variable size superblock sections: */ struct bch_sb_field {
@@ -2251,7 +2262,7 @@ enum btree_id_flags {x (deleted_inodes,16 ,BTREE_ID_SNAPSHOTS , \ BIT_ULL(KEY_TYPE_set)) \x (logged_ops,17 ,0 , \ - 0) + BIT_ULL(KEY_TYPE_logged_op_truncate))enum btree_id {#define x(name, nr, ...) BTREE_ID_##name = nr,
fs/bcachefs/bkey_methods.c +1 -0
@@ -10,6 +10,7 @@#include "error.h" #include "extents.h" #include "inode.h" +#include "io_misc.h" #include "lru.h" #include "quota.h" #include "reflink.h"
fs/bcachefs/io_misc.c +47 -17
95 lines not on this page · raw message
fs/bcachefs/io_misc.h +9 -0
@@ -18,6 +18,15 @@ int bch2_fpunch_at(struct btree_trans *, struct btree_iter *, subvol_inum, u64, s64 *);int bch2_fpunch (struct bch_fs *c,subvol_inum ,u64 ,u64 ,s64 *); +void bch2_logged_op_truncate_to_text (struct printbuf *,struct bch_fs *,struct bkey_s_c ); + +#define bch2_bkey_ops_logged_op_truncate ((struct bkey_ops) { \ + .val_to_text = bch2_logged_op_truncate_to_text, \ + .min_val_size = 24, \ +}) + +int bch2_resume_logged_op_truncate (struct btree_trans *,struct bkey_i *); +int bch2_truncate (struct bch_fs *,subvol_inum ,u64 ,u64 *);int bch2_fcollapse_finsert (struct bch_fs *,subvol_inum ,u64 ,u64 ,bool ,s64 *);
fs/bcachefs/logged_ops.c +1 -0
@@ -4,6 +4,7 @@#include "bkey_buf.h" #include "btree_update.h" #include "error.h" +#include "io_misc.h" #include "logged_ops.h" struct bch_logged_op_fn {
fs/bcachefs/logged_ops.h +2 -1
@@ -4,7 +4,8 @@#include "bkey.h" -#define BCH_LOGGED_OPS () +#define BCH_LOGGED_OPS () \ + x(truncate)static inline int bch2_logged_op_update (struct btree_trans *trans,struct bkey_i *op) {
Signature · 1 line
2.40.1
PATCH 3/3 bcachefs: Log finsert/fcollapse operations
In reply to Kent OverstreetNow that we have the logged operations btree, we can make finsert/fcollapse atomic w.r.t. unclean shutdown as well.
This adds bch_logged_op_finsert to represent the state of an finsert or fcollapse, which is a bit more complicated than truncate since we need to track our position in the "shift extents" operation.
--- fs/bcachefs/bcachefs_format.h | 23 +++- fs/bcachefs/io_misc.c | 211 +++++++++++++++++++--------------- fs/bcachefs/io_misc.h | 10 ++ fs/bcachefs/logged_ops.h | 3 +- 4 files changed, 152 insertions(+), 95 deletions(-)
fs/bcachefs/bcachefs_format.h +21 -2
@@ -371,7 +371,8 @@ static inline void bkey_init(struct bkey *k) x(inode_v3, 29) \ x(bucket_gens, 30) \ x(snapshot_tree, 31) \ - x(logged_op_truncate, 32) + x(logged_op_truncate, 32) \ + x(logged_op_finsert, 33)enum bch_bkey_type {#define x(name, nr) KEY_TYPE_##name = nr,
@@ -1194,6 +1195,23 @@ struct bch_logged_op_truncate {__le64 new_i_size; }; +enum logged_op_finsert_state { + LOGGED_OP_FINSERT_start, + LOGGED_OP_FINSERT_shift_extents, + LOGGED_OP_FINSERT_finish, +}; + +struct bch_logged_op_finsert { +struct bch_val v; +__u8 state; +__u8 pad[3 ]; +__le32 subvol; +__le64 inum; +__le64 dst_offset; +__le64 src_offset; +__le64 pos; +}; +/* Optional/variable size superblock sections: */ struct bch_sb_field {
@@ -2262,7 +2280,8 @@ enum btree_id_flags {x (deleted_inodes,16 ,BTREE_ID_SNAPSHOTS , \ BIT_ULL(KEY_TYPE_set)) \x (logged_ops,17 ,0 , \ - BIT_ULL(KEY_TYPE_logged_op_truncate)) +BIT_ULL (KEY_TYPE_logged_op_truncate)| \ + BIT_ULL(KEY_TYPE_logged_op_finsert))enum btree_id {#define x(name, nr, ...) BTREE_ID_##name = nr,
fs/bcachefs/io_misc.c +119 -92
270 lines not on this page · raw message
fs/bcachefs/io_misc.h +10 -0
@@ -28,6 +28,16 @@ void bch2_logged_op_truncate_to_text(struct printbuf *, struct bch_fs *, structint bch2_resume_logged_op_truncate (struct btree_trans *,struct bkey_i *);int bch2_truncate (struct bch_fs *,subvol_inum ,u64 ,u64 *); + +void bch2_logged_op_finsert_to_text (struct printbuf *,struct bch_fs *,struct bkey_s_c ); + +#define bch2_bkey_ops_logged_op_finsert ((struct bkey_ops) { \ + .val_to_text = bch2_logged_op_finsert_to_text, \ + .min_val_size = 24, \ +}) + +int bch2_resume_logged_op_finsert (struct btree_trans *,struct bkey_i *); +int bch2_fcollapse_finsert (struct bch_fs *,subvol_inum ,u64 ,u64 ,bool ,s64 *);#endif /* _BCACHEFS_IO_MISC_H */
fs/bcachefs/logged_ops.h +2 -1
@@ -5,7 +5,8 @@#include "bkey.h" #define BCH_LOGGED_OPS () \ - x(truncate) + x(truncate) \ + x(finsert)static inline int bch2_logged_op_update (struct btree_trans *trans,struct bkey_i *op) {
Signature · 1 line
2.40.1