xfs: refactor and improve inode iteration
v6 cover xfs: refactor and improve inode iteration
Hi all,
This next series refactors all the inode walking code in XFS into a single set of helper functions. The goal is to separate the mechanics of iterating a subset of inode in the filesystem from bulkstat.
First we clean up a few weird things in XFS, then build a generic inode iteration function. Next, we convert the bulkstat ioctl to use it, then fix a few things from some of the code we saved from the old bulkstat inode iteration code. After that, we restructure the code slightly to support the inumbers functionality, and then port the inumbers ioctl to it too.
Finally, we introduce a parallel inode walk feature to speed up quotacheck on large filesystems. The justification for this part is a little questionable since it needs further discovery of what hardware and software this works best on. It's also an open question of whether or not bulkstat could be optimized further.
If you're going to start using this mess, you probably ought to just pull from my git trees, which are linked below.
This is an extraordinary way to destroy everything. Enjoy! Comments and questions are, as always, welcome.
--D
kernel git tree:
https://git.kernel.org/cgit/linux/kernel/git/djwong/xfs-linux.git/log/?h=parallel-iwalk
xfsprogs git tree:
https://git.kernel.org/cgit/linux/kernel/git/djwong/xfsprogs-dev.git/log/?h=parallel-iwalk
PATCH v6 1/15 xfs: create iterator error codes
From: Darrick J. Wong <[email protected]>
Currently, xfs doesn't have generic error codes defined for "stop iterating"; we just reuse the XFS_BTREE_QUERY_* return values. This looks a little weird if we're not actually iterating a btree index. Before we start adding more iterators, we should create general XFS_ITER_{CONTINUE,ABORT} return values and define the XFS_BTREE_QUERY_* ones from that.
--- fs/xfs/libxfs/xfs_alloc.c | 2 +- fs/xfs/libxfs/xfs_btree.h | 4 ++-- fs/xfs/libxfs/xfs_shared.h | 6 ++++++ fs/xfs/scrub/agheader.c | 4 ++-- fs/xfs/scrub/repair.c | 4 ++-- fs/xfs/xfs_dquot.c | 2 +- 6 files changed, 14 insertions(+), 8 deletions(-)
fs/xfs/libxfs/xfs_alloc.c +1 -1
@@ -3146,7 +3146,7 @@ xfs_alloc_has_record(/* * Walk all the blocks in the AGFL. The @walk_fn can return any negative -* error code or XFS_BTREE_QUERY_RANGE_ABORT. +* error code or XFS_ITER_*. */ int xfs_agfl_walk(
fs/xfs/libxfs/xfs_btree.h +2 -2
@@ -465,8 +465,8 @@ uint xfs_btree_compute_maxlevels(uint *limits, unsigned long len);unsigned long long xfs_btree_calc_size (uint *limits,unsigned long long len);/* return codes */ -#define XFS_BTREE_QUERY_RANGE_CONTINUE 0/* keep iterating */ -#define XFS_BTREE_QUERY_RANGE_ABORT 1/* stop iterating */ +#define XFS_BTREE_QUERY_RANGE_CONTINUE (XFS_ITER_CONTINUE)/* keep iterating */ +#define XFS_BTREE_QUERY_RANGE_ABORT (XFS_ITER_ABORT)/* stop iterating */ typedef int (*xfs_btree_query_range_fn )(struct xfs_btree_cur *cur,union xfs_btree_rec *rec,void *priv);
fs/xfs/libxfs/xfs_shared.h +6 -0
@@ -136,4 +136,10 @@ void xfs_symlink_local_to_remote(struct xfs_trans *tp, struct xfs_buf *bp, struct xfs_inode *ip, struct xfs_ifork *ifp);xfs_failaddr_t xfs_symlink_shortform_verify (struct xfs_inode *ip); +/* Keep iterating the data structure. */ +#define XFS_ITER_CONTINUE (0) + +/* Stop iterating the data structure. */ +#define XFS_ITER_ABORT (1) +#endif /* __XFS_SHARED_H__ */
fs/xfs/scrub/agheader.c +2 -2
@@ -646,7 +646,7 @@ xchk_agfl_block(xchk_agfl_block_xref (sc, agbno);if (sc->sm->sm_flags &XFS_SCRUB_OFLAG_CORRUPT ) -return XFS_BTREE_QUERY_RANGE_ABORT ; +return XFS_ITER_ABORT ;return 0 ; }
@@ -737,7 +737,7 @@ xchk_agfl(/* Check the blocks in the AGFL. */ error =xfs_agfl_walk (sc->mp,XFS_BUF_TO_AGF (sc->sa.agf_bp), sc->sa.agfl_bp, xchk_agfl_block, &sai); -if (error ==XFS_BTREE_QUERY_RANGE_ABORT ) { +if (error ==XFS_ITER_ABORT ) { error =0 ; goto out_free; }
fs/xfs/scrub/repair.c +2 -2
@@ -672,7 +672,7 @@ xrep_findroot_agfl_walk( {xfs_agblock_t *agbno = priv; -return (*agbno == bno) ?XFS_BTREE_QUERY_RANGE_ABORT :0 ; +return (*agbno == bno) ?XFS_ITER_ABORT :0 ; }/* Does this block match the btree information passed in? */
@@ -702,7 +702,7 @@ xrep_findroot_block(if (owner ==XFS_RMAP_OWN_AG ) { error =xfs_agfl_walk (mp, ri->agf, ri->agfl_bp, xrep_findroot_agfl_walk, &agbno); -if (error ==XFS_BTREE_QUERY_RANGE_ABORT ) +if (error ==XFS_ITER_ABORT )return 0 ;if (error)return error;
fs/xfs/xfs_dquot.c +1 -1
@@ -1243,7 +1243,7 @@ xfs_qm_exit(void)/* * Iterate every dquot of a particular type. The caller must ensure that the * particular quota type is active. iter_fn can return negative error codes, -* or XFS_BTREE_QUERY_RANGE_ABORT to indicate that it wants to stop iterating. +* or XFS_ITER_ABORT to indicate that it wants to stop iterating. */ int xfs_qm_dqiterate(
PATCH v6 2/15 xfs: create simplified inode walk function
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Create a new iterator function to simplify walking inodes in an XFS filesystem. This new iterator will replace the existing open-coded walking that goes on in various places.
--- fs/xfs/Makefile | 1 fs/xfs/libxfs/xfs_ialloc_btree.c | 37 +++- fs/xfs/libxfs/xfs_ialloc_btree.h | 3 fs/xfs/xfs_itable.c | 5 fs/xfs/xfs_itable.h | 8 + fs/xfs/xfs_iwalk.c | 389 ++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_iwalk.h | 19 ++ fs/xfs/xfs_trace.h | 40 ++++ 8 files changed, 496 insertions(+), 6 deletions(-) create mode 100644 fs/xfs/xfs_iwalk.c create mode 100644 fs/xfs/xfs_iwalk.h
fs/xfs/Makefile +1 -0
@@ -80,6 +80,7 @@ xfs-y += xfs_aops.o \ xfs_iops.o \ xfs_inode.o \ xfs_itable.o \ + xfs_iwalk.o \ xfs_message.o \ xfs_mount.o \ xfs_mru_cache.o \
fs/xfs/libxfs/xfs_ialloc_btree.c +33 -4
@@ -564,6 +564,36 @@ xfs_inobt_max_size( XFS_INODES_PER_CHUNK); } +/* Read AGI and create inobt cursor. */ +int +xfs_inobt_cur ( +struct xfs_mount *mp, +struct xfs_trans *tp, +xfs_agnumber_t agno, +xfs_btnum_t which, +struct xfs_btree_cur **curpp, +struct xfs_buf **agi_bpp) +{ +struct xfs_btree_cur *cur; +int error; + +ASSERT (*agi_bpp ==NULL ); +ASSERT (*curpp ==NULL ); + + error =xfs_ialloc_read_agi (mp, tp, agno, agi_bpp); +if (error) +return error; + + cur =xfs_inobt_init_cursor (mp, tp, *agi_bpp, agno, which); +if (!cur) { +xfs_trans_brelse (tp, *agi_bpp); + *agi_bpp =NULL ; +return -ENOMEM ; + } + *curpp = cur; +return 0 ; +} + static int xfs_inobt_count_blocks( struct xfs_mount *mp,
@@ -572,15 +602,14 @@ xfs_inobt_count_blocks(xfs_btnum_t btnum, xfs_extlen_t *tree_blocks) { - struct xfs_buf *agbp; -struct xfs_btree_cur *cur; + struct xfs_buf *agbp = NULL; +struct xfs_btree_cur *cur =NULL ;int error; - error =xfs_ialloc_read_agi (mp, tp, agno, &agbp); + error =xfs_inobt_cur (mp, tp, agno, btnum, &cur, &agbp);if (error)return error; - cur =xfs_inobt_init_cursor (mp, tp, agbp, agno, btnum); error =xfs_btree_count_blocks (cur, tree_blocks);xfs_btree_del_cursor (cur, error);xfs_trans_brelse (tp, agbp);
fs/xfs/libxfs/xfs_ialloc_btree.h +3 -0
@@ -64,5 +64,8 @@ int xfs_finobt_calc_reserves(struct xfs_mount *mp, struct xfs_trans *tp, xfs_agnumber_t agno, xfs_extlen_t *ask, xfs_extlen_t *used);extern xfs_extlen_t xfs_iallocbt_calc_size (struct xfs_mount *mp,unsigned long long len); +int xfs_inobt_cur (struct xfs_mount *mp,struct xfs_trans *tp, +xfs_agnumber_t agno,xfs_btnum_t btnum, +struct xfs_btree_cur **curpp,struct xfs_buf **agi_bpp);#endif /* __XFS_IALLOC_BTREE_H__ */
fs/xfs/xfs_itable.c +3 -2
@@ -19,6 +19,7 @@#include "xfs_trace.h" #include "xfs_icache.h" #include "xfs_health.h" +#include "xfs_iwalk.h" /* * Return stat information for one inode.
@@ -161,7 +162,7 @@ xfs_bulkstat_one(* Loop over all clusters in a chunk for a given incore inode allocation btree * record. Do a readahead if there are any allocated inodes in that cluster. */ -STATIC void +void xfs_bulkstat_ichunk_ra( struct xfs_mount *mp, xfs_agnumber_t agno,
@@ -195,7 +196,7 @@ xfs_bulkstat_ichunk_ra(* are some left allocated, update the data for the pointed-to record as well as * return the count of grabbed inodes. */ -STATIC int +int xfs_bulkstat_grab_ichunk( struct xfs_btree_cur *cur, /* btree cursor */ xfs_agino_t agino, /* starting inode of chunk */
fs/xfs/xfs_itable.h +8 -0
@@ -84,4 +84,12 @@ xfs_inumbers(void __user *buffer,/* buffer with inode info */ inumbers_fmt_pf formatter); +/* Temporarily needed while we refactor functions. */ +struct xfs_btree_cur ; +struct xfs_inobt_rec_incore ; +void xfs_bulkstat_ichunk_ra (struct xfs_mount *mp,xfs_agnumber_t agno, +struct xfs_inobt_rec_incore *irec); +int xfs_bulkstat_grab_ichunk (struct xfs_btree_cur *cur,xfs_agino_t agino, +int *icount,struct xfs_inobt_rec_incore *irec); +#endif /* __XFS_ITABLE_H__ */
fs/xfs/xfs_iwalk.c new file +389 -0
389 lines not on this page · raw message
fs/xfs/xfs_iwalk.h new file +19 -0
@@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* +* Copyright (C) 2019 Oracle. All Rights Reserved. +* Author: Darrick J. Wong <[email protected]> +*/ +#ifndef __XFS_IWALK_H__ +#define __XFS_IWALK_H__ + +/* Walk all inodes in the filesystem starting from @startino. */ +typedef int (*xfs_iwalk_fn )(struct xfs_mount *mp,struct xfs_trans *tp, +xfs_ino_t ino,void *data); +/* Return values for xfs_iwalk_fn. */ +#define XFS_IWALK_CONTINUE (XFS_ITER_CONTINUE) +#define XFS_IWALK_ABORT (XFS_ITER_ABORT) + +int xfs_iwalk (struct xfs_mount *mp,struct xfs_trans *tp,xfs_ino_t startino, +xfs_iwalk_fn iwalk_fn,unsigned int inode_records,void *data); + +#endif /* __XFS_IWALK_H__ */
fs/xfs/xfs_trace.h +40 -0
@@ -3516,6 +3516,46 @@ DEFINE_EVENT(xfs_inode_corrupt_class, name, \DEFINE_INODE_CORRUPT_EVENT (xfs_inode_mark_sick);DEFINE_INODE_CORRUPT_EVENT (xfs_inode_mark_healthy); +TRACE_EVENT (xfs_iwalk_ag, + TP_PROTO(struct xfs_mount *mp, xfs_agnumber_t agno, + xfs_agino_t startino), +TP_ARGS (mp, agno, startino), +TP_STRUCT__entry ( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) +__field (xfs_agino_t, startino) + ), +TP_fast_assign ( + __entry->dev = mp->m_super->s_dev; + __entry->agno = agno; + __entry->startino = startino; + ), +TP_printk ("dev %d:%d agno %d startino %u" , +MAJOR (__entry->dev),MINOR (__entry->dev), __entry->agno, + __entry->startino) +) + +TRACE_EVENT (xfs_iwalk_ag_rec, + TP_PROTO(struct xfs_mount *mp, xfs_agnumber_t agno, + struct xfs_inobt_rec_incore *irec), +TP_ARGS (mp, agno, irec), +TP_STRUCT__entry ( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, startino) +__field (uint64_t, freemask) + ), +TP_fast_assign ( + __entry->dev = mp->m_super->s_dev; + __entry->agno = agno; + __entry->startino = irec->ir_startino; + __entry->freemask = irec->ir_free; + ), +TP_printk ("dev %d:%d agno %d startino %u freemask 0x%llx" , +MAJOR (__entry->dev),MINOR (__entry->dev), __entry->agno, + __entry->startino, __entry->freemask) +) +#endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH
Re: [PATCH 02/15] xfs: create simplified inode walk function
Quoted 8 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Create a new iterator function to simplify walking inodes in an XFS filesystem. This new iterator will replace the existing open-coded walking that goes on in various places.
Signed-off-by Darrick J. Wong <[email protected]>---
Looks good:
Quoted 617 lines
fs/xfs/Makefile | 1 fs/xfs/libxfs/xfs_ialloc_btree.c | 37 +++- fs/xfs/libxfs/xfs_ialloc_btree.h | 3 fs/xfs/xfs_itable.c | 5 fs/xfs/xfs_itable.h | 8 + fs/xfs/xfs_iwalk.c | 389 ++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_iwalk.h | 19 ++ fs/xfs/xfs_trace.h | 40 ++++ 8 files changed, 496 insertions(+), 6 deletions(-) create mode 100644 fs/xfs/xfs_iwalk.c create mode 100644 fs/xfs/xfs_iwalk.hfs/xfs/Makefile +1
-0@@ -80,6 +80,7 @@ xfs-y += xfs_aops.o \ xfs_iops.o \ xfs_inode.o \ xfs_itable.o \ + xfs_iwalk.o \ xfs_message.o \ xfs_mount.o \ xfs_mru_cache.o \fs/xfs/libxfs/xfs_ialloc_btree.c +33
-4@@ -564,6 +564,36 @@ xfs_inobt_max_size( XFS_INODES_PER_CHUNK); } +/* Read AGI and create inobt cursor. */ +int +xfs_inobt_cur ( +struct xfs_mount *mp, +struct xfs_trans *tp, +xfs_agnumber_t agno, +xfs_btnum_t which, +struct xfs_btree_cur **curpp, +struct xfs_buf **agi_bpp) +{ +struct xfs_btree_cur *cur; +int error; + +ASSERT (*agi_bpp ==NULL ); +ASSERT (*curpp ==NULL ); + + error =xfs_ialloc_read_agi (mp, tp, agno, agi_bpp); +if (error) +return error; + + cur =xfs_inobt_init_cursor (mp, tp, *agi_bpp, agno, which); +if (!cur) { +xfs_trans_brelse (tp, *agi_bpp); + *agi_bpp =NULL ; +return -ENOMEM ; + } + *curpp = cur; +return 0 ; +} + static int xfs_inobt_count_blocks( struct xfs_mount *mp,@@ -572,15 +602,14 @@ xfs_inobt_count_blocks(xfs_btnum_t btnum, xfs_extlen_t *tree_blocks) { - struct xfs_buf *agbp; -struct xfs_btree_cur *cur; + struct xfs_buf *agbp = NULL; +struct xfs_btree_cur *cur =NULL ;int error; - error =xfs_ialloc_read_agi (mp, tp, agno, &agbp); + error =xfs_inobt_cur (mp, tp, agno, btnum, &cur, &agbp);if (error)return error; - cur =xfs_inobt_init_cursor (mp, tp, agbp, agno, btnum); error =xfs_btree_count_blocks (cur, tree_blocks);xfs_btree_del_cursor (cur, error);xfs_trans_brelse (tp, agbp);fs/xfs/libxfs/xfs_ialloc_btree.h +3
-0@@ -64,5 +64,8 @@ int xfs_finobt_calc_reserves(struct xfs_mount *mp, struct xfs_trans *tp, xfs_agnumber_t agno, xfs_extlen_t *ask, xfs_extlen_t *used);extern xfs_extlen_t xfs_iallocbt_calc_size (struct xfs_mount *mp,unsigned long long len); +int xfs_inobt_cur (struct xfs_mount *mp,struct xfs_trans *tp, +xfs_agnumber_t agno,xfs_btnum_t btnum, +struct xfs_btree_cur **curpp,struct xfs_buf **agi_bpp);#endif /* __XFS_IALLOC_BTREE_H__ */ fs/xfs/xfs_itable.c +3
-2@@ -19,6 +19,7 @@#include "xfs_trace.h" #include "xfs_icache.h" #include "xfs_health.h" +#include "xfs_iwalk.h" /* * Return stat information for one inode. @@ -161,7 +162,7 @@ xfs_bulkstat_one(* Loop over all clusters in a chunk for a given incore inode allocation btree * record. Do a readahead if there are any allocated inodes in that cluster. */ -STATIC void +void xfs_bulkstat_ichunk_ra( struct xfs_mount *mp, xfs_agnumber_t agno,@@ -195,7 +196,7 @@ xfs_bulkstat_ichunk_ra(* are some left allocated, update the data for the pointed-to record as well as * return the count of grabbed inodes. */ -STATIC int +int xfs_bulkstat_grab_ichunk( struct xfs_btree_cur *cur, /* btree cursor */ xfs_agino_t agino, /* starting inode of chunk */fs/xfs/xfs_itable.h +8
-0@@ -84,4 +84,12 @@ xfs_inumbers(void __user *buffer,/* buffer with inode info */ inumbers_fmt_pf formatter); +/* Temporarily needed while we refactor functions. */ +struct xfs_btree_cur ; +struct xfs_inobt_rec_incore ; +void xfs_bulkstat_ichunk_ra (struct xfs_mount *mp,xfs_agnumber_t agno, +struct xfs_inobt_rec_incore *irec); +int xfs_bulkstat_grab_ichunk (struct xfs_btree_cur *cur,xfs_agino_t agino, +int *icount,struct xfs_inobt_rec_incore *irec); +#endif /* __XFS_ITABLE_H__ */ fs/xfs/xfs_iwalk.c new file +389
-0@@ -0,0 +1,389 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* +* Copyright (C) 2019 Oracle. All Rights Reserved. +* Author: Darrick J. Wong <[email protected]> +*/ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_iwalk.h" +#include "xfs_itable.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_icache.h" +#include "xfs_health.h" +#include "xfs_trans.h" + +/* +* Walking Inodes in the Filesystem +* ================================ +* +* This iterator function walks a subset of filesystem inodes in increasing +* order from @startino until there are no more inodes. For each allocated +* inode it finds, it calls a walk function with the relevant inode number and +* a pointer to caller-provided data. The walk function can return the usual +* negative error code to stop the iteration; 0 to continue the iteration; or +* XFS_IWALK_ABORT to stop the iteration. This return value is returned to the +* caller. +* +* Internally, we allow the walk function to do anything, which means that we +* cannot maintain the inobt cursor or our lock on the AGI buffer. We +* therefore cache the inobt records in kernel memory and only call the walk +* function when our memory buffer is full. @nr_recs is the number of records +* that we've cached, and @sz_recs is the size of our cache. +* +* It is the responsibility of the walk function to ensure it accesses +* allocated inodes, as the inobt records may be stale by the time they are +* acted upon. +*/ + +struct xfs_iwalk_ag { +struct xfs_mount *mp; +struct xfs_trans *tp; + +/* Where do we start the traversal? */ +xfs_ino_t startino; + +/* Array of inobt records we cache. */ +struct xfs_inobt_rec_incore *recs; + +/* Number of entries allocated for the @recs array. */ +unsigned int sz_recs; + +/* Number of entries in the @recs array that are in use. */ +unsigned int nr_recs; + +/* Inode walk function and data pointer. */ +xfs_iwalk_fn iwalk_fn; +void *data; +}; + +/* Allocate memory for a walk. */ +STATIC int +xfs_iwalk_alloc ( +struct xfs_iwalk_ag *iwag) +{ +size_t size; + +ASSERT (iwag->recs ==NULL ); + iwag->nr_recs =0 ; + +/* Allocate a prefetch buffer for inobt records. */ + size = iwag->sz_recs *sizeof (struct xfs_inobt_rec_incore ); + iwag->recs =kmem_alloc (size,KM_MAYFAIL ); +if (iwag->recs ==NULL ) +return -ENOMEM ; + +return 0 ; +} + +/* Free memory we allocated for a walk. */ +STATIC void +xfs_iwalk_free ( +struct xfs_iwalk_ag *iwag) +{ +kmem_free (iwag->recs); + iwag->recs =NULL ; +} + +/* For each inuse inode in each cached inobt record, call our function. */ +STATIC int +xfs_iwalk_ag_recs ( +struct xfs_iwalk_ag *iwag) +{ +struct xfs_mount *mp = iwag->mp; +struct xfs_trans *tp = iwag->tp; +xfs_ino_t ino; +unsigned int i, j; +xfs_agnumber_t agno; +int error; + + agno =XFS_INO_TO_AGNO (mp, iwag->startino); +for (i =0 ; i < iwag->nr_recs; i++) { +struct xfs_inobt_rec_incore *irec = &iwag->recs[i]; + +trace_xfs_iwalk_ag_rec (mp, agno, irec); + +for (j =0 ; j <XFS_INODES_PER_CHUNK ; j++) { +/* Skip if this inode is free */ +if (XFS_INOBT_MASK (j) & irec->ir_free) +continue ; + +/* Otherwise call our function. */ + ino =XFS_AGINO_TO_INO (mp, agno, irec->ir_startino + j); + error = iwag->iwalk_fn (mp, tp, ino, iwag->data); +if (error) +return error; + } + } + +return 0 ; +} + +/* Delete cursor and let go of AGI. */ +static inline void +xfs_iwalk_del_inobt ( +struct xfs_trans *tp, +struct xfs_btree_cur **curpp, +struct xfs_buf **agi_bpp, +int error) +{ +if (*curpp) { +xfs_btree_del_cursor (*curpp, error); + *curpp =NULL ; + } +if (*agi_bpp) { +xfs_trans_brelse (tp, *agi_bpp); + *agi_bpp =NULL ; + } +} + +/* +* Set ourselves up for walking inobt records starting from a given point in +* the filesystem. +* +* If caller passed in a nonzero start inode number, load the record from the +* inobt and make the record look like all the inodes before agino are free so +* that we skip them, and then move the cursor to the next inobt record. This +* is how we support starting an iwalk in the middle of an inode chunk. +* +* If the caller passed in a start number of zero, move the cursor to the first +* inobt record. +* +* The caller is responsible for cleaning up the cursor and buffer pointer +* regardless of the error status. +*/ +STATIC int +xfs_iwalk_ag_start ( +struct xfs_iwalk_ag *iwag, +xfs_agnumber_t agno, +xfs_agino_t agino, +struct xfs_btree_cur **curpp, +struct xfs_buf **agi_bpp, +int *has_more) +{ +struct xfs_mount *mp = iwag->mp; +struct xfs_trans *tp = iwag->tp; +int icount; +int error; + +/* Set up a fresh cursor and empty the inobt cache. */ + iwag->nr_recs =0 ; + error =xfs_inobt_cur (mp, tp, agno,XFS_BTNUM_INO , curpp, agi_bpp); +if (error) +return error; + +/* Starting at the beginning of the AG? That's easy! */ +if (agino ==0 ) +return xfs_inobt_lookup (*curpp,0 ,XFS_LOOKUP_GE , has_more); + +/* +* Otherwise, we have to grab the inobt record where we left off, stuff +* the record into our cache, and then see if there are more records. +* We require a lookup cache of at least two elements so that we don't +* have to deal with tearing down the cursor to walk the records. +*/ + error =xfs_bulkstat_grab_ichunk (*curpp, agino -1 , &icount, + &iwag->recs[iwag->nr_recs]); +if (error) +return error; +if (icount) + iwag->nr_recs++; + +/* +* The prefetch calculation is supposed to give us a large enough inobt +* record cache that grab_ichunk can stage a partial first record and +* the loop body can cache a record without having to check for cache +* space until after it reads an inobt record. +*/ +ASSERT (iwag->nr_recs < iwag->sz_recs); + +return xfs_btree_increment (*curpp,0 , has_more); +} + +/* +* The inobt record cache is full, so preserve the inobt cursor state and +* run callbacks on the cached inobt records. When we're done, restore the +* cursor state to wherever the cursor would have been had the cache not been +* full (and therefore we could've just incremented the cursor) if *@has_more +* is true. On exit, *@has_more will indicate whether or not the caller should +* try for more inode records. +*/ +STATIC int +xfs_iwalk_run_callbacks ( +struct xfs_iwalk_ag *iwag, +xfs_agnumber_t agno, +struct xfs_btree_cur **curpp, +struct xfs_buf **agi_bpp, +int *has_more) +{ +struct xfs_mount *mp = iwag->mp; +struct xfs_trans *tp = iwag->tp; +struct xfs_inobt_rec_incore *irec; +xfs_agino_t restart; +int error; + +ASSERT (iwag->nr_recs >0 ); + +/* Delete cursor but remember the last record we cached... */ +xfs_iwalk_del_inobt (tp, curpp, agi_bpp,0 ); + irec = &iwag->recs[iwag->nr_recs -1 ]; + restart = irec->ir_startino +XFS_INODES_PER_CHUNK -1 ; + + error =xfs_iwalk_ag_recs (iwag); +if (error) +return error; + +/* ...empty the cache... */ + iwag->nr_recs =0 ; + +if (!has_more) +return 0 ; + +/* ...and recreate the cursor just past where we left off. */ + error =xfs_inobt_cur (mp, tp, agno,XFS_BTNUM_INO , curpp, agi_bpp); +if (error) +return error; + +return xfs_inobt_lookup (*curpp, restart,XFS_LOOKUP_GE , has_more); +} + +/* Walk all inodes in a single AG, from @iwag->startino to the end of the AG. */ +STATIC int +xfs_iwalk_ag ( +struct xfs_iwalk_ag *iwag) +{ +struct xfs_mount *mp = iwag->mp; +struct xfs_trans *tp = iwag->tp; +struct xfs_buf *agi_bp =NULL ; +struct xfs_btree_cur *cur =NULL ; +xfs_agnumber_t agno; +xfs_agino_t agino; +int has_more; +int error =0 ; + +/* Set up our cursor at the right place in the inode btree. */ + agno =XFS_INO_TO_AGNO (mp, iwag->startino); + agino =XFS_INO_TO_AGINO (mp, iwag->startino); + error =xfs_iwalk_ag_start (iwag, agno, agino, &cur, &agi_bp, &has_more); + +while (!error && has_more) { +struct xfs_inobt_rec_incore *irec; + +cond_resched (); + +/* Fetch the inobt record. */ + irec = &iwag->recs[iwag->nr_recs]; + error =xfs_inobt_get_rec (cur, irec, &has_more); +if (error || !has_more) +break ; + +/* No allocated inodes in this chunk; skip it. */ +if (irec->ir_freecount == irec->ir_count) { + error =xfs_btree_increment (cur,0 , &has_more); +if (error) +break ; +continue ; + } + +/* +* Start readahead for this inode chunk in anticipation of +* walking the inodes. +*/ +xfs_bulkstat_ichunk_ra (mp, agno, irec); + +/* +* If there's space in the buffer for more records, increment +* the btree cursor and grab more. +*/ +if (++iwag->nr_recs < iwag->sz_recs) { + error =xfs_btree_increment (cur,0 , &has_more); +if (error || !has_more) +break ; +continue ; + } + +/* +* Otherwise, we need to save cursor state and run the callback +* function on the cached records. The run_callbacks function +* is supposed to return a cursor pointing to the record where +* we would be if we had been able to increment like above. +*/ +ASSERT (has_more); + error =xfs_iwalk_run_callbacks (iwag, agno, &cur, &agi_bp, + &has_more); + } + +if (iwag->nr_recs ==0 || error) + goto out; + +/* Walk the unprocessed records in the cache. */ + error =xfs_iwalk_run_callbacks (iwag, agno, &cur, &agi_bp, &has_more); + +out: +xfs_iwalk_del_inobt (tp, &cur, &agi_bp, error); +return error; +} + +/* +* Given the number of inodes to prefetch, set the number of inobt records that +* we cache in memory, which controls the number of inodes we try to read +* ahead. +*/ +static inline unsigned int +xfs_iwalk_prefetch ( +unsigned int inode_records) +{ +return PAGE_SIZE *4 /sizeof (struct xfs_inobt_rec_incore ); +} + +/* +* Walk all inodes in the filesystem starting from @startino. The @iwalk_fn +* will be called for each allocated inode, being passed the inode's number and +* @data. @max_prefetch controls how many inobt records' worth of inodes we +* try to readahead. +*/ +int +xfs_iwalk ( +struct xfs_mount *mp, +struct xfs_trans *tp, +xfs_ino_t startino, +xfs_iwalk_fn iwalk_fn, +unsigned int inode_records, +void *data) +{ +struct xfs_iwalk_ag iwag = { + .mp = mp, + .tp = tp, + .iwalk_fn = iwalk_fn, + .data = data, + .startino = startino, + .sz_recs =xfs_iwalk_prefetch (inode_records), + }; +xfs_agnumber_t agno =XFS_INO_TO_AGNO (mp, startino); +int error; + +ASSERT (agno < mp->m_sb.sb_agcount); + + error =xfs_iwalk_alloc (&iwag); +if (error) +return error; + +for (; agno < mp->m_sb.sb_agcount; agno++) { + error =xfs_iwalk_ag (&iwag); +if (error) +break ; + iwag.startino =XFS_AGINO_TO_INO (mp, agno +1 ,0 ); + } + +xfs_iwalk_free (&iwag); +return error; +}fs/xfs/xfs_iwalk.h new file +19
-0@@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* +* Copyright (C) 2019 Oracle. All Rights Reserved. +* Author: Darrick J. Wong <[email protected]> +*/ +#ifndef __XFS_IWALK_H__ +#define __XFS_IWALK_H__ + +/* Walk all inodes in the filesystem starting from @startino. */ +typedef int (*xfs_iwalk_fn )(struct xfs_mount *mp,struct xfs_trans *tp, +xfs_ino_t ino,void *data); +/* Return values for xfs_iwalk_fn. */ +#define XFS_IWALK_CONTINUE (XFS_ITER_CONTINUE) +#define XFS_IWALK_ABORT (XFS_ITER_ABORT) + +int xfs_iwalk (struct xfs_mount *mp,struct xfs_trans *tp,xfs_ino_t startino, +xfs_iwalk_fn iwalk_fn,unsigned int inode_records,void *data); + +#endif /* __XFS_IWALK_H__ */ fs/xfs/xfs_trace.h +40
-0@@ -3516,6 +3516,46 @@ DEFINE_EVENT(xfs_inode_corrupt_class, name, \DEFINE_INODE_CORRUPT_EVENT (xfs_inode_mark_sick);DEFINE_INODE_CORRUPT_EVENT (xfs_inode_mark_healthy); +TRACE_EVENT (xfs_iwalk_ag, + TP_PROTO(struct xfs_mount *mp, xfs_agnumber_t agno, + xfs_agino_t startino), +TP_ARGS (mp, agno, startino), +TP_STRUCT__entry ( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) +__field (xfs_agino_t, startino) + ), +TP_fast_assign ( + __entry->dev = mp->m_super->s_dev; + __entry->agno = agno; + __entry->startino = startino; + ), +TP_printk ("dev %d:%d agno %d startino %u" , +MAJOR (__entry->dev),MINOR (__entry->dev), __entry->agno, + __entry->startino) +) + +TRACE_EVENT (xfs_iwalk_ag_rec, + TP_PROTO(struct xfs_mount *mp, xfs_agnumber_t agno, + struct xfs_inobt_rec_incore *irec), +TP_ARGS (mp, agno, irec), +TP_STRUCT__entry ( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, startino) +__field (uint64_t, freemask) + ), +TP_fast_assign ( + __entry->dev = mp->m_super->s_dev; + __entry->agno = agno; + __entry->startino = irec->ir_startino; + __entry->freemask = irec->ir_free; + ), +TP_printk ("dev %d:%d agno %d startino %u freemask 0x%llx" , +MAJOR (__entry->dev),MINOR (__entry->dev), __entry->agno, + __entry->startino, __entry->freemask) +) +#endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH
PATCH v6 3/15 xfs: convert quotacheck to use the new iwalk functions
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Convert quotacheck to use the new iwalk iterator to dig through the inodes.
--- fs/xfs/xfs_qm.c | 63 +++++++++++++++++-------------------------------------- 1 file changed, 20 insertions(+), 43 deletions(-)
fs/xfs/xfs_qm.c +20 -43
@@ -14,7 +14,7 @@#include "xfs_mount.h" #include "xfs_inode.h" #include "xfs_ialloc.h" -#include "xfs_itable.h" +#include "xfs_iwalk.h" #include "xfs_quota.h" #include "xfs_error.h" #include "xfs_bmap.h"
@@ -1118,17 +1118,15 @@ xfs_qm_quotacheck_dqadjust(/* ARGSUSED */ STATIC intxfs_qm_dqusage_adjust ( -xfs_mount_t *mp,/* mount point for filesystem */ -xfs_ino_t ino,/* inode number to get data for */ -void __user *buffer,/* not used */ -int ubsize,/* not used */ -int *ubused,/* not used */ -int *res)/* result code value */ +struct xfs_mount *mp, +struct xfs_trans *tp, +xfs_ino_t ino, +void *data) { -xfs_inode_t *ip; -xfs_qcnt_t nblks; -xfs_filblks_t rtblks =0 ;/* total rt blks */ -int error; +struct xfs_inode *ip; +xfs_qcnt_t nblks; +xfs_filblks_t rtblks =0 ;/* total rt blks */ +int error;ASSERT (XFS_IS_QUOTA_RUNNING (mp));
@@ -1136,20 +1134,18 @@ xfs_qm_dqusage_adjust(* rootino must have its resources accounted for, not so with the quota * inodes. */ -if (xfs_is_quota_inode (&mp->m_sb, ino)) { - *res =BULKSTAT_RV_NOTHING ; -return -EINVAL ; - } +if (xfs_is_quota_inode (&mp->m_sb, ino)) +return 0 ;/* * We don't _need_ to take the ilock EXCL here because quotacheck runs * at mount time and therefore nobody will be racing chown/chproj. */ - error =xfs_iget (mp,NULL , ino,XFS_IGET_DONTCACHE ,0 , &ip); -if (error) { - *res =BULKSTAT_RV_NOTHING ; + error =xfs_iget (mp, tp, ino,XFS_IGET_DONTCACHE ,0 , &ip); +if (error == -EINVAL || error == -ENOENT ) +return 0 ; +if (error)return error; - }ASSERT (ip->i_delayed_blks ==0 );
@@ -1157,7 +1153,7 @@ xfs_qm_dqusage_adjust(struct xfs_ifork *ifp =XFS_IFORK_PTR (ip,XFS_DATA_FORK );if (!(ifp->if_flags &XFS_IFEXTENTS )) { - error =xfs_iread_extents (NULL , ip,XFS_DATA_FORK ); + error =xfs_iread_extents (tp, ip,XFS_DATA_FORK );if (error) goto error0; }
@@ -1200,13 +1196,8 @@ xfs_qm_dqusage_adjust( goto error0; } -xfs_irele (ip); - *res =BULKSTAT_RV_DIDONE ; -return 0 ; - error0:xfs_irele (ip); - *res =BULKSTAT_RV_GIVEUP ;return error; }
@@ -1270,18 +1261,13 @@ STATIC intxfs_qm_quotacheck ( xfs_mount_t *mp) { -int done, count, error, error2; -xfs_ino_t lastino; -size_t structsz; +int error, error2;uint flags;LIST_HEAD (buffer_list);struct xfs_inode *uip = mp->m_quotainfo->qi_uquotaip;struct xfs_inode *gip = mp->m_quotainfo->qi_gquotaip;struct xfs_inode *pip = mp->m_quotainfo->qi_pquotaip; - count =INT_MAX ; - structsz =1 ; - lastino =0 ; flags =0 ;ASSERT (uip || gip || pip);
@@ -1318,18 +1304,9 @@ xfs_qm_quotacheck( flags |=XFS_PQUOTA_CHKD ; } -do { -/* -* Iterate thru all the inodes in the file system, -* adjusting the corresponding dquot counters in core. -*/ - error =xfs_bulkstat (mp, &lastino, &count, - xfs_qm_dqusage_adjust, - structsz,NULL , &done); -if (error) -break ; - - }while (!done); + error =xfs_iwalk (mp,NULL ,0 , xfs_qm_dqusage_adjust,0 ,NULL ); +if (error) + goto error_return;/* * We've made all the changes that we need to make incore. Flush them
PATCH v6 4/15 xfs: bulkstat should copy lastip whenever userspace supplies one
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
When userspace passes in a @lastip pointer we should copy the results back, even if the @ocount pointer is NULL.
--- fs/xfs/xfs_ioctl.c | 13 ++++++------- fs/xfs/xfs_ioctl32.c | 13 ++++++------- 2 files changed, 12 insertions(+), 14 deletions(-)
fs/xfs/xfs_ioctl.c +6 -7
@@ -768,14 +768,13 @@ xfs_ioc_bulkstat(if (error)return error; -if (bulkreq.ocount !=NULL ) { -if (copy_to_user (bulkreq.lastip, &inlast, -sizeof (xfs_ino_t))) -return -EFAULT ; +if (bulkreq.lastip !=NULL && +copy_to_user (bulkreq.lastip, &inlast,sizeof (xfs_ino_t))) +return -EFAULT ; -if (copy_to_user (bulkreq.ocount, &count,sizeof (count))) -return -EFAULT ; - } +if (bulkreq.ocount !=NULL && +copy_to_user (bulkreq.ocount, &count,sizeof (count))) +return -EFAULT ;return 0 ; }
fs/xfs/xfs_ioctl32.c +6 -7
@@ -310,14 +310,13 @@ xfs_compat_ioc_bulkstat(if (error)return error; -if (bulkreq.ocount !=NULL ) { -if (copy_to_user (bulkreq.lastip, &inlast, -sizeof (xfs_ino_t))) -return -EFAULT ; +if (bulkreq.lastip !=NULL && +copy_to_user (bulkreq.lastip, &inlast,sizeof (xfs_ino_t))) +return -EFAULT ; -if (copy_to_user (bulkreq.ocount, &count,sizeof (count))) -return -EFAULT ; - } +if (bulkreq.ocount !=NULL && +copy_to_user (bulkreq.ocount, &count,sizeof (count))) +return -EFAULT ;return 0 ; }
PATCH v6 5/15 xfs: remove unnecessary includes of xfs_itable.h
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Don't include xfs_itable.h in files that don't need it.
--- fs/xfs/scrub/common.c | 1 - fs/xfs/scrub/dir.c | 1 - fs/xfs/scrub/scrub.c | 1 - fs/xfs/xfs_trace.c | 1 - 4 files changed, 4 deletions(-)
fs/xfs/scrub/common.c +0 -1
@@ -17,7 +17,6 @@#include "xfs_sb.h" #include "xfs_inode.h" #include "xfs_icache.h" -#include "xfs_itable.h" #include "xfs_alloc.h" #include "xfs_alloc_btree.h" #include "xfs_bmap.h"
fs/xfs/scrub/dir.c +0 -1
@@ -17,7 +17,6 @@#include "xfs_sb.h" #include "xfs_inode.h" #include "xfs_icache.h" -#include "xfs_itable.h" #include "xfs_da_format.h" #include "xfs_da_btree.h" #include "xfs_dir2.h"
fs/xfs/scrub/scrub.c +0 -1
@@ -17,7 +17,6 @@#include "xfs_sb.h" #include "xfs_inode.h" #include "xfs_icache.h" -#include "xfs_itable.h" #include "xfs_alloc.h" #include "xfs_alloc_btree.h" #include "xfs_bmap.h"
fs/xfs/xfs_trace.c +0 -1
@@ -16,7 +16,6 @@#include "xfs_btree.h" #include "xfs_da_btree.h" #include "xfs_ialloc.h" -#include "xfs_itable.h" #include "xfs_alloc.h" #include "xfs_bmap.h" #include "xfs_attr.h"
PATCH v6 6/15 xfs: convert bulkstat to new iwalk infrastructure
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Create a new ibulk structure incore to help us deal with bulk inode stat state tracking and then convert the bulkstat code to use the new iwalk iterator. This disentangles inode walking from bulk stat control for simpler code and enables us to isolate the formatter functions to the ioctl handling code.
--- fs/xfs/xfs_ioctl.c | 70 ++++++-- fs/xfs/xfs_ioctl.h | 5 + fs/xfs/xfs_ioctl32.c | 92 +++++------ fs/xfs/xfs_itable.c | 428 +++++++++++++++----------------------------------- fs/xfs/xfs_itable.h | 76 +++------ 5 files changed, 265 insertions(+), 406 deletions(-)
fs/xfs/xfs_ioctl.c +53 -17
@@ -14,6 +14,7 @@#include "xfs_ioctl.h" #include "xfs_alloc.h" #include "xfs_rtalloc.h" +#include "xfs_iwalk.h" #include "xfs_itable.h" #include "xfs_error.h" #include "xfs_attr.h"
@@ -721,16 +722,29 @@ xfs_ioc_space(return error; } +/* Return 0 on success or positive error */ +int +xfs_bulkstat_one_fmt ( +struct xfs_ibulk *breq, +const struct xfs_bstat *bstat) +{ +if (copy_to_user (breq->ubuffer, bstat,sizeof (*bstat))) +return -EFAULT ; +return xfs_ibulk_advance (breq,sizeof (struct xfs_bstat )); +} + STATIC intxfs_ioc_bulkstat (xfs_mount_t *mp,unsigned int cmd,void __user *arg) { -xfs_fsop_bulkreq_t bulkreq; -int count;/* # of records returned */ -xfs_ino_t inlast;/* last inode number */ -int done; +struct xfs_fsop_bulkreq bulkreq; +struct xfs_ibulk breq = { + .mp = mp, + .ocount =0 , + }; +xfs_ino_t lastino;int error;/* done = 1 if there are more stats to get and if bulkstat */
@@ -745,35 +759,57 @@ xfs_ioc_bulkstat(if (copy_from_user (&bulkreq, arg,sizeof (xfs_fsop_bulkreq_t)))return -EFAULT ; -if (copy_from_user (&inlast, bulkreq.lastip,sizeof (__s64))) +if (copy_from_user (&lastino, bulkreq.lastip,sizeof (__s64)))return -EFAULT ; -if ((count = bulkreq.icount) <=0 ) +if (bulkreq.icount <=0 )return -EINVAL ;if (bulkreq.ubuffer ==NULL )return -EINVAL ; -if (cmd ==XFS_IOC_FSINUMBERS ) - error =xfs_inumbers (mp, &inlast, &count, + breq.ubuffer = bulkreq.ubuffer; + breq.icount = bulkreq.icount; + +/* +* FSBULKSTAT_SINGLE expects that *lastip contains the inode number +* that we want to stat. However, FSINUMBERS and FSBULKSTAT expect +* that *lastip contains either zero or the number of the last inode to +* be examined by the previous call and return results starting with +* the next inode after that. The new bulk request back end functions +* take the inode to start with, so we have to compute the startino +* parameter from lastino to maintain correct function. lastino == 0 +* is a special case because it has traditionally meant "first inode +* in filesystem". +*/ +if (cmd ==XFS_IOC_FSINUMBERS ) { +int count = breq.icount; + + breq.startino = lastino; + error =xfs_inumbers (mp, &breq.startino, &count, bulkreq.ubuffer, xfs_inumbers_fmt); -else if (cmd ==XFS_IOC_FSBULKSTAT_SINGLE ) - error =xfs_bulkstat_one (mp, inlast, bulkreq.ubuffer, -sizeof (xfs_bstat_t),NULL , &done); -else /* XFS_IOC_FSBULKSTAT */ - error =xfs_bulkstat (mp, &inlast, &count, xfs_bulkstat_one, -sizeof (xfs_bstat_t), bulkreq.ubuffer, - &done); + breq.ocount = count; + lastino = breq.startino; + }else if (cmd ==XFS_IOC_FSBULKSTAT_SINGLE ) { + breq.startino = lastino; + breq.icount =1 ; + error =xfs_bulkstat_one (&breq, xfs_bulkstat_one_fmt); + lastino = breq.startino; + }else {/* XFS_IOC_FSBULKSTAT */ + breq.startino = lastino ? lastino +1 :0 ; + error =xfs_bulkstat (&breq, xfs_bulkstat_one_fmt); + lastino = breq.startino -1 ; + }if (error)return error;if (bulkreq.lastip !=NULL && -copy_to_user (bulkreq.lastip, &inlast,sizeof (xfs_ino_t))) +copy_to_user (bulkreq.lastip, &lastino,sizeof (xfs_ino_t)))return -EFAULT ;if (bulkreq.ocount !=NULL && -copy_to_user (bulkreq.ocount, &count,sizeof (count))) +copy_to_user (bulkreq.ocount, &breq.ocount,sizeof (__s32)))return -EFAULT ;return 0 ;
fs/xfs/xfs_ioctl.h +5 -0
@@ -77,4 +77,9 @@ xfs_set_dmattrs(uint evmask, uint16_t state); +struct xfs_ibulk ; +struct xfs_bstat ; + +int xfs_bulkstat_one_fmt (struct xfs_ibulk *breq,const struct xfs_bstat *bstat); +#endif
fs/xfs/xfs_ioctl32.c +47 -45
160 lines not on this page · raw message
fs/xfs/xfs_itable.c +132 -296
495 lines not on this page · raw message
fs/xfs/xfs_itable.h +28 -48
91 lines not on this page · raw message
PATCH v6 7/15 xfs: calculate inode walk prefetch more carefully
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
The existing inode walk prefetch is based on the old bulkstat code, which simply allocated 4 pages worth of memory and prefetched that many inobt records, regardless of however many inodes the caller requested. 65536 inodes is a lot to prefetch (~32M on x64, ~512M on arm64) so let's scale things down a little more intelligently based on the number of inodes requested, etc.
--- fs/xfs/xfs_iwalk.c | 46 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-)
fs/xfs/xfs_iwalk.c +44 -2
@@ -333,16 +333,58 @@ xfs_iwalk_ag(return error; } +/* +* We experimentally determined that the reduction in ioctl call overhead +* diminishes when userspace asks for more than 2048 inodes, so we'll cap +* prefetch at this point. +*/ +#define MAX_IWALK_PREFETCH (2048U) +/* * Given the number of inodes to prefetch, set the number of inobt records that * we cache in memory, which controls the number of inodes we try to read -* ahead. +* ahead. Set the maximum if @inode_records == 0. */ static inline unsigned int xfs_iwalk_prefetch (unsigned int inode_records) { -return PAGE_SIZE *4 /sizeof (struct xfs_inobt_rec_incore ); +unsigned int inobt_records; + +/* +* If the caller didn't tell us the number of inodes they wanted, +* assume the maximum prefetch possible for best performance. +* Otherwise, cap prefetch at that maximum so that we don't start an +* absurd amount of prefetch. +*/ +if (inode_records ==0 ) + inode_records =MAX_IWALK_PREFETCH ; + inode_records =min (inode_records,MAX_IWALK_PREFETCH ); + +/* Round the inode count up to a full chunk. */ + inode_records =round_up (inode_records,XFS_INODES_PER_CHUNK ); + +/* +* In order to convert the number of inodes to prefetch into an +* estimate of the number of inobt records to cache, we require a +* conversion factor that reflects our expectations of the average +* loading factor of an inode chunk. Based on data gathered, most +* (but not all) filesystems manage to keep the inode chunks totally +* full, so we'll underestimate slightly so that our readahead will +* still deliver the performance we want on aging filesystems: +* +* inobt = inodes / (INODES_PER_CHUNK * (4 / 5)); +* +* The funny math is to avoid division. +*/ + inobt_records = (inode_records *5 ) / (4 *XFS_INODES_PER_CHUNK ); + +/* +* Allocate enough space to prefetch at least two inobt records so that +* we can cache both the record where the iwalk started and the next +* record. This simplifies the AG inode walk loop setup code. +*/ +return max (inobt_records,2U ); }/*
Re: [PATCH 07/15] xfs: calculate inode walk prefetch more carefully
Quoted 11 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
The existing inode walk prefetch is based on the old bulkstat code, which simply allocated 4 pages worth of memory and prefetched that many inobt records, regardless of however many inodes the caller requested. 65536 inodes is a lot to prefetch (~32M on x64, ~512M on arm64) so let's scale things down a little more intelligently based on the number of inodes requested, etc.
Signed-off-by Darrick J. Wong <[email protected]>---
A few nits..
Quoted 9 lines
fs/xfs/xfs_iwalk.c | 46 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-)fs/xfs/xfs_iwalk.c +0
-0
fs/xfs/xfs_iwalk.c xfs_iwalk_ag( · line 342
Quoted 10 lines
fs/xfs/xfs_iwalk.c +7
-0@@ -333,16 +333,58 @@ xfs_iwalk_ag(return error; } +/* +* We experimentally determined that the reduction in ioctl call overhead +* diminishes when userspace asks for more than 2048 inodes, so we'll cap +* prefetch at this point. +*/ +#define MAX_IWALK_PREFETCH (2048U) +
Something like IWALK_MAX_INODE_PREFETCH is a bit more clear IMO.
fs/xfs/xfs_iwalk.c xfs_iwalk_ag( · line 350
Quoted 9 lines
fs/xfs/xfs_iwalk.c +1
-1/* * Given the number of inodes to prefetch, set the number of inobt records that * we cache in memory, which controls the number of inodes we try to read -* ahead. +* ahead. Set the maximum if @inode_records == 0. */ static inline unsigned int xfs_iwalk_prefetch (unsigned int inode_records)
Perhaps this should be called 'inodes' since the function converts this value to inode records?
Quoted 20 lines
fs/xfs/xfs_iwalk.c +18
-1{ -return PAGE_SIZE *4 /sizeof (struct xfs_inobt_rec_incore ); +unsigned int inobt_records; + +/* +* If the caller didn't tell us the number of inodes they wanted, +* assume the maximum prefetch possible for best performance. +* Otherwise, cap prefetch at that maximum so that we don't start an +* absurd amount of prefetch. +*/ +if (inode_records ==0 ) + inode_records =MAX_IWALK_PREFETCH ; + inode_records =min (inode_records,MAX_IWALK_PREFETCH ); + +/* Round the inode count up to a full chunk. */ + inode_records =round_up (inode_records,XFS_INODES_PER_CHUNK ); + +/* +* In order to convert the number of inodes to prefetch into an +* estimate of the number of inobt records to cache, we require a
fs/xfs/xfs_iwalk.c xfs_iwalk_ag( · line 379
Quoted 10 lines
fs/xfs/xfs_iwalk.c +10
-0+* conversion factor that reflects our expectations of the average +* loading factor of an inode chunk. Based on data gathered, most +* (but not all) filesystems manage to keep the inode chunks totally +* full, so we'll underestimate slightly so that our readahead will +* still deliver the performance we want on aging filesystems: +* +* inobt = inodes / (INODES_PER_CHUNK * (4 / 5)); +* +* The funny math is to avoid division. +*/
The last bit of this comment is unclear. What do you mean by "avoid division?"
With those nits fixed up:
Quoted 12 lines
fs/xfs/xfs_iwalk.c +8
-0+ inobt_records = (inode_records *5 ) / (4 *XFS_INODES_PER_CHUNK ); + +/* +* Allocate enough space to prefetch at least two inobt records so that +* we can cache both the record where the iwalk started and the next +* record. This simplifies the AG inode walk loop setup code. +*/ +return max (inobt_records,2U ); }/*
Re: [PATCH 07/15] xfs: calculate inode walk prefetch more carefully
Quoted 27 lines · Brian Foster
Quoted 11 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
The existing inode walk prefetch is based on the old bulkstat code, which simply allocated 4 pages worth of memory and prefetched that many inobt records, regardless of however many inodes the caller requested. 65536 inodes is a lot to prefetch (~32M on x64, ~512M on arm64) so let's scale things down a little more intelligently based on the number of inodes requested, etc.
Signed-off-by Darrick J. Wong <[email protected]>---
A few nits..
Quoted 12 lines
fs/xfs/xfs_iwalk.c | 46 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-)fs/xfs/xfs_iwalk.c +0
-0@@ -333,16 +333,58 @@ xfs_iwalk_ag(return error; }
Quoted 9 lines
fs/xfs/xfs_iwalk.c xfs_iwalk_ag( · line 342
Quoted 7 lines
fs/xfs/xfs_iwalk.c +7
-0+/* +* We experimentally determined that the reduction in ioctl call overhead +* diminishes when userspace asks for more than 2048 inodes, so we'll cap +* prefetch at this point. +*/ +#define MAX_IWALK_PREFETCH (2048U) +Something like
IWALK_MAX_INODE_PREFETCHis a bit more clear IMO.
<nod>
Quoted 10 lines
fs/xfs/xfs_iwalk.c xfs_iwalk_ag( · line 350
Quoted 9 lines
fs/xfs/xfs_iwalk.c +1
-1/* * Given the number of inodes to prefetch, set the number of inobt records that * we cache in memory, which controls the number of inodes we try to read -* ahead. +* ahead. Set the maximum if @inode_records == 0. */ static inline unsigned int xfs_iwalk_prefetch (unsigned int inode_records)
Quoted 2 lines
Perhaps this should be called 'inodes' since the function converts this value to inode records?
ok, I see how that could be a little confusing.
Quoted 25 lines
Quoted 25 lines
fs/xfs/xfs_iwalk.c +23
-1{ -return PAGE_SIZE *4 /sizeof (struct xfs_inobt_rec_incore ); +unsigned int inobt_records; + +/* +* If the caller didn't tell us the number of inodes they wanted, +* assume the maximum prefetch possible for best performance. +* Otherwise, cap prefetch at that maximum so that we don't start an +* absurd amount of prefetch. +*/ +if (inode_records ==0 ) + inode_records =MAX_IWALK_PREFETCH ; + inode_records =min (inode_records,MAX_IWALK_PREFETCH ); + +/* Round the inode count up to a full chunk. */ + inode_records =round_up (inode_records,XFS_INODES_PER_CHUNK ); + +/* +* In order to convert the number of inodes to prefetch into an +* estimate of the number of inobt records to cache, we require a +* conversion factor that reflects our expectations of the average +* loading factor of an inode chunk. Based on data gathered, most +* (but not all) filesystems manage to keep the inode chunks totally +* full, so we'll underestimate slightly so that our readahead will +* still deliver the performance we want on aging filesystems:
Quoted 8 lines
fs/xfs/xfs_iwalk.c xfs_iwalk_ag( · line 379
Quoted 5 lines
fs/xfs/xfs_iwalk.c +5
-0+* +* inobt = inodes / (INODES_PER_CHUNK * (4 / 5)); +* +* The funny math is to avoid division. +*/ The last bit of this comment is unclear. What do you mean by "avoid division?"
"..to avoid 64-bit integer division."
Quoted 16 lines
With those nits fixed up:
Reviewed-by Brian Foster <[email protected]>Quoted 12 lines
fs/xfs/xfs_iwalk.c +8
-0+ inobt_records = (inode_records *5 ) / (4 *XFS_INODES_PER_CHUNK ); + +/* +* Allocate enough space to prefetch at least two inobt records so that +* we can cache both the record where the iwalk started and the next +* record. This simplifies the AG inode walk loop setup code. +*/ +return max (inobt_records,2U ); }/*
PATCH v6 8/15 xfs: move bulkstat ichunk helpers to iwalk code
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Now that we've reworked the bulkstat code to use iwalk, we can move the old bulkstat ichunk helpers to xfs_iwalk.c. No functional changes here.
--- fs/xfs/xfs_itable.c | 93 ------------------------------------------------- fs/xfs/xfs_itable.h | 8 ---- fs/xfs/xfs_iwalk.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 93 insertions(+), 104 deletions(-)
fs/xfs/xfs_itable.c +0 -93
@@ -189,99 +189,6 @@ xfs_bulkstat_one(return error; } -/* -* Loop over all clusters in a chunk for a given incore inode allocation btree -* record. Do a readahead if there are any allocated inodes in that cluster. -*/ -void -xfs_bulkstat_ichunk_ra ( -struct xfs_mount *mp, -xfs_agnumber_t agno, -struct xfs_inobt_rec_incore *irec) -{ -struct xfs_ino_geometry *igeo =M_IGEO (mp); -xfs_agblock_t agbno; -struct blk_plug plug; -int i;/* inode chunk index */ - - agbno =XFS_AGINO_TO_AGBNO (mp, irec->ir_startino); - -blk_start_plug (&plug); -for (i =0 ; - i <XFS_INODES_PER_CHUNK ; - i += igeo->inodes_per_cluster, - agbno += igeo->blocks_per_cluster) { -if (xfs_inobt_maskn (i, igeo->inodes_per_cluster) & - ~irec->ir_free) { -xfs_btree_reada_bufs (mp, agno, agbno, - igeo->blocks_per_cluster, - &xfs_inode_buf_ops); - } - } -blk_finish_plug (&plug); -} - -/* -* Lookup the inode chunk that the given inode lives in and then get the record -* if we found the chunk. If the inode was not the last in the chunk and there -* are some left allocated, update the data for the pointed-to record as well as -* return the count of grabbed inodes. -*/ -int -xfs_bulkstat_grab_ichunk ( -struct xfs_btree_cur *cur,/* btree cursor */ -xfs_agino_t agino,/* starting inode of chunk */ -int *icount,/* return # of inodes grabbed */ -struct xfs_inobt_rec_incore *irec)/* btree record */ -{ -int idx;/* index into inode chunk */ -int stat; -int error =0 ; - -/* Lookup the inode chunk that this inode lives in */ - error =xfs_inobt_lookup (cur, agino,XFS_LOOKUP_LE , &stat); -if (error) -return error; -if (!stat) { - *icount =0 ; -return error; - } - -/* Get the record, should always work */ - error =xfs_inobt_get_rec (cur, irec, &stat); -if (error) -return error; -XFS_WANT_CORRUPTED_RETURN (cur->bc_mp, stat ==1 ); - -/* Check if the record contains the inode in request */ -if (irec->ir_startino +XFS_INODES_PER_CHUNK <= agino) { - *icount =0 ; -return 0 ; - } - - idx = agino - irec->ir_startino +1 ; -if (idx <XFS_INODES_PER_CHUNK && - (xfs_inobt_maskn (idx,XFS_INODES_PER_CHUNK - idx) & ~irec->ir_free)) { -int i; - -/* We got a right chunk with some left inodes allocated at it. -* Grab the chunk record. Mark all the uninteresting inodes -* free -- because they're before our start point. -*/ -for (i =0 ; i < idx; i++) { -if (XFS_INOBT_MASK (i) & ~irec->ir_free) - irec->ir_freecount++; - } - - irec->ir_free |=xfs_inobt_maskn (0 , idx); - *icount = irec->ir_count - irec->ir_freecount; - } - -return 0 ; -} - -#define XFS_BULKSTAT_UBLEFT (ubleft) ((ubleft) >= statstruct_size) - static int xfs_bulkstat_iwalk( struct xfs_mount *mp,
fs/xfs/xfs_itable.h +0 -8
@@ -64,12 +64,4 @@ xfs_inumbers(void __user *buffer,/* buffer with inode info */ inumbers_fmt_pf formatter); -/* Temporarily needed while we refactor functions. */ -struct xfs_btree_cur ; -struct xfs_inobt_rec_incore ; -void xfs_bulkstat_ichunk_ra (struct xfs_mount *mp,xfs_agnumber_t agno, -struct xfs_inobt_rec_incore *irec); -int xfs_bulkstat_grab_ichunk (struct xfs_btree_cur *cur,xfs_agino_t agino, -int *icount,struct xfs_inobt_rec_incore *irec); -#endif /* __XFS_ITABLE_H__ */
fs/xfs/xfs_iwalk.c +93 -3
120 lines not on this page · raw message
PATCH v6 9/15 xfs: change xfs_iwalk_grab_ichunk to use startino, not lastino
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Now that the inode chunk grabbing function is a static function in the iwalk code, change its behavior so that @agino is the inode where we want to /start/ the iteration. This reduces cognitive friction with the callers and simplifes the code.
--- fs/xfs/xfs_iwalk.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-)
fs/xfs/xfs_iwalk.c +17 -20
@@ -98,10 +98,10 @@ xfs_iwalk_ichunk_ra( }/* -* Lookup the inode chunk that the given inode lives in and then get the record -* if we found the chunk. If the inode was not the last in the chunk and there -* are some left allocated, update the data for the pointed-to record as well as -* return the count of grabbed inodes. +* Lookup the inode chunk that the given @agino lives in and then get the +* record if we found the chunk. Set the bits in @irec's free mask that +* correspond to the inodes before @agino so that we skip them. This is how we +* restart an inode walk that was interrupted in the middle of an inode record. */ STATIC int xfs_iwalk_grab_ichunk(
@@ -112,6 +112,7 @@ xfs_iwalk_grab_ichunk( {int idx;/* index into inode chunk */ int stat; +int i;int error =0 ;/* Lookup the inode chunk that this inode lives in */
@@ -135,24 +136,20 @@ xfs_iwalk_grab_ichunk(return 0 ; } - idx = agino - irec->ir_startino +1 ; -if (idx <XFS_INODES_PER_CHUNK && - (xfs_inobt_maskn (idx,XFS_INODES_PER_CHUNK - idx) & ~irec->ir_free)) { -int i; + idx = agino - irec->ir_startino; -/* We got a right chunk with some left inodes allocated at it. -* Grab the chunk record. Mark all the uninteresting inodes -* free -- because they're before our start point. -*/ -for (i =0 ; i < idx; i++) { -if (XFS_INOBT_MASK (i) & ~irec->ir_free) - irec->ir_freecount++; - } - - irec->ir_free |=xfs_inobt_maskn (0 , idx); - *icount = irec->ir_count - irec->ir_freecount; +/* +* We got a right chunk with some left inodes allocated at it. Grab +* the chunk record. Mark all the uninteresting inodes free because +* they're before our start point. +*/ +for (i =0 ; i < idx; i++) { +if (XFS_INOBT_MASK (i) & ~irec->ir_free) + irec->ir_freecount++; } + irec->ir_free |=xfs_inobt_maskn (0 , idx); + *icount = irec->ir_count - irec->ir_freecount;return 0 ; }
@@ -281,7 +278,7 @@ xfs_iwalk_ag_start(* We require a lookup cache of at least two elements so that we don't * have to deal with tearing down the cursor to walk the records. */ - error =xfs_iwalk_grab_ichunk (*curpp, agino -1 , &icount, + error =xfs_iwalk_grab_ichunk (*curpp, agino, &icount, &iwag->recs[iwag->nr_recs]);if (error)return error;
PATCH v6 10/15 xfs: clean up long conditionals in xfs_iwalk_ichunk_ra
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Refactor xfs_iwalk_ichunk_ra to avoid long conditionals.
--- fs/xfs/xfs_iwalk.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)
fs/xfs/xfs_iwalk.c +6 -6
@@ -83,16 +83,16 @@ xfs_iwalk_ichunk_ra( agbno =XFS_AGINO_TO_AGBNO (mp, irec->ir_startino);blk_start_plug (&plug); -for (i =0 ; - i <XFS_INODES_PER_CHUNK ; - i += igeo->inodes_per_cluster, - agbno += igeo->blocks_per_cluster) { -if (xfs_inobt_maskn (i, igeo->inodes_per_cluster) & - ~irec->ir_free) { +for (i =0 ; i <XFS_INODES_PER_CHUNK ; i += igeo->inodes_per_cluster) { +xfs_inofree_t imask; + + imask =xfs_inobt_maskn (i, igeo->inodes_per_cluster); +if (imask & ~irec->ir_free) {xfs_btree_reada_bufs (mp, agno, agbno, igeo->blocks_per_cluster, &xfs_inode_buf_ops); } + agbno += igeo->blocks_per_cluster; }blk_finish_plug (&plug); }
PATCH v6 11/15 xfs: refactor xfs_iwalk_grab_ichunk
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
In preparation for reusing the iwalk code for the inogrp walking code (aka INUMBERS), move the initial inobt lookup and retrieval code out of xfs_iwalk_grab_ichunk so that we call the masking code only when we need to trim out the inodes that came before the cursor in the inobt record (aka BULKSTAT).
--- fs/xfs/xfs_iwalk.c | 79 ++++++++++++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 40 deletions(-)
fs/xfs/xfs_iwalk.c +39 -40
@@ -98,43 +98,17 @@ xfs_iwalk_ichunk_ra( }/* -* Lookup the inode chunk that the given @agino lives in and then get the -* record if we found the chunk. Set the bits in @irec's free mask that -* correspond to the inodes before @agino so that we skip them. This is how we -* restart an inode walk that was interrupted in the middle of an inode record. +* Set the bits in @irec's free mask that correspond to the inodes before +* @agino so that we skip them. This is how we restart an inode walk that was +* interrupted in the middle of an inode record. */ -STATIC int -xfs_iwalk_grab_ichunk ( -struct xfs_btree_cur *cur,/* btree cursor */ +STATIC void +xfs_iwalk_adjust_start (xfs_agino_t agino,/* starting inode of chunk */ -int *icount,/* return # of inodes grabbed */ struct xfs_inobt_rec_incore *irec)/* btree record */ {int idx;/* index into inode chunk */ -int stat;int i; -int error =0 ; - -/* Lookup the inode chunk that this inode lives in */ - error =xfs_inobt_lookup (cur, agino,XFS_LOOKUP_LE , &stat); -if (error) -return error; -if (!stat) { - *icount =0 ; -return error; - } - -/* Get the record, should always work */ - error =xfs_inobt_get_rec (cur, irec, &stat); -if (error) -return error; -XFS_WANT_CORRUPTED_RETURN (cur->bc_mp, stat ==1 ); - -/* Check if the record contains the inode in request */ -if (irec->ir_startino +XFS_INODES_PER_CHUNK <= agino) { - *icount =0 ; -return 0 ; - } idx = agino - irec->ir_startino;
@@ -149,8 +123,6 @@ xfs_iwalk_grab_ichunk( } irec->ir_free |=xfs_inobt_maskn (0 , idx); - *icount = irec->ir_count - irec->ir_freecount; -return 0 ; }/* Allocate memory for a walk. */
@@ -259,7 +231,7 @@ xfs_iwalk_ag_start( {struct xfs_mount *mp = iwag->mp;struct xfs_trans *tp = iwag->tp; -int icount; +struct xfs_inobt_rec_incore *irec;int error;/* Set up a fresh cursor and empty the inobt cache. */
@@ -275,15 +247,40 @@ xfs_iwalk_ag_start(/* * Otherwise, we have to grab the inobt record where we left off, stuff * the record into our cache, and then see if there are more records. -* We require a lookup cache of at least two elements so that we don't -* have to deal with tearing down the cursor to walk the records. +* We require a lookup cache of at least two elements so that the +* caller doesn't have to deal with tearing down the cursor to walk the +* records. */ - error =xfs_iwalk_grab_ichunk (*curpp, agino, &icount, - &iwag->recs[iwag->nr_recs]); + error =xfs_inobt_lookup (*curpp, agino,XFS_LOOKUP_LE , has_more); +if (error) +return error; + +/* +* If the LE lookup at @agino yields no records, jump ahead to the +* inobt cursor increment to see if there are more records to process. +*/ +if (!*has_more) + goto out_advance; + +/* Get the record, should always work */ + irec = &iwag->recs[iwag->nr_recs]; + error =xfs_inobt_get_rec (*curpp, irec, has_more);if (error)return error; -if (icount) - iwag->nr_recs++; +XFS_WANT_CORRUPTED_RETURN (mp, *has_more ==1 ); + +/* +* If the LE lookup yielded an inobt record before the cursor position, +* skip it and see if there's another one after it. +*/ +if (irec->ir_startino +XFS_INODES_PER_CHUNK <= agino) + goto out_advance; + +/* +* If agino fell in the middle of the inode record, make it look like +* the inodes up to agino are free so that we don't return them again. +*/ +xfs_iwalk_adjust_start (agino, irec);/* * The prefetch calculation is supposed to give us a large enough inobt
@@ -291,8 +288,10 @@ xfs_iwalk_ag_start(* the loop body can cache a record without having to check for cache * space until after it reads an inobt record. */ + iwag->nr_recs++;ASSERT (iwag->nr_recs < iwag->sz_recs); +out_advance:return xfs_btree_increment (*curpp,0 , has_more); }
PATCH v6 12/15 xfs: refactor iwalk code to handle walking inobt records
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Refactor xfs_iwalk_ag_start and xfs_iwalk_ag so that the bits that are particular to bulkstat (trimming the start irec, starting inode readahead, and skipping empty groups) can be controlled via flags in the iwag structure.
This enables us to add a new function to walk all inobt records which will be used for the new INUMBERS implementation in the next patch.
--- fs/xfs/xfs_iwalk.c | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++- fs/xfs/xfs_iwalk.h | 12 ++++++ 2 files changed, 121 insertions(+), 3 deletions(-)
fs/xfs/xfs_iwalk.c +109 -3
@@ -62,7 +62,18 @@ struct xfs_iwalk_ag {/* Inode walk function and data pointer. */ xfs_iwalk_fn iwalk_fn; +xfs_inobt_walk_fn inobt_walk_fn;void *data; + +/* +* Make it look like the inodes up to startino are free so that +* bulkstat can start its inode iteration at the correct place without +* needing to special case everywhere. +*/ + unsigned int trim_start:1; + +/* Skip empty inobt records? */ + unsigned int skip_empty:1; };/*
@@ -171,6 +182,16 @@ xfs_iwalk_ag_recs(trace_xfs_iwalk_ag_rec (mp, agno, irec); +if (iwag->inobt_walk_fn) { + error = iwag->inobt_walk_fn (mp, tp, agno, irec, + iwag->data); +if (error) +return error; + } + +if (!iwag->iwalk_fn) +continue ; +for (j =0 ; j <XFS_INODES_PER_CHUNK ; j++) {/* Skip if this inode is free */ if (XFS_INOBT_MASK (j) & irec->ir_free)
@@ -280,7 +301,8 @@ xfs_iwalk_ag_start(* If agino fell in the middle of the inode record, make it look like * the inodes up to agino are free so that we don't return them again. */ -xfs_iwalk_adjust_start (agino, irec); +if (iwag->trim_start) +xfs_iwalk_adjust_start (agino, irec);/* * The prefetch calculation is supposed to give us a large enough inobt
@@ -373,7 +395,7 @@ xfs_iwalk_ag(break ;/* No allocated inodes in this chunk; skip it. */ -if (irec->ir_freecount == irec->ir_count) { +if (iwag->skip_empty && irec->ir_freecount == irec->ir_count) { error =xfs_btree_increment (cur,0 , &has_more);if (error)break ;
@@ -384,7 +406,8 @@ xfs_iwalk_ag(* Start readahead for this inode chunk in anticipation of * walking the inodes. */ -xfs_iwalk_ichunk_ra (mp, agno, irec); +if (iwag->iwalk_fn) +xfs_iwalk_ichunk_ra (mp, agno, irec);/* * If there's space in the buffer for more records, increment
@@ -495,6 +518,89 @@ xfs_iwalk( .data = data, .startino = startino, .sz_recs = xfs_iwalk_prefetch(inode_records), + .trim_start = 1, + .skip_empty = 1, + }; +xfs_agnumber_t agno =XFS_INO_TO_AGNO (mp, startino); +int error; + +ASSERT (agno < mp->m_sb.sb_agcount); + + error =xfs_iwalk_alloc (&iwag); +if (error) +return error; + +for (; agno < mp->m_sb.sb_agcount; agno++) { + error =xfs_iwalk_ag (&iwag); +if (error) +break ; + iwag.startino =XFS_AGINO_TO_INO (mp, agno +1 ,0 ); + } + +xfs_iwalk_free (&iwag); +return error; +} + +/* +* Allow callers to cache up to a page's worth of inobt records. This reflects +* the existing inumbers prefetching behavior. Since the inobt walk does not +* itself do anything with the inobt records, we can set a fairly high limit +* here. +*/ +#define MAX_INOBT_WALK_PREFETCH \ + (PAGE_SIZE / sizeof(struct xfs_inobt_rec_incore)) + +/* +* Given the number of records that the user wanted, set the number of inobt +* records that we buffer in memory. Set the maximum if @inobt_records == 0. +*/ +static inline unsigned int +xfs_inobt_walk_prefetch ( +unsigned int inobt_records) +{ +/* +* If the caller didn't tell us the number of inobt records they +* wanted, assume the maximum prefetch possible for best performance. +*/ +if (inobt_records ==0 ) + inobt_records =MAX_INOBT_WALK_PREFETCH ; + +/* +* Allocate enough space to prefetch at least two inobt records so that +* we can cache both the record where the iwalk started and the next +* record. This simplifies the AG inode walk loop setup code. +*/ + inobt_records =max (inobt_records,2U ); + +/* +* Cap prefetch at that maximum so that we don't use an absurd amount +* of memory. +*/ + return min_t(unsigned int, inobt_records, MAX_INOBT_WALK_PREFETCH); +} + +/* +* Walk all inode btree records in the filesystem starting from @startino. The +* @inobt_walk_fn will be called for each btree record, being passed the incore +* record and @data. @max_prefetch controls how many inobt records we try to +* cache ahead of time. +*/ +int +xfs_inobt_walk ( +struct xfs_mount *mp, +struct xfs_trans *tp, +xfs_ino_t startino, +xfs_inobt_walk_fn inobt_walk_fn, +unsigned int inobt_records, +void *data) +{ +struct xfs_iwalk_ag iwag = { + .mp = mp, + .tp = tp, + .inobt_walk_fn = inobt_walk_fn, + .data = data, + .startino = startino, + .sz_recs =xfs_inobt_walk_prefetch (inobt_records), };xfs_agnumber_t agno =XFS_INO_TO_AGNO (mp, startino);int error;
fs/xfs/xfs_iwalk.h +12 -0
@@ -16,4 +16,16 @@ typedef int (*xfs_iwalk_fn)(struct xfs_mount *mp, struct xfs_trans *tp,int xfs_iwalk (struct xfs_mount *mp,struct xfs_trans *tp,xfs_ino_t startino,xfs_iwalk_fn iwalk_fn,unsigned int inode_records,void *data); +/* Walk all inode btree records in the filesystem starting from @startino. */ +typedef int (*xfs_inobt_walk_fn )(struct xfs_mount *mp,struct xfs_trans *tp, +xfs_agnumber_t agno, +const struct xfs_inobt_rec_incore *irec, +void *data); +/* Return value (for xfs_inobt_walk_fn) that aborts the walk immediately. */ +#define XFS_INOBT_WALK_ABORT (XFS_IWALK_ABORT) + +int xfs_inobt_walk (struct xfs_mount *mp,struct xfs_trans *tp, +xfs_ino_t startino,xfs_inobt_walk_fn inobt_walk_fn, +unsigned int inobt_records,void *data); +#endif /* __XFS_IWALK_H__ */
PATCH v6 13/15 xfs: refactor INUMBERS to use iwalk functions
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Now that we have generic functions to walk inode records, refactor the INUMBERS implementation to use it.
--- fs/xfs/xfs_ioctl.c | 20 ++++-- fs/xfs/xfs_ioctl.h | 2 + fs/xfs/xfs_ioctl32.c | 35 ++++------- fs/xfs/xfs_itable.c | 164 +++++++++++++++++++------------------------------- fs/xfs/xfs_itable.h | 22 +------ 5 files changed, 93 insertions(+), 150 deletions(-)
fs/xfs/xfs_ioctl.c +13 -7
@@ -733,6 +733,16 @@ xfs_bulkstat_one_fmt(return xfs_ibulk_advance (breq,sizeof (struct xfs_bstat )); } +int +xfs_inumbers_fmt ( +struct xfs_ibulk *breq, +const struct xfs_inogrp *igrp) +{ +if (copy_to_user (breq->ubuffer, igrp,sizeof (*igrp))) +return -EFAULT ; +return xfs_ibulk_advance (breq,sizeof (struct xfs_inogrp )); +} + STATIC int xfs_ioc_bulkstat( xfs_mount_t *mp,
@@ -783,13 +793,9 @@ xfs_ioc_bulkstat(* in filesystem". */ if (cmd ==XFS_IOC_FSINUMBERS ) { -int count = breq.icount; - - breq.startino = lastino; - error =xfs_inumbers (mp, &breq.startino, &count, - bulkreq.ubuffer, xfs_inumbers_fmt); - breq.ocount = count; - lastino = breq.startino; + breq.startino = lastino ? lastino +1 :0 ; + error =xfs_inumbers (&breq, xfs_inumbers_fmt); + lastino = breq.startino -1 ; }else if (cmd ==XFS_IOC_FSBULKSTAT_SINGLE ) { breq.startino = lastino; breq.icount =1 ;
fs/xfs/xfs_ioctl.h +2 -0
@@ -79,7 +79,9 @@ xfs_set_dmattrs(struct xfs_ibulk ;struct xfs_bstat ; +struct xfs_inogrp ;int xfs_bulkstat_one_fmt (struct xfs_ibulk *breq,const struct xfs_bstat *bstat); +int xfs_inumbers_fmt (struct xfs_ibulk *breq,const struct xfs_inogrp *igrp);#endif
fs/xfs/xfs_ioctl32.c +13 -22
@@ -87,22 +87,17 @@ xfs_compat_growfs_rt_copyin( STATIC intxfs_inumbers_fmt_compat ( -void __user *ubuffer, -const struct xfs_inogrp *buffer, -long count, -long *written) +struct xfs_ibulk *breq, +const struct xfs_inogrp *igrp) { -compat_xfs_inogrp_t __user *p32 = ubuffer; -long i; +struct compat_xfs_inogrp __user *p32 = breq->ubuffer; -for (i =0 ; i < count; i++) { -if (put_user (buffer[i].xi_startino, &p32[i].xi_startino) || -put_user (buffer[i].xi_alloccount, &p32[i].xi_alloccount) || -put_user (buffer[i].xi_allocmask, &p32[i].xi_allocmask)) -return -EFAULT ; - } - *written = count *sizeof (*p32); -return 0 ; +if (put_user (igrp->xi_startino, &p32->xi_startino) || +put_user (igrp->xi_alloccount, &p32->xi_alloccount) || +put_user (igrp->xi_allocmask, &p32->xi_allocmask)) +return -EFAULT ; + +return xfs_ibulk_advance (breq,sizeof (struct compat_xfs_inogrp )); }#else
@@ -228,7 +223,7 @@ xfs_compat_ioc_bulkstat(* to userpace memory via bulkreq.ubuffer. Normally the compat * functions and structure size are the correct ones to use ... */ -inumbers_fmt_pf inumbers_func = xfs_inumbers_fmt_compat; +inumbers_fmt_pf inumbers_func = xfs_inumbers_fmt_compat;bulkstat_one_fmt_pf bs_one_func = xfs_bulkstat_one_fmt_compat;#ifdef CONFIG_X86_X32
@@ -290,13 +285,9 @@ xfs_compat_ioc_bulkstat(* in filesystem". */ if (cmd ==XFS_IOC_FSINUMBERS_32 ) { -int count = breq.icount; - - breq.startino = lastino; - error =xfs_inumbers (mp, &breq.startino, &count, - bulkreq.ubuffer, inumbers_func); - breq.ocount = count; - lastino = breq.startino; + breq.startino = lastino ? lastino +1 :0 ; + error =xfs_inumbers (&breq, inumbers_func); + lastino = breq.startino -1 ; }else if (cmd ==XFS_IOC_FSBULKSTAT_SINGLE_32 ) { breq.startino = lastino; breq.icount =1 ;
fs/xfs/xfs_itable.c +62 -102
183 lines not on this page · raw message
fs/xfs/xfs_itable.h +3 -19
@@ -43,25 +43,9 @@ typedef int (*bulkstat_one_fmt_pf)(struct xfs_ibulk *breq,int xfs_bulkstat_one (struct xfs_ibulk *breq,bulkstat_one_fmt_pf formatter);int xfs_bulkstat (struct xfs_ibulk *breq,bulkstat_one_fmt_pf formatter); -typedef int (*inumbers_fmt_pf )( -void __user *ubuffer,/* buffer to write to */ -const xfs_inogrp_t *buffer,/* buffer to read from */ -long count,/* # of elements to read */ -long *written);/* # of bytes written */ +typedef int (*inumbers_fmt_pf )(struct xfs_ibulk *breq, +const struct xfs_inogrp *igrp); -int -xfs_inumbers_fmt ( -void __user *ubuffer,/* buffer to write to */ -const xfs_inogrp_t *buffer,/* buffer to read from */ -long count,/* # of elements to read */ -long *written);/* # of bytes written */ - -int /* error status */ -xfs_inumbers ( -xfs_mount_t *mp,/* mount point for filesystem */ -xfs_ino_t *last,/* last inode returned */ -int *count,/* size of buffer/count returned */ -void __user *buffer,/* buffer with inode info */ -inumbers_fmt_pf formatter); +int xfs_inumbers (struct xfs_ibulk *breq,inumbers_fmt_pf formatter);#endif /* __XFS_ITABLE_H__ */
PATCH v6 14/15 xfs: multithreaded iwalk implementation
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Create a parallel iwalk implementation and switch quotacheck to use it.
--- fs/xfs/Makefile | 1 fs/xfs/xfs_globals.c | 3 + fs/xfs/xfs_iwalk.c | 82 +++++++++++++++++++++++++++++++++ fs/xfs/xfs_iwalk.h | 2 + fs/xfs/xfs_pwork.c | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_pwork.h | 58 +++++++++++++++++++++++ fs/xfs/xfs_qm.c | 2 - fs/xfs/xfs_sysctl.h | 6 ++ fs/xfs/xfs_sysfs.c | 40 ++++++++++++++++ fs/xfs/xfs_trace.h | 18 +++++++ 10 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 fs/xfs/xfs_pwork.c create mode 100644 fs/xfs/xfs_pwork.h
fs/xfs/Makefile +1 -0
@@ -84,6 +84,7 @@ xfs-y += xfs_aops.o \ xfs_message.o \ xfs_mount.o \ xfs_mru_cache.o \ + xfs_pwork.o \ xfs_reflink.o \ xfs_stats.o \ xfs_super.o \
fs/xfs/xfs_globals.c +3 -0
@@ -31,6 +31,9 @@ xfs_param_t xfs_params = { .fstrm_timer = { 1, 30*100, 3600*100}, .eofb_timer = { 1, 300, 3600*24}, .cowb_timer = { 1, 1800, 3600*24}, +#ifdef DEBUG + .pwork_threads = { -1, -1, NR_CPUS }, +#endif }; struct xfs_globals xfs_globals = {
fs/xfs/xfs_iwalk.c +82 -0
130 lines not on this page · raw message
fs/xfs/xfs_iwalk.h +2 -0
@@ -15,6 +15,8 @@ typedef int (*xfs_iwalk_fn)(struct xfs_mount *mp, struct xfs_trans *tp,int xfs_iwalk (struct xfs_mount *mp,struct xfs_trans *tp,xfs_ino_t startino,xfs_iwalk_fn iwalk_fn,unsigned int inode_records,void *data); +int xfs_iwalk_threaded (struct xfs_mount *mp,xfs_ino_t startino, +xfs_iwalk_fn iwalk_fn,unsigned int inode_records,void *data);/* Walk all inode btree records in the filesystem starting from @startino. */ typedef int (*xfs_inobt_walk_fn)(struct xfs_mount *mp, struct xfs_trans *tp,
fs/xfs/xfs_pwork.c new file +126 -0
126 lines not on this page · raw message
fs/xfs/xfs_pwork.h new file +58 -0
58 lines not on this page · raw message
fs/xfs/xfs_qm.c +1 -1
@@ -1304,7 +1304,7 @@ xfs_qm_quotacheck( flags |=XFS_PQUOTA_CHKD ; } - error =xfs_iwalk (mp,NULL ,0 , xfs_qm_dqusage_adjust,0 ,NULL ); + error =xfs_iwalk_threaded (mp,0 , xfs_qm_dqusage_adjust,0 ,NULL );if (error) goto error_return;
fs/xfs/xfs_sysctl.h +6 -0
@@ -37,6 +37,9 @@ typedef struct xfs_param {xfs_sysctl_val_t fstrm_timer;/* Filestream dir-AG assoc'n timeout. */ xfs_sysctl_val_t eofb_timer;/* Interval between eofb scan wakeups */ xfs_sysctl_val_t cowb_timer;/* Interval between cowb scan wakeups */ +#ifdef DEBUG +xfs_sysctl_val_t pwork_threads;/* Parallel workqueue thread count */ +#endif } xfs_param_t;/*
@@ -82,6 +85,9 @@ enum {extern xfs_param_t xfs_params;struct xfs_globals { +#ifdef DEBUG +int pwork_threads;/* parallel workqueue threads */ +#endif int log_recovery_delay;/* log recovery delay (secs) */ int mount_delay;/* mount setup delay (secs) */ bool bug_on_assert;/* BUG() the kernel on assert failure */
fs/xfs/xfs_sysfs.c +40 -0
@@ -206,11 +206,51 @@ always_cow_show( }XFS_SYSFS_ATTR_RW (always_cow); +#ifdef DEBUG +/* +* Override how many threads the parallel work queue is allowed to create. +* This has to be a debug-only global (instead of an errortag) because one of +* the main users of parallel workqueues is mount time quotacheck. +*/ +STATIC ssize_t +pwork_threads_store ( +struct kobject *kobject, +const char *buf, +size_t count) +{ +int ret; +int val; + + ret =kstrtoint (buf,0 , &val); +if (ret) +return ret; + +if (val <0 || val >num_possible_cpus ()) +return -EINVAL ; + + xfs_globals.pwork_threads = val; + +return count; +} + +STATIC ssize_t +pwork_threads_show ( +struct kobject *kobject, +char *buf) +{ +return snprintf (buf,PAGE_SIZE ,"%d\n" , xfs_globals.pwork_threads); +} +XFS_SYSFS_ATTR_RW (pwork_threads); +#endif /* DEBUG */ +static struct attribute *xfs_dbg_attrs[] = {ATTR_LIST (bug_on_assert),ATTR_LIST (log_recovery_delay),ATTR_LIST (mount_delay),ATTR_LIST (always_cow), +#ifdef DEBUG +ATTR_LIST (pwork_threads), +#endifNULL , };
fs/xfs/xfs_trace.h +18 -0
@@ -3556,6 +3556,24 @@ TRACE_EVENT(xfs_iwalk_ag_rec, __entry->startino, __entry->freemask) ) +TRACE_EVENT (xfs_pwork_init, + TP_PROTO(struct xfs_mount *mp, unsigned int nr_threads, pid_t pid), +TP_ARGS (mp, nr_threads, pid), +TP_STRUCT__entry ( + __field(dev_t, dev) + __field(unsigned int, nr_threads) +__field (pid_t, pid) + ), +TP_fast_assign ( + __entry->dev = mp->m_super->s_dev; + __entry->nr_threads = nr_threads; + __entry->pid = pid; + ), +TP_printk ("dev %d:%d nr_threads %u pid %u" , +MAJOR (__entry->dev),MINOR (__entry->dev), + __entry->nr_threads, __entry->pid) +) +#endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH
Re: [PATCH 14/15] xfs: multithreaded iwalk implementation
Quoted 38 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Create a parallel iwalk implementation and switch quotacheck to use it.
Signed-off-by Darrick J. Wong <[email protected]>--- fs/xfs/Makefile | 1 fs/xfs/xfs_globals.c | 3 + fs/xfs/xfs_iwalk.c | 82 +++++++++++++++++++++++++++++++++ fs/xfs/xfs_iwalk.h | 2 + fs/xfs/xfs_pwork.c | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_pwork.h | 58 +++++++++++++++++++++++ fs/xfs/xfs_qm.c | 2 - fs/xfs/xfs_sysctl.h | 6 ++ fs/xfs/xfs_sysfs.c | 40 ++++++++++++++++ fs/xfs/xfs_trace.h | 18 +++++++ 10 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 fs/xfs/xfs_pwork.c create mode 100644 fs/xfs/xfs_pwork.hfs/xfs/Makefile +1
-0@@ -84,6 +84,7 @@ xfs-y += xfs_aops.o \ xfs_message.o \ xfs_mount.o \ xfs_mru_cache.o \ + xfs_pwork.o \ xfs_reflink.o \ xfs_stats.o \ xfs_super.o \fs/xfs/xfs_globals.c +0
-0
fs/xfs/xfs_globals.c xfs_param_t xfs_params = { · line 35
Quoted 5 lines
fs/xfs/xfs_globals.c +2
-0@@ -31,6 +31,9 @@ xfs_param_t xfs_params = { .fstrm_timer = { 1, 30*100, 3600*100}, .eofb_timer = { 1, 300, 3600*24}, .cowb_timer = { 1, 1800, 3600*24}, +#ifdef DEBUG + .pwork_threads = { -1, -1, NR_CPUS },
So I noticed that /sys/fs/xfs/debug/pwork_threads was still 0 by default with this patch, then realized that we've only initialized it in xfs_params and not xfs_globals. The sysfs handlers added by this patch use the latter whereas the former looks related to /proc. It looks like this should be fixed up to init the global field to -1 and probably drop the xfs_params bits since we don't expose the value via /proc.
Quoted 4 lines
fs/xfs/xfs_globals.c +1
-0+#endif };struct xfs_globals xfs_globals = {
...
Quoted 6 lines
fs/xfs/xfs_pwork.c new file +0
-0@@ -0,0 +1,126 @@
...
Quoted 15 lines
fs/xfs/xfs_pwork.c new file +15
-0+/* +* Return the amount of parallelism that the data device can handle, or 0 for +* no limit. +*/ +unsigned int +xfs_pwork_guess_datadev_parallelism ( +struct xfs_mount *mp) +{ +struct xfs_buftarg *btp = mp->m_ddev_targp; +int iomin; +int ioopt; + +if (blk_queue_nonrot (btp->bt_bdev->bd_queue)) +return num_online_cpus (); +
fs/xfs/xfs_pwork.c line 126
Quoted 10 lines
fs/xfs/xfs_pwork.c new file +10
-0+if (mp->m_sb.sb_width && mp->m_sb.sb_unit) +return mp->m_sb.sb_width / mp->m_sb.sb_unit; + + iomin =bdev_io_min (btp->bt_bdev); + ioopt =bdev_io_opt (btp->bt_bdev); +if (iomin && ioopt) +return ioopt / iomin; + +return 1 ; +}
I may have lost track of where we left off with this but IIRC we still needed some numbers with regard to multi-device storage where sb_width/sb_unit doesn't necessarily describe parallelism (i.e. RAID5/6). I'm not sure what your goal is for this patch vs. the rest of the series, but I'd be fine with merging this with just the non-rotational bit for now if we add some kind of conservative maximum (4? 8?) to the heuristic. It seems kind of risky to me to parallelize 100s of AGs just because we might have that many CPUs/AGs on a particular storage setup (we may also generate warning messages if a system has a CPU count that exceeds the workqueue limit).
...
Quoted 20 lines
fs/xfs/xfs_sysfs.c +12
-0@@ -206,11 +206,51 @@ always_cow_show( }XFS_SYSFS_ATTR_RW (always_cow); +#ifdef DEBUG +/* +* Override how many threads the parallel work queue is allowed to create. +* This has to be a debug-only global (instead of an errortag) because one of +* the main users of parallel workqueues is mount time quotacheck. +*/ +STATIC ssize_t +pwork_threads_store ( +struct kobject *kobject, +const char *buf, +size_t count) +{
fs/xfs/xfs_sysfs.c always_cow_show( · line 230
Quoted 10 lines
fs/xfs/xfs_sysfs.c +10
-0+int ret; +int val; + + ret =kstrtoint (buf,0 , &val); +if (ret) +return ret; + +if (val <0 || val >num_possible_cpus ()) +return -EINVAL ; +
We need to allow assignment of -1 now too.
Brian
Quoted 56 lines
fs/xfs/xfs_sysfs.c +18
-0+ xfs_globals.pwork_threads = val; + +return count; +} + +STATIC ssize_t +pwork_threads_show ( +struct kobject *kobject, +char *buf) +{ +return snprintf (buf,PAGE_SIZE ,"%d\n" , xfs_globals.pwork_threads); +} +XFS_SYSFS_ATTR_RW (pwork_threads); +#endif /* DEBUG */ +static struct attribute *xfs_dbg_attrs[] = {ATTR_LIST (bug_on_assert),ATTR_LIST (log_recovery_delay),ATTR_LIST (mount_delay),ATTR_LIST (always_cow), +#ifdef DEBUG +ATTR_LIST (pwork_threads), +#endifNULL , };fs/xfs/xfs_trace.h +18
-0@@ -3556,6 +3556,24 @@ TRACE_EVENT(xfs_iwalk_ag_rec, __entry->startino, __entry->freemask) ) +TRACE_EVENT (xfs_pwork_init, + TP_PROTO(struct xfs_mount *mp, unsigned int nr_threads, pid_t pid), +TP_ARGS (mp, nr_threads, pid), +TP_STRUCT__entry ( + __field(dev_t, dev) + __field(unsigned int, nr_threads) +__field (pid_t, pid) + ), +TP_fast_assign ( + __entry->dev = mp->m_super->s_dev; + __entry->nr_threads = nr_threads; + __entry->pid = pid; + ), +TP_printk ("dev %d:%d nr_threads %u pid %u" , +MAJOR (__entry->dev),MINOR (__entry->dev), + __entry->nr_threads, __entry->pid) +) +#endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH
Re: [PATCH 14/15] xfs: multithreaded iwalk implementation
Quoted 43 lines · Brian Foster
Quoted 42 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Create a parallel iwalk implementation and switch quotacheck to use it.
Signed-off-by Darrick J. Wong <[email protected]>--- fs/xfs/Makefile | 1 fs/xfs/xfs_globals.c | 3 + fs/xfs/xfs_iwalk.c | 82 +++++++++++++++++++++++++++++++++ fs/xfs/xfs_iwalk.h | 2 + fs/xfs/xfs_pwork.c | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_pwork.h | 58 +++++++++++++++++++++++ fs/xfs/xfs_qm.c | 2 - fs/xfs/xfs_sysctl.h | 6 ++ fs/xfs/xfs_sysfs.c | 40 ++++++++++++++++ fs/xfs/xfs_trace.h | 18 +++++++ 10 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 fs/xfs/xfs_pwork.c create mode 100644 fs/xfs/xfs_pwork.hfs/xfs/Makefile +1
-0@@ -84,6 +84,7 @@ xfs-y += xfs_aops.o \ xfs_message.o \ xfs_mount.o \ xfs_mru_cache.o \ + xfs_pwork.o \ xfs_reflink.o \ xfs_stats.o \ xfs_super.o \fs/xfs/xfs_globals.c +1
-0@@ -31,6 +31,9 @@ xfs_param_t xfs_params = { .fstrm_timer = { 1, 30*100, 3600*100}, .eofb_timer = { 1, 300, 3600*24}, .cowb_timer = { 1, 1800, 3600*24}, +#ifdef DEBUG
Quoted 8 lines
fs/xfs/xfs_globals.c xfs_param_t xfs_params = { · line 35
Quoted 1 line
fs/xfs/xfs_globals.c +1
-0+ .pwork_threads = { -1, -1, NR_CPUS },So I noticed that
/sys/fs/xfs/debug/pwork_threadswas still 0 by default with this patch, then realized that we've only initialized it inxfs_paramsand notxfs_globals. The sysfs handlers added by this patch use the latter whereas the former looks related to /proc. It looks like this should be fixed up to init the global field to -1 and probably drop thexfs_paramsbits since we don't expose the value via /proc.
Oh, you're right. I'll drop the xfs_params bits.
Quoted 41 lines
Quoted 5 lines
fs/xfs/xfs_globals.c +1
-0+#endif };struct xfs_globals xfs_globals = {...
Quoted 6 lines
fs/xfs/xfs_pwork.c new file +0
-0@@ -0,0 +1,126 @@...
Quoted 15 lines
fs/xfs/xfs_pwork.c new file +15
-0+/* +* Return the amount of parallelism that the data device can handle, or 0 for +* no limit. +*/ +unsigned int +xfs_pwork_guess_datadev_parallelism ( +struct xfs_mount *mp) +{ +struct xfs_buftarg *btp = mp->m_ddev_targp; +int iomin; +int ioopt; + +if (blk_queue_nonrot (btp->bt_bdev->bd_queue)) +return num_online_cpus (); +fs/xfs/xfs_pwork.c line 126
Quoted 10 lines
fs/xfs/xfs_pwork.c new file +10
-0+if (mp->m_sb.sb_width && mp->m_sb.sb_unit) +return mp->m_sb.sb_width / mp->m_sb.sb_unit; + + iomin =bdev_io_min (btp->bt_bdev); + ioopt =bdev_io_opt (btp->bt_bdev); +if (iomin && ioopt) +return ioopt / iomin; + +return 1 ; +}I may have lost track of where we left off with this but IIRC we still needed some numbers with regard to multi-device storage where
Quoted 8 lines
sb_width/sb_unit doesn't necessarily describe parallelism (i.e. RAID5/6). I'm not sure what your goal is for this patch vs. the rest of the series, but I'd be fine with merging this with just the non-rotational bit for now if we add some kind of conservative maximum (4? 8?) to the heuristic. It seems kind of risky to me to parallelize 100s of AGs just because we might have that many CPUs/AGs on a particular storage setup (we may also generate warning messages if a system has a CPU count that exceeds the workqueue limit).
Hm, how about the most conservative threading that I can think of?
unsigned int
xfs_pwork_guess_datadev_parallelism(
struct xfs_mount *mp)
{
return blk_queue_nonrot(...) ? 2 : 1;
}Since that did seem to yield speedups for everything except the dumb USB mass storage case. Then I can work on building a case for more threads (and figuring out the maximums) for 5.4...
--D
Quoted 26 lines
...
Quoted 24 lines
fs/xfs/xfs_sysfs.c +16
-0@@ -206,11 +206,51 @@ always_cow_show( }XFS_SYSFS_ATTR_RW (always_cow); +#ifdef DEBUG +/* +* Override how many threads the parallel work queue is allowed to create. +* This has to be a debug-only global (instead of an errortag) because one of +* the main users of parallel workqueues is mount time quotacheck. +*/ +STATIC ssize_t +pwork_threads_store ( +struct kobject *kobject, +const char *buf, +size_t count) +{ +int ret; +int val; + + ret =kstrtoint (buf,0 , &val);
Quoted 8 lines
fs/xfs/xfs_sysfs.c always_cow_show( · line 230
Quoted 6 lines
fs/xfs/xfs_sysfs.c +6
-0+if (ret) +return ret; + +if (val <0 || val >num_possible_cpus ()) +return -EINVAL ; +We need to allow assignment of -1 now too.
<nod>
--D
Quoted 58 lines
Brian
Quoted 56 lines
fs/xfs/xfs_sysfs.c +18
-0+ xfs_globals.pwork_threads = val; + +return count; +} + +STATIC ssize_t +pwork_threads_show ( +struct kobject *kobject, +char *buf) +{ +return snprintf (buf,PAGE_SIZE ,"%d\n" , xfs_globals.pwork_threads); +} +XFS_SYSFS_ATTR_RW (pwork_threads); +#endif /* DEBUG */ +static struct attribute *xfs_dbg_attrs[] = {ATTR_LIST (bug_on_assert),ATTR_LIST (log_recovery_delay),ATTR_LIST (mount_delay),ATTR_LIST (always_cow), +#ifdef DEBUG +ATTR_LIST (pwork_threads), +#endifNULL , };fs/xfs/xfs_trace.h +18
-0@@ -3556,6 +3556,24 @@ TRACE_EVENT(xfs_iwalk_ag_rec, __entry->startino, __entry->freemask) ) +TRACE_EVENT (xfs_pwork_init, + TP_PROTO(struct xfs_mount *mp, unsigned int nr_threads, pid_t pid), +TP_ARGS (mp, nr_threads, pid), +TP_STRUCT__entry ( + __field(dev_t, dev) + __field(unsigned int, nr_threads) +__field (pid_t, pid) + ), +TP_fast_assign ( + __entry->dev = mp->m_super->s_dev; + __entry->nr_threads = nr_threads; + __entry->pid = pid; + ), +TP_printk ("dev %d:%d nr_threads %u pid %u" , +MAJOR (__entry->dev),MINOR (__entry->dev), + __entry->nr_threads, __entry->pid) +) +#endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH
Re: [PATCH 14/15] xfs: multithreaded iwalk implementation
Quoted 110 lines · Darrick J. Wong
Quoted 43 lines · Brian Foster
Quoted 42 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Create a parallel iwalk implementation and switch quotacheck to use it.
Signed-off-by Darrick J. Wong <[email protected]>--- fs/xfs/Makefile | 1 fs/xfs/xfs_globals.c | 3 + fs/xfs/xfs_iwalk.c | 82 +++++++++++++++++++++++++++++++++ fs/xfs/xfs_iwalk.h | 2 + fs/xfs/xfs_pwork.c | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_pwork.h | 58 +++++++++++++++++++++++ fs/xfs/xfs_qm.c | 2 - fs/xfs/xfs_sysctl.h | 6 ++ fs/xfs/xfs_sysfs.c | 40 ++++++++++++++++ fs/xfs/xfs_trace.h | 18 +++++++ 10 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 fs/xfs/xfs_pwork.c create mode 100644 fs/xfs/xfs_pwork.hfs/xfs/Makefile +1
-0@@ -84,6 +84,7 @@ xfs-y += xfs_aops.o \ xfs_message.o \ xfs_mount.o \ xfs_mru_cache.o \ + xfs_pwork.o \ xfs_reflink.o \ xfs_stats.o \ xfs_super.o \fs/xfs/xfs_globals.c +1
-0@@ -31,6 +31,9 @@ xfs_param_t xfs_params = { .fstrm_timer = { 1, 30*100, 3600*100}, .eofb_timer = { 1, 300, 3600*24}, .cowb_timer = { 1, 1800, 3600*24}, +#ifdef DEBUGQuoted 8 lines
fs/xfs/xfs_globals.c xfs_param_t xfs_params = { · line 35
Quoted 1 line
fs/xfs/xfs_globals.c +1
-0+ .pwork_threads = { -1, -1, NR_CPUS },So I noticed that
/sys/fs/xfs/debug/pwork_threadswas still 0 by default with this patch, then realized that we've only initialized it inxfs_paramsand notxfs_globals. The sysfs handlers added by this patch use the latter whereas the former looks related to /proc. It looks like this should be fixed up to init the global field to -1 and probably drop thexfs_paramsbits since we don't expose the value via /proc.Oh, you're right. I'll drop the
xfs_paramsbits.Quoted 41 lines
Quoted 5 lines
fs/xfs/xfs_globals.c +1
-0+#endif };struct xfs_globals xfs_globals = {...
Quoted 6 lines
fs/xfs/xfs_pwork.c new file +0
-0@@ -0,0 +1,126 @@...
Quoted 15 lines
fs/xfs/xfs_pwork.c new file +15
-0+/* +* Return the amount of parallelism that the data device can handle, or 0 for +* no limit. +*/ +unsigned int +xfs_pwork_guess_datadev_parallelism ( +struct xfs_mount *mp) +{ +struct xfs_buftarg *btp = mp->m_ddev_targp; +int iomin; +int ioopt; + +if (blk_queue_nonrot (btp->bt_bdev->bd_queue)) +return num_online_cpus (); +fs/xfs/xfs_pwork.c line 126
Quoted 10 lines
fs/xfs/xfs_pwork.c new file +10
-0+if (mp->m_sb.sb_width && mp->m_sb.sb_unit) +return mp->m_sb.sb_width / mp->m_sb.sb_unit; + + iomin =bdev_io_min (btp->bt_bdev); + ioopt =bdev_io_opt (btp->bt_bdev); +if (iomin && ioopt) +return ioopt / iomin; + +return 1 ; +}I may have lost track of where we left off with this but IIRC we still needed some numbers with regard to multi-device storage where
Quoted 8 lines
sb_width/sb_unit doesn't necessarily describe parallelism (i.e. RAID5/6). I'm not sure what your goal is for this patch vs. the rest of the series, but I'd be fine with merging this with just the non-rotational bit for now if we add some kind of conservative maximum (4? 8?) to the heuristic. It seems kind of risky to me to parallelize 100s of AGs just because we might have that many CPUs/AGs on a particular storage setup (we may also generate warning messages if a system has a CPU count that exceeds the workqueue limit).
Hm, how about the most conservative threading that I can think of?
unsigned int
xfs_pwork_guess_datadev_parallelism(
structxfs_mount*mp)
Quoted 8 lines
{ return blk_queue_nonrot(...) ? 2 : 1; }Since that did seem to yield speedups for everything except the dumb USB mass storage case. Then I can work on building a case for more threads (and figuring out the maximums) for 5.4...
Ack, that sounds good to me. Then we can get this code in sooner and worry about opening up the parallelization incrementally based on more focused testing/analysis.
Brian
Quoted 99 lines
--D
Quoted 26 lines
...
Quoted 24 lines
fs/xfs/xfs_sysfs.c +16
-0@@ -206,11 +206,51 @@ always_cow_show( }XFS_SYSFS_ATTR_RW (always_cow); +#ifdef DEBUG +/* +* Override how many threads the parallel work queue is allowed to create. +* This has to be a debug-only global (instead of an errortag) because one of +* the main users of parallel workqueues is mount time quotacheck. +*/ +STATIC ssize_t +pwork_threads_store ( +struct kobject *kobject, +const char *buf, +size_t count) +{ +int ret; +int val; + + ret =kstrtoint (buf,0 , &val);Quoted 8 lines
fs/xfs/xfs_sysfs.c always_cow_show( · line 230
Quoted 6 lines
fs/xfs/xfs_sysfs.c +6
-0+if (ret) +return ret; + +if (val <0 || val >num_possible_cpus ()) +return -EINVAL ; +We need to allow assignment of -1 now too.
<nod>
--D
Quoted 58 lines
Brian
Quoted 56 lines
fs/xfs/xfs_sysfs.c +18
-0+ xfs_globals.pwork_threads = val; + +return count; +} + +STATIC ssize_t +pwork_threads_show ( +struct kobject *kobject, +char *buf) +{ +return snprintf (buf,PAGE_SIZE ,"%d\n" , xfs_globals.pwork_threads); +} +XFS_SYSFS_ATTR_RW (pwork_threads); +#endif /* DEBUG */ +static struct attribute *xfs_dbg_attrs[] = {ATTR_LIST (bug_on_assert),ATTR_LIST (log_recovery_delay),ATTR_LIST (mount_delay),ATTR_LIST (always_cow), +#ifdef DEBUG +ATTR_LIST (pwork_threads), +#endifNULL , };fs/xfs/xfs_trace.h +18
-0@@ -3556,6 +3556,24 @@ TRACE_EVENT(xfs_iwalk_ag_rec, __entry->startino, __entry->freemask) ) +TRACE_EVENT (xfs_pwork_init, + TP_PROTO(struct xfs_mount *mp, unsigned int nr_threads, pid_t pid), +TP_ARGS (mp, nr_threads, pid), +TP_STRUCT__entry ( + __field(dev_t, dev) + __field(unsigned int, nr_threads) +__field (pid_t, pid) + ), +TP_fast_assign ( + __entry->dev = mp->m_super->s_dev; + __entry->nr_threads = nr_threads; + __entry->pid = pid; + ), +TP_printk ("dev %d:%d nr_threads %u pid %u" , +MAJOR (__entry->dev),MINOR (__entry->dev), + __entry->nr_threads, __entry->pid) +) +#endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH
PATCH v2 14/15 xfs: multithreaded iwalk implementation
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Create a parallel iwalk implementation and switch quotacheck to use it.
---
v2: conservative thread count estimates; clean out the xfs_param_t stuff
--- fs/xfs/Makefile | 1 fs/xfs/xfs_globals.c | 3 + fs/xfs/xfs_iwalk.c | 82 ++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_iwalk.h | 2 + fs/xfs/xfs_pwork.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_pwork.h | 58 +++++++++++++++++++++++++ fs/xfs/xfs_qm.c | 2 - fs/xfs/xfs_sysctl.h | 3 + fs/xfs/xfs_sysfs.c | 40 +++++++++++++++++ fs/xfs/xfs_trace.h | 18 ++++++++ 10 files changed, 323 insertions(+), 1 deletion(-) create mode 100644 fs/xfs/xfs_pwork.c create mode 100644 fs/xfs/xfs_pwork.h
fs/xfs/Makefile +1 -0
@@ -92,6 +92,7 @@ xfs-y += xfs_aops.o \ xfs_message.o \ xfs_mount.o \ xfs_mru_cache.o \ + xfs_pwork.o \ xfs_reflink.o \ xfs_stats.o \ xfs_super.o \
fs/xfs/xfs_globals.c +3 -0
@@ -40,4 +40,7 @@ struct xfs_globals xfs_globals = {#else .bug_on_assert = false, /* assert failures WARN() */#endif +#ifdef DEBUG + .pwork_threads = -1, /* automatic thread detection */ +#endif };
fs/xfs/xfs_iwalk.c +82 -0
130 lines not on this page · raw message
fs/xfs/xfs_iwalk.h +2 -0
@@ -15,6 +15,8 @@ typedef int (*xfs_iwalk_fn)(struct xfs_mount *mp, struct xfs_trans *tp,int xfs_iwalk (struct xfs_mount *mp,struct xfs_trans *tp,xfs_ino_t startino,xfs_iwalk_fn iwalk_fn,unsigned int inode_records,void *data); +int xfs_iwalk_threaded (struct xfs_mount *mp,xfs_ino_t startino, +xfs_iwalk_fn iwalk_fn,unsigned int inode_records,void *data);/* Walk all inode btree records in the filesystem starting from @startino. */ typedef int (*xfs_inobt_walk_fn)(struct xfs_mount *mp, struct xfs_trans *tp,
fs/xfs/xfs_pwork.c new file +115 -0
115 lines not on this page · raw message
fs/xfs/xfs_pwork.h new file +58 -0
58 lines not on this page · raw message
fs/xfs/xfs_qm.c +1 -1
@@ -1300,7 +1300,7 @@ xfs_qm_quotacheck( flags |=XFS_PQUOTA_CHKD ; } - error =xfs_iwalk (mp,NULL ,0 , xfs_qm_dqusage_adjust,0 ,NULL ); + error =xfs_iwalk_threaded (mp,0 , xfs_qm_dqusage_adjust,0 ,NULL );if (error) goto error_return;
fs/xfs/xfs_sysctl.h +3 -0
@@ -82,6 +82,9 @@ enum {extern xfs_param_t xfs_params;struct xfs_globals { +#ifdef DEBUG +int pwork_threads;/* parallel workqueue threads */ +#endif int log_recovery_delay;/* log recovery delay (secs) */ int mount_delay;/* mount setup delay (secs) */ bool bug_on_assert;/* BUG() the kernel on assert failure */
fs/xfs/xfs_sysfs.c +40 -0
@@ -204,11 +204,51 @@ always_cow_show( }XFS_SYSFS_ATTR_RW (always_cow); +#ifdef DEBUG +/* +* Override how many threads the parallel work queue is allowed to create. +* This has to be a debug-only global (instead of an errortag) because one of +* the main users of parallel workqueues is mount time quotacheck. +*/ +STATIC ssize_t +pwork_threads_store ( +struct kobject *kobject, +const char *buf, +size_t count) +{ +int ret; +int val; + + ret =kstrtoint (buf,0 , &val); +if (ret) +return ret; + +if (val <-1 || val >num_possible_cpus ()) +return -EINVAL ; + + xfs_globals.pwork_threads = val; + +return count; +} + +STATIC ssize_t +pwork_threads_show ( +struct kobject *kobject, +char *buf) +{ +return snprintf (buf,PAGE_SIZE ,"%d\n" , xfs_globals.pwork_threads); +} +XFS_SYSFS_ATTR_RW (pwork_threads); +#endif /* DEBUG */ +static struct attribute *xfs_dbg_attrs[] = {ATTR_LIST (bug_on_assert),ATTR_LIST (log_recovery_delay),ATTR_LIST (mount_delay),ATTR_LIST (always_cow), +#ifdef DEBUG +ATTR_LIST (pwork_threads), +#endifNULL , };
fs/xfs/xfs_trace.h +18 -0
@@ -3557,6 +3557,24 @@ TRACE_EVENT(xfs_iwalk_ag_rec, __entry->startino, __entry->freemask) ) +TRACE_EVENT (xfs_pwork_init, + TP_PROTO(struct xfs_mount *mp, unsigned int nr_threads, pid_t pid), +TP_ARGS (mp, nr_threads, pid), +TP_STRUCT__entry ( + __field(dev_t, dev) + __field(unsigned int, nr_threads) +__field (pid_t, pid) + ), +TP_fast_assign ( + __entry->dev = mp->m_super->s_dev; + __entry->nr_threads = nr_threads; + __entry->pid = pid; + ), +TP_printk ("dev %d:%d nr_threads %u pid %u" , +MAJOR (__entry->dev),MINOR (__entry->dev), + __entry->nr_threads, __entry->pid) +) +#endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH
Re: [PATCH v2 14/15] xfs: multithreaded iwalk implementation
Quoted 8 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Create a parallel iwalk implementation and switch quotacheck to use it.
Signed-off-by Darrick J. Wong <[email protected]>---
v2: conservative thread count estimates; clean out thexfs_param_tstuff
---
Quoted 490 lines
fs/xfs/Makefile | 1 fs/xfs/xfs_globals.c | 3 + fs/xfs/xfs_iwalk.c | 82 ++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_iwalk.h | 2 + fs/xfs/xfs_pwork.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_pwork.h | 58 +++++++++++++++++++++++++ fs/xfs/xfs_qm.c | 2 - fs/xfs/xfs_sysctl.h | 3 + fs/xfs/xfs_sysfs.c | 40 +++++++++++++++++ fs/xfs/xfs_trace.h | 18 ++++++++ 10 files changed, 323 insertions(+), 1 deletion(-) create mode 100644 fs/xfs/xfs_pwork.c create mode 100644 fs/xfs/xfs_pwork.hfs/xfs/Makefile +1
-0@@ -92,6 +92,7 @@ xfs-y += xfs_aops.o \ xfs_message.o \ xfs_mount.o \ xfs_mru_cache.o \ + xfs_pwork.o \ xfs_reflink.o \ xfs_stats.o \ xfs_super.o \fs/xfs/xfs_globals.c +3
-0@@ -40,4 +40,7 @@ struct xfs_globals xfs_globals = {#else .bug_on_assert = false, /* assert failures WARN() */#endif +#ifdef DEBUG + .pwork_threads = -1, /* automatic thread detection */ +#endif };fs/xfs/xfs_iwalk.c +82
-0@@ -20,6 +20,7 @@#include "xfs_icache.h" #include "xfs_health.h" #include "xfs_trans.h" +#include "xfs_pwork.h" /* * Walking Inodes in the Filesystem @@ -45,6 +46,9 @@*/ struct xfs_iwalk_ag { +/* parallel work control data; will be null if single threaded */ +struct xfs_pwork pwork; +struct xfs_mount *mp;struct xfs_trans *tp;@@ -182,6 +186,9 @@ xfs_iwalk_ag_recs(trace_xfs_iwalk_ag_rec (mp, agno, irec); +if (xfs_pwork_want_abort (&iwag->pwork)) +return 0 ; +if (iwag->inobt_walk_fn) { error = iwag->inobt_walk_fn (mp, tp, agno, irec, iwag->data);@@ -193,6 +200,9 @@ xfs_iwalk_ag_recs(continue ;for (j =0 ; j <XFS_INODES_PER_CHUNK ; j++) { +if (xfs_pwork_want_abort (&iwag->pwork)) +return 0 ; +/* Skip if this inode is free */ if (XFS_INOBT_MASK (j) & irec->ir_free)continue ;@@ -387,6 +397,8 @@ xfs_iwalk_ag(struct xfs_inobt_rec_incore *irec;cond_resched (); +if (xfs_pwork_want_abort (&iwag->pwork)) + goto out;/* Fetch the inobt record. */ irec = &iwag->recs[iwag->nr_recs];@@ -520,6 +532,7 @@ xfs_iwalk( .sz_recs = xfs_iwalk_prefetch(inode_records), .trim_start = 1, .skip_empty = 1, + .pwork = XFS_PWORK_SINGLE_THREADED, };xfs_agnumber_t agno =XFS_INO_TO_AGNO (mp, startino);int error;@@ -541,6 +554,74 @@ xfs_iwalk(return error; } +/* Run per-thread iwalk work. */ +static int +xfs_iwalk_ag_work ( +struct xfs_mount *mp, +struct xfs_pwork *pwork) +{ +struct xfs_iwalk_ag *iwag; +int error =0 ; + + iwag = container_of(pwork, struct xfs_iwalk_ag, pwork); +if (xfs_pwork_want_abort (pwork)) + goto out; + + error =xfs_iwalk_alloc (iwag); +if (error) + goto out; + + error =xfs_iwalk_ag (iwag); +xfs_iwalk_free (iwag); +out: +kmem_free (iwag); +return error; +} + +/* +* Walk all the inodes in the filesystem using multiple threads to process each +* AG. +*/ +int +xfs_iwalk_threaded ( +struct xfs_mount *mp, +xfs_ino_t startino, +xfs_iwalk_fn iwalk_fn, +unsigned int inode_records, +void *data) +{ +struct xfs_pwork_ctl pctl; +xfs_agnumber_t agno =XFS_INO_TO_AGNO (mp, startino); +unsigned int nr_threads; +int error; + +ASSERT (agno < mp->m_sb.sb_agcount); + + nr_threads =xfs_pwork_guess_datadev_parallelism (mp); + error =xfs_pwork_init (mp, &pctl, xfs_iwalk_ag_work,"xfs_iwalk" , + nr_threads); +if (error) +return error; + +for (; agno < mp->m_sb.sb_agcount; agno++) { +struct xfs_iwalk_ag *iwag; + +if (xfs_pwork_ctl_want_abort (&pctl)) +break ; + + iwag =kmem_zalloc (sizeof (struct xfs_iwalk_ag ),KM_SLEEP ); + iwag->mp = mp; + iwag->iwalk_fn = iwalk_fn; + iwag->data = data; + iwag->startino = startino; + iwag->sz_recs =xfs_iwalk_prefetch (inode_records); +xfs_pwork_queue (&pctl, &iwag->pwork); + startino =XFS_AGINO_TO_INO (mp, agno +1 ,0 ); + } + +return xfs_pwork_destroy (&pctl); +} +/* * Allow callers to cache up to a page's worth of inobt records. This reflects * the existing inumbers prefetching behavior. Since the inobt walk does not @@ -601,6 +682,7 @@ xfs_inobt_walk( .data = data, .startino = startino, .sz_recs = xfs_inobt_walk_prefetch(inobt_records), + .pwork = XFS_PWORK_SINGLE_THREADED, };xfs_agnumber_t agno =XFS_INO_TO_AGNO (mp, startino);int error;fs/xfs/xfs_iwalk.h +2
-0@@ -15,6 +15,8 @@ typedef int (*xfs_iwalk_fn)(struct xfs_mount *mp, struct xfs_trans *tp,int xfs_iwalk (struct xfs_mount *mp,struct xfs_trans *tp,xfs_ino_t startino,xfs_iwalk_fn iwalk_fn,unsigned int inode_records,void *data); +int xfs_iwalk_threaded (struct xfs_mount *mp,xfs_ino_t startino, +xfs_iwalk_fn iwalk_fn,unsigned int inode_records,void *data);/* Walk all inode btree records in the filesystem starting from @startino. */ typedef int (*xfs_inobt_walk_fn)(struct xfs_mount *mp, struct xfs_trans *tp,fs/xfs/xfs_pwork.c new file +115
-0@@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* +* Copyright (C) 2019 Oracle. All Rights Reserved. +* Author: Darrick J. Wong <[email protected]> +*/ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_trace.h" +#include "xfs_sysctl.h" +#include "xfs_pwork.h" + +/* +* Parallel Work Queue +* =================== +* +* Abstract away the details of running a large and "obviously" parallelizable +* task across multiple CPUs. Callers initialize the pwork control object with +* a desired level of parallelization and a work function. Next, they embed +* struct xfs_pwork in whatever structure they use to pass work context to a +* worker thread and queue that pwork. The work function will be passed the +* pwork item when it is run (from process context) and any returned error will +* be recorded in xfs_pwork_ctl.error. Work functions should check for errors +* and abort if necessary; the non-zeroness of xfs_pwork_ctl.error does not +* stop workqueue item processing. +* +* This is the rough equivalent of the xfsprogs workqueue code, though we can't +* reuse that name here. +*/ + +/* Invoke our caller's function. */ +static void +xfs_pwork_work ( +struct work_struct *work) +{ +struct xfs_pwork *pwork; +struct xfs_pwork_ctl *pctl; +int error; + + pwork = container_of(work, struct xfs_pwork, work); + pctl = pwork->pctl; + error = pctl->work_fn (pctl->mp, pwork); +if (error && !pctl->error) + pctl->error = error; +} + +/* +* Set up control data for parallel work. @work_fn is the function that will +* be called. @tag will be written into the kernel threads. @nr_threads is +* the level of parallelism desired, or 0 for no limit. +*/ +int +xfs_pwork_init ( +struct xfs_mount *mp, +struct xfs_pwork_ctl *pctl, +xfs_pwork_work_fn work_fn, +const char *tag, +unsigned int nr_threads) +{ +#ifdef DEBUG +if (xfs_globals.pwork_threads >=0 ) + nr_threads = xfs_globals.pwork_threads; +#endif +trace_xfs_pwork_init (mp, nr_threads, current->pid); + + pctl->wq =alloc_workqueue ("%s-%d" ,WQ_FREEZABLE , nr_threads, tag, + current->pid); +if (!pctl->wq) +return -ENOMEM ; + pctl->work_fn = work_fn; + pctl->error =0 ; + pctl->mp = mp; + +return 0 ; +} + +/* Queue some parallel work. */ +void +xfs_pwork_queue ( +struct xfs_pwork_ctl *pctl, +struct xfs_pwork *pwork) +{ +INIT_WORK (&pwork->work, xfs_pwork_work); + pwork->pctl = pctl; +queue_work (pctl->wq, &pwork->work); +} + +/* Wait for the work to finish and tear down the control structure. */ +int +xfs_pwork_destroy ( +struct xfs_pwork_ctl *pctl) +{ +destroy_workqueue (pctl->wq); + pctl->wq =NULL ; +return pctl->error; +} + +/* +* Return the amount of parallelism that the data device can handle, or 0 for +* no limit. +*/ +unsigned int +xfs_pwork_guess_datadev_parallelism ( +struct xfs_mount *mp) +{ +/* +* For now we'll go with the most conservative setting possible, +* which is two threads for an SSD and 1 thread everywhere else. +*/ +return blk_queue_nonrot (btp->bt_bdev->bd_queue) ?2 :1 ; +}fs/xfs/xfs_pwork.h new file +58
-0@@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* +* Copyright (C) 2019 Oracle. All Rights Reserved. +* Author: Darrick J. Wong <[email protected]> +*/ +#ifndef __XFS_PWORK_H__ +#define __XFS_PWORK_H__ + +struct xfs_pwork ; +struct xfs_mount ; + +typedef int (*xfs_pwork_work_fn )(struct xfs_mount *mp,struct xfs_pwork *pwork); + +/* +* Parallel work coordination structure. +*/ +struct xfs_pwork_ctl { +struct workqueue_struct *wq; +struct xfs_mount *mp; +xfs_pwork_work_fn work_fn; +int error; +}; + +/* +* Embed this parallel work control item inside your own work structure, +* then queue work with it. +*/ +struct xfs_pwork { +struct work_struct work; +struct xfs_pwork_ctl *pctl; +}; + +#define XFS_PWORK_SINGLE_THREADED { .pctl = NULL } + +/* Have we been told to abort? */ +static inline bool +xfs_pwork_ctl_want_abort ( +struct xfs_pwork_ctl *pctl) +{ +return pctl && pctl->error; +} + +/* Have we been told to abort? */ +static inline bool +xfs_pwork_want_abort ( +struct xfs_pwork *pwork) +{ +return xfs_pwork_ctl_want_abort (pwork->pctl); +} + +int xfs_pwork_init (struct xfs_mount *mp,struct xfs_pwork_ctl *pctl, +xfs_pwork_work_fn work_fn,const char *tag, +unsigned int nr_threads); +void xfs_pwork_queue (struct xfs_pwork_ctl *pctl,struct xfs_pwork *pwork); +int xfs_pwork_destroy (struct xfs_pwork_ctl *pctl); +unsigned int xfs_pwork_guess_datadev_parallelism (struct xfs_mount *mp); + +#endif /* __XFS_PWORK_H__ */ fs/xfs/xfs_qm.c +1
-1@@ -1300,7 +1300,7 @@ xfs_qm_quotacheck( flags |=XFS_PQUOTA_CHKD ; } - error =xfs_iwalk (mp,NULL ,0 , xfs_qm_dqusage_adjust,0 ,NULL ); + error =xfs_iwalk_threaded (mp,0 , xfs_qm_dqusage_adjust,0 ,NULL );if (error) goto error_return;fs/xfs/xfs_sysctl.h +3
-0@@ -82,6 +82,9 @@ enum {extern xfs_param_t xfs_params;struct xfs_globals { +#ifdef DEBUG +int pwork_threads;/* parallel workqueue threads */ +#endif int log_recovery_delay;/* log recovery delay (secs) */ int mount_delay;/* mount setup delay (secs) */ bool bug_on_assert;/* BUG() the kernel on assert failure */ fs/xfs/xfs_sysfs.c +40
-0@@ -204,11 +204,51 @@ always_cow_show( }XFS_SYSFS_ATTR_RW (always_cow); +#ifdef DEBUG +/* +* Override how many threads the parallel work queue is allowed to create. +* This has to be a debug-only global (instead of an errortag) because one of +* the main users of parallel workqueues is mount time quotacheck. +*/ +STATIC ssize_t +pwork_threads_store ( +struct kobject *kobject, +const char *buf, +size_t count) +{ +int ret; +int val; + + ret =kstrtoint (buf,0 , &val); +if (ret) +return ret; + +if (val <-1 || val >num_possible_cpus ()) +return -EINVAL ; + + xfs_globals.pwork_threads = val; + +return count; +} + +STATIC ssize_t +pwork_threads_show ( +struct kobject *kobject, +char *buf) +{ +return snprintf (buf,PAGE_SIZE ,"%d\n" , xfs_globals.pwork_threads); +} +XFS_SYSFS_ATTR_RW (pwork_threads); +#endif /* DEBUG */ +static struct attribute *xfs_dbg_attrs[] = {ATTR_LIST (bug_on_assert),ATTR_LIST (log_recovery_delay),ATTR_LIST (mount_delay),ATTR_LIST (always_cow), +#ifdef DEBUG +ATTR_LIST (pwork_threads), +#endifNULL , };fs/xfs/xfs_trace.h +18
-0@@ -3557,6 +3557,24 @@ TRACE_EVENT(xfs_iwalk_ag_rec, __entry->startino, __entry->freemask) ) +TRACE_EVENT (xfs_pwork_init, + TP_PROTO(struct xfs_mount *mp, unsigned int nr_threads, pid_t pid), +TP_ARGS (mp, nr_threads, pid), +TP_STRUCT__entry ( + __field(dev_t, dev) + __field(unsigned int, nr_threads) +__field (pid_t, pid) + ), +TP_fast_assign ( + __entry->dev = mp->m_super->s_dev; + __entry->nr_threads = nr_threads; + __entry->pid = pid; + ), +TP_printk ("dev %d:%d nr_threads %u pid %u" , +MAJOR (__entry->dev),MINOR (__entry->dev), + __entry->nr_threads, __entry->pid) +) +#endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH
PATCH v6 15/15 xfs: poll waiting for quotacheck
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
Create a pwork destroy function that uses polling instead of uninterruptible sleep to wait for work items to finish so that we can touch the softlockup watchdog. IOWs, gross hack.
--- fs/xfs/xfs_iwalk.c | 3 +++ fs/xfs/xfs_iwalk.h | 3 ++- fs/xfs/xfs_pwork.c | 19 +++++++++++++++++++ fs/xfs/xfs_pwork.h | 3 +++ fs/xfs/xfs_qm.c | 2 +- 5 files changed, 28 insertions(+), 2 deletions(-)
fs/xfs/xfs_iwalk.c +3 -0
@@ -588,6 +588,7 @@ xfs_iwalk_threaded(xfs_ino_t startino, xfs_iwalk_fn iwalk_fn, unsigned int inode_records, + bool polled, void *data) { struct xfs_pwork_ctl pctl;
@@ -619,6 +620,8 @@ xfs_iwalk_threaded( startino =XFS_AGINO_TO_INO (mp, agno +1 ,0 ); } +if (polled) +xfs_pwork_poll (&pctl);return xfs_pwork_destroy (&pctl); }
fs/xfs/xfs_iwalk.h +2 -1
@@ -16,7 +16,8 @@ typedef int (*xfs_iwalk_fn)(struct xfs_mount *mp, struct xfs_trans *tp,int xfs_iwalk (struct xfs_mount *mp,struct xfs_trans *tp,xfs_ino_t startino,xfs_iwalk_fn iwalk_fn,unsigned int inode_records,void *data);int xfs_iwalk_threaded (struct xfs_mount *mp,xfs_ino_t startino, -xfs_iwalk_fn iwalk_fn,unsigned int inode_records,void *data); +xfs_iwalk_fn iwalk_fn,unsigned int inode_records,bool poll, +void *data);/* Walk all inode btree records in the filesystem starting from @startino. */ typedef int (*xfs_inobt_walk_fn)(struct xfs_mount *mp, struct xfs_trans *tp,
fs/xfs/xfs_pwork.c +19 -0
@@ -13,6 +13,7 @@#include "xfs_trace.h" #include "xfs_sysctl.h" #include "xfs_pwork.h" +#include <linux/nmi.h> /* * Parallel Work Queue
@@ -46,6 +47,8 @@ xfs_pwork_work( error = pctl->work_fn (pctl->mp, pwork);if (error && !pctl->error) pctl->error = error; +if (atomic_dec_and_test (&pctl->nr_work)) +wake_up (&pctl->poll_wait); }/*
@@ -74,6 +77,8 @@ xfs_pwork_init( pctl->work_fn = work_fn; pctl->error =0 ; pctl->mp = mp; +atomic_set (&pctl->nr_work,0 ); +init_waitqueue_head (&pctl->poll_wait);return 0 ; }
@@ -86,6 +91,7 @@ xfs_pwork_queue( {INIT_WORK (&pwork->work, xfs_pwork_work); pwork->pctl = pctl; +atomic_inc (&pctl->nr_work);queue_work (pctl->wq, &pwork->work); }
@@ -99,6 +105,19 @@ xfs_pwork_destroy(return pctl->error; } +/* +* Wait for the work to finish by polling completion status and touch the soft +* lockup watchdog. This is for callers such as mount which hold locks. +*/ +void +xfs_pwork_poll ( +struct xfs_pwork_ctl *pctl) +{ +while (wait_event_timeout (pctl->poll_wait, +atomic_read (&pctl->nr_work) ==0 ,HZ ) ==0 ) +touch_softlockup_watchdog (); +} +/* * Return the amount of parallelism that the data device can handle, or 0 for * no limit.
fs/xfs/xfs_pwork.h +3 -0
@@ -18,6 +18,8 @@ struct xfs_pwork_ctl {struct workqueue_struct *wq;struct xfs_mount *mp;xfs_pwork_work_fn work_fn; +struct wait_queue_head poll_wait; +atomic_t nr_work;int error; };
@@ -53,6 +55,7 @@ int xfs_pwork_init(struct xfs_mount *mp, struct xfs_pwork_ctl *pctl, unsigned int nr_threads);void xfs_pwork_queue (struct xfs_pwork_ctl *pctl,struct xfs_pwork *pwork);int xfs_pwork_destroy (struct xfs_pwork_ctl *pctl); +void xfs_pwork_poll (struct xfs_pwork_ctl *pctl);unsigned int xfs_pwork_guess_datadev_parallelism (struct xfs_mount *mp);#endif /* __XFS_PWORK_H__ */
fs/xfs/xfs_qm.c +1 -1
@@ -1304,7 +1304,7 @@ xfs_qm_quotacheck( flags |=XFS_PQUOTA_CHKD ; } - error =xfs_iwalk_threaded (mp,0 , xfs_qm_dqusage_adjust,0 ,NULL ); + error =xfs_iwalk_threaded (mp,0 , xfs_qm_dqusage_adjust,0 , true,NULL );if (error) goto error_return;