few mm helpers for bcachefs
Subsystems (3)
cover few mm helpers for bcachefs
memalloc_flags_{save,restore} - cleanup of the existing duplication in our memalloc_*_{save,restore} functions
PF_MEMALLOC_NORECLAIM, PF_MEMALLOC_NOWARN: we've been talking about getting away from gfp flags for years, but we've been lacking some PF_MEMALLOC_* equivalents that we need to make that happen - this adds some of the missing ones.
they're needed for calling alloc_inode() in bcachefs with btree locks held (so that we can make lookup() properly transactional), since it doesn't take a gfp flags param and adding it would require changing methods in every fs.
Kent Overstreet (3):
mm: introduce memalloc_flags_{save,restore}
mm: introduce PF_MEMALLOC_NORECLAIM, PF_MEMALLOC_NOWARN
mempool: kvmalloc pool
include/linux/mempool.h | 13 +++++++++ include/linux/sched.h | 4 +-- include/linux/sched/mm.h | 60 ++++++++++++++++++++++++++-------------- mm/mempool.c | 13 +++++++++ 4 files changed, 67 insertions(+), 23 deletions(-)
Signature · 1 line
2.43.0
PATCH 1/3 mm: introduce memalloc_flags_{save,restore}
Our proliferation of memalloc_*_{save,restore} APIs is getting a bit silly, this adds a generic version and converts the existing save/restore functions to wrappers.
--- include/linux/sched/mm.h | 43 ++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-)
include/linux/sched/mm.h +26 -17
@@ -306,6 +306,24 @@ static inline void might_alloc(gfp_t gfp_mask)might_sleep_if (gfpflags_allow_blocking (gfp_mask)); } +/** +* memalloc_flags_save - Add a PF_* flag to current->flags, save old value +* +* This allows PF_* flags to be conveniently added, irrespective of current +* value, and then the old version restored with memalloc_flags_restore(). +*/ +static inline unsigned memalloc_flags_save (unsigned flags) +{ +unsigned oldflags = ~current->flags & flags; + current->flags |= flags; +return oldflags; +} + +static inline void memalloc_flags_restore (unsigned flags) +{ + current->flags &= ~flags; +} +/** * memalloc_noio_save - Marks implicit GFP_NOIO allocation scope. *
@@ -319,9 +337,7 @@ static inline void might_alloc(gfp_t gfp_mask)*/ static inline unsigned int memalloc_noio_save (void ) { -unsigned int flags = current->flags &PF_MEMALLOC_NOIO ; - current->flags |=PF_MEMALLOC_NOIO ; -return flags; +return memalloc_flags_save (PF_MEMALLOC_NOIO ); }/**
@@ -334,7 +350,7 @@ static inline unsigned int memalloc_noio_save(void)*/ static inline void memalloc_noio_restore (unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC_NOIO ) | flags; +memalloc_flags_restore (flags); }/**
@@ -350,9 +366,7 @@ static inline void memalloc_noio_restore(unsigned int flags)*/ static inline unsigned int memalloc_nofs_save (void ) { -unsigned int flags = current->flags &PF_MEMALLOC_NOFS ; - current->flags |=PF_MEMALLOC_NOFS ; -return flags; +return memalloc_flags_save (PF_MEMALLOC_NOFS ); }/**
@@ -365,32 +379,27 @@ static inline unsigned int memalloc_nofs_save(void)*/ static inline void memalloc_nofs_restore (unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC_NOFS ) | flags; +memalloc_flags_restore (flags); }static inline unsigned int memalloc_noreclaim_save (void ) { -unsigned int flags = current->flags &PF_MEMALLOC ; - current->flags |=PF_MEMALLOC ; -return flags; +return memalloc_flags_save (PF_MEMALLOC ); }static inline void memalloc_noreclaim_restore (unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC ) | flags; +memalloc_flags_restore (flags); }static inline unsigned int memalloc_pin_save (void ) { -unsigned int flags = current->flags &PF_MEMALLOC_PIN ; - - current->flags |=PF_MEMALLOC_PIN ; -return flags; +return memalloc_flags_save (PF_MEMALLOC_PIN ); }static inline void memalloc_pin_restore (unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC_PIN ) | flags; +memalloc_flags_restore (flags); }#ifdef CONFIG_MEMCG
Signature · 1 line
2.43.0
Re: [PATCH 1/3] mm: introduce memalloc_flags_{save,restore}
Quoted 10 lines · Kent Overstreet
Our proliferation of memalloc_*_{save,restore} APIs is getting a bit silly, this adds a generic version and converts the existing save/restore functions to wrappers.
Signed-off-by Kent Overstreet <[email protected]>Cc Vlastimil Babka <[email protected]>Cc Matthew Wilcox <[email protected]>Cc Michal Hocko <[email protected]>Cc Darrick J. Wong <[email protected]>
This will also make it possible to do combinations of flags in a single call, in case some ad-hoc combinations are needed only in few places that don't warrant a designated helper.
Quoted 102 lines
--- include/linux/sched/mm.h | 43 ++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-)include/linux/sched/mm.h +26
-17@@ -306,6 +306,24 @@ static inline void might_alloc(gfp_t gfp_mask)might_sleep_if (gfpflags_allow_blocking (gfp_mask)); } +/** +* memalloc_flags_save - Add a PF_* flag to current->flags, save old value +* +* This allows PF_* flags to be conveniently added, irrespective of current +* value, and then the old version restored with memalloc_flags_restore(). +*/ +static inline unsigned memalloc_flags_save (unsigned flags) +{ +unsigned oldflags = ~current->flags & flags; + current->flags |= flags; +return oldflags; +} + +static inline void memalloc_flags_restore (unsigned flags) +{ + current->flags &= ~flags; +} +/** * memalloc_noio_save - Marks implicit GFP_NOIO allocation scope. * @@ -319,9 +337,7 @@ static inline void might_alloc(gfp_t gfp_mask)*/ static inline unsigned int memalloc_noio_save (void ) { -unsigned int flags = current->flags &PF_MEMALLOC_NOIO ; - current->flags |=PF_MEMALLOC_NOIO ; -return flags; +return memalloc_flags_save (PF_MEMALLOC_NOIO ); }/** @@ -334,7 +350,7 @@ static inline unsigned int memalloc_noio_save(void)*/ static inline void memalloc_noio_restore (unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC_NOIO ) | flags; +memalloc_flags_restore (flags); }/** @@ -350,9 +366,7 @@ static inline void memalloc_noio_restore(unsigned int flags)*/ static inline unsigned int memalloc_nofs_save (void ) { -unsigned int flags = current->flags &PF_MEMALLOC_NOFS ; - current->flags |=PF_MEMALLOC_NOFS ; -return flags; +return memalloc_flags_save (PF_MEMALLOC_NOFS ); }/** @@ -365,32 +379,27 @@ static inline unsigned int memalloc_nofs_save(void)*/ static inline void memalloc_nofs_restore (unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC_NOFS ) | flags; +memalloc_flags_restore (flags); }static inline unsigned int memalloc_noreclaim_save (void ) { -unsigned int flags = current->flags &PF_MEMALLOC ; - current->flags |=PF_MEMALLOC ; -return flags; +return memalloc_flags_save (PF_MEMALLOC ); }static inline void memalloc_noreclaim_restore (unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC ) | flags; +memalloc_flags_restore (flags); }static inline unsigned int memalloc_pin_save (void ) { -unsigned int flags = current->flags &PF_MEMALLOC_PIN ; - - current->flags |=PF_MEMALLOC_PIN ; -return flags; +return memalloc_flags_save (PF_MEMALLOC_PIN ); }static inline void memalloc_pin_restore (unsigned int flags) { - current->flags = (current->flags & ~PF_MEMALLOC_PIN ) | flags; +memalloc_flags_restore (flags); }#ifdef CONFIG_MEMCG
PATCH 2/3 mm: introduce PF_MEMALLOC_NORECLAIM, PF_MEMALLOC_NOWARN
In reply to Kent OverstreetIntroduce PF_MEMALLOC_* equivalents of some GFP_ flags:
PF_MEMALLOC_NORECLAIM -> GFP_NOWAITPF_MEMALLOC_NOWARN -> __GFP_NOWARN
--- include/linux/sched.h | 4 ++-- include/linux/sched/mm.h | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-)
include/linux/sched.h +2 -2
@@ -1755,8 +1755,8 @@ extern struct pid *cad_pid;* I am cleaning dirty pages from some other bdi. */ #define PF_KTHREAD 0x00200000/* I am a kernel thread */ #define PF_RANDOMIZE 0x00400000/* Randomize virtual address space */ -#define PF__HOLE__00800000 0x00800000 -#define PF__HOLE__01000000 0x01000000 +#define PF_MEMALLOC_NORECLAIM 0x00800000/* All allocation requests will inherit __GFP_NOWARN */ +#define PF_MEMALLOC_NOWARN 0x01000000/* All allocation requests will inherit __GFP_NOWARN */ #define PF__HOLE__02000000 0x02000000#define PF_NO_SETAFFINITY 0x04000000/* Userland is not allowed to meddle with cpus_mask */ #define PF_MCE_EARLY 0x08000000/* Early kill for mce process policy */
include/linux/sched/mm.h +13 -4
@@ -236,16 +236,25 @@ static inline gfp_t current_gfp_context(gfp_t flags) {unsigned int pflags =READ_ONCE (current->flags); -if (unlikely (pflags & (PF_MEMALLOC_NOIO |PF_MEMALLOC_NOFS |PF_MEMALLOC_PIN ))) { +if (unlikely (pflags & (PF_MEMALLOC_NOIO | +PF_MEMALLOC_NOFS | +PF_MEMALLOC_NORECLAIM | +PF_MEMALLOC_NOWARN | +PF_MEMALLOC_PIN ))) {/* -* NOIO implies both NOIO and NOFS and it is a weaker context -* so always make sure it makes precedence +* Stronger flags before weaker flags: +* NORECLAIM implies NOIO, which in turn implies NOFS */ -if (pflags &PF_MEMALLOC_NOIO ) +if (pflags &PF_MEMALLOC_NORECLAIM ) + flags &= ~__GFP_DIRECT_RECLAIM; +else if (pflags &PF_MEMALLOC_NOIO ) flags &= ~(__GFP_IO | __GFP_FS);else if (pflags &PF_MEMALLOC_NOFS ) flags &= ~__GFP_FS; +if (pflags &PF_MEMALLOC_NOWARN ) + flags |= __GFP_NOWARN; +if (pflags &PF_MEMALLOC_PIN ) flags &= ~__GFP_MOVABLE; }
Signature · 1 line
2.43.0
Re: [PATCH 2/3] mm: introduce PF_MEMALLOC_NORECLAIM, PF_MEMALLOC_NOWARN
Quoted 3 lines · Kent Overstreet
Introduce PF_MEMALLOC_* equivalents of some
GFP_flags:
PF_MEMALLOC_NORECLAIM->GFP_NOWAIT
In an ideal world, this would be nice, but we are in a world with implicit "too small to fail" guarantees that has so far been impossible to get away from [1] for small order GFP_KERNEL allocations, and this scoping would be only safe if no allocations underneath relied on this behavior. But how to ensure that's the case?
[1] https://lwn.net/Articles/723317/
Quoted 62 lines
PF_MEMALLOC_NOWARN->__GFP_NOWARNCc Vlastimil Babka <[email protected]>Cc Matthew Wilcox <[email protected]>Cc Michal Hocko <[email protected]>Cc Darrick J. Wong <[email protected]>Signed-off-by Kent Overstreet <[email protected]>--- include/linux/sched.h | 4 ++-- include/linux/sched/mm.h | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-)include/linux/sched.h +2
-2@@ -1755,8 +1755,8 @@ extern struct pid *cad_pid;* I am cleaning dirty pages from some other bdi. */ #define PF_KTHREAD 0x00200000/* I am a kernel thread */ #define PF_RANDOMIZE 0x00400000/* Randomize virtual address space */ -#define PF__HOLE__00800000 0x00800000 -#define PF__HOLE__01000000 0x01000000 +#define PF_MEMALLOC_NORECLAIM 0x00800000/* All allocation requests will inherit __GFP_NOWARN */ +#define PF_MEMALLOC_NOWARN 0x01000000/* All allocation requests will inherit __GFP_NOWARN */ #define PF__HOLE__02000000 0x02000000#define PF_NO_SETAFFINITY 0x04000000/* Userland is not allowed to meddle with cpus_mask */ #define PF_MCE_EARLY 0x08000000/* Early kill for mce process policy */ include/linux/sched/mm.h +13
-4@@ -236,16 +236,25 @@ static inline gfp_t current_gfp_context(gfp_t flags) {unsigned int pflags =READ_ONCE (current->flags); -if (unlikely (pflags & (PF_MEMALLOC_NOIO |PF_MEMALLOC_NOFS |PF_MEMALLOC_PIN ))) { +if (unlikely (pflags & (PF_MEMALLOC_NOIO | +PF_MEMALLOC_NOFS | +PF_MEMALLOC_NORECLAIM | +PF_MEMALLOC_NOWARN | +PF_MEMALLOC_PIN ))) {/* -* NOIO implies both NOIO and NOFS and it is a weaker context -* so always make sure it makes precedence +* Stronger flags before weaker flags: +* NORECLAIM implies NOIO, which in turn implies NOFS */ -if (pflags &PF_MEMALLOC_NOIO ) +if (pflags &PF_MEMALLOC_NORECLAIM ) + flags &= ~__GFP_DIRECT_RECLAIM; +else if (pflags &PF_MEMALLOC_NOIO ) flags &= ~(__GFP_IO | __GFP_FS);else if (pflags &PF_MEMALLOC_NOFS ) flags &= ~__GFP_FS; +if (pflags &PF_MEMALLOC_NOWARN ) + flags |= __GFP_NOWARN; +if (pflags &PF_MEMALLOC_PIN ) flags &= ~__GFP_MOVABLE; }
Re: [PATCH 2/3] mm: introduce PF_MEMALLOC_NORECLAIM, PF_MEMALLOC_NOWARN
Quoted 10 lines · Vlastimil Babka
Quoted 3 lines · Kent Overstreet
Introduce PF_MEMALLOC_* equivalents of some
GFP_flags:
PF_MEMALLOC_NORECLAIM->GFP_NOWAITIn an ideal world, this would be nice, but we are in a world with implicit "too small to fail" guarantees that has so far been impossible to get away from [1] for small order
GFP_KERNELallocations, and this scoping would be only safe if no allocations underneath relied on this behavior. But how to ensure that's the case?
Right http://lkml.kernel.org/r/Zbu_yyChbCO6b2Lj@tiehlicka
Quoted 1 line
Signature · 2 lines
Michal Hocko SUSE Labs
Re: [PATCH 2/3] mm: introduce PF_MEMALLOC_NORECLAIM, PF_MEMALLOC_NOWARN
In reply to Vlastimil BabkaQuoted 10 lines · Vlastimil Babka
Quoted 3 lines · Kent Overstreet
Introduce PF_MEMALLOC_* equivalents of some
GFP_flags:
PF_MEMALLOC_NORECLAIM->GFP_NOWAITIn an ideal world, this would be nice, but we are in a world with implicit "too small to fail" guarantees that has so far been impossible to get away from [1] for small order
GFP_KERNELallocations, and this scoping would be only safe if no allocations underneath relied on this behavior. But how to ensure that's the case?
Fault injection. You can't know if code works if it never gets tested, and if small allocations don't fail in practice, then you need fault injection.
But there's a code pattern that absolutely requires GFP_NOWAIT. Say you've got locks held and you want to allocate memory:
p = kmalloc(GFP_NOWAIT);
if (p)
goto success;
unlock();
p = kmalloc(GFP_KERNEL);
/* unwind and retry, or tryrelock, depending on what you're doing */
that is - try the allocation nonblocking, then unlock or unwind, then try it GFP_KERNEL.
bcachefs uses this heavily because we've got bch2_trans_unlock() and bch2_trans_relock(); relock succeeds iff nothing else took write locks on the nodes he had locked before - so we can safely use GFP_KERNEL without causing deadlocks, only the occasional transaction restart.
but: the first GFP_NOWAIT allocation, before using GFP_KERNEL, is absolutely required - calling unlock() has to be a slowpath operation, otherwise it will livelock when multiple threads are contending for the same locks.
More broadly, there's a bunch of other GFP_NOWAIT uses in the kernel, and we're _not_ going to kill them off, and we are trying to kill off gfp_t for this kind of purpose - so we need this.
PATCH 3/3 mempool: kvmalloc pool
In reply to Kent OverstreetAdd mempool_init_kvmalloc_pool() and mempool_create_kvmalloc_pool(), which wrap kvmalloc() instead of kmalloc() - kmalloc() with a vmalloc() fallback.
This is part of a bcachefs cleanup - dropping an internal kvpmalloc() helper (which predates kvmalloc()) along with mempool helpers; this replaces the bcachefs-private kvpmalloc_pool.
--- include/linux/mempool.h | 13 +++++++++++++ mm/mempool.c | 13 +++++++++++++ 2 files changed, 26 insertions(+)
include/linux/mempool.h +13 -0
@@ -94,6 +94,19 @@ static inline mempool_t *mempool_create_kmalloc_pool(int min_nr, size_t size) (void *) size); } +void *mempool_kvmalloc (gfp_t gfp_mask,void *pool_data); +void mempool_kvfree (void *element,void *pool_data); + +static inline int mempool_init_kvmalloc_pool (mempool_t *pool,int min_nr,size_t size) +{ +return mempool_init (pool, min_nr, mempool_kvmalloc, mempool_kvfree, (void *) size); +} + +static inline mempool_t *mempool_create_kvmalloc_pool (int min_nr,size_t size) +{ +return mempool_create (min_nr, mempool_kvmalloc, mempool_kvfree, (void *) size); +} +/* * A mempool_alloc_t and mempool_free_t for a simple page allocator that * allocates pages of the order specified by pool_data
mm/mempool.c +13 -0
@@ -543,6 +543,19 @@ void mempool_kfree(void *element, void *pool_data) }EXPORT_SYMBOL (mempool_kfree); +void *mempool_kvmalloc (gfp_t gfp_mask,void *pool_data) +{ +size_t size = (size_t )pool_data; +return kvmalloc (size, gfp_mask); +} +EXPORT_SYMBOL (mempool_kvmalloc); + +void mempool_kvfree (void *element,void *pool_data) +{ +kvfree (element); +} +EXPORT_SYMBOL (mempool_kvfree); +/* * A simple mempool-backed page allocator that allocates pages * of the order specified by pool_data.
Signature · 1 line
2.43.0