mm/slub: perform free consistency checks before call_rcu
Subsystems (3)
RFC PATCH 1/2 mm/slub: perform free consistency checks before call_rcu
For SLAB_TYPESAFE_BY_RCU caches we use call_rcu to perform empty slab freeing. The rcu callback rcu_free_slab() calls __free_slab() that currently includes checking the slab consistency for caches with SLAB_CONSISTENCY_CHECKS flags. This check needs the slab->objects field to be intact.
Because in the next patch we want to allow rcu_head in struct slab to become larger in debug configurations and thus potentially overwrite more fields through a union than slab_list, we want to limit the fields used in rcu_free_slab(). Thus move the consistency checks to free_slab() before call_rcu(). This can be done safely even for SLAB_TYPESAFE_BY_RCU caches where accesses to the objects can still occur after freeing them.
As a result, only the slab->slab_cache field has to be physically separate from rcu_head for the freeing callback to work. We also save some cycles in the rcu callback for caches with consistency checks enabled.
--- mm/slub.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)
mm/slub.c +10 -10
@@ -2036,14 +2036,6 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab)int order =folio_order (folio);int pages =1 << order; -if (kmem_cache_debug_flags (s,SLAB_CONSISTENCY_CHECKS )) { -void *p; - -slab_pad_check (s, slab); -for_each_object (p, s,slab_address (slab), slab->objects) -check_object (s, slab, p,SLUB_RED_INACTIVE ); - } -__slab_clear_pfmemalloc (slab);__folio_clear_slab (folio); folio->mapping =NULL ;
@@ -2062,9 +2054,17 @@ static void rcu_free_slab(struct rcu_head *h)static void free_slab (struct kmem_cache *s,struct slab *slab) { -if (unlikely (s->flags &SLAB_TYPESAFE_BY_RCU )) { +if (kmem_cache_debug_flags (s,SLAB_CONSISTENCY_CHECKS )) { +void *p; + +slab_pad_check (s, slab); +for_each_object (p, s,slab_address (slab), slab->objects) +check_object (s, slab, p,SLUB_RED_INACTIVE ); + } + +if (unlikely (s->flags &SLAB_TYPESAFE_BY_RCU ))call_rcu (&slab->rcu_head, rcu_free_slab); - }else +else __free_slab (s, slab); }
Signature · 1 line
2.37.2
RFC PATCH 2/2 mm/sl[au]b: rearrange struct slab fields to allow larger rcu_head
Joel reports [1] that increasing the rcu_head size for debugging purposes used to work before struct slab was split from struct page, but now runs into the various SLAB_MATCH() sanity checks of the layout.
This is because the rcu_head in struct page is in union with large sub-structures and has space to grow without exceeding their size, while in struct slab (for SLAB and SLUB) it's in union only with a list_head.
On closer inspection (and after the previous patch) we can put all fields except slab_cache to a union with rcu_head, as slab_cache is sufficient for the rcu freeing callbacks to work and the rest can be overwritten by rcu_head without causing issues.
This is only somewhat complicated by the need to keep SLUB's freelist+counters aligned for cmpxchg_double. As a result the fields need to be reordered so that slab_cache is first (after page flags) and the union with rcu_head follows. For consistency, do that for SLAB as well, although not necessary there.
As a result, the rcu_head field in struct page and struct slab is no longer at the same offset, but that doesn't matter as there is no casting that would rely on that in the slab freeing callbacks, so we can just drop the respective SLAB_MATCH() check.
Also we need to update the SLAB_MATCH() for compound_head to reflect the new ordering.
While at it, also add a static_assert to check the alignment needed for cmpxchg_double so mistakes are found sooner than a runtime GPF.
[1] https://lore.kernel.org/all/[email protected]/
--- mm/slab.h | 54 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 22 deletions(-)
mm/slab.h +32 -22
@@ -11,37 +11,43 @@ struct slab {#if defined(CONFIG_SLAB ) +struct kmem_cache *slab_cache;union { -struct list_head slab_list; +struct { +struct list_head slab_list; +void *freelist;/* array of free object indexes */ +void *s_mem;/* first object */ + };struct rcu_head rcu_head; }; -struct kmem_cache *slab_cache; -void *freelist;/* array of free object indexes */ -void *s_mem;/* first object */ unsigned int active;#elif defined(CONFIG_SLUB ) -union { -struct list_head slab_list; -struct rcu_head rcu_head; -#ifdef CONFIG_SLUB_CPU_PARTIAL -struct { -struct slab *next; -int slabs;/* Nr of slabs left */ - }; -#endif - };struct kmem_cache *slab_cache; -/* Double-word boundary */ -void *freelist;/* first free object */ union { -unsigned long counters;struct { -unsigned inuse:16 ; -unsigned objects:15 ; -unsigned frozen:1 ; +union { +struct list_head slab_list; +#ifdef CONFIG_SLUB_CPU_PARTIAL +struct { +struct slab *next; +int slabs;/* Nr of slabs left */ + }; +#endif + }; +/* Double-word boundary */ +void *freelist;/* first free object */ +union { +unsigned long counters; +struct { +unsigned inuse:16 ; +unsigned objects:15 ; +unsigned frozen:1 ; + }; + }; }; +struct rcu_head rcu_head; };unsigned int __unused;
@@ -66,9 +72,10 @@ struct slab {#define SLAB_MATCH (pg, sl) \ static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl))SLAB_MATCH (flags, __page_flags); -SLAB_MATCH (compound_head, slab_list);/* Ensure bit 0 is clear */ #ifndef CONFIG_SLOB -SLAB_MATCH (rcu_head, rcu_head); +SLAB_MATCH (compound_head, slab_cache);/* Ensure bit 0 is clear */ +#else +SLAB_MATCH (compound_head, slab_list);/* Ensure bit 0 is clear */ #endif SLAB_MATCH (_refcount, __page_refcount);#ifdef CONFIG_MEMCG
@@ -76,6 +83,9 @@ SLAB_MATCH(memcg_data, memcg_data);#endif #undef SLAB_MATCHstatic_assert (sizeof (struct slab ) <=sizeof (struct page )); +#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE ) && defined(CONFIG_SLUB ) +static_assert (IS_ALIGNED (offsetof(struct slab , freelist),16 )); +#endif /** * folio_slab - Converts from folio to slab.
Signature · 1 line
2.37.2
Re: [RFC PATCH 2/2] mm/sl[au]b: rearrange struct slab fields to allow larger rcu_head
Quoted 132 lines · Vlastimil Babka
Joel reports [1] that increasing the
rcu_headsize for debugging purposes used to work before struct slab was split from struct page, but now runs into the variousSLAB_MATCH()sanity checks of the layout.This is because the
rcu_headin struct page is in union with large sub-structures and has space to grow without exceeding their size, while in struct slab (for SLAB and SLUB) it's in union only with alist_head.On closer inspection (and after the previous patch) we can put all fields except
slab_cacheto a union withrcu_head, asslab_cacheis sufficient for the rcu freeing callbacks to work and the rest can be overwritten byrcu_headwithout causing issues.This is only somewhat complicated by the need to keep SLUB's freelist+counters aligned for
cmpxchg_double. As a result the fields need to be reordered so thatslab_cacheis first (after page flags) and the union withrcu_headfollows. For consistency, do that for SLAB as well, although not necessary there.As a result, the
rcu_headfield in struct page and struct slab is no longer at the same offset, but that doesn't matter as there is no casting that would rely on that in the slab freeing callbacks, so we can just drop the respectiveSLAB_MATCH()check.Also we need to update the
SLAB_MATCH()forcompound_headto reflect the new ordering.While at it, also add a
static_assertto check the alignment needed forcmpxchg_doubleso mistakes are found sooner than a runtime GPF.[1] https://lore.kernel.org/all/[email protected]/
Reported-by Joel Fernandes <[email protected]>Signed-off-by Vlastimil Babka <[email protected]>--- mm/slab.h | 54 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 22 deletions(-)mm/slab.h +32
-22@@ -11,37 +11,43 @@ struct slab {#if defined(CONFIG_SLAB ) +struct kmem_cache *slab_cache;union { -struct list_head slab_list; +struct { +struct list_head slab_list; +void *freelist;/* array of free object indexes */ +void *s_mem;/* first object */ + };struct rcu_head rcu_head; }; -struct kmem_cache *slab_cache; -void *freelist;/* array of free object indexes */ -void *s_mem;/* first object */ unsigned int active;#elif defined(CONFIG_SLUB ) -union { -struct list_head slab_list; -struct rcu_head rcu_head; -#ifdef CONFIG_SLUB_CPU_PARTIAL -struct { -struct slab *next; -int slabs;/* Nr of slabs left */ - }; -#endif - };struct kmem_cache *slab_cache; -/* Double-word boundary */ -void *freelist;/* first free object */ union { -unsigned long counters;struct { -unsigned inuse:16 ; -unsigned objects:15 ; -unsigned frozen:1 ; +union { +struct list_head slab_list; +#ifdef CONFIG_SLUB_CPU_PARTIAL +struct { +struct slab *next; +int slabs;/* Nr of slabs left */ + }; +#endif + }; +/* Double-word boundary */ +void *freelist;/* first free object */ +union { +unsigned long counters; +struct { +unsigned inuse:16 ; +unsigned objects:15 ; +unsigned frozen:1 ; + }; + }; }; +struct rcu_head rcu_head; };unsigned int __unused;@@ -66,9 +72,10 @@ struct slab {#define SLAB_MATCH (pg, sl) \ static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl))SLAB_MATCH (flags, __page_flags); -SLAB_MATCH (compound_head, slab_list);/* Ensure bit 0 is clear */ #ifndef CONFIG_SLOB -SLAB_MATCH (rcu_head, rcu_head); +SLAB_MATCH (compound_head, slab_cache);/* Ensure bit 0 is clear */ +#else +SLAB_MATCH (compound_head, slab_list);/* Ensure bit 0 is clear */ #endif SLAB_MATCH (_refcount, __page_refcount);#ifdef CONFIG_MEMCG @@ -76,6 +83,9 @@ SLAB_MATCH(memcg_data, memcg_data);#endif #undef SLAB_MATCHstatic_assert (sizeof (struct slab ) <=sizeof (struct page )); +#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE ) && defined(CONFIG_SLUB ) +static_assert (IS_ALIGNED (offsetof(struct slab , freelist),16 )); +#endif /** * folio_slab - Converts from folio to slab. Signature · 1 line
2.37.2
Looks sane to me.
For slab part:
Signature · 2 lines
Thanks, Hyeonggon
Re: [RFC PATCH 2/2] mm/sl[au]b: rearrange struct slab fields to allow larger rcu_head
In reply to Vlastimil BabkaQuoted 27 lines · Vlastimil Babka
Joel reports [1] that increasing the
rcu_headsize for debugging purposes used to work before struct slab was split from struct page, but now runs into the variousSLAB_MATCH()sanity checks of the layout.This is because the
rcu_headin struct page is in union with large sub-structures and has space to grow without exceeding their size, while in struct slab (for SLAB and SLUB) it's in union only with alist_head.On closer inspection (and after the previous patch) we can put all fields except
slab_cacheto a union withrcu_head, asslab_cacheis sufficient for the rcu freeing callbacks to work and the rest can be overwritten byrcu_headwithout causing issues.This is only somewhat complicated by the need to keep SLUB's freelist+counters aligned for
cmpxchg_double. As a result the fields need to be reordered so thatslab_cacheis first (after page flags) and the union withrcu_headfollows. For consistency, do that for SLAB as well, although not necessary there.As a result, the
rcu_headfield in struct page and struct slab is no longer at the same offset, but that doesn't matter as there is no casting that would rely on that in the slab freeing callbacks, so we can just drop the respectiveSLAB_MATCH()check.Also we need to update the
SLAB_MATCH()forcompound_headto reflect the new ordering.
Quoted 7 lines
While at it, also add a
static_assertto check the alignment needed forcmpxchg_doubleso mistakes are found sooner than a runtime GPF.[1] https://lore.kernel.org/all/[email protected]/
Reported-by Joel Fernandes <[email protected]>Signed-off-by Vlastimil Babka <[email protected]>
Both patches now pushed to slab.git for-6.1/fit_rcu_head and for-next
Quoted 95 lines
--- mm/slab.h | 54 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 22 deletions(-)mm/slab.h +32
-22@@ -11,37 +11,43 @@ struct slab {#if defined(CONFIG_SLAB ) +struct kmem_cache *slab_cache;union { -struct list_head slab_list; +struct { +struct list_head slab_list; +void *freelist;/* array of free object indexes */ +void *s_mem;/* first object */ + };struct rcu_head rcu_head; }; -struct kmem_cache *slab_cache; -void *freelist;/* array of free object indexes */ -void *s_mem;/* first object */ unsigned int active;#elif defined(CONFIG_SLUB ) -union { -struct list_head slab_list; -struct rcu_head rcu_head; -#ifdef CONFIG_SLUB_CPU_PARTIAL -struct { -struct slab *next; -int slabs;/* Nr of slabs left */ - }; -#endif - };struct kmem_cache *slab_cache; -/* Double-word boundary */ -void *freelist;/* first free object */ union { -unsigned long counters;struct { -unsigned inuse:16 ; -unsigned objects:15 ; -unsigned frozen:1 ; +union { +struct list_head slab_list; +#ifdef CONFIG_SLUB_CPU_PARTIAL +struct { +struct slab *next; +int slabs;/* Nr of slabs left */ + }; +#endif + }; +/* Double-word boundary */ +void *freelist;/* first free object */ +union { +unsigned long counters; +struct { +unsigned inuse:16 ; +unsigned objects:15 ; +unsigned frozen:1 ; + }; + }; }; +struct rcu_head rcu_head; };unsigned int __unused;@@ -66,9 +72,10 @@ struct slab {#define SLAB_MATCH (pg, sl) \ static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl))SLAB_MATCH (flags, __page_flags); -SLAB_MATCH (compound_head, slab_list);/* Ensure bit 0 is clear */ #ifndef CONFIG_SLOB -SLAB_MATCH (rcu_head, rcu_head); +SLAB_MATCH (compound_head, slab_cache);/* Ensure bit 0 is clear */ +#else +SLAB_MATCH (compound_head, slab_list);/* Ensure bit 0 is clear */ #endif SLAB_MATCH (_refcount, __page_refcount);#ifdef CONFIG_MEMCG @@ -76,6 +83,9 @@ SLAB_MATCH(memcg_data, memcg_data);#endif #undef SLAB_MATCHstatic_assert (sizeof (struct slab ) <=sizeof (struct page )); +#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE ) && defined(CONFIG_SLUB ) +static_assert (IS_ALIGNED (offsetof(struct slab , freelist),16 )); +#endif /** * folio_slab - Converts from folio to slab.
Quoted 52 lines · Vlastimil Babka
For
SLAB_TYPESAFE_BY_RCUcaches we usecall_rcuto perform empty slab freeing. The rcu callbackrcu_free_slab()calls__free_slab()that currently includes checking the slab consistency for caches withSLAB_CONSISTENCY_CHECKSflags. This check needs theslab->objectsfield to be intact.Because in the next patch we want to allow
rcu_headin struct slab to become larger in debug configurations and thus potentially overwrite more fields through a union thanslab_list, we want to limit the fields used inrcu_free_slab(). Thus move the consistency checks tofree_slab()beforecall_rcu(). This can be done safely even forSLAB_TYPESAFE_BY_RCUcaches where accesses to the objects can still occur after freeing them.As a result, only the
slab->slab_cachefield has to be physically separate fromrcu_headfor the freeing callback to work. We also save some cycles in the rcu callback for caches with consistency checks enabled.Signed-off-by Vlastimil Babka <[email protected]>--- mm/slub.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)mm/slub.c +4
-9@@ -2036,14 +2036,6 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab)int order =folio_order (folio);int pages =1 << order; -if (kmem_cache_debug_flags (s,SLAB_CONSISTENCY_CHECKS )) { -void *p; - -slab_pad_check (s, slab); -for_each_object (p, s,slab_address (slab), slab->objects) -check_object (s, slab, p,SLUB_RED_INACTIVE ); - } -__slab_clear_pfmemalloc (slab);__folio_clear_slab (folio); folio->mapping =NULL ;@@ -2062,9 +2054,17 @@ static void rcu_free_slab(struct rcu_head *h)static void free_slab (struct kmem_cache *s,struct slab *slab) { -if (unlikely (s->flags &SLAB_TYPESAFE_BY_RCU )) { +if (kmem_cache_debug_flags (s,SLAB_CONSISTENCY_CHECKS )) { +void *p; + +slab_pad_check (s, slab);
mm/slub.c static void rcu_free_slab(struct rcu_head *h) · line 2069
Quoted 10 lines
mm/slub.c +6
-1+for_each_object (p, s,slab_address (slab), slab->objects) +check_object (s, slab, p,SLUB_RED_INACTIVE ); + } + +if (unlikely (s->flags &SLAB_TYPESAFE_BY_RCU ))call_rcu (&slab->rcu_head, rcu_free_slab); - }else +else __free_slab (s, slab); }
So this allows corrupting 'counters' with patch 2.
The code looks still safe to me as we do only
redzone checking for SLAB_TYPESAFE_RCU caches.
Quoted 3 lines
Signature · 1 line
2.37.2
Signature · 2 lines
Thanks, Hyeonggon