provide the flexibility to enable KFENCE
Subsystems (2)
v2 cover provide the flexibility to enable KFENCE
Hi, I'm sending v2 about (re-)enabling KFENCE.
I've removed description about use case and just focus on flexibility.
Thanks.
v2:
Take KFENCE_WARN_ON() into account. Do not allow re-enabling KFENCE if it once disabled by warn.
Modify func names and comments.
RFC/v1: https://lore.kernel.org/all/[email protected]/
Tianchen Ding (2):
kfence: Allow re-enabling KFENCE after system startup
kfence: Alloc kfence_pool after system startup
mm/kfence/core.c | 114 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 93 insertions(+), 21 deletions(-)
Signature · 1 line
2.27.0
PATCH v2 1/2 kfence: Allow re-enabling KFENCE after system startup
If once KFENCE is disabled by:
echo 0 > /sys/module/kfence/parameters/sample_interval KFENCE could never be re-enabled until next rebooting.
Allow re-enabling it by writing a positive num to sample_interval.
--- mm/kfence/core.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-)
mm/kfence/core.c +18 -3
@@ -38,14 +38,17 @@#define KFENCE_WARN_ON (cond) \ ({ \ const bool __cond = WARN_ON(cond); \ - if (unlikely(__cond)) \ + if (unlikely(__cond)) { \ WRITE_ONCE(kfence_enabled, false); \ + disabled_by_warn = true; \ + } \ __cond; \ })/* === Data ================================================================= */ static bool kfence_enabled__read_mostly ; +static bool disabled_by_warn__read_mostly ;unsigned long kfence_sample_interval__read_mostly =CONFIG_KFENCE_SAMPLE_INTERVAL ;EXPORT_SYMBOL_GPL (kfence_sample_interval);/* Export for test modules. */
@@ -55,6 +58,7 @@ EXPORT_SYMBOL_GPL(kfence_sample_interval); /* Export for test modules. */#endif #define MODULE_PARAM_PREFIX "kfence." +static int kfence_enable_late (void );static int param_set_sample_interval (const char *val,const struct kernel_param *kp) {unsigned long num;
@@ -65,10 +69,11 @@ static int param_set_sample_interval(const char *val, const struct kernel_paramif (!num)/* Using 0 to indicate KFENCE is disabled. */ WRITE_ONCE (kfence_enabled, false); -else if (!READ_ONCE (kfence_enabled) && system_state !=SYSTEM_BOOTING ) -return -EINVAL ;/* Cannot (re-)enable KFENCE on-the-fly. */ *((unsigned long *)kp->arg) = num; + +if (num && !READ_ONCE (kfence_enabled) && system_state !=SYSTEM_BOOTING ) +return disabled_by_warn ? -EINVAL :kfence_enable_late ();return 0 ; }
@@ -787,6 +792,16 @@ void __init kfence_init(void) (void *)(__kfence_pool + KFENCE_POOL_SIZE)); } +static int kfence_enable_late (void ) +{ +if (!__kfence_pool) +return -EINVAL ; + +WRITE_ONCE (kfence_enabled, true); +queue_delayed_work (system_unbound_wq, &kfence_timer,0 ); +return 0 ; +} +void kfence_shutdown_cache (struct kmem_cache *s) {unsigned long flags;
Signature · 1 line
2.27.0
PATCH v2 2/2 kfence: Alloc kfence_pool after system startup
In reply to Tianchen DingAllow enabling KFENCE after system startup by allocating its pool via the page allocator. This provides the flexibility to enable KFENCE even if it wasn't enabled at boot time.
--- mm/kfence/core.c | 99 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 78 insertions(+), 21 deletions(-)
mm/kfence/core.c +78 -21
@@ -96,7 +96,7 @@ static unsigned long kfence_skip_covered_thresh __read_mostly = 75;module_param_named (skip_covered_thresh, kfence_skip_covered_thresh, ulong,0644 );/* The pool of pages used for guard pages and objects. */ -char *__kfence_pool__ro_after_init ; +char *__kfence_pool__read_mostly ;EXPORT_SYMBOL (__kfence_pool);/* Export for test modules. */ /*
@@ -537,17 +537,19 @@ static void rcu_guarded_free(struct rcu_head *h)kfence_guarded_free ((void *)meta->addr, meta, false); } -static bool __init kfence_init_pool (void ) +/* +* Initialization of the KFENCE pool after its allocation. +* Returns 0 on success; otherwise returns the address up to +* which partial initialization succeeded. +*/ +static unsigned long kfence_init_pool (void ) {unsigned long addr = (unsigned long )__kfence_pool;struct page *pages;int i; -if (!__kfence_pool) -return false; -if (!arch_kfence_init_pool ()) - goto err; +return addr; pages =virt_to_page (addr);
@@ -565,7 +567,7 @@ static bool __init kfence_init_pool(void)/* Verify we do not have a compound head page. */ if (WARN_ON (compound_head (&pages[i]) != &pages[i])) - goto err; +return addr;__SetPageSlab (&pages[i]); }
@@ -578,7 +580,7 @@ static bool __init kfence_init_pool(void)*/ for (i =0 ; i <2 ; i++) {if (unlikely (!kfence_protect (addr))) - goto err; +return addr; addr +=PAGE_SIZE ; }
@@ -595,7 +597,7 @@ static bool __init kfence_init_pool(void)/* Protect the right redzone. */ if (unlikely (!kfence_protect (addr +PAGE_SIZE ))) - goto err; +return addr; addr +=2 *PAGE_SIZE ; }
@@ -608,9 +610,21 @@ static bool __init kfence_init_pool(void)*/ kmemleak_free (__kfence_pool); -return true; +return 0 ; +} + +static bool __init kfence_init_pool_early (void ) +{ +unsigned long addr; + +if (!__kfence_pool) +return false; + + addr =kfence_init_pool (); + +if (!addr) +return true; -err:/* * Only release unprotected pages, and do not try to go back and change * page attributes due to risk of failing to do so as well. If changing
@@ -623,6 +637,22 @@ static bool __init kfence_init_pool(void)return false; } +static bool kfence_init_pool_late (void ) +{ +unsigned long addr, free_pages; + + addr =kfence_init_pool (); + +if (!addr) +return true; + +/* Same as above. */ + free_pages = (KFENCE_POOL_SIZE - (addr - (unsigned long )__kfence_pool)) /PAGE_SIZE ; +free_contig_range (page_to_pfn (virt_to_page (addr)), free_pages); + __kfence_pool =NULL ; +return false; +} +/* === DebugFS Interface ==================================================== */ static int stats_show (struct seq_file *seq,void *v)
@@ -771,31 +801,58 @@ void __init kfence_alloc_pool(void)pr_err ("failed to allocate pool\n" ); } +static void kfence_init_enable (void ) +{ +if (!IS_ENABLED (CONFIG_KFENCE_STATIC_KEYS )) +static_branch_enable (&kfence_allocation_key); +WRITE_ONCE (kfence_enabled, true); +queue_delayed_work (system_unbound_wq, &kfence_timer,0 ); +pr_info ("initialized - using %lu bytes for %d objects at 0x%p-0x%p\n" ,KFENCE_POOL_SIZE , +CONFIG_KFENCE_NUM_OBJECTS , (void *)__kfence_pool, + (void *)(__kfence_pool +KFENCE_POOL_SIZE )); +} +void __init kfence_init (void ) { + stack_hash_seed = (u32 )random_get_entropy (); +/* Setting kfence_sample_interval to 0 on boot disables KFENCE. */ if (!kfence_sample_interval)return ; - stack_hash_seed = (u32 )random_get_entropy (); -if (!kfence_init_pool ()) { +if (!kfence_init_pool_early ()) {pr_err ("%s failed\n" , __func__);return ; } -if (!IS_ENABLED (CONFIG_KFENCE_STATIC_KEYS )) -static_branch_enable (&kfence_allocation_key); -WRITE_ONCE (kfence_enabled, true); -queue_delayed_work (system_unbound_wq, &kfence_timer,0 ); -pr_info ("initialized - using %lu bytes for %d objects at 0x%p-0x%p\n" ,KFENCE_POOL_SIZE , -CONFIG_KFENCE_NUM_OBJECTS , (void *)__kfence_pool, - (void *)(__kfence_pool +KFENCE_POOL_SIZE )); +kfence_init_enable (); +} + +static int kfence_init_late (void ) +{ +const unsigned long nr_pages =KFENCE_POOL_SIZE /PAGE_SIZE ; +struct page *pages; + + pages =alloc_contig_pages (nr_pages,GFP_KERNEL , first_online_node,NULL ); + +if (!pages) +return -ENOMEM ; + + __kfence_pool =page_to_virt (pages); + +if (!kfence_init_pool_late ()) { +pr_err ("%s failed\n" , __func__); +return -EBUSY ; + } + +kfence_init_enable (); +return 0 ; }static int kfence_enable_late (void ) {if (!__kfence_pool) -return -EINVAL ; +return kfence_init_late ();WRITE_ONCE (kfence_enabled, true);queue_delayed_work (system_unbound_wq, &kfence_timer,0 );
Signature · 1 line
2.27.0
Marco Elver
maintainerRe: [PATCH v2 2/2] kfence: Alloc kfence_pool after system startup
On Sat, 5 Mar 2022 at 15:49, Tianchen Ding <[email protected]> wrote:
[...]
Quoted 8 lines
mm/kfence/core.c +6
-0+static int kfence_init_late (void ) +{ +const unsigned long nr_pages =KFENCE_POOL_SIZE /PAGE_SIZE ; +struct page *pages; + + pages =alloc_contig_pages (nr_pages,GFP_KERNEL , first_online_node,NULL );
mm/kfence/core.c:836:17: error: implicit declaration of function ‘alloc_contig_pages’ [-Werror=implicit-function-declaration]
This doesn't build without CMA. See ifdef CONFIG_CONTIG_ALLOC in gfp.h, which declares alloc_contig_pages.
Will alloc_pages() work as you expect? If so, perhaps only use alloc_contig_pages() #ifdef CONFIG_CONTIG_ALLOC.
Thanks,
-- Marco
Re: [PATCH v2 2/2] kfence: Alloc kfence_pool after system startup
Quoted 9 lines · Marco Elver
On Sat, 5 Mar 2022 at 15:49, Tianchen Ding <[email protected]> wrote:
[...]Quoted 6 lines
mm/kfence/core.c +6
-0+static int kfence_init_late (void ) +{ +const unsigned long nr_pages =KFENCE_POOL_SIZE /PAGE_SIZE ; +struct page *pages; + + pages =alloc_contig_pages (nr_pages,GFP_KERNEL , first_online_node,NULL );
Quoted 8 lines
Quoted 1 line
mm/kfence/core.c:836:17: error: implicit declaration of function ‘alloc_contig_pages’ [-Werror=implicit-function-declaration]This doesn't build without CMA. See ifdef
CONFIG_CONTIG_ALLOCingfp.h, which declaresalloc_contig_pages.Will
alloc_pages()work as you expect? If so, perhaps only usealloc_contig_pages()#ifdefCONFIG_CONTIG_ALLOC.
alloc_pages() will be fine. We could free "tail" pages after inited. Will send v3 soon.
Quoted 2 lines
Thanks,
-- Marco
Re: [PATCH v2 2/2] kfence: Alloc kfence_pool after system startup
Quoted 16 lines · Tianchen Ding
Quoted 15 lines · Marco Elver
On Sat, 5 Mar 2022 at 15:49, Tianchen Ding <[email protected]> wrote:
[...]Quoted 9 lines
mm/kfence/core.c +6
-0+static int kfence_init_late (void ) +{ + const unsigned long nr_pages = KFENCE_POOL_SIZE / PAGE_SIZE; + struct page *pages; + + pages = alloc_contig_pages(nr_pages, GFP_KERNEL,
first_online_node, NULL);
mm/kfence/core.c:836:17: error: implicit declaration of function ‘alloc_contig_pages’ [-Werror=implicit-function-declaration]This doesn't build without CMA. See ifdef
CONFIG_CONTIG_ALLOCingfp.h, which declaresalloc_contig_pages.
Quoted 7 lines
Quoted 3 lines
Will
alloc_pages()work as you expect? If so, perhaps only usealloc_contig_pages()#ifdefCONFIG_CONTIG_ALLOC.
alloc_pages()will be fine. We could free "tail" pages after inited. Will send v3 soon.
Oh, I remember why we use alloc_contig_pages()...alloc_pages() (or alloc_pages_exact()) only support pages less than MAX_ORDER (default 11). The alloc would fail when KFENCE_NUM_OBJECTS >= 512.
So the design would be:
ifndef CONFIG_CONTIG_ALLOC and KFENCE_NUM_OBJECTS exceeds MAX_ORDER, we do not support alloc KFENCE pool after system startup.
Quoted 3 lines
Quoted 2 lines
Thanks,
-- Marco