x86/ftrace: Cure boot time W+X mapping
Subsystems (15)
Peter Zijlstra
maintainercover x86/ftrace: Cure boot time W+X mapping
Hi,
These few patches re-work and re-order boot things enough to avoid ftrace creating boot time W+X maps.
The patches compile and boot for the one config I tested things on (with ftrace=function enabled; *slooooow*).
I've pushed them out for the robots to have a go at here:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git x86/mm.poke_me
Peter Zijlstra
maintainerPATCH 1/5 mm: Move mm_cachep initialization to mm_init()
In order to allow using mm_alloc() much earlier, move initializing mm_cachep into mm_init().
--- include/linux/sched/task.h | 1 + init/main.c | 1 + kernel/fork.c | 32 ++++++++++++++++++-------------- 3 files changed, 20 insertions(+), 14 deletions(-)
include/linux/sched/task.h +1 -0
@@ -65,6 +65,7 @@ extern void sched_dead(struct task_strucvoid __noreturn do_task_dead (void );void __noreturn make_task_dead (int signr); +extern void mm_cache_init (void );extern void proc_caches_init (void );extern void fork_init (void );
init/main.c +1 -0
@@ -860,6 +860,7 @@ static void __init mm_init(void)/* Should be run after espfix64 is set up. */ pti_init ();kmsan_init_runtime (); +mm_cache_init (); }#ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET
kernel/fork.c +18 -14
@@ -3015,10 +3015,27 @@ static void sighand_ctor(void *data)init_waitqueue_head (&sighand->signalfd_wqh); } -void __init proc_caches_init (void ) +void __init mm_cache_init (void ) {unsigned int mm_size; +/* +* The mm_cpumask is located at the end of mm_struct, and is +* dynamically sized based on the maximum CPU number this system +* can have, taking hotplug into account (nr_cpu_ids). +*/ + mm_size =sizeof (struct mm_struct ) +cpumask_size (); + + mm_cachep =kmem_cache_create_usercopy ("mm_struct" , + mm_size,ARCH_MIN_MMSTRUCT_ALIGN , +SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , + offsetof(struct mm_struct , saved_auxv), + sizeof_field(struct mm_struct, saved_auxv), +NULL ); +} + +void __init proc_caches_init (void ) +{ sighand_cachep = kmem_cache_create("sighand_cache", sizeof(struct sighand_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
@@ -3036,19 +3053,6 @@ void __init proc_caches_init(void)SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , NULL); -/* -* The mm_cpumask is located at the end of mm_struct, and is -* dynamically sized based on the maximum CPU number this system -* can have, taking hotplug into account (nr_cpu_ids). -*/ - mm_size =sizeof (struct mm_struct ) +cpumask_size (); - - mm_cachep =kmem_cache_create_usercopy ("mm_struct" , - mm_size,ARCH_MIN_MMSTRUCT_ALIGN , -SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , - offsetof(struct mm_struct , saved_auxv), - sizeof_field(struct mm_struct, saved_auxv), -NULL ); vm_area_cachep =KMEM_CACHE (vm_area_struct,SLAB_PANIC |SLAB_ACCOUNT );mmap_init ();nsproxy_cache_init ();
[tip: x86/mm] mm: Move mm_cachep initialization to mm_init()
The following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: a2e87952bf54b99e8d560c095a2c75ebc676e1fb Gitweb: https://git.kernel.org/tip/a2e87952bf54b99e8d560c095a2c75ebc676e1fb Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 25 Oct 2022 21:38:18 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 01 Nov 2022 13:43:56 +01:00
mm: Move mm_cachep initialization to mm_init()
In order to allow using mm_alloc() much earlier, move initializing mm_cachep into mm_init().
--- include/linux/sched/task.h | 1 + init/main.c | 1 + kernel/fork.c | 32 ++++++++++++++++++-------------- 3 files changed, 20 insertions(+), 14 deletions(-)
include/linux/sched/task.h +1 -0
@@ -65,6 +65,7 @@ extern void sched_dead(struct task_struct *p);void __noreturn do_task_dead (void );void __noreturn make_task_dead (int signr); +extern void mm_cache_init (void );extern void proc_caches_init (void );extern void fork_init (void );
init/main.c +1 -0
@@ -860,6 +860,7 @@ static void __init mm_init(void)/* Should be run after espfix64 is set up. */ pti_init ();kmsan_init_runtime (); +mm_cache_init (); }#ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET
kernel/fork.c +18 -14
@@ -3015,10 +3015,27 @@ static void sighand_ctor(void *data)init_waitqueue_head (&sighand->signalfd_wqh); } -void __init proc_caches_init (void ) +void __init mm_cache_init (void ) {unsigned int mm_size; +/* +* The mm_cpumask is located at the end of mm_struct, and is +* dynamically sized based on the maximum CPU number this system +* can have, taking hotplug into account (nr_cpu_ids). +*/ + mm_size =sizeof (struct mm_struct ) +cpumask_size (); + + mm_cachep =kmem_cache_create_usercopy ("mm_struct" , + mm_size,ARCH_MIN_MMSTRUCT_ALIGN , +SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , + offsetof(struct mm_struct , saved_auxv), + sizeof_field(struct mm_struct, saved_auxv), +NULL ); +} + +void __init proc_caches_init (void ) +{ sighand_cachep = kmem_cache_create("sighand_cache", sizeof(struct sighand_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
@@ -3036,19 +3053,6 @@ void __init proc_caches_init(void)SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , NULL); -/* -* The mm_cpumask is located at the end of mm_struct, and is -* dynamically sized based on the maximum CPU number this system -* can have, taking hotplug into account (nr_cpu_ids). -*/ - mm_size =sizeof (struct mm_struct ) +cpumask_size (); - - mm_cachep =kmem_cache_create_usercopy ("mm_struct" , - mm_size,ARCH_MIN_MMSTRUCT_ALIGN , -SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , - offsetof(struct mm_struct , saved_auxv), - sizeof_field(struct mm_struct, saved_auxv), -NULL ); vm_area_cachep =KMEM_CACHE (vm_area_struct,SLAB_PANIC |SLAB_ACCOUNT );mmap_init ();nsproxy_cache_init ();
[tip: x86/mm] mm: Move mm_cachep initialization to mm_init()
In reply to Peter ZijlstraThe following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: af80602799681c78f14fbe20b6185a56020dedee Gitweb: https://git.kernel.org/tip/af80602799681c78f14fbe20b6185a56020dedee Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 25 Oct 2022 21:38:18 +02:00 Committer: Dave Hansen <[email protected]> CommitterDate: Thu, 15 Dec 2022 10:37:26 -08:00
mm: Move mm_cachep initialization to mm_init()
In order to allow using mm_alloc() much earlier, move initializing mm_cachep into mm_init().
--- include/linux/sched/task.h | 1 + init/main.c | 1 + kernel/fork.c | 32 ++++++++++++++++++-------------- 3 files changed, 20 insertions(+), 14 deletions(-)
include/linux/sched/task.h +1 -0
@@ -65,6 +65,7 @@ extern void sched_dead(struct task_struct *p);void __noreturn do_task_dead (void );void __noreturn make_task_dead (int signr); +extern void mm_cache_init (void );extern void proc_caches_init (void );extern void fork_init (void );
init/main.c +1 -0
@@ -860,6 +860,7 @@ static void __init mm_init(void)/* Should be run after espfix64 is set up. */ pti_init ();kmsan_init_runtime (); +mm_cache_init (); }#ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET
kernel/fork.c +18 -14
@@ -3015,10 +3015,27 @@ static void sighand_ctor(void *data)init_waitqueue_head (&sighand->signalfd_wqh); } -void __init proc_caches_init (void ) +void __init mm_cache_init (void ) {unsigned int mm_size; +/* +* The mm_cpumask is located at the end of mm_struct, and is +* dynamically sized based on the maximum CPU number this system +* can have, taking hotplug into account (nr_cpu_ids). +*/ + mm_size =sizeof (struct mm_struct ) +cpumask_size (); + + mm_cachep =kmem_cache_create_usercopy ("mm_struct" , + mm_size,ARCH_MIN_MMSTRUCT_ALIGN , +SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , + offsetof(struct mm_struct , saved_auxv), + sizeof_field(struct mm_struct, saved_auxv), +NULL ); +} + +void __init proc_caches_init (void ) +{ sighand_cachep = kmem_cache_create("sighand_cache", sizeof(struct sighand_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
@@ -3036,19 +3053,6 @@ void __init proc_caches_init(void)SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , NULL); -/* -* The mm_cpumask is located at the end of mm_struct, and is -* dynamically sized based on the maximum CPU number this system -* can have, taking hotplug into account (nr_cpu_ids). -*/ - mm_size =sizeof (struct mm_struct ) +cpumask_size (); - - mm_cachep =kmem_cache_create_usercopy ("mm_struct" , - mm_size,ARCH_MIN_MMSTRUCT_ALIGN , -SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , - offsetof(struct mm_struct , saved_auxv), - sizeof_field(struct mm_struct, saved_auxv), -NULL ); vm_area_cachep =KMEM_CACHE (vm_area_struct,SLAB_PANIC |SLAB_ACCOUNT );mmap_init ();nsproxy_cache_init ();
Peter Zijlstra
maintainerPATCH 2/5 x86/mm: Use mm_alloc() in poking_init()
In reply to Peter ZijlstraInstead of duplicating init_mm, allocate a fresh mm. The advantage is that mm_alloc() has much simpler dependencies. Additionally it makes more conceptual sense, init_mm has no (and must not have) user state to duplicate.
--- arch/x86/mm/init.c | 2 +- include/linux/sched/task.h | 1 - kernel/fork.c | 5 ----- 3 files changed, 1 insertion(+), 7 deletions(-)
arch/x86/mm/init.c +1 -1
@@ -801,7 +801,7 @@ void __init poking_init(void)spinlock_t *ptl;pte_t *ptep; - poking_mm =copy_init_mm (); + poking_mm =mm_alloc ();BUG_ON (!poking_mm);/*
include/linux/sched/task.h +0 -1
@@ -91,7 +91,6 @@ extern void exit_itimers(struct task_strextern pid_t kernel_clone (struct kernel_clone_args *kargs);struct task_struct *create_io_thread (int (*fn)(void *),void *arg,int node);struct task_struct *fork_idle (int ); -struct mm_struct *copy_init_mm (void );extern pid_t kernel_thread (int (*fn)(void *),void *arg,unsigned long flags);extern pid_t user_mode_thread (int (*fn)(void *),void *arg,unsigned long flags);extern long kernel_wait4 (pid_t ,int __user *,int ,struct rusage *);
kernel/fork.c +0 -5
@@ -2592,11 +2592,6 @@ struct task_struct * __init fork_idle(inreturn task; } -struct mm_struct *copy_init_mm (void ) -{ -return dup_mm (NULL , &init_mm); -} -/* * This is like kernel_clone(), but shaved down and tailored to just * creating io_uring workers. It returns a created task, or an error pointer.
[tip: x86/mm] x86/mm: Use mm_alloc() in poking_init()
The following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 107b6828a7cde2de0bb293588a59892831cef78b Gitweb: https://git.kernel.org/tip/107b6828a7cde2de0bb293588a59892831cef78b Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 25 Oct 2022 21:38:21 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 01 Nov 2022 13:43:57 +01:00
x86/mm: Use mm_alloc() in poking_init()
Instead of duplicating init_mm, allocate a fresh mm. The advantage is that mm_alloc() has much simpler dependencies. Additionally it makes more conceptual sense, init_mm has no (and must not have) user state to duplicate.
--- arch/x86/mm/init.c | 2 +- include/linux/sched/task.h | 1 - kernel/fork.c | 5 ----- 3 files changed, 1 insertion(+), 7 deletions(-)
arch/x86/mm/init.c +1 -1
@@ -801,7 +801,7 @@ void __init poking_init(void)spinlock_t *ptl;pte_t *ptep; - poking_mm =copy_init_mm (); + poking_mm =mm_alloc ();BUG_ON (!poking_mm);/*
include/linux/sched/task.h +0 -1
@@ -91,7 +91,6 @@ extern void exit_itimers(struct task_struct *);extern pid_t kernel_clone (struct kernel_clone_args *kargs);struct task_struct *create_io_thread (int (*fn)(void *),void *arg,int node);struct task_struct *fork_idle (int ); -struct mm_struct *copy_init_mm (void );extern pid_t kernel_thread (int (*fn)(void *),void *arg,unsigned long flags);extern pid_t user_mode_thread (int (*fn)(void *),void *arg,unsigned long flags);extern long kernel_wait4 (pid_t ,int __user *,int ,struct rusage *);
kernel/fork.c +0 -5
@@ -2592,11 +2592,6 @@ struct task_struct * __init fork_idle(int cpu)return task; } -struct mm_struct *copy_init_mm (void ) -{ -return dup_mm (NULL , &init_mm); -} -/* * This is like kernel_clone(), but shaved down and tailored to just * creating io_uring workers. It returns a created task, or an error pointer.
[tip: x86/mm] x86/mm: Use mm_alloc() in poking_init()
In reply to Peter ZijlstraThe following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 3f4c8211d982099be693be9aa7d6fc4607dff290 Gitweb: https://git.kernel.org/tip/3f4c8211d982099be693be9aa7d6fc4607dff290 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 25 Oct 2022 21:38:21 +02:00 Committer: Dave Hansen <[email protected]> CommitterDate: Thu, 15 Dec 2022 10:37:26 -08:00
x86/mm: Use mm_alloc() in poking_init()
Instead of duplicating init_mm, allocate a fresh mm. The advantage is that mm_alloc() has much simpler dependencies. Additionally it makes more conceptual sense, init_mm has no (and must not have) user state to duplicate.
--- arch/x86/mm/init.c | 2 +- include/linux/sched/task.h | 1 - kernel/fork.c | 5 ----- 3 files changed, 1 insertion(+), 7 deletions(-)
arch/x86/mm/init.c +1 -1
@@ -801,7 +801,7 @@ void __init poking_init(void)spinlock_t *ptl;pte_t *ptep; - poking_mm =copy_init_mm (); + poking_mm =mm_alloc ();BUG_ON (!poking_mm);/*
include/linux/sched/task.h +0 -1
@@ -91,7 +91,6 @@ extern void exit_itimers(struct task_struct *);extern pid_t kernel_clone (struct kernel_clone_args *kargs);struct task_struct *create_io_thread (int (*fn)(void *),void *arg,int node);struct task_struct *fork_idle (int ); -struct mm_struct *copy_init_mm (void );extern pid_t kernel_thread (int (*fn)(void *),void *arg,unsigned long flags);extern pid_t user_mode_thread (int (*fn)(void *),void *arg,unsigned long flags);extern long kernel_wait4 (pid_t ,int __user *,int ,struct rusage *);
kernel/fork.c +0 -5
@@ -2592,11 +2592,6 @@ struct task_struct * __init fork_idle(int cpu)return task; } -struct mm_struct *copy_init_mm (void ) -{ -return dup_mm (NULL , &init_mm); -} -/* * This is like kernel_clone(), but shaved down and tailored to just * creating io_uring workers. It returns a created task, or an error pointer.
Peter Zijlstra
maintainerPATCH 3/5 x86/mm: Initialize text poking earlier
In reply to Peter ZijlstraMove poking_init() up a bunch; specifically move it right after mm_init() which is right before ftrace_init().
This will allow simplifying ftrace text poking which currently has a bunch of exceptions for early boot.
--- init/main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
init/main.c +1 -2
@@ -996,7 +996,7 @@ asmlinkage __visible void __init __no_sasort_main_extable ();trap_init ();mm_init (); - +poking_init ();ftrace_init ();/* trace_printk can be enabled here */
@@ -1135,7 +1135,6 @@ asmlinkage __visible void __init __no_sataskstats_init_early ();delayacct_init (); -poking_init ();check_bugs ();acpi_subsystem_init ();
[tip: x86/mm] x86/mm: Initialize text poking earlier
The following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 4b6f3a4cd681fa54e67c2987dfebe413cd8d5a59 Gitweb: https://git.kernel.org/tip/4b6f3a4cd681fa54e67c2987dfebe413cd8d5a59 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 25 Oct 2022 21:38:25 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 01 Nov 2022 13:43:57 +01:00
x86/mm: Initialize text poking earlier
Move poking_init() up a bunch; specifically move it right after mm_init() which is right before ftrace_init().
This will allow simplifying ftrace text poking which currently has a bunch of exceptions for early boot.
--- init/main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
init/main.c +1 -2
@@ -996,7 +996,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)sort_main_extable ();trap_init ();mm_init (); - +poking_init ();ftrace_init ();/* trace_printk can be enabled here */
@@ -1135,7 +1135,6 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)taskstats_init_early ();delayacct_init (); -poking_init ();check_bugs ();acpi_subsystem_init ();
[tip: x86/mm] x86/mm: Initialize text poking earlier
In reply to Peter ZijlstraThe following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 5b93a83649c7cba3a15eb7e8959b250841acb1b1 Gitweb: https://git.kernel.org/tip/5b93a83649c7cba3a15eb7e8959b250841acb1b1 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 25 Oct 2022 21:38:25 +02:00 Committer: Dave Hansen <[email protected]> CommitterDate: Thu, 15 Dec 2022 10:37:26 -08:00
x86/mm: Initialize text poking earlier
Move poking_init() up a bunch; specifically move it right after mm_init() which is right before ftrace_init().
This will allow simplifying ftrace text poking which currently has a bunch of exceptions for early boot.
--- init/main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
init/main.c +1 -2
@@ -996,7 +996,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)sort_main_extable ();trap_init ();mm_init (); - +poking_init ();ftrace_init ();/* trace_printk can be enabled here */
@@ -1135,7 +1135,6 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)taskstats_init_early ();delayacct_init (); -poking_init ();check_bugs ();acpi_subsystem_init ();
Peter Zijlstra
maintainerPATCH 4/5 x86/ftrace: Remove SYSTEM_BOOTING exceptions
In reply to Peter ZijlstraNow that text_poke is available before ftrace, remove the SYSTEM_BOOTING exceptions.
Specifically, this cures a W+X case during boot.
--- arch/x86/kernel/alternative.c | 10 ---------- arch/x86/kernel/ftrace.c | 3 +-- 2 files changed, 1 insertion(+), 12 deletions(-)
arch/x86/kernel/alternative.c +0 -10
@@ -1681,11 +1681,6 @@ void __ref text_poke_queue(void *addr, c {struct text_poke_loc *tp; -if (unlikely (system_state ==SYSTEM_BOOTING )) { -text_poke_early (addr, opcode, len); -return ; - } -text_poke_flush (addr); tp = &tp_vec[tp_vec_nr++];
@@ -1707,11 +1702,6 @@ void __ref text_poke_bp(void *addr, cons {struct text_poke_loc tp; -if (unlikely (system_state ==SYSTEM_BOOTING )) { -text_poke_early (addr, opcode, len); -return ; - } -text_poke_loc_init (&tp, addr, opcode, len, emulate);text_poke_bp_batch (&tp,1 ); }
arch/x86/kernel/ftrace.c +1 -2
@@ -415,8 +415,7 @@ create_trampoline(struct ftrace_ops *opsset_vm_flush_reset_perms (trampoline); -if (likely (system_state !=SYSTEM_BOOTING )) -set_memory_ro ((unsigned long )trampoline, npages); +set_memory_ro ((unsigned long )trampoline, npages);set_memory_x ((unsigned long )trampoline, npages);return (unsigned long )trampoline; fail:
Steven Rostedt
maintainerRe: [PATCH 4/5] x86/ftrace: Remove SYSTEM_BOOTING exceptions
Quoted 4 lines · On Tue, 25 Oct 2022 22:07:00 +0200 Peter Zijlstra <[email protected]> wrote:
Now that
text_pokeis available before ftrace, remove theSYSTEM_BOOTINGexceptions.Specifically, this cures a W+X case during boot.
We have W+X all over the place (the entire kernel text). And I don't think we really want this.
This will slow down boots in general, as it will cause all static_branches to use this memory page logic. And I don't think we really want to do that at boot up when we don't need to.
I would change this to:
if (unlikely(system_state == SYSTEM_BOOTING) &&
core_kernel_text((unsigned long)addr)) {This way we still do memcpy() on all core kernel text which is still writable. It was the ftrace allocated trampoline that caused issues, not the locations that were being updated.
-- Steve
Quoted 46 lines
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/kernel/alternative.c | 10 ---------- arch/x86/kernel/ftrace.c | 3 +-- 2 files changed, 1 insertion(+), 12 deletions(-)arch/x86/kernel/alternative.c +0
-10@@ -1681,11 +1681,6 @@ void __ref text_poke_queue(void *addr, c {struct text_poke_loc *tp; -if (unlikely (system_state ==SYSTEM_BOOTING )) { -text_poke_early (addr, opcode, len); -return ; - } -text_poke_flush (addr); tp = &tp_vec[tp_vec_nr++];@@ -1707,11 +1702,6 @@ void __ref text_poke_bp(void *addr, cons {struct text_poke_loc tp; -if (unlikely (system_state ==SYSTEM_BOOTING )) { -text_poke_early (addr, opcode, len); -return ; - } -text_poke_loc_init (&tp, addr, opcode, len, emulate);text_poke_bp_batch (&tp,1 ); }arch/x86/kernel/ftrace.c +1
-2@@ -415,8 +415,7 @@ create_trampoline(struct ftrace_ops *opsset_vm_flush_reset_perms (trampoline); -if (likely (system_state !=SYSTEM_BOOTING )) -set_memory_ro ((unsigned long )trampoline, npages); +set_memory_ro ((unsigned long )trampoline, npages);set_memory_x ((unsigned long )trampoline, npages);return (unsigned long )trampoline; fail:
Peter Zijlstra
maintainerRe: [PATCH 4/5] x86/ftrace: Remove SYSTEM_BOOTING exceptions
Quoted 6 lines · Steven Rostedt
Quoted 3 lines · On Tue, 25 Oct 2022 22:07:00 +0200 Peter Zijlstra <[email protected]> wrote:
Now that
text_pokeis available before ftrace, remove theSYSTEM_BOOTINGexceptions.
Quoted 8 lines
Quoted 1 line
Specifically, this cures a W+X case during boot.
We have W+X all over the place (the entire kernel text). And I don't think we really want this.
This will slow down boots in general, as it will cause all
static_branchesto use this memory page logic. And I don't think we really want to do that at boot up when we don't need to.
Both static_call and jump_label explicitly call text_poke_early() when appropriate.
Quoted 8 lines
I would change this to:
if (unlikely(system_state == SYSTEM_BOOTING) && core_kernel_text((unsigned long)addr)) {This way we still do
memcpy()on all core kernel text which is still writable. It was the ftrace allocated trampoline that caused issues, not the locations that were being updated.
I would suggest changing ftrace to call text_poke_early() when appropriate if it matters (it already does a little of that); doing a boot test with and without my patch 4 on shows no noticable overhead over being horribly slow either way.
[tip: x86/mm] x86/ftrace: Remove SYSTEM_BOOTING exceptions
In reply to Peter ZijlstraThe following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 52a56f20bb7c34ed4b48466ad2d443165fad942f Gitweb: https://git.kernel.org/tip/52a56f20bb7c34ed4b48466ad2d443165fad942f Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 25 Oct 2022 21:39:47 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 01 Nov 2022 13:43:58 +01:00
x86/ftrace: Remove SYSTEM_BOOTING exceptions
Now that text_poke is available before ftrace, remove the SYSTEM_BOOTING exceptions.
Specifically, this cures a W+X case during boot.
--- arch/x86/kernel/alternative.c | 10 ---------- arch/x86/kernel/ftrace.c | 3 +-- 2 files changed, 1 insertion(+), 12 deletions(-)
arch/x86/kernel/alternative.c +0 -10
@@ -1681,11 +1681,6 @@ void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const voi {struct text_poke_loc *tp; -if (unlikely (system_state ==SYSTEM_BOOTING )) { -text_poke_early (addr, opcode, len); -return ; - } -text_poke_flush (addr); tp = &tp_vec[tp_vec_nr++];
@@ -1707,11 +1702,6 @@ void __ref text_poke_bp(void *addr, const void *opcode, size_t len, const void * {struct text_poke_loc tp; -if (unlikely (system_state ==SYSTEM_BOOTING )) { -text_poke_early (addr, opcode, len); -return ; - } -text_poke_loc_init (&tp, addr, opcode, len, emulate);text_poke_bp_batch (&tp,1 ); }
arch/x86/kernel/ftrace.c +1 -2
@@ -415,8 +415,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)set_vm_flush_reset_perms (trampoline); -if (likely (system_state !=SYSTEM_BOOTING )) -set_memory_ro ((unsigned long )trampoline, npages); +set_memory_ro ((unsigned long )trampoline, npages);set_memory_x ((unsigned long )trampoline, npages);return (unsigned long )trampoline; fail:
[tip: x86/mm] x86/ftrace: Remove SYSTEM_BOOTING exceptions
In reply to Peter ZijlstraThe following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: eb7d389d5b2b3c453332abc41c3eea73290cc006 Gitweb: https://git.kernel.org/tip/eb7d389d5b2b3c453332abc41c3eea73290cc006 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 25 Oct 2022 21:39:47 +02:00 Committer: Dave Hansen <[email protected]> CommitterDate: Thu, 15 Dec 2022 10:37:26 -08:00
x86/ftrace: Remove SYSTEM_BOOTING exceptions
Now that text_poke is available before ftrace, remove the SYSTEM_BOOTING exceptions.
Specifically, this cures a W+X case during boot.
--- arch/x86/kernel/alternative.c | 10 ---------- arch/x86/kernel/ftrace.c | 3 +-- 2 files changed, 1 insertion(+), 12 deletions(-)
arch/x86/kernel/alternative.c +0 -10
@@ -1681,11 +1681,6 @@ void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const voi {struct text_poke_loc *tp; -if (unlikely (system_state ==SYSTEM_BOOTING )) { -text_poke_early (addr, opcode, len); -return ; - } -text_poke_flush (addr); tp = &tp_vec[tp_vec_nr++];
@@ -1707,11 +1702,6 @@ void __ref text_poke_bp(void *addr, const void *opcode, size_t len, const void * {struct text_poke_loc tp; -if (unlikely (system_state ==SYSTEM_BOOTING )) { -text_poke_early (addr, opcode, len); -return ; - } -text_poke_loc_init (&tp, addr, opcode, len, emulate);text_poke_bp_batch (&tp,1 ); }
arch/x86/kernel/ftrace.c +1 -2
@@ -415,8 +415,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)set_vm_flush_reset_perms (trampoline); -if (likely (system_state !=SYSTEM_BOOTING )) -set_memory_ro ((unsigned long )trampoline, npages); +set_memory_ro ((unsigned long )trampoline, npages);set_memory_x ((unsigned long )trampoline, npages);return (unsigned long )trampoline; fail:
Peter Zijlstra
maintainerPATCH 5/5 x86/mm: Do verify W^X at boot up
In reply to Peter ZijlstraStraight up revert of commit:
a970174d7a10 ("x86/mm: Do not verify W^X at boot up")
now that the root cause has been fixed.
--- arch/x86/mm/pat/set_memory.c | 4 ---- 1 file changed, 4 deletions(-)
arch/x86/mm/pat/set_memory.c +0 -4
@@ -587,10 +587,6 @@ static inline pgprot_t verify_rwx(pgprot {unsigned long end; -/* Kernel text is rw at boot up */ -if (system_state ==SYSTEM_BOOTING ) -return new; -/* * 32-bit has some unfixable W+X issues, like EFI code * and writeable data being in the same page. Disable
[tip: x86/mm] x86/mm: Do verify W^X at boot up
The following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 237c7e967566ca7048fd7e74951fccb026f92df0 Gitweb: https://git.kernel.org/tip/237c7e967566ca7048fd7e74951fccb026f92df0 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 25 Oct 2022 21:39:43 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 01 Nov 2022 13:43:58 +01:00
x86/mm: Do verify W^X at boot up
Straight up revert of commit:
a970174d7a10 ("x86/mm: Do not verify W^X at boot up")
now that the root cause has been fixed.
--- arch/x86/mm/pat/set_memory.c | 4 ---- 1 file changed, 4 deletions(-)
arch/x86/mm/pat/set_memory.c +0 -4
@@ -587,10 +587,6 @@ static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long star {unsigned long end; -/* Kernel text is rw at boot up */ -if (system_state ==SYSTEM_BOOTING ) -return new; -/* * 32-bit has some unfixable W+X issues, like EFI code * and writeable data being in the same page. Disable
[tip: x86/mm] x86/mm: Do verify W^X at boot up
In reply to Peter ZijlstraThe following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 414ebf148cb5c5fa727ec51fdb69c4ab82dccf3b Gitweb: https://git.kernel.org/tip/414ebf148cb5c5fa727ec51fdb69c4ab82dccf3b Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 25 Oct 2022 21:39:43 +02:00 Committer: Dave Hansen <[email protected]> CommitterDate: Thu, 15 Dec 2022 10:37:26 -08:00
x86/mm: Do verify W^X at boot up
Straight up revert of commit:
a970174d7a10 ("x86/mm: Do not verify W^X at boot up")
now that the root cause has been fixed.
--- arch/x86/mm/pat/set_memory.c | 4 ---- 1 file changed, 4 deletions(-)
arch/x86/mm/pat/set_memory.c +0 -4
@@ -587,10 +587,6 @@ static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long star {unsigned long end; -/* Kernel text is rw at boot up */ -if (system_state ==SYSTEM_BOOTING ) -return new; -/* * 32-bit has some unfixable W+X issues, like EFI code * and writeable data being in the same page. Disable
Quoted 3 lines · Peter Zijlstra
These few patches re-work and re-order boot things enough to avoid ftrace creating boot time W+X maps.
Thanks, looks fine.
Quoted 2 lines
The patches compile and boot for the one config I tested things on (with ftrace=function enabled; *slooooow*).
So this might be just tracing overhead, but it might also be that you slowed down text_poke() at bootup a _lot_.
The only part that the NX^W checking cared about was that
- if (likely(system_state != SYSTEM_BOOTING))
- set_memory_ro((unsigned long)trampoline, npages);
+ set_memory_ro((unsigned long)trampoline, npages);
set_memory_x((unsigned long)trampoline, npages);for the create_trampoline(), because without the 'set_memory_ro()', the 'set_memory_x()' will complain.
It does strike me that it's stupid to make those be two calls that do exactly the same thing, and we should have a combined "set it read-only and executable" function, but that's a separate issue.
The slowness is probably not the trampilines, but just the regular "text_poke of kernel text" that we probably want to keep special just because otherwise it's _so_ slow to do for every alternative etc.
Linus
Steven Rostedt
maintainerQuoted 3 lines · On Tue, 25 Oct 2022 16:07:25 -0700 Linus Torvalds…
The slowness is probably not the trampilines, but just the regular "
text_pokeof kernel text" that we probably want to keep special just because otherwise it's _so_ slow to do for every alternative etc.
Yes. That's why I recommended to change patch 4 to:
if (unlikely(system_state == SYSTEM_BOOTING) &&
core_kernel_text((unsigned long)addr)) {
text_poke_early(addr, opcode, len);
return;
}The only special case we had was the ftrace trampoline that was dynamically allocated, and there's paths that require updating it (when you add two function callbacks to the same location).
-- Steve
Peter Zijlstra
maintainerQuoted 3 lines · Linus Torvalds
It does strike me that it's stupid to make those be two calls that do exactly the same thing, and we should have a combined "set it read-only and executable" function, but that's a separate issue.
Right, and we have it all over the place. Something like the below perhaps? I'll feed it to the robots, see if something breaks.
Quoted 3 lines
The slowness is probably not the trampilines, but just the regular "
text_pokeof kernel text" that we probably want to keep special just because otherwise it's _so_ slow to do for every alternative etc.
I tried with and without the patches, it's dead slow either way and I couldn't spot a noticable difference between the two -- so I'm assuming it's simply the trace overhead, not the trace-enable time.
---
arch/arm/mach-omap1/sram-init.c +2 -4
@@ -74,8 +74,7 @@ void *omap_sram_push(void *funcp, unsign dst =fncpy (sram, funcp, size); -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages);return dst; }
@@ -126,8 +125,7 @@ static void __init omap_detect_and_map_s base = (unsigned long )omap_sram_base; pages =PAGE_ALIGN (omap_sram_size) /PAGE_SIZE ; -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages); }static void (*_omap_sram_reprogram_clock)(u32 dpllctl,u32 ckctl);
arch/arm/mach-omap2/sram.c +2 -4
@@ -96,8 +96,7 @@ void *omap_sram_push(void *funcp, unsign dst =fncpy (sram, funcp, size); -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages);return dst; }
@@ -217,8 +216,7 @@ static void __init omap2_map_sram(void) base = (unsigned long )omap_sram_base; pages =PAGE_ALIGN (omap_sram_size) /PAGE_SIZE ; -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages); } static void (*_omap2_sram_ddr_init)(u32 *slow_dll_ctrl, u32 fast_dll_ctrl,
arch/powerpc/kernel/kprobes.c +3 -4
@@ -134,10 +134,9 @@ void *alloc_insn_page(void)if (!page)return NULL ; -if (strict_module_rwx_enabled ()) { -set_memory_ro ((unsigned long )page,1 ); -set_memory_x ((unsigned long )page,1 ); - } +if (strict_module_rwx_enabled ()) +set_memory_rox ((unsigned long )page,1 ); +return page; }
arch/x86/kernel/ftrace.c +1 -2
@@ -415,8 +415,7 @@ create_trampoline(struct ftrace_ops *opsset_vm_flush_reset_perms (trampoline); -set_memory_ro ((unsigned long )trampoline, npages); -set_memory_x ((unsigned long )trampoline, npages); +set_memory_rox ((unsigned long )trampoline, npages);return (unsigned long )trampoline; fail:tramp_free (trampoline);
arch/x86/kernel/kprobes/core.c +1 -6
@@ -415,17 +415,12 @@ void *alloc_insn_page(void)return NULL ;set_vm_flush_reset_perms (page); -/* -* First make the page read-only, and only then make it executable to -* prevent it from being W+X in between. -*/ -set_memory_ro ((unsigned long )page,1 );/* * TODO: Once additional kernel code protection mechanisms are set, ensure * that the page was not maliciously altered and it is still zeroed. */ -set_memory_x ((unsigned long )page,1 ); +set_memory_rox ((unsigned long )page,1 );return page; }
drivers/misc/sram-exec.c +1 -4
@@ -106,10 +106,7 @@ void *sram_exec_copy(struct gen_pool *po dst_cpy =fncpy (dst, src, size); - ret =set_memory_ro ((unsigned long )base, pages); -if (ret) - goto error_out; - ret =set_memory_x ((unsigned long )base, pages); + ret =set_memory_rox ((unsigned long )base, pages);if (ret) goto error_out;
include/linux/filter.h +1 -2
@@ -860,8 +860,7 @@ static inline void bpf_prog_lock_ro(strustatic inline void bpf_jit_binary_lock_ro (struct bpf_binary_header *hdr) {set_vm_flush_reset_perms (hdr); -set_memory_ro ((unsigned long )hdr, hdr->size >>PAGE_SHIFT ); -set_memory_x ((unsigned long )hdr, hdr->size >>PAGE_SHIFT ); +set_memory_rox ((unsigned long )hdr, hdr->size >>PAGE_SHIFT ); }int sk_filter_trim_cap (struct sock *sk,struct sk_buff *skb,unsigned int cap);
include/linux/set_memory.h +8 -0
@@ -14,6 +14,14 @@ static inline int set_memory_x(unsignedstatic inline int set_memory_nx (unsigned long addr,int numpages) {return 0 ; }#endif +static inline int set_memory_rox (unsigned long addr,int numpages) +{ +int ret =set_memory_ro (addr, numpages); +if (ret) +return ret; +return set_memory_x (addr, numpages); +} +#ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP static inline int set_direct_map_invalid_noflush (struct page *page) {
kernel/bpf/bpf_struct_ops.c +1 -2
@@ -494,8 +494,7 @@ static int bpf_struct_ops_map_update_elerefcount_set (&kvalue->refcnt,1 );bpf_map_inc (map); -set_memory_ro ((long )st_map->image,1 ); -set_memory_x ((long )st_map->image,1 ); +set_memory_rox ((long )st_map->image,1 ); err = st_ops->reg (kdata);if (likely (!err)) {/* Pair with smp_load_acquire() during lookup_elem().
kernel/bpf/core.c +2 -4
@@ -864,8 +864,7 @@ static struct bpf_prog_pack *alloc_new_plist_add_tail (&pack->list, &pack_list);set_vm_flush_reset_perms (pack->ptr); -set_memory_ro ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE ); -set_memory_x ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE ); +set_memory_rox ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE );return pack; }
@@ -883,8 +882,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_if (ptr) {bpf_fill_ill_insns (ptr, size);set_vm_flush_reset_perms (ptr); -set_memory_ro ((unsigned long )ptr, size /PAGE_SIZE ); -set_memory_x ((unsigned long )ptr, size /PAGE_SIZE ); +set_memory_rox ((unsigned long )ptr, size /PAGE_SIZE ); } goto out; }
kernel/bpf/trampoline.c +1 -2
@@ -468,8 +468,7 @@ static int bpf_trampoline_update(structif (err <0 ) goto out; -set_memory_ro ((long )im->image,1 ); -set_memory_x ((long )im->image,1 ); +set_memory_rox ((long )im->image,1 );WARN_ON (tr->cur_image && tr->selector ==0 );WARN_ON (!tr->cur_image && tr->selector);
net/bpf/bpf_dummy_struct_ops.c +1 -2
@@ -124,8 +124,7 @@ int bpf_struct_ops_test_run(struct bpf_pif (err <0 ) goto out; -set_memory_ro ((long )image,1 ); -set_memory_x ((long )image,1 ); +set_memory_rox ((long )image,1 ); prog_ret =dummy_ops_call_op (image, args); err =dummy_ops_copy_args (args);
Quoted 3 lines · Peter Zijlstra
Right, and we have it all over the place. Something like the below perhaps? I'll feed it to the robots, see if something breaks.
I was nodding along with the patches like this:
arch/arm/mach-omap1/sram-init.c void *omap_sram_push(void *funcp, unsign · line 77
Quoted 3 lines
arch/arm/mach-omap1/sram-init.c +1
-2-set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages);
but then I got to this part:
include/linux/set_memory.h static inline int set_memory_x(unsigned · line 23
Quoted 7 lines
include/linux/set_memory.h +7
-0+static inline int set_memory_rox (unsigned long addr,int numpages) +{ +int ret =set_memory_ro (addr, numpages); +if (ret) +return ret; +return set_memory_x (addr, numpages); +}
and that's when I went "no, I really meant make it one single call".
set_memory_ro() and set_memory_x() basically end up doing the exact same thing, just with different bits. So it's not only silly to have the callers do two different calls, it's silly to have the *implementation* do two different scans of the page tables and page merging/splitting.
I think in the case of x86, the set_memory_rox() function would basically just be
int set_memory_rox(unsigned long addr, int numpages)
{
pgprot_t clr = __pgprot(_PAGE_RW);
pgprot_t set = { 0 };
if (__supported_pte_mask & _PAGE_NX)
set.pgprot |= _PAGE_NX;
return change_page_attr_set_clr(&addr, numpages, set, clr, 0, NULL);
}or something close to that. (NOTE! The above was cobbled together in the MUA, hasn't seen a compiler much less been booted, and might be completely broken for some reason - it's meant to be the concept, not some kind of final word).
IOW, the whole "set_rox" is really just a _single_ change_page_attr_set_clr() call.
Maybe you meant to do that, and this patch was just prep-work for the arch code being the second stage?
Linus
Peter Zijlstra
maintainerQuoted 2 lines · Linus Torvalds
Maybe you meant to do that, and this patch was just prep-work for the arch code being the second stage?
Yeah; also, since this is cross arch, we need a fallback. Anyway; robots hated on me for missing a few includes. I'll go prod at this more.
Peter Zijlstra
maintainerQuoted 8 lines · Peter Zijlstra
Quoted 2 lines · Linus Torvalds
Maybe you meant to do that, and this patch was just prep-work for the arch code being the second stage?
Yeah; also, since this is cross arch, we need a fallback. Anyway; robots hated on me for missing a few includes. I'll go prod at this more.
Got around to it; I added the below patch on top and things seem to still boot so it must be good :-)
---
Subject: x86/mm: Implement native set_memory_rox()
From: Peter Zijlstra <[email protected]>
Date: Sat Oct 29 13:19:31 CEST 2022
Provide a native implementation of set_memory_rox(), avoiding the double set_memory_ro();set_memory_x(); calls.
--- arch/x86/include/asm/set_memory.h | 3 +++ arch/x86/mm/pat/set_memory.c | 10 ++++++++++ include/linux/set_memory.h | 2 ++ 3 files changed, 15 insertions(+)
arch/x86/include/asm/set_memory.h +3 -0
@@ -6,6 +6,9 @@#include <asm/page.h> #include <asm-generic/set_memory.h> +#define set_memory_rox set_memory_rox +int set_memory_rox (unsigned long addr,int numpages); +/* * The set_memory_* API can be used to change various attributes of a virtual * address range. The attributes include:
arch/x86/mm/pat/set_memory.c +10 -0
@@ -2025,6 +2025,16 @@ int set_memory_ro(unsigned long addr, inreturn change_page_attr_clear (&addr, numpages,__pgprot (_PAGE_RW),0 ); } +int set_memory_rox (unsigned long addr,int numpages) +{ +pgprot_t clr =__pgprot (_PAGE_RW); + +if (__supported_pte_mask & _PAGE_NX) + clr.pgprot |= _PAGE_NX; + +return change_page_attr_clear (&addr, numpages, clr,0 ); +} +int set_memory_rw (unsigned long addr,int numpages) {return change_page_attr_set (&addr, numpages,__pgprot (_PAGE_RW),0 );
include/linux/set_memory.h +2 -0
@@ -14,6 +14,7 @@ static inline int set_memory_x(unsignedstatic inline int set_memory_nx (unsigned long addr,int numpages) {return 0 ; }#endif +#ifndef set_memory_roxstatic inline int set_memory_rox (unsigned long addr,int numpages) {int ret =set_memory_ro (addr, numpages);
@@ -21,6 +22,7 @@ static inline int set_memory_rox(unsignereturn ret; return set_memory_x(addr, numpages); } +#endif #ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP static inline int set_direct_map_invalid_noflush (struct page *page)
Quoted 3 lines · Peter Zijlstra
Got around to it; I added the below patch on top and things seem to still boot so it must be good :-)
Thanks, looks good to me, and as I see your simpler version, I realized that my broken MUA version wasn't "rox", it was "ronx".
Linus
[tip: x86/mm] mm: Introduce set_memory_rox()
In reply to Peter ZijlstraThe following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 1f6eae43052889579dae56eae275003b9a876c21 Gitweb: https://git.kernel.org/tip/1f6eae43052889579dae56eae275003b9a876c21 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 26 Oct 2022 12:13:03 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 01 Nov 2022 13:43:58 +01:00
mm: Introduce set_memory_rox()
Because endlessly repeating:
set_memory_ro() set_memory_x()
is getting tedious.
--- arch/arm/mach-omap1/sram-init.c | 8 +++----- arch/arm/mach-omap2/sram.c | 8 +++----- arch/powerpc/kernel/kprobes.c | 9 ++++----- arch/x86/kernel/ftrace.c | 5 ++--- arch/x86/kernel/kprobes/core.c | 9 ++------- drivers/misc/sram-exec.c | 7 ++----- include/linux/filter.h | 3 +-- include/linux/set_memory.h | 8 ++++++++ kernel/bpf/bpf_struct_ops.c | 3 +-- kernel/bpf/core.c | 6 ++---- kernel/bpf/trampoline.c | 3 +-- net/bpf/bpf_dummy_struct_ops.c | 3 +-- 12 files changed, 30 insertions(+), 42 deletions(-)
arch/arm/mach-omap1/sram-init.c +3 -5
@@ -10,11 +10,11 @@#include <linux/kernel.h> #include <linux/init.h> #include <linux/io.h> +#include <linux/set_memory.h> #include <asm/fncpy.h> #include <asm/tlb.h> #include <asm/cacheflush.h> -#include <asm/set_memory.h> #include <asm/mach/map.h>
@@ -74,8 +74,7 @@ void *omap_sram_push(void *funcp, unsigned long size) dst =fncpy (sram, funcp, size); -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages);return dst; }
@@ -126,8 +125,7 @@ static void __init omap_detect_and_map_sram(void) base = (unsigned long )omap_sram_base; pages =PAGE_ALIGN (omap_sram_size) /PAGE_SIZE ; -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages); }static void (*_omap_sram_reprogram_clock)(u32 dpllctl,u32 ckctl);
arch/arm/mach-omap2/sram.c +3 -5
@@ -14,11 +14,11 @@#include <linux/kernel.h> #include <linux/init.h> #include <linux/io.h> +#include <linux/set_memory.h> #include <asm/fncpy.h> #include <asm/tlb.h> #include <asm/cacheflush.h> -#include <asm/set_memory.h> #include <asm/mach/map.h>
@@ -96,8 +96,7 @@ void *omap_sram_push(void *funcp, unsigned long size) dst =fncpy (sram, funcp, size); -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages);return dst; }
@@ -217,8 +216,7 @@ static void __init omap2_map_sram(void) base = (unsigned long )omap_sram_base; pages =PAGE_ALIGN (omap_sram_size) /PAGE_SIZE ; -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages); } static void (*_omap2_sram_ddr_init)(u32 *slow_dll_ctrl, u32 fast_dll_ctrl,
arch/powerpc/kernel/kprobes.c +4 -5
@@ -20,12 +20,12 @@#include <linux/kdebug.h> #include <linux/slab.h> #include <linux/moduleloader.h> +#include <linux/set_memory.h> #include <asm/code-patching.h> #include <asm/cacheflush.h> #include <asm/sstep.h> #include <asm/sections.h> #include <asm/inst.h> -#include <asm/set_memory.h> #include <linux/uaccess.h> DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
@@ -134,10 +134,9 @@ void *alloc_insn_page(void)if (!page)return NULL ; -if (strict_module_rwx_enabled ()) { -set_memory_ro ((unsigned long )page,1 ); -set_memory_x ((unsigned long )page,1 ); - } +if (strict_module_rwx_enabled ()) +set_memory_rox ((unsigned long )page,1 ); +return page; }
arch/x86/kernel/ftrace.c +2 -3
@@ -24,10 +24,10 @@#include <linux/module.h> #include <linux/memory.h> #include <linux/vmalloc.h> +#include <linux/set_memory.h> #include <trace/syscall.h> -#include <asm/set_memory.h> #include <asm/kprobes.h> #include <asm/ftrace.h> #include <asm/nops.h>
@@ -415,8 +415,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)set_vm_flush_reset_perms (trampoline); -set_memory_ro ((unsigned long )trampoline, npages); -set_memory_x ((unsigned long )trampoline, npages); +set_memory_rox ((unsigned long )trampoline, npages);return (unsigned long )trampoline; fail:tramp_free (trampoline);
arch/x86/kernel/kprobes/core.c +2 -7
@@ -43,6 +43,7 @@#include <linux/objtool.h> #include <linux/vmalloc.h> #include <linux/pgtable.h> +#include <linux/set_memory.h> #include <asm/text-patching.h> #include <asm/cacheflush.h>
@@ -51,7 +52,6 @@#include <asm/alternative.h> #include <asm/insn.h> #include <asm/debugreg.h> -#include <asm/set_memory.h> #include <asm/ibt.h> #include "common.h"
@@ -415,17 +415,12 @@ void *alloc_insn_page(void)return NULL ;set_vm_flush_reset_perms (page); -/* -* First make the page read-only, and only then make it executable to -* prevent it from being W+X in between. -*/ -set_memory_ro ((unsigned long )page,1 );/* * TODO: Once additional kernel code protection mechanisms are set, ensure * that the page was not maliciously altered and it is still zeroed. */ -set_memory_x ((unsigned long )page,1 ); +set_memory_rox ((unsigned long )page,1 );return page; }
drivers/misc/sram-exec.c +2 -5
@@ -10,9 +10,9 @@#include <linux/genalloc.h> #include <linux/mm.h> #include <linux/sram.h> +#include <linux/set_memory.h> #include <asm/fncpy.h> -#include <asm/set_memory.h> #include "sram.h"
@@ -106,10 +106,7 @@ void *sram_exec_copy(struct gen_pool *pool, void *dst, void *src, dst_cpy =fncpy (dst, src, size); - ret =set_memory_ro ((unsigned long )base, pages); -if (ret) - goto error_out; - ret =set_memory_x ((unsigned long )base, pages); + ret =set_memory_rox ((unsigned long )base, pages);if (ret) goto error_out;
include/linux/filter.h +1 -2
@@ -860,8 +860,7 @@ static inline void bpf_prog_lock_ro(struct bpf_prog *fp)static inline void bpf_jit_binary_lock_ro (struct bpf_binary_header *hdr) {set_vm_flush_reset_perms (hdr); -set_memory_ro ((unsigned long )hdr, hdr->size >>PAGE_SHIFT ); -set_memory_x ((unsigned long )hdr, hdr->size >>PAGE_SHIFT ); +set_memory_rox ((unsigned long )hdr, hdr->size >>PAGE_SHIFT ); }int sk_filter_trim_cap (struct sock *sk,struct sk_buff *skb,unsigned int cap);
include/linux/set_memory.h +8 -0
@@ -14,6 +14,14 @@ static inline int set_memory_x(unsigned long addr, int numpages) { return 0; }static inline int set_memory_nx (unsigned long addr,int numpages) {return 0 ; }#endif +static inline int set_memory_rox (unsigned long addr,int numpages) +{ +int ret =set_memory_ro (addr, numpages); +if (ret) +return ret; +return set_memory_x (addr, numpages); +} +#ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP static inline int set_direct_map_invalid_noflush (struct page *page) {
kernel/bpf/bpf_struct_ops.c +1 -2
@@ -494,8 +494,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,refcount_set (&kvalue->refcnt,1 );bpf_map_inc (map); -set_memory_ro ((long )st_map->image,1 ); -set_memory_x ((long )st_map->image,1 ); +set_memory_rox ((long )st_map->image,1 ); err = st_ops->reg (kdata);if (likely (!err)) {/* Pair with smp_load_acquire() during lookup_elem().
kernel/bpf/core.c +2 -4
@@ -864,8 +864,7 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_inslist_add_tail (&pack->list, &pack_list);set_vm_flush_reset_perms (pack->ptr); -set_memory_ro ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE ); -set_memory_x ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE ); +set_memory_rox ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE );return pack; }
@@ -883,8 +882,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)if (ptr) {bpf_fill_ill_insns (ptr, size);set_vm_flush_reset_perms (ptr); -set_memory_ro ((unsigned long )ptr, size /PAGE_SIZE ); -set_memory_x ((unsigned long )ptr, size /PAGE_SIZE ); +set_memory_rox ((unsigned long )ptr, size /PAGE_SIZE ); } goto out; }
kernel/bpf/trampoline.c +1 -2
@@ -468,8 +468,7 @@ again:if (err <0 ) goto out; -set_memory_ro ((long )im->image,1 ); -set_memory_x ((long )im->image,1 ); +set_memory_rox ((long )im->image,1 );WARN_ON (tr->cur_image && tr->selector ==0 );WARN_ON (!tr->cur_image && tr->selector);
net/bpf/bpf_dummy_struct_ops.c +1 -2
@@ -124,8 +124,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,if (err <0 ) goto out; -set_memory_ro ((long )image,1 ); -set_memory_x ((long )image,1 ); +set_memory_rox ((long )image,1 ); prog_ret =dummy_ops_call_op (image, args); err =dummy_ops_copy_args (args);
[tip: x86/mm] mm: Introduce set_memory_rox()
In reply to Peter ZijlstraThe following commit has been merged into the x86/mm branch of tip:
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: d48567c9a0d1e605639f8a8705a61bbb55fb4e84 Gitweb: https://git.kernel.org/tip/d48567c9a0d1e605639f8a8705a61bbb55fb4e84 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 26 Oct 2022 12:13:03 +02:00 Committer: Dave Hansen <[email protected]> CommitterDate: Thu, 15 Dec 2022 10:37:26 -08:00
mm: Introduce set_memory_rox()
Because endlessly repeating:
set_memory_ro() set_memory_x()
is getting tedious.
--- arch/arm/mach-omap1/sram-init.c | 8 +++----- arch/arm/mach-omap2/sram.c | 8 +++----- arch/powerpc/kernel/kprobes.c | 9 ++++----- arch/x86/kernel/ftrace.c | 5 ++--- arch/x86/kernel/kprobes/core.c | 9 ++------- drivers/misc/sram-exec.c | 7 ++----- include/linux/filter.h | 3 +-- include/linux/set_memory.h | 8 ++++++++ kernel/bpf/bpf_struct_ops.c | 3 +-- kernel/bpf/core.c | 6 ++---- kernel/bpf/trampoline.c | 3 +-- net/bpf/bpf_dummy_struct_ops.c | 3 +-- 12 files changed, 30 insertions(+), 42 deletions(-)
arch/arm/mach-omap1/sram-init.c +3 -5
@@ -10,11 +10,11 @@#include <linux/kernel.h> #include <linux/init.h> #include <linux/io.h> +#include <linux/set_memory.h> #include <asm/fncpy.h> #include <asm/tlb.h> #include <asm/cacheflush.h> -#include <asm/set_memory.h> #include <asm/mach/map.h>
@@ -74,8 +74,7 @@ void *omap_sram_push(void *funcp, unsigned long size) dst =fncpy (sram, funcp, size); -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages);return dst; }
@@ -126,8 +125,7 @@ static void __init omap_detect_and_map_sram(void) base = (unsigned long )omap_sram_base; pages =PAGE_ALIGN (omap_sram_size) /PAGE_SIZE ; -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages); }static void (*_omap_sram_reprogram_clock)(u32 dpllctl,u32 ckctl);
arch/arm/mach-omap2/sram.c +3 -5
@@ -14,11 +14,11 @@#include <linux/kernel.h> #include <linux/init.h> #include <linux/io.h> +#include <linux/set_memory.h> #include <asm/fncpy.h> #include <asm/tlb.h> #include <asm/cacheflush.h> -#include <asm/set_memory.h> #include <asm/mach/map.h>
@@ -96,8 +96,7 @@ void *omap_sram_push(void *funcp, unsigned long size) dst =fncpy (sram, funcp, size); -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages);return dst; }
@@ -217,8 +216,7 @@ static void __init omap2_map_sram(void) base = (unsigned long )omap_sram_base; pages =PAGE_ALIGN (omap_sram_size) /PAGE_SIZE ; -set_memory_ro (base, pages); -set_memory_x (base, pages); +set_memory_rox (base, pages); } static void (*_omap2_sram_ddr_init)(u32 *slow_dll_ctrl, u32 fast_dll_ctrl,
arch/powerpc/kernel/kprobes.c +4 -5
@@ -20,12 +20,12 @@#include <linux/kdebug.h> #include <linux/slab.h> #include <linux/moduleloader.h> +#include <linux/set_memory.h> #include <asm/code-patching.h> #include <asm/cacheflush.h> #include <asm/sstep.h> #include <asm/sections.h> #include <asm/inst.h> -#include <asm/set_memory.h> #include <linux/uaccess.h> DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
@@ -134,10 +134,9 @@ void *alloc_insn_page(void)if (!page)return NULL ; -if (strict_module_rwx_enabled ()) { -set_memory_ro ((unsigned long )page,1 ); -set_memory_x ((unsigned long )page,1 ); - } +if (strict_module_rwx_enabled ()) +set_memory_rox ((unsigned long )page,1 ); +return page; }
arch/x86/kernel/ftrace.c +2 -3
@@ -24,10 +24,10 @@#include <linux/module.h> #include <linux/memory.h> #include <linux/vmalloc.h> +#include <linux/set_memory.h> #include <trace/syscall.h> -#include <asm/set_memory.h> #include <asm/kprobes.h> #include <asm/ftrace.h> #include <asm/nops.h>
@@ -415,8 +415,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)set_vm_flush_reset_perms (trampoline); -set_memory_ro ((unsigned long )trampoline, npages); -set_memory_x ((unsigned long )trampoline, npages); +set_memory_rox ((unsigned long )trampoline, npages);return (unsigned long )trampoline; fail:tramp_free (trampoline);
arch/x86/kernel/kprobes/core.c +2 -7
@@ -43,6 +43,7 @@#include <linux/objtool.h> #include <linux/vmalloc.h> #include <linux/pgtable.h> +#include <linux/set_memory.h> #include <asm/text-patching.h> #include <asm/cacheflush.h>
@@ -51,7 +52,6 @@#include <asm/alternative.h> #include <asm/insn.h> #include <asm/debugreg.h> -#include <asm/set_memory.h> #include <asm/ibt.h> #include "common.h"
@@ -415,17 +415,12 @@ void *alloc_insn_page(void)return NULL ;set_vm_flush_reset_perms (page); -/* -* First make the page read-only, and only then make it executable to -* prevent it from being W+X in between. -*/ -set_memory_ro ((unsigned long )page,1 );/* * TODO: Once additional kernel code protection mechanisms are set, ensure * that the page was not maliciously altered and it is still zeroed. */ -set_memory_x ((unsigned long )page,1 ); +set_memory_rox ((unsigned long )page,1 );return page; }
drivers/misc/sram-exec.c +2 -5
@@ -10,9 +10,9 @@#include <linux/genalloc.h> #include <linux/mm.h> #include <linux/sram.h> +#include <linux/set_memory.h> #include <asm/fncpy.h> -#include <asm/set_memory.h> #include "sram.h"
@@ -106,10 +106,7 @@ void *sram_exec_copy(struct gen_pool *pool, void *dst, void *src, dst_cpy =fncpy (dst, src, size); - ret =set_memory_ro ((unsigned long )base, pages); -if (ret) - goto error_out; - ret =set_memory_x ((unsigned long )base, pages); + ret =set_memory_rox ((unsigned long )base, pages);if (ret) goto error_out;
include/linux/filter.h +1 -2
@@ -860,8 +860,7 @@ static inline void bpf_prog_lock_ro(struct bpf_prog *fp)static inline void bpf_jit_binary_lock_ro (struct bpf_binary_header *hdr) {set_vm_flush_reset_perms (hdr); -set_memory_ro ((unsigned long )hdr, hdr->size >>PAGE_SHIFT ); -set_memory_x ((unsigned long )hdr, hdr->size >>PAGE_SHIFT ); +set_memory_rox ((unsigned long )hdr, hdr->size >>PAGE_SHIFT ); }int sk_filter_trim_cap (struct sock *sk,struct sk_buff *skb,unsigned int cap);
include/linux/set_memory.h +8 -0
@@ -14,6 +14,14 @@ static inline int set_memory_x(unsigned long addr, int numpages) { return 0; }static inline int set_memory_nx (unsigned long addr,int numpages) {return 0 ; }#endif +static inline int set_memory_rox (unsigned long addr,int numpages) +{ +int ret =set_memory_ro (addr, numpages); +if (ret) +return ret; +return set_memory_x (addr, numpages); +} +#ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP static inline int set_direct_map_invalid_noflush (struct page *page) {
kernel/bpf/bpf_struct_ops.c +1 -2
@@ -494,8 +494,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,refcount_set (&kvalue->refcnt,1 );bpf_map_inc (map); -set_memory_ro ((long )st_map->image,1 ); -set_memory_x ((long )st_map->image,1 ); +set_memory_rox ((long )st_map->image,1 ); err = st_ops->reg (kdata);if (likely (!err)) {/* Pair with smp_load_acquire() during lookup_elem().
kernel/bpf/core.c +2 -4
@@ -864,8 +864,7 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_inslist_add_tail (&pack->list, &pack_list);set_vm_flush_reset_perms (pack->ptr); -set_memory_ro ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE ); -set_memory_x ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE ); +set_memory_rox ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE );return pack; }
@@ -883,8 +882,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)if (ptr) {bpf_fill_ill_insns (ptr, size);set_vm_flush_reset_perms (ptr); -set_memory_ro ((unsigned long )ptr, size /PAGE_SIZE ); -set_memory_x ((unsigned long )ptr, size /PAGE_SIZE ); +set_memory_rox ((unsigned long )ptr, size /PAGE_SIZE ); } goto out; }
kernel/bpf/trampoline.c +1 -2
@@ -468,8 +468,7 @@ again:if (err <0 ) goto out; -set_memory_ro ((long )im->image,1 ); -set_memory_x ((long )im->image,1 ); +set_memory_rox ((long )im->image,1 );WARN_ON (tr->cur_image && tr->selector ==0 );WARN_ON (!tr->cur_image && tr->selector);
net/bpf/bpf_dummy_struct_ops.c +1 -2
@@ -124,8 +124,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,if (err <0 ) goto out; -set_memory_ro ((long )image,1 ); -set_memory_x ((long )image,1 ); +set_memory_rox ((long )image,1 ); prog_ret =dummy_ops_call_op (image, args); err =dummy_ops_copy_args (args);