Skip to contentSkip to conversation

objtool/x86: noinstr vs PARAVIRT

RFC PATCH Applied Peter Zijlstra 3 participants 25 messages Jun 21, 2021 – Jun 22, 2021

Subsystems (7)

v1 of 2 Applied Latest is v2 →

lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

cover objtool/x86: noinstr vs PARAVIRT

Hi!

Per long standing request from Thomas, this here mostly sorts the whole noinstr validation vs PARAVIRT.

It isn't exactly pretty or robust, but it does work.

So far only tested on x86_64-defconfig+kvm_guest.config+xen.config. I did a quick allyesconfig build, but still saw some funnies with that, will poke at them.

author maintainer
#1 lore raw

PATCH 1/18 x86/entry: Fix noinstr fail in __do_fast_syscall_32()

vmlinux.o: warning: objtool: __do_fast_syscall_32()+0xf5: call to trace_hardirqs_off() leaves .noinstr.text section

Fixes 5d5675df792f ("x86/entry: Fix entry/exit mismatch on failed fast 32-bit syscalls")
---
 arch/x86/entry/common.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/entry/common.c +1 -1
@@ -166,8 +166,8 @@ static noinstr bool __do_fast_syscall_32
 		/* User code screwed up. */
 		regs->ax = -EFAULT;
 
-		instrumentation_end();
 		local_irq_disable();
+		instrumentation_end();
 		irqentry_exit_to_user_mode(regs);
 		return false;
 	}
merge bot Applied
#2 lore raw

[tip: objtool/urgent] x86/entry: Fix noinstr fail in __do_fast_syscall_32()

The following commit has been merged into the objtool/urgent branch of tip:

The following commit has been merged into the objtool/urgent branch of tip:

Commit-ID:     240001d4e3041832e8a2654adc3ccf1683132b92
Gitweb:        https://git.kernel.org/tip/240001d4e3041832e8a2654adc3ccf1683132b92
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Mon, 21 Jun 2021 13:12:34 +02:00
Committer:     Ingo Molnar <[email protected]>
CommitterDate: Tue, 22 Jun 2021 13:56:42 +02:00

x86/entry: Fix noinstr fail in __do_fast_syscall_32()

Fix:

vmlinux.o: warning: objtool: __do_fast_syscall_32()+0xf5: call to trace_hardirqs_off() leaves .noinstr.text section

Fixes 5d5675df792f ("x86/entry: Fix entry/exit mismatch on failed fast 32-bit syscalls")
---
 arch/x86/entry/common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/entry/common.c +1 -1
index 7b2542b..a6bf516 100644
@@ -130,8 +130,8 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
 		/* User code screwed up. */
 		regs->ax = -EFAULT;
 
-		instrumentation_end();
 		local_irq_disable();
+		instrumentation_end();
 		irqentry_exit_to_user_mode(regs);
 		return false;
 	}
author maintainer
#3 lore raw

PATCH 2/18 x86/xen: Fix noinstr fail in xen_pv_evtchn_do_upcall()

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: xen_pv_evtchn_do_upcall()+0x23: call to irq_enter_rcu() leaves .noinstr.text section

Fixes 359f01d1816f ("x86/entry: Use run_sysvec_on_irqstack_cond() for XEN upcall")
---
 arch/x86/entry/common.c |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
arch/x86/entry/common.c +2 -1
@@ -304,15 +304,16 @@ __visible noinstr void xen_pv_evtchn_do_
 	irqentry_state_t state = irqentry_enter(regs);
 	bool inhcall;
 
+	instrumentation_begin();
 	run_sysvec_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs);
 
 	inhcall = get_and_clear_inhcall();
 	if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) {
-		instrumentation_begin();
 		irqentry_exit_cond_resched();
 		instrumentation_end();
 		restore_inhcall(inhcall);
 	} else {
+		instrumentation_end();
 		irqentry_exit(regs, state);
 	}
 }
merge bot Applied
#4 lore raw

[tip: objtool/urgent] x86/xen: Fix noinstr fail in xen_pv_evtchn_do_upcall()

The following commit has been merged into the objtool/urgent branch of tip:

The following commit has been merged into the objtool/urgent branch of tip:

Commit-ID:     84e60065df9ef03759115a7e48c04bbc0d292165
Gitweb:        https://git.kernel.org/tip/84e60065df9ef03759115a7e48c04bbc0d292165
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Mon, 21 Jun 2021 13:12:35 +02:00
Committer:     Ingo Molnar <[email protected]>
CommitterDate: Tue, 22 Jun 2021 13:56:42 +02:00

x86/xen: Fix noinstr fail in xen_pv_evtchn_do_upcall()

Fix:

vmlinux.o: warning: objtool: xen_pv_evtchn_do_upcall()+0x23: call to irq_enter_rcu() leaves .noinstr.text section

Fixes 359f01d1816f ("x86/entry: Use run_sysvec_on_irqstack_cond() for XEN upcall")
---
 arch/x86/entry/common.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
arch/x86/entry/common.c +2 -1
index a6bf516..04bce95 100644
@@ -269,15 +269,16 @@ __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs)
 	irqentry_state_t state = irqentry_enter(regs);
 	bool inhcall;
 
+	instrumentation_begin();
 	run_sysvec_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs);
 
 	inhcall = get_and_clear_inhcall();
 	if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) {
-		instrumentation_begin();
 		irqentry_exit_cond_resched();
 		instrumentation_end();
 		restore_inhcall(inhcall);
 	} else {
+		instrumentation_end();
 		irqentry_exit(regs, state);
 	}
 }
author maintainer
#5 lore raw

PATCH 3/18 x86/xen: Fix noinstr fail in exc_xen_unknown_trap()

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: exc_xen_unknown_trap()+0x7: call to printk() leaves .noinstr.text section

Fixes 2e92493637a0 ("x86/xen: avoid warning in Xen pv guest with CONFIG_AMD_MEM_ENCRYPT enabled")
---
 arch/x86/xen/enlighten_pv.c |    2 ++
 1 file changed, 2 insertions(+)
arch/x86/xen/enlighten_pv.c +2 -0
@@ -592,8 +592,10 @@ DEFINE_IDTENTRY_RAW(xenpv_exc_debug)
 DEFINE_IDTENTRY_RAW(exc_xen_unknown_trap)
 {
 	/* This should never happen and there is no way to handle it. */
+	instrumentation_begin();
 	pr_err("Unknown trap in Xen PV mode.");
 	BUG();
+	instrumentation_end();
 }
 
 #ifdef CONFIG_X86_MCE
merge bot Applied
#6 lore raw

[tip: objtool/urgent] x86/xen: Fix noinstr fail in exc_xen_unknown_trap()

The following commit has been merged into the objtool/urgent branch of tip:

The following commit has been merged into the objtool/urgent branch of tip:

Commit-ID:     4c9c26f1e67648f41f28f8c997c5c9467a3dbbe4
Gitweb:        https://git.kernel.org/tip/4c9c26f1e67648f41f28f8c997c5c9467a3dbbe4
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Mon, 21 Jun 2021 13:12:36 +02:00
Committer:     Ingo Molnar <[email protected]>
CommitterDate: Tue, 22 Jun 2021 13:56:42 +02:00

x86/xen: Fix noinstr fail in exc_xen_unknown_trap()

Fix:

vmlinux.o: warning: objtool: exc_xen_unknown_trap()+0x7: call to printk() leaves .noinstr.text section

Fixes 2e92493637a0 ("x86/xen: avoid warning in Xen pv guest with CONFIG_AMD_MEM_ENCRYPT enabled")
---
 arch/x86/xen/enlighten_pv.c | 2 ++
 1 file changed, 2 insertions(+)
arch/x86/xen/enlighten_pv.c +2 -0
index e87699a..0314942 100644
@@ -592,8 +592,10 @@ DEFINE_IDTENTRY_RAW(xenpv_exc_debug)
 DEFINE_IDTENTRY_RAW(exc_xen_unknown_trap)
 {
 	/* This should never happen and there is no way to handle it. */
+	instrumentation_begin();
 	pr_err("Unknown trap in Xen PV mode.");
 	BUG();
+	instrumentation_end();
 }
 
 #ifdef CONFIG_X86_MCE
author maintainer
#7 lore raw

PATCH 4/18 x86: Always inline task_size_max()

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: handle_bug()+0x10: call to task_size_max() leaves .noinstr.text section

When #UD isn't a BUG, we shouldn't violate noinstr (we'll still probably die, but that's another story).

Fixes 025768a966a3 ("x86/cpu: Use alternative to generate the TASK_SIZE_MAX constant")
---
 arch/x86/include/asm/page_64.h |    2 +-
 arch/x86/kernel/traps.c        |    9 ++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)
arch/x86/include/asm/page_64.h +1 -1
@@ -75,7 +75,7 @@ void copy_page(void *to, void *from);
  *
  * With page table isolation enabled, we map the LDT in ... [stay tuned]
  */
-static inline unsigned long task_size_max(void)
+static __always_inline unsigned long task_size_max(void)
 {
 	unsigned long ret;
 
merge bot Applied
#8 lore raw

[tip: objtool/urgent] x86: Always inline task_size_max()

The following commit has been merged into the objtool/urgent branch of tip:

The following commit has been merged into the objtool/urgent branch of tip:

Commit-ID:     1f008d46f1243899d27fd034ab5c41985bd16cee
Gitweb:        https://git.kernel.org/tip/1f008d46f1243899d27fd034ab5c41985bd16cee
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Mon, 21 Jun 2021 13:12:37 +02:00
Committer:     Ingo Molnar <[email protected]>
CommitterDate: Tue, 22 Jun 2021 13:56:43 +02:00

x86: Always inline task_size_max()

Fix:

vmlinux.o: warning: objtool: handle_bug()+0x10: call to task_size_max() leaves .noinstr.text section

When #UD isn't a BUG, we shouldn't violate noinstr (we'll still probably die, but that's another story).

Fixes 025768a966a3 ("x86/cpu: Use alternative to generate the TASK_SIZE_MAX constant")
---
 arch/x86/include/asm/page_64.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/include/asm/page_64.h +1 -1
index ca840fe..4bde0dc 100644
@@ -75,7 +75,7 @@ void copy_page(void *to, void *from);
  *
  * With page table isolation enabled, we map the LDT in ... [stay tuned]
  */
-static inline unsigned long task_size_max(void)
+static __always_inline unsigned long task_size_max(void)
 {
 	unsigned long ret;
 
author maintainer
#9 lore raw

PATCH 5/18 locking/lockdep: Improve noinstr vs errors

In reply to Peter Zijlstra

Better handle the failure paths.

vmlinux.o: warning: objtool: debug_locks_off()+0x23: call to console_verbose() leaves .noinstr.text section vmlinux.o: warning: objtool: debug_locks_off()+0x19: call to __kasan_check_write() leaves .noinstr.text section

debug_locks_off+0x19/0x40:
instrument_atomic_write at include/linux/instrumented.h:86 (inlined by) __debug_locks_off at include/linux/debug_locks.h:17 (inlined by) debug_locks_off at lib/debug_locks.c:41

Fixes 6eebad1ad303 ("lockdep: __always_inline more for noinstr")

---

include/linux/debug_locks.h +2 -0
index 2915f56ad421..edb5c186b0b7 100644
@@ -27,8 +27,10 @@ extern int debug_locks_off(void);
 	int __ret = 0;							\
 									\
 	if (!oops_in_progress && unlikely(c)) {				\
+		instrumentation_begin();				\
 		if (debug_locks_off() && !debug_locks_silent)		\
 			WARN(1, "DEBUG_LOCKS_WARN_ON(%s)", #c);		\
+		instrumentation_end();					\
 		__ret = 1;						\
 	}								\
 	__ret;								\
kernel/locking/lockdep.c +3 -1
index cb8af6a3e154..6fc60d363d26 100644
@@ -843,7 +843,7 @@ static int count_matching_names(struct lock_class *new_class)
 }
 
 /* used from NMI context -- must be lockless */
-static __always_inline struct lock_class *
+static noinstr struct lock_class *
 look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
 {
 	struct lockdep_subclass_key *key;
@@ -851,12 +851,14 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
 	struct lock_class *class;
 
 	if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) {
+		instrumentation_begin();
 		debug_locks_off();
 		printk(KERN_ERR
 			"BUG: looking up invalid subclass: %u\n", subclass);
 		printk(KERN_ERR
 			"turning off the locking correctness validator.\n");
 		dump_stack();
+		instrumentation_end();
 		return NULL;
 	}
 
lib/debug_locks.c +1 -1
index 06d3135bd184..a75ee30b77cb 100644
@@ -36,7 +36,7 @@ EXPORT_SYMBOL_GPL(debug_locks_silent);
 /*
  * Generic 'turn off all lock debugging' function:
  */
-noinstr int debug_locks_off(void)
+int debug_locks_off(void)
 {
 	if (debug_locks && __debug_locks_off()) {
 		if (!debug_locks_silent) {
merge bot Applied
#10 lore raw

[tip: objtool/urgent] locking/lockdep: Improve noinstr vs errors

The following commit has been merged into the objtool/urgent branch of tip:

The following commit has been merged into the objtool/urgent branch of tip:

Commit-ID:     49faa77759b211fff344898edc23bb780707fff5
Gitweb:        https://git.kernel.org/tip/49faa77759b211fff344898edc23bb780707fff5
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Mon, 21 Jun 2021 13:12:38 +02:00
Committer:     Ingo Molnar <[email protected]>
CommitterDate: Tue, 22 Jun 2021 13:56:43 +02:00

locking/lockdep: Improve noinstr vs errors

Better handle the failure paths.

vmlinux.o: warning: objtool: debug_locks_off()+0x23: call to console_verbose() leaves .noinstr.text section vmlinux.o: warning: objtool: debug_locks_off()+0x19: call to __kasan_check_write() leaves .noinstr.text section

  debug_locks_off+0x19/0x40:
  instrument_atomic_write at include/linux/instrumented.h:86
  (inlined by) __debug_locks_off at include/linux/debug_locks.h:17
  (inlined by) debug_locks_off at lib/debug_locks.c:41
Fixes 6eebad1ad303 ("lockdep: __always_inline more for noinstr")
---
 include/linux/debug_locks.h | 2 ++
 kernel/locking/lockdep.c    | 4 +++-
 lib/debug_locks.c           | 2 +-
 3 files changed, 6 insertions(+), 2 deletions(-)
include/linux/debug_locks.h +2 -0
index 2915f56..edb5c18 100644
@@ -27,8 +27,10 @@ extern int debug_locks_off(void);
 	int __ret = 0;							\
 									\
 	if (!oops_in_progress && unlikely(c)) {				\
+		instrumentation_begin();				\
 		if (debug_locks_off() && !debug_locks_silent)		\
 			WARN(1, "DEBUG_LOCKS_WARN_ON(%s)", #c);		\
+		instrumentation_end();					\
 		__ret = 1;						\
 	}								\
 	__ret;								\
kernel/locking/lockdep.c +3 -1
index 7641bd4..e323130 100644
@@ -843,7 +843,7 @@ static int count_matching_names(struct lock_class *new_class)
 }
 
 /* used from NMI context -- must be lockless */
-static __always_inline struct lock_class *
+static noinstr struct lock_class *
 look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
 {
 	struct lockdep_subclass_key *key;
@@ -851,12 +851,14 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
 	struct lock_class *class;
 
 	if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) {
+		instrumentation_begin();
 		debug_locks_off();
 		printk(KERN_ERR
 			"BUG: looking up invalid subclass: %u\n", subclass);
 		printk(KERN_ERR
 			"turning off the locking correctness validator.\n");
 		dump_stack();
+		instrumentation_end();
 		return NULL;
 	}
 
lib/debug_locks.c +1 -1
index 06d3135..a75ee30 100644
@@ -36,7 +36,7 @@ EXPORT_SYMBOL_GPL(debug_locks_silent);
 /*
  * Generic 'turn off all lock debugging' function:
  */
-noinstr int debug_locks_off(void)
+int debug_locks_off(void)
 {
 	if (debug_locks && __debug_locks_off()) {
 		if (!debug_locks_silent) {
author maintainer
#11 lore raw

RFC PATCH 6/18 x86/paravirt: Mark arch_local_irq_*() __always_inline

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: lockdep_hardirqs_on()+0x72: call to arch_local_save_flags() leaves .noinstr.text section vmlinux.o: warning: objtool: lockdep_hardirqs_off()+0x73: call to arch_local_save_flags() leaves .noinstr.text section vmlinux.o: warning: objtool: match_held_lock()+0x11f: call to arch_local_save_flags() leaves .noinstr.text section

vmlinux.o: warning: objtool: lock_is_held_type()+0x4e: call to arch_local_irq_save() leaves .noinstr.text section

vmlinux.o: warning: objtool: lock_is_held_type()+0x65: call to arch_local_irq_disable() leaves .noinstr.text section

vmlinux.o: warning: objtool: lock_is_held_type()+0xfe: call to arch_local_irq_enable() leaves .noinstr.text section

It makes no sense to not inline these things.

---
 arch/x86/include/asm/paravirt.h |    8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
arch/x86/include/asm/paravirt.h +4 -4
@@ -681,23 +681,23 @@ bool __raw_callee_save___native_vcpu_is_
 	((struct paravirt_callee_save) { func })
 
 #ifdef CONFIG_PARAVIRT_XXL
-static inline notrace unsigned long arch_local_save_flags(void)
+static __always_inline unsigned long arch_local_save_flags(void)
 {
 	return PVOP_ALT_CALLEE0(unsigned long, irq.save_fl, "pushf; pop %%rax;",
 				ALT_NOT(X86_FEATURE_XENPV));
 }
 
-static inline notrace void arch_local_irq_disable(void)
+static __always_inline void arch_local_irq_disable(void)
 {
 	PVOP_ALT_VCALLEE0(irq.irq_disable, "cli;", ALT_NOT(X86_FEATURE_XENPV));
 }
 
-static inline notrace void arch_local_irq_enable(void)
+static __always_inline void arch_local_irq_enable(void)
 {
 	PVOP_ALT_VCALLEE0(irq.irq_enable, "sti;", ALT_NOT(X86_FEATURE_XENPV));
 }
 
-static inline notrace unsigned long arch_local_irq_save(void)
+static __always_inline unsigned long arch_local_irq_save(void)
 {
 	unsigned long f;
 
author maintainer
#12 lore raw

RFC PATCH 7/18 x86/paravirt: Use PVOP_* for paravirt calls

In reply to Peter Zijlstra

Doing unconditional indirect calls through the pv_ops vector is weird.

---
 arch/x86/include/asm/paravirt.h |    8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
arch/x86/include/asm/paravirt.h +4 -4
@@ -52,11 +52,11 @@ void __init paravirt_set_cap(void);
 /* The paravirtualized I/O functions */
 static inline void slow_down_io(void)
 {
-	pv_ops.cpu.io_delay();
+	PVOP_VCALL0(cpu.io_delay);
 #ifdef REALLY_SLOW_IO
-	pv_ops.cpu.io_delay();
-	pv_ops.cpu.io_delay();
-	pv_ops.cpu.io_delay();
+	PVOP_VCALL0(cpu.io_delay);
+	PVOP_VCALL0(cpu.io_delay);
+	PVOP_VCALL0(cpu.io_delay);
 #endif
 }
 
author maintainer
#13 lore raw

RFC PATCH 8/18 x86/xen: Make read_cr2() noinstr

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: pv_ops[41]: native_read_cr2
vmlinux.o: warning: objtool: pv_ops[41]: xen_read_cr2
vmlinux.o: warning: objtool: pv_ops[41]: xen_read_cr2_direct
vmlinux.o: warning: objtool: exc_double_fault()+0x15: call to pv_ops[41]() leaves .noinstr.text section

---
 arch/x86/include/asm/paravirt.h |    2 +-
 arch/x86/kernel/paravirt.c      |    7 ++++++-
 arch/x86/xen/xen-asm.S          |    2 ++
 3 files changed, 9 insertions(+), 2 deletions(-)
arch/x86/include/asm/paravirt.h +1 -1
@@ -133,7 +133,7 @@ static inline void write_cr0(unsigned lo
 	PVOP_VCALL1(cpu.write_cr0, x);
 }
 
-static inline unsigned long read_cr2(void)
+static __always_inline unsigned long read_cr2(void)
 {
 	return PVOP_ALT_CALLEE0(unsigned long, mmu.read_cr2,
 				"mov %%cr2, %%rax;",
arch/x86/kernel/paravirt.c +6 -1
@@ -238,6 +238,11 @@ struct pv_info pv_info = {
 /* 64-bit pagetable entries */
 #define PTE_IDENT	__PV_IS_CALLEE_SAVE(_paravirt_ident_64)
 
+static noinstr unsigned long pv_native_read_cr2(void)
+{
+	return native_read_cr2();
+}
+
 struct paravirt_patch_template pv_ops = {
 	/* Cpu ops. */
 	.cpu.io_delay		= native_io_delay,
@@ -298,7 +303,7 @@ struct paravirt_patch_template pv_ops =
 	.mmu.exit_mmap		= paravirt_nop,
 
 #ifdef CONFIG_PARAVIRT_XXL
-	.mmu.read_cr2		= __PV_IS_CALLEE_SAVE(native_read_cr2),
+	.mmu.read_cr2		= __PV_IS_CALLEE_SAVE(pv_native_read_cr2),
 	.mmu.write_cr2		= native_write_cr2,
 	.mmu.read_cr3		= __native_read_cr3,
 	.mmu.write_cr3		= native_write_cr3,
arch/x86/xen/xen-asm.S +2 -0
@@ -102,6 +102,7 @@ SYM_FUNC_START(check_events)
 	ret
 SYM_FUNC_END(check_events)
 
+.pushsection .noinstr.text, "ax"
 SYM_FUNC_START(xen_read_cr2)
 	FRAME_BEGIN
 	_ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
@@ -116,6 +117,7 @@ SYM_FUNC_START(xen_read_cr2_direct)
 	FRAME_END
 	ret
 SYM_FUNC_END(xen_read_cr2_direct);
+.popsection
 
 .macro xen_pv_trap name
 SYM_CODE_START(xen_\name)
author maintainer
#14 lore raw

RFC PATCH 9/18 x86/xen: Make write_cr2() noinstr

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: pv_ops[42]: native_write_cr2
vmlinux.o: warning: objtool: pv_ops[42]: xen_write_cr2
vmlinux.o: warning: objtool: exc_nmi()+0x127: call to pv_ops[42]() leaves .noinstr.text section

---
 arch/x86/include/asm/paravirt.h |    2 +-
 arch/x86/kernel/paravirt.c      |    7 ++++++-
 arch/x86/xen/mmu_pv.c           |    3 ++-
 3 files changed, 9 insertions(+), 3 deletions(-)
arch/x86/include/asm/paravirt.h +1 -1
@@ -140,7 +140,7 @@ static __always_inline unsigned long rea
 				ALT_NOT(X86_FEATURE_XENPV));
 }
 
-static inline void write_cr2(unsigned long x)
+static __always_inline void write_cr2(unsigned long x)
 {
 	PVOP_VCALL1(mmu.write_cr2, x);
 }
arch/x86/kernel/paravirt.c +6 -1
@@ -243,6 +243,11 @@ static noinstr unsigned long pv_native_r
 	return native_read_cr2();
 }
 
+static noinstr void pv_native_write_cr2(unsigned long val)
+{
+	native_write_cr2(val);
+}
+
 struct paravirt_patch_template pv_ops = {
 	/* Cpu ops. */
 	.cpu.io_delay		= native_io_delay,
@@ -304,7 +309,7 @@ struct paravirt_patch_template pv_ops =
 
 #ifdef CONFIG_PARAVIRT_XXL
 	.mmu.read_cr2		= __PV_IS_CALLEE_SAVE(pv_native_read_cr2),
-	.mmu.write_cr2		= native_write_cr2,
+	.mmu.write_cr2		= pv_native_write_cr2,
 	.mmu.read_cr3		= __native_read_cr3,
 	.mmu.write_cr3		= native_write_cr3,
 
arch/x86/xen/mmu_pv.c +2 -1
@@ -1204,7 +1204,8 @@ static void __init xen_pagetable_init(vo
 	xen_remap_memory();
 	xen_setup_mfn_list_list();
 }
-static void xen_write_cr2(unsigned long cr2)
+
+static noinstr void xen_write_cr2(unsigned long cr2)
 {
 	this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
 }
author maintainer
#15 lore raw

RFC PATCH 10/18 x86/xen: Make get_debugreg() noinstr

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: pv_ops[1]: xen_get_debugreg
vmlinux.o: warning: objtool: pv_ops[1]: native_get_debugreg
vmlinux.o: warning: objtool: exc_debug()+0x25: call to pv_ops[1]() leaves .noinstr.text section

---
 arch/x86/include/asm/paravirt.h      |    2 +-
 arch/x86/include/asm/xen/hypercall.h |    2 +-
 arch/x86/kernel/paravirt.c           |    8 ++++++--
 arch/x86/xen/enlighten_pv.c          |    2 +-
 4 files changed, 9 insertions(+), 5 deletions(-)
arch/x86/include/asm/paravirt.h +1 -1
@@ -113,7 +113,7 @@ static inline void __cpuid(unsigned int
 /*
  * These special macros can be used to get or set a debugging register
  */
-static inline unsigned long paravirt_get_debugreg(int reg)
+static __always_inline unsigned long paravirt_get_debugreg(int reg)
 {
 	return PVOP_CALL1(unsigned long, cpu.get_debugreg, reg);
 }
arch/x86/include/asm/xen/hypercall.h +1 -1
@@ -314,7 +314,7 @@ HYPERVISOR_set_debugreg(int reg, unsigne
 	return _hypercall2(int, set_debugreg, reg, value);
 }
 
-static inline unsigned long
+static __always_inline unsigned long
 HYPERVISOR_get_debugreg(int reg)
 {
 	return _hypercall1(unsigned long, get_debugreg, reg);
arch/x86/kernel/paravirt.c +6 -2
@@ -248,13 +248,18 @@ static noinstr void pv_native_write_cr2(
 	native_write_cr2(val);
 }
 
+static noinstr unsigned long pv_native_get_debugreg(int regno)
+{
+	return native_get_debugreg(regno);
+}
+
 struct paravirt_patch_template pv_ops = {
 	/* Cpu ops. */
 	.cpu.io_delay		= native_io_delay,
 
 #ifdef CONFIG_PARAVIRT_XXL
 	.cpu.cpuid		= native_cpuid,
-	.cpu.get_debugreg	= native_get_debugreg,
+	.cpu.get_debugreg	= pv_native_get_debugreg,
 	.cpu.set_debugreg	= native_set_debugreg,
 	.cpu.read_cr0		= native_read_cr0,
 	.cpu.write_cr0		= native_write_cr0,
@@ -382,7 +387,6 @@ struct paravirt_patch_template pv_ops =
 
 #ifdef CONFIG_PARAVIRT_XXL
 /* At this point, native_get/set_debugreg has real function entries */
-NOKPROBE_SYMBOL(native_get_debugreg);
 NOKPROBE_SYMBOL(native_set_debugreg);
 NOKPROBE_SYMBOL(native_load_idt);
 
arch/x86/xen/enlighten_pv.c +1 -1
@@ -317,7 +317,7 @@ static void xen_set_debugreg(int reg, un
 	HYPERVISOR_set_debugreg(reg, val);
 }
 
-static unsigned long xen_get_debugreg(int reg)
+static noinstr unsigned long xen_get_debugreg(int reg)
 {
 	return HYPERVISOR_get_debugreg(reg);
 }
author maintainer
#16 lore raw

RFC PATCH 11/18 x86/xen: Make set_debugreg() noinstr

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: pv_ops[2]: xen_set_debugreg
vmlinux.o: warning: objtool: pv_ops[2]: native_set_debugreg
vmlinux.o: warning: objtool: exc_debug()+0x3b: call to pv_ops[2]() leaves .noinstr.text section

---
 arch/x86/include/asm/paravirt.h      |    2 +-
 arch/x86/include/asm/xen/hypercall.h |    2 +-
 arch/x86/kernel/paravirt.c           |    9 ++++++---
 arch/x86/xen/enlighten_pv.c          |    2 +-
 4 files changed, 9 insertions(+), 6 deletions(-)
arch/x86/include/asm/paravirt.h +1 -1
@@ -118,7 +118,7 @@ static __always_inline unsigned long par
 	return PVOP_CALL1(unsigned long, cpu.get_debugreg, reg);
 }
 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
-static inline void set_debugreg(unsigned long val, int reg)
+static __always_inline void set_debugreg(unsigned long val, int reg)
 {
 	PVOP_VCALL2(cpu.set_debugreg, reg, val);
 }
arch/x86/include/asm/xen/hypercall.h +1 -1
@@ -308,7 +308,7 @@ HYPERVISOR_platform_op(struct xen_platfo
 	return _hypercall1(int, platform_op, op);
 }
 
-static inline int
+static __always_inline int
 HYPERVISOR_set_debugreg(int reg, unsigned long value)
 {
 	return _hypercall2(int, set_debugreg, reg, value);
arch/x86/kernel/paravirt.c +6 -3
@@ -253,6 +253,11 @@ static noinstr unsigned long pv_native_g
 	return native_get_debugreg(regno);
 }
 
+static noinstr void pv_native_set_debugreg(int regno, unsigned long val)
+{
+	native_set_debugreg(regno, val);
+}
+
 struct paravirt_patch_template pv_ops = {
 	/* Cpu ops. */
 	.cpu.io_delay		= native_io_delay,
@@ -260,7 +265,7 @@ struct paravirt_patch_template pv_ops =
 #ifdef CONFIG_PARAVIRT_XXL
 	.cpu.cpuid		= native_cpuid,
 	.cpu.get_debugreg	= pv_native_get_debugreg,
-	.cpu.set_debugreg	= native_set_debugreg,
+	.cpu.set_debugreg	= pv_native_set_debugreg,
 	.cpu.read_cr0		= native_read_cr0,
 	.cpu.write_cr0		= native_write_cr0,
 	.cpu.write_cr4		= native_write_cr4,
@@ -386,8 +391,6 @@ struct paravirt_patch_template pv_ops =
 };
 
 #ifdef CONFIG_PARAVIRT_XXL
-/* At this point, native_get/set_debugreg has real function entries */
-NOKPROBE_SYMBOL(native_set_debugreg);
 NOKPROBE_SYMBOL(native_load_idt);
 
 void (*paravirt_iret)(void) = native_iret;
arch/x86/xen/enlighten_pv.c +1 -1
@@ -312,7 +312,7 @@ static void __init xen_init_capabilities
 	}
 }
 
-static void xen_set_debugreg(int reg, unsigned long val)
+static noinstr void xen_set_debugreg(int reg, unsigned long val)
 {
 	HYPERVISOR_set_debugreg(reg, val);
 }
author maintainer
#17 lore raw

RFC PATCH 12/18 x86/xen: Make save_fl() noinstr

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: pv_ops[30]: native_save_fl
vmlinux.o: warning: objtool: pv_ops[30]: __raw_callee_save_xen_save_fl
vmlinux.o: warning: objtool: pv_ops[30]: xen_save_fl_direct
vmlinux.o: warning: objtool: lockdep_hardirqs_off()+0x73: call to pv_ops[30]() leaves .noinstr.text section

---
 arch/x86/include/asm/paravirt.h |    7 +++++--
 arch/x86/kernel/irqflags.S      |    2 ++
 arch/x86/xen/irq.c              |    4 ++--
 arch/x86/xen/xen-asm.S          |   32 ++++++++++++++++----------------
 4 files changed, 25 insertions(+), 20 deletions(-)
arch/x86/include/asm/paravirt.h +5 -2
@@ -653,10 +653,10 @@ bool __raw_callee_save___native_vcpu_is_
  * functions.
  */
 #define PV_THUNK_NAME(func) "__raw_callee_save_" #func
-#define PV_CALLEE_SAVE_REGS_THUNK(func)					\
+#define __PV_CALLEE_SAVE_REGS_THUNK(func, section)			\
 	extern typeof(func) __raw_callee_save_##func;			\
 									\
-	asm(".pushsection .text;"					\
+	asm(".pushsection " section ", \"ax\";"				\
 	    ".globl " PV_THUNK_NAME(func) ";"				\
 	    ".type " PV_THUNK_NAME(func) ", @function;"			\
 	    PV_THUNK_NAME(func) ":"					\
@@ -669,6 +669,9 @@ bool __raw_callee_save___native_vcpu_is_
 	    ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";"	\
 	    ".popsection")
 
+#define PV_CALLEE_SAVE_REGS_THUNK(func)			\
+	__PV_CALLEE_SAVE_REGS_THUNK(func, ".text")
+
 /* Get a reference to a callee-save function */
 #define PV_CALLEE_SAVE(func)						\
 	((struct paravirt_callee_save) { __raw_callee_save_##func })
arch/x86/kernel/irqflags.S +2 -0
@@ -7,9 +7,11 @@
 /*
  * unsigned long native_save_fl(void)
  */
+.pushsection .noinstr.text, "ax"
 SYM_FUNC_START(native_save_fl)
 	pushf
 	pop %_ASM_AX
 	ret
 SYM_FUNC_END(native_save_fl)
+.popsection
 EXPORT_SYMBOL(native_save_fl)
arch/x86/xen/irq.c +2 -2
@@ -24,7 +24,7 @@ void xen_force_evtchn_callback(void)
 	(void)HYPERVISOR_xen_version(0, NULL);
 }
 
-asmlinkage __visible unsigned long xen_save_fl(void)
+asmlinkage __visible noinstr unsigned long xen_save_fl(void)
 {
 	struct vcpu_info *vcpu;
 	unsigned long flags;
@@ -40,7 +40,7 @@ asmlinkage __visible unsigned long xen_s
 	*/
 	return (-flags) & X86_EFLAGS_IF;
 }
-PV_CALLEE_SAVE_REGS_THUNK(xen_save_fl);
+__PV_CALLEE_SAVE_REGS_THUNK(xen_save_fl, ".noinstr.text");
 
 asmlinkage __visible void xen_irq_disable(void)
 {
arch/x86/xen/xen-asm.S +16 -16
@@ -58,22 +58,6 @@ SYM_FUNC_START(xen_irq_disable_direct)
 SYM_FUNC_END(xen_irq_disable_direct)
 
 /*
- * (xen_)save_fl is used to get the current interrupt enable status.
- * Callers expect the status to be in X86_EFLAGS_IF, and other bits
- * may be set in the return value.  We take advantage of this by
- * making sure that X86_EFLAGS_IF has the right value (and other bits
- * in that byte are 0), but other bits in the return value are
- * undefined.  We need to toggle the state of the bit, because Xen and
- * x86 use opposite senses (mask vs enable).
- */
-SYM_FUNC_START(xen_save_fl_direct)
-	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
-	setz %ah
-	addb %ah, %ah
-	ret
-SYM_FUNC_END(xen_save_fl_direct)
-
-/*
  * Force an event check by making a hypercall, but preserve regs
  * before making the call.
  */
@@ -103,6 +87,22 @@ SYM_FUNC_START(check_events)
 SYM_FUNC_END(check_events)
 
 .pushsection .noinstr.text, "ax"
+/*
+ * (xen_)save_fl is used to get the current interrupt enable status.
+ * Callers expect the status to be in X86_EFLAGS_IF, and other bits
+ * may be set in the return value.  We take advantage of this by
+ * making sure that X86_EFLAGS_IF has the right value (and other bits
+ * in that byte are 0), but other bits in the return value are
+ * undefined.  We need to toggle the state of the bit, because Xen and
+ * x86 use opposite senses (mask vs enable).
+ */
+SYM_FUNC_START(xen_save_fl_direct)
+	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
+	setz %ah
+	addb %ah, %ah
+	ret
+SYM_FUNC_END(xen_save_fl_direct)
+
 SYM_FUNC_START(xen_read_cr2)
 	FRAME_BEGIN
 	_ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
author maintainer
#18 lore raw

RFC PATCH 13/18 x86/xen: Make hypercall_page noinstr

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: xen_set_debugreg()+0x3: call to hypercall_page() leaves .noinstr.text section vmlinux.o: warning: objtool: xen_get_debugreg()+0x3: call to hypercall_page() leaves .noinstr.text section vmlinux.o: warning: objtool: xen_irq_enable()+0x24: call to hypercall_page() leaves .noinstr.text section

---
 arch/x86/xen/xen-head.S |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/xen/xen-head.S +1 -1
@@ -64,7 +64,7 @@ SYM_CODE_END(asm_cpu_bringup_and_idle)
 #endif
 #endif
 
-.pushsection .text
+.pushsection .noinstr.text, "ax"
 	.balign PAGE_SIZE
 SYM_CODE_START(hypercall_page)
 	.rept (PAGE_SIZE / 32)
author maintainer
#19 lore raw

RFC PATCH 14/18 x86/xen: Make irq_enable() noinstr

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: pv_ops[32]: native_irq_enable
vmlinux.o: warning: objtool: pv_ops[32]: __raw_callee_save_xen_irq_enable
vmlinux.o: warning: objtool: pv_ops[32]: xen_irq_enable_direct
vmlinux.o: warning: objtool: lock_is_held_type()+0xfe: call to pv_ops[32]() leaves .noinstr.text section

---
 arch/x86/kernel/paravirt.c |    7 ++++-
 arch/x86/xen/irq.c         |    4 +--
 arch/x86/xen/xen-asm.S     |   56 ++++++++++++++++++++++-----------------------
 3 files changed, 36 insertions(+), 31 deletions(-)
arch/x86/kernel/paravirt.c +6 -1
@@ -258,6 +258,11 @@ static noinstr void pv_native_set_debugr
 	native_set_debugreg(regno, val);
 }
 
+static noinstr void pv_native_irq_enable(void)
+{
+	native_irq_enable();
+}
+
 struct paravirt_patch_template pv_ops = {
 	/* Cpu ops. */
 	.cpu.io_delay		= native_io_delay,
@@ -302,7 +307,7 @@ struct paravirt_patch_template pv_ops =
 	/* Irq ops. */
 	.irq.save_fl		= __PV_IS_CALLEE_SAVE(native_save_fl),
 	.irq.irq_disable	= __PV_IS_CALLEE_SAVE(native_irq_disable),
-	.irq.irq_enable		= __PV_IS_CALLEE_SAVE(native_irq_enable),
+	.irq.irq_enable		= __PV_IS_CALLEE_SAVE(pv_native_irq_enable),
 	.irq.safe_halt		= native_safe_halt,
 	.irq.halt		= native_halt,
 #endif /* CONFIG_PARAVIRT_XXL */
arch/x86/xen/irq.c +2 -2
@@ -53,7 +53,7 @@ asmlinkage __visible void xen_irq_disabl
 }
 PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
 
-asmlinkage __visible void xen_irq_enable(void)
+asmlinkage __visible noinstr void xen_irq_enable(void)
 {
 	struct vcpu_info *vcpu;
 
@@ -76,7 +76,7 @@ asmlinkage __visible void xen_irq_enable
 
 	preempt_enable();
 }
-PV_CALLEE_SAVE_REGS_THUNK(xen_irq_enable);
+__PV_CALLEE_SAVE_REGS_THUNK(xen_irq_enable, ".noinstr.text");
 
 static void xen_safe_halt(void)
 {
arch/x86/xen/xen-asm.S +28 -28

74 lines not on this page · raw message

author maintainer
#20 lore raw

RFC PATCH 15/18 x86/xen: Make irq_disable() noinstr

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: pv_ops[31]: native_irq_disable
vmlinux.o: warning: objtool: pv_ops[31]: __raw_callee_save_xen_irq_disable
vmlinux.o: warning: objtool: pv_ops[31]: xen_irq_disable_direct
vmlinux.o: warning: objtool: lock_is_held_type()+0x5b: call to pv_ops[31]() leaves .noinstr.text section

---
 arch/x86/kernel/paravirt.c |    7 ++++++-
 arch/x86/xen/irq.c         |    4 ++--
 arch/x86/xen/xen-asm.S     |    3 +--
 3 files changed, 9 insertions(+), 5 deletions(-)
arch/x86/kernel/paravirt.c +6 -1
@@ -263,6 +263,11 @@ static noinstr void pv_native_irq_enable
 	native_irq_enable();
 }
 
+static noinstr void pv_native_irq_disable(void)
+{
+	native_irq_disable();
+}
+
 struct paravirt_patch_template pv_ops = {
 	/* Cpu ops. */
 	.cpu.io_delay		= native_io_delay,
@@ -306,7 +311,7 @@ struct paravirt_patch_template pv_ops =
 
 	/* Irq ops. */
 	.irq.save_fl		= __PV_IS_CALLEE_SAVE(native_save_fl),
-	.irq.irq_disable	= __PV_IS_CALLEE_SAVE(native_irq_disable),
+	.irq.irq_disable	= __PV_IS_CALLEE_SAVE(pv_native_irq_disable),
 	.irq.irq_enable		= __PV_IS_CALLEE_SAVE(pv_native_irq_enable),
 	.irq.safe_halt		= native_safe_halt,
 	.irq.halt		= native_halt,
arch/x86/xen/irq.c +2 -2
@@ -42,7 +42,7 @@ asmlinkage __visible noinstr unsigned lo
 }
 __PV_CALLEE_SAVE_REGS_THUNK(xen_save_fl, ".noinstr.text");
 
-asmlinkage __visible void xen_irq_disable(void)
+asmlinkage __visible noinstr void xen_irq_disable(void)
 {
 	/* There's a one instruction preempt window here.  We need to
 	   make sure we're don't switch CPUs between getting the vcpu
@@ -51,7 +51,7 @@ asmlinkage __visible void xen_irq_disabl
 	this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
 	preempt_enable_no_resched();
 }
-PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
+__PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable, ".noinstr.text");
 
 asmlinkage __visible noinstr void xen_irq_enable(void)
 {
arch/x86/xen/xen-asm.S +1 -2
@@ -21,6 +21,7 @@
 #include <linux/init.h>
 #include <linux/linkage.h>
 
+.pushsection .noinstr.text, "ax"
 /*
  * Disabling events is simply a matter of making the event mask
  * non-zero.
@@ -30,8 +31,6 @@ SYM_FUNC_START(xen_irq_disable_direct)
 	ret
 SYM_FUNC_END(xen_irq_disable_direct)
 
-.pushsection .noinstr.text, "ax"
-
 /*
  * Force an event check by making a hypercall, but preserve regs
  * before making the call.
author maintainer
#21 lore raw

RFC PATCH 16/18 x86/xen: Mark xen_force_evtchn_callback() noinstr

In reply to Peter Zijlstra

vmlinux.o: warning: objtool: check_events()+0xd: call to xen_force_evtchn_callback() leaves .noinstr.text section

---
 arch/x86/include/asm/xen/hypercall.h |    2 +-
 arch/x86/xen/irq.c                   |    2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
arch/x86/include/asm/xen/hypercall.h +1 -1
@@ -358,7 +358,7 @@ HYPERVISOR_event_channel_op(int cmd, voi
 	return _hypercall2(int, event_channel_op, cmd, arg);
 }
 
-static inline int
+static __always_inline int
 HYPERVISOR_xen_version(int cmd, void *arg)
 {
 	return _hypercall2(int, xen_version, cmd, arg);
arch/x86/xen/irq.c +1 -1
@@ -19,7 +19,7 @@
  * callback mask. We do this in a very simple manner, by making a call
  * down into Xen. The pending flag will be checked by Xen on return.
  */
-void xen_force_evtchn_callback(void)
+noinstr void xen_force_evtchn_callback(void)
 {
 	(void)HYPERVISOR_xen_version(0, NULL);
 }
author maintainer
#22 lore raw

RFC PATCH 17/18 x86/xen: Rework the xen_{cpu,irq,mmu}_ops[] arrays

In reply to Peter Zijlstra

In order to allow objtool to make sense of all the various paravirt functions, it needs to either parse whole pv_ops[] tables, or observe individual assignments in the form:

bf87: 48 c7 05 00 00 00 00 00 00 00 00 movq $0x0,0x0(%rip)

    bf92 <xen_init_spinlocks+0x5f>
    bf8a: R_X86_64_PC32     pv_ops+0x268

As is, xen_cpu_ops[] is at offset +0 in pv_ops[] and could thus be parsed as a 'normal' pv_ops[] table, however xen_irq_ops[] and xen_mmu_ops[] are not.

Worse, both the latter two are compiled into the individual assignment for by current GCC, but that's not something one can rely on.

Therefore, convert all three into full pv_ops[] tables. This has the benefit of not needing to teach objtool about the offsets and resulting in more conservative code-gen.

---
 arch/x86/xen/enlighten_pv.c |   66 ++++++++++++++++---------------
 arch/x86/xen/irq.c          |   17 ++++----
 arch/x86/xen/mmu_pv.c       |   92 ++++++++++++++++++++++----------------------
 3 files changed, 91 insertions(+), 84 deletions(-)
arch/x86/xen/enlighten_pv.c +34 -32
@@ -1051,52 +1051,54 @@ static const struct pv_info xen_info __i
 	.name = "Xen",
 };
 
-static const struct pv_cpu_ops xen_cpu_ops __initconst = {
-	.cpuid = xen_cpuid,
+static const typeof(pv_ops) xen_cpu_ops __initconst = {
+	.cpu = {
+		.cpuid = xen_cpuid,
 
-	.set_debugreg = xen_set_debugreg,
-	.get_debugreg = xen_get_debugreg,
+		.set_debugreg = xen_set_debugreg,
+		.get_debugreg = xen_get_debugreg,
 
-	.read_cr0 = xen_read_cr0,
-	.write_cr0 = xen_write_cr0,
+		.read_cr0 = xen_read_cr0,
+		.write_cr0 = xen_write_cr0,
 
-	.write_cr4 = xen_write_cr4,
+		.write_cr4 = xen_write_cr4,
 
-	.wbinvd = native_wbinvd,
+		.wbinvd = native_wbinvd,
 
-	.read_msr = xen_read_msr,
-	.write_msr = xen_write_msr,
+		.read_msr = xen_read_msr,
+		.write_msr = xen_write_msr,
 
-	.read_msr_safe = xen_read_msr_safe,
-	.write_msr_safe = xen_write_msr_safe,
+		.read_msr_safe = xen_read_msr_safe,
+		.write_msr_safe = xen_write_msr_safe,
 
-	.read_pmc = xen_read_pmc,
+		.read_pmc = xen_read_pmc,
 
-	.load_tr_desc = paravirt_nop,
-	.set_ldt = xen_set_ldt,
-	.load_gdt = xen_load_gdt,
-	.load_idt = xen_load_idt,
-	.load_tls = xen_load_tls,
-	.load_gs_index = xen_load_gs_index,
+		.load_tr_desc = paravirt_nop,
+		.set_ldt = xen_set_ldt,
+		.load_gdt = xen_load_gdt,
+		.load_idt = xen_load_idt,
+		.load_tls = xen_load_tls,
+		.load_gs_index = xen_load_gs_index,
 
-	.alloc_ldt = xen_alloc_ldt,
-	.free_ldt = xen_free_ldt,
+		.alloc_ldt = xen_alloc_ldt,
+		.free_ldt = xen_free_ldt,
 
-	.store_tr = xen_store_tr,
+		.store_tr = xen_store_tr,
 
-	.write_ldt_entry = xen_write_ldt_entry,
-	.write_gdt_entry = xen_write_gdt_entry,
-	.write_idt_entry = xen_write_idt_entry,
-	.load_sp0 = xen_load_sp0,
+		.write_ldt_entry = xen_write_ldt_entry,
+		.write_gdt_entry = xen_write_gdt_entry,
+		.write_idt_entry = xen_write_idt_entry,
+		.load_sp0 = xen_load_sp0,
 
 #ifdef CONFIG_X86_IOPL_IOPERM
-	.invalidate_io_bitmap = xen_invalidate_io_bitmap,
-	.update_io_bitmap = xen_update_io_bitmap,
+		.invalidate_io_bitmap = xen_invalidate_io_bitmap,
+		.update_io_bitmap = xen_update_io_bitmap,
 #endif
-	.io_delay = xen_io_delay,
+		.io_delay = xen_io_delay,
 
-	.start_context_switch = paravirt_start_context_switch,
-	.end_context_switch = xen_end_context_switch,
+		.start_context_switch = paravirt_start_context_switch,
+		.end_context_switch = xen_end_context_switch,
+	},
 };
 
 static void xen_restart(char *msg)
@@ -1232,7 +1234,7 @@ asmlinkage __visible void __init xen_sta
 
 	/* Install Xen paravirt ops */
 	pv_info = xen_info;
-	pv_ops.cpu = xen_cpu_ops;
+	pv_ops.cpu = xen_cpu_ops.cpu;
 	paravirt_iret = xen_iret;
 	xen_init_irq_ops();
 
arch/x86/xen/irq.c +10 -7
@@ -94,17 +94,20 @@ static void xen_halt(void)
 		xen_safe_halt();
 }
 
-static const struct pv_irq_ops xen_irq_ops __initconst = {
-	.save_fl = PV_CALLEE_SAVE(xen_save_fl),
-	.irq_disable = PV_CALLEE_SAVE(xen_irq_disable),
-	.irq_enable = PV_CALLEE_SAVE(xen_irq_enable),
+static const typeof(pv_ops) xen_irq_ops __initconst = {
+	.irq = {
 
-	.safe_halt = xen_safe_halt,
-	.halt = xen_halt,
+		.save_fl = PV_CALLEE_SAVE(xen_save_fl),
+		.irq_disable = PV_CALLEE_SAVE(xen_irq_disable),
+		.irq_enable = PV_CALLEE_SAVE(xen_irq_enable),
+
+		.safe_halt = xen_safe_halt,
+		.halt = xen_halt,
+	},
 };
 
 void __init xen_init_irq_ops(void)
 {
-	pv_ops.irq = xen_irq_ops;
+	pv_ops.irq = xen_irq_ops.irq;
 	x86_init.irqs.intr_init = xen_init_IRQ;
 }
arch/x86/xen/mmu_pv.c +47 -45

121 lines not on this page · raw message

author maintainer
#23 lore raw

RFC PATCH 18/18 objtool: Support pv_opsindirect calls for noinstr

In reply to Peter Zijlstra

Normally objtool will now follow indirect calls; there is no need.

However, this becomes a problem with noinstr validation; if there's an indirect call from noinstr code, we very much need to know it is to another noinstr function. Luckily there aren't many indirect calls in entry code with the obvious exception of paravirt. As such, noinstr validation didn't work with paravirt kernels.

In order to track pv_ops[] call targets, objtool reads the static pv_ops[] tables as well as direct assignments to the pv_ops[] array, provided the compiler makes them a single instruction like:

bf87: 48 c7 05 00 00 00 00 00 00 00 00 movq $0x0,0x0(%rip)

    bf92 <xen_init_spinlocks+0x5f>
    bf8a: R_X86_64_PC32     pv_ops+0x268

There are, as of yet, no warnings for when this goes wrong :/

Using the functions found with the above means, all pv_ops[] calls are now subject to noinstr validation.

---
 lib/Kconfig.debug                       |    2 
 tools/objtool/arch/x86/decode.c         |   34 ++++++-
 tools/objtool/check.c                   |  151 +++++++++++++++++++++++++++++---
 tools/objtool/include/objtool/arch.h    |    2 
 tools/objtool/include/objtool/elf.h     |    1 
 tools/objtool/include/objtool/objtool.h |    9 +
 tools/objtool/objtool.c                 |   22 ++++
 7 files changed, 208 insertions(+), 13 deletions(-)
lib/Kconfig.debug +1 -1
@@ -445,7 +445,7 @@ config STACK_VALIDATION
 
 config VMLINUX_VALIDATION
 	bool
-	depends on STACK_VALIDATION && DEBUG_ENTRY && !PARAVIRT
+	depends on STACK_VALIDATION && DEBUG_ENTRY
 	default y
 
 config VMLINUX_MAP
tools/objtool/arch/x86/decode.c +33 -1
@@ -20,6 +20,7 @@
 #include <objtool/arch.h>
 #include <objtool/warn.h>
 #include <objtool/endianness.h>
+#include <objtool/builtin.h>
 #include <arch/elf.h>
 
 static int is_x86_64(const struct elf *elf)
@@ -102,12 +103,13 @@ unsigned long arch_jump_destination(stru
 #define rm_is_mem(reg)	(mod_is_mem() && !is_RIP() && rm_is(reg))
 #define rm_is_reg(reg)	(mod_is_reg() && modrm_rm == (reg))
 
-int arch_decode_instruction(const struct elf *elf, const struct section *sec,
+int arch_decode_instruction(struct objtool_file *file, const struct section *sec,
 			    unsigned long offset, unsigned int maxlen,
 			    unsigned int *len, enum insn_type *type,
 			    unsigned long *immediate,
 			    struct list_head *ops_list)
 {
+	const struct elf *elf = file->elf;
 	struct insn insn;
 	int x86_64, ret;
 	unsigned char op1, op2,
@@ -544,6 +546,36 @@ int arch_decode_instruction(const struct
 		*type = INSN_RETURN;
 		break;
 
+	case 0xc7: /* mov imm, r/m */
+		if (!noinstr)
+			break;
+
+		if (insn.length == 3+4+4 && !strncmp(sec->name, ".init.text", 10)) {
+			struct reloc *immr, *disp;
+			struct symbol *func;
+			int idx;
+
+			immr = find_reloc_by_dest(elf, (void *)sec, offset+3);
+			disp = find_reloc_by_dest(elf, (void *)sec, offset+7);
+
+			if (!immr || strcmp(immr->sym->name, "pv_ops"))
+				break;
+
+			idx = (immr->addend + 8) / sizeof(void *);
+
+			func = disp->sym;
+			if (disp->sym->type == STT_SECTION)
+				func = find_symbol_by_offset(disp->sym->sec, disp->addend);
+			if (!func) {
+				WARN("no func for pv_ops[]");
+				return -1;
+			}
+
+			objtool_pv_add(file, idx, func);
+		}
+
+		break;
+
 	case 0xcf: /* iret */
 		/*
 		 * Handle sync_core(), which has an IRET to self.
tools/objtool/check.c +141 -10

231 lines not on this page · raw message

tools/objtool/include/objtool/arch.h +1 -1
@@ -69,7 +69,7 @@ struct instruction;
 
 void arch_initial_func_cfi_state(struct cfi_init_state *state);
 
-int arch_decode_instruction(const struct elf *elf, const struct section *sec,
+int arch_decode_instruction(struct objtool_file *file, const struct section *sec,
 			    unsigned long offset, unsigned int maxlen,
 			    unsigned int *len, enum insn_type *type,
 			    unsigned long *immediate,
tools/objtool/include/objtool/elf.h +1 -0
@@ -57,6 +57,7 @@ struct symbol {
 	struct symbol *pfunc, *cfunc, *alias;
 	bool uaccess_safe;
 	bool static_call_tramp;
+	struct list_head pv_target;
 };
 
 struct reloc {
tools/objtool/include/objtool/objtool.h +9 -0
@@ -14,6 +14,11 @@
 
 #define __weak __attribute__((weak))
 
+struct pv_state {
+	bool clean;
+	struct list_head targets;
+};
+
 struct objtool_file {
 	struct elf *elf;
 	struct list_head insn_list;
@@ -25,10 +30,14 @@ struct objtool_file {
 
 	unsigned long jl_short, jl_long;
 	unsigned long jl_nop_short, jl_nop_long;
+
+	struct pv_state *pv_ops;
 };
 
 struct objtool_file *objtool_open_read(const char *_objname);
 
+void objtool_pv_add(struct objtool_file *file, int idx, struct symbol *func);
+
 int check(struct objtool_file *file);
 int orc_dump(const char *objname);
 int orc_create(struct objtool_file *file);
tools/objtool/objtool.c +22 -0
@@ -135,6 +135,28 @@ struct objtool_file *objtool_open_read(c
 	return &file;
 }
 
+void objtool_pv_add(struct objtool_file *f, int idx, struct symbol *func)
+{
+	if (!noinstr)
+		return;
+
+	if (!f->pv_ops) {
+		WARN("paravirt confusion");
+		return;
+	}
+
+	/*
+	 * These functions will be patched into native code,
+	 * see paravirt_patch().
+	 */
+	if (!strcmp(func->name, "_paravirt_nop") ||
+	    !strcmp(func->name, "_paravirt_ident_64"))
+		return;
+
+	list_add(&func->pv_target, &f->pv_ops[idx].targets);
+	f->pv_ops[idx].clean = false;
+}
+
 static void cmd_usage(void)
 {
 	unsigned int i, longest = 0;
Reviewed-by on the v1 series
#24 lore raw
In reply to Peter Zijlstra
Quoted 9 lines · Peter Zijlstra

Hi!

Per long standing request from Thomas, this here mostly sorts the whole noinstr validation vs PARAVIRT.

It isn't exactly pretty or robust, but it does work.

So far only tested on x86_64-defconfig+kvm_guest.config+xen.config. I did a quick allyesconfig build, but still saw some funnies with that, will poke at them.

Didn't see anything weird in these patches.

Thanks,

Ingo