Skip to contentSkip to conversation

x86/msr: add idle version of wrmsr trace

PATCH In review FirstName LastName 2 participants 3 messages Aug 23, 2022 – Aug 24, 2022

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 x86/msr: add idle version of wrmsr trace

From: Peter Shier <[email protected]>

With commit bf5835bcdb963 ("intel_idle: Disable IBRS during long idle"), enabling wrmsr trace with CONFIG_LOCKDEP causes "suspicious rcu_dereference_check() usage" warning because do_trace_write_msr does not use trace_write_msr_rcuidle.

Add idle versions of needed routines and change intel_idle_ibrs to use it.

Sample warning:
============================
WARNING: suspicious RCU usage
6.0.0-dbg-DEV #7 Tainted: G S O
-----------------------------
arch/x86/include/asm/msr-trace.h:48 suspicious rcu_dereference_check() usage!

other info that might help us debug this:

rcu_scheduler_active = 2, debug_locks = 1
RCU used illegally from extended quiescent state!
no locks held by swapper/59/0.

stack backtrace:
CPU: 59 PID: 0 Comm: swapper/59 Tainted: G S         O       6.0.0-dbg-DEV #7
Call Trace:
 dump_stack_lvl
 dump_stack
 lockdep_rcu_suspicious
 trace_write_msr
 do_trace_write_msr
 intel_idle_ibrs
 cpuidle_enter_state
 cpuidle_enter
 do_idle
 cpu_startup_entry
 start_secondary
 secondary_startup_64_no_verify

Tested on skylake using:
echo "msr:write_msr" >/sys/kernel/debug/tracing/set_event with and without patch.

---
 arch/x86/include/asm/msr.h | 16 ++++++++++++++++
 arch/x86/lib/msr.c         |  6 ++++++
 drivers/idle/intel_idle.c  |  4 ++--
 3 files changed, 24 insertions(+), 2 deletions(-)
arch/x86/include/asm/msr.h +16 -0
index 65ec1965cd28..248cc58b7758 100644
@@ -62,10 +62,12 @@ DECLARE_TRACEPOINT(read_msr);
 DECLARE_TRACEPOINT(write_msr);
 DECLARE_TRACEPOINT(rdpmc);
 extern void do_trace_write_msr(unsigned int msr, u64 val, int failed);
+extern void do_trace_write_msr_idle(unsigned int msr, u64 val, int failed);
 extern void do_trace_read_msr(unsigned int msr, u64 val, int failed);
 extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed);
 #else
 static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {}
+extern void do_trace_write_msr_idle(unsigned int msr, u64 val, int failed) {}
 static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {}
 static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {}
 #endif
@@ -148,6 +150,15 @@ native_write_msr(unsigned int msr, u32 low, u32 high)
 		do_trace_write_msr(msr, ((u64)high << 32 | low), 0);
 }
 
+static inline void notrace
+native_write_msr_idle(unsigned int msr, u32 low, u32 high)
+{
+	__wrmsr(msr, low, high);
+
+	if (tracepoint_enabled(write_msr))
+		do_trace_write_msr_idle(msr, ((u64)high << 32 | low), 0);
+}
+
 /* Can be uninlined because referenced by paravirt */
 static inline int notrace
 native_write_msr_safe(unsigned int msr, u32 low, u32 high)
@@ -262,6 +273,11 @@ static inline void wrmsrl(unsigned int msr, u64 val)
 	native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32));
 }
 
+static inline void wrmsrl_idle(unsigned int msr, u64 val)
+{
+	native_write_msr_idle(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32));
+}
+
 /* wrmsr with exception handling */
 static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high)
 {
arch/x86/lib/msr.c +6 -0
index b09cd2ad426c..58fdf0f13850 100644
@@ -121,6 +121,12 @@ void do_trace_write_msr(unsigned int msr, u64 val, int failed)
 EXPORT_SYMBOL(do_trace_write_msr);
 EXPORT_TRACEPOINT_SYMBOL(write_msr);
 
+void do_trace_write_msr_idle(unsigned int msr, u64 val, int failed)
+{
+	trace_write_msr_rcuidle(msr, val, failed);
+}
+EXPORT_SYMBOL(do_trace_write_msr_idle);
+
 void do_trace_read_msr(unsigned int msr, u64 val, int failed)
 {
 	trace_read_msr(msr, val, failed);
drivers/idle/intel_idle.c +2 -2
index 3e101719689a..bdecd2638c59 100644
@@ -187,12 +187,12 @@ static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
 	int ret;
 
 	if (smt_active)
-		wrmsrl(MSR_IA32_SPEC_CTRL, 0);
+		wrmsrl_idle(MSR_IA32_SPEC_CTRL, 0);
 
 	ret = __intel_idle(dev, drv, index);
 
 	if (smt_active)
-		wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl);
+		wrmsrl_idle(MSR_IA32_SPEC_CTRL, spec_ctrl);
 
 	return ret;
 }
#1 lore raw

Dear FirstName LastName, pleae fix your MUA.

Quoted 7 lines · FirstName LastName

From: Peter Shier <[email protected]>

With commit bf5835bcdb963 ("intel_idle: Disable IBRS during long idle"), enabling wrmsr trace with CONFIG_LOCKDEP causes "suspicious rcu_dereference_check() usage" warning because do_trace_write_msr does not use trace_write_msr_rcuidle.

No, the right thing here is to not do tracing at all.

*sigh* I should go finish this series:

https://lore.kernel.org/lkml/[email protected]/

---

drivers/idle/intel_idle.c +2 -2
index 3e101719689a..6e82b2df29cb 100644
@@ -187,12 +187,12 @@ static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
 	int ret;
 
 	if (smt_active)
-		wrmsrl(MSR_IA32_SPEC_CTRL, 0);
+		__native_wrmsr(MSR_IA32_SPEC_CTRL, 0);
 
 	ret = __intel_idle(dev, drv, index);
 
 	if (smt_active)
-		wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl);
+		__native_wrmsr(MSR_IA32_SPEC_CTRL, spec_ctrl);
 
 	return ret;
 }
#2 lore raw
Quoted 9 lines · Peter Zijlstra
drivers/idle/intel_idle.c +0 -1
index 3e101719689a..6e82b2df29cb 100644
@@ -187,12 +187,12 @@ static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
 	int ret;
 
 	if (smt_active)
-		wrmsrl(MSR_IA32_SPEC_CTRL, 0);

drivers/idle/intel_idle.c static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev, · line 198

Quoted 10 lines
drivers/idle/intel_idle.c +2 -1
+		__native_wrmsr(MSR_IA32_SPEC_CTRL, 0);
 
 	ret = __intel_idle(dev, drv, index);
 
 	if (smt_active)
-		wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl);
+		__native_wrmsr(MSR_IA32_SPEC_CTRL, spec_ctrl);
 
 	return ret;
 }

Clearly I should not be sending email before having wake-up-juice.. that should of course be native_wrmsr(SPEC_CTRL, val, 0) etc..