x86/mtrr: Allow MTRR updates on multiple CPUs in parallel
cover x86/mtrr: Allow MTRR updates on multiple CPUs in parallel
Today MTRR updates are serialized to not happen on multiple CPUs at the same time, as the related coding is using global variables.
On huge machines with lots of CPUs this can result in problems, as such updates are happening through stop_machine(), which will call the MTRR update function with interrupts off on all CPUs at the same time. The interrupts will be switched on only after the last CPU has finished the MTRR update. As the update is required to run in uncached mode, it can take easily several milliseconds on each CPU, resulting in the whole process to need several seconds. This in turn can cause the watchdog to trigger and to recognize a hard system lockup.
This series is changing the behavior by allowing the MTRR update to happen on all CPUs in parallel.
Juergen Gross (4):
x86/mtrr: Move cache_enable() and cache_disable() to mtrr/generic.c x86/mtrr: Introduce MTRR work state structure
x86/mtrr: Add a prepare_set hook to mtrr_ops
x86/mtrr: Drop cache_disable_lock
arch/x86/include/asm/cacheinfo.h | 2 - arch/x86/include/asm/mtrr.h | 2 - arch/x86/kernel/cpu/cacheinfo.c | 80 +---------------- arch/x86/kernel/cpu/mtrr/generic.c | 137 ++++++++++++++++++++++++----- arch/x86/kernel/cpu/mtrr/mtrr.c | 3 + arch/x86/kernel/cpu/mtrr/mtrr.h | 2 + 6 files changed, 120 insertions(+), 106 deletions(-)
Signature · 1 line
2.52.0
PATCH 1/4 x86/mtrr: Move cache_enable() and cache_disable() to mtrr/generic.c
cache_enable() and cache_disable() are used for generic MTRR code only. Move them and related stuff to mtrr/generic.c, allowing to make them static. This requires to move the cache_enable() and cache_disable() calls from cache_cpu_init() into mtrr_generic_set_state().
This allows to make mtrr_enable() and mtrr_disable() static, too.
While moving the code, drop the comment related to the PAT MSR, as this is not true anymore.
No change of functionality.
--- arch/x86/include/asm/cacheinfo.h | 2 - arch/x86/include/asm/mtrr.h | 2 - arch/x86/kernel/cpu/cacheinfo.c | 80 +--------------------------- arch/x86/kernel/cpu/mtrr/generic.c | 83 +++++++++++++++++++++++++++++- 4 files changed, 82 insertions(+), 85 deletions(-)
arch/x86/include/asm/cacheinfo.h +0 -2
@@ -7,8 +7,6 @@ extern unsigned int memory_caching_control;#define CACHE_MTRR 0x01#define CACHE_PAT 0x02 -void cache_disable (void ); -void cache_enable (void );void set_cache_aps_delayed_init (bool val);bool get_cache_aps_delayed_init (void );void cache_bp_init (void );
arch/x86/include/asm/mtrr.h +0 -2
@@ -58,8 +58,6 @@ extern int mtrr_del(int reg, unsigned long base, unsigned long size);extern int mtrr_del_page (int reg,unsigned long base,unsigned long size);extern int mtrr_trim_uncached_memory (unsigned long end_pfn);extern int amd_special_default_mtrr (void ); -void mtrr_disable (void ); -void mtrr_enable (void );void mtrr_generic_set_state (void );# else static inline void guest_force_mtrr_state(struct mtrr_var_range *var,
arch/x86/kernel/cpu/cacheinfo.c +1 -79
93 lines not on this page · raw message
arch/x86/kernel/cpu/mtrr/generic.c +81 -2
110 lines not on this page · raw message
Signature · 1 line
2.52.0
PATCH 2/4 x86/mtrr: Introduce MTRR work state structure
In reply to Juergen GrossInstead of using static variables for storing local state across cache_disable() ... cache_enable(), use a structure allocated on the stack for the same purpose.
--- arch/x86/kernel/cpu/mtrr/generic.c | 58 ++++++++++++++++-------------- 1 file changed, 31 insertions(+), 27 deletions(-)
arch/x86/kernel/cpu/mtrr/generic.c +31 -27
@@ -905,18 +905,19 @@ static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)return changed; } -static u32 deftype_lo, deftype_hi; +struct mtrr_work_state { +unsigned long cr4; +u32 lo; +u32 hi; +};/** * set_mtrr_state - Set the MTRR state for this CPU. * -* NOTE: The CPU must already be in a safe state for MTRR changes, including -* measures that only a single CPU can be active in set_mtrr_state() in -* order to not be subject to races for usage of deftype_lo. This is -* accomplished by taking cache_disable_lock. +* NOTE: The CPU must already be in a safe state for MTRR changes. * RETURNS: 0 if no changes made, else a mask indicating what was changed. */ -static unsigned long set_mtrr_state (void ) +static unsigned long set_mtrr_state (struct mtrr_work_state *state) {unsigned long change_mask =0 ;unsigned int i;
@@ -933,10 +934,10 @@ static unsigned long set_mtrr_state(void)* Set_mtrr_restore restores the old value of MTRRdefType, * so to set it we fiddle with the saved value: */ -if ((deftype_lo &MTRR_DEF_TYPE_TYPE ) != mtrr_state.def_type || - ((deftype_lo &MTRR_DEF_TYPE_ENABLE ) >>MTRR_STATE_SHIFT ) != mtrr_state.enabled) { +if ((state->lo &MTRR_DEF_TYPE_TYPE ) != mtrr_state.def_type || + ((state->lo &MTRR_DEF_TYPE_ENABLE ) >>MTRR_STATE_SHIFT ) != mtrr_state.enabled) { - deftype_lo = (deftype_lo &MTRR_DEF_TYPE_DISABLE ) | + state->lo = (state->lo &MTRR_DEF_TYPE_DISABLE ) | mtrr_state.def_type | (mtrr_state.enabled <<MTRR_STATE_SHIFT ); change_mask |=MTRR_CHANGE_MASK_DEFTYPE ;
@@ -945,19 +946,19 @@ static unsigned long set_mtrr_state(void)return change_mask; } -static void mtrr_disable (void ) +static void mtrr_disable (struct mtrr_work_state *state) {/* Save MTRR state */ -rdmsr (MSR_MTRRdefType, deftype_lo, deftype_hi); +rdmsr (MSR_MTRRdefType, state->lo, state->hi);/* Disable MTRRs, and set the default type to uncached */ -mtrr_wrmsr (MSR_MTRRdefType, deftype_lo &MTRR_DEF_TYPE_DISABLE , deftype_hi); +mtrr_wrmsr (MSR_MTRRdefType, state->lo &MTRR_DEF_TYPE_DISABLE , state->hi); } -static void mtrr_enable (void ) +static void mtrr_enable (struct mtrr_work_state *state) {/* Intel (P6) standard MTRRs */ -mtrr_wrmsr (MSR_MTRRdefType, deftype_lo, deftype_hi); +mtrr_wrmsr (MSR_MTRRdefType, state->lo, state->hi); }/*
@@ -969,7 +970,6 @@ static void mtrr_enable(void)* The caller must ensure that local interrupts are disabled and * are reenabled after cache_enable() has been called. */ -static unsigned long saved_cr4;static DEFINE_RAW_SPINLOCK (cache_disable_lock);/*
@@ -983,7 +983,8 @@ static void maybe_flush_caches(void)wbinvd (); } -static void cache_disable (void )__acquires (cache_disable_lock) +static void cache_disable (struct mtrr_work_state *state) +__acquires (cache_disable_lock) {unsigned long cr0;
@@ -1002,8 +1003,8 @@ static void cache_disable(void) __acquires(cache_disable_lock)/* Save value of CR4 and clear Page Global Enable (bit 7) */ if (cpu_feature_enabled (X86_FEATURE_PGE )) { - saved_cr4 =__read_cr4 (); -__write_cr4 (saved_cr4 & ~X86_CR4_PGE ); + state->cr4 =__read_cr4 (); +__write_cr4 (state->cr4 & ~X86_CR4_PGE ); }/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
@@ -1011,26 +1012,27 @@ static void cache_disable(void) __acquires(cache_disable_lock)flush_tlb_local ();if (cpu_feature_enabled (X86_FEATURE_MTRR )) -mtrr_disable (); +mtrr_disable (state);maybe_flush_caches (); } -static void cache_enable (void )__releases (cache_disable_lock) +static void cache_enable (struct mtrr_work_state *state) +__releases (cache_disable_lock) {/* Flush TLBs (no need to flush caches - they are disabled) */ count_vm_tlb_event (NR_TLB_LOCAL_FLUSH_ALL );flush_tlb_local ();if (cpu_feature_enabled (X86_FEATURE_MTRR )) -mtrr_enable (); +mtrr_enable (state);/* Enable caches */ write_cr0 (read_cr0 () & ~X86_CR0_CD );/* Restore value of CR4 */ if (cpu_feature_enabled (X86_FEATURE_PGE )) -__write_cr4 (saved_cr4); +__write_cr4 (state->cr4);raw_spin_unlock (&cache_disable_lock); }
@@ -1038,11 +1040,12 @@ static void cache_enable(void) __releases(cache_disable_lock)void mtrr_generic_set_state (void ) {unsigned long mask, count; +struct mtrr_work_state state; -cache_disable (); +cache_disable (&state);/* Actually set the state */ - mask =set_mtrr_state (); + mask =set_mtrr_state (&state);/* Use the atomic bitops to update the global mask */ for (count =0 ; count <sizeof (mask) *8 ; ++count) {
@@ -1051,7 +1054,7 @@ void mtrr_generic_set_state(void) mask >>=1 ; } -cache_enable (); +cache_enable (&state); }/**
@@ -1069,11 +1072,12 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base, {unsigned long flags;struct mtrr_var_range *vr; +struct mtrr_work_state state; vr = &mtrr_state.var_ranges[reg];local_irq_save (flags); -cache_disable (); +cache_disable (&state);if (size ==0 ) {/*
@@ -1092,7 +1096,7 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base,mtrr_wrmsr (MTRRphysMask_MSR (reg), vr->mask_lo, vr->mask_hi); } -cache_enable (); +cache_enable (&state);local_irq_restore (flags); }
Signature · 1 line
2.52.0
kernel test robot
build botRe: [PATCH 2/4] x86/mtrr: Introduce MTRR work state structure
kernel test robot noticed the following build warnings:
Hi Juergen,
kernel test robot noticed the following build warnings:
[auto build test WARNING on tip/x86/core]
[also build test WARNING on tip/master linus/master v6.19-rc6 next-20260121] [cannot apply to tip/auto-latest]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Juergen-Gross/x86-mtrr-Move-cache_enable-and-cache_disable-to-mtrr-generic-c/20260121-230320 base: tip/x86/core patch link: https://lore.kernel.org/r/20260121141106.755458-3-jgross%40suse.com patch subject: [PATCH 2/4] x86/mtrr: Introduce MTRR work state structure config: x86_64-allnoconfig (https://download.01.org/0day-ci/archive/20260122/[email protected]/config) compiler: clang version 20.1.8 (https://github.com/llvm/llvm-project 87f0227cb60147a26a1eeb4fb06e3b505e9c7261) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260122/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> Warning: arch/x86/kernel/cpu/mtrr/generic.c:920 function parameter 'state' not described in 'set_mtrr_state' >> Warning: arch/x86/kernel/cpu/mtrr/generic.c:920 function parameter 'state' not described in 'set_mtrr_state'
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Re: [PATCH 2/4] x86/mtrr: Introduce MTRR work state structure
Quoted 19 lines · kernel test robot
Hi Juergen,
kernel test robot noticed the following build warnings:
[auto build test WARNING on tip/x86/core]
[also build test WARNING on tip/master linus/master v6.19-rc6 next-20260121] [cannot apply to tip/auto-latest]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]url: https://github.com/intel-lab-lkp/linux/commits/Juergen-Gross/x86-mtrr-Move-cache_enable-and-cache_disable-to-mtrr-generic-c/20260121-230320 base: tip/x86/core patch link: https://lore.kernel.org/r/20260121141106.755458-3-jgross%40suse.com patch subject: [PATCH 2/4] x86/mtrr: Introduce MTRR work state structure config: x86_64-allnoconfig (https://download.01.org/0day-ci/archive/20260122/[email protected]/config) compiler: clang version 20.1.8 (https://github.com/llvm/llvm-project 87f0227cb60147a26a1eeb4fb06e3b505e9c7261) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260122/[email protected]/reproduce)
Quoted 10 lines
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/All warnings (new ones prefixed by >>):
Quoted 2 lines
Quoted 2 lines
Warning:
arch/x86/kernel/cpu/mtrr/generic.c:920function parameter 'state' not described in 'set_mtrr_state'
Warning:arch/x86/kernel/cpu/mtrr/generic.c:920function parameter 'state' not described in 'set_mtrr_state'
Just a note: I have a local fix for that, but will send V2 only after some more feedback.
Juergen
PATCH 3/4 x86/mtrr: Add a prepare_set hook to mtrr_ops
In reply to Juergen GrossIn order to prepare dropping the cache_disable_lock add a new hook to struct mtrr_ops, allowing to set some global state before calling the .set hook on all active CPUs.
Move setting of mtrr_state.var_ranges[] from generic_set_mtrr() to the new prepare hook. Note that doing that only once outside the cache_disable_lock is fine, as generic_set_mtrr() is called via set_mtrr() only and this call is protected by mtrr_mutex.
--- arch/x86/kernel/cpu/mtrr/generic.c | 32 ++++++++++++++++++++++++------ arch/x86/kernel/cpu/mtrr/mtrr.c | 3 +++ arch/x86/kernel/cpu/mtrr/mtrr.h | 2 ++ 3 files changed, 31 insertions(+), 6 deletions(-)
arch/x86/kernel/cpu/mtrr/generic.c +26 -6
@@ -1057,6 +1057,31 @@ void mtrr_generic_set_state(void)cache_enable (&state); } +/** +* generic_prepare_set_mtrr - set variable MTRR register data in mtrr_state +* +* @reg: The register to set. +* @base: The base address of the region. +* @size: The size of the region. If this is 0 the region is disabled. +* @type: The type of the region. +* +* Returns nothing. +*/ +static void generic_prepare_set_mtrr (unsigned int reg,unsigned long base, +unsigned long size,mtrr_type type) +{ +struct mtrr_var_range *vr = &mtrr_state.var_ranges[reg]; + +if (size ==0 ) { +memset (vr,0 ,sizeof (struct mtrr_var_range )); + }else { + vr->base_lo = base <<PAGE_SHIFT | type; + vr->base_hi = (base >> (32 -PAGE_SHIFT )) & ~phys_hi_rsvd; + vr->mask_lo = -size <<PAGE_SHIFT |MTRR_PHYSMASK_V ; + vr->mask_hi = (-size >> (32 -PAGE_SHIFT )) & ~phys_hi_rsvd; + } +} +/** * generic_set_mtrr - set variable MTRR register on the local CPU. *
@@ -1085,13 +1110,7 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base,* clear the relevant mask register to disable a range. */ mtrr_wrmsr (MTRRphysMask_MSR (reg),0 ,0 ); -memset (vr,0 ,sizeof (struct mtrr_var_range )); }else { - vr->base_lo = base <<PAGE_SHIFT | type; - vr->base_hi = (base >> (32 -PAGE_SHIFT )) & ~phys_hi_rsvd; - vr->mask_lo = -size <<PAGE_SHIFT |MTRR_PHYSMASK_V ; - vr->mask_hi = (-size >> (32 -PAGE_SHIFT )) & ~phys_hi_rsvd; -mtrr_wrmsr (MTRRphysBase_MSR (reg), vr->base_lo, vr->base_hi);mtrr_wrmsr (MTRRphysMask_MSR (reg), vr->mask_lo, vr->mask_hi); }
@@ -1156,6 +1175,7 @@ int positive_have_wrcomb(void)const struct mtrr_ops generic_mtrr_ops = { .get = generic_get_mtrr, .get_free_region = generic_get_free_region, + .prepare_set = generic_prepare_set_mtrr, .set = generic_set_mtrr, .validate_add_page = generic_validate_add_page, .have_wrcomb = generic_have_wrcomb,
arch/x86/kernel/cpu/mtrr/mtrr.c +3 -0
@@ -175,6 +175,9 @@ static void set_mtrr(unsigned int reg, unsigned long base, unsigned long size, .smp_type = type }; +if (mtrr_if->prepare_set) + mtrr_if->prepare_set (reg, base, size, type); +stop_machine_cpuslocked (mtrr_rendezvous_handler, &data, cpu_online_mask);generic_rebuild_map ();
arch/x86/kernel/cpu/mtrr/mtrr.h +2 -0
@@ -17,6 +17,8 @@ extern unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];struct mtrr_ops {u32 var_regs; +void (*prepare_set)(unsigned int reg,unsigned long base, +unsigned long size,mtrr_type type);void (*set)(unsigned int reg,unsigned long base,unsigned long size,mtrr_type type); void (*get)(unsigned int reg, unsigned long *base,
Signature · 1 line
2.52.0
PATCH 4/4 x86/mtrr: Drop cache_disable_lock
In reply to Juergen GrossNow that no global state is modified under cache_disable_lock, it can be dropped.
All required serialization is done via mtrr_mutex and cpus_read_lock(), ensuring that only one set_mtrr() can be active at any time and that this call can't run concurrently with CPU bringup.
The main advantages are a faster boot of machines with lots of CPUs, and avoiding hard lockups on those machines in case mtrr_generic_set_state() takes too long in uncached mode, resulting in other CPUs waiting for seconds to get the cache_disable_lock.
This has been seen happening in more than 1% of all boots on an Intel machine with 960 CPUs. With this patch applied boot was always successful.
---
I was considering applying a "Fixes:" tag, but this could only reference the initial kernel git commit, as MTRR support predates git, and the problem existed since MTRRs became a thing.
arch/x86/kernel/cpu/mtrr/generic.c | 12 ------------ 1 file changed, 12 deletions(-)
arch/x86/kernel/cpu/mtrr/generic.c +0 -12
@@ -970,7 +970,6 @@ static void mtrr_enable(struct mtrr_work_state *state)* The caller must ensure that local interrupts are disabled and * are reenabled after cache_enable() has been called. */ -static DEFINE_RAW_SPINLOCK (cache_disable_lock);/* * Cache flushing is the most time-consuming step when programming the
@@ -984,17 +983,9 @@ static void maybe_flush_caches(void) }static void cache_disable (struct mtrr_work_state *state) -__acquires (cache_disable_lock) {unsigned long cr0; -/* -* This is not ideal since the cache is only flushed/disabled -* for this CPU while the MTRRs are changed, but changing this -* requires more invasive changes to the way the kernel boots. -*/ -raw_spin_lock (&cache_disable_lock); -/* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */ cr0 =read_cr0 () |X86_CR0_CD ;write_cr0 (cr0);
@@ -1018,7 +1009,6 @@ static void cache_disable(struct mtrr_work_state *state) }static void cache_enable (struct mtrr_work_state *state) -__releases (cache_disable_lock) {/* Flush TLBs (no need to flush caches - they are disabled) */ count_vm_tlb_event (NR_TLB_LOCAL_FLUSH_ALL );
@@ -1033,8 +1023,6 @@ static void cache_enable(struct mtrr_work_state *state)/* Restore value of CR4 */ if (cpu_feature_enabled (X86_FEATURE_PGE ))__write_cr4 (state->cr4); - -raw_spin_unlock (&cache_disable_lock); }void mtrr_generic_set_state (void )
Signature · 1 line
2.52.0