Patch "kaiser: load_new_mm_cr3() let SWITCH_USER_CR3 flush user" has been added to the 4.4-stable tree

This is a note to let you know that I've just added the patch titled

kaiser: load_new_mm_cr3() let SWITCH_USER_CR3 flush user

to the 4.4-stable tree which can be found at:
http://www.kernel.org/git/?p=linux/kernel/git/stable/stable-queue.git;a=summary

The filename of the patch is:
kaiser-load_new_mm_cr3-let-switch_user_cr3-flush-user.patch
and it can be found in the queue-4.4 subdirectory.

If you, or anyone else, feels it should not be added to the stable tree, please let <[email protected]> know about it.

From: Hugh Dickins <[email protected]>
Date: Thu, 17 Aug 2017 15:00:37 -0700
Subject: kaiser: load_new_mm_cr3() let SWITCH_USER_CR3 flush user

From: Hugh Dickins <[email protected]>

We have many machines (Westmere, Sandybridge, Ivybridge) supporting PCID but not INVPCID: on these load_new_mm_cr3() simply crashed.

Flushing user context inside load_new_mm_cr3() without the use of invpcid is difficult: momentarily switch from kernel to user context and back to do so? I'm not sure whether that can be safely done at all, and would risk polluting user context with kernel internals, and kernel context with stale user externals.

Instead, follow the hint in the comment that was there: change X86_CR3_PCID_USER_VAR to be a per-cpu variable, then load_new_mm_cr3() can leave a note in it, for SWITCH_USER_CR3 on return to userspace to flush user context TLB, instead of default X86_CR3_PCID_USER_NOFLUSH.

Which works well enough that there's no need to do it this way only when invpcid is unsupported: it's a good alternative to invpcid here. But there's a couple of inlines in asm/tlbflush.h that need to do the same trick, so it's best to localize all this per-cpu business in mm/kaiser.c: moving that part of the initialization from setup_pcid() to kaiser_setup_pcid(); with kaiser_flush_tlb_on_return_to_user() the function for noting an X86_CR3_PCID_USER_FLUSH. And let's keep a KAISER_SHADOW_PGD_OFFSET in there, to avoid the extra OR on exit.

I did try to make the feature tests in asm/tlbflush.h more consistent with each other: there seem to be far too many ways of performing such tests, and I don't have a good grasp of their differences. At first I converted them all to be static_cpu_has(): but that proved to be a mistake, as the comment in __native_flush_tlb_single() hints; so then I reversed and made them all this_cpu_has(). Probably all gratuitous change, but that's the way it's working at present.

I am slightly bothered by the way non-per-cpu X86_CR3_PCID_KERN_VAR gets re-initialized by each cpu (before and after these changes):
no problem when (as usual) all cpus on a machine have the same features, but in principle incorrect. However, my experiment to per-cpu-ify that one did not end well...

---
 arch/x86/include/asm/kaiser.h   |   18 +++++++-----
 arch/x86/include/asm/tlbflush.h |   58 +++++++++++++++++++++++++++-------------
 arch/x86/kernel/cpu/common.c    |   22 ---------------
 arch/x86/mm/kaiser.c            |   50 ++++++++++++++++++++++++++++++----
 arch/x86/mm/tlb.c               |   46 ++++++++++++-------------------
 5 files changed, 114 insertions(+), 80 deletions(-)

@@ -32,13 +32,12 @@ movq \reg, %cr3
 .macro _SWITCH_TO_USER_CR3 reg
 movq %cr3, \reg
 andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg
-/*
- * This can obviously be one instruction by putting the
- * KAISER_SHADOW_PGD_OFFSET bit in the X86_CR3_PCID_USER_VAR.
- * But, just leave it now for simplicity.
- */
-orq  X86_CR3_PCID_USER_VAR, \reg
-orq  $(KAISER_SHADOW_PGD_OFFSET), \reg
+orq  PER_CPU_VAR(X86_CR3_PCID_USER_VAR), \reg
+js   9f
+// FLUSH this time, reset to NOFLUSH for next time
+// But if nopcid?  Consider using 0x80 for user pcid?
+movb $(0x80), PER_CPU_VAR(X86_CR3_PCID_USER_VAR+7)
+9:
 movq \reg, %cr3
 .endm
 

@@ -90,6 +89,11 @@ movq PER_CPU_VAR(unsafe_stack_register_b
 */
 DECLARE_PER_CPU_USER_MAPPED(unsigned long, unsafe_stack_register_backup);
 
+extern unsigned long X86_CR3_PCID_KERN_VAR;
+DECLARE_PER_CPU(unsigned long, X86_CR3_PCID_USER_VAR);
+
+extern char __per_cpu_user_mapped_start[], __per_cpu_user_mapped_end[];
+
 /**
  *  kaiser_add_mapping - map a virtual memory part to the shadow (user) mapping
  *  @addr: the start address of the range

@@ -321,33 +321,12 @@ static __always_inline void setup_smap(s
 	}
 }
 
-/*
- * These can have bit 63 set, so we can not just use a plain "or"
- * instruction to get their value or'd into CR3.  It would take
- * another register.  So, we use a memory reference to these
- * instead.
- *
- * This is also handy because systems that do not support
- * PCIDs just end up or'ing a 0 into their CR3, which does
- * no harm.
- */
-__aligned(PAGE_SIZE) unsigned long X86_CR3_PCID_KERN_VAR = 0;
-__aligned(PAGE_SIZE) unsigned long X86_CR3_PCID_USER_VAR = 0;
-
 static void setup_pcid(struct cpuinfo_x86 *c)
 {
 	if (cpu_has(c, X86_FEATURE_PCID)) {
 		if (cpu_has(c, X86_FEATURE_PGE)) {
 			cr4_set_bits(X86_CR4_PCIDE);
 			/*
-			 * These variables are used by the entry/exit
-			 * code to change PCIDs.
-			 */
-#ifdef CONFIG_KAISER
-			X86_CR3_PCID_KERN_VAR = X86_CR3_PCID_KERN_NOFLUSH;
-			X86_CR3_PCID_USER_VAR = X86_CR3_PCID_USER_NOFLUSH;
-#endif
-			/*
 			 * INVPCID has two "groups" of types:
 			 * 1/2: Invalidate an individual address
 			 * 3/4: Invalidate all contexts

@@ -372,6 +351,7 @@ static void setup_pcid(struct cpuinfo_x8
 			clear_cpu_cap(c, X86_FEATURE_PCID);
 		}
 	}
+	kaiser_setup_pcid();
 }
 
 /*

Patches currently in stable-queue which might be from [email protected] are

queue-4.4/kaiser-vmstat-show-nr_kaisertable-as-nr_overhead.patch queue-4.4/kaiser-add-nokaiser-boot-option-using-alternative.patch queue-4.4/kaiser-fix-unlikely-error-in-alloc_ldt_struct.patch queue-4.4/kaiser-_pgd_alloc-without-__gfp_repeat-to-avoid-stalls.patch queue-4.4/kaiser-kaiser_flush_tlb_on_return_to_user-check-pcid.patch queue-4.4/x86-paravirt-dont-patch-flush_tlb_single.patch queue-4.4/kaiser-merged-update.patch queue-4.4/kaiser-delete-kaiser_real_switch-option.patch queue-4.4/kaiser-kaiser_remove_mapping-move-along-the-pgd.patch queue-4.4/kaiser-fix-perf-crashes.patch queue-4.4/kaiser-drop-is_atomic-arg-to-kaiser_pagetable_walk.patch queue-4.4/kaiser-load_new_mm_cr3-let-switch_user_cr3-flush-user.patch queue-4.4/kaiser-enhanced-by-kernel-and-user-pcids.patch queue-4.4/kaiser-x86_cr3_pcid_noflush-and-x86_cr3_pcid_user.patch queue-4.4/kaiser-use-alternative-instead-of-x86_cr3_pcid_noflush.patch queue-4.4/kaiser-stack-map-page_size-at-thread_size-page_size.patch queue-4.4/kaiser-name-that-0x1000-kaiser_shadow_pgd_offset.patch queue-4.4/kaiser-fix-regs-to-do_nmi-ifndef-config_kaiser.patch queue-4.4/kaiser-do-not-set-_page_nx-on-pgd_none.patch queue-4.4/kaiser-tidied-up-asm-kaiser.h-somewhat.patch queue-4.4/kaiser-cleanups-while-trying-for-gold-link.patch queue-4.4/kaiser-tidied-up-kaiser_add-remove_mapping-slightly.patch queue-4.4/kaiser-fix-build-and-fixme-in-alloc_ldt_struct.patch queue-4.4/kaiser-kernel-address-isolation.patch queue-4.4/kaiser-enomem-if-kaiser_pagetable_walk-null.patch queue-4.4/kaiser-asm-tlbflush.h-handle-nopge-at-lower-level.patch queue-4.4/kaiser-paranoid_entry-pass-cr3-need-to-paranoid_exit.patch queue-4.4/kaiser-kaiser-depends-on-smp.patch queue-4.4/kaiser-pcid-0-for-kernel-and-128-for-user.patch