x86 TLB flush cleanups, moving toward PCID support

PATCH Applied Andy Lutomirski 4 participants 15 messages May 26, 2017

Subsystems (9)

Memory management x86 KVM perf events +5 more subsystems

lore

Share & tools

lore

Andy Lutomirski

author maintainer

2017-05-26 00:47 UTC #0 lore raw

v3 cover x86 TLB flush cleanups, moving toward PCID support

As I've been working on polishing my PCID code, a major problem I've encountered is that there are too many x86 TLB flushing code paths and that they have too many inconsequential differences. The result was that earlier versions of the PCID code were a colossal mess and very difficult to understand.

This series goes a long way toward cleaning up the mess. With all the patches applied, there is a single function that contains the meat of the code to flush the TLB on a given CPU, and all the tlb flushing APIs call it for both local and remote CPUs.

This series should only adversely affect the kernel in a couple of minor ways:

- It makes smp_mb() unconditional when flushing TLBs. We used to use the TLB flush itself to mostly avoid smp_mb() on the initiating CPU.

- On UP kernels, we lose the dubious optimization of inlining nerfed variants of all the TLB flush APIs. This bloats the kernel a tiny bit, although it should increase performance, since the SMP versions were better.

Patch 8 in here is a little bit off topic. It's a cleanup that's also needed before PCID can go in, but it's not directly about TLB flushing.

Changes from v1:
- Rebased onto tip:x86/mm to pick up UV and Xen changes.
- Drop the patches that Ingo already applied.

Changes from RFC:
- Fixed missing call to arch_tlbbatch_flush().
- "Be more consistent wrt PAGE_SHIFT vs PAGE_SIZE in tlb flush code" is new
- Misc typos fixed.
- Actually compiles when UV is enabled.

Andy Lutomirski (8):
x86/mm: Pass flush_tlb_info to flush_tlb_others() etc x86/mm: Change the leave_mm() condition for local TLB flushes x86/mm: Refactor flush_tlb_mm_range() to merge local and remote cases x86/mm: Use new merged flush logic in arch_tlbbatch_flush() x86/mm: Remove the UP tlbflush code; always use the formerly SMP code x86/mm: Rework lazy TLB to track the actual loaded mm x86/mm: Be more consistent wrt PAGE_SHIFT vs PAGE_SIZE in tlb flush
code
x86,kvm: Teach KVM's VMX code that CR3 isn't a constant

 arch/x86/Kconfig                      |   2 +-
 arch/x86/events/core.c                |   3 +-
 arch/x86/include/asm/hardirq.h        |   2 +-
 arch/x86/include/asm/mmu.h            |   6 -
 arch/x86/include/asm/mmu_context.h    |  21 +-
 arch/x86/include/asm/paravirt.h       |   6 +-
 arch/x86/include/asm/paravirt_types.h |   5 +-
 arch/x86/include/asm/tlbbatch.h       |   2 -
 arch/x86/include/asm/tlbflush.h       | 104 ++-------
 arch/x86/include/asm/uv/uv.h          |   9 +-
 arch/x86/kernel/ldt.c                 |   7 +-
 arch/x86/kvm/vmx.c                    |  21 +-
 arch/x86/mm/init.c                    |   4 +-
 arch/x86/mm/tlb.c                     | 389 ++++++++++++++++------------------
 arch/x86/platform/uv/tlb_uv.c         |  10 +-
 arch/x86/xen/mmu_pv.c                 |  61 +++---
 16 files changed, 281 insertions(+), 371 deletions(-)

Signature · 7 lines

2.9.4

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Andy Lutomirski

author maintainer

2017-05-26 00:47 UTC #1 lore raw

PATCH v3 1/8 x86/mm: Pass flush_tlb_info to flush_tlb_others() etc

Rather than passing all the contents of flush_tlb_info to flush_tlb_others(), pass a pointer to the structure directly. For consistency, this also removes the unnecessary cpu parameter from uv_flush_tlb_others() to make its signature match the other *flush_tlb_others() functions.

This serves two purposes:

- It will dramatically simplify future patches that change struct flush_tlb_info, which I'm planning to do.

- struct flush_tlb_info is an adequate description of what to do for a local flush, too, so by reusing it we can remove duplicated code between local and remove flushes in a future patch.

Cc Rik van Riel <[email protected]>

Cc Dave Hansen <[email protected]>

Cc Nadav Amit <[email protected]>

Cc Michal Hocko <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Andy Lutomirski <[email protected]>

---
 arch/x86/include/asm/paravirt.h       |  6 ++--
 arch/x86/include/asm/paravirt_types.h |  5 ++-
 arch/x86/include/asm/tlbflush.h       | 19 ++++++-----
 arch/x86/include/asm/uv/uv.h          |  9 ++---
 arch/x86/mm/tlb.c                     | 64 +++++++++++++++++------------------
 arch/x86/platform/uv/tlb_uv.c         | 10 +++---
 arch/x86/xen/mmu_pv.c                 | 10 +++---
 7 files changed, 59 insertions(+), 64 deletions(-)

arch/x86/include/asm/paravirt.h +2 -4

index 55fa56fe4e45..9a15739d9f4b 100644
@@ -312,11 +312,9 @@ static inline void __flush_tlb_single(unsigned long addr)
 }
 
 static inline void flush_tlb_others(const struct cpumask *cpumask,
-				    struct mm_struct *mm,
-				    unsigned long start,
-				    unsigned long end)
+				    const struct flush_tlb_info *info)
 {
-	PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end);
+	PVOP_VCALL2(pv_mmu_ops.flush_tlb_others, cpumask, info);
 }
 
 static inline int paravirt_pgd_alloc(struct mm_struct *mm)

arch/x86/include/asm/paravirt_types.h +2 -3

index 7465d6fe336f..cb976bab6299 100644
@@ -51,6 +51,7 @@ struct mm_struct;
 struct desc_struct;
 struct task_struct;
 struct cpumask;
+struct flush_tlb_info;
 
 /*
  * Wrapper type for pointers to code which uses the non-standard

@@ -223,9 +224,7 @@ struct pv_mmu_ops {
 	void (*flush_tlb_kernel)(void);
 	void (*flush_tlb_single)(unsigned long addr);
 	void (*flush_tlb_others)(const struct cpumask *cpus,
-				 struct mm_struct *mm,
-				 unsigned long start,
-				 unsigned long end);
+				 const struct flush_tlb_info *info);
 
 	/* Hooks for allocating and freeing a pagetable top-level */
 	int  (*pgd_alloc)(struct mm_struct *mm);

arch/x86/include/asm/tlbflush.h +11 -8

index 8f6e2f87511b..6f439ac92026 100644
@@ -220,12 +220,18 @@ static inline void __flush_tlb_one(unsigned long addr)
  *  - flush_tlb_page(vma, vmaddr) flushes one page
  *  - flush_tlb_range(vma, start, end) flushes a range of pages
  *  - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
- *  - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus
+ *  - flush_tlb_others(cpumask, info) flushes TLBs on other cpus
  *
  * ..but the i386 has somewhat limited tlb flushing capabilities,
  * and page-granular flushes are available only on i486 and up.
  */
 
+struct flush_tlb_info {
+	struct mm_struct *mm;
+	unsigned long start;
+	unsigned long end;
+};
+
 #ifndef CONFIG_SMP
 
 /* "_up" is for UniProcessor.

@@ -279,9 +285,7 @@ static inline void flush_tlb_mm_range(struct mm_struct *mm,
 }
 
 static inline void native_flush_tlb_others(const struct cpumask *cpumask,
-					   struct mm_struct *mm,
-					   unsigned long start,
-					   unsigned long end)
+					   const struct flush_tlb_info *info)
 {
 }

@@ -317,8 +321,7 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long a)
 }
 
 void native_flush_tlb_others(const struct cpumask *cpumask,
-				struct mm_struct *mm,
-				unsigned long start, unsigned long end);
+			     const struct flush_tlb_info *info);
 
 #define TLBSTATE_OK	1
 #define TLBSTATE_LAZY	2

@@ -340,8 +343,8 @@ extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch);
 #endif	/* SMP */
 
 #ifndef CONFIG_PARAVIRT
-#define flush_tlb_others(mask, mm, start, end)	\
-	native_flush_tlb_others(mask, mm, start, end)
+#define flush_tlb_others(mask, info)	\
+	native_flush_tlb_others(mask, info)
 #endif
 
 #endif /* _ASM_X86_TLBFLUSH_H */

arch/x86/include/asm/uv/uv.h +3 -6

index 6686820feae9..d2db4517930c 100644
@@ -15,10 +15,7 @@ extern void uv_cpu_init(void);
 extern void uv_nmi_init(void);
 extern void uv_system_init(void);
 extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
-						 struct mm_struct *mm,
-						 unsigned long start,
-						 unsigned long end,
-						 unsigned int cpu);
+						 const struct flush_tlb_info *info);
 
 #else	/* X86_UV */

@@ -28,8 +25,8 @@ static inline int is_uv_hubless(void)	{ return 0; }
 static inline void uv_cpu_init(void)	{ }
 static inline void uv_system_init(void)	{ }
 static inline const struct cpumask *
-uv_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm,
-		    unsigned long start, unsigned long end, unsigned int cpu)
+uv_flush_tlb_others(const struct cpumask *cpumask,
+		    const struct flush_tlb_info *info)
 { return cpumask; }
 
 #endif	/* X86_UV */

arch/x86/mm/tlb.c +32 ~~-32~~

152 lines not on this page · raw message

arch/x86/platform/uv/tlb_uv.c +4 -6

index 42e65fee5673..5a22d77ffc2a 100644
@@ -1121,11 +1121,9 @@ static int set_distrib_bits(struct cpumask *flush_mask, struct bau_control *bcp,
  * done.  The returned pointer is valid till preemption is re-enabled.
  */
 const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
-						struct mm_struct *mm,
-						unsigned long start,
-						unsigned long end,
-						unsigned int cpu)
+					  const struct flush_tlb_info *info)
 {
+	unsigned int cpu = smp_processor_id();
 	int locals = 0, remotes = 0, hubs = 0;
 	struct bau_desc *bau_desc;
 	struct cpumask *flush_mask;

@@ -1179,8 +1177,8 @@ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
 
 	record_send_statistics(stat, locals, hubs, remotes, bau_desc);
 
-	if (!end || (end - start) <= PAGE_SIZE)
-		address = start;
+	if (!info->end || (info->end - info->start) <= PAGE_SIZE)
+		address = info->start;
 	else
 		address = TLB_FLUSH_ALL;

arch/x86/xen/mmu_pv.c +5 -5

index 1f386d7fdf70..4b926c6b813c 100644
@@ -1366,8 +1366,7 @@ static void xen_flush_tlb_single(unsigned long addr)
 }
 
 static void xen_flush_tlb_others(const struct cpumask *cpus,
-				 struct mm_struct *mm, unsigned long start,
-				 unsigned long end)
+				 const struct flush_tlb_info *info)
 {
 	struct {
 		struct mmuext_op op;

@@ -1379,7 +1378,7 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
 	} *args;
 	struct multicall_space mcs;
 
-	trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
+	trace_xen_mmu_flush_tlb_others(cpus, info->mm, info->start, info->end);
 
 	if (cpumask_empty(cpus))
 		return;		/* nothing to do */

@@ -1393,9 +1392,10 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
 	cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
 
 	args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
-	if (end != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) {
+	if (info->end != TLB_FLUSH_ALL &&
+	    (info->end - info->start) <= PAGE_SIZE) {
 		args->op.cmd = MMUEXT_INVLPG_MULTI;
-		args->op.arg1.linear_addr = start;
+		args->op.arg1.linear_addr = info->start;
 	}
 
 	MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);

Signature · 7 lines

2.9.4

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Andy Lutomirski

author maintainer

2017-05-26 00:47 UTC #2 lore raw

PATCH v3 2/8 x86/mm: Change the leave_mm() condition for local TLB flushes

In reply to Andy Lutomirski

On a remote TLB flush, we leave_mm() if we're TLBSTATE_LAZY. For a local flush_tlb_mm_range(), we leave_mm() if !current->mm. These are approximately the same condition -- the scheduler sets lazy TLB mode when switching to a thread with no mm.

I'm about to merge the local and remote flush code, but for ease of verifying and bisecting the patch, I want the local and remote flush behavior to match first. This patch changes the local code to match the remote code.

Cc Rik van Riel <[email protected]>

Cc Dave Hansen <[email protected]>

Cc Nadav Amit <[email protected]>

Cc Michal Hocko <[email protected]>

Cc Arjan van de Ven <[email protected]>

Signed-off-by Andy Lutomirski <[email protected]>

---
 arch/x86/mm/tlb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

arch/x86/mm/tlb.c +1 -1

index 776469cc54e0..3143c9a180e5 100644
@@ -311,7 +311,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
 		goto out;
 	}
 
-	if (!current->mm) {
+	if (this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK) {
 		leave_mm(smp_processor_id());
 
 		/* Synchronize with switch_mm. */

Signature · 7 lines

2.9.4

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Rik van Riel

2017-05-26 01:39 UTC #3 lore raw

Re: [PATCH v3 2/8] x86/mm: Change the leave_mm() condition for local TLB flushes

Quoted 10 lines · Andy Lutomirski

+++ b/arch/x86/mm/tlb.c

(quoted diff) +0 -0

@@ -311,7 +311,7 @@ void flush_tlb_mm_range(struct mm_struct *mm,
 unsigned long start,
 A 		goto out;
 A 	}

arch/x86/mm/tlb.c +1 -1

-	if (!current->mm) {
+	if (this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK) {

A leave_mm(smp_processor_id());

Unless -mm changed leave_mm (I did not check), this is not quite correct yet.

The reason is leave_mm (at least in the latest Linus tree) ignores the cpu argument for one of its checks.

You should probably fix that in an earlier patch,
assuming you haven't already done so in -mm.

void leave_mm(int cpu)
{
A A A A A A A A struct mm_struct *active_mm = this_cpu_read(cpu_tlbstate.active_mm);
A A A A A A A A if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) A A A A A A A A A A A A A A A A BUG();
A A A A A A A A if (cpumask_test_cpu(cpu, mm_cpumask(active_mm))) { A A A A A A A A A A A A A A A A cpumask_clear_cpu(cpu, mm_cpumask(active_mm)); A A A A A A A A A A A A A A A A load_cr3(swapper_pg_dir);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Andy Lutomirski

author maintainer

2017-05-26 02:02 UTC #4 lore raw

Re: [PATCH v3 2/8] x86/mm: Change the leave_mm() condition for local TLB flushes

Quoted 24 lines · Rik van Riel

arch/x86/mm/tlb.c void flush_tlb_mm_range(struct mm_struct *mm, · line 316
Quoted 10 lines · Andy Lutomirski
+++ b/arch/x86/mm/tlb.c
arch/x86/mm/tlb.c +1 -1
@@ -311,7 +311,7 @@ void flush_tlb_mm_range(struct mm_struct *mm,
 unsigned long start,
              goto out;
      }
 
-     if (!current->mm) {
+     if (this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK) {
              leave_mm(smp_processor_id());
Unless -mm changed leave_mm (I did not check), this is not quite correct yet.
The reason is leave_mm (at least in the latest Linus tree) ignores the cpu argument for one of its checks.
You should probably fix that in an earlier patch, assuming you haven't already done so in -mm.
void leave_mm(int cpu)
{
struct mm_struct *active_mm =

Quoted 6 lines

this_cpu_read(cpu_tlbstate.active_mm);
        if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
                BUG();
        if (cpumask_test_cpu(cpu, mm_cpumask(active_mm))) {
                cpumask_clear_cpu(cpu, mm_cpumask(active_mm));
                load_cr3(swapper_pg_dir);

I agree it's odd, but what's the bug? Both before and after, leave_mm needed to be called with cpu == smp_processor_id(), and smp_processor_id() warns if it's called in a preemptible context.

Rik van Riel

2017-05-26 13:25 UTC #5 lore raw

Re: [PATCH v3 2/8] x86/mm: Change the leave_mm() condition for local TLB flushes

Quoted 33 lines · Andy Lutomirski

Quoted 21 lines · Rik van Riel
Review void flush_tlb_mm_range(struct mm_struct *mm, · line 316
Quoted 10 lines · Andy Lutomirski
+++ b/arch/x86/mm/tlb.c
(quoted diff) +1 -1
@@ -311,7 +311,7 @@ void flush_tlb_mm_range(struct mm_struct *mm,
 unsigned long start,
 A A A A A A A A A A A A A A goto out;
 A A A A A A }
 
-A A A A A if (!current->mm) {
+A A A A A if (this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK) {
 A A A A A A A A A A A A A A leave_mm(smp_processor_id());
Unless -mm changed leave_mm (I did not check), this is not quite correct yet.
The reason is leave_mm (at least in the latest Linus tree) ignores the cpu argument for one of its checks.
You should probably fix that in an earlier patch,
assuming you haven't already done so in -mm.
Quoted 9 lines
void leave_mm(int cpu)
{
A A A A A A A A struct mm_struct *active_mm = this_cpu_read(cpu_tlbstate.active_mm);
A A A A A A A A if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) A A A A A A A A A A A A A A A A BUG();
A A A A A A A A if (cpumask_test_cpu(cpu, mm_cpumask(active_mm))) { A A A A A A A A A A A A A A A A cpumask_clear_cpu(cpu, mm_cpumask(active_mm)); A A A A A A A A A A A A A A A A load_cr3(swapper_pg_dir);

Quoted 4 lines

I agree it's odd, but what's the bug?A A Both before and after, leave_mm
needed to be called with cpu == smp_processor_id(), and smp_processor_id() warns if it's called in a preemptible context.

Indeed, you are right. Looking at too much code at once...

Nadav Amit

2017-05-26 01:43 UTC #6 lore raw

Re: [PATCH v3 2/8] x86/mm: Change the leave_mm() condition for local TLB flushes

In reply to Andy Lutomirski

Quoted 28 lines · Andy Lutomirski

On a remote TLB flush, we leave_mm() if we're TLBSTATE_LAZY. For a local flush_tlb_mm_range(), we leave_mm() if !current->mm. These are approximately the same condition -- the scheduler sets lazy TLB mode when switching to a thread with no mm.
I'm about to merge the local and remote flush code, but for ease of verifying and bisecting the patch, I want the local and remote flush behavior to match first. This patch changes the local code to match the remote code.
Cc Rik van Riel <[email protected]>
Cc Dave Hansen <[email protected]>
Cc Nadav Amit <[email protected]>
Cc Michal Hocko <[email protected]>
Cc Arjan van de Ven <[email protected]>
Signed-off-by Andy Lutomirski <[email protected]>
---
arch/x86/mm/tlb.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/mm/tlb.c +0 -0
index 776469cc54e0..3143c9a180e5 100644

arch/x86/mm/tlb.c void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, · line 315

Quoted 6 lines

arch/x86/mm/tlb.c +1 -1

@@ -311,7 +311,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
 		goto out;
 	}
 
-	if (!current->mm) {
+	if (this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK) {
 		leave_mm(smp_processor_id());

Maybe it is an overkill, but you may want to have two variants: leave_mm() and leave_mm_irq_off(). Currently, leave_mm() does not disable IRQs, but in patch 6 it does. Here you indeed need to disable IRQs, but the cases in prior to this patch - you do not.

Andy Lutomirski

author maintainer

2017-05-26 00:47 UTC #7 lore raw

PATCH v3 3/8 x86/mm: Refactor flush_tlb_mm_range() to merge local and remote cases

In reply to Andy Lutomirski

The local flush path is very similar to the remote flush path. Merge them.

This is intended to make no difference to behavior whatsoever. It removes some code and will make future changes to the flushing mechanics simpler.

This patch does remove one small optimization: flush_tlb_mm_range() now has an unconditional smp_mb() instead of using MOV to CR3 or INVLPG as a full barrier when applicable. I think this is okay for a few reasons. First, smp_mb() is quite cheap compared to the cost of a TLB flush. Second, this rearrangement makes a bigger optimization available: with some work on the SMP function call code, we could do the local and remote flushes in parallel. Third, I'm planning a rework of the TLB flush algorithm that will require an atomic operation at the beginning of each flush, and that operation will replace the smp_mb().

Cc Rik van Riel <[email protected]>

Cc Dave Hansen <[email protected]>

Cc Nadav Amit <[email protected]>

Cc Michal Hocko <[email protected]>

Cc Andrew Morton <[email protected]>

Cc Arjan van de Ven <[email protected]>

Signed-off-by Andy Lutomirski <[email protected]>

---
 arch/x86/include/asm/tlbflush.h |   1 -
 arch/x86/mm/tlb.c               | 113 +++++++++++++++++-----------------------
 2 files changed, 48 insertions(+), 66 deletions(-)

arch/x86/include/asm/tlbflush.h +0 -1

index 6f439ac92026..9934c7c99213 100644
@@ -225,7 +225,6 @@ static inline void __flush_tlb_one(unsigned long addr)
  * ..but the i386 has somewhat limited tlb flushing capabilities,
  * and page-granular flushes are available only on i486 and up.
  */
-
 struct flush_tlb_info {
 	struct mm_struct *mm;
 	unsigned long start;

arch/x86/mm/tlb.c +48 ~~-65~~

159 lines not on this page · raw message

Signature · 7 lines

2.9.4

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Andy Lutomirski

author maintainer

2017-05-26 00:47 UTC #8 lore raw

PATCH v3 4/8 x86/mm: Use new merged flush logic in arch_tlbbatch_flush()

In reply to Andy Lutomirski

Now there's only one copy of the local tlb flush logic for non-kernel pages on SMP kernels.

The only functional change is that arch_tlbbatch_flush() will now leave_mm() on the local CPU if that CPU is in the batch and is in TLBSTATE_LAZY mode.

Cc Rik van Riel <[email protected]>

Cc Dave Hansen <[email protected]>

Cc Nadav Amit <[email protected]>

Cc Michal Hocko <[email protected]>

Cc Arjan van de Ven <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Andy Lutomirski <[email protected]>

---
 arch/x86/mm/tlb.c | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

arch/x86/mm/tlb.c +2 -6

index 12b8812e8926..c03b4a0ce58c 100644
@@ -382,12 +382,8 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
 
 	int cpu = get_cpu();
 
-	if (cpumask_test_cpu(cpu, &batch->cpumask)) {
-		count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
-		local_flush_tlb();
-		trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL);
-	}
-
+	if (cpumask_test_cpu(cpu, &batch->cpumask))
+		flush_tlb_func_local(&info, TLB_LOCAL_SHOOTDOWN);
 	if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids)
 		flush_tlb_others(&batch->cpumask, &info);
 	cpumask_clear(&batch->cpumask);

Signature · 7 lines

2.9.4

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Andy Lutomirski

author maintainer

2017-05-26 00:48 UTC #9 lore raw

PATCH v3 5/8 x86/mm: Remove the UP tlbflush code; always use the formerly SMP code

In reply to Andy Lutomirski

The UP tlbflush generates somewhat nicer code than the SMP version. Aside from that, it's fallen quite a bit behind the SMP code:

- flush_tlb_mm_range() didn't flush individual pages if the range was small.

- The lazy TLB code was much weaker. This usually wouldn't matter, but, if a kernel thread flushed its lazy "active_mm" more than once (due to reclaim or similar), it wouldn't be unlazied and would instead pointlessly flush repeatedly.

- Tracepoints were missing.

Aside from that, simply having the UP code around was a maintanence burden, since it means that any change to the TLB flush code had to make sure not to break it.

Simplify everything by deleting the UP code.

Cc Rik van Riel <[email protected]>

Cc Dave Hansen <[email protected]>

Cc Nadav Amit <[email protected]>

Cc Michal Hocko <[email protected]>

Cc Andrew Morton <[email protected]>

Cc Arjan van de Ven <[email protected]>

Signed-off-by Andy Lutomirski <[email protected]>

---
 arch/x86/Kconfig                   |  2 +-
 arch/x86/include/asm/hardirq.h     |  2 +-
 arch/x86/include/asm/mmu.h         |  6 ---
 arch/x86/include/asm/mmu_context.h |  2 -
 arch/x86/include/asm/tlbbatch.h    |  2 -
 arch/x86/include/asm/tlbflush.h    | 76 +-------------------------------------
 arch/x86/mm/init.c                 |  2 -
 arch/x86/mm/tlb.c                  | 17 +--------
 8 files changed, 5 insertions(+), 104 deletions(-)

arch/x86/Kconfig +1 -1

index cd18994a9555..d65f87e083d1 100644
@@ -69,7 +69,7 @@ config X86
 	select ARCH_USE_BUILTIN_BSWAP
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
-	select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH if SMP
+	select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
 	select ARCH_WANT_FRAME_POINTERS
 	select ARCH_WANTS_DYNAMIC_TASK_STRUCT
 	select BUILDTIME_EXTABLE_SORT

arch/x86/include/asm/hardirq.h +1 -1

index 59405a248fc2..9b76cd331990 100644
@@ -22,8 +22,8 @@ typedef struct {
 #ifdef CONFIG_SMP
 	unsigned int irq_resched_count;
 	unsigned int irq_call_count;
-	unsigned int irq_tlb_count;
 #endif
+	unsigned int irq_tlb_count;
 #ifdef CONFIG_X86_THERMAL_VECTOR
 	unsigned int irq_thermal_count;
 #endif

arch/x86/include/asm/mmu.h +0 -6

index f9813b6d8b80..79b647a7ebd0 100644
@@ -37,12 +37,6 @@ typedef struct {
 #endif
 } mm_context_t;
 
-#ifdef CONFIG_SMP
 void leave_mm(int cpu);
-#else
-static inline void leave_mm(int cpu)
-{
-}
-#endif
 
 #endif /* _ASM_X86_MMU_H */

arch/x86/include/asm/mmu_context.h +0 -2

index 68b329d77b3a..187c39470a0b 100644
@@ -99,10 +99,8 @@ static inline void load_mm_ldt(struct mm_struct *mm)
 
 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
 {
-#ifdef CONFIG_SMP
 	if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
 		this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
-#endif
 }
 
 static inline int init_new_context(struct task_struct *tsk,

arch/x86/include/asm/tlbbatch.h +0 -2

index 01a6de16fb96..f4a6ff352a0e 100644
@@ -3,7 +3,6 @@
 
 #include <linux/cpumask.h>
 
-#ifdef CONFIG_SMP
 struct arch_tlbflush_unmap_batch {
 	/*
 	 * Each bit set is a CPU that potentially has a TLB entry for one of

@@ -11,6 +10,5 @@ struct arch_tlbflush_unmap_batch {
 	 */
 	struct cpumask cpumask;
 };
-#endif
 
 #endif /* _ARCH_X86_TLBBATCH_H */

arch/x86/include/asm/tlbflush.h +1 ~~-75~~

102 lines not on this page · raw message

arch/x86/mm/init.c +0 -2

index cbc87ea98751..c61183b57427 100644
@@ -811,10 +811,8 @@ void __init zone_sizes_init(void)
 }
 
 DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = {
-#ifdef CONFIG_SMP
 	.active_mm = &init_mm,
 	.state = 0,
-#endif
 	.cr4 = ~0UL,	/* fail hard if we screw up cr4 shadow initialization */
 };
 EXPORT_SYMBOL_GPL(cpu_tlbstate);

arch/x86/mm/tlb.c +2 ~~-15~~

index c03b4a0ce58c..da1416c77bfb 100644
@@ -15,7 +15,7 @@
 #include <linux/debugfs.h>
 
 /*
- *	Smarter SMP flushing macros.
+ *	TLB flushing, formerly SMP-only
  *		c/o Linus Torvalds.
  *
  *	These mean you can really definitely utterly forget about

@@ -28,8 +28,6 @@
  *	Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi
  */
 
-#ifdef CONFIG_SMP
-
 /*
  * We cannot call mmdrop() because we are in interrupt context,
  * instead update mm->cpu_vm_mask.

@@ -53,8 +51,6 @@ void leave_mm(int cpu)
 }
 EXPORT_SYMBOL_GPL(leave_mm);
 
-#endif /* CONFIG_SMP */
-
 void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	       struct task_struct *tsk)
 {

@@ -85,10 +81,8 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 				set_pgd(pgd, init_mm.pgd[stack_pgd_index]);
 		}
 
-#ifdef CONFIG_SMP
 		this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
 		this_cpu_write(cpu_tlbstate.active_mm, next);
-#endif
 
 		cpumask_set_cpu(cpu, mm_cpumask(next));

@@ -146,9 +140,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 		if (unlikely(prev->context.ldt != next->context.ldt))
 			load_mm_ldt(next);
 #endif
-	}
-#ifdef CONFIG_SMP
-	  else {
+	} else {
 		this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
 		BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);

@@ -175,11 +167,8 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 			load_mm_ldt(next);
 		}
 	}
-#endif
 }
 
-#ifdef CONFIG_SMP
-
 /*
  * The flush IPI assumes that a thread switch happens in this order:
  * [cpu0: the cpu that switches]

@@ -436,5 +425,3 @@ static int __init create_tlb_single_page_flush_ceiling(void)
 	return 0;
 }
 late_initcall(create_tlb_single_page_flush_ceiling);
-
-#endif /* CONFIG_SMP */

Signature · 7 lines

2.9.4

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Andy Lutomirski

author maintainer

2017-05-26 00:48 UTC #10 lore raw

PATCH v3 6/8 x86/mm: Rework lazy TLB to track the actual loaded mm

In reply to Andy Lutomirski

Lazy TLB state is currently managed in a rather baroque manner. AFAICT, there are three possible states:

- Non-lazy. This means that we're running a user thread or a kernel thread that has called use_mm(). current->mm == current->active_mm == cpu_tlbstate.active_mm and cpu_tlbstate.state == TLBSTATE_OK.

- Lazy with user mm. We're running a kernel thread without an mm and we're borrowing an mm_struct. We have current->mm == NULL, current->active_mm == cpu_tlbstate.active_mm, cpu_tlbstate.state != TLBSTATE_OK (i.e. TLBSTATE_LAZY or 0). The current cpu is set in mm_cpumask(current->active_mm). CR3 points to current->active_mm->pgd. The TLB is up to date.

- Lazy with init_mm. This happens when we call leave_mm(). We have current->mm == NULL, current->active_mm == cpu_tlbstate.active_mm, but that mm is only relelvant insofar as the scheduler is tracking it for refcounting. cpu_tlbstate.state != TLBSTATE_OK. The current cpu is clear in mm_cpumask(current->active_mm). CR3 points to swapper_pg_dir, i.e. init_mm->pgd.

This patch simplifies the situation. Other than perf, x86 stops caring about current->active_mm at all. We have cpu_tlbstate.loaded_mm pointing to the mm that CR3 references. The TLB is always up to date for that mm. leave_mm() just switches us to init_mm. There are no longer any special cases for mm_cpumask, and switch_mm() switches mms without worrying about laziness.

After this patch, cpu_tlbstate.state serves only to tell the TLB flush code whether it may switch to init_mm instead of doing a normal flush.

This makes fairly extensive changes to xen_exit_mmap(), which used to look a bit like black magic.

Perf is unchanged. With or without this change, perf may behave a bit erratically if it tries to read user memory in kernel thread context. We should build on this patch to teach perf to never look at user memory when cpu_tlbstate.loaded_mm != current->mm.

Cc Rik van Riel <[email protected]>

Cc Dave Hansen <[email protected]>

Cc Nadav Amit <[email protected]>

Cc Michal Hocko <[email protected]>

Cc Andrew Morton <[email protected]>

Cc Arjan van de Ven <[email protected]>

Signed-off-by Andy Lutomirski <[email protected]>

---
 arch/x86/events/core.c          |   3 +-
 arch/x86/include/asm/tlbflush.h |  12 ++-
 arch/x86/kernel/ldt.c           |   7 +-
 arch/x86/mm/init.c              |   2 +-
 arch/x86/mm/tlb.c               | 208 ++++++++++++++++++++--------------------
 arch/x86/xen/mmu_pv.c           |  51 +++++-----
 6 files changed, 143 insertions(+), 140 deletions(-)

arch/x86/events/core.c +1 -2

index 580b60f5ac83..77a33096728d 100644
@@ -2101,8 +2101,7 @@ static int x86_pmu_event_init(struct perf_event *event)
 
 static void refresh_pce(void *ignored)
 {
-	if (current->active_mm)
-		load_mm_cr4(current->active_mm);
+	load_mm_cr4(this_cpu_read(cpu_tlbstate.loaded_mm));
 }
 
 static void x86_pmu_event_mapped(struct perf_event *event)

arch/x86/include/asm/tlbflush.h +10 -2

index dbb5a9f0fed8..388c2463fde6 100644
@@ -66,7 +66,13 @@ static inline void invpcid_flush_all_nonglobals(void)
 #endif
 
 struct tlb_state {
-	struct mm_struct *active_mm;
+	/*
+	 * cpu_tlbstate.loaded_mm should match CR3 whenever interrupts
+	 * are on.  This means that it may not match current->active_mm,
+	 * which will contain the previous user mm when we're in lazy TLB
+	 * mode even if we've already switched back to swapper_pg_dir.
+	 */
+	struct mm_struct *loaded_mm;
 	int state;
 
 	/*

@@ -256,7 +262,9 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
 static inline void reset_lazy_tlbstate(void)
 {
 	this_cpu_write(cpu_tlbstate.state, 0);
-	this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
+	this_cpu_write(cpu_tlbstate.loaded_mm, &init_mm);
+
+	WARN_ON(read_cr3() != __pa_symbol(swapper_pg_dir));
 }
 
 static inline void arch_tlbbatch_add_mm(struct arch_tlbflush_unmap_batch *batch,

arch/x86/kernel/ldt.c +4 -3

index d4a15831ac58..de503e7a64ad 100644
@@ -22,14 +22,15 @@
 #include <asm/syscalls.h>
 
 /* context.lock is held for us, so we don't need any locking. */
-static void flush_ldt(void *current_mm)
+static void flush_ldt(void *__mm)
 {
+	struct mm_struct *mm = __mm;
 	mm_context_t *pc;
 
-	if (current->active_mm != current_mm)
+	if (this_cpu_read(cpu_tlbstate.loaded_mm) != mm)
 		return;
 
-	pc = &current->active_mm->context;
+	pc = &mm->context;
 	set_ldt(pc->ldt->entries, pc->ldt->size);
 }

arch/x86/mm/init.c +1 -1

index c61183b57427..88ee942cb47d 100644
@@ -811,7 +811,7 @@ void __init zone_sizes_init(void)
 }
 
 DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = {
-	.active_mm = &init_mm,
+	.loaded_mm = &init_mm,
 	.state = 0,
 	.cr4 = ~0UL,	/* fail hard if we screw up cr4 shadow initialization */
 };

arch/x86/mm/tlb.c +104 ~~-104~~

246 lines not on this page · raw message

arch/x86/xen/mmu_pv.c +23 ~~-28~~

91 lines not on this page · raw message

Signature · 7 lines

2.9.4

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Andy Lutomirski

author maintainer

2017-05-26 00:48 UTC #11 lore raw

PATCH v3 7/8 x86/mm: Be more consistent wrt PAGE_SHIFT vs PAGE_SIZE in tlb flush code

In reply to Andy Lutomirski

Nadav pointed out that some code used PAGE_SIZE and other code used PAGE_SHIFT. Use PAGE_SHIFT instead of multiplying or dividing by PAGE_SIZE.

Requested-by Nadav Amit <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Dave Hansen <[email protected]>

Cc Nadav Amit <[email protected]>

Cc Michal Hocko <[email protected]>

Cc Arjan van de Ven <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Andy Lutomirski <[email protected]>

---
 arch/x86/mm/tlb.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

arch/x86/mm/tlb.c +2 -3

index 4bfadb869a1e..4bcec510174c 100644
@@ -220,8 +220,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
 		trace_tlb_flush(reason, TLB_FLUSH_ALL);
 	} else {
 		unsigned long addr;
-		unsigned long nr_pages =
-			(f->end - f->start) / PAGE_SIZE;
+		unsigned long nr_pages = (f->end - f->start) >> PAGE_SHIFT;
 		addr = f->start;
 		while (addr < f->end) {
 			__flush_tlb_single(addr);

@@ -351,7 +350,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 
 	/* Balance as user space task's flush, a bit conservative */
 	if (end == TLB_FLUSH_ALL ||
-	    (end - start) > tlb_single_page_flush_ceiling * PAGE_SIZE) {
+	    (end - start) > tlb_single_page_flush_ceiling << PAGE_SHIFT) {
 		on_each_cpu(do_flush_tlb_all, NULL, 1);
 	} else {
 		struct flush_tlb_info info;

Signature · 7 lines

2.9.4

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Andy Lutomirski

author maintainer

2017-05-26 00:47 UTC #12 lore raw

PATCH v3 8/8 x86,kvm: Teach KVM's VMX code that CR3 isn't a constant

In reply to Andy Lutomirski

When PCID is enabled, CR3's PCID bits can change during context switches, so KVM won't be able to treat CR3 as a per-mm constant any more.

I structured this like the existing CR4 handling. Under ordinary circumstances (PCID disabled or if the current PCID and the value that's already in the VMCS match), then we won't do an extra VMCS write, and we'll never do an extra direct CR3 read. The overhead should be minimal.

I disallowed using the new helper in non-atomic context because PCID support will cause CR3 to stop being constant in non-atomic process context.

(Frankly, it also scares me a bit that KVM ever treated CR3 as constant, but it looks like it was okay before.)

Cc Paolo Bonzini <[email protected]>

Cc Radim Krčmář <[email protected]>

Cc [email protected]

Cc Rik van Riel <[email protected]>

Cc Dave Hansen <[email protected]>

Cc Nadav Amit <[email protected]>

Cc Michal Hocko <[email protected]>

Cc Andrew Morton <[email protected]>

Cc Arjan van de Ven <[email protected]>

Signed-off-by Andy Lutomirski <[email protected]>

---
 arch/x86/include/asm/mmu_context.h | 19 +++++++++++++++++++
 arch/x86/kvm/vmx.c                 | 21 ++++++++++++++++++---
 2 files changed, 37 insertions(+), 3 deletions(-)

arch/x86/include/asm/mmu_context.h +19 -0

index 187c39470a0b..f20d7ea47095 100644
@@ -266,4 +266,23 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
 	return __pkru_allows_pkey(vma_pkey(vma), write);
 }
 
+
+/*
+ * This can be used from process context to figure out what the value of
+ * CR3 is without needing to do a (slow) read_cr3().
+ *
+ * It's intended to be used for code like KVM that sneakily changes CR3
+ * and needs to restore it.  It needs to be used very carefully.
+ */
+static inline unsigned long __get_current_cr3_fast(void)
+{
+	unsigned long cr3 = __pa(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd);
+
+	/* For now, be very restrictive about when this can be called. */
+	VM_WARN_ON(in_nmi() || !in_atomic());
+
+	VM_BUG_ON(cr3 != read_cr3());
+	return cr3;
+}
+
 #endif /* _ASM_X86_MMU_CONTEXT_H */

arch/x86/kvm/vmx.c +18 -3

index 72f78396bc09..b7b36c9ffa3d 100644
@@ -48,6 +48,7 @@
 #include <asm/kexec.h>
 #include <asm/apic.h>
 #include <asm/irq_remapping.h>
+#include <asm/mmu_context.h>
 
 #include "trace.h"
 #include "pmu.h"

@@ -596,6 +597,7 @@ struct vcpu_vmx {
 		int           gs_ldt_reload_needed;
 		int           fs_reload_needed;
 		u64           msr_host_bndcfgs;
+		unsigned long vmcs_host_cr3;	/* May not match real cr3 */
 		unsigned long vmcs_host_cr4;	/* May not match real cr4 */
 	} host_state;
 	struct {

@@ -5012,12 +5014,19 @@ static void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
 	u32 low32, high32;
 	unsigned long tmpl;
 	struct desc_ptr dt;
-	unsigned long cr0, cr4;
+	unsigned long cr0, cr3, cr4;
 
 	cr0 = read_cr0();
 	WARN_ON(cr0 & X86_CR0_TS);
 	vmcs_writel(HOST_CR0, cr0);  /* 22.2.3 */
-	vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */
+
+	/*
+	 * Save the most likely value for this task's CR3 in the VMCS.
+	 * We can't use __get_current_cr3_fast() because we're not atomic.
+	 */
+	cr3 = read_cr3();
+	vmcs_writel(HOST_CR3, cr3);		/* 22.2.3  FIXME: shadow tables */
+	vmx->host_state.vmcs_host_cr3 = cr3;
 
 	/* Save the most likely value for this task's CR4 in the VMCS. */
 	cr4 = cr4_read_shadow();

@@ -8843,7 +8852,7 @@ static void vmx_arm_hv_timer(struct kvm_vcpu *vcpu)
 static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
-	unsigned long debugctlmsr, cr4;
+	unsigned long debugctlmsr, cr3, cr4;
 
 	/* Don't enter VMX if guest state is invalid, let the exit handler
 	   start emulation until we arrive back to a valid state */

@@ -8865,6 +8874,12 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 	if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
 		vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
 
+	cr3 = __get_current_cr3_fast();
+	if (unlikely(cr3 != vmx->host_state.vmcs_host_cr3)) {
+		vmcs_writel(HOST_CR3, cr3);
+		vmx->host_state.vmcs_host_cr3 = cr3;
+	}
+
 	cr4 = cr4_read_shadow();
 	if (unlikely(cr4 != vmx->host_state.vmcs_host_cr4)) {
 		vmcs_writel(HOST_CR4, cr4);

Signature · 7 lines

2.9.4

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Paolo Bonzini

maintainer Applied

2017-05-26 15:30 UTC #13 lore raw

Re: [PATCH v3 8/8] x86,kvm: Teach KVM's VMX code that CR3 isn't a constant

Quoted 115 lines · Andy Lutomirski

When PCID is enabled, CR3's PCID bits can change during context switches, so KVM won't be able to treat CR3 as a per-mm constant any more.

I disallowed using the new helper in non-atomic context because PCID support will cause CR3 to stop being constant in non-atomic process context.

(Frankly, it also scares me a bit that KVM ever treated CR3 as constant, but it looks like it was okay before.)

Cc Paolo Bonzini <[email protected]>

Cc Radim Krčmář <[email protected]>

Cc [email protected]

Cc Rik van Riel <[email protected]>

Cc Dave Hansen <[email protected]>

Cc Nadav Amit <[email protected]>

Cc Michal Hocko <[email protected]>

Cc Andrew Morton <[email protected]>

Cc Arjan van de Ven <[email protected]>

Signed-off-by Andy Lutomirski <[email protected]>

---
 arch/x86/include/asm/mmu_context.h | 19 +++++++++++++++++++
 arch/x86/kvm/vmx.c                 | 21 ++++++++++++++++++---
 2 files changed, 37 insertions(+), 3 deletions(-)

arch/x86/include/asm/mmu_context.h +19 -0

index 187c39470a0b..f20d7ea47095 100644
@@ -266,4 +266,23 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
 	return __pkru_allows_pkey(vma_pkey(vma), write);
 }
 
+
+/*
+ * This can be used from process context to figure out what the value of
+ * CR3 is without needing to do a (slow) read_cr3().
+ *
+ * It's intended to be used for code like KVM that sneakily changes CR3
+ * and needs to restore it.  It needs to be used very carefully.
+ */
+static inline unsigned long __get_current_cr3_fast(void)
+{
+	unsigned long cr3 = __pa(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd);
+
+	/* For now, be very restrictive about when this can be called. */
+	VM_WARN_ON(in_nmi() || !in_atomic());
+
+	VM_BUG_ON(cr3 != read_cr3());
+	return cr3;
+}
+
 #endif /* _ASM_X86_MMU_CONTEXT_H */

arch/x86/kvm/vmx.c +12 -3

index 72f78396bc09..b7b36c9ffa3d 100644
@@ -48,6 +48,7 @@
 #include <asm/kexec.h>
 #include <asm/apic.h>
 #include <asm/irq_remapping.h>
+#include <asm/mmu_context.h>
 
 #include "trace.h"
 #include "pmu.h"

@@ -596,6 +597,7 @@ struct vcpu_vmx {
 		int           gs_ldt_reload_needed;
 		int           fs_reload_needed;
 		u64           msr_host_bndcfgs;
+		unsigned long vmcs_host_cr3;	/* May not match real cr3 */
 		unsigned long vmcs_host_cr4;	/* May not match real cr4 */
 	} host_state;
 	struct {

@@ -5012,12 +5014,19 @@ static void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
 	u32 low32, high32;
 	unsigned long tmpl;
 	struct desc_ptr dt;
-	unsigned long cr0, cr4;
+	unsigned long cr0, cr3, cr4;
 
 	cr0 = read_cr0();
 	WARN_ON(cr0 & X86_CR0_TS);
 	vmcs_writel(HOST_CR0, cr0);  /* 22.2.3 */
-	vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */
+
+	/*
+	 * Save the most likely value for this task's CR3 in the VMCS.
+	 * We can't use __get_current_cr3_fast() because we're not atomic.
+	 */
+	cr3 = read_cr3();
+	vmcs_writel(HOST_CR3, cr3);		/* 22.2.3  FIXME: shadow tables */
+	vmx->host_state.vmcs_host_cr3 = cr3;
 
 	/* Save the most likely value for this task's CR4 in the VMCS. */
 	cr4 = cr4_read_shadow();

@@ -8843,7 +8852,7 @@ static void vmx_arm_hv_timer(struct kvm_vcpu *vcpu)
 static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
-	unsigned long debugctlmsr, cr4;
+	unsigned long debugctlmsr, cr3, cr4;
 
 	/* Don't enter VMX if guest state is invalid, let the exit handler
 	   start emulation until we arrive back to a valid state */

@@ -8865,6 +8874,12 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 	if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
 		vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);

arch/x86/kvm/vmx.c static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) · line 8885

Quoted 10 lines

arch/x86/kvm/vmx.c +6 -0

 
+	cr3 = __get_current_cr3_fast();
+	if (unlikely(cr3 != vmx->host_state.vmcs_host_cr3)) {
+		vmcs_writel(HOST_CR3, cr3);
+		vmx->host_state.vmcs_host_cr3 = cr3;
+	}
+
 	cr4 = cr4_read_shadow();
 	if (unlikely(cr4 != vmx->host_state.vmcs_host_cr4)) {
 		vmcs_writel(HOST_CR4, cr4);

Queued, thanks. If anybody needs a topic branch, please holler.

Paolo

Paolo Bonzini

maintainer

2017-05-26 16:05 UTC #14 lore raw

Re: [PATCH v3 8/8] x86,kvm: Teach KVM's VMX code that CR3 isn't a constant

Quoted 118 lines · Paolo Bonzini

Quoted 115 lines · Andy Lutomirski

When PCID is enabled, CR3's PCID bits can change during context switches, so KVM won't be able to treat CR3 as a per-mm constant any more.

I disallowed using the new helper in non-atomic context because PCID support will cause CR3 to stop being constant in non-atomic process context.

(Frankly, it also scares me a bit that KVM ever treated CR3 as constant, but it looks like it was okay before.)

Cc Paolo Bonzini <[email protected]>

Cc Radim Krčmář <[email protected]>

Cc [email protected]

Cc Rik van Riel <[email protected]>

Cc Dave Hansen <[email protected]>

Cc Nadav Amit <[email protected]>

Cc Michal Hocko <[email protected]>

Cc Andrew Morton <[email protected]>

Cc Arjan van de Ven <[email protected]>

Signed-off-by Andy Lutomirski <[email protected]>

---
 arch/x86/include/asm/mmu_context.h | 19 +++++++++++++++++++
 arch/x86/kvm/vmx.c                 | 21 ++++++++++++++++++---
 2 files changed, 37 insertions(+), 3 deletions(-)

arch/x86/include/asm/mmu_context.h +19 -0

index 187c39470a0b..f20d7ea47095 100644
@@ -266,4 +266,23 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
 	return __pkru_allows_pkey(vma_pkey(vma), write);
 }
 
+
+/*
+ * This can be used from process context to figure out what the value of
+ * CR3 is without needing to do a (slow) read_cr3().
+ *
+ * It's intended to be used for code like KVM that sneakily changes CR3
+ * and needs to restore it.  It needs to be used very carefully.
+ */
+static inline unsigned long __get_current_cr3_fast(void)
+{
+	unsigned long cr3 = __pa(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd);
+
+	/* For now, be very restrictive about when this can be called. */
+	VM_WARN_ON(in_nmi() || !in_atomic());
+
+	VM_BUG_ON(cr3 != read_cr3());
+	return cr3;
+}
+
 #endif /* _ASM_X86_MMU_CONTEXT_H */

arch/x86/kvm/vmx.c +12 -3

index 72f78396bc09..b7b36c9ffa3d 100644
@@ -48,6 +48,7 @@
 #include <asm/kexec.h>
 #include <asm/apic.h>
 #include <asm/irq_remapping.h>
+#include <asm/mmu_context.h>
 
 #include "trace.h"
 #include "pmu.h"

@@ -596,6 +597,7 @@ struct vcpu_vmx {
 		int           gs_ldt_reload_needed;
 		int           fs_reload_needed;
 		u64           msr_host_bndcfgs;
+		unsigned long vmcs_host_cr3;	/* May not match real cr3 */
 		unsigned long vmcs_host_cr4;	/* May not match real cr4 */
 	} host_state;
 	struct {

@@ -5012,12 +5014,19 @@ static void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
 	u32 low32, high32;
 	unsigned long tmpl;
 	struct desc_ptr dt;
-	unsigned long cr0, cr4;
+	unsigned long cr0, cr3, cr4;
 
 	cr0 = read_cr0();
 	WARN_ON(cr0 & X86_CR0_TS);
 	vmcs_writel(HOST_CR0, cr0);  /* 22.2.3 */
-	vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */
+
+	/*
+	 * Save the most likely value for this task's CR3 in the VMCS.
+	 * We can't use __get_current_cr3_fast() because we're not atomic.
+	 */
+	cr3 = read_cr3();
+	vmcs_writel(HOST_CR3, cr3);		/* 22.2.3  FIXME: shadow tables */
+	vmx->host_state.vmcs_host_cr3 = cr3;
 
 	/* Save the most likely value for this task's CR4 in the VMCS. */
 	cr4 = cr4_read_shadow();

@@ -8843,7 +8852,7 @@ static void vmx_arm_hv_timer(struct kvm_vcpu *vcpu)
 static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
-	unsigned long debugctlmsr, cr4;
+	unsigned long debugctlmsr, cr3, cr4;
 
 	/* Don't enter VMX if guest state is invalid, let the exit handler
 	   start emulation until we arrive back to a valid state */

@@ -8865,6 +8874,12 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 	if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
 		vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);

Quoted 12 lines

arch/x86/kvm/vmx.c static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) · line 8885
Quoted 10 lines
arch/x86/kvm/vmx.c +6 -0
+	cr3 = __get_current_cr3_fast();
+	if (unlikely(cr3 != vmx->host_state.vmcs_host_cr3)) {
+		vmcs_writel(HOST_CR3, cr3);
+		vmx->host_state.vmcs_host_cr3 = cr3;
+	}
+
 	cr4 = cr4_read_shadow();
 	if (unlikely(cr4 != vmx->host_state.vmcs_host_cr4)) {
 		vmcs_writel(HOST_CR4, cr4);
Queued, thanks. If anybody needs a topic branch, please holler.

Ah, no, it depends on the others. Note to self, compile first, answer second.

Paolo