Skip to contentSkip to conversation

x86/entry: simplify RESTORE_CR3

PATCH In review Lai Jiangshan 2 participants 4 messages May 26, 2020 – May 29, 2020

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover x86/entry: simplify RESTORE_CR3

When I searched percpu data touched by entry code for #DB protection[1], it seems to me RESTORE_CR3() does too much work, this patchset simplifies it.

Patch 1 enhances 21e944591102("x86/mm: Optimize RESTORE_CR3") for kernel CR3.

Patch 2 *reverts* 21e944591102("x86/mm: Optimize RESTORE_CR3") for User CR3.

Lai Jiangshan (2):
x86/entry: Don't write to CR3 when restoring to kernel CR3 x86/entry: always flush user CR3 in RESTORE_CR3

 arch/x86/entry/calling.h  | 36 ++++++++----------------------------
 arch/x86/entry/entry_64.S |  6 +++---
 2 files changed, 11 insertions(+), 31 deletions(-)
Signature · 1 line
2.20.1
#1 lore raw

PATCH 1/2 x86/entry: Don't write to CR3 when restoring to kernel CR3

Skip resuming KERNEL pages since it is already KERNEL CR3

---
 arch/x86/entry/calling.h | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)
arch/x86/entry/calling.h +4 -9
index 1c7f13bb6728..505246185624 100644
@@ -268,14 +268,13 @@ For 32-bit we have the following conventions - kernel is built with
 .macro RESTORE_CR3 scratch_reg:req save_reg:req
 	ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
 
-	ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
-
 	/*
-	 * KERNEL pages can always resume with NOFLUSH as we do
-	 * explicit flushes.
+	 * Skip resuming KERNEL pages since it is already KERNEL CR3.
 	 */
 	bt	$PTI_USER_PGTABLE_BIT, \save_reg
-	jnc	.Lnoflush_\@
+	jnc	.Lend_\@
+
+	ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
 
 	/*
 	 * Check if there's a pending flush for the user ASID we're
@@ -293,10 +292,6 @@ For 32-bit we have the following conventions - kernel is built with
 	SET_NOFLUSH_BIT \save_reg
 
 .Lwrcr3_\@:
-	/*
-	 * The CR3 write could be avoided when not changing its value,
-	 * but would require a CR3 read *and* a scratch register.
-	 */
 	movq	\save_reg, %cr3
 .Lend_\@:
 .endm
Signature · 1 line
2.20.1
#2 lore raw

PATCH 2/2 x86/entry: always flush user CR3 in RESTORE_CR3

In reply to Lai Jiangshan

RESTORE_CR3 is called when CPL==0 or #DF, it is unlikely CPL==0&cr3==userCR3 and #DF itself is unlikely case. There is no much overhead to always flush userCR3.

---
 arch/x86/entry/calling.h  | 27 ++++++---------------------
 arch/x86/entry/entry_64.S |  6 +++---
 2 files changed, 9 insertions(+), 24 deletions(-)
arch/x86/entry/calling.h +6 -21
index 505246185624..ff26e4eb7063 100644
@@ -265,33 +265,18 @@ For 32-bit we have the following conventions - kernel is built with
 .Ldone_\@:
 .endm
 
-.macro RESTORE_CR3 scratch_reg:req save_reg:req
+.macro RESTORE_CR3 save_reg:req
 	ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
 
 	/*
 	 * Skip resuming KERNEL pages since it is already KERNEL CR3.
+	 *
+	 * RESTORE_CR3 is called when CPL==0 or #DF, it is unlikely
+	 * CPL==0&cr3==userCR3 and #DF itself is unlikely case.
+	 * There is no much overhead to always flush userCR3.
 	 */
 	bt	$PTI_USER_PGTABLE_BIT, \save_reg
 	jnc	.Lend_\@
-
-	ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
-
-	/*
-	 * Check if there's a pending flush for the user ASID we're
-	 * about to set.
-	 */
-	movq	\save_reg, \scratch_reg
-	andq	$(0x7FF), \scratch_reg
-	bt	\scratch_reg, THIS_CPU_user_pcid_flush_mask
-	jnc	.Lnoflush_\@
-
-	btr	\scratch_reg, THIS_CPU_user_pcid_flush_mask
-	jmp	.Lwrcr3_\@
-
-.Lnoflush_\@:
-	SET_NOFLUSH_BIT \save_reg
-
-.Lwrcr3_\@:
 	movq	\save_reg, %cr3
 .Lend_\@:
 .endm
@@ -306,7 +291,7 @@ For 32-bit we have the following conventions - kernel is built with
 .endm
 .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
 .endm
-.macro RESTORE_CR3 scratch_reg:req save_reg:req
+.macro RESTORE_CR3 save_reg:req
 .endm
 
 #endif
arch/x86/entry/entry_64.S +3 -3
index d983a0d4bc73..46efa842a45e 100644
@@ -1283,13 +1283,13 @@ SYM_CODE_START_LOCAL(paranoid_exit)
 	jnz	.Lparanoid_exit_no_swapgs
 	TRACE_IRQS_IRETQ
 	/* Always restore stashed CR3 value (see paranoid_entry) */
-	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
+	RESTORE_CR3	save_reg=%r14
 	SWAPGS_UNSAFE_STACK
 	jmp	restore_regs_and_return_to_kernel
 .Lparanoid_exit_no_swapgs:
 	TRACE_IRQS_IRETQ_DEBUG
 	/* Always restore stashed CR3 value (see paranoid_entry) */
-	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
+	RESTORE_CR3	save_reg=%r14
 	jmp restore_regs_and_return_to_kernel
 SYM_CODE_END(paranoid_exit)
 
@@ -1703,7 +1703,7 @@ end_repeat_nmi:
 	call	exc_nmi
 
 	/* Always restore stashed CR3 value (see paranoid_entry) */
-	RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
+	RESTORE_CR3 save_reg=%r14
 
 	testl	%ebx, %ebx			/* swapgs needed? */
 	jnz	nmi_restore
Signature · 1 line
2.20.1
maintainer
#3 lore raw
In reply to Lai Jiangshan
Quoted 10 lines · Lai Jiangshan

When I searched percpu data touched by entry code for #DB protection[1], it seems to me RESTORE_CR3() does too much work, this patchset simplifies it.

Patch 1 enhances 21e944591102("x86/mm: Optimize RESTORE_CR3") for kernel CR3.

Patch 2 *reverts* 21e944591102("x86/mm: Optimize RESTORE_CR3") for User CR3.

This series looks correct, but I don't think it's 5.8 material. I also want to try moving all this code to C, so it's possible this little series will become obsolete.

Quoted 17 lines

Lai Jiangshan (2):
x86/entry: Don't write to CR3 when restoring to kernel CR3 x86/entry: always flush user CR3 in RESTORE_CR3

 arch/x86/entry/calling.h  | 36 ++++++++----------------------------
 arch/x86/entry/entry_64.S |  6 +++---
 2 files changed, 11 insertions(+), 31 deletions(-)

--
2.20.1