Skip to contentSkip to conversation

Introduce a REQUIRE_NX Kconfig option

PATCH Superseded Alejandro Vallejo 3 participants 14 messages Jun 15, 2023 – Jun 23, 2023

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover Introduce a REQUIRE_NX Kconfig option

v2:
* Created assembly override for cases where XD_DISABLE is set
* Turned rdmsrl() into rdmsr_safe() when reading IA32_MSR_MISC_ENABLE in C

This option hardens Xen by forcing it to write secure (NX-enhanced) PTEs regardless of the runtime NX feature bit in boot_cpu_data. This prevents an attacker with partial write support from affecting Xen's PTE generation logic by overriding the NX feature flag. The patch asserts support for the NX bit in PTEs at boot time and if so short-circuits the cpu_has_nx macro to 1.

Alejandro Vallejo (2):
x86/boot: Clear XD_DISABLE from the early boot path
x86: Add Kconfig option to require NX bit support

 xen/arch/x86/Kconfig                  | 16 ++++++
 xen/arch/x86/boot/head.S              | 73 ++++++++++++++++++++++++---
 xen/arch/x86/boot/trampoline.S        |  3 +-
 xen/arch/x86/cpu/intel.c              | 32 ++++++------
 xen/arch/x86/efi/efi-boot.h           |  9 ++++
 xen/arch/x86/include/asm/cpufeature.h |  3 +-
 xen/arch/x86/include/asm/msr-index.h  |  2 +-
 7 files changed, 111 insertions(+), 27 deletions(-)
Signature · 1 line
2.34.1
#1 lore raw

PATCH v2 1/2 x86/boot: Clear XD_DISABLE from the early boot path

Intel CPUs have a bit in MSR_IA32_MISC_ENABLE that may prevent the NX bit from being advertised. Clear it unconditionally if we can't find the NX feature right away on boot.

The conditions for the MSR being read on early boot are (in this order):

* Long Mode is supported
* NX isn't advertised
* The vendor is Intel

The order of checks has been chosen carefully so a virtualized Xen on a hypervisor that doesn't emulate that MSR (but supports NX) doesn't triple fault trying to access the non-existing MSR.

While at it, make sure we use rdmsr_safe rather than rdmsrl in the Intel-specific init path so we don't accidentally crash if the MSR isn't emulated while Xen is virtualized.

---
 xen/arch/x86/boot/head.S             | 60 ++++++++++++++++++++++++----
 xen/arch/x86/cpu/intel.c             | 32 +++++++--------
 xen/arch/x86/include/asm/msr-index.h |  2 +-
 3 files changed, 69 insertions(+), 25 deletions(-)
xen/arch/x86/boot/head.S +52 -8
index 09bebf8635..ce62eae6f3 100644
@@ -142,8 +142,8 @@ efi_platform:
 
         .section .init.text, "ax", @progbits
 
-bad_cpu:
-        add     $sym_offs(.Lbad_cpu_msg),%esi   # Error message
+.Lbad_cpu:
+        add     $sym_offs(.Lbad_cpu_msg),%esi
         jmp     .Lget_vtb
 not_multiboot:
         add     $sym_offs(.Lbad_ldr_msg),%esi   # Error message
@@ -647,15 +647,59 @@ trampoline_setup:
         cpuid
 1:      mov     %edx, CPUINFO_FEATURE_OFFSET(X86_FEATURE_LM) + sym_esi(boot_cpu_data)
 
-        /* Check for NX. Adjust EFER setting if available. */
+        /* Check for availability of long mode. */
+        bt      $cpufeat_bit(X86_FEATURE_LM),%edx
+        jnc     .Lbad_cpu
+
+        /* Check for NX */
         bt      $cpufeat_bit(X86_FEATURE_NX), %edx
+        jc     .Lhas_nx_bit
+
+        /*
+         * NX appears to be unsupported, but it might be hidden.
+         *
+         * Intel CPUs (may) implement MSR_IA32_MISC_ENABLE. Among other
+         * things this MSR has a bit that artificially hides NX support in
+         * CPUID. Xen _really_ wants that feature enabled if present, so we
+         * have to determine if (a) the MSR exists and if so (b) clear the
+         * bit.
+         *
+         * For native boots this is perfectly fine because the MSR was
+         * introduced before Netburst, which was the first family to
+         * provide 64bit support. So we're safe simply accessing it as long
+         * as long mode support has already been checked.
+         *
+         * For the virtualized case the MSR might not be emulated though,
+         * so we make sure to do an initial check for NX in order to bypass
+         * this MSR read
+         */
+        xor     %eax,%eax
+        cpuid
+        cmpl    $X86_VENDOR_INTEL_EBX,%ebx
+        jnz     .Lno_nx_bit
+        cmpl    $X86_VENDOR_INTEL_EDX,%edx
+        jnz     .Lno_nx_bit
+        cmpl    $X86_VENDOR_INTEL_ECX,%ecx
+        jnz     .Lno_nx_bit
+
+        /* Clear the XD_DISABLE bit */
+        movl    $MSR_IA32_MISC_ENABLE, %ecx
+        rdmsr
+        btrl    $2, %edx
         jnc     1f
-        orb     $EFER_NXE >> 8, 1 + sym_esi(trampoline_efer)
-1:
+        wrmsr
+        orb     $MSR_IA32_MISC_ENABLE_XD_DISABLE >> 32, 4 + sym_esi(trampoline_misc_enable_off)
 
-        /* Check for availability of long mode. */
-        bt      $cpufeat_bit(X86_FEATURE_LM),%edx
-        jnc     bad_cpu
+1:      /* Check again for NX */
+        mov     $0x80000001,%eax
+        cpuid
+        bt      $cpufeat_bit(X86_FEATURE_NX), %edx
+        jnc     .Lno_nx_bit
+
+.Lhas_nx_bit:
+        /* Adjust EFER is NX is present */
+        orb     $EFER_NXE >> 8, 1 + sym_esi(trampoline_efer)
+.Lno_nx_bit:
 
         /* Stash TSC to calculate a good approximation of time-since-boot */
         rdtsc
xen/arch/x86/cpu/intel.c +16 -16
index 168cd58f36..46b0cd8dbb 100644
@@ -305,23 +305,23 @@ static void cf_check early_init_intel(struct cpuinfo_x86 *c)
 		c->x86_cache_alignment = 128;
 
 	/* Unmask CPUID levels and NX if masked: */
-	rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
-
-	disable = misc_enable & (MSR_IA32_MISC_ENABLE_LIMIT_CPUID |
-				 MSR_IA32_MISC_ENABLE_XD_DISABLE);
-	if (disable) {
-		wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable & ~disable);
-		bootsym(trampoline_misc_enable_off) |= disable;
-		bootsym(trampoline_efer) |= EFER_NXE;
-	}
+	if (rdmsr_safe(MSR_IA32_MISC_ENABLE, misc_enable) == 0) {
+		disable = misc_enable & (MSR_IA32_MISC_ENABLE_LIMIT_CPUID |
+					 MSR_IA32_MISC_ENABLE_XD_DISABLE);
+		if (disable) {
+			wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable & ~disable);
+			bootsym(trampoline_misc_enable_off) |= disable;
+			bootsym(trampoline_efer) |= EFER_NXE;
+		}
 
-	if (disable & MSR_IA32_MISC_ENABLE_LIMIT_CPUID)
-		printk(KERN_INFO "revised cpuid level: %d\n",
-		       cpuid_eax(0));
-	if (disable & MSR_IA32_MISC_ENABLE_XD_DISABLE) {
-		write_efer(read_efer() | EFER_NXE);
-		printk(KERN_INFO
-		       "re-enabled NX (Execute Disable) protection\n");
+		if (disable & MSR_IA32_MISC_ENABLE_LIMIT_CPUID)
+			printk(KERN_INFO "revised cpuid level: %d\n",
+			       cpuid_eax(0));
+		if (disable & MSR_IA32_MISC_ENABLE_XD_DISABLE) {
+			write_efer(read_efer() | EFER_NXE);
+			printk(KERN_INFO
+			       "re-enabled NX (Execute Disable) protection\n");
+		}
 	}
 
 	/* CPUID workaround for Intel 0F33/0F34 CPU */
xen/arch/x86/include/asm/msr-index.h +1 -1
index 2749e433d2..4f861c0bb4 100644
@@ -502,7 +502,7 @@
 #define MSR_IA32_MISC_ENABLE_MONITOR_ENABLE (1<<18)
 #define MSR_IA32_MISC_ENABLE_LIMIT_CPUID  (1<<22)
 #define MSR_IA32_MISC_ENABLE_XTPR_DISABLE (1<<23)
-#define MSR_IA32_MISC_ENABLE_XD_DISABLE	(1ULL << 34)
+#define MSR_IA32_MISC_ENABLE_XD_DISABLE   (_AC(1, ULL) << 34)
 
 #define MSR_IA32_TSC_DEADLINE		0x000006E0
 #define MSR_IA32_ENERGY_PERF_BIAS	0x000001b0
Signature · 1 line
2.34.1
#7 lore raw

PATCH v2 2/2 x86: Add Kconfig option to require NX bit support

In reply to Alejandro Vallejo

This option hardens Xen by forcing it to write secure (NX-enhanced) PTEs regardless of the runtime NX feature bit in boot_cpu_data. This prevents an attacker with partial write support from affecting Xen's PTE generation logic by overriding the NX feature flag. The patch asserts support for the NX bit in PTEs at boot time and if so short-circuits the cpu_has_nx macro to 1.

It has the nice benefit of replacing many instances of runtime checks with folded constants. This has several knock-on effects that improve codegen, saving 2.5KiB off the text section.

The config option defaults to OFF for compatibility with previous behaviour.

---
 xen/arch/x86/Kconfig                  | 16 ++++++++++++++++
 xen/arch/x86/boot/head.S              | 15 ++++++++++++++-
 xen/arch/x86/boot/trampoline.S        |  3 ++-
 xen/arch/x86/efi/efi-boot.h           |  9 +++++++++
 xen/arch/x86/include/asm/cpufeature.h |  3 ++-
 5 files changed, 43 insertions(+), 3 deletions(-)
xen/arch/x86/Kconfig +16 -0
index 406445a358..fa97d4cccc 100644
@@ -307,6 +307,22 @@ config MEM_SHARING
 	bool "Xen memory sharing support (UNSUPPORTED)" if UNSUPPORTED
 	depends on HVM
 
+config REQUIRE_NX
+	bool "Require NX bit support"
+	help
+	  No-eXecute (also called XD "eXecute Disable" and DEP "Data
+	  Execution Prevention") is a security feature designed originally
+	  to combat buffer overflow attacks by marking regions of memory
+	  which the CPU must not interpret as instructions.
+
+	  The NX feature exists in every 64bit CPU except for some very
+	  early Pentium 4 Prescott machines.
+
+	  Enabling this option will improve Xen's security by removing
+	  cases where Xen could be tricked into thinking that the feature
+	  was unavailable. However, if enabled, Xen will no longer boot on
+	  any CPU which is lacking NX support.
+
 endmenu
 
 source "common/Kconfig"
xen/arch/x86/boot/head.S +14 -1
index ce62eae6f3..ec1e80ef68 100644
@@ -123,6 +123,7 @@ multiboot2_header:
 .Lbad_ldr_nih: .asciz "ERR: EFI ImageHandle is not provided by bootloader!"
 .Lbad_efi_msg: .asciz "ERR: EFI IA-32 platforms are not supported!"
 .Lbag_alg_msg: .asciz "ERR: Xen must be loaded at a 2Mb boundary!"
+.Lno_nx_bit_msg: .asciz "ERR: Not an NX-bit capable CPU!"
 
         .section .init.data, "aw", @progbits
         .align 4
@@ -151,6 +152,11 @@ not_multiboot:
 .Lnot_aligned:
         add     $sym_offs(.Lbag_alg_msg),%esi   # Error message
         jmp     .Lget_vtb
+#ifdef CONFIG_REQUIRE_NX
+.Lno_nx_bit:
+        add     $sym_offs(.Lno_nx_bit_msg),%esi
+        jmp     .Lget_vtb
+#endif
 .Lmb2_no_st:
         /*
          * Here we are on EFI platform. vga_text_buffer was zapped earlier
@@ -651,7 +657,12 @@ trampoline_setup:
         bt      $cpufeat_bit(X86_FEATURE_LM),%edx
         jnc     .Lbad_cpu
 
-        /* Check for NX */
+        /*
+         * Check for NX
+         *   - If Xen was compiled requiring it simply assert it's
+         *     supported. The trampoline already has the right constant.
+         *   - Otherwise, update the trampoline EFER mask accordingly.
+         */
         bt      $cpufeat_bit(X86_FEATURE_NX), %edx
         jc     .Lhas_nx_bit
 
@@ -697,9 +708,11 @@ trampoline_setup:
         jnc     .Lno_nx_bit
 
 .Lhas_nx_bit:
+#ifndef CONFIG_REQUIRE_NX
         /* Adjust EFER is NX is present */
         orb     $EFER_NXE >> 8, 1 + sym_esi(trampoline_efer)
 .Lno_nx_bit:
+#endif
 
         /* Stash TSC to calculate a good approximation of time-since-boot */
         rdtsc
xen/arch/x86/boot/trampoline.S +2 -1
index c6005fa33d..b8ab0ffdcb 100644
@@ -147,7 +147,8 @@ GLOBAL(trampoline_misc_enable_off)
 
 /* EFER OR-mask for boot paths.  SCE conditional on PV support, NX added when available. */
 GLOBAL(trampoline_efer)
-        .long   EFER_LME | (EFER_SCE * IS_ENABLED(CONFIG_PV))
+        .long   EFER_LME | (EFER_SCE * IS_ENABLED(CONFIG_PV)) | \
+                (EFER_NXE * IS_ENABLED(CONFIG_REQUIRE_NX))
 
 GLOBAL(trampoline_xen_phys_start)
         .long   0
xen/arch/x86/efi/efi-boot.h +9 -0
index c94e53d139..84700559bb 100644
@@ -751,6 +751,15 @@ static void __init efi_arch_cpu(void)
     {
         caps[FEATURESET_e1d] = cpuid_edx(0x80000001);
 
+        /*
+         * This check purposefully doesn't use cpu_has_nx because
+         * cpu_has_nx bypasses the boot_cpu_data read if Xen was compiled
+         * with CONFIG_REQUIRE_NX
+         */
+        if ( IS_ENABLED(CONFIG_REQUIRE_NX) &&
+             !boot_cpu_has(X86_FEATURE_NX) )
+            blexit(L"This Xen build requires NX bit support.");
+
         if ( cpu_has_nx )
             trampoline_efer |= EFER_NXE;
     }
xen/arch/x86/include/asm/cpufeature.h +2 -1
index ace31e3b1f..610532da43 100644
@@ -91,7 +91,8 @@ static inline bool boot_cpu_has(unsigned int feat)
 #define cpu_has_hypervisor      boot_cpu_has(X86_FEATURE_HYPERVISOR)
 
 /* CPUID level 0x80000001.edx */
-#define cpu_has_nx              boot_cpu_has(X86_FEATURE_NX)
+#define cpu_has_nx              (IS_ENABLED(CONFIG_REQUIRE_NX) || \
+                                 boot_cpu_has(X86_FEATURE_NX))
 #define cpu_has_page1gb         boot_cpu_has(X86_FEATURE_PAGE1GB)
 #define cpu_has_rdtscp          boot_cpu_has(X86_FEATURE_RDTSCP)
 #define cpu_has_3dnow_ext       boot_cpu_has(X86_FEATURE_3DNOWEXT)
Signature · 1 line
2.34.1
#10 lore raw

PATCH 1/1 x86/boot: Clean up early error asm

In reply to Alejandro Vallejo

The asm forming early error handling is a mix of local and non-local symbols, and has some pointless comments. Drop the "# Error message" comments, tweaking the style on modified lines, and make the symbols local.

However, leave behind one real symbol so this logic disassembles nicely without merging in to acpi_boot_init(), which is the thing that happens to be immediately prior in my build.

No functional change.

---
CC: Jan Beulich <[email protected]>
CC: Roger Pau Monné <[email protected]>
CC: Wei Liu <[email protected]>
CC: Alejandro Vallejo <[email protected]>

Done in order to simplfy Alejandro's NX series a little.

---
 xen/arch/x86/boot/head.S | 33 +++++++++++++++++++--------------
 1 file changed, 19 insertions(+), 14 deletions(-)
xen/arch/x86/boot/head.S +19 -14
index 09bebf8635d0..d52dbc752e29 100644
@@ -142,25 +142,27 @@ efi_platform:
 
         .section .init.text, "ax", @progbits
 
-bad_cpu:
-        add     $sym_offs(.Lbad_cpu_msg),%esi   # Error message
+early_error:
+
+.Lbad_cpu:
+        add     $sym_offs(.Lbad_cpu_msg), %esi
         jmp     .Lget_vtb
-not_multiboot:
-        add     $sym_offs(.Lbad_ldr_msg),%esi   # Error message
+.Lnot_multiboot:
+        add     $sym_offs(.Lbad_ldr_msg), %esi
         jmp     .Lget_vtb
 .Lnot_aligned:
-        add     $sym_offs(.Lbag_alg_msg),%esi   # Error message
+        add     $sym_offs(.Lbag_alg_msg), %esi
         jmp     .Lget_vtb
 .Lmb2_no_st:
         /*
          * Here we are on EFI platform. vga_text_buffer was zapped earlier
          * because there is pretty good chance that VGA is unavailable.
          */
-        add     $sym_offs(.Lbad_ldr_nst),%esi   # Error message
+        add     $sym_offs(.Lbad_ldr_nst), %esi
         jmp     .Lget_vtb
 .Lmb2_no_ih:
         /* Ditto. */
-        add     $sym_offs(.Lbad_ldr_nih),%esi   # Error message
+        add     $sym_offs(.Lbad_ldr_nih), %esi
         jmp     .Lget_vtb
 .Lmb2_no_bs:
         /*
@@ -168,7 +170,7 @@ not_multiboot:
          * via start label. Then reliable vga_text_buffer zap is impossible
          * in Multiboot2 scanning loop and we have to zero %edi below.
          */
-        add     $sym_offs(.Lbad_ldr_nbs),%esi   # Error message
+        add     $sym_offs(.Lbad_ldr_nbs), %esi
         xor     %edi,%edi                       # No VGA text buffer
         jmp     .Lprint_err
 .Lmb2_efi_ia_32:
@@ -176,11 +178,11 @@ not_multiboot:
          * Here we are on EFI IA-32 platform. Then reliable vga_text_buffer zap is
          * impossible in Multiboot2 scanning loop and we have to zero %edi below.
          */
-        add     $sym_offs(.Lbad_efi_msg),%esi   # Error message
+        add     $sym_offs(.Lbad_efi_msg), %esi
         xor     %edi,%edi                       # No VGA text buffer
         jmp     .Lprint_err
 .Lget_vtb:
-        mov     sym_esi(vga_text_buffer),%edi
+        mov     sym_esi(vga_text_buffer), %edi
 .Lprint_err:
         lodsb
         test    %al,%al        # Terminate on '\0' sentinel
@@ -202,6 +204,9 @@ not_multiboot:
 .Lhalt: hlt
         jmp     .Lhalt
 
+        .size early_error, . - early_error
+        .type early_error, @function
+
         .code64
 
 __efi64_mb2_start:
@@ -221,8 +226,8 @@ __efi64_mb2_start:
         cmp     $MULTIBOOT2_BOOTLOADER_MAGIC,%eax
         je      .Lefi_multiboot2_proto
 
-        /* Jump to not_multiboot after switching CPU to x86_32 mode. */
-        lea     not_multiboot(%rip),%r15
+        /* Jump to .Lnot_multiboot after switching CPU to x86_32 mode. */
+        lea     .Lnot_multiboot(%rip), %r15
         jmp     x86_32_switch
 
 .Lefi_multiboot2_proto:
@@ -464,7 +469,7 @@ __start:
 
         /* Check for Multiboot bootloader. */
         cmp     $MULTIBOOT_BOOTLOADER_MAGIC,%eax
-        jne     not_multiboot
+        jne     .Lnot_multiboot
 
         /* Get mem_lower from Multiboot information. */
         testb   $MBI_MEMLIMITS,MB_flags(%ebx)
@@ -655,7 +660,7 @@ trampoline_setup:
 
         /* Check for availability of long mode. */
         bt      $cpufeat_bit(X86_FEATURE_LM),%edx
-        jnc     bad_cpu
+        jnc     .Lbad_cpu
 
         /* Stash TSC to calculate a good approximation of time-since-boot */
         rdtsc
Signature · 1 line
2.30.2
#2 lore raw

Re: [PATCH v2 1/2] x86/boot: Clear XD_DISABLE from the early boot path

In reply to Alejandro Vallejo
Quoted 17 lines · Alejandro Vallejo
xen/arch/x86/boot/head.S +2 -2
index 09bebf8635..ce62eae6f3 100644
@@ -142,8 +142,8 @@ efi_platform:
 
         .section .init.text, "ax", @progbits
 
-bad_cpu:
-        add     $sym_offs(.Lbad_cpu_msg),%esi   # Error message
+.Lbad_cpu:
+        add     $sym_offs(.Lbad_cpu_msg),%esi
         jmp     .Lget_vtb
 not_multiboot:
         add     $sym_offs(.Lbad_ldr_msg),%esi   # Error message
@@ -647,15 +647,59 @@ trampoline_setup:
         cpuid

xen/arch/x86/boot/head.S trampoline_setup: · line 656

Quoted 10 lines
xen/arch/x86/boot/head.S +6 -1
 1:      mov     %edx, CPUINFO_FEATURE_OFFSET(X86_FEATURE_LM) + sym_esi(boot_cpu_data)
 
-        /* Check for NX. Adjust EFER setting if available. */
+        /* Check for availability of long mode. */
+        bt      $cpufeat_bit(X86_FEATURE_LM),%edx
+        jnc     .Lbad_cpu
+
+        /* Check for NX */
         bt      $cpufeat_bit(X86_FEATURE_NX), %edx
+        jc     .Lhas_nx_bit

This part of the diff is far more legible when rebased over the 0.5/2 patch I've just emailed out.  I've got the rebase locally if you want it.

Although I'd suggest that this label name be .Lgot_nx.

Quoted 8 lines
xen/arch/x86/boot/head.S +8 -0
+
+        /*
+         * NX appears to be unsupported, but it might be hidden.
+         *
+         * Intel CPUs (may) implement MSR_IA32_MISC_ENABLE. Among other
+         * things this MSR has a bit that artificially hides NX support in
+         * CPUID. Xen _really_ wants that feature enabled if present, so we
+         * have to determine if (a) the MSR exists and if so (b) clear the

xen/arch/x86/boot/head.S trampoline_setup: · line 674

Quoted 10 lines
xen/arch/x86/boot/head.S +10 -0
+         * bit.
+         *
+         * For native boots this is perfectly fine because the MSR was
+         * introduced before Netburst, which was the first family to
+         * provide 64bit support. So we're safe simply accessing it as long
+         * as long mode support has already been checked.
+         *
+         * For the virtualized case the MSR might not be emulated though,
+         * so we make sure to do an initial check for NX in order to bypass
+         * this MSR read

I'd suggest reordering this a bit, and swapping some what for why.  How about:

---
NX appears to be unsupported, but it might be hidden.

The feature is part of the AMD64 spec, but the very first Intel 64bit CPUs lacked the feature, and thereafter there was a firmware knob to disable the feature.  Undo the disable if possible.

All 64bit Intel CPUs support this MSR.  If virtualised, expect the hypervisor to either emulate the MSR or give us NX.
---

xen/arch/x86/boot/head.S trampoline_setup: · line 678

Quoted 4 lines
xen/arch/x86/boot/head.S +4 -0
+         */
+        xor     %eax,%eax
+        cpuid
+        cmpl    $X86_VENDOR_INTEL_EBX,%ebx

Where possible, spaces after commas and without size suffixes (the l on cmpl here).  While not relevant here

xen/arch/x86/boot/head.S trampoline_setup: · line 679

Quoted 1 line
xen/arch/x86/boot/head.S +1 -0
+        jnz     .Lno_nx_bit

The "_bit" is still redundant.  Simply .Lno_nx will do.

xen/arch/x86/boot/head.S trampoline_setup: · line 688

Quoted 9 lines
xen/arch/x86/boot/head.S +9 -0
+        cmpl    $X86_VENDOR_INTEL_EDX,%edx
+        jnz     .Lno_nx_bit
+        cmpl    $X86_VENDOR_INTEL_ECX,%ecx
+        jnz     .Lno_nx_bit
+
+        /* Clear the XD_DISABLE bit */
+        movl    $MSR_IA32_MISC_ENABLE, %ecx
+        rdmsr
+        btrl    $2, %edx

It's unfortunate that we don't have an asm-safe ilog2().  Oh well.

xen/arch/x86/boot/head.S trampoline_setup: · line 693

Quoted 10 lines
xen/arch/x86/boot/head.S +3 -5
         jnc     1f
-        orb     $EFER_NXE >> 8, 1 + sym_esi(trampoline_efer)
-1:
+        wrmsr
+        orb     $MSR_IA32_MISC_ENABLE_XD_DISABLE >> 32, 4 + sym_esi(trampoline_misc_enable_off)
 
-        /* Check for availability of long mode. */
-        bt      $cpufeat_bit(X86_FEATURE_LM),%edx
-        jnc     bad_cpu
+1:      /* Check again for NX */

Failing to clear XD_DISABLE wants to go to .Lno_nx, not to re-scan CPUID having done nothing to the MSR.

xen/arch/x86/boot/head.S trampoline_setup: · line 700

Quoted 7 lines
xen/arch/x86/boot/head.S +7 -0
+        mov     $0x80000001,%eax
+        cpuid
+        bt      $cpufeat_bit(X86_FEATURE_NX), %edx
+        jnc     .Lno_nx_bit
+
+.Lhas_nx_bit:
+        /* Adjust EFER is NX is present */

"Adjust EFER, given that NX is present".

Quoted 8 lines
xen/arch/x86/cpu/intel.c +0 -0
index 168cd58f36..46b0cd8dbb 100644
@@ -305,23 +305,23 @@ static void cf_check early_init_intel(struct cpuinfo_x86 *c)
 		c->x86_cache_alignment = 128;
 
 	/* Unmask CPUID levels and NX if masked: */

xen/arch/x86/cpu/intel.c static void cf_check early_init_intel(struct cpuinfo_x86 *c) · line 308

Quoted 10 lines
xen/arch/x86/cpu/intel.c +1 -9
-	rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
-
-	disable = misc_enable & (MSR_IA32_MISC_ENABLE_LIMIT_CPUID |
-				 MSR_IA32_MISC_ENABLE_XD_DISABLE);
-	if (disable) {
-		wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable & ~disable);
-		bootsym(trampoline_misc_enable_off) |= disable;
-		bootsym(trampoline_efer) |= EFER_NXE;
-	}
+	if (rdmsr_safe(MSR_IA32_MISC_ENABLE, misc_enable) == 0) {

There's no need to change rdmsrl() to rdmsr_safe(), and not doing so will shrink this diff a lot.

The only thing you need to alter the re-enable NX printk(), which probably wants to move ahead of the "if (disable)" and source itself from bootsym(trampoline_misc_enable_off) instead.

~Andrew

#8 lore raw

Re: [PATCH v2 2/2] x86: Add Kconfig option to require NX bit support

In reply to Alejandro Vallejo

xen/arch/x86/Kconfig config MEM_SHARING · line 311

Quoted 10 lines · Alejandro Vallejo
xen/arch/x86/Kconfig +2 -0
index 406445a358..fa97d4cccc 100644
@@ -307,6 +307,22 @@ config MEM_SHARING
 	bool "Xen memory sharing support (UNSUPPORTED)" if UNSUPPORTED
 	depends on HVM
 
+config REQUIRE_NX
+	bool "Require NX bit support"

"Require NX (No eXecute) support".

Quoted 22 lines
xen/arch/x86/Kconfig +14 -0
+	help
+	  No-eXecute (also called XD "eXecute Disable" and DEP "Data
+	  Execution Prevention") is a security feature designed originally
+	  to combat buffer overflow attacks by marking regions of memory
+	  which the CPU must not interpret as instructions.
+
+	  The NX feature exists in every 64bit CPU except for some very
+	  early Pentium 4 Prescott machines.
+
+	  Enabling this option will improve Xen's security by removing
+	  cases where Xen could be tricked into thinking that the feature
+	  was unavailable. However, if enabled, Xen will no longer boot on
+	  any CPU which is lacking NX support.
+
 endmenu
 
 source "common/Kconfig"
xen/arch/x86/boot/head.S +0 -0
index ce62eae6f3..ec1e80ef68 100644

xen/arch/x86/boot/head.S multiboot2_header: · line 126

Quoted 4 lines
xen/arch/x86/boot/head.S +1 -0
@@ -123,6 +123,7 @@ multiboot2_header:
 .Lbad_ldr_nih: .asciz "ERR: EFI ImageHandle is not provided by bootloader!"
 .Lbad_efi_msg: .asciz "ERR: EFI IA-32 platforms are not supported!"
 .Lbag_alg_msg: .asciz "ERR: Xen must be loaded at a 2Mb boundary!"
+.Lno_nx_bit_msg: .asciz "ERR: Not an NX-bit capable CPU!"

Still two too many "bit"'s in this line.

With these two adjusted, Reviewed-by: Andrew Cooper <[email protected]>

Reviewed-by on v1 1/1
#11 lore raw

Re: [PATCH 0.5/2] x86/boot: Clean up early error asm

In reply to Andrew Cooper
Quoted 11 lines · Andrew Cooper

The asm forming early error handling is a mix of local and non-local symbols, and has some pointless comments. Drop the "# Error message" comments, tweaking the style on modified lines, and make the symbols local.

However, leave behind one real symbol so this logic disassembles nicely without merging in to acpi_boot_init(), which is the thing that happens to be immediately prior in my build.

No functional change.

with one request:

xen/arch/x86/boot/head.S efi_platform: · line 145

Quoted 9 lines
xen/arch/x86/boot/head.S +1 -2
@@ -142,25 +142,27 @@ efi_platform:
 
         .section .init.text, "ax", @progbits
 
-bad_cpu:
-        add     $sym_offs(.Lbad_cpu_msg),%esi   # Error message
+early_error:

This symbol, which isn't really used except by .size/.type below, could do with a comment saying that it's intentionally here despite having no real reference.

xen/arch/x86/boot/head.S not_multiboot: · line 208

Quoted 6 lines
xen/arch/x86/boot/head.S +2 -0
@@ -202,6 +204,9 @@ not_multiboot:
 .Lhalt: hlt
         jmp     .Lhalt
 
+        .size early_error, . - early_error
+        .type early_error, @function

At the 1st and 2nd glance this looks unrelated to this patch. Which isn't to say I'm opposed, but of course once we have settled on an annotation model, it'll need touching again anyway.

Jan

#9 lore raw

Re: [PATCH v2 2/2] x86: Add Kconfig option to require NX bit support

In reply to Alejandro Vallejo
Quoted 7 lines · Alejandro Vallejo

This option hardens Xen by forcing it to write secure (NX-enhanced) PTEs regardless of the runtime NX feature bit in boot_cpu_data. This prevents an attacker with partial write support from affecting Xen's PTE generation logic by overriding the NX feature flag. The patch asserts support for the NX bit in PTEs at boot time and if so short-circuits the cpu_has_nx macro to 1.

Quoted 8 lines

It has the nice benefit of replacing many instances of runtime checks with folded constants. This has several knock-on effects that improve codegen, saving 2.5KiB off the text section.

The config option defaults to OFF for compatibility with previous behaviour.

Just one nit on top of Andrew's comments:

xen/arch/x86/boot/head.S trampoline_setup: · line 715

Quoted 9 lines
xen/arch/x86/boot/head.S +2 -0
@@ -697,9 +708,11 @@ trampoline_setup:
         jnc     .Lno_nx_bit
 
 .Lhas_nx_bit:
+#ifndef CONFIG_REQUIRE_NX
         /* Adjust EFER is NX is present */
         orb     $EFER_NXE >> 8, 1 + sym_esi(trampoline_efer)
 .Lno_nx_bit:
+#endif

In the comment the first "is" likely was meant to be "if".

Jan

#12 lore raw

Re: [PATCH 0.5/2] x86/boot: Clean up early error asm

In reply to Jan Beulich
Quoted 5 lines · Jan Beulich
Quoted 4 lines · Andrew Cooper

The asm forming early error handling is a mix of local and non-local symbols, and has some pointless comments. Drop the "# Error message" comments, tweaking the style on modified lines, and make the symbols local.

Quoted 8 lines
Quoted 7 lines

However, leave behind one real symbol so this logic disassembles nicely without merging in to acpi_boot_init(), which is the thing that happens to be immediately prior in my build.

No functional change.

Thanks.

Quoted 6 lines

with one request:

Quoted 4 lines
xen/arch/x86/boot/head.S +0 -0
@@ -142,25 +142,27 @@ efi_platform:
 
Quoted 8 lines

xen/arch/x86/boot/head.S efi_platform: · line 145

Quoted 5 lines
xen/arch/x86/boot/head.S +1 -2
         .section .init.text, "ax", @progbits
 
-bad_cpu:
-        add     $sym_offs(.Lbad_cpu_msg),%esi   # Error message
+early_error:

This symbol, which isn't really used except by .size/.type below, could do with a comment saying that it's intentionally here despite having no real reference.

Ok.  /* Here to make the disassembly better. */ ?

The .Lbad_cpu jmp will end up disassembling to early_error+0, so it is technically referenced, but I see your point.

Quoted 9 lines

xen/arch/x86/boot/head.S not_multiboot: · line 208

Quoted 6 lines
xen/arch/x86/boot/head.S +2 -0
@@ -202,6 +204,9 @@ not_multiboot:
 .Lhalt: hlt
         jmp     .Lhalt
 
+        .size early_error, . - early_error
+        .type early_error, @function

At the 1st and 2nd glance this looks unrelated to this patch. Which isn't to say I'm opposed, but of course once we have settled on an annotation model, it'll need touching again anyway.

I know it's going to need changing, but in the short term I'm at least making sure the metadata is correct on new additions to asm code.

~Andrew

#13 lore raw

Re: [PATCH 0.5/2] x86/boot: Clean up early error asm

Quoted 25 lines · Andrew Cooper
Quoted 5 lines · Jan Beulich
Quoted 4 lines · Andrew Cooper

The asm forming early error handling is a mix of local and non-local symbols, and has some pointless comments. Drop the "# Error message" comments, tweaking the style on modified lines, and make the symbols local.

Quoted 8 lines
Quoted 7 lines

However, leave behind one real symbol so this logic disassembles nicely without merging in to acpi_boot_init(), which is the thing that happens to be immediately prior in my build.

No functional change.

Thanks.

Quoted 8 lines

with one request:

Quoted 6 lines
xen/arch/x86/boot/head.S +0 -0
@@ -142,25 +142,27 @@ efi_platform:
 
         .section .init.text, "ax", @progbits
 
Quoted 8 lines
Quoted 6 lines

xen/arch/x86/boot/head.S efi_platform: · line 145

Quoted 3 lines
xen/arch/x86/boot/head.S +1 -2
-bad_cpu:
-        add     $sym_offs(.Lbad_cpu_msg),%esi   # Error message
+early_error:

This symbol, which isn't really used except by .size/.type below, could do with a comment saying that it's intentionally here despite having no real reference.

Ok.  /* Here to make the disassembly better. */ ?

Yes please.

Jan

#3 lore raw

Re: [PATCH v2 1/2] x86/boot: Clear XD_DISABLE from the early boot path

In reply to Andrew Cooper

Sure, to everything before this

Quoted 7 lines
Quoted 7 lines
xen/arch/x86/cpu/intel.c +0 -0
index 168cd58f36..46b0cd8dbb 100644
@@ -305,23 +305,23 @@ static void cf_check early_init_intel(struct cpuinfo_x86 *c)
 		c->x86_cache_alignment = 128;
 
Quoted 13 lines

xen/arch/x86/cpu/intel.c static void cf_check early_init_intel(struct cpuinfo_x86 *c) · line 308

Quoted 11 lines
xen/arch/x86/cpu/intel.c +1 -9
 	/* Unmask CPUID levels and NX if masked: */
-	rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
-
-	disable = misc_enable & (MSR_IA32_MISC_ENABLE_LIMIT_CPUID |
-				 MSR_IA32_MISC_ENABLE_XD_DISABLE);
-	if (disable) {
-		wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable & ~disable);
-		bootsym(trampoline_misc_enable_off) |= disable;
-		bootsym(trampoline_efer) |= EFER_NXE;
-	}
+	if (rdmsr_safe(MSR_IA32_MISC_ENABLE, misc_enable) == 0) {

There's no need to change rdmsrl() to rdmsr_safe(),

I thought we established before some hypervisors might not implement it. In that case this function would crash. More gracefully than a triple fault, sure, but still not a very friendly thing to do.

Quoted 1 line

and not doing so will shrink this diff a lot.

This particular one, yes. That said, looking more carefully I'd say I should remove the XD_DISABLE parts of those lines altogether. It is unconditionally cleared on boot now and will never ever have a non-zero value at this point.

Quoted 3 lines

The only thing you need to alter the re-enable NX printk(), which probably wants to move ahead of the "if (disable)" and source itself from bootsym(trampoline_misc_enable_off) instead.

Alternatively I could just get rid of the printk. As it is after this patch even the BSP starts off with NX enabled. EFER is updated as early as the trampoline, which is as early as it would've happened before regardless of XD_DISABLE.

Alejandro

#4 lore raw

Re: [PATCH v2 1/2] x86/boot: Clear XD_DISABLE from the early boot path

Quoted 22 lines · Alejandro Vallejo

Sure, to everything before this

Quoted 7 lines
Quoted 7 lines
xen/arch/x86/cpu/intel.c +0 -0
index 168cd58f36..46b0cd8dbb 100644
@@ -305,23 +305,23 @@ static void cf_check early_init_intel(struct cpuinfo_x86 *c)
 		c->x86_cache_alignment = 128;
 
Quoted 13 lines

xen/arch/x86/cpu/intel.c static void cf_check early_init_intel(struct cpuinfo_x86 *c) · line 308

Quoted 11 lines
xen/arch/x86/cpu/intel.c +1 -9
 	/* Unmask CPUID levels and NX if masked: */
-	rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
-
-	disable = misc_enable & (MSR_IA32_MISC_ENABLE_LIMIT_CPUID |
-				 MSR_IA32_MISC_ENABLE_XD_DISABLE);
-	if (disable) {
-		wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable & ~disable);
-		bootsym(trampoline_misc_enable_off) |= disable;
-		bootsym(trampoline_efer) |= EFER_NXE;
-	}
+	if (rdmsr_safe(MSR_IA32_MISC_ENABLE, misc_enable) == 0) {

There's no need to change rdmsrl() to rdmsr_safe(),

Quoted 3 lines

I thought we established before some hypervisors might not implement it. In that case this function would crash. More gracefully than a triple fault, sure, but still not a very friendly thing to do.

Yet then in early boot code you also don't (and can't) recover from getting #GP(0), in case that might really happen.

Jan

#5 lore raw

Re: [PATCH v2 1/2] x86/boot: Clear XD_DISABLE from the early boot path

Quoted 23 lines · Jan Beulich
Quoted 22 lines · Alejandro Vallejo

Sure, to everything before this

Quoted 20 lines
Quoted 8 lines
xen/arch/x86/cpu/intel.c +0 -0
index 168cd58f36..46b0cd8dbb 100644
@@ -305,23 +305,23 @@ static void cf_check early_init_intel(struct cpuinfo_x86 *c)
 		c->x86_cache_alignment = 128;
 
 	/* Unmask CPUID levels and NX if masked: */

xen/arch/x86/cpu/intel.c static void cf_check early_init_intel(struct cpuinfo_x86 *c) · line 308

Quoted 10 lines
xen/arch/x86/cpu/intel.c +1 -9
-	rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
-
-	disable = misc_enable & (MSR_IA32_MISC_ENABLE_LIMIT_CPUID |
-				 MSR_IA32_MISC_ENABLE_XD_DISABLE);
-	if (disable) {
-		wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable & ~disable);
-		bootsym(trampoline_misc_enable_off) |= disable;
-		bootsym(trampoline_efer) |= EFER_NXE;
-	}
+	if (rdmsr_safe(MSR_IA32_MISC_ENABLE, misc_enable) == 0) {

There's no need to change rdmsrl() to rdmsr_safe(),

Quoted 8 lines
Quoted 3 lines

I thought we established before some hypervisors might not implement it. In that case this function would crash. More gracefully than a triple fault, sure, but still not a very friendly thing to do.

Yet then in early boot code you also don't (and can't) recover from getting #GP(0), in case that might really happen.

Jan

That's true, strictly speaking, but that case is restricted to the incredibly unlikely case of "no NX and no MSR". As is, if we ever boot on an Intel machine without that MSR we'll hit #GP(0), regardless of NX.

Not that it matters a whole lot though. It's pretty unlikely we'll ever trip there.

Alejandro

#6 lore raw

Re: [PATCH v2 1/2] x86/boot: Clear XD_DISABLE from the early boot path

Quoted 28 lines · Alejandro Vallejo
Quoted 27 lines · Jan Beulich
Quoted 25 lines · Alejandro Vallejo

Sure, to everything before this

Quoted 7 lines
Quoted 7 lines
xen/arch/x86/cpu/intel.c +0 -0
index 168cd58f36..46b0cd8dbb 100644
@@ -305,23 +305,23 @@ static void cf_check early_init_intel(struct cpuinfo_x86 *c)
 		c->x86_cache_alignment = 128;
 
Quoted 13 lines

xen/arch/x86/cpu/intel.c static void cf_check early_init_intel(struct cpuinfo_x86 *c) · line 308

Quoted 11 lines
xen/arch/x86/cpu/intel.c +1 -9
 	/* Unmask CPUID levels and NX if masked: */
-	rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
-
-	disable = misc_enable & (MSR_IA32_MISC_ENABLE_LIMIT_CPUID |
-				 MSR_IA32_MISC_ENABLE_XD_DISABLE);
-	if (disable) {
-		wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable & ~disable);
-		bootsym(trampoline_misc_enable_off) |= disable;
-		bootsym(trampoline_efer) |= EFER_NXE;
-	}
+	if (rdmsr_safe(MSR_IA32_MISC_ENABLE, misc_enable) == 0) {

There's no need to change rdmsrl() to rdmsr_safe(),

I thought we established before some hypervisors might not implement it. In that case this function would crash. More gracefully than a triple fault, sure, but still not a very friendly thing to do.

Quoted 6 lines
Quoted 3 lines

Yet then in early boot code you also don't (and can't) recover from getting #GP(0), in case that might really happen.

That's true, strictly speaking, but that case is restricted to the incredibly unlikely case of "no NX and no MSR". As is, if we ever boot on an Intel machine without that MSR we'll hit #GP(0), regardless of NX.

Well, my comment was meant to support Andrew's earlier one. Why check the MSR access here when we don't earlier on? Plus isn't it the case that the MSR can be implied to exist when NX is (in principle) available, as that "opt-out" existed specifically when Intel first introduced NX on their CPUs? If so,
- when NX is not available, we would have crashed much earlier,
- when NX is available (or was turned on successfully) the MSR exists either because we wrote to it earlier or because we're on a newer CPU.

Jan