Skip to contentSkip to conversation

mm, x86/cc: Implement support for unaccepted memory

PATCH Superseded Kirill A. Shutemov 21 participants 200 messages Jun 14, 2022 – Dec 1, 2022 Page 2 of 2

Subsystems (16)

v7 of 14 Superseded Latest is v14 →

lore
Share & tools
lore
#187 lore raw

PATCH v4 4/4 x86/sev: Add SNP-specific unaccepted memory support

In reply to Tom Lendacky

Add SNP-specific hooks to the unaccepted memory support in the boot path (__accept_memory()) and the core kernel (accept_memory()) in order to support booting SNP guests when unaccepted memory is present. Without this support, SNP guests will fail to boot and/or panic() when unaccepted memory is present in the EFI memory map.

The process of accepting memory under SNP involves invoking the hypervisor to perform a page state change for the page to private memory and then issuing a PVALIDATE instruction to accept the page.

Since the boot path and the core kernel paths perform similar operations, move the pvalidate_pages() and vmgexit_psc() functions into sev-shared.c to avoid code duplication.

Create the new header file arch/x86/boot/compressed/sev.h because adding the function declaration to any of the existing SEV related header files pulls in too many other header files, causing the build to fail.

---
 arch/x86/Kconfig                |   1 +
 arch/x86/boot/compressed/mem.c  |   3 +
 arch/x86/boot/compressed/sev.c  |  54 ++++++++++++++-
 arch/x86/boot/compressed/sev.h  |  23 +++++++
 arch/x86/include/asm/sev.h      |   3 +
 arch/x86/kernel/sev-shared.c    | 104 +++++++++++++++++++++++++++++
 arch/x86/kernel/sev.c           | 112 ++++----------------------------
 arch/x86/mm/unaccepted_memory.c |   4 ++
 8 files changed, 205 insertions(+), 99 deletions(-)
 create mode 100644 arch/x86/boot/compressed/sev.h
arch/x86/Kconfig +1 -0
index 34146ecc5bdd..0ad53c3533c2 100644
@@ -1553,6 +1553,7 @@ config AMD_MEM_ENCRYPT
 	select INSTRUCTION_DECODER
 	select ARCH_HAS_CC_PLATFORM
 	select X86_MEM_ENCRYPT
+	select UNACCEPTED_MEMORY
 	help
 	  Say yes to enable support for the encryption of system memory.
 	  This requires an AMD processor that supports Secure Memory
arch/x86/boot/compressed/mem.c +3 -0
index 48e36e640da1..3e19dc0da0d7 100644
@@ -6,6 +6,7 @@
 #include "find.h"
 #include "math.h"
 #include "tdx.h"
+#include "sev.h"
 #include <asm/shared/tdx.h>
 
 #define PMD_SHIFT	21
@@ -39,6 +40,8 @@ static inline void __accept_memory(phys_addr_t start, phys_addr_t end)
 	/* Platform-specific memory-acceptance call goes here */
 	if (is_tdx_guest())
 		tdx_accept_memory(start, end);
+	else if (sev_snp_enabled())
+		snp_accept_memory(start, end);
 	else
 		error("Cannot accept memory: unknown platform\n");
 }
arch/x86/boot/compressed/sev.c +53 -1

66 lines not on this page · raw message

arch/x86/boot/compressed/sev.h new file +23 -0
new file mode 100644
index 000000000000..fc725a981b09
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * AMD SEV header for early boot related functions.
+ *
+ * Author: Tom Lendacky <[email protected]>
+ */
+
+#ifndef BOOT_COMPRESSED_SEV_H
+#define BOOT_COMPRESSED_SEV_H
+
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+
+bool sev_snp_enabled(void);
+void snp_accept_memory(phys_addr_t start, phys_addr_t end);
+
+#else
+
+static inline bool sev_snp_enabled(void) { return false; }
+static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { }
+
+#endif
+
+#endif
arch/x86/include/asm/sev.h +3 -0
index 0007ab04ac5f..9297aab0c79e 100644
@@ -206,6 +206,7 @@ void snp_set_wakeup_secondary_cpu(void);
 bool snp_init(struct boot_params *bp);
 void snp_abort(void);
 int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, unsigned long *fw_err);
+void snp_accept_memory(phys_addr_t start, phys_addr_t end);
 #else
 static inline void sev_es_ist_enter(struct pt_regs *regs) { }
 static inline void sev_es_ist_exit(void) { }
@@ -230,6 +231,8 @@ static inline int snp_issue_guest_request(u64 exit_code, struct snp_req_data *in
 {
 	return -ENOTTY;
 }
+
+static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { }
 #endif
 
 #endif
arch/x86/kernel/sev-shared.c +104 -0

113 lines not on this page · raw message

arch/x86/kernel/sev.c +14 -98

130 lines not on this page · raw message

arch/x86/mm/unaccepted_memory.c +4 -0
index 9ec2304272dc..b86ad6a8ddf5 100644
@@ -9,6 +9,7 @@
 #include <asm/setup.h>
 #include <asm/shared/tdx.h>
 #include <asm/unaccepted_memory.h>
+#include <asm/sev.h>
 
 /* Protects unaccepted memory bitmap */
 static DEFINE_SPINLOCK(unaccepted_memory_lock);
@@ -66,6 +67,9 @@ void accept_memory(phys_addr_t start, phys_addr_t end)
 		if (cpu_feature_enabled(X86_FEATURE_TDX_GUEST)) {
 			tdx_accept_memory(range_start * PMD_SIZE,
 					  range_end * PMD_SIZE);
+		} else if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) {
+			snp_accept_memory(range_start * PMD_SIZE,
+					  range_end * PMD_SIZE);
 		} else {
 			panic("Cannot accept memory: unknown platform\n");
 		}
Signature · 1 line
2.37.2
#188 lore raw

Re: [PATCH v4 4/4] x86/sev: Add SNP-specific unaccepted memory support

Quoted 10 lines

Add SNP-specific hooks to the unaccepted memory support in the boot path (__accept_memory()) and the core kernel (accept_memory()) in order to support booting SNP guests when unaccepted memory is present. Without this support, SNP guests will fail to boot and/or panic() when unaccepted memory is present in the EFI memory map.

The process of accepting memory under SNP involves invoking the hypervisor to perform a page state change for the page to private memory and then issuing a PVALIDATE instruction to accept the page.

Thanks for this update! Tests show the boot performance shaves off a good few seconds over eager acceptance, and it'll get better when we have on-demand pinning.

The uncaught #VC exception is still there for 256GB machines and larger though.

Signature · 1 line
-Dionna Glaze, PhD (she/her)
#189 lore raw

Re: [PATCH v4 4/4] x86/sev: Add SNP-specific unaccepted memory support

Quoted 11 lines · Dionna Amalie Glaze
Quoted 10 lines

Add SNP-specific hooks to the unaccepted memory support in the boot path (__accept_memory()) and the core kernel (accept_memory()) in order to support booting SNP guests when unaccepted memory is present. Without this support, SNP guests will fail to boot and/or panic() when unaccepted memory is present in the EFI memory map.

The process of accepting memory under SNP involves invoking the hypervisor to perform a page state change for the page to private memory and then issuing a PVALIDATE instruction to accept the page.

Quoted 5 lines

Thanks for this update! Tests show the boot performance shaves off a good few seconds over eager acceptance, and it'll get better when we have on-demand pinning.

The uncaught #VC exception is still there for 256GB machines and larger though.

Any chance of getting a stack trace when this occurs, e.g. adding a WARN_ON() in vc_handle_exitcode() (assuming it happens when logging is enabled)?

Thanks,
Tom

Quoted 1 line
#190 lore raw

v5 cover Provide SEV-SNP support for unaccepted memory

In reply to Kirill A. Shutemov

This series adds SEV-SNP support for unaccepted memory to the patch series titled:

[PATCHv7 00/14] mm, x86/cc: Implement support for unaccepted memory

Currently, when changing the state of a page under SNP, the page state change structure is kmalloc()'d. This lead to hangs during boot when accepting memory because the allocation can trigger the need to accept more memory. Additionally, the page state change operations are not optimized under Linux since it was expected that all memory has been validated already, resulting in poor performance when adding basic support for unaccepted memory.

This series consists of six patches:
- Two pre-patch fixes which can be taken regardless of this series.

- A pre-patch to switch from a kmalloc()'d page state change structure to a (smaller) stack-based page state change structure.

- A pre-patch to allow the use of the early boot GHCB in the core kernel path.

- A pre-patch to allow for use of 2M page state change requests and 2M page validation.

- SNP support for unaccepted memory.

The series is based off of and tested against Kirill Shutemov's tree:
https://github.com/intel/tdx.git guest-unaccepted-memory

---

Changes since v4:
- Two fixes for when an unsigned int used as the number of pages to process, it needs to be converted to an unsigned long before being used to calculate ending addresses, otherwise a value >= 0x100000 results in adding 0 in the calculation.
- Commit message and comment updates.

Changes since v3:
- Reworks the PSC process to greatly improve performance:
- Optimize the PSC process to use 2M pages when applicable.
- Optimize the page validation process to use 2M pages when applicable.
- Use the early GHCB in both the decompression phase and core kernel boot phase in order to minimize the use of the MSR protocol. The MSR protocol only allows for a single 4K page to be updated at a time.
- Move the ghcb_percpu_ready flag into the sev_config structure and rename it to ghcbs_initialized.

Changes since v2:
- Improve code comments in regards to when to use the per-CPU GHCB vs the MSR protocol and why a single global value is valid for both the BSP and APs.
- Add a comment related to the number of PSC entries and how it can impact the size of the struct and, therefore, stack usage.
- Add a WARN_ON_ONCE() for invoking vmgexit_psc() when per-CPU GHCBs haven't been created or registered, yet.
- Use the compiler support for clearing the PSC struct instead of issuing memset().

Changes since v1:
- Change from using a per-CPU PSC structure to a (smaller) stack PSC structure.

Tom Lendacky (6):
x86/sev: Fix calculation of end address based on number of pages x86/sev: Fix calculation of end address based on number of pages x86/sev: Put PSC struct on the stack in prep for unaccepted memory
support
x86/sev: Allow for use of the early boot GHCB for PSC requests x86/sev: Use large PSC requests if applicable
x86/sev: Add SNP-specific unaccepted memory support

 arch/x86/Kconfig                  |   1 +
 arch/x86/boot/compressed/mem.c    |   3 +
 arch/x86/boot/compressed/sev.c    |  54 ++++++-
 arch/x86/boot/compressed/sev.h    |  23 +++
 arch/x86/include/asm/sev-common.h |   9 +-
 arch/x86/include/asm/sev.h        |   7 +
 arch/x86/kernel/sev-shared.c      | 104 +++++++++++++
 arch/x86/kernel/sev.c             | 250 +++++++++++++-----------------
 arch/x86/mm/unaccepted_memory.c   |   4 +
 9 files changed, 307 insertions(+), 148 deletions(-)
 create mode 100644 arch/x86/boot/compressed/sev.h
Signature · 1 line
2.37.3
#191 lore raw

PATCH v5 1/6 x86/sev: Fix calculation of end address based on number of pages

When calculating an end address based on an unsigned int number of pages, the number of pages must be cast to an unsigned long so that any value greater than or equal to 0x100000 does not result in zero after the shift.

Fixes 5e5ccff60a29 ("x86/sev: Add helper for validating pages in early enc attribute changes")
---
 arch/x86/kernel/sev.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/sev.c +2 -2
index c05f0124c410..cac56540929d 100644
@@ -649,7 +649,7 @@ static void pvalidate_pages(unsigned long vaddr, unsigned int npages, bool valid
 	int rc;
 
 	vaddr = vaddr & PAGE_MASK;
-	vaddr_end = vaddr + (npages << PAGE_SHIFT);
+	vaddr_end = vaddr + ((unsigned long)npages << PAGE_SHIFT);
 
 	while (vaddr < vaddr_end) {
 		rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
@@ -666,7 +666,7 @@ static void __init early_set_pages_state(unsigned long paddr, unsigned int npage
 	u64 val;
 
 	paddr = paddr & PAGE_MASK;
-	paddr_end = paddr + (npages << PAGE_SHIFT);
+	paddr_end = paddr + ((unsigned long)npages << PAGE_SHIFT);
 
 	while (paddr < paddr_end) {
 		/*
Signature · 1 line
2.37.3
#192 lore raw

Re: [PATCH v5 1/6] x86/sev: Fix calculation of end address based on number of pages

arch/x86/kernel/sev.c static void pvalidate_pages(unsigned long vaddr, unsigned int npages, bool valid · line 652

Quoted 8 lines · Tom Lendacky
arch/x86/kernel/sev.c +1 -1
@@ -649,7 +649,7 @@ static void pvalidate_pages(unsigned long vaddr, unsigned int npages, bool valid
 	int rc;
 
 	vaddr = vaddr & PAGE_MASK;
-	vaddr_end = vaddr + (npages << PAGE_SHIFT);
+	vaddr_end = vaddr + ((unsigned long)npages << PAGE_SHIFT);

Could we please just fix the fragile typing that cascaded down to this point?

Shouldn't 'npages' in this interface be a long?

Quoted 6 lines
arch/x86/kernel/sev.c +0 -0
 struct x86_guest {
        void (*enc_status_change_prepare)(unsigned long vaddr, int npages, bool enc);
        bool (*enc_status_change_finish)(unsigned long vaddr, int npages, bool enc);
        bool (*enc_tlb_flush_required)(bool enc);
        bool (*enc_cache_flush_required)(void);
};
#193 lore raw

Re: [PATCH v5 1/6] x86/sev: Fix calculation of end address based on number of pages

Quoted 6 lines · Dave Hansen
Quoted 5 lines · Tom Lendacky
arch/x86/kernel/sev.c +0 -0
@@ -649,7 +649,7 @@ static void pvalidate_pages(unsigned long vaddr, unsigned int npages, bool valid
  	int rc;
 
Quoted 8 lines

arch/x86/kernel/sev.c static void pvalidate_pages(unsigned long vaddr, unsigned int npages, bool valid · line 652

Quoted 3 lines
arch/x86/kernel/sev.c +1 -1
  	vaddr = vaddr & PAGE_MASK;
-	vaddr_end = vaddr + (npages << PAGE_SHIFT);
+	vaddr_end = vaddr + ((unsigned long)npages << PAGE_SHIFT);

Could we please just fix the fragile typing that cascaded down to this point?

Shouldn't 'npages' in this interface be a long?

I'll take a look at that.

Thanks,
Tom

Quoted 7 lines
Quoted 7 lines
arch/x86/kernel/sev.c +0 -0
 struct x86_guest {
         void (*enc_status_change_prepare)(unsigned long vaddr, int npages, bool enc);
         bool (*enc_status_change_finish)(unsigned long vaddr, int npages, bool enc);
         bool (*enc_tlb_flush_required)(bool enc);
         bool (*enc_cache_flush_required)(void);
};
Tested-by on v5 1/6
#194 lore raw

Re: [PATCH v5 1/6] x86/sev: Fix calculation of end address based on number of pages

In reply to Tom Lendacky
Quoted 6 lines

When calculating an end address based on an unsigned int number of pages, the number of pages must be cast to an unsigned long so that any value greater than or equal to 0x100000 does not result in zero after the shift.

Fixes 5e5ccff60a29 ("x86/sev: Add helper for validating pages in early enc attribute changes")
Signature · 1 line
-Dionna Glaze, PhD (she/her)
#195 lore raw

PATCH v5 2/6 x86/sev: Fix calculation of end address based on number of pages

In reply to Tom Lendacky

When calculating an end address based on an unsigned int number of pages, the number of pages must be cast to an unsigned long so that any value greater than or equal to 0x100000 does not result in zero after the shift.

Fixes dc3f3d2474b8 ("x86/mm: Validate memory when changing the C-bit")
---
 arch/x86/kernel/sev.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kernel/sev.c +1 -1
index cac56540929d..c90a47c39f6b 100644
@@ -875,7 +875,7 @@ static void set_pages_state(unsigned long vaddr, unsigned int npages, int op)
 		panic("SNP: failed to allocate memory for PSC descriptor\n");
 
 	vaddr = vaddr & PAGE_MASK;
-	vaddr_end = vaddr + (npages << PAGE_SHIFT);
+	vaddr_end = vaddr + ((unsigned long)npages << PAGE_SHIFT);
 
 	while (vaddr < vaddr_end) {
 		/* Calculate the last vaddr that fits in one struct snp_psc_desc. */
Signature · 1 line
2.37.3
#196 lore raw

PATCH v5 3/6 x86/sev: Put PSC struct on the stack in prep for unaccepted memory support

In reply to Tom Lendacky

In advance of providing support for unaccepted memory, switch from using kmalloc() for allocating the Page State Change (PSC) structure to using a local variable that lives on the stack. This is needed to avoid a possible recursive call into set_pages_state() if the kmalloc() call requires (more) memory to be accepted, which would result in a hang.

The current size of the PSC struct is 2,032 bytes. To make the struct more stack friendly, reduce the number of PSC entries from 253 down to 64, resulting in a size of 520 bytes. This is a nice compromise on struct size and total PSC requests while still allowing parallel PSC operations across vCPUs.

If the reduction in PSC entries results in any kind of performance issue (that is not seen at the moment), use of a larger static PSC struct, with fallback to the smaller stack version, can be investigated.

For more background info on this decision, see the subthread in the Link:
tag below.

---
 arch/x86/include/asm/sev-common.h |  9 +++++++--
 arch/x86/kernel/sev.c             | 10 ++--------
 2 files changed, 9 insertions(+), 10 deletions(-)
arch/x86/include/asm/sev-common.h +7 -2
index b8357d6ecd47..8ddfdbe521d4 100644
@@ -106,8 +106,13 @@ enum psc_op {
 #define GHCB_HV_FT_SNP			BIT_ULL(0)
 #define GHCB_HV_FT_SNP_AP_CREATION	BIT_ULL(1)
 
-/* SNP Page State Change NAE event */
-#define VMGEXIT_PSC_MAX_ENTRY		253
+/*
+ * SNP Page State Change NAE event
+ *   The VMGEXIT_PSC_MAX_ENTRY determines the size of the PSC structure, which
+ *   is a local stack variable in set_pages_state(). Do not increase this value
+ *   without evaluating the impact to stack usage.
+ */
+#define VMGEXIT_PSC_MAX_ENTRY		64
 
 struct psc_hdr {
 	u16 cur_entry;
arch/x86/kernel/sev.c +2 -8
index c90a47c39f6b..664a4de91757 100644
@@ -868,11 +868,7 @@ static void __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
 static void set_pages_state(unsigned long vaddr, unsigned int npages, int op)
 {
 	unsigned long vaddr_end, next_vaddr;
-	struct snp_psc_desc *desc;
-
-	desc = kmalloc(sizeof(*desc), GFP_KERNEL_ACCOUNT);
-	if (!desc)
-		panic("SNP: failed to allocate memory for PSC descriptor\n");
+	struct snp_psc_desc desc;
 
 	vaddr = vaddr & PAGE_MASK;
 	vaddr_end = vaddr + ((unsigned long)npages << PAGE_SHIFT);
@@ -882,12 +878,10 @@ static void set_pages_state(unsigned long vaddr, unsigned int npages, int op)
 		next_vaddr = min_t(unsigned long, vaddr_end,
 				   (VMGEXIT_PSC_MAX_ENTRY * PAGE_SIZE) + vaddr);
 
-		__set_pages_state(desc, vaddr, next_vaddr, op);
+		__set_pages_state(&desc, vaddr, next_vaddr, op);
 
 		vaddr = next_vaddr;
 	}
-
-	kfree(desc);
 }
 
 void snp_set_memory_shared(unsigned long vaddr, unsigned int npages)
Signature · 1 line
2.37.3
#197 lore raw

PATCH v5 4/6 x86/sev: Allow for use of the early boot GHCB for PSC requests

In reply to Tom Lendacky

Using a GHCB for a page stage change (as opposed to the MSR protocol) allows for multiple pages to be processed in a single request. In prep for early PSC requests in support of unaccepted memory, update the invocation of vmgexit_psc() to be able to use the early boot GHCB and not just the per-CPU GHCB structure.

In order to use the proper GHCB (early boot vs per-CPU), set a flag that indicates when the per-CPU GHCBs are available and registered. For APs, the per-CPU GHCBs are created before they are started and registered upon startup, so this flag can be used globally for the BSP and APs instead of creating a per-CPU flag. This will allow for a significant reduction in the number of MSR protocol page state change requests when accepting memory.

---
 arch/x86/kernel/sev.c | 61 +++++++++++++++++++++++++++----------------
 1 file changed, 38 insertions(+), 23 deletions(-)
arch/x86/kernel/sev.c +38 -23
index 664a4de91757..0b958d77abb4 100644
@@ -117,7 +117,19 @@ static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
 
 struct sev_config {
 	__u64 debug		: 1,
-	      __reserved	: 63;
+
+	      /*
+	       * A flag used by __set_pages_state() that indicates when the
+	       * per-CPU GHCB has been created and registered and thus can be
+	       * used by the BSP instead of the early boot GHCB.
+	       *
+	       * For APs, the per-CPU GHCB is created before they are started
+	       * and registered upon startup, so this flag can be used globally
+	       * for the BSP and APs.
+	       */
+	      ghcbs_initialized	: 1,
+
+	      __reserved	: 62;
 };
 
 static struct sev_config sev_cfg __read_mostly;
@@ -660,7 +672,7 @@ static void pvalidate_pages(unsigned long vaddr, unsigned int npages, bool valid
 	}
 }
 
-static void __init early_set_pages_state(unsigned long paddr, unsigned int npages, enum psc_op op)
+static void early_set_pages_state(unsigned long paddr, unsigned int npages, enum psc_op op)
 {
 	unsigned long paddr_end;
 	u64 val;
@@ -742,26 +754,13 @@ void __init snp_prep_memory(unsigned long paddr, unsigned int sz, enum psc_op op
 		WARN(1, "invalid memory op %d\n", op);
 }
 
-static int vmgexit_psc(struct snp_psc_desc *desc)
+static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
 {
 	int cur_entry, end_entry, ret = 0;
 	struct snp_psc_desc *data;
-	struct ghcb_state state;
 	struct es_em_ctxt ctxt;
-	unsigned long flags;
-	struct ghcb *ghcb;
 
-	/*
-	 * __sev_get_ghcb() needs to run with IRQs disabled because it is using
-	 * a per-CPU GHCB.
-	 */
-	local_irq_save(flags);
-
-	ghcb = __sev_get_ghcb(&state);
-	if (!ghcb) {
-		ret = 1;
-		goto out_unlock;
-	}
+	vc_ghcb_invalidate(ghcb);
 
 	/* Copy the input desc into GHCB shared buffer */
 	data = (struct snp_psc_desc *)ghcb->shared_buffer;
@@ -818,20 +817,18 @@ static int vmgexit_psc(struct snp_psc_desc *desc)
 	}
 
 out:
-	__sev_put_ghcb(&state);
-
-out_unlock:
-	local_irq_restore(flags);
-
 	return ret;
 }
 
 static void __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
 			      unsigned long vaddr_end, int op)
 {
+	struct ghcb_state state;
 	struct psc_hdr *hdr;
 	struct psc_entry *e;
+	unsigned long flags;
 	unsigned long pfn;
+	struct ghcb *ghcb;
 	int i;
 
 	hdr = &data->hdr;
@@ -861,8 +858,20 @@ static void __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
 		i++;
 	}
 
-	if (vmgexit_psc(data))
+	local_irq_save(flags);
+
+	if (sev_cfg.ghcbs_initialized)
+		ghcb = __sev_get_ghcb(&state);
+	else
+		ghcb = boot_ghcb;
+
+	if (!ghcb || vmgexit_psc(ghcb, data))
 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
+
+	if (sev_cfg.ghcbs_initialized)
+		__sev_put_ghcb(&state);
+
+	local_irq_restore(flags);
 }
 
 static void set_pages_state(unsigned long vaddr, unsigned int npages, int op)
@@ -870,6 +879,10 @@ static void set_pages_state(unsigned long vaddr, unsigned int npages, int op)
 	unsigned long vaddr_end, next_vaddr;
 	struct snp_psc_desc desc;
 
+	/* Use the MSR protocol when a GHCB is not available. */
+	if (!boot_ghcb)
+		return early_set_pages_state(__pa(vaddr), npages, op);
+
 	vaddr = vaddr & PAGE_MASK;
 	vaddr_end = vaddr + ((unsigned long)npages << PAGE_SHIFT);
 
@@ -1248,6 +1261,8 @@ void setup_ghcb(void)
 		if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
 			snp_register_per_cpu_ghcb();
 
+		sev_cfg.ghcbs_initialized = true;
+
 		return;
 	}
 
Signature · 1 line
2.37.3
#198 lore raw

PATCH v5 5/6 x86/sev: Use large PSC requests if applicable

In reply to Tom Lendacky

In advance of providing support for unaccepted memory, request 2M Page State Change (PSC) requests when the address range allows for it. By using a 2M page size, more PSC operations can be handled in a single request to the hypervisor. The hypervisor will determine if it can accommodate the larger request by checking the mapping in the nested page table. If mapped as a large page, then the 2M page request can be performed, otherwise the 2M page request will be broken down into 512 4K page requests. This is still more efficient than having the guest perform multiple PSC requests in order to process the 512 4K pages.

In conjunction with the 2M PSC requests, attempt to perform the associated PVALIDATE instruction of the page using the 2M page size. If PVALIDATE fails with a size mismatch, then fallback to validating 512 4K pages. To do this, page validation is modified to work with the PSC structure and not just a virtual address range.

---
 arch/x86/include/asm/sev.h |   4 ++
 arch/x86/kernel/sev.c      | 125 ++++++++++++++++++++++++-------------
 2 files changed, 84 insertions(+), 45 deletions(-)
arch/x86/include/asm/sev.h +4 -0
index 19514524f0f8..0007ab04ac5f 100644
@@ -79,11 +79,15 @@ extern void vc_no_ghcb(void);
 extern void vc_boot_ghcb(void);
 extern bool handle_vc_boot_ghcb(struct pt_regs *regs);
 
+/* PVALIDATE return codes */
+#define PVALIDATE_FAIL_SIZEMISMATCH	6
+
 /* Software defined (when rFlags.CF = 1) */
 #define PVALIDATE_FAIL_NOUPDATE		255
 
 /* RMP page size */
 #define RMP_PG_SIZE_4K			0
+#define RMP_PG_SIZE_2M			1
 
 #define RMPADJUST_VMSA_PAGE_BIT		BIT(16)
 
arch/x86/kernel/sev.c +80 -45

228 lines not on this page · raw message

Signature · 1 line
2.37.3
#199 lore raw

PATCH v5 6/6 x86/sev: Add SNP-specific unaccepted memory support

In reply to Tom Lendacky

Add SNP-specific hooks to the unaccepted memory support in the boot path (__accept_memory()) and the core kernel (accept_memory()) in order to support booting SNP guests when unaccepted memory is present. Without this support, SNP guests will fail to boot and/or panic() when unaccepted memory is present in the EFI memory map.

The process of accepting memory under SNP involves invoking the hypervisor to perform a page state change for the page to private memory and then issuing a PVALIDATE instruction to accept the page.

Since the boot path and the core kernel paths perform similar operations, move the pvalidate_pages() and vmgexit_psc() functions into sev-shared.c to avoid code duplication.

Create the new header file arch/x86/boot/compressed/sev.h because adding the function declaration to any of the existing SEV related header files pulls in too many other header files, causing the build to fail.

---
 arch/x86/Kconfig                |   1 +
 arch/x86/boot/compressed/mem.c  |   3 +
 arch/x86/boot/compressed/sev.c  |  54 ++++++++++++++-
 arch/x86/boot/compressed/sev.h  |  23 +++++++
 arch/x86/include/asm/sev.h      |   3 +
 arch/x86/kernel/sev-shared.c    | 104 +++++++++++++++++++++++++++++
 arch/x86/kernel/sev.c           | 112 ++++----------------------------
 arch/x86/mm/unaccepted_memory.c |   4 ++
 8 files changed, 205 insertions(+), 99 deletions(-)
 create mode 100644 arch/x86/boot/compressed/sev.h
arch/x86/Kconfig +1 -0
index 34146ecc5bdd..0ad53c3533c2 100644
@@ -1553,6 +1553,7 @@ config AMD_MEM_ENCRYPT
 	select INSTRUCTION_DECODER
 	select ARCH_HAS_CC_PLATFORM
 	select X86_MEM_ENCRYPT
+	select UNACCEPTED_MEMORY
 	help
 	  Say yes to enable support for the encryption of system memory.
 	  This requires an AMD processor that supports Secure Memory
arch/x86/boot/compressed/mem.c +3 -0
index 48e36e640da1..3e19dc0da0d7 100644
@@ -6,6 +6,7 @@
 #include "find.h"
 #include "math.h"
 #include "tdx.h"
+#include "sev.h"
 #include <asm/shared/tdx.h>
 
 #define PMD_SHIFT	21
@@ -39,6 +40,8 @@ static inline void __accept_memory(phys_addr_t start, phys_addr_t end)
 	/* Platform-specific memory-acceptance call goes here */
 	if (is_tdx_guest())
 		tdx_accept_memory(start, end);
+	else if (sev_snp_enabled())
+		snp_accept_memory(start, end);
 	else
 		error("Cannot accept memory: unknown platform\n");
 }
arch/x86/boot/compressed/sev.c +53 -1

66 lines not on this page · raw message

arch/x86/boot/compressed/sev.h new file +23 -0
new file mode 100644
index 000000000000..fc725a981b09
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * AMD SEV header for early boot related functions.
+ *
+ * Author: Tom Lendacky <[email protected]>
+ */
+
+#ifndef BOOT_COMPRESSED_SEV_H
+#define BOOT_COMPRESSED_SEV_H
+
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+
+bool sev_snp_enabled(void);
+void snp_accept_memory(phys_addr_t start, phys_addr_t end);
+
+#else
+
+static inline bool sev_snp_enabled(void) { return false; }
+static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { }
+
+#endif
+
+#endif
arch/x86/include/asm/sev.h +3 -0
index 0007ab04ac5f..9297aab0c79e 100644
@@ -206,6 +206,7 @@ void snp_set_wakeup_secondary_cpu(void);
 bool snp_init(struct boot_params *bp);
 void snp_abort(void);
 int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, unsigned long *fw_err);
+void snp_accept_memory(phys_addr_t start, phys_addr_t end);
 #else
 static inline void sev_es_ist_enter(struct pt_regs *regs) { }
 static inline void sev_es_ist_exit(void) { }
@@ -230,6 +231,8 @@ static inline int snp_issue_guest_request(u64 exit_code, struct snp_req_data *in
 {
 	return -ENOTTY;
 }
+
+static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { }
 #endif
 
 #endif
arch/x86/kernel/sev-shared.c +104 -0

113 lines not on this page · raw message

arch/x86/kernel/sev.c +14 -98

130 lines not on this page · raw message

arch/x86/mm/unaccepted_memory.c +4 -0
index 9ec2304272dc..b86ad6a8ddf5 100644
@@ -9,6 +9,7 @@
 #include <asm/setup.h>
 #include <asm/shared/tdx.h>
 #include <asm/unaccepted_memory.h>
+#include <asm/sev.h>
 
 /* Protects unaccepted memory bitmap */
 static DEFINE_SPINLOCK(unaccepted_memory_lock);
@@ -66,6 +67,9 @@ void accept_memory(phys_addr_t start, phys_addr_t end)
 		if (cpu_feature_enabled(X86_FEATURE_TDX_GUEST)) {
 			tdx_accept_memory(range_start * PMD_SIZE,
 					  range_end * PMD_SIZE);
+		} else if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) {
+			snp_accept_memory(range_start * PMD_SIZE,
+					  range_end * PMD_SIZE);
 		} else {
 			panic("Cannot accept memory: unknown platform\n");
 		}
Signature · 1 line
2.37.3