Skip to contentSkip to conversation

Allow disabling RAPL during SNP initialization

PATCH In review Tycho Andersen 3 participants 13 messages Apr 27 – Apr 28

Subsystems (8)
Documentation x86 KVM Kselftest +4 more subsystems
lore
Share & tools
lore
#0 lore raw

v2 cover Allow disabling RAPL during SNP initialization

From: "Tycho Andersen (AMD)" <[email protected]>

The PLATYPUS (https://platypusattack.com/platypus.pdf) attack paper offers method for exfiltration via a low-resolution side channel using Running Average Power Limit. Newer versions of the SEV firmware offer a way to freeze these counters while initialized, so export this as an option.

Changes from v1:
* add a blurb from the platypus attack paper about the side channel
* move the kernel parameter from the ccp driver to kvm_amd to make it more obvious
* split off the SEV-ES feature detection into its own series:
https://lore.kernel.org/all/[email protected]/

v1 is here: https://lore.kernel.org/all/[email protected]/

Tycho Andersen (AMD) (4):
crypto/ccp: Pass init_args to __sev_snp_init_locked() crypto/ccp: Support setting RAPL_DIS in SNP_INIT_EX
KVM: SEV: Add the kvm-amd.rapl_disable module parameter
KVM: selftests: Add a smoke test support for RAPL_DIS

 .../admin-guide/kernel-parameters.txt         |  5 +++
 arch/x86/kvm/svm/sev.c                        |  8 +++++
 drivers/crypto/ccp/sev-dev.c                  | 35 +++++++++++++------
 include/linux/psp-sev.h                       |  2 ++
 tools/testing/selftests/kvm/include/x86/sev.h |  1 +
 .../selftests/kvm/x86/sev_smoke_test.c        | 24 ++++++++++++-
 6 files changed, 63 insertions(+), 12 deletions(-)

base-commit: 39f1c201b93f4ff71631bac72cff6eb155f976a4

Signature · 1 line
2.53.0
#1 lore raw

PATCH v2 1/4 crypto/ccp: Pass init_args to __sev_snp_init_locked()

From: "Tycho Andersen (AMD)" <[email protected]>

Rather than splitting this off, pass the whole struct so that __sev_snp_init_locked() will have access to any other structure members that are added in the future.

No functional change intended.

---
 drivers/crypto/ccp/sev-dev.c | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)
drivers/crypto/ccp/sev-dev.c +11 -10
index d1e9e0ac63b6..bf54a3fadb28 100644
@@ -1351,7 +1351,7 @@ static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg)
 	return 0;
 }
 
-static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
+static int __sev_snp_init_locked(struct sev_platform_init_args *args)
 {
 	struct sev_data_range_list *snp_range_list __free(kfree) = NULL;
 	struct psp_device *psp = psp_master;
@@ -1421,9 +1421,9 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
 
 		memset(&data, 0, sizeof(data));
 
-		if (max_snp_asid) {
+		if (args->max_snp_asid) {
 			data.ciphertext_hiding_en = 1;
-			data.max_snp_asid = max_snp_asid;
+			data.max_snp_asid = args->max_snp_asid;
 		}
 
 		data.init_rmp = 1;
@@ -1458,20 +1458,20 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
 	 */
 	wbinvd_on_all_cpus();
 
-	rc = __sev_do_cmd_locked(cmd, arg, error);
+	rc = __sev_do_cmd_locked(cmd, arg, &args->error);
 	if (rc) {
 		dev_err(sev->dev, "SEV-SNP: %s failed rc %d, error %#x\n",
 			cmd == SEV_CMD_SNP_INIT_EX ? "SNP_INIT_EX" : "SNP_INIT",
-			rc, *error);
+			rc, args->error);
 		return rc;
 	}
 
 	/* Prepare for first SNP guest launch after INIT. */
 	wbinvd_on_all_cpus();
-	rc = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, error);
+	rc = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, &args->error);
 	if (rc) {
 		dev_err(sev->dev, "SEV-SNP: SNP_DF_FLUSH failed rc %d, error %#x\n",
-			rc, *error);
+			rc, args->error);
 		return rc;
 	}
 
@@ -1651,7 +1651,7 @@ static int _sev_platform_init_locked(struct sev_platform_init_args *args)
 	if (sev->sev_plat_status.state == SEV_STATE_INIT)
 		return 0;
 
-	rc = __sev_snp_init_locked(&args->error, args->max_snp_asid);
+	rc = __sev_snp_init_locked(args);
 	if (rc && rc != -ENODEV)
 		return rc;
 
@@ -1732,9 +1732,10 @@ static int sev_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_req
 
 static int snp_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required)
 {
-	int error, rc;
+	struct sev_platform_init_args args = {};
+	int rc;
 
-	rc = __sev_snp_init_locked(&error, 0);
+	rc = __sev_snp_init_locked(&args);
 	if (rc) {
 		argp->error = SEV_RET_INVALID_PLATFORM_STATE;
 		return rc;
Signature · 1 line
2.53.0
#2 lore raw

PATCH v2 2/4 crypto/ccp: Support setting RAPL_DIS in SNP_INIT_EX

In reply to Tycho Andersen

From: "Tycho Andersen (AMD)" <[email protected]>

From the PLATYPUS [1] attack paper:

We exploit unprivileged access to the Intel Running Average Power Limit (RAPL) interface that exposes values directly correlated with power consumption, forming a low-resolution side channel.

The SEV firmware offers a mechanism to freeze RAPL counters across all cores during SNP initialization via the RAPL_DIS bit in SNP_INIT_EX. The counters remain frozen while SNP is initialized, and resume after an SNP shutdown.

The SEV firmware also has a RAPL_DIS policy bit, allowing guests to enforce that RAPL is disabled on a system before running. Since the kernel had no way to set the RAPL_DIS bit during SNP init, trying to set the policy bit would always result in a failed launch.

Allow setting the RAPL_DIS bit during SNP_INIT_EX via struct sev_platform_init_args.

If the hardware does not support RAPL_DIS, set the rapl_disable parameter to false so that consumers can detect when it was not actually initialized.

[1]: https://platypusattack.com/platypus.pdf

---
 drivers/crypto/ccp/sev-dev.c | 14 +++++++++++++-
 include/linux/psp-sev.h      |  2 ++
 2 files changed, 15 insertions(+), 1 deletion(-)
drivers/crypto/ccp/sev-dev.c +13 -1
index bf54a3fadb28..6223d63e676e 100644
@@ -1365,8 +1365,11 @@ static int __sev_snp_init_locked(struct sev_platform_init_args *args)
 
 	sev = psp->sev_data;
 
-	if (sev->snp_initialized)
+	if (sev->snp_initialized) {
+		if (args->rapl_disable && !sev->snp_plat_status.rapl_dis)
+			args->rapl_disable = false;
 		return 0;
+	}
 
 	if (!sev_version_greater_or_equal(SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR)) {
 		dev_dbg(sev->dev, "SEV-SNP support requires firmware version >= %d:%d\n",
@@ -1376,6 +1379,12 @@ static int __sev_snp_init_locked(struct sev_platform_init_args *args)
 
 	snp_prepare();
 
+	if (args->rapl_disable && !(sev->snp_feat_info_0.ecx & SNP_RAPL_DISABLE_SUPPORTED)) {
+		dev_info(sev->dev,
+			"SEV: RAPL_DIS requested, but not supported\n");
+		args->rapl_disable = false;
+	}
+
 	/*
 	 * Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list
 	 * of system physical address ranges to convert into HV-fixed page
@@ -1426,6 +1435,9 @@ static int __sev_snp_init_locked(struct sev_platform_init_args *args)
 			data.max_snp_asid = args->max_snp_asid;
 		}
 
+		if (args->rapl_disable)
+			data.rapl_dis = 1;
+
 		data.init_rmp = 1;
 		data.list_paddr_en = 1;
 		data.list_paddr = __psp_pa(snp_range_list);
include/linux/psp-sev.h +2 -0
index d5099a2baca5..55ffc098d573 100644
@@ -848,11 +848,13 @@ struct sev_data_snp_shutdown_ex {
  *  unless psp_init_on_probe module param is set
  * @max_snp_asid: When non-zero, enable ciphertext hiding and specify the
  *  maximum ASID that can be used for an SEV-SNP guest.
+ * @rapl_disable: Whether or not to set the RAPL_DIS bit during SNP_INIT_EX.
  */
 struct sev_platform_init_args {
 	int error;
 	bool probe;
 	unsigned int max_snp_asid;
+	bool rapl_disable;
 };
 
 /**
Signature · 1 line
2.53.0
#3 lore raw

PATCH v2 3/4 KVM: SEV: Add the kvm-amd.rapl_disable module parameter

In reply to Tycho Andersen

From: "Tycho Andersen (AMD)" <[email protected]>

Add a user-visible way to set the RAPL_DIS bit for SNP init.

Since setting RAPL_DIS affects the whole system, put the module parameter in kvm_amd instead of in the CCP driver to hopefully make it more obvious to admins.

---
 Documentation/admin-guide/kernel-parameters.txt | 5 +++++
 arch/x86/kvm/svm/sev.c                          | 8 ++++++++
 2 files changed, 13 insertions(+)
Documentation/admin-guide/kernel-parameters.txt +5 -0
index 4d0f545fb3ec..2b50eed8664c 100644
@@ -3207,6 +3207,11 @@ Kernel parameters
 			max_snp_asid == min_sev_asid-1, will effectively make
 			SEV-ES unusable.
 
+	kvm-amd.rapl_disable=	[KVM,AMD] Whether to disable RAPL
+			(Running Average Power Limit) when initializing the SNP
+			firmware. This disables the counters for the entire system until an
+			SNP shutdown command is issued.
+
 	kvm-arm.mode=
 			[KVM,ARM,EARLY] Select one of KVM/arm64's modes of
 			operation.
arch/x86/kvm/svm/sev.c +8 -0
index c2126b3c3072..c2a30a3d6f50 100644
@@ -66,6 +66,10 @@ module_param_named(sev_snp, sev_snp_enabled, bool, 0444);
 static unsigned int __ro_after_init nr_ciphertext_hiding_asids;
 module_param_named(ciphertext_hiding_asids, nr_ciphertext_hiding_asids, uint, 0444);
 
+static bool rapl_disable;
+module_param(rapl_disable, bool, 0444);
+MODULE_PARM_DESC(rapl_disable, " if true, disable RAPL during SNP Initialization");
+
 #define AP_RESET_HOLD_NONE		0
 #define AP_RESET_HOLD_NAE_EVENT		1
 #define AP_RESET_HOLD_MSR_PROTO		2
@@ -3163,6 +3167,7 @@ void __init sev_hardware_setup(void)
 out:
 	if (sev_enabled) {
 		init_args.probe = true;
+		init_args.rapl_disable = rapl_disable;
 
 		if (sev_is_snp_ciphertext_hiding_supported())
 			init_args.max_snp_asid = min(nr_ciphertext_hiding_asids,
@@ -3174,6 +3179,9 @@ void __init sev_hardware_setup(void)
 			sev_snp_supported = is_sev_snp_initialized();
 
 		if (sev_snp_supported) {
+			if (!init_args.rapl_disable)
+				rapl_disable = false;
+
 			snp_supported_policy_bits = sev_get_snp_policy_bits() &
 						    KVM_SNP_POLICY_MASK_VALID;
 			nr_ciphertext_hiding_asids = init_args.max_snp_asid;
Signature · 1 line
2.53.0
#4 lore raw

Re: [PATCH v2 3/4] KVM: SEV: Add the kvm-amd.rapl_disable module parameter

Quoted 19 lines · Tycho Andersen

From: "Tycho Andersen (AMD)" <[email protected]>

Add a user-visible way to set the RAPL_DIS bit for SNP init.

Since setting RAPL_DIS affects the whole system, put the module parameter in kvm_amd instead of in the CCP driver to hopefully make it more obvious to admins.

---
 Documentation/admin-guide/kernel-parameters.txt | 5 +++++
 arch/x86/kvm/svm/sev.c                          | 8 ++++++++
 2 files changed, 13 insertions(+)
Documentation/admin-guide/kernel-parameters.txt +0 -0
index 4d0f545fb3ec..2b50eed8664c 100644

Documentation/admin-guide/kernel-parameters.txt Kernel parameters · line 3213

Quoted 7 lines
Documentation/admin-guide/kernel-parameters.txt +4 -0
@@ -3207,6 +3207,11 @@ Kernel parameters
 			max_snp_asid == min_sev_asid-1, will effectively make
 			SEV-ES unusable.
 
+	kvm-amd.rapl_disable=	[KVM,AMD] Whether to disable RAPL
+			(Running Average Power Limit) when initializing the SNP
+			firmware. This disables the counters for the entire system until an
+			SNP shutdown command is issued.

I'm pretty sure I said this earlier: KVM absolutely should not be able to disable RAPL for the entire system. That needs to be a power management thing. KVM then needs to communicate (and enforce?) the policy to userspace.

#5 lore raw

Re: [PATCH v2 3/4] KVM: SEV: Add the kvm-amd.rapl_disable module parameter

Quoted 23 lines · Sean Christopherson
Quoted 22 lines · Tycho Andersen

From: "Tycho Andersen (AMD)" <[email protected]>

Add a user-visible way to set the RAPL_DIS bit for SNP init.

Since setting RAPL_DIS affects the whole system, put the module parameter in kvm_amd instead of in the CCP driver to hopefully make it more obvious to admins.

---
 Documentation/admin-guide/kernel-parameters.txt | 5 +++++
 arch/x86/kvm/svm/sev.c                          | 8 ++++++++
 2 files changed, 13 insertions(+)
Documentation/admin-guide/kernel-parameters.txt +0 -0
index 4d0f545fb3ec..2b50eed8664c 100644
@@ -3207,6 +3207,11 @@ Kernel parameters
 			max_snp_asid == min_sev_asid-1, will effectively make
 			SEV-ES unusable.
 
Quoted 7 lines

Documentation/admin-guide/kernel-parameters.txt Kernel parameters · line 3213

Quoted 4 lines
Documentation/admin-guide/kernel-parameters.txt +4 -0
+	kvm-amd.rapl_disable=	[KVM,AMD] Whether to disable RAPL
+			(Running Average Power Limit) when initializing the SNP
+			firmware. This disables the counters for the entire system until an
+			SNP shutdown command is issued.

I'm pretty sure I said this earlier: KVM absolutely should not be able to disable RAPL for the entire system. That needs to be a power management thing.

You definitely noted "not CCP", I don't think I quite understood what that meant though:
https://lore.kernel.org/all/[email protected]/

I'm a little worried that putting it in power management will generate some weird dependencies, or weakref symbols that can't change things if they are loaded independently of kvm_amd or something. But let me see what I can come up with.

Quoted 2 lines

KVM then needs to communicate (and enforce?) the policy to userspace.

KVM doesn't need to enforce anything, the SEV firmware will generate a launch error for policy violation if it's not supported.

For communicating to userspace if it's not a kvm module parameter, one option is to mask it off in sev_get_snp_supported_policy() if it was initialized without the support. Then it'll be visible via KVM_X86_SNP_POLICY_BITS.

Thanks,

Tycho

#6 lore raw

Re: [PATCH v2 3/4] KVM: SEV: Add the kvm-amd.rapl_disable module parameter

Quoted 32 lines · Tycho Andersen
Quoted 23 lines · Sean Christopherson
Quoted 22 lines · Tycho Andersen

From: "Tycho Andersen (AMD)" <[email protected]>

Add a user-visible way to set the RAPL_DIS bit for SNP init.

Since setting RAPL_DIS affects the whole system, put the module parameter in kvm_amd instead of in the CCP driver to hopefully make it more obvious to admins.

---
 Documentation/admin-guide/kernel-parameters.txt | 5 +++++
 arch/x86/kvm/svm/sev.c                          | 8 ++++++++
 2 files changed, 13 insertions(+)
Documentation/admin-guide/kernel-parameters.txt +0 -0
index 4d0f545fb3ec..2b50eed8664c 100644
@@ -3207,6 +3207,11 @@ Kernel parameters
 			max_snp_asid == min_sev_asid-1, will effectively make
 			SEV-ES unusable.
 
Quoted 7 lines

Documentation/admin-guide/kernel-parameters.txt Kernel parameters · line 3213

Quoted 4 lines
Documentation/admin-guide/kernel-parameters.txt +4 -0
+	kvm-amd.rapl_disable=	[KVM,AMD] Whether to disable RAPL
+			(Running Average Power Limit) when initializing the SNP
+			firmware. This disables the counters for the entire system until an
+			SNP shutdown command is issued.

I'm pretty sure I said this earlier: KVM absolutely should not be able to disable RAPL for the entire system. That needs to be a power management thing.

Quoted 8 lines

You definitely noted "not CCP", I don't think I quite understood what that meant though:
https://lore.kernel.org/all/[email protected]/

I'm a little worried that putting it in power management will generate some weird dependencies, or weakref symbols that can't change things if they are loaded independently of kvm_amd or something. But let me see what I can come up with.

Ugh, and it's not even powerman per se, it's actually a module in perf. Oof.

I 100% agree it'll be tricky, but I also stand by comments that neither the CCP driver or KVM should be allowed to silently pull the rug out from under the RAPL module.

Quoted 10 lines
Quoted 2 lines

KVM then needs to communicate (and enforce?) the policy to userspace.

KVM doesn't need to enforce anything, the SEV firmware will generate a launch error for policy violation if it's not supported.

For communicating to userspace if it's not a kvm module parameter, one option is to mask it off in sev_get_snp_supported_policy() if it was initialized without the support. Then it'll be visible via KVM_X86_SNP_POLICY_BITS.

Ya, this is what I was envisioning.

maintainer
#7 lore raw

Re: [PATCH v2 3/4] KVM: SEV: Add the kvm-amd.rapl_disable module parameter

Quoted 42 lines · Sean Christopherson
Quoted 32 lines · Tycho Andersen
Quoted 23 lines · Sean Christopherson
Quoted 22 lines · Tycho Andersen

From: "Tycho Andersen (AMD)" <[email protected]>

Add a user-visible way to set the RAPL_DIS bit for SNP init.

Since setting RAPL_DIS affects the whole system, put the module parameter in kvm_amd instead of in the CCP driver to hopefully make it more obvious to admins.

---
 Documentation/admin-guide/kernel-parameters.txt | 5 +++++
 arch/x86/kvm/svm/sev.c                          | 8 ++++++++
 2 files changed, 13 insertions(+)
Documentation/admin-guide/kernel-parameters.txt +0 -0
index 4d0f545fb3ec..2b50eed8664c 100644
@@ -3207,6 +3207,11 @@ Kernel parameters
 			max_snp_asid == min_sev_asid-1, will effectively make
 			SEV-ES unusable.
 
Quoted 7 lines

Documentation/admin-guide/kernel-parameters.txt Kernel parameters · line 3213

Quoted 4 lines
Documentation/admin-guide/kernel-parameters.txt +4 -0
+	kvm-amd.rapl_disable=	[KVM,AMD] Whether to disable RAPL
+			(Running Average Power Limit) when initializing the SNP
+			firmware. This disables the counters for the entire system until an
+			SNP shutdown command is issued.

I'm pretty sure I said this earlier: KVM absolutely should not be able to disable RAPL for the entire system. That needs to be a power management thing.

Quoted 8 lines

You definitely noted "not CCP", I don't think I quite understood what that meant though:
https://lore.kernel.org/all/[email protected]/

I'm a little worried that putting it in power management will generate some weird dependencies, or weakref symbols that can't change things if they are loaded independently of kvm_amd or something. But let me see what I can come up with.

Quoted 5 lines

Ugh, and it's not even powerman per se, it's actually a module in perf. Oof.

I 100% agree it'll be tricky, but I also stand by comments that neither the CCP driver or KVM should be allowed to silently pull the rug out from under the RAPL module.

Maybe something that can be added to the current sev= kernel command line parameter, e.g. sev=norapl, or such? Maybe even with a kernel config option for a default value? On SNP_SHUTDOWN it will be re-enabled if it was disabled.

Quoted 9 lines
Quoted 9 lines
Quoted 3 lines

KVM then needs to communicate (and enforce?) the policy to userspace.

KVM doesn't need to enforce anything, the SEV firmware will generate a launch error for policy violation if it's not supported.

For communicating to userspace if it's not a kvm module parameter, one option is to mask it off in sev_get_snp_supported_policy() if it was

Did you mean sev_get_snp_policy_bits() or were you referring to the KVM ioctl() for retrieving them?

Quoted 4 lines
Quoted 2 lines

initialized without the support. Then it'll be visible via KVM_X86_SNP_POLICY_BITS.

Ya, this is what I was envisioning.

It's still a valid policy bit (if supported by the platform), so I don't think masking it off is appropriate.

Thanks,
Tom

#8 lore raw

Re: [PATCH v2 3/4] KVM: SEV: Add the kvm-amd.rapl_disable module parameter

Quoted 15 lines · Tom Lendacky
Quoted 14 lines · Sean Christopherson
Quoted 12 lines · Tycho Andersen
Quoted 2 lines · Sean Christopherson

I'm pretty sure I said this earlier: KVM absolutely should not be able to disable RAPL for the entire system. That needs to be a power management thing.

You definitely noted "not CCP", I don't think I quite understood what that meant though:
https://lore.kernel.org/all/[email protected]/

I'm a little worried that putting it in power management will generate some weird dependencies, or weakref symbols that can't change things if they are loaded independently of kvm_amd or something. But let me see what I can come up with.

Quoted 8 lines
Quoted 5 lines

Ugh, and it's not even powerman per se, it's actually a module in perf. Oof.

I 100% agree it'll be tricky, but I also stand by comments that neither the CCP driver or KVM should be allowed to silently pull the rug out from under the RAPL module.

Maybe something that can be added to the current sev= kernel command line parameter, e.g. sev=norapl, or such?

Yeah. The only question I have is if we expect end users to want to disable RAPL at runtime. If so, then we probably want a sysfs knob or something.

However, letting RAPL be toggled on/off will introduce some amount of complexity, as the kernel would need to negotiate/coordinate with the RAPL perf module and with the CPP driver to ensure RAPL stays in the "correct" state. E.g. if the perf module is loaded, then RAPL is effectively pinned "on". And if SNP has been initialized with RAPL_DIS, then RAPL is effectively pinned "off". Blech.

Quoted 1 line

Maybe even with a kernel config option for a default value?

Probably overkill?

Quoted 1 line

On SNP_SHUTDOWN it will be re-enabled if it was disabled.

Stating the obvious, if we do this, we open the can of worms I described above.

Quoted 12 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines

KVM then needs to communicate (and enforce?) the policy to userspace.

KVM doesn't need to enforce anything, the SEV firmware will generate a launch error for policy violation if it's not supported.

For communicating to userspace if it's not a kvm module parameter, one option is to mask it off in sev_get_snp_supported_policy() if it was

Did you mean sev_get_snp_policy_bits() or were you referring to the KVM ioctl() for retrieving them?

Quoted 7 lines
Quoted 4 lines
Quoted 2 lines

initialized without the support. Then it'll be visible via KVM_X86_SNP_POLICY_BITS.

Ya, this is what I was envisioning.

It's still a valid policy bit (if supported by the platform), so I don't think masking it off is appropriate.

But it's not fully supported, no? I.e. won't the VM fail if it requests RAPL_DIS?

Ooh, presumably the subtle difference is that on a platform without RAPL_DIS at all, the VM will successfully launch and thus could run with RAPL enabled even if the VM requested RAPL_DIS?

#9 lore raw

Re: [PATCH v2 3/4] KVM: SEV: Add the kvm-amd.rapl_disable module parameter

Quoted 34 lines · Sean Christopherson
Quoted 15 lines · Tom Lendacky
Quoted 14 lines · Sean Christopherson
Quoted 12 lines · Tycho Andersen
Quoted 2 lines · Sean Christopherson

I'm pretty sure I said this earlier: KVM absolutely should not be able to disable RAPL for the entire system. That needs to be a power management thing.

You definitely noted "not CCP", I don't think I quite understood what that meant though:
https://lore.kernel.org/all/[email protected]/

I'm a little worried that putting it in power management will generate some weird dependencies, or weakref symbols that can't change things if they are loaded independently of kvm_amd or something. But let me see what I can come up with.

Quoted 8 lines
Quoted 5 lines

Ugh, and it's not even powerman per se, it's actually a module in perf. Oof.

I 100% agree it'll be tricky, but I also stand by comments that neither the CCP driver or KVM should be allowed to silently pull the rug out from under the RAPL module.

Maybe something that can be added to the current sev= kernel command line parameter, e.g. sev=norapl, or such?

Yeah. The only question I have is if we expect end users to want to disable RAPL at runtime. If so, then we probably want a sysfs knob or something.

However, letting RAPL be toggled on/off will introduce some amount of complexity, as the kernel would need to negotiate/coordinate with the RAPL perf module and with the CPP driver to ensure RAPL stays in the "correct" state. E.g. if the perf module is loaded, then RAPL is effectively pinned "on". And if SNP has been initialized with RAPL_DIS, then RAPL is effectively pinned "off". Blech.

Quoted 7 lines
Quoted 1 line

Maybe even with a kernel config option for a default value?

Probably overkill?

Quoted 1 line

On SNP_SHUTDOWN it will be re-enabled if it was disabled.

Stating the obvious, if we do this, we open the can of worms I described above.

Unfortunately that's how the firmware works and since we do a shutdown on module unload, if you have ccp=m this is the behavior.

Maybe it makes sense to go the other way: have perf look for a ccp symbol that's loaded that says whether RAPL is usable or not, and refuse to allow access to the counters if it is? But it looks like there are several UAPIs for this (perf, /dev/amd-hsmp-*, sysfs), so it's not just one place, which is also ugly.

Quoted 11 lines
Quoted 11 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines

KVM then needs to communicate (and enforce?) the policy to userspace.

KVM doesn't need to enforce anything, the SEV firmware will generate a launch error for policy violation if it's not supported.

For communicating to userspace if it's not a kvm module parameter, one option is to mask it off in sev_get_snp_supported_policy() if it was

Did you mean sev_get_snp_policy_bits() or were you referring to the KVM ioctl() for retrieving them?

I was thinking of the ioctl() for retrieving them, but doing the masking in sev_get_snp_policy_bits() since it would be able to remember whether RAPL_DIS was set or not. Of course I merged the two in my head when typing the sentence :)

Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
Quoted 2 lines

initialized without the support. Then it'll be visible via KVM_X86_SNP_POLICY_BITS.

Ya, this is what I was envisioning.

Quoted 8 lines
Quoted 2 lines

It's still a valid policy bit (if supported by the platform), so I don't think masking it off is appropriate.

But it's not fully supported, no? I.e. won't the VM fail if it requests RAPL_DIS?

Ooh, presumably the subtle difference is that on a platform without RAPL_DIS at all, the VM will successfully launch and thus could run with RAPL enabled even if the VM requested RAPL_DIS?

I haven't tested this, but I would hope what you describe an error. I think Tom means it's supported by the architecture, it just needs to be enabled via reconfiguration.

Tycho

#10 lore raw

Re: [PATCH v2 3/4] KVM: SEV: Add the kvm-amd.rapl_disable module parameter

Quoted 37 lines · Tycho Andersen
Quoted 36 lines · Sean Christopherson
Quoted 15 lines · Tom Lendacky
Quoted 14 lines · Sean Christopherson
Quoted 12 lines · Tycho Andersen
Quoted 2 lines · Sean Christopherson

I'm pretty sure I said this earlier: KVM absolutely should not be able to disable RAPL for the entire system. That needs to be a power management thing.

You definitely noted "not CCP", I don't think I quite understood what that meant though:
https://lore.kernel.org/all/[email protected]/

I'm a little worried that putting it in power management will generate some weird dependencies, or weakref symbols that can't change things if they are loaded independently of kvm_amd or something. But let me see what I can come up with.

Quoted 8 lines
Quoted 5 lines

Ugh, and it's not even powerman per se, it's actually a module in perf. Oof.

I 100% agree it'll be tricky, but I also stand by comments that neither the CCP driver or KVM should be allowed to silently pull the rug out from under the RAPL module.

Maybe something that can be added to the current sev= kernel command line parameter, e.g. sev=norapl, or such?

Yeah. The only question I have is if we expect end users to want to disable RAPL at runtime. If so, then we probably want a sysfs knob or something.

However, letting RAPL be toggled on/off will introduce some amount of complexity, as the kernel would need to negotiate/coordinate with the RAPL perf module and with the CPP driver to ensure RAPL stays in the "correct" state. E.g. if the perf module is loaded, then RAPL is effectively pinned "on". And if SNP has been initialized with RAPL_DIS, then RAPL is effectively pinned "off". Blech.

Quoted 1 line

Maybe even with a kernel config option for a default value?

Quoted 8 lines
Quoted 5 lines

Probably overkill?

Quoted 1 line

On SNP_SHUTDOWN it will be re-enabled if it was disabled.

Stating the obvious, if we do this, we open the can of worms I described above.

Unfortunately that's how the firmware works and since we do a shutdown on module unload, if you have ccp=m this is the behavior.

Right, but that's just at the hardware level. The kernel can still leave RAPL "disabled" at a software level, i.e. can still disallow loading the RAPL perf module.

Quoted 3 lines

Maybe it makes sense to go the other way: have perf look for a ccp symbol that's loaded that says whether RAPL is usable or not, and refuse to allow access to the counters if it is?

Yeah, this is what I suggesting. Or rather, trying to suggest :-)

Quoted 39 lines

But it looks like there are several UAPIs for this (perf, /dev/amd-hsmp-*, sysfs), so it's not just one place, which is also ugly.

Quoted 11 lines
Quoted 11 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines

KVM then needs to communicate (and enforce?) the policy to userspace.

KVM doesn't need to enforce anything, the SEV firmware will generate a launch error for policy violation if it's not supported.

For communicating to userspace if it's not a kvm module parameter, one option is to mask it off in sev_get_snp_supported_policy() if it was

Did you mean sev_get_snp_policy_bits() or were you referring to the KVM ioctl() for retrieving them?

I was thinking of the ioctl() for retrieving them, but doing the masking in sev_get_snp_policy_bits() since it would be able to remember whether RAPL_DIS was set or not. Of course I merged the two in my head when typing the sentence :)

Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
Quoted 2 lines

initialized without the support. Then it'll be visible via KVM_X86_SNP_POLICY_BITS.

Ya, this is what I was envisioning.

Quoted 8 lines
Quoted 2 lines

It's still a valid policy bit (if supported by the platform), so I don't think masking it off is appropriate.

But it's not fully supported, no? I.e. won't the VM fail if it requests RAPL_DIS?

Ooh, presumably the subtle difference is that on a platform without RAPL_DIS at all, the VM will successfully launch and thus could run with RAPL enabled even if the VM requested RAPL_DIS?

I haven't tested this, but I would hope what you describe an error. I think Tom means it's supported by the architecture, it just needs to be enabled via reconfiguration.

Tycho

maintainer
#11 lore raw

Re: [PATCH v2 3/4] KVM: SEV: Add the kvm-amd.rapl_disable module parameter

In reply to Sean Christopherson
Quoted 34 lines · Sean Christopherson
Quoted 15 lines · Tom Lendacky
Quoted 14 lines · Sean Christopherson
Quoted 12 lines · Tycho Andersen
Quoted 2 lines · Sean Christopherson

I'm pretty sure I said this earlier: KVM absolutely should not be able to disable RAPL for the entire system. That needs to be a power management thing.

You definitely noted "not CCP", I don't think I quite understood what that meant though:
https://lore.kernel.org/all/[email protected]/

I'm a little worried that putting it in power management will generate some weird dependencies, or weakref symbols that can't change things if they are loaded independently of kvm_amd or something. But let me see what I can come up with.

Quoted 8 lines
Quoted 5 lines

Ugh, and it's not even powerman per se, it's actually a module in perf. Oof.

I 100% agree it'll be tricky, but I also stand by comments that neither the CCP driver or KVM should be allowed to silently pull the rug out from under the RAPL module.

Maybe something that can be added to the current sev= kernel command line parameter, e.g. sev=norapl, or such?

Yeah. The only question I have is if we expect end users to want to disable RAPL at runtime. If so, then we probably want a sysfs knob or something.

However, letting RAPL be toggled on/off will introduce some amount of complexity, as the kernel would need to negotiate/coordinate with the RAPL perf module and with the CPP driver to ensure RAPL stays in the "correct" state. E.g. if the perf module is loaded, then RAPL is effectively pinned "on". And if SNP has been initialized with RAPL_DIS, then RAPL is effectively pinned "off". Blech.

Quoted 3 lines
Quoted 1 line

Maybe even with a kernel config option for a default value?

Probably overkill?

Yeah, probably.

Quoted 22 lines
Quoted 2 lines

On SNP_SHUTDOWN it will be re-enabled if it was disabled.

Stating the obvious, if we do this, we open the can of worms I described above.

Quoted 17 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines

KVM then needs to communicate (and enforce?) the policy to userspace.

KVM doesn't need to enforce anything, the SEV firmware will generate a launch error for policy violation if it's not supported.

For communicating to userspace if it's not a kvm module parameter, one option is to mask it off in sev_get_snp_supported_policy() if it was

Did you mean sev_get_snp_policy_bits() or were you referring to the KVM ioctl() for retrieving them?

Quoted 4 lines
Quoted 2 lines

initialized without the support. Then it'll be visible via KVM_X86_SNP_POLICY_BITS.

Ya, this is what I was envisioning.

Quoted 8 lines
Quoted 2 lines

It's still a valid policy bit (if supported by the platform), so I don't think masking it off is appropriate.

But it's not fully supported, no? I.e. won't the VM fail if it requests RAPL_DIS?

Ooh, presumably the subtle difference is that on a platform without RAPL_DIS at all, the VM will successfully launch and thus could run with RAPL enabled even if the VM requested RAPL_DIS?

No, the VM launch will fail in that case, too. But I thought the difference there would be getting an INVALID_PARAM for specifying an unsupported policy bit, but I think you still get POLICY_FAILURE, so maybe it doesn't matter.

Thanks,
Tom

#12 lore raw

PATCH v2 4/4 KVM: selftests: Add a smoke test support for RAPL_DIS

In reply to Tycho Andersen

From: "Tycho Andersen (AMD)" <[email protected]>

If the hardware supports the RAPL_DIS policy bit and the ccp has been loaded with the RAPL_DIS bit set, make sure a VM can actually start using it.

---
 tools/testing/selftests/kvm/include/x86/sev.h |  1 +
 .../selftests/kvm/x86/sev_smoke_test.c        | 24 ++++++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)
tools/testing/selftests/kvm/include/x86/sev.h +1 -0
index 1af44c151d60..2bbac9cd192a 100644
@@ -28,6 +28,7 @@ enum sev_guest_state {
 #define SNP_POLICY_SMT		(1ULL << 16)
 #define SNP_POLICY_RSVD_MBO	(1ULL << 17)
 #define SNP_POLICY_DBG		(1ULL << 19)
+#define SNP_POLICY_RAPL_DIS	(1ULL << 23)
 
 #define GHCB_MSR_TERM_REQ	0x100
 
tools/testing/selftests/kvm/x86/sev_smoke_test.c +23 -1
index 1a49ee391586..15c848749de6 100644
@@ -243,6 +243,18 @@ static void test_sev_smoke(void *guest, u32 type, u64 policy)
 	}
 }
 
+static u64 supported_policy_mask(void)
+{
+	int kvm_fd = open_kvm_dev_path_or_exit();
+	u64 policy_mask = 0;
+
+	kvm_device_attr_get(kvm_fd, KVM_X86_GRP_SEV,
+			    KVM_X86_SNP_POLICY_BITS,
+			    &policy_mask);
+	close(kvm_fd);
+	return policy_mask;
+}
+
 int main(int argc, char *argv[])
 {
 	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SEV));
@@ -252,8 +264,18 @@ int main(int argc, char *argv[])
 	if (kvm_cpu_has(X86_FEATURE_SEV_ES))
 		test_sev_smoke(guest_sev_es_code, KVM_X86_SEV_ES_VM, SEV_POLICY_ES);
 
-	if (kvm_cpu_has(X86_FEATURE_SEV_SNP))
+	if (kvm_cpu_has(X86_FEATURE_SEV_SNP)) {
+		u64 supported_policy = supported_policy_mask();
+
 		test_sev_smoke(guest_snp_code, KVM_X86_SNP_VM, snp_default_policy());
 
+		if (supported_policy & SNP_POLICY_RAPL_DIS &&
+		    kvm_get_module_param_bool("kvm_amd", "rapl_disable")) {
+			u64 policy = snp_default_policy() | SNP_POLICY_RAPL_DIS;
+
+			test_sev_smoke(guest_snp_code, KVM_X86_SNP_VM, policy);
+		}
+	}
+
 	return 0;
 }
Signature · 1 line
2.53.0