KVM CPU frequency change hypercalls

drivers/cpufreq/cpufreq_userspace.c +4 -0

+int cpufreq_userspace_freq_up(int cpu)
+{
+	unsigned int curfreq, nextminfreq;
+	unsigned int ret = 0;

drivers/cpufreq/cpufreq_userspace.c +6 -0

+	mutex_unlock(&userspace_mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(cpufreq_userspace_freq_up);
+++ kvm-pvfreq/include/linux/cpufreq.h	2017-01-31 14:20:00.508613672 -0200

include/linux/cpufreq.h +1 -0

+#endif
 #endif /* _LINUX_CPUFREQ_H */

For most VMs, modifying the host frequency is an undesired operation. Introduce ioctl to enable the guest to modify host CPU frequency.

Signed-off-by Marcelo Tosatti <[email protected]>

---
 arch/x86/include/asm/kvm_host.h |    2 ++
 arch/x86/include/uapi/asm/kvm.h |    5 +++++
 arch/x86/kvm/x86.c              |   20 ++++++++++++++++++++
 include/uapi/linux/kvm.h        |    3 +++
 virt/kvm/kvm_main.c             |    2 ++
 5 files changed, 32 insertions(+)

Index: kvm-pvfreq/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +9 -0

+		struct kvm_vcpu_allow_freq freq;
+
+		memset(&freq, 0, sizeof(struct kvm_vcpu_allow_freq));
+		r = -EFAULT;
+		if (copy_to_user(&freq, argp, sizeof(freq)))
+			break;
+		r = 0;
+		break;
+	}
 	default:
 		r = -EINVAL;
 	}

Index: kvm-pvfreq/include/uapi/linux/kvm.h
===================================================================

include/uapi/linux/kvm.h +3 -0

@@ -871,6 +871,7 @@
 #define KVM_CAP_S390_USER_INSTR0 130
 #define KVM_CAP_MSI_DEVID 131
 #define KVM_CAP_PPC_HTM 132
+#define KVM_CAP_ALLOW_FREQ_HC 133
 
 #ifdef KVM_CAP_IRQ_ROUTING
 

@@ -1281,6 +1282,8 @@
 #define KVM_S390_GET_IRQ_STATE	  _IOW(KVMIO, 0xb6, struct kvm_s390_irq_state)
 /* Available with KVM_CAP_X86_SMM */
 #define KVM_SMI                   _IO(KVMIO,   0xb7)
+#define KVM_SET_VCPU_ALLOW_FREQ_HC   _IO(KVMIO,   0xb8)
+#define KVM_GET_VCPU_ALLOW_FREQ_HC   _IO(KVMIO,   0xb9)
 
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
 #define KVM_DEV_ASSIGN_PCI_2_3		(1 << 1)

Index: kvm-pvfreq/arch/x86/include/uapi/asm/kvm.h
===================================================================

arch/x86/include/uapi/asm/kvm.h +5 -0

@@ -357,4 +357,9 @@
 #define KVM_X86_QUIRK_LINT0_REENABLED	(1 << 0)
 #define KVM_X86_QUIRK_CD_NW_CLEARED	(1 << 1)
 
+struct kvm_vcpu_allow_freq {
+	__u16 enable;
+	__u16 pad[7];
+};
+
 #endif /* _ASM_X86_KVM_H */

Index: kvm-pvfreq/virt/kvm/kvm_main.c
===================================================================

virt/kvm/kvm_main.c +1 -0

+		return 1;
 	default:
 		break;
 	}

Index: kvm-pvfreq/arch/x86/include/asm/kvm_host.h
===================================================================

arch/x86/include/asm/kvm_host.h +2 -0

@@ -678,6 +678,8 @@
 
 	/* GPA available (AMD only) */
 	bool gpa_available;
+
+	bool allow_freq_hypercall;
 };
 
 struct kvm_lpage_info {

2017-02-02 15:47-0200, Marcelo Tosatti:

Quoted 14 lines

For most VMs, modifying the host frequency is an undesired operation. Introduce ioctl to enable the guest to modify host CPU frequency.

Signed-off-by Marcelo Tosatti <[email protected]>

---
 arch/x86/include/asm/kvm_host.h |    2 ++
 arch/x86/include/uapi/asm/kvm.h |    5 +++++
 arch/x86/kvm/x86.c              |   20 ++++++++++++++++++++
 include/uapi/linux/kvm.h        |    3 +++
 virt/kvm/kvm_main.c             |    2 ++
 5 files changed, 32 insertions(+)

arch/x86/kvm/x86.c line 3668

Quoted 9 lines

Index: kvm-pvfreq/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +1 -0

@@ -3665,6 +3665,26 @@
 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
 		break;
 	}
+	case KVM_SET_VCPU_ALLOW_FREQ_HC: {

Just enable the frequency hypercalls with KVM_ENABLE_CAP ioctl and get rid of this ioctl.
(I don't think that we want to allow disabling this capability.)

arch/x86/kvm/x86.c line 3674

Quoted 6 lines

arch/x86/kvm/x86.c +6 -0

+		struct kvm_vcpu_allow_freq freq;
+
+		r = -EFAULT;
+		if (copy_from_user(&freq, argp, sizeof(freq)))
+			goto out;
+		vcpu->arch.allow_freq_hypercall = freq.enable;

Enabling the capability should also set a bit in KVM_CPUID_FEATURES, a well-behaved guest won't use it otherwise.

arch/x86/kvm/x86.c line 3678

Quoted 4 lines

arch/x86/kvm/x86.c +4 -0

+		r = 0;
+		break;
+	}
+	case KVM_GET_VCPU_ALLOW_FREQ_HC: {

And this ioctl has no use, so it should be omitted.

(Userspace doesn't learn anything it doesn't already know.
The feature is unsafe, so the default must be 0.)

Quoted 51 lines

arch/x86/kvm/x86.c +9 -0

+		struct kvm_vcpu_allow_freq freq;
+
+		memset(&freq, 0, sizeof(struct kvm_vcpu_allow_freq));
+		r = -EFAULT;
+		if (copy_to_user(&freq, argp, sizeof(freq)))
+			break;
+		r = 0;
+		break;
+	}
 	default:
 		r = -EINVAL;
 	}

Index: kvm-pvfreq/include/uapi/linux/kvm.h
===================================================================

include/uapi/linux/kvm.h +3 -0

@@ -871,6 +871,7 @@
 #define KVM_CAP_S390_USER_INSTR0 130
 #define KVM_CAP_MSI_DEVID 131
 #define KVM_CAP_PPC_HTM 132
+#define KVM_CAP_ALLOW_FREQ_HC 133
 
 #ifdef KVM_CAP_IRQ_ROUTING
 

@@ -1281,6 +1282,8 @@
 #define KVM_S390_GET_IRQ_STATE	  _IOW(KVMIO, 0xb6, struct kvm_s390_irq_state)
 /* Available with KVM_CAP_X86_SMM */
 #define KVM_SMI                   _IO(KVMIO,   0xb7)
+#define KVM_SET_VCPU_ALLOW_FREQ_HC   _IO(KVMIO,   0xb8)
+#define KVM_GET_VCPU_ALLOW_FREQ_HC   _IO(KVMIO,   0xb9)
 
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
 #define KVM_DEV_ASSIGN_PCI_2_3		(1 << 1)

Index: kvm-pvfreq/arch/x86/include/uapi/asm/kvm.h
===================================================================

arch/x86/include/uapi/asm/kvm.h +5 -0

@@ -357,4 +357,9 @@
 #define KVM_X86_QUIRK_LINT0_REENABLED	(1 << 0)
 #define KVM_X86_QUIRK_CD_NW_CLEARED	(1 << 1)
 
+struct kvm_vcpu_allow_freq {
+	__u16 enable;
+	__u16 pad[7];
+};
+
 #endif /* _ASM_X86_KVM_H */

Index: kvm-pvfreq/virt/kvm/kvm_main.c
===================================================================

virt/kvm/kvm_main.c +0 -0

virt/kvm/kvm_main.c line 2941

Quoted 5 lines

virt/kvm/kvm_main.c +1 -0

@@ -2938,6 +2938,8 @@
 #endif
 	case KVM_CAP_MAX_VCPU_ID:
 		return KVM_MAX_VCPU_ID;
+	case KVM_CAP_ALLOW_FREQ_HC:

This can share an existing return 1 and would be elsewhere with KVM_ENABLE_CAP.

Quoted 15 lines

2017-02-02 15:47-0200, Marcelo Tosatti:

Quoted 14 lines

For most VMs, modifying the host frequency is an undesired operation. Introduce ioctl to enable the guest to modify host CPU frequency.

Signed-off-by Marcelo Tosatti <[email protected]>

---
 arch/x86/include/asm/kvm_host.h |    2 ++
 arch/x86/include/uapi/asm/kvm.h |    5 +++++
 arch/x86/kvm/x86.c              |   20 ++++++++++++++++++++
 include/uapi/linux/kvm.h        |    3 +++
 virt/kvm/kvm_main.c             |    2 ++
 5 files changed, 32 insertions(+)

Quoted 13 lines

arch/x86/kvm/x86.c line 3668

Quoted 9 lines

Index: kvm-pvfreq/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +1 -0

@@ -3665,6 +3665,26 @@
 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
 		break;
 	}
+	case KVM_SET_VCPU_ALLOW_FREQ_HC: {

Just enable the frequency hypercalls with KVM_ENABLE_CAP ioctl and get rid of this ioctl.
(I don't think that we want to allow disabling this capability.)

Not sure. What if you change the role of vcpus and now want to change vcpu-1 from a realtime vcpu (one where only DPDK runs) to a multi-user vcpu without a reboot?

Doing it manually doesn't sound useful outside of testing ... is DPDK actually being used "dynamically"?
(I thought that the setup is decided when the host boots.)

Implement min/max/up/down frequency change KVM hypercalls. To be used by DPDK implementation.

Also allow such hypercalls from guest userspace.

Signed-off-by Marcelo Tosatti <[email protected]>

---
 Documentation/virtual/kvm/hypercalls.txt |   45 +++++++++++++++++++
 arch/x86/kvm/x86.c                       |   71 ++++++++++++++++++++++++++++++-
 include/uapi/linux/kvm_para.h            |    5 ++
 3 files changed, 120 insertions(+), 1 deletion(-)

Index: kvm-pvfreq/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +48 -0

@@ -6219,10 +6219,58 @@
 	kvm_x86_ops->refresh_apicv_exec_ctrl(vcpu);
 }
 
+#ifdef CONFIG_CPU_FREQ_GOV_USERSPACE
+/* call into cpufreq-userspace governor */
+static int kvm_pvfreq_up(struct kvm_vcpu *vcpu)
+{
+	int ret;
+	int cpu = get_cpu();
+
+	ret = cpufreq_userspace_freq_up(cpu);
+	put_cpu();
+
+	return ret;
+}
+
+static int kvm_pvfreq_down(struct kvm_vcpu *vcpu)
+{
+	int ret;
+	int cpu = get_cpu();
+
+	ret = cpufreq_userspace_freq_down(cpu);
+	put_cpu();
+
+	return ret;
+}
+
+static int kvm_pvfreq_max(struct kvm_vcpu *vcpu)
+{
+	int ret;
+	int cpu = get_cpu();
+
+	ret = cpufreq_userspace_freq_max(cpu);
+	put_cpu();
+
+	return ret;
+}
+
+static int kvm_pvfreq_min(struct kvm_vcpu *vcpu)
+{
+	int ret;
+	int cpu = get_cpu();
+
+	ret = cpufreq_userspace_freq_min(cpu);
+	put_cpu();
+
+	return ret;
+}
+#endif
+
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
 {
 	unsigned long nr, a0, a1, a2, a3, ret;
 	int op_64_bit, r;
+	bool cpl_check;
 
 	r = kvm_skip_emulated_instruction(vcpu);
 

@@ -6246,7 +6294,13 @@
 		a3 &= 0xFFFFFFFF;
 	}
 

arch/x86/kvm/x86.c +8 -1

 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
 {
 	unsigned long nr, a0, a1, a2, a3, ret;
 	int op_64_bit, r;
+	bool cpl_check;
 
 	r = kvm_skip_emulated_instruction(vcpu);
 

@@ -6246,7 +6294,13 @@
 		a3 &= 0xFFFFFFFF;
 	}
 
-	if (kvm_x86_ops->get_cpl(vcpu) != 0) {
+	cpl_check = true;
+	if (nr == KVM_HC_FREQ_UP || nr == KVM_HC_FREQ_DOWN ||
+	    nr == KVM_HC_FREQ_MIN || nr == KVM_HC_FREQ_MAX)
+		if (vcpu->arch.allow_freq_hypercall == true)
+			cpl_check = false;
+
+	if (cpl_check == true && kvm_x86_ops->get_cpl(vcpu) != 0) {
 		ret = -KVM_EPERM;
 		goto out;
 	}

2017-02-02 15:47-0200, Marcelo Tosatti:

arch/x86/kvm/x86.c

Quoted 13 lines

Implement min/max/up/down frequency change KVM hypercalls. To be used by DPDK implementation.

Also allow such hypercalls from guest userspace.

Signed-off-by Marcelo Tosatti <[email protected]>

---
Index: kvm-pvfreq/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +0 -0

@@ -6219,10 +6219,58 @@

[Here lived copy-paste.]

Quoted 24 lines

arch/x86/kvm/x86.c +8 -1

 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
 {
 	unsigned long nr, a0, a1, a2, a3, ret;
 	int op_64_bit, r;
+	bool cpl_check;
 
 	r = kvm_skip_emulated_instruction(vcpu);
 

@@ -6246,7 +6294,13 @@
 		a3 &= 0xFFFFFFFF;
 	}
 
-	if (kvm_x86_ops->get_cpl(vcpu) != 0) {
+	cpl_check = true;
+	if (nr == KVM_HC_FREQ_UP || nr == KVM_HC_FREQ_DOWN ||
+	    nr == KVM_HC_FREQ_MIN || nr == KVM_HC_FREQ_MAX)
+		if (vcpu->arch.allow_freq_hypercall == true)
+			cpl_check = false;
+
+	if (cpl_check == true && kvm_x86_ops->get_cpl(vcpu) != 0) {
 		ret = -KVM_EPERM;
 		goto out;
 	}

arch/x86/kvm/x86.c line 6319

Quoted 4 lines

arch/x86/kvm/x86.c +1 -0

@@ -6262,6 +6316,21 @@
 	case KVM_HC_CLOCK_PAIRING:
 		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
 		break;
+#ifdef CONFIG_CPU_FREQ_GOV_USERSPACE

CONFIG_CPU_FREQ_GOV_USERSPACE should be checked when enabling the capability.

arch/x86/kvm/x86.c line 6331

Quoted 12 lines

arch/x86/kvm/x86.c +12 -0

+	case KVM_HC_FREQ_UP:
+		ret = kvm_pvfreq_up(vcpu);
+		break;
+	case KVM_HC_FREQ_DOWN:
+		ret = kvm_pvfreq_down(vcpu);
+		break;
+	case KVM_HC_FREQ_MAX:
+		ret = kvm_pvfreq_max(vcpu);
+		break;
+	case KVM_HC_FREQ_MIN:
+		ret = kvm_pvfreq_min(vcpu);
+		break;

Having 4 hypercalls for this is an overkill.
You can make it one hypercall with an argument.

And the argument doesn't have to be enum {UP, DOWN, MAX, MIN}, but an int, which would also allow you to do -2 steps.

A number over the capabilites of stepping would just map to MAX/MIN.

Avoiding an absolute scale for interface simplifies migration, where the guest cannot really depend much on this. Except that calling it with MIN (INT_MIN) will get the minimum and MAX (INT_MAX) the maximum frequency.

Plese explictly say in documentation that things like the number of steps, which the guest can learn by doing MAX and then -1 until the hypercall fails, is undefined and should not be depended upon.

Userspace might still want know the number of steps to avoid useless hypercall -- I think we should return a different value when the limit is reached, not just after the guest wants to go past it.

arch/x86/kvm/x86.c line 6336

Quoted 5 lines

arch/x86/kvm/x86.c +2 -0

+#endif
+
 	default:
 		ret = -KVM_ENOSYS;
 		break;

And thinking more about migration, userspace cannot learn the current frequency (at least MIN/MAX), so the new host will just pick at random, which will break userspace's expectations that it cannot increase or decrease the frequency. Is migration left for the future, because DPDK doesn't migrate anyway?

Thanks.

Quoted 1 line

You can make it one hypercall with an argument.

Fine.

Quoted 2 lines

And the argument doesn't have to be enum {UP, DOWN, MAX, MIN}, but an int, which would also allow you to do -2 steps.

Are you suggesting to have an integer to signify the number of steps up or down.

Quoted 1 line

A number over the capabilites of stepping would just map to MAX/MIN.

Then MAX == any positive value above the number of steps

     MIN == any negative value below the negative of number of steps

Sure.

Quoted 4 lines

Avoiding an absolute scale for interface simplifies migration, where the guest cannot really depend much on this. Except that calling it with MIN (INT_MIN) will get the minimum and MAX (INT_MAX) the maximum frequency.

Are you suggesting for the hypercall to return the maximum/minimum frequency if called with the highest integer and lowest negative integer respectively? (That same hypercall).

Quoted 3 lines

Plese explictly say in documentation that things like the number of steps, which the guest can learn by doing MAX and then -1 until the hypercall fails, is undefined and should not be depended upon.

Sure, because it fails over migration.

Quoted 3 lines

Userspace might still want know the number of steps to avoid useless hypercall -- I think we should return a different value when the limit is reached, not just after the guest wants to go past it.

Are you suggesting to return a different value when going from

max-1 -> max
and
min+1 -> min

frequencies?

Fine.

Quoted 12 lines

arch/x86/kvm/x86.c line 6336

Quoted 5 lines

arch/x86/kvm/x86.c +2 -0

+#endif
+
 	default:
 		ret = -KVM_ENOSYS;
 		break;

And thinking more about migration, userspace cannot learn the current frequency (at least MIN/MAX), so the new host will just pick at random, which will break userspace's expectations that it cannot increase or decrease the frequency. Is migration left for the future, because DPDK doesn't migrate anyway?

Quoted 1 line

Thanks.

The new host should start with the highest frequency always. Then the frequency tuning algorithm can reduce frequency afterwards.

Migration is a desired feature for DPDK, so it should be supported (thats one reason why virtio-net drivers are used in the guest BTW).

Implement KVM hypercalls for the guest
to issue frequency changes.

Current situation with DPDK and frequency changes is as follows:
An algorithm in the guest decides when to increase/decrease frequency based on the queue length of the device.

On the host, a power manager daemon is used to listen for frequency change requests (on another core) and issue these requests.

However frequency changes are performance sensitive events because:
On a change from low load condition to max load condition, the frequency should be raised as soon as possible. Sending a virtio-serial notification to another pCPU, waiting for that pCPU to initiate an IPI to the requestor pCPU to change frequency, is slower and more cache costly than a direct hypercall to host to switch the frequency.

If the pCPU where the power manager daemon is running is not busy spinning on requests from the isolated DPDK vcpus, there is also the cost of HLT wakeup for that pCPU.

Moreover, the daemon serves multiple VMs, meaning that the scheme is subject to additional delays from queueing of power change requests from VMs.

A direct hypercall from userspace is the fastest most direct method for the guest to change frequency and does not suffer from the issues above.

The usage scenario for this hypercalls is for pinned vCPUs <-> pCPUs.

Implement KVM hypercalls for the guest
to issue frequency changes.

Current situation with DPDK and frequency changes is as follows:
An algorithm in the guest decides when to increase/decrease frequency based on the queue length of the device.

On the host, a power manager daemon is used to listen for frequency change requests (on another core) and issue these requests.

However frequency changes are performance sensitive events because:
On a change from low load condition to max load condition, the frequency should be raised as soon as possible. Sending a virtio-serial notification to another pCPU, waiting for that pCPU to initiate an IPI to the requestor pCPU to change frequency, is slower and more cache costly than a direct hypercall to host to switch the frequency.

If the pCPU where the power manager daemon is running is not busy spinning on requests from the isolated DPDK vcpus, there is also the cost of HLT wakeup for that pCPU.

Moreover, the daemon serves multiple VMs, meaning that the scheme is subject to additional delays from queueing of power change requests from VMs.

A direct hypercall from userspace is the fastest most direct method for the guest to change frequency and does not suffer from the issues above.

The usage scenario for this hypercalls is for pinned vCPUs <-> pCPUs.

2017-02-02 15:47-0200, Marcelo Tosatti:

Quoted 6 lines

Implement KVM hypercalls for the guest
to issue frequency changes.

Current situation with DPDK and frequency changes is as follows:
An algorithm in the guest decides when to increase/decrease frequency based on the queue length of the device.

Does the algorithm compute with the magnitude of frequency steps?

(e.g. if CPU can step with 200 MHz granularity, does the algorithm ever
do 400 MHz at once, because it assumes that frequency would be enough to handle the load?)

Quoted 12 lines

On the host, a power manager daemon is used to listen for frequency change requests (on another core) and issue these requests.

However frequency changes are performance sensitive events because:
On a change from low load condition to max load condition, the frequency should be raised as soon as possible. Sending a virtio-serial notification to another pCPU, waiting for that pCPU to initiate an IPI to the requestor pCPU to change frequency, is slower and more cache costly than a direct hypercall to host to switch the frequency.

Quoted 7 lines

If the pCPU where the power manager daemon is running is not busy spinning on requests from the isolated DPDK vcpus, there is also the cost of HLT wakeup for that pCPU.

Moreover, the daemon serves multiple VMs, meaning that the scheme is subject to additional delays from queueing of power change requests from VMs.

(Wow, this must be bringing humanity to its doom faster than the heat it
helps to eliminate.)

Quoted 3 lines

A direct hypercall from userspace is the fastest most direct method for the guest to change frequency and does not suffer from the issues above.

Right, userspace on bare-metal cannot change frequency directly.

Quoted 1 line

The usage scenario for this hypercalls is for pinned vCPUs <-> pCPUs.

And pinned tasks <-> vCPUs, because the guest kernel has no idea what frequency is being used or desired on its virtualware,

so the kernel
cannot even change frequency without introducing a bug ...

I'm not happy about this hole through layers of isolations.

The domain of valid users is very small and a problem is that any program with access to /dev/kvm gains the ability to change host CPU frequency if the host happens to use the userspace governor.

We should at least enable this feature only if /dev/kvm is root-only.

Quoted 10 lines

2017-02-02 15:47-0200, Marcelo Tosatti:

Quoted 6 lines

Implement KVM hypercalls for the guest
to issue frequency changes.

Current situation with DPDK and frequency changes is as follows:
An algorithm in the guest decides when to increase/decrease frequency based on the queue length of the device.

Does the algorithm compute with the magnitude of frequency steps?

Quoted 3 lines

(e.g. if CPU can step with 200 MHz granularity, does the algorithm ever
do 400 MHz at once, because it assumes that frequency would be enough to handle the load?)

No, it does not know the frequency directly. It only "knows" the frequency indirectly by the size of the network queue (that is, if the network queue is above a threshold, then frequency is "too low" and should be increased).

Quoted 5 lines

Quoted 3 lines

A direct hypercall from userspace is the fastest most direct method for the guest to change frequency and does not suffer from the issues above.

Right, userspace on bare-metal cannot change frequency directly.

Yes it can: write to sysfs (not sure what you meant).

Quoted 4 lines

Quoted 1 line

The usage scenario for this hypercalls is for pinned vCPUs <-> pCPUs.

And pinned tasks <-> vCPUs, because the guest kernel has no idea what frequency is being used or desired on its virtualware,

And it does not have to know...

Quoted 2 lines

so the kernel
cannot even change frequency without introducing a bug ...

Not sure what are you thinking, please be more verbose.

Quoted 5 lines

I'm not happy about this hole through layers of isolations.

The domain of valid users is very small and a problem is that any program with access to /dev/kvm gains the ability to change host CPU frequency if the host happens to use the userspace governor.

Yes.

Quoted 1 line

We should at least enable this feature only if /dev/kvm is root-only.

Fine, can change that, will fix in -v2. Maybe there is a capability to change frequency... should require that capability (or root if there is none).

One reason why we have a kernel/userspace split is to allow sharing of CPU time. Each application then its state that the kernel keeps track of and saves/restores while time-multiplexing.

Our frequency scaling interface goes against the idea -- guest kernel cannot schedule multiple userspaces on the same vCPU, because they could conflict by overriding frequency.

i.e. our feature implies userspace tasks pinned to isolated vCPUs.

Quoted 4 lines · Radim Krcmar

One reason why we have a kernel/userspace split is to allow sharing of CPU time. Each application then its state that the kernel keeps track of and saves/restores while time-multiplexing.

Quoted 5 lines

Our frequency scaling interface goes against the idea -- guest kernel cannot schedule multiple userspaces on the same vCPU, because they could conflict by overriding frequency.

i.e. our feature implies userspace tasks pinned to isolated vCPUs.

That's bad. This feature is broken by design unless it does proper save/restore across preemption.

You don't need a hypercall. Add a cpufreq driver in DPDK that doesn't use sysfs, and connect it to a daemon in the host through virtio-serial or vsock.

Paolo

Quoted 1 line

Quoted 1 line

i.e. our feature implies userspace tasks pinned to isolated vCPUs.

This is how cpufreq-userspace works:

2.2 Governor
------------

On all other cpufreq implementations, these boundaries still need to be set. Then, a "governor" must be selected. Such a "governor" decides what speed the processor shall run within the boundaries. One such "governor" is the "userspace" governor. This one allows the user - or a yet-to-implement userspace program - to decide what specific speed the processor shall run at.

Quoted 2 lines

That's bad. This feature is broken by design unless it does proper save/restore across preemption.

Whats the current usecase, or forseeable future usecase, for save/restore across preemption again? (which would validate the broken by design claim).

Quoted 6 lines · Marcelo Tosatti

Quoted 1 line

Quoted 1 line

i.e. our feature implies userspace tasks pinned to isolated vCPUs.

This is how cpufreq-userspace works:

2.2 Governor
------------

Quoted 6 lines

On all other cpufreq implementations, these boundaries still need to be set. Then, a "governor" must be selected. Such a "governor" decides what speed the processor shall run within the boundaries. One such "governor" is the "userspace" governor. This one allows the user - or a yet-to-implement userspace program - to decide what specific speed the processor shall run at.

The userspace program sets a policy for the whole system.

Quoted 6 lines

Quoted 2 lines

That's bad. This feature is broken by design unless it does proper save/restore across preemption.

Whats the current usecase, or forseeable future usecase, for save/restore across preemption again? (which would validate the broken by design claim).

Stop a guest that is using cpufreq, start a guest that is not using it. The second guest's performance now depends on the state that the first guest left in cpufreq.

I think this is abusing the userspace governor. Unfortunately cpufreq governors cannot be stacked.

Paolo

Quoted 8 lines

Quoted 7 lines

On all other cpufreq implementations, these boundaries still need to be set. Then, a "governor" must be selected. Such a "governor" decides what speed the processor shall run within the boundaries. One such "governor" is the "userspace" governor. This one allows the user - or a yet-to-implement userspace program - to decide what specific speed the processor shall run at.

The userspace program sets a policy for the whole system.

No, its per cpu.

Quoted 5 lines

Quoted 5 lines

Quoted 2 lines

That's bad. This feature is broken by design unless it does proper save/restore across preemption.

Whats the current usecase, or forseeable future usecase, for save/restore across preemption again? (which would validate the broken by design claim).

Quoted 3 lines

Stop a guest that is using cpufreq, start a guest that is not using it. The second guest's performance now depends on the state that the first guest left in cpufreq.

Nothing forbids the host to implement switching with the current hypercall interface: all you need is a scheduler hook.

Quoted 9 lines · Marcelo Tosatti

Quoted 8 lines

Quoted 7 lines

On all other cpufreq implementations, these boundaries still need to be set. Then, a "governor" must be selected. Such a "governor" decides what speed the processor shall run within the boundaries. One such "governor" is the "userspace" governor. This one allows the user - or a yet-to-implement userspace program - to decide what specific speed the processor shall run at.

The userspace program sets a policy for the whole system.

No, its per cpu.

Yeah, what I mean is that userspace program can be per-CPU, but it looks at all the processes running on that CPU ("the whole system"). This is very different from a guest, which is isolated.

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 2 lines

That's bad. This feature is broken by design unless it does proper save/restore across preemption.

Whats the current usecase, or forseeable future usecase, for save/restore across preemption again? (which would validate the broken by design claim).

Quoted 7 lines

Quoted 3 lines

Stop a guest that is using cpufreq, start a guest that is not using it. The second guest's performance now depends on the state that the first guest left in cpufreq.

Nothing forbids the host to implement switching with the current hypercall interface: all you need is a scheduler hook.

Can it be done in vcpu_load/vcpu_put? But you still would have two components (KVM and sysfs) potentially fighting over the frequency, and that's still a bit ugly.

Paolo

Quoted 9 lines

Quoted 9 lines

Quoted 6 lines

Quoted 3 lines

Whats the current usecase, or forseeable future usecase, for save/restore across preemption again? (which would validate the broken by design claim).

Stop a guest that is using cpufreq, start a guest that is not using it. The second guest's performance now depends on the state that the first guest left in cpufreq.

Nothing forbids the host to implement switching with the current hypercall interface: all you need is a scheduler hook.

Quoted 3 lines

Can it be done in vcpu_load/vcpu_put? But you still would have two components (KVM and sysfs) potentially fighting over the frequency, and that's still a bit ugly.

Change the frequency at vcpu_load/vcpu_put? Yes: call into cpufreq-userspace. But there is no notion of "per-task frequency" on the Linux kernel (which was the starting point of this subthread).

But if you configure all CPUs in the system as cpufreq-userspace, then some other (userspace program) has to decide the frequency for the other CPUs.

Which agent would do that and why? Thats why i initially said "whats the usecase".

Quoted 9 lines · Marcelo Tosatti

Quoted 9 lines

Quoted 9 lines

Quoted 6 lines

Quoted 3 lines

Whats the current usecase, or forseeable future usecase, for save/restore across preemption again? (which would validate the broken by design claim).

Stop a guest that is using cpufreq, start a guest that is not using it. The second guest's performance now depends on the state that the first guest left in cpufreq.

Nothing forbids the host to implement switching with the current hypercall interface: all you need is a scheduler hook.

Quoted 7 lines

Quoted 3 lines

Can it be done in vcpu_load/vcpu_put? But you still would have two components (KVM and sysfs) potentially fighting over the frequency, and that's still a bit ugly.

Change the frequency at vcpu_load/vcpu_put? Yes: call into cpufreq-userspace. But there is no notion of "per-task frequency" on the Linux kernel (which was the starting point of this subthread).

There isn't, but this patchset is providing a direct path from a task to cpufreq-userspace. This is as close as you can get to a per-task frequency.

Quoted 4 lines

But if you configure all CPUs in the system as cpufreq-userspace, then some other (userspace program) has to decide the frequency for the other CPUs.

Quoted 2 lines

Which agent would do that and why? Thats why i initially said "whats the usecase".

You could just pin them at the highest non-TurboBoost frequency until a guest runs. That's assuming that they are idle and, because of isol_cpus/nohz_full, they would be almost always in deep C state anyway.

Quoted 13 lines · Marcelo Tosatti

Quoted 12 lines

Quoted 9 lines

Quoted 6 lines

Quoted 3 lines

Whats the current usecase, or forseeable future usecase, for save/restore across preemption again? (which would validate the broken by design claim).

Stop a guest that is using cpufreq, start a guest that is not using it. The second guest's performance now depends on the state that the first guest left in cpufreq.

Nothing forbids the host to implement switching with the current hypercall interface: all you need is a scheduler hook.

Can it be done in vcpu_load/vcpu_put? But you still would have two components (KVM and sysfs) potentially fighting over the frequency, and that's still a bit ugly.

Quoted 3 lines

Change the frequency at vcpu_load/vcpu_put? Yes: call into cpufreq-userspace. But there is no notion of "per-task frequency" on the Linux kernel (which was the starting point of this subthread).

There isn't, but this patchset is providing a direct path from a task to cpufreq-userspace. This is as close as you can get to a per-task frequency.

Quoted 4 lines

But if you configure all CPUs in the system as cpufreq-userspace, then some other (userspace program) has to decide the frequency for the other CPUs.

Quoted 2 lines

Which agent would do that and why? Thats why i initially said "whats the usecase".

You could just pin them at the highest non-TurboBoost frequency until a guest runs. That's assuming that they are idle and, because of isol_cpus/nohz_full, they would be almost always in deep C state anyway.

Paolo

Quoted 20 lines · Paolo Bonzini

Quoted 16 lines · Marcelo Tosatti

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 3 lines

Whats the current usecase, or forseeable future usecase, for save/restore across preemption again? (which would validate the broken by design claim).

Stop a guest that is using cpufreq, start a guest that is not using it. The second guest's performance now depends on the state that the first guest left in cpufreq.

Quoted 6 lines

Quoted 3 lines

Nothing forbids the host to implement switching with the current hypercall interface: all you need is a scheduler hook.

Can it be done in vcpu_load/vcpu_put? But you still would have two components (KVM and sysfs) potentially fighting over the frequency, and that's still a bit ugly.

Change the frequency at vcpu_load/vcpu_put? Yes: call into cpufreq-userspace. But there is no notion of "per-task frequency" on the Linux kernel (which was the starting point of this subthread).

Quoted 2 lines

There isn't, but this patchset is providing a direct path from a task to cpufreq-userspace. This is as close as you can get to a per-task frequency.

Cpufreq-userspace is supposed to be used by tasks in userspace. Thats why its called "userspace".

Quoted 10 lines

Quoted 6 lines

But if you configure all CPUs in the system as cpufreq-userspace, then some other (userspace program) has to decide the frequency for the other CPUs.

Which agent would do that and why? Thats why i initially said "whats the usecase".

You could just pin them at the highest non-TurboBoost frequency until a guest runs. That's assuming that they are idle and, because of isol_cpus/nohz_full, they would be almost always in deep C state anyway.

The original claim of the thread was: "this feature (frequency hypercalls) works for pinned vcpu<->pcpu, pcpu dedicated exclusively to vcpu case, lets try to extend this to other cases".

Which is a valid and useful direction to go.

However there is no user for multiple vcpus in the same pcpu now.

If there were multiple vcpus, all of them requesting a given frequency, it would be necessary to:

1) Maintain frequency of the pcpu to the highest frequencies.

OR

2) Since switching frequencies can take up to 70us (*) (depends on processor), its generally not worthwhile to switch frequencies between task switches.

Quoted 21 lines · Marcelo Tosatti

Quoted 20 lines · Paolo Bonzini

Quoted 16 lines · Marcelo Tosatti

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 3 lines

Whats the current usecase, or forseeable future usecase, for save/restore across preemption again? (which would validate the broken by design claim).

Stop a guest that is using cpufreq, start a guest that is not using it. The second guest's performance now depends on the state that the first guest left in cpufreq.

Quoted 6 lines

Quoted 3 lines

Nothing forbids the host to implement switching with the current hypercall interface: all you need is a scheduler hook.

Can it be done in vcpu_load/vcpu_put? But you still would have two components (KVM and sysfs) potentially fighting over the frequency, and that's still a bit ugly.

Change the frequency at vcpu_load/vcpu_put? Yes: call into cpufreq-userspace. But there is no notion of "per-task frequency" on the Linux kernel (which was the starting point of this subthread).

Quoted 5 lines

Quoted 2 lines

There isn't, but this patchset is providing a direct path from a task to cpufreq-userspace. This is as close as you can get to a per-task frequency.

Cpufreq-userspace is supposed to be used by tasks in userspace. Thats why its called "userspace".

I think the intended usecase is to have a daemon handling a systemwide policy. Examples are the historical (and now obsolete) users such as cpufreqd, cpudyn, powernowd, or cpuspeed. The user alternatively can play the role of the daemon by writing to sysfs.

I've never seen userspace tasks talking to cpufreq-userspace to set their own running frequency. If DPDK does it, that's nasty in my opinion

and we should find an interface that works best for both DPDK
and KVM. Which should be done on linux-pm like Rafael suggested.

Quoted 15 lines

Quoted 10 lines

Quoted 6 lines

But if you configure all CPUs in the system as cpufreq-userspace, then some other (userspace program) has to decide the frequency for the other CPUs.

Which agent would do that and why? Thats why i initially said "whats the usecase".

You could just pin them at the highest non-TurboBoost frequency until a guest runs. That's assuming that they are idle and, because of isol_cpus/nohz_full, they would be almost always in deep C state anyway.

The original claim of the thread was: "this feature (frequency hypercalls) works for pinned vcpu<->pcpu, pcpu dedicated exclusively to vcpu case, lets try to extend this to other cases".

Quoted 3 lines

Which is a valid and useful direction to go.

However there is no user for multiple vcpus in the same pcpu now.

You are still ignoring the case of one guest started after another, or of another program started on a CPU that formerly was used by KVM. They don't have to be multiple users at the same time.

Quoted 8 lines

If there were multiple vcpus, all of them requesting a given frequency, it would be necessary to:

1) Maintain frequency of the pcpu to the highest frequencies.

OR

Quoted 3 lines

2) Since switching frequencies can take up to 70us (*) (depends on processor), its generally not worthwhile to switch frequencies between task switches.

Is latency that important, or is rather overhead the one to pay attention to? The slides you linked
(http://www.ena-hpc.org/2013/pdf/04.pdf) at page 17 suggest it's around 10us.

One possibility is to do (1) if you have multiple tasks on the run queue (or fallback to what is specified in sysfs) and (2) if you only have one task.

Anyway, please repost with Cc to linux-pm so that we can restart the discussion there.

Paolo