Add support for RAPL MSRs in KVM/Qemu
PATCH 1/1 Add support for RAPL MSRs in KVM/Qemu
Starting with the "Sandy Bridge" generation, Intel CPUs provide a RAPL interface (Running Average Power Limit) for advertising the accumulated energy consumption of various power domains (e.g. CPU packages, DRAM, etc.).
The consumption is reported via MSRs (model specific registers) like MSR_PKG_ENERGY_STATUS for the CPU package power domain. These MSRs are 64 bits registers that represent the accumulated energy consumption in micro Joules. They are updated by microcode every ~1ms.
For now, KVM always returns 0 when the guest requests the value of these MSRs. Use the KVM MSR filtering mechanism to allow QEMU handle these MSRs dynamically in userspace.
To limit the amount of system calls for every MSR call, create a new thread in QEMU that updates the "virtual" MSR values asynchronously.
Each vCPU has its own vMSR to reflect the independence of vCPUs. The thread updates the vMSR values with the ratio of energy consumed of the whole physical CPU package the vCPU thread runs on and the thread's utime and stime values.
All other non-vCPU threads are also taken into account. Their energy consumption is evenly distributed among all vCPUs threads running on the same physical CPU package.
This feature is activated with -accel kvm,rapl=true.
Actual limitation:
- Works only on Intel host CPU because AMD CPUs are using different MSR adresses.
- Only the Package Power-Plane (MSR_PKG_ENERGY_STATUS) is reported at the moment.
- Since each vCPU has an independent vMSR value, the vCPU topology must be changed to match that reality. There must be a single vCPU per virtual socket (e.g.: -smp 4,sockets=4). Accessing pkg-0 energy will give vCPU 0 energy, pkg-1 will give vCPU 1 energy, etc.
---
Notes:
This is a follow-up of the RFC sent a month exactly ago [1].
RFC -> v1
---------
- Add vmsr_* in front of all vmsr specific function
- Change malloc()/calloc()... with all glib equivalent
- Pre-allocate all dynamic memories when possible
- Add a Documentation of implementation, limitation and usage
Regards,
Anthony
[1]: https://lore.kernel.org/qemu-devel/[email protected]/
accel/kvm/kvm-all.c | 13 ++ docs/specs/index.rst | 1 + docs/specs/rapl-msr.rst | 128 +++++++++++++++ include/sysemu/kvm_int.h | 11 ++ target/i386/cpu.h | 8 + target/i386/kvm/kvm.c | 301 ++++++++++++++++++++++++++++++++++ target/i386/kvm/meson.build | 1 + target/i386/kvm/vmsr_energy.c | 178 ++++++++++++++++++++ target/i386/kvm/vmsr_energy.h | 81 +++++++++ 9 files changed, 722 insertions(+) create mode 100644 docs/specs/rapl-msr.rst create mode 100644 target/i386/kvm/vmsr_energy.c create mode 100644 target/i386/kvm/vmsr_energy.h
accel/kvm/kvm-all.c +13 -0
@@ -3746,6 +3746,12 @@ static void kvm_set_dirty_ring_size(Object *obj, Visitor *v, s->kvm_dirty_ring_size = value; } +static void kvm_set_kvm_rapl (Object *obj,bool value,Error **errp) +{ +KVMState *s =KVM_STATE (obj); + s->msr_energy.enable = value; +} +static void kvm_accel_instance_init (Object *obj) {KVMState *s =KVM_STATE (obj);
@@ -3763,6 +3769,7 @@ static void kvm_accel_instance_init(Object *obj) s->xen_version =0 ; s->xen_gnttab_max_frames =64 ; s->xen_evtchn_max_pirq =256 ; + s->msr_energy.enable = false; }/**
@@ -3803,6 +3810,12 @@ static void kvm_accel_class_init(ObjectClass *oc, void *data)object_class_property_set_description (oc,"dirty-ring-size" ,"Size of KVM dirty page ring buffer (default: 0, i.e. use bitmap)" ); +object_class_property_add_bool (oc,"rapl" , +NULL , + kvm_set_kvm_rapl); +object_class_property_set_description (oc,"rapl" , +"Allow energy related MSRs for RAPL interface in Guest" ); +kvm_arch_accel_class_init (oc); }
docs/specs/index.rst +1 -0
@@ -24,3 +24,4 @@ guest hardware that is specific to QEMU. acpi_erst sev-guest-firmware fw_cfg + rapl-msr
docs/specs/rapl-msr.rst new file +128 -0
128 lines not on this page · raw message
include/sysemu/kvm_int.h +11 -0
@@ -47,6 +47,16 @@ typedef struct KVMMemoryListener {#define KVM_MSI_HASHTAB_SIZE 256 +struct KVMMsrEnergy { +bool enable; +QemuThread msr_thr; +int cpus; +uint64_t *msr_value; +uint64_t msr_unit; +uint64_t msr_limit; +uint64_t msr_info; +}; +enum KVMDirtyRingReaperState {KVM_DIRTY_RING_REAPER_NONE =0 ,/* The reaper is sleeping */
@@ -117,6 +127,7 @@ struct KVMStateuint32_t kvm_dirty_ring_size;/* Number of dirty GFNs per ring */ bool kvm_dirty_ring_with_bitmap;struct KVMDirtyRingReaper reaper; +struct KVMMsrEnergy msr_energy;NotifyVmexitOption notify_vmexit;uint32_t notify_window;uint32_t xen_version;
target/i386/cpu.h +8 -0
@@ -396,6 +396,10 @@ typedef enum X86Seg {#define MSR_IA32_TSX_CTRL 0x122#define MSR_IA32_TSCDEADLINE 0x6e0#define MSR_IA32_PKRS 0x6e1 +#define MSR_RAPL_POWER_UNIT 0x00000606 +#define MSR_PKG_POWER_LIMIT 0x00000610 +#define MSR_PKG_ENERGY_STATUS 0x00000611 +#define MSR_PKG_POWER_INFO 0x00000614#define MSR_ARCH_LBR_CTL 0x000014ce#define MSR_ARCH_LBR_DEPTH 0x000014cf#define MSR_ARCH_LBR_FROM_0 0x00001500
@@ -1775,6 +1779,10 @@ typedef struct CPUArchState {uintptr_t retaddr; +/* RAPL MSR */ +uint64_t msr_rapl_power_unit; +uint64_t msr_pkg_energy_status; +/* Fields up to this point are cleared by a CPU reset */ struct {} end_reset_fields;
target/i386/kvm/kvm.c +301 -0
336 lines not on this page · raw message
target/i386/kvm/meson.build +1 -0
@@ -5,6 +5,7 @@ i386_softmmu_kvm_ss = ss.source_set() i386_softmmu_kvm_ss.add(files( 'kvm.c', 'kvm-cpu.c', + 'vmsr_energy.c', )) i386_softmmu_kvm_ss.add(when: 'CONFIG_XEN_EMU', if_true: files('xen-emu.c'))
target/i386/kvm/vmsr_energy.c new file +178 -0
178 lines not on this page · raw message
target/i386/kvm/vmsr_energy.h new file +81 -0
81 lines not on this page · raw message
Signature · 1 line
2.40.1
Quoted 22 lines · Anthony Harivel
Starting with the "Sandy Bridge" generation, Intel CPUs provide a RAPL interface (Running Average Power Limit) for advertising the accumulated energy consumption of various power domains (e.g. CPU packages, DRAM, etc.).
The consumption is reported via MSRs (model specific registers) like
MSR_PKG_ENERGY_STATUSfor the CPU package power domain. These MSRs are 64 bits registers that represent the accumulated energy consumption in micro Joules. They are updated by microcode every ~1ms.For now, KVM always returns 0 when the guest requests the value of these MSRs. Use the KVM MSR filtering mechanism to allow QEMU handle these MSRs dynamically in userspace.
To limit the amount of system calls for every MSR call, create a new thread in QEMU that updates the "virtual" MSR values asynchronously.
Each vCPU has its own vMSR to reflect the independence of vCPUs. The thread updates the vMSR values with the ratio of energy consumed of the whole physical CPU package the vCPU thread runs on and the thread's utime and stime values.
Quoted 5 lines
All other non-vCPU threads are also taken into account. Their energy consumption is evenly distributed among all vCPUs threads running on the same physical CPU package.
This feature is activated with -accel kvm,rapl=true.
I suppose this should be a CPU flag instead? -cpu xxx,rapl=on.
Marcelo Tosatti, Jun 28, 2023 at 19:26:
Hi Marcelo,
Quoted 4 lines · Anthony Harivel
This feature is activated with -accel kvm,rapl=true.
Quoted 1 line
I suppose this should be a CPU flag instead? -cpu xxx,rapl=on.
It's possible yes then I might need to make sure that '-accel=kvm' is selected because this rapl feature depends on the KVM msr filtering mecanism.
Like '-cpu host' is checking that KVM or HVF is selected.
Thanks
Anthony