Skip to contentSkip to conversation

Add support for RAPL MSRs in KVM/Qemu

PATCH In review Anthony Harivel 2 participants 3 messages Jun 16, 2023 – Jun 28, 2023 Arm/Amlogic Meson SoC support

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Add support for RAPL MSRs in KVM/Qemu

Starting with the "Sandy Bridge" generation, Intel CPUs provide a RAPL interface (Running Average Power Limit) for advertising the accumulated energy consumption of various power domains (e.g. CPU packages, DRAM, etc.).

The consumption is reported via MSRs (model specific registers) like MSR_PKG_ENERGY_STATUS for the CPU package power domain. These MSRs are 64 bits registers that represent the accumulated energy consumption in micro Joules. They are updated by microcode every ~1ms.

For now, KVM always returns 0 when the guest requests the value of these MSRs. Use the KVM MSR filtering mechanism to allow QEMU handle these MSRs dynamically in userspace.

To limit the amount of system calls for every MSR call, create a new thread in QEMU that updates the "virtual" MSR values asynchronously.

Each vCPU has its own vMSR to reflect the independence of vCPUs. The thread updates the vMSR values with the ratio of energy consumed of the whole physical CPU package the vCPU thread runs on and the thread's utime and stime values.

All other non-vCPU threads are also taken into account. Their energy consumption is evenly distributed among all vCPUs threads running on the same physical CPU package.

This feature is activated with -accel kvm,rapl=true.

Actual limitation:
- Works only on Intel host CPU because AMD CPUs are using different MSR adresses.

- Only the Package Power-Plane (MSR_PKG_ENERGY_STATUS) is reported at the moment.

- Since each vCPU has an independent vMSR value, the vCPU topology must be changed to match that reality. There must be a single vCPU per virtual socket (e.g.: -smp 4,sockets=4). Accessing pkg-0 energy will give vCPU 0 energy, pkg-1 will give vCPU 1 energy, etc.

---

Notes:
This is a follow-up of the RFC sent a month exactly ago [1].

RFC -> v1
---------

- Add vmsr_* in front of all vmsr specific function

- Change malloc()/calloc()... with all glib equivalent

- Pre-allocate all dynamic memories when possible

- Add a Documentation of implementation, limitation and usage

Regards,
Anthony

[1]: https://lore.kernel.org/qemu-devel/[email protected]/

 accel/kvm/kvm-all.c           |  13 ++
 docs/specs/index.rst          |   1 +
 docs/specs/rapl-msr.rst       | 128 +++++++++++++++
 include/sysemu/kvm_int.h      |  11 ++
 target/i386/cpu.h             |   8 +
 target/i386/kvm/kvm.c         | 301 ++++++++++++++++++++++++++++++++++
 target/i386/kvm/meson.build   |   1 +
 target/i386/kvm/vmsr_energy.c | 178 ++++++++++++++++++++
 target/i386/kvm/vmsr_energy.h |  81 +++++++++
 9 files changed, 722 insertions(+)
 create mode 100644 docs/specs/rapl-msr.rst
 create mode 100644 target/i386/kvm/vmsr_energy.c
 create mode 100644 target/i386/kvm/vmsr_energy.h
accel/kvm/kvm-all.c +13 -0
index 7679f397aec0..33bbdb598533 100644
@@ -3746,6 +3746,12 @@ static void kvm_set_dirty_ring_size(Object *obj, Visitor *v,
     s->kvm_dirty_ring_size = value;
 }
 
+static void kvm_set_kvm_rapl(Object *obj, bool value, Error **errp)
+{
+    KVMState *s = KVM_STATE(obj);
+    s->msr_energy.enable = value;
+}
+
 static void kvm_accel_instance_init(Object *obj)
 {
     KVMState *s = KVM_STATE(obj);
@@ -3763,6 +3769,7 @@ static void kvm_accel_instance_init(Object *obj)
     s->xen_version = 0;
     s->xen_gnttab_max_frames = 64;
     s->xen_evtchn_max_pirq = 256;
+    s->msr_energy.enable = false;
 }
 
 /**
@@ -3803,6 +3810,12 @@ static void kvm_accel_class_init(ObjectClass *oc, void *data)
     object_class_property_set_description(oc, "dirty-ring-size",
         "Size of KVM dirty page ring buffer (default: 0, i.e. use bitmap)");
 
+    object_class_property_add_bool(oc, "rapl",
+                                   NULL,
+                                   kvm_set_kvm_rapl);
+    object_class_property_set_description(oc, "rapl",
+        "Allow energy related MSRs for RAPL interface in Guest");
+
     kvm_arch_accel_class_init(oc);
 }
 
docs/specs/index.rst +1 -0
index e58be38c41c7..5c2fa3d65877 100644
@@ -24,3 +24,4 @@ guest hardware that is specific to QEMU.
    acpi_erst
    sev-guest-firmware
    fw_cfg
+   rapl-msr
docs/specs/rapl-msr.rst new file +128 -0

128 lines not on this page · raw message

include/sysemu/kvm_int.h +11 -0
index 511b42bde5c4..09e4d6809b05 100644
@@ -47,6 +47,16 @@ typedef struct KVMMemoryListener {
 
 #define KVM_MSI_HASHTAB_SIZE    256
 
+struct KVMMsrEnergy {
+    bool enable;
+    QemuThread msr_thr;
+    int cpus;
+    uint64_t *msr_value;
+    uint64_t msr_unit;
+    uint64_t msr_limit;
+    uint64_t msr_info;
+};
+
 enum KVMDirtyRingReaperState {
     KVM_DIRTY_RING_REAPER_NONE = 0,
     /* The reaper is sleeping */
@@ -117,6 +127,7 @@ struct KVMState
     uint32_t kvm_dirty_ring_size;   /* Number of dirty GFNs per ring */
     bool kvm_dirty_ring_with_bitmap;
     struct KVMDirtyRingReaper reaper;
+    struct KVMMsrEnergy msr_energy;
     NotifyVmexitOption notify_vmexit;
     uint32_t notify_window;
     uint32_t xen_version;
target/i386/cpu.h +8 -0
index cd047e04102f..506a96867b45 100644
@@ -396,6 +396,10 @@ typedef enum X86Seg {
 #define MSR_IA32_TSX_CTRL		0x122
 #define MSR_IA32_TSCDEADLINE            0x6e0
 #define MSR_IA32_PKRS                   0x6e1
+#define MSR_RAPL_POWER_UNIT             0x00000606
+#define MSR_PKG_POWER_LIMIT             0x00000610
+#define MSR_PKG_ENERGY_STATUS           0x00000611
+#define MSR_PKG_POWER_INFO              0x00000614
 #define MSR_ARCH_LBR_CTL                0x000014ce
 #define MSR_ARCH_LBR_DEPTH              0x000014cf
 #define MSR_ARCH_LBR_FROM_0             0x00001500
@@ -1775,6 +1779,10 @@ typedef struct CPUArchState {
 
     uintptr_t retaddr;
 
+    /* RAPL MSR */
+    uint64_t msr_rapl_power_unit;
+    uint64_t msr_pkg_energy_status;
+
     /* Fields up to this point are cleared by a CPU reset */
     struct {} end_reset_fields;
 
target/i386/kvm/kvm.c +301 -0

336 lines not on this page · raw message

target/i386/kvm/meson.build +1 -0
index 322272091bce..9cdc93c6c439 100644
@@ -5,6 +5,7 @@ i386_softmmu_kvm_ss = ss.source_set()
 i386_softmmu_kvm_ss.add(files(
   'kvm.c',
   'kvm-cpu.c',
+  'vmsr_energy.c',
 ))
 
 i386_softmmu_kvm_ss.add(when: 'CONFIG_XEN_EMU', if_true: files('xen-emu.c'))
target/i386/kvm/vmsr_energy.c new file +178 -0

178 lines not on this page · raw message

target/i386/kvm/vmsr_energy.h new file +81 -0

81 lines not on this page · raw message

Signature · 1 line
2.40.1
#1 lore raw
Quoted 22 lines · Anthony Harivel

Starting with the "Sandy Bridge" generation, Intel CPUs provide a RAPL interface (Running Average Power Limit) for advertising the accumulated energy consumption of various power domains (e.g. CPU packages, DRAM, etc.).

The consumption is reported via MSRs (model specific registers) like MSR_PKG_ENERGY_STATUS for the CPU package power domain. These MSRs are 64 bits registers that represent the accumulated energy consumption in micro Joules. They are updated by microcode every ~1ms.

For now, KVM always returns 0 when the guest requests the value of these MSRs. Use the KVM MSR filtering mechanism to allow QEMU handle these MSRs dynamically in userspace.

To limit the amount of system calls for every MSR call, create a new thread in QEMU that updates the "virtual" MSR values asynchronously.

Each vCPU has its own vMSR to reflect the independence of vCPUs. The thread updates the vMSR values with the ratio of energy consumed of the whole physical CPU package the vCPU thread runs on and the thread's utime and stime values.

Quoted 5 lines

All other non-vCPU threads are also taken into account. Their energy consumption is evenly distributed among all vCPUs threads running on the same physical CPU package.

This feature is activated with -accel kvm,rapl=true.

I suppose this should be a CPU flag instead? -cpu xxx,rapl=on.

#2 lore raw

Marcelo Tosatti, Jun 28, 2023 at 19:26:

Hi Marcelo,

Quoted 4 lines · Anthony Harivel

This feature is activated with -accel kvm,rapl=true.

Quoted 1 line

I suppose this should be a CPU flag instead? -cpu xxx,rapl=on.

It's possible yes then I might need to make sure that '-accel=kvm' is selected because this rapl feature depends on the KVM msr filtering mecanism.

Like '-cpu host' is checking that KVM or HVF is selected.

Thanks
Anthony