Skip to contentSkip to conversation

Add TCG & KVM support for MSR_CORE_THREAD_COUNT

PATCH Applied Alexander Graf 2 participants 5 messages Oct 4, 2022 – Oct 5, 2022 KVM

lore
Share & tools
lore
#0 lore raw

cover Add TCG & KVM support for MSR_CORE_THREAD_COUNT

Commit 027ac0cb516 ("target/i386/hvf: add rdmsr 35H MSR_CORE_THREAD_COUNT") added support for the MSR_CORE_THREAD_COUNT MSR to HVF. This MSR is mandatory to execute macOS when run with -cpu host,+hypervisor.

This patch set adds support for the very same MSR to TCG as well as KVM - as long as host KVM is recent enough to support MSR trapping.

With this support added, I can successfully execute macOS guests in KVM with an APFS enabled OVMF build, a valid applesmc plus OSK and

-cpu Skylake-Client,+invtsc,+hypervisor

Alex

Alexander Graf (3):
x86: Implement MSR_CORE_THREAD_COUNT MSR
i386: kvm: Add support for MSR filtering
KVM: x86: Implement MSR_CORE_THREAD_COUNT MSR

 target/i386/kvm/kvm.c                | 145 +++++++++++++++++++++++++++
 target/i386/kvm/kvm_i386.h           |  11 ++
 target/i386/tcg/sysemu/misc_helper.c |   5 +
 3 files changed, 161 insertions(+)
Signature · 1 line
2.37.0 (Apple Git-136)
#1 lore raw

PATCH 1/3 x86: Implement MSR_CORE_THREAD_COUNT MSR

Intel CPUs starting with Haswell-E implement a new MSR called MSR_CORE_THREAD_COUNT which exposes the number of threads and cores inside of a package.

This MSR is used by XNU to populate internal data structures and not implementing it prevents virtual machines with more than 1 vCPU from booting if the emulated CPU generation is at least Haswell-E.

This patch propagates the existing hvf logic from patch 027ac0cb516 ("target/i386/hvf: add rdmsr 35H MSR_CORE_THREAD_COUNT") to TCG.

---
 target/i386/tcg/sysemu/misc_helper.c | 5 +++++
 1 file changed, 5 insertions(+)
target/i386/tcg/sysemu/misc_helper.c +5 -0
index 1328aa656f..e1528b7f80 100644
@@ -450,6 +450,11 @@ void helper_rdmsr(CPUX86State *env)
      case MSR_IA32_UCODE_REV:
         val = x86_cpu->ucode_rev;
         break;
+    case MSR_CORE_THREAD_COUNT: {
+        CPUState *cs = CPU(x86_cpu);
+        val = (cs->nr_threads * cs->nr_cores) | (cs->nr_cores << 16);
+        break;
+    }
     default:
         if ((uint32_t)env->regs[R_ECX] >= MSR_MC0_CTL
             && (uint32_t)env->regs[R_ECX] < MSR_MC0_CTL +
Signature · 1 line
2.37.0 (Apple Git-136)
#2 lore raw

PATCH 2/3 i386: kvm: Add support for MSR filtering

In reply to Alexander Graf

KVM has grown support to deflect arbitrary MSRs to user space since Linux 5.10. For now we don't expect to make a lot of use of this feature, so let's expose it the easiest way possible: With up to 16 individually maskable MSRs.

This patch adds a kvm_filter_msr() function that other code can call to install a hook on KVM MSR reads or writes.

---
 target/i386/kvm/kvm.c      | 124 +++++++++++++++++++++++++++++++++++++
 target/i386/kvm/kvm_i386.h |  11 ++++
 2 files changed, 135 insertions(+)
target/i386/kvm/kvm.c +124 -0
index a1fd1f5379..ea53092dd0 100644
@@ -139,6 +139,8 @@ static struct kvm_cpuid2 *cpuid_cache;
 static struct kvm_cpuid2 *hv_cpuid_cache;
 static struct kvm_msr_list *kvm_feature_msrs;
 
+static KVMMSRHandlers msr_handlers[KVM_MSR_FILTER_MAX_RANGES];
+
 #define BUS_LOCK_SLICE_TIME 1000000000ULL /* ns */
 static RateLimit bus_lock_ratelimit_ctrl;
 static int kvm_get_one_msr(X86CPU *cpu, int index, uint64_t *value);
@@ -2588,6 +2590,16 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
         }
     }
 
+    if (kvm_vm_check_extension(s, KVM_CAP_X86_USER_SPACE_MSR)) {
+        ret = kvm_vm_enable_cap(s, KVM_CAP_X86_USER_SPACE_MSR, 0,
+                                KVM_MSR_EXIT_REASON_FILTER);
+        if (ret) {
+            error_report("Could not enable user space MSRs: %s",
+                         strerror(-ret));
+            exit(1);
+        }
+    }
+
     return 0;
 }
 
@@ -5077,6 +5089,108 @@ void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg)
     }
 }
 
+static bool kvm_install_msr_filters(KVMState *s)
+{
+    uint64_t zero = 0;
+    struct kvm_msr_filter filter = {
+        .flags = KVM_MSR_FILTER_DEFAULT_ALLOW,
+    };
+    int r, i, j = 0;
+
+    for (i = 0; i < KVM_MSR_FILTER_MAX_RANGES; i++) {
+        KVMMSRHandlers *handler = &msr_handlers[i];
+        if (handler->msr) {
+            struct kvm_msr_filter_range *range = &filter.ranges[j++];
+
+            *range = (struct kvm_msr_filter_range) {
+                .flags = 0,
+                .nmsrs = 1,
+                .base = handler->msr,
+                .bitmap = (__u8 *)&zero,
+            };
+
+            if (handler->rdmsr) {
+                range->flags |= KVM_MSR_FILTER_READ;
+            }
+
+            if (handler->wrmsr) {
+                range->flags |= KVM_MSR_FILTER_WRITE;
+            }
+        }
+    }
+
+    r = kvm_vm_ioctl(s, KVM_X86_SET_MSR_FILTER, &filter);
+    if (r) {
+        return false;
+    }
+
+    return true;
+}
+
+bool kvm_filter_msr(KVMState *s, uint32_t msr, QEMURDMSRHandler *rdmsr,
+                    QEMUWRMSRHandler *wrmsr)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) {
+        if (!msr_handlers[i].msr) {
+            msr_handlers[i] = (KVMMSRHandlers) {
+                .msr = msr,
+                .rdmsr = rdmsr,
+                .wrmsr = wrmsr,
+            };
+
+            if (!kvm_install_msr_filters(s)) {
+                msr_handlers[i] = (KVMMSRHandlers) { };
+                return false;
+            }
+
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static int kvm_handle_rdmsr(X86CPU *cpu, struct kvm_run *run)
+{
+    int i;
+    bool r;
+
+    for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) {
+        KVMMSRHandlers *handler = &msr_handlers[i];
+        if (run->msr.index == handler->msr) {
+            if (handler->rdmsr) {
+                r = handler->rdmsr(cpu, handler->msr,
+                                   (uint64_t *)&run->msr.data);
+                run->msr.error = r ? 0 : 1;
+                return 0;
+            }
+        }
+    }
+
+    assert(false);
+}
+
+static int kvm_handle_wrmsr(X86CPU *cpu, struct kvm_run *run)
+{
+    int i;
+    bool r;
+
+    for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) {
+        KVMMSRHandlers *handler = &msr_handlers[i];
+        if (run->msr.index == handler->msr) {
+            if (handler->wrmsr) {
+                r = handler->wrmsr(cpu, handler->msr, run->msr.data);
+                run->msr.error = r ? 0 : 1;
+                return 0;
+            }
+        }
+    }
+
+    assert(false);
+}
+
 static bool has_sgx_provisioning;
 
 static bool __kvm_enable_sgx_provisioning(KVMState *s)
@@ -5176,6 +5290,16 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
         /* already handled in kvm_arch_post_run */
         ret = 0;
         break;
+    case KVM_EXIT_X86_RDMSR:
+        /* We only enable MSR filtering, any other exit is bogus */
+        assert(run->msr.reason == KVM_MSR_EXIT_REASON_FILTER);
+        ret = kvm_handle_rdmsr(cpu, run);
+        break;
+    case KVM_EXIT_X86_WRMSR:
+        /* We only enable MSR filtering, any other exit is bogus */
+        assert(run->msr.reason == KVM_MSR_EXIT_REASON_FILTER);
+        ret = kvm_handle_wrmsr(cpu, run);
+        break;
     default:
         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
         ret = -1;
target/i386/kvm/kvm_i386.h +11 -0
index 4124912c20..2ed586c11b 100644
@@ -54,4 +54,15 @@ uint64_t kvm_swizzle_msi_ext_dest_id(uint64_t address);
 bool kvm_enable_sgx_provisioning(KVMState *s);
 void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask);
 
+typedef bool QEMURDMSRHandler(X86CPU *cpu, uint32_t msr, uint64_t *val);
+typedef bool QEMUWRMSRHandler(X86CPU *cpu, uint32_t msr, uint64_t val);
+typedef struct kvm_msr_handlers {
+    uint32_t msr;
+    QEMURDMSRHandler *rdmsr;
+    QEMUWRMSRHandler *wrmsr;
+} KVMMSRHandlers;
+
+bool kvm_filter_msr(KVMState *s, uint32_t msr, QEMURDMSRHandler *rdmsr,
+                    QEMUWRMSRHandler *wrmsr);
+
 #endif
Signature · 1 line
2.37.0 (Apple Git-136)
#3 lore raw

PATCH 3/3 KVM: x86: Implement MSR_CORE_THREAD_COUNT MSR

In reply to Alexander Graf

The MSR_CORE_THREAD_COUNT MSR describes CPU package topology, such as number of threads and cores for a given package. This is information that QEMU has readily available and can provide through the new user space MSR deflection interface.

This patch propagates the existing hvf logic from patch 027ac0cb516 ("target/i386/hvf: add rdmsr 35H MSR_CORE_THREAD_COUNT") to KVM.

---
 target/i386/kvm/kvm.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
target/i386/kvm/kvm.c +21 -0
index ea53092dd0..791e995389 100644
@@ -2403,6 +2403,17 @@ static int kvm_get_supported_msrs(KVMState *s)
     return ret;
 }
 
+static bool kvm_rdmsr_core_thread_count(X86CPU *cpu, uint32_t msr,
+                                        uint64_t *val)
+{
+    CPUState *cs = CPU(cpu);
+
+    *val = cs->nr_threads * cs->nr_cores; /* thread count, bits 15..0 */
+    *val |= ((uint32_t)cs->nr_cores << 16); /* core count, bits 31..16 */
+
+    return true;
+}
+
 static Notifier smram_machine_done;
 static KVMMemoryListener smram_listener;
 static AddressSpace smram_address_space;
@@ -2591,6 +2602,8 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     }
 
     if (kvm_vm_check_extension(s, KVM_CAP_X86_USER_SPACE_MSR)) {
+        bool r;
+
         ret = kvm_vm_enable_cap(s, KVM_CAP_X86_USER_SPACE_MSR, 0,
                                 KVM_MSR_EXIT_REASON_FILTER);
         if (ret) {
@@ -2598,6 +2611,14 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
                          strerror(-ret));
             exit(1);
         }
+
+        r = kvm_filter_msr(s, MSR_CORE_THREAD_COUNT,
+                           kvm_rdmsr_core_thread_count, NULL);
+        if (!r) {
+            error_report("Could not install MSR_CORE_THREAD_COUNT handler: %s",
+                         strerror(-ret));
+            exit(1);
+        }
     }
 
     return 0;
Signature · 1 line
2.37.0 (Apple Git-136)
maintainer Applied
#4 lore raw
In reply to Alexander Graf
Quoted 21 lines · Alexander Graf

Commit 027ac0cb516 ("target/i386/hvf: add rdmsr 35H MSR_CORE_THREAD_COUNT") added support for the MSR_CORE_THREAD_COUNT MSR to HVF. This MSR is mandatory to execute macOS when run with -cpu host,+hypervisor.

This patch set adds support for the very same MSR to TCG as well as KVM - as long as host KVM is recent enough to support MSR trapping.

With this support added, I can successfully execute macOS guests in KVM with an APFS enabled OVMF build, a valid applesmc plus OSK and

-cpu Skylake-Client,+invtsc,+hypervisor

Alex

Alexander Graf (3):
x86: Implement MSR_CORE_THREAD_COUNT MSR
i386: kvm: Add support for MSR filtering
KVM: x86: Implement MSR_CORE_THREAD_COUNT MSR

Quoted 5 lines
  target/i386/kvm/kvm.c                | 145 +++++++++++++++++++++++++++
  target/i386/kvm/kvm_i386.h           |  11 ++
  target/i386/tcg/sysemu/misc_helper.c |   5 +
  3 files changed, 161 insertions(+)

Queued, thanks!

Paolo