Skip to contentSkip to conversation

Improve instruction and CSR emulation in KVM RISC-V

PATCH In review Anup Patel 2 participants 5 messages Jun 15, 2022 – Jul 4, 2022

Subsystems (3)
lore
Share & tools
lore

Anup Patel

author
#0 lore raw

v2 cover Improve instruction and CSR emulation in KVM RISC-V

Currently, the instruction emulation for MMIO traps and Virtual instruction traps co-exist with general VCPU exit handling. The instruction and CSR emulation will grow with upcoming SBI PMU, AIA, and Nested virtualization in KVM RISC-V. In addition, we also need a mechanism to allow user-space emulate certain CSRs under certain situation (example, host has AIA support but user-space does not wants to use in-kernel AIA IMSIC and APLIC support).

This series improves instruction and CSR emulation in KVM RISC-V to make it extensible based on above.

These patches can also be found in riscv_kvm_csr_v2 branch at:
https://github.com/avpatel/linux.git

Changes since v1:
- Added a switch-case in PATCH3 to process MMIO, CSR, and SBI returned from user-space
- Removed hard-coding in PATCH3 for determining type of CSR instruction

Anup Patel (3):
RISC-V: KVM: Factor-out instruction emulation into separate sources
RISC-V: KVM: Add extensible system instruction emulation framework
RISC-V: KVM: Add extensible CSR emulation framework

 arch/riscv/include/asm/kvm_host.h           |  16 +-
 arch/riscv/include/asm/kvm_vcpu_insn.h      |  48 ++
 arch/riscv/kvm/Makefile                     |   1 +
 arch/riscv/kvm/vcpu.c                       |  34 +-
 arch/riscv/kvm/vcpu_exit.c                  | 490 +----------------
 arch/riscv/kvm/{vcpu_exit.c => vcpu_insn.c} | 563 +++++++++++---------
 include/uapi/linux/kvm.h                    |   8 +
 7 files changed, 392 insertions(+), 768 deletions(-)
 create mode 100644 arch/riscv/include/asm/kvm_vcpu_insn.h
 copy arch/riscv/kvm/{vcpu_exit.c => vcpu_insn.c} (63%)
Signature · 1 line
2.34.1
author
#1 lore raw

PATCH v2 1/3 RISC-V: KVM: Factor-out instruction emulation into separate sources

The instruction and CSR emulation for VCPU is going to grow over time due to upcoming AIA, PMU, Nested and other virtualization features.

Let us factor-out VCPU instruction emulation from vcpu_exit.c to a separate source dedicated for this purpose.

---
 arch/riscv/include/asm/kvm_host.h           |  11 +-
 arch/riscv/include/asm/kvm_vcpu_insn.h      |  33 ++
 arch/riscv/kvm/Makefile                     |   1 +
 arch/riscv/kvm/vcpu_exit.c                  | 490 +-------------------
 arch/riscv/kvm/{vcpu_exit.c => vcpu_insn.c} | 309 +++---------
 5 files changed, 101 insertions(+), 743 deletions(-)
 create mode 100644 arch/riscv/include/asm/kvm_vcpu_insn.h
 copy arch/riscv/kvm/{vcpu_exit.c => vcpu_insn.c} (67%)
arch/riscv/include/asm/kvm_host.h +1 -10
index 319c8aeb42af..03103b86dd86 100644
@@ -15,6 +15,7 @@
 #include <linux/spinlock.h>
 #include <asm/csr.h>
 #include <asm/kvm_vcpu_fp.h>
+#include <asm/kvm_vcpu_insn.h>
 #include <asm/kvm_vcpu_timer.h>
 
 #define KVM_MAX_VCPUS			1024
@@ -90,14 +91,6 @@ struct kvm_arch {
 	struct kvm_guest_timer timer;
 };
 
-struct kvm_mmio_decode {
-	unsigned long insn;
-	int insn_len;
-	int len;
-	int shift;
-	int return_handled;
-};
-
 struct kvm_sbi_context {
 	int return_handled;
 };
@@ -303,14 +296,12 @@ void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu);
 
 void __kvm_riscv_unpriv_trap(void);
 
-void kvm_riscv_vcpu_wfi(struct kvm_vcpu *vcpu);
 unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu,
 					 bool read_insn,
 					 unsigned long guest_addr,
 					 struct kvm_cpu_trap *trap);
 void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu,
 				  struct kvm_cpu_trap *trap);
-int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run);
 int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
 			struct kvm_cpu_trap *trap);
 
arch/riscv/include/asm/kvm_vcpu_insn.h new file +33 -0
new file mode 100644
index 000000000000..4e3ba4e84d0f
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022 Ventana Micro Systems Inc.
+ */
+
+#ifndef __KVM_VCPU_RISCV_INSN_H
+#define __KVM_VCPU_RISCV_INSN_H
+
+struct kvm_vcpu;
+struct kvm_run;
+struct kvm_cpu_trap;
+
+struct kvm_mmio_decode {
+	unsigned long insn;
+	int insn_len;
+	int len;
+	int shift;
+	int return_handled;
+};
+
+void kvm_riscv_vcpu_wfi(struct kvm_vcpu *vcpu);
+int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
+				struct kvm_cpu_trap *trap);
+
+int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run,
+			     unsigned long fault_addr,
+			     unsigned long htinst);
+int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run,
+			      unsigned long fault_addr,
+			      unsigned long htinst);
+int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run);
+
+#endif
arch/riscv/kvm/Makefile +1 -0
index e5c56182f48f..019df9208bdd 100644
@@ -17,6 +17,7 @@ kvm-y += mmu.o
 kvm-y += vcpu.o
 kvm-y += vcpu_exit.o
 kvm-y += vcpu_fp.o
+kvm-y += vcpu_insn.o
 kvm-y += vcpu_switch.o
 kvm-y += vcpu_sbi.o
 kvm-$(CONFIG_RISCV_SBI_V01) += vcpu_sbi_v01.o
arch/riscv/kvm/vcpu_exit.c +7 -483

524 lines not on this page · raw message

arch/riscv/kvm/vcpu_insn.c renamed from arch/riscv/kvm/vcpu_exit.c +59 -250

352 lines not on this page · raw message

Signature · 1 line
2.34.1
author
#2 lore raw

PATCH v2 2/3 RISC-V: KVM: Add extensible system instruction emulation framework

In reply to Anup Patel

We will be emulating more system instructions in near future with upcoming AIA, PMU, Nested and other virtualization features.

To accommodate above, we add an extensible system instruction emulation framework in vcpu_insn.c.

---
 arch/riscv/include/asm/kvm_vcpu_insn.h |  9 +++
 arch/riscv/kvm/vcpu_insn.c             | 82 +++++++++++++++++++++++---
 2 files changed, 82 insertions(+), 9 deletions(-)
arch/riscv/include/asm/kvm_vcpu_insn.h +9 -0
index 4e3ba4e84d0f..3351eb61a251 100644
@@ -18,6 +18,15 @@ struct kvm_mmio_decode {
 	int return_handled;
 };
 
+/* Return values used by function emulating a particular instruction */
+enum kvm_insn_return {
+	KVM_INSN_EXIT_TO_USER_SPACE = 0,
+	KVM_INSN_CONTINUE_NEXT_SEPC,
+	KVM_INSN_CONTINUE_SAME_SEPC,
+	KVM_INSN_ILLEGAL_TRAP,
+	KVM_INSN_VIRTUAL_TRAP
+};
+
 void kvm_riscv_vcpu_wfi(struct kvm_vcpu *vcpu);
 int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
 				struct kvm_cpu_trap *trap);
arch/riscv/kvm/vcpu_insn.c +73 -9

105 lines not on this page · raw message

Signature · 1 line
2.34.1
author
#3 lore raw

PATCH v2 3/3 RISC-V: KVM: Add extensible CSR emulation framework

In reply to Anup Patel

We add an extensible CSR emulation framework which is based upon the existing system instruction emulation. This will be useful to upcoming AIA, PMU, Nested and other virtualization features.

The CSR emulation framework also has provision to emulate CSR in user space but this will be used only in very specific cases such as AIA IMSIC CSR emulation in user space or vendor specific CSR emulation in user space.

By default, all CSRs not handled by KVM RISC-V will be redirected back to Guest VCPU as illegal instruction trap.

---
 arch/riscv/include/asm/kvm_host.h      |   5 +
 arch/riscv/include/asm/kvm_vcpu_insn.h |   6 +
 arch/riscv/kvm/vcpu.c                  |  34 +++--
 arch/riscv/kvm/vcpu_insn.c             | 172 ++++++++++++++++++++++++-
 include/uapi/linux/kvm.h               |   8 ++
 5 files changed, 209 insertions(+), 16 deletions(-)
arch/riscv/include/asm/kvm_host.h +5 -0
index 03103b86dd86..a54744d7e1ba 100644
@@ -64,6 +64,8 @@ struct kvm_vcpu_stat {
 	u64 wfi_exit_stat;
 	u64 mmio_exit_user;
 	u64 mmio_exit_kernel;
+	u64 csr_exit_user;
+	u64 csr_exit_kernel;
 	u64 exits;
 };
 
@@ -209,6 +211,9 @@ struct kvm_vcpu_arch {
 	/* MMIO instruction details */
 	struct kvm_mmio_decode mmio_decode;
 
+	/* CSR instruction details */
+	struct kvm_csr_decode csr_decode;
+
 	/* SBI context */
 	struct kvm_sbi_context sbi_context;
 
arch/riscv/include/asm/kvm_vcpu_insn.h +6 -0
index 3351eb61a251..350011c83581 100644
@@ -18,6 +18,11 @@ struct kvm_mmio_decode {
 	int return_handled;
 };
 
+struct kvm_csr_decode {
+	unsigned long insn;
+	int return_handled;
+};
+
 /* Return values used by function emulating a particular instruction */
 enum kvm_insn_return {
 	KVM_INSN_EXIT_TO_USER_SPACE = 0,
@@ -28,6 +33,7 @@ enum kvm_insn_return {
 };
 
 void kvm_riscv_vcpu_wfi(struct kvm_vcpu *vcpu);
+int kvm_riscv_vcpu_csr_return(struct kvm_vcpu *vcpu, struct kvm_run *run);
 int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
 				struct kvm_cpu_trap *trap);
 
arch/riscv/kvm/vcpu.c +20 -14
index 7f4ad5e4373a..21bbb9e83982 100644
@@ -26,6 +26,8 @@ const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
 	STATS_DESC_COUNTER(VCPU, wfi_exit_stat),
 	STATS_DESC_COUNTER(VCPU, mmio_exit_user),
 	STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
+	STATS_DESC_COUNTER(VCPU, csr_exit_user),
+	STATS_DESC_COUNTER(VCPU, csr_exit_kernel),
 	STATS_DESC_COUNTER(VCPU, exits)
 };
 
@@ -851,22 +853,26 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 
 	kvm_vcpu_srcu_read_lock(vcpu);
 
-	/* Process MMIO value returned from user-space */
-	if (run->exit_reason == KVM_EXIT_MMIO) {
+	switch (run->exit_reason) {
+	case KVM_EXIT_MMIO:
+		/* Process MMIO value returned from user-space */
 		ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run);
-		if (ret) {
-			kvm_vcpu_srcu_read_unlock(vcpu);
-			return ret;
-		}
-	}
-
-	/* Process SBI value returned from user-space */
-	if (run->exit_reason == KVM_EXIT_RISCV_SBI) {
+		break;
+	case KVM_EXIT_RISCV_SBI:
+		/* Process SBI value returned from user-space */
 		ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run);
-		if (ret) {
-			kvm_vcpu_srcu_read_unlock(vcpu);
-			return ret;
-		}
+		break;
+	case KVM_EXIT_RISCV_CSR:
+		/* Process CSR value returned from user-space */
+		ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run);
+		break;
+	default:
+		ret = 0;
+		break;
+	}
+	if (ret) {
+		kvm_vcpu_srcu_read_unlock(vcpu);
+		return ret;
 	}
 
 	if (run->immediate_exit) {
arch/riscv/kvm/vcpu_insn.c +170 -2

203 lines not on this page · raw message

include/uapi/linux/kvm.h +8 -0
index 5088bd9f1922..c48fd3d1c45b 100644
@@ -270,6 +270,7 @@ struct kvm_xen_exit {
 #define KVM_EXIT_X86_BUS_LOCK     33
 #define KVM_EXIT_XEN              34
 #define KVM_EXIT_RISCV_SBI        35
+#define KVM_EXIT_RISCV_CSR        36
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -496,6 +497,13 @@ struct kvm_run {
 			unsigned long args[6];
 			unsigned long ret[2];
 		} riscv_sbi;
+		/* KVM_EXIT_RISCV_CSR */
+		struct {
+			unsigned long csr_num;
+			unsigned long new_value;
+			unsigned long write_mask;
+			unsigned long ret_value;
+		} riscv_csr;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
Signature · 1 line
2.34.1
maintainer
#4 lore raw
In reply to Anup Patel
Quoted 19 lines · Anup Patel

Currently, the instruction emulation for MMIO traps and Virtual instruction traps co-exist with general VCPU exit handling. The instruction and CSR emulation will grow with upcoming SBI PMU, AIA, and Nested virtualization in KVM RISC-V. In addition, we also need a mechanism to allow user-space emulate certain CSRs under certain situation (example, host has AIA support but user-space does not wants to use in-kernel AIA IMSIC and APLIC support).

This series improves instruction and CSR emulation in KVM RISC-V to make it extensible based on above.

These patches can also be found in riscv_kvm_csr_v2 branch at:
https://github.com/avpatel/linux.git

Changes since v1:
- Added a switch-case in PATCH3 to process MMIO, CSR, and SBI returned from user-space
- Removed hard-coding in PATCH3 for determining type of CSR instruction

Quoted 4 lines

Anup Patel (3):
RISC-V: KVM: Factor-out instruction emulation into separate sources
RISC-V: KVM: Add extensible system instruction emulation framework
RISC-V: KVM: Add extensible CSR emulation framework

I have queued this series for 5.20

Thanks,
Anup

Quoted 15 lines
 arch/riscv/include/asm/kvm_host.h           |  16 +-
 arch/riscv/include/asm/kvm_vcpu_insn.h      |  48 ++
 arch/riscv/kvm/Makefile                     |   1 +
 arch/riscv/kvm/vcpu.c                       |  34 +-
 arch/riscv/kvm/vcpu_exit.c                  | 490 +----------------
 arch/riscv/kvm/{vcpu_exit.c => vcpu_insn.c} | 563 +++++++++++---------
 include/uapi/linux/kvm.h                    |   8 +
 7 files changed, 392 insertions(+), 768 deletions(-)
 create mode 100644 arch/riscv/include/asm/kvm_vcpu_insn.h
 copy arch/riscv/kvm/{vcpu_exit.c => vcpu_insn.c} (63%)

--
2.34.1