kvm-unit-tests: Add a func to run instruction in emulator

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Signed-off-by Arthur Chunqi Li <[email protected]>

---
 x86/emulator.c |   67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 mode change 100644 => 100755 x86/emulator.c

x86/emulator.c +67 -0

old mode 100644
new mode 100755
index 96576e5..770e8f7
@@ -11,6 +11,13 @@ int fails, tests;
 
 static int exceptions;
 
+struct regs {
+	u64 rax, rbx, rcx, rdx;
+	u64 rsi, rdi, rsp, rbp;
+	u64 rip, rflags;
+};
+static struct regs inregs, outregs;
+
 void report(const char *name, int result)
 {
 	++tests;

@@ -685,6 +692,66 @@ static void test_shld_shrd(u32 *mem)
     report("shrd (cl)", *mem == ((0x12345678 >> 3) | (5u << 29)));
 }
 
+static void trap_emulator(uint64_t *mem, uint8_t *insn_page,
+			     uint8_t *alt_insn_page, void *insn_ram,
+			     uint8_t *alt_insn, int alt_insn_length)
+{
+	ulong *cr3 = (ulong *)read_cr3();
+	int i;
+	static struct regs save;
+
+	// Pad with RET instructions
+	memset(insn_page, 0xc3, 4096);
+	memset(alt_insn_page, 0xc3, 4096);
+
+	// Place a trapping instruction in the page to trigger a VMEXIT
+	insn_page[0] = 0x89; // mov %eax, (%rax)
+	insn_page[1] = 0x00;
+	insn_page[2] = 0x90; // nop
+	insn_page[3] = 0xc3; // ret
+
+	// Place the instruction we want the hypervisor to see in the alternate page
+	for (i=0; i<alt_insn_length; i++)
+		alt_insn_page[i] = alt_insn[i];
+	save = inregs;
+	
+	// Load the code TLB with insn_page, but point the page tables at
+	// alt_insn_page (and keep the data TLB clear, for AMD decode assist).
+	// This will make the CPU trap on the insn_page instruction but the
+	// hypervisor will see alt_insn_page.
+	install_page(cr3, virt_to_phys(insn_page), insn_ram);
+	invlpg(insn_ram);
+	// Load code TLB
+	asm volatile("call *%0" : : "r"(insn_ram + 3));
+	install_page(cr3, virt_to_phys(alt_insn_page), insn_ram);
+	// Trap, let hypervisor emulate at alt_insn_page
+	asm volatile(
+		"xchg %%rbx, 8+%[save] \n\t"
+		"xchg %%rcx, 16+%[save] \n\t"
+		"xchg %%rdx, 24+%[save] \n\t"
+		"xchg %%rsi, 32+%[save] \n\t"
+		"xchg %%rdi, 40+%[save] \n\t"
+
+		"call *%1\n\t"
+
+		"mov %%rax, 0+%[save] \n\t"
+		"xchg %%rbx, 8+%[save] \n\t"
+		"xchg %%rcx, 16+%[save] \n\t"
+		"xchg %%rdx, 24+%[save] \n\t"
+		"xchg %%rsi, 32+%[save] \n\t"
+		"xchg %%rdi, 40+%[save] \n\t"
+		"mov %%rsp, 48+%[save] \n\t"
+		"mov %%rbp, 56+%[save] \n\t"
+		/* Save RFLAGS in outregs*/
+		"pushf \n\t"
+		"popq 72+%[save] \n\t"
+		: [save]"+m"(save)
+		: "r"(insn_ram), "a"(mem)
+		: "memory", "cc"
+		);
+	outregs = save;
+}
+
 static void advance_rip_by_3_and_note_exception(struct ex_regs *regs)
 {
     ++exceptions;

Signature · 1 line

1.7.9.5

Quoted 5 lines · Arthur Chunqi Li

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

%rsp and %rbp because of ret on instruction page? Use the same trick realmode.c test uses: have the code that sets/saves registers in insn_page/alt_insn_page itself and copy the instruction you want to test into the page itself instead of doing call.

Not sure what is so special about %rflags.

Quoted 90 lines

Signed-off-by Arthur Chunqi Li <[email protected]>

---
 x86/emulator.c |   67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 mode change 100644 => 100755 x86/emulator.c

x86/emulator.c +67 -0

old mode 100644
new mode 100755
index 96576e5..770e8f7
@@ -11,6 +11,13 @@ int fails, tests;
 
 static int exceptions;
 
+struct regs {
+     u64 rax, rbx, rcx, rdx;
+     u64 rsi, rdi, rsp, rbp;
+     u64 rip, rflags;
+};
+static struct regs inregs, outregs;
+
 void report(const char *name, int result)
 {
      ++tests;

@@ -685,6 +692,66 @@ static void test_shld_shrd(u32 *mem)
     report("shrd (cl)", *mem == ((0x12345678 >> 3) | (5u << 29)));
 }
 
+static void trap_emulator(uint64_t *mem, uint8_t *insn_page,
+                          uint8_t *alt_insn_page, void *insn_ram,
+                          uint8_t *alt_insn, int alt_insn_length)
+{
+     ulong *cr3 = (ulong *)read_cr3();
+     int i;
+     static struct regs save;
+
+     // Pad with RET instructions
+     memset(insn_page, 0xc3, 4096);
+     memset(alt_insn_page, 0xc3, 4096);
+
+     // Place a trapping instruction in the page to trigger a VMEXIT
+     insn_page[0] = 0x89; // mov %eax, (%rax)
+     insn_page[1] = 0x00;
+     insn_page[2] = 0x90; // nop
+     insn_page[3] = 0xc3; // ret
+
+     // Place the instruction we want the hypervisor to see in the alternate page
+     for (i=0; i<alt_insn_length; i++)
+             alt_insn_page[i] = alt_insn[i];
+     save = inregs;
+
+     // Load the code TLB with insn_page, but point the page tables at
+     // alt_insn_page (and keep the data TLB clear, for AMD decode assist).
+     // This will make the CPU trap on the insn_page instruction but the
+     // hypervisor will see alt_insn_page.
+     install_page(cr3, virt_to_phys(insn_page), insn_ram);
+     invlpg(insn_ram);
+     // Load code TLB
+     asm volatile("call *%0" : : "r"(insn_ram + 3));
+     install_page(cr3, virt_to_phys(alt_insn_page), insn_ram);
+     // Trap, let hypervisor emulate at alt_insn_page
+     asm volatile(
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+
+             "call *%1\n\t"
+
+             "mov %%rax, 0+%[save] \n\t"
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+             "mov %%rsp, 48+%[save] \n\t"
+             "mov %%rbp, 56+%[save] \n\t"
+             /* Save RFLAGS in outregs*/
+             "pushf \n\t"
+             "popq 72+%[save] \n\t"
+             : [save]"+m"(save)
+             : "r"(insn_ram), "a"(mem)
+             : "memory", "cc"
+             );
+     outregs = save;
+}
+

Quoted 5 lines

x86/emulator.c +0 -1

 static void advance_rip_by_3_and_note_exception(struct ex_regs *regs)
 {
     ++exceptions;
--

1.7.9.5

--
Gleb.

Quoted 6 lines · Gleb Natapov

Quoted 5 lines · Arthur Chunqi Li

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Quoted 4 lines

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

I don't know why set %rax before call alt_insn_page can cause error. I use "xchg %%rax, 0+%[save]" before "call *%1" and the %rcx is not set correctly.

Quoted 5 lines

%rsp and %rbp because of ret on instruction page? Use the same trick realmode.c test uses: have the code that sets/saves registers in insn_page/alt_insn_page itself and copy the instruction you want to test into the page itself instead of doing call.

I don't know how instructions between calling insn_page and alt_insn_page are executed (function install_page and some other instructions before call *%1". If these insns are executed after insn_page is called, changes before the trapping instruction may affect the executing of these instructions.

Quoted 93 lines

Not sure what is so special about %rflags.

Quoted 90 lines

Signed-off-by Arthur Chunqi Li <[email protected]>

---
 x86/emulator.c |   67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 mode change 100644 => 100755 x86/emulator.c

x86/emulator.c +67 -0

old mode 100644
new mode 100755
index 96576e5..770e8f7
@@ -11,6 +11,13 @@ int fails, tests;
 
 static int exceptions;
 
+struct regs {
+     u64 rax, rbx, rcx, rdx;
+     u64 rsi, rdi, rsp, rbp;
+     u64 rip, rflags;
+};
+static struct regs inregs, outregs;
+
 void report(const char *name, int result)
 {
      ++tests;

@@ -685,6 +692,66 @@ static void test_shld_shrd(u32 *mem)
     report("shrd (cl)", *mem == ((0x12345678 >> 3) | (5u << 29)));
 }
 
+static void trap_emulator(uint64_t *mem, uint8_t *insn_page,
+                          uint8_t *alt_insn_page, void *insn_ram,
+                          uint8_t *alt_insn, int alt_insn_length)
+{
+     ulong *cr3 = (ulong *)read_cr3();
+     int i;
+     static struct regs save;
+
+     // Pad with RET instructions
+     memset(insn_page, 0xc3, 4096);
+     memset(alt_insn_page, 0xc3, 4096);
+
+     // Place a trapping instruction in the page to trigger a VMEXIT
+     insn_page[0] = 0x89; // mov %eax, (%rax)
+     insn_page[1] = 0x00;
+     insn_page[2] = 0x90; // nop
+     insn_page[3] = 0xc3; // ret
+
+     // Place the instruction we want the hypervisor to see in the alternate page
+     for (i=0; i<alt_insn_length; i++)
+             alt_insn_page[i] = alt_insn[i];
+     save = inregs;
+
+     // Load the code TLB with insn_page, but point the page tables at
+     // alt_insn_page (and keep the data TLB clear, for AMD decode assist).
+     // This will make the CPU trap on the insn_page instruction but the
+     // hypervisor will see alt_insn_page.
+     install_page(cr3, virt_to_phys(insn_page), insn_ram);
+     invlpg(insn_ram);
+     // Load code TLB
+     asm volatile("call *%0" : : "r"(insn_ram + 3));
+     install_page(cr3, virt_to_phys(alt_insn_page), insn_ram);
+     // Trap, let hypervisor emulate at alt_insn_page
+     asm volatile(
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+
+             "call *%1\n\t"
+
+             "mov %%rax, 0+%[save] \n\t"
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+             "mov %%rsp, 48+%[save] \n\t"
+             "mov %%rbp, 56+%[save] \n\t"
+             /* Save RFLAGS in outregs*/
+             "pushf \n\t"
+             "popq 72+%[save] \n\t"
+             : [save]"+m"(save)
+             : "r"(insn_ram), "a"(mem)
+             : "memory", "cc"
+             );
+     outregs = save;
+}
+

Quoted 8 lines

Quoted 5 lines

x86/emulator.c +0 -1

 static void advance_rip_by_3_and_note_exception(struct ex_regs *regs)
 {
     ++exceptions;
--

1.7.9.5

--
Gleb.

--
Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

Quoted 7 lines · 李春奇

Quoted 6 lines · Gleb Natapov

Quoted 5 lines · Arthur Chunqi Li

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Quoted 7 lines

Quoted 4 lines

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

I don't know why set %rax before call alt_insn_page can cause error. I use "xchg %%rax, 0+%[save]" before "call *%1" and the %rcx is not set correctly.

We better find this out :)

Quoted 6 lines

Quoted 6 lines

%rsp and %rbp because of ret on instruction page? Use the same trick realmode.c test uses: have the code that sets/saves registers in insn_page/alt_insn_page itself and copy the instruction you want to test into the page itself instead of doing call.

Quoted 6 lines

I don't know how instructions between calling insn_page and alt_insn_page are executed (function install_page and some other instructions before call *%1". If these insns are executed after insn_page is called, changes before the trapping instruction may affect the executing of these instructions.

Not sure what do you mean here.

Quoted 105 lines

Quoted 94 lines

Not sure what is so special about %rflags.

Quoted 90 lines

Signed-off-by Arthur Chunqi Li <[email protected]>

---
 x86/emulator.c |   67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 mode change 100644 => 100755 x86/emulator.c

x86/emulator.c +67 -0

old mode 100644
new mode 100755
index 96576e5..770e8f7
@@ -11,6 +11,13 @@ int fails, tests;
 
 static int exceptions;
 
+struct regs {
+     u64 rax, rbx, rcx, rdx;
+     u64 rsi, rdi, rsp, rbp;
+     u64 rip, rflags;
+};
+static struct regs inregs, outregs;
+
 void report(const char *name, int result)
 {
      ++tests;

@@ -685,6 +692,66 @@ static void test_shld_shrd(u32 *mem)
     report("shrd (cl)", *mem == ((0x12345678 >> 3) | (5u << 29)));
 }
 
+static void trap_emulator(uint64_t *mem, uint8_t *insn_page,
+                          uint8_t *alt_insn_page, void *insn_ram,
+                          uint8_t *alt_insn, int alt_insn_length)
+{
+     ulong *cr3 = (ulong *)read_cr3();
+     int i;
+     static struct regs save;
+
+     // Pad with RET instructions
+     memset(insn_page, 0xc3, 4096);
+     memset(alt_insn_page, 0xc3, 4096);
+
+     // Place a trapping instruction in the page to trigger a VMEXIT
+     insn_page[0] = 0x89; // mov %eax, (%rax)
+     insn_page[1] = 0x00;
+     insn_page[2] = 0x90; // nop
+     insn_page[3] = 0xc3; // ret
+
+     // Place the instruction we want the hypervisor to see in the alternate page
+     for (i=0; i<alt_insn_length; i++)
+             alt_insn_page[i] = alt_insn[i];
+     save = inregs;
+
+     // Load the code TLB with insn_page, but point the page tables at
+     // alt_insn_page (and keep the data TLB clear, for AMD decode assist).
+     // This will make the CPU trap on the insn_page instruction but the
+     // hypervisor will see alt_insn_page.
+     install_page(cr3, virt_to_phys(insn_page), insn_ram);
+     invlpg(insn_ram);
+     // Load code TLB
+     asm volatile("call *%0" : : "r"(insn_ram + 3));
+     install_page(cr3, virt_to_phys(alt_insn_page), insn_ram);
+     // Trap, let hypervisor emulate at alt_insn_page
+     asm volatile(
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+
+             "call *%1\n\t"
+
+             "mov %%rax, 0+%[save] \n\t"
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+             "mov %%rsp, 48+%[save] \n\t"
+             "mov %%rbp, 56+%[save] \n\t"
+             /* Save RFLAGS in outregs*/
+             "pushf \n\t"
+             "popq 72+%[save] \n\t"
+             : [save]"+m"(save)
+             : "r"(insn_ram), "a"(mem)
+             : "memory", "cc"
+             );
+     outregs = save;
+}
+

Quoted 8 lines

Quoted 5 lines

x86/emulator.c +0 -1

 static void advance_rip_by_3_and_note_exception(struct ex_regs *regs)
 {
     ++exceptions;
--

1.7.9.5

--
Gleb.

Quoted 6 lines

--
Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

--
Gleb.

Quoted 8 lines · Gleb Natapov

Quoted 7 lines · 李春奇

Quoted 6 lines · Gleb Natapov

Quoted 5 lines · Arthur Chunqi Li

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Quoted 8 lines

Quoted 7 lines

Quoted 4 lines

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

I don't know why set %rax before call alt_insn_page can cause error. I use "xchg %%rax, 0+%[save]" before "call *%1" and the %rcx is not set correctly.

We better find this out :)

I will try to trace it.

Quoted 12 lines

Quoted 12 lines

Quoted 6 lines

%rsp and %rbp because of ret on instruction page? Use the same trick realmode.c test uses: have the code that sets/saves registers in insn_page/alt_insn_page itself and copy the instruction you want to test into the page itself instead of doing call.

I don't know how instructions between calling insn_page and alt_insn_page are executed (function install_page and some other instructions before call *%1". If these insns are executed after insn_page is called, changes before the trapping instruction may affect the executing of these instructions.

Quoted 1 line

Not sure what do you mean here.

Simply, I mean what is the executing sequence in that piece of codes. Why instruction in alt_insn_page will be emulated?

Quoted 111 lines

Quoted 104 lines

Quoted 93 lines

Not sure what is so special about %rflags.

Quoted 90 lines

Signed-off-by Arthur Chunqi Li <[email protected]>

---
 x86/emulator.c |   67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 mode change 100644 => 100755 x86/emulator.c

x86/emulator.c +67 -0

old mode 100644
new mode 100755
index 96576e5..770e8f7
@@ -11,6 +11,13 @@ int fails, tests;
 
 static int exceptions;
 
+struct regs {
+     u64 rax, rbx, rcx, rdx;
+     u64 rsi, rdi, rsp, rbp;
+     u64 rip, rflags;
+};
+static struct regs inregs, outregs;
+
 void report(const char *name, int result)
 {
      ++tests;

@@ -685,6 +692,66 @@ static void test_shld_shrd(u32 *mem)
     report("shrd (cl)", *mem == ((0x12345678 >> 3) | (5u << 29)));
 }
 
+static void trap_emulator(uint64_t *mem, uint8_t *insn_page,
+                          uint8_t *alt_insn_page, void *insn_ram,
+                          uint8_t *alt_insn, int alt_insn_length)
+{
+     ulong *cr3 = (ulong *)read_cr3();
+     int i;
+     static struct regs save;
+
+     // Pad with RET instructions
+     memset(insn_page, 0xc3, 4096);
+     memset(alt_insn_page, 0xc3, 4096);
+
+     // Place a trapping instruction in the page to trigger a VMEXIT
+     insn_page[0] = 0x89; // mov %eax, (%rax)
+     insn_page[1] = 0x00;
+     insn_page[2] = 0x90; // nop
+     insn_page[3] = 0xc3; // ret
+
+     // Place the instruction we want the hypervisor to see in the alternate page
+     for (i=0; i<alt_insn_length; i++)
+             alt_insn_page[i] = alt_insn[i];
+     save = inregs;
+
+     // Load the code TLB with insn_page, but point the page tables at
+     // alt_insn_page (and keep the data TLB clear, for AMD decode assist).
+     // This will make the CPU trap on the insn_page instruction but the
+     // hypervisor will see alt_insn_page.
+     install_page(cr3, virt_to_phys(insn_page), insn_ram);
+     invlpg(insn_ram);
+     // Load code TLB
+     asm volatile("call *%0" : : "r"(insn_ram + 3));
+     install_page(cr3, virt_to_phys(alt_insn_page), insn_ram);
+     // Trap, let hypervisor emulate at alt_insn_page
+     asm volatile(
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+
+             "call *%1\n\t"
+
+             "mov %%rax, 0+%[save] \n\t"
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+             "mov %%rsp, 48+%[save] \n\t"
+             "mov %%rbp, 56+%[save] \n\t"
+             /* Save RFLAGS in outregs*/
+             "pushf \n\t"
+             "popq 72+%[save] \n\t"
+             : [save]"+m"(save)
+             : "r"(insn_ram), "a"(mem)
+             : "memory", "cc"
+             );
+     outregs = save;
+}
+

Quoted 8 lines

Quoted 5 lines

x86/emulator.c +0 -1

 static void advance_rip_by_3_and_note_exception(struct ex_regs *regs)
 {
     ++exceptions;
--

1.7.9.5

--
Gleb.

Quoted 6 lines

--
Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

Quoted 2 lines

--
Gleb.

--
Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

Quoted 7 lines · 李春奇

Quoted 6 lines · Gleb Natapov

Quoted 5 lines · Arthur Chunqi Li

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Quoted 7 lines

Quoted 4 lines

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

I don't know why set %rax before call alt_insn_page can cause error. I use "xchg %%rax, 0+%[save]" before "call *%1" and the %rcx is not set correctly.

We better find this out :)

Quoted 6 lines

Quoted 6 lines

%rsp and %rbp because of ret on instruction page? Use the same trick realmode.c test uses: have the code that sets/saves registers in insn_page/alt_insn_page itself and copy the instruction you want to test into the page itself instead of doing call.

Quoted 6 lines

I don't know how instructions between calling insn_page and alt_insn_page are executed (function install_page and some other instructions before call *%1". If these insns are executed after insn_page is called, changes before the trapping instruction may affect the executing of these instructions.

Not sure what do you mean here.

Quoted 104 lines

Quoted 93 lines

Not sure what is so special about %rflags.

Quoted 90 lines

Signed-off-by Arthur Chunqi Li <[email protected]>

---
 x86/emulator.c |   67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 mode change 100644 => 100755 x86/emulator.c

x86/emulator.c +67 -0

old mode 100644
new mode 100755
index 96576e5..770e8f7
@@ -11,6 +11,13 @@ int fails, tests;
 
 static int exceptions;
 
+struct regs {
+     u64 rax, rbx, rcx, rdx;
+     u64 rsi, rdi, rsp, rbp;
+     u64 rip, rflags;
+};
+static struct regs inregs, outregs;
+
 void report(const char *name, int result)
 {
      ++tests;

@@ -685,6 +692,66 @@ static void test_shld_shrd(u32 *mem)
     report("shrd (cl)", *mem == ((0x12345678 >> 3) | (5u << 29)));
 }
 
+static void trap_emulator(uint64_t *mem, uint8_t *insn_page,
+                          uint8_t *alt_insn_page, void *insn_ram,
+                          uint8_t *alt_insn, int alt_insn_length)
+{
+     ulong *cr3 = (ulong *)read_cr3();
+     int i;
+     static struct regs save;
+
+     // Pad with RET instructions
+     memset(insn_page, 0xc3, 4096);
+     memset(alt_insn_page, 0xc3, 4096);
+
+     // Place a trapping instruction in the page to trigger a VMEXIT
+     insn_page[0] = 0x89; // mov %eax, (%rax)
+     insn_page[1] = 0x00;
+     insn_page[2] = 0x90; // nop
+     insn_page[3] = 0xc3; // ret
+
+     // Place the instruction we want the hypervisor to see in the alternate page
+     for (i=0; i<alt_insn_length; i++)
+             alt_insn_page[i] = alt_insn[i];
+     save = inregs;
+
+     // Load the code TLB with insn_page, but point the page tables at
+     // alt_insn_page (and keep the data TLB clear, for AMD decode assist).
+     // This will make the CPU trap on the insn_page instruction but the
+     // hypervisor will see alt_insn_page.
+     install_page(cr3, virt_to_phys(insn_page), insn_ram);
+     invlpg(insn_ram);
+     // Load code TLB
+     asm volatile("call *%0" : : "r"(insn_ram + 3));
+     install_page(cr3, virt_to_phys(alt_insn_page), insn_ram);
+     // Trap, let hypervisor emulate at alt_insn_page
+     asm volatile(
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+
+             "call *%1\n\t"
+
+             "mov %%rax, 0+%[save] \n\t"
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+             "mov %%rsp, 48+%[save] \n\t"
+             "mov %%rbp, 56+%[save] \n\t"
+             /* Save RFLAGS in outregs*/
+             "pushf \n\t"
+             "popq 72+%[save] \n\t"
+             : [save]"+m"(save)
+             : "r"(insn_ram), "a"(mem)
+             : "memory", "cc"
+             );
+     outregs = save;
+}
+

Quoted 8 lines

Quoted 5 lines

x86/emulator.c +0 -1

 static void advance_rip_by_3_and_note_exception(struct ex_regs *regs)
 {
     ++exceptions;
--

1.7.9.5

--
Gleb.

Quoted 6 lines

--
Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

--
Gleb.

Quoted 10 lines · Gleb Natapov

Quoted 9 lines · 李春奇

Quoted 8 lines · Gleb Natapov

Quoted 5 lines · Arthur Chunqi Li

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

Quoted 6 lines

Quoted 5 lines

Quoted 2 lines

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

I don't know why set %rax before call alt_insn_page can cause error. I use "xchg %%rax, 0+%[save]" before "call *%1" and the %rcx is not set correctly.

We better find this out :)

I found that before calling alt_insn_page, address of "mem" is saved to %rax, why?

Quoted 125 lines

Quoted 6 lines

Quoted 6 lines

%rsp and %rbp because of ret on instruction page? Use the same trick realmode.c test uses: have the code that sets/saves registers in insn_page/alt_insn_page itself and copy the instruction you want to test into the page itself instead of doing call.

Quoted 6 lines

I don't know how instructions between calling insn_page and alt_insn_page are executed (function install_page and some other instructions before call *%1". If these insns are executed after insn_page is called, changes before the trapping instruction may affect the executing of these instructions.

Not sure what do you mean here.

Quoted 104 lines

Quoted 93 lines

Not sure what is so special about %rflags.

Quoted 90 lines

Signed-off-by Arthur Chunqi Li <[email protected]>

---
 x86/emulator.c |   67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 mode change 100644 => 100755 x86/emulator.c

x86/emulator.c +67 -0

old mode 100644
new mode 100755
index 96576e5..770e8f7
@@ -11,6 +11,13 @@ int fails, tests;
 
 static int exceptions;
 
+struct regs {
+     u64 rax, rbx, rcx, rdx;
+     u64 rsi, rdi, rsp, rbp;
+     u64 rip, rflags;
+};
+static struct regs inregs, outregs;
+
 void report(const char *name, int result)
 {
      ++tests;

@@ -685,6 +692,66 @@ static void test_shld_shrd(u32 *mem)
     report("shrd (cl)", *mem == ((0x12345678 >> 3) | (5u << 29)));
 }
 
+static void trap_emulator(uint64_t *mem, uint8_t *insn_page,
+                          uint8_t *alt_insn_page, void *insn_ram,
+                          uint8_t *alt_insn, int alt_insn_length)
+{
+     ulong *cr3 = (ulong *)read_cr3();
+     int i;
+     static struct regs save;
+
+     // Pad with RET instructions
+     memset(insn_page, 0xc3, 4096);
+     memset(alt_insn_page, 0xc3, 4096);
+
+     // Place a trapping instruction in the page to trigger a VMEXIT
+     insn_page[0] = 0x89; // mov %eax, (%rax)
+     insn_page[1] = 0x00;
+     insn_page[2] = 0x90; // nop
+     insn_page[3] = 0xc3; // ret
+
+     // Place the instruction we want the hypervisor to see in the alternate page
+     for (i=0; i<alt_insn_length; i++)
+             alt_insn_page[i] = alt_insn[i];
+     save = inregs;
+
+     // Load the code TLB with insn_page, but point the page tables at
+     // alt_insn_page (and keep the data TLB clear, for AMD decode assist).
+     // This will make the CPU trap on the insn_page instruction but the
+     // hypervisor will see alt_insn_page.
+     install_page(cr3, virt_to_phys(insn_page), insn_ram);
+     invlpg(insn_ram);
+     // Load code TLB
+     asm volatile("call *%0" : : "r"(insn_ram + 3));
+     install_page(cr3, virt_to_phys(alt_insn_page), insn_ram);
+     // Trap, let hypervisor emulate at alt_insn_page
+     asm volatile(
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+
+             "call *%1\n\t"
+
+             "mov %%rax, 0+%[save] \n\t"
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+             "mov %%rsp, 48+%[save] \n\t"
+             "mov %%rbp, 56+%[save] \n\t"
+             /* Save RFLAGS in outregs*/
+             "pushf \n\t"
+             "popq 72+%[save] \n\t"
+             : [save]"+m"(save)
+             : "r"(insn_ram), "a"(mem)
+             : "memory", "cc"
+             );
+     outregs = save;
+}
+

Quoted 8 lines

Quoted 5 lines

x86/emulator.c +0 -1

 static void advance_rip_by_3_and_note_exception(struct ex_regs *regs)
 {
     ++exceptions;
--

1.7.9.5

--
Gleb.

Quoted 6 lines

--
Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

Quoted 2 lines

--
Gleb.

--
Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

Quoted 13 lines · 李春奇

Quoted 12 lines · Gleb Natapov

Quoted 11 lines · 李春奇

Quoted 10 lines · Gleb Natapov

Quoted 5 lines · Arthur Chunqi Li

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

Quoted 6 lines

Quoted 4 lines

Quoted 3 lines

I don't know why set %rax before call alt_insn_page can cause error. I use "xchg %%rax, 0+%[save]" before "call *%1" and the %rcx is not set correctly.

We better find this out :)

I found that before calling alt_insn_page, address of "mem" is saved to %rax, why?

Because instruction that we use to trigger vmexit is mov %eax, (%rax) so MMOI address mem is loaded into %rax before jumping into it.

Quoted 131 lines

Quoted 126 lines

Quoted 7 lines

Quoted 7 lines

%rsp and %rbp because of ret on instruction page? Use the same trick realmode.c test uses: have the code that sets/saves registers in insn_page/alt_insn_page itself and copy the instruction you want to test into the page itself instead of doing call.

Quoted 6 lines

I don't know how instructions between calling insn_page and alt_insn_page are executed (function install_page and some other instructions before call *%1". If these insns are executed after insn_page is called, changes before the trapping instruction may affect the executing of these instructions.

Not sure what do you mean here.

Quoted 104 lines

Quoted 93 lines

Not sure what is so special about %rflags.

Quoted 90 lines

Signed-off-by Arthur Chunqi Li <[email protected]>

---
 x86/emulator.c |   67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 mode change 100644 => 100755 x86/emulator.c

x86/emulator.c +67 -0

old mode 100644
new mode 100755
index 96576e5..770e8f7
@@ -11,6 +11,13 @@ int fails, tests;
 
 static int exceptions;
 
+struct regs {
+     u64 rax, rbx, rcx, rdx;
+     u64 rsi, rdi, rsp, rbp;
+     u64 rip, rflags;
+};
+static struct regs inregs, outregs;
+
 void report(const char *name, int result)
 {
      ++tests;

@@ -685,6 +692,66 @@ static void test_shld_shrd(u32 *mem)
     report("shrd (cl)", *mem == ((0x12345678 >> 3) | (5u << 29)));
 }
 
+static void trap_emulator(uint64_t *mem, uint8_t *insn_page,
+                          uint8_t *alt_insn_page, void *insn_ram,
+                          uint8_t *alt_insn, int alt_insn_length)
+{
+     ulong *cr3 = (ulong *)read_cr3();
+     int i;
+     static struct regs save;
+
+     // Pad with RET instructions
+     memset(insn_page, 0xc3, 4096);
+     memset(alt_insn_page, 0xc3, 4096);
+
+     // Place a trapping instruction in the page to trigger a VMEXIT
+     insn_page[0] = 0x89; // mov %eax, (%rax)
+     insn_page[1] = 0x00;
+     insn_page[2] = 0x90; // nop
+     insn_page[3] = 0xc3; // ret
+
+     // Place the instruction we want the hypervisor to see in the alternate page
+     for (i=0; i<alt_insn_length; i++)
+             alt_insn_page[i] = alt_insn[i];
+     save = inregs;
+
+     // Load the code TLB with insn_page, but point the page tables at
+     // alt_insn_page (and keep the data TLB clear, for AMD decode assist).
+     // This will make the CPU trap on the insn_page instruction but the
+     // hypervisor will see alt_insn_page.
+     install_page(cr3, virt_to_phys(insn_page), insn_ram);
+     invlpg(insn_ram);
+     // Load code TLB
+     asm volatile("call *%0" : : "r"(insn_ram + 3));
+     install_page(cr3, virt_to_phys(alt_insn_page), insn_ram);
+     // Trap, let hypervisor emulate at alt_insn_page
+     asm volatile(
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+
+             "call *%1\n\t"
+
+             "mov %%rax, 0+%[save] \n\t"
+             "xchg %%rbx, 8+%[save] \n\t"
+             "xchg %%rcx, 16+%[save] \n\t"
+             "xchg %%rdx, 24+%[save] \n\t"
+             "xchg %%rsi, 32+%[save] \n\t"
+             "xchg %%rdi, 40+%[save] \n\t"
+             "mov %%rsp, 48+%[save] \n\t"
+             "mov %%rbp, 56+%[save] \n\t"
+             /* Save RFLAGS in outregs*/
+             "pushf \n\t"
+             "popq 72+%[save] \n\t"
+             : [save]"+m"(save)
+             : "r"(insn_ram), "a"(mem)
+             : "memory", "cc"
+             );
+     outregs = save;
+}
+

Quoted 8 lines

Quoted 5 lines

x86/emulator.c +0 -1

 static void advance_rip_by_3_and_note_exception(struct ex_regs *regs)
 {
     ++exceptions;
--

1.7.9.5

--
Gleb.

Quoted 6 lines

--
Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

Quoted 2 lines

--
Gleb.

Quoted 6 lines

--
Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

--
Gleb.

Quoted 20 lines · Gleb Natapov

Quoted 19 lines · 李春奇

Quoted 8 lines · Gleb Natapov

Quoted 7 lines · 李春奇

Quoted 6 lines · Gleb Natapov

Quoted 5 lines · Arthur Chunqi Li

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Quoted 8 lines

Quoted 7 lines

Quoted 4 lines

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

I don't know why set %rax before call alt_insn_page can cause error. I use "xchg %%rax, 0+%[save]" before "call *%1" and the %rcx is not set correctly.

We better find this out :)

I found that before calling alt_insn_page, address of "mem" is saved to %rax, why?

Quoted 2 lines

Because instruction that we use to trigger vmexit is mov %eax, (%rax) so MMOI address mem is loaded into %rax before jumping into it.

I think this is why changing %rax will cause error. If we use mov %eax, (%rax) to trigger vmexit, and %rax is changed before calling alt_insn_page, codes in alt_insn_page will not be executed and return directly.
I changed the codes which trigger vmexit to "mov %eax, (%r8)" and set "mem" to %r8 before calling alt_insn_page, it runs OK.

Besides, I also don't know if changed %rflags may cause some unpredictable actions, so now we just treat it with no error :)

Quoted 21 lines · 李春奇

Quoted 20 lines · Gleb Natapov

Quoted 19 lines · 李春奇

Quoted 8 lines · Gleb Natapov

Quoted 7 lines · 李春奇

Quoted 6 lines · Gleb Natapov

Quoted 5 lines · Arthur Chunqi Li

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Quoted 8 lines

Quoted 7 lines

Quoted 4 lines

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

I don't know why set %rax before call alt_insn_page can cause error. I use "xchg %%rax, 0+%[save]" before "call *%1" and the %rcx is not set correctly.

We better find this out :)

I found that before calling alt_insn_page, address of "mem" is saved to %rax, why?

Quoted 9 lines

Quoted 2 lines

Because instruction that we use to trigger vmexit is mov %eax, (%rax) so MMOI address mem is loaded into %rax before jumping into it.

I think this is why changing %rax will cause error. If we use mov %eax, (%rax) to trigger vmexit, and %rax is changed before calling alt_insn_page, codes in alt_insn_page will not be executed and return directly.
I changed the codes which trigger vmexit to "mov %eax, (%r8)" and set "mem" to %r8 before calling alt_insn_page, it runs OK.

Just use an instruction that does not use registers at all. mov $1, addr where addr is immediate and encoded from mem parameter.

Quoted 2 lines

Besides, I also don't know if changed %rflags may cause some unpredictable actions, so now we just treat it with no error :)

If test sets rflags to a value that causes crashes this is a test bug, no need to prevent this from happening.

--
Gleb.

I have finished the infrastructure but after changing test_mmx_movq_mf test case, it return error:
unhandled excecption 6
Return value from qemu: 15

If I don't change %rsp and %rbp, it runs OK.
So I wonder if this test case is strictly reply on %rsp and %rbp?

Quoted 34 lines · 李春奇

Quoted 21 lines · Gleb Natapov

Quoted 20 lines · 李春奇

Quoted 19 lines · Gleb Natapov

Quoted 8 lines · 李春奇

Quoted 7 lines · Gleb Natapov

Quoted 6 lines · Arthur Chunqi Li

Quoted 5 lines

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Quoted 8 lines

Quoted 7 lines

Quoted 4 lines

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

I don't know why set %rax before call alt_insn_page can cause error. I use "xchg %%rax, 0+%[save]" before "call *%1" and the %rcx is not set correctly.

We better find this out :)

I found that before calling alt_insn_page, address of "mem" is saved to %rax, why?

Quoted 9 lines

Quoted 2 lines

Because instruction that we use to trigger vmexit is mov %eax, (%rax) so MMOI address mem is loaded into %rax before jumping into it.

I think this is why changing %rax will cause error. If we use mov %eax, (%rax) to trigger vmexit, and %rax is changed before calling alt_insn_page, codes in alt_insn_page will not be executed and return directly.
I changed the codes which trigger vmexit to "mov %eax, (%r8)" and set "mem" to %r8 before calling alt_insn_page, it runs OK.

Just use an instruction that does not use registers at all. mov $1, addr where addr is immediate and encoded from mem parameter.

Quoted 7 lines

Quoted 2 lines

Besides, I also don't know if changed %rflags may cause some unpredictable actions, so now we just treat it with no error :)

If test sets rflags to a value that causes crashes this is a test bug, no need to prevent this from happening.

--
Gleb.

Signature · 5 lines

Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

Quoted 8 lines · 李春奇

I have finished the infrastructure but after changing test_mmx_movq_mf test case, it return error:
unhandled excecption 6
Return value from qemu: 15

If I don't change %rsp and %rbp, it runs OK.
So I wonder if this test case is strictly reply on %rsp and %rbp?

I can't help without seeing the code.

Quoted 45 lines · Gleb Natapov

Quoted 34 lines · 李春奇

Quoted 21 lines · Gleb Natapov

Quoted 20 lines · 李春奇

Quoted 19 lines · Gleb Natapov

Quoted 8 lines · 李春奇

Quoted 7 lines · Gleb Natapov

Quoted 6 lines · Arthur Chunqi Li

Quoted 5 lines

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Quoted 8 lines

Quoted 7 lines

Quoted 4 lines

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

I don't know why set %rax before call alt_insn_page can cause error. I use "xchg %%rax, 0+%[save]" before "call *%1" and the %rcx is not set correctly.

We better find this out :)

I found that before calling alt_insn_page, address of "mem" is saved to %rax, why?

Quoted 9 lines

Quoted 2 lines

Because instruction that we use to trigger vmexit is mov %eax, (%rax) so MMOI address mem is loaded into %rax before jumping into it.

I think this is why changing %rax will cause error. If we use mov %eax, (%rax) to trigger vmexit, and %rax is changed before calling alt_insn_page, codes in alt_insn_page will not be executed and return directly.
I changed the codes which trigger vmexit to "mov %eax, (%r8)" and set "mem" to %r8 before calling alt_insn_page, it runs OK.

Just use an instruction that does not use registers at all. mov $1, addr where addr is immediate and encoded from mem parameter.

Quoted 7 lines

Quoted 2 lines

Besides, I also don't know if changed %rflags may cause some unpredictable actions, so now we just treat it with no error :)

If test sets rflags to a value that causes crashes this is a test bug, no need to prevent this from happening.

--
Gleb.

Quoted 6 lines

--
Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

--
Gleb.

I have trouble to send the address of inregs into alt_insn_page, so I use r9 and r10 to carry %rsp and %rbp into it. And r8 is used to trigger vmexit.

I paste the relevant functions as follows:

static void trap_emulator(uint64_t *mem, uint8_t *insn_page,
uint8_t *alt_insn_page, void *insn_ram, uint8_t *alt_insn, int alt_insn_length)

{
    ulong *cr3 = (ulong *)read_cr3();
    int i;
    static struct regs save;

    // Pad with RET instructions
    memset(insn_page, 0x90, 4096);
    memset(alt_insn_page, 0x90, 4096);

    // Place a trapping instruction in the page to trigger a VMEXIT
    insn_page[0] = 0x49; //xchg   %rsp, %r9
    insn_page[1] = 0x87;
    insn_page[2] = 0xe1;
    insn_page[3] = 0x49; //xchg   %rbp, %r10
    insn_page[4] = 0x87;
    insn_page[5] = 0xea;
    insn_page[6] = 0x41; // mov %eax, (%r8)
    insn_page[7] = 0x89;
    insn_page[8] = 0x00; // ret

    // Place the instruction we want the hypervisor to see in the alternate page
    for (i=0; i<alt_insn_length; i++)
        alt_insn_page[i+6] = alt_insn[i];
    i+=6;
    insn_page[i++] = 0x49; //xchg   %rsp, %r9
    insn_page[i++] = 0x87;
    insn_page[i++] = 0xe1;
    insn_page[i++] = 0x49; //xchg   %rbp, %r10
    insn_page[i++] = 0x87;
    insn_page[i++] = 0xea;
    insn_page[i++] = 0xc3; // ret
    save = inregs;
    // Load the code TLB with insn_page, but point the page tables at
    // alt_insn_page (and keep the data TLB clear, for AMD decode assist).
    // This will make the CPU trap on the insn_page instruction but the
    // hypervisor will see alt_insn_page.
    install_page(cr3, virt_to_phys(insn_page), insn_ram);
    invlpg(insn_ram);
    // Load code TLB
    asm volatile("call *%0" : : "r"(insn_ram));
    install_page(cr3, virt_to_phys(alt_insn_page), insn_ram);
    // Trap, let hypervisor emulate at alt_insn_page
    asm volatile(
    "mov %2, %%r8\n\r"
    "push 72+%[save]; popf\n\t"
    "xchg %%rax, 0+%[save]\n\t"
    "xchg %%rbx, 8+%[save] \n\t"
    "xchg %%rcx, 16+%[save] \n\t"
    "xchg %%rdx, 24+%[save] \n\t"
    "xchg %%rsi, 32+%[save] \n\t"
    "xchg %%rdi, 40+%[save] \n\t"
    "xchg %%r9, 48+%[save] \n\t" // %rsp in %r9
    "xchg %%r10, 56+%[save] \n\t" // %rbp in %r10

    "call *%1\n\t"

    "xchg %%rax, 0+%[save] \n\t"
    "xchg %%rbx, 8+%[save] \n\t"
    "xchg %%rcx, 16+%[save] \n\t"
    "xchg %%rdx, 24+%[save] \n\t"
    "xchg %%rsi, 32+%[save] \n\t"
    "xchg %%rdi, 40+%[save] \n\t"
    "xchg %%r9, 48+%[save] \n\t" // %rsp in %r9
    "xchg %%r10, 56+%[save] \n\t" // %rbp in %r10
    /* Save RFLAGS in outregs*/
    "pushf \n\t"
    "pop 72+%[save] \n\t"
    : [save]"=m"(save)
    : "r"(insn_ram), "r"(mem)
    : "memory", "cc", "r9", "r10", "r8"
    );
    outregs = save;
}

static void test_mmx_movq_mf(uint64_t *mem, uint8_t *insn_page,
uint8_t *alt_insn_page, void *insn_ram)

{
    uint16_t fcw = 0;  // all exceptions unmasked
    uint8_t alt_insn[] = {0x0f, 0x7f, 0x00}; // movq %mm0, (%rax)

    write_cr0(read_cr0() & ~6);  // TS, EM
    exceptions = 0;
    handle_exception(MF_VECTOR, advance_rip_by_3_and_note_exception);
    asm volatile("fninit; fldcw %0" : : "m"(fcw));
    asm volatile("fldz; fldz; fdivp"); // generate exception

    inregs = (struct regs){ .rsp=0, .rbp=0 };
    trap_emulator(mem, insn_page, alt_insn_page, insn_ram,
alt_insn, 3);
    // exit MMX mode
    asm volatile("fnclex; emms");
    report("movq mmx generates #MF2", exceptions == 1);
    handle_exception(MF_VECTOR, 0);
}

Quoted 63 lines · Gleb Natapov

Quoted 8 lines · 李春奇

I have finished the infrastructure but after changing test_mmx_movq_mf test case, it return error:
unhandled excecption 6
Return value from qemu: 15

If I don't change %rsp and %rbp, it runs OK.
So I wonder if this test case is strictly reply on %rsp and %rbp?

I can't help without seeing the code.

Quoted 51 lines · Gleb Natapov

Quoted 34 lines · 李春奇

Quoted 21 lines · Gleb Natapov

Quoted 20 lines · 李春奇

Quoted 19 lines · Gleb Natapov

Quoted 8 lines · 李春奇

Quoted 7 lines · Gleb Natapov

Quoted 6 lines · Arthur Chunqi Li

Quoted 5 lines

Add a function trap_emulator to run an instruction in emulator. Set inregs first (%rax, %rsp, %rbp, %rflags have special usage and cannot set in inregs), put instruction codec in alt_insn and call func with alt_insn_length. Get results in outregs.

Quoted 8 lines

Quoted 7 lines

Quoted 4 lines

Why %rax, %rsp, %rbp, %rflags cannot be set in inregs?

%rax because trapping instruction uses it? Use one that does not use register at all: MOV r/m32, imm32

I don't know why set %rax before call alt_insn_page can cause error. I use "xchg %%rax, 0+%[save]" before "call *%1" and the %rcx is not set correctly.

We better find this out :)

I found that before calling alt_insn_page, address of "mem" is saved to %rax, why?

Quoted 9 lines

Quoted 2 lines

Because instruction that we use to trigger vmexit is mov %eax, (%rax) so MMOI address mem is loaded into %rax before jumping into it.

I think this is why changing %rax will cause error. If we use mov %eax, (%rax) to trigger vmexit, and %rax is changed before calling alt_insn_page, codes in alt_insn_page will not be executed and return directly.
I changed the codes which trigger vmexit to "mov %eax, (%r8)" and set "mem" to %r8 before calling alt_insn_page, it runs OK.

Just use an instruction that does not use registers at all. mov $1, addr where addr is immediate and encoded from mem parameter.

Quoted 7 lines

Quoted 2 lines

Besides, I also don't know if changed %rflags may cause some unpredictable actions, so now we just treat it with no error :)

If test sets rflags to a value that causes crashes this is a test bug, no need to prevent this from happening.

--
Gleb.

--
Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China

Quoted 2 lines

--
Gleb.

Signature · 5 lines

Arthur Chunqi Li
Department of Computer Science
School of EECS
Peking University
Beijing, China