[PATCH][RFC] Emulating real mode with x86_emulate

Anthony Liguori

author

2007-03-25 23:35 UTC #0 lore raw

RFC PATCH 1/1 Emulating real mode with x86_emulate

Howdy,

Attached is a patch that begins to lay down the infrastructure for emulating real mode with x86_emulate(). With a little more refactoring, I think it could also replace the SVM emulator.

The patch introduces an HVMOP hypercall to set a flag in the hvm vcpu struct to signal that real mode should be emulated with x86_emulate instead of using vm86. This is to make development a little bit easier since x86_emulate is not quite there yet wrt 16 bit emulation. It can be enabled by passing -emulate-16bit to qemu-dm (I use a wrapper script similar to qemu-dm.debug).

The VT code keeps track of the whether it's in the emulator and loops on the do_resume path in x86_emulate. I think this code probably should be refactored into the common HVM code although this would require changing some of the HVM ops. This would allow SVM to use the x86_emulate to handle individual instructions.

There are some issues to work out. Namely, x86_emulate appears to want blocking PIO calls which isn't conducive to the wait PIO works today in HVM. This is only a problem for instructions at the moment. I'm also a bit confused about how to properly loop in the emulator. schedule_tail is not meant to return so perhaps we should loop on emulating == 1 instead of hypercall_preempt_check? I didn't think the hypervisor was preemptable though.

The current code doesn't handle non-flat segments as I don't think hvm_copy_from/to_guest handles it (which I assume it would need to).

However, it is enough to start running instructions in x86_emulate so it's enough to start working on enhancing that.

Regards,

Anthony Liguori

tools/firmware/hvmloader/hvmloader.c +25 -9

diff -r 3fd9b0c71b8c tools/firmware/hvmloader/hvmloader.c
@@ -349,6 +349,14 @@ static void cmos_write_memory_size(void)
     cmos_outb(0x35, (uint8_t)( alt_mem >> 8));
 }
 
+#define DOMID_SELF (0x7FF0U)
+
+#define HVMOP_emulate_realmode 6
+struct xen_hvm_op_emulate_realmode
+{
+    uint16_t domid;
+};
+
 int main(void)
 {
     int acpi_sz;

@@ -401,15 +409,23 @@ int main(void)
 
     if ( !check_amd() )
     {
-        printf("Loading VMXAssist ...\n");
-        memcpy((void *)VMXASSIST_PHYSICAL_ADDRESS,
-               vmxassist, sizeof(vmxassist));
-
-        printf("VMX go ...\n");
-        __asm__ __volatile__(
-            "jmp *%%eax"
-            : : "a" (VMXASSIST_PHYSICAL_ADDRESS), "d" (0)
-            );
+        printf("Loading VMXAssist ... %x\n", inl(0x595));
+        if (inl(0x595) == 0xdeadbeef) {
+            struct xen_hvm_op_emulate_realmode op;
+
+            printf("foo\n");
+            op.domid = DOMID_SELF;
+            hypercall_hvm_op(HVMOP_emulate_realmode, &op);
+        } else {
+            memcpy((void *)VMXASSIST_PHYSICAL_ADDRESS,
+                   vmxassist, sizeof(vmxassist));
+
+            printf("VMX go ...\n");
+            __asm__ __volatile__(
+                                 "jmp *%%eax"
+                                 : : "a" (VMXASSIST_PHYSICAL_ADDRESS), "d" (0)
+                                 );
+        }
     }
 
     printf("Invoking ROMBIOS ...\n");

tools/ioemu/hw/pc.c +9 -0

diff -r 3fd9b0c71b8c tools/ioemu/hw/pc.c
@@ -316,6 +316,13 @@ static uint32_t ioport92_read(void *opaq
     return ioport_get_a20() << 1;
 }
 
+static uint32_t emulate_16bit_read(void *opaque, uint32_t addr)
+{
+    if (emulate_16bit)
+	return 0xdeadbeef;
+    return 0;
+}
+
 /***********************************************************/
 /* Bochs BIOS debug ports */

@@ -728,6 +735,8 @@ static void pc_init1(uint64_t ram_size, 
     
     bochs_bios_init();
 
+    register_ioport_read(0x595, 1, 4, emulate_16bit_read, NULL);
+
 #ifndef CONFIG_DM
     if (linux_boot) {
         uint8_t bootsect[512];

tools/ioemu/vl.c +6 -0

diff -r 3fd9b0c71b8c tools/ioemu/vl.c
@@ -161,6 +161,7 @@ int vnc_display = -1;
 #endif
 int acpi_enabled = 0;
 int fd_bootchk = 1;
+int emulate_16bit;
 
 extern int vcpus;

@@ -5535,6 +5536,7 @@ enum {
     QEMU_OPTION_vncviewer,
     QEMU_OPTION_vncunused,
     QEMU_OPTION_vnclisten,
+    QEMU_OPTION_emulate_16bit,
 };
 
 typedef struct QEMUOption {

@@ -5614,6 +5616,7 @@ const QEMUOption qemu_options[] = {
     { "vncviewer", 0, QEMU_OPTION_vncviewer },
     { "vncunused", 0, QEMU_OPTION_vncunused },
     { "vnclisten", HAS_ARG, QEMU_OPTION_vnclisten },
+    { "emulate-16bit", 0, QEMU_OPTION_emulate_16bit },
     
     /* temporary options */
     { "usb", 0, QEMU_OPTION_usb },

@@ -6522,6 +6525,9 @@ int main(int argc, char **argv)
             case QEMU_OPTION_vnclisten:
                 parse_host(&vnclisten_addr, optarg);
                 break;
+	    case QEMU_OPTION_emulate_16bit:
+		emulate_16bit = 1;
+		break;
             }
         }
     }

tools/ioemu/vl.h +1 -0

diff -r 3fd9b0c71b8c tools/ioemu/vl.h
@@ -156,6 +156,7 @@ extern void *shared_vram;
 
 extern FILE *logfile;
 
+int emulate_16bit;
 
 #if defined(__i386__) || defined(__x86_64__)

xen/arch/x86/hvm/hvm.c +39 -0

diff -r 3fd9b0c71b8c xen/arch/x86/hvm/hvm.c
@@ -185,6 +185,41 @@ static int hvmop_drain_io(
 
  out:
     rcu_unlock_domain(d);
+    return rc;
+}
+
+static int hvmop_emulate_realmode(
+    XEN_GUEST_HANDLE(xen_hvm_emulate_realmode_t) uop)
+{
+    struct xen_hvm_emulate_realmode op;
+    struct domain *d;
+    struct vcpu *v;
+    int rc;
+
+    printk("hvmop_emulate_realmode\n");
+
+    if ( copy_from_guest(&op, uop, 1) )
+        return -EFAULT;
+
+    if ( op.domid != DOMID_SELF )
+        return -EPERM;
+
+    d = rcu_lock_current_domain();
+    if ( d == NULL )
+        return -ESRCH;
+
+    printk("guest requests real mode emulation\n");
+
+    for_each_vcpu(d, v)
+    {
+        v->arch.hvm_vcpu.emulate_realmode = 1;
+    }
+
+    rc = 0;
+
+    rcu_unlock_domain(d);
+    printk("foo %d\n", __LINE__);
+
     return rc;
 }

@@ -963,6 +998,10 @@ long do_hvm_op(unsigned long op, XEN_GUE
             guest_handle_cast(arg, xen_hvm_drain_io_t));
         break;
 
+    case HVMOP_emulate_realmode:
+        rc = hvmop_emulate_realmode(
+            guest_handle_cast(arg, xen_hvm_emulate_realmode_t));
+        break;
 
     default:
     {

xen/arch/x86/hvm/vmx/vmcs.c +141 -1

154 lines not on this page · raw message

xen/arch/x86/hvm/vmx/vmx.c +18 -2

diff -r 3fd9b0c71b8c xen/arch/x86/hvm/vmx/vmx.c
@@ -1865,7 +1865,7 @@ static int vmx_assist(struct vcpu *v, in
     return 0;
 }
 
-static int vmx_set_cr0(unsigned long value)
+int vmx_set_cr0(unsigned long value)
 {
     struct vcpu *v = current;
     unsigned long mfn;

@@ -1982,13 +1982,29 @@ static int vmx_set_cr0(unsigned long val
             }
         }
 
-        if ( vmx_assist(v, VMX_ASSIST_INVOKE) )
+        if ( v->arch.hvm_vcpu.emulate_realmode )
+        {
+            eip = __vmread(GUEST_RIP);
+            HVM_DBG_LOG(DBG_LEVEL_1,
+                        "Transfering control to x86_emulate %%eip 0x%lx\n", eip);
+            v->arch.hvm_vmx.emulating = 1;
+            return 1;
+        }
+        else if ( vmx_assist(v, VMX_ASSIST_INVOKE) )
         {
             eip = __vmread(GUEST_RIP);
             HVM_DBG_LOG(DBG_LEVEL_1,
                         "Transfering control to vmxassist %%eip 0x%lx\n", eip);
             return 0; /* do not update eip! */
         }
+    }
+    else if ( v->arch.hvm_vmx.emulating )
+    {
+        eip = __vmread(GUEST_RIP);
+        HVM_DBG_LOG(DBG_LEVEL_1,
+                    "Enabling CR0.PE at %%eip 0x%lx\n", eip);
+        v->arch.hvm_vmx.emulating = 0;
+        return 1;
     }
     else if ( v->arch.hvm_vmx.vmxassist_enabled )
     {

xen/include/asm-x86/hvm/vcpu.h +1 -0

diff -r 3fd9b0c71b8c xen/include/asm-x86/hvm/vcpu.h
@@ -44,6 +44,7 @@ struct hvm_vcpu {
 
     /* Flags */
     int                 flag_dr_dirty;
+    unsigned long       emulate_realmode;
 
     union {
         struct arch_vmx_struct vmx;

xen/include/asm-x86/hvm/vmx/vmcs.h +1 -0

diff -r 3fd9b0c71b8c xen/include/asm-x86/hvm/vmx/vmcs.h
@@ -77,6 +77,7 @@ struct arch_vmx_struct {
     unsigned long        cpu_cr2; /* save CR2 */
     unsigned long        cpu_cr3;
     struct vmx_msr_state msr_state;
+    unsigned long        emulating;
     unsigned long        vmxassist_enabled:1;
 };

xen/include/public/hvm/hvm_op.h +9 -0

diff -r 3fd9b0c71b8c xen/include/public/hvm/hvm_op.h
@@ -78,4 +78,13 @@ typedef struct xen_hvm_drain_io xen_hvm_
 typedef struct xen_hvm_drain_io xen_hvm_drain_io_t;
 DEFINE_XEN_GUEST_HANDLE(xen_hvm_drain_io_t);
 
+/* Enter into 16 bit emulation */
+#define HVMOP_emulate_realmode    6
+struct xen_hvm_emulate_realmode {
+    /* Should be DOMID_SELF */
+    domid_t  domid;
+};
+typedef struct xen_hvm_emulate_realmode xen_hvm_emulate_realmode_t;
+DEFINE_XEN_GUEST_HANDLE(xen_hvm_emulate_realmode_t);
+
 #endif /* __XEN_PUBLIC_HVM_HVM_OP_H__ */

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Kamble, Nitin A

2007-03-29 21:17 UTC #1 lore raw

Hi Anthony,
I tried your patch at my end with snapshot of xen-unstable on
20070326. And here are my findings.
- The --emulate-16bit option for qemu did not work. qemu is not
accepting this as an valid command line option.
- I tried hardcoding this to 1 in the qemu code, and found that the
port x595 is not reflecting it correctly to hvmloader.
- Then I tried forcing in the hvmloader to use the emulator instead
of vmxassit, and I see the the emulator is not getting the right context. Following is the console log at this point.

(XEN) HVM3: HVM
Loader
(XEN) HVM3: Detected Xen
v3.0-unstable
(XEN) HVM3: Writing SMBIOS
tables ...
(XEN) HVM3: Loading
ROMBIOS ...
(XEN) irq.c:210: Dom3 PCI link 0 changed 0 ->
5
(XEN) HVM3: PCI-ISA link 0 routed to
IRQ5
(XEN) irq.c:210: Dom3 PCI link 1 changed 0 ->
7
(XEN) HVM3: PCI-ISA link 1 routed to
IRQ7
(XEN) irq.c:210: Dom3 PCI link 2 changed 0 ->
10
(XEN) HVM3: PCI-ISA link 2 routed to
IRQ10
(XEN) irq.c:210: Dom3 PCI link 3 changed 0 ->
11
(XEN) HVM3: PCI-ISA link 3 routed to
IRQ11
(XEN) HVM3: pci dev 01:1 bar 20 size 00000010:
0000c001
(XEN) HVM3: pci dev 01:2
INTA->IRQ7
(XEN) HVM3: pci dev 02:0 bar 10 size 02000000:
f0000008
(XEN) HVM3: pci dev 02:0 bar 14 size 00001000:
f2000000
(XEN) HVM3: pci dev 03:0 bar 10 size 00000100:
0000c101
(XEN) HVM3: pci dev 03:0 bar 14 size 01000000:
f3000008
(XEN) HVM3: pci dev 03:0
INTA->IRQ11
(XEN) HVM3: pci dev 04:0 bar 10 size 00000100:
0000c201
(XEN) HVM3: pci dev 04:0 bar 14 size 00000100:
f4000000
(XEN) HVM3: pci dev 04:0
INTA->IRQ5
(XEN) HVM3: Creating MP
tables ...
(XEN) HVM3: Loading Cirrus
VGABIOS ...
(XEN) HVM3: Loading
ACPI ...
(XEN) HVM3: Not Loading VMXAssist ...
ffffffff
(XEN) HVM3:
foo
(XEN)
hvmop_emulate_realmode
(XEN) guest requests real mode
emulation
(XEN) foo
221
(XEN) HVM3: Invoking
ROMBIOS ...
(XEN) Transfering control to x86_emulate %eip
0x0
(XEN) hvm.c:446:d3 Triple fault on VCPU0 - invoking HVM system reset.

The log shows that emulator is not getting the right cpu context. How much of testing have you done with this code? I am not able to proceed to emulation of 1 instruction with the patch. How many instruction could you emulate with the patch?

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

Quoted 40 lines · Anthony Liguori

Howdy,
Attached is a patch that begins to lay down the infrastructure for emulating real mode with x86_emulate(). With a little more refactoring,
I think it could also replace the SVM emulator.
The patch introduces an HVMOP hypercall to set a flag in the hvm vcpu struct to signal that real mode should be emulated with x86_emulate instead of using vm86. This is to make development a little bit easier
since x86_emulate is not quite there yet wrt 16 bit emulation. It can be enabled by passing -emulate-16bit to qemu-dm (I use a wrapper script
similar to qemu-dm.debug).
The VT code keeps track of the whether it's in the emulator and loops on
the do_resume path in x86_emulate. I think this code probably should be
refactored into the common HVM code although this would require changing
some of the HVM ops. This would allow SVM to use the x86_emulate to handle individual instructions.
There are some issues to work out. Namely, x86_emulate appears to want
blocking PIO calls which isn't conducive to the wait PIO works today in
HVM. This is only a problem for instructions at the moment. I'm also a
bit confused about how to properly loop in the emulator. schedule_tail
is not meant to return so perhaps we should loop on emulating == 1 instead of hypercall_preempt_check? I didn't think the hypervisor was preemptable though.
The current code doesn't handle non-flat segments as I don't think hvm_copy_from/to_guest handles it (which I assume it would need to).

Quoted 8 lines

However, it is enough to start running instructions in x86_emulate so it's enough to start working on enhancing that.
Regards,
Anthony Liguori

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Anthony Liguori

author

2007-03-29 23:24 UTC #2 lore raw

Hi Nitin,

Thanks for trying out the patch

Quoted 5 lines · Nitin A

Hi Anthony,
I tried your patch at my end with snapshot of xen-unstable on
20070326. And here are my findings.
- The --emulate-16bit option for qemu did not work. qemu is not
accepting this as an valid command line option.

Perhaps you did --emulate-16bit instead of -emulate-16bit? The patch definitely has the added option.

Quoted 2 lines

- I tried hardcoding this to 1 in the qemu code, and found that the
port x595 is not reflecting it correctly to hvmloader.

Are you sure you don't have something stale?

Quoted 7 lines

- Then I tried forcing in the hvmloader to use the emulator instead
of vmxassit, and I see the the emulator is not getting the right context. Following is the console log at this point.
(XEN) HVM3: HVM Loader (XEN) HVM3: Detected Xen v3.0-unstable (XEN) HVM3: Writing SMBIOS tables ... (XEN) HVM3: Loading ROMBIOS ... (XEN) irq.c:210: Dom3 PCI link 0 changed 0 -> 5 (XEN) HVM3: PCI-ISA link 0 routed to IRQ5 (XEN) irq.c:210: Dom3 PCI link 1 changed 0 -> 7 (XEN) HVM3: PCI-ISA link 1 routed to IRQ7 (XEN) irq.c:210: Dom3 PCI link 2 changed 0 -> 10 (XEN) HVM3: PCI-ISA link 2 routed to IRQ10 (XEN) irq.c:210: Dom3 PCI link 3 changed 0 -> 11 (XEN) HVM3: PCI-ISA link 3 routed to IRQ11 (XEN) HVM3: pci dev 01:1 bar 20 size 00000010: 0000c001 (XEN) HVM3: pci dev 01:2 INTA->IRQ7 (XEN) HVM3: pci dev 02:0 bar 10 size 02000000: f0000008 (XEN) HVM3: pci dev 02:0 bar 14 size 00001000: f2000000 (XEN) HVM3: pci dev 03:0 bar 10 size 00000100: 0000c101 (XEN) HVM3: pci dev 03:0 bar 14 size 01000000: f3000008 (XEN) HVM3: pci dev 03:0 INTA->IRQ11 (XEN) HVM3: pci dev 04:0 bar 10 size 00000100: 0000c201 (XEN) HVM3: pci dev 04:0 bar 14 size 00000100: f4000000 (XEN) HVM3: pci dev 04:0 INTA->IRQ5 (XEN) HVM3: Creating MP tables ... (XEN) HVM3: Loading Cirrus VGABIOS ... (XEN) HVM3: Loading ACPI ... (XEN) HVM3: Not Loading VMXAssist ... ffffffff (XEN) HVM3: foo (XEN) hvmop_emulate_realmode (XEN) guest requests real mode emulation (XEN) foo 221 (XEN) HVM3: Invoking ROMBIOS ... (XEN) Transfering control to x86_emulate %eip 0x0 (XEN) hvm.c:446:d3 Triple fault on VCPU0 - invoking HVM system reset.
The log shows that emulator is not getting the right cpu context.
How much of testing have you done with this code? I am not able to proceed to emulation of 1 instruction with the patch. How many instruction could you emulate with the patch?

I suspect you have a partial install or something. I have never seen this happen.

You should see x86_emulate run for a good number of instructions. The odd thing about your log is that the transfering control to x86_emulate message uses the EIP from the vmcs. If this is the first time the emulator is being invoked, how can eip be 0? There's nothing in the emulate path that would touch the vmcs.

How exactly are you forcing the use of the emulator instead of vmxassist?

Regards,

Anthony Liguori

Quoted 47 lines

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.
Quoted 38 lines · Anthony Liguori
Howdy,
Attached is a patch that begins to lay down the infrastructure for emulating real mode with x86_emulate(). With a little more refactoring, I think it could also replace the SVM emulator.
The patch introduces an HVMOP hypercall to set a flag in the hvm vcpu struct to signal that real mode should be emulated with x86_emulate instead of using vm86. This is to make development a little bit easier since x86_emulate is not quite there yet wrt 16 bit emulation. It can be enabled by passing -emulate-16bit to qemu-dm (I use a wrapper script similar to qemu-dm.debug).
The VT code keeps track of the whether it's in the emulator and loops on the do_resume path in x86_emulate. I think this code probably should be refactored into the common HVM code although this would require changing some of the HVM ops. This would allow SVM to use the x86_emulate to handle individual instructions.
There are some issues to work out. Namely, x86_emulate appears to want blocking PIO calls which isn't conducive to the wait PIO works today in HVM. This is only a problem for instructions at the moment. I'm also a bit confused about how to properly loop in the emulator. schedule_tail is not meant to return so perhaps we should loop on emulating == 1 instead of hypercall_preempt_check? I didn't think the hypervisor was preemptable though.
The current code doesn't handle non-flat segments as I don't think hvm_copy_from/to_guest handles it (which I assume it would need to).
However, it is enough to start running instructions in x86_emulate so it's enough to start working on enhancing that.
Regards,
Anthony Liguori

Kamble, Nitin A

2007-03-29 23:52 UTC #3 lore raw

Hi Anthony,

Quoted 3 lines

Perhaps you did --emulate-16bit instead of -emulate-16bit? The patch definitely has the added option.

You are right, it was -emulate-16bit.

Quoted 4 lines

Quoted 2 lines
- I tried hardcoding this to 1 in the qemu code, and found that the
port x595 is not reflecting it correctly to hvmloader.
Are you sure you don't have something stale?

I started with a fresh tree for your patch. I also tried rebuilding the quemu from start.

Quoted 2 lines

I suspect you have a partial install or something. I have never seen this happen.

I have already tried few times. i will try one more time with newer fresh xen-unstable tree. Some of the debug messages were coming from my prinks.

Quoted 9 lines

You should see x86_emulate run for a good number of instructions. The odd thing about your log is that the transfering control to x86_emulate message uses the EIP from the vmcs. If this is the first time the emulator is being invoked, how can eip be 0? There's nothing in the emulate path that would touch the vmcs.
How exactly are you forcing the use of the emulator instead of vmxassist?

I basically forced emulate_16bit varilable in qemu to 1.

int acpi_enabled = 0;
int fd_bootchk = 1;
int emulate_16bit = 1;

and in the hvmloader instead of checking the output of the port 595, I replaced it with if (1).
like this:

#if 0
        if (inl(0x595) == 0xdeadbeef) {
#else
        if (1) {

#endif

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Anthony Liguori

author

2007-03-30 00:11 UTC #4 lore raw

Quoted 4 lines · Nitin A

Hi Anthony,
Quoted 3 lines
Perhaps you did --emulate-16bit instead of -emulate-16bit? The patch definitely has the added option.

Quoted 8 lines

You are right, it was -emulate-16bit.
Quoted 5 lines
Quoted 2 lines
- I tried hardcoding this to 1 in the qemu code, and found that the
port x595 is not reflecting it correctly to hvmloader.
Are you sure you don't have something stale?
I started with a fresh tree for your patch. I also tried rebuilding the quemu from start.

Okay, you'll need to make sure that hvmloader and qemu are both being rebuilt and reinstalled.

Quoted 14 lines

Quoted 2 lines
I suspect you have a partial install or something. I have never seen this happen.
I have already tried few times. i will try one more time with newer fresh xen-unstable tree. Some of the debug messages were coming from my prinks.
Quoted 5 lines
You should see x86_emulate run for a good number of instructions. The odd thing about your log is that the transfering control to x86_emulate message uses the EIP from the vmcs. If this is the first time the emulator is being invoked, how can eip be 0? There's nothing in the emulate path that would touch the vmcs.
How exactly are you forcing the use of the emulator instead of vmxassist?
I basically forced emulate_16bit varilable in qemu to 1.
int acpi_enabled = 0;
int fd_bootchk = 1;
int emulate_16bit* = 1;*

Quoted 7 lines

and in the hvmloader instead of checking the output of the port 595, I replaced it with if (1). like this:
#if 0
        if (inl(0x595) == 0xdeadbeef) {
#else
* if (1) {*
#endif

The really curious thing is that eip from the VMCS is set to 0. eip should be address of the EIP of whatever movl cr0 instruction that occurs when returning from main() in hvmloader.

I guess it's possible that if you are using a version of unstable that predates Keir's recent hvmloader changes, you wouldn't have hvmloader switching to real mode. That could explain some of the weirdness you're seeing. What changeset are you on?

Regards,

Anthony Liguori

Quoted 7 lines

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

Kamble, Nitin A

2007-03-30 00:59 UTC #5 lore raw

Quoted 3 lines · Anthony Liguori

Okay, you'll need to make sure that hvmloader and qemu are both being rebuilt and reinstalled.

I am sure both are built with the patch and installed correctly.

Quoted 4 lines

The really curious thing is that eip from the VMCS is set to 0. eip should be address of the EIP of whatever movl cr0 instruction that occurs when returning from main() in hvmloader.

I also find that x86_emulate() is never called while emulate_realmode is set in the hypervisor.

Quoted 4 lines

I guess it's possible that if you are using a version of unstable that predates Keir's recent hvmloader changes, you wouldn't have hvmloader switching to real mode. That could explain some of the weirdness you're seeing. What changeset are you on?

So far I am using: r14560_20070326

Now I am trying with: r14631_20070329

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Kamble, Nitin A

2007-03-30 02:20 UTC #6 lore raw

Hi Anthony,
With the new revision I am able to get the -emulate-16bit command line
switch working.

(XEN) HVM2: Creating MP
tables ...
(XEN) HVM2: Loading Cirrus
VGABIOS ...
(XEN) HVM2: Loading
ACPI ...
(XEN) HVM2: Loading VMXAssist ...
deadbeef
(XEN) HVM2:
foo
(XEN)
hvmop_emulate_realmode
(XEN) guest requests real mode
emulation
(XEN) foo
221
(XEN) HVM2: Invoking
ROMBIOS ...
(XEN) hvm.c:446:d2 Triple fault on VCPU0 - invoking HVM system reset.

I have not added any debug lines in the code yet, and I don't know how far is the emulator going. I will check it tomorrow.

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

Quoted 26 lines · Nitin A Kamble

Quoted 3 lines · Anthony Liguori
Okay, you'll need to make sure that hvmloader and qemu are both being rebuilt and reinstalled.
I am sure both are built with the patch and installed correctly.
Quoted 4 lines
The really curious thing is that eip from the VMCS is set to 0. eip should be address of the EIP of whatever movl cr0 instruction that occurs when returning from main() in hvmloader.
I also find that x86_emulate() is never called while emulate_realmode is set in the hypervisor.
Quoted 4 lines
I guess it's possible that if you are using a version of unstable that predates Keir's recent hvmloader changes, you wouldn't have hvmloader switching to real mode. That could explain some of the weirdness you're seeing. What changeset are you on?
So far I am using: r14560_20070326

Quoted 7 lines

Now I am trying with: r14631_20070329
Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Anthony Liguori

author

2007-03-30 03:20 UTC #7 lore raw

Quoted 5 lines · Nitin A

Hi Anthony,
With the new revision I am able to get the -emulate-16bit command
line switch working.
(XEN) HVM2: Creating MP tables ... (XEN) HVM2: Loading Cirrus VGABIOS ... (XEN) HVM2: Loading ACPI ... (XEN) HVM2: Loading VMXAssist ... deadbeef (XEN) HVM2: foo (XEN) hvmop_emulate_realmode (XEN) guest requests real mode emulation (XEN) foo 221 (XEN) HVM2: Invoking ROMBIOS ... (XEN) hvm.c:446:d2 Triple fault on VCPU0 - invoking HVM system reset.

The Triple fault you're seeing here is terribly curious. Also the "deadbeef" output. Just to sanity check, I threw the following printk in vmcs.c

    while (!hypercall_preempt_check()) {
+        printk("eip = 0x%x\n", regs->eip);
        if (x86_emulate(&ctxt, &em_ops)) {

And I get the following output with a FC5 guest:

(XEN) hvmop_emulate_realmode (XEN) guest requests real mode emulation (XEN) foo 221 (XEN) eip = 0xd338d (XEN) eip = 0xd338e (XEN) eip = 0xffbf0000 (XEN) failed to emulate instruction at %eip = 0xd338d (XEN) domain_crash_sync called from vmcs.c:625 (XEN) Domain 1 (vcpu#0) crashed on cpu#0: (XEN) ----[ Xen-3.0-unstable x86_32 debug=n Not tainted ]---- (XEN) CPU: 0 (XEN) EIP: 0010:[<000d338d>] (XEN) EFLAGS: 00000002 CONTEXT: hvm (XEN) eax: 00000076 ebx: 000d7324 ecx: 000d7324 edx: 000000e9 (XEN) esi: 000d4e54 edi: 000d3380 ebp: 000d72a8 esp: 000d72a8 (XEN) cr0: 00050032 cr4: 00000651 cr3: 00000000 cr2: 00000000 (XEN) ds: 0000 es: 0000 fs: 0000 gs: 0000 ss: 0018 cs: 0010

So, perhaps it's the guest you're using? Clearly, we're running in x86_emulate and hitting a 16 bit instruction we can't handle. N.B. the printk in the error path for x86_emulate is wrong. I should be looking at regs->eip, not GUEST_RIP since that wouldn't have been updated again.

Regards,

Anthony Liguori

Quoted 35 lines

I have not added any debug lines in the code yet, and I don't know how far is the emulator going. I will check it tomorrow.
Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.
Quoted 24 lines · Nitin A Kamble
Quoted 3 lines · Anthony Liguori
Okay, you'll need to make sure that hvmloader and qemu are both being rebuilt and reinstalled.
I am sure both are built with the patch and installed correctly.
Quoted 3 lines
The really curious thing is that eip from the VMCS is set to 0. eip should be address of the EIP of whatever movl cr0 instruction that occurs when returning from main() in hvmloader.
I also find that x86_emulate() is never called while emulate_realmode is set in the hypervisor.
Quoted 2 lines
I guess it's possible that if you are using a version of unstable that predates Keir's recent hvmloader changes, you wouldn't have hvmloader switching to real mode. That could explain some of the weirdness you're seeing. What changeset are you on?
So far I am using: r14560_20070326
Now I am trying with: r14631_20070329
Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

Anthony Liguori

author

2007-03-30 03:55 UTC #8 lore raw

Ugh! Sorry that got munged. Let me try again:

(XEN) hvmop_emulate_realmode
(XEN) guest requests real mode emulation
(XEN) foo 221
(XEN) eip = 0xd338d
(XEN) eip = 0xd338e
(XEN) eip = 0xffbf0000
(XEN) failed to emulate instruction at %eip = 0xd338d (XEN) domain_crash_sync called from vmcs.c:625
(XEN) Domain 1 (vcpu#0) crashed on cpu#0:
(XEN) ----[ Xen-3.0-unstable x86_32 debug=n Not tainted ]----
(XEN) CPU: 0
(XEN) EIP: 0010:[<000d338d>]
(XEN) EFLAGS: 00000002 CONTEXT: hvm

(XEN) eax: 00000076   ebx: 000d7324   ecx: 000d7324   edx: 000000e9
(XEN) esi: 000d4e54   edi: 000d3380   ebp: 000d72a8   esp: 000d72a8
(XEN) cr0: 00050032   cr4: 00000651   cr3: 00000000   cr2: 00000000

(XEN) ds: 0000 es: 0000 fs: 0000 gs: 0000 ss: 0018 cs: 0010

Regards,

Anthony Liguori

Quoted 65 lines · Anthony Liguori

Quoted 5 lines · Nitin A
Hi Anthony,
With the new revision I am able to get the -emulate-16bit command
line switch working.
(XEN) HVM2: Creating MP tables ... (XEN) HVM2: Loading Cirrus VGABIOS ... (XEN) HVM2: Loading ACPI ... (XEN) HVM2: Loading VMXAssist ... deadbeef (XEN) HVM2: foo (XEN) hvmop_emulate_realmode (XEN) guest requests real mode emulation (XEN) foo 221 (XEN) HVM2: Invoking ROMBIOS ... (XEN) hvm.c:446:d2 Triple fault on VCPU0 - invoking HVM system reset.
The Triple fault you're seeing here is terribly curious. Also the "deadbeef" output. Just to sanity check, I threw the following printk in vmcs.c
    while (!hypercall_preempt_check()) {
+        printk("eip = 0x%x\n", regs->eip);
        if (x86_emulate(&ctxt, &em_ops)) {
And I get the following output with a FC5 guest:
(XEN) hvmop_emulate_realmode (XEN) guest requests real mode emulation (XEN) foo 221 (XEN) eip = 0xd338d (XEN) eip = 0xd338e (XEN) eip = 0xffbf0000 (XEN) failed to emulate instruction at %eip = 0xd338d (XEN) domain_crash_sync called from vmcs.c:625 (XEN) Domain 1 (vcpu#0) crashed on cpu#0: (XEN) ----[ Xen-3.0-unstable x86_32 debug=n Not tainted ]---- (XEN) CPU: 0 (XEN) EIP: 0010:[<000d338d>] (XEN) EFLAGS: 00000002 CONTEXT: hvm (XEN) eax: 00000076 ebx: 000d7324 ecx: 000d7324 edx: 000000e9 (XEN) esi: 000d4e54 edi: 000d3380 ebp: 000d72a8 esp: 000d72a8 (XEN) cr0: 00050032 cr4: 00000651 cr3: 00000000 cr2: 00000000 (XEN) ds: 0000 es: 0000 fs: 0000 gs: 0000 ss: 0018 cs: 0010 So, perhaps it's the guest you're using? Clearly, we're running in x86_emulate and hitting a 16 bit instruction we can't handle. N.B. the printk in the error path for x86_emulate is wrong. I should be looking at regs->eip, not GUEST_RIP since that wouldn't have been updated again.
Regards,
Anthony Liguori
Quoted 7 lines
I have not added any debug lines in the code yet, and I don't know how far is the emulator going. I will check it tomorrow.
Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
Quoted 30 lines
The mind is like a parachute; it works much better when it's open.
Quoted 25 lines · Nitin A Kamble
Quoted 3 lines · Anthony Liguori
Okay, you'll need to make sure that hvmloader and qemu are both being rebuilt and reinstalled.
I am sure both are built with the patch and installed correctly.
Quoted 3 lines
The really curious thing is that eip from the VMCS is set to 0. eip should be address of the EIP of whatever movl cr0 instruction that occurs when returning from main() in hvmloader.
I also find that x86_emulate() is never called while emulate_realmode is set in the hypervisor.
Quoted 2 lines
I guess it's possible that if you are using a version of unstable that predates Keir's recent hvmloader changes, you wouldn't have hvmloader switching to real mode. That could explain some of the weirdness you're seeing. What changeset are you on?
So far I am using: r14560_20070326
Now I am trying with: r14631_20070329
Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Kamble, Nitin A

2007-03-30 18:53 UTC #9 lore raw

In reply to Anthony Liguori

Quoted 7 lines · Anthony Liguori

Quoted 2 lines
...
(XEN) hvm.c:446:d2 Triple fault on VCPU0 - invoking HVM system reset.
The Triple fault you're seeing here is terribly curious. Also the "deadbeef" output. Just to sanity check, I threw the following printk in vmcs.c

Anthony,
To me the tripple fault makes sense.

Your patch enables emulation only when "emulating" is set to 1.

void arch_vmx_do_resume(struct vcpu *v)

 {
     if ( v->arch.hvm_vmx.active_cpu == smp_processor_id() )
@@ -508,7 +644,11 @@ void arch_vmx_do_resume(struct vcpu *v)
     }
 
     hvm_do_resume(v);

- reset_stack_and_jump(vmx_asm_do_vmentry);
+
+ if (v->arch.hvm_vmx.emulating)
+ vmx_do_emulate(v);
+ else
+ reset_stack_and_jump(vmx_asm_do_vmentry);
}

And it is turned on only when guest (hvmloader) sets up CR0.

-static int vmx_set_cr0(unsigned long value)
+int vmx_set_cr0(unsigned long value)

 {
     struct vcpu *v = current;
     unsigned long mfn;
@@ -1982,13 +1982,29 @@ static int vmx_set_cr0(unsigned long val
             }
         }

-        if ( vmx_assist(v, VMX_ASSIST_INVOKE) )
+        if ( v->arch.hvm_vcpu.emulate_realmode )
+        {

+ eip = __vmread(GUEST_RIP);
+ HVM_DBG_LOG(DBG_LEVEL_1,
+ "Transfering control to x86_emulate %%eip 0x%lx
\n", eip);
+ v->arch.hvm_vmx.emulating = 1;
+ return 1;
+ }
+ else if ( vmx_assist(v, VMX_ASSIST_INVOKE) )

And I don't see any code in the hvmloader for setting cr0 before returning from the main.

So the code flow is returning from main, which is causing the tripple fault.

I observe the vmx_do_emulate is never getting called.

I think set cr0 instruction is needed just after the emulate_realmode hypercall in the hvmloader code.

Have you added more code lateron after sending the patch out?

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

Quoted 40 lines

     while (!hypercall_preempt_check()) {
+        printk("eip = 0x%x\n", regs->eip);
         if (x86_emulate(&ctxt, &em_ops)) {
And I get the following output with a FC5 guest:
(XEN)
hvmop_emulate_realmode
(XEN) guest requests real mode
emulation
(XEN) foo
221
(XEN) eip =
0xd338d
(XEN) eip =
0xd338e
(XEN) eip =
0xffbf0000
(XEN) failed to emulate instruction at %eip = 0xd338d
(XEN) domain_crash_sync called from
vmcs.c:625
(XEN) Domain 1 (vcpu#0) crashed on
cpu#0:
(XEN) ----[ Xen-3.0-unstable x86_32 debug=n Not tainted ]----
(XEN) CPU:
0
(XEN) EIP:
0010:[<000d338d>]
(XEN) EFLAGS: 00000002 CONTEXT:
hvm
(XEN) eax: 00000076 ebx: 000d7324 ecx: 000d7324 edx:
000000e9
(XEN) esi: 000d4e54 edi: 000d3380 ebp: 000d72a8 esp:
000d72a8
(XEN) cr0: 00050032 cr4: 00000651 cr3: 00000000 cr2:
00000000
(XEN) ds: 0000 es: 0000 fs: 0000 gs: 0000 ss: 0018 cs: 0010

Quoted 8 lines

So, perhaps it's the guest you're using? Clearly, we're running in x86_emulate and hitting a 16 bit instruction we can't handle. N.B. the printk in the error path for x86_emulate is wrong. I should be looking at regs->eip, not GUEST_RIP since that wouldn't have been updated again.
Regards,
Anthony Liguori

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Anthony Liguori

author

2007-03-30 19:00 UTC #10 lore raw

Quoted 53 lines · Nitin A

Quoted 5 lines · Anthony Liguori
Quoted 1 line
... (XEN) hvm.c:446:d2 Triple fault on VCPU0 - invoking HVM system reset.
The Triple fault you're seeing here is terribly curious. Also the "deadbeef" output. Just to sanity check, I threw the following printk in vmcs.c
Anthony,
To me the tripple fault makes sense.
Your patch enables emulation only when "emulating" is set to 1.
void arch_vmx_do_resume(struct vcpu *v)
{
     if ( v->arch.hvm_vmx.active_cpu == smp_processor_id() )
(quoted diff) +5 -1
@@ -508,7 +644,11 @@ void arch_vmx_do_resume(struct vcpu *v)
     }
 
     hvm_do_resume(v);
-    reset_stack_and_jump(vmx_asm_do_vmentry);
+
+    if (v->arch.hvm_vmx.emulating)
+        vmx_do_emulate(v);
+    else
+        reset_stack_and_jump(vmx_asm_do_vmentry);
 }
 
 
And it is turned on only when guest (hvmloader) sets up CR0.
(quoted diff) +1 -1
-static int vmx_set_cr0(unsigned long value)
+int vmx_set_cr0(unsigned long value)
{
     struct vcpu *v = current;
     unsigned long mfn;
(quoted diff) +9 -1
@@ -1982,13 +1982,29 @@ static int vmx_set_cr0(unsigned long val
             }
         }
 
-        if ( vmx_assist(v, VMX_ASSIST_INVOKE) )
+        if ( v->arch.hvm_vcpu.emulate_realmode )
+        {
+            eip = __vmread(GUEST_RIP);
+            HVM_DBG_LOG(DBG_LEVEL_1,
+                        "Transfering control to x86_emulate %%eip 
 0x%lx\n", eip);
+            v->arch.hvm_vmx.emulating = 1;
+            return 1;
+        }
+        else if ( vmx_assist(v, VMX_ASSIST_INVOKE) )
         {
 

Quoted 1 line

And I don't see any code in the hvmloader for setting cr0 before returning from the main.

My tip is 14462:3fd9b0c71b8c and in hvmloader.c, there is:

asm(

   "    .text                       \n"
   "    .globl _start               \n"
   "_start:                         \n"
   /* C runtime kickoff. */
   "    cld                         \n"
   "    cli                         \n"
   "    movl $stack_top,%esp        \n"
   "    movl %esp,%ebp              \n"
   "    call main                   \n"
   /* Relocate real-mode trampoline to 0x0. */
   "    mov  $trampoline_start,%esi \n"
   "    xor  %edi,%edi              \n"
   "    mov  $trampoline_end,%ecx   \n"
   "    sub  %esi,%ecx              \n"
   "    rep  movsb                  \n"
<snip>
   /* Enter real mode, reload all segment registers and IDT. */
   "    ljmp $0x8,$0x0              \n"
   "trampoline_start: .code16       \n"
   "    mov  %eax,%cr0              \n"

That change was pretty recent so unless it was reverted it should Just Work.

Regards,

Anthony Liguori

Quoted 31 lines

So the code flow is returning from main, which is causing the tripple fault.
I observe the vmx_do_emulate is never getting called.
I think set cr0 instruction is needed just after the emulate_realmode hypercall in the hvmloader code.
Have you added more code lateron after sending the patch out?
Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.
Quoted 14 lines
     while (!hypercall_preempt_check()) {
+        printk("eip = 0x%x\n", regs->eip);
         if (x86_emulate(&ctxt, &em_ops)) {
And I get the following output with a FC5 guest:
(XEN) hvmop_emulate_realmode (XEN) guest requests real mode emulation (XEN) foo 221 (XEN) eip = 0xd338d (XEN) eip = 0xd338e (XEN) eip = 0xffbf0000 (XEN) failed to emulate instruction at %eip = 0xd338d (XEN) domain_crash_sync called from vmcs.c:625 (XEN) Domain 1 (vcpu#0) crashed on cpu#0: (XEN) ----[ Xen-3.0-unstable x86_32 debug=n Not tainted ]---- (XEN) CPU: 0 (XEN) EIP: 0010:[<000d338d>] (XEN) EFLAGS: 00000002 CONTEXT: hvm (XEN) eax: 00000076 ebx: 000d7324 ecx: 000d7324 edx: 000000e9 (XEN) esi: 000d4e54 edi: 000d3380 ebp: 000d72a8 esp: 000d72a8 (XEN) cr0: 00050032 cr4: 00000651 cr3: 00000000 cr2: 00000000 (XEN) ds: 0000 es: 0000 fs: 0000 gs: 0000 ss: 0018 cs: 0010
So, perhaps it's the guest you're using? Clearly, we're running in x86_emulate and hitting a 16 bit instruction we can't handle. N.B. the printk in the error path for x86_emulate is wrong. I should be looking at regs->eip, not GUEST_RIP since that wouldn't have been updated again.
Regards,
Anthony Liguori

Kamble, Nitin A

2007-03-30 21:08 UTC #11 lore raw

Hi Anthony,

Quoted 10 lines · Anthony Liguori

/* Enter real mode, reload all segment registers and IDT. */
" ljmp $0x8,$0x0 \n"
"trampoline_start: .code16 \n"
" mov %eax,%cr0 \n"
That change was pretty recent so unless it was reverted it should Just Work.
Regards,
Anthony Liguori

Now I see cr0 is updated after returning from main.
I enabled the debug log in the xen code now.

(XEN) HVM2: Creating MP
tables ...
(XEN) HVM2: Loading Cirrus
VGABIOS ...
(XEN) HVM2: Loading
ACPI ...
(XEN) HVM2: Loading VMXAssist ...
deadbeef
(XEN) HVM2:
foo
(XEN)
hvmop_emulate_realmode
(XEN) guest requests real mode
emulation
(XEN) foo
221
(XEN) HVM2: Invoking
ROMBIOS ...
(XEN) vmx_vmexit_handler called. eip =
0x0
(XEN) vmx_cr_access called
eip=0x0
(XEN) mov_to_cr 0 called
eip=0x0
(XEN) vmx_set_cr0 called
eip=0x0
(XEN) Transfering -- control to x86_emulate eip
0x0
(XEN) hvm.c:446:d2 Triple fault on VCPU0 - invoking HVM system reset.

It shows cr0 is getting modified. But the eip is still 0x0.

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Anthony Liguori

author

2007-03-30 21:20 UTC #21 lore raw

In reply to Andi Kleen

Quoted 11 lines · Andi Kleen

Quoted 4 lines · Anthony Liguori
Attached is a patch that begins to lay down the infrastructure for emulating real mode with x86_emulate(). With a little more refactoring, I think it could also replace the SVM emulator.
I thought the roadmap direction of this was to migrate the context into qemu and let its JIT handle it? Or has that been dropped?

There are some hard problems with migrating into QEMU. Namely, the code that QEMU generates isn't SMP safe (atomic instructions lose their atomicity).

Regards,

Anthony Liguori

Quoted 2 lines

-Andi

Kamble, Nitin A

2007-03-30 21:24 UTC #12 lore raw

In reply to Kamble, Nitin A

Anthony,
eip of 0x0 is the right address.

Because the trampoline_Start code is relocated to address 0.

/* Relocate real-mode trampoline to 0x0. */
" mov $trampoline_start,%esi \n"
" xor %edi,%edi \n"
" mov $trampoline_end,%ecx \n"
" sub %esi,%ecx \n"
" rep movsb \n"

Still there is no progress in the emulation. The guest is dying with triple-fault just like before.

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

Quoted 49 lines · Nitin A Kamble

Hi Anthony,
Quoted 10 lines · Anthony Liguori
/* Enter real mode, reload all segment registers and IDT. */
" ljmp $0x8,$0x0 \n"
"trampoline_start: .code16 \n"
" mov %eax,%cr0 \n"
That change was pretty recent so unless it was reverted it should Just Work.
Regards,
Anthony Liguori
Now I see cr0 is updated after returning from main.
I enabled the debug log in the xen code now.
(XEN) HVM2: Creating MP
tables ...
(XEN) HVM2: Loading Cirrus
VGABIOS ...
(XEN) HVM2: Loading
ACPI ...
(XEN) HVM2: Loading VMXAssist ...
deadbeef
(XEN) HVM2:
foo
(XEN)
hvmop_emulate_realmode
(XEN) guest requests real mode
emulation
(XEN) foo
221
(XEN) HVM2: Invoking
ROMBIOS ...
(XEN) vmx_vmexit_handler called. eip =
0x0
(XEN) vmx_cr_access called
eip=0x0
(XEN) mov_to_cr 0 called
eip=0x0
(XEN) vmx_set_cr0 called
eip=0x0
(XEN) Transfering -- control to x86_emulate eip
0x0
(XEN) hvm.c:446:d2 Triple fault on VCPU0 - invoking HVM system reset.

Quoted 8 lines

It shows cr0 is getting modified. But the eip is still 0x0.
Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Andi Kleen

2007-03-30 21:25 UTC #22 lore raw

In reply to Anthony Liguori

Quoted 3 lines

There are some hard problems with migrating into QEMU. Namely, the code that QEMU generates isn't SMP safe (atomic instructions lose their atomicity)

I see.

You're worrying that real mode startup trampolines of OS rely on atomic instructions? I can't think of much other possible real mode multi CPU code.

-Andi

Anthony Liguori

author

2007-03-30 22:00 UTC #23 lore raw

Quoted 8 lines · Andi Kleen

Quoted 2 lines
There are some hard problems with migrating into QEMU. Namely, the code that QEMU generates isn't SMP safe (atomic instructions lose their atomicity)
I see.
You're worrying that real mode startup trampolines of OS rely on atomic instructions? I can't think of much other possible real mode multi CPU code.

No, that's a concern for using V2E to handle 32 bit code. The big problem for handling real mode code is dealing with synchronization for device state. With the devices split between QEMU and within the hypervisor, this becomes even more complicated.

Any of the interesting guests that use big real mode also depend on being able to synchronize this device state.

Regards,

Anthony Liguori

Quoted 2 lines

-Andi

Andi Kleen

2007-03-30 22:10 UTC #20 lore raw

In reply to Anthony Liguori

Quoted 3 lines · Anthony Liguori

Attached is a patch that begins to lay down the infrastructure for emulating real mode with x86_emulate(). With a little more refactoring, I think it could also replace the SVM emulator.

I thought the roadmap direction of this was to migrate the context into qemu and let its JIT handle it? Or has that been dropped?

-Andi

Anthony Liguori

author

2007-03-30 22:11 UTC #13 lore raw

In reply to Kamble, Nitin A

Quoted 14 lines · Nitin A

Anthony,
eip of 0x0 is the right address.
Because the trampoline_Start code is relocated to address 0.
/* Relocate real-mode trampoline to 0x0. */
" mov $trampoline_start,%esi \n"
" xor %edi,%edi \n"
" mov $trampoline_end,%ecx \n"
" sub %esi,%ecx \n"
" rep movsb \n"
Still there is no progress in the emulation. The guest is dying with triple-fault just like before.

Quoted 30 lines

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.
Quoted 22 lines · Nitin A Kamble
Hi Anthony,
Quoted 11 lines · Anthony Liguori
/* Enter real mode, reload all segment registers and IDT. */
" ljmp $0x8,$0x0 \n"
"trampoline_start: .code16 \n"
" mov %eax,%cr0 \n"
That change was pretty recent so unless it was reverted it should Just Work.
Regards,
Anthony Liguori
Now I see cr0 is updated after returning from main.
I enabled the debug log in the xen code now.
(XEN) HVM2: Creating MP tables ... (XEN) HVM2: Loading Cirrus VGABIOS ... (XEN) HVM2: Loading ACPI ... (XEN) HVM2: Loading VMXAssist ... deadbeef (XEN) HVM2: foo (XEN) hvmop_emulate_realmode
(XEN) guest requests real mode emulation (XEN) foo 221 (XEN) HVM2: Invoking ROMBIOS ... (XEN) vmx_vmexit_handler called. eip = 0x0 (XEN) vmx_cr_access called eip=0x0 (XEN) mov_to_cr 0 called eip=0x0 (XEN) vmx_set_cr0 called eip=0x0 (XEN) Transfering -- control to x86_emulate eip 0x0 (XEN) hvm.c:446:d2 Triple fault on VCPU0 - invoking HVM system reset.
It shows cr0 is getting modified. But the eip is still 0x0.

set_cr0 is returning 1 though which should increment eip to the next instruction.

I'm a bit perplexed about my eip now and also why your eip is still 0. It should be the instruction following the mov cr0.

Regards,

Anthony Liguori

Quoted 8 lines

Quoted 7 lines
Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

Kamble, Nitin A

2007-04-02 18:45 UTC #14 lore raw

Quoted 11 lines · Anthony Liguori

set_cr0 is returning 1 though which should increment eip to the next instruction.
I'm a bit perplexed about my eip now and also why your eip is still 0.
It should be the instruction following the mov cr0.
Regards,
Anthony Liguori

Hi Anthony,
I don't see any code doing context save/restore like
vmx_world_save , vmx_world_restore in the current code for the hyperviser based emulator.
Also the function arch_vmx_do_resume() is called at the time of vcpu schedule, so it is not right to call the vmx_do_emulate() from there.

The code would need vmx_world_save/restore in the code path before returning to vmx_asm_vmexit_handler from vmx_vmexit_handler. Without that I don't see it can emulate any instructions.

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Anthony Liguori

author

2007-04-02 18:54 UTC #15 lore raw

Quoted 7 lines · Nitin A

Quoted 5 lines · Anthony Liguori
set_cr0 is returning 1 though which should increment eip to the next instruction.
I'm a bit perplexed about my eip now and also why your eip is still 0. It should be the instruction following the mov cr0.

Quoted 8 lines

Quoted 4 lines
Regards,
Anthony Liguori
Hi Anthony,
I don't see any code doing context save/restore like
vmx_world_save , vmx_world_restore in the current code for the hyperviser based emulator.

Before calling x86_emulate, we use hvm_store_cpu_guest_regs() to copy the guest state into a regs struct (which happens to be the vcpu's reg struct). This reads directly from the VMCS via vmread() so it should be okay. I don't think a vmx_world_save/restore is actually needed although perhaps I'm missing something?

Quoted 1 line

Also the function arch_vmx_do_resume() is called at the time of vcpu schedule, so it is not right to call the vmx_do_emulate() from there.

Right, the idea was to have x86_emulate() be called instead of vmentry(). I thought that being in the set_cr0 path would ensure that we go through do_resume() again. Is this assumption incorrect?

I didn't want to just stick it in the set_cr0 path because we ought to be able to pull the emulation loop into common code for SVM/VT and the do_resume path seems like the only place where there's common place to hook right now.

Regards,

Anthony Liguori

Quoted 9 lines

The code would need vmx_world_save/restore in the code path before returning to vmx_asm_vmexit_handler from vmx_vmexit_handler. Without that I don't see it can emulate any instructions.
Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

Kamble, Nitin A

2007-04-02 23:52 UTC #16 lore raw

Quoted 6 lines · Anthony Liguori

Before calling x86_emulate, we use hvm_store_cpu_guest_regs() to copy the guest state into a regs struct (which happens to be the vcpu's reg struct). This reads directly from the VMCS via vmread() so it should be
okay. I don't think a vmx_world_save/restore is actually needed although perhaps I'm missing something?

It may be ok to use hvm_store_cpu_guest_regs() for 1st few instructions, but I think it is not complete enough for emulator use.

Quoted 9 lines

Quoted 2 lines
Also the function arch_vmx_do_resume() is called at the time of
vcpu
Quoted 1 line
schedule, so it is not right to call the vmx_do_emulate() from
there.
Right, the idea was to have x86_emulate() be called instead of vmentry(). I thought that being in the set_cr0 path would ensure that we go through do_resume() again. Is this assumption incorrect?

Yes, This assumption is not right. arch_vmx_do_resume() is assigned to schedule tail, so that the vcpu context is saved/restored when another vcpu is scheduled on the physical cpu.

Quoted 5 lines

I didn't want to just stick it in the set_cr0 path because we ought to be able to pull the emulation loop into common code for SVM/VT and the do_resume path seems like the only place where there's common place to hook right now.

I thought the emulator will be needed only for VMX; why is it needed for SVM?

Also calling the x86_emulate() to emulate multiple instructions from vmx_do_resume() will block the physical cpu from other vcpus. I think we discussed the approach of using the non-root context for for emulator within the Xen. Or did I misunderstanding it?

Quoted 4 lines

Regards,
Anthony Liguori

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Keir Fraser

2007-04-03 06:25 UTC #17 lore raw

Quoted 7 lines · Nitin A

Quoted 3 lines
Right, the idea was to have x86_emulate() be called instead of vmentry(). I thought that being in the set_cr0 path would ensure that we go through do_resume() again. Is this assumption incorrect?
Yes, This assumption is not right. arch_vmx_do_resume() is assigned to schedule tail, so that the vcpu context is saved/restored when another vcpu is scheduled on the physical cpu.

If you want to keep the check off the Œfast¹ return path (which actually isn¹t all that fast as yet, sadly) then you can kick the SCHEDULE_SOFTIRQ from the set_cr0 path. This will take you the really slow path, through the scheduler, but will ultimately guarantee that you go through arch_vmx_do_resume() before returning to guest context.

Quoted 7 lines

Quoted 5 lines
I didn't want to just stick it in the set_cr0 path because we ought to be able to pull the emulation loop into common code for SVM/VT and the do_resume path seems like the only place where there's common place to hook right now.
I thought the emulator will be needed only for VMX; why is it needed for SVM?

It would be really nice to make as much of this infrastructure as possible HVM-generic so that any HVM implementation can fall back to emulation to handle corner cases. In SVM case this may just be a few tricky instructions (SMSW springs to mind), but we¹d like to share code if possible.

Quoted 5 lines

Also calling the x86_emulate() to emulate multiple instructions from vmx_do_resume() will block the physical cpu from other vcpus. I think we discussed the approach of using the non-root context for for emulator within the Xen. Or did I misunderstanding it?

I think you did. It¹s sufficient to check for softirq work at emulated-instruction boundaries.

-- Keir

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Anthony Liguori

author

2007-04-03 14:03 UTC #18 lore raw

In reply to Kamble, Nitin A

Quoted 7 lines · Nitin A

Quoted 5 lines · Anthony Liguori
Before calling x86_emulate, we use hvm_store_cpu_guest_regs() to copy the guest state into a regs struct (which happens to be the vcpu's reg struct). This reads directly from the VMCS via vmread() so it should be okay. I don't think a vmx_world_save/restore is actually needed although perhaps I'm missing something?
It may be ok to use hvm_store_cpu_guest_regs() for 1st few instructions, but I think it is not complete enough for emulator use.

What is missing? x86_emulate() only uses info in the regs (it calls out to function pointers for special registers). The GP registers should be kept up-to-date on vmexit and hvm_store_cpu_guest_regs() should sync the remainder of the register state.

Is there a specific item you think is missing?

Quoted 8 lines

Quoted 7 lines
Quoted 3 lines
Also the function arch_vmx_do_resume() is called at the time of vcpu schedule, so it is not right to call the vmx_do_emulate() from there.
Right, the idea was to have x86_emulate() be called instead of vmentry(). I thought that being in the set_cr0 path would ensure that we go through do_resume() again. Is this assumption incorrect?
Yes, This assumption is not right. arch_vmx_do_resume() is assigned to schedule tail, so that the vcpu context is saved/restored when another vcpu is scheduled on the physical cpu.

Hrm, okay. Manually invoking the scheduler is probably a reasonable place to start. It would be nice to clean things up though so that wasn't necessary.

Quoted 6 lines

Quoted 5 lines
I didn't want to just stick it in the set_cr0 path because we ought to be able to pull the emulation loop into common code for SVM/VT and the do_resume path seems like the only place where there's common place to hook right now.
I thought the emulator will be needed only for VMX; why is it needed for SVM?

As Keir mentioned, there are some corner cases where emulation is needed. Also, there is some opportunity to simplify things by using the emulator. For instance, instead of decoding a PIO instruction using the info in the VMCS/VMCB (none of which is actually common to VT/SVM), we may find that it is simpler to just call out to x86_emulate() and let it decode and dispatch the PIO operation.

In fact, a large number of the exits can be handled in this way. I have no clue if this would impact performance in a significant way but it would definitely simplify things.

Quoted 1 line

Also calling the x86_emulate() to emulate multiple instructions from vmx_do_resume() will block the physical cpu from other vcpus.

That's what the hypercall_preempt_check() is for.

Quoted 1 line

I think we discussed the approach of using the non-root context for for emulator within the Xen. Or did I misunderstanding it?

We discussed quite a few approaches :-) I thought we settled on doing the emulation within Xen. I'm not sure what would be gained from a non-root context other than better security assurances.

Regards,

Anthony Liguori

Quoted 11 lines

Quoted 4 lines
Regards,
Anthony Liguori
Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

Kamble, Nitin A

2007-04-05 01:24 UTC #19 lore raw

Hi Anthony, Keir,
As per your suggestion I added the SCHEDULE_SOFTIRQ in the set_cr0
path. And now I see the code path going to vmx_do_emulate() for emulation. I added the emulation code for ljmp and it is able to go forward 1 instruction now.

(XEN) HVM2: Loading Cirrus
VGABIOS ...
(XEN) HVM2: Loading
ACPI ...
(XEN) HVM2: Loading VMXAssist ...
deadbeef
(XEN) HVM2:
foo
(XEN)
hvmop_emulate_realmode
(XEN) guest requests real mode
emulation
(XEN) foo
221
(XEN) HVM2: Invoking
ROMBIOS ...
(XEN) vmx_vmexit_handler called. eip =
0x0
(XEN) vmx_cr_access called
eip=0x0
(XEN) mov_to_cr 0 called
eip=0x0
(XEN) vmx_set_cr0 called
eip=0x0
(XEN) Transfering -- control to x86_emulate eip 0x0
(XEN) cr access done: updating eip from eip = 0x0
(XEN) updated
eip=0x3
(XEN) arch_vmx_do_resume: starting emulation
(XEN) vmx_do_emulate:
hvm_store_cpu_guest_regs
(XEN) vmx_do_emulate: hypercall_preempt_check() = 0
(XEN) vmx_do_emulate: cs:eip =
0008:0003
(XEN) x86_emulate realmode:
0008:0003
(XEN) jmp far seg:offset (Xea)
handled
(XEN) vmx_do_emulate: cs:eip =
0000:0008
(XEN) x86_emulate realmode:
0000:0008
(XEN) x86_emulate.c:2441:d2 Instr:
8e
(XEN) failed to emulate instruction at eip = 0x8
(XEN) domain_crash_sync called from vmcs.c:625
(XEN) Domain 2 (vcpu#0) crashed on
cpu#0:
(XEN) ----[ Xen-3.0-unstable x86_32p debug=n Not tainted ]----
(XEN) CPU:
0
(XEN) EIP:
0008:[<00000003>]
(XEN) EFLAGS: 00000046 CONTEXT:
hvm
(XEN) eax: 00000000 ebx: 00000000 ecx: 00000000 edx:
00000000
(XEN) esi: 00000000 edi: 00000000 ebp: 00000000 esp:
00000000
(XEN) cr0: 00000000 cr4: 00000650 cr3: 00000000 cr2:
00000000
(XEN) ds: 0000 es: 0000 fs: 0000 gs: 0000 ss: 0010 cs: 0008

I think getting more instruction emulated will be easy now. I will keep posted about the progress.

Thanks & Regards,
Nitin
Open Source Technology Center, Intel Corporation.
-------------------------------------------------------------------------
The mind is like a parachute; it works much better when it's open.

Quoted 78 lines · Anthony Liguori

Quoted 2 lines · Nitin A
Quoted 1 line · Anthony Liguori
Before calling x86_emulate, we use hvm_store_cpu_guest_regs() to
copy
Quoted 1 line
Quoted 1 line
the guest state into a regs struct (which happens to be the vcpu's
reg
Quoted 1 line
Quoted 1 line
struct). This reads directly from the VMCS via vmread() so it
should be
Quoted 4 lines
Quoted 2 lines
okay. I don't think a vmx_world_save/restore is actually needed although perhaps I'm missing something?
It may be ok to use hvm_store_cpu_guest_regs() for 1st few instructions, but I think it is not complete enough for emulator
use.
What is missing? x86_emulate() only uses info in the regs (it calls out
to function pointers for special registers). The GP registers should be
kept up-to-date on vmexit and hvm_store_cpu_guest_regs() should sync the
remainder of the register state.
Is there a specific item you think is missing?
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Also the function arch_vmx_do_resume() is called at the time of
vcpu
Quoted 1 line
Quoted 1 line
Quoted 1 line
schedule, so it is not right to call the vmx_do_emulate() from
there.
Quoted 3 lines
Quoted 3 lines
Right, the idea was to have x86_emulate() be called instead of vmentry(). I thought that being in the set_cr0 path would ensure
that
Quoted 2 lines
Quoted 1 line
we go through do_resume() again. Is this assumption incorrect?
Yes, This assumption is not right. arch_vmx_do_resume() is assigned
to
Quoted 1 line
schedule tail, so that the vcpu context is saved/restored when
another
Quoted 1 line
vcpu is scheduled on the physical cpu.
Hrm, okay. Manually invoking the scheduler is probably a reasonable place to start. It would be nice to clean things up though so that wasn't necessary.
Quoted 2 lines
Quoted 2 lines
I didn't want to just stick it in the set_cr0 path because we ought
to
Quoted 1 line
Quoted 1 line
be able to pull the emulation loop into common code for SVM/VT and
the
Quoted 1 line
Quoted 1 line
do_resume path seems like the only place where there's common place
to
Quoted 3 lines
Quoted 1 line
hook right now.
I thought the emulator will be needed only for VMX; why is it needed for SVM?
As Keir mentioned, there are some corner cases where emulation is needed. Also, there is some opportunity to simplify things by using the
emulator. For instance, instead of decoding a PIO instruction using the
info in the VMCS/VMCB (none of which is actually common to VT/SVM), we may find that it is simpler to just call out to x86_emulate() and let it
decode and dispatch the PIO operation.
In fact, a large number of the exits can be handled in this way. I have
no clue if this would impact performance in a significant way but it would definitely simplify things.
Quoted 2 lines
Also calling the x86_emulate() to emulate multiple instructions from vmx_do_resume() will block the physical cpu from other vcpus.
That's what the hypercall_preempt_check() is for.
Quoted 2 lines
I think we discussed the approach of using the non-root context for for emulator within the Xen. Or did I misunderstanding it?

Quoted 12 lines

We discussed quite a few approaches :-) I thought we settled on doing the emulation within Xen. I'm not sure what would be gained from a non-root context other than better security assurances.
Regards,
Anthony Liguori
Quoted 4 lines
Quoted 4 lines
Regards,
Anthony Liguori

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel