Skip to contentSkip to conversation

i386 Vmi interface definition

RFC PATCH In review Zachary Amsden 4 participants 6 messages Mar 13, 2006 – Mar 22, 2006

lore
Share & tools
lore
#0 lore raw

RFC PATCH 3/24 i386 Vmi interface definition

Master definition of VMI interface, including calls, constants, and interface version.

include/asm-i386/mach-vmi/paravirtualInterface.h +188 -0
Index: linux-2.6.16-rc5/include/asm-i386/mach-vmi/paravirtualInterface.h
===================================================================
@@ -0,0 +1,188 @@
+/*
+ * paravirtualInterface.h --
+ *
+ *      Header file for paravirtualization interface and definitions
+ *      for the hypervisor option ROM tables.
+ *
+ *      Copyright (C) 2005, VMWare, Inc.
+ *
+ */
+
+#ifndef _PARAVIRTUAL_INTERFACE_H_
+#define _PARAVIRTUAL_INTERFACE_H_
+
+#include "vmiCalls.h"
+
+/*
+ *---------------------------------------------------------------------
+ *
+ *  VMI Option ROM API
+ *
+ *---------------------------------------------------------------------
+ */
+#define VDEF(call) VMI_CALL_##call,
+typedef enum VMICall {
+   VMI_CALLS
+   NUM_VMI_CALLS
+} VMICall;
+#undef VDEF
+
+#define VMI_SIGNATURE 0x696d5663   /* "cVmi" */
+
+#define VMI_API_REV_MAJOR          13
+#define VMI_API_REV_MINOR          0
+
+/* Flags used by VMI_Reboot call */
+#define VMI_REBOOT_SOFT          0x0
+#define VMI_REBOOT_HARD          0x1
+
+/* Flags used by VMI_{Notify|Release}Page call */
+#define VMI_PAGE_PT              0x01
+#define VMI_PAGE_PD              0x02
+#define VMI_PAGE_PAE             0x04
+#define VMI_PAGE_PDP             0x04
+#define VMI_PAGE_PML4            0x08
+
+/* Flags used by VMI_FlushTLB call */
+#define VMI_FLUSH_TLB            0x01
+#define VMI_FLUSH_GLOBAL         0x02
+
+/* Flags used by VMI_FlushSync call */
+#define VMI_FLUSH_PT_UPDATES     0x80
+
+/* The number of VMI address translation slot */
+#define VMI_LINEAR_MAP_SLOTS    4
+
+/* The cycle counters. */
+#define VMI_CYCLES_REAL         0
+#define VMI_CYCLES_AVAILABLE    1
+#define VMI_CYCLES_STOLEN       2
+
+/* The alarm interface 'flags' bits. [TBD: exact format of 'flags'] */
+#define VMI_ALARM_COUNTERS      2
+
+#define VMI_ALARM_COUNTER_MASK  0x000000ff
+
+#define VMI_ALARM_WIRED_IRQ0    0x00000000
+#define VMI_ALARM_WIRED_LVTT    0x00010000
+
+#define VMI_ALARM_IS_ONESHOT    0x00000000
+#define VMI_ALARM_IS_PERIODIC   0x00000100
+
+
+/*
+ *---------------------------------------------------------------------
+ *
+ *  Generic VROM structures and definitions
+ *
+ *---------------------------------------------------------------------
+ */
+
+/* VROM call table definitions */
+#define VROM_CALL_LEN             32
+
+typedef struct VROMCallEntry {
+   char f[VROM_CALL_LEN];
+} VROMCallEntry;
+
+typedef struct VROMHeader {
+   VMI_UINT16          romSignature;             // option ROM signature
+   VMI_INT8            romLength;                // ROM length in 512 byte chunks
+   unsigned char       romEntry[4];              // 16-bit code entry point
+   VMI_UINT8           romPad0;                  // 4-byte align pad
+   VMI_UINT32          vRomSignature;            // VROM identification signature
+   VMI_UINT8           APIVersionMinor;          // Minor version of API
+   VMI_UINT8           APIVersionMajor;          // Major version of API
+   VMI_UINT8           reserved0;                // Reserved for expansion
+   VMI_UINT8           reserved1;                // Reserved for expansion
+   VMI_UINT32          reserved2;                // Reserved for expansion
+   VMI_UINT32          reserved3;                // Reserved for private use
+   VMI_UINT16          pciHeaderOffset;          // Offset to PCI OPROM header
+   VMI_UINT16          pnpHeaderOffset;          // Offset to PnP OPROM header
+   VMI_UINT32          romPad3;                  // PnP reserverd / VMI reserved
+   VROMCallEntry       romCallReserved[3];       // Reserved call slots
+} VROMHeader;
+
+typedef struct VROMCallTable {
+   VROMCallEntry    vromCall[128];           // @ 0x80: ROM calls 4-127
+} VROMCallTable;
+
+/* State needed to start an application processor in an SMP system. */
+typedef struct APState {
+   VMI_UINT32 cr0;
+   VMI_UINT32 cr2;
+   VMI_UINT32 cr3;
+   VMI_UINT32 cr4;
+
+   VMI_UINT64 efer;
+
+   VMI_UINT32 eip;
+   VMI_UINT32 eflags;
+   VMI_UINT32 eax;
+   VMI_UINT32 ebx;
+   VMI_UINT32 ecx;
+   VMI_UINT32 edx;
+   VMI_UINT32 esp;
+   VMI_UINT32 ebp;
+   VMI_UINT32 esi;
+   VMI_UINT32 edi;
+   VMI_UINT16 cs;
+   VMI_UINT16 ss;
+   VMI_UINT16 ds;
+   VMI_UINT16 es;
+   VMI_UINT16 fs;
+   VMI_UINT16 gs;
+   VMI_UINT16 ldtr;
+
+   VMI_UINT16 gdtr_limit;
+   VMI_UINT32 gdtr_base;
+   VMI_UINT32 idtr_base;
+   VMI_UINT16 idtr_limit;
+} APState;
+
+// Historical 3.X revisions
+//#define MIN_VMI_API_REV_MINOR	        1 /* GetFlags_CLI is used */
+//#define MIN_VMI_API_REV_MINOR	        2 /* STI_SYSEXIT is used */
+//#define MIN_VMI_API_REV_MINOR	        3 /* IN/OUT are used */
+//#define MIN_VMI_API_REV_MINOR         4 /* Deferred calls used */
+//#define MIN_VMI_API_REV_MINOR		5 /* SetIOPLMask is used */
+
+// 4.X revisions
+//#define MIN_VMI_API_REV_MINOR		0 /* IN/OUT binary compat */
+
+// 5.X revisions
+//#define MIN_VMI_API_REV_MINOR		0
+//#define MIN_VMI_API_REV_MINOR		1 /* APIC read/write */
+//#define MIN_VMI_API_REV_MINOR         2 /* Send IPI */
+//#define MIN_VMI_API_REV_MINOR         3 /* IO Delay */
+//#define MIN_VMI_API_REV_MINOR         4 /* Timer API */
+//#define MIN_VMI_API_REV_MINOR         5 /* SMP VMI-Timer */
+
+// 6.X revisions
+//#define MIN_VMI_API_REV_MINOR         0 /* VMI Timer periodic alarms */
+
+// 7.X revisions
+//#define MIN_VMI_API_REV_MINOR         0 /* Dropped 6 VMI calls */
+//#define MIN_VMI_API_REV_MINOR         1 /* Wallclock */
+//#define MIN_VMI_API_REV_MINOR         2 /* RDTSC compatiblity */
+
+// 8.X revisions			
+//#define MIN_VMI_API_REV_MINOR         0 /* Xen compatible DT */
+//#define MIN_VMI_API_REV_MINOR         1 /* WallclockUpdated */
+
+// 9.X revisions
+//#define MIN_VMI_API_REV_MINOR		0 /* Removed HookCallback */
+
+// 10.X revisions
+//#define MIN_VMI_API_REV_MINOR		0 /* UpdateIDT -> WriteIDTEntry */
+
+// 11.X revisions
+//#define MIN_VMI_API_REV_MINOR		0 /* UpdateTSS -> UpdateKernelStack */
+
+// 12.X revisions
+//#define MIN_VMI_API_REV_MINOR		0 /* Drop APICSendIPI */
+
+// 13.X revisions
+#define MIN_VMI_API_REV_MINOR		0 /* Remove shared area allocation */
+
+#endif
include/asm-i386/mach-vmi/vmiCalls.h +98 -0

98 lines not on this page · raw message

#1 lore raw

include/asm-i386/mach-vmi/paravirtualInterface.h line 86

Quoted 9 lines · * Zachary Amsden ([email protected]) wrote:

Master definition of VMI interface, including calls, constants, and interface version.

include/asm-i386/mach-vmi/paravirtualInterface.h +6 -0
+/* VROM call table definitions */
+#define VROM_CALL_LEN             32
+
+typedef struct VROMCallEntry {
+   char f[VROM_CALL_LEN];
+} VROMCallEntry;

And the call entry is meant to be handled in whatever mechanism hypervisor prefers for its entry points (ABI constraints notwithstanding) as in, arbitrary software interrupt, or call gate, etc? I guess for transparent
it has to, since those would be local calls. Quite similar to the
hypercall entry point that Xen places on the hypercall_page, so easily compatible.

Quoted 7 lines
include/asm-i386/mach-vmi/paravirtualInterface.h +7 -0
+typedef struct VROMHeader {
+   VMI_UINT16          romSignature;             // option ROM signature
+   VMI_INT8            romLength;                // ROM length in 512 byte chunks
+   unsigned char       romEntry[4];              // 16-bit code entry point
+   VMI_UINT8           romPad0;                  // 4-byte align pad
+   VMI_UINT32          vRomSignature;            // VROM identification signature
+   VMI_UINT8           APIVersionMinor;          // Minor version of API

include/asm-i386/mach-vmi/paravirtualInterface.h line 104

Quoted 10 lines
include/asm-i386/mach-vmi/paravirtualInterface.h +10 -0
+   VMI_UINT8           APIVersionMajor;          // Major version of API
+   VMI_UINT8           reserved0;                // Reserved for expansion
+   VMI_UINT8           reserved1;                // Reserved for expansion
+   VMI_UINT32          reserved2;                // Reserved for expansion
+   VMI_UINT32          reserved3;                // Reserved for private use
+   VMI_UINT16          pciHeaderOffset;          // Offset to PCI OPROM header
+   VMI_UINT16          pnpHeaderOffset;          // Offset to PnP OPROM header
+   VMI_UINT32          romPad3;                  // PnP reserverd / VMI reserved
+   VROMCallEntry       romCallReserved[3];       // Reserved call slots
+} VROMHeader;

The document is slightly more descriptive. The above reserved slots are shown as:

	char		reserved[32];
	char		elfHeader[64];

But that's only 3 (0-2). I think I'm missing some small bit of magic.

include/asm-i386/mach-vmi/paravirtualInterface.h line 108

Quoted 3 lines
include/asm-i386/mach-vmi/paravirtualInterface.h +3 -0
+typedef struct VROMCallTable {
+   VROMCallEntry    vromCall[128];           // @ 0x80: ROM calls 4-127
+} VROMCallTable;

That comment eludes me. Are 0-3 special somehow (IOW, I thought it was just 0-2 as per above), and is it suggesting int 0x80?

include/asm-i386/mach-vmi/paravirtualInterface.h line 151

Quoted 9 lines
include/asm-i386/mach-vmi/paravirtualInterface.h +9 -0
+// Historical 3.X revisions
+//#define MIN_VMI_API_REV_MINOR	        1 /* GetFlags_CLI is used */
+//#define MIN_VMI_API_REV_MINOR	        2 /* STI_SYSEXIT is used */
+//#define MIN_VMI_API_REV_MINOR	        3 /* IN/OUT are used */
+//#define MIN_VMI_API_REV_MINOR         4 /* Deferred calls used */
+//#define MIN_VMI_API_REV_MINOR		5 /* SetIOPLMask is used */
+
+// 4.X revisions
+//#define MIN_VMI_API_REV_MINOR		0 /* IN/OUT binary compat */

Probably not the best format for keeping changelog entries. Although it's worth keeping the info somehwere in documentation perhaps.

thanks,
-chris

#2 lore raw
Quoted 7 lines · Chris Wright
Quoted 3 lines · * Zachary Amsden ([email protected]) wrote:

Master definition of VMI interface, including calls, constants, and interface version.

Quoted 14 lines

include/asm-i386/mach-vmi/paravirtualInterface.h line 86

Quoted 7 lines
include/asm-i386/mach-vmi/paravirtualInterface.h +6 -0
+/* VROM call table definitions */
+#define VROM_CALL_LEN             32
+
+typedef struct VROMCallEntry {
+   char f[VROM_CALL_LEN];
+} VROMCallEntry;

And the call entry is meant to be handled in whatever mechanism hypervisor prefers for its entry points (ABI constraints notwithstanding) as in, arbitrary software interrupt, or call gate, etc? I guess for transparent
it has to, since those would be local calls. Quite similar to the
hypercall entry point that Xen places on the hypercall_page, so easily compatible.

See below.

Quoted 9 lines

The document is slightly more descriptive. The above reserved slots are shown as:

	char		reserved[32];
	char		elfHeader[64];

But that's only 3 (0-2). I think I'm missing some small bit of magic.

Quoted 8 lines

include/asm-i386/mach-vmi/paravirtualInterface.h line 108

Quoted 4 lines
include/asm-i386/mach-vmi/paravirtualInterface.h +3 -0
+typedef struct VROMCallTable {
+   VROMCallEntry    vromCall[128];           // @ 0x80: ROM calls 4-127
+} VROMCallTable;

That comment eludes me. Are 0-3 special somehow (IOW, I thought it was just 0-2 as per above), and is it suggesting int 0x80?

Yeah, most of this is rather crufty - it is in transition. We had a full blown ROM image with 32-byte aligned stub points at one point for all of the VMI calls. In fact, it still is in that form, and it was required to overlap exactly with a native ROM that was built into Linux. See patch 6, VMI magic fixes for the details.

The machinery is already in place to do this, and it is a very nice thing that Xen has decided to adopt a similar approach (to the ROM) of publishing hypervisor code. I think they even use 32-byte alignment as well. The power of an indirection at this layer is just too attractive, and once you decide on a single binary image, I think it is inevitable that everyone will converge on the same idea. The same concept surfaces over and over - vsyscall being another example. You're basically dynamically linking in code at runtime, which is a pretty common thing to do, and gives you a very powerful redirection interface.

But we discovered that we couldn't achieve native performance, even using directly linked calls into the ROM. Calling out for sti / cli / popf / pushf on fast paths is just too expensive. We had to inline the native code to match native performance. And this leaves an opportunity for flexibility and performance in the hypervisor implementation.

You don't want direct calls to 32-byte stubs; in fact, as Joshua found, you can't get optimal performance in a hypervisor that way. What you need is the ability to preferentially inline certain hot calls into the VMI layer by using NOP padding. The non-hot calls can call out to the hypercall page.

Publishing code is the first step - inlining is the second, and it gets you back the hit you took when indirecting your fast paths.

Zach

#3 lore raw
In reply to Zachary Amsden
Quoted 2 lines · Zachary Amsden

Master definition of VMI interface, including calls, constants, and interface version.

This is a totally horrible style. There's absolutely no need to find your own sized integer types, please use the standard kernel ones. Also don't use camel case and #pack but rather __attribute__. Also please avoid // comments.

Also please remove all the historical version garbage, we don't care about that.

#4 lore raw
Quoted 6 lines · Christoph Hellwig
Quoted 3 lines · Zachary Amsden

Master definition of VMI interface, including calls, constants, and interface version.

Quoted 8 lines

This is a totally horrible style. There's absolutely no need to find your own sized integer types, please use the standard kernel ones. Also don't use camel case and #pack but rather __attribute__. Also please avoid // comments.

Also please remove all the historical version garbage, we don't care about that.

Ugly, isn't it. The collision of two source styles has left some scars. And fixing it is not yet finished. We know about these problems, and we are working on getting rid of them. The historical version garbage is rather important to us internally, and these bits are not yet fully polished, so you get to see it to. I think you'll find us respecting Linux conventions a lot more in the later patches.

Zach