Skip to contentSkip to conversation

Add #VC exception handling for AMD SEV-ES

PATCH Superseded Varad Gautam 5 participants 27 messages Jan 20, 2022 – Feb 8, 2022

Subsystems (2)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

cover Add #VC exception handling for AMD SEV-ES

For AMD SEV-ES, kvm-unit-tests currently rely on UEFI to set up a #VC exception handler.

This series adds #VC exception processing from Linux to kvm-unit-tests.

Based on the kvm-unit-tests uefi branch at
2fbec2578 ("x86 UEFI: Make _NO_FILE_4Uhere_ work w/ BOOTX64.EFI")

Tree: https://github.com/varadgautam/kvm-unit-tests/tree/vc-handler-v1

Varad Gautam (13):
x86/efi: Allow specifying AMD SEV/SEV-ES guest launch policy
x86: AMD SEV-ES: Setup #VC exception handler for AMD SEV-ES
x86: Move svm.h to lib/x86/
lib: x86: Import insn decoder from Linux
x86: AMD SEV-ES: Pull related GHCB definitions and helpers from Linux
x86: AMD SEV-ES: Prepare for #VC processing
x86: AMD SEV-ES: Handle WBINVD #VC
lib/x86: Move xsave helpers to lib/
x86: AMD SEV-ES: Handle CPUID #VC
x86: AMD SEV-ES: Handle RDTSC/RDTSCP #VC
x86: AMD SEV-ES: Handle MSR #VC
x86: AMD SEV-ES: Handle IOIO #VC
x86: AMD SEV-ES: Handle string IO for IOIO #VC

 Makefile                   |    3 +
 configure                  |   16 +
 lib/x86/amd_sev.c          |    3 +-
 lib/x86/amd_sev.h          |  107 +++
 lib/x86/amd_sev_vc.c       |  504 ++++++++++++
 lib/x86/desc.c             |   15 +
 lib/x86/desc.h             |    1 +
 lib/x86/insn/inat-tables.c | 1566 ++++++++++++++++++++++++++++++++++++
 lib/x86/insn/inat.c        |   86 ++
 lib/x86/insn/inat.h        |  233 ++++++
 lib/x86/insn/inat_types.h  |   18 +
 lib/x86/insn/insn.c        |  778 ++++++++++++++++++
 lib/x86/insn/insn.h        |  280 +++++++
 lib/x86/setup.c            |   10 +
 {x86 => lib/x86}/svm.h     |   37 +
 lib/x86/xsave.c            |   37 +
 lib/x86/xsave.h            |   16 +
 x86/Makefile.common        |    4 +
 x86/Makefile.x86_64        |    1 +
 x86/efi/README.md          |    5 +
 x86/efi/run                |   16 +
 x86/svm.c                  |    2 +-
 x86/svm_tests.c            |    2 +-
 x86/xsave.c                |   43 +-
 24 files changed, 3738 insertions(+), 45 deletions(-)
 create mode 100644 lib/x86/amd_sev_vc.c
 create mode 100644 lib/x86/insn/inat-tables.c
 create mode 100644 lib/x86/insn/inat.c
 create mode 100644 lib/x86/insn/inat.h
 create mode 100644 lib/x86/insn/inat_types.h
 create mode 100644 lib/x86/insn/insn.c
 create mode 100644 lib/x86/insn/insn.h
 rename {x86 => lib/x86}/svm.h (93%)
 create mode 100644 lib/x86/xsave.c
 create mode 100644 lib/x86/xsave.h
Signature · 1 line
2.32.0
#1 lore raw

PATCH 1/13 x86/efi: Allow specifying AMD SEV/SEV-ES guest launch policy

Make x86/efi/run check for AMDSEV envvar and set SEV/SEV-ES parameters on the qemu cmdline.

AMDSEV can be set to sev or sev-es.

---
 x86/efi/README.md |  5 +++++
 x86/efi/run       | 16 ++++++++++++++++
 2 files changed, 21 insertions(+)
x86/efi/README.md +5 -0
index a39f509..1222b30 100644
@@ -30,6 +30,11 @@ the env variable `EFI_UEFI`:
 
     EFI_UEFI=/path/to/OVMF.fd ./x86/efi/run ./x86/msr.efi
 
+To run the tests under AMD SEV/SEV-ES, set env variable `AMDSEV=sev` or
+`AMDSEV=sev-es`. This adds the desired guest policy to qemu command line.
+
+    AMDSEV=sev-es EFI_UEFI=/path/to/OVMF.fd ./x86/efi/run ./x86/amd_sev.efi
+
 ## Code structure
 
 ### Code from GNU-EFI
x86/efi/run +16 -0
index ac368a5..b48f626 100755
@@ -43,6 +43,21 @@ fi
 mkdir -p "$EFI_CASE_DIR"
 cp "$EFI_SRC/$EFI_CASE.efi" "$EFI_CASE_BINARY"
 
+amdsev_opts=
+if [ -n "$AMDSEV" ]; then
+	policy=
+	if [ "$AMDSEV" = "sev" ]; then
+		policy="0x1"
+	elif [ "$AMDSEV" = "sev-es" ]; then
+		policy="0x5"
+	else
+		echo "Cannot set AMDSEV policy. AMDSEV must be one of 'sev', 'sev-es'."
+		exit 2
+	fi
+
+	amdsev_opts="-object sev-guest,id=sev0,cbitpos=51,reduced-phys-bits=1,policy=$policy -machine memory-encryption=sev0"
+fi
+
 # Run test case with 256MiB QEMU memory. QEMU default memory size is 128MiB.
 # After UEFI boot up and we call `LibMemoryMap()`, the largest consecutive
 # memory region is ~42MiB. Although this is sufficient for many test cases to
@@ -61,4 +76,5 @@ cp "$EFI_SRC/$EFI_CASE.efi" "$EFI_CASE_BINARY"
 	-nographic \
 	-m 256 \
 	"$@" \
+	$amdsev_opts \
 	-smp "$EFI_SMP"
Signature · 1 line
2.32.0
#2 lore raw

Re: [kvm-unit-tests 01/13] x86/efi: Allow specifying AMD SEV/SEV-ES guest launch policy

Quoted 38 lines · Varad Gautam

Make x86/efi/run check for AMDSEV envvar and set SEV/SEV-ES parameters on the qemu cmdline.

AMDSEV can be set to sev or sev-es.

---
  x86/efi/README.md |  5 +++++
  x86/efi/run       | 16 ++++++++++++++++
  2 files changed, 21 insertions(+)
x86/efi/README.md +5 -0
index a39f509..1222b30 100644
@@ -30,6 +30,11 @@ the env variable `EFI_UEFI`:
 
      EFI_UEFI=/path/to/OVMF.fd ./x86/efi/run ./x86/msr.efi
 
+To run the tests under AMD SEV/SEV-ES, set env variable `AMDSEV=sev` or
+`AMDSEV=sev-es`. This adds the desired guest policy to qemu command line.
+
+    AMDSEV=sev-es EFI_UEFI=/path/to/OVMF.fd ./x86/efi/run ./x86/amd_sev.efi
+
  ## Code structure
 
  ### Code from GNU-EFI
x86/efi/run +3 -0
index ac368a5..b48f626 100755
@@ -43,6 +43,21 @@ fi
  mkdir -p "$EFI_CASE_DIR"
  cp "$EFI_SRC/$EFI_CASE.efi" "$EFI_CASE_BINARY"
 
+amdsev_opts=
+if [ -n "$AMDSEV" ]; then
+	policy=

x86/efi/run fi · line 58

Quoted 10 lines
x86/efi/run +10 -0
+	if [ "$AMDSEV" = "sev" ]; then
+		policy="0x1"
+	elif [ "$AMDSEV" = "sev-es" ]; then
+		policy="0x5"
+	else
+		echo "Cannot set AMDSEV policy. AMDSEV must be one of 'sev', 'sev-es'."
+		exit 2
+	fi
+
+	amdsev_opts="-object sev-guest,id=sev0,cbitpos=51,reduced-phys-bits=1,policy=$policy -machine memory-encryption=sev0"

This won't work on Naples or Rome systems because the cbitpos is 47 on those machines. You'll need to use CPUID to obtain the proper position for the system on which you are running.

You can use the cpuid command to get Fn8000001F_EBX[5:0] or I've used the following to find it from a bash script if you don't want to rely on the cpuid command being present:

EBX=$(dd if=/dev/cpu/0/cpuid ibs=16 count=32 skip=134217728 | tail -c 16 | od -An -t u4 -j 4 -N 4 | sed -re 's|^ *||') CBITPOS=$((EBX & 0x3f))

where 134217728 == 0x80000000

(I'm sure there's probably an easier way, but this works for me, but does rely on CONFIG_X86_CPUID)

Thanks,
Tom

Quoted 12 lines
x86/efi/run +3 -0
+fi
+
  # Run test case with 256MiB QEMU memory. QEMU default memory size is 128MiB.
  # After UEFI boot up and we call `LibMemoryMap()`, the largest consecutive
  # memory region is ~42MiB. Although this is sufficient for many test cases to
@@ -61,4 +76,5 @@ cp "$EFI_SRC/$EFI_CASE.efi" "$EFI_CASE_BINARY"
  	-nographic \
  	-m 256 \
  	"$@" \
+	$amdsev_opts \
  	-smp "$EFI_SMP"
#3 lore raw

Re: [kvm-unit-tests 01/13] x86/efi: Allow specifying AMD SEV/SEV-ES guest launch policy

In reply to Varad Gautam
Quoted 33 lines · Varad Gautam

Make x86/efi/run check for AMDSEV envvar and set SEV/SEV-ES parameters on the qemu cmdline.

AMDSEV can be set to sev or sev-es.

---
 x86/efi/README.md |  5 +++++
 x86/efi/run       | 16 ++++++++++++++++
 2 files changed, 21 insertions(+)
x86/efi/README.md +5 -0
index a39f509..1222b30 100644
@@ -30,6 +30,11 @@ the env variable `EFI_UEFI`:
 
     EFI_UEFI=/path/to/OVMF.fd ./x86/efi/run ./x86/msr.efi
 
+To run the tests under AMD SEV/SEV-ES, set env variable `AMDSEV=sev` or
+`AMDSEV=sev-es`. This adds the desired guest policy to qemu command line.
+
+    AMDSEV=sev-es EFI_UEFI=/path/to/OVMF.fd ./x86/efi/run ./x86/amd_sev.efi
+
 ## Code structure
 
 ### Code from GNU-EFI
x86/efi/run +0 -0
index ac368a5..b48f626 100755

x86/efi/run fi · line 52

Quoted 10 lines
x86/efi/run +7 -0
@@ -43,6 +43,21 @@ fi
 mkdir -p "$EFI_CASE_DIR"
 cp "$EFI_SRC/$EFI_CASE.efi" "$EFI_CASE_BINARY"
 
+amdsev_opts=
+if [ -n "$AMDSEV" ]; then
+       policy=
+       if [ "$AMDSEV" = "sev" ]; then
+               policy="0x1"
+       elif [ "$AMDSEV" = "sev-es" ]; then
+               policy="0x5"

Rather than bare constants, I think we should define these as bit fields. Something like:

$ git diff
x86/efi/run +6 -2
index b48f626b0435..427865807720 100755
@@ -19,6 +19,10 @@ source config.mak
 : "${EFI_SMP:=1}"
 : "${EFI_CASE:=$(basename $1 .efi)}"
 
+# Define guest policy bits, used to form QEMU command line.
+readonly SEV_POLICY_NODBG=$(( 1 << 0 ))
+readonly SEV_POLICY_ES=$(( 1 << 2 ))
+
 if [ ! -f "$EFI_UEFI" ]; then
        echo "UEFI firmware not found: $EFI_UEFI"
        echo "Please install the UEFI firmware to this path"
@@ -47,9 +51,9 @@ amdsev_opts=
 if [ -n "$AMDSEV" ]; then
        policy=
        if [ "$AMDSEV" = "sev" ]; then
-               policy="0x1"
+               policy="$(( $GUEST_POLICY_NODBG ))"
        elif [ "$AMDSEV" = "sev-es" ]; then
-               policy="0x5"
+               policy="$(( $GUEST_POLICY_NODBG | $GUEST_POLICY_ES ))"
        else
                echo "Cannot set AMDSEV policy. AMDSEV must be one of

'sev', 'sev-es'."
exit 2

Quoted 18 lines
x86/efi/run +9 -1
+       else
+               echo "Cannot set AMDSEV policy. AMDSEV must be one of 'sev', 'sev-es'."
+               exit 2
+       fi
+
+       amdsev_opts="-object sev-guest,id=sev0,cbitpos=51,reduced-phys-bits=1,policy=$policy -machine memory-encryption=sev0"
+fi
+
 # Run test case with 256MiB QEMU memory. QEMU default memory size is 128MiB.
 # After UEFI boot up and we call `LibMemoryMap()`, the largest consecutive
 # memory region is ~42MiB. Although this is sufficient for many test cases to
@@ -61,4 +76,5 @@ cp "$EFI_SRC/$EFI_CASE.efi" "$EFI_CASE_BINARY"
        -nographic \
        -m 256 \
        "$@" \
+       $amdsev_opts \
        -smp "$EFI_SMP"
--
Quoted 2 lines

2.32.0

We don't have QEMU working on our internal setup because it doesn't support INIT_EX. So I wasn't able to test this patch in its entirety. But this patch seems pretty reasonable, independent of the rest of the series. Depending on maintainer review bandwidth, it might make sense to send this patch on its own (with Tom's feedback incorporated), outside of the series, so it can get merged quicker.

#4 lore raw

PATCH 2/13 x86: AMD SEV-ES: Setup #VC exception handler for AMD SEV-ES

In reply to Varad Gautam

AMD SEV-ES defines a new guest exception that gets triggered on some vmexits to allow the guest to control what state gets shared with the host. kvm-unit-tests currently relies on UEFI to provide this #VC exception handler.

Switch the tests to install their own #VC handler on early bootup to process these exits, just after GHCB has been mapped.

If --amdsev-efi-vc is passed during ./configure, the tests will continue using the UEFI #VC handler.

---
 Makefile             |  3 +++
 configure            | 16 ++++++++++++++++
 lib/x86/amd_sev.c    |  3 ++-
 lib/x86/amd_sev.h    |  1 +
 lib/x86/amd_sev_vc.c | 14 ++++++++++++++
 lib/x86/desc.c       | 15 +++++++++++++++
 lib/x86/desc.h       |  1 +
 lib/x86/setup.c      | 10 ++++++++++
 x86/Makefile.common  |  1 +
 9 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 lib/x86/amd_sev_vc.c
Makefile +3 -0
index 4f4ad23..94a0162 100644
@@ -46,6 +46,9 @@ else
 $(error Cannot build $(ARCH_NAME) tests as EFI apps)
 endif
 EFI_CFLAGS := -DTARGET_EFI
+ifeq ($(AMDSEV_EFI_VC),y)
+EFI_CFLAGS += -DAMDSEV_EFI_VC
+endif
 # The following CFLAGS and LDFLAGS come from:
 #   - GNU-EFI/Makefile.defaults
 #   - GNU-EFI/apps/Makefile
configure +16 -0
index 41372ef..c687d9f 100755
@@ -29,6 +29,7 @@ host_key_document=
 page_size=
 earlycon=
 target_efi=
+amdsev_efi_vc=
 
 usage() {
     cat <<-EOF
@@ -71,6 +72,8 @@ usage() {
 	                           Specify a PL011 compatible UART at address ADDR. Supported
 	                           register stride is 32 bit only.
 	    --target-efi           Boot and run from UEFI
+	    --amdsev-efi-vc        Use UEFI-provided #VC handlers on AMD SEV/ES. Requires
+	                           --target-efi.
 EOF
     exit 1
 }
@@ -138,6 +141,9 @@ while [[ "$1" = -* ]]; do
 	--target-efi)
 	    target_efi=y
 	    ;;
+	--amdsev-efi-vc)
+	    amdsev_efi_vc=y
+	    ;;
 	--help)
 	    usage
 	    ;;
@@ -197,8 +203,17 @@ elif [ "$processor" = "arm" ]; then
     processor="cortex-a15"
 fi
 
+if [ "$amdsev_efi_vc" ] && [ "$arch" != "x86_64" ]; then
+    echo "--amdsev-efi-vc requires arch x86_64."
+    usage
+fi
+
 if [ "$arch" = "i386" ] || [ "$arch" = "x86_64" ]; then
     testdir=x86
+    if [ "$amdsev_efi_vc" ] && [ -z "$target_efi" ]; then
+        echo "--amdsev-efi-vc requires --target-efi."
+        usage
+    fi
 elif [ "$arch" = "arm" ] || [ "$arch" = "arm64" ]; then
     testdir=arm
     if [ "$target" = "qemu" ]; then
@@ -356,6 +371,7 @@ WA_DIVIDE=$wa_divide
 GENPROTIMG=${GENPROTIMG-genprotimg}
 HOST_KEY_DOCUMENT=$host_key_document
 TARGET_EFI=$target_efi
+AMDSEV_EFI_VC=$amdsev_efi_vc
 EOF
 if [ "$arch" = "arm" ] || [ "$arch" = "arm64" ]; then
     echo "TARGET=$target" >> config.mak
lib/x86/amd_sev.c +2 -1
index 6672214..bde126b 100644
@@ -14,6 +14,7 @@
 #include "x86/vm.h"
 
 static unsigned short amd_sev_c_bit_pos;
+phys_addr_t ghcb_addr;
 
 bool amd_sev_enabled(void)
 {
@@ -126,7 +127,7 @@ void setup_ghcb_pte(pgd_t *page_table)
 	 * function searches GHCB's L1 pte, creates corresponding L1 ptes if not
 	 * found, and unsets the c-bit of GHCB's L1 pte.
 	 */
-	phys_addr_t ghcb_addr, ghcb_base_addr;
+	phys_addr_t ghcb_base_addr;
 	pteval_t *pte;
 
 	/* Read the current GHCB page addr */
lib/x86/amd_sev.h +1 -0
index 6a10f84..afbacf3 100644
@@ -54,6 +54,7 @@ efi_status_t setup_amd_sev(void);
 bool amd_sev_es_enabled(void);
 efi_status_t setup_amd_sev_es(void);
 void setup_ghcb_pte(pgd_t *page_table);
+void handle_sev_es_vc(struct ex_regs *regs);
 
 unsigned long long get_amd_sev_c_bit_mask(void);
 unsigned long long get_amd_sev_addr_upperbound(void);
lib/x86/amd_sev_vc.c new file +14 -0
new file mode 100644
index 0000000..8226121
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include "amd_sev.h"
+
+extern phys_addr_t ghcb_addr;
+
+void handle_sev_es_vc(struct ex_regs *regs)
+{
+	struct ghcb *ghcb = (struct ghcb *) ghcb_addr;
+	if (!ghcb) {
+		/* TODO: kill guest */
+		return;
+	}
+}
lib/x86/desc.c +15 -0
index 16b7256..73aa866 100644
@@ -3,6 +3,9 @@
 #include "processor.h"
 #include <setjmp.h>
 #include "apic-defs.h"
+#ifdef TARGET_EFI
+#include "amd_sev.h"
+#endif
 
 /* Boot-related data structures */
 
@@ -228,6 +231,9 @@ EX_E(ac, 17);
 EX(mc, 18);
 EX(xm, 19);
 EX_E(cp, 21);
+#ifdef TARGET_EFI
+EX_E(vc, 29);
+#endif
 
 asm (".pushsection .text \n\t"
      "__handle_exception: \n\t"
@@ -293,6 +299,15 @@ void setup_idt(void)
     handle_exception(13, check_exception_table);
 }
 
+void setup_amd_sev_es_vc(void)
+{
+	if (!amd_sev_es_enabled())
+		return;
+
+	set_idt_entry(29, &vc_fault, 0);
+	handle_exception(29, handle_sev_es_vc);
+}
+
 unsigned exception_vector(void)
 {
     unsigned char vector;
lib/x86/desc.h +1 -0
index b65539e..4fcbf9f 100644
@@ -220,6 +220,7 @@ void set_intr_alt_stack(int e, void *fn);
 void print_current_tss_info(void);
 handler handle_exception(u8 v, handler fn);
 void unhandled_exception(struct ex_regs *regs, bool cpu);
+void setup_amd_sev_es_vc(void);
 
 bool test_for_exception(unsigned int ex, void (*trigger_func)(void *data),
 			void *data);
lib/x86/setup.c +10 -0
index bbd3468..6013602 100644
@@ -327,6 +327,16 @@ efi_status_t setup_efi(efi_bootinfo_t *efi_bootinfo)
 	smp_init();
 	setup_page_table();
 
+#ifndef AMDSEV_EFI_VC
+	if (amd_sev_es_enabled()) {
+		/*
+		 * Switch away from the UEFI-installed #VC handler.
+		 * GHCB has already been mapped at this point.
+		 */
+		setup_amd_sev_es_vc();
+	}
+#endif /* AMDSEV_EFI_VC */
+
 	return EFI_SUCCESS;
 }
 
x86/Makefile.common +1 -0
index 984444e..65d16e7 100644
@@ -24,6 +24,7 @@ cflatobjs += lib/x86/fault_test.o
 cflatobjs += lib/x86/delay.o
 ifeq ($(TARGET_EFI),y)
 cflatobjs += lib/x86/amd_sev.o
+cflatobjs += lib/x86/amd_sev_vc.o
 cflatobjs += lib/efi.o
 cflatobjs += x86/efi/reloc_x86_64.o
 endif
Signature · 1 line
2.32.0
Tested-by on v1 2/13
#5 lore raw

Re: [kvm-unit-tests 02/13] x86: AMD SEV-ES: Setup #VC exception handler for AMD SEV-ES

Quoted 53 lines · Varad Gautam

AMD SEV-ES defines a new guest exception that gets triggered on some vmexits to allow the guest to control what state gets shared with the host. kvm-unit-tests currently relies on UEFI to provide this #VC exception handler.

Switch the tests to install their own #VC handler on early bootup to process these exits, just after GHCB has been mapped.

If --amdsev-efi-vc is passed during ./configure, the tests will continue using the UEFI #VC handler.

---
 Makefile             |  3 +++
 configure            | 16 ++++++++++++++++
 lib/x86/amd_sev.c    |  3 ++-
 lib/x86/amd_sev.h    |  1 +
 lib/x86/amd_sev_vc.c | 14 ++++++++++++++
 lib/x86/desc.c       | 15 +++++++++++++++
 lib/x86/desc.h       |  1 +
 lib/x86/setup.c      | 10 ++++++++++
 x86/Makefile.common  |  1 +
 9 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 lib/x86/amd_sev_vc.c
Makefile +3 -0
index 4f4ad23..94a0162 100644
@@ -46,6 +46,9 @@ else
 $(error Cannot build $(ARCH_NAME) tests as EFI apps)
 endif
 EFI_CFLAGS := -DTARGET_EFI
+ifeq ($(AMDSEV_EFI_VC),y)
+EFI_CFLAGS += -DAMDSEV_EFI_VC
+endif
 # The following CFLAGS and LDFLAGS come from:
 #   - GNU-EFI/Makefile.defaults
 #   - GNU-EFI/apps/Makefile
configure +1 -0
index 41372ef..c687d9f 100755
@@ -29,6 +29,7 @@ host_key_document=
 page_size=
 earlycon=
 target_efi=
+amdsev_efi_vc=
 
 usage() {
     cat <<-EOF

configure usage() { · line 76

Quoted 5 lines
configure +2 -0
@@ -71,6 +72,8 @@ usage() {
                                   Specify a PL011 compatible UART at address ADDR. Supported
                                   register stride is 32 bit only.
            --target-efi           Boot and run from UEFI
+           --amdsev-efi-vc        Use UEFI-provided #VC handlers on AMD SEV/ES. Requires
+                                  --target-efi.

What's the rationale to make the built-in #VC handler default? Note:
I'm not actually saying I have an opinion, one way or the other. But I think the commit description (and cover letter) should explain why we prefer what we're doing.

Quoted 149 lines
configure +13 -0
 EOF
     exit 1
 }
@@ -138,6 +141,9 @@ while [[ "$1" = -* ]]; do
        --target-efi)
            target_efi=y
            ;;
+       --amdsev-efi-vc)
+           amdsev_efi_vc=y
+           ;;
        --help)
            usage
            ;;
@@ -197,8 +203,17 @@ elif [ "$processor" = "arm" ]; then
     processor="cortex-a15"
 fi
 
+if [ "$amdsev_efi_vc" ] && [ "$arch" != "x86_64" ]; then
+    echo "--amdsev-efi-vc requires arch x86_64."
+    usage
+fi
+
 if [ "$arch" = "i386" ] || [ "$arch" = "x86_64" ]; then
     testdir=x86
+    if [ "$amdsev_efi_vc" ] && [ -z "$target_efi" ]; then
+        echo "--amdsev-efi-vc requires --target-efi."
+        usage
+    fi
 elif [ "$arch" = "arm" ] || [ "$arch" = "arm64" ]; then
     testdir=arm
     if [ "$target" = "qemu" ]; then
@@ -356,6 +371,7 @@ WA_DIVIDE=$wa_divide
 GENPROTIMG=${GENPROTIMG-genprotimg}
 HOST_KEY_DOCUMENT=$host_key_document
 TARGET_EFI=$target_efi
+AMDSEV_EFI_VC=$amdsev_efi_vc
 EOF
 if [ "$arch" = "arm" ] || [ "$arch" = "arm64" ]; then
     echo "TARGET=$target" >> config.mak
lib/x86/amd_sev.c +2 -1
index 6672214..bde126b 100644
@@ -14,6 +14,7 @@
 #include "x86/vm.h"
 
 static unsigned short amd_sev_c_bit_pos;
+phys_addr_t ghcb_addr;
 
 bool amd_sev_enabled(void)
 {
@@ -126,7 +127,7 @@ void setup_ghcb_pte(pgd_t *page_table)
         * function searches GHCB's L1 pte, creates corresponding L1 ptes if not
         * found, and unsets the c-bit of GHCB's L1 pte.
         */
-       phys_addr_t ghcb_addr, ghcb_base_addr;
+       phys_addr_t ghcb_base_addr;
        pteval_t *pte;
 
        /* Read the current GHCB page addr */
lib/x86/amd_sev.h +1 -0
index 6a10f84..afbacf3 100644
@@ -54,6 +54,7 @@ efi_status_t setup_amd_sev(void);
 bool amd_sev_es_enabled(void);
 efi_status_t setup_amd_sev_es(void);
 void setup_ghcb_pte(pgd_t *page_table);
+void handle_sev_es_vc(struct ex_regs *regs);
 
 unsigned long long get_amd_sev_c_bit_mask(void);
 unsigned long long get_amd_sev_addr_upperbound(void);
lib/x86/amd_sev_vc.c new file +14 -0
new file mode 100644
index 0000000..8226121
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include "amd_sev.h"
+
+extern phys_addr_t ghcb_addr;
+
+void handle_sev_es_vc(struct ex_regs *regs)
+{
+       struct ghcb *ghcb = (struct ghcb *) ghcb_addr;
+       if (!ghcb) {
+               /* TODO: kill guest */
+               return;
+       }
+}
lib/x86/desc.c +15 -0
index 16b7256..73aa866 100644
@@ -3,6 +3,9 @@
 #include "processor.h"
 #include <setjmp.h>
 #include "apic-defs.h"
+#ifdef TARGET_EFI
+#include "amd_sev.h"
+#endif
 
 /* Boot-related data structures */
 
@@ -228,6 +231,9 @@ EX_E(ac, 17);
 EX(mc, 18);
 EX(xm, 19);
 EX_E(cp, 21);
+#ifdef TARGET_EFI
+EX_E(vc, 29);
+#endif
 
 asm (".pushsection .text \n\t"
      "__handle_exception: \n\t"
@@ -293,6 +299,15 @@ void setup_idt(void)
     handle_exception(13, check_exception_table);
 }
 
+void setup_amd_sev_es_vc(void)
+{
+       if (!amd_sev_es_enabled())
+               return;
+
+       set_idt_entry(29, &vc_fault, 0);
+       handle_exception(29, handle_sev_es_vc);
+}
+
 unsigned exception_vector(void)
 {
     unsigned char vector;
lib/x86/desc.h +1 -0
index b65539e..4fcbf9f 100644
@@ -220,6 +220,7 @@ void set_intr_alt_stack(int e, void *fn);
 void print_current_tss_info(void);
 handler handle_exception(u8 v, handler fn);
 void unhandled_exception(struct ex_regs *regs, bool cpu);
+void setup_amd_sev_es_vc(void);
 
 bool test_for_exception(unsigned int ex, void (*trigger_func)(void *data),
                        void *data);
lib/x86/setup.c +0 -0
index bbd3468..6013602 100644

lib/x86/setup.c efi_status_t setup_efi(efi_bootinfo_t *efi_bootinfo) · line 331

Quoted 5 lines
lib/x86/setup.c +2 -0
@@ -327,6 +327,16 @@ efi_status_t setup_efi(efi_bootinfo_t *efi_bootinfo)
        smp_init();
        setup_page_table();
 
+#ifndef AMDSEV_EFI_VC
+       if (amd_sev_es_enabled()) {

This is a tautology. setup_amd_sev_es_vc() returns immediately upon !amd_sev_es_enabled(). Therefore, I think we should remove the check here.

Quoted 24 lines
lib/x86/setup.c +8 -0
+               /*
+                * Switch away from the UEFI-installed #VC handler.
+                * GHCB has already been mapped at this point.
+                */
+               setup_amd_sev_es_vc();
+       }
+#endif /* AMDSEV_EFI_VC */
+
        return EFI_SUCCESS;
 }
 
x86/Makefile.common +1 -1
index 984444e..65d16e7 100644
@@ -24,6 +24,7 @@ cflatobjs += lib/x86/fault_test.o
 cflatobjs += lib/x86/delay.o
 ifeq ($(TARGET_EFI),y)
 cflatobjs += lib/x86/amd_sev.o
+cflatobjs += lib/x86/amd_sev_vc.o
 cflatobjs += lib/efi.o
 cflatobjs += x86/efi/reloc_x86_64.o
 endif
--
Quoted 2 lines

2.32.0

Thank you for adding this patch to the series!

I left a comment above asking about the default #VC handler, above. And I'd like to expand on that question with my thoughts here.

As far as I understand, mostly from offline conversations with Sean, a big motivation for the built-in #VC handler is that in the (very) long term, people will build a guest firmware without a #VC handler. In other words, eventually (I assume this is many years down the road), the guest firmware will be enlightened enough to avoid NAEs altogether, and instead explicitly call VMGEXIT explicitly, rather than implicitly after triggering a #VC exception. To me, this all sounds very far (e.g., 5-10 years) out. And thus, I still question the value in adding the built-in #VC handler with this as the justification now. However, it's clear from prior discussions that this is the direction preferred by the community. So I'm not pushing back. I'm just making sure we're all on the same page here. Please let me know if I'm mis-understanding this rationale or missing any reasons for why folks want a built-in #VC handler. And again, I think the cover letter should be updated with all the reasons why we're doing what we're doing.

Where I think this work could be useful in the shorter term is testing. We have three pieces of code to test:
1. Host-side VMGEXIT handler.
2. Guest-side firmware #VC handler.
3. Guest-side kernel #VC handler.

Technically, I guess there are multiple #VC handlers within each guest-side component, depending on the boot stage. But let's consider the steady state handlers that operate after initial guest-side boot stages for now.

Using the UEFI's builtin #VC handler tests:
- Guest-side firmware #VC handler + host-side VMGEXIT handler.

Notably, it does _not_ test the guest-side kernel #VC handler. Looking ahead in the series, I can see that the code for the new builtin kvm-unit-tests #VC handler is basically copied from the Linux kernel #VC handler. Therefore, we could use this work to test:
- Guest-side kernel #VC handler + host-side VMGEXIT handler.

The difficulty with using this approach to test the guest-side #VC handler is that we now have two copies of the code that can get out of
sync: one in the kernel and a 2nd in kvm-unit-tests. However, maybe this is OK. And maybe we can even try to ameliorate this issue with some scripts to compare the two codes or some policy that when we update the Linux kernel proper we send a follow-up patch to kvm-unit-tests.

Another approach to test the guest-side kernel #VC handler would be to add tests to run from within a Linux guest. E.g., maybe Linux selftests could work here. We actually have something like this internally, But the way it was written would make it non-trivial to upstream.

Thoughts?

#6 lore raw

Re: [kvm-unit-tests 02/13] x86: AMD SEV-ES: Setup #VC exception handler for AMD SEV-ES

Hi Marc,

Quoted 2 lines · Marc Orr

Please let me know if I'm mis-understanding this rationale or missing any reasons for why folks want a built-in #VC handler.

There are a couple of reasons which come all come down to one goal:
Robustnes of the kvm-unit-tests.

If kvm-unit-tests continue to use the firmware #VC handler after ExitBootServices there needs to be a contract between the test framework and the firmware about:

1) Page-table layout - The page table needs to map the firmware and the shared GHCB used by the firmware.

2) The firmware is required to keep its #VC handler in the current IDT for kvm-unit-tests to find it and copy the #VC entry into its own IDT.

3) The firmware #VC handler might use state which is not available anymore after ExitBootServices.

4) If the firmware uses the kvm-unit-test GHCB after ExitBootServices, it has the get the GHCB address from the GHCB MSR, requiring an identity mapping. Moreover it requires to keep the address of the GHCB in the MSR at all times where a #VC could happen. This could be a problem when we start to add SEV-ES specific tests to the unit-tests, explcitily testing the MSR protocol.

It is easy to violate this implicit protocol and breaking kvm-unit-tests just by a new version of OVMF being used. I think that is not a very robust approach and a separate #VC handler in the unit-test framework makes sense even now.

Regards,

Signature · 10 lines
Jörg Rödel
[email protected]

SUSE Software Solutions Germany GmbH
Maxfeldstr. 5
90409 Nürnberg
Germany
 
(HRB 36809, AG Nürnberg)
Geschäftsführer: Ivo Totev
#7 lore raw

Re: [kvm-unit-tests 02/13] x86: AMD SEV-ES: Setup #VC exception handler for AMD SEV-ES

Quoted 14 lines · Joerg Roedel

Hi Marc,

Quoted 2 lines · Marc Orr

Please let me know if I'm mis-understanding this rationale or missing any reasons for why folks want a built-in #VC handler.

There are a couple of reasons which come all come down to one goal:
Robustnes of the kvm-unit-tests.

If kvm-unit-tests continue to use the firmware #VC handler after ExitBootServices there needs to be a contract between the test framework and the firmware about:

Quoted 6 lines

1) Page-table layout - The page table needs to map the firmware and the shared GHCB used by the firmware.

2) The firmware is required to keep its #VC handler in the current IDT for kvm-unit-tests to find it and copy the #VC entry into its own IDT.

Yeah. I think we already resolved these first two issues in the initial patch set.

Quoted 2 lines

3) The firmware #VC handler might use state which is not available anymore after ExitBootServices.

Of all the issues listed, this one seems the most serious.

Quoted 7 lines

4) If the firmware uses the kvm-unit-test GHCB after ExitBootServices, it has the get the GHCB address from the GHCB MSR, requiring an identity mapping. Moreover it requires to keep the address of the GHCB in the MSR at all times where a #VC could happen. This could be a problem when we start to add SEV-ES specific tests to the unit-tests, explcitily testing the MSR protocol.

Ack. I'd think we could require tests to save/restore the GHCB MSR.

Quoted 4 lines

It is easy to violate this implicit protocol and breaking kvm-unit-tests just by a new version of OVMF being used. I think that is not a very robust approach and a separate #VC handler in the unit-test framework makes sense even now.

Thanks for the explanation! I hope we can keep the UEFI #VC handler working, because like I mentioned, I think this work can be used to test that code inside of UEFI. But I guess time will tell.

Of all the points listed above, I think point #3 is the most concerning. The others seem like they can be managed.

Nonetheless, based on this explanation plus prior mailing list discussions, it is clear that the preference is to make the built-in #VC handler the default. My only request to Varad is to update the cover letter/patch descriptions with a summary of this discussion. Also, it might be worth adding a comment in the configure script mentioning that the built-in #VC handler is the default due to robustness and future-proofing concerns.

Regarding code review and testing, I can help with the following:
- Compare the patches being pulled into kvm-unit-tests to what's in the Linux kernel and add my Reviewed-by tags if I don't see any meaningful discrepancies.
- Test the entire series on Google's setup, which doesn't use QEMU and add my Tested-by tag accordingly. My previous Tested-by tags were on individual patches. I have not yet tested the entire series.

Please let me know if this is useful. If not, I wouldn't spend the time :-).

Quoted 14 lines

Regards,

--
Jörg Rödel
[email protected]

SUSE Software Solutions Germany GmbH
Maxfeldstr. 5
90409 Nürnberg
Germany

(HRB 36809, AG Nürnberg)
Geschäftsführer: Ivo Totev

#8 lore raw

Re: [kvm-unit-tests 02/13] x86: AMD SEV-ES: Setup #VC exception handler for AMD SEV-ES

Quoted 21 lines · Marc Orr
Quoted 2 lines · Joerg Roedel

3) The firmware #VC handler might use state which is not available anymore after ExitBootServices.

Of all the issues listed, this one seems the most serious.

Quoted 7 lines

4) If the firmware uses the kvm-unit-test GHCB after ExitBootServices, it has the get the GHCB address from the GHCB MSR, requiring an identity mapping. Moreover it requires to keep the address of the GHCB in the MSR at all times where a #VC could happen. This could be a problem when we start to add SEV-ES specific tests to the unit-tests, explcitily testing the MSR protocol.

Ack. I'd think we could require tests to save/restore the GHCB MSR.

Quoted 4 lines

It is easy to violate this implicit protocol and breaking kvm-unit-tests just by a new version of OVMF being used. I think that is not a very robust approach and a separate #VC handler in the unit-test framework makes sense even now.

Quoted 6 lines

Thanks for the explanation! I hope we can keep the UEFI #VC handler working, because like I mentioned, I think this work can be used to test that code inside of UEFI. But I guess time will tell.

Of all the points listed above, I think point #3 is the most concerning. The others seem like they can be managed.

5) Debug. I don't want to have to reverse engineer assembly code to understand why a #VC handler isn't doing what I expect, or to a debug the exchanges between guest and host.

Quoted 2 lines · Varad Gautam

If --amdsev-efi-vc is passed during ./configure, the tests will continue using the UEFI #VC handler.

Why bother? I would prefer we ditch the UEFI #VC handler entirely and not give users the option to using anything but the built-in handler. What do we gain other than complexity?

#9 lore raw

Re: [kvm-unit-tests 02/13] x86: AMD SEV-ES: Setup #VC exception handler for AMD SEV-ES

Quoted 27 lines · Sean Christopherson
Quoted 25 lines · Marc Orr
Quoted 2 lines · Joerg Roedel

3) The firmware #VC handler might use state which is not available anymore after ExitBootServices.

Of all the issues listed, this one seems the most serious.

Quoted 7 lines

4) If the firmware uses the kvm-unit-test GHCB after ExitBootServices, it has the get the GHCB address from the GHCB MSR, requiring an identity mapping. Moreover it requires to keep the address of the GHCB in the MSR at all times where a #VC could happen. This could be a problem when we start to add SEV-ES specific tests to the unit-tests, explcitily testing the MSR protocol.

Ack. I'd think we could require tests to save/restore the GHCB MSR.

Quoted 4 lines

It is easy to violate this implicit protocol and breaking kvm-unit-tests just by a new version of OVMF being used. I think that is not a very robust approach and a separate #VC handler in the unit-test framework makes sense even now.

Thanks for the explanation! I hope we can keep the UEFI #VC handler working, because like I mentioned, I think this work can be used to test that code inside of UEFI. But I guess time will tell.

Quoted 6 lines
Quoted 2 lines

Of all the points listed above, I think point #3 is the most concerning. The others seem like they can be managed.

5) Debug. I don't want to have to reverse engineer assembly code to understand why a #VC handler isn't doing what I expect, or to a debug the exchanges between guest and host.

Ack. But this can also be viewed as a benefit. Because the bug is probably something that should be followed up and fixed inside of UEFI.

And that's my end goal. Can we reuse this work to test the #VC handler in the UEFI?

This shouldn't be onerous. Because the #VC should follow the APM and GHCB specs. And both UEFI and kvm-unit-tests #VC handlers should be coded to those specs. If they're not, then one of them has a bug.

Quoted 7 lines · Varad Gautam
Quoted 2 lines

If --amdsev-efi-vc is passed during ./configure, the tests will continue using the UEFI #VC handler.

Why bother? I would prefer we ditch the UEFI #VC handler entirely and not give users the option to using anything but the built-in handler. What do we gain other than complexity?

See above. If we keep the ability to run off the UEFI #VC handler then we can get continuous testing running inside of Google to verify the UEFI used to launch every SEV VM on Google cloud.

#10 lore raw

Re: [kvm-unit-tests 02/13] x86: AMD SEV-ES: Setup #VC exception handler for AMD SEV-ES

Quoted 31 lines · Marc Orr
Quoted 30 lines · Sean Christopherson
Quoted 27 lines · Marc Orr
Quoted 2 lines · Joerg Roedel

3) The firmware #VC handler might use state which is not available anymore after ExitBootServices.

Of all the issues listed, this one seems the most serious.

Quoted 7 lines

4) If the firmware uses the kvm-unit-test GHCB after ExitBootServices, it has the get the GHCB address from the GHCB MSR, requiring an identity mapping. Moreover it requires to keep the address of the GHCB in the MSR at all times where a #VC could happen. This could be a problem when we start to add SEV-ES specific tests to the unit-tests, explcitily testing the MSR protocol.

Ack. I'd think we could require tests to save/restore the GHCB MSR.

Quoted 4 lines

It is easy to violate this implicit protocol and breaking kvm-unit-tests just by a new version of OVMF being used. I think that is not a very robust approach and a separate #VC handler in the unit-test framework makes sense even now.

Thanks for the explanation! I hope we can keep the UEFI #VC handler working, because like I mentioned, I think this work can be used to test that code inside of UEFI. But I guess time will tell.

Of all the points listed above, I think point #3 is the most concerning. The others seem like they can be managed.

Quoted 7 lines
Quoted 3 lines

5) Debug. I don't want to have to reverse engineer assembly code to understand why a #VC handler isn't doing what I expect, or to a debug the exchanges between guest and host.

Ack. But this can also be viewed as a benefit. Because the bug is probably something that should be followed up and fixed inside of UEFI.

But how would we know it's a bug? E.g. IMO, UEFI should be enlightened to _never_ take a #VC, at which point its #VC handle can be changed to panic and using such a UEIF would cause KUT to fail.

Quoted 11 lines

And that's my end goal. Can we reuse this work to test the #VC handler in the UEFI?

This shouldn't be onerous. Because the #VC should follow the APM and GHCB specs. And both UEFI and kvm-unit-tests #VC handlers should be coded to those specs. If they're not, then one of them has a bug.

Quoted 4 lines · Varad Gautam
Quoted 2 lines

If --amdsev-efi-vc is passed during ./configure, the tests will continue using the UEFI #VC handler.

Quoted 7 lines
Quoted 3 lines

Why bother? I would prefer we ditch the UEFI #VC handler entirely and not give users the option to using anything but the built-in handler. What do we gain other than complexity?

See above. If we keep the ability to run off the UEFI #VC handler then we can get continuous testing running inside of Google to verify the UEFI used to launch every SEV VM on Google cloud.

I'm not super opposed to the idea, but I really do think that taking a #VC in guest UEFI is a bug in and of itself.

#11 lore raw

Re: [kvm-unit-tests 02/13] x86: AMD SEV-ES: Setup #VC exception handler for AMD SEV-ES

Quoted 37 lines · Sean Christopherson
Quoted 35 lines · Marc Orr
Quoted 33 lines · Sean Christopherson
Quoted 21 lines · Marc Orr
Quoted 2 lines · Joerg Roedel

3) The firmware #VC handler might use state which is not available anymore after ExitBootServices.

Of all the issues listed, this one seems the most serious.

Quoted 7 lines

4) If the firmware uses the kvm-unit-test GHCB after ExitBootServices, it has the get the GHCB address from the GHCB MSR, requiring an identity mapping. Moreover it requires to keep the address of the GHCB in the MSR at all times where a #VC could happen. This could be a problem when we start to add SEV-ES specific tests to the unit-tests, explcitily testing the MSR protocol.

Ack. I'd think we could require tests to save/restore the GHCB MSR.

Quoted 4 lines

It is easy to violate this implicit protocol and breaking kvm-unit-tests just by a new version of OVMF being used. I think that is not a very robust approach and a separate #VC handler in the unit-test framework makes sense even now.

Quoted 6 lines

Thanks for the explanation! I hope we can keep the UEFI #VC handler working, because like I mentioned, I think this work can be used to test that code inside of UEFI. But I guess time will tell.

Of all the points listed above, I think point #3 is the most concerning. The others seem like they can be managed.

5) Debug. I don't want to have to reverse engineer assembly code to understand why a #VC handler isn't doing what I expect, or to a debug the exchanges between guest and host.

Quoted 7 lines
Quoted 3 lines

Ack. But this can also be viewed as a benefit. Because the bug is probably something that should be followed up and fixed inside of UEFI.

But how would we know it's a bug? E.g. IMO, UEFI should be enlightened to _never_ take a #VC, at which point its #VC handle can be changed to panic and using such a UEIF would cause KUT to fail.

Yeah. If we end up with enlightened UEFIs that skip handling some/all #VC exceptions, then using the UEFI #VC handler from kvm-unit-tests doesn't make any sense :-).

Quoted 15 lines
Quoted 15 lines

And that's my end goal. Can we reuse this work to test the #VC handler in the UEFI?

This shouldn't be onerous. Because the #VC should follow the APM and GHCB specs. And both UEFI and kvm-unit-tests #VC handlers should be coded to those specs. If they're not, then one of them has a bug.

Quoted 7 lines · Varad Gautam
Quoted 2 lines

If --amdsev-efi-vc is passed during ./configure, the tests will continue using the UEFI #VC handler.

Why bother? I would prefer we ditch the UEFI #VC handler entirely and not give users the option to using anything but the built-in handler. What do we gain other than complexity?

Quoted 6 lines
Quoted 3 lines

See above. If we keep the ability to run off the UEFI #VC handler then we can get continuous testing running inside of Google to verify the UEFI used to launch every SEV VM on Google cloud.

I'm not super opposed to the idea, but I really do think that taking a #VC in guest UEFI is a bug in and of itself.

Understood.

#12 lore raw

Re: [kvm-unit-tests 02/13] x86: AMD SEV-ES: Setup #VC exception handler for AMD SEV-ES

In reply to Marc Orr

Hi Marc,

Quoted 9 lines · Marc Orr

Regarding code review and testing, I can help with the following:
- Compare the patches being pulled into kvm-unit-tests to what's in the Linux kernel and add my Reviewed-by tags if I don't see any meaningful discrepancies.
- Test the entire series on Google's setup, which doesn't use QEMU and add my Tested-by tag accordingly. My previous Tested-by tags were on individual patches. I have not yet tested the entire series.

Please let me know if this is useful. If not, I wouldn't spend the time :-).

I think it is definitly useful to run this in Googles environment too to get it tested and possible bugs ruled out. That can only help the upstream integration of these patches :)

Varad discussed an idea with me today where the core VC handling code could be put into a library and used by the kernel and unit-tests and possibly others as well. The has the benefit that the kvm-unit-tests would also test the kernels VC handler paths. But it is probably something for the future.

Regards,

Signature · 10 lines
Jörg Rödel
[email protected]

SUSE Software Solutions Germany GmbH
Maxfeldstr. 5
90409 Nürnberg
Germany
 
(HRB 36809, AG Nürnberg)
Geschäftsführer: Ivo Totev
#13 lore raw

Re: [kvm-unit-tests 02/13] x86: AMD SEV-ES: Setup #VC exception handler for AMD SEV-ES

Quoted 12 lines · Joerg Roedel

Hi Marc,

Quoted 8 lines · Marc Orr

Regarding code review and testing, I can help with the following:
- Compare the patches being pulled into kvm-unit-tests to what's in the Linux kernel and add my Reviewed-by tags if I don't see any meaningful discrepancies.
- Test the entire series on Google's setup, which doesn't use QEMU and add my Tested-by tag accordingly. My previous Tested-by tags were on individual patches. I have not yet tested the entire series.

Quoted 5 lines
Quoted 1 line

Please let me know if this is useful. If not, I wouldn't spend the time :-).

I think it is definitly useful to run this in Googles environment too to get it tested and possible bugs ruled out. That can only help the upstream integration of these patches :)

SGTM. Thank you for the feedback.

Quoted 5 lines

Varad discussed an idea with me today where the core VC handling code could be put into a library and used by the kernel and unit-tests and possibly others as well. The has the benefit that the kvm-unit-tests would also test the kernels VC handler paths. But it is probably something for the future.

Yes, I like this idea a lot. This is basically what our internal test framework does. So if we can eventually convert this code to be a #VC as a library, that can be used from kvm-unit-tests, Linux kernel, UEFI, etc., that would be great.

#14 lore raw

PATCH 3/13 x86: Move svm.h to lib/x86/

In reply to Varad Gautam

to share common definitions across testcases and lib/.

---
 {x86 => lib/x86}/svm.h | 0
 x86/svm.c              | 2 +-
 x86/svm_tests.c        | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename {x86 => lib/x86}/svm.h (100%)
lib/x86/svm.h renamed from x86/svm.h +0 -0
similarity index 100%
rename from x86/svm.h
rename to lib/x86/svm.h
x86/svm.c +1 -1
index 3f94b2a..7cfef9e 100644
@@ -2,7 +2,7 @@
  * Framework for testing nested virtualization
  */
 
-#include "svm.h"
+#include "x86/svm.h"
 #include "libcflat.h"
 #include "processor.h"
 #include "desc.h"
x86/svm_tests.c +1 -1
index 8ad6122..5cc4642 100644
@@ -1,4 +1,4 @@
-#include "svm.h"
+#include "x86/svm.h"
 #include "libcflat.h"
 #include "processor.h"
 #include "desc.h"
Signature · 1 line
2.32.0
#15 lore raw

PATCH 4/13 lib: x86: Import insn decoder from Linux

In reply to Varad Gautam

Processing #VC exceptions on AMD SEV-ES requires instruction decoding logic to set up the right GHCB state before exiting to the host.

Pull in the instruction decoder from Linux for this purpose.

Origin: Linux 64222515138e43da1fcf288f0289ef1020427b87

---
 lib/x86/insn/inat-tables.c | 1566 ++++++++++++++++++++++++++++++++++++
 lib/x86/insn/inat.c        |   86 ++
 lib/x86/insn/inat.h        |  233 ++++++
 lib/x86/insn/inat_types.h  |   18 +
 lib/x86/insn/insn.c        |  778 ++++++++++++++++++
 lib/x86/insn/insn.h        |  280 +++++++
 x86/Makefile.common        |    2 +
 7 files changed, 2963 insertions(+)
 create mode 100644 lib/x86/insn/inat-tables.c
 create mode 100644 lib/x86/insn/inat.c
 create mode 100644 lib/x86/insn/inat.h
 create mode 100644 lib/x86/insn/inat_types.h
 create mode 100644 lib/x86/insn/insn.c
 create mode 100644 lib/x86/insn/insn.h
lib/x86/insn/inat-tables.c new file +1566 -0
new file mode 100644
index 0000000..3e5fdba
@@ -0,0 +1,1566 @@
+/*
+ * x86 opcode map generated from x86-opcode-map.txt
+ *
+ * Origin: Linux 64222515138e43da1fcf288f0289ef1020427b87
+ *   arch/x86/lib/inat-tables.c
+ */
+
+/* Table: one byte opcode */
+const insn_attr_t inat_primary_table[INAT_OPCODE_TABLE_SIZE] = {
+	[0x00] = INAT_MODRM,
+	[0x01] = INAT_MODRM,
+	[0x02] = INAT_MODRM,
+	[0x03] = INAT_MODRM,
+	[0x04] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x05] = INAT_MAKE_IMM(INAT_IMM_VWORD32),
+	[0x08] = INAT_MODRM,
+	[0x09] = INAT_MODRM,
+	[0x0a] = INAT_MODRM,
+	[0x0b] = INAT_MODRM,
+	[0x0c] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x0d] = INAT_MAKE_IMM(INAT_IMM_VWORD32),
+	[0x0f] = INAT_MAKE_ESCAPE(1),
+	[0x10] = INAT_MODRM,
+	[0x11] = INAT_MODRM,
+	[0x12] = INAT_MODRM,
+	[0x13] = INAT_MODRM,
+	[0x14] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x15] = INAT_MAKE_IMM(INAT_IMM_VWORD32),
+	[0x18] = INAT_MODRM,
+	[0x19] = INAT_MODRM,
+	[0x1a] = INAT_MODRM,
+	[0x1b] = INAT_MODRM,
+	[0x1c] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x1d] = INAT_MAKE_IMM(INAT_IMM_VWORD32),
+	[0x20] = INAT_MODRM,
+	[0x21] = INAT_MODRM,
+	[0x22] = INAT_MODRM,
+	[0x23] = INAT_MODRM,
+	[0x24] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x25] = INAT_MAKE_IMM(INAT_IMM_VWORD32),
+	[0x26] = INAT_MAKE_PREFIX(INAT_PFX_ES),
+	[0x28] = INAT_MODRM,
+	[0x29] = INAT_MODRM,
+	[0x2a] = INAT_MODRM,
+	[0x2b] = INAT_MODRM,
+	[0x2c] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x2d] = INAT_MAKE_IMM(INAT_IMM_VWORD32),
+	[0x2e] = INAT_MAKE_PREFIX(INAT_PFX_CS),
+	[0x30] = INAT_MODRM,
+	[0x31] = INAT_MODRM,
+	[0x32] = INAT_MODRM,
+	[0x33] = INAT_MODRM,
+	[0x34] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x35] = INAT_MAKE_IMM(INAT_IMM_VWORD32),
+	[0x36] = INAT_MAKE_PREFIX(INAT_PFX_SS),
+	[0x38] = INAT_MODRM,
+	[0x39] = INAT_MODRM,
+	[0x3a] = INAT_MODRM,
+	[0x3b] = INAT_MODRM,
+	[0x3c] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x3d] = INAT_MAKE_IMM(INAT_IMM_VWORD32),
+	[0x3e] = INAT_MAKE_PREFIX(INAT_PFX_DS),
+	[0x40] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x41] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x42] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x43] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x44] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x45] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x46] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x47] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x48] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x49] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x4a] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x4b] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x4c] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x4d] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x4e] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x4f] = INAT_MAKE_PREFIX(INAT_PFX_REX),
+	[0x50] = INAT_FORCE64,
+	[0x51] = INAT_FORCE64,
+	[0x52] = INAT_FORCE64,
+	[0x53] = INAT_FORCE64,
+	[0x54] = INAT_FORCE64,
+	[0x55] = INAT_FORCE64,
+	[0x56] = INAT_FORCE64,
+	[0x57] = INAT_FORCE64,
+	[0x58] = INAT_FORCE64,
+	[0x59] = INAT_FORCE64,
+	[0x5a] = INAT_FORCE64,
+	[0x5b] = INAT_FORCE64,
+	[0x5c] = INAT_FORCE64,
+	[0x5d] = INAT_FORCE64,
+	[0x5e] = INAT_FORCE64,
+	[0x5f] = INAT_FORCE64,
+	[0x62] = INAT_MODRM | INAT_MAKE_PREFIX(INAT_PFX_EVEX),
+	[0x63] = INAT_MODRM | INAT_MODRM,
+	[0x64] = INAT_MAKE_PREFIX(INAT_PFX_FS),
+	[0x65] = INAT_MAKE_PREFIX(INAT_PFX_GS),
+	[0x66] = INAT_MAKE_PREFIX(INAT_PFX_OPNDSZ),
+	[0x67] = INAT_MAKE_PREFIX(INAT_PFX_ADDRSZ),
+	[0x68] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x69] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_MODRM,
+	[0x6a] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_FORCE64,
+	[0x6b] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM,
+	[0x70] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x71] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x72] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x73] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x74] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x75] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x76] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x77] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x78] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x79] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x7a] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x7b] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x7c] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x7d] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x7e] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x7f] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0x80] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_MAKE_GROUP(1),
+	[0x81] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_MODRM | INAT_MAKE_GROUP(1),
+	[0x82] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_MAKE_GROUP(1),
+	[0x83] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_MAKE_GROUP(1),
+	[0x84] = INAT_MODRM,
+	[0x85] = INAT_MODRM,
+	[0x86] = INAT_MODRM,
+	[0x87] = INAT_MODRM,
+	[0x88] = INAT_MODRM,
+	[0x89] = INAT_MODRM,
+	[0x8a] = INAT_MODRM,
+	[0x8b] = INAT_MODRM,
+	[0x8c] = INAT_MODRM,
+	[0x8d] = INAT_MODRM,
+	[0x8e] = INAT_MODRM,
+	[0x8f] = INAT_MAKE_GROUP(2) | INAT_MODRM | INAT_FORCE64,
+	[0x9a] = INAT_MAKE_IMM(INAT_IMM_PTR),
+	[0x9c] = INAT_FORCE64,
+	[0x9d] = INAT_FORCE64,
+	[0xa0] = INAT_MOFFSET,
+	[0xa1] = INAT_MOFFSET,
+	[0xa2] = INAT_MOFFSET,
+	[0xa3] = INAT_MOFFSET,
+	[0xa8] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xa9] = INAT_MAKE_IMM(INAT_IMM_VWORD32),
+	[0xb0] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xb1] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xb2] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xb3] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xb4] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xb5] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xb6] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xb7] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xb8] = INAT_MAKE_IMM(INAT_IMM_VWORD),
+	[0xb9] = INAT_MAKE_IMM(INAT_IMM_VWORD),
+	[0xba] = INAT_MAKE_IMM(INAT_IMM_VWORD),
+	[0xbb] = INAT_MAKE_IMM(INAT_IMM_VWORD),
+	[0xbc] = INAT_MAKE_IMM(INAT_IMM_VWORD),
+	[0xbd] = INAT_MAKE_IMM(INAT_IMM_VWORD),
+	[0xbe] = INAT_MAKE_IMM(INAT_IMM_VWORD),
+	[0xbf] = INAT_MAKE_IMM(INAT_IMM_VWORD),
+	[0xc0] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_MAKE_GROUP(3),
+	[0xc1] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_MAKE_GROUP(3),
+	[0xc2] = INAT_MAKE_IMM(INAT_IMM_WORD) | INAT_FORCE64,
+	[0xc4] = INAT_MODRM | INAT_MAKE_PREFIX(INAT_PFX_VEX3),
+	[0xc5] = INAT_MODRM | INAT_MAKE_PREFIX(INAT_PFX_VEX2),
+	[0xc6] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_MAKE_GROUP(4),
+	[0xc7] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_MODRM | INAT_MAKE_GROUP(5),
+	[0xc8] = INAT_MAKE_IMM(INAT_IMM_WORD) | INAT_SCNDIMM,
+	[0xc9] = INAT_FORCE64,
+	[0xca] = INAT_MAKE_IMM(INAT_IMM_WORD),
+	[0xcd] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xd0] = INAT_MODRM | INAT_MAKE_GROUP(3),
+	[0xd1] = INAT_MODRM | INAT_MAKE_GROUP(3),
+	[0xd2] = INAT_MODRM | INAT_MAKE_GROUP(3),
+	[0xd3] = INAT_MODRM | INAT_MAKE_GROUP(3),
+	[0xd4] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xd5] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xd8] = INAT_MODRM,
+	[0xd9] = INAT_MODRM,
+	[0xda] = INAT_MODRM,
+	[0xdb] = INAT_MODRM,
+	[0xdc] = INAT_MODRM,
+	[0xdd] = INAT_MODRM,
+	[0xde] = INAT_MODRM,
+	[0xdf] = INAT_MODRM,
+	[0xe0] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_FORCE64,
+	[0xe1] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_FORCE64,
+	[0xe2] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_FORCE64,
+	[0xe3] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_FORCE64,
+	[0xe4] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xe5] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xe6] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xe7] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+	[0xe8] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0xe9] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0xea] = INAT_MAKE_IMM(INAT_IMM_PTR),
+	[0xeb] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_FORCE64,
+	[0xf0] = INAT_MAKE_PREFIX(INAT_PFX_LOCK),
+	[0xf2] = INAT_MAKE_PREFIX(INAT_PFX_REPNE) | INAT_MAKE_PREFIX(INAT_PFX_REPNE),
+	[0xf3] = INAT_MAKE_PREFIX(INAT_PFX_REPE) | INAT_MAKE_PREFIX(INAT_PFX_REPE),
+	[0xf6] = INAT_MODRM | INAT_MAKE_GROUP(6),
+	[0xf7] = INAT_MODRM | INAT_MAKE_GROUP(7),
+	[0xfe] = INAT_MAKE_GROUP(8),
+	[0xff] = INAT_MAKE_GROUP(9),
+};
+
+/* Table: 2-byte opcode (0x0f) */
+const insn_attr_t inat_escape_table_1[INAT_OPCODE_TABLE_SIZE] = {
+	[0x00] = INAT_MAKE_GROUP(10),
+	[0x01] = INAT_MAKE_GROUP(11),
+	[0x02] = INAT_MODRM,
+	[0x03] = INAT_MODRM,
+	[0x0d] = INAT_MAKE_GROUP(12),
+	[0x0f] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM,
+	[0x10] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x11] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x12] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x13] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x14] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x15] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x16] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x17] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x18] = INAT_MAKE_GROUP(13),
+	[0x1a] = INAT_MODRM | INAT_VARIANT,
+	[0x1b] = INAT_MODRM | INAT_VARIANT,
+	[0x1c] = INAT_MAKE_GROUP(14),
+	[0x1e] = INAT_MAKE_GROUP(15),
+	[0x1f] = INAT_MODRM,
+	[0x20] = INAT_MODRM,
+	[0x21] = INAT_MODRM,
+	[0x22] = INAT_MODRM,
+	[0x23] = INAT_MODRM,
+	[0x28] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x29] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x2a] = INAT_MODRM | INAT_VARIANT,
+	[0x2b] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x2c] = INAT_MODRM | INAT_VARIANT,
+	[0x2d] = INAT_MODRM | INAT_VARIANT,
+	[0x2e] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x2f] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x38] = INAT_MAKE_ESCAPE(2),
+	[0x3a] = INAT_MAKE_ESCAPE(3),
+	[0x40] = INAT_MODRM,
+	[0x41] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x42] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x43] = INAT_MODRM,
+	[0x44] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x45] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x46] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x47] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x48] = INAT_MODRM,
+	[0x49] = INAT_MODRM,
+	[0x4a] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x4b] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x4c] = INAT_MODRM,
+	[0x4d] = INAT_MODRM,
+	[0x4e] = INAT_MODRM,
+	[0x4f] = INAT_MODRM,
+	[0x50] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x51] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x52] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x53] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x54] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x55] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x56] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x57] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x58] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x59] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x5a] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x5b] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x5c] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x5d] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x5e] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x5f] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x60] = INAT_MODRM | INAT_VARIANT,
+	[0x61] = INAT_MODRM | INAT_VARIANT,
+	[0x62] = INAT_MODRM | INAT_VARIANT,
+	[0x63] = INAT_MODRM | INAT_VARIANT,
+	[0x64] = INAT_MODRM | INAT_VARIANT,
+	[0x65] = INAT_MODRM | INAT_VARIANT,
+	[0x66] = INAT_MODRM | INAT_VARIANT,
+	[0x67] = INAT_MODRM | INAT_VARIANT,
+	[0x68] = INAT_MODRM | INAT_VARIANT,
+	[0x69] = INAT_MODRM | INAT_VARIANT,
+	[0x6a] = INAT_MODRM | INAT_VARIANT,
+	[0x6b] = INAT_MODRM | INAT_VARIANT,
+	[0x6c] = INAT_VARIANT,
+	[0x6d] = INAT_VARIANT,
+	[0x6e] = INAT_MODRM | INAT_VARIANT,
+	[0x6f] = INAT_MODRM | INAT_VARIANT,
+	[0x70] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+	[0x71] = INAT_MAKE_GROUP(16),
+	[0x72] = INAT_MAKE_GROUP(17),
+	[0x73] = INAT_MAKE_GROUP(18),
+	[0x74] = INAT_MODRM | INAT_VARIANT,
+	[0x75] = INAT_MODRM | INAT_VARIANT,
+	[0x76] = INAT_MODRM | INAT_VARIANT,
+	[0x77] = INAT_VEXOK | INAT_VEXOK,
+	[0x78] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x79] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x7a] = INAT_VARIANT,
+	[0x7b] = INAT_VARIANT,
+	[0x7c] = INAT_VARIANT,
+	[0x7d] = INAT_VARIANT,
+	[0x7e] = INAT_MODRM | INAT_VARIANT,
+	[0x7f] = INAT_MODRM | INAT_VARIANT,
+	[0x80] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x81] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x82] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x83] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x84] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x85] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x86] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x87] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x88] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x89] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x8a] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x8b] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x8c] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x8d] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x8e] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x8f] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64,
+	[0x90] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x91] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x92] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x93] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x94] = INAT_MODRM,
+	[0x95] = INAT_MODRM,
+	[0x96] = INAT_MODRM,
+	[0x97] = INAT_MODRM,
+	[0x98] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x99] = INAT_MODRM | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x9a] = INAT_MODRM,
+	[0x9b] = INAT_MODRM,
+	[0x9c] = INAT_MODRM,
+	[0x9d] = INAT_MODRM,
+	[0x9e] = INAT_MODRM,
+	[0x9f] = INAT_MODRM,
+	[0xa0] = INAT_FORCE64,
+	[0xa1] = INAT_FORCE64,
+	[0xa3] = INAT_MODRM,
+	[0xa4] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM,
+	[0xa5] = INAT_MODRM,
+	[0xa6] = INAT_MAKE_GROUP(19),
+	[0xa7] = INAT_MAKE_GROUP(20),
+	[0xa8] = INAT_FORCE64,
+	[0xa9] = INAT_FORCE64,
+	[0xab] = INAT_MODRM,
+	[0xac] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM,
+	[0xad] = INAT_MODRM,
+	[0xae] = INAT_MAKE_GROUP(21),
+	[0xaf] = INAT_MODRM,
+	[0xb0] = INAT_MODRM,
+	[0xb1] = INAT_MODRM,
+	[0xb2] = INAT_MODRM,
+	[0xb3] = INAT_MODRM,
+	[0xb4] = INAT_MODRM,
+	[0xb5] = INAT_MODRM,
+	[0xb6] = INAT_MODRM,
+	[0xb7] = INAT_MODRM,
+	[0xb8] = INAT_VARIANT,
+	[0xb9] = INAT_MAKE_GROUP(22),
+	[0xba] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_MAKE_GROUP(23),
+	[0xbb] = INAT_MODRM,
+	[0xbc] = INAT_MODRM | INAT_VARIANT,
+	[0xbd] = INAT_MODRM | INAT_VARIANT,
+	[0xbe] = INAT_MODRM,
+	[0xbf] = INAT_MODRM,
+	[0xc0] = INAT_MODRM,
+	[0xc1] = INAT_MODRM,
+	[0xc2] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0xc3] = INAT_MODRM,
+	[0xc4] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+	[0xc5] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+	[0xc6] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0xc7] = INAT_MAKE_GROUP(24),
+	[0xd0] = INAT_VARIANT,
+	[0xd1] = INAT_MODRM | INAT_VARIANT,
+	[0xd2] = INAT_MODRM | INAT_VARIANT,
+	[0xd3] = INAT_MODRM | INAT_VARIANT,
+	[0xd4] = INAT_MODRM | INAT_VARIANT,
+	[0xd5] = INAT_MODRM | INAT_VARIANT,
+	[0xd6] = INAT_VARIANT,
+	[0xd7] = INAT_MODRM | INAT_VARIANT,
+	[0xd8] = INAT_MODRM | INAT_VARIANT,
+	[0xd9] = INAT_MODRM | INAT_VARIANT,
+	[0xda] = INAT_MODRM | INAT_VARIANT,
+	[0xdb] = INAT_MODRM | INAT_VARIANT,
+	[0xdc] = INAT_MODRM | INAT_VARIANT,
+	[0xdd] = INAT_MODRM | INAT_VARIANT,
+	[0xde] = INAT_MODRM | INAT_VARIANT,
+	[0xdf] = INAT_MODRM | INAT_VARIANT,
+	[0xe0] = INAT_MODRM | INAT_VARIANT,
+	[0xe1] = INAT_MODRM | INAT_VARIANT,
+	[0xe2] = INAT_MODRM | INAT_VARIANT,
+	[0xe3] = INAT_MODRM | INAT_VARIANT,
+	[0xe4] = INAT_MODRM | INAT_VARIANT,
+	[0xe5] = INAT_MODRM | INAT_VARIANT,
+	[0xe6] = INAT_VARIANT,
+	[0xe7] = INAT_MODRM | INAT_VARIANT,
+	[0xe8] = INAT_MODRM | INAT_VARIANT,
+	[0xe9] = INAT_MODRM | INAT_VARIANT,
+	[0xea] = INAT_MODRM | INAT_VARIANT,
+	[0xeb] = INAT_MODRM | INAT_VARIANT,
+	[0xec] = INAT_MODRM | INAT_VARIANT,
+	[0xed] = INAT_MODRM | INAT_VARIANT,
+	[0xee] = INAT_MODRM | INAT_VARIANT,
+	[0xef] = INAT_MODRM | INAT_VARIANT,
+	[0xf0] = INAT_VARIANT,
+	[0xf1] = INAT_MODRM | INAT_VARIANT,
+	[0xf2] = INAT_MODRM | INAT_VARIANT,
+	[0xf3] = INAT_MODRM | INAT_VARIANT,
+	[0xf4] = INAT_MODRM | INAT_VARIANT,
+	[0xf5] = INAT_MODRM | INAT_VARIANT,
+	[0xf6] = INAT_MODRM | INAT_VARIANT,
+	[0xf7] = INAT_MODRM | INAT_VARIANT,
+	[0xf8] = INAT_MODRM | INAT_VARIANT,
+	[0xf9] = INAT_MODRM | INAT_VARIANT,
+	[0xfa] = INAT_MODRM | INAT_VARIANT,
+	[0xfb] = INAT_MODRM | INAT_VARIANT,
+	[0xfc] = INAT_MODRM | INAT_VARIANT,
+	[0xfd] = INAT_MODRM | INAT_VARIANT,
+	[0xfe] = INAT_MODRM | INAT_VARIANT,
+};
+const insn_attr_t inat_escape_table_1_1[INAT_OPCODE_TABLE_SIZE] = {
+	[0x10] = INAT_MODRM | INAT_VEXOK,
+	[0x11] = INAT_MODRM | INAT_VEXOK,
+	[0x12] = INAT_MODRM | INAT_VEXOK,
+	[0x13] = INAT_MODRM | INAT_VEXOK,
+	[0x14] = INAT_MODRM | INAT_VEXOK,
+	[0x15] = INAT_MODRM | INAT_VEXOK,
+	[0x16] = INAT_MODRM | INAT_VEXOK,
+	[0x17] = INAT_MODRM | INAT_VEXOK,
+	[0x1a] = INAT_MODRM,
+	[0x1b] = INAT_MODRM,
+	[0x28] = INAT_MODRM | INAT_VEXOK,
+	[0x29] = INAT_MODRM | INAT_VEXOK,
+	[0x2a] = INAT_MODRM,
+	[0x2b] = INAT_MODRM | INAT_VEXOK,
+	[0x2c] = INAT_MODRM,
+	[0x2d] = INAT_MODRM,
+	[0x2e] = INAT_MODRM | INAT_VEXOK,
+	[0x2f] = INAT_MODRM | INAT_VEXOK,
+	[0x41] = INAT_MODRM | INAT_VEXOK,
+	[0x42] = INAT_MODRM | INAT_VEXOK,
+	[0x44] = INAT_MODRM | INAT_VEXOK,
+	[0x45] = INAT_MODRM | INAT_VEXOK,
+	[0x46] = INAT_MODRM | INAT_VEXOK,
+	[0x47] = INAT_MODRM | INAT_VEXOK,
+	[0x4a] = INAT_MODRM | INAT_VEXOK,
+	[0x4b] = INAT_MODRM | INAT_VEXOK,
+	[0x50] = INAT_MODRM | INAT_VEXOK,
+	[0x51] = INAT_MODRM | INAT_VEXOK,
+	[0x54] = INAT_MODRM | INAT_VEXOK,
+	[0x55] = INAT_MODRM | INAT_VEXOK,
+	[0x56] = INAT_MODRM | INAT_VEXOK,
+	[0x57] = INAT_MODRM | INAT_VEXOK,
+	[0x58] = INAT_MODRM | INAT_VEXOK,
+	[0x59] = INAT_MODRM | INAT_VEXOK,
+	[0x5a] = INAT_MODRM | INAT_VEXOK,
+	[0x5b] = INAT_MODRM | INAT_VEXOK,
+	[0x5c] = INAT_MODRM | INAT_VEXOK,
+	[0x5d] = INAT_MODRM | INAT_VEXOK,
+	[0x5e] = INAT_MODRM | INAT_VEXOK,
+	[0x5f] = INAT_MODRM | INAT_VEXOK,
+	[0x60] = INAT_MODRM | INAT_VEXOK,
+	[0x61] = INAT_MODRM | INAT_VEXOK,
+	[0x62] = INAT_MODRM | INAT_VEXOK,
+	[0x63] = INAT_MODRM | INAT_VEXOK,
+	[0x64] = INAT_MODRM | INAT_VEXOK,
+	[0x65] = INAT_MODRM | INAT_VEXOK,
+	[0x66] = INAT_MODRM | INAT_VEXOK,
+	[0x67] = INAT_MODRM | INAT_VEXOK,
+	[0x68] = INAT_MODRM | INAT_VEXOK,
+	[0x69] = INAT_MODRM | INAT_VEXOK,
+	[0x6a] = INAT_MODRM | INAT_VEXOK,
+	[0x6b] = INAT_MODRM | INAT_VEXOK,
+	[0x6c] = INAT_MODRM | INAT_VEXOK,
+	[0x6d] = INAT_MODRM | INAT_VEXOK,
+	[0x6e] = INAT_MODRM | INAT_VEXOK,
+	[0x6f] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0x70] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x74] = INAT_MODRM | INAT_VEXOK,
+	[0x75] = INAT_MODRM | INAT_VEXOK,
+	[0x76] = INAT_MODRM | INAT_VEXOK,
+	[0x78] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x79] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7a] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7b] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7c] = INAT_MODRM | INAT_VEXOK,
+	[0x7d] = INAT_MODRM | INAT_VEXOK,
+	[0x7e] = INAT_MODRM | INAT_VEXOK,
+	[0x7f] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0x90] = INAT_MODRM | INAT_VEXOK,
+	[0x91] = INAT_MODRM | INAT_VEXOK,
+	[0x92] = INAT_MODRM | INAT_VEXOK,
+	[0x93] = INAT_MODRM | INAT_VEXOK,
+	[0x98] = INAT_MODRM | INAT_VEXOK,
+	[0x99] = INAT_MODRM | INAT_VEXOK,
+	[0xbc] = INAT_MODRM,
+	[0xbd] = INAT_MODRM,
+	[0xc2] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0xc4] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0xc5] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0xc6] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0xd0] = INAT_MODRM | INAT_VEXOK,
+	[0xd1] = INAT_MODRM | INAT_VEXOK,
+	[0xd2] = INAT_MODRM | INAT_VEXOK,
+	[0xd3] = INAT_MODRM | INAT_VEXOK,
+	[0xd4] = INAT_MODRM | INAT_VEXOK,
+	[0xd5] = INAT_MODRM | INAT_VEXOK,
+	[0xd6] = INAT_MODRM | INAT_VEXOK,
+	[0xd7] = INAT_MODRM | INAT_VEXOK,
+	[0xd8] = INAT_MODRM | INAT_VEXOK,
+	[0xd9] = INAT_MODRM | INAT_VEXOK,
+	[0xda] = INAT_MODRM | INAT_VEXOK,
+	[0xdb] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0xdc] = INAT_MODRM | INAT_VEXOK,
+	[0xdd] = INAT_MODRM | INAT_VEXOK,
+	[0xde] = INAT_MODRM | INAT_VEXOK,
+	[0xdf] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0xe0] = INAT_MODRM | INAT_VEXOK,
+	[0xe1] = INAT_MODRM | INAT_VEXOK,
+	[0xe2] = INAT_MODRM | INAT_VEXOK,
+	[0xe3] = INAT_MODRM | INAT_VEXOK,
+	[0xe4] = INAT_MODRM | INAT_VEXOK,
+	[0xe5] = INAT_MODRM | INAT_VEXOK,
+	[0xe6] = INAT_MODRM | INAT_VEXOK,
+	[0xe7] = INAT_MODRM | INAT_VEXOK,
+	[0xe8] = INAT_MODRM | INAT_VEXOK,
+	[0xe9] = INAT_MODRM | INAT_VEXOK,
+	[0xea] = INAT_MODRM | INAT_VEXOK,
+	[0xeb] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0xec] = INAT_MODRM | INAT_VEXOK,
+	[0xed] = INAT_MODRM | INAT_VEXOK,
+	[0xee] = INAT_MODRM | INAT_VEXOK,
+	[0xef] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0xf1] = INAT_MODRM | INAT_VEXOK,
+	[0xf2] = INAT_MODRM | INAT_VEXOK,
+	[0xf3] = INAT_MODRM | INAT_VEXOK,
+	[0xf4] = INAT_MODRM | INAT_VEXOK,
+	[0xf5] = INAT_MODRM | INAT_VEXOK,
+	[0xf6] = INAT_MODRM | INAT_VEXOK,
+	[0xf7] = INAT_MODRM | INAT_VEXOK,
+	[0xf8] = INAT_MODRM | INAT_VEXOK,
+	[0xf9] = INAT_MODRM | INAT_VEXOK,
+	[0xfa] = INAT_MODRM | INAT_VEXOK,
+	[0xfb] = INAT_MODRM | INAT_VEXOK,
+	[0xfc] = INAT_MODRM | INAT_VEXOK,
+	[0xfd] = INAT_MODRM | INAT_VEXOK,
+	[0xfe] = INAT_MODRM | INAT_VEXOK,
+};
+const insn_attr_t inat_escape_table_1_2[INAT_OPCODE_TABLE_SIZE] = {
+	[0x10] = INAT_MODRM | INAT_VEXOK,
+	[0x11] = INAT_MODRM | INAT_VEXOK,
+	[0x12] = INAT_MODRM | INAT_VEXOK,
+	[0x16] = INAT_MODRM | INAT_VEXOK,
+	[0x1a] = INAT_MODRM,
+	[0x1b] = INAT_MODRM,
+	[0x2a] = INAT_MODRM | INAT_VEXOK,
+	[0x2c] = INAT_MODRM | INAT_VEXOK,
+	[0x2d] = INAT_MODRM | INAT_VEXOK,
+	[0x51] = INAT_MODRM | INAT_VEXOK,
+	[0x52] = INAT_MODRM | INAT_VEXOK,
+	[0x53] = INAT_MODRM | INAT_VEXOK,
+	[0x58] = INAT_MODRM | INAT_VEXOK,
+	[0x59] = INAT_MODRM | INAT_VEXOK,
+	[0x5a] = INAT_MODRM | INAT_VEXOK,
+	[0x5b] = INAT_MODRM | INAT_VEXOK,
+	[0x5c] = INAT_MODRM | INAT_VEXOK,
+	[0x5d] = INAT_MODRM | INAT_VEXOK,
+	[0x5e] = INAT_MODRM | INAT_VEXOK,
+	[0x5f] = INAT_MODRM | INAT_VEXOK,
+	[0x6f] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0x70] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x78] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x79] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7a] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7b] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7e] = INAT_MODRM | INAT_VEXOK,
+	[0x7f] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0xb8] = INAT_MODRM,
+	[0xbc] = INAT_MODRM,
+	[0xbd] = INAT_MODRM,
+	[0xc2] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0xd6] = INAT_MODRM,
+	[0xe6] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+};
+const insn_attr_t inat_escape_table_1_3[INAT_OPCODE_TABLE_SIZE] = {
+	[0x10] = INAT_MODRM | INAT_VEXOK,
+	[0x11] = INAT_MODRM | INAT_VEXOK,
+	[0x12] = INAT_MODRM | INAT_VEXOK,
+	[0x1a] = INAT_MODRM,
+	[0x1b] = INAT_MODRM,
+	[0x2a] = INAT_MODRM | INAT_VEXOK,
+	[0x2c] = INAT_MODRM | INAT_VEXOK,
+	[0x2d] = INAT_MODRM | INAT_VEXOK,
+	[0x51] = INAT_MODRM | INAT_VEXOK,
+	[0x58] = INAT_MODRM | INAT_VEXOK,
+	[0x59] = INAT_MODRM | INAT_VEXOK,
+	[0x5a] = INAT_MODRM | INAT_VEXOK,
+	[0x5c] = INAT_MODRM | INAT_VEXOK,
+	[0x5d] = INAT_MODRM | INAT_VEXOK,
+	[0x5e] = INAT_MODRM | INAT_VEXOK,
+	[0x5f] = INAT_MODRM | INAT_VEXOK,
+	[0x6f] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x70] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x78] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x79] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7a] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7b] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7c] = INAT_MODRM | INAT_VEXOK,
+	[0x7d] = INAT_MODRM | INAT_VEXOK,
+	[0x7f] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x92] = INAT_MODRM | INAT_VEXOK,
+	[0x93] = INAT_MODRM | INAT_VEXOK,
+	[0xbc] = INAT_MODRM,
+	[0xbd] = INAT_MODRM,
+	[0xc2] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0xd0] = INAT_MODRM | INAT_VEXOK,
+	[0xd6] = INAT_MODRM,
+	[0xe6] = INAT_MODRM | INAT_VEXOK,
+	[0xf0] = INAT_MODRM | INAT_VEXOK,
+};
+
+/* Table: 3-byte opcode 1 (0x0f 0x38) */
+const insn_attr_t inat_escape_table_2[INAT_OPCODE_TABLE_SIZE] = {
+	[0x00] = INAT_MODRM | INAT_VARIANT,
+	[0x01] = INAT_MODRM | INAT_VARIANT,
+	[0x02] = INAT_MODRM | INAT_VARIANT,
+	[0x03] = INAT_MODRM | INAT_VARIANT,
+	[0x04] = INAT_MODRM | INAT_VARIANT,
+	[0x05] = INAT_MODRM | INAT_VARIANT,
+	[0x06] = INAT_MODRM | INAT_VARIANT,
+	[0x07] = INAT_MODRM | INAT_VARIANT,
+	[0x08] = INAT_MODRM | INAT_VARIANT,
+	[0x09] = INAT_MODRM | INAT_VARIANT,
+	[0x0a] = INAT_MODRM | INAT_VARIANT,
+	[0x0b] = INAT_MODRM | INAT_VARIANT,
+	[0x0c] = INAT_VARIANT,
+	[0x0d] = INAT_VARIANT,
+	[0x0e] = INAT_VARIANT,
+	[0x0f] = INAT_VARIANT,
+	[0x10] = INAT_VARIANT,
+	[0x11] = INAT_VARIANT,
+	[0x12] = INAT_VARIANT,
+	[0x13] = INAT_VARIANT,
+	[0x14] = INAT_VARIANT,
+	[0x15] = INAT_VARIANT,
+	[0x16] = INAT_VARIANT,
+	[0x17] = INAT_VARIANT,
+	[0x18] = INAT_VARIANT,
+	[0x19] = INAT_VARIANT,
+	[0x1a] = INAT_VARIANT,
+	[0x1b] = INAT_VARIANT,
+	[0x1c] = INAT_MODRM | INAT_VARIANT,
+	[0x1d] = INAT_MODRM | INAT_VARIANT,
+	[0x1e] = INAT_MODRM | INAT_VARIANT,
+	[0x1f] = INAT_VARIANT,
+	[0x20] = INAT_VARIANT,
+	[0x21] = INAT_VARIANT,
+	[0x22] = INAT_VARIANT,
+	[0x23] = INAT_VARIANT,
+	[0x24] = INAT_VARIANT,
+	[0x25] = INAT_VARIANT,
+	[0x26] = INAT_VARIANT,
+	[0x27] = INAT_VARIANT,
+	[0x28] = INAT_VARIANT,
+	[0x29] = INAT_VARIANT,
+	[0x2a] = INAT_VARIANT,
+	[0x2b] = INAT_VARIANT,
+	[0x2c] = INAT_VARIANT,
+	[0x2d] = INAT_VARIANT,
+	[0x2e] = INAT_VARIANT,
+	[0x2f] = INAT_VARIANT,
+	[0x30] = INAT_VARIANT,
+	[0x31] = INAT_VARIANT,
+	[0x32] = INAT_VARIANT,
+	[0x33] = INAT_VARIANT,
+	[0x34] = INAT_VARIANT,
+	[0x35] = INAT_VARIANT,
+	[0x36] = INAT_VARIANT,
+	[0x37] = INAT_VARIANT,
+	[0x38] = INAT_VARIANT,
+	[0x39] = INAT_VARIANT,
+	[0x3a] = INAT_VARIANT,
+	[0x3b] = INAT_VARIANT,
+	[0x3c] = INAT_VARIANT,
+	[0x3d] = INAT_VARIANT,
+	[0x3e] = INAT_VARIANT,
+	[0x3f] = INAT_VARIANT,
+	[0x40] = INAT_VARIANT,
+	[0x41] = INAT_VARIANT,
+	[0x42] = INAT_VARIANT,
+	[0x43] = INAT_VARIANT,
+	[0x44] = INAT_VARIANT,
+	[0x45] = INAT_VARIANT,
+	[0x46] = INAT_VARIANT,
+	[0x47] = INAT_VARIANT,
+	[0x4c] = INAT_VARIANT,
+	[0x4d] = INAT_VARIANT,
+	[0x4e] = INAT_VARIANT,
+	[0x4f] = INAT_VARIANT,
+	[0x50] = INAT_VARIANT,
+	[0x51] = INAT_VARIANT,
+	[0x52] = INAT_VARIANT,
+	[0x53] = INAT_VARIANT,
+	[0x54] = INAT_VARIANT,
+	[0x55] = INAT_VARIANT,
+	[0x58] = INAT_VARIANT,
+	[0x59] = INAT_VARIANT,
+	[0x5a] = INAT_VARIANT,
+	[0x5b] = INAT_VARIANT,
+	[0x62] = INAT_VARIANT,
+	[0x63] = INAT_VARIANT,
+	[0x64] = INAT_VARIANT,
+	[0x65] = INAT_VARIANT,
+	[0x66] = INAT_VARIANT,
+	[0x68] = INAT_VARIANT,
+	[0x70] = INAT_VARIANT,
+	[0x71] = INAT_VARIANT,
+	[0x72] = INAT_VARIANT,
+	[0x73] = INAT_VARIANT,
+	[0x75] = INAT_VARIANT,
+	[0x76] = INAT_VARIANT,
+	[0x77] = INAT_VARIANT,
+	[0x78] = INAT_VARIANT,
+	[0x79] = INAT_VARIANT,
+	[0x7a] = INAT_VARIANT,
+	[0x7b] = INAT_VARIANT,
+	[0x7c] = INAT_VARIANT,
+	[0x7d] = INAT_VARIANT,
+	[0x7e] = INAT_VARIANT,
+	[0x7f] = INAT_VARIANT,
+	[0x80] = INAT_VARIANT,
+	[0x81] = INAT_VARIANT,
+	[0x82] = INAT_VARIANT,
+	[0x83] = INAT_VARIANT,
+	[0x88] = INAT_VARIANT,
+	[0x89] = INAT_VARIANT,
+	[0x8a] = INAT_VARIANT,
+	[0x8b] = INAT_VARIANT,
+	[0x8c] = INAT_VARIANT,
+	[0x8d] = INAT_VARIANT,
+	[0x8e] = INAT_VARIANT,
+	[0x8f] = INAT_VARIANT,
+	[0x90] = INAT_VARIANT,
+	[0x91] = INAT_VARIANT,
+	[0x92] = INAT_VARIANT,
+	[0x93] = INAT_VARIANT,
+	[0x96] = INAT_VARIANT,
+	[0x97] = INAT_VARIANT,
+	[0x98] = INAT_VARIANT,
+	[0x99] = INAT_VARIANT,
+	[0x9a] = INAT_VARIANT,
+	[0x9b] = INAT_VARIANT,
+	[0x9c] = INAT_VARIANT,
+	[0x9d] = INAT_VARIANT,
+	[0x9e] = INAT_VARIANT,
+	[0x9f] = INAT_VARIANT,
+	[0xa0] = INAT_VARIANT,
+	[0xa1] = INAT_VARIANT,
+	[0xa2] = INAT_VARIANT,
+	[0xa3] = INAT_VARIANT,
+	[0xa6] = INAT_VARIANT,
+	[0xa7] = INAT_VARIANT,
+	[0xa8] = INAT_VARIANT,
+	[0xa9] = INAT_VARIANT,
+	[0xaa] = INAT_VARIANT,
+	[0xab] = INAT_VARIANT,
+	[0xac] = INAT_VARIANT,
+	[0xad] = INAT_VARIANT,
+	[0xae] = INAT_VARIANT,
+	[0xaf] = INAT_VARIANT,
+	[0xb4] = INAT_VARIANT,
+	[0xb5] = INAT_VARIANT,
+	[0xb6] = INAT_VARIANT,
+	[0xb7] = INAT_VARIANT,
+	[0xb8] = INAT_VARIANT,
+	[0xb9] = INAT_VARIANT,
+	[0xba] = INAT_VARIANT,
+	[0xbb] = INAT_VARIANT,
+	[0xbc] = INAT_VARIANT,
+	[0xbd] = INAT_VARIANT,
+	[0xbe] = INAT_VARIANT,
+	[0xbf] = INAT_VARIANT,
+	[0xc4] = INAT_VARIANT,
+	[0xc6] = INAT_MAKE_GROUP(25),
+	[0xc7] = INAT_MAKE_GROUP(26),
+	[0xc8] = INAT_MODRM | INAT_VARIANT,
+	[0xc9] = INAT_MODRM,
+	[0xca] = INAT_MODRM | INAT_VARIANT,
+	[0xcb] = INAT_MODRM | INAT_VARIANT,
+	[0xcc] = INAT_MODRM | INAT_VARIANT,
+	[0xcd] = INAT_MODRM | INAT_VARIANT,
+	[0xcf] = INAT_VARIANT,
+	[0xdb] = INAT_VARIANT,
+	[0xdc] = INAT_VARIANT,
+	[0xdd] = INAT_VARIANT,
+	[0xde] = INAT_VARIANT,
+	[0xdf] = INAT_VARIANT,
+	[0xf0] = INAT_MODRM | INAT_MODRM | INAT_VARIANT,
+	[0xf1] = INAT_MODRM | INAT_MODRM | INAT_VARIANT,
+	[0xf2] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xf3] = INAT_MAKE_GROUP(27),
+	[0xf5] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_VARIANT,
+	[0xf6] = INAT_MODRM | INAT_VARIANT,
+	[0xf7] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_VARIANT,
+	[0xf8] = INAT_VARIANT,
+	[0xf9] = INAT_MODRM,
+};
+const insn_attr_t inat_escape_table_2_1[INAT_OPCODE_TABLE_SIZE] = {
+	[0x00] = INAT_MODRM | INAT_VEXOK,
+	[0x01] = INAT_MODRM | INAT_VEXOK,
+	[0x02] = INAT_MODRM | INAT_VEXOK,
+	[0x03] = INAT_MODRM | INAT_VEXOK,
+	[0x04] = INAT_MODRM | INAT_VEXOK,
+	[0x05] = INAT_MODRM | INAT_VEXOK,
+	[0x06] = INAT_MODRM | INAT_VEXOK,
+	[0x07] = INAT_MODRM | INAT_VEXOK,
+	[0x08] = INAT_MODRM | INAT_VEXOK,
+	[0x09] = INAT_MODRM | INAT_VEXOK,
+	[0x0a] = INAT_MODRM | INAT_VEXOK,
+	[0x0b] = INAT_MODRM | INAT_VEXOK,
+	[0x0c] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x0d] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x0e] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x0f] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x10] = INAT_MODRM | INAT_MODRM | INAT_VEXOK,
+	[0x11] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x12] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x13] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x14] = INAT_MODRM | INAT_MODRM | INAT_VEXOK,
+	[0x15] = INAT_MODRM | INAT_MODRM | INAT_VEXOK,
+	[0x16] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MODRM | INAT_VEXOK,
+	[0x17] = INAT_MODRM | INAT_VEXOK,
+	[0x18] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x19] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MODRM | INAT_VEXOK,
+	[0x1a] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MODRM | INAT_VEXOK,
+	[0x1b] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x1c] = INAT_MODRM | INAT_VEXOK,
+	[0x1d] = INAT_MODRM | INAT_VEXOK,
+	[0x1e] = INAT_MODRM | INAT_VEXOK,
+	[0x1f] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x20] = INAT_MODRM | INAT_VEXOK,
+	[0x21] = INAT_MODRM | INAT_VEXOK,
+	[0x22] = INAT_MODRM | INAT_VEXOK,
+	[0x23] = INAT_MODRM | INAT_VEXOK,
+	[0x24] = INAT_MODRM | INAT_VEXOK,
+	[0x25] = INAT_MODRM | INAT_VEXOK,
+	[0x26] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x27] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x28] = INAT_MODRM | INAT_VEXOK,
+	[0x29] = INAT_MODRM | INAT_VEXOK,
+	[0x2a] = INAT_MODRM | INAT_VEXOK,
+	[0x2b] = INAT_MODRM | INAT_VEXOK,
+	[0x2c] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MODRM | INAT_VEXOK,
+	[0x2d] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MODRM | INAT_VEXOK,
+	[0x2e] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x2f] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x30] = INAT_MODRM | INAT_VEXOK,
+	[0x31] = INAT_MODRM | INAT_VEXOK,
+	[0x32] = INAT_MODRM | INAT_VEXOK,
+	[0x33] = INAT_MODRM | INAT_VEXOK,
+	[0x34] = INAT_MODRM | INAT_VEXOK,
+	[0x35] = INAT_MODRM | INAT_VEXOK,
+	[0x36] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MODRM | INAT_VEXOK,
+	[0x37] = INAT_MODRM | INAT_VEXOK,
+	[0x38] = INAT_MODRM | INAT_VEXOK,
+	[0x39] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0x3a] = INAT_MODRM | INAT_VEXOK,
+	[0x3b] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0x3c] = INAT_MODRM | INAT_VEXOK,
+	[0x3d] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0x3e] = INAT_MODRM | INAT_VEXOK,
+	[0x3f] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0x40] = INAT_MODRM | INAT_VEXOK | INAT_MODRM | INAT_VEXOK,
+	[0x41] = INAT_MODRM | INAT_VEXOK,
+	[0x42] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x43] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x44] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x45] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x46] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MODRM | INAT_VEXOK,
+	[0x47] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x4c] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x4d] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x4e] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x4f] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x50] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x51] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x52] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x53] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x54] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x55] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x58] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x59] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MODRM | INAT_VEXOK,
+	[0x5a] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MODRM | INAT_VEXOK,
+	[0x5b] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x62] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x63] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x64] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x65] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x66] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x70] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x71] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x72] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x73] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x75] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x76] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x77] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x78] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x79] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x7a] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7b] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7c] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7d] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7e] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x7f] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x80] = INAT_MODRM,
+	[0x81] = INAT_MODRM,
+	[0x82] = INAT_MODRM,
+	[0x83] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x88] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x89] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x8a] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x8b] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x8c] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x8d] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x8e] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x8f] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x90] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MODRM | INAT_VEXOK,
+	[0x91] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MODRM | INAT_VEXOK,
+	[0x92] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x93] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x96] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x97] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x98] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x99] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x9a] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x9b] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x9c] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x9d] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x9e] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x9f] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xa0] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xa1] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xa2] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xa3] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xa6] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xa7] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xa8] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xa9] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xaa] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xab] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xac] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xad] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xae] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xaf] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xb4] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xb5] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xb6] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xb7] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xb8] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xb9] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xba] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xbb] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xbc] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xbd] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xbe] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xbf] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xc4] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xc8] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xca] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xcb] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xcc] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xcd] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xcf] = INAT_MODRM | INAT_VEXOK,
+	[0xdb] = INAT_MODRM | INAT_VEXOK,
+	[0xdc] = INAT_MODRM | INAT_VEXOK,
+	[0xdd] = INAT_MODRM | INAT_VEXOK,
+	[0xde] = INAT_MODRM | INAT_VEXOK,
+	[0xdf] = INAT_MODRM | INAT_VEXOK,
+	[0xf0] = INAT_MODRM,
+	[0xf1] = INAT_MODRM,
+	[0xf5] = INAT_MODRM,
+	[0xf6] = INAT_MODRM,
+	[0xf7] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xf8] = INAT_MODRM,
+};
+const insn_attr_t inat_escape_table_2_2[INAT_OPCODE_TABLE_SIZE] = {
+	[0x10] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x11] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x12] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x13] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x14] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x15] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x20] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x21] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x22] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x23] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x24] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x25] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x26] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x27] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x28] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x29] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x2a] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x30] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x31] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x32] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x33] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x34] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x35] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x38] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x39] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x3a] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x52] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x72] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xf5] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xf6] = INAT_MODRM,
+	[0xf7] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xf8] = INAT_MODRM,
+};
+const insn_attr_t inat_escape_table_2_3[INAT_OPCODE_TABLE_SIZE] = {
+	[0x52] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x53] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x68] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x72] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x9a] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x9b] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xaa] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xab] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xf0] = INAT_MODRM | INAT_MODRM,
+	[0xf1] = INAT_MODRM | INAT_MODRM,
+	[0xf5] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xf6] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xf7] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0xf8] = INAT_MODRM,
+};
+
+/* Table: 3-byte opcode 2 (0x0f 0x3a) */
+const insn_attr_t inat_escape_table_3[INAT_OPCODE_TABLE_SIZE] = {
+	[0x00] = INAT_VARIANT,
+	[0x01] = INAT_VARIANT,
+	[0x02] = INAT_VARIANT,
+	[0x03] = INAT_VARIANT,
+	[0x04] = INAT_VARIANT,
+	[0x05] = INAT_VARIANT,
+	[0x06] = INAT_VARIANT,
+	[0x08] = INAT_VARIANT,
+	[0x09] = INAT_VARIANT,
+	[0x0a] = INAT_VARIANT,
+	[0x0b] = INAT_VARIANT,
+	[0x0c] = INAT_VARIANT,
+	[0x0d] = INAT_VARIANT,
+	[0x0e] = INAT_VARIANT,
+	[0x0f] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+	[0x14] = INAT_VARIANT,
+	[0x15] = INAT_VARIANT,
+	[0x16] = INAT_VARIANT,
+	[0x17] = INAT_VARIANT,
+	[0x18] = INAT_VARIANT,
+	[0x19] = INAT_VARIANT,
+	[0x1a] = INAT_VARIANT,
+	[0x1b] = INAT_VARIANT,
+	[0x1d] = INAT_VARIANT,
+	[0x1e] = INAT_VARIANT,
+	[0x1f] = INAT_VARIANT,
+	[0x20] = INAT_VARIANT,
+	[0x21] = INAT_VARIANT,
+	[0x22] = INAT_VARIANT,
+	[0x23] = INAT_VARIANT,
+	[0x25] = INAT_VARIANT,
+	[0x26] = INAT_VARIANT,
+	[0x27] = INAT_VARIANT,
+	[0x30] = INAT_VARIANT,
+	[0x31] = INAT_VARIANT,
+	[0x32] = INAT_VARIANT,
+	[0x33] = INAT_VARIANT,
+	[0x38] = INAT_VARIANT,
+	[0x39] = INAT_VARIANT,
+	[0x3a] = INAT_VARIANT,
+	[0x3b] = INAT_VARIANT,
+	[0x3e] = INAT_VARIANT,
+	[0x3f] = INAT_VARIANT,
+	[0x40] = INAT_VARIANT,
+	[0x41] = INAT_VARIANT,
+	[0x42] = INAT_VARIANT,
+	[0x43] = INAT_VARIANT,
+	[0x44] = INAT_VARIANT,
+	[0x46] = INAT_VARIANT,
+	[0x4a] = INAT_VARIANT,
+	[0x4b] = INAT_VARIANT,
+	[0x4c] = INAT_VARIANT,
+	[0x50] = INAT_VARIANT,
+	[0x51] = INAT_VARIANT,
+	[0x54] = INAT_VARIANT,
+	[0x55] = INAT_VARIANT,
+	[0x56] = INAT_VARIANT,
+	[0x57] = INAT_VARIANT,
+	[0x60] = INAT_VARIANT,
+	[0x61] = INAT_VARIANT,
+	[0x62] = INAT_VARIANT,
+	[0x63] = INAT_VARIANT,
+	[0x66] = INAT_VARIANT,
+	[0x67] = INAT_VARIANT,
+	[0x70] = INAT_VARIANT,
+	[0x71] = INAT_VARIANT,
+	[0x72] = INAT_VARIANT,
+	[0x73] = INAT_VARIANT,
+	[0xcc] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM,
+	[0xce] = INAT_VARIANT,
+	[0xcf] = INAT_VARIANT,
+	[0xdf] = INAT_VARIANT,
+	[0xf0] = INAT_VARIANT,
+};
+const insn_attr_t inat_escape_table_3_1[INAT_OPCODE_TABLE_SIZE] = {
+	[0x00] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x01] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x02] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x03] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x04] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x05] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x06] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x08] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x09] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x0a] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x0b] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x0c] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x0d] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x0e] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x0f] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x14] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x15] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x16] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x17] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x18] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x19] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x1a] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x1b] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x1d] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x1e] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x1f] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x20] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x21] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x22] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x23] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x25] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x26] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x27] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x30] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x31] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x32] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x33] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x38] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x39] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY | INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x3a] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x3b] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x3e] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x3f] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x40] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x41] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x42] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x43] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x44] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x46] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x4a] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x4b] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x4c] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x50] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x51] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x54] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x55] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x56] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x57] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x60] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x61] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x62] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x63] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x66] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x67] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x70] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x71] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x72] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x73] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0xce] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0xcf] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0xdf] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+};
+const insn_attr_t inat_escape_table_3_3[INAT_OPCODE_TABLE_SIZE] = {
+	[0xf0] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+};
+
+/* GrpTable: Grp1 */
+
+/* GrpTable: Grp1A */
+
+/* GrpTable: Grp2 */
+
+/* GrpTable: Grp3_1 */
+const insn_attr_t inat_group_table_6[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM,
+	[0x1] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM,
+	[0x2] = INAT_MODRM,
+	[0x3] = INAT_MODRM,
+	[0x4] = INAT_MODRM,
+	[0x5] = INAT_MODRM,
+	[0x6] = INAT_MODRM,
+	[0x7] = INAT_MODRM,
+};
+
+/* GrpTable: Grp3_2 */
+const insn_attr_t inat_group_table_7[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_MODRM,
+	[0x1] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_MODRM,
+	[0x2] = INAT_MODRM,
+	[0x3] = INAT_MODRM,
+	[0x4] = INAT_MODRM,
+	[0x5] = INAT_MODRM,
+	[0x6] = INAT_MODRM,
+	[0x7] = INAT_MODRM,
+};
+
+/* GrpTable: Grp4 */
+const insn_attr_t inat_group_table_8[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MODRM,
+	[0x1] = INAT_MODRM,
+};
+
+/* GrpTable: Grp5 */
+const insn_attr_t inat_group_table_9[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MODRM,
+	[0x1] = INAT_MODRM,
+	[0x2] = INAT_MODRM | INAT_FORCE64,
+	[0x3] = INAT_MODRM,
+	[0x4] = INAT_MODRM | INAT_FORCE64,
+	[0x5] = INAT_MODRM,
+	[0x6] = INAT_MODRM | INAT_FORCE64,
+};
+
+/* GrpTable: Grp6 */
+const insn_attr_t inat_group_table_10[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MODRM,
+	[0x1] = INAT_MODRM,
+	[0x2] = INAT_MODRM,
+	[0x3] = INAT_MODRM,
+	[0x4] = INAT_MODRM,
+	[0x5] = INAT_MODRM,
+};
+
+/* GrpTable: Grp7 */
+const insn_attr_t inat_group_table_11[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MODRM,
+	[0x1] = INAT_MODRM,
+	[0x2] = INAT_MODRM,
+	[0x3] = INAT_MODRM,
+	[0x4] = INAT_MODRM,
+	[0x5] = INAT_VARIANT,
+	[0x6] = INAT_MODRM,
+	[0x7] = INAT_MODRM,
+};
+const insn_attr_t inat_group_table_11_2[INAT_GROUP_TABLE_SIZE] = {
+	[0x5] = INAT_MODRM,
+};
+
+/* GrpTable: Grp8 */
+
+/* GrpTable: Grp9 */
+const insn_attr_t inat_group_table_24[INAT_GROUP_TABLE_SIZE] = {
+	[0x1] = INAT_MODRM,
+	[0x6] = INAT_MODRM | INAT_MODRM | INAT_VARIANT,
+	[0x7] = INAT_MODRM | INAT_MODRM | INAT_VARIANT,
+};
+const insn_attr_t inat_group_table_24_1[INAT_GROUP_TABLE_SIZE] = {
+	[0x6] = INAT_MODRM,
+};
+const insn_attr_t inat_group_table_24_2[INAT_GROUP_TABLE_SIZE] = {
+	[0x6] = INAT_MODRM,
+	[0x7] = INAT_MODRM,
+};
+
+/* GrpTable: Grp10 */
+
+/* GrpTable: Grp11A */
+const insn_attr_t inat_group_table_4[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM,
+	[0x7] = INAT_MAKE_IMM(INAT_IMM_BYTE),
+};
+
+/* GrpTable: Grp11B */
+const insn_attr_t inat_group_table_5[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_MODRM,
+	[0x7] = INAT_MAKE_IMM(INAT_IMM_VWORD32),
+};
+
+/* GrpTable: Grp12 */
+const insn_attr_t inat_group_table_16[INAT_GROUP_TABLE_SIZE] = {
+	[0x2] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+	[0x4] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+	[0x6] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+};
+const insn_attr_t inat_group_table_16_1[INAT_GROUP_TABLE_SIZE] = {
+	[0x2] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x4] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x6] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+};
+
+/* GrpTable: Grp13 */
+const insn_attr_t inat_group_table_17[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_VARIANT,
+	[0x1] = INAT_VARIANT,
+	[0x2] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+	[0x4] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+	[0x6] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+};
+const insn_attr_t inat_group_table_17_1[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x1] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x2] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x4] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK | INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x6] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+};
+
+/* GrpTable: Grp14 */
+const insn_attr_t inat_group_table_18[INAT_GROUP_TABLE_SIZE] = {
+	[0x2] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+	[0x3] = INAT_VARIANT,
+	[0x6] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VARIANT,
+	[0x7] = INAT_VARIANT,
+};
+const insn_attr_t inat_group_table_18_1[INAT_GROUP_TABLE_SIZE] = {
+	[0x2] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x3] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x6] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+	[0x7] = INAT_MAKE_IMM(INAT_IMM_BYTE) | INAT_MODRM | INAT_VEXOK,
+};
+
+/* GrpTable: Grp15 */
+const insn_attr_t inat_group_table_21[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_VARIANT,
+	[0x1] = INAT_VARIANT,
+	[0x2] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x3] = INAT_MODRM | INAT_VEXOK | INAT_VARIANT,
+	[0x4] = INAT_VARIANT,
+	[0x5] = INAT_VARIANT,
+	[0x6] = INAT_VARIANT,
+};
+const insn_attr_t inat_group_table_21_1[INAT_GROUP_TABLE_SIZE] = {
+	[0x6] = INAT_MODRM,
+};
+const insn_attr_t inat_group_table_21_2[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MODRM,
+	[0x1] = INAT_MODRM,
+	[0x2] = INAT_MODRM,
+	[0x3] = INAT_MODRM,
+	[0x4] = INAT_MODRM,
+	[0x5] = INAT_MODRM,
+	[0x6] = INAT_MODRM | INAT_MODRM,
+};
+const insn_attr_t inat_group_table_21_3[INAT_GROUP_TABLE_SIZE] = {
+	[0x6] = INAT_MODRM,
+};
+
+/* GrpTable: Grp16 */
+const insn_attr_t inat_group_table_13[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MODRM,
+	[0x1] = INAT_MODRM,
+	[0x2] = INAT_MODRM,
+	[0x3] = INAT_MODRM,
+};
+
+/* GrpTable: Grp17 */
+const insn_attr_t inat_group_table_27[INAT_GROUP_TABLE_SIZE] = {
+	[0x1] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x2] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+	[0x3] = INAT_MODRM | INAT_VEXOK | INAT_VEXONLY,
+};
+
+/* GrpTable: Grp18 */
+const insn_attr_t inat_group_table_25[INAT_GROUP_TABLE_SIZE] = {
+	[0x1] = INAT_VARIANT,
+	[0x2] = INAT_VARIANT,
+	[0x5] = INAT_VARIANT,
+	[0x6] = INAT_VARIANT,
+};
+const insn_attr_t inat_group_table_25_1[INAT_GROUP_TABLE_SIZE] = {
+	[0x1] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x2] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x5] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x6] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+};
+
+/* GrpTable: Grp19 */
+const insn_attr_t inat_group_table_26[INAT_GROUP_TABLE_SIZE] = {
+	[0x1] = INAT_VARIANT,
+	[0x2] = INAT_VARIANT,
+	[0x5] = INAT_VARIANT,
+	[0x6] = INAT_VARIANT,
+};
+const insn_attr_t inat_group_table_26_1[INAT_GROUP_TABLE_SIZE] = {
+	[0x1] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x2] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x5] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+	[0x6] = INAT_MODRM | INAT_VEXOK | INAT_EVEXONLY,
+};
+
+/* GrpTable: Grp20 */
+const insn_attr_t inat_group_table_14[INAT_GROUP_TABLE_SIZE] = {
+	[0x0] = INAT_MODRM,
+};
+
+/* GrpTable: Grp21 */
+const insn_attr_t inat_group_table_15[INAT_GROUP_TABLE_SIZE] = {
+	[0x1] = INAT_VARIANT,
+};
+const insn_attr_t inat_group_table_15_2[INAT_GROUP_TABLE_SIZE] = {
+	[0x1] = INAT_MODRM,
+};
+
+/* GrpTable: GrpP */
+
+/* GrpTable: GrpPDLK */
+
+/* GrpTable: GrpRNG */
+
+#ifndef __BOOT_COMPRESSED
+
+/* Escape opcode map array */
+const insn_attr_t * const inat_escape_tables[INAT_ESC_MAX + 1][INAT_LSTPFX_MAX + 1] = {
+	[1][0] = inat_escape_table_1,
+	[1][1] = inat_escape_table_1_1,
+	[1][2] = inat_escape_table_1_2,
+	[1][3] = inat_escape_table_1_3,
+	[2][0] = inat_escape_table_2,
+	[2][1] = inat_escape_table_2_1,
+	[2][2] = inat_escape_table_2_2,
+	[2][3] = inat_escape_table_2_3,
+	[3][0] = inat_escape_table_3,
+	[3][1] = inat_escape_table_3_1,
+	[3][3] = inat_escape_table_3_3,
+};
+
+/* Group opcode map array */
+const insn_attr_t * const inat_group_tables[INAT_GRP_MAX + 1][INAT_LSTPFX_MAX + 1] = {
+	[4][0] = inat_group_table_4,
+	[5][0] = inat_group_table_5,
+	[6][0] = inat_group_table_6,
+	[7][0] = inat_group_table_7,
+	[8][0] = inat_group_table_8,
+	[9][0] = inat_group_table_9,
+	[10][0] = inat_group_table_10,
+	[11][0] = inat_group_table_11,
+	[11][2] = inat_group_table_11_2,
+	[13][0] = inat_group_table_13,
+	[14][0] = inat_group_table_14,
+	[15][0] = inat_group_table_15,
+	[15][2] = inat_group_table_15_2,
+	[16][0] = inat_group_table_16,
+	[16][1] = inat_group_table_16_1,
+	[17][0] = inat_group_table_17,
+	[17][1] = inat_group_table_17_1,
+	[18][0] = inat_group_table_18,
+	[18][1] = inat_group_table_18_1,
+	[21][0] = inat_group_table_21,
+	[21][1] = inat_group_table_21_1,
+	[21][2] = inat_group_table_21_2,
+	[21][3] = inat_group_table_21_3,
+	[24][0] = inat_group_table_24,
+	[24][1] = inat_group_table_24_1,
+	[24][2] = inat_group_table_24_2,
+	[25][0] = inat_group_table_25,
+	[25][1] = inat_group_table_25_1,
+	[26][0] = inat_group_table_26,
+	[26][1] = inat_group_table_26_1,
+	[27][0] = inat_group_table_27,
+};
+
+/* AVX opcode map array */
+const insn_attr_t * const inat_avx_tables[X86_VEX_M_MAX + 1][INAT_LSTPFX_MAX + 1] = {
+	[1][0] = inat_escape_table_1,
+	[1][1] = inat_escape_table_1_1,
+	[1][2] = inat_escape_table_1_2,
+	[1][3] = inat_escape_table_1_3,
+	[2][0] = inat_escape_table_2,
+	[2][1] = inat_escape_table_2_1,
+	[2][2] = inat_escape_table_2_2,
+	[2][3] = inat_escape_table_2_3,
+	[3][0] = inat_escape_table_3,
+	[3][1] = inat_escape_table_3_1,
+	[3][3] = inat_escape_table_3_3,
+};
+
+#else /* !__BOOT_COMPRESSED */
+
+/* Escape opcode map array */
+static const insn_attr_t *inat_escape_tables[INAT_ESC_MAX + 1][INAT_LSTPFX_MAX + 1];
+
+/* Group opcode map array */
+static const insn_attr_t *inat_group_tables[INAT_GRP_MAX + 1][INAT_LSTPFX_MAX + 1];
+
+/* AVX opcode map array */
+static const insn_attr_t *inat_avx_tables[X86_VEX_M_MAX + 1][INAT_LSTPFX_MAX + 1];
+
+static void inat_init_tables(void)
+{
+	/* Print Escape opcode map array */
+	inat_escape_tables[1][0] = inat_escape_table_1;
+	inat_escape_tables[1][1] = inat_escape_table_1_1;
+	inat_escape_tables[1][2] = inat_escape_table_1_2;
+	inat_escape_tables[1][3] = inat_escape_table_1_3;
+	inat_escape_tables[2][0] = inat_escape_table_2;
+	inat_escape_tables[2][1] = inat_escape_table_2_1;
+	inat_escape_tables[2][2] = inat_escape_table_2_2;
+	inat_escape_tables[2][3] = inat_escape_table_2_3;
+	inat_escape_tables[3][0] = inat_escape_table_3;
+	inat_escape_tables[3][1] = inat_escape_table_3_1;
+	inat_escape_tables[3][3] = inat_escape_table_3_3;
+
+	/* Print Group opcode map array */
+	inat_group_tables[4][0] = inat_group_table_4;
+	inat_group_tables[5][0] = inat_group_table_5;
+	inat_group_tables[6][0] = inat_group_table_6;
+	inat_group_tables[7][0] = inat_group_table_7;
+	inat_group_tables[8][0] = inat_group_table_8;
+	inat_group_tables[9][0] = inat_group_table_9;
+	inat_group_tables[10][0] = inat_group_table_10;
+	inat_group_tables[11][0] = inat_group_table_11;
+	inat_group_tables[11][2] = inat_group_table_11_2;
+	inat_group_tables[13][0] = inat_group_table_13;
+	inat_group_tables[14][0] = inat_group_table_14;
+	inat_group_tables[15][0] = inat_group_table_15;
+	inat_group_tables[15][2] = inat_group_table_15_2;
+	inat_group_tables[16][0] = inat_group_table_16;
+	inat_group_tables[16][1] = inat_group_table_16_1;
+	inat_group_tables[17][0] = inat_group_table_17;
+	inat_group_tables[17][1] = inat_group_table_17_1;
+	inat_group_tables[18][0] = inat_group_table_18;
+	inat_group_tables[18][1] = inat_group_table_18_1;
+	inat_group_tables[21][0] = inat_group_table_21;
+	inat_group_tables[21][1] = inat_group_table_21_1;
+	inat_group_tables[21][2] = inat_group_table_21_2;
+	inat_group_tables[21][3] = inat_group_table_21_3;
+	inat_group_tables[24][0] = inat_group_table_24;
+	inat_group_tables[24][1] = inat_group_table_24_1;
+	inat_group_tables[24][2] = inat_group_table_24_2;
+	inat_group_tables[25][0] = inat_group_table_25;
+	inat_group_tables[25][1] = inat_group_table_25_1;
+	inat_group_tables[26][0] = inat_group_table_26;
+	inat_group_tables[26][1] = inat_group_table_26_1;
+	inat_group_tables[27][0] = inat_group_table_27;
+
+	/* Print AVX opcode map array */
+	inat_avx_tables[1][0] = inat_escape_table_1;
+	inat_avx_tables[1][1] = inat_escape_table_1_1;
+	inat_avx_tables[1][2] = inat_escape_table_1_2;
+	inat_avx_tables[1][3] = inat_escape_table_1_3;
+	inat_avx_tables[2][0] = inat_escape_table_2;
+	inat_avx_tables[2][1] = inat_escape_table_2_1;
+	inat_avx_tables[2][2] = inat_escape_table_2_2;
+	inat_avx_tables[2][3] = inat_escape_table_2_3;
+	inat_avx_tables[3][0] = inat_escape_table_3;
+	inat_avx_tables[3][1] = inat_escape_table_3_1;
+	inat_avx_tables[3][3] = inat_escape_table_3_3;
+}
+#endif
lib/x86/insn/inat.c new file +86 -0

86 lines not on this page · raw message

lib/x86/insn/inat.h new file +233 -0

233 lines not on this page · raw message

lib/x86/insn/inat_types.h new file +18 -0
new file mode 100644
index 0000000..5e4ef12
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _ASM_X86_INAT_TYPES_H
+#define _ASM_X86_INAT_TYPES_H
+/*
+ * x86 instruction attributes
+ *
+ * Written by Masami Hiramatsu <[email protected]>
+ *
+ * Origin: Linux 64222515138e43da1fcf288f0289ef1020427b87
+ *   tools/arch/x86/include/asm/inat_types.h
+ */
+
+/* Instruction attributes */
+typedef unsigned int insn_attr_t;
+typedef unsigned char insn_byte_t;
+typedef signed int insn_value_t;
+
+#endif
lib/x86/insn/insn.c new file +778 -0

778 lines not on this page · raw message

lib/x86/insn/insn.h new file +280 -0

280 lines not on this page · raw message

x86/Makefile.common +2 -0
index 65d16e7..825318c 100644
@@ -25,6 +25,8 @@ cflatobjs += lib/x86/delay.o
 ifeq ($(TARGET_EFI),y)
 cflatobjs += lib/x86/amd_sev.o
 cflatobjs += lib/x86/amd_sev_vc.o
+cflatobjs += lib/x86/insn/insn.o
+cflatobjs += lib/x86/insn/inat.o
 cflatobjs += lib/efi.o
 cflatobjs += x86/efi/reloc_x86_64.o
 endif
Signature · 1 line
2.32.0
#16 lore raw

PATCH 5/13 x86: AMD SEV-ES: Pull related GHCB definitions and helpers from Linux

In reply to Varad Gautam

Origin: Linux 64222515138e43da1fcf288f0289ef1020427b87

Suppress -Waddress-of-packed-member to allow taking addresses on struct ghcb / struct vmcb_save_area fields.

---
 lib/x86/amd_sev.h   | 106 ++++++++++++++++++++++++++++++++++++++++++++
 lib/x86/svm.h       |  37 ++++++++++++++++
 x86/Makefile.x86_64 |   1 +
 3 files changed, 144 insertions(+)
lib/x86/amd_sev.h +106 -0
index afbacf3..ed71c18 100644
@@ -18,6 +18,49 @@
 #include "desc.h"
 #include "asm/page.h"
 #include "efi.h"
+#include "processor.h"
+#include "insn/insn.h"
+#include "svm.h"
+
+struct __attribute__ ((__packed__)) ghcb {
+	struct vmcb_save_area save;
+	u8 reserved_save[2048 - sizeof(struct vmcb_save_area)];
+
+	u8 shared_buffer[2032];
+
+	u8 reserved_1[10];
+	u16 protocol_version;	/* negotiated SEV-ES/GHCB protocol version */
+	u32 ghcb_usage;
+};
+
+/* SEV definitions from linux's include/asm/sev.h */
+#define GHCB_PROTO_OUR		0x0001UL
+#define GHCB_PROTOCOL_MAX	1ULL
+#define GHCB_DEFAULT_USAGE	0ULL
+
+#define	VMGEXIT()			{ asm volatile("rep; vmmcall\n\r"); }
+
+enum es_result {
+	ES_OK,			/* All good */
+	ES_UNSUPPORTED,		/* Requested operation not supported */
+	ES_VMM_ERROR,		/* Unexpected state from the VMM */
+	ES_DECODE_FAILED,	/* Instruction decoding failed */
+	ES_EXCEPTION,		/* Instruction caused exception */
+	ES_RETRY,		/* Retry instruction emulation */
+};
+
+struct es_fault_info {
+	unsigned long vector;
+	unsigned long error_code;
+	unsigned long cr2;
+};
+
+/* ES instruction emulation context */
+struct es_em_ctxt {
+	struct ex_regs *regs;
+	struct insn insn;
+	struct es_fault_info fi;
+};
 
 /*
  * AMD Programmer's Manual Volume 3
@@ -59,6 +102,69 @@ void handle_sev_es_vc(struct ex_regs *regs);
 unsigned long long get_amd_sev_c_bit_mask(void);
 unsigned long long get_amd_sev_addr_upperbound(void);
 
+static int _test_bit(int nr, const volatile unsigned long *addr)
+{
+	const volatile unsigned long *word = addr + BIT_WORD(nr);
+	unsigned long mask = BIT_MASK(nr);
+
+	return (*word & mask) != 0;
+}
+
+/* GHCB Accessor functions from Linux's include/asm/svm.h */
+
+#define GHCB_BITMAP_IDX(field)							\
+	(offsetof(struct vmcb_save_area, field) / sizeof(u64))
+
+#define DEFINE_GHCB_ACCESSORS(field)						\
+	static inline bool ghcb_##field##_is_valid(const struct ghcb *ghcb)	\
+	{									\
+		return _test_bit(GHCB_BITMAP_IDX(field),				\
+				(unsigned long *)&ghcb->save.valid_bitmap);	\
+	}									\
+										\
+	static inline u64 ghcb_get_##field(struct ghcb *ghcb)			\
+	{									\
+		return ghcb->save.field;					\
+	}									\
+										\
+	static inline u64 ghcb_get_##field##_if_valid(struct ghcb *ghcb)	\
+	{									\
+		return ghcb_##field##_is_valid(ghcb) ? ghcb->save.field : 0;	\
+	}									\
+										\
+	static inline void ghcb_set_##field(struct ghcb *ghcb, u64 value)	\
+	{									\
+		set_bit(GHCB_BITMAP_IDX(field),				\
+			  (u8 *)&ghcb->save.valid_bitmap);		\
+		ghcb->save.field = value;					\
+	}
+
+DEFINE_GHCB_ACCESSORS(cpl)
+DEFINE_GHCB_ACCESSORS(rip)
+DEFINE_GHCB_ACCESSORS(rsp)
+DEFINE_GHCB_ACCESSORS(rax)
+DEFINE_GHCB_ACCESSORS(rcx)
+DEFINE_GHCB_ACCESSORS(rdx)
+DEFINE_GHCB_ACCESSORS(rbx)
+DEFINE_GHCB_ACCESSORS(rbp)
+DEFINE_GHCB_ACCESSORS(rsi)
+DEFINE_GHCB_ACCESSORS(rdi)
+DEFINE_GHCB_ACCESSORS(r8)
+DEFINE_GHCB_ACCESSORS(r9)
+DEFINE_GHCB_ACCESSORS(r10)
+DEFINE_GHCB_ACCESSORS(r11)
+DEFINE_GHCB_ACCESSORS(r12)
+DEFINE_GHCB_ACCESSORS(r13)
+DEFINE_GHCB_ACCESSORS(r14)
+DEFINE_GHCB_ACCESSORS(r15)
+DEFINE_GHCB_ACCESSORS(sw_exit_code)
+DEFINE_GHCB_ACCESSORS(sw_exit_info_1)
+DEFINE_GHCB_ACCESSORS(sw_exit_info_2)
+DEFINE_GHCB_ACCESSORS(sw_scratch)
+DEFINE_GHCB_ACCESSORS(xcr0)
+
+#define MSR_AMD64_SEV_ES_GHCB          0xc0010130
+
 #endif /* TARGET_EFI */
 
 #endif /* _X86_AMD_SEV_H_ */
lib/x86/svm.h +37 -0
index f74b13a..f046455 100644
@@ -197,6 +197,42 @@ struct __attribute__ ((__packed__)) vmcb_save_area {
 	u64 br_to;
 	u64 last_excp_from;
 	u64 last_excp_to;
+
+	/*
+	 * The following part of the save area is valid only for
+	 * SEV-ES guests when referenced through the GHCB or for
+	 * saving to the host save area.
+	 */
+	u8 reserved_7[72];
+	u32 spec_ctrl;          /* Guest version of SPEC_CTRL at 0x2E0 */
+	u8 reserved_7b[4];
+	u32 pkru;
+	u8 reserved_7a[20];
+	u64 reserved_8;         /* rax already available at 0x01f8 */
+	u64 rcx;
+	u64 rdx;
+	u64 rbx;
+	u64 reserved_9;         /* rsp already available at 0x01d8 */
+	u64 rbp;
+	u64 rsi;
+	u64 rdi;
+	u64 r8;
+	u64 r9;
+	u64 r10;
+	u64 r11;
+	u64 r12;
+	u64 r13;
+	u64 r14;
+	u64 r15;
+	u8 reserved_10[16];
+	u64 sw_exit_code;
+	u64 sw_exit_info_1;
+	u64 sw_exit_info_2;
+	u64 sw_scratch;
+	u8 reserved_11[56];
+	u64 xcr0;
+	u8 valid_bitmap[16];
+	u64 x87_state_gpa;
 };
 
 struct __attribute__ ((__packed__)) vmcb {
@@ -297,6 +333,7 @@ struct __attribute__ ((__packed__)) vmcb {
 #define	SVM_EXIT_WRITE_DR6 	0x036
 #define	SVM_EXIT_WRITE_DR7 	0x037
 #define SVM_EXIT_EXCP_BASE      0x040
+#define SVM_EXIT_LAST_EXCP     0x05f
 #define SVM_EXIT_INTR		0x060
 #define SVM_EXIT_NMI		0x061
 #define SVM_EXIT_SMI		0x062
x86/Makefile.x86_64 +1 -0
index 3963840..eeb321b 100644
@@ -13,6 +13,7 @@ endif
 
 fcf_protection_full := $(call cc-option, -fcf-protection=full,)
 COMMON_CFLAGS += -mno-red-zone -mno-sse -mno-sse2 $(fcf_protection_full)
+COMMON_CFLAGS += -Wno-address-of-packed-member
 
 cflatobjs += lib/x86/setjmp64.o
 cflatobjs += lib/x86/intel-iommu.o
Signature · 1 line
2.32.0
#17 lore raw

PATCH 6/13 x86: AMD SEV-ES: Prepare for #VC processing

In reply to Varad Gautam

Lay the groundwork for processing #VC exceptions in the handler. This includes clearing the GHCB, decoding the insn that triggered this #VC, and continuing execution after the exception has been processed.

---
 lib/x86/amd_sev_vc.c | 78 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
lib/x86/amd_sev_vc.c +78 -0
index 8226121..142f2cd 100644
@@ -1,14 +1,92 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 #include "amd_sev.h"
+#include "svm.h"
 
 extern phys_addr_t ghcb_addr;
 
+static void vc_ghcb_invalidate(struct ghcb *ghcb)
+{
+	ghcb->save.sw_exit_code = 0;
+	memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
+}
+
+static bool vc_decoding_needed(unsigned long exit_code)
+{
+	/* Exceptions don't require to decode the instruction */
+	return !(exit_code >= SVM_EXIT_EXCP_BASE &&
+		 exit_code <= SVM_EXIT_LAST_EXCP);
+}
+
+static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
+{
+	unsigned char buffer[MAX_INSN_SIZE];
+	int ret;
+
+	memcpy(buffer, (unsigned char *)ctxt->regs->rip, MAX_INSN_SIZE);
+
+	ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
+	if (ret < 0)
+		return ES_DECODE_FAILED;
+	else
+		return ES_OK;
+}
+
+static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
+				      struct ex_regs *regs,
+				      unsigned long exit_code)
+{
+	enum es_result ret = ES_OK;
+
+	memset(ctxt, 0, sizeof(*ctxt));
+	ctxt->regs = regs;
+
+	if (vc_decoding_needed(exit_code))
+		ret = vc_decode_insn(ctxt);
+
+	return ret;
+}
+
+static void vc_finish_insn(struct es_em_ctxt *ctxt)
+{
+	ctxt->regs->rip += ctxt->insn.length;
+}
+
+static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
+					 struct ghcb *ghcb,
+					 unsigned long exit_code)
+{
+	enum es_result result;
+
+	switch (exit_code) {
+	default:
+		/*
+		 * Unexpected #VC exception
+		 */
+		result = ES_UNSUPPORTED;
+	}
+
+	return result;
+}
+
 void handle_sev_es_vc(struct ex_regs *regs)
 {
 	struct ghcb *ghcb = (struct ghcb *) ghcb_addr;
+	unsigned long exit_code = regs->error_code;
+	struct es_em_ctxt ctxt;
+	enum es_result result;
+
 	if (!ghcb) {
 		/* TODO: kill guest */
 		return;
 	}
+
+	vc_ghcb_invalidate(ghcb);
+	result = vc_init_em_ctxt(&ctxt, regs, exit_code);
+	if (result == ES_OK)
+		result = vc_handle_exitcode(&ctxt, ghcb, exit_code);
+	if (result == ES_OK)
+		vc_finish_insn(&ctxt);
+
+	return;
 }
Signature · 1 line
2.32.0
#18 lore raw

PATCH 7/13 x86: AMD SEV-ES: Handle WBINVD #VC

In reply to Varad Gautam

Add Linux's sev_es_ghcb_hv_call helper to allow VMGEXIT from the #VC handler, and handle SVM_EXIT_WBINVD using this.

---
 lib/x86/amd_sev_vc.c | 66 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
lib/x86/amd_sev_vc.c +66 -0
index 142f2cd..27a3ed0 100644
@@ -52,6 +52,69 @@ static void vc_finish_insn(struct es_em_ctxt *ctxt)
 	ctxt->regs->rip += ctxt->insn.length;
 }
 
+static inline u64 lower_bits(u64 val, unsigned int bits)
+{
+	u64 mask = (1ULL << bits) - 1;
+
+	return (val & mask);
+}
+
+static inline void sev_es_wr_ghcb_msr(u64 val)
+{
+	wrmsr(MSR_AMD64_SEV_ES_GHCB, val);
+}
+
+static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
+					  struct es_em_ctxt *ctxt,
+					  u64 exit_code, u64 exit_info_1,
+					  u64 exit_info_2)
+{
+	enum es_result ret;
+
+	/* Fill in protocol and format specifiers */
+	ghcb->protocol_version = GHCB_PROTOCOL_MAX;
+	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
+
+	ghcb_set_sw_exit_code(ghcb, exit_code);
+	ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
+	ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
+
+	sev_es_wr_ghcb_msr(__pa(ghcb));
+	VMGEXIT();
+
+	if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) {
+		u64 info = ghcb->save.sw_exit_info_2;
+		unsigned long v;
+
+		info = ghcb->save.sw_exit_info_2;
+		v = info & SVM_EVTINJ_VEC_MASK;
+
+		/* Check if exception information from hypervisor is sane. */
+		if ((info & SVM_EVTINJ_VALID) &&
+		    ((v == GP_VECTOR) || (v == UD_VECTOR)) &&
+		    ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
+			ctxt->fi.vector = v;
+			if (info & SVM_EVTINJ_VALID_ERR)
+				ctxt->fi.error_code = info >> 32;
+			ret = ES_EXCEPTION;
+		} else {
+			ret = ES_VMM_ERROR;
+		}
+	} else if (ghcb->save.sw_exit_info_1 & 0xffffffff) {
+		ret = ES_VMM_ERROR;
+	} else {
+		ret = ES_OK;
+	}
+
+	return ret;
+}
+
+static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,
+				       struct es_em_ctxt *ctxt)
+{
+	return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);
+}
+
 static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 					 struct ghcb *ghcb,
 					 unsigned long exit_code)
@@ -59,6 +122,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 	enum es_result result;
 
 	switch (exit_code) {
+	case SVM_EXIT_WBINVD:
+		result = vc_handle_wbinvd(ghcb, ctxt);
+		break;
 	default:
 		/*
 		 * Unexpected #VC exception
Signature · 1 line
2.32.0
#19 lore raw

Re: [kvm-unit-tests 07/13] x86: AMD SEV-ES: Handle WBINVD #VC

Quoted 2 lines · Varad Gautam

Add Linux's sev_es_ghcb_hv_call helper to allow VMGEXIT from the #VC handler, and handle SVM_EXIT_WBINVD using this.

There are no WBINVDs in KUT outside of nVMX or nSVM tests, and there never should be. I.e. this patch is unnecessary.

#20 lore raw

PATCH 8/13 lib/x86: Move xsave helpers to lib/

In reply to Varad Gautam

Processing CPUID #VC for AMD SEV-ES requires copying xcr0 into GHCB. Move the xsave read/write helpers used by xsave testcase to lib/x86 to share as common code.

---
 lib/x86/xsave.c     | 37 +++++++++++++++++++++++++++++++++++++
 lib/x86/xsave.h     | 16 ++++++++++++++++
 x86/Makefile.common |  1 +
 x86/xsave.c         | 43 +------------------------------------------
 4 files changed, 55 insertions(+), 42 deletions(-)
 create mode 100644 lib/x86/xsave.c
 create mode 100644 lib/x86/xsave.h
lib/x86/xsave.c new file +37 -0
new file mode 100644
index 0000000..1c0f16e
@@ -0,0 +1,37 @@
+#include "libcflat.h"
+#include "xsave.h"
+#include "processor.h"
+
+int xgetbv_checking(u32 index, u64 *result)
+{
+    u32 eax, edx;
+
+    asm volatile(ASM_TRY("1f")
+            ".byte 0x0f,0x01,0xd0\n\t" /* xgetbv */
+            "1:"
+            : "=a" (eax), "=d" (edx)
+            : "c" (index));
+    *result = eax + ((u64)edx << 32);
+    return exception_vector();
+}
+
+int xsetbv_checking(u32 index, u64 value)
+{
+    u32 eax = value;
+    u32 edx = value >> 32;
+
+    asm volatile(ASM_TRY("1f")
+            ".byte 0x0f,0x01,0xd1\n\t" /* xsetbv */
+            "1:"
+            : : "a" (eax), "d" (edx), "c" (index));
+    return exception_vector();
+}
+
+uint64_t get_supported_xcr0(void)
+{
+    struct cpuid r;
+    r = cpuid_indexed(0xd, 0);
+    printf("eax %x, ebx %x, ecx %x, edx %x\n",
+            r.a, r.b, r.c, r.d);
+    return r.a + ((u64)r.d << 32);
+}
lib/x86/xsave.h new file +16 -0
new file mode 100644
index 0000000..f1851a3
@@ -0,0 +1,16 @@
+#ifndef _X86_XSAVE_H_
+#define _X86_XSAVE_H_
+
+#define X86_CR4_OSXSAVE			0x00040000
+#define XCR_XFEATURE_ENABLED_MASK       0x00000000
+#define XCR_XFEATURE_ILLEGAL_MASK       0x00000010
+
+#define XSTATE_FP       0x1
+#define XSTATE_SSE      0x2
+#define XSTATE_YMM      0x4
+
+int xgetbv_checking(u32 index, u64 *result);
+int xsetbv_checking(u32 index, u64 value);
+uint64_t get_supported_xcr0(void);
+
+#endif
x86/Makefile.common +1 -0
index 825318c..d5c4d64 100644
@@ -22,6 +22,7 @@ cflatobjs += lib/x86/acpi.o
 cflatobjs += lib/x86/stack.o
 cflatobjs += lib/x86/fault_test.o
 cflatobjs += lib/x86/delay.o
+cflatobjs += lib/x86/xsave.o
 ifeq ($(TARGET_EFI),y)
 cflatobjs += lib/x86/amd_sev.o
 cflatobjs += lib/x86/amd_sev_vc.o
x86/xsave.c +1 -42

55 lines not on this page · raw message

Signature · 1 line
2.32.0
#21 lore raw

PATCH 9/13 x86: AMD SEV-ES: Handle CPUID #VC

In reply to Varad Gautam

Using Linux's CPUID #VC processing logic.

---
 lib/x86/amd_sev_vc.c | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
lib/x86/amd_sev_vc.c +41 -0
index 27a3ed0..91f57e0 100644
@@ -2,6 +2,7 @@
 
 #include "amd_sev.h"
 #include "svm.h"
+#include "x86/xsave.h"
 
 extern phys_addr_t ghcb_addr;
 
@@ -115,6 +116,43 @@ static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,
 	return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);
 }
 
+static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
+				      struct es_em_ctxt *ctxt)
+{
+	struct ex_regs *regs = ctxt->regs;
+	u32 cr4 = read_cr4();
+	enum es_result ret;
+
+	ghcb_set_rax(ghcb, regs->rax);
+	ghcb_set_rcx(ghcb, regs->rcx);
+
+	if (cr4 & X86_CR4_OSXSAVE) {
+		/* Safe to read xcr0 */
+		u64 xcr0;
+		xgetbv_checking(XCR_XFEATURE_ENABLED_MASK, &xcr0);
+		ghcb_set_xcr0(ghcb, xcr0);
+	} else
+		/* xgetbv will cause #GP - use reset value for xcr0 */
+		ghcb_set_xcr0(ghcb, 1);
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
+	if (ret != ES_OK)
+		return ret;
+
+	if (!(ghcb_rax_is_valid(ghcb) &&
+	      ghcb_rbx_is_valid(ghcb) &&
+	      ghcb_rcx_is_valid(ghcb) &&
+	      ghcb_rdx_is_valid(ghcb)))
+		return ES_VMM_ERROR;
+
+	regs->rax = ghcb->save.rax;
+	regs->rbx = ghcb->save.rbx;
+	regs->rcx = ghcb->save.rcx;
+	regs->rdx = ghcb->save.rdx;
+
+	return ES_OK;
+}
+
 static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 					 struct ghcb *ghcb,
 					 unsigned long exit_code)
@@ -125,6 +163,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 	case SVM_EXIT_WBINVD:
 		result = vc_handle_wbinvd(ghcb, ctxt);
 		break;
+	case SVM_EXIT_CPUID:
+		result = vc_handle_cpuid(ghcb, ctxt);
+		break;
 	default:
 		/*
 		 * Unexpected #VC exception
Signature · 1 line
2.32.0
#22 lore raw

PATCH 10/13 x86: AMD SEV-ES: Handle RDTSC/RDTSCP #VC

In reply to Varad Gautam

Using Linux's RDTSC #VC processing logic.

---
 lib/x86/amd_sev_vc.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
lib/x86/amd_sev_vc.c +27 -0
index 91f57e0..45b7ad1 100644
@@ -153,6 +153,29 @@ static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
 	return ES_OK;
 }
 
+static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
+				      struct es_em_ctxt *ctxt,
+				      unsigned long exit_code)
+{
+	bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
+	enum es_result ret;
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
+	if (ret != ES_OK)
+		return ret;
+
+	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
+	     (!rdtscp || ghcb_rcx_is_valid(ghcb))))
+		return ES_VMM_ERROR;
+
+	ctxt->regs->rax = ghcb->save.rax;
+	ctxt->regs->rdx = ghcb->save.rdx;
+	if (rdtscp)
+		ctxt->regs->rcx = ghcb->save.rcx;
+
+	return ES_OK;
+}
+
 static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 					 struct ghcb *ghcb,
 					 unsigned long exit_code)
@@ -166,6 +189,10 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 	case SVM_EXIT_CPUID:
 		result = vc_handle_cpuid(ghcb, ctxt);
 		break;
+	case SVM_EXIT_RDTSC:
+	case SVM_EXIT_RDTSCP:
+		result = vc_handle_rdtsc(ghcb, ctxt, exit_code);
+		break;
 	default:
 		/*
 		 * Unexpected #VC exception
Signature · 1 line
2.32.0
#23 lore raw

Re: [kvm-unit-tests 10/13] x86: AMD SEV-ES: Handle RDTSC/RDTSCP #VC

Quoted 1 line · Varad Gautam

Using Linux's RDTSC #VC processing logic.

I vote we drop this patch and instead treat #VC on RDTSC{,P} as a failure. KVM should never intercept RDTSC or RDTSCP except to inject #UD on RDTSCP, and IMO any hypervisor that deviates from the behavior for an SEV-ES guest is being malicious. That the SEV spec even allows reflecting them as #VC is ridiculous.

#24 lore raw

PATCH 11/13 x86: AMD SEV-ES: Handle MSR #VC

In reply to Varad Gautam

Using Linux's MSR #VC processing logic.

---
 lib/x86/amd_sev_vc.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
lib/x86/amd_sev_vc.c +28 -0
index 45b7ad1..28203df 100644
@@ -176,6 +176,31 @@ static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
 	return ES_OK;
 }
 
+static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	struct ex_regs *regs = ctxt->regs;
+	enum es_result ret;
+	u64 exit_info_1;
+
+	/* Is it a WRMSR? */
+	exit_info_1 = (ctxt->insn.opcode.bytes[1] == 0x30) ? 1 : 0;
+
+	ghcb_set_rcx(ghcb, regs->rcx);
+	if (exit_info_1) {
+		ghcb_set_rax(ghcb, regs->rax);
+		ghcb_set_rdx(ghcb, regs->rdx);
+	}
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0);
+
+	if ((ret == ES_OK) && (!exit_info_1)) {
+		regs->rax = ghcb->save.rax;
+		regs->rdx = ghcb->save.rdx;
+	}
+
+	return ret;
+}
+
 static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 					 struct ghcb *ghcb,
 					 unsigned long exit_code)
@@ -193,6 +218,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 	case SVM_EXIT_RDTSCP:
 		result = vc_handle_rdtsc(ghcb, ctxt, exit_code);
 		break;
+	case SVM_EXIT_MSR:
+		result = vc_handle_msr(ghcb, ctxt);
+		break;
 	default:
 		/*
 		 * Unexpected #VC exception
Signature · 1 line
2.32.0
#25 lore raw

PATCH 12/13 x86: AMD SEV-ES: Handle IOIO #VC

In reply to Varad Gautam

Using Linux's IOIO #VC processing logic.

---
 lib/x86/amd_sev_vc.c | 146 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 146 insertions(+)
lib/x86/amd_sev_vc.c +146 -0
index 28203df..bb912e4 100644
@@ -201,6 +201,149 @@ static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 	return ret;
 }
 
+#define IOIO_TYPE_STR  BIT(2)
+#define IOIO_TYPE_IN   1
+#define IOIO_TYPE_INS  (IOIO_TYPE_IN | IOIO_TYPE_STR)
+#define IOIO_TYPE_OUT  0
+#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
+
+#define IOIO_REP       BIT(3)
+
+#define IOIO_ADDR_64   BIT(9)
+#define IOIO_ADDR_32   BIT(8)
+#define IOIO_ADDR_16   BIT(7)
+
+#define IOIO_DATA_32   BIT(6)
+#define IOIO_DATA_16   BIT(5)
+#define IOIO_DATA_8    BIT(4)
+
+#define IOIO_SEG_ES    (0 << 10)
+#define IOIO_SEG_DS    (3 << 10)
+
+static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
+{
+	struct insn *insn = &ctxt->insn;
+	*exitinfo = 0;
+
+	switch (insn->opcode.bytes[0]) {
+	/* INS opcodes */
+	case 0x6c:
+	case 0x6d:
+		*exitinfo |= IOIO_TYPE_INS;
+		*exitinfo |= IOIO_SEG_ES;
+		*exitinfo |= (ctxt->regs->rdx & 0xffff) << 16;
+		break;
+
+	/* OUTS opcodes */
+	case 0x6e:
+	case 0x6f:
+		*exitinfo |= IOIO_TYPE_OUTS;
+		*exitinfo |= IOIO_SEG_DS;
+		*exitinfo |= (ctxt->regs->rdx & 0xffff) << 16;
+		break;
+
+	/* IN immediate opcodes */
+	case 0xe4:
+	case 0xe5:
+		*exitinfo |= IOIO_TYPE_IN;
+		*exitinfo |= (u8)insn->immediate.value << 16;
+		break;
+
+	/* OUT immediate opcodes */
+	case 0xe6:
+	case 0xe7:
+		*exitinfo |= IOIO_TYPE_OUT;
+		*exitinfo |= (u8)insn->immediate.value << 16;
+		break;
+
+	/* IN register opcodes */
+	case 0xec:
+	case 0xed:
+		*exitinfo |= IOIO_TYPE_IN;
+		*exitinfo |= (ctxt->regs->rdx & 0xffff) << 16;
+		break;
+
+	/* OUT register opcodes */
+	case 0xee:
+	case 0xef:
+		*exitinfo |= IOIO_TYPE_OUT;
+		*exitinfo |= (ctxt->regs->rdx & 0xffff) << 16;
+		break;
+
+	default:
+		return ES_DECODE_FAILED;
+	}
+
+	switch (insn->opcode.bytes[0]) {
+	case 0x6c:
+	case 0x6e:
+	case 0xe4:
+	case 0xe6:
+	case 0xec:
+	case 0xee:
+		/* Single byte opcodes */
+		*exitinfo |= IOIO_DATA_8;
+		break;
+	default:
+		/* Length determined by instruction parsing */
+		*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
+						     : IOIO_DATA_32;
+	}
+	switch (insn->addr_bytes) {
+	case 2:
+		*exitinfo |= IOIO_ADDR_16;
+		break;
+	case 4:
+		*exitinfo |= IOIO_ADDR_32;
+		break;
+	case 8:
+		*exitinfo |= IOIO_ADDR_64;
+		break;
+	}
+
+	if (insn_has_rep_prefix(insn))
+		*exitinfo |= IOIO_REP;
+
+	return ES_OK;
+}
+
+static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	struct ex_regs *regs = ctxt->regs;
+	u64 exit_info_1;
+	enum es_result ret;
+
+	ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
+	if (ret != ES_OK)
+		return ret;
+
+	if (exit_info_1 & IOIO_TYPE_STR) {
+		ret = ES_VMM_ERROR;
+	} else {
+		/* IN/OUT into/from rAX */
+
+		int bits = (exit_info_1 & 0x70) >> 1;
+		u64 rax = 0;
+
+		if (!(exit_info_1 & IOIO_TYPE_IN))
+			rax = lower_bits(regs->rax, bits);
+
+		ghcb_set_rax(ghcb, rax);
+
+		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
+		if (ret != ES_OK)
+			return ret;
+
+		if (exit_info_1 & IOIO_TYPE_IN) {
+			if (!ghcb_rax_is_valid(ghcb))
+				return ES_VMM_ERROR;
+			regs->rax = lower_bits(ghcb->save.rax, bits);
+		}
+	}
+
+	return ret;
+}
+
 static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 					 struct ghcb *ghcb,
 					 unsigned long exit_code)
@@ -221,6 +364,9 @@ static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 	case SVM_EXIT_MSR:
 		result = vc_handle_msr(ghcb, ctxt);
 		break;
+	case SVM_EXIT_IOIO:
+		result = vc_handle_ioio(ghcb, ctxt);
+		break;
 	default:
 		/*
 		 * Unexpected #VC exception
Signature · 1 line
2.32.0
#26 lore raw

PATCH 13/13 x86: AMD SEV-ES: Handle string IO for IOIO #VC

In reply to Varad Gautam

Using Linux's IOIO #VC processing logic.

---
 lib/x86/amd_sev_vc.c | 108 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 106 insertions(+), 2 deletions(-)
lib/x86/amd_sev_vc.c +106 -2
index bb912e4..24db992 100644
@@ -307,10 +307,46 @@ static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
 	return ES_OK;
 }
 
+static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
+					  void *src, unsigned char *buf,
+					  unsigned int data_size,
+					  unsigned int count,
+					  bool backwards)
+{
+	int i, b = backwards ? -1 : 1;
+
+	for (i = 0; i < count; i++) {
+		void *s = src + (i * data_size * b);
+		unsigned char *d = buf + (i * data_size);
+
+		memcpy(d, s, data_size);
+	}
+
+	return ES_OK;
+}
+
+static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
+					   void *dst, unsigned char *buf,
+					   unsigned int data_size,
+					   unsigned int count,
+					   bool backwards)
+{
+	int i, s = backwards ? -1 : 1;
+
+	for (i = 0; i < count; i++) {
+		void *d = dst + (i * data_size * s);
+		unsigned char *b = buf + (i * data_size);
+
+		memcpy(d, b, data_size);
+	}
+
+	return ES_OK;
+}
+
 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 {
 	struct ex_regs *regs = ctxt->regs;
-	u64 exit_info_1;
+	u64 exit_info_1, exit_info_2;
 	enum es_result ret;
 
 	ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
@@ -318,7 +354,75 @@ static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 		return ret;
 
 	if (exit_info_1 & IOIO_TYPE_STR) {
-		ret = ES_VMM_ERROR;
+		/* (REP) INS/OUTS */
+
+		bool df = ((regs->rflags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
+		unsigned int io_bytes, exit_bytes;
+		unsigned int ghcb_count, op_count;
+		unsigned long es_base;
+		u64 sw_scratch;
+
+		/*
+		 * For the string variants with rep prefix the amount of in/out
+		 * operations per #VC exception is limited so that the kernel
+		 * has a chance to take interrupts and re-schedule while the
+		 * instruction is emulated.
+		 */
+		io_bytes   = (exit_info_1 >> 4) & 0x7;
+		ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
+
+		op_count    = (exit_info_1 & IOIO_REP) ? regs->rcx : 1;
+		exit_info_2 = op_count < ghcb_count ? op_count : ghcb_count;
+		exit_bytes  = exit_info_2 * io_bytes;
+
+		es_base = 0;
+
+		/* Read bytes of OUTS into the shared buffer */
+		if (!(exit_info_1 & IOIO_TYPE_IN)) {
+			ret = vc_insn_string_read(ctxt,
+					       (void *)(es_base + regs->rsi),
+					       ghcb->shared_buffer, io_bytes,
+					       exit_info_2, df);
+			if (ret)
+				return ret;
+		}
+
+		/*
+		 * Issue an VMGEXIT to the HV to consume the bytes from the
+		 * shared buffer or to have it write them into the shared buffer
+		 * depending on the instruction: OUTS or INS.
+		 */
+		sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
+		ghcb_set_sw_scratch(ghcb, sw_scratch);
+		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
+					  exit_info_1, exit_info_2);
+		if (ret != ES_OK)
+			return ret;
+
+		/* Read bytes from shared buffer into the guest's destination. */
+		if (exit_info_1 & IOIO_TYPE_IN) {
+			ret = vc_insn_string_write(ctxt,
+						   (void *)(es_base + regs->rdi),
+						   ghcb->shared_buffer, io_bytes,
+						   exit_info_2, df);
+			if (ret)
+				return ret;
+
+			if (df)
+				regs->rdi -= exit_bytes;
+			else
+				regs->rdi += exit_bytes;
+		} else {
+			if (df)
+				regs->rsi -= exit_bytes;
+			else
+				regs->rsi += exit_bytes;
+		}
+
+		if (exit_info_1 & IOIO_REP)
+			regs->rcx -= exit_info_2;
+
+		ret = regs->rcx ? ES_RETRY : ES_OK;
 	} else {
 		/* IN/OUT into/from rAX */
 
Signature · 1 line
2.32.0