Skip to contentSkip to conversation

TCTI TCG backend for acceleration on non-JIT AArch64

RFC PATCH In review Joelle van Dyne 2 participants 3 messages Jul 22, 2025 – Jul 23, 2025 Arm/Amlogic Meson SoC support

lore
Share & tools
lore
#0 lore raw

cover TCTI TCG backend for acceleration on non-JIT AArch64

The following patch is from work by Katherine Temkin to add a JITless backend for aarch64. The aarch64-tcti target for TCG uses pre-compiled "gadgets" which are snippets of code for every TCG op x all operands and then at runtime will "thread" together the gadgets with jumps after each gadget. This results in significant speedup against TCI but is still much slower than JIT.

This backend is mainly useful for iOS, which does not allow JIT in distributed apps. We ported the feature from v5.2 to v10.0 but will rebase it to master if there is interest. Would this backend be useful for mainline QEMU?

Joelle van Dyne (1):
tcg/tcti: add TCTI TCG backend for acceleration on non-JIT AArch64

 docs/devel/tcg-tcti.rst                       | 1140 +++++++++
 meson.build                                   |   10 +
 include/accel/tcg/getpc.h                     |    6 +-
 include/disas/dis-asm.h                       |    1 +
 include/tcg/tcg-opc.h                         |   12 +
 include/tcg/tcg.h                             |    2 +-
 tcg/aarch64-tcti/tcg-target-con-set.h         |   32 +
 tcg/aarch64-tcti/tcg-target-con-str.h         |   20 +
 tcg/aarch64-tcti/tcg-target-has.h             |  132 +
 tcg/aarch64-tcti/tcg-target-mo.h              |   13 +
 tcg/aarch64-tcti/tcg-target-reg-bits.h        |   16 +
 tcg/aarch64-tcti/tcg-target.h                 |  107 +
 host/include/generic/host/atomic128-cas.h.inc |    3 +-
 tcg/aarch64-tcti/tcg-target-opc.h.inc         |   15 +
 accel/tcg/cputlb.c                            |    3 +-
 accel/tcg/tcg-accel-ops.c                     |    8 +
 tcg/optimize.c                                |    2 +
 tcg/region.c                                  |   11 +-
 tcg/tcg-op.c                                  |   27 +
 tcg/tcg.c                                     |   19 +-
 tcg/aarch64-tcti/tcg-target.c.inc             | 2250 +++++++++++++++++
 meson_options.txt                             |    2 +
 scripts/meson-buildoptions.sh                 |    5 +
 tcg/aarch64-tcti/tcti-gadget-gen.py           | 1192 +++++++++
 tcg/meson.build                               |   71 +-
 25 files changed, 5082 insertions(+), 17 deletions(-)
 create mode 100644 docs/devel/tcg-tcti.rst
 create mode 100644 tcg/aarch64-tcti/tcg-target-con-set.h
 create mode 100644 tcg/aarch64-tcti/tcg-target-con-str.h
 create mode 100644 tcg/aarch64-tcti/tcg-target-has.h
 create mode 100644 tcg/aarch64-tcti/tcg-target-mo.h
 create mode 100644 tcg/aarch64-tcti/tcg-target-reg-bits.h
 create mode 100644 tcg/aarch64-tcti/tcg-target.h
 create mode 100644 tcg/aarch64-tcti/tcg-target-opc.h.inc
 create mode 100644 tcg/aarch64-tcti/tcg-target.c.inc
 create mode 100755 tcg/aarch64-tcti/tcti-gadget-gen.py
Signature · 1 line
2.41.0
#1 lore raw

RFC PATCH 1/1 tcg/tcti: add TCTI TCG backend for acceleration on non-JIT AArch64

Introduce a new backend TCTI for TCG which pre-compiles gadgets for every possible TCG op x operands and then "threads" them together at runtime. This results in huge binary sizes but as a result, the emulation speed is significantly faster than the TCI interpreter.

---
 docs/devel/tcg-tcti.rst                       | 1140 +++++++++
 meson.build                                   |   10 +
 include/accel/tcg/getpc.h                     |    6 +-
 include/disas/dis-asm.h                       |    1 +
 include/tcg/tcg-opc.h                         |   12 +
 include/tcg/tcg.h                             |    2 +-
 tcg/aarch64-tcti/tcg-target-con-set.h         |   32 +
 tcg/aarch64-tcti/tcg-target-con-str.h         |   20 +
 tcg/aarch64-tcti/tcg-target-has.h             |  132 +
 tcg/aarch64-tcti/tcg-target-mo.h              |   13 +
 tcg/aarch64-tcti/tcg-target-reg-bits.h        |   16 +
 tcg/aarch64-tcti/tcg-target.h                 |  107 +
 host/include/generic/host/atomic128-cas.h.inc |    3 +-
 tcg/aarch64-tcti/tcg-target-opc.h.inc         |   15 +
 accel/tcg/cputlb.c                            |    3 +-
 accel/tcg/tcg-accel-ops.c                     |    8 +
 tcg/optimize.c                                |    2 +
 tcg/region.c                                  |   11 +-
 tcg/tcg-op.c                                  |   27 +
 tcg/tcg.c                                     |   19 +-
 tcg/aarch64-tcti/tcg-target.c.inc             | 2250 +++++++++++++++++
 meson_options.txt                             |    2 +
 scripts/meson-buildoptions.sh                 |    5 +
 tcg/aarch64-tcti/tcti-gadget-gen.py           | 1192 +++++++++
 tcg/meson.build                               |   71 +-
 25 files changed, 5082 insertions(+), 17 deletions(-)
 create mode 100644 docs/devel/tcg-tcti.rst
 create mode 100644 tcg/aarch64-tcti/tcg-target-con-set.h
 create mode 100644 tcg/aarch64-tcti/tcg-target-con-str.h
 create mode 100644 tcg/aarch64-tcti/tcg-target-has.h
 create mode 100644 tcg/aarch64-tcti/tcg-target-mo.h
 create mode 100644 tcg/aarch64-tcti/tcg-target-reg-bits.h
 create mode 100644 tcg/aarch64-tcti/tcg-target.h
 create mode 100644 tcg/aarch64-tcti/tcg-target-opc.h.inc
 create mode 100644 tcg/aarch64-tcti/tcg-target.c.inc
 create mode 100755 tcg/aarch64-tcti/tcti-gadget-gen.py
docs/devel/tcg-tcti.rst new file +1140 -0
new file mode 100644
index 0000000000..047f4b8c07
@@ -0,0 +1,1140 @@
+.. _tcg_tcti:
+
+QEMU Tiny-Code Threaded Interpreter (AArch64)
+=============================================
+
+A TCG backend that chains together JOP/ROP-ish gadgets to massively
+reduce interpreter overhead vs TCI. Platform-dependent; but usable when
+JIT isn't available; e.g. on platforms that lack WX mappings. The
+general idea squish the addresses of a gadget sequence into a “queue”
+and then write each gadget so it ends in a “dequeue-jump”.
+
+Execution occurs by jumping into the first gadget, and letting it just
+play back some linear-overhead native code sequences for a while.
+
+Since TCG-TCI is optimized for sets of 16 GP registers and aarch64 has
+30, we could easily keep JIT/QEMU and guest state separate, and since
+16*16 is reasonably small we could actually have a set of reasonable
+gadgets for each combination of operands.
+
+Register Convention
+-------------------
+
+====== =====================
+Regs   Use
+====== =====================
+x1-x15 Guest Registers
+x24    TCTI temporary
+x25    saved IP during call
+x26    TCTI temporary
+x27    TCTI temporary
+x28    Thread-stream pointer
+x30    Link register
+SP     Stack Pointer, host
+PC     Program Counter, host
+====== =====================
+
+In pseudocode:
+
+====== ===================================
+Symbol Meaning
+====== ===================================
+Rd     stand-in for destination register
+Rn     stand-in for first source register
+Rm     stand-in for second source register
+====== ===================================
+
+Gadget Structure
+----------------
+
+End of gadget
+~~~~~~~~~~~~~
+
+Each gadget ends by advancing our bytecode pointer, and then executing
+from thew new location.
+
+.. code:: asm
+
+   # Load our next gadget address from our bytecode stream, advancing it, and jump to the next gadget.
+
+   ldr x27, [x28], #8\n
+   br x27
+
+Calling into QEMU’s C codebase
+------------------------------
+
+When calling into C, we lose control over which registers are used.
+Accordingly, we’ll need to save registers relevant to TCTI:
+
+.. code:: asm
+
+   str x25,      [sp, #-16]!
+   stp x14, x15, [sp, #-16]!
+   stp x12, x13, [sp, #-16]!
+   stp x10, x11, [sp, #-16]!
+   stp x8,  x9,  [sp, #-16]!
+   stp x6,  x7,  [sp, #-16]!
+   stp x4,  x5,  [sp, #-16]!
+   stp x2,  x3,  [sp, #-16]!
+   stp x0,  x1,  [sp, #-16]!
+   stp x28, lr,  [sp, #-16]!
+
+Upon returning to the gadget stream, we’ll then restore them.
+
+.. code:: asm
+
+   ldp x28, lr, [sp], #16
+   ldp x0,  x1, [sp], #16
+   ldp x2,  x3, [sp], #16
+   ldp x4,  x5, [sp], #16
+   ldp x6,  x7, [sp], #16
+   ldp x8,  x9, [sp], #16
+   ldp x10, x11, [sp], #16
+   ldp x12, x13, [sp], #16
+   ldp x14, x15, [sp], #16
+   ldr x25,      [sp], #16
+
+TCG Operations
+--------------
+
+Each operation needs an implementation for every platform; and probably
+a set of gadgets for each possible set of operands.
+
+At 14 GP registers, that means that
+
+1 operand => 16 gadgets 2 operands => 256 gadgets 3 operands => 4096
+gadgets
+
+call
+~~~~
+
+Calls a helper function by address.
+
+| **IR Format**: ``br <ptr address>``
+| **Gadget type:** single
+
+.. code:: asm
+
+       # Get our C runtime function's location as a pointer-sized immediate...
+       "ldr x27, [x28], #8",
+
+       # Store our TB return address for our helper. This is necessary so the GETPC()
+       # macro works correctly as used in helper functions.
+       "str x28, [x25]",
+
+       # Prepare ourselves to call into our C runtime...
+       *C_CALL_PROLOGUE,
+
+       # ... perform the call itself ...
+       "blr x27",
+
+       # Save the result of our call for later.
+       "mov x27, x0",
+
+       # ... and restore our environment.
+       *C_CALL_EPILOGUE,
+
+       # Restore our return value.
+       "mov x0, x27"
+
+br
+~~
+
+Branches to a given immediate address. Branches are
+
+| **IR Format**: ``br <ptr address>``
+| **Gadget type:** single
+
+.. code:: asm
+
+   # Use our immediate argument as our new bytecode-pointer location.
+   ldr x28, [x28]
+
+setcond_i32
+~~~~~~~~~~~
+
+Performs a comparison between two 32-bit operands.
+
+| **IR Format**: ``setcond32 <cond>, Rd, Rn, Rm``
+| **Gadget type:** treated as 10 operations with variants for every
+  ``Rd``/``Rn``/``Rm`` (40,960)
+
+.. code:: asm
+
+   subs Wd, Wn, Wm
+   cset Wd, <cond>
+
+========= ============
+QEMU Cond AArch64 Cond
+========= ============
+EQ        EQ
+NE        NE
+LT        LT
+GE        GE
+LE        LE
+GT        GT
+LTU       LO
+GEU       HS
+LEU       LS
+GTU       HI
+========= ============
+
+setcond_i64
+~~~~~~~~~~~
+
+Performs a comparison between two 32-bit operands.
+
+| **IR Format**: ``setcond64 <cond>, Rd, Rn, Rm``
+| **Gadget type:** treated as 10 operations with variants for every
+  ``Rd``/``Rn``/``Rm`` (40,960)
+
+.. code:: asm
+
+   subs Xd, Xn, Xm
+   cset Xd, <cond>
+
+Comparison chart is the same as the ``_i32`` variant.
+
+brcond_i32
+~~~~~~~~~~
+
+Compares two 32-bit numbers, and branches if the comparison is true.
+
+| **IR Format**: ``brcond Rn, Rm, <cond>``
+| **Gadget type:** treated as 10 operations with variants for every
+  ``Rn``/``Rm`` (2560)
+
+.. code:: asm
+
+   # Perform our comparison and conditional branch.
+   subs Wrz, Wn, Wm
+   br<cond> taken
+
+       # Consume the branch target, without using it.
+       add x28, x28, #8
+
+       # Perform our end-of-instruction epilogue.
+       <epilogue here>
+
+   taken:
+
+       # Update our bytecode pointer to take the label.
+       ldr x28, [x28]
+
+Comparison chart is the same as in ``setcond_i32`` .
+
+brcond_i64
+~~~~~~~~~~
+
+Compares two 64-bit numbers, and branches if the comparison is true.
+
+| **IR Format**: ``brcond Rn, Rm, <cond>``
+| **Gadget type:** treated as 10 operations with variants for every
+  ``Rn``/``Rm`` (2560)
+
+.. code:: asm
+
+   # Perform our comparison and conditional branch.
+   subs Xrz, Xn, Xm
+   br<cond> taken
+
+       # Consume the branch target, without using it.
+       add x28, x28, #8
+
+       # Perform our end-of-instruction epilogue.
+       <epilogue here>
+
+   taken:
+
+       # Update our bytecode pointer to take the label.
+       ldr x28, [x28]
+
+Comparison chart is the same as in ``setcond_i32`` .
+
+mov_i32
+~~~~~~~
+
+Moves a value from a register to another register.
+
+| **IR Format**: ``mov Rd, Rn``
+| **Gadget type:** gadget per ``Rd`` + ``Rn`` combo (256)
+
+.. code:: asm
+
+   mov Rd, Rn
+
+mov_i64
+~~~~~~~
+
+Moves a value from a register to another register.
+
+| **IR Format**: ``mov Rd, Rn``
+| **Gadget type:** gadget per ``Rd`` + ``Rn`` combo (256)
+
+.. code:: asm
+
+   mov Xd, Xn
+
+tci_movi_i32
+~~~~~~~~~~~~
+
+Moves an 32b immediate into a register.
+
+| **IR Format**: ``mov Rd, #imm32``
+| **Gadget type:** gadget per ``Rd`` (16)
+
+.. code:: asm
+
+   ldr w27, [x28], #4
+   mov Wd, w27
+
+tci_movi_i64
+~~~~~~~~~~~~
+
+Moves an 64b immediate into a register.
+
+| **IR Format**: ``mov Rd, #imm64``
+| **Gadget type:** gadget per ``Rd`` (16)
+
+.. code:: asm
+
+   ldr x27, [x28], #4
+   mov Xd, x27
+
+ld8u_i32 / ld8u_i64
+~~~~~~~~~~~~~~~~~~~
+
+Load byte from host memory to register.
+
+| **IR Format**: ``ldr Rd, Rn, <signed offset>``
+| **Gadget type:** gadget per ``Rd`` & ``Rn`` (256)
+
+.. code:: asm
+
+   ldrsw x27, [x28], #4
+   ldrb Xd, [Xn, x27]
+
+ld8s_i32 / ld8s_i64
+~~~~~~~~~~~~~~~~~~~
+
+Load byte from host memory to register; sign extending.
+
+| **IR Format**: ``ldr Rd, Rn, <signed offset>``
+| **Gadget type:** gadget per ``Rd`` & ``Rn`` (256)
+
+.. code:: asm
+
+   ldrsw x27, [x28], #4
+   ldrsb Xd, [Xn, x27]
+
+ld16u_i32 / ld16u_i64
+~~~~~~~~~~~~~~~~~~~~~
+
+Load 16b from host memory to register.
+
+| **IR Format**: ``ldr Rd, Rn, <signed offset>``
+| **Gadget type:** gadget per ``Rd`` & ``Rn`` (256)
+
+.. code:: asm
+
+   ldrsw x27, [x28], #4
+   ldrh Wd, [Xn, x27]
+
+ld16s_i32 / ld16s_i64
+~~~~~~~~~~~~~~~~~~~~~
+
+Load 16b from host memory to register; sign extending.
+
+| **IR Format**: ``ldr Rd, Rn, <signed offset>``
+| **Gadget type:** gadget per ``Rd`` & ``Rn`` (256)
+
+.. code:: asm
+
+   ldrsw x27, [x28], #4
+   ldrsh Xd, [Xn, x27]
+
+ld32u_i32 / ld32u_i64
+~~~~~~~~~~~~~~~~~~~~~
+
+Load 32b from host memory to register.
+
+| **IR Format**: ``ldr Rd, Rn, <signed offset>``
+| **Gadget type:** gadget per ``Rd`` & ``Rn`` (256)
+
+.. code:: asm
+
+   ldrsw x27, [x28], #4
+   ldr Wd, [Xn, x27]
+
+ld32s_i64
+~~~~~~~~~
+
+Load 32b from host memory to register; sign extending.
+
+| **IR Format**: ``ldr Rd, Rn, <signed offset>``
+| **Gadget type:** gadget per ``Rd`` & ``Rn`` (256)
+
+.. code:: asm
+
+   ldrsw x27, [x28], #4
+   ldrsw Xd, [Xn, x27]
+
+ld_i64
+~~~~~~
+
+Load 64b from host memory to register.
+
+| **IR Format**: ``ldr Rd, Rn, <signed offset>``
+| **Gadget type:** gadget per ``Rd`` & ``Rn`` (256)
+
+.. code:: asm
+
+   ldrsw x27, [x28], #4
+   ldr Xd, [Xn, x27]
+
+st8_i32 / st8_i64
+~~~~~~~~~~~~~~~~~
+
+Stores byte from register to host memory.
+
+| **IR Format**: ``str Rd, Rn, <signed offset>``
+| **Gadget type:** gadget per ``Rd`` & ``Rn`` (256)
+
+.. code:: asm
+
+   ldrsw x27, [x28], #4
+   strb Wd, [Xn, x27]
+
+st16_i32 / st16_i64
+~~~~~~~~~~~~~~~~~~~
+
+Stores 16b from register to host memory.
+
+| **IR Format**: ``str Rd, Rn, <signed offset>``
+| **Gadget type:** gadget per ``Rd`` & ``Rn`` (256)
+
+.. code:: asm
+
+   ldrsw x27, [x28], #4
+   strh Wd, [Xn, x27]
+
+st_i32 / st32_i64
+~~~~~~~~~~~~~~~~~
+
+Stores 32b from register to host memory.
+
+| **IR Format**: ``str Rd, Rn, <signed offset>``
+| **Gadget type:** gadget per ``Rd`` & ``Rn`` (256)
+
+.. code:: asm
+
+   ldrsw x27, [x28], #4
+   str Wd, [Xn, x27]
+
+st_i64
+~~~~~~
+
+Stores 64b from register to host memory.
+
+| **IR Format**: ``str Rd, Rn, <signed offset>``
+| **Gadget type:** gadget per ``Rd`` & ``Rn`` (256)
+
+.. code:: asm
+
+   ldrsw x27, [x28], #4
+   str Xd, [Xn, x27]
+
+qemu_ld_i32
+~~~~~~~~~~~
+
+Loads 32b from *guest* memory to register.
+
+| **IR Format**: ``ld Rd, <foreign/guest pointer>, <memory operation>``
+| **Gadget type:** thunk per ``Rd`` into C impl?
+
+qemu_ld_i64
+~~~~~~~~~~~
+
+Loads 64b from *guest* memory to register.
+
+| **IR Format**: ``ld Rd, <foreign/guest pointer>, <memory operation>``
+| **Gadget type:** thunk per ``Rd`` into C impl?
+
+qemu_st_i32
+~~~~~~~~~~~
+
+Stores 32b from a register to *guest* memory.
+
+| **IR Format**: ``st Rd, <foreign/guest pointer>, <memory operation>``
+| **Gadget type:** thunk per ``Rd`` into C impl
+
+qemu_st_i64
+~~~~~~~~~~~
+
+Stores 64b from a register to *guest* memory.
+
+| **IR Format**: ``st Rd, <foreign/guest pointer>, <memory operation>``
+| **Gadget type:** thunk per ``Rd`` into C impl?
+
+Note
+^^^^
+
+See note on ``qemu_ld_i32``.
+
+add_i32
+~~~~~~~
+
+Adds two 32-bit numbers.
+
+| **IR Format**: ``add Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   add Wd, Wn, Wm
+
+add_i64
+~~~~~~~
+
+Adds two 64-bit numbers.
+
+| **IR Format**: ``add Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   add Xd, Xn, Xm
+
+sub_i32
+~~~~~~~
+
+Subtracts two 32-bit numbers.
+
+| **IR Format**: ``add Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   Sub Wd, Wn, Wm
+
+sub_i64
+~~~~~~~
+
+Subtracts two 64-bit numbers.
+
+| **IR Format**: ``sub Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   sub Xd, Xn, Xm
+
+mul_i32
+~~~~~~~
+
+Multiplies two 32-bit numbers.
+
+| **IR Format**: ``mul Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   mul Wd, Wn, Wm
+
+mul_i64
+~~~~~~~
+
+Multiplies two 64-bit numbers.
+
+| **IR Format**: ``mul Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   mul Xd, Xn, Xm
+
+div_i32
+~~~~~~~
+
+Divides two 32-bit numbers; considering them signed.
+
+| **IR Format**: ``div Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   sdiv Wd, Wn, Wm
+
+div_i64
+~~~~~~~
+
+Divides two 64-bit numbers; considering them signed.
+
+| **IR Format**: ``div Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   sdiv Xd, Xn, Xm
+
+divu_i32
+~~~~~~~~
+
+Divides two 32-bit numbers; considering them unsigned.
+
+| **IR Format**: ``div Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   udiv Wd, Wn, Wm
+
+divu_i64
+~~~~~~~~
+
+Divides two 32-bit numbers; considering them unsigned.
+
+| **IR Format**: ``div Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   udiv Xd, Xn, Xm
+
+rem_i32
+~~~~~~~
+
+Computes the division remainder (modulus) of two 32-bit numbers;
+considering them signed.
+
+| **IR Format**: ``rem Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   sdiv    w27, Wn, Wm
+   msub    Wd, w27, Wm, Wn
+
+rem_i64
+~~~~~~~
+
+Computes the division remainder (modulus) of two 64-bit numbers;
+considering them signed.
+
+| **IR Format**: ``rem Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   sdiv    x27, Xn, Xm
+   msub    Xd, x27, Xm, Xn
+
+remu_i32
+~~~~~~~~
+
+Computes the division remainder (modulus) of two 32-bit numbers;
+considering them unsigned.
+
+| **IR Format**: ``rem Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   udiv    w27, Wn, Wm
+   msub    Wd, w27, Wm, Wn
+
+remu_i64
+~~~~~~~~
+
+Computes the division remainder (modulus) of two 32-bit numbers;
+considering them unsigned.
+
+| **IR Format**: ``rem Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   udiv    x27, Xn, Xm
+   msub    Xd, x27, Xm, Xn
+
+not_i32
+~~~~~~~
+
+Logically inverts a 32-bit number.
+
+| **IR Format**: ``not Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   mvn Wd, Wn
+
+not_i64
+~~~~~~~
+
+Logically inverts a 64-bit number.
+
+| **IR Format**: ``not Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   mvn Xd, Xn
+
+neg_i32
+~~~~~~~
+
+Arithmetically inverts (two’s compliment) a 32-bit number.
+
+| **IR Format**: ``not Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   neg Wd, Wn
+
+neg_i64
+~~~~~~~
+
+Arithmetically inverts (two’s compliment) a 64-bit number.
+
+| **IR Format**: ``not Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   neg Xd, Xn
+
+and_i32
+~~~~~~~
+
+Logically ANDs two 32-bit numbers.
+
+| **IR Format**: ``and Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   and Wd, Wn, Wm
+
+and_i64
+~~~~~~~
+
+Logically ANDs two 64-bit numbers.
+
+| **IR Format**: ``and Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   and Xd, Xn, Xm
+
+or_i32
+~~~~~~
+
+Logically ORs two 32-bit numbers.
+
+| **IR Format**: ``or Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   or Wd, Wn, Wm
+
+or_i64
+~~~~~~
+
+Logically ORs two 64-bit numbers.
+
+| **IR Format**: ``or Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   or Xd, Xn, Xm
+
+xor_i32
+~~~~~~~
+
+Logically XORs two 32-bit numbers.
+
+| **IR Format**: ``xor Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   eor Wd, Wn, Wm
+
+xor_i64
+~~~~~~~
+
+Logically XORs two 64-bit numbers.
+
+| **IR Format**: ``xor Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   eor Xd, Xn, Xm
+
+shl_i32
+~~~~~~~
+
+Logically shifts a 32-bit number left.
+
+| **IR Format**: ``shl Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   lsl Wd, Wn, Wm
+
+shl_i64
+~~~~~~~
+
+Logically shifts a 64-bit number left.
+
+| **IR Format**: ``shl Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   lsl Xd, Xn, Xm
+
+shr_i32
+~~~~~~~
+
+Logically shifts a 32-bit number right.
+
+| **IR Format**: ``shr Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   lsr Wd, Wn, Wm
+
+shr_i64
+~~~~~~~
+
+Logically shifts a 64-bit number right.
+
+| **IR Format**: ``shr Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   lsr Xd, Xn, Xm
+
+sar_i32
+~~~~~~~
+
+Arithmetically shifts a 32-bit number right.
+
+| **IR Format**: ``sar Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   asr Wd, Wn, Wm
+
+sar_i64
+~~~~~~~
+
+Arithmetically shifts a 64-bit number right.
+
+| **IR Format**: ``sar Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   asr Xd, Xn, Xm
+
+rotl_i32
+~~~~~~~~
+
+Rotates a 32-bit number left.
+
+| **IR Format**: ``rotl Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   rol Wd, Wn, Wm
+
+rotl_i64
+~~~~~~~~
+
+Rotates a 64-bit number left.
+
+| **IR Format**: ``rotl Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   rol Xd, Xn, Xm
+
+rotr_i32
+~~~~~~~~
+
+Rotates a 32-bit number right.
+
+| **IR Format**: ``rotr Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   ror Wd, Wn, Wm
+
+rotr_i64
+~~~~~~~~
+
+Rotates a 64-bit number right.
+
+| **IR Format**: ``rotr Rd, Rn, Rm``
+| **Gadget type:** gadget per ``Rd``, ``Rn``, ``Rm`` (4096)
+
+.. code:: asm
+
+   ror Xd, Xn, Xm
+
+deposit_i32
+~~~~~~~~~~~
+
+Optional; not currently implementing.
+
+deposit_i64
+~~~~~~~~~~~
+
+Optional; not currently implementing.
+
+ext8s_i32
+~~~~~~~~~
+
+Sign extends the lower 8b of a register into a 32b destination.
+
+| **IR Format**: ``ext8s Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   sxtb Wd, Wn
+
+ext8s_i64
+~~~~~~~~~
+
+Sign extends the lower 8b of a register into a 64b destination.
+
+| **IR Format**: ``ext8s Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   sxtb Xd, Wn
+
+ext8u_i32
+~~~~~~~~~
+
+Zero extends the lower 8b of a register into a 32b destination.
+
+| **IR Format**: ``ext8u Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   and Xd, Xn, #0xff
+
+ext8u_i64
+~~~~~~~~~
+
+Zero extends the lower 8b of a register into a 64b destination.
+
+| **IR Format**: ``ext8u Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   and Xd, Xn, #0xff
+
+ext16s_i32
+~~~~~~~~~~
+
+Sign extends the lower 16b of a register into a 32b destination.
+
+| **IR Format**: ``ext16s Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   sxth Xd, Wn
+
+ext16s_i64
+~~~~~~~~~~
+
+Sign extends the lower 16b of a register into a 64b destination.
+
+| **IR Format**: ``ext16s Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   sxth Xd, Wn
+
+ext16u_i32
+~~~~~~~~~~
+
+Zero extends the lower 16b of a register into a 32b destination.
+
+| **IR Format**: ``ext16u Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   and Wd, Wn, #0xffff
+
+ext16u_i64
+~~~~~~~~~~
+
+Zero extends the lower 16b of a register into a 32b destination.
+
+| **IR Format**: ``ext16u Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   and Wd, Wn, #0xffff
+
+ext32s_i64
+~~~~~~~~~~
+
+Sign extends the lower 32b of a register into a 64b destination.
+
+| **IR Format**: ``ext32s Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   sxtw Xd, Wn
+
+ext32u_i64
+~~~~~~~~~~
+
+Zero extends the lower 32b of a register into a 64b destination.
+
+| **IR Format**: ``ext32s Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   sxtw Xd, Wn
+
+ext_i32_i64
+~~~~~~~~~~~
+
+Sign extends the lower 32b of a register into a 64b destination.
+
+| **IR Format**: ``ext32s Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   sxtw Xd, Wn
+
+extu_i32_i64
+~~~~~~~~~~~~
+
+Zero extends the lower 32b of a register into a 32b destination.
+
+| **IR Format**: ``ext32u Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   and Xd, Xn, #0xffffffff
+
+bswap16_i32
+~~~~~~~~~~~
+
+Byte-swaps a 16b quantity.
+
+| **IR Format**: ``bswap16 Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   rev     w27, Wn
+   lsr     Wd, w27, #16
+
+bswap16_i64
+~~~~~~~~~~~
+
+Byte-swaps a 16b quantity.
+
+| **IR Format**: ``bswap16 Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   rev     w27, Wn
+   lsr     Wd, w27, #16
+
+bswap32_i32
+~~~~~~~~~~~
+
+Byte-swaps a 32b quantity.
+
+| **IR Format**: ``bswap32 Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   rev     Wd, Wn
+
+bswap32_i64
+~~~~~~~~~~~
+
+Byte-swaps a 32b quantity.
+
+| **IR Format**: ``bswap32 Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   rev     Wd, Wn
+
+bswap64_i64
+~~~~~~~~~~~
+
+Byte-swaps a 64b quantity.
+
+| **IR Format**: ``bswap64 Rd, Rn``
+| **Gadget type:** gadget per ``Rd``, ``Rn`` (256)
+
+.. code:: asm
+
+   rev     Xd, Xn
+
+exit_tb
+~~~~~~~
+
+Exits the translation block. Has no gadget; but instead inserts the
+address of the translation block epilogue.
+
+mb
+~~
+
+Memory barrier.
+
+| **IR Format**: ``mb <type>``
+| **Gadget type:** gadget per type
+
+.. code:: asm
+
+   # !!! TODO
+
+.. _note-1:
+
+Note
+^^^^
+
+We still need to look up out how to map QEMU MB types map to AArch64
+ones. This might take nuance.
meson.build +10 -0
index 8ec796d835..02dc705553 100644
@@ -53,6 +53,7 @@ bsd_oses = ['gnu/kfreebsd', 'freebsd', 'netbsd', 'openbsd', 'dragonfly', 'darwin
 supported_oses = ['windows', 'freebsd', 'netbsd', 'openbsd', 'darwin', 'sunos', 'linux']
 supported_cpus = ['ppc', 'ppc64', 's390x', 'riscv32', 'riscv64', 'x86', 'x86_64',
   'arm', 'aarch64', 'loongarch64', 'mips', 'mips64', 'sparc64']
+tcti_supported_cpus = ['aarch64']
 
 cpu = host_machine.cpu_family()
 
@@ -912,6 +913,12 @@ if get_option('tcg').allowed()
   endif
   if get_option('tcg_interpreter')
     tcg_arch = 'tci'
+  elif get_option('tcg_threaded_interpreter')
+    if cpu not in tcti_supported_cpus
+      error('Unsupported CPU @0@ for TCTI, try --enable-tcg-interpreter'.format(cpu))
+    else
+      tcg_arch = '@0@-tcti'.format(cpu)
+    endif
   elif host_arch == 'x86_64'
     tcg_arch = 'i386'
   elif host_arch == 'ppc64'
@@ -2526,6 +2533,7 @@ config_host_data.set('CONFIG_SOLARIS', host_os == 'sunos')
 if get_option('tcg').allowed()
   config_host_data.set('CONFIG_TCG', 1)
   config_host_data.set('CONFIG_TCG_INTERPRETER', tcg_arch == 'tci')
+  config_host_data.set('CONFIG_TCG_THREADED_INTERPRETER', tcg_arch.endswith('tcti'))
 endif
 config_host_data.set('CONFIG_TPM', have_tpm)
 config_host_data.set('CONFIG_TSAN', get_option('tsan'))
@@ -4662,6 +4670,8 @@ summary_info += {'TCG support':       config_all_accel.has_key('CONFIG_TCG')}
 if config_all_accel.has_key('CONFIG_TCG')
   if get_option('tcg_interpreter')
     summary_info += {'TCG backend':   'TCI (TCG with bytecode interpreter, slow)'}
+  elif get_option('tcg_threaded_interpreter')
+    summary_info += {'TCG backend':   'TCTI (TCG with threaded-dispatch bytecode interpreter, experimental and slow; but faster than TCI)'}
   else
     summary_info += {'TCG backend':   'native (@0@)'.format(cpu)}
   endif
include/accel/tcg/getpc.h +5 -1
index 8a97ce34e7..3060565b05 100644
@@ -13,10 +13,14 @@
 #endif
 
 /* GETPC is the true target of the return instruction that we'll execute.  */
-#ifdef CONFIG_TCG_INTERPRETER
+#if defined(CONFIG_TCG_INTERPRETER)
 extern __thread uintptr_t tci_tb_ptr;
 # define GETPC() tci_tb_ptr
+#elif defined(CONFIG_TCG_THREADED_INTERPRETER)
+extern __thread uintptr_t tcti_call_return_address;
+# define GETPC() tcti_call_return_address
 #else
+/* Note that this is correct for TCTI also; whose gadget behaves like native code. */
 # define GETPC() \
     ((uintptr_t)__builtin_extract_return_addr(__builtin_return_address(0)))
 #endif
include/disas/dis-asm.h +1 -0
index 3b50ecfb54..c68eaa4736 100644
@@ -412,6 +412,7 @@ typedef struct disassemble_info {
 typedef int (*disassembler_ftype) (bfd_vma, disassemble_info *);
 
 int print_insn_tci(bfd_vma, disassemble_info*);
+int print_insn_tcti(bfd_vma, disassemble_info*);
 int print_insn_big_mips         (bfd_vma, disassemble_info*);
 int print_insn_little_mips      (bfd_vma, disassemble_info*);
 int print_insn_nanomips         (bfd_vma, disassemble_info*);
include/tcg/tcg-opc.h +12 -0
index 5bf78b0764..9f73771149 100644
@@ -40,7 +40,11 @@ DEF(mb, 0, 0, 1, TCG_OPF_NOT_PRESENT)
 DEF(mov_i32, 1, 1, 0, TCG_OPF_NOT_PRESENT)
 DEF(setcond_i32, 1, 2, 1, 0)
 DEF(negsetcond_i32, 1, 2, 1, 0)
+#if defined(CONFIG_TCG_THREADED_INTERPRETER)
+DEF(movcond_i32, 1, 4, 1, TCG_OPF_NOT_PRESENT)
+#else
 DEF(movcond_i32, 1, 4, 1, 0)
+#endif
 /* load/store */
 DEF(ld8u_i32, 1, 1, 1, 0)
 DEF(ld8s_i32, 1, 1, 1, 0)
@@ -105,7 +109,11 @@ DEF(ctpop_i32, 1, 1, 0, 0)
 DEF(mov_i64, 1, 1, 0, TCG_OPF_NOT_PRESENT)
 DEF(setcond_i64, 1, 2, 1, 0)
 DEF(negsetcond_i64, 1, 2, 1, 0)
+#if defined(CONFIG_TCG_THREADED_INTERPRETER)
+DEF(movcond_i64, 1, 4, 1, TCG_OPF_NOT_PRESENT)
+#else
 DEF(movcond_i64, 1, 4, 1, 0)
+#endif
 /* load/store */
 DEF(ld8u_i64, 1, 1, 1, 0)
 DEF(ld8s_i64, 1, 1, 1, 0)
@@ -183,7 +191,11 @@ DEF(insn_start, 0, 0, DATA64_ARGS, TCG_OPF_NOT_PRESENT)
 
 DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_EXIT | TCG_OPF_BB_END | TCG_OPF_NOT_PRESENT)
 DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_EXIT | TCG_OPF_BB_END | TCG_OPF_NOT_PRESENT)
+#if defined(CONFIG_TCG_THREADED_INTERPRETER)
+DEF(goto_ptr, 0, 1, 0, TCG_OPF_BB_EXIT | TCG_OPF_BB_END | TCG_OPF_NOT_PRESENT)
+#else
 DEF(goto_ptr, 0, 1, 0, TCG_OPF_BB_EXIT | TCG_OPF_BB_END)
+#endif
 
 DEF(plugin_cb, 0, 0, 1, TCG_OPF_NOT_PRESENT)
 DEF(plugin_mem_cb, 0, 1, 1, TCG_OPF_NOT_PRESENT)
include/tcg/tcg.h +1 -1
index 84d99508b6..74f0c7580e 100644
@@ -940,7 +940,7 @@ static inline size_t tcg_current_code_size(TCGContext *s)
 #define TB_EXIT_IDXMAX    1
 #define TB_EXIT_REQUESTED 3
 
-#ifdef CONFIG_TCG_INTERPRETER
+#if defined(CONFIG_TCG_INTERPRETER) || defined(CONFIG_TCG_THREADED_INTERPRETER)
 uintptr_t tcg_qemu_tb_exec(CPUArchState *env, const void *tb_ptr);
 #else
 typedef uintptr_t tcg_prologue_fn(CPUArchState *env, const void *tb_ptr);
tcg/aarch64-tcti/tcg-target-con-set.h new file +32 -0
new file mode 100644
index 0000000000..a0b91bb320
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * TCI target-specific constraint sets.
+ * Copyright (c) 2021 Linaro
+ */
+
+/*
+ * C_On_Im(...) defines a constraint set with <n> outputs and <m> inputs.
+ * Each operand should be a sequence of constraint letters as defined by
+ * tcg-target-con-str.h; the constraint combination is inclusive or.
+ */
+
+// Simple register functions.
+C_O0_I1(r)
+C_O0_I2(r, r)
+C_O0_I3(r, r, r)
+//C_O0_I4(r, r, r, r)
+C_O1_I1(r, r)
+C_O1_I2(r, r, r)
+//C_O1_I4(r, r, r, r, r)
+//C_O2_I1(r, r, r)
+//C_O2_I2(r, r, r, r)
+//C_O2_I4(r, r, r, r, r, r)
+
+// Vector functions.
+C_O1_I1(w, w)
+C_O1_I1(w, r)
+C_O0_I2(w, r)
+C_O1_I1(w, wr)
+C_O1_I2(w, w, w)
+C_O1_I3(w, w, w, w)
+C_O1_I2(w, 0, w)
\ No newline at end of file
tcg/aarch64-tcti/tcg-target-con-str.h new file +20 -0
new file mode 100644
index 0000000000..94d06d3e74
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Define TCI target-specific operand constraints.
+ * Copyright (c) 2021 Linaro
+ */
+
+/*
+ * Define constraint letters for register sets:
+ * REGS(letter, register_mask)
+ */
+REGS('r', TCG_MASK_GP_REGISTERS)
+REGS('w', TCG_MASK_VECTOR_REGISTERS)
+
+/*
+ * Define constraint letters for constants:
+ * CONST(letter, TCG_CT_CONST_* bit set)
+ */
+
+// Simple 64-bit immediates.
+CONST('I', 0xFFFFFFFFFFFFFFFF)
tcg/aarch64-tcti/tcg-target-has.h new file +132 -0

132 lines not on this page · raw message

tcg/aarch64-tcti/tcg-target-mo.h new file +13 -0
new file mode 100644
index 0000000000..d246f7fefe
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Define target-specific memory model
+ * Copyright (c) 2009, 2011 Stefan Weil
+ */
+
+#ifndef TCG_TARGET_MO_H
+#define TCG_TARGET_MO_H
+
+// We'll need to enforce memory ordering with barriers.
+#define TCG_TARGET_DEFAULT_MO  (0)
+ 
+#endif
tcg/aarch64-tcti/tcg-target-reg-bits.h new file +16 -0
new file mode 100644
index 0000000000..43cf075f6f
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Define target-specific register size
+ * Copyright (c) 2009, 2011 Stefan Weil
+ */
+
+#ifndef TCG_TARGET_REG_BITS_H
+#define TCG_TARGET_REG_BITS_H
+
+#if UINTPTR_MAX == UINT64_MAX
+# define TCG_TARGET_REG_BITS 64
+#else
+# error Unknown pointer size for tci target
+#endif
+
+#endif
tcg/aarch64-tcti/tcg-target.h new file +107 -0

107 lines not on this page · raw message

host/include/generic/host/atomic128-cas.h.inc +2 -1
index 6b40cc2271..6c788450ea 100644
@@ -11,7 +11,8 @@
 #ifndef HOST_ATOMIC128_CAS_H
 #define HOST_ATOMIC128_CAS_H
 
-#if defined(CONFIG_ATOMIC128)
+/* FIXME: this doesn't work in TCTI */
+#if defined(CONFIG_ATOMIC128) && !defined(CONFIG_TCG_THREADED_INTERPRETER)
 static inline Int128 ATTRIBUTE_ATOMIC128_OPT
 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
 {
tcg/aarch64-tcti/tcg-target-opc.h.inc new file +15 -0
new file mode 100644
index 0000000000..5382315c41
@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2019 Linaro
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.
+ *
+ * See the COPYING file in the top-level directory for details.
+ *
+ * Target-specific opcodes for host vector expansion.  These will be
+ * emitted by tcg_expand_vec_op.  For those familiar with GCC internals,
+ * consider these to be UNSPEC with names.
+ */
+
+DEF(aa64_sshl_vec, 1, 2, 0, TCG_OPF_VECTOR)
+DEF(aa64_sli_vec, 1, 2, 1, TCG_OPF_VECTOR)
accel/tcg/cputlb.c +2 -1
index fb22048876..1040af2d22 100644
@@ -2890,7 +2890,8 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
 #include "atomic_template.h"
 #endif
 
-#if defined(CONFIG_ATOMIC128) || HAVE_CMPXCHG128
+/* FIXME: this doesn't work in TCTI */
+#if (defined(CONFIG_ATOMIC128) && !defined(CONFIG_TCG_THREADED_INTERPRETER)) || HAVE_CMPXCHG128
 #define DATA_SIZE 16
 #include "atomic_template.h"
 #endif
accel/tcg/tcg-accel-ops.c +8 -0
index d9b662efe3..5564b483a8 100644
@@ -64,6 +64,14 @@ void tcg_cpu_init_cflags(CPUState *cpu, bool parallel)
 
     cflags |= parallel ? CF_PARALLEL : 0;
     cflags |= icount_enabled() ? CF_USE_ICOUNT : 0;
+#if defined(CONFIG_TCG_THREADED_INTERPRETER)
+    /*
+     * GOTO_PTR is too complex to emit a simple gadget for.
+     * We'll let C handle it, since the overhead is similar.
+     */
+    cflags |= CF_NO_GOTO_PTR;
+    cpu->cflags_next_tb = CF_NO_GOTO_PTR;
+#endif
     tcg_cflags_set(cpu, cflags);
 }
 
tcg/optimize.c +2 -0
index f922f86a1d..418c068fe4 100644
@@ -2958,6 +2958,7 @@ void tcg_optimize(TCGContext *s)
         case INDEX_op_ld32u_i64:
             done = fold_tcg_ld(&ctx, op);
             break;
+#if !defined(CONFIG_TCG_THREADED_INTERPRETER) /* FIXME: this breaks TCTI */
         case INDEX_op_ld_i32:
         case INDEX_op_ld_i64:
         case INDEX_op_ld_vec:
@@ -2973,6 +2974,7 @@ void tcg_optimize(TCGContext *s)
         case INDEX_op_st_vec:
             done = fold_tcg_st_memcopy(&ctx, op);
             break;
+#endif
         case INDEX_op_mb:
             done = fold_mb(&ctx, op);
             break;
tcg/region.c +7 -4
index 478ec051c4..70996b5ab1 100644
@@ -568,7 +568,7 @@ static int alloc_code_gen_buffer_anon(size_t size, int prot,
     return prot;
 }
 
-#ifndef CONFIG_TCG_INTERPRETER
+#if !defined(CONFIG_TCG_INTERPRETER) && !defined(CONFIG_TCG_THREADED_INTERPRETER)
 #ifdef CONFIG_POSIX
 #include "qemu/memfd.h"
 
@@ -670,7 +670,7 @@ static int alloc_code_gen_buffer_splitwx_vmremap(size_t size, Error **errp)
 
 static int alloc_code_gen_buffer_splitwx(size_t size, Error **errp)
 {
-#ifndef CONFIG_TCG_INTERPRETER
+#if !defined(CONFIG_TCG_INTERPRETER) && !defined(CONFIG_TCG_THREADED_INTERPRETER)
 # ifdef CONFIG_DARWIN
     return alloc_code_gen_buffer_splitwx_vmremap(size, errp);
 # endif
@@ -710,7 +710,10 @@ static int alloc_code_gen_buffer(size_t size, int splitwx, Error **errp)
      */
     prot = PROT_NONE;
     flags = MAP_PRIVATE | MAP_ANONYMOUS;
-#ifdef CONFIG_DARWIN
+#if defined(CONFIG_TCG_INTERPRETER) || defined(CONFIG_TCG_THREADED_INTERPRETER)
+    /* The tcg interpreter does not need execute permission. */
+    prot = PROT_READ | PROT_WRITE;
+#elif defined(CONFIG_DARWIN)
     /* Applicable to both iOS and macOS (Apple Silicon). */
     if (!splitwx) {
         flags |= MAP_JIT;
@@ -816,7 +819,7 @@ void tcg_region_init(size_t tb_size, int splitwx, unsigned max_cpus)
      * Work with the page protections set up with the initial mapping.
      */
     need_prot = PROT_READ | PROT_WRITE;
-#ifndef CONFIG_TCG_INTERPRETER
+#if !defined(CONFIG_TCG_INTERPRETER) && !defined(CONFIG_TCG_THREADED_INTERPRETER)
     if (tcg_splitwx_diff == 0) {
         need_prot |= host_prot_read_exec();
     }
tcg/tcg-op.c +27 -0
index fec6d678a2..e58b3fefd3 100644
@@ -1150,7 +1150,18 @@ void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
     } else if (cond == TCG_COND_NEVER) {
         tcg_gen_mov_i32(ret, v2);
     } else {
+#if defined(CONFIG_TCG_THREADED_INTERPRETER)
+        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
+        TCGv_i32 t1 = tcg_temp_ebb_new_i32();
+        tcg_gen_negsetcond_i32(cond, t0, c1, c2);
+        tcg_gen_and_i32(t1, v1, t0);
+        tcg_gen_andc_i32(ret, v2, t0);
+        tcg_gen_or_i32(ret, ret, t1);
+        tcg_temp_free_i32(t0);
+        tcg_temp_free_i32(t1);
+#else
         tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
+#endif
     }
 }
 
@@ -3002,8 +3013,23 @@ void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
     } else if (cond == TCG_COND_NEVER) {
         tcg_gen_mov_i64(ret, v2);
     } else if (TCG_TARGET_REG_BITS == 64) {
+#if defined(CONFIG_TCG_THREADED_INTERPRETER)
+        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
+        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
+        tcg_gen_negsetcond_i64(cond, t0, c1, c2);
+        tcg_gen_and_i64(t1, v1, t0);
+        tcg_gen_andc_i64(ret, v2, t0);
+        tcg_gen_or_i64(ret, ret, t1);
+        tcg_temp_free_i64(t0);
+        tcg_temp_free_i64(t1);
+#else
         tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
+#endif
     } else {
+#if defined(CONFIG_TCG_THREADED_INTERPRETER)
+        /* we do not support 32-bit TCTI */
+        g_assert_not_reached();
+#else
         TCGv_i32 t0 = tcg_temp_ebb_new_i32();
         TCGv_i32 zero = tcg_constant_i32(0);
 
@@ -3017,6 +3043,7 @@ void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
                             TCGV_HIGH(v1), TCGV_HIGH(v2));
 
         tcg_temp_free_i32(t0);
+#endif
     }
 }
 
tcg/tcg.c +12 -7
index dfd48b8264..229687c0c2 100644
@@ -251,7 +251,7 @@ TCGv_env tcg_env;
 const void *tcg_code_gen_epilogue;
 uintptr_t tcg_splitwx_diff;
 
-#ifndef CONFIG_TCG_INTERPRETER
+#if !defined(CONFIG_TCG_INTERPRETER) && !defined(CONFIG_TCG_THREADED_INTERPRETER)
 tcg_prologue_fn *tcg_qemu_tb_exec;
 #endif
 
@@ -956,7 +956,7 @@ static const TCGConstraintSet constraint_sets[] = {
 
 #include "tcg-target.c.inc"
 
-#ifndef CONFIG_TCG_INTERPRETER
+#if !defined(CONFIG_TCG_INTERPRETER) && !defined(CONFIG_TCG_THREADED_INTERPRETER)
 /* Validate CPUTLBDescFast placement. */
 QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
                         sizeof(CPUNegativeOffsetState))
@@ -1593,7 +1593,7 @@ void tcg_prologue_init(void)
     s->code_buf = s->code_gen_ptr;
     s->data_gen_ptr = NULL;
 
-#ifndef CONFIG_TCG_INTERPRETER
+#if !defined(CONFIG_TCG_INTERPRETER) && !defined(CONFIG_TCG_THREADED_INTERPRETER)
     tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
 #endif
 
@@ -1612,7 +1612,7 @@ void tcg_prologue_init(void)
     prologue_size = tcg_current_code_size(s);
     perf_report_prologue(s->code_gen_ptr, prologue_size);
 
-#ifndef CONFIG_TCG_INTERPRETER
+#if !defined(CONFIG_TCG_INTERPRETER) && !defined(CONFIG_TCG_THREADED_INTERPRETER)
     flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
                         (uintptr_t)s->code_buf, prologue_size);
 #endif
@@ -1649,7 +1649,7 @@ void tcg_prologue_init(void)
         }
     }
 
-#ifndef CONFIG_TCG_INTERPRETER
+#if !defined(CONFIG_TCG_INTERPRETER) && !defined(CONFIG_TCG_THREADED_INTERPRETER)
     /*
      * Assert that goto_ptr is implemented completely, setting an epilogue.
      * For tci, we use NULL as the signal to return from the interpreter,
@@ -2137,6 +2137,12 @@ bool tcg_op_supported(TCGOpcode op, TCGType type, unsigned flags)
     }
 
     switch (op) {
+    case INDEX_op_goto_ptr:
+#if defined(CONFIG_TCG_THREADED_INTERPRETER)
+        return false;
+#else
+        return true;
+#endif
     case INDEX_op_discard:
     case INDEX_op_set_label:
     case INDEX_op_call:
@@ -2145,7 +2151,6 @@ bool tcg_op_supported(TCGOpcode op, TCGType type, unsigned flags)
     case INDEX_op_insn_start:
     case INDEX_op_exit_tb:
     case INDEX_op_goto_tb:
-    case INDEX_op_goto_ptr:
     case INDEX_op_qemu_ld_i32:
     case INDEX_op_qemu_st_i32:
     case INDEX_op_qemu_ld_i64:
@@ -6498,7 +6503,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start)
         return -2;
     }
 
-#ifndef CONFIG_TCG_INTERPRETER
+#if !defined(CONFIG_TCG_INTERPRETER) && !defined(CONFIG_TCG_THREADED_INTERPRETER)
     /* flush instruction cache */
     flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
                         (uintptr_t)s->code_buf,
tcg/aarch64-tcti/tcg-target.c.inc new file +2250 -0

1,231 lines not on this page · raw message

meson_options.txt +2 -0
index 59d973bca0..92c6efeb34 100644

Lines of this file are not shown. View the raw message.

scripts/meson-buildoptions.sh +5 -0
index 3e8e00852b..8eb1954243 100644

Lines of this file are not shown. View the raw message.

tcg/aarch64-tcti/tcti-gadget-gen.py new file +1192 -0

1,192 lines not on this page · raw message

tcg/meson.build +69 -2

71 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.41.0

Part of the diff is not shown here. View the raw message.

#2 lore raw
In reply to Joelle van Dyne
Quoted 9 lines · Joelle van Dyne

The following patch is from work by Katherine Temkin to add a JITless backend for aarch64. The aarch64-tcti target for TCG uses pre-compiled "gadgets" which are snippets of code for every TCG op x all operands and then at runtime will "thread" together the gadgets with jumps after each gadget. This results in significant speedup against TCI but is still much slower than JIT.

This backend is mainly useful for iOS, which does not allow JIT in distributed apps. We ported the feature from v5.2 to v10.0 but will rebase it to master if there is interest. Would this backend be useful for mainline QEMU?

That's certainly interesting.

(1) The generated gadget code is excessively large: 75MiB of code, 18MiB of tables.

$ ld -r -o z.o *gadgets.c.o && size z.o

text data
74556216 18358784

Have you done experiments with the number of available registers to see at what point you get diminishing returns? The combinatorics on 16 registers is not in your favor.

I would expect you could make do with e.g. 6 writable registers, plus SP and ENV. Note that SP and ENV may be read in controlled ways, but not written. You would never generate a gadget that writes to either. They need not be accessed in other ways, such as the source of a multiply.

Have you done experiments with 2-operand matching constraints? I.e. "d*=m" instead of "d=n*m".
That will also dramatically affect combinatorics.

Have you experimented with, for the most part, generating *only* 64-bit code? For many operations, the 32-bit operation can be implemented by the 64-bit instruction simply by ignoring the high 32 bits of the output. E.g. add, sub, mul, logicals, left-shift. That would avoid repeating some gadgets for _i32 and _i64.

(2) The use of __attribute__((naked)) means this is clang-specific.

I'm really not sure why you're generating C and using naked, as opposed to simply generating assembly directly. By generating assembly, you could also emit the correct unwind info for the gadgets. Or, one set of unwind info for *all* of the gadgets.

I'll also note that it takes up to 5 minutes for clang to compile some of these files, so using the assembler instead of the compiler would help with that as well.

(3) The tables of gadgets are writable, placed in .data.

With C, it's trivial to simply place the "const" correctly to place these tables in mostly-read-only memory (.data.rel.ro; constant after runtime relocation).

With assembly, it's trivial to generate 32-bit pc-relative offsets instead of raw pointers, which would allow truly read-only memory (.rodata). This would require trivial changes to the read of the tables within tcg_out_*gadget.

(4) Some of the other changes are unacceptable, such as the ifdefs for movcond.

I'm surprised about that one, really. I suppose you were worried about combinatorics on a 5-operand operation. But there's no reason you need to keep comparisons next to uses.

You are currently generating C*N^3 versions of setcond_i64, setcond_i32, negsetcond_i64, negsetcond_i32, brcond_i64, and brcond_i32.

But instead you can generate N^2 versions of compare_i32 and compare_i64, and then chain that to C versions of brcond, C*N versions of setcond and negsetcond, and C*N^2 versions of movcond. The cpu flags retain the comparison state across the two gadgets.

(5) docs/devel/tcg-tcti.rst generates a build error.

This is caused by not being linked to the index.

(6) Do you actually see much speed-up from the vector code?

(7) To be properly reviewed, this would have to be split into many many patches.

(8) True interest in this is probably limited to iOS, and thus an aarch64 host.

I guess I'm ok with that. I would insist that the code remain buildable on any aarch64 host so that we can test it in CI.

I have thought about using gcc's computed goto instead of assembly gadgets, which would allow this general scheme to work on any host. We would be at the mercy of the quality of code generation though, and the size of the generated function might break the compiler -- this has happened before for machine-generated interpreters. But it would avoid two interpreter implementations, which would be a big plus.

Here's an incomplete example:

uintptr_t tcg_qemu_tb_exec(CPUArchState *env, const void *v_tb_ptr)

{
    static void * const table[20]
        __asm__("tci_table")  __attribute__((used)) = {
        &&mov_0_1,
        &&mov_0_2,
        &&mov_0_3,
        &&mov_1_0,
        &&mov_1_2,
        &&mov_1_3,
        &&mov_2_0,
        &&mov_2_1,
        &&mov_2_3,
        &&mov_3_0,
        &&mov_3_1,
        &&mov_3_2,
        &&stq_0_sp,
        &&stq_1_sp,
        &&stq_2_sp,
        &&stq_3_sp,
        &&ldq_0_sp,
        &&ldq_1_sp,
        &&ldq_2_sp,
        &&ldq_3_sp,
    };

    const intptr_t *tb_ptr = v_tb_ptr;
    tcg_target_ulong r0 = 0, r1 = 0, r2 = 0, r3 = 0;
    uint64_t stack[(TCG_STATIC_CALL_ARGS_SIZE + TCG_STATIC_FRAME_SIZE)
                   / sizeof(uint64_t)];
    uintptr_t sp = (uintptr_t)stack;

#define NEXT goto *tb_ptr++

    NEXT;

    mov_0_1: r0 = r1; NEXT;
    mov_0_2: r0 = r2; NEXT;
    mov_0_3: r0 = r3; NEXT;
    mov_1_0: r1 = r0; NEXT;
    mov_1_2: r1 = r2; NEXT;
    mov_1_3: r1 = r3; NEXT;
    mov_2_0: r2 = r0; NEXT;
    mov_2_1: r2 = r1; NEXT;
    mov_2_3: r2 = r3; NEXT;
    mov_3_0: r3 = r0; NEXT;
    mov_3_1: r3 = r1; NEXT;
    mov_3_2: r3 = r2; NEXT;
    stq_0_sp: *(uint64_t *)(sp + *tb_ptr++) = r0; NEXT;
    stq_1_sp: *(uint64_t *)(sp + *tb_ptr++) = r1; NEXT;
    stq_2_sp: *(uint64_t *)(sp + *tb_ptr++) = r2; NEXT;
    stq_3_sp: *(uint64_t *)(sp + *tb_ptr++) = r3; NEXT;
    ldq_0_sp: r0 = *(uint64_t *)(sp + *tb_ptr++); NEXT;
    ldq_1_sp: r1 = *(uint64_t *)(sp + *tb_ptr++); NEXT;
    ldq_2_sp: r2 = *(uint64_t *)(sp + *tb_ptr++); NEXT;
    ldq_3_sp: r3 = *(uint64_t *)(sp + *tb_ptr++); NEXT;

#undef NEXT

    return 0;
}

This compiles to fragments like

// ldq_x_sp

  3c:   f9400420        ldr     x0, [x1, #8]
  40:   f8410425        ldr     x5, [x1], #16
  44:   f86668a5        ldr     x5, [x5, x6]
  48:   d61f0000        br      x0

// stq_x_sp

  7c:   aa0103e7        mov     x7, x1
  80:   f84104e0        ldr     x0, [x7], #16
  84:   f8266805        str     x5, [x0, x6]
  88:   f9400420        ldr     x0, [x1, #8]
  8c:   aa0703e1        mov     x1, x7
  90:   d61f0000        br      x0

// mov_x_y

 154:   f8408420        ldr     x0, [x1], #8
 158:   aa0403e2        mov     x2, x4
 15c:   d61f0000        br      x0

While stq has two unnecessary moves, the other two are optimal.

Obviously, the function would be not be hand-written in a complete implementation.

r~