Skip to contentSkip to conversation

tcg aarch64 improvements

RFC Superseded Richard Henderson 1 participant 15 messages Aug 12, 2013

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

cover tcg aarch64 improvements

Several of these patches need cleaning up, but brings the state of the port in line with the other targets.

The last patch is dependant on another patch set I've got in the queue for 1.7 (http://patchwork.ozlabs.org/patch/264736/), but is otherwise independant.

The patch set is at git://github.com/rth7680/qemu.git tcg-aarch

r~

Richard Henderson (14):
tcg-aarch64: Allow immediate operands to add and sub
tcg-aarch64: Allow immediate operands to and, or, xor
tcg-aarch64: Allow immediate operands to compare
tcg-aarch64: Convert from opcode enums to insn enums
tcg-aarch64: Support andc, orc, eqv, not
tcg-aarch64: Handle zero as first argument to sub
tcg-aarch64: Support movcond
tcg-aarch64: Support deposit
tcg-aarch64: Support add2, sub2
tcg-aarch64: Support div, mulu2
tcg-aarch64: Improve tcg_out_movi
tcg-aarch64: Avoid add with zero in tlb load
tcg-aarch64: Use adrp in tcg_out_movi
tcg-aarch64: Pass return address to load/store helpers directly.

 include/exec/exec-all.h     |   16 +-
 tcg/aarch64/bitmask-table.c |   85 +
 tcg/aarch64/bitmask-table.h | 5342 +++++++++++++++++++++++++++++++++++++++++++
 tcg/aarch64/tcg-target.c    |  979 +++++---
 tcg/aarch64/tcg-target.h    |   38 +-
 5 files changed, 6143 insertions(+), 317 deletions(-)
 create mode 100644 tcg/aarch64/bitmask-table.c
 create mode 100644 tcg/aarch64/bitmask-table.h
Signature · 1 line
1.8.3.1
#1 lore raw

RFC 1/14 tcg-aarch64: Allow immediate operands to add and sub

Use signed 25-bit operands, because two 12-bit operations is smaller than movz+movk+add.

---
 tcg/aarch64/tcg-target.c | 107 +++++++++++++++++++++++++++--------------------
 1 file changed, 61 insertions(+), 46 deletions(-)
tcg/aarch64/tcg-target.c +61 -46
index 41a17f8..ed32f64 100644
@@ -106,6 +106,8 @@ static inline void patch_reloc(uint8_t *code_ptr, int type,
     }
 }
 
+#define TCG_CT_CONST_S25  0x100
+
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct,
                                    const char **pct_str)
@@ -129,6 +131,9 @@ static int target_parse_constraint(TCGArgConstraint *ct,
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_X3);
 #endif
         break;
+    case 'A': /* 25-bit signed, to be added or subtracted.  */
+        ct->ct |= TCG_CT_CONST_S25;
+        break;
     default:
         return -1;
     }
@@ -146,6 +151,9 @@ static inline int tcg_target_const_match(tcg_target_long val,
     if (ct & TCG_CT_CONST) {
         return 1;
     }
+    if ((ct & TCG_CT_CONST_S25) && sextract32(val, 0, 25) == val) {
+        return 1;
+    }
 
     return 0;
 }
@@ -200,12 +208,14 @@ enum aarch64_ldst_op_type { /* type of operation */
 };
 
 enum aarch64_arith_opc {
-    ARITH_AND = 0x0a,
-    ARITH_ADD = 0x0b,
-    ARITH_OR = 0x2a,
+    ARITH_AND  = 0x0a,
+    ARITH_ADD  = 0x0b,
+    ARITH_ADDI = 0x11,
+    ARITH_OR   = 0x2a,
     ARITH_ADDS = 0x2b,
-    ARITH_XOR = 0x4a,
-    ARITH_SUB = 0x4b,
+    ARITH_XOR  = 0x4a,
+    ARITH_SUB  = 0x4b,
+    ARITH_SUBI = 0x51,
     ARITH_ANDS = 0x6a,
     ARITH_SUBS = 0x6b,
 };
@@ -453,6 +463,20 @@ static inline void tcg_out_arith(TCGContext *s, enum aarch64_arith_opc opc,
     tcg_out32(s, base | rm << 16 | shift | rn << 5 | rd);
 }
 
+static inline void tcg_out_aimm(TCGContext *s, enum aarch64_arith_opc opc,
+                                int ext, TCGReg rd, TCGReg rn, uint64_t aimm)
+{
+    unsigned int base = (ext ? 0x80 | opc : opc) << 24;
+
+    if (aimm > 0xfff) {
+        assert((aimm & 0xfff) == 0);
+        aimm >>= 12;
+        base |= 1 << 22; /* apply LSL 12 */
+        assert(aimm <= 0xfff);
+    }
+    tcg_out32(s, base | (aimm << 10) | (rn << 5) | rd);
+}
+
 static inline void tcg_out_mul(TCGContext *s, int ext,
                                TCGReg rd, TCGReg rn, TCGReg rm)
 {
@@ -732,44 +756,27 @@ static inline void tcg_out_uxt(TCGContext *s, int s_bits,
     tcg_out_ubfm(s, 0, rd, rn, 0, bits);
 }
 
-static inline void tcg_out_addi(TCGContext *s, int ext,
-                                TCGReg rd, TCGReg rn, unsigned int aimm)
+static void tcg_out_addi(TCGContext *s, int ext, TCGReg rd, TCGReg rn,
+                         tcg_target_long aimm)
 {
-    /* add immediate aimm unsigned 12bit value (with LSL 0 or 12) */
-    /* using ADD 0x11000000 | (ext) | (aimm << 10) | (rn << 5) | rd */
-    unsigned int base = ext ? 0x91000000 : 0x11000000;
+    enum aarch64_arith_opc opc = ARITH_ADDI;
+    tcg_target_long lo, hi;
 
-    if (aimm <= 0xfff) {
-        aimm <<= 10;
-    } else {
-        /* we can only shift left by 12, on assert we cannot represent */
-        assert(!(aimm & 0xfff));
-        assert(aimm <= 0xfff000);
-        base |= 1 << 22; /* apply LSL 12 */
-        aimm >>= 2;
+    if (aimm < 0) {
+        aimm = -aimm;
+        opc = ARITH_SUBI;
     }
+    hi = aimm & 0xfff000;
+    lo = aimm & 0xfff;
+    assert(aimm == hi + lo);
 
-    tcg_out32(s, base | aimm | (rn << 5) | rd);
-}
-
-static inline void tcg_out_subi(TCGContext *s, int ext,
-                                TCGReg rd, TCGReg rn, unsigned int aimm)
-{
-    /* sub immediate aimm unsigned 12bit value (with LSL 0 or 12) */
-    /* using SUB 0x51000000 | (ext) | (aimm << 10) | (rn << 5) | rd */
-    unsigned int base = ext ? 0xd1000000 : 0x51000000;
-
-    if (aimm <= 0xfff) {
-        aimm <<= 10;
-    } else {
-        /* we can only shift left by 12, on assert we cannot represent */
-        assert(!(aimm & 0xfff));
-        assert(aimm <= 0xfff000);
-        base |= 1 << 22; /* apply LSL 12 */
-        aimm >>= 2;
+    if (hi != 0) {
+        tcg_out_aimm(s, opc, ext, rd, rn, hi);
+        rn = rd;
+    }
+    if (lo != 0 || rd != rn) {
+        tcg_out_aimm(s, opc, ext, rd, rn, lo);
     }
-
-    tcg_out32(s, base | aimm | (rn << 5) | rd);
 }
 
 static inline void tcg_out_nop(TCGContext *s)
@@ -1180,13 +1187,21 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_add_i64:
         ext = 1; /* fall through */
     case INDEX_op_add_i32:
-        tcg_out_arith(s, ARITH_ADD, ext, args[0], args[1], args[2], 0);
+        if (const_args[2]) {
+            tcg_out_addi(s, ext, args[0], args[1], args[2]);
+        } else {
+            tcg_out_arith(s, ARITH_ADD, ext, args[0], args[1], args[2], 0);
+        }
         break;
 
     case INDEX_op_sub_i64:
         ext = 1; /* fall through */
     case INDEX_op_sub_i32:
-        tcg_out_arith(s, ARITH_SUB, ext, args[0], args[1], args[2], 0);
+        if (const_args[2]) {
+            tcg_out_addi(s, ext, args[0], args[1], -args[2]);
+        } else {
+            tcg_out_arith(s, ARITH_SUB, ext, args[0], args[1], args[2], 0);
+        }
         break;
 
     case INDEX_op_and_i64:
@@ -1391,10 +1406,10 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_st32_i64, { "r", "r" } },
     { INDEX_op_st_i64, { "r", "r" } },
 
-    { INDEX_op_add_i32, { "r", "r", "r" } },
-    { INDEX_op_add_i64, { "r", "r", "r" } },
-    { INDEX_op_sub_i32, { "r", "r", "r" } },
-    { INDEX_op_sub_i64, { "r", "r", "r" } },
+    { INDEX_op_add_i32, { "r", "r", "rA" } },
+    { INDEX_op_add_i64, { "r", "r", "rA" } },
+    { INDEX_op_sub_i32, { "r", "r", "rA" } },
+    { INDEX_op_sub_i64, { "r", "r", "rA" } },
     { INDEX_op_mul_i32, { "r", "r", "r" } },
     { INDEX_op_mul_i64, { "r", "r", "r" } },
     { INDEX_op_and_i32, { "r", "r", "r" } },
@@ -1518,8 +1533,8 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     }
 
     /* make stack space for TCG locals */
-    tcg_out_subi(s, 1, TCG_REG_SP, TCG_REG_SP,
-                 frame_size_tcg_locals * TCG_TARGET_STACK_ALIGN);
+    tcg_out_addi(s, 1, TCG_REG_SP, TCG_REG_SP,
+                 -frame_size_tcg_locals * TCG_TARGET_STACK_ALIGN);
     /* inform TCG about how to find TCG locals with register, offset, size */
     tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE,
                   CPU_TEMP_BUF_NLONGS * sizeof(long));
Signature · 1 line
1.8.3.1
#2 lore raw

RFC 2/14 tcg-aarch64: Allow immediate operands to and, or, xor

In reply to Richard Henderson

It isn't obvious how to have the makefile generate bitmask-table.h, so check it in along with the generator.

---
 tcg/aarch64/bitmask-table.c |   85 +
 tcg/aarch64/bitmask-table.h | 5342 +++++++++++++++++++++++++++++++++++++++++++
 tcg/aarch64/tcg-target.c    |   82 +-
 3 files changed, 5500 insertions(+), 9 deletions(-)
 create mode 100644 tcg/aarch64/bitmask-table.c
 create mode 100644 tcg/aarch64/bitmask-table.h
tcg/aarch64/bitmask-table.c new file +85 -0
new file mode 100644
index 0000000..03a1a52
@@ -0,0 +1,85 @@
+#include <stdint.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+typedef struct {
+    uint64_t imm;
+    unsigned enc;
+} imm_enc;
+
+static int do_cmp (const void *x1, const void *x2)
+{
+  const imm_enc *i1 = x1;
+  const imm_enc *i2 = x2;
+
+  if (i1->imm < i2->imm)
+    return -1;
+  if (i1->imm > i2->imm)
+    return +1;
+  return 0;
+}
+
+#define N 5334
+static imm_enc bitmasks[N];
+
+int main(void)
+{
+    unsigned int log_e;
+    unsigned int n = 0;
+
+    for (log_e = 1; log_e <= 6; log_e++) {
+        unsigned s, e = 1 << log_e;
+        uint64_t mask = (e == 64 ? 0 : 1ULL << e) - 1;
+        unsigned s_mask = (0x3e << log_e) & 0x3f;
+
+        for (s = 0; s < e - 1; s++) {
+            unsigned r;
+            for (r = 0; r < e; r++) {
+                /* set s+1 consecutive bits to 1 (s < 64) */
+                uint64_t imm = (1ULL << (s + 1)) - 1;
+
+                /* rotate right by r */
+                if (r != 0) {
+                    imm = ((imm >> r) | (imm << (e - r))) & mask;
+                }
+
+                /* replicate the constant depending on SIMD size */
+                switch (log_e) {
+                case 1: imm |= (imm <<  2);
+                case 2: imm |= (imm <<  4);
+                case 3: imm |= (imm <<  8);
+                case 4: imm |= (imm << 16);
+                case 5: imm |= (imm << 32);
+                case 6:
+                  break;
+                default:
+                  abort();
+                }
+                assert (n < N);
+                bitmasks[n].imm = imm;
+                bitmasks[n].enc = (e == 64) << 12 | (r << 6) | s | s_mask;
+                n++;
+            }
+        }
+    }
+    assert (n == N);
+
+    qsort (bitmasks, N, sizeof(bitmasks[0]), do_cmp);
+
+    printf("/* Auto-generated from bitmask-table.c.  */\n\n");
+
+    printf("static uint64_t const bitmask_imm[] = {\n");
+    for (n = 0; n < N; n += 2) {
+        printf("    %#018lx, %#018lx,\n", bitmasks[n].imm, bitmasks[n+1].imm);
+    }
+    printf("};\n\n");
+
+    printf("static uint16_t const bitmask_enc[] = {\n");
+    for (n = 0; n < N; n += 2) {
+        printf("    %#06x, %#06x,\n", bitmasks[n].enc, bitmasks[n+1].enc);
+    }
+    printf("};\n\n");
+
+    return 0;
+}
tcg/aarch64/bitmask-table.h new file +5342 -0

2,915 lines not on this page · raw message

tcg/aarch64/tcg-target.c +73 -9

82 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
1.8.3.1

Part of the diff is not shown here. View the raw message.

#3 lore raw

RFC 3/14 tcg-aarch64: Allow immediate operands to compare

In reply to Richard Henderson
---
 tcg/aarch64/tcg-target.c | 74 +++++++++++++++++++++++++++++++++---------------
 1 file changed, 51 insertions(+), 23 deletions(-)
tcg/aarch64/tcg-target.c +51 -23
index e8370a9..76595b4 100644
@@ -109,6 +109,7 @@ static inline void patch_reloc(uint8_t *code_ptr, int type,
 #define TCG_CT_CONST_S25  0x100
 #define TCG_CT_CONST_LI32 0x200
 #define TCG_CT_CONST_LI64 0x400
+#define TCG_CT_CONST_CMP  0x800
 
 #include "bitmask-table.h"
 
@@ -135,6 +136,14 @@ static int find_bitmask32(uint32_t val)
     return find_bitmask64(((uint64_t)val << 32) | val);
 }
 
+static int can_cmpi(tcg_target_long val)
+{
+    if (val < 0) {
+        val = ~val;
+    }
+    return (val & ~0xfff) == 0 || (val & ~0xfff000) == 0;
+}
+
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct,
                                    const char **pct_str)
@@ -161,6 +170,9 @@ static int target_parse_constraint(TCGArgConstraint *ct,
     case 'A': /* 25-bit signed, to be added or subtracted.  */
         ct->ct |= TCG_CT_CONST_S25;
         break;
+    case 'C': /* a 12-bit constant (maybe inverted) to be used with compare. */
+        ct->ct |= TCG_CT_CONST_CMP;
+        break;
     case 'K': /* logical immediate 32-bit */
         ct->ct |= TCG_CT_CONST_LI32;
         break;
@@ -193,6 +205,9 @@ static inline int tcg_target_const_match(tcg_target_long val,
     if ((ct & TCG_CT_CONST_LI32) && find_bitmask32(val) >= 0) {
         return 1;
     }
+    if ((ct & TCG_CT_CONST_CMP) && can_cmpi(val)) {
+        return 1;
+    }
 
     return 0;
 }
@@ -247,19 +262,21 @@ enum aarch64_ldst_op_type { /* type of operation */
 };
 
 enum aarch64_arith_opc {
-    ARITH_AND  = 0x0a,
-    ARITH_ADD  = 0x0b,
-    ARITH_ADDI = 0x11,
-    ARITH_ANDI = 0x12,
-    ARITH_OR   = 0x2a,
-    ARITH_ADDS = 0x2b,
-    ARITH_ORI  = 0x32,
-    ARITH_XOR  = 0x4a,
-    ARITH_SUB  = 0x4b,
-    ARITH_SUBI = 0x51,
-    ARITH_XORI = 0x52,
-    ARITH_ANDS = 0x6a,
-    ARITH_SUBS = 0x6b,
+    ARITH_AND   = 0x0a,
+    ARITH_ADD   = 0x0b,
+    ARITH_ADDI  = 0x11,
+    ARITH_ANDI  = 0x12,
+    ARITH_OR    = 0x2a,
+    ARITH_ADDS  = 0x2b,
+    ARITH_ADDSI = 0x31,
+    ARITH_ORI   = 0x32,
+    ARITH_XOR   = 0x4a,
+    ARITH_SUB   = 0x4b,
+    ARITH_SUBI  = 0x51,
+    ARITH_XORI  = 0x52,
+    ARITH_ANDS  = 0x6a,
+    ARITH_SUBS  = 0x6b,
+    ARITH_SUBSI = 0x71,
 };
 
 enum aarch64_srr_opc {
@@ -609,11 +626,22 @@ static inline void tcg_out_rotl(TCGContext *s, int ext,
     tcg_out_extr(s, ext, rd, rn, rn, bits - (m & max));
 }
 
-static inline void tcg_out_cmp(TCGContext *s, int ext, TCGReg rn, TCGReg rm,
-                               int shift_imm)
+static void tcg_out_cmp(TCGContext *s, int ext, TCGReg a,
+                        tcg_target_long b, bool const_b)
 {
-    /* Using CMP alias SUBS wzr, Wn, Wm */
-    tcg_out_arith(s, ARITH_SUBS, ext, TCG_REG_XZR, rn, rm, shift_imm);
+    if (const_b) {
+        /* Using CMP alias SUBS xzr, Xn, const */
+        enum aarch64_arith_opc opc = ARITH_SUBSI;
+
+        if (b < 0) {
+            b = ~b;
+            opc = ARITH_ADDSI;
+        }
+        tcg_out_aimm(s, opc, ext, TCG_REG_XZR, a, b);
+    } else {
+        /* Using CMP alias SUBS wzr, Wn, Wm */
+        tcg_out_arith(s, ARITH_SUBS, ext, TCG_REG_XZR, a, b, 0);
+    }
 }
 
 static inline void tcg_out_cset(TCGContext *s, int ext, TCGReg rd, TCGCond c)
@@ -1348,14 +1376,14 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_brcond_i64:
         ext = 1; /* fall through */
     case INDEX_op_brcond_i32: /* CMP 0, 1, cond(2), label 3 */
-        tcg_out_cmp(s, ext, args[0], args[1], 0);
+        tcg_out_cmp(s, ext, args[0], args[1], const_args[1]);
         tcg_out_goto_label_cond(s, args[2], args[3]);
         break;
 
     case INDEX_op_setcond_i64:
         ext = 1; /* fall through */
     case INDEX_op_setcond_i32:
-        tcg_out_cmp(s, ext, args[1], args[2], 0);
+        tcg_out_cmp(s, ext, args[1], args[2], const_args[2]);
         tcg_out_cset(s, 0, args[0], args[3]);
         break;
 
@@ -1494,10 +1522,10 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_rotl_i64, { "r", "r", "ri" } },
     { INDEX_op_rotr_i64, { "r", "r", "ri" } },
 
-    { INDEX_op_brcond_i32, { "r", "r" } },
-    { INDEX_op_setcond_i32, { "r", "r", "r" } },
-    { INDEX_op_brcond_i64, { "r", "r" } },
-    { INDEX_op_setcond_i64, { "r", "r", "r" } },
+    { INDEX_op_brcond_i32, { "r", "rC" } },
+    { INDEX_op_setcond_i32, { "r", "r", "rC" } },
+    { INDEX_op_brcond_i64, { "r", "rC" } },
+    { INDEX_op_setcond_i64, { "r", "r", "rC" } },
 
     { INDEX_op_qemu_ld8u, { "r", "l" } },
     { INDEX_op_qemu_ld8s, { "r", "l" } },
Signature · 1 line
1.8.3.1
#4 lore raw

RFC 4/14 tcg-aarch64: Convert from opcode enums to insn enums

In reply to Richard Henderson

The difference being that INSN_* has been pre-shifted into place. This should result in less runtime shifting of constants.

The patch could be split into smaller pieces for clarity...

---
 tcg/aarch64/tcg-target.c | 518 ++++++++++++++++++++++++-----------------------
 1 file changed, 269 insertions(+), 249 deletions(-)
tcg/aarch64/tcg-target.c +269 -249
index 76595b4..3640486 100644
@@ -212,7 +212,76 @@ static inline int tcg_target_const_match(tcg_target_long val,
     return 0;
 }
 
-enum aarch64_cond_code {
+typedef enum {
+    /* Logical immediate instructions */
+    INSN_ANDI  = 0x12000000,
+    INSN_ORRI  = 0x32000000,
+    INSN_EORI  = 0x52000000,
+
+    /* Logical shifted register instructions */
+    INSN_AND   = 0x0a000000,
+    INSN_BIC   = 0x0a200000,
+    INSN_ORR   = 0x2a000000,
+    INSN_ORN   = 0x2a200000,
+    INSN_EOR   = 0x4a000000,
+    INSN_EON   = 0x4a200000,
+
+    /* Move wide immediate instructions */
+    INSN_MOVN  = 0x12800000,
+    INSN_MOVZ  = 0x52800000,
+    INSN_MOVK  = 0x72800000,
+
+    /* Add/subtract immediate instructions */
+    INSN_ADDI  = 0x11000000,
+    INSN_ADDSI = 0x31000000,
+    INSN_SUBI  = 0x51000000,
+    INSN_SUBSI = 0x71000000,
+
+    /* Add/subtract shifted register instructions */
+    INSN_ADD   = 0x0b000000,
+    INSN_SUB   = 0x4b000000,
+    INSN_SUBS  = 0x6b000000,
+
+    /* Data-processing (1 source) instructions */
+    INSN_REV16 = 0x5ac00400,
+    INSN_REVx  = 0xdac00c00,
+    INSN_REVw  = 0x5ac00800,
+
+    /* Data-processing (2 source) instructions */
+    INSN_LSLV  = 0x1ac02000,
+    INSN_LSRV  = 0x1ac02400,
+    INSN_ASRV  = 0x1ac02800,
+    INSN_RORV  = 0x1ac02c00,
+    INSN_MUL   = 0x1b007c00,  /* MADD alias with Ra = xzr */
+
+    /* Bitfield instructions */
+    INSN_SBFM  = 0x13000000,
+    INSN_UBFM  = 0x53000000,
+    INSN_EXTR  = 0x13800000,
+
+    /* Conditional select instructions */
+    INSN_CSINC = 0x1a800400,
+
+    /* Branch instructions */
+    INSN_B     = 0x14000000,
+    INSN_BL    = 0x94000000,
+    INSN_BR    = 0xd61f0000,
+    INSN_BLR   = 0xd63f0000,
+    INSN_RET   = 0xd65f0000,
+    INSN_B_C   = 0x54000000,
+
+    /* System instructions */
+    INSN_NOP   = 0xd503201f,
+} AArch64Insn;
+
+typedef enum {
+    E32 = 0,
+    E64 = 0x80000000u,
+} AArch64Ext;
+
+#define EXT(cond) (cond ? E64 : E32)
+
+typedef enum {
     COND_EQ = 0x0,
     COND_NE = 0x1,
     COND_CS = 0x2,     /* Unsigned greater or equal */
@@ -231,9 +300,9 @@ enum aarch64_cond_code {
     COND_LE = 0xd,
     COND_AL = 0xe,
     COND_NV = 0xf, /* behaves like COND_AL here */
-};
+} AArch64Cond;
 
-static const enum aarch64_cond_code tcg_cond_to_aarch64[] = {
+static const AArch64Cond tcg_cond_to_aarch64[] = {
     [TCG_COND_EQ] = COND_EQ,
     [TCG_COND_NE] = COND_NE,
     [TCG_COND_LT] = COND_LT,
@@ -261,31 +330,6 @@ enum aarch64_ldst_op_type { /* type of operation */
     LDST_LD_S_W = 0xc,  /* load and sign-extend into Wt */
 };
 
-enum aarch64_arith_opc {
-    ARITH_AND   = 0x0a,
-    ARITH_ADD   = 0x0b,
-    ARITH_ADDI  = 0x11,
-    ARITH_ANDI  = 0x12,
-    ARITH_OR    = 0x2a,
-    ARITH_ADDS  = 0x2b,
-    ARITH_ADDSI = 0x31,
-    ARITH_ORI   = 0x32,
-    ARITH_XOR   = 0x4a,
-    ARITH_SUB   = 0x4b,
-    ARITH_SUBI  = 0x51,
-    ARITH_XORI  = 0x52,
-    ARITH_ANDS  = 0x6a,
-    ARITH_SUBS  = 0x6b,
-    ARITH_SUBSI = 0x71,
-};
-
-enum aarch64_srr_opc {
-    SRR_SHL = 0x0,
-    SRR_SHR = 0x4,
-    SRR_SAR = 0x8,
-    SRR_ROR = 0xc
-};
-
 static inline enum aarch64_ldst_op_data
 aarch64_ldst_get_data(TCGOpcode tcg_op)
 {
@@ -395,46 +439,78 @@ static inline void tcg_out_ldst_12(TCGContext *s,
               | op_type << 20 | scaled_uimm << 10 | rn << 5 | rd);
 }
 
-static inline void tcg_out_movr(TCGContext *s, int ext, TCGReg rd, TCGReg src)
+/* Suitable for add/sub and logical shifted register instructions.  */
+static inline void tcg_out_arith(TCGContext *s, AArch64Insn insn,
+                                 AArch64Ext ext, TCGReg rd, TCGReg rn,
+                                 TCGReg rm, int shift_imm)
 {
-    /* register to register move using MOV (shifted register with no shift) */
-    /* using MOV 0x2a0003e0 | (shift).. */
-    unsigned int base = ext ? 0xaa0003e0 : 0x2a0003e0;
-    tcg_out32(s, base | src << 16 | rd);
+    unsigned int shift;
+    if (shift_imm == 0) {
+        shift = 0;
+    } else if (shift_imm > 0) {
+        shift = shift_imm << 10 | 1 << 22;
+    } else /* (shift_imm < 0) */ {
+        shift = (-shift_imm) << 10;
+    }
+    tcg_out32(s, insn | ext | shift | rm << 16 | rn << 5 | rd);
 }
 
-static inline void tcg_out_movi_aux(TCGContext *s,
-                                    TCGReg rd, uint64_t value)
+static inline void tcg_out_aimm(TCGContext *s, AArch64Insn insn,
+                                AArch64Ext ext, TCGReg rd, TCGReg rn,
+                                uint64_t aimm)
 {
-    uint32_t half, base, shift, movk = 0;
-    /* construct halfwords of the immediate with MOVZ/MOVK with LSL */
-    /* using MOVZ 0x52800000 | extended reg.. */
-    base = (value > 0xffffffff) ? 0xd2800000 : 0x52800000;
-    /* count trailing zeros in 16 bit steps, mapping 64 to 0. Emit the
-       first MOVZ with the half-word immediate skipping the zeros, with a shift
-       (LSL) equal to this number. Then morph all next instructions into MOVKs.
+    if (aimm > 0xfff) {
+        assert((aimm & 0xfff) == 0);
+        aimm >>= 12;
+        assert(aimm <= 0xfff);
+        aimm |= 1 << 12; /* apply LSL 12 */
+    }
+    tcg_out32(s, insn | ext | aimm << 10 | rn << 5 | rd);
+}
+
+static void tcg_out_limm(TCGContext *s, AArch64Insn insn, AArch64Ext ext,
+                         TCGReg rd, TCGReg rn, tcg_target_long val)
+{
+    int index = (ext ? find_bitmask64(val) : find_bitmask32(val));
+    assert(index >= 0);
+    tcg_out32(s, insn | ext | bitmask_enc[index] << 10 | rn << 5 | rd);
+}
+
+/* Register-register move, assuming XSP not XZR.  */
+static inline void tcg_out_movr(TCGContext *s, AArch64Ext ext,
+                                TCGReg dest, TCGReg src)
+{
+    tcg_out_aimm(s, INSN_ADDI, ext, dest, src, 0);
+}
+
+static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
+                         tcg_target_long value)
+{
+    AArch64Insn insn = INSN_MOVZ;
+
+    if (type == TCG_TYPE_I32) {
+        value = (uint32_t)value;
+    }
+
+    /* Construct halfwords of the immediate with MOVZ/MOVK with LSL.
+       Count trailing zeros in 16 bit steps, mapping 64 to 0.  Emit the
+       first MOVZ with the half-word immediate skipping the zeros, with
+       a shift (LSL) equal to this number.  Then all other insns are MOVKs.
        Zero the processed half-word in the value, continue until empty.
        We build the final result 16bits at a time with up to 4 instructions,
        but do not emit instructions for 16bit zero holes. */
     do {
-        shift = ctz64(value) & (63 & -16);
-        half = (value >> shift) & 0xffff;
-        tcg_out32(s, base | movk | shift << 17 | half << 5 | rd);
-        movk = 0x20000000; /* morph next MOVZs into MOVKs */
+        unsigned shift = ctz64(value) & (63 & -16);
+        unsigned half = (value >> shift) & 0xffff;
+        AArch64Ext ext = EXT(shift >= 32);
+
+        tcg_out32(s, insn | ext | shift << 17 | half << 5 | rd);
+
+        insn = INSN_MOVK;
         value &= ~(0xffffUL << shift);
     } while (value);
 }
 
-static inline void tcg_out_movi(TCGContext *s, TCGType type,
-                                TCGReg rd, tcg_target_long value)
-{
-    if (type == TCG_TYPE_I64) {
-        tcg_out_movi_aux(s, rd, value);
-    } else {
-        tcg_out_movi_aux(s, rd, value & 0xffffffff);
-    }
-}
-
 static inline void tcg_out_ldst_r(TCGContext *s,
                                   enum aarch64_ldst_op_data op_data,
                                   enum aarch64_ldst_op_type op_type,
@@ -475,19 +551,11 @@ static inline void tcg_out_ldst(TCGContext *s, enum aarch64_ldst_op_data data,
     tcg_out_ldst_r(s, data, type, rd, rn, TCG_REG_TMP);
 }
 
-/* mov alias implemented with add immediate, useful to move to/from SP */
-static inline void tcg_out_movr_sp(TCGContext *s, int ext, TCGReg rd, TCGReg rn)
-{
-    /* using ADD 0x11000000 | (ext) | rn << 5 | rd */
-    unsigned int base = ext ? 0x91000000 : 0x11000000;
-    tcg_out32(s, base | rn << 5 | rd);
-}
-
 static inline void tcg_out_mov(TCGContext *s,
                                TCGType type, TCGReg ret, TCGReg arg)
 {
     if (ret != arg) {
-        tcg_out_movr(s, type == TCG_TYPE_I64, ret, arg);
+        tcg_out_movr(s, EXT(type == TCG_TYPE_I64), ret, arg);
     }
 }
 
@@ -505,90 +573,35 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
                  arg, arg1, arg2);
 }
 
-static inline void tcg_out_arith(TCGContext *s, enum aarch64_arith_opc opc,
-                                 int ext, TCGReg rd, TCGReg rn, TCGReg rm,
-                                 int shift_imm)
-{
-    /* Using shifted register arithmetic operations */
-    /* if extended register operation (64bit) just OR with 0x80 << 24 */
-    unsigned int shift, base = ext ? (0x80 | opc) << 24 : opc << 24;
-    if (shift_imm == 0) {
-        shift = 0;
-    } else if (shift_imm > 0) {
-        shift = shift_imm << 10 | 1 << 22;
-    } else /* (shift_imm < 0) */ {
-        shift = (-shift_imm) << 10;
-    }
-    tcg_out32(s, base | rm << 16 | shift | rn << 5 | rd);
-}
-
-static inline void tcg_out_aimm(TCGContext *s, enum aarch64_arith_opc opc,
-                                int ext, TCGReg rd, TCGReg rn, uint64_t aimm)
-{
-    unsigned int base = (ext ? 0x80 | opc : opc) << 24;
-
-    if (aimm > 0xfff) {
-        assert((aimm & 0xfff) == 0);
-        aimm >>= 12;
-        base |= 1 << 22; /* apply LSL 12 */
-        assert(aimm <= 0xfff);
-    }
-    tcg_out32(s, base | (aimm << 10) | (rn << 5) | rd);
-}
-
-static void tcg_out_limm(TCGContext *s, enum aarch64_arith_opc opc, int ext,
-                         TCGReg rd, TCGReg rn, tcg_target_long val)
-{
-    int index = (ext ? find_bitmask64(val) : find_bitmask32(val));
-    unsigned base = (ext ? 0x80 | opc : opc) << 24;
-
-    assert(index >= 0);
-    tcg_out32(s, base | bitmask_enc[index] << 10 | rn << 5 | rd);
-}
-
-static inline void tcg_out_mul(TCGContext *s, int ext,
-                               TCGReg rd, TCGReg rn, TCGReg rm)
+static inline void tcg_out_data2(TCGContext *s, AArch64Insn insn,
+                                 AArch64Ext ext, TCGReg rd,
+                                 TCGReg rn, TCGReg rm)
 {
-    /* Using MADD 0x1b000000 with Ra = wzr alias MUL 0x1b007c00 */
-    unsigned int base = ext ? 0x9b007c00 : 0x1b007c00;
-    tcg_out32(s, base | rm << 16 | rn << 5 | rd);
+    tcg_out32(s, insn | ext | rm << 16 | rn << 5 | rd);
 }
 
-static inline void tcg_out_shiftrot_reg(TCGContext *s,
-                                        enum aarch64_srr_opc opc, int ext,
-                                        TCGReg rd, TCGReg rn, TCGReg rm)
+static inline void tcg_out_ubfm(TCGContext *s, AArch64Ext ext, TCGReg rd,
+                                TCGReg rn, unsigned int a, unsigned int b)
 {
-    /* using 2-source data processing instructions 0x1ac02000 */
-    unsigned int base = ext ? 0x9ac02000 : 0x1ac02000;
-    tcg_out32(s, base | rm << 16 | opc << 8 | rn << 5 | rd);
-}
-
-static inline void tcg_out_ubfm(TCGContext *s, int ext, TCGReg rd, TCGReg rn,
-                                unsigned int a, unsigned int b)
-{
-    /* Using UBFM 0x53000000 Wd, Wn, a, b */
-    unsigned int base = ext ? 0xd3400000 : 0x53000000;
+    unsigned int base = INSN_UBFM | (ext ? 0x80400000 : 0);
     tcg_out32(s, base | a << 16 | b << 10 | rn << 5 | rd);
 }
 
-static inline void tcg_out_sbfm(TCGContext *s, int ext, TCGReg rd, TCGReg rn,
-                                unsigned int a, unsigned int b)
+static inline void tcg_out_sbfm(TCGContext *s, AArch64Ext ext, TCGReg rd,
+                                TCGReg rn, unsigned int a, unsigned int b)
 {
-    /* Using SBFM 0x13000000 Wd, Wn, a, b */
-    unsigned int base = ext ? 0x93400000 : 0x13000000;
+    unsigned int base = INSN_SBFM | (ext ? 0x80400000 : 0);
     tcg_out32(s, base | a << 16 | b << 10 | rn << 5 | rd);
 }
 
-static inline void tcg_out_extr(TCGContext *s, int ext, TCGReg rd,
+static inline void tcg_out_extr(TCGContext *s, AArch64Ext ext, TCGReg rd,
                                 TCGReg rn, TCGReg rm, unsigned int a)
 {
-    /* Using EXTR 0x13800000 Wd, Wn, Wm, a */
-    unsigned int base = ext ? 0x93c00000 : 0x13800000;
-    tcg_out32(s, base | rm << 16 | a << 10 | rn << 5 | rd);
+    tcg_out32(s, INSN_EXTR | ext | rm << 16 | a << 10 | rn << 5 | rd);
 }
 
-static inline void tcg_out_shl(TCGContext *s, int ext,
-                               TCGReg rd, TCGReg rn, unsigned int m)
+static inline void tcg_out_shli(TCGContext *s, AArch64Ext ext, TCGReg rd,
+                                TCGReg rn, unsigned int m)
 {
     int bits, max;
     bits = ext ? 64 : 32;
@@ -596,29 +609,29 @@ static inline void tcg_out_shl(TCGContext *s, int ext,
     tcg_out_ubfm(s, ext, rd, rn, bits - (m & max), max - (m & max));
 }
 
-static inline void tcg_out_shr(TCGContext *s, int ext,
-                               TCGReg rd, TCGReg rn, unsigned int m)
+static inline void tcg_out_shri(TCGContext *s, AArch64Ext ext, TCGReg rd,
+                                TCGReg rn, unsigned int m)
 {
     int max = ext ? 63 : 31;
     tcg_out_ubfm(s, ext, rd, rn, m & max, max);
 }
 
-static inline void tcg_out_sar(TCGContext *s, int ext,
-                               TCGReg rd, TCGReg rn, unsigned int m)
+static inline void tcg_out_sari(TCGContext *s, AArch64Ext ext, TCGReg rd,
+                                TCGReg rn, unsigned int m)
 {
     int max = ext ? 63 : 31;
     tcg_out_sbfm(s, ext, rd, rn, m & max, max);
 }
 
-static inline void tcg_out_rotr(TCGContext *s, int ext,
-                                TCGReg rd, TCGReg rn, unsigned int m)
+static inline void tcg_out_rotri(TCGContext *s, AArch64Ext ext, TCGReg rd,
+                                 TCGReg rn, unsigned int m)
 {
     int max = ext ? 63 : 31;
     tcg_out_extr(s, ext, rd, rn, rn, m & max);
 }
 
-static inline void tcg_out_rotl(TCGContext *s, int ext,
-                                TCGReg rd, TCGReg rn, unsigned int m)
+static inline void tcg_out_rotli(TCGContext *s, AArch64Ext ext, TCGReg rd,
+                                 TCGReg rn, unsigned int m)
 {
     int bits, max;
     bits = ext ? 64 : 32;
@@ -626,29 +639,31 @@ static inline void tcg_out_rotl(TCGContext *s, int ext,
     tcg_out_extr(s, ext, rd, rn, rn, bits - (m & max));
 }
 
-static void tcg_out_cmp(TCGContext *s, int ext, TCGReg a,
+static void tcg_out_cmp(TCGContext *s, AArch64Ext ext, TCGReg a,
                         tcg_target_long b, bool const_b)
 {
     if (const_b) {
         /* Using CMP alias SUBS xzr, Xn, const */
-        enum aarch64_arith_opc opc = ARITH_SUBSI;
+        AArch64Insn insn = INSN_SUBSI;
 
         if (b < 0) {
             b = ~b;
-            opc = ARITH_ADDSI;
+            insn = INSN_ADDSI;
         }
-        tcg_out_aimm(s, opc, ext, TCG_REG_XZR, a, b);
+        tcg_out_aimm(s, insn, ext, TCG_REG_XZR, a, b);
     } else {
         /* Using CMP alias SUBS wzr, Wn, Wm */
-        tcg_out_arith(s, ARITH_SUBS, ext, TCG_REG_XZR, a, b, 0);
+        tcg_out_arith(s, INSN_SUBS, ext, TCG_REG_XZR, a, b, 0);
     }
 }
 
-static inline void tcg_out_cset(TCGContext *s, int ext, TCGReg rd, TCGCond c)
+static inline void tcg_out_cset(TCGContext *s, AArch64Ext ext,
+                                TCGReg rd, TCGCond c)
 {
     /* Using CSET alias of CSINC 0x1a800400 Xd, XZR, XZR, invert(cond) */
-    unsigned int base = ext ? 0x9a9f07e0 : 0x1a9f07e0;
-    tcg_out32(s, base | tcg_cond_to_aarch64[tcg_invert_cond(c)] << 12 | rd);
+    unsigned int base = INSN_CSINC | TCG_REG_XZR << 16 | TCG_REG_XZR << 5;
+    unsigned int cond = tcg_cond_to_aarch64[tcg_invert_cond(c)];
+    tcg_out32(s, base | ext | cond << 12 | rd);
 }
 
 static inline void tcg_out_goto(TCGContext *s, tcg_target_long target)
@@ -661,7 +676,7 @@ static inline void tcg_out_goto(TCGContext *s, tcg_target_long target)
         tcg_abort();
     }
 
-    tcg_out32(s, 0x14000000 | (offset & 0x03ffffff));
+    tcg_out32(s, INSN_B | (offset & 0x03ffffff));
 }
 
 static inline void tcg_out_goto_noaddr(TCGContext *s)
@@ -672,7 +687,7 @@ static inline void tcg_out_goto_noaddr(TCGContext *s)
        Mask away possible garbage in the high bits for the first translation,
        while keeping the offset bits for retranslation. */
     uint32_t insn;
-    insn = (tcg_in32(s) & 0x03ffffff) | 0x14000000;
+    insn = (tcg_in32(s) & 0x03ffffff) | INSN_B;
     tcg_out32(s, insn);
 }
 
@@ -681,7 +696,7 @@ static inline void tcg_out_goto_cond_noaddr(TCGContext *s, TCGCond c)
     /* see comments in tcg_out_goto_noaddr */
     uint32_t insn;
     insn = tcg_in32(s) & (0x07ffff << 5);
-    insn |= 0x54000000 | tcg_cond_to_aarch64[c];
+    insn |= INSN_B_C | tcg_cond_to_aarch64[c];
     tcg_out32(s, insn);
 }
 
@@ -697,17 +712,17 @@ static inline void tcg_out_goto_cond(TCGContext *s, TCGCond c,
     }
 
     offset &= 0x7ffff;
-    tcg_out32(s, 0x54000000 | tcg_cond_to_aarch64[c] | offset << 5);
+    tcg_out32(s, INSN_B_C | tcg_cond_to_aarch64[c] | offset << 5);
 }
 
 static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
 {
-    tcg_out32(s, 0xd63f0000 | reg << 5);
+    tcg_out32(s, INSN_BLR | reg << 5);
 }
 
 static inline void tcg_out_gotor(TCGContext *s, TCGReg reg)
 {
-    tcg_out32(s, 0xd61f0000 | reg << 5);
+    tcg_out32(s, INSN_BR | reg << 5);
 }
 
 static inline void tcg_out_call(TCGContext *s, tcg_target_long target)
@@ -720,7 +735,7 @@ static inline void tcg_out_call(TCGContext *s, tcg_target_long target)
         tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, target);
         tcg_out_callr(s, TCG_REG_TMP);
     } else {
-        tcg_out32(s, 0x94000000 | (offset & 0x03ffffff));
+        tcg_out32(s, INSN_BL | (offset & 0x03ffffff));
     }
 }
 
@@ -741,7 +756,7 @@ aarch64_limm(unsigned int m, unsigned int r)
    to test a 32bit reg against 0xff000000, pass M = 8,  R = 8.
    to test a 32bit reg against 0xff0000ff, pass M = 16, R = 8.
  */
-static inline void tcg_out_tst(TCGContext *s, int ext, TCGReg rn,
+static inline void tcg_out_tst(TCGContext *s, AArch64Ext ext, TCGReg rn,
                                unsigned int m, unsigned int r)
 {
     /* using TST alias of ANDS XZR, Xn,#bimm64 0x7200001f */
@@ -750,8 +765,8 @@ static inline void tcg_out_tst(TCGContext *s, int ext, TCGReg rn,
 }
 
 /* and a register with a bit pattern, similarly to TST, no flags change */
-static inline void tcg_out_andi(TCGContext *s, int ext, TCGReg rd, TCGReg rn,
-                                unsigned int m, unsigned int r)
+static inline void tcg_out_andi(TCGContext *s, AArch64Ext ext, TCGReg rd,
+                                TCGReg rn, unsigned int m, unsigned int r)
 {
     /* using AND 0x12000000 */
     unsigned int base = ext ? 0x92400000 : 0x12000000;
@@ -760,8 +775,7 @@ static inline void tcg_out_andi(TCGContext *s, int ext, TCGReg rd, TCGReg rn,
 
 static inline void tcg_out_ret(TCGContext *s)
 {
-    /* emit RET { LR } */
-    tcg_out32(s, 0xd65f03c0);
+    tcg_out32(s, INSN_RET | TCG_REG_LR << 5);
 }
 
 void aarch64_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
@@ -804,25 +818,23 @@ static inline void tcg_out_goto_label_cond(TCGContext *s,
     }
 }
 
-static inline void tcg_out_rev(TCGContext *s, int ext, TCGReg rd, TCGReg rm)
+static inline void tcg_out_rev(TCGContext *s, AArch64Ext ext,
+                               TCGReg rd, TCGReg rm)
 {
-    /* using REV 0x5ac00800 */
-    unsigned int base = ext ? 0xdac00c00 : 0x5ac00800;
-    tcg_out32(s, base | rm << 5 | rd);
+    AArch64Insn insn = ext ? INSN_REVx : INSN_REVw;
+    tcg_out32(s, insn | rm << 5 | rd);
 }
 
-static inline void tcg_out_rev16(TCGContext *s, int ext, TCGReg rd, TCGReg rm)
+static inline void tcg_out_rev16(TCGContext *s, AArch64Ext ext,
+                                 TCGReg rd, TCGReg rm)
 {
-    /* using REV16 0x5ac00400 */
-    unsigned int base = ext ? 0xdac00400 : 0x5ac00400;
-    tcg_out32(s, base | rm << 5 | rd);
+    tcg_out32(s, INSN_REV16 | ext | rm << 5 | rd);
 }
 
-static inline void tcg_out_sxt(TCGContext *s, int ext, int s_bits,
+static inline void tcg_out_sxt(TCGContext *s, AArch64Ext ext, int s_bits,
                                TCGReg rd, TCGReg rn)
 {
-    /* using ALIASes SXTB 0x13001c00, SXTH 0x13003c00, SXTW 0x93407c00
-       of SBFM Xd, Xn, #0, #7|15|31 */
+    /* using ALIASes SXTB, SXTH, SXTW of SBFM Xd, Xn, #0, #7|15|31 */
     int bits = 8 * (1 << s_bits) - 1;
     tcg_out_sbfm(s, ext, rd, rn, 0, bits);
 }
@@ -830,38 +842,37 @@ static inline void tcg_out_sxt(TCGContext *s, int ext, int s_bits,
 static inline void tcg_out_uxt(TCGContext *s, int s_bits,
                                TCGReg rd, TCGReg rn)
 {
-    /* using ALIASes UXTB 0x53001c00, UXTH 0x53003c00
-       of UBFM Wd, Wn, #0, #7|15 */
+    /* using ALIASes UXTB, UXTH, of UBFM Wd, Wn, #0, #7|15 */
     int bits = 8 * (1 << s_bits) - 1;
     tcg_out_ubfm(s, 0, rd, rn, 0, bits);
 }
 
-static void tcg_out_addi(TCGContext *s, int ext, TCGReg rd, TCGReg rn,
+static void tcg_out_addi(TCGContext *s, AArch64Ext ext, TCGReg rd, TCGReg rn,
                          tcg_target_long aimm)
 {
-    enum aarch64_arith_opc opc = ARITH_ADDI;
+    AArch64Insn insn = INSN_ADDI;
     tcg_target_long lo, hi;
 
     if (aimm < 0) {
         aimm = -aimm;
-        opc = ARITH_SUBI;
+        insn = INSN_SUBI;
     }
     hi = aimm & 0xfff000;
     lo = aimm & 0xfff;
     assert(aimm == hi + lo);
 
     if (hi != 0) {
-        tcg_out_aimm(s, opc, ext, rd, rn, hi);
+        tcg_out_aimm(s, insn, ext, rd, rn, hi);
         rn = rd;
     }
     if (lo != 0 || rd != rn) {
-        tcg_out_aimm(s, opc, ext, rd, rn, lo);
+        tcg_out_aimm(s, insn, ext, rd, rn, lo);
     }
 }
 
 static inline void tcg_out_nop(TCGContext *s)
 {
-    tcg_out32(s, 0xd503201f);
+    tcg_out32(s, INSN_NOP);
 }
 
 #ifdef CONFIG_SOFTMMU
@@ -887,17 +898,23 @@ static const void * const qemu_st_helpers[4] = {
 
 static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 {
+    int opc = lb->opc;
+    int s_bits = opc & 3;
+
     reloc_pc19(lb->label_ptr[0], (tcg_target_long)s->code_ptr);
-    tcg_out_movr(s, 1, TCG_REG_X0, TCG_AREG0);
-    tcg_out_movr(s, (TARGET_LONG_BITS == 64), TCG_REG_X1, lb->addrlo_reg);
+
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
+    tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_X1, lb->addrlo_reg);
     tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, lb->mem_index);
     tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP,
-                 (tcg_target_long)qemu_ld_helpers[lb->opc & 3]);
+                 (tcg_target_long)qemu_ld_helpers[s_bits]);
     tcg_out_callr(s, TCG_REG_TMP);
-    if (lb->opc & 0x04) {
-        tcg_out_sxt(s, 1, lb->opc & 3, lb->datalo_reg, TCG_REG_X0);
+
+    if (opc & 0x04) {
+        tcg_out_sxt(s, E64, s_bits, lb->datalo_reg, TCG_REG_X0);
     } else {
-        tcg_out_movr(s, 1, lb->datalo_reg, TCG_REG_X0);
+        tcg_out_mov(s, s_bits == 3 ? TCG_TYPE_I64 : TCG_TYPE_I32,
+                    lb->datalo_reg, TCG_REG_X0);
     }
 
     tcg_out_goto(s, (tcg_target_long)lb->raddr);
@@ -905,14 +922,17 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 
 static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 {
+    int s_bits = lb->opc;
+
     reloc_pc19(lb->label_ptr[0], (tcg_target_long)s->code_ptr);
 
-    tcg_out_movr(s, 1, TCG_REG_X0, TCG_AREG0);
-    tcg_out_movr(s, (TARGET_LONG_BITS == 64), TCG_REG_X1, lb->addrlo_reg);
-    tcg_out_movr(s, 1, TCG_REG_X2, lb->datalo_reg);
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
+    tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_X1, lb->addrlo_reg);
+    tcg_out_mov(s, s_bits == 3 ? TCG_TYPE_I64 : TCG_TYPE_I32,
+                TCG_REG_X2, lb->datalo_reg);
     tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X3, lb->mem_index);
     tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP,
-                 (tcg_target_long)qemu_st_helpers[lb->opc & 3]);
+                 (tcg_target_long)qemu_st_helpers[s_bits]);
     tcg_out_callr(s, TCG_REG_TMP);
 
     tcg_out_nop(s);
@@ -980,10 +1000,10 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg,
     /* Add any "high bits" from the tlb offset to the env address into X2,
        to take advantage of the LSL12 form of the addi instruction.
        X2 = env + (tlb_offset & 0xfff000) */
-    tcg_out_addi(s, 1, TCG_REG_X2, base, tlb_offset & 0xfff000);
+    tcg_out_addi(s, E64, TCG_REG_X2, base, tlb_offset & 0xfff000);
     /* Merge the tlb index contribution into X2.
        X2 = X2 + (X0 << CPU_TLB_ENTRY_BITS) */
-    tcg_out_arith(s, ARITH_ADD, 1, TCG_REG_X2, TCG_REG_X2,
+    tcg_out_arith(s, INSN_ADD, E64, TCG_REG_X2, TCG_REG_X2,
                   TCG_REG_X0, -CPU_TLB_ENTRY_BITS);
     /* Merge "low bits" from tlb offset, load the tlb comparator into X0.
        X0 = load [X2 + (tlb_offset & 0x000fff)] */
@@ -997,7 +1017,7 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg,
                  (is_read ? offsetof(CPUTLBEntry, addr_read)
                   : offsetof(CPUTLBEntry, addr_write)));
     /* Perform the address comparison. */
-    tcg_out_cmp(s, (TARGET_LONG_BITS == 64), TCG_REG_X0, TCG_REG_X3, 0);
+    tcg_out_cmp(s, EXT(TARGET_LONG_BITS == 64), TCG_REG_X0, TCG_REG_X3, 0);
     *label_ptr = s->code_ptr;
     /* If not equal, we jump to the slow path. */
     tcg_out_goto_cond_noaddr(s, TCG_COND_NE);
@@ -1024,8 +1044,8 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int opc, TCGReg data_r,
     case 1 | 4:
         if (TCG_LDST_BSWAP) {
             tcg_out_ldst_r(s, LDST_16, LDST_LD, data_r, addr_r, off_r);
-            tcg_out_rev16(s, 0, data_r, data_r);
-            tcg_out_sxt(s, 1, 1, data_r, data_r);
+            tcg_out_rev16(s, E32, data_r, data_r);
+            tcg_out_sxt(s, E64, 1, data_r, data_r);
         } else {
             tcg_out_ldst_r(s, LDST_16, LDST_LD_S_X, data_r, addr_r, off_r);
         }
@@ -1033,14 +1053,14 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int opc, TCGReg data_r,
     case 2:
         tcg_out_ldst_r(s, LDST_32, LDST_LD, data_r, addr_r, off_r);
         if (TCG_LDST_BSWAP) {
-            tcg_out_rev(s, 0, data_r, data_r);
+            tcg_out_rev(s, E32, data_r, data_r);
         }
         break;
     case 2 | 4:
         if (TCG_LDST_BSWAP) {
             tcg_out_ldst_r(s, LDST_32, LDST_LD, data_r, addr_r, off_r);
-            tcg_out_rev(s, 0, data_r, data_r);
-            tcg_out_sxt(s, 1, 2, data_r, data_r);
+            tcg_out_rev(s, E32, data_r, data_r);
+            tcg_out_sxt(s, E64, 2, data_r, data_r);
         } else {
             tcg_out_ldst_r(s, LDST_32, LDST_LD_S_X, data_r, addr_r, off_r);
         }
@@ -1048,7 +1068,7 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int opc, TCGReg data_r,
     case 3:
         tcg_out_ldst_r(s, LDST_64, LDST_LD, data_r, addr_r, off_r);
         if (TCG_LDST_BSWAP) {
-            tcg_out_rev(s, 1, data_r, data_r);
+            tcg_out_rev(s, E64, data_r, data_r);
         }
         break;
     default:
@@ -1065,7 +1085,7 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int opc, TCGReg data_r,
         break;
     case 1:
         if (TCG_LDST_BSWAP) {
-            tcg_out_rev16(s, 0, TCG_REG_TMP, data_r);
+            tcg_out_rev16(s, E32, TCG_REG_TMP, data_r);
             tcg_out_ldst_r(s, LDST_16, LDST_ST, TCG_REG_TMP, addr_r, off_r);
         } else {
             tcg_out_ldst_r(s, LDST_16, LDST_ST, data_r, addr_r, off_r);
@@ -1073,7 +1093,7 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int opc, TCGReg data_r,
         break;
     case 2:
         if (TCG_LDST_BSWAP) {
-            tcg_out_rev(s, 0, TCG_REG_TMP, data_r);
+            tcg_out_rev(s, E32, TCG_REG_TMP, data_r);
             tcg_out_ldst_r(s, LDST_32, LDST_ST, TCG_REG_TMP, addr_r, off_r);
         } else {
             tcg_out_ldst_r(s, LDST_32, LDST_ST, data_r, addr_r, off_r);
@@ -1081,7 +1101,7 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int opc, TCGReg data_r,
         break;
     case 3:
         if (TCG_LDST_BSWAP) {
-            tcg_out_rev(s, 1, TCG_REG_TMP, data_r);
+            tcg_out_rev(s, E64, TCG_REG_TMP, data_r);
             tcg_out_ldst_r(s, LDST_64, LDST_ST, TCG_REG_TMP, addr_r, off_r);
         } else {
             tcg_out_ldst_r(s, LDST_64, LDST_ST, data_r, addr_r, off_r);
@@ -1196,7 +1216,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 {
     /* ext will be set in the switch below, which will fall through to the
        common code. It triggers the use of extended regs where appropriate. */
-    int ext = 0;
+    AArch64Ext ext = E32;
 
     switch (opc) {
     case INDEX_op_exit_tb:
@@ -1252,9 +1272,10 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_mov_i64:
-        ext = 1; /* fall through */
+        tcg_out_mov(s, TCG_TYPE_I64, args[0], args[1]);
+        break;
     case INDEX_op_mov_i32:
-        tcg_out_movr(s, ext, args[0], args[1]);
+        tcg_out_mov(s, TCG_TYPE_I32, args[0], args[1]);
         break;
 
     case INDEX_op_movi_i64:
@@ -1265,123 +1286,122 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_add_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_add_i32:
         if (const_args[2]) {
             tcg_out_addi(s, ext, args[0], args[1], args[2]);
         } else {
-            tcg_out_arith(s, ARITH_ADD, ext, args[0], args[1], args[2], 0);
+            tcg_out_arith(s, INSN_ADD, ext, args[0], args[1], args[2], 0);
         }
         break;
 
     case INDEX_op_sub_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_sub_i32:
         if (const_args[2]) {
             tcg_out_addi(s, ext, args[0], args[1], -args[2]);
         } else {
-            tcg_out_arith(s, ARITH_SUB, ext, args[0], args[1], args[2], 0);
+            tcg_out_arith(s, INSN_SUB, ext, args[0], args[1], args[2], 0);
         }
         break;
 
     case INDEX_op_and_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_and_i32:
         if (const_args[2]) {
-            tcg_out_limm(s, ARITH_ANDI, ext, args[0], args[1], args[2]);
+            tcg_out_limm(s, INSN_ANDI, ext, args[0], args[1], args[2]);
         } else {
-            tcg_out_arith(s, ARITH_AND, ext, args[0], args[1], args[2], 0);
+            tcg_out_arith(s, INSN_AND, ext, args[0], args[1], args[2], 0);
         }
         break;
 
     case INDEX_op_or_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_or_i32:
         if (const_args[2]) {
-            tcg_out_limm(s, ARITH_ORI, ext, args[0], args[1], args[2]);
+            tcg_out_limm(s, INSN_ORRI, ext, args[0], args[1], args[2]);
         } else {
-            tcg_out_arith(s, ARITH_OR, ext, args[0], args[1], args[2], 0);
+            tcg_out_arith(s, INSN_ORR, ext, args[0], args[1], args[2], 0);
         }
         break;
 
     case INDEX_op_xor_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_xor_i32:
         if (const_args[2]) {
-            tcg_out_limm(s, ARITH_XORI, ext, args[0], args[1], args[2]);
+            tcg_out_limm(s, INSN_EORI, ext, args[0], args[1], args[2]);
         } else {
-            tcg_out_arith(s, ARITH_XOR, ext, args[0], args[1], args[2], 0);
+            tcg_out_arith(s, INSN_EOR, ext, args[0], args[1], args[2], 0);
         }
         break;
 
     case INDEX_op_mul_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_mul_i32:
-        tcg_out_mul(s, ext, args[0], args[1], args[2]);
+        tcg_out_data2(s, INSN_MUL, ext, args[0], args[1], args[2]);
         break;
 
     case INDEX_op_shl_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_shl_i32:
         if (const_args[2]) {    /* LSL / UBFM Wd, Wn, (32 - m) */
-            tcg_out_shl(s, ext, args[0], args[1], args[2]);
+            tcg_out_shli(s, ext, args[0], args[1], args[2]);
         } else {                /* LSL / LSLV */
-            tcg_out_shiftrot_reg(s, SRR_SHL, ext, args[0], args[1], args[2]);
+            tcg_out_data2(s, INSN_LSLV, ext, args[0], args[1], args[2]);
         }
         break;
 
     case INDEX_op_shr_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_shr_i32:
         if (const_args[2]) {    /* LSR / UBFM Wd, Wn, m, 31 */
-            tcg_out_shr(s, ext, args[0], args[1], args[2]);
+            tcg_out_shri(s, ext, args[0], args[1], args[2]);
         } else {                /* LSR / LSRV */
-            tcg_out_shiftrot_reg(s, SRR_SHR, ext, args[0], args[1], args[2]);
+            tcg_out_data2(s, INSN_LSRV, ext, args[0], args[1], args[2]);
         }
         break;
 
     case INDEX_op_sar_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_sar_i32:
         if (const_args[2]) {    /* ASR / SBFM Wd, Wn, m, 31 */
-            tcg_out_sar(s, ext, args[0], args[1], args[2]);
+            tcg_out_sari(s, ext, args[0], args[1], args[2]);
         } else {                /* ASR / ASRV */
-            tcg_out_shiftrot_reg(s, SRR_SAR, ext, args[0], args[1], args[2]);
+            tcg_out_data2(s, INSN_ASRV, ext, args[0], args[1], args[2]);
         }
         break;
 
     case INDEX_op_rotr_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_rotr_i32:
         if (const_args[2]) {    /* ROR / EXTR Wd, Wm, Wm, m */
-            tcg_out_rotr(s, ext, args[0], args[1], args[2]);
+            tcg_out_rotri(s, ext, args[0], args[1], args[2]);
         } else {                /* ROR / RORV */
-            tcg_out_shiftrot_reg(s, SRR_ROR, ext, args[0], args[1], args[2]);
+            tcg_out_data2(s, INSN_RORV, ext, args[0], args[1], args[2]);
         }
         break;
 
     case INDEX_op_rotl_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_rotl_i32:     /* same as rotate right by (32 - m) */
         if (const_args[2]) {    /* ROR / EXTR Wd, Wm, Wm, 32 - m */
-            tcg_out_rotl(s, ext, args[0], args[1], args[2]);
+            tcg_out_rotli(s, ext, args[0], args[1], args[2]);
         } else {
-            tcg_out_arith(s, ARITH_SUB, 0,
+            tcg_out_arith(s, INSN_SUB, E32,
                           TCG_REG_TMP, TCG_REG_XZR, args[2], 0);
-            tcg_out_shiftrot_reg(s, SRR_ROR, ext,
-                                 args[0], args[1], TCG_REG_TMP);
+            tcg_out_data2(s, INSN_RORV, ext, args[0], args[1], TCG_REG_TMP);
         }
         break;
 
     case INDEX_op_brcond_i64:
-        ext = 1; /* fall through */
-    case INDEX_op_brcond_i32: /* CMP 0, 1, cond(2), label 3 */
+        ext = E64; /* fall through */
+    case INDEX_op_brcond_i32:
         tcg_out_cmp(s, ext, args[0], args[1], const_args[1]);
         tcg_out_goto_label_cond(s, args[2], args[3]);
         break;
 
     case INDEX_op_setcond_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_setcond_i32:
         tcg_out_cmp(s, ext, args[1], args[2], const_args[2]);
         tcg_out_cset(s, 0, args[0], args[3]);
@@ -1425,7 +1445,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_bswap64_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_bswap32_i64:
     case INDEX_op_bswap32_i32:
         tcg_out_rev(s, ext, args[0], args[1]);
@@ -1436,17 +1456,17 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_ext8s_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_ext8s_i32:
         tcg_out_sxt(s, ext, 0, args[0], args[1]);
         break;
     case INDEX_op_ext16s_i64:
-        ext = 1; /* fall through */
+        ext = E64; /* fall through */
     case INDEX_op_ext16s_i32:
         tcg_out_sxt(s, ext, 1, args[0], args[1]);
         break;
     case INDEX_op_ext32s_i64:
-        tcg_out_sxt(s, 1, 2, args[0], args[1]);
+        tcg_out_sxt(s, E64, 2, args[0], args[1]);
         break;
     case INDEX_op_ext8u_i64:
     case INDEX_op_ext8u_i32:
@@ -1457,7 +1477,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_uxt(s, 1, args[0], args[1]);
         break;
     case INDEX_op_ext32u_i64:
-        tcg_out_movr(s, 0, args[0], args[1]);
+        tcg_out_movr(s, E32, args[0], args[1]);
         break;
 
     default:
@@ -1616,7 +1636,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
                       TCG_REG_FP, TCG_REG_LR, frame_size_callee_saved);
 
     /* FP -> callee_saved */
-    tcg_out_movr_sp(s, 1, TCG_REG_FP, TCG_REG_SP);
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_FP, TCG_REG_SP);
 
     /* store callee-preserved regs x19..x28 using FP -> callee_saved */
     for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
@@ -1625,7 +1645,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     }
 
     /* make stack space for TCG locals */
-    tcg_out_addi(s, 1, TCG_REG_SP, TCG_REG_SP,
+    tcg_out_addi(s, E64, TCG_REG_SP, TCG_REG_SP,
                  -frame_size_tcg_locals * TCG_TARGET_STACK_ALIGN);
     /* inform TCG about how to find TCG locals with register, offset, size */
     tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE,
@@ -1644,7 +1664,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     tb_ret_addr = s->code_ptr;
 
     /* remove TCG locals stack space */
-    tcg_out_addi(s, 1, TCG_REG_SP, TCG_REG_SP,
+    tcg_out_addi(s, E64, TCG_REG_SP, TCG_REG_SP,
                  frame_size_tcg_locals * TCG_TARGET_STACK_ALIGN);
 
     /* restore registers x19..x28.
Signature · 1 line
1.8.3.1
#5 lore raw

RFC 5/14 tcg-aarch64: Support andc, orc, eqv, not

In reply to Richard Henderson
---
 tcg/aarch64/tcg-target.c | 44 ++++++++++++++++++++++++++++++++++++++++++++
 tcg/aarch64/tcg-target.h | 16 ++++++++--------
 2 files changed, 52 insertions(+), 8 deletions(-)
tcg/aarch64/tcg-target.c +44 -0
index 3640486..7aeb3cd 100644
@@ -1314,6 +1314,15 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
             tcg_out_arith(s, INSN_AND, ext, args[0], args[1], args[2], 0);
         }
         break;
+    case INDEX_op_andc_i64:
+        ext = E64; /* fall through */
+    case INDEX_op_andc_i32:
+        if (const_args[2]) {
+            tcg_out_limm(s, INSN_ANDI, ext, args[0], args[1], ~args[2]);
+        } else {
+            tcg_out_arith(s, INSN_BIC, ext, args[0], args[1], args[2], 0);
+        }
+        break;
 
     case INDEX_op_or_i64:
         ext = E64; /* fall through */
@@ -1325,6 +1334,16 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
+    case INDEX_op_orc_i64:
+        ext = E64; /* fall through */
+    case INDEX_op_orc_i32:
+        if (const_args[2]) {
+            tcg_out_limm(s, INSN_ORRI, ext, args[0], args[1], ~args[2]);
+        } else {
+            tcg_out_arith(s, INSN_ORN, ext, args[0], args[1], args[2], 0);
+        }
+        break;
+
     case INDEX_op_xor_i64:
         ext = E64; /* fall through */
     case INDEX_op_xor_i32:
@@ -1335,6 +1354,22 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         }
         break;
 
+    case INDEX_op_eqv_i64:
+        ext = E64; /* fall through */
+    case INDEX_op_eqv_i32:
+        if (const_args[2]) {
+            tcg_out_limm(s, INSN_EORI, ext, args[0], args[1], ~args[2]);
+        } else {
+            tcg_out_arith(s, INSN_EON, ext, args[0], args[1], args[2], 0);
+        }
+        break;
+
+    case INDEX_op_not_i64:
+        ext = E64; /* fall through */
+    case INDEX_op_not_i32:
+        tcg_out_arith(s, INSN_ORN, ext, args[0], TCG_REG_XZR, args[1], 0);
+        break;
+
     case INDEX_op_mul_i64:
         ext = E64; /* fall through */
     case INDEX_op_mul_i32:
@@ -1530,6 +1565,15 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_or_i64, { "r", "r", "rL" } },
     { INDEX_op_xor_i32, { "r", "r", "rK" } },
     { INDEX_op_xor_i64, { "r", "r", "rL" } },
+    { INDEX_op_andc_i32, { "r", "r", "rK" } },
+    { INDEX_op_andc_i64, { "r", "r", "rL" } },
+    { INDEX_op_orc_i32, { "r", "r", "rK" } },
+    { INDEX_op_orc_i64, { "r", "r", "rL" } },
+    { INDEX_op_eqv_i32, { "r", "r", "rK" } },
+    { INDEX_op_eqv_i64, { "r", "r", "rL" } },
+
+    { INDEX_op_not_i32, { "r", "r" } },
+    { INDEX_op_not_i64, { "r", "r" } },
 
     { INDEX_op_shl_i32, { "r", "r", "ri" } },
     { INDEX_op_shr_i32, { "r", "r", "ri" } },
tcg/aarch64/tcg-target.h +8 -8
index 51e5092..ffe9429 100644
@@ -47,12 +47,12 @@ typedef enum {
 #define TCG_TARGET_HAS_ext16u_i32       1
 #define TCG_TARGET_HAS_bswap16_i32      1
 #define TCG_TARGET_HAS_bswap32_i32      1
-#define TCG_TARGET_HAS_not_i32          0
+#define TCG_TARGET_HAS_not_i32          1
 #define TCG_TARGET_HAS_neg_i32          0
 #define TCG_TARGET_HAS_rot_i32          1
-#define TCG_TARGET_HAS_andc_i32         0
-#define TCG_TARGET_HAS_orc_i32          0
-#define TCG_TARGET_HAS_eqv_i32          0
+#define TCG_TARGET_HAS_andc_i32         1
+#define TCG_TARGET_HAS_orc_i32          1
+#define TCG_TARGET_HAS_eqv_i32          1
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
@@ -73,12 +73,12 @@ typedef enum {
 #define TCG_TARGET_HAS_bswap16_i64      1
 #define TCG_TARGET_HAS_bswap32_i64      1
 #define TCG_TARGET_HAS_bswap64_i64      1
-#define TCG_TARGET_HAS_not_i64          0
+#define TCG_TARGET_HAS_not_i64          1
 #define TCG_TARGET_HAS_neg_i64          0
 #define TCG_TARGET_HAS_rot_i64          1
-#define TCG_TARGET_HAS_andc_i64         0
-#define TCG_TARGET_HAS_orc_i64          0
-#define TCG_TARGET_HAS_eqv_i64          0
+#define TCG_TARGET_HAS_andc_i64         1
+#define TCG_TARGET_HAS_orc_i64          1
+#define TCG_TARGET_HAS_eqv_i64          1
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      0
Signature · 1 line
1.8.3.1
#6 lore raw

RFC 6/14 tcg-aarch64: Handle zero as first argument to sub

In reply to Richard Henderson

In order to properly handle neg, as generated by TCG generic code.

---
 tcg/aarch64/tcg-target.c | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)
tcg/aarch64/tcg-target.c +17 -4
index 7aeb3cd..88bbfd2 100644
@@ -110,6 +110,7 @@ static inline void patch_reloc(uint8_t *code_ptr, int type,
 #define TCG_CT_CONST_LI32 0x200
 #define TCG_CT_CONST_LI64 0x400
 #define TCG_CT_CONST_CMP  0x800
+#define TCG_CT_CONST_ZERO 0x1000
 
 #include "bitmask-table.h"
 
@@ -179,6 +180,9 @@ static int target_parse_constraint(TCGArgConstraint *ct,
     case 'L': /* logical immediate 64-bit */
         ct->ct |= TCG_CT_CONST_LI64;
         break;
+    case 'Z': /* zero */
+        ct->ct |= TCG_CT_CONST_ZERO;
+        break;
     default:
         return -1;
     }
@@ -208,6 +212,9 @@ static inline int tcg_target_const_match(tcg_target_long val,
     if ((ct & TCG_CT_CONST_CMP) && can_cmpi(val)) {
         return 1;
     }
+    if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
+        return 1;
+    }
 
     return 0;
 }
@@ -1218,6 +1225,10 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
        common code. It triggers the use of extended regs where appropriate. */
     AArch64Ext ext = E32;
 
+    /* Some operands are defined with "rZ" constraint, a register or
+       the zero register.  These need not actually test args[I] == 0.  */
+#define REG0(I)  (const_args[I] ? TCG_REG_XZR : args[I])
+
     switch (opc) {
     case INDEX_op_exit_tb:
         tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, args[0]);
@@ -1299,9 +1310,9 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         ext = E64; /* fall through */
     case INDEX_op_sub_i32:
         if (const_args[2]) {
-            tcg_out_addi(s, ext, args[0], args[1], -args[2]);
+            tcg_out_addi(s, ext, args[0], REG0(1), -args[2]);
         } else {
-            tcg_out_arith(s, INSN_SUB, ext, args[0], args[1], args[2], 0);
+            tcg_out_arith(s, INSN_SUB, ext, args[0], REG0(1), args[2], 0);
         }
         break;
 
@@ -1518,6 +1529,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     default:
         tcg_abort(); /* opcode not implemented */
     }
+
+#undef REG0
 }
 
 static const TCGTargetOpDef aarch64_op_defs[] = {
@@ -1555,8 +1568,8 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
 
     { INDEX_op_add_i32, { "r", "r", "rA" } },
     { INDEX_op_add_i64, { "r", "r", "rA" } },
-    { INDEX_op_sub_i32, { "r", "r", "rA" } },
-    { INDEX_op_sub_i64, { "r", "r", "rA" } },
+    { INDEX_op_sub_i32, { "r", "rZ", "rA" } },
+    { INDEX_op_sub_i64, { "r", "rZ", "rA" } },
     { INDEX_op_mul_i32, { "r", "r", "r" } },
     { INDEX_op_mul_i64, { "r", "r", "r" } },
     { INDEX_op_and_i32, { "r", "r", "rK" } },
Signature · 1 line
1.8.3.1
#7 lore raw

RFC 7/14 tcg-aarch64: Support movcond

In reply to Richard Henderson
---
 tcg/aarch64/tcg-target.c | 19 ++++++++++++++++++-
 tcg/aarch64/tcg-target.h |  4 ++--
 2 files changed, 20 insertions(+), 3 deletions(-)
tcg/aarch64/tcg-target.c +18 -1
index 88bbfd2..f0febc9 100644
@@ -267,6 +267,7 @@ typedef enum {
     INSN_EXTR  = 0x13800000,
 
     /* Conditional select instructions */
+    INSN_CSEL  = 0x1a800000,
     INSN_CSINC = 0x1a800400,
 
     /* Branch instructions */
@@ -673,6 +674,13 @@ static inline void tcg_out_cset(TCGContext *s, AArch64Ext ext,
     tcg_out32(s, base | ext | cond << 12 | rd);
 }
 
+static inline void tcg_out_csel(TCGContext *s, AArch64Ext ext, TCGReg rd,
+                                TCGReg rn, TCGReg rm, TCGCond c)
+{
+    unsigned int cond = tcg_cond_to_aarch64[c];
+    tcg_out32(s, INSN_CSEL | ext | rm << 16 | cond << 12 | rn << 5 | rd);
+}
+
 static inline void tcg_out_goto(TCGContext *s, tcg_target_long target)
 {
     tcg_target_long offset;
@@ -1453,6 +1461,13 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_cset(s, 0, args[0], args[3]);
         break;
 
+    case INDEX_op_movcond_i64:
+        ext = E64; /* fall through */
+    case INDEX_op_movcond_i32:
+        tcg_out_cmp(s, ext, args[1], args[2], const_args[2]);
+        tcg_out_csel(s, ext, args[0], REG0(3), REG0(4), args[5]);
+        break;
+
     case INDEX_op_qemu_ld8u:
         tcg_out_qemu_ld(s, args, 0 | 0);
         break;
@@ -1600,9 +1615,11 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_rotr_i64, { "r", "r", "ri" } },
 
     { INDEX_op_brcond_i32, { "r", "rC" } },
-    { INDEX_op_setcond_i32, { "r", "r", "rC" } },
     { INDEX_op_brcond_i64, { "r", "rC" } },
+    { INDEX_op_setcond_i32, { "r", "r", "rC" } },
     { INDEX_op_setcond_i64, { "r", "r", "rC" } },
+    { INDEX_op_movcond_i32, { "r", "r", "rC", "rZ", "rZ" } },
+    { INDEX_op_movcond_i64, { "r", "r", "rC", "rZ", "rZ" } },
 
     { INDEX_op_qemu_ld8u, { "r", "l" } },
     { INDEX_op_qemu_ld8s, { "r", "l" } },
tcg/aarch64/tcg-target.h +2 -2
index ffe9429..7d87870 100644
@@ -56,7 +56,7 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
-#define TCG_TARGET_HAS_movcond_i32      0
+#define TCG_TARGET_HAS_movcond_i32      1
 #define TCG_TARGET_HAS_add2_i32         0
 #define TCG_TARGET_HAS_sub2_i32         0
 #define TCG_TARGET_HAS_mulu2_i32        0
@@ -82,7 +82,7 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      0
-#define TCG_TARGET_HAS_movcond_i64      0
+#define TCG_TARGET_HAS_movcond_i64      1
 #define TCG_TARGET_HAS_add2_i64         0
 #define TCG_TARGET_HAS_sub2_i64         0
 #define TCG_TARGET_HAS_mulu2_i64        0
Signature · 1 line
1.8.3.1
#8 lore raw

RFC 8/14 tcg-aarch64: Support deposit

In reply to Richard Henderson
---
 tcg/aarch64/tcg-target.c | 25 +++++++++++++++++++++++++
 tcg/aarch64/tcg-target.h |  4 ++--
 2 files changed, 27 insertions(+), 2 deletions(-)
tcg/aarch64/tcg-target.c +25 -0
index f0febc9..3474ca4 100644
@@ -262,6 +262,7 @@ typedef enum {
     INSN_MUL   = 0x1b007c00,  /* MADD alias with Ra = xzr */
 
     /* Bitfield instructions */
+    INSN_BFM   = 0x33000000,
     INSN_SBFM  = 0x13000000,
     INSN_UBFM  = 0x53000000,
     INSN_EXTR  = 0x13800000,
@@ -588,6 +589,12 @@ static inline void tcg_out_data2(TCGContext *s, AArch64Insn insn,
     tcg_out32(s, insn | ext | rm << 16 | rn << 5 | rd);
 }
 
+static inline void tcg_out_bfm(TCGContext *s, AArch64Ext ext, TCGReg rd,
+                               TCGReg rn, unsigned int a, unsigned int b)
+{
+    tcg_out32(s, INSN_BFM | ext | a << 16 | b << 10 | rn << 5 | rd);
+}
+
 static inline void tcg_out_ubfm(TCGContext *s, AArch64Ext ext, TCGReg rd,
                                 TCGReg rn, unsigned int a, unsigned int b)
 {
@@ -647,6 +654,15 @@ static inline void tcg_out_rotli(TCGContext *s, AArch64Ext ext, TCGReg rd,
     tcg_out_extr(s, ext, rd, rn, rn, bits - (m & max));
 }
 
+static inline void tcg_out_dep(TCGContext *s, AArch64Ext ext, TCGReg rd,
+                               TCGReg rn, unsigned lsb, unsigned width)
+{
+    unsigned size = ext ? 64 : 32;
+    unsigned a = (size - lsb) & (size - 1);
+    unsigned b = width - 1;
+    tcg_out_bfm(s, ext, rd, rn, a, b);
+}
+
 static void tcg_out_cmp(TCGContext *s, AArch64Ext ext, TCGReg a,
                         tcg_target_long b, bool const_b)
 {
@@ -1541,6 +1557,12 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_movr(s, E32, args[0], args[1]);
         break;
 
+    case INDEX_op_deposit_i64:
+        ext = E64; /* fall through */
+    case INDEX_op_deposit_i32:
+        tcg_out_dep(s, ext, args[0], REG0(2), args[3], args[4]);
+        break;
+    
     default:
         tcg_abort(); /* opcode not implemented */
     }
@@ -1654,6 +1676,9 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_ext16u_i64, { "r", "r" } },
     { INDEX_op_ext32u_i64, { "r", "r" } },
 
+    { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
+    { INDEX_op_deposit_i64, { "r", "0", "rZ" } },
+
     { -1 },
 };
 
tcg/aarch64/tcg-target.h +2 -2
index 7d87870..4082172 100644
@@ -55,7 +55,7 @@ typedef enum {
 #define TCG_TARGET_HAS_eqv_i32          1
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
-#define TCG_TARGET_HAS_deposit_i32      0
+#define TCG_TARGET_HAS_deposit_i32      1
 #define TCG_TARGET_HAS_movcond_i32      1
 #define TCG_TARGET_HAS_add2_i32         0
 #define TCG_TARGET_HAS_sub2_i32         0
@@ -81,7 +81,7 @@ typedef enum {
 #define TCG_TARGET_HAS_eqv_i64          1
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
-#define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_deposit_i64      1
 #define TCG_TARGET_HAS_movcond_i64      1
 #define TCG_TARGET_HAS_add2_i64         0
 #define TCG_TARGET_HAS_sub2_i64         0
Signature · 1 line
1.8.3.1
#9 lore raw

RFC 9/14 tcg-aarch64: Support add2, sub2

In reply to Richard Henderson
---
 tcg/aarch64/tcg-target.c | 110 ++++++++++++++++++++++++++++++++++++-----------
 tcg/aarch64/tcg-target.h |   8 ++--
 2 files changed, 90 insertions(+), 28 deletions(-)
tcg/aarch64/tcg-target.c +86 -24
index 3474ca4..967526b 100644
@@ -109,8 +109,10 @@ static inline void patch_reloc(uint8_t *code_ptr, int type,
 #define TCG_CT_CONST_S25  0x100
 #define TCG_CT_CONST_LI32 0x200
 #define TCG_CT_CONST_LI64 0x400
-#define TCG_CT_CONST_CMP  0x800
-#define TCG_CT_CONST_ZERO 0x1000
+#define TCG_CT_CONST_ZERO 0x800
+#define TCG_CT_CONST_AI   0x1000
+#define TCG_CT_CONST_AIC  0x2000
+#define TCG_CT_CONST_AIN  0x4000
 
 #include "bitmask-table.h"
 
@@ -137,14 +139,11 @@ static int find_bitmask32(uint32_t val)
     return find_bitmask64(((uint64_t)val << 32) | val);
 }
 
-static int can_cmpi(tcg_target_long val)
+static int is_aimm(tcg_target_long val)
 {
-    if (val < 0) {
-        val = ~val;
-    }
     return (val & ~0xfff) == 0 || (val & ~0xfff000) == 0;
 }
-
+    
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct,
                                    const char **pct_str)
@@ -168,12 +167,9 @@ static int target_parse_constraint(TCGArgConstraint *ct,
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_X3);
 #endif
         break;
-    case 'A': /* 25-bit signed, to be added or subtracted.  */
+    case 'I': /* 25-bit signed, to be added or subtracted.  */
         ct->ct |= TCG_CT_CONST_S25;
         break;
-    case 'C': /* a 12-bit constant (maybe inverted) to be used with compare. */
-        ct->ct |= TCG_CT_CONST_CMP;
-        break;
     case 'K': /* logical immediate 32-bit */
         ct->ct |= TCG_CT_CONST_LI32;
         break;
@@ -183,6 +179,15 @@ static int target_parse_constraint(TCGArgConstraint *ct,
     case 'Z': /* zero */
         ct->ct |= TCG_CT_CONST_ZERO;
         break;
+    case 'A': /* an arithmetic immediate.  */
+        ct->ct |= TCG_CT_CONST_AI;
+        break;
+    case 'C': /* a complimented arithmetic immediate.  */
+        ct->ct |= TCG_CT_CONST_AIC;
+        break;
+    case 'N': /* a negated arithmetic immediate.  */
+        ct->ct |= TCG_CT_CONST_AIN;
+        break;
     default:
         return -1;
     }
@@ -209,10 +214,16 @@ static inline int tcg_target_const_match(tcg_target_long val,
     if ((ct & TCG_CT_CONST_LI32) && find_bitmask32(val) >= 0) {
         return 1;
     }
-    if ((ct & TCG_CT_CONST_CMP) && can_cmpi(val)) {
+    if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
         return 1;
     }
-    if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
+    if ((ct & TCG_CT_CONST_AI) && is_aimm(val)) {
+        return 1;
+    }
+    if ((ct & TCG_CT_CONST_AIC) && is_aimm(~val)) {
+        return 1;
+    }
+    if ((ct & TCG_CT_CONST_AIN) && is_aimm(-val)) {
         return 1;
     }
 
@@ -246,9 +257,14 @@ typedef enum {
 
     /* Add/subtract shifted register instructions */
     INSN_ADD   = 0x0b000000,
+    INSN_ADDS  = 0x2b000000,
     INSN_SUB   = 0x4b000000,
     INSN_SUBS  = 0x6b000000,
 
+    /* Add/subtract with carry instructions */
+    INSN_ADC   = 0x1a000000,
+    INSN_SBC   = 0x5a000000,
+
     /* Data-processing (1 source) instructions */
     INSN_REV16 = 0x5ac00400,
     INSN_REVx  = 0xdac00c00,
@@ -1242,6 +1258,33 @@ static inline void tcg_out_load_pair(TCGContext *s, TCGReg addr,
     tcg_out32(s, 0xa9400000 | idx << 16 | r2 << 10 | addr << 5 | r1);
 }
 
+static inline void tcg_out_addsub2(TCGContext *s, AArch64Ext ext, TCGReg rl,
+                                   TCGReg rh, TCGReg al, TCGReg ah,
+                                   tcg_target_long bl, TCGReg bh,
+                                   bool const_bl, bool sub)
+{
+    TCGReg orig_rl = rl;
+
+    if (rl == ah || rl == bh) {
+        rl = TCG_REG_TMP;
+    }
+    if (const_bl) {
+        if ((bl < 0) ^ sub) {
+            tcg_out_aimm(s, INSN_SUBSI, ext, rl, al, -bl);
+        } else {
+            tcg_out_aimm(s, INSN_ADDSI, ext, rl, al, bl);
+        }
+    } else {
+        tcg_out_arith(s, sub ? INSN_SUBS : INSN_ADDS, ext, rl, al, bl, 0);
+    }
+
+    tcg_out_arith(s, sub ? INSN_ADC : INSN_SBC, ext, rh, ah, bh, 0);
+
+    if (rl != orig_rl) {
+        tcg_out_movr(s, ext, orig_rl, rl);
+    }
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg *args, const int *const_args)
 {
@@ -1562,7 +1605,21 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_deposit_i32:
         tcg_out_dep(s, ext, args[0], REG0(2), args[3], args[4]);
         break;
-    
+
+    case INDEX_op_add2_i64:
+        ext = E64; /* fall through */
+    case INDEX_op_add2_i32:
+        tcg_out_addsub2(s, ext, args[0], args[1], REG0(2), REG0(3), args[4],
+                        REG0(5), const_args[4], false);
+        break;
+
+    case INDEX_op_sub2_i64:
+        ext = E64; /* fall through */
+    case INDEX_op_sub2_i32:
+        tcg_out_addsub2(s, ext, args[0], args[1], REG0(2), REG0(3), args[4],
+                        REG0(5), const_args[4], true);
+        break;
+
     default:
         tcg_abort(); /* opcode not implemented */
     }
@@ -1603,10 +1660,10 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_st32_i64, { "r", "r" } },
     { INDEX_op_st_i64, { "r", "r" } },
 
-    { INDEX_op_add_i32, { "r", "r", "rA" } },
-    { INDEX_op_add_i64, { "r", "r", "rA" } },
-    { INDEX_op_sub_i32, { "r", "rZ", "rA" } },
-    { INDEX_op_sub_i64, { "r", "rZ", "rA" } },
+    { INDEX_op_add_i32, { "r", "r", "rI" } },
+    { INDEX_op_add_i64, { "r", "r", "rI" } },
+    { INDEX_op_sub_i32, { "r", "rZ", "rI" } },
+    { INDEX_op_sub_i64, { "r", "rZ", "rI" } },
     { INDEX_op_mul_i32, { "r", "r", "r" } },
     { INDEX_op_mul_i64, { "r", "r", "r" } },
     { INDEX_op_and_i32, { "r", "r", "rK" } },
@@ -1636,12 +1693,12 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_rotl_i64, { "r", "r", "ri" } },
     { INDEX_op_rotr_i64, { "r", "r", "ri" } },
 
-    { INDEX_op_brcond_i32, { "r", "rC" } },
-    { INDEX_op_brcond_i64, { "r", "rC" } },
-    { INDEX_op_setcond_i32, { "r", "r", "rC" } },
-    { INDEX_op_setcond_i64, { "r", "r", "rC" } },
-    { INDEX_op_movcond_i32, { "r", "r", "rC", "rZ", "rZ" } },
-    { INDEX_op_movcond_i64, { "r", "r", "rC", "rZ", "rZ" } },
+    { INDEX_op_brcond_i32, { "r", "rAC" } },
+    { INDEX_op_brcond_i64, { "r", "rAC" } },
+    { INDEX_op_setcond_i32, { "r", "r", "rAC" } },
+    { INDEX_op_setcond_i64, { "r", "r", "rAC" } },
+    { INDEX_op_movcond_i32, { "r", "r", "rAC", "rZ", "rZ" } },
+    { INDEX_op_movcond_i64, { "r", "r", "rAC", "rZ", "rZ" } },
 
     { INDEX_op_qemu_ld8u, { "r", "l" } },
     { INDEX_op_qemu_ld8s, { "r", "l" } },
@@ -1679,6 +1736,11 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
     { INDEX_op_deposit_i64, { "r", "0", "rZ" } },
 
+    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rAN", "rZ" } },
+    { INDEX_op_add2_i64, { "r", "r", "rZ", "rZ", "rAN", "rZ" } },
+    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rAN", "rZ" } },
+    { INDEX_op_sub2_i64, { "r", "r", "rZ", "rZ", "rAN", "rZ" } },
+
     { -1 },
 };
 
tcg/aarch64/tcg-target.h +4 -4
index 4082172..0c71048 100644
@@ -57,8 +57,8 @@ typedef enum {
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      1
 #define TCG_TARGET_HAS_movcond_i32      1
-#define TCG_TARGET_HAS_add2_i32         0
-#define TCG_TARGET_HAS_sub2_i32         0
+#define TCG_TARGET_HAS_add2_i32         1
+#define TCG_TARGET_HAS_sub2_i32         1
 #define TCG_TARGET_HAS_mulu2_i32        0
 #define TCG_TARGET_HAS_muls2_i32        0
 
@@ -83,8 +83,8 @@ typedef enum {
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      1
 #define TCG_TARGET_HAS_movcond_i64      1
-#define TCG_TARGET_HAS_add2_i64         0
-#define TCG_TARGET_HAS_sub2_i64         0
+#define TCG_TARGET_HAS_add2_i64         1
+#define TCG_TARGET_HAS_sub2_i64         1
 #define TCG_TARGET_HAS_mulu2_i64        0
 #define TCG_TARGET_HAS_muls2_i64        0
 
Signature · 1 line
1.8.3.1
#10 lore raw

RFC 10/14 tcg-aarch64: Support div, mulu2

In reply to Richard Henderson
---
 tcg/aarch64/tcg-target.c | 42 ++++++++++++++++++++++++++++++++++++++++++
 tcg/aarch64/tcg-target.h |  6 +++---
 2 files changed, 45 insertions(+), 3 deletions(-)
tcg/aarch64/tcg-target.c +42 -0
index 967526b..920c63c 100644
@@ -276,6 +276,9 @@ typedef enum {
     INSN_ASRV  = 0x1ac02800,
     INSN_RORV  = 0x1ac02c00,
     INSN_MUL   = 0x1b007c00,  /* MADD alias with Ra = xzr */
+    INSN_UMULH = 0x9bc07c00,
+    INSN_UDIV  = 0x1ac00800,
+    INSN_SDIV  = 0x1ac00c00,
 
     /* Bitfield instructions */
     INSN_BFM   = 0x33000000,
@@ -1285,6 +1288,26 @@ static inline void tcg_out_addsub2(TCGContext *s, AArch64Ext ext, TCGReg rl,
     }
 }
 
+static inline void tcg_out_mulu2(TCGContext *s, TCGReg rl, TCGReg rh,
+                                 TCGReg a, TCGReg b)
+{
+    if (rl == a || rl == b) {
+        if (rh == a || rh == b) {
+            /* Overlap on both outputs -- use a temporary.  */
+            tcg_out_data2(s, INSN_MUL, E64, TCG_REG_TMP, a, b);
+            tcg_out_data2(s, INSN_UMULH, E64, rh, a, b);
+            tcg_out_mov(s, TCG_TYPE_I64, rl, TCG_REG_TMP);
+        } else {
+            /* Overlap on low output -- do high part first.  */
+            tcg_out_data2(s, INSN_UMULH, E64, rh, a, b);
+            tcg_out_data2(s, INSN_MUL, E64, rl, a, b);
+        }
+    } else {
+        tcg_out_data2(s, INSN_MUL, E64, rl, a, b);
+        tcg_out_data2(s, INSN_UMULH, E64, rh, a, b);
+    }
+}
+
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                        const TCGArg *args, const int *const_args)
 {
@@ -1453,6 +1476,16 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_mul_i32:
         tcg_out_data2(s, INSN_MUL, ext, args[0], args[1], args[2]);
         break;
+    case INDEX_op_div_i64:
+        ext = E64; /* fall through */
+    case INDEX_op_div_i32:
+        tcg_out_data2(s, INSN_SDIV, ext, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_divu_i64:
+        ext = E64; /* fall through */
+    case INDEX_op_divu_i32:
+        tcg_out_data2(s, INSN_UDIV, ext, args[0], args[1], args[2]);
+        break;
 
     case INDEX_op_shl_i64:
         ext = E64; /* fall through */
@@ -1620,6 +1653,10 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
                         REG0(5), const_args[4], true);
         break;
 
+    case INDEX_op_mulu2_i64:
+        tcg_out_mulu2(s, args[0], args[1], args[2], args[3]);
+        break;
+
     default:
         tcg_abort(); /* opcode not implemented */
     }
@@ -1666,6 +1703,10 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_sub_i64, { "r", "rZ", "rI" } },
     { INDEX_op_mul_i32, { "r", "r", "r" } },
     { INDEX_op_mul_i64, { "r", "r", "r" } },
+    { INDEX_op_div_i32, { "r", "r", "r" } },
+    { INDEX_op_div_i64, { "r", "r", "r" } },
+    { INDEX_op_divu_i32, { "r", "r", "r" } },
+    { INDEX_op_divu_i64, { "r", "r", "r" } },
     { INDEX_op_and_i32, { "r", "r", "rK" } },
     { INDEX_op_and_i64, { "r", "r", "rL" } },
     { INDEX_op_or_i32, { "r", "r", "rK" } },
@@ -1740,6 +1781,7 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_add2_i64, { "r", "r", "rZ", "rZ", "rAN", "rZ" } },
     { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rAN", "rZ" } },
     { INDEX_op_sub2_i64, { "r", "r", "rZ", "rZ", "rAN", "rZ" } },
+    { INDEX_op_mulu2_i64, { "r", "r", "r", "r" } },
 
     { -1 },
 };
tcg/aarch64/tcg-target.h +3 -3
index 0c71048..507e192 100644
@@ -39,7 +39,7 @@ typedef enum {
 #define TCG_TARGET_CALL_STACK_OFFSET    0
 
 /* optional instructions */
-#define TCG_TARGET_HAS_div_i32          0
+#define TCG_TARGET_HAS_div_i32          1
 #define TCG_TARGET_HAS_rem_i32          0
 #define TCG_TARGET_HAS_ext8s_i32        1
 #define TCG_TARGET_HAS_ext16s_i32       1
@@ -62,7 +62,7 @@ typedef enum {
 #define TCG_TARGET_HAS_mulu2_i32        0
 #define TCG_TARGET_HAS_muls2_i32        0
 
-#define TCG_TARGET_HAS_div_i64          0
+#define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rem_i64          0
 #define TCG_TARGET_HAS_ext8s_i64        1
 #define TCG_TARGET_HAS_ext16s_i64       1
@@ -85,7 +85,7 @@ typedef enum {
 #define TCG_TARGET_HAS_movcond_i64      1
 #define TCG_TARGET_HAS_add2_i64         1
 #define TCG_TARGET_HAS_sub2_i64         1
-#define TCG_TARGET_HAS_mulu2_i64        0
+#define TCG_TARGET_HAS_mulu2_i64        1
 #define TCG_TARGET_HAS_muls2_i64        0
 
 enum {
Signature · 1 line
1.8.3.1
#11 lore raw

RFC 11/14 tcg-aarch64: Improve tcg_out_movi

In reply to Richard Henderson

Handle small positive and negative numbers early. Check for logical immediates. Check if using MOVN for the first set helps.

---
 tcg/aarch64/tcg-target.c | 85 +++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 66 insertions(+), 19 deletions(-)
tcg/aarch64/tcg-target.c +66 -19
index 920c63c..02ab278 100644
@@ -511,32 +511,79 @@ static inline void tcg_out_movr(TCGContext *s, AArch64Ext ext,
     tcg_out_aimm(s, INSN_ADDI, ext, dest, src, 0);
 }
 
+static inline void tcg_out_movwi(TCGContext *s, AArch64Insn insn,
+                                 AArch64Ext ext, TCGReg rd,
+                                 uint16_t value, int shift)
+{
+    tcg_out32(s, insn | ext | shift << 17 | value << 5 | rd);
+}
+
 static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
                          tcg_target_long value)
 {
-    AArch64Insn insn = INSN_MOVZ;
+    tcg_target_long valid = (type == TCG_TYPE_I32 ? 0xffffffffull : -1ull);
+    AArch64Insn insn;
+    AArch64Ext ext;
+    int i, wantinv, shift;
+
+    value &= valid;
+    
+    /* Check small positive values.  */
+    if ((value & ~0xffff) == 0) {
+        tcg_out_movwi(s, INSN_MOVZ, E32, rd, value, 0);
+        return;
+    }
+
+    /* Check small negative values.  */
+    if ((~value & valid & ~0xffff) == 0) {
+        tcg_out_movwi(s, INSN_MOVN, EXT(type == TCG_TYPE_I64), rd, ~value, 0);
+        return;
+    }
+        
+    /* Check for bitfield immediates.  */
+    if ((value & ~0xffffffffull) == 0) {
+        i = find_bitmask32(value);
+        ext = E32;
+    } else {
+        i = find_bitmask64(value);
+        ext = E64;
+    }
+    if (i >= 0) {
+        tcg_out32(s, INSN_ORRI | TCG_REG_XZR << 5 | ext
+                  | bitmask_enc[i] << 10 | rd);
+        return;
+    }
 
-    if (type == TCG_TYPE_I32) {
-        value = (uint32_t)value;
+    /* Would it take fewer insns to load the inverse?  */
+    wantinv = 0;
+    for (i = 0; i < 64; i += 16) {
+        if (((value >> i) & 0xffff) == 0) {
+            wantinv--;
+        }
+        if (((~value >> i) & 0xffff) == 0) {
+            wantinv++;
+        }
     }
 
-    /* Construct halfwords of the immediate with MOVZ/MOVK with LSL.
-       Count trailing zeros in 16 bit steps, mapping 64 to 0.  Emit the
-       first MOVZ with the half-word immediate skipping the zeros, with
-       a shift (LSL) equal to this number.  Then all other insns are MOVKs.
-       Zero the processed half-word in the value, continue until empty.
-       We build the final result 16bits at a time with up to 4 instructions,
-       but do not emit instructions for 16bit zero holes. */
-    do {
-        unsigned shift = ctz64(value) & (63 & -16);
-        unsigned half = (value >> shift) & 0xffff;
-        AArch64Ext ext = EXT(shift >= 32);
-
-        tcg_out32(s, insn | ext | shift << 17 | half << 5 | rd);
-
-        insn = INSN_MOVK;
+    if (wantinv > 0) {
+        value = ~value;
+        insn = INSN_MOVN;
+        valid = -1;
+    } else {
+        insn = INSN_MOVZ;
+        valid = 0;
+    }
+
+    /* Perform the first round specially, to handle the inverse.  */
+    shift = ctz64(value) & (63 & -16);
+    tcg_out_movwi(s, insn, ext, rd, value >> shift, shift);
+    value &= ~(0xffffUL << shift);
+
+    while (value) {
+        shift = ctz64(value) & (63 & -16);
+        tcg_out_movwi(s, INSN_MOVK, ext, rd, (value ^ valid) >> shift, shift);
         value &= ~(0xffffUL << shift);
-    } while (value);
+    }
 }
 
 static inline void tcg_out_ldst_r(TCGContext *s,
Signature · 1 line
1.8.3.1
#12 lore raw

RFC 12/14 tcg-aarch64: Avoid add with zero in tlb load

In reply to Richard Henderson

Some guest env are small enough to reach the tlb with only a 12-bit addition.

---
 tcg/aarch64/tcg-target.c | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)
tcg/aarch64/tcg-target.c +18 -7
index 02ab278..3ea5db7 100644
@@ -1076,47 +1076,58 @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, int opc,
    slow path for the failure case, which will be patched later when finalizing
    the slow path. Generated code returns the host addend in X1,
    clobbers X0,X2,X3,TMP. */
-static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg,
-            int s_bits, uint8_t **label_ptr, int mem_index, int is_read)
+static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, int s_bits,
+                             uint8_t **label_ptr, int mem_index, int is_read)
 {
     TCGReg base = TCG_AREG0;
     int tlb_offset = is_read ?
         offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
         : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
+
     /* Extract the TLB index from the address into X0.
        X0<CPU_TLB_BITS:0> =
        addr_reg<TARGET_PAGE_BITS+CPU_TLB_BITS:TARGET_PAGE_BITS> */
     tcg_out_ubfm(s, (TARGET_LONG_BITS == 64), TCG_REG_X0, addr_reg,
                  TARGET_PAGE_BITS, TARGET_PAGE_BITS + CPU_TLB_BITS);
+
     /* Store the page mask part of the address and the low s_bits into X3.
        Later this allows checking for equality and alignment at the same time.
        X3 = addr_reg & (PAGE_MASK | ((1 << s_bits) - 1)) */
     tcg_out_andi(s, (TARGET_LONG_BITS == 64), TCG_REG_X3, addr_reg,
                  (TARGET_LONG_BITS - TARGET_PAGE_BITS) + s_bits,
                  (TARGET_LONG_BITS - TARGET_PAGE_BITS));
+
     /* Add any "high bits" from the tlb offset to the env address into X2,
        to take advantage of the LSL12 form of the addi instruction.
-       X2 = env + (tlb_offset & 0xfff000) */
-    tcg_out_addi(s, E64, TCG_REG_X2, base, tlb_offset & 0xfff000);
+       base = env + (tlb_offset & 0xfff000) */
+    if (tlb_offset & 0xfff000) {
+        tcg_out_addi(s, E64, TCG_REG_X2, base, tlb_offset & 0xfff000);
+        base = TCG_REG_X2;
+    }
+
     /* Merge the tlb index contribution into X2.
-       X2 = X2 + (X0 << CPU_TLB_ENTRY_BITS) */
-    tcg_out_arith(s, INSN_ADD, E64, TCG_REG_X2, TCG_REG_X2,
+       X2 = base + (X0 << CPU_TLB_ENTRY_BITS) */
+    tcg_out_arith(s, INSN_ADD, E64, TCG_REG_X2, base,
                   TCG_REG_X0, -CPU_TLB_ENTRY_BITS);
+
     /* Merge "low bits" from tlb offset, load the tlb comparator into X0.
        X0 = load [X2 + (tlb_offset & 0x000fff)] */
     tcg_out_ldst(s, TARGET_LONG_BITS == 64 ? LDST_64 : LDST_32,
                  LDST_LD, TCG_REG_X0, TCG_REG_X2,
                  (tlb_offset & 0xfff));
+
     /* Load the tlb addend. Do that early to avoid stalling.
        X1 = load [X2 + (tlb_offset & 0xfff) + offsetof(addend)] */
     tcg_out_ldst(s, LDST_64, LDST_LD, TCG_REG_X1, TCG_REG_X2,
                  (tlb_offset & 0xfff) + (offsetof(CPUTLBEntry, addend)) -
                  (is_read ? offsetof(CPUTLBEntry, addr_read)
                   : offsetof(CPUTLBEntry, addr_write)));
+
     /* Perform the address comparison. */
     tcg_out_cmp(s, EXT(TARGET_LONG_BITS == 64), TCG_REG_X0, TCG_REG_X3, 0);
-    *label_ptr = s->code_ptr;
+
     /* If not equal, we jump to the slow path. */
+    *label_ptr = s->code_ptr;
     tcg_out_goto_cond_noaddr(s, TCG_COND_NE);
 }
 
Signature · 1 line
1.8.3.1
#13 lore raw

RFC 13/14 tcg-aarch64: Use adrp in tcg_out_movi

In reply to Richard Henderson

Loading an qemu pointer as an immediate happens often:

- exit_tb $0x7fa8140013
+ exit_tb $0x7f81ee0013
...

- :  d2800260        mov     x0, #0x13
- :  f2b50280        movk    x0, #0xa814, lsl #16
- :  f2c00fe0        movk    x0, #0x7f, lsl #32
+ :  90ff1000        adrp    x0, 0x7f81ee0000
+ :  91004c00        add     x0, x0, #0x13
---
 tcg/aarch64/tcg-target.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
tcg/aarch64/tcg-target.c +18 -0
index 3ea5db7..a03da58 100644
@@ -298,6 +298,10 @@ typedef enum {
     INSN_RET   = 0xd65f0000,
     INSN_B_C   = 0x54000000,
 
+    /* PC relative addressing instructions */
+    INSN_ADR   = 0x10000000,
+    INSN_ADRP  = 0x90000000,
+
     /* System instructions */
     INSN_NOP   = 0xd503201f,
 } AArch64Insn;
@@ -554,6 +558,20 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
         return;
     }
 
+    /* Look for host pointer values within 4G of the PC.  This happens
+       often when loading pointers to QEMU's data structures.  */
+    valid = (value >> 12) - ((intptr_t)s->code_ptr >> 12);
+    if (valid == sextract64(valid, 0, 21)) {
+        insn  = INSN_ADRP | rd;
+        insn |= (valid & 3) << 29;
+        insn |= (valid & 0x1ffffc) << (5 - 2);
+        tcg_out32(s, insn);
+        if (value & 0xfff) {
+            tcg_out_aimm(s, INSN_ADDI, ext, rd, rd, value & 0xfff);
+        }
+        return;
+    }
+
     /* Would it take fewer insns to load the inverse?  */
     wantinv = 0;
     for (i = 0; i < 64; i += 16) {
Signature · 1 line
1.8.3.1
#14 lore raw

RFC 14/14 tcg-aarch64: Pass return address to load/store helpers directly.

In reply to Richard Henderson
---
 include/exec/exec-all.h  | 16 +----------
 tcg/aarch64/tcg-target.c | 69 ++++++++++++++++++++++++++----------------------
 2 files changed, 39 insertions(+), 46 deletions(-)
include/exec/exec-all.h +1 -15
index b3402a1..10e680d 100644
@@ -325,7 +325,7 @@ extern uintptr_t tci_tb_ptr;
    (5) post-process (e.g. stack adjust)
    (6) jump to corresponding code of the next of fast path
  */
-# if defined(__i386__) || defined(__x86_64__)
+# if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__)
 #  define GETRA() ((uintptr_t)__builtin_return_address(0))
 /* The return address argument for ldst is passed directly.  */
 #  define GETPC_LDST()  (abort(), 0)
@@ -334,20 +334,6 @@ extern uintptr_t tci_tb_ptr;
 #  define GETPC_LDST() ((uintptr_t) ((*(int32_t *)(GETRA() - 4)) - 1))
 # elif defined(__arm__)
 #  define GETPC_EXT()  GETPC()
-# elif defined(__aarch64__)
-#  define GETRA()       ((uintptr_t)__builtin_return_address(0))
-#  define GETPC_LDST()  tcg_getpc_ldst(GETRA())
-static inline uintptr_t tcg_getpc_ldst(uintptr_t ra)
-{
-    int32_t b;
-    ra += 4;                    /* skip one instruction */
-    b = *(int32_t *)ra;         /* load the branch insn */
-    b = (b << 6) >> (6 - 2);    /* extract the displacement */
-    ra += b;                    /* apply the displacement  */
-    ra -= 4;                    /* return a pointer into the current opcode,
-                                   not the start of the next opcode  */
-    return ra;
-}
 # else
 #  error "CONFIG_QEMU_LDST_OPTIMIZATION needs GETPC_LDST() implementation!"
 # endif
tcg/aarch64/tcg-target.c +38 -31

115 lines not on this page · raw message

Signature · 1 line
1.8.3.1