Skip to contentSkip to conversation

Generate strided vector loads/stores with tcg nodes

PATCH In review Chao Liu 2 participants 5 messages Apr 17 – Apr 24 RISC-V

lore
Share & tools
lore

Chao Liu

author
#0 lore raw

v10 cover Generate strided vector loads/stores with tcg nodes

Hi Alistair,

In patch v10:

- Update the email address of the patch submitter.
- Rebase on the latest riscv-to-apply.next branch.

History of changes:

patch v9:
- Simplify the implementation of gen_check_vext_elem_mask():
remove the mask argument, compute the mask directly inside the function, and eliminate redundant code.
- Limit the bit width to 8 bits when loading the mask from memory.
- Remove the vreg argument in gen_ldst_vreg().
https://lore.kernel.org/qemu-devel/[email protected]/

patch v8:
- Use the right TCGv type for each variable — for example, make mask_elem type TCGv_i64.
- Use tcg_gen_trunc_i64_ptr() to change TCGv types — don't use C-style casting.
- Use TCG_COND_TSTNE, not TCG_COND_NE in tcg_gen_brcond_i64() to represent:
if (vext_elem_mask(v0, i) != 0)
https://lore.kernel.org/qemu-devel/[email protected]/

patch v7:
- Standardize the subject line of patch 1 and remove the trailing period.
- Split into sub-functions to improve the patch's code readability and facilitate review.
- Use more faster TCG ops, use tcg_gen_andi_tl() instead of tcg_gen_rem_tl().
- Add a tested-by signature for patch 2, as Eric has already tested it.
https://lore.kernel.org/qemu-devel/[email protected]/

patch v6:
- If a strided vector memory access instruction has non-zero vstart, check it through vlse/vsse helpers function.
- Adjust the tcg test Makefile.
https://lore.kernel.org/qemu-devel/[email protected]/

Patch v5:
- Removed the redundant call to mark_vs_dirty(s) within the gen_ldst_stride_main_loop() function.
https://lore.kernel.org/qemu-riscv/[email protected]/

Patch v4:
- Use ctz32() replace to for-loop
https://lore.kernel.org/qemu-devel/[email protected]/

Patch v3:
- Fix the get_log2() function:
https://lore.kernel.org/qemu-riscv/[email protected]/T/#t
- Add test for vlsseg8e32 instruction.
- Rebase on top of the latest master.

Patch v2:
- Split the TCG node emulation of the complex strided load/store operation into two separate functions to simplify the implementation:
https://lore.kernel.org/qemu-riscv/[email protected]/

Patch v1:
- Paolo submitted the initial version of the patch.
https://lore.kernel.org/qemu-devel/[email protected]/

Thanks,
Chao

Chao Liu (2):
target/riscv: Use tcg nodes for strided vector ld/st generation tests/tcg/riscv64: Add test for vlsseg8e32 instruction

 target/riscv/insn_trans/trans_rvv.c.inc   | 354 ++++++++++++++++++++--
 tests/tcg/riscv64/Makefile.softmmu-target |   7 +-
 tests/tcg/riscv64/test-vlsseg8e32.S       | 107 +++++++
 3 files changed, 450 insertions(+), 18 deletions(-)
 create mode 100644 tests/tcg/riscv64/test-vlsseg8e32.S

--
2.53.0

author
#1 lore raw

PATCH v10 1/2 target/riscv: Use tcg nodes for strided vector ld/st generation

This commit improves the performance of QEMU when emulating strided vector loads and stores by substituting the call for the helper function with the generation of equivalent TCG operations.

PS:

An implementation is permitted to cause an illegal instruction if vstart is not 0 and it is set to a value that can not be produced implicitly by the implementation, but memory accesses will generally always need to deal with page faults.

So, if a strided vector memory access instruction has non-zero vstart, check it through vlse/vsse helpers function.

---
 target/riscv/insn_trans/trans_rvv.c.inc | 354 ++++++++++++++++++++++--
 1 file changed, 337 insertions(+), 17 deletions(-)
target/riscv/insn_trans/trans_rvv.c.inc +337 -17
index 5b72926b3c..4cf3643cbd 100644
@@ -16,6 +16,7 @@
  */
 #include "tcg/tcg-op-gvec.h"
 #include "tcg/tcg-gvec-desc.h"
+#include "tcg/tcg-temp-internal.h"
 #include "internals.h"
 
 static inline bool is_overlapped(const int8_t astart, int8_t asize,
@@ -872,15 +873,290 @@ static bool st_us_mask_check(DisasContext *s, arg_vsm_v *a, uint8_t eew)
 GEN_VEXT_TRANS(vlm_v, MO_8, vlm_v, ld_us_mask_op, ld_us_mask_check)
 GEN_VEXT_TRANS(vsm_v, MO_8, vsm_v, st_us_mask_op, st_us_mask_check)
 
+/*
+ * MAXSZ returns the maximum vector size can be operated in bytes,
+ * which is used in GVEC IR when vl_eq_vlmax flag is set to true
+ * to accelerate vector operation.
+ */
+static inline uint32_t MAXSZ(DisasContext *s)
+{
+    int max_sz = s->cfg_ptr->vlenb << 3;
+    return max_sz >> (3 - s->lmul);
+}
+
+static inline uint32_t get_log2(uint32_t a)
+{
+    assert(is_power_of_2(a));
+    return ctz32(a);
+}
+
+typedef void gen_tl_ldst(TCGv, TCGv_ptr, tcg_target_long);
+
+static void gen_ldst_vreg(DisasContext *s, TCGv_i64 dest_offs, TCGv_i64 addr,
+                          gen_tl_ldst *ld_fn, gen_tl_ldst *st_fn, bool is_load)
+{
+    MemOp atomicity = (s->sew == 0) ? MO_ATOM_NONE : MO_ATOM_IFALIGN_PAIR;
+    TCGv_ptr dest_ptr = tcg_temp_new_ptr();
+    TCGv_i64 vreg = tcg_temp_new_i64();
+    tcg_gen_trunc_i64_ptr(dest_ptr, dest_offs);
+
+    if (is_load) {
+        tcg_gen_qemu_ld_tl(vreg, addr, s->mem_idx, MO_LE | s->sew | atomicity);
+        st_fn(vreg, dest_ptr, 0);
+    } else {
+        ld_fn(vreg, dest_ptr, 0);
+        tcg_gen_qemu_st_tl(vreg, addr, s->mem_idx, MO_LE | s->sew | atomicity);
+    }
+    tcg_temp_free_ptr(dest_ptr);
+    tcg_temp_free_i64(vreg);
+}
+
+/*
+ * Check whether the i bit of the mask is 0 or 1.
+ *
+ * static inline int vext_elem_mask(void *v0, int index)
+ * {
+ *     int idx = index / 64;
+ *     int pos = index % 64;
+ *     return (((uint64_t *)v0)[idx] >> pos) & 1;
+ * }
+ *
+ * And
+ *
+ * if (vext_elem_mask(v0, i) != 0) {
+ *     goto label;
+ * }
+ */
+static void gen_check_vext_elem_mask(DisasContext *s, TCGLabel *label,
+                                     TCGv_i64 mask_offs)
+{
+    TCGv_i64 temp = tcg_temp_new_i64();
+    TCGv_ptr ptr = tcg_temp_new_ptr();
+    TCGv_i64 elem = tcg_temp_new_i64();
+
+    tcg_gen_shri_tl(temp, mask_offs, 3);
+    tcg_gen_trunc_i64_ptr(ptr, temp);
+    tcg_gen_add_ptr(ptr, ptr, tcg_env);
+
+    tcg_gen_ld8u_i64(elem, ptr, 0);
+    tcg_gen_andi_tl(temp, mask_offs, 7);
+    tcg_gen_shr_tl(elem, elem, temp);
+    tcg_gen_brcond_i64(TCG_COND_TSTNE, elem, tcg_constant_i64(1), label);
+
+    tcg_temp_free_i64(temp);
+    tcg_temp_free_ptr(ptr);
+    tcg_temp_free_i64(elem);
+}
+
+static void gen_vext_set_elems_1s(TCGv dest, TCGv_i64 mask_offs, int sew,
+                                  gen_tl_ldst *st_fn, bool is_load)
+{
+    if (is_load) {
+        TCGv_ptr ptr = tcg_temp_new_ptr();
+        tcg_gen_shli_tl(mask_offs, mask_offs, sew);
+        tcg_gen_add_tl(mask_offs, mask_offs, dest);
+        tcg_gen_trunc_i64_ptr(ptr, mask_offs);
+        st_fn(tcg_constant_tl(-1), ptr, 0);
+        tcg_temp_free_ptr(ptr);
+    }
+}
+
+/*
+ * Simulate the strided load/store main loop:
+ *
+ * for (i = env->vstart; i < env->vl; env->vstart = ++i) {
+ *     k = 0;
+ *     while (k < nf) {
+ *         if (!vm && !vext_elem_mask(v0, i)) {
+ *             vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
+ *                               (i + k * max_elems + 1) * esz);
+ *             k++;
+ *             continue;
+ *         }
+ *         target_ulong addr = base + stride * i + (k << log2_esz);
+ *         ldst(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+ *         k++;
+ *     }
+ * }
+ */
+static void gen_ldst_stride_main_loop(DisasContext *s, TCGv dest, uint32_t rs1,
+                                      uint32_t rs2, uint32_t vm, uint32_t nf,
+                                      gen_tl_ldst *ld_fn, gen_tl_ldst *st_fn,
+                                      bool is_load)
+{
+    TCGv_i64 addr = tcg_temp_new_i64();
+    TCGv base = get_gpr(s, rs1, EXT_NONE);
+    TCGv stride = get_gpr(s, rs2, EXT_NONE);
+
+    TCGv i = tcg_temp_new();
+    TCGv i_esz = tcg_temp_new();
+    TCGv k = tcg_temp_new();
+    TCGv k_esz = tcg_temp_new();
+    TCGv k_max = tcg_temp_new();
+    TCGv_i64 mask_offs = tcg_temp_new_i64();
+    TCGv_i64 dest_offs = tcg_temp_new_i64();
+    TCGv_i64 stride_offs = tcg_temp_new_i64();
+
+    uint32_t max_elems = MAXSZ(s) >> s->sew;
+
+    TCGLabel *start = gen_new_label();
+    TCGLabel *end = gen_new_label();
+    TCGLabel *start_k = gen_new_label();
+    TCGLabel *inc_k = gen_new_label();
+    TCGLabel *end_k = gen_new_label();
+
+    /* Start of outer loop. */
+    tcg_gen_mov_tl(i, cpu_vstart);
+    gen_set_label(start);
+    tcg_gen_brcond_tl(TCG_COND_GE, i, cpu_vl, end);
+    tcg_gen_shli_tl(i_esz, i, s->sew);
+
+    /* Start of inner loop. */
+    tcg_gen_movi_tl(k, 0);
+    gen_set_label(start_k);
+    tcg_gen_brcond_tl(TCG_COND_GE, k, tcg_constant_tl(nf), end_k);
+
+    /*
+     * If we are in mask agnostic regime and the operation is not unmasked we
+     * set the inactive elements to 1.
+     */
+    if (!vm && s->vma) {
+        TCGLabel *active_element = gen_new_label();
+        /* (i + k * max_elems) * esz */
+        tcg_gen_shli_tl(mask_offs, k, get_log2(max_elems << s->sew));
+        tcg_gen_add_tl(mask_offs, mask_offs, i_esz);
+
+        /*
+         * Check whether the i bit of the mask is 0 or 1.
+         * If it is 0, set masked-off elements;
+         * otherwise, directly load/store the vector register.
+         */
+        gen_check_vext_elem_mask(s, active_element, mask_offs);
+
+        /*
+         * Set masked-off elements in the destination vector register to 1s.
+         * Store instructions simply skip this bit as memory ops access memory
+         * only for active elements.
+         */
+        gen_vext_set_elems_1s(dest, mask_offs, s->sew, st_fn, is_load);
+
+        tcg_gen_br(inc_k);
+        gen_set_label(active_element);
+    }
+
+    /*
+     * The element is active, calculate the address with stride:
+     * target_ulong addr = base + stride * i + (k << log2_esz);
+     */
+    tcg_gen_mul_tl(stride_offs, stride, i);
+    tcg_gen_shli_tl(k_esz, k, s->sew);
+    tcg_gen_add_tl(stride_offs, stride_offs, k_esz);
+    tcg_gen_add_tl(addr, base, stride_offs);
+
+    /* Calculate the offset in the dst/src vector register. */
+    tcg_gen_shli_tl(k_max, k, get_log2(max_elems));
+    tcg_gen_add_tl(dest_offs, i, k_max);
+    tcg_gen_shli_tl(dest_offs, dest_offs, s->sew);
+    tcg_gen_add_tl(dest_offs, dest_offs, dest);
+
+    /* Load/Store vector register. */
+    gen_ldst_vreg(s, dest_offs, addr, ld_fn, st_fn, is_load);
+
+    /*
+     * We don't execute the load/store above if the element was inactive.
+     * We jump instead directly to incrementing k and continuing the loop.
+     */
+    if (!vm && s->vma) {
+        gen_set_label(inc_k);
+    }
+    tcg_gen_addi_tl(k, k, 1);
+    tcg_gen_br(start_k);
+
+    /* End of the inner loop. */
+    gen_set_label(end_k);
+
+    tcg_gen_addi_tl(i, i, 1);
+    tcg_gen_mov_tl(cpu_vstart, i);
+    tcg_gen_br(start);
+
+    /* End of the outer loop. */
+    gen_set_label(end);
+
+    return;
+}
+
+/*
+ * Set the tail bytes of the strided loads/stores to 1:
+ *
+ * for (k = 0; k < nf; ++k) {
+ *     cnt = (k * max_elems + vl) * esz;
+ *     tot = (k * max_elems + max_elems) * esz;
+ *     for (i = cnt; i < tot; i += esz) {
+ *         store_1s(-1, vd[vl+i]);
+ *     }
+ * }
+ */
+static void gen_ldst_stride_tail_loop(DisasContext *s, TCGv dest, uint32_t nf,
+                                      gen_tl_ldst *st_fn)
+{
+    TCGv i = tcg_temp_new();
+    TCGv k = tcg_temp_new();
+    TCGv tail_cnt = tcg_temp_new();
+    TCGv tail_tot = tcg_temp_new();
+    TCGv tail_addr = tcg_temp_new();
+
+    TCGLabel *start = gen_new_label();
+    TCGLabel *end = gen_new_label();
+    TCGLabel *start_i = gen_new_label();
+    TCGLabel *end_i = gen_new_label();
+
+    uint32_t max_elems_b = MAXSZ(s);
+    uint32_t esz = 1 << s->sew;
+
+    /* Start of the outer loop. */
+    tcg_gen_movi_tl(k, 0);
+    tcg_gen_shli_tl(tail_cnt, cpu_vl, s->sew);
+    tcg_gen_movi_tl(tail_tot, max_elems_b);
+    tcg_gen_add_tl(tail_addr, dest, tail_cnt);
+    gen_set_label(start);
+    tcg_gen_brcond_tl(TCG_COND_GE, k, tcg_constant_tl(nf), end);
+
+    /* Start of the inner loop. */
+    tcg_gen_mov_tl(i, tail_cnt);
+    gen_set_label(start_i);
+    tcg_gen_brcond_tl(TCG_COND_GE, i, tail_tot, end_i);
+
+    /* store_1s(-1, vd[vl+i]); */
+    st_fn(tcg_constant_tl(-1), (TCGv_ptr)tail_addr, 0);
+    tcg_gen_addi_tl(tail_addr, tail_addr, esz);
+    tcg_gen_addi_tl(i, i, esz);
+    tcg_gen_br(start_i);
+
+    /* End of the inner loop. */
+    gen_set_label(end_i);
+
+    /* Update the counts */
+    tcg_gen_addi_tl(tail_cnt, tail_cnt, max_elems_b);
+    tcg_gen_addi_tl(tail_tot, tail_cnt, max_elems_b);
+    tcg_gen_addi_tl(k, k, 1);
+    tcg_gen_br(start);
+
+    /* End of the outer loop. */
+    gen_set_label(end);
+
+    return;
+}
+
 /*
  *** stride load and store
  */
 typedef void gen_helper_ldst_stride(TCGv_ptr, TCGv_ptr, TCGv,
                                     TCGv, TCGv_env, TCGv_i32);
 
-static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
-                              uint32_t data, gen_helper_ldst_stride *fn,
-                              DisasContext *s)
+static
+bool gen_call_helper_ldst_stride(uint32_t vd, uint32_t rs1, uint32_t rs2,
+                                 uint32_t data, gen_helper_ldst_stride *fn,
+                                 DisasContext *s)
 {
     TCGv_ptr dest, mask;
     TCGv base, stride;
@@ -904,11 +1180,66 @@ static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
     return true;
 }
 
+static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
+                              uint32_t data, gen_helper_ldst_stride *fn,
+                              DisasContext *s, bool is_load)
+{
+    if (!s->vstart_eq_zero) {
+        /* vstart != 0 helper slowpath */
+        return gen_call_helper_ldst_stride(vd, rs1, rs2, data, fn, s);
+    }
+
+    TCGv dest = tcg_temp_new();
+
+    uint32_t nf = FIELD_EX32(data, VDATA, NF);
+    uint32_t vm = FIELD_EX32(data, VDATA, VM);
+
+    /* Destination register and mask register */
+    tcg_gen_addi_tl(dest, (TCGv)tcg_env, vreg_ofs(s, vd));
+
+    /*
+     * Select the appropriate load/store to retrieve data from the vector
+     * register given a specific sew.
+     */
+    static gen_tl_ldst * const ld_fns[4] = {
+        tcg_gen_ld8u_tl, tcg_gen_ld16u_tl,
+        tcg_gen_ld32u_tl, tcg_gen_ld_tl
+    };
+
+    static gen_tl_ldst * const st_fns[4] = {
+        tcg_gen_st8_tl, tcg_gen_st16_tl,
+        tcg_gen_st32_tl, tcg_gen_st_tl
+    };
+
+    gen_tl_ldst *ld_fn = ld_fns[s->sew];
+    gen_tl_ldst *st_fn = st_fns[s->sew];
+
+    if (ld_fn == NULL || st_fn == NULL) {
+        return false;
+    }
+
+    mark_vs_dirty(s);
+
+    gen_ldst_stride_main_loop(s, dest, rs1, rs2, vm, nf, ld_fn, st_fn, is_load);
+
+    tcg_gen_movi_tl(cpu_vstart, 0);
+
+    /*
+     * Set the tail bytes to 1 if tail agnostic:
+     */
+    if (s->vta != 0 && is_load) {
+        gen_ldst_stride_tail_loop(s, dest, nf, st_fn);
+    }
+
+    finalize_rvv_inst(s);
+    return true;
+}
+
 static bool ld_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
 {
     uint32_t data = 0;
     gen_helper_ldst_stride *fn;
-    static gen_helper_ldst_stride * const fns[4] = {
+    static gen_helper_ldst_stride *const fns[4] = {
         gen_helper_vlse8_v, gen_helper_vlse16_v,
         gen_helper_vlse32_v, gen_helper_vlse64_v
     };
@@ -924,7 +1255,7 @@ static bool ld_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
     data = FIELD_DP32(data, VDATA, NF, a->nf);
     data = FIELD_DP32(data, VDATA, VTA, s->vta);
     data = FIELD_DP32(data, VDATA, VMA, s->vma);
-    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s);
+    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s, true);
 }
 
 static bool ld_stride_check(DisasContext *s, arg_rnfvm* a, uint8_t eew)
@@ -958,7 +1289,7 @@ static bool st_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
         return false;
     }
 
-    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s);
+    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s, false);
 }
 
 static bool st_stride_check(DisasContext *s, arg_rnfvm* a, uint8_t eew)
@@ -1281,17 +1612,6 @@ GEN_LDST_WHOLE_TRANS(vs8r_v, int8_t, 8, false)
  *** Vector Integer Arithmetic Instructions
  */
 
-/*
- * MAXSZ returns the maximum vector size can be operated in bytes,
- * which is used in GVEC IR when vl_eq_vlmax flag is set to true
- * to accelerate vector operation.
- */
-static inline uint32_t MAXSZ(DisasContext *s)
-{
-    int max_sz = s->cfg_ptr->vlenb * 8;
-    return max_sz >> (3 - s->lmul);
-}
-
 static bool opivv_check(DisasContext *s, arg_rmrr *a)
 {
     return require_rvv(s) &&
Signature · 1 line
2.53.0
#2 lore raw

Re: [PATCH v10 1/2] target/riscv: Use tcg nodes for strided vector ld/st generation

Hi Chao,

Quoted 59 lines · Chao Liu

This commit improves the performance of QEMU when emulating strided vector loads and stores by substituting the call for the helper function with the generation of equivalent TCG operations.

PS:

An implementation is permitted to cause an illegal instruction if vstart is not 0 and it is set to a value that can not be produced implicitly by the implementation, but memory accesses will generally always need to deal with page faults.

So, if a strided vector memory access instruction has non-zero vstart, check it through vlse/vsse helpers function.

---
  target/riscv/insn_trans/trans_rvv.c.inc | 354 ++++++++++++++++++++++--
  1 file changed, 337 insertions(+), 17 deletions(-)
target/riscv/insn_trans/trans_rvv.c.inc +21 -0
index 5b72926b3c..4cf3643cbd 100644
@@ -16,6 +16,7 @@
   */
  #include "tcg/tcg-op-gvec.h"
  #include "tcg/tcg-gvec-desc.h"
+#include "tcg/tcg-temp-internal.h"
  #include "internals.h"
 
  static inline bool is_overlapped(const int8_t astart, int8_t asize,
@@ -872,15 +873,290 @@ static bool st_us_mask_check(DisasContext *s, arg_vsm_v *a, uint8_t eew)
  GEN_VEXT_TRANS(vlm_v, MO_8, vlm_v, ld_us_mask_op, ld_us_mask_check)
  GEN_VEXT_TRANS(vsm_v, MO_8, vsm_v, st_us_mask_op, st_us_mask_check)
 
+/*
+ * MAXSZ returns the maximum vector size can be operated in bytes,
+ * which is used in GVEC IR when vl_eq_vlmax flag is set to true
+ * to accelerate vector operation.
+ */
+static inline uint32_t MAXSZ(DisasContext *s)
+{
+    int max_sz = s->cfg_ptr->vlenb << 3;
+    return max_sz >> (3 - s->lmul);
+}
+
+static inline uint32_t get_log2(uint32_t a)
+{
+    assert(is_power_of_2(a));
+    return ctz32(a);
+}
+
+typedef void gen_tl_ldst(TCGv, TCGv_ptr, tcg_target_long);
+
+static void gen_ldst_vreg(DisasContext *s, TCGv_i64 dest_offs, TCGv_i64 addr,

target/riscv/insn_trans/trans_rvv.c.inc static bool st_us_mask_check(DisasContext *s, arg_vsm_v *a, uint8_t eew) · line 905

Quoted 10 lines
target/riscv/insn_trans/trans_rvv.c.inc +10 -0
+                          gen_tl_ldst *ld_fn, gen_tl_ldst *st_fn, bool is_load)
+{
+    MemOp atomicity = (s->sew == 0) ? MO_ATOM_NONE : MO_ATOM_IFALIGN_PAIR;
+    TCGv_ptr dest_ptr = tcg_temp_new_ptr();
+    TCGv_i64 vreg = tcg_temp_new_i64();
+    tcg_gen_trunc_i64_ptr(dest_ptr, dest_offs);
+
+    if (is_load) {
+        tcg_gen_qemu_ld_tl(vreg, addr, s->mem_idx, MO_LE | s->sew | atomicity);
+        st_fn(vreg, dest_ptr, 0);

So I'm not sure how we missed this for this long hehehe but this won't build for riscv32-softmmu:

In file included from ../target/riscv/translate.c:1207:
../target/riscv/insn_trans/trans_rvv.c.inc: In function ‘gen_ldst_vreg’:
../target/riscv/insn_trans/trans_rvv.c.inc:904:28: error: passing argument 1 of ‘tcg_gen_qemu_ld_i32’ from incompatible pointer type [-Werror=incompatible-pointer-types]

  904 |         tcg_gen_qemu_ld_tl(vreg, addr, s->mem_idx, MO_LE | s->sew | atomicity);
      |                            ^~~~
      |                            |
      |                            TCGv_i64 {aka struct TCGv_i64_d *}

In file included from ../target/riscv/translate.c:22:
/home/dbarboza/work/qemu/include/tcg/tcg-op.h:50:30: note: expected ‘TCGv_i32’ {aka ‘struct TCGv_i32_d *’} but argument is of type ‘TCGv_i64’ {aka ‘struct TCGv_i64_d *’}
50 | tcg_gen_qemu_ld_i32(TCGv_i32 v, TCGv a, TCGArg i, MemOp m)

The issue here is that you're using tcg_gen_qemu_ld_tl() with a TCGv_i64 arg. This works normally in a 64 bit but, for a riscv32 build, tcg_gen_qemu_ld_tl will default to tcg_gen_qemu_ld_i32 (from tcg-op.h):

#if TARGET_LONG_BITS == 32
typedef TCGv_i32 TCGv;
#define tcg_temp_new() tcg_temp_new_i32()
#define tcg_global_mem_new tcg_global_mem_new_i32 #define tcgv_tl_temp tcgv_i32_temp
#define tcg_gen_qemu_ld_tl tcg_gen_qemu_ld_i32 <========= #define tcg_gen_qemu_st_tl tcg_gen_qemu_st_i32

Given that vreg is a TCGv_i64 what would work here is use tcg_gen_qemu_ld_i64 directly. But note that this code needs to work for 32 bits too, therefore I'm not sure if vreg should be fixed in 64 bits like we're doing here.

I suggest using vreg as 'TCGv" and then initiating it via "tcg_temp_new()". This will create either a i32 or a i64 var depending on the build target. You can then keep using tcg_gen_qemu_ld_tl().

Note that this is not the only build error I caught and there might be others similars. You'll have to test the patch building with riscv32-softmmu to address all of them.

Thanks,
Daniel

Quoted 364 lines
target/riscv/insn_trans/trans_rvv.c.inc +306 -17
+    } else {
+        ld_fn(vreg, dest_ptr, 0);
+        tcg_gen_qemu_st_tl(vreg, addr, s->mem_idx, MO_LE | s->sew | atomicity);
+    }
+    tcg_temp_free_ptr(dest_ptr);
+    tcg_temp_free_i64(vreg);
+}
+
+/*
+ * Check whether the i bit of the mask is 0 or 1.
+ *
+ * static inline int vext_elem_mask(void *v0, int index)
+ * {
+ *     int idx = index / 64;
+ *     int pos = index % 64;
+ *     return (((uint64_t *)v0)[idx] >> pos) & 1;
+ * }
+ *
+ * And
+ *
+ * if (vext_elem_mask(v0, i) != 0) {
+ *     goto label;
+ * }
+ */
+static void gen_check_vext_elem_mask(DisasContext *s, TCGLabel *label,
+                                     TCGv_i64 mask_offs)
+{
+    TCGv_i64 temp = tcg_temp_new_i64();
+    TCGv_ptr ptr = tcg_temp_new_ptr();
+    TCGv_i64 elem = tcg_temp_new_i64();
+
+    tcg_gen_shri_tl(temp, mask_offs, 3);
+    tcg_gen_trunc_i64_ptr(ptr, temp);
+    tcg_gen_add_ptr(ptr, ptr, tcg_env);
+
+    tcg_gen_ld8u_i64(elem, ptr, 0);
+    tcg_gen_andi_tl(temp, mask_offs, 7);
+    tcg_gen_shr_tl(elem, elem, temp);
+    tcg_gen_brcond_i64(TCG_COND_TSTNE, elem, tcg_constant_i64(1), label);
+
+    tcg_temp_free_i64(temp);
+    tcg_temp_free_ptr(ptr);
+    tcg_temp_free_i64(elem);
+}
+
+static void gen_vext_set_elems_1s(TCGv dest, TCGv_i64 mask_offs, int sew,
+                                  gen_tl_ldst *st_fn, bool is_load)
+{
+    if (is_load) {
+        TCGv_ptr ptr = tcg_temp_new_ptr();
+        tcg_gen_shli_tl(mask_offs, mask_offs, sew);
+        tcg_gen_add_tl(mask_offs, mask_offs, dest);
+        tcg_gen_trunc_i64_ptr(ptr, mask_offs);
+        st_fn(tcg_constant_tl(-1), ptr, 0);
+        tcg_temp_free_ptr(ptr);
+    }
+}
+
+/*
+ * Simulate the strided load/store main loop:
+ *
+ * for (i = env->vstart; i < env->vl; env->vstart = ++i) {
+ *     k = 0;
+ *     while (k < nf) {
+ *         if (!vm && !vext_elem_mask(v0, i)) {
+ *             vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
+ *                               (i + k * max_elems + 1) * esz);
+ *             k++;
+ *             continue;
+ *         }
+ *         target_ulong addr = base + stride * i + (k << log2_esz);
+ *         ldst(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+ *         k++;
+ *     }
+ * }
+ */
+static void gen_ldst_stride_main_loop(DisasContext *s, TCGv dest, uint32_t rs1,
+                                      uint32_t rs2, uint32_t vm, uint32_t nf,
+                                      gen_tl_ldst *ld_fn, gen_tl_ldst *st_fn,
+                                      bool is_load)
+{
+    TCGv_i64 addr = tcg_temp_new_i64();
+    TCGv base = get_gpr(s, rs1, EXT_NONE);
+    TCGv stride = get_gpr(s, rs2, EXT_NONE);
+
+    TCGv i = tcg_temp_new();
+    TCGv i_esz = tcg_temp_new();
+    TCGv k = tcg_temp_new();
+    TCGv k_esz = tcg_temp_new();
+    TCGv k_max = tcg_temp_new();
+    TCGv_i64 mask_offs = tcg_temp_new_i64();
+    TCGv_i64 dest_offs = tcg_temp_new_i64();
+    TCGv_i64 stride_offs = tcg_temp_new_i64();
+
+    uint32_t max_elems = MAXSZ(s) >> s->sew;
+
+    TCGLabel *start = gen_new_label();
+    TCGLabel *end = gen_new_label();
+    TCGLabel *start_k = gen_new_label();
+    TCGLabel *inc_k = gen_new_label();
+    TCGLabel *end_k = gen_new_label();
+
+    /* Start of outer loop. */
+    tcg_gen_mov_tl(i, cpu_vstart);
+    gen_set_label(start);
+    tcg_gen_brcond_tl(TCG_COND_GE, i, cpu_vl, end);
+    tcg_gen_shli_tl(i_esz, i, s->sew);
+
+    /* Start of inner loop. */
+    tcg_gen_movi_tl(k, 0);
+    gen_set_label(start_k);
+    tcg_gen_brcond_tl(TCG_COND_GE, k, tcg_constant_tl(nf), end_k);
+
+    /*
+     * If we are in mask agnostic regime and the operation is not unmasked we
+     * set the inactive elements to 1.
+     */
+    if (!vm && s->vma) {
+        TCGLabel *active_element = gen_new_label();
+        /* (i + k * max_elems) * esz */
+        tcg_gen_shli_tl(mask_offs, k, get_log2(max_elems << s->sew));
+        tcg_gen_add_tl(mask_offs, mask_offs, i_esz);
+
+        /*
+         * Check whether the i bit of the mask is 0 or 1.
+         * If it is 0, set masked-off elements;
+         * otherwise, directly load/store the vector register.
+         */
+        gen_check_vext_elem_mask(s, active_element, mask_offs);
+
+        /*
+         * Set masked-off elements in the destination vector register to 1s.
+         * Store instructions simply skip this bit as memory ops access memory
+         * only for active elements.
+         */
+        gen_vext_set_elems_1s(dest, mask_offs, s->sew, st_fn, is_load);
+
+        tcg_gen_br(inc_k);
+        gen_set_label(active_element);
+    }
+
+    /*
+     * The element is active, calculate the address with stride:
+     * target_ulong addr = base + stride * i + (k << log2_esz);
+     */
+    tcg_gen_mul_tl(stride_offs, stride, i);
+    tcg_gen_shli_tl(k_esz, k, s->sew);
+    tcg_gen_add_tl(stride_offs, stride_offs, k_esz);
+    tcg_gen_add_tl(addr, base, stride_offs);
+
+    /* Calculate the offset in the dst/src vector register. */
+    tcg_gen_shli_tl(k_max, k, get_log2(max_elems));
+    tcg_gen_add_tl(dest_offs, i, k_max);
+    tcg_gen_shli_tl(dest_offs, dest_offs, s->sew);
+    tcg_gen_add_tl(dest_offs, dest_offs, dest);
+
+    /* Load/Store vector register. */
+    gen_ldst_vreg(s, dest_offs, addr, ld_fn, st_fn, is_load);
+
+    /*
+     * We don't execute the load/store above if the element was inactive.
+     * We jump instead directly to incrementing k and continuing the loop.
+     */
+    if (!vm && s->vma) {
+        gen_set_label(inc_k);
+    }
+    tcg_gen_addi_tl(k, k, 1);
+    tcg_gen_br(start_k);
+
+    /* End of the inner loop. */
+    gen_set_label(end_k);
+
+    tcg_gen_addi_tl(i, i, 1);
+    tcg_gen_mov_tl(cpu_vstart, i);
+    tcg_gen_br(start);
+
+    /* End of the outer loop. */
+    gen_set_label(end);
+
+    return;
+}
+
+/*
+ * Set the tail bytes of the strided loads/stores to 1:
+ *
+ * for (k = 0; k < nf; ++k) {
+ *     cnt = (k * max_elems + vl) * esz;
+ *     tot = (k * max_elems + max_elems) * esz;
+ *     for (i = cnt; i < tot; i += esz) {
+ *         store_1s(-1, vd[vl+i]);
+ *     }
+ * }
+ */
+static void gen_ldst_stride_tail_loop(DisasContext *s, TCGv dest, uint32_t nf,
+                                      gen_tl_ldst *st_fn)
+{
+    TCGv i = tcg_temp_new();
+    TCGv k = tcg_temp_new();
+    TCGv tail_cnt = tcg_temp_new();
+    TCGv tail_tot = tcg_temp_new();
+    TCGv tail_addr = tcg_temp_new();
+
+    TCGLabel *start = gen_new_label();
+    TCGLabel *end = gen_new_label();
+    TCGLabel *start_i = gen_new_label();
+    TCGLabel *end_i = gen_new_label();
+
+    uint32_t max_elems_b = MAXSZ(s);
+    uint32_t esz = 1 << s->sew;
+
+    /* Start of the outer loop. */
+    tcg_gen_movi_tl(k, 0);
+    tcg_gen_shli_tl(tail_cnt, cpu_vl, s->sew);
+    tcg_gen_movi_tl(tail_tot, max_elems_b);
+    tcg_gen_add_tl(tail_addr, dest, tail_cnt);
+    gen_set_label(start);
+    tcg_gen_brcond_tl(TCG_COND_GE, k, tcg_constant_tl(nf), end);
+
+    /* Start of the inner loop. */
+    tcg_gen_mov_tl(i, tail_cnt);
+    gen_set_label(start_i);
+    tcg_gen_brcond_tl(TCG_COND_GE, i, tail_tot, end_i);
+
+    /* store_1s(-1, vd[vl+i]); */
+    st_fn(tcg_constant_tl(-1), (TCGv_ptr)tail_addr, 0);
+    tcg_gen_addi_tl(tail_addr, tail_addr, esz);
+    tcg_gen_addi_tl(i, i, esz);
+    tcg_gen_br(start_i);
+
+    /* End of the inner loop. */
+    gen_set_label(end_i);
+
+    /* Update the counts */
+    tcg_gen_addi_tl(tail_cnt, tail_cnt, max_elems_b);
+    tcg_gen_addi_tl(tail_tot, tail_cnt, max_elems_b);
+    tcg_gen_addi_tl(k, k, 1);
+    tcg_gen_br(start);
+
+    /* End of the outer loop. */
+    gen_set_label(end);
+
+    return;
+}
+
  /*
   *** stride load and store
   */
  typedef void gen_helper_ldst_stride(TCGv_ptr, TCGv_ptr, TCGv,
                                      TCGv, TCGv_env, TCGv_i32);
 
-static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
-                              uint32_t data, gen_helper_ldst_stride *fn,
-                              DisasContext *s)
+static
+bool gen_call_helper_ldst_stride(uint32_t vd, uint32_t rs1, uint32_t rs2,
+                                 uint32_t data, gen_helper_ldst_stride *fn,
+                                 DisasContext *s)
  {
      TCGv_ptr dest, mask;
      TCGv base, stride;
@@ -904,11 +1180,66 @@ static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
      return true;
  }
 
+static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
+                              uint32_t data, gen_helper_ldst_stride *fn,
+                              DisasContext *s, bool is_load)
+{
+    if (!s->vstart_eq_zero) {
+        /* vstart != 0 helper slowpath */
+        return gen_call_helper_ldst_stride(vd, rs1, rs2, data, fn, s);
+    }
+
+    TCGv dest = tcg_temp_new();
+
+    uint32_t nf = FIELD_EX32(data, VDATA, NF);
+    uint32_t vm = FIELD_EX32(data, VDATA, VM);
+
+    /* Destination register and mask register */
+    tcg_gen_addi_tl(dest, (TCGv)tcg_env, vreg_ofs(s, vd));
+
+    /*
+     * Select the appropriate load/store to retrieve data from the vector
+     * register given a specific sew.
+     */
+    static gen_tl_ldst * const ld_fns[4] = {
+        tcg_gen_ld8u_tl, tcg_gen_ld16u_tl,
+        tcg_gen_ld32u_tl, tcg_gen_ld_tl
+    };
+
+    static gen_tl_ldst * const st_fns[4] = {
+        tcg_gen_st8_tl, tcg_gen_st16_tl,
+        tcg_gen_st32_tl, tcg_gen_st_tl
+    };
+
+    gen_tl_ldst *ld_fn = ld_fns[s->sew];
+    gen_tl_ldst *st_fn = st_fns[s->sew];
+
+    if (ld_fn == NULL || st_fn == NULL) {
+        return false;
+    }
+
+    mark_vs_dirty(s);
+
+    gen_ldst_stride_main_loop(s, dest, rs1, rs2, vm, nf, ld_fn, st_fn, is_load);
+
+    tcg_gen_movi_tl(cpu_vstart, 0);
+
+    /*
+     * Set the tail bytes to 1 if tail agnostic:
+     */
+    if (s->vta != 0 && is_load) {
+        gen_ldst_stride_tail_loop(s, dest, nf, st_fn);
+    }
+
+    finalize_rvv_inst(s);
+    return true;
+}
+
  static bool ld_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
  {
      uint32_t data = 0;
      gen_helper_ldst_stride *fn;
-    static gen_helper_ldst_stride * const fns[4] = {
+    static gen_helper_ldst_stride *const fns[4] = {
          gen_helper_vlse8_v, gen_helper_vlse16_v,
          gen_helper_vlse32_v, gen_helper_vlse64_v
      };
@@ -924,7 +1255,7 @@ static bool ld_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
      data = FIELD_DP32(data, VDATA, NF, a->nf);
      data = FIELD_DP32(data, VDATA, VTA, s->vta);
      data = FIELD_DP32(data, VDATA, VMA, s->vma);
-    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s);
+    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s, true);
  }
 
  static bool ld_stride_check(DisasContext *s, arg_rnfvm* a, uint8_t eew)
@@ -958,7 +1289,7 @@ static bool st_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
          return false;
      }
 
-    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s);
+    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s, false);
  }
 
  static bool st_stride_check(DisasContext *s, arg_rnfvm* a, uint8_t eew)
@@ -1281,17 +1612,6 @@ GEN_LDST_WHOLE_TRANS(vs8r_v, int8_t, 8, false)
   *** Vector Integer Arithmetic Instructions
   */
 
-/*
- * MAXSZ returns the maximum vector size can be operated in bytes,
- * which is used in GVEC IR when vl_eq_vlmax flag is set to true
- * to accelerate vector operation.
- */
-static inline uint32_t MAXSZ(DisasContext *s)
-{
-    int max_sz = s->cfg_ptr->vlenb * 8;
-    return max_sz >> (3 - s->lmul);
-}
-
  static bool opivv_check(DisasContext *s, arg_rmrr *a)
  {
      return require_rvv(s) &&
author
#3 lore raw

Re: [PATCH v10 1/2] target/riscv: Use tcg nodes for strided vector ld/st generation

Quoted 82 lines · Daniel Henrique Barboza

Hi Chao,

Quoted 57 lines · Chao Liu

This commit improves the performance of QEMU when emulating strided vector loads and stores by substituting the call for the helper function with the generation of equivalent TCG operations.

PS:

An implementation is permitted to cause an illegal instruction if vstart is not 0 and it is set to a value that can not be produced implicitly by the implementation, but memory accesses will generally always need to deal with page faults.

So, if a strided vector memory access instruction has non-zero vstart, check it through vlse/vsse helpers function.

---
  target/riscv/insn_trans/trans_rvv.c.inc | 354 ++++++++++++++++++++++--
  1 file changed, 337 insertions(+), 17 deletions(-)
target/riscv/insn_trans/trans_rvv.c.inc +21 -0
index 5b72926b3c..4cf3643cbd 100644
@@ -16,6 +16,7 @@
   */
  #include "tcg/tcg-op-gvec.h"
  #include "tcg/tcg-gvec-desc.h"
+#include "tcg/tcg-temp-internal.h"
  #include "internals.h"
  static inline bool is_overlapped(const int8_t astart, int8_t asize,
@@ -872,15 +873,290 @@ static bool st_us_mask_check(DisasContext *s, arg_vsm_v *a, uint8_t eew)
  GEN_VEXT_TRANS(vlm_v, MO_8, vlm_v, ld_us_mask_op, ld_us_mask_check)
  GEN_VEXT_TRANS(vsm_v, MO_8, vsm_v, st_us_mask_op, st_us_mask_check)
+/*
+ * MAXSZ returns the maximum vector size can be operated in bytes,
+ * which is used in GVEC IR when vl_eq_vlmax flag is set to true
+ * to accelerate vector operation.
+ */
+static inline uint32_t MAXSZ(DisasContext *s)
+{
+    int max_sz = s->cfg_ptr->vlenb << 3;
+    return max_sz >> (3 - s->lmul);
+}
+
+static inline uint32_t get_log2(uint32_t a)
+{
+    assert(is_power_of_2(a));
+    return ctz32(a);
+}
+
+typedef void gen_tl_ldst(TCGv, TCGv_ptr, tcg_target_long);
+
+static void gen_ldst_vreg(DisasContext *s, TCGv_i64 dest_offs, TCGv_i64 addr,

target/riscv/insn_trans/trans_rvv.c.inc static bool st_us_mask_check(DisasContext *s, arg_vsm_v *a, uint8_t eew) · line 904

Quoted 10 lines
target/riscv/insn_trans/trans_rvv.c.inc +10 -0
+                          gen_tl_ldst *ld_fn, gen_tl_ldst *st_fn, bool is_load)
+{
+    MemOp atomicity = (s->sew == 0) ? MO_ATOM_NONE : MO_ATOM_IFALIGN_PAIR;
+    TCGv_ptr dest_ptr = tcg_temp_new_ptr();
+    TCGv_i64 vreg = tcg_temp_new_i64();
+    tcg_gen_trunc_i64_ptr(dest_ptr, dest_offs);
+
+    if (is_load) {
+        tcg_gen_qemu_ld_tl(vreg, addr, s->mem_idx, MO_LE | s->sew | atomicity);
+        st_fn(vreg, dest_ptr, 0);

So I'm not sure how we missed this for this long hehehe but this won't build for riscv32-softmmu:

In file included from ../target/riscv/translate.c:1207:
../target/riscv/insn_trans/trans_rvv.c.inc: In function ‘gen_ldst_vreg’:
../target/riscv/insn_trans/trans_rvv.c.inc:904:28: error: passing argument 1 of ‘tcg_gen_qemu_ld_i32’ from incompatible pointer type [-Werror=incompatible-pointer-types]

  904 |         tcg_gen_qemu_ld_tl(vreg, addr, s->mem_idx, MO_LE | s->sew | atomicity);
      |                            ^~~~
      |                            |
      |                            TCGv_i64 {aka struct TCGv_i64_d *}
Quoted 9 lines

In file included from ../target/riscv/translate.c:22:
/home/dbarboza/work/qemu/include/tcg/tcg-op.h:50:30: note: expected ‘TCGv_i32’ {aka ‘struct TCGv_i32_d *’} but argument is of type ‘TCGv_i64’ {aka ‘struct TCGv_i64_d *’}
50 | tcg_gen_qemu_ld_i32(TCGv_i32 v, TCGv a, TCGArg i, MemOp m)

The issue here is that you're using tcg_gen_qemu_ld_tl() with a TCGv_i64 arg. This works normally in a 64 bit but, for a riscv32 build, tcg_gen_qemu_ld_tl will default to tcg_gen_qemu_ld_i32 (from tcg-op.h):

Oh, I was too focused on the riscv64 implementation before. Thanks for helping compile and test!

Quoted 9 lines

#if TARGET_LONG_BITS == 32
typedef TCGv_i32 TCGv;
#define tcg_temp_new() tcg_temp_new_i32()
#define tcg_global_mem_new tcg_global_mem_new_i32 #define tcgv_tl_temp tcgv_i32_temp
#define tcg_gen_qemu_ld_tl tcg_gen_qemu_ld_i32 <========= #define tcg_gen_qemu_st_tl tcg_gen_qemu_st_i32

Quoted 8 lines

Given that vreg is a TCGv_i64 what would work here is use tcg_gen_qemu_ld_i64 directly. But note that this code needs to work for 32 bits too, therefore I'm not sure if vreg should be fixed in 64 bits like we're doing here.

I suggest using vreg as 'TCGv" and then initiating it via "tcg_temp_new()". This will create either a i32 or a i64 var depending on the build target. You can then keep using tcg_gen_qemu_ld_tl().

Yes, you're right. I'll fix it in the next version and make sure the full CI passes.

Thanks,
Chao

Quoted 370 lines

Note that this is not the only build error I caught and there might be others similars. You'll have to test the patch building with riscv32-softmmu to address all of them.

Thanks,
Daniel

Quoted 358 lines
target/riscv/insn_trans/trans_rvv.c.inc +306 -17
+    } else {
+        ld_fn(vreg, dest_ptr, 0);
+        tcg_gen_qemu_st_tl(vreg, addr, s->mem_idx, MO_LE | s->sew | atomicity);
+    }
+    tcg_temp_free_ptr(dest_ptr);
+    tcg_temp_free_i64(vreg);
+}
+
+/*
+ * Check whether the i bit of the mask is 0 or 1.
+ *
+ * static inline int vext_elem_mask(void *v0, int index)
+ * {
+ *     int idx = index / 64;
+ *     int pos = index % 64;
+ *     return (((uint64_t *)v0)[idx] >> pos) & 1;
+ * }
+ *
+ * And
+ *
+ * if (vext_elem_mask(v0, i) != 0) {
+ *     goto label;
+ * }
+ */
+static void gen_check_vext_elem_mask(DisasContext *s, TCGLabel *label,
+                                     TCGv_i64 mask_offs)
+{
+    TCGv_i64 temp = tcg_temp_new_i64();
+    TCGv_ptr ptr = tcg_temp_new_ptr();
+    TCGv_i64 elem = tcg_temp_new_i64();
+
+    tcg_gen_shri_tl(temp, mask_offs, 3);
+    tcg_gen_trunc_i64_ptr(ptr, temp);
+    tcg_gen_add_ptr(ptr, ptr, tcg_env);
+
+    tcg_gen_ld8u_i64(elem, ptr, 0);
+    tcg_gen_andi_tl(temp, mask_offs, 7);
+    tcg_gen_shr_tl(elem, elem, temp);
+    tcg_gen_brcond_i64(TCG_COND_TSTNE, elem, tcg_constant_i64(1), label);
+
+    tcg_temp_free_i64(temp);
+    tcg_temp_free_ptr(ptr);
+    tcg_temp_free_i64(elem);
+}
+
+static void gen_vext_set_elems_1s(TCGv dest, TCGv_i64 mask_offs, int sew,
+                                  gen_tl_ldst *st_fn, bool is_load)
+{
+    if (is_load) {
+        TCGv_ptr ptr = tcg_temp_new_ptr();
+        tcg_gen_shli_tl(mask_offs, mask_offs, sew);
+        tcg_gen_add_tl(mask_offs, mask_offs, dest);
+        tcg_gen_trunc_i64_ptr(ptr, mask_offs);
+        st_fn(tcg_constant_tl(-1), ptr, 0);
+        tcg_temp_free_ptr(ptr);
+    }
+}
+
+/*
+ * Simulate the strided load/store main loop:
+ *
+ * for (i = env->vstart; i < env->vl; env->vstart = ++i) {
+ *     k = 0;
+ *     while (k < nf) {
+ *         if (!vm && !vext_elem_mask(v0, i)) {
+ *             vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
+ *                               (i + k * max_elems + 1) * esz);
+ *             k++;
+ *             continue;
+ *         }
+ *         target_ulong addr = base + stride * i + (k << log2_esz);
+ *         ldst(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+ *         k++;
+ *     }
+ * }
+ */
+static void gen_ldst_stride_main_loop(DisasContext *s, TCGv dest, uint32_t rs1,
+                                      uint32_t rs2, uint32_t vm, uint32_t nf,
+                                      gen_tl_ldst *ld_fn, gen_tl_ldst *st_fn,
+                                      bool is_load)
+{
+    TCGv_i64 addr = tcg_temp_new_i64();
+    TCGv base = get_gpr(s, rs1, EXT_NONE);
+    TCGv stride = get_gpr(s, rs2, EXT_NONE);
+
+    TCGv i = tcg_temp_new();
+    TCGv i_esz = tcg_temp_new();
+    TCGv k = tcg_temp_new();
+    TCGv k_esz = tcg_temp_new();
+    TCGv k_max = tcg_temp_new();
+    TCGv_i64 mask_offs = tcg_temp_new_i64();
+    TCGv_i64 dest_offs = tcg_temp_new_i64();
+    TCGv_i64 stride_offs = tcg_temp_new_i64();
+
+    uint32_t max_elems = MAXSZ(s) >> s->sew;
+
+    TCGLabel *start = gen_new_label();
+    TCGLabel *end = gen_new_label();
+    TCGLabel *start_k = gen_new_label();
+    TCGLabel *inc_k = gen_new_label();
+    TCGLabel *end_k = gen_new_label();
+
+    /* Start of outer loop. */
+    tcg_gen_mov_tl(i, cpu_vstart);
+    gen_set_label(start);
+    tcg_gen_brcond_tl(TCG_COND_GE, i, cpu_vl, end);
+    tcg_gen_shli_tl(i_esz, i, s->sew);
+
+    /* Start of inner loop. */
+    tcg_gen_movi_tl(k, 0);
+    gen_set_label(start_k);
+    tcg_gen_brcond_tl(TCG_COND_GE, k, tcg_constant_tl(nf), end_k);
+
+    /*
+     * If we are in mask agnostic regime and the operation is not unmasked we
+     * set the inactive elements to 1.
+     */
+    if (!vm && s->vma) {
+        TCGLabel *active_element = gen_new_label();
+        /* (i + k * max_elems) * esz */
+        tcg_gen_shli_tl(mask_offs, k, get_log2(max_elems << s->sew));
+        tcg_gen_add_tl(mask_offs, mask_offs, i_esz);
+
+        /*
+         * Check whether the i bit of the mask is 0 or 1.
+         * If it is 0, set masked-off elements;
+         * otherwise, directly load/store the vector register.
+         */
+        gen_check_vext_elem_mask(s, active_element, mask_offs);
+
+        /*
+         * Set masked-off elements in the destination vector register to 1s.
+         * Store instructions simply skip this bit as memory ops access memory
+         * only for active elements.
+         */
+        gen_vext_set_elems_1s(dest, mask_offs, s->sew, st_fn, is_load);
+
+        tcg_gen_br(inc_k);
+        gen_set_label(active_element);
+    }
+
+    /*
+     * The element is active, calculate the address with stride:
+     * target_ulong addr = base + stride * i + (k << log2_esz);
+     */
+    tcg_gen_mul_tl(stride_offs, stride, i);
+    tcg_gen_shli_tl(k_esz, k, s->sew);
+    tcg_gen_add_tl(stride_offs, stride_offs, k_esz);
+    tcg_gen_add_tl(addr, base, stride_offs);
+
+    /* Calculate the offset in the dst/src vector register. */
+    tcg_gen_shli_tl(k_max, k, get_log2(max_elems));
+    tcg_gen_add_tl(dest_offs, i, k_max);
+    tcg_gen_shli_tl(dest_offs, dest_offs, s->sew);
+    tcg_gen_add_tl(dest_offs, dest_offs, dest);
+
+    /* Load/Store vector register. */
+    gen_ldst_vreg(s, dest_offs, addr, ld_fn, st_fn, is_load);
+
+    /*
+     * We don't execute the load/store above if the element was inactive.
+     * We jump instead directly to incrementing k and continuing the loop.
+     */
+    if (!vm && s->vma) {
+        gen_set_label(inc_k);
+    }
+    tcg_gen_addi_tl(k, k, 1);
+    tcg_gen_br(start_k);
+
+    /* End of the inner loop. */
+    gen_set_label(end_k);
+
+    tcg_gen_addi_tl(i, i, 1);
+    tcg_gen_mov_tl(cpu_vstart, i);
+    tcg_gen_br(start);
+
+    /* End of the outer loop. */
+    gen_set_label(end);
+
+    return;
+}
+
+/*
+ * Set the tail bytes of the strided loads/stores to 1:
+ *
+ * for (k = 0; k < nf; ++k) {
+ *     cnt = (k * max_elems + vl) * esz;
+ *     tot = (k * max_elems + max_elems) * esz;
+ *     for (i = cnt; i < tot; i += esz) {
+ *         store_1s(-1, vd[vl+i]);
+ *     }
+ * }
+ */
+static void gen_ldst_stride_tail_loop(DisasContext *s, TCGv dest, uint32_t nf,
+                                      gen_tl_ldst *st_fn)
+{
+    TCGv i = tcg_temp_new();
+    TCGv k = tcg_temp_new();
+    TCGv tail_cnt = tcg_temp_new();
+    TCGv tail_tot = tcg_temp_new();
+    TCGv tail_addr = tcg_temp_new();
+
+    TCGLabel *start = gen_new_label();
+    TCGLabel *end = gen_new_label();
+    TCGLabel *start_i = gen_new_label();
+    TCGLabel *end_i = gen_new_label();
+
+    uint32_t max_elems_b = MAXSZ(s);
+    uint32_t esz = 1 << s->sew;
+
+    /* Start of the outer loop. */
+    tcg_gen_movi_tl(k, 0);
+    tcg_gen_shli_tl(tail_cnt, cpu_vl, s->sew);
+    tcg_gen_movi_tl(tail_tot, max_elems_b);
+    tcg_gen_add_tl(tail_addr, dest, tail_cnt);
+    gen_set_label(start);
+    tcg_gen_brcond_tl(TCG_COND_GE, k, tcg_constant_tl(nf), end);
+
+    /* Start of the inner loop. */
+    tcg_gen_mov_tl(i, tail_cnt);
+    gen_set_label(start_i);
+    tcg_gen_brcond_tl(TCG_COND_GE, i, tail_tot, end_i);
+
+    /* store_1s(-1, vd[vl+i]); */
+    st_fn(tcg_constant_tl(-1), (TCGv_ptr)tail_addr, 0);
+    tcg_gen_addi_tl(tail_addr, tail_addr, esz);
+    tcg_gen_addi_tl(i, i, esz);
+    tcg_gen_br(start_i);
+
+    /* End of the inner loop. */
+    gen_set_label(end_i);
+
+    /* Update the counts */
+    tcg_gen_addi_tl(tail_cnt, tail_cnt, max_elems_b);
+    tcg_gen_addi_tl(tail_tot, tail_cnt, max_elems_b);
+    tcg_gen_addi_tl(k, k, 1);
+    tcg_gen_br(start);
+
+    /* End of the outer loop. */
+    gen_set_label(end);
+
+    return;
+}
+
  /*
   *** stride load and store
   */
  typedef void gen_helper_ldst_stride(TCGv_ptr, TCGv_ptr, TCGv,
                                      TCGv, TCGv_env, TCGv_i32);
-static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
-                              uint32_t data, gen_helper_ldst_stride *fn,
-                              DisasContext *s)
+static
+bool gen_call_helper_ldst_stride(uint32_t vd, uint32_t rs1, uint32_t rs2,
+                                 uint32_t data, gen_helper_ldst_stride *fn,
+                                 DisasContext *s)
  {
      TCGv_ptr dest, mask;
      TCGv base, stride;
@@ -904,11 +1180,66 @@ static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
      return true;
  }
+static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
+                              uint32_t data, gen_helper_ldst_stride *fn,
+                              DisasContext *s, bool is_load)
+{
+    if (!s->vstart_eq_zero) {
+        /* vstart != 0 helper slowpath */
+        return gen_call_helper_ldst_stride(vd, rs1, rs2, data, fn, s);
+    }
+
+    TCGv dest = tcg_temp_new();
+
+    uint32_t nf = FIELD_EX32(data, VDATA, NF);
+    uint32_t vm = FIELD_EX32(data, VDATA, VM);
+
+    /* Destination register and mask register */
+    tcg_gen_addi_tl(dest, (TCGv)tcg_env, vreg_ofs(s, vd));
+
+    /*
+     * Select the appropriate load/store to retrieve data from the vector
+     * register given a specific sew.
+     */
+    static gen_tl_ldst * const ld_fns[4] = {
+        tcg_gen_ld8u_tl, tcg_gen_ld16u_tl,
+        tcg_gen_ld32u_tl, tcg_gen_ld_tl
+    };
+
+    static gen_tl_ldst * const st_fns[4] = {
+        tcg_gen_st8_tl, tcg_gen_st16_tl,
+        tcg_gen_st32_tl, tcg_gen_st_tl
+    };
+
+    gen_tl_ldst *ld_fn = ld_fns[s->sew];
+    gen_tl_ldst *st_fn = st_fns[s->sew];
+
+    if (ld_fn == NULL || st_fn == NULL) {
+        return false;
+    }
+
+    mark_vs_dirty(s);
+
+    gen_ldst_stride_main_loop(s, dest, rs1, rs2, vm, nf, ld_fn, st_fn, is_load);
+
+    tcg_gen_movi_tl(cpu_vstart, 0);
+
+    /*
+     * Set the tail bytes to 1 if tail agnostic:
+     */
+    if (s->vta != 0 && is_load) {
+        gen_ldst_stride_tail_loop(s, dest, nf, st_fn);
+    }
+
+    finalize_rvv_inst(s);
+    return true;
+}
+
  static bool ld_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
  {
      uint32_t data = 0;
      gen_helper_ldst_stride *fn;
-    static gen_helper_ldst_stride * const fns[4] = {
+    static gen_helper_ldst_stride *const fns[4] = {
          gen_helper_vlse8_v, gen_helper_vlse16_v,
          gen_helper_vlse32_v, gen_helper_vlse64_v
      };
@@ -924,7 +1255,7 @@ static bool ld_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
      data = FIELD_DP32(data, VDATA, NF, a->nf);
      data = FIELD_DP32(data, VDATA, VTA, s->vta);
      data = FIELD_DP32(data, VDATA, VMA, s->vma);
-    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s);
+    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s, true);
  }
  static bool ld_stride_check(DisasContext *s, arg_rnfvm* a, uint8_t eew)
@@ -958,7 +1289,7 @@ static bool st_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
          return false;
      }
-    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s);
+    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s, false);
  }
  static bool st_stride_check(DisasContext *s, arg_rnfvm* a, uint8_t eew)
@@ -1281,17 +1612,6 @@ GEN_LDST_WHOLE_TRANS(vs8r_v, int8_t, 8, false)
   *** Vector Integer Arithmetic Instructions
   */
-/*
- * MAXSZ returns the maximum vector size can be operated in bytes,
- * which is used in GVEC IR when vl_eq_vlmax flag is set to true
- * to accelerate vector operation.
- */
-static inline uint32_t MAXSZ(DisasContext *s)
-{
-    int max_sz = s->cfg_ptr->vlenb * 8;
-    return max_sz >> (3 - s->lmul);
-}
-
  static bool opivv_check(DisasContext *s, arg_rmrr *a)
  {
      return require_rvv(s) &&
author
#4 lore raw

PATCH v10 2/2 tests/tcg/riscv64: Add test for vlsseg8e32 instruction

In reply to Chao Liu

This case, it copied 64 bytes from a0 to a1 with vlsseg8e32.

---
 tests/tcg/riscv64/Makefile.softmmu-target |   7 +-
 tests/tcg/riscv64/test-vlsseg8e32.S       | 107 ++++++++++++++++++++++
 2 files changed, 113 insertions(+), 1 deletion(-)
 create mode 100644 tests/tcg/riscv64/test-vlsseg8e32.S
tests/tcg/riscv64/Makefile.softmmu-target +6 -1
index 82be8a2c91..05c8f851e0 100644
@@ -7,7 +7,7 @@ VPATH += $(TEST_SRC)
 
 LINK_SCRIPT = $(TEST_SRC)/semihost.ld
 LDFLAGS = -T $(LINK_SCRIPT)
-CFLAGS += -g -Og
+CFLAGS += -march=rv64gcv -mabi=lp64d -g -Og
 
 %.o: %.S
 	$(CC) $(CFLAGS) $< -Wa,--noexecstack -c -o $@
@@ -41,5 +41,10 @@ comma:= ,
 run-test-crc32: test-crc32
 	$(call run-test, $<, $(QEMU) -cpu rv64$(comma)xlrbr=true $(QEMU_OPTS)$<)
 
+EXTRA_RUNS += run-vlsseg8e32
+run-vlsseg8e32: QEMU_OPTS := -cpu rv64,v=true $(QEMU_OPTS)
+run-vlsseg8e32: test-vlsseg8e32
+	$(call run-test, $<, $(QEMU) $(QEMU_OPTS)$<)
+
 # We don't currently support the multiarch system tests
 undefine MULTIARCH_TESTS
tests/tcg/riscv64/test-vlsseg8e32.S new file +107 -0

107 lines not on this page · raw message

Signature · 1 line
2.53.0