Skip to contentSkip to conversation

target/xtensa: rework zero overhead loops implementation

PATCH Superseded Max Filippov 2 participants 7 messages Jan 11, 2019

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 target/xtensa: rework zero overhead loops implementation

Don't invalidate TB with the end of zero overhead loop when LBEG or LEND change. Instead encode the distance from the start of the page where the TB starts to the LEND in the TB cs_base and generate loopback code when the next PC matches encoded LEND. Distance to a destination within the same page and up to a maximum instruction length into the next page is encoded literally, otherwise it's zero. The distance from LEND to LBEG is also encoded in the cs_base: it's encoded literally when less than 256 or as 0 otherwise. This allows for TB chaining for the loopback branch at the end of a loop for the most common loop sizes.

With this change the resulting emulation speed is about 10% higher in softmmu mode on uClibc-ng and LTP tests. Emulation speed in linux user mode is a few percent lower because there's no direct TB chaining between different memory pages. Testing with lower limit on direct TB chainig range shows gradual slowdown to ~15% for the block size of 64 bytes and ~50% for the block size of 32 bytes.

---
Changes v1->v2:
- drop LINKABLE_*;
- add comment about LEND offset encoding;

 target/xtensa/cpu.h          | 32 ++++++++++++++++++++++++++
 target/xtensa/helper.h       |  2 --
 target/xtensa/op_helper.c    | 24 --------------------
 target/xtensa/overlay_tool.h |  1 +
 target/xtensa/translate.c    | 53 +++++++++++++-------------------------------
 5 files changed, 49 insertions(+), 63 deletions(-)
target/xtensa/cpu.h +32 -0
index 34e5ccd9f1d6..6c2636d06c5e 100644
@@ -400,6 +400,7 @@ struct XtensaConfig {
     int excm_level;
     int ndepc;
     unsigned inst_fetch_width;
+    unsigned max_insn_size;
     uint32_t vecbase;
     uint32_t exception_vector[EXC_MAX];
     unsigned ninterrupt;
@@ -695,6 +696,11 @@ static inline int cpu_mmu_index(CPUXtensaState *env, bool ifetch)
 #define XTENSA_TBFLAG_CALLINC_MASK 0x180000
 #define XTENSA_TBFLAG_CALLINC_SHIFT 19
 
+#define XTENSA_CSBASE_LEND_MASK 0x0000ffff
+#define XTENSA_CSBASE_LEND_SHIFT 0
+#define XTENSA_CSBASE_LBEG_OFF_MASK 0x00ff0000
+#define XTENSA_CSBASE_LBEG_OFF_SHIFT 16
+
 static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc,
         target_ulong *cs_base, uint32_t *flags)
 {
@@ -706,6 +712,32 @@ static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc,
     *flags |= xtensa_get_ring(env);
     if (env->sregs[PS] & PS_EXCM) {
         *flags |= XTENSA_TBFLAG_EXCM;
+    } else if (xtensa_option_enabled(env->config, XTENSA_OPTION_LOOP)) {
+        target_ulong lend_dist =
+            env->sregs[LEND] - (env->pc & -(1u << TARGET_PAGE_BITS));
+
+        /*
+         * 0 in the csbase_lend field means that there may not be a loopback
+         * for any instruction that starts inside this page. Any other value
+         * means that an instruction that ends at this offset from the page
+         * start may loop back.
+         *
+         * lend_dist is 0 when LEND points to the start of the page, but
+         * no instruction that starts inside this page may end at offset 0,
+         * so it's still correct.
+         *
+         * When an instruction ends at a page boundary it may only start in
+         * the previous page. lend_dist will be encoded as TARGET_PAGE_SIZE
+         * for the TB that contains this instruction.
+         */
+        if (lend_dist < (1u << TARGET_PAGE_BITS) + env->config->max_insn_size) {
+            target_ulong lbeg_off = env->sregs[LEND] - env->sregs[LBEG];
+
+            *cs_base = lend_dist;
+            if (lbeg_off < 256) {
+                *cs_base |= lbeg_off << XTENSA_CSBASE_LBEG_OFF_SHIFT;
+            }
+        }
     }
     if (xtensa_option_enabled(env->config, XTENSA_OPTION_EXTENDED_L32R) &&
             (env->sregs[LITBASE] & 1)) {
target/xtensa/helper.h +0 -2
index 10153c245360..2ebba0b2c2bf 100644
@@ -12,8 +12,6 @@ DEF_HELPER_2(rotw, void, env, i32)
 DEF_HELPER_3(window_check, noreturn, env, i32, i32)
 DEF_HELPER_1(restore_owb, void, env)
 DEF_HELPER_2(movsp, void, env, i32)
-DEF_HELPER_2(wsr_lbeg, void, env, i32)
-DEF_HELPER_2(wsr_lend, void, env, i32)
 #ifndef CONFIG_USER_ONLY
 DEF_HELPER_1(simcall, void, env)
 #endif
target/xtensa/op_helper.c +0 -24
index e4b42ab3e56c..078aeb6c2c94 100644
@@ -107,13 +107,6 @@ static void tb_invalidate_virtual_addr(CPUXtensaState *env, uint32_t vaddr)
     }
 }
 
-#else
-
-static void tb_invalidate_virtual_addr(CPUXtensaState *env, uint32_t vaddr)
-{
-    tb_invalidate_phys_addr(vaddr);
-}
-
 #endif
 
 void HELPER(exception)(CPUXtensaState *env, uint32_t excp)
@@ -370,23 +363,6 @@ void HELPER(movsp)(CPUXtensaState *env, uint32_t pc)
     }
 }
 
-void HELPER(wsr_lbeg)(CPUXtensaState *env, uint32_t v)
-{
-    if (env->sregs[LBEG] != v) {
-        tb_invalidate_virtual_addr(env, env->sregs[LEND] - 1);
-        env->sregs[LBEG] = v;
-    }
-}
-
-void HELPER(wsr_lend)(CPUXtensaState *env, uint32_t v)
-{
-    if (env->sregs[LEND] != v) {
-        tb_invalidate_virtual_addr(env, env->sregs[LEND] - 1);
-        env->sregs[LEND] = v;
-        tb_invalidate_virtual_addr(env, env->sregs[LEND] - 1);
-    }
-}
-
 void HELPER(dump_state)(CPUXtensaState *env)
 {
     XtensaCPU *cpu = xtensa_env_get_cpu(env);
target/xtensa/overlay_tool.h +1 -0
index ee37a04a176c..12609a0d0c1e 100644
@@ -457,6 +457,7 @@
     .nareg = XCHAL_NUM_AREGS, \
     .ndepc = (XCHAL_XEA_VERSION >= 2), \
     .inst_fetch_width = XCHAL_INST_FETCH_WIDTH, \
+    .max_insn_size = XCHAL_MAX_INSTRUCTION_SIZE, \
     EXCEPTIONS_SECTION, \
     INTERRUPTS_SECTION, \
     TLB_SECTION, \
target/xtensa/translate.c +16 -37

144 lines not on this page · raw message

Signature · 1 line
2.11.0
Reviewed-by on v2 1/1
#1 lore raw
Quoted 10 lines · Max Filippov

Don't invalidate TB with the end of zero overhead loop when LBEG or LEND change. Instead encode the distance from the start of the page where the TB starts to the LEND in the TB cs_base and generate loopback code when the next PC matches encoded LEND. Distance to a destination within the same page and up to a maximum instruction length into the next page is encoded literally, otherwise it's zero. The distance from LEND to LBEG is also encoded in the cs_base: it's encoded literally when less than 256 or as 0 otherwise. This allows for TB chaining for the loopback branch at the end of a loop for the most common loop sizes.

Quoted 8 lines

With this change the resulting emulation speed is about 10% higher in softmmu mode on uClibc-ng and LTP tests. Emulation speed in linux user mode is a few percent lower because there's no direct TB chaining between different memory pages. Testing with lower limit on direct TB chainig range shows gradual slowdown to ~15% for the block size of 64 bytes and ~50% for the block size of 32 bytes.

target/xtensa/cpu.h static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc, · line 723

Quoted 5 lines
target/xtensa/cpu.h +5 -0
+        /*
+         * 0 in the csbase_lend field means that there may not be a loopback
+         * for any instruction that starts inside this page. Any other value
+         * means that an instruction that ends at this offset from the page
+         * start may loop back.

Nit: s/may/will/g

Using "may" makes it seem like we may have missed a case that should have looped back.

r~

#2 lore raw
Quoted 30 lines · Richard Henderson
Quoted 10 lines · Max Filippov

Don't invalidate TB with the end of zero overhead loop when LBEG or LEND change. Instead encode the distance from the start of the page where the TB starts to the LEND in the TB cs_base and generate loopback code when the next PC matches encoded LEND. Distance to a destination within the same page and up to a maximum instruction length into the next page is encoded literally, otherwise it's zero. The distance from LEND to LBEG is also encoded in the cs_base: it's encoded literally when less than 256 or as 0 otherwise. This allows for TB chaining for the loopback branch at the end of a loop for the most common loop sizes.

Quoted 8 lines

With this change the resulting emulation speed is about 10% higher in softmmu mode on uClibc-ng and LTP tests. Emulation speed in linux user mode is a few percent lower because there's no direct TB chaining between different memory pages. Testing with lower limit on direct TB chainig range shows gradual slowdown to ~15% for the block size of 64 bytes and ~50% for the block size of 32 bytes.

target/xtensa/cpu.h static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc, · line 723

Quoted 5 lines
target/xtensa/cpu.h +5 -0
+        /*
+         * 0 in the csbase_lend field means that there may not be a loopback
+         * for any instruction that starts inside this page. Any other value
+         * means that an instruction that ends at this offset from the page
+         * start may loop back.
Quoted 4 lines

Nit: s/may/will/g

Using "may" makes it seem like we may have missed a case that should have looped back.

Using "will" makes it seem like it always loops back, but it doesn't do it when LCOUNT is 0.

Signature · 2 lines
Thanks.
-- Max
#3 lore raw
Quoted 6 lines · Max Filippov
Quoted 6 lines
Quoted 5 lines
target/xtensa/cpu.h +5 -0
+        /*
+         * 0 in the csbase_lend field means that there may not be a loopback
+         * for any instruction that starts inside this page. Any other value
+         * means that an instruction that ends at this offset from the page
+         * start may loop back.
Quoted 7 lines
Quoted 4 lines

Nit: s/may/will/g

Using "may" makes it seem like we may have missed a case that should have looped back.

Using "will" makes it seem like it always loops back, but it doesn't do it when LCOUNT is 0.

Err, no, you covered that in the first sentence, where 0 means that it will not loop back.

r~

#4 lore raw
Quoted 10 lines · Richard Henderson
Quoted 8 lines · Max Filippov
Quoted 8 lines

target/xtensa/cpu.h static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc, · line 723

Quoted 5 lines
target/xtensa/cpu.h +5 -0
+        /*
+         * 0 in the csbase_lend field means that there may not be a loopback
+         * for any instruction that starts inside this page. Any other value
+         * means that an instruction that ends at this offset from the page
+         * start may loop back.

Nit: s/may/will/g

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

Using "may" makes it seem like we may have missed a case that should have looped back.

Using "will" makes it seem like it always loops back, but it doesn't do it when LCOUNT is 0.

Err, no, you covered that in the first sentence, where 0 means that it will not loop back.

What I'm saying is that "Any other value means that an instruction that ends at this offset from the page start will loop back." sounds like that instruction will loop back unconditionally. LCOUNT = 0 is the condition when this doesn't happen, and it's not encoded in the TB flags. It's checked in the generated loopback code.

Signature · 2 lines
Thanks.
-- Max
#5 lore raw
Quoted 19 lines · Max Filippov
Quoted 16 lines · Richard Henderson
Quoted 13 lines · Max Filippov
Quoted 10 lines

target/xtensa/cpu.h static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc, · line 723

Quoted 5 lines
target/xtensa/cpu.h +5 -0
+        /*
+         * 0 in the csbase_lend field means that there may not be a loopback
+         * for any instruction that starts inside this page. Any other value
+         * means that an instruction that ends at this offset from the page
+         * start may loop back.

Nit: s/may/will/g

Using "may" makes it seem like we may have missed a case that should have looped back.

Using "will" makes it seem like it always loops back, but it doesn't do it when LCOUNT is 0.

Quoted 8 lines
Quoted 2 lines

Err, no, you covered that in the first sentence, where 0 means that it will not loop back.

What I'm saying is that "Any other value means that an instruction that ends at this offset from the page start will loop back." sounds like that instruction will loop back unconditionally. LCOUNT = 0 is the condition when this doesn't happen, and it's not encoded in the TB flags. It's checked in the generated loopback code.

Maybe "Any other value means that an instruction that ends at this offset from the page start may loop back and will need loopback code to be generated"?

Signature · 2 lines
Thanks.
-- Max
#6 lore raw
Quoted 2 lines · Max Filippov

Maybe "Any other value means that an instruction that ends at this offset from the page start may loop back and will need loopback code to be generated"?

Much better, thanks.

r~