hw/ppc: Fix clock update drift
PATCH 1/1 hw/ppc: Fix clock update drift
The clock update logic reads the clock twice to compute the new clock value, with a value derived from the later time subtracted from a value derived from the earlier time. The delta causes time to be lost.
This can ultimately result in time becoming unsynchronized between CPUs and that can cause OS lockups, timeouts, watchdogs, etc. This can be seen running a KVM guest (that causes lots of TB updates) on a powernv SMP machine.
Fix this by reading the clock once.
---
I also made a test case that can trigger this with kvm-unit-tests, but it's been taking me a while to get that upstreamed.
Thanks,
Nick
hw/ppc/ppc.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-)
hw/ppc/ppc.c +17 -16
@@ -535,23 +535,24 @@ static inline void cpu_ppc_store_tb(ppc_tb_t *tb_env, uint64_t vmclk,void cpu_ppc_store_tbl (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0xFFFFFFFF00000000ULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, tb | (uint64_t )value); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, tb | (uint64_t )value); }static inline void _cpu_ppc_store_tbu (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0x00000000FFFFFFFFULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, ((uint64_t )value <<32 ) | tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, + ((uint64_t )value <<32 ) | tb); }void cpu_ppc_store_tbu (CPUPPCState *env,uint32_t value)
@@ -584,23 +585,24 @@ uint32_t cpu_ppc_load_atbu (CPUPPCState *env)void cpu_ppc_store_atbl (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->atb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->atb_offset); tb &=0xFFFFFFFF00000000ULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->atb_offset, tb | (uint64_t )value); +cpu_ppc_store_tb (tb_env, clock, &tb_env->atb_offset, tb | (uint64_t )value); }void cpu_ppc_store_atbu (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->atb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->atb_offset); tb &=0x00000000FFFFFFFFULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->atb_offset, ((uint64_t )value <<32 ) | tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->atb_offset, + ((uint64_t )value <<32 ) | tb); }uint64_t cpu_ppc_load_vtb (CPUPPCState *env)
@@ -622,14 +624,13 @@ void cpu_ppc_store_vtb(CPUPPCState *env, uint64_t value)void cpu_ppc_store_tbu40 (CPUPPCState *env,uint64_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0xFFFFFFUL ; tb |= (value & ~0xFFFFFFUL ); -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, tb); }static void cpu_ppc_tb_stop (CPUPPCState *env)
Signature · 1 line
2.40.1
Quoted 9 lines · Nicholas Piggin
The clock update logic reads the clock twice to compute the new clock value, with a value derived from the later time subtracted from a value derived from the earlier time. The delta causes time to be lost.
This can ultimately result in time becoming unsynchronized between CPUs and that can cause OS lockups, timeouts, watchdogs, etc. This can be seen running a KVM guest (that causes lots of TB updates) on a powernv SMP machine.
Quoted 4 lines
Fix this by reading the clock once.
Signed-off-by Nicholas Piggin <[email protected]>
Thanks,
C.
Quoted 94 lines
---
I also made a test case that can trigger this with kvm-unit-tests, but it's been taking me a while to get that upstreamed.Thanks,
Nickhw/ppc/ppc.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-)hw/ppc/ppc.c +17
-16@@ -535,23 +535,24 @@ static inline void cpu_ppc_store_tb(ppc_tb_t *tb_env, uint64_t vmclk,void cpu_ppc_store_tbl (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0xFFFFFFFF00000000ULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, tb | (uint64_t )value); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, tb | (uint64_t )value); }static inline void _cpu_ppc_store_tbu (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0x00000000FFFFFFFFULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, ((uint64_t )value <<32 ) | tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, + ((uint64_t )value <<32 ) | tb); }void cpu_ppc_store_tbu (CPUPPCState *env,uint32_t value)@@ -584,23 +585,24 @@ uint32_t cpu_ppc_load_atbu (CPUPPCState *env)void cpu_ppc_store_atbl (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->atb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->atb_offset); tb &=0xFFFFFFFF00000000ULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->atb_offset, tb | (uint64_t )value); +cpu_ppc_store_tb (tb_env, clock, &tb_env->atb_offset, tb | (uint64_t )value); }void cpu_ppc_store_atbu (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->atb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->atb_offset); tb &=0x00000000FFFFFFFFULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->atb_offset, ((uint64_t )value <<32 ) | tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->atb_offset, + ((uint64_t )value <<32 ) | tb); }uint64_t cpu_ppc_load_vtb (CPUPPCState *env)@@ -622,14 +624,13 @@ void cpu_ppc_store_vtb(CPUPPCState *env, uint64_t value)void cpu_ppc_store_tbu40 (CPUPPCState *env,uint64_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0xFFFFFFUL ; tb |= (value & ~0xFFFFFFUL ); -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, tb); }static void cpu_ppc_tb_stop (CPUPPCState *env)
Quoted 9 lines · Nicholas Piggin
The clock update logic reads the clock twice to compute the new clock value, with a value derived from the later time subtracted from a value derived from the earlier time. The delta causes time to be lost.
This can ultimately result in time becoming unsynchronized between CPUs and that can cause OS lockups, timeouts, watchdogs, etc. This can be seen running a KVM guest (that causes lots of TB updates) on a powernv SMP machine.
Quoted 5 lines
Fix this by reading the clock once.
Signed-off-by Nicholas Piggin <[email protected]>---
Nice! It must have been a nightmare to debug.
Do you know where that ATB spr comes from? I can't relate it to the ISA.
Quoted 93 lines
I also made a test case that can trigger this with kvm-unit-tests, but it's been taking me a while to get that upstreamed.
Thanks,
Nickhw/ppc/ppc.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-)hw/ppc/ppc.c +17
-16@@ -535,23 +535,24 @@ static inline void cpu_ppc_store_tb(ppc_tb_t *tb_env, uint64_t vmclk,void cpu_ppc_store_tbl (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0xFFFFFFFF00000000ULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, tb | (uint64_t )value); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, tb | (uint64_t )value); }static inline void _cpu_ppc_store_tbu (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0x00000000FFFFFFFFULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, ((uint64_t )value <<32 ) | tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, + ((uint64_t )value <<32 ) | tb); }void cpu_ppc_store_tbu (CPUPPCState *env,uint32_t value)@@ -584,23 +585,24 @@ uint32_t cpu_ppc_load_atbu (CPUPPCState *env)void cpu_ppc_store_atbl (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->atb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->atb_offset); tb &=0xFFFFFFFF00000000ULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->atb_offset, tb | (uint64_t )value); +cpu_ppc_store_tb (tb_env, clock, &tb_env->atb_offset, tb | (uint64_t )value); }void cpu_ppc_store_atbu (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->atb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->atb_offset); tb &=0x00000000FFFFFFFFULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->atb_offset, ((uint64_t )value <<32 ) | tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->atb_offset, + ((uint64_t )value <<32 ) | tb); }uint64_t cpu_ppc_load_vtb (CPUPPCState *env)@@ -622,14 +624,13 @@ void cpu_ppc_store_vtb(CPUPPCState *env, uint64_t value)void cpu_ppc_store_tbu40 (CPUPPCState *env,uint64_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0xFFFFFFUL ; tb |= (value & ~0xFFFFFFUL ); -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, tb); }static void cpu_ppc_tb_stop (CPUPPCState *env)
Quoted 14 lines · Frederic Barrat
Quoted 11 lines · Nicholas Piggin
The clock update logic reads the clock twice to compute the new clock value, with a value derived from the later time subtracted from a value derived from the earlier time. The delta causes time to be lost.
This can ultimately result in time becoming unsynchronized between CPUs and that can cause OS lockups, timeouts, watchdogs, etc. This can be seen running a KVM guest (that causes lots of TB updates) on a powernv SMP machine.
Fix this by reading the clock once.
Quoted 7 lines
Quoted 3 lines
Signed-off-by Nicholas Piggin <[email protected]>---
Nice! It must have been a nightmare to debug.
Do you know where that ATB spr comes from? I can't relate it to the ISA.
You should look at the e500 CPU (BookE) specs.
C.
Quoted 20 lines · Cédric Le Goater
Quoted 19 lines · Frederic Barrat
Quoted 14 lines · Nicholas Piggin
The clock update logic reads the clock twice to compute the new clock value, with a value derived from the later time subtracted from a value derived from the earlier time. The delta causes time to be lost.
This can ultimately result in time becoming unsynchronized between CPUs and that can cause OS lockups, timeouts, watchdogs, etc. This can be seen running a KVM guest (that causes lots of TB updates) on a powernv SMP machine.
Fix this by reading the clock once.
Signed-off-by Nicholas Piggin <[email protected]>---
Quoted 4 lines
Quoted 2 lines
Nice! It must have been a nightmare to debug.
Do you know where that ATB spr comes from? I can't relate it to the ISA.You should look at the e500 CPU (BookE) specs.
Yeah it's in 2.07 in the Book E. It doesn't look like Linux uses it anywhere so I didn't actually test it, but the pattern looks the same as the others so I think it's should be right.
Thanks,
Nick
Daniel Henrique Barboza
AppliedQuoted 9 lines · Nicholas Piggin
The clock update logic reads the clock twice to compute the new clock value, with a value derived from the later time subtracted from a value derived from the earlier time. The delta causes time to be lost.
This can ultimately result in time becoming unsynchronized between CPUs and that can cause OS lockups, timeouts, watchdogs, etc. This can be seen running a KVM guest (that causes lots of TB updates) on a powernv SMP machine.
Quoted 5 lines
Fix this by reading the clock once.
Signed-off-by Nicholas Piggin <[email protected]>---
Queued in gitlab.com/danielhb/qemu/tree/ppc-next after adding the "Fixes" tag Cedric mentioned. Thanks,
Daniel
Quoted 93 lines
I also made a test case that can trigger this with kvm-unit-tests, but it's been taking me a while to get that upstreamed.
Thanks,
Nickhw/ppc/ppc.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-)hw/ppc/ppc.c +17
-16@@ -535,23 +535,24 @@ static inline void cpu_ppc_store_tb(ppc_tb_t *tb_env, uint64_t vmclk,void cpu_ppc_store_tbl (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0xFFFFFFFF00000000ULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, tb | (uint64_t )value); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, tb | (uint64_t )value); }static inline void _cpu_ppc_store_tbu (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0x00000000FFFFFFFFULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, ((uint64_t )value <<32 ) | tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, + ((uint64_t )value <<32 ) | tb); }void cpu_ppc_store_tbu (CPUPPCState *env,uint32_t value)@@ -584,23 +585,24 @@ uint32_t cpu_ppc_load_atbu (CPUPPCState *env)void cpu_ppc_store_atbl (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->atb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->atb_offset); tb &=0xFFFFFFFF00000000ULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->atb_offset, tb | (uint64_t )value); +cpu_ppc_store_tb (tb_env, clock, &tb_env->atb_offset, tb | (uint64_t )value); }void cpu_ppc_store_atbu (CPUPPCState *env,uint32_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), tb_env->atb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->atb_offset); tb &=0x00000000FFFFFFFFULL ; -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->atb_offset, ((uint64_t )value <<32 ) | tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->atb_offset, + ((uint64_t )value <<32 ) | tb); }uint64_t cpu_ppc_load_vtb (CPUPPCState *env)@@ -622,14 +624,13 @@ void cpu_ppc_store_vtb(CPUPPCState *env, uint64_t value)void cpu_ppc_store_tbu40 (CPUPPCState *env,uint64_t value) {ppc_tb_t *tb_env = env->tb_env; +int64_t clock =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL );uint64_t tb; - tb =cpu_ppc_get_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - tb_env->tb_offset); + tb =cpu_ppc_get_tb (tb_env, clock, tb_env->tb_offset); tb &=0xFFFFFFUL ; tb |= (value & ~0xFFFFFFUL ); -cpu_ppc_store_tb (tb_env,qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ), - &tb_env->tb_offset, tb); +cpu_ppc_store_tb (tb_env, clock, &tb_env->tb_offset, tb); }static void cpu_ppc_tb_stop (CPUPPCState *env)