Skip to contentSkip to conversation

riscv: Timer support

PATCH In review Andrew Jones 1 participant 7 messages Aug 28, 2024 – Sep 3, 2024

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover riscv: Timer support

We already have some timer / delay support but we leave a lot to the unit test authors for deciding what timer to use (SBI vs. Sstc) and setting it. Provide an API that prefers Sstc and falls back to SBI.

This found a bug in QEMU's Sstc that I'll try to find time to fix. If we start a timer with a long delay and then stop it before it has expired, QEMU still delivers the interrupt. The only way to avoid getting it on QEMU is to disable the timer irq. The BPI, which also has Sstc, behaves as expected though, i.e. even with the timer irq always enabled we won't get a timer irq if we stop it before it had a chance to expire. Disabling Sstc on QEMU, which falls back to SBI TIME, also behaves as expected.

Andrew Jones (3):
riscv: Introduce local_timer_init
riscv: Share sbi_time_ecall with framework
riscv: Provide timer_start and timer_stop

 lib/riscv/asm/sbi.h   |  1 +
 lib/riscv/asm/timer.h |  3 +++
 lib/riscv/sbi.c       |  5 +++++
 lib/riscv/setup.c     |  2 ++
 lib/riscv/smp.c       |  2 ++
 lib/riscv/timer.c     | 46 +++++++++++++++++++++++++++++++++++++++++++
 riscv/sbi.c           | 18 ++++-------------
 7 files changed, 63 insertions(+), 14 deletions(-)
Signature · 1 line
2.45.2
#1 lore raw

PATCH 1/3 riscv: Introduce local_timer_init

When Sstc is available make sure that even if we enable timer interrupts nothing will happen. This is necessary for cases where the unit tests actually intend to use the SBI TIME extension and aren't thinking about Sstc at all, like the SBI TIME test in riscv/sbi where we can now remove the initialization.

---
 lib/riscv/asm/timer.h |  1 +
 lib/riscv/setup.c     |  2 ++
 lib/riscv/smp.c       |  2 ++
 lib/riscv/timer.c     | 13 +++++++++++++
 riscv/sbi.c           |  5 -----
 5 files changed, 18 insertions(+), 5 deletions(-)
lib/riscv/asm/timer.h +1 -0
index b3514d3f6a78..fd12251a3a6b 100644
@@ -5,6 +5,7 @@
 #include <asm/csr.h>
 
 extern void timer_get_frequency(void);
+extern void local_timer_init(void);
 
 static inline uint64_t timer_get_cycles(void)
 {
lib/riscv/setup.c +2 -0
index 9a16f00093d7..7c4321b1c30f 100644
@@ -210,6 +210,7 @@ void setup(const void *fdt, phys_addr_t freemem_start)
 	cpu_init();
 	timer_get_frequency();
 	thread_info_init();
+	local_timer_init();
 	io_init();
 
 	ret = dt_get_bootargs(&bootargs);
@@ -276,6 +277,7 @@ efi_status_t setup_efi(efi_bootinfo_t *efi_bootinfo)
 	cpu_init();
 	timer_get_frequency();
 	thread_info_init();
+	local_timer_init();
 	io_init();
 	initrd_setup();
 
lib/riscv/smp.c +2 -0
index 4d373e0a29a8..18d0393c0cc2 100644
@@ -14,6 +14,7 @@
 #include <asm/processor.h>
 #include <asm/sbi.h>
 #include <asm/smp.h>
+#include <asm/timer.h>
 
 cpumask_t cpu_present_mask;
 cpumask_t cpu_online_mask;
@@ -27,6 +28,7 @@ secondary_func_t secondary_cinit(struct secondary_data *data)
 
 	__mmu_enable(data->satp);
 	thread_info_init();
+	local_timer_init();
 	info = current_thread_info();
 	set_cpu_online(info->cpu, true);
 	smp_send_event();
lib/riscv/timer.c +13 -0
index d78d254c8eca..92826d6ec3fe 100644
@@ -4,7 +4,11 @@
  */
 #include <libcflat.h>
 #include <devicetree.h>
+#include <limits.h>
+#include <asm/csr.h>
+#include <asm/isa.h>
 #include <asm/setup.h>
+#include <asm/smp.h>
 #include <asm/timer.h>
 
 void timer_get_frequency(void)
@@ -26,3 +30,12 @@ void timer_get_frequency(void)
 	data = (u32 *)prop->data;
 	timebase_frequency = fdt32_to_cpu(*data);
 }
+
+void local_timer_init(void)
+{
+	if (cpu_has_extension(smp_processor_id(), ISA_SSTC)) {
+		csr_write(CSR_STIMECMP, ULONG_MAX);
+		if (__riscv_xlen == 32)
+			csr_write(CSR_STIMECMPH, ULONG_MAX);
+	}
+}
riscv/sbi.c +0 -5
index 01697aed3457..e8598fe721a6 100644
@@ -258,11 +258,6 @@ static void check_time(void)
 
 	install_irq_handler(IRQ_S_TIMER, timer_irq_handler);
 	local_irq_enable();
-	if (cpu_has_extension(smp_processor_id(), ISA_SSTC)) {
-		csr_write(CSR_STIMECMP, ULONG_MAX);
-		if (__riscv_xlen == 32)
-			csr_write(CSR_STIMECMPH, ULONG_MAX);
-	}
 	timer_irq_enable();
 
 	timer_check_set_timer(false);
Signature · 1 line
2.45.2
#2 lore raw

Re: [kvm-unit-tests PATCH 1/3] riscv: Introduce local_timer_init

Quoted 20 lines · Andrew Jones

When Sstc is available make sure that even if we enable timer interrupts nothing will happen. This is necessary for cases where the unit tests actually intend to use the SBI TIME extension and aren't thinking about Sstc at all, like the SBI TIME test in riscv/sbi where we can now remove the initialization.

---
 lib/riscv/asm/timer.h |  1 +
 lib/riscv/setup.c     |  2 ++
 lib/riscv/smp.c       |  2 ++
 lib/riscv/timer.c     | 13 +++++++++++++
 riscv/sbi.c           |  5 -----
 5 files changed, 18 insertions(+), 5 deletions(-)
lib/riscv/asm/timer.h +0 -0
index b3514d3f6a78..fd12251a3a6b 100644

lib/riscv/asm/timer.h line 9

Quoted 5 lines
lib/riscv/asm/timer.h +1 -0
@@ -5,6 +5,7 @@
 #include <asm/csr.h>
 
 extern void timer_get_frequency(void);
+extern void local_timer_init(void);
 

I've renamed the new function to local_hart_init and put it in processor.c instead of timer.c. This is because going forward there will be other non-timer-related CSRs that need to be set at init time and we can just lump them all together.

Thanks,
drew

#3 lore raw

PATCH 2/3 riscv: Share sbi_time_ecall with framework

In reply to Andrew Jones

Setting timers is a useful thing to do for all types of tests. Not every platform will have Sstc so make the SBI TIME extension available as well.

---
 lib/riscv/asm/sbi.h |  1 +
 lib/riscv/sbi.c     |  5 +++++
 riscv/sbi.c         | 13 ++++---------
 3 files changed, 10 insertions(+), 9 deletions(-)
lib/riscv/asm/sbi.h +1 -0
index a864e268437b..4a35cf38da70 100644
@@ -67,6 +67,7 @@ struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0,
 void sbi_shutdown(void);
 struct sbiret sbi_hart_start(unsigned long hartid, unsigned long entry, unsigned long sp);
 struct sbiret sbi_send_ipi(unsigned long hart_mask, unsigned long hart_mask_base);
+struct sbiret sbi_set_timer(unsigned long stime_value);
 long sbi_probe(int ext);
 
 #endif /* !__ASSEMBLY__ */
lib/riscv/sbi.c +5 -0
index 19d58ab73593..07660e422cbb 100644
@@ -44,6 +44,11 @@ struct sbiret sbi_send_ipi(unsigned long hart_mask, unsigned long hart_mask_base
 	return sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, hart_mask, hart_mask_base, 0, 0, 0, 0);
 }
 
+struct sbiret sbi_set_timer(unsigned long stime_value)
+{
+	return sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, 0, 0, 0, 0);
+}
+
 long sbi_probe(int ext)
 {
 	struct sbiret ret;
riscv/sbi.c +4 -9
index e8598fe721a6..85cb7e589bdc 100644
@@ -35,11 +35,6 @@ static struct sbiret __base_sbi_ecall(int fid, unsigned long arg0)
 	return sbi_ecall(SBI_EXT_BASE, fid, arg0, 0, 0, 0, 0, 0);
 }
 
-static struct sbiret __time_sbi_ecall(unsigned long stime_value)
-{
-	return sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, 0, 0, 0, 0);
-}
-
 static struct sbiret __dbcn_sbi_ecall(int fid, unsigned long arg0, unsigned long arg1, unsigned long arg2)
 {
 	return sbi_ecall(SBI_EXT_DBCN, fid, arg0, arg1, arg2, 0, 0, 0);
@@ -198,7 +193,7 @@ static void timer_irq_handler(struct pt_regs *regs)
 	if (timer_info.mask_timer_irq)
 		timer_irq_disable();
 	else
-		__time_sbi_ecall(ULONG_MAX);
+		sbi_set_timer(ULONG_MAX);
 
 	if (!timer_irq_pending())
 		timer_info.timer_irq_cleared = true;
@@ -217,7 +212,7 @@ static void timer_check_set_timer(bool mask_timer_irq)
 
 	timer_info = (struct timer_info){ .mask_timer_irq = mask_timer_irq };
 	begin = timer_get_cycles();
-	ret = __time_sbi_ecall(begin + d);
+	ret = sbi_set_timer(begin + d);
 
 	report(!ret.error, "set timer%s", mask_test_str);
 	if (ret.error)
@@ -268,10 +263,10 @@ static void check_time(void)
 		report_skip("timer irq enable bit is not writable, skipping mask irq test");
 
 	timer_irq_disable();
-	__time_sbi_ecall(0);
+	sbi_set_timer(0);
 	pending = timer_irq_pending();
 	report(pending, "timer immediately pending by setting timer to 0");
-	__time_sbi_ecall(ULONG_MAX);
+	sbi_set_timer(ULONG_MAX);
 	if (pending)
 		report(!timer_irq_pending(), "pending timer cleared while masked");
 	else
Signature · 1 line
2.45.2
#4 lore raw

PATCH 3/3 riscv: Provide timer_start and timer_stop

In reply to Andrew Jones

For unit tests that need a timer but don't care if they use Sstc or SBI TIME, provide timer_start and timer_stop which will try Sstc first and fallback to SBI TIME.

---
 lib/riscv/asm/timer.h |  2 ++
 lib/riscv/timer.c     | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
lib/riscv/asm/timer.h +2 -0
index fd12251a3a6b..9e790a97bd24 100644
@@ -6,6 +6,8 @@
 
 extern void timer_get_frequency(void);
 extern void local_timer_init(void);
+extern void timer_start(unsigned long duration_us);
+extern void timer_stop(void);
 
 static inline uint64_t timer_get_cycles(void)
 {
lib/riscv/timer.c +33 -0
index 92826d6ec3fe..67fd031ab95f 100644
@@ -6,7 +6,9 @@
 #include <devicetree.h>
 #include <limits.h>
 #include <asm/csr.h>
+#include <asm/delay.h>
 #include <asm/isa.h>
+#include <asm/sbi.h>
 #include <asm/setup.h>
 #include <asm/smp.h>
 #include <asm/timer.h>
@@ -39,3 +41,34 @@ void local_timer_init(void)
 			csr_write(CSR_STIMECMPH, ULONG_MAX);
 	}
 }
+
+void timer_start(unsigned long duration_us)
+{
+	uint64_t next = timer_get_cycles() + usec_to_cycles((uint64_t)duration_us);
+
+	if (cpu_has_extension(smp_processor_id(), ISA_SSTC)) {
+		csr_write(CSR_STIMECMP, (unsigned long)next);
+		if (__riscv_xlen == 32)
+			csr_write(CSR_STIMECMPH, (unsigned long)(next >> 32));
+	} else if (sbi_probe(SBI_EXT_TIME)) {
+		struct sbiret ret = sbi_set_timer(next);
+		assert(ret.error == SBI_SUCCESS);
+		assert(!(next >> 32));
+	} else {
+		assert_msg(false, "No timer to start!");
+	}
+}
+
+void timer_stop(void)
+{
+	if (cpu_has_extension(smp_processor_id(), ISA_SSTC)) {
+		csr_write(CSR_STIMECMP, ULONG_MAX);
+		if (__riscv_xlen == 32)
+			csr_write(CSR_STIMECMPH, ULONG_MAX);
+	} else if (sbi_probe(SBI_EXT_TIME)) {
+		struct sbiret ret = sbi_set_timer(ULONG_MAX);
+		assert(ret.error == SBI_SUCCESS);
+	} else {
+		assert_msg(false, "No timer to stop!");
+	}
+}
Signature · 1 line
2.45.2
#5 lore raw

[kvm-unit-tests PATCH 4/3] riscv: QEMU Sstc timer stop workaround

In reply to Andrew Jones

QEMU doesn't stop a pending timer when UINT64_MAX is written to stimecmp, see QEMU commit ae0edf2188b3 ("target/riscv: No need to re-start QEMU timer when timecmp == UINT64_MAX"). We should probably change that in QEMU, but we need a solution in kvm-unit-tests anyway in order to support older QEMU versions. A bit of an ugly workaround is to simply subtract one from UINT64_MAX, which is still a really big number, but not the exact number QEMU is using to decide it should skip the timer update.

---
 lib/riscv/timer.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)
lib/riscv/timer.c +9 -2
index 67fd031ab95f..28e1626607f7 100644
@@ -62,9 +62,16 @@ void timer_start(unsigned long duration_us)
 void timer_stop(void)
 {
 	if (cpu_has_extension(smp_processor_id(), ISA_SSTC)) {
-		csr_write(CSR_STIMECMP, ULONG_MAX);
+		/*
+		 * Subtract one from ULONG_MAX to workaround QEMU using that
+		 * exact number to decide *not* to update the timer. IOW, if
+		 * we used ULONG_MAX, then we wouldn't stop the timer at all,
+		 * but one less is still a big number ("infinity") and it gets
+		 * QEMU to do what we want.
+		 */
+		csr_write(CSR_STIMECMP, ULONG_MAX - 1);
 		if (__riscv_xlen == 32)
-			csr_write(CSR_STIMECMPH, ULONG_MAX);
+			csr_write(CSR_STIMECMPH, ULONG_MAX - 1);
 	} else if (sbi_probe(SBI_EXT_TIME)) {
 		struct sbiret ret = sbi_set_timer(ULONG_MAX);
 		assert(ret.error == SBI_SUCCESS);
Signature · 1 line
2.45.2
#6 lore raw
In reply to Andrew Jones
Quoted 19 lines · Andrew Jones

We already have some timer / delay support but we leave a lot to the unit test authors for deciding what timer to use (SBI vs. Sstc) and setting it. Provide an API that prefers Sstc and falls back to SBI.

This found a bug in QEMU's Sstc that I'll try to find time to fix. If we start a timer with a long delay and then stop it before it has expired, QEMU still delivers the interrupt. The only way to avoid getting it on QEMU is to disable the timer irq. The BPI, which also has Sstc, behaves as expected though, i.e. even with the timer irq always enabled we won't get a timer irq if we stop it before it had a chance to expire. Disabling Sstc on QEMU, which falls back to SBI TIME, also behaves as expected.

Andrew Jones (3):
riscv: Introduce local_timer_init
riscv: Share sbi_time_ecall with framework
riscv: Provide timer_start and timer_stop

Quoted 12 lines
 lib/riscv/asm/sbi.h   |  1 +
 lib/riscv/asm/timer.h |  3 +++
 lib/riscv/sbi.c       |  5 +++++
 lib/riscv/setup.c     |  2 ++
 lib/riscv/smp.c       |  2 ++
 lib/riscv/timer.c     | 46 +++++++++++++++++++++++++++++++++++++++++++
 riscv/sbi.c           | 18 ++++-------------
 7 files changed, 63 insertions(+), 14 deletions(-)
Signature · 1 line
2.45.2

Queued on riscv/sbi, https://gitlab.com/jones-drew/kvm-unit-tests/-/commits/riscv%2Fsbi