Skip to contentSkip to conversation

riscv: sbi: Add support to test HSM extension

PATCH Superseded James Raphael Tiovalen 2 participants 8 messages Aug 10, 2024 – Aug 12, 2024

Subsystems (3)

v1 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#0 lore raw

cover riscv: sbi: Add support to test HSM extension

This patch series adds support for testing all 4 functions of the HSM extension as defined in the RISC-V SBI specification. The first 2 patches add some helper routines to prepare for the HSM test, while the last patch adds the actual test for the HSM extension.

James Raphael Tiovalen (3):
riscv: sbi: Add IPI extension support
riscv: sbi: Add HSM extension functions
riscv: sbi: Add tests for HSM extension

 riscv/Makefile      |   7 +-
 lib/riscv/asm/sbi.h |  23 ++++
 lib/riscv/sbi.c     |  15 +++
 riscv/sbi-asm.S     |  38 ++++++
 riscv/sbi.c         | 285 ++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 365 insertions(+), 3 deletions(-)
 create mode 100644 riscv/sbi-asm.S

--
2.43.0

#1 lore raw

PATCH 1/3 riscv: sbi: Add IPI extension support

Add IPI EID and FID constants and a helper function to perform the IPI SBI ecall.

---
 lib/riscv/asm/sbi.h | 5 +++++
 riscv/sbi.c         | 5 +++++
 2 files changed, 10 insertions(+)
lib/riscv/asm/sbi.h +5 -0
index 73ab5438..6b485dd3 100644
@@ -17,6 +17,7 @@
 enum sbi_ext_id {
 	SBI_EXT_BASE = 0x10,
 	SBI_EXT_TIME = 0x54494d45,
+	SBI_EXT_IPI = 0x735049,
 	SBI_EXT_HSM = 0x48534d,
 	SBI_EXT_SRST = 0x53525354,
 };
@@ -42,6 +43,10 @@ enum sbi_ext_time_fid {
 	SBI_EXT_TIME_SET_TIMER = 0,
 };
 
+enum sbi_ext_ipi_fid {
+	SBI_EXT_IPI_SEND_IPI = 0,
+};
+
 struct sbiret {
 	long error;
 	long value;
riscv/sbi.c +5 -0
index 2438c497..08bd6a95 100644
@@ -32,6 +32,11 @@ static struct sbiret __time_sbi_ecall(unsigned long stime_value)
 	return sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, 0, 0, 0, 0);
 }
 
+static struct sbiret __ipi_sbi_ecall(unsigned long hart_mask, unsigned long hart_mask_base)
+{
+	return sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, hart_mask, hart_mask_base, 0, 0, 0, 0);
+}
+
 static bool env_or_skip(const char *env)
 {
 	if (!getenv(env)) {
Signature · 1 line
2.43.0
#2 lore raw

Re: [kvm-unit-tests PATCH 1/3] riscv: sbi: Add IPI extension support

Quoted 37 lines · James Raphael Tiovalen

Add IPI EID and FID constants and a helper function to perform the IPI SBI ecall.

---
 lib/riscv/asm/sbi.h | 5 +++++
 riscv/sbi.c         | 5 +++++
 2 files changed, 10 insertions(+)
lib/riscv/asm/sbi.h +5 -0
index 73ab5438..6b485dd3 100644
@@ -17,6 +17,7 @@
 enum sbi_ext_id {
 	SBI_EXT_BASE = 0x10,
 	SBI_EXT_TIME = 0x54494d45,
+	SBI_EXT_IPI = 0x735049,
 	SBI_EXT_HSM = 0x48534d,
 	SBI_EXT_SRST = 0x53525354,
 };
@@ -42,6 +43,10 @@ enum sbi_ext_time_fid {
 	SBI_EXT_TIME_SET_TIMER = 0,
 };
 
+enum sbi_ext_ipi_fid {
+	SBI_EXT_IPI_SEND_IPI = 0,
+};
+
 struct sbiret {
 	long error;
 	long value;
riscv/sbi.c +0 -0
index 2438c497..08bd6a95 100644

riscv/sbi.c static struct sbiret __time_sbi_ecall(unsigned long stime_value) · line 38

Quoted 7 lines
riscv/sbi.c +4 -0
@@ -32,6 +32,11 @@ static struct sbiret __time_sbi_ecall(unsigned long stime_value)
 	return sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, 0, 0, 0, 0);
 }
 
+static struct sbiret __ipi_sbi_ecall(unsigned long hart_mask, unsigned long hart_mask_base)
+{
+	return sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, hart_mask, hart_mask_base, 0, 0, 0, 0);
+}

This will likely come in handy for other tests. Let's add it with the name sbi_send_ipi() to lib/riscv/sbi.c instead.

Thanks,
drew

Quoted 12 lines
riscv/sbi.c +1 -0
+
 static bool env_or_skip(const char *env)
 {
 	if (!getenv(env)) {
Signature · 7 lines
2.43.0


-- 
kvm-riscv mailing list
kvm-riscv at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/kvm-riscv
#3 lore raw

PATCH 2/3 riscv: sbi: Add HSM extension functions

In reply to James Raphael Tiovalen

Add helper functions to perform hart-related operations to prepare for the HSM tests. Also add the HSM state IDs and default suspend type constants.

---
 lib/riscv/asm/sbi.h | 18 ++++++++++++++++++
 lib/riscv/sbi.c     | 15 +++++++++++++++
 2 files changed, 33 insertions(+)
lib/riscv/asm/sbi.h +18 -0
index 6b485dd3..5cebc4d9 100644
@@ -47,6 +47,21 @@ enum sbi_ext_ipi_fid {
 	SBI_EXT_IPI_SEND_IPI = 0,
 };
 
+enum sbi_ext_hsm_sid {
+	SBI_EXT_HSM_STARTED = 0,
+	SBI_EXT_HSM_STOPPED,
+	SBI_EXT_HSM_START_PENDING,
+	SBI_EXT_HSM_STOP_PENDING,
+	SBI_EXT_HSM_SUSPENDED,
+	SBI_EXT_HSM_SUSPEND_PENDING,
+	SBI_EXT_HSM_RESUME_PENDING,
+};
+
+enum sbi_ext_hsm_hart_suspend_type {
+	SBI_EXT_HSM_HART_SUSPEND_RETENTIVE = 0,
+	SBI_EXT_HSM_HART_SUSPEND_NON_RETENTIVE = 0x80000000,
+};
+
 struct sbiret {
 	long error;
 	long value;
@@ -59,6 +74,9 @@ struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0,
 
 void sbi_shutdown(void);
 struct sbiret sbi_hart_start(unsigned long hartid, unsigned long entry, unsigned long sp);
+struct sbiret sbi_hart_stop(void);
+struct sbiret sbi_hart_get_status(unsigned long hartid);
+struct sbiret sbi_hart_suspend(uint32_t suspend_type, unsigned long resume_addr, unsigned long opaque);
 long sbi_probe(int ext);
 
 #endif /* !__ASSEMBLY__ */
lib/riscv/sbi.c +15 -0
index 3d4236e5..a5c18450 100644
@@ -39,6 +39,21 @@ struct sbiret sbi_hart_start(unsigned long hartid, unsigned long entry, unsigned
 	return sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_START, hartid, entry, sp, 0, 0, 0);
 }
 
+struct sbiret sbi_hart_stop(void)
+{
+	return sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STOP, 0, 0, 0, 0, 0, 0);
+}
+
+struct sbiret sbi_hart_get_status(unsigned long hartid)
+{
+	return sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STATUS, hartid, 0, 0, 0, 0, 0);
+}
+
+struct sbiret sbi_hart_suspend(uint32_t suspend_type, unsigned long resume_addr, unsigned long opaque)
+{
+	return sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_SUSPEND, suspend_type, resume_addr, opaque, 0, 0, 0);
+}
+
 long sbi_probe(int ext)
 {
 	struct sbiret ret;
Signature · 1 line
2.43.0
#4 lore raw

Re: [kvm-unit-tests PATCH 2/3] riscv: sbi: Add HSM extension functions

Quoted 58 lines · James Raphael Tiovalen

Add helper functions to perform hart-related operations to prepare for the HSM tests. Also add the HSM state IDs and default suspend type constants.

---
 lib/riscv/asm/sbi.h | 18 ++++++++++++++++++
 lib/riscv/sbi.c     | 15 +++++++++++++++
 2 files changed, 33 insertions(+)
lib/riscv/asm/sbi.h +18 -0
index 6b485dd3..5cebc4d9 100644
@@ -47,6 +47,21 @@ enum sbi_ext_ipi_fid {
 	SBI_EXT_IPI_SEND_IPI = 0,
 };
 
+enum sbi_ext_hsm_sid {
+	SBI_EXT_HSM_STARTED = 0,
+	SBI_EXT_HSM_STOPPED,
+	SBI_EXT_HSM_START_PENDING,
+	SBI_EXT_HSM_STOP_PENDING,
+	SBI_EXT_HSM_SUSPENDED,
+	SBI_EXT_HSM_SUSPEND_PENDING,
+	SBI_EXT_HSM_RESUME_PENDING,
+};
+
+enum sbi_ext_hsm_hart_suspend_type {
+	SBI_EXT_HSM_HART_SUSPEND_RETENTIVE = 0,
+	SBI_EXT_HSM_HART_SUSPEND_NON_RETENTIVE = 0x80000000,
+};
+
 struct sbiret {
 	long error;
 	long value;
@@ -59,6 +74,9 @@ struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0,
 
 void sbi_shutdown(void);
 struct sbiret sbi_hart_start(unsigned long hartid, unsigned long entry, unsigned long sp);
+struct sbiret sbi_hart_stop(void);
+struct sbiret sbi_hart_get_status(unsigned long hartid);
+struct sbiret sbi_hart_suspend(uint32_t suspend_type, unsigned long resume_addr, unsigned long opaque);
 long sbi_probe(int ext);
 
 #endif /* !__ASSEMBLY__ */
lib/riscv/sbi.c +4 -0
index 3d4236e5..a5c18450 100644
@@ -39,6 +39,21 @@ struct sbiret sbi_hart_start(unsigned long hartid, unsigned long entry, unsigned
 	return sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_START, hartid, entry, sp, 0, 0, 0);
 }
 
+struct sbiret sbi_hart_stop(void)
+{
+	return sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STOP, 0, 0, 0, 0, 0, 0);
+}

lib/riscv/sbi.c struct sbiret sbi_hart_start(unsigned long hartid, unsigned long entry, unsigned · line 55

Quoted 10 lines
lib/riscv/sbi.c +10 -0
+
+struct sbiret sbi_hart_get_status(unsigned long hartid)
+{
+	return sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STATUS, hartid, 0, 0, 0, 0, 0);
+}
+
+struct sbiret sbi_hart_suspend(uint32_t suspend_type, unsigned long resume_addr, unsigned long opaque)
+{
+	return sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_SUSPEND, suspend_type, resume_addr, opaque, 0, 0, 0);
+}

Let's put sbi_hart_suspend in riscv/sbi.c for now. I can't think of why that would useful outside of the HSM tests. stop and get-status are good to have in the library though.

Thanks,
drew

Quoted 12 lines
lib/riscv/sbi.c +1 -0
+
 long sbi_probe(int ext)
 {
 	struct sbiret ret;
Signature · 7 lines
2.43.0


-- 
kvm-riscv mailing list
kvm-riscv at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/kvm-riscv
#5 lore raw

PATCH 3/3 riscv: sbi: Add tests for HSM extension

In reply to James Raphael Tiovalen

Add some tests for all of the HSM extension functions.

---
 riscv/Makefile  |   7 +-
 riscv/sbi-asm.S |  38 +++++++
 riscv/sbi.c     | 280 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 322 insertions(+), 3 deletions(-)
 create mode 100644 riscv/sbi-asm.S
riscv/Makefile +4 -3
index b0cd613f..c0fd8684 100644
@@ -21,6 +21,7 @@ all: $(tests)
 $(TEST_DIR)/sieve.$(exe): AUXFLAGS = 0x1
 
 cstart.o = $(TEST_DIR)/cstart.o
+sbi-asm.o = $(TEST_DIR)/sbi-asm.o
 
 cflatobjs += lib/alloc.o
 cflatobjs += lib/alloc_page.o
@@ -97,7 +98,7 @@ cflatobjs += lib/efi.o
 .PRECIOUS: %.so
 
 %.so: EFI_LDFLAGS += -defsym=EFI_SUBSYSTEM=0xa --no-undefined
-%.so: %.o $(FLATLIBS) $(SRCDIR)/riscv/efi/elf_riscv64_efi.lds $(cstart.o) %.aux.o
+%.so: %.o $(FLATLIBS) $(SRCDIR)/riscv/efi/elf_riscv64_efi.lds $(cstart.o) $(sbi-asm.o) %.aux.o
 	$(LD) $(EFI_LDFLAGS) -o $@ -T $(SRCDIR)/riscv/efi/elf_riscv64_efi.lds \
 		$(filter %.o, $^) $(FLATLIBS) $(EFI_LIBS)
 
@@ -113,7 +114,7 @@ cflatobjs += lib/efi.o
 		-O binary $^ $@
 else
 %.elf: LDFLAGS += -pie -n -z notext
-%.elf: %.o $(FLATLIBS) $(SRCDIR)/riscv/flat.lds $(cstart.o) %.aux.o
+%.elf: %.o $(FLATLIBS) $(SRCDIR)/riscv/flat.lds $(cstart.o) $(sbi-asm.o) %.aux.o
 	$(LD) $(LDFLAGS) -o $@ -T $(SRCDIR)/riscv/flat.lds \
 		$(filter %.o, $^) $(FLATLIBS)
 	@chmod a-x $@
@@ -125,7 +126,7 @@ else
 endif
 
 generated-files = $(asm-offsets)
-$(tests:.$(exe)=.o) $(cstart.o) $(cflatobjs): $(generated-files)
+$(tests:.$(exe)=.o) $(cstart.o) $(sbi-asm.o) $(cflatobjs): $(generated-files)
 
 arch_clean: asm_offsets_clean
 	$(RM) $(TEST_DIR)/*.{o,flat,elf,so,efi,debug} \
riscv/sbi-asm.S new file +38 -0
new file mode 100644
index 00000000..6d348c88
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Helper assembly code routines for RISC-V SBI extension tests.
+ *
+ * Copyright (C) 2024, James Raphael Tiovalen <[email protected]>
+ */
+#define __ASSEMBLY__
+#include <asm/asm.h>
+#include <asm/asm-offsets.h>
+#include <asm/csr.h>
+#include <asm/page.h>
+
+.section .text
+.balign 4
+.global check_hart_start
+check_hart_start:
+	csrr t0, CSR_SATP
+	bnez t0, hart_start_checks_failed
+	csrr t0, CSR_SSTATUS
+	andi t1, t0, SR_SIE
+	bnez t1, hart_start_checks_failed
+	bne a0, a1, hart_start_checks_failed
+	la t0, hart_start_works
+	li t1, 1
+	sb t1, 0(t0)
+hart_start_checks_failed:
+	la t0, stop_test_hart
+	lb t1, 0(t0)
+	beqz t1, hart_start_checks_failed
+	call sbi_hart_stop
+	j halt
+
+.section .data
+.balign PAGE_SIZE
+.global hart_start_works
+hart_start_works:	.byte 0
+.global stop_test_hart
+stop_test_hart:		.byte 0
riscv/sbi.c +280 -0

304 lines not on this page · raw message

Signature · 1 line
2.43.0
#6 lore raw

Re: [kvm-unit-tests PATCH 3/3] riscv: sbi: Add tests for HSM extension

Quoted 15 lines · James Raphael Tiovalen

Add some tests for all of the HSM extension functions.

---
 riscv/Makefile  |   7 +-
 riscv/sbi-asm.S |  38 +++++++
 riscv/sbi.c     | 280 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 322 insertions(+), 3 deletions(-)
 create mode 100644 riscv/sbi-asm.S
riscv/Makefile +0 -0
index b0cd613f..c0fd8684 100644

riscv/Makefile all: $(tests) · line 24

Quoted 4 lines
riscv/Makefile +1 -0
@@ -21,6 +21,7 @@ all: $(tests)
 $(TEST_DIR)/sieve.$(exe): AUXFLAGS = 0x1
 
 cstart.o = $(TEST_DIR)/cstart.o
+sbi-asm.o = $(TEST_DIR)/sbi-asm.o

We should be able to just add sbi-asm.o to cflatobjs. It doesn't need special treatment.

Quoted 36 lines
riscv/Makefile +3 -3
 cflatobjs += lib/alloc.o
 cflatobjs += lib/alloc_page.o
@@ -97,7 +98,7 @@ cflatobjs += lib/efi.o
 .PRECIOUS: %.so
 
 %.so: EFI_LDFLAGS += -defsym=EFI_SUBSYSTEM=0xa --no-undefined
-%.so: %.o $(FLATLIBS) $(SRCDIR)/riscv/efi/elf_riscv64_efi.lds $(cstart.o) %.aux.o
+%.so: %.o $(FLATLIBS) $(SRCDIR)/riscv/efi/elf_riscv64_efi.lds $(cstart.o) $(sbi-asm.o) %.aux.o
 	$(LD) $(EFI_LDFLAGS) -o $@ -T $(SRCDIR)/riscv/efi/elf_riscv64_efi.lds \
 		$(filter %.o, $^) $(FLATLIBS) $(EFI_LIBS)
 
@@ -113,7 +114,7 @@ cflatobjs += lib/efi.o
 		-O binary $^ $@
 else
 %.elf: LDFLAGS += -pie -n -z notext
-%.elf: %.o $(FLATLIBS) $(SRCDIR)/riscv/flat.lds $(cstart.o) %.aux.o
+%.elf: %.o $(FLATLIBS) $(SRCDIR)/riscv/flat.lds $(cstart.o) $(sbi-asm.o) %.aux.o
 	$(LD) $(LDFLAGS) -o $@ -T $(SRCDIR)/riscv/flat.lds \
 		$(filter %.o, $^) $(FLATLIBS)
 	@chmod a-x $@
@@ -125,7 +126,7 @@ else
 endif
 
 generated-files = $(asm-offsets)
-$(tests:.$(exe)=.o) $(cstart.o) $(cflatobjs): $(generated-files)
+$(tests:.$(exe)=.o) $(cstart.o) $(sbi-asm.o) $(cflatobjs): $(generated-files)
 
 arch_clean: asm_offsets_clean
 	$(RM) $(TEST_DIR)/*.{o,flat,elf,so,efi,debug} \
riscv/sbi-asm.S new file +0 -0
new file mode 100644
index 00000000..6d348c88

riscv/sbi-asm.S line 9

Quoted 9 lines
riscv/sbi-asm.S new file +9 -0
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Helper assembly code routines for RISC-V SBI extension tests.
+ *
+ * Copyright (C) 2024, James Raphael Tiovalen <[email protected]>
+ */
+#define __ASSEMBLY__
+#include <asm/asm.h>
+#include <asm/asm-offsets.h>

Don't need to include asm-offsets.h

riscv/sbi-asm.S line 11

Quoted 2 lines
riscv/sbi-asm.S new file +2 -0
+#include <asm/csr.h>
+#include <asm/page.h>

Don't need to include page.h

Quoted 9 lines
riscv/sbi-asm.S new file +9 -0
+
+.section .text
+.balign 4
+.global check_hart_start
+check_hart_start:
+	csrr t0, CSR_SATP
+	bnez t0, hart_start_checks_failed
+	csrr t0, CSR_SSTATUS
+	andi t1, t0, SR_SIE

riscv/sbi-asm.S line 30

Quoted 10 lines
riscv/sbi-asm.S new file +10 -0
+	bnez t1, hart_start_checks_failed
+	bne a0, a1, hart_start_checks_failed
+	la t0, hart_start_works
+	li t1, 1
+	sb t1, 0(t0)
+hart_start_checks_failed:
+	la t0, stop_test_hart
+	lb t1, 0(t0)
+	beqz t1, hart_start_checks_failed
+	call sbi_hart_stop

We can't call C code unless we setup the stack.

riscv/sbi-asm.S line 31

Quoted 1 line
riscv/sbi-asm.S new file +1 -0
+	j halt

I think we want a variable named 'sbi_hsm_hart_start_checks' which is a bitmap, where each bit represents the result of a single test. Something like

#define SBI_HSM_START_TEST_DONE       (1 << 0)
#define SBI_HSM_START_TEST_SATP       (1 << 1)
#define SBI_HSM_START_TEST_SIE        (1 << 2)
#define SBI_HSM_START_TEST_HARTID_A1  (1 << 4)

sbi_hsm_check_hart_start:

    li     t0, 0
    csrr   t1, CSR_SATP
    bnez   t1, 1f
    li     t0, SBI_HSM_START_TEST_SATP
1:  csrr   t1, CSR_STATUS
    andi   t1, t1, SR_SIE
    bnez   t1, 2f
    orr    t0, t0, SBI_HSM_START_TEST_SIE
2:  bne    a0, a1, 3f
    orr    t0, t0, SBI_HSM_START_TEST_HARTID_A1
3:  orr    t0, t0, SBI_HSM_START_TEST_DONE
    la     t1, sbi_hsm_hart_start_checks
    REG_S  t0, 0(t1)
4:  la     t0, sbi_hsm_stop_hart
    REG_L  t0, 0(t0)
    pause
    beqz   t0, 4b
    li     a7, 0x48534d		/* HSM extension ID */
    li     a6, 1		/* hart stop FID */
    ecall
    j      halt
Quoted 20 lines
riscv/sbi-asm.S new file +7 -0
+
+.section .data
+.balign PAGE_SIZE
+.global hart_start_works
+hart_start_works:	.byte 0
+.global stop_test_hart
+stop_test_hart:		.byte 0
riscv/sbi.c +1 -0
index 08bd6a95..53986c9e 100644
@@ -5,6 +5,7 @@
  * Copyright (C) 2023, Ventana Micro Systems Inc., Andrew Jones <[email protected]>
  */
 #include <libcflat.h>
+#include <cpumask.h>
 #include <stdlib.h>
 #include <limits.h>
 #include <asm/barrier.h>

riscv/sbi.c line 21

Quoted 6 lines
riscv/sbi.c +3 -0
@@ -15,6 +16,9 @@
 #include <asm/sbi.h>
 #include <asm/smp.h>
 #include <asm/timer.h>
+#include <asm/io.h>
+#include <asm/page.h>
+#include <asm/setup.h>

nit: prefer alphabetic order unless order matters (but we should fix it if order does matter, except for libcflat.h needing to be first)

riscv/sbi.c static void check_time(void) · line 261

Quoted 9 lines
riscv/sbi.c +2 -0
 static void help(void)
 {
@@ -253,6 +257,281 @@ static void check_time(void)
 	report_prefix_pop();
 }
 
+struct hsm_info {
+	bool stages[2];

Please write a comment explaining the stages of the test(s).

Quoted 8 lines
riscv/sbi.c +8 -0
+	bool retentive_suspend_hart;
+	bool stop_hart;
+};
+
+static struct hsm_info hsm_info[NR_CPUS];
+extern void check_hart_start(void);
+extern unsigned char stop_test_hart;
+extern unsigned char hart_start_works;

riscv/sbi.c static void check_time(void) · line 279

Quoted 10 lines
riscv/sbi.c +10 -0
+
+static void hart_execute(void)
+{
+	struct sbiret ret;
+	unsigned long hartid = current_thread_info()->hartid;
+
+	hsm_info[hartid].stages[0] = true;
+
+	while (true) {
+		if (hsm_info[hartid].retentive_suspend_hart) {

We need to use READ_ONCE() when using data in this way to ensure the compiler doesn't generate code that either loops forever or never loops.

riscv/sbi.c static void check_time(void) · line 286

Quoted 7 lines
riscv/sbi.c +7 -0
+			hsm_info[hartid].retentive_suspend_hart = false;
+			ret = sbi_hart_suspend(SBI_EXT_HSM_HART_SUSPEND_RETENTIVE, __pa(NULL), __pa(NULL));
+			if (ret.error)
+				report_fail("failed to retentive suspend hart %ld", hartid);
+			else
+				hsm_info[hartid].stages[1] = true;
+

nit: stray blank line

riscv/sbi.c static void check_time(void) · line 287

Quoted 1 line
riscv/sbi.c +1 -0
+		} else if (hsm_info[hartid].stop_hart) {

Also needs READ_ONCE

Why all the separate booleans for this state machine instead of enum with different command IDs.

riscv/sbi.c static void check_time(void) · line 296

Quoted 9 lines
riscv/sbi.c +9 -0
+			break;
+		} else {
+			cpu_relax();
+		}
+	}
+
+	ret = sbi_hart_stop();
+	if (ret.error)
+		report_fail("failed to stop hart %ld", hartid);

If we failed to stop we should call halt() here.

Quoted 16 lines
riscv/sbi.c +16 -0
+}
+
+static void check_hsm(void)
+{
+	struct sbiret ret;
+	unsigned long hartid;
+	int cpu;
+	unsigned long hart_mask = 0;
+	bool ipi_failed = false;
+	unsigned int stage = 0;
+
+	report_prefix_push("hsm");
+
+	if (!sbi_probe(SBI_EXT_HSM)) {
+		report_skip("hsm extension not available");
+		report_prefix_pop();

riscv/sbi.c static void check_time(void) · line 322

Quoted 10 lines
riscv/sbi.c +10 -0
+		return;
+	}
+
+	report_prefix_push("hart_get_status");
+
+	hartid = current_thread_info()->hartid;
+	ret = sbi_hart_get_status(hartid);
+
+	if (ret.error) {
+		report_fail("current hartid is invalid");

Need to check/output error, not assume that it's invalid-param

riscv/sbi.c static void check_time(void) · line 324

Quoted 2 lines
riscv/sbi.c +2 -0
+		report_prefix_pop();
+		report_prefix_pop();

As a separate patch, we should add a

void report_prefix_popn(int n)

 {
	while (n--)
		report_prefix_pop();

}

and / or

void report_prefix_clear(void)
{

 	spin_lock(&lock);
	prefixes[0] = '\0';
	spin_unlock(&lock);

}

function to lib/report.c and then use it in places like this.

Quoted 26 lines
riscv/sbi.c +26 -0
+		return;
+	} else if (ret.value != SBI_EXT_HSM_STARTED) {
+		report_fail("current hart is not started");
+		report_prefix_pop();
+		report_prefix_pop();
+		return;
+	}
+
+	report_pass("status of current hart is started");
+
+	report_prefix_pop();
+
+	report_prefix_push("hart_start");
+
+	ret = sbi_hart_start(hartid, virt_to_phys(&hart_execute), __pa(NULL));
+	report(ret.error == SBI_ERR_ALREADY_AVAILABLE, "boot hart is already started");
+
+	ret = sbi_hart_start(ULONG_MAX, virt_to_phys(&hart_execute), __pa(NULL));
+	report(ret.error == SBI_ERR_INVALID_PARAM, "invalid hartid check");
+
+	if (nr_cpus < 2) {
+		report_skip("no other cpus to run the remaining hsm tests on");
+		report_prefix_pop();
+		report_prefix_pop();
+		return;
+	}

riscv/sbi.c static void check_time(void) · line 360

Quoted 10 lines
riscv/sbi.c +10 -0
+
+	for_each_present_cpu(cpu) {
+		if (cpu != smp_processor_id()) {
+			hartid = cpus[cpu].hartid;
+			break;
+		}
+	}
+
+	ret = sbi_hart_start(hartid, virt_to_phys(&check_hart_start), hartid);
+

nit: remove this blank line

riscv/sbi.c static void check_time(void) · line 362

Quoted 2 lines
riscv/sbi.c +2 -0
+	if (ret.error) {
+		report_fail("failed to start test hart");

output the error

riscv/sbi.c static void check_time(void) · line 370

Quoted 8 lines
riscv/sbi.c +8 -0
+		report_prefix_pop();
+		report_prefix_pop();
+		return;
+	}
+
+	ret = sbi_hart_get_status(hartid);
+
+	while (!ret.error && (ret.value == SBI_EXT_HSM_STOPPED))

nit: unnecessary (), same comment for similar conditions below

riscv/sbi.c static void check_time(void) · line 371

Quoted 1 line
riscv/sbi.c +1 -0
+		ret = sbi_hart_get_status(hartid);

Need to check if we broke out of the loop due to error here and report the error if we did. I just a helper function to use for all these

static void hart_wait_on_status(unsigned long hartid, enum sbi_ext_hsm_sid status)

 {
    sbiret ret = sbi_hart_get_status(hartid);
    while (!ret.error && ret.value == status)
        ret = sbi_hart_get_status(hartid);
    if (ret.error)
        report_fail("got %ld while waiting on status %u for hart %lx\n", ret.error, status, hartid);

}

riscv/sbi.c static void check_time(void) · line 379

Quoted 8 lines
riscv/sbi.c +8 -0
+
+	while (!ret.error && (ret.value == SBI_EXT_HSM_START_PENDING))
+		ret = sbi_hart_get_status(hartid);
+
+	report(!ret.error && (ret.value == SBI_EXT_HSM_STARTED),
+	       "test hart with hartid %ld successfully started", hartid);
+
+	while (!hart_start_works)

while (!(READ_ONCE(sbi_hsm_hart_start_checks) & SBI_HSM_START_TEST_DONE))

riscv/sbi.c static void check_time(void) · line 383

Quoted 4 lines
riscv/sbi.c +4 -0
+		cpu_relax();
+
+	report(hart_start_works,
+	       "test hart %ld successfully executed code", hartid);

report(sbi_hsm_hart_start_checks & SBI_HSM_START_TEST_SATP, "satp is zero"); report(sbi_hsm_hart_start_checks & ... report(sbi_hsm_hart_start_checks & ...

riscv/sbi.c static void check_time(void) · line 385

Quoted 2 lines
riscv/sbi.c +2 -0
+
+	stop_test_hart = true;

WRITE_ONCE(sbi_hsm_stop_hart, 1);

Quoted 21 lines
riscv/sbi.c +21 -0
+
+	ret = sbi_hart_get_status(hartid);
+
+	while (!ret.error && (ret.value == SBI_EXT_HSM_STARTED))
+		ret = sbi_hart_get_status(hartid);
+
+	while (!ret.error && (ret.value == SBI_EXT_HSM_STOP_PENDING))
+		ret = sbi_hart_get_status(hartid);
+
+	report(!ret.error && (ret.value == SBI_EXT_HSM_STOPPED),
+	       "test hart %ld successfully stopped", hartid);
+
+	for_each_present_cpu(cpu) {
+		if (cpu != smp_processor_id())
+			smp_boot_secondary(cpu, hart_execute);
+	}
+	for_each_present_cpu(cpu) {
+		if (cpu != smp_processor_id()) {
+			hartid = cpus[cpu].hartid;
+			ret = sbi_hart_get_status(hartid);
+

riscv/sbi.c static void check_time(void) · line 417

Quoted 10 lines
riscv/sbi.c +10 -0
+			while (!ret.error && (ret.value == SBI_EXT_HSM_STOPPED))
+				ret = sbi_hart_get_status(hartid);
+
+			while (!ret.error && (ret.value == SBI_EXT_HSM_START_PENDING))
+				ret = sbi_hart_get_status(hartid);
+
+			report(!ret.error && (ret.value == SBI_EXT_HSM_STARTED),
+			       "new hart with hartid %ld successfully started", hartid);
+		}
+	}

Instead of the two for loops, we can use on_cpus(hart_execute, NULL) and then wait on a cpumask for each to report that they've entered hart_execute and then check their HSM status. (See riscv/selftest.c for some inspiration.)

riscv/sbi.c static void check_time(void) · line 424

Quoted 8 lines
riscv/sbi.c +8 -0
+
+
+	for_each_present_cpu(cpu) {
+		if (cpu != smp_processor_id()) {
+			hartid = cpus[cpu].hartid;
+
+			while (!hsm_info[hartid].stages[stage])
+				cpu_relax();

You probably want cpumasks instead of these arrays. You won't need READ_ONCE with cpumasks.

Quoted 5 lines
riscv/sbi.c +5 -0
+
+			report(hsm_info[hartid].stages[stage],
+			       "hart %ld successfully executed stage %d code", hartid, stage + 1);
+		}
+	}

riscv/sbi.c static void check_time(void) · line 439

Quoted 10 lines
riscv/sbi.c +10 -0
+
+	stage++;
+
+	report_prefix_pop();
+
+	report_prefix_push("hart_suspend");
+
+	if (sbi_probe(SBI_EXT_IPI)) {
+		for_each_present_cpu(cpu) {
+			if (cpu != smp_processor_id()) {

Reduce indentation and CSR reads by saving smp_processor_id() once at the top of this function into a variable, e.g. 'me', and then using continue in the loops

   for_each_present_cpu(cpu) {
       if (cpu == me)
          continue;
       ...
   }
Quoted 13 lines
riscv/sbi.c +13 -0
+				hartid = cpus[cpu].hartid;
+				hsm_info[hartid].retentive_suspend_hart = true;
+				hart_mask |= 1UL << hartid;
+			}
+		}
+
+		for_each_present_cpu(cpu) {
+			if (cpu != smp_processor_id()) {
+				hartid = cpus[cpu].hartid;
+				ret = sbi_hart_get_status(hartid);
+
+				while (!ret.error && (ret.value == SBI_EXT_HSM_STARTED))
+					ret = sbi_hart_get_status(hartid);

riscv/sbi.c static void check_time(void) · line 462

Quoted 10 lines
riscv/sbi.c +10 -0
+
+				while (!ret.error && (ret.value == SBI_EXT_HSM_SUSPEND_PENDING))
+					ret = sbi_hart_get_status(hartid);
+
+				report(!ret.error && (ret.value == SBI_EXT_HSM_SUSPENDED),
+				       "hart %ld successfully retentive suspended", hartid);
+			}
+		}
+
+		ret = __ipi_sbi_ecall(hart_mask, 0UL);

no need for the UL

riscv/sbi.c static void check_time(void) · line 465

Quoted 3 lines
riscv/sbi.c +3 -0
+		if (ret.error) {
+			ipi_failed = true;
+			report_fail("failed to send ipi to retentive suspended harts");

output the error number

Quoted 22 lines
riscv/sbi.c +22 -0
+		} else {
+			for_each_present_cpu(cpu) {
+				if (cpu != smp_processor_id()) {
+					hartid = cpus[cpu].hartid;
+					ret = sbi_hart_get_status(hartid);
+
+					while (!ret.error && (ret.value == SBI_EXT_HSM_SUSPENDED))
+						ret = sbi_hart_get_status(hartid);
+
+					while (!ret.error && (ret.value == SBI_EXT_HSM_RESUME_PENDING))
+						ret = sbi_hart_get_status(hartid);
+
+					report(!ret.error && (ret.value == SBI_EXT_HSM_STARTED),
+					       "hart %ld successfully retentive resumed", hartid);
+				}
+			}
+
+			for_each_present_cpu(cpu) {
+				if (cpu != smp_processor_id()) {
+					hartid = cpus[cpu].hartid;
+
+					while (!hsm_info[hartid].stages[stage])

riscv/sbi.c static void check_time(void) · line 497

Quoted 10 lines
riscv/sbi.c +10 -0
+						cpu_relax();
+
+					report(hsm_info[hartid].stages[stage],
+					       "hart %ld successfully executed stage %d code",
+					       hartid, stage + 1);
+				}
+			}
+		}
+	} else {
+		report_skip("skipping tests since ipi extension is unavailable");

^ suspension

Quoted 22 lines
riscv/sbi.c +22 -0
+	}
+
+	report_prefix_pop();
+
+	report_prefix_push("hart_stop");
+
+	if (!ipi_failed) {
+		for_each_present_cpu(cpu) {
+			if (cpu != smp_processor_id()) {
+				hartid = cpus[cpu].hartid;
+				hsm_info[hartid].stop_hart = true;
+			}
+		}
+
+		for_each_present_cpu(cpu) {
+			if (cpu != smp_processor_id()) {
+				hartid = cpus[cpu].hartid;
+				ret = sbi_hart_get_status(hartid);
+
+				while (!ret.error && (ret.value == SBI_EXT_HSM_STARTED))
+					ret = sbi_hart_get_status(hartid);
+

riscv/sbi.c static void check_time(void) · line 529

Quoted 10 lines
riscv/sbi.c +10 -0
+				while (!ret.error && (ret.value == SBI_EXT_HSM_STOP_PENDING))
+					ret = sbi_hart_get_status(hartid);
+
+				report(!ret.error && (ret.value == SBI_EXT_HSM_STOPPED),
+				       "hart %ld successfully stopped", hartid);
+			}
+		}
+	} else {
+		report_skip("skipping tests since ipi failed to be sent");
+	}

What about non-retentive suspend tests?a

I think we can model these tests by breaking them into functions and then using on_cpus() for each rather than have the stages stuff.

Thanks,
drew

Quoted 24 lines
riscv/sbi.c +6 -0
+
+	report_prefix_pop();
+	report_prefix_pop();
+}
+
 int main(int argc, char **argv)
 {
 
@@ -264,6 +543,7 @@ int main(int argc, char **argv)
 	report_prefix_push("sbi");
 	check_base();
 	check_time();
+	check_hsm();
 
 	return report_summary();
 }
Signature · 7 lines
2.43.0


-- 
kvm-riscv mailing list
kvm-riscv at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/kvm-riscv
#7 lore raw
In reply to James Raphael Tiovalen
Quoted 18 lines · James Raphael Tiovalen

This patch series adds support for testing all 4 functions of the HSM extension as defined in the RISC-V SBI specification. The first 2 patches add some helper routines to prepare for the HSM test, while the last patch adds the actual test for the HSM extension.

James Raphael Tiovalen (3):
riscv: sbi: Add IPI extension support
riscv: sbi: Add HSM extension functions
riscv: sbi: Add tests for HSM extension

 riscv/Makefile      |   7 +-
 lib/riscv/asm/sbi.h |  23 ++++
 lib/riscv/sbi.c     |  15 +++
 riscv/sbi-asm.S     |  38 ++++++
 riscv/sbi.c         | 285 ++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 365 insertions(+), 3 deletions(-)
 create mode 100644 riscv/sbi-asm.S
Quoted 3 lines

--
2.43.0

Hi James,

When you send v2, please base on riscv/sbi, https://gitlab.com/jones-drew/kvm-unit-tests/-/commits/riscv%2Fsbi

Thanks,
drew