riscv: introduce basic UART support and interrupts for hypervisor mode
cover riscv: introduce basic UART support and interrupts for hypervisor mode
The patch series introduces basic UART support (in interrupt mode) and support of interrupts for hypervisor mode.
To implement this the following has been added:
- APLIC and IMISC initialization.
- Introduce of intc_hw_operations abstraction.
- Introduce some APLIC and IMSIC operations.
- Introduce init_IRQ(), platform_get_irq() and setup_irq() functions.
- Update do_trap() handler to handle IRQ_S_EXT.
- Introduce some other functions such as: get_s_time(), smp_clear_cpu_maps(), ioremap().
- Enable UART.
Oleksii Kurochko (14):
xen/riscv: implement get_s_time()
xen/riscv: introduce smp_clear_cpu_maps()
xen/riscv: introduce ioremap()
xen/riscv: introduce init_IRQ()
xen/riscv: introduce platform_get_irq()
xen/riscv: riscv_of_processor_hartid() implementation xen/riscv: Introduce intc_hw_operations abstraction xen/riscv: imsic_init() implementation
xen/riscv: aplic_init() implementation
xen/riscv: implementation of aplic and imsic operations xen/riscv: add external interrupt handling for hypervisor mode xen/riscv: implement setup_irq()
xen/riscv: initialize interrupt controller xen/riscv: add basic UART support
xen/arch/riscv/Kconfig | 13 + xen/arch/riscv/Makefile | 3 + xen/arch/riscv/aplic.c | 302 +++++++++++++++++++++++ xen/arch/riscv/imsic.c | 374 +++++++++++++++++++++++++++++ xen/arch/riscv/include/asm/aplic.h | 89 +++++++ xen/arch/riscv/include/asm/imsic.h | 88 +++++++ xen/arch/riscv/include/asm/intc.h | 31 +++ xen/arch/riscv/include/asm/irq.h | 37 ++- xen/arch/riscv/include/asm/smp.h | 5 + xen/arch/riscv/include/asm/time.h | 6 + xen/arch/riscv/intc.c | 58 +++++ xen/arch/riscv/irq.c | 223 +++++++++++++++++ xen/arch/riscv/pt.c | 19 ++ xen/arch/riscv/setup.c | 23 ++ xen/arch/riscv/smpboot.c | 83 +++++++ xen/arch/riscv/stubs.c | 16 -- xen/arch/riscv/time.c | 7 + xen/arch/riscv/traps.c | 18 ++ xen/drivers/char/Kconfig | 1 - 19 files changed, 1378 insertions(+), 18 deletions(-) create mode 100644 xen/arch/riscv/imsic.c create mode 100644 xen/arch/riscv/include/asm/aplic.h create mode 100644 xen/arch/riscv/include/asm/imsic.h create mode 100644 xen/arch/riscv/irq.c create mode 100644 xen/arch/riscv/smpboot.c
Signature · 1 line
2.49.0
PATCH 1/14 xen/riscv: implement get_s_time()
Also tick_to_ns() is implemeted as it is used in get_s_time().
--- xen/arch/riscv/include/asm/time.h | 6 ++++++ xen/arch/riscv/stubs.c | 5 ----- xen/arch/riscv/time.c | 7 +++++++ 3 files changed, 13 insertions(+), 5 deletions(-)
xen/arch/riscv/include/asm/time.h +6 -0
@@ -3,6 +3,7 @@#define ASM__RISCV__TIME_H #include <xen/bug.h> +#include <xen/lib.h> #include <xen/types.h> #include <asm/csr.h>
@@ -23,6 +24,11 @@ static inline cycles_t get_cycles(void) return csr_read(CSR_TIME); } +static inline s_time_t ticks_to_ns (uint64_t ticks) +{ +return muldiv64 (ticks,SECONDS (1 ),1000 * cpu_khz); +} +void preinit_xen_time (void );#endif /* ASM__RISCV__TIME_H */
xen/arch/riscv/stubs.c +0 -5
@@ -27,11 +27,6 @@ nodemask_t __read_mostly node_online_map = { { [0] = 1UL } };/* time.c */ -s_time_t get_s_time (void ) -{ -BUG_ON ("unimplemented" ); -} -int reprogram_timer (s_time_t timeout) {BUG_ON ("unimplemented" );
xen/arch/riscv/time.c +7 -0
@@ -4,10 +4,17 @@#include <xen/init.h> #include <xen/lib.h> #include <xen/sections.h> +#include <xen/types.h> unsigned long __ro_after_init cpu_khz;/* CPU clock frequency in kHz. */ uint64_t __ro_after_init boot_clock_cycles; +s_time_t get_s_time (void ) +{ +uint64_t ticks =get_cycles () - boot_clock_cycles; +return ticks_to_ns (ticks); +} +/* Set up the timer on the boot CPU (early init function) */ static void __init preinit_dt_xen_time (void ) {
Signature · 1 line
2.49.0
Re: [PATCH v1 01/14] xen/riscv: implement get_s_time()
xen/arch/riscv/include/asm/time.h static inline cycles_t get_cycles(void) · line 30
Quoted 8 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/time.h +4
-0@@ -23,6 +24,11 @@ static inline cycles_t get_cycles(void)return csr_read (CSR_TIME ); } +static inline s_time_t ticks_to_ns (uint64_t ticks) +{ +return muldiv64 (ticks,SECONDS (1 ),1000 * cpu_khz); +}
Why the extra multiplication by 1000? I.e. why not "muldiv64(ticks, MILLISECONDS(1), cpu_khz)", getting away with just one multiplication and a reduced risk of encountering intermediate overflow (affecting only hypothetical above 4THz CPUs then)?
Quoted 5 lines
xen/arch/riscv/time.c +0
-0@@ -4,10 +4,17 @@#include <xen/init.h> #include <xen/lib.h>
xen/arch/riscv/time.c line 15
Quoted 10 lines
xen/arch/riscv/time.c +5
-0#include <xen/sections.h> +#include <xen/types.h> unsigned long __ro_after_init cpu_khz;/* CPU clock frequency in kHz. */ uint64_t __ro_after_init boot_clock_cycles; +s_time_t get_s_time (void ) +{ +uint64_t ticks =get_cycles () - boot_clock_cycles; +return ticks_to_ns (ticks);
Nit: Blank line between declaration(s) and statement(s) please, as well as ahead of the main "return" of a function.
Happy to make both adjustments upon committing, so long as you agree; then:
Jan
Re: [PATCH v1 01/14] xen/riscv: implement get_s_time()
Quoted 5 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/time.h +0
-0@@ -23,6 +24,11 @@ static inline cycles_t get_cycles(void)return csr_read (CSR_TIME ); }
Quoted 8 lines
xen/arch/riscv/include/asm/time.h static inline cycles_t get_cycles(void) · line 30
Quoted 4 lines
xen/arch/riscv/include/asm/time.h +4
-0+static inline s_time_t ticks_to_ns (uint64_t ticks) +{ +return muldiv64 (ticks,SECONDS (1 ),1000 * cpu_khz); +}Why the extra multiplication by 1000? I.e. why not "
muldiv64(ticks, MILLISECONDS(1), cpu_khz)", getting away with just one multiplication and a reduced risk of encountering intermediate overflow (affecting only hypothetical above 4THz CPUs then)?
Multiplication by 1000 was needed to convert khz to hz, but yes, your option would be better.
Quoted 19 lines
Quoted 6 lines
xen/arch/riscv/time.c +0
-0@@ -4,10 +4,17 @@#include <xen/init.h> #include <xen/lib.h> xen/arch/riscv/time.c line 15
Quoted 10 lines
xen/arch/riscv/time.c +5
-0#include <xen/sections.h> +#include <xen/types.h> unsigned long __ro_after_init cpu_khz;/* CPU clock frequency in kHz. */ uint64_t __ro_after_init boot_clock_cycles; +s_time_t get_s_time (void ) +{ +uint64_t ticks =get_cycles () - boot_clock_cycles; +return ticks_to_ns (ticks);Nit: Blank line between declaration(s) and statement(s) please, as well as ahead of the main "return" of a function.
Quoted 2 lines
Happy to make both adjustments upon committing, so long as you agree; then:
Reviewed-by Jan Beulich<[email protected]>
I'll be happy with that.
Thank you very much.
~ Oleksii
PATCH 2/14 xen/riscv: introduce smp_clear_cpu_maps()
In reply to Oleksii KurochkoInitialize cpu_{possible, online, present}_map by using smp_clear_cpu_maps().
Drop DEFINE_PER_CPU(unsigned int, cpu_id) from stubs.c as this variable isn't expected to be used in RISC-V at all.
Move declaration of cpu_{possible,online,present}_map from stubs.c to smpboot.c as now smpboot.c is now introduced.
Other defintions keep in stubs.c as they are not initialized and not needed, at the moment.
--- xen/arch/riscv/Makefile | 1 + xen/arch/riscv/include/asm/smp.h | 2 ++ xen/arch/riscv/setup.c | 2 ++ xen/arch/riscv/smpboot.c | 15 +++++++++++++++ xen/arch/riscv/stubs.c | 6 ------ 5 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 xen/arch/riscv/smpboot.c
xen/arch/riscv/Makefile +1 -0
@@ -10,6 +10,7 @@ obj-y += sbi.o obj-y += setup.o obj-y += shutdown.o obj-y += smp.o +obj-y += smpboot.o obj-y += stubs.o obj-y += time.o obj-y += traps.o
xen/arch/riscv/include/asm/smp.h +2 -0
@@ -26,6 +26,8 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,void setup_tp (unsigned int cpuid); +void smp_clear_cpu_maps (void ); +#endif /*
xen/arch/riscv/setup.c +2 -0
@@ -72,6 +72,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,remove_identity_mapping (); +smp_clear_cpu_maps (); +set_processor_id (0 );set_cpuid_to_hartid (0 , bootcpu_id);
xen/arch/riscv/smpboot.c new file +15 -0
@@ -0,0 +1,15 @@ +#include <xen/cpumask.h> +#include <xen/init.h> + +cpumask_t cpu_online_map; +cpumask_t cpu_present_map; +cpumask_t cpu_possible_map; + +void __init smp_clear_cpu_maps (void ) +{ +cpumask_clear (&cpu_possible_map); +cpumask_clear (&cpu_online_map); +cpumask_set_cpu (0 , &cpu_possible_map); +cpumask_set_cpu (0 , &cpu_online_map); +cpumask_copy (&cpu_present_map, &cpu_possible_map); +}
xen/arch/riscv/stubs.c +0 -6
@@ -11,12 +11,6 @@/* smpboot.c */ -cpumask_t cpu_online_map; -cpumask_t cpu_present_map; -cpumask_t cpu_possible_map; - -/* ID of the PCPU we're running on */ -DEFINE_PER_CPU(unsigned int, cpu_id);/* XXX these seem awfully x86ish... */ /* representing HT siblings of each logical CPU */ DEFINE_PER_CPU_READ_MOSTLY (cpumask_var_t, cpu_sibling_mask);
Signature · 1 line
2.49.0
Re: [PATCH v1 02/14] xen/riscv: introduce smp_clear_cpu_maps()
Quoted 64 lines · Oleksii Kurochko
Initialize cpu_{possible, online, present}_map by using
smp_clear_cpu_maps().Drop
DEFINE_PER_CPU(unsigned int, cpu_id)fromstubs.cas this variable isn't expected to be used in RISC-V at all.Move declaration of cpu_{possible,online,present}_map from
stubs.ctosmpboot.cas nowsmpboot.cis now introduced.
Other defintions keep instubs.cas they are not initialized and not needed, at the moment.Signed-off-by Oleksii Kurochko <[email protected]>--- xen/arch/riscv/Makefile | 1 + xen/arch/riscv/include/asm/smp.h | 2 ++ xen/arch/riscv/setup.c | 2 ++ xen/arch/riscv/smpboot.c | 15 +++++++++++++++ xen/arch/riscv/stubs.c | 6 ------ 5 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 xen/arch/riscv/smpboot.cxen/arch/riscv/Makefile +1
-0@@ -10,6 +10,7 @@ obj-y += sbi.o obj-y += setup.o obj-y += shutdown.o obj-y += smp.o +obj-y += smpboot.o obj-y += stubs.o obj-y += time.o obj-y += traps.oxen/arch/riscv/include/asm/smp.h +2
-0@@ -26,6 +26,8 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,void setup_tp (unsigned int cpuid); +void smp_clear_cpu_maps (void ); +#endif /* xen/arch/riscv/setup.c +2
-0@@ -72,6 +72,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,remove_identity_mapping (); +smp_clear_cpu_maps (); +set_processor_id (0 );set_cpuid_to_hartid (0 , bootcpu_id);xen/arch/riscv/smpboot.c new file +0
-0
xen/arch/riscv/smpboot.c line 6
Quoted 6 lines
xen/arch/riscv/smpboot.c new file +6
-0@@ -0,0 +1,15 @@ +#include <xen/cpumask.h> +#include <xen/init.h> + +cpumask_t cpu_online_map; +cpumask_t cpu_present_map; +cpumask_t cpu_possible_map;
__read_mostly for all of them, perhaps (if CPU hotplug isn't expected to be supported) even __ro_after_init for the latter two?
As to cpu_possible_map - do you predict that you'll actually use it? Arm does (and instead has only a fake cpu_present_map), but on x86 we get away without.
xen/arch/riscv/smpboot.c line 11
Quoted 4 lines
xen/arch/riscv/smpboot.c new file +4
-0+void __init smp_clear_cpu_maps (void ) +{ +cpumask_clear (&cpu_possible_map); +cpumask_clear (&cpu_online_map);
What's the point of these? All three maps start out fully zeroed.
xen/arch/riscv/smpboot.c line 13
Quoted 2 lines
xen/arch/riscv/smpboot.c new file +2
-0+cpumask_set_cpu (0 , &cpu_possible_map); +cpumask_set_cpu (0 , &cpu_online_map);
These are contradicting the name of the function. The somewhat equivalent function we have on x86 is smp_prepare_boot_cpu().
xen/arch/riscv/smpboot.c line 14
Quoted 1 line
xen/arch/riscv/smpboot.c new file +1
-0+cpumask_copy (&cpu_present_map, &cpu_possible_map);
Another cpumask_set_cpu() is probably cheaper here then.
Jan
Re: [PATCH v1 02/14] xen/riscv: introduce smp_clear_cpu_maps()
Quoted 58 lines · Jan Beulich
Quoted 57 lines · Oleksii Kurochko
Initialize cpu_{possible, online, present}_map by using
smp_clear_cpu_maps().Drop
DEFINE_PER_CPU(unsigned int, cpu_id)fromstubs.cas this variable isn't expected to be used in RISC-V at all.Move declaration of cpu_{possible,online,present}_map from
stubs.ctosmpboot.cas nowsmpboot.cis now introduced.
Other defintions keep instubs.cas they are not initialized and not needed, at the moment.Signed-off-by Oleksii Kurochko<[email protected]>--- xen/arch/riscv/Makefile | 1 + xen/arch/riscv/include/asm/smp.h | 2 ++ xen/arch/riscv/setup.c | 2 ++ xen/arch/riscv/smpboot.c | 15 +++++++++++++++ xen/arch/riscv/stubs.c | 6 ------ 5 files changed, 20 insertions(+), 6 deletions(-)create mode 100644
xen/arch/riscv/smpboot.cxen/arch/riscv/Makefile +1
-0@@ -10,6 +10,7 @@ obj-y += sbi.o obj-y += setup.o obj-y += shutdown.o obj-y += smp.o +obj-y += smpboot.o obj-y += stubs.o obj-y += time.o obj-y += traps.oxen/arch/riscv/include/asm/smp.h +2
-0@@ -26,6 +26,8 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,void setup_tp (unsigned int cpuid); +void smp_clear_cpu_maps (void ); +#endif /* xen/arch/riscv/setup.c +2
-0@@ -72,6 +72,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,remove_identity_mapping (); +smp_clear_cpu_maps (); +set_processor_id (0 );
Quoted 15 lines
xen/arch/riscv/smpboot.c line 6
Quoted 13 lines
xen/arch/riscv/setup.c +0
-0set_cpuid_to_hartid (0 , bootcpu_id);xen/arch/riscv/smpboot.c new file +6
-0@@ -0,0 +1,15 @@ +#include <xen/cpumask.h> +#include <xen/init.h> + +cpumask_t cpu_online_map; +cpumask_t cpu_present_map; +cpumask_t cpu_possible_map;
__read_mostlyfor all of them, perhaps (if CPU hotplug isn't expected to be supported) even__ro_after_initfor the latter two?
We have been living without CPU hotplug support for a long time in the downstream branch, but I can't say whether it is expected to be supported in the future or not. To ensure we can add such an option later without changing the attributes of cpu_online_map variable, I prefer to use|__read_mostly| here and __ro_after_init for cpu_possible_map.
Quoted 4 lines
As to
cpu_possible_map- do you predict that you'll actually use it? Arm does (and instead has only a fakecpu_present_map), but on x86 we get away without.
I checked how it is used now in downstream latest branch and it isn't really used only during initialization smp_clear_cpu_maps() and smp_prepare_cpus() so we can skip it for RISC-V too.
Quoted 6 lines
xen/arch/riscv/smpboot.c line 11
Quoted 5 lines
xen/arch/riscv/smpboot.c new file +4
-0+void __init smp_clear_cpu_maps (void ) +{ +cpumask_clear (&cpu_possible_map); +cpumask_clear (&cpu_online_map);What's the point of these? All three maps start out fully zeroed.
It could be really dropped. I saw your patch for Arm, I'll align the current patch with that changes.
Quoted 8 lines
xen/arch/riscv/smpboot.c line 13
Quoted 3 lines
xen/arch/riscv/smpboot.c new file +2
-0+cpumask_set_cpu (0 , &cpu_possible_map); +cpumask_set_cpu (0 , &cpu_online_map);These are contradicting the name of the function. The somewhat equivalent function we have on x86 is
smp_prepare_boot_cpu().xen/arch/riscv/smpboot.c line 14
Quoted 1 line
xen/arch/riscv/smpboot.c new file +1
-0+cpumask_copy (&cpu_present_map, &cpu_possible_map);Another
cpumask_set_cpu()is probably cheaper here then.
What do you mean by cheaper here?
~ Oleksii
Re: [PATCH v1 02/14] xen/riscv: introduce smp_clear_cpu_maps()
Quoted 13 lines · Oleksii Kurochko
Quoted 6 lines · Jan Beulich
xen/arch/riscv/smpboot.c line 11
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c new file +4
-0+void __init smp_clear_cpu_maps (void ) +{ +cpumask_clear (&cpu_possible_map); +cpumask_clear (&cpu_online_map);What's the point of these? All three maps start out fully zeroed.
It could be really dropped. I saw your patch for Arm, I'll align the current patch with that changes.
Quoted 2 lines
Quoted 2 lines
xen/arch/riscv/smpboot.c new file +2
-0+cpumask_set_cpu (0 , &cpu_possible_map); +cpumask_set_cpu (0 , &cpu_online_map);
Quoted 7 lines
Quoted 5 lines
These are contradicting the name of the function. The somewhat equivalent function we have on x86 is
smp_prepare_boot_cpu().Quoted 1 line
+
cpumask_copy(&cpu_present_map, &cpu_possible_map);Another
cpumask_set_cpu()is probably cheaper here then.What do you mean by cheaper here?
Less code to execute to achieve the same effect.
Jan
PATCH 3/14 xen/riscv: introduce ioremap()
In reply to Oleksii KurochkoBased on RISC-V unpriviliged spec ( Version 20240411 ):
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. I/O regions in the address space should be considered non-cacheable regions in the PMAs for those regions. Such regions can be considered coherent by the PMA if they are not cached by any agent. ``` and [1]: ``` The current riscv linux implementation requires SOC system to support memory coherence between all I/O devices and CPUs. But some SOC systems cannot maintain the coherence and they need support cache clean/invalid operations to synchronize data. Current implementation is no problem with SiFive FU540, because FU540 keeps all IO devices and DMA master devices coherence with CPU. But to a traditional SOC vendor, it may already have a stable non-coherency SOC system, the need is simply to replace the CPU with RV CPU and rebuild the whole system with IO-coherency is very expensive. ```
and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.
[1] https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/
--- xen/arch/riscv/Kconfig | 12 ++++++++++++ xen/arch/riscv/pt.c | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+)
xen/arch/riscv/Kconfig +12 -0
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG string default "arch/riscv/configs/tiny64_defconfig" +config HAS_SVPBMT + bool + help + This config enables usage of Svpbmt ISA-extension ( Supervisor-mode: + page-based memory types). + + The memory type for a page contains a combination of attributes + that indicate the cacheability, idempotency, and ordering + properties for access to that page. + + The Svpbmt extension is only available on 64-bit cpus. + menu "Architecture Features" source "arch/Kconfig"
xen/arch/riscv/pt.c +19 -0
@@ -7,6 +7,7 @@#include <xen/pfn.h> #include <xen/pmap.h> #include <xen/spinlock.h> +#include <xen/vmap.h> #include <asm/fixmap.h> #include <asm/flushtlb.h>
@@ -548,3 +549,21 @@ void clear_fixmap(unsigned int map) FIXMAP_ADDR(map) + PAGE_SIZE) != 0 )BUG (); } + +void *ioremap (paddr_t pa,size_t len) +{ +mfn_t mfn =_mfn (PFN_DOWN (pa)); +unsigned int offs = pa & (PAGE_SIZE -1 ); +unsigned int nr =PFN_UP (offs + len); + +#ifdef CONFIG_HAS_SVPBMT +#error "an introduction of PAGE_HYPERVISOR_IOREMAP is needed for __vmap()" +#endif + +void *ptr =__vmap (&mfn, nr,1 ,1 ,PAGE_HYPERVISOR ,VMAP_DEFAULT ); + +if ( !ptr ) +return NULL ; + +return ptr + offs; +}
Signature · 1 line
2.49.0
Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()
Quoted 40 lines · Oleksii Kurochko
Based on RISC-V unpriviliged spec ( Version 20240411 ):
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. I/O regions in the address space should be considered non-cacheable regions in the PMAs for those regions. Such regions can be considered coherent by the PMA if they are not cached by any agent. ``` and [1]: ``` The current riscv linux implementation requires SOC system to support memory coherence between all I/O devices and CPUs. But some SOC systems cannot maintain the coherence and they need support cache clean/invalid operations to synchronize data. Current implementation is no problem with SiFive FU540, because FU540 keeps all IO devices and DMA master devices coherence with CPU. But to a traditional SOC vendor, it may already have a stable non-coherency SOC system, the need is simply to replace the CPU with RV CPU and rebuild the whole system with IO-coherency is very expensive. ```and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the
PAGE_HYPERVISORattribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to updateioremap()is not overlooked.
But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...
Quoted 7 lines
xen/arch/riscv/Kconfig +1
-0@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG string default "arch/riscv/configs/tiny64_defconfig" +config HAS_SVPBMT
xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28
Quoted 10 lines
xen/arch/riscv/Kconfig +10
-0+ bool + help + This config enables usage of Svpbmt ISA-extension ( Supervisor-mode: + page-based memory types). + + The memory type for a page contains a combination of attributes + that indicate the cacheability, idempotency, and ordering + properties for access to that page. + + The Svpbmt extension is only available on 64-bit cpus.
... I kind of expect this extension (or anything else that there might be) will need making use of.
Quoted 4 lines
xen/arch/riscv/Kconfig +0
-0@@ -548,3 +549,21 @@ void clear_fixmap(unsigned int map) FIXMAP_ADDR(map) + PAGE_SIZE) != 0 ) BUG(); }
xen/arch/riscv/Kconfig void clear_fixmap(unsigned int map) · line 561
Quoted 10 lines
xen/arch/riscv/Kconfig +10
-0+ +void *ioremap(paddr_t pa, size_t len) +{ + mfn_t mfn = _mfn(PFN_DOWN(pa)); + unsigned int offs = pa & (PAGE_SIZE - 1); + unsigned int nr = PFN_UP(offs + len); + +#ifdef CONFIG_HAS_SVPBMT + #error "an introduction of PAGE_HYPERVISOR_IOREMAP is needed for __vmap()" +#endif
While, as per above, I don't think this can stay, just in case: As indicated earlier, pre-processor directives want to have the # in the first column.
Jan
Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko
Based on RISC-V unpriviliged spec ( Version 20240411 ):
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. I/O regions in the address space should be considered non-cacheable regions in the PMAs for those regions. Such regions can be considered coherent by the PMA if they are not cached by any agent. ``` and [1]: ``` The current riscv linux implementation requires SOC system to support memory coherence between all I/O devices and CPUs. But some SOC systems cannot maintain the coherence and they need support cache clean/invalid operations to synchronize data. Current implementation is no problem with SiFive FU540, because FU540 keeps all IO devices and DMA master devices coherence with CPU. But to a traditional SOC vendor, it may already have a stable non-coherency SOC system, the need is simply to replace the CPU with RV CPU and rebuild the whole system with IO-coherency is very expensive. ```and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the
PAGE_HYPERVISORattribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to updateioremap()is not overlooked.But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...
Quoted 6 lines
xen/arch/riscv/Kconfig +0
-0@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG string default "arch/riscv/configs/tiny64_defconfig"
Quoted 13 lines
xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28
Quoted 11 lines
xen/arch/riscv/Kconfig +11
-0+config HAS_SVPBMT + bool + help + This config enables usage of Svpbmt ISA-extension ( Supervisor-mode: + page-based memory types). + + The memory type for a page contains a combination of attributes + that indicate the cacheability, idempotency, and ordering + properties for access to that page. + + The Svpbmt extension is only available on 64-bit cpus.... I kind of expect this extension (or anything else that there might be) will need making use of.
In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.
In the case of QEMU, I assume it is QEMU's responsibility to properly emulate accesses to device memory regions. Since QEMU does not appear to provide registers for configuring PMA, it seems that PMA is not emulated. Additionally, QEMU does not emulate caches.
Based on that, I expect that it is the responsibility of the firmware or the hardware itself to provide the correct PMA configuration.
I want to note that even Svpbmt is available PMA settings could be used or be ovewritten by Svpbmt's attribute value.
I will update the commit message for more clearness.
Quoted 17 lines
Quoted 5 lines
xen/arch/riscv/Kconfig +0
-0@@ -548,3 +549,21 @@ void clear_fixmap(unsigned int map) FIXMAP_ADDR(map) + PAGE_SIZE) != 0 ) BUG(); }xen/arch/riscv/Kconfig void clear_fixmap(unsigned int map) · line 561
Quoted 10 lines
xen/arch/riscv/Kconfig +10
-0+ +void *ioremap(paddr_t pa, size_t len) +{ + mfn_t mfn = _mfn(PFN_DOWN(pa)); + unsigned int offs = pa & (PAGE_SIZE - 1); + unsigned int nr = PFN_UP(offs + len); + +#ifdef CONFIG_HAS_SVPBMT + #error "an introduction of PAGE_HYPERVISOR_IOREMAP is needed for __vmap()" +#endifWhile, as per above, I don't think this can stay, just in case: As indicated earlier, pre-processor directives want to have the # in the first column.
I think it can be safely dropped now, and|PAGE_HYPERVISOR_IOREMAP| could be introduced and simply PTE's PMBT bits can be ignored in|pt_update_entry()| if Svpbmt isn't implemented.
~ Oleksii
Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()
Quoted 69 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko
Based on RISC-V unpriviliged spec ( Version 20240411 ):
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. I/O regions in the address space should be considered non-cacheable regions in the PMAs for those regions. Such regions can be considered coherent by the PMA if they are not cached by any agent. ``` and [1]: ``` The current riscv linux implementation requires SOC system to support memory coherence between all I/O devices and CPUs. But some SOC systems cannot maintain the coherence and they need support cache clean/invalid operations to synchronize data. Current implementation is no problem with SiFive FU540, because FU540 keeps all IO devices and DMA master devices coherence with CPU. But to a traditional SOC vendor, it may already have a stable non-coherency SOC system, the need is simply to replace the CPU with RV CPU and rebuild the whole system with IO-coherency is very expensive. ```and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the
PAGE_HYPERVISORattribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to updateioremap()is not overlooked.But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...
Quoted 6 lines
xen/arch/riscv/Kconfig +0
-0@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG string default "arch/riscv/configs/tiny64_defconfig"Quoted 13 lines
xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28
Quoted 11 lines
xen/arch/riscv/Kconfig +11
-0+config HAS_SVPBMT + bool + help + This config enables usage of Svpbmt ISA-extension ( Supervisor-mode: + page-based memory types). + + The memory type for a page contains a combination of attributes + that indicate the cacheability, idempotency, and ordering + properties for access to that page. + + The Svpbmt extension is only available on 64-bit cpus.... I kind of expect this extension (or anything else that there might be) will need making use of.
Quoted 4 lines
In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.
How would things work if there was a need to map a RAM page uncacheable (via ioremap() or otherwise)?
Jan
Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()
Quoted 68 lines · Jan Beulich
Quoted 67 lines · Oleksii Kurochko
Quoted 66 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko
Based on RISC-V unpriviliged spec ( Version 20240411 ):
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. I/O regions in the address space should be considered non-cacheable regions in the PMAs for those regions. Such regions can be considered coherent by the PMA if they are not cached by any agent. ``` and [1]: ``` The current riscv linux implementation requires SOC system to support memory coherence between all I/O devices and CPUs. But some SOC systems cannot maintain the coherence and they need support cache clean/invalid operations to synchronize data. Current implementation is no problem with SiFive FU540, because FU540 keeps all IO devices and DMA master devices coherence with CPU. But to a traditional SOC vendor, it may already have a stable non-coherency SOC system, the need is simply to replace the CPU with RV CPU and rebuild the whole system with IO-coherency is very expensive. ```and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the
PAGE_HYPERVISORattribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to updateioremap()is not overlooked.But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...
Quoted 7 lines
xen/arch/riscv/Kconfig +1
-0@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG string default "arch/riscv/configs/tiny64_defconfig" +config HAS_SVPBMTxen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28
Quoted 10 lines
xen/arch/riscv/Kconfig +10
-0+ bool + help + This config enables usage of Svpbmt ISA-extension ( Supervisor-mode: + page-based memory types). + + The memory type for a page contains a combination of attributes + that indicate the cacheability, idempotency, and ordering + properties for access to that page. + + The Svpbmt extension is only available on 64-bit cpus.... I kind of expect this extension (or anything else that there might be) will need making use of.
Quoted 6 lines
Quoted 4 lines
In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.
How would things work if there was a need to map a RAM page uncacheable (via
ioremap()or otherwise)?
My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.
The question is if non-cacheable RAM page is really needed if we have a coherency?
~ Oleksii
Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()
Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko
Based on RISC-V unpriviliged spec ( Version 20240411 ):
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. I/O regions in the address space should be considered non-cacheable regions in the PMAs for those regions. Such regions can be considered coherent by the PMA if they are not cached by any agent. ``` and [1]: ``` The current riscv linux implementation requires SOC system to support memory coherence between all I/O devices and CPUs. But some SOC systems cannot maintain the coherence and they need support cache clean/invalid operations to synchronize data. Current implementation is no problem with SiFive FU540, because FU540 keeps all IO devices and DMA master devices coherence with CPU. But to a traditional SOC vendor, it may already have a stable non-coherency SOC system, the need is simply to replace the CPU with RV CPU and rebuild the whole system with IO-coherency is very expensive. ```and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the
PAGE_HYPERVISORattribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to updateioremap()is not overlooked.But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...
Quoted 6 lines
xen/arch/riscv/Kconfig +0
-0@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG string default "arch/riscv/configs/tiny64_defconfig"Quoted 13 lines
xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28
Quoted 11 lines
xen/arch/riscv/Kconfig +11
-0+config HAS_SVPBMT + bool + help + This config enables usage of Svpbmt ISA-extension ( Supervisor-mode: + page-based memory types). + + The memory type for a page contains a combination of attributes + that indicate the cacheability, idempotency, and ordering + properties for access to that page. + + The Svpbmt extension is only available on 64-bit cpus.... I kind of expect this extension (or anything else that there might be) will need making use of.
In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.
Quoted 7 lines
Quoted 2 lines
How would things work if there was a need to map a RAM page uncacheable (via
ioremap()or otherwise)?My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.
The question is if non-cacheable RAM page is really needed if we have a coherency?
Aiui coherency here is among CPUs. Properties of devices in the system are largely unknown? (Beyond this there may also be special situations in which one really cares about data going directly to RAM.)
Jan
Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()
Quoted 74 lines · Jan Beulich
Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko
Based on RISC-V unpriviliged spec ( Version 20240411 ):
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. I/O regions in the address space should be considered non-cacheable regions in the PMAs for those regions. Such regions can be considered coherent by the PMA if they are not cached by any agent. ``` and [1]: ``` The current riscv linux implementation requires SOC system to support memory coherence between all I/O devices and CPUs. But some SOC systems cannot maintain the coherence and they need support cache clean/invalid operations to synchronize data. Current implementation is no problem with SiFive FU540, because FU540 keeps all IO devices and DMA master devices coherence with CPU. But to a traditional SOC vendor, it may already have a stable non-coherency SOC system, the need is simply to replace the CPU with RV CPU and rebuild the whole system with IO-coherency is very expensive. ```and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the
PAGE_HYPERVISORattribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to updateioremap()is not overlooked.But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...
Quoted 6 lines
xen/arch/riscv/Kconfig +0
-0@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG string default "arch/riscv/configs/tiny64_defconfig"Quoted 13 lines
xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28
Quoted 11 lines
xen/arch/riscv/Kconfig +11
-0+config HAS_SVPBMT + bool + help + This config enables usage of Svpbmt ISA-extension ( Supervisor-mode: + page-based memory types). + + The memory type for a page contains a combination of attributes + that indicate the cacheability, idempotency, and ordering + properties for access to that page. + + The Svpbmt extension is only available on 64-bit cpus.... I kind of expect this extension (or anything else that there might be) will need making use of.
In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.
Quoted 7 lines
Quoted 6 lines
Quoted 2 lines
How would things work if there was a need to map a RAM page uncacheable (via
ioremap()or otherwise)?My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.
The question is if non-cacheable RAM page is really needed if we have a coherency?
Aiui coherency here is among CPUs.
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. ```
Based on this from the spec, coherency here is not only among CPUs.
Quoted 2 lines
Properties of devices in the system are
largely unknown?
Yes, but still not sure what kind of property requires ioremap() which won't work without Svpmbt. Could you please tell me an example?
Quoted 2 lines
(Beyond this there may also be special situations in which one really cares about data going directly to RAM.)
If there are such special cases, I assume that the firmware or hardware (in the case of fixed PMA) will provide a non-cacheable region. In that case, the user should be aware of this region and use it for those specific scenarios.
~ Oleksii
Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()
Quoted 95 lines · Oleksii Kurochko
Quoted 74 lines · Jan Beulich
Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko
Based on RISC-V unpriviliged spec ( Version 20240411 ):
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. I/O regions in the address space should be considered non-cacheable regions in the PMAs for those regions. Such regions can be considered coherent by the PMA if they are not cached by any agent. ``` and [1]: ``` The current riscv linux implementation requires SOC system to support memory coherence between all I/O devices and CPUs. But some SOC systems cannot maintain the coherence and they need support cache clean/invalid operations to synchronize data. Current implementation is no problem with SiFive FU540, because FU540 keeps all IO devices and DMA master devices coherence with CPU. But to a traditional SOC vendor, it may already have a stable non-coherency SOC system, the need is simply to replace the CPU with RV CPU and rebuild the whole system with IO-coherency is very expensive. ```and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the
PAGE_HYPERVISORattribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to updateioremap()is not overlooked.But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...
Quoted 6 lines
xen/arch/riscv/Kconfig +0
-0@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG string default "arch/riscv/configs/tiny64_defconfig"Quoted 13 lines
xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28
Quoted 11 lines
xen/arch/riscv/Kconfig +11
-0+config HAS_SVPBMT + bool + help + This config enables usage of Svpbmt ISA-extension ( Supervisor-mode: + page-based memory types). + + The memory type for a page contains a combination of attributes + that indicate the cacheability, idempotency, and ordering + properties for access to that page. + + The Svpbmt extension is only available on 64-bit cpus.... I kind of expect this extension (or anything else that there might be) will need making use of.
In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.
Quoted 7 lines
Quoted 6 lines
Quoted 2 lines
How would things work if there was a need to map a RAM page uncacheable (via
ioremap()or otherwise)?My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.
The question is if non-cacheable RAM page is really needed if we have a coherency?
Aiui coherency here is among CPUs.
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. ```
Quoted 8 lines
Based on this from the spec, coherency here is not only among CPUs.
Quoted 2 lines
Properties of devices in the system are
largely unknown?Yes, but still not sure what kind of property requires
ioremap()which won't work without Svpmbt. Could you please tell me an example?
Well, above you said they all need to access memory coherently. That's the "property" I was referring to.
Quoted 5 lines
Quoted 2 lines
(Beyond this there may also be special situations in which one really cares about data going directly to RAM.)
If there are such special cases, I assume that the firmware or hardware (in the case of fixed PMA) will provide a non-cacheable region.
How could they? Firmware may be unaware of specific properties of specific devices a user adds to a system.
Jan
Quoted 5 lines
In that case, the user should be
aware of this region and use it for those specific scenarios.~ Oleksii
Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()
Quoted 97 lines · Jan Beulich
Quoted 96 lines · Oleksii Kurochko
Quoted 74 lines · Jan Beulich
Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko
Based on RISC-V unpriviliged spec ( Version 20240411 ):
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. I/O regions in the address space should be considered non-cacheable regions in the PMAs for those regions. Such regions can be considered coherent by the PMA if they are not cached by any agent. ``` and [1]: ``` The current riscv linux implementation requires SOC system to support memory coherence between all I/O devices and CPUs. But some SOC systems cannot maintain the coherence and they need support cache clean/invalid operations to synchronize data. Current implementation is no problem with SiFive FU540, because FU540 keeps all IO devices and DMA master devices coherence with CPU. But to a traditional SOC vendor, it may already have a stable non-coherency SOC system, the need is simply to replace the CPU with RV CPU and rebuild the whole system with IO-coherency is very expensive. ```and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the
PAGE_HYPERVISORattribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to updateioremap()is not overlooked.But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...
Quoted 6 lines
xen/arch/riscv/Kconfig +0
-0@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG string default "arch/riscv/configs/tiny64_defconfig"Quoted 13 lines
xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28
Quoted 11 lines
xen/arch/riscv/Kconfig +11
-0+config HAS_SVPBMT + bool + help + This config enables usage of Svpbmt ISA-extension ( Supervisor-mode: + page-based memory types). + + The memory type for a page contains a combination of attributes + that indicate the cacheability, idempotency, and ordering + properties for access to that page. + + The Svpbmt extension is only available on 64-bit cpus.... I kind of expect this extension (or anything else that there might be) will need making use of.
In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.
Quoted 7 lines
Quoted 6 lines
Quoted 2 lines
How would things work if there was a need to map a RAM page uncacheable (via
ioremap()or otherwise)?My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.
The question is if non-cacheable RAM page is really needed if we have a coherency?
Aiui coherency here is among CPUs.
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. ```Based on this from the spec, coherency here is not only among CPUs.
Quoted 6 lines
Quoted 4 lines
Quoted 2 lines
Properties of devices in the system are
largely unknown?Yes, but still not sure what kind of property requires
ioremap()which won't work without Svpmbt. Could you please tell me an example?Well, above you said they all need to access memory coherently. That's the "property" I was referring to.
Do you mean that device could have a property which tell that it would like to have non-cachable region used for that? I haven't seen such property in device tree files.
Do we have in Xen cases when Xen wants to have map part of RAM as non-cachebale and it is only the one option?
I am also thinking why it can't be used cachable region + barrier (if we don't have memory coherency for everything).
Anyway, if it isn't an option to have mapped cacheble region + barrier then there is no any choice and the support of Svpmbt is required.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
(Beyond this there may also be special situations in which one really cares about data going directly to RAM.)
If there are such special cases, I assume that the firmware or hardware (in the case of fixed PMA) will provide a non-cacheable region.
How could they? Firmware may be unaware of specific properties of specific devices a user adds to a system.
(this is not real case, just thoughts) Firmware could by default provide part of RAM as non-cacheable region and then hypervisor/kernel use this region for allocation. But I agree that it isn't the best thing to manage that.
~ Oleksii
Quoted 5 lines
Quoted 5 lines
In that case, the user should be
aware of this region and use it for those specific scenarios.~ Oleksii
Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()
Quoted 103 lines · Oleksii Kurochko
Quoted 101 lines · Jan Beulich
Quoted 100 lines · Oleksii Kurochko
Quoted 74 lines · Jan Beulich
Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko
Based on RISC-V unpriviliged spec ( Version 20240411 ):
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. I/O regions in the address space should be considered non-cacheable regions in the PMAs for those regions. Such regions can be considered coherent by the PMA if they are not cached by any agent. ``` and [1]: ``` The current riscv linux implementation requires SOC system to support memory coherence between all I/O devices and CPUs. But some SOC systems cannot maintain the coherence and they need support cache clean/invalid operations to synchronize data. Current implementation is no problem with SiFive FU540, because FU540 keeps all IO devices and DMA master devices coherence with CPU. But to a traditional SOC vendor, it may already have a stable non-coherency SOC system, the need is simply to replace the CPU with RV CPU and rebuild the whole system with IO-coherency is very expensive. ```and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the
PAGE_HYPERVISORattribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to updateioremap()is not overlooked.But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...
Quoted 6 lines
xen/arch/riscv/Kconfig +0
-0@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG string default "arch/riscv/configs/tiny64_defconfig"Quoted 13 lines
xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28
Quoted 11 lines
xen/arch/riscv/Kconfig +11
-0+config HAS_SVPBMT + bool + help + This config enables usage of Svpbmt ISA-extension ( Supervisor-mode: + page-based memory types). + + The memory type for a page contains a combination of attributes + that indicate the cacheability, idempotency, and ordering + properties for access to that page. + + The Svpbmt extension is only available on 64-bit cpus.... I kind of expect this extension (or anything else that there might be) will need making use of.
In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.
Quoted 7 lines
Quoted 6 lines
Quoted 2 lines
How would things work if there was a need to map a RAM page uncacheable (via
ioremap()or otherwise)?My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.
The question is if non-cacheable RAM page is really needed if we have a coherency?
Aiui coherency here is among CPUs.
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. ```Based on this from the spec, coherency here is not only among CPUs.
Quoted 2 lines
Properties of devices in the system are
largely unknown?Yes, but still not sure what kind of property requires
ioremap()which won't work without Svpmbt. Could you please tell me an example?
Quoted 8 lines
Quoted 2 lines
Well, above you said they all need to access memory coherently. That's the "property" I was referring to.
Do you mean that device could have a property which tell that it would like to have non-cachable region used for that? I haven't seen such property in device tree files.
Do we have in Xen cases when Xen wants to have map part of RAM as non-cachebale and it is only the one option?
On x86 we have the case that IOMMUs may access memory non-coherently. This is particular means that IOMMU page table updates (which necessarily live in RAM) need to be done quite carefully. As it's all our code, we deal with the situation by issuing cache flushes, avoiding the need for UC mappings.
Graphics engines may have similar constraints, aiui. With the driver code not being part of Xen, we wouldn't be able to use a similar "simplification" there. UC mappings would be pretty much unavoidable.
Quoted 2 lines
I am also thinking why it can't be used cachable region + barrier (if we don't have memory coherency for everything).
Not sure what exactly you're asking here (if anything). An answer would very likely depend on the specific kind of barrier you're thinking about. The question would be what, if any, effect a barrier would have on the cache(s).
Quoted 2 lines
Anyway, if it isn't an option to have mapped cacheble region + barrier then there is no any choice and the support of Svpmbt is required.
Quite possible.
Jan
Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()
Quoted 110 lines · Jan Beulich
Quoted 102 lines · Oleksii Kurochko
Quoted 101 lines · Jan Beulich
Quoted 100 lines · Oleksii Kurochko
Quoted 74 lines · Jan Beulich
Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko
Based on RISC-V unpriviliged spec ( Version 20240411 ):
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. I/O regions in the address space should be considered non-cacheable regions in the PMAs for those regions. Such regions can be considered coherent by the PMA if they are not cached by any agent. ``` and [1]: ``` The current riscv linux implementation requires SOC system to support memory coherence between all I/O devices and CPUs. But some SOC systems cannot maintain the coherence and they need support cache clean/invalid operations to synchronize data. Current implementation is no problem with SiFive FU540, because FU540 keeps all IO devices and DMA master devices coherence with CPU. But to a traditional SOC vendor, it may already have a stable non-coherency SOC system, the need is simply to replace the CPU with RV CPU and rebuild the whole system with IO-coherency is very expensive. ```and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the
PAGE_HYPERVISORattribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to updateioremap()is not overlooked.But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...
Quoted 6 lines
xen/arch/riscv/Kconfig +0
-0@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG string default "arch/riscv/configs/tiny64_defconfig"Quoted 13 lines
xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28
Quoted 11 lines
xen/arch/riscv/Kconfig +11
-0+config HAS_SVPBMT + bool + help + This config enables usage of Svpbmt ISA-extension ( Supervisor-mode: + page-based memory types). + + The memory type for a page contains a combination of attributes + that indicate the cacheability, idempotency, and ordering + properties for access to that page. + + The Svpbmt extension is only available on 64-bit cpus.... I kind of expect this extension (or anything else that there might be) will need making use of.
In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.
Quoted 7 lines
Quoted 6 lines
Quoted 2 lines
How would things work if there was a need to map a RAM page uncacheable (via
ioremap()or otherwise)?My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.
The question is if non-cacheable RAM page is really needed if we have a coherency?
Aiui coherency here is among CPUs.
``` For implementations that conform to the RISC-V Unix Platform Specification, I/O devices and DMA operations are required to access memory coherently and via strongly ordered I/O channels. Therefore, accesses to regular main memory regions that are concurrently accessed by external devices can also use the standard synchronization mechanisms. Implementations that do not conform to the Unix Platform Specification and/or in which devices do not access memory coherently will need to use mechanisms (which are currently platform-specific or device-specific) to enforce coherency. ```Based on this from the spec, coherency here is not only among CPUs.
Quoted 2 lines
Properties of devices in the system are
largely unknown?Yes, but still not sure what kind of property requires
ioremap()which won't work without Svpmbt. Could you please tell me an example?Quoted 7 lines
Quoted 2 lines
Well, above you said they all need to access memory coherently. That's the "property" I was referring to.
Do you mean that device could have a property which tell that it would like to have non-cachable region used for that? I haven't seen such property in device tree files.
Do we have in Xen cases when Xen wants to have map part of RAM as non-cachebale and it is only the one option?
Quoted 8 lines
On x86 we have the case that IOMMUs may access memory non-coherently. This is particular means that IOMMU page table updates (which necessarily live in RAM) need to be done quite carefully. As it's all our code, we deal with the situation by issuing cache flushes, avoiding the need for UC mappings.
Graphics engines may have similar constraints, aiui. With the driver code not being part of Xen, we wouldn't be able to use a similar "simplification" there. UC mappings would be pretty much unavoidable.
For this case, it would be better to have Svpmbt.
I would like to noted that Svpmbt isn't supported by RV32 architectures. For such cases, it will be still needed to play with PMA.
I found today a patch in Linux kernel which does something similar to what I wrote in one of my previous replies:
[0]https://lore.kernel.org/all/[email protected]/
In the cover letter [0] it is mentioned the following:
On some RISC-V platforms, including StarFive JH7100 and ESWIN EIC7700,
RAM is mapped to multiple physical address ranges, with each alias
having a different set of statically-determined Physical Memory
Attributes (PMAs). Software selects the PMAs for a page by choosing a
PFN from the corresponding physical address range. On these platforms,
this is the only way to allocate noncached memory for use with
noncoherent DMA.
So firmware should configure PMAs so some part of RAM is noncached and then kernel get this info based on the binding:
https://patchew.org/linux/[email protected]/[email protected]/
Considering that this feature isn't available even in Linux kernel, we can start with assumption that all our SoCs will support Svpmbt.
We don't really care about StarFive JH7100 as it doesn't support H extension, but we potentially should care about ESWIN EIC7700, which support H extension and doesn't support Svpmbt extension according to a datasheet publicly available:
Each EIC7700X core is configured to support the RV64I base ISA, as well as the Multiply (M), Atomic(A), Single-Precision Floating Point (F), Double-Precision Floating Point (D), Compressed (C), CSR Instructions (Zicsr), Instruction-Fetch Fence (Zifencei), Address Calculation (Zba), Basic Bit Manipulation (Zbb), and Count Overflow and Mode-Based Filtering (Sscofpmf) RISC‑V extensions. This is captured by the RISC‑V extension string: RV64GC_Zba_Zbb_Sscofpmf.
Quoted 6 lines
Quoted 3 lines
I am also thinking why it can't be used cachable region + barrier (if we don't have memory coherency for everything).
Not sure what exactly you're asking here (if anything). An answer would very likely depend on the specific kind of barrier you're thinking about. The question would be what, if any, effect a barrier would have on the cache(s).
I confused barrier with cache flushes (when I wrote that I thought about the case of DMA that we don't really should have requirement of non-cachable memory for DMA as it is enough to have memory fence between use of DMA memory and MMIO that triggers the dma), basically I meant what you wrote above about x86's IOMMUs.
~ Oleksii
Quoted 6 lines
Quoted 3 lines
Anyway, if it isn't an option to have mapped cacheble region + barrier then there is no any choice and the support of Svpmbt is required.
Quite possible.
Jan
PATCH 4/14 xen/riscv: introduce init_IRQ()
In reply to Oleksii KurochkoImplement init_IRQ() to initalize various IRQs.
Currently, this function initializes the irq_desc[] array, which stores IRQ descriptors containing various information about each IRQ, such as the type of hardware handling, whether the IRQ is disabled, etc.
The initialization is basic at this point and includes setting IRQ_TYPE_INVALID as the IRQ type, assigning the IRQ number ( which is just a consequent index of irq_desc[] array ) to desc->irq, and setting desc->action to NULL.
Additionally, the function init_irq_data() is introduced to initialize the IRQ descriptors for all IRQs in the system.
Also, define IRQ_TYPE_* which are the same as the existing device tree definitions for convenience.
--- xen/arch/riscv/Makefile | 1 + xen/arch/riscv/include/asm/irq.h | 24 +++++++++++++++++ xen/arch/riscv/irq.c | 44 ++++++++++++++++++++++++++++++++ xen/arch/riscv/setup.c | 3 +++ xen/arch/riscv/stubs.c | 5 ---- 5 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 xen/arch/riscv/irq.c
xen/arch/riscv/Makefile +1 -0
@@ -3,6 +3,7 @@ obj-y += cpufeature.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-y += entry.o obj-y += intc.o +obj-y += irq.o obj-y += mm.o obj-y += pt.o obj-$(CONFIG_RISCV_64) += riscv64/
xen/arch/riscv/include/asm/irq.h +24 -0
@@ -3,6 +3,28 @@#define ASM__RISCV__IRQ_H #include <xen/bug.h> +#include <xen/device_tree.h> + +#define NR_IRQS 1024 + +/* +* TODO: Should IRQ_TYPE_* be moved to xen/irq.h and wrapped into +* #ifdef CONFIG_HAS_DEVICE_TREE? +*/ +/* +* These defines correspond to the Xen internal representation of the +* IRQ types. We choose to make them the same as the existing device +* tree definitions for convenience. +*/ +#define IRQ_TYPE_NONE DT_IRQ_TYPE_NONE +#define IRQ_TYPE_EDGE_RISING DT_IRQ_TYPE_EDGE_RISING +#define IRQ_TYPE_EDGE_FALLING DT_IRQ_TYPE_EDGE_FALLING +#define IRQ_TYPE_EDGE_BOTH DT_IRQ_TYPE_EDGE_BOTH +#define IRQ_TYPE_LEVEL_HIGH DT_IRQ_TYPE_LEVEL_HIGH +#define IRQ_TYPE_LEVEL_LOW DT_IRQ_TYPE_LEVEL_LOW +#define IRQ_TYPE_LEVEL_MASK DT_IRQ_TYPE_LEVEL_MASK +#define IRQ_TYPE_SENSE_MASK DT_IRQ_TYPE_SENSE_MASK +#define IRQ_TYPE_INVALID DT_IRQ_TYPE_INVALID/* TODO */ #define nr_irqs 0U
@@ -25,6 +47,8 @@ static inline void arch_move_irqs(struct vcpu *v) BUG_ON("unimplemented"); } +void init_IRQ (void ); +#endif /* ASM__RISCV__IRQ_H */ /*
xen/arch/riscv/irq.c new file +44 -0
44 lines not on this page · raw message
xen/arch/riscv/setup.c +3 -0
@@ -6,6 +6,7 @@#include <xen/compile.h> #include <xen/device_tree.h> #include <xen/init.h> +#include <xen/irq.h> #include <xen/mm.h> #include <xen/shutdown.h> #include <xen/vmap.h>
@@ -127,6 +128,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,panic ("Booting using ACPI isn't supported\n" ); } +init_IRQ (); +riscv_fill_hwcap ();preinit_xen_time ();
xen/arch/riscv/stubs.c +0 -5
@@ -107,11 +107,6 @@ void irq_ack_none(struct irq_desc *desc)BUG_ON ("unimplemented" ); } -int arch_init_one_irq_desc (struct irq_desc *desc) -{ -BUG_ON ("unimplemented" ); -} -void smp_send_state_dump (unsigned int cpu) {BUG_ON ("unimplemented" );
Signature · 1 line
2.49.0
Re: [PATCH v1 04/14] xen/riscv: introduce init_IRQ()
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/irq.h +0
-0@@ -3,6 +3,28 @@#define ASM__RISCV__IRQ_H
xen/arch/riscv/include/asm/irq.h line 13
Quoted 10 lines
xen/arch/riscv/include/asm/irq.h +8
-0#include <xen/bug.h> +#include <xen/device_tree.h> + +#define NR_IRQS 1024 + +/* +* TODO: Should IRQ_TYPE_* be moved to xen/irq.h and wrapped into +* #ifdef CONFIG_HAS_DEVICE_TREE? +*/
Wouldn't that be more like asm-generic/dt-irq.h (or irq-dt.h)? The field where these values are stored is an arch-specific one, after all.
Quoted 18 lines
xen/arch/riscv/irq.c new file +15
-0@@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +/* +* RISC-V Trap handlers +* +* Copyright (c) 2024 Vates +*/ + +#include <xen/bug.h> +#include <xen/init.h> +#include <xen/irq.h> + +static irq_desc_t irq_desc[NR_IRQS ]; + +int arch_init_one_irq_desc (struct irq_desc *desc)
xen/arch/riscv/irq.c line 25
Quoted 10 lines
xen/arch/riscv/irq.c new file +10
-0+{ + desc->arch.type =IRQ_TYPE_INVALID ; +return 0 ; +} + +static int __init init_irq_data (void ) +{ +int irq; + +for ( irq =0 ; irq <NR_IRQS ; irq++ )
For this the variable would better be of unsigned type. I realize though that ...
xen/arch/riscv/irq.c line 33
Quoted 8 lines
xen/arch/riscv/irq.c new file +8
-0+ { +struct irq_desc *desc =irq_to_desc (irq); +int rc =init_one_irq_desc (desc); + +if ( rc ) +return rc; + + desc->irq = irq;
... it's a plain int field that we're storing into here. I don't think I can spot any place where a negative value would be stored into there.
Jan
Re: [PATCH v1 04/14] xen/riscv: introduce init_IRQ()
Quoted 6 lines · Jan Beulich
Quoted 5 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/irq.h +0
-0@@ -3,6 +3,28 @@#define ASM__RISCV__IRQ_H
Quoted 11 lines
xen/arch/riscv/include/asm/irq.h line 13
Quoted 9 lines
xen/arch/riscv/include/asm/irq.h +8
-0#include <xen/bug.h> +#include <xen/device_tree.h> + +#define NR_IRQS 1024 + +/* +* TODO: Should IRQ_TYPE_* be moved to xen/irq.h and wrapped into +* #ifdef CONFIG_HAS_DEVICE_TREE? +*/ Wouldn't that be more like
asm-generic/dt-irq.h(orirq-dt.h)? The field where these values are stored is an arch-specific one, after all.
It would be much better. Thanks!
~ Oleksii
PATCH 5/14 xen/riscv: introduce platform_get_irq()
In reply to Oleksii Kurochkoplatform_get_irq() recieves information about device's irq ( type and irq number ) from device tree node and using this information update irq descriptor in irq_desc[] array.
Introduce dt_irq_xlate and initialize with aplic_irq_xlate() as it is used by dt_device_get_irq() which is called by platform_get_irq().
--- xen/arch/riscv/aplic.c | 19 +++++++++++++++ xen/arch/riscv/include/asm/irq.h | 3 +++ xen/arch/riscv/irq.c | 41 ++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+)
xen/arch/riscv/aplic.c +19 -0
@@ -11,6 +11,7 @@#include <xen/errno.h> #include <xen/init.h> +#include <xen/irq.h> #include <xen/sections.h> #include <xen/types.h>
@@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = { .hw_version = INTC_APLIC, }; +static int aplic_irq_xlate (const uint32_t *intspec,unsigned int intsize, +unsigned int *out_hwirq, +unsigned int *out_type) +{ +if ( intsize <2 ) +return -EINVAL ; + +/* Mapping 1:1 */ + *out_hwirq = intspec[0 ]; + +if ( out_type ) + *out_type = intspec[1 ] &IRQ_TYPE_SENSE_MASK ; + +return 0 ; +} +static int __init aplic_preinit (struct dt_device_node *node,const void *dat) {if ( aplic_info.node )
@@ -35,6 +52,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat) aplic_info.node = node; + dt_irq_xlate = aplic_irq_xlate; +return 0 ; }
xen/arch/riscv/include/asm/irq.h +3 -0
@@ -47,6 +47,9 @@ static inline void arch_move_irqs(struct vcpu *v) BUG_ON("unimplemented"); } +struct dt_device_node ; +int platform_get_irq (const struct dt_device_node *device,int index); +void init_IRQ (void );#endif /* ASM__RISCV__IRQ_H */
xen/arch/riscv/irq.c +41 -0
52 lines not on this page · raw message
Signature · 1 line
2.49.0
Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()
xen/arch/riscv/aplic.c static struct intc_info __ro_after_init aplic_info = { · line 25
Quoted 5 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +1
-0@@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = { .hw_version = INTC_APLIC, }; +static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
As you start adding functions calling indirectly, please consider adding cf_check right away, even if right now this has no effect on RISC-V. That'll save you from going through the entire RISC-V subtree later on to find them all.
Quoted 30 lines
xen/arch/riscv/aplic.c +17
-0+ unsigned int *out_hwirq, + unsigned int *out_type) +{ +if ( intsize <2 ) +return -EINVAL ; + +/* Mapping 1:1 */ + *out_hwirq = intspec[0 ]; + +if ( out_type ) + *out_type = intspec[1 ] &IRQ_TYPE_SENSE_MASK ; + +return 0 ; +} +static int __init aplic_preinit (struct dt_device_node *node,const void *dat) {if ( aplic_info.node )@@ -35,6 +52,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat) aplic_info.node = node; + dt_irq_xlate = aplic_irq_xlate; +return 0 ; }
xen/arch/riscv/include/asm/irq.h static inline void arch_move_irqs(struct vcpu *v) · line 51
Quoted 5 lines
xen/arch/riscv/include/asm/irq.h +2
-0@@ -47,6 +47,9 @@ static inline void arch_move_irqs(struct vcpu *v)BUG_ON ("unimplemented" ); } +struct dt_device_node ; +int platform_get_irq (const struct dt_device_node *device,int index);
And I assume callers of this will appear later in the series.
Quoted 24 lines
xen/arch/riscv/irq.c +13
-0@@ -7,11 +7,52 @@*/ #include <xen/bug.h> +#include <xen/device_tree.h> +#include <xen/errno.h> #include <xen/init.h> #include <xen/irq.h> static irq_desc_t irq_desc[NR_IRQS ]; +static bool irq_validate_new_type (unsigned int curr,unsigned int new) +{ +return (curr ==IRQ_TYPE_INVALID || curr == new ); +} + +static int irq_set_type (unsigned int irq,unsigned int type) +{ +unsigned long flags; +struct irq_desc *desc =irq_to_desc (irq); +int ret = -EBUSY ; +
xen/arch/riscv/irq.c line 37
Quoted 10 lines
xen/arch/riscv/irq.c +10
-0+spin_lock_irqsave (&desc->lock, flags); + +if ( !irq_validate_new_type (desc->arch.type, type) ) + goto err; + + desc->arch.type = type; + + ret =0 ; + +err:
Labels indented by at least one blank please.
Quoted 4 lines
xen/arch/riscv/irq.c +4
-0+spin_unlock_irqrestore (&desc->lock, flags); + +return ret; +}
xen/arch/riscv/irq.c line 51
Quoted 10 lines
xen/arch/riscv/irq.c +10
-0+ +int platform_get_irq (const struct dt_device_node *device,int index) +{ +struct dt_irq dt_irq; + +if (dt_device_get_irq (device, index, &dt_irq) ) +return -1 ; + +if (irq_set_type (dt_irq.irq, dt_irq.type) ) +return -1 ;
Can you please return proper -E... values, perhaps ones coming back from the functions called?
Jan
Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()
Quoted 9 lines · Jan Beulich
xen/arch/riscv/aplic.c static struct intc_info __ro_after_init aplic_info = { · line 25
Quoted 5 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +1
-0@@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = { .hw_version = INTC_APLIC, }; +static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,As you start adding functions calling indirectly, please consider adding
cf_checkright away, even if right now this has no effect on RISC-V. That'll save you from going through the entire RISC-V subtree later on to find them all.
Sure. I thought that it is a feature for x86 as I haven't seen such attribute for Arm and RISC-V in GCC manuals.
Quoted 34 lines
Quoted 34 lines
xen/arch/riscv/aplic.c +17
-0+ unsigned int *out_hwirq, + unsigned int *out_type) +{ +if ( intsize <2 ) +return -EINVAL ; + +/* Mapping 1:1 */ + *out_hwirq = intspec[0 ]; + +if ( out_type ) + *out_type = intspec[1 ] &IRQ_TYPE_SENSE_MASK ; + +return 0 ; +} +static int __init aplic_preinit (struct dt_device_node *node,const void *dat) {if ( aplic_info.node )@@ -35,6 +52,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat) aplic_info.node = node; + dt_irq_xlate = aplic_irq_xlate; +return 0 ; }xen/arch/riscv/include/asm/irq.h +0
-0@@ -47,6 +47,9 @@ static inline void arch_move_irqs(struct vcpu *v)BUG_ON ("unimplemented" ); }
Quoted 3 lines
xen/arch/riscv/include/asm/irq.h static inline void arch_move_irqs(struct vcpu *v) · line 51
Quoted 2 lines
xen/arch/riscv/include/asm/irq.h +2
-0+struct dt_device_node ; +int platform_get_irq (const struct dt_device_node *device,int index);And I assume callers of this will appear later in the series.
Yes, it will be called ns16550_uart_dt_init() when CONFIG_NS16550 will be enabled for RISC-V.
Quoted 37 lines
Quoted 25 lines
xen/arch/riscv/irq.c +13
-0@@ -7,11 +7,52 @@*/ #include <xen/bug.h> +#include <xen/device_tree.h> +#include <xen/errno.h> #include <xen/init.h> #include <xen/irq.h> static irq_desc_t irq_desc[NR_IRQS ]; +static bool irq_validate_new_type (unsigned int curr,unsigned int new) +{ +return (curr ==IRQ_TYPE_INVALID || curr == new ); +} + +static int irq_set_type (unsigned int irq,unsigned int type) +{ +unsigned long flags; +struct irq_desc *desc =irq_to_desc (irq); +int ret = -EBUSY ; +xen/arch/riscv/irq.c line 37
Quoted 10 lines
xen/arch/riscv/irq.c +10
-0+spin_lock_irqsave (&desc->lock, flags); + +if ( !irq_validate_new_type (desc->arch.type, type) ) + goto err; + + desc->arch.type = type; + + ret =0 ; + +err:Labels indented by at least one blank please.
Quoted 16 lines
Quoted 4 lines
xen/arch/riscv/irq.c +4
-0+spin_unlock_irqrestore (&desc->lock, flags); + +return ret; +}xen/arch/riscv/irq.c line 51
Quoted 10 lines
xen/arch/riscv/irq.c +10
-0+ +int platform_get_irq (const struct dt_device_node *device,int index) +{ +struct dt_irq dt_irq; + +if (dt_device_get_irq (device, index, &dt_irq) ) +return -1 ; + +if (irq_set_type (dt_irq.irq, dt_irq.type) ) +return -1 ;Can you please return proper -E... values, perhaps ones coming back from the functions called?
Sure, I will use -EINVAL. (or ,perhaps, it will be better to introduce ret and return what dt_device_get_irq()/irq_set_type() returns in the case of failure.
~ Oleksii
Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()
Quoted 6 lines · Oleksii Kurochko
Quoted 5 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +0
-0@@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = { .hw_version = INTC_APLIC, };
Quoted 7 lines
Quoted 4 lines
xen/arch/riscv/aplic.c static struct intc_info __ro_after_init aplic_info = { · line 25
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,As you start adding functions calling indirectly, please consider adding
cf_checkright away, even if right now this has no effect on RISC-V. That'll save you from going through the entire RISC-V subtree later on to find them all.Sure. I thought that it is a feature for x86 as I haven't seen such attribute for Arm and RISC-V in GCC manuals.
And that looks to be correct. I was under the (admittedly vague) impression Arm64 had something equivalent in hardware, which then merely needs enabling in the compiler. Not sure about RISC-V, but seeing the endless flow of patches enabling new extensions in binutils, it would perhaps even be surprising if nothing along these lines was already in the works somewhere.
Jan
Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()
Quoted 11 lines · Jan Beulich
Quoted 10 lines · Oleksii Kurochko
Quoted 9 lines · Jan Beulich
xen/arch/riscv/aplic.c static struct intc_info __ro_after_init aplic_info = { · line 25
Quoted 5 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +1
-0@@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = { .hw_version = INTC_APLIC, }; +static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,As you start adding functions calling indirectly, please consider adding
cf_checkright away, even if right now this has no effect on RISC-V. That'll save you from going through the entire RISC-V subtree later on to find them all.
Quoted 7 lines
Quoted 2 lines
Sure. I thought that it is a feature for x86 as I haven't seen such attribute for Arm and RISC-V in GCC manuals.
And that looks to be correct. I was under the (admittedly vague) impression Arm64 had something equivalent in hardware, which then merely needs enabling in the compiler. Not sure about RISC-V, but seeing the endless flow of patches enabling new extensions in binutils, it would perhaps even be surprising if nothing along these lines was already in the works somewhere.
You are right, something is already in the work:
-https://github.com/riscv/riscv-cfi -https://lore.kernel.org/lkml/[email protected]/ (interesting that
they are enabling it for U-mode)
~ Oleksii
PATCH 6/14 xen/riscv: riscv_of_processor_hartid() implementation
In reply to Oleksii KurochkoImplements riscv_of_processor_hartid() to get the hart ID of the given device tree node and do some checks if CPU is available and given device tree node has proper riscv,isa property.
As a helper function of_get_cpu_hwid() is introduced to deal specifically with reg propery of a CPU device node.
--- xen/arch/riscv/include/asm/smp.h | 3 ++ xen/arch/riscv/smpboot.c | 68 ++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+)
xen/arch/riscv/include/asm/smp.h +3 -0
@@ -26,6 +26,9 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,void setup_tp (unsigned int cpuid); +struct dt_device_node ; +int riscv_of_processor_hartid (struct dt_device_node *node,unsigned long *hart); +void smp_clear_cpu_maps (void );#endif
xen/arch/riscv/smpboot.c +68 -0
76 lines not on this page · raw message
Signature · 1 line
2.49.0
Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +0
-0@@ -13,3 +16,68 @@ void __init smp_clear_cpu_maps(void)cpumask_set_cpu (0 , &cpu_online_map);cpumask_copy (&cpu_present_map, &cpu_possible_map); }
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 28
Quoted 10 lines
xen/arch/riscv/smpboot.c +10
-0+ +/** +* of_get_cpu_hwid - Get the hardware ID from a CPU device node +* +* @cpun: CPU number(logical index) for which device node is required +* @thread: The local thread number to get the hardware ID for. +* +* Return: The hardware ID for the CPU node or ~0ULL if not found. +*/ +static uint64_t of_get_cpu_hwid (struct dt_device_node *cpun,unsigned int thread)
What does the "of" prefix stand for here? Looking at the function body I'm really at a loss. (I was first guessing something like OpenFirmware, but there's nothing here that would support that.)
As you're only fetching data - can cpun be pointer-to-const?
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37
Quoted 9 lines
xen/arch/riscv/smpboot.c +9
-0+{ +const __be32 *cell; +int ac; +uint32_t len; + + ac =dt_n_addr_cells (cpun); + cell =dt_get_property (cpun,"reg" , &len); +if ( !cell || !ac || ((sizeof (*cell) * ac * (thread +1 )) > len) ) +return ~0ULL ;
I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 40
Quoted 2 lines
xen/arch/riscv/smpboot.c +2
-0+ cell += ac * thread; +return dt_read_number (cell, ac);
Nit (you know what)
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 47
Quoted 5 lines
xen/arch/riscv/smpboot.c +5
-0+/* +* Returns the hart ID of the given device tree node, or -ENODEV if the node +* isn't an enabled and valid RISC-V hart node. +*/ +int riscv_of_processor_hartid (struct dt_device_node *node,unsigned long *hart)
Similar question as above: What's "of" and what significance does the "riscv" prefix have in RISC-V code?
Const-ness question again for "node".
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 58
Quoted 11 lines
xen/arch/riscv/smpboot.c +11
-0+{ +const char *isa; + +if ( !dt_device_is_compatible (node,"riscv" ) ) + { +printk ("Found incompatible CPU\n" ); +return -ENODEV ; + } + + *hart = (unsigned long )of_get_cpu_hwid (node,0 ); +if ( *hart == ~0UL )
While for RV64 this won't matter, the difference in types (uint64_t returned, unsigned long used) is still puzzling me. What's the deal?
Jan
Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation
Quoted 10 lines · Jan Beulich
Quoted 9 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +5
-0@@ -13,3 +16,68 @@ void __init smp_clear_cpu_maps(void)cpumask_set_cpu (0 , &cpu_online_map);cpumask_copy (&cpu_present_map, &cpu_possible_map); } + +/** +* of_get_cpu_hwid - Get the hardware ID from a CPU device node +* +* @cpun: CPU number(logical index) for which device node is required
Quoted 8 lines
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 28
Quoted 5 lines
xen/arch/riscv/smpboot.c +5
-0+* @thread: The local thread number to get the hardware ID for. +* +* Return: The hardware ID for the CPU node or ~0ULL if not found. +*/ +static uint64_t of_get_cpu_hwid (struct dt_device_node *cpun,unsigned int thread)What does the "of" prefix stand for here? Looking at the function body I'm really at a loss. (I was first guessing something like OpenFirmware, but there's nothing here that would support that.)
I copy this function from Linux kernel. But you are right, "of" means OpenFirmware or Open Firmware Device Tree and is used for the functions which work with device tree.
I'll rename to dt_get_cpu_hwid() to follow the naming of device tree's functions name in Xen.
Quoted 2 lines
As you're only fetching data - can cpun be pointer-to-const?
Sure, it can be. I'll update that.
Quoted 13 lines
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37
Quoted 10 lines
xen/arch/riscv/smpboot.c +9
-0+{ +const __be32 *cell; +int ac; +uint32_t len; + + ac =dt_n_addr_cells (cpun); + cell =dt_get_property (cpun,"reg" , &len); +if ( !cell || !ac || ((sizeof (*cell) * ac * (thread +1 )) > len) ) +return ~0ULL ;I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).
Based on the DT specification alone, the|reg| value could refer to either a CPU or a thread ID:
``` The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for the CPU/threads represented by the CPU node. If a CPU supports more than one thread (i.e. multiple streams of execution) the reg prop-erty is an array with 1 element per thread. ```
My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).
Interestingly, the Linux kernel always uses|thread = 0|.
We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|reg| property contains only one entry, representing the HART (CPU) ID:
``` Software can determine the number of threads by dividing the size of reg by the parent node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support is required. ```
Does that approach make sense, or should we stick with the current implementation?
Quoted 5 lines
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 40
Quoted 3 lines
xen/arch/riscv/smpboot.c +2
-0+ cell += ac * thread; +return dt_read_number (cell, ac);Nit (you know what)
Quoted 7 lines
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 47
Quoted 5 lines
xen/arch/riscv/smpboot.c +5
-0+/* +* Returns the hart ID of the given device tree node, or -ENODEV if the node +* isn't an enabled and valid RISC-V hart node. +*/ +int riscv_of_processor_hartid (struct dt_device_node *node,unsigned long *hart)Similar question as above: What's "of" and what significance does the "riscv" prefix have in RISC-V code?
I will drop usage of 'of' in Xen and change it to 'dt'.
Quoted 16 lines
Const-ness question again for "node".
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 58
Quoted 11 lines
xen/arch/riscv/smpboot.c +11
-0+{ +const char *isa; + +if ( !dt_device_is_compatible (node,"riscv" ) ) + { +printk ("Found incompatible CPU\n" ); +return -ENODEV ; + } + + *hart = (unsigned long )of_get_cpu_hwid (node,0 ); +if ( *hart == ~0UL )While for RV64 this won't matter, the difference in types (
uint64_treturned, unsigned long used) is still puzzling me. What's the deal?
No specific reason, just overlooked that moment. I think we could use just drop this cast.
The reason for uint64_t as a return type is that dt_read_number() returns u64.
~ Oleksii
Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation
Quoted 32 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37
Quoted 9 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +9
-0+{ +const __be32 *cell; +int ac; +uint32_t len; + + ac =dt_n_addr_cells (cpun); + cell =dt_get_property (cpun,"reg" , &len); +if ( !cell || !ac || ((sizeof (*cell) * ac * (thread +1 )) > len) ) +return ~0ULL ;I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).
Based on the DT specification alone, the|
reg| value could refer to either a CPU or a thread ID:``` The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for the CPU/threads represented by the CPU node. If a CPU supports more than one thread (i.e. multiple streams of execution) the reg prop-erty is an array with 1 element per thread. ```My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).Interestingly, the Linux kernel always uses|thread = 0|.
We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|
reg| property contains only one entry, representing the HART (CPU) ID:
Quoted 7 lines
``` Software can determine the number of threads by dividing the size of reg by the parent node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support is required. ```Does that approach make sense, or should we stick with the current implementation?
If extra enabling is required to make multi-thread CPUs work, then panic()ing (not so much ASSERT()ing) may make sense, for the time being. Better would be if we could use all threads in a system right away.
Jan
Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation
Quoted 38 lines · Jan Beulich
Quoted 37 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37
Quoted 9 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +9
-0+{ +const __be32 *cell; +int ac; +uint32_t len; + + ac =dt_n_addr_cells (cpun); + cell =dt_get_property (cpun,"reg" , &len); +if ( !cell || !ac || ((sizeof (*cell) * ac * (thread +1 )) > len) ) +return ~0ULL ;I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).
Based on the DT specification alone, the|
reg| value could refer to either a CPU or a thread ID:``` The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for the CPU/threads represented by the CPU node. If a CPU supports more than one thread (i.e. multiple streams of execution) the reg prop-erty is an array with 1 element per thread. ```My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).Interestingly, the Linux kernel always uses|thread = 0|.
We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|
reg| property contains only one entry, representing the HART (CPU) ID:``` Software can determine the number of threads by dividing the size of reg by the parent node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support is required. ```
Quoted 4 lines
Quoted 1 line
Does that approach make sense, or should we stick with the current implementation?
If extra enabling is required to make multi-thread CPUs work, then panic()ing (not so much ASSERT()ing) may make sense, for the time being. Better would be if we could use all threads in a system right away.
Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id by passing thread argument (thread -> the local thread number to get the hardware ID for). So by calling:
dt_get_cpu_hwid(cpu0, 0) -> it will return hardware id of thread 0 of cpu0 dt_get_cpu_hwid(cpu0, 1) -> it will return hardware id of thread 1 of cpu0
...
In our case we assume that SMP isn't supported so that is why it is used only dt_get_cpu_hwid(cpu0, 0).
If one day, SMP will be enabled then it will be needed to change a callers of dt_get_cpu_hwid().
I will add a check in the caller.
~ Oleksii
Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation
Quoted 25 lines · Oleksii Kurochko
Quoted 23 lines · Jan Beulich
Quoted 22 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37
Quoted 9 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +9
-0+{ +const __be32 *cell; +int ac; +uint32_t len; + + ac =dt_n_addr_cells (cpun); + cell =dt_get_property (cpun,"reg" , &len); +if ( !cell || !ac || ((sizeof (*cell) * ac * (thread +1 )) > len) ) +return ~0ULL ;I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).
Based on the DT specification alone, the|
reg| value could refer to either a CPU or a thread ID:``` The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for the CPU/threads represented by the CPU node. If a CPU supports more than one thread (i.e. multiple streams of execution) the reg prop-erty is an array with 1 element per thread. ```
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).
Note the terminology ("CPU") you used here.
Quoted 22 lines
Quoted 10 lines
Quoted 10 lines
Interestingly, the Linux kernel always uses|thread = 0|.
We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|
reg| property contains only one entry, representing the HART (CPU) ID:``` Software can determine the number of threads by dividing the size of reg by the parent node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support is required. ```Quoted 4 lines
Quoted 1 line
Does that approach make sense, or should we stick with the current implementation?
If extra enabling is required to make multi-thread CPUs work, then panic()ing (not so much ASSERT()ing) may make sense, for the time being. Better would be if we could use all threads in a system right away.
Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id by passing
threadargument (thread-> the local thread number to get the hardware ID for). So by calling:
dt_get_cpu_hwid(cpu0, 0)-> it will return hardware id of thread 0 of cpu0dt_get_cpu_hwid(cpu0, 1)-> it will return hardware id of thread 1 of cpu0
...
Quoted 3 lines
In our case we assume that SMP isn't supported so that is why it is used only
dt_get_cpu_hwid(cpu0, 0).If one day, SMP will be enabled then it will be needed to change a callers of
dt_get_cpu_hwid().
I assume you meant SMT in both places you wrote SMP? But my main point here is:
If enumeration gives you "thread <N> of core <M>" (using x86 terminology), you need to be quite careful with what you call "CPU". Things need to be entirely unambiguous, taking into account what internally in (common code) Xen we call a "CPU". You certainly may call "CPU" what is a collection of threads / harts, but you then need to clarify this in a prominent comment somewhere, and you need to be entirely consistent throughout the RISC-V sub-tree.
Jan
Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation
Quoted 52 lines · Jan Beulich
Quoted 24 lines · Oleksii Kurochko
Quoted 23 lines · Jan Beulich
Quoted 22 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich
xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37
Quoted 9 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +9
-0+{ +const __be32 *cell; +int ac; +uint32_t len; + + ac =dt_n_addr_cells (cpun); + cell =dt_get_property (cpun,"reg" , &len); +if ( !cell || !ac || ((sizeof (*cell) * ac * (thread +1 )) > len) ) +return ~0ULL ;I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).
Based on the DT specification alone, the|
reg| value could refer to either a CPU or a thread ID:``` The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for the CPU/threads represented by the CPU node. If a CPU supports more than one thread (i.e. multiple streams of execution) the reg prop-erty is an array with 1 element per thread. ```Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).Note the terminology ("CPU") you used here.
Quoted 21 lines
Quoted 10 lines
Quoted 10 lines
Interestingly, the Linux kernel always uses|thread = 0|.
We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|
reg| property contains only one entry, representing the HART (CPU) ID:``` Software can determine the number of threads by dividing the size of reg by the parent node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support is required. ```Quoted 4 lines
Quoted 1 line
Does that approach make sense, or should we stick with the current implementation?
If extra enabling is required to make multi-thread CPUs work, then panic()ing (not so much ASSERT()ing) may make sense, for the time being. Better would be if we could use all threads in a system right away.
Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id by passing
threadargument (thread-> the local thread number to get the hardware ID for). So by calling:
dt_get_cpu_hwid(cpu0, 0)-> it will return hardware id of thread 0 of cpu0dt_get_cpu_hwid(cpu0, 1)-> it will return hardware id of thread 1 of cpu0
...
Quoted 4 lines
Quoted 3 lines
In our case we assume that SMP isn't supported so that is why it is used only
dt_get_cpu_hwid(cpu0, 0).If one day, SMP will be enabled then it will be needed to change a callers of
dt_get_cpu_hwid().I assume you meant SMT in both places you wrote SMP?
Yes, it should be SMT.
Quoted 7 lines
But my main point here is:
If enumeration gives you "thread <N> of core <M>" (using x86 terminology), you need to be quite careful with what you call "CPU". Things need to be entirely unambiguous, taking into account what internally in (common code) Xen we call a "CPU". You certainly may call "CPU" what is a collection of threads / harts, but you then need to clarify this in a prominent comment somewhere, and you need to be entirely consistent throughout the RISC-V sub-tree.
╭────────────────────╮ │ CPU │ ← 1 physical processor (chip) │ ┌───────┬─────────┐ │ │ │ Core 0│ Core 1 │ │ ← 2 cores (for example) │ │ ┌──┬──┐ ┌──┬──┐ │ │ │ │Thr0 Thr1 Thr0 Thr1│ ← 2 threads on each core (SMT) │ └───────┴─────────┘ │ ╰────────────────────╯
I want to double check what Xen call a "CPU". I thought that Xen uses word CPU to describe a core, right?
What you wrote above "thread <N> of core <M> (using x86 terminology)" is also correlated with RISC-V terminology:
A component is termed a core if it contains an independent instruction fetch unit. A RISC-V-compatible core might support multiple RISC-V-compatible hardware threads, or harts, through multithreading
I checked RISC-V's DTS binding and it seems it is a little bit contradictory to DTS spec, where it is mentioned that reg property is used to describe how many threads a cpu has when SMP is used, but in RISC-V's dts binding they are describing a hardware execution context:
This document uses some terminology common to the RISC-V community
that is not widely used, the definitions of which are listed here:
hart: A hardware execution context, which contains all the state mandated by the RISC-V ISA: a PC and some registers. This terminology is designed to disambiguate software's view of execution contexts from any particular microarchitectural implementation strategy. For example, an Intel laptop containing one socket with two cores, each of which has two hyperthreads, could be described as having four harts.
So in RISC-V's DTS binding they are describing only hardware threads what makes things more confusing in terms what kind terminology from Xen point of view should be used.
And based on what is written in RISC-V's dts binding:
For example, an Intel laptop containing one socket with
two cores, each of which has two hyperthreads, could be described as
having four harts.
It would be more logical to drop 'thread' argument of riscv_of_get_cpu_hwid(const struct dt_device_node *cpun). And then the question is what to do with the name of variable cpun? As it could be still confusing. Or, at least, I can add the comment that CPUn in terms of RISC-V means hart (hardware thread). And then will it be needed to add such comment for each usage of word "CPU"?
~ Oleksii
Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation
Quoted 77 lines · Oleksii Kurochko
Quoted 52 lines · Jan Beulich
Quoted 24 lines · Oleksii Kurochko
Quoted 23 lines · Jan Beulich
Quoted 22 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich
Review
Quoted 9 lines · Oleksii Kurochko
(quoted diff) +9
-0+{ + const __be32 *cell; + int ac; + uint32_t len; + + ac = dt_n_addr_cells(cpun); + cell = dt_get_property(cpun, "reg", &len); + if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) ) + return ~0ULL;I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).
Based on the DT specification alone, the|
reg| value could refer to either a CPU or a thread ID:``` The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for the CPU/threads represented by the CPU node. If a CPU supports more than one thread (i.e. multiple streams of execution) the reg prop-erty is an array with 1 element per thread. ```Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).Note the terminology ("CPU") you used here.
Quoted 21 lines
Quoted 10 lines
Quoted 10 lines
Interestingly, the Linux kernel always uses|thread = 0|.
We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|
reg| property contains only one entry, representing the HART (CPU) ID:``` Software can determine the number of threads by dividing the size of reg by the parent node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support is required. ```Quoted 4 lines
Quoted 1 line
Does that approach make sense, or should we stick with the current implementation?
If extra enabling is required to make multi-thread CPUs work, then panic()ing (not so much ASSERT()ing) may make sense, for the time being. Better would be if we could use all threads in a system right away.
Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id by passing
threadargument (thread-> the local thread number to get the hardware ID for). So by calling:
dt_get_cpu_hwid(cpu0, 0)-> it will return hardware id of thread 0 of cpu0dt_get_cpu_hwid(cpu0, 1)-> it will return hardware id of thread 1 of cpu0
...Quoted 4 lines
Quoted 3 lines
In our case we assume that SMP isn't supported so that is why it is used only
dt_get_cpu_hwid(cpu0, 0).If one day, SMP will be enabled then it will be needed to change a callers of
dt_get_cpu_hwid().I assume you meant SMT in both places you wrote SMP?
Yes, it should be SMT.
Quoted 7 lines
But my main point here is:
If enumeration gives you "thread <N> of core <M>" (using x86 terminology), you need to be quite careful with what you call "CPU". Things need to be entirely unambiguous, taking into account what internally in (common code) Xen we call a "CPU". You certainly may call "CPU" what is a collection of threads / harts, but you then need to clarify this in a prominent comment somewhere, and you need to be entirely consistent throughout the RISC-V sub-tree.╭────────────────────╮ │ CPU │ ← 1 physical processor (chip) │ ┌───────┬─────────┐ │ │ │ Core 0│ Core 1 │ │ ← 2 cores (for example) │ │ ┌──┬──┐ ┌──┬──┐ │ │ │ │Thr0 Thr1 Thr0 Thr1│ ← 2 threads on each core (SMT) │ └───────┴─────────┘ │ ╰────────────────────╯
Quoted 2 lines
I want to double check what Xen call a "CPU". I thought that Xen uses word CPU to describe a core, right?
No, see e.g. cpumask.h - it's a hart (as per below) that we internally describe as CPU (leaving aside potentially ambiguous comments here and there, which is what I'd like to prevent from the start for RISC-V).
Quoted 24 lines
What you wrote above "thread <N> of core <M> (using x86 terminology)" is also correlated with RISC-V terminology:
A component is termed a core if it contains an independent instruction fetch unit. A RISC-V-compatible core might support multiple RISC-V-compatible hardware threads, or harts, through multithreadingI checked RISC-V's DTS binding and it seems it is a little bit contradictory to DTS spec, where it is mentioned that reg property is used to describe how many threads a cpu has when SMP is used, but in RISC-V's dts binding they are describing a hardware execution context:
This document uses some terminology common to the RISC-V community
that is not widely used, the definitions of which are listed here:hart: A hardware execution context, which contains all the state mandated by the RISC-V ISA: a PC and some registers. This terminology is designed to disambiguate software's view of execution contexts from any particular microarchitectural implementation strategy. For example, an Intel laptop containing one socket with two cores, each of which has two hyperthreads, could be described as having four harts.
So in RISC-V's DTS binding they are describing only hardware threads what makes things more confusing in terms what kind terminology from Xen point of view should be used.
Quoted 7 lines
And based on what is written in RISC-V's dts binding:
For example, an Intel laptop containing one socket with
two cores, each of which has two hyperthreads, could be described as
having four harts.
It would be more logical to drop 'thread' argument ofriscv_of_get_cpu_hwid(const struct dt_device_node *cpun). And then the question is what to do with the name of variable cpun? As it could be still confusing. Or, at least, I can add the comment that CPUn in terms of RISC-V means hart (hardware thread).
If it's the normal thing you call a CPU, I don't think much commentary is necessary. Then please just make sure you don't call anything else "CPU", especially in identifiers.
Quoted 2 lines
And then will it be needed to
add such comment for each usage of word "CPU"?
No, that would go too far in any event. Hence why I said "clarify this in a prominent comment somewhere".
Jan
PATCH 7/14 xen/riscv: Introduce intc_hw_operations abstraction
In reply to Oleksii KurochkoIntroduce the intc_hw_operations structure to encapsulate interrupt controller-specific data and operations. This structure includes:
- A pointer to interrupt controller information (intc_info)
- Callbacks to initialize the controller and set IRQ type/priority
- A reference to an interupt controller descriptor (host_irq_type)
Also introduce generic helper functions:
- intc_init(): Initializes the interrupt controller
- register_intc_ops(): Registers the intc_hw_operations implementation
- intc_route_irq_to_xen(): Configures IRQ routing to Xen, setting handler, type, and priority
Most of these functions act as thin wrappers around the corresponding callbacks in intc_hw_operations.
This abstraction lays the groundwork for supporting multiple interrupt controller types (e.g., PLIC, APLIC) in extensible way.
This patch is based on the changes from [1].
[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7cfb4bd4748ca268142497ac5c327d2766fb342d
--- xen/arch/riscv/include/asm/intc.h | 23 ++++++++++++++ xen/arch/riscv/intc.c | 51 +++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+)
xen/arch/riscv/include/asm/intc.h +23 -0
@@ -17,6 +17,29 @@ struct intc_info { const struct dt_device_node *node; }; +struct intc_hw_operations { +/* Hold intc hw information */ +const struct intc_info *info; +/* Initialize the intc and the boot CPU */ +int (*init)(void ); + +/* hw_irq_controller to enable/disable/eoi host irq */ +hw_irq_controller *host_irq_type; + +/* Set IRQ type */ +void (*set_irq_type)(struct irq_desc *desc,unsigned int type); +/* Set IRQ priority */ +void (*set_irq_priority)(struct irq_desc *desc,unsigned int priority); + +}; +void intc_preinit (void ); +void intc_init (void ); + +void register_intc_ops (const struct intc_hw_operations *ops); + +struct irq_desc ; +void intc_route_irq_to_xen (struct irq_desc *desc,unsigned int priority); +#endif /* ASM__RISCV__INTERRUPT_CONTOLLER_H */
xen/arch/riscv/intc.c +51 -0
63 lines not on this page · raw message
Signature · 1 line
2.49.0
Re: [PATCH v1 07/14] xen/riscv: Introduce intc_hw_operations abstraction
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/intc.h +0
-0@@ -17,6 +17,29 @@ struct intc_info {const struct dt_device_node *node;
xen/arch/riscv/include/asm/intc.h struct intc_info { · line 27
Quoted 10 lines
xen/arch/riscv/include/asm/intc.h +8
-0}; +struct intc_hw_operations { +/* Hold intc hw information */ +const struct intc_info *info; +/* Initialize the intc and the boot CPU */ +int (*init)(void ); + +/* hw_irq_controller to enable/disable/eoi host irq */ +hw_irq_controller *host_irq_type;
Pointer-to-const perhaps?
Quoted 4 lines
xen/arch/riscv/include/asm/intc.h +4
-0+/* Set IRQ type */ +void (*set_irq_type)(struct irq_desc *desc,unsigned int type); +/* Set IRQ priority */ +void (*set_irq_priority)(struct irq_desc *desc,unsigned int priority);
xen/arch/riscv/include/asm/intc.h struct intc_info { · line 42
Quoted 10 lines
xen/arch/riscv/include/asm/intc.h +8
-0+ +}; +void intc_preinit (void ); +void intc_init (void ); + +void register_intc_ops (const struct intc_hw_operations *ops); + +struct irq_desc ;
If it's needed here at all, it needs to move up, as some of the hook pointers already use the type.
Quoted 6 lines
xen/arch/riscv/intc.c +0
-0@@ -1,9 +1,21 @@/* SPDX-License-Identifier: GPL-2.0-only */ #include <xen/acpi.h>
xen/arch/riscv/intc.c line 13
Quoted 10 lines
xen/arch/riscv/intc.c +7
-0+#include <xen/bug.h> #include <xen/device_tree.h> #include <xen/init.h> +#include <xen/irq.h> #include <xen/lib.h> +#include <xen/spinlock.h> + +#include <asm/intc.h> + +static const struct intc_hw_operations *intc_hw_ops;
__ro_after_init perhaps?
xen/arch/riscv/intc.c line 15
Quoted 2 lines
xen/arch/riscv/intc.c +2
-0+ +void register_intc_ops (const struct intc_hw_operations *ops)
__init perhaps?
Quoted 38 lines
xen/arch/riscv/intc.c +31
-0+{ + intc_hw_ops = ops; +}void __init intc_preinit (void ) {@@ -12,3 +24,42 @@ void __init intc_preinit(void)else panic ("ACPI interrupt controller preinit() isn't implemented\n" ); } + +void __init intc_init (void ) +{ +ASSERT (intc_hw_ops); + +if ( intc_hw_ops->init () ) +panic ("Failed to initialize the interrupt controller drivers\n" ); +} + +/* desc->irq needs to be disabled before calling this function */ +static void intc_set_irq_type (struct irq_desc *desc,unsigned int type) +{ +ASSERT (test_bit (_IRQ_DISABLED, &desc->status)); +ASSERT (spin_is_locked (&desc->lock)); +ASSERT (type !=IRQ_TYPE_INVALID ); +ASSERT (intc_hw_ops && intc_hw_ops->set_irq_type); + + intc_hw_ops->set_irq_type (desc, type); +} + +static void intc_set_irq_priority (struct irq_desc *desc,unsigned int priority) +{ +ASSERT (intc_hw_ops && intc_hw_ops->set_irq_priority); + + intc_hw_ops->set_irq_priority (desc, priority); +} + +void intc_route_irq_to_xen (struct irq_desc *desc,unsigned int priority)
xen/arch/riscv/intc.c void __init intc_preinit(void) · line 64
Quoted 10 lines
xen/arch/riscv/intc.c +10
-0+{ +ASSERT (test_bit (_IRQ_DISABLED, &desc->status)); +ASSERT (spin_is_locked (&desc->lock)); +/* Can't route interrupts that don't exist */ +ASSERT (intc_hw_ops && desc->irq < intc_hw_ops->info->nr_irqs); + + desc->handler = intc_hw_ops->host_irq_type; + +intc_set_irq_type (desc, desc->arch.type); +intc_set_irq_priority (desc, priority);
If these are going to remain the sole callers of the two functions, I'd question the need for the separate functions. Some of the assertions done there would then actually be redundant.
If not, is there a reason intc_set_irq_priority() doesn't have a lock-held check?
Jan
Re: [PATCH v1 07/14] xen/riscv: Introduce intc_hw_operations abstraction
Quoted 7 lines · Jan Beulich
Quoted 6 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/intc.h +0
-0@@ -17,6 +17,29 @@ struct intc_info {const struct dt_device_node *node; };
Quoted 9 lines
xen/arch/riscv/include/asm/intc.h struct intc_info { · line 27
Quoted 8 lines
xen/arch/riscv/include/asm/intc.h +8
-0+struct intc_hw_operations { +/* Hold intc hw information */ +const struct intc_info *info; +/* Initialize the intc and the boot CPU */ +int (*init)(void ); + +/* hw_irq_controller to enable/disable/eoi host irq */ +hw_irq_controller *host_irq_type;Pointer-to-const perhaps?
It could be pointer-to-const. I'll updat
Quoted 10 lines
Quoted 10 lines
xen/arch/riscv/include/asm/intc.h +7
-0+/* Set IRQ type */ +void (*set_irq_type)(struct irq_desc *desc,unsigned int type); +/* Set IRQ priority */ +void (*set_irq_priority)(struct irq_desc *desc,unsigned int priority); + +}; +void intc_preinit (void );
Quoted 7 lines
xen/arch/riscv/include/asm/intc.h struct intc_info { · line 42
Quoted 5 lines
xen/arch/riscv/include/asm/intc.h +5
-0+void intc_init (void ); + +void register_intc_ops (const struct intc_hw_operations *ops); + +struct irq_desc ;If it's needed here at all, it needs to move up, as some of the hook pointers already use the type.
It could be dropped, but then we would need to include|<xen/irq.h>|, and I figured the fewer includes we have, the better.
Quoted 19 lines
Quoted 7 lines
xen/arch/riscv/intc.c +0
-0@@ -1,9 +1,21 @@/* SPDX-License-Identifier: GPL-2.0-only */ #include <xen/acpi.h> xen/arch/riscv/intc.c line 13
Quoted 10 lines
xen/arch/riscv/intc.c +7
-0+#include <xen/bug.h> #include <xen/device_tree.h> #include <xen/init.h> +#include <xen/irq.h> #include <xen/lib.h> +#include <xen/spinlock.h> + +#include <asm/intc.h> + +static const struct intc_hw_operations *intc_hw_ops;
__ro_after_initperhaps?
Quoted 3 lines
xen/arch/riscv/intc.c line 15
Quoted 2 lines
xen/arch/riscv/intc.c +2
-0+ +void register_intc_ops (const struct intc_hw_operations *ops)
__initperhaps?
For both, yes, it should be __ro_after_init and __init.
Quoted 44 lines
Quoted 44 lines
xen/arch/riscv/intc.c +36
-0+{ + intc_hw_ops = ops; +}void __init intc_preinit (void ) {@@ -12,3 +24,42 @@ void __init intc_preinit(void)else panic ("ACPI interrupt controller preinit() isn't implemented\n" ); } + +void __init intc_init (void ) +{ +ASSERT (intc_hw_ops); + +if ( intc_hw_ops->init () ) +panic ("Failed to initialize the interrupt controller drivers\n" ); +} + +/* desc->irq needs to be disabled before calling this function */ +static void intc_set_irq_type (struct irq_desc *desc,unsigned int type) +{ +ASSERT (test_bit (_IRQ_DISABLED, &desc->status)); +ASSERT (spin_is_locked (&desc->lock)); +ASSERT (type !=IRQ_TYPE_INVALID ); +ASSERT (intc_hw_ops && intc_hw_ops->set_irq_type); + + intc_hw_ops->set_irq_type (desc, type); +} + +static void intc_set_irq_priority (struct irq_desc *desc,unsigned int priority) +{ +ASSERT (intc_hw_ops && intc_hw_ops->set_irq_priority); + + intc_hw_ops->set_irq_priority (desc, priority); +} + +void intc_route_irq_to_xen (struct irq_desc *desc,unsigned int priority) +{ +ASSERT (test_bit (_IRQ_DISABLED, &desc->status)); +ASSERT (spin_is_locked (&desc->lock)); +/* Can't route interrupts that don't exist */ +ASSERT (intc_hw_ops && desc->irq < intc_hw_ops->info->nr_irqs);
Quoted 8 lines
xen/arch/riscv/intc.c void __init intc_preinit(void) · line 64
Quoted 5 lines
xen/arch/riscv/intc.c +5
-0+ + desc->handler = intc_hw_ops->host_irq_type; + +intc_set_irq_type (desc, desc->arch.type); +intc_set_irq_priority (desc, priority);If these are going to remain the sole callers of the two functions, I'd question the need for the separate functions. Some of the assertions done there would then actually be redundant.
Likely they will be reused for similar function but for guest to route an interrupt to a guest. And, perhaps, only intc_set_irq_type() will be re-used outside intc_route_irq_to_{xen,guest}...
Quoted 2 lines
If not, is there a reason
intc_set_irq_priority()doesn't have a lock-held check?
Do you mean this one: ASSERT(spin_is_locked(&desc->lock));
...
And it is the reason why intc_set_irq_priority() doesn't have this check as it is checked in the caller.
But I think just to be on a safe side, it could be better to add this check to intc_set_irq_priority().
~ Oleksii
PATCH 8/14 xen/riscv: imsic_init() implementation
In reply to Oleksii Kurochkoimsic_init() is introduced to parse device tree node, which has the following bindings [2], and based on the parsed information update IMSIC configuration which is stored in imsic_cfg.
The following helpers are introduces for imsic_init() usage:
- imsic_parse_node() parses IMSIC node from DTS
- imsic_get_parent_hartid() returns the hart ( CPU ) ID of the given device tree node.
This patch is based on the code from [1].
Since Microchip originally developed imsic.{c,h}, an internal discussion with them led to the decision to use the MIT license.
[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/0b1a94f2bc3bb1a81cd26bb75f0bf578f84cb4d4
[2] https://elixir.bootlin.com/linux/v6.12/source/Documentation/devicetree/bindings/interrupt-controller/riscv,imsics.yaml
--- xen/arch/riscv/Makefile | 1 + xen/arch/riscv/imsic.c | 286 +++++++++++++++++++++++++++++ xen/arch/riscv/include/asm/imsic.h | 66 +++++++ 3 files changed, 353 insertions(+) create mode 100644 xen/arch/riscv/imsic.c create mode 100644 xen/arch/riscv/include/asm/imsic.h
xen/arch/riscv/Makefile +1 -0
@@ -2,6 +2,7 @@ obj-y += aplic.o obj-y += cpufeature.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-y += entry.o +obj-y += imsic.o obj-y += intc.o obj-y += irq.o obj-y += mm.o
xen/arch/riscv/imsic.c new file +286 -0
286 lines not on this page · raw message
xen/arch/riscv/include/asm/imsic.h new file +66 -0
66 lines not on this page · raw message
Signature · 1 line
2.49.0
Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation
xen/arch/riscv/imsic.c line 9
Quoted 12 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c new file +9
-0@@ -0,0 +1,286 @@ +/* SPDX-License-Identifier: MIT */ + +/* +* xen/arch/riscv/imsic.c +* +* RISC-V Incoming MSI Controller support +* +* (c) 2023 Microchip Technology Inc. +* (c) 2024 Vates
No 2025 here (if already the years matter)?
Quoted 4 lines
xen/arch/riscv/imsic.c new file +4
-0+*/ + +#include <xen/const.h> +#include <xen/device_tree.h>
xen/arch/riscv/imsic.c line 23
Quoted 10 lines
xen/arch/riscv/imsic.c new file +10
-0+#include <xen/errno.h> +#include <xen/init.h> +#include <xen/macros.h> +#include <xen/xmalloc.h> + +#include <asm/imsic.h> + +static struct imsic_config imsic_cfg; + +const struct imsic_config *imsic_get_config (void )
Does this need to return a pointer to non-const?
xen/arch/riscv/imsic.c line 36
Quoted 13 lines
xen/arch/riscv/imsic.c new file +13
-0+{ +return &imsic_cfg; +} + +static int __init imsic_get_parent_hartid (struct dt_device_node *node, +unsigned int index, +unsigned long *hartid) +{ +int res; +unsigned long hart; +struct dt_phandle_args args; + +/* Try the new-style interrupts-extended first */
The comment says "first", but then ...
xen/arch/riscv/imsic.c line 48
Quoted 12 lines
xen/arch/riscv/imsic.c new file +12
-0+ res =dt_parse_phandle_with_args (node,"interrupts-extended" , +"#interrupt-cells" , index, &args); +if ( !res ) + { + res =riscv_of_processor_hartid (args.np->parent, &hart); +if ( res <0 ) +return -EINVAL ; + + *hartid = hart; + } +return res; +}
... nothing else is being tried.
Also, nit: Blank line please ahead of the main "return" of a function.
Further - any particular reason to discard riscv_of_processor_hartid()'s error code on the error path?
xen/arch/riscv/imsic.c line 50
Quoted 2 lines
xen/arch/riscv/imsic.c new file +2
-0+ +
Nit: No double blank lines please (and I wish I wouldn't need to repeat this any further).
Quoted 9 lines
xen/arch/riscv/imsic.c new file +9
-0+static int imsic_parse_node (struct dt_device_node *node, +unsigned int *nr_parent_irqs) +{ +int rc; +unsigned int tmp; +paddr_t base_addr; + +/* Find number of parent interrupts */ + *nr_parent_irqs =dt_number_of_irq (node);
xen/arch/riscv/imsic.c line 69
Quoted 10 lines
xen/arch/riscv/imsic.c new file +10
-0+if ( !*nr_parent_irqs ) + { +printk (XENLOG_ERR "%s: no parent irqs available\n" , node->name); +return -ENOENT ; + } + +/* Find number of guest index bits in MSI address */ + rc =dt_property_read_u32 (node,"riscv,guest-index-bits" , + &imsic_cfg.guest_index_bits); +if ( !rc )
It is confusing to store a bool return value in a local "int" variable, just to then use it as boolean. Is the local var needed at all here?
Quoted 10 lines
xen/arch/riscv/imsic.c new file +10
-0+ imsic_cfg.guest_index_bits =0 ; + tmp =BITS_PER_LONG -IMSIC_MMIO_PAGE_SHIFT ; +if ( tmp < imsic_cfg.guest_index_bits ) + { +printk (XENLOG_ERR "%s: guest index bits too big\n" , node->name); +return -ENOENT ; + } + +/* Find number of HART index bits */ + rc = dt_property_read_u32(node, "riscv,hart-index-bits",
xen/arch/riscv/imsic.c line 89
Quoted 10 lines
xen/arch/riscv/imsic.c new file +10
-0+ &imsic_cfg.hart_index_bits); +if ( !rc ) + { +/* Assume default value */ + imsic_cfg.hart_index_bits =fls (*nr_parent_irqs); +if (BIT (imsic_cfg.hart_index_bits,UL ) < *nr_parent_irqs ) + imsic_cfg.hart_index_bits++; + } + tmp =BITS_PER_LONG -IMSIC_MMIO_PAGE_SHIFT - + imsic_cfg.guest_index_bits;
tmp -= imsic_cfg.guest_index_bits;
? (And then similarly further down.)
Quoted 38 lines
xen/arch/riscv/imsic.c new file +38
-0+if ( tmp < imsic_cfg.hart_index_bits ) + { +printk (XENLOG_ERR "%s: HART index bits too big\n" , node->name); +return -ENOENT ; + } + +/* Find number of group index bits */ + rc =dt_property_read_u32 (node,"riscv,group-index-bits" , + &imsic_cfg.group_index_bits); +if ( !rc ) + imsic_cfg.group_index_bits =0 ; + tmp =BITS_PER_LONG -IMSIC_MMIO_PAGE_SHIFT - + imsic_cfg.guest_index_bits - imsic_cfg.hart_index_bits; +if ( tmp < imsic_cfg.group_index_bits ) + { +printk (XENLOG_ERR "%s: group index bits too big\n" , node->name); +return -ENOENT ; + } + +/* Find first bit position of group index */ + tmp =IMSIC_MMIO_PAGE_SHIFT *2 ; + rc =dt_property_read_u32 (node,"riscv,group-index-shift" , + &imsic_cfg.group_index_shift); +if ( !rc ) + imsic_cfg.group_index_shift = tmp; +if ( imsic_cfg.group_index_shift < tmp ) + { +printk (XENLOG_ERR "%s: group index shift too small\n" , node->name); +return -ENOENT ; + } + tmp = imsic_cfg.group_index_bits + imsic_cfg.group_index_shift -1 ; +if ( tmp >=BITS_PER_LONG ) + { +printk (XENLOG_ERR "%s: group index shift too big\n" , node->name); +return -EINVAL ; + } + +/* Find number of interrupt identities */
xen/arch/riscv/imsic.c line 137
Quoted 10 lines
xen/arch/riscv/imsic.c new file +10
-0+ rc =dt_property_read_u32 (node,"riscv,num-ids" , &imsic_cfg.nr_ids); +if ( !rc ) + { +printk (XENLOG_ERR "%s: number of interrupt identities not found\n" , + node->name); +return -ENOENT ; + } + + if ( (imsic_cfg.nr_ids < IMSIC_MIN_ID) || + (imsic_cfg.nr_ids >= IMSIC_MAX_ID) ||
Something named "max" normally wants to decribe the highest valid value, not the first out-of-range one.
xen/arch/riscv/imsic.c line 148
Quoted 11 lines
xen/arch/riscv/imsic.c new file +11
-0+ ((imsic_cfg.nr_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) ) + { +printk (XENLOG_ERR "%s: invalid number of interrupt identities\n" , + node->name); +return -EINVAL ; + } + +/* Compute base address */ + imsic_cfg.nr_mmios =0 ; + rc =dt_device_get_address (node, imsic_cfg.nr_mmios, &base_addr,NULL ); +if (rc)
Nit: Style.
xen/arch/riscv/imsic.c line 151
Quoted 3 lines
xen/arch/riscv/imsic.c new file +3
-0+ { +printk (XENLOG_ERR "%s: first MMIO resource not found\n" , node->name); +return -EINVAL ;
Discarding "rc" again?
xen/arch/riscv/imsic.c line 159
Quoted 8 lines
xen/arch/riscv/imsic.c new file +8
-0+ } + + imsic_cfg.base_addr = base_addr; + imsic_cfg.base_addr &= ~(BIT (imsic_cfg.guest_index_bits + + imsic_cfg.hart_index_bits + +IMSIC_MMIO_PAGE_SHIFT ,UL ) -1 ); + imsic_cfg.base_addr &= ~((BIT (imsic_cfg.group_index_bits,UL ) -1 ) << + imsic_cfg.group_index_shift);
Besides indentation being bogus here, why is it that you need to mask bits off of the value read from DT? Wouldn't the expectation be that you get back the true base address?
xen/arch/riscv/imsic.c line 164
Quoted 4 lines
xen/arch/riscv/imsic.c new file +4
-0+/* Find number of MMIO register sets */ + imsic_cfg.nr_mmios++; +while ( !dt_device_get_address (node, imsic_cfg.nr_mmios, &base_addr,NULL ) ) + imsic_cfg.nr_mmios++;
And the base addresses of these aren't of interest? Oh, I see they're fetched again further down.
Also - use do-while here?
Quoted 12 lines
xen/arch/riscv/imsic.c new file +12
-0+return 0 ; +} + +int __init imsic_init (struct dt_device_node *node) +{ +int rc; +unsigned long reloff, hartid; +uint32_t nr_parent_irqs, index, nr_handlers =0 ; +paddr_t base_addr; + +/* Parse IMSIC node */ + rc =imsic_parse_node (node, &nr_parent_irqs);
xen/arch/riscv/imsic.c line 187
Quoted 10 lines
xen/arch/riscv/imsic.c new file +10
-0+if ( rc ) +return rc; + +/* Allocate MMIO resource array */ + imsic_cfg.mmios = xzalloc_array(struct imsic_mmios, imsic_cfg.nr_mmios); +if ( !imsic_cfg.mmios ) +return -ENOMEM ; + +/* check MMIO register sets */ +for (int i =0 ; i < imsic_cfg.nr_mmios; i++ )
No plain int please for anything that can only be non-negative.
Quoted 6 lines
xen/arch/riscv/imsic.c new file +6
-0+ { + rc =dt_device_get_address (node, i, &imsic_cfg.mmios[i].base_addr, + &imsic_cfg.mmios[i].size); +if ( rc ) + { + printk(XENLOG_ERR "%s: unable to parse MMIO regset %d\n",
xen/arch/riscv/imsic.c line 203
Quoted 10 lines
xen/arch/riscv/imsic.c new file +10
-0+ node->name, i); + goto imsic_init_err; + } + + base_addr = imsic_cfg.mmios[i].base_addr; + base_addr &= ~(BIT (imsic_cfg.guest_index_bits + + imsic_cfg.hart_index_bits + +IMSIC_MMIO_PAGE_SHIFT ,UL ) -1 ); + base_addr &= ~((BIT (imsic_cfg.group_index_bits,UL ) -1 ) << + imsic_cfg.group_index_shift);
Indentation again.
xen/arch/riscv/imsic.c line 210
Quoted 7 lines
xen/arch/riscv/imsic.c new file +7
-0+if ( base_addr != imsic_cfg.base_addr ) + { + rc = -EINVAL ; +printk (XENLOG_ERR "%s: address mismatch for regset %d\n" , + node->name, i); + goto imsic_init_err; + }
Oh, all of the addresses need to (sufficiently) match.
Quoted 62 lines
xen/arch/riscv/imsic.c new file +62
-0+ } + +/* Configure handlers for target CPUs */ +for (int i =0 ; i < nr_parent_irqs; i++ ) + { + rc =imsic_get_parent_hartid (node, i, &hartid); +if ( rc ) + { +printk (XENLOG_WARNING "%s: hart ID for parent irq%d not found\n" , + node->name, i); +continue ; + } + +if ( hartid >NR_CPUS ) + { +printk (XENLOG_WARNING "%s: unsupported hart ID=%lu for parent irq%d\n" , + node->name, hartid, i); +continue ; + } + +/* Find MMIO location of MSI page */ + index = imsic_cfg.nr_mmios; + reloff = i *BIT (imsic_cfg.guest_index_bits,UL ) *IMSIC_MMIO_PAGE_SZ ; +for (int j =0 ; imsic_cfg.nr_mmios; j++ ) + { +if ( reloff < imsic_cfg.mmios[j].size ) + { + index = j; +break ; + } + +/* +* MMIO region size may not be aligned to +* BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ +* if holes are present. +*/ + reloff -=ROUNDUP (imsic_cfg.mmios[j].size, +BIT (imsic_cfg.guest_index_bits,UL ) *IMSIC_MMIO_PAGE_SZ ); + } + +if ( index >= imsic_cfg.nr_mmios ) + { +printk (XENLOG_WARNING "%s: MMIO not found for parent irq%d\n" , + node->name, i); +continue ; + } + +if ( !IS_ALIGNED (imsic_cfg.msi[hartid].base_addr + reloff,PAGE_SIZE ) ) + { +printk (XENLOG_WARNING "%s: MMIO address 0x%lx is not aligned on a page\n" , + node->name, imsic_cfg.msi[hartid].base_addr + reloff); + imsic_cfg.msi[hartid].offset =0 ; + imsic_cfg.msi[hartid].base_addr =0 ; +continue ; + } + + imsic_cfg.mmios[index].harts[hartid] = true; + imsic_cfg.msi[hartid].base_addr = imsic_cfg.mmios[index].base_addr; + imsic_cfg.msi[hartid].offset = reloff; + nr_handlers++; + } +
xen/arch/riscv/imsic.c line 282
Quoted 10 lines
xen/arch/riscv/imsic.c new file +10
-0+if ( !nr_handlers ) + { +printk (XENLOG_ERR "%s: No CPU handlers found\n" , node->name); + rc = -ENODEV ; + goto imsic_init_err; + } + +return 0 ; + +imsic_init_err:
Labels indented by at least one blank please.
xen/arch/riscv/imsic.c line 283
Quoted 1 line
xen/arch/riscv/imsic.c new file +1
-0+xfree (imsic_cfg.mmios);
Better use XFREE() in cases like this one?
Quoted 23 lines
xen/arch/riscv/include/asm/imsic.h new file +20
-0@@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: MIT */ + +/* +* xen/arch/riscv/imsic.h +* +* RISC-V Incoming MSI Controller support +* +* (c) 2023 Microchip Technology Inc. +*/ + +#ifndef ASM__RISCV__IMSIC_H +#define ASM__RISCV__IMSIC_H + +#include <xen/types.h> + +#define IMSIC_MMIO_PAGE_SHIFT 12 +#define IMSIC_MMIO_PAGE_SZ (1UL << IMSIC_MMIO_PAGE_SHIFT) + +#define IMSIC_MIN_ID 63 +#define IMSIC_MAX_ID 2048
xen/arch/riscv/include/asm/imsic.h line 30
Quoted 10 lines
xen/arch/riscv/include/asm/imsic.h new file +10
-0+ +struct imsic_msi { +paddr_t base_addr; +unsigned long offset; +}; + +struct imsic_mmios { +paddr_t base_addr; +unsigned long size; +bool harts[NR_CPUS ];
An array of bool - won't a bitmap do here? Even then I wouldn't be overly happy to see it dimensioned by NR_CPUS.
Quoted 17 lines
xen/arch/riscv/include/asm/imsic.h new file +17
-0+}; + +struct imsic_config { +/* base address */ +paddr_t base_addr; + +/* Bits representing Guest index, HART index, and Group index */ +unsigned int guest_index_bits; +unsigned int hart_index_bits; +unsigned int group_index_bits; +unsigned int group_index_shift; + +/* imsic phandle */ +unsigned int phandle; + +/* number of parent irq */ +unsigned int nr_parent_irqs;
xen/arch/riscv/include/asm/imsic.h line 57
Quoted 10 lines
xen/arch/riscv/include/asm/imsic.h new file +10
-0+ +/* number off interrupt identities */ +unsigned int nr_ids; + +/* mmios */ +unsigned int nr_mmios; +struct imsic_mmios *mmios; + +/* MSI */ +struct imsic_msi msi[NR_CPUS ];
You surely can avoid wasting perhaps a lot of memory by allocating this based on the number of CPUs in use?
xen/arch/riscv/include/asm/imsic.h line 61
Quoted 4 lines
xen/arch/riscv/include/asm/imsic.h new file +4
-0+}; + +struct dt_device_node ; +int imsic_init (struct dt_device_node *n);
Misra demands that parameter names match between declaration and definition.
xen/arch/riscv/include/asm/imsic.h line 63
Quoted 1 line
xen/arch/riscv/include/asm/imsic.h new file +1
-0+struct imsic_config ;
I don't think you need this, as it's ...
xen/arch/riscv/include/asm/imsic.h line 64
Quoted 1 line
xen/arch/riscv/include/asm/imsic.h new file +1
-0+const struct imsic_config *imsic_get_config (void );
... not used as parameter type (where its scope would otherwise be wrongly limited).
Jan
Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation
Quoted 14 lines · Jan Beulich
xen/arch/riscv/imsic.c line 9
Quoted 12 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c new file +9
-0@@ -0,0 +1,286 @@ +/* SPDX-License-Identifier: MIT */ + +/* +* xen/arch/riscv/imsic.c +* +* RISC-V Incoming MSI Controller support +* +* (c) 2023 Microchip Technology Inc. +* (c) 2024 Vates No 2025 here (if already the years matter)?
I think it doesn't really matter and it could be just dropped.
Quoted 16 lines
Quoted 5 lines
xen/arch/riscv/imsic.c new file +4
-0+*/ + +#include <xen/const.h> +#include <xen/device_tree.h> xen/arch/riscv/imsic.c line 23
Quoted 10 lines
xen/arch/riscv/imsic.c new file +10
-0+#include <xen/errno.h> +#include <xen/init.h> +#include <xen/macros.h> +#include <xen/xmalloc.h> + +#include <asm/imsic.h> + +static struct imsic_config imsic_cfg; + +const struct imsic_config *imsic_get_config (void )Does this need to return a pointer to non-const?
No, I think it could be const struct imsic_config * const as a return type. It isn't expected that the caller will change a pointer.
Quoted 16 lines
Quoted 4 lines
xen/arch/riscv/imsic.c new file +3
-0+{ +return &imsic_cfg; +}xen/arch/riscv/imsic.c line 36
Quoted 10 lines
xen/arch/riscv/imsic.c new file +10
-0+ +static int __init imsic_get_parent_hartid (struct dt_device_node *node, +unsigned int index, +unsigned long *hartid) +{ +int res; +unsigned long hart; +struct dt_phandle_args args; + +/* Try the new-style interrupts-extended first */ The comment says "first", but then ...
Quoted 13 lines
xen/arch/riscv/imsic.c line 48
Quoted 12 lines
xen/arch/riscv/imsic.c new file +12
-0+ res =dt_parse_phandle_with_args (node,"interrupts-extended" , +"#interrupt-cells" , index, &args); +if ( !res ) + { + res =riscv_of_processor_hartid (args.np->parent, &hart); +if ( res <0 ) +return -EINVAL ; + + *hartid = hart; + } +return res; +}... nothing else is being tried.
A stale comment, we decided to support only interrupt-extended to be in line with Linux device tree bindings.
I'll drop this comment.
Quoted 5 lines
Also, nit: Blank line please ahead of the main "return" of a function.
Further - any particular reason to discard riscv_of_processor_hartid()'s error code on the error path?
No particular reason, just overlooked that we could really return just res.
Quoted 2 lines
xen/arch/riscv/imsic.c line 52
Quoted 2 lines
xen/arch/riscv/imsic.c new file +2
-0+static int imsic_parse_node (struct dt_device_node *node, +unsigned int *nr_parent_irqs)
(for me: fix an indentation)
Quoted 11 lines
Quoted 11 lines
xen/arch/riscv/imsic.c new file +11
-0+{ +int rc; +unsigned int tmp; +paddr_t base_addr; + +/* Find number of parent interrupts */ + *nr_parent_irqs =dt_number_of_irq (node); +if ( !*nr_parent_irqs ) + { +printk (XENLOG_ERR "%s: no parent irqs available\n" , node->name); +return -ENOENT ;
Quoted 8 lines
xen/arch/riscv/imsic.c line 69
Quoted 6 lines
xen/arch/riscv/imsic.c new file +6
-0+ } + +/* Find number of guest index bits in MSI address */ + rc =dt_property_read_u32 (node,"riscv,guest-index-bits" , + &imsic_cfg.guest_index_bits); +if ( !rc )It is confusing to store a bool return value in a local "int" variable, just to then use it as boolean. Is the local var needed at all here?
'int` is used because of ...:
rc = dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL);
... at the end of imsic_parse_node().
Agree, we can just drop rc variable and return err code explicitly.
Quoted 21 lines
Quoted 11 lines
xen/arch/riscv/imsic.c new file +10
-0+ imsic_cfg.guest_index_bits =0 ; + tmp =BITS_PER_LONG -IMSIC_MMIO_PAGE_SHIFT ; +if ( tmp < imsic_cfg.guest_index_bits ) + { +printk (XENLOG_ERR "%s: guest index bits too big\n" , node->name); +return -ENOENT ; + } + +/* Find number of HART index bits */ + rc = dt_property_read_u32(node, "riscv,hart-index-bits",xen/arch/riscv/imsic.c line 89
Quoted 10 lines
xen/arch/riscv/imsic.c new file +10
-0+ &imsic_cfg.hart_index_bits); +if ( !rc ) + { +/* Assume default value */ + imsic_cfg.hart_index_bits =fls (*nr_parent_irqs); +if (BIT (imsic_cfg.hart_index_bits,UL ) < *nr_parent_irqs ) + imsic_cfg.hart_index_bits++; + } + tmp =BITS_PER_LONG -IMSIC_MMIO_PAGE_SHIFT - + imsic_cfg.guest_index_bits;
Quoted 3 lines
tmp -= imsic_cfg.guest_index_bits;? (And then similarly further down.)
Yes, it would be simpler.
Quoted 42 lines
Quoted 42 lines
xen/arch/riscv/imsic.c new file +42
-0+if ( tmp < imsic_cfg.hart_index_bits ) + { +printk (XENLOG_ERR "%s: HART index bits too big\n" , node->name); +return -ENOENT ; + } + +/* Find number of group index bits */ + rc =dt_property_read_u32 (node,"riscv,group-index-bits" , + &imsic_cfg.group_index_bits); +if ( !rc ) + imsic_cfg.group_index_bits =0 ; + tmp =BITS_PER_LONG -IMSIC_MMIO_PAGE_SHIFT - + imsic_cfg.guest_index_bits - imsic_cfg.hart_index_bits; +if ( tmp < imsic_cfg.group_index_bits ) + { +printk (XENLOG_ERR "%s: group index bits too big\n" , node->name); +return -ENOENT ; + } + +/* Find first bit position of group index */ + tmp =IMSIC_MMIO_PAGE_SHIFT *2 ; + rc =dt_property_read_u32 (node,"riscv,group-index-shift" , + &imsic_cfg.group_index_shift); +if ( !rc ) + imsic_cfg.group_index_shift = tmp; +if ( imsic_cfg.group_index_shift < tmp ) + { +printk (XENLOG_ERR "%s: group index shift too small\n" , node->name); +return -ENOENT ; + } + tmp = imsic_cfg.group_index_bits + imsic_cfg.group_index_shift -1 ; +if ( tmp >=BITS_PER_LONG ) + { +printk (XENLOG_ERR "%s: group index shift too big\n" , node->name); +return -EINVAL ; + } + +/* Find number of interrupt identities */ + rc =dt_property_read_u32 (node,"riscv,num-ids" , &imsic_cfg.nr_ids); +if ( !rc ) + { + printk(XENLOG_ERR "%s: number of interrupt identities not found\n",
Quoted 8 lines
xen/arch/riscv/imsic.c line 137
Quoted 6 lines
xen/arch/riscv/imsic.c new file +6
-0+ node->name); +return -ENOENT ; + } + + if ( (imsic_cfg.nr_ids < IMSIC_MIN_ID) || + (imsic_cfg.nr_ids >= IMSIC_MAX_ID) ||Something named "max" normally wants to decribe the highest valid value, not the first out-of-range one.
But it is the maximum of identities for IMISIC according to the AIA specs:
``` ( MRIF stands for Memory-Resident Interrupt File ) All MRIFs are the size to accommodate 2047 valid interrupt identities, the maximum allowed for an IMSIC interrupt file. If a system’s actual IMSICs have interrupt files that implement only N interrupt identities, N < 2047, then the contents of MRIFs for identities greater than N may be ignored by softwareand ```
and prefix IMSIC (IMO)emphasizes this. Would it be better to rename to IMSIC_MAX_ALLOWED_ID? And then IMSIC_MIN_ALLOWED_ID?
Quoted 12 lines
xen/arch/riscv/imsic.c line 148
Quoted 12 lines
xen/arch/riscv/imsic.c new file +11
-0+ ((imsic_cfg.nr_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) ) + { +printk (XENLOG_ERR "%s: invalid number of interrupt identities\n" , + node->name); +return -EINVAL ; + } + +/* Compute base address */ + imsic_cfg.nr_mmios =0 ; + rc =dt_device_get_address (node, imsic_cfg.nr_mmios, &base_addr,NULL ); +if (rc)
Quoted 6 lines
Nit: Style.
xen/arch/riscv/imsic.c line 151
Quoted 3 lines
xen/arch/riscv/imsic.c new file +3
-0+ { +printk (XENLOG_ERR "%s: first MMIO resource not found\n" , node->name); +return -EINVAL ;Discarding "rc" again?
Agree, it could be just rc, but considering of that this functions always returns -EINVAL in case of the error and that rc could be dropped at all, we just explicitly return here -EINVAL.
Quoted 9 lines
xen/arch/riscv/imsic.c line 159
Quoted 9 lines
xen/arch/riscv/imsic.c new file +8
-0+ } + + imsic_cfg.base_addr = base_addr; + imsic_cfg.base_addr &= ~(BIT (imsic_cfg.guest_index_bits + + imsic_cfg.hart_index_bits + +IMSIC_MMIO_PAGE_SHIFT ,UL ) -1 ); + imsic_cfg.base_addr &= ~((BIT (imsic_cfg.group_index_bits,UL ) -1 ) << + imsic_cfg.group_index_shift);
Quoted 3 lines
Besides indentation being bogus here, why is it that you need to mask bits off of the value read from DT? Wouldn't the expectation be that you get back the true base address?
The group index is used to differentiate between clusters/groups. For example, consider two clusters: - Cluster 1 with cpu0 and cpu1 - Cluster 2 with cpu2 and cpu3 Then, the reg property in the IMSIC node will include two entries: reg = <0x0 0xd100000 0x0 0x20000>, <0x0 0x2d100000 0x0 0x20000>; riscv,guest-index-bits = <3>; riscv,hart-index-bits = <2>; riscv,group-index-bits = <1>; riscv,group-index-shift = <29>; In this example: The group index is 1 bit wide (group-index-bits = <1>), It is located at bit 29 (group-index-shift = <29>) of the address.
so imsic_cfg.group_index_bits will be used to distinguish clusters, but they must have
the same base address and this is the reason why the mask is applied.
Quoted 7 lines
xen/arch/riscv/imsic.c line 164
Quoted 5 lines
xen/arch/riscv/imsic.c new file +4
-0+/* Find number of MMIO register sets */ + imsic_cfg.nr_mmios++; +while ( !dt_device_get_address (node, imsic_cfg.nr_mmios, &base_addr,NULL ) ) + imsic_cfg.nr_mmios++;And the base addresses of these aren't of interest? Oh, I see they're fetched again further down.
It is fetched again because we are using imsic_cfg.nr_mmios to calculate the size of imsic_cfg.mmios array:
imsic_cfg.mmios = xzalloc_array(struct imsic_mmios, imsic_cfg.nr_mmios);
Quoted 2 lines
Also - use do-while here?
It could be, I'll update that.
Quoted 11 lines
xen/arch/riscv/imsic.c line 210
Quoted 7 lines
xen/arch/riscv/imsic.c new file +7
-0+if ( base_addr != imsic_cfg.base_addr ) + { + rc = -EINVAL ; +printk (XENLOG_ERR "%s: address mismatch for regset %d\n" , + node->name, i); + goto imsic_init_err; + }Oh, all of the addresses need to (sufficiently) match.
xen/arch/riscv/imsic.c line 283
Quoted 1 line
xen/arch/riscv/imsic.c new file +1
-0+xfree (imsic_cfg.mmios);Better use
XFREE()in cases like this one?
I think, yes.
Quoted 28 lines
Quoted 28 lines
xen/arch/riscv/include/asm/imsic.h new file +24
-0@@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: MIT */ + +/* +* xen/arch/riscv/imsic.h +* +* RISC-V Incoming MSI Controller support +* +* (c) 2023 Microchip Technology Inc. +*/ + +#ifndef ASM__RISCV__IMSIC_H +#define ASM__RISCV__IMSIC_H + +#include <xen/types.h> + +#define IMSIC_MMIO_PAGE_SHIFT 12 +#define IMSIC_MMIO_PAGE_SZ (1UL << IMSIC_MMIO_PAGE_SHIFT) + +#define IMSIC_MIN_ID 63 +#define IMSIC_MAX_ID 2048 + +struct imsic_msi { + paddr_t base_addr; + unsigned long offset;
Quoted 8 lines
xen/arch/riscv/include/asm/imsic.h line 30
Quoted 6 lines
xen/arch/riscv/include/asm/imsic.h new file +6
-0+}; + +struct imsic_mmios { +paddr_t base_addr; +unsigned long size; +bool harts[NR_CPUS ];An array of bool - won't a bitmap do here? Even then I wouldn't be overly happy to see it dimensioned by
NR_CPUS.
Bitmap will fit here well. But for DECLARE_BITMAP() is necessary the size of bitmap so NR_CPUS should be used again.
Could you please remind me why it isn't good to use it?
Because NR_CPUS not always equal to an amount of physical cpus? Should I use non-static version of bitmap declaration? (if we have such...)
Quoted 22 lines
Quoted 22 lines
xen/arch/riscv/include/asm/imsic.h new file +21
-0+}; + +struct imsic_config { +/* base address */ +paddr_t base_addr; + +/* Bits representing Guest index, HART index, and Group index */ +unsigned int guest_index_bits; +unsigned int hart_index_bits; +unsigned int group_index_bits; +unsigned int group_index_shift; + +/* imsic phandle */ +unsigned int phandle; + +/* number of parent irq */ +unsigned int nr_parent_irqs; + +/* number off interrupt identities */ +unsigned int nr_ids; +
Quoted 8 lines
xen/arch/riscv/include/asm/imsic.h line 57
Quoted 6 lines
xen/arch/riscv/include/asm/imsic.h new file +6
-0+/* mmios */ +unsigned int nr_mmios; +struct imsic_mmios *mmios; + +/* MSI */ +struct imsic_msi msi[NR_CPUS ];You surely can avoid wasting perhaps a lot of memory by allocating this based on the number of CPUs in use?
It make sense. I'll allocate then this dynamically.
Thanks!
~ Oleksii
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation
Quoted 23 lines · Jan Beulich
Quoted 23 lines
Quoted 11 lines
xen/arch/riscv/imsic.c line 159
Quoted 8 lines
xen/arch/riscv/imsic.c new file +8
-0+ } + + imsic_cfg.base_addr = base_addr; + imsic_cfg.base_addr &= ~(BIT (imsic_cfg.guest_index_bits + + imsic_cfg.hart_index_bits + +IMSIC_MMIO_PAGE_SHIFT ,UL ) -1 ); + imsic_cfg.base_addr &= ~((BIT (imsic_cfg.group_index_bits,UL ) -1 ) << + imsic_cfg.group_index_shift);Besides indentation being bogus here, why is it that you need to mask bits off of the value read from DT? Wouldn't the expectation be that you get back the true base address?
The group index is used to differentiate between clusters/groups. For example, consider two clusters: - Cluster 1 with cpu0 and cpu1 - Cluster 2 with cpu2 and cpu3 Then, the reg property in the IMSIC node will include two entries: reg = <
0x00xd1000000x00x20000>, <0x00x2d1000000x00x20000>; riscv,guest-index-bits = <3>; riscv,hart-index-bits = <2>; riscv,group-index-bits = <1>; riscv,group-index-shift = <29>; In this
example: The group index is 1 bit wide (group-index-bits = <1>), It is located at bit 29 (group-index-shift = <29>) of the address.so imsic_cfg.group_index_bits will be used to distinguish clusters, but they must have
Quoted 7 lines
Quoted 1 line
the same base address and this is the reason why the mask is applied.
Well, yes, but that doesn't answer my question. All of what you say makes sense for the loop elsewhere retrieving all the addresses. Here you retrieved only the first of them, and in your example no masking would be needed here either. To re-phrase my question: Can the address observed in the first entry be other than the lowest one across the set of all entries?
It doesn't mentioned explicitly in riscv,imsiscs binding that reg property is sorted.
I can write:
reg = <0x0 0x2d100000 0x0 0x20000>,<0x0 0xd100000 0x0 0x20000>;
And a dtc compiler will compile.
Quoted 2 lines
If not, can there be bits set across all entries that need masking off from all of them?
I think that other bits (Hart Index and Guest Index) excepts base address bits are 0 in dts, but it doesn't guarantee explicitly by IMSIC's dts bindinding.
Quoted 36 lines
Quoted 36 lines
Quoted 36 lines
Quoted 24 lines
xen/arch/riscv/include/asm/imsic.h new file +20
-0@@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: MIT */ + +/* +* xen/arch/riscv/imsic.h +* +* RISC-V Incoming MSI Controller support +* +* (c) 2023 Microchip Technology Inc. +*/ + +#ifndef ASM__RISCV__IMSIC_H +#define ASM__RISCV__IMSIC_H + +#include <xen/types.h> + +#define IMSIC_MMIO_PAGE_SHIFT 12 +#define IMSIC_MMIO_PAGE_SZ (1UL << IMSIC_MMIO_PAGE_SHIFT) + +#define IMSIC_MIN_ID 63 +#define IMSIC_MAX_ID 2048xen/arch/riscv/include/asm/imsic.h line 30
Quoted 10 lines
xen/arch/riscv/include/asm/imsic.h new file +10
-0+ +struct imsic_msi { +paddr_t base_addr; +unsigned long offset; +}; + +struct imsic_mmios { +paddr_t base_addr; +unsigned long size; +bool harts[NR_CPUS ];An array of bool - won't a bitmap do here? Even then I wouldn't be overly happy to see it dimensioned by
NR_CPUS.
Quoted 8 lines
Quoted 4 lines
Bitmap will fit here well. But for
DECLARE_BITMAP()is necessary the size of bitmap soNR_CPUSshould be used again.
Could you please remind me why it isn't good to use it? BecauseNR_CPUSnot always equal to an amount of physical cpus?"Not equal" wouldn't be overly problematic. But NR_CPUS=4000 and the actual number of CPUs being 4 would be wasteful in general. More when its wider than a bit that's needed per CPU, but where would you draw the line if you permitted use of
NR_CPUShere?
Hard to say. It seems it will be just better to use apporoach you suggested below.
Thanks.
~ Oleksii
Quoted 40 lines
Quoted 2 lines
Should I use non-static version of bitmap declaration? (if we have such...)
That's simply "unsigned long *" then, or - at the tail of a dynamically allocated struct - possibly unsigned long[].
Quoted 29 lines
Quoted 21 lines
Quoted 21 lines
xen/arch/riscv/include/asm/imsic.h new file +21
-0+}; + +struct imsic_config { +/* base address */ +paddr_t base_addr; + +/* Bits representing Guest index, HART index, and Group index */ +unsigned int guest_index_bits; +unsigned int hart_index_bits; +unsigned int group_index_bits; +unsigned int group_index_shift; + +/* imsic phandle */ +unsigned int phandle; + +/* number of parent irq */ +unsigned int nr_parent_irqs; + +/* number off interrupt identities */ +unsigned int nr_ids; +Quoted 8 lines
xen/arch/riscv/include/asm/imsic.h line 57
Quoted 6 lines
xen/arch/riscv/include/asm/imsic.h new file +6
-0+/* mmios */ +unsigned int nr_mmios; +struct imsic_mmios *mmios; + +/* MSI */ +struct imsic_msi msi[NR_CPUS ];You surely can avoid wasting perhaps a lot of memory by allocating this based on the number of CPUs in use?
Quoted 1 line
It make sense. I'll allocate then this dynamically.
Or, as per above, when put at the tail and the struct itself is dynamically allocated, use struct imsic_msi[]. We even have dedicated
xmalloc()flavors for this kind of allocation.Jan
Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation
In reply to a message not in the archiveQuoted 77 lines · Jan Beulich
Quoted 35 lines
Quoted 27 lines
Quoted 27 lines
xen/arch/riscv/include/asm/imsic.h new file +24
-0@@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: MIT */ + +/* +* xen/arch/riscv/imsic.h +* +* RISC-V Incoming MSI Controller support +* +* (c) 2023 Microchip Technology Inc. +*/ + +#ifndef ASM__RISCV__IMSIC_H +#define ASM__RISCV__IMSIC_H + +#include <xen/types.h> + +#define IMSIC_MMIO_PAGE_SHIFT 12 +#define IMSIC_MMIO_PAGE_SZ (1UL << IMSIC_MMIO_PAGE_SHIFT) + +#define IMSIC_MIN_ID 63 +#define IMSIC_MAX_ID 2048 + +struct imsic_msi { + paddr_t base_addr; + unsigned long offset;Quoted 8 lines
xen/arch/riscv/include/asm/imsic.h line 30
Quoted 6 lines
xen/arch/riscv/include/asm/imsic.h new file +6
-0+}; + +struct imsic_mmios { +paddr_t base_addr; +unsigned long size; +bool harts[NR_CPUS ];An array of bool - won't a bitmap do here? Even then I wouldn't be overly happy to see it dimensioned by
NR_CPUS.Quoted 4 lines
Bitmap will fit here well. But for
DECLARE_BITMAP()is necessary the size of bitmap soNR_CPUSshould be used again.
Could you please remind me why it isn't good to use it? BecauseNR_CPUSnot always equal to an amount of physical cpus?"Not equal" wouldn't be overly problematic. But NR_CPUS=4000 and the actual number of CPUs being 4 would be wasteful in general. More when its wider than a bit that's needed per CPU, but where would you draw the line if you permitted use of
NR_CPUShere?Quoted 1 line
Should I use non-static version of bitmap declaration? (if we have such...)
That's simply "unsigned long *" then, or - at the tail of a dynamically allocated struct - possibly unsigned long[].
Quoted 29 lines
Quoted 29 lines
Quoted 17 lines
xen/arch/riscv/include/asm/imsic.h new file +17
-0+}; + +struct imsic_config { +/* base address */ +paddr_t base_addr; + +/* Bits representing Guest index, HART index, and Group index */ +unsigned int guest_index_bits; +unsigned int hart_index_bits; +unsigned int group_index_bits; +unsigned int group_index_shift; + +/* imsic phandle */ +unsigned int phandle; + +/* number of parent irq */ +unsigned int nr_parent_irqs;xen/arch/riscv/include/asm/imsic.h line 57
Quoted 10 lines
xen/arch/riscv/include/asm/imsic.h new file +10
-0+ +/* number off interrupt identities */ +unsigned int nr_ids; + +/* mmios */ +unsigned int nr_mmios; +struct imsic_mmios *mmios; + +/* MSI */ +struct imsic_msi msi[NR_CPUS ];You surely can avoid wasting perhaps a lot of memory by allocating this based on the number of CPUs in use?
Quoted 4 lines
Quoted 1 line
It make sense. I'll allocate then this dynamically.
Or, as per above, when put at the tail and the struct itself is dynamically allocated, use struct imsic_msi[]. We even have dedicated
xmalloc()flavors for this kind of allocation.
Do you mean xzalloc_flex_struct()?
I think, I can't use for both of the cases (allocation of mmios and msi). For msi[] then it is needed to allocate imsic_config also dynamically, isn't it? So something like:
imsic_config = xzalloc_flex_struct(struct imsic_config, msi, NR_CPUS).
But now it is allocated statically.
For *mmios and harts[] (a member inside struct imsic_mmios):
mmios = xzalloc_flex_struct(struct imsic_mmios, harts, NR_CPUS); // NR_CPUs just for example...
It will allocate only one mmios, but it is needed mmios[nr_mmios].
Maybe, something like _xmalloc((offsetof(struct imsic_mmios, harts[NR_CPUS])) * NR_CPUS, sizeof(struct imsic_mmios)) will work.
Am I missing something?
~ Oleksii
PATCH 9/14 xen/riscv: aplic_init() implementation
In reply to Oleksii Kurochkoaplic_init() function does the following few things:
- checks that IMSIC in device tree node ( by checking msi-parent property in APLIC node ) is present as current one implmenetaion of AIA is supported only MSI method.
- initialize IMSIC based on IMSIC device tree node
- Read value of APLIC's paddr start/end and size.
- Map aplic.regs
- Setup APLIC initial state interrupts (disable all interrupts, set interrupt type and default priority, confgifure APLIC domaincfg) by calling aplic_init_hw_interrutps().
aplic_init() is based on the code from [1] and [2].
Since Microchip originally developed aplic.c, an internal discussion with them led to the decision to use the MIT license.
[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7cfb4bd4748ca268142497ac5c327d2766fb342d
[2] https://gitlab.com/xen-project/people/olkur/xen/-/commit/392a531bfad39bf4656ce8128e004b241b8b3f3e
--- xen/arch/riscv/aplic.c | 97 ++++++++++++++++++++++++++++++ xen/arch/riscv/include/asm/aplic.h | 77 ++++++++++++++++++++++++ xen/arch/riscv/include/asm/intc.h | 3 + xen/arch/riscv/include/asm/irq.h | 1 - 4 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 xen/arch/riscv/include/asm/aplic.h
xen/arch/riscv/aplic.c +97 -0
@@ -9,19 +9,112 @@* Copyright (c) 2024-2025 Vates */ +#include <xen/device_tree.h> #include <xen/errno.h> #include <xen/init.h> #include <xen/irq.h> +#include <xen/mm.h> #include <xen/sections.h> #include <xen/types.h> +#include <xen/vmap.h> +#include <asm/aplic.h> #include <asm/device.h> +#include <asm/imsic.h> #include <asm/intc.h> +#include <asm/riscv_encoding.h> + +#define APLIC_DEFAULT_PRIORITY 1 + +static struct aplic_priv aplic;static struct intc_info __ro_after_init aplic_info = { .hw_version =INTC_APLIC , }; +static void __init aplic_init_hw_interrupts (void ) +{ +int i; + +/* Disable all interrupts */ +for ( i =0 ; i <= aplic_info.nr_irqs; i +=32 ) + aplic.regs->clrie[i] =-1U ; + +/* Set interrupt type and default priority for all interrupts */ +for ( i =1 ; i <= aplic_info.nr_irqs; i++ ) + { + aplic.regs->sourcecfg[i -1 ] =0 ; + aplic.regs->target[i -1 ] =APLIC_DEFAULT_PRIORITY ; + } + +/* Clear APLIC domaincfg */ + aplic.regs->domaincfg =APLIC_DOMAINCFG_IE |APLIC_DOMAINCFG_DM ; +} + +static int __init aplic_init (void ) +{ +int rc; +dt_phandle imsic_phandle; +uint32_t irq_range[2 ]; +const __be32 *prop; +uint64_t size, paddr; +struct dt_device_node *imsic_node; +const struct dt_device_node *node = aplic_info.node; + +/* check for associated imsic node */ + rc =dt_property_read_u32 (node,"msi-parent" , &imsic_phandle); +if ( !rc ) +panic ("%s: IDC mode not supported\n" , node->full_name); + + imsic_node =dt_find_node_by_phandle (imsic_phandle); +if ( !imsic_node ) +panic ("%s: unable to find IMSIC node\n" , node->full_name); + +/* check imsic mode */ + rc =dt_property_read_u32_array (imsic_node,"interrupts-extended" , + irq_range,ARRAY_SIZE (irq_range)); +if ( rc && (rc != -EOVERFLOW ) ) +panic ("%s: unable to find interrupt-extended in %s node\n" , + node->full_name, imsic_node->full_name); + +if ( irq_range[1 ] ==IRQ_M_EXT ) +/* machine mode imsic node, ignore this aplic node */ +return 0 ; + + rc =imsic_init (imsic_node); +if ( rc ) +panic ("%s: Failded to initialize IMSIC\n" , node->full_name); + +/* Find out number of interrupt sources */ + rc =dt_property_read_u32 (node,"riscv,num-sources" , &aplic_info.nr_irqs); +if ( !rc ) +panic ("%s: failed to get number of interrupt sources\n" , + node->full_name); + + prop =dt_get_property (node,"reg" ,NULL ); +dt_get_range (&prop, node, &paddr, &size); +if ( !paddr ) +panic ("%s: first MMIO resource not found\n" , node->full_name); + + aplic.paddr_start = paddr; + aplic.paddr_end = paddr + size; + aplic.size = size; + + aplic.regs =ioremap (paddr, size); +if ( !aplic.regs ) +panic ("%s: unable to map\n" , node->full_name); + +/* Setup initial state APLIC interrupts */ +aplic_init_hw_interrupts (); + +return 0 ; +} + +static const struct intc_hw_operations __ro_after_init aplic_ops = { + .info = &aplic_info, + .init = aplic_init, +}; +static int aplic_irq_xlate (const uint32_t *intspec,unsigned int intsize,unsigned int *out_hwirq,unsigned int *out_type)
@@ -52,8 +145,12 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat) aplic_info.node = node; + aplic.imsic_cfg =imsic_get_config (); + dt_irq_xlate = aplic_irq_xlate; +register_intc_ops (&aplic_ops); +return 0 ; }
xen/arch/riscv/include/asm/aplic.h new file +77 -0
77 lines not on this page · raw message
xen/arch/riscv/include/asm/intc.h +3 -0
@@ -15,6 +15,9 @@ enum intc_version {struct intc_info {enum intc_version hw_version;const struct dt_device_node *node; + +/* number of irqs */ +unsigned int nr_irqs; };struct intc_hw_operations {
xen/arch/riscv/include/asm/irq.h +0 -1
@@ -27,7 +27,6 @@#define IRQ_TYPE_INVALID DT_IRQ_TYPE_INVALID/* TODO */ -#define nr_irqs 0U#define nr_static_irqs 0#define arch_hwdom_irqs (domid) 0U
Signature · 1 line
2.49.0
Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation
Quoted 32 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +13
-0@@ -9,19 +9,112 @@* Copyright (c) 2024-2025 Vates */ +#include <xen/device_tree.h> #include <xen/errno.h> #include <xen/init.h> #include <xen/irq.h> +#include <xen/mm.h> #include <xen/sections.h> #include <xen/types.h> +#include <xen/vmap.h> +#include <asm/aplic.h> #include <asm/device.h> +#include <asm/imsic.h> #include <asm/intc.h> +#include <asm/riscv_encoding.h> + +#define APLIC_DEFAULT_PRIORITY 1 + +static struct aplic_priv aplic;static struct intc_info __ro_after_init aplic_info = { .hw_version =INTC_APLIC , }; +static void __init aplic_init_hw_interrupts (void ) +{ +int i;
xen/arch/riscv/aplic.c line 47
Quoted 10 lines
xen/arch/riscv/aplic.c +10
-0+ +/* Disable all interrupts */ +for ( i =0 ; i <= aplic_info.nr_irqs; i +=32 ) + aplic.regs->clrie[i] =-1U ; + +/* Set interrupt type and default priority for all interrupts */ +for ( i =1 ; i <= aplic_info.nr_irqs; i++ ) + { + aplic.regs->sourcecfg[i -1 ] =0 ; + aplic.regs->target[i -1 ] =APLIC_DEFAULT_PRIORITY ;
A field named "target" is written with a priority value?
xen/arch/riscv/aplic.c line 51
Quoted 4 lines
xen/arch/riscv/aplic.c +4
-0+ } + +/* Clear APLIC domaincfg */ + aplic.regs->domaincfg =APLIC_DOMAINCFG_IE |APLIC_DOMAINCFG_DM ;
The statement doesn't like like there was any "clearing" here.
xen/arch/riscv/aplic.c line 64
Quoted 13 lines
xen/arch/riscv/aplic.c +13
-0+} + +static int __init aplic_init (void ) +{ +int rc; +dt_phandle imsic_phandle; +uint32_t irq_range[2 ]; +const __be32 *prop; +uint64_t size, paddr; +struct dt_device_node *imsic_node; +const struct dt_device_node *node = aplic_info.node; + +/* check for associated imsic node */
Nit: Comment style (also elsewhere).
Quoted 4 lines
xen/arch/riscv/aplic.c +4
-0+ rc =dt_property_read_u32 (node,"msi-parent" , &imsic_phandle); +if ( !rc ) +panic ("%s: IDC mode not supported\n" , node->full_name); +
xen/arch/riscv/aplic.c line 78
Quoted 10 lines
xen/arch/riscv/aplic.c +10
-0+ imsic_node =dt_find_node_by_phandle (imsic_phandle); +if ( !imsic_node ) +panic ("%s: unable to find IMSIC node\n" , node->full_name); + +/* check imsic mode */ + rc =dt_property_read_u32_array (imsic_node,"interrupts-extended" , + irq_range,ARRAY_SIZE (irq_range)); +if ( rc && (rc != -EOVERFLOW ) ) +panic ("%s: unable to find interrupt-extended in %s node\n" , + node->full_name, imsic_node->full_name);
Why exactly is EOVERFLOW tolerable here?
Quoted 12 lines
xen/arch/riscv/aplic.c +12
-0+if ( irq_range[1 ] ==IRQ_M_EXT ) +/* machine mode imsic node, ignore this aplic node */ +return 0 ; + + rc =imsic_init (imsic_node); +if ( rc ) +panic ("%s: Failded to initialize IMSIC\n" , node->full_name); + +/* Find out number of interrupt sources */ + rc =dt_property_read_u32 (node,"riscv,num-sources" , &aplic_info.nr_irqs); +if ( !rc ) + panic("%s: failed to get number of interrupt sources\n",
xen/arch/riscv/aplic.c line 101
Quoted 10 lines
xen/arch/riscv/aplic.c +10
-0+ node->full_name); + + prop =dt_get_property (node,"reg" ,NULL ); +dt_get_range (&prop, node, &paddr, &size); +if ( !paddr ) +panic ("%s: first MMIO resource not found\n" , node->full_name); + + aplic.paddr_start = paddr; + aplic.paddr_end = paddr + size; + aplic.size = size;
Why do all three need recording? Isn't a (start,size) tuple sufficient (and unambiguous)?
xen/arch/riscv/aplic.c line 113
Quoted 11 lines
xen/arch/riscv/aplic.c +11
-0+ aplic.regs =ioremap (paddr, size); +if ( !aplic.regs ) +panic ("%s: unable to map\n" , node->full_name); + +/* Setup initial state APLIC interrupts */ +aplic_init_hw_interrupts (); + +return 0 ; +} + +static const struct intc_hw_operations __ro_after_init aplic_ops = {
const or __ro_after_init?
Quoted 70 lines
xen/arch/riscv/include/asm/aplic.h new file +67
-0@@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: MIT */ + +/* +* xen/arch/riscv/aplic.h +* +* RISC-V Advanced Platform-Level Interrupt Controller support +* +* Copyright (c) 2023 Microchip. +*/ + +#ifndef ASM__RISCV__APLIC_H +#define ASM__RISCV__APLIC_H + +#include <xen/types.h> + +#include <asm/imsic.h> + +#define APLIC_DOMAINCFG_IE BIT(8, UL) +#define APLIC_DOMAINCFG_DM BIT(2, UL) + +struct aplic_regs { + uint32_t domaincfg; + uint32_t sourcecfg[1023]; + uint8_t _reserved1[0xBC0]; + + uint32_t mmsiaddrcfg; + uint32_t mmsiaddrcfgh; + uint32_t smsiaddrcfg; + uint32_t smsiaddrcfgh; + uint8_t _reserved2[0x30]; + + uint32_t setip[32]; + uint8_t _reserved3[92]; + + uint32_t setipnum; + uint8_t _reserved4[0x20]; + + uint32_t in_clrip[32]; + uint8_t _reserved5[92]; + + uint32_t clripnum; + uint8_t _reserved6[32]; + + uint32_t setie[32]; + uint8_t _reserved7[92]; + + uint32_t setienum; + uint8_t _reserved8[32]; + + uint32_t clrie[32]; + uint8_t _reserved9[92]; + + uint32_t clrienum; + uint8_t _reserved10[32]; + + uint32_t setipnum_le; + uint32_t setipnum_be; + uint8_t _reserved11[4088]; + + uint32_t genmsi; + uint32_t target[1023]; +}; + +struct aplic_priv { + /* base physical address and size */ + paddr_t paddr_start; + paddr_t paddr_end;
xen/arch/riscv/include/asm/aplic.h line 77
Quoted 10 lines
xen/arch/riscv/include/asm/aplic.h new file +10
-0+size_t size; + +/* registers */ +volatile struct aplic_regs *regs; + +/* imsic configuration */ + const struct imsic_config *imsic_cfg; +}; + +#endif /* ASM__RISCV__APLIC_H */
Does all of this really need to live in a non-private header?
xen/arch/riscv/include/asm/irq.h line 30
Quoted 7 lines
xen/arch/riscv/include/asm/irq.h +0
-1@@ -27,7 +27,6 @@#define IRQ_TYPE_INVALID DT_IRQ_TYPE_INVALID/* TODO */ -#define nr_irqs 0U
How come this is simply no longer needed, i.e. without any replacement? Hmm, looks like the only use in common code has gone away. Yet then this still doesn't really look to belong here (especially if not mentioned in the description).
Jan
Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation
Quoted 30 lines · Jan Beulich
Quoted 29 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +10
-0@@ -9,19 +9,112 @@* Copyright (c) 2024-2025 Vates */ +#include <xen/device_tree.h> #include <xen/errno.h> #include <xen/init.h> #include <xen/irq.h> +#include <xen/mm.h> #include <xen/sections.h> #include <xen/types.h> +#include <xen/vmap.h> +#include <asm/aplic.h> #include <asm/device.h> +#include <asm/imsic.h> #include <asm/intc.h> +#include <asm/riscv_encoding.h> + +#define APLIC_DEFAULT_PRIORITY 1 + +static struct aplic_priv aplic;static struct intc_info __ro_after_init aplic_info = { .hw_version =INTC_APLIC , };
Quoted 14 lines
xen/arch/riscv/aplic.c line 47
Quoted 13 lines
xen/arch/riscv/aplic.c +13
-0+static void __init aplic_init_hw_interrupts (void ) +{ +int i; + +/* Disable all interrupts */ +for ( i =0 ; i <= aplic_info.nr_irqs; i +=32 ) + aplic.regs->clrie[i] =-1U ; + +/* Set interrupt type and default priority for all interrupts */ +for ( i =1 ; i <= aplic_info.nr_irqs; i++ ) + { + aplic.regs->sourcecfg[i -1 ] =0 ; + aplic.regs->target[i -1 ] =APLIC_DEFAULT_PRIORITY ;A field named "target" is written with a priority value?
Low bits of target register contains Interrupt Priority bits which can't be zero according to AIA spec:
``` 4.5.16.1. Active source, direct delivery mode For an active interrupt source , if the domain is configured in direct delivery mode (domaincfg.DM = 0), then register target[ ] has this format: bits 31:18 Hart Index (WLRL) bits 7:0 IPRIO (WARL) All other register bits are reserved and read as zeros. Hart Index is a WLRL field that specifies the hart to which interrupts from this source will be delivered. Field IPRIO (Interrupt Priority) specifies the priority number for the interrupt source. This field is a WARL unsigned integer of IPRIOLEN bits, where IPRIOLEN is a constant parameter for the given APLIC, in the range of 1 to 8. Only values 1 through are allowed for IPRIO, not zero. A write to a target register sets IPRIO equal to bits :0 of the 32-bit value written, unless those bits are all zeros, in which case the priority number is set to 1 instead. (If IPRIOLEN = 1, these rules cause IPRIO to be effectively read-only with value 1.) ```
Quoted 6 lines
xen/arch/riscv/aplic.c line 51
Quoted 5 lines
xen/arch/riscv/aplic.c +4
-0+ } + +/* Clear APLIC domaincfg */ + aplic.regs->domaincfg =APLIC_DOMAINCFG_IE |APLIC_DOMAINCFG_DM ;The statement doesn't like like there was any "clearing" here.
But all other bits, except|APLIC_DOMAINCFG_{IE, DM}|, are set to zero. I think we can remove this comment to avoid confusion.
Quoted 7 lines
Quoted 7 lines
xen/arch/riscv/aplic.c +7
-0+ rc =dt_property_read_u32 (node,"msi-parent" , &imsic_phandle); +if ( !rc ) +panic ("%s: IDC mode not supported\n" , node->full_name); + + imsic_node =dt_find_node_by_phandle (imsic_phandle); +if ( !imsic_node ) +panic ("%s: unable to find IMSIC node\n" , node->full_name);
Quoted 8 lines
xen/arch/riscv/aplic.c line 78
Quoted 7 lines
xen/arch/riscv/aplic.c +7
-0+ +/* check imsic mode */ + rc =dt_property_read_u32_array (imsic_node,"interrupts-extended" , + irq_range,ARRAY_SIZE (irq_range)); +if ( rc && (rc != -EOVERFLOW ) ) +panic ("%s: unable to find interrupt-extended in %s node\n" , + node->full_name, imsic_node->full_name);Why exactly is EOVERFLOW tolerable here?
QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode. For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only interested in the S-mode IMSIC node.
The IMSIC node includes this information in the|"interrupts-extended"| property, which has the following format:
interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.
For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:
if ( irq_range[1] == IRQ_M_EXT )
Thereby dt_property_read_u32_array() will return -EOVERFLOW in the case when a platfrom has more then one CPU as we passed irq_range[2] as an argument but the amount of values in "interrupts-extended" property will be (2 * CPUS_NUM).
I can update the comment above dt_property_read_u32_array() for more clearness.
Quoted 16 lines
Quoted 16 lines
xen/arch/riscv/aplic.c +16
-0+if ( irq_range[1 ] ==IRQ_M_EXT ) +/* machine mode imsic node, ignore this aplic node */ +return 0 ; + + rc =imsic_init (imsic_node); +if ( rc ) +panic ("%s: Failded to initialize IMSIC\n" , node->full_name); + +/* Find out number of interrupt sources */ + rc =dt_property_read_u32 (node,"riscv,num-sources" , &aplic_info.nr_irqs); +if ( !rc ) +panic ("%s: failed to get number of interrupt sources\n" , + node->full_name); + + prop =dt_get_property (node,"reg" ,NULL ); +dt_get_range (&prop, node, &paddr, &size);
Quoted 8 lines
xen/arch/riscv/aplic.c line 101
Quoted 6 lines
xen/arch/riscv/aplic.c +6
-0+if ( !paddr ) +panic ("%s: first MMIO resource not found\n" , node->full_name); + + aplic.paddr_start = paddr; + aplic.paddr_end = paddr + size; + aplic.size = size;Why do all three need recording? Isn't a (start,size) tuple sufficient (and unambiguous)?
(start,size) will be enough. I'll drop aplic.paddr_end.
Quoted 13 lines
xen/arch/riscv/aplic.c line 113
Quoted 12 lines
xen/arch/riscv/aplic.c +11
-0+ aplic.regs =ioremap (paddr, size); +if ( !aplic.regs ) +panic ("%s: unable to map\n" , node->full_name); + +/* Setup initial state APLIC interrupts */ +aplic_init_hw_interrupts (); + +return 0 ; +} + +static const struct intc_hw_operations __ro_after_init aplic_ops = {const or
__ro_after_init?
What’s wrong with using both?|const| ensures the variable can't be changed at compile time, while|__ro_after_init| makes it read-only at runtime after initialization is complete.
Probably,|__initconst| would be a better fit:
static const struct intc_hw_operations __initconst aplic_ops = {
Or even|__initconstrel|, since the|struct intc_hw_operations| contains pointers.
Quoted 74 lines
Quoted 74 lines
xen/arch/riscv/include/asm/aplic.h new file +70
-0@@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: MIT */ + +/* +* xen/arch/riscv/aplic.h +* +* RISC-V Advanced Platform-Level Interrupt Controller support +* +* Copyright (c) 2023 Microchip. +*/ + +#ifndef ASM__RISCV__APLIC_H +#define ASM__RISCV__APLIC_H + +#include <xen/types.h> + +#include <asm/imsic.h> + +#define APLIC_DOMAINCFG_IE BIT(8, UL) +#define APLIC_DOMAINCFG_DM BIT(2, UL) + +struct aplic_regs { + uint32_t domaincfg; + uint32_t sourcecfg[1023]; + uint8_t _reserved1[0xBC0]; + + uint32_t mmsiaddrcfg; + uint32_t mmsiaddrcfgh; + uint32_t smsiaddrcfg; + uint32_t smsiaddrcfgh; + uint8_t _reserved2[0x30]; + + uint32_t setip[32]; + uint8_t _reserved3[92]; + + uint32_t setipnum; + uint8_t _reserved4[0x20]; + + uint32_t in_clrip[32]; + uint8_t _reserved5[92]; + + uint32_t clripnum; + uint8_t _reserved6[32]; + + uint32_t setie[32]; + uint8_t _reserved7[92]; + + uint32_t setienum; + uint8_t _reserved8[32]; + + uint32_t clrie[32]; + uint8_t _reserved9[92]; + + uint32_t clrienum; + uint8_t _reserved10[32]; + + uint32_t setipnum_le; + uint32_t setipnum_be; + uint8_t _reserved11[4088]; + + uint32_t genmsi; + uint32_t target[1023]; +}; + +struct aplic_priv { + /* base physical address and size */ + paddr_t paddr_start; + paddr_t paddr_end; + size_t size; + + /* registers */
Quoted 8 lines
xen/arch/riscv/include/asm/aplic.h line 77
Quoted 7 lines
xen/arch/riscv/include/asm/aplic.h new file +7
-0+volatile struct aplic_regs *regs; + +/* imsic configuration */ + const struct imsic_config *imsic_cfg; +}; + +#endif /* ASM__RISCV__APLIC_H */ Does all of this really need to live in a non-private header?
struct aplic_priv is used in different files:
- in aplic.c to define aplic variable.
- in vaplic.c (which isn't intoduced yet) is used in several places:https://gitlab.com/xen-project/people/olkur/xen/-/blob/latest/xen/arch/riscv/vaplic.c#L41
struct aplic_regs is used only in aplic.c (at least, at the moment) so could be moved to aplic.c, but I don't see too much sense.
Quoted 6 lines
Quoted 6 lines
xen/arch/riscv/include/asm/irq.h +0
-0@@ -27,7 +27,6 @@#define IRQ_TYPE_INVALID DT_IRQ_TYPE_INVALID
Quoted 6 lines
xen/arch/riscv/include/asm/irq.h line 30
Quoted 2 lines
xen/arch/riscv/include/asm/irq.h +0
-1/* TODO */ -#define nr_irqs 0UHow come this is simply no longer needed, i.e. without any replacement? Hmm, looks like the only use in common code has gone away. Yet then this still doesn't really look to belong here (especially if not mentioned in the description).
I missed that it is used in xen/common/domain.c when CONFIG_HAS_PIRQ=y, but this config isn't selected for RISC-V.
I think that I have to revert this change.
~ Oleksii
Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation
Quoted 29 lines · Oleksii Kurochko
Quoted 16 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +4
-0+ rc =dt_property_read_u32 (node,"msi-parent" , &imsic_phandle); +if ( !rc ) +panic ("%s: IDC mode not supported\n" , node->full_name); +xen/arch/riscv/aplic.c line 78
Quoted 10 lines
xen/arch/riscv/aplic.c +10
-0+ imsic_node =dt_find_node_by_phandle (imsic_phandle); +if ( !imsic_node ) +panic ("%s: unable to find IMSIC node\n" , node->full_name); + +/* check imsic mode */ + rc =dt_property_read_u32_array (imsic_node,"interrupts-extended" , + irq_range,ARRAY_SIZE (irq_range)); +if ( rc && (rc != -EOVERFLOW ) ) +panic ("%s: unable to find interrupt-extended in %s node\n" , + node->full_name, imsic_node->full_name);Why exactly is EOVERFLOW tolerable here?
QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode. For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only interested in the S-mode IMSIC node.
The IMSIC node includes this information in the|"interrupts-extended"| property, which has the following format:
interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:
Quoted 7 lines
if ( irq_range[1] == IRQ_M_EXT )Thereby
dt_property_read_u32_array()will return-EOVERFLOWin the case when a platfrom has more then one CPU as we passed irq_range[2] as an argument but the amount of values in "interrupts-extended" property will be (2 *CPUS_NUM).I can update the comment above
dt_property_read_u32_array()for more clearness.
Yet my question remains: Why would it be okay to ignore the remaining entries, and hence accept -EOVERFLOW as kind-of-success?
Quoted 13 lines
Quoted 4 lines
Quoted 4 lines
xen/arch/riscv/aplic.c +4
-0+ aplic.regs =ioremap (paddr, size); +if ( !aplic.regs ) +panic ("%s: unable to map\n" , node->full_name); +Quoted 8 lines
xen/arch/riscv/aplic.c line 113
Quoted 7 lines
xen/arch/riscv/aplic.c +7
-0+/* Setup initial state APLIC interrupts */ +aplic_init_hw_interrupts (); + +return 0 ; +} + +static const struct intc_hw_operations __ro_after_init aplic_ops = {const or
__ro_after_init?
Quoted 2 lines
What’s wrong with using both?|const| ensures the variable can't be changed at compile time, while|__ro_after_init| makes it read-only at runtime after initialization is complete.
No, const makes it read-only at compile- and run-time. __ro_after_init, putting the item into a special section, makes it writable at init-time. Due to the const, the compiler wouldn't emit any writes. But we can also avoid stray writes by having the item live in .rodata.
Quoted 4 lines
Probably,|__initconst| would be a better fit:
static const struct intc_hw_operations __initconst aplic_ops = {Or even|__initconstrel|, since the|struct intc_hw_operations| contains pointers.
Well, if this variable isn't accessed post-init, sure. That seems pretty unlikely though, considering it contains pointers to hook functions.
Quoted 82 lines
Quoted 73 lines
Quoted 73 lines
xen/arch/riscv/include/asm/aplic.h new file +70
-0@@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: MIT */ + +/* +* xen/arch/riscv/aplic.h +* +* RISC-V Advanced Platform-Level Interrupt Controller support +* +* Copyright (c) 2023 Microchip. +*/ + +#ifndef ASM__RISCV__APLIC_H +#define ASM__RISCV__APLIC_H + +#include <xen/types.h> + +#include <asm/imsic.h> + +#define APLIC_DOMAINCFG_IE BIT(8, UL) +#define APLIC_DOMAINCFG_DM BIT(2, UL) + +struct aplic_regs { + uint32_t domaincfg; + uint32_t sourcecfg[1023]; + uint8_t _reserved1[0xBC0]; + + uint32_t mmsiaddrcfg; + uint32_t mmsiaddrcfgh; + uint32_t smsiaddrcfg; + uint32_t smsiaddrcfgh; + uint8_t _reserved2[0x30]; + + uint32_t setip[32]; + uint8_t _reserved3[92]; + + uint32_t setipnum; + uint8_t _reserved4[0x20]; + + uint32_t in_clrip[32]; + uint8_t _reserved5[92]; + + uint32_t clripnum; + uint8_t _reserved6[32]; + + uint32_t setie[32]; + uint8_t _reserved7[92]; + + uint32_t setienum; + uint8_t _reserved8[32]; + + uint32_t clrie[32]; + uint8_t _reserved9[92]; + + uint32_t clrienum; + uint8_t _reserved10[32]; + + uint32_t setipnum_le; + uint32_t setipnum_be; + uint8_t _reserved11[4088]; + + uint32_t genmsi; + uint32_t target[1023]; +}; + +struct aplic_priv { + /* base physical address and size */ + paddr_t paddr_start; + paddr_t paddr_end; + size_t size; + + /* registers */Quoted 8 lines
xen/arch/riscv/include/asm/aplic.h line 77
Quoted 7 lines
xen/arch/riscv/include/asm/aplic.h new file +7
-0+volatile struct aplic_regs *regs; + +/* imsic configuration */ + const struct imsic_config *imsic_cfg; +}; + +#endif /* ASM__RISCV__APLIC_H */ Does all of this really need to live in a non-private header?
Quoted 3 lines
struct
aplic_privis used in different files:
- inaplic.cto defineaplicvariable.
- invaplic.c(which isn't intoduced yet) is used in several places:https://gitlab.com/xen-project/people/olkur/xen/-/blob/latest/xen/arch/riscv/vaplic.c#L41
Which would still call for a private header (xen/arch/riscv/aplic.h).
Quoted 2 lines
struct
aplic_regsis used only inaplic.c(at least, at the moment) so could be moved toaplic.c, but I don't see too much sense.
It is generally good practice to limit the scope of things as much as possible. Just to avoid (or make more noticeable) mis-uses or layering violations, for example.
Quoted 12 lines
Quoted 11 lines
xen/arch/riscv/include/asm/irq.h line 30
Quoted 7 lines
xen/arch/riscv/include/asm/irq.h +0
-1@@ -27,7 +27,6 @@#define IRQ_TYPE_INVALID DT_IRQ_TYPE_INVALID/* TODO */ -#define nr_irqs 0UHow come this is simply no longer needed, i.e. without any replacement? Hmm, looks like the only use in common code has gone away. Yet then this still doesn't really look to belong here (especially if not mentioned in the description).
Quoted 3 lines
I missed that it is used in
xen/common/domain.cwhenCONFIG_HAS_PIRQ=y, but this config isn't selected for RISC-V.
I think that I have to revert this change.
I don't think you need to, as long as you don't mean to select HAS_PIRQ for RISC-V. It's just that the change looks entirely unrelated _here_.
Jan
Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation
Quoted 31 lines · Jan Beulich
Quoted 30 lines · Oleksii Kurochko
Quoted 16 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +4
-0+ rc =dt_property_read_u32 (node,"msi-parent" , &imsic_phandle); +if ( !rc ) +panic ("%s: IDC mode not supported\n" , node->full_name); +xen/arch/riscv/aplic.c line 78
Quoted 10 lines
xen/arch/riscv/aplic.c +10
-0+ imsic_node =dt_find_node_by_phandle (imsic_phandle); +if ( !imsic_node ) +panic ("%s: unable to find IMSIC node\n" , node->full_name); + +/* check imsic mode */ + rc =dt_property_read_u32_array (imsic_node,"interrupts-extended" , + irq_range,ARRAY_SIZE (irq_range)); +if ( rc && (rc != -EOVERFLOW ) ) +panic ("%s: unable to find interrupt-extended in %s node\n" , + node->full_name, imsic_node->full_name);Why exactly is EOVERFLOW tolerable here?
QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode. For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only interested in the S-mode IMSIC node.
The IMSIC node includes this information in the|"interrupts-extended"| property, which has the following format:
interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:
if ( irq_range[1] == IRQ_M_EXT )
Quoted 7 lines
Quoted 5 lines
Thereby
dt_property_read_u32_array()will return-EOVERFLOWin the case when a platfrom has more then one CPU as we passed irq_range[2] as an argument but the amount of values in "interrupts-extended" property will be (2 *CPUS_NUM).I can update the comment above
dt_property_read_u32_array()for more clearness.Yet my question remains: Why would it be okay to ignore the remaining entries, and hence accept
-EOVERFLOWas kind-of-success?
Because for other entries the IMSIC mode will be the same and the difference will be only in interrupt controller's phandle which we don't care about in this function and cares only about in imisic_init(), look at usage of imsic_get_parent_hartid().
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
xen/arch/riscv/aplic.c line 113
Quoted 12 lines
xen/arch/riscv/aplic.c +11
-0+ aplic.regs =ioremap (paddr, size); +if ( !aplic.regs ) +panic ("%s: unable to map\n" , node->full_name); + +/* Setup initial state APLIC interrupts */ +aplic_init_hw_interrupts (); + +return 0 ; +} + +static const struct intc_hw_operations __ro_after_init aplic_ops = {const or
__ro_after_init?
Quoted 6 lines
Quoted 2 lines
What’s wrong with using both?|const| ensures the variable can't be changed at compile time, while|__ro_after_init| makes it read-only at runtime after initialization is complete.
No, const makes it read-only at compile- and run-time.__ro_after_init, putting the item into a special section, makes it writable at init-time. Due to the const, the compiler wouldn't emit any writes. But we can also avoid stray writes by having the item live in .rodata.
Oh, right, const will add the variable to .rodata.
Then I think it is enough to have const as aplic_ops is going to be initialized once and then only read will happen.
Quoted 7 lines
Quoted 5 lines
Probably,|__initconst| would be a better fit:
static const struct intc_hw_operations __initconst aplic_ops = {Or even|__initconstrel|, since the|struct intc_hw_operations| contains pointers.
Well, if this variable isn't accessed post-init, sure. That seems pretty unlikely though, considering it contains pointers to hook functions.
Sure, .init section is going to be freed after init-time.
~ Oleksii
Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation
Quoted 37 lines · Oleksii Kurochko
Quoted 35 lines · Jan Beulich
Quoted 34 lines · Oleksii Kurochko
Quoted 16 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +4
-0+ rc =dt_property_read_u32 (node,"msi-parent" , &imsic_phandle); +if ( !rc ) +panic ("%s: IDC mode not supported\n" , node->full_name); +xen/arch/riscv/aplic.c line 78
Quoted 10 lines
xen/arch/riscv/aplic.c +10
-0+ imsic_node =dt_find_node_by_phandle (imsic_phandle); +if ( !imsic_node ) +panic ("%s: unable to find IMSIC node\n" , node->full_name); + +/* check imsic mode */ + rc =dt_property_read_u32_array (imsic_node,"interrupts-extended" , + irq_range,ARRAY_SIZE (irq_range)); +if ( rc && (rc != -EOVERFLOW ) ) +panic ("%s: unable to find interrupt-extended in %s node\n" , + node->full_name, imsic_node->full_name);Why exactly is EOVERFLOW tolerable here?
QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode. For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only interested in the S-mode IMSIC node.
The IMSIC node includes this information in the|"interrupts-extended"| property, which has the following format:
interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:
if ( irq_range[1] == IRQ_M_EXT )Thereby
dt_property_read_u32_array()will return-EOVERFLOWin the case when a platfrom has more then one CPU as we passed irq_range[2] as an argument but the amount of values in "interrupts-extended" property will be (2 *CPUS_NUM).
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
I can update the comment above
dt_property_read_u32_array()for more clearness.Yet my question remains: Why would it be okay to ignore the remaining entries, and hence accept
-EOVERFLOWas kind-of-success?Because for other entries the IMSIC mode will be the same and the difference will be only in interrupt controller's phandle
And we can blindly take this for granted? Would you mind extending the comment that's there to include this aspect?
Jan
Quoted 2 lines
which we don't care about in this function and cares only about in
imisic_init(), look at usage ofimsic_get_parent_hartid().
Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation
Quoted 37 lines · Jan Beulich
Quoted 36 lines · Oleksii Kurochko
Quoted 35 lines · Jan Beulich
Quoted 34 lines · Oleksii Kurochko
Quoted 16 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +4
-0+ rc =dt_property_read_u32 (node,"msi-parent" , &imsic_phandle); +if ( !rc ) +panic ("%s: IDC mode not supported\n" , node->full_name); +xen/arch/riscv/aplic.c line 78
Quoted 10 lines
xen/arch/riscv/aplic.c +10
-0+ imsic_node =dt_find_node_by_phandle (imsic_phandle); +if ( !imsic_node ) +panic ("%s: unable to find IMSIC node\n" , node->full_name); + +/* check imsic mode */ + rc =dt_property_read_u32_array (imsic_node,"interrupts-extended" , + irq_range,ARRAY_SIZE (irq_range)); +if ( rc && (rc != -EOVERFLOW ) ) +panic ("%s: unable to find interrupt-extended in %s node\n" , + node->full_name, imsic_node->full_name);Why exactly is EOVERFLOW tolerable here?
QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode. For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only interested in the S-mode IMSIC node.
The IMSIC node includes this information in the|"interrupts-extended"| property, which has the following format:
interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:
if ( irq_range[1] == IRQ_M_EXT )Thereby
dt_property_read_u32_array()will return-EOVERFLOWin the case when a platfrom has more then one CPU as we passed irq_range[2] as an argument but the amount of values in "interrupts-extended" property will be (2 *CPUS_NUM).
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
I can update the comment above
dt_property_read_u32_array()for more clearness.Yet my question remains: Why would it be okay to ignore the remaining entries, and hence accept
-EOVERFLOWas kind-of-success?Because for other entries the IMSIC mode will be the same and the difference will be only in interrupt controller's phandle
And we can blindly take this for granted? Would you mind extending the comment that's there to include this aspect?
I tried to compile dtc with different modes in interrupt-extends and compilation doesn't failed, so it's not really granted.
Just to be sure, I'll check all items of interrupts-extend not just the first one.
~ Oleksii
Quoted 2 lines
Quoted 2 lines
which we don't care about in this function and cares only about in
imisic_init(), look at usage ofimsic_get_parent_hartid().
PATCH 10/14 xen/riscv: implementation of aplic and imsic operations
In reply to Oleksii KurochkoIntroduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
- aplic_irq_startup()
- aplic_irq_shutdown()
- aplic_irq_enable()
- aplic_irq_disable()
- aplic_irq_ack()
- aplic_host_irq_end()
- aplic_set_irq_affinity()
As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().
For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is introduced to get hart id.
Also, introduce additional interrupt controller h/w operations and host_irq_type for APLIC:
- aplic_host_irq_type
- aplic_set_irq_priority()
- aplic_set_irq_type()
Patch is based on the code from [1].
[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7390e2365828b83e27ead56b03114a56e3699dd5
--- xen/arch/riscv/aplic.c | 169 ++++++++++++++++++++++++++++- xen/arch/riscv/imsic.c | 63 +++++++++++ xen/arch/riscv/include/asm/aplic.h | 12 ++ xen/arch/riscv/include/asm/imsic.h | 15 +++ 4 files changed, 258 insertions(+), 1 deletion(-)
xen/arch/riscv/aplic.c +168 -1
@@ -15,6 +15,7 @@#include <xen/irq.h> #include <xen/mm.h> #include <xen/sections.h> +#include <xen/spinlock.h> #include <xen/types.h> #include <xen/vmap.h>
@@ -110,9 +111,173 @@ static int __init aplic_init(void)return 0 ; } -static const struct intc_hw_operations __ro_after_init aplic_ops = { +static void aplic_set_irq_type (struct irq_desc *desc,unsigned int type) +{ +unsigned int irq = desc->irq -1 ; + +spin_lock (&aplic.lock); +switch (type) { +case IRQ_TYPE_EDGE_RISING : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_EDGE_RISE ; +break ; +case IRQ_TYPE_EDGE_FALLING : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_EDGE_FALL ; +break ; +case IRQ_TYPE_LEVEL_HIGH : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_LEVEL_HIGH ; +break ; +case IRQ_TYPE_LEVEL_LOW : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_LEVEL_LOW ; +break ; +default : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_INACTIVE ; +break ; + } +spin_unlock (&aplic.lock); +} + +static void aplic_set_irq_priority (struct irq_desc *desc, +unsigned int priority) +{ +/* No priority, do nothing */ +} + +static void aplic_irq_enable (struct irq_desc *desc) +{ +unsigned long flags; + +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used); + +ASSERT (spin_is_locked (&desc->lock)); + +spin_lock_irqsave (&aplic.lock, flags); + +clear_bit (_IRQ_DISABLED, &desc->status); + +/* enable interrupt in IMSIC */ +imsic_irq_enable (desc->irq); + +/* enable interrupt in APLIC */ + aplic.regs->setienum = desc->irq; + +spin_unlock_irqrestore (&aplic.lock, flags); +} + +static void aplic_irq_disable (struct irq_desc *desc) +{ +unsigned long flags; + +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used); + +ASSERT (spin_is_locked (&desc->lock)); + +spin_lock_irqsave (&aplic.lock, flags); + +set_bit (_IRQ_DISABLED, &desc->status); + +/* disable interrupt in APLIC */ + aplic.regs->clrienum = desc->irq; + +/* disable interrupt in IMSIC */ +imsic_irq_disable (desc->irq); + +spin_unlock_irqrestore (&aplic.lock, flags); +} + +static unsigned int aplic_irq_startup (struct irq_desc *desc) +{ +aplic_irq_enable (desc); + +return 0 ; +} + +static void aplic_irq_shutdown (struct irq_desc *desc) +{ +aplic_irq_disable (desc); +} + +static void aplic_irq_ack (struct irq_desc *desc) +{ +/* nothing to do */ +} + +static void aplic_host_irq_end (struct irq_desc *desc) +{ +/* nothing to do */ +} + +static unsigned int aplic_get_cpu_from_mask (const cpumask_t *cpumask) +{ +unsigned int cpu; +cpumask_t possible_mask; + +cpumask_and (&possible_mask, cpumask, &cpu_possible_map); + cpu =cpumask_any (&possible_mask); + +return cpu; +} + +static void aplic_set_irq_affinity (struct irq_desc *desc,const cpumask_t *mask) +{ +unsigned int cpu; +uint64_t group_index, base_ppn; +uint32_t hhxw, lhxw ,hhxs, value; +const struct imsic_config *imsic = aplic.imsic_cfg; + +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used); + +ASSERT (!cpumask_empty (mask)); + +spin_lock (&aplic.lock); + + cpu =cpuid_to_hartid (aplic_get_cpu_from_mask (mask)); + hhxw = imsic->group_index_bits; + lhxw = imsic->hart_index_bits; + hhxs = imsic->group_index_shift -IMSIC_MMIO_PAGE_SHIFT *2 ; + base_ppn = imsic->msi[cpu].base_addr >>IMSIC_MMIO_PAGE_SHIFT ; + +/* update hart and EEID in the target register */ + group_index = (base_ppn >> (hhxs +12 )) & (BIT (hhxw,UL ) -1 ); + value = desc->irq; + value |= cpu <<APLIC_TARGET_HART_IDX_SHIFT ; + value |= group_index << (lhxw +APLIC_TARGET_HART_IDX_SHIFT ) ; + aplic.regs->target[desc->irq -1 ] = value; + +spin_unlock (&aplic.lock); +} + +static hw_irq_controller aplic_host_irq_type = { + .typename ="aplic" , + .startup = aplic_irq_startup, + .shutdown = aplic_irq_shutdown, + .enable = aplic_irq_enable, + .disable = aplic_irq_disable, + .ack = aplic_irq_ack, + .end = aplic_host_irq_end, + .set_affinity = aplic_set_irq_affinity, +}; + +static const struct intc_hw_operations aplic_ops = { .info = &aplic_info, .init = aplic_init, + .host_irq_type = &aplic_host_irq_type, + .set_irq_priority = aplic_set_irq_priority, + .set_irq_type = aplic_set_irq_type, }; static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
@@ -149,6 +314,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat) dt_irq_xlate = aplic_irq_xlate; +spin_lock_init (&aplic.lock); +register_intc_ops (&aplic_ops);return 0 ;
xen/arch/riscv/imsic.c +63 -0
81 lines not on this page · raw message
xen/arch/riscv/include/asm/aplic.h +12 -0
@@ -18,6 +18,15 @@#define APLIC_DOMAINCFG_IE BIT(8, UL)#define APLIC_DOMAINCFG_DM BIT(2, UL) +#define APLIC_SOURCECFG_SM_INACTIVE 0x0 +#define APLIC_SOURCECFG_SM_DETACH 0x1 +#define APLIC_SOURCECFG_SM_EDGE_RISE 0x4 +#define APLIC_SOURCECFG_SM_EDGE_FALL 0x5 +#define APLIC_SOURCECFG_SM_LEVEL_HIGH 0x6 +#define APLIC_SOURCECFG_SM_LEVEL_LOW 0x7 + +#define APLIC_TARGET_HART_IDX_SHIFT 18 +struct aplic_regs {uint32_t domaincfg;uint32_t sourcecfg[1023 ];
@@ -70,6 +79,9 @@ struct aplic_priv {/* registers */ volatile struct aplic_regs *regs; +/* lock */ +spinlock_t lock; +/* imsic configuration */ const struct imsic_config *imsic_cfg; };
xen/arch/riscv/include/asm/imsic.h +15 -0
@@ -11,6 +11,7 @@#ifndef ASM__RISCV__IMSIC_H #define ASM__RISCV__IMSIC_H +#include <xen/spinlock.h> #include <xen/types.h> #define IMSIC_MMIO_PAGE_SHIFT 12
@@ -19,6 +20,11 @@#define IMSIC_MIN_ID 63#define IMSIC_MAX_ID 2048 +#define IMSIC_EIP0 0x80 +#define IMSIC_EIPx_BITS 32 + +#define IMSIC_EIE0 0xC0 +struct imsic_msi {paddr_t base_addr;unsigned long offset;
@@ -55,6 +61,12 @@ struct imsic_config {/* MSI */ struct imsic_msi msi[NR_CPUS ]; + +/* a check that IMSIC is used */ +bool is_used; + +/* lock */ +spinlock_t lock; };struct dt_device_node ;
@@ -63,4 +75,7 @@ int imsic_init(struct dt_device_node *n);struct imsic_config ;const struct imsic_config *imsic_get_config (void ); +void imsic_irq_enable (unsigned int hwirq); +void imsic_irq_disable (unsigned int hwirq); +#endif /* ASM__RISCV__IMSIC_H */
Signature · 1 line
2.49.0
Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations
Quoted 14 lines · Oleksii Kurochko
Introduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
-aplic_irq_startup()
-aplic_irq_shutdown()
-aplic_irq_enable()
-aplic_irq_disable()
-aplic_irq_ack()
-aplic_host_irq_end()
-aplic_set_irq_affinity()As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().
Quoted 8 lines
For the purpose of
aplic_set_irq_affinity()aplic_get_cpu_from_mask()is introduced to get hart id.Also, introduce additional interrupt controller h/w operations and
host_irq_typefor APLIC:
-aplic_host_irq_type
-aplic_set_irq_priority()
-aplic_set_irq_type()
Yet these two functions nor the hooks they're used to populate are entirely unused here. Since they're also outside of the common IRQ handling machinery, it's unclear how one would sanely ack such a change.
Quoted 11 lines
xen/arch/riscv/aplic.c +1
-0@@ -15,6 +15,7 @@#include <xen/irq.h> #include <xen/mm.h> #include <xen/sections.h> +#include <xen/spinlock.h> #include <xen/types.h> #include <xen/vmap.h>
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 116
Quoted 7 lines
xen/arch/riscv/aplic.c +3
-1@@ -110,9 +111,173 @@ static int __init aplic_init(void)return 0 ; } -static const struct intc_hw_operations __ro_after_init aplic_ops = { +static void aplic_set_irq_type (struct irq_desc *desc,unsigned int type) +{ +unsigned int irq = desc->irq -1 ;
Why this adjustment by 1 (and yet both items being named "irq")?
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 120
Quoted 3 lines
xen/arch/riscv/aplic.c +3
-0+spin_lock (&aplic.lock); +switch (type) { +case IRQ_TYPE_EDGE_RISING :
Nit (style): Missing blanks, brace on its own line, case labels indented like their containing switch().
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 123
Quoted 3 lines
xen/arch/riscv/aplic.c +3
-0+ aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_EDGE_RISE ; +break ; +case IRQ_TYPE_EDGE_FALLING :
Blank lines please between non-fall-through case blocks.
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 134
Quoted 11 lines
xen/arch/riscv/aplic.c +11
-0+ aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_EDGE_FALL ; +break ; +case IRQ_TYPE_LEVEL_HIGH : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_LEVEL_HIGH ; +break ; +case IRQ_TYPE_LEVEL_LOW : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_LEVEL_LOW ; +break ; +default : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_INACTIVE ; +break ;
Is the default: label legitimate to be reached?
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143
Quoted 9 lines
xen/arch/riscv/aplic.c +9
-0+ } +spin_unlock (&aplic.lock); +} + +static void aplic_set_irq_priority (struct irq_desc *desc, +unsigned int priority) +{ +/* No priority, do nothing */ +}
Since the function dopes nothing, wouldn't it be better to omit it and have the (future) caller check for a NULL pointer ahead of making the (indirect) call? Same remark for other handlers (below) which also do nothing.
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154
Quoted 10 lines
xen/arch/riscv/aplic.c +10
-0+static void aplic_irq_enable (struct irq_desc *desc) +{ +unsigned long flags; + +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used);
Such an extra field, used only for assertions, is pretty odd. Can't you use any of the other fields to achieve the same effect?
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 153
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+ASSERT (spin_is_locked (&desc->lock));
If this lock (which is an IRQ-safe one) is necessarily held, ...
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+spin_lock_irqsave (&aplic.lock, flags);
... you can use just spin_lock() here.
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 160
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+clear_bit (_IRQ_DISABLED, &desc->status);
Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)
I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.
In any event this can be done outside of the APLIC-locked region, I think.
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 162
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+/* enable interrupt in IMSIC */
May I remind you of Xen comment style?
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 166
Quoted 4 lines
xen/arch/riscv/aplic.c +4
-0+imsic_irq_enable (desc->irq); + +/* enable interrupt in APLIC */ + aplic.regs->setienum = desc->irq;
Are you sure you want to use plain assignments for MMIO accesses? I'd have expected writel() to be used here. (And only later I realized that I didn't spot the same already higher up from here.)
From the vague understanding I've gained so far: Isn't the APLIC closer to the CPU and the IMSIC closer to the device? If so, wouldn't you want to enable at the APLIC before enabling at the IMSIC? But of course that also depends on what exactly happens in the window while one is already enabled and the other is still disabled. (Later) From the code you add to imsic.c it looks like it's the other way around, as the IMSIC is accessed through CSRs.
Quoted 30 lines
xen/arch/riscv/aplic.c +30
-0+spin_unlock_irqrestore (&aplic.lock, flags); +} + +static void aplic_irq_disable (struct irq_desc *desc) +{ +unsigned long flags; + +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used); + +ASSERT (spin_is_locked (&desc->lock)); + +spin_lock_irqsave (&aplic.lock, flags); + +set_bit (_IRQ_DISABLED, &desc->status); + +/* disable interrupt in APLIC */ + aplic.regs->clrienum = desc->irq; + +/* disable interrupt in IMSIC */ +imsic_irq_disable (desc->irq); + +spin_unlock_irqrestore (&aplic.lock, flags); +} + +static unsigned int aplic_irq_startup (struct irq_desc *desc)
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 207
Quoted 10 lines
xen/arch/riscv/aplic.c +10
-0+{ +aplic_irq_enable (desc); + +return 0 ; +} + +static void aplic_irq_shutdown (struct irq_desc *desc) +{ +aplic_irq_disable (desc); +}
You don't really need a separate hook function here, do you?
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 214
Quoted 6 lines
xen/arch/riscv/aplic.c +6
-0+static void aplic_irq_ack (struct irq_desc *desc) +{ +/* nothing to do */ +} + +static void aplic_host_irq_end (struct irq_desc *desc)
What's the "host" in the identifier about?
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 221
Quoted 7 lines
xen/arch/riscv/aplic.c +7
-0+{ +/* nothing to do */ +} + +static unsigned int aplic_get_cpu_from_mask (const cpumask_t *cpumask) +{ +unsigned int cpu;
No real need for this variable?
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 225
Quoted 4 lines
xen/arch/riscv/aplic.c +4
-0+cpumask_t possible_mask; + +cpumask_and (&possible_mask, cpumask, &cpu_possible_map); + cpu =cpumask_any (&possible_mask);
Why would you use cpu_possible_map here? That includes any offline CPUs. I think you need to use cpu_online_map here.
Quoted 6 lines
xen/arch/riscv/aplic.c +6
-0+return cpu; +} + +static void aplic_set_irq_affinity (struct irq_desc *desc,const cpumask_t *mask) +{ +unsigned int cpu;
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 242
Quoted 10 lines
xen/arch/riscv/aplic.c +10
-0+uint64_t group_index, base_ppn; +uint32_t hhxw, lhxw ,hhxs, value; +const struct imsic_config *imsic = aplic.imsic_cfg; + +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used);
Use the local variable you have made yourself?
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 246
Quoted 3 lines
xen/arch/riscv/aplic.c +3
-0+ASSERT (!cpumask_empty (mask)); + +spin_lock (&aplic.lock);
Aiui the lock can be acquired quite a bit later. It ought to be needed only around the actual write to the hardware register.
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 255
Quoted 8 lines
xen/arch/riscv/aplic.c +8
-0+ cpu =cpuid_to_hartid (aplic_get_cpu_from_mask (mask)); + hhxw = imsic->group_index_bits; + lhxw = imsic->hart_index_bits; + hhxs = imsic->group_index_shift -IMSIC_MMIO_PAGE_SHIFT *2 ; + base_ppn = imsic->msi[cpu].base_addr >>IMSIC_MMIO_PAGE_SHIFT ; + +/* update hart and EEID in the target register */ + group_index = (base_ppn >> (hhxs +12 )) & (BIT (hhxw,UL ) -1 );
What's this magic 12 in here? Not IMSIC_MMIO_PAGE_SHIFT I suppose?
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 264
Quoted 9 lines
xen/arch/riscv/aplic.c +9
-0+ value = desc->irq; + value |= cpu <<APLIC_TARGET_HART_IDX_SHIFT ; + value |= group_index << (lhxw +APLIC_TARGET_HART_IDX_SHIFT ) ; + aplic.regs->target[desc->irq -1 ] = value; + +spin_unlock (&aplic.lock); +} + +static hw_irq_controller aplic_host_irq_type = {
const?
Quoted 19 lines
xen/arch/riscv/imsic.c +7
-0@@ -14,12 +14,68 @@#include <xen/errno.h> #include <xen/init.h> #include <xen/macros.h> +#include <xen/spinlock.h> #include <xen/xmalloc.h> #include <asm/imsic.h> static struct imsic_config imsic_cfg; +#define imsic_csr_set (c, v) \ +do { \ + csr_write(CSR_SISELECT, c); \ + csr_set(CSR_SIREG, v); \ +} while (0) +
xen/arch/riscv/imsic.c line 39
Quoted 10 lines
xen/arch/riscv/imsic.c +10
-0+#define imsic_csr_clear (c, v) \ +do { \ + csr_write(CSR_SISELECT, c); \ + csr_clear(CSR_SIREG, v); \ +} while (0) + +static void imsic_local_eix_update (unsigned long base_id,unsigned long num_id, +bool pend,bool val) +{ +unsigned long i, isel, ireg;
These can be constrained to inside the outer loop below.
xen/arch/riscv/imsic.c line 46
Quoted 7 lines
xen/arch/riscv/imsic.c +7
-0+unsigned long id = base_id, last_id = base_id + num_id; + +while ( id < last_id ) + { + isel = id / __riscv_xlen; + isel *= __riscv_xlen / IMSIC_EIPx_BITS; + isel += (pend) ?IMSIC_EIP0 :IMSIC_EIE0 ;
Nit: Why the parentheses?
xen/arch/riscv/imsic.c line 52
Quoted 5 lines
xen/arch/riscv/imsic.c +5
-0+ ireg =0 ; +for ( i = id & (__riscv_xlen -1 ); + (id < last_id) && (i < __riscv_xlen); + i++, id++ ) + ireg |= (1 << i);
I wonder if this calculation really needs a loop. Afaict it's just a consecutive set of bits you mean to set.
xen/arch/riscv/imsic.c line 66
Quoted 13 lines
xen/arch/riscv/imsic.c +13
-0+if ( val ) +imsic_csr_set (isel, ireg); +else +imsic_csr_clear (isel, ireg); + } +} + +void imsic_irq_enable (unsigned int hwirq) +{ +unsigned long flags; + +spin_lock_irqsave (&imsic_cfg.lock, flags); +imsic_local_eix_update (hwirq,1 , false, true);
No subtraction of 1 here? Also, why "hwirq" and not just "irq"?
Quoted 16 lines
xen/arch/riscv/imsic.c +12
-0+spin_unlock_irqrestore (&imsic_cfg.lock, flags); +} + +void imsic_irq_disable (unsigned int hwirq) +{ +unsigned long flags; + +spin_lock_irqsave (&imsic_cfg.lock, flags); +imsic_local_eix_update (hwirq,1 , false, false); +spin_unlock_irqrestore (&imsic_cfg.lock, flags); +} +const struct imsic_config *imsic_get_config (void ) {return &imsic_cfg;
xen/arch/riscv/imsic.c int __init imsic_init(struct dt_device_node *node) · line 339
Quoted 7 lines
xen/arch/riscv/imsic.c +4
-0@@ -277,6 +333,13 @@ int __init imsic_init(struct dt_device_node *node) goto imsic_init_err; } +spin_lock_init (&imsic_cfg.lock); + +/* Enable local interrupt delivery */ +imsic_ids_local_delivery (true);
What's this? I can't find the function/macro here, nor in patch 08, nor in staging.
Jan
Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations
Quoted 18 lines · Jan Beulich
Quoted 17 lines · Oleksii Kurochko
Introduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
-aplic_irq_startup()
-aplic_irq_shutdown()
-aplic_irq_enable()
-aplic_irq_disable()
-aplic_irq_ack()
-aplic_host_irq_end()
-aplic_set_irq_affinity()As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().
For the purpose of
aplic_set_irq_affinity()aplic_get_cpu_from_mask()is introduced to get hart id.
Quoted 8 lines
Quoted 5 lines
Also, introduce additional interrupt controller h/w operations and
host_irq_typefor APLIC:
-aplic_host_irq_type
-aplic_set_irq_priority()
-aplic_set_irq_type()Yet these two functions nor the hooks they're used to populate are entirely unused here. Since they're also outside of the common IRQ handling machinery, it's unclear how one would sanely ack such a change.
They will be called by intc_route_irq_to_xen() from setup_irq() during firt time the IRQ is setup.
Quoted 15 lines
Quoted 15 lines
xen/arch/riscv/aplic.c +1
-0@@ -15,6 +15,7 @@#include <xen/irq.h> #include <xen/mm.h> #include <xen/sections.h> +#include <xen/spinlock.h> #include <xen/types.h> #include <xen/vmap.h> @@ -110,9 +111,173 @@ static int __init aplic_init(void)return 0 ; }
Quoted 5 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 116
Quoted 4 lines
xen/arch/riscv/aplic.c +3
-1-static const struct intc_hw_operations __ro_after_init aplic_ops = { +static void aplic_set_irq_type (struct irq_desc *desc,unsigned int type) +{ +unsigned int irq = desc->irq -1 ;Why this adjustment by 1 (and yet both items being named "irq")?
Interrupt 0 isn't possible based on the spec:
``` Each of an APLIC’s interrupt sources has a fixed unique identity number in the range 1 to N, where N is the total number of sources at the APLIC. The number zero is not a valid interrupt identity number at an APLIC. The maximum number of interrupt sources an APLIC may support is 1023. ``` and interrupt 1 will correspond to bit 0 in sourcecfg[] register, interrupt 2 ->sourcecfg[1] and so on. And that is the reason why we need -1. >> + aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_FALL; >> + break; >> + case IRQ_TYPE_LEVEL_HIGH: >> + aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_HIGH; >> + break; >> + case IRQ_TYPE_LEVEL_LOW: >> + aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_LOW; >> + break; >> + default: >> + aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_INACTIVE; >> + break; > Is the default: label legitimate to be reached? From the spec: ```
0 Inactive Inactive in this domain (and not delegated) 1 Detached Active, detached from the source wire 2–3 — Reserved 4 Edge1 Active, edge-sensitive; interrupt asserted on rising edge 5 Edge0 Active, edge-sensitive; interrupt asserted on falling edge 6 Level1 Active, level-sensitive; interrupt asserted when high 7 Level0 Active, level-sensitive; interrupt asserted when low ``` It seems to me like APLIC_SOURCECFG_SM_INACTIVE just covers cases (0-3) and inactive IRQ pretty safe to as a default value.
Quoted 13 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143
Quoted 10 lines
xen/arch/riscv/aplic.c +9
-0+ } +spin_unlock (&aplic.lock); +} + +static void aplic_set_irq_priority (struct irq_desc *desc, +unsigned int priority) +{ +/* No priority, do nothing */ +}Since the function dopes nothing, wouldn't it be better to omit it and have the (future) caller check for a NULL pointer ahead of making the (indirect) call? Same remark for other handlers (below) which also do nothing.
I thought about that too, but it could be some cases when the stub is introduced with temporary BUG_ON("unimplemented") inside just to not miss to implement it when it will be necessary.
If we will have only the caller check then we could miss to implement such stubs.
Quoted 13 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154
Quoted 11 lines
xen/arch/riscv/aplic.c +10
-0+static void aplic_irq_enable (struct irq_desc *desc) +{ +unsigned long flags; + +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used);Such an extra field, used only for assertions, is pretty odd. Can't you use any of the other fields to achieve the same effect?
in aplic_init() there is:
/* check for associated imsic node */
rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
if ( !rc )
panic("%s: IDC mode not supported\n", node->full_name);So we will have panic() anyway if MSI mode isn't supported. As an option we can just drop the ASSERT.
Or introduce static variable in aplic.c aplic_mode, init it in aplic_init() and use it in ASSERT().
Quoted 10 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 136
Quoted 2 lines
xen/arch/riscv/aplic.c +1
-0+ASSERT (spin_is_locked (&desc->lock));If this lock (which is an IRQ-safe one) is necessarily held, ...
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 137
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+spin_lock_irqsave (&aplic.lock, flags);... you can use just
spin_lock()here.xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 160
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+clear_bit (_IRQ_DISABLED, &desc->status);Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)
I haven't thought about that. Likely non-atomic bitop could be used here.
Quoted 5 lines
I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.
In any event this can be done outside of the APLIC-locked region, I think.
Considering that we are doing that under desc->lock, agree we can move that outside the APLIC-locked region.
Quoted 7 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 166
Quoted 4 lines
xen/arch/riscv/aplic.c +4
-0+imsic_irq_enable (desc->irq); + +/* enable interrupt in APLIC */ + aplic.regs->setienum = desc->irq;Are you sure you want to use plain assignments for MMIO accesses? I'd have expected
writel()to be used here. (And only later I realized that I didn't spot the same already higher up from here.)
Good point. I have to update that with writel()...
Quoted 8 lines
From the vague understanding I've gained so far: Isn't the APLIC closer to
the CPU and the IMSIC closer to the device? If so, wouldn't you want to enable at the APLIC before enabling at the IMSIC? But of course that also depends on what exactly happens in the window while one is already enabled and the other is still disabled. (Later) From the code you add toimsic.cit looks like it's the other way around, as the IMSIC is accessed through CSRs.
From the AIA spec:
``` An Incoming MSI Controller (IMSIC) is an optional RISC-V hardware component that is closely coupled with a hart, one IMSIC per hart. An IMSIC receives and records incoming message-signaled interrupts (MSIs) for a hart, and signals to the hart when there are pending and enabled interrupts to be serviced. ```
Based on the figure 2 (Interrupt delivery by MSIs when harts have IMSICs for receiving them) of AIA spechttps://github.com/riscv/riscv-aia/blob/main/src/intrsWithIMSICs.png IMSIC is more close to CPU and APLIC is more close to the device. The external interrupt controller is APLIC and it only sends a MSI message for a CPU.
The logical flow of an interrupt to a hart with an IMSIC would be:
1. A physical interrupt signal arrives at the APLIC.
2. The APLIC, if configured for MSI delivery mode (domaincfg.DM = 1) and if the specific interrupt source is active and enabled within its domain (controlled by sourcecfg[i] and the global Interrupt Enable bit IE in domaincfg), will generate an MSI.
3. This MSI is then sent to the target hart's IMSIC. The APLIC needs to know the MSI target address for each hart, which can be hardwired or configured through registers like mmsiaddrcfg and mmsiaddrcfgh.
4. The receiving hart's IMSIC records this MSI as a pending interrupt.
5. If the corresponding interrupt identity is enabled within the IMSIC's interrupt file, the IMSIC will signal the hart, typically by setting the MEIP or SEIP bit in the mip CSR (or sip CSR).
Generally, I think that the order in which enable interrupts doesn't really matter as if you were to enable the IMSIC to receive a certain interrupt before the APLIC was configured to send it (or had a pending interrupt from the device), the IMSIC would simply be waiting for an MSI that wouldn't arrive.
Similarly, if the APLIC sends an MSI for an interrupt that is not enabled in the IMSIC, the interrupt would remain pending in the IMSIC but wouldn't trigger an interrupt at the hart.
IMO, the order which is used now in the code is pretty logical.
Does it make sense?
Quoted 34 lines
Quoted 34 lines
xen/arch/riscv/aplic.c +33
-0+spin_unlock_irqrestore (&aplic.lock, flags); +} + +static void aplic_irq_disable (struct irq_desc *desc) +{ +unsigned long flags; + +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used); + +ASSERT (spin_is_locked (&desc->lock)); + +spin_lock_irqsave (&aplic.lock, flags); + +set_bit (_IRQ_DISABLED, &desc->status); + +/* disable interrupt in APLIC */ + aplic.regs->clrienum = desc->irq; + +/* disable interrupt in IMSIC */ +imsic_irq_disable (desc->irq); + +spin_unlock_irqrestore (&aplic.lock, flags); +} + +static unsigned int aplic_irq_startup (struct irq_desc *desc) +{ +aplic_irq_enable (desc); +
Quoted 8 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 207
Quoted 7 lines
xen/arch/riscv/aplic.c +7
-0+return 0 ; +} + +static void aplic_irq_shutdown (struct irq_desc *desc) +{ +aplic_irq_disable (desc); +}You don't really need a separate hook function here, do you?
With such implementation it is really not needed to have a hook so I will drop it.
Quoted 7 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 214
Quoted 6 lines
xen/arch/riscv/aplic.c +6
-0+static void aplic_irq_ack (struct irq_desc *desc) +{ +/* nothing to do */ +} + +static void aplic_host_irq_end (struct irq_desc *desc)What's the "host" in the identifier about?
It was copied that from Arm and my understanding that it means Xen-related IRQ as they also have:
```
/* XXX different for level vs edge */
static hw_irq_controller gicv2_host_irq_type = {
...
.end = gicv2_host_irq_end,
...
};
static hw_irq_controller gicv2_guest_irq_type = {
...
.end = gicv2_guest_irq_end,
...
};
```Quoted 9 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 221
Quoted 8 lines
xen/arch/riscv/aplic.c +7
-0+{ +/* nothing to do */ +} + +static unsigned int aplic_get_cpu_from_mask (const cpumask_t *cpumask) +{ +unsigned int cpu;No real need for this variable?
Yes, it could be dropped.
Quoted 7 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 225
Quoted 5 lines
xen/arch/riscv/aplic.c +4
-0+cpumask_t possible_mask; + +cpumask_and (&possible_mask, cpumask, &cpu_possible_map); + cpu =cpumask_any (&possible_mask);Why would you use
cpu_possible_maphere? That includes any offline CPUs. I think you need to usecpu_online_maphere.
It makes sense, Ill switch to cpu_online_map.
Quoted 10 lines
Quoted 10 lines
xen/arch/riscv/aplic.c +9
-0+return cpu; +} + +static void aplic_set_irq_affinity (struct irq_desc *desc,const cpumask_t *mask) +{ +unsigned int cpu; +uint64_t group_index, base_ppn; +uint32_t hhxw, lhxw ,hhxs, value; +const struct imsic_config *imsic = aplic.imsic_cfg;
Quoted 8 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154
Quoted 7 lines
xen/arch/riscv/aplic.c +7
-0+ +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used);Use the local variable you have made yourself?
What do you mean by local here?
Quoted 15 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 246
Quoted 4 lines
xen/arch/riscv/aplic.c +3
-0+ASSERT (!cpumask_empty (mask)); + +spin_lock (&aplic.lock);Aiui the lock can be acquired quite a bit later. It ought to be needed only around the actual write to the hardware register.
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 255
Quoted 8 lines
xen/arch/riscv/aplic.c +8
-0+ cpu =cpuid_to_hartid (aplic_get_cpu_from_mask (mask)); + hhxw = imsic->group_index_bits; + lhxw = imsic->hart_index_bits; + hhxs = imsic->group_index_shift -IMSIC_MMIO_PAGE_SHIFT *2 ; + base_ppn = imsic->msi[cpu].base_addr >>IMSIC_MMIO_PAGE_SHIFT ; + +/* update hart and EEID in the target register */ + group_index = (base_ppn >> (hhxs +12 )) & (BIT (hhxw,UL ) -1 );
Quoted 1 line
What's this magic 12 in here? Not
IMSIC_MMIO_PAGE_SHIFTI suppose?
In the AIA spec they are using 12 explicitly:https://github.com/riscv/riscv-aia/blob/main/src/AdvPLIC.adoc#AdvPLIC-MSIAddrs
Quoted 9 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 233
Quoted 7 lines
xen/arch/riscv/aplic.c +7
-0+unsigned long id = base_id, last_id = base_id + num_id; + +while ( id < last_id ) + { + isel = id / __riscv_xlen; + isel *= __riscv_xlen / IMSIC_EIPx_BITS; + isel += (pend) ?IMSIC_EIP0 :IMSIC_EIE0 ;Nit: Why the parentheses?
Quoted 7 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 238
Quoted 5 lines
xen/arch/riscv/aplic.c +5
-0+ ireg =0 ; +for ( i = id & (__riscv_xlen -1 ); + (id < last_id) && (i < __riscv_xlen); + i++, id++ ) + ireg |= (1 << i);I wonder if this calculation really needs a loop. Afaict it's just a consecutive set of bits you mean to set.
Good point. I will double-check.
Quoted 15 lines
Quoted 4 lines
xen/arch/riscv/aplic.c +3
-0+if ( val ) +imsic_csr_set (isel, ireg); +else xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 251
Quoted 10 lines
xen/arch/riscv/aplic.c +10
-0+imsic_csr_clear (isel, ireg); + } +} + +void imsic_irq_enable (unsigned int hwirq) +{ +unsigned long flags; + +spin_lock_irqsave (&imsic_cfg.lock, flags); +imsic_local_eix_update (hwirq,1 , false, true);No subtraction of 1 here? Also, why "hwirq" and not just "irq"?
From the spec:
``` When an interrupt file supports distinct interrupt identities, valid identity numbers are between 1 and inclusive. The identity numbers within this range are said to be implemented by the interrupt file; numbers outside this range are not implemented. The number zero is never a valid interrupt identity. ... Bit positions in a valid eiek register that don’t correspond to a supported interrupt identity (such as bit 0 of eie0) are read-only zeros. ```
So in EIx registers interrupt i corresponds to bit i in comparison wiht APLIC's sourcecfg which starts from 0.
Quoted 20 lines
Quoted 20 lines
xen/arch/riscv/aplic.c +12
-0+spin_unlock_irqrestore (&imsic_cfg.lock, flags); +} + +void imsic_irq_disable (unsigned int hwirq) +{ +unsigned long flags; + +spin_lock_irqsave (&imsic_cfg.lock, flags); +imsic_local_eix_update (hwirq,1 , false, false); +spin_unlock_irqrestore (&imsic_cfg.lock, flags); +} +const struct imsic_config *imsic_get_config (void ) {return &imsic_cfg;@@ -277,6 +333,13 @@ int __init imsic_init(struct dt_device_node *node) goto imsic_init_err; }
Quoted 6 lines
xen/arch/riscv/aplic.c int __init imsic_init(struct dt_device_node *node) · line 339
Quoted 4 lines
xen/arch/riscv/aplic.c +4
-0+spin_lock_init (&imsic_cfg.lock); + +/* Enable local interrupt delivery */ +imsic_ids_local_delivery (true);What's this? I can't find the function/macro here, nor in patch 08, nor in staging.
It is defined in imsic.c:
```
void imsic_ids_local_delivery(bool enable)
{
if ( enable )
{
imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD);
imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY);
}
else
{
imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD);
imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY);
}
}
```Thanks for review.
~ Oleksii
Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations
Quoted 24 lines · Oleksii Kurochko
Quoted 23 lines · Jan Beulich
Quoted 22 lines · Oleksii Kurochko
Introduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
-aplic_irq_startup()
-aplic_irq_shutdown()
-aplic_irq_enable()
-aplic_irq_disable()
-aplic_irq_ack()
-aplic_host_irq_end()
-aplic_set_irq_affinity()As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().
For the purpose of
aplic_set_irq_affinity()aplic_get_cpu_from_mask()is introduced to get hart id.Also, introduce additional interrupt controller h/w operations and
host_irq_typefor APLIC:
-aplic_host_irq_type
-aplic_set_irq_priority()
-aplic_set_irq_type()
Quoted 6 lines
Quoted 3 lines
Yet these two functions nor the hooks they're used to populate are entirely unused here. Since they're also outside of the common IRQ handling machinery, it's unclear how one would sanely ack such a change.
They will be called by
intc_route_irq_to_xen()fromsetup_irq()during firt time the IRQ is setup.
Perhaps move their introduction to there then? We don't do any Misra checking yet lon RISC-V, but imo it's still good practice to avoid introducing new violations, even if only temporarily.
Quoted 20 lines
Quoted 14 lines
Quoted 14 lines
xen/arch/riscv/aplic.c +1
-0@@ -15,6 +15,7 @@#include <xen/irq.h> #include <xen/mm.h> #include <xen/sections.h> +#include <xen/spinlock.h> #include <xen/types.h> #include <xen/vmap.h> @@ -110,9 +111,173 @@ static int __init aplic_init(void)return 0 ; }Quoted 5 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 116
Quoted 4 lines
xen/arch/riscv/aplic.c +3
-1-static const struct intc_hw_operations __ro_after_init aplic_ops = { +static void aplic_set_irq_type (struct irq_desc *desc,unsigned int type) +{ +unsigned int irq = desc->irq -1 ;Why this adjustment by 1 (and yet both items being named "irq")?
Quoted 8 lines
Interrupt 0 isn't possible based on the spec:
```
Each of an APLIC’s interrupt sources has a fixed unique identity number in the range 1 to N, where N is the total number of sources at the APLIC. The number zero is not a valid interrupt identity number at an APLIC. The maximum number of interrupt sources an APLIC may support is 1023. ``` and interrupt 1 will correspond to bit 0 in sourcecfg[] register, interrupt 2 ->sourcecfg[1] and so on. And that is the reason why we need -1.
Okay, fine. But what about the part of the question in parentheses?
Quoted 13 lines
Quoted 11 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 134
Quoted 11 lines
xen/arch/riscv/aplic.c +11
-0+ aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_EDGE_FALL ; +break ; +case IRQ_TYPE_LEVEL_HIGH : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_LEVEL_HIGH ; +break ; +case IRQ_TYPE_LEVEL_LOW : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_LEVEL_LOW ; +break ; +default : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_INACTIVE ; +break ;Quoted 1 line
Is the default: label legitimate to be reached?
Quoted 10 lines
From the spec:
```
0 Inactive Inactive in this domain (and not delegated) 1 Detached Active, detached from the source wire 2–3 — Reserved 4 Edge1 Active, edge-sensitive; interrupt asserted on rising edge 5 Edge0 Active, edge-sensitive; interrupt asserted on falling edge 6 Level1 Active, level-sensitive; interrupt asserted when high 7 Level0 Active, level-sensitive; interrupt asserted when low ``` It seems to me likeAPLIC_SOURCECFG_SM_INACTIVEjust covers cases (0-3) and inactive IRQ pretty safe to as a default value.
I fear this doesn't answer my question, which is to a large part related to the Xen code, and only to some degree to the spec.
Quoted 9 lines
Quoted 8 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143
Quoted 5 lines
xen/arch/riscv/aplic.c +5
-0+static void aplic_set_irq_priority (struct irq_desc *desc, +unsigned int priority) +{ +/* No priority, do nothing */ +}Since the function dopes nothing, wouldn't it be better to omit it and have the (future) caller check for a NULL pointer ahead of making the (indirect) call? Same remark for other handlers (below) which also do nothing.
Quoted 4 lines
I thought about that too, but it could be some cases when the stub is introduced with temporary
BUG_ON("unimplemented")inside just to not miss to implement it when it will be necessary.
If we will have only the caller check then we could miss to implement such stubs.
I guess I don't understand the concern.
Quoted 13 lines
Quoted 4 lines
Quoted 4 lines
xen/arch/riscv/aplic.c +4
-0+static void aplic_irq_enable (struct irq_desc *desc) +{ +unsigned long flags; +Quoted 8 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154
Quoted 6 lines
xen/arch/riscv/aplic.c +6
-0+/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used);Such an extra field, used only for assertions, is pretty odd. Can't you use any of the other fields to achieve the same effect?
Quoted 8 lines
in
aplic_init()there is:/* check for associated imsic node */ rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle); if ( !rc ) panic("%s: IDC mode not supported\n", node->full_name);So we will have
panic()anyway if MSI mode isn't supported. As an option we can just drop the ASSERT.
Since they serve primarily as a reminder where changes would need making, I'd prefer if they could be kept.
Quoted 2 lines
Or introduce static variable in
aplic.caplic_mode, init it inaplic_init()and use it inASSERT().
This would then again be used solely for assertions, aiui? As said, I think it would be preferable if some already existing indicator could be used for this purpose.
Quoted 11 lines
Quoted 9 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+ASSERT (spin_is_locked (&desc->lock));If this lock (which is an IRQ-safe one) is necessarily held, ...
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 144
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+spin_lock_irqsave (&aplic.lock, flags);... you can use just
spin_lock()here.xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 160
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+clear_bit (_IRQ_DISABLED, &desc->status);Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)
I haven't thought about that. Likely non-atomic bitop could be used here.
And then - does it need to be a bitop? Aiui that's what Arm uses, while x86 doesn't. And I see no reason to use other than plain C operators here. If Arm was switched, presumably all the redundant (and misnamed) _IRQ_* constants could go away, with just the IRQ_* ones left.
Quoted 2 lines
Quoted 2 lines
I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.
What about this part?
Quoted 59 lines
Quoted 1 line
In any event this can be done outside of the APLIC-locked region, I think.
Considering that we are doing that under
desc->lock, agree we can move that outside the APLIC-locked region.Quoted 7 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 166
Quoted 4 lines
xen/arch/riscv/aplic.c +4
-0+imsic_irq_enable (desc->irq); + +/* enable interrupt in APLIC */ + aplic.regs->setienum = desc->irq;Are you sure you want to use plain assignments for MMIO accesses? I'd have expected
writel()to be used here. (And only later I realized that I didn't spot the same already higher up from here.)Good point. I have to update that with
writel()...Quoted 8 lines
From the vague understanding I've gained so far: Isn't the APLIC closer to
the CPU and the IMSIC closer to the device? If so, wouldn't you want to enable at the APLIC before enabling at the IMSIC? But of course that also depends on what exactly happens in the window while one is already enabled and the other is still disabled. (Later) From the code you add toimsic.cit looks like it's the other way around, as the IMSIC is accessed through CSRs.From the AIA spec:
``` An Incoming MSI Controller (IMSIC) is an optional RISC-V hardware component that is closely coupled with a hart, one IMSIC per hart. An IMSIC receives and records incoming message-signaled interrupts (MSIs) for a hart, and signals to the hart when there are pending and enabled interrupts to be serviced. ```Based on the figure 2 (Interrupt delivery by MSIs when harts have IMSICs for receiving them) of AIA spechttps://github.com/riscv/riscv-aia/blob/main/src/intrsWithIMSICs.png IMSIC is more close to CPU and APLIC is more close to the device. The external interrupt controller is APLIC and it only sends a MSI message for a CPU.
The logical flow of an interrupt to a hart with an IMSIC would be:
1. A physical interrupt signal arrives at the APLIC.
2. The APLIC, if configured for MSI delivery mode (domaincfg.DM = 1) and if the specific interrupt source is active and enabled within its domain (controlled by sourcecfg[i] and the global Interrupt Enable bit IE in domaincfg), will generate an MSI.
3. This MSI is then sent to the target hart's IMSIC. The APLIC needs to know the MSI target address for each hart, which can be hardwired or configured through registers like mmsiaddrcfg and mmsiaddrcfgh.
4. The receiving hart's IMSIC records this MSI as a pending interrupt.
5. If the corresponding interrupt identity is enabled within the IMSIC's interrupt file, the IMSIC will signal the hart, typically by setting the MEIP or SEIP bit in the mip CSR (or sip CSR).Generally, I think that the order in which enable interrupts doesn't really matter as if you were to enable the IMSIC to receive a certain interrupt before the APLIC was configured to send it (or had a pending interrupt from the device), the IMSIC would simply be waiting for an MSI that wouldn't arrive.
Similarly, if the APLIC sends an MSI for an interrupt that is not enabled in the IMSIC, the interrupt would remain pending in the IMSIC but wouldn't trigger an interrupt at the hart.
Quoted 3 lines
IMO, the order which is used now in the code is pretty logical.
Does it make sense?
Except for the "doesn't really matter" - yes. In a reply to a later patch I indicated I realized that IMSIC is what's closer to the CPU (and hence later in the chain of interrupt delivery actions).
Quoted 55 lines
Quoted 33 lines
Quoted 33 lines
xen/arch/riscv/aplic.c +33
-0+spin_unlock_irqrestore (&aplic.lock, flags); +} + +static void aplic_irq_disable (struct irq_desc *desc) +{ +unsigned long flags; + +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used); + +ASSERT (spin_is_locked (&desc->lock)); + +spin_lock_irqsave (&aplic.lock, flags); + +set_bit (_IRQ_DISABLED, &desc->status); + +/* disable interrupt in APLIC */ + aplic.regs->clrienum = desc->irq; + +/* disable interrupt in IMSIC */ +imsic_irq_disable (desc->irq); + +spin_unlock_irqrestore (&aplic.lock, flags); +} + +static unsigned int aplic_irq_startup (struct irq_desc *desc) +{ +aplic_irq_enable (desc); +Quoted 8 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 207
Quoted 7 lines
xen/arch/riscv/aplic.c +7
-0+return 0 ; +} + +static void aplic_irq_shutdown (struct irq_desc *desc) +{ +aplic_irq_disable (desc); +}You don't really need a separate hook function here, do you?
With such implementation it is really not needed to have a hook so I will drop it.
Quoted 7 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 214
Quoted 6 lines
xen/arch/riscv/aplic.c +6
-0+static void aplic_irq_ack (struct irq_desc *desc) +{ +/* nothing to do */ +} + +static void aplic_host_irq_end (struct irq_desc *desc)What's the "host" in the identifier about?
It was copied that from Arm and my understanding that it means Xen-related IRQ as they also have:
Quoted 14 lines
```
/* XXX different for level vs edge */
static hw_irq_controller gicv2_host_irq_type = {
...
.end = gicv2_host_irq_end,
...
};
static hw_irq_controller gicv2_guest_irq_type = {
...
.end = gicv2_guest_irq_end,
...
};
```And you expect to end up with a similar distinction on RISC-V? There's nothing like that on x86, just to mention it.
Quoted 15 lines
Quoted 6 lines
Quoted 6 lines
xen/arch/riscv/aplic.c +6
-0+static void aplic_set_irq_affinity (struct irq_desc *desc,const cpumask_t *mask) +{ +unsigned int cpu; +uint64_t group_index, base_ppn; +uint32_t hhxw, lhxw ,hhxs, value; +const struct imsic_config *imsic = aplic.imsic_cfg;Quoted 8 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154
Quoted 7 lines
xen/arch/riscv/aplic.c +7
-0+ +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used);Use the local variable you have made yourself?
Quoted 1 line
What do you mean by local here?
Just a few lines up you latch aplic.imsic_cfg into the local "imsic".
Quoted 14 lines
Quoted 14 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 246
Quoted 3 lines
xen/arch/riscv/aplic.c +3
-0+ASSERT (!cpumask_empty (mask)); + +spin_lock (&aplic.lock);Aiui the lock can be acquired quite a bit later. It ought to be needed only around the actual write to the hardware register.
Quoted 8 lines
xen/arch/riscv/aplic.c +8
-0+ cpu =cpuid_to_hartid (aplic_get_cpu_from_mask (mask)); + hhxw = imsic->group_index_bits; + lhxw = imsic->hart_index_bits; + hhxs = imsic->group_index_shift -IMSIC_MMIO_PAGE_SHIFT *2 ; + base_ppn = imsic->msi[cpu].base_addr >>IMSIC_MMIO_PAGE_SHIFT ; + +/* update hart and EEID in the target register */ + group_index = (base_ppn >> (hhxs +12 )) & (BIT (hhxw,UL ) -1 );
Quoted 3 lines
Quoted 1 line
What's this magic 12 in here? Not
IMSIC_MMIO_PAGE_SHIFTI suppose?In the AIA spec they are using 12 explicitly:https://github.com/riscv/riscv-aia/blob/main/src/AdvPLIC.adoc#AdvPLIC-MSIAddrs
In the spec that's fine, but please make yourself a constant with a suitable name then, to be used here. Just consider what would happen if we used literal 12 everywhere PAGE_SHIFT was meant.
Quoted 23 lines
Quoted 7 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 225
Quoted 6 lines
xen/arch/riscv/aplic.c +6
-0+void imsic_irq_enable (unsigned int hwirq) +{ +unsigned long flags; + +spin_lock_irqsave (&imsic_cfg.lock, flags); +imsic_local_eix_update (hwirq,1 , false, true);No subtraction of 1 here? Also, why "hwirq" and not just "irq"?
From the spec:
``` When an interrupt file supports distinct interrupt identities, valid identity numbers are between 1 and inclusive. The identity numbers within this range are said to be implemented by the interrupt file; numbers outside this range are not implemented. The number zero is never a valid interrupt identity. ... Bit positions in a valid eiek register that don’t correspond to a supported interrupt identity (such as bit 0 of eie0) are read-only zeros. ```
Quoted 1 line
So in EIx registers interrupt i corresponds to bit i in comparison wiht APLIC's sourcecfg which starts from 0.
Confusing, but what do you do.
Quoted 12 lines
Quoted 10 lines
xen/arch/riscv/aplic.c int __init imsic_init(struct dt_device_node *node) · line 339
Quoted 8 lines
xen/arch/riscv/aplic.c +4
-0@@ -277,6 +333,13 @@ int __init imsic_init(struct dt_device_node *node) goto imsic_init_err; } +spin_lock_init (&imsic_cfg.lock); + +/* Enable local interrupt delivery */ +imsic_ids_local_delivery (true);What's this? I can't find the function/macro here, nor in patch 08, nor in staging.
It is defined in
imsic.c:
Quoted 15 lines
```
void imsic_ids_local_delivery(bool enable)
{
if ( enable )
{
imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD);
imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY);
}
else
{
imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD);
imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY);
}
}
```No, it's not. As noted in the reply to a later patch, it's only introduced there. Hence the build will break between the two patches.
Jan
Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations
Quoted 25 lines · Jan Beulich
Quoted 24 lines · Oleksii Kurochko
Quoted 23 lines · Jan Beulich
Quoted 22 lines · Oleksii Kurochko
Introduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
-aplic_irq_startup()
-aplic_irq_shutdown()
-aplic_irq_enable()
-aplic_irq_disable()
-aplic_irq_ack()
-aplic_host_irq_end()
-aplic_set_irq_affinity()As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().
For the purpose of
aplic_set_irq_affinity()aplic_get_cpu_from_mask()is introduced to get hart id.Also, introduce additional interrupt controller h/w operations and
host_irq_typefor APLIC:
-aplic_host_irq_type
-aplic_set_irq_priority()
-aplic_set_irq_type()
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Yet these two functions nor the hooks they're used to populate are entirely unused here. Since they're also outside of the common IRQ handling machinery, it's unclear how one would sanely ack such a change.
They will be called by
intc_route_irq_to_xen()fromsetup_irq()during firt time the IRQ is setup.Perhaps move their introduction to there then? We don't do any Misra checking yet lon RISC-V, but imo it's still good practice to avoid introducing new violations, even if only temporarily.
Okay, I will move their introduction to there.
Btw, what is needed to add Misra checking for RISC-V? I started to think that, probably, it will make sense to do that from the start.
Quoted 27 lines
Quoted 19 lines
Quoted 14 lines
Quoted 14 lines
xen/arch/riscv/aplic.c +1
-0@@ -15,6 +15,7 @@#include <xen/irq.h> #include <xen/mm.h> #include <xen/sections.h> +#include <xen/spinlock.h> #include <xen/types.h> #include <xen/vmap.h> @@ -110,9 +111,173 @@ static int __init aplic_init(void)return 0 ; }Quoted 5 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 116
Quoted 4 lines
xen/arch/riscv/aplic.c +3
-1-static const struct intc_hw_operations __ro_after_init aplic_ops = { +static void aplic_set_irq_type (struct irq_desc *desc,unsigned int type) +{ +unsigned int irq = desc->irq -1 ;Why this adjustment by 1 (and yet both items being named "irq")?
Quoted 8 lines
Interrupt 0 isn't possible based on the spec:
```
Each of an APLIC’s interrupt sources has a fixed unique identity number in the range 1 to N, where N is the total number of sources at the APLIC. The number zero is not a valid interrupt identity number at an APLIC. The maximum number of interrupt sources an APLIC may support is 1023. ``` and interrupt 1 will correspond to bit 0 in sourcecfg[] register, interrupt 2 ->sourcecfg[1] and so on. And that is the reason why we need -1.
Quoted 1 line
Okay, fine. But what about the part of the question in parentheses?
Sorry, missed to write that I'll change irq to irq_bit or something like that.
Quoted 22 lines
Quoted 12 lines
Quoted 11 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 134
Quoted 11 lines
xen/arch/riscv/aplic.c +11
-0+ aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_EDGE_FALL ; +break ; +case IRQ_TYPE_LEVEL_HIGH : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_LEVEL_HIGH ; +break ; +case IRQ_TYPE_LEVEL_LOW : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_LEVEL_LOW ; +break ; +default : + aplic.regs->sourcecfg[irq] =APLIC_SOURCECFG_SM_INACTIVE ; +break ;Quoted 1 line
Is the default: label legitimate to be reached?
Quoted 10 lines
From the spec:
```
0 Inactive Inactive in this domain (and not delegated) 1 Detached Active, detached from the source wire 2–3 — Reserved 4 Edge1 Active, edge-sensitive; interrupt asserted on rising edge 5 Edge0 Active, edge-sensitive; interrupt asserted on falling edge 6 Level1 Active, level-sensitive; interrupt asserted when high 7 Level0 Active, level-sensitive; interrupt asserted when low ``` It seems to me likeAPLIC_SOURCECFG_SM_INACTIVEjust covers cases (0-3) and inactive IRQ pretty safe to as a default value.
Quoted 2 lines
I fear this doesn't answer my question, which is to a large part related to the Xen code, and only to some degree to the spec.
From Xen code point of view, I am not sure if it legitimate or not. I've not any issue, at the moment, with such implementation, but to be on a safe side I'll implement default case as panic("...").
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143
Quoted 5 lines
xen/arch/riscv/aplic.c +5
-0+static void aplic_set_irq_priority (struct irq_desc *desc, +unsigned int priority) +{ +/* No priority, do nothing */ +}Since the function dopes nothing, wouldn't it be better to omit it and have the (future) caller check for a NULL pointer ahead of making the (indirect) call? Same remark for other handlers (below) which also do nothing.
Quoted 5 lines
Quoted 4 lines
I thought about that too, but it could be some cases when the stub is introduced with temporary
BUG_ON("unimplemented")inside just to not miss to implement it when it will be necessary.
If we will have only the caller check then we could miss to implement such stubs.I guess I don't understand the concern.
for example, if we will have the following code:
void some_caller(struct irq_desc *desc)
{
if ( desc->handler->set_affinity )
desc->handler->set_affinity(desc, cpu_mask);}
Then we will skip the call of handler->set_affinity() (if it was just initialized with .set_affinity = NULL) without any notification. And it is fine specifically in this case as aplic_set_irq_priority() does nothing.
But what about the cases if it is a function which will have some implementation in the future but doesn't have implementation for now. Then without notification that this function is unimplemented we could skip something what really matters.
But I think that your initial comment was just about the function which basically does nothing. Am i right?
Quoted 20 lines
Quoted 12 lines
Quoted 4 lines
Quoted 4 lines
xen/arch/riscv/aplic.c +4
-0+static void aplic_irq_enable (struct irq_desc *desc) +{ +unsigned long flags; +Quoted 8 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154
Quoted 6 lines
xen/arch/riscv/aplic.c +6
-0+/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used);Such an extra field, used only for assertions, is pretty odd. Can't you use any of the other fields to achieve the same effect?
Quoted 8 lines
in
aplic_init()there is:/* check for associated imsic node */ rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle); if ( !rc ) panic("%s: IDC mode not supported\n", node->full_name);So we will have
panic()anyway if MSI mode isn't supported. As an option we can just drop the ASSERT.
Quoted 8 lines
Since they serve primarily as a reminder where changes would need making, I'd prefer if they could be kept.
Quoted 2 lines
Or introduce static variable in
aplic.caplic_mode, init it inaplic_init()and use it inASSERT().This would then again be used solely for assertions, aiui? As said, I think it would be preferable if some already existing indicator could be used for this purpose.
I think that not solely, for example, if IMSIC isn't available then we should skip the calls of imsic_irq_enable(), at least, and this variable could be used for that purpose.
But I will double check if we have better indicator. At the moment, I don't think we have better, probably, except checking of aplic.regs->domaincfg if bit APLIC_DOMAINCFG_DM is set.
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+ASSERT (spin_is_locked (&desc->lock));If this lock (which is an IRQ-safe one) is necessarily held, ...
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 144
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+spin_lock_irqsave (&aplic.lock, flags);... you can use just
spin_lock()here.
Quoted 8 lines
Quoted 4 lines
Quoted 3 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 160
Quoted 1 line
xen/arch/riscv/aplic.c +1
-0+clear_bit (_IRQ_DISABLED, &desc->status);Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)
I haven't thought about that. Likely non-atomic bitop could be used here.
And then - does it need to be a bitop? Aiui that's what Arm uses, while x86 doesn't. And I see no reason to use other than plain C operators here. If Arm was switched, presumably all the redundant (and misnamed) _IRQ_* constants could go away, with just the IRQ_* ones left.
The reason for a bitop in Arm is explained in this commithttps://gitlab.com/xen-project/xen/-/commit/50d8fe8fcbab2440cfeeb65c4765868398652473
but all the places where plain C operators were changed to bitops are actually executed under|spin_lock_irqsave(&desc->lock, flags). By quick look I found only two places one in __setup_irq() but it is called by the functions which do ||spin_lock_irqsave(&desc->lock, flags) and in vgic_v2_fold_lr_state(). Maybe, I'm missing something.| |RISC-V won't have something similar to ||vgic_v2_fold_lr_state|(), but __setup_irq() is used in a similar way. It can be added ASSERT(spin_is_lock(&desc->lock))
and then it will also safe to use non-bitop function.
Probably, it is a little bit safer to use always bitops for desc->status.
||
Quoted 3 lines
Quoted 2 lines
Quoted 2 lines
I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.
What about this part?
As I understand, based on Arm, code then Xen enables interrupts corresponding to devices assigned to dom0/domU before booting dom0/domU, resulting in the possibility of receiving an interrupt and not knowing what to do with it. So it is needed for enablement of IRQs when the guest requests it and not unconditionally at boot time.
Quoted 61 lines
Quoted 61 lines
Quoted 33 lines
Quoted 33 lines
xen/arch/riscv/aplic.c +33
-0+spin_unlock_irqrestore (&aplic.lock, flags); +} + +static void aplic_irq_disable (struct irq_desc *desc) +{ +unsigned long flags; + +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used); + +ASSERT (spin_is_locked (&desc->lock)); + +spin_lock_irqsave (&aplic.lock, flags); + +set_bit (_IRQ_DISABLED, &desc->status); + +/* disable interrupt in APLIC */ + aplic.regs->clrienum = desc->irq; + +/* disable interrupt in IMSIC */ +imsic_irq_disable (desc->irq); + +spin_unlock_irqrestore (&aplic.lock, flags); +} + +static unsigned int aplic_irq_startup (struct irq_desc *desc) +{ +aplic_irq_enable (desc); +Quoted 8 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 207
Quoted 7 lines
xen/arch/riscv/aplic.c +7
-0+return 0 ; +} + +static void aplic_irq_shutdown (struct irq_desc *desc) +{ +aplic_irq_disable (desc); +}You don't really need a separate hook function here, do you?
With such implementation it is really not needed to have a hook so I will drop it.
Quoted 7 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 214
Quoted 6 lines
xen/arch/riscv/aplic.c +6
-0+static void aplic_irq_ack (struct irq_desc *desc) +{ +/* nothing to do */ +} + +static void aplic_host_irq_end (struct irq_desc *desc)What's the "host" in the identifier about?
It was copied that from Arm and my understanding that it means Xen-related IRQ as they also have:
```
/* XXX different for level vs edge */
statichw_irq_controllergicv2_host_irq_type= {
...
.end =gicv2_host_irq_end,
...
};
Quoted 8 lines
Quoted 6 lines
static
hw_irq_controllergicv2_guest_irq_type= {
...
.end =gicv2_guest_irq_end,
...
};
```And you expect to end up with a similar distinction on RISC-V? There's nothing like that on x86, just to mention it.
Yes, if one day support for guest interrupts without IMSIC support will be added for APLIC. (at the moment, we are planning only to have APLIC+IMSIC support as this way is hypervisor-aware)
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 242
Quoted 13 lines
xen/arch/riscv/aplic.c +13
-0+static void aplic_set_irq_affinity (struct irq_desc *desc,const cpumask_t *mask) +{ +unsigned int cpu; +uint64_t group_index, base_ppn; +uint32_t hhxw, lhxw ,hhxs, value; +const struct imsic_config *imsic = aplic.imsic_cfg; + +/* +* TODO: Currently, APLIC is supported only with MSI interrupts. +* If APLIC without MSI interrupts is required in the future, +* this function will need to be updated accordingly. +*/ +ASSERT (aplic.imsic_cfg->is_used);Use the local variable you have made yourself?
Quoted 2 lines
Quoted 1 line
What do you mean by local here?
Just a few lines up you latch aplic.imsic_cfg into the local "imsic".
Oh, sure, if *->is_used will still present in the next patch series then I'll re-use here "imsic".
~ Oleksii
Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations
Quoted 31 lines · Oleksii Kurochko
Quoted 30 lines · Jan Beulich
Quoted 29 lines · Oleksii Kurochko
Quoted 18 lines · Jan Beulich
Quoted 17 lines · Oleksii Kurochko
Introduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
-aplic_irq_startup()
-aplic_irq_shutdown()
-aplic_irq_enable()
-aplic_irq_disable()
-aplic_irq_ack()
-aplic_host_irq_end()
-aplic_set_irq_affinity()As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().
For the purpose of
aplic_set_irq_affinity()aplic_get_cpu_from_mask()is introduced to get hart id.Quoted 8 lines
Quoted 5 lines
Also, introduce additional interrupt controller h/w operations and
host_irq_typefor APLIC:
-aplic_host_irq_type
-aplic_set_irq_priority()
-aplic_set_irq_type()Yet these two functions nor the hooks they're used to populate are entirely unused here. Since they're also outside of the common IRQ handling machinery, it's unclear how one would sanely ack such a change.
They will be called by
intc_route_irq_to_xen()fromsetup_irq()during firt time the IRQ is setup.
Quoted 8 lines
Quoted 3 lines
Perhaps move their introduction to there then? We don't do any Misra checking yet lon RISC-V, but imo it's still good practice to avoid introducing new violations, even if only temporarily.
Okay, I will move their introduction to there.
Btw, what is needed to add Misra checking for RISC-V? I started to think that, probably, it will make sense to do that from the start.
Best I can do is point you at what is done for Arm and x86. You may want to ask people more familiar with the CI aspects involved here.
Quoted 25 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143
Quoted 5 lines
xen/arch/riscv/aplic.c +5
-0+static void aplic_set_irq_priority (struct irq_desc *desc, +unsigned int priority) +{ +/* No priority, do nothing */ +}Since the function dopes nothing, wouldn't it be better to omit it and have the (future) caller check for a NULL pointer ahead of making the (indirect) call? Same remark for other handlers (below) which also do nothing.
Quoted 5 lines
Quoted 4 lines
I thought about that too, but it could be some cases when the stub is introduced with temporary
BUG_ON("unimplemented")inside just to not miss to implement it when it will be necessary.
If we will have only the caller check then we could miss to implement such stubs.I guess I don't understand the concern.
for example, if we will have the following code:
void some_caller(struct irq_desc *desc) { if ( desc->handler->set_affinity ) desc->handler->set_affinity(desc, cpu_mask); }Then we will skip the call of
handler->set_affinity()(if it was just initialized with .set_affinity = NULL) without any notification. And it is fine specifically in this case asaplic_set_irq_priority()does nothing.
Quoted 6 lines
But what about the cases if it is a function which will have some implementation in the future but doesn't have implementation for now. Then without notification that this function is unimplemented we could skip something what really matters.
But I think that your initial comment was just about the function which basically does nothing. Am i right?
Since indirect calls are not only more expensive (often; not sure about RISC-V) but also pose speculative concerns, having such just to do nothing simply seems like moving in the wrong direction.
Quoted 15 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 1 line
+ ASSERT(spin_is_locked(&desc->lock));
If this lock (which is an IRQ-safe one) is necessarily held, ...
Quoted 1 line
+
spin_lock_irqsave(&aplic.lock, flags);... you can use just
spin_lock()here.Quoted 8 lines
Quoted 4 lines
Quoted 3 lines
Quoted 1 line
+
clear_bit(_IRQ_DISABLED, &desc->status);Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)
I haven't thought about that. Likely non-atomic bitop could be used here.
And then - does it need to be a bitop? Aiui that's what Arm uses, while x86 doesn't. And I see no reason to use other than plain C operators here. If Arm was switched, presumably all the redundant (and misnamed) _IRQ_* constants could go away, with just the IRQ_* ones left.
Quoted 8 lines
The reason for a bitop in Arm is explained in this commithttps://gitlab.com/xen-project/xen/-/commit/50d8fe8fcbab2440cfeeb65c4765868398652473 but all the places where plain C operators were changed to bitops are actually executed under|spin_lock_irqsave(&desc->lock, flags). By quick look I found only two places one in
__setup_irq()but it is called by the functions which do ||spin_lock_irqsave(&desc->lock, flags) and invgic_v2_fold_lr_state(). Maybe, I'm missing something.|
|RISC-V won't have something similar to ||vgic_v2_fold_lr_state|(), but__setup_irq()is used in a similar way. It can be added ASSERT(spin_is_lock(&desc->lock)) and then it will also safe to use non-bitop function.
Probably, it is a little bit safer to use always bitops fordesc->status.
||
I question that. If any accesses outside of locked regions were needed (as the description of that commit suggests), then the situation would be different.
Btw, you not wrapping lines and you adding strange | instances doesn't help readability of your replies.
Quoted 8 lines
Quoted 3 lines
Quoted 2 lines
Quoted 2 lines
I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.
What about this part?
As I understand, based on Arm, code then Xen enables interrupts corresponding to devices assigned to dom0/domU before booting dom0/domU, resulting in the possibility of receiving an interrupt and not knowing what to do with it. So it is needed for enablement of IRQs when the guest requests it and not unconditionally at boot time.
I fear I don't understand this. The way we do things on x86 doesn't leave us in such a situation.
Jan
Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations
Quoted 22 lines · Jan Beulich
Quoted 14 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 1 line
+ ASSERT(spin_is_locked(&desc->lock));
If this lock (which is an IRQ-safe one) is necessarily held, ...
Quoted 1 line
+
spin_lock_irqsave(&aplic.lock, flags);... you can use just
spin_lock()here.Quoted 8 lines
Quoted 4 lines
Quoted 3 lines
Quoted 1 line
+
clear_bit(_IRQ_DISABLED, &desc->status);Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)
I haven't thought about that. Likely non-atomic bitop could be used here.
And then - does it need to be a bitop? Aiui that's what Arm uses, while x86 doesn't. And I see no reason to use other than plain C operators here. If Arm was switched, presumably all the redundant (and misnamed) _IRQ_* constants could go away, with just the IRQ_* ones left.
Quoted 8 lines
The reason for a bitop in Arm is explained in this commithttps://gitlab.com/xen-project/xen/-/commit/50d8fe8fcbab2440cfeeb65c4765868398652473 but all the places where plain C operators were changed to bitops are actually executed under|spin_lock_irqsave(&desc->lock, flags). By quick look I found only two places one in
__setup_irq()but it is called by the functions which do ||spin_lock_irqsave(&desc->lock, flags) and invgic_v2_fold_lr_state(). Maybe, I'm missing something.|
|RISC-V won't have something similar to ||vgic_v2_fold_lr_state|(), but__setup_irq()is used in a similar way. It can be added ASSERT(spin_is_lock(&desc->lock)) and then it will also safe to use non-bitop function.
Probably, it is a little bit safer to use always bitops fordesc->status.
||
Quoted 2 lines
I question that. If any accesses outside of locked regions were needed (as the description of that commit suggests), then the situation would be different.
Okay, then at the moment there is no such cases and I'll use plain C operator instead of clear/set_bit().
Quoted 6 lines
Btw, you not wrapping lines and you adding strange | instances doesn't help readability of your replies.
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
What about this part?
As I understand, based on Arm, code then Xen enables interrupts corresponding to devices assigned to dom0/domU before booting dom0/domU, resulting in the possibility of receiving an interrupt and not knowing what to do with it. So it is needed for enablement of IRQs when the guest requests it and not unconditionally at boot time.
I fear I don't understand this. The way we do things on x86 doesn't leave us in such a situation.
On Arm, the physical interrupts would be enabled when the interrupt is initially routed and in case guest is booting with interrupt disabled, it could introduce a problem when guest enabled interrupts it will already have a pending interrupt for which it isn't ready.
How is it handled the case when a device isn't quiescing at the boot time in x86?
But I just realized the way how interrupts are enabled in RISC-V for guest won't lead to such case. The interrupt will be enabled only when guest's device driver will request that. So this setting/clearing of IRQ_DISABLED could be dropped for RISC-V.
~ Oleksii
Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations
In reply to Oleksii KurochkoQuoted 14 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c +2
-0@@ -14,12 +14,68 @@#include <xen/errno.h> #include <xen/init.h> #include <xen/macros.h> +#include <xen/spinlock.h> #include <xen/xmalloc.h> #include <asm/imsic.h> static struct imsic_config imsic_cfg; +#define imsic_csr_set (c, v) \
xen/arch/riscv/imsic.c line 34
Quoted 10 lines
xen/arch/riscv/imsic.c +10
-0+do { \ +csr_write (CSR_SISELECT , c); \ +csr_set (CSR_SIREG , v); \ +} while (0) + +#define imsic_csr_clear (c, v) \ +do { \ + csr_write(CSR_SISELECT, c); \ + csr_clear(CSR_SIREG, v); \ +} while (0)
Coming back to these (the later patch adds one more here): How expensive are these CSR writes? IOW would it perhaps make sense to maintain a local cache of the last written SISELECT value, to avoid writing the same one again if the same windowed register needs accessing twice in a row?
Jan
Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations
Quoted 14 lines · Jan Beulich
Quoted 13 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c +1
-0@@ -14,12 +14,68 @@#include <xen/errno.h> #include <xen/init.h> #include <xen/macros.h> +#include <xen/spinlock.h> #include <xen/xmalloc.h> #include <asm/imsic.h> static struct imsic_config imsic_cfg;
Quoted 15 lines
xen/arch/riscv/imsic.c line 34
Quoted 11 lines
xen/arch/riscv/imsic.c +11
-0+#define imsic_csr_set (c, v) \ +do { \ + csr_write(CSR_SISELECT, c); \ + csr_set(CSR_SIREG, v); \ +} while (0) + +#define imsic_csr_clear (c, v) \ +do { \ + csr_write(CSR_SISELECT, c); \ + csr_clear(CSR_SIREG, v); \ +} while (0)Coming back to these (the later patch adds one more here): How expensive are these CSR writes? IOW would it perhaps make sense to maintain a local cache of the last written SISELECT value, to avoid writing the same one again if the same windowed register needs accessing twice in a row?
CSRs belong to the HART, so access to them is very fast.
~ Oleksii
Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations
Quoted 32 lines · Oleksii Kurochko
Quoted 14 lines · Jan Beulich
Quoted 13 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c +1
-0@@ -14,12 +14,68 @@#include <xen/errno.h> #include <xen/init.h> #include <xen/macros.h> +#include <xen/spinlock.h> #include <xen/xmalloc.h> #include <asm/imsic.h> static struct imsic_config imsic_cfg;Quoted 15 lines
xen/arch/riscv/imsic.c line 34
Quoted 11 lines
xen/arch/riscv/imsic.c +11
-0+#define imsic_csr_set (c, v) \ +do { \ + csr_write(CSR_SISELECT, c); \ + csr_set(CSR_SIREG, v); \ +} while (0) + +#define imsic_csr_clear (c, v) \ +do { \ + csr_write(CSR_SISELECT, c); \ + csr_clear(CSR_SIREG, v); \ +} while (0)Coming back to these (the later patch adds one more here): How expensive are these CSR writes? IOW would it perhaps make sense to maintain a local cache of the last written SISELECT value, to avoid writing the same one again if the same windowed register needs accessing twice in a row?
Quoted 1 line
CSRs belong to the HART, so access to them is very fast.
Can you back this by any data? I view CSRs as somewhat similar to x86'es MSRs, and access (writes in particular) to some of them is rather slow.
Jan
Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations
Quoted 31 lines · Jan Beulich
Quoted 30 lines · Oleksii Kurochko
Quoted 29 lines · Jan Beulich
Quoted 14 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c +2
-0@@ -14,12 +14,68 @@#include <xen/errno.h> #include <xen/init.h> #include <xen/macros.h> +#include <xen/spinlock.h> #include <xen/xmalloc.h> #include <asm/imsic.h> static struct imsic_config imsic_cfg; +#define imsic_csr_set (c, v) \xen/arch/riscv/imsic.c line 34
Quoted 10 lines
xen/arch/riscv/imsic.c +10
-0+do { \ +csr_write (CSR_SISELECT , c); \ +csr_set (CSR_SIREG , v); \ +} while (0) + +#define imsic_csr_clear (c, v) \ +do { \ + csr_write(CSR_SISELECT, c); \ + csr_clear(CSR_SIREG, v); \ +} while (0)Coming back to these (the later patch adds one more here): How expensive are these CSR writes? IOW would it perhaps make sense to maintain a local cache of the last written SISELECT value, to avoid writing the same one again if the same windowed register needs accessing twice in a row?
Quoted 3 lines
Quoted 1 line
CSRs belong to the HART, so access to them is very fast.
Can you back this by any data? I view CSRs as somewhat similar to x86'es MSRs, and access (writes in particular) to some of them is rather slow.
CSR read 1 cycle, CSR write 7 cycles on Microchip platform. ~ Oleksii
Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations
Quoted 33 lines · Oleksii Kurochko
Quoted 31 lines · Jan Beulich
Quoted 30 lines · Oleksii Kurochko
Quoted 29 lines · Jan Beulich
Quoted 14 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c +2
-0@@ -14,12 +14,68 @@#include <xen/errno.h> #include <xen/init.h> #include <xen/macros.h> +#include <xen/spinlock.h> #include <xen/xmalloc.h> #include <asm/imsic.h> static struct imsic_config imsic_cfg; +#define imsic_csr_set (c, v) \xen/arch/riscv/imsic.c line 34
Quoted 10 lines
xen/arch/riscv/imsic.c +10
-0+do { \ +csr_write (CSR_SISELECT , c); \ +csr_set (CSR_SIREG , v); \ +} while (0) + +#define imsic_csr_clear (c, v) \ +do { \ + csr_write(CSR_SISELECT, c); \ + csr_clear(CSR_SIREG, v); \ +} while (0)Coming back to these (the later patch adds one more here): How expensive are these CSR writes? IOW would it perhaps make sense to maintain a local cache of the last written SISELECT value, to avoid writing the same one again if the same windowed register needs accessing twice in a row?
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
CSRs belong to the HART, so access to them is very fast.
Can you back this by any data? I view CSRs as somewhat similar to x86'es MSRs, and access (writes in particular) to some of them is rather slow.
CSR read 1 cycle, CSR write 7 cycles on Microchip platform. ~ Oleksii
And that's an in-order platform, i.e. cycle count being all that matters for performance? No other (e.g. latency) effect on subsequent insns?
Further, how does this compare to the outlined alternative, especially if we assumed that the respective cacheline would be hot and hence usually in L1 cache?
Jan
PATCH 11/14 xen/riscv: add external interrupt handling for hypervisor mode
In reply to Oleksii KurochkoImplement functions necessarry to have working external interrupts in hypervisor mode. The following changes are done:
- Add a common function intc_handle_external_irq() to call APLIC specific function to handle an interrupt.
- Update do_trap() function to handle IRQ_S_EXT case; add the check to catch case when cause of trap is an interrupt.
- Add handle_interrrupt() member to intc_hw_operations structure.
- Enable local interrupt delivery for IMSIC by implementation and calling of imsic_ids_local_delivery() in imsic_init(); additionally introduce helper imsic_csr_write() to update IMSIC_EITHRESHOLD and IMSIC_EITHRESHOLD.
- Enable hypervisor external interrupts.
- Implement aplic_handler_interrupt() and use it to init ->handle_interrupt member of intc_hw_operations for APLIC.
- Add implementation of do_IRQ() to dispatch the interrupt.
The current patch is based on the code from [1].
[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7390e2365828b83e27ead56b03114a56e3699dd5
--- xen/arch/riscv/aplic.c | 19 +++++++++++++ xen/arch/riscv/imsic.c | 25 +++++++++++++++++ xen/arch/riscv/include/asm/imsic.h | 7 +++++ xen/arch/riscv/include/asm/intc.h | 5 ++++ xen/arch/riscv/include/asm/irq.h | 3 +++ xen/arch/riscv/intc.c | 7 +++++ xen/arch/riscv/irq.c | 43 ++++++++++++++++++++++++++++++ xen/arch/riscv/traps.c | 18 +++++++++++++ 8 files changed, 127 insertions(+)
xen/arch/riscv/aplic.c +19 -0
@@ -261,6 +261,21 @@ static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)spin_unlock (&aplic.lock); } +static void aplic_handle_interrupt (unsigned long cause,struct cpu_user_regs *regs) +{ +/* disable to avoid more external interrupts */ +csr_clear (CSR_SIE ,1UL <<IRQ_S_EXT ); + +/* clear the pending bit */ +csr_clear (CSR_SIP ,1UL <<IRQ_S_EXT ); + +/* dispatch the interrupt */ +do_IRQ (regs,csr_swap (CSR_STOPEI ,0 ) >>TOPI_IID_SHIFT ); + +/* enable external interrupts */ +csr_set (CSR_SIE ,1UL <<IRQ_S_EXT ); +} +static hw_irq_controller aplic_host_irq_type = { .typename ="aplic" , .startup = aplic_irq_startup,
@@ -278,6 +293,7 @@ static const struct intc_hw_operations aplic_ops = { .host_irq_type = &aplic_host_irq_type, .set_irq_priority = aplic_set_irq_priority, .set_irq_type = aplic_set_irq_type, + .handle_interrupt = aplic_handle_interrupt, }; static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
@@ -318,6 +334,9 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)register_intc_ops (&aplic_ops); +/* Enable supervisor external interrupt */ +csr_set (CSR_SIE ,1UL <<IRQ_S_EXT ); +return 0 ; }
xen/arch/riscv/imsic.c +25 -0
@@ -19,8 +19,19 @@#include <asm/imsic.h> +#define IMSIC_DISABLE_EIDELIVERY 0 +#define IMSIC_ENABLE_EIDELIVERY 1 +#define IMSIC_DISABLE_EITHRESHOLD 1 +#define IMSIC_ENABLE_EITHRESHOLD 0 +static struct imsic_config imsic_cfg; +#define imsic_csr_write (c, v) \ +do { \ + csr_write(CSR_SISELECT, c); \ + csr_write(CSR_SIREG, v); \ +} while (0) +#define imsic_csr_set (c, v) \ do { \ csr_write(CSR_SISELECT, c); \
@@ -33,6 +44,20 @@ do { \csr_clear (CSR_SIREG , v); \ }while (0 ) +void imsic_ids_local_delivery(bool enable) +{ +if ( enable ) + { +imsic_csr_write (IMSIC_EITHRESHOLD ,IMSIC_ENABLE_EITHRESHOLD ); +imsic_csr_write (IMSIC_EIDELIVERY ,IMSIC_ENABLE_EIDELIVERY ); + } +else + { +imsic_csr_write (IMSIC_EITHRESHOLD ,IMSIC_DISABLE_EITHRESHOLD ); +imsic_csr_write (IMSIC_EIDELIVERY ,IMSIC_DISABLE_EIDELIVERY ); + } +} +static void imsic_local_eix_update (unsigned long base_id,unsigned long num_id,bool pend,bool val) {
xen/arch/riscv/include/asm/imsic.h +7 -0
@@ -12,6 +12,7 @@#define ASM__RISCV__IMSIC_H #include <xen/spinlock.h> +#include <xen/stdbool.h> #include <xen/types.h> #define IMSIC_MMIO_PAGE_SHIFT 12
@@ -20,6 +21,10 @@#define IMSIC_MIN_ID 63#define IMSIC_MAX_ID 2048 +#define IMSIC_EIDELIVERY 0x70 + +#define IMSIC_EITHRESHOLD 0x72 +#define IMSIC_EIP0 0x80#define IMSIC_EIPx_BITS 32
@@ -78,4 +83,6 @@ const struct imsic_config *imsic_get_config(void);void imsic_irq_enable (unsigned int hwirq);void imsic_irq_disable (unsigned int hwirq); +void imsic_ids_local_delivery (bool enable); +#endif /* ASM__RISCV__IMSIC_H */
xen/arch/riscv/include/asm/intc.h +5 -0
@@ -34,6 +34,8 @@ struct intc_hw_operations {/* Set IRQ priority */ void (*set_irq_priority)(struct irq_desc *desc,unsigned int priority); +/* handle external interrupt */ +void (*handle_interrupt)(unsigned long cause,struct cpu_user_regs *regs); };void intc_preinit (void );
@@ -45,4 +47,7 @@ void register_intc_ops(const struct intc_hw_operations *ops);struct irq_desc ;void intc_route_irq_to_xen (struct irq_desc *desc,unsigned int priority); +struct cpu_user_regs ; +void intc_handle_external_irqs (unsigned long cause,struct cpu_user_regs *regs); +#endif /* ASM__RISCV__INTERRUPT_CONTOLLER_H */
xen/arch/riscv/include/asm/irq.h +3 -0
@@ -51,6 +51,9 @@ int platform_get_irq(const struct dt_device_node *device, int index);void init_IRQ (void ); +struct cpu_user_regs ; +void do_IRQ (struct cpu_user_regs *regs,unsigned int irq); +#endif /* ASM__RISCV__IRQ_H */ /*
xen/arch/riscv/intc.c +7 -0
@@ -51,6 +51,13 @@ static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority) intc_hw_ops->set_irq_priority (desc, priority); } +void intc_handle_external_irqs (unsigned long cause,struct cpu_user_regs *regs) +{ +ASSERT (intc_hw_ops && intc_hw_ops->handle_interrupt); + + intc_hw_ops->handle_interrupt (cause, regs); +} +void intc_route_irq_to_xen (struct irq_desc *desc,unsigned int priority) {ASSERT (test_bit (_IRQ_DISABLED, &desc->status));
xen/arch/riscv/irq.c +43 -0
52 lines not on this page · raw message
xen/arch/riscv/traps.c +18 -0
@@ -11,6 +11,7 @@#include <xen/nospec.h> #include <xen/sched.h> +#include <asm/intc.h> #include <asm/processor.h> #include <asm/riscv_encoding.h> #include <asm/traps.h>
@@ -128,6 +129,23 @@ void do_trap(struct cpu_user_regs *cpu_regs) } fallthrough;default : +if ( cause &CAUSE_IRQ_FLAG ) + { +/* Handle interrupt */ +unsigned long icause = cause & ~CAUSE_IRQ_FLAG ; + +switch ( icause ) + { +case IRQ_S_EXT : +intc_handle_external_irqs (cause, cpu_regs); +break ; +default : +break ; + } + +break ; + } +do_unexpected_trap (cpu_regs);break ; }
Signature · 1 line
2.49.0
Re: [PATCH v1 11/14] xen/riscv: add external interrupt handling for hypervisor mode
Quoted 9 lines · Oleksii Kurochko
Implement functions necessarry to have working external interrupts in hypervisor mode. The following changes are done:
- Add a common functionintc_handle_external_irq()to call APLIC specific function to handle an interrupt.
- Updatedo_trap()function to handleIRQ_S_EXTcase; add the check to catch case when cause of trap is an interrupt.
- Addhandle_interrrupt()member tointc_hw_operationsstructure.
- Enable local interrupt delivery for IMSIC by implementation and calling ofimsic_ids_local_delivery()inimsic_init();
Ah, here is where that call really belongs (see my question on the earlier patch). Please make sure you series builds okay at every patch boundary.
xen/arch/riscv/aplic.c static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask) · line 267
Quoted 10 lines
xen/arch/riscv/aplic.c +4
-0@@ -261,6 +261,21 @@ static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)spin_unlock (&aplic.lock); } +static void aplic_handle_interrupt (unsigned long cause,struct cpu_user_regs *regs) +{ +/* disable to avoid more external interrupts */ +csr_clear (CSR_SIE ,1UL <<IRQ_S_EXT );
Didn't I see you use BIT() elsewhere? Would be nice to be overall consistent at least within related code.
xen/arch/riscv/aplic.c static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask) · line 277
Quoted 9 lines
xen/arch/riscv/aplic.c +9
-0+/* clear the pending bit */ +csr_clear (CSR_SIP ,1UL <<IRQ_S_EXT ); + +/* dispatch the interrupt */ +do_IRQ (regs,csr_swap (CSR_STOPEI ,0 ) >>TOPI_IID_SHIFT ); + +/* enable external interrupts */ +csr_set (CSR_SIE ,1UL <<IRQ_S_EXT ); +}
Why does "cause" need passing into here? I realize the function is used ...
xen/arch/riscv/aplic.c static const struct intc_hw_operations aplic_ops = { · line 297
Quoted 6 lines
xen/arch/riscv/aplic.c +1
-0@@ -278,6 +293,7 @@ static const struct intc_hw_operations aplic_ops = { .host_irq_type = &aplic_host_irq_type, .set_irq_priority = aplic_set_irq_priority, .set_irq_type = aplic_set_irq_type, + .handle_interrupt = aplic_handle_interrupt, };
... as a hook, yet it's still unclear whether (why) any other such hook would need the cause to be passed in.
xen/arch/riscv/aplic.c do { \ · line 47
Quoted 5 lines
xen/arch/riscv/aplic.c +1
-0@@ -33,6 +44,20 @@ do { \csr_clear (CSR_SIREG , v); \ }while (0 ) +void imsic_ids_local_delivery(bool enable)
__init as long as the sole caller is such?
Quoted 12 lines
xen/arch/riscv/include/asm/intc.h +2
-0@@ -34,6 +34,8 @@ struct intc_hw_operations {/* Set IRQ priority */ void (*set_irq_priority)(struct irq_desc *desc,unsigned int priority); +/* handle external interrupt */ +void (*handle_interrupt)(unsigned long cause,struct cpu_user_regs *regs); };void intc_preinit (void );
xen/arch/riscv/include/asm/intc.h void register_intc_ops(const struct intc_hw_operations *ops); · line 50
Quoted 4 lines
xen/arch/riscv/include/asm/intc.h +1
-0@@ -45,4 +47,7 @@ void register_intc_ops(const struct intc_hw_operations *ops);struct irq_desc ;void intc_route_irq_to_xen (struct irq_desc *desc,unsigned int priority); +struct cpu_user_regs ;
This is too late - you've used it above already. It either can be dropped, or needs to move up.
xen/arch/riscv/intc.c static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority) · line 56
Quoted 9 lines
xen/arch/riscv/intc.c +3
-0@@ -51,6 +51,13 @@ static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority) intc_hw_ops->set_irq_priority (desc, priority); } +void intc_handle_external_irqs (unsigned long cause,struct cpu_user_regs *regs) +{ +ASSERT (intc_hw_ops && intc_hw_ops->handle_interrupt);
I don't view such checks (on every interrupt) as very useful. If you checked once early on - okay. But here you gain nothing at all ...
xen/arch/riscv/intc.c static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority) · line 58
Quoted 1 line
xen/arch/riscv/intc.c +1
-0+ intc_hw_ops->handle_interrupt (cause, regs);
... towards the use here, when considering a release build.
Quoted 10 lines
xen/arch/riscv/intc.c +6
-0@@ -83,3 +87,42 @@ void __init init_IRQ(void)if (init_irq_data () <0 )panic ("initialization of IRQ data failed\n" ); } + +/* Dispatch an interrupt */ +void do_IRQ (struct cpu_user_regs *regs,unsigned int irq) +{ +struct irq_desc *desc =irq_to_desc (irq); +struct irqaction *action;
xen/arch/riscv/intc.c void __init init_IRQ(void) · line 105
Quoted 10 lines
xen/arch/riscv/intc.c +10
-0+ +irq_enter (); + +spin_lock (&desc->lock); + desc->handler->ack (desc); + +if (test_bit (_IRQ_DISABLED, &desc->status) ) + goto out; + +set_bit (_IRQ_INPROGRESS, &desc->status);
Same comment as on the earlier patch - atomic bitop when in a suitably locked region?
xen/arch/riscv/intc.c void __init init_IRQ(void) · line 110
Quoted 5 lines
xen/arch/riscv/intc.c +5
-0+ action = desc->action; + +spin_unlock_irq (&desc->lock); + +#ifndef CONFIG_IRQ_HAS_MULTIPLE_ACTION
Stolen from Arm? What's this about?
xen/arch/riscv/intc.c void __init init_IRQ(void) · line 121
Quoted 11 lines
xen/arch/riscv/intc.c +11
-0+ action->handler (irq, action->dev_id); +#else +do { + action->handler (irq, action->dev_id); + action = action->next; + }while ( action ); +#endif /* CONFIG_IRQ_HAS_MULTIPLE_ACTION */ + +spin_lock_irq (&desc->lock); + +clear_bit (_IRQ_INPROGRESS, &desc->status);
See above.
xen/arch/riscv/intc.c void __init init_IRQ(void) · line 122
Quoted 1 line
xen/arch/riscv/intc.c +1
-0+out:
Nit (you know what).
Quoted 5 lines
xen/arch/riscv/intc.c +1
-0@@ -128,6 +129,23 @@ void do_trap(struct cpu_user_regs *cpu_regs) } fallthrough;default : +if ( cause &CAUSE_IRQ_FLAG )
xen/arch/riscv/intc.c void do_trap(struct cpu_user_regs *cpu_regs) · line 142
Quoted 10 lines
xen/arch/riscv/intc.c +10
-0+ { +/* Handle interrupt */ +unsigned long icause = cause & ~CAUSE_IRQ_FLAG ; + +switch ( icause ) + { +case IRQ_S_EXT : +intc_handle_external_irqs (cause, cpu_regs); +break ; +default :
Nit: Blank line please between non-fall-through case blocks.
Jan
Re: [PATCH v1 11/14] xen/riscv: add external interrupt handling for hypervisor mode
Quoted 12 lines · Jan Beulich
xen/arch/riscv/aplic.c static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask) · line 277
Quoted 10 lines
xen/arch/riscv/aplic.c +9
-0+/* clear the pending bit */ +csr_clear (CSR_SIP ,1UL <<IRQ_S_EXT ); + +/* dispatch the interrupt */ +do_IRQ (regs,csr_swap (CSR_STOPEI ,0 ) >>TOPI_IID_SHIFT ); + +/* enable external interrupts */ +csr_set (CSR_SIE ,1UL <<IRQ_S_EXT ); +}Why does "cause" need passing into here? I realize the function is used ...
Quoted 8 lines
xen/arch/riscv/aplic.c static const struct intc_hw_operations aplic_ops = { · line 297
Quoted 6 lines
xen/arch/riscv/aplic.c +1
-0@@ -278,6 +293,7 @@ static const struct intc_hw_operations aplic_ops = { .host_irq_type = &aplic_host_irq_type, .set_irq_priority = aplic_set_irq_priority, .set_irq_type = aplic_set_irq_type, + .handle_interrupt = aplic_handle_interrupt, };... as a hook, yet it's still unclear whether (why) any other such hook would need the cause to be passed in.
I don't remember a particular reason, but it could have been dropped. If, for some reason, the cause is needed in|handle_interrupt()|, it can be retrieved explicitly from a register.
Quoted 7 lines
xen/arch/riscv/imsic.c do { \ · line 47
Quoted 6 lines
xen/arch/riscv/imsic.c +1
-0@@ -33,6 +44,20 @@ do { \csr_clear (CSR_SIREG , v); \ }while (0 ) +void imsic_ids_local_delivery(bool enable)
__initas long as the sole caller is such?
Yes, it make sense. Also, I noticed some other functions that could be __init in imsic.c (but likely you mentioned that in the previous patches).
Quoted 6 lines
Quoted 6 lines
xen/arch/riscv/intc.c +0
-0@@ -51,6 +51,13 @@ static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority) intc_hw_ops->set_irq_priority (desc, priority); }
Quoted 8 lines
xen/arch/riscv/intc.c static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority) · line 56
Quoted 3 lines
xen/arch/riscv/intc.c +3
-0+void intc_handle_external_irqs (unsigned long cause,struct cpu_user_regs *regs) +{ +ASSERT (intc_hw_ops && intc_hw_ops->handle_interrupt);I don't view such checks (on every interrupt) as very useful. If you checked once early on - okay. But here you gain nothing at all ...
xen/arch/riscv/intc.c static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority) · line 58
Quoted 1 line
xen/arch/riscv/intc.c +1
-0+ intc_hw_ops->handle_interrupt (cause, regs);... towards the use here, when considering a release build.
I will try to find a better place then.
Quoted 16 lines
Quoted 16 lines
xen/arch/riscv/intc.c +10
-0@@ -83,3 +87,42 @@ void __init init_IRQ(void)if (init_irq_data () <0 )panic ("initialization of IRQ data failed\n" ); } + +/* Dispatch an interrupt */ +void do_IRQ (struct cpu_user_regs *regs,unsigned int irq) +{ +struct irq_desc *desc =irq_to_desc (irq); +struct irqaction *action; + +irq_enter (); + +spin_lock (&desc->lock);
Quoted 8 lines
xen/arch/riscv/intc.c void __init init_IRQ(void) · line 105
Quoted 6 lines
xen/arch/riscv/intc.c +6
-0+ desc->handler->ack (desc); + +if (test_bit (_IRQ_DISABLED, &desc->status) ) + goto out; + +set_bit (_IRQ_INPROGRESS, &desc->status);Same comment as on the earlier patch - atomic bitop when in a suitably locked region?
Agree, it could be used non-atomic bitop.
Quoted 7 lines
xen/arch/riscv/intc.c void __init init_IRQ(void) · line 110
Quoted 6 lines
xen/arch/riscv/intc.c +5
-0+ action = desc->action; + +spin_unlock_irq (&desc->lock); + +#ifndef CONFIG_IRQ_HAS_MULTIPLE_ACTION Stolen from Arm? What's this about?
Yes, it is stolen from Arm. I thought that it is a generic one, but the config is defined inside Arm's config.h.
Then it could be dropped now as I don't know, at the moment, the cases when it is neeeded to exectute several handler for an irq for RISC-V.
Thanks.
~ Oleksii
Re: [PATCH v1 11/14] xen/riscv: add external interrupt handling for hypervisor mode
Quoted 10 lines · Oleksii Kurochko
Quoted 6 lines
xen/arch/riscv/irq.c void __init init_IRQ(void) · line 111
Quoted 5 lines
xen/arch/riscv/irq.c +5
-0+ action = desc->action; + +spin_unlock_irq (&desc->lock); + +#ifndef CONFIG_IRQ_HAS_MULTIPLE_ACTION Stolen from Arm? What's this about?
Yes, it is stolen from Arm. I thought that it is a generic one, but the config is defined inside Arm's
config.h.
Then it could be dropped now as I don't know, at the moment, the cases when it is neeeded to exectute several handler for an irq for RISC-V.
Probably, IOMMU may setup multiple handler for the same interrupt. I'll double check that.
~ Oleksii
PATCH 12/14 xen/riscv: implement setup_irq()
In reply to Oleksii KurochkoIntroduce support for IRQ setup on RISC-V by implementing setup_irq() and __setup_irq(), adapted and extended from an initial implementation by [1].
__setup_irq() does the following:
- Sets up an IRQ action.
- Validates that shared IRQs have non-NULL dev_id and are only used when existing handlers allow sharing.
- Uses smp_mb() to enforce memory ordering after assigning desc->action to ensure visibility before enabling the IRQ.
- Supports multi-action setups via CONFIG_IRQ_HAS_MULTIPLE_ACTION.
setup_irq() does the following:
- Converts IRQ number to descriptor and acquires its lock.
- Rejects registration if the IRQ is already assigned to a guest domain, printing an error.
- Delegates the core setup to __setup_irq().
- On first-time setup, disables the IRQ, routes it to Xen using intc_route_irq_to_xen(), sets default CPU affinity (current CPU), calls the handler’s startup routine, and finally enables the IRQ.
irq_set_affinity() invokes set_affinity callback from the IRQ handler if present.
Defined IRQ_NO_PRIORITY as default priority used when routing IRQs to Xen.
[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7390e2365828b83e27ead56b03114a56e3699dd5
--- xen/arch/riscv/include/asm/irq.h | 6 ++ xen/arch/riscv/irq.c | 95 ++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+)
xen/arch/riscv/include/asm/irq.h +6 -0
@@ -26,6 +26,8 @@#define IRQ_TYPE_SENSE_MASK DT_IRQ_TYPE_SENSE_MASK#define IRQ_TYPE_INVALID DT_IRQ_TYPE_INVALID +#define IRQ_NO_PRIORITY 0 +/* TODO */ #define nr_static_irqs 0#define arch_hwdom_irqs (domid) 0U
@@ -54,6 +56,10 @@ void init_IRQ(void);struct cpu_user_regs ;void do_IRQ (struct cpu_user_regs *regs,unsigned int irq); +struct irq_desc ; +struct cpumask_t ; +void irq_set_affinity (struct irq_desc *desc,const cpumask_t *cpu_mask); +#endif /* ASM__RISCV__IRQ_H */ /*
xen/arch/riscv/irq.c +95 -0
108 lines not on this page · raw message
Signature · 1 line
2.49.0
Re: [PATCH v1 12/14] xen/riscv: implement setup_irq()
xen/arch/riscv/include/asm/irq.h void init_IRQ(void); · line 56
Quoted 13 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/irq.h +2
-0@@ -26,6 +26,8 @@#define IRQ_TYPE_SENSE_MASK DT_IRQ_TYPE_SENSE_MASK#define IRQ_TYPE_INVALID DT_IRQ_TYPE_INVALID +#define IRQ_NO_PRIORITY 0 +/* TODO */ #define nr_static_irqs 0#define arch_hwdom_irqs (domid) 0U@@ -54,6 +56,10 @@ void init_IRQ(void);struct cpu_user_regs ;
Seeing this and ...
xen/arch/riscv/include/asm/irq.h void init_IRQ(void); · line 60
Quoted 4 lines
xen/arch/riscv/include/asm/irq.h +2
-0void do_IRQ (struct cpu_user_regs *regs,unsigned int irq); +struct irq_desc ; +struct cpumask_t ;
... now these - all of such forward decls may want to collectively live in a central place higher up in the file.
xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 63
Quoted 6 lines
xen/arch/riscv/include/asm/irq.h +2
-0@@ -57,6 +59,99 @@ int platform_get_irq(const struct dt_device_node *device, int index)return dt_irq.irq; } +static int __setup_irq (struct irq_desc *desc,unsigned int irqflags, +struct irqaction *new)
Irrespective of you possibly having found it like this elsewhere, may I suggest that in new code we avoid leading double underscores? A single one will do here.
xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 69
Quoted 6 lines
xen/arch/riscv/include/asm/irq.h +6
-0+{ +bool shared = irqflags &IRQF_SHARED ; + +ASSERT (new !=NULL ); + +/* Sanity checks:
Nit: Comment style (and there are many more issues below).
xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 74
Quoted 5 lines
xen/arch/riscv/include/asm/irq.h +5
-0+* - if the IRQ is marked as shared +* - dev_id is not NULL when IRQF_SHARED is set +*/ +if ( desc->action !=NULL && (!test_bit (_IRQF_SHARED, &desc->status) + || !shared) )
Nit: Operator placement and indentation.
You're probably better off this way anyway:
if ( desc->action != NULL &&
(!test_bit(_IRQF_SHARED, &desc->status) || !shared) )xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 80
Quoted 6 lines
xen/arch/riscv/include/asm/irq.h +6
-0+return -EINVAL ; +if ( shared && new->dev_id ==NULL ) +return -EINVAL ; + +if ( shared ) +set_bit (_IRQF_SHARED, &desc->status);
See comments on earlier patches.
xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 87
Quoted 7 lines
xen/arch/riscv/include/asm/irq.h +7
-0+#ifdef CONFIG_IRQ_HAS_MULTIPLE_ACTION + new->next = desc->action; +smp_mb (); +#endif + + desc->action = new; +smp_mb ();
Aren't smp_wmb() sufficient on both places? If not, I think comments want adding.
xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 93
Quoted 6 lines
xen/arch/riscv/include/asm/irq.h +6
-0+return 0 ; +} + +void irq_set_affinity (struct irq_desc *desc,const cpumask_t *cpu_mask) +{ +if ( desc !=NULL )
Can desc really be NULL here? Isn't desc->lock required to be held?
xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 104
Quoted 11 lines
xen/arch/riscv/include/asm/irq.h +11
-0+ desc->handler->set_affinity (desc, cpu_mask); +} + +int setup_irq (unsigned int irq,unsigned int irqflags,struct irqaction *new) +{ +int rc; +unsigned long flags; +struct irq_desc *desc; +bool disabled; + + desc =irq_to_desc (irq);
Make this the variable's initializer?
Quoted 14 lines
xen/arch/riscv/include/asm/irq.h +14
-0+spin_lock_irqsave (&desc->lock, flags); + + disabled = (desc->action ==NULL ); + +if (test_bit (_IRQ_GUEST, &desc->status) ) + { +spin_unlock_irqrestore (&desc->lock, flags); +/* +* TODO: would be nice to have functionality to print which domain owns +* an IRQ. +*/ +printk (XENLOG_ERR "ERROR: IRQ %u is already in use by a domain\n" , irq); +return -EBUSY ; + }
xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 128
Quoted 10 lines
xen/arch/riscv/include/asm/irq.h +10
-0+ + rc =__setup_irq (desc, irqflags, new); +if ( rc ) + goto err; + +/* First time the IRQ is setup */ +if ( disabled ) + { +/* disable irq by default */ +set_bit (_IRQ_DISABLED, &desc->status);
Shouldn't this be set when we make it here?
Quoted 4 lines
xen/arch/riscv/include/asm/irq.h +4
-0+/* route interrupt to xen */ +intc_route_irq_to_xen (desc,IRQ_NO_PRIORITY ); + +/*
xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 142
Quoted 10 lines
xen/arch/riscv/include/asm/irq.h +10
-0+* we don't care for now which CPU will receive the +* interrupt +* +* TODO: Handle case where IRQ is setup on different CPU than +* the targeted CPU and the priority. +*/ +irq_set_affinity (desc,cpumask_of (smp_processor_id ())); + desc->handler->startup (desc); +/* enable irq */ +clear_bit (_IRQ_DISABLED, &desc->status);
Now it turns out this is really done twice: Once in aplic_irq_enable(), and once here.
xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 145
Quoted 3 lines
xen/arch/riscv/include/asm/irq.h +3
-0+ } + +err:
Nit (yet once again).
Jan
Re: [PATCH v1 12/14] xen/riscv: implement setup_irq()
On 4/15/25 5:55 PM, Jan Beulich wrote
Quoted 9 lines
xen/arch/riscv/irq.c int platform_get_irq(const struct dt_device_node *device, int index) · line 88
Quoted 7 lines
xen/arch/riscv/irq.c +7
-0+#ifdef CONFIG_IRQ_HAS_MULTIPLE_ACTION + new->next = desc->action; +smp_mb (); +#endif + + desc->action = new; +smp_mb ();Aren't
smp_wmb()sufficient on both places? If not, I think comments want adding.
smp_wmb() will be sufficient but I think the barrier could be dropped at all as __setup_irq() is called only in setup_irq() and __setup_irq() call is wrapped by spinlock_{un}lock_irqsave() where spinlock_unlock_*() will put barrier.
Quoted 8 lines
xen/arch/riscv/irq.c int platform_get_irq(const struct dt_device_node *device, int index) · line 95
Quoted 7 lines
xen/arch/riscv/irq.c +6
-0+return 0 ; +} + +void irq_set_affinity (struct irq_desc *desc,const cpumask_t *cpu_mask) +{ +if ( desc !=NULL )Can desc really be NULL here?
It can't as irq_desc[] isn't dynamically allocated.
Quoted 1 line
Isn't
desc->lockrequired to be held?
It is and it is called in setup_irq() which calls spin_lock_irqsave().
Anyway, I think it could be dropped at all and use 'desc->handler->set_affinity(desc, cpu_mask);' explicitly in setup_irq().
Quoted 17 lines
Quoted 17 lines
xen/arch/riscv/irq.c +17
-0+spin_lock_irqsave (&desc->lock, flags); + + disabled = (desc->action ==NULL ); + +if (test_bit (_IRQ_GUEST, &desc->status) ) + { +spin_unlock_irqrestore (&desc->lock, flags); +/* +* TODO: would be nice to have functionality to print which domain owns +* an IRQ. +*/ +printk (XENLOG_ERR "ERROR: IRQ %u is already in use by a domain\n" , irq); +return -EBUSY ; + } + + rc =__setup_irq (desc, irqflags, new); +if ( rc )
Quoted 8 lines
xen/arch/riscv/irq.c int platform_get_irq(const struct dt_device_node *device, int index) · line 131
Quoted 7 lines
xen/arch/riscv/irq.c +7
-0+ goto err; + +/* First time the IRQ is setup */ +if ( disabled ) + { +/* disable irq by default */ +set_bit (_IRQ_DISABLED, &desc->status);Shouldn't this be set when we make it here?
It should be. I'll drop the setting of _IRQ_DISABLED.
Quoted 15 lines
Quoted 5 lines
xen/arch/riscv/irq.c +4
-0+/* route interrupt to xen */ +intc_route_irq_to_xen (desc,IRQ_NO_PRIORITY ); + +/* xen/arch/riscv/irq.c int platform_get_irq(const struct dt_device_node *device, int index) · line 146
Quoted 10 lines
xen/arch/riscv/irq.c +10
-0+* we don't care for now which CPU will receive the +* interrupt +* +* TODO: Handle case where IRQ is setup on different CPU than +* the targeted CPU and the priority. +*/ +irq_set_affinity (desc,cpumask_of (smp_processor_id ())); + desc->handler->startup (desc); +/* enable irq */ +clear_bit (_IRQ_DISABLED, &desc->status);
Quoted 2 lines
Now it turns out this is really done twice: Once in
aplic_irq_enable(), and once here.
Agree, this is a job of *_startup()->*_aplic_irq_enable(). I'll drop that too.
~ Oleksii
Re: [PATCH v1 12/14] xen/riscv: implement setup_irq()
Quoted 12 lines · Oleksii Kurochko
On 4/15/25 5:55 PM, Jan Beulich wrote
Quoted 11 lines
Quoted 11 lines
xen/arch/riscv/irq.c +11
-0+/* +* we don't care for now which CPU will receive the +* interrupt +* +* TODO: Handle case where IRQ is setup on different CPU than +* the targeted CPU and the priority. +*/ +irq_set_affinity (desc,cpumask_of (smp_processor_id ())); + desc->handler->startup (desc); +/* enable irq */ +clear_bit (_IRQ_DISABLED, &desc->status);
Quoted 4 lines
Quoted 2 lines
Now it turns out this is really done twice: Once in
aplic_irq_enable(), and once here.Agree, this is a job of *_startup()->*_aplic_irq_enable(). I'll drop that too.
Wait - see my comment there. I think it belongs here, not there.
Jan
PATCH 13/14 xen/riscv: initialize interrupt controller
In reply to Oleksii KurochkoCall intc_init() to do basic initialization steps for APLIC and IMISC.
--- xen/arch/riscv/setup.c | 2 ++ 1 file changed, 2 insertions(+)
xen/arch/riscv/setup.c +2 -0
@@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,intc_preinit (); +intc_init (); +printk ("All set up\n" );machine_halt ();
Signature · 1 line
2.49.0
Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller
Quoted 3 lines · Oleksii Kurochko
Call
intc_init()to do basic initialization steps for APLIC and IMISC.Signed-off-by Oleksii Kurochko <[email protected]>
yet ...
xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 143
Quoted 11 lines
xen/arch/riscv/setup.c +2
-0@@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,intc_preinit (); +intc_init (); +printk ("All set up\n" );machine_halt ();
... this being everything here I wonder if this can't be folded with the patch where the function is introduced.
Jan
Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller
Quoted 13 lines · Jan Beulich
Quoted 3 lines · Oleksii Kurochko
Call
intc_init()to do basic initialization steps for APLIC and IMISC.Signed-off-by Oleksii Kurochko<[email protected]>Acked-by Jan Beulich<[email protected]>yet ...
Quoted 6 lines
xen/arch/riscv/setup.c +0
-0@@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,intc_preinit ();
Quoted 7 lines
xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 143
Quoted 5 lines
xen/arch/riscv/setup.c +2
-0+intc_init (); +printk ("All set up\n" );machine_halt ();... this being everything here I wonder if this can't be folded with the patch where the function is introduced.
Sure, it can be folded. I will do that to reduce patch series.
~ Oleksii
Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller
Quoted 16 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich
Quoted 3 lines · Oleksii Kurochko
Call
intc_init()to do basic initialization steps for APLIC and IMISC.Signed-off-by Oleksii Kurochko<[email protected]>Acked-by Jan Beulich<[email protected]>yet ...
Quoted 6 lines
xen/arch/riscv/setup.c +0
-0@@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,intc_preinit ();
Quoted 8 lines
Quoted 7 lines
xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 143
Quoted 5 lines
xen/arch/riscv/setup.c +2
-0+intc_init (); +printk ("All set up\n" );machine_halt ();... this being everything here I wonder if this can't be folded with the patch where the function is introduced.
Sure, it can be folded. I will do that to reduce patch series.
I doubled checked and, at the moment, when intc_init() is introduced:
void __init intc_init(void)
{
ASSERT(intc_hw_ops);
if ( intc_hw_ops->init() )
panic("Failed to initialize the interrupt controller drivers\n");
}intc_hw_ops isn't registered as they are registered in the next two patches after intriduction of intc_hw_ops.
~ Oleksii
Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller
Quoted 36 lines · Oleksii Kurochko
Quoted 16 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich
Quoted 3 lines · Oleksii Kurochko
Call
intc_init()to do basic initialization steps for APLIC and IMISC.Signed-off-by Oleksii Kurochko<[email protected]>Acked-by Jan Beulich<[email protected]>yet ...
Quoted 6 lines
xen/arch/riscv/setup.c +0
-0@@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,intc_preinit ();Quoted 8 lines
Quoted 7 lines
xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 143
Quoted 5 lines
xen/arch/riscv/setup.c +2
-0+intc_init (); +printk ("All set up\n" );machine_halt ();... this being everything here I wonder if this can't be folded with the patch where the function is introduced.
Sure, it can be folded. I will do that to reduce patch series.
I doubled checked and, at the moment, when
intc_init()is introduced:
void__initintc_init(void){ ASSERT(intc_hw_ops); if ( intc_hw_ops->init() ) panic("Failed to initialize the interrupt controller drivers\n"); }
Quoted 2 lines
intc_hw_opsisn't registered as they are registered in the next two patches after intriduction ofintc_hw_ops.
Which then feels wrong anyway; you're then merely leveraging that the function has no caller, which (as said elsewhere) shouldn't be the case at the very least for Misra's sake. So I expect some re-ordering to be necessary. Or you may want to introduce the function empty and add the intc_hw_ops uses as intc_hw_ops is introduced.
Jan
PATCH 14/14 xen/riscv: add basic UART support
In reply to Oleksii KurochkoUpdate Kconfig to select GENERIC_UART_INIT for basic UART init ( find a dt node and call device specific device_init() ).
Drop default n if RISCV statement for config HAS_NS16550 as now ns16550 is ready to be compiled and used by RISC-V.
Initialize a minimal amount of stuff to have UART and Xen console:
- Initialize uart by calling uart_init().
- Initialize Xen console by calling console_init_{pre,post}irq().
- Initialize timer and its internal lists which are used by init_timer() which is called by ns16550_init_postirq(); otherwise "Unhandled exception: Store/AMO Page Fault" occurs.
- Enable local interrupt to recieve an input from UART
--- xen/arch/riscv/Kconfig | 1 + xen/arch/riscv/setup.c | 16 ++++++++++++++++ xen/drivers/char/Kconfig | 1 - 3 files changed, 17 insertions(+), 1 deletion(-)
xen/arch/riscv/Kconfig +1 -0
@@ -2,6 +2,7 @@ config RISCV def_bool y select FUNCTION_ALIGNMENT_16B select GENERIC_BUG_FRAME + select GENERIC_UART_INIT select HAS_DEVICE_TREE select HAS_PMAP select HAS_UBSAN
xen/arch/riscv/setup.c +16 -0
@@ -4,11 +4,16 @@#include <xen/bug.h> #include <xen/bootfdt.h> #include <xen/compile.h> +#include <xen/console.h> #include <xen/device_tree.h> #include <xen/init.h> #include <xen/irq.h> +#include <xen/keyhandler.h> #include <xen/mm.h> +#include <xen/percpu.h> +#include <xen/serial.h> #include <xen/shutdown.h> +#include <xen/timer.h> #include <xen/vmap.h> #include <xen/xvmalloc.h>
@@ -73,6 +78,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,remove_identity_mapping (); +percpu_init_areas (); +smp_clear_cpu_maps ();set_processor_id (0 );
@@ -136,8 +143,17 @@ void __init noreturn start_xen(unsigned long bootcpu_id,intc_preinit (); +uart_init (); +console_init_preirq (); +intc_init (); +timer_init (); + +local_irq_enable (); + +console_init_postirq (); +printk ("All set up\n" );machine_halt ();
xen/drivers/char/Kconfig +0 -1
@@ -3,7 +3,6 @@ config GENERIC_UART_INIT config HAS_NS16550 bool "NS16550 UART driver" if ARM - default n if RISCV default y help This selects the 16550-series UART support. For most systems, say Y.
Signature · 1 line
2.49.0
Re: [PATCH v1 14/14] xen/riscv: add basic UART support
xen/arch/riscv/setup.c line 11
Quoted 11 lines · Oleksii Kurochko
xen/arch/riscv/setup.c +2
-0@@ -4,11 +4,16 @@#include <xen/bug.h> #include <xen/bootfdt.h> #include <xen/compile.h> +#include <xen/console.h> #include <xen/device_tree.h> #include <xen/init.h> #include <xen/irq.h> +#include <xen/keyhandler.h>
Why's this one needed?
xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 81
Quoted 13 lines
xen/arch/riscv/setup.c +4
-0#include <xen/mm.h> +#include <xen/percpu.h> +#include <xen/serial.h> #include <xen/shutdown.h> +#include <xen/timer.h> #include <xen/vmap.h> #include <xen/xvmalloc.h> @@ -73,6 +78,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,remove_identity_mapping (); +percpu_init_areas ();
I'll trust you that it's needed now, but the addition looks unrelated here, and also isn't mentioned as intentional in the description.
xen/drivers/char/Kconfig config GENERIC_UART_INIT · line 6
Quoted 8 lines
xen/drivers/char/Kconfig +0
-1@@ -3,7 +3,6 @@ config GENERIC_UART_INIT config HAS_NS16550 bool "NS16550 UART driver" if ARM - default n if RISCV default y
Just to double-check: Unlike Arm you don't want this to be user-(un)selectable on RISC-V?
Jan
Re: [PATCH v1 14/14] xen/riscv: add basic UART support
Quoted 13 lines · Jan Beulich
xen/arch/riscv/setup.c line 11
Quoted 11 lines · Oleksii Kurochko
xen/arch/riscv/setup.c +2
-0@@ -4,11 +4,16 @@#include <xen/bug.h> #include <xen/bootfdt.h> #include <xen/compile.h> +#include <xen/console.h> #include <xen/device_tree.h> #include <xen/init.h> #include <xen/irq.h> +#include <xen/keyhandler.h> Why's this one needed?
It isn't needed anymore, just missed to drop.
I thought that it would be needed to test UART working fine by checking if Ctrl+AAA is working.
Quoted 9 lines
Quoted 9 lines
xen/arch/riscv/setup.c +3
-0#include <xen/mm.h> +#include <xen/percpu.h> +#include <xen/serial.h> #include <xen/shutdown.h> +#include <xen/timer.h> #include <xen/vmap.h> #include <xen/xvmalloc.h>
Quoted 7 lines
xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 81
Quoted 5 lines
xen/arch/riscv/setup.c +1
-0@@ -73,6 +78,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,remove_identity_mapping (); +percpu_init_areas ();I'll trust you that it's needed now, but the addition looks unrelated here, and also isn't mentioned as intentional in the description.
I had this patch when I used polling mode for UART, for this case percpu is used to receive serial port info:
struct serial_port *port = this_cpu(poll_port);
So percpu isn't really needed at the current development state. I'll drop this change or as an option move to separate patch.
Quoted 10 lines
xen/drivers/char/Kconfig config GENERIC_UART_INIT · line 6
Quoted 8 lines
xen/drivers/char/Kconfig +0
-1@@ -3,7 +3,6 @@ config GENERIC_UART_INIT config HAS_NS16550 bool "NS16550 UART driver" if ARM - default n if RISCV default yJust to double-check: Unlike Arm you don't want this to be user-(un)selectable on RISC-V?
Thanks for noticing that. I want to have this selectable by user. I will add RISC-V here.
~ Oleksii
Re: [PATCH v1 14/14] xen/riscv: add basic UART support
Quoted 6 lines · Oleksii Kurochko
Quoted 5 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/drivers/char/Kconfig +0
-0@@ -3,7 +3,6 @@ config GENERIC_UART_INIT
Quoted 8 lines
Quoted 6 lines
xen/drivers/char/Kconfig config GENERIC_UART_INIT · line 6
Quoted 4 lines
xen/drivers/char/Kconfig +0
-1config HAS_NS16550 bool "NS16550 UART driver" if ARM - default n if RISCV default yJust to double-check: Unlike Arm you don't want this to be user-(un)selectable on RISC-V?
Thanks for noticing that. I want to have this selectable by user. I will add RISC-V here.
At which point we may want to consider whether the condition on the prompt wouldn't better become "!X86".
Jan