Skip to contentSkip to conversation

riscv: introduce basic UART support and interrupts for hypervisor mode

PATCH Superseded Oleksii Kurochko 2 participants 79 messages Apr 8, 2025 – Apr 30, 2025 1 message not in the archive RISC-V

v1 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

cover riscv: introduce basic UART support and interrupts for hypervisor mode

The patch series introduces basic UART support (in interrupt mode) and support of interrupts for hypervisor mode.

To implement this the following has been added:
- APLIC and IMISC initialization.
- Introduce of intc_hw_operations abstraction.
- Introduce some APLIC and IMSIC operations.
- Introduce init_IRQ(), platform_get_irq() and setup_irq() functions.
- Update do_trap() handler to handle IRQ_S_EXT.
- Introduce some other functions such as: get_s_time(), smp_clear_cpu_maps(), ioremap().
- Enable UART.

Oleksii Kurochko (14):
xen/riscv: implement get_s_time()
xen/riscv: introduce smp_clear_cpu_maps()
xen/riscv: introduce ioremap()
xen/riscv: introduce init_IRQ()
xen/riscv: introduce platform_get_irq()
xen/riscv: riscv_of_processor_hartid() implementation xen/riscv: Introduce intc_hw_operations abstraction xen/riscv: imsic_init() implementation
xen/riscv: aplic_init() implementation
xen/riscv: implementation of aplic and imsic operations xen/riscv: add external interrupt handling for hypervisor mode xen/riscv: implement setup_irq()
xen/riscv: initialize interrupt controller xen/riscv: add basic UART support

 xen/arch/riscv/Kconfig             |  13 +
 xen/arch/riscv/Makefile            |   3 +
 xen/arch/riscv/aplic.c             | 302 +++++++++++++++++++++++
 xen/arch/riscv/imsic.c             | 374 +++++++++++++++++++++++++++++
 xen/arch/riscv/include/asm/aplic.h |  89 +++++++
 xen/arch/riscv/include/asm/imsic.h |  88 +++++++
 xen/arch/riscv/include/asm/intc.h  |  31 +++
 xen/arch/riscv/include/asm/irq.h   |  37 ++-
 xen/arch/riscv/include/asm/smp.h   |   5 +
 xen/arch/riscv/include/asm/time.h  |   6 +
 xen/arch/riscv/intc.c              |  58 +++++
 xen/arch/riscv/irq.c               | 223 +++++++++++++++++
 xen/arch/riscv/pt.c                |  19 ++
 xen/arch/riscv/setup.c             |  23 ++
 xen/arch/riscv/smpboot.c           |  83 +++++++
 xen/arch/riscv/stubs.c             |  16 --
 xen/arch/riscv/time.c              |   7 +
 xen/arch/riscv/traps.c             |  18 ++
 xen/drivers/char/Kconfig           |   1 -
 19 files changed, 1378 insertions(+), 18 deletions(-)
 create mode 100644 xen/arch/riscv/imsic.c
 create mode 100644 xen/arch/riscv/include/asm/aplic.h
 create mode 100644 xen/arch/riscv/include/asm/imsic.h
 create mode 100644 xen/arch/riscv/irq.c
 create mode 100644 xen/arch/riscv/smpboot.c
Signature · 1 line
2.49.0
#1 lore raw

PATCH 1/14 xen/riscv: implement get_s_time()

Also tick_to_ns() is implemeted as it is used in get_s_time().

---
 xen/arch/riscv/include/asm/time.h | 6 ++++++
 xen/arch/riscv/stubs.c            | 5 -----
 xen/arch/riscv/time.c             | 7 +++++++
 3 files changed, 13 insertions(+), 5 deletions(-)
xen/arch/riscv/include/asm/time.h +6 -0
index e8d9ffec57..0f6aa99ab1 100644
@@ -3,6 +3,7 @@
 #define ASM__RISCV__TIME_H
 
 #include <xen/bug.h>
+#include <xen/lib.h>
 #include <xen/types.h>
 #include <asm/csr.h>
 
@@ -23,6 +24,11 @@ static inline cycles_t get_cycles(void)
     return csr_read(CSR_TIME);
 }
 
+static inline s_time_t ticks_to_ns(uint64_t ticks)
+{
+    return muldiv64(ticks, SECONDS(1), 1000 * cpu_khz);
+}
+
 void preinit_xen_time(void);
 
 #endif /* ASM__RISCV__TIME_H */
xen/arch/riscv/stubs.c +0 -5
index a1d64534cd..83416d3350 100644
@@ -27,11 +27,6 @@ nodemask_t __read_mostly node_online_map = { { [0] = 1UL } };
 
 /* time.c */
 
-s_time_t get_s_time(void)
-{
-    BUG_ON("unimplemented");
-}
-
 int reprogram_timer(s_time_t timeout)
 {
     BUG_ON("unimplemented");
xen/arch/riscv/time.c +7 -0
index 905bb13eb4..81e06781f8 100644
@@ -4,10 +4,17 @@
 #include <xen/init.h>
 #include <xen/lib.h>
 #include <xen/sections.h>
+#include <xen/types.h>
 
 unsigned long __ro_after_init cpu_khz; /* CPU clock frequency in kHz. */
 uint64_t __ro_after_init boot_clock_cycles;
 
+s_time_t get_s_time(void)
+{
+    uint64_t ticks = get_cycles() - boot_clock_cycles;
+    return ticks_to_ns(ticks);
+}
+
 /* Set up the timer on the boot CPU (early init function) */
 static void __init preinit_dt_xen_time(void)
 {
Signature · 1 line
2.49.0
Reviewed-by on v1 1/14
#2 lore raw

Re: [PATCH v1 01/14] xen/riscv: implement get_s_time()

xen/arch/riscv/include/asm/time.h static inline cycles_t get_cycles(void) · line 30

Quoted 8 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/time.h +4 -0
@@ -23,6 +24,11 @@ static inline cycles_t get_cycles(void)
     return csr_read(CSR_TIME);
 }
 
+static inline s_time_t ticks_to_ns(uint64_t ticks)
+{
+    return muldiv64(ticks, SECONDS(1), 1000 * cpu_khz);
+}

Why the extra multiplication by 1000? I.e. why not "muldiv64(ticks, MILLISECONDS(1), cpu_khz)", getting away with just one multiplication and a reduced risk of encountering intermediate overflow (affecting only hypothetical above 4THz CPUs then)?

Quoted 5 lines
xen/arch/riscv/time.c +0 -0
@@ -4,10 +4,17 @@
 #include <xen/init.h>
 #include <xen/lib.h>

xen/arch/riscv/time.c line 15

Quoted 10 lines
xen/arch/riscv/time.c +5 -0
 #include <xen/sections.h>
+#include <xen/types.h>
 
 unsigned long __ro_after_init cpu_khz; /* CPU clock frequency in kHz. */
 uint64_t __ro_after_init boot_clock_cycles;
 
+s_time_t get_s_time(void)
+{
+    uint64_t ticks = get_cycles() - boot_clock_cycles;
+    return ticks_to_ns(ticks);

Nit: Blank line between declaration(s) and statement(s) please, as well as ahead of the main "return" of a function.

Happy to make both adjustments upon committing, so long as you agree; then:

Jan

#3 lore raw

Re: [PATCH v1 01/14] xen/riscv: implement get_s_time()

Quoted 5 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/time.h +0 -0
@@ -23,6 +24,11 @@ static inline cycles_t get_cycles(void)
      return csr_read(CSR_TIME);
  }
 
Quoted 8 lines

xen/arch/riscv/include/asm/time.h static inline cycles_t get_cycles(void) · line 30

Quoted 4 lines
xen/arch/riscv/include/asm/time.h +4 -0
+static inline s_time_t ticks_to_ns(uint64_t ticks)
+{
+    return muldiv64(ticks, SECONDS(1), 1000 * cpu_khz);
+}

Why the extra multiplication by 1000? I.e. why not "muldiv64(ticks, MILLISECONDS(1), cpu_khz)", getting away with just one multiplication and a reduced risk of encountering intermediate overflow (affecting only hypothetical above 4THz CPUs then)?

Multiplication by 1000 was needed to convert khz to hz, but yes, your option would be better.

Quoted 19 lines
Quoted 6 lines
xen/arch/riscv/time.c +0 -0
@@ -4,10 +4,17 @@
  #include <xen/init.h>
  #include <xen/lib.h>

xen/arch/riscv/time.c line 15

Quoted 10 lines
xen/arch/riscv/time.c +5 -0
  #include <xen/sections.h>
+#include <xen/types.h>
 
  unsigned long __ro_after_init cpu_khz; /* CPU clock frequency in kHz. */
  uint64_t __ro_after_init boot_clock_cycles;
 
+s_time_t get_s_time(void)
+{
+    uint64_t ticks = get_cycles() - boot_clock_cycles;
+    return ticks_to_ns(ticks);

Nit: Blank line between declaration(s) and statement(s) please, as well as ahead of the main "return" of a function.

Quoted 2 lines

Happy to make both adjustments upon committing, so long as you agree; then:

I'll be happy with that.

Thank you very much.

~ Oleksii

#4 lore raw

PATCH 2/14 xen/riscv: introduce smp_clear_cpu_maps()

In reply to Oleksii Kurochko

Initialize cpu_{possible, online, present}_map by using smp_clear_cpu_maps().

Drop DEFINE_PER_CPU(unsigned int, cpu_id) from stubs.c as this variable isn't expected to be used in RISC-V at all.

Move declaration of cpu_{possible,online,present}_map from stubs.c to smpboot.c as now smpboot.c is now introduced.
Other defintions keep in stubs.c as they are not initialized and not needed, at the moment.

---
 xen/arch/riscv/Makefile          |  1 +
 xen/arch/riscv/include/asm/smp.h |  2 ++
 xen/arch/riscv/setup.c           |  2 ++
 xen/arch/riscv/smpboot.c         | 15 +++++++++++++++
 xen/arch/riscv/stubs.c           |  6 ------
 5 files changed, 20 insertions(+), 6 deletions(-)
 create mode 100644 xen/arch/riscv/smpboot.c
xen/arch/riscv/Makefile +1 -0
index 0c6c4a38a3..f551bf32a2 100644
@@ -10,6 +10,7 @@ obj-y += sbi.o
 obj-y += setup.o
 obj-y += shutdown.o
 obj-y += smp.o
+obj-y += smpboot.o
 obj-y += stubs.o
 obj-y += time.o
 obj-y += traps.o
xen/arch/riscv/include/asm/smp.h +2 -0
index 5e170b57b3..188c033718 100644
@@ -26,6 +26,8 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,
 
 void setup_tp(unsigned int cpuid);
 
+void smp_clear_cpu_maps(void);
+
 #endif
 
 /*
xen/arch/riscv/setup.c +2 -0
index 4e416f6e44..7f68f3f5b7 100644
@@ -72,6 +72,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     remove_identity_mapping();
 
+    smp_clear_cpu_maps();
+
     set_processor_id(0);
 
     set_cpuid_to_hartid(0, bootcpu_id);
xen/arch/riscv/smpboot.c new file +15 -0
new file mode 100644
index 0000000000..0f4dcc28e1
@@ -0,0 +1,15 @@
+#include <xen/cpumask.h>
+#include <xen/init.h>
+
+cpumask_t cpu_online_map;
+cpumask_t cpu_present_map;
+cpumask_t cpu_possible_map;
+
+void __init smp_clear_cpu_maps(void)
+{
+    cpumask_clear(&cpu_possible_map);
+    cpumask_clear(&cpu_online_map);
+    cpumask_set_cpu(0, &cpu_possible_map);
+    cpumask_set_cpu(0, &cpu_online_map);
+    cpumask_copy(&cpu_present_map, &cpu_possible_map);
+}
xen/arch/riscv/stubs.c +0 -6
index 83416d3350..fdcf91054e 100644
@@ -11,12 +11,6 @@
 
 /* smpboot.c */
 
-cpumask_t cpu_online_map;
-cpumask_t cpu_present_map;
-cpumask_t cpu_possible_map;
-
-/* ID of the PCPU we're running on */
-DEFINE_PER_CPU(unsigned int, cpu_id);
 /* XXX these seem awfully x86ish... */
 /* representing HT siblings of each logical CPU */
 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_mask);
Signature · 1 line
2.49.0
#5 lore raw

Re: [PATCH v1 02/14] xen/riscv: introduce smp_clear_cpu_maps()

Quoted 64 lines · Oleksii Kurochko

Initialize cpu_{possible, online, present}_map by using smp_clear_cpu_maps().

Drop DEFINE_PER_CPU(unsigned int, cpu_id) from stubs.c as this variable isn't expected to be used in RISC-V at all.

Move declaration of cpu_{possible,online,present}_map from stubs.c to smpboot.c as now smpboot.c is now introduced.
Other defintions keep in stubs.c as they are not initialized and not needed, at the moment.

---
 xen/arch/riscv/Makefile          |  1 +
 xen/arch/riscv/include/asm/smp.h |  2 ++
 xen/arch/riscv/setup.c           |  2 ++
 xen/arch/riscv/smpboot.c         | 15 +++++++++++++++
 xen/arch/riscv/stubs.c           |  6 ------
 5 files changed, 20 insertions(+), 6 deletions(-)
 create mode 100644 xen/arch/riscv/smpboot.c
xen/arch/riscv/Makefile +1 -0
index 0c6c4a38a3..f551bf32a2 100644
@@ -10,6 +10,7 @@ obj-y += sbi.o
 obj-y += setup.o
 obj-y += shutdown.o
 obj-y += smp.o
+obj-y += smpboot.o
 obj-y += stubs.o
 obj-y += time.o
 obj-y += traps.o
xen/arch/riscv/include/asm/smp.h +2 -0
index 5e170b57b3..188c033718 100644
@@ -26,6 +26,8 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,
 
 void setup_tp(unsigned int cpuid);
 
+void smp_clear_cpu_maps(void);
+
 #endif
 
 /*
xen/arch/riscv/setup.c +2 -0
index 4e416f6e44..7f68f3f5b7 100644
@@ -72,6 +72,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     remove_identity_mapping();
 
+    smp_clear_cpu_maps();
+
     set_processor_id(0);
 
     set_cpuid_to_hartid(0, bootcpu_id);
xen/arch/riscv/smpboot.c new file +0 -0
new file mode 100644
index 0000000000..0f4dcc28e1

xen/arch/riscv/smpboot.c line 6

Quoted 6 lines
xen/arch/riscv/smpboot.c new file +6 -0
@@ -0,0 +1,15 @@
+#include <xen/cpumask.h>
+#include <xen/init.h>
+
+cpumask_t cpu_online_map;
+cpumask_t cpu_present_map;
+cpumask_t cpu_possible_map;

__read_mostly for all of them, perhaps (if CPU hotplug isn't expected to be supported) even __ro_after_init for the latter two?

As to cpu_possible_map - do you predict that you'll actually use it? Arm does (and instead has only a fake cpu_present_map), but on x86 we get away without.

xen/arch/riscv/smpboot.c line 11

Quoted 4 lines
xen/arch/riscv/smpboot.c new file +4 -0
+void __init smp_clear_cpu_maps(void)
+{
+    cpumask_clear(&cpu_possible_map);
+    cpumask_clear(&cpu_online_map);

What's the point of these? All three maps start out fully zeroed.

xen/arch/riscv/smpboot.c line 13

Quoted 2 lines
xen/arch/riscv/smpboot.c new file +2 -0
+    cpumask_set_cpu(0, &cpu_possible_map);
+    cpumask_set_cpu(0, &cpu_online_map);

These are contradicting the name of the function. The somewhat equivalent function we have on x86 is smp_prepare_boot_cpu().

xen/arch/riscv/smpboot.c line 14

Quoted 1 line
xen/arch/riscv/smpboot.c new file +1 -0
+    cpumask_copy(&cpu_present_map, &cpu_possible_map);

Another cpumask_set_cpu() is probably cheaper here then.

Jan

#6 lore raw

Re: [PATCH v1 02/14] xen/riscv: introduce smp_clear_cpu_maps()

Quoted 58 lines · Jan Beulich
Quoted 57 lines · Oleksii Kurochko

Initialize cpu_{possible, online, present}_map by using smp_clear_cpu_maps().

Drop DEFINE_PER_CPU(unsigned int, cpu_id) from stubs.c as this variable isn't expected to be used in RISC-V at all.

Move declaration of cpu_{possible,online,present}_map from stubs.c to smpboot.c as now smpboot.c is now introduced.
Other defintions keep in stubs.c as they are not initialized and not needed, at the moment.

---
  xen/arch/riscv/Makefile          |  1 +
  xen/arch/riscv/include/asm/smp.h |  2 ++
  xen/arch/riscv/setup.c           |  2 ++
  xen/arch/riscv/smpboot.c         | 15 +++++++++++++++
  xen/arch/riscv/stubs.c           |  6 ------
  5 files changed, 20 insertions(+), 6 deletions(-)

create mode 100644 xen/arch/riscv/smpboot.c

xen/arch/riscv/Makefile +1 -0
index 0c6c4a38a3..f551bf32a2 100644
@@ -10,6 +10,7 @@ obj-y += sbi.o
  obj-y += setup.o
  obj-y += shutdown.o
  obj-y += smp.o
+obj-y += smpboot.o
  obj-y += stubs.o
  obj-y += time.o
  obj-y += traps.o
xen/arch/riscv/include/asm/smp.h +2 -0
index 5e170b57b3..188c033718 100644
@@ -26,6 +26,8 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,
 
  void setup_tp(unsigned int cpuid);
 
+void smp_clear_cpu_maps(void);
+
  #endif
 
  /*
xen/arch/riscv/setup.c +2 -0
index 4e416f6e44..7f68f3f5b7 100644
@@ -72,6 +72,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
      remove_identity_mapping();
 
+    smp_clear_cpu_maps();
+
      set_processor_id(0);
 
Quoted 15 lines

xen/arch/riscv/smpboot.c line 6

Quoted 13 lines
xen/arch/riscv/setup.c +0 -0
      set_cpuid_to_hartid(0, bootcpu_id);
xen/arch/riscv/smpboot.c new file +6 -0
new file mode 100644
index 0000000000..0f4dcc28e1
@@ -0,0 +1,15 @@
+#include <xen/cpumask.h>
+#include <xen/init.h>
+
+cpumask_t cpu_online_map;
+cpumask_t cpu_present_map;
+cpumask_t cpu_possible_map;

__read_mostly for all of them, perhaps (if CPU hotplug isn't expected to be supported) even __ro_after_init for the latter two?

We have been living without CPU hotplug support for a long time in the downstream branch, but I can't say whether it is expected to be supported in the future or not. To ensure we can add such an option later without changing the attributes of cpu_online_map variable, I prefer to use|__read_mostly| here and __ro_after_init for cpu_possible_map.

Quoted 4 lines

As to cpu_possible_map - do you predict that you'll actually use it? Arm does (and instead has only a fake cpu_present_map), but on x86 we get away without.

I checked how it is used now in downstream latest branch and it isn't really used only during initialization smp_clear_cpu_maps() and smp_prepare_cpus() so we can skip it for RISC-V too.

Quoted 6 lines

xen/arch/riscv/smpboot.c line 11

Quoted 5 lines
xen/arch/riscv/smpboot.c new file +4 -0
+void __init smp_clear_cpu_maps(void)
+{
+    cpumask_clear(&cpu_possible_map);
+    cpumask_clear(&cpu_online_map);

What's the point of these? All three maps start out fully zeroed.

It could be really dropped. I saw your patch for Arm, I'll align the current patch with that changes.

Quoted 8 lines

xen/arch/riscv/smpboot.c line 13

Quoted 3 lines
xen/arch/riscv/smpboot.c new file +2 -0
+    cpumask_set_cpu(0, &cpu_possible_map);
+    cpumask_set_cpu(0, &cpu_online_map);

These are contradicting the name of the function. The somewhat equivalent function we have on x86 is smp_prepare_boot_cpu().

xen/arch/riscv/smpboot.c line 14

Quoted 1 line
xen/arch/riscv/smpboot.c new file +1 -0
+    cpumask_copy(&cpu_present_map, &cpu_possible_map);

Another cpumask_set_cpu() is probably cheaper here then.

What do you mean by cheaper here?

~ Oleksii

#7 lore raw

Re: [PATCH v1 02/14] xen/riscv: introduce smp_clear_cpu_maps()

Quoted 13 lines · Oleksii Kurochko
Quoted 6 lines · Jan Beulich

xen/arch/riscv/smpboot.c line 11

Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c new file +4 -0
+void __init smp_clear_cpu_maps(void)
+{
+    cpumask_clear(&cpu_possible_map);
+    cpumask_clear(&cpu_online_map);

What's the point of these? All three maps start out fully zeroed.

It could be really dropped. I saw your patch for Arm, I'll align the current patch with that changes.

Quoted 2 lines
Quoted 2 lines
xen/arch/riscv/smpboot.c new file +2 -0
+    cpumask_set_cpu(0, &cpu_possible_map);
+    cpumask_set_cpu(0, &cpu_online_map);
Quoted 7 lines
Quoted 5 lines

These are contradicting the name of the function. The somewhat equivalent function we have on x86 is smp_prepare_boot_cpu().

Quoted 1 line

+ cpumask_copy(&cpu_present_map, &cpu_possible_map);

Another cpumask_set_cpu() is probably cheaper here then.

What do you mean by cheaper here?

Less code to execute to achieve the same effect.

Jan

#8 lore raw

PATCH 3/14 xen/riscv: introduce ioremap()

In reply to Oleksii Kurochko

Based on RISC-V unpriviliged spec ( Version 20240411 ):

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.

[1] https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/

---
 xen/arch/riscv/Kconfig | 12 ++++++++++++
 xen/arch/riscv/pt.c    | 19 +++++++++++++++++++
 2 files changed, 31 insertions(+)
xen/arch/riscv/Kconfig +12 -0
index d882e0a059..27086cca9c 100644
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
 	string
 	default "arch/riscv/configs/tiny64_defconfig"
 
+config HAS_SVPBMT
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.
+
 menu "Architecture Features"
 
 source "arch/Kconfig"
xen/arch/riscv/pt.c +19 -0
index 857619d48d..e2f49e2f97 100644
@@ -7,6 +7,7 @@
 #include <xen/pfn.h>
 #include <xen/pmap.h>
 #include <xen/spinlock.h>
+#include <xen/vmap.h>
 
 #include <asm/fixmap.h>
 #include <asm/flushtlb.h>
@@ -548,3 +549,21 @@ void clear_fixmap(unsigned int map)
                               FIXMAP_ADDR(map) + PAGE_SIZE) != 0 )
         BUG();
 }
+
+void *ioremap(paddr_t pa, size_t len)
+{
+    mfn_t mfn = _mfn(PFN_DOWN(pa));
+    unsigned int offs = pa & (PAGE_SIZE - 1);
+    unsigned int nr = PFN_UP(offs + len);
+
+#ifdef CONFIG_HAS_SVPBMT
+    #error "an introduction of PAGE_HYPERVISOR_IOREMAP is needed for __vmap()"
+#endif
+
+    void *ptr = __vmap(&mfn, nr, 1, 1, PAGE_HYPERVISOR, VMAP_DEFAULT);
+
+    if ( !ptr )
+        return NULL;
+
+    return ptr + offs;
+}
Signature · 1 line
2.49.0
#9 lore raw

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

Quoted 40 lines · Oleksii Kurochko

Based on RISC-V unpriviliged spec ( Version 20240411 ):

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.

Quoted 1 line

[1] https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/

But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...

Quoted 7 lines
xen/arch/riscv/Kconfig +1 -0
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
 	string
 	default "arch/riscv/configs/tiny64_defconfig"
 
+config HAS_SVPBMT

xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28

Quoted 10 lines
xen/arch/riscv/Kconfig +10 -0
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.

... I kind of expect this extension (or anything else that there might be) will need making use of.

Quoted 4 lines
xen/arch/riscv/Kconfig +0 -0
@@ -548,3 +549,21 @@ void clear_fixmap(unsigned int map)
                               FIXMAP_ADDR(map) + PAGE_SIZE) != 0 )
         BUG();
 }

xen/arch/riscv/Kconfig void clear_fixmap(unsigned int map) · line 561

Quoted 10 lines
xen/arch/riscv/Kconfig +10 -0
+
+void *ioremap(paddr_t pa, size_t len)
+{
+    mfn_t mfn = _mfn(PFN_DOWN(pa));
+    unsigned int offs = pa & (PAGE_SIZE - 1);
+    unsigned int nr = PFN_UP(offs + len);
+
+#ifdef CONFIG_HAS_SVPBMT
+    #error "an introduction of PAGE_HYPERVISOR_IOREMAP is needed for __vmap()"
+#endif

While, as per above, I don't think this can stay, just in case: As indicated earlier, pre-processor directives want to have the # in the first column.

Jan

#10 lore raw

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko

Based on RISC-V unpriviliged spec ( Version 20240411 ):

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.

Quoted 1 line

[1]https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/

But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...

Quoted 6 lines
xen/arch/riscv/Kconfig +0 -0
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
  	string
  	default "arch/riscv/configs/tiny64_defconfig"
 
Quoted 13 lines

xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28

Quoted 11 lines
xen/arch/riscv/Kconfig +11 -0
+config HAS_SVPBMT
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.

... I kind of expect this extension (or anything else that there might be) will need making use of.

In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.

In the case of QEMU, I assume it is QEMU's responsibility to properly emulate accesses to device memory regions. Since QEMU does not appear to provide registers for configuring PMA, it seems that PMA is not emulated. Additionally, QEMU does not emulate caches.

Based on that, I expect that it is the responsibility of the firmware or the hardware itself to provide the correct PMA configuration.

I want to note that even Svpbmt is available PMA settings could be used or be ovewritten by Svpbmt's attribute value.

I will update the commit message for more clearness.

Quoted 17 lines
Quoted 5 lines
xen/arch/riscv/Kconfig +0 -0
@@ -548,3 +549,21 @@ void clear_fixmap(unsigned int map)
                                FIXMAP_ADDR(map) + PAGE_SIZE) != 0 )
          BUG();
  }

xen/arch/riscv/Kconfig void clear_fixmap(unsigned int map) · line 561

Quoted 10 lines
xen/arch/riscv/Kconfig +10 -0
+
+void *ioremap(paddr_t pa, size_t len)
+{
+    mfn_t mfn = _mfn(PFN_DOWN(pa));
+    unsigned int offs = pa & (PAGE_SIZE - 1);
+    unsigned int nr = PFN_UP(offs + len);
+
+#ifdef CONFIG_HAS_SVPBMT
+    #error "an introduction of PAGE_HYPERVISOR_IOREMAP is needed for __vmap()"
+#endif

While, as per above, I don't think this can stay, just in case: As indicated earlier, pre-processor directives want to have the # in the first column.

I think it can be safely dropped now, and|PAGE_HYPERVISOR_IOREMAP| could be introduced and simply PTE's PMBT bits can be ignored in|pt_update_entry()| if Svpbmt isn't implemented.

~ Oleksii

#11 lore raw

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

Quoted 69 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko

Based on RISC-V unpriviliged spec ( Version 20240411 ):

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.

Quoted 1 line

[1]https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/

But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...

Quoted 6 lines
xen/arch/riscv/Kconfig +0 -0
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
  	string
  	default "arch/riscv/configs/tiny64_defconfig"
 
Quoted 13 lines

xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28

Quoted 11 lines
xen/arch/riscv/Kconfig +11 -0
+config HAS_SVPBMT
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.

... I kind of expect this extension (or anything else that there might be) will need making use of.

Quoted 4 lines

In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.

How would things work if there was a need to map a RAM page uncacheable (via ioremap() or otherwise)?

Jan

#12 lore raw

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

Quoted 68 lines · Jan Beulich
Quoted 67 lines · Oleksii Kurochko
Quoted 66 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko

Based on RISC-V unpriviliged spec ( Version 20240411 ):

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.

Quoted 1 line

[1]https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/

But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...

Quoted 7 lines
xen/arch/riscv/Kconfig +1 -0
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
   	string
   	default "arch/riscv/configs/tiny64_defconfig"
 
+config HAS_SVPBMT

xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28

Quoted 10 lines
xen/arch/riscv/Kconfig +10 -0
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.

... I kind of expect this extension (or anything else that there might be) will need making use of.

Quoted 6 lines
Quoted 4 lines

In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.

How would things work if there was a need to map a RAM page uncacheable (via ioremap() or otherwise)?

My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.

The question is if non-cacheable RAM page is really needed if we have a coherency?

~ Oleksii

#13 lore raw

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko

Based on RISC-V unpriviliged spec ( Version 20240411 ):

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.

Quoted 1 line

[1]https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/

But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...

Quoted 6 lines
xen/arch/riscv/Kconfig +0 -0
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
   	string
   	default "arch/riscv/configs/tiny64_defconfig"
 
Quoted 13 lines

xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28

Quoted 11 lines
xen/arch/riscv/Kconfig +11 -0
+config HAS_SVPBMT
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.

... I kind of expect this extension (or anything else that there might be) will need making use of.

In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.

Quoted 7 lines
Quoted 2 lines

How would things work if there was a need to map a RAM page uncacheable (via ioremap() or otherwise)?

My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.

The question is if non-cacheable RAM page is really needed if we have a coherency?

Aiui coherency here is among CPUs. Properties of devices in the system are largely unknown? (Beyond this there may also be special situations in which one really cares about data going directly to RAM.)

Jan

#14 lore raw

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

Quoted 74 lines · Jan Beulich
Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko

Based on RISC-V unpriviliged spec ( Version 20240411 ):

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.

Quoted 1 line

[1]https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/

But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...

Quoted 6 lines
xen/arch/riscv/Kconfig +0 -0
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
    	string
    	default "arch/riscv/configs/tiny64_defconfig"
 
Quoted 13 lines

xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28

Quoted 11 lines
xen/arch/riscv/Kconfig +11 -0
+config HAS_SVPBMT
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.

... I kind of expect this extension (or anything else that there might be) will need making use of.

In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.

Quoted 7 lines
Quoted 6 lines
Quoted 2 lines

How would things work if there was a need to map a RAM page uncacheable (via ioremap() or otherwise)?

My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.

The question is if non-cacheable RAM page is really needed if we have a coherency?

Aiui coherency here is among CPUs.

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.
```

Based on this from the spec, coherency here is not only among CPUs.

Quoted 2 lines

Properties of devices in the system are
largely unknown?

Yes, but still not sure what kind of property requires ioremap() which won't work without Svpmbt. Could you please tell me an example?

Quoted 2 lines

(Beyond this there may also be special situations in which one really cares about data going directly to RAM.)

If there are such special cases, I assume that the firmware or hardware (in the case of fixed PMA) will provide a non-cacheable region. In that case, the user should be aware of this region and use it for those specific scenarios.

~ Oleksii

#15 lore raw

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

Quoted 95 lines · Oleksii Kurochko
Quoted 74 lines · Jan Beulich
Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko

Based on RISC-V unpriviliged spec ( Version 20240411 ):

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.

Quoted 1 line

[1]https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/

But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...

Quoted 6 lines
xen/arch/riscv/Kconfig +0 -0
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
    	string
    	default "arch/riscv/configs/tiny64_defconfig"
 
Quoted 13 lines

xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28

Quoted 11 lines
xen/arch/riscv/Kconfig +11 -0
+config HAS_SVPBMT
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.

... I kind of expect this extension (or anything else that there might be) will need making use of.

In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.

Quoted 7 lines
Quoted 6 lines
Quoted 2 lines

How would things work if there was a need to map a RAM page uncacheable (via ioremap() or otherwise)?

My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.

The question is if non-cacheable RAM page is really needed if we have a coherency?

Aiui coherency here is among CPUs.

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.
```
Quoted 8 lines

Based on this from the spec, coherency here is not only among CPUs.

Quoted 2 lines

Properties of devices in the system are
largely unknown?

Yes, but still not sure what kind of property requires ioremap() which won't work without Svpmbt. Could you please tell me an example?

Well, above you said they all need to access memory coherently. That's the "property" I was referring to.

Quoted 5 lines
Quoted 2 lines

(Beyond this there may also be special situations in which one really cares about data going directly to RAM.)

If there are such special cases, I assume that the firmware or hardware (in the case of fixed PMA) will provide a non-cacheable region.

How could they? Firmware may be unaware of specific properties of specific devices a user adds to a system.

Jan

Quoted 5 lines

In that case, the user should be
aware of this region and use it for those specific scenarios.

~ Oleksii

#16 lore raw

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

Quoted 97 lines · Jan Beulich
Quoted 96 lines · Oleksii Kurochko
Quoted 74 lines · Jan Beulich
Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko

Based on RISC-V unpriviliged spec ( Version 20240411 ):

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.

Quoted 1 line

[1]https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/

But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...

Quoted 6 lines
xen/arch/riscv/Kconfig +0 -0
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
     	string
     	default "arch/riscv/configs/tiny64_defconfig"
 
Quoted 13 lines

xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28

Quoted 11 lines
xen/arch/riscv/Kconfig +11 -0
+config HAS_SVPBMT
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.

... I kind of expect this extension (or anything else that there might be) will need making use of.

In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.

Quoted 7 lines
Quoted 6 lines
Quoted 2 lines

How would things work if there was a need to map a RAM page uncacheable (via ioremap() or otherwise)?

My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.

The question is if non-cacheable RAM page is really needed if we have a coherency?

Aiui coherency here is among CPUs.

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.
```

Based on this from the spec, coherency here is not only among CPUs.

Quoted 6 lines
Quoted 4 lines
Quoted 2 lines

Properties of devices in the system are
largely unknown?

Yes, but still not sure what kind of property requires ioremap() which won't work without Svpmbt. Could you please tell me an example?

Well, above you said they all need to access memory coherently. That's the "property" I was referring to.

Do you mean that device could have a property which tell that it would like to have non-cachable region used for that? I haven't seen such property in device tree files.

Do we have in Xen cases when Xen wants to have map part of RAM as non-cachebale and it is only the one option?

I am also thinking why it can't be used cachable region + barrier (if we don't have memory coherency for everything).

Anyway, if it isn't an option to have mapped cacheble region + barrier then there is no any choice and the support of Svpmbt is required.

Quoted 7 lines
Quoted 5 lines
Quoted 3 lines

(Beyond this there may also be special situations in which one really cares about data going directly to RAM.)

If there are such special cases, I assume that the firmware or hardware (in the case of fixed PMA) will provide a non-cacheable region.

How could they? Firmware may be unaware of specific properties of specific devices a user adds to a system.

(this is not real case, just thoughts) Firmware could by default provide part of RAM as non-cacheable region and then hypervisor/kernel use this region for allocation. But I agree that it isn't the best thing to manage that.

~ Oleksii

Quoted 5 lines
Quoted 5 lines

In that case, the user should be
aware of this region and use it for those specific scenarios.

~ Oleksii

#17 lore raw

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

Quoted 103 lines · Oleksii Kurochko
Quoted 101 lines · Jan Beulich
Quoted 100 lines · Oleksii Kurochko
Quoted 74 lines · Jan Beulich
Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko

Based on RISC-V unpriviliged spec ( Version 20240411 ):

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.

Quoted 1 line

[1]https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/

But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...

Quoted 6 lines
xen/arch/riscv/Kconfig +0 -0
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
     	string
     	default "arch/riscv/configs/tiny64_defconfig"
 
Quoted 13 lines

xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28

Quoted 11 lines
xen/arch/riscv/Kconfig +11 -0
+config HAS_SVPBMT
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.

... I kind of expect this extension (or anything else that there might be) will need making use of.

In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.

Quoted 7 lines
Quoted 6 lines
Quoted 2 lines

How would things work if there was a need to map a RAM page uncacheable (via ioremap() or otherwise)?

My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.

The question is if non-cacheable RAM page is really needed if we have a coherency?

Aiui coherency here is among CPUs.

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.
```

Based on this from the spec, coherency here is not only among CPUs.

Quoted 2 lines

Properties of devices in the system are
largely unknown?

Yes, but still not sure what kind of property requires ioremap() which won't work without Svpmbt. Could you please tell me an example?

Quoted 8 lines
Quoted 2 lines

Well, above you said they all need to access memory coherently. That's the "property" I was referring to.

Do you mean that device could have a property which tell that it would like to have non-cachable region used for that? I haven't seen such property in device tree files.

Do we have in Xen cases when Xen wants to have map part of RAM as non-cachebale and it is only the one option?

On x86 we have the case that IOMMUs may access memory non-coherently. This is particular means that IOMMU page table updates (which necessarily live in RAM) need to be done quite carefully. As it's all our code, we deal with the situation by issuing cache flushes, avoiding the need for UC mappings.

Graphics engines may have similar constraints, aiui. With the driver code not being part of Xen, we wouldn't be able to use a similar "simplification" there. UC mappings would be pretty much unavoidable.

Quoted 2 lines

I am also thinking why it can't be used cachable region + barrier (if we don't have memory coherency for everything).

Not sure what exactly you're asking here (if anything). An answer would very likely depend on the specific kind of barrier you're thinking about. The question would be what, if any, effect a barrier would have on the cache(s).

Quoted 2 lines

Anyway, if it isn't an option to have mapped cacheble region + barrier then there is no any choice and the support of Svpmbt is required.

Quite possible.

Jan

#18 lore raw

Re: [PATCH v1 03/14] xen/riscv: introduce ioremap()

Quoted 110 lines · Jan Beulich
Quoted 102 lines · Oleksii Kurochko
Quoted 101 lines · Jan Beulich
Quoted 100 lines · Oleksii Kurochko
Quoted 74 lines · Jan Beulich
Quoted 73 lines · Oleksii Kurochko
Quoted 72 lines · Jan Beulich
Quoted 71 lines · Oleksii Kurochko
Quoted 53 lines · Jan Beulich
Quoted 40 lines · Oleksii Kurochko

Based on RISC-V unpriviliged spec ( Version 20240411 ):

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.

I/O regions in the address space should be considered non-cacheable
regions in the PMAs for those regions. Such regions can be considered coherent
by the PMA if they are not cached by any agent.
```
and [1]:
```
The current riscv linux implementation requires SOC system to support
memory coherence between all I/O devices and CPUs. But some SOC systems
cannot maintain the coherence and they need support cache clean/invalid
operations to synchronize data.

Current implementation is no problem with SiFive FU540, because FU540
keeps all IO devices and DMA master devices coherence with CPU. But to a
traditional SOC vendor, it may already have a stable non-coherency SOC
system, the need is simply to replace the CPU with RV CPU and rebuild
the whole system with IO-coherency is very expensive.
```

and the fact that all known ( to me ) CPUs that support the H-extension and that ones is going to be supported by Xen have memory coherency between all I/O devices and CPUs, so it is currently safe to use the PAGE_HYPERVISOR attribute.
However, in cases where a platform does not support memory coherency, it should support CMO extensions and Svpbmt. In this scenario, updates to ioremap will be necessary.
For now, a compilation error will be generated to ensure that the need to update ioremap() is not overlooked.

Quoted 1 line

[1]https://patchwork.kernel.org/project/linux-riscv/patch/[email protected]/

But MMIO access correctness isn't just a matter of coherency. There may not be any caching involved in most cases, or else you may observe significantly delayed or even dropped (folded with later ones) writes, and reads may be serviced from the cache instead of going to actual MMIO. Therefore ...

Quoted 6 lines
xen/arch/riscv/Kconfig +0 -0
@@ -15,6 +15,18 @@ config ARCH_DEFCONFIG
      	string
      	default "arch/riscv/configs/tiny64_defconfig"
 
Quoted 13 lines

xen/arch/riscv/Kconfig config ARCH_DEFCONFIG · line 28

Quoted 11 lines
xen/arch/riscv/Kconfig +11 -0
+config HAS_SVPBMT
+	bool
+	help
+	  This config enables usage of Svpbmt ISA-extension ( Supervisor-mode:
+	  page-based memory types).
+
+	  The memory type for a page contains a combination of attributes
+	  that indicate the cacheability, idempotency, and ordering
+	  properties for access to that page.
+
+	  The Svpbmt extension is only available on 64-bit cpus.

... I kind of expect this extension (or anything else that there might be) will need making use of.

In cases where the Svpbmt extension isn't available, PMA (Physical Memory Attributes) is used to control which memory regions are cacheable, non-cacheable, readable, writable, etc. PMA is configured in M-mode by the firmware (e.g., OpenSBI), as is done in Andes cores, or it can be fixed at design time, as in SiFive cores.

Quoted 7 lines
Quoted 6 lines
Quoted 2 lines

How would things work if there was a need to map a RAM page uncacheable (via ioremap() or otherwise)?

My understanding is that Svpbmt is only needed when someone wants to change the memory attribute of a page set by PMA.

The question is if non-cacheable RAM page is really needed if we have a coherency?

Aiui coherency here is among CPUs.

```
For implementations that conform to the RISC-V Unix Platform Specification,
I/O devices and DMA operations are required to access memory coherently and
via strongly ordered I/O channels. Therefore, accesses to regular main memory
regions that are concurrently accessed by external devices can also use the
standard synchronization mechanisms. Implementations that do not conform
to the Unix Platform Specification and/or in which devices do not access
memory coherently will need to use mechanisms
(which are currently platform-specific or device-specific) to enforce
coherency.
```

Based on this from the spec, coherency here is not only among CPUs.

Quoted 2 lines

Properties of devices in the system are
largely unknown?

Yes, but still not sure what kind of property requires ioremap() which won't work without Svpmbt. Could you please tell me an example?

Quoted 7 lines
Quoted 2 lines

Well, above you said they all need to access memory coherently. That's the "property" I was referring to.

Do you mean that device could have a property which tell that it would like to have non-cachable region used for that? I haven't seen such property in device tree files.

Do we have in Xen cases when Xen wants to have map part of RAM as non-cachebale and it is only the one option?

Quoted 8 lines

On x86 we have the case that IOMMUs may access memory non-coherently. This is particular means that IOMMU page table updates (which necessarily live in RAM) need to be done quite carefully. As it's all our code, we deal with the situation by issuing cache flushes, avoiding the need for UC mappings.

Graphics engines may have similar constraints, aiui. With the driver code not being part of Xen, we wouldn't be able to use a similar "simplification" there. UC mappings would be pretty much unavoidable.

For this case, it would be better to have Svpmbt.

I would like to noted that Svpmbt isn't supported by RV32 architectures. For such cases, it will be still needed to play with PMA.
I found today a patch in Linux kernel which does something similar to what I wrote in one of my previous replies:
[0]https://lore.kernel.org/all/[email protected]/
In the cover letter [0] it is mentioned the following:
On some RISC-V platforms, including StarFive JH7100 and ESWIN EIC7700,
RAM is mapped to multiple physical address ranges, with each alias
having a different set of statically-determined Physical Memory
Attributes (PMAs). Software selects the PMAs for a page by choosing a
PFN from the corresponding physical address range. On these platforms,
this is the only way to allocate noncached memory for use with
noncoherent DMA.

So firmware should configure PMAs so some part of RAM is noncached and then kernel get this info based on the binding:
https://patchew.org/linux/[email protected]/[email protected]/

Considering that this feature isn't available even in Linux kernel, we can start with assumption that all our SoCs will support Svpmbt.

We don't really care about StarFive JH7100 as it doesn't support H extension, but we potentially should care about ESWIN EIC7700, which support H extension and doesn't support Svpmbt extension according to a datasheet publicly available:
Each EIC7700X core is configured to support the RV64I base ISA, as well as the Multiply (M), Atomic(A), Single-Precision Floating Point (F), Double-Precision Floating Point (D), Compressed (C), CSR Instructions (Zicsr), Instruction-Fetch Fence (Zifencei), Address Calculation (Zba), Basic Bit Manipulation (Zbb), and Count Overflow and Mode-Based Filtering (Sscofpmf) RISC‑V extensions. This is captured by the RISC‑V extension string: RV64GC_Zba_Zbb_Sscofpmf.

Quoted 6 lines
Quoted 3 lines

I am also thinking why it can't be used cachable region + barrier (if we don't have memory coherency for everything).

Not sure what exactly you're asking here (if anything). An answer would very likely depend on the specific kind of barrier you're thinking about. The question would be what, if any, effect a barrier would have on the cache(s).

I confused barrier with cache flushes (when I wrote that I thought about the case of DMA that we don't really should have requirement of non-cachable memory for DMA as it is enough to have memory fence between use of DMA memory and MMIO that triggers the dma), basically I meant what you wrote above about x86's IOMMUs.

~ Oleksii

Quoted 6 lines
Quoted 3 lines

Anyway, if it isn't an option to have mapped cacheble region + barrier then there is no any choice and the support of Svpmbt is required.

Quite possible.

Jan

#19 lore raw

PATCH 4/14 xen/riscv: introduce init_IRQ()

In reply to Oleksii Kurochko

Implement init_IRQ() to initalize various IRQs.

Currently, this function initializes the irq_desc[] array, which stores IRQ descriptors containing various information about each IRQ, such as the type of hardware handling, whether the IRQ is disabled, etc.
The initialization is basic at this point and includes setting IRQ_TYPE_INVALID as the IRQ type, assigning the IRQ number ( which is just a consequent index of irq_desc[] array ) to desc->irq, and setting desc->action to NULL.

Additionally, the function init_irq_data() is introduced to initialize the IRQ descriptors for all IRQs in the system.

Also, define IRQ_TYPE_* which are the same as the existing device tree definitions for convenience.

---
 xen/arch/riscv/Makefile          |  1 +
 xen/arch/riscv/include/asm/irq.h | 24 +++++++++++++++++
 xen/arch/riscv/irq.c             | 44 ++++++++++++++++++++++++++++++++
 xen/arch/riscv/setup.c           |  3 +++
 xen/arch/riscv/stubs.c           |  5 ----
 5 files changed, 72 insertions(+), 5 deletions(-)
 create mode 100644 xen/arch/riscv/irq.c
xen/arch/riscv/Makefile +1 -0
index f551bf32a2..457e8e88a4 100644
@@ -3,6 +3,7 @@ obj-y += cpufeature.o
 obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
 obj-y += entry.o
 obj-y += intc.o
+obj-y += irq.o
 obj-y += mm.o
 obj-y += pt.o
 obj-$(CONFIG_RISCV_64) += riscv64/
xen/arch/riscv/include/asm/irq.h +24 -0
index 2a48da2651..8f936b7d01 100644
@@ -3,6 +3,28 @@
 #define ASM__RISCV__IRQ_H
 
 #include <xen/bug.h>
+#include <xen/device_tree.h>
+
+#define NR_IRQS 1024
+
+/*
+ * TODO: Should IRQ_TYPE_* be moved to xen/irq.h and wrapped into
+ * #ifdef CONFIG_HAS_DEVICE_TREE?
+ */
+/*
+ * These defines correspond to the Xen internal representation of the
+ * IRQ types. We choose to make them the same as the existing device
+ * tree definitions for convenience.
+ */
+#define IRQ_TYPE_NONE           DT_IRQ_TYPE_NONE
+#define IRQ_TYPE_EDGE_RISING    DT_IRQ_TYPE_EDGE_RISING
+#define IRQ_TYPE_EDGE_FALLING   DT_IRQ_TYPE_EDGE_FALLING
+#define IRQ_TYPE_EDGE_BOTH      DT_IRQ_TYPE_EDGE_BOTH
+#define IRQ_TYPE_LEVEL_HIGH     DT_IRQ_TYPE_LEVEL_HIGH
+#define IRQ_TYPE_LEVEL_LOW      DT_IRQ_TYPE_LEVEL_LOW
+#define IRQ_TYPE_LEVEL_MASK     DT_IRQ_TYPE_LEVEL_MASK
+#define IRQ_TYPE_SENSE_MASK     DT_IRQ_TYPE_SENSE_MASK
+#define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
 
 /* TODO */
 #define nr_irqs 0U
@@ -25,6 +47,8 @@ static inline void arch_move_irqs(struct vcpu *v)
     BUG_ON("unimplemented");
 }
 
+void init_IRQ(void);
+
 #endif /* ASM__RISCV__IRQ_H */
 
 /*
xen/arch/riscv/irq.c new file +44 -0

44 lines not on this page · raw message

xen/arch/riscv/setup.c +3 -0
index 7f68f3f5b7..a3189697da 100644
@@ -6,6 +6,7 @@
 #include <xen/compile.h>
 #include <xen/device_tree.h>
 #include <xen/init.h>
+#include <xen/irq.h>
 #include <xen/mm.h>
 #include <xen/shutdown.h>
 #include <xen/vmap.h>
@@ -127,6 +128,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
         panic("Booting using ACPI isn't supported\n");
     }
 
+    init_IRQ();
+
     riscv_fill_hwcap();
 
     preinit_xen_time();
xen/arch/riscv/stubs.c +0 -5
index fdcf91054e..e396b67cd3 100644
@@ -107,11 +107,6 @@ void irq_ack_none(struct irq_desc *desc)
     BUG_ON("unimplemented");
 }
 
-int arch_init_one_irq_desc(struct irq_desc *desc)
-{
-    BUG_ON("unimplemented");
-}
-
 void smp_send_state_dump(unsigned int cpu)
 {
     BUG_ON("unimplemented");
Signature · 1 line
2.49.0
#20 lore raw

Re: [PATCH v1 04/14] xen/riscv: introduce init_IRQ()

Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/irq.h +0 -0
@@ -3,6 +3,28 @@
 #define ASM__RISCV__IRQ_H

xen/arch/riscv/include/asm/irq.h line 13

Quoted 10 lines
xen/arch/riscv/include/asm/irq.h +8 -0
 
 #include <xen/bug.h>
+#include <xen/device_tree.h>
+
+#define NR_IRQS 1024
+
+/*
+ * TODO: Should IRQ_TYPE_* be moved to xen/irq.h and wrapped into
+ * #ifdef CONFIG_HAS_DEVICE_TREE?
+ */

Wouldn't that be more like asm-generic/dt-irq.h (or irq-dt.h)? The field where these values are stored is an arch-specific one, after all.

Quoted 18 lines
xen/arch/riscv/irq.c new file +15 -0
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+/*
+ * RISC-V Trap handlers
+ *
+ * Copyright (c) 2024 Vates
+ */
+
+#include <xen/bug.h>
+#include <xen/init.h>
+#include <xen/irq.h>
+
+static irq_desc_t irq_desc[NR_IRQS];
+
+int arch_init_one_irq_desc(struct irq_desc *desc)

xen/arch/riscv/irq.c line 25

Quoted 10 lines
xen/arch/riscv/irq.c new file +10 -0
+{
+    desc->arch.type = IRQ_TYPE_INVALID;
+    return 0;
+}
+
+static int __init init_irq_data(void)
+{
+    int irq;
+
+    for ( irq = 0; irq < NR_IRQS; irq++ )

For this the variable would better be of unsigned type. I realize though that ...

xen/arch/riscv/irq.c line 33

Quoted 8 lines
xen/arch/riscv/irq.c new file +8 -0
+    {
+        struct irq_desc *desc = irq_to_desc(irq);
+        int rc = init_one_irq_desc(desc);
+
+        if ( rc )
+            return rc;
+
+        desc->irq = irq;

... it's a plain int field that we're storing into here. I don't think I can spot any place where a negative value would be stored into there.

Jan

#21 lore raw

Re: [PATCH v1 04/14] xen/riscv: introduce init_IRQ()

Quoted 6 lines · Jan Beulich
Quoted 5 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/irq.h +0 -0
@@ -3,6 +3,28 @@
  #define ASM__RISCV__IRQ_H
 
Quoted 11 lines

xen/arch/riscv/include/asm/irq.h line 13

Quoted 9 lines
xen/arch/riscv/include/asm/irq.h +8 -0
  #include <xen/bug.h>
+#include <xen/device_tree.h>
+
+#define NR_IRQS 1024
+
+/*
+ * TODO: Should IRQ_TYPE_* be moved to xen/irq.h and wrapped into
+ * #ifdef CONFIG_HAS_DEVICE_TREE?
+ */

Wouldn't that be more like asm-generic/dt-irq.h (or irq-dt.h)? The field where these values are stored is an arch-specific one, after all.

It would be much better. Thanks!

~ Oleksii

#22 lore raw

PATCH 5/14 xen/riscv: introduce platform_get_irq()

In reply to Oleksii Kurochko

platform_get_irq() recieves information about device's irq ( type and irq number ) from device tree node and using this information update irq descriptor in irq_desc[] array.

Introduce dt_irq_xlate and initialize with aplic_irq_xlate() as it is used by dt_device_get_irq() which is called by platform_get_irq().

---
 xen/arch/riscv/aplic.c           | 19 +++++++++++++++
 xen/arch/riscv/include/asm/irq.h |  3 +++
 xen/arch/riscv/irq.c             | 41 ++++++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+)
xen/arch/riscv/aplic.c +19 -0
index caba8f8993..6dc040af6f 100644
@@ -11,6 +11,7 @@
 
 #include <xen/errno.h>
 #include <xen/init.h>
+#include <xen/irq.h>
 #include <xen/sections.h>
 #include <xen/types.h>
 
@@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = {
     .hw_version = INTC_APLIC,
 };
 
+static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
+                           unsigned int *out_hwirq,
+                           unsigned int *out_type)
+{
+    if ( intsize < 2 )
+        return -EINVAL;
+
+    /* Mapping 1:1 */
+    *out_hwirq = intspec[0];
+
+    if ( out_type )
+        *out_type = intspec[1] & IRQ_TYPE_SENSE_MASK;
+
+    return 0;
+}
+
 static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 {
     if ( aplic_info.node )
@@ -35,6 +52,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 
     aplic_info.node = node;
 
+    dt_irq_xlate = aplic_irq_xlate;
+
     return 0;
 }
 
xen/arch/riscv/include/asm/irq.h +3 -0
index 8f936b7d01..ff1c95e0be 100644
@@ -47,6 +47,9 @@ static inline void arch_move_irqs(struct vcpu *v)
     BUG_ON("unimplemented");
 }
 
+struct dt_device_node;
+int platform_get_irq(const struct dt_device_node *device, int index);
+
 void init_IRQ(void);
 
 #endif /* ASM__RISCV__IRQ_H */
xen/arch/riscv/irq.c +41 -0

52 lines not on this page · raw message

Signature · 1 line
2.49.0
#23 lore raw

Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()

xen/arch/riscv/aplic.c static struct intc_info __ro_after_init aplic_info = { · line 25

Quoted 5 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +1 -0
@@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = {
     .hw_version = INTC_APLIC,
 };
 
+static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,

As you start adding functions calling indirectly, please consider adding cf_check right away, even if right now this has no effect on RISC-V. That'll save you from going through the entire RISC-V subtree later on to find them all.

Quoted 30 lines
xen/arch/riscv/aplic.c +17 -0
+                           unsigned int *out_hwirq,
+                           unsigned int *out_type)
+{
+    if ( intsize < 2 )
+        return -EINVAL;
+
+    /* Mapping 1:1 */
+    *out_hwirq = intspec[0];
+
+    if ( out_type )
+        *out_type = intspec[1] & IRQ_TYPE_SENSE_MASK;
+
+    return 0;
+}
+
 static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 {
     if ( aplic_info.node )
@@ -35,6 +52,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 
     aplic_info.node = node;
 
+    dt_irq_xlate = aplic_irq_xlate;
+
     return 0;
 }
 

xen/arch/riscv/include/asm/irq.h static inline void arch_move_irqs(struct vcpu *v) · line 51

Quoted 5 lines
xen/arch/riscv/include/asm/irq.h +2 -0
@@ -47,6 +47,9 @@ static inline void arch_move_irqs(struct vcpu *v)
     BUG_ON("unimplemented");
 }
 
+struct dt_device_node;
+int platform_get_irq(const struct dt_device_node *device, int index);

And I assume callers of this will appear later in the series.

Quoted 24 lines
xen/arch/riscv/irq.c +13 -0
@@ -7,11 +7,52 @@
  */
 
 #include <xen/bug.h>
+#include <xen/device_tree.h>
+#include <xen/errno.h>
 #include <xen/init.h>
 #include <xen/irq.h>
 
 static irq_desc_t irq_desc[NR_IRQS];
 
+static bool irq_validate_new_type(unsigned int curr, unsigned int new)
+{
+    return (curr == IRQ_TYPE_INVALID || curr == new );
+}
+
+static int irq_set_type(unsigned int irq, unsigned int type)
+{
+    unsigned long flags;
+    struct irq_desc *desc = irq_to_desc(irq);
+    int ret = -EBUSY;
+

xen/arch/riscv/irq.c line 37

Quoted 10 lines
xen/arch/riscv/irq.c +10 -0
+    spin_lock_irqsave(&desc->lock, flags);
+
+    if ( !irq_validate_new_type(desc->arch.type, type) )
+        goto err;
+
+    desc->arch.type = type;
+
+    ret = 0;
+
+err:

Labels indented by at least one blank please.

Quoted 4 lines
xen/arch/riscv/irq.c +4 -0
+    spin_unlock_irqrestore(&desc->lock, flags);
+
+    return ret;
+}

xen/arch/riscv/irq.c line 51

Quoted 10 lines
xen/arch/riscv/irq.c +10 -0
+
+int platform_get_irq(const struct dt_device_node *device, int index)
+{
+    struct dt_irq dt_irq;
+
+    if ( dt_device_get_irq(device, index, &dt_irq) )
+        return -1;
+
+    if ( irq_set_type(dt_irq.irq, dt_irq.type) )
+        return -1;

Can you please return proper -E... values, perhaps ones coming back from the functions called?

Jan

#24 lore raw

Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()

Quoted 9 lines · Jan Beulich

xen/arch/riscv/aplic.c static struct intc_info __ro_after_init aplic_info = { · line 25

Quoted 5 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +1 -0
@@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = {
      .hw_version = INTC_APLIC,
  };
 
+static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,

As you start adding functions calling indirectly, please consider adding cf_check right away, even if right now this has no effect on RISC-V. That'll save you from going through the entire RISC-V subtree later on to find them all.

Sure. I thought that it is a feature for x86 as I haven't seen such attribute for Arm and RISC-V in GCC manuals.

Quoted 34 lines
Quoted 34 lines
xen/arch/riscv/aplic.c +17 -0
+                           unsigned int *out_hwirq,
+                           unsigned int *out_type)
+{
+    if ( intsize < 2 )
+        return -EINVAL;
+
+    /* Mapping 1:1 */
+    *out_hwirq = intspec[0];
+
+    if ( out_type )
+        *out_type = intspec[1] & IRQ_TYPE_SENSE_MASK;
+
+    return 0;
+}
+
  static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
  {
      if ( aplic_info.node )
@@ -35,6 +52,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 
      aplic_info.node = node;
 
+    dt_irq_xlate = aplic_irq_xlate;
+
      return 0;
  }
 
xen/arch/riscv/include/asm/irq.h +0 -0
@@ -47,6 +47,9 @@ static inline void arch_move_irqs(struct vcpu *v)
      BUG_ON("unimplemented");
  }
 
Quoted 3 lines

xen/arch/riscv/include/asm/irq.h static inline void arch_move_irqs(struct vcpu *v) · line 51

Quoted 2 lines
xen/arch/riscv/include/asm/irq.h +2 -0
+struct dt_device_node;
+int platform_get_irq(const struct dt_device_node *device, int index);

And I assume callers of this will appear later in the series.

Yes, it will be called ns16550_uart_dt_init() when CONFIG_NS16550 will be enabled for RISC-V.

Quoted 37 lines
Quoted 25 lines
xen/arch/riscv/irq.c +13 -0
@@ -7,11 +7,52 @@
   */
 
  #include <xen/bug.h>
+#include <xen/device_tree.h>
+#include <xen/errno.h>
  #include <xen/init.h>
  #include <xen/irq.h>
 
  static irq_desc_t irq_desc[NR_IRQS];
 
+static bool irq_validate_new_type(unsigned int curr, unsigned int new)
+{
+    return (curr == IRQ_TYPE_INVALID || curr == new );
+}
+
+static int irq_set_type(unsigned int irq, unsigned int type)
+{
+    unsigned long flags;
+    struct irq_desc *desc = irq_to_desc(irq);
+    int ret = -EBUSY;
+

xen/arch/riscv/irq.c line 37

Quoted 10 lines
xen/arch/riscv/irq.c +10 -0
+    spin_lock_irqsave(&desc->lock, flags);
+
+    if ( !irq_validate_new_type(desc->arch.type, type) )
+        goto err;
+
+    desc->arch.type = type;
+
+    ret = 0;
+
+err:

Labels indented by at least one blank please.

Quoted 16 lines
Quoted 4 lines
xen/arch/riscv/irq.c +4 -0
+    spin_unlock_irqrestore(&desc->lock, flags);
+
+    return ret;
+}

xen/arch/riscv/irq.c line 51

Quoted 10 lines
xen/arch/riscv/irq.c +10 -0
+
+int platform_get_irq(const struct dt_device_node *device, int index)
+{
+    struct dt_irq dt_irq;
+
+    if ( dt_device_get_irq(device, index, &dt_irq) )
+        return -1;
+
+    if ( irq_set_type(dt_irq.irq, dt_irq.type) )
+        return -1;

Can you please return proper -E... values, perhaps ones coming back from the functions called?

Sure, I will use -EINVAL. (or ,perhaps, it will be better to introduce ret and return what dt_device_get_irq()/irq_set_type() returns in the case of failure.

~ Oleksii

#25 lore raw

Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()

Quoted 6 lines · Oleksii Kurochko
Quoted 5 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +0 -0
@@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = {
      .hw_version = INTC_APLIC,
  };
 
Quoted 7 lines
Quoted 4 lines

xen/arch/riscv/aplic.c static struct intc_info __ro_after_init aplic_info = { · line 25

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,

As you start adding functions calling indirectly, please consider adding cf_check right away, even if right now this has no effect on RISC-V. That'll save you from going through the entire RISC-V subtree later on to find them all.

Sure. I thought that it is a feature for x86 as I haven't seen such attribute for Arm and RISC-V in GCC manuals.

And that looks to be correct. I was under the (admittedly vague) impression Arm64 had something equivalent in hardware, which then merely needs enabling in the compiler. Not sure about RISC-V, but seeing the endless flow of patches enabling new extensions in binutils, it would perhaps even be surprising if nothing along these lines was already in the works somewhere.

Jan

#26 lore raw

Re: [PATCH v1 05/14] xen/riscv: introduce platform_get_irq()

Quoted 11 lines · Jan Beulich
Quoted 10 lines · Oleksii Kurochko
Quoted 9 lines · Jan Beulich

xen/arch/riscv/aplic.c static struct intc_info __ro_after_init aplic_info = { · line 25

Quoted 5 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +1 -0
@@ -21,6 +22,22 @@ static struct intc_info __ro_after_init aplic_info = {
       .hw_version = INTC_APLIC,
   };
 
+static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,

As you start adding functions calling indirectly, please consider adding cf_check right away, even if right now this has no effect on RISC-V. That'll save you from going through the entire RISC-V subtree later on to find them all.

Quoted 7 lines
Quoted 2 lines

Sure. I thought that it is a feature for x86 as I haven't seen such attribute for Arm and RISC-V in GCC manuals.

And that looks to be correct. I was under the (admittedly vague) impression Arm64 had something equivalent in hardware, which then merely needs enabling in the compiler. Not sure about RISC-V, but seeing the endless flow of patches enabling new extensions in binutils, it would perhaps even be surprising if nothing along these lines was already in the works somewhere.

You are right, something is already in the work:
-https://github.com/riscv/riscv-cfi -https://lore.kernel.org/lkml/[email protected]/ (interesting that
they are enabling it for U-mode)

~ Oleksii

#27 lore raw

PATCH 6/14 xen/riscv: riscv_of_processor_hartid() implementation

In reply to Oleksii Kurochko

Implements riscv_of_processor_hartid() to get the hart ID of the given device tree node and do some checks if CPU is available and given device tree node has proper riscv,isa property.

As a helper function of_get_cpu_hwid() is introduced to deal specifically with reg propery of a CPU device node.

---
 xen/arch/riscv/include/asm/smp.h |  3 ++
 xen/arch/riscv/smpboot.c         | 68 ++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
xen/arch/riscv/include/asm/smp.h +3 -0
index 188c033718..9b68f1e27a 100644
@@ -26,6 +26,9 @@ static inline void set_cpuid_to_hartid(unsigned long cpuid,
 
 void setup_tp(unsigned int cpuid);
 
+struct dt_device_node;
+int riscv_of_processor_hartid(struct dt_device_node *node, unsigned long *hart);
+
 void smp_clear_cpu_maps(void);
 
 #endif
xen/arch/riscv/smpboot.c +68 -0

76 lines not on this page · raw message

Signature · 1 line
2.49.0
#28 lore raw

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +0 -0
@@ -13,3 +16,68 @@ void __init smp_clear_cpu_maps(void)
     cpumask_set_cpu(0, &cpu_online_map);
     cpumask_copy(&cpu_present_map, &cpu_possible_map);
 }

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 28

Quoted 10 lines
xen/arch/riscv/smpboot.c +10 -0
+
+/**
+ * of_get_cpu_hwid - Get the hardware ID from a CPU device node
+ *
+ * @cpun: CPU number(logical index) for which device node is required
+ * @thread: The local thread number to get the hardware ID for.
+ *
+ * Return: The hardware ID for the CPU node or ~0ULL if not found.
+ */
+static uint64_t of_get_cpu_hwid(struct dt_device_node *cpun, unsigned int thread)

What does the "of" prefix stand for here? Looking at the function body I'm really at a loss. (I was first guessing something like OpenFirmware, but there's nothing here that would support that.)

As you're only fetching data - can cpun be pointer-to-const?

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37

Quoted 9 lines
xen/arch/riscv/smpboot.c +9 -0
+{
+    const __be32 *cell;
+    int ac;
+    uint32_t len;
+
+    ac = dt_n_addr_cells(cpun);
+    cell = dt_get_property(cpun, "reg", &len);
+    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
+        return ~0ULL;

I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 40

Quoted 2 lines
xen/arch/riscv/smpboot.c +2 -0
+    cell += ac * thread;
+    return dt_read_number(cell, ac);

Nit (you know what)

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 47

Quoted 5 lines
xen/arch/riscv/smpboot.c +5 -0
+/*
+ * Returns the hart ID of the given device tree node, or -ENODEV if the node
+ * isn't an enabled and valid RISC-V hart node.
+ */
+int riscv_of_processor_hartid(struct dt_device_node *node, unsigned long *hart)

Similar question as above: What's "of" and what significance does the "riscv" prefix have in RISC-V code?

Const-ness question again for "node".

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 58

Quoted 11 lines
xen/arch/riscv/smpboot.c +11 -0
+{
+    const char *isa;
+
+    if ( !dt_device_is_compatible(node, "riscv") )
+    {
+        printk("Found incompatible CPU\n");
+        return -ENODEV;
+    }
+
+    *hart = (unsigned long) of_get_cpu_hwid(node, 0);
+    if ( *hart == ~0UL )

While for RV64 this won't matter, the difference in types (uint64_t returned, unsigned long used) is still puzzling me. What's the deal?

Jan

#29 lore raw

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

Quoted 10 lines · Jan Beulich
Quoted 9 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +5 -0
@@ -13,3 +16,68 @@ void __init smp_clear_cpu_maps(void)
      cpumask_set_cpu(0, &cpu_online_map);
      cpumask_copy(&cpu_present_map, &cpu_possible_map);
  }
+
+/**
+ * of_get_cpu_hwid - Get the hardware ID from a CPU device node
+ *
+ * @cpun: CPU number(logical index) for which device node is required
Quoted 8 lines

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 28

Quoted 5 lines
xen/arch/riscv/smpboot.c +5 -0
+ * @thread: The local thread number to get the hardware ID for.
+ *
+ * Return: The hardware ID for the CPU node or ~0ULL if not found.
+ */
+static uint64_t of_get_cpu_hwid(struct dt_device_node *cpun, unsigned int thread)

What does the "of" prefix stand for here? Looking at the function body I'm really at a loss. (I was first guessing something like OpenFirmware, but there's nothing here that would support that.)

I copy this function from Linux kernel. But you are right, "of" means OpenFirmware or Open Firmware Device Tree and is used for the functions which work with device tree.
I'll rename to dt_get_cpu_hwid() to follow the naming of device tree's functions name in Xen.

Quoted 2 lines

As you're only fetching data - can cpun be pointer-to-const?

Sure, it can be. I'll update that.

Quoted 13 lines

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37

Quoted 10 lines
xen/arch/riscv/smpboot.c +9 -0
+{
+    const __be32 *cell;
+    int ac;
+    uint32_t len;
+
+    ac = dt_n_addr_cells(cpun);
+    cell = dt_get_property(cpun, "reg", &len);
+    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
+        return ~0ULL;

I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).

Based on the DT specification alone, the|reg| value could refer to either a CPU or a thread ID:

```
The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
the CPU/threads represented by the CPU node. If a CPU supports more than one thread
(i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
per thread.
```

My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).

Interestingly, the Linux kernel always uses|thread = 0|.

We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|reg| property contains only one entry, representing the HART (CPU) ID:

```
  Software can determine the number of threads by dividing the size of reg by the parent
  node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
  is required.
```

Does that approach make sense, or should we stick with the current implementation?

Quoted 5 lines

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 40

Quoted 3 lines
xen/arch/riscv/smpboot.c +2 -0
+    cell += ac * thread;
+    return dt_read_number(cell, ac);

Nit (you know what)

Quoted 7 lines

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 47

Quoted 5 lines
xen/arch/riscv/smpboot.c +5 -0
+/*
+ * Returns the hart ID of the given device tree node, or -ENODEV if the node
+ * isn't an enabled and valid RISC-V hart node.
+ */
+int riscv_of_processor_hartid(struct dt_device_node *node, unsigned long *hart)

Similar question as above: What's "of" and what significance does the "riscv" prefix have in RISC-V code?

I will drop usage of 'of' in Xen and change it to 'dt'.

Quoted 16 lines

Const-ness question again for "node".

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 58

Quoted 11 lines
xen/arch/riscv/smpboot.c +11 -0
+{
+    const char *isa;
+
+    if ( !dt_device_is_compatible(node, "riscv") )
+    {
+        printk("Found incompatible CPU\n");
+        return -ENODEV;
+    }
+
+    *hart = (unsigned long) of_get_cpu_hwid(node, 0);
+    if ( *hart == ~0UL )

While for RV64 this won't matter, the difference in types (uint64_t returned, unsigned long used) is still puzzling me. What's the deal?

No specific reason, just overlooked that moment. I think we could use just drop this cast.
The reason for uint64_t as a return type is that dt_read_number() returns u64.

~ Oleksii

#30 lore raw

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

Quoted 32 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37

Quoted 9 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +9 -0
+{
+    const __be32 *cell;
+    int ac;
+    uint32_t len;
+
+    ac = dt_n_addr_cells(cpun);
+    cell = dt_get_property(cpun, "reg", &len);
+    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
+        return ~0ULL;

I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).

Based on the DT specification alone, the|reg| value could refer to either a CPU or a thread ID:

```
The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
the CPU/threads represented by the CPU node. If a CPU supports more than one thread
(i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
per thread.
```

My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).

Interestingly, the Linux kernel always uses|thread = 0|.

We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|reg| property contains only one entry, representing the HART (CPU) ID:

Quoted 7 lines
```
   Software can determine the number of threads by dividing the size of reg by the parent
   node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
   is required.
```

Does that approach make sense, or should we stick with the current implementation?

If extra enabling is required to make multi-thread CPUs work, then panic()ing (not so much ASSERT()ing) may make sense, for the time being. Better would be if we could use all threads in a system right away.

Jan

#31 lore raw

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

Quoted 38 lines · Jan Beulich
Quoted 37 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37

Quoted 9 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +9 -0
+{
+    const __be32 *cell;
+    int ac;
+    uint32_t len;
+
+    ac = dt_n_addr_cells(cpun);
+    cell = dt_get_property(cpun, "reg", &len);
+    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
+        return ~0ULL;

I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).

Based on the DT specification alone, the|reg| value could refer to either a CPU or a thread ID:

```
The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
the CPU/threads represented by the CPU node. If a CPU supports more than one thread
(i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
per thread.
```

My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).

Interestingly, the Linux kernel always uses|thread = 0|.

We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|reg| property contains only one entry, representing the HART (CPU) ID:

```
    Software can determine the number of threads by dividing the size of reg by the parent
    node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
    is required.
```
Quoted 4 lines
Quoted 1 line

Does that approach make sense, or should we stick with the current implementation?

If extra enabling is required to make multi-thread CPUs work, then panic()ing (not so much ASSERT()ing) may make sense, for the time being. Better would be if we could use all threads in a system right away.

Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id by passing thread argument (thread -> the local thread number to get the hardware ID for). So by calling:
dt_get_cpu_hwid(cpu0, 0) -> it will return hardware id of thread 0 of cpu0 dt_get_cpu_hwid(cpu0, 1) -> it will return hardware id of thread 1 of cpu0
...

In our case we assume that SMP isn't supported so that is why it is used only dt_get_cpu_hwid(cpu0, 0).

If one day, SMP will be enabled then it will be needed to change a callers of dt_get_cpu_hwid().

I will add a check in the caller.

~ Oleksii

#32 lore raw

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

Quoted 25 lines · Oleksii Kurochko
Quoted 23 lines · Jan Beulich
Quoted 22 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37

Quoted 9 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +9 -0
+{
+    const __be32 *cell;
+    int ac;
+    uint32_t len;
+
+    ac = dt_n_addr_cells(cpun);
+    cell = dt_get_property(cpun, "reg", &len);
+    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
+        return ~0ULL;

I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).

Based on the DT specification alone, the|reg| value could refer to either a CPU or a thread ID:

```
The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
the CPU/threads represented by the CPU node. If a CPU supports more than one thread
(i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
per thread.
```
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).

Note the terminology ("CPU") you used here.

Quoted 22 lines
Quoted 10 lines
Quoted 10 lines

Interestingly, the Linux kernel always uses|thread = 0|.

We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|reg| property contains only one entry, representing the HART (CPU) ID:

```
    Software can determine the number of threads by dividing the size of reg by the parent
    node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
    is required.
```
Quoted 4 lines
Quoted 1 line

Does that approach make sense, or should we stick with the current implementation?

If extra enabling is required to make multi-thread CPUs work, then panic()ing (not so much ASSERT()ing) may make sense, for the time being. Better would be if we could use all threads in a system right away.

Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id by passing thread argument (thread -> the local thread number to get the hardware ID for). So by calling:
dt_get_cpu_hwid(cpu0, 0) -> it will return hardware id of thread 0 of cpu0 dt_get_cpu_hwid(cpu0, 1) -> it will return hardware id of thread 1 of cpu0
...

Quoted 3 lines

In our case we assume that SMP isn't supported so that is why it is used only dt_get_cpu_hwid(cpu0, 0).

If one day, SMP will be enabled then it will be needed to change a callers of dt_get_cpu_hwid().

I assume you meant SMT in both places you wrote SMP? But my main point here is:
If enumeration gives you "thread <N> of core <M>" (using x86 terminology), you need to be quite careful with what you call "CPU". Things need to be entirely unambiguous, taking into account what internally in (common code) Xen we call a "CPU". You certainly may call "CPU" what is a collection of threads / harts, but you then need to clarify this in a prominent comment somewhere, and you need to be entirely consistent throughout the RISC-V sub-tree.

Jan

#33 lore raw

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

Quoted 52 lines · Jan Beulich
Quoted 24 lines · Oleksii Kurochko
Quoted 23 lines · Jan Beulich
Quoted 22 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich

xen/arch/riscv/smpboot.c void __init smp_clear_cpu_maps(void) · line 37

Quoted 9 lines · Oleksii Kurochko
xen/arch/riscv/smpboot.c +9 -0
+{
+    const __be32 *cell;
+    int ac;
+    uint32_t len;
+
+    ac = dt_n_addr_cells(cpun);
+    cell = dt_get_property(cpun, "reg", &len);
+    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
+        return ~0ULL;

I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).

Based on the DT specification alone, the|reg| value could refer to either a CPU or a thread ID:

```
The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
the CPU/threads represented by the CPU node. If a CPU supports more than one thread
(i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
per thread.
```
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).

Note the terminology ("CPU") you used here.

Quoted 21 lines
Quoted 10 lines
Quoted 10 lines

Interestingly, the Linux kernel always uses|thread = 0|.

We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|reg| property contains only one entry, representing the HART (CPU) ID:

```
     Software can determine the number of threads by dividing the size of reg by the parent
     node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
     is required.
```
Quoted 4 lines
Quoted 1 line

Does that approach make sense, or should we stick with the current implementation?

If extra enabling is required to make multi-thread CPUs work, then panic()ing (not so much ASSERT()ing) may make sense, for the time being. Better would be if we could use all threads in a system right away.

Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id by passing thread argument (thread -> the local thread number to get the hardware ID for). So by calling:
dt_get_cpu_hwid(cpu0, 0) -> it will return hardware id of thread 0 of cpu0 dt_get_cpu_hwid(cpu0, 1) -> it will return hardware id of thread 1 of cpu0
...

Quoted 4 lines
Quoted 3 lines

In our case we assume that SMP isn't supported so that is why it is used only dt_get_cpu_hwid(cpu0, 0).

If one day, SMP will be enabled then it will be needed to change a callers of dt_get_cpu_hwid().

I assume you meant SMT in both places you wrote SMP?

Yes, it should be SMT.

Quoted 7 lines

But my main point here is:
If enumeration gives you "thread <N> of core <M>" (using x86 terminology), you need to be quite careful with what you call "CPU". Things need to be entirely unambiguous, taking into account what internally in (common code) Xen we call a "CPU". You certainly may call "CPU" what is a collection of threads / harts, but you then need to clarify this in a prominent comment somewhere, and you need to be entirely consistent throughout the RISC-V sub-tree.

╭────────────────────╮
│        CPU          │  ← 1 physical processor (chip)
│ ┌───────┬─────────┐ │
│ │ Core 0│ Core 1  │ │  ← 2 cores (for example)
│ │ ┌──┬──┐ ┌──┬──┐ │ │
│ │Thr0 Thr1 Thr0 Thr1│ ← 2 threads on each core (SMT)
│ └───────┴─────────┘ │
╰────────────────────╯

I want to double check what Xen call a "CPU". I thought that Xen uses word CPU to describe a core, right?

What you wrote above "thread <N> of core <M> (using x86 terminology)" is also correlated with RISC-V terminology:
A component is termed a core if it contains an independent instruction fetch unit. A RISC-V-compatible core might support multiple RISC-V-compatible hardware threads, or harts, through multithreading

I checked RISC-V's DTS binding and it seems it is a little bit contradictory to DTS spec, where it is mentioned that reg property is used to describe how many threads a cpu has when SMP is used, but in RISC-V's dts binding they are describing a hardware execution context:
This document uses some terminology common to the RISC-V community
that is not widely used, the definitions of which are listed here:

hart: A hardware execution context, which contains all the state mandated by the RISC-V ISA: a PC and some registers. This terminology is designed to disambiguate software's view of execution contexts from any particular microarchitectural implementation strategy. For example, an Intel laptop containing one socket with two cores, each of which has two hyperthreads, could be described as having four harts.

So in RISC-V's DTS binding they are describing only hardware threads what makes things more confusing in terms what kind terminology from Xen point of view should be used.

And based on what is written in RISC-V's dts binding:
For example, an Intel laptop containing one socket with
two cores, each of which has two hyperthreads, could be described as
having four harts.
It would be more logical to drop 'thread' argument of riscv_of_get_cpu_hwid(const struct dt_device_node *cpun). And then the question is what to do with the name of variable cpun? As it could be still confusing. Or, at least, I can add the comment that CPUn in terms of RISC-V means hart (hardware thread). And then will it be needed to add such comment for each usage of word "CPU"?

~ Oleksii

#34 lore raw

Re: [PATCH v1 06/14] xen/riscv: riscv_of_processor_hartid() implementation

Quoted 77 lines · Oleksii Kurochko
Quoted 52 lines · Jan Beulich
Quoted 24 lines · Oleksii Kurochko
Quoted 23 lines · Jan Beulich
Quoted 22 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich

Review

Quoted 9 lines · Oleksii Kurochko
(quoted diff) +9 -0
+{
+    const __be32 *cell;
+    int ac;
+    uint32_t len;
+
+    ac = dt_n_addr_cells(cpun);
+    cell = dt_get_property(cpun, "reg", &len);
+    if ( !cell || !ac || ((sizeof(*cell) * ac * (thread + 1)) > len) )
+        return ~0ULL;

I'm sorry for my lack of DT knowledge, but what's "thread" representing here? You only pass in 0 below, so it's unclear whether it's what one might expect (the thread number on a multi-threaded core).

Based on the DT specification alone, the|reg| value could refer to either a CPU or a thread ID:

```
The value of reg is a <prop-encoded-array> that defines a unique CPU/thread id for
the CPU/threads represented by the CPU node. If a CPU supports more than one thread
(i.e. multiple streams of execution) the reg prop-erty is an array with 1 element
per thread.
```
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

My understanding is that the term/thread/ was used in the Linux kernel to cover both cases.
When SMT isn't supported, the CPU can be considered to have a single thread. For example, RISC-V uses the term/hardware thread/ to describe a hart (i.e., a CPU).

Note the terminology ("CPU") you used here.

Quoted 21 lines
Quoted 10 lines
Quoted 10 lines

Interestingly, the Linux kernel always uses|thread = 0|.

We could potentially drop this ambiguity and introduce an|ASSERT()| to check that the|reg| property contains only one entry, representing the HART (CPU) ID:

```
     Software can determine the number of threads by dividing the size of reg by the parent
     node’s #address-cells. If `|reg`| has more than one entry, it would simply SMT support
     is required.
```
Quoted 4 lines
Quoted 1 line

Does that approach make sense, or should we stick with the current implementation?

If extra enabling is required to make multi-thread CPUs work, then panic()ing (not so much ASSERT()ing) may make sense, for the time being. Better would be if we could use all threads in a system right away.

Actually, this function is ready to be used for multi-thread CPUs. A caller can request hardware id by passing thread argument (thread -> the local thread number to get the hardware ID for). So by calling:
dt_get_cpu_hwid(cpu0, 0) -> it will return hardware id of thread 0 of cpu0 dt_get_cpu_hwid(cpu0, 1) -> it will return hardware id of thread 1 of cpu0
...

Quoted 4 lines
Quoted 3 lines

In our case we assume that SMP isn't supported so that is why it is used only dt_get_cpu_hwid(cpu0, 0).

If one day, SMP will be enabled then it will be needed to change a callers of dt_get_cpu_hwid().

I assume you meant SMT in both places you wrote SMP?

Yes, it should be SMT.

Quoted 7 lines

But my main point here is:
If enumeration gives you "thread <N> of core <M>" (using x86 terminology), you need to be quite careful with what you call "CPU". Things need to be entirely unambiguous, taking into account what internally in (common code) Xen we call a "CPU". You certainly may call "CPU" what is a collection of threads / harts, but you then need to clarify this in a prominent comment somewhere, and you need to be entirely consistent throughout the RISC-V sub-tree.

╭────────────────────╮
│        CPU          │  ← 1 physical processor (chip)
│ ┌───────┬─────────┐ │
│ │ Core 0│ Core 1  │ │  ← 2 cores (for example)
│ │ ┌──┬──┐ ┌──┬──┐ │ │
│ │Thr0 Thr1 Thr0 Thr1│ ← 2 threads on each core (SMT)
│ └───────┴─────────┘ │
╰────────────────────╯
Quoted 2 lines

I want to double check what Xen call a "CPU". I thought that Xen uses word CPU to describe a core, right?

No, see e.g. cpumask.h - it's a hart (as per below) that we internally describe as CPU (leaving aside potentially ambiguous comments here and there, which is what I'd like to prevent from the start for RISC-V).

Quoted 24 lines

What you wrote above "thread <N> of core <M> (using x86 terminology)" is also correlated with RISC-V terminology:
A component is termed a core if it contains an independent instruction fetch unit. A RISC-V-compatible core might support multiple RISC-V-compatible hardware threads, or harts, through multithreading

I checked RISC-V's DTS binding and it seems it is a little bit contradictory to DTS spec, where it is mentioned that reg property is used to describe how many threads a cpu has when SMP is used, but in RISC-V's dts binding they are describing a hardware execution context:
This document uses some terminology common to the RISC-V community
that is not widely used, the definitions of which are listed here:

hart: A hardware execution context, which contains all the state mandated by the RISC-V ISA: a PC and some registers. This terminology is designed to disambiguate software's view of execution contexts from any particular microarchitectural implementation strategy. For example, an Intel laptop containing one socket with two cores, each of which has two hyperthreads, could be described as having four harts.

So in RISC-V's DTS binding they are describing only hardware threads what makes things more confusing in terms what kind terminology from Xen point of view should be used.

Quoted 7 lines

And based on what is written in RISC-V's dts binding:
For example, an Intel laptop containing one socket with
two cores, each of which has two hyperthreads, could be described as
having four harts.
It would be more logical to drop 'thread' argument of riscv_of_get_cpu_hwid(const struct dt_device_node *cpun). And then the question is what to do with the name of variable cpun? As it could be still confusing. Or, at least, I can add the comment that CPUn in terms of RISC-V means hart (hardware thread).

If it's the normal thing you call a CPU, I don't think much commentary is necessary. Then please just make sure you don't call anything else "CPU", especially in identifiers.

Quoted 2 lines

And then will it be needed to
add such comment for each usage of word "CPU"?

No, that would go too far in any event. Hence why I said "clarify this in a prominent comment somewhere".

Jan

#35 lore raw

PATCH 7/14 xen/riscv: Introduce intc_hw_operations abstraction

In reply to Oleksii Kurochko

Introduce the intc_hw_operations structure to encapsulate interrupt controller-specific data and operations. This structure includes:
- A pointer to interrupt controller information (intc_info)
- Callbacks to initialize the controller and set IRQ type/priority
- A reference to an interupt controller descriptor (host_irq_type)

Also introduce generic helper functions:
- intc_init(): Initializes the interrupt controller
- register_intc_ops(): Registers the intc_hw_operations implementation
- intc_route_irq_to_xen(): Configures IRQ routing to Xen, setting handler, type, and priority

Most of these functions act as thin wrappers around the corresponding callbacks in intc_hw_operations.

This abstraction lays the groundwork for supporting multiple interrupt controller types (e.g., PLIC, APLIC) in extensible way.

This patch is based on the changes from [1].

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7cfb4bd4748ca268142497ac5c327d2766fb342d

---
 xen/arch/riscv/include/asm/intc.h | 23 ++++++++++++++
 xen/arch/riscv/intc.c             | 51 +++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)
xen/arch/riscv/include/asm/intc.h +23 -0
index 52ba196d87..0d498b10f4 100644
@@ -17,6 +17,29 @@ struct intc_info {
     const struct dt_device_node *node;
 };
 
+struct intc_hw_operations {
+    /* Hold intc hw information */
+    const struct intc_info *info;
+    /* Initialize the intc and the boot CPU */
+    int (*init)(void);
+
+    /* hw_irq_controller to enable/disable/eoi host irq */
+    hw_irq_controller *host_irq_type;
+
+    /* Set IRQ type */
+    void (*set_irq_type)(struct irq_desc *desc, unsigned int type);
+    /* Set IRQ priority */
+    void (*set_irq_priority)(struct irq_desc *desc, unsigned int priority);
+
+};
+
 void intc_preinit(void);
 
+void intc_init(void);
+
+void register_intc_ops(const struct intc_hw_operations *ops);
+
+struct irq_desc;
+void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority);
+
 #endif /* ASM__RISCV__INTERRUPT_CONTOLLER_H */
xen/arch/riscv/intc.c +51 -0

63 lines not on this page · raw message

Signature · 1 line
2.49.0
#36 lore raw

Re: [PATCH v1 07/14] xen/riscv: Introduce intc_hw_operations abstraction

Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/intc.h +0 -0
@@ -17,6 +17,29 @@ struct intc_info {
     const struct dt_device_node *node;

xen/arch/riscv/include/asm/intc.h struct intc_info { · line 27

Quoted 10 lines
xen/arch/riscv/include/asm/intc.h +8 -0
 };
 
+struct intc_hw_operations {
+    /* Hold intc hw information */
+    const struct intc_info *info;
+    /* Initialize the intc and the boot CPU */
+    int (*init)(void);
+
+    /* hw_irq_controller to enable/disable/eoi host irq */
+    hw_irq_controller *host_irq_type;

Pointer-to-const perhaps?

Quoted 4 lines
xen/arch/riscv/include/asm/intc.h +4 -0
+    /* Set IRQ type */
+    void (*set_irq_type)(struct irq_desc *desc, unsigned int type);
+    /* Set IRQ priority */
+    void (*set_irq_priority)(struct irq_desc *desc, unsigned int priority);

xen/arch/riscv/include/asm/intc.h struct intc_info { · line 42

Quoted 10 lines
xen/arch/riscv/include/asm/intc.h +8 -0
+
+};
+
 void intc_preinit(void);
 
+void intc_init(void);
+
+void register_intc_ops(const struct intc_hw_operations *ops);
+
+struct irq_desc;

If it's needed here at all, it needs to move up, as some of the hook pointers already use the type.

Quoted 6 lines
xen/arch/riscv/intc.c +0 -0
@@ -1,9 +1,21 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 
 #include <xen/acpi.h>

xen/arch/riscv/intc.c line 13

Quoted 10 lines
xen/arch/riscv/intc.c +7 -0
+#include <xen/bug.h>
 #include <xen/device_tree.h>
 #include <xen/init.h>
+#include <xen/irq.h>
 #include <xen/lib.h>
+#include <xen/spinlock.h>
+
+#include <asm/intc.h>
+
+static const struct intc_hw_operations *intc_hw_ops;

__ro_after_init perhaps?

xen/arch/riscv/intc.c line 15

Quoted 2 lines
xen/arch/riscv/intc.c +2 -0
+
+void register_intc_ops(const struct intc_hw_operations *ops)

__init perhaps?

Quoted 38 lines
xen/arch/riscv/intc.c +31 -0
+{
+    intc_hw_ops = ops;
+}
 
 void __init intc_preinit(void)
 {
@@ -12,3 +24,42 @@ void __init intc_preinit(void)
     else
         panic("ACPI interrupt controller preinit() isn't implemented\n");
 }
+
+void __init intc_init(void)
+{
+    ASSERT(intc_hw_ops);
+
+    if ( intc_hw_ops->init() )
+        panic("Failed to initialize the interrupt controller drivers\n");
+}
+
+/* desc->irq needs to be disabled before calling this function */
+static void intc_set_irq_type(struct irq_desc *desc, unsigned int type)
+{
+    ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
+    ASSERT(spin_is_locked(&desc->lock));
+    ASSERT(type != IRQ_TYPE_INVALID);
+    ASSERT(intc_hw_ops && intc_hw_ops->set_irq_type);
+
+    intc_hw_ops->set_irq_type(desc, type);
+}
+
+static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority)
+{
+    ASSERT(intc_hw_ops && intc_hw_ops->set_irq_priority);
+
+    intc_hw_ops->set_irq_priority(desc, priority);
+}
+
+void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority)

xen/arch/riscv/intc.c void __init intc_preinit(void) · line 64

Quoted 10 lines
xen/arch/riscv/intc.c +10 -0
+{
+    ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
+    ASSERT(spin_is_locked(&desc->lock));
+    /* Can't route interrupts that don't exist */
+    ASSERT(intc_hw_ops && desc->irq < intc_hw_ops->info->nr_irqs);
+
+    desc->handler = intc_hw_ops->host_irq_type;
+
+    intc_set_irq_type(desc, desc->arch.type);
+    intc_set_irq_priority(desc, priority);

If these are going to remain the sole callers of the two functions, I'd question the need for the separate functions. Some of the assertions done there would then actually be redundant.

If not, is there a reason intc_set_irq_priority() doesn't have a lock-held check?

Jan

#37 lore raw

Re: [PATCH v1 07/14] xen/riscv: Introduce intc_hw_operations abstraction

Quoted 7 lines · Jan Beulich
Quoted 6 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/intc.h +0 -0
@@ -17,6 +17,29 @@ struct intc_info {
      const struct dt_device_node *node;
  };
 
Quoted 9 lines

xen/arch/riscv/include/asm/intc.h struct intc_info { · line 27

Quoted 8 lines
xen/arch/riscv/include/asm/intc.h +8 -0
+struct intc_hw_operations {
+    /* Hold intc hw information */
+    const struct intc_info *info;
+    /* Initialize the intc and the boot CPU */
+    int (*init)(void);
+
+    /* hw_irq_controller to enable/disable/eoi host irq */
+    hw_irq_controller *host_irq_type;

Pointer-to-const perhaps?

It could be pointer-to-const. I'll updat

Quoted 10 lines
Quoted 10 lines
xen/arch/riscv/include/asm/intc.h +7 -0
+    /* Set IRQ type */
+    void (*set_irq_type)(struct irq_desc *desc, unsigned int type);
+    /* Set IRQ priority */
+    void (*set_irq_priority)(struct irq_desc *desc, unsigned int priority);
+
+};
+
  void intc_preinit(void);
 
Quoted 7 lines

xen/arch/riscv/include/asm/intc.h struct intc_info { · line 42

Quoted 5 lines
xen/arch/riscv/include/asm/intc.h +5 -0
+void intc_init(void);
+
+void register_intc_ops(const struct intc_hw_operations *ops);
+
+struct irq_desc;

If it's needed here at all, it needs to move up, as some of the hook pointers already use the type.

It could be dropped, but then we would need to include|<xen/irq.h>|, and I figured the fewer includes we have, the better.

Quoted 19 lines
Quoted 7 lines
xen/arch/riscv/intc.c +0 -0
@@ -1,9 +1,21 @@
  /* SPDX-License-Identifier: GPL-2.0-only */
 
  #include <xen/acpi.h>

xen/arch/riscv/intc.c line 13

Quoted 10 lines
xen/arch/riscv/intc.c +7 -0
+#include <xen/bug.h>
  #include <xen/device_tree.h>
  #include <xen/init.h>
+#include <xen/irq.h>
  #include <xen/lib.h>
+#include <xen/spinlock.h>
+
+#include <asm/intc.h>
+
+static const struct intc_hw_operations *intc_hw_ops;

__ro_after_init perhaps?

Quoted 3 lines

xen/arch/riscv/intc.c line 15

Quoted 2 lines
xen/arch/riscv/intc.c +2 -0
+
+void register_intc_ops(const struct intc_hw_operations *ops)

__init perhaps?

For both, yes, it should be __ro_after_init and __init.

Quoted 44 lines
Quoted 44 lines
xen/arch/riscv/intc.c +36 -0
+{
+    intc_hw_ops = ops;
+}
 
  void __init intc_preinit(void)
  {
@@ -12,3 +24,42 @@ void __init intc_preinit(void)
      else
          panic("ACPI interrupt controller preinit() isn't implemented\n");
  }
+
+void __init intc_init(void)
+{
+    ASSERT(intc_hw_ops);
+
+    if ( intc_hw_ops->init() )
+        panic("Failed to initialize the interrupt controller drivers\n");
+}
+
+/* desc->irq needs to be disabled before calling this function */
+static void intc_set_irq_type(struct irq_desc *desc, unsigned int type)
+{
+    ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
+    ASSERT(spin_is_locked(&desc->lock));
+    ASSERT(type != IRQ_TYPE_INVALID);
+    ASSERT(intc_hw_ops && intc_hw_ops->set_irq_type);
+
+    intc_hw_ops->set_irq_type(desc, type);
+}
+
+static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority)
+{
+    ASSERT(intc_hw_ops && intc_hw_ops->set_irq_priority);
+
+    intc_hw_ops->set_irq_priority(desc, priority);
+}
+
+void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority)
+{
+    ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
+    ASSERT(spin_is_locked(&desc->lock));
+    /* Can't route interrupts that don't exist */
+    ASSERT(intc_hw_ops && desc->irq < intc_hw_ops->info->nr_irqs);
Quoted 8 lines

xen/arch/riscv/intc.c void __init intc_preinit(void) · line 64

Quoted 5 lines
xen/arch/riscv/intc.c +5 -0
+
+    desc->handler = intc_hw_ops->host_irq_type;
+
+    intc_set_irq_type(desc, desc->arch.type);
+    intc_set_irq_priority(desc, priority);

If these are going to remain the sole callers of the two functions, I'd question the need for the separate functions. Some of the assertions done there would then actually be redundant.

Likely they will be reused for similar function but for guest to route an interrupt to a guest. And, perhaps, only intc_set_irq_type() will be re-used outside intc_route_irq_to_{xen,guest}...

Quoted 2 lines

If not, is there a reason intc_set_irq_priority() doesn't have a lock-held check?

Do you mean this one: ASSERT(spin_is_locked(&desc->lock));

...
And it is the reason why intc_set_irq_priority() doesn't have this check as it is checked in the caller.

But I think just to be on a safe side, it could be better to add this check to intc_set_irq_priority().

~ Oleksii

#38 lore raw

PATCH 8/14 xen/riscv: imsic_init() implementation

In reply to Oleksii Kurochko

imsic_init() is introduced to parse device tree node, which has the following bindings [2], and based on the parsed information update IMSIC configuration which is stored in imsic_cfg.

The following helpers are introduces for imsic_init() usage:
- imsic_parse_node() parses IMSIC node from DTS
- imsic_get_parent_hartid() returns the hart ( CPU ) ID of the given device tree node.

This patch is based on the code from [1].

Since Microchip originally developed imsic.{c,h}, an internal discussion with them led to the decision to use the MIT license.

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/0b1a94f2bc3bb1a81cd26bb75f0bf578f84cb4d4
[2] https://elixir.bootlin.com/linux/v6.12/source/Documentation/devicetree/bindings/interrupt-controller/riscv,imsics.yaml

---
 xen/arch/riscv/Makefile            |   1 +
 xen/arch/riscv/imsic.c             | 286 +++++++++++++++++++++++++++++
 xen/arch/riscv/include/asm/imsic.h |  66 +++++++
 3 files changed, 353 insertions(+)
 create mode 100644 xen/arch/riscv/imsic.c
 create mode 100644 xen/arch/riscv/include/asm/imsic.h
xen/arch/riscv/Makefile +1 -0
index 457e8e88a4..baa499a72d 100644
@@ -2,6 +2,7 @@ obj-y += aplic.o
 obj-y += cpufeature.o
 obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
 obj-y += entry.o
+obj-y += imsic.o
 obj-y += intc.o
 obj-y += irq.o
 obj-y += mm.o
xen/arch/riscv/imsic.c new file +286 -0

286 lines not on this page · raw message

xen/arch/riscv/include/asm/imsic.h new file +66 -0

66 lines not on this page · raw message

Signature · 1 line
2.49.0
#39 lore raw

Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation

xen/arch/riscv/imsic.c line 9

Quoted 12 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c new file +9 -0
@@ -0,0 +1,286 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/imsic.c
+ *
+ * RISC-V Incoming MSI Controller support
+ *
+ * (c) 2023 Microchip Technology Inc.
+ * (c) 2024 Vates

No 2025 here (if already the years matter)?

Quoted 4 lines
xen/arch/riscv/imsic.c new file +4 -0
+ */
+
+#include <xen/const.h>
+#include <xen/device_tree.h>

xen/arch/riscv/imsic.c line 23

Quoted 10 lines
xen/arch/riscv/imsic.c new file +10 -0
+#include <xen/errno.h>
+#include <xen/init.h>
+#include <xen/macros.h>
+#include <xen/xmalloc.h>
+
+#include <asm/imsic.h>
+
+static struct imsic_config imsic_cfg;
+
+const struct imsic_config *imsic_get_config(void)

Does this need to return a pointer to non-const?

xen/arch/riscv/imsic.c line 36

Quoted 13 lines
xen/arch/riscv/imsic.c new file +13 -0
+{
+    return &imsic_cfg;
+}
+
+static int __init imsic_get_parent_hartid(struct dt_device_node *node,
+                                          unsigned int index,
+                                          unsigned long *hartid)
+{
+    int res;
+    unsigned long hart;
+    struct dt_phandle_args args;
+
+    /* Try the new-style interrupts-extended first */

The comment says "first", but then ...

xen/arch/riscv/imsic.c line 48

Quoted 12 lines
xen/arch/riscv/imsic.c new file +12 -0
+    res = dt_parse_phandle_with_args(node, "interrupts-extended",
+                                     "#interrupt-cells", index, &args);
+    if ( !res )
+    {
+        res = riscv_of_processor_hartid(args.np->parent, &hart);
+        if ( res < 0 )
+            return -EINVAL;
+
+        *hartid = hart;
+    }
+    return res;
+}

... nothing else is being tried.

Also, nit: Blank line please ahead of the main "return" of a function.

Further - any particular reason to discard riscv_of_processor_hartid()'s error code on the error path?

xen/arch/riscv/imsic.c line 50

Quoted 2 lines
xen/arch/riscv/imsic.c new file +2 -0
+
+

Nit: No double blank lines please (and I wish I wouldn't need to repeat this any further).

Quoted 9 lines
xen/arch/riscv/imsic.c new file +9 -0
+static int imsic_parse_node(struct dt_device_node *node,
+                     unsigned int *nr_parent_irqs)
+{
+    int rc;
+    unsigned int tmp;
+    paddr_t base_addr;
+
+    /* Find number of parent interrupts */
+    *nr_parent_irqs = dt_number_of_irq(node);

xen/arch/riscv/imsic.c line 69

Quoted 10 lines
xen/arch/riscv/imsic.c new file +10 -0
+    if ( !*nr_parent_irqs )
+    {
+        printk(XENLOG_ERR "%s: no parent irqs available\n", node->name);
+        return -ENOENT;
+    }
+
+    /* Find number of guest index bits in MSI address */
+    rc = dt_property_read_u32(node, "riscv,guest-index-bits",
+                              &imsic_cfg.guest_index_bits);
+    if ( !rc )

It is confusing to store a bool return value in a local "int" variable, just to then use it as boolean. Is the local var needed at all here?

Quoted 10 lines
xen/arch/riscv/imsic.c new file +10 -0
+        imsic_cfg.guest_index_bits = 0;
+    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT;
+    if ( tmp < imsic_cfg.guest_index_bits )
+    {
+        printk(XENLOG_ERR "%s: guest index bits too big\n", node->name);
+        return -ENOENT;
+    }
+
+    /* Find number of HART index bits */
+    rc = dt_property_read_u32(node, "riscv,hart-index-bits",

xen/arch/riscv/imsic.c line 89

Quoted 10 lines
xen/arch/riscv/imsic.c new file +10 -0
+                              &imsic_cfg.hart_index_bits);
+    if ( !rc )
+    {
+        /* Assume default value */
+        imsic_cfg.hart_index_bits = fls(*nr_parent_irqs);
+        if ( BIT(imsic_cfg.hart_index_bits, UL) < *nr_parent_irqs )
+            imsic_cfg.hart_index_bits++;
+    }
+    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT -
+          imsic_cfg.guest_index_bits;
    tmp -= imsic_cfg.guest_index_bits;

? (And then similarly further down.)

Quoted 38 lines
xen/arch/riscv/imsic.c new file +38 -0
+    if ( tmp < imsic_cfg.hart_index_bits )
+    {
+        printk(XENLOG_ERR "%s: HART index bits too big\n", node->name);
+        return -ENOENT;
+    }
+
+    /* Find number of group index bits */
+    rc = dt_property_read_u32(node, "riscv,group-index-bits",
+                              &imsic_cfg.group_index_bits);
+    if ( !rc )
+        imsic_cfg.group_index_bits = 0;
+    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT -
+          imsic_cfg.guest_index_bits - imsic_cfg.hart_index_bits;
+    if ( tmp < imsic_cfg.group_index_bits )
+    {
+        printk(XENLOG_ERR "%s: group index bits too big\n", node->name);
+        return -ENOENT;
+    }
+
+    /* Find first bit position of group index */
+    tmp = IMSIC_MMIO_PAGE_SHIFT * 2;
+    rc = dt_property_read_u32(node, "riscv,group-index-shift",
+                              &imsic_cfg.group_index_shift);
+    if ( !rc )
+        imsic_cfg.group_index_shift = tmp;
+    if ( imsic_cfg.group_index_shift < tmp )
+    {
+        printk(XENLOG_ERR "%s: group index shift too small\n", node->name);
+        return -ENOENT;
+    }
+    tmp = imsic_cfg.group_index_bits + imsic_cfg.group_index_shift - 1;
+    if ( tmp >= BITS_PER_LONG )
+    {
+        printk(XENLOG_ERR "%s: group index shift too big\n", node->name);
+        return -EINVAL;
+    }
+
+    /* Find number of interrupt identities */

xen/arch/riscv/imsic.c line 137

Quoted 10 lines
xen/arch/riscv/imsic.c new file +10 -0
+    rc = dt_property_read_u32(node, "riscv,num-ids", &imsic_cfg.nr_ids);
+    if ( !rc )
+    {
+        printk(XENLOG_ERR "%s: number of interrupt identities not found\n",
+               node->name);
+        return -ENOENT;
+    }
+
+    if ( (imsic_cfg.nr_ids < IMSIC_MIN_ID) ||
+         (imsic_cfg.nr_ids >= IMSIC_MAX_ID) ||

Something named "max" normally wants to decribe the highest valid value, not the first out-of-range one.

xen/arch/riscv/imsic.c line 148

Quoted 11 lines
xen/arch/riscv/imsic.c new file +11 -0
+         ((imsic_cfg.nr_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) )
+    {
+        printk(XENLOG_ERR "%s: invalid number of interrupt identities\n",
+               node->name);
+        return -EINVAL;
+    }
+
+    /* Compute base address */
+    imsic_cfg.nr_mmios = 0;
+    rc = dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL);
+    if (rc)

Nit: Style.

xen/arch/riscv/imsic.c line 151

Quoted 3 lines
xen/arch/riscv/imsic.c new file +3 -0
+    {
+        printk(XENLOG_ERR "%s: first MMIO resource not found\n", node->name);
+        return -EINVAL;

Discarding "rc" again?

xen/arch/riscv/imsic.c line 159

Quoted 8 lines
xen/arch/riscv/imsic.c new file +8 -0
+    }
+
+    imsic_cfg.base_addr = base_addr;
+    imsic_cfg.base_addr &= ~(BIT(imsic_cfg.guest_index_bits +
+                   imsic_cfg.hart_index_bits +
+                   IMSIC_MMIO_PAGE_SHIFT, UL) - 1);
+    imsic_cfg.base_addr &= ~((BIT(imsic_cfg.group_index_bits, UL) - 1) <<
+                   imsic_cfg.group_index_shift);

Besides indentation being bogus here, why is it that you need to mask bits off of the value read from DT? Wouldn't the expectation be that you get back the true base address?

xen/arch/riscv/imsic.c line 164

Quoted 4 lines
xen/arch/riscv/imsic.c new file +4 -0
+    /* Find number of MMIO register sets */
+    imsic_cfg.nr_mmios++;
+    while ( !dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL) )
+        imsic_cfg.nr_mmios++;

And the base addresses of these aren't of interest? Oh, I see they're fetched again further down.

Also - use do-while here?

Quoted 12 lines
xen/arch/riscv/imsic.c new file +12 -0
+    return 0;
+}
+
+int __init imsic_init(struct dt_device_node *node)
+{
+    int rc;
+    unsigned long reloff, hartid;
+    uint32_t nr_parent_irqs, index, nr_handlers = 0;
+    paddr_t base_addr;
+
+    /* Parse IMSIC node */
+    rc = imsic_parse_node(node, &nr_parent_irqs);

xen/arch/riscv/imsic.c line 187

Quoted 10 lines
xen/arch/riscv/imsic.c new file +10 -0
+    if ( rc )
+        return rc;
+
+    /* Allocate MMIO resource array */
+    imsic_cfg.mmios = xzalloc_array(struct imsic_mmios, imsic_cfg.nr_mmios);
+    if ( !imsic_cfg.mmios )
+        return -ENOMEM;
+
+    /* check MMIO register sets */
+    for ( int i = 0; i < imsic_cfg.nr_mmios; i++ )

No plain int please for anything that can only be non-negative.

Quoted 6 lines
xen/arch/riscv/imsic.c new file +6 -0
+    {
+        rc = dt_device_get_address(node, i, &imsic_cfg.mmios[i].base_addr,
+                                   &imsic_cfg.mmios[i].size);
+        if ( rc )
+        {
+            printk(XENLOG_ERR "%s:  unable to parse MMIO regset %d\n",

xen/arch/riscv/imsic.c line 203

Quoted 10 lines
xen/arch/riscv/imsic.c new file +10 -0
+                   node->name, i);
+            goto imsic_init_err;
+        }
+
+        base_addr = imsic_cfg.mmios[i].base_addr;
+        base_addr &= ~(BIT(imsic_cfg.guest_index_bits +
+                   imsic_cfg.hart_index_bits +
+                   IMSIC_MMIO_PAGE_SHIFT, UL) - 1);
+        base_addr &= ~((BIT(imsic_cfg.group_index_bits, UL) - 1) <<
+                   imsic_cfg.group_index_shift);

Indentation again.

xen/arch/riscv/imsic.c line 210

Quoted 7 lines
xen/arch/riscv/imsic.c new file +7 -0
+        if ( base_addr != imsic_cfg.base_addr )
+        {
+            rc = -EINVAL;
+            printk(XENLOG_ERR "%s: address mismatch for regset %d\n",
+                   node->name, i);
+            goto imsic_init_err;
+        }

Oh, all of the addresses need to (sufficiently) match.

Quoted 62 lines
xen/arch/riscv/imsic.c new file +62 -0
+    }
+
+    /* Configure handlers for target CPUs */
+    for ( int i = 0; i < nr_parent_irqs; i++ )
+    {
+        rc = imsic_get_parent_hartid(node, i, &hartid);
+        if ( rc )
+        {
+            printk(XENLOG_WARNING "%s: hart ID for parent irq%d not found\n",
+                   node->name, i);
+            continue;
+        }
+
+        if ( hartid > NR_CPUS )
+        {
+            printk(XENLOG_WARNING "%s: unsupported hart ID=%lu for parent irq%d\n",
+                   node->name, hartid, i);
+            continue;
+        }
+
+        /* Find MMIO location of MSI page */
+        index = imsic_cfg.nr_mmios;
+        reloff = i * BIT(imsic_cfg.guest_index_bits, UL) * IMSIC_MMIO_PAGE_SZ;
+        for ( int j = 0; imsic_cfg.nr_mmios; j++ )
+        {
+            if ( reloff < imsic_cfg.mmios[j].size )
+            {
+                index = j;
+                break;
+            }
+
+            /*
+             * MMIO region size may not be aligned to
+             * BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ
+             * if holes are present.
+             */
+            reloff -= ROUNDUP(imsic_cfg.mmios[j].size,
+                BIT(imsic_cfg.guest_index_bits, UL) * IMSIC_MMIO_PAGE_SZ);
+        }
+
+        if ( index >= imsic_cfg.nr_mmios )
+        {
+            printk(XENLOG_WARNING "%s: MMIO not found for parent irq%d\n",
+                   node->name, i);
+            continue;
+        }
+
+        if ( !IS_ALIGNED(imsic_cfg.msi[hartid].base_addr + reloff, PAGE_SIZE) )
+        {
+            printk(XENLOG_WARNING "%s: MMIO address 0x%lx is not aligned on a page\n",
+                   node->name, imsic_cfg.msi[hartid].base_addr + reloff);
+            imsic_cfg.msi[hartid].offset = 0;
+            imsic_cfg.msi[hartid].base_addr = 0;
+            continue;
+        }
+
+        imsic_cfg.mmios[index].harts[hartid] = true;
+        imsic_cfg.msi[hartid].base_addr = imsic_cfg.mmios[index].base_addr;
+        imsic_cfg.msi[hartid].offset = reloff;
+        nr_handlers++;
+    }
+

xen/arch/riscv/imsic.c line 282

Quoted 10 lines
xen/arch/riscv/imsic.c new file +10 -0
+    if ( !nr_handlers )
+    {
+        printk(XENLOG_ERR "%s: No CPU handlers found\n", node->name);
+        rc = -ENODEV;
+        goto imsic_init_err;
+    }
+
+    return 0;
+
+imsic_init_err:

Labels indented by at least one blank please.

xen/arch/riscv/imsic.c line 283

Quoted 1 line
xen/arch/riscv/imsic.c new file +1 -0
+    xfree(imsic_cfg.mmios);

Better use XFREE() in cases like this one?

Quoted 23 lines
xen/arch/riscv/include/asm/imsic.h new file +20 -0
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/imsic.h
+ *
+ * RISC-V Incoming MSI Controller support
+ *
+ * (c) 2023 Microchip Technology Inc.
+ */
+
+#ifndef ASM__RISCV__IMSIC_H
+#define ASM__RISCV__IMSIC_H
+
+#include <xen/types.h>
+
+#define IMSIC_MMIO_PAGE_SHIFT   12
+#define IMSIC_MMIO_PAGE_SZ      (1UL << IMSIC_MMIO_PAGE_SHIFT)
+
+#define IMSIC_MIN_ID            63
+#define IMSIC_MAX_ID            2048

xen/arch/riscv/include/asm/imsic.h line 30

Quoted 10 lines
xen/arch/riscv/include/asm/imsic.h new file +10 -0
+
+struct imsic_msi {
+    paddr_t base_addr;
+    unsigned long offset;
+};
+
+struct imsic_mmios {
+    paddr_t base_addr;
+    unsigned long size;
+    bool harts[NR_CPUS];

An array of bool - won't a bitmap do here? Even then I wouldn't be overly happy to see it dimensioned by NR_CPUS.

Quoted 17 lines
xen/arch/riscv/include/asm/imsic.h new file +17 -0
+};
+
+struct imsic_config {
+    /* base address */
+    paddr_t base_addr;
+
+    /* Bits representing Guest index, HART index, and Group index */
+    unsigned int guest_index_bits;
+    unsigned int hart_index_bits;
+    unsigned int group_index_bits;
+    unsigned int group_index_shift;
+
+    /* imsic phandle */
+    unsigned int phandle;
+
+    /* number of parent irq */
+    unsigned int nr_parent_irqs;

xen/arch/riscv/include/asm/imsic.h line 57

Quoted 10 lines
xen/arch/riscv/include/asm/imsic.h new file +10 -0
+
+    /* number off interrupt identities */
+    unsigned int nr_ids;
+
+    /* mmios */
+    unsigned int nr_mmios;
+    struct imsic_mmios *mmios;
+
+    /* MSI */
+    struct imsic_msi msi[NR_CPUS];

You surely can avoid wasting perhaps a lot of memory by allocating this based on the number of CPUs in use?

xen/arch/riscv/include/asm/imsic.h line 61

Quoted 4 lines
xen/arch/riscv/include/asm/imsic.h new file +4 -0
+};
+
+struct dt_device_node;
+int imsic_init(struct dt_device_node *n);

Misra demands that parameter names match between declaration and definition.

xen/arch/riscv/include/asm/imsic.h line 63

Quoted 1 line
xen/arch/riscv/include/asm/imsic.h new file +1 -0
+struct imsic_config;

I don't think you need this, as it's ...

xen/arch/riscv/include/asm/imsic.h line 64

Quoted 1 line
xen/arch/riscv/include/asm/imsic.h new file +1 -0
+const struct imsic_config *imsic_get_config(void);

... not used as parameter type (where its scope would otherwise be wrongly limited).

Jan

#40 lore raw

Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation

Quoted 14 lines · Jan Beulich

xen/arch/riscv/imsic.c line 9

Quoted 12 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c new file +9 -0
@@ -0,0 +1,286 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/imsic.c
+ *
+ * RISC-V Incoming MSI Controller support
+ *
+ * (c) 2023 Microchip Technology Inc.
+ * (c) 2024 Vates

No 2025 here (if already the years matter)?

I think it doesn't really matter and it could be just dropped.

Quoted 16 lines
Quoted 5 lines
xen/arch/riscv/imsic.c new file +4 -0
+ */
+
+#include <xen/const.h>
+#include <xen/device_tree.h>

xen/arch/riscv/imsic.c line 23

Quoted 10 lines
xen/arch/riscv/imsic.c new file +10 -0
+#include <xen/errno.h>
+#include <xen/init.h>
+#include <xen/macros.h>
+#include <xen/xmalloc.h>
+
+#include <asm/imsic.h>
+
+static struct imsic_config imsic_cfg;
+
+const struct imsic_config *imsic_get_config(void)

Does this need to return a pointer to non-const?

No, I think it could be const struct imsic_config * const as a return type. It isn't expected that the caller will change a pointer.

Quoted 16 lines
Quoted 4 lines
xen/arch/riscv/imsic.c new file +3 -0
+{
+    return &imsic_cfg;
+}

xen/arch/riscv/imsic.c line 36

Quoted 10 lines
xen/arch/riscv/imsic.c new file +10 -0
+
+static int __init imsic_get_parent_hartid(struct dt_device_node *node,
+                                          unsigned int index,
+                                          unsigned long *hartid)
+{
+    int res;
+    unsigned long hart;
+    struct dt_phandle_args args;
+
+    /* Try the new-style interrupts-extended first */

The comment says "first", but then ...

Quoted 13 lines

xen/arch/riscv/imsic.c line 48

Quoted 12 lines
xen/arch/riscv/imsic.c new file +12 -0
+    res = dt_parse_phandle_with_args(node, "interrupts-extended",
+                                     "#interrupt-cells", index, &args);
+    if ( !res )
+    {
+        res = riscv_of_processor_hartid(args.np->parent, &hart);
+        if ( res < 0 )
+            return -EINVAL;
+
+        *hartid = hart;
+    }
+    return res;
+}

... nothing else is being tried.

A stale comment, we decided to support only interrupt-extended to be in line with Linux device tree bindings.
I'll drop this comment.

Quoted 5 lines

Also, nit: Blank line please ahead of the main "return" of a function.

Further - any particular reason to discard riscv_of_processor_hartid()'s error code on the error path?

No particular reason, just overlooked that we could really return just res.

Quoted 2 lines

xen/arch/riscv/imsic.c line 52

Quoted 2 lines
xen/arch/riscv/imsic.c new file +2 -0
+static int imsic_parse_node(struct dt_device_node *node,
+                     unsigned int *nr_parent_irqs)

(for me: fix an indentation)

Quoted 11 lines
Quoted 11 lines
xen/arch/riscv/imsic.c new file +11 -0
+{
+    int rc;
+    unsigned int tmp;
+    paddr_t base_addr;
+
+    /* Find number of parent interrupts */
+    *nr_parent_irqs = dt_number_of_irq(node);
+    if ( !*nr_parent_irqs )
+    {
+        printk(XENLOG_ERR "%s: no parent irqs available\n", node->name);
+        return -ENOENT;
Quoted 8 lines

xen/arch/riscv/imsic.c line 69

Quoted 6 lines
xen/arch/riscv/imsic.c new file +6 -0
+    }
+
+    /* Find number of guest index bits in MSI address */
+    rc = dt_property_read_u32(node, "riscv,guest-index-bits",
+                              &imsic_cfg.guest_index_bits);
+    if ( !rc )

It is confusing to store a bool return value in a local "int" variable, just to then use it as boolean. Is the local var needed at all here?

'int` is used because of ...:
rc = dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL);
... at the end of imsic_parse_node().

Agree, we can just drop rc variable and return err code explicitly.

Quoted 21 lines
Quoted 11 lines
xen/arch/riscv/imsic.c new file +10 -0
+        imsic_cfg.guest_index_bits = 0;
+    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT;
+    if ( tmp < imsic_cfg.guest_index_bits )
+    {
+        printk(XENLOG_ERR "%s: guest index bits too big\n", node->name);
+        return -ENOENT;
+    }
+
+    /* Find number of HART index bits */
+    rc = dt_property_read_u32(node, "riscv,hart-index-bits",

xen/arch/riscv/imsic.c line 89

Quoted 10 lines
xen/arch/riscv/imsic.c new file +10 -0
+                              &imsic_cfg.hart_index_bits);
+    if ( !rc )
+    {
+        /* Assume default value */
+        imsic_cfg.hart_index_bits = fls(*nr_parent_irqs);
+        if ( BIT(imsic_cfg.hart_index_bits, UL) < *nr_parent_irqs )
+            imsic_cfg.hart_index_bits++;
+    }
+    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT -
+          imsic_cfg.guest_index_bits;
Quoted 3 lines
     tmp -= imsic_cfg.guest_index_bits;

? (And then similarly further down.)

Yes, it would be simpler.

Quoted 42 lines
Quoted 42 lines
xen/arch/riscv/imsic.c new file +42 -0
+    if ( tmp < imsic_cfg.hart_index_bits )
+    {
+        printk(XENLOG_ERR "%s: HART index bits too big\n", node->name);
+        return -ENOENT;
+    }
+
+    /* Find number of group index bits */
+    rc = dt_property_read_u32(node, "riscv,group-index-bits",
+                              &imsic_cfg.group_index_bits);
+    if ( !rc )
+        imsic_cfg.group_index_bits = 0;
+    tmp = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT -
+          imsic_cfg.guest_index_bits - imsic_cfg.hart_index_bits;
+    if ( tmp < imsic_cfg.group_index_bits )
+    {
+        printk(XENLOG_ERR "%s: group index bits too big\n", node->name);
+        return -ENOENT;
+    }
+
+    /* Find first bit position of group index */
+    tmp = IMSIC_MMIO_PAGE_SHIFT * 2;
+    rc = dt_property_read_u32(node, "riscv,group-index-shift",
+                              &imsic_cfg.group_index_shift);
+    if ( !rc )
+        imsic_cfg.group_index_shift = tmp;
+    if ( imsic_cfg.group_index_shift < tmp )
+    {
+        printk(XENLOG_ERR "%s: group index shift too small\n", node->name);
+        return -ENOENT;
+    }
+    tmp = imsic_cfg.group_index_bits + imsic_cfg.group_index_shift - 1;
+    if ( tmp >= BITS_PER_LONG )
+    {
+        printk(XENLOG_ERR "%s: group index shift too big\n", node->name);
+        return -EINVAL;
+    }
+
+    /* Find number of interrupt identities */
+    rc = dt_property_read_u32(node, "riscv,num-ids", &imsic_cfg.nr_ids);
+    if ( !rc )
+    {
+        printk(XENLOG_ERR "%s: number of interrupt identities not found\n",
Quoted 8 lines

xen/arch/riscv/imsic.c line 137

Quoted 6 lines
xen/arch/riscv/imsic.c new file +6 -0
+               node->name);
+        return -ENOENT;
+    }
+
+    if ( (imsic_cfg.nr_ids < IMSIC_MIN_ID) ||
+         (imsic_cfg.nr_ids >= IMSIC_MAX_ID) ||

Something named "max" normally wants to decribe the highest valid value, not the first out-of-range one.

But it is the maximum of identities for IMISIC according to the AIA specs:

```
( MRIF stands for Memory-Resident Interrupt File )

All MRIFs are the size to accommodate 2047 valid interrupt identities,
the maximum allowed for an IMSIC interrupt file. If a system’s actual IMSICs
have interrupt files that implement only N interrupt identities, N < 2047,
then the contents of MRIFs for identities greater than N may be ignored by
softwareand
```

and prefix IMSIC (IMO)emphasizes this. Would it be better to rename to IMSIC_MAX_ALLOWED_ID? And then IMSIC_MIN_ALLOWED_ID?

Quoted 12 lines

xen/arch/riscv/imsic.c line 148

Quoted 12 lines
xen/arch/riscv/imsic.c new file +11 -0
+         ((imsic_cfg.nr_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) )
+    {
+        printk(XENLOG_ERR "%s: invalid number of interrupt identities\n",
+               node->name);
+        return -EINVAL;
+    }
+
+    /* Compute base address */
+    imsic_cfg.nr_mmios = 0;
+    rc = dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL);
+    if (rc)
Quoted 6 lines

Nit: Style.

xen/arch/riscv/imsic.c line 151

Quoted 3 lines
xen/arch/riscv/imsic.c new file +3 -0
+    {
+        printk(XENLOG_ERR "%s: first MMIO resource not found\n", node->name);
+        return -EINVAL;

Discarding "rc" again?

Agree, it could be just rc, but considering of that this functions always returns -EINVAL in case of the error and that rc could be dropped at all, we just explicitly return here -EINVAL.

Quoted 9 lines

xen/arch/riscv/imsic.c line 159

Quoted 9 lines
xen/arch/riscv/imsic.c new file +8 -0
+    }
+
+    imsic_cfg.base_addr = base_addr;
+    imsic_cfg.base_addr &= ~(BIT(imsic_cfg.guest_index_bits +
+                   imsic_cfg.hart_index_bits +
+                   IMSIC_MMIO_PAGE_SHIFT, UL) - 1);
+    imsic_cfg.base_addr &= ~((BIT(imsic_cfg.group_index_bits, UL) - 1) <<
+                   imsic_cfg.group_index_shift);
Quoted 3 lines

Besides indentation being bogus here, why is it that you need to mask bits off of the value read from DT? Wouldn't the expectation be that you get back the true base address?

The group index is used to differentiate between clusters/groups. For example, consider two clusters: - Cluster 1 with cpu0 and cpu1 - Cluster 2 with cpu2 and cpu3 Then, the reg property in the IMSIC node will include two entries: reg = <0x0 0xd100000 0x0 0x20000>, <0x0 0x2d100000 0x0 0x20000>; riscv,guest-index-bits = <3>; riscv,hart-index-bits = <2>; riscv,group-index-bits = <1>; riscv,group-index-shift = <29>; In this example: The group index is 1 bit wide (group-index-bits = <1>), It is located at bit 29 (group-index-shift = <29>) of the address.

so imsic_cfg.group_index_bits will be used to distinguish clusters, but they must have

the same base address and this is the reason why the mask is applied.

Quoted 7 lines

xen/arch/riscv/imsic.c line 164

Quoted 5 lines
xen/arch/riscv/imsic.c new file +4 -0
+    /* Find number of MMIO register sets */
+    imsic_cfg.nr_mmios++;
+    while ( !dt_device_get_address(node, imsic_cfg.nr_mmios, &base_addr, NULL) )
+        imsic_cfg.nr_mmios++;

And the base addresses of these aren't of interest? Oh, I see they're fetched again further down.

It is fetched again because we are using imsic_cfg.nr_mmios to calculate the size of imsic_cfg.mmios array:
imsic_cfg.mmios = xzalloc_array(struct imsic_mmios, imsic_cfg.nr_mmios);

Quoted 2 lines

Also - use do-while here?

It could be, I'll update that.

Quoted 11 lines

xen/arch/riscv/imsic.c line 210

Quoted 7 lines
xen/arch/riscv/imsic.c new file +7 -0
+        if ( base_addr != imsic_cfg.base_addr )
+        {
+            rc = -EINVAL;
+            printk(XENLOG_ERR "%s: address mismatch for regset %d\n",
+                   node->name, i);
+            goto imsic_init_err;
+        }

Oh, all of the addresses need to (sufficiently) match.

xen/arch/riscv/imsic.c line 283

Quoted 1 line
xen/arch/riscv/imsic.c new file +1 -0
+    xfree(imsic_cfg.mmios);

Better use XFREE() in cases like this one?

I think, yes.

Quoted 28 lines
Quoted 28 lines
xen/arch/riscv/include/asm/imsic.h new file +24 -0
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/imsic.h
+ *
+ * RISC-V Incoming MSI Controller support
+ *
+ * (c) 2023 Microchip Technology Inc.
+ */
+
+#ifndef ASM__RISCV__IMSIC_H
+#define ASM__RISCV__IMSIC_H
+
+#include <xen/types.h>
+
+#define IMSIC_MMIO_PAGE_SHIFT   12
+#define IMSIC_MMIO_PAGE_SZ      (1UL << IMSIC_MMIO_PAGE_SHIFT)
+
+#define IMSIC_MIN_ID            63
+#define IMSIC_MAX_ID            2048
+
+struct imsic_msi {
+    paddr_t base_addr;
+    unsigned long offset;
Quoted 8 lines

xen/arch/riscv/include/asm/imsic.h line 30

Quoted 6 lines
xen/arch/riscv/include/asm/imsic.h new file +6 -0
+};
+
+struct imsic_mmios {
+    paddr_t base_addr;
+    unsigned long size;
+    bool harts[NR_CPUS];

An array of bool - won't a bitmap do here? Even then I wouldn't be overly happy to see it dimensioned by NR_CPUS.

Bitmap will fit here well. But for DECLARE_BITMAP() is necessary the size of bitmap so NR_CPUS should be used again.
Could you please remind me why it isn't good to use it?
Because NR_CPUS not always equal to an amount of physical cpus? Should I use non-static version of bitmap declaration? (if we have such...)

Quoted 22 lines
Quoted 22 lines
xen/arch/riscv/include/asm/imsic.h new file +21 -0
+};
+
+struct imsic_config {
+    /* base address */
+    paddr_t base_addr;
+
+    /* Bits representing Guest index, HART index, and Group index */
+    unsigned int guest_index_bits;
+    unsigned int hart_index_bits;
+    unsigned int group_index_bits;
+    unsigned int group_index_shift;
+
+    /* imsic phandle */
+    unsigned int phandle;
+
+    /* number of parent irq */
+    unsigned int nr_parent_irqs;
+
+    /* number off interrupt identities */
+    unsigned int nr_ids;
+
Quoted 8 lines

xen/arch/riscv/include/asm/imsic.h line 57

Quoted 6 lines
xen/arch/riscv/include/asm/imsic.h new file +6 -0
+    /* mmios */
+    unsigned int nr_mmios;
+    struct imsic_mmios *mmios;
+
+    /* MSI */
+    struct imsic_msi msi[NR_CPUS];

You surely can avoid wasting perhaps a lot of memory by allocating this based on the number of CPUs in use?

It make sense. I'll allocate then this dynamically.

Thanks!

~ Oleksii

A message that is not in the archive (sent off-list or not yet received): [email protected]

#42 lore raw

Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation

Quoted 23 lines · Jan Beulich
Quoted 23 lines
Quoted 11 lines

xen/arch/riscv/imsic.c line 159

Quoted 8 lines
xen/arch/riscv/imsic.c new file +8 -0
+    }
+
+    imsic_cfg.base_addr = base_addr;
+    imsic_cfg.base_addr &= ~(BIT(imsic_cfg.guest_index_bits +
+                   imsic_cfg.hart_index_bits +
+                   IMSIC_MMIO_PAGE_SHIFT, UL) - 1);
+    imsic_cfg.base_addr &= ~((BIT(imsic_cfg.group_index_bits, UL) - 1) <<
+                   imsic_cfg.group_index_shift);

Besides indentation being bogus here, why is it that you need to mask bits off of the value read from DT? Wouldn't the expectation be that you get back the true base address?

The group index is used to differentiate between clusters/groups. For example, consider two clusters: - Cluster 1 with cpu0 and cpu1 - Cluster 2 with cpu2 and cpu3 Then, the reg property in the IMSIC node will include two entries: reg = <0x0 0xd100000 0x0 0x20000>, <0x0 0x2d100000 0x0 0x20000>; riscv,guest-index-bits = <3>; riscv,hart-index-bits = <2>; riscv,group-index-bits = <1>; riscv,group-index-shift = <29>; In this
example: The group index is 1 bit wide (group-index-bits = <1>), It is located at bit 29 (group-index-shift = <29>) of the address.

so imsic_cfg.group_index_bits will be used to distinguish clusters, but they must have

Quoted 7 lines
Quoted 1 line

the same base address and this is the reason why the mask is applied.

Well, yes, but that doesn't answer my question. All of what you say makes sense for the loop elsewhere retrieving all the addresses. Here you retrieved only the first of them, and in your example no masking would be needed here either. To re-phrase my question: Can the address observed in the first entry be other than the lowest one across the set of all entries?

It doesn't mentioned explicitly in riscv,imsiscs binding that reg property is sorted.
I can write:
reg = <0x0 0x2d100000 0x0 0x20000>,<0x0 0xd100000 0x0 0x20000>;
And a dtc compiler will compile.

Quoted 2 lines

If not, can there be bits set across all entries that need masking off from all of them?

I think that other bits (Hart Index and Guest Index) excepts base address bits are 0 in dts, but it doesn't guarantee explicitly by IMSIC's dts bindinding.

Quoted 36 lines
Quoted 36 lines
Quoted 36 lines
Quoted 24 lines
xen/arch/riscv/include/asm/imsic.h new file +20 -0
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/imsic.h
+ *
+ * RISC-V Incoming MSI Controller support
+ *
+ * (c) 2023 Microchip Technology Inc.
+ */
+
+#ifndef ASM__RISCV__IMSIC_H
+#define ASM__RISCV__IMSIC_H
+
+#include <xen/types.h>
+
+#define IMSIC_MMIO_PAGE_SHIFT   12
+#define IMSIC_MMIO_PAGE_SZ      (1UL << IMSIC_MMIO_PAGE_SHIFT)
+
+#define IMSIC_MIN_ID            63
+#define IMSIC_MAX_ID            2048

xen/arch/riscv/include/asm/imsic.h line 30

Quoted 10 lines
xen/arch/riscv/include/asm/imsic.h new file +10 -0
+
+struct imsic_msi {
+    paddr_t base_addr;
+    unsigned long offset;
+};
+
+struct imsic_mmios {
+    paddr_t base_addr;
+    unsigned long size;
+    bool harts[NR_CPUS];

An array of bool - won't a bitmap do here? Even then I wouldn't be overly happy to see it dimensioned by NR_CPUS.

Quoted 8 lines
Quoted 4 lines

Bitmap will fit here well. But for DECLARE_BITMAP() is necessary the size of bitmap so NR_CPUS should be used again.
Could you please remind me why it isn't good to use it? Because NR_CPUS not always equal to an amount of physical cpus?

"Not equal" wouldn't be overly problematic. But NR_CPUS=4000 and the actual number of CPUs being 4 would be wasteful in general. More when its wider than a bit that's needed per CPU, but where would you draw the line if you permitted use of NR_CPUS here?

Hard to say. It seems it will be just better to use apporoach you suggested below.

Thanks.

~ Oleksii

Quoted 40 lines
Quoted 2 lines

Should I use non-static version of bitmap declaration? (if we have such...)

That's simply "unsigned long *" then, or - at the tail of a dynamically allocated struct - possibly unsigned long[].

Quoted 29 lines
Quoted 21 lines
Quoted 21 lines
xen/arch/riscv/include/asm/imsic.h new file +21 -0
+};
+
+struct imsic_config {
+    /* base address */
+    paddr_t base_addr;
+
+    /* Bits representing Guest index, HART index, and Group index */
+    unsigned int guest_index_bits;
+    unsigned int hart_index_bits;
+    unsigned int group_index_bits;
+    unsigned int group_index_shift;
+
+    /* imsic phandle */
+    unsigned int phandle;
+
+    /* number of parent irq */
+    unsigned int nr_parent_irqs;
+
+    /* number off interrupt identities */
+    unsigned int nr_ids;
+
Quoted 8 lines

xen/arch/riscv/include/asm/imsic.h line 57

Quoted 6 lines
xen/arch/riscv/include/asm/imsic.h new file +6 -0
+    /* mmios */
+    unsigned int nr_mmios;
+    struct imsic_mmios *mmios;
+
+    /* MSI */
+    struct imsic_msi msi[NR_CPUS];

You surely can avoid wasting perhaps a lot of memory by allocating this based on the number of CPUs in use?

Quoted 1 line

It make sense. I'll allocate then this dynamically.

Or, as per above, when put at the tail and the struct itself is dynamically allocated, use struct imsic_msi[]. We even have dedicated xmalloc() flavors for this kind of allocation.

Jan

#43 lore raw

Re: [PATCH v1 08/14] xen/riscv: imsic_init() implementation

In reply to a message not in the archive
Quoted 77 lines · Jan Beulich
Quoted 35 lines
Quoted 27 lines
Quoted 27 lines
xen/arch/riscv/include/asm/imsic.h new file +24 -0
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/imsic.h
+ *
+ * RISC-V Incoming MSI Controller support
+ *
+ * (c) 2023 Microchip Technology Inc.
+ */
+
+#ifndef ASM__RISCV__IMSIC_H
+#define ASM__RISCV__IMSIC_H
+
+#include <xen/types.h>
+
+#define IMSIC_MMIO_PAGE_SHIFT   12
+#define IMSIC_MMIO_PAGE_SZ      (1UL << IMSIC_MMIO_PAGE_SHIFT)
+
+#define IMSIC_MIN_ID            63
+#define IMSIC_MAX_ID            2048
+
+struct imsic_msi {
+    paddr_t base_addr;
+    unsigned long offset;
Quoted 8 lines

xen/arch/riscv/include/asm/imsic.h line 30

Quoted 6 lines
xen/arch/riscv/include/asm/imsic.h new file +6 -0
+};
+
+struct imsic_mmios {
+    paddr_t base_addr;
+    unsigned long size;
+    bool harts[NR_CPUS];

An array of bool - won't a bitmap do here? Even then I wouldn't be overly happy to see it dimensioned by NR_CPUS.

Quoted 4 lines

Bitmap will fit here well. But for DECLARE_BITMAP() is necessary the size of bitmap so NR_CPUS should be used again.
Could you please remind me why it isn't good to use it? Because NR_CPUS not always equal to an amount of physical cpus?

"Not equal" wouldn't be overly problematic. But NR_CPUS=4000 and the actual number of CPUs being 4 would be wasteful in general. More when its wider than a bit that's needed per CPU, but where would you draw the line if you permitted use of NR_CPUS here?

Quoted 1 line

Should I use non-static version of bitmap declaration? (if we have such...)

That's simply "unsigned long *" then, or - at the tail of a dynamically allocated struct - possibly unsigned long[].

Quoted 29 lines
Quoted 29 lines
Quoted 17 lines
xen/arch/riscv/include/asm/imsic.h new file +17 -0
+};
+
+struct imsic_config {
+    /* base address */
+    paddr_t base_addr;
+
+    /* Bits representing Guest index, HART index, and Group index */
+    unsigned int guest_index_bits;
+    unsigned int hart_index_bits;
+    unsigned int group_index_bits;
+    unsigned int group_index_shift;
+
+    /* imsic phandle */
+    unsigned int phandle;
+
+    /* number of parent irq */
+    unsigned int nr_parent_irqs;

xen/arch/riscv/include/asm/imsic.h line 57

Quoted 10 lines
xen/arch/riscv/include/asm/imsic.h new file +10 -0
+
+    /* number off interrupt identities */
+    unsigned int nr_ids;
+
+    /* mmios */
+    unsigned int nr_mmios;
+    struct imsic_mmios *mmios;
+
+    /* MSI */
+    struct imsic_msi msi[NR_CPUS];

You surely can avoid wasting perhaps a lot of memory by allocating this based on the number of CPUs in use?

Quoted 4 lines
Quoted 1 line

It make sense. I'll allocate then this dynamically.

Or, as per above, when put at the tail and the struct itself is dynamically allocated, use struct imsic_msi[]. We even have dedicated xmalloc() flavors for this kind of allocation.

Do you mean xzalloc_flex_struct()?

I think, I can't use for both of the cases (allocation of mmios and msi). For msi[] then it is needed to allocate imsic_config also dynamically, isn't it? So something like:
imsic_config = xzalloc_flex_struct(struct imsic_config, msi, NR_CPUS).
But now it is allocated statically.

For *mmios and harts[] (a member inside struct imsic_mmios):
mmios = xzalloc_flex_struct(struct imsic_mmios, harts, NR_CPUS); // NR_CPUs just for example...
It will allocate only one mmios, but it is needed mmios[nr_mmios].
Maybe, something like _xmalloc((offsetof(struct imsic_mmios, harts[NR_CPUS])) * NR_CPUS, sizeof(struct imsic_mmios)) will work.

Am I missing something?

~ Oleksii

#44 lore raw

PATCH 9/14 xen/riscv: aplic_init() implementation

In reply to Oleksii Kurochko

aplic_init() function does the following few things:
- checks that IMSIC in device tree node ( by checking msi-parent property in APLIC node ) is present as current one implmenetaion of AIA is supported only MSI method.
- initialize IMSIC based on IMSIC device tree node
- Read value of APLIC's paddr start/end and size.
- Map aplic.regs
- Setup APLIC initial state interrupts (disable all interrupts, set interrupt type and default priority, confgifure APLIC domaincfg) by calling aplic_init_hw_interrutps().

aplic_init() is based on the code from [1] and [2].

Since Microchip originally developed aplic.c, an internal discussion with them led to the decision to use the MIT license.

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7cfb4bd4748ca268142497ac5c327d2766fb342d
[2] https://gitlab.com/xen-project/people/olkur/xen/-/commit/392a531bfad39bf4656ce8128e004b241b8b3f3e

---
 xen/arch/riscv/aplic.c             | 97 ++++++++++++++++++++++++++++++
 xen/arch/riscv/include/asm/aplic.h | 77 ++++++++++++++++++++++++
 xen/arch/riscv/include/asm/intc.h  |  3 +
 xen/arch/riscv/include/asm/irq.h   |  1 -
 4 files changed, 177 insertions(+), 1 deletion(-)
 create mode 100644 xen/arch/riscv/include/asm/aplic.h
xen/arch/riscv/aplic.c +97 -0
index 6dc040af6f..d1aa835c3e 100644
@@ -9,19 +9,112 @@
  * Copyright (c) 2024-2025 Vates
  */
 
+#include <xen/device_tree.h>
 #include <xen/errno.h>
 #include <xen/init.h>
 #include <xen/irq.h>
+#include <xen/mm.h>
 #include <xen/sections.h>
 #include <xen/types.h>
+#include <xen/vmap.h>
 
+#include <asm/aplic.h>
 #include <asm/device.h>
+#include <asm/imsic.h>
 #include <asm/intc.h>
+#include <asm/riscv_encoding.h>
+
+#define APLIC_DEFAULT_PRIORITY  1
+
+static struct aplic_priv aplic;
 
 static struct intc_info __ro_after_init aplic_info = {
     .hw_version = INTC_APLIC,
 };
 
+static void __init aplic_init_hw_interrupts(void)
+{
+    int i;
+
+    /* Disable all interrupts */
+    for ( i = 0; i <= aplic_info.nr_irqs; i += 32 )
+        aplic.regs->clrie[i] = -1U;
+
+    /* Set interrupt type and default priority for all interrupts */
+    for ( i = 1; i <= aplic_info.nr_irqs; i++ )
+    {
+        aplic.regs->sourcecfg[i - 1] = 0;
+        aplic.regs->target[i - 1] = APLIC_DEFAULT_PRIORITY;
+    }
+
+    /* Clear APLIC domaincfg */
+    aplic.regs->domaincfg = APLIC_DOMAINCFG_IE | APLIC_DOMAINCFG_DM;
+}
+
+static int __init aplic_init(void)
+{
+    int rc;
+    dt_phandle imsic_phandle;
+    uint32_t irq_range[2];
+    const __be32 *prop;
+    uint64_t size, paddr;
+    struct dt_device_node *imsic_node;
+    const struct dt_device_node *node = aplic_info.node;
+
+    /* check for associated imsic node */
+    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
+    if ( !rc )
+        panic("%s: IDC mode not supported\n", node->full_name);
+
+    imsic_node = dt_find_node_by_phandle(imsic_phandle);
+    if ( !imsic_node )
+        panic("%s: unable to find IMSIC node\n", node->full_name);
+
+    /* check imsic mode */
+    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
+                                    irq_range, ARRAY_SIZE(irq_range));
+    if ( rc && (rc != -EOVERFLOW) )
+        panic("%s: unable to find interrupt-extended in %s node\n",
+               node->full_name, imsic_node->full_name);
+
+    if ( irq_range[1] == IRQ_M_EXT )
+        /* machine mode imsic node, ignore this aplic node */
+        return 0;
+
+    rc = imsic_init(imsic_node);
+    if ( rc )
+        panic("%s: Failded to initialize IMSIC\n", node->full_name);
+
+    /* Find out number of interrupt sources */
+    rc = dt_property_read_u32(node, "riscv,num-sources", &aplic_info.nr_irqs);
+    if ( !rc )
+        panic("%s: failed to get number of interrupt sources\n",
+              node->full_name);
+
+    prop = dt_get_property(node, "reg", NULL);
+    dt_get_range(&prop, node, &paddr, &size);
+    if ( !paddr )
+        panic("%s: first MMIO resource not found\n", node->full_name);
+
+    aplic.paddr_start = paddr;
+    aplic.paddr_end = paddr + size;
+    aplic.size = size;
+
+    aplic.regs = ioremap(paddr, size);
+    if ( !aplic.regs )
+        panic("%s: unable to map\n", node->full_name);
+
+    /* Setup initial state APLIC interrupts */
+    aplic_init_hw_interrupts();
+
+    return 0;
+}
+
+static const struct intc_hw_operations __ro_after_init aplic_ops = {
+    .info                = &aplic_info,
+    .init                = aplic_init,
+};
+
 static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
                            unsigned int *out_hwirq,
                            unsigned int *out_type)
@@ -52,8 +145,12 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 
     aplic_info.node = node;
 
+    aplic.imsic_cfg = imsic_get_config();
+
     dt_irq_xlate = aplic_irq_xlate;
 
+    register_intc_ops(&aplic_ops);
+
     return 0;
 }
 
xen/arch/riscv/include/asm/aplic.h new file +77 -0

77 lines not on this page · raw message

xen/arch/riscv/include/asm/intc.h +3 -0
index 0d498b10f4..db53caa07b 100644
@@ -15,6 +15,9 @@ enum intc_version {
 struct intc_info {
     enum intc_version hw_version;
     const struct dt_device_node *node;
+
+    /* number of irqs */
+    unsigned int nr_irqs;
 };
 
 struct intc_hw_operations {
xen/arch/riscv/include/asm/irq.h +0 -1
index ff1c95e0be..163a478d78 100644
@@ -27,7 +27,6 @@
 #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
 
 /* TODO */
-#define nr_irqs 0U
 #define nr_static_irqs 0
 #define arch_hwdom_irqs(domid) 0U
 
Signature · 1 line
2.49.0
#45 lore raw

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

Quoted 32 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +13 -0
@@ -9,19 +9,112 @@
  * Copyright (c) 2024-2025 Vates
  */
 
+#include <xen/device_tree.h>
 #include <xen/errno.h>
 #include <xen/init.h>
 #include <xen/irq.h>
+#include <xen/mm.h>
 #include <xen/sections.h>
 #include <xen/types.h>
+#include <xen/vmap.h>
 
+#include <asm/aplic.h>
 #include <asm/device.h>
+#include <asm/imsic.h>
 #include <asm/intc.h>
+#include <asm/riscv_encoding.h>
+
+#define APLIC_DEFAULT_PRIORITY  1
+
+static struct aplic_priv aplic;
 
 static struct intc_info __ro_after_init aplic_info = {
     .hw_version = INTC_APLIC,
 };
 
+static void __init aplic_init_hw_interrupts(void)
+{
+    int i;

xen/arch/riscv/aplic.c line 47

Quoted 10 lines
xen/arch/riscv/aplic.c +10 -0
+
+    /* Disable all interrupts */
+    for ( i = 0; i <= aplic_info.nr_irqs; i += 32 )
+        aplic.regs->clrie[i] = -1U;
+
+    /* Set interrupt type and default priority for all interrupts */
+    for ( i = 1; i <= aplic_info.nr_irqs; i++ )
+    {
+        aplic.regs->sourcecfg[i - 1] = 0;
+        aplic.regs->target[i - 1] = APLIC_DEFAULT_PRIORITY;

A field named "target" is written with a priority value?

xen/arch/riscv/aplic.c line 51

Quoted 4 lines
xen/arch/riscv/aplic.c +4 -0
+    }
+
+    /* Clear APLIC domaincfg */
+    aplic.regs->domaincfg = APLIC_DOMAINCFG_IE | APLIC_DOMAINCFG_DM;

The statement doesn't like like there was any "clearing" here.

xen/arch/riscv/aplic.c line 64

Quoted 13 lines
xen/arch/riscv/aplic.c +13 -0
+}
+
+static int __init aplic_init(void)
+{
+    int rc;
+    dt_phandle imsic_phandle;
+    uint32_t irq_range[2];
+    const __be32 *prop;
+    uint64_t size, paddr;
+    struct dt_device_node *imsic_node;
+    const struct dt_device_node *node = aplic_info.node;
+
+    /* check for associated imsic node */

Nit: Comment style (also elsewhere).

Quoted 4 lines
xen/arch/riscv/aplic.c +4 -0
+    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
+    if ( !rc )
+        panic("%s: IDC mode not supported\n", node->full_name);
+

xen/arch/riscv/aplic.c line 78

Quoted 10 lines
xen/arch/riscv/aplic.c +10 -0
+    imsic_node = dt_find_node_by_phandle(imsic_phandle);
+    if ( !imsic_node )
+        panic("%s: unable to find IMSIC node\n", node->full_name);
+
+    /* check imsic mode */
+    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
+                                    irq_range, ARRAY_SIZE(irq_range));
+    if ( rc && (rc != -EOVERFLOW) )
+        panic("%s: unable to find interrupt-extended in %s node\n",
+               node->full_name, imsic_node->full_name);

Why exactly is EOVERFLOW tolerable here?

Quoted 12 lines
xen/arch/riscv/aplic.c +12 -0
+    if ( irq_range[1] == IRQ_M_EXT )
+        /* machine mode imsic node, ignore this aplic node */
+        return 0;
+
+    rc = imsic_init(imsic_node);
+    if ( rc )
+        panic("%s: Failded to initialize IMSIC\n", node->full_name);
+
+    /* Find out number of interrupt sources */
+    rc = dt_property_read_u32(node, "riscv,num-sources", &aplic_info.nr_irqs);
+    if ( !rc )
+        panic("%s: failed to get number of interrupt sources\n",

xen/arch/riscv/aplic.c line 101

Quoted 10 lines
xen/arch/riscv/aplic.c +10 -0
+              node->full_name);
+
+    prop = dt_get_property(node, "reg", NULL);
+    dt_get_range(&prop, node, &paddr, &size);
+    if ( !paddr )
+        panic("%s: first MMIO resource not found\n", node->full_name);
+
+    aplic.paddr_start = paddr;
+    aplic.paddr_end = paddr + size;
+    aplic.size = size;

Why do all three need recording? Isn't a (start,size) tuple sufficient (and unambiguous)?

xen/arch/riscv/aplic.c line 113

Quoted 11 lines
xen/arch/riscv/aplic.c +11 -0
+    aplic.regs = ioremap(paddr, size);
+    if ( !aplic.regs )
+        panic("%s: unable to map\n", node->full_name);
+
+    /* Setup initial state APLIC interrupts */
+    aplic_init_hw_interrupts();
+
+    return 0;
+}
+
+static const struct intc_hw_operations __ro_after_init aplic_ops = {

const or __ro_after_init?

Quoted 70 lines
xen/arch/riscv/include/asm/aplic.h new file +67 -0
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/aplic.h
+ *
+ * RISC-V Advanced Platform-Level Interrupt Controller support
+ *
+ * Copyright (c) 2023 Microchip.
+ */
+
+#ifndef ASM__RISCV__APLIC_H
+#define ASM__RISCV__APLIC_H
+
+#include <xen/types.h>
+
+#include <asm/imsic.h>
+
+#define APLIC_DOMAINCFG_IE      BIT(8, UL)
+#define APLIC_DOMAINCFG_DM      BIT(2, UL)
+
+struct aplic_regs {
+    uint32_t domaincfg;
+    uint32_t sourcecfg[1023];
+    uint8_t _reserved1[0xBC0];
+
+    uint32_t mmsiaddrcfg;
+    uint32_t mmsiaddrcfgh;
+    uint32_t smsiaddrcfg;
+    uint32_t smsiaddrcfgh;
+    uint8_t _reserved2[0x30];
+
+    uint32_t setip[32];
+    uint8_t _reserved3[92];
+
+    uint32_t setipnum;
+    uint8_t _reserved4[0x20];
+
+    uint32_t in_clrip[32];
+    uint8_t _reserved5[92];
+
+    uint32_t clripnum;
+    uint8_t _reserved6[32];
+
+    uint32_t setie[32];
+    uint8_t _reserved7[92];
+
+    uint32_t setienum;
+    uint8_t _reserved8[32];
+
+    uint32_t clrie[32];
+    uint8_t _reserved9[92];
+
+    uint32_t clrienum;
+    uint8_t _reserved10[32];
+
+    uint32_t setipnum_le;
+    uint32_t setipnum_be;
+    uint8_t _reserved11[4088];
+
+    uint32_t genmsi;
+    uint32_t target[1023];
+};
+
+struct aplic_priv {
+    /* base physical address and size */
+    paddr_t paddr_start;
+    paddr_t paddr_end;

xen/arch/riscv/include/asm/aplic.h line 77

Quoted 10 lines
xen/arch/riscv/include/asm/aplic.h new file +10 -0
+    size_t  size;
+
+    /* registers */
+    volatile struct aplic_regs *regs;
+
+    /* imsic configuration */
+    const struct imsic_config *imsic_cfg;
+};
+
+#endif /* ASM__RISCV__APLIC_H */

Does all of this really need to live in a non-private header?

xen/arch/riscv/include/asm/irq.h line 30

Quoted 7 lines
xen/arch/riscv/include/asm/irq.h +0 -1
@@ -27,7 +27,6 @@
 #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
 
 /* TODO */
-#define nr_irqs 0U

How come this is simply no longer needed, i.e. without any replacement? Hmm, looks like the only use in common code has gone away. Yet then this still doesn't really look to belong here (especially if not mentioned in the description).

Jan

#46 lore raw

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

Quoted 30 lines · Jan Beulich
Quoted 29 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +10 -0
@@ -9,19 +9,112 @@
   * Copyright (c) 2024-2025 Vates
   */
 
+#include <xen/device_tree.h>
  #include <xen/errno.h>
  #include <xen/init.h>
  #include <xen/irq.h>
+#include <xen/mm.h>
  #include <xen/sections.h>
  #include <xen/types.h>
+#include <xen/vmap.h>
 
+#include <asm/aplic.h>
  #include <asm/device.h>
+#include <asm/imsic.h>
  #include <asm/intc.h>
+#include <asm/riscv_encoding.h>
+
+#define APLIC_DEFAULT_PRIORITY  1
+
+static struct aplic_priv aplic;
 
  static struct intc_info __ro_after_init aplic_info = {
      .hw_version = INTC_APLIC,
  };
 
Quoted 14 lines

xen/arch/riscv/aplic.c line 47

Quoted 13 lines
xen/arch/riscv/aplic.c +13 -0
+static void __init aplic_init_hw_interrupts(void)
+{
+    int i;
+
+    /* Disable all interrupts */
+    for ( i = 0; i <= aplic_info.nr_irqs; i += 32 )
+        aplic.regs->clrie[i] = -1U;
+
+    /* Set interrupt type and default priority for all interrupts */
+    for ( i = 1; i <= aplic_info.nr_irqs; i++ )
+    {
+        aplic.regs->sourcecfg[i - 1] = 0;
+        aplic.regs->target[i - 1] = APLIC_DEFAULT_PRIORITY;

A field named "target" is written with a priority value?

Low bits of target register contains Interrupt Priority bits which can't be zero according to AIA spec:

```
4.5.16.1. Active source, direct delivery mode
For an active interrupt source , if the domain is configured in direct delivery mode (domaincfg.DM = 0),
then register target[ ] has this format:
  bits 31:18 Hart Index (WLRL)
  bits 7:0 IPRIO (WARL)

All other register bits are reserved and read as zeros.
Hart Index is a WLRL field that specifies the hart to which interrupts from this source will be delivered.
Field IPRIO (Interrupt Priority) specifies the priority number for the interrupt source. This field is
a WARL unsigned integer of IPRIOLEN bits, where IPRIOLEN is a constant parameter for the given APLIC, in
the range of 1 to 8. Only values 1 through are allowed for IPRIO, not zero. A write to a target register
sets IPRIO equal to bits :0 of the 32-bit value written, unless those bits are all zeros, in which case
the priority number is set to 1 instead. (If IPRIOLEN = 1, these rules cause IPRIO to be effectively
read-only with value 1.)
```
Quoted 6 lines

xen/arch/riscv/aplic.c line 51

Quoted 5 lines
xen/arch/riscv/aplic.c +4 -0
+    }
+
+    /* Clear APLIC domaincfg */
+    aplic.regs->domaincfg = APLIC_DOMAINCFG_IE | APLIC_DOMAINCFG_DM;

The statement doesn't like like there was any "clearing" here.

But all other bits, except|APLIC_DOMAINCFG_{IE, DM}|, are set to zero. I think we can remove this comment to avoid confusion.

Quoted 7 lines
Quoted 7 lines
xen/arch/riscv/aplic.c +7 -0
+    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
+    if ( !rc )
+        panic("%s: IDC mode not supported\n", node->full_name);
+
+    imsic_node = dt_find_node_by_phandle(imsic_phandle);
+    if ( !imsic_node )
+        panic("%s: unable to find IMSIC node\n", node->full_name);
Quoted 8 lines

xen/arch/riscv/aplic.c line 78

Quoted 7 lines
xen/arch/riscv/aplic.c +7 -0
+
+    /* check imsic mode */
+    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
+                                    irq_range, ARRAY_SIZE(irq_range));
+    if ( rc && (rc != -EOVERFLOW) )
+        panic("%s: unable to find interrupt-extended in %s node\n",
+               node->full_name, imsic_node->full_name);

Why exactly is EOVERFLOW tolerable here?

QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode. For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only interested in the S-mode IMSIC node.

The IMSIC node includes this information in the|"interrupts-extended"| property, which has the following format:
interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.

For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:

if ( irq_range[1] == IRQ_M_EXT )

Thereby dt_property_read_u32_array() will return -EOVERFLOW in the case when a platfrom has more then one CPU as we passed irq_range[2] as an argument but the amount of values in "interrupts-extended" property will be (2 * CPUS_NUM).

I can update the comment above dt_property_read_u32_array() for more clearness.

Quoted 16 lines
Quoted 16 lines
xen/arch/riscv/aplic.c +16 -0
+    if ( irq_range[1] == IRQ_M_EXT )
+        /* machine mode imsic node, ignore this aplic node */
+        return 0;
+
+    rc = imsic_init(imsic_node);
+    if ( rc )
+        panic("%s: Failded to initialize IMSIC\n", node->full_name);
+
+    /* Find out number of interrupt sources */
+    rc = dt_property_read_u32(node, "riscv,num-sources", &aplic_info.nr_irqs);
+    if ( !rc )
+        panic("%s: failed to get number of interrupt sources\n",
+              node->full_name);
+
+    prop = dt_get_property(node, "reg", NULL);
+    dt_get_range(&prop, node, &paddr, &size);
Quoted 8 lines

xen/arch/riscv/aplic.c line 101

Quoted 6 lines
xen/arch/riscv/aplic.c +6 -0
+    if ( !paddr )
+        panic("%s: first MMIO resource not found\n", node->full_name);
+
+    aplic.paddr_start = paddr;
+    aplic.paddr_end = paddr + size;
+    aplic.size = size;

Why do all three need recording? Isn't a (start,size) tuple sufficient (and unambiguous)?

(start,size) will be enough. I'll drop aplic.paddr_end.

Quoted 13 lines

xen/arch/riscv/aplic.c line 113

Quoted 12 lines
xen/arch/riscv/aplic.c +11 -0
+    aplic.regs = ioremap(paddr, size);
+    if ( !aplic.regs )
+        panic("%s: unable to map\n", node->full_name);
+
+    /* Setup initial state APLIC interrupts */
+    aplic_init_hw_interrupts();
+
+    return 0;
+}
+
+static const struct intc_hw_operations __ro_after_init aplic_ops = {

const or __ro_after_init?

What’s wrong with using both?|const| ensures the variable can't be changed at compile time, while|__ro_after_init| makes it read-only at runtime after initialization is complete.

Probably,|__initconst| would be a better fit:
static const struct intc_hw_operations __initconst aplic_ops = {

Or even|__initconstrel|, since the|struct intc_hw_operations| contains pointers.

Quoted 74 lines
Quoted 74 lines
xen/arch/riscv/include/asm/aplic.h new file +70 -0
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/aplic.h
+ *
+ * RISC-V Advanced Platform-Level Interrupt Controller support
+ *
+ * Copyright (c) 2023 Microchip.
+ */
+
+#ifndef ASM__RISCV__APLIC_H
+#define ASM__RISCV__APLIC_H
+
+#include <xen/types.h>
+
+#include <asm/imsic.h>
+
+#define APLIC_DOMAINCFG_IE      BIT(8, UL)
+#define APLIC_DOMAINCFG_DM      BIT(2, UL)
+
+struct aplic_regs {
+    uint32_t domaincfg;
+    uint32_t sourcecfg[1023];
+    uint8_t _reserved1[0xBC0];
+
+    uint32_t mmsiaddrcfg;
+    uint32_t mmsiaddrcfgh;
+    uint32_t smsiaddrcfg;
+    uint32_t smsiaddrcfgh;
+    uint8_t _reserved2[0x30];
+
+    uint32_t setip[32];
+    uint8_t _reserved3[92];
+
+    uint32_t setipnum;
+    uint8_t _reserved4[0x20];
+
+    uint32_t in_clrip[32];
+    uint8_t _reserved5[92];
+
+    uint32_t clripnum;
+    uint8_t _reserved6[32];
+
+    uint32_t setie[32];
+    uint8_t _reserved7[92];
+
+    uint32_t setienum;
+    uint8_t _reserved8[32];
+
+    uint32_t clrie[32];
+    uint8_t _reserved9[92];
+
+    uint32_t clrienum;
+    uint8_t _reserved10[32];
+
+    uint32_t setipnum_le;
+    uint32_t setipnum_be;
+    uint8_t _reserved11[4088];
+
+    uint32_t genmsi;
+    uint32_t target[1023];
+};
+
+struct aplic_priv {
+    /* base physical address and size */
+    paddr_t paddr_start;
+    paddr_t paddr_end;
+    size_t  size;
+
+    /* registers */
Quoted 8 lines

xen/arch/riscv/include/asm/aplic.h line 77

Quoted 7 lines
xen/arch/riscv/include/asm/aplic.h new file +7 -0
+    volatile struct aplic_regs *regs;
+
+    /* imsic configuration */
+    const struct imsic_config *imsic_cfg;
+};
+
+#endif /* ASM__RISCV__APLIC_H */

Does all of this really need to live in a non-private header?

struct aplic_priv is used in different files:
- in aplic.c to define aplic variable.
- in vaplic.c (which isn't intoduced yet) is used in several places:https://gitlab.com/xen-project/people/olkur/xen/-/blob/latest/xen/arch/riscv/vaplic.c#L41

struct aplic_regs is used only in aplic.c (at least, at the moment) so could be moved to aplic.c, but I don't see too much sense.

Quoted 6 lines
Quoted 6 lines
xen/arch/riscv/include/asm/irq.h +0 -0
@@ -27,7 +27,6 @@
  #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
 
Quoted 6 lines

xen/arch/riscv/include/asm/irq.h line 30

Quoted 2 lines
xen/arch/riscv/include/asm/irq.h +0 -1
  /* TODO */
-#define nr_irqs 0U

How come this is simply no longer needed, i.e. without any replacement? Hmm, looks like the only use in common code has gone away. Yet then this still doesn't really look to belong here (especially if not mentioned in the description).

I missed that it is used in xen/common/domain.c when CONFIG_HAS_PIRQ=y, but this config isn't selected for RISC-V.
I think that I have to revert this change.

~ Oleksii

#47 lore raw

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

Quoted 29 lines · Oleksii Kurochko
Quoted 16 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +4 -0
+    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
+    if ( !rc )
+        panic("%s: IDC mode not supported\n", node->full_name);
+

xen/arch/riscv/aplic.c line 78

Quoted 10 lines
xen/arch/riscv/aplic.c +10 -0
+    imsic_node = dt_find_node_by_phandle(imsic_phandle);
+    if ( !imsic_node )
+        panic("%s: unable to find IMSIC node\n", node->full_name);
+
+    /* check imsic mode */
+    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
+                                    irq_range, ARRAY_SIZE(irq_range));
+    if ( rc && (rc != -EOVERFLOW) )
+        panic("%s: unable to find interrupt-extended in %s node\n",
+               node->full_name, imsic_node->full_name);

Why exactly is EOVERFLOW tolerable here?

QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode. For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only interested in the S-mode IMSIC node.

The IMSIC node includes this information in the|"interrupts-extended"| property, which has the following format:
interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.

For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:

Quoted 7 lines
   if ( irq_range[1] == IRQ_M_EXT )

Thereby dt_property_read_u32_array() will return -EOVERFLOW in the case when a platfrom has more then one CPU as we passed irq_range[2] as an argument but the amount of values in "interrupts-extended" property will be (2 * CPUS_NUM).

I can update the comment above dt_property_read_u32_array() for more clearness.

Yet my question remains: Why would it be okay to ignore the remaining entries, and hence accept -EOVERFLOW as kind-of-success?

Quoted 13 lines
Quoted 4 lines
Quoted 4 lines
xen/arch/riscv/aplic.c +4 -0
+    aplic.regs = ioremap(paddr, size);
+    if ( !aplic.regs )
+        panic("%s: unable to map\n", node->full_name);
+
Quoted 8 lines

xen/arch/riscv/aplic.c line 113

Quoted 7 lines
xen/arch/riscv/aplic.c +7 -0
+    /* Setup initial state APLIC interrupts */
+    aplic_init_hw_interrupts();
+
+    return 0;
+}
+
+static const struct intc_hw_operations __ro_after_init aplic_ops = {

const or __ro_after_init?

Quoted 2 lines

What’s wrong with using both?|const| ensures the variable can't be changed at compile time, while|__ro_after_init| makes it read-only at runtime after initialization is complete.

No, const makes it read-only at compile- and run-time. __ro_after_init, putting the item into a special section, makes it writable at init-time. Due to the const, the compiler wouldn't emit any writes. But we can also avoid stray writes by having the item live in .rodata.

Quoted 4 lines

Probably,|__initconst| would be a better fit:

   static const struct intc_hw_operations __initconst aplic_ops = {

Or even|__initconstrel|, since the|struct intc_hw_operations| contains pointers.

Well, if this variable isn't accessed post-init, sure. That seems pretty unlikely though, considering it contains pointers to hook functions.

Quoted 82 lines
Quoted 73 lines
Quoted 73 lines
xen/arch/riscv/include/asm/aplic.h new file +70 -0
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * xen/arch/riscv/aplic.h
+ *
+ * RISC-V Advanced Platform-Level Interrupt Controller support
+ *
+ * Copyright (c) 2023 Microchip.
+ */
+
+#ifndef ASM__RISCV__APLIC_H
+#define ASM__RISCV__APLIC_H
+
+#include <xen/types.h>
+
+#include <asm/imsic.h>
+
+#define APLIC_DOMAINCFG_IE      BIT(8, UL)
+#define APLIC_DOMAINCFG_DM      BIT(2, UL)
+
+struct aplic_regs {
+    uint32_t domaincfg;
+    uint32_t sourcecfg[1023];
+    uint8_t _reserved1[0xBC0];
+
+    uint32_t mmsiaddrcfg;
+    uint32_t mmsiaddrcfgh;
+    uint32_t smsiaddrcfg;
+    uint32_t smsiaddrcfgh;
+    uint8_t _reserved2[0x30];
+
+    uint32_t setip[32];
+    uint8_t _reserved3[92];
+
+    uint32_t setipnum;
+    uint8_t _reserved4[0x20];
+
+    uint32_t in_clrip[32];
+    uint8_t _reserved5[92];
+
+    uint32_t clripnum;
+    uint8_t _reserved6[32];
+
+    uint32_t setie[32];
+    uint8_t _reserved7[92];
+
+    uint32_t setienum;
+    uint8_t _reserved8[32];
+
+    uint32_t clrie[32];
+    uint8_t _reserved9[92];
+
+    uint32_t clrienum;
+    uint8_t _reserved10[32];
+
+    uint32_t setipnum_le;
+    uint32_t setipnum_be;
+    uint8_t _reserved11[4088];
+
+    uint32_t genmsi;
+    uint32_t target[1023];
+};
+
+struct aplic_priv {
+    /* base physical address and size */
+    paddr_t paddr_start;
+    paddr_t paddr_end;
+    size_t  size;
+
+    /* registers */
Quoted 8 lines

xen/arch/riscv/include/asm/aplic.h line 77

Quoted 7 lines
xen/arch/riscv/include/asm/aplic.h new file +7 -0
+    volatile struct aplic_regs *regs;
+
+    /* imsic configuration */
+    const struct imsic_config *imsic_cfg;
+};
+
+#endif /* ASM__RISCV__APLIC_H */

Does all of this really need to live in a non-private header?

Quoted 3 lines

struct aplic_priv is used in different files:
- in aplic.c to define aplic variable.
- in vaplic.c (which isn't intoduced yet) is used in several places:https://gitlab.com/xen-project/people/olkur/xen/-/blob/latest/xen/arch/riscv/vaplic.c#L41

Which would still call for a private header (xen/arch/riscv/aplic.h).

Quoted 2 lines

struct aplic_regs is used only in aplic.c (at least, at the moment) so could be moved to aplic.c, but I don't see too much sense.

It is generally good practice to limit the scope of things as much as possible. Just to avoid (or make more noticeable) mis-uses or layering violations, for example.

Quoted 12 lines
Quoted 11 lines

xen/arch/riscv/include/asm/irq.h line 30

Quoted 7 lines
xen/arch/riscv/include/asm/irq.h +0 -1
@@ -27,7 +27,6 @@
  #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
 
  /* TODO */
-#define nr_irqs 0U

How come this is simply no longer needed, i.e. without any replacement? Hmm, looks like the only use in common code has gone away. Yet then this still doesn't really look to belong here (especially if not mentioned in the description).

Quoted 3 lines

I missed that it is used in xen/common/domain.c when CONFIG_HAS_PIRQ=y, but this config isn't selected for RISC-V.
I think that I have to revert this change.

I don't think you need to, as long as you don't mean to select HAS_PIRQ for RISC-V. It's just that the change looks entirely unrelated _here_.

Jan

#48 lore raw

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

Quoted 31 lines · Jan Beulich
Quoted 30 lines · Oleksii Kurochko
Quoted 16 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +4 -0
+    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
+    if ( !rc )
+        panic("%s: IDC mode not supported\n", node->full_name);
+

xen/arch/riscv/aplic.c line 78

Quoted 10 lines
xen/arch/riscv/aplic.c +10 -0
+    imsic_node = dt_find_node_by_phandle(imsic_phandle);
+    if ( !imsic_node )
+        panic("%s: unable to find IMSIC node\n", node->full_name);
+
+    /* check imsic mode */
+    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
+                                    irq_range, ARRAY_SIZE(irq_range));
+    if ( rc && (rc != -EOVERFLOW) )
+        panic("%s: unable to find interrupt-extended in %s node\n",
+               node->full_name, imsic_node->full_name);

Why exactly is EOVERFLOW tolerable here?

QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode. For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only interested in the S-mode IMSIC node.

The IMSIC node includes this information in the|"interrupts-extended"| property, which has the following format:
interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.

For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:

    if ( irq_range[1] == IRQ_M_EXT )
Quoted 7 lines
Quoted 5 lines

Thereby dt_property_read_u32_array() will return -EOVERFLOW in the case when a platfrom has more then one CPU as we passed irq_range[2] as an argument but the amount of values in "interrupts-extended" property will be (2 * CPUS_NUM).

I can update the comment above dt_property_read_u32_array() for more clearness.

Yet my question remains: Why would it be okay to ignore the remaining entries, and hence accept -EOVERFLOW as kind-of-success?

Because for other entries the IMSIC mode will be the same and the difference will be only in interrupt controller's phandle which we don't care about in this function and cares only about in imisic_init(), look at usage of imsic_get_parent_hartid().

Quoted 13 lines
Quoted 13 lines
Quoted 13 lines

xen/arch/riscv/aplic.c line 113

Quoted 12 lines
xen/arch/riscv/aplic.c +11 -0
+    aplic.regs = ioremap(paddr, size);
+    if ( !aplic.regs )
+        panic("%s: unable to map\n", node->full_name);
+
+    /* Setup initial state APLIC interrupts */
+    aplic_init_hw_interrupts();
+
+    return 0;
+}
+
+static const struct intc_hw_operations __ro_after_init aplic_ops = {

const or __ro_after_init?

Quoted 6 lines
Quoted 2 lines

What’s wrong with using both?|const| ensures the variable can't be changed at compile time, while|__ro_after_init| makes it read-only at runtime after initialization is complete.

No, const makes it read-only at compile- and run-time.__ro_after_init, putting the item into a special section, makes it writable at init-time. Due to the const, the compiler wouldn't emit any writes. But we can also avoid stray writes by having the item live in .rodata.

Oh, right, const will add the variable to .rodata.
Then I think it is enough to have const as aplic_ops is going to be initialized once and then only read will happen.

Quoted 7 lines
Quoted 5 lines

Probably,|__initconst| would be a better fit:

    static const struct intc_hw_operations __initconst aplic_ops = {

Or even|__initconstrel|, since the|struct intc_hw_operations| contains pointers.

Well, if this variable isn't accessed post-init, sure. That seems pretty unlikely though, considering it contains pointers to hook functions.

Sure, .init section is going to be freed after init-time.

~ Oleksii

#49 lore raw

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

Quoted 37 lines · Oleksii Kurochko
Quoted 35 lines · Jan Beulich
Quoted 34 lines · Oleksii Kurochko
Quoted 16 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +4 -0
+    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
+    if ( !rc )
+        panic("%s: IDC mode not supported\n", node->full_name);
+

xen/arch/riscv/aplic.c line 78

Quoted 10 lines
xen/arch/riscv/aplic.c +10 -0
+    imsic_node = dt_find_node_by_phandle(imsic_phandle);
+    if ( !imsic_node )
+        panic("%s: unable to find IMSIC node\n", node->full_name);
+
+    /* check imsic mode */
+    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
+                                    irq_range, ARRAY_SIZE(irq_range));
+    if ( rc && (rc != -EOVERFLOW) )
+        panic("%s: unable to find interrupt-extended in %s node\n",
+               node->full_name, imsic_node->full_name);

Why exactly is EOVERFLOW tolerable here?

QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode. For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only interested in the S-mode IMSIC node.

The IMSIC node includes this information in the|"interrupts-extended"| property, which has the following format:
interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.

For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:

    if ( irq_range[1] == IRQ_M_EXT )

Thereby dt_property_read_u32_array() will return -EOVERFLOW in the case when a platfrom has more then one CPU as we passed irq_range[2] as an argument but the amount of values in "interrupts-extended" property will be (2 * CPUS_NUM).

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

I can update the comment above dt_property_read_u32_array() for more clearness.

Yet my question remains: Why would it be okay to ignore the remaining entries, and hence accept -EOVERFLOW as kind-of-success?

Because for other entries the IMSIC mode will be the same and the difference will be only in interrupt controller's phandle

And we can blindly take this for granted? Would you mind extending the comment that's there to include this aspect?

Jan

Quoted 2 lines

which we don't care about in this function and cares only about in imisic_init(), look at usage of imsic_get_parent_hartid().

#50 lore raw

Re: [PATCH v1 09/14] xen/riscv: aplic_init() implementation

Quoted 37 lines · Jan Beulich
Quoted 36 lines · Oleksii Kurochko
Quoted 35 lines · Jan Beulich
Quoted 34 lines · Oleksii Kurochko
Quoted 16 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/arch/riscv/aplic.c +4 -0
+    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
+    if ( !rc )
+        panic("%s: IDC mode not supported\n", node->full_name);
+

xen/arch/riscv/aplic.c line 78

Quoted 10 lines
xen/arch/riscv/aplic.c +10 -0
+    imsic_node = dt_find_node_by_phandle(imsic_phandle);
+    if ( !imsic_node )
+        panic("%s: unable to find IMSIC node\n", node->full_name);
+
+    /* check imsic mode */
+    rc = dt_property_read_u32_array(imsic_node, "interrupts-extended",
+                                    irq_range, ARRAY_SIZE(irq_range));
+    if ( rc && (rc != -EOVERFLOW) )
+        panic("%s: unable to find interrupt-extended in %s node\n",
+               node->full_name, imsic_node->full_name);

Why exactly is EOVERFLOW tolerable here?

QEMU generates two IMSIC device tree nodes: one for M-mode and one for S-mode. For the hypervisor, we don’t really care about the M-mode IMSIC node — we're only interested in the S-mode IMSIC node.

The IMSIC node includes this information in the|"interrupts-extended"| property, which has the following format:
interrupt-extended = {<interrupt-controller-phandle>, <machine_mode>},...
The number of such|<phandle, mode>| pairs depends on the number of CPUs the platform has.

For our purposes, to determine whether the IMSIC node corresponds to M-mode or not, it’s sufficient to read only the first pair and check the mode like this:

     if ( irq_range[1] == IRQ_M_EXT )

Thereby dt_property_read_u32_array() will return -EOVERFLOW in the case when a platfrom has more then one CPU as we passed irq_range[2] as an argument but the amount of values in "interrupts-extended" property will be (2 * CPUS_NUM).

Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line

I can update the comment above dt_property_read_u32_array() for more clearness.

Yet my question remains: Why would it be okay to ignore the remaining entries, and hence accept -EOVERFLOW as kind-of-success?

Because for other entries the IMSIC mode will be the same and the difference will be only in interrupt controller's phandle

And we can blindly take this for granted? Would you mind extending the comment that's there to include this aspect?

I tried to compile dtc with different modes in interrupt-extends and compilation doesn't failed, so it's not really granted.
Just to be sure, I'll check all items of interrupts-extend not just the first one.

~ Oleksii

Quoted 2 lines
Quoted 2 lines

which we don't care about in this function and cares only about in imisic_init(), look at usage of imsic_get_parent_hartid().

#51 lore raw

PATCH 10/14 xen/riscv: implementation of aplic and imsic operations

In reply to Oleksii Kurochko

Introduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
- aplic_irq_startup()
- aplic_irq_shutdown()
- aplic_irq_enable()
- aplic_irq_disable()
- aplic_irq_ack()
- aplic_host_irq_end()
- aplic_set_irq_affinity()

As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().

For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is introduced to get hart id.

Also, introduce additional interrupt controller h/w operations and host_irq_type for APLIC:
- aplic_host_irq_type
- aplic_set_irq_priority()
- aplic_set_irq_type()

Patch is based on the code from [1].

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7390e2365828b83e27ead56b03114a56e3699dd5

---
 xen/arch/riscv/aplic.c             | 169 ++++++++++++++++++++++++++++-
 xen/arch/riscv/imsic.c             |  63 +++++++++++
 xen/arch/riscv/include/asm/aplic.h |  12 ++
 xen/arch/riscv/include/asm/imsic.h |  15 +++
 4 files changed, 258 insertions(+), 1 deletion(-)
xen/arch/riscv/aplic.c +168 -1
index d1aa835c3e..4b60cb9a77 100644
@@ -15,6 +15,7 @@
 #include <xen/irq.h>
 #include <xen/mm.h>
 #include <xen/sections.h>
+#include <xen/spinlock.h>
 #include <xen/types.h>
 #include <xen/vmap.h>
 
@@ -110,9 +111,173 @@ static int __init aplic_init(void)
     return 0;
 }
 
-static const struct intc_hw_operations __ro_after_init aplic_ops = {
+static void aplic_set_irq_type(struct irq_desc *desc, unsigned int type)
+{
+    unsigned int irq = desc->irq - 1;
+
+    spin_lock(&aplic.lock);
+    switch(type) {
+        case IRQ_TYPE_EDGE_RISING:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_RISE;
+            break;
+        case IRQ_TYPE_EDGE_FALLING:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_FALL;
+            break;
+        case IRQ_TYPE_LEVEL_HIGH:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_HIGH;
+            break;
+        case IRQ_TYPE_LEVEL_LOW:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_LOW;
+            break;
+        default:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_INACTIVE;
+            break;
+    }
+    spin_unlock(&aplic.lock);
+}
+
+static void aplic_set_irq_priority(struct irq_desc *desc,
+                                   unsigned int priority)
+{
+    /* No priority, do nothing */
+}
+
+static void aplic_irq_enable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);
+
+    ASSERT(spin_is_locked(&desc->lock));
+
+    spin_lock_irqsave(&aplic.lock, flags);
+
+    clear_bit(_IRQ_DISABLED, &desc->status);
+
+    /* enable interrupt in IMSIC */
+    imsic_irq_enable(desc->irq);
+
+    /* enable interrupt in APLIC */
+    aplic.regs->setienum = desc->irq;
+
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static void aplic_irq_disable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);
+
+    ASSERT(spin_is_locked(&desc->lock));
+
+    spin_lock_irqsave(&aplic.lock, flags);
+
+    set_bit(_IRQ_DISABLED, &desc->status);
+
+    /* disable interrupt in APLIC */
+    aplic.regs->clrienum = desc->irq;
+
+    /* disable interrupt in IMSIC */
+    imsic_irq_disable(desc->irq);
+
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static unsigned int aplic_irq_startup(struct irq_desc *desc)
+{
+    aplic_irq_enable(desc);
+
+    return 0;
+}
+
+static void aplic_irq_shutdown(struct irq_desc *desc)
+{
+    aplic_irq_disable(desc);
+}
+
+static void aplic_irq_ack(struct irq_desc *desc)
+{
+    /* nothing to do */
+}
+
+static void aplic_host_irq_end(struct irq_desc *desc)
+{
+    /* nothing to do */
+}
+
+static unsigned int aplic_get_cpu_from_mask(const cpumask_t *cpumask)
+{
+    unsigned int cpu;
+    cpumask_t possible_mask;
+
+    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
+    cpu = cpumask_any(&possible_mask);
+
+    return cpu;
+}
+
+static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
+{
+    unsigned int cpu;
+    uint64_t group_index, base_ppn;
+    uint32_t hhxw, lhxw ,hhxs, value;
+    const struct imsic_config *imsic = aplic.imsic_cfg;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);
+
+    ASSERT(!cpumask_empty(mask));
+
+    spin_lock(&aplic.lock);
+
+    cpu = cpuid_to_hartid(aplic_get_cpu_from_mask(mask));
+    hhxw = imsic->group_index_bits;
+    lhxw = imsic->hart_index_bits;
+    hhxs = imsic->group_index_shift - IMSIC_MMIO_PAGE_SHIFT * 2;
+    base_ppn = imsic->msi[cpu].base_addr >> IMSIC_MMIO_PAGE_SHIFT;
+
+    /* update hart and EEID in the target register */
+    group_index = (base_ppn >> (hhxs + 12)) & (BIT(hhxw, UL) - 1);
+    value = desc->irq;
+    value |= cpu << APLIC_TARGET_HART_IDX_SHIFT;
+    value |= group_index << (lhxw + APLIC_TARGET_HART_IDX_SHIFT) ;
+    aplic.regs->target[desc->irq - 1] = value;
+
+    spin_unlock(&aplic.lock);
+}
+
+static hw_irq_controller aplic_host_irq_type = {
+    .typename     = "aplic",
+    .startup      = aplic_irq_startup,
+    .shutdown     = aplic_irq_shutdown,
+    .enable       = aplic_irq_enable,
+    .disable      = aplic_irq_disable,
+    .ack          = aplic_irq_ack,
+    .end          = aplic_host_irq_end,
+    .set_affinity = aplic_set_irq_affinity,
+};
+
+static const struct intc_hw_operations aplic_ops = {
     .info                = &aplic_info,
     .init                = aplic_init,
+    .host_irq_type       = &aplic_host_irq_type,
+    .set_irq_priority    = aplic_set_irq_priority,
+    .set_irq_type        = aplic_set_irq_type,
 };
 
 static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
@@ -149,6 +314,8 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 
     dt_irq_xlate = aplic_irq_xlate;
 
+    spin_lock_init(&aplic.lock);
+
     register_intc_ops(&aplic_ops);
 
     return 0;
xen/arch/riscv/imsic.c +63 -0

81 lines not on this page · raw message

xen/arch/riscv/include/asm/aplic.h +12 -0
index 94b3d0b616..ce858663a9 100644
@@ -18,6 +18,15 @@
 #define APLIC_DOMAINCFG_IE      BIT(8, UL)
 #define APLIC_DOMAINCFG_DM      BIT(2, UL)
 
+#define APLIC_SOURCECFG_SM_INACTIVE     0x0
+#define APLIC_SOURCECFG_SM_DETACH       0x1
+#define APLIC_SOURCECFG_SM_EDGE_RISE    0x4
+#define APLIC_SOURCECFG_SM_EDGE_FALL    0x5
+#define APLIC_SOURCECFG_SM_LEVEL_HIGH   0x6
+#define APLIC_SOURCECFG_SM_LEVEL_LOW    0x7
+
+#define APLIC_TARGET_HART_IDX_SHIFT 18
+
 struct aplic_regs {
     uint32_t domaincfg;
     uint32_t sourcecfg[1023];
@@ -70,6 +79,9 @@ struct aplic_priv {
     /* registers */
     volatile struct aplic_regs *regs;
 
+    /* lock */
+    spinlock_t lock;
+
     /* imsic configuration */
     const struct imsic_config *imsic_cfg;
 };
xen/arch/riscv/include/asm/imsic.h +15 -0
index 126e651863..d2c0178529 100644
@@ -11,6 +11,7 @@
 #ifndef ASM__RISCV__IMSIC_H
 #define ASM__RISCV__IMSIC_H
 
+#include <xen/spinlock.h>
 #include <xen/types.h>
 
 #define IMSIC_MMIO_PAGE_SHIFT   12
@@ -19,6 +20,11 @@
 #define IMSIC_MIN_ID            63
 #define IMSIC_MAX_ID            2048
 
+#define IMSIC_EIP0              0x80
+#define IMSIC_EIPx_BITS         32
+
+#define IMSIC_EIE0              0xC0
+
 struct imsic_msi {
     paddr_t base_addr;
     unsigned long offset;
@@ -55,6 +61,12 @@ struct imsic_config {
 
     /* MSI */
     struct imsic_msi msi[NR_CPUS];
+
+    /* a check that IMSIC is used */
+    bool is_used;
+
+    /* lock */
+    spinlock_t lock;
 };
 
 struct dt_device_node;
@@ -63,4 +75,7 @@ int imsic_init(struct dt_device_node *n);
 struct imsic_config;
 const struct imsic_config *imsic_get_config(void);
 
+void imsic_irq_enable(unsigned int hwirq);
+void imsic_irq_disable(unsigned int hwirq);
+
 #endif /* ASM__RISCV__IMSIC_H */
Signature · 1 line
2.49.0
#52 lore raw

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

Quoted 14 lines · Oleksii Kurochko

Introduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
- aplic_irq_startup()
- aplic_irq_shutdown()
- aplic_irq_enable()
- aplic_irq_disable()
- aplic_irq_ack()
- aplic_host_irq_end()
- aplic_set_irq_affinity()

As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().

Quoted 8 lines

For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is introduced to get hart id.

Also, introduce additional interrupt controller h/w operations and host_irq_type for APLIC:
- aplic_host_irq_type
- aplic_set_irq_priority()
- aplic_set_irq_type()

Yet these two functions nor the hooks they're used to populate are entirely unused here. Since they're also outside of the common IRQ handling machinery, it's unclear how one would sanely ack such a change.

Quoted 11 lines
xen/arch/riscv/aplic.c +1 -0
@@ -15,6 +15,7 @@
 #include <xen/irq.h>
 #include <xen/mm.h>
 #include <xen/sections.h>
+#include <xen/spinlock.h>
 #include <xen/types.h>
 #include <xen/vmap.h>
 

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 116

Quoted 7 lines
xen/arch/riscv/aplic.c +3 -1
@@ -110,9 +111,173 @@ static int __init aplic_init(void)
     return 0;
 }
 
-static const struct intc_hw_operations __ro_after_init aplic_ops = {
+static void aplic_set_irq_type(struct irq_desc *desc, unsigned int type)
+{
+    unsigned int irq = desc->irq - 1;

Why this adjustment by 1 (and yet both items being named "irq")?

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 120

Quoted 3 lines
xen/arch/riscv/aplic.c +3 -0
+    spin_lock(&aplic.lock);
+    switch(type) {
+        case IRQ_TYPE_EDGE_RISING:

Nit (style): Missing blanks, brace on its own line, case labels indented like their containing switch().

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 123

Quoted 3 lines
xen/arch/riscv/aplic.c +3 -0
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_RISE;
+            break;
+        case IRQ_TYPE_EDGE_FALLING:

Blank lines please between non-fall-through case blocks.

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 134

Quoted 11 lines
xen/arch/riscv/aplic.c +11 -0
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_FALL;
+            break;
+        case IRQ_TYPE_LEVEL_HIGH:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_HIGH;
+            break;
+        case IRQ_TYPE_LEVEL_LOW:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_LOW;
+            break;
+        default:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_INACTIVE;
+            break;

Is the default: label legitimate to be reached?

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143

Quoted 9 lines
xen/arch/riscv/aplic.c +9 -0
+    }
+    spin_unlock(&aplic.lock);
+}
+
+static void aplic_set_irq_priority(struct irq_desc *desc,
+                                   unsigned int priority)
+{
+    /* No priority, do nothing */
+}

Since the function dopes nothing, wouldn't it be better to omit it and have the (future) caller check for a NULL pointer ahead of making the (indirect) call? Same remark for other handlers (below) which also do nothing.

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154

Quoted 10 lines
xen/arch/riscv/aplic.c +10 -0
+static void aplic_irq_enable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);

Such an extra field, used only for assertions, is pretty odd. Can't you use any of the other fields to achieve the same effect?

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 153

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    ASSERT(spin_is_locked(&desc->lock));

If this lock (which is an IRQ-safe one) is necessarily held, ...

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    spin_lock_irqsave(&aplic.lock, flags);

... you can use just spin_lock() here.

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 160

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    clear_bit(_IRQ_DISABLED, &desc->status);

Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)

I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.

In any event this can be done outside of the APLIC-locked region, I think.

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 162

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    /* enable interrupt in IMSIC */

May I remind you of Xen comment style?

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 166

Quoted 4 lines
xen/arch/riscv/aplic.c +4 -0
+    imsic_irq_enable(desc->irq);
+
+    /* enable interrupt in APLIC */
+    aplic.regs->setienum = desc->irq;

Are you sure you want to use plain assignments for MMIO accesses? I'd have expected writel() to be used here. (And only later I realized that I didn't spot the same already higher up from here.)

From the vague understanding I've gained so far: Isn't the APLIC closer to the CPU and the IMSIC closer to the device? If so, wouldn't you want to enable at the APLIC before enabling at the IMSIC? But of course that also depends on what exactly happens in the window while one is already enabled and the other is still disabled. (Later) From the code you add to imsic.c it looks like it's the other way around, as the IMSIC is accessed through CSRs.

Quoted 30 lines
xen/arch/riscv/aplic.c +30 -0
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static void aplic_irq_disable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);
+
+    ASSERT(spin_is_locked(&desc->lock));
+
+    spin_lock_irqsave(&aplic.lock, flags);
+
+    set_bit(_IRQ_DISABLED, &desc->status);
+
+    /* disable interrupt in APLIC */
+    aplic.regs->clrienum = desc->irq;
+
+    /* disable interrupt in IMSIC */
+    imsic_irq_disable(desc->irq);
+
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static unsigned int aplic_irq_startup(struct irq_desc *desc)

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 207

Quoted 10 lines
xen/arch/riscv/aplic.c +10 -0
+{
+    aplic_irq_enable(desc);
+
+    return 0;
+}
+
+static void aplic_irq_shutdown(struct irq_desc *desc)
+{
+    aplic_irq_disable(desc);
+}

You don't really need a separate hook function here, do you?

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 214

Quoted 6 lines
xen/arch/riscv/aplic.c +6 -0
+static void aplic_irq_ack(struct irq_desc *desc)
+{
+    /* nothing to do */
+}
+
+static void aplic_host_irq_end(struct irq_desc *desc)

What's the "host" in the identifier about?

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 221

Quoted 7 lines
xen/arch/riscv/aplic.c +7 -0
+{
+    /* nothing to do */
+}
+
+static unsigned int aplic_get_cpu_from_mask(const cpumask_t *cpumask)
+{
+    unsigned int cpu;

No real need for this variable?

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 225

Quoted 4 lines
xen/arch/riscv/aplic.c +4 -0
+    cpumask_t possible_mask;
+
+    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
+    cpu = cpumask_any(&possible_mask);

Why would you use cpu_possible_map here? That includes any offline CPUs. I think you need to use cpu_online_map here.

Quoted 6 lines
xen/arch/riscv/aplic.c +6 -0
+    return cpu;
+}
+
+static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
+{
+    unsigned int cpu;

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 242

Quoted 10 lines
xen/arch/riscv/aplic.c +10 -0
+    uint64_t group_index, base_ppn;
+    uint32_t hhxw, lhxw ,hhxs, value;
+    const struct imsic_config *imsic = aplic.imsic_cfg;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);

Use the local variable you have made yourself?

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 246

Quoted 3 lines
xen/arch/riscv/aplic.c +3 -0
+    ASSERT(!cpumask_empty(mask));
+
+    spin_lock(&aplic.lock);

Aiui the lock can be acquired quite a bit later. It ought to be needed only around the actual write to the hardware register.

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 255

Quoted 8 lines
xen/arch/riscv/aplic.c +8 -0
+    cpu = cpuid_to_hartid(aplic_get_cpu_from_mask(mask));
+    hhxw = imsic->group_index_bits;
+    lhxw = imsic->hart_index_bits;
+    hhxs = imsic->group_index_shift - IMSIC_MMIO_PAGE_SHIFT * 2;
+    base_ppn = imsic->msi[cpu].base_addr >> IMSIC_MMIO_PAGE_SHIFT;
+
+    /* update hart and EEID in the target register */
+    group_index = (base_ppn >> (hhxs + 12)) & (BIT(hhxw, UL) - 1);

What's this magic 12 in here? Not IMSIC_MMIO_PAGE_SHIFT I suppose?

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 264

Quoted 9 lines
xen/arch/riscv/aplic.c +9 -0
+    value = desc->irq;
+    value |= cpu << APLIC_TARGET_HART_IDX_SHIFT;
+    value |= group_index << (lhxw + APLIC_TARGET_HART_IDX_SHIFT) ;
+    aplic.regs->target[desc->irq - 1] = value;
+
+    spin_unlock(&aplic.lock);
+}
+
+static hw_irq_controller aplic_host_irq_type = {

const?

Quoted 19 lines
xen/arch/riscv/imsic.c +7 -0
@@ -14,12 +14,68 @@
 #include <xen/errno.h>
 #include <xen/init.h>
 #include <xen/macros.h>
+#include <xen/spinlock.h>
 #include <xen/xmalloc.h>
 
 #include <asm/imsic.h>
 
 static struct imsic_config imsic_cfg;
 
+#define imsic_csr_set(c, v)     \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_set(CSR_SIREG, v);      \
+} while (0)
+

xen/arch/riscv/imsic.c line 39

Quoted 10 lines
xen/arch/riscv/imsic.c +10 -0
+#define imsic_csr_clear(c, v)   \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_clear(CSR_SIREG, v);    \
+} while (0)
+
+static void imsic_local_eix_update(unsigned long base_id, unsigned long num_id,
+                                   bool pend, bool val)
+{
+    unsigned long i, isel, ireg;

These can be constrained to inside the outer loop below.

xen/arch/riscv/imsic.c line 46

Quoted 7 lines
xen/arch/riscv/imsic.c +7 -0
+    unsigned long id = base_id, last_id = base_id + num_id;
+
+    while ( id < last_id )
+    {
+        isel = id / __riscv_xlen;
+        isel *= __riscv_xlen / IMSIC_EIPx_BITS;
+        isel += (pend) ? IMSIC_EIP0 : IMSIC_EIE0;

Nit: Why the parentheses?

xen/arch/riscv/imsic.c line 52

Quoted 5 lines
xen/arch/riscv/imsic.c +5 -0
+        ireg = 0;
+        for ( i = id & (__riscv_xlen - 1);
+              (id < last_id) && (i < __riscv_xlen);
+              i++, id++ )
+            ireg |= (1 << i);

I wonder if this calculation really needs a loop. Afaict it's just a consecutive set of bits you mean to set.

xen/arch/riscv/imsic.c line 66

Quoted 13 lines
xen/arch/riscv/imsic.c +13 -0
+        if ( val )
+            imsic_csr_set(isel, ireg);
+        else
+            imsic_csr_clear(isel, ireg);
+    }
+}
+
+void imsic_irq_enable(unsigned int hwirq)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&imsic_cfg.lock, flags);
+    imsic_local_eix_update(hwirq, 1, false, true);

No subtraction of 1 here? Also, why "hwirq" and not just "irq"?

Quoted 16 lines
xen/arch/riscv/imsic.c +12 -0
+    spin_unlock_irqrestore(&imsic_cfg.lock, flags);
+}
+
+void imsic_irq_disable(unsigned int hwirq)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&imsic_cfg.lock, flags);
+    imsic_local_eix_update(hwirq, 1, false, false);
+    spin_unlock_irqrestore(&imsic_cfg.lock, flags);
+}
+
 const struct imsic_config *imsic_get_config(void)
 {
     return &imsic_cfg;

xen/arch/riscv/imsic.c int __init imsic_init(struct dt_device_node *node) · line 339

Quoted 7 lines
xen/arch/riscv/imsic.c +4 -0
@@ -277,6 +333,13 @@ int __init imsic_init(struct dt_device_node *node)
         goto imsic_init_err;
     }
 
+    spin_lock_init(&imsic_cfg.lock);
+
+    /* Enable local interrupt delivery */
+    imsic_ids_local_delivery(true);

What's this? I can't find the function/macro here, nor in patch 08, nor in staging.

Jan

#53 lore raw

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

Quoted 18 lines · Jan Beulich
Quoted 17 lines · Oleksii Kurochko

Introduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
- aplic_irq_startup()
- aplic_irq_shutdown()
- aplic_irq_enable()
- aplic_irq_disable()
- aplic_irq_ack()
- aplic_host_irq_end()
- aplic_set_irq_affinity()

As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().

For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is introduced to get hart id.

Quoted 8 lines
Quoted 5 lines

Also, introduce additional interrupt controller h/w operations and host_irq_type for APLIC:
- aplic_host_irq_type
- aplic_set_irq_priority()
- aplic_set_irq_type()

Yet these two functions nor the hooks they're used to populate are entirely unused here. Since they're also outside of the common IRQ handling machinery, it's unclear how one would sanely ack such a change.

They will be called by intc_route_irq_to_xen() from setup_irq() during firt time the IRQ is setup.

Quoted 15 lines
Quoted 15 lines
xen/arch/riscv/aplic.c +1 -0
@@ -15,6 +15,7 @@
  #include <xen/irq.h>
  #include <xen/mm.h>
  #include <xen/sections.h>
+#include <xen/spinlock.h>
  #include <xen/types.h>
  #include <xen/vmap.h>
 
@@ -110,9 +111,173 @@ static int __init aplic_init(void)
      return 0;
  }
 
Quoted 5 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 116

Quoted 4 lines
xen/arch/riscv/aplic.c +3 -1
-static const struct intc_hw_operations __ro_after_init aplic_ops = {
+static void aplic_set_irq_type(struct irq_desc *desc, unsigned int type)
+{
+    unsigned int irq = desc->irq - 1;

Why this adjustment by 1 (and yet both items being named "irq")?

Interrupt 0 isn't possible based on the spec:

```
Each of an APLIC’s interrupt sources has a fixed unique identity number in the range 1 to N, where N is the total number of sources at the APLIC. The number zero is not a valid interrupt identity number at an APLIC. The maximum number of interrupt sources an APLIC may support is 1023. ``` and interrupt 1 will correspond to bit 0 in sourcecfg[] register, interrupt 2 ->sourcecfg[1] and so on. And that is the reason why we need -1.

>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_FALL;
>> +            break;
>> +        case IRQ_TYPE_LEVEL_HIGH:
>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_HIGH;
>> +            break;
>> +        case IRQ_TYPE_LEVEL_LOW:
>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_LOW;
>> +            break;
>> +        default:
>> +            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_INACTIVE;
>> +            break;
> Is the default: label legitimate to be reached?

From the spec:
```

0 Inactive Inactive in this domain (and not delegated) 1 Detached Active, detached from the source wire 2–3 — Reserved 4 Edge1 Active, edge-sensitive; interrupt asserted on rising edge 5 Edge0 Active, edge-sensitive; interrupt asserted on falling edge 6 Level1 Active, level-sensitive; interrupt asserted when high 7 Level0 Active, level-sensitive; interrupt asserted when low ``` It seems to me like APLIC_SOURCECFG_SM_INACTIVE just covers cases (0-3) and inactive IRQ pretty safe to as a default value.

Quoted 13 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143

Quoted 10 lines
xen/arch/riscv/aplic.c +9 -0
+    }
+    spin_unlock(&aplic.lock);
+}
+
+static void aplic_set_irq_priority(struct irq_desc *desc,
+                                   unsigned int priority)
+{
+    /* No priority, do nothing */
+}

Since the function dopes nothing, wouldn't it be better to omit it and have the (future) caller check for a NULL pointer ahead of making the (indirect) call? Same remark for other handlers (below) which also do nothing.

I thought about that too, but it could be some cases when the stub is introduced with temporary BUG_ON("unimplemented") inside just to not miss to implement it when it will be necessary.
If we will have only the caller check then we could miss to implement such stubs.

Quoted 13 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154

Quoted 11 lines
xen/arch/riscv/aplic.c +10 -0
+static void aplic_irq_enable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);

Such an extra field, used only for assertions, is pretty odd. Can't you use any of the other fields to achieve the same effect?

in aplic_init() there is:

    /* check for associated imsic node */
    rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
    if ( !rc )
        panic("%s: IDC mode not supported\n", node->full_name);

So we will have panic() anyway if MSI mode isn't supported. As an option we can just drop the ASSERT.

Or introduce static variable in aplic.c aplic_mode, init it in aplic_init() and use it in ASSERT().

Quoted 10 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 136

Quoted 2 lines
xen/arch/riscv/aplic.c +1 -0
+    ASSERT(spin_is_locked(&desc->lock));

If this lock (which is an IRQ-safe one) is necessarily held, ...

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 137

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    spin_lock_irqsave(&aplic.lock, flags);

... you can use just spin_lock() here.

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 160

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    clear_bit(_IRQ_DISABLED, &desc->status);

Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)

I haven't thought about that. Likely non-atomic bitop could be used here.

Quoted 5 lines

I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.

In any event this can be done outside of the APLIC-locked region, I think.

Considering that we are doing that under desc->lock, agree we can move that outside the APLIC-locked region.

Quoted 7 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 166

Quoted 4 lines
xen/arch/riscv/aplic.c +4 -0
+    imsic_irq_enable(desc->irq);
+
+    /* enable interrupt in APLIC */
+    aplic.regs->setienum = desc->irq;

Are you sure you want to use plain assignments for MMIO accesses? I'd have expected writel() to be used here. (And only later I realized that I didn't spot the same already higher up from here.)

Good point. I have to update that with writel()...

Quoted 8 lines

From the vague understanding I've gained so far: Isn't the APLIC closer to
the CPU and the IMSIC closer to the device? If so, wouldn't you want to enable at the APLIC before enabling at the IMSIC? But of course that also depends on what exactly happens in the window while one is already enabled and the other is still disabled. (Later) From the code you add to imsic.c it looks like it's the other way around, as the IMSIC is accessed through CSRs.

From the AIA spec:

```
An Incoming MSI Controller (IMSIC) is an optional RISC-V hardware component
that is closely coupled with a hart, one IMSIC per hart. An IMSIC receives
and records incoming message-signaled interrupts (MSIs) for a hart, and
signals to the hart when there are pending and enabled interrupts to be
serviced.
```

Based on the figure 2 (Interrupt delivery by MSIs when harts have IMSICs for receiving them) of AIA spechttps://github.com/riscv/riscv-aia/blob/main/src/intrsWithIMSICs.png IMSIC is more close to CPU and APLIC is more close to the device. The external interrupt controller is APLIC and it only sends a MSI message for a CPU.

The logical flow of an interrupt to a hart with an IMSIC would be:
1. A physical interrupt signal arrives at the APLIC.
2. The APLIC, if configured for MSI delivery mode (domaincfg.DM = 1) and if the specific interrupt source is active and enabled within its domain (controlled by sourcecfg[i] and the global Interrupt Enable bit IE in domaincfg), will generate an MSI.
3. This MSI is then sent to the target hart's IMSIC. The APLIC needs to know the MSI target address for each hart, which can be hardwired or configured through registers like mmsiaddrcfg and mmsiaddrcfgh.
4. The receiving hart's IMSIC records this MSI as a pending interrupt.
5. If the corresponding interrupt identity is enabled within the IMSIC's interrupt file, the IMSIC will signal the hart, typically by setting the MEIP or SEIP bit in the mip CSR (or sip CSR).

Generally, I think that the order in which enable interrupts doesn't really matter as if you were to enable the IMSIC to receive a certain interrupt before the APLIC was configured to send it (or had a pending interrupt from the device), the IMSIC would simply be waiting for an MSI that wouldn't arrive.
Similarly, if the APLIC sends an MSI for an interrupt that is not enabled in the IMSIC, the interrupt would remain pending in the IMSIC but wouldn't trigger an interrupt at the hart.

IMO, the order which is used now in the code is pretty logical.

Does it make sense?

Quoted 34 lines
Quoted 34 lines
xen/arch/riscv/aplic.c +33 -0
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static void aplic_irq_disable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);
+
+    ASSERT(spin_is_locked(&desc->lock));
+
+    spin_lock_irqsave(&aplic.lock, flags);
+
+    set_bit(_IRQ_DISABLED, &desc->status);
+
+    /* disable interrupt in APLIC */
+    aplic.regs->clrienum = desc->irq;
+
+    /* disable interrupt in IMSIC */
+    imsic_irq_disable(desc->irq);
+
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static unsigned int aplic_irq_startup(struct irq_desc *desc)
+{
+    aplic_irq_enable(desc);
+
Quoted 8 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 207

Quoted 7 lines
xen/arch/riscv/aplic.c +7 -0
+    return 0;
+}
+
+static void aplic_irq_shutdown(struct irq_desc *desc)
+{
+    aplic_irq_disable(desc);
+}

You don't really need a separate hook function here, do you?

With such implementation it is really not needed to have a hook so I will drop it.

Quoted 7 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 214

Quoted 6 lines
xen/arch/riscv/aplic.c +6 -0
+static void aplic_irq_ack(struct irq_desc *desc)
+{
+    /* nothing to do */
+}
+
+static void aplic_host_irq_end(struct irq_desc *desc)

What's the "host" in the identifier about?

It was copied that from Arm and my understanding that it means Xen-related IRQ as they also have:

```
/* XXX different for level vs edge */
static hw_irq_controller gicv2_host_irq_type = {
...
    .end          = gicv2_host_irq_end,
...
};

static hw_irq_controller gicv2_guest_irq_type = {
...
    .end          = gicv2_guest_irq_end,
...
};
```
Quoted 9 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 221

Quoted 8 lines
xen/arch/riscv/aplic.c +7 -0
+{
+    /* nothing to do */
+}
+
+static unsigned int aplic_get_cpu_from_mask(const cpumask_t *cpumask)
+{
+    unsigned int cpu;

No real need for this variable?

Yes, it could be dropped.

Quoted 7 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 225

Quoted 5 lines
xen/arch/riscv/aplic.c +4 -0
+    cpumask_t possible_mask;
+
+    cpumask_and(&possible_mask, cpumask, &cpu_possible_map);
+    cpu = cpumask_any(&possible_mask);

Why would you use cpu_possible_map here? That includes any offline CPUs. I think you need to use cpu_online_map here.

It makes sense, Ill switch to cpu_online_map.

Quoted 10 lines
Quoted 10 lines
xen/arch/riscv/aplic.c +9 -0
+    return cpu;
+}
+
+static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
+{
+    unsigned int cpu;
+    uint64_t group_index, base_ppn;
+    uint32_t hhxw, lhxw ,hhxs, value;
+    const struct imsic_config *imsic = aplic.imsic_cfg;
Quoted 8 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154

Quoted 7 lines
xen/arch/riscv/aplic.c +7 -0
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);

Use the local variable you have made yourself?

What do you mean by local here?

Quoted 15 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 246

Quoted 4 lines
xen/arch/riscv/aplic.c +3 -0
+    ASSERT(!cpumask_empty(mask));
+
+    spin_lock(&aplic.lock);

Aiui the lock can be acquired quite a bit later. It ought to be needed only around the actual write to the hardware register.

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 255

Quoted 8 lines
xen/arch/riscv/aplic.c +8 -0
+    cpu = cpuid_to_hartid(aplic_get_cpu_from_mask(mask));
+    hhxw = imsic->group_index_bits;
+    lhxw = imsic->hart_index_bits;
+    hhxs = imsic->group_index_shift - IMSIC_MMIO_PAGE_SHIFT * 2;
+    base_ppn = imsic->msi[cpu].base_addr >> IMSIC_MMIO_PAGE_SHIFT;
+
+    /* update hart and EEID in the target register */
+    group_index = (base_ppn >> (hhxs + 12)) & (BIT(hhxw, UL) - 1);
Quoted 1 line

What's this magic 12 in here? Not IMSIC_MMIO_PAGE_SHIFT I suppose?

In the AIA spec they are using 12 explicitly:https://github.com/riscv/riscv-aia/blob/main/src/AdvPLIC.adoc#AdvPLIC-MSIAddrs

Quoted 9 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 233

Quoted 7 lines
xen/arch/riscv/aplic.c +7 -0
+    unsigned long id = base_id, last_id = base_id + num_id;
+
+    while ( id < last_id )
+    {
+        isel = id / __riscv_xlen;
+        isel *= __riscv_xlen / IMSIC_EIPx_BITS;
+        isel += (pend) ? IMSIC_EIP0 : IMSIC_EIE0;

Nit: Why the parentheses?

Quoted 7 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 238

Quoted 5 lines
xen/arch/riscv/aplic.c +5 -0
+        ireg = 0;
+        for ( i = id & (__riscv_xlen - 1);
+              (id < last_id) && (i < __riscv_xlen);
+              i++, id++ )
+            ireg |= (1 << i);

I wonder if this calculation really needs a loop. Afaict it's just a consecutive set of bits you mean to set.

Good point. I will double-check.

Quoted 15 lines
Quoted 4 lines
xen/arch/riscv/aplic.c +3 -0
+        if ( val )
+            imsic_csr_set(isel, ireg);
+        else

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 251

Quoted 10 lines
xen/arch/riscv/aplic.c +10 -0
+            imsic_csr_clear(isel, ireg);
+    }
+}
+
+void imsic_irq_enable(unsigned int hwirq)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&imsic_cfg.lock, flags);
+    imsic_local_eix_update(hwirq, 1, false, true);

No subtraction of 1 here? Also, why "hwirq" and not just "irq"?

From the spec:

```

When an interrupt file supports distinct interrupt identities, valid identity numbers are between 1
and inclusive. The identity numbers within this range are said to be implemented by the interrupt
file; numbers outside this range are not implemented. The number zero is never a valid interrupt
identity.
...

Bit positions in a valid eiek register that don’t correspond to a supported interrupt identity (such as bit 0 of eie0) are read-only zeros.


```

So in EIx registers interrupt i corresponds to bit i in comparison wiht APLIC's sourcecfg which starts from 0.

Quoted 20 lines
Quoted 20 lines
xen/arch/riscv/aplic.c +12 -0
+    spin_unlock_irqrestore(&imsic_cfg.lock, flags);
+}
+
+void imsic_irq_disable(unsigned int hwirq)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&imsic_cfg.lock, flags);
+    imsic_local_eix_update(hwirq, 1, false, false);
+    spin_unlock_irqrestore(&imsic_cfg.lock, flags);
+}
+
  const struct imsic_config *imsic_get_config(void)
  {
      return &imsic_cfg;
@@ -277,6 +333,13 @@ int __init imsic_init(struct dt_device_node *node)
          goto imsic_init_err;
      }
 
Quoted 6 lines

xen/arch/riscv/aplic.c int __init imsic_init(struct dt_device_node *node) · line 339

Quoted 4 lines
xen/arch/riscv/aplic.c +4 -0
+    spin_lock_init(&imsic_cfg.lock);
+
+    /* Enable local interrupt delivery */
+    imsic_ids_local_delivery(true);

What's this? I can't find the function/macro here, nor in patch 08, nor in staging.

It is defined in imsic.c:

```
void imsic_ids_local_delivery(bool enable)
{
    if ( enable )
    {
        imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD);
        imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY);
    }
    else
    {
        imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD);
        imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY);
    }
}
```

Thanks for review.

~ Oleksii

#54 lore raw

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

Quoted 24 lines · Oleksii Kurochko
Quoted 23 lines · Jan Beulich
Quoted 22 lines · Oleksii Kurochko

Introduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
- aplic_irq_startup()
- aplic_irq_shutdown()
- aplic_irq_enable()
- aplic_irq_disable()
- aplic_irq_ack()
- aplic_host_irq_end()
- aplic_set_irq_affinity()

As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().

For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is introduced to get hart id.

Also, introduce additional interrupt controller h/w operations and host_irq_type for APLIC:
- aplic_host_irq_type
- aplic_set_irq_priority()
- aplic_set_irq_type()

Quoted 6 lines
Quoted 3 lines

Yet these two functions nor the hooks they're used to populate are entirely unused here. Since they're also outside of the common IRQ handling machinery, it's unclear how one would sanely ack such a change.

They will be called by intc_route_irq_to_xen() from setup_irq() during firt time the IRQ is setup.

Perhaps move their introduction to there then? We don't do any Misra checking yet lon RISC-V, but imo it's still good practice to avoid introducing new violations, even if only temporarily.

Quoted 20 lines
Quoted 14 lines
Quoted 14 lines
xen/arch/riscv/aplic.c +1 -0
@@ -15,6 +15,7 @@
  #include <xen/irq.h>
  #include <xen/mm.h>
  #include <xen/sections.h>
+#include <xen/spinlock.h>
  #include <xen/types.h>
  #include <xen/vmap.h>
 
@@ -110,9 +111,173 @@ static int __init aplic_init(void)
      return 0;
  }
 
Quoted 5 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 116

Quoted 4 lines
xen/arch/riscv/aplic.c +3 -1
-static const struct intc_hw_operations __ro_after_init aplic_ops = {
+static void aplic_set_irq_type(struct irq_desc *desc, unsigned int type)
+{
+    unsigned int irq = desc->irq - 1;

Why this adjustment by 1 (and yet both items being named "irq")?

Quoted 8 lines

Interrupt 0 isn't possible based on the spec:
```
Each of an APLIC’s interrupt sources has a fixed unique identity number in the range 1 to N, where N is the total number of sources at the APLIC. The number zero is not a valid interrupt identity number at an APLIC. The maximum number of interrupt sources an APLIC may support is 1023. ``` and interrupt 1 will correspond to bit 0 in sourcecfg[] register, interrupt 2 ->sourcecfg[1] and so on. And that is the reason why we need -1.

Okay, fine. But what about the part of the question in parentheses?

Quoted 13 lines
Quoted 11 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 134

Quoted 11 lines
xen/arch/riscv/aplic.c +11 -0
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_FALL;
+            break;
+        case IRQ_TYPE_LEVEL_HIGH:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_HIGH;
+            break;
+        case IRQ_TYPE_LEVEL_LOW:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_LOW;
+            break;
+        default:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_INACTIVE;
+            break;
Quoted 1 line

Is the default: label legitimate to be reached?

Quoted 10 lines

From the spec:
```
0 Inactive Inactive in this domain (and not delegated) 1 Detached Active, detached from the source wire 2–3 — Reserved 4 Edge1 Active, edge-sensitive; interrupt asserted on rising edge 5 Edge0 Active, edge-sensitive; interrupt asserted on falling edge 6 Level1 Active, level-sensitive; interrupt asserted when high 7 Level0 Active, level-sensitive; interrupt asserted when low ``` It seems to me like APLIC_SOURCECFG_SM_INACTIVE just covers cases (0-3) and inactive IRQ pretty safe to as a default value.

I fear this doesn't answer my question, which is to a large part related to the Xen code, and only to some degree to the spec.

Quoted 9 lines
Quoted 8 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143

Quoted 5 lines
xen/arch/riscv/aplic.c +5 -0
+static void aplic_set_irq_priority(struct irq_desc *desc,
+                                   unsigned int priority)
+{
+    /* No priority, do nothing */
+}

Since the function dopes nothing, wouldn't it be better to omit it and have the (future) caller check for a NULL pointer ahead of making the (indirect) call? Same remark for other handlers (below) which also do nothing.

Quoted 4 lines

I thought about that too, but it could be some cases when the stub is introduced with temporary BUG_ON("unimplemented") inside just to not miss to implement it when it will be necessary.
If we will have only the caller check then we could miss to implement such stubs.

I guess I don't understand the concern.

Quoted 13 lines
Quoted 4 lines
Quoted 4 lines
xen/arch/riscv/aplic.c +4 -0
+static void aplic_irq_enable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
Quoted 8 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154

Quoted 6 lines
xen/arch/riscv/aplic.c +6 -0
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);

Such an extra field, used only for assertions, is pretty odd. Can't you use any of the other fields to achieve the same effect?

Quoted 8 lines

in aplic_init() there is:

     /* check for associated imsic node */
     rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
     if ( !rc )
         panic("%s: IDC mode not supported\n", node->full_name);

So we will have panic() anyway if MSI mode isn't supported. As an option we can just drop the ASSERT.

Since they serve primarily as a reminder where changes would need making, I'd prefer if they could be kept.

Quoted 2 lines

Or introduce static variable in aplic.c aplic_mode, init it in aplic_init() and use it in ASSERT().

This would then again be used solely for assertions, aiui? As said, I think it would be preferable if some already existing indicator could be used for this purpose.

Quoted 11 lines
Quoted 9 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    ASSERT(spin_is_locked(&desc->lock));

If this lock (which is an IRQ-safe one) is necessarily held, ...

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 144

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    spin_lock_irqsave(&aplic.lock, flags);

... you can use just spin_lock() here.

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 160

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    clear_bit(_IRQ_DISABLED, &desc->status);

Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)

I haven't thought about that. Likely non-atomic bitop could be used here.

And then - does it need to be a bitop? Aiui that's what Arm uses, while x86 doesn't. And I see no reason to use other than plain C operators here. If Arm was switched, presumably all the redundant (and misnamed) _IRQ_* constants could go away, with just the IRQ_* ones left.

Quoted 2 lines
Quoted 2 lines

I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.

What about this part?

Quoted 59 lines
Quoted 1 line

In any event this can be done outside of the APLIC-locked region, I think.

Considering that we are doing that under desc->lock, agree we can move that outside the APLIC-locked region.

Quoted 7 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 166

Quoted 4 lines
xen/arch/riscv/aplic.c +4 -0
+    imsic_irq_enable(desc->irq);
+
+    /* enable interrupt in APLIC */
+    aplic.regs->setienum = desc->irq;

Are you sure you want to use plain assignments for MMIO accesses? I'd have expected writel() to be used here. (And only later I realized that I didn't spot the same already higher up from here.)

Good point. I have to update that with writel()...

Quoted 8 lines

From the vague understanding I've gained so far: Isn't the APLIC closer to
the CPU and the IMSIC closer to the device? If so, wouldn't you want to enable at the APLIC before enabling at the IMSIC? But of course that also depends on what exactly happens in the window while one is already enabled and the other is still disabled. (Later) From the code you add to imsic.c it looks like it's the other way around, as the IMSIC is accessed through CSRs.

From the AIA spec:

```
An Incoming MSI Controller (IMSIC) is an optional RISC-V hardware component
that is closely coupled with a hart, one IMSIC per hart. An IMSIC receives
and records incoming message-signaled interrupts (MSIs) for a hart, and
signals to the hart when there are pending and enabled interrupts to be
serviced.
```

Based on the figure 2 (Interrupt delivery by MSIs when harts have IMSICs for receiving them) of AIA spechttps://github.com/riscv/riscv-aia/blob/main/src/intrsWithIMSICs.png IMSIC is more close to CPU and APLIC is more close to the device. The external interrupt controller is APLIC and it only sends a MSI message for a CPU.

The logical flow of an interrupt to a hart with an IMSIC would be:
1. A physical interrupt signal arrives at the APLIC.
2. The APLIC, if configured for MSI delivery mode (domaincfg.DM = 1) and if the specific interrupt source is active and enabled within its domain (controlled by sourcecfg[i] and the global Interrupt Enable bit IE in domaincfg), will generate an MSI.
3. This MSI is then sent to the target hart's IMSIC. The APLIC needs to know the MSI target address for each hart, which can be hardwired or configured through registers like mmsiaddrcfg and mmsiaddrcfgh.
4. The receiving hart's IMSIC records this MSI as a pending interrupt.
5. If the corresponding interrupt identity is enabled within the IMSIC's interrupt file, the IMSIC will signal the hart, typically by setting the MEIP or SEIP bit in the mip CSR (or sip CSR).

Generally, I think that the order in which enable interrupts doesn't really matter as if you were to enable the IMSIC to receive a certain interrupt before the APLIC was configured to send it (or had a pending interrupt from the device), the IMSIC would simply be waiting for an MSI that wouldn't arrive.
Similarly, if the APLIC sends an MSI for an interrupt that is not enabled in the IMSIC, the interrupt would remain pending in the IMSIC but wouldn't trigger an interrupt at the hart.

Quoted 3 lines

IMO, the order which is used now in the code is pretty logical.

Does it make sense?

Except for the "doesn't really matter" - yes. In a reply to a later patch I indicated I realized that IMSIC is what's closer to the CPU (and hence later in the chain of interrupt delivery actions).

Quoted 55 lines
Quoted 33 lines
Quoted 33 lines
xen/arch/riscv/aplic.c +33 -0
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static void aplic_irq_disable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);
+
+    ASSERT(spin_is_locked(&desc->lock));
+
+    spin_lock_irqsave(&aplic.lock, flags);
+
+    set_bit(_IRQ_DISABLED, &desc->status);
+
+    /* disable interrupt in APLIC */
+    aplic.regs->clrienum = desc->irq;
+
+    /* disable interrupt in IMSIC */
+    imsic_irq_disable(desc->irq);
+
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static unsigned int aplic_irq_startup(struct irq_desc *desc)
+{
+    aplic_irq_enable(desc);
+
Quoted 8 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 207

Quoted 7 lines
xen/arch/riscv/aplic.c +7 -0
+    return 0;
+}
+
+static void aplic_irq_shutdown(struct irq_desc *desc)
+{
+    aplic_irq_disable(desc);
+}

You don't really need a separate hook function here, do you?

With such implementation it is really not needed to have a hook so I will drop it.

Quoted 7 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 214

Quoted 6 lines
xen/arch/riscv/aplic.c +6 -0
+static void aplic_irq_ack(struct irq_desc *desc)
+{
+    /* nothing to do */
+}
+
+static void aplic_host_irq_end(struct irq_desc *desc)

What's the "host" in the identifier about?

It was copied that from Arm and my understanding that it means Xen-related IRQ as they also have:

Quoted 14 lines
```
/* XXX different for level vs edge */
static hw_irq_controller gicv2_host_irq_type = {
...
     .end          = gicv2_host_irq_end,
...
};

static hw_irq_controller gicv2_guest_irq_type = {
...
     .end          = gicv2_guest_irq_end,
...
};
```

And you expect to end up with a similar distinction on RISC-V? There's nothing like that on x86, just to mention it.

Quoted 15 lines
Quoted 6 lines
Quoted 6 lines
xen/arch/riscv/aplic.c +6 -0
+static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
+{
+    unsigned int cpu;
+    uint64_t group_index, base_ppn;
+    uint32_t hhxw, lhxw ,hhxs, value;
+    const struct imsic_config *imsic = aplic.imsic_cfg;
Quoted 8 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154

Quoted 7 lines
xen/arch/riscv/aplic.c +7 -0
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);

Use the local variable you have made yourself?

Quoted 1 line

What do you mean by local here?

Just a few lines up you latch aplic.imsic_cfg into the local "imsic".

Quoted 14 lines
Quoted 14 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 246

Quoted 3 lines
xen/arch/riscv/aplic.c +3 -0
+    ASSERT(!cpumask_empty(mask));
+
+    spin_lock(&aplic.lock);

Aiui the lock can be acquired quite a bit later. It ought to be needed only around the actual write to the hardware register.

Quoted 8 lines
xen/arch/riscv/aplic.c +8 -0
+    cpu = cpuid_to_hartid(aplic_get_cpu_from_mask(mask));
+    hhxw = imsic->group_index_bits;
+    lhxw = imsic->hart_index_bits;
+    hhxs = imsic->group_index_shift - IMSIC_MMIO_PAGE_SHIFT * 2;
+    base_ppn = imsic->msi[cpu].base_addr >> IMSIC_MMIO_PAGE_SHIFT;
+
+    /* update hart and EEID in the target register */
+    group_index = (base_ppn >> (hhxs + 12)) & (BIT(hhxw, UL) - 1);
Quoted 3 lines
Quoted 1 line

What's this magic 12 in here? Not IMSIC_MMIO_PAGE_SHIFT I suppose?

In the AIA spec they are using 12 explicitly:https://github.com/riscv/riscv-aia/blob/main/src/AdvPLIC.adoc#AdvPLIC-MSIAddrs

In the spec that's fine, but please make yourself a constant with a suitable name then, to be used here. Just consider what would happen if we used literal 12 everywhere PAGE_SHIFT was meant.

Quoted 23 lines
Quoted 7 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 225

Quoted 6 lines
xen/arch/riscv/aplic.c +6 -0
+void imsic_irq_enable(unsigned int hwirq)
+{
+    unsigned long flags;
+
+    spin_lock_irqsave(&imsic_cfg.lock, flags);
+    imsic_local_eix_update(hwirq, 1, false, true);

No subtraction of 1 here? Also, why "hwirq" and not just "irq"?

From the spec:

```

When an interrupt file supports distinct interrupt identities, valid identity numbers are between 1
and inclusive. The identity numbers within this range are said to be implemented by the interrupt
file; numbers outside this range are not implemented. The number zero is never a valid interrupt
identity.
...

Bit positions in a valid eiek register that don’t correspond to a 
supported interrupt identity (such as bit 0 of eie0) are read-only zeros.


```
Quoted 1 line

So in EIx registers interrupt i corresponds to bit i in comparison wiht APLIC's sourcecfg which starts from 0.

Confusing, but what do you do.

Quoted 12 lines
Quoted 10 lines

xen/arch/riscv/aplic.c int __init imsic_init(struct dt_device_node *node) · line 339

Quoted 8 lines
xen/arch/riscv/aplic.c +4 -0
@@ -277,6 +333,13 @@ int __init imsic_init(struct dt_device_node *node)
          goto imsic_init_err;
      }
 
+    spin_lock_init(&imsic_cfg.lock);
+
+    /* Enable local interrupt delivery */
+    imsic_ids_local_delivery(true);

What's this? I can't find the function/macro here, nor in patch 08, nor in staging.

It is defined in imsic.c:

Quoted 15 lines
```
void imsic_ids_local_delivery(bool enable)
{
     if ( enable )
     {
         imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD);
         imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY);
     }
     else
     {
         imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD);
         imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY);
     }
}
```

No, it's not. As noted in the reply to a later patch, it's only introduced there. Hence the build will break between the two patches.

Jan

#55 lore raw

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

Quoted 25 lines · Jan Beulich
Quoted 24 lines · Oleksii Kurochko
Quoted 23 lines · Jan Beulich
Quoted 22 lines · Oleksii Kurochko

Introduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
- aplic_irq_startup()
- aplic_irq_shutdown()
- aplic_irq_enable()
- aplic_irq_disable()
- aplic_irq_ack()
- aplic_host_irq_end()
- aplic_set_irq_affinity()

As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().

For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is introduced to get hart id.

Also, introduce additional interrupt controller h/w operations and host_irq_type for APLIC:
- aplic_host_irq_type
- aplic_set_irq_priority()
- aplic_set_irq_type()

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines

Yet these two functions nor the hooks they're used to populate are entirely unused here. Since they're also outside of the common IRQ handling machinery, it's unclear how one would sanely ack such a change.

They will be called by intc_route_irq_to_xen() from setup_irq() during firt time the IRQ is setup.

Perhaps move their introduction to there then? We don't do any Misra checking yet lon RISC-V, but imo it's still good practice to avoid introducing new violations, even if only temporarily.

Okay, I will move their introduction to there.

Btw, what is needed to add Misra checking for RISC-V? I started to think that, probably, it will make sense to do that from the start.

Quoted 27 lines
Quoted 19 lines
Quoted 14 lines
Quoted 14 lines
xen/arch/riscv/aplic.c +1 -0
@@ -15,6 +15,7 @@
   #include <xen/irq.h>
   #include <xen/mm.h>
   #include <xen/sections.h>
+#include <xen/spinlock.h>
   #include <xen/types.h>
   #include <xen/vmap.h>
 
@@ -110,9 +111,173 @@ static int __init aplic_init(void)
       return 0;
   }
 
Quoted 5 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 116

Quoted 4 lines
xen/arch/riscv/aplic.c +3 -1
-static const struct intc_hw_operations __ro_after_init aplic_ops = {
+static void aplic_set_irq_type(struct irq_desc *desc, unsigned int type)
+{
+    unsigned int irq = desc->irq - 1;

Why this adjustment by 1 (and yet both items being named "irq")?

Quoted 8 lines

Interrupt 0 isn't possible based on the spec:
```
Each of an APLIC’s interrupt sources has a fixed unique identity number in the range 1 to N, where N is the total number of sources at the APLIC. The number zero is not a valid interrupt identity number at an APLIC. The maximum number of interrupt sources an APLIC may support is 1023. ``` and interrupt 1 will correspond to bit 0 in sourcecfg[] register, interrupt 2 ->sourcecfg[1] and so on. And that is the reason why we need -1.

Quoted 1 line

Okay, fine. But what about the part of the question in parentheses?

Sorry, missed to write that I'll change irq to irq_bit or something like that.

Quoted 22 lines
Quoted 12 lines
Quoted 11 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 134

Quoted 11 lines
xen/arch/riscv/aplic.c +11 -0
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_EDGE_FALL;
+            break;
+        case IRQ_TYPE_LEVEL_HIGH:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_HIGH;
+            break;
+        case IRQ_TYPE_LEVEL_LOW:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_LEVEL_LOW;
+            break;
+        default:
+            aplic.regs->sourcecfg[irq] = APLIC_SOURCECFG_SM_INACTIVE;
+            break;
Quoted 1 line

Is the default: label legitimate to be reached?

Quoted 10 lines

From the spec:
```
0 Inactive Inactive in this domain (and not delegated) 1 Detached Active, detached from the source wire 2–3 — Reserved 4 Edge1 Active, edge-sensitive; interrupt asserted on rising edge 5 Edge0 Active, edge-sensitive; interrupt asserted on falling edge 6 Level1 Active, level-sensitive; interrupt asserted when high 7 Level0 Active, level-sensitive; interrupt asserted when low ``` It seems to me like APLIC_SOURCECFG_SM_INACTIVE just covers cases (0-3) and inactive IRQ pretty safe to as a default value.

Quoted 2 lines

I fear this doesn't answer my question, which is to a large part related to the Xen code, and only to some degree to the spec.

From Xen code point of view, I am not sure if it legitimate or not. I've not any issue, at the moment, with such implementation, but to be on a safe side I'll implement default case as panic("...").

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143

Quoted 5 lines
xen/arch/riscv/aplic.c +5 -0
+static void aplic_set_irq_priority(struct irq_desc *desc,
+                                   unsigned int priority)
+{
+    /* No priority, do nothing */
+}

Since the function dopes nothing, wouldn't it be better to omit it and have the (future) caller check for a NULL pointer ahead of making the (indirect) call? Same remark for other handlers (below) which also do nothing.

Quoted 5 lines
Quoted 4 lines

I thought about that too, but it could be some cases when the stub is introduced with temporary BUG_ON("unimplemented") inside just to not miss to implement it when it will be necessary.
If we will have only the caller check then we could miss to implement such stubs.

I guess I don't understand the concern.

for example, if we will have the following code:
void some_caller(struct irq_desc *desc)

  {
      if ( desc->handler->set_affinity )
          desc->handler->set_affinity(desc, cpu_mask);

}

Then we will skip the call of handler->set_affinity() (if it was just initialized with .set_affinity = NULL) without any notification. And it is fine specifically in this case as aplic_set_irq_priority() does nothing.

But what about the cases if it is a function which will have some implementation in the future but doesn't have implementation for now. Then without notification that this function is unimplemented we could skip something what really matters.

But I think that your initial comment was just about the function which basically does nothing. Am i right?

Quoted 20 lines
Quoted 12 lines
Quoted 4 lines
Quoted 4 lines
xen/arch/riscv/aplic.c +4 -0
+static void aplic_irq_enable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
Quoted 8 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 154

Quoted 6 lines
xen/arch/riscv/aplic.c +6 -0
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);

Such an extra field, used only for assertions, is pretty odd. Can't you use any of the other fields to achieve the same effect?

Quoted 8 lines

in aplic_init() there is:

      /* check for associated imsic node */
      rc = dt_property_read_u32(node, "msi-parent", &imsic_phandle);
      if ( !rc )
          panic("%s: IDC mode not supported\n", node->full_name);

So we will have panic() anyway if MSI mode isn't supported. As an option we can just drop the ASSERT.

Quoted 8 lines

Since they serve primarily as a reminder where changes would need making, I'd prefer if they could be kept.

Quoted 2 lines

Or introduce static variable in aplic.c aplic_mode, init it in aplic_init() and use it in ASSERT().

This would then again be used solely for assertions, aiui? As said, I think it would be preferable if some already existing indicator could be used for this purpose.

I think that not solely, for example, if IMSIC isn't available then we should skip the calls of imsic_irq_enable(), at least, and this variable could be used for that purpose.

But I will double check if we have better indicator. At the moment, I don't think we have better, probably, except checking of aplic.regs->domaincfg if bit APLIC_DOMAINCFG_DM is set.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    ASSERT(spin_is_locked(&desc->lock));

If this lock (which is an IRQ-safe one) is necessarily held, ...

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 144

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    spin_lock_irqsave(&aplic.lock, flags);

... you can use just spin_lock() here.

Quoted 8 lines
Quoted 4 lines
Quoted 3 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 160

Quoted 1 line
xen/arch/riscv/aplic.c +1 -0
+    clear_bit(_IRQ_DISABLED, &desc->status);

Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)

I haven't thought about that. Likely non-atomic bitop could be used here.

And then - does it need to be a bitop? Aiui that's what Arm uses, while x86 doesn't. And I see no reason to use other than plain C operators here. If Arm was switched, presumably all the redundant (and misnamed) _IRQ_* constants could go away, with just the IRQ_* ones left.

The reason for a bitop in Arm is explained in this commithttps://gitlab.com/xen-project/xen/-/commit/50d8fe8fcbab2440cfeeb65c4765868398652473
but all the places where plain C operators were changed to bitops are actually executed under|spin_lock_irqsave(&desc->lock, flags). By quick look I found only two places one in __setup_irq() but it is called by the functions which do ||spin_lock_irqsave(&desc->lock, flags) and in vgic_v2_fold_lr_state(). Maybe, I'm missing something.| |RISC-V won't have something similar to ||vgic_v2_fold_lr_state|(), but __setup_irq() is used in a similar way. It can be added ASSERT(spin_is_lock(&desc->lock))
and then it will also safe to use non-bitop function.
Probably, it is a little bit safer to use always bitops for desc->status.
||

Quoted 3 lines
Quoted 2 lines
Quoted 2 lines

I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.

What about this part?

As I understand, based on Arm, code then Xen enables interrupts corresponding to devices assigned to dom0/domU before booting dom0/domU, resulting in the possibility of receiving an interrupt and not knowing what to do with it. So it is needed for enablement of IRQs when the guest requests it and not unconditionally at boot time.

Quoted 61 lines
Quoted 61 lines
Quoted 33 lines
Quoted 33 lines
xen/arch/riscv/aplic.c +33 -0
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static void aplic_irq_disable(struct irq_desc *desc)
+{
+    unsigned long flags;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);
+
+    ASSERT(spin_is_locked(&desc->lock));
+
+    spin_lock_irqsave(&aplic.lock, flags);
+
+    set_bit(_IRQ_DISABLED, &desc->status);
+
+    /* disable interrupt in APLIC */
+    aplic.regs->clrienum = desc->irq;
+
+    /* disable interrupt in IMSIC */
+    imsic_irq_disable(desc->irq);
+
+    spin_unlock_irqrestore(&aplic.lock, flags);
+}
+
+static unsigned int aplic_irq_startup(struct irq_desc *desc)
+{
+    aplic_irq_enable(desc);
+
Quoted 8 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 207

Quoted 7 lines
xen/arch/riscv/aplic.c +7 -0
+    return 0;
+}
+
+static void aplic_irq_shutdown(struct irq_desc *desc)
+{
+    aplic_irq_disable(desc);
+}

You don't really need a separate hook function here, do you?

With such implementation it is really not needed to have a hook so I will drop it.

Quoted 7 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 214

Quoted 6 lines
xen/arch/riscv/aplic.c +6 -0
+static void aplic_irq_ack(struct irq_desc *desc)
+{
+    /* nothing to do */
+}
+
+static void aplic_host_irq_end(struct irq_desc *desc)

What's the "host" in the identifier about?

It was copied that from Arm and my understanding that it means Xen-related IRQ as they also have:
```
/* XXX different for level vs edge */
static hw_irq_controller gicv2_host_irq_type = {
...
.end = gicv2_host_irq_end,
...
};

Quoted 8 lines
Quoted 6 lines

static hw_irq_controller gicv2_guest_irq_type = {
...
.end = gicv2_guest_irq_end,
...
};
```

And you expect to end up with a similar distinction on RISC-V? There's nothing like that on x86, just to mention it.

Yes, if one day support for guest interrupts without IMSIC support will be added for APLIC. (at the moment, we are planning only to have APLIC+IMSIC support as this way is hypervisor-aware)

Quoted 14 lines
Quoted 14 lines
Quoted 14 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 242

Quoted 13 lines
xen/arch/riscv/aplic.c +13 -0
+static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
+{
+    unsigned int cpu;
+    uint64_t group_index, base_ppn;
+    uint32_t hhxw, lhxw ,hhxs, value;
+    const struct imsic_config *imsic = aplic.imsic_cfg;
+
+    /*
+     * TODO: Currently, APLIC is supported only with MSI interrupts.
+     *       If APLIC without MSI interrupts is required in the future,
+     *       this function will need to be updated accordingly.
+     */
+    ASSERT(aplic.imsic_cfg->is_used);

Use the local variable you have made yourself?

Quoted 2 lines
Quoted 1 line

What do you mean by local here?

Just a few lines up you latch aplic.imsic_cfg into the local "imsic".

Oh, sure, if *->is_used will still present in the next patch series then I'll re-use here "imsic".

~ Oleksii

#56 lore raw

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

Quoted 31 lines · Oleksii Kurochko
Quoted 30 lines · Jan Beulich
Quoted 29 lines · Oleksii Kurochko
Quoted 18 lines · Jan Beulich
Quoted 17 lines · Oleksii Kurochko

Introduce interrupt controller descriptor for host APLIC to describe the low-lovel hardare. It includes implementation of the following functions:
- aplic_irq_startup()
- aplic_irq_shutdown()
- aplic_irq_enable()
- aplic_irq_disable()
- aplic_irq_ack()
- aplic_host_irq_end()
- aplic_set_irq_affinity()

As APLIC is used in MSI mode it requires to enable/disable interrupts not only for APLIC but also for IMSIC. Thereby for the purpose of aplic_irq_{enable,disable}() it is introduced imsic_irq_{enable,disable)().

For the purpose of aplic_set_irq_affinity() aplic_get_cpu_from_mask() is introduced to get hart id.

Quoted 8 lines
Quoted 5 lines

Also, introduce additional interrupt controller h/w operations and host_irq_type for APLIC:
- aplic_host_irq_type
- aplic_set_irq_priority()
- aplic_set_irq_type()

Yet these two functions nor the hooks they're used to populate are entirely unused here. Since they're also outside of the common IRQ handling machinery, it's unclear how one would sanely ack such a change.

They will be called by intc_route_irq_to_xen() from setup_irq() during firt time the IRQ is setup.

Quoted 8 lines
Quoted 3 lines

Perhaps move their introduction to there then? We don't do any Misra checking yet lon RISC-V, but imo it's still good practice to avoid introducing new violations, even if only temporarily.

Okay, I will move their introduction to there.

Btw, what is needed to add Misra checking for RISC-V? I started to think that, probably, it will make sense to do that from the start.

Best I can do is point you at what is done for Arm and x86. You may want to ask people more familiar with the CI aspects involved here.

Quoted 25 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

xen/arch/riscv/aplic.c static int __init aplic_init(void) · line 143

Quoted 5 lines
xen/arch/riscv/aplic.c +5 -0
+static void aplic_set_irq_priority(struct irq_desc *desc,
+                                   unsigned int priority)
+{
+    /* No priority, do nothing */
+}

Since the function dopes nothing, wouldn't it be better to omit it and have the (future) caller check for a NULL pointer ahead of making the (indirect) call? Same remark for other handlers (below) which also do nothing.

Quoted 5 lines
Quoted 4 lines

I thought about that too, but it could be some cases when the stub is introduced with temporary BUG_ON("unimplemented") inside just to not miss to implement it when it will be necessary.
If we will have only the caller check then we could miss to implement such stubs.

I guess I don't understand the concern.

for example, if we will have the following code:

   void some_caller(struct irq_desc *desc)
   {
       if ( desc->handler->set_affinity )
           desc->handler->set_affinity(desc, cpu_mask);
   }

Then we will skip the call of handler->set_affinity() (if it was just initialized with .set_affinity = NULL) without any notification. And it is fine specifically in this case as aplic_set_irq_priority() does nothing.

Quoted 6 lines

But what about the cases if it is a function which will have some implementation in the future but doesn't have implementation for now. Then without notification that this function is unimplemented we could skip something what really matters.

But I think that your initial comment was just about the function which basically does nothing. Am i right?

Since indirect calls are not only more expensive (often; not sure about RISC-V) but also pose speculative concerns, having such just to do nothing simply seems like moving in the wrong direction.

Quoted 15 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 1 line

+ ASSERT(spin_is_locked(&desc->lock));

If this lock (which is an IRQ-safe one) is necessarily held, ...

Quoted 1 line

+ spin_lock_irqsave(&aplic.lock, flags);

... you can use just spin_lock() here.

Quoted 8 lines
Quoted 4 lines
Quoted 3 lines
Quoted 1 line

+ clear_bit(_IRQ_DISABLED, &desc->status);

Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)

I haven't thought about that. Likely non-atomic bitop could be used here.

And then - does it need to be a bitop? Aiui that's what Arm uses, while x86 doesn't. And I see no reason to use other than plain C operators here. If Arm was switched, presumably all the redundant (and misnamed) _IRQ_* constants could go away, with just the IRQ_* ones left.

Quoted 8 lines

The reason for a bitop in Arm is explained in this commithttps://gitlab.com/xen-project/xen/-/commit/50d8fe8fcbab2440cfeeb65c4765868398652473 but all the places where plain C operators were changed to bitops are actually executed under|spin_lock_irqsave(&desc->lock, flags). By quick look I found only two places one in __setup_irq() but it is called by the functions which do ||spin_lock_irqsave(&desc->lock, flags) and in vgic_v2_fold_lr_state(). Maybe, I'm missing something.|
|RISC-V won't have something similar to ||vgic_v2_fold_lr_state|(), but __setup_irq() is used in a similar way. It can be added ASSERT(spin_is_lock(&desc->lock)) and then it will also safe to use non-bitop function.
Probably, it is a little bit safer to use always bitops for desc->status.
||

I question that. If any accesses outside of locked regions were needed (as the description of that commit suggests), then the situation would be different.

Btw, you not wrapping lines and you adding strange | instances doesn't help readability of your replies.

Quoted 8 lines
Quoted 3 lines
Quoted 2 lines
Quoted 2 lines

I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.

What about this part?

As I understand, based on Arm, code then Xen enables interrupts corresponding to devices assigned to dom0/domU before booting dom0/domU, resulting in the possibility of receiving an interrupt and not knowing what to do with it. So it is needed for enablement of IRQs when the guest requests it and not unconditionally at boot time.

I fear I don't understand this. The way we do things on x86 doesn't leave us in such a situation.

Jan

#57 lore raw

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

Quoted 22 lines · Jan Beulich
Quoted 14 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 1 line

+ ASSERT(spin_is_locked(&desc->lock));

If this lock (which is an IRQ-safe one) is necessarily held, ...

Quoted 1 line

+ spin_lock_irqsave(&aplic.lock, flags);

... you can use just spin_lock() here.

Quoted 8 lines
Quoted 4 lines
Quoted 3 lines
Quoted 1 line

+ clear_bit(_IRQ_DISABLED, &desc->status);

Why an atomic bitop when desc is locked? (And yes, I ought to raise the same question on Arm code also doing so.)

I haven't thought about that. Likely non-atomic bitop could be used here.

And then - does it need to be a bitop? Aiui that's what Arm uses, while x86 doesn't. And I see no reason to use other than plain C operators here. If Arm was switched, presumably all the redundant (and misnamed) _IRQ_* constants could go away, with just the IRQ_* ones left.

Quoted 8 lines

The reason for a bitop in Arm is explained in this commithttps://gitlab.com/xen-project/xen/-/commit/50d8fe8fcbab2440cfeeb65c4765868398652473 but all the places where plain C operators were changed to bitops are actually executed under|spin_lock_irqsave(&desc->lock, flags). By quick look I found only two places one in __setup_irq() but it is called by the functions which do ||spin_lock_irqsave(&desc->lock, flags) and in vgic_v2_fold_lr_state(). Maybe, I'm missing something.|
|RISC-V won't have something similar to ||vgic_v2_fold_lr_state|(), but __setup_irq() is used in a similar way. It can be added ASSERT(spin_is_lock(&desc->lock)) and then it will also safe to use non-bitop function.
Probably, it is a little bit safer to use always bitops for desc->status.
||

Quoted 2 lines

I question that. If any accesses outside of locked regions were needed (as the description of that commit suggests), then the situation would be different.

Okay, then at the moment there is no such cases and I'll use plain C operator instead of clear/set_bit().

Quoted 6 lines

Btw, you not wrapping lines and you adding strange | instances doesn't help readability of your replies.

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

I'm uncertain about this bit setting anyway - on x86 we would only fiddle with it for IRQs not in use, not while enabling/disabling one.

Quoted 7 lines
Quoted 5 lines
Quoted 1 line

What about this part?

As I understand, based on Arm, code then Xen enables interrupts corresponding to devices assigned to dom0/domU before booting dom0/domU, resulting in the possibility of receiving an interrupt and not knowing what to do with it. So it is needed for enablement of IRQs when the guest requests it and not unconditionally at boot time.

I fear I don't understand this. The way we do things on x86 doesn't leave us in such a situation.

On Arm, the physical interrupts would be enabled when the interrupt is initially routed and in case guest is booting with interrupt disabled, it could introduce a problem when guest enabled interrupts it will already have a pending interrupt for which it isn't ready.

How is it handled the case when a device isn't quiescing at the boot time in x86?

But I just realized the way how interrupts are enabled in RISC-V for guest won't lead to such case. The interrupt will be enabled only when guest's device driver will request that. So this setting/clearing of IRQ_DISABLED could be dropped for RISC-V.

~ Oleksii

#58 lore raw

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

In reply to Oleksii Kurochko
Quoted 14 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c +2 -0
@@ -14,12 +14,68 @@
 #include <xen/errno.h>
 #include <xen/init.h>
 #include <xen/macros.h>
+#include <xen/spinlock.h>
 #include <xen/xmalloc.h>
 
 #include <asm/imsic.h>
 
 static struct imsic_config imsic_cfg;
 
+#define imsic_csr_set(c, v)     \

xen/arch/riscv/imsic.c line 34

Quoted 10 lines
xen/arch/riscv/imsic.c +10 -0
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_set(CSR_SIREG, v);      \
+} while (0)
+
+#define imsic_csr_clear(c, v)   \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_clear(CSR_SIREG, v);    \
+} while (0)

Coming back to these (the later patch adds one more here): How expensive are these CSR writes? IOW would it perhaps make sense to maintain a local cache of the last written SISELECT value, to avoid writing the same one again if the same windowed register needs accessing twice in a row?

Jan

#59 lore raw

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

Quoted 14 lines · Jan Beulich
Quoted 13 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c +1 -0
@@ -14,12 +14,68 @@
  #include <xen/errno.h>
  #include <xen/init.h>
  #include <xen/macros.h>
+#include <xen/spinlock.h>
  #include <xen/xmalloc.h>
 
  #include <asm/imsic.h>
 
  static struct imsic_config imsic_cfg;
 
Quoted 15 lines

xen/arch/riscv/imsic.c line 34

Quoted 11 lines
xen/arch/riscv/imsic.c +11 -0
+#define imsic_csr_set(c, v)     \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_set(CSR_SIREG, v);      \
+} while (0)
+
+#define imsic_csr_clear(c, v)   \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_clear(CSR_SIREG, v);    \
+} while (0)

Coming back to these (the later patch adds one more here): How expensive are these CSR writes? IOW would it perhaps make sense to maintain a local cache of the last written SISELECT value, to avoid writing the same one again if the same windowed register needs accessing twice in a row?

CSRs belong to the HART, so access to them is very fast.

~ Oleksii

#60 lore raw

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

Quoted 32 lines · Oleksii Kurochko
Quoted 14 lines · Jan Beulich
Quoted 13 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c +1 -0
@@ -14,12 +14,68 @@
  #include <xen/errno.h>
  #include <xen/init.h>
  #include <xen/macros.h>
+#include <xen/spinlock.h>
  #include <xen/xmalloc.h>
 
  #include <asm/imsic.h>
 
  static struct imsic_config imsic_cfg;
 
Quoted 15 lines

xen/arch/riscv/imsic.c line 34

Quoted 11 lines
xen/arch/riscv/imsic.c +11 -0
+#define imsic_csr_set(c, v)     \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_set(CSR_SIREG, v);      \
+} while (0)
+
+#define imsic_csr_clear(c, v)   \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_clear(CSR_SIREG, v);    \
+} while (0)

Coming back to these (the later patch adds one more here): How expensive are these CSR writes? IOW would it perhaps make sense to maintain a local cache of the last written SISELECT value, to avoid writing the same one again if the same windowed register needs accessing twice in a row?

Quoted 1 line

CSRs belong to the HART, so access to them is very fast.

Can you back this by any data? I view CSRs as somewhat similar to x86'es MSRs, and access (writes in particular) to some of them is rather slow.

Jan

#61 lore raw

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

Quoted 31 lines · Jan Beulich
Quoted 30 lines · Oleksii Kurochko
Quoted 29 lines · Jan Beulich
Quoted 14 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c +2 -0
@@ -14,12 +14,68 @@
   #include <xen/errno.h>
   #include <xen/init.h>
   #include <xen/macros.h>
+#include <xen/spinlock.h>
   #include <xen/xmalloc.h>
 
   #include <asm/imsic.h>
 
   static struct imsic_config imsic_cfg;
 
+#define imsic_csr_set(c, v)     \

xen/arch/riscv/imsic.c line 34

Quoted 10 lines
xen/arch/riscv/imsic.c +10 -0
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_set(CSR_SIREG, v);      \
+} while (0)
+
+#define imsic_csr_clear(c, v)   \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_clear(CSR_SIREG, v);    \
+} while (0)

Coming back to these (the later patch adds one more here): How expensive are these CSR writes? IOW would it perhaps make sense to maintain a local cache of the last written SISELECT value, to avoid writing the same one again if the same windowed register needs accessing twice in a row?

Quoted 3 lines
Quoted 1 line

CSRs belong to the HART, so access to them is very fast.

Can you back this by any data? I view CSRs as somewhat similar to x86'es MSRs, and access (writes in particular) to some of them is rather slow.

CSR read 1 cycle, CSR write 7 cycles on Microchip platform. ~ Oleksii

#62 lore raw

Re: [PATCH v1 10/14] xen/riscv: implementation of aplic and imsic operations

Quoted 33 lines · Oleksii Kurochko
Quoted 31 lines · Jan Beulich
Quoted 30 lines · Oleksii Kurochko
Quoted 29 lines · Jan Beulich
Quoted 14 lines · Oleksii Kurochko
xen/arch/riscv/imsic.c +2 -0
@@ -14,12 +14,68 @@
   #include <xen/errno.h>
   #include <xen/init.h>
   #include <xen/macros.h>
+#include <xen/spinlock.h>
   #include <xen/xmalloc.h>
 
   #include <asm/imsic.h>
 
   static struct imsic_config imsic_cfg;
 
+#define imsic_csr_set(c, v)     \

xen/arch/riscv/imsic.c line 34

Quoted 10 lines
xen/arch/riscv/imsic.c +10 -0
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_set(CSR_SIREG, v);      \
+} while (0)
+
+#define imsic_csr_clear(c, v)   \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_clear(CSR_SIREG, v);    \
+} while (0)

Coming back to these (the later patch adds one more here): How expensive are these CSR writes? IOW would it perhaps make sense to maintain a local cache of the last written SISELECT value, to avoid writing the same one again if the same windowed register needs accessing twice in a row?

Quoted 5 lines
Quoted 3 lines
Quoted 1 line

CSRs belong to the HART, so access to them is very fast.

Can you back this by any data? I view CSRs as somewhat similar to x86'es MSRs, and access (writes in particular) to some of them is rather slow.

CSR read 1 cycle, CSR write 7 cycles on Microchip platform. ~ Oleksii

And that's an in-order platform, i.e. cycle count being all that matters for performance? No other (e.g. latency) effect on subsequent insns?

Further, how does this compare to the outlined alternative, especially if we assumed that the respective cacheline would be hot and hence usually in L1 cache?

Jan

#63 lore raw

PATCH 11/14 xen/riscv: add external interrupt handling for hypervisor mode

In reply to Oleksii Kurochko

Implement functions necessarry to have working external interrupts in hypervisor mode. The following changes are done:
- Add a common function intc_handle_external_irq() to call APLIC specific function to handle an interrupt.
- Update do_trap() function to handle IRQ_S_EXT case; add the check to catch case when cause of trap is an interrupt.
- Add handle_interrrupt() member to intc_hw_operations structure.
- Enable local interrupt delivery for IMSIC by implementation and calling of imsic_ids_local_delivery() in imsic_init(); additionally introduce helper imsic_csr_write() to update IMSIC_EITHRESHOLD and IMSIC_EITHRESHOLD.
- Enable hypervisor external interrupts.
- Implement aplic_handler_interrupt() and use it to init ->handle_interrupt member of intc_hw_operations for APLIC.
- Add implementation of do_IRQ() to dispatch the interrupt.

The current patch is based on the code from [1].

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7390e2365828b83e27ead56b03114a56e3699dd5

---
 xen/arch/riscv/aplic.c             | 19 +++++++++++++
 xen/arch/riscv/imsic.c             | 25 +++++++++++++++++
 xen/arch/riscv/include/asm/imsic.h |  7 +++++
 xen/arch/riscv/include/asm/intc.h  |  5 ++++
 xen/arch/riscv/include/asm/irq.h   |  3 +++
 xen/arch/riscv/intc.c              |  7 +++++
 xen/arch/riscv/irq.c               | 43 ++++++++++++++++++++++++++++++
 xen/arch/riscv/traps.c             | 18 +++++++++++++
 8 files changed, 127 insertions(+)
xen/arch/riscv/aplic.c +19 -0
index 4b60cb9a77..38b57ed1ac 100644
@@ -261,6 +261,21 @@ static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
     spin_unlock(&aplic.lock);
 }
 
+static void aplic_handle_interrupt(unsigned long cause, struct cpu_user_regs *regs)
+{
+    /* disable to avoid more external interrupts */
+    csr_clear(CSR_SIE, 1UL << IRQ_S_EXT);
+
+    /* clear the pending bit */
+    csr_clear(CSR_SIP, 1UL << IRQ_S_EXT);
+
+    /* dispatch the interrupt */
+    do_IRQ(regs, csr_swap(CSR_STOPEI, 0) >> TOPI_IID_SHIFT);
+
+    /* enable external interrupts */
+    csr_set(CSR_SIE, 1UL << IRQ_S_EXT);
+}
+
 static hw_irq_controller aplic_host_irq_type = {
     .typename     = "aplic",
     .startup      = aplic_irq_startup,
@@ -278,6 +293,7 @@ static const struct intc_hw_operations aplic_ops = {
     .host_irq_type       = &aplic_host_irq_type,
     .set_irq_priority    = aplic_set_irq_priority,
     .set_irq_type        = aplic_set_irq_type,
+    .handle_interrupt    = aplic_handle_interrupt,
 };
 
 static int aplic_irq_xlate(const uint32_t *intspec, unsigned int intsize,
@@ -318,6 +334,9 @@ static int __init aplic_preinit(struct dt_device_node *node, const void *dat)
 
     register_intc_ops(&aplic_ops);
 
+    /* Enable supervisor external interrupt */
+    csr_set(CSR_SIE, 1UL << IRQ_S_EXT);
+
     return 0;
 }
 
xen/arch/riscv/imsic.c +25 -0
index 8198d008ef..e00f2d69df 100644
@@ -19,8 +19,19 @@
 
 #include <asm/imsic.h>
 
+#define IMSIC_DISABLE_EIDELIVERY    0
+#define IMSIC_ENABLE_EIDELIVERY     1
+#define IMSIC_DISABLE_EITHRESHOLD   1
+#define IMSIC_ENABLE_EITHRESHOLD    0
+
 static struct imsic_config imsic_cfg;
 
+#define imsic_csr_write(c, v)   \
+do {                            \
+    csr_write(CSR_SISELECT, c); \
+    csr_write(CSR_SIREG, v);    \
+} while (0)
+
 #define imsic_csr_set(c, v)     \
 do {                            \
     csr_write(CSR_SISELECT, c); \
@@ -33,6 +44,20 @@ do {                            \
     csr_clear(CSR_SIREG, v);    \
 } while (0)
 
+void imsic_ids_local_delivery(bool enable)
+{
+    if ( enable )
+    {
+        imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD);
+        imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY);
+    }
+    else
+    {
+        imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD);
+        imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY);
+    }
+}
+
 static void imsic_local_eix_update(unsigned long base_id, unsigned long num_id,
                                    bool pend, bool val)
 {
xen/arch/riscv/include/asm/imsic.h +7 -0
index d2c0178529..b2c674f271 100644
@@ -12,6 +12,7 @@
 #define ASM__RISCV__IMSIC_H
 
 #include <xen/spinlock.h>
+#include <xen/stdbool.h>
 #include <xen/types.h>
 
 #define IMSIC_MMIO_PAGE_SHIFT   12
@@ -20,6 +21,10 @@
 #define IMSIC_MIN_ID            63
 #define IMSIC_MAX_ID            2048
 
+#define IMSIC_EIDELIVERY        0x70
+
+#define IMSIC_EITHRESHOLD       0x72
+
 #define IMSIC_EIP0              0x80
 #define IMSIC_EIPx_BITS         32
 
@@ -78,4 +83,6 @@ const struct imsic_config *imsic_get_config(void);
 void imsic_irq_enable(unsigned int hwirq);
 void imsic_irq_disable(unsigned int hwirq);
 
+void imsic_ids_local_delivery(bool enable);
+
 #endif /* ASM__RISCV__IMSIC_H */
xen/arch/riscv/include/asm/intc.h +5 -0
index db53caa07b..e4363af87d 100644
@@ -34,6 +34,8 @@ struct intc_hw_operations {
     /* Set IRQ priority */
     void (*set_irq_priority)(struct irq_desc *desc, unsigned int priority);
 
+    /* handle external interrupt */
+    void (*handle_interrupt)(unsigned long cause, struct cpu_user_regs *regs);
 };
 
 void intc_preinit(void);
@@ -45,4 +47,7 @@ void register_intc_ops(const struct intc_hw_operations *ops);
 struct irq_desc;
 void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority);
 
+struct cpu_user_regs;
+void intc_handle_external_irqs(unsigned long cause, struct cpu_user_regs *regs);
+
 #endif /* ASM__RISCV__INTERRUPT_CONTOLLER_H */
xen/arch/riscv/include/asm/irq.h +3 -0
index 163a478d78..9558d3fa61 100644
@@ -51,6 +51,9 @@ int platform_get_irq(const struct dt_device_node *device, int index);
 
 void init_IRQ(void);
 
+struct cpu_user_regs;
+void do_IRQ(struct cpu_user_regs *regs, unsigned int irq);
+
 #endif /* ASM__RISCV__IRQ_H */
 
 /*
xen/arch/riscv/intc.c +7 -0
index 8274897d8c..41a4310ead 100644
@@ -51,6 +51,13 @@ static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority)
     intc_hw_ops->set_irq_priority(desc, priority);
 }
 
+void intc_handle_external_irqs(unsigned long cause, struct cpu_user_regs *regs)
+{
+    ASSERT(intc_hw_ops && intc_hw_ops->handle_interrupt);
+
+    intc_hw_ops->handle_interrupt(cause, regs);
+}
+
 void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority)
 {
     ASSERT(test_bit(_IRQ_DISABLED, &desc->status));
xen/arch/riscv/irq.c +43 -0

52 lines not on this page · raw message

xen/arch/riscv/traps.c +18 -0
index ea3638a54f..da5813e34a 100644
@@ -11,6 +11,7 @@
 #include <xen/nospec.h>
 #include <xen/sched.h>
 
+#include <asm/intc.h>
 #include <asm/processor.h>
 #include <asm/riscv_encoding.h>
 #include <asm/traps.h>
@@ -128,6 +129,23 @@ void do_trap(struct cpu_user_regs *cpu_regs)
         }
         fallthrough;
     default:
+        if ( cause & CAUSE_IRQ_FLAG )
+        {
+            /* Handle interrupt */
+            unsigned long icause = cause & ~CAUSE_IRQ_FLAG;
+
+            switch ( icause )
+            {
+            case IRQ_S_EXT:
+                intc_handle_external_irqs(cause, cpu_regs);
+                break;
+            default:
+                break;
+            }
+
+            break;
+        }
+
         do_unexpected_trap(cpu_regs);
         break;
     }
Signature · 1 line
2.49.0
#64 lore raw

Re: [PATCH v1 11/14] xen/riscv: add external interrupt handling for hypervisor mode

Quoted 9 lines · Oleksii Kurochko

Implement functions necessarry to have working external interrupts in hypervisor mode. The following changes are done:
- Add a common function intc_handle_external_irq() to call APLIC specific function to handle an interrupt.
- Update do_trap() function to handle IRQ_S_EXT case; add the check to catch case when cause of trap is an interrupt.
- Add handle_interrrupt() member to intc_hw_operations structure.
- Enable local interrupt delivery for IMSIC by implementation and calling of imsic_ids_local_delivery() in imsic_init();

Ah, here is where that call really belongs (see my question on the earlier patch). Please make sure you series builds okay at every patch boundary.

xen/arch/riscv/aplic.c static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask) · line 267

Quoted 10 lines
xen/arch/riscv/aplic.c +4 -0
@@ -261,6 +261,21 @@ static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask)
     spin_unlock(&aplic.lock);
 }
 
+static void aplic_handle_interrupt(unsigned long cause, struct cpu_user_regs *regs)
+{
+    /* disable to avoid more external interrupts */
+    csr_clear(CSR_SIE, 1UL << IRQ_S_EXT);

Didn't I see you use BIT() elsewhere? Would be nice to be overall consistent at least within related code.

xen/arch/riscv/aplic.c static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask) · line 277

Quoted 9 lines
xen/arch/riscv/aplic.c +9 -0
+    /* clear the pending bit */
+    csr_clear(CSR_SIP, 1UL << IRQ_S_EXT);
+
+    /* dispatch the interrupt */
+    do_IRQ(regs, csr_swap(CSR_STOPEI, 0) >> TOPI_IID_SHIFT);
+
+    /* enable external interrupts */
+    csr_set(CSR_SIE, 1UL << IRQ_S_EXT);
+}

Why does "cause" need passing into here? I realize the function is used ...

xen/arch/riscv/aplic.c static const struct intc_hw_operations aplic_ops = { · line 297

Quoted 6 lines
xen/arch/riscv/aplic.c +1 -0
@@ -278,6 +293,7 @@ static const struct intc_hw_operations aplic_ops = {
     .host_irq_type       = &aplic_host_irq_type,
     .set_irq_priority    = aplic_set_irq_priority,
     .set_irq_type        = aplic_set_irq_type,
+    .handle_interrupt    = aplic_handle_interrupt,
 };

... as a hook, yet it's still unclear whether (why) any other such hook would need the cause to be passed in.

xen/arch/riscv/aplic.c do { \ · line 47

Quoted 5 lines
xen/arch/riscv/aplic.c +1 -0
@@ -33,6 +44,20 @@ do {                            \
     csr_clear(CSR_SIREG, v);    \
 } while (0)
 
+void imsic_ids_local_delivery(bool enable)

__init as long as the sole caller is such?

Quoted 12 lines
xen/arch/riscv/include/asm/intc.h +2 -0
@@ -34,6 +34,8 @@ struct intc_hw_operations {
     /* Set IRQ priority */
     void (*set_irq_priority)(struct irq_desc *desc, unsigned int priority);
 
+    /* handle external interrupt */
+    void (*handle_interrupt)(unsigned long cause, struct cpu_user_regs *regs);
 };
 
 void intc_preinit(void);

xen/arch/riscv/include/asm/intc.h void register_intc_ops(const struct intc_hw_operations *ops); · line 50

Quoted 4 lines
xen/arch/riscv/include/asm/intc.h +1 -0
@@ -45,4 +47,7 @@ void register_intc_ops(const struct intc_hw_operations *ops);
 struct irq_desc;
 void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority);
 
+struct cpu_user_regs;

This is too late - you've used it above already. It either can be dropped, or needs to move up.

xen/arch/riscv/intc.c static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority) · line 56

Quoted 9 lines
xen/arch/riscv/intc.c +3 -0
@@ -51,6 +51,13 @@ static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority)
     intc_hw_ops->set_irq_priority(desc, priority);
 }
 
+void intc_handle_external_irqs(unsigned long cause, struct cpu_user_regs *regs)
+{
+    ASSERT(intc_hw_ops && intc_hw_ops->handle_interrupt);

I don't view such checks (on every interrupt) as very useful. If you checked once early on - okay. But here you gain nothing at all ...

xen/arch/riscv/intc.c static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority) · line 58

Quoted 1 line
xen/arch/riscv/intc.c +1 -0
+    intc_hw_ops->handle_interrupt(cause, regs);

... towards the use here, when considering a release build.

Quoted 10 lines
xen/arch/riscv/intc.c +6 -0
@@ -83,3 +87,42 @@ void __init init_IRQ(void)
     if ( init_irq_data() < 0 )
         panic("initialization of IRQ data failed\n");
 }
+
+/* Dispatch an interrupt */
+void do_IRQ(struct cpu_user_regs *regs, unsigned int irq)
+{
+    struct irq_desc *desc = irq_to_desc(irq);
+    struct irqaction *action;

xen/arch/riscv/intc.c void __init init_IRQ(void) · line 105

Quoted 10 lines
xen/arch/riscv/intc.c +10 -0
+
+    irq_enter();
+
+    spin_lock(&desc->lock);
+    desc->handler->ack(desc);
+
+    if ( test_bit(_IRQ_DISABLED, &desc->status) )
+        goto out;
+
+    set_bit(_IRQ_INPROGRESS, &desc->status);

Same comment as on the earlier patch - atomic bitop when in a suitably locked region?

xen/arch/riscv/intc.c void __init init_IRQ(void) · line 110

Quoted 5 lines
xen/arch/riscv/intc.c +5 -0
+    action = desc->action;
+
+    spin_unlock_irq(&desc->lock);
+
+#ifndef CONFIG_IRQ_HAS_MULTIPLE_ACTION

Stolen from Arm? What's this about?

xen/arch/riscv/intc.c void __init init_IRQ(void) · line 121

Quoted 11 lines
xen/arch/riscv/intc.c +11 -0
+    action->handler(irq, action->dev_id);
+#else
+    do {
+        action->handler(irq, action->dev_id);
+        action = action->next;
+    } while ( action );
+#endif /* CONFIG_IRQ_HAS_MULTIPLE_ACTION */
+
+    spin_lock_irq(&desc->lock);
+
+    clear_bit(_IRQ_INPROGRESS, &desc->status);

See above.

xen/arch/riscv/intc.c void __init init_IRQ(void) · line 122

Quoted 1 line
xen/arch/riscv/intc.c +1 -0
+out:

Nit (you know what).

Quoted 5 lines
xen/arch/riscv/intc.c +1 -0
@@ -128,6 +129,23 @@ void do_trap(struct cpu_user_regs *cpu_regs)
         }
         fallthrough;
     default:
+        if ( cause & CAUSE_IRQ_FLAG )

xen/arch/riscv/intc.c void do_trap(struct cpu_user_regs *cpu_regs) · line 142

Quoted 10 lines
xen/arch/riscv/intc.c +10 -0
+        {
+            /* Handle interrupt */
+            unsigned long icause = cause & ~CAUSE_IRQ_FLAG;
+
+            switch ( icause )
+            {
+            case IRQ_S_EXT:
+                intc_handle_external_irqs(cause, cpu_regs);
+                break;
+            default:

Nit: Blank line please between non-fall-through case blocks.

Jan

#65 lore raw

Re: [PATCH v1 11/14] xen/riscv: add external interrupt handling for hypervisor mode

Quoted 12 lines · Jan Beulich

xen/arch/riscv/aplic.c static void aplic_set_irq_affinity(struct irq_desc *desc, const cpumask_t *mask) · line 277

Quoted 10 lines
xen/arch/riscv/aplic.c +9 -0
+    /* clear the pending bit */
+    csr_clear(CSR_SIP, 1UL << IRQ_S_EXT);
+
+    /* dispatch the interrupt */
+    do_IRQ(regs, csr_swap(CSR_STOPEI, 0) >> TOPI_IID_SHIFT);
+
+    /* enable external interrupts */
+    csr_set(CSR_SIE, 1UL << IRQ_S_EXT);
+}

Why does "cause" need passing into here? I realize the function is used ...

Quoted 8 lines

xen/arch/riscv/aplic.c static const struct intc_hw_operations aplic_ops = { · line 297

Quoted 6 lines
xen/arch/riscv/aplic.c +1 -0
@@ -278,6 +293,7 @@ static const struct intc_hw_operations aplic_ops = {
      .host_irq_type       = &aplic_host_irq_type,
      .set_irq_priority    = aplic_set_irq_priority,
      .set_irq_type        = aplic_set_irq_type,
+    .handle_interrupt    = aplic_handle_interrupt,
  };

... as a hook, yet it's still unclear whether (why) any other such hook would need the cause to be passed in.

I don't remember a particular reason, but it could have been dropped. If, for some reason, the cause is needed in|handle_interrupt()|, it can be retrieved explicitly from a register.

Quoted 7 lines

xen/arch/riscv/imsic.c do { \ · line 47

Quoted 6 lines
xen/arch/riscv/imsic.c +1 -0
@@ -33,6 +44,20 @@ do {                            \
      csr_clear(CSR_SIREG, v);    \
  } while (0)
 
+void imsic_ids_local_delivery(bool enable)

__init as long as the sole caller is such?

Yes, it make sense. Also, I noticed some other functions that could be __init in imsic.c (but likely you mentioned that in the previous patches).

Quoted 6 lines
Quoted 6 lines
xen/arch/riscv/intc.c +0 -0
@@ -51,6 +51,13 @@ static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority)
      intc_hw_ops->set_irq_priority(desc, priority);
  }
 
Quoted 8 lines

xen/arch/riscv/intc.c static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority) · line 56

Quoted 3 lines
xen/arch/riscv/intc.c +3 -0
+void intc_handle_external_irqs(unsigned long cause, struct cpu_user_regs *regs)
+{
+    ASSERT(intc_hw_ops && intc_hw_ops->handle_interrupt);

I don't view such checks (on every interrupt) as very useful. If you checked once early on - okay. But here you gain nothing at all ...

xen/arch/riscv/intc.c static void intc_set_irq_priority(struct irq_desc *desc, unsigned int priority) · line 58

Quoted 1 line
xen/arch/riscv/intc.c +1 -0
+    intc_hw_ops->handle_interrupt(cause, regs);

... towards the use here, when considering a release build.

I will try to find a better place then.

Quoted 16 lines
Quoted 16 lines
xen/arch/riscv/intc.c +10 -0
@@ -83,3 +87,42 @@ void __init init_IRQ(void)
      if ( init_irq_data() < 0 )
          panic("initialization of IRQ data failed\n");
  }
+
+/* Dispatch an interrupt */
+void do_IRQ(struct cpu_user_regs *regs, unsigned int irq)
+{
+    struct irq_desc *desc = irq_to_desc(irq);
+    struct irqaction *action;
+
+    irq_enter();
+
+    spin_lock(&desc->lock);
Quoted 8 lines

xen/arch/riscv/intc.c void __init init_IRQ(void) · line 105

Quoted 6 lines
xen/arch/riscv/intc.c +6 -0
+    desc->handler->ack(desc);
+
+    if ( test_bit(_IRQ_DISABLED, &desc->status) )
+        goto out;
+
+    set_bit(_IRQ_INPROGRESS, &desc->status);

Same comment as on the earlier patch - atomic bitop when in a suitably locked region?

Agree, it could be used non-atomic bitop.

Quoted 7 lines

xen/arch/riscv/intc.c void __init init_IRQ(void) · line 110

Quoted 6 lines
xen/arch/riscv/intc.c +5 -0
+    action = desc->action;
+
+    spin_unlock_irq(&desc->lock);
+
+#ifndef CONFIG_IRQ_HAS_MULTIPLE_ACTION

Stolen from Arm? What's this about?

Yes, it is stolen from Arm. I thought that it is a generic one, but the config is defined inside Arm's config.h.
Then it could be dropped now as I don't know, at the moment, the cases when it is neeeded to exectute several handler for an irq for RISC-V.

Thanks.

~ Oleksii

#66 lore raw

Re: [PATCH v1 11/14] xen/riscv: add external interrupt handling for hypervisor mode

Quoted 10 lines · Oleksii Kurochko
Quoted 6 lines

xen/arch/riscv/irq.c void __init init_IRQ(void) · line 111

Quoted 5 lines
xen/arch/riscv/irq.c +5 -0
+    action = desc->action;
+
+    spin_unlock_irq(&desc->lock);
+
+#ifndef CONFIG_IRQ_HAS_MULTIPLE_ACTION

Stolen from Arm? What's this about?

Yes, it is stolen from Arm. I thought that it is a generic one, but the config is defined inside Arm's config.h.
Then it could be dropped now as I don't know, at the moment, the cases when it is neeeded to exectute several handler for an irq for RISC-V.

Probably, IOMMU may setup multiple handler for the same interrupt. I'll double check that.

~ Oleksii

#67 lore raw

PATCH 12/14 xen/riscv: implement setup_irq()

In reply to Oleksii Kurochko

Introduce support for IRQ setup on RISC-V by implementing setup_irq() and __setup_irq(), adapted and extended from an initial implementation by [1].

__setup_irq() does the following:
- Sets up an IRQ action.
- Validates that shared IRQs have non-NULL dev_id and are only used when existing handlers allow sharing.
- Uses smp_mb() to enforce memory ordering after assigning desc->action to ensure visibility before enabling the IRQ.
- Supports multi-action setups via CONFIG_IRQ_HAS_MULTIPLE_ACTION.

setup_irq() does the following:
- Converts IRQ number to descriptor and acquires its lock.
- Rejects registration if the IRQ is already assigned to a guest domain, printing an error.
- Delegates the core setup to __setup_irq().
- On first-time setup, disables the IRQ, routes it to Xen using intc_route_irq_to_xen(), sets default CPU affinity (current CPU), calls the handler’s startup routine, and finally enables the IRQ.

irq_set_affinity() invokes set_affinity callback from the IRQ handler if present.

Defined IRQ_NO_PRIORITY as default priority used when routing IRQs to Xen.

[1] https://gitlab.com/xen-project/people/olkur/xen/-/commit/7390e2365828b83e27ead56b03114a56e3699dd5

---
 xen/arch/riscv/include/asm/irq.h |  6 ++
 xen/arch/riscv/irq.c             | 95 ++++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
xen/arch/riscv/include/asm/irq.h +6 -0
index 9558d3fa61..bba3a97e3e 100644
@@ -26,6 +26,8 @@
 #define IRQ_TYPE_SENSE_MASK     DT_IRQ_TYPE_SENSE_MASK
 #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
 
+#define IRQ_NO_PRIORITY 0
+
 /* TODO */
 #define nr_static_irqs 0
 #define arch_hwdom_irqs(domid) 0U
@@ -54,6 +56,10 @@ void init_IRQ(void);
 struct cpu_user_regs;
 void do_IRQ(struct cpu_user_regs *regs, unsigned int irq);
 
+struct irq_desc;
+struct cpumask_t;
+void irq_set_affinity(struct irq_desc *desc, const cpumask_t *cpu_mask);
+
 #endif /* ASM__RISCV__IRQ_H */
 
 /*
xen/arch/riscv/irq.c +95 -0

108 lines not on this page · raw message

Signature · 1 line
2.49.0
#68 lore raw

Re: [PATCH v1 12/14] xen/riscv: implement setup_irq()

xen/arch/riscv/include/asm/irq.h void init_IRQ(void); · line 56

Quoted 13 lines · Oleksii Kurochko
xen/arch/riscv/include/asm/irq.h +2 -0
@@ -26,6 +26,8 @@
 #define IRQ_TYPE_SENSE_MASK     DT_IRQ_TYPE_SENSE_MASK
 #define IRQ_TYPE_INVALID        DT_IRQ_TYPE_INVALID
 
+#define IRQ_NO_PRIORITY 0
+
 /* TODO */
 #define nr_static_irqs 0
 #define arch_hwdom_irqs(domid) 0U
@@ -54,6 +56,10 @@ void init_IRQ(void);
 struct cpu_user_regs;

Seeing this and ...

xen/arch/riscv/include/asm/irq.h void init_IRQ(void); · line 60

Quoted 4 lines
xen/arch/riscv/include/asm/irq.h +2 -0
 void do_IRQ(struct cpu_user_regs *regs, unsigned int irq);
 
+struct irq_desc;
+struct cpumask_t;

... now these - all of such forward decls may want to collectively live in a central place higher up in the file.

xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 63

Quoted 6 lines
xen/arch/riscv/include/asm/irq.h +2 -0
@@ -57,6 +59,99 @@ int platform_get_irq(const struct dt_device_node *device, int index)
     return dt_irq.irq;
 }
 
+static int __setup_irq(struct irq_desc *desc, unsigned int irqflags,
+                       struct irqaction *new)

Irrespective of you possibly having found it like this elsewhere, may I suggest that in new code we avoid leading double underscores? A single one will do here.

xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 69

Quoted 6 lines
xen/arch/riscv/include/asm/irq.h +6 -0
+{
+    bool shared = irqflags & IRQF_SHARED;
+
+    ASSERT(new != NULL);
+
+    /* Sanity checks:

Nit: Comment style (and there are many more issues below).

xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 74

Quoted 5 lines
xen/arch/riscv/include/asm/irq.h +5 -0
+     *  - if the IRQ is marked as shared
+     *  - dev_id is not NULL when IRQF_SHARED is set
+     */
+    if ( desc->action != NULL && (!test_bit(_IRQF_SHARED, &desc->status)
+         || !shared) )

Nit: Operator placement and indentation.

You're probably better off this way anyway:

    if ( desc->action != NULL &&
         (!test_bit(_IRQF_SHARED, &desc->status) || !shared) )

xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 80

Quoted 6 lines
xen/arch/riscv/include/asm/irq.h +6 -0
+        return -EINVAL;
+    if ( shared && new->dev_id == NULL )
+        return -EINVAL;
+
+    if ( shared )
+        set_bit(_IRQF_SHARED, &desc->status);

See comments on earlier patches.

xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 87

Quoted 7 lines
xen/arch/riscv/include/asm/irq.h +7 -0
+#ifdef CONFIG_IRQ_HAS_MULTIPLE_ACTION
+    new->next = desc->action;
+    smp_mb();
+#endif
+
+    desc->action = new;
+    smp_mb();

Aren't smp_wmb() sufficient on both places? If not, I think comments want adding.

xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 93

Quoted 6 lines
xen/arch/riscv/include/asm/irq.h +6 -0
+    return 0;
+}
+
+void irq_set_affinity(struct irq_desc *desc, const cpumask_t *cpu_mask)
+{
+    if ( desc != NULL )

Can desc really be NULL here? Isn't desc->lock required to be held?

xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 104

Quoted 11 lines
xen/arch/riscv/include/asm/irq.h +11 -0
+        desc->handler->set_affinity(desc, cpu_mask);
+}
+
+int setup_irq(unsigned int irq, unsigned int irqflags, struct irqaction *new)
+{
+    int rc;
+    unsigned long flags;
+    struct irq_desc *desc;
+    bool disabled;
+
+    desc = irq_to_desc(irq);

Make this the variable's initializer?

Quoted 14 lines
xen/arch/riscv/include/asm/irq.h +14 -0
+    spin_lock_irqsave(&desc->lock, flags);
+
+    disabled = (desc->action == NULL);
+
+    if ( test_bit(_IRQ_GUEST, &desc->status) )
+    {
+        spin_unlock_irqrestore(&desc->lock, flags);
+        /*
+         * TODO: would be nice to have functionality to print which domain owns
+         *       an IRQ.
+         */
+        printk(XENLOG_ERR "ERROR: IRQ %u is already in use by a domain\n", irq);
+        return -EBUSY;
+    }

xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 128

Quoted 10 lines
xen/arch/riscv/include/asm/irq.h +10 -0
+
+    rc = __setup_irq(desc, irqflags, new);
+    if ( rc )
+        goto err;
+
+    /* First time the IRQ is setup */
+    if ( disabled )
+    {
+        /* disable irq by default */
+        set_bit(_IRQ_DISABLED, &desc->status);

Shouldn't this be set when we make it here?

Quoted 4 lines
xen/arch/riscv/include/asm/irq.h +4 -0
+        /* route interrupt to xen */
+        intc_route_irq_to_xen(desc, IRQ_NO_PRIORITY);
+
+        /*

xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 142

Quoted 10 lines
xen/arch/riscv/include/asm/irq.h +10 -0
+         * we don't care for now which CPU will receive the
+         * interrupt
+         *
+         * TODO: Handle case where IRQ is setup on different CPU than
+         * the targeted CPU and the priority.
+         */
+        irq_set_affinity(desc, cpumask_of(smp_processor_id()));
+        desc->handler->startup(desc);
+        /* enable irq */
+        clear_bit(_IRQ_DISABLED, &desc->status);

Now it turns out this is really done twice: Once in aplic_irq_enable(), and once here.

xen/arch/riscv/include/asm/irq.h int platform_get_irq(const struct dt_device_node *device, int index) · line 145

Quoted 3 lines
xen/arch/riscv/include/asm/irq.h +3 -0
+    }
+
+err:

Nit (yet once again).

Jan

#69 lore raw

Re: [PATCH v1 12/14] xen/riscv: implement setup_irq()

On 4/15/25 5:55 PM, Jan Beulich wrote

Quoted 9 lines

xen/arch/riscv/irq.c int platform_get_irq(const struct dt_device_node *device, int index) · line 88

Quoted 7 lines
xen/arch/riscv/irq.c +7 -0
+#ifdef CONFIG_IRQ_HAS_MULTIPLE_ACTION
+    new->next = desc->action;
+    smp_mb();
+#endif
+
+    desc->action = new;
+    smp_mb();

Aren't smp_wmb() sufficient on both places? If not, I think comments want adding.

smp_wmb() will be sufficient but I think the barrier could be dropped at all as __setup_irq() is called only in setup_irq() and __setup_irq() call is wrapped by spinlock_{un}lock_irqsave() where spinlock_unlock_*() will put barrier.

Quoted 8 lines

xen/arch/riscv/irq.c int platform_get_irq(const struct dt_device_node *device, int index) · line 95

Quoted 7 lines
xen/arch/riscv/irq.c +6 -0
+    return 0;
+}
+
+void irq_set_affinity(struct irq_desc *desc, const cpumask_t *cpu_mask)
+{
+    if ( desc != NULL )

Can desc really be NULL here?

It can't as irq_desc[] isn't dynamically allocated.

Quoted 1 line

Isn't desc->lock required to be held?

It is and it is called in setup_irq() which calls spin_lock_irqsave().
Anyway, I think it could be dropped at all and use 'desc->handler->set_affinity(desc, cpu_mask);' explicitly in setup_irq().

Quoted 17 lines
Quoted 17 lines
xen/arch/riscv/irq.c +17 -0
+    spin_lock_irqsave(&desc->lock, flags);
+
+    disabled = (desc->action == NULL);
+
+    if ( test_bit(_IRQ_GUEST, &desc->status) )
+    {
+        spin_unlock_irqrestore(&desc->lock, flags);
+        /*
+         * TODO: would be nice to have functionality to print which domain owns
+         *       an IRQ.
+         */
+        printk(XENLOG_ERR "ERROR: IRQ %u is already in use by a domain\n", irq);
+        return -EBUSY;
+    }
+
+    rc = __setup_irq(desc, irqflags, new);
+    if ( rc )
Quoted 8 lines

xen/arch/riscv/irq.c int platform_get_irq(const struct dt_device_node *device, int index) · line 131

Quoted 7 lines
xen/arch/riscv/irq.c +7 -0
+        goto err;
+
+    /* First time the IRQ is setup */
+    if ( disabled )
+    {
+        /* disable irq by default */
+        set_bit(_IRQ_DISABLED, &desc->status);

Shouldn't this be set when we make it here?

It should be. I'll drop the setting of _IRQ_DISABLED.

Quoted 15 lines
Quoted 5 lines
xen/arch/riscv/irq.c +4 -0
+        /* route interrupt to xen */
+        intc_route_irq_to_xen(desc, IRQ_NO_PRIORITY);
+
+        /*

xen/arch/riscv/irq.c int platform_get_irq(const struct dt_device_node *device, int index) · line 146

Quoted 10 lines
xen/arch/riscv/irq.c +10 -0
+         * we don't care for now which CPU will receive the
+         * interrupt
+         *
+         * TODO: Handle case where IRQ is setup on different CPU than
+         * the targeted CPU and the priority.
+         */
+        irq_set_affinity(desc, cpumask_of(smp_processor_id()));
+        desc->handler->startup(desc);
+        /* enable irq */
+        clear_bit(_IRQ_DISABLED, &desc->status);
Quoted 2 lines

Now it turns out this is really done twice: Once in aplic_irq_enable(), and once here.

Agree, this is a job of *_startup()->*_aplic_irq_enable(). I'll drop that too.

~ Oleksii

#70 lore raw

Re: [PATCH v1 12/14] xen/riscv: implement setup_irq()

Quoted 12 lines · Oleksii Kurochko

On 4/15/25 5:55 PM, Jan Beulich wrote

Quoted 11 lines
Quoted 11 lines
xen/arch/riscv/irq.c +11 -0
+        /*
+         * we don't care for now which CPU will receive the
+         * interrupt
+         *
+         * TODO: Handle case where IRQ is setup on different CPU than
+         * the targeted CPU and the priority.
+         */
+        irq_set_affinity(desc, cpumask_of(smp_processor_id()));
+        desc->handler->startup(desc);
+        /* enable irq */
+        clear_bit(_IRQ_DISABLED, &desc->status);
Quoted 4 lines
Quoted 2 lines

Now it turns out this is really done twice: Once in aplic_irq_enable(), and once here.

Agree, this is a job of *_startup()->*_aplic_irq_enable(). I'll drop that too.

Wait - see my comment there. I think it belongs here, not there.

Jan

#71 lore raw

PATCH 13/14 xen/riscv: initialize interrupt controller

In reply to Oleksii Kurochko

Call intc_init() to do basic initialization steps for APLIC and IMISC.

---
 xen/arch/riscv/setup.c | 2 ++
 1 file changed, 2 insertions(+)
xen/arch/riscv/setup.c +2 -0
index a3189697da..9765bcbb08 100644
@@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     intc_preinit();
 
+    intc_init();
+
     printk("All set up\n");
 
     machine_halt();
Signature · 1 line
2.49.0
Acked-by on v1 13/14
#72 lore raw

Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller

Quoted 3 lines · Oleksii Kurochko

Call intc_init() to do basic initialization steps for APLIC and IMISC.

yet ...

xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 143

Quoted 11 lines
xen/arch/riscv/setup.c +2 -0
@@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     intc_preinit();
 
+    intc_init();
+
     printk("All set up\n");
 
     machine_halt();

... this being everything here I wonder if this can't be folded with the patch where the function is introduced.

Jan

#73 lore raw

Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller

Quoted 13 lines · Jan Beulich
Quoted 3 lines · Oleksii Kurochko

Call intc_init() to do basic initialization steps for APLIC and IMISC.

yet ...

Quoted 6 lines
xen/arch/riscv/setup.c +0 -0
@@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
      intc_preinit();
 
Quoted 7 lines

xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 143

Quoted 5 lines
xen/arch/riscv/setup.c +2 -0
+    intc_init();
+
      printk("All set up\n");
 
      machine_halt();

... this being everything here I wonder if this can't be folded with the patch where the function is introduced.

Sure, it can be folded. I will do that to reduce patch series.

~ Oleksii

#74 lore raw

Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller

Quoted 16 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich
Quoted 3 lines · Oleksii Kurochko

Call intc_init() to do basic initialization steps for APLIC and IMISC.

yet ...

Quoted 6 lines
xen/arch/riscv/setup.c +0 -0
@@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
      intc_preinit();
 
Quoted 8 lines
Quoted 7 lines

xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 143

Quoted 5 lines
xen/arch/riscv/setup.c +2 -0
+    intc_init();
+
      printk("All set up\n");
 
      machine_halt();

... this being everything here I wonder if this can't be folded with the patch where the function is introduced.

Sure, it can be folded. I will do that to reduce patch series.

I doubled checked and, at the moment, when intc_init() is introduced:
void __init intc_init(void)

{
    ASSERT(intc_hw_ops);

    if ( intc_hw_ops->init() )
        panic("Failed to initialize the interrupt controller drivers\n");
}

intc_hw_ops isn't registered as they are registered in the next two patches after intriduction of intc_hw_ops.

~ Oleksii

#75 lore raw

Re: [PATCH v1 13/14] xen/riscv: initialize interrupt controller

Quoted 36 lines · Oleksii Kurochko
Quoted 16 lines · Oleksii Kurochko
Quoted 13 lines · Jan Beulich
Quoted 3 lines · Oleksii Kurochko

Call intc_init() to do basic initialization steps for APLIC and IMISC.

yet ...

Quoted 6 lines
xen/arch/riscv/setup.c +0 -0
@@ -136,6 +136,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
      intc_preinit();
 
Quoted 8 lines
Quoted 7 lines

xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 143

Quoted 5 lines
xen/arch/riscv/setup.c +2 -0
+    intc_init();
+
      printk("All set up\n");
 
      machine_halt();

... this being everything here I wonder if this can't be folded with the patch where the function is introduced.

Sure, it can be folded. I will do that to reduce patch series.

I doubled checked and, at the moment, when intc_init() is introduced:
void __init intc_init(void)

{
     ASSERT(intc_hw_ops);

     if ( intc_hw_ops->init() )
         panic("Failed to initialize the interrupt controller drivers\n");
}
Quoted 2 lines

intc_hw_ops isn't registered as they are registered in the next two patches after intriduction of intc_hw_ops.

Which then feels wrong anyway; you're then merely leveraging that the function has no caller, which (as said elsewhere) shouldn't be the case at the very least for Misra's sake. So I expect some re-ordering to be necessary. Or you may want to introduce the function empty and add the intc_hw_ops uses as intc_hw_ops is introduced.

Jan

#76 lore raw

PATCH 14/14 xen/riscv: add basic UART support

In reply to Oleksii Kurochko

Update Kconfig to select GENERIC_UART_INIT for basic UART init ( find a dt node and call device specific device_init() ).

Drop default n if RISCV statement for config HAS_NS16550 as now ns16550 is ready to be compiled and used by RISC-V.

Initialize a minimal amount of stuff to have UART and Xen console:
- Initialize uart by calling uart_init().
- Initialize Xen console by calling console_init_{pre,post}irq().
- Initialize timer and its internal lists which are used by init_timer() which is called by ns16550_init_postirq(); otherwise "Unhandled exception: Store/AMO Page Fault" occurs.
- Enable local interrupt to recieve an input from UART

---
 xen/arch/riscv/Kconfig   |  1 +
 xen/arch/riscv/setup.c   | 16 ++++++++++++++++
 xen/drivers/char/Kconfig |  1 -
 3 files changed, 17 insertions(+), 1 deletion(-)
xen/arch/riscv/Kconfig +1 -0
index 27086cca9c..f5ba7a5a78 100644
@@ -2,6 +2,7 @@ config RISCV
 	def_bool y
 	select FUNCTION_ALIGNMENT_16B
 	select GENERIC_BUG_FRAME
+	select GENERIC_UART_INIT
 	select HAS_DEVICE_TREE
 	select HAS_PMAP
 	select HAS_UBSAN
xen/arch/riscv/setup.c +16 -0
index 9765bcbb08..b5fd660a4b 100644
@@ -4,11 +4,16 @@
 #include <xen/bug.h>
 #include <xen/bootfdt.h>
 #include <xen/compile.h>
+#include <xen/console.h>
 #include <xen/device_tree.h>
 #include <xen/init.h>
 #include <xen/irq.h>
+#include <xen/keyhandler.h>
 #include <xen/mm.h>
+#include <xen/percpu.h>
+#include <xen/serial.h>
 #include <xen/shutdown.h>
+#include <xen/timer.h>
 #include <xen/vmap.h>
 #include <xen/xvmalloc.h>
 
@@ -73,6 +78,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     remove_identity_mapping();
 
+    percpu_init_areas();
+
     smp_clear_cpu_maps();
 
     set_processor_id(0);
@@ -136,8 +143,17 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     intc_preinit();
 
+    uart_init();
+    console_init_preirq();
+
     intc_init();
 
+    timer_init();
+
+    local_irq_enable();
+
+    console_init_postirq();
+
     printk("All set up\n");
 
     machine_halt();
xen/drivers/char/Kconfig +0 -1
index e6e12bb413..01fa31fb2b 100644
@@ -3,7 +3,6 @@ config GENERIC_UART_INIT
 
 config HAS_NS16550
 	bool "NS16550 UART driver" if ARM
-	default n if RISCV
 	default y
 	help
 	  This selects the 16550-series UART support. For most systems, say Y.
Signature · 1 line
2.49.0
#77 lore raw

Re: [PATCH v1 14/14] xen/riscv: add basic UART support

xen/arch/riscv/setup.c line 11

Quoted 11 lines · Oleksii Kurochko
xen/arch/riscv/setup.c +2 -0
@@ -4,11 +4,16 @@
 #include <xen/bug.h>
 #include <xen/bootfdt.h>
 #include <xen/compile.h>
+#include <xen/console.h>
 #include <xen/device_tree.h>
 #include <xen/init.h>
 #include <xen/irq.h>
+#include <xen/keyhandler.h>

Why's this one needed?

xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 81

Quoted 13 lines
xen/arch/riscv/setup.c +4 -0
 #include <xen/mm.h>
+#include <xen/percpu.h>
+#include <xen/serial.h>
 #include <xen/shutdown.h>
+#include <xen/timer.h>
 #include <xen/vmap.h>
 #include <xen/xvmalloc.h>
 
@@ -73,6 +78,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     remove_identity_mapping();
 
+    percpu_init_areas();

I'll trust you that it's needed now, but the addition looks unrelated here, and also isn't mentioned as intentional in the description.

xen/drivers/char/Kconfig config GENERIC_UART_INIT · line 6

Quoted 8 lines
xen/drivers/char/Kconfig +0 -1
@@ -3,7 +3,6 @@ config GENERIC_UART_INIT
 
 config HAS_NS16550
 	bool "NS16550 UART driver" if ARM
-	default n if RISCV
 	default y

Just to double-check: Unlike Arm you don't want this to be user-(un)selectable on RISC-V?

Jan

#78 lore raw

Re: [PATCH v1 14/14] xen/riscv: add basic UART support

Quoted 13 lines · Jan Beulich

xen/arch/riscv/setup.c line 11

Quoted 11 lines · Oleksii Kurochko
xen/arch/riscv/setup.c +2 -0
@@ -4,11 +4,16 @@
  #include <xen/bug.h>
  #include <xen/bootfdt.h>
  #include <xen/compile.h>
+#include <xen/console.h>
  #include <xen/device_tree.h>
  #include <xen/init.h>
  #include <xen/irq.h>
+#include <xen/keyhandler.h>

Why's this one needed?

It isn't needed anymore, just missed to drop.
I thought that it would be needed to test UART working fine by checking if Ctrl+AAA is working.

Quoted 9 lines
Quoted 9 lines
xen/arch/riscv/setup.c +3 -0
  #include <xen/mm.h>
+#include <xen/percpu.h>
+#include <xen/serial.h>
  #include <xen/shutdown.h>
+#include <xen/timer.h>
  #include <xen/vmap.h>
  #include <xen/xvmalloc.h>
 
Quoted 7 lines

xen/arch/riscv/setup.c void __init noreturn start_xen(unsigned long bootcpu_id, · line 81

Quoted 5 lines
xen/arch/riscv/setup.c +1 -0
@@ -73,6 +78,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
      remove_identity_mapping();
 
+    percpu_init_areas();

I'll trust you that it's needed now, but the addition looks unrelated here, and also isn't mentioned as intentional in the description.

I had this patch when I used polling mode for UART, for this case percpu is used to receive serial port info:
struct serial_port *port = this_cpu(poll_port);

So percpu isn't really needed at the current development state. I'll drop this change or as an option move to separate patch.

Quoted 10 lines

xen/drivers/char/Kconfig config GENERIC_UART_INIT · line 6

Quoted 8 lines
xen/drivers/char/Kconfig +0 -1
@@ -3,7 +3,6 @@ config GENERIC_UART_INIT
 
  config HAS_NS16550
  	bool "NS16550 UART driver" if ARM
-	default n if RISCV
  	default y

Just to double-check: Unlike Arm you don't want this to be user-(un)selectable on RISC-V?

Thanks for noticing that. I want to have this selectable by user. I will add RISC-V here.

~ Oleksii

#79 lore raw

Re: [PATCH v1 14/14] xen/riscv: add basic UART support

Quoted 6 lines · Oleksii Kurochko
Quoted 5 lines · Jan Beulich
Quoted 4 lines · Oleksii Kurochko
xen/drivers/char/Kconfig +0 -0
@@ -3,7 +3,6 @@ config GENERIC_UART_INIT
 
Quoted 8 lines
Quoted 6 lines

xen/drivers/char/Kconfig config GENERIC_UART_INIT · line 6

Quoted 4 lines
xen/drivers/char/Kconfig +0 -1
  config HAS_NS16550
  	bool "NS16550 UART driver" if ARM
-	default n if RISCV
  	default y

Just to double-check: Unlike Arm you don't want this to be user-(un)selectable on RISC-V?

Thanks for noticing that. I want to have this selectable by user. I will add RISC-V here.

At which point we may want to consider whether the condition on the prompt wouldn't better become "!X86".

Jan