OpenRISC SMP Support
v2 cover OpenRISC SMP Support
Hello,
This series adds SMP support for OpenRISC. The platform is based on the work that Stefan Kristiansson did around 2012 implemented in Verilog and run on FPGAs. I have been working to upstream this work, these are my QEMU patches I have been used to help with testing.
I have documented the platform in the OpenRISC 1.2 specification proposal available here:
https://github.com/stffrdhrn/doc/raw/arch-1.2-proposal/openrisc-arch-1.2-rev0.pdf
My latest kernel patches are available here:
https://github.com/stffrdhrn/linux.git openrisc-4.14-smp-qspinlock-v3
-Stafford
Changes since v1
* Don't numcores and coreid to state suggested by Richard
Stafford Horne (5):
openrisc/ompic: Add OpenRISC Multicore PIC (OMPIC) target/openrisc: Make coreid and numcores variable openrisc/cputimer: Perparation for Multicore
openrisc: Initial SMP support
openrisc: Only kick cpu on timeout, not on update
default-configs/or1k-softmmu.mak | 1 + hw/intc/Makefile.objs | 1 + hw/intc/ompic.c | 179 +++++++++++++++++++++++++++++++++++++++ hw/openrisc/cputimer.c | 64 ++++++++++---- hw/openrisc/openrisc_sim.c | 84 +++++++++++++----- target/openrisc/cpu.c | 1 - target/openrisc/cpu.h | 4 +- target/openrisc/machine.c | 1 - target/openrisc/sys_helper.c | 9 +- 9 files changed, 298 insertions(+), 46 deletions(-) create mode 100644 hw/intc/ompic.c
Signature · 1 line
2.13.6
PATCH v2 1/5 openrisc/ompic: Add OpenRISC Multicore PIC (OMPIC)
Add OpenRISC Multicore PIC which handles inter processor interrupts (IPI) between cores. In OpenRISC all device interrupts are routed to each core enabling this device to be simple.
--- default-configs/or1k-softmmu.mak | 1 + hw/intc/Makefile.objs | 1 + hw/intc/ompic.c | 179 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 hw/intc/ompic.c
default-configs/or1k-softmmu.mak +1 -0
@@ -2,3 +2,4 @@ CONFIG_SERIAL=y CONFIG_OPENCORES_ETH=y +CONFIG_OMPIC=y
hw/intc/Makefile.objs +1 -0
@@ -43,3 +43,4 @@ obj-$(CONFIG_ASPEED_SOC) += aspeed_vic.o obj-$(CONFIG_ARM_GIC) += arm_gicv3_cpuif.o obj-$(CONFIG_MIPS_CPS) += mips_gic.o obj-$(CONFIG_NIOS2) += nios2_iic.o +obj-$(CONFIG_OMPIC) += ompic.o
hw/intc/ompic.c new file +179 -0
179 lines not on this page · raw message
Signature · 1 line
2.13.6
PATCH v2 2/5 target/openrisc: Make coreid and numcores variable
In reply to Stafford HornePreviously coreid and numcores were hard coded as 0 and 1 respectively as OpenRISC QEMU did not have multicore support.
Multicore support is now being added so these registers need to have configured values.
--- target/openrisc/sys_helper.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
target/openrisc/sys_helper.c +3 -2
@@ -23,6 +23,7 @@#include "exec/exec-all.h" #include "exec/helper-proto.h" #include "exception.h" +#include "sysemu/sysemu.h" #define TO_SPR (group, number) (((group) << 11) + (number))
@@ -249,10 +250,10 @@ target_ulong HELPER(mfspr)(CPUOpenRISCState *env,return env->esr;case TO_SPR (0 ,128 ):/* COREID */ -return 0 ; +return cpu->parent_obj.cpu_index;case TO_SPR (0 ,129 ):/* NUMCORES */ -return 1 ; +return max_cpus; case TO_SPR(0, 1024) ... TO_SPR(0, 1024 + (16 * 32)): /* Shadow GPRs */ idx = (spr -1024 );
Signature · 1 line
2.13.6
PATCH v2 2/5 target/openrisc: Make coreid and numcores variable
Quoted 10 lines · Stafford Horne
Previously coreid and numcores were hard coded as 0 and 1 respectively as OpenRISC QEMU did not have multicore support.
Multicore support is now being added so these registers need to have configured values.
Signed-off-by Stafford Horne <[email protected]>--- target/openrisc/sys_helper.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
r~
PATCH v2 2/5 target/openrisc: Make coreid and numcores variable
Quoted 12 lines · Richard Henderson
Quoted 10 lines · Stafford Horne
Previously coreid and numcores were hard coded as 0 and 1 respectively as OpenRISC QEMU did not have multicore support.
Multicore support is now being added so these registers need to have configured values.
Signed-off-by Stafford Horne <[email protected]>--- target/openrisc/sys_helper.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
Quoted 4 lines
Reviewed-by Richard Henderson <[email protected]>r~
Thank you for the review.
-Stafford
PATCH v2 3/5 openrisc/cputimer: Perparation for Multicore
In reply to Stafford HorneIn order to support multicore system we move some of the previously static state variables into the state of each core.
On the other hand in order to allow timers to be synced between each code the ttcr (tick timer count register) is moved out of the core. This is not as per real hardware spec which has a separate timer counter per core, but it seems the most simple way to keep each clock in sync.
--- hw/openrisc/cputimer.c | 62 +++++++++++++++++++++++++++++++++----------- target/openrisc/cpu.c | 1 - target/openrisc/cpu.h | 4 ++- target/openrisc/machine.c | 1 - target/openrisc/sys_helper.c | 4 +-- 5 files changed, 52 insertions(+), 20 deletions(-)
hw/openrisc/cputimer.c +47 -15
@@ -25,39 +25,56 @@#define TIMER_PERIOD 50/* 50 ns period for 20 MHz timer */ -/* The time when TTCR changes */ -static uint64_t last_clk; -static int is_counting; +/* Tick Timer global state to allow all cores to be in sync */ +typedef struct OR1KTimerState { +uint32_t ttcr; +uint64_t last_clk; +}OR1KTimerState ; +static OR1KTimerState *or1k_timer; + +void cpu_openrisc_count_set (OpenRISCCPU *cpu,uint32_t val) +{ + or1k_timer->ttcr = val; +} + +uint32_t cpu_openrisc_count_get (OpenRISCCPU *cpu) +{ +return or1k_timer->ttcr; +} + +/* Add elapsed ticks to ttcr */ void cpu_openrisc_count_update (OpenRISCCPU *cpu) {uint64_t now; -if (!is_counting) { +if (!cpu->env.is_counting) {return ; } now =qemu_clock_get_ns (QEMU_CLOCK_VIRTUAL ); - cpu->env.ttcr += (uint32_t )((now - last_clk) /TIMER_PERIOD ); - last_clk = now; + or1k_timer->ttcr += (uint32_t )((now - or1k_timer->last_clk) + /TIMER_PERIOD ); + or1k_timer->last_clk = now; } +/* Update the next timeout time as difference between ttmr and ttcr */ void cpu_openrisc_timer_update (OpenRISCCPU *cpu) {uint32_t wait;uint64_t now, next; -if (!is_counting) { +if (!cpu->env.is_counting) {return ; }cpu_openrisc_count_update (cpu); - now = last_clk; + now = or1k_timer->last_clk; -if ((cpu->env.ttmr &TTMR_TP ) <= (cpu->env.ttcr &TTMR_TP )) { - wait =TTMR_TP - (cpu->env.ttcr &TTMR_TP ) +1 ; +if ((cpu->env.ttmr &TTMR_TP ) <= (or1k_timer->ttcr &TTMR_TP )) { + wait =TTMR_TP - (or1k_timer->ttcr &TTMR_TP ) +1 ; wait += cpu->env.ttmr &TTMR_TP ; }else { - wait = (cpu->env.ttmr &TTMR_TP ) - (cpu->env.ttcr &TTMR_TP ); + wait = (cpu->env.ttmr &TTMR_TP ) - (or1k_timer->ttcr &TTMR_TP ); } next = now + (uint64_t )wait *TIMER_PERIOD ;timer_mod (cpu->env.timer, next);
@@ -66,7 +83,7 @@ void cpu_openrisc_timer_update(OpenRISCCPU *cpu)void cpu_openrisc_count_start (OpenRISCCPU *cpu) { - is_counting =1 ; + cpu->env.is_counting =1 ;cpu_openrisc_count_update (cpu); }
@@ -74,7 +91,7 @@ void cpu_openrisc_count_stop(OpenRISCCPU *cpu) {timer_del (cpu->env.timer);cpu_openrisc_count_update (cpu); - is_counting =0 ; + cpu->env.is_counting =0 ; }static void openrisc_timer_cb (void *opaque)
@@ -93,7 +110,7 @@ static void openrisc_timer_cb(void *opaque)case TIMER_NONE :break ;case TIMER_INTR : - cpu->env.ttcr =0 ; + or1k_timer->ttcr =0 ;break ;case TIMER_SHOT :cpu_openrisc_count_stop (cpu);
@@ -105,9 +122,24 @@ static void openrisc_timer_cb(void *opaque)cpu_openrisc_timer_update (cpu); } +static const VMStateDescription vmstate_or1k_timer = { + .name ="or1k_timer" , + .version_id =1 , + .minimum_version_id =1 , + .fields = (VMStateField []) { +VMSTATE_UINT32 (ttcr, OR1KTimerState), +VMSTATE_UINT64 (last_clk, OR1KTimerState), +VMSTATE_END_OF_LIST () + } +}; +void cpu_openrisc_clock_init (OpenRISCCPU *cpu) { cpu->env.timer =timer_new_ns (QEMU_CLOCK_VIRTUAL , &openrisc_timer_cb, cpu); cpu->env.ttmr =0x00000000 ; - cpu->env.ttcr =0x00000000 ; + +if (or1k_timer ==NULL ) { + or1k_timer =g_new0 (OR1KTimerState,1 ); +vmstate_register (NULL ,0 , &vmstate_or1k_timer, or1k_timer); + } }
target/openrisc/cpu.c +0 -1
@@ -61,7 +61,6 @@ static void openrisc_cpu_reset(CPUState *s) cpu->env.picsr =0x00000000 ; cpu->env.ttmr =0x00000000 ; - cpu->env.ttcr =0x00000000 ;#endif }
target/openrisc/cpu.h +3 -1
@@ -315,7 +315,7 @@ typedef struct CPUOpenRISCState {QEMUTimer *timer;uint32_t ttmr;/* Timer tick mode register */ -uint32_t ttcr;/* Timer tick count register */ +int is_counting;uint32_t picmr;/* Interrupt mask register */ uint32_t picsr;/* Interrupt contrl register*/
@@ -373,6 +373,8 @@ void cpu_openrisc_pic_init(OpenRISCCPU *cpu);/* hw/openrisc_timer.c */ void cpu_openrisc_clock_init (OpenRISCCPU *cpu); +uint32_t cpu_openrisc_count_get (OpenRISCCPU *cpu); +void cpu_openrisc_count_set (OpenRISCCPU *cpu,uint32_t val);void cpu_openrisc_count_update (OpenRISCCPU *cpu);void cpu_openrisc_timer_update (OpenRISCCPU *cpu);void cpu_openrisc_count_start (OpenRISCCPU *cpu);
target/openrisc/machine.c +0 -1
@@ -147,7 +147,6 @@ static const VMStateDescription vmstate_env = {VMSTATE_TIMER_PTR (timer, CPUOpenRISCState),VMSTATE_UINT32 (ttmr, CPUOpenRISCState), -VMSTATE_UINT32 (ttcr, CPUOpenRISCState),VMSTATE_UINT32 (picmr, CPUOpenRISCState), VMSTATE_UINT32(picsr, CPUOpenRISCState),
target/openrisc/sys_helper.c +2 -2
@@ -189,7 +189,7 @@ void HELPER(mtspr)(CPUOpenRISCState *env,break ;case TO_SPR (10 ,1 ):/* TTCR */ - env->ttcr = rb; +cpu_openrisc_count_set (cpu, rb);if (env->ttmr &TIMER_NONE ) {return ; }
@@ -312,7 +312,7 @@ target_ulong HELPER(mfspr)(CPUOpenRISCState *env,case TO_SPR (10 ,1 ):/* TTCR */ cpu_openrisc_count_update (cpu); -return env->ttcr; +return cpu_openrisc_count_get (cpu);default :break ;
Signature · 1 line
2.13.6
PATCH v2 4/5 openrisc: Initial SMP support
In reply to Stafford HorneWire in ompic and add basic support for SMP. The OpenRISC is special in that interrupts for devices are routed to each core's PIC. This is achieved using the qemu_irq_split utility, but this currently limits OpenRISC to 2 cores.
This models the reference architecture described in the OpenRISC spec 1.2 proposal.
https://github.com/stffrdhrn/doc/raw/arch-1.2-proposal/openrisc-arch-1.2-rev0.pdf
The changes to the intialization of the sim include:
CPU Reset
o Reset each cpu to the bootstrap PC rather than only a single cpu as done before.
o During Kernel loading the bootstrap PC is saved in a static global.
Network Initialization
o Connect the interrupt to each CPU
o Use more simple sysbus_mmio_map() rather than memory_region_add_subregion()
Sim Initialization
o Initialize the pic and tick timer per cpu
o Wire in the OMPIC if SMP is enabled
o Wire the serial irq to each CPU using qemu_irq_split()
--- hw/openrisc/openrisc_sim.c | 84 +++++++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 23 deletions(-)
hw/openrisc/openrisc_sim.c +61 -23
@@ -35,36 +35,60 @@#define KERNEL_LOAD_ADDR 0x100 +static struct openrisc_boot_info { +uint32_t bootstrap_pc; +} boot_info; +static void main_cpu_reset (void *opaque) {OpenRISCCPU *cpu = opaque; +CPUState *cs =CPU (cpu);cpu_reset (CPU (cpu)); + +cpu_set_pc (cs, boot_info.bootstrap_pc); } -static void openrisc_sim_net_init (MemoryRegion *address_space, -hwaddr base, -hwaddr descriptors, -qemu_irq irq,NICInfo *nd) +static void openrisc_sim_net_init (hwaddr base,hwaddr descriptors, +int num_cpus,qemu_irq **cpu_irqs, +int irq_pin,NICInfo *nd) {DeviceState *dev;SysBusDevice *s; +int i; dev =qdev_create (NULL ,"open_eth" );qdev_set_nic_properties (dev, nd);qdev_init_nofail (dev); s =SYS_BUS_DEVICE (dev); -sysbus_connect_irq (s,0 , irq); -memory_region_add_subregion (address_space, base, -sysbus_mmio_get_region (s,0 )); -memory_region_add_subregion (address_space, descriptors, -sysbus_mmio_get_region (s,1 )); +for (i =0 ; i < num_cpus; i++) { +sysbus_connect_irq (s,0 , cpu_irqs[i][irq_pin]); + } +sysbus_mmio_map (s,0 , base); +sysbus_mmio_map (s,1 , descriptors); } -static void cpu_openrisc_load_kernel (ram_addr_t ram_size, -const char *kernel_filename, -OpenRISCCPU *cpu) +static void openrisc_sim_ompic_init (hwaddr base,int num_cpus, +qemu_irq **cpu_irqs,int irq_pin) +{ +DeviceState *dev; +SysBusDevice *s; +int i; + + dev =qdev_create (NULL ,"or1k-ompic" ); +qdev_prop_set_uint32 (dev,"num-cpus" , num_cpus); +qdev_init_nofail (dev); + + s =SYS_BUS_DEVICE (dev); +for (i =0 ; i < num_cpus; i++) { +sysbus_connect_irq (s, i, cpu_irqs[i][irq_pin]); + } +sysbus_mmio_map (s,0 , base); +} + +static void openrisc_load_kernel (ram_addr_t ram_size, +const char *kernel_filename) {long kernel_size;uint64_t elf_entry;
@@ -83,6 +107,9 @@ static void cpu_openrisc_load_kernel(ram_addr_t ram_size, kernel_size =load_image_targphys (kernel_filename,KERNEL_LOAD_ADDR , ram_size -KERNEL_LOAD_ADDR ); + } + +if (entry <=0 ) { entry =KERNEL_LOAD_ADDR ; }
@@ -91,7 +118,7 @@ static void cpu_openrisc_load_kernel(ram_addr_t ram_size, kernel_filename);exit (1 ); } - cpu->env.pc = entry; + boot_info.bootstrap_pc = entry; } }
@@ -102,6 +129,8 @@ static void openrisc_sim_init(MachineState *machine)const char *kernel_filename = machine->kernel_filename;OpenRISCCPU *cpu =NULL ;MemoryRegion *ram; +qemu_irq *cpu_irqs[2 ]; +qemu_irq serial_irq;int n;if (!cpu_model) {
@@ -114,33 +143,42 @@ static void openrisc_sim_init(MachineState *machine)fprintf (stderr,"Unable to find CPU definition!\n" );exit (1 ); } +cpu_openrisc_pic_init (cpu); + cpu_irqs[n] = (qemu_irq *) cpu->env.irq; + +cpu_openrisc_clock_init (cpu); +qemu_register_reset (main_cpu_reset, cpu); -main_cpu_reset (cpu); } ram =g_malloc (sizeof (*ram));memory_region_init_ram (ram,NULL ,"openrisc.ram" , ram_size, &error_fatal);memory_region_add_subregion (get_system_memory (),0 , ram); -cpu_openrisc_pic_init (cpu); -cpu_openrisc_clock_init (cpu); +if (nd_table[0 ].used) { +openrisc_sim_net_init (0x92000000 ,0x92000400 , smp_cpus, + cpu_irqs,4 , nd_table); + } -serial_mm_init (get_system_memory (),0x90000000 ,0 , cpu->env.irq[2 ], -115200 , serial_hds[0 ],DEVICE_NATIVE_ENDIAN ); +if (smp_cpus >1 ) { +openrisc_sim_ompic_init (0x98000000 , smp_cpus, cpu_irqs,1 ); -if (nd_table[0 ].used) { -openrisc_sim_net_init (get_system_memory (),0x92000000 , -0x92000400 , cpu->env.irq[4 ], nd_table); + serial_irq =qemu_irq_split (cpu_irqs[0 ][2 ], cpu_irqs[1 ][2 ]); + }else { + serial_irq = cpu_irqs[0 ][2 ]; } -cpu_openrisc_load_kernel (ram_size, kernel_filename, cpu); +serial_mm_init (get_system_memory (),0x90000000 ,0 , serial_irq, +115200 , serial_hds[0 ],DEVICE_NATIVE_ENDIAN ); + +openrisc_load_kernel (ram_size, kernel_filename); }static void openrisc_sim_machine_init (MachineClass *mc) { mc->desc ="or1k simulation" ; mc->init = openrisc_sim_init; - mc->max_cpus =1 ; + mc->max_cpus =2 ; mc->is_default =1 ; }
Signature · 1 line
2.13.6
PATCH v2 5/5 openrisc: Only kick cpu on timeout, not on update
In reply to Stafford HornePreviously we were kicking the cpu on every update. This caused problems noticeable in SMP configurations where one CPU got pinned continuously servicing timer exceptions.
--- hw/openrisc/cputimer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
hw/openrisc/cputimer.c +1 -1
@@ -78,7 +78,6 @@ void cpu_openrisc_timer_update(OpenRISCCPU *cpu) } next = now + (uint64_t )wait *TIMER_PERIOD ;timer_mod (cpu->env.timer, next); -qemu_cpu_kick (CPU (cpu)); }void cpu_openrisc_count_start (OpenRISCCPU *cpu)
@@ -120,6 +119,7 @@ static void openrisc_timer_cb(void *opaque) }cpu_openrisc_timer_update (cpu); +qemu_cpu_kick (CPU (cpu)); }static const VMStateDescription vmstate_or1k_timer = {
Signature · 1 line
2.13.6