target-arm: KVM to TCG migration
v2 cover target-arm: KVM to TCG migration
QEMU at the moment doesn't allow the migration between KVM and TCG, in both directions. This limitation is due first of all to the different set of coprocessor registers supported by KVM and TCG, but also in the way the coprocessors values are copied from the incoming data to the local QEMU structures.
This set of patches is focused on the KVM to TCG migration using the CortexA15 processor (in both sides) and proposes a way to mitigate all the blocking discrepancies between the two processors.
The major changes carried by this patch set could be summed as following:
- Excluded any AArch64 register definition from a AArch32 only execution since KVM has none of those registers defined when running on an ARMv7 processor.
- KVM migrates the values of the supported CSSIDR registers as coprocessors cp17 registers. We can do the same in TCG adopting the same encoding used in KVM.
- Some registers are partially supported in TCG mostly because some of their bits reflect tightly the underlying real hardware.
In some cases TCG simply ignores those bits considering only the modelled ones, in some other these bits have the default value.
In both cases however, the values carried by these registers could differ from those migrated by KVM: this is why it is necessary a mechanism to ignore part of a register when we are copying its value from the incoming data. For this purpose a new field is introduced (attr_mask) inside ARMCPRegInfo to explicitly specify, where necessary, which bits are to be considered when comparing the incoming values with the current ones.
- Some definitions of coprocessors registers have been slightly modified to guarantee backward compatibility with the AArch32 definitions. TTBCR, for instance, is one of these registers.
This set of patches has been tested by booting a guest running the Linux 3.13 and 3.14 kernel in a host using KVM.
Two types of hosts have been used: ARM FastModels and the ARM Chromebook. Then, in the middle of the kernel boot, the guest was migrated to a x86 host running TCG.
The machine model used is vexpress-a15 with cortex-a15 as CPU.
Alvise Rigo (7):
Decouple AArch64 cp registers from AArch32
Migrate CCSIDR registers values as cp17 registers
Add a way to mask some unimplemented bits of cp registers Exclude IC bit from L2CTLR register and add attributes mask Make TTBR0/1 and TTBRC cp registers backward compatible with AArch32
ids
Added ADFSR/AIFSR and REVIDR cp registers
Minor cp registers changes for KVM -> TCG migration
target-arm/cpu.c | 72 +++++++++++++++++++++++++++-- target-arm/cpu.h | 2 + target-arm/helper.c | 129 ++++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 179 insertions(+), 24 deletions(-)
Signature · 1 line
1.8.3.2
PATCH v2 1/7 target-arm: Decouple AArch64 cp registers
Relying on the AArch64 views for the migration of cp registers values makes impossible to have a successful migration between TCG and KVM because the latter uses the AArch32 indexes format: force the AArch32 processor to not generate the cp registers belonging to the 64bit architecture.
--- target-arm/helper.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-)
target-arm/helper.c +13 -2
@@ -2313,6 +2313,14 @@ static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,/* Private utility function for define_one_arm_cp_reg_with_opaque(): * add a single reginfo struct to the hash table. */ +CPUARMState *env = &cpu->env; +int is_a64 =arm_feature (env,ARM_FEATURE_AARCH64 ); +if (((r->state ==ARM_CP_STATE_BOTH && state ==ARM_CP_STATE_AA64 + && !is_a64) || (r->state ==ARM_CP_STATE_AA64 && !is_a64))) { +/* no need of the AArch64 cp definition */ +return ; + } +uint32_t *key =g_new (uint32_t,1 );ARMCPRegInfo *r2 =g_memdup (r,sizeof (ARMCPRegInfo));int is64 = (r->type &ARM_CP_64BIT ) ?1 :0 ;
@@ -2322,9 +2330,12 @@ static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,* view handles that. AArch64 also handles reset. * We assume it is a cp15 register. */ +if (is_a64) { +/* this is an AArch64 view */ + r2->type |=ARM_CP_NO_MIGRATE ; + r2->resetfn = arm_cp_reset_ignore; + } r2->cp =15 ; - r2->type |=ARM_CP_NO_MIGRATE ; - r2->resetfn = arm_cp_reset_ignore;#ifdef HOST_WORDS_BIGENDIAN if (r2->fieldoffset) { r2->fieldoffset +=sizeof (uint32_t );
Signature · 1 line
1.8.3.2
PATCH v2 2/7 target-arm: Migrate CCSIDR registers
In reply to Alvise RigoSince KVM migrates the values of the CCSIDR registes as cp17 coprocessors registers values, we do the same in TCG, in such a way to not let the migration fail.
The values of these registers will be read by the guest with the usual mechanism (writing into CSSELR to select the desired Cache Size ID register and then reading it from the CCSIDR register).
--- target-arm/cpu.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+)
target-arm/cpu.c +64 -0
@@ -692,6 +692,51 @@ static uint64_t a15_l2ctlr_read(CPUARMState *env, const ARMCPRegInfo *ri) }#endif +static void cssidr_id_decode (const ARMCPRegInfo *ri, +uint32_t *level,uint32_t *instr_bit) +{ +/* the Cortex-A15 doesn't support more than 2 levels of cache, +* so both the instruction bit and level bits of the CSSELR encoding +* stay in the first 3 bits - opc2 bits */ + *level = (ri->opc2 >>1 ) &3 ; + *instr_bit = ri->opc2 &1 ; +} + +static uint64_t a15_read_ccsidr (CPUARMState *env,const ARMCPRegInfo *ri) +{ +ARMCPU *cpu =arm_env_get_cpu (env); +uint32_t level, instr_bit; +cssidr_id_decode (ri, &level, &instr_bit); + +switch (level) { +case 0 :/* both L1 caches supported */ +return cpu->ccsidr[instr_bit]; +case 1 :/* only one L2 unified cache */ +if (!instr_bit) { +return cpu->ccsidr[2 ]; + } + } + +return 0 ; +} + +static void a15_write_ccsidr (CPUARMState *env,const ARMCPRegInfo *ri, +uint64_t value) +{ +ARMCPU *cpu =arm_env_get_cpu (env); +uint32_t level, instr_bit; +cssidr_id_decode (ri, &level, &instr_bit); + +switch (level) { +case 0 :/* both L1 caches supported */ + cpu->ccsidr[instr_bit] = value; +case 1 :/* only one L2 unified cache */ +if (!instr_bit) { + cpu->ccsidr[2 ] = value; + } + } +} +static const ARMCPRegInfo cortexa15_cp_reginfo[] = { #ifndef CONFIG_USER_ONLY { .name ="L2CTLR" , .cp =15 , .crn =9 , .crm =0 , .opc1 =1 , .opc2 =2 ,
@@ -700,6 +745,25 @@ static const ARMCPRegInfo cortexa15_cp_reginfo[] = {#endif { .name = "L2ECTLR", .cp = 15, .crn = 9, .crm = 0, .opc1 = 1, .opc2 = 3, .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 }, + /* The following registers follows the CSSELR econding and are only used + * to migrate their data; the default values are set by cortex_a15_initfn. + * Their values are read by the system through the CCSELR and CCSIDR + * coprocessor registers. */ + { .name = "L1_DCACHE", + .cp = 17, .opc1 = 0, .opc2 = 0, .crn = 0, .crm = 0, + .access = PL0_RW, .resetfn = arm_cp_reset_ignore, + .writefn = a15_write_ccsidr, .readfn = a15_read_ccsidr, + .state = ARM_CP_STATE_AA32, }, + { .name = "L1_ICACHE", + .cp = 17, .opc1 = 0, .opc2 = 1, .crn = 0, .crm = 0, + .access = PL0_RW, .resetfn = arm_cp_reset_ignore, + .writefn = a15_write_ccsidr, .readfn = a15_read_ccsidr, + .state = ARM_CP_STATE_AA32, }, + { .name = "L2_CACHE", + .cp = 17, .opc1 = 0, .opc2 = 2, .crn = 0, .crm = 0, + .access = PL0_RW, .resetfn = arm_cp_reset_ignore, + .writefn = a15_write_ccsidr, .readfn = a15_read_ccsidr, + .state = ARM_CP_STATE_AA32, }, REGINFO_SENTINEL };
Signature · 1 line
1.8.3.2
PATCH v2 3/7 target-arm: Add a way to mask some
In reply to Alvise RigoOne of the issues blocking the migration from KVM to TCG is that the former processor reads the coprocessors values directly from the underlying hardware. TCG however, for the way it emulates some registers, is not able to verify the incoming values, failing the migration.
Add a mask to the ARMCPRegInfo structure to exclude the bits not supported of the incoming registers values.
In case of an outgoing migration, the unsupported data is retrieved from the cpreg_vmstate array.
--- target-arm/cpu.h | 1 + target-arm/helper.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-)
target-arm/cpu.h +1 -0
@@ -955,6 +955,7 @@ struct ARMCPRegInfo {* fieldoffset is 0 then no reset will be done. */ CPResetFn *resetfn; +uint64_t attr_mask; };/* Macros which are lvalues for the field in CPUARMState for the
target-arm/helper.c +47 -2
67 lines not on this page · raw message
Signature · 1 line
1.8.3.2
PATCH v2 4/7 target-arm: Exclude IC bit from L2CTLR
In reply to Alvise RigoThe real hardware seems to not set the Interrupt Controller bit of the L2CTLR cp register; on the contrary it could set some other bits regarding RAM features that are not modelled in TCG, so we can mask them out.
--- target-arm/cpu.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
target-arm/cpu.c +3 -5
@@ -685,10 +685,8 @@ static void cortex_a9_initfn(Object *obj)#ifndef CONFIG_USER_ONLY static uint64_t a15_l2ctlr_read (CPUARMState *env,const ARMCPRegInfo *ri) { -/* Linux wants the number of processors from here. -* Might as well set the interrupt-controller bit too. -*/ -return ((smp_cpus -1 ) <<24 ) | (1 <<23 ); +/* Linux wants the number of processors from here. */ +return (smp_cpus -1 ) <<24 ; }#endif
@@ -741,7 +739,7 @@ static const ARMCPRegInfo cortexa15_cp_reginfo[] = { #ifndef CONFIG_USER_ONLY { .name = "L2CTLR", .cp = 15, .crn = 9, .crm = 0, .opc1 = 1, .opc2 = 2, .access = PL1_RW, .resetvalue = 0, .readfn = a15_l2ctlr_read, - .writefn = arm_cp_write_ignore, }, + .writefn = arm_cp_write_ignore, .attr_mask = (3 << 24),}, #endif { .name = "L2ECTLR", .cp = 15, .crn = 9, .crm = 0, .opc1 = 1, .opc2 = 3, .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
Signature · 1 line
1.8.3.2
PATCH v2 5/7 target-arm: Make TTBR0/1 and TTBRC cp
In reply to Alvise RigoRevisit the definitions of the cp registers TTBR0/1 and TTBRC in such a way to use the AArch32 ids format when the guest is using a 32bit model.
--- target-arm/helper.c | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-)
target-arm/helper.c +27 -11
@@ -1390,6 +1390,16 @@ static void vmsa_ttbcr_raw_write(CPUARMState *env, const ARMCPRegInfo *ri, env->cp15.c2_base_mask = ~((uint32_t )0x3fffu >> maskshift); } +static void vmsa_control_raw_write (CPUARMState *env,const ARMCPRegInfo *ri, +uint64_t value) +{ +if (ri->state ==ARM_CP_STATE_AA64 ) { +raw_write (env, ri, value); + }else { +vmsa_ttbcr_raw_write (env, ri, value); + } +} +static void vmsa_ttbcr_write (CPUARMState *env,const ARMCPRegInfo *ri,uint64_t value) {
@@ -1417,6 +1427,16 @@ static void vmsa_tcr_el1_write(CPUARMState *env, const ARMCPRegInfo *ri, env->cp15.c2_control = value; } +static void vmsa_control_write (CPUARMState *env,const ARMCPRegInfo *ri, +uint64_t value) +{ +if (ri->state ==ARM_CP_STATE_AA64 ) { +vmsa_tcr_el1_write (env, ri, value); + }else { +vmsa_ttbcr_write (env, ri, value); + } +} +static void vmsa_ttbr_write (CPUARMState *env,const ARMCPRegInfo *ri,uint64_t value) {
@@ -1439,20 +1459,16 @@ static const ARMCPRegInfo vmsa_cp_reginfo[] = { { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH, .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0, .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el1), - .writefn = vmsa_ttbr_write, .resetvalue = 0 }, + .writefn = vmsa_ttbr_write, .resetvalue = 0, .type = ARM_CP_NO_MIGRATE, }, { .name = "TTBR1_EL1", .state = ARM_CP_STATE_BOTH, .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 1, .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el1), - .writefn = vmsa_ttbr_write, .resetvalue = 0 }, - { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64, + .writefn = vmsa_ttbr_write, .resetvalue = 0, .type = ARM_CP_NO_MIGRATE, }, + { .name = "TCR_EL1", .state = ARM_CP_STATE_BOTH, .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2, - .access = PL1_RW, .writefn = vmsa_tcr_el1_write, - .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write, + .access = PL1_RW, .writefn = vmsa_control_write, + .resetfn = vmsa_ttbcr_reset, .raw_writefn = vmsa_control_raw_write, .fieldoffset = offsetof(CPUARMState, cp15.c2_control) }, - { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2, - .access = PL1_RW, .type = ARM_CP_NO_MIGRATE, .writefn = vmsa_ttbcr_write, - .resetfn = arm_cp_reset_ignore, .raw_writefn = vmsa_ttbcr_raw_write, - .fieldoffset = offsetoflow32(CPUARMState, cp15.c2_control) }, { .name = "DFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0, .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c6_data), .resetvalue = 0, },
@@ -1683,11 +1699,11 @@ static const ARMCPRegInfo lpae_cp_reginfo[] = { .access = PL1_RW, .type = ARM_CP_64BIT, .readfn = par64_read, .writefn = par64_write, .resetfn = par64_reset }, { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0, - .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_NO_MIGRATE, + .access = PL1_RW, .type = ARM_CP_64BIT, .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el1), .writefn = vmsa_ttbr_write, .resetfn = arm_cp_reset_ignore }, { .name = "TTBR1", .cp = 15, .crm = 2, .opc1 = 1, - .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_NO_MIGRATE, + .access = PL1_RW, .type = ARM_CP_64BIT, .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el1), .writefn = vmsa_ttbr_write, .resetfn = arm_cp_reset_ignore },REGINFO_SENTINEL
Signature · 1 line
1.8.3.2
PATCH v2 6/7 target-arm: Added ADFSR/AIFSR and REVIDR
In reply to Alvise RigoThese registers are required in TCG because they are migrated by KVM:
their absence from the cpreg table leads to a migration failure.
--- target-arm/cpu.h | 1 + target-arm/helper.c | 9 +++++++++ 2 files changed, 10 insertions(+)
target-arm/cpu.h +1 -0
@@ -170,6 +170,7 @@ typedef struct CPUARMState {/* System control coprocessor (cp15) */ struct {uint32_t c0_cpuid; +uint32_t c0_revid;uint64_t c0_cssel;/* Cache size selection. */ uint64_t c1_sys;/* System control register. */ uint64_t c1_coproc;/* Coprocessor access register. */
target-arm/helper.c +9 -0
@@ -1456,6 +1456,10 @@ static const ARMCPRegInfo vmsa_cp_reginfo[] = { { .name = "IFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1, .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c5_insn), .resetvalue = 0, }, + { .name = "ADFSR", .cp = 15, .crn = 5, .crm = 1, .opc1 = 0, .opc2 = 0, + .access = PL1_R, .resetvalue = 0, .type = ARM_CP_CONST, }, + { .name = "AIFSR", .cp = 15, .crn = 5, .crm = 1, .opc1 = 0, .opc2 = 1, + .access = PL1_R, .resetvalue = 0, .type = ARM_CP_CONST, }, { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH, .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0, .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el1),
@@ -2148,6 +2152,11 @@ void register_cp_regs_for_features(ARMCPU *cpu) .writefn = arm_cp_write_ignore, .raw_writefn = raw_write, .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid), .type = ARM_CP_OVERRIDE }, + { .name = "REVIDR", + .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 6, + .access = PL1_R, .resetvalue = 0, + .writefn = arm_cp_write_ignore, .raw_writefn = raw_write, + .fieldoffset = offsetof(CPUARMState, cp15.c0_revid),}, { .name = "MIDR_EL1", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 0, .opc2 = 0, .crn = 0, .crm = 0, .access = PL1_R, .resetvalue = cpu->midr, .type = ARM_CP_CONST },
Signature · 1 line
1.8.3.2
PATCH v2 7/7 target-arm: Minor cp registers changes
In reply to Alvise RigoMinor changes still required to enable KVM -> TCG migration:
* The MPIDR register is now migrated, as KVM does.
* Implemented the SMP bit of the AUXCR register, the other bits are masked through attr_mask because not supported.
* Added the attr_mask to ID_PFR0 to exclude the Jazelle bits (in TCG these bits are set in the reset value of the register, probably we should consider to remove them).
--- target-arm/helper.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-)
target-arm/helper.c +14 -4
@@ -1660,7 +1660,7 @@ static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)static const ARMCPRegInfo mpidr_cp_reginfo[] = { { .name ="MPIDR" , .state =ARM_CP_STATE_BOTH , .opc0 =3 , .crn =0 , .crm =0 , .opc1 =0 , .opc2 =5 , - .access =PL1_R , .readfn = mpidr_read, .type =ARM_CP_NO_MIGRATE }, + .access =PL1_R , .readfn = mpidr_read },REGINFO_SENTINEL };
@@ -1911,6 +1911,16 @@ static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)return CP_ACCESS_OK ; } +static uint64_t auxcr_reg_read (CPUARMState *env,const ARMCPRegInfo *ri) +{ +if (smp_cpus >1 ) { +/* set the SMP bit accordingly with the number of CPUs */ +return 1 <<6 ; + } + +return 0 ; +} +static void define_aarch64_debug_regs (ARMCPU *cpu) {/* Define breakpoint and watchpoint registers. These do nothing
@@ -1957,7 +1967,7 @@ void register_cp_regs_for_features(ARMCPU *cpu)ARMCPRegInfo v6_idregs[] = { { .name ="ID_PFR0" , .cp =15 , .crn =0 , .crm =1 , .opc1 =0 , .opc2 =0 , .access =PL1_R , .type =ARM_CP_CONST , - .resetvalue = cpu->id_pfr0 }, + .resetvalue = cpu->id_pfr0, .attr_mask = ~(0xF <<8 )}, { .name ="ID_PFR1" , .cp =15 , .crn =0 , .crm =1 , .opc1 =0 , .opc2 =1 , .access =PL1_R , .type =ARM_CP_CONST , .resetvalue = cpu->id_pfr1 },
@@ -2219,8 +2229,8 @@ void register_cp_regs_for_features(ARMCPU *cpu)if (arm_feature (env,ARM_FEATURE_AUXCR )) {ARMCPRegInfo auxcr = { .name ="AUXCR" , .cp =15 , .crn =1 , .crm =0 , .opc1 =0 , .opc2 =1 , - .access =PL1_RW , .type =ARM_CP_CONST , - .resetvalue = cpu->reset_auxcr + .access =PL1_RW , .resetvalue =0 , .readfn = auxcr_reg_read, + .writefn = arm_cp_write_ignore, .attr_mask = (1 <<6 ), };define_one_arm_cp_reg (cpu, &auxcr); }
Signature · 1 line
1.8.3.2