target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migration
PATCH 1/1 target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migration
This patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration due to incorrect privilege state restoration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive. This patch ensures all cores are properly set to runnable state after migration.
Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.
--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
target/riscv/kvm/kvm-cpu.c +8 -0
@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);
ret = kvm_get_one_reg(cs, id, ®);
@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)
return ret;
}+ /* Save guest privilege level before migration */
+ reg = env->priv;
+ if (reg != 3) {+ ret = kvm_set_one_reg(cs, RISCV_CORE_REG(env, mode), ®);
+ if (ret) {
+ return ret;
+ }
+ }
+
for (i = 1; i < 32; i++) {
uint64_t id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CORE, i);
reg = env->gpr[i];
@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)
return ret;
}+ /* Ensure all non-core 0 CPUs are runnable after migration */
+ if ((level == KVM_PUT_FULL_STATE) && (cs->cpu_index != 0)) {
+ RISCVCPU *cpu = RISCV_CPU(cs);
+ ret = kvm_riscv_sync_mpstate_to_kvm(cpu, KVM_MP_STATE_RUNNABLE);
+ if (ret) {+ return ret;
+ }
+ }
if (KVM_PUT_RESET_STATE == level) {
RISCVCPU *cpu = RISCV_CPU(cs);
if (cs->cpu_index == 0) {Signature · 4 lines
2.34.1 ______________________www.ultrarisc.com 重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
This is v2 of this patch with no functional changes; adding CC.
--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
target/riscv/kvm/kvm-cpu.c +8 -0
@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);
ret = kvm_get_one_reg(cs, id, ®);
@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)
return ret;
}+ /* Save guest privilege level before migration */
+ reg = env->priv;
+ if (reg != 3) {+ ret = kvm_set_one_reg(cs, RISCV_CORE_REG(env, mode), ®);
+ if (ret) {
+ return ret;
+ }
+ }
+
for (i = 1; i < 32; i++) {
uint64_t id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CORE, i);
reg = env->gpr[i];
@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)
return ret;
}+ /* Ensure all non-core 0 CPUs are runnable after migration */
+ if ((level == KVM_PUT_FULL_STATE) && (cs->cpu_index != 0)) {
+ RISCVCPU *cpu = RISCV_CPU(cs);
+ ret = kvm_riscv_sync_mpstate_to_kvm(cpu, KVM_MP_STATE_RUNNABLE);
+ if (ret) {+ return ret;
+ }
+ }
if (KVM_PUT_RESET_STATE == level) {
RISCVCPU *cpu = RISCV_CPU(cs);
if (cs->cpu_index == 0) {Signature · 1 line
2.34.1
Quoted 78 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-05-19 17:41:36 (星期一)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected]
主题: [PATCH] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration due to incorrect privilege state restoration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive. This patch ensures all cores are properly set to runnable state after migration.
Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.Signed-off-by Xie Bo <[email protected]>--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +8
-0@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);ret = kvm_get_one_reg(cs, id, ®);target/riscv/kvm/kvm-cpu.c +17
-0@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/* Save guest privilege level before migration */ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/* Ensure all non-core 0 CPUs are runnable after migration */ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )) { +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); +if (ret) { +return ret; + } + }if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) {Signature · 4 lines
2.34.1 ______________________www.ultrarisc.com 重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
PATCH 1/1 target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migration
This is v3 of this patch to fix patch format
This patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive.
Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.
--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
target/riscv/kvm/kvm-cpu.c +23 -0
@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/*Restore the guest's privileged level after migration*/ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); ret =kvm_get_one_reg (cs, id, ®);
@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/*Save guest privilege level before migration*/ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];
@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/*Ensure all non-core 0 CPUs are runnable after migration*/ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )){ +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); + } +if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) {
--
2.34.1
Quoted 151 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-05-26 15:45:52 (星期一)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected], [email protected], [email protected] 主题: Re: [PATCH V2] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis is v2 of this patch with no functional changes; adding CC.
--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +8
-0@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);ret = kvm_get_one_reg(cs, id, ®);target/riscv/kvm/kvm-cpu.c +17
-0@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/* Save guest privilege level before migration */ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/* Ensure all non-core 0 CPUs are runnable after migration */ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )) { +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); +if (ret) { +return ret; + } + }if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) {Signature · 1 line
2.34.1Quoted 78 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-05-19 17:41:36 (星期一)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected]
主题: [PATCH] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration due to incorrect privilege state restoration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive. This patch ensures all cores are properly set to runnable state after migration.
Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.Signed-off-by Xie Bo <[email protected]>--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +8
-0@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);ret = kvm_get_one_reg(cs, id, ®);target/riscv/kvm/kvm-cpu.c +17
-0@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/* Save guest privilege level before migration */ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/* Ensure all non-core 0 CPUs are runnable after migration */ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )) { +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); +if (ret) { +return ret; + } + }if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) {Signature · 4 lines
2.34.1 ______________________www.ultrarisc.com 重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
Hi XieBo,
Could you give the method on how to test it?
Thanks,
Zhiwei
获取Outlook for Android<https://aka.ms/AAb9ysg>
________________________________
From: [email protected] <[email protected]> on behalf of 谢波 <[email protected]>
Sent: Friday, July 11, 2025 5:28:10 PM
To: [email protected] <[email protected]>
Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Subject: [PATCH for v10.0.0] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migration
This is v3 of this patch to fix patch format
This patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive.
Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.
--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
target/riscv/kvm/kvm-cpu.c +23 -0
@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/*Restore the guest's privileged level after migration*/ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); ret =kvm_get_one_reg (cs, id, ®);
@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/*Save guest privilege level before migration*/ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];
@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/*Ensure all non-core 0 CPUs are runnable after migration*/ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )){ +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); + } +if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) {
--
2.34.1
Quoted 145 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-05-26 15:45:52 (星期一)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected], [email protected], [email protected] 主题: Re: [PATCH V2] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis is v2 of this patch with no functional changes; adding CC.
--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +8
-0@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);ret = kvm_get_one_reg(cs, id, ®);target/riscv/kvm/kvm-cpu.c +17
-1@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/* Save guest privilege level before migration */ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/* Ensure all non-core 0 CPUs are runnable after migration */ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )) { +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); +if (ret) { +return ret; + } + }if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) { --2.34.1
Quoted 76 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-05-19 17:41:36 (星期一)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected]
主题: [PATCH] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration due to incorrect privilege state restoration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive. This patch ensures all cores are properly set to runnable state after migration.
Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.Signed-off-by Xie Bo <[email protected]>--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +8
-0@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);ret = kvm_get_one_reg(cs, id, ®);target/riscv/kvm/kvm-cpu.c +17
-1@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/* Save guest privilege level before migration */ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/* Ensure all non-core 0 CPUs are runnable after migration */ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )) { +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); +if (ret) { +return ret; + } + }if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) { --2.34.1
Quoted 6 lines
Quoted 2 lines
______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
Hi Zhiwei,
You need a machine that supports hardware virtualization and add the --accel kvm option when running QEMU; the migration works the same way as in TCG mode.
Best regards,
XieBo
-----原始邮件-----
发件人:"Zhiwei LIU" <[email protected]>
发送时间:2025-07-12 06:46:14 (星期六)
收件人: 谢波 <[email protected]>, "[email protected]" <[email protected]>
抄送: "[email protected]" <[email protected]>, "[email protected]" <[email protected]>, "[email protected]" <[email protected]>, "[email protected]" <[email protected]>, "[email protected]" <[email protected]> 主题: Re: [PATCH for v10.0.0] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migration
Hi XieBo,
Could you give the method on how to test it?
Thanks,
Zhiwei
获取Outlook for Android
From: [email protected] <[email protected]> on behalf of 谢波 <[email protected]>
Sent: Friday, July 11, 2025 5:28:10 PM
To: [email protected] <[email protected]>
Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Subject: [PATCH for v10.0.0] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migration
This is v3 of this patch to fix patch format
This patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive.
Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.
--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
target/riscv/kvm/kvm-cpu.c +23 -0
@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/*Restore the guest's privileged level after migration*/ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); ret =kvm_get_one_reg (cs, id, ®);
@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/*Save guest privilege level before migration*/ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];
@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/*Ensure all non-core 0 CPUs are runnable after migration*/ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )){ +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); + } +if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) {
--
2.34.1
Quoted 145 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-05-26 15:45:52 (星期一)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected], [email protected], [email protected] 主题: Re: [PATCH V2] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis is v2 of this patch with no functional changes; adding CC.
--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +8
-0@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);ret = kvm_get_one_reg(cs, id, ®);target/riscv/kvm/kvm-cpu.c +17
-1@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/* Save guest privilege level before migration */ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/* Ensure all non-core 0 CPUs are runnable after migration */ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )) { +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); +if (ret) { +return ret; + } + }if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) { --2.34.1
Quoted 76 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-05-19 17:41:36 (星期一)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected]
主题: [PATCH] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration due to incorrect privilege state restoration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive. This patch ensures all cores are properly set to runnable state after migration.
Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.Signed-off-by Xie Bo <[email protected]>--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +8
-0@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);ret = kvm_get_one_reg(cs, id, ®);target/riscv/kvm/kvm-cpu.c +17
-1@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/* Save guest privilege level before migration */ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/* Ensure all non-core 0 CPUs are runnable after migration */ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )) { +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); +if (ret) { +return ret; + } + }if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) { --2.34.1
Quoted 6 lines
Quoted 2 lines
______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
PATCH v4 1/1 target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migration
In reply to 谢波This is v4 of this patch to add the function of saving and restoring the running status of vCPU during migration
This patch fixes two critical bugs in QEMU with KVM:
Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive. This patch addresses both problems to ensure stable guest operation after migration.
--- target/riscv/cpu.h | 1 + target/riscv/kvm/kvm-cpu.c | 63 ++++++++++++++++++++++++++++++++---- target/riscv/kvm/kvm_riscv.h | 3 +- target/riscv/machine.c | 1 + 4 files changed, 61 insertions(+), 7 deletions(-)
target/riscv/cpu.h +1 -0
@@ -256,6 +256,7 @@ struct CPUArchState {#endif target_ulong priv; +uint32_t mp_state;/*current multiprocessor state of this vCPU*/ /* CSRs for execution environment configuration */ uint64_t menvcfg;target_ulong senvcfg;
target/riscv/kvm/kvm-cpu.c +57 -6
107 lines not on this page · raw message
target/riscv/kvm/kvm_riscv.h +2 -1
@@ -28,7 +28,8 @@ void kvm_riscv_aia_create(MachineState *machine, uint64_t group_shift, uint64_t aplic_base, uint64_t imsic_base, uint64_t guest_num);void riscv_kvm_aplic_request (void *opaque,int irq,int level); -int kvm_riscv_sync_mpstate_to_kvm (RISCVCPU *cpu,int state); +int kvm_riscv_sync_mpstate_to_kvm (CPUState *cs); +int kvm_riscv_sync_mpstate_to_qemu (CPUState *cs);void riscv_kvm_cpu_finalize_features (RISCVCPU *cpu,Error **errp);uint64_t kvm_riscv_get_timebase_frequency (RISCVCPU *cpu);
target/riscv/machine.c +1 -0
@@ -422,6 +422,7 @@ const VMStateDescription vmstate_riscv_cpu = {VMSTATE_UNUSED (4 ),VMSTATE_UINT32 (env.misa_ext_mask,RISCVCPU ),VMSTATE_UINTTL (env.priv,RISCVCPU ), +VMSTATE_UINT32 (env.mp_state,RISCVCPU ),VMSTATE_BOOL (env.virt_enabled,RISCVCPU ),VMSTATE_UINT64 (env.resetvec,RISCVCPU ), VMSTATE_UINTTL(env.mhartid, RISCVCPU),
--
Quoted 229 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-07-11 17:28:10 (星期五)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected], [email protected], [email protected] 主题: [PATCH for v10.0.0] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis is v3 of this patch to fix patch format
This patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive.Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.Signed-off-by Xie Bo <[email protected]>--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +23
-1@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/*Restore the guest's privileged level after migration*/ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); ret =kvm_get_one_reg (cs, id, ®);@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/*Save guest privilege level before migration*/ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/*Ensure all non-core 0 CPUs are runnable after migration*/ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )){ +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); + } +if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) { --2.34.1
Quoted 151 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-05-26 15:45:52 (星期一)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected], [email protected], [email protected] 主题: Re: [PATCH V2] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis is v2 of this patch with no functional changes; adding CC.
--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +8
-0@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);ret = kvm_get_one_reg(cs, id, ®);target/riscv/kvm/kvm-cpu.c +17
-0@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/* Save guest privilege level before migration */ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/* Ensure all non-core 0 CPUs are runnable after migration */ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )) { +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); +if (ret) { +return ret; + } + }if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) {Signature · 1 line
2.34.1Quoted 78 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-05-19 17:41:36 (星期一)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected]
主题: [PATCH] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration due to incorrect privilege state restoration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive. This patch ensures all cores are properly set to runnable state after migration.
Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.Signed-off-by Xie Bo <[email protected]>--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +8
-0@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);ret = kvm_get_one_reg(cs, id, ®);target/riscv/kvm/kvm-cpu.c +17
-0@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/* Save guest privilege level before migration */ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/* Ensure all non-core 0 CPUs are runnable after migration */ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )) { +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); +if (ret) { +return ret; + } + }if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) {Signature · 4 lines
2.34.1 ______________________www.ultrarisc.com 重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
Quoted 2 lines
______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
Quoted 20 lines · 谢波
This is v4 of this patch to add the function of saving and restoring the running status of vCPU during migration
This patch fixes two critical bugs in QEMU with KVM:
Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive. This patch addresses both problems to ensure stable guest operation after migration.Signed-off-by Xie Bo <[email protected]>--- target/riscv/cpu.h | 1 + target/riscv/kvm/kvm-cpu.c | 63 ++++++++++++++++++++++++++++++++---- target/riscv/kvm/kvm_riscv.h | 3 +- target/riscv/machine.c | 1 + 4 files changed, 61 insertions(+), 7 deletions(-)target/riscv/cpu.h +0
-0
target/riscv/cpu.h struct CPUArchState { · line 259
Quoted 4 lines
target/riscv/cpu.h +1
-0@@ -256,6 +256,7 @@ struct CPUArchState {#endif target_ulong priv; +uint32_t mp_state;/*current multiprocessor state of this vCPU*/
This patch has several formatting problems, such as missing spaces between the '/*' and the text. Also please capitalize first letters of sentences and use punctuation, such as periods, in comments.
Try running scripts/checkpatch.pl to find formatting issues.
target/riscv/kvm/kvm-cpu.c static int kvm_riscv_get_regs_core(CPUState *cs) · line 579
Quoted 12 lines
target/riscv/cpu.h +0
-0/* CSRs for execution environment configuration */ uint64_t menvcfg;target_ulong senvcfg;target/riscv/kvm/kvm-cpu.c +1
-0@@ -576,6 +576,15 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/*Save the guest's privileged state before migration*/
This comment can be dropped since we know the purpose of get-regs.
target/riscv/kvm/kvm-cpu.c static int kvm_riscv_get_regs_core(CPUState *cs) · line 584
Quoted 5 lines
target/riscv/kvm/kvm-cpu.c +5
-0+ ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=PRV_M ) {
Missing space between 'if' and '('. Hopefully checkpatch complains about stuff like that. Also priv == M should never be true for a KVM guest.
target/riscv/kvm/kvm-cpu.c static int kvm_riscv_put_regs_core(CPUState *cs) · line 613
Quoted 11 lines
target/riscv/kvm/kvm-cpu.c +4
-0+ env->priv = reg; + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); ret =kvm_get_one_reg (cs, id, ®);@@ -601,6 +610,16 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/*Restore the guest's privileged state after migration*/
This comment can be dropped since we know the purpose of put-regs.
target/riscv/kvm/kvm-cpu.c static int kvm_riscv_put_regs_core(CPUState *cs) · line 616
Quoted 3 lines
target/riscv/kvm/kvm-cpu.c +3
-0+ reg = env->priv; + +if (reg !=PRV_M ) {
Should never be true.
Quoted 15 lines
target/riscv/kvm/kvm-cpu.c +8
-0+ ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1244,22 +1263,46 @@ int kvm_arch_get_registers(CPUState *cs, Error **errp)return ret; } + ret =kvm_riscv_sync_mpstate_to_qemu (cs); +if (ret) {
target/riscv/kvm/kvm-cpu.c int kvm_arch_get_registers(CPUState *cs, Error **errp) · line 1276
Quoted 10 lines
target/riscv/kvm/kvm-cpu.c +5
-1+return ret; + } +return ret; } -int kvm_riscv_sync_mpstate_to_kvm (RISCVCPU *cpu,int state) +int kvm_riscv_sync_mpstate_to_kvm (CPUState *cs) { +CPURISCVState *env = &RISCV_CPU (cs)->env;
Need a blank line here.
target/riscv/kvm/kvm-cpu.c int kvm_arch_get_registers(CPUState *cs, Error **errp) · line 1282
Quoted 8 lines
target/riscv/kvm/kvm-cpu.c +2
-2if (cap_has_mp_state) {struct kvm_mp_state mp_state = { - .mp_state = state + .mp_state = env->mp_state }; -int ret =kvm_vcpu_ioctl (CPU (cpu),KVM_SET_MP_STATE , &mp_state); +int ret =kvm_vcpu_ioctl (cs,KVM_SET_MP_STATE , &mp_state);
Put 'int ret' at the top of the function.
target/riscv/kvm/kvm-cpu.c int kvm_arch_get_registers(CPUState *cs, Error **errp) · line 1284
Quoted 2 lines
target/riscv/kvm/kvm-cpu.c +2
-0+if (ret) { + fprintf(stderr, "%s: failed to sync MP_STATE %d/%s\n",
We should add "to KVM" to this error message.
target/riscv/kvm/kvm-cpu.c int kvm_arch_get_registers(CPUState *cs, Error **errp) · line 1286
Quoted 2 lines
target/riscv/kvm/kvm-cpu.c +2
-0+ __func__, ret, strerror(-ret)); +return -1 ;
We should return 'ret' instead of -1. So just change this to
if (ret) {
fprintf(...);}
return ret;
Quoted 10 lines
target/riscv/kvm/kvm-cpu.c +10
-0+ } + } + +return 0 ; +} + +int kvm_riscv_sync_mpstate_to_qemu (CPUState *cs) +{ +CPURISCVState *env = &RISCV_CPU (cs)->env; +if (cap_has_mp_state) {
target/riscv/kvm/kvm-cpu.c int kvm_arch_get_registers(CPUState *cs, Error **errp) · line 1306
Quoted 10 lines
target/riscv/kvm/kvm-cpu.c +4
-0+struct kvm_mp_state mp_state; + +int ret =kvm_vcpu_ioctl (cs,KVM_GET_MP_STATE , &mp_state);if (ret) {fprintf (stderr,"%s: failed to sync MP_STATE %d/%s\n" , __func__, ret,strerror (-ret));return -1 ; } + env->mp_state = mp_state.mp_state; }
All the same comments as above for kvm_riscv_sync_mpstate_to_kvm.
Quoted 12 lines
target/riscv/kvm/kvm-cpu.c +3
-2return 0 ;@@ -1290,16 +1333,24 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp) }if (KVM_PUT_RESET_STATE == level) { -RISCVCPU *cpu =RISCV_CPU (cs); +CPURISCVState *env = &RISCV_CPU (cs)->env;if (cs->cpu_index ==0 ) { - ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); + env->mp_state =KVM_MP_STATE_RUNNABLE ; + ret =kvm_riscv_sync_mpstate_to_kvm (cs);
target/riscv/kvm/kvm-cpu.c int kvm_arch_put_registers(CPUState *cs, int level, Error **errp) · line 1348
Quoted 10 lines
target/riscv/kvm/kvm-cpu.c +3
-1}else { - ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_STOPPED ); + env->mp_state =KVM_MP_STATE_STOPPED ; + ret =kvm_riscv_sync_mpstate_to_kvm (cs); }if (ret) {return ret; } } +else {
The else should be up after the '}'
Quoted 27 lines
target/riscv/kvm/kvm-cpu.c +5
-0+ ret =kvm_riscv_sync_mpstate_to_kvm (cs); +if (ret) { +return ret; + } + }return ret; }target/riscv/kvm/kvm_riscv.h +2
-1@@ -28,7 +28,8 @@ void kvm_riscv_aia_create(MachineState *machine, uint64_t group_shift, uint64_t aplic_base, uint64_t imsic_base, uint64_t guest_num);void riscv_kvm_aplic_request (void *opaque,int irq,int level); -int kvm_riscv_sync_mpstate_to_kvm (RISCVCPU *cpu,int state); +int kvm_riscv_sync_mpstate_to_kvm (CPUState *cs); +int kvm_riscv_sync_mpstate_to_qemu (CPUState *cs);void riscv_kvm_cpu_finalize_features (RISCVCPU *cpu,Error **errp);uint64_t kvm_riscv_get_timebase_frequency (RISCVCPU *cpu);target/riscv/machine.c +0
-0
target/riscv/machine.c const VMStateDescription vmstate_riscv_cpu = { · line 428
Quoted 7 lines
target/riscv/machine.c +1
-0@@ -422,6 +422,7 @@ const VMStateDescription vmstate_riscv_cpu = {VMSTATE_UNUSED (4 ),VMSTATE_UINT32 (env.misa_ext_mask,RISCVCPU ),VMSTATE_UINTTL (env.priv,RISCVCPU ), +VMSTATE_UINT32 (env.mp_state,RISCVCPU ),VMSTATE_BOOL (env.virt_enabled,RISCVCPU ),VMSTATE_UINT64 (env.resetvec,RISCVCPU ), VMSTATE_UINTTL(env.mhartid, RISCVCPU),
This requires a vmstate version bump.
None of the stuff below this point should be in the patch.
Thanks,
drew
Quoted 237 lines
--
Quoted 229 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-07-11 17:28:10 (星期五)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected], [email protected], [email protected] 主题: [PATCH for v10.0.0] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis is v3 of this patch to fix patch format
This patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive.Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.Signed-off-by Xie Bo <[email protected]>--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +23
-1@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/*Restore the guest's privileged level after migration*/ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); ret =kvm_get_one_reg (cs, id, ®);@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/*Save guest privilege level before migration*/ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/*Ensure all non-core 0 CPUs are runnable after migration*/ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )){ +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); + } +if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) { --2.34.1
Quoted 151 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-05-26 15:45:52 (星期一)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected], [email protected], [email protected] 主题: Re: [PATCH V2] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis is v2 of this patch with no functional changes; adding CC.
--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +8
-0@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);ret = kvm_get_one_reg(cs, id, ®);target/riscv/kvm/kvm-cpu.c +17
-0@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/* Save guest privilege level before migration */ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/* Ensure all non-core 0 CPUs are runnable after migration */ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )) { +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); +if (ret) { +return ret; + } + }if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) {Signature · 1 line
2.34.1Quoted 78 lines
-----原始邮件-----
发件人: 谢波 <[email protected]>
发送时间:2025-05-19 17:41:36 (星期一)
收件人: [email protected]
抄送: [email protected], [email protected], [email protected]
主题: [PATCH] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migrationThis patch fixes two critical issues in QEMU with KVM:
1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user mode, the guest may fail to function correctly after migration due to incorrect privilege state restoration.
2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0) remains functional, while all other cores become unresponsive. This patch ensures all cores are properly set to runnable state after migration.
Changes include:
- Properly restoring guest privileged state during register synchronization.
- Correctly updating multi-core state after migration to ensure all cores are active.Signed-off-by Xie Bo <[email protected]>--- target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)target/riscv/kvm/kvm-cpu.c +8
-0@@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; +/* Restore guest privilege level after migration */ + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +if (reg !=3 ) { + env->priv = reg; + }for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i);ret = kvm_get_one_reg(cs, id, ®);target/riscv/kvm/kvm-cpu.c +17
-0@@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } +/* Save guest privilege level before migration */ + reg = env->priv; +if (reg !=3 ) { + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } +/* Ensure all non-core 0 CPUs are runnable after migration */ +if ((level ==KVM_PUT_FULL_STATE ) && (cs->cpu_index !=0 )) { +RISCVCPU *cpu =RISCV_CPU (cs); + ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); +if (ret) { +return ret; + } + }if (KVM_PUT_RESET_STATE == level) {RISCVCPU *cpu =RISCV_CPU (cs);if (cs->cpu_index ==0 ) {Signature · 4 lines
2.34.1 ______________________www.ultrarisc.com 重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.Quoted 2 lines
______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.______________________www.ultrarisc.com
重要提示:本邮件包括附件的内容是受法律保护的保密信息,如果您不是指定收件人,请立即将本邮件删除,法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定,如无我方的正式书面澄清或授权,不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT NOTICE: This email, including its attachment if any, is confidential. If you are not the intended recipient, please delete it from your computer immediately. Any disclosure, copying, or distribution of this message, or taking of any action based on it is strictly prohibited. Any opinions and suggestions contained in this email are subject to the terms and conditions expressed and defined by us and should not be relied upon unconditionally under any circumstances unless they are confirmed in official written clarification or authorization from us. Thank you for your understanding and cooperation.All rights reserved.
v5 cover target/riscv:Fix riscv64 kvm migration
This series(v5) replaces the earlier single patch "[PATCH v4 for v10.0.0] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after QEMU+KVM migration".
Changes since v4:
- Split the patch into 2 patches
- Fix formatting issues
- Vmstate version bump
Patch 1 sets the initial value of the vCPU to S-mode in KVM mode and add mp_state.
Patch 2 adds saving and restoring of the vCPU's privilege mode and running states, fixing the migration functionality in KVM mode. The functionality of Patch 2 depends on Patch 1 to take effect.
Based on these two patches, we have successfully performed live migration of virtual machines between two physical machines.
Xie Bo (2):
target/riscv:Set KVM initial privilege mode and mp_state target/riscv/kvm:Fixed the issue of resume after QEMU+KVM migration
target/riscv/cpu.c | 11 ++++++- target/riscv/cpu.h | 2 ++ target/riscv/kvm/kvm-cpu.c | 59 ++++++++++++++++++++++++++++-------- target/riscv/kvm/kvm_riscv.h | 3 +- target/riscv/machine.c | 5 +-- 5 files changed, 63 insertions(+), 17 deletions(-)
Signature · 1 line
2.43.0
PATCH v5 1/2 target/riscv:Set-KVM-initial-privilege-mode and mp_state
For KVM mode, the privilege mode should not include M-mode, and the initial value should be set to S-mode. Additionally,patch 2 adds the implementation of putting the vCPU privilege mode to KVM. When the vCPU runs for the first time, QEMU will first put the privilege state to KVM.If the initial value is set to M-mode, KVM will encounter an error.
In addition, this patch introduces the 'mp_state' field to RISC-V vCPUs,following the convention used by KVM on x86. The 'mp_state' reflects the multiprocessor state of a vCPU, and is used to control whether the vCPU is runnable by KVM.
- The bootstrap processor (cpu_index == 0) is initialized with KVM_MP_STATE_RUNNABLE.
- All other vCPUs are initialized with KVM_MP_STATE_STOPPED.
--- target/riscv/cpu.c | 11 ++++++++++- target/riscv/cpu.h | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-)
target/riscv/cpu.c +10 -1
@@ -1042,7 +1042,16 @@ static void riscv_cpu_reset_hold(Object *obj, ResetType type) }#ifndef CONFIG_USER_ONLY env->misa_mxl = mcc->misa_mxl_max; - env->priv =PRV_M ; +if (kvm_enabled ()) { + env->priv =PRV_S ; + }else { + env->priv =PRV_M ; + } +if (cs->cpu_index ==0 ) { + env->mp_state =KVM_MP_STATE_RUNNABLE ; + }else { + env->mp_state =KVM_MP_STATE_STOPPED ; + } env->mstatus &= ~(MSTATUS_MIE |MSTATUS_MPRV );if (env->misa_mxl >MXL_RV32 ) {/*
target/riscv/cpu.h +2 -0
@@ -256,6 +256,8 @@ struct CPUArchState {#endif target_ulong priv; +/* Current multiprocessor state of this vCPU. */ +uint32_t mp_state;/* CSRs for execution environment configuration */ uint64_t menvcfg;target_ulong senvcfg;
Signature · 1 line
2.43.0
Re: [PATCH v5 for v10.0.0 1/2] target/riscv:Set-KVM-initial-privilege-mode and mp_state
Remove the dashes in the summary (Set-KVM-initial-privilege-mode).
Quoted 1 line · Xie Bo
For KVM mode, the privilege mode should not include M-mode, and the initial value should be set to S-mode. Additionally,patch 2 adds the implementation of putting the vCPU privilege mode to KVM. When the vCPU runs for the first time, QEMU will first put the privilege state to KVM.If the initial value is set to M-mode, KVM will encounter an error.
Don't reference "patch 2" in a commit message. You may say something like "a following patch".
Quoted 4 lines
In addition, this patch introduces the '
mp_state' field to RISC-V vCPUs,following the convention used by KVM on x86. The 'mp_state' reflects the multiprocessor state of a vCPU, and is used to control whether the vCPU is runnable by KVM.
- The bootstrap processor (cpu_index== 0) is initialized withKVM_MP_STATE_RUNNABLE.
- All other vCPUs are initialized withKVM_MP_STATE_STOPPED.
This is just stating what the code does. What we should put here is _why_ we choose cpu_index 0 to be the only runnable cpu. Is there some reason? Or is it just because it's the first cpu and we'd otherwise need to randomly select a boot cpu.
Use newlines to format commit messages, breaking around 70 chars. The text is also missing space after commas and periods.
Quoted 45 lines
Signed-off-by Xie Bo <[email protected]>--- target/riscv/cpu.c | 11 ++++++++++- target/riscv/cpu.h | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-)target/riscv/cpu.c +10
-1@@ -1042,7 +1042,16 @@ static void riscv_cpu_reset_hold(Object *obj, ResetType type) }#ifndef CONFIG_USER_ONLY env->misa_mxl = mcc->misa_mxl_max; - env->priv =PRV_M ; +if (kvm_enabled ()) { + env->priv =PRV_S ; + }else { + env->priv =PRV_M ; + } +if (cs->cpu_index ==0 ) { + env->mp_state =KVM_MP_STATE_RUNNABLE ; + }else { + env->mp_state =KVM_MP_STATE_STOPPED ; + } env->mstatus &= ~(MSTATUS_MIE |MSTATUS_MPRV );if (env->misa_mxl >MXL_RV32 ) {/* target/riscv/cpu.h +2
-0@@ -256,6 +256,8 @@ struct CPUArchState {#endif target_ulong priv; +/* Current multiprocessor state of this vCPU. */ +uint32_t mp_state;/* CSRs for execution environment configuration */ uint64_t menvcfg;target_ulong senvcfg;Signature · 1 line
2.43.0
Thanks,
drew
PATCH v5 2/2 target/riscv/kvm:Fixed the issue of resume after QEMU+KVM migration
In reply to Xie BoThis patch fixes two migration issues for virtual machines in KVM mode:
1.It saves and restores the vCPU's privilege mode to ensure that the vCPU's privilege mode is correct after migration.
2.It saves and restores the vCPU's mp_state (runnable or stopped) and includes this state in the migration sequence, upgrading the vmstate version to ensure that the vCPU's mp_state is correct after migration.
--- target/riscv/kvm/kvm-cpu.c | 59 ++++++++++++++++++++++++++++-------- target/riscv/kvm/kvm_riscv.h | 3 +- target/riscv/machine.c | 5 +-- 3 files changed, 51 insertions(+), 16 deletions(-)
target/riscv/kvm/kvm-cpu.c +46 -13
@@ -576,6 +576,12 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + env->priv = reg; +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); ret =kvm_get_one_reg (cs, id, ®);
@@ -601,6 +607,12 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } + reg = env->priv; + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];
@@ -1244,25 +1256,51 @@ int kvm_arch_get_registers(CPUState *cs, Error **errp)return ret; } + ret =kvm_riscv_sync_mpstate_to_qemu (cs); +if (ret) { +return ret; + } +return ret; } -int kvm_riscv_sync_mpstate_to_kvm (RISCVCPU *cpu,int state) +int kvm_riscv_sync_mpstate_to_kvm (CPUState *cs) { +int ret =0 ; +CPURISCVState *env = &RISCV_CPU (cs)->env; +if (cap_has_mp_state) {struct kvm_mp_state mp_state = { - .mp_state = state + .mp_state = env->mp_state }; -int ret =kvm_vcpu_ioctl (CPU (cpu),KVM_SET_MP_STATE , &mp_state); + ret =kvm_vcpu_ioctl (cs,KVM_SET_MP_STATE , &mp_state);if (ret) { -fprintf (stderr,"%s: failed to sync MP_STATE %d/%s\n" , +fprintf (stderr,"%s: failed to sync MP_STATE to KVM %d/%s\n" , __func__, ret,strerror (-ret)); -return -1 ; } } -return 0 ; +return ret; +} + +int kvm_riscv_sync_mpstate_to_qemu (CPUState *cs) +{ +int ret =0 ; +CPURISCVState *env = &RISCV_CPU (cs)->env; + +if (cap_has_mp_state) { +struct kvm_mp_state mp_state; + + ret =kvm_vcpu_ioctl (cs,KVM_GET_MP_STATE , &mp_state); +if (ret) { +fprintf (stderr,"%s: failed to sync MP_STATE to QEMU %d/%s\n" , + __func__, ret,strerror (-ret)); + } + env->mp_state = mp_state.mp_state; + } + +return ret; }int kvm_arch_put_registers (CPUState *cs,int level,Error **errp)
@@ -1289,13 +1327,8 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret; } -if (KVM_PUT_RESET_STATE == level) { -RISCVCPU *cpu =RISCV_CPU (cs); -if (cs->cpu_index ==0 ) { - ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); - }else { - ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_STOPPED ); - } +if (level >=KVM_PUT_RESET_STATE ) { + ret =kvm_riscv_sync_mpstate_to_kvm (cs);if (ret) {return ret; }
target/riscv/kvm/kvm_riscv.h +2 -1
@@ -28,7 +28,8 @@ void kvm_riscv_aia_create(MachineState *machine, uint64_t group_shift, uint64_t aplic_base, uint64_t imsic_base, uint64_t guest_num);void riscv_kvm_aplic_request (void *opaque,int irq,int level); -int kvm_riscv_sync_mpstate_to_kvm (RISCVCPU *cpu,int state); +int kvm_riscv_sync_mpstate_to_kvm (CPUState *cs); +int kvm_riscv_sync_mpstate_to_qemu (CPUState *cs);void riscv_kvm_cpu_finalize_features (RISCVCPU *cpu,Error **errp);uint64_t kvm_riscv_get_timebase_frequency (RISCVCPU *cpu);
target/riscv/machine.c +3 -2
@@ -401,8 +401,8 @@ static const VMStateDescription vmstate_ssp = {const VMStateDescription vmstate_riscv_cpu = { .name ="cpu" , - .version_id =10 , - .minimum_version_id =10 , + .version_id =11 , + .minimum_version_id =11 , .post_load = riscv_cpu_post_load, .fields = (const VMStateField []) {VMSTATE_UINTTL_ARRAY (env.gpr,RISCVCPU ,32 ),
@@ -422,6 +422,7 @@ const VMStateDescription vmstate_riscv_cpu = {VMSTATE_UNUSED (4 ),VMSTATE_UINT32 (env.misa_ext_mask,RISCVCPU ),VMSTATE_UINTTL (env.priv,RISCVCPU ), +VMSTATE_UINT32 (env.mp_state,RISCVCPU ),VMSTATE_BOOL (env.virt_enabled,RISCVCPU ),VMSTATE_UINT64 (env.resetvec,RISCVCPU ), VMSTATE_UINTTL(env.mhartid, RISCVCPU),
Signature · 1 line
2.43.0
Re: [PATCH v5 for v10.0.0 2/2] target/riscv/kvm:Fixed the issue of resume after QEMU+KVM migration
sed s/Fixed the issue of/Fix/ <<<$SUBJECT
Quoted 1 line · Xie Bo
This patch fixes two migration issues for virtual machines in KVM mode:
Don't start commit messages with "This patch".
Quoted 4 lines
1.It saves and restores the vCPU's privilege mode to ensure that the vCPU's privilege mode is correct after migration.
2.It saves and restores the vCPU'smp_state(runnable or stopped) and includes this state in the migration sequence, upgrading the vmstate version to ensure that the vCPU'smp_stateis correct after migration.
Missing spaces after periods. Break lines around 70 chars.
Quoted 80 lines
Signed-off-by Xie Bo <[email protected]>--- target/riscv/kvm/kvm-cpu.c | 59 ++++++++++++++++++++++++++++-------- target/riscv/kvm/kvm_riscv.h | 3 +- target/riscv/machine.c | 5 +-- 3 files changed, 51 insertions(+), 16 deletions(-)target/riscv/kvm/kvm-cpu.c +31
-6@@ -576,6 +576,12 @@ static int kvm_riscv_get_regs_core(CPUState *cs) } env->pc = reg; + ret =kvm_get_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } + env->priv = reg; +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); ret =kvm_get_one_reg (cs, id, ®);@@ -601,6 +607,12 @@ static int kvm_riscv_put_regs_core(CPUState *cs)return ret; } + reg = env->priv; + ret =kvm_set_one_reg (cs,RISCV_CORE_REG (env, mode), ®); +if (ret) { +return ret; + } +for (i =1 ; i <32 ; i++) {uint64_t id =kvm_riscv_reg_id_ulong (env,KVM_REG_RISCV_CORE , i); reg = env->gpr[i];@@ -1244,25 +1256,51 @@ int kvm_arch_get_registers(CPUState *cs, Error **errp)return ret; } + ret =kvm_riscv_sync_mpstate_to_qemu (cs); +if (ret) { +return ret; + } +return ret; } -int kvm_riscv_sync_mpstate_to_kvm (RISCVCPU *cpu,int state) +int kvm_riscv_sync_mpstate_to_kvm (CPUState *cs) { +int ret =0 ; +CPURISCVState *env = &RISCV_CPU (cs)->env; +if (cap_has_mp_state) {struct kvm_mp_state mp_state = { - .mp_state = state + .mp_state = env->mp_state }; -int ret =kvm_vcpu_ioctl (CPU (cpu),KVM_SET_MP_STATE , &mp_state); + ret =kvm_vcpu_ioctl (cs,KVM_SET_MP_STATE , &mp_state);if (ret) { -fprintf (stderr,"%s: failed to sync MP_STATE %d/%s\n" , +fprintf (stderr,"%s: failed to sync MP_STATE to KVM %d/%s\n" , __func__, ret,strerror (-ret)); -return -1 ; } } -return 0 ; +return ret; +} + +int kvm_riscv_sync_mpstate_to_qemu (CPUState *cs) +{ +int ret =0 ; +CPURISCVState *env = &RISCV_CPU (cs)->env;
target/riscv/kvm/kvm-cpu.c int kvm_arch_get_registers(CPUState *cs, Error **errp) · line 1300
Quoted 10 lines
target/riscv/kvm/kvm-cpu.c +10
-0+ +if (cap_has_mp_state) { +struct kvm_mp_state mp_state; + + ret =kvm_vcpu_ioctl (cs,KVM_GET_MP_STATE , &mp_state); +if (ret) { +fprintf (stderr,"%s: failed to sync MP_STATE to QEMU %d/%s\n" , + __func__, ret,strerror (-ret)); + } + env->mp_state = mp_state.mp_state;
We shouldn't still do this assignment when the ioctl fails.
Quoted 8 lines
target/riscv/kvm/kvm-cpu.c +3
-0+ } + +return ret; }int kvm_arch_put_registers (CPUState *cs,int level,Error **errp)@@ -1289,13 +1327,8 @@ int kvm_arch_put_registers(CPUState *cs, int level, Error **errp)return ret;
target/riscv/kvm/kvm-cpu.c int kvm_arch_put_registers(CPUState *cs, int level, Error **errp) · line 1330
Quoted 10 lines
target/riscv/kvm/kvm-cpu.c +1
-7} -if (KVM_PUT_RESET_STATE == level) { -RISCVCPU *cpu =RISCV_CPU (cs); -if (cs->cpu_index ==0 ) { - ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_RUNNABLE ); - }else { - ret =kvm_riscv_sync_mpstate_to_kvm (cpu,KVM_MP_STATE_STOPPED ); - } +if (level >=KVM_PUT_RESET_STATE ) {
Changing this to >= makes sense, but it probably deserves a justification in the commit message.
Quoted 44 lines
target/riscv/kvm/kvm-cpu.c +1
-0+ ret =kvm_riscv_sync_mpstate_to_kvm (cs);if (ret) {return ret; }target/riscv/kvm/kvm_riscv.h +2
-1@@ -28,7 +28,8 @@ void kvm_riscv_aia_create(MachineState *machine, uint64_t group_shift, uint64_t aplic_base, uint64_t imsic_base, uint64_t guest_num);void riscv_kvm_aplic_request (void *opaque,int irq,int level); -int kvm_riscv_sync_mpstate_to_kvm (RISCVCPU *cpu,int state); +int kvm_riscv_sync_mpstate_to_kvm (CPUState *cs); +int kvm_riscv_sync_mpstate_to_qemu (CPUState *cs);void riscv_kvm_cpu_finalize_features (RISCVCPU *cpu,Error **errp);uint64_t kvm_riscv_get_timebase_frequency (RISCVCPU *cpu);target/riscv/machine.c +3
-2@@ -401,8 +401,8 @@ static const VMStateDescription vmstate_ssp = {const VMStateDescription vmstate_riscv_cpu = { .name ="cpu" , - .version_id =10 , - .minimum_version_id =10 , + .version_id =11 , + .minimum_version_id =11 , .post_load = riscv_cpu_post_load, .fields = (const VMStateField []) {VMSTATE_UINTTL_ARRAY (env.gpr,RISCVCPU ,32 ),@@ -422,6 +422,7 @@ const VMStateDescription vmstate_riscv_cpu = {VMSTATE_UNUSED (4 ),VMSTATE_UINT32 (env.misa_ext_mask,RISCVCPU ),VMSTATE_UINTTL (env.priv,RISCVCPU ), +VMSTATE_UINT32 (env.mp_state,RISCVCPU ),VMSTATE_BOOL (env.virt_enabled,RISCVCPU ),VMSTATE_UINT64 (env.resetvec,RISCVCPU ), VMSTATE_UINTTL(env.mhartid, RISCVCPU),Signature · 1 line
2.43.0
Thanks,
drew