x86: add a new SMP bring up way for tboot case
Subsystems (2)
PATCH v2 1/1 x86: add a new SMP bring up way for tboot case
tboot provides a better AP wakeup mechanism based on cpu MWAIT feature for OS/VMM. With this mechanism, system will boot faster and will NOT require VT to be enabled. But it requires that OS/VMM must have support it, otherwise system can never boot up.
Once this mechanism is enabled, tboot will put APs waiting in MWAIT loops before launching kernel. kernel can check the new flag field in v6 tboot shared page for the hint. If the bit TB_FLAG_AP_WAKE_SUPPORT in flag field is set, kernel BSP has to write the monitored memory (tboot->ap_wake_trigger) to bring APs out of MWAIT loops. The sipi vector should be written in tboot->ap_wake_addr before waking up APs.
--- arch/x86/kernel/smpboot.c | 21 +++++++++++---------- arch/x86/kernel/tboot.c | 17 +++++++++++++++++ include/linux/tboot.h | 17 +++++++++++++++++ 3 files changed, 45 insertions(+), 10 deletions(-)
arch/x86/kernel/smpboot.c +11 -10
@@ -662,8 +662,7 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle)/* start_ip had better be page-aligned! */ unsigned long start_ip = real_mode_header->trampoline_start; -unsigned long boot_error =0 ; -int timeout; +int timeout, boot_error =0 ;/* Just in case we booted with a single CPU. */ alternatives_enable_smp ();
@@ -710,14 +709,16 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle) } } -/* -* Kick the secondary CPU. Use the method in the APIC driver -* if it's defined - or use an INIT boot APIC message otherwise: -*/ -if (apic->wakeup_secondary_cpu) - boot_error = apic->wakeup_secondary_cpu (apicid, start_ip); -else - boot_error =wakeup_secondary_cpu_via_init (apicid, start_ip); +if (!tboot_wake_up (apicid, start_ip)) { +/* +* Kick the secondary CPU. Use the method in the APIC driver +* if it's defined - or use an INIT boot APIC message otherwise: +*/ +if (apic->wakeup_secondary_cpu) + boot_error = apic->wakeup_secondary_cpu (apicid, start_ip); +else + boot_error =wakeup_secondary_cpu_via_init (apicid, start_ip); + }if (!boot_error) {/*
arch/x86/kernel/tboot.c +17 -0
@@ -101,6 +101,8 @@ void __init tboot_probe(void)pr_debug ("shutdown_entry: 0x%x\n" , tboot->shutdown_entry);pr_debug ("tboot_base: 0x%08x\n" , tboot->tboot_base);pr_debug ("tboot_size: 0x%x\n" , tboot->tboot_size); +if (tboot->version >=6 ) +pr_info ("flags: 0x%08x\n" , tboot->flags); }static pgd_t *tboot_pg_dir;
@@ -453,3 +455,18 @@ int tboot_force_iommu(void)return 1 ; } + +int tboot_wake_up (int apicid,unsigned long sipi_vec) +{ +if (!tboot_enabled ()) +return 0 ; + +if ((tboot->version <6 ) || + !(tboot->flags &TB_FLAG_AP_WAKE_SUPPORT )) +return 0 ; + + tboot->ap_wake_addr = sipi_vec; + tboot->ap_wake_trigger = apicid; + +return 1 ; +}
include/linux/tboot.h +17 -0
@@ -124,8 +124,23 @@ struct tboot {/* number of processors in wait-for-SIPI */ u32 num_in_wfs; + +/* +* version 6+ fields: +*/ + +u32 flags; + +/* phys addr of kernel/VMM SIPI vector */ +u64 ap_wake_addr; + +/* kernel/VMM writes APIC ID to wake AP */ +u32 ap_wake_trigger; } __packed; +/* kernel/VMM use INIT-SIPI-SIPI if clear, ap_wake_* if set */ +#define TB_FLAG_AP_WAKE_SUPPORT 0X00000001 +/* * UUID for tboot data struct to facilitate matching * defined as {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} by tboot, which is
@@ -146,6 +161,7 @@ extern void tboot_shutdown(u32 shutdown_type);extern struct acpi_table_header *tboot_get_dmar_table (struct acpi_table_header *dmar_tbl);extern int tboot_force_iommu (void ); +extern int tboot_wake_up (int apicid,unsigned long sipi_vec);#else
@@ -156,6 +172,7 @@ extern int tboot_force_iommu(void); do { } while (0)#define tboot_get_dmar_table (dmar_tbl) (dmar_tbl)#define tboot_force_iommu () 0 +#define tboot_wake_up (apicid, sipi_vec) 0#endif /* !CONFIG_INTEL_TXT */
Signature · 1 line
1.7.9.5
Any comments on this patch?
Jimmy
Ren, Qiaowei wrote on 2012-10-11:
arch/x86/kernel/smpboot.c static int __cpuinit do_boot_cpu(int apicid, int
Quoted 35 lines
tboot provides a better AP wakeup mechanism based on cpu MWAIT feature for OS/VMM. With this mechanism, system will boot faster and will NOT require VT to be enabled. But it requires that OS/VMM must have support it, otherwise system can never boot up.
Once this mechanism is enabled, tboot will put APs waiting in MWAIT loops before launching kernel. kernel can check the new flag field in v6 tboot shared page for the hint. If the bit
TB_FLAG_AP_WAKE_SUPPORTin flag field is set, kernel BSP has to write the monitored memory (tboot->ap_wake_trigger) to bring APs out of MWAIT loops. The sipi vector should be written intboot->ap_wake_addrbefore waking up APs.Signed-off-by Qiaowei Ren <[email protected]>Signed-off-by Xiaoyan Zhang <[email protected]>Signed-off-by Gang Wei <[email protected]>--- arch/x86/kernel/smpboot.c | 21 +++++++++++---------- arch/x86/kernel/tboot.c | 17 +++++++++++++++++ include/linux/tboot.h | 17 +++++++++++++++++ 3 files changed, 45 insertions(+), 10 deletions(-)arch/x86/kernel/smpboot.c +1
-2@@ -662,8 +662,7 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle)/* start_ip had better be page-aligned! */ unsigned long start_ip = real_mode_header->trampoline_start; -unsigned long boot_error =0 ; -int timeout; +int timeout, boot_error =0 ;/* Just in case we booted with a single CPU. */ alternatives_enable_smp ();@@ -710,14 +709,16 @@ static int __cpuinit do_boot_cpu(int apicid, int
cpu,
arch/x86/kernel/smpboot.c static int __cpuinit do_boot_cpu(int apicid, int · line 713
Quoted 4 lines
arch/x86/kernel/smpboot.c +0
-1struct task_struct *idle) } } -/* - * Kick the secondary CPU. Use the method in the APIC driver
- *
Quoted 1 line
if it's defined - or use an INIT boot APIC message otherwise: - */
- if
Quoted 2 lines
(
apic->wakeup_secondary_cpu) -boot_error=apic->wakeup_secondary_cpu(apicid, start_ip); - else -
boot_error =
Quoted 2 lines
wakeup_secondary_cpu_via_init(apicid, start_ip); + if
(!tboot_wake_up(apicid,start_ip)) { + /* + *
Kick the secondary CPU.
Quoted 1 line
Use the method in the APIC driver + * if it's defined - or use
an INIT
Quoted 1 line
boot APIC message otherwise: + */ + if
(apic->wakeup_secondary_cpu)
arch/x86/kernel/smpboot.c
Quoted 1 line
arch/x86/kernel/smpboot.c +1
-0+ boot_error = apic->wakeup_secondary_cpu(apicid,
start_ip); + else
arch/x86/kernel/smpboot.c
Quoted 1 line
arch/x86/kernel/smpboot.c +1
-0+ boot_error = wakeup_secondary_cpu_via_init(apicid,
start_ip); + }
Quoted 76 lines
arch/x86/kernel/smpboot.c +0
-0if (!boot_error) {/* arch/x86/kernel/tboot.c +17
-0@@ -101,6 +101,8 @@ void __init tboot_probe(void)pr_debug ("shutdown_entry: 0x%x\n" , tboot->shutdown_entry);pr_debug ("tboot_base: 0x%08x\n" , tboot->tboot_base);pr_debug ("tboot_size: 0x%x\n" , tboot->tboot_size); +if (tboot->version >=6 ) +pr_info ("flags: 0x%08x\n" , tboot->flags); }static pgd_t *tboot_pg_dir;@@ -453,3 +455,18 @@ int tboot_force_iommu(void)return 1 ; } + +int tboot_wake_up (int apicid,unsigned long sipi_vec) +{ +if (!tboot_enabled ()) +return 0 ; + +if ((tboot->version <6 ) || + !(tboot->flags &TB_FLAG_AP_WAKE_SUPPORT )) +return 0 ; + + tboot->ap_wake_addr = sipi_vec; + tboot->ap_wake_trigger = apicid; + +return 1 ; +}include/linux/tboot.h +17
-0@@ -124,8 +124,23 @@ struct tboot {/* number of processors in wait-for-SIPI */ u32 num_in_wfs; + +/* +* version 6+ fields: +*/ + +u32 flags; + +/* phys addr of kernel/VMM SIPI vector */ +u64 ap_wake_addr; + +/* kernel/VMM writes APIC ID to wake AP */ +u32 ap_wake_trigger; } __packed; +/* kernel/VMM use INIT-SIPI-SIPI if clear, ap_wake_* if set */ +#define TB_FLAG_AP_WAKE_SUPPORT 0X00000001 +/* * UUID for tboot data struct to facilitate matching * defined as {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} by tboot, which is @@ -146,6 +161,7 @@ extern void tboot_shutdown(u32 shutdown_type);extern struct acpi_table_header *tboot_get_dmar_table (struct acpi_table_header *dmar_tbl);extern int tboot_force_iommu (void ); +extern int tboot_wake_up (int apicid,unsigned long sipi_vec);#else @@ -156,6 +172,7 @@ extern int tboot_force_iommu(void); do { } while (0)#define tboot_get_dmar_table (dmar_tbl) (dmar_tbl)#define tboot_force_iommu () 0 +#define tboot_wake_up (apicid, sipi_vec) 0#endif /* !CONFIG_INTEL_TXT */
Any comments on this patch?
Thanks,
Qiaowei
-----Original Message-----
From: Ren, Qiaowei
Sent: Thursday, October 11, 2012 6:11 PM
To: Thomas Gleixner; Ingo Molnar; H. Peter Anvin; [email protected]
Cc: Maliszewski, Richard L; [email protected]; [email protected]; Ren, Qiaowei; Xiaoyan Zhang; Wei, Gang
Subject: [PATCH v2] x86: add a new SMP bring up way for tboot case
tboot provides a better AP wakeup mechanism based on cpu MWAIT feature for OS/VMM. With this mechanism, system will boot faster and will NOT require VT to be enabled. But it requires that OS/VMM must have support it, otherwise system can never boot up.
Once this mechanism is enabled, tboot will put APs waiting in MWAIT loops before launching kernel. kernel can check the new flag field in v6 tboot shared page for the hint. If the bit TB_FLAG_AP_WAKE_SUPPORT in flag field is set, kernel BSP has to write the monitored memory (tboot->ap_wake_trigger) to bring APs out of MWAIT loops. The sipi vector should be written in tboot->ap_wake_addr before waking up APs.
--- arch/x86/kernel/smpboot.c | 21 +++++++++++---------- arch/x86/kernel/tboot.c | 17 +++++++++++++++++ include/linux/tboot.h | 17 +++++++++++++++++ 3 files changed, 45 insertions(+), 10 deletions(-)
arch/x86/kernel/smpboot.c +11 -10
@@ -662,8 +662,7 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle)/* start_ip had better be page-aligned! */ unsigned long start_ip = real_mode_header->trampoline_start; -unsigned long boot_error =0 ; -int timeout; +int timeout, boot_error =0 ;/* Just in case we booted with a single CPU. */ alternatives_enable_smp ();
@@ -710,14 +709,16 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle) } } -/* -* Kick the secondary CPU. Use the method in the APIC driver -* if it's defined - or use an INIT boot APIC message otherwise: -*/ -if (apic->wakeup_secondary_cpu) - boot_error = apic->wakeup_secondary_cpu (apicid, start_ip); -else - boot_error =wakeup_secondary_cpu_via_init (apicid, start_ip); +if (!tboot_wake_up (apicid, start_ip)) { +/* +* Kick the secondary CPU. Use the method in the APIC driver +* if it's defined - or use an INIT boot APIC message otherwise: +*/ +if (apic->wakeup_secondary_cpu) + boot_error = apic->wakeup_secondary_cpu (apicid, start_ip); +else + boot_error =wakeup_secondary_cpu_via_init (apicid, start_ip); + }if (!boot_error) {/*
arch/x86/kernel/tboot.c +16 -0
@@ -101,6 +101,8 @@ void __init tboot_probe(void)pr_debug ("shutdown_entry: 0x%x\n" , tboot->shutdown_entry);pr_debug ("tboot_base: 0x%08x\n" , tboot->tboot_base);pr_debug ("tboot_size: 0x%x\n" , tboot->tboot_size); +if (tboot->version >=6 ) +pr_info ("flags: 0x%08x\n" , tboot->flags); }static pgd_t *tboot_pg_dir;
@@ -453,3 +455,18 @@ int tboot_force_iommu(void)return 1 ; } + +int tboot_wake_up (int apicid,unsigned long sipi_vec) { +if (!tboot_enabled ()) +return 0 ; + +if ((tboot->version <6 ) || + !(tboot->flags &TB_FLAG_AP_WAKE_SUPPORT )) +return 0 ; + + tboot->ap_wake_addr = sipi_vec; + tboot->ap_wake_trigger = apicid; + +return 1 ; +}
include/linux/tboot.h +15 -0
@@ -124,8 +124,23 @@ struct tboot {/* number of processors in wait-for-SIPI */ u32 num_in_wfs; + +/* +* version 6+ fields: +*/ + +u32 flags; + +/* phys addr of kernel/VMM SIPI vector */ +u64 ap_wake_addr; + +/* kernel/VMM writes APIC ID to wake AP */ +u32 ap_wake_trigger; } __packed; +/* kernel/VMM use INIT-SIPI-SIPI if clear, ap_wake_* if set */ #define +TB_FLAG_AP_WAKE_SUPPORT 0X00000001 +/* * UUID for tboot data struct to facilitate matching * defined as {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} by tboot, which is @@ -146,6 +161,7 @@ extern void tboot_shutdown(u32 shutdown_type); extern struct acpi_table_header *tboot_get_dmar_table(
struct acpi_table_header *dmar_tbl); extern int tboot_force_iommu(void);
+extern int tboot_wake_up(int apicid, unsigned long sipi_vec);
#else
@@ -156,6 +172,7 @@ extern int tboot_force_iommu(void);
do { } while (0)
#define tboot_get_dmar_table(dmar_tbl) (dmar_tbl)
#define tboot_force_iommu() 0
+#define tboot_wake_up(apicid, sipi_vec) 0
#endif /* !CONFIG_INTEL_TXT */
--
1.7.9.5
H. Peter Anvin
reviewerQuoted 4 lines · Qiaowei
Any comments on this patch?
Thanks,
Qiaowei
The biggest question is probably if we can use an existing hook of some sort.
Overriding the apic method is probably not the right way to go, though.
tglx, do you have any opinions here?
Furthermore, this really is not nice:
arch/x86/kernel/tboot.c int tboot_force_iommu(void) · line 471
Quoted 13 lines
arch/x86/kernel/tboot.c +13
-0+int tboot_wake_up (int apicid,unsigned long sipi_vec) { +if (!tboot_enabled ()) +return 0 ; + +if ((tboot->version <6 ) || + !(tboot->flags &TB_FLAG_AP_WAKE_SUPPORT )) +return 0 ; + + tboot->ap_wake_addr = sipi_vec; + tboot->ap_wake_trigger = apicid; + +return 1 ; +}
Not only don't you set boot_error for the code above it, but there is absolutely no indication how that does its job (are those active operations? If so they should use writel()), nor does it include any kind of synchronization.
-hpa
Quoted 12 lines
-----Original Message-----
From: H. Peter Anvin [mailto:[email protected]]
Sent: Wednesday, March 20, 2013 11:02 AM
To: Ren, Qiaowei
Cc: Thomas Gleixner; Ingo Molnar; [email protected]; Maliszewski, Richard L; [email protected]; [email protected]; Xiaoyan Zhang; Wei, Gang
Subject: Re: [PATCH v2] x86: add a new SMP bring up way for tboot caseQuoted 2 lines · Qiaowei
Any comments on this patch?
Quoted 8 lines
Quoted 2 lines
Thanks,
QiaoweiThe biggest question is probably if we can use an existing hook of some sort.
Overriding the apic method is probably not the right way to go, though.
tglx, do you have any opinions here?
If we can't use an existing hook, could you please give me any suggestion about other opinions here?
Quoted 11 lines
Furthermore, this really is not nice:
Quoted 9 lines
arch/x86/kernel/tboot.c +9
-0+int tboot_wake_up (int apicid,unsigned long sipi_vec) { +if (!tboot_enabled ()) +return 0 ; + +if ((tboot->version <6 ) || + !(tboot->flags &TB_FLAG_AP_WAKE_SUPPORT )) +return 0 ; + + tboot->ap_wake_addr = sipi_vec;
Quoted 9 lines
arch/x86/kernel/tboot.c int tboot_force_iommu(void) · line 471
Quoted 4 lines
arch/x86/kernel/tboot.c +4
-0+ tboot->ap_wake_trigger = apicid; + +return 1 ; +}Not only don't you set
boot_errorfor the code above it, but there is absolutely no indication how that does its job (are those active operations? If so they should usewritel()), nor does it include any kind of synchronization.
Ok. I should add some comments to explain how that does its job. But I guess boot_error don't have to be set inside this function.
Thanks,
Qiaowei