vm_event: fix race-condition when disabling monitor events
PATCH v2 1/3 xen/monitor: Control register values
In reply to Tamas K LengyelExtend the monitor_op domctl to include option that enables controlling what values certain registers are permitted to hold by a monitor subscriber.
--- xen/arch/x86/hvm/hvm.c | 25 ++++++++++++++++--------- xen/arch/x86/monitor.c | 10 +++++++++- xen/include/asm-x86/domain.h | 1 + xen/include/public/domctl.h | 1 + 4 files changed, 27 insertions(+), 10 deletions(-)
xen/arch/x86/hvm/hvm.c +16 -9
@@ -2263,7 +2263,8 @@ int hvm_set_cr0(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR0 , value, old_value) ) +if (hvm_monitor_crX (CR0 , value, old_value) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr0 =1 ;
@@ -2362,7 +2363,8 @@ int hvm_set_cr3(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR3 , value, old) ) +if (hvm_monitor_crX (CR3 , value, old) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr3 =1 ;
@@ -2443,7 +2445,8 @@ int hvm_set_cr4(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR4 , value, old_cr) ) +if (hvm_monitor_crX (CR4 , value, old_cr) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr4 =1 ;
@@ -3587,13 +3590,17 @@ int hvm_msr_write_intercept(unsigned int msr, uint64_t msr_content,ASSERT (v->arch.vm_event); -/* The actual write will occur in hvm_do_resume() (if permitted). */ - v->arch.vm_event->write_data.do_write.msr =1 ; - v->arch.vm_event->write_data.msr = msr; - v->arch.vm_event->write_data.value = msr_content; -hvm_monitor_msr (msr, msr_content, msr_old_content); -return X86EMUL_OKAY ; + +if ( v->domain->arch.monitor.control_register_values ) + { +/* The actual write will occur in hvm_do_resume(), if permitted. */ + v->arch.vm_event->write_data.do_write.msr =1 ; + v->arch.vm_event->write_data.msr = msr; + v->arch.vm_event->write_data.value = msr_content; + +return X86EMUL_OKAY ; + } }if ( (ret =guest_wrmsr (v, msr, msr_content)) !=X86EMUL_UNHANDLEABLE )
xen/arch/x86/monitor.c +9 -1
@@ -144,7 +144,15 @@ int arch_monitor_domctl_event(struct domain *d, struct xen_domctl_monitor_op *mop) {struct arch_domain *ad = &d->arch; -bool requested_status = (XEN_DOMCTL_MONITOR_OP_ENABLE == mop->op); +bool requested_status; + +if (XEN_DOMCTL_MONITOR_OP_CONTROL_REGISTERS == mop->op ) + { + ad->monitor.control_register_values = true; +return 0 ; + } + + requested_status = (XEN_DOMCTL_MONITOR_OP_ENABLE == mop->op);switch ( mop->event ) {
xen/include/asm-x86/domain.h +1 -0
@@ -416,6 +416,7 @@ struct arch_domain* This is used to filter out pagefaults. */ unsigned int inguest_pagefault_disabled : 1; + unsigned int control_register_values : 1;struct monitor_msr_bitmap *msr_bitmap;uint64_t write_ctrlreg_mask[4 ]; } monitor;
xen/include/public/domctl.h +1 -0
@@ -1025,6 +1025,7 @@ struct xen_domctl_psr_cmt_op {#define XEN_DOMCTL_MONITOR_OP_DISABLE 1#define XEN_DOMCTL_MONITOR_OP_GET_CAPABILITIES 2#define XEN_DOMCTL_MONITOR_OP_EMULATE_EACH_REP 3 +#define XEN_DOMCTL_MONITOR_OP_CONTROL_REGISTERS 4#define XEN_DOMCTL_MONITOR_EVENT_WRITE_CTRLREG 0#define XEN_DOMCTL_MONITOR_EVENT_MOV_TO_MSR 1
Signature · 1 line
2.26.1
PATCH v2 2/3 xen/vm_event: add vm_event_check_pending_op
In reply to Tamas K LengyelPerform sanity checking when shutting vm_event down to determine whether it is safe to do so. Error out with -EAGAIN in case pending operations have been found for the domain.
--- xen/arch/x86/vm_event.c | 23 +++++++++++++++++++++++ xen/common/vm_event.c | 17 ++++++++++++++--- xen/include/asm-arm/vm_event.h | 7 +++++++ xen/include/asm-x86/vm_event.h | 2 ++ 4 files changed, 46 insertions(+), 3 deletions(-)
xen/arch/x86/vm_event.c +23 -0
@@ -297,6 +297,29 @@ void vm_event_emulate_check(struct vcpu *v, vm_event_response_t *rsp) }; } +bool vm_event_check_pending_op (const struct vcpu *v) +{ +struct monitor_write_data *w = &v->arch.vm_event->write_data; + +if ( !v->arch.vm_event->sync_event ) +return false; + +if ( w->do_write.cr0 ) +return true; +if ( w->do_write.cr3 ) +return true; +if ( w->do_write.cr4 ) +return true; +if ( w->do_write.msr ) +return true; +if ( v->arch.vm_event->set_gprs ) +return true; +if ( v->arch.vm_event->emulate_flags ) +return true; + +return false; +} +/* * Local variables: * mode: C
xen/common/vm_event.c +14 -3
@@ -183,6 +183,7 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)if (vm_event_check_ring (ved) ) {struct vcpu *v; +bool pending_op = false;spin_lock (&ved->lock);
@@ -192,9 +193,6 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)return -EBUSY ; } -/* Free domU's event channel and leave the other one unbound */ -free_xen_event_channel (d, ved->xen_port); -/* Unblock all vCPUs */ for_each_vcpu ( d, v ) {
@@ -203,8 +201,21 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)vcpu_unpause (v); ved->blocked--; } + +if (vm_event_check_pending_op (v) ) + pending_op = true; } +/* vm_event ops are still pending until vCPUs get scheduled */ +if ( pending_op ) + { +spin_unlock (&ved->lock); +return -EAGAIN ; + } + +/* Free domU's event channel and leave the other one unbound */ +free_xen_event_channel (d, ved->xen_port); +destroy_ring_for_helper (&ved->ring_page, ved->ring_pg_struct);vm_event_cleanup_domain (d);
xen/include/asm-arm/vm_event.h +7 -0
@@ -58,4 +58,11 @@ void vm_event_sync_event(struct vcpu *v, bool value) /* Not supported on ARM. */ } +static inline +bool vm_event_check_pending_op (const struct vcpu *v) +{ +/* Not supported on ARM. */ +return false; +} +#endif /* __ASM_ARM_VM_EVENT_H__ */
xen/include/asm-x86/vm_event.h +2 -0
@@ -54,4 +54,6 @@ void vm_event_emulate_check(struct vcpu *v, vm_event_response_t *rsp);void vm_event_sync_event (struct vcpu *v,bool value); +bool vm_event_check_pending_op (const struct vcpu *v); +#endif /* __ASM_X86_VM_EVENT_H__ */
Signature · 1 line
2.26.1
PATCH v2 3/3 xen/vm_event: Add safe to disable vm_event
In reply to Tamas K LengyelInstead of having to repeatedly try to disable vm_events, request a specific vm_event to be sent when the domain is safe to continue with shutting down the vm_event interface.
--- xen/arch/x86/hvm/hvm.c | 38 ++++++++++++++++++++++++++----- xen/arch/x86/hvm/monitor.c | 14 ++++++++++++ xen/arch/x86/monitor.c | 13 +++++++++++ xen/include/asm-x86/domain.h | 1 + xen/include/asm-x86/hvm/monitor.h | 1 + xen/include/public/domctl.h | 2 ++ xen/include/public/vm_event.h | 8 +++++++ 7 files changed, 71 insertions(+), 6 deletions(-)
xen/arch/x86/hvm/hvm.c +32 -6
@@ -563,15 +563,41 @@ void hvm_do_resume(struct vcpu *v) v->arch.hvm.inject_event.vector =HVM_EVENT_VECTOR_UNSET ; } -if (unlikely (v->arch.vm_event) && v->arch.monitor.next_interrupt_enabled ) +if (unlikely (v->arch.vm_event) ) { -struct x86_event info; +struct domain *d = v->domain; + +if ( v->arch.monitor.next_interrupt_enabled ) + { +struct x86_event info; + +if (hvm_get_pending_event (v, &info) ) + { +hvm_monitor_interrupt (info.vector, info.type, info.error_code, + info.cr2); + v->arch.monitor.next_interrupt_enabled = false; + } + } -if (hvm_get_pending_event (v, &info) ) +if ( d->arch.monitor.safe_to_disable ) { -hvm_monitor_interrupt (info.vector, info.type, info.error_code, - info.cr2); - v->arch.monitor.next_interrupt_enabled = false; +const struct vcpu *check_vcpu; +bool pending_op = false; + + for_each_vcpu ( d, check_vcpu ) + { +if (vm_event_check_pending_op (check_vcpu) ) + { + pending_op = true; +break ; + } + } + +if ( !pending_op ) + { +hvm_monitor_safe_to_disable (); + d->arch.monitor.safe_to_disable = false; + } } } }
xen/arch/x86/hvm/monitor.c +14 -0
@@ -300,6 +300,20 @@ bool hvm_monitor_check_p2m(unsigned long gla, gfn_t gfn, uint32_t pfec,return monitor_traps (curr, true, &req) >=0 ; } +void hvm_monitor_safe_to_disable (void ) +{ +struct vcpu *curr = current; +struct arch_domain *ad = &curr->domain->arch; +vm_event_request_t req = {}; + +if ( !ad->monitor.safe_to_disable ) +return ; + + req.reason =VM_EVENT_REASON_SAFE_TO_DISABLE ; + +monitor_traps (curr,0 , &req); +} +/* * Local variables: * mode: C
xen/arch/x86/monitor.c +13 -0
@@ -339,6 +339,19 @@ int arch_monitor_domctl_event(struct domain *d,break ; } +case XEN_DOMCTL_MONITOR_EVENT_SAFE_TO_DISABLE : + { +bool old_status = ad->monitor.safe_to_disable; + +if (unlikely (old_status == requested_status) ) +return -EEXIST ; + +domain_pause (d); + ad->monitor.safe_to_disable = requested_status; +domain_unpause (d); +break ; + } +default :/* * Should not be reached unless arch_monitor_get_capabilities() is
xen/include/asm-x86/domain.h +1 -0
@@ -417,6 +417,7 @@ struct arch_domain*/ unsigned int inguest_pagefault_disabled : 1; unsigned int control_register_values : 1; + unsigned int safe_to_disable : 1;struct monitor_msr_bitmap *msr_bitmap;uint64_t write_ctrlreg_mask[4 ]; } monitor;
xen/include/asm-x86/hvm/monitor.h +1 -0
@@ -52,6 +52,7 @@ bool hvm_monitor_emul_unimplemented(void);bool hvm_monitor_check_p2m (unsigned long gla,gfn_t gfn,uint32_t pfec,uint16_t kind); +void hvm_monitor_safe_to_disable (void );#endif /* __ASM_X86_HVM_MONITOR_H__ */
xen/include/public/domctl.h +2 -0
@@ -1040,6 +1040,8 @@ struct xen_domctl_psr_cmt_op {#define XEN_DOMCTL_MONITOR_EVENT_EMUL_UNIMPLEMENTED 10/* Enabled by default */ #define XEN_DOMCTL_MONITOR_EVENT_INGUEST_PAGEFAULT 11 +/* Always async, disables automaticaly on first event */ +#define XEN_DOMCTL_MONITOR_EVENT_SAFE_TO_DISABLE 12struct xen_domctl_monitor_op {uint32_t op;/* XEN_DOMCTL_MONITOR_OP_* */
xen/include/public/vm_event.h +8 -0
@@ -159,6 +159,14 @@#define VM_EVENT_REASON_DESCRIPTOR_ACCESS 13/* Current instruction is not implemented by the emulator */ #define VM_EVENT_REASON_EMUL_UNIMPLEMENTED 14 +/* +* When shutting down vm_event it may not be immediately safe to complete the +* process as some vCPUs may be pending synchronization. This async event +* type can be used to receive a notification when its safe to finish disabling +* the vm_event interface. All other event types need to be disabled before +* registering to this one. +*/ +#define VM_EVENT_REASON_SAFE_TO_DISABLE 15/* Supported values for the vm_event_write_ctrlreg index. */ #define VM_EVENT_X86_CR0 0
Signature · 1 line
2.26.1
v2 cover vm_event: fix race-condition when disabling monitor events
For the last couple years we have received numerous reports from users of monitor vm_events of spurious guest crashes when using events. In particular, it has observed that the problem occurs when vm_events are being disabled. The nature of the guest crash varied widely and has only occured occasionally. This made debugging the issue particularly hard. We had discussions about this issue even here on the xen-devel mailinglist with no luck figuring it out.
The bug has now been identified as a race-condition between register event handling and disabling the vm_event interface.
Patch 96760e2fba100d694300a81baddb5740e0f8c0ee, "vm_event: deny register writes if refused by vm_event reply" is the patch that introduced the error. In this patch emulation of register write events can be postponed until the corresponding vm_event handler decides whether to allow such write to take place. Unfortunately this can only be implemented by performing the deny/allow step when the vCPU gets scheduled. Due to that postponed emulation of the event if the user decides to pause the VM in the vm_event handler and then disable events, the entire emulation step is skipped the next time the vCPU is resumed. Even if the user doesn't pause during the vm_event handling but exits immediately and disables vm_event, the situation becomes racey as disabling vm_event may succeed before the guest's vCPUs get scheduled with the pending emulation task. This has been particularly the case with VMS that have several vCPUs as after the VM is unpaused it may actually take a long time before all vCPUs get scheduled.
The only solution currently is to poll each vCPU before vm_events are disabled to verify they had been scheduled before it is safe to disable vm_events. The following patches resolve this issue in a much nicer way.
Patch 1 adds an option to the monitor_op domctl that needs to be specified if
the user wants to actually use the postponed register-write handling mechanism. If that option is not specified then handling is performed the same way as before patch 96760e2fba100d694300a81baddb5740e0f8c0ee.
Patch 2 performs sanity checking when disabling vm_events to determine whether
its safe to free all vm_event structures. The vCPUs still get unpaused to allow them to get scheduled and perform any of their pending operations, but otherwise an -EAGAIN error is returned signaling to the user that they need to wait and try again disabling the interface.
Patch 3 adds a vm_event specifically to signal to the user when it is safe to
continue disabling the interface.
Shout out to our friends at CERT.pl for stumbling upon a crucial piece of information that lead to finally squashing this nasty bug.
v2: minor adjustments based on Jan's comments
Tamas K Lengyel (3):
xen/monitor: Control register values
xen/vm_event: add vm_event_check_pending_op
xen/vm_event: Add safe to disable vm_event
xen/arch/x86/hvm/hvm.c | 63 +++++++++++++++++++++++-------- xen/arch/x86/hvm/monitor.c | 14 +++++++ xen/arch/x86/monitor.c | 23 ++++++++++- xen/arch/x86/vm_event.c | 23 +++++++++++ xen/common/vm_event.c | 17 +++++++-- xen/include/asm-arm/vm_event.h | 7 ++++ xen/include/asm-x86/domain.h | 2 + xen/include/asm-x86/hvm/monitor.h | 1 + xen/include/asm-x86/vm_event.h | 2 + xen/include/public/domctl.h | 3 ++ xen/include/public/vm_event.h | 8 ++++ 11 files changed, 144 insertions(+), 19 deletions(-)
Signature · 1 line
2.26.1
Quoted 41 lines · Tamas K Lengyel
For the last couple years we have received numerous reports from users of monitor
vm_eventsof spurious guest crashes when using events. In particular, it has observed that the problem occurs whenvm_eventsare being disabled. The nature of the guest crash varied widely and has only occured occasionally. This made debugging the issue particularly hard. We had discussions about this issue even here on the xen-devel mailinglist with no luck figuring it out.The bug has now been identified as a race-condition between register event handling and disabling the
vm_eventinterface.Patch
96760e2fba100d694300a81baddb5740e0f8c0ee, "vm_event: deny register writes if refused byvm_eventreply" is the patch that introduced the error. In this patch emulation of register write events can be postponed until the correspondingvm_eventhandler decides whether to allow such write to take place. Unfortunately this can only be implemented by performing the deny/allow step when the vCPU gets scheduled. Due to that postponed emulation of the event if the user decides to pause the VM in thevm_eventhandler and then disable events, the entire emulation step is skipped the next time the vCPU is resumed. Even if the user doesn't pause during thevm_eventhandling but exits immediately and disablesvm_event, the situation becomes racey as disablingvm_eventmay succeed before the guest's vCPUs get scheduled with the pending emulation task. This has been particularly the case with VMS that have several vCPUs as after the VM is unpaused it may actually take a long time before all vCPUs get scheduled.The only solution currently is to poll each vCPU before
vm_eventsare disabled to verify they had been scheduled before it is safe to disablevm_events. The following patches resolve this issue in a much nicer way.Patch 1 adds an option to the
monitor_opdomctl that needs to be specified if
the user wants to actually use the postponed register-write handling mechanism. If that option is not specified then handling is performed the same way as before patch96760e2fba100d694300a81baddb5740e0f8c0ee.Patch 2 performs sanity checking when disabling
vm_eventsto determine whether
its safe to free allvm_eventstructures. The vCPUs still get unpaused to allow them to get scheduled and perform any of their pending operations, but otherwise an-EAGAINerror is returned signaling to the user that they need to wait and try again disabling the interface.
Quoted 7 lines
Patch 3 adds a
vm_eventspecifically to signal to the user when it is safe to
continue disabling the interface.Shout out to our friends at CERT.pl for stumbling upon a crucial piece of information that lead to finally squashing this nasty bug.
v2: minor adjustments based on Jan's comments
Patch ping.
Tamas
Re: [PATCH v2 for-4.14 1/3] xen/monitor: Control register values
In reply to Tamas K LengyelQuoted 3 lines · Tamas K Lengyel
Extend the
monitor_opdomctl to include option that enables controlling what values certain registers are permitted to hold by a monitor subscriber.
I think the change could benefit for some more detail commit message here. Why is this useful?
There already seems to be some support for gating MSR writes, which seems to be expanded by this commit?
Is it solving some kind of bug reported?
Quoted 33 lines
Signed-off-by Tamas K Lengyel <[email protected]>--- xen/arch/x86/hvm/hvm.c | 25 ++++++++++++++++--------- xen/arch/x86/monitor.c | 10 +++++++++- xen/include/asm-x86/domain.h | 1 + xen/include/public/domctl.h | 1 + 4 files changed, 27 insertions(+), 10 deletions(-)xen/arch/x86/hvm/hvm.c +4
-2@@ -2263,7 +2263,8 @@ int hvm_set_cr0(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR0 , value, old_value) ) +if (hvm_monitor_crX (CR0 , value, old_value) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr0 =1 ;@@ -2362,7 +2363,8 @@ int hvm_set_cr3(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR3 , value, old) ) +if (hvm_monitor_crX (CR3 , value, old) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr3 =1 ;
xen/arch/x86/hvm/hvm.c int hvm_set_cr4(unsigned long value, bool may_defer) · line 2449
Quoted 6 lines
xen/arch/x86/hvm/hvm.c +2
-1@@ -2443,7 +2445,8 @@ int hvm_set_cr4(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR4 , value, old_cr) ) +if (hvm_monitor_crX (CR4 , value, old_cr) && + v->domain->arch.monitor.control_register_values )
I think you could return control_register_values in hvm_monitor_crX instead of having to add the check to each caller?
Quoted 6 lines
xen/arch/x86/hvm/hvm.c +0
-0{/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr4 =1 ;@@ -3587,13 +3590,17 @@ int hvm_msr_write_intercept(unsigned int msr, uint64_t msr_content,ASSERT (v->arch.vm_event);
xen/arch/x86/hvm/hvm.c int hvm_msr_write_intercept(unsigned int msr, uint64_t msr_content, · line 3595
Quoted 10 lines
xen/arch/x86/hvm/hvm.c +2
-6-/* The actual write will occur in hvm_do_resume() (if permitted). */ - v->arch.vm_event->write_data.do_write.msr =1 ; - v->arch.vm_event->write_data.msr = msr; - v->arch.vm_event->write_data.value = msr_content; -hvm_monitor_msr (msr, msr_content, msr_old_content); -return X86EMUL_OKAY ; + +if ( v->domain->arch.monitor.control_register_values )
Is there any value in limiting control_register_values to MSR that represent control registers, like EFER and XSS?
xen/arch/x86/hvm/hvm.c int hvm_msr_write_intercept(unsigned int msr, uint64_t msr_content, · line 3603
Quoted 8 lines
xen/arch/x86/hvm/hvm.c +8
-0+ { +/* The actual write will occur in hvm_do_resume(), if permitted. */ + v->arch.vm_event->write_data.do_write.msr =1 ; + v->arch.vm_event->write_data.msr = msr; + v->arch.vm_event->write_data.value = msr_content; + +return X86EMUL_OKAY ; + }
You seem to change the previous flow of the function here, that would just call hvm_monitor_msr and return previously.
Don't you need to move the return from outside the added if condition in order to keep previous behavior? Or else the write is committed straight away.
Quoted 8 lines
xen/arch/x86/hvm/hvm.c +0
-0}if ( (ret =guest_wrmsr (v, msr, msr_content)) !=X86EMUL_UNHANDLEABLE )xen/arch/x86/monitor.c +0
-0
xen/arch/x86/monitor.c int arch_monitor_domctl_event(struct domain *d, · line 151
Quoted 9 lines
xen/arch/x86/monitor.c +5
-1@@ -144,7 +144,15 @@ int arch_monitor_domctl_event(struct domain *d, struct xen_domctl_monitor_op *mop) {struct arch_domain *ad = &d->arch; -bool requested_status = (XEN_DOMCTL_MONITOR_OP_ENABLE == mop->op); +bool requested_status; + +if (XEN_DOMCTL_MONITOR_OP_CONTROL_REGISTERS == mop->op ) + { + ad->monitor.control_register_values = true;
I think strictly speaking you need to use 1 here, since this variable is not a boolean.
Thanks, Roger.
Re: [PATCH v2 for-4.14 2/3] xen/vm_event: add vm_event_check_pending_op
In reply to Tamas K LengyelQuoted 17 lines · Tamas K Lengyel
Perform sanity checking when shutting
vm_eventdown to determine whether it is safe to do so. Error out with-EAGAINin case pending operations have been found for the domain.Signed-off-by Tamas K Lengyel <[email protected]>--- xen/arch/x86/vm_event.c | 23 +++++++++++++++++++++++ xen/common/vm_event.c | 17 ++++++++++++++--- xen/include/asm-arm/vm_event.h | 7 +++++++ xen/include/asm-x86/vm_event.h | 2 ++ 4 files changed, 46 insertions(+), 3 deletions(-)xen/arch/x86/vm_event.c +0
-0
xen/arch/x86/vm_event.c void vm_event_emulate_check(struct vcpu *v, vm_event_response_t *rsp) · line 302
Quoted 6 lines
xen/arch/x86/vm_event.c +3
-0@@ -297,6 +297,29 @@ void vm_event_emulate_check(struct vcpu *v, vm_event_response_t *rsp) }; } +bool vm_event_check_pending_op (const struct vcpu *v) +{ +struct monitor_write_data *w = &v->arch.vm_event->write_data;
const
Quoted 6 lines
xen/arch/x86/vm_event.c +6
-0+ +if ( !v->arch.vm_event->sync_event ) +return false; + +if ( w->do_write.cr0 ) +return true;
xen/arch/x86/vm_event.c void vm_event_emulate_check(struct vcpu *v, vm_event_response_t *rsp) · line 318
Quoted 10 lines
xen/arch/x86/vm_event.c +10
-0+if ( w->do_write.cr3 ) +return true; +if ( w->do_write.cr4 ) +return true; +if ( w->do_write.msr ) +return true; +if ( v->arch.vm_event->set_gprs ) +return true; +if ( v->arch.vm_event->emulate_flags ) +return true;
Can you please group this into a single if, ie:
if ( w->do_write.cr0 || w->do_write.cr3 || ... )
return true;
Quoted 30 lines
xen/arch/x86/vm_event.c +4
-0+ +return false; +} +/* * Local variables: * mode: C xen/common/vm_event.c +1
-3@@ -183,6 +183,7 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)if (vm_event_check_ring (ved) ) {struct vcpu *v; +bool pending_op = false;spin_lock (&ved->lock);@@ -192,9 +193,6 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)return -EBUSY ; } -/* Free domU's event channel and leave the other one unbound */ -free_xen_event_channel (d, ved->xen_port); -/* Unblock all vCPUs */ for_each_vcpu ( d, v ) {
xen/common/vm_event.c static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved) · line 206
Quoted 6 lines
xen/common/vm_event.c +3
-0@@ -203,8 +201,21 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)vcpu_unpause (v); ved->blocked--; } + +if (vm_event_check_pending_op (v) ) + pending_op = true;
You could just do:
pending_op |= vm_event_check_pending_op(v);
and avoid the initialization of pending_op above. Or alternatively:
if ( !pending_op && vm_event_check_pending_op(v) )
pending_op = true;
Which avoid repeated calls to vm_event_check_pending_op when at least one vCPU is known to be busy.
xen/common/vm_event.c static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved) · line 213
Quoted 7 lines
xen/common/vm_event.c +5
-0} +/* vm_event ops are still pending until vCPUs get scheduled */ +if ( pending_op ) + { +spin_unlock (&ved->lock); +return -EAGAIN ;
What happens when this gets called from vm_event_cleanup?
AFAICT the result there is ignored, and could leak the vm_event allocated memory?
Thanks, Roger.
Re: [PATCH v2 for-4.14 2/3] xen/vm_event: add vm_event_check_pending_op
Quoted 88 lines · Roger Pau Monné
Quoted 17 lines · Tamas K Lengyel
Perform sanity checking when shutting
vm_eventdown to determine whether it is safe to do so. Error out with-EAGAINin case pending operations have been found for the domain.Signed-off-by Tamas K Lengyel <[email protected]>--- xen/arch/x86/vm_event.c | 23 +++++++++++++++++++++++ xen/common/vm_event.c | 17 ++++++++++++++--- xen/include/asm-arm/vm_event.h | 7 +++++++ xen/include/asm-x86/vm_event.h | 2 ++ 4 files changed, 46 insertions(+), 3 deletions(-)xen/arch/x86/vm_event.c +0
-0xen/arch/x86/vm_event.c void vm_event_emulate_check(struct vcpu *v, vm_event_response_t *rsp) · line 302
Quoted 6 lines
xen/arch/x86/vm_event.c +3
-0@@ -297,6 +297,29 @@ void vm_event_emulate_check(struct vcpu *v, vm_event_response_t *rsp) }; } +bool vm_event_check_pending_op (const struct vcpu *v) +{ +struct monitor_write_data *w = &v->arch.vm_event->write_data;const
Quoted 6 lines
xen/arch/x86/vm_event.c +6
-0+ +if ( !v->arch.vm_event->sync_event ) +return false; + +if ( w->do_write.cr0 ) +return true;xen/arch/x86/vm_event.c void vm_event_emulate_check(struct vcpu *v, vm_event_response_t *rsp) · line 318
Quoted 10 lines
xen/arch/x86/vm_event.c +10
-0+if ( w->do_write.cr3 ) +return true; +if ( w->do_write.cr4 ) +return true; +if ( w->do_write.msr ) +return true; +if ( v->arch.vm_event->set_gprs ) +return true; +if ( v->arch.vm_event->emulate_flags ) +return true;Can you please group this into a single if, ie:
if (
w->do_write.cr0||w->do_write.cr3|| ... )return true;Quoted 30 lines
xen/arch/x86/vm_event.c +4
-0+ +return false; +} +/* * Local variables: * mode: C xen/common/vm_event.c +1
-3@@ -183,6 +183,7 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)if (vm_event_check_ring (ved) ) {struct vcpu *v; +bool pending_op = false;spin_lock (&ved->lock);@@ -192,9 +193,6 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)return -EBUSY ; } -/* Free domU's event channel and leave the other one unbound */ -free_xen_event_channel (d, ved->xen_port); -/* Unblock all vCPUs */ for_each_vcpu ( d, v ) {xen/common/vm_event.c static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved) · line 206
Quoted 6 lines
xen/common/vm_event.c +3
-0@@ -203,8 +201,21 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)vcpu_unpause (v); ved->blocked--; } + +if (vm_event_check_pending_op (v) ) + pending_op = true;You could just do:
Quoted 3 lines
pending_op|=vm_event_check_pending_op(v);and avoid the initialization of
pending_opabove.
The initialization has to stay, or it couldn't be |= afaict.
Quoted 7 lines
Or alternatively:
if ( !pending_op &&
vm_event_check_pending_op(v))pending_op = true;Which avoid repeated calls to
vm_event_check_pending_opwhen at least one vCPU is known to be busy.
if ( !pending_op )
pending_op = vm_event_check_pending_op(v);
?
Jan
Re: [PATCH v2 for-4.14 1/3] xen/monitor: Control register values
In reply to Roger Pau MonnéQuoted 8 lines · Roger Pau Monné
Quoted 3 lines · Tamas K Lengyel
Extend the
monitor_opdomctl to include option that enables controlling what values certain registers are permitted to hold by a monitor subscriber.I think the change could benefit for some more detail commit message here. Why is this useful?
You would have to ask the Bitdefender folks who made the feature. I don't use it. Here we are just making it optional as it is buggy so it is disabled by default.
Quoted 3 lines
There already seems to be some support for gating MSR writes, which seems to be expanded by this commit?
We don't expand on any existing features, we make an existing feature optional.
Quoted 2 lines
Is it solving some kind of bug reported?
It does, please take a look at the cover letter.
Quoted 37 lines
Quoted 37 lines
Signed-off-by Tamas K Lengyel <[email protected]>--- xen/arch/x86/hvm/hvm.c | 25 ++++++++++++++++--------- xen/arch/x86/monitor.c | 10 +++++++++- xen/include/asm-x86/domain.h | 1 + xen/include/public/domctl.h | 1 + 4 files changed, 27 insertions(+), 10 deletions(-)xen/arch/x86/hvm/hvm.c +4
-2@@ -2263,7 +2263,8 @@ int hvm_set_cr0(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR0 , value, old_value) ) +if (hvm_monitor_crX (CR0 , value, old_value) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr0 =1 ;@@ -2362,7 +2363,8 @@ int hvm_set_cr3(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR3 , value, old) ) +if (hvm_monitor_crX (CR3 , value, old) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr3 =1 ;@@ -2443,7 +2445,8 @@ int hvm_set_cr4(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event);
Quoted 6 lines
xen/arch/x86/hvm/hvm.c int hvm_set_cr4(unsigned long value, bool may_defer) · line 2449
Quoted 3 lines
xen/arch/x86/hvm/hvm.c +2
-1-if (hvm_monitor_crX (CR4 , value, old_cr) ) +if (hvm_monitor_crX (CR4 , value, old_cr) && + v->domain->arch.monitor.control_register_values )I think you could return
control_register_valuesinhvm_monitor_crXinstead of having to add the check to each caller?
We could, but this way the code is more consistent.
Quoted 8 lines
Quoted 8 lines
xen/arch/x86/hvm/hvm.c +0
-0{/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr4 =1 ;@@ -3587,13 +3590,17 @@ int hvm_msr_write_intercept(unsigned int msr, uint64_t msr_content,ASSERT (v->arch.vm_event);
Quoted 12 lines
xen/arch/x86/hvm/hvm.c int hvm_msr_write_intercept(unsigned int msr, uint64_t msr_content, · line 3595
Quoted 9 lines
xen/arch/x86/hvm/hvm.c +2
-6-/* The actual write will occur in hvm_do_resume() (if permitted). */ - v->arch.vm_event->write_data.do_write.msr =1 ; - v->arch.vm_event->write_data.msr = msr; - v->arch.vm_event->write_data.value = msr_content; -hvm_monitor_msr (msr, msr_content, msr_old_content); -return X86EMUL_OKAY ; + +if ( v->domain->arch.monitor.control_register_values )Is there any value in limiting
control_register_valuesto MSR that represent control registers, like EFER and XSS?
I don't know, you would have to ask Bitdefender about it who made this feature.
Quoted 10 lines
xen/arch/x86/hvm/hvm.c int hvm_msr_write_intercept(unsigned int msr, uint64_t msr_content, · line 3603
Quoted 9 lines
xen/arch/x86/hvm/hvm.c +8
-0+ { +/* The actual write will occur in hvm_do_resume(), if permitted. */ + v->arch.vm_event->write_data.do_write.msr =1 ; + v->arch.vm_event->write_data.msr = msr; + v->arch.vm_event->write_data.value = msr_content; + +return X86EMUL_OKAY ; + }
Quoted 6 lines
You seem to change the previous flow of the function here, that would just call
hvm_monitor_msrand return previously.Don't you need to move the return from outside the added if condition in order to keep previous behavior? Or else the write is committed straight away.
That's exactly what we want to achieve. Postponing the write is buggy. We want to make that feature optional. Before Bitdefender contributed that feature writes were always commited straight away, so with this patch we are actually reverting default behavior to what it was like to start with.
Quoted 13 lines
Quoted 13 lines
xen/arch/x86/hvm/hvm.c +0
-0}if ( (ret =guest_wrmsr (v, msr, msr_content)) !=X86EMUL_UNHANDLEABLE )xen/arch/x86/monitor.c +0
-1@@ -144,7 +144,15 @@ int arch_monitor_domctl_event(struct domain *d, struct xen_domctl_monitor_op *mop) {struct arch_domain *ad = &d->arch; -bool requested_status = (XEN_DOMCTL_MONITOR_OP_ENABLE == mop->op);
Quoted 8 lines
xen/arch/x86/monitor.c int arch_monitor_domctl_event(struct domain *d, · line 151
Quoted 5 lines
xen/arch/x86/monitor.c +5
-0+bool requested_status; + +if (XEN_DOMCTL_MONITOR_OP_CONTROL_REGISTERS == mop->op ) + { + ad->monitor.control_register_values = true;I think strictly speaking you need to use 1 here, since this variable is not a boolean.
Sure.
Thanks,
Tamas
Re: [PATCH v2 for-4.14 2/3] xen/vm_event: add vm_event_check_pending_op
In reply to Roger Pau MonnéQuoted 107 lines · Roger Pau Monné
Quoted 17 lines · Tamas K Lengyel
Perform sanity checking when shutting
vm_eventdown to determine whether it is safe to do so. Error out with-EAGAINin case pending operations have been found for the domain.Signed-off-by Tamas K Lengyel <[email protected]>--- xen/arch/x86/vm_event.c | 23 +++++++++++++++++++++++ xen/common/vm_event.c | 17 ++++++++++++++--- xen/include/asm-arm/vm_event.h | 7 +++++++ xen/include/asm-x86/vm_event.h | 2 ++ 4 files changed, 46 insertions(+), 3 deletions(-)xen/arch/x86/vm_event.c +0
-0xen/arch/x86/vm_event.c void vm_event_emulate_check(struct vcpu *v, vm_event_response_t *rsp) · line 302
Quoted 6 lines
xen/arch/x86/vm_event.c +3
-0@@ -297,6 +297,29 @@ void vm_event_emulate_check(struct vcpu *v, vm_event_response_t *rsp) }; } +bool vm_event_check_pending_op (const struct vcpu *v) +{ +struct monitor_write_data *w = &v->arch.vm_event->write_data;const
Quoted 6 lines
xen/arch/x86/vm_event.c +6
-0+ +if ( !v->arch.vm_event->sync_event ) +return false; + +if ( w->do_write.cr0 ) +return true;xen/arch/x86/vm_event.c void vm_event_emulate_check(struct vcpu *v, vm_event_response_t *rsp) · line 318
Quoted 10 lines
xen/arch/x86/vm_event.c +10
-0+if ( w->do_write.cr3 ) +return true; +if ( w->do_write.cr4 ) +return true; +if ( w->do_write.msr ) +return true; +if ( v->arch.vm_event->set_gprs ) +return true; +if ( v->arch.vm_event->emulate_flags ) +return true;Can you please group this into a single if, ie:
if (
w->do_write.cr0||w->do_write.cr3|| ... )return true;Quoted 30 lines
xen/arch/x86/vm_event.c +4
-0+ +return false; +} +/* * Local variables: * mode: C xen/common/vm_event.c +1
-3@@ -183,6 +183,7 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)if (vm_event_check_ring (ved) ) {struct vcpu *v; +bool pending_op = false;spin_lock (&ved->lock);@@ -192,9 +193,6 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)return -EBUSY ; } -/* Free domU's event channel and leave the other one unbound */ -free_xen_event_channel (d, ved->xen_port); -/* Unblock all vCPUs */ for_each_vcpu ( d, v ) {xen/common/vm_event.c static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved) · line 206
Quoted 6 lines
xen/common/vm_event.c +3
-0@@ -203,8 +201,21 @@ static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved)vcpu_unpause (v); ved->blocked--; } + +if (vm_event_check_pending_op (v) ) + pending_op = true;You could just do:
pending_op|=vm_event_check_pending_op(v);and avoid the initialization of
pending_opabove. Or alternatively:if ( !pending_op &&
vm_event_check_pending_op(v))pending_op = true;Which avoid repeated calls to
vm_event_check_pending_opwhen at least one vCPU is known to be busy.xen/common/vm_event.c static int vm_event_disable(struct domain *d, struct vm_event_domain **p_ved) · line 213
Quoted 7 lines
xen/common/vm_event.c +5
-0} +/* vm_event ops are still pending until vCPUs get scheduled */ +if ( pending_op ) + { +spin_unlock (&ved->lock); +return -EAGAIN ;
Quoted 4 lines
What happens when this gets called from
vm_event_cleanup?AFAICT the result there is ignored, and could leak the
vm_eventallocated memory?
Thanks for the feedback. I'm going to drop this patch at let Bitdefender pick it up if they feel like fixing their buggy feature. As things stand for my use-case I only need patch 1 from this series.
Tamas
Re: [PATCH v2 for-4.14 1/3] xen/monitor: Control register values
In reply to Tamas K LengyelQuoted 7 lines · Tamas K Lengyel
Quoted 6 lines · Roger Pau Monné
Quoted 3 lines · Tamas K Lengyel
Extend the
monitor_opdomctl to include option that enables controlling what values certain registers are permitted to hold by a monitor subscriber.
Quoted 6 lines
Quoted 2 lines
I think the change could benefit for some more detail commit message here. Why is this useful?
You would have to ask the Bitdefender folks who made the feature. I don't use it. Here we are just making it optional as it is buggy so it is disabled by default.
Now that's exactly the opposite of what I had derived from the description here so far. Perhaps an at least weak indication that you want to reword this. For example, from your reply to Roger I understand it's rather that the new flag allows to "suppress" the controlling (since presumably you don't change default behavior), rather then "enabling" it.
Quoted 9 lines
Quoted 2 lines
There already seems to be some support for gating MSR writes, which seems to be expanded by this commit?
We don't expand on any existing features, we make an existing feature optional.
Quoted 2 lines
Is it solving some kind of bug reported?
It does, please take a look at the cover letter.
Please can such important aspects also go in the commit message, so they're available later without needing to hunt down mail threads (besides this way being more readily available to reviewers)?
Jan
Re: [PATCH v2 for-4.14 1/3] xen/monitor: Control register values
Quoted 16 lines · Jan Beulich
Quoted 7 lines · Tamas K Lengyel
Quoted 6 lines · Roger Pau Monné
Quoted 3 lines · Tamas K Lengyel
Extend the
monitor_opdomctl to include option that enables controlling what values certain registers are permitted to hold by a monitor subscriber.Quoted 6 lines
Quoted 2 lines
I think the change could benefit for some more detail commit message here. Why is this useful?
You would have to ask the Bitdefender folks who made the feature. I don't use it. Here we are just making it optional as it is buggy so it is disabled by default.
Quoted 6 lines
Now that's exactly the opposite of what I had derived from the description here so far. Perhaps an at least weak indication that you want to reword this. For example, from your reply to Roger I understand it's rather that the new flag allows to "suppress" the controlling (since presumably you don't change default behavior), rather then "enabling" it.
What we are adding is a domctl you need to call that enables this feature. It's not an option to suppress it. It shouldn't have been enabled by default to begin with. That was a mistake when the feature was contributed and it is buggy.
Quoted 7 lines
Quoted 7 lines
Quoted 3 lines
There already seems to be some support for gating MSR writes, which seems to be expanded by this commit?
We don't expand on any existing features, we make an existing feature optional.
Quoted 1 line
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
Is it solving some kind of bug reported?
It does, please take a look at the cover letter.
Please can such important aspects also go in the commit message, so they're available later without needing to hunt down mail threads (besides this way being more readily available to reviewers)?
Noted.
Thanks,
Tamas
Re: [PATCH v2 for-4.14 3/3] xen/vm_event: Add safe to disable vm_event
In reply to Tamas K LengyelQuoted 1 line · Tamas K Lengyel
Instead of having to repeatedly try to disable
vm_events,
Why not use a hypercall continuation instead so that this is all hidden from the caller?
I take that the current interface requires the user to repeatedly issue hypercalls in order to disable vm_events until one of those succeeds?
Quoted 43 lines
request a specific
vm_eventto be sent when the domain is safe to continue with shutting down thevm_eventinterface.Signed-off-by Tamas K Lengyel <[email protected]>--- xen/arch/x86/hvm/hvm.c | 38 ++++++++++++++++++++++++++----- xen/arch/x86/hvm/monitor.c | 14 ++++++++++++ xen/arch/x86/monitor.c | 13 +++++++++++ xen/include/asm-x86/domain.h | 1 + xen/include/asm-x86/hvm/monitor.h | 1 + xen/include/public/domctl.h | 2 ++ xen/include/public/vm_event.h | 8 +++++++ 7 files changed, 71 insertions(+), 6 deletions(-)xen/arch/x86/hvm/hvm.c +15
-3@@ -563,15 +563,41 @@ void hvm_do_resume(struct vcpu *v) v->arch.hvm.inject_event.vector =HVM_EVENT_VECTOR_UNSET ; } -if (unlikely (v->arch.vm_event) && v->arch.monitor.next_interrupt_enabled ) +if (unlikely (v->arch.vm_event) ) { -struct x86_event info; +struct domain *d = v->domain; + +if ( v->arch.monitor.next_interrupt_enabled ) + { +struct x86_event info; + +if (hvm_get_pending_event (v, &info) ) + { +hvm_monitor_interrupt (info.vector, info.type, info.error_code, + info.cr2); + v->arch.monitor.next_interrupt_enabled = false; + } + } -if (hvm_get_pending_event (v, &info) ) +if ( d->arch.monitor.safe_to_disable )
xen/arch/x86/hvm/hvm.c void hvm_do_resume(struct vcpu *v) · line 589
Quoted 10 lines
xen/arch/x86/hvm/hvm.c +6
-3{ -hvm_monitor_interrupt (info.vector, info.type, info.error_code, - info.cr2); - v->arch.monitor.next_interrupt_enabled = false; +const struct vcpu *check_vcpu; +bool pending_op = false; + + for_each_vcpu ( d, check_vcpu ) + { +if (vm_event_check_pending_op (check_vcpu) )
Don't you need some kind of lock here, since you are poking at another vCPU which could be modifying any of those bits?
Quoted 19 lines
xen/arch/x86/hvm/hvm.c +11
-0+ { + pending_op = true; +break ; + } + } + +if ( !pending_op ) + { +hvm_monitor_safe_to_disable (); + d->arch.monitor.safe_to_disable = false; + } } } }xen/arch/x86/hvm/monitor.c +0
-0
xen/arch/x86/hvm/monitor.c bool hvm_monitor_check_p2m(unsigned long gla, gfn_t gfn, uint32_t pfec, · line 306
Quoted 7 lines
xen/arch/x86/hvm/monitor.c +4
-0@@ -300,6 +300,20 @@ bool hvm_monitor_check_p2m(unsigned long gla, gfn_t gfn, uint32_t pfec,return monitor_traps (curr, true, &req) >=0 ; } +void hvm_monitor_safe_to_disable (void ) +{ +struct vcpu *curr = current; +struct arch_domain *ad = &curr->domain->arch;
const
xen/arch/x86/hvm/monitor.c bool hvm_monitor_check_p2m(unsigned long gla, gfn_t gfn, uint32_t pfec, · line 310
Quoted 4 lines
xen/arch/x86/hvm/monitor.c +4
-0+vm_event_request_t req = {}; + +if ( !ad->monitor.safe_to_disable ) +return ;
Should this rather be an ASSERT? I don't think you are supposed to call hvm_monitor_safe_to_disable when the bit is not set?
xen/arch/x86/hvm/monitor.c bool hvm_monitor_check_p2m(unsigned long gla, gfn_t gfn, uint32_t pfec, · line 312
Quoted 2 lines
xen/arch/x86/hvm/monitor.c +2
-0+ + req.reason =VM_EVENT_REASON_SAFE_TO_DISABLE ;
I think you cat set the field at definition time.
Quoted 14 lines
xen/arch/x86/hvm/monitor.c +4
-0+ +monitor_traps (curr,0 , &req); +} +/* * Local variables: * mode: C xen/arch/x86/monitor.c +0
-0@@ -339,6 +339,19 @@ int arch_monitor_domctl_event(struct domain *d,break ; }
xen/arch/x86/monitor.c int arch_monitor_domctl_event(struct domain *d, · line 350
Quoted 10 lines
xen/arch/x86/monitor.c +9
-0+case XEN_DOMCTL_MONITOR_EVENT_SAFE_TO_DISABLE : + { +bool old_status = ad->monitor.safe_to_disable; + +if (unlikely (old_status == requested_status) ) +return -EEXIST ; + +domain_pause (d); + ad->monitor.safe_to_disable = requested_status;
Maybe I'm missing something, but I don't see any check that others events are disabled before safe_to_disable is set?
In the same way, you should prevent setting any events when safe_to_disable is set IMO, likely returning -EBUSY in both cases.
Thanks, Roger.
Re: [PATCH v2 for-4.14 1/3] xen/monitor: Control register values
In reply to Tamas K LengyelQuoted 24 lines · Tamas K Lengyel
Quoted 16 lines · Jan Beulich
Quoted 7 lines · Tamas K Lengyel
Quoted 6 lines · Roger Pau Monné
Quoted 3 lines · Tamas K Lengyel
Extend the
monitor_opdomctl to include option that enables controlling what values certain registers are permitted to hold by a monitor subscriber.Quoted 6 lines
Quoted 2 lines
I think the change could benefit for some more detail commit message here. Why is this useful?
You would have to ask the Bitdefender folks who made the feature. I don't use it. Here we are just making it optional as it is buggy so it is disabled by default.
Quoted 6 lines
Now that's exactly the opposite of what I had derived from the description here so far. Perhaps an at least weak indication that you want to reword this. For example, from your reply to Roger I understand it's rather that the new flag allows to "suppress" the controlling (since presumably you don't change default behavior), rather then "enabling" it.
Quoted 4 lines
What we are adding is a domctl you need to call that enables this feature. It's not an option to suppress it. It shouldn't have been enabled by default to begin with. That was a mistake when the feature was contributed and it is buggy.
Okay, in this case it's important to point out that you alter default behavior. The BitDefender folks may not like this, yet they've been surprisingly silent so far.
Jan
Re: [PATCH v2 for-4.14 1/3] xen/monitor: Control register values
In reply to Tamas K LengyelQuoted 18 lines · Tamas K Lengyel
Quoted 8 lines · Roger Pau Monné
Quoted 3 lines · Tamas K Lengyel
Extend the
monitor_opdomctl to include option that enables controlling what values certain registers are permitted to hold by a monitor subscriber.I think the change could benefit for some more detail commit message here. Why is this useful?
You would have to ask the Bitdefender folks who made the feature. I don't use it. Here we are just making it optional as it is buggy so it is disabled by default.
Quoted 3 lines
There already seems to be some support for gating MSR writes, which seems to be expanded by this commit?
Quoted 6 lines
We don't expand on any existing features, we make an existing feature optional.
Quoted 2 lines
Is it solving some kind of bug reported?
It does, please take a look at the cover letter.
Please copy the relevant bits here for reference.
Quoted 37 lines
Quoted 37 lines
Quoted 37 lines
Signed-off-by Tamas K Lengyel <[email protected]>--- xen/arch/x86/hvm/hvm.c | 25 ++++++++++++++++--------- xen/arch/x86/monitor.c | 10 +++++++++- xen/include/asm-x86/domain.h | 1 + xen/include/public/domctl.h | 1 + 4 files changed, 27 insertions(+), 10 deletions(-)xen/arch/x86/hvm/hvm.c +4
-2@@ -2263,7 +2263,8 @@ int hvm_set_cr0(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR0 , value, old_value) ) +if (hvm_monitor_crX (CR0 , value, old_value) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr0 =1 ;@@ -2362,7 +2363,8 @@ int hvm_set_cr3(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR3 , value, old) ) +if (hvm_monitor_crX (CR3 , value, old) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr3 =1 ;@@ -2443,7 +2445,8 @@ int hvm_set_cr4(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event);
Quoted 8 lines
Quoted 6 lines
xen/arch/x86/hvm/hvm.c int hvm_set_cr4(unsigned long value, bool may_defer) · line 2449
Quoted 3 lines
xen/arch/x86/hvm/hvm.c +2
-1-if (hvm_monitor_crX (CR4 , value, old_cr) ) +if (hvm_monitor_crX (CR4 , value, old_cr) && + v->domain->arch.monitor.control_register_values )I think you could return
control_register_valuesinhvm_monitor_crXinstead of having to add the check to each caller?We could, but this way the code is more consistent.
OK, I guess it's a matter of taste. I would rather prefer those checks to be confined to hvm_monitor_crX because then the generic code is not polluted with monitor checks, but that's likely just my taste.
Quoted 40 lines
Quoted 8 lines
Quoted 8 lines
xen/arch/x86/hvm/hvm.c +0
-0{/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr4 =1 ;@@ -3587,13 +3590,17 @@ int hvm_msr_write_intercept(unsigned int msr, uint64_t msr_content,ASSERT (v->arch.vm_event);Quoted 12 lines
xen/arch/x86/hvm/hvm.c int hvm_msr_write_intercept(unsigned int msr, uint64_t msr_content, · line 3595
Quoted 9 lines
xen/arch/x86/hvm/hvm.c +2
-6-/* The actual write will occur in hvm_do_resume() (if permitted). */ - v->arch.vm_event->write_data.do_write.msr =1 ; - v->arch.vm_event->write_data.msr = msr; - v->arch.vm_event->write_data.value = msr_content; -hvm_monitor_msr (msr, msr_content, msr_old_content); -return X86EMUL_OKAY ; + +if ( v->domain->arch.monitor.control_register_values )Is there any value in limiting
control_register_valuesto MSR that represent control registers, like EFER and XSS?I don't know, you would have to ask Bitdefender about it who made this feature.
Quoted 10 lines
xen/arch/x86/hvm/hvm.c int hvm_msr_write_intercept(unsigned int msr, uint64_t msr_content, · line 3603
Quoted 9 lines
xen/arch/x86/hvm/hvm.c +8
-0+ { +/* The actual write will occur in hvm_do_resume(), if permitted. */ + v->arch.vm_event->write_data.do_write.msr =1 ; + v->arch.vm_event->write_data.msr = msr; + v->arch.vm_event->write_data.value = msr_content; + +return X86EMUL_OKAY ; + }Quoted 6 lines
You seem to change the previous flow of the function here, that would just call
hvm_monitor_msrand return previously.Don't you need to move the return from outside the added if condition in order to keep previous behavior? Or else the write is committed straight away.
Quoted 5 lines
That's exactly what we want to achieve. Postponing the write is buggy. We want to make that feature optional. Before Bitdefender contributed that feature writes were always commited straight away, so with this patch we are actually reverting default behavior to what it was like to start with.
Oh, could this be made clear on the commit message then?
When I first saw the code I assumed this was wrong (I'm likely not familiar enough with the code anyway).
Thanks, Roger.
Re: [PATCH v2 for-4.14 1/3] xen/monitor: Control register values
Quoted 36 lines · Roger Pau Monné
Quoted 35 lines · Tamas K Lengyel
Quoted 33 lines · Roger Pau Monné
Quoted 23 lines · Tamas K Lengyel
xen/arch/x86/hvm/hvm.c +4
-2@@ -2263,7 +2263,8 @@ int hvm_set_cr0(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR0 , value, old_value) ) +if (hvm_monitor_crX (CR0 , value, old_value) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr0 =1 ;@@ -2362,7 +2363,8 @@ int hvm_set_cr3(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR3 , value, old) ) +if (hvm_monitor_crX (CR3 , value, old) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr3 =1 ;xen/arch/x86/hvm/hvm.c int hvm_set_cr4(unsigned long value, bool may_defer) · line 2449
Quoted 6 lines
xen/arch/x86/hvm/hvm.c +2
-1@@ -2443,7 +2445,8 @@ int hvm_set_cr4(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR4 , value, old_cr) ) +if (hvm_monitor_crX (CR4 , value, old_cr) && + v->domain->arch.monitor.control_register_values )I think you could return
control_register_valuesinhvm_monitor_crXinstead of having to add the check to each caller?
Quoted 5 lines
Quoted 1 line
We could, but this way the code is more consistent.
OK, I guess it's a matter of taste. I would rather prefer those checks to be confined to
hvm_monitor_crXbecause then the generic code is not polluted with monitor checks, but that's likely just my taste.
+1
Jan
Re: [PATCH v2 for-4.14 3/3] xen/vm_event: Add safe to disable vm_event
In reply to Roger Pau MonnéQuoted 10 lines · Roger Pau Monné
Quoted 1 line · Tamas K Lengyel
Instead of having to repeatedly try to disable
vm_events,Why not use a hypercall continuation instead so that this is all hidden from the caller?
I take that the current interface requires the user to repeatedly issue hypercalls in order to disable
vm_eventsuntil one of those succeeds?
No, it succeeds right away. And then the guest crashes in unique and unpredictable ways.
Quoted 128 lines
Quoted 44 lines
request a specific
vm_eventto be sent when the domain is safe to continue with shutting down thevm_eventinterface.Signed-off-by Tamas K Lengyel <[email protected]>--- xen/arch/x86/hvm/hvm.c | 38 ++++++++++++++++++++++++++----- xen/arch/x86/hvm/monitor.c | 14 ++++++++++++ xen/arch/x86/monitor.c | 13 +++++++++++ xen/include/asm-x86/domain.h | 1 + xen/include/asm-x86/hvm/monitor.h | 1 + xen/include/public/domctl.h | 2 ++ xen/include/public/vm_event.h | 8 +++++++ 7 files changed, 71 insertions(+), 6 deletions(-)xen/arch/x86/hvm/hvm.c +15
-3@@ -563,15 +563,41 @@ void hvm_do_resume(struct vcpu *v) v->arch.hvm.inject_event.vector =HVM_EVENT_VECTOR_UNSET ; } -if (unlikely (v->arch.vm_event) && v->arch.monitor.next_interrupt_enabled ) +if (unlikely (v->arch.vm_event) ) { -struct x86_event info; +struct domain *d = v->domain; + +if ( v->arch.monitor.next_interrupt_enabled ) + { +struct x86_event info; + +if (hvm_get_pending_event (v, &info) ) + { +hvm_monitor_interrupt (info.vector, info.type, info.error_code, + info.cr2); + v->arch.monitor.next_interrupt_enabled = false; + } + } -if (hvm_get_pending_event (v, &info) ) +if ( d->arch.monitor.safe_to_disable )xen/arch/x86/hvm/hvm.c void hvm_do_resume(struct vcpu *v) · line 589
Quoted 10 lines
xen/arch/x86/hvm/hvm.c +6
-3{ -hvm_monitor_interrupt (info.vector, info.type, info.error_code, - info.cr2); - v->arch.monitor.next_interrupt_enabled = false; +const struct vcpu *check_vcpu; +bool pending_op = false; + + for_each_vcpu ( d, check_vcpu ) + { +if (vm_event_check_pending_op (check_vcpu) )Don't you need some kind of lock here, since you are poking at another vCPU which could be modifying any of those bits?
Quoted 19 lines
xen/arch/x86/hvm/hvm.c +11
-0+ { + pending_op = true; +break ; + } + } + +if ( !pending_op ) + { +hvm_monitor_safe_to_disable (); + d->arch.monitor.safe_to_disable = false; + } } } }xen/arch/x86/hvm/monitor.c +0
-0xen/arch/x86/hvm/monitor.c bool hvm_monitor_check_p2m(unsigned long gla, gfn_t gfn, uint32_t pfec, · line 306
Quoted 7 lines
xen/arch/x86/hvm/monitor.c +4
-0@@ -300,6 +300,20 @@ bool hvm_monitor_check_p2m(unsigned long gla, gfn_t gfn, uint32_t pfec,return monitor_traps (curr, true, &req) >=0 ; } +void hvm_monitor_safe_to_disable (void ) +{ +struct vcpu *curr = current; +struct arch_domain *ad = &curr->domain->arch;const
xen/arch/x86/hvm/monitor.c bool hvm_monitor_check_p2m(unsigned long gla, gfn_t gfn, uint32_t pfec, · line 310
Quoted 4 lines
xen/arch/x86/hvm/monitor.c +4
-0+vm_event_request_t req = {}; + +if ( !ad->monitor.safe_to_disable ) +return ;Should this rather be an ASSERT? I don't think you are supposed to call
hvm_monitor_safe_to_disablewhen the bit is not set?xen/arch/x86/hvm/monitor.c bool hvm_monitor_check_p2m(unsigned long gla, gfn_t gfn, uint32_t pfec, · line 312
Quoted 2 lines
xen/arch/x86/hvm/monitor.c +2
-0+ + req.reason =VM_EVENT_REASON_SAFE_TO_DISABLE ;I think you cat set the field at definition time.
Quoted 14 lines
xen/arch/x86/hvm/monitor.c +4
-0+ +monitor_traps (curr,0 , &req); +} +/* * Local variables: * mode: C xen/arch/x86/monitor.c +0
-0@@ -339,6 +339,19 @@ int arch_monitor_domctl_event(struct domain *d,break ; }xen/arch/x86/monitor.c int arch_monitor_domctl_event(struct domain *d, · line 350
Quoted 10 lines
xen/arch/x86/monitor.c +9
-0+case XEN_DOMCTL_MONITOR_EVENT_SAFE_TO_DISABLE : + { +bool old_status = ad->monitor.safe_to_disable; + +if (unlikely (old_status == requested_status) ) +return -EEXIST ; + +domain_pause (d); + ad->monitor.safe_to_disable = requested_status;Maybe I'm missing something, but I don't see any check that others events are disabled before
safe_to_disableis set?
Quoted 4 lines
In the same way, you should prevent setting any events when
safe_to_disableis set IMO, likely returning-EBUSYin both cases.Thanks, Roger.
Thanks for the feedback again. I won't have the bandwidth to address these so I'm dropping this patch. If Bitdefender is so inclined to pick-up later they are welcome to do so. This is only needed if their buggy feature is enabled.
Tamas
Re: [PATCH v2 for-4.14 1/3] xen/monitor: Control register values
In reply to Jan BeulichQuoted 38 lines · Jan Beulich
Quoted 36 lines · Roger Pau Monné
Quoted 35 lines · Tamas K Lengyel
Quoted 33 lines · Roger Pau Monné
Quoted 23 lines · Tamas K Lengyel
xen/arch/x86/hvm/hvm.c +4
-2@@ -2263,7 +2263,8 @@ int hvm_set_cr0(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR0 , value, old_value) ) +if (hvm_monitor_crX (CR0 , value, old_value) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr0 =1 ;@@ -2362,7 +2363,8 @@ int hvm_set_cr3(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR3 , value, old) ) +if (hvm_monitor_crX (CR3 , value, old) && + v->domain->arch.monitor.control_register_values ) {/* The actual write will occur in hvm_do_resume(), if permitted. */ v->arch.vm_event->write_data.do_write.cr3 =1 ;xen/arch/x86/hvm/hvm.c int hvm_set_cr4(unsigned long value, bool may_defer) · line 2449
Quoted 6 lines
xen/arch/x86/hvm/hvm.c +2
-1@@ -2443,7 +2445,8 @@ int hvm_set_cr4(unsigned long value, bool may_defer) {ASSERT (v->arch.vm_event); -if (hvm_monitor_crX (CR4 , value, old_cr) ) +if (hvm_monitor_crX (CR4 , value, old_cr) && + v->domain->arch.monitor.control_register_values )I think you could return
control_register_valuesinhvm_monitor_crXinstead of having to add the check to each caller?
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
We could, but this way the code is more consistent.
OK, I guess it's a matter of taste. I would rather prefer those checks to be confined to
hvm_monitor_crXbecause then the generic code is not polluted with monitor checks, but that's likely just my taste.+1
OK.
Re: [PATCH v2 for-4.14 1/3] xen/monitor: Control register values
In reply to Roger Pau MonnéQuoted 20 lines · Roger Pau Monné
Quoted 18 lines · Tamas K Lengyel
Quoted 8 lines · Roger Pau Monné
Quoted 3 lines · Tamas K Lengyel
Extend the
monitor_opdomctl to include option that enables controlling what values certain registers are permitted to hold by a monitor subscriber.I think the change could benefit for some more detail commit message here. Why is this useful?
You would have to ask the Bitdefender folks who made the feature. I don't use it. Here we are just making it optional as it is buggy so it is disabled by default.
Quoted 3 lines
There already seems to be some support for gating MSR writes, which seems to be expanded by this commit?
Quoted 8 lines
Quoted 6 lines
We don't expand on any existing features, we make an existing feature optional.
Quoted 2 lines
Is it solving some kind of bug reported?
It does, please take a look at the cover letter.
Please copy the relevant bits here for reference.
Sure. As things stand I'm dropping patch 2 & 3 anyway so I'll just use the cover letter as the commit message.
Tamas
Re: [PATCH v2 for-4.14 1/3] xen/monitor: Control register values
In reply to Jan BeulichQuoted 26 lines · Jan Beulich
Quoted 24 lines · Tamas K Lengyel
Quoted 22 lines · Jan Beulich
Quoted 7 lines · Tamas K Lengyel
Quoted 6 lines · Roger Pau Monné
Quoted 3 lines · Tamas K Lengyel
Extend the
monitor_opdomctl to include option that enables controlling what values certain registers are permitted to hold by a monitor subscriber.Quoted 6 lines
Quoted 2 lines
I think the change could benefit for some more detail commit message here. Why is this useful?
You would have to ask the Bitdefender folks who made the feature. I don't use it. Here we are just making it optional as it is buggy so it is disabled by default.
Now that's exactly the opposite of what I had derived from the description here so far. Perhaps an at least weak indication that you want to reword this. For example, from your reply to Roger I understand it's rather that the new flag allows to "suppress" the controlling (since presumably you don't change default behavior), rather then "enabling" it.
Quoted 8 lines
Quoted 4 lines
What we are adding is a domctl you need to call that enables this feature. It's not an option to suppress it. It shouldn't have been enabled by default to begin with. That was a mistake when the feature was contributed and it is buggy.
Okay, in this case it's important to point out that you alter default behavior. The BitDefender folks may not like this, yet they've been surprisingly silent so far.
Well, it was Bitdefender who altered the default behavior. We are reverting their mistake and making it optional. But I can certainly make that more clear.
Tamas
Re: [PATCH v2 for-4.14 1/3] xen/monitor: Control register values
Quoted 32 lines · Tamas K Lengyel
Quoted 31 lines · Jan Beulich
Quoted 28 lines · Tamas K Lengyel
Quoted 16 lines · Jan Beulich
Quoted 7 lines · Tamas K Lengyel
Quoted 6 lines · Roger Pau Monné
Quoted 3 lines · Tamas K Lengyel
Extend the
monitor_opdomctl to include option that enables controlling what values certain registers are permitted to hold by a monitor subscriber.Quoted 6 lines
Quoted 2 lines
I think the change could benefit for some more detail commit message here. Why is this useful?
You would have to ask the Bitdefender folks who made the feature. I don't use it. Here we are just making it optional as it is buggy so it is disabled by default.
Quoted 6 lines
Now that's exactly the opposite of what I had derived from the description here so far. Perhaps an at least weak indication that you want to reword this. For example, from your reply to Roger I understand it's rather that the new flag allows to "suppress" the controlling (since presumably you don't change default behavior), rather then "enabling" it.
What we are adding is a domctl you need to call that enables this feature. It's not an option to suppress it. It shouldn't have been enabled by default to begin with. That was a mistake when the feature was contributed and it is buggy.
Quoted 7 lines
Quoted 3 lines
Okay, in this case it's important to point out that you alter default behavior. The BitDefender folks may not like this, yet they've been surprisingly silent so far.
Well, it was Bitdefender who altered the default behavior. We are reverting their mistake and making it optional. But I can certainly make that more clear.
I would like some input from the Bitdefender guys. Is there a bugfix planned for this feature?
It would be nice to get this in before 4.14 releases.
Roger.