Skip to contentSkip to conversation

x86/domctl: Don't pause the whole domain if only getting vcpu state

RFC PATCH In review Alexandru Isaila 2 participants 2 messages Feb 23, 2018 – Apr 24, 2018

lore
Share & tools
lore
#0 lore raw

RFC PATCH v3 1/1 x86/domctl: Don't pause the whole domain if only getting vcpu state

This patch adds the hvm_save_one_cpu_ctxt() function. It optimizes by only pausing the vcpu on all HVMSR_PER_VCPU save callbacks where only data for one VCPU is required.

---
Changes since V2:
- Added functions for all the save_*_one cases
- Added unpause to the error case in the hvm_save_one().

---
 tools/tests/vhpet/emul.h       |   3 +-
 tools/tests/vhpet/main.c       |   2 +-
 xen/arch/x86/cpu/mcheck/vmce.c |  29 +++-
 xen/arch/x86/domctl.c          |   2 -
 xen/arch/x86/hvm/hpet.c        |   2 +-
 xen/arch/x86/hvm/hvm.c         | 329 +++++++++++++++++++++++++----------------
 xen/arch/x86/hvm/i8254.c       |   2 +-
 xen/arch/x86/hvm/irq.c         |   6 +-
 xen/arch/x86/hvm/mtrr.c        |  58 +++++---
 xen/arch/x86/hvm/pmtimer.c     |   2 +-
 xen/arch/x86/hvm/rtc.c         |   2 +-
 xen/arch/x86/hvm/save.c        |  78 +++++++---
 xen/arch/x86/hvm/vioapic.c     |   2 +-
 xen/arch/x86/hvm/viridian.c    |  30 +++-
 xen/arch/x86/hvm/vlapic.c      |  23 ++-
 xen/arch/x86/hvm/vpic.c        |   2 +-
 xen/include/asm-x86/hvm/hvm.h  |   2 +
 xen/include/asm-x86/hvm/save.h |   5 +-
 18 files changed, 381 insertions(+), 198 deletions(-)
tools/tests/vhpet/emul.h +2 -1
index 383acff..99d5bbd 100644
@@ -296,7 +296,8 @@ struct hvm_hw_hpet
 };
 
 typedef int (*hvm_save_handler)(struct domain *d,
-                                hvm_domain_context_t *h);
+                                hvm_domain_context_t *h,
+                                unsigned int instance);
 typedef int (*hvm_load_handler)(struct domain *d,
                                 hvm_domain_context_t *h);
 
tools/tests/vhpet/main.c +1 -1
index 6fe65ea..3d8e7f5 100644
@@ -177,7 +177,7 @@ void __init hvm_register_savevm(uint16_t typecode,
 
 int do_save(uint16_t typecode, struct domain *d, hvm_domain_context_t *h)
 {
-    return hvm_sr_handlers[typecode].save(d, h);
+    return hvm_sr_handlers[typecode].save(d, h, d->max_vcpus);
 }
 
 int do_load(uint16_t typecode, struct domain *d, hvm_domain_context_t *h)
xen/arch/x86/cpu/mcheck/vmce.c +21 -8
index e07cd2f..98b12e0 100644
@@ -349,20 +349,33 @@ int vmce_wrmsr(uint32_t msr, uint64_t val)
     return ret;
 }
 
-static int vmce_save_vcpu_ctxt(struct domain *d, hvm_domain_context_t *h)
+void vmce_save_vcpu_ctxt_one(struct vcpu *v, struct hvm_vmce_vcpu *ctxt)
+{
+    ctxt->caps = v->arch.vmce.mcg_cap;
+    ctxt->mci_ctl2_bank0 = v->arch.vmce.bank[0].mci_ctl2;
+    ctxt->mci_ctl2_bank1 = v->arch.vmce.bank[1].mci_ctl2;
+    ctxt->mcg_ext_ctl = v->arch.vmce.mcg_ext_ctl;
+}
+
+static int vmce_save_vcpu_ctxt(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     struct vcpu *v;
     int err = 0;
 
-    for_each_vcpu ( d, v )
+    if( instance < d->max_vcpus )
+    {
+        struct hvm_vmce_vcpu ctxt;
+
+        v = d->vcpu[instance];
+        vmce_save_vcpu_ctxt_one(v, &ctxt);
+
+        err = hvm_save_entry(VMCE_VCPU, v->vcpu_id, h, &ctxt);
+    }
+    else for_each_vcpu ( d, v )
     {
-        struct hvm_vmce_vcpu ctxt = {
-            .caps = v->arch.vmce.mcg_cap,
-            .mci_ctl2_bank0 = v->arch.vmce.bank[0].mci_ctl2,
-            .mci_ctl2_bank1 = v->arch.vmce.bank[1].mci_ctl2,
-            .mcg_ext_ctl = v->arch.vmce.mcg_ext_ctl,
-        };
+        struct hvm_vmce_vcpu ctxt;
 
+        vmce_save_vcpu_ctxt_one(v, &ctxt);
         err = hvm_save_entry(VMCE_VCPU, v->vcpu_id, h, &ctxt);
         if ( err )
             break;
xen/arch/x86/domctl.c +0 -2
index 540ba08..d3c4e14 100644
@@ -624,12 +624,10 @@ long arch_do_domctl(
              !is_hvm_domain(d) )
             break;
 
-        domain_pause(d);
         ret = hvm_save_one(d, domctl->u.hvmcontext_partial.type,
                            domctl->u.hvmcontext_partial.instance,
                            domctl->u.hvmcontext_partial.buffer,
                            &domctl->u.hvmcontext_partial.bufsz);
-        domain_unpause(d);
 
         if ( !ret )
             copyback = true;
xen/arch/x86/hvm/hpet.c +1 -1
index 3ea895a..56f4691 100644
@@ -509,7 +509,7 @@ static const struct hvm_mmio_ops hpet_mmio_ops = {
 };
 
 
-static int hpet_save(struct domain *d, hvm_domain_context_t *h)
+static int hpet_save(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     HPETState *hp = domain_vhpet(d);
     struct vcpu *v = pt_global_vcpu_target(d);
xen/arch/x86/hvm/hvm.c +205 -124

397 lines not on this page · raw message

xen/arch/x86/hvm/i8254.c +1 -1
index 992f08d..143b64d 100644
@@ -390,7 +390,7 @@ void pit_stop_channel0_irq(PITState *pit)
     spin_unlock(&pit->lock);
 }
 
-static int pit_save(struct domain *d, hvm_domain_context_t *h)
+static int pit_save(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     PITState *pit = domain_vpit(d);
     int rc;
xen/arch/x86/hvm/irq.c +3 -3
index e425df9..dbbf769 100644
@@ -598,7 +598,7 @@ static int __init dump_irq_info_key_init(void)
 }
 __initcall(dump_irq_info_key_init);
 
-static int irq_save_pci(struct domain *d, hvm_domain_context_t *h)
+static int irq_save_pci(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     struct hvm_irq *hvm_irq = hvm_domain_irq(d);
     unsigned int asserted, pdev, pintx;
@@ -630,7 +630,7 @@ static int irq_save_pci(struct domain *d, hvm_domain_context_t *h)
     return rc;
 }
 
-static int irq_save_isa(struct domain *d, hvm_domain_context_t *h)
+static int irq_save_isa(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     struct hvm_irq *hvm_irq = hvm_domain_irq(d);
 
@@ -638,7 +638,7 @@ static int irq_save_isa(struct domain *d, hvm_domain_context_t *h)
     return ( hvm_save_entry(ISA_IRQ, 0, h, &hvm_irq->isa_irq) );
 }
 
-static int irq_save_link(struct domain *d, hvm_domain_context_t *h)
+static int irq_save_link(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     struct hvm_irq *hvm_irq = hvm_domain_irq(d);
 
xen/arch/x86/hvm/mtrr.c +36 -22

72 lines not on this page · raw message

xen/arch/x86/hvm/pmtimer.c +1 -1
index b70c299..21dcdeb 100644
@@ -249,7 +249,7 @@ static int handle_pmt_io(
     return X86EMUL_OKAY;
 }
 
-static int acpi_save(struct domain *d, hvm_domain_context_t *h)
+static int acpi_save(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     struct hvm_hw_acpi *acpi = &d->arch.hvm_domain.acpi;
     PMTState *s = &d->arch.hvm_domain.pl_time->vpmt;
xen/arch/x86/hvm/rtc.c +1 -1
index bcfa169..83f339d 100644
@@ -737,7 +737,7 @@ void rtc_migrate_timers(struct vcpu *v)
 }
 
 /* Save RTC hardware state */
-static int rtc_save(struct domain *d, hvm_domain_context_t *h)
+static int rtc_save(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     RTCState *s = domain_vrtc(d);
     int rc;
xen/arch/x86/hvm/save.c +60 -18

114 lines not on this page · raw message

xen/arch/x86/hvm/vioapic.c +1 -1
index 97b419f..34d6907 100644
@@ -569,7 +569,7 @@ int vioapic_get_trigger_mode(const struct domain *d, unsigned int gsi)
     return vioapic->redirtbl[pin].fields.trig_mode;
 }
 
-static int ioapic_save(struct domain *d, hvm_domain_context_t *h)
+static int ioapic_save(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     struct hvm_vioapic *s;
 
xen/arch/x86/hvm/viridian.c +23 -7
index f0fa59d..6415954 100644
@@ -994,7 +994,7 @@ out:
     return HVM_HCALL_completed;
 }
 
-static int viridian_save_domain_ctxt(struct domain *d, hvm_domain_context_t *h)
+static int viridian_save_domain_ctxt(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     struct hvm_viridian_domain_context ctxt = {
         .time_ref_count = d->arch.hvm_domain.viridian.time_ref_count.val,
@@ -1030,18 +1030,34 @@ static int viridian_load_domain_ctxt(struct domain *d, hvm_domain_context_t *h)
 HVM_REGISTER_SAVE_RESTORE(VIRIDIAN_DOMAIN, viridian_save_domain_ctxt,
                           viridian_load_domain_ctxt, 1, HVMSR_PER_DOM);
 
-static int viridian_save_vcpu_ctxt(struct domain *d, hvm_domain_context_t *h)
+void viridian_save_vcpu_ctxt_one(struct vcpu *v, struct hvm_viridian_vcpu_context *ctxt)
+{
+    ctxt->vp_assist_msr = v->arch.hvm_vcpu.viridian.vp_assist.msr.raw;
+    ctxt->vp_assist_vector = v->arch.hvm_vcpu.viridian.vp_assist.vector;
+}
+
+static int viridian_save_vcpu_ctxt(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     struct vcpu *v;
 
     if ( !is_viridian_domain(d) )
         return 0;
 
-    for_each_vcpu( d, v ) {
-        struct hvm_viridian_vcpu_context ctxt = {
-            .vp_assist_msr = v->arch.hvm_vcpu.viridian.vp_assist.msr.raw,
-            .vp_assist_vector = v->arch.hvm_vcpu.viridian.vp_assist.vector,
-        };
+    if( instance < d->max_vcpus )
+    {
+        struct hvm_viridian_vcpu_context ctxt;
+
+        v = d->vcpu[instance];
+
+        viridian_save_vcpu_ctxt_one(v, &ctxt);
+
+        if ( hvm_save_entry(VIRIDIAN_VCPU, v->vcpu_id, h, &ctxt) != 0 )
+            return 1;
+    }
+    else for_each_vcpu( d, v ) {
+        struct hvm_viridian_vcpu_context ctxt;
+
+        viridian_save_vcpu_ctxt_one(v, &ctxt);
 
         if ( hvm_save_entry(VIRIDIAN_VCPU, v->vcpu_id, h, &ctxt) != 0 )
             return 1;
xen/arch/x86/hvm/vlapic.c +19 -4
index 4bfc53e..591631a 100644
@@ -1387,7 +1387,7 @@ static void lapic_rearm(struct vlapic *s)
     s->timer_last_update = s->pt.last_plt_gtime;
 }
 
-static int lapic_save_hidden(struct domain *d, hvm_domain_context_t *h)
+static int lapic_save_hidden(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     struct vcpu *v;
     struct vlapic *s;
@@ -1396,7 +1396,13 @@ static int lapic_save_hidden(struct domain *d, hvm_domain_context_t *h)
     if ( !has_vlapic(d) )
         return 0;
 
-    for_each_vcpu ( d, v )
+    if( instance < d->max_vcpus )
+    {
+        v = d->vcpu[instance];
+        s = vcpu_vlapic(v);
+        rc = hvm_save_entry(LAPIC, v->vcpu_id, h, &s->hw);
+    }
+    else for_each_vcpu ( d, v )
     {
         s = vcpu_vlapic(v);
         if ( (rc = hvm_save_entry(LAPIC, v->vcpu_id, h, &s->hw)) != 0 )
@@ -1406,7 +1412,7 @@ static int lapic_save_hidden(struct domain *d, hvm_domain_context_t *h)
     return rc;
 }
 
-static int lapic_save_regs(struct domain *d, hvm_domain_context_t *h)
+static int lapic_save_regs(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     struct vcpu *v;
     struct vlapic *s;
@@ -1415,7 +1421,16 @@ static int lapic_save_regs(struct domain *d, hvm_domain_context_t *h)
     if ( !has_vlapic(d) )
         return 0;
 
-    for_each_vcpu ( d, v )
+    if( instance < d->max_vcpus )
+    {
+        v = d->vcpu[instance];
+        if ( hvm_funcs.sync_pir_to_irr )
+            hvm_funcs.sync_pir_to_irr(v);
+
+        s = vcpu_vlapic(v);
+        rc = hvm_save_entry(LAPIC_REGS, v->vcpu_id, h, s->regs);
+    }
+    else for_each_vcpu ( d, v )
     {
         if ( hvm_funcs.sync_pir_to_irr )
             hvm_funcs.sync_pir_to_irr(v);
xen/arch/x86/hvm/vpic.c +1 -1
index e160bbd..6b77f3c 100644
@@ -371,7 +371,7 @@ static int vpic_intercept_elcr_io(
     return X86EMUL_OKAY;
 }
 
-static int vpic_save(struct domain *d, hvm_domain_context_t *h)
+static int vpic_save(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {
     struct hvm_hw_vpic *s;
     int i;
xen/include/asm-x86/hvm/hvm.h +2 -0
index b687e03..9f13112 100644
@@ -609,6 +609,8 @@ static inline bool altp2m_vcpu_emulate_ve(struct vcpu *v)
     return false;
 }
 
+void hvm_save_cpu_ctxt_one(struct vcpu *v, struct hvm_hw_cpu *ctxt);
+
 /* Check CR4/EFER values */
 const char *hvm_efer_valid(const struct vcpu *v, uint64_t value,
                            signed int cr0_pg);
xen/include/asm-x86/hvm/save.h +3 -2
index f889e8f..a2c39c4 100644
@@ -95,8 +95,9 @@ static inline uint16_t hvm_load_instance(struct hvm_domain_context *h)
  * The save handler may save multiple instances of a type into the buffer;
  * the load handler will be called once for each instance found when
  * restoring.  Both return non-zero on error. */
-typedef int (*hvm_save_handler) (struct domain *d, 
-                                 hvm_domain_context_t *h);
+typedef int (*hvm_save_handler) (struct domain *d,
+                                 hvm_domain_context_t *h,
+                                 unsigned int instance);
 typedef int (*hvm_load_handler) (struct domain *d,
                                  hvm_domain_context_t *h);
 
Signature · 7 lines
2.7.4


_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
#1 lore raw
Quoted 18 lines
Quoted 1 line
Quoted 1 line · On 23.02.18 at 16:32, <[email protected]> wrote:
xen/arch/x86/cpu/mcheck/vmce.c +9 -1
@@ -349,20 +349,33 @@ int vmce_wrmsr(uint32_t msr, uint64_t val)
     return ret;
 }
 
-static int vmce_save_vcpu_ctxt(struct domain *d, hvm_domain_context_t *h)
+void vmce_save_vcpu_ctxt_one(struct vcpu *v, struct hvm_vmce_vcpu *ctxt)
+{
+    ctxt->caps = v->arch.vmce.mcg_cap;
+    ctxt->mci_ctl2_bank0 = v->arch.vmce.bank[0].mci_ctl2;
+    ctxt->mci_ctl2_bank1 = v->arch.vmce.bank[1].mci_ctl2;
+    ctxt->mcg_ext_ctl = v->arch.vmce.mcg_ext_ctl;
+}
+
+static int vmce_save_vcpu_ctxt(struct domain *d, hvm_domain_context_t *h, unsigned int instance)
 {

xen/arch/x86/cpu/mcheck/vmce.c int vmce_wrmsr(uint32_t msr, uint64_t val) · line 370

Quoted 10 lines
xen/arch/x86/cpu/mcheck/vmce.c +6 -1
     struct vcpu *v;
     int err = 0;
 
-    for_each_vcpu ( d, v )
+    if( instance < d->max_vcpus )
+    {
+        struct hvm_vmce_vcpu ctxt;
+
+        v = d->vcpu[instance];
+        vmce_save_vcpu_ctxt_one(v, &ctxt);

Please add a NULL check for v between these two lines, like we do everywhere else when accessing d->vcpu[].

Quoted 6 lines
xen/arch/x86/cpu/mcheck/vmce.c +3 -2
+        err = hvm_save_entry(VMCE_VCPU, v->vcpu_id, h, &ctxt);
+    }
+    else for_each_vcpu ( d, v )
     {
-        struct hvm_vmce_vcpu ctxt = {
-            .caps = v->arch.vmce.mcg_cap,

xen/arch/x86/cpu/mcheck/vmce.c int vmce_wrmsr(uint32_t msr, uint64_t val) · line 381

Quoted 10 lines
xen/arch/x86/cpu/mcheck/vmce.c +2 -4
-            .mci_ctl2_bank0 = v->arch.vmce.bank[0].mci_ctl2,
-            .mci_ctl2_bank1 = v->arch.vmce.bank[1].mci_ctl2,
-            .mcg_ext_ctl = v->arch.vmce.mcg_ext_ctl,
-        };
+        struct hvm_vmce_vcpu ctxt;
 
+        vmce_save_vcpu_ctxt_one(v, &ctxt);
         err = hvm_save_entry(VMCE_VCPU, v->vcpu_id, h, &ctxt);
         if ( err )
             break;

Note the recurring pattern throughout the patch:

    if ( instance < d->max_vcpus )
    {
        ...
        xyz_save_one();
    }
    else for_each_vcpu ( d, v )
    {
        ...
        xyz_save_one();
    }

I think you want to move this into the caller, such that each specific handler only deals with a single instance. I also think you want to split the patch, introducing the save_one() functions one at a time (used by their save() counterparts), and only then dropping the save() ones in favor of doing the loop in the caller. The final patch would then add the single instance case you're after.

Also please pay attention to coding style.

Jan

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel