Skip to contentSkip to conversation

tools: Delete XEN_DOMCTL_disable_migrate

PATCH In review Andrew Cooper 3 participants 5 messages Sep 11, 2020 – Sep 16, 2020 Arm/actions semi architecture

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 tools: Delete XEN_DOMCTL_disable_migrate

It is conceptually wrong for this information to exist in the hypervisor in the first place. Only the toolstack is capable of correctly reasoning about the non-migrateability of guests.

This hypercall has only ever existed to control the visibility of the Invariant TSC flag to the guest. Now that we have properly disentanged that and moved ITSC into the guests CPUID policy, delete this hypercall.

Furthermore, this fixes a corner case where Xen would override the toolstacks choice of ITSC for a xenstore stubdomain. That said, init-xenstore-domain doesn't currently ask for ITSC, and MiniOS doesn't know what ITSC is either, so nothing actually changes in practice.

---
CC: Jan Beulich <[email protected]>
CC: Roger Pau Monné <[email protected]>
CC: Wei Liu <[email protected]>
CC: Stefano Stabellini <[email protected]>
CC: Julien Grall <[email protected]>
CC: Volodymyr Babchuk <[email protected]>
CC: Ian Jackson <[email protected]>
CC: Wei Liu <[email protected]>
CC: Juergen Gross <[email protected]>
CC: Marek Marczykowski-Górecki <[email protected]>

---
 tools/libs/ctrl/include/xenctrl.h   |  2 --
 tools/libs/ctrl/xc_domain.c         |  9 ---------
 tools/libxl/libxl_dom.c             |  6 ------
 tools/python/xen/lowlevel/xc/xc.c   | 21 ---------------------
 xen/arch/arm/domctl.c               |  4 ----
 xen/arch/x86/cpuid.c                |  3 ++-
 xen/arch/x86/domctl.c               |  5 -----
 xen/common/domain.c                 |  4 ----
 xen/include/public/domctl.h         |  8 +-------
 xen/include/xen/sched.h             |  2 --
 xen/xsm/flask/hooks.c               |  1 -
 xen/xsm/flask/policy/access_vectors |  2 +-
 12 files changed, 4 insertions(+), 63 deletions(-)
tools/libs/ctrl/include/xenctrl.h +0 -2
index 0a921a95fa..73e9535fc8 100644
@@ -1351,8 +1351,6 @@ int xc_domain_get_tsc_info(xc_interface *xch,
                            uint32_t *gtsc_khz,
                            uint32_t *incarnation);
 
-int xc_domain_disable_migrate(xc_interface *xch, uint32_t domid);
-
 int xc_domain_maximum_gpfn(xc_interface *xch, uint32_t domid, xen_pfn_t *gpfns);
 
 int xc_domain_nr_gpfns(xc_interface *xch, uint32_t domid, xen_pfn_t *gpfns);
tools/libs/ctrl/xc_domain.c +0 -9
index 43fab50c06..e7cea4a17d 100644
@@ -815,15 +815,6 @@ int xc_domain_set_time_offset(xc_interface *xch,
     return do_domctl(xch, &domctl);
 }
 
-int xc_domain_disable_migrate(xc_interface *xch, uint32_t domid)
-{
-    DECLARE_DOMCTL;
-    domctl.cmd = XEN_DOMCTL_disable_migrate;
-    domctl.domain = domid;
-    domctl.u.disable_migrate.disable = 1;
-    return do_domctl(xch, &domctl);
-}
-
 int xc_domain_set_tsc_info(xc_interface *xch,
                            uint32_t domid,
                            uint32_t tsc_mode,
tools/libxl/libxl_dom.c +0 -6
index e2dca64aa1..597a6826d1 100644
@@ -257,12 +257,6 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid,
         return ERROR_FAIL;
     }
 
-    if (libxl_defbool_val(d_config->b_info.disable_migrate) &&
-        xc_domain_disable_migrate(ctx->xch, domid) != 0) {
-        LOG(ERROR, "Couldn't set nomigrate");
-        return ERROR_FAIL;
-    }
-
     /*
      * Check if the domain has any CPU or node affinity already. If not, try
      * to build up the latter via automatic NUMA placement. In fact, in case
tools/python/xen/lowlevel/xc/xc.c +0 -21
index 8c7b184f0b..ce123f0409 100644
@@ -1469,20 +1469,6 @@ static PyObject *pyxc_domain_set_tsc_info(XcObject *self, PyObject *args)
     return zero;
 }
 
-static PyObject *pyxc_domain_disable_migrate(XcObject *self, PyObject *args)
-{
-    uint32_t dom;
-
-    if (!PyArg_ParseTuple(args, "i", &dom))
-        return NULL;
-
-    if (xc_domain_disable_migrate(self->xc_handle, dom) != 0)
-        return pyxc_error_to_exception(self->xc_handle);
-
-    Py_INCREF(zero);
-    return zero;
-}
-
 static PyObject *pyxc_domain_send_trigger(XcObject *self,
                                           PyObject *args,
                                           PyObject *kwds)
@@ -2306,13 +2292,6 @@ static PyMethodDef pyxc_methods[] = {
       "                   1=always emulate 2=never emulate\n"
       "Returns: [int] 0 on success; -1 on error.\n" },
 
-    { "domain_disable_migrate",
-      (PyCFunction)pyxc_domain_disable_migrate,
-      METH_VARARGS, "\n"
-      "Marks domain as non-migratable AND non-restoreable\n"
-      " dom        [int]: Domain whose TSC mode is being set.\n"
-      "Returns: [int] 0 on success; -1 on error.\n" },
-
     { "domain_send_trigger",
       (PyCFunction)pyxc_domain_send_trigger,
       METH_VARARGS | METH_KEYWORDS, "\n"
xen/arch/arm/domctl.c +0 -4
index 9da88b8c64..b7d27f37df 100644
@@ -146,10 +146,6 @@ long arch_do_domctl(struct xen_domctl *domctl, struct domain *d,
         return 0;
     }
 
-    case XEN_DOMCTL_disable_migrate:
-        d->disable_migrate = domctl->u.disable_migrate.disable;
-        return 0;
-
     case XEN_DOMCTL_vuart_op:
     {
         int rc;
xen/arch/x86/cpuid.c +2 -1
index 23425790e1..dde3e95284 100644
@@ -708,7 +708,8 @@ int init_domain_cpuid_policy(struct domain *d)
     if ( !p )
         return -ENOMEM;
 
-    if ( d->disable_migrate )
+    /* The hardware domain can't migrate.  Give it ITSC if available. */
+    if ( is_hardware_domain(d) )
         p->extd.itsc = cpu_has_itsc;
 
     /*
xen/arch/x86/domctl.c +0 -5
index 7c644add92..2ee29a0f78 100644
@@ -1277,11 +1277,6 @@ long arch_do_domctl(
 
         break;
 
-    case XEN_DOMCTL_disable_migrate:
-        d->disable_migrate = domctl->u.disable_migrate.disable;
-        recalculate_cpuid_policy(d);
-        break;
-
     case XEN_DOMCTL_get_cpu_policy:
         /* Process the CPUID leaves. */
         if ( guest_handle_is_null(domctl->u.cpu_policy.cpuid_policy) )
xen/common/domain.c +0 -4
index f0f9c62feb..61ffccde0c 100644
@@ -371,7 +371,6 @@ struct domain *domain_create(domid_t domid,
         if ( hardware_domid < 0 || hardware_domid >= DOMID_FIRST_RESERVED )
             panic("The value of hardware_dom must be a valid domain ID\n");
 
-        d->disable_migrate = true;
         old_hwdom = hardware_domain;
         hardware_domain = d;
     }
@@ -452,9 +451,6 @@ struct domain *domain_create(domid_t domid,
         watchdog_domain_init(d);
         init_status |= INIT_watchdog;
 
-        if ( is_xenstore_domain(d) )
-            d->disable_migrate = true;
-
         d->iomem_caps = rangeset_new(d, "I/O Memory", RANGESETF_prettyprint_hex);
         d->irq_caps   = rangeset_new(d, "Interrupts", 0);
         if ( !d->iomem_caps || !d->irq_caps )
xen/include/public/domctl.h +1 -7
index 9464a9058a..be8bd79d9d 100644
@@ -730,11 +730,6 @@ struct xen_domctl_hvmcontext_partial {
     XEN_GUEST_HANDLE_64(uint8) buffer;  /* OUT: buffer to write record into */
 };
 
-/* XEN_DOMCTL_disable_migrate */
-struct xen_domctl_disable_migrate {
-    uint32_t disable; /* IN: 1: disable migration and restore */
-};
-
 
 /* XEN_DOMCTL_gettscinfo */
 /* XEN_DOMCTL_settscinfo */
@@ -1191,7 +1186,7 @@ struct xen_domctl {
 #define XEN_DOMCTL_gethvmcontext_partial         55
 #define XEN_DOMCTL_vm_event_op                   56
 #define XEN_DOMCTL_mem_sharing_op                57
-#define XEN_DOMCTL_disable_migrate               58
+/* #define XEN_DOMCTL_disable_migrate            58 - Obsolete */
 #define XEN_DOMCTL_gettscinfo                    59
 #define XEN_DOMCTL_settscinfo                    60
 #define XEN_DOMCTL_getpageframeinfo3             61
@@ -1242,7 +1237,6 @@ struct xen_domctl {
         struct xen_domctl_ioport_permission ioport_permission;
         struct xen_domctl_hypercall_init    hypercall_init;
         struct xen_domctl_settimeoffset     settimeoffset;
-        struct xen_domctl_disable_migrate   disable_migrate;
         struct xen_domctl_tsc_info          tsc_info;
         struct xen_domctl_hvmcontext        hvmcontext;
         struct xen_domctl_hvmcontext_partial hvmcontext_partial;
xen/include/xen/sched.h +0 -2
index ac53519d7f..e9ce43cc9a 100644
@@ -401,8 +401,6 @@ struct domain
     bool             is_privileged;
     /* Can this guest access the Xen console? */
     bool             is_console;
-    /* Non-migratable and non-restoreable? */
-    bool             disable_migrate;
     /* Is this guest being debugged by dom0? */
     bool             debugger_attached;
     /*
xen/xsm/flask/hooks.c +0 -1
index a314bf85ce..fab5d30c3a 100644
@@ -710,7 +710,6 @@ static int flask_domctl(struct domain *d, int cmd)
         return current_has_perm(d, SECCLASS_DOMAIN, DOMAIN__SETDEBUGGING);
 
     case XEN_DOMCTL_subscribe:
-    case XEN_DOMCTL_disable_migrate:
         return current_has_perm(d, SECCLASS_DOMAIN, DOMAIN__SET_MISC_INFO);
 
     case XEN_DOMCTL_set_virq_handler:
xen/xsm/flask/policy/access_vectors +1 -1
index b87c99ea98..fde5162c7e 100644
@@ -176,7 +176,7 @@ class domain
     getpodtarget
 # XENMEM_set_pod_target
     setpodtarget
-# XEN_DOMCTL_subscribe, XEN_DOMCTL_disable_migrate
+# XEN_DOMCTL_subscribe
     set_misc_info
 # XEN_DOMCTL_set_virq_handler
     set_virq_handler
Signature · 1 line
2.11.0
#1 lore raw
Quoted 3 lines · Andrew Cooper

It is conceptually wrong for this information to exist in the hypervisor in the first place. Only the toolstack is capable of correctly reasoning about the non-migrateability of guests.

But isn't it the purpose of the domctl to tell Xen about the tool stack's decision in this regard?

Quoted 6 lines

This hypercall has only ever existed to control the visibility of the Invariant TSC flag to the guest. Now that we have properly disentanged that and moved ITSC into the guests CPUID policy, delete this hypercall.

Furthermore, this fixes a corner case where Xen would override the toolstacks choice of ITSC for a xenstore stubdomain.

I'm afraid I don't fully understand: A xenstore stubdom can't be migrated (or at least it isn't supposed to be), can it? In which case - what's wrong with exposing to it even by default a feature it may be able to make use of? IOW ...

xen/arch/x86/cpuid.c int init_domain_cpuid_policy(struct domain *d) · line 713

Quoted 10 lines
xen/arch/x86/cpuid.c +2 -1
@@ -708,7 +708,8 @@ int init_domain_cpuid_policy(struct domain *d)
     if ( !p )
         return -ENOMEM;
 
-    if ( d->disable_migrate )
+    /* The hardware domain can't migrate.  Give it ITSC if available. */
+    if ( is_hardware_domain(d) )
         p->extd.itsc = cpu_has_itsc;

... why not include is_xenstore_domain() here that you remove from
...

xen/arch/x86/cpuid.c struct domain *domain_create(domid_t domid, · line 456

Quoted 6 lines
xen/arch/x86/cpuid.c +0 -2
@@ -452,9 +451,6 @@ struct domain *domain_create(domid_t domid,
         watchdog_domain_init(d);
         init_status |= INIT_watchdog;
 
-        if ( is_xenstore_domain(d) )
-            d->disable_migrate = true;

... here? On the tool stack side the change here only deletes code, i.e. I don't see you taking care of the default enabling there either. Am I overlooking any logic that now causes the feature to be requested for the xenstore domain without you needing to add any code?

Quoted 24 lines
xen/include/public/domctl.h +1 -6
@@ -730,11 +730,6 @@ struct xen_domctl_hvmcontext_partial {
     XEN_GUEST_HANDLE_64(uint8) buffer;  /* OUT: buffer to write record into */
 };
 
-/* XEN_DOMCTL_disable_migrate */
-struct xen_domctl_disable_migrate {
-    uint32_t disable; /* IN: 1: disable migration and restore */
-};
-
 
 /* XEN_DOMCTL_gettscinfo */
 /* XEN_DOMCTL_settscinfo */
@@ -1191,7 +1186,7 @@ struct xen_domctl {
 #define XEN_DOMCTL_gethvmcontext_partial         55
 #define XEN_DOMCTL_vm_event_op                   56
 #define XEN_DOMCTL_mem_sharing_op                57
-#define XEN_DOMCTL_disable_migrate               58
+/* #define XEN_DOMCTL_disable_migrate            58 - Obsolete */
 #define XEN_DOMCTL_gettscinfo                    59
 #define XEN_DOMCTL_settscinfo                    60
 #define XEN_DOMCTL_getpageframeinfo3             61

xen/include/public/domctl.h struct xen_domctl { · line 1242

Quoted 7 lines
xen/include/public/domctl.h +0 -1
@@ -1242,7 +1237,6 @@ struct xen_domctl {
         struct xen_domctl_ioport_permission ioport_permission;
         struct xen_domctl_hypercall_init    hypercall_init;
         struct xen_domctl_settimeoffset     settimeoffset;
-        struct xen_domctl_disable_migrate   disable_migrate;
         struct xen_domctl_tsc_info          tsc_info;
         struct xen_domctl_hvmcontext        hvmcontext;
         struct xen_domctl_hvmcontext_partial hvmcontext_partial;

Deletion of sub-ops, just like their modification, requires the interface version to get bumped if it wasn't already during a release cycle. I know you dislike the underlying concept, but as long as the interface version continues to exist (with its present meaning) I'm afraid it needs bumping for any backwards- incompatible change.

Jan

#2 lore raw
Quoted 6 lines · Jan Beulich
Quoted 3 lines · Andrew Cooper

It is conceptually wrong for this information to exist in the hypervisor in the first place. Only the toolstack is capable of correctly reasoning about the non-migrateability of guests.

But isn't it the purpose of the domctl to tell Xen about the tool stack's decision in this regard?

There is nothing (not even ITSC handling, which was buggy) which Xen can legitimately do with the information.

It is conceptually wrong, and a layering violation, for Xen to have this information, or to be making any assumptions about it.

Quoted 8 lines
Quoted 6 lines

This hypercall has only ever existed to control the visibility of the Invariant TSC flag to the guest. Now that we have properly disentanged that and moved ITSC into the guests CPUID policy, delete this hypercall.

Furthermore, this fixes a corner case where Xen would override the toolstacks choice of ITSC for a xenstore stubdomain.

I'm afraid I don't fully understand: A xenstore stubdom can't be migrated (or at least it isn't supposed to be), can it?

That is some very wild assumptions.

What if someone is trying to debug a time related issue, and wants to turn itsc off?

Quoted 3 lines

In which
case - what's wrong with exposing to it even by default a feature it may be able to make use of?

Because silently trampling the configuration chosen by the toolstack isn't acceptable.

Quoted 19 lines

IOW ...

xen/arch/x86/cpuid.c int init_domain_cpuid_policy(struct domain *d) · line 713

Quoted 10 lines
xen/arch/x86/cpuid.c +2 -1
@@ -708,7 +708,8 @@ int init_domain_cpuid_policy(struct domain *d)
     if ( !p )
         return -ENOMEM;
 
-    if ( d->disable_migrate )
+    /* The hardware domain can't migrate.  Give it ITSC if available. */
+    if ( is_hardware_domain(d) )
         p->extd.itsc = cpu_has_itsc;

... why not include is_xenstore_domain() here that you remove from
...

Quoted 4 lines
xen/arch/x86/cpuid.c +0 -0
@@ -452,9 +451,6 @@ struct domain *domain_create(domid_t domid,
         watchdog_domain_init(d);
         init_status |= INIT_watchdog;
 
Quoted 7 lines

xen/arch/x86/cpuid.c struct domain *domain_create(domid_t domid, · line 456

Quoted 2 lines
xen/arch/x86/cpuid.c +0 -2
-        if ( is_xenstore_domain(d) )
-            d->disable_migrate = true;

... here? On the tool stack side the change here only deletes code, i.e. I don't see you taking care of the default enabling there either. Am I overlooking any logic that now causes the feature to be requested for the xenstore domain without you needing to add any code?

The toolstack (legitimately) has knowledge of nomigrate, and will even implicitly turn on ITSC as a side effect, but will also honour explicit requests to turn it on or off.

Quoted 29 lines
Quoted 29 lines
xen/include/public/domctl.h +1 -7
@@ -730,11 +730,6 @@ struct xen_domctl_hvmcontext_partial {
     XEN_GUEST_HANDLE_64(uint8) buffer;  /* OUT: buffer to write record into */
 };
 
-/* XEN_DOMCTL_disable_migrate */
-struct xen_domctl_disable_migrate {
-    uint32_t disable; /* IN: 1: disable migration and restore */
-};
-
 
 /* XEN_DOMCTL_gettscinfo */
 /* XEN_DOMCTL_settscinfo */
@@ -1191,7 +1186,7 @@ struct xen_domctl {
 #define XEN_DOMCTL_gethvmcontext_partial         55
 #define XEN_DOMCTL_vm_event_op                   56
 #define XEN_DOMCTL_mem_sharing_op                57
-#define XEN_DOMCTL_disable_migrate               58
+/* #define XEN_DOMCTL_disable_migrate            58 - Obsolete */
 #define XEN_DOMCTL_gettscinfo                    59
 #define XEN_DOMCTL_settscinfo                    60
 #define XEN_DOMCTL_getpageframeinfo3             61
@@ -1242,7 +1237,6 @@ struct xen_domctl {
         struct xen_domctl_ioport_permission ioport_permission;
         struct xen_domctl_hypercall_init    hypercall_init;
         struct xen_domctl_settimeoffset     settimeoffset;
-        struct xen_domctl_disable_migrate   disable_migrate;
         struct xen_domctl_tsc_info          tsc_info;
Quoted 8 lines

xen/include/public/domctl.h struct xen_domctl { · line 1242

Quoted 2 lines
xen/include/public/domctl.h +0 -0
         struct xen_domctl_hvmcontext        hvmcontext;
         struct xen_domctl_hvmcontext_partial hvmcontext_partial;

Deletion of sub-ops, just like their modification, requires the interface version to get bumped if it wasn't already during a release cycle. I know you dislike the underlying concept, but as long as the interface version continues to exist (with its present meaning) I'm afraid it needs bumping for any backwards- incompatible change.

I can fix it on commit.  I don't waste time tracking whether it has had its conditional bump.

~Andrew

Acked-by
#3 lore raw
Quoted 27 lines · Andrew Cooper
Quoted 18 lines · Jan Beulich

xen/arch/x86/cpuid.c int init_domain_cpuid_policy(struct domain *d) · line 713

Quoted 10 lines · Andrew Cooper
xen/arch/x86/cpuid.c +2 -1
@@ -708,7 +708,8 @@ int init_domain_cpuid_policy(struct domain *d)
     if ( !p )
         return -ENOMEM;
 
-    if ( d->disable_migrate )
+    /* The hardware domain can't migrate.  Give it ITSC if available. */
+    if ( is_hardware_domain(d) )
         p->extd.itsc = cpu_has_itsc;

... why not include is_xenstore_domain() here that you remove from
...

Quoted 4 lines
xen/arch/x86/cpuid.c +0 -0
@@ -452,9 +451,6 @@ struct domain *domain_create(domid_t domid,
         watchdog_domain_init(d);
         init_status |= INIT_watchdog;
 
Quoted 7 lines

xen/arch/x86/cpuid.c struct domain *domain_create(domid_t domid, · line 456

Quoted 2 lines
xen/arch/x86/cpuid.c +0 -2
-        if ( is_xenstore_domain(d) )
-            d->disable_migrate = true;

... here? On the tool stack side the change here only deletes code, i.e. I don't see you taking care of the default enabling there either. Am I overlooking any logic that now causes the feature to be requested for the xenstore domain without you needing to add any code?

Quoted 3 lines

The toolstack (legitimately) has knowledge of nomigrate, and will even implicitly turn on ITSC as a side effect, but will also honour explicit requests to turn it on or off.

Okay, it this is the case then ...

Quoted 38 lines
Quoted 29 lines
Quoted 29 lines
xen/include/public/domctl.h +1 -7
@@ -730,11 +730,6 @@ struct xen_domctl_hvmcontext_partial {
     XEN_GUEST_HANDLE_64(uint8) buffer;  /* OUT: buffer to write record into */
 };
 
-/* XEN_DOMCTL_disable_migrate */
-struct xen_domctl_disable_migrate {
-    uint32_t disable; /* IN: 1: disable migration and restore */
-};
-
 
 /* XEN_DOMCTL_gettscinfo */
 /* XEN_DOMCTL_settscinfo */
@@ -1191,7 +1186,7 @@ struct xen_domctl {
 #define XEN_DOMCTL_gethvmcontext_partial         55
 #define XEN_DOMCTL_vm_event_op                   56
 #define XEN_DOMCTL_mem_sharing_op                57
-#define XEN_DOMCTL_disable_migrate               58
+/* #define XEN_DOMCTL_disable_migrate            58 - Obsolete */
 #define XEN_DOMCTL_gettscinfo                    59
 #define XEN_DOMCTL_settscinfo                    60
 #define XEN_DOMCTL_getpageframeinfo3             61
@@ -1242,7 +1237,6 @@ struct xen_domctl {
         struct xen_domctl_ioport_permission ioport_permission;
         struct xen_domctl_hypercall_init    hypercall_init;
         struct xen_domctl_settimeoffset     settimeoffset;
-        struct xen_domctl_disable_migrate   disable_migrate;
         struct xen_domctl_tsc_info          tsc_info;
Quoted 8 lines

xen/include/public/domctl.h struct xen_domctl { · line 1242

Quoted 2 lines
xen/include/public/domctl.h +0 -0
         struct xen_domctl_hvmcontext        hvmcontext;
         struct xen_domctl_hvmcontext_partial hvmcontext_partial;

Deletion of sub-ops, just like their modification, requires the interface version to get bumped if it wasn't already during a release cycle. I know you dislike the underlying concept, but as long as the interface version continues to exist (with its present meaning) I'm afraid it needs bumping for any backwards- incompatible change.

Quoted 2 lines

I can fix it on commit.  I don't waste time tracking whether it has had its conditional bump.

... with this taken care of

Jan

Acked-by on v1 1/1
#4 lore raw
In reply to Andrew Cooper

Andrew Cooper writes ("[PATCH] tools: Delete XEN_DOMCTL_disable_migrate"):

Quoted 4 lines

It is conceptually wrong for this information to exist in the hypervisor in the first place. Only the toolstack is capable of correctly reasoning about the non-migrateability of guests.

Quoted 8 lines

This hypercall has only ever existed to control the visibility of the Invariant TSC flag to the guest. Now that we have properly disentanged that and moved ITSC into the guests CPUID policy, delete this hypercall.

Furthermore, this fixes a corner case where Xen would override the toolstacks choice of ITSC for a xenstore stubdomain. That said, init-xenstore-domain doesn't currently ask for ITSC, and MiniOS doesn't know what ITSC is either, so nothing actually changes in practice.

For the tools parts,