Domain Groups - VMM Support

diffstat domgrps-vmm.patch

 arch/ia64/xen/xensetup.c                 |    7
 arch/powerpc/powerpc64/hypercall_table.S |    1
 arch/powerpc/setup.c                     |    7
 arch/x86/setup.c                         |    8
 arch/x86/x86_32/entry.S                  |    2
 arch/x86/x86_64/entry.S                  |    3
 common/Makefile                          |    2
 common/domain.c                          |    8
 common/domctl.c                          |   10
 common/domgrp.c                          |  315 +++++++++++++++++++++++++++++++
 common/domgrpctl.c                       |  134 +++++++++++++
 include/public/domctl.h                  |    2
 include/public/domgrpctl.h               |   86 ++++++++
 include/public/xen.h                     |    7
 include/xen/domgrp.h                     |   36 +++
 include/xen/hypercall.h                  |    5
 include/xen/sched.h                      |   26 ++

17 files changed, 657 insertions(+), 2 deletions(-)

diff -urN xen-unstable/xen/arch/ia64/xen/xensetup.c xen-unstable-trp-domgrps-rebase-tip/xen/arch/ia64/xen/xensetup.c
@@ -15,6 +15,7 @@
 #include <xen/version.h>
 #include <xen/console.h>
 #include <xen/domain.h>
+#include <xen/domgrp.h>
 #include <xen/serial.h>
 #include <xen/trace.h>
 #include <xen/keyhandler.h>

@@ -550,6 +551,10 @@
 
     expose_p2m_init();
 
+    /* initialize domain groups */
+    if (init_domain_groups())
+        panic("Error creating default groups\n");
+
     /* Create initial domain 0. */
     dom0 = domain_create(0, 0, DOM0_SSIDREF);
     if (dom0 == NULL)

@@ -559,6 +564,8 @@
         panic("Cannot allocate dom0 vcpu 0\n");
 
     dom0->is_privileged = 1;
+    if (add_dom_to_grp(dom0, 0))
+        panic("Error adding dom0 to grp0\n");
 
     /*
      * We're going to setup domain0 using the module(s) that we stashed safely

diff -urN xen-unstable/xen/arch/powerpc/powerpc64/hypercall_table.S xen-unstable-trp-domgrps-rebase-tip/xen/arch/powerpc/powerpc64/hypercall_table.S
@@ -41,6 +41,7 @@
         .quad 0 /* do_hvm_op */
         .quad do_sysctl             /* 35 */
         .quad do_domctl
+        .quad do_domgrpctl
         .rept NR_hypercalls-((.-__hypercall_table)/8)
         .quad do_ni_hypercall
         .endr

diff -urN xen-unstable/xen/arch/powerpc/setup.c xen-unstable-trp-domgrps-rebase-tip/xen/arch/powerpc/setup.c
@@ -32,6 +32,7 @@
 #include <xen/trace.h>
 #include <xen/mm.h>
 #include <xen/domain.h>
+#include <xen/domgrp.h>
 #include <xen/gdbstub.h>
 #include <xen/symbols.h>
 #include <xen/keyhandler.h>

@@ -365,6 +366,10 @@
     /* This cannot be called before secondary cpus are marked online.  */
     percpu_free_unused_areas();
 
+    /* initialize domain groups */
+    if (init_domain_groups())
+        panic("Error creating default groups\n");
+
     /* Create initial domain 0. */
     dom0 = domain_create(0, 0, DOM0_SSIDREF);
     if (dom0 == NULL)

@@ -375,6 +380,8 @@
     dom0->vcpu[0]->cpu_affinity = cpumask_of_cpu(0);
 
     dom0->is_privileged = 1;
+    if (add_dom_to_grp(dom0, 0))
+        panic("Error adding dom0 to grp0\n");
 
     /* scrub_heap_pages() requires IRQs enabled, and we're post IRQ setup... */
     local_irq_enable();

diff -urN xen-unstable/xen/arch/x86/setup.c xen-unstable-trp-domgrps-rebase-tip/xen/arch/x86/setup.c
@@ -3,6 +3,7 @@
 #include <xen/lib.h>
 #include <xen/sched.h>
 #include <xen/domain.h>
+#include <xen/domgrp.h>
 #include <xen/serial.h>
 #include <xen/softirq.h>
 #include <xen/acpi.h>

@@ -955,6 +956,10 @@
     if ( opt_watchdog ) 
         watchdog_enable();
 
+    /* initialize domain groups */
+    if (init_domain_groups())
+        panic("Error creating default groups\n");
+
     /* Create initial domain 0. */
     dom0 = domain_create(0, 0, DOM0_SSIDREF);
     if ( (dom0 == NULL) || (alloc_vcpu(dom0, 0, 0) == NULL) )

@@ -962,6 +967,9 @@
 
     dom0->is_privileged = 1;
 
+    if (add_dom_to_grp(dom0, 0))
+        panic("Error adding dom0 to grp0\n");
+
     /* Grab the DOM0 command line. */
     cmdline = (char *)(mod[0].string ? __va(mod[0].string) : NULL);
     if ( (cmdline != NULL) || (kextra != NULL) )

diff -urN xen-unstable/xen/arch/x86/x86_32/entry.S xen-unstable-trp-domgrps-rebase-tip/xen/arch/x86/x86_32/entry.S
@@ -682,6 +682,7 @@
         .long do_sysctl             /* 35 */
         .long do_domctl
         .long do_kexec_op
+        .long do_domgrpctl
         .rept NR_hypercalls-((.-hypercall_table)/4)
         .long do_ni_hypercall
         .endr

@@ -725,6 +726,7 @@
         .byte 1 /* do_sysctl            */  /* 35 */
         .byte 1 /* do_domctl            */
         .byte 2 /* do_kexec_op          */
+        .byte 1 /* do_domgrpctl         */
         .rept NR_hypercalls-(.-hypercall_args_table)
         .byte 0 /* do_ni_hypercall      */
         .endr

diff -urN xen-unstable/xen/arch/x86/x86_64/entry.S xen-unstable-trp-domgrps-rebase-tip/xen/arch/x86/x86_64/entry.S
@@ -672,6 +672,7 @@
         .quad do_sysctl             /* 35 */
         .quad do_domctl
         .quad do_kexec_op
+        .quad do_domgrpctl
         .rept NR_hypercalls-((.-hypercall_table)/8)
         .quad do_ni_hypercall
         .endr

@@ -715,7 +716,7 @@
         .byte 1 /* do_sysctl            */  /* 35 */
         .byte 1 /* do_domctl            */
         .byte 2 /* do_kexec             */
-        .byte 1 /* do_xsm_op            */
+        .byte 1 /* do_domgrpctl         */
         .rept NR_hypercalls-(.-hypercall_args_table)
         .byte 0 /* do_ni_hypercall      */
         .endr

diff -urN xen-unstable/xen/common/domain.c xen-unstable-trp-domgrps-rebase-tip/xen/common/domain.c
@@ -11,6 +11,7 @@
 #include <xen/errno.h>
 #include <xen/sched.h>
 #include <xen/domain.h>
+#include <xen/domgrp.h>
 #include <xen/mm.h>
 #include <xen/event.h>
 #include <xen/time.h>

@@ -57,6 +58,7 @@
 
     memset(d, 0, sizeof(*d));
     d->domain_id = domid;
+    d->group_id = INVAL_GROUP_ID;
 
     if ( xsm_alloc_security_domain(d) != 0 )
     {

@@ -241,6 +243,8 @@
         rcu_assign_pointer(*pd, d);
         rcu_assign_pointer(domain_hash[DOMAIN_HASH(domid)], d);
         spin_unlock(&domlist_update_lock);
+
+        add_dom_to_grp(d, NULL_GROUP_ID);
     }
 
     return d;

@@ -388,6 +392,8 @@
 {
     struct vcpu *v;
 
+    del_dom_from_grp(d);
+
     if ( d->domain_id == 0 )
         dom0_shutdown(reason);
 

@@ -522,6 +528,8 @@
     if ( _atomic_read(old) != 0 )
         return;
 
+    del_dom_from_grp(d);
+
     /* Delete from task list and task hashtable. */
     spin_lock(&domlist_update_lock);
     pd = &domain_list;

diff -urN xen-unstable/xen/common/domctl.c xen-unstable-trp-domgrps-rebase-tip/xen/common/domctl.c
@@ -12,6 +12,7 @@
 #include <xen/mm.h>
 #include <xen/sched.h>
 #include <xen/domain.h>
+#include <xen/domgrp.h>
 #include <xen/event.h>
 #include <xen/domain_page.h>
 #include <xen/trace.h>

@@ -94,8 +95,10 @@
     u64 cpu_time = 0;
     int flags = XEN_DOMINF_blocked;
     struct vcpu_runstate_info runstate;
-    
+    struct domain_group *grp;
+
     info->domain = d->domain_id;
+    info->group = d->group_id;
     info->nr_online_vcpus = 0;
     
     /* 

@@ -136,6 +139,9 @@
     info->shared_info_frame = mfn_to_gmfn(d, __pa(d->shared_info)>>PAGE_SHIFT);
 
     memcpy(info->handle, d->handle, sizeof(xen_domain_handle_t));
+    grp = find_grp_by_id(info->group);
+    if (grp)
+        memcpy(info->dg_handle, grp->handle, sizeof(xen_domain_group_handle_t));
 }
 
 static unsigned int default_vcpu0_location(void)

@@ -256,6 +262,7 @@
     {
         struct domain *d = rcu_lock_domain_by_id(op->domain);
         ret = -ESRCH;
+
         if ( d != NULL )
         {
             ret = xsm_pausedomain(d);

@@ -426,6 +433,7 @@
     {
         struct domain *d = rcu_lock_domain_by_id(op->domain);
         ret = -ESRCH;
+
         if ( d != NULL )
         {
             ret = xsm_destroydomain(d) ? : domain_kill(d);

diff -urN xen-unstable/xen/common/Makefile xen-unstable-trp-domgrps-rebase-tip/xen/common/Makefile
@@ -1,6 +1,8 @@
 obj-y += bitmap.o
 obj-y += domctl.o
+obj-y += domgrpctl.o
 obj-y += domain.o
+obj-y += domgrp.o
 obj-y += event_channel.o
 obj-y += grant_table.o
 obj-y += kernel.o

diff -urN xen-unstable/xen/include/public/domctl.h xen-unstable-trp-domgrps-rebase-tip/xen/include/public/domctl.h
@@ -67,6 +67,7 @@
 struct xen_domctl_getdomaininfo {
     /* OUT variables. */
     domid_t  domain;              /* Also echoed in domctl.domain */
+    dgid_t   group;
  /* Domain is scheduled to die. */
 #define _XEN_DOMINF_dying     0
 #define XEN_DOMINF_dying      (1U<<_XEN_DOMINF_dying)

@@ -103,6 +104,7 @@
     uint32_t max_vcpu_id;        /* Maximum VCPUID in use by this domain. */
     uint32_t ssidref;
     xen_domain_handle_t handle;
+    xen_domain_group_handle_t dg_handle;
 };
 typedef struct xen_domctl_getdomaininfo xen_domctl_getdomaininfo_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_getdomaininfo_t);

diff -urN xen-unstable/xen/include/public/xen.h xen-unstable-trp-domgrps-rebase-tip/xen/include/public/xen.h
@@ -80,6 +80,7 @@
 #define __HYPERVISOR_sysctl               35
 #define __HYPERVISOR_domctl               36
 #define __HYPERVISOR_kexec_op             37
+#define __HYPERVISOR_domgrpctl            38
 
 /* Architecture-specific hypercall definitions. */
 #define __HYPERVISOR_arch_0               48

@@ -297,6 +298,8 @@
 
 typedef uint16_t domid_t;
 
+typedef uint16_t dgid_t;
+
 /* Domain ids >= DOMID_FIRST_RESERVED cannot be used for ordinary domains. */
 #define DOMID_FIRST_RESERVED (0x7FF0U)
 

@@ -520,7 +523,9 @@
     unsigned long nr_pt_frames; /* Number of bootstrap p.t. frames.       */
     unsigned long mfn_list;     /* VIRTUAL address of page-frame list.    */
     unsigned long mod_start;    /* VIRTUAL address of pre-loaded module.  */
+                                /* or VIRTUAL address of module_t array   */
     unsigned long mod_len;      /* Size (bytes) of pre-loaded module.     */
+                                /* or length of module_t array.           */
     int8_t cmd_line[MAX_GUEST_CMDLINE];
 };
 typedef struct start_info start_info_t;

@@ -579,6 +584,8 @@
 
 typedef uint8_t xen_domain_handle_t[16];
 
+typedef uint8_t xen_domain_group_handle_t[16];
+
 /* Turn a plain number into a C unsigned long constant. */
 #define __mk_unsigned_long(x) x ## UL
 #define mk_unsigned_long(x) __mk_unsigned_long(x)

diff -urN xen-unstable/xen/include/xen/domgrp.h xen-unstable-trp-domgrps-rebase-tip/xen/include/xen/domgrp.h
@@ -0,0 +1,36 @@
+/******************************************************************************
+ * domgrp.h
+ *
+ * Generic domain group-handling functions.
+ *
+ * Author: Chris Bookholt ([email protected])
+ */
+
+#ifndef __XEN_DOM_GROUP_H__
+#define __XEN_DOM_GROUP_H__
+
+#include <public/domgrpctl.h>
+
+extern struct list_head domgrplist;
+
+void get_grp_info(struct domain_group *grp, xen_domgrpctl_getgrpinfo_t * info);
+
+struct domain_group *find_grp_by_id(dgid_t dgid);
+
+uint16_t get_new_group_id(void);
+
+struct domain_group *domain_group_create(dgid_t dgid);
+
+int del_dom_from_grp(struct domain *old_dom);
+
+int add_dom_to_grp(struct domain *dom, dgid_t dgid);
+
+int pause_grp(dgid_t dgid);
+
+int unpause_grp(dgid_t dgid);
+
+int domain_group_destroy(dgid_t dgid);
+
+int init_domain_groups(void);
+
+#endif				/* __XEN_DOM_GROUP_H__ */

diff -urN xen-unstable/xen/include/xen/hypercall.h xen-unstable-trp-domgrps-rebase-tip/xen/include/xen/hypercall.h
@@ -10,6 +10,7 @@
 #include <xen/time.h>
 #include <public/xen.h>
 #include <public/domctl.h>
+#include <public/domgrpctl.h>
 #include <public/sysctl.h>
 #include <public/platform.h>
 #include <public/event_channel.h>

@@ -35,6 +36,10 @@
     XEN_GUEST_HANDLE(xen_domctl_t) u_domctl);
 
 extern long
+do_domgrpctl(
+    XEN_GUEST_HANDLE(xen_domgrpctl_t) u_domgrpctl);
+
+extern long
 do_sysctl(
     XEN_GUEST_HANDLE(xen_sysctl_t) u_sysctl);
 

diff -urN xen-unstable/xen/include/xen/sched.h xen-unstable-trp-domgrps-rebase-tip/xen/include/xen/sched.h
@@ -9,6 +9,7 @@
 #include <xen/shared.h>
 #include <public/xen.h>
 #include <public/domctl.h>
+#include <public/domgrpctl.h>
 #include <public/vcpu.h>
 #include <public/xsm/acm.h>
 #include <xen/time.h>

@@ -19,6 +20,7 @@
 #include <xen/xenoprof.h>
 #include <xen/rcupdate.h>
 #include <xen/irq.h>
+#include <xen/init.h>
 
 #ifdef CONFIG_COMPAT
 #include <compat/vcpu.h>

@@ -27,6 +29,9 @@
 
 extern unsigned long volatile jiffies;
 
+extern spinlock_t domgrplist_update_lock;
+extern rcu_read_lock_t domgrplist_read_lock;
+
 /* A global pointer to the initial domain (DOM0). */
 extern struct domain *dom0;
 

@@ -145,9 +150,13 @@
 {
     domid_t          domain_id;
 
+    dgid_t           group_id;        /* TODO: replace struct group ptr */
+    struct list_head member;
+
     shared_info_t   *shared_info;     /* shared data area */
 
     spinlock_t       big_lock;
+    spinlock_t      *grp_big_lock;
 
     spinlock_t       page_alloc_lock; /* protects all the following fields  */
     struct list_head page_list;       /* linked list, of size tot_pages     */

@@ -233,6 +242,23 @@
     spinlock_t hypercall_deadlock_mutex;
 };
 
+struct domain_group
+{
+    dgid_t                       group_id;
+    uint16_t                     size;
+
+    struct list_head             groups;
+
+    struct list_head             member_list;
+
+    spinlock_t                   grp_big_lock;
+
+    spinlock_t                   member_list_update_lock;
+    rcu_read_lock_t              member_list_read_lock;
+
+    xen_domain_group_handle_t    handle;
+};
+
 struct domain_setup_info
 {
     /* Initialised by caller. */

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Hello,

Samuel,

You're right, those comments snuck in from another project. An interest of our team is domain decomposition. Members of the team created a domain builder separate from dom0 (conceptually similar to Derek Murray's recent project based on Rolf Neugebauer's "initial domains" patch) that accepts grub modules as input. So, your suspicion is correct that the comments don't belong in this patch and are related to modifications used to accommodate several modules and command line arguments.

Thanks for catching my mistake. I also welcome such an attentive critique of the proposed grouping functionality. :)

Cheers,
Chris

Samuel,

You're right, those comments snuck in from another project. An interest of our team is domain decomposition. Members of the team created a domain builder separate from dom0 (conceptually similar to Derek Murray's recent project based on Rolf Neugebauer's "initial domains" patch) that accepts grub modules as input. So, your suspicion is correct that the comments don't belong in this patch and are related to modifications used to accommodate several modules and command line arguments.

Thanks for catching my mistake. I also welcome such an attentive critique of the proposed grouping functionality. :)

Cheers,
Chris

Hello,

Actually I thought I had already answered this, but I can't find a trace...

The patch looked sane to me, but maybe it would be useful to see it merged with the other group patch proposed by Mike D. Day. I guess it is mostly a matter of plumbing both together.

Samuel

I think there was some mischief from the xen-devel list server late last week. Please disregard my duplicated response, assuming your question was answered.

I'm in the process of updating my domgrps patches to facilitate scheduling information for compatibility with Mike's schegrps. I'll certainly post the revisions once completed.

Cheers,
Chris

A magic argument ("multimodules") is appended to the VMM boot parameters, which signals that the mod_start and mod_len variables are to be interpreted in the context of multiple grub modules.

At the moment the changes are not suitable for submission primarily because they're applicable to a stale release of xen. I've asked the folks responsible to work something up for general consumption. We'll test it internally and then submit it for public discussion.

Cheers,
Chris

I have no objection to making domgrpctl become a set of sub-ops to the existing domain management hypercall. In fact, with the predecessor to domctl (dom0_op) that's how it was implemented. During the domctl developmental churn I separated them to make my life easier, but now it's just an artifact.

I agree. There's more to groups than what the two of us have released thus far and adding new features should be made as painless as possible. My suggestion is to save this type of overhaul until we settle on an agreeable functionality merge.

I'm also having a go at the merge and will release a patch set RFC hopefully well before leaving on travel at the end of next week.

With regard to the merge, I've got a question for you. In schedgrps, when a domain joins a group the assumption is that credits are thereafter shared among group members (for as long as it's a member). I think there are cases where it makes sense to group domains without implicitly sharing credits (the null-group in domgrps is such a case). So, I'd like to make it optional for members to delegate their VCPUs to the group. The question is, do you have an opinion whether domains should have to opt-in or opt-out of sharing their credits when joining a group?

Cheers,
Chris

Hi Mike,

Here's a rough cut of the domgrps/schedgrps merge. It includes a slightly revised version of domgrps with some enhancements for performance and compatibility with schedgrps. I'd appreciate your feedback to make sure the merged result accurately represents your intent.

DomGrps changes:
- add per-domgrp scheduler-specific data

SchedDom Changes:
- when each domgrp is created, make a csched_dom struct to act as the group representative
- when doms join the group, delegate its VCPUs to its representative (instead of a master dom)
- removed group management functions, changes to tools, and command line interface

As of right now, when a domain joins a group it must relinquish its VCPU(s) to the group. In the near future I'd like to add the following:
- configuration option choose whether a dom will relinquish its VCPUs to the group
- modify xm to show whether members have relinquished their VCPUs
- documentation :)

I haven't done any benchmarks, but on the fast path the extra code should be nearly identical to your original schedgrps implementation, so performance should be the very similar. Other than the merge itself, the practical benefit is that schegrps are now persistent across reboots and host migration (although I haven't tested the latter).

The attached patches should apply cleanly to xen-unstable.hg cs 16975. The intended patch order is: domgrps-vmm.patch, schedgrps.patch, domgrps-tools.patch.

Cheers,
Chris

Hmmm... again the generic domgrp stuff seems like a huge amount of code with few moving parts that would affect user experience of Xen. The original schedgrps code by comparison is tiny. I don't mind taking big gobs of code if there are genuine use cases, but I won't big pieces of 'architecture' without good reason. I'm inclined to take Mike's patches roughly as he previously posted them.

-- Keir

Use cases for domain groups that exist today:
- domain management (e.g. group migration)
- scheduler integration (Mike's schedgrps)

Use cases coming down the pike:
- XSM integration (e.g. express policy in terms of groups)
- domain relationships (e.g. failure handling)
- resource management (e.g. allocating a resource to a set of domains)
- set_target/IS_PRIV_FOR integration (e.g. to establish privilege over a set of domains)

These use cases stem from where we see domain decomposition headed and the challenges it creates.

Can some notion of groups can be implemented in each case without an underlying VMM-backed, non-hierarchical group architecture? You bet. However, each implementation is likely to have a separate management interface and impose (possibly conflicting) hierarchies between domains. For such independent components to take advantage of one another, each would have to translate their respective notions of groups, assuming such a translation exists.

On the issue of code size, take Mike's schedgrps for example, which was very small as originally posted. After integration with domgrps, it shrank to less than 40% of its original size (259 insertions down from 681) and it no longer induced a domain hierarchy.

Could domgrps be smaller? Yes, and thanks to Mike's recent suggestions, I have some concrete ways to make that happen. If you have some specific areas where you think domgrps is unnecessarily large, I'm willing to make changes. Another way to look at the code size issue is that the integration with schedgrps shaved off well over over 400 LOC, mostly by removing redundant management code from schedgrps. At that rate it would take two, maybe three more group-aware projects for the domgrps architecture to break even in terms of code size.

But it sounds like the main objection is lack of existing use cases. They're coming... slowly. The best I can say is that I'm working to identify and mitigate future challenges before they cause problems. Is there critical mass for a generic group architecture yet? I think so, but the case should only get stronger with time.

Cheers,
Chris

If credit-sharing is made configurable (as you would surely want it to be if domgrps are to have other uses) then a reasonable number of those lines of code will reappear, and spread across tools and hypervisor.

I'm driven by concrete use cases. Several of the upcoming uses you mention need careful consideration of what they are useful for, to determine the best way to design them into the system. Take resource sharing. Stub domains sharing scheduler credits with the HVM guest is a rather special case, and one where a master/slave relationship is not unreasonable (and hence in this case I think it is arguable whether it is actually a good fit with domgrps after all). If you really want resource sharing amongst a peer group of domains, what does that actually mean? Does each domain have a private allocation of resource plus membership of one or more pools? If so, which resource set satisfies each resource request by a particular domain? Even if not, how is resource contention between domains belonging to the resource pool (e.g., for the static assignment of memory pages to that resource pool) managed and resolved? The obvious place would be the dom0 control plane -- in which case these resource pools can probably be implemented as a high-level tools abstraction with no direct hypervisor involvement at all.

My fear is that a nice domgrps infrastructure doesn't itself resolve the hard devils in the details, for all that it sounds architecturally neat.

-- Keir

Actually, in my former research team in Bordeaux, they would like to write a small domain that computes the scheduling of a bunch of others, for parallel scientific computing.

Samuel

Well that sounds plausible, but I'm not sure how resource pools or domgrps would help with that. It sounds like a delegation mechanism (of privilege and/or resource) would be more appropriate.

-- Keir

Sure, delegation will be needed, but there needs to be a way to specify which domains that scheduler has control on.

Samuel

It's not clear to me that supporting any more than a delegation relation across pairs of domains is needed at the hypervisor level. Nor whether a richer grouping mechanism in this case would lead to greater run-time efficiency, fewer lines of code, or clearer code.

-- Keir

Without testing, I'd say that making the domgrps/schedgrps merge provide logically equivalent functionality to the original schedgrps requires the addition of 1 conditional statement in the VMM:

diff -r 82d60475e726 xen/common/domgrp.c
@@ -215,7 +215,8 @@ int add_dom_to_grp(struct domain *dom, d
        grp->size++;
 
        /* add dom to schedgrp */
-       sched_join_group(dom); // FIXME: should be conditional/optional
+       if (dgid == NULL_GROUP_ID)
+               sched_join_group(dom);
       out:
        return ret;
 }

In the domgrps model, all ungrouped domains are in the "null group." And in schedgrps model, all grouped domains automatically relinquish their VCPUs. So by preventing domains in the null group from relinquishing their VCPUs, the above change make the two implementations effectively equivalent: only explicitly grouped domains relinquish their VCPUs.

Additional configuration options will be desirable and will take code, but in this case domgrps hasn't created any new problems.

I agree. It's not unreasonable, but it's also not strictly necessary. I would like to avoid any mechanism that induces a domain hierarchy without need.

It might be that, in the case of resource management, the VMM doesn't have to be involved, but the tool chain would benefit from a common configuration system and API to converse about groups of domains.

You're right, it definitely doesn't make the devils go away. My goal is more modest. I'm advocating for a common framework where tools and the VMM can use group information in a way that's relationship agnostic.

-Chris