Skip to contentSkip to conversation

Xen Share: Simplified I/O Mechanism

PATCH In review Rusty Russell 4 participants 19 messages Jun 6, 2006 – Jun 14, 2006 Arm/actions semi architecture

lore
Share & tools
lore
#0 lore raw

PATCH 1/9 Xen Share: Simplified I/O Mechanism

I think this is ready for experimental inclusion. A number of people have asked about it, and I think it would benefit from exposure to bright minds. It provides additional mechanisms, and need not replace existing ones.

My main concern is that having the hypervisor decrement an arbitrary address on notification is wonderful for driver authors, but requires the x86 hypervisor to keep mappings: restricting addresses to within the shared pages would be easier for the hypervisor...

Feedback encouraged!
Rusty.
---
Subject: Xen share core

This introduces a page "share" mechanism to xen: an alternative to both cross-domain binding of event channels, and grant tables.

Dom0 can create sharable pages, which returns a handle. It can then grant access permission to other domains. Any domain which has permission can request access using that handle, which binds an event channel to that share and returns a unique peerid for that domain:
this is useful for arbitration on multi-way shared pages ("you are user #3").

A watch & trigger mechanism creates a simple event mechanism: a watch on an arbitrary "watch number" associated with the share causes the hypervisor to decrement an address when a trigger is performed on that watch number: if the location is decremented to zero, an event channel is raised.

Finally, a scatter-gather list mechanism allows the domains to associate their pages with arbitrary queue numbers in the shared region, to transport bulk data (effectively by having the hypervisor do "DMA" between domains).

The patch includes an abstraction layer so architectures which don't want virtual or machine addresses from the kernel can change that, and also so that architectures can allocate the sharable pages as they wish.

xen/arch/x86/Makefile +1 -0
diff -r d5f98d23427a xen/arch/x86/Makefile
@@ -30,6 +30,7 @@
 obj-y += physdev.o
 obj-y += rwlock.o
 obj-y += setup.o
+obj-y += share.o
 obj-y += shutdown.o
 obj-y += smp.o
 obj-y += smpboot.o
xen/arch/x86/x86_32/entry.S +2 -0
diff -r d5f98d23427a xen/arch/x86/x86_32/entry.S
@@ -648,6 +648,7 @@
         .long do_xenoprof_op
         .long do_event_channel_op
         .long do_physdev_op
+        .long do_share_op
         .rept NR_hypercalls-((.-hypercall_table)/4)
         .long do_ni_hypercall
         .endr
@@ -687,6 +688,7 @@
         .byte 2 /* do_xenoprof_op       */
         .byte 2 /* do_event_channel_op  */
         .byte 2 /* do_physdev_op        */
+        .byte 5 /* do_share_op          */
         .rept NR_hypercalls-(.-hypercall_args_table)
         .byte 0 /* do_ni_hypercall      */
         .endr
xen/common/Makefile +1 -0
diff -r d5f98d23427a xen/common/Makefile
@@ -16,6 +16,7 @@
 obj-y += sched_credit.o
 obj-y += sched_sedf.o
 obj-y += schedule.o
+obj-y += share.o
 obj-y += softirq.o
 obj-y += string.o
 obj-y += symbols.o
xen/common/dom0_ops.c +22 -0
diff -r d5f98d23427a xen/common/dom0_ops.c
@@ -16,6 +16,7 @@
 #include <xen/domain_page.h>
 #include <xen/trace.h>
 #include <xen/console.h>
+#include <xen/share.h>
 #include <xen/iocap.h>
 #include <xen/guest_access.h>
 #include <asm/current.h>
@@ -634,6 +635,27 @@
         }
     }
     break;
+    case DOM0_CREATESHAREDPAGES:
+    {
+        ret = create_shared_pages(op->u.createsharedpages.num);
+    }
+    break;
+    case DOM0_DESTROYSHAREDPAGES:
+    {
+        ret = destroy_shared_pages(op->u.destroysharedpages.share_ref);
+    }
+    break;
+    case DOM0_GRANTSHAREDPAGES:
+    {
+        struct domain *d; 
+        ret = -ESRCH;
+        d = find_domain_by_id(op->u.grantsharedpages.domain);
+        if ( d != NULL )
+        {
+            ret = grant_shared_pages(op->u.grantsharedpages.share_ref, d);
+        }
+    }
+    break;
 
     case DOM0_IRQ_PERMISSION:
     {
xen/common/domain.c +2 -0
diff -r d5f98d23427a xen/common/domain.c
@@ -16,6 +16,7 @@
 #include <xen/console.h>
 #include <xen/softirq.h>
 #include <xen/domain_page.h>
+#include <xen/share.h>
 #include <xen/rangeset.h>
 #include <xen/guest_access.h>
 #include <xen/hypercall.h>
@@ -304,6 +305,7 @@
     grant_table_destroy(d);
 
     arch_domain_destroy(d);
+    free_shares(d);
 
     free_domain(d);
 
xen/include/asm-x86/mm.h +44 -0

53 lines not on this page · raw message

xen/include/public/dom0_ops.h +25 -0
diff -r d5f98d23427a xen/include/public/dom0_ops.h
@@ -513,6 +513,28 @@
 };
 typedef struct dom0_hypercall_init dom0_hypercall_init_t;
 DEFINE_XEN_GUEST_HANDLE(dom0_hypercall_init_t);
+
+#define DOM0_CREATESHAREDPAGES 49
+struct dom0_createsharedpages {
+    uint32_t num;
+};
+typedef struct dom0_createsharedpages dom0_createsharedpages_t;
+DEFINE_XEN_GUEST_HANDLE(dom0_createsharedpages_t);
+
+#define DOM0_GRANTSHAREDPAGES 50
+struct dom0_grantsharedpages {
+    unsigned long share_ref;
+    domid_t domain;
+};
+typedef struct dom0_grantsharedpages dom0_grantsharedpages_t;
+DEFINE_XEN_GUEST_HANDLE(dom0_grantsharedpages_t);
+
+#define DOM0_DESTROYSHAREDPAGES 51
+struct dom0_destroysharedpages {
+    unsigned long share_ref;
+};
+typedef struct dom0_destroysharedpages dom0_destroysharedpages_t;
+DEFINE_XEN_GUEST_HANDLE(dom0_destroysharedpages_t);
 
 struct dom0_op {
     uint32_t cmd;
@@ -555,6 +577,9 @@
         struct dom0_irq_permission    irq_permission;
         struct dom0_iomem_permission  iomem_permission;
         struct dom0_hypercall_init    hypercall_init;
+        struct dom0_createsharedpages createsharedpages;
+        struct dom0_grantsharedpages  grantsharedpages;
+        struct dom0_destroysharedpages destroysharedpages;
         uint8_t                       pad[128];
     } u;
 };
xen/include/public/xen.h +1 -0
diff -r d5f98d23427a xen/include/public/xen.h
@@ -64,6 +64,7 @@
 #define __HYPERVISOR_xenoprof_op          31
 #define __HYPERVISOR_event_channel_op     32
 #define __HYPERVISOR_physdev_op           33
+#define __HYPERVISOR_share_op             33
 
 /* Architecture-specific hypercall definitions. */
 #define __HYPERVISOR_arch_0               48
xen/arch/x86/share.c new file +36 -0
diff -r d5f98d23427a xen/arch/x86/share.c
@@ -0,0 +1,36 @@
+#include <xen/share.h>
+#include <xen/mm.h>
+#include <asm/share.h>
+
+struct page_info *arch_alloc_shared_pages(unsigned int order, share_ref_t *ref)
+{
+	struct page_info *page;
+	void *addr;
+	int i;
+
+	/* x86 uses normal xen heap pages to share. */
+	addr = alloc_xenheap_pages(order);
+	if (!addr)
+		return NULL;
+
+	for(i=0;i<(1<<order);i++) {
+		clear_page(addr+i*PAGE_SIZE);
+		page = virt_to_page(addr+i*PAGE_SIZE);
+		page_set_owner(page, NULL);
+		/* Domain pointer must be visible before updating refcnt. */
+		wmb();
+		page->count_info = PGC_allocated|1;
+		page->u.inuse.type_info = PGT_writable_page|PGT_validated;
+		BUG_ON(page->u.inuse._domain);
+	}
+
+	/* x86 simply uses page frame numbers as share_refs. */
+	page = virt_to_page(addr);
+	*ref = page_to_mfn(page);
+	return page;
+}
+
+void arch_free_shared_pages(struct page_info *page, unsigned int order)
+{
+	free_xenheap_pages(page_to_virt(page), order);
+}
xen/common/share.c new file +926 -0

926 lines not on this page · raw message

xen/include/asm-x86/share.h new file +7 -0
diff -r d5f98d23427a xen/include/asm-x86/share.h
@@ -0,0 +1,7 @@
+#ifndef __XEN_ASM_SHARE_H
+#define __XEN_ASM_SHARE_H
+
+struct page_info *arch_alloc_shared_pages(unsigned int order, share_ref_t *ref);
+void arch_free_shared_pages(struct page_info *pfn, unsigned int order);
+
+#endif	/* __XEN_ASM_SHARE_H */
xen/include/public/share.h new file +48 -0

48 lines not on this page · raw message

xen/include/xen/share.h new file +13 -0
diff -r d5f98d23427a xen/include/xen/share.h
@@ -0,0 +1,13 @@
+#ifndef _XEN_SHARE_H
+#define _XEN_SHARE_H
+#include <public/share.h>
+
+struct domain;
+/* DOM0 ops */
+share_ref_t create_shared_pages(unsigned int num);
+int grant_shared_pages(share_ref_t share_ref, struct domain *d);
+int destroy_shared_pages(share_ref_t share_ref);
+
+/* Domain is dying, release shares */
+void free_shares(struct domain *domain);
+#endif
Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#1 lore raw

PATCH 2/9 Linux kernel infrastructure for Xen Share access

On the Linux kernel side, we provide some wrappers for accessing shared pages. They are currently reference-counted, because a future patch allows userspace to access shared pages, and the Xen interface will refuse the second request for access by the same domain.

The entire hypercall interface is arch-wrapped, which is probably overkill, but I wasn't entirely sure of the needs of non-x86 architectures. Some of this should almost certainly be in common code.

linux-2.6-xen-sparse/include/asm-i386/mach-xen/asm/share.h new file +62 -0
diff -r 6d476981e3a5 -r 07a00d96357d linux-2.6-xen-sparse/include/asm-i386/mach-xen/asm/share.h
@@ -0,0 +1,62 @@
+#ifndef __ASM_XEN_I386_SHARE_H
+#define __ASM_XEN_I386_SHARE_H
+#include <linux/types.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <xen/interface/share.h>
+
+struct xen_share
+{
+	struct list_head list;
+	atomic_t use;
+	share_ref_t share_ref;
+	unsigned num_pages;
+	void *addr;
+	int event_channel;
+	int peerid;
+	int irq;
+	struct list_head handlers;
+};
+
+struct xen_share_handler
+{
+	struct list_head list;
+	void (*handler)(struct xen_share_handler *h);
+};
+
+/* Map a shared area.  Returns PTR_ERR(errno) on fail. */
+struct xen_share *xen_share_get(share_ref_t share_ref, unsigned pages);
+
+/* Set up handler for events. */
+void xen_share_add_handler(struct xen_share *s, struct xen_share_handler *h);
+
+/* Remove handler. */
+void xen_share_remove_handler(struct xen_share *s,
+			      struct xen_share_handler *h);
+
+/* Unmap a shared area (irq unbound if not done already). */
+void xen_share_put(struct xen_share *share);
+
+/* Register this sg list (physical kernel addresses).  Returns 0 on success. */
+int xen_sg_register(struct xen_share *share, int dirmask, u32 queue, u32 *lenp,
+		    unsigned int num_sgs, const struct xen_sg sg[]);
+
+/* Unregister this sg list: give first phys address of sg. */
+void xen_sg_unregister(struct xen_share *share, unsigned long sgaddr);
+
+/* Transfer this sg list (physical kernel addresses).  Returns len xferred. */
+int xen_sg_xfer(struct xen_share *share, u32 queue, int dir,
+		unsigned int num_sgs, const struct xen_sg sg[]);
+
+/* Place watch on this trigger.  Returns 0 on success. */
+int xen_share_watch(struct xen_share *share, int triggernum, u32 *resultp);
+
+/* Remove watch on this trigger. */
+void xen_share_unwatch(struct xen_share *share, int triggernum);
+
+/* Trigger a watch.  Returns num watching on success. */
+int xen_share_trigger(struct xen_share *share, int triggernum);
+
+/* Map a share into a vma (for userspace mmap). */
+int xen_share_map(struct xen_share *share, struct vm_area_struct *vma);
+#endif	/* __ASM_XEN_I386_SHARE_H */
linux-2.6-xen-sparse/arch/i386/kernel/share-xen.c new file +280 -0

280 lines not on this page · raw message

linux-2.6-xen-sparse/arch/i386/kernel/Makefile +1 -0
diff -r 6d476981e3a5 -r 07a00d96357d linux-2.6-xen-sparse/arch/i386/kernel/Makefile
@@ -88,6 +88,7 @@
 include $(srctree)/scripts/Makefile.xen
 
 obj-y += fixup.o
+obj-y += share-xen.o
 microcode-$(subst m,y,$(CONFIG_MICROCODE)) := microcode-xen.o
 n-obj-xen := i8259.o timers/ reboot.o smpboot.o trampoline.o
 
linux-2.6-xen-sparse/arch/i386/mm/ioremap-xen.c +3 -0
diff -r 6d476981e3a5 -r 07a00d96357d linux-2.6-xen-sparse/arch/i386/mm/ioremap-xen.c
@@ -123,8 +123,11 @@
 	/* Same as remap_pfn_range(). */
 	vma->vm_flags |= VM_IO | VM_RESERVED;
 
+	/* FIXME: xenshare needs to pass DOMID_SELF. Check it's save to remove
+	 * the check.
 	if (domid == DOMID_SELF)
 		return -EINVAL;
+	*/
 
 	return __direct_remap_pfn_range(
 		vma->vm_mm, address, mfn, size, prot, domid);
linux-2.6-xen-sparse/include/asm-i386/mach-xen/asm/hypercall.h +6 -0
diff -r 6d476981e3a5 -r 07a00d96357d linux-2.6-xen-sparse/include/asm-i386/mach-xen/asm/hypercall.h
@@ -359,5 +359,11 @@
 	return _hypercall2(int, xenoprof_op, op, arg);
 }
 
+static inline long
+HYPERVISOR_share(
+	int op, long arg1, long arg2, long arg3, long arg4)
+{
+	return _hypercall5(long, share_op, op, arg1, arg2, arg3, arg4);
+}
 
 #endif /* __HYPERCALL_H__ */
patches/linux-2.6.12/get_vm_area.patch new file +9 -0
diff -r 6d476981e3a5 -r 07a00d96357d patches/linux-2.6.12/get_vm_area.patch
@@ -0,0 +1,9 @@
+diff -Naur linux-2.6.12/mm/vmalloc.c linux-2.6.12.post/mm/vmalloc.c
+--- linux-2.6.12/mm/vmalloc.c    2005-06-18 05:48:29.000000000 +1000
++++ linux-2.6.12.post/mm/vmalloc.c        2006-01-10 16:56:36.000000000 +1100
+@@ -247,6 +247,7 @@
+ {
+        return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END);
+ }
++EXPORT_SYMBOL(get_vm_area);
+
Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#2 lore raw

Re: [PATCH 2/9] Linux kernel infrastructure for Xen Share access

Quoted 8 lines · Rusty Russell

The entire hypercall interface is arch-wrapped, which is probably overkill, but I wasn't entirely sure of the needs of non-x86 architectures. Some of this should almost certainly be in common code.

diff -r 6d476981e3a5 -r 07a00d96357d linux-2.6-xen-sparse/include/asm-i386/mach-xen/asm/share.h

+struct xen_share *create_share(share_ref_t share_ref, unsigned pages)

<snip>

linux-2.6-xen-sparse/arch/i386/kernel/share-xen.c line 124

Quoted 5 lines
linux-2.6-xen-sparse/arch/i386/kernel/share-xen.c new file +5 -0
+
+	prot = __pgprot(_PAGE_PRESENT|_PAGE_RW|_PAGE_DIRTY|_PAGE_ACCESSED);
+	err = direct_kernel_remap_pfn_range((unsigned long)vma->addr,
+					    share_ref, pages * PAGE_SIZE,
+					    prot, DOMID_SELF);

Using share_ref as the 2nd parameter to remap_pfn_range it becomes clear that share_ref is really a page frame number. This is also made clear in
xen/arch/x86/share.c.

My question is: is it useful to abstract the share reference when it is always going to be a page number? Is there any architecture where it wouldn't be feasible to refer to a share by page number?

Really this is just a readability issue.

Mike

#3 lore raw

PATCH 3/9 privcmd interface addition to support share operations from Dom0 userspace

In reply to Rusty Russell

Subject: privcmd interface addition to support share operations from Dom0 userspace

We create to simple privcmd ops to create and grant access to shares.

linux-2.6-xen-sparse/drivers/xen/privcmd/privcmd.c +1 -0
diff -r 6d476981e3a5 -r 07a00d96357d linux-2.6-xen-sparse/drivers/xen/privcmd/privcmd.c
@@ -268,6 +268,7 @@
 	set_bit(__HYPERVISOR_xen_version,      hypercall_permission_map);
 	set_bit(__HYPERVISOR_sched_op,         hypercall_permission_map);
 	set_bit(__HYPERVISOR_sched_op_compat,  hypercall_permission_map);
+	set_bit(__HYPERVISOR_share_op,         hypercall_permission_map);
 	set_bit(__HYPERVISOR_event_channel_op_compat,
 		hypercall_permission_map);
 
tools/libxc/xc_domain.c +19 -0
diff -r 6d476981e3a5 -r 07a00d96357d tools/libxc/xc_domain.c
@@ -482,6 +482,25 @@
     return rc;
 
 }
+
+share_ref_t xc_create_shared_pages(int xc_handle, unsigned int num_pages)
+{
+    DECLARE_DOM0_OP;
+
+    op.cmd = DOM0_CREATESHAREDPAGES;
+    op.u.createsharedpages.num = num_pages;
+    return do_dom0_op(xc_handle, &op);
+}
+
+int xc_grant_shared_pages(int xc_handle, domid_t domid, share_ref_t share_ref)
+{
+    DECLARE_DOM0_OP;
+
+    op.cmd = DOM0_GRANTSHAREDPAGES;
+    op.u.grantsharedpages.domain = domid;
+    op.u.grantsharedpages.share_ref = share_ref;
+    return do_dom0_op(xc_handle, &op);
+}    
 
 int xc_domain_irq_permission(int xc_handle,
                              uint32_t domid,
tools/libxc/xenctrl.h +5 -0
diff -r 6d476981e3a5 -r 07a00d96357d tools/libxc/xenctrl.h
@@ -21,6 +21,7 @@
 #include <xen/memory.h>
 #include <xen/acm.h>
 #include <xen/acm_ops.h>
+#include <xen/share.h>
 
 #ifdef __ia64__
 #define XC_PAGE_SHIFT           14
@@ -584,6 +585,10 @@
 
 int xc_version(int xc_handle, int cmd, void *arg);
 
+/* Create & add permissions to sharable pages. */
+share_ref_t xc_create_shared_pages(int xc_handle, unsigned int num_pages);
+int xc_grant_shared_pages(int xc_handle, domid_t domid, share_ref_t share_ref);
+
 /*
  * MMU updates.
  */
Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#4 lore raw

PATCH 4/9 /dev/xenshare for accessing/mapping shared pages from userspace

In reply to Rusty Russell

To manipulate shared pages from userspace, we use a simple device. Userspace can gain access to a share by handle, mmap it, place a watch, trigger them, and do scatter-gather transfers.

FIXME: Should use vm_insert_page these days.

linux-2.6-xen-sparse/drivers/xen/Makefile +1 -0
diff -r 125c7cd65739 linux-2.6-xen-sparse/drivers/xen/Makefile
@@ -8,6 +8,7 @@
 obj-y	+= balloon/
 obj-y	+= privcmd/
 obj-y	+= xenbus/
+obj-y	+= xenshare.o
 
 obj-$(CONFIG_XEN_BLKDEV_BACKEND)	+= blkback/
 obj-$(CONFIG_XEN_NETDEV_BACKEND)	+= netback/
linux-2.6-xen-sparse/drivers/xen/xenshare.c new file +365 -0

365 lines not on this page · raw message

linux-2.6-xen-sparse/include/xen/public/xenshare.h new file +77 -0

77 lines not on this page · raw message

Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#5 lore raw

PATCH 5/9 Vdevice share in start_info

In reply to Rusty Russell

This patch adds a single entry on the end of the start_info, to provide a simplified bus called "vdevice". This is completely backwards compatible: old kernels will ignore it, old tools set it to zero.

The vdevice bus is a simple array of structures: now that a share reference is sufficient to describe devices, and devices no longer map other domains' pages, this is all we need. The page is initially populated with devices (although currently it's always empty when a domain starts).

Devices are added by setting a zero entry, and triggering watch number
1. They are acknowledged by setting bits in the appropriate entry and triggering watch 0. Devices are removed the same way.

This majority of this patch is plumbing it into the tools.

xen/include/public/xen.h +1 -0
diff -r cddc595b285b xen/include/public/xen.h
@@ -455,6 +455,7 @@
     unsigned long mod_start;    /* VIRTUAL address of pre-loaded module.  */
     unsigned long mod_len;      /* Size (bytes) of pre-loaded module.     */
     int8_t cmd_line[MAX_GUEST_CMDLINE];
+    unsigned long vdevice_share; /* share_ref of vdevice config page. */
 };
 typedef struct start_info start_info_t;
 
tools/libxc/xc_linux_build.c +16 -9
diff -r cddc595b285b tools/libxc/xc_linux_build.c
@@ -457,7 +457,8 @@
                        unsigned long flags,
                        unsigned int store_evtchn, unsigned long *store_mfn,
                        unsigned int console_evtchn, unsigned long *console_mfn,
-                       uint32_t required_features[XENFEAT_NR_SUBMAPS])
+                       uint32_t required_features[XENFEAT_NR_SUBMAPS],
+                       unsigned long vdevice_share)
 {
     unsigned long *page_array = NULL;
     struct load_funcs load_funcs;
@@ -549,6 +550,7 @@
     start_info->store_evtchn = store_evtchn;
     start_info->console_mfn   = nr_pages - 1;
     start_info->console_evtchn = console_evtchn;
+    start_info->vdevice_share  = vdevice_share;
     start_info->nr_pages       = nr_pages; // FIXME?: nr_pages - 2 ????
     if ( initrd->len != 0 )
     {
@@ -612,7 +614,8 @@
                        unsigned long flags,
                        unsigned int store_evtchn, unsigned long *store_mfn,
                        unsigned int console_evtchn, unsigned long *console_mfn,
-                       uint32_t required_features[XENFEAT_NR_SUBMAPS])
+                       uint32_t required_features[XENFEAT_NR_SUBMAPS],
+                       unsigned long vdevice_share)
 {
     unsigned long *page_array = NULL;
     unsigned long count, i, hypercall_pfn;
@@ -977,6 +980,7 @@
     start_info->store_evtchn = store_evtchn;
     start_info->console_mfn   = guest_console_mfn;
     start_info->console_evtchn = console_evtchn;
+    start_info->vdevice_share  = vdevice_share;
     if ( initrd->len != 0 )
     {
         start_info->mod_start    = vinitrd_start;
@@ -1044,7 +1048,8 @@
                                    unsigned int store_evtchn,
                                    unsigned long *store_mfn,
                                    unsigned int console_evtchn,
-                                   unsigned long *console_mfn)
+                                   unsigned long *console_mfn,
+                                   unsigned long vdevice_share)
 {
     dom0_op_t launch_op;
     DECLARE_DOM0_OP;
@@ -1097,8 +1102,8 @@
                      &vstack_start, ctxt, cmdline,
                      op.u.getdomaininfo.shared_info_frame,
                      flags, store_evtchn, store_mfn,
-                     console_evtchn, console_mfn,
-                     features_bitmap) < 0 )
+                     console_evtchn, console_mfn, features_bitmap,
+                     vdevice_share) < 0 )
     {
         ERROR("Error constructing guest OS");
         goto error_out;
@@ -1204,7 +1209,8 @@
                        unsigned int store_evtchn,
                        unsigned long *store_mfn,
                        unsigned int console_evtchn,
-                       unsigned long *console_mfn)
+                       unsigned long *console_mfn,
+                       unsigned long vdevice_share)
 {
     int            sts;
     char          *img_buf;
@@ -1245,7 +1251,7 @@
     sts = xc_linux_build_internal(xc_handle, domid, img_buf, img_len,
                                   &initrd_info, cmdline, features, flags,
                                   store_evtchn, store_mfn,
-                                  console_evtchn, console_mfn);
+                                  console_evtchn, console_mfn, vdevice_share);
 
  out:
     /* The inflation routines may pass back the same buffer so be */
@@ -1270,7 +1276,8 @@
                    unsigned int store_evtchn,
                    unsigned long *store_mfn,
                    unsigned int console_evtchn,
-                   unsigned long *console_mfn)
+                   unsigned long *console_mfn,
+                   unsigned long vdevice_share)
 {
     char *image = NULL;
     unsigned long image_size;
@@ -1302,7 +1309,7 @@
     sts = xc_linux_build_internal(xc_handle, domid, image, image_size,
                                   &initrd_info, cmdline, features, flags,
                                   store_evtchn, store_mfn,
-                                  console_evtchn, console_mfn);
+                                  console_evtchn, console_mfn, vdevice_share);
 
  error_out:
     free(image);
tools/libxc/xc_linux_restore.c +3 -1
diff -r cddc595b285b tools/libxc/xc_linux_restore.c
@@ -105,7 +105,8 @@
 int xc_linux_restore(int xc_handle, int io_fd,
                      uint32_t dom, unsigned long nr_pfns,
                      unsigned int store_evtchn, unsigned long *store_mfn,
-                     unsigned int console_evtchn, unsigned long *console_mfn)
+                     unsigned int console_evtchn, unsigned long *console_mfn,
+                     unsigned long vdevice_share)
 {
     DECLARE_DOM0_OP;
     int rc = 1, i, n;
@@ -518,6 +519,7 @@
     start_info->store_evtchn                 = store_evtchn;
     *console_mfn = start_info->console_mfn   = p2m[start_info->console_mfn];
     start_info->console_evtchn               = console_evtchn;
+    start_info->vdevice_share                  = vdevice_share;
     munmap(start_info, PAGE_SIZE);
 
     /* Uncanonicalise each GDT frame number. */
tools/libxc/xenguest.h +5 -3
diff -r cddc595b285b tools/libxc/xenguest.h
@@ -40,7 +40,7 @@
 int xc_linux_restore(int xc_handle, int io_fd, uint32_t dom,
                      unsigned long nr_pfns, unsigned int store_evtchn,
                      unsigned long *store_mfn, unsigned int console_evtchn,
-                     unsigned long *console_mfn);
+                     unsigned long *console_mfn, unsigned long vdevice_share);
 
 /**
  * This function will create a domain for a paravirtualized Linux
@@ -68,7 +68,8 @@
                    unsigned int store_evtchn,
                    unsigned long *store_mfn,
                    unsigned int console_evtchn,
-                   unsigned long *console_mfn);
+                   unsigned long *console_mfn,
+		   unsigned long vdevice_share);
 
 /**
  * This function will create a domain for a paravirtualized Linux
@@ -100,7 +101,8 @@
                        unsigned int store_evtchn,
                        unsigned long *store_mfn,
                        unsigned int console_evtchn,
-                       unsigned long *console_mfn);
+                       unsigned long *console_mfn,
+                       unsigned long vdevice_share);
 
 int xc_hvm_build(int xc_handle,
                  uint32_t domid,
tools/python/xen/lowlevel/xc/xc.c +53 -5

90 lines not on this page · raw message

tools/python/xen/xend/XendDomainInfo.py +10 -0
diff -r cddc595b285b tools/python/xen/xend/XendDomainInfo.py
@@ -452,6 +452,7 @@
         self.store_mfn = None
         self.console_port = None
         self.console_mfn = None
+        self.vdevice_share = None
 
         self.vmWatch = None
         self.shutdownWatch = None
@@ -730,6 +731,7 @@
         f('console/ring-ref', self.console_mfn)
         f('store/port',       self.store_port)
         f('store/ring-ref',   self.store_mfn)
+        f('vdevice-share',    self.vdevice_share)
 
         to_store.update(self.vcpuDomDetails())
 
@@ -785,6 +787,8 @@
         """For use only by image.py and XendCheckpoint.py."""
         return self.store_port
 
+    def getVdevicePage(self):
+	return self.vdevice_share
 
     def getConsolePort(self):
         """For use only by image.py and XendCheckpoint.py"""
@@ -1285,6 +1289,7 @@
                                                   0, 0)
 
             self.createChannels()
+            self.createVdevicePage()
 
             channel_details = self.image.createImage()
 
@@ -1444,6 +1449,11 @@
             log.exception("Exception in alloc_unbound(%d)", self.domid)
             raise
 
+    def createVdevicePage(self):
+	"""Create a shared Vdevice page for a domain
+	"""
+	self.vdevice_share = xc.create_shared_pages(1)
+	xc.grant_shared_pages(dom=self.domid, mfn=self.vdevice_share)
 
     ## public:
 
tools/python/xen/xend/image.py +5 -2
diff -r cddc595b285b tools/python/xen/xend/image.py
@@ -171,13 +171,15 @@
     ostype = "linux"
 
     def buildDomain(self):
-        store_evtchn = self.vm.getStorePort()
+        store_evtchn = self.vm.getStorePort() 
         console_evtchn = self.vm.getConsolePort()
+        vdevice_share = self.vm.getVdevicePage()
 
         log.debug("dom            = %d", self.vm.getDomid())
         log.debug("image          = %s", self.kernel)
         log.debug("store_evtchn   = %d", store_evtchn)
         log.debug("console_evtchn = %d", console_evtchn)
+        log.debug("vdevide_mfn    = %#lx", vdevice_share)
         log.debug("cmdline        = %s", self.cmdline)
         log.debug("ramdisk        = %s", self.ramdisk)
         log.debug("vcpus          = %d", self.vm.getVCpuCount())
@@ -189,7 +191,8 @@
                               console_evtchn = console_evtchn,
                               cmdline        = self.cmdline,
                               ramdisk        = self.ramdisk,
-                              features       = self.vm.getFeatures())
+                              features       = self.vm.getFeatures(),
+                              vdevice_share    = vdevice_share)
 
 class HVMImageHandler(ImageHandler):
 
tools/xcutils/xc_restore.c +5 -4
diff -r cddc595b285b tools/xcutils/xc_restore.c
@@ -19,11 +19,11 @@
 {
     unsigned int xc_fd, io_fd, domid, nr_pfns, store_evtchn, console_evtchn;
     int ret;
-    unsigned long store_mfn, console_mfn;
+    unsigned long store_mfn, console_mfn, vdevice_share;
 
-    if (argc != 7)
+    if (argc != 8)
 	errx(1,
-	     "usage: %s xcfd iofd domid nr_pfns store_evtchn console_evtchn",
+	     "usage: %s xcfd iofd domid nr_pfns store_evtchn console_evtchn vdevice_share",
 	     argv[0]);
 
     xc_fd = atoi(argv[1]);
@@ -32,9 +32,10 @@
     nr_pfns = atoi(argv[4]);
     store_evtchn = atoi(argv[5]);
     console_evtchn = atoi(argv[6]);
+    vdevice_share = atoi(argv[7]);
 
     ret = xc_linux_restore(xc_fd, io_fd, domid, nr_pfns, store_evtchn,
-			   &store_mfn, console_evtchn, &console_mfn);
+			   &store_mfn, console_evtchn, &console_mfn, vdevice_share);
     if (ret == 0) {
 	printf("store-mfn %li\n", store_mfn);
 	printf("console-mfn %li\n", console_mfn);
xen/include/public/io/vdevice.h new file +23 -0
diff -r cddc595b285b linux-2.6-xen-sparse/include/xen/public/io/vdevice.h
@@ -0,0 +1,23 @@
+#ifndef __XEN_PUBLIC_IO_VDEVICE_H__
+#define __XEN_PUBLIC_IO_VDEVICE_H__
+
+/* These status bits are generic.  256 and above is device specific. */
+#define VDEVICE_S_ACKNOWLEDGE	1 /* We have seen device. */
+#define VDEVICE_S_MAPPED	2 /* We have mapped device OK */
+#define VDEVICE_S_DRIVER	4 /* We have found a driver */
+#define VDEVICE_S_DRIVER_OK	8 /* Driver says OK! */
+#define VDEVICE_S_FAILED	128 /* Something actually failed */
+
+struct vdevice_id {
+	uint32_t type;
+	uint32_t features;
+};
+
+/* We have a page of these descriptors in the vdevice page. */
+struct vdevice_desc {
+	struct vdevice_id id;
+	uint32_t nr_pages;
+	uint32_t status;
+	uint64_t shared_ref;
+};
+#endif	/* __LINUX_VDEVICE_H__ */
Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#6 lore raw

PATCH 6/9 Linux support for vdevice bus

In reply to Rusty Russell

Subject: Linux support for vdevice bus

This patch provides the Linux implementation of the vdevice bus.

FIXME: currently it does not support save/restore of the domain: it should call stop before shutting down, and remap shares afterwards before calling reconnect. This depends on exactly what we do with shared pages on restore.

linux-2.6-xen-sparse/drivers/xen/Makefile +1 -0
diff -r 520f3bf7d3f0 linux-2.6-xen-sparse/drivers/xen/Makefile
@@ -8,6 +8,7 @@
 obj-y	+= balloon/
 obj-y	+= privcmd/
 obj-y	+= xenbus/
+obj-y	+= vdevice/
 obj-y	+= xenshare.o
 
 obj-$(CONFIG_XEN_BLKDEV_BACKEND)	+= blkback/
linux-2.6-xen-sparse/drivers/xen/vdevice/Makefile new file +1 -0
diff -r 520f3bf7d3f0 linux-2.6-xen-sparse/drivers/xen/vdevice/Makefile
@@ -0,0 +1,1 @@
+obj-y := vdevice.o
linux-2.6-xen-sparse/drivers/xen/vdevice/vdevice.c new file +286 -0

286 lines not on this page · raw message

linux-2.6-xen-sparse/include/linux/vdevice.h new file +40 -0
diff -r 520f3bf7d3f0 linux-2.6-xen-sparse/include/linux/vdevice.h
@@ -0,0 +1,40 @@
+#ifndef _LINUX_VDEVICE_H_
+#define _LINUX_VDEVICE_H_
+
+#include <linux/device.h>
+#include <xen/interface/io/vdevice.h>
+#include <xen/interface/share.h>
+#include <asm/share.h>
+
+struct vdevice {
+	/* Unique busid */
+	int vdevice_index;
+
+	/* Shared region for this device. */
+	struct xen_share *share;
+
+	struct device dev;
+	struct vdevice_id id;
+
+	/* Driver can hang data off here. */
+	void *private;
+};
+
+struct vdevice_driver {
+	/* I can drive the following type of device(s) */
+	char *name;
+	struct module *owner;
+	const struct vdevice_id *ids;
+	int (*probe)(struct vdevice *dev, const struct vdevice_id *id);
+	void (*remove)(struct vdevice *dev);
+
+	void (*stop)(struct vdevice *dev);
+	int (*reconnect)(struct vdevice *dev);
+
+	struct device_driver driver;
+};
+
+extern int register_vdevice_driver(struct vdevice_driver *drv);
+extern void unregister_vdevice_driver(struct vdevice_driver *drv);
+
+#endif /* _LINUX_VDEVICE_H_ */
Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#7 lore raw

Re: [PATCH 6/9] Linux support for vdevice bus

Quoted 8 lines · Rusty Russell

Subject: Linux support for vdevice bus

This patch provides the Linux implementation of the vdevice bus.

FIXME: currently it does not support save/restore of the domain: it should call stop before shutting down, and remap shares afterwards before calling reconnect. This depends on exactly what we do with shared pages on restore.

In general I find the 'remember to suspend on save' approach that we are currently using for xenbus and drivers problematic, and I much favor a 'reset on resume' approach instead. Sometimes when doing a save (or migration, or, in my case, self-migration), the domain wants to continue running after the save, and then having to shut down all external devices just to immediately resume them is inelegant and often creates a lot of trouble. If we are to change the IPC/sharing mechanism (and you make some good arguments for that), I think we should design for 'reset on resume' rather than 'suspend-on-save'.

Just my DKK0.02.

Jacob

#8 lore raw

Re: [PATCH 6/9] Linux support for vdevice bus

Quoted 10 lines · Jacob Gorm Hansen
Quoted 8 lines · Rusty Russell

Subject: Linux support for vdevice bus

This patch provides the Linux implementation of the vdevice bus.

FIXME: currently it does not support save/restore of the domain: it should call stop before shutting down, and remap shares afterwards before calling reconnect. This depends on exactly what we do with shared pages on restore.

Quoted 9 lines

In general I find the 'remember to suspend on save' approach that we are currently using for xenbus and drivers problematic, and I much favor a 'reset on resume' approach instead. Sometimes when doing a save (or migration, or, in my case, self-migration), the domain wants to continue running after the save, and then having to shut down all external devices just to immediately resume them is inelegant and often creates a lot of trouble. If we are to change the IPC/sharing mechanism (and you make some good arguments for that), I think we should design for 'reset on resume' rather than 'suspend-on-save'.

Thanks for this input Jacob!

Things are simpler enough that this might not be an issue, but...

Let's look at the shared network device as an example. It maps the share_ref it's told to in the vdevice page, and is reading and writing to that mapped page (although not in an interrupt, unlike the block driver).

On restore, the previously mapped pages will no longer be valid, and the new share_ref needs to be mapped. Even if we remap the new share_refs in place on restore before allowing interrupts, the driver might process an interrupt before being told it has to reset, and get confused.

So if we really want all the action to happen on restore, I think we need a stop!() function for drivers called before interrupts are re-enabled, and then we can call restart() later at our leisure.

The driver, other than ensuring that "stop" can be called with interrupts off, doesn't have to know whether it's called during save or restore.

Seem reasonable?
Rusty.

Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#9 lore raw

Re: [PATCH 6/9] Linux support for vdevice bus

Quoted 9 lines · Rusty Russell

So if we really want all the action to happen on restore, I think we need a stop!() function for drivers called before interrupts are re-enabled, and then we can call restart() later at our leisure.

The driver, other than ensuring that "stop" can be called with interrupts off, doesn't have to know whether it's called during save or restore.

Seem reasonable?

Yes, that sounds fine to me.

Jacob

#10 lore raw

PATCH 7/9 vdevice tool for manipulating vdevice bus

In reply to Rusty Russell

Subject: vdevice tool for manipulating vdevice bus

This creates a simple C tool for list, adding and removing vdevices. This is a demonstration (although, in my opinion, and important one, because it shows how simple this is).

This functionality should also be added to the python tools.

.hgignore +1 -0
diff -r 570d423310b5 .hgignore
@@ -146,6 +146,7 @@
 ^tools/security/secpol_tool$
 ^tools/security/xen/.*$
 ^tools/tests/test_x86_emulator$
+^tools/vdevice/vdevice$
 ^tools/vnet/gc$
 ^tools/vnet/gc.*/.*$
 ^tools/vnet/vnet-module/.*\.ko$
tools/Makefile +1 -0
diff -r 570d423310b5 tools/Makefile
@@ -13,6 +13,7 @@
 SUBDIRS += console
 SUBDIRS += xenmon
 SUBDIRS += guest-headers
+SUBDIRS += vdevice
 ifeq ($(VTPM_TOOLS),y)
 SUBDIRS += vtpm_manager
 SUBDIRS += vtpm
tools/vdevice/Makefile new file +34 -0
diff -r 570d423310b5 tools/vdevice/Makefile
@@ -0,0 +1,34 @@
+XEN_ROOT=../..
+include $(XEN_ROOT)/tools/Rules.mk
+
+INSTALL         = install
+INSTALL_DATA	= $(INSTALL) -m0644
+INSTALL_PROG    = $(INSTALL) -m0755
+INSTALL_DIR     = $(INSTALL) -d -m0755
+
+PROFILE=#-pg
+BASECFLAGS=-Wall -g -Werror
+# Make gcc generate dependencies.
+BASECFLAGS += -Wp,-MD,.$(@F).d
+PROG_DEP = .*.d
+BASECFLAGS+= -O3 $(PROFILE)
+BASECFLAGS+= -I$(XEN_ROOT)/tools/libxc
+BASECFLAGS+= -I$(XEN_ROOT)/tools/xenstore
+BASECFLAGS+= -I.
+
+CFLAGS  += $(BASECFLAGS)
+LDFLAGS += $(PROFILE) -L$(XEN_LIBXC) -L$(XEN_XENSTORE)
+
+all: vdevice
+
+clean:
+	rm -f vdevice *.o .*.d
+
+vdevice: vdevice.o
+	$(LINK.o) $^ $(LOADLIBES) $(LDLIBS) -lxenctrl -lxenstore -o $@
+
+install: vdevice
+	$(INSTALL_DIR) -p $(DESTDIR)/usr/sbin
+	$(INSTALL_PROG) vdevice $(DESTDIR)/usr/sbin
+
+-include $(PROG_DEP)
tools/vdevice/vdevice.c new file +571 -0

571 lines not on this page · raw message

Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#11 lore raw

PATCH 8/9 Xen Share Net Device

In reply to Rusty Russell

Subject: Xen Share Net Device

This is a simple network device using the vdevice bus and the share info. It is primitive, in that it does not handle fragmented skbs (for no particularly good reason).

The main feature of this device is that it demonstrates how a N:N device like a virtual intra-domain LAN can be implemented.

linux-2.6-xen-sparse/drivers/xen/Makefile +1 -0
diff -r c0c781af505d linux-2.6-xen-sparse/drivers/xen/Makefile
@@ -9,6 +9,7 @@
 obj-y	+= privcmd/
 obj-y	+= xenbus/
 obj-y	+= vdevice/
+obj-m	+= sharenet/
 obj-y	+= xenshare.o
 
 obj-$(CONFIG_XEN_BLKDEV_BACKEND)	+= blkback/
tools/vdevice/vdevice.c +47 -0

59 lines not on this page · raw message

linux-2.6-xen-sparse/drivers/xen/sharenet/Makefile new file +1 -0
diff -r c0c781af505d linux-2.6-xen-sparse/drivers/xen/sharenet/Makefile
@@ -0,0 +1,1 @@
+obj-m	:= xensnet.o
linux-2.6-xen-sparse/drivers/xen/sharenet/xensnet.c new file +527 -0

527 lines not on this page · raw message

Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#12 lore raw

PATCH 9/9 Simple Xenshare Block Device and userspace backend

In reply to Rusty Russell

This is a simple block device, with the backend in userspace. While performance is good, it's not production-ready because it need to keep track of which backend process to kill when device goes away (look for "So very, very wrong"), and also should use separate device numbers for front and backend.

.hgignore +1 -0
diff -r 20b744b2c4c0 .hgignore
@@ -147,6 +147,7 @@
 ^tools/security/xen/.*$
 ^tools/tests/test_x86_emulator$
 ^tools/vdevice/vdevice$
+^tools/vdevice/xensblk$
 ^tools/vnet/gc$
 ^tools/vnet/gc.*/.*$
 ^tools/vnet/vnet-module/.*\.ko$
linux-2.6-xen-sparse/drivers/xen/Makefile +1 -0
diff -r 20b744b2c4c0 linux-2.6-xen-sparse/drivers/xen/Makefile
@@ -10,6 +10,7 @@
 obj-y	+= xenbus/
 obj-y	+= vdevice/
 obj-m	+= sharenet/
+obj-m	+= shareblock/
 obj-y	+= xenshare.o
 
 obj-$(CONFIG_XEN_BLKDEV_BACKEND)	+= blkback/
tools/examples/Makefile +1 -1
diff -r 20b744b2c4c0 tools/examples/Makefile
@@ -35,7 +35,7 @@
 XEN_SCRIPT_DATA += vtpm-migration.sh
 
 XEN_HOTPLUG_DIR = /etc/hotplug
-XEN_HOTPLUG_SCRIPTS = xen-backend.agent
+XEN_HOTPLUG_SCRIPTS = xen-backend.agent vdevice.agent
 
 UDEV_RULES_DIR = /etc/udev
 UDEV_RULES = xen-backend.rules
tools/vdevice/Makefile +8 -3
diff -r 20b744b2c4c0 tools/vdevice/Makefile
@@ -19,16 +19,21 @@
 CFLAGS  += $(BASECFLAGS)
 LDFLAGS += $(PROFILE) -L$(XEN_LIBXC) -L$(XEN_XENSTORE)
 
-all: vdevice
+all: vdevice xensblk
 
 clean:
-	rm -f vdevice *.o .*.d
+	rm -f vdevice xensblk *.o
 
 vdevice: vdevice.o
 	$(LINK.o) $^ $(LOADLIBES) $(LDLIBS) -lxenctrl -lxenstore -o $@
 
-install: vdevice
+xensblk: xensblk.o
+	$(LINK.o) $^ $(LOADLIBES) $(LDLIBS) -lxenctrl -o $@
+
+
+install: vdevice xensblk
 	$(INSTALL_DIR) -p $(DESTDIR)/usr/sbin
 	$(INSTALL_PROG) vdevice $(DESTDIR)/usr/sbin
+	$(INSTALL_PROG) xensblk $(DESTDIR)/usr/sbin
 
 -include $(PROG_DEP)
tools/vdevice/vdevice.c +63 -0

81 lines not on this page · raw message

linux-2.6-xen-sparse/drivers/xen/shareblock/Makefile new file +1 -0
diff -r 20b744b2c4c0 linux-2.6-xen-sparse/drivers/xen/shareblock/Makefile
@@ -0,0 +1,1 @@
+obj-m	+= xensblk.o
linux-2.6-xen-sparse/drivers/xen/shareblock/xensblk.c new file +293 -0

293 lines not on this page · raw message

linux-2.6-xen-sparse/include/xen/public/xensblk.h new file +38 -0
diff -r 20b744b2c4c0 linux-2.6-xen-sparse/include/xen/public/xensblk.h
@@ -0,0 +1,38 @@
+#ifndef _LINUX_XENSBLK_H
+#define _LINUX_XENSBLK_H
+
+/* Possible device types */
+#define XENSBLK_DEVTYPE_FILE 1 	/* Followed by path of file to serve. */
+
+struct xensblk_page
+{
+	/* Filled in by the creation tool */
+	uint16_t device_type;
+	uint16_t flags;
+
+	/* Contents depends on device_type */
+	char device_specific_info[256];
+
+	/* Filled in by server. */
+	uint32_t capacity_in_blocks;
+	uint32_t error;
+
+	/* Request 0 = read, 1 = write. */
+	uint32_t req_type;
+	/* Length (sectors). */
+	uint32_t num;
+	/* Sector to read/write (multiply by 512 for offset). */
+	uint64_t sector;
+};
+
+/* Triggered by client when op ready (except write: xfer triggers) */
+#define XENSBLK_OP_READY 1
+
+/* Triggered by server has finished op (except read: xfer triggers) */
+#define XENSBLK_OP_DONE 2
+
+/* Where the server and client queue their sg lists */
+#define XENSBLK_CLIENT_QUEUE 0
+#define XENSBLK_SERVER_QUEUE 1
+
+#endif	/* _LINUX_XENSBLK_H */
tools/examples/vdevice.agent new file +18 -0
diff -r 20b744b2c4c0 tools/examples/vdevice.agent
@@ -0,0 +1,18 @@
+#! /bin/sh
+
+PATH=/etc/xen/scripts:$PATH
+
+exec > /tmp/log 2>&1
+echo VDEVICE called: $DEVPATH
+
+if [ `cat /sys/$DEVPATH/type` == 2 ]; then
+    case "$ACTION" in
+	add)
+	    /usr/sbin/xensblk $DEVPATH >> /tmp/log 2>&1 &
+	    ;;
+	remove)
+	    # FIXME: So very, very wrong...
+	    killall /usr/sbin/xensblk
+	    ;;
+    esac
+fi
tools/examples/vdevice.rules new file +3 -0
diff -r 20b744b2c4c0 tools/examples/vdevice.rules
@@ -0,0 +1,3 @@
+SUBSYSTEM=="vdevice", SYSFS{type}=="2", ACTION=="add", RUN+="/usr/sbin/xensblk %p &"
+# FIXME: So very, very wrong...
+SUBSYSTEM=="vdevice", SYSFS{type}=="2", ACTION=="remove", RUN+="killall /usr/sbin/xensblk"
tools/vdevice/xensblk.c new file +269 -0

269 lines not on this page · raw message

Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#13 lore raw
In reply to Rusty Russell
Quoted 22 lines · Rusty Russell

static inline int get_page(struct page_info *page,
struct domain *domain)

xen/include/asm-x86/mm.h +20 -0
+{
+    u32 x, nx, y = page->count_info;
+    u32 d, nd = page->u.inuse._domain;
+    u32 _domain = pickle_domptr(domain);
+
+    do {
+        x  = y;
+        nx = x + 1;
+        d  = nd;
+        if ( unlikely((x & PGC_count_mask) == 0) ||  /* Not allocated? */
+             unlikely((nx & PGC_count_mask) == 0) ) /* Count overflow? */
+        {
+	    if ( !_shadow_mode_refcounts(domain) )
+		DPRINTK("Error pfn %lx: rd=%p, od=%p, caf=%08x, taf=%" PRtype_info "\n",
+			page_to_pfn(page), domain, unpickle_domptr(d),
+			x, page->u.inuse.type_info);
+	    return 0;
+        }
+	if ( unlikely(d != _domain) )               /* Wrong owner? */
+	    return try_shared_page(page, domain);

xen/include/asm-x86/mm.h line 219

Quoted 10 lines
xen/include/asm-x86/mm.h +10 -0
+        __asm__ __volatile__(
+            LOCK_PREFIX "cmpxchg8b %3"
+            : "=d" (nd), "=a" (y), "=c" (d),
+              "=m" (*(volatile u64 *)(&page->count_info))
+            : "0" (d), "1" (x), "c" (d), "b" (nx) );
+    }
+    while ( unlikely(nd != d) || unlikely(y != x) );
+
+    return 1;
+}

What is the "=c" (d) there for? And doesn't cmpxchg8b modify the zero flag---is it necessary to clobber the condition code register?

xen/include/public/xen.h line 67

Quoted 8 lines

diff -r d5f98d23427a xen/include/public/xen.h

xen/include/public/xen.h +1 -0
@@ -64,6 +64,7 @@
 #define __HYPERVISOR_xenoprof_op          31
 #define __HYPERVISOR_event_channel_op     32
 #define __HYPERVISOR_physdev_op           33
+#define __HYPERVISOR_share_op             33

Sharing no 33?

#14 lore raw
Quoted 14 lines · Harry Butterworth

xen/include/asm-x86/mm.h line 219

Quoted 10 lines · Rusty Russell
xen/include/asm-x86/mm.h +10 -0
+        __asm__ __volatile__(
+            LOCK_PREFIX "cmpxchg8b %3"
+            : "=d" (nd), "=a" (y), "=c" (d),
+              "=m" (*(volatile u64 *)(&page->count_info))
+            : "0" (d), "1" (x), "c" (d), "b" (nx) );
+    }
+    while ( unlikely(nd != d) || unlikely(y != x) );
+
+    return 1;
+}

What is the "=c" (d) there for? And doesn't cmpxchg8b modify the zero flag---is it necessary to clobber the condition code register?

Good questions. I copied this code from below, though, so I can conveniently punt on this one...

Quoted 10 lines

xen/include/public/xen.h line 67

Quoted 8 lines

diff -r d5f98d23427a xen/include/public/xen.h

xen/include/public/xen.h +1 -0
@@ -64,6 +64,7 @@
 #define __HYPERVISOR_xenoprof_op          31
 #define __HYPERVISOR_event_channel_op     32
 #define __HYPERVISOR_physdev_op           33
+#define __HYPERVISOR_share_op             33

Sharing no 33?

Oops. Good catch, this must have been my sloppy merging. I've fixed this locally.

Thanks!
Rusty.

Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#15 lore raw
In reply to Rusty Russell
Quoted 2 lines · Rusty Russell

This introduces a page "share" mechanism to xen: an alternative to both cross-domain binding of event channels, and grant tables.

Why do you think an alternative to event channels and grant tables is needed?

Personally, I think there is a need for a more convenient _high-level_ API that more directly meets the split-drivers' requirements for inter-domain messaging and bulk data transfer but this seems to me to be another low-level API and doesn't seem significantly more convenient to use than grant-tables and event-channels.

Is the n>2-way sharing feature the only benefit over grant-tables or do you think there are other benefits to your approach?

#16 lore raw
Quoted 6 lines · Harry Butterworth
Quoted 2 lines · Rusty Russell

This introduces a page "share" mechanism to xen: an alternative to both cross-domain binding of event channels, and grant tables.

Why do you think an alternative to event channels and grant tables is needed?

Fundamentally, I think having a simplified mechanism for inter-domain communication keeps us honest: we can benchmark against it and look at the code size, if nothing else.

Quoted 5 lines

Personally, I think there is a need for a more convenient _high-level_ API that more directly meets the split-drivers' requirements for inter-domain messaging and bulk data transfer but this seems to me to be another low-level API and doesn't seem significantly more convenient to use than grant-tables and event-channels.

The isolation it provides forms a better basis for a higher-level mechanism, IMHO. Neither side really knows, or cares, where the events and data are really going to. There is no mapping of other domain's memory, with all the trust and coordination issues that we had to deal with (and didn't completely!) with the current mechanisms.

For example, if you wanted to use this as a remote mechanism for communication with another machine, you could. If you wanted to substitute backends without the front end knowing, you could. If the frontend or backend dies, there is no restriction on cleaning up its memory.

Quoted 2 lines

Is the n>2-way sharing feature the only benefit over grant-tables or do you think there are other benefits to your approach?

Other than the above theoretical advantages, I found it far easier to implement a Linux driver on top of this, than it was to implement it on top of grant tables, event channel binding and xenbus. Easier to implement means easier to optimize, easier to port, and easier to debug.

Hope that clarifies!
Rusty.

Signature · 1 line
 ccontrol: http://ccontrol.ozlabs.org
#17 lore raw
Quoted 7 lines · Rusty Russell
Quoted 6 lines · Harry Butterworth
Quoted 2 lines · Rusty Russell

This introduces a page "share" mechanism to xen: an alternative to both cross-domain binding of event channels, and grant tables.

Why do you think an alternative to event channels and grant tables is needed?

<big snip>

Quoted 3 lines

If the
frontend or backend dies, there is no restriction on cleaning up its memory.

This is a good answer. In my use of grant-tables for the USB driver, I ended up with the possibility of one side having to poll indefinitely for the other side to free up its memory with no guarantee of this ever happening. The memory couldn't be reused until it was freed because the other side had a writeable mapping (reusing even just a readable mapping would have been a security hole).

I think a fix for this problem is a significant win for your approach over grant-tables.

Quoted 1 line

Hope that clarifies!

Yes thanks.

Harry.

#18 lore raw
In reply to Rusty Russell
Quoted 4 lines · Rusty Russell

Finally, a scatter-gather list mechanism allows the domains to associate their pages with arbitrary queue numbers in the shared region, to transport bulk data (effectively by having the hypervisor do "DMA" between domains).

The sg transfer mechanism avoids bounce buffers and does a straight copy
from_user or to_user below. So this is an alternative to page
flipping, right? It would be interesting to see how the direct copy performs compared to page flipping on various architectures. Page table updates are not free and processors are getting more efficient at bulk dma.

Quoted 18 lines
Quoted 1 line

+/* Copy from src to dst, return amount copied. */

xen/common/share.c new file +17 -0
+static int do_copy(const struct sg_list *sgdst, const struct sg_list *sgsrc,
+		   unsigned long (*copy)(paddr_t, paddr_t, unsigned long))
+{
+	unsigned long totlen, src, dst, srcoff, dstoff;
+	int ret = 0;
+
+	totlen = 0;
+	src = dst = 0;
+	srcoff = dstoff = 0;
+	while (src < sgsrc->num_sg) {
+		unsigned long len;
+		len = min(sgsrc->sg[src].len - srcoff,
+			  sgdst->sg[dst].len - dstoff);
+
+		len = copy(sgdst->sg[dst].addr+dstoff,
+			   sgsrc->sg[src].addr+srcoff,
+			   len);