Skip to contentSkip to conversation

stubdom: PCI passthrough support via PV-PCI

PATCH In review Samuel Thibault 3 participants 7 messages Jul 2, 2008

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 stubdom: PCI passthrough support via PV-PCI

stubdom: PCI passthrough support via PV-PCI

extras/mini-os/include/pcifront.h new file +15 -0
diff -r 0361bcf5d310 extras/mini-os/include/pcifront.h
@@ -0,0 +1,15 @@
+#include <types.h>
+#include <xen/io/pciif.h>
+struct pcifront_dev;
+struct pcifront_dev *init_pcifront(char *nodename);
+void pcifront_scan(struct pcifront_dev *dev, void (*fun)(unsigned int domain, unsigned int bus, unsigned slot, unsigned int fun));
+void pcifront_op(struct pcifront_dev *dev, struct xen_pci_op *op);
+void shutdown_pcifront(struct pcifront_dev *dev);
+int pcifront_conf_read(struct pcifront_dev *dev,
+                       unsigned int dom,
+                       unsigned int bus, unsigned int slot, unsigned long fun,
+                       unsigned int off, unsigned int size, unsigned int *val);
+int pcifront_conf_write(struct pcifront_dev *dev,
+                        unsigned int dom,
+                        unsigned int bus, unsigned int slot, unsigned long fun,
+                        unsigned int off, unsigned int size, unsigned int val);
extras/mini-os/include/posix/sys/mman.h +3 -0
diff -r 0361bcf5d310 extras/mini-os/include/posix/sys/mman.h
@@ -9,6 +9,9 @@
 #define MAP_PRIVATE	0x02
 #define MAP_ANON	0x20
 
+/* Pages are always resident anyway */
+#define MAP_LOCKED	0x0
+
 #define MAP_FAILED	((void*)0)
 
 void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset);
extras/mini-os/kernel.c +26 -0
diff -r 0361bcf5d310 extras/mini-os/kernel.c
@@ -40,6 +40,7 @@
 #include <netfront.h>
 #include <blkfront.h>
 #include <fbfront.h>
+#include <pcifront.h>
 #include <fs.h>
 #include <xmalloc.h>
 #include <fcntl.h>
@@ -431,6 +432,27 @@
     }
 }
 
+static struct pcifront_dev *pci_dev;
+
+static void pcifront_thread(void *p)
+{
+    void print(unsigned int domain, unsigned int bus, unsigned int slot, unsigned int fun)
+    {
+        unsigned int vendor, device, rev, class;
+
+        pcifront_conf_read(pci_dev, domain, bus, slot, fun, 0x00, 2, &vendor);
+        pcifront_conf_read(pci_dev, domain, bus, slot, fun, 0x02, 2, &device);
+        pcifront_conf_read(pci_dev, domain, bus, slot, fun, 0x08, 1, &rev);
+        pcifront_conf_read(pci_dev, domain, bus, slot, fun, 0x0a, 2, &class);
+
+        printk("%04x:%02x:%02x.%02x %04x: %04x:%04x (rev %02x)\n", domain, bus, slot, fun, class, vendor, device, rev);
+    }
+
+    pci_dev = init_pcifront(NULL);
+    printk("PCI devices:\n");
+    pcifront_scan(pci_dev, print);
+}
+
 static void fs_thread(void *p)
 {
     init_fs_frontend();
@@ -446,6 +468,7 @@
     create_thread("blkfront", blkfront_thread, si);
     create_thread("fbfront", fbfront_thread, si);
     create_thread("kbdfront", kbdfront_thread, si);
+    create_thread("pcifront", pcifront_thread, si);
     create_thread("fs-frontend", fs_thread, si);
     return 0;
 }
@@ -524,6 +547,9 @@
     if (kbd_dev)
         shutdown_kbdfront(kbd_dev);
 
+    if (pci_dev)
+        shutdown_pcifront(pci_dev);
+
     /* TODO: fs import */
 
     local_irq_disable();
extras/mini-os/pcifront.c new file +278 -0

278 lines not on this page · raw message

stubdom/Makefile +7 -3
diff -r 0361bcf5d310 stubdom/Makefile
@@ -132,16 +132,20 @@
 pciutils-$(LIBPCI_VERSION).tar.bz2:
 	$(WGET) http://www.kernel.org/pub/software/utils/pciutils/pciutils-$(LIBPCI_VERSION).tar.bz2
 
+pciutils-$(LIBPCI_VERSION): pciutils-$(LIBPCI_VERSION).tar.bz2
+	tar xjf $<
+	patch -d $@ -p1 < pciutils.patch
+	touch $@
+
 LIBPCI_STAMPFILE=$(CROSS_ROOT)/$(GNU_TARGET_ARCH)-xen-elf/lib/libpci.a
 .PHONY: cross-libpci
 cross-libpci: $(LIBPCI_STAMPFILE)
-$(LIBPCI_STAMPFILE): pciutils-$(LIBPCI_VERSION).tar.bz2 $(NEWLIB_STAMPFILE) $(ZLIB_STAMPFILE)
-	tar xjf $<
+$(LIBPCI_STAMPFILE): pciutils-$(LIBPCI_VERSION) $(NEWLIB_STAMPFILE) $(ZLIB_STAMPFILE)
 	( cd pciutils-$(LIBPCI_VERSION) && \
 	  cp ../libpci.config.h lib/config.h && \
 	  echo '#define PCILIB_VERSION "$(LIBPCI_VERSION)"' >> lib/config.h && \
 	  cp ../libpci.config.mak lib/config.mk && \
-	  $(MAKE) CC="$(GNU_TARGET_ARCH)-xen-elf-gcc $(TARGET_CFLAGS)" lib/libpci.a && \
+	  $(MAKE) CC="$(GNU_TARGET_ARCH)-xen-elf-gcc $(TARGET_CFLAGS) -I$(realpath $(MINI_OS)/include)" lib/libpci.a && \
 	  $(INSTALL_DATA) lib/libpci.a $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/lib/ && \
 	  $(INSTALL_DIR) $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/include/pci && \
 	  $(INSTALL_DATA) lib/{config,header,pci,types}.h $(CROSS_PREFIX)/$(GNU_TARGET_ARCH)-xen-elf/include/pci/ \
stubdom/libpci.config.h +1 -1
diff -r 0361bcf5d310 stubdom/libpci.config.h
@@ -1,4 +1,4 @@
-#define PCI_OS_STUBDOM
+#define PCI_OS_MINIOS
 #define PCI_HAVE_STDINT_H
 #define PCI_PATH_IDS_DIR "."
 #define PCI_COMPRESSED_IDS
stubdom/libpci.config.mak +5 -0
diff -r 0361bcf5d310 stubdom/libpci.config.mak
@@ -1,2 +1,7 @@
 LIBZ=-lz
 LDLIBS+=$(LIBZ)
+PCI_OS_MINIOS=1
+PCI_HAVE_STDINT_H=1
+PCI_PATH_IDS_DIR=.
+PCI_COMPRESSED_IDS=1
+PCI_IDS=pci.ids.gz
stubdom/pciutils.patch new file +299 -0

299 lines not on this page · raw message

tools/ioemu/Makefile.target +1 -1
diff -r 0361bcf5d310 tools/ioemu/Makefile.target
@@ -358,7 +358,7 @@
 endif
 
 ifdef CONFIG_STUBDOM
-#CONFIG_PASSTHROUGH=1
+CONFIG_PASSTHROUGH=1
 else
   ifeq (,$(wildcard /usr/include/pci))
 $(warning *** pciutils-devl package not found - missing /usr/include/pci)
tools/ioemu/hw/pass-through.c +1 -0
diff -r 0361bcf5d310 tools/ioemu/hw/pass-through.c
@@ -515,6 +515,7 @@
         PT_LOG("Error: couldn't locate device in libpci structures\n");
         return NULL;
     }
+    pci_fill_info(pci_dev, PCI_FILL_IRQ | PCI_FILL_BASES | PCI_FILL_ROM_BASE | PCI_FILL_SIZES);
 
     if ( e_devfn == PT_VIRT_DEVFN_AUTO ) {
         /*indicate a static assignment(not hotplug), so find a free PCI hot plug slot */
tools/ioemu/xenstore.c +2 -2
diff -r 0361bcf5d310 tools/ioemu/xenstore.c
@@ -322,7 +322,7 @@
 
     /* get the pci pass-through parameter */
     if (pasprintf(&buf, "/local/domain/0/backend/pci/%u/%u/num_devs",
-                  domid, pci_devid) == -1)
+                  hvm_domid, pci_devid) == -1)
         goto out;
 
     free(params);
@@ -333,7 +333,7 @@
 
     for ( i = 0; i < num; i++ ) {
         if (pasprintf(&buf, "/local/domain/0/backend/pci/%u/%u/dev-%d",
-                    domid, pci_devid, i) != -1) {
+                    hvm_domid, pci_devid, i) != -1) {
             free(dev);
             dev = xs_read(xsh, XBT_NULL, buf, &len);
#1 lore raw

Re: PCI passthrough support via PV-PCI

Note however that it can not work as such because ioemu needs to perform some domctls, which the hypervisor will not let the stub domain do. We need to find a secure way to allow a domain to call memory/IO mapping and IRQ/MSI binding domctls when it is allowed to, i.e. probably have xend tell the hypervisor which PCI cards are controlled by which domains and then accept domctls for their resources.

Samuel

#2 lore raw

Re: Re: PCI passthrough support via PV-PCI

Shouldn't they happen from xend or pciback?

-- Keir

Quoted 13 lines · Samuel Thibault

Note however that it can not work as such because ioemu needs to perform some domctls, which the hypervisor will not let the stub domain do. We need to find a secure way to allow a domain to call memory/IO mapping and IRQ/MSI binding domctls when it is allowed to, i.e. probably have xend tell the hypervisor which PCI cards are controlled by which domains and then accept domctls for their resources.

Samuel

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#3 lore raw

Re: Re: PCI passthrough support via PV-PCI

Quoted 1 line · Keir Fraser, le Wed 02 Jul 2008 13:59:42 +0100, a écrit :

Shouldn't they happen from xend or pciback?

For PV guests it looks like something is done indeed, because else it wouldn't work. It is probably a matter of having domctl permit the stub domain to transfer these rights to the HVM guest.

Samuel

#4 lore raw

Re: Re: PCI passthrough support via PV-PCI

[Samuel Thibault]

Quoted 6 lines
Quoted 1 line · Keir Fraser, le Wed 02 Jul 2008 13:59:42 +0100, a écrit :

Shouldn't they happen from xend or pciback?

For PV guests it looks like something is done indeed, because else it wouldn't work. It is probably a matter of having domctl permit the stub domain to transfer these rights to the HVM guest.

When I added VT-d support for PV guests I made xend do the device assignment iff the destination was a PV domain. Is there any reason why device assignment can not be performed by xend for HVM guests as well?

Other resource assignment (IOMEM and IRQs) still seems to happen in xend for HVM guests as well as for PV guest. What other domctls does ioemu need to perform?

eSk

#5 lore raw

Re: Re: PCI passthrough support via PV-PCI

Quoted 8 lines · Espen Skoglund, le Wed 02 Jul 2008 15:18:16 +0100, a écrit :

[Samuel Thibault]

Quoted 6 lines
Quoted 1 line · Keir Fraser, le Wed 02 Jul 2008 13:59:42 +0100, a écrit :

Shouldn't they happen from xend or pciback?

For PV guests it looks like something is done indeed, because else it wouldn't work. It is probably a matter of having domctl permit the stub domain to transfer these rights to the HVM guest.

Quoted 8 lines

When I added VT-d support for PV guests I made xend do the device assignment iff the destination was a PV domain. Is there any reason why device assignment can not be performed by xend for HVM guests as well?

Other resource assignment (IOMEM and IRQs) still seems to happen in xend for HVM guests as well as for PV guest. What other domctls does ioemu need to perform?

ioemu needs to update the mappings, see the pt_iomem_map() function.

Samuel

#6 lore raw

Re: Re: PCI passthrough support via PV-PCI

Quoted 10 lines · Samuel Thibault
Quoted 8 lines

When I added VT-d support for PV guests I made xend do the device assignment iff the destination was a PV domain. Is there any reason why device assignment can not be performed by xend for HVM guests as well?

Other resource assignment (IOMEM and IRQs) still seems to happen in xend for HVM guests as well as for PV guest. What other domctls does ioemu need to perform?

ioemu needs to update the mappings, see the pt_iomem_map() function.

I suggest the domctls should set up access permissions (e.g., by use of existing iomem_premission domctl). And then extend XENMEM_add_to_physmap to allow (re-)mapping of permitted I/O ranges into pseudophysical address space. The new passthru-specific domctls may well be ill conceived.

-- Keir