Skip to contentSkip to conversation

ioemu: Allow any unused PCI device to be used for pass-through

RFC In review Simon Horman 1 participant 5 messages Mar 13, 2009 – Mar 20, 2009

lore
Share & tools
lore
#0 lore raw

cover ioemu: Allow any unused PCI device to be used for pass-through

The current limit of 2 pass-through devices seems to be rather small. These patches remove that limit and allow any unused PCI device to be used for pass-through, including hotplug.

It seems that the first few (4) devices will be used for other purposes by HVM domains. However, it seems to me to be clean to make this code generic and cope with any device on the bus - saving the need for subsequent changes if the number of devices HVM domains use by default ever changes.

There are also xend and hvmloader patches required for this change. I will submit them separately.

Signature · 3 lines
Simon Horman
  VA Linux Systems Japan K.K., Sydney, Australia Satellite Office
  H: www.vergenet.net/~horms/             W: www.valinux.co.jp/en
#1 lore raw

RFC 1/2 Use test_pci_slot() to test if a php slot is valid, in use or free

---

 hw/pass-through.c |    8 ++++----
 hw/piix4acpi.c    |    2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)
hw/pass-through.c +4 -4
Index: ioemu-remote/hw/pass-through.c
===================================================================
@@ -833,7 +833,7 @@ static int __insert_to_pci_slot(int bus,
     if ( slot >= PHP_SLOT_START && slot < PHP_SLOT_END )
     {
         php_slot = PCI_TO_PHP_SLOT(slot);
-        if ( !dpci_infos.php_devs[php_slot].valid )
+        if ( !test_pci_slot(slot) )
         {
             goto found;
         }
@@ -847,7 +847,7 @@ static int __insert_to_pci_slot(int bus,
     /* slot == 0, pick up a free one */
     for ( i = 0; i < PHP_SLOT_LEN; i++ )
     {
-        if ( !dpci_infos.php_devs[i].valid )
+        if ( !test_pci_slot(PHP_TO_PCI_SLOT(i)) )
         {
             php_slot = i;
             goto found;
@@ -3728,13 +3728,13 @@ int unregister_real_device(int php_slot)
     uint32_t bdf = 0;
     int rc = -1;
 
-    if ( php_slot < 0 || php_slot >= PHP_SLOT_LEN )
+    if ( test_pci_slot(PHP_TO_PCI_SLOT(php_slot)) != 1 )
        return -1;
 
     php_dev = &dpci_infos.php_devs[php_slot];
     assigned_device = php_dev->pt_dev;
 
-    if ( !assigned_device || !php_dev->valid )
+    if ( !assigned_device )
         return -1;
 
     pci_dev = assigned_device->pci_dev;
hw/piix4acpi.c +1 -1
Index: ioemu-remote/hw/piix4acpi.c
===================================================================
@@ -454,7 +454,7 @@ void acpi_php_del(int pci_slot)
     GPEState *s = &gpe_state;
     int php_slot = PCI_TO_PHP_SLOT(pci_slot);
 
-    if ( pci_slot < PHP_SLOT_START || pci_slot >= PHP_SLOT_END ) {
+    if ( test_pci_slot(pci_slot) < 0 ) {
         fprintf(logfile, "not find the pci slot %d when hot remove.\n", pci_slot);
 
         return;
Signature · 4 lines
-- 
Simon Horman
  VA Linux Systems Japan K.K., Sydney, Australia Satellite Office
  H: www.vergenet.net/~horms/             W: www.valinux.co.jp/en
#2 lore raw

RFC 2/2 ioemu: Allow any unused PCI device to be used for pass-through

In reply to Simon Horman

Allow any unused PCI device to be used for pass-through. This includes allowing these devices to be used for hotplug.

There are companion xend and hvmloader patches.

---

 hw/pass-through.c |  113 ++++++++++++++++++++++-------------------------------
 hw/pci.c          |    8 ++-
 hw/pci.h          |   12 +----
 hw/piix4acpi.c    |   68 ++++++++++++++++++-------------
 4 files changed, 96 insertions(+), 105 deletions(-)
hw/pass-through.c +47 -66
Index: ioemu-remote/hw/pass-through.c
===================================================================
@@ -39,7 +39,7 @@ struct php_dev {
 };
 struct dpci_infos {
 
-    struct php_dev php_devs[PHP_SLOT_LEN];
+    struct php_dev php_devs[NR_PCI_DEV];
 
     PCIBus *e_bus;
     struct pci_access *pci_access;
@@ -819,51 +819,41 @@ static int get_next_keyval(char **option
 }
 
 /* Insert a new pass-through device into a specific pci slot.
- * input  dom:bus:dev.func@slot, chose free one if slot == 0
- * return -1: required slot not available
- *         0: no free hotplug slots, but normal slot should okay
- *        >0: the new hotplug slot
+ * input  dom:bus:dev.func@slot, chose free one if slot == AUTO_PHP_SLOT
+ * return -2: requested slot not available
+ *        -1: no free slots
+ *        >=0: the new hotplug slot
  */
 static int __insert_to_pci_slot(int bus, int dev, int func, int slot,
                                 char *opt)
 {
-    int i, php_slot;
+    PCIBus *e_bus = dpci_infos.e_bus;
 
     /* preferred virt pci slot */
-    if ( slot >= PHP_SLOT_START && slot < PHP_SLOT_END )
+    if ( slot != AUTO_PHP_SLOT)
     {
-        php_slot = PCI_TO_PHP_SLOT(slot);
-        if ( !test_pci_slot(slot) )
-        {
+        if ( !test_pci_slot(slot) && !pci_devfn_in_use(e_bus, slot << 3) )
             goto found;
-        }
-        else
-            return -1;
+        return -2;
     }
 
-    if ( slot != 0 )
-        return -1;
-
     /* slot == 0, pick up a free one */
-    for ( i = 0; i < PHP_SLOT_LEN; i++ )
+    for ( slot = 0; slot < NR_PCI_DEV; slot++ )
     {
-        if ( !test_pci_slot(PHP_TO_PCI_SLOT(i)) )
-        {
-            php_slot = i;
+        if ( !test_pci_slot(slot) && !pci_devfn_in_use(e_bus, slot << 3) )
             goto found;
-        }
     }
 
     /* not found */
-    return 0;
+    return -1;
 
 found:
-    dpci_infos.php_devs[php_slot].valid  = 1;
-    dpci_infos.php_devs[php_slot].r_bus  = bus;
-    dpci_infos.php_devs[php_slot].r_dev  = dev;
-    dpci_infos.php_devs[php_slot].r_func = func;
-    dpci_infos.php_devs[php_slot].opt = opt;
-    return PHP_TO_PCI_SLOT(php_slot);
+    dpci_infos.php_devs[slot].valid  = 1;
+    dpci_infos.php_devs[slot].r_bus  = bus;
+    dpci_infos.php_devs[slot].r_dev  = dev;
+    dpci_infos.php_devs[slot].r_func = func;
+    dpci_infos.php_devs[slot].opt = opt;
+    return slot;
 }
 
 /* Insert a new pass-through device into a specific pci slot.
@@ -872,7 +862,7 @@ found:
 int insert_to_pci_slot(char *bdf_slt)
 {
     int seg, bus, dev, func, slot;
-    char *bdf_str, *slt_str, *opt;
+    char *bdf_str, *slt_str, *opt, *endptr;
     const char *delim="@";
 
     bdf_str = strsep(&bdf_slt, delim);
@@ -888,23 +878,20 @@ int insert_to_pci_slot(char *bdf_slt)
 
 }
 
-/* Test if a pci slot has a device
+/* Test if a pci slot has a PHP device
  * 1:  present
  * 0:  not present
- * -1: invalide pci slot input
+ * -1: invalid pci slot input
  */
 int test_pci_slot(int slot)
 {
-    int php_slot;
-
-    if ( slot < PHP_SLOT_START || slot >= PHP_SLOT_END )
+    if ( slot < 0 || slot >= NR_PCI_DEV )
         return -1;
 
-    php_slot = PCI_TO_PHP_SLOT(slot);
-    if ( dpci_infos.php_devs[php_slot].valid )
+    if ( dpci_infos.php_devs[slot].valid )
         return 1;
-    else
-        return 0;
+
+    return 0;
 }
 
 /* find the pci slot for pass-through dev with specified BDF */
@@ -919,15 +906,13 @@ int bdf_to_slot(char *bdf_str)
     }
 
     /* locate the virtual pci slot for this VTd device */
-    for ( i = 0; i < PHP_SLOT_LEN; i++ )
+    for ( i = 0; i < NR_PCI_DEV; i++ )
     {
         if ( dpci_infos.php_devs[i].valid &&
            dpci_infos.php_devs[i].r_bus == bus &&
            dpci_infos.php_devs[i].r_dev  == dev &&
            dpci_infos.php_devs[i].r_func == func )
-        {
-            return PHP_TO_PCI_SLOT(i);
-        }
+            return i;
     }
 
     return -1;
@@ -3562,7 +3547,7 @@ struct pt_dev * register_real_device(PCI
     struct pci_dev *pci_dev;
     uint8_t e_device, e_intx;
     struct pci_config_cf8 machine_bdf;
-    int free_pci_slot = -1;
+    int free_slot = -1;
     char *key, *val;
     int msi_translate;
 
@@ -3587,13 +3572,13 @@ struct pt_dev * register_real_device(PCI
 
     if ( e_devfn == PT_VIRT_DEVFN_AUTO ) {
         /*indicate a static assignment(not hotplug), so find a free PCI hot plug slot */
-        free_pci_slot = __insert_to_pci_slot(r_bus, r_dev, r_func, 0, NULL);
-        if ( free_pci_slot > 0 )
-            e_devfn = free_pci_slot  << 3;
-        else {
+        free_slot = __insert_to_pci_slot(r_bus, r_dev, r_func,
+                                         AUTO_PHP_SLOT, NULL);
+        if ( free_slot < 0 ) {
             PT_LOG("Error: no free virtual PCI hot plug slot, thus no live migration.\n");
             return NULL;
         }
+        e_devfn = free_slot << 3;
     }
 
     msi_translate = direct_pci_msitranslate;
@@ -3634,8 +3619,8 @@ struct pt_dev * register_real_device(PCI
         return NULL;
     }
 
-    if ( free_pci_slot > 0 )
-        dpci_infos.php_devs[PCI_TO_PHP_SLOT(free_pci_slot)].pt_dev = assigned_device;
+    if ( free_slot > 0 )
+        dpci_infos.php_devs[free_slot].pt_dev = assigned_device;
 
     assigned_device->pci_dev = pci_dev;
     assigned_device->msi_trans_cap = msi_translate;
@@ -3718,7 +3703,7 @@ out:
     return assigned_device;
 }
 
-int unregister_real_device(int php_slot)
+int unregister_real_device(int slot)
 {
     struct php_dev *php_dev;
     struct pci_dev *pci_dev;
@@ -3728,10 +3713,10 @@ int unregister_real_device(int php_slot)
     uint32_t bdf = 0;
     int rc = -1;
 
-    if ( test_pci_slot(PHP_TO_PCI_SLOT(php_slot)) != 1 )
+    if ( test_pci_slot(slot) != 1 )
        return -1;
 
-    php_dev = &dpci_infos.php_devs[php_slot];
+    php_dev = &dpci_infos.php_devs[slot];
     assigned_device = php_dev->pt_dev;
 
     if ( !assigned_device )
@@ -3789,15 +3774,14 @@ int unregister_real_device(int php_slot)
     return 0;
 }
 
-int power_on_php_slot(int php_slot)
+int power_on_php_slot(int slot)
 {
-    struct php_dev *php_dev = &dpci_infos.php_devs[php_slot];
-    int pci_slot = php_slot + PHP_SLOT_START;
+    struct php_dev *php_dev = &dpci_infos.php_devs[slot];
     struct pt_dev *pt_dev;
     pt_dev =
         register_real_device(dpci_infos.e_bus,
             "DIRECT PCI",
-            pci_slot << 3,
+            slot << 3,
             php_dev->r_bus,
             php_dev->r_dev,
             php_dev->r_func,
@@ -3820,7 +3804,7 @@ int power_off_php_slot(int php_slot)
 
 int pt_init(PCIBus *e_bus, const char *direct_pci)
 {
-    int seg, b, d, f, php_slot = 0, status = -1;
+    int seg, b, d, f, slot, status = -1;
     struct pt_dev *pt_dev;
     struct pci_access *pci_access;
     char *vslots;
@@ -3856,7 +3840,9 @@ int pt_init(PCIBus *e_bus, const char *d
     vslots = qemu_mallocz ( strlen(direct_pci) / 3 );
 
     /* Assign given devices to guest */
-    while ( next_bdf(&direct_pci_p, &seg, &b, &d, &f, &opt) )
+    for ( slot = 0;
+          slot < NR_PCI_DEV && next_bdf(&direct_pci_p, &seg, &b, &d, &f, &opt);
+          slot++ )
     {
         /* Register real device with the emulated bus */
         pt_dev = register_real_device(e_bus, "DIRECT PCI", PT_VIRT_DEVFN_AUTO,
@@ -3868,16 +3854,11 @@ int pt_init(PCIBus *e_bus, const char *d
         }
 
         /* Record the virtual slot info */
-        if ( php_slot < PHP_SLOT_LEN &&
-              dpci_infos.php_devs[php_slot].pt_dev == pt_dev )
-        {
-            sprintf(slot_str, "0x%x;", PHP_TO_PCI_SLOT(php_slot));
-        }
-        else
-            sprintf(slot_str, "0x%x;", 0);
+        sprintf(slot_str, "0x%02x;",
+                dpci_infos.php_devs[slot].pt_dev == pt_dev ? slot :
+                AUTO_PHP_SLOT);
 
         strcat(vslots, slot_str);
-        php_slot++;
     }
 
     /* Write virtual slots info to xenstore for Control panel use */
hw/pci.c +6 -2
Index: ioemu-remote/hw/pci.c
===================================================================
@@ -130,6 +130,11 @@ int pci_device_load(PCIDevice *s, QEMUFi
     return 0;
 }
 
+int pci_devfn_in_use(PCIBus *bus, int devfn)
+{
+        return bus->devices[devfn] ? 1 : 0;
+}
+
 /* -1 for devfn means auto assign */
 PCIDevice *pci_register_device(PCIBus *bus, const char *name,
                                int instance_size, int devfn,
@@ -140,8 +145,7 @@ PCIDevice *pci_register_device(PCIBus *b
 
     if (devfn < 0) {
         for(devfn = bus->devfn_min ; devfn < 256; devfn += 8) {
-            if ( !bus->devices[devfn] &&
-                 !( devfn >= PHP_DEVFN_START && devfn < PHP_DEVFN_END ) )
+            if ( !pci_devfn_in_use(bus, devfn) )
                 goto found;
         }
         return NULL;
hw/pci.h +4 -8
Index: ioemu-remote/hw/pci.h
===================================================================
@@ -66,6 +66,8 @@ struct PCIDevice {
 extern char direct_pci_str[];
 extern int direct_pci_msitranslate;
 
+int pci_devfn_in_use(PCIBus *bus, int devfn);
+
 PCIDevice *pci_register_device(PCIBus *bus, const char *name,
                                int instance_size, int devfn,
                                PCIConfigReadFunc *config_read,
@@ -101,14 +103,8 @@ void pci_info(void);
 PCIBus *pci_bridge_init(PCIBus *bus, int devfn, uint32_t id,
                         pci_map_irq_fn map_irq, const char *name);
 
-/* PCI slot 6~7 support ACPI PCI hot plug */
-#define PHP_SLOT_START  (6)
-#define PHP_SLOT_END    (8)
-#define PHP_SLOT_LEN    (PHP_SLOT_END - PHP_SLOT_START)
-#define PHP_TO_PCI_SLOT(x) (x + PHP_SLOT_START)
-#define PCI_TO_PHP_SLOT(x) (x - PHP_SLOT_START)
-#define PHP_DEVFN_START (PHP_SLOT_START << 3)
-#define PHP_DEVFN_END   (PHP_SLOT_END << 3)
+#define NR_PCI_DEV 32
+#define AUTO_PHP_SLOT NR_PCI_DEV
 
 int insert_to_pci_slot(char*);
 int test_pci_slot(int);
hw/piix4acpi.c +39 -29

168 lines not on this page · raw message

Signature · 4 lines
-- 
Simon Horman
  VA Linux Systems Japan K.K., Sydney, Australia Satellite Office
  H: www.vergenet.net/~horms/             W: www.valinux.co.jp/en
#3 lore raw
In reply to Simon Horman
Quoted 11 lines · Simon Horman

The current limit of 2 pass-through devices seems to be rather small. These patches remove that limit and allow any unused PCI device to be used for pass-through, including hotplug.

It seems that the first few (4) devices will be used for other purposes by HVM domains. However, it seems to me to be clean to make this code generic and cope with any device on the bus - saving the need for subsequent changes if the number of devices HVM domains use by default ever changes.

There are also xend and hvmloader patches required for this change. I will submit them separately.

Hi Ian,

please consider applying these patches as they are required for hotplug to work since the firmware and xend portions where applied to xen-unstable.hg as:

changeset:   19362:986d0ed147ff
user:        Keir Fraser <[email protected]>
date:        Tue Mar 17 10:48:52 2009 +0000
summary:     hvmloader: Allow any unused PCI device to be used for PCI

hotplug

changeset:   19361:6e3e841f0259
user:        Keir Fraser <[email protected]>
date:        Tue Mar 17 10:42:32 2009 +0000
summary:     xend: Use AUTO_PHP_SLOT_STR=0xff rather than 0x0 for auto pci

allocation

Thanks

Signature · 3 lines
Simon Horman
  VA Linux Systems Japan K.K., Sydney, Australia Satellite Office
  H: www.vergenet.net/~horms/             W: www.valinux.co.jp/en
#4 lore raw
Quoted 25 lines · Simon Horman
Quoted 11 lines · Simon Horman

The current limit of 2 pass-through devices seems to be rather small. These patches remove that limit and allow any unused PCI device to be used for pass-through, including hotplug.

It seems that the first few (4) devices will be used for other purposes by HVM domains. However, it seems to me to be clean to make this code generic and cope with any device on the bus - saving the need for subsequent changes if the number of devices HVM domains use by default ever changes.

There are also xend and hvmloader patches required for this change. I will submit them separately.

Hi Ian,

please consider applying these patches as they are required for hotplug to work since the firmware and xend portions where applied to xen-unstable.hg as:

changeset:   19362:986d0ed147ff
user:        Keir Fraser <[email protected]>
date:        Tue Mar 17 10:48:52 2009 +0000
summary:     hvmloader: Allow any unused PCI device to be used for PCI

hotplug

Quoted 8 lines
changeset:   19361:6e3e841f0259
user:        Keir Fraser <[email protected]>
date:        Tue Mar 17 10:42:32 2009 +0000
summary:     xend: Use AUTO_PHP_SLOT_STR=0xff rather than 0x0 for auto pci

allocation

Thanks

Hi,

I found a regression introduced by the 2nd patch, in the way that it treats vslots in pt_init.

pt_init() iterates through the PHP slots independantly of the assignment that occurs inside __insert_to_pci_slot which is called by register_real_device(). It assumes that vslots are assigned in order sarting at PHP_SLOT_START.

This was valid before my change, although why it didn't take the simpler option of just checking what value had been assigned to pt_dev->dev.devfn in register_real_device() is a mystery to me. Its also a mystery to what valid circumstance could lead to pt_init() using 0 (a.k.a. unknown?) as the vslot.

My patch made one the assumptions that pt_init() made about slot numbers invalid. That is, they don't start at PHP_SLOT_START, they start wherever there is a free device.

A simple solution seems to be to use the value assigned to pt_dev->dev.devfn in register_real_device().

The fix is as per the patch below. I will fold it into my previous patch series and repost.

hw/pass-through.c +3 -7
Index: ioemu-remote/hw/pass-through.c
===================================================================
@@ -3804,7 +3804,7 @@ int power_off_php_slot(int slot)
 
 int pt_init(PCIBus *e_bus, const char *direct_pci)
 {
-    int seg, b, d, f, slot, status = -1;
+    int seg, b, d, f, status = -1;
     struct pt_dev *pt_dev;
     struct pci_access *pci_access;
     char *vslots;
@@ -3840,9 +3840,7 @@ int pt_init(PCIBus *e_bus, const char *d
     vslots = qemu_mallocz ( strlen(direct_pci) / 3 );
 
     /* Assign given devices to guest */
-    for ( slot = 0;
-          slot < NR_PCI_DEV && next_bdf(&direct_pci_p, &seg, &b, &d, &f, &opt);
-          slot++ )
+    while ( next_bdf(&direct_pci_p, &seg, &b, &d, &f, &opt) )
     {
         /* Register real device with the emulated bus */
         pt_dev = register_real_device(e_bus, "DIRECT PCI", PT_VIRT_DEVFN_AUTO,
@@ -3854,9 +3852,7 @@ int pt_init(PCIBus *e_bus, const char *d
         }
 
         /* Record the virtual slot info */
-        sprintf(slot_str, "0x%02x;",
-                dpci_infos.php_devs[slot].pt_dev == pt_dev ? slot :
-                AUTO_PHP_SLOT);
+        sprintf(slot_str, "0x%02x;", PCI_SLOT(pt_dev->dev.devfn));
 
         strcat(vslots, slot_str);
     }