Skip to contentSkip to conversation

ioemu: specify the slot for pass-through devices

RFC In review Simon Horman 1 participant 6 messages Mar 20, 2009 – Mar 23, 2009

lore
Share & tools
lore
#0 lore raw

cover ioemu: specify the slot for pass-through devices

[ I am "reposting" these because yesterday's initial post
never made it off my PC ]

Currently a slot may be specified for a hot-plug device, but not for a pass-through device that is inserted at boot time. This patch adds support for the latter.

The syntax is:

     BUS:DEV.FUNC[@VSLOT]
     e.g: 0000:00:1d:0@7

This may be important as recent changes that allow any free PCI slot to be used for pass-through (and hotplug) may case pass-through devices to be assigned in different locations to before. Amongst other things, specifying the slot will allow users to move them back, if there is a need.

These changes need the patches that I posted today as the series "[patch 0/2] ioemu: Allow any unused PCI device to be used for pass-through"

There is also a xend portion of this patch, which will be posted separately.

Signature · 3 lines
Simon Horman
  VA Linux Systems Japan K.K., Sydney, Australia Satellite Office
  H: www.vergenet.net/~horms/             W: www.valinux.co.jp/en
#1 lore raw

RFC 1/4 ioemu: Register dpci_infos.php_devs.pt_dev in a common location

Currently power_on_php_slot() sets dpci_infos.php_devs[i].pt_dev itself after the call to register_real_device(). While pt_init relies on power_on_php_slot() to set this pointer.

It seems sensible to use the same behaviour for both callers.

There are no other callers of register_real_device()

hw/pass-through.c +2 -5
Index: ioemu-remote/hw/pass-through.c
===================================================================
@@ -3547,7 +3547,7 @@ struct pt_dev * register_real_device(PCI
     struct pci_dev *pci_dev;
     uint8_t e_device, e_intx;
     struct pci_config_cf8 machine_bdf;
-    int free_slot = -1;
+    int free_slot;
     char *key, *val;
     int msi_translate;
 
@@ -3619,8 +3619,7 @@ struct pt_dev * register_real_device(PCI
         return NULL;
     }
 
-    if ( free_slot > 0 )
-        dpci_infos.php_devs[free_slot].pt_dev = assigned_device;
+    dpci_infos.php_devs[PCI_SLOT(e_devfn, 0)].pt_dev = assigned_device;
 
     assigned_device->pci_dev = pci_dev;
     assigned_device->msi_trans_cap = msi_translate;
@@ -3791,8 +3790,6 @@ int power_on_php_slot(int slot)
 
     php_dev->opt = NULL;
 
-    php_dev->pt_dev = pt_dev;
-
     return 0;
 
 }
Signature · 4 lines
-- 
Simon Horman
  VA Linux Systems Japan K.K., Sydney, Australia Satellite Office
  H: www.vergenet.net/~horms/             W: www.valinux.co.jp/en
#2 lore raw

RFC 2/4 lSubject: ioemu: Do slot parsing inside of next_bdf

In reply to Simon Horman

Currently only hotplug provides vslot information from xend. A subsequent patch will have xend provide this information for boot-time inserted pass-through devices too.

With this in mind, this patch makes some infrastructure to parse bdf + slot information.

hw/pass-through.c +27 -15
Index: ioemu-remote/hw/pass-through.c
===================================================================
@@ -770,9 +770,10 @@ static int token_value(char *token)
     return strtol(token, NULL, 16);
 }
 
-static int next_bdf(char **str, int *seg, int *bus, int *dev, int *func, char **opt)
+static int next_bdf(char **str, int *seg, int *bus, int *dev, int *func,
+                    char **opt, int *vslot)
 {
-    char *token;
+    char *token, *endptr;
     const char *delim = ":.-";
 
     if ( !(*str) ||
@@ -789,12 +790,28 @@ static int next_bdf(char **str, int *seg
     *dev  = token_value(token);
 
     token  = strsep(str, delim);
-    *opt = strchr(token, ',');
+
+    *opt = strchr(token, '@');
+    if (*opt)
+    {
+        *(*opt)++ = '\0';
+        *vslot = token_value(*opt);
+    }
+    else
+    {
+        *vslot = AUTO_PHP_SLOT;
+        *opt = token;
+    }
+
+    *opt = strchr(*opt, ',');
     if (*opt)
         *(*opt)++ = '\0';
 
     *func  = token_value(token);
 
+    token = strsep(str, delim);
+    *vslot  = token_value(token);
+
     return 1;
 }
 
@@ -862,14 +879,9 @@ found:
 int insert_to_pci_slot(char *bdf_slt)
 {
     int seg, bus, dev, func, slot;
-    char *bdf_str, *slt_str, *opt, *endptr;
-    const char *delim="@";
-
-    bdf_str = strsep(&bdf_slt, delim);
-    slt_str = bdf_slt;
-    slot = token_value(slt_str);
+    char *opt;
 
-    if ( !next_bdf(&bdf_str, &seg, &bus, &dev, &func, &opt))
+    if ( !next_bdf(&bdf_slt, &seg, &bus, &dev, &func, &opt, &slot) )
     {
         return -1;
     }
@@ -897,10 +909,10 @@ int test_pci_slot(int slot)
 /* find the pci slot for pass-through dev with specified BDF */
 int bdf_to_slot(char *bdf_str)
 {
-    int seg, bus, dev, func, i;
+    int seg, bus, dev, func, slot, i;
     char *opt;
 
-    if ( !next_bdf(&bdf_str, &seg, &bus, &dev, &func, &opt))
+    if ( !next_bdf(&bdf_str, &seg, &bus, &dev, &func, &opt, &slot))
     {
         return -1;
     }
@@ -3619,7 +3631,7 @@ struct pt_dev * register_real_device(PCI
         return NULL;
     }
 
-    dpci_infos.php_devs[PCI_SLOT(e_devfn, 0)].pt_dev = assigned_device;
+    dpci_infos.php_devs[PCI_SLOT(e_devfn)].pt_dev = assigned_device;
 
     assigned_device->pci_dev = pci_dev;
     assigned_device->msi_trans_cap = msi_translate;
@@ -3801,7 +3813,7 @@ int power_off_php_slot(int slot)
 
 int pt_init(PCIBus *e_bus, const char *direct_pci)
 {
-    int seg, b, d, f, status = -1;
+    int seg, b, d, f, s, status = -1;
     struct pt_dev *pt_dev;
     struct pci_access *pci_access;
     char *vslots;
@@ -3837,7 +3849,7 @@ int pt_init(PCIBus *e_bus, const char *d
     vslots = qemu_mallocz ( strlen(direct_pci) / 3 );
 
     /* Assign given devices to guest */
-    while ( next_bdf(&direct_pci_p, &seg, &b, &d, &f, &opt) )
+    while ( next_bdf(&direct_pci_p, &seg, &b, &d, &f, &opt, &s) )
     {
         /* Register real device with the emulated bus */
         pt_dev = register_real_device(e_bus, "DIRECT PCI", PT_VIRT_DEVFN_AUTO,
Signature · 4 lines
-- 
Simon Horman
  VA Linux Systems Japan K.K., Sydney, Australia Satellite Office
  H: www.vergenet.net/~horms/             W: www.valinux.co.jp/en
#3 lore raw

Re: [rfc 2/4] ioemu: Do slot parsing inside of next_bdf

Currently only hotplug provides vslot information from xend. A subsequent patch will have xend provide this information for boot-time inserted pass-through devices too.

With this in mind, this patch makes some infrastructure to parse bdf + slot information.

---

* Sat, 21 Mar 2009 09:24:37 +1100
Initial Public Release

* Mon, 23 Mar 2009 20:38:40 +1100
Remove a bogus second assignment of *vslot* inside next_bdf(). Unfortunately this crept into the version I posted while my testing was against a versino without it.

hw/pass-through.c +24 -15
Index: ioemu-remote/hw/pass-through.c
===================================================================
@@ -770,9 +770,10 @@ static int token_value(char *token)
     return strtol(token, NULL, 16);
 }
 
-static int next_bdf(char **str, int *seg, int *bus, int *dev, int *func, char **opt)
+static int next_bdf(char **str, int *seg, int *bus, int *dev, int *func,
+                    char **opt, int *vslot)
 {
-    char *token;
+    char *token, *endptr;
     const char *delim = ":.-";
 
     if ( !(*str) ||
@@ -789,7 +790,20 @@ static int next_bdf(char **str, int *seg
     *dev  = token_value(token);
 
     token  = strsep(str, delim);
-    *opt = strchr(token, ',');
+
+    *opt = strchr(token, '@');
+    if (*opt)
+    {
+        *(*opt)++ = '\0';
+        *vslot = token_value(*opt);
+    }
+    else
+    {
+        *vslot = AUTO_PHP_SLOT;
+        *opt = token;
+    }
+
+    *opt = strchr(*opt, ',');
     if (*opt)
         *(*opt)++ = '\0';
 
@@ -862,14 +876,9 @@ found:
 int insert_to_pci_slot(char *bdf_slt)
 {
     int seg, bus, dev, func, slot;
-    char *bdf_str, *slt_str, *opt, *endptr;
-    const char *delim="@";
-
-    bdf_str = strsep(&bdf_slt, delim);
-    slt_str = bdf_slt;
-    slot = token_value(slt_str);
+    char *opt;
 
-    if ( !next_bdf(&bdf_str, &seg, &bus, &dev, &func, &opt))
+    if ( !next_bdf(&bdf_slt, &seg, &bus, &dev, &func, &opt, &slot) )
     {
         return -1;
     }
@@ -897,10 +906,10 @@ int test_pci_slot(int slot)
 /* find the pci slot for pass-through dev with specified BDF */
 int bdf_to_slot(char *bdf_str)
 {
-    int seg, bus, dev, func, i;
+    int seg, bus, dev, func, slot, i;
     char *opt;
 
-    if ( !next_bdf(&bdf_str, &seg, &bus, &dev, &func, &opt))
+    if ( !next_bdf(&bdf_str, &seg, &bus, &dev, &func, &opt, &slot))
     {
         return -1;
     }
@@ -3619,7 +3628,7 @@ struct pt_dev * register_real_device(PCI
         return NULL;
     }
 
-    dpci_infos.php_devs[PCI_SLOT(e_devfn, 0)].pt_dev = assigned_device;
+    dpci_infos.php_devs[PCI_SLOT(e_devfn)].pt_dev = assigned_device;
 
     assigned_device->pci_dev = pci_dev;
     assigned_device->msi_trans_cap = msi_translate;
@@ -3801,7 +3810,7 @@ int power_off_php_slot(int slot)
 
 int pt_init(PCIBus *e_bus, const char *direct_pci)
 {
-    int seg, b, d, f, status = -1;
+    int seg, b, d, f, s, status = -1;
     struct pt_dev *pt_dev;
     struct pci_access *pci_access;
     char *vslots;
@@ -3837,7 +3846,7 @@ int pt_init(PCIBus *e_bus, const char *d
     vslots = qemu_mallocz ( strlen(direct_pci) / 3 );
 
     /* Assign given devices to guest */
-    while ( next_bdf(&direct_pci_p, &seg, &b, &d, &f, &opt) )
+    while ( next_bdf(&direct_pci_p, &seg, &b, &d, &f, &opt, &s) )
     {
         /* Register real device with the emulated bus */
         pt_dev = register_real_device(e_bus, "DIRECT PCI", PT_VIRT_DEVFN_AUTO,
#4 lore raw

RFC 3/4 ioemu: make __insert_to_pci_slot indempotent

In reply to Simon Horman
hw/pass-through.c +14 -4
Index: ioemu-remote/hw/pass-through.c
===================================================================
@@ -835,6 +835,16 @@ static int get_next_keyval(char **option
     return 0;
 }
 
+static int pci_slot_match(int bus, int dev, int func, int slot)
+{
+    if (test_pci_slot(slot) == 1 &&
+        dpci_infos.php_devs[slot].r_bus == bus &&
+        dpci_infos.php_devs[slot].r_dev  == dev &&
+        dpci_infos.php_devs[slot].r_func == func )
+        return 1;
+    return 0;
+}
+
 /* Insert a new pass-through device into a specific pci slot.
  * input  dom:bus:dev.func@slot, chose free one if slot == AUTO_PHP_SLOT
  * return -2: requested slot not available
@@ -851,6 +861,9 @@ static int __insert_to_pci_slot(int bus,
     {
         if ( !test_pci_slot(slot) && !pci_devfn_in_use(e_bus, slot << 3) )
             goto found;
+        if ( pci_slot_match(bus, dev, func, slot) )
+            /* The slot is already here, just return */
+            return slot;
         return -2;
     }
 
@@ -920,10 +933,7 @@ int bdf_to_slot(char *bdf_str)
     /* locate the virtual pci slot for this VTd device */
     for ( i = 0; i < NR_PCI_DEV; i++ )
     {
-        if ( dpci_infos.php_devs[i].valid &&
-           dpci_infos.php_devs[i].r_bus == bus &&
-           dpci_infos.php_devs[i].r_dev  == dev &&
-           dpci_infos.php_devs[i].r_func == func )
+        if ( pci_slot_match(bus, dev, func, slot) )
             return i;
     }
 
Signature · 4 lines
-- 
Simon Horman
  VA Linux Systems Japan K.K., Sydney, Australia Satellite Office
  H: www.vergenet.net/~horms/             W: www.valinux.co.jp/en
#5 lore raw

RFC 4/4 ioemu: allow xend to specify the slot for pass-through devices

In reply to Simon Horman

Currently a slot may be specified for a hot-plug device, but not for a pass-through device that is inserted at boot time. This patch adds support for the latter.

The syntax is:
BUS:DEV.FUNC[@VSLOT]
e.g: 0000:00:1d:0@7

This may be important as recent changes that allow any free PCI slot to be used for pass-through (and hotplug) may case pass-through devices to be assigned in different locations to before. Amongst other things, specifying the slot will allow users to move them back, if there is a need.

There is also a xend portion of this patch, which will be posted separately.

---

Some implementation details

* Pass the slot parsed from xend to register_real_device(). If none was passed, next_bdf() will supply AUTO_PHP_SLOT.

* Make use of the fact that __insert_to_pci_slot() is now indempotent and always call it in register_real_device()

* Change the error message int he case where __insert_to_pci_slot() fails to not refer to hot-plut. e_devfn will always
be non-auto in the hotplug case as register_real_device() is called by power_on_php_slot() which is called by acpi_php_add() which is called by acpi_php_add(). acpi_php_add() handles calling __insert_to_pci_slot() and acpi_php_add() handles the case where the call failed, before calling power_on_php_slot().

* Remove PT_VIRT_DEVFN_AUTO, it is no longer used.

hw/pass-through.c +9 -15
Index: ioemu-remote/hw/pass-through.c
===================================================================
@@ -3560,7 +3560,7 @@ static int pt_pmcsr_reg_restore(struct p
 }
 
 struct pt_dev * register_real_device(PCIBus *e_bus,
-        const char *e_dev_name, int e_devfn, uint8_t r_bus, uint8_t r_dev,
+        const char *e_dev_name, int e_slot, uint8_t r_bus, uint8_t r_dev,
         uint8_t r_func, uint32_t machine_irq, struct pci_access *pci_access,
         char *opt)
 {
@@ -3569,7 +3569,6 @@ struct pt_dev * register_real_device(PCI
     struct pci_dev *pci_dev;
     uint8_t e_device, e_intx;
     struct pci_config_cf8 machine_bdf;
-    int free_slot;
     char *key, *val;
     int msi_translate;
 
@@ -3592,15 +3591,10 @@ struct pt_dev * register_real_device(PCI
     pci_fill_info(pci_dev, PCI_FILL_IRQ | PCI_FILL_BASES | PCI_FILL_ROM_BASE | PCI_FILL_SIZES);
     pt_libpci_fixup(pci_dev);
 
-    if ( e_devfn == PT_VIRT_DEVFN_AUTO ) {
-        /*indicate a static assignment(not hotplug), so find a free PCI hot plug slot */
-        free_slot = __insert_to_pci_slot(r_bus, r_dev, r_func,
-                                         AUTO_PHP_SLOT, NULL);
-        if ( free_slot < 0 ) {
-            PT_LOG("Error: no free virtual PCI hot plug slot, thus no live migration.\n");
-            return NULL;
-        }
-        e_devfn = free_slot << 3;
+    e_slot = __insert_to_pci_slot(r_bus, r_dev, r_func, e_slot, NULL);
+    if ( e_slot < 0 ) {
+        PT_LOG("Error: no free virtual PCI slot\n");
+        return NULL;
     }
 
     msi_translate = direct_pci_msitranslate;
@@ -3633,7 +3627,7 @@ struct pt_dev * register_real_device(PCI
 
     /* Register device */
     assigned_device = (struct pt_dev *) pci_register_device(e_bus, e_dev_name,
-                                sizeof(struct pt_dev), e_devfn,
+                                sizeof(struct pt_dev), PCI_DEVFN(e_slot, 0),
                                 pt_pci_read_config, pt_pci_write_config);
     if ( assigned_device == NULL )
     {
@@ -3641,7 +3635,7 @@ struct pt_dev * register_real_device(PCI
         return NULL;
     }
 
-    dpci_infos.php_devs[PCI_SLOT(e_devfn)].pt_dev = assigned_device;
+    dpci_infos.php_devs[e_slot].pt_dev = assigned_device;
 
     assigned_device->pci_dev = pci_dev;
     assigned_device->msi_trans_cap = msi_translate;
@@ -3802,7 +3796,7 @@ int power_on_php_slot(int slot)
     pt_dev =
         register_real_device(dpci_infos.e_bus,
             "DIRECT PCI",
-            slot << 3,
+            slot,
             php_dev->r_bus,
             php_dev->r_dev,
             php_dev->r_func,
@@ -3862,7 +3856,7 @@ int pt_init(PCIBus *e_bus, const char *d
     while ( next_bdf(&direct_pci_p, &seg, &b, &d, &f, &opt, &s) )
     {
         /* Register real device with the emulated bus */
-        pt_dev = register_real_device(e_bus, "DIRECT PCI", PT_VIRT_DEVFN_AUTO,
+        pt_dev = register_real_device(e_bus, "DIRECT PCI", s,
             b, d, f, PT_MACHINE_IRQ_AUTO, pci_access, opt);
         if ( pt_dev == NULL )
         {
hw/pass-through.h +0 -1
Index: ioemu-remote/hw/pass-through.h
===================================================================
@@ -39,7 +39,6 @@
 // #define PT_DEBUG_PCI_CONFIG_ACCESS
 
 #define PT_MACHINE_IRQ_AUTO (0xFFFFFFFF)
-#define PT_VIRT_DEVFN_AUTO  (-1)
 
 /* Misc PCI constants that should be moved to a separate library :) */
 #define PCI_CONFIG_SIZE         (256)
Signature · 4 lines
-- 
Simon Horman
  VA Linux Systems Japan K.K., Sydney, Australia Satellite Office
  H: www.vergenet.net/~horms/             W: www.valinux.co.jp/en