Skip to contentSkip to conversation

xend: Update info['platform']['pci']

PATCH In review Masaki Kanno 3 participants 8 messages May 28, 2009 – Jun 1, 2009

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 xend: Update info['platform']['pci']

Hi,

This patch updates info['platform']['pci'] for PCI devices assignment to domains.

When a domain is started, xend confirms by using xc.test_assign_device whether PCI devices can be assigned to the domain.
For the confirmation, info['platform']['pci'] must be an appropriate value. However, info['platform']['pci'] may be not appropriate. Because info['platform']['pci'] isn't almost always updated even if the PCI device configuration of the domain was changed by using xm pci-attach/detach.
This patch updates info['platform']['pci'] to the appropriate value when domains are started.

Best regards,
Kan

tools/python/xen/util/pci.py +12 -4
diff -r fe68405201d2 tools/python/xen/util/pci.py
@@ -14,6 +14,7 @@
 import time
 import threading
 from xen.util import utils
+from xen.xend import sxp
 
 PROC_PCI_PATH = '/proc/bus/pci/devices'
 PROC_PCI_NUM_RESOURCES = 7
@@ -140,10 +141,17 @@
     return (domain, bus, slot, func)
 
 def assigned_or_requested_vslot(dev):
-    if dev.has_key("vslot"):
-        return dev["vslot"]
-    if dev.has_key("requested_vslot"):
-        return dev["requested_vslot"]
+    if isinstance(dev, types.DictType):
+        if dev.has_key("vslot"):
+            return dev["vslot"]
+        if dev.has_key("requested_vslot"):
+            return dev["requested_vslot"]
+    elif isinstance(dev, (types.ListType, types.TupleType)):
+        vslot = sxp.child_value(dev, 'vslot', None)
+        if not vslot:
+            vslot = sxp.child_value(dev, 'requested_vslot', None)
+        if vslot:
+            return vslot
     raise PciDeviceVslotMissing("%s" % dev)
 
 def find_sysfs_mnt():
tools/python/xen/xend/XendConfig.py +24 -0
diff -r fe68405201d2 tools/python/xen/xend/XendConfig.py
@@ -37,6 +37,7 @@
 from xen.xend.server.BlktapController import blktap_disk_types
 from xen.xend.server.netif import randomMAC
 from xen.util.blkif import blkdev_name_to_number, blkdev_uname_to_file
+from xen.util.pci import assigned_or_requested_vslot
 from xen.util import xsconstants
 import xen.util.auxbin
 
@@ -2186,3 +2187,26 @@
 
     def is_hap(self):
         return self['platform'].get('hap', 0)
+
+    def update_platform_pci(self):
+        if not self.is_hvm():
+            return
+
+        pci = []
+        for dev_type, dev_info in self.all_devices_sxpr():
+            if dev_type != 'pci':
+                continue
+            for dev in sxp.children(dev_info, 'dev'):
+                domain = sxp.child_value(dev, 'domain')
+                bus = sxp.child_value(dev, 'bus')
+                slot = sxp.child_value(dev, 'slot')
+                func = sxp.child_value(dev, 'func')
+                vslot = assigned_or_requested_vslot(dev) 
+                opts = ''
+                for opt in sxp.child_value(dev, 'opts', []):
+                    if opts:
+                        opts += ','
+                    opts += '%s=%s' % (opt[0], str(opt[1]))
+                pci.append([domain, bus, slot, func, vslot, opts])
+        self['platform']['pci'] = pci
+
tools/python/xen/xend/XendDomainInfo.py +3 -20
diff -r fe68405201d2 tools/python/xen/xend/XendDomainInfo.py
@@ -893,18 +893,9 @@
             if num_devs == 0:
                 if self.info.is_hvm():
                     self.destroyDevice('pci', devid, True)
-                    del self.info['devices'][dev_uuid]
-                    platform = self.info['platform']
-                    orig_dev_num = len(platform['pci'])
-                    # TODO: can use this to keep some info to ask high level
-                    # management tools to hot insert a new passthrough dev
-                    # after migration
-                    if orig_dev_num != 0:
-                        #platform['pci'] = ["%dDEVs" % orig_dev_num]
-                        platform['pci'] = []
                 else:
                     self.destroyDevice('pci', devid)
-                    del self.info['devices'][dev_uuid]
+                del self.info['devices'][dev_uuid]
         else:
             new_dev_sxp = ['pci']
             for cur_dev in sxp.children(existing_dev_info, 'dev'):
@@ -923,18 +914,9 @@
             dev_uuid = sxp.child_value(existing_dev_info, 'uuid')
             self.info.device_update(dev_uuid, new_dev_sxp)
 
-            # If there is only 'vscsi' in new_dev_sxp, remove the config.
+            # If there is no device left, remove config.
             if len(sxp.children(new_dev_sxp, 'dev')) == 0:
                 del self.info['devices'][dev_uuid]
-                if self.info.is_hvm():
-                    platform = self.info['platform']
-                    orig_dev_num = len(platform['pci'])
-                    # TODO: can use this to keep some info to ask high level
-                    # management tools to hot insert a new passthrough dev
-                    # after migration
-                    if orig_dev_num != 0:
-                        #platform['pci'] = ["%dDEVs" % orig_dev_num]
-                        platform['pci'] = []
 
         xen.xend.XendDomain.instance().managed_config_save(self)
 
@@ -2463,6 +2445,7 @@
                               (self.getVCpuCount() * 100))
 
         # Test whether the devices can be assigned with VT-d
+        self.info.update_platform_pci()
         pci = self.info["platform"].get("pci")
         pci_str = ''
         if pci and len(pci) > 0:
tools/python/xen/xend/server/pciif.py +13 -2
diff -r fe68405201d2 tools/python/xen/xend/server/pciif.py
@@ -223,6 +223,11 @@
                     except IndexError:
                         dev_dict['vslot'] = AUTO_PHP_SLOT_STR
 
+                #append opts info
+                opts = self.readBackend(devid, 'opts-%d' % i)
+                if opts is not None:
+                    dev_dict['opts'] = opts
+
                 pci_devs.append(dev_dict)
 
         result['devs'] = pci_devs
@@ -243,8 +248,14 @@
         
         for dev in devs:
             dev_sxpr = ['dev']
-            for dev_item in dev.items():
-                dev_sxpr.append(list(dev_item))
+            for dev_key, dev_val in dev.items():
+                if dev_key == 'opts':
+                    opts = []
+                    for opt in dev_val.split(','):
+                        opts.append(opt.split('='))
+                    dev_sxpr.append(['opts', opts])
+                else:
+                    dev_sxpr.append([dev_key, dev_val])
             sxpr.append(dev_sxpr)
         
         for key, val in configDict.items():

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#1 lore raw

On Thu, May 28, 2009 at 04:01:50PM +0900, Masaki Kanno wrote:
Content-Description: Mail message body

Quoted 5 lines

Hi,

This patch updates info['platform']['pci'] for PCI devices assignment to domains.

Quoted 9 lines

When a domain is started, xend confirms by using xc.test_assign_device whether PCI devices can be assigned to the domain.
For the confirmation, info['platform']['pci'] must be an appropriate value. However, info['platform']['pci'] may be not appropriate. Because info['platform']['pci'] isn't almost always updated even if the PCI device configuration of the domain was changed by using xm pci-attach/detach.
This patch updates info['platform']['pci'] to the appropriate value when domains are started.

Hi Kanno-san,

I am curious to know how you exercise this code.
Did you notice a bug under some conditions?

#2 lore raw
Quoted 17 lines · Simon Horman

On Thu, May 28, 2009 at 04:01:50PM +0900, Masaki Kanno wrote:
Content-Description: Mail message body

Quoted 5 lines

Hi,

This patch updates info['platform']['pci'] for PCI devices assignment to domains.

Quoted 9 lines

When a domain is started, xend confirms by using xc.test_assign_device whether PCI devices can be assigned to the domain.
For the confirmation, info['platform']['pci'] must be an appropriate value. However, info['platform']['pci'] may be not appropriate. Because info['platform']['pci'] isn't almost always updated even if the PCI device configuration of the domain was changed by using xm pci-attach/detach.
This patch updates info['platform']['pci'] to the appropriate value when domains are started.

Quoted 4 lines

Hi Kanno-san,

I am curious to know how you exercise this code.
Did you notice a bug under some conditions?

Hi Simon,

Sorry for late response.

I noticed the bug when I tried the following operations.

1) xm new vm1
2) xm list --long vm1

    (domain
    <<snip>>
        (image
            (hvm
    <<snip>>
                (pci ((0x0 0x01 0x00 0x0 0x6 )))
 3) xm pci-attach vm1 00:1a.0 7
 4) xm list --long vm1
    (domain
    <<snip>>
        (image
            (hvm
    <<snip>>
                (pci ((0x0 0x01 0x00 0x0 0x6 )))

The value of /domain/image/hvm/pci is a value of info['platform']['pci']. The value wasn't updated when I attached the PCI device to the domain by using xm pci-attach.

The value is updated by my patch. Nonetheless, the value isn't still updated when xm pci-attach/detach are executed. The value is updated when starting domains.

Best regards,
Kan

#3 lore raw
Quoted 47 lines · Masaki Kanno
Quoted 17 lines · Simon Horman

On Thu, May 28, 2009 at 04:01:50PM +0900, Masaki Kanno wrote:
Content-Description: Mail message body

Quoted 5 lines

Hi,

This patch updates info['platform']['pci'] for PCI devices assignment to domains.

Quoted 9 lines

When a domain is started, xend confirms by using xc.test_assign_device whether PCI devices can be assigned to the domain.
For the confirmation, info['platform']['pci'] must be an appropriate value. However, info['platform']['pci'] may be not appropriate. Because info['platform']['pci'] isn't almost always updated even if the PCI device configuration of the domain was changed by using xm pci-attach/detach.
This patch updates info['platform']['pci'] to the appropriate value when domains are started.

Quoted 4 lines

Hi Kanno-san,

I am curious to know how you exercise this code.
Did you notice a bug under some conditions?

Hi Simon,

Sorry for late response.

I noticed the bug when I tried the following operations.

1) xm new vm1
2) xm list --long vm1

    (domain
    <<snip>>
        (image
            (hvm
    <<snip>>
                (pci ((0x0 0x01 0x00 0x0 0x6 )))
 3) xm pci-attach vm1 00:1a.0 7
 4) xm list --long vm1
    (domain
    <<snip>>
        (image
            (hvm
    <<snip>>
                (pci ((0x0 0x01 0x00 0x0 0x6 )))
Quoted 7 lines

The value of /domain/image/hvm/pci is a value of info['platform']['pci']. The value wasn't updated when I attached the PCI device to the domain by using xm pci-attach.

The value is updated by my patch. Nonetheless, the value isn't still updated when xm pci-attach/detach are executed. The value is updated when starting domains.

Understood. Do you think that also needs to be fixed?

#4 lore raw
Quoted 52 lines · Simon Horman
Quoted 51 lines · Masaki Kanno
Quoted 17 lines · Simon Horman

On Thu, May 28, 2009 at 04:01:50PM +0900, Masaki Kanno wrote:
Content-Description: Mail message body

Quoted 5 lines

Hi,

This patch updates info['platform']['pci'] for PCI devices assignment to domains.

Quoted 9 lines

When a domain is started, xend confirms by using xc.test_assign_device whether PCI devices can be assigned to the domain.
For the confirmation, info['platform']['pci'] must be an appropriate value. However, info['platform']['pci'] may be not appropriate. Because info['platform']['pci'] isn't almost always updated even if the PCI device configuration of the domain was changed by using xm pci-attach/detach.
This patch updates info['platform']['pci'] to the appropriate value when domains are started.

Quoted 4 lines

Hi Kanno-san,

I am curious to know how you exercise this code.
Did you notice a bug under some conditions?

Hi Simon,

Sorry for late response.

I noticed the bug when I tried the following operations.

1) xm new vm1
2) xm list --long vm1

    (domain
    <<snip>>
        (image
            (hvm
    <<snip>>
                (pci ((0x0 0x01 0x00 0x0 0x6 )))
 3) xm pci-attach vm1 00:1a.0 7
 4) xm list --long vm1
    (domain
    <<snip>>
        (image
            (hvm
    <<snip>>
                (pci ((0x0 0x01 0x00 0x0 0x6 )))

The value of /domain/image/hvm/pci is a value of info['platform']['pci']. The value wasn't updated when I attached the PCI device to the domain by using xm pci-attach.

Quoted 5 lines
Quoted 3 lines

The value is updated by my patch. Nonetheless, the value isn't still updated when xm pci-attach/detach are executed. The value is updated when starting domains.

Understood. Do you think that also needs to be fixed?

Hi Simon,

For now, I think that doesn't need to be fixed.

Best regards,
Kan

#5 lore raw
In reply to Masaki Kanno

Masaki,
How about removing the 'pci' config from the info['platform']? We already have the info['devices'] to kept the pci/vif/disk config, so duplicated info in ['platform']['pci'] would cause synchronization troubles.

There are only several places using info['platform']['pci'], so take place of them with _getDeviceInfo_pci('0') should be okay.

Thanks,

Quoted 15 lines · Masaki Kanno

Hi,

This patch updates info['platform']['pci'] for PCI devices assignment to domains.

When a domain is started, xend confirms by using xc.test_assign_device whether PCI devices can be assigned to the domain.
For the confirmation, info['platform']['pci'] must be an appropriate value. However, info['platform']['pci'] may be not appropriate. Because info['platform']['pci'] isn't almost always updated even if the PCI device configuration of the domain was changed by using xm pci-attach/detach.
This patch updates info['platform']['pci'] to the appropriate value when domains are started.

Quoted 6 lines

Best regards,
Kan

Content-Description: ATT00001.txt

Quoted 4 lines

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Signature · 2 lines
best rgds,
edwin
#6 lore raw
Quoted 10 lines · Edwin

Masaki,
How about removing the 'pci' config from the info['platform']? We already have
the info['devices'] to kept the pci/vif/disk config, so duplicated info in ['platform']['pci'] would cause synchronization troubles.

There are only several places using info['platform']['pci'], so take place of
them with _getDeviceInfo_pci('0') should be okay.

Hi Edwin,

As far as I know, info['platform']['pci'] is referred in the following places.
- [email protected]
- [email protected]

Are there any others than the above? If there are others, I will confirm them.

Best regards,
Kan

Quoted 39 lines · Masaki Kanno
Quoted 15 lines

Hi,

This patch updates info['platform']['pci'] for PCI devices assignment to domains.

When a domain is started, xend confirms by using xc.test_assign_device whether PCI devices can be assigned to the domain.
For the confirmation, info['platform']['pci'] must be an appropriate value. However, info['platform']['pci'] may be not appropriate. Because info['platform']['pci'] isn't almost always updated even if the PCI device configuration of the domain was changed by using xm pci-attach/detach.
This patch updates info['platform']['pci'] to the appropriate value when domains are started.

Quoted 6 lines

Best regards,
Kan

Content-Description: ATT00001.txt

Quoted 4 lines

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Signature · 7 lines
best rgds,
edwin

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
#7 lore raw

Masaki,
Attached is my patch to fix another bug caused by wrong synchronization of info['platform']['pci']. Just FYI. I don't know if it still can apply on unstable. We also need make sure old saved image can be restored with this fix.

Thanks,

Quoted 55 lines · Masaki Kanno
Quoted 6 lines · Edwin

Masaki,
How about removing the 'pci' config from the info['platform']? We already have the info['devices'] to kept the pci/vif/disk config, so duplicated info in ['platform']['pci'] would cause synchronization troubles.

There are only several places using info['platform']['pci'], so take place of them with _getDeviceInfo_pci('0') should be okay.

Hi Edwin,

As far as I know, info['platform']['pci'] is referred in the following places.
- [email protected]
- [email protected]

Are there any others than the above? If there are others, I will confirm them.

Best regards,
Kan

Quoted 31 lines · Masaki Kanno
Quoted 5 lines

Hi,

This patch updates info['platform']['pci'] for PCI devices assignment to domains.

Quoted 9 lines

When a domain is started, xend confirms by using xc.test_assign_device whether PCI devices can be assigned to the domain. For the confirmation, info['platform']['pci'] must be an appropriate value. However, info['platform']['pci'] may be not appropriate. Because info['platform']['pci'] isn't almost always updated even if the PCI device configuration of the domain was changed by using xm pci-attach/detach. This patch updates info['platform']['pci'] to the appropriate value when domains are started.

Best regards,
Kan

Content-Description: ATT00001.txt

Quoted 5 lines

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Signature · 7 lines
best rgds,
edwin

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Signature · 2 lines
best rgds,
edwin
tools/python/xen/xend/XendDomainInfo.py +12 -23
Index: hv/tools/python/xen/xend/XendDomainInfo.py
===================================================================
@@ -851,18 +851,9 @@ class XendDomainInfo:
             if num_devs == 0:
                 if self.info.is_hvm():
                     self.destroyDevice('pci', devid, True)
-                    del self.info['devices'][dev_uuid]
-                    platform = self.info['platform']
-                    orig_dev_num = len(platform['pci'])
-                    # TODO: can use this to keep some info to ask high level
-                    # management tools to hot insert a new passthrough dev
-                    # after migration
-                    if orig_dev_num != 0:
-                        #platform['pci'] = ["%dDEVs" % orig_dev_num]
-                        platform['pci'] = []
                 else:
                     self.destroyDevice('pci', devid)
-                    del self.info['devices'][dev_uuid]
+                del self.info['devices'][dev_uuid]
         else:
             new_dev_sxp = ['pci']
             for cur_dev in sxp.children(existing_dev_info, 'dev'):
@@ -884,15 +875,6 @@ class XendDomainInfo:
             # If there is only 'vscsi' in new_dev_sxp, remove the config.
             if len(sxp.children(new_dev_sxp, 'dev')) == 0:
                 del self.info['devices'][dev_uuid]
-                if self.info.is_hvm():
-                    platform = self.info['platform']
-                    orig_dev_num = len(platform['pci'])
-                    # TODO: can use this to keep some info to ask high level
-                    # management tools to hot insert a new passthrough dev
-                    # after migration
-                    if orig_dev_num != 0:
-                        #platform['pci'] = ["%dDEVs" % orig_dev_num]
-                        platform['pci'] = []
 
         xen.xend.XendDomain.instance().managed_config_save(self)
 
@@ -2388,11 +2370,18 @@ class XendDomainInfo:
                               (self.getVCpuCount() * 100))
 
         # Test whether the devices can be assigned with VT-d
-        pci = self.info["platform"].get("pci")
         pci_str = ''
-        if pci and len(pci) > 0:
-            pci = map(lambda x: x[0:4], pci)  # strip options 
-            pci_str = str(pci)
+        pci_list = []
+        devid = '0'
+        dev_info = self._getDeviceInfo_pci(devid)
+        if dev_info is not None:
+            dev_uuid = sxp.child_value(dev_info, 'uuid')
+            pci_conf = self.info['devices'][dev_uuid][1]
+            pci_devs = pci_conf['devs']
+            for x in pci_devs:
+                pci_list.append([x['domain'], x['bus'], x['slot'], x['func']])
+            pci_str = str(pci_list)
+
         if hvm and pci_str:
             bdf = xc.test_assign_device(0, pci_str)
             if bdf != 0:
tools/python/xen/xend/XendConfig.py +0 -2
Index: hv/tools/python/xen/xend/XendConfig.py
===================================================================
@@ -163,7 +163,6 @@ XENAPI_PLATFORM_CFG_TYPES = {
     'vncpasswd': str,
     'vncunused': int,
     'xauthority': str,
-    'pci': str,
     'vhpt': int,
     'guest_os_type': str,
     'hap': int,
@@ -214,7 +213,6 @@ XENAPI_CFG_TYPES = {
     'other_config': dict,
     'target': int,
     'security_label': str,
-    'pci': str,
     'cpuid' : dict,
     'cpuid_check' : dict,
     'machine_address_size': int,

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel