xend: passthrough: loosen the pci co-assignment for pv guest

In current xend, we can not assign co-assignment devices to different
guests, even for pv guests. This patch loosens the policy for pv
guest: if none of the co-assignment devices have been assigned to hvm
guest, we can assign the devices to different pv guests.

The patch also adds a detection: if a device has been assigned to
guest, we can't try to "xm pci-attach" it to the same guest again.

Signed-off-by: Dexuan Cui <dexuan.cui@intel.com>

tools/python/xen/util/pci.py

@@ -7,6 +7,7 @@
 
 import sys
 import os, os.path
+import errno
 import resource
 import re
 import types
@@ -571,7 +572,7 @@ class PciDeviceParseError(Exception):
     def __init__(self,msg):
         self.message = msg
     def __str__(self):
-        return 'Error Parsing PCI Device Info: '+self.message
+        return self.message
 
 class PciDeviceAssignmentError(Exception):
     def __init__(self,msg):
@@ -880,7 +881,13 @@ class PciDevice:
         os.close(fd)
 
     def detect_dev_info(self):
-        class_dev = self.pci_conf_read16(PCI_CLASS_DEVICE)
+        try:
+            class_dev = self.pci_conf_read16(PCI_CLASS_DEVICE)
+        except OSError, (err, strerr):
+            if err == errno.ENOENT:
+                strerr = "the device doesn't exist?"
+            raise PciDeviceParseError('%s: %s' %\
+                (self.name, strerr))
         pos = self.find_cap_offset(PCI_CAP_ID_EXP)
         if class_dev == PCI_CLASS_BRIDGE_PCI:
             if pos == 0:
@@ -941,7 +948,7 @@ class PciDevice:
                 ', but it is not owned by pciback or pci-stub.'
             raise PciDeviceAssignmentError(err_msg % (pci_dev, self.name))
 
-    def do_FLR(self):
+    def do_FLR(self, is_hvm):
         """ Perform FLR (Functional Level Reset) for the device.
         """
         if self.dev_type == DEV_TYPE_PCIe_ENDPOINT:
@@ -958,6 +965,10 @@ class PciDevice:
                     self.do_FLR_for_integrated_device()
                 else:
                     funcs = self.find_all_the_multi_functions()
+
+                    if not is_hvm and (len(funcs) > 1):
+                        return
+
                     self.devs_check_driver(funcs)
 
                     parent = pci_dict_to_bdf_str(self.find_parent())
@@ -982,6 +993,10 @@ class PciDevice:
                     # Remove the element 0 which is a bridge
                     target_bus = devs[0]
                     del devs[0]
+
+                    if not is_hvm and (len(devs) > 1):
+                        return
+
                     self.devs_check_driver(devs)
 
                     # Do Secondary Bus Reset.

tools/python/xen/xend/XendDomain.py

@@ -423,7 +423,7 @@ class XendDomain:
                     log.exception("Unable to recreate domain")
                     try:
                         xc.domain_pause(domid)
-                        XendDomainInfo.do_FLR(domid)
+                        XendDomainInfo.do_FLR(domid, dom['hvm'])
                         xc.domain_destroy(domid)
                     except:
                         log.exception("Hard destruction of domain failed: %d" %
@@ -1276,7 +1276,8 @@ class XendDomain:
         else:
             try:
                 xc.domain_pause(int(domid))
-                XendDomainInfo.do_FLR(int(domid))
+                dom = self.domains[int(domid)]
+                XendDomainInfo.do_FLR(int(domid), dom.info.is_hvm())
                 val = xc.domain_destroy(int(domid))
             except ValueError:
                 raise XendInvalidDomain(domid)

tools/python/xen/xend/XendDomainInfo.py

@@ -302,7 +302,7 @@ from xen.xend.server.pciif import parse_pci_name, PciDevice,\
     get_assigned_pci_devices, get_all_assigned_pci_devices
 
 
-def do_FLR(domid):
+def do_FLR(domid, is_hvm):
     dev_str_list = get_assigned_pci_devices(domid)
 
     for dev_str in dev_str_list:
@@ -311,7 +311,7 @@ def do_FLR(domid):
         except Exception, e:
             raise VmError("pci: failed to locate device and "+
                     "parse it's resources - "+str(e))
-        dev.do_FLR()
+        dev.do_FLR(is_hvm)
 
 class XendDomainInfo:
     """An object represents a domain.
@@ -693,19 +693,14 @@ class XendDomainInfo:
 
         # Test whether the device is owned by pciback. For instance, we can't
         # hotplug a device being used by Dom0 itself to an HVM guest.
-        from xen.xend.server.pciif import PciDevice, parse_pci_name
         try:
             pci_device = PciDevice(new_dev)
         except Exception, e:
             raise VmError("pci: failed to locate device and "+
-                    "parse it's resources - "+str(e))
+                    "parse its resources - "+str(e))
         if pci_device.driver!='pciback' and pci_device.driver!='pci-stub':
-            raise VmError(("pci: PCI Backend does not own device "+ \
-                    "%s\n"+ \
-                    "See the pciback.hide kernel "+ \
-                    "command-line parameter or\n"+ \
-                    "bind your slot/device to the PCI backend using sysfs" \
-                    )%(pci_device.name))
+            raise VmError(("pci: PCI Backend and pci-stub don't own device %s")\
+                            %pci_device.name)
 
         # Check non-page-aligned MMIO BAR.
         if pci_device.has_non_page_aligned_bar and arch.type != "ia64":
@@ -1148,7 +1143,7 @@ class XendDomainInfo:
             pci_device = PciDevice(pci_dev)
         except Exception, e:
             raise VmError("pci: failed to locate device and "+
-                    "parse it's resources - "+str(e))
+                    "parse its resources - "+str(e))
         coassignment_list = pci_device.find_coassigned_devices()
         coassignment_list.remove(pci_device.name)
         assigned_pci_device_str_list = self._get_assigned_pci_devices()
@@ -2458,6 +2453,22 @@ class XendDomainInfo:
             pci = map(lambda x: x[0:4], pci)  # strip options 
             pci_str = str(pci)
 
+        # This test is done for both pv and hvm guest.
+        for p in pci:
+            pci_name = '%04x:%02x:%02x.%x' % \
+                (parse_hex(p[0]), parse_hex(p[1]), parse_hex(p[2]), parse_hex(p[3]))
+            try:
+                pci_device = PciDevice(parse_pci_name(pci_name))
+            except Exception, e:
+                raise VmError("pci: failed to locate device and "+
+                    "parse its resources - "+str(e))
+            if pci_device.driver!='pciback' and pci_device.driver!='pci-stub':
+                raise VmError(("pci: PCI Backend and pci-stub don't own device %s")\
+                                %pci_device.name)
+            if pci_name in get_all_assigned_pci_devices():
+                raise VmError("failed to assign device %s that has"
+                              " already been assigned to other domain." % pci_name)
+
         if hvm and pci_str != '':
             bdf = xc.test_assign_device(0, pci_str)
             if bdf != 0:
@@ -2469,18 +2480,10 @@ class XendDomainInfo:
                 devfn = (bdf >> 8) & 0xff
                 dev = (devfn >> 3) & 0x1f
                 func = devfn & 0x7
-                raise VmError("failed to assign device(%x:%x.%x): maybe it has"
+                raise VmError("failed to assign device %02x:%02x.%x: maybe it has"
                               " already been assigned to other domain, or maybe"
                               " it doesn't exist." % (bus, dev, func))
 
-        # This test is done for both pv and hvm guest.
-        for p in pci:
-            pci_name = '%04x:%02x:%02x.%x' % \
-                (parse_hex(p[0]), parse_hex(p[1]), parse_hex(p[2]), parse_hex(p[3]))
-            if pci_name in get_all_assigned_pci_devices():
-                raise VmError("failed to assign device %s that has"
-                              " already been assigned to other domain." % pci_name)
-
         # register the domain in the list 
         from xen.xend import XendDomain
         XendDomain.instance().add_domain(self)
@@ -2810,7 +2813,7 @@ class XendDomainInfo:
             try:
                 xc.domain_destroy_hook(self.domid)
                 xc.domain_pause(self.domid)
-                do_FLR(self.domid)
+                do_FLR(self.domid, self.info.is_hvm())
                 xc.domain_destroy(self.domid)
                 for state in DOM_STATES_OLD:
                     self.info[state] = 0

tools/python/xen/xend/server/pciif.py

@@ -279,7 +279,7 @@ class PciController(DevController):
             dev = PciDevice(pci_dev)
         except Exception, e:
             raise VmError("pci: failed to locate device and "+
-                    "parse it's resources - "+str(e))
+                    "parse its resources - "+str(e))
 
         if dev.driver!='pciback' and dev.driver!='pci-stub':
             raise VmError(("pci: PCI Backend and pci-stub don't own "+ \
@@ -373,7 +373,7 @@ class PciController(DevController):
                 dev = PciDevice(pci_dev)
             except Exception, e:
                 raise VmError("pci: failed to locate device and "+
-                        "parse it's resources - "+str(e))
+                        "parse its resources - "+str(e))
             if (dev.dev_type == DEV_TYPE_PCIe_ENDPOINT) and not dev.pcie_flr:
                 if dev.bus == 0:
                     # We cope with this case by using the Dstate transition
@@ -383,6 +383,9 @@ class PciController(DevController):
                         ' method or some vendor specific methods if available.'
                     log.warn(err_msg % dev.name)
                 else:
+                    if not self.vm.info.is_hvm():
+                        continue
+
                     funcs = dev.find_all_the_multi_functions()
                     dev.devs_check_driver(funcs)
                     for f in funcs:
@@ -403,6 +406,9 @@ class PciController(DevController):
                             ' specific methods if available.'
                         log.warn(err_msg % dev.name)
                 else:
+                    if not self.vm.info.is_hvm():
+                        continue
+
                     # All devices behind the uppermost PCI/PCI-X bridge must be\
                     # co-assigned to the same guest.
                     devs_str = dev.find_coassigned_pci_devices(True)
@@ -455,7 +461,7 @@ class PciController(DevController):
             dev = PciDevice(pci_dev)
         except Exception, e:
             raise VmError("pci: failed to locate device and "+
-                    "parse it's resources - "+str(e))
+                    "parse its resources - "+str(e))
 
         if dev.driver!='pciback' and dev.driver!='pci-stub':
             raise VmError(("pci: PCI Backend and pci-stub don't own device "+ \
@@ -463,7 +469,7 @@ class PciController(DevController):
 
         # Need to do FLR here before deassign device in order to terminate
         # DMA transaction, etc
-        dev.do_FLR()
+        dev.do_FLR(self.vm.info.is_hvm())
 
         bdf = xc.deassign_device(fe_domid, pci_dict_to_xc_str(pci_dev))
         pci_str = pci_dict_to_bdf_str(pci_dev)

tools/python/xen/xm/main.py

@@ -2538,6 +2538,10 @@ def xm_pci_attach(args):
 
     (dom, dev) = parse_pci_configuration(params, config_pci_opts)
 
+    attached = attached_pci_dict(dom)
+
+    attached_dev = map(lambda x: find_attached(attached, x, False), dev)
+
     head_dev = dev.pop(0)
     xm_pci_attach_one(dom, head_dev)
 
@@ -2724,19 +2728,24 @@ def xm_network_detach(args):
         arg_check(args, 'network-detach', 2, 3)
         detach(args, 'vif')
 
-def find_attached(attached, key):
+def find_attached(attached, key, detaching):
     l = filter(lambda dev: pci_dict_cmp(dev, key), attached)
 
-    if len(l) == 0:
-         raise OptionError("pci: device is not attached: " +
-                           pci_dict_to_bdf_str(key))
-
-    # There shouldn't ever be more than one match,
-    # but perhaps an exception should be thrown if there is
-    return l[0]
+    if detaching:
+        if  len(l) == 0:
+             raise OptionError("pci: device %s is not attached!" %\
+                               pci_dict_to_bdf_str(key))
+        # There shouldn't ever be more than one match,
+        # but perhaps an exception should be thrown if there is
+        return l[0]
+    else:
+        if len(l) == 1:
+            raise  OptionError("pci: device %s has been attached! " %\
+                               pci_dict_to_bdf_str(key))
+        return None
 
 def find_attached_devfn(attached, key):
-    pci_dev = find_attached(attached, key)
+    pci_dev = find_attached(attached, key, True)
     return pci_dev['vdevfn']
 
 def xm_pci_detach(args):
@@ -2745,7 +2754,7 @@ def xm_pci_detach(args):
     (dom, dev) = parse_pci_configuration(args)
     attached = attached_pci_dict(dom)
 
-    attached_dev = map(lambda x: find_attached(attached, x), dev)
+    attached_dev = map(lambda x: find_attached(attached, x, True), dev)
 
     def f(pci_dev):
         vdevfn = int(pci_dev['vdevfn'], 16)