Enable HVM guest VT-d device hotplug via a simple ACPI hotplug device model.

** Currently only 2 virtual hotplug pci slots(6~7) are created so more
   than 2 vtd dev can't be hotplugged, but we can easily extend it in
   future.

Three new commands are added:
"xm pci-list domid" show the current assigned vtd device, like:
VSlt  domain   bus   slot   func
0x6      0x0  0x02   0x00    0x0

"xm pci-detach" hot remove the specified vtd device by the virtual
slot, like:
xm pci-detach EdwinHVMDomainVtd 6

"xm pci-attach DomainID dom bus dev func [vslot]" hot add a new vtd
device in the vslot. If no vslot specified, a free slot will be picked
up. e.g. to insert '0000:03:00.0':
xm pci-attach EdwinHVMDomainVtd 0 3 0 0

** guest pci hotplug
linux: pls. use 2.6.X and enable ACPI PCI hotplug ( Bus options=> PCI
hotplug => ACPI PCI hotplug driver )
windows: 2000/xp/2003/vista are all okay

Signed-off-by: Zhai Edwin <edwin.zhai@intel.com>

tools/python/xen/xend/XendDomainInfo.py

@@ -516,6 +516,131 @@ class XendDomainInfo:
         asserts.isCharConvertible(key)
         self.storeDom("control/sysrq", '%c' % key)
 
+    def sync_pcidev_info(self):
+
+        if not self.info.is_hvm():
+            return
+
+        devid = '0'
+        dev_info = self._getDeviceInfo_pci(devid)
+        if dev_info is None:
+            return
+
+        # get the virtual slot info from xenstore
+        dev_uuid = sxp.child_value(dev_info, 'uuid')
+        pci_conf = self.info['devices'][dev_uuid][1]
+        pci_devs = pci_conf['devs']
+
+        count = 0
+        vslots = None
+        while vslots is None and count < 20:
+            vslots = xstransact.Read("/local/domain/0/backend/pci/%u/%s/vslots"
+                              % (self.getDomid(), devid))
+            time.sleep(0.1)
+            count += 1
+        if vslots is None:
+            log.error("Device model didn't tell the vslots for PCI device")
+            return
+
+        #delete last delim
+        if vslots[-1] == ";":
+            vslots = vslots[:-1]
+
+        slot_list = vslots.split(';')
+        if len(slot_list) != len(pci_devs):
+            log.error("Device model's pci dev num dismatch")
+            return
+
+        #update the vslot info
+        count = 0;
+        for x in pci_devs:
+            x['vslt'] = slot_list[count]
+            count += 1
+
+
+    def pci_device_create(self, dev_config):
+        log.debug("XendDomainInfo.pci_device_create: %s" % scrub_password(dev_config))
+
+        if not self.info.is_hvm():
+            raise VmError("only HVM guest support pci attach")
+
+        #all the PCI devs share one conf node
+        devid = '0'
+
+        dev_type = sxp.name(dev_config)
+        new_devs = sxp.child_value(dev_config, 'devs')
+        new_dev = new_devs[0]
+        dev_info = self._getDeviceInfo_pci(devid)#from self.info['devices']
+
+        #check conflict before trigger hotplug event
+        if dev_info is not None:
+            dev_uuid = sxp.child_value(dev_info, 'uuid')
+            pci_conf = self.info['devices'][dev_uuid][1]
+            pci_devs = pci_conf['devs']
+            for x in pci_devs:
+                if (int(x['vslt'], 16) == int(new_dev['vslt'], 16) and
+                   int(x['vslt'], 16) != 0 ):
+                    raise VmError("vslot %s already have a device." % (new_dev['vslt']))
+
+                if (int(x['domain'], 16) == int(new_dev['domain'], 16) and
+                   int(x['bus'], 16)    == int(new_dev['bus'], 16) and
+                   int(x['slot'], 16)   == int(new_dev['slot'], 16) and
+                   int(x['func'], 16)   == int(new_dev['func'], 16) ):
+                    raise VmError("device is already inserted")
+
+        # Test whether the devices can be assigned with VT-d
+        pci_str = "%s, %s, %s, %s" % (new_dev['domain'],
+                new_dev['bus'],
+                new_dev['slot'],
+                new_dev['func'])
+        bdf = xc.test_assign_device(self.domid, pci_str)
+        if bdf != 0:
+            bus = (bdf >> 16) & 0xff
+            devfn = (bdf >> 8) & 0xff
+            dev = (devfn >> 3) & 0x1f
+            func = devfn & 0x7
+            raise VmError("Fail to hot insert device(%x:%x.%x): maybe VT-d is "
+                          "not enabled, or the device is not exist, or it "
+                          "has already been assigned to other domain"
+                          % (bus, dev, func))
+
+        bdf_str = "%s:%s:%s.%s@%s" % (new_dev['domain'],
+                new_dev['bus'],
+                new_dev['slot'],
+                new_dev['func'],
+                new_dev['vslt'])
+        self.image.signalDeviceModel('pci-ins', 'pci-inserted', bdf_str)
+
+        # update the virtual pci slot
+        vslt = xstransact.Read("/local/domain/0/device-model/%i/parameter"
+                          % self.getDomid())
+        new_dev['vslt'] = vslt
+
+        if dev_info is None:
+            # create a new one from scrach
+            dev_cfg_sxp = [dev_type,
+                ['dev',
+                  ['domain', new_dev['domain']],
+                  ['bus',    new_dev['bus']],
+                  ['slot',   new_dev['slot']],
+                  ['func',   new_dev['func']],
+                  ['vslt',   new_dev['vslt']]
+                ]]
+            dev_uuid = self.info.device_add(dev_type, cfg_sxp = dev_cfg_sxp)
+            dev_config_dict = self.info['devices'][dev_uuid][1]
+            try:
+                dev_config_dict['devid'] = devid = \
+                    self._createDevice(dev_type, dev_config_dict)
+                self._waitForDevice(dev_type, devid)
+            except VmError, ex:
+                raise ex
+        else:
+            # update the pci config to add the new dev
+            pci_devs.extend(new_devs)
+            self._reconfigureDevice('pci', devid, pci_conf)
+
+        return self.getDeviceController('pci').sxpr(devid)
+
     def device_create(self, dev_config):
         """Create a new device.
 
@@ -524,6 +649,11 @@ class XendDomainInfo:
         """
         log.debug("XendDomainInfo.device_create: %s" % scrub_password(dev_config))
         dev_type = sxp.name(dev_config)
+
+        if dev_type == 'pci':
+            rc = self.pci_device_create(dev_config)
+            return rc
+
         dev_uuid = self.info.device_add(dev_type, cfg_sxp = dev_config)
         dev_config_dict = self.info['devices'][dev_uuid][1]
         log.debug("XendDomainInfo.device_create: %s" % scrub_password(dev_config_dict))
@@ -584,10 +714,65 @@ class XendDomainInfo:
         for devclass in XendDevices.valid_devices():
             self.getDeviceController(devclass).waitForDevices()
 
+    def destroyPCIDevice(self, vslot):
+        log.debug("destroyPCIDevice called %s", vslot)
+
+        if not self.info.is_hvm():
+            raise VmError("only HVM guest support pci detach")
+
+        #all the PCI devs share one conf node
+        devid = '0'
+        vslot = int(vslot)
+        dev_info = self._getDeviceInfo_pci('0')#from self.info['devices']
+        dev_uuid = sxp.child_value(dev_info, 'uuid')
+
+        #delete the pci bdf config under the pci device
+        pci_conf = self.info['devices'][dev_uuid][1]
+        pci_len = len(pci_conf['devs'])
+
+        #find the pass-through device with the virtual slot
+        devnum = 0
+        for x in pci_conf['devs']:
+            if int(x['vslt'], 16) == vslot:
+                break
+            devnum += 1
+
+        if devnum >= pci_len:
+            raise VmError("Device @ vslot 0x%x doesn't exist." % (vslot))
+
+        if vslot == 0:
+            raise VmError("Device @ vslot 0x%x do not support hotplug." % (vslot))
+
+        bdf_str = "%s:%s:%s.%s" % (x['domain'], x['bus'], x['slot'], x['func'])
+        log.info("destroyPCIDevice:%s:%s!", x, bdf_str)
+
+        self.image.signalDeviceModel('pci-rem', 'pci-removed', bdf_str)
+
+        if pci_len > 1:
+            del pci_conf['devs'][devnum]
+            self._reconfigureDevice('pci', devid, pci_conf)
+        else:
+            self.getDeviceController('pci').destroyDevice(devid, True)
+            del self.info['devices'][dev_uuid]
+            platform = self.info['platform']
+            orig_dev_num = len(platform['pci'])
+
+            #need remove the pci config
+            #TODO:can use this to keep some info to ask high level management tools to hot insert a new passthrough dev after migration
+            if orig_dev_num != 0:
+#                platform['pci'] = ["%dDEVs" % orig_dev_num]
+                platform['pci'] = []
+
+        return 0
+
     def destroyDevice(self, deviceClass, devid, force = False, rm_cfg = False):
         log.debug("XendDomainInfo.destroyDevice: deviceClass = %s, device = %s",
                   deviceClass, devid)
 
+        if deviceClass == 'dpci':
+            rc = self.destroyPCIDevice(devid)
+            return rc
+
         if rm_cfg:
             # Convert devid to device number.  A device number is
             # needed to remove its configuration.
@@ -647,6 +832,14 @@ class XendDomainInfo:
         return rc
 
     def getDeviceSxprs(self, deviceClass):
+        if deviceClass == 'pci':
+            dev_info = self._getDeviceInfo_pci('0')#from self.info['devices']
+            if dev_info is None:
+                return []
+            dev_uuid = sxp.child_value(dev_info, 'uuid')
+            pci_devs = self.info['devices'][dev_uuid][1]['devs']
+            pci_len = len(pci_devs)
+            return pci_devs
         if self._stateGet() in (DOM_STATE_RUNNING, DOM_STATE_PAUSED, DOM_STATE_CRASHED):
             return self.getDeviceController(deviceClass).sxprs()
         else:
@@ -683,6 +876,12 @@ class XendDomainInfo:
             if devid == dev:
                 return dev_info
 
+    def _getDeviceInfo_pci(self, devid):
+        for dev_type, dev_info in self.info.all_devices_sxpr():
+            if dev_type != 'pci':
+                continue
+            return dev_info
+        return None
 
     def setMemoryTarget(self, target):
         """Set the memory target of this domain.
@@ -1543,6 +1742,9 @@ class XendDomainInfo:
         if self.image:
             self.image.createDeviceModel()
 
+        #if have pass-through devs, need the virtual pci slots info from qemu
+        self.sync_pcidev_info()
+
     def _releaseDevices(self, suspend = False):
         """Release all domain's devices.  Nothrow guarantee."""
         if self.image:

tools/python/xen/xend/image.py

@@ -300,23 +300,42 @@ class ImageHandler:
         self.vm.storeDom("image/device-model-pid", self.pid)
         log.info("device model pid: %d", self.pid)
 
-    def saveDeviceModel(self):
+    def signalDeviceModel(self, cmd, ret, par = None):
         if self.device_model is None:
             return
-        # Signal the device model to pause itself and save its state
+        # Signal the device model to for action
+        if cmd is '' or ret is '':
+            raise VmError('need valid command and result when signal device model')
+
+        orig_state = xstransact.Read("/local/domain/0/device-model/%i/state"
+                                % self.vm.getDomid())
+
+        if par is not None:
+            xstransact.Store("/local/domain/0/device-model/%i"
+                             % self.vm.getDomid(), ('parameter', par))
+
         xstransact.Store("/local/domain/0/device-model/%i"
-                         % self.vm.getDomid(), ('command', 'save'))
+                         % self.vm.getDomid(), ('command', cmd))
         # Wait for confirmation.  Could do this with a watch but we'd
         # still end up spinning here waiting for the watch to fire. 
         state = ''
         count = 0
-        while state != 'paused':
+        while state != ret:
             state = xstransact.Read("/local/domain/0/device-model/%i/state"
                                     % self.vm.getDomid())
             time.sleep(0.1)
             count += 1
             if count > 100:
-                raise VmError('Timed out waiting for device model to save')
+                raise VmError('Timed out waiting for device model action')
+
+        #resotre orig state
+        xstransact.Store("/local/domain/0/device-model/%i"
+                         % self.vm.getDomid(), ('state', orig_state))
+        log.info("signalDeviceModel:restore dm state to %s", orig_state)
+
+    def saveDeviceModel(self):
+        # Signal the device model to pause itself and save its state
+        self.signalDeviceModel('save', 'paused')
 
     def resumeDeviceModel(self):
         if self.device_model is None:
@@ -479,7 +498,7 @@ class HVMImageHandler(ImageHandler):
 
         dmargs = [ 'boot', 'fda', 'fdb', 'soundhw',
                    'localtime', 'serial', 'stdvga', 'isa',
-                   'acpi', 'usb', 'usbdevice', 'pci' ]
+                   'acpi', 'usb', 'usbdevice' ]
 
         for a in dmargs:
             v = vmConfig['platform'].get(a)

tools/python/xen/xend/server/DevController.py

@@ -412,6 +412,14 @@ class DevController:
         return result
 
 
+    def removeBackend(self, devid, *args):
+        frontpath = self.frontendPath(devid)
+        backpath = xstransact.Read(frontpath, "backend")
+        if backpath:
+            return xstransact.Remove(backpath, *args)
+        else:
+            raise VmError("Device %s not connected" % devid)
+
     def readBackend(self, devid, *args):
         frontpath = self.frontendPath(devid)
         backpath = xstransact.Read(frontpath, "backend")

tools/python/xen/xend/server/pciif.py

@@ -18,6 +18,7 @@
 
 
 import types
+import time
 
 from xen.xend import sxp
 from xen.xend.XendError import VmError
@@ -62,25 +63,62 @@ class PciController(DevController):
             
         back = {}
         pcidevid = 0
+        vslots = ""
         for pci_config in config.get('devs', []):
             domain = parse_hex(pci_config.get('domain', 0))
             bus = parse_hex(pci_config.get('bus', 0))
             slot = parse_hex(pci_config.get('slot', 0))
             func = parse_hex(pci_config.get('func', 0))            
+
+            vslt = pci_config.get('vslt')
+            if vslt is not None:
+                vslots = vslots + vslt + ";"
+
             self.setupDevice(domain, bus, slot, func)
             back['dev-%i' % pcidevid] = "%04x:%02x:%02x.%02x" % \
                                         (domain, bus, slot, func)
             pcidevid += 1
 
+        if vslots != "":
+            back['vslots'] = vslots
+
         back['num_devs']=str(pcidevid)
         back['uuid'] = config.get('uuid','')
         return (0, back, {})
 
+    def reconfigureDevice(self, _, config):
+        """@see DevController.reconfigureDevice"""
+        #currently only support config changes by hot insert/remove pass-through dev
+        #delete all the devices in xenstore
+        (devid, new_back, new_front) = self.getDeviceDetails(config)
+        num_devs = self.readBackend(devid, 'num_devs')
+        for i in range(int(num_devs)):
+            self.removeBackend(devid, 'dev-%d' % i)
+        self.removeBackend(devid, 'num_devs')
+
+        #create new devices config
+        num_devs = new_back['num_devs']
+        for i in range(int(num_devs)):
+            dev_no = 'dev-%d' % i
+            self.writeBackend(devid, dev_no, new_back[dev_no])
+        self.writeBackend(devid, 'num_devs', num_devs)
+
+        if new_back['vslots'] is not None:
+            self.writeBackend(devid, 'vslots', new_back['vslots'])
+
+        return new_back.get('uuid')
+
     def getDeviceConfiguration(self, devid, transaction = None):
         result = DevController.getDeviceConfiguration(self, devid, transaction)
         num_devs = self.readBackend(devid, 'num_devs')
         pci_devs = []
         
+        vslots = self.readBackend(devid, 'vslots')
+        if vslots is not None:
+            if vslots[-1] == ";":
+                vslots = vslots[:-1]
+            slot_list = vslots.split(';')
+
         for i in range(int(num_devs)):
             dev_config = self.readBackend(devid, 'dev-%d' % i)
 
@@ -91,10 +129,16 @@ class PciController(DevController):
             
             if pci_match!=None:
                 pci_dev_info = pci_match.groupdict()
-                pci_devs.append({'domain': '0x%(domain)s' % pci_dev_info,
+                dev_dict = {'domain': '0x%(domain)s' % pci_dev_info,
                                  'bus': '0x%(bus)s' % pci_dev_info,
                                  'slot': '0x%(slot)s' % pci_dev_info,
-                                 'func': '0x%(func)s' % pci_dev_info})
+                                 'func': '0x%(func)s' % pci_dev_info}
+
+                #append vslot info
+                if vslots is not None:
+                    dev_dict['vslt'] = slot_list[i]
+
+                pci_devs.append(dev_dict)
 
         result['devs'] = pci_devs
         result['uuid'] = self.readBackend(devid, 'uuid')

tools/python/xen/xm/main.py

@@ -175,6 +175,12 @@ SUBCOMMAND_HELP = {
     'vnet-delete'   :  ('<VnetId>', 'Delete a Vnet.'),
     'vnet-list'     :  ('[-l|--long]', 'List Vnets.'),
     'vtpm-list'     :  ('<Domain> [--long]', 'List virtual TPM devices.'),
+    'pci-attach '   :  ('<Domain> <dom> <bus> <slot> <func> [virtual slot]',
+                        'Insert a new pass-through pci device.'),
+    'pci-detach '   :  ('<Domain> <virtual slot>',
+                        'Remove a domain\'s pass-through pci device.'),
+    'pci-list'     :  ('<Domain>',
+                        'List pass-through pci devices for a domain.'),
 
     # security
 
@@ -335,6 +341,9 @@ device_commands = [
     "network-detach",
     "network-list",
     "vtpm-list",
+    "pci-attach",
+    "pci-detach",
+    "pci-list",
     ]
 
 vnet_commands = [
@@ -2051,6 +2060,31 @@ def xm_vtpm_list(args):
                    % ni)
 
 
+def xm_pci_list(args):
+    (use_long, params) = arg_check_for_resource_list(args, "pci-list")
+
+    dom = params[0]
+
+    devs = server.xend.domain.getDeviceSxprs(dom, 'pci')
+
+    if len(devs) == 0:
+        return
+
+    has_vslt = devs[0].has_key('vslt')
+    if has_vslt:
+        hdr_str = 'VSlt domain   bus   slot   func'
+        fmt_str =  "%(vslt)-3s    %(domain)-3s  %(bus)-3s   %(slot)-3s    %(func)-3s    "
+    else:
+        hdr_str = 'domain   bus   slot   func'
+        fmt_str =  "%(domain)-3s  %(bus)-3s   %(slot)-3s    %(func)-3s    "
+    hdr = 0
+
+    for x in devs:
+        if hdr == 0:
+            print (hdr_str)
+            hdr = 1
+        print ( fmt_str % x )
+
 def parse_block_configuration(args):
     dom = args[0]
 
@@ -2198,6 +2232,29 @@ def xm_network_attach(args):
             vif.append(vif_param)
         server.xend.domain.device_create(dom, vif)
 
+def parse_pci_configuration(args):
+    dom = args[0]
+
+    if len(args) == 6:
+        vslt = args[5]
+    else:
+        vslt = '0x0' #chose a free virtual PCI slot
+
+    pci = ['pci',
+          ['devs',
+            [{'domain': "0x%x" % int(args[1], 16),
+              'bus':    "0x%x" % int(args[2], 16),
+              'slot':   "0x%x" % int(args[3], 16),
+              'func':   "0x%x" % int(args[4], 16),
+              'vslt':   "0x%x" % int(vslt,    16)}]
+          ]]
+
+    return (dom, pci)
+
+def xm_pci_attach(args):
+    arg_check(args, 'xm_pci_attach', 5, 6)
+    (dom, pci) = parse_pci_configuration(args)
+    server.xend.domain.device_create(dom, pci)
 
 def detach(args, deviceClass):
     rm_cfg = True
@@ -2263,6 +2320,12 @@ def xm_network_detach(args):
         detach(args, 'vif')
 
 
+def xm_pci_detach(args):
+    arg_check(args, 'xm_pci_detach', 2, 2)
+    dom = args[0]
+    dev = args[1]
+    server.xend.domain.destroyDevice(dom, 'dpci', dev)
+
 def xm_vnet_list(args):
     xenapi_unsupported()
     try:
@@ -2452,6 +2515,10 @@ commands = {
     "vnet-delete": xm_vnet_delete,
     # vtpm
     "vtpm-list": xm_vtpm_list,
+    #pci
+    "pci-attach": xm_pci_attach,
+    "pci-detach": xm_pci_detach,
+    "pci-list": xm_pci_list,
     }
 
 ## The commands supported by a separate argument parser in xend.xm.