Tools changes for PCI front/back drivers.

Replace the old pciif DevController class with a new one that
configures the PCI backend. A util class detects the resource usage of
the specified PCI devices and pciif interacts with Xen to permit a
driver domain to directly access those physical I/O resources.

Signed-off-by: Ryan Wilson <hap9@epoch.ncsc.mil>

tools/python/xen/util/pci.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+#
+# PCI Device Information Class
+# - Helps obtain information about which I/O resources a PCI device needs
+#
+#   Author: Ryan Wilson <hap9@epoch.ncsc.mil>
+
+import sys
+import os, os.path
+
+PROC_MNT_PATH = '/proc/mounts'
+PROC_PCI_PATH = '/proc/bus/pci/devices'
+PROC_PCI_NUM_RESOURCES = 7
+
+SYSFS_PCI_DEVS_PATH = '/bus/pci/devices'
+SYSFS_PCI_DEV_RESOURCE_PATH = '/resource'
+SYSFS_PCI_DEV_IRQ_PATH = '/irq'
+SYSFS_PCI_DEV_DRIVER_DIR_PATH = '/driver'
+
+PCI_BAR_IO = 0x01
+PCI_BAR_IO_MASK = ~0x03
+PCI_BAR_MEM_MASK = ~0x0f
+
+# Definitions from Linux: include/linux/pci.h
+def PCI_DEVFN(slot, func):
+    return ((((slot) & 0x1f) << 3) | ((func) & 0x07))
+
+def find_sysfs_mnt():
+    mounts_file = open(PROC_MNT_PATH,'r')
+
+    for line in mounts_file:
+        sline = line.split()
+        if len(sline)<3:
+            continue
+
+        if sline[2]=='sysfs':
+            return sline[1]
+
+    return None
+
+class PciDeviceNotFoundError(Exception):
+    def __init__(self,domain,bus,slot,func):
+        self.domain = domain
+        self.bus = bus
+        self.slot = slot
+        self.func = func
+        self.name = "%04x:%02x:%02x.%01x"%(domain, bus, slot, func)
+    
+    def __str__(self):
+        return ('PCI Device %s Not Found' % (self.name))
+
+class PciDeviceParseError(Exception):
+    def __init__(self,msg):
+        self.message = msg
+    def __str__(self):
+        return 'Error Parsing PCI Device Info: '+self.message
+
+class PciDevice:
+    def __init__(self, domain, bus, slot, func):
+        self.domain = domain
+        self.bus = bus
+        self.slot = slot
+        self.func = func
+        self.name = "%04x:%02x:%02x.%01x"%(domain, bus, slot, func)
+        self.irq = 0
+        self.iomem = []
+        self.ioports = []
+        self.driver = None
+
+        if not self.get_info_from_sysfs():
+            self.get_info_from_proc()
+
+    def get_info_from_sysfs(self):
+        try:
+            sysfs_mnt = find_sysfs_mnt()
+        except IOError, (errno, strerr):
+            raise PciDeviceParseError(('Failed to locate sysfs mount: %s (%d)' %
+                (PROC_PCI_PATH, strerr, errno)))
+
+        if sysfs_mnt == None:
+            return False
+
+        path = sysfs_mnt+SYSFS_PCI_DEVS_PATH+'/'+ \
+                self.name+SYSFS_PCI_DEV_RESOURCE_PATH
+        try:
+            resource_file = open(path,'r')
+
+            for i in range(7):
+                line = resource_file.readline()
+                sline = line.split()
+                if len(sline)<3:
+                    continue
+
+                start = int(sline[0],16)
+                end = int(sline[1],16)
+                flags = int(sline[2],16)
+                size = end-start+1
+
+                if start!=0:
+                    if flags&PCI_BAR_IO:
+                        self.ioports.append( (start,size) )
+                    else:
+                        self.iomem.append( (start,size) )
+
+        except IOError, (errno, strerr):
+            raise PciDeviceParseError(('Failed to open & read %s: %s (%d)' %
+                (path, strerr, errno)))
+
+        path = sysfs_mnt+SYSFS_PCI_DEVS_PATH+'/'+ \
+                self.name+SYSFS_PCI_DEV_IRQ_PATH
+        try:
+            self.irq = int(open(path,'r').readline())
+        except IOError, (errno, strerr):
+            raise PciDeviceParseError(('Failed to open & read %s: %s (%d)' %
+                (path, strerr, errno)))
+
+        path = sysfs_mnt+SYSFS_PCI_DEVS_PATH+'/'+ \
+                self.name+SYSFS_PCI_DEV_DRIVER_DIR_PATH
+        try:
+            self.driver = os.path.basename(os.readlink(path))
+        except IOError, (errno, strerr):
+            raise PciDeviceParseError(('Failed to read %s: %s (%d)' %
+                (path, strerr, errno)))
+
+        return True
+        
+    def get_info_from_proc(self):
+        bus_devfn = '%02x%02x' % (self.bus,PCI_DEVFN(self.slot,self.func))
+
+        # /proc/bus/pci/devices doesn't expose domains
+        if self.domain!=0:
+            raise PciDeviceParseError("Can't yet detect resource usage by "+
+                    "devices in other domains through proc!")
+
+        try:
+            proc_pci_file = open(PROC_PCI_PATH,'r')
+        except IOError, (errno, strerr):
+            raise PciDeviceParseError(('Failed to open %s: %s (%d)' %
+                (PROC_PCI_PATH, strerr, errno)))
+
+        for line in proc_pci_file:
+            sline = line.split()
+            if len(sline)<(PROC_PCI_NUM_RESOURCES*2+3):
+                continue
+
+            if sline[0]==bus_devfn:
+                self.dissect_proc_pci_line(sline)
+                break
+        else:
+            raise PciDeviceNotFoundError(self.domain, self.bus,
+                    self.slot, self.func)
+
+    def dissect_proc_pci_line(self, sline):
+        self.irq = int(sline[2],16)
+        start_idx = 3
+        for i in range(PROC_PCI_NUM_RESOURCES):
+            flags = int(sline[start_idx+i],16)
+            size = int(sline[start_idx+i+PROC_PCI_NUM_RESOURCES],16)
+            if flags&PCI_BAR_IO:
+                start = flags&PCI_BAR_IO_MASK
+                if start!=0:
+                    self.ioports.append( (start,size) )
+            else:
+                start = flags&PCI_BAR_MEM_MASK
+                if start!=0:
+                    self.iomem.append( (start,size) )
+
+        # detect driver module name
+        driver_idx = PROC_PCI_NUM_RESOURCES*2+3
+        if len(sline)>driver_idx:
+            self.driver = sline[driver_idx]
+
+    def __str__(self):
+        str = "PCI Device %s\n" % (self.name)
+        for (start,size) in self.ioports:
+            str = str + "IO Port 0x%02x [size=%d]\n"%(start,size)
+        for (start,size) in self.iomem:
+            str = str + "IO Mem 0x%02x [size=%d]\n"%(start,size)
+        str = str + "IRQ %d"%(self.irq)
+        return str
+
+def main():
+    if len(sys.argv)<5:
+        print "Usage: %s <domain> <bus> <slot> <func>\n"
+        sys.exit(2)
+
+    dev = PciDevice(int(sys.argv[1],16), int(sys.argv[2],16),
+            int(sys.argv[3],16), int(sys.argv[4],16))
+    print str(dev)
+
+if __name__=='__main__':
+    main()

tools/python/xen/xend/server/pciif.py

@@ -19,29 +19,28 @@
 
 import types
 
-import xen.lowlevel.xc;
-
 from xen.xend import sxp
 from xen.xend.XendError import VmError
+from xen.xend.XendLogging import log
+
+from xen.xend.xenstore.xstransact import xstransact
 
 from xen.xend.server.DevController import DevController
 
+import xen.lowlevel.xc
+
+from xen.util.pci import PciDevice
+import resource
 
 xc = xen.lowlevel.xc.xc()
 
-
-def parse_pci(val):
-    """Parse a pci field.
-    """
-    if isinstance(val, types.StringType):
-        radix = 10
-        if val.startswith('0x') or val.startswith('0X'):
-            radix = 16
-        v = int(val, radix)
-    else:
-        v = val
-    return v
-
+#Calculate PAGE_SHIFT: number of bits to shift an address to get the page number
+PAGE_SIZE = resource.getpagesize()
+PAGE_SHIFT = 0
+t = PAGE_SIZE
+while not (t&1):
+    t>>=1
+    PAGE_SHIFT+=1
 
 class PciController(DevController):
 
@@ -51,32 +50,110 @@ class PciController(DevController):
 
     def getDeviceDetails(self, config):
         """@see DevController.getDeviceDetails"""
+        #log.debug('pci config='+sxp.to_string(config))
 
-        def get_param(field):
+        def get_param(config, field, default=None):
             try:
                 val = sxp.child_value(config, field)
 
                 if not val:
-                    raise VmError('pci: Missing %s config setting' % field)
+                    if default==None:
+                        raise VmError('pci: Missing %s config setting' % field)
+                    else:
+                        return default
 
-                return parse_pci(val)
+                if isinstance(val, types.StringType):
+                    return int(val, 16)
+                else:
+                    return val
             except:
-                raise VmError('pci: Invalid config setting %s: %s' %
+                if default==None:
+                    raise VmError('pci: Invalid config setting %s: %s' %
                               (field, val))
+                else:
+                    return default
         
-        bus  = get_param('bus')
-        dev  = get_param('dev')
-        func = get_param('func')
+        back = {}
 
-        rc = xc.physdev_pci_access_modify(dom    = self.getDomid(),
-                                          bus    = bus,
-                                          dev    = dev,
-                                          func   = func,
-                                          enable = True)
-        if rc < 0:
-            #todo non-fatal
-            raise VmError(
-                'pci: Failed to configure device: bus=%s dev=%s func=%s' %
-                (bus, dev, func))
+        val = sxp.child_value(config, 'dev')
+        if isinstance(val, list):
+            pcidevid = 0
+            for dev_config in sxp.children(config, 'dev'):
+                domain = get_param(dev_config, 'domain', 0)
+                bus = get_param(dev_config,'bus')
+                slot = get_param(dev_config,'slot')
+                func = get_param(dev_config,'func')
 
-        return (dev, {}, {})
+                self.setupDevice(domain, bus, slot, func)
+
+                back['dev-%i'%(pcidevid)]="%04x:%02x:%02x.%02x"% \
+                        (domain, bus, slot, func)
+                pcidevid+=1
+            
+            back['num_devs']=str(pcidevid)
+
+        else:
+            # Xen 2.0 configuration compatibility
+            domain = get_param(dev_config, 'domain', 0)
+            bus  = get_param(config, 'bus')
+            slot = get_param(config, 'dev')
+            func = get_param(config, 'func')
+
+            self.setupDevice(domain, bus, slot, func)
+
+            back['dev-0']="%04x:%02x:%02x.%02x"%(domain, bus, slot, func)
+            back['num_devs']=str(1)
+
+        return (0, back, {})
+
+    def setupDevice(self, domain, bus, slot, func):
+        """ Attach I/O resources for device to frontend domain
+        """
+        fe_domid = self.getDomid()
+
+        try:
+            dev = PciDevice(domain, bus, slot, func)
+        except Exception, e:
+            raise VmError("pci: failed to locate device and "+
+                    "parse it's resources - %s"+str(e))
+
+        if dev.driver!='pciback':
+            raise VmError(("pci: PCI Backend does not own device "+
+                    "%s\n"+
+                    "See the pciback.hide kernel "+
+                    "command-line parameter")%(dev.name))
+
+        for (start, size) in dev.ioports:
+            log.debug('pci: enabling ioport 0x%x/0x%x'%(start,size))
+            rc = xc.domain_ioport_permission(dom = fe_domid, first_port = start,
+                    nr_ports = size, allow_access = True)
+            if rc<0:
+                raise VmError(('pci: failed to configure I/O ports on device '+
+                            '%s - errno=%d')&(dev.name,rc))
+            
+        for (start, size) in dev.iomem:
+            # Convert start/size from bytes to page frame sizes
+            start_pfn = start>>PAGE_SHIFT
+            # Round number of pages up to nearest page boundary (if not on one)
+            nr_pfns = (size+(PAGE_SIZE-1))>>PAGE_SHIFT
+
+            log.debug('pci: enabling iomem 0x%x/0x%x pfn 0x%x/0x%x'% \
+                    (start,size,start_pfn,nr_pfns))
+            rc = xc.domain_iomem_permission(dom = fe_domid,
+                    first_pfn = start_pfn,
+                    nr_pfns = nr_pfns,
+                    allow_access = True)
+            if rc<0:
+                raise VmError(('pci: failed to configure I/O memory on device '+
+                            '%s - errno=%d')&(dev.name,rc))
+
+        if dev.irq>0:
+            log.debug('pci: enabling irq %d'%dev.irq)
+            rc = xc.domain_irq_permission(dom = fe_domid, pirq = dev.irq,
+                    allow_access = True)
+            if rc<0:
+                raise VmError(('pci: failed to configure irq on device '+
+                            '%s - errno=%d')&(dev.name,rc))
+
+    def waitForBackend(self,devid):
+        return (0, "ok - no hotplug")

tools/python/xen/xm/create.py

@@ -26,6 +26,7 @@ import sys
 import socket
 import commands
 import time
+import re
 
 import xen.lowlevel.xc
 
@@ -240,10 +241,10 @@ gopts.var('disk', val='phy:DEV,VDEV,MODE[,DOM]',
           backend driver domain to use for the disk.
           The option may be repeated to add more than one disk.""")
 
-gopts.var('pci', val='BUS,DEV,FUNC',
+gopts.var('pci', val='BUS:DEV.FUNC',
           fn=append_value, default=[],
           use="""Add a PCI device to a domain, using given params (in hex).
-         For example '-pci c0,02,1a'.
+         For example '-pci c0:02.1a'.
          The option may be repeated to add more than one pci device.""")
 
 gopts.var('ioports', val='FROM[-TO]',
@@ -461,8 +462,13 @@ def configure_disks(config_devs, vals):
 def configure_pci(config_devs, vals):
     """Create the config for pci devices.
     """
-    for (bus, dev, func) in vals.pci:
-        config_pci = ['pci', ['bus', bus], ['dev', dev], ['func', func]]
+    config_pci = []
+    for (domain, bus, slot, func) in vals.pci:
+        config_pci.append(['dev', ['domain', domain], ['bus', bus], \
+                        ['slot', slot], ['func', func]])
+
+    if len(config_pci)>0:
+        config_pci.insert(0, 'pci')
         config_devs.append(['device', config_pci])
 
 def configure_ioports(config_devs, vals):
@@ -624,13 +630,20 @@ def preprocess_disk(vals):
 def preprocess_pci(vals):
     if not vals.pci: return
     pci = []
-    for v in vals.pci:
-        d = v.split(',')
-        if len(d) != 3:
-            err('Invalid pci specifier: ' + v)
-        # Components are in hex: add hex specifier.
-        hexd = map(lambda v: '0x'+v, d)
-        pci.append(hexd)
+    for pci_dev_str in vals.pci:
+        pci_match = re.match(r"((?P<domain>[0-9a-fA-F]{1,4})[:,])?" + \
+                r"(?P<bus>[0-9a-fA-F]{1,2})[:,]" + \
+                r"(?P<slot>[0-9a-fA-F]{1,2})[.,]" + \
+                r"(?P<func>[0-9a-fA-F])", pci_dev_str)
+        if pci_match!=None:
+            pci_dev_info = pci_match.groupdict('0')
+            try:
+                pci.append( ('0x'+pci_dev_info['domain'], \
+                        '0x'+pci_dev_info['bus'], \
+                        '0x'+pci_dev_info['slot'], \
+                        '0x'+pci_dev_info['func']))
+            except IndexError:
+                err('Error in PCI slot syntax "%s"'%(pci_dev_str))
     vals.pci = pci
 
 def preprocess_ioports(vals):