MSI 4/6: remove io_mem permission for MSI-X, since MSI-X

facilities are allocted through and located in PCI BAR.

Signed-off-by: Jiang Yunhong <yunhong.jiang@intel.com>
Signed-off-by: Shan Haitao <haitao.shan@intel.com>

tools/python/xen/util/pci.py

@@ -7,6 +7,7 @@
 
 import sys
 import os, os.path
+import resource
 
 PROC_MNT_PATH = '/proc/mounts'
 PROC_PCI_PATH = '/proc/bus/pci/devices'
@@ -14,6 +15,7 @@ PROC_PCI_NUM_RESOURCES = 7
 
 SYSFS_PCI_DEVS_PATH = '/bus/pci/devices'
 SYSFS_PCI_DEV_RESOURCE_PATH = '/resource'
+SYSFS_PCI_DEV_CONFIG_PATH = '/config'
 SYSFS_PCI_DEV_IRQ_PATH = '/irq'
 SYSFS_PCI_DEV_DRIVER_DIR_PATH = '/driver'
 SYSFS_PCI_DEV_VENDOR_PATH = '/vendor'
@@ -24,7 +26,20 @@ SYSFS_PCI_DEV_SUBDEVICE_PATH = '/subsystem_device'
 PCI_BAR_IO = 0x01
 PCI_BAR_IO_MASK = ~0x03
 PCI_BAR_MEM_MASK = ~0x0f
+PCI_STATUS_CAP_MASK = 0x10
+PCI_STATUS_OFFSET = 0x6
+PCI_CAP_OFFSET = 0x34
+MSIX_BIR_MASK = 0x7
 
+#Calculate PAGE_SHIFT: number of bits to shift an address to get the page number
+PAGE_SIZE = resource.getpagesize()
+PAGE_SHIFT = 0
+t = PAGE_SIZE
+while not (t&1):
+    t>>=1
+    PAGE_SHIFT+=1
+
+PAGE_MASK=~(PAGE_SIZE - 1)
 # Definitions from Linux: include/linux/pci.h
 def PCI_DEVFN(slot, func):
     return ((((slot) & 0x1f) << 3) | ((func) & 0x07))
@@ -74,10 +89,72 @@ class PciDevice:
         self.device = None
         self.subvendor = None
         self.subdevice = None
-
+        self.msix = 0
+        self.msix_iomem = []
         self.get_info_from_sysfs()
 
+    def find_capability(self, type):
+        try:
+            sysfs_mnt = find_sysfs_mnt()
+        except IOError, (errno, strerr):
+            raise PciDeviceParseError(('Failed to locate sysfs mount: %s (%d)' %
+                (PROC_PCI_PATH, strerr, errno)))
+
+        if sysfs_mnt == None:
+            return False
+        path = sysfs_mnt+SYSFS_PCI_DEVS_PATH+'/'+ \
+               self.name+SYSFS_PCI_DEV_CONFIG_PATH
+        try:
+            conf_file = open(path, 'rb')
+            conf_file.seek(PCI_STATUS_OFFSET)
+            status = ord(conf_file.read(1))
+            if status&PCI_STATUS_CAP_MASK:
+                conf_file.seek(PCI_CAP_OFFSET)
+                capa_pointer = ord(conf_file.read(1))
+                while capa_pointer:
+                    conf_file.seek(capa_pointer)
+                    capa_id = ord(conf_file.read(1))
+                    capa_pointer = ord(conf_file.read(1))
+                    if capa_id == type:
+                        # get the type
+                        message_cont_lo = ord(conf_file.read(1))
+                        message_cont_hi = ord(conf_file.read(1))
+                        self.msix=1
+                        self.msix_entries = message_cont_lo + \
+                                            message_cont_hi << 8
+                        t_off=conf_file.read(4)
+                        p_off=conf_file.read(4)
+                        self.table_offset=ord(t_off[0]) | (ord(t_off[1])<<8) | \
+                                          (ord(t_off[2])<<16)|  \
+                                          (ord(t_off[3])<<24)
+                        self.pba_offset=ord(p_off[0]) | (ord(p_off[1]) << 8)| \
+                                        (ord(p_off[2])<<16) | \
+                                        (ord(p_off[3])<<24)
+                        self.table_index = self.table_offset & MSIX_BIR_MASK
+                        self.table_offset = self.table_offset & ~MSIX_BIR_MASK
+                        self.pba_index = self.pba_offset & MSIX_BIR_MASK
+                        self.pba_offset = self.pba_offset & ~MSIX_BIR_MASK
+                        break
+        except IOError, (errno, strerr):
+            raise PciDeviceParseError(('Failed to locate sysfs mount: %s (%d)' %
+                (PROC_PCI_PATH, strerr, errno)))
+
+    def remove_msix_iomem(self, index, start, size):
+        if (index == self.table_index):
+            table_start = start+self.table_offset
+            table_end = table_start + self.msix_entries * 16
+            table_start = table_start & PAGE_MASK
+            table_end = (table_end + PAGE_SIZE) & PAGE_MASK
+            self.msix_iomem.append((table_start, table_end-table_start))
+        if (index==self.pba_index):
+            pba_start = start + self.pba_offset
+            pba_end = pba_start + self.msix_entries/8
+            pba_start = pba_start & PAGE_MASK
+            pba_end = (pba_end + PAGE_SIZE) & PAGE_MASK
+            self.msix_iomem.append((pba_start, pba_end-pba_start))
+
     def get_info_from_sysfs(self):
+        self.find_capability(0x11)
         try:
             sysfs_mnt = find_sysfs_mnt()
         except IOError, (errno, strerr):
@@ -108,6 +185,10 @@ class PciDevice:
                         self.ioports.append( (start,size) )
                     else:
                         self.iomem.append( (start,size) )
+                    if (self.msix):
+                        self.remove_msix_iomem(i, start, size)
+
+
 
         except IOError, (errno, strerr):
             raise PciDeviceParseError(('Failed to open & read %s: %s (%d)' %

tools/python/xen/xend/server/pciif.py

@@ -278,6 +278,19 @@ class PciController(DevController):
                 raise VmError(('pci: failed to map irq on device '+
                             '%s - errno=%d')%(dev.name,rc))
 
+        if dev.msix:
+            for (start, size) in dev.msix_iomem:
+                start_pfn = start>>PAGE_SHIFT
+                nr_pfns = (size+(PAGE_SIZE-1))>>PAGE_SHIFT
+                log.debug('pci-msix: remove permission for 0x%x/0x%x 0x%x/0x%x' % \
+                         (start,size, start_pfn, nr_pfns))
+                rc = xc.domain_iomem_permission(domid = fe_domid,
+                                                first_pfn = start_pfn,
+                                                nr_pfns = nr_pfns,
+                                                allow_access = False)
+                if rc<0:
+                    raise VmError(('pci: failed to remove msi-x iomem'))
+
         if dev.irq>0:
             log.debug('pci: enabling irq %d'%dev.irq)
             rc = xc.domain_irq_permission(domid =  fe_domid, pirq = dev.irq,