diff --git a/xen/drivers/passthrough/amd/iommu.h b/xen/drivers/passthrough/amd/iommu.h index 5ae86cdec8..3c702eb517 100644 --- a/xen/drivers/passthrough/amd/iommu.h +++ b/xen/drivers/passthrough/amd/iommu.h @@ -237,7 +237,8 @@ int amd_iommu_init_late(void); int amd_iommu_update_ivrs_mapping_acpi(void); int iov_adjust_irq_affinities(void); -int amd_iommu_quarantine_init(struct domain *d); +int amd_iommu_quarantine_init(struct pci_dev *pdev, bool scratch_page); +void amd_iommu_quarantine_teardown(struct pci_dev *pdev); /* mapping functions */ int __must_check amd_iommu_map_page(struct domain *d, dfn_t dfn, diff --git a/xen/drivers/passthrough/amd/iommu_map.c b/xen/drivers/passthrough/amd/iommu_map.c index e36595f289..cf6f01b633 100644 --- a/xen/drivers/passthrough/amd/iommu_map.c +++ b/xen/drivers/passthrough/amd/iommu_map.c @@ -598,64 +598,138 @@ int amd_iommu_get_reserved_device_memory(iommu_grdm_t *func, void *ctxt) return 0; } -int __init amd_iommu_quarantine_init(struct domain *d) +static int fill_qpt(union amd_iommu_pte *this, unsigned int level, + struct page_info *pgs[IOMMU_MAX_PT_LEVELS]) { - struct domain_iommu *hd = dom_iommu(d); - unsigned long end_gfn = - 1ul << (DEFAULT_DOMAIN_ADDRESS_WIDTH - PAGE_SHIFT); - unsigned int level = amd_iommu_get_paging_mode(end_gfn); - union amd_iommu_pte *table; + struct domain_iommu *hd = dom_iommu(dom_io); + unsigned int i; + int rc = 0; - if ( hd->arch.amd.root_table ) + for ( i = 0; !rc && i < PTE_PER_TABLE_SIZE; ++i ) { - ASSERT_UNREACHABLE(); - return 0; - } + union amd_iommu_pte *pte = &this[i], *next; - spin_lock(&hd->arch.mapping_lock); - - hd->arch.amd.root_table = iommu_alloc_pgtable(hd); - if ( !hd->arch.amd.root_table ) - goto out; - - table = __map_domain_page(hd->arch.amd.root_table); - while ( level ) - { - struct page_info *pg; - unsigned int i; - - /* - * The pgtable allocator is fine for the leaf page, as well as - * page table pages, and the resulting allocations are always - * zeroed. - */ - pg = iommu_alloc_pgtable(hd); - if ( !pg ) - break; - - for ( i = 0; i < PTE_PER_TABLE_SIZE; i++ ) + if ( !pte->pr ) { - union amd_iommu_pte *pde = &table[i]; + if ( !pgs[level] ) + { + /* + * The pgtable allocator is fine for the leaf page, as well as + * page table pages, and the resulting allocations are always + * zeroed. + */ + pgs[level] = iommu_alloc_pgtable(hd); + if ( !pgs[level] ) + { + rc = -ENOMEM; + break; + } + + if ( level ) + { + next = __map_domain_page(pgs[level]); + rc = fill_qpt(next, level - 1, pgs); + unmap_domain_page(next); + } + } /* * PDEs are essentially a subset of PTEs, so this function * is fine to use even at the leaf. */ - set_iommu_pde_present(pde, mfn_x(page_to_mfn(pg)), level - 1, - false, true); + set_iommu_pde_present(pte, mfn_x(page_to_mfn(pgs[level])), level, + true, true); + } + else if ( level && pte->next_level ) + { + next = map_domain_page(_mfn(pte->mfn)); + rc = fill_qpt(next, level - 1, pgs); + unmap_domain_page(next); } - - unmap_domain_page(table); - table = __map_domain_page(pg); - level--; } - unmap_domain_page(table); - out: - spin_unlock(&hd->arch.mapping_lock); + return rc; +} - /* Pages leaked in failure case */ - return level ? -ENOMEM : 0; +int amd_iommu_quarantine_init(struct pci_dev *pdev, bool scratch_page) +{ + struct domain_iommu *hd = dom_iommu(dom_io); + unsigned long end_gfn = + 1ul << (DEFAULT_DOMAIN_ADDRESS_WIDTH - PAGE_SHIFT); + unsigned int level = amd_iommu_get_paging_mode(end_gfn); + unsigned int req_id = get_dma_requestor_id(pdev->seg, pdev->sbdf.bdf); + const struct ivrs_mappings *ivrs_mappings = get_ivrs_mappings(pdev->seg); + int rc; + + ASSERT(pcidevs_locked()); + ASSERT(!hd->arch.amd.root_table); + ASSERT(page_list_empty(&hd->arch.pgtables.list)); + + if ( !scratch_page && !ivrs_mappings[req_id].unity_map ) + return 0; + + ASSERT(pdev->arch.pseudo_domid != DOMID_INVALID); + + if ( pdev->arch.amd.root_table ) + { + clear_domain_page(pdev->arch.leaf_mfn); + return 0; + } + + pdev->arch.amd.root_table = iommu_alloc_pgtable(hd); + if ( !pdev->arch.amd.root_table ) + return -ENOMEM; + + /* Transiently install the root into DomIO, for iommu_identity_mapping(). */ + hd->arch.amd.root_table = pdev->arch.amd.root_table; + + rc = amd_iommu_reserve_domain_unity_map(dom_io, + ivrs_mappings[req_id].unity_map, + 0); + + iommu_identity_map_teardown(dom_io); + hd->arch.amd.root_table = NULL; + + if ( rc ) + AMD_IOMMU_WARN("%pp: quarantine unity mapping failed\n", &pdev->sbdf); + else if ( scratch_page ) + { + union amd_iommu_pte *root; + struct page_info *pgs[IOMMU_MAX_PT_LEVELS] = {}; + + spin_lock(&hd->arch.mapping_lock); + + root = __map_domain_page(pdev->arch.amd.root_table); + rc = fill_qpt(root, level - 1, pgs); + unmap_domain_page(root); + + pdev->arch.leaf_mfn = page_to_mfn(pgs[0]); + + spin_unlock(&hd->arch.mapping_lock); + } + + page_list_move(&pdev->arch.pgtables_list, &hd->arch.pgtables.list); + + if ( rc ) + amd_iommu_quarantine_teardown(pdev); + + return rc; +} + +void amd_iommu_quarantine_teardown(struct pci_dev *pdev) +{ + struct domain_iommu *hd = dom_iommu(dom_io); + + ASSERT(pcidevs_locked()); + + if ( !pdev->arch.amd.root_table ) + return; + + ASSERT(page_list_empty(&hd->arch.pgtables.list)); + page_list_move(&hd->arch.pgtables.list, &pdev->arch.pgtables_list); + while ( iommu_free_pgtables(dom_io) == -ERESTART ) + /* nothing */; + pdev->arch.amd.root_table = NULL; } /* diff --git a/xen/drivers/passthrough/amd/pci_amd_iommu.c b/xen/drivers/passthrough/amd/pci_amd_iommu.c index 6092a2932b..4792b34bc2 100644 --- a/xen/drivers/passthrough/amd/pci_amd_iommu.c +++ b/xen/drivers/passthrough/amd/pci_amd_iommu.c @@ -26,7 +26,7 @@ #include "../ats.h" /* dom_io is used as a sentinel for quarantined devices */ -#define QUARANTINE_SKIP(d) ((d) == dom_io && !dom_iommu(d)->arch.amd.root_table) +#define QUARANTINE_SKIP(d, p) ((d) == dom_io && !(p)->arch.amd.root_table) static bool_t __read_mostly init_done; @@ -125,8 +125,10 @@ static int __must_check amd_iommu_setup_domain_device( u8 bus = pdev->bus; struct domain_iommu *hd = dom_iommu(domain); const struct ivrs_mappings *ivrs_dev; + const struct page_info *root_pg; + domid_t domid; - if ( QUARANTINE_SKIP(domain) ) + if ( QUARANTINE_SKIP(domain, pdev) ) return 0; BUG_ON(!hd->arch.amd.paging_mode || !iommu->dev_table.buffer); @@ -147,14 +149,25 @@ static int __must_check amd_iommu_setup_domain_device( dte = &table[req_id]; ivrs_dev = &get_ivrs_mappings(iommu->seg)[req_id]; + if ( domain != dom_io ) + { + root_pg = hd->arch.amd.root_table; + domid = domain->domain_id; + } + else + { + root_pg = pdev->arch.amd.root_table; + domid = pdev->arch.pseudo_domid; + } + spin_lock_irqsave(&iommu->lock, flags); if ( !dte->v || !dte->tv ) { /* bind DTE to domain page-tables */ rc = amd_iommu_set_root_page_table( - dte, page_to_maddr(hd->arch.amd.root_table), - domain->domain_id, hd->arch.amd.paging_mode, sr_flags); + dte, page_to_maddr(root_pg), domid, + hd->arch.amd.paging_mode, sr_flags); if ( rc ) { ASSERT(rc < 0); @@ -181,7 +194,7 @@ static int __must_check amd_iommu_setup_domain_device( amd_iommu_flush_device(iommu, req_id); } - else if ( dte->pt_root != mfn_x(page_to_mfn(hd->arch.amd.root_table)) ) + else if ( dte->pt_root != mfn_x(page_to_mfn(root_pg)) ) { /* * Strictly speaking if the device is the only one with this requestor @@ -194,8 +207,8 @@ static int __must_check amd_iommu_setup_domain_device( rc = -EOPNOTSUPP; else rc = amd_iommu_set_root_page_table( - dte, page_to_maddr(hd->arch.amd.root_table), - domain->domain_id, hd->arch.amd.paging_mode, sr_flags); + dte, page_to_maddr(root_pg), domid, + hd->arch.amd.paging_mode, sr_flags); if ( rc < 0 ) { spin_unlock_irqrestore(&iommu->lock, flags); @@ -214,6 +227,7 @@ static int __must_check amd_iommu_setup_domain_device( * intended anyway. */ !pdev->domain->is_dying && + pdev->domain != dom_io && (any_pdev_behind_iommu(pdev->domain, pdev, iommu) || pdev->phantom_stride) ) AMD_IOMMU_WARN(" %pp: reassignment may cause %pd data corruption\n", @@ -246,9 +260,8 @@ static int __must_check amd_iommu_setup_domain_device( AMD_IOMMU_DEBUG("Setup I/O page table: device id = %#x, type = %#x, " "root table = %#"PRIx64", " "domain = %d, paging mode = %d\n", - req_id, pdev->type, - page_to_maddr(hd->arch.amd.root_table), - domain->domain_id, hd->arch.amd.paging_mode); + req_id, pdev->type, page_to_maddr(root_pg), + domid, hd->arch.amd.paging_mode); ASSERT(pcidevs_locked()); @@ -327,7 +340,7 @@ int amd_iommu_alloc_root(struct domain *d) { struct domain_iommu *hd = dom_iommu(d); - if ( unlikely(!hd->arch.amd.root_table) ) + if ( unlikely(!hd->arch.amd.root_table) && d != dom_io ) { hd->arch.amd.root_table = iommu_alloc_pgtable(hd); if ( !hd->arch.amd.root_table ) @@ -391,7 +404,7 @@ static void amd_iommu_disable_domain_device(const struct domain *domain, int req_id; u8 bus = pdev->bus; - if ( QUARANTINE_SKIP(domain) ) + if ( QUARANTINE_SKIP(domain, pdev) ) return; ASSERT(pcidevs_locked()); @@ -430,7 +443,7 @@ static void amd_iommu_disable_domain_device(const struct domain *domain, AMD_IOMMU_DEBUG("Disable: device id = %#x, " "domain = %d, paging mode = %d\n", - req_id, domain->domain_id, + req_id, dte->domain_id, dom_iommu(domain)->arch.amd.paging_mode); } else @@ -453,7 +466,7 @@ static int reassign_device(struct domain *source, struct domain *target, return -ENODEV; } - if ( !QUARANTINE_SKIP(target) ) + if ( !QUARANTINE_SKIP(target, pdev) ) { rc = amd_iommu_setup_domain_device(target, iommu, devfn, pdev); if ( rc ) @@ -655,6 +668,8 @@ static int amd_iommu_remove_device(u8 devfn, struct pci_dev *pdev) AMD_IOMMU_WARN("%pd: unity unmapping failed for %pp\n", pdev->domain, &pdev->sbdf); + amd_iommu_quarantine_teardown(pdev); + iommu_free_domid(pdev->arch.pseudo_domid, iommu->domid_map); pdev->arch.pseudo_domid = DOMID_INVALID; diff --git a/xen/drivers/passthrough/iommu.c b/xen/drivers/passthrough/iommu.c index 6334370109..caaba62c88 100644 --- a/xen/drivers/passthrough/iommu.c +++ b/xen/drivers/passthrough/iommu.c @@ -443,21 +443,22 @@ int iommu_iotlb_flush_all(struct domain *d, unsigned int flush_flags) return rc; } -static int __init iommu_quarantine_init(void) +int iommu_quarantine_dev_init(device_t *dev) { const struct domain_iommu *hd = dom_iommu(dom_io); - int rc; - dom_io->options |= XEN_DOMCTL_CDF_iommu; - - rc = iommu_domain_init(dom_io, 0); - if ( rc || iommu_quarantine < IOMMU_quarantine_scratch_page ) - return rc; - - if ( !hd->platform_ops->quarantine_init ) + if ( !iommu_quarantine || !hd->platform_ops->quarantine_init ) return 0; - return hd->platform_ops->quarantine_init(dom_io); + return iommu_call(hd->platform_ops, quarantine_init, + dev, iommu_quarantine == IOMMU_quarantine_scratch_page); +} + +static int __init iommu_quarantine_init(void) +{ + dom_io->options |= XEN_DOMCTL_CDF_iommu; + + return iommu_domain_init(dom_io, 0); } int __init iommu_setup(void) diff --git a/xen/drivers/passthrough/pci.c b/xen/drivers/passthrough/pci.c index dd1c0ea329..395958698e 100644 --- a/xen/drivers/passthrough/pci.c +++ b/xen/drivers/passthrough/pci.c @@ -852,9 +852,16 @@ static int deassign_device(struct domain *d, uint16_t seg, uint8_t bus, return -ENODEV; /* De-assignment from dom_io should de-quarantine the device */ - target = ((pdev->quarantine || iommu_quarantine) && - pdev->domain != dom_io) ? - dom_io : hardware_domain; + if ( (pdev->quarantine || iommu_quarantine) && pdev->domain != dom_io ) + { + ret = iommu_quarantine_dev_init(pci_to_dev(pdev)); + if ( ret ) + return ret; + + target = dom_io; + } + else + target = hardware_domain; while ( pdev->phantom_stride ) { @@ -1424,6 +1431,13 @@ static int assign_device(struct domain *d, u16 seg, u8 bus, u8 devfn, u32 flag) if ( rc ) goto done; + if ( pdev->domain != dom_io ) + { + rc = iommu_quarantine_dev_init(pci_to_dev(pdev)); + if ( rc ) + goto done; + } + pdev->fault.count = 0; if ( (rc = hd->platform_ops->assign_device(d, devfn, pci_to_dev(pdev), flag)) ) diff --git a/xen/drivers/passthrough/vtd/iommu.c b/xen/drivers/passthrough/vtd/iommu.c index 37339aa044..22242ab208 100644 --- a/xen/drivers/passthrough/vtd/iommu.c +++ b/xen/drivers/passthrough/vtd/iommu.c @@ -45,6 +45,11 @@ /* dom_io is used as a sentinel for quarantined devices */ #define QUARANTINE_SKIP(d, pgd_maddr) ((d) == dom_io && !(pgd_maddr)) +#define DEVICE_DOMID(d, pdev) ((d) != dom_io ? (d)->domain_id \ + : (pdev)->arch.pseudo_domid) +#define DEVICE_PGTABLE(d, pdev) ((d) != dom_io \ + ? dom_iommu(d)->arch.vtd.pgd_maddr \ + : (pdev)->arch.vtd.pgd_maddr) /* Possible unfiltered LAPIC/MSI messages from untrusted sources? */ bool __read_mostly untrusted_msi; @@ -88,13 +93,18 @@ static int get_iommu_did(domid_t domid, const struct vtd_iommu *iommu, #define DID_FIELD_WIDTH 16 #define DID_HIGH_OFFSET 8 + +/* + * This function may have "context" passed as NULL, to merely obtain a DID + * for "domid". + */ static int context_set_domain_id(struct context_entry *context, domid_t domid, struct vtd_iommu *iommu) { unsigned long nr_dom, i; int found = 0; - ASSERT(spin_is_locked(&iommu->lock)); + ASSERT(pcidevs_locked()); nr_dom = cap_ndoms(iommu->cap); i = find_first_bit(iommu->domid_bitmap, nr_dom); @@ -120,8 +130,13 @@ static int context_set_domain_id(struct context_entry *context, } set_bit(i, iommu->domid_bitmap); - context->hi &= ~(((1 << DID_FIELD_WIDTH) - 1) << DID_HIGH_OFFSET); - context->hi |= (i & ((1 << DID_FIELD_WIDTH) - 1)) << DID_HIGH_OFFSET; + + if ( context ) + { + context->hi &= ~(((1 << DID_FIELD_WIDTH) - 1) << DID_HIGH_OFFSET); + context->hi |= (i & ((1 << DID_FIELD_WIDTH) - 1)) << DID_HIGH_OFFSET; + } + return 0; } @@ -171,8 +186,12 @@ static void check_cleanup_domid_map(const struct domain *d, const struct pci_dev *exclude, struct vtd_iommu *iommu) { - bool found = any_pdev_behind_iommu(d, exclude, iommu); + bool found; + if ( d == dom_io ) + return; + + found = any_pdev_behind_iommu(d, exclude, iommu); /* * Hidden devices are associated with DomXEN but usable by the hardware * domain. Hence they need considering here as well. @@ -1426,7 +1445,7 @@ int domain_context_mapping_one( domid = iommu->domid_map[prev_did]; if ( domid < DOMID_FIRST_RESERVED ) prev_dom = rcu_lock_domain_by_id(domid); - else if ( domid == DOMID_IO ) + else if ( pdev ? domid == pdev->arch.pseudo_domid : domid > DOMID_MASK ) prev_dom = rcu_lock_domain(dom_io); if ( !prev_dom ) { @@ -1582,15 +1601,12 @@ int domain_context_mapping_one( { if ( !prev_dom ) ret = domain_context_unmap_one(domain, iommu, bus, devfn, - domain->domain_id); + DEVICE_DOMID(domain, pdev)); else if ( prev_dom != domain ) /* Avoid infinite recursion. */ - { - hd = dom_iommu(prev_dom); ret = domain_context_mapping_one(prev_dom, iommu, bus, devfn, pdev, - domain->domain_id, - hd->arch.vtd.pgd_maddr, + DEVICE_DOMID(prev_dom, pdev), + DEVICE_PGTABLE(prev_dom, pdev), mode & MAP_WITH_RMRR) < 0; - } else ret = 1; @@ -1612,7 +1628,7 @@ static int domain_context_mapping(struct domain *domain, u8 devfn, { const struct acpi_drhd_unit *drhd = acpi_find_matched_drhd_unit(pdev); const struct acpi_rmrr_unit *rmrr; - paddr_t pgd_maddr = dom_iommu(domain)->arch.vtd.pgd_maddr; + paddr_t pgd_maddr = DEVICE_PGTABLE(domain, pdev); domid_t orig_domid = pdev->arch.pseudo_domid; int ret = 0; unsigned int i, mode = 0; @@ -1641,7 +1657,7 @@ static int domain_context_mapping(struct domain *domain, u8 devfn, break; } - if ( domain != pdev->domain ) + if ( domain != pdev->domain && pdev->domain != dom_io ) { if ( pdev->domain->is_dying ) mode |= MAP_OWNER_DYING; @@ -1683,8 +1699,8 @@ static int domain_context_mapping(struct domain *domain, u8 devfn, if ( iommu_debug ) printk(VTDPREFIX "%pd:PCIe: map %pp\n", domain, &PCI_SBDF3(seg, bus, devfn)); - ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn, - pdev, domain->domain_id, pgd_maddr, + ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn, pdev, + DEVICE_DOMID(domain, pdev), pgd_maddr, mode); if ( ret > 0 ) ret = 0; @@ -1710,8 +1726,8 @@ static int domain_context_mapping(struct domain *domain, u8 devfn, domain, &PCI_SBDF3(seg, bus, devfn)); ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn, - pdev, domain->domain_id, pgd_maddr, - mode); + pdev, DEVICE_DOMID(domain, pdev), + pgd_maddr, mode); if ( ret < 0 ) break; prev_present = ret; @@ -1739,8 +1755,8 @@ static int domain_context_mapping(struct domain *domain, u8 devfn, */ if ( ret >= 0 ) ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn, - NULL, domain->domain_id, pgd_maddr, - mode); + NULL, DEVICE_DOMID(domain, pdev), + pgd_maddr, mode); /* * Devices behind PCIe-to-PCI/PCIx bridge may generate different @@ -1755,8 +1771,8 @@ static int domain_context_mapping(struct domain *domain, u8 devfn, if ( !ret && pdev_type(seg, bus, devfn) == DEV_TYPE_PCIe2PCI_BRIDGE && (secbus != pdev->bus || pdev->devfn != 0) ) ret = domain_context_mapping_one(domain, drhd->iommu, secbus, 0, - NULL, domain->domain_id, pgd_maddr, - mode); + NULL, DEVICE_DOMID(domain, pdev), + pgd_maddr, mode); if ( ret ) { @@ -1798,9 +1814,6 @@ int domain_context_unmap_one( int iommu_domid, rc, ret; bool_t flush_dev_iotlb; - if ( QUARANTINE_SKIP(domain, dom_iommu(domain)->arch.vtd.pgd_maddr) ) - return 0; - ASSERT(pcidevs_locked()); spin_lock(&iommu->lock); @@ -1902,7 +1915,7 @@ static const struct acpi_drhd_unit *domain_context_unmap( printk(VTDPREFIX "%pd:PCIe: unmap %pp\n", domain, &PCI_SBDF3(seg, bus, devfn)); ret = domain_context_unmap_one(domain, iommu, bus, devfn, - domain->domain_id); + DEVICE_DOMID(domain, pdev)); if ( !ret && devfn == pdev->devfn && ats_device(pdev, drhd) > 0 ) disable_ats_device(pdev); @@ -1916,7 +1929,7 @@ static const struct acpi_drhd_unit *domain_context_unmap( printk(VTDPREFIX "%pd:PCI: unmap %pp\n", domain, &PCI_SBDF3(seg, bus, devfn)); ret = domain_context_unmap_one(domain, iommu, bus, devfn, - domain->domain_id); + DEVICE_DOMID(domain, pdev)); if ( ret ) break; @@ -1939,18 +1952,12 @@ static const struct acpi_drhd_unit *domain_context_unmap( break; } + ret = domain_context_unmap_one(domain, iommu, tmp_bus, tmp_devfn, + DEVICE_DOMID(domain, pdev)); /* PCIe to PCI/PCIx bridge */ - if ( pdev_type(seg, tmp_bus, tmp_devfn) == DEV_TYPE_PCIe2PCI_BRIDGE ) - { - ret = domain_context_unmap_one(domain, iommu, tmp_bus, tmp_devfn, - domain->domain_id); - if ( !ret ) - ret = domain_context_unmap_one(domain, iommu, secbus, 0, - domain->domain_id); - } - else /* Legacy PCI bridge */ - ret = domain_context_unmap_one(domain, iommu, tmp_bus, tmp_devfn, - domain->domain_id); + if ( !ret && pdev_type(seg, tmp_bus, tmp_devfn) == DEV_TYPE_PCIe2PCI_BRIDGE ) + ret = domain_context_unmap_one(domain, iommu, secbus, 0, + DEVICE_DOMID(domain, pdev)); break; @@ -1961,7 +1968,7 @@ static const struct acpi_drhd_unit *domain_context_unmap( } if ( !ret && pdev->devfn == devfn && - !QUARANTINE_SKIP(domain, dom_iommu(domain)->arch.vtd.pgd_maddr) ) + !QUARANTINE_SKIP(domain, pdev->arch.vtd.pgd_maddr) ) check_cleanup_domid_map(domain, pdev, iommu); return drhd; @@ -1994,6 +2001,26 @@ static void iommu_domain_teardown(struct domain *d) XFREE(hd->arch.vtd.iommu_bitmap); } +static void quarantine_teardown(struct pci_dev *pdev, + const struct acpi_drhd_unit *drhd) +{ + struct domain_iommu *hd = dom_iommu(dom_io); + + ASSERT(pcidevs_locked()); + + if ( !pdev->arch.vtd.pgd_maddr ) + return; + + ASSERT(page_list_empty(&hd->arch.pgtables.list)); + page_list_move(&hd->arch.pgtables.list, &pdev->arch.pgtables_list); + while ( iommu_free_pgtables(dom_io) == -ERESTART ) + /* nothing */; + pdev->arch.vtd.pgd_maddr = 0; + + if ( drhd ) + cleanup_domid_map(pdev->arch.pseudo_domid, drhd->iommu); +} + static int __must_check intel_iommu_map_page(struct domain *d, dfn_t dfn, mfn_t mfn, unsigned int flags, unsigned int *flush_flags) @@ -2218,6 +2245,8 @@ static int intel_iommu_remove_device(u8 devfn, struct pci_dev *pdev) rmrr->end_address, 0); } + quarantine_teardown(pdev, drhd); + if ( drhd ) { iommu_free_domid(pdev->arch.pseudo_domid, @@ -2576,7 +2605,7 @@ static int reassign_device_ownership( { int ret; - if ( !QUARANTINE_SKIP(target, dom_iommu(target)->arch.vtd.pgd_maddr) ) + if ( !QUARANTINE_SKIP(target, pdev->arch.vtd.pgd_maddr) ) { if ( !has_arch_pdevs(target) ) vmx_pi_hooks_assign(target); @@ -2592,7 +2621,7 @@ static int reassign_device_ownership( ret = domain_context_mapping(target, devfn, pdev); if ( !ret && pdev->devfn == devfn && - !QUARANTINE_SKIP(source, dom_iommu(source)->arch.vtd.pgd_maddr) ) + !QUARANTINE_SKIP(source, pdev->arch.vtd.pgd_maddr) ) { const struct acpi_drhd_unit *drhd = acpi_find_matched_drhd_unit(pdev); @@ -2913,69 +2942,135 @@ static void vtd_dump_page_tables(struct domain *d) agaw_to_level(hd->arch.vtd.agaw), 0, 0); } -static int __init intel_iommu_quarantine_init(struct domain *d) +static int fill_qpt(struct dma_pte *this, unsigned int level, + struct page_info *pgs[6]) { - struct domain_iommu *hd = dom_iommu(d); - struct page_info *pg; - struct dma_pte *parent; - unsigned int agaw = width_to_agaw(DEFAULT_DOMAIN_ADDRESS_WIDTH); - unsigned int level = agaw_to_level(agaw); + struct domain_iommu *hd = dom_iommu(dom_io); + unsigned int i; int rc = 0; - spin_lock(&hd->arch.mapping_lock); - - if ( hd->arch.vtd.pgd_maddr ) + for ( i = 0; !rc && i < PTE_NUM; ++i ) { - ASSERT_UNREACHABLE(); - goto out; + struct dma_pte *pte = &this[i], *next; + + if ( !dma_pte_present(*pte) ) + { + if ( !pgs[level] ) + { + /* + * The pgtable allocator is fine for the leaf page, as well as + * page table pages, and the resulting allocations are always + * zeroed. + */ + pgs[level] = iommu_alloc_pgtable(hd); + if ( !pgs[level] ) + { + rc = -ENOMEM; + break; + } + + if ( level ) + { + next = map_vtd_domain_page(page_to_maddr(pgs[level])); + rc = fill_qpt(next, level - 1, pgs); + unmap_vtd_domain_page(next); + } + } + + dma_set_pte_addr(*pte, page_to_maddr(pgs[level])); + dma_set_pte_readable(*pte); + dma_set_pte_writable(*pte); + } + else if ( level && !dma_pte_superpage(*pte) ) + { + next = map_vtd_domain_page(dma_pte_addr(*pte)); + rc = fill_qpt(next, level - 1, pgs); + unmap_vtd_domain_page(next); + } } + return rc; +} + +static int intel_iommu_quarantine_init(struct pci_dev *pdev, bool scratch_page) +{ + struct domain_iommu *hd = dom_iommu(dom_io); + struct page_info *pg; + unsigned int agaw = width_to_agaw(DEFAULT_DOMAIN_ADDRESS_WIDTH); + unsigned int level = agaw_to_level(agaw); + const struct acpi_drhd_unit *drhd; + const struct acpi_rmrr_unit *rmrr; + unsigned int i, bdf; + bool rmrr_found = false; + int rc; + + ASSERT(pcidevs_locked()); + ASSERT(!hd->arch.vtd.pgd_maddr); + ASSERT(page_list_empty(&hd->arch.pgtables.list)); + + if ( pdev->arch.vtd.pgd_maddr ) + { + clear_domain_page(pdev->arch.leaf_mfn); + return 0; + } + + drhd = acpi_find_matched_drhd_unit(pdev); + if ( !drhd ) + return -ENODEV; + pg = iommu_alloc_pgtable(hd); - - rc = -ENOMEM; if ( !pg ) - goto out; + return -ENOMEM; + rc = context_set_domain_id(NULL, pdev->arch.pseudo_domid, drhd->iommu); + + /* Transiently install the root into DomIO, for iommu_identity_mapping(). */ hd->arch.vtd.pgd_maddr = page_to_maddr(pg); - parent = map_vtd_domain_page(hd->arch.vtd.pgd_maddr); - while ( level ) + for_each_rmrr_device ( rmrr, bdf, i ) { - uint64_t maddr; - unsigned int offset; + if ( rc ) + break; - /* - * The pgtable allocator is fine for the leaf page, as well as - * page table pages, and the resulting allocations are always - * zeroed. - */ - pg = iommu_alloc_pgtable(hd); - - if ( !pg ) - goto out; - - maddr = page_to_maddr(pg); - for ( offset = 0; offset < PTE_NUM; offset++ ) + if ( rmrr->segment == pdev->seg && bdf == pdev->sbdf.bdf ) { - struct dma_pte *pte = &parent[offset]; + rmrr_found = true; - dma_set_pte_addr(*pte, maddr); - dma_set_pte_readable(*pte); + rc = iommu_identity_mapping(dom_io, p2m_access_rw, + rmrr->base_address, rmrr->end_address, + 0); + if ( rc ) + printk(XENLOG_ERR VTDPREFIX + "%pp: RMRR quarantine mapping failed\n", + &pdev->sbdf); } - iommu_sync_cache(parent, PAGE_SIZE); - - unmap_vtd_domain_page(parent); - parent = map_vtd_domain_page(maddr); - level--; } - unmap_vtd_domain_page(parent); - rc = 0; + iommu_identity_map_teardown(dom_io); + hd->arch.vtd.pgd_maddr = 0; + pdev->arch.vtd.pgd_maddr = page_to_maddr(pg); - out: - spin_unlock(&hd->arch.mapping_lock); + if ( !rc && scratch_page ) + { + struct dma_pte *root; + struct page_info *pgs[6] = {}; + + spin_lock(&hd->arch.mapping_lock); + + root = map_vtd_domain_page(pdev->arch.vtd.pgd_maddr); + rc = fill_qpt(root, level - 1, pgs); + unmap_vtd_domain_page(root); + + pdev->arch.leaf_mfn = page_to_mfn(pgs[0]); + + spin_unlock(&hd->arch.mapping_lock); + } + + page_list_move(&pdev->arch.pgtables_list, &hd->arch.pgtables.list); + + if ( rc || (!scratch_page && !rmrr_found) ) + quarantine_teardown(pdev, drhd); - /* Pages may be leaked in failure case */ return rc; } diff --git a/xen/drivers/passthrough/vtd/iommu.h b/xen/drivers/passthrough/vtd/iommu.h index 025895c7d0..f2a0f28298 100644 --- a/xen/drivers/passthrough/vtd/iommu.h +++ b/xen/drivers/passthrough/vtd/iommu.h @@ -482,7 +482,7 @@ struct vtd_iommu { u32 nr_pt_levels; u64 cap; u64 ecap; - spinlock_t lock; /* protect context, domain ids */ + spinlock_t lock; /* protect context */ spinlock_t register_lock; /* protect iommu register handling */ u64 root_maddr; /* root entry machine address */ nodeid_t node; diff --git a/xen/include/asm-x86/pci.h b/xen/include/asm-x86/pci.h index f944017128..c8e1a9ecdb 100644 --- a/xen/include/asm-x86/pci.h +++ b/xen/include/asm-x86/pci.h @@ -1,6 +1,8 @@ #ifndef __X86_PCI_H__ #define __X86_PCI_H__ +#include + #define CF8_BDF(cf8) ( ((cf8) & 0x00ffff00) >> 8) #define CF8_ADDR_LO(cf8) ( (cf8) & 0x000000fc) #define CF8_ADDR_HI(cf8) ( ((cf8) & 0x0f000000) >> 16) @@ -18,7 +20,18 @@ struct arch_pci_dev { * them don't race (de)initialization and hence don't strictly need any * locking. */ + union { + /* Subset of struct arch_iommu's fields, to be used in dom_io. */ + struct { + uint64_t pgd_maddr; + } vtd; + struct { + struct page_info *root_table; + } amd; + }; domid_t pseudo_domid; + mfn_t leaf_mfn; + struct page_list_head pgtables_list; }; int pci_conf_write_intercept(unsigned int seg, unsigned int bdf, diff --git a/xen/include/xen/iommu.h b/xen/include/xen/iommu.h index 6b2cdffa4a..92b2d23f0b 100644 --- a/xen/include/xen/iommu.h +++ b/xen/include/xen/iommu.h @@ -233,7 +233,7 @@ typedef int iommu_grdm_t(xen_pfn_t start, xen_ulong_t nr, u32 id, void *ctxt); struct iommu_ops { int (*init)(struct domain *d); void (*hwdom_init)(struct domain *d); - int (*quarantine_init)(struct domain *d); + int (*quarantine_init)(device_t *dev, bool scratch_page); int (*add_device)(u8 devfn, device_t *dev); int (*enable_device)(device_t *dev); int (*remove_device)(u8 devfn, device_t *dev); @@ -350,6 +350,7 @@ int __must_check iommu_suspend(void); void iommu_resume(void); void iommu_crash_shutdown(void); int iommu_get_reserved_device_memory(iommu_grdm_t *, void *); +int iommu_quarantine_dev_init(device_t *dev); #ifdef CONFIG_HAS_PCI int iommu_do_pci_domctl(struct xen_domctl *, struct domain *d,