diff --git a/xen/arch/x86/hvm/emulate.c b/xen/arch/x86/hvm/emulate.c index f2bc6967df..d3006f094a 100644 --- a/xen/arch/x86/hvm/emulate.c +++ b/xen/arch/x86/hvm/emulate.c @@ -2741,7 +2741,7 @@ int hvm_emulate_one_mmio(unsigned long mfn, unsigned long gla) .write = mmio_ro_emulated_write, .validate = hvmemul_validate, }; - struct mmio_ro_emulate_ctxt mmio_ro_ctxt = { .cr2 = gla }; + struct mmio_ro_emulate_ctxt mmio_ro_ctxt = { .cr2 = gla, .mfn = _mfn(mfn) }; struct hvm_emulate_ctxt ctxt; const struct x86_emulate_ops *ops; unsigned int seg, bdf; diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c index 018d44a08b..74e58c653e 100644 --- a/xen/arch/x86/hvm/hvm.c +++ b/xen/arch/x86/hvm/hvm.c @@ -2025,8 +2025,8 @@ int hvm_hap_nested_page_fault(paddr_t gpa, unsigned long gla, goto out_put_gfn; } - if ( (p2mt == p2m_mmio_direct) && is_hardware_domain(currd) && - npfec.write_access && npfec.present && + if ( (p2mt == p2m_mmio_direct) && npfec.write_access && npfec.present && + (is_hardware_domain(currd) || subpage_mmio_write_accept(mfn, gla)) && (hvm_emulate_one_mmio(mfn_x(mfn), gla) == X86EMUL_OKAY) ) { rc = 1; diff --git a/xen/arch/x86/include/asm/mm.h b/xen/arch/x86/include/asm/mm.h index 71a29b2cb3..6c7e66ee21 100644 --- a/xen/arch/x86/include/asm/mm.h +++ b/xen/arch/x86/include/asm/mm.h @@ -522,9 +522,32 @@ extern struct rangeset *mmio_ro_ranges; void memguard_guard_stack(void *p); void memguard_unguard_stack(void *p); +/* + * Add more precise r/o marking for a MMIO page. Range specified here + * will still be R/O, but the rest of the page (not marked as R/O via another + * call) will have writes passed through. + * The start address and the size must be aligned to MMIO_RO_SUBPAGE_GRAN. + * + * This API cannot be used for overlapping ranges, nor for pages already added + * to mmio_ro_ranges separately. + * + * Since there is currently no subpage_mmio_ro_remove(), relevant device should + * not be hot-unplugged. + * + * Return values: + * - negative: error + * - 0: success + */ +#define MMIO_RO_SUBPAGE_GRAN 8 +int subpage_mmio_ro_add(paddr_t start, size_t size); +bool subpage_mmio_write_accept(mfn_t mfn, unsigned long gla); + struct mmio_ro_emulate_ctxt { unsigned long cr2; + /* Used only for mmcfg case */ unsigned int seg, bdf; + /* Used only for non-mmcfg case */ + mfn_t mfn; }; int cf_check mmio_ro_emulated_write( diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index d537a799bc..11933321a3 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -153,6 +153,17 @@ struct rangeset *__read_mostly mmio_ro_ranges; static uint32_t __ro_after_init base_disallow_mask; +/* Handling sub-page read-only MMIO regions */ +struct subpage_ro_range { + struct list_head list; + mfn_t mfn; + void __iomem *mapped; + DECLARE_BITMAP(ro_elems, PAGE_SIZE / MMIO_RO_SUBPAGE_GRAN); +}; + +static LIST_HEAD_RO_AFTER_INIT(subpage_ro_ranges); +static DEFINE_SPINLOCK(subpage_ro_lock); + /* Global bit is allowed to be set on L1 PTEs. Intended for user mappings. */ #define L1_DISALLOW_MASK ((base_disallow_mask | _PAGE_GNTTAB) & ~_PAGE_GLOBAL) @@ -4915,6 +4926,253 @@ long arch_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) return rc; } +static struct subpage_ro_range *subpage_mmio_find_page(mfn_t mfn) +{ + struct subpage_ro_range *entry; + + list_for_each_entry(entry, &subpage_ro_ranges, list) + if ( mfn_eq(entry->mfn, mfn) ) + return entry; + + return NULL; +} + +/* + * Mark part of the page as R/O. + * Returns: + * - 0 on success - first range in the page + * - 1 on success - subsequent range in the page + * - <0 on error + */ +static int __init subpage_mmio_ro_add_page( + mfn_t mfn, + unsigned int offset_s, + unsigned int offset_e) +{ + struct subpage_ro_range *entry = NULL; + bool new_entry = false; + unsigned int i; + + entry = subpage_mmio_find_page(mfn); + if ( !entry ) + { + entry = xzalloc(struct subpage_ro_range); + if ( !entry ) + return -ENOMEM; + entry->mfn = mfn; + list_add(&entry->list, &subpage_ro_ranges); + new_entry = true; + } + + for ( i = offset_s; i <= offset_e; i += MMIO_RO_SUBPAGE_GRAN ) + { + bool oldbit = __test_and_set_bit(i / MMIO_RO_SUBPAGE_GRAN, + entry->ro_elems); + ASSERT(!oldbit); + } + + return !new_entry; +} + +static void __init subpage_mmio_ro_remove_page( + mfn_t mfn, + unsigned int offset_s, + unsigned int offset_e) +{ + struct subpage_ro_range *entry = NULL; + unsigned int i; + + entry = subpage_mmio_find_page(mfn); + if ( !entry ) + return; + + for ( i = offset_s; i <= offset_e; i += MMIO_RO_SUBPAGE_GRAN ) + __clear_bit(i / MMIO_RO_SUBPAGE_GRAN, entry->ro_elems); + + if ( !bitmap_empty(entry->ro_elems, PAGE_SIZE / MMIO_RO_SUBPAGE_GRAN) ) + return; + + list_del(&entry->list); + if ( entry->mapped ) + iounmap(entry->mapped); + xfree(entry); +} + +int __init subpage_mmio_ro_add( + paddr_t start, + size_t size) +{ + mfn_t mfn_start = maddr_to_mfn(start); + paddr_t end = start + size - 1; + mfn_t mfn_end = maddr_to_mfn(end); + unsigned int offset_end = 0; + int rc; + bool subpage_start, subpage_end; + + ASSERT(IS_ALIGNED(start, MMIO_RO_SUBPAGE_GRAN)); + ASSERT(IS_ALIGNED(size, MMIO_RO_SUBPAGE_GRAN)); + if ( !IS_ALIGNED(start, MMIO_RO_SUBPAGE_GRAN) || + !IS_ALIGNED(size, MMIO_RO_SUBPAGE_GRAN) ) + return -EINVAL; + + if ( !size ) + return 0; + + if ( mfn_eq(mfn_start, mfn_end) ) + { + /* Both starting and ending parts handled at once */ + subpage_start = PAGE_OFFSET(start) || PAGE_OFFSET(end) != PAGE_SIZE - 1; + subpage_end = false; + } + else + { + subpage_start = PAGE_OFFSET(start); + subpage_end = PAGE_OFFSET(end) != PAGE_SIZE - 1; + } + + if ( subpage_start ) + { + offset_end = mfn_eq(mfn_start, mfn_end) ? + PAGE_OFFSET(end) : + (PAGE_SIZE - 1); + rc = subpage_mmio_ro_add_page(mfn_start, + PAGE_OFFSET(start), + offset_end); + if ( rc < 0 ) + goto err_unlock; + /* Check if not marking R/W part of a page intended to be fully R/O */ + ASSERT(rc || !rangeset_contains_singleton(mmio_ro_ranges, + mfn_x(mfn_start))); + } + + if ( subpage_end ) + { + rc = subpage_mmio_ro_add_page(mfn_end, 0, PAGE_OFFSET(end)); + if ( rc < 0 ) + goto err_unlock_remove; + /* Check if not marking R/W part of a page intended to be fully R/O */ + ASSERT(rc || !rangeset_contains_singleton(mmio_ro_ranges, + mfn_x(mfn_end))); + } + + rc = rangeset_add_range(mmio_ro_ranges, mfn_x(mfn_start), mfn_x(mfn_end)); + if ( rc ) + goto err_remove; + + return 0; + + err_remove: + if ( subpage_end ) + subpage_mmio_ro_remove_page(mfn_end, 0, PAGE_OFFSET(end)); + err_unlock_remove: + if ( subpage_start ) + subpage_mmio_ro_remove_page(mfn_start, PAGE_OFFSET(start), offset_end); + err_unlock: + return rc; +} + +static void __iomem *subpage_mmio_map_page( + struct subpage_ro_range *entry) +{ + void __iomem *mapped_page; + + if ( entry->mapped ) + return entry->mapped; + + mapped_page = ioremap(mfn_to_maddr(entry->mfn), PAGE_SIZE); + + spin_lock(&subpage_ro_lock); + /* Re-check under the lock */ + if ( entry->mapped ) + { + spin_unlock(&subpage_ro_lock); + if ( mapped_page ) + iounmap(mapped_page); + return entry->mapped; + } + + entry->mapped = mapped_page; + spin_unlock(&subpage_ro_lock); + return entry->mapped; +} + +static void subpage_mmio_write_emulate( + mfn_t mfn, + unsigned int offset, + const void *data, + unsigned int len) +{ + struct subpage_ro_range *entry; + volatile void __iomem *addr; + + entry = subpage_mmio_find_page(mfn); + if ( !entry ) + /* Do not print message for pages without any writable parts. */ + return; + + if ( test_bit(offset / MMIO_RO_SUBPAGE_GRAN, entry->ro_elems) ) + { + write_ignored: + gprintk(XENLOG_WARNING, + "ignoring write to R/O MMIO 0x%"PRI_mfn"%03x len %u\n", + mfn_x(mfn), offset, len); + return; + } + + addr = subpage_mmio_map_page(entry); + if ( !addr ) + { + gprintk(XENLOG_ERR, + "Failed to map page for MMIO write at 0x%"PRI_mfn"%03x\n", + mfn_x(mfn), offset); + return; + } + + switch ( len ) + { + case 1: + writeb(*(const uint8_t*)data, addr); + break; + case 2: + writew(*(const uint16_t*)data, addr); + break; + case 4: + writel(*(const uint32_t*)data, addr); + break; + case 8: + writeq(*(const uint64_t*)data, addr); + break; + default: + /* mmio_ro_emulated_write() already validated the size */ + ASSERT_UNREACHABLE(); + goto write_ignored; + } +} + +#ifdef CONFIG_HVM +bool subpage_mmio_write_accept(mfn_t mfn, unsigned long gla) +{ + unsigned int offset = PAGE_OFFSET(gla); + const struct subpage_ro_range *entry; + + entry = subpage_mmio_find_page(mfn); + if ( !entry ) + return false; + + if ( !test_bit(offset / MMIO_RO_SUBPAGE_GRAN, entry->ro_elems) ) + { + /* + * We don't know the write size at this point yet, so it could be + * an unaligned write, but accept it here anyway and deal with it + * later. + */ + return true; + } + + return false; +} +#endif + int cf_check mmio_ro_emulated_write( enum x86_segment seg, unsigned long offset, @@ -4933,6 +5191,9 @@ int cf_check mmio_ro_emulated_write( return X86EMUL_UNHANDLEABLE; } + subpage_mmio_write_emulate(mmio_ro_ctxt->mfn, PAGE_OFFSET(offset), + p_data, bytes); + return X86EMUL_OKAY; } diff --git a/xen/arch/x86/pv/ro-page-fault.c b/xen/arch/x86/pv/ro-page-fault.c index 6a739f9e9a..372187e9a0 100644 --- a/xen/arch/x86/pv/ro-page-fault.c +++ b/xen/arch/x86/pv/ro-page-fault.c @@ -333,8 +333,10 @@ static int mmio_ro_do_page_fault(struct x86_emulate_ctxt *ctxt, ctxt->data = &mmio_ro_ctxt; if ( pci_ro_mmcfg_decode(mfn_x(mfn), &mmio_ro_ctxt.seg, &mmio_ro_ctxt.bdf) ) return x86_emulate(ctxt, &mmcfg_intercept_ops); - else - return x86_emulate(ctxt, &mmio_ro_emulate_ops); + + mmio_ro_ctxt.mfn = mfn; + + return x86_emulate(ctxt, &mmio_ro_emulate_ops); } int pv_ro_page_fault(unsigned long addr, struct cpu_user_regs *regs)