diff --git a/xen/arch/x86/include/asm/config.h b/xen/arch/x86/include/asm/config.h index 3553bf89dc..282363cd2c 100644 --- a/xen/arch/x86/include/asm/config.h +++ b/xen/arch/x86/include/asm/config.h @@ -249,11 +249,30 @@ extern unsigned long xen_phys_start; #define GDT_LDT_VIRT_START PERDOMAIN_VIRT_SLOT(0) #define GDT_LDT_VIRT_END (GDT_LDT_VIRT_START + (GDT_LDT_MBYTES << 20)) -/* The address of a particular VCPU's GDT or LDT. */ +/* + * The address of a particular VCPU's GDT or LDT; each is the architectural + * maximum of 64k. + */ +#define GDT_VIRT_SIZE KB(64) #define GDT_VIRT_START(v) \ (PERDOMAIN_VIRT_START + ((v)->vcpu_id << GDT_LDT_VCPU_VA_SHIFT)) #define LDT_VIRT_START(v) \ - (GDT_VIRT_START(v) + (64*1024)) + (GDT_VIRT_START(v) + GDT_VIRT_SIZE) + +/* + * The vCPU's effective root page table, mapped read-only in the last + * page of its GDT: the page after Xen's reserved range, beyond the + * GDT limit Xen loads (LAST_RESERVED_GDT_BYTE), so no selector can + * reach it. "Effective" is the root page table the vCPU is meant to + * run on after any transformation: the guest's own, or its shadow + * under shadow paging. XPTI does not transform it further but copies + * from it: the exit path reads the guest-visible L4 slots from here. + * Adding a reserved GDT page would mean moving the window out of the + * GDT first; pv_map_eff_root_pt() checks the layout at build time. + */ +#define PV_EFF_ROOT_PT_PAGE (FIRST_RESERVED_GDT_PAGE + 1) +#define PV_EFF_ROOT_PT_OFFSET (PV_EFF_ROOT_PT_PAGE << PAGE_SHIFT) +#define PV_EFF_ROOT_PT(v) (GDT_VIRT_START(v) + PV_EFF_ROOT_PT_OFFSET) /* map_domain_page() map cache. The second per-domain-mapping sub-area. */ #define MAPCACHE_VCPU_ENTRIES (CONFIG_PAGING_LEVELS * CONFIG_PAGING_LEVELS) diff --git a/xen/arch/x86/include/asm/desc.h b/xen/arch/x86/include/asm/desc.h index 85fae6b2f9..d15ae69035 100644 --- a/xen/arch/x86/include/asm/desc.h +++ b/xen/arch/x86/include/asm/desc.h @@ -6,6 +6,16 @@ /* * Xen reserves a memory page of GDT entries. * No guest GDT entries exist beyond the Xen reserved area. + * + * The page after the reserved area, the last of the GDT's 64k, is the + * PV effective root-table window (PV_EFF_ROOT_PT_PAGE, config.h): + * beyond the GDT limit Xen loads (LAST_RESERVED_GDT_BYTE), so no + * selector reaches it. Xen uses only a few of the reserved page's + * 512 slots, so more Xen descriptors never need another page; if one + * were needed regardless, the window would have to move out of the + * GDT first (into a per-domain sub-area of its own), as this is the + * GDT's last page. pv_map_eff_root_pt() checks the layout at build + * time. */ #define NR_RESERVED_GDT_PAGES 1 #define NR_RESERVED_GDT_BYTES (NR_RESERVED_GDT_PAGES * PAGE_SIZE) diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index d02c9862d3..6095f8e623 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -511,13 +511,52 @@ void share_xen_page_with_guest(struct page_info *page, struct domain *d, nrspin_unlock(&d->page_alloc_lock); } +/* + * Map @mfn read-only at v's PV_EFF_ROOT_PT() window. The window lives in + * v's per-domain area, so v need not be current; the local flush is for + * when it is, with its own page tables loaded. + */ +static void pv_map_eff_root_pt(const struct vcpu *v, mfn_t mfn) +{ + /* + * The window must lie beyond the GDT limit Xen loads, so that no + * selector reaches it, and within the GDT's 64k. + */ + BUILD_BUG_ON(PV_EFF_ROOT_PT_OFFSET <= LAST_RESERVED_GDT_BYTE); + BUILD_BUG_ON(PV_EFF_ROOT_PT_OFFSET + PAGE_SIZE > GDT_VIRT_SIZE); + + l1e_write(pv_gdt_ptes(v) + PV_EFF_ROOT_PT_PAGE, + l1e_from_mfn(mfn, __PAGE_HYPERVISOR_RO)); + flush_tlb_one_local(PV_EFF_ROOT_PT(v)); +} + +/* + * Clear the window. Its lifetime is v->arch.cr3's: make_cr3() writes it + * wherever v->arch.cr3 is set, and vcpu_destroy_pagetables() clears it where + * v->arch.cr3 is zeroed. In shadow mode both name a top-level shadow, which + * shadow_teardown() releases before that point; nothing uses either once the + * vCPU can no longer run. + */ +static void pv_unmap_eff_root_pt(const struct vcpu *v) +{ + l1e_write(pv_gdt_ptes(v) + PV_EFF_ROOT_PT_PAGE, l1e_empty()); + flush_tlb_one_local(PV_EFF_ROOT_PT(v)); +} + void make_cr3(struct vcpu *v, mfn_t mfn) { struct domain *d = v->domain; v->arch.cr3 = mfn_x(mfn) << PAGE_SHIFT; - if ( is_pv_domain(d) && d->arch.pv.pcid ) - v->arch.cr3 |= get_pcid_bits(v, false); + + if ( is_pv_domain(d) ) + { + if ( d->arch.pv.pcid ) + v->arch.cr3 |= get_pcid_bits(v, false); + + if ( d->arch.pv.xpti ) + pv_map_eff_root_pt(v, mfn); + } } void write_ptbase(struct vcpu *v) @@ -3223,6 +3262,8 @@ int vcpu_destroy_pagetables(struct vcpu *v) return rc; v->arch.cr3 = 0; + if ( is_pv_domain(v->domain) && v->domain->arch.pv.xpti ) + pv_unmap_eff_root_pt(v); /* * Get the top-level guest page; either the guest_table itself, for diff --git a/xen/arch/x86/x86_64/asm-offsets.c b/xen/arch/x86/x86_64/asm-offsets.c index 9d45364026..2872904ae2 100644 --- a/xen/arch/x86/x86_64/asm-offsets.c +++ b/xen/arch/x86/x86_64/asm-offsets.c @@ -81,6 +81,7 @@ void __dummy__(void) #undef OFFSET_EF + OFFSET(VCPU_id, struct vcpu, vcpu_id); OFFSET(VCPU_processor, struct vcpu, processor); OFFSET(VCPU_domain, struct vcpu, domain); OFFSET(VCPU_vcpu_info, struct vcpu, vcpu_info_area.map); diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S index 33f6788984..34c045fab8 100644 --- a/xen/arch/x86/x86_64/entry.S +++ b/xen/arch/x86/x86_64/entry.S @@ -201,9 +201,14 @@ FUNC_LOCAL(restore_all_guest) movabs $PADDR_MASK & PAGE_MASK, %rsi movabs $DIRECTMAP_VIRT_START, %rcx and %rsi, %rdi - and %r9, %rsi add %rcx, %rdi + + /* Effective root page table is mapped at PV_EFF_ROOT_PT(v) */ + mov VCPU_id(%rbx), %esi + shl $GDT_LDT_VCPU_VA_SHIFT, %rsi + movabs $GDT_LDT_VIRT_START + PV_EFF_ROOT_PT_OFFSET, %rcx add %rcx, %rsi + mov $ROOT_PAGETABLE_FIRST_XEN_SLOT, %ecx mov root_table_offset(SH_LINEAR_PT_VIRT_START)*8(%rsi), %r8 mov %r8, root_table_offset(SH_LINEAR_PT_VIRT_START)*8(%rdi)