diff --git a/docs/misc/xen-command-line.markdown b/docs/misc/xen-command-line.markdown index cdc78623c3..6197847c3d 100644 --- a/docs/misc/xen-command-line.markdown +++ b/docs/misc/xen-command-line.markdown @@ -1362,6 +1362,20 @@ Flag to enable Memory Protection Keys. The protection-key feature provides an additional mechanism by which IA-32e paging controls access to usermode addresses. +### pcid (x86) +> `= | xpti=` + +> Default: `xpti` + +> Can be modified at runtime (change takes effect only for domains created + afterwards) + +If available, control usage of the PCID feature of the processor for +64-bit pv-domains. PCID can be used either for no domain at all (`false`), +for all of them (`true`), only for those subject to XPTI (`xpti`) or for +those not subject to XPTI (`no-xpti`). The feature is used only in case +INVPCID is supported and not disabled via `invpcid=false`. + ### psr (Intel) > `= List of ( cmt: | rmid_max: | cat: | cos_max: | cdp: )` diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index 4e00b44975..324ca90bfb 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -47,7 +47,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -66,6 +68,49 @@ #include #include +static __read_mostly enum { + PCID_OFF, + PCID_ALL, + PCID_XPTI, + PCID_NOXPTI +} opt_pcid = PCID_XPTI; + +static __init int parse_pcid(const char *s) +{ + int rc = 0; + + switch ( parse_bool(s) ) + { + case 0: + opt_pcid = PCID_OFF; + break; + + case 1: + opt_pcid = PCID_ALL; + break; + + default: + switch ( parse_boolean("xpti", s, NULL) ) + { + case 0: + opt_pcid = PCID_NOXPTI; + break; + + case 1: + opt_pcid = PCID_XPTI; + break; + + default: + rc = -EINVAL; + break; + } + break; + } + + return rc; +} +custom_param("pcid", parse_pcid); + DEFINE_PER_CPU(struct vcpu *, curr_vcpu); static void default_idle(void); @@ -395,6 +440,7 @@ int switch_compat(struct domain *d) d->arch.x87_fip_width = 4; d->arch.pv_domain.xpti = false; + d->arch.pv_domain.pcid = false; return 0; @@ -679,6 +725,29 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags, d->arch.pv_domain.xpti = opt_xpti & (is_hardware_domain(d) ? OPT_XPTI_DOM0 : OPT_XPTI_DOMU); + + if ( !is_pv_32bit_domain(d) && use_invpcid && cpu_has_pcid ) + switch ( opt_pcid ) + { + case PCID_OFF: + break; + + case PCID_ALL: + d->arch.pv_domain.pcid = true; + break; + + case PCID_XPTI: + d->arch.pv_domain.pcid = d->arch.pv_domain.xpti; + break; + + case PCID_NOXPTI: + d->arch.pv_domain.pcid = !d->arch.pv_domain.xpti; + break; + + default: + ASSERT_UNREACHABLE(); + break; + } } /* initialize default tsc behavior in case tools don't */ diff --git a/xen/arch/x86/domain_build.c b/xen/arch/x86/domain_build.c index a6ff5f4acd..20f1c65091 100644 --- a/xen/arch/x86/domain_build.c +++ b/xen/arch/x86/domain_build.c @@ -1002,6 +1002,7 @@ int __init construct_dom0( { d->arch.is_32bit_pv = d->arch.has_32bit_shinfo = 1; d->arch.pv_domain.xpti = false; + d->arch.pv_domain.pcid = false; v->vcpu_info = (void *)&d->shared_info->compat.vcpu_info[0]; if ( setup_compat_arg_xlat(v) != 0 ) BUG(); @@ -1327,7 +1328,7 @@ int __init construct_dom0( update_cr3(v); /* We run on dom0's page tables for the final part of the build process. */ - switch_cr3_cr4(v->arch.cr3, read_cr4()); + switch_cr3_cr4(cr3_pa(v->arch.cr3), read_cr4()); mapcache_override_current(v); /* Copy the OS image and free temporary buffer. */ diff --git a/xen/arch/x86/flushtlb.c b/xen/arch/x86/flushtlb.c index 6d86a6b260..d389197c41 100644 --- a/xen/arch/x86/flushtlb.c +++ b/xen/arch/x86/flushtlb.c @@ -14,6 +14,7 @@ #include #include #include +#include /* Debug builds: Wrap frequently to stress-test the wrap logic. */ #ifdef NDEBUG @@ -95,6 +96,7 @@ void switch_cr3_cr4(unsigned long cr3, unsigned long cr4) { unsigned long flags, old_cr4; u32 t; + unsigned long old_pcid = cr3_pcid(read_cr3()); /* This non-reentrant function is sometimes called in interrupt context. */ local_irq_save(flags); @@ -104,14 +106,34 @@ void switch_cr3_cr4(unsigned long cr3, unsigned long cr4) old_cr4 = read_cr4(); if ( old_cr4 & X86_CR4_PGE ) { + /* + * X86_CR4_PGE set means PCID is inactive. + * We have to purge the TLB via flipping cr4.pge. + */ old_cr4 = cr4 & ~X86_CR4_PGE; write_cr4(old_cr4); } + else if ( use_invpcid ) + /* + * Flushing the TLB via INVPCID is necessary only in case PCIDs are + * in use, which is true only with INVPCID being available. + * Without PCID usage the following write_cr3() will purge the TLB + * (we are in the cr4.pge off path) of all entries. + * Using invpcid_flush_all_nonglobals() seems to be faster than + * invpcid_flush_all(), so use that. + */ + invpcid_flush_all_nonglobals(); write_cr3(cr3); if ( old_cr4 != cr4 ) write_cr4(cr4); + else if ( old_pcid != cr3_pcid(cr3) ) + /* + * Make sure no TLB entries related to the old PCID created between + * flushing the TLB and writing the new %cr3 value remain in the TLB. + */ + invpcid_flush_single_context(old_pcid); post_flush(t); @@ -141,8 +163,29 @@ unsigned int flush_area_local(const void *va, unsigned int flags) * are various errata surrounding INVLPG usage on superpages, and * a full flush is in any case not *that* expensive. */ - asm volatile ( "invlpg %0" - : : "m" (*(const char *)(va)) : "memory" ); + if ( read_cr4() & X86_CR4_PCIDE ) + { + unsigned long addr = (unsigned long)va; + + /* + * Flush the addresses for all potential address spaces. + * We can't check the current domain for being subject to + * XPTI as current might be the idle vcpu while we still have + * some XPTI domain TLB entries. + * Using invpcid is okay here, as with PCID enabled we always + * have global pages disabled. + */ + invpcid_flush_one(PCID_PV_PRIV, addr); + invpcid_flush_one(PCID_PV_USER, addr); + if ( !cpu_has_no_xpti ) + { + invpcid_flush_one(PCID_PV_PRIV | PCID_PV_XPTI, addr); + invpcid_flush_one(PCID_PV_USER | PCID_PV_XPTI, addr); + } + } + else + asm volatile ( "invlpg %0" + : : "m" (*(const char *)(va)) : "memory" ); } else do_tlb_flush(); diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index 40f708d2cc..c80f842599 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -494,7 +494,11 @@ void free_shared_domheap_page(struct page_info *page) void make_cr3(struct vcpu *v, unsigned long mfn) { + struct domain *d = v->domain; + v->arch.cr3 = mfn << PAGE_SHIFT; + if ( is_pv_domain(d) && d->arch.pv_domain.pcid ) + v->arch.cr3 |= get_pcid_bits(v, false); } unsigned long pv_guest_cr4_to_real_cr4(const struct vcpu *v) @@ -505,7 +509,12 @@ unsigned long pv_guest_cr4_to_real_cr4(const struct vcpu *v) cr4 = v->arch.pv_vcpu.ctrlreg[4] & ~X86_CR4_DE; cr4 |= mmu_cr4_features & (X86_CR4_PSE | X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_OSXSAVE | X86_CR4_FSGSBASE); - cr4 |= d->arch.pv_domain.xpti ? 0 : X86_CR4_PGE; + + if ( d->arch.pv_domain.pcid ) + cr4 |= X86_CR4_PCIDE; + else if ( !d->arch.pv_domain.xpti ) + cr4 |= X86_CR4_PGE; + cr4 |= d->arch.vtsc ? X86_CR4_TSD : 0; return cr4; @@ -518,12 +527,14 @@ void write_ptbase(struct vcpu *v) new_cr4 = (is_pv_vcpu(v) && !is_idle_vcpu(v)) ? pv_guest_cr4_to_real_cr4(v) - : ((read_cr4() & ~X86_CR4_TSD) | X86_CR4_PGE); + : ((read_cr4() & ~(X86_CR4_PCIDE | X86_CR4_TSD)) | X86_CR4_PGE); if ( is_pv_vcpu(v) && v->domain->arch.pv_domain.xpti ) { cpu_info->root_pgt_changed = true; cpu_info->pv_cr3 = __pa(this_cpu(root_pgt)); + if ( new_cr4 & X86_CR4_PCIDE ) + cpu_info->pv_cr3 |= get_pcid_bits(v, true); switch_cr3_cr4(v->arch.cr3, new_cr4); } else diff --git a/xen/arch/x86/x86_64/traps.c b/xen/arch/x86/x86_64/traps.c index eb956ae9f6..8ad09b524d 100644 --- a/xen/arch/x86/x86_64/traps.c +++ b/xen/arch/x86/x86_64/traps.c @@ -280,12 +280,22 @@ void toggle_guest_mode(struct vcpu *v) void toggle_guest_pt(struct vcpu *v) { + const struct domain *d = v->domain; + if ( is_pv_32bit_vcpu(v) ) return; v->arch.flags ^= TF_kernel_mode; update_cr3(v); - get_cpu_info()->root_pgt_changed = true; + if ( d->arch.pv_domain.xpti ) + { + struct cpu_info *cpu_info = get_cpu_info(); + + cpu_info->root_pgt_changed = true; + cpu_info->pv_cr3 = __pa(this_cpu(root_pgt)) | + (d->arch.pv_domain.pcid + ? get_pcid_bits(v, true) : 0); + } /* Don't flush user global mappings from the TLB. Don't tick TLB clock. */ write_cr3(v->arch.cr3); diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h index 01158f6a17..df7064fce4 100644 --- a/xen/include/asm-x86/domain.h +++ b/xen/include/asm-x86/domain.h @@ -259,6 +259,8 @@ struct pv_domain /* XPTI active? */ bool xpti; + /* Use PCID feature? */ + bool pcid; /* map_domain_page() mapping cache. */ struct mapcache_domain mapcache; @@ -266,6 +268,35 @@ struct pv_domain struct cpuidmasks *cpuidmasks; }; +/* + * PCID values for the address spaces of 64-bit pv domains: + * + * We are using 4 PCID values for a 64 bit pv domain subject to XPTI: + * - hypervisor active and guest in kernel mode PCID 0 + * - hypervisor active and guest in user mode PCID 1 + * - guest active and in kernel mode PCID 2 + * - guest active and in user mode PCID 3 + * + * Without XPTI only 2 values are used: + * - guest in kernel mode PCID 0 + * - guest in user mode PCID 1 + */ + +#define PCID_PV_PRIV 0x0000 /* Used for other domains, too. */ +#define PCID_PV_USER 0x0001 +#define PCID_PV_XPTI 0x0002 /* To be ORed to above values. */ + +/* + * Return additional PCID specific cr3 bits. + * + * Note that X86_CR3_NOFLUSH will not be readable in cr3. Anyone consuming + * v->arch.cr3 should mask away X86_CR3_NOFLUSH and X86_CR3_PCIDMASK in case + * the value is used to address the root page table. + */ +#define get_pcid_bits(v, is_xpti) \ + (X86_CR3_NOFLUSH | ((is_xpti) ? PCID_PV_XPTI : 0) | \ + (((v)->arch.flags & TF_kernel_mode) ? PCID_PV_PRIV : PCID_PV_USER)) + struct monitor_write_data { struct { unsigned int msr : 1; @@ -612,7 +643,7 @@ unsigned long pv_guest_cr4_to_real_cr4(const struct vcpu *v); #define real_cr4_to_pv_guest_cr4(c) \ ((c) & ~(X86_CR4_PGE | X86_CR4_PSE | X86_CR4_TSD | \ X86_CR4_OSXSAVE | X86_CR4_SMEP | \ - X86_CR4_FSGSBASE | X86_CR4_SMAP)) + X86_CR4_FSGSBASE | X86_CR4_SMAP | X86_CR4_PCIDE)) void domain_cpuid(const struct domain *d, unsigned int input, diff --git a/xen/include/asm-x86/processor.h b/xen/include/asm-x86/processor.h index 00f9d3ebb9..1482e4755d 100644 --- a/xen/include/asm-x86/processor.h +++ b/xen/include/asm-x86/processor.h @@ -370,6 +370,9 @@ static inline unsigned long read_cr4(void) static inline void write_cr4(unsigned long val) { + /* No global pages in case of PCIDs enabled! */ + ASSERT(!(val & X86_CR4_PGE) || !(val & X86_CR4_PCIDE)); + get_cpu_info()->cr4 = val; asm volatile ( "mov %0,%%cr4" : : "r" (val) ); }