diff --git a/xen/arch/x86/domain_page.c b/xen/arch/x86/domain_page.c index eac5e3304f..72c00194f3 100644 --- a/xen/arch/x86/domain_page.c +++ b/xen/arch/x86/domain_page.c @@ -18,48 +18,40 @@ #include #include -static DEFINE_PER_CPU(struct vcpu *, override); - static inline struct vcpu *mapcache_current_vcpu(void) { - /* In the common case we use the mapcache of the running VCPU. */ - struct vcpu *v = this_cpu(override) ?: current; + struct vcpu *v = this_cpu(pgtable_vcpu); + struct vcpu *curr = current; /* - * When current isn't properly set up yet, this is equivalent to - * running in an idle vCPU (callers must check for NULL). + * During early boot pgtable_vcpu is not set, callers must handle NULL. + * Non-PV domains don't have a mapcache, the directmap covers all physical + * address space. */ - if ( !v ) + if ( !v || !is_pv_vcpu(v) ) return NULL; /* - * When using efi runtime page tables, we have the equivalent of the idle - * domain's page tables but current may point at another domain's VCPU. - * Return NULL as though current is not properly set up yet. + * If we are in a lazy context-switch state from a PV vCPU do a full switch + * to the idle vCPU now, otherwise an incoming FLUSH_VCPU_STATE IPI would + * change the page tables under our feet an invalidate any in-use mapcache + * entries. */ - if ( efi_rs_using_pgtables() ) - return NULL; - - /* - * If guest_table is NULL, and we are running a paravirtualised guest, - * then it means we are running on the idle domain's page table and must - * therefore use its mapcache. - */ - if ( unlikely(pagetable_is_null(v->arch.guest_table)) && is_pv_vcpu(v) ) + if ( unlikely(this_cpu(curr_vcpu) != curr) ) { - /* If we really are idling, perform lazy context switch now. */ - if ( (v = idle_vcpu[smp_processor_id()]) == current ) - sync_local_execstate(); + ASSERT(curr == idle_vcpu[smp_processor_id()]); + sync_local_execstate(); /* We must now be running on the idle page table. */ ASSERT(cr3_pa(read_cr3()) == __pa(idle_pg_table)); } - return v; -} - -void __init mapcache_override_current(struct vcpu *v) -{ - this_cpu(override) = v; + /* + * At this point we can guarantee Xen is not in lazy context switch: either + * the code above will have synced the state, or an incoming + * FLUSH_VCPU_STATE IPI has done so behind our back. Use ACCESS_ONCE to + * ensure the compiler never returns the locally cached pgtable_vcpu value. + */ + return ACCESS_ONCE(this_cpu(pgtable_vcpu)); } #define mapcache_l2_entry(e) ((e) >> PAGETABLE_ORDER) diff --git a/xen/arch/x86/flushtlb.c b/xen/arch/x86/flushtlb.c index 09e676c151..928bca66b4 100644 --- a/xen/arch/x86/flushtlb.c +++ b/xen/arch/x86/flushtlb.c @@ -111,7 +111,9 @@ static void do_tlb_flush(void) local_irq_restore(flags); } -void switch_cr3_cr4(unsigned long cr3, unsigned long cr4) +DEFINE_PER_CPU(struct vcpu *, pgtable_vcpu); + +void switch_cr3_cr4(struct vcpu *v, unsigned long cr3, unsigned long cr4) { unsigned long flags, old_cr4; u32 t = 0; @@ -155,6 +157,7 @@ void switch_cr3_cr4(unsigned long cr3, unsigned long cr4) if ( (old_cr4 & X86_CR4_PCIDE) > (cr4 & X86_CR4_PCIDE) ) cr3 |= X86_CR3_NOFLUSH; write_cr3(cr3); + this_cpu(pgtable_vcpu) = v; if ( old_cr4 != cr4 ) write_cr4(cr4); diff --git a/xen/arch/x86/include/asm/domain.h b/xen/arch/x86/include/asm/domain.h index 828f42c3e4..10d2b9fe25 100644 --- a/xen/arch/x86/include/asm/domain.h +++ b/xen/arch/x86/include/asm/domain.h @@ -75,7 +75,6 @@ struct mapcache_domain { int mapcache_domain_init(struct domain *d); int mapcache_vcpu_init(struct vcpu *v); -void mapcache_override_current(struct vcpu *v); /* x86/64: toggle guest between kernel and user modes. */ void toggle_guest_mode(struct vcpu *v); diff --git a/xen/arch/x86/include/asm/flushtlb.h b/xen/arch/x86/include/asm/flushtlb.h index 7bcbca2b7f..345677eb72 100644 --- a/xen/arch/x86/include/asm/flushtlb.h +++ b/xen/arch/x86/include/asm/flushtlb.h @@ -104,7 +104,7 @@ static inline void invlpg(const void *p) } /* Write pagetable base and implicitly tick the tlbflush clock. */ -void switch_cr3_cr4(unsigned long cr3, unsigned long cr4); +void switch_cr3_cr4(struct vcpu *v, unsigned long cr3, unsigned long cr4); /* flush_* flag fields: */ /* diff --git a/xen/arch/x86/include/asm/processor.h b/xen/arch/x86/include/asm/processor.h index 2e087c6257..d2cacdfedb 100644 --- a/xen/arch/x86/include/asm/processor.h +++ b/xen/arch/x86/include/asm/processor.h @@ -328,6 +328,9 @@ DECLARE_PER_CPU(struct tss_page, tss_page); DECLARE_PER_CPU(root_pgentry_t *, root_pgt); +/* vCPU of the currently loaded page-tables. */ +DECLARE_PER_CPU(struct vcpu *, pgtable_vcpu); + extern void write_ptbase(struct vcpu *v); /* PAUSE (encoding: REP NOP) is a good thing to insert into busy-wait loops. */ diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index 2b23bf2e7a..d02c9862d3 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -535,7 +535,7 @@ void write_ptbase(struct vcpu *v) cpu_info->pv_cr3 = __pa(this_cpu(root_pgt)); if ( new_cr4 & X86_CR4_PCIDE ) cpu_info->pv_cr3 |= get_pcid_bits(v, true); - switch_cr3_cr4(v->arch.cr3, new_cr4); + switch_cr3_cr4(v, v->arch.cr3, new_cr4); } else { @@ -543,7 +543,7 @@ void write_ptbase(struct vcpu *v) cpu_info->use_pv_cr3 = false; cpu_info->xen_cr3 = 0; /* switch_cr3_cr4() serializes. */ - switch_cr3_cr4(v->arch.cr3, new_cr4); + switch_cr3_cr4(v, v->arch.cr3, new_cr4); cpu_info->pv_cr3 = 0; } } diff --git a/xen/arch/x86/pv/dom0_build.c b/xen/arch/x86/pv/dom0_build.c index 37729091df..42bc530c0f 100644 --- a/xen/arch/x86/pv/dom0_build.c +++ b/xen/arch/x86/pv/dom0_build.c @@ -828,8 +828,7 @@ static int __init dom0_construct(const struct boot_domain *bd) update_cr3(v); /* We run on dom0's page tables for the final part of the build process. */ - switch_cr3_cr4(cr3_pa(v->arch.cr3), read_cr4()); - mapcache_override_current(v); + switch_cr3_cr4(v, cr3_pa(v->arch.cr3), read_cr4()); /* Copy the OS image and free temporary buffer. */ elf.dest_base = (void*)vkern_start; @@ -838,8 +837,7 @@ static int __init dom0_construct(const struct boot_domain *bd) rc = elf_load_binary(&elf); if ( rc < 0 ) { - mapcache_override_current(NULL); - switch_cr3_cr4(current->arch.cr3, read_cr4()); + switch_cr3_cr4(current, current->arch.cr3, read_cr4()); printk("Failed to load the kernel binary\n"); goto out; } @@ -850,8 +848,7 @@ static int __init dom0_construct(const struct boot_domain *bd) if ( (parms.virt_hypercall < v_start) || (parms.virt_hypercall >= v_end) ) { - mapcache_override_current(NULL); - switch_cr3_cr4(current->arch.cr3, read_cr4()); + switch_cr3_cr4(current, current->arch.cr3, read_cr4()); printk("Invalid HYPERCALL_PAGE field in ELF notes.\n"); return -EINVAL; } @@ -992,8 +989,7 @@ static int __init dom0_construct(const struct boot_domain *bd) #endif /* Return to idle domain's page tables. */ - mapcache_override_current(NULL); - switch_cr3_cr4(current->arch.cr3, read_cr4()); + switch_cr3_cr4(current, current->arch.cr3, read_cr4()); update_domain_wallclock_time(d); diff --git a/xen/arch/x86/pv/domain.c b/xen/arch/x86/pv/domain.c index ef4f442e73..d9e52f5f88 100644 --- a/xen/arch/x86/pv/domain.c +++ b/xen/arch/x86/pv/domain.c @@ -451,6 +451,8 @@ static void _toggle_guest_pt(struct vcpu *v) pagetable_t old_shadow; unsigned long cr3; + ASSERT(local_irq_is_enabled()); + v->arch.flags ^= TF_kernel_mode; guest_update = v->arch.flags & TF_kernel_mode; old_shadow = update_cr3(v); @@ -473,15 +475,22 @@ static void _toggle_guest_pt(struct vcpu *v) { cr3 &= ~X86_CR3_NOFLUSH; + local_irq_disable(); if ( unlikely(mfn_eq(pagetable_get_mfn(old_shadow), maddr_to_mfn(cr3))) ) { - cr3 = idle_vcpu[v->processor]->arch.cr3; /* Also suppress runstate/time area updates below. */ guest_update = false; + + cr3 = idle_vcpu[v->processor]->arch.cr3; + this_cpu(pgtable_vcpu) = idle_vcpu[v->processor]; } + + write_cr3(cr3); + local_irq_enable(); } - write_cr3(cr3); + else + write_cr3(cr3); if ( !pagetable_is_null(old_shadow) ) shadow_put_top_level(v->domain, old_shadow); diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c index 27628800a8..b37feab3be 100644 --- a/xen/arch/x86/smpboot.c +++ b/xen/arch/x86/smpboot.c @@ -1063,6 +1063,7 @@ static int cpu_smpboot_alloc(unsigned int cpu) info->current_vcpu = idle_vcpu[cpu]; /* set_current() */ per_cpu(curr_vcpu, cpu) = idle_vcpu[cpu]; + per_cpu(pgtable_vcpu, cpu) = idle_vcpu[cpu]; gdt = per_cpu(gdt, cpu) ?: alloc_xenheap_pages(0, memflags); if ( gdt == NULL ) diff --git a/xen/common/efi/common-stub.c b/xen/common/efi/common-stub.c index 77f138a6c5..7b12005bea 100644 --- a/xen/common/efi/common-stub.c +++ b/xen/common/efi/common-stub.c @@ -7,11 +7,6 @@ bool efi_enabled(unsigned int feature) return false; } -bool efi_rs_using_pgtables(void) -{ - return false; -} - unsigned long efi_get_time(void) { BUG(); diff --git a/xen/common/efi/runtime.c b/xen/common/efi/runtime.c index 30d649ca5c..feb09acf75 100644 --- a/xen/common/efi/runtime.c +++ b/xen/common/efi/runtime.c @@ -49,7 +49,6 @@ const CHAR16 *__read_mostly efi_fw_vendor; const EFI_RUNTIME_SERVICES *__read_mostly efi_rs; #ifndef CONFIG_ARM /* TODO - disabled until implemented on ARM */ static DEFINE_SPINLOCK(efi_rs_lock); -static unsigned int efi_rs_on_cpu = NR_CPUS; #endif UINTN __read_mostly efi_memmap_size; @@ -92,6 +91,11 @@ struct efi_rs_state efi_rs_enter(void) if ( mfn_eq(efi_l4_mfn, INVALID_MFN) ) return state; + /* + * If in lazy idle context switch state sync now to avoid an incoming + * FLUSH_VCPU_STATE IPI changing the loaded page-tables. + */ + sync_local_execstate(); state.cr3 = read_cr3(); save_fpu_enable(); asm volatile ( "fnclex; fldcw %0" :: "m" (fcw) ); @@ -99,8 +103,6 @@ struct efi_rs_state efi_rs_enter(void) spin_lock(&efi_rs_lock); - efi_rs_on_cpu = smp_processor_id(); - /* prevent fixup_page_fault() from doing anything */ irq_enter(); @@ -115,7 +117,8 @@ struct efi_rs_state efi_rs_enter(void) lgdt(&gdt_desc); } - switch_cr3_cr4(mfn_to_maddr(efi_l4_mfn), read_cr4()); + switch_cr3_cr4(idle_vcpu[smp_processor_id()], mfn_to_maddr(efi_l4_mfn), + read_cr4()); /* * At the time of writing (2022), no UEFI firwmare is CET-IBT compatible. @@ -143,7 +146,7 @@ void efi_rs_leave(struct efi_rs_state *state) if ( state->msr_s_cet ) wrmsrl(MSR_S_CET, state->msr_s_cet); - switch_cr3_cr4(state->cr3, read_cr4()); + switch_cr3_cr4(curr, state->cr3, read_cr4()); if ( is_pv_vcpu(curr) && !is_idle_vcpu(curr) ) { struct desc_ptr gdt_desc = { @@ -154,18 +157,10 @@ void efi_rs_leave(struct efi_rs_state *state) lgdt(&gdt_desc); } irq_exit(); - efi_rs_on_cpu = NR_CPUS; spin_unlock(&efi_rs_lock); vcpu_restore_fpu_nonlazy(curr, true); } -bool efi_rs_using_pgtables(void) -{ - return !mfn_eq(efi_l4_mfn, INVALID_MFN) && - (smp_processor_id() == efi_rs_on_cpu) && - (read_cr3() == mfn_to_maddr(efi_l4_mfn)); -} - unsigned long efi_get_time(void) { EFI_TIME time; diff --git a/xen/include/xen/efi.h b/xen/include/xen/efi.h index 723cb80852..9953197ee5 100644 --- a/xen/include/xen/efi.h +++ b/xen/include/xen/efi.h @@ -40,7 +40,6 @@ extern bool efi_secure_boot; void efi_init_memory(void); bool efi_boot_mem_unused(unsigned long *start, unsigned long *end); -bool efi_rs_using_pgtables(void); unsigned long efi_get_time(void); void efi_halt_system(void); void efi_reset_system(bool warm);