diff --git a/xen/arch/arm/include/asm/arm32/flushtlb.h b/xen/arch/arm/include/asm/arm32/flushtlb.h index 61c25a3189..5483be08fb 100644 --- a/xen/arch/arm/include/asm/arm32/flushtlb.h +++ b/xen/arch/arm/include/asm/arm32/flushtlb.h @@ -57,6 +57,9 @@ static inline void __flush_xen_tlb_one(vaddr_t va) asm volatile(STORE_CP32(0, TLBIMVAHIS) : : "r" (va) : "memory"); } +/* Only for ARM64_WORKAROUND_REPEAT_TLBI */ +static inline void __tlb_repeat_sync(void) {} + #endif /* __ASM_ARM_ARM32_FLUSHTLB_H__ */ /* * Local variables: diff --git a/xen/arch/arm/include/asm/arm64/flushtlb.h b/xen/arch/arm/include/asm/arm64/flushtlb.h index 3b99c11b50..1606b26bf2 100644 --- a/xen/arch/arm/include/asm/arm64/flushtlb.h +++ b/xen/arch/arm/include/asm/arm64/flushtlb.h @@ -12,9 +12,14 @@ * ARM64_WORKAROUND_REPEAT_TLBI: * Modification of the translation table for a virtual address might lead to * read-after-read ordering violation. - * The workaround repeats TLBI+DSB ISH operation for all the TLB flush - * operations. While this is strictly not necessary, we don't want to - * take any risk. + * The workaround repeats TLBI+DSB ISH operation for broadcast TLB flush + * operations. The workaround is not needed for local operations. + * + * It is sufficient for the additional TLBI to use *any* operation which will + * be broadcast, regardless of which translation regime or stage of translation + * the operation applies to. TLBI VALE2IS is used passing XZR. While there is + * an identity mapping there, it's only used during suspend/resume, CPU on/off, + * so the impact (performance if any) is negligible. * * For Xen page-tables the ISB will discard any instructions fetched * from the old mappings. @@ -26,69 +31,90 @@ * Note that for local TLB flush, using non-shareable (nsh) is sufficient * (see D5-4929 in ARM DDI 0487H.a). Although, the memory barrier in * for the workaround is left as inner-shareable to match with Linux - * v6.1-rc8. + * v6.19. */ -#define TLB_HELPER(name, tlbop, sh) \ +#define TLB_HELPER_LOCAL(name, tlbop) \ static inline void name(void) \ { \ asm_inline volatile ( \ - "dsb " # sh "st;" \ + "dsb nshst;" \ "tlbi " # tlbop ";" \ - ALTERNATIVE( \ - "nop; nop;", \ - "dsb ish;" \ - "tlbi " # tlbop ";", \ - ARM64_WORKAROUND_REPEAT_TLBI, \ - CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \ - "dsb " # sh ";" \ + "dsb nsh;" \ "isb;" \ : : : "memory"); \ } +#define TLB_HELPER(name, tlbop) \ +static inline void name(void) \ +{ \ + asm_inline volatile ( \ + "dsb ishst;" \ + "tlbi " # tlbop ";" \ + ALTERNATIVE( \ + "nop; nop;", \ + "dsb ish;" \ + "tlbi vale2is, xzr;", \ + ARM64_WORKAROUND_REPEAT_TLBI, \ + CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \ + "dsb ish;" \ + "isb;" \ + : : : "memory"); \ +} + +/* Flush local TLBs, current VMID only. */ +TLB_HELPER_LOCAL(flush_guest_tlb_local, vmalls12e1) + +/* Flush innershareable TLBs, current VMID only */ +TLB_HELPER(flush_guest_tlb, vmalls12e1is) + +/* Flush local TLBs, all VMIDs, non-hypervisor mode */ +TLB_HELPER_LOCAL(flush_all_guests_tlb_local, alle1) + +/* Flush innershareable TLBs, all VMIDs, non-hypervisor mode */ +TLB_HELPER(flush_all_guests_tlb, alle1is) + +/* Flush all hypervisor mappings from the TLB of the local processor. */ +TLB_HELPER_LOCAL(flush_xen_tlb_local, alle2) + +#undef TLB_HELPER_LOCAL +#undef TLB_HELPER + /* * FLush TLB by VA. This will likely be used in a loop, so the caller * is responsible to use the appropriate memory barriers before/after * the sequence. - * - * See above about the ARM64_WORKAROUND_REPEAT_TLBI sequence. */ -#define TLB_HELPER_VA(name, tlbop) \ -static inline void name(vaddr_t va) \ -{ \ - asm_inline volatile ( \ - "tlbi " # tlbop ", %0;" \ - ALTERNATIVE( \ - "nop; nop;", \ - "dsb ish;" \ - "tlbi " # tlbop ", %0;", \ - ARM64_WORKAROUND_REPEAT_TLBI, \ - CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \ - : : "r" (va >> PAGE_SHIFT) : "memory"); \ -} - -/* Flush local TLBs, current VMID only. */ -TLB_HELPER(flush_guest_tlb_local, vmalls12e1, nsh) - -/* Flush innershareable TLBs, current VMID only */ -TLB_HELPER(flush_guest_tlb, vmalls12e1is, ish) - -/* Flush local TLBs, all VMIDs, non-hypervisor mode */ -TLB_HELPER(flush_all_guests_tlb_local, alle1, nsh) - -/* Flush innershareable TLBs, all VMIDs, non-hypervisor mode */ -TLB_HELPER(flush_all_guests_tlb, alle1is, ish) - -/* Flush all hypervisor mappings from the TLB of the local processor. */ -TLB_HELPER(flush_xen_tlb_local, alle2, nsh) /* Flush TLB of local processor for address va. */ -TLB_HELPER_VA(__flush_xen_tlb_one_local, vae2) +static inline void __flush_xen_tlb_one_local(vaddr_t va) +{ + asm_inline volatile ( + "tlbi vae2, %0" : : "r" (va >> PAGE_SHIFT) : "memory"); +} /* Flush TLB of all processors in the inner-shareable domain for address va. */ -TLB_HELPER_VA(__flush_xen_tlb_one, vae2is) +static inline void __flush_xen_tlb_one(vaddr_t va) +{ + asm_inline volatile ( + "tlbi vae2is, %0" : : "r" (va >> PAGE_SHIFT) : "memory"); +} -#undef TLB_HELPER -#undef TLB_HELPER_VA +/* + * ARM64_WORKAROUND_REPEAT_TLBI: + * For all relevant erratas it is only necessary to execute a single + * additional TLBI;DSB sequence after any number of TLBIs are completed by DSB. + */ +static inline void __tlb_repeat_sync(void) +{ + asm_inline volatile ( + ALTERNATIVE( + "nop; nop;", + "tlbi vale2is, xzr;" + "dsb ish;", + ARM64_WORKAROUND_REPEAT_TLBI, + CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) + : : : "memory"); +} #endif /* __ASM_ARM_ARM64_FLUSHTLB_H__ */ /* diff --git a/xen/arch/arm/include/asm/flushtlb.h b/xen/arch/arm/include/asm/flushtlb.h index e45fb6d97b..c292c3c00d 100644 --- a/xen/arch/arm/include/asm/flushtlb.h +++ b/xen/arch/arm/include/asm/flushtlb.h @@ -65,6 +65,7 @@ static inline void flush_xen_tlb_range_va(vaddr_t va, va += PAGE_SIZE; } dsb(ish); /* Ensure the TLB invalidation has completed */ + __tlb_repeat_sync(); isb(); } diff --git a/xen/arch/arm/include/asm/mmu/layout.h b/xen/arch/arm/include/asm/mmu/layout.h index 19c0ec63a5..feafc14ebf 100644 --- a/xen/arch/arm/include/asm/mmu/layout.h +++ b/xen/arch/arm/include/asm/mmu/layout.h @@ -23,6 +23,10 @@ * * Reserved to identity map Xen * + * Note: As part of ARM64_WORKAROUND_REPEAT_TLBI, VA 0 is used for an extra + * TLBI operation given its rare use (only identity mapping) and thus + * negligible performance impact. + * * 0x00000a0000000000 - 0x00000a7fffffffff (512GB, L0 slot [20]) * (Relative offsets) * 0 - 2M Unmapped