diff --git a/xen/arch/x86/flushtlb.c b/xen/arch/x86/flushtlb.c index 0c5a1de443..471b3e31c4 100644 --- a/xen/arch/x86/flushtlb.c +++ b/xen/arch/x86/flushtlb.c @@ -235,7 +235,7 @@ unsigned int flush_area_local(const void *va, unsigned int flags) if ( flags & FLUSH_CACHE ) { const struct cpuinfo_x86 *c = ¤t_cpu_data; - unsigned long i, sz = 0; + unsigned long sz = 0; if ( order < (BITS_PER_LONG - PAGE_SHIFT) ) sz = 1UL << (order + PAGE_SHIFT); @@ -245,12 +245,7 @@ unsigned int flush_area_local(const void *va, unsigned int flags) c->x86_clflush_size && c->x86_cache_size && sz && ((sz >> 10) < c->x86_cache_size) ) { - alternative("", "sfence", X86_FEATURE_CLFLUSHOPT); - for ( i = 0; i < sz; i += c->x86_clflush_size ) - alternative_input("ds; clflush %0", - "data16 clflush %0", /* clflushopt */ - X86_FEATURE_CLFLUSHOPT, - "m" (((const char *)va)[i])); + cache_flush(va, sz); flags &= ~FLUSH_CACHE; } else @@ -265,7 +260,7 @@ unsigned int flush_area_local(const void *va, unsigned int flags) return flags; } -void cache_writeback(const void *addr, unsigned int size) +void cache_flush(const void *addr, unsigned int size) { /* * This function may be called before current_cpu_data is established. @@ -274,6 +269,38 @@ void cache_writeback(const void *addr, unsigned int size) unsigned int clflush_size = current_cpu_data.x86_clflush_size ?: 16; const void *end = addr + size; + addr -= (unsigned long)addr & (clflush_size - 1); + for ( ; addr < end; addr += clflush_size ) + { + /* + * Note regarding the "ds" prefix use: it's faster to do a clflush + * + prefix than a clflush + nop, and hence the prefix is added instead + * of letting the alternative framework fill the gap by appending nops. + */ + alternative_io("ds; clflush %[p]", + "data16 clflush %[p]", /* clflushopt */ + X86_FEATURE_CLFLUSHOPT, + /* no outputs */, + [p] "m" (*(const char *)(addr))); + } + + alternative("", "sfence", X86_FEATURE_CLFLUSHOPT); +} + +void cache_writeback(const void *addr, unsigned int size) +{ + unsigned int clflush_size; + const void *end = addr + size; + + /* Fall back to CLFLUSH{,OPT} when CLWB isn't available. */ + if ( !boot_cpu_has(X86_FEATURE_CLWB) ) + return cache_flush(addr, size); + + /* + * This function may be called before current_cpu_data is established. + * Hence a fallback is needed to prevent the loop below becoming infinite. + */ + clflush_size = current_cpu_data.x86_clflush_size ?: 16; addr -= (unsigned long)addr & (clflush_size - 1); for ( ; addr < end; addr += clflush_size ) { @@ -296,24 +323,15 @@ void cache_writeback(const void *addr, unsigned int size) #else # define INPUT(addr) "a" (addr), BASE_INPUT(addr) #endif - /* - * Note regarding the "ds" prefix use: it's faster to do a clflush - * + prefix than a clflush + nop, and hence the prefix is added instead - * of letting the alternative framework fill the gap by appending nops. - */ - alternative_io_2("ds; clflush %[p]", - "data16 clflush %[p]", /* clflushopt */ - X86_FEATURE_CLFLUSHOPT, - CLWB_ENCODING, - X86_FEATURE_CLWB, /* no outputs */, - INPUT(addr)); + + asm volatile (CLWB_ENCODING :: INPUT(addr)); + #undef INPUT #undef BASE_INPUT #undef CLWB_ENCODING } - alternative_2("", "sfence", X86_FEATURE_CLFLUSHOPT, - "sfence", X86_FEATURE_CLWB); + asm volatile ("sfence" ::: "memory"); } unsigned int guest_flush_tlb_flags(const struct domain *d) diff --git a/xen/arch/x86/include/asm/cache.h b/xen/arch/x86/include/asm/cache.h index 424dc5b7b9..e4770efb22 100644 --- a/xen/arch/x86/include/asm/cache.h +++ b/xen/arch/x86/include/asm/cache.h @@ -13,6 +13,7 @@ #ifndef __ASSEMBLY__ +void cache_flush(const void *addr, unsigned int size); void cache_writeback(const void *addr, unsigned int size); #endif diff --git a/xen/common/grant_table.c b/xen/common/grant_table.c index febbe12eab..3918e6de6b 100644 --- a/xen/common/grant_table.c +++ b/xen/common/grant_table.c @@ -3447,7 +3447,7 @@ gnttab_swap_grant_ref(XEN_GUEST_HANDLE_PARAM(gnttab_swap_grant_ref_t) uop, return 0; } -static int cache_flush(const gnttab_cache_flush_t *cflush, grant_ref_t *cur_ref) +static int _cache_flush(const gnttab_cache_flush_t *cflush, grant_ref_t *cur_ref) { struct domain *d, *owner; struct page_info *page; @@ -3541,7 +3541,7 @@ gnttab_cache_flush(XEN_GUEST_HANDLE_PARAM(gnttab_cache_flush_t) uop, return -EFAULT; for ( ; ; ) { - int ret = cache_flush(&op, cur_ref); + int ret = _cache_flush(&op, cur_ref); if ( ret < 0 ) return ret;