diff --git a/xen/arch/x86/acpi/cpu_idle.c b/xen/arch/x86/acpi/cpu_idle.c index 15fe2e9f9e..b1f606c0b9 100644 --- a/xen/arch/x86/acpi/cpu_idle.c +++ b/xen/arch/x86/acpi/cpu_idle.c @@ -56,6 +56,7 @@ #include #include #include +#include /*#define DEBUG_PM_CX*/ @@ -399,8 +400,14 @@ void mwait_idle_with_hints(unsigned int eax, unsigned int ecx) */ if ( (expires > NOW() || expires == 0) && !softirq_pending(cpu) ) { + struct cpu_info *info = get_cpu_info(); + cpumask_set_cpu(cpu, &cpuidle_mwait_flags); + + spec_ctrl_enter_idle(info); __mwait(eax, ecx); + spec_ctrl_exit_idle(info); + cpumask_clear_cpu(cpu, &cpuidle_mwait_flags); } @@ -415,6 +422,8 @@ static void acpi_processor_ffh_cstate_enter(struct acpi_processor_cx *cx) static void acpi_idle_do_entry(struct acpi_processor_cx *cx) { + struct cpu_info *info = get_cpu_info(); + switch ( cx->entry_method ) { case ACPI_CSTATE_EM_FFH: @@ -422,15 +431,19 @@ static void acpi_idle_do_entry(struct acpi_processor_cx *cx) acpi_processor_ffh_cstate_enter(cx); return; case ACPI_CSTATE_EM_SYSIO: + spec_ctrl_enter_idle(info); /* IO port based C-state */ inb(cx->address); /* Dummy wait op - must do something useless after P_LVL2 read because chipsets cannot guarantee that STPCLK# signal gets asserted in time to freeze execution properly. */ inl(pmtmr_ioport); + spec_ctrl_exit_idle(info); return; case ACPI_CSTATE_EM_HALT: + spec_ctrl_enter_idle(info); safe_halt(); + spec_ctrl_exit_idle(info); local_irq_disable(); return; } @@ -558,7 +571,13 @@ static void acpi_processor_idle(void) if ( pm_idle_save ) pm_idle_save(); else + { + struct cpu_info *info = get_cpu_info(); + + spec_ctrl_enter_idle(info); safe_halt(); + spec_ctrl_exit_idle(info); + } return; } @@ -737,6 +756,7 @@ void acpi_dead_idle(void) * Otherwise, CPU may still hold dirty data, breaking cache coherency, * leading to strange errors. */ + spec_ctrl_enter_idle(get_cpu_info()); wbinvd(); while ( 1 ) @@ -766,6 +786,7 @@ void acpi_dead_idle(void) u32 address = cx->address; u32 pmtmr_ioport_local = pmtmr_ioport; + spec_ctrl_enter_idle(get_cpu_info()); wbinvd(); while ( 1 ) diff --git a/xen/arch/x86/cpu/mwait-idle.c b/xen/arch/x86/cpu/mwait-idle.c index 07d8794f22..123851e6a8 100644 --- a/xen/arch/x86/cpu/mwait-idle.c +++ b/xen/arch/x86/cpu/mwait-idle.c @@ -58,6 +58,7 @@ #include #include #include +#include #include #define MWAIT_IDLE_VERSION "0.4" @@ -543,7 +544,13 @@ static void mwait_idle(void) if (pm_idle_save) pm_idle_save(); else + { + struct cpu_info *info = get_cpu_info(); + + spec_ctrl_enter_idle(info); safe_halt(); + spec_ctrl_exit_idle(info); + } return; } diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index d8d87cdc6f..3304eb15ac 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -54,6 +54,7 @@ #include #include #include +#include #include #include #include @@ -75,9 +76,15 @@ static void paravirt_ctxt_switch_to(struct vcpu *v); static void default_idle(void) { + struct cpu_info *info = get_cpu_info(); + local_irq_disable(); if ( cpu_is_haltable(smp_processor_id()) ) + { + spec_ctrl_enter_idle(info); safe_halt(); + spec_ctrl_exit_idle(info); + } else local_irq_enable(); } @@ -89,6 +96,7 @@ void default_dead_idle(void) * held by the CPUs spinning here indefinitely, and get discarded by * a subsequent INIT. */ + spec_ctrl_enter_idle(get_cpu_info()); wbinvd(); for ( ; ; ) halt(); diff --git a/xen/include/asm-x86/spec_ctrl.h b/xen/include/asm-x86/spec_ctrl.h index 41dce1527c..39823afe63 100644 --- a/xen/include/asm-x86/spec_ctrl.h +++ b/xen/include/asm-x86/spec_ctrl.h @@ -20,7 +20,9 @@ #ifndef __X86_SPEC_CTRL_H__ #define __X86_SPEC_CTRL_H__ +#include #include +#include void init_speculation_mitigations(void); @@ -35,6 +37,38 @@ static inline void init_shadow_spec_ctrl_state(void) info->bti_ist_info = default_bti_ist_info; } +/* WARNING! `ret`, `call *`, `jmp *` not safe after this call. */ +static always_inline void spec_ctrl_enter_idle(struct cpu_info *info) +{ + uint32_t val = 0; + + /* + * Latch the new shadow value, then enable shadowing, then update the MSR. + * There are no SMP issues here; only local processor ordering concerns. + */ + info->shadow_spec_ctrl = val; + barrier(); + info->use_shadow_spec_ctrl = 1; + barrier(); + asm volatile ( ALTERNATIVE(ASM_NOP3, "wrmsr", X86_FEATURE_XEN_IBRS_SET) + :: "a" (val), "c" (MSR_SPEC_CTRL), "d" (0) : "memory" ); +} + +/* WARNING! `ret`, `call *`, `jmp *` not safe before this call. */ +static always_inline void spec_ctrl_exit_idle(struct cpu_info *info) +{ + uint32_t val = SPEC_CTRL_IBRS; + + /* + * Disable shadowing before updating the MSR. There are no SMP issues + * here; only local processor ordering concerns. + */ + info->use_shadow_spec_ctrl = 0; + barrier(); + asm volatile ( ALTERNATIVE(ASM_NOP3, "wrmsr", X86_FEATURE_XEN_IBRS_SET) + :: "a" (val), "c" (MSR_SPEC_CTRL), "d" (0) : "memory" ); +} + #endif /* !__X86_SPEC_CTRL_H__ */ /*