diff --git a/xen/arch/x86/hvm/vmx/entry.S b/xen/arch/x86/hvm/vmx/entry.S index 5f5de45a13..cdde76e138 100644 --- a/xen/arch/x86/hvm/vmx/entry.S +++ b/xen/arch/x86/hvm/vmx/entry.S @@ -87,17 +87,39 @@ UNLIKELY_END(realmode) /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ /* SPEC_CTRL_EXIT_TO_VMX Req: %rsp=regs/cpuinfo Clob: */ - DO_SPEC_CTRL_COND_VERW + /* + * All speculation safety work happens to be elsewhere. VERW is after + * popping the GPRs, while restoring the guest MSR_SPEC_CTRL is left + * to the MSR load list. + */ mov VCPU_hvm_guest_cr2(%rbx),%rax + mov %rax, %cr2 + + /* + * We need to perform two conditional actions (VERW, and Resume vs + * Launch) after popping GPRs. With some cunning, we can encode both + * of these in eflags together. + * + * Parity is only calculated over the bottom byte of the answer, while + * Sign is simply the top bit. + * + * Therefore, the final OR instruction ends up producing: + * SF = VCPU_vmx_launched + * PF = !SCF_verw + */ + BUILD_BUG_ON(SCF_verw & ~0xff) + movzbl VCPU_vmx_launched(%rbx), %ecx + shl $31, %ecx + movzbl CPUINFO_spec_ctrl_flags(%rsp), %eax + and $SCF_verw, %eax + or %eax, %ecx pop %r15 pop %r14 pop %r13 pop %r12 pop %rbp - mov %rax,%cr2 - cmpb $0,VCPU_vmx_launched(%rbx) pop %rbx pop %r11 pop %r10 @@ -108,7 +130,13 @@ UNLIKELY_END(realmode) pop %rdx pop %rsi pop %rdi - je .Lvmx_launch + + jpe .L_skip_verw + /* VERW clobbers ZF, but preserves all others, including SF. */ + verw STK_REL(CPUINFO_verw_sel, CPUINFO_error_code)(%rsp) +.L_skip_verw: + + jns .Lvmx_launch /*.Lvmx_resume:*/ VMRESUME diff --git a/xen/arch/x86/x86_64/asm-offsets.c b/xen/arch/x86/x86_64/asm-offsets.c index 9d105161db..eb4a619891 100644 --- a/xen/arch/x86/x86_64/asm-offsets.c +++ b/xen/arch/x86/x86_64/asm-offsets.c @@ -133,6 +133,7 @@ void __dummy__(void) #endif OFFSET(CPUINFO_guest_cpu_user_regs, struct cpu_info, guest_cpu_user_regs); + OFFSET(CPUINFO_error_code, struct cpu_info, guest_cpu_user_regs.error_code); OFFSET(CPUINFO_verw_sel, struct cpu_info, verw_sel); OFFSET(CPUINFO_current_vcpu, struct cpu_info, current_vcpu); OFFSET(CPUINFO_per_cpu_offset, struct cpu_info, per_cpu_offset); diff --git a/xen/include/asm-x86/asm_defns.h b/xen/include/asm-x86/asm_defns.h index d480a4461b..4ab54f41cf 100644 --- a/xen/include/asm-x86/asm_defns.h +++ b/xen/include/asm-x86/asm_defns.h @@ -81,6 +81,14 @@ register unsigned long current_stack_pointer asm("rsp"); #ifdef __ASSEMBLY__ +.macro BUILD_BUG_ON condstr, cond:vararg + .if \cond + .error "Condition \"\condstr\" not satisfied" + .endif +.endm +/* preprocessor macro to make error message more user friendly */ +#define BUILD_BUG_ON(cond) BUILD_BUG_ON #cond, cond + #ifdef HAVE_AS_QUOTED_SYM #define SUBSECTION_LBL(tag) \ .ifndef .L.tag; \ diff --git a/xen/include/asm-x86/spec_ctrl_asm.h b/xen/include/asm-x86/spec_ctrl_asm.h index c47399dbf2..b5e5669436 100644 --- a/xen/include/asm-x86/spec_ctrl_asm.h +++ b/xen/include/asm-x86/spec_ctrl_asm.h @@ -164,6 +164,13 @@ #endif .endm +/* + * Helper to improve the readibility of stack dispacements with %rsp in + * unusual positions. Both @field and @top_of_stack should be constants from + * the same object. @top_of_stack should be where %rsp is currently pointing. + */ +#define STK_REL(field, top_of_stk) ((field) - (top_of_stk)) + .macro DO_SPEC_CTRL_COND_VERW /* * Requires %rsp=cpuinfo