bitkeeper revision 1.1175 (420bacd51yx4f66j0Yv3tnUwEmeDtw)

Fix x86/64 SYSCALL handling. Handle return to compatibility mode as
well as long mode.
Signed-off-by: keir.fraser@cl.cam.ac.uk

xen/arch/x86/x86_64/entry.S

@@ -24,14 +24,21 @@ restore_all_guest:
         jc    failsafe_callback
         RESTORE_ALL
         testw $TRAP_syscall,4(%rsp)
-        jz    1f
+        jz    iret_exit_to_guest
+
         addq  $8,%rsp
-        popq  %rcx
-        addq  $8,%rsp
-        popq  %r11
-        popq  %rsp
+        popq  %rcx                    # RIP
+        popq  %r11                    # CS
+        cmpw  $__GUEST_CS32,%r11
+        popq  %r11                    # RFLAGS
+        popq  %rsp                    # RSP
+        je    1f
         sysretq
-1:      addq  $8,%rsp
+1:      sysretl
+
+        ALIGN
+iret_exit_to_guest:
+        addq  $8,%rsp
 FLT1:   iretq
 .section .fixup,"ax"
 FIX1:   popq  -15*8-8(%rsp)            # error_code/entry_vector
@@ -89,16 +96,24 @@ restore_all_xen:
         iretq
 
 /*
- * %rax                            = hypercall vector
- * %rdi, %rsi, %rdx, %r10, %r8, %9 = hypercall arguments
- * %r11, %rcx                      = SYSCALL-saved %rflags and %rip
- * NB. We must move %r10 to %rcx for C function-calling ABI.
+ * When entering SYSCALL from kernel mode:
+ *  %rax                            = hypercall vector
+ *  %rdi, %rsi, %rdx, %r10, %r8, %9 = hypercall arguments
+ *  %r11, %rcx                      = SYSCALL-saved %rflags and %rip
+ *  NB. We must move %r10 to %rcx for C function-calling ABI.
+ *
+ * When entering SYSCALL from user mode:
+ *  Vector directly to the registered arch.syscall_addr.
+ *
+ * Initial work is done by per-CPU stack trampolines. At this point %rsp
+ * has been initialised to point at the correct Xen stack, and %rsp, %rflags
+ * and %cs have been saved. All other registers are still to be saved onto
+ * the stack, starting with %rip, and an appropriate %ss must be saved into
+ * the space left by the trampoline.
  */
         ALIGN
 ENTRY(syscall_enter)
-        movl  $__GUEST_SS,8(%rsp)
-        pushq %r11
-        pushq $__GUEST_CS
+        movl  $__GUEST_SS,24(%rsp)
         pushq %rcx
         pushq $0
         movl  $TRAP_syscall,4(%rsp)

xen/arch/x86/x86_64/traps.c

@@ -196,16 +196,24 @@ void __init percpu_traps_init(void)
     stack[9] = 0x25;
     *(u32 *)&stack[10] = (stack_bottom - &stack[14]) - 16;
 
+    /* pushq %r11 */
+    stack[14] = 0x41;
+    stack[15] = 0x53;
+
+    /* pushq $__GUEST_CS64 */
+    stack[16] = 0x68;
+    *(u32 *)&stack[17] = __GUEST_CS64;
+
     /* jmp syscall_enter */
-    stack[14] = 0xe9;
-    *(u32 *)&stack[15] = (char *)syscall_enter - &stack[19];
+    stack[21] = 0xe9;
+    *(u32 *)&stack[22] = (char *)syscall_enter - &stack[26];
 
     /*
      * Trampoline for SYSCALL entry from compatibility mode.
      */
 
     /* Skip the long-mode entry trampoline. */
-    stack = &stack[19];
+    stack = &stack[26];
     wrmsr(MSR_CSTAR, (unsigned long)stack, ((unsigned long)stack>>32)); 
 
     /* movq %rsp, saversp(%rip) */
@@ -220,9 +228,17 @@ void __init percpu_traps_init(void)
     stack[9] = 0x25;
     *(u32 *)&stack[10] = (stack_bottom - &stack[14]) - 16;
 
+    /* pushq %r11 */
+    stack[14] = 0x41;
+    stack[15] = 0x53;
+
+    /* pushq $__GUEST_CS32 */
+    stack[16] = 0x68;
+    *(u32 *)&stack[17] = __GUEST_CS32;
+
     /* jmp syscall_enter */
-    stack[14] = 0xe9;
-    *(u32 *)&stack[15] = (char *)syscall_enter - &stack[19];
+    stack[21] = 0xe9;
+    *(u32 *)&stack[22] = (char *)syscall_enter - &stack[26];
 
     /*
      * Common SYSCALL parameters.

xen/include/asm-x86/config.h

@@ -202,7 +202,9 @@ extern void __out_of_line_bug(int line) __attribute__((noreturn));
 #define __HYPERVISOR_DS32 0x0818
 #define __HYPERVISOR_DS   __HYPERVISOR_DS64
 
-#define __GUEST_CS        0x0833
+#define __GUEST_CS64      0x0833
+#define __GUEST_CS32      0x0823
+#define __GUEST_CS        __GUEST_CS64
 #define __GUEST_DS        0x0000
 #define __GUEST_SS        0x082b