[HVM] Cache segment-register contents during PTE-update emulations.

Also clean up page-fault propagation to inject the correct error
code and CR2 value.
Signed-off-by: Keir Fraser <keir@xensource.com>

xen/arch/x86/hvm/svm/svm.c

@@ -812,9 +812,13 @@ static void svm_vcpu_destroy(struct vcpu *v)
     svm_destroy_vmcb(v);
 }
 
-static void svm_hvm_inject_exception(unsigned int trapnr, int errcode)
+static void svm_hvm_inject_exception(
+    unsigned int trapnr, int errcode, unsigned long cr2)
 {
-    svm_inject_exception(current, trapnr, (errcode != -1), errcode);
+    struct vcpu *v = current;
+    svm_inject_exception(v, trapnr, (errcode != -1), errcode);
+    if ( trapnr == TRAP_page_fault )
+        v->arch.hvm_svm.vmcb->cr2 = v->arch.hvm_svm.cpu_cr2 = cr2;
 }
 
 int start_svm(void)

xen/arch/x86/hvm/vmx/vmx.c

@@ -715,9 +715,13 @@ static void vmx_update_host_cr3(struct vcpu *v)
     __vmwrite(HOST_CR3, v->arch.cr3);
 }
 
-static void vmx_inject_exception(unsigned int trapnr, int errcode)
+static void vmx_inject_exception(
+    unsigned int trapnr, int errcode, unsigned long cr2)
 {
-    vmx_inject_hw_exception(current, trapnr, errcode);
+    struct vcpu *v = current;
+    vmx_inject_hw_exception(v, trapnr, errcode);
+    if ( trapnr == TRAP_page_fault )
+        v->arch.hvm_vmx.cpu_cr2 = cr2;
 }
 
 /* Setup HVM interfaces */

xen/arch/x86/mm/shadow/common.c

@@ -69,42 +69,52 @@ int _shadow_mode_refcounts(struct domain *d)
 /* x86 emulator support for the shadow code
  */
 
+struct segment_register *hvm_get_seg_reg(
+    enum x86_segment seg, struct sh_emulate_ctxt *sh_ctxt)
+{
+    struct segment_register *seg_reg = &sh_ctxt->seg_reg[seg];
+    if ( !__test_and_set_bit(seg, &sh_ctxt->valid_seg_regs) )
+        hvm_get_segment_register(current, seg, seg_reg);
+    return seg_reg;
+}
+
 static int hvm_translate_linear_addr(
     enum x86_segment seg,
     unsigned long offset,
     unsigned int bytes,
     unsigned int is_write,
+    struct sh_emulate_ctxt *sh_ctxt,
     unsigned long *paddr)
 {
-    struct segment_register creg, dreg;
+    struct segment_register *creg, *dreg;
     unsigned long limit, addr = offset;
     uint32_t last_byte;
 
-    hvm_get_segment_register(current, x86_seg_cs, &creg);
-    hvm_get_segment_register(current, seg,        &dreg);
+    creg = hvm_get_seg_reg(x86_seg_cs, sh_ctxt);
+    dreg = hvm_get_seg_reg(seg,        sh_ctxt);
 
-    if ( !creg.attr.fields.l || !hvm_long_mode_enabled(current) )
+    if ( !creg->attr.fields.l || !hvm_long_mode_enabled(current) )
     {
         /*
          * COMPATIBILITY MODE: Apply segment checks and add base.
          */
 
         /* If this is a store, is the segment a writable data segment? */
-        if ( is_write && ((dreg.attr.fields.type & 0xa) != 0x2) )
+        if ( is_write && ((dreg->attr.fields.type & 0xa) != 0x2) )
             goto gpf;
 
         /* Calculate the segment limit, including granularity flag. */
-        limit = dreg.limit;
-        if ( dreg.attr.fields.g )
+        limit = dreg->limit;
+        if ( dreg->attr.fields.g )
             limit = (limit << 12) | 0xfff;
 
         last_byte = offset + bytes - 1;
 
         /* Is this a grows-down data segment? Special limit check if so. */
-        if ( (dreg.attr.fields.type & 0xc) == 0x4 )
+        if ( (dreg->attr.fields.type & 0xc) == 0x4 )
         {
             /* Is upper limit 0xFFFF or 0xFFFFFFFF? */
-            if ( !dreg.attr.fields.db )
+            if ( !dreg->attr.fields.db )
                 last_byte = (uint16_t)last_byte;
 
             /* Check first byte and last byte against respective bounds. */
@@ -118,7 +128,7 @@ static int hvm_translate_linear_addr(
          * Hardware truncates to 32 bits in compatibility mode.
          * It does not truncate to 16 bits in 16-bit address-size mode.
          */
-        addr = (uint32_t)(addr + dreg.base);
+        addr = (uint32_t)(addr + dreg->base);
     }
     else
     {
@@ -127,7 +137,7 @@ static int hvm_translate_linear_addr(
          */
 
         if ( (seg == x86_seg_fs) || (seg == x86_seg_gs) )
-            addr += dreg.base;
+            addr += dreg->base;
 
         if ( !is_canonical_address(addr) )
             goto gpf;
@@ -138,7 +148,7 @@ static int hvm_translate_linear_addr(
 
  gpf:
     /* Inject #GP(0). */
-    hvm_inject_exception(TRAP_gp_fault, 0);
+    hvm_inject_exception(TRAP_gp_fault, 0, 0);
     return X86EMUL_PROPAGATE_FAULT;
 }
 
@@ -149,10 +159,12 @@ sh_x86_emulate_read(enum x86_segment seg,
                     unsigned int bytes,
                     struct x86_emulate_ctxt *ctxt)
 {
+    struct sh_emulate_ctxt *sh_ctxt =
+        container_of(ctxt, struct sh_emulate_ctxt, ctxt);
     unsigned long addr;
-    int rc;
+    int rc, errcode;
 
-    rc = hvm_translate_linear_addr(seg, offset, bytes, 0, &addr);
+    rc = hvm_translate_linear_addr(seg, offset, bytes, 0, sh_ctxt, &addr);
     if ( rc )
         return rc;
 
@@ -161,7 +173,7 @@ sh_x86_emulate_read(enum x86_segment seg,
     //        It entirely ignores the permissions in the page tables.
     //        In this case, that is only a user vs supervisor access check.
     //
-    if ( hvm_copy_from_guest_virt(val, addr, bytes) == 0 )
+    if ( (rc = hvm_copy_from_guest_virt(val, addr, bytes)) == 0 )
     {
 #if 0
         struct vcpu *v = current;
@@ -176,6 +188,8 @@ sh_x86_emulate_read(enum x86_segment seg,
      * was mapped here.  This should never happen: we're here because
      * of a write fault at the end of the instruction we're emulating. */ 
     SHADOW_PRINTK("read failed to va %#lx\n", addr);
+    errcode = ring_3(sh_ctxt->ctxt.regs) ? PFEC_user_mode : 0;
+    hvm_inject_exception(TRAP_page_fault, errcode, addr + bytes - rc);
     return X86EMUL_PROPAGATE_FAULT;
 }
 
@@ -186,11 +200,13 @@ sh_x86_emulate_write(enum x86_segment seg,
                      unsigned int bytes,
                      struct x86_emulate_ctxt *ctxt)
 {
+    struct sh_emulate_ctxt *sh_ctxt =
+        container_of(ctxt, struct sh_emulate_ctxt, ctxt);
     struct vcpu *v = current;
     unsigned long addr;
     int rc;
 
-    rc = hvm_translate_linear_addr(seg, offset, bytes, 1, &addr);
+    rc = hvm_translate_linear_addr(seg, offset, bytes, 1, sh_ctxt, &addr);
     if ( rc )
         return rc;
 
@@ -198,7 +214,8 @@ sh_x86_emulate_write(enum x86_segment seg,
     SHADOW_PRINTK("d=%u v=%u a=%#lx v=%#lx bytes=%u\n",
                   v->domain->domain_id, v->vcpu_id, addr, val, bytes);
 #endif
-    return v->arch.shadow.mode->x86_emulate_write(v, addr, &val, bytes, ctxt);
+    return v->arch.shadow.mode->x86_emulate_write(
+        v, addr, &val, bytes, sh_ctxt);
 }
 
 static int 
@@ -209,11 +226,13 @@ sh_x86_emulate_cmpxchg(enum x86_segment seg,
                        unsigned int bytes,
                        struct x86_emulate_ctxt *ctxt)
 {
+    struct sh_emulate_ctxt *sh_ctxt =
+        container_of(ctxt, struct sh_emulate_ctxt, ctxt);
     struct vcpu *v = current;
     unsigned long addr;
     int rc;
 
-    rc = hvm_translate_linear_addr(seg, offset, bytes, 1, &addr);
+    rc = hvm_translate_linear_addr(seg, offset, bytes, 1, sh_ctxt, &addr);
     if ( rc )
         return rc;
 
@@ -221,8 +240,8 @@ sh_x86_emulate_cmpxchg(enum x86_segment seg,
     SHADOW_PRINTK("d=%u v=%u a=%#lx o?=%#lx n:=%#lx bytes=%u\n",
                    v->domain->domain_id, v->vcpu_id, addr, old, new, bytes);
 #endif
-    return v->arch.shadow.mode->x86_emulate_cmpxchg(v, addr, old, new,
-                                                     bytes, ctxt);
+    return v->arch.shadow.mode->x86_emulate_cmpxchg(
+        v, addr, old, new, bytes, sh_ctxt);
 }
 
 static int 
@@ -234,11 +253,13 @@ sh_x86_emulate_cmpxchg8b(enum x86_segment seg,
                          unsigned long new_hi,
                          struct x86_emulate_ctxt *ctxt)
 {
+    struct sh_emulate_ctxt *sh_ctxt =
+        container_of(ctxt, struct sh_emulate_ctxt, ctxt);
     struct vcpu *v = current;
     unsigned long addr;
     int rc;
 
-    rc = hvm_translate_linear_addr(seg, offset, 8, 1, &addr);
+    rc = hvm_translate_linear_addr(seg, offset, 8, 1, sh_ctxt, &addr);
     if ( rc )
         return rc;
 
@@ -247,8 +268,8 @@ sh_x86_emulate_cmpxchg8b(enum x86_segment seg,
                    v->domain->domain_id, v->vcpu_id, addr, old_hi, old_lo,
                    new_hi, new_lo, ctxt);
 #endif
-    return v->arch.shadow.mode->x86_emulate_cmpxchg8b(v, addr, old_lo, old_hi,
-                                                       new_lo, new_hi, ctxt);
+    return v->arch.shadow.mode->x86_emulate_cmpxchg8b(
+        v, addr, old_lo, old_hi, new_lo, new_hi, sh_ctxt);
 }
 
 

xen/arch/x86/mm/shadow/multi.c

@@ -2582,7 +2582,7 @@ static int sh_page_fault(struct vcpu *v,
     mfn_t gmfn, sl1mfn=_mfn(0);
     shadow_l1e_t sl1e, *ptr_sl1e;
     paddr_t gpa;
-    struct x86_emulate_ctxt emul_ctxt;
+    struct sh_emulate_ctxt emul_ctxt;
     int r, mmio;
     fetch_type_t ft = 0;
 
@@ -2808,17 +2808,17 @@ static int sh_page_fault(struct vcpu *v,
     return EXCRET_fault_fixed;
 
  emulate:
-    /* Take the register set we were called with */
-    if ( is_hvm_domain(d) )
-        hvm_store_cpu_guest_regs(v, regs, NULL);
-    emul_ctxt.regs = regs;
-    emul_ctxt.mode = (is_hvm_domain(d) ?
-                      hvm_guest_x86_mode(v) : X86EMUL_MODE_HOST);
+    if ( !is_hvm_domain(d) )
+        goto not_a_shadow_fault;
+
+    hvm_store_cpu_guest_regs(v, regs, NULL);
+    emul_ctxt.ctxt.regs = regs;
+    emul_ctxt.ctxt.mode = (is_hvm_domain(d) ?
+                           hvm_guest_x86_mode(v) : X86EMUL_MODE_HOST);
+    emul_ctxt.valid_seg_regs = 0;
 
     SHADOW_PRINTK("emulate: eip=%#lx\n", regs->eip);
 
-    v->arch.shadow.propagate_fault = 0;
-
     /*
      * We do not emulate user writes. Instead we use them as a hint that the
      * page is no longer a page table. This behaviour differs from native, but
@@ -2826,7 +2826,7 @@ static int sh_page_fault(struct vcpu *v,
      * We also disallow guest PTE updates from within Xen.
      */
     if ( (regs->error_code & PFEC_user_mode) || !guest_mode(regs) ||
-         x86_emulate_memop(&emul_ctxt, &shadow_emulator_ops) )
+         x86_emulate_memop(&emul_ctxt.ctxt, &shadow_emulator_ops) )
     {
         SHADOW_PRINTK("emulator failure, unshadowing mfn %#lx\n", 
                        mfn_x(gmfn));
@@ -2835,19 +2835,10 @@ static int sh_page_fault(struct vcpu *v,
          * to support more operations in the emulator.  More likely, 
          * though, this is a hint that this page should not be shadowed. */
         shadow_remove_all_shadows(v, gmfn);
-        /* This means that actual missing operations will cause the 
-         * guest to loop on the same page fault. */
-        goto done;
     }
 
-    /* Emulation triggered another page fault? */
-    if ( v->arch.shadow.propagate_fault )
-        goto not_a_shadow_fault;
-
     /* Emulator has changed the user registers: write back */
-    if ( is_hvm_domain(d) )
-        hvm_load_cpu_guest_regs(v, regs);
-
+    hvm_load_cpu_guest_regs(v, regs);
     goto done;
 
  mmio:
@@ -3786,11 +3777,11 @@ int sh_remove_l3_shadow(struct vcpu *v, mfn_t sl4mfn, mfn_t sl3mfn)
  * or NULL for error. */
 static inline void * emulate_map_dest(struct vcpu *v,
                                       unsigned long vaddr,
-                                      struct x86_emulate_ctxt *ctxt,
+                                      struct sh_emulate_ctxt *sh_ctxt,
                                       mfn_t *mfnp)
 {
     walk_t gw;
-    u32 flags;
+    u32 flags, errcode;
     gfn_t gfn;
     mfn_t mfn;
 
@@ -3801,13 +3792,17 @@ static inline void * emulate_map_dest(struct vcpu *v,
     sh_audit_gw(v, &gw);
     unmap_walk(v, &gw);
 
-    if ( !(flags & _PAGE_PRESENT) 
-         || !(flags & _PAGE_RW) 
-         || (!(flags & _PAGE_USER) && ring_3(ctxt->regs)) )
+    if ( !(flags & _PAGE_PRESENT) )
     {
-        /* This write would have faulted even on bare metal */
-        v->arch.shadow.propagate_fault = 1;
-        return NULL;
+        errcode = 0;
+        goto page_fault;
+    }
+
+    if ( !(flags & _PAGE_RW) ||
+         (!(flags & _PAGE_USER) && ring_3(sh_ctxt->ctxt.regs)) )
+    {
+        errcode = PFEC_page_present;
+        goto page_fault;
     }
 
     /* Attempted a write to a bad gfn? This should never happen:
@@ -3817,11 +3812,18 @@ static inline void * emulate_map_dest(struct vcpu *v,
     ASSERT(sh_mfn_is_a_page_table(mfn));
     *mfnp = mfn;
     return sh_map_domain_page(mfn) + (vaddr & ~PAGE_MASK);
+
+ page_fault:
+    errcode |= PFEC_write_access;
+    if ( ring_3(sh_ctxt->ctxt.regs) )
+        errcode |= PFEC_user_mode;
+    hvm_inject_exception(TRAP_page_fault, errcode, vaddr);
+    return NULL;
 }
 
 int
 sh_x86_emulate_write(struct vcpu *v, unsigned long vaddr, void *src,
-                      u32 bytes, struct x86_emulate_ctxt *ctxt)
+                      u32 bytes, struct sh_emulate_ctxt *sh_ctxt)
 {
     mfn_t mfn;
     void *addr;
@@ -3832,7 +3834,7 @@ sh_x86_emulate_write(struct vcpu *v, unsigned long vaddr, void *src,
     ASSERT(shadow_lock_is_acquired(v->domain));
     ASSERT(((vaddr & ~PAGE_MASK) + bytes) <= PAGE_SIZE);
 
-    if ( (addr = emulate_map_dest(v, vaddr, ctxt, &mfn)) == NULL )
+    if ( (addr = emulate_map_dest(v, vaddr, sh_ctxt, &mfn)) == NULL )
         return X86EMUL_PROPAGATE_FAULT;
 
     memcpy(addr, src, bytes);
@@ -3850,7 +3852,7 @@ sh_x86_emulate_write(struct vcpu *v, unsigned long vaddr, void *src,
 int
 sh_x86_emulate_cmpxchg(struct vcpu *v, unsigned long vaddr, 
                         unsigned long old, unsigned long new,
-                        unsigned int bytes, struct x86_emulate_ctxt *ctxt)
+                        unsigned int bytes, struct sh_emulate_ctxt *sh_ctxt)
 {
     mfn_t mfn;
     void *addr;
@@ -3863,7 +3865,7 @@ sh_x86_emulate_cmpxchg(struct vcpu *v, unsigned long vaddr,
     if ( vaddr & (bytes-1) )
         return X86EMUL_UNHANDLEABLE;
 
-    if ( (addr = emulate_map_dest(v, vaddr, ctxt, &mfn)) == NULL )
+    if ( (addr = emulate_map_dest(v, vaddr, sh_ctxt, &mfn)) == NULL )
         return X86EMUL_PROPAGATE_FAULT;
 
     switch ( bytes )
@@ -3899,7 +3901,7 @@ int
 sh_x86_emulate_cmpxchg8b(struct vcpu *v, unsigned long vaddr, 
                           unsigned long old_lo, unsigned long old_hi,
                           unsigned long new_lo, unsigned long new_hi,
-                          struct x86_emulate_ctxt *ctxt)
+                          struct sh_emulate_ctxt *sh_ctxt)
 {
     mfn_t mfn;
     void *addr;
@@ -3911,7 +3913,7 @@ sh_x86_emulate_cmpxchg8b(struct vcpu *v, unsigned long vaddr,
     if ( vaddr & 7 )
         return X86EMUL_UNHANDLEABLE;
 
-    if ( (addr = emulate_map_dest(v, vaddr, ctxt, &mfn)) == NULL )
+    if ( (addr = emulate_map_dest(v, vaddr, sh_ctxt, &mfn)) == NULL )
         return X86EMUL_PROPAGATE_FAULT;
 
     old = (((u64) old_hi) << 32) | (u64) old_lo;

xen/arch/x86/mm/shadow/private.h

@@ -506,6 +506,19 @@ static inline void sh_unpin(struct vcpu *v, mfn_t smfn)
     }
 }
 
+
+/**************************************************************************/
+/* PTE-write emulation. */
+
+struct sh_emulate_ctxt {
+    struct x86_emulate_ctxt ctxt;
+
+    /* Cache of segment registers already gathered for this emulation. */
+    unsigned int valid_seg_regs;
+    struct segment_register seg_reg[6];
+};
+
+
 #endif /* _XEN_SHADOW_PRIVATE_H */
 
 /*

xen/include/asm-x86/domain.h

@@ -147,8 +147,6 @@ struct shadow_vcpu {
     unsigned long last_writeable_pte_smfn;
     /* HVM guest: paging enabled (CR0.PG)?  */
     unsigned int translate_enabled:1;
-    /* Emulated fault needs to be propagated to guest? */
-    unsigned int propagate_fault:1;
 };
 
 struct arch_vcpu

xen/include/asm-x86/hvm/hvm.h

@@ -110,7 +110,8 @@ struct hvm_function_table {
     void (*stts)(struct vcpu *v);
     void (*set_tsc_offset)(struct vcpu *v, u64 offset);
 
-    void (*inject_exception)(unsigned int trapnr, int errcode);
+    void (*inject_exception)(unsigned int trapnr, int errcode,
+                             unsigned long cr2);
 
     void (*init_ap_context)(struct vcpu_guest_context *ctxt,
                             int vcpuid, int trampoline_vector);
@@ -225,9 +226,9 @@ hvm_init_ap_context(struct vcpu_guest_context *ctxt,
 }
 
 static inline void
-hvm_inject_exception(unsigned int trapnr, int errcode)
+hvm_inject_exception(unsigned int trapnr, int errcode, unsigned long cr2)
 {
-    hvm_funcs.inject_exception(trapnr, errcode);
+    hvm_funcs.inject_exception(trapnr, errcode, cr2);
 }
 
 int hvm_bringup_ap(int vcpuid, int trampoline_vector);

xen/include/asm-x86/shadow.h

@@ -246,7 +246,7 @@ shadow_vcpu_mode_translate(struct vcpu *v)
 /**************************************************************************/
 /* Mode-specific entry points into the shadow code */
 
-struct x86_emulate_ctxt;
+struct sh_emulate_ctxt;
 struct shadow_paging_mode {
     int           (*page_fault            )(struct vcpu *v, unsigned long va,
                                             struct cpu_user_regs *regs);
@@ -267,18 +267,18 @@ struct shadow_paging_mode {
     void          (*detach_old_tables     )(struct vcpu *v);
     int           (*x86_emulate_write     )(struct vcpu *v, unsigned long va,
                                             void *src, u32 bytes,
-                                            struct x86_emulate_ctxt *ctxt);
+                                            struct sh_emulate_ctxt *sh_ctxt);
     int           (*x86_emulate_cmpxchg   )(struct vcpu *v, unsigned long va,
                                             unsigned long old, 
                                             unsigned long new,
                                             unsigned int bytes,
-                                            struct x86_emulate_ctxt *ctxt);
+                                            struct sh_emulate_ctxt *sh_ctxt);
     int           (*x86_emulate_cmpxchg8b )(struct vcpu *v, unsigned long va,
                                             unsigned long old_lo, 
                                             unsigned long old_hi, 
                                             unsigned long new_lo,
                                             unsigned long new_hi,
-                                            struct x86_emulate_ctxt *ctxt);
+                                            struct sh_emulate_ctxt *sh_ctxt);
     mfn_t         (*make_monitor_table    )(struct vcpu *v);
     void          (*destroy_monitor_table )(struct vcpu *v, mfn_t mmfn);
     void *        (*guest_map_l1e         )(struct vcpu *v, unsigned long va,