Out-of-sync L1 shadows: OOS snapshot.

Make snapshots of guest pages on unsync to allow faster revalidation
of OOS pages.

Signed-off-by: Gianluca Guida <gianluca.guida@eu.citrix.com>

xen/arch/x86/mm/shadow/common.c

@@ -72,7 +72,10 @@ void shadow_vcpu_init(struct vcpu *v)
     int i;
 
     for ( i = 0; i < SHADOW_OOS_PAGES; i++ )
+    {
         v->arch.paging.shadow.oos[i] = _mfn(INVALID_MFN);
+        v->arch.paging.shadow.oos_snapshot[i] = _mfn(INVALID_MFN);
+    }
 #endif
 
     v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode, 3);
@@ -562,7 +565,7 @@ void oos_audit_hash_is_present(struct domain *d, mfn_t gmfn)
 #endif
 
 /* Update the shadow, but keep the page out of sync. */
-static inline void _sh_resync_l1(struct vcpu *v, mfn_t gmfn)
+static inline void _sh_resync_l1(struct vcpu *v, mfn_t gmfn, mfn_t snpmfn)
 {
     struct page_info *pg = mfn_to_page(gmfn);
 
@@ -571,12 +574,12 @@ static inline void _sh_resync_l1(struct vcpu *v, mfn_t gmfn)
 
     /* Call out to the appropriate per-mode resyncing function */
     if ( pg->shadow_flags & SHF_L1_32 )
-        SHADOW_INTERNAL_NAME(sh_resync_l1, 2)(v, gmfn);
+        SHADOW_INTERNAL_NAME(sh_resync_l1, 2)(v, gmfn, snpmfn);
     else if ( pg->shadow_flags & SHF_L1_PAE )
-        SHADOW_INTERNAL_NAME(sh_resync_l1, 3)(v, gmfn);
+        SHADOW_INTERNAL_NAME(sh_resync_l1, 3)(v, gmfn, snpmfn);
 #if CONFIG_PAGING_LEVELS >= 4
     else if ( pg->shadow_flags & SHF_L1_64 )
-        SHADOW_INTERNAL_NAME(sh_resync_l1, 4)(v, gmfn);
+        SHADOW_INTERNAL_NAME(sh_resync_l1, 4)(v, gmfn, snpmfn);
 #endif
 }
 
@@ -728,7 +731,7 @@ static int oos_remove_write_access(struct vcpu *v, mfn_t gmfn, unsigned long va)
 
 
 /* Pull all the entries on an out-of-sync page back into sync. */
-static void _sh_resync(struct vcpu *v, mfn_t gmfn, unsigned long va)
+static void _sh_resync(struct vcpu *v, mfn_t gmfn, unsigned long va, mfn_t snp)
 {
     struct page_info *pg = mfn_to_page(gmfn);
 
@@ -753,7 +756,7 @@ static void _sh_resync(struct vcpu *v, mfn_t gmfn, unsigned long va)
     pg->shadow_flags &= ~SHF_oos_may_write;
 
     /* Update the shadows with current guest entries. */
-    _sh_resync_l1(v, gmfn);
+    _sh_resync_l1(v, gmfn, snp);
 
     /* Now we know all the entries are synced, and will stay that way */
     pg->shadow_flags &= ~SHF_out_of_sync;
@@ -764,27 +767,41 @@ static void _sh_resync(struct vcpu *v, mfn_t gmfn, unsigned long va)
 /* Add an MFN to the list of out-of-sync guest pagetables */
 static void oos_hash_add(struct vcpu *v, mfn_t gmfn, unsigned long va)
 {
-    int idx;
+    int idx, oidx, swap = 0;
+    void *gptr, *gsnpptr;
     mfn_t *oos = v->arch.paging.shadow.oos;
     unsigned long *oos_va = v->arch.paging.shadow.oos_va;
+    mfn_t *oos_snapshot = v->arch.paging.shadow.oos_snapshot;
 
     idx = mfn_x(gmfn) % SHADOW_OOS_PAGES;
+    oidx = idx;
+
     if ( mfn_valid(oos[idx]) 
          && (mfn_x(oos[idx]) % SHADOW_OOS_PAGES) == idx )
     {
         /* Punt the current occupant into the next slot */
         SWAP(oos[idx], gmfn);
         SWAP(oos_va[idx], va);
+        swap = 1;
         idx = (idx + 1) % SHADOW_OOS_PAGES;
     }
     if ( mfn_valid(oos[idx]) )
    {
         /* Crush the current occupant. */
-        _sh_resync(v, oos[idx], oos_va[idx]);
+        _sh_resync(v, oos[idx], oos_va[idx], oos_snapshot[idx]);
         perfc_incr(shadow_unsync_evict);
     }
     oos[idx] = gmfn;
     oos_va[idx] = va;
+
+    if ( swap )
+        SWAP(oos_snapshot[idx], oos_snapshot[oidx]);
+
+    gptr = sh_map_domain_page(oos[oidx]);
+    gsnpptr = sh_map_domain_page(oos_snapshot[oidx]);
+    memcpy(gsnpptr, gptr, PAGE_SIZE);
+    sh_unmap_domain_page(gptr);
+    sh_unmap_domain_page(gsnpptr);
 }
 
 /* Remove an MFN from the list of out-of-sync guest pagetables */
@@ -814,25 +831,52 @@ static void oos_hash_remove(struct vcpu *v, mfn_t gmfn)
     BUG();
 }
 
+mfn_t oos_snapshot_lookup(struct vcpu *v, mfn_t gmfn)
+{
+    int idx;
+    mfn_t *oos;
+    mfn_t *oos_snapshot;
+    struct domain *d = v->domain;
+    
+    for_each_vcpu(d, v) 
+    {
+        oos = v->arch.paging.shadow.oos;
+        oos_snapshot = v->arch.paging.shadow.oos_snapshot;
+        idx = mfn_x(gmfn) % SHADOW_OOS_PAGES;
+        if ( mfn_x(oos[idx]) != mfn_x(gmfn) )
+            idx = (idx + 1) % SHADOW_OOS_PAGES;
+        if ( mfn_x(oos[idx]) == mfn_x(gmfn) )
+        {
+            return oos_snapshot[idx];
+        }
+    }
+
+    SHADOW_ERROR("gmfn %lx was OOS but not in hash table\n", mfn_x(gmfn));
+    BUG();
+    return _mfn(INVALID_MFN);
+}
+
 /* Pull a single guest page back into sync */
 void sh_resync(struct vcpu *v, mfn_t gmfn)
 {
     int idx;
     mfn_t *oos;
     unsigned long *oos_va;
+    mfn_t *oos_snapshot;
     struct domain *d = v->domain;
 
     for_each_vcpu(d, v) 
     {
         oos = v->arch.paging.shadow.oos;
         oos_va = v->arch.paging.shadow.oos_va;
+        oos_snapshot = v->arch.paging.shadow.oos_snapshot;
         idx = mfn_x(gmfn) % SHADOW_OOS_PAGES;
         if ( mfn_x(oos[idx]) != mfn_x(gmfn) )
             idx = (idx + 1) % SHADOW_OOS_PAGES;
         
         if ( mfn_x(oos[idx]) == mfn_x(gmfn) )
         {
-            _sh_resync(v, gmfn, oos_va[idx]);
+            _sh_resync(v, gmfn, oos_va[idx], oos_snapshot[idx]);
             oos[idx] = _mfn(INVALID_MFN);
             return;
         }
@@ -873,6 +917,7 @@ void sh_resync_all(struct vcpu *v, int skip, int this, int others, int do_lockin
     struct vcpu *other;
     mfn_t *oos = v->arch.paging.shadow.oos;
     unsigned long *oos_va = v->arch.paging.shadow.oos_va;
+    mfn_t *oos_snapshot = v->arch.paging.shadow.oos_snapshot;
 
     SHADOW_PRINTK("d=%d, v=%d\n", v->domain->domain_id, v->vcpu_id);
 
@@ -892,7 +937,7 @@ void sh_resync_all(struct vcpu *v, int skip, int this, int others, int do_lockin
         if ( mfn_valid(oos[idx]) )
         {
             /* Write-protect and sync contents */
-            _sh_resync(v, oos[idx], oos_va[idx]);
+            _sh_resync(v, oos[idx], oos_va[idx], oos_snapshot[idx]);
             oos[idx] = _mfn(INVALID_MFN);
         }
 
@@ -914,7 +959,7 @@ void sh_resync_all(struct vcpu *v, int skip, int this, int others, int do_lockin
 
         oos = other->arch.paging.shadow.oos;
         oos_va = other->arch.paging.shadow.oos_va;
-
+        oos_snapshot = other->arch.paging.shadow.oos_snapshot;
         for ( idx = 0; idx < SHADOW_OOS_PAGES; idx++ ) 
         {
             if ( !mfn_valid(oos[idx]) )
@@ -925,12 +970,12 @@ void sh_resync_all(struct vcpu *v, int skip, int this, int others, int do_lockin
                 /* Update the shadows and leave the page OOS. */
                 if ( sh_skip_sync(v, oos[idx]) )
                     continue;
-                _sh_resync_l1(other, oos[idx]);
+                _sh_resync_l1(other, oos[idx], oos_snapshot[idx]);
             }
             else
             {
                 /* Write-protect and sync contents */
-                _sh_resync(other, oos[idx], oos_va[idx]);
+                _sh_resync(other, oos[idx], oos_va[idx], oos_snapshot[idx]);
                 oos[idx] = _mfn(INVALID_MFN);
             }
         }
@@ -1233,7 +1278,8 @@ shadow_order(unsigned int shadow_type)
         0, /* SH_type_l3_64_shadow   */
         0, /* SH_type_l4_64_shadow   */
         2, /* SH_type_p2m_table      */
-        0  /* SH_type_monitor_table  */
+        0, /* SH_type_monitor_table  */
+        0  /* SH_type_oos_snapshot   */
         };
     ASSERT(shadow_type < SH_type_unused);
     return type_to_order[shadow_type];
@@ -2765,6 +2811,17 @@ static void sh_update_paging_modes(struct vcpu *v)
         for ( i = 0; i < SHADOW_OOS_FT_HASH * SHADOW_OOS_FT_ENTRIES; i++ )
             v->arch.paging.shadow.oos_fixups[i].gmfn = _mfn(INVALID_MFN);
     }
+     
+    if ( mfn_x(v->arch.paging.shadow.oos_snapshot[0]) == INVALID_MFN )
+    {
+        int i;
+        for(i = 0; i < SHADOW_OOS_PAGES; i++)
+        {
+            shadow_prealloc(d, SH_type_oos_snapshot, 1);
+            v->arch.paging.shadow.oos_snapshot[i] =
+                shadow_alloc(d, SH_type_oos_snapshot, 0);
+        }
+    }
 #endif /* OOS */
 
     // Valid transitions handled by this function:
@@ -3112,6 +3169,14 @@ void shadow_teardown(struct domain *d)
             free_xenheap_pages(v->arch.paging.shadow.oos_fixups,
                                SHADOW_OOS_FT_ORDER);
         }
+
+        {
+            int i;
+            mfn_t *oos_snapshot = v->arch.paging.shadow.oos_snapshot;
+            for(i = 0; i < SHADOW_OOS_PAGES; i++)
+                if ( mfn_valid(oos_snapshot[i]) )
+                    shadow_free(d, oos_snapshot[i]);
+        }
 #endif /* OOS */
     }
 #endif /* (SHADOW_OPTIMIZATIONS & (SHOPT_VIRTUAL_TLB|SHOPT_OUT_OF_SYNC)) */

xen/arch/x86/mm/shadow/multi.c

@@ -2607,6 +2607,9 @@ static int validate_gl1e(struct vcpu *v, void *new_ge, mfn_t sl1mfn, void *se)
     mfn_t gmfn;
     p2m_type_t p2mt;
     int result = 0;
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+    mfn_t gl1mfn;
+#endif /* OOS */
 
     perfc_incr(shadow_validate_gl1e_calls);
 
@@ -2614,8 +2617,25 @@ static int validate_gl1e(struct vcpu *v, void *new_ge, mfn_t sl1mfn, void *se)
     gmfn = gfn_to_mfn(v->domain, gfn, &p2mt);
 
     l1e_propagate_from_guest(v, new_gl1e, gmfn, &new_sl1e, ft_prefetch, p2mt);
-    
     result |= shadow_set_l1e(v, sl1p, new_sl1e, sl1mfn);
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+    gl1mfn = _mfn(mfn_to_shadow_page(sl1mfn)->backpointer);
+    if ( mfn_valid(gl1mfn) 
+         && mfn_is_out_of_sync(gl1mfn) )
+    {
+        /* Update the OOS snapshot. */
+        mfn_t snpmfn = oos_snapshot_lookup(v, gl1mfn);
+        guest_l1e_t *snp;
+
+        ASSERT(mfn_valid(snpmfn));
+
+        snp = sh_map_domain_page(snpmfn);
+        snp[guest_index(new_ge)] = new_gl1e;
+        sh_unmap_domain_page(snp);
+    }
+#endif /* OOS */
+
     return result;
 }
 
@@ -2626,24 +2646,44 @@ static int validate_gl1e(struct vcpu *v, void *new_ge, mfn_t sl1mfn, void *se)
  * revalidates the guest entry that corresponds to it.
  * N.B. This function is called with the vcpu that unsynced the page,
  *      *not* the one that is causing it to be resynced. */
-void sh_resync_l1(struct vcpu *v, mfn_t gmfn)
+void sh_resync_l1(struct vcpu *v, mfn_t gl1mfn, mfn_t snpmfn)
 {
     mfn_t sl1mfn;
     shadow_l1e_t *sl1p;
-    guest_l1e_t *gl1p, *gp;
+    guest_l1e_t *gl1p, *gp, *snp;
     int rc = 0;
 
-    sl1mfn = get_shadow_status(v, gmfn, SH_type_l1_shadow);
+    ASSERT(mfn_valid(snpmfn));
+
+    sl1mfn = get_shadow_status(v, gl1mfn, SH_type_l1_shadow);
     ASSERT(mfn_valid(sl1mfn)); /* Otherwise we would not have been called */
 
-    gp = sh_map_domain_page(gmfn);
+    snp = sh_map_domain_page(snpmfn);
+    gp = sh_map_domain_page(gl1mfn);
     gl1p = gp;
 
-    SHADOW_FOREACH_L1E(sl1mfn, sl1p, &gl1p, 0, {
-        rc |= validate_gl1e(v, gl1p, sl1mfn, sl1p);
+   SHADOW_FOREACH_L1E(sl1mfn, sl1p, &gl1p, 0, {
+        guest_l1e_t gl1e = *gl1p;
+        guest_l1e_t *snpl1p = (guest_l1e_t *)snp + guest_index(gl1p);
+
+        if ( memcmp(snpl1p, &gl1e, sizeof(gl1e)) )
+        {
+            gfn_t gfn;
+            mfn_t gmfn;
+            p2m_type_t p2mt;
+            shadow_l1e_t nsl1e;
+
+            gfn = guest_l1e_get_gfn(gl1e);
+            gmfn = gfn_to_mfn(v->domain, gfn, &p2mt);
+            l1e_propagate_from_guest(v, gl1e, gmfn, &nsl1e, ft_prefetch, p2mt);
+            rc |= shadow_set_l1e(v, sl1p, nsl1e, sl1mfn);
+            
+            *snpl1p = gl1e;
+        }
     });
 
     sh_unmap_domain_page(gp);
+    sh_unmap_domain_page(snp);
 
     /* Setting shadow L1 entries should never need us to flush the TLB */
     ASSERT(!(rc & SHADOW_SET_FLUSH));
@@ -2891,6 +2931,10 @@ static void sh_prefetch(struct vcpu *v, walk_t *gw,
     shadow_l1e_t sl1e;
     u32 gflags;
     p2m_type_t p2mt;
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+    guest_l1e_t *snpl1p = NULL;
+#endif /* OOS */
+
 
     /* Prefetch no further than the end of the _shadow_ l1 MFN */
     dist = (PAGE_SIZE - ((unsigned long)ptr_sl1e & ~PAGE_MASK)) / sizeof sl1e;
@@ -2903,6 +2947,17 @@ static void sh_prefetch(struct vcpu *v, walk_t *gw,
         /* Normal guest page; grab the next guest entry */
         gl1p = sh_map_domain_page(gw->l1mfn);
         gl1p += guest_l1_table_offset(gw->va);
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+        if ( mfn_is_out_of_sync(gw->l1mfn) )
+        {
+            mfn_t snpmfn = oos_snapshot_lookup(v, gw->l1mfn);
+
+            ASSERT(mfn_valid(snpmfn));
+            snpl1p = sh_map_domain_page(snpmfn);
+            snpl1p += guest_l1_table_offset(gw->va);
+        }
+#endif /* OOS */
     }
 
     for ( i = 1; i < dist ; i++ ) 
@@ -2940,9 +2995,18 @@ static void sh_prefetch(struct vcpu *v, walk_t *gw,
         /* Propagate the entry.  */
         l1e_propagate_from_guest(v, gl1e, gmfn, &sl1e, ft_prefetch, p2mt);
         (void) shadow_set_l1e(v, ptr_sl1e + i, sl1e, sl1mfn);
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+        if ( snpl1p != NULL )
+            snpl1p[i] = gl1e;
+#endif /* OOS */
     }
     if ( gl1p != NULL )
         sh_unmap_domain_page(gl1p);
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+    if ( snpl1p != NULL )
+        sh_unmap_domain_page(snpl1p);
+#endif /* OOS */
 }
 
 #endif /* SHADOW_OPTIMIZATIONS & SHOPT_PREFETCH */
@@ -3228,6 +3292,22 @@ static int sh_page_fault(struct vcpu *v,
     l1e_propagate_from_guest(v, gw.l1e, gmfn, &sl1e, ft, p2mt);
     r = shadow_set_l1e(v, ptr_sl1e, sl1e, sl1mfn);
 
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+    if ( mfn_valid(gw.l1mfn) 
+         && mfn_is_out_of_sync(gw.l1mfn) )
+    {
+        /* Update the OOS snapshot. */
+        mfn_t snpmfn = oos_snapshot_lookup(v, gw.l1mfn);
+        guest_l1e_t *snp;
+        
+        ASSERT(mfn_valid(snpmfn));
+        
+        snp = sh_map_domain_page(snpmfn);
+        snp[guest_l1_table_offset(va)] = gw.l1e;
+        sh_unmap_domain_page(snp);
+    }
+#endif /* OOS */
+
 #if SHADOW_OPTIMIZATIONS & SHOPT_PREFETCH
     /* Prefetch some more shadow entries */
     sh_prefetch(v, &gw, ptr_sl1e, sl1mfn);

xen/arch/x86/mm/shadow/multi.h

@@ -119,7 +119,7 @@ SHADOW_INTERNAL_NAME(sh_paging_mode, GUEST_LEVELS);
 #if SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC
 extern void 
 SHADOW_INTERNAL_NAME(sh_resync_l1, GUEST_LEVELS)
-    (struct vcpu *v, mfn_t gmfn);
+     (struct vcpu *v, mfn_t gmfn, mfn_t snpmfn);
 
 extern int
 SHADOW_INTERNAL_NAME(sh_safe_not_to_sync, GUEST_LEVELS)

xen/arch/x86/mm/shadow/private.h

@@ -196,9 +196,9 @@ struct shadow_page_info
         u32 tlbflush_timestamp;
     };
     struct {
-        unsigned int type:4;      /* What kind of shadow is this? */
+        unsigned int type:5;      /* What kind of shadow is this? */
         unsigned int pinned:1;    /* Is the shadow pinned? */
-        unsigned int count:27;    /* Reference count */
+        unsigned int count:26;    /* Reference count */
         u32 mbz;                  /* Must be zero: this is where the owner 
                                    * field lives in a non-shadow page */
     } __attribute__((packed));
@@ -243,7 +243,8 @@ static inline void shadow_check_page_struct_offsets(void) {
 #define SH_type_max_shadow    (13U)
 #define SH_type_p2m_table     (14U) /* in use as the p2m table */
 #define SH_type_monitor_table (15U) /* in use as a monitor table */
-#define SH_type_unused        (16U)
+#define SH_type_oos_snapshot  (16U) /* in use as OOS snapshot */
+#define SH_type_unused        (17U)
 
 /* 
  * What counts as a pinnable shadow?
@@ -466,6 +467,8 @@ shadow_sync_other_vcpus(struct vcpu *v, int do_locking)
 }
 
 void oos_audit_hash_is_present(struct domain *d, mfn_t gmfn);
+mfn_t oos_snapshot_lookup(struct vcpu *v, mfn_t gmfn);
+
 #endif /* (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC) */
 
 /******************************************************************************

xen/include/asm-x86/domain.h

@@ -129,6 +129,7 @@ struct shadow_vcpu {
     /* Shadow out-of-sync: pages that this vcpu has let go out of sync */
     mfn_t oos[SHADOW_OOS_PAGES];
     unsigned long oos_va[SHADOW_OOS_PAGES];
+    mfn_t oos_snapshot[SHADOW_OOS_PAGES];
     struct oos_fixup {
         mfn_t gmfn;
         mfn_t smfn;