Replace shadow pagetable code with shadow2.

.hgtags

@@ -15,3 +15,13 @@ fb875591fd72e15c31879c0e9034d99b80225595 RELEASE-2.0.4
 c8fdb0caa77b429cf47f9707926e83947778cb48 RELEASE-3.0.0
 af0573e9e5258db0a9d28aa954dd302ddd2c2d23 3.0.2-rc
 d0d3fef37685be264a7f52201f8ef44c030daad3 3.0.2-branched
+6e864d7de9db066f92bea505d256bfe286200fed last-code-review
+a898a6510c5db4e3d1f69d40fcacb540643b0f22 mainline
+bfa6f4a0c594bc0ebd896437d69857b58dab0988 last-code-review
+fc6cbf31bd883bc76ceb97f4b817ac88078d696a latest patch to unstable
+8e55c5c1147589b7a6a1875384d4317aec7ccf84 mainline
+2d2ed4d9b1c14aeee29dfdd77acd6017d31290cd mainline
+0e32095a7b4611d18a82052a9d5b23e474f91af9 mainline
+88e6bd5e2b5439f97e1d50a8724103c619aeaadf mainline
+5233c4b076b9aa073eff63508461b7bfa597737c mainline
+fda70200da01b89d5339342df6c0db372369a16d mainline

tools/examples/xmexample.hvm

@@ -27,6 +27,10 @@ builder='hvm'
 #          and modules. Allocating less than 32MBs is not recommended.
 memory = 128
 
+# Shadow pagetable memory for the domain, in MB.
+# Should be at least 2KB per MB of domain memory, plus a few MB per vcpu.
+shadow_memory = 8
+
 # A name for your domain. All domains must have different names.
 name = "ExampleHVMDomain"
 

tools/libxc/xc_domain.c

@@ -213,21 +213,28 @@ int xc_shadow_control(int xc_handle,
                       unsigned int sop,
                       unsigned long *dirty_bitmap,
                       unsigned long pages,
-                      xc_shadow_control_stats_t *stats )
+                      unsigned long *mb,
+                      uint32_t mode,
+                      xc_shadow_control_stats_t *stats)
 {
     int rc;
     DECLARE_DOM0_OP;
     op.cmd = DOM0_SHADOW_CONTROL;
     op.u.shadow_control.domain = (domid_t)domid;
     op.u.shadow_control.op     = sop;
-    set_xen_guest_handle(op.u.shadow_control.dirty_bitmap, dirty_bitmap);
     op.u.shadow_control.pages  = pages;
+    op.u.shadow_control.mb     = mb ? *mb : 0;
+    op.u.shadow_control.mode   = mode;
+    set_xen_guest_handle(op.u.shadow_control.dirty_bitmap, dirty_bitmap);
 
     rc = do_dom0_op(xc_handle, &op);
 
     if ( stats )
         memcpy(stats, &op.u.shadow_control.stats,
                sizeof(xc_shadow_control_stats_t));
+    
+    if ( mb ) 
+        *mb = op.u.shadow_control.mb;
 
     return (rc == 0) ? op.u.shadow_control.pages : rc;
 }
@@ -391,7 +398,7 @@ int xc_domain_memory_populate_physmap(int xc_handle,
 
     if ( err > 0 )
     {
-        DPRINTF("Failed deallocation for dom %d: %ld pages order %d\n",
+        DPRINTF("Failed allocation for dom %d: %ld pages order %d\n",
                 domid, nr_extents, extent_order);
         errno = EBUSY;
         err = -1;

tools/libxc/xc_hvm_build.c

@@ -396,6 +396,19 @@ static int xc_hvm_build_internal(int xc_handle,
         goto error_out;
     }
 
+    /* HVM domains must be put into shadow2 mode at the start of day */
+    if ( xc_shadow_control(xc_handle, domid, DOM0_SHADOW2_CONTROL_OP_ENABLE,
+                           NULL, 0, NULL, 
+                           DOM0_SHADOW2_CONTROL_FLAG_ENABLE 
+                           | DOM0_SHADOW2_CONTROL_FLAG_REFCOUNT
+                           | DOM0_SHADOW2_CONTROL_FLAG_TRANSLATE
+                           | DOM0_SHADOW2_CONTROL_FLAG_EXTERNAL, 
+                           NULL) ) 
+    {
+        PERROR("Could not enable shadow paging for domain.\n");
+        goto error_out;
+    }        
+
     memset(ctxt, 0, sizeof(*ctxt));
 
     ctxt->flags = VGCF_HVM_GUEST;

tools/libxc/xc_linux_build.c

@@ -972,7 +972,7 @@ static int setup_guest(int xc_handle,
         /* Enable shadow translate mode */
         if ( xc_shadow_control(xc_handle, dom,
                                DOM0_SHADOW_CONTROL_OP_ENABLE_TRANSLATE,
-                               NULL, 0, NULL) < 0 )
+                               NULL, 0, NULL, 0, NULL) < 0 )
         {
             PERROR("Could not enable translation mode");
             goto error_out;

tools/libxc/xc_linux_save.c

@@ -338,13 +338,13 @@ static int analysis_phase(int xc_handle, uint32_t domid, int max_pfn,
         int i;
 
         xc_shadow_control(xc_handle, domid, DOM0_SHADOW_CONTROL_OP_CLEAN,
-                          arr, max_pfn, NULL);
+                          arr, max_pfn, NULL, 0, NULL);
         DPRINTF("#Flush\n");
         for ( i = 0; i < 40; i++ ) {
             usleep(50000);
             now = llgettimeofday();
             xc_shadow_control(xc_handle, domid, DOM0_SHADOW_CONTROL_OP_PEEK,
-                              NULL, 0, &stats);
+                              NULL, 0, NULL, 0, &stats);
 
             DPRINTF("now= %lld faults= %" PRId32 " dirty= %" PRId32
                     " dirty_net= %" PRId32 " dirty_block= %" PRId32"\n",
@@ -727,7 +727,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
 
         if (xc_shadow_control(xc_handle, dom,
                               DOM0_SHADOW_CONTROL_OP_ENABLE_LOGDIRTY,
-                              NULL, 0, NULL ) < 0) {
+                              NULL, 0, NULL, 0, NULL) < 0) {
             ERR("Couldn't enable shadow mode");
             goto out;
         }
@@ -879,7 +879,7 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
                but this is fast enough for the moment. */
             if (!last_iter && xc_shadow_control(
                     xc_handle, dom, DOM0_SHADOW_CONTROL_OP_PEEK,
-                    to_skip, max_pfn, NULL) != max_pfn) {
+                    to_skip, max_pfn, NULL, 0, NULL) != max_pfn) {
                 ERR("Error peeking shadow bitmap");
                 goto out;
             }
@@ -1084,8 +1084,9 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
                         (unsigned long)ctxt.user_regs.edx);
             }
 
-            if (xc_shadow_control(xc_handle, dom, DOM0_SHADOW_CONTROL_OP_CLEAN,
-                                  to_send, max_pfn, &stats ) != max_pfn) {
+            if (xc_shadow_control(xc_handle, dom, 
+                                  DOM0_SHADOW_CONTROL_OP_CLEAN, to_send, 
+                                  max_pfn, NULL, 0, &stats) != max_pfn) {
                 ERR("Error flushing shadow PT");
                 goto out;
             }
@@ -1174,8 +1175,9 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
  out:
 
     if (live) {
-        if(xc_shadow_control(xc_handle, dom, DOM0_SHADOW_CONTROL_OP_OFF,
-                             NULL, 0, NULL ) < 0) {
+        if(xc_shadow_control(xc_handle, dom, 
+                             DOM0_SHADOW_CONTROL_OP_OFF,
+                             NULL, 0, NULL, 0, NULL) < 0) {
             DPRINTF("Warning - couldn't disable shadow mode");
         }
     }

tools/libxc/xenctrl.h

@@ -323,6 +323,8 @@ int xc_shadow_control(int xc_handle,
                       unsigned int sop,
                       unsigned long *dirty_bitmap,
                       unsigned long pages,
+                      unsigned long *mb,
+                      uint32_t mode,
                       xc_shadow_control_stats_t *stats);
 
 int xc_bvtsched_global_set(int xc_handle,

tools/misc/xc_shadow.c

@@ -60,6 +60,8 @@ int main(int argc, char *argv[])
                            mode, 
                            NULL,
                            0,
+                           NULL,
+                           0,
                            NULL) < 0 )
     {    
         fprintf(stderr, "Error reseting performance counters: %d (%s)\n",

tools/python/xen/lowlevel/xc/xc.c

@@ -669,6 +669,59 @@ static PyObject *pyxc_sedf_domain_get(XcObject *self, PyObject *args)
                          "weight",    weight);
 }
 
+static PyObject *pyxc_shadow_control(PyObject *self,
+                                     PyObject *args,
+                                     PyObject *kwds)
+{
+    XcObject *xc = (XcObject *)self;
+
+    uint32_t dom;
+    int op=0;
+
+    static char *kwd_list[] = { "dom", "op", NULL };
+
+    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i|i", kwd_list, 
+                                      &dom, &op) )
+        return NULL;
+    
+    if ( xc_shadow_control(xc->xc_handle, dom, op, NULL, 0, NULL, 0, NULL) 
+         < 0 )
+        return PyErr_SetFromErrno(xc_error);
+    
+    Py_INCREF(zero);
+    return zero;
+}
+
+static PyObject *pyxc_shadow_mem_control(PyObject *self,
+                                         PyObject *args,
+                                         PyObject *kwds)
+{
+    XcObject *xc = (XcObject *)self;
+    int op;
+    uint32_t dom;
+    int mbarg = -1;
+    unsigned long mb;
+
+    static char *kwd_list[] = { "dom", "mb", NULL };
+
+    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i|i", kwd_list, 
+                                      &dom, &mbarg) )
+        return NULL;
+    
+    if ( mbarg < 0 ) 
+        op = DOM0_SHADOW2_CONTROL_OP_GET_ALLOCATION;
+    else 
+    {
+        mb = mbarg;
+        op = DOM0_SHADOW2_CONTROL_OP_SET_ALLOCATION;
+    }
+    if ( xc_shadow_control(xc->xc_handle, dom, op, NULL, 0, &mb, 0, NULL) < 0 )
+        return PyErr_SetFromErrno(xc_error);
+    
+    mbarg = mb;
+    return Py_BuildValue("i", mbarg);
+}
+
 static PyObject *pyxc_sched_credit_domain_set(XcObject *self,
                                               PyObject *args,
                                               PyObject *kwds)
@@ -1119,6 +1172,22 @@ static PyMethodDef pyxc_methods[] = {
       "Returns [dict]: information about Xen"
       "        [None]: on failure.\n" },
 
+    { "shadow_control", 
+      (PyCFunction)pyxc_shadow_control, 
+      METH_VARARGS | METH_KEYWORDS, "\n"
+      "Set parameter for shadow pagetable interface\n"
+      " dom [int]:   Identifier of domain.\n"
+      " op [int, 0]: operation\n\n"
+      "Returns: [int] 0 on success; -1 on error.\n" },
+
+    { "shadow_mem_control", 
+      (PyCFunction)pyxc_shadow_mem_control, 
+      METH_VARARGS | METH_KEYWORDS, "\n"
+      "Set or read shadow pagetable memory use\n"
+      " dom [int]:   Identifier of domain.\n"
+      " mb [int, -1]: MB of shadow memory this domain should have.\n\n"
+      "Returns: [int] MB of shadow memory in use by this domain.\n" },
+
     { "domain_setmaxmem", 
       (PyCFunction)pyxc_domain_setmaxmem, 
       METH_VARARGS, "\n"

tools/python/xen/xend/XendDomain.py

@@ -532,6 +532,30 @@ class XendDomain:
         except Exception, ex:
             raise XendError(str(ex))
 
+    def domain_shadow_control(self, domid, op):
+        """Shadow page control."""
+        dominfo = self.domain_lookup(domid)
+        try:
+            return xc.shadow_control(dominfo.getDomid(), op)
+        except Exception, ex:
+            raise XendError(str(ex))
+
+    def domain_shadow_mem_get(self, domid):
+        """Get shadow pagetable memory allocation."""
+        dominfo = self.domain_lookup(domid)
+        try:
+            return xc.shadow_mem_control(dominfo.getDomid())
+        except Exception, ex:
+            raise XendError(str(ex))
+
+    def domain_shadow_mem_set(self, domid, mb):
+        """Set shadow pagetable memory allocation."""
+        dominfo = self.domain_lookup(domid)
+        try:
+            return xc.shadow_mem_control(dominfo.getDomid(), mb=mb)
+        except Exception, ex:
+            raise XendError(str(ex))
+
     def domain_sched_credit_get(self, domid):
         """Get credit scheduler parameters for a domain.
         """

tools/python/xen/xend/XendDomainInfo.py

@@ -30,6 +30,7 @@ import string
 import time
 import threading
 import os
+import math
 
 import xen.lowlevel.xc
 from xen.util import asserts
@@ -126,16 +127,17 @@ VM_CONFIG_PARAMS = [
 # don't come out of xc in the same form as they are specified in the config
 # file, so those are handled separately.
 ROUNDTRIPPING_CONFIG_ENTRIES = [
-    ('uuid',       str),
-    ('vcpus',      int),
-    ('vcpu_avail', int),
-    ('cpu_weight', float),
-    ('memory',     int),
-    ('maxmem',     int),
-    ('bootloader', str),
+    ('uuid',            str),
+    ('vcpus',           int),
+    ('vcpu_avail',      int),
+    ('cpu_weight',      float),
+    ('memory',          int),
+    ('shadow_memory',   int),
+    ('maxmem',          int),
+    ('bootloader',      str),
     ('bootloader_args', str),
-    ('features', str),
-    ('localtime', int),
+    ('features',        str),
+    ('localtime',       int),
     ]
 
 ROUNDTRIPPING_CONFIG_ENTRIES += VM_CONFIG_PARAMS
@@ -146,12 +148,13 @@ ROUNDTRIPPING_CONFIG_ENTRIES += VM_CONFIG_PARAMS
 # entries written to the store that cannot be reconfigured on-the-fly.
 #
 VM_STORE_ENTRIES = [
-    ('uuid',       str),
-    ('vcpus',      int),
-    ('vcpu_avail', int),
-    ('memory',     int),
-    ('maxmem',     int),
-    ('start_time', float),
+    ('uuid',          str),
+    ('vcpus',         int),
+    ('vcpu_avail',    int),
+    ('memory',        int),
+    ('shadow_memory', int),
+    ('maxmem',        int),
+    ('start_time',    float),
     ]
 
 VM_STORE_ENTRIES += VM_CONFIG_PARAMS
@@ -572,6 +575,7 @@ class XendDomainInfo:
             defaultInfo('vcpu_avail',   lambda: (1 << self.info['vcpus']) - 1)
 
             defaultInfo('memory',       lambda: 0)
+            defaultInfo('shadow_memory', lambda: 0)
             defaultInfo('maxmem',       lambda: 0)
             defaultInfo('bootloader',   lambda: None)
             defaultInfo('bootloader_args', lambda: None)            
@@ -1280,7 +1284,18 @@ class XendDomainInfo:
             xc.domain_setmaxmem(self.domid, self.info['maxmem'] * 1024)
 
             m = self.image.getDomainMemory(self.info['memory'] * 1024)
-            balloon.free(m)
+
+            # get the domain's shadow memory requirement
+            sm = int(math.ceil(self.image.getDomainShadowMemory(m) / 1024.0))
+            if self.info['shadow_memory'] > sm:
+                sm = self.info['shadow_memory']
+
+            # Make sure there's enough RAM available for the domain
+            balloon.free(m + sm * 1024)
+
+            # Set up the shadow memory
+            sm = xc.shadow_mem_control(self.domid, mb=sm)
+            self.info['shadow_memory'] = sm
 
             init_reservation = self.info['memory'] * 1024
             if os.uname()[4] in ('ia64', 'ppc64'):

tools/python/xen/xend/image.py

@@ -153,6 +153,12 @@ class ImageHandler:
                 mem_kb += 4*1024;
         return mem_kb
 
+    def getDomainShadowMemory(self, mem_kb):
+        """@return The minimum shadow memory required, in KiB, for a domain 
+        with mem_kb KiB of RAM."""
+        # PV domains don't need any shadow memory
+        return 0
+
     def buildDomain(self):
         """Build the domain. Define in subclass."""
         raise NotImplementedError()
@@ -364,6 +370,17 @@ class HVMImageHandler(ImageHandler):
             extra_pages = int( math.ceil( extra_mb*1024 / page_kb ))
         return mem_kb + extra_pages * page_kb
 
+    def getDomainShadowMemory(self, mem_kb):
+        """@return The minimum shadow memory required, in KiB, for a domain 
+        with mem_kb KiB of RAM."""
+        if os.uname()[4] in ('ia64', 'ppc64'):
+            # Explicit shadow memory is not a concept 
+            return 0
+        else:
+            # 1MB per vcpu plus 4Kib/Mib of RAM.  This is higher than 
+            # the minimum that Xen would allocate if no value were given.
+            return 1024 * self.vm.getVCpuCount() + mem_kb / 256
+
     def register_shutdown_watch(self):
         """ add xen store watch on control/shutdown """
         self.shutdownWatch = xswatch(self.vm.dompath + "/control/shutdown", \

tools/python/xen/xm/create.py

@@ -158,6 +158,10 @@ gopts.var('maxmem', val='MEMORY',
           fn=set_int, default=None,
           use="Maximum domain memory in MB.")
 
+gopts.var('shadow_memory', val='MEMORY',
+          fn=set_int, default=0,
+          use="Domain shadow memory in MB.")
+
 gopts.var('cpu', val='CPU',
           fn=set_int, default=None,
           use="CPU to run the VCPU0 on.")
@@ -666,8 +670,9 @@ def make_config(vals):
             if v:
                 config.append([n, v])
 
-    map(add_conf, ['name', 'memory', 'maxmem', 'restart', 'on_poweroff',
-                   'on_reboot', 'on_crash', 'vcpus', 'features'])
+    map(add_conf, ['name', 'memory', 'maxmem', 'shadow_memory',
+                   'restart', 'on_poweroff', 'on_reboot', 'on_crash',
+                   'vcpus', 'features'])
 
     if vals.uuid is not None:
         config.append(['uuid', vals.uuid])

xen/arch/x86/Makefile

@@ -8,7 +8,6 @@ subdir-$(x86_32) += x86_32
 subdir-$(x86_64) += x86_64
 
 obj-y += apic.o
-obj-y += audit.o
 obj-y += bitops.o
 obj-y += compat.o
 obj-y += delay.o
@@ -41,12 +40,21 @@ obj-y += usercopy.o
 obj-y += x86_emulate.o
 
 ifneq ($(pae),n)
-obj-$(x86_32) += shadow.o shadow_public.o shadow_guest32.o shadow_guest32pae.o
+obj-$(x86_32) += shadow2-common.o shadow2_g2_on_s3.o shadow2_g3_on_s3.o
 else
-obj-$(x86_32) += shadow32.o
+obj-$(x86_32) += shadow2-common.o shadow2_g2_on_s2.o
 endif
 
-obj-$(x86_64) += shadow.o shadow_public.o shadow_guest32.o shadow_guest32pae.o
+obj-$(x86_64) += shadow2-common.o shadow2_g4_on_s4.o shadow2_g3_on_s3.o \
+                 shadow2_g2_on_s3.o
+
+guest_levels  = $(subst g,,$(filter g%,$(subst ., ,$(subst _, ,$(subst shadow2_,,$(1))))))
+shadow_levels = $(subst s,,$(filter s%,$(subst ., ,$(subst _, ,$(subst shadow2_,,$(1))))))
+shadow2_defns = -DGUEST_PAGING_LEVELS=$(call guest_levels,$(1)) \
+                -DSHADOW_PAGING_LEVELS=$(call shadow_levels,$(1))
+
+shadow2_%.o: shadow2.c $(HDRS) Makefile
+	$(CC) $(CFLAGS) $(call shadow2_defns,$(@F)) -c $< -o $@
 
 obj-$(crash_debug) += gdbstub.o
 

xen/arch/x86/audit.c

@@ -1,984 +0,0 @@
-/******************************************************************************
- * arch/x86/audit.c
- * 
- * Copyright (c) 2002-2005 K A Fraser
- * Copyright (c) 2004 Christian Limpach
- * Copyright (c) 2005 Michael A Fetterman
- * 
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- * 
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-#include <xen/config.h>
-#include <xen/init.h>
-#include <xen/kernel.h>
-#include <xen/lib.h>
-#include <xen/mm.h>
-#include <xen/perfc.h>
-#include <asm/shadow.h>
-#include <asm/page.h>
-#include <asm/flushtlb.h>
-
-/* XXX SMP bug -- these should not be statics... */
-static int ttot=0, ctot=0, io_mappings=0, lowmem_mappings=0;
-static int l1, l2, oos_count, page_count;
-
-#define FILE_AND_LINE 0
-
-#if FILE_AND_LINE
-#define adjust(_p, _a) _adjust((_p), (_a), __FILE__, __LINE__)
-#define ADJUST_EXTRA_ARGS ,const char *file, int line
-#define APRINTK(_f, _a...) printk(_f " %s:%d\n", ## _a, file, line)
-#else
-#define adjust _adjust
-#define ADJUST_EXTRA_ARGS
-#define APRINTK(_f, _a...) printk(_f "\n", ##_a)
-#endif
-
-int audit_adjust_pgtables(struct domain *d, int dir, int noisy)
-{
-    int errors = 0;
-    int shadow_refcounts = !!shadow_mode_refcounts(d);
-    int shadow_enabled = !!shadow_mode_enabled(d);
-
-    int l2limit( unsigned long mfn )
-    {
-
-        if ( shadow_mode_external(d) )
-            return L2_PAGETABLE_ENTRIES;
-
-#ifdef __i386__
-#ifdef CONFIG_X86_PAE
-        /* 32b PAE */
-        if ( (( mfn_to_page(mfn)->u.inuse.type_info & PGT_va_mask ) 
-	    >> PGT_va_shift) == 3 )
-            return l2_table_offset(HYPERVISOR_VIRT_START);
-        else
-            return L2_PAGETABLE_ENTRIES;
-#else
-        /* 32b non-PAE */
-        return DOMAIN_ENTRIES_PER_L2_PAGETABLE;
-#endif
-#else
-        /* 64b */
-        return 0; /* XXX x86/64 XXX */
-#endif
-    }
-
-    void _adjust(struct page_info *page, int adjtype ADJUST_EXTRA_ARGS)
-    {
-        int count;
-
-        if ( adjtype )
-        {
-            /* adjust the type count */
-            int tcount = page->u.inuse.type_info & PGT_count_mask;
-            tcount += dir;
-            ttot++;
-
-            if ( page_get_owner(page) == NULL )
-            {
-                APRINTK("adjust(mfn=%lx, dir=%d, adjtype=%d) owner=NULL",
-                        page_to_mfn(page), dir, adjtype);
-                errors++;
-            }
-
-            if ( tcount < 0 )
-            {
-                APRINTK("Audit %d: type count went below zero "
-                        "mfn=%lx t=%" PRtype_info " ot=%x",
-                        d->domain_id, page_to_mfn(page),
-                        page->u.inuse.type_info,
-                        page->tlbflush_timestamp);
-                errors++;
-            }
-            else if ( (tcount & ~PGT_count_mask) != 0 )
-            {
-                APRINTK("Audit %d: type count overflowed "
-                        "mfn=%lx t=%" PRtype_info " ot=%x",
-                        d->domain_id, page_to_mfn(page),
-                        page->u.inuse.type_info,
-                        page->tlbflush_timestamp);
-                errors++;
-            }
-            else
-                page->u.inuse.type_info += dir;
-        }
-
-        /* adjust the general count */
-        count = (page->count_info & PGC_count_mask) + dir;
-        ctot++;
-
-        if ( count < 0 )
-        {
-            APRINTK("Audit %d: general count went below zero "
-                    "mfn=%lx t=%" PRtype_info " ot=%x",
-                    d->domain_id, page_to_mfn(page),
-                    page->u.inuse.type_info,
-                    page->tlbflush_timestamp);
-            errors++;
-        }
-        else if ( (count & ~PGT_count_mask) != 0 )
-        {
-            APRINTK("Audit %d: general count overflowed "
-                    "mfn=%lx t=%" PRtype_info " ot=%x",
-                    d->domain_id, page_to_mfn(page),
-                    page->u.inuse.type_info,
-                    page->tlbflush_timestamp);
-            errors++;
-        }
-        else
-            page->count_info += dir;
-    }
-
-    void adjust_l2_page(unsigned long mfn, int shadow)
-    {
-        l2_pgentry_t *pt = map_domain_page(mfn);
-        int i;
-        u32 page_type;
-
-        for ( i = 0; i < l2limit(mfn); i++ )
-        {
-            if ( l2e_get_flags(pt[i]) & _PAGE_PRESENT )
-            {
-	        unsigned long l1mfn = l2e_get_pfn(pt[i]);
-                struct page_info *l1page = mfn_to_page(l1mfn);
-
-                if ( noisy )
-                {
-                    if ( shadow )
-                    {
-                        if ( page_get_owner(l1page) != NULL )
-                        {
-                            printk("L2: Bizarre shadow L1 page mfn=%lx "
-                                   "belonging to a domain %p (id=%d)\n",
-                                   l1mfn,
-                                   page_get_owner(l1page),
-                                   page_get_owner(l1page)->domain_id);
-                            errors++;
-                            continue;
-                        }
-
-                        page_type = l1page->u.inuse.type_info & PGT_type_mask;
-                        if ( page_type != PGT_l1_shadow )
-                        {
-                            printk("Audit %d: [Shadow L2 mfn=%lx i=%x] "
-                                   "Expected Shadow L1 t=%" PRtype_info 
-				   " mfn=%lx\n",
-                                   d->domain_id, mfn, i,
-                                   l1page->u.inuse.type_info, l1mfn);
-                            errors++;
-                        }
-                    }
-                    else
-                    {
-                        if ( page_get_owner(l1page) != d )
-                        {
-                            printk("L2: Skip bizarre L1 page mfn=%lx "
-                                   "belonging to other dom %p (id=%d)\n",
-                                   l1mfn,
-                                   page_get_owner(l1page),
-                                   (page_get_owner(l1page)
-                                    ? page_get_owner(l1page)->domain_id
-                                    : -1));
-                            errors++;
-                            continue;
-                        }
-
-                        page_type = l1page->u.inuse.type_info & PGT_type_mask;
-                        if ( page_type == PGT_l2_page_table )
-                        {
-                            printk("Audit %d: [%x] Found %s Linear PT "
-                                   "t=%" PRtype_info " mfn=%lx\n",
-                                   d->domain_id, i, (l1mfn==mfn) ? "Self" : "Other",
-                                   l1page->u.inuse.type_info, l1mfn);
-                        }
-                        else if ( page_type != PGT_l1_page_table )
-                        {
-                            printk("Audit %d: [L2 mfn=%lx i=%x] "
-                                   "Expected L1 t=%" PRtype_info " mfn=%lx\n",
-                                   d->domain_id, mfn, i,
-                                   l1page->u.inuse.type_info, l1mfn);
-                            errors++;
-                        }
-                    }
-                }
-
-                adjust(l1page, !shadow);
-            }
-        }
-
-        if ( shadow_mode_translate(d) && !shadow_mode_external(d) )
-        {
-            unsigned long hl2mfn =
-                l2e_get_pfn(pt[l2_table_offset(LINEAR_PT_VIRT_START)]);
-            struct page_info *hl2page = mfn_to_page(hl2mfn);
-            adjust(hl2page, 0);
-        }
-
-        unmap_domain_page(pt);
-    }
-
-    void adjust_hl2_page(unsigned long hl2mfn)
-    {
-        l2_pgentry_t *pt = map_domain_page(hl2mfn);
-        int i;
-
-        for ( i = 0; i < l2limit(hl2mfn); i++ )
-        {
-            if ( l2e_get_flags(pt[i]) & _PAGE_PRESENT )
-            {
-                unsigned long mfn = l2e_get_pfn(pt[i]);
-                struct page_info *gpage = mfn_to_page(mfn);
-
-                if ( mfn < 0x100 )
-                {
-                    lowmem_mappings++;
-                    continue;
-                }
-
-                if ( !mfn_valid(mfn) )
-                {
-                    io_mappings++;
-                    continue;
-                }
-
-                if ( noisy )
-                {
-                    if ( page_get_owner(gpage) != d )
-                    {
-                        printk("Audit %d: [hl2mfn=%lx,i=%x] Skip foreign page "
-                               "dom=%p (id=%d) mfn=%lx c=%08x t=%"
-			       PRtype_info "\n",
-                               d->domain_id, hl2mfn, i,
-                               page_get_owner(gpage),
-                               page_get_owner(gpage)->domain_id,
-                               mfn,
-                               gpage->count_info,
-                               gpage->u.inuse.type_info);
-                        continue;
-                    }
-                }
-                adjust(gpage, 0);
-            }
-        }
-
-        unmap_domain_page(pt);
-    }
-
-    void adjust_l1_page(unsigned long l1mfn)
-    {
-        l1_pgentry_t *pt = map_domain_page(l1mfn);
-        int i;
-
-        for ( i = 0; i < L1_PAGETABLE_ENTRIES; i++ )
-        {
-            if ( l1e_get_flags(pt[i]) & _PAGE_PRESENT )
-            {
-                unsigned long mfn = l1e_get_pfn(pt[i]);
-                struct page_info *gpage = mfn_to_page(mfn);
-
-                if ( mfn < 0x100 )
-                {
-                    lowmem_mappings++;
-                    continue;
-                }
-
-                if ( !mfn_valid(mfn) )
-                {
-                    io_mappings++;
-                    continue;
-                }
-
-                if ( noisy )
-                {
-                    if ( l1e_get_flags(pt[i]) & _PAGE_RW )
-                    {
-                        // If it's not a writable page, complain.
-                        //
-                        if ( !((gpage->u.inuse.type_info & PGT_type_mask) ==
-                               PGT_writable_page) )
-                        {
-                            printk("Audit %d: [l1mfn=%lx, i=%x] Illegal RW "
-                                   "t=%" PRtype_info " mfn=%lx\n",
-                                   d->domain_id, l1mfn, i,
-                                   gpage->u.inuse.type_info, mfn);
-                            errors++;
-                        }
-
-                        if ( shadow_refcounts &&
-                             page_is_page_table(gpage) &&
-                             ! page_out_of_sync(gpage) )
-                        {
-                            printk("Audit %d: [l1mfn=%lx, i=%x] Illegal RW of "
-                                   "page table mfn=%lx\n",
-                                   d->domain_id, l1mfn, i, mfn);
-                            errors++;
-                        }
-                    }		   
-
-                    if ( page_get_owner(gpage) != d )
-                    {
-                        printk("Audit %d: [l1mfn=%lx,i=%x] Skip foreign page "
-                               "dom=%p (id=%d) mfn=%lx c=%08x t=%" 
-			       PRtype_info "\n",
-                               d->domain_id, l1mfn, i,
-                               page_get_owner(gpage),
-                               page_get_owner(gpage)->domain_id,
-                               mfn,
-                               gpage->count_info,
-                               gpage->u.inuse.type_info);
-                        continue;
-                    }
-                }
-
-                adjust(gpage, (l1e_get_flags(pt[i]) & _PAGE_RW) ? 1 : 0);
-            }
-        }
-
-        unmap_domain_page(pt);
-    }
-
-    void adjust_shadow_tables(void)
-    {
-        struct shadow_status *a;
-        unsigned long smfn, gmfn;
-        struct page_info *page;
-        int i;
-
-        for ( i = 0; i < shadow_ht_buckets; i++ )
-        {
-            a = &d->arch.shadow_ht[i];
-            while ( a && a->gpfn_and_flags )
-            {
-                gmfn = gmfn_to_mfn(d, a->gpfn_and_flags & PGT_mfn_mask);
-                smfn = a->smfn;
-                page = mfn_to_page(smfn);
-
-                switch ( a->gpfn_and_flags & PGT_type_mask ) {
-                case PGT_writable_pred:
-                    break;
-                case PGT_snapshot:
-                    adjust(mfn_to_page(gmfn), 0);
-                    break;
-                case PGT_l1_shadow:
-                    adjust(mfn_to_page(gmfn), 0);
-                    if ( shadow_refcounts )
-                        adjust_l1_page(smfn);
-                    if ( page->u.inuse.type_info & PGT_pinned )
-                        adjust(page, 0);
-                    break;
-                case PGT_hl2_shadow:
-                    adjust(mfn_to_page(gmfn), 0);
-                    if ( shadow_refcounts )
-                        adjust_hl2_page(smfn);
-                    if ( page->u.inuse.type_info & PGT_pinned )
-                        adjust(page, 0);
-                    break;
-                case PGT_l2_shadow:
-                    adjust(mfn_to_page(gmfn), 0);
-                    adjust_l2_page(smfn, 1);
-                    if ( page->u.inuse.type_info & PGT_pinned )
-                        adjust(page, 0);
-                    break;
-                default:
-                    BUG();
-                    break;
-                }
-
-                a = a->next;
-            }
-        }
-    }
-
-    void adjust_oos_list(void)
-    {
-        struct out_of_sync_entry *oos;
-
-        if ( (oos = d->arch.out_of_sync) )
-            ASSERT(shadow_enabled);
-
-        while ( oos )
-        {
-            adjust(mfn_to_page(oos->gmfn), 0);
-
-            // Only use entries that have low bits clear...
-            //
-            if ( !(oos->writable_pl1e & (sizeof(l1_pgentry_t)-1)) )
-                adjust(mfn_to_page(oos->writable_pl1e >> PAGE_SHIFT), 0);
-
-            if ( oos->snapshot_mfn != SHADOW_SNAPSHOT_ELSEWHERE )
-                adjust(mfn_to_page(oos->snapshot_mfn), 0);
-
-            oos = oos->next;
-            oos_count++;
-        }
-    }
-
-    void adjust_for_pgtbase(void)
-    {
-        struct vcpu *v;
-
-        for_each_vcpu(d, v)
-        {
-            if ( !pagetable_is_null(v->arch.guest_table) )
-                adjust(mfn_to_page(pagetable_get_pfn(v->arch.guest_table)),
-                       !shadow_mode_refcounts(d));
-            if ( !pagetable_is_null(v->arch.shadow_table) )
-                adjust(mfn_to_page(pagetable_get_pfn(v->arch.shadow_table)),
-                       0);
-            if ( v->arch.monitor_shadow_ref )
-                adjust(mfn_to_page(v->arch.monitor_shadow_ref), 0);
-        }
-    }
-
-    void adjust_guest_pages(void)
-    {
-        struct list_head *list_ent = d->page_list.next;
-        struct page_info *page;
-        unsigned long mfn, snapshot_mfn;
-
-        while ( list_ent != &d->page_list )
-        {
-            u32 page_type;
-
-            page = list_entry(list_ent, struct page_info, list);
-            snapshot_mfn = mfn = page_to_mfn(page);
-            page_type = page->u.inuse.type_info & PGT_type_mask;
-
-            BUG_ON(page_get_owner(page) != d);
-
-            page_count++;
-
-            if ( shadow_enabled && !shadow_refcounts &&
-                 page_out_of_sync(page) )
-            {
-                unsigned long gpfn = mfn_to_gmfn(d, mfn);
-                ASSERT( VALID_M2P(gpfn) );
-                snapshot_mfn = __shadow_status(d, gpfn, PGT_snapshot);
-                ASSERT( snapshot_mfn );
-            }
-
-            switch ( page_type )
-            {
-            case PGT_l2_page_table:
-                l2++;
-
-                if ( noisy )
-                {
-                    if ( shadow_refcounts )
-                    {
-                        printk("Audit %d: found an L2 guest page "
-                               "mfn=%lx t=%" PRtype_info " c=%08x while in shadow mode\n",
-                               d->domain_id, mfn, page->u.inuse.type_info,
-                               page->count_info);
-                        errors++;
-                    }
-
-                    if ( (page->u.inuse.type_info & PGT_count_mask) != 0 )
-                    {
-                        if ( (page->u.inuse.type_info & PGT_validated) !=
-                             PGT_validated )
-                        {
-                            printk("Audit %d: L2 mfn=%lx not validated %"
-				   PRtype_info "\n",
-                                   d->domain_id, mfn, page->u.inuse.type_info);
-                            errors++;
-                        }
-
-                    }
-                }
-
-                if ( page->u.inuse.type_info & PGT_pinned )
-                    adjust(page, 1);
-
-                if ( page->u.inuse.type_info & PGT_validated )
-                    adjust_l2_page(snapshot_mfn, 0);
-
-                break;
-
-            case PGT_l1_page_table:
-                l1++;
-
-                if ( noisy )
-                {
-                    if ( shadow_refcounts )
-                    {
-                        printk("found an L1 guest page mfn=%lx t=%" 
-			       PRtype_info " c=%08x "
-                               "while in shadow mode\n",
-                               mfn, page->u.inuse.type_info, page->count_info);
-                        errors++;
-                    }
-
-                    if ( (page->u.inuse.type_info & PGT_count_mask) != 0 )
-                    {
-                        if ( (page->u.inuse.type_info & PGT_validated) !=
-                             PGT_validated )
-                        {
-                            printk("Audit %d: L1 not validated mfn=%lx t=%"
-				   PRtype_info "\n",
-                                   d->domain_id, mfn, page->u.inuse.type_info);
-                            errors++;
-                        }
-                    }
-                }
-                
-                if ( page->u.inuse.type_info & PGT_pinned )
-                    adjust(page, 1);
-
-                if ( page->u.inuse.type_info & PGT_validated )
-                    adjust_l1_page(snapshot_mfn);
-
-                break;
-
-            case PGT_gdt_page:
-                ASSERT( !page_out_of_sync(page) );
-                adjust(page, 1);
-                break;
-
-            case PGT_ldt_page:
-                ASSERT( !page_out_of_sync(page) );
-                adjust(page, 1);
-                break;
-
-            case PGT_writable_page:
-                if ( shadow_refcounts )
-                {
-                    // In shadow mode, writable pages can get pinned by
-                    // paravirtualized guests that think they are pinning
-                    // their L1s and/or L2s.
-                    //
-                    if ( page->u.inuse.type_info & PGT_pinned )
-                        adjust(page, 1);
-                }
-            }
-
-            list_ent = page->list.next;
-        }
-    }
-
-    adjust_for_pgtbase();
-
-    adjust_guest_pages();
-
-    if ( shadow_enabled )
-    {
-        adjust_oos_list();
-        adjust_shadow_tables();
-    }
-
-    adjust(virt_to_page(d->shared_info), 1);
-
-    return errors;
-}
-
-
-#ifndef NDEBUG
-
-void audit_pagelist(struct domain *d)
-{
-    struct list_head *list_ent;
-    int xenpages, totpages;
-
-    list_ent = d->xenpage_list.next;
-    for ( xenpages = 0; (list_ent != &d->xenpage_list); xenpages++ )
-    {
-        list_ent = list_ent->next;
-    }
-    list_ent = d->page_list.next;
-    for ( totpages = 0; (list_ent != &d->page_list); totpages++ )
-    {
-        list_ent = list_ent->next;
-    }
-
-    if ( xenpages != d->xenheap_pages ||
-         totpages != d->tot_pages )
-    {
-        printk("ARGH! dom %d: xen=%d %d, pages=%d %d\n", d->domain_id,
-               xenpages, d->xenheap_pages, 
-               totpages, d->tot_pages );
-    }
-}
-
-void _audit_domain(struct domain *d, int flags)
-{
-    int shadow_refcounts = !!shadow_mode_refcounts(d);
-
-    void scan_for_pfn_in_mfn(struct domain *d, unsigned long xmfn,
-                             unsigned long mfn)
-    {
-        struct page_info *page = mfn_to_page(mfn);
-        l1_pgentry_t *pt = map_domain_page(mfn);
-        int i;
-
-        for ( i = 0; i < L1_PAGETABLE_ENTRIES; i++ )
-        {
-            if ( (l1e_get_flags(pt[i]) & _PAGE_PRESENT) && 
-                 (l1e_get_pfn(pt[i]) == xmfn) )
-                printk("     found dom=%d mfn=%lx t=%" PRtype_info " c=%08x "
-                       "pt[i=%x]=%" PRIpte "\n",
-                       d->domain_id, mfn, page->u.inuse.type_info,
-                       page->count_info, i, l1e_get_intpte(pt[i]));
-        }
-
-        unmap_domain_page(pt);           
-    }
-
-    void scan_for_pfn_in_grant_table(struct domain *d, unsigned xmfn)
-    {
-        int i;
-        struct active_grant_entry *act = d->grant_table->active;
-
-        spin_lock(&d->grant_table->lock);
-
-        for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
-        {
-            if ( act[i].pin && (act[i].frame == xmfn) )
-            {
-                printk("     found active grant table entry i=%d dom=%d pin=%d\n",
-                       i, act[i].domid, act[i].pin);
-            }
-        }
-
-        spin_unlock(&d->grant_table->lock);
-    }
-
-    void scan_for_pfn(struct domain *d, unsigned long xmfn)
-    {
-        scan_for_pfn_in_grant_table(d, xmfn);
-
-        if ( !shadow_mode_enabled(d) )
-        {
-            struct list_head *list_ent = d->page_list.next;
-            struct page_info *page;
-
-            while ( list_ent != &d->page_list )
-            {
-                page = list_entry(list_ent, struct page_info, list);
-
-                switch ( page->u.inuse.type_info & PGT_type_mask )
-                {
-                case PGT_l1_page_table:
-                case PGT_l2_page_table:
-                    scan_for_pfn_in_mfn(d, xmfn, page_to_mfn(page));
-                    break;
-                default:
-                    break;
-                }
-
-                list_ent = page->list.next;
-            }
-        }
-        else
-        {
-            struct shadow_status *a;
-            int i;
-            
-            for ( i = 0; i < shadow_ht_buckets; i++ )
-            {
-                a = &d->arch.shadow_ht[i];
-                while ( a && a->gpfn_and_flags )
-                {
-                    switch ( a->gpfn_and_flags & PGT_type_mask )
-                    {
-                    case PGT_l1_shadow:
-                    case PGT_l2_shadow:
-                    case PGT_hl2_shadow:
-                        scan_for_pfn_in_mfn(d, xmfn, a->smfn);
-                        break;
-                    case PGT_snapshot:
-                    case PGT_writable_pred:
-                        break;
-                    default:
-                        BUG();
-                        break;
-                    }
-                    a = a->next;
-                }
-            }
-        }
-    }
-
-    void scan_for_pfn_remote(unsigned long xmfn)
-    {
-        struct domain *e;
-        for_each_domain ( e )
-            scan_for_pfn( e, xmfn );
-    } 
-
-    unsigned long mfn;
-    struct list_head *list_ent;
-    struct page_info *page;
-    int errors = 0;
-
-    if ( (d != current->domain) && shadow_mode_translate(d) )
-    {
-        printk("skipping audit domain of translated domain %d "
-               "from other context\n",
-               d->domain_id);
-        return;
-    }
-
-    if ( d != current->domain )
-        domain_pause(d);
-
-    // Maybe we should just be using BIGLOCK?
-    //
-    if ( !(flags & AUDIT_SHADOW_ALREADY_LOCKED) )
-        shadow_lock(d);
-
-    spin_lock(&d->page_alloc_lock);
-
-    audit_pagelist(d);
-
-    /* PHASE 0 */
-
-    list_ent = d->page_list.next;
-    while ( list_ent != &d->page_list )
-    {
-        u32 page_type;
-        unsigned long pfn;
-
-        page = list_entry(list_ent, struct page_info, list);
-        mfn = page_to_mfn(page);
-        page_type = page->u.inuse.type_info & PGT_type_mask;
-
-        BUG_ON(page_get_owner(page) != d);
-
-        if ( (page->u.inuse.type_info & PGT_count_mask) >
-             (page->count_info & PGC_count_mask) )
-        {
-            printk("taf(%" PRtype_info ") > caf(%08x) mfn=%lx\n",
-                   page->u.inuse.type_info, page->count_info, mfn);
-            errors++;
-        }
-
-        if ( shadow_mode_refcounts(d) &&
-             (page_type == PGT_writable_page) &&
-             !(page->u.inuse.type_info & PGT_validated) )
-        {
-            printk("shadow mode writable page not validated mfn=%lx " 
-		   "t=%" PRtype_info  " c=%08x\n",
-                   mfn, page->u.inuse.type_info, page->count_info);
-            errors++;
-        }
- 
-#if 0   /* SYSV shared memory pages plus writeable files. */
-        if ( page_type == PGT_writable_page && 
-             (page->u.inuse.type_info & PGT_count_mask) > 1 )
-        {
-            printk("writeable page with type count >1: "
-                   "mfn=%lx t=%" PRtype_info " c=%08x\n",
-                  mfn,
-                  page->u.inuse.type_info,
-                  page->count_info );
-            errors++;
-            scan_for_pfn_remote(mfn);
-        }
-#endif
-
-        if ( page_type == PGT_none && 
-             (page->u.inuse.type_info & PGT_count_mask) > 0 )
-        {
-            printk("normal page with type count >0: mfn=%lx t=%" PRtype_info " c=%08x\n",
-                  mfn,
-                  page->u.inuse.type_info,
-                  page->count_info );
-            errors++;
-        }
-
-        if ( page_out_of_sync(page) )
-        {
-            if ( !page_is_page_table(page) )
-            {
-                printk("out of sync page mfn=%lx is not a page table\n", mfn);
-                errors++;
-            }
-            pfn = mfn_to_gmfn(d, mfn);
-            if ( !__shadow_status(d, pfn, PGT_snapshot) )
-            {
-                printk("out of sync page mfn=%lx doesn't have a snapshot\n",
-                       mfn);
-                errors++;
-            }
-            if ( shadow_refcounts
-                 ? (page_type != PGT_writable_page)
-                 : !(page_type && (page_type <= PGT_l4_page_table)) )
-            {
-                printk("out of sync page mfn=%lx has strange type "
-                       "t=%" PRtype_info  " c=%08x\n",
-                       mfn, page->u.inuse.type_info, page->count_info);
-                errors++;
-            }
-        }
-
-        /* Use tlbflush_timestamp to store original type_info. */
-        page->tlbflush_timestamp = page->u.inuse.type_info;
-
-        list_ent = page->list.next;
-    }
-
-    /* PHASE 1 */
-    io_mappings = lowmem_mappings = 0;
-
-    errors += audit_adjust_pgtables(d, -1, 1);
-
-    if ( !(flags & AUDIT_QUIET) &&
-         ((io_mappings > 0) || (lowmem_mappings > 0)) )
-        printk("Audit %d: Found %d lowmem mappings and %d io mappings\n",
-               d->domain_id, lowmem_mappings, io_mappings);
-
-    /* PHASE 2 */
-
-    list_ent = d->page_list.next;
-    while ( list_ent != &d->page_list )
-    {
-        page = list_entry(list_ent, struct page_info, list);
-        mfn = page_to_mfn(page);
-
-        switch ( page->u.inuse.type_info & PGT_type_mask)
-        {
-        case PGT_l1_page_table:
-        case PGT_l2_page_table:
-        case PGT_l3_page_table:
-        case PGT_l4_page_table:
-            if ( (page->u.inuse.type_info & PGT_count_mask) != 0 )
-            {
-                printk("Audit %d: type count!=0 t=%" PRtype_info " ot=%x c=%x mfn=%lx\n",
-                       d->domain_id, page->u.inuse.type_info, 
-                       page->tlbflush_timestamp,
-                       page->count_info, mfn);
-                errors++;
-                scan_for_pfn_remote(mfn);
-            }
-            break;
-        case PGT_none:
-        case PGT_writable_page:
-        case PGT_gdt_page:
-        case PGT_ldt_page:
-            if ( (page->u.inuse.type_info & PGT_count_mask) != 0 )
-            {
-                printk("Audit %d: type count!=0 t=%" PRtype_info " ot=%x c=%x mfn=%lx\n",
-                       d->domain_id, page->u.inuse.type_info, 
-                       page->tlbflush_timestamp,
-                       page->count_info, mfn);
-                //errors++;
-            }
-            break;
-        default:
-            BUG(); // XXX fix me...
-        }
-        
-        if ( (page->count_info & PGC_count_mask) != 1 )
-        {
-            printk("Audit %d: gen count!=1 (c=%x) t=%" PRtype_info " ot=%x mfn=%lx\n",
-                   d->domain_id,
-                   page->count_info,
-                   page->u.inuse.type_info, 
-                   page->tlbflush_timestamp, mfn );
-            //errors++;
-            scan_for_pfn_remote(mfn);
-        }
-
-        list_ent = page->list.next;
-    }
-
-    if ( shadow_mode_enabled(d) )
-    {
-        struct shadow_status *a;
-        struct page_info *page;
-        u32 page_type;
-        int i;
-
-        for ( i = 0; i < shadow_ht_buckets; i++ )
-        {
-            a = &d->arch.shadow_ht[i];
-            while ( a && a->gpfn_and_flags )
-            {
-                page = mfn_to_page(a->smfn);
-                page_type = a->gpfn_and_flags & PGT_type_mask;
-
-                switch ( page_type ) {
-                case PGT_l1_shadow:
-                case PGT_l2_shadow:
-                case PGT_hl2_shadow:
-                case PGT_snapshot:
-                    if ( ((page->u.inuse.type_info & PGT_type_mask) != page_type ) ||
-                         (page->count_info != 0) )
-                    {
-                        printk("Audit %d: shadow page counts wrong "
-                               "mfn=%lx t=%" PRtype_info " c=%08x\n",
-                               d->domain_id, page_to_mfn(page),
-                               page->u.inuse.type_info,
-                               page->count_info);
-                        printk("a->gpfn_and_flags=%"PRIx64"\n",
-                               (u64)a->gpfn_and_flags);
-                        errors++;
-                    }
-                    break;
-                case PGT_writable_pred:
-                    // XXX - nothing to check?
-                    break;
-
-                default:
-                    BUG();
-                    break;
-                }
-
-                a = a->next;
-            }
-        }
-    }
-
-    /* PHASE 3 */
-    ctot = ttot = page_count = l1 = l2 = oos_count = 0;
-
-    audit_adjust_pgtables(d, 1, 0);
-
-#if 0
-    // This covers our sins of trashing the tlbflush_timestamps...
-    //
-    local_flush_tlb();
-#endif
-
-    spin_unlock(&d->page_alloc_lock);
-
-    if ( !(flags & AUDIT_QUIET) )
-        printk("Audit dom%d Done. "
-               "pages=%d oos=%d l1=%d l2=%d ctot=%d ttot=%d\n",
-               d->domain_id, page_count, oos_count, l1, l2, ctot, ttot);
-
-    if ( !(flags & AUDIT_SHADOW_ALREADY_LOCKED) )
-        shadow_unlock(d);
-
-    if ( d != current->domain )
-        domain_unpause(d);
-
-    if ( errors && !(flags & AUDIT_ERRORS_OK) )
-        BUG();
-}
-
-void audit_domains(void)
-{
-    struct domain *d;
-    for_each_domain ( d )
-        audit_domain(d);
-}
-
-void audit_domains_key(unsigned char key)
-{
-    audit_domains();
-}
-#endif

xen/arch/x86/dom0_ops.c

@@ -89,7 +89,7 @@ long arch_do_dom0_op(struct dom0_op *op, XEN_GUEST_HANDLE(dom0_op_t) u_dom0_op)
         d = find_domain_by_id(op->u.shadow_control.domain);
         if ( d != NULL )
         {
-            ret = shadow_mode_control(d, &op->u.shadow_control);
+            ret = shadow2_control_op(d, &op->u.shadow_control, u_dom0_op);
             put_domain(d);
             copy_to_guest(u_dom0_op, op, 1);
         } 

xen/arch/x86/domain.c

@@ -134,13 +134,6 @@ struct vcpu *alloc_vcpu_struct(struct domain *d, unsigned int vcpu_id)
     v->arch.perdomain_ptes =
         d->arch.mm_perdomain_pt + (vcpu_id << GDT_LDT_VCPU_SHIFT);
 
-    v->arch.guest_vtable  = __linear_l2_table;
-    v->arch.shadow_vtable = __shadow_linear_l2_table;
-#if defined(__x86_64__)
-    v->arch.guest_vl3table = __linear_l3_table;
-    v->arch.guest_vl4table = __linear_l4_table;
-#endif
-
     pae_l3_cache_init(&v->arch.pae_l3_cache);
 
     return v;
@@ -155,9 +148,7 @@ int arch_domain_create(struct domain *d)
 {
     l1_pgentry_t gdt_l1e;
     int vcpuid, pdpt_order;
-#ifdef __x86_64__
     int i;
-#endif
 
     pdpt_order = get_order_from_bytes(PDPT_L1_ENTRIES * sizeof(l1_pgentry_t));
     d->arch.mm_perdomain_pt = alloc_xenheap_pages(pdpt_order);
@@ -202,8 +193,12 @@ int arch_domain_create(struct domain *d)
 
 #endif /* __x86_64__ */
 
-    shadow_lock_init(d);
-    INIT_LIST_HEAD(&d->arch.free_shadow_frames);
+    shadow2_lock_init(d);
+    for ( i = 0; i <= SHADOW2_MAX_ORDER; i++ )
+        INIT_LIST_HEAD(&d->arch.shadow2_freelists[i]);
+    INIT_LIST_HEAD(&d->arch.shadow2_p2m_freelist);
+    INIT_LIST_HEAD(&d->arch.shadow2_p2m_inuse);
+    INIT_LIST_HEAD(&d->arch.shadow2_toplevel_shadows);
 
     if ( !is_idle_domain(d) )
     {
@@ -234,6 +229,8 @@ int arch_domain_create(struct domain *d)
 
 void arch_domain_destroy(struct domain *d)
 {
+    shadow2_final_teardown(d);
+
     free_xenheap_pages(
         d->arch.mm_perdomain_pt,
         get_order_from_bytes(PDPT_L1_ENTRIES * sizeof(l1_pgentry_t)));
@@ -328,14 +325,6 @@ int arch_set_info_guest(
         if ( !hvm_initialize_guest_resources(v) )
             return -EINVAL;
     }
-    else if ( shadow_mode_refcounts(d) )
-    {
-        if ( !get_page(mfn_to_page(cr3_pfn), d) )
-        {
-            destroy_gdt(v);
-            return -EINVAL;
-        }
-    }
     else
     {
         if ( !get_page_and_type(mfn_to_page(cr3_pfn), d,
@@ -344,9 +333,16 @@ int arch_set_info_guest(
             destroy_gdt(v);
             return -EINVAL;
         }
-    }
+    }    
 
-    update_pagetables(v);
+    /* Shadow2: make sure the domain has enough shadow memory to
+     * boot another vcpu */
+    if ( shadow2_mode_enabled(d) 
+         && d->arch.shadow2_total_pages < shadow2_min_acceptable_pages(d) )
+    {
+        destroy_gdt(v);
+        return -ENOMEM;
+    }
 
     if ( v->vcpu_id == 0 )
         update_domain_wallclock_time(d);
@@ -354,6 +350,11 @@ int arch_set_info_guest(
     /* Don't redo final setup */
     set_bit(_VCPUF_initialised, &v->vcpu_flags);
 
+    if ( shadow2_mode_enabled(d) )
+        shadow2_update_paging_modes(v);
+
+    update_cr3(v);
+
     return 0;
 }
 
@@ -669,7 +670,6 @@ static void __context_switch(void)
             loaddebug(&n->arch.guest_context, 6);
             loaddebug(&n->arch.guest_context, 7);
         }
-
         n->arch.ctxt_switch_to(n);
     }
 
@@ -927,29 +927,34 @@ void domain_relinquish_resources(struct domain *d)
     /* Drop the in-use references to page-table bases. */
     for_each_vcpu ( d, v )
     {
-        if ( (pfn = pagetable_get_pfn(v->arch.guest_table)) != 0 )
+        /* Drop ref to guest_table (from new_guest_cr3(), svm/vmx cr3 handling,
+         * or sh2_update_paging_modes()) */
+        pfn = pagetable_get_pfn(v->arch.guest_table);
+        if ( pfn != 0 )
         {
-            if ( !shadow_mode_refcounts(d) )
-                put_page_type(mfn_to_page(pfn));
-            put_page(mfn_to_page(pfn));
-
+            if ( shadow2_mode_refcounts(d) )
+                put_page(mfn_to_page(pfn));
+            else
+                put_page_and_type(mfn_to_page(pfn));
             v->arch.guest_table = pagetable_null();
         }
 
-        if ( (pfn = pagetable_get_pfn(v->arch.guest_table_user)) != 0 )
+#ifdef __x86_64__
+        /* Drop ref to guest_table_user (from MMUEXT_NEW_USER_BASEPTR) */
+        pfn = pagetable_get_pfn(v->arch.guest_table_user);
+        if ( pfn != 0 )
         {
-            if ( !shadow_mode_refcounts(d) )
-                put_page_type(mfn_to_page(pfn));
-            put_page(mfn_to_page(pfn));
-
+            put_page_and_type(mfn_to_page(pfn));
             v->arch.guest_table_user = pagetable_null();
         }
+#endif
     }
 
     if ( d->vcpu[0] && hvm_guest(d->vcpu[0]) )
         hvm_relinquish_guest_resources(d);
 
-    shadow_mode_disable(d);
+    /* Tear down shadow mode stuff. */
+    shadow2_teardown(d);
 
     /*
      * Relinquish GDT mappings. No need for explicit unmapping of the LDT as
@@ -964,26 +969,23 @@ void domain_relinquish_resources(struct domain *d)
 
     /* Free page used by xen oprofile buffer */
     free_xenoprof_pages(d);
-
 }
 
 void arch_dump_domain_info(struct domain *d)
 {
-    if ( shadow_mode_enabled(d) )
+    if ( shadow2_mode_enabled(d) )
     {
-        printk("    shadow mode: ");
-        if ( shadow_mode_refcounts(d) )
+        printk("    shadow2 mode: ");
+        if ( d->arch.shadow2_mode & SHM2_enable )
+            printk("enabled ");
+        if ( shadow2_mode_refcounts(d) )
             printk("refcounts ");
-        if ( shadow_mode_write_all(d) )
-            printk("write_all ");
-        if ( shadow_mode_log_dirty(d) )
+        if ( shadow2_mode_log_dirty(d) )
             printk("log_dirty ");
-        if ( shadow_mode_translate(d) )
+        if ( shadow2_mode_translate(d) )
             printk("translate ");
-        if ( shadow_mode_external(d) )
+        if ( shadow2_mode_external(d) )
             printk("external ");
-        if ( shadow_mode_wr_pt_pte(d) )
-            printk("wr_pt_pte ");
         printk("\n");
     }
 }

xen/arch/x86/domain_build.c

@@ -683,8 +683,11 @@ int construct_dom0(struct domain *d,
     for ( i = 1; i < opt_dom0_max_vcpus; i++ )
         (void)alloc_vcpu(d, i, i);
 
-    /* Set up monitor table */
-    update_pagetables(v);
+    /* Set up CR3 value for write_ptbase */
+    if ( shadow2_mode_enabled(v->domain) )
+        shadow2_update_paging_modes(v);
+    else
+        update_cr3(v);
 
     /* Install the new page tables. */
     local_irq_disable();
@@ -796,10 +799,8 @@ int construct_dom0(struct domain *d,
     new_thread(v, dsi.v_kernentry, vstack_end, vstartinfo_start);
 
     if ( opt_dom0_shadow )
-    {
-        shadow_mode_enable(d, SHM_enable);
-        update_pagetables(v);
-    }
+        if ( shadow2_test_enable(d) == 0 ) 
+            shadow2_update_paging_modes(v);
 
     if ( supervisor_mode_kernel )
     {

xen/arch/x86/hvm/hvm.c

@@ -30,6 +30,7 @@
 #include <xen/hypercall.h>
 #include <xen/guest_access.h>
 #include <xen/event.h>
+#include <xen/shadow.h>
 #include <asm/current.h>
 #include <asm/e820.h>
 #include <asm/io.h>
@@ -42,10 +43,6 @@
 #include <asm/spinlock.h>
 #include <asm/hvm/hvm.h>
 #include <asm/hvm/support.h>
-#include <asm/shadow.h>
-#if CONFIG_PAGING_LEVELS >= 3
-#include <asm/shadow_64.h>
-#endif
 #include <public/sched.h>
 #include <public/hvm/ioreq.h>
 #include <public/version.h>
@@ -61,7 +58,7 @@ struct hvm_function_table hvm_funcs;
 static void hvm_zap_mmio_range(
     struct domain *d, unsigned long pfn, unsigned long nr_pfn)
 {
-    unsigned long i, val = INVALID_MFN;
+    unsigned long i;
 
     ASSERT(d == current->domain);
 
@@ -70,7 +67,8 @@ static void hvm_zap_mmio_range(
         if ( pfn + i >= 0xfffff )
             break;
 
-        __copy_to_user(&phys_to_machine_mapping[pfn + i], &val, sizeof (val));
+        if ( VALID_MFN(gmfn_to_mfn(d, pfn + i)) )
+            guest_remove_page(d, pfn + i);
     }
 }
 
@@ -262,11 +260,13 @@ void hvm_setup_platform(struct domain* d)
     if ( !hvm_guest(v) || (v->vcpu_id != 0) )
         return;
 
+#if 0 /* SHADOW2 does not have this */
     if ( shadow_direct_map_init(d) == 0 )
     {
         printk("Can not allocate shadow direct map for HVM domain.\n");
         domain_crash_synchronous();
     }
+#endif
 
     hvm_zap_iommu_pages(d);
 
@@ -380,6 +380,8 @@ void hvm_hlt(unsigned long rflags)
  */
 int hvm_copy(void *buf, unsigned long vaddr, int size, int dir)
 {
+    struct vcpu *v = current;
+    unsigned long gfn;
     unsigned long mfn;
     char *addr;
     int count;
@@ -389,10 +391,9 @@ int hvm_copy(void *buf, unsigned long vaddr, int size, int dir)
         if (count > size)
             count = size;
 
-        if (hvm_paging_enabled(current))
-            mfn = gva_to_mfn(vaddr);
-        else
-            mfn = get_mfn_from_gpfn(vaddr >> PAGE_SHIFT);
+        gfn = shadow2_gva_to_gfn(v, vaddr);
+        mfn = mfn_x(sh2_vcpu_gfn_to_mfn(v, gfn));
+
         if (mfn == INVALID_MFN)
             return 0;
 
@@ -545,7 +546,7 @@ void hvm_do_hypercall(struct cpu_user_regs *pregs)
         return;
     }
 
-    if ( current->domain->arch.ops->guest_paging_levels == PAGING_L4 )
+    if ( current->arch.shadow2->guest_levels == 4 )
     {
         pregs->rax = hvm_hypercall64_table[pregs->rax](pregs->rdi,
                                                        pregs->rsi,

xen/arch/x86/hvm/platform.c

@@ -21,7 +21,7 @@
 #include <xen/config.h>
 #include <xen/types.h>
 #include <xen/mm.h>
-#include <asm/shadow.h>
+#include <xen/shadow.h>
 #include <xen/domain_page.h>
 #include <asm/page.h>
 #include <xen/event.h>
@@ -35,9 +35,6 @@
 #include <xen/lib.h>
 #include <xen/sched.h>
 #include <asm/current.h>
-#if CONFIG_PAGING_LEVELS >= 3
-#include <asm/shadow_64.h>
-#endif
 
 #define DECODE_success  1
 #define DECODE_failure  0
@@ -724,7 +721,7 @@ void send_pio_req(struct cpu_user_regs *regs, unsigned long port,
 
     if (pvalid) {
         if (hvm_paging_enabled(current))
-            p->u.pdata = (void *) gva_to_gpa(value);
+            p->u.data = shadow2_gva_to_gpa(current, value);
         else
             p->u.pdata = (void *) value; /* guest VA == guest PA */
     } else
@@ -774,7 +771,7 @@ void send_mmio_req(
 
     if (pvalid) {
         if (hvm_paging_enabled(v))
-            p->u.pdata = (void *) gva_to_gpa(value);
+            p->u.data = shadow2_gva_to_gpa(v, value);
         else
             p->u.pdata = (void *) value; /* guest VA == guest PA */
     } else

xen/arch/x86/hvm/svm/svm.c

@@ -26,9 +26,10 @@
 #include <xen/irq.h>
 #include <xen/softirq.h>
 #include <xen/hypercall.h>
+#include <xen/domain_page.h>
 #include <asm/current.h>
 #include <asm/io.h>
-#include <asm/shadow.h>
+#include <asm/shadow2.h>
 #include <asm/regs.h>
 #include <asm/cpufeature.h>
 #include <asm/processor.h>
@@ -43,10 +44,6 @@
 #include <asm/hvm/svm/emulate.h>
 #include <asm/hvm/svm/vmmcall.h>
 #include <asm/hvm/svm/intr.h>
-#include <asm/shadow.h>
-#if CONFIG_PAGING_LEVELS >= 3
-#include <asm/shadow_64.h>
-#endif
 #include <public/sched.h>
 
 #define SVM_EXTRA_DEBUG
@@ -414,7 +411,7 @@ static int svm_realmode(struct vcpu *v)
     return (eflags & X86_EFLAGS_VM) || !(cr0 & X86_CR0_PE);
 }
 
-static int svm_instruction_length(struct vcpu *v)
+int svm_guest_x86_mode(struct vcpu *v)
 {
     struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb;
     unsigned long cr0 = vmcb->cr0, eflags = vmcb->rflags, mode;
@@ -423,10 +420,20 @@ static int svm_instruction_length(struct vcpu *v)
         mode = vmcb->cs.attributes.fields.l ? 8 : 4;
     else
         mode = (eflags & X86_EFLAGS_VM) || !(cr0 & X86_CR0_PE) ? 2 : 4;
-    return svm_instrlen(guest_cpu_user_regs(), mode);
+    return mode;
 }
 
-static unsigned long svm_get_ctrl_reg(struct vcpu *v, unsigned int num)
+int svm_instruction_length(struct vcpu *v)
+{
+    return svm_instrlen(guest_cpu_user_regs(), svm_guest_x86_mode(v));
+}
+
+void svm_update_host_cr3(struct vcpu *v)
+{
+    /* SVM doesn't have a HOST_CR3 equivalent to update. */
+}
+
+unsigned long svm_get_ctrl_reg(struct vcpu *v, unsigned int num)
 {
     switch ( num )
     {
@@ -436,6 +443,8 @@ static unsigned long svm_get_ctrl_reg(struct vcpu *v, unsigned int num)
         return v->arch.hvm_svm.cpu_cr2;
     case 3:
         return v->arch.hvm_svm.cpu_cr3;
+    case 4:
+        return v->arch.hvm_svm.cpu_shadow_cr4;
     default:
         BUG();
     }
@@ -526,8 +535,6 @@ static void svm_init_hypercall_page(struct domain *d, void *hypercall_page)
 }
 
 
-
-
 int svm_dbg_on = 0;
 
 static inline int svm_do_debugout(unsigned long exit_code)
@@ -647,6 +654,11 @@ static void svm_load_cpu_guest_regs(
     svm_load_cpu_user_regs(v, regs);
 }
 
+int svm_long_mode_enabled(struct vcpu *v)
+{
+    return SVM_LONG_GUEST(v);
+}
+
 
 
 static void arch_svm_do_launch(struct vcpu *v) 
@@ -726,7 +738,6 @@ static void svm_ctxt_switch_to(struct vcpu *v)
 static void svm_final_setup_guest(struct vcpu *v)
 {
     struct domain *d = v->domain;
-    struct vcpu *vc;
 
     v->arch.schedule_tail    = arch_svm_do_launch;
     v->arch.ctxt_switch_from = svm_ctxt_switch_from;
@@ -735,9 +746,12 @@ static void svm_final_setup_guest(struct vcpu *v)
     if ( v != d->vcpu[0] )
         return;
 
-    /* Initialize monitor page table */
-    for_each_vcpu( d, vc )
-        vc->arch.monitor_table = pagetable_null();
+    if ( !shadow2_mode_external(d) )
+    {
+        DPRINTK("Can't init HVM for dom %u vcpu %u: "
+                "not in shadow2 external mode\n", d->domain_id, v->vcpu_id);
+        domain_crash(d);
+    }
 
     /* 
      * Required to do this once per domain
@@ -745,13 +759,6 @@ static void svm_final_setup_guest(struct vcpu *v)
      */
     memset(&d->shared_info->evtchn_mask[0], 0xff, 
            sizeof(d->shared_info->evtchn_mask));       
-
-    /* 
-     * Put the domain in shadow mode even though we're going to be using
-     * the shared 1:1 page table initially. It shouldn't hurt 
-     */
-    shadow_mode_enable(d, SHM_enable|SHM_refcounts|
-                       SHM_translate|SHM_external|SHM_wr_pt_pte);
 }
 
 
@@ -809,9 +816,13 @@ int start_svm(void)
 
     hvm_funcs.realmode = svm_realmode;
     hvm_funcs.paging_enabled = svm_paging_enabled;
+    hvm_funcs.long_mode_enabled = svm_long_mode_enabled;
+    hvm_funcs.guest_x86_mode = svm_guest_x86_mode;
     hvm_funcs.instruction_length = svm_instruction_length;
     hvm_funcs.get_guest_ctrl_reg = svm_get_ctrl_reg;
 
+    hvm_funcs.update_host_cr3 = svm_update_host_cr3;
+    
     hvm_funcs.stts = svm_stts;
     hvm_funcs.set_tsc_offset = svm_set_tsc_offset;
 
@@ -834,7 +845,6 @@ static void svm_relinquish_guest_resources(struct domain *d)
             continue;
 
         destroy_vmcb(&v->arch.hvm_svm);
-        free_monitor_pagetable(v);
         kill_timer(&v->arch.hvm_vcpu.hlt_timer);
         if ( hvm_apic_support(v->domain) && (VLAPIC(v) != NULL) ) 
         {
@@ -851,8 +861,6 @@ static void svm_relinquish_guest_resources(struct domain *d)
 
     if ( d->arch.hvm_domain.buffered_io_va )
         unmap_domain_page_global((void *)d->arch.hvm_domain.buffered_io_va);
-
-    shadow_direct_map_clean(d);
 }
 
 
@@ -894,7 +902,6 @@ static int svm_do_page_fault(unsigned long va, struct cpu_user_regs *regs)
 {
     struct vcpu *v = current;
     unsigned long eip;
-    unsigned long gpa; /* FIXME: PAE */
     int result;
     struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb;
 
@@ -907,43 +914,7 @@ static int svm_do_page_fault(unsigned long va, struct cpu_user_regs *regs)
             va, eip, (unsigned long)regs->error_code);
 //#endif
 
-    if ( !svm_paging_enabled(v) )
-    {
-        if ( shadow_direct_map_fault(va, regs) ) 
-            return 1;
-
-        handle_mmio(va, va);
-        return 1;
-    }
-
-
-    gpa = gva_to_gpa(va);
-
-    /* Use 1:1 page table to identify MMIO address space */
-    if (mmio_space(gpa))
-    {
-        /* No support for APIC */
-        if (!hvm_apic_support(v->domain) && gpa >= 0xFEC00000)
-        { 
-            int inst_len;
-            inst_len = svm_instruction_length(v);
-            if (inst_len == -1)
-            {
-                printf("%s: INST_LEN - Unable to decode properly\n", __func__);
-                domain_crash_synchronous();
-            }
-
-            __update_guest_eip(vmcb, inst_len);
-
-            return 1;
-        }
-
-        handle_mmio(va, gpa);
-
-        return 1;
-    }
-    
-    result = shadow_fault(va, regs);
+    result = shadow2_fault(va, regs); 
 
     if( result ) {
         /* Let's make sure that the Guest TLB is flushed */
@@ -1035,19 +1006,12 @@ static void svm_vmexit_do_cpuid(struct vmcb_struct *vmcb, unsigned long input,
             clear_bit(X86_FEATURE_APIC, &edx);
         }
 
-#if CONFIG_PAGING_LEVELS < 3
-        clear_bit(X86_FEATURE_PAE, &edx);
-        clear_bit(X86_FEATURE_PSE, &edx);
-        clear_bit(X86_FEATURE_PSE36, &edx);
-#else
-        if ( v->domain->arch.ops->guest_paging_levels == PAGING_L2 )
-        {
-            if ( !v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
-                clear_bit(X86_FEATURE_PAE, &edx);
-            clear_bit(X86_FEATURE_PSE, &edx);
-            clear_bit(X86_FEATURE_PSE36, &edx);
-        }
+#if CONFIG_PAGING_LEVELS >= 3
+        if ( !v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
 #endif
+            clear_bit(X86_FEATURE_PAE, &edx);
+        clear_bit(X86_FEATURE_PSE36, &edx);
+
         /* Clear out reserved bits. */
         ecx &= ~SVM_VCPU_CPUID_L1_ECX_RESERVED;
         edx &= ~SVM_VCPU_CPUID_L1_EDX_RESERVED;
@@ -1097,23 +1061,12 @@ static void svm_vmexit_do_cpuid(struct vmcb_struct *vmcb, unsigned long input,
         clear_bit(X86_FEATURE_SYSCALL & 31, &edx);
 #endif
 
-#if CONFIG_PAGING_LEVELS < 3
-        clear_bit(X86_FEATURE_NX & 31, &edx);
-        clear_bit(X86_FEATURE_PAE, &edx);
-        clear_bit(X86_FEATURE_PSE, &edx);
-        clear_bit(X86_FEATURE_PSE36, &edx);
-#else
-        if ( v->domain->arch.ops->guest_paging_levels == PAGING_L2 )
-        {
-            if ( !v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
-            {
-                clear_bit(X86_FEATURE_NX & 31, &edx);
-                clear_bit(X86_FEATURE_PAE, &edx);
-            }
-            clear_bit(X86_FEATURE_PSE, &edx);
-            clear_bit(X86_FEATURE_PSE36, &edx);
-        }
+
+#if CONFIG_PAGING_LEVELS >= 3
+        if ( !v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
 #endif
+            clear_bit(X86_FEATURE_PAE, &edx);
+        clear_bit(X86_FEATURE_PSE36, &edx);
 
         /* Make SVM feature invisible to the guest. */
         clear_bit(X86_FEATURE_SVME & 31, &ecx);
@@ -1555,6 +1508,7 @@ static int svm_set_cr0(unsigned long value)
     unsigned long mfn;
     int paging_enabled;
     struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb;
+    unsigned long old_base_mfn;
   
     ASSERT(vmcb);
 
@@ -1600,54 +1554,21 @@ static int svm_set_cr0(unsigned long value)
             set_bit(SVM_CPU_STATE_LMA_ENABLED,
                     &v->arch.hvm_svm.cpu_state);
             vmcb->efer |= (EFER_LMA | EFER_LME);
-            if (!shadow_set_guest_paging_levels(v->domain, PAGING_L4) )
-            {
-                printk("Unsupported guest paging levels\n");
-                domain_crash_synchronous(); /* need to take a clean path */
-            }
         }
-        else
 #endif  /* __x86_64__ */
-        {
-#if CONFIG_PAGING_LEVELS >= 3
-            /* seems it's a 32-bit or 32-bit PAE guest */
-            if ( test_bit(SVM_CPU_STATE_PAE_ENABLED,
-                        &v->arch.hvm_svm.cpu_state) )
-            {
-                /* The guest enables PAE first and then it enables PG, it is
-                 * really a PAE guest */
-                if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L3) )
-                {
-                    printk("Unsupported guest paging levels\n");
-                    domain_crash_synchronous();
-                }
-            }
-            else
-            {
-                if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L2) )
-                {
-                    printk("Unsupported guest paging levels\n");
-                    domain_crash_synchronous(); /* need to take a clean path */
-                }
-            }
-#endif
-        }
 
         /* Now arch.guest_table points to machine physical. */
+        old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
         v->arch.guest_table = pagetable_from_pfn(mfn);
-        update_pagetables(v);
+        if ( old_base_mfn )
+            put_page(mfn_to_page(old_base_mfn));
+        shadow2_update_paging_modes(v);
 
         HVM_DBG_LOG(DBG_LEVEL_VMMU, "New arch.guest_table = %lx", 
                 (unsigned long) (mfn << PAGE_SHIFT));
 
+        vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3; 
         set_bit(ARCH_SVM_VMCB_ASSIGN_ASID, &v->arch.hvm_svm.flags);
-        vmcb->cr3 = pagetable_get_paddr(v->arch.shadow_table);
-
-        /* arch->shadow_table should hold the next CR3 for shadow */
-        HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %lx, mfn = %lx\n", 
-                    v->arch.hvm_svm.cpu_cr3, mfn);
-
-        return 1;
     }
 
     if ( !((value & X86_CR0_PE) && (value & X86_CR0_PG)) && paging_enabled )
@@ -1667,17 +1588,16 @@ static int svm_set_cr0(unsigned long value)
             svm_inject_exception(v, TRAP_gp_fault, 1, 0);
             return 0;
         }
-
-        clear_all_shadow_status( v->domain );
+        shadow2_update_paging_modes(v);
+        vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3;
         set_bit(ARCH_SVM_VMCB_ASSIGN_ASID, &v->arch.hvm_svm.flags);
-        vmcb->cr3 = pagetable_get_paddr(v->domain->arch.phys_table);
     }
     else if ( (value & (X86_CR0_PE | X86_CR0_PG)) == X86_CR0_PE )
     {
         /* we should take care of this kind of situation */
-        clear_all_shadow_status(v->domain);
+        shadow2_update_paging_modes(v);
+        vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3;
         set_bit(ARCH_SVM_VMCB_ASSIGN_ASID, &v->arch.hvm_svm.flags);
-        vmcb->cr3 = pagetable_get_paddr(v->domain->arch.phys_table);
     }
 
     return 1;
@@ -1786,7 +1706,7 @@ static int mov_to_cr(int gpreg, int cr, struct cpu_user_regs *regs)
             mfn = get_mfn_from_gpfn(value >> PAGE_SHIFT);
             if (mfn != pagetable_get_pfn(v->arch.guest_table))
                 __hvm_bug(regs);
-            shadow_sync_all(v->domain);
+            shadow2_update_cr3(v);
         }
         else 
         {
@@ -1812,14 +1732,10 @@ static int mov_to_cr(int gpreg, int cr, struct cpu_user_regs *regs)
             /*
              * arch.shadow_table should now hold the next CR3 for shadow
              */
-#if CONFIG_PAGING_LEVELS >= 3
-            if ( v->domain->arch.ops->guest_paging_levels == PAGING_L3 )
-                shadow_sync_all(v->domain);
-#endif
             v->arch.hvm_svm.cpu_cr3 = value;
-            update_pagetables(v);
+            update_cr3(v);
+            vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3; 
             HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %lx", value);
-            vmcb->cr3 = pagetable_get_paddr(v->arch.shadow_table);
         }
         break;
     }
@@ -1839,12 +1755,6 @@ static int mov_to_cr(int gpreg, int cr, struct cpu_user_regs *regs)
 #if CONFIG_PAGING_LEVELS >= 3
                 unsigned long mfn, old_base_mfn;
 
-                if( !shadow_set_guest_paging_levels(v->domain, PAGING_L3) )
-                {
-                    printk("Unsupported guest paging levels\n");
-                    domain_crash_synchronous(); /* need to take a clean path */
-                }
-
                 if ( !VALID_MFN(mfn = get_mfn_from_gpfn(
                                     v->arch.hvm_svm.cpu_cr3 >> PAGE_SHIFT)) ||
                      !get_page(mfn_to_page(mfn), v->domain) )
@@ -1853,21 +1763,20 @@ static int mov_to_cr(int gpreg, int cr, struct cpu_user_regs *regs)
                     domain_crash_synchronous(); /* need to take a clean path */
                 }
 
-                old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
-                if ( old_base_mfn )
-                    put_page(mfn_to_page(old_base_mfn));
-
                 /*
                  * Now arch.guest_table points to machine physical.
                  */
 
+                old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
                 v->arch.guest_table = pagetable_from_pfn(mfn);
-                update_pagetables(v);
+                if ( old_base_mfn )
+                    put_page(mfn_to_page(old_base_mfn));
+                shadow2_update_paging_modes(v);
 
                 HVM_DBG_LOG(DBG_LEVEL_VMMU, "New arch.guest_table = %lx",
                             (unsigned long) (mfn << PAGE_SHIFT));
 
-                vmcb->cr3 = pagetable_get_paddr(v->arch.shadow_table);
+                vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3; 
 
                 /*
                  * arch->shadow_table should hold the next CR3 for shadow
@@ -1876,33 +1785,6 @@ static int mov_to_cr(int gpreg, int cr, struct cpu_user_regs *regs)
                 HVM_DBG_LOG(DBG_LEVEL_VMMU, 
                             "Update CR3 value = %lx, mfn = %lx",
                             v->arch.hvm_svm.cpu_cr3, mfn);
-#endif
-            }
-            else
-            {
-                /*  The guest is a 64 bit or 32-bit PAE guest. */
-#if CONFIG_PAGING_LEVELS >= 3
-                if ( (v->domain->arch.ops != NULL) &&
-                        v->domain->arch.ops->guest_paging_levels == PAGING_L2)
-                {
-                    /* Seems the guest first enables PAE without enabling PG,
-                     * it must enable PG after that, and it is a 32-bit PAE
-                     * guest */
-
-                    if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L3))
-                    {
-                        printk("Unsupported guest paging levels\n");
-                        domain_crash_synchronous();
-                    }                   
-                }
-                else
-                {
-                    if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L4))
-                    {
-                        printk("Unsupported guest paging levels\n");
-                        domain_crash_synchronous();
-                    }
-                }
 #endif
             }
         }
@@ -1926,7 +1808,7 @@ static int mov_to_cr(int gpreg, int cr, struct cpu_user_regs *regs)
         if ((old_cr ^ value) & (X86_CR4_PSE | X86_CR4_PGE | X86_CR4_PAE))
         {
             set_bit(ARCH_SVM_VMCB_ASSIGN_ASID, &v->arch.hvm_svm.flags);
-            shadow_sync_all(v->domain);
+            shadow2_update_paging_modes(v);
         }
         break;
     }
@@ -2267,7 +2149,7 @@ void svm_handle_invlpg(const short invlpga, struct cpu_user_regs *regs)
 
     /* Overkill, we may not this */
     set_bit(ARCH_SVM_VMCB_ASSIGN_ASID, &v->arch.hvm_svm.flags);
-    shadow_invlpg(v, g_vaddr);
+    shadow2_invlpg(v, g_vaddr);
 }
 
 
@@ -2638,7 +2520,7 @@ void walk_shadow_and_guest_pt(unsigned long gva)
     struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb;
     unsigned long gpa;
 
-    gpa = gva_to_gpa( gva );
+    gpa = shadow2_gva_to_gpa(current, gva);
     printk( "gva = %lx, gpa=%lx, gCR3=%x\n", gva, gpa, (u32)vmcb->cr3 );
     if( !svm_paging_enabled(v) || mmio_space(gpa) )
        return;
@@ -2662,8 +2544,12 @@ void walk_shadow_and_guest_pt(unsigned long gva)
     __copy_from_user(&gpte, &linear_pg_table[ l1_linear_offset(gva) ],
                      sizeof(gpte) );
     printk( "G-PTE = %x, flags=%x\n", gpte.l1, l1e_get_flags(gpte) );
-    __copy_from_user( &spte, &phys_to_machine_mapping[ l1e_get_pfn( gpte ) ],
+
+    BUG(); // need to think about this, and convert usage of
+           // phys_to_machine_mapping to use pagetable format...
+    __copy_from_user( &spte, &phys_to_machine_mapping[ l1e_get_pfn( gpte ) ], 
                       sizeof(spte) );
+
     printk( "S-PTE = %x, flags=%x\n", spte.l1, l1e_get_flags(spte));
 }
 #endif /* SVM_WALK_GUEST_PAGES */
@@ -2704,7 +2590,8 @@ asmlinkage void svm_vmexit_handler(struct cpu_user_regs regs)
 
     if (svm_dbg_on && exit_reason == VMEXIT_EXCEPTION_PF) 
     {
-        if (svm_paging_enabled(v) && !mmio_space(gva_to_gpa(vmcb->exitinfo2)))
+        if (svm_paging_enabled(v) && 
+            !mmio_space(shadow2_gva_to_gpa(current, vmcb->exitinfo2)))
         {
             printk("I%08ld,ExC=%s(%d),IP=%x:%llx,I1=%llx,I2=%llx,INT=%llx, "
                    "gpa=%llx\n", intercepts_counter,
@@ -2713,7 +2600,7 @@ asmlinkage void svm_vmexit_handler(struct cpu_user_regs regs)
 		    (unsigned long long) vmcb->exitinfo1,
 		    (unsigned long long) vmcb->exitinfo2,
 		    (unsigned long long) vmcb->exitintinfo.bytes,
-            (unsigned long long) gva_to_gpa( vmcb->exitinfo2 ) );
+            (unsigned long long) shadow2_gva_to_gpa(current, vmcb->exitinfo2));
         }
         else 
         {
@@ -2757,7 +2644,7 @@ asmlinkage void svm_vmexit_handler(struct cpu_user_regs regs)
         && ( ( vmcb->exitinfo2 == vmcb->rip )
         || vmcb->exitintinfo.bytes) )
     {
-       if (svm_paging_enabled(v) && !mmio_space(gva_to_gpa(vmcb->exitinfo2)))     
+       if (svm_paging_enabled(v) && !mmio_space(gva_to_gpa(vmcb->exitinfo2)))
            walk_shadow_and_guest_pt( vmcb->exitinfo2 );
     }
 #endif

xen/arch/x86/hvm/svm/vmcb.c

@@ -380,8 +380,8 @@ void svm_do_launch(struct vcpu *v)
         printk("%s: phys_table   = %lx\n", __func__, pt);
     }
 
-    /* At launch we always use the phys_table */
-    vmcb->cr3 = pagetable_get_paddr(v->domain->arch.phys_table);
+    /* Set cr3 from hw_cr3 even when guest-visible paging is not enabled */
+    vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3; 
 
     if (svm_dbg_on) 
     {

xen/arch/x86/hvm/vlapic.c

@@ -21,7 +21,8 @@
 #include <xen/types.h>
 #include <xen/mm.h>
 #include <xen/xmalloc.h>
-#include <asm/shadow.h>
+#include <xen/shadow.h>
+#include <xen/domain_page.h>
 #include <asm/page.h>
 #include <xen/event.h>
 #include <xen/trace.h>

xen/arch/x86/hvm/vmx/vmcs.c

@@ -34,12 +34,8 @@
 #include <asm/flushtlb.h>
 #include <xen/event.h>
 #include <xen/kernel.h>
-#include <asm/shadow.h>
 #include <xen/keyhandler.h>
-
-#if CONFIG_PAGING_LEVELS >= 3
-#include <asm/shadow_64.h>
-#endif
+#include <asm/shadow2.h>
 
 static int vmcs_size;
 static int vmcs_order;
@@ -238,7 +234,7 @@ static void vmx_set_host_env(struct vcpu *v)
 
 static void vmx_do_launch(struct vcpu *v)
 {
-/* Update CR3, GDT, LDT, TR */
+/* Update CR3, CR0, CR4, GDT, LDT, TR */
     unsigned int  error = 0;
     unsigned long cr0, cr4;
 
@@ -276,8 +272,11 @@ static void vmx_do_launch(struct vcpu *v)
     error |= __vmwrite(GUEST_TR_BASE, 0);
     error |= __vmwrite(GUEST_TR_LIMIT, 0xff);
 
-    __vmwrite(GUEST_CR3, pagetable_get_paddr(v->domain->arch.phys_table));
-    __vmwrite(HOST_CR3, pagetable_get_paddr(v->arch.monitor_table));
+    shadow2_update_paging_modes(v);
+    printk("%s(): GUEST_CR3<=%08lx, HOST_CR3<=%08lx\n",
+           __func__, v->arch.hvm_vcpu.hw_cr3, v->arch.cr3);
+    __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
+    __vmwrite(HOST_CR3, v->arch.cr3);
 
     v->arch.schedule_tail = arch_vmx_do_resume;
 

xen/arch/x86/hvm/vmx/vmx.c

@@ -26,9 +26,9 @@
 #include <xen/softirq.h>
 #include <xen/domain_page.h>
 #include <xen/hypercall.h>
+#include <xen/perfc.h>
 #include <asm/current.h>
 #include <asm/io.h>
-#include <asm/shadow.h>
 #include <asm/regs.h>
 #include <asm/cpufeature.h>
 #include <asm/processor.h>
@@ -40,10 +40,7 @@
 #include <asm/hvm/vmx/vmx.h>
 #include <asm/hvm/vmx/vmcs.h>
 #include <asm/hvm/vmx/cpu.h>
-#include <asm/shadow.h>
-#if CONFIG_PAGING_LEVELS >= 3
-#include <asm/shadow_64.h>
-#endif
+#include <asm/shadow2.h>
 #include <public/sched.h>
 #include <public/hvm/ioreq.h>
 #include <asm/hvm/vpic.h>
@@ -69,11 +66,16 @@ static int vmx_initialize_guest_resources(struct vcpu *v)
     if ( v->vcpu_id != 0 )
         return 1;
 
+    if ( !shadow2_mode_external(d) )
+    {
+        DPRINTK("Can't init HVM for dom %u vcpu %u: "
+                "not in shadow2 external mode\n", 
+                d->domain_id, v->vcpu_id);
+        domain_crash(d);
+    }
+
     for_each_vcpu ( d, vc )
     {
-        /* Initialize monitor page table */
-        vc->arch.monitor_table = pagetable_null();
-
         memset(&vc->arch.hvm_vmx, 0, sizeof(struct arch_vmx_struct));
 
         if ( (rc = vmx_create_vmcs(vc)) != 0 )
@@ -107,6 +109,7 @@ static int vmx_initialize_guest_resources(struct vcpu *v)
 
         vc->arch.hvm_vmx.io_bitmap_a = io_bitmap_a;
         vc->arch.hvm_vmx.io_bitmap_b = io_bitmap_b;
+
     }
 
     /*
@@ -116,11 +119,6 @@ static int vmx_initialize_guest_resources(struct vcpu *v)
     memset(&d->shared_info->evtchn_mask[0], 0xff,
            sizeof(d->shared_info->evtchn_mask));
 
-    /* Put the domain in shadow mode even though we're going to be using
-     * the shared 1:1 page table initially. It shouldn't hurt */
-    shadow_mode_enable(
-        d, SHM_enable|SHM_refcounts|SHM_translate|SHM_external|SHM_wr_pt_pte);
-
     return 1;
 }
 
@@ -133,7 +131,6 @@ static void vmx_relinquish_guest_resources(struct domain *d)
         vmx_destroy_vmcs(v);
         if ( !test_bit(_VCPUF_initialised, &v->vcpu_flags) )
             continue;
-        free_monitor_pagetable(v);
         kill_timer(&v->arch.hvm_vcpu.hlt_timer);
         if ( hvm_apic_support(v->domain) && (VLAPIC(v) != NULL) )
         {
@@ -153,8 +150,6 @@ static void vmx_relinquish_guest_resources(struct domain *d)
 
     if ( d->arch.hvm_domain.buffered_io_va )
         unmap_domain_page_global((void *)d->arch.hvm_domain.buffered_io_va);
-
-    shadow_direct_map_clean(d);
 }
 
 #ifdef __x86_64__
@@ -595,14 +590,6 @@ static void vmx_load_cpu_guest_regs(struct vcpu *v, struct cpu_user_regs *regs)
     vmx_vmcs_exit(v);
 }
 
-static int vmx_realmode(struct vcpu *v)
-{
-    unsigned long rflags;
-
-    __vmread(GUEST_RFLAGS, &rflags);
-    return rflags & X86_EFLAGS_VM;
-}
-
 static int vmx_instruction_length(struct vcpu *v)
 {
     unsigned long inst_len;
@@ -622,6 +609,8 @@ static unsigned long vmx_get_ctrl_reg(struct vcpu *v, unsigned int num)
         return v->arch.hvm_vmx.cpu_cr2;
     case 3:
         return v->arch.hvm_vmx.cpu_cr3;
+    case 4:
+        return v->arch.hvm_vmx.cpu_shadow_cr4;
     default:
         BUG();
     }
@@ -753,9 +742,13 @@ static void vmx_setup_hvm_funcs(void)
 
     hvm_funcs.realmode = vmx_realmode;
     hvm_funcs.paging_enabled = vmx_paging_enabled;
+    hvm_funcs.long_mode_enabled = vmx_long_mode_enabled;
+    hvm_funcs.guest_x86_mode = vmx_guest_x86_mode;
     hvm_funcs.instruction_length = vmx_instruction_length;
     hvm_funcs.get_guest_ctrl_reg = vmx_get_ctrl_reg;
 
+    hvm_funcs.update_host_cr3 = vmx_update_host_cr3;
+
     hvm_funcs.stts = vmx_stts;
     hvm_funcs.set_tsc_offset = vmx_set_tsc_offset;
 
@@ -855,53 +848,25 @@ static void inline __update_guest_eip(unsigned long inst_len)
     __vmwrite(GUEST_INTERRUPTIBILITY_INFO, 0);
 }
 
-
 static int vmx_do_page_fault(unsigned long va, struct cpu_user_regs *regs)
 {
-    unsigned long gpa; /* FIXME: PAE */
     int result;
 
 #if 0 /* keep for debugging */
     {
-        unsigned long eip;
+        unsigned long eip, cs;
 
+        __vmread(GUEST_CS_BASE, &cs);
         __vmread(GUEST_RIP, &eip);
         HVM_DBG_LOG(DBG_LEVEL_VMMU,
-                    "vmx_do_page_fault = 0x%lx, eip = %lx, error_code = %lx",
-                    va, eip, (unsigned long)regs->error_code);
+                    "vmx_do_page_fault = 0x%lx, cs_base=%lx, "
+                    "eip = %lx, error_code = %lx\n",
+                    va, cs, eip, (unsigned long)regs->error_code);
     }
 #endif
 
-    if ( !vmx_paging_enabled(current) )
-    {
-        /* construct 1-to-1 direct mapping */
-        if ( shadow_direct_map_fault(va, regs) ) 
-            return 1;
+    result = shadow2_fault(va, regs);
 
-        handle_mmio(va, va);
-        TRACE_VMEXIT (2,2);
-        return 1;
-    }
-    gpa = gva_to_gpa(va);
-
-    /* Use 1:1 page table to identify MMIO address space */
-    if ( mmio_space(gpa) ){
-        struct vcpu *v = current;
-        /* No support for APIC */
-        if (!hvm_apic_support(v->domain) && gpa >= 0xFEC00000) { 
-            u32 inst_len;
-            __vmread(VM_EXIT_INSTRUCTION_LEN, &(inst_len));
-            __update_guest_eip(inst_len);
-            return 1;
-        }
-        TRACE_VMEXIT (2,2);
-        /* in the case of MMIO, we are more interested in gpa than in va */
-        TRACE_VMEXIT (4,gpa);
-        handle_mmio(va, gpa);
-        return 1;
-    }
-
-    result = shadow_fault(va, regs);
     TRACE_VMEXIT (2,result);
 #if 0
     if ( !result )
@@ -972,23 +937,11 @@ static void vmx_vmexit_do_cpuid(struct cpu_user_regs *regs)
                 clear_bit(X86_FEATURE_APIC, &edx);
             }
     
-#if CONFIG_PAGING_LEVELS < 3
-            edx &= ~(bitmaskof(X86_FEATURE_PAE)  |
-                     bitmaskof(X86_FEATURE_PSE)  |
-                     bitmaskof(X86_FEATURE_PSE36));
-#else
-            if ( v->domain->arch.ops->guest_paging_levels == PAGING_L2 )
-            {
-                if ( v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
-                    clear_bit(X86_FEATURE_PSE36, &edx);
-                else
-                {
-                    clear_bit(X86_FEATURE_PAE, &edx);
-                    clear_bit(X86_FEATURE_PSE, &edx);
-                    clear_bit(X86_FEATURE_PSE36, &edx);
-                }
-            }
+#if CONFIG_PAGING_LEVELS >= 3
+            if ( !v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
 #endif
+                clear_bit(X86_FEATURE_PAE, &edx);
+            clear_bit(X86_FEATURE_PSE36, &edx);
 
             ebx &= NUM_THREADS_RESET_MASK;  
 
@@ -1086,7 +1039,7 @@ static void vmx_vmexit_do_invlpg(unsigned long va)
      * We do the safest things first, then try to update the shadow
      * copying from guest
      */
-    shadow_invlpg(v, va);
+    shadow2_invlpg(v, va);
 }
 
 
@@ -1307,11 +1260,8 @@ vmx_world_restore(struct vcpu *v, struct vmx_assist_context *c)
 
     error |= __vmwrite(CR0_READ_SHADOW, c->cr0);
 
-    if (!vmx_paging_enabled(v)) {
-        HVM_DBG_LOG(DBG_LEVEL_VMMU, "switching to vmxassist. use phys table");
-        __vmwrite(GUEST_CR3, pagetable_get_paddr(v->domain->arch.phys_table));
+    if (!vmx_paging_enabled(v))
         goto skip_cr3;
-    }
 
     if (c->cr3 == v->arch.hvm_vmx.cpu_cr3) {
         /*
@@ -1325,7 +1275,6 @@ vmx_world_restore(struct vcpu *v, struct vmx_assist_context *c)
             domain_crash_synchronous();
             return 0;
         }
-        shadow_sync_all(v->domain);
     } else {
         /*
          * If different, make a shadow. Check if the PDBR is valid
@@ -1348,13 +1297,17 @@ vmx_world_restore(struct vcpu *v, struct vmx_assist_context *c)
          * arch.shadow_table should now hold the next CR3 for shadow
          */
         v->arch.hvm_vmx.cpu_cr3 = c->cr3;
-        update_pagetables(v);
-        HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %x", c->cr3);
-        __vmwrite(GUEST_CR3, pagetable_get_paddr(v->arch.shadow_table));
     }
 
  skip_cr3:
 
+    shadow2_update_paging_modes(v);
+    if (!vmx_paging_enabled(v))
+        HVM_DBG_LOG(DBG_LEVEL_VMMU, "switching to vmxassist. use phys table");
+    else
+        HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %x", c->cr3);
+    __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
+
     error |= __vmread(CR4_READ_SHADOW, &old_cr4);
     error |= __vmwrite(GUEST_CR4, (c->cr4 | VMX_CR4_HOST_MASK));
     error |= __vmwrite(CR4_READ_SHADOW, c->cr4);
@@ -1485,6 +1438,7 @@ static int vmx_set_cr0(unsigned long value)
     int paging_enabled;
     unsigned long vm_entry_value;
     unsigned long old_cr0;
+    unsigned long old_base_mfn;
 
     /*
      * CR0: We don't want to lose PE and PG.
@@ -1514,7 +1468,8 @@ static int vmx_set_cr0(unsigned long value)
             v->arch.hvm_vmx.cpu_cr3 >> PAGE_SHIFT)) ||
              !get_page(mfn_to_page(mfn), v->domain) )
         {
-            printk("Invalid CR3 value = %lx", v->arch.hvm_vmx.cpu_cr3);
+            printk("Invalid CR3 value = %lx (mfn=%lx)\n", 
+                   v->arch.hvm_vmx.cpu_cr3, mfn);
             domain_crash_synchronous(); /* need to take a clean path */
         }
 
@@ -1539,51 +1494,22 @@ static int vmx_set_cr0(unsigned long value)
             __vmread(VM_ENTRY_CONTROLS, &vm_entry_value);
             vm_entry_value |= VM_ENTRY_CONTROLS_IA32E_MODE;
             __vmwrite(VM_ENTRY_CONTROLS, vm_entry_value);
-
-            if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L4) )
-            {
-                printk("Unsupported guest paging levels\n");
-                domain_crash_synchronous(); /* need to take a clean path */
-            }
         }
-        else
-#endif  /* __x86_64__ */
-        {
-#if CONFIG_PAGING_LEVELS >= 3
-            /* seems it's a 32-bit or 32-bit PAE guest */
-
-            if ( test_bit(VMX_CPU_STATE_PAE_ENABLED,
-                        &v->arch.hvm_vmx.cpu_state) )
-            {
-                /* The guest enables PAE first and then it enables PG, it is
-                 * really a PAE guest */
-                if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L3) )
-                {
-                    printk("Unsupported guest paging levels\n");
-                    domain_crash_synchronous();
-                }
-            }
-            else
-            {
-                if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L2) )
-                {
-                    printk("Unsupported guest paging levels\n");
-                    domain_crash_synchronous(); /* need to take a clean path */
-                }
-            }
 #endif
-        }
 
         /*
          * Now arch.guest_table points to machine physical.
          */
+        old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
         v->arch.guest_table = pagetable_from_pfn(mfn);
-        update_pagetables(v);
+        if (old_base_mfn)
+            put_page(mfn_to_page(old_base_mfn));
+        shadow2_update_paging_modes(v);
 
         HVM_DBG_LOG(DBG_LEVEL_VMMU, "New arch.guest_table = %lx",
                     (unsigned long) (mfn << PAGE_SHIFT));
 
-        __vmwrite(GUEST_CR3, pagetable_get_paddr(v->arch.shadow_table));
+        __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
         /*
          * arch->shadow_table should hold the next CR3 for shadow
          */
@@ -1625,7 +1551,6 @@ static int vmx_set_cr0(unsigned long value)
             }
         }
 
-        clear_all_shadow_status(v->domain);
         if ( vmx_assist(v, VMX_ASSIST_INVOKE) ) {
             set_bit(VMX_CPU_STATE_ASSIST_ENABLED, &v->arch.hvm_vmx.cpu_state);
             __vmread(GUEST_RIP, &eip);
@@ -1651,9 +1576,8 @@ static int vmx_set_cr0(unsigned long value)
     }
     else if ( (value & (X86_CR0_PE | X86_CR0_PG)) == X86_CR0_PE )
     {
-        /* we should take care of this kind of situation */
-        clear_all_shadow_status(v->domain);
-        __vmwrite(GUEST_CR3, pagetable_get_paddr(v->domain->arch.phys_table));
+        __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
+        shadow2_update_paging_modes(v);
     }
 
     return 1;
@@ -1738,7 +1662,7 @@ static int mov_to_cr(int gp, int cr, struct cpu_user_regs *regs)
             mfn = get_mfn_from_gpfn(value >> PAGE_SHIFT);
             if (mfn != pagetable_get_pfn(v->arch.guest_table))
                 __hvm_bug(regs);
-            shadow_sync_all(v->domain);
+            shadow2_update_cr3(v);
         } else {
             /*
              * If different, make a shadow. Check if the PDBR is valid
@@ -1759,16 +1683,11 @@ static int mov_to_cr(int gp, int cr, struct cpu_user_regs *regs)
             /*
              * arch.shadow_table should now hold the next CR3 for shadow
              */
-#if CONFIG_PAGING_LEVELS >= 3
-            if ( v->domain->arch.ops->guest_paging_levels == PAGING_L3 )
-                shadow_sync_all(v->domain);
-#endif
-
             v->arch.hvm_vmx.cpu_cr3 = value;
-            update_pagetables(v);
+            update_cr3(v);
             HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %lx",
                         value);
-            __vmwrite(GUEST_CR3, pagetable_get_paddr(v->arch.shadow_table));
+            __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
         }
         break;
     }
@@ -1786,12 +1705,6 @@ static int mov_to_cr(int gp, int cr, struct cpu_user_regs *regs)
 #if CONFIG_PAGING_LEVELS >= 3
                 unsigned long mfn, old_base_mfn;
 
-                if( !shadow_set_guest_paging_levels(v->domain, PAGING_L3) )
-                {
-                    printk("Unsupported guest paging levels\n");
-                    domain_crash_synchronous(); /* need to take a clean path */
-                }
-
                 if ( !VALID_MFN(mfn = get_mfn_from_gpfn(
                                     v->arch.hvm_vmx.cpu_cr3 >> PAGE_SHIFT)) ||
                      !get_page(mfn_to_page(mfn), v->domain) )
@@ -1800,21 +1713,20 @@ static int mov_to_cr(int gp, int cr, struct cpu_user_regs *regs)
                     domain_crash_synchronous(); /* need to take a clean path */
                 }
 
-                old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
-                if ( old_base_mfn )
-                    put_page(mfn_to_page(old_base_mfn));
 
                 /*
                  * Now arch.guest_table points to machine physical.
                  */
 
+                old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
                 v->arch.guest_table = pagetable_from_pfn(mfn);
-                update_pagetables(v);
+                if ( old_base_mfn )
+                    put_page(mfn_to_page(old_base_mfn));
 
                 HVM_DBG_LOG(DBG_LEVEL_VMMU, "New arch.guest_table = %lx",
                             (unsigned long) (mfn << PAGE_SHIFT));
 
-                __vmwrite(GUEST_CR3, pagetable_get_paddr(v->arch.shadow_table));
+                __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
 
                 /*
                  * arch->shadow_table should hold the next CR3 for shadow
@@ -1822,27 +1734,6 @@ static int mov_to_cr(int gp, int cr, struct cpu_user_regs *regs)
 
                 HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %lx, mfn = %lx",
                             v->arch.hvm_vmx.cpu_cr3, mfn);
-#endif
-            }
-            else
-            {
-                /*  The guest is a 64 bit or 32-bit PAE guest. */
-#if CONFIG_PAGING_LEVELS >= 3
-                if ( (v->domain->arch.ops != NULL) &&
-                        v->domain->arch.ops->guest_paging_levels == PAGING_L2)
-                {
-                    /* Seems the guest first enables PAE without enabling PG,
-                     * it must enable PG after that, and it is a 32-bit PAE
-                     * guest */
-
-                    if ( !shadow_set_guest_paging_levels(v->domain,
-                                                            PAGING_L3) )
-                    {
-                        printk("Unsupported guest paging levels\n");
-                        /* need to take a clean path */
-                        domain_crash_synchronous();
-                    }
-                }
 #endif
             }
         }
@@ -1864,8 +1755,7 @@ static int mov_to_cr(int gp, int cr, struct cpu_user_regs *regs)
          * all TLB entries except global entries.
          */
         if ( (old_cr ^ value) & (X86_CR4_PSE | X86_CR4_PGE | X86_CR4_PAE) )
-            shadow_sync_all(v->domain);
-
+            shadow2_update_paging_modes(v);
         break;
     }
     default:

xen/arch/x86/setup.c

@@ -532,8 +532,6 @@ void __init __start_xen(multiboot_info_t *mbi)
     if ( opt_watchdog ) 
         watchdog_enable();
 
-    shadow_mode_init();
-
     /* initialize access control security module */
     acm_init(&initrdidx, mbi, initial_images_start);
 

xen/arch/x86/shadow_guest32.c

@@ -1,16 +0,0 @@
-#define GUEST_PGENTRY_32
-
-#include "shadow.c"
-struct shadow_ops MODE_64_2_HANDLER = {
-    .guest_paging_levels        = 2,
-    .invlpg                     = shadow_invlpg_64,
-    .fault                      = shadow_fault_64,
-    .update_pagetables          = shadow_update_pagetables,
-    .sync_all                   = sync_all,
-    .remove_all_write_access    = remove_all_write_access,
-    .do_update_va_mapping       = do_update_va_mapping,
-    .mark_mfn_out_of_sync       = mark_mfn_out_of_sync,
-    .is_out_of_sync             = is_out_of_sync,
-    .gva_to_gpa                 = gva_to_gpa_64,
-};
-

xen/arch/x86/shadow_guest32pae.c

@@ -1,16 +0,0 @@
-#define GUEST_32PAE
-
-#include "shadow.c"
-struct shadow_ops MODE_64_PAE_HANDLER = {
-    .guest_paging_levels              = 3,
-    .invlpg                     = shadow_invlpg_64,
-    .fault                      = shadow_fault_64,
-    .update_pagetables          = shadow_update_pagetables,
-    .sync_all                   = sync_all,
-    .remove_all_write_access    = remove_all_write_access,
-    .do_update_va_mapping       = do_update_va_mapping,
-    .mark_mfn_out_of_sync       = mark_mfn_out_of_sync,
-    .is_out_of_sync             = is_out_of_sync,
-    .gva_to_gpa                 = gva_to_gpa_64,
-};
-

xen/arch/x86/smpboot.c

@@ -896,7 +896,7 @@ static int __devinit do_boot_cpu(int apicid, int cpu)
 	v = alloc_idle_vcpu(cpu);
 	BUG_ON(v == NULL);
 
-	v->arch.monitor_table = pagetable_from_paddr(__pa(idle_pg_table));
+	v->arch.cr3 = __pa(idle_pg_table);
 
 	/* start_eip had better be page-aligned! */
 	start_eip = setup_trampoline();

xen/arch/x86/traps.c

@@ -277,6 +277,21 @@ void show_stack(struct cpu_user_regs *regs)
     show_trace(regs);
 }
 
+void show_xen_trace()
+{
+    struct cpu_user_regs regs;
+#ifdef __x86_64
+    __asm__("movq %%rsp,%0" : "=m" (regs.rsp));
+    __asm__("movq %%rbp,%0" : "=m" (regs.rbp));
+    __asm__("leaq 0(%%rip),%0" : "=a" (regs.rip));
+#else
+    __asm__("movl %%esp,%0" : "=m" (regs.esp));
+    __asm__("movl %%ebp,%0" : "=m" (regs.ebp));
+    __asm__("call 1f; 1: popl %0" : "=a" (regs.eip));
+#endif
+    show_trace(&regs);
+}
+
 void show_stack_overflow(unsigned long esp)
 {
 #ifdef MEMORY_GUARD
@@ -861,8 +876,8 @@ static int fixup_page_fault(unsigned long addr, struct cpu_user_regs *regs)
 
     if ( unlikely(IN_HYPERVISOR_RANGE(addr)) )
     {
-        if ( shadow_mode_external(d) && guest_mode(regs) )
-            return shadow_fault(addr, regs);
+        if ( shadow2_mode_external(d) && guest_mode(regs) )
+            return shadow2_fault(addr, regs);
         if ( (addr >= GDT_LDT_VIRT_START) && (addr < GDT_LDT_VIRT_END) )
             return handle_gdt_ldt_mapping_fault(
                 addr - GDT_LDT_VIRT_START, regs);
@@ -873,15 +888,15 @@ static int fixup_page_fault(unsigned long addr, struct cpu_user_regs *regs)
         return (spurious_page_fault(addr, regs) ? EXCRET_not_a_fault : 0);
     }
 
-    if ( unlikely(shadow_mode_enabled(d)) )
-        return shadow_fault(addr, regs);
-
     if ( likely(VM_ASSIST(d, VMASST_TYPE_writable_pagetables)) &&
          guest_kernel_mode(v, regs) &&
          ((regs->error_code & (PGERR_write_access|PGERR_page_present)) ==
           (PGERR_write_access|PGERR_page_present)) )
         return ptwr_do_page_fault(d, addr, regs) ? EXCRET_fault_fixed : 0;
 
+    if ( shadow2_mode_enabled(d) )
+        return shadow2_fault(addr, regs);
+
     return 0;
 }
 
@@ -906,6 +921,13 @@ asmlinkage int do_page_fault(struct cpu_user_regs *regs)
 
     perfc_incrc(page_faults);
 
+    if ( shadow2_mode_enabled(current->domain) )
+        debugtrace_printk("%s %s %d dom=%d eip=%p cr2=%p code=%d cs=%x\n",
+                          __func__, __FILE__, __LINE__,
+                          current->domain->domain_id,
+                          (void *)regs->eip, (void *)addr, regs->error_code,
+                          regs->cs);
+
     if ( unlikely((rc = fixup_page_fault(addr, regs)) != 0) )
         return rc;
 

xen/arch/x86/x86_32/domain_page.c

@@ -15,6 +15,7 @@
 #include <asm/current.h>
 #include <asm/flushtlb.h>
 #include <asm/hardirq.h>
+#include <asm/hvm/support.h>
 
 static inline struct vcpu *mapcache_current_vcpu(void)
 {
@@ -58,10 +59,10 @@ void *map_domain_page(unsigned long pfn)
     cache = &v->domain->arch.mapcache;
 
     hashent = &cache->vcpu_maphash[vcpu].hash[MAPHASH_HASHFN(pfn)];
-    if ( hashent->pfn == pfn )
+    if ( hashent->pfn == pfn && (idx = hashent->idx) != MAPHASHENT_NOTINUSE )
     {
-        idx = hashent->idx;
         hashent->refcnt++;
+        ASSERT(idx < MAPCACHE_ENTRIES);
         ASSERT(hashent->refcnt != 0);
         ASSERT(l1e_get_pfn(cache->l1tab[idx]) == pfn);
         goto out;
@@ -178,6 +179,30 @@ void mapcache_init(struct domain *d)
                 MAPHASHENT_NOTINUSE;
 }
 
+paddr_t mapped_domain_page_to_maddr(void *va) 
+/* Convert a pointer in a mapped domain page to a machine address. 
+ * Takes any pointer that's valid for use in unmap_domain_page() */
+{
+    unsigned int idx;
+    struct vcpu *v;
+    struct mapcache *cache;
+    unsigned long pfn;
+
+    ASSERT(!in_irq());
+
+    ASSERT((void *)MAPCACHE_VIRT_START <= va);
+    ASSERT(va < (void *)MAPCACHE_VIRT_END);
+
+    v = mapcache_current_vcpu();
+
+    cache = &v->domain->arch.mapcache;
+
+    idx = ((unsigned long)va - MAPCACHE_VIRT_START) >> PAGE_SHIFT;
+    pfn = l1e_get_pfn(cache->l1tab[idx]);
+    return ((paddr_t) pfn << PAGE_SHIFT 
+            | ((unsigned long) va & ~PAGE_MASK));
+}
+
 #define GLOBALMAP_BITS (IOREMAP_MBYTES << (20 - PAGE_SHIFT))
 static unsigned long inuse[BITS_TO_LONGS(GLOBALMAP_BITS)];
 static unsigned long garbage[BITS_TO_LONGS(GLOBALMAP_BITS)];
@@ -233,6 +258,8 @@ void unmap_domain_page_global(void *va)
     l1_pgentry_t *pl1e;
     unsigned int idx;
 
+    ASSERT((__va >= IOREMAP_VIRT_START) && (__va <= (IOREMAP_VIRT_END - 1)));
+
     /* /First/, we zap the PTE. */
     pl2e = virt_to_xen_l2e(__va);
     pl1e = l2e_to_l1e(*pl2e) + l1_table_offset(__va);

xen/arch/x86/x86_32/mm.c

@@ -75,8 +75,7 @@ void __init paging_init(void)
     printk("PAE disabled.\n");
 #endif
 
-    idle_vcpu[0]->arch.monitor_table =
-        pagetable_from_paddr(__pa(idle_pg_table));
+    idle_vcpu[0]->arch.cr3 = __pa(idle_pg_table);
 
     if ( cpu_has_pge )
     {

xen/arch/x86/x86_64/mm.c

@@ -81,8 +81,7 @@ void __init paging_init(void)
     l2_pgentry_t *l2_ro_mpt;
     struct page_info *pg;
 
-    idle_vcpu[0]->arch.monitor_table =
-        pagetable_from_paddr(__pa(idle_pg_table));
+    idle_vcpu[0]->arch.cr3 = __pa(idle_pg_table);
 
     /* Create user-accessible L2 directory to map the MPT for guests. */
     l3_ro_mpt = alloc_xenheap_page();

xen/arch/x86/x86_64/traps.c

@@ -84,7 +84,8 @@ void show_page_walk(unsigned long addr)
     l4e = l4t[l4_table_offset(addr)];
     mfn = l4e_get_pfn(l4e);
     pfn = get_gpfn_from_mfn(mfn);
-    printk(" L4 = %"PRIpte" %016lx\n", l4e_get_intpte(l4e), pfn);
+    printk(" L4[0x%lx] = %"PRIpte" %016lx\n",
+           l4_table_offset(addr), l4e_get_intpte(l4e), pfn);
     if ( !(l4e_get_flags(l4e) & _PAGE_PRESENT) )
         return;
 
@@ -92,7 +93,8 @@ void show_page_walk(unsigned long addr)
     l3e = l3t[l3_table_offset(addr)];
     mfn = l3e_get_pfn(l3e);
     pfn = get_gpfn_from_mfn(mfn);
-    printk("  L3 = %"PRIpte" %016lx\n", l3e_get_intpte(l3e), pfn);
+    printk("  L3[0x%lx] = %"PRIpte" %016lx\n",
+           l3_table_offset(addr), l3e_get_intpte(l3e), pfn);
     if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
         return;
 
@@ -100,7 +102,8 @@ void show_page_walk(unsigned long addr)
     l2e = l2t[l2_table_offset(addr)];
     mfn = l2e_get_pfn(l2e);
     pfn = get_gpfn_from_mfn(mfn);
-    printk("   L2 = %"PRIpte" %016lx %s\n", l2e_get_intpte(l2e), pfn,
+    printk("   L2[0x%lx] = %"PRIpte" %016lx %s\n",
+           l2_table_offset(addr), l2e_get_intpte(l2e), pfn,
            (l2e_get_flags(l2e) & _PAGE_PSE) ? "(PSE)" : "");
     if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) ||
          (l2e_get_flags(l2e) & _PAGE_PSE) )
@@ -110,7 +113,8 @@ void show_page_walk(unsigned long addr)
     l1e = l1t[l1_table_offset(addr)];
     mfn = l1e_get_pfn(l1e);
     pfn = get_gpfn_from_mfn(mfn);
-    printk("    L1 = %"PRIpte" %016lx\n", l1e_get_intpte(l1e), pfn);
+    printk("    L1[0x%lx] = %"PRIpte" %016lx\n",
+           l1_table_offset(addr), l1e_get_intpte(l1e), pfn);
 }
 
 asmlinkage void double_fault(void);
@@ -162,7 +166,7 @@ void toggle_guest_mode(struct vcpu *v)
 {
     v->arch.flags ^= TF_kernel_mode;
     __asm__ __volatile__ ( "swapgs" );
-    update_pagetables(v);
+    update_cr3(v);
     write_ptbase(v);
 }
 

xen/common/acm_ops.c

@@ -26,7 +26,6 @@
 #include <xen/trace.h>
 #include <xen/console.h>
 #include <xen/guest_access.h>
-#include <asm/shadow.h>
 #include <public/sched_ctl.h>
 #include <acm/acm_hooks.h>
 

xen/common/grant_table.c

@@ -434,7 +434,7 @@ __gnttab_unmap_grant_ref(
 
     /* If just unmapped a writable mapping, mark as dirtied */
     if ( !(flags & GNTMAP_readonly) )
-         gnttab_log_dirty(rd, frame);
+         gnttab_mark_dirty(rd, frame);
 
     if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
          !(flags & GNTMAP_readonly) )
@@ -731,7 +731,7 @@ __release_grant_for_copy(
     const unsigned long r_frame = act->frame;
 
     if ( !readonly )
-        gnttab_log_dirty(rd, r_frame);
+        gnttab_mark_dirty(rd, r_frame);
 
     spin_lock(&rd->grant_table->lock);
     if ( readonly )

xen/common/keyhandler.c

@@ -241,9 +241,6 @@ static void read_clocks(unsigned char key)
 }
 
 extern void dump_runq(unsigned char key);
-#ifndef NDEBUG
-extern void audit_domains_key(unsigned char key);
-#endif
 
 #ifdef PERF_COUNTERS
 extern void perfc_printall(unsigned char key);
@@ -261,10 +258,16 @@ static void do_debug_key(unsigned char key, struct cpu_user_regs *regs)
 #ifndef NDEBUG
 static void debugtrace_key(unsigned char key)
 {
-    debugtrace_send_to_console = !debugtrace_send_to_console;
-    debugtrace_dump();
-    printk("debugtrace_printk now writing to %s.\n",
-           debugtrace_send_to_console ? "console" : "buffer");
+    debugtrace_toggle();
+}
+
+static void shadow2_audit_key(unsigned char key)
+{
+    extern int shadow2_audit_enable;
+
+    shadow2_audit_enable = !shadow2_audit_enable;
+    printk("%s shadow2_audit_enable=%d\n",
+           __func__, shadow2_audit_enable);
 }
 #endif
 
@@ -288,7 +291,7 @@ void initialize_keytable(void)
 
 #ifndef NDEBUG
     register_keyhandler(
-        'o', audit_domains_key,  "audit domains >0 EXPERIMENTAL");
+        'O', shadow2_audit_key,  "toggle shadow2 audits");
     register_keyhandler(
         'T', debugtrace_key, "toggle debugtrace to console/buffer");
 #endif

xen/common/memory.c

@@ -126,6 +126,11 @@ populate_physmap(
             for ( j = 0; j < (1 << extent_order); j++ )
                 guest_physmap_add_page(d, gpfn + j, mfn + j);
         }
+        else if ( unlikely(shadow2_mode_translate(d)) )
+        {
+            for ( j = 0; j < (1 << extent_order); j++ )
+                shadow2_guest_physmap_add_page(d, gpfn + j, mfn + j);
+        }
         else
         {
             for ( j = 0; j < (1 << extent_order); j++ )
@@ -153,7 +158,7 @@ guest_remove_page(
     if ( unlikely(!mfn_valid(mfn)) )
     {
         DPRINTK("Domain %u page number %lx invalid\n",
-                d->domain_id, mfn);
+                d->domain_id, gmfn);
         return 0;
     }
             
@@ -179,7 +184,7 @@ guest_remove_page(
                 (unsigned long)page->count_info, page->u.inuse.type_info);
     }
 
-    guest_physmap_remove_page(d, gmfn, mfn);
+    shadow2_guest_physmap_remove_page(d, gmfn, mfn);
 
     put_page(page);
 
@@ -250,7 +255,7 @@ translate_gpfn_list(
     if ( (d = find_domain_by_id(op.domid)) == NULL )
         return -ESRCH;
 
-    if ( !shadow_mode_translate(d) )
+    if ( !(shadow_mode_translate(d) || shadow2_mode_translate(d)) )
     {
         put_domain(d);
         return -EINVAL;

xen/drivers/char/console.c

@@ -569,7 +569,7 @@ int console_getc(void)
 #ifndef NDEBUG
 
 /* Send output direct to console, or buffer it? */
-int debugtrace_send_to_console;
+static volatile int debugtrace_send_to_console;
 
 static char        *debugtrace_buf; /* Debug-trace buffer */
 static unsigned int debugtrace_prd; /* Producer index     */
@@ -578,17 +578,11 @@ static unsigned int debugtrace_used;
 static DEFINE_SPINLOCK(debugtrace_lock);
 integer_param("debugtrace", debugtrace_kilobytes);
 
-void debugtrace_dump(void)
+static void debugtrace_dump_worker(void)
 {
-    unsigned long flags;
-
     if ( (debugtrace_bytes == 0) || !debugtrace_used )
         return;
 
-    watchdog_disable();
-
-    spin_lock_irqsave(&debugtrace_lock, flags);
-
     printk("debugtrace_dump() starting\n");
 
     /* Print oldest portion of the ring. */
@@ -602,15 +596,47 @@ void debugtrace_dump(void)
     memset(debugtrace_buf, '\0', debugtrace_bytes);
 
     printk("debugtrace_dump() finished\n");
+}
+
+void debugtrace_toggle(void)
+{
+    unsigned long flags;
+
+    watchdog_disable();
+    spin_lock_irqsave(&debugtrace_lock, flags);
+
+    // dump the buffer *before* toggling, in case the act of dumping the
+    // buffer itself causes more printk's...
+    //
+    printk("debugtrace_printk now writing to %s.\n",
+           !debugtrace_send_to_console ? "console": "buffer");
+    if ( !debugtrace_send_to_console )
+        debugtrace_dump_worker();
+
+    debugtrace_send_to_console = !debugtrace_send_to_console;
 
     spin_unlock_irqrestore(&debugtrace_lock, flags);
+    watchdog_enable();
 
+}
+
+void debugtrace_dump(void)
+{
+    unsigned long flags;
+
+    watchdog_disable();
+    spin_lock_irqsave(&debugtrace_lock, flags);
+
+    debugtrace_dump_worker();
+
+    spin_unlock_irqrestore(&debugtrace_lock, flags);
     watchdog_enable();
 }
 
 void debugtrace_printk(const char *fmt, ...)
 {
     static char    buf[1024];
+    static u32 count;
 
     va_list       args;
     char         *p;
@@ -625,8 +651,10 @@ void debugtrace_printk(const char *fmt, ...)
 
     ASSERT(debugtrace_buf[debugtrace_bytes - 1] == 0);
 
+    sprintf(buf, "%u ", ++count);
+
     va_start(args, fmt);
-    (void)vsnprintf(buf, sizeof(buf), fmt, args);
+    (void)vsnprintf(buf + strlen(buf), sizeof(buf), fmt, args);
     va_end(args);
 
     if ( debugtrace_send_to_console )

xen/include/asm-x86/bitops.h

@@ -75,6 +75,24 @@ static __inline__ void clear_bit(int nr, volatile void * addr)
 		:"=m" (ADDR)
 		:"dIr" (nr));
 }
+
+/**
+ * __clear_bit - Clears a bit in memory
+ * @nr: Bit to clear
+ * @addr: Address to start counting from
+ *
+ * Unlike clear_bit(), this function is non-atomic and may be reordered.
+ * If it's called on the same region of memory simultaneously, the effect
+ * may be that only one operation succeeds.
+ */
+static __inline__ void __clear_bit(int nr, volatile void * addr)
+{
+	__asm__(
+		"btrl %1,%0"
+		:"=m" (ADDR)
+		:"dIr" (nr));
+}
+
 #define smp_mb__before_clear_bit()	barrier()
 #define smp_mb__after_clear_bit()	barrier()
 

xen/include/asm-x86/config.h

@@ -79,9 +79,14 @@
 
 #ifndef __ASSEMBLY__
 extern unsigned long _end; /* standard ELF symbol */
-#endif /* __ASSEMBLY__ */
 
-#define FORCE_CRASH() __asm__ __volatile__ ( "ud2" )
+static inline void FORCE_CRASH(void) __attribute__((noreturn,always_inline)); 
+static inline void FORCE_CRASH(void) 
+{
+    __asm__ __volatile__ ( "ud2" );
+    while(1);
+}
+#endif /* __ASSEMBLY__ */
 
 #if defined(__x86_64__)
 
@@ -149,9 +154,14 @@ extern unsigned long _end; /* standard ELF symbol */
 /* Slot 256: read-only guest-accessible machine-to-phys translation table. */
 #define RO_MPT_VIRT_START       (PML4_ADDR(256))
 #define RO_MPT_VIRT_END         (RO_MPT_VIRT_START + PML4_ENTRY_BYTES/2)
+
+// current unused?
+#if 0
 /* Slot 257: read-only guest-accessible linear page table. */
 #define RO_LINEAR_PT_VIRT_START (PML4_ADDR(257))
 #define RO_LINEAR_PT_VIRT_END   (RO_LINEAR_PT_VIRT_START + PML4_ENTRY_BYTES)
+#endif
+
 /* Slot 258: linear page table (guest table). */
 #define LINEAR_PT_VIRT_START    (PML4_ADDR(258))
 #define LINEAR_PT_VIRT_END      (LINEAR_PT_VIRT_START + PML4_ENTRY_BYTES)
@@ -175,7 +185,7 @@ extern unsigned long _end; /* standard ELF symbol */
 #define DIRECTMAP_VIRT_START    (PML4_ADDR(262))
 #define DIRECTMAP_VIRT_END      (DIRECTMAP_VIRT_START + PML4_ENTRY_BYTES*2)
 
-#define PGT_base_page_table PGT_l4_page_table
+#define PGT_base_page_table     PGT_l4_page_table
 
 #define __HYPERVISOR_CS64 0xe010
 #define __HYPERVISOR_CS32 0xe008
@@ -274,9 +284,9 @@ extern unsigned long _end; /* standard ELF symbol */
     (L2_PAGETABLE_LAST_XEN_SLOT - L2_PAGETABLE_FIRST_XEN_SLOT + 1)
 
 #ifdef CONFIG_X86_PAE
-# define PGT_base_page_table PGT_l3_page_table
+# define PGT_base_page_table     PGT_l3_page_table
 #else
-# define PGT_base_page_table PGT_l2_page_table
+# define PGT_base_page_table     PGT_l2_page_table
 #endif
 
 #define __HYPERVISOR_CS 0xe008

xen/include/asm-x86/domain.h

@@ -73,42 +73,42 @@ struct arch_domain
     /* I/O-port admin-specified access capabilities. */
     struct rangeset *ioport_caps;
 
-    /* Shadow mode status and controls. */
-    struct shadow_ops *ops;
-    unsigned int shadow_mode;  /* flags to control shadow table operation */
-    unsigned int shadow_nest;  /* Recursive depth of shadow_lock() nesting */
-
-    /* shadow hashtable */
-    struct shadow_status *shadow_ht;
-    struct shadow_status *shadow_ht_free;
-    struct shadow_status *shadow_ht_extras; /* extra allocation units */
-    unsigned int shadow_extras_count;
-
-    /* shadow dirty bitmap */
-    unsigned long *shadow_dirty_bitmap;
-    unsigned int shadow_dirty_bitmap_size;  /* in pages, bit per page */
-
-    /* shadow mode stats */
-    unsigned int shadow_page_count;
-    unsigned int hl2_page_count;
-    unsigned int snapshot_page_count;
-
-    unsigned int shadow_fault_count;
-    unsigned int shadow_dirty_count;
-
-    /* full shadow mode */
-    struct out_of_sync_entry *out_of_sync; /* list of out-of-sync pages */
-    struct out_of_sync_entry *out_of_sync_free;
-    struct out_of_sync_entry *out_of_sync_extras;
-    unsigned int out_of_sync_extras_count;
-
-    struct list_head free_shadow_frames;
-
-    pagetable_t         phys_table;         /* guest 1:1 pagetable */
+    /* HVM stuff */
     struct hvm_domain   hvm_domain;
 
     /* Shadow-translated guest: Pseudophys base address of reserved area. */
     unsigned long first_reserved_pfn;
+
+    /* Shadow2 stuff */
+    u32               shadow2_mode;  /* flags to control shadow operation */
+    spinlock_t        shadow2_lock;  /* shadow2 domain lock */
+    int               shadow2_locker; /* processor which holds the lock */
+    const char       *shadow2_locker_function; /* Func that took it */
+    struct list_head  shadow2_freelists[SHADOW2_MAX_ORDER + 1]; 
+    struct list_head  shadow2_p2m_freelist;
+    struct list_head  shadow2_p2m_inuse;
+    struct list_head  shadow2_toplevel_shadows;
+    unsigned int      shadow2_total_pages;  /* number of pages allocated */
+    unsigned int      shadow2_free_pages;   /* number of pages on freelists */
+    unsigned int      shadow2_p2m_pages;    /* number of pages in p2m map */
+
+    /* Shadow2 hashtable */
+    struct shadow2_hash_entry *shadow2_hash_table;
+    struct shadow2_hash_entry *shadow2_hash_freelist;
+    struct shadow2_hash_entry *shadow2_hash_allocations;
+    int shadow2_hash_walking;  /* Some function is walking the hash table */
+
+    /* Shadow log-dirty bitmap */
+    unsigned long *shadow_dirty_bitmap;
+    unsigned int shadow_dirty_bitmap_size;  /* in pages, bit per page */
+
+    /* Shadow log-dirty mode stats */
+    unsigned int shadow_fault_count;
+    unsigned int shadow_dirty_count;
+
+    /* Shadow translated domain: P2M mapping */
+    pagetable_t phys_table;
+
 } __cacheline_aligned;
 
 #ifdef CONFIG_X86_PAE
@@ -166,25 +166,34 @@ struct arch_vcpu
      */
     l1_pgentry_t *perdomain_ptes;
 
-    pagetable_t  guest_table_user;      /* x86/64: user-space pagetable. */
-    pagetable_t  guest_table;           /* (MA) guest notion of cr3 */
-    pagetable_t  shadow_table;          /* (MA) shadow of guest */
-    pagetable_t  monitor_table;         /* (MA) used in hypervisor */
-
-    l2_pgentry_t *guest_vtable;         /* virtual address of pagetable */
-    l2_pgentry_t *shadow_vtable;        /* virtual address of shadow_table */
-    l2_pgentry_t *monitor_vtable;		/* virtual address of monitor_table */
-    l1_pgentry_t *hl2_vtable;			/* virtual address of hl2_table */
-
 #ifdef CONFIG_X86_64
-    l3_pgentry_t *guest_vl3table;
-    l4_pgentry_t *guest_vl4table;
+    pagetable_t guest_table_user;       /* (MFN) x86/64 user-space pagetable */
 #endif
+    pagetable_t guest_table;            /* (MFN) guest notion of cr3 */
+    /* guest_table holds a ref to the page, and also a type-count unless
+     * shadow refcounts are in use */
+    pagetable_t shadow_table;           /* (MFN) shadow of guest */
+    pagetable_t monitor_table;          /* (MFN) hypervisor PT (for HVM) */
+    unsigned long cr3;           	    /* (MA) value to install in HW CR3 */
 
-    unsigned long monitor_shadow_ref;
+    void *guest_vtable;                 /* virtual address of pagetable */
+    void *shadow_vtable;                /* virtual address of shadow_table */
+    root_pgentry_t *monitor_vtable;		/* virtual address of monitor_table */
 
     /* Current LDT details. */
     unsigned long shadow_ldt_mapcnt;
+
+    /* Shadow2 stuff */
+    /* -- pointers to mode-specific entry points */
+    struct shadow2_entry_points *shadow2; 
+    unsigned long last_emulated_mfn;    /* last mfn we emulated a write to */
+    u8 shadow2_propagate_fault;         /* emulated fault needs to be */
+                                        /* propagated to guest */
+#if CONFIG_PAGING_LEVELS >= 3
+    u8 shadow2_pae_flip_pending;        /* shadow update requires this PAE cpu
+                                         * to recopy/install its L3 table.
+                                         */
+#endif
 } __cacheline_aligned;
 
 /* shorthands to improve code legibility */

xen/include/asm-x86/grant_table.h

@@ -31,7 +31,7 @@ int destroy_grant_host_mapping(
 #define gnttab_shared_gmfn(d, t, i)                     \
     (mfn_to_gmfn(d, gnttab_shared_mfn(d, t, i)))
 
-#define gnttab_log_dirty(d, f) mark_dirty((d), (f))
+#define gnttab_mark_dirty(d, f) mark_dirty((d), (f))
 
 static inline void gnttab_clear_flag(unsigned long nr, uint16_t *addr)
 {

xen/include/asm-x86/hvm/hvm.h

@@ -56,9 +56,16 @@ struct hvm_function_table {
      */
     int (*realmode)(struct vcpu *v);
     int (*paging_enabled)(struct vcpu *v);
+    int (*long_mode_enabled)(struct vcpu *v);
+    int (*guest_x86_mode)(struct vcpu *v);
     int (*instruction_length)(struct vcpu *v);
     unsigned long (*get_guest_ctrl_reg)(struct vcpu *v, unsigned int num);
 
+    /* 
+     * Re-set the value of CR3 that Xen runs on when handling VM exits
+     */
+    void (*update_host_cr3)(struct vcpu *v);
+
     /*
      * Update specifics of the guest state:
      * 1) TS bit in guest cr0 
@@ -133,12 +140,30 @@ hvm_paging_enabled(struct vcpu *v)
     return hvm_funcs.paging_enabled(v);
 }
 
+static inline int
+hvm_long_mode_enabled(struct vcpu *v)
+{
+    return hvm_funcs.long_mode_enabled(v);
+}
+
+static inline int
+hvm_guest_x86_mode(struct vcpu *v)
+{
+    return hvm_funcs.guest_x86_mode(v);
+}
+
 static inline int
 hvm_instruction_length(struct vcpu *v)
 {
     return hvm_funcs.instruction_length(v);
 }
 
+static inline void
+hvm_update_host_cr3(struct vcpu *v)
+{
+    hvm_funcs.update_host_cr3(v);
+}
+
 void hvm_hypercall_page_initialise(struct domain *d,
                                    void *hypercall_page);
 

xen/include/asm-x86/hvm/support.h

@@ -116,10 +116,13 @@ enum hval_bitmaps {
 #define DBG_LEVEL_IOAPIC            (1 << 9)
 
 extern unsigned int opt_hvm_debug_level;
-#define HVM_DBG_LOG(level, _f, _a...)           \
-    if ( (level) & opt_hvm_debug_level )        \
-        printk("[HVM:%d.%d] <%s> " _f "\n",     \
-               current->domain->domain_id, current->vcpu_id, __func__, ## _a)
+#define HVM_DBG_LOG(level, _f, _a...)                                         \
+    do {                                                                      \
+        if ( (level) & opt_hvm_debug_level )                                  \
+            printk("[HVM:%d.%d] <%s> " _f "\n",                               \
+                   current->domain->domain_id, current->vcpu_id, __func__,    \
+                   ## _a);                                                    \
+    } while (0)
 #else
 #define HVM_DBG_LOG(level, _f, _a...)
 #endif

xen/include/asm-x86/hvm/vcpu.h

@@ -29,6 +29,7 @@
 #define HVM_VCPU_INIT_SIPI_SIPI_STATE_WAIT_SIPI     1
 
 struct hvm_vcpu {
+    unsigned long       hw_cr3;     /* value we give to HW to use */
     unsigned long       ioflags;
     struct hvm_io_op    io_op;
     struct vlapic       *vlapic;
@@ -40,6 +41,11 @@ struct hvm_vcpu {
 
     int                 xen_port;
 
+#if CONFIG_PAGING_LEVELS >= 3
+    l3_pgentry_t hvm_lowmem_l3tab[4]
+    __attribute__((__aligned__(32)));
+#endif
+
     /* Flags */
     int                 flag_dr_dirty;
 

xen/include/asm-x86/hvm/vmx/vmcs.h

@@ -87,6 +87,7 @@ struct arch_vmx_struct {
 
     unsigned long        cpu_cr0; /* copy of guest CR0 */
     unsigned long        cpu_shadow_cr0; /* copy of guest read shadow CR0 */
+    unsigned long        cpu_shadow_cr4; /* copy of guest read shadow CR4 */
     unsigned long        cpu_cr2; /* save CR2 */
     unsigned long        cpu_cr3;
     unsigned long        cpu_state;

xen/include/asm-x86/hvm/vmx/vmx.h

@@ -298,6 +298,9 @@ static always_inline void __vmwrite_vcpu(
     case GUEST_CR0:
         v->arch.hvm_vmx.cpu_cr0 = value;
         break;
+    case CR4_READ_SHADOW:
+        v->arch.hvm_vmx.cpu_shadow_cr4 = value;
+        break;
     case CPU_BASED_VM_EXEC_CONTROL:
         v->arch.hvm_vmx.cpu_based_exec_control = value;
         break;
@@ -317,11 +320,14 @@ static always_inline void __vmread_vcpu(
     case GUEST_CR0:
         *value = v->arch.hvm_vmx.cpu_cr0;
         break;
+    case CR4_READ_SHADOW:
+        *value = v->arch.hvm_vmx.cpu_shadow_cr4;
+        break;
     case CPU_BASED_VM_EXEC_CONTROL:
         *value = v->arch.hvm_vmx.cpu_based_exec_control;
         break;
     default:
-        printk("__vmread_cpu: invalid field %lx\n", field);
+        printk("__vmread_vcpu: invalid field %lx\n", field);
         break;
     }
 }
@@ -342,6 +348,7 @@ static inline int __vmwrite(unsigned long field, unsigned long value)
     switch ( field ) {
     case CR0_READ_SHADOW:
     case GUEST_CR0:
+    case CR4_READ_SHADOW:
     case CPU_BASED_VM_EXEC_CONTROL:
         __vmwrite_vcpu(v, field, value);
         break;
@@ -404,6 +411,46 @@ static inline int vmx_paging_enabled(struct vcpu *v)
     return (cr0 & X86_CR0_PE) && (cr0 & X86_CR0_PG);
 }
 
+/* Works only for vcpu == current */
+static inline int vmx_long_mode_enabled(struct vcpu *v)
+{
+    ASSERT(v == current);
+    return VMX_LONG_GUEST(current);
+}
+
+/* Works only for vcpu == current */
+static inline int vmx_realmode(struct vcpu *v)
+{
+    unsigned long rflags;
+    ASSERT(v == current);
+
+    __vmread(GUEST_RFLAGS, &rflags);
+    return rflags & X86_EFLAGS_VM;
+}
+
+/* Works only for vcpu == current */
+static inline void vmx_update_host_cr3(struct vcpu *v)
+{
+    ASSERT(v == current);
+    __vmwrite(HOST_CR3, v->arch.cr3);
+}
+
+static inline int vmx_guest_x86_mode(struct vcpu *v)
+{
+    unsigned long cs_ar_bytes;
+    ASSERT(v == current);
+
+    if ( vmx_long_mode_enabled(v) )
+    {
+        __vmread(GUEST_CS_AR_BYTES, &cs_ar_bytes);
+        return (cs_ar_bytes & (1u<<13)) ? 8 : 4;
+    }
+    if ( vmx_realmode(v) )
+        return 2;
+    __vmread(GUEST_CS_AR_BYTES, &cs_ar_bytes);
+    return (cs_ar_bytes & (1u<<14)) ? 4 : 2;
+}
+
 static inline int vmx_pgbit_test(struct vcpu *v)
 {
     unsigned long cr0;

xen/include/asm-x86/mm.h

@@ -20,7 +20,11 @@
 struct page_info
 {
     /* Each frame can be threaded onto a doubly-linked list. */
-    struct list_head list;
+    union {
+        struct list_head list;
+        /* Shadow2 uses this field as an up-pointer in lower-level shadows */
+        paddr_t up;
+    };
 
     /* Reference count and various PGC_xxx flags and fields. */
     u32 count_info;
@@ -46,8 +50,20 @@ struct page_info
 
     } u;
 
-    /* Timestamp from 'TLB clock', used to reduce need for safety flushes. */
-    u32 tlbflush_timestamp;
+    union {
+        /* Timestamp from 'TLB clock', used to reduce need for safety
+         * flushes.  Only valid on a) free pages, and b) guest pages with a
+         * zero type count. */
+        u32 tlbflush_timestamp;
+
+        /* Only used on guest pages with a shadow.
+         * Guest pages with a shadow must have a non-zero type count, so this
+         * does not conflict with the tlbflush timestamp. */
+        u32 shadow2_flags;
+
+        // XXX -- we expect to add another field here, to be used for min/max
+        // purposes, which is only used for shadow pages.
+    };
 };
 
  /* The following page types are MUTUALLY EXCLUSIVE. */
@@ -60,6 +76,7 @@ struct page_info
 #define PGT_ldt_page        (6U<<29) /* using this page in an LDT? */
 #define PGT_writable_page   (7U<<29) /* has writable mappings of this page? */
 
+#ifndef SHADOW2
 #define PGT_l1_shadow       PGT_l1_page_table
 #define PGT_l2_shadow       PGT_l2_page_table
 #define PGT_l3_shadow       PGT_l3_page_table
@@ -69,14 +86,16 @@ struct page_info
 #define PGT_writable_pred   (7U<<29) /* predicted gpfn with writable ref */
 
 #define PGT_fl1_shadow      (5U<<29)
+#endif
+
 #define PGT_type_mask       (7U<<29) /* Bits 29-31. */
 
- /* Has this page been validated for use as its current type? */
-#define _PGT_validated      28
-#define PGT_validated       (1U<<_PGT_validated)
  /* Owning guest has pinned this page to its current type? */
-#define _PGT_pinned         27
+#define _PGT_pinned         28
 #define PGT_pinned          (1U<<_PGT_pinned)
+ /* Has this page been validated for use as its current type? */
+#define _PGT_validated      27
+#define PGT_validated       (1U<<_PGT_validated)
 #if defined(__i386__)
  /* The 11 most significant bits of virt address if this is a page table. */
 #define PGT_va_shift        16
@@ -98,6 +117,7 @@ struct page_info
  /* 16-bit count of uses of this frame as its current type. */
 #define PGT_count_mask      ((1U<<16)-1)
 
+#ifndef SHADOW2
 #ifdef __x86_64__
 #define PGT_high_mfn_shift  52
 #define PGT_high_mfn_mask   (0xfffUL << PGT_high_mfn_shift)
@@ -112,19 +132,53 @@ struct page_info
 #define PGT_score_shift     23
 #define PGT_score_mask      (((1U<<4)-1)<<PGT_score_shift)
 #endif
+#endif /* SHADOW2 */
 
  /* Cleared when the owning guest 'frees' this page. */
 #define _PGC_allocated      31
 #define PGC_allocated       (1U<<_PGC_allocated)
- /* Set when fullshadow mode marks a page out-of-sync */
+ /* Set on a *guest* page to mark it out-of-sync with its shadow */
 #define _PGC_out_of_sync     30
 #define PGC_out_of_sync     (1U<<_PGC_out_of_sync)
- /* Set when fullshadow mode is using a page as a page table */
+ /* Set when is using a page as a page table */
 #define _PGC_page_table      29
 #define PGC_page_table      (1U<<_PGC_page_table)
  /* 29-bit count of references to this frame. */
 #define PGC_count_mask      ((1U<<29)-1)
 
+/* shadow2 uses the count_info on shadow pages somewhat differently */
+/* NB: please coordinate any changes here with the SH2F's in shadow2.h */
+#define PGC_SH2_none           (0U<<28) /* on the shadow2 free list */
+#define PGC_SH2_min_shadow     (1U<<28)
+#define PGC_SH2_l1_32_shadow   (1U<<28) /* shadowing a 32-bit L1 guest page */
+#define PGC_SH2_fl1_32_shadow  (2U<<28) /* L1 shadow for a 32b 4M superpage */
+#define PGC_SH2_l2_32_shadow   (3U<<28) /* shadowing a 32-bit L2 guest page */
+#define PGC_SH2_l1_pae_shadow  (4U<<28) /* shadowing a pae L1 page */
+#define PGC_SH2_fl1_pae_shadow (5U<<28) /* L1 shadow for pae 2M superpg */
+#define PGC_SH2_l2_pae_shadow  (6U<<28) /* shadowing a pae L2-low page */
+#define PGC_SH2_l2h_pae_shadow (7U<<28) /* shadowing a pae L2-high page */
+#define PGC_SH2_l3_pae_shadow  (8U<<28) /* shadowing a pae L3 page */
+#define PGC_SH2_l1_64_shadow   (9U<<28) /* shadowing a 64-bit L1 page */
+#define PGC_SH2_fl1_64_shadow (10U<<28) /* L1 shadow for 64-bit 2M superpg */
+#define PGC_SH2_l2_64_shadow  (11U<<28) /* shadowing a 64-bit L2 page */
+#define PGC_SH2_l3_64_shadow  (12U<<28) /* shadowing a 64-bit L3 page */
+#define PGC_SH2_l4_64_shadow  (13U<<28) /* shadowing a 64-bit L4 page */
+#define PGC_SH2_max_shadow    (13U<<28)
+#define PGC_SH2_p2m_table     (14U<<28) /* in use as the p2m table */
+#define PGC_SH2_monitor_table (15U<<28) /* in use as a monitor table */
+#define PGC_SH2_unused        (15U<<28)
+
+#define PGC_SH2_type_mask     (15U<<28)
+#define PGC_SH2_type_shift          28
+
+#define PGC_SH2_pinned         (1U<<27)
+
+#define _PGC_SH2_log_dirty          26
+#define PGC_SH2_log_dirty      (1U<<26)
+
+/* 26 bit ref count for shadow pages */
+#define PGC_SH2_count_mask    ((1U<<26) - 1)
+
 /* We trust the slab allocator in slab.c, and our use of it. */
 #define PageSlab(page)	    (1)
 #define PageSetSlab(page)   ((void)0)
@@ -134,16 +188,24 @@ struct page_info
 
 #if defined(__i386__)
 #define pickle_domptr(_d)   ((u32)(unsigned long)(_d))
-#define unpickle_domptr(_d) ((struct domain *)(unsigned long)(_d))
+static inline struct domain *unpickle_domptr(u32 _domain)
+{ return (_domain & 1) ? NULL : (void *)_domain; }
 #define PRtype_info "08lx" /* should only be used for printk's */
 #elif defined(__x86_64__)
 static inline struct domain *unpickle_domptr(u32 _domain)
-{ return (_domain == 0) ? NULL : __va(_domain); }
+{ return ((_domain == 0) || (_domain & 1)) ? NULL : __va(_domain); }
 static inline u32 pickle_domptr(struct domain *domain)
 { return (domain == NULL) ? 0 : (u32)__pa(domain); }
 #define PRtype_info "016lx"/* should only be used for printk's */
 #endif
 
+/* The order of the largest allocation unit we use for shadow pages */
+#if CONFIG_PAGING_LEVELS == 2
+#define SHADOW2_MAX_ORDER 0 /* Only ever need 4k allocations */
+#else  
+#define SHADOW2_MAX_ORDER 2 /* Need up to 16k allocs for 32-bit on PAE/64 */
+#endif
+
 #define page_get_owner(_p)    (unpickle_domptr((_p)->u.inuse._domain))
 #define page_set_owner(_p,_d) ((_p)->u.inuse._domain = pickle_domptr(_d))
 
@@ -165,7 +227,7 @@ extern void invalidate_shadow_ldt(struct vcpu *d);
 extern int shadow_remove_all_write_access(
     struct domain *d, unsigned long gmfn, unsigned long mfn);
 extern u32 shadow_remove_all_access( struct domain *d, unsigned long gmfn);
-extern int _shadow_mode_refcounts(struct domain *d);
+extern int _shadow2_mode_refcounts(struct domain *d);
 
 static inline void put_page(struct page_info *page)
 {
@@ -197,8 +259,8 @@ static inline int get_page(struct page_info *page,
              unlikely((nx & PGC_count_mask) == 0) || /* Count overflow? */
              unlikely(d != _domain) )                /* Wrong owner? */
         {
-            if ( !_shadow_mode_refcounts(domain) )
-                DPRINTK("Error pfn %lx: rd=%p, od=%p, caf=%08x, taf=%"
+            if ( !_shadow2_mode_refcounts(domain) )
+                DPRINTK("Error pfn %lx: rd=%p, od=%p, caf=%08x, taf=%" 
                         PRtype_info "\n",
                         page_to_mfn(page), domain, unpickle_domptr(d),
                         x, page->u.inuse.type_info);
@@ -254,6 +316,16 @@ static inline int page_is_removable(struct page_info *page)
     ASSERT(((_p)->count_info & PGC_count_mask) != 0);          \
     ASSERT(page_get_owner(_p) == (_d))
 
+// Quick test for whether a given page can be represented directly in CR3.
+//
+#if CONFIG_PAGING_LEVELS == 3
+#define MFN_FITS_IN_CR3(_MFN) !(mfn_x(_MFN) >> 20)
+
+/* returns a lowmem machine address of the copied L3 root table */
+unsigned long
+pae_copy_root(struct vcpu *v, l3_pgentry_t *l3tab);
+#endif /* CONFIG_PAGING_LEVELS == 3 */
+
 int check_descriptor(struct desc_struct *d);
 
 /*
@@ -271,29 +343,44 @@ int check_descriptor(struct desc_struct *d);
 #define set_gpfn_from_mfn(mfn, pfn) (machine_to_phys_mapping[(mfn)] = (pfn))
 #define get_gpfn_from_mfn(mfn)      (machine_to_phys_mapping[(mfn)])
 
+
+#define mfn_to_gmfn(_d, mfn)                            \
+    ( (shadow2_mode_translate(_d))                      \
+      ? get_gpfn_from_mfn(mfn)                          \
+      : (mfn) )
+
+#define gmfn_to_mfn(_d, gpfn)  mfn_x(sh2_gfn_to_mfn(_d, gpfn))
+
+
 /*
  * The phys_to_machine_mapping is the reversed mapping of MPT for full
  * virtualization.  It is only used by shadow_mode_translate()==true
  * guests, so we steal the address space that would have normally
  * been used by the read-only MPT map.
  */
-#define phys_to_machine_mapping ((unsigned long *)RO_MPT_VIRT_START)
-#define NR_P2M_TABLE_ENTRIES    ((unsigned long *)RO_MPT_VIRT_END \
-                                 - phys_to_machine_mapping)
+#define phys_to_machine_mapping ((l1_pgentry_t *)RO_MPT_VIRT_START)
 #define INVALID_MFN             (~0UL)
 #define VALID_MFN(_mfn)         (!((_mfn) & (1U<<31)))
 
-#define set_mfn_from_gpfn(pfn, mfn) (phys_to_machine_mapping[(pfn)] = (mfn))
 static inline unsigned long get_mfn_from_gpfn(unsigned long pfn)
 {
-    unsigned long mfn;
+    l1_pgentry_t l1e = l1e_empty();
+    int ret;
 
-    if ( unlikely(pfn >= NR_P2M_TABLE_ENTRIES) ||
-         unlikely(__copy_from_user(&mfn, &phys_to_machine_mapping[pfn],
-                                   sizeof(mfn))) )
-	mfn = INVALID_MFN;
+#if CONFIG_PAGING_LEVELS > 2
+    if ( pfn > (RO_MPT_VIRT_END - RO_MPT_VIRT_START) / sizeof (l1_pgentry_t) ) 
+        /* This pfn is higher than the p2m map can hold */
+        return INVALID_MFN;
+#endif
 
-    return mfn;
+    ret = __copy_from_user(&l1e,
+                               &phys_to_machine_mapping[pfn],
+                               sizeof(l1e));
+
+    if ( (ret == 0) && (l1e_get_flags(l1e) & _PAGE_PRESENT) )
+        return l1e_get_pfn(l1e);
+
+    return INVALID_MFN;
 }
 
 #ifdef MEMORY_GUARD
@@ -333,6 +420,7 @@ void audit_domains(void);
 #endif
 
 int new_guest_cr3(unsigned long pfn);
+void make_cr3(struct vcpu *v, unsigned long mfn);
 
 void propagate_page_fault(unsigned long addr, u16 error_code);
 

xen/include/asm-x86/msr.h

@@ -112,6 +112,10 @@ static inline void wrmsrl(unsigned int msr, __u64 val)
 #define MSR_IA32_VMX_EXIT_CTLS_MSR              0x483
 #define MSR_IA32_VMX_ENTRY_CTLS_MSR             0x484
 #define MSR_IA32_VMX_MISC_MSR                   0x485
+#define MSR_IA32_VMX_CR0_FIXED0                 0x486
+#define MSR_IA32_VMX_CR0_FIXED1                 0x487
+#define MSR_IA32_VMX_CR4_FIXED0                 0x488
+#define MSR_IA32_VMX_CR4_FIXED1                 0x489
 #define IA32_FEATURE_CONTROL_MSR                0x3a
 #define IA32_FEATURE_CONTROL_MSR_LOCK           0x1
 #define IA32_FEATURE_CONTROL_MSR_ENABLE_VMXON   0x4

xen/include/asm-x86/page-guest32.h

@@ -89,15 +89,8 @@ static inline l2_pgentry_32_t l2e_from_paddr_32(paddr_t pa, unsigned int flags)
 
 #define linear_l1_table_32                                                 \
     ((l1_pgentry_32_t *)(LINEAR_PT_VIRT_START))
-#define __linear_l2_table_32                                                 \
-    ((l2_pgentry_32_t *)(LINEAR_PT_VIRT_START +                            \
-                     (LINEAR_PT_VIRT_START >> (PAGETABLE_ORDER<<0))))
 
 #define linear_pg_table_32 linear_l1_table_32
-#define linear_l2_table_32(_ed) ((_ed)->arch.guest_vtable)
-
-#define va_to_l1mfn_32(_ed, _va) \
-    (l2e_get_pfn(linear_l2_table(_ed)[_va>>L2_PAGETABLE_SHIFT]))
 
 #endif /* __X86_PAGE_GUEST_H__ */
 

xen/include/asm-x86/page.h

@@ -233,26 +233,18 @@ typedef struct { u64 pfn; } pagetable_t;
      + DOMAIN_ENTRIES_PER_L4_PAGETABLE)
 #endif
 
-#define LINEAR_PT_OFFSET (LINEAR_PT_VIRT_START & VADDR_MASK)
-#define linear_l1_table                                             \
-    ((l1_pgentry_t *)(LINEAR_PT_VIRT_START))
-#define __linear_l2_table                                           \
-    ((l2_pgentry_t *)(LINEAR_PT_VIRT_START +                        \
-                     (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<0))))
-#define __linear_l3_table                                           \
-    ((l3_pgentry_t *)(LINEAR_PT_VIRT_START +                        \
-                     (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<0)) +   \
-                     (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<1))))
-#define __linear_l4_table                                           \
-    ((l4_pgentry_t *)(LINEAR_PT_VIRT_START +                        \
-                     (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<0)) +   \
-                     (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<1)) +   \
-                     (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<2))))
+/* Where to find each level of the linear mapping */
+#define __linear_l1_table ((l1_pgentry_t *)(LINEAR_PT_VIRT_START))
+#define __linear_l2_table \
+ ((l2_pgentry_t *)(__linear_l1_table + l1_linear_offset(LINEAR_PT_VIRT_START)))
+#define __linear_l3_table \
+ ((l3_pgentry_t *)(__linear_l2_table + l2_linear_offset(LINEAR_PT_VIRT_START)))
+#define __linear_l4_table \
+ ((l4_pgentry_t *)(__linear_l3_table + l3_linear_offset(LINEAR_PT_VIRT_START)))
 
+#define linear_l1_table __linear_l1_table
 #define linear_pg_table linear_l1_table
-#define linear_l2_table(v) ((v)->arch.guest_vtable)
-#define linear_l3_table(v) ((v)->arch.guest_vl3table)
-#define linear_l4_table(v) ((v)->arch.guest_vl4table)
+#define linear_l2_table(v) ((l2_pgentry_t *)(v)->arch.guest_vtable)
 
 #ifndef __ASSEMBLY__
 #if CONFIG_PAGING_LEVELS == 3
@@ -294,6 +286,7 @@ extern void paging_init(void);
 #define _PAGE_AVAIL1   0x400U
 #define _PAGE_AVAIL2   0x800U
 #define _PAGE_AVAIL    0xE00U
+#define _PAGE_PSE_PAT 0x1000U
 
 /*
  * Debug option: Ensure that granted mappings are not implicitly unmapped.
@@ -307,9 +300,9 @@ extern void paging_init(void);
 #endif
 
 /*
- * Disallow unused flag bits plus PAT, PSE and GLOBAL. Also disallow GNTTAB
- * if we are using it for grant-table debugging. Permit the NX bit if the
- * hardware supports it.
+ * Disallow unused flag bits plus PAT, PSE and GLOBAL.
+ * Also disallow GNTTAB if we are using it for grant-table debugging.
+ * Permit the NX bit if the hardware supports it.
  */
 #define BASE_DISALLOW_MASK ((0xFFFFF180U | _PAGE_GNTTAB) & ~_PAGE_NX)
 

xen/include/asm-x86/perfc_defn.h

@@ -144,4 +144,57 @@ PERFCOUNTER_CPU(remove_write_predicted, "remove_write predict hit&exit")
 PERFCOUNTER_CPU(remove_write_bad_prediction, "remove_write bad prediction")
 PERFCOUNTER_CPU(update_hl2e_invlpg,     "update_hl2e calls invlpg")
 
+/* Shadow2 counters */
+PERFCOUNTER_CPU(shadow2_alloc,          "calls to shadow2_alloc")
+PERFCOUNTER_CPU(shadow2_alloc_tlbflush, "shadow2_alloc flushed TLBs")
+PERFSTATUS(shadow2_alloc_count,         "number of shadow pages in use")
+PERFCOUNTER_CPU(shadow2_free,           "calls to shadow2_free")
+PERFCOUNTER_CPU(shadow2_prealloc_1,     "shadow2 recycles old shadows")
+PERFCOUNTER_CPU(shadow2_prealloc_2,     "shadow2 recycles in-use shadows")
+PERFCOUNTER_CPU(shadow2_linear_map_failed, "shadow2 hit read-only linear map")
+PERFCOUNTER_CPU(shadow2_a_update,       "shadow2 A bit update")
+PERFCOUNTER_CPU(shadow2_ad_update,      "shadow2 A&D bit update")
+PERFCOUNTER_CPU(shadow2_fault,          "calls to shadow2_fault")
+PERFCOUNTER_CPU(shadow2_fault_bail_bad_gfn, "shadow2_fault guest bad gfn")
+PERFCOUNTER_CPU(shadow2_fault_bail_not_present, 
+                                        "shadow2_fault guest not-present")
+PERFCOUNTER_CPU(shadow2_fault_bail_nx,  "shadow2_fault guest NX fault")
+PERFCOUNTER_CPU(shadow2_fault_bail_ro_mapping, "shadow2_fault guest R/W fault")
+PERFCOUNTER_CPU(shadow2_fault_bail_user_supervisor, 
+                                        "shadow2_fault guest U/S fault")
+PERFCOUNTER_CPU(shadow2_fault_emulate_read, "shadow2_fault emulates a read")
+PERFCOUNTER_CPU(shadow2_fault_emulate_write, "shadow2_fault emulates a write")
+PERFCOUNTER_CPU(shadow2_fault_emulate_failed, "shadow2_fault emulator fails")
+PERFCOUNTER_CPU(shadow2_fault_mmio,     "shadow2_fault handled as mmio")
+PERFCOUNTER_CPU(shadow2_fault_fixed,    "shadow2_fault fixed fault")
+PERFCOUNTER_CPU(shadow2_ptwr_emulate,   "shadow2 causes ptwr to emulate")
+PERFCOUNTER_CPU(shadow2_validate_gl1e_calls, "calls to shadow2_validate_gl1e")
+PERFCOUNTER_CPU(shadow2_validate_gl2e_calls, "calls to shadow2_validate_gl2e")
+PERFCOUNTER_CPU(shadow2_validate_gl3e_calls, "calls to shadow2_validate_gl3e")
+PERFCOUNTER_CPU(shadow2_validate_gl4e_calls, "calls to shadow2_validate_gl4e")
+PERFCOUNTER_CPU(shadow2_hash_lookups,   "calls to shadow2_hash_lookup")
+PERFCOUNTER_CPU(shadow2_hash_lookup_head, "shadow2 hash hit in bucket head")
+PERFCOUNTER_CPU(shadow2_hash_lookup_miss, "shadow2 hash misses")
+PERFCOUNTER_CPU(shadow2_get_shadow_status, "calls to get_shadow_status")
+PERFCOUNTER_CPU(shadow2_hash_inserts,   "calls to shadow2_hash_insert")
+PERFCOUNTER_CPU(shadow2_hash_deletes,   "calls to shadow2_hash_delete")
+PERFCOUNTER_CPU(shadow2_writeable,      "shadow2 removes write access")
+PERFCOUNTER_CPU(shadow2_writeable_h_1,  "shadow2 writeable: 32b w2k3")
+PERFCOUNTER_CPU(shadow2_writeable_h_2,  "shadow2 writeable: 32pae w2k3")
+PERFCOUNTER_CPU(shadow2_writeable_h_3,  "shadow2 writeable: 64b w2k3")
+PERFCOUNTER_CPU(shadow2_writeable_h_4,  "shadow2 writeable: 32b linux low")
+PERFCOUNTER_CPU(shadow2_writeable_bf,   "shadow2 writeable brute-force")
+PERFCOUNTER_CPU(shadow2_mappings,       "shadow2 removes all mappings")
+PERFCOUNTER_CPU(shadow2_mappings_bf,    "shadow2 rm-mappings brute-force")
+PERFCOUNTER_CPU(shadow2_early_unshadow, "shadow2 unshadows for fork/exit")
+PERFCOUNTER_CPU(shadow2_early_unshadow_top, "shadow2 unhooks for fork/exit")
+PERFCOUNTER_CPU(shadow2_unshadow,       "shadow2 unshadows a page")
+PERFCOUNTER_CPU(shadow2_up_pointer,     "shadow2 unshadow by up-pointer")
+PERFCOUNTER_CPU(shadow2_unshadow_bf,    "shadow2 unshadow brute-force")
+PERFCOUNTER_CPU(shadow2_get_page_fail,  "shadow2_get_page_from_l1e failed")
+PERFCOUNTER_CPU(shadow2_guest_walk,     "shadow2 walks guest tables")
+PERFCOUNTER_CPU(shadow2_walk_cache_hit, "shadow2 walk-cache hits")
+PERFCOUNTER_CPU(shadow2_walk_cache_miss, "shadow2 walk-cache misses")
+
+
 /*#endif*/ /* __XEN_PERFC_DEFN_H__ */

xen/include/asm-x86/processor.h

@@ -545,6 +545,7 @@ extern always_inline void prefetchw(const void *x)
 #endif
 
 void show_stack(struct cpu_user_regs *regs);
+void show_xen_trace(void);
 void show_stack_overflow(unsigned long esp);
 void show_registers(struct cpu_user_regs *regs);
 void show_execution_state(struct cpu_user_regs *regs);

xen/include/asm-x86/shadow2-multi.h

@@ -0,0 +1,116 @@
+/******************************************************************************
+ * arch/x86/shadow2-multi.h
+ *
+ * Shadow2 declarations which will be multiply compiled.
+ * Parts of this code are Copyright (c) 2006 by XenSource Inc.
+ * Parts of this code are Copyright (c) 2006 by Michael A Fetterman
+ * Parts based on earlier work by Michael A Fetterman, Ian Pratt et al.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+extern int 
+SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl1e, SHADOW_LEVELS, GUEST_LEVELS)(
+    struct vcpu *v, mfn_t gl1mfn, void *new_gl1p, u32 size);
+extern int 
+SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl2e, SHADOW_LEVELS, GUEST_LEVELS)(
+    struct vcpu *v, mfn_t gl2mfn, void *new_gl2p, u32 size);
+extern int 
+SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl2he, SHADOW_LEVELS, GUEST_LEVELS)(
+    struct vcpu *v, mfn_t gl2mfn, void *new_gl2p, u32 size);
+extern int 
+SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl3e, SHADOW_LEVELS, GUEST_LEVELS)(
+    struct vcpu *v, mfn_t gl3mfn, void *new_gl3p, u32 size);
+extern int 
+SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl4e, SHADOW_LEVELS, GUEST_LEVELS)(
+    struct vcpu *v, mfn_t gl4mfn, void *new_gl4p, u32 size);
+
+extern void 
+SHADOW2_INTERNAL_NAME(sh2_destroy_l1_shadow, SHADOW_LEVELS, GUEST_LEVELS)(
+    struct vcpu *v, mfn_t smfn);
+extern void 
+SHADOW2_INTERNAL_NAME(sh2_destroy_l2_shadow, SHADOW_LEVELS, GUEST_LEVELS)(
+    struct vcpu *v, mfn_t smfn);
+extern void 
+SHADOW2_INTERNAL_NAME(sh2_destroy_l3_shadow, SHADOW_LEVELS, GUEST_LEVELS)(
+    struct vcpu *v, mfn_t smfn);
+extern void 
+SHADOW2_INTERNAL_NAME(sh2_destroy_l4_shadow, SHADOW_LEVELS, GUEST_LEVELS)(
+    struct vcpu *v, mfn_t smfn);
+
+extern void
+SHADOW2_INTERNAL_NAME(sh2_unpin_all_l3_subshadows, 3, 3)
+    (struct vcpu *v, mfn_t smfn);
+
+extern void 
+SHADOW2_INTERNAL_NAME(sh2_unhook_32b_mappings, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl2mfn);
+extern void 
+SHADOW2_INTERNAL_NAME(sh2_unhook_pae_mappings, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl3mfn);
+extern void 
+SHADOW2_INTERNAL_NAME(sh2_unhook_64b_mappings, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl4mfn);
+
+extern int
+SHADOW2_INTERNAL_NAME(sh2_remove_write_access, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl1mfn, mfn_t readonly_mfn);
+extern int
+SHADOW2_INTERNAL_NAME(sh2_remove_all_mappings, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl1mfn, mfn_t target_mfn);
+
+extern void
+SHADOW2_INTERNAL_NAME(sh2_clear_shadow_entry, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, void *ep, mfn_t smfn);
+
+extern int
+SHADOW2_INTERNAL_NAME(sh2_remove_l1_shadow, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl2mfn, mfn_t sl1mfn);
+extern int
+SHADOW2_INTERNAL_NAME(sh2_remove_l2_shadow, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl3mfn, mfn_t sl2mfn);
+extern int
+SHADOW2_INTERNAL_NAME(sh2_remove_l3_shadow, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl4mfn, mfn_t sl3mfn);
+
+#if SHADOW2_AUDIT & SHADOW2_AUDIT_ENTRIES
+int 
+SHADOW2_INTERNAL_NAME(sh2_audit_l1_table, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl1mfn, mfn_t x);
+int 
+SHADOW2_INTERNAL_NAME(sh2_audit_fl1_table, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl1mfn, mfn_t x);
+int 
+SHADOW2_INTERNAL_NAME(sh2_audit_l2_table, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl2mfn, mfn_t x);
+int 
+SHADOW2_INTERNAL_NAME(sh2_audit_l3_table, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl3mfn, mfn_t x);
+int 
+SHADOW2_INTERNAL_NAME(sh2_audit_l4_table, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t sl4mfn, mfn_t x);
+#endif
+
+#if SHADOW_LEVELS == GUEST_LEVELS
+extern mfn_t
+SHADOW2_INTERNAL_NAME(sh2_make_monitor_table, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v);
+extern void
+SHADOW2_INTERNAL_NAME(sh2_destroy_monitor_table, SHADOW_LEVELS, GUEST_LEVELS)
+    (struct vcpu *v, mfn_t mmfn);
+#endif
+
+extern struct shadow2_entry_points 
+SHADOW2_INTERNAL_NAME(shadow2_entry, SHADOW_LEVELS, GUEST_LEVELS);

xen/include/asm-x86/shadow2-private.h

@@ -0,0 +1,612 @@
+/******************************************************************************
+ * arch/x86/shadow2-private.h
+ *
+ * Shadow2 code that is private, and does not need to be multiply compiled.
+ * Parts of this code are Copyright (c) 2006 by XenSource Inc.
+ * Parts of this code are Copyright (c) 2006 by Michael A Fetterman
+ * Parts based on earlier work by Michael A Fetterman, Ian Pratt et al.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#ifndef _XEN_SHADOW2_PRIVATE_H
+#define _XEN_SHADOW2_PRIVATE_H
+
+// In order to override the definition of mfn_to_page, we make sure page.h has
+// been included...
+#include <asm/page.h>
+#include <xen/domain_page.h>
+#include <asm/x86_emulate.h>
+#include <asm/hvm/support.h>
+
+
+/******************************************************************************
+ * Definitions for the use of the "available" bits in the shadow PTEs.
+ *
+ * Review of the low 12 bits of a shadow page table entry:
+ *
+ *         in a guest:                      in a shadow:
+ * Bit 11: _PAGE_AVAIL2, aka _PAGE_GNTTAB
+ * Bit 10: _PAGE_AVAIL1                     _PAGE_SHADOW_RW ("SW" below)
+ * Bit  9: _PAGE_AVAIL0                     _PAGE_SHADOW_PRESENT ("SP" below)
+ * Bit  8: _PAGE_GLOBAL                     _PAGE_SHADOW_MMIO ("MMIO" below),
+ *                                          aka _PAGE_SHADOW_GUEST_NOT_PRESENT
+ * Bit  7: _PAGE_PSE, aka _PAGE_PAT
+ * Bit  6: _PAGE_DIRTY
+ * Bit  5: _PAGE_ACCESSED
+ * Bit  4: _PAGE_PCD
+ * Bit  3: _PAGE_PWT
+ * Bit  2: _PAGE_USER
+ * Bit  1: _PAGE_RW ("GW" below)
+ * Bit  0: _PAGE_PRESENT ("GP" below)
+ *
+ * Given a guest entry, as shown below, we can expect the following in the
+ * corresponding shadow entry:
+ *
+ * Guest entry  Shadow entry      Commentary
+ * -----------  ----------------  ---------------------------------------------
+ *       Maps     
+ * GP GW  IO    GP SP GW SW MMIO 
+ * -- -- ----   -- -- -- -- ----
+ *  -  -   -     0  0  0  0   0   The guest entry has not yet been shadowed.
+ *  0  -   -     0  0  0  0   1   The guest entry is marked not-present.
+ *  1  1  no     ?  1  ?  1   0   Writable entry in the guest.
+ *  1  0  no     ?  1  0  0   0   Read-only entry in the guest.
+ *  1  1  yes    0  1  ?  1   1   Writable MMIO mapping in the guest.
+ *  1  0  yes    0  1  0  0   1   Read-only MMIO mapping in the guest.
+ *
+ * Normally, we would expect that GP=1 in the guest to imply GP=1 in the
+ * shadow, and similarly for GW=1.  However, various functionality that may be
+ * implemented via the shadow can cause GP or GW to be cleared in such cases.
+ * A & D bit emulation is a prime example of such functionality.
+ *
+ * If _PAGE_SHADOW_PRESENT is zero, then the _PAGE_PRESENT bit in that same
+ * entry will always be zero, too.
+
+ * Bit 11 is used in debug builds as the _PAGE_GNTTAB bit in PV guests.  It is
+ * currently available for random (ab)use in shadow entries.
+ *
+ * Bit 8 (the global bit) could be propagated from an HVM guest to the shadow,
+ * but currently there is no benefit, as the guest's TLB is flushed on every
+ * transition of CR3 anyway due to the HVM exit/re-entry.
+ *
+ * In shadow entries in which the _PAGE_SHADOW_PRESENT is set, bit 8 is used
+ * as the _PAGE_SHADOW_MMIO bit.  In such entries, if _PAGE_SHADOW_MMIO is
+ * set, then the entry contains the *gfn* directly from the corresponding
+ * guest entry (not an mfn!!).
+ *
+ * Bit 7 is set in a guest L2 to signify a superpage entry.  The current
+ * shadow code splinters superpage mappings into 512 or 1024 4K mappings; the
+ * resulting shadow L1 table is called an FL1.  Note that there is no guest
+ * page that corresponds to an FL1.
+ *
+ * Bit 7 in a guest L1 is the PAT2 bit.  Currently we do not support PAT in
+ * this shadow code.
+ *
+ * Bit 6 is the dirty bit.
+ *
+ * Bit 5 is the accessed bit.
+ *
+ * Bit 4 is the cache disable bit.  If set in a guest, the hardware is
+ * supposed to refuse to cache anything found via this entry.  It can be set
+ * in an L4e, L3e, L2e, or L1e.  This shadow code currently does not support
+ * cache disable bits.  They are silently ignored.
+ *
+ * Bit 4 is a guest L1 is also the PAT1 bit.  Currently we do not support PAT
+ * in this shadow code.
+ *
+ * Bit 3 is the cache write-thru bit.  If set in a guest, the hardware is
+ * supposed to use write-thru instead of write-back caching for anything found
+ * via this entry.  It can be set in an L4e, L3e, L2e, or L1e.  This shadow
+ * code currently does not support cache write-thru bits.  They are silently
+ * ignored.
+ *
+ * Bit 3 is a guest L1 is also the PAT0 bit.  Currently we do not support PAT
+ * in this shadow code.
+ *
+ * Bit 2 is the user bit.
+ *
+ * Bit 1 is the read-write bit.
+ *
+ * Bit 0 is the present bit.
+ */
+
+// Copy of the _PAGE_RW bit from the guest's PTE, appropriately zero'ed by
+// the appropriate shadow rules.
+#define _PAGE_SHADOW_RW                 _PAGE_AVAIL1
+
+// Copy of the _PAGE_PRESENT bit from the guest's PTE
+#define _PAGE_SHADOW_PRESENT            _PAGE_AVAIL0
+
+// The matching guest entry maps MMIO space
+#define _PAGE_SHADOW_MMIO               _PAGE_GLOBAL
+
+// Shadow flags value used when the guest is not present
+#define _PAGE_SHADOW_GUEST_NOT_PRESENT  _PAGE_GLOBAL
+
+
+/******************************************************************************
+ * Debug and error-message output
+ */
+#define SHADOW2_PRINTK(_f, _a...)                                     \
+    debugtrace_printk("sh2: %s(): " _f, __func__, ##_a)
+#define SHADOW2_ERROR(_f, _a...)                                      \
+    printk("sh2 error: %s(): " _f, __func__, ##_a)
+#define SHADOW2_DEBUG(flag, _f, _a...)                                \
+    do {                                                              \
+        if (SHADOW2_DEBUG_ ## flag)                                   \
+            debugtrace_printk("sh2debug: %s(): " _f, __func__, ##_a); \
+    } while (0)
+
+// The flags for use with SHADOW2_DEBUG:
+#define SHADOW2_DEBUG_PROPAGATE         0
+#define SHADOW2_DEBUG_MAKE_SHADOW       0
+#define SHADOW2_DEBUG_DESTROY_SHADOW    0
+#define SHADOW2_DEBUG_P2M               0
+#define SHADOW2_DEBUG_A_AND_D           0
+#define SHADOW2_DEBUG_EMULATE           0
+#define SHADOW2_DEBUG_LOGDIRTY          1
+
+
+/******************************************************************************
+ * Auditing routines 
+ */
+
+#if SHADOW2_AUDIT & SHADOW2_AUDIT_ENTRIES_FULL
+extern void shadow2_audit_tables(struct vcpu *v);
+#else
+#define shadow2_audit_tables(_v) do {} while(0)
+#endif
+
+#if SHADOW2_AUDIT & SHADOW2_AUDIT_P2M
+extern void shadow2_audit_p2m(struct domain *d);
+#else
+#define shadow2_audit_p2m(_d) do {} while(0)
+#endif
+
+
+/******************************************************************************
+ * Mechanism for double-checking the optimized pagefault path: this
+ * structure contains a record of actions taken by the fault handling
+ * code.  In paranoid mode, the fast-path code fills out one of these
+ * structures (but doesn't take any actual action) and then the normal 
+ * path fills in another.  When the fault handler finishes, the 
+ * two are compared */
+
+#ifdef SHADOW2_OPTIMIZATION_PARANOIA
+
+typedef struct shadow2_action_log sh2_log_t;
+struct shadow2_action_log {
+    paddr_t ad[CONFIG_PAGING_LEVELS];  /* A & D bits propagated here */
+    paddr_t mmio;                      /* Address of an mmio operation */
+    int rv;                            /* Result of the fault handler */
+};
+
+/* There are two logs, one for the fast path, one for the normal path */
+enum sh2_log_type { log_slow = 0, log_fast= 1 };
+
+/* Alloc and zero the logs */
+static inline void sh2_init_log(struct vcpu *v) 
+{
+    if ( unlikely(!v->arch.shadow2_action_log) ) 
+        v->arch.shadow2_action_log = xmalloc_array(sh2_log_t, 2);
+    ASSERT(v->arch.shadow2_action_log);
+    memset(v->arch.shadow2_action_log, 0, 2 * sizeof (sh2_log_t));
+}
+
+/* Log an A&D-bit update */
+static inline void sh2_log_ad(struct vcpu *v, paddr_t e, unsigned int level)
+{
+    v->arch.shadow2_action_log[v->arch.shadow2_action_index].ad[level] = e;
+}
+
+/* Log an MMIO address */
+static inline void sh2_log_mmio(struct vcpu *v, paddr_t m)
+{
+    v->arch.shadow2_action_log[v->arch.shadow2_action_index].mmio = m;
+}
+
+/* Log the result */
+static inline void sh2_log_rv(struct vcpu *v, int rv)
+{
+    v->arch.shadow2_action_log[v->arch.shadow2_action_index].rv = rv;
+}
+
+/* Set which mode we're in */
+static inline void sh2_set_log_mode(struct vcpu *v, enum sh2_log_type t) 
+{
+    v->arch.shadow2_action_index = t;
+}
+
+/* Know not to take action, because we're only checking the mechanism */
+static inline int sh2_take_no_action(struct vcpu *v) 
+{
+    return (v->arch.shadow2_action_index == log_fast);
+}
+
+#else /* Non-paranoid mode: these logs do not exist */
+
+#define sh2_init_log(_v) do { (void)(_v); } while(0)
+#define sh2_set_log_mode(_v,_t) do { (void)(_v); } while(0)
+#define sh2_log_ad(_v,_e,_l) do { (void)(_v),(void)(_e),(void)(_l); } while (0)
+#define sh2_log_mmio(_v,_m) do { (void)(_v),(void)(_m); } while (0)
+#define sh2_log_rv(_v,_r) do { (void)(_v),(void)(_r); } while (0)
+#define sh2_take_no_action(_v) (((void)(_v)), 0)
+
+#endif /* SHADOW2_OPTIMIZATION_PARANOIA */
+
+
+/******************************************************************************
+ * Macro for dealing with the naming of the internal names of the
+ * shadow code's external entry points.
+ */
+#define SHADOW2_INTERNAL_NAME_HIDDEN(name, shadow_levels, guest_levels) \
+    name ## __shadow_ ## shadow_levels ## _guest_ ## guest_levels
+#define SHADOW2_INTERNAL_NAME(name, shadow_levels, guest_levels) \
+    SHADOW2_INTERNAL_NAME_HIDDEN(name, shadow_levels, guest_levels)
+
+#if CONFIG_PAGING_LEVELS == 2
+#define GUEST_LEVELS  2
+#define SHADOW_LEVELS 2
+#include <asm/shadow2-multi.h>
+#undef GUEST_LEVELS
+#undef SHADOW_LEVELS
+#endif /* CONFIG_PAGING_LEVELS == 2 */
+
+#if CONFIG_PAGING_LEVELS == 3
+#define GUEST_LEVELS  2
+#define SHADOW_LEVELS 3
+#include <asm/shadow2-multi.h>
+#undef GUEST_LEVELS
+#undef SHADOW_LEVELS
+
+#define GUEST_LEVELS  3
+#define SHADOW_LEVELS 3
+#include <asm/shadow2-multi.h>
+#undef GUEST_LEVELS
+#undef SHADOW_LEVELS
+#endif /* CONFIG_PAGING_LEVELS == 3 */
+
+#if CONFIG_PAGING_LEVELS == 4
+#define GUEST_LEVELS  2
+#define SHADOW_LEVELS 3
+#include <asm/shadow2-multi.h>
+#undef GUEST_LEVELS
+#undef SHADOW_LEVELS
+
+#define GUEST_LEVELS  3
+#define SHADOW_LEVELS 3
+#include <asm/shadow2-multi.h>
+#undef GUEST_LEVELS
+#undef SHADOW_LEVELS
+
+#define GUEST_LEVELS  3
+#define SHADOW_LEVELS 4
+#include <asm/shadow2-multi.h>
+#undef GUEST_LEVELS
+#undef SHADOW_LEVELS
+
+#define GUEST_LEVELS  4
+#define SHADOW_LEVELS 4
+#include <asm/shadow2-multi.h>
+#undef GUEST_LEVELS
+#undef SHADOW_LEVELS
+#endif /* CONFIG_PAGING_LEVELS == 4 */
+
+
+/******************************************************************************
+ * Various function declarations 
+ */
+
+/* x86 emulator support */
+extern struct x86_emulate_ops shadow2_emulator_ops;
+
+/* Hash table functions */
+mfn_t shadow2_hash_lookup(struct vcpu *v, unsigned long n, u8 t);
+void  shadow2_hash_insert(struct vcpu *v, unsigned long n, u8 t, mfn_t smfn);
+void  shadow2_hash_delete(struct vcpu *v, unsigned long n, u8 t, mfn_t smfn);
+
+/* shadow promotion */
+void shadow2_promote(struct vcpu *v, mfn_t gmfn, u32 type);
+void shadow2_demote(struct vcpu *v, mfn_t gmfn, u32 type);
+
+/* Shadow page allocation functions */
+void  shadow2_prealloc(struct domain *d, unsigned int order);
+mfn_t shadow2_alloc(struct domain *d, 
+                    u32 shadow_type,
+                    unsigned long backpointer);
+void  shadow2_free(struct domain *d, mfn_t smfn);
+
+/* Function to convert a shadow to log-dirty */
+void shadow2_convert_to_log_dirty(struct vcpu *v, mfn_t smfn);
+
+/* Dispatcher function: call the per-mode function that will unhook the
+ * non-Xen mappings in this top-level shadow mfn */
+void shadow2_unhook_mappings(struct vcpu *v, mfn_t smfn);
+
+/* Re-sync copies of PAE shadow L3 tables if they have been changed */
+void sh2_pae_recopy(struct domain *d);
+
+/* Install the xen mappings in various flavours of shadow */
+void sh2_install_xen_entries_in_l4(struct vcpu *v, mfn_t gl4mfn, mfn_t sl4mfn);
+void sh2_install_xen_entries_in_l2h(struct vcpu *v, mfn_t sl2hmfn);
+void sh2_install_xen_entries_in_l3(struct vcpu *v, mfn_t gl3mfn, mfn_t sl3mfn);
+void sh2_install_xen_entries_in_l2(struct vcpu *v, mfn_t gl2mfn, mfn_t sl2mfn);
+
+
+/******************************************************************************
+ * MFN/page-info handling 
+ */
+
+// Override mfn_to_page from asm/page.h, which was #include'd above,
+// in order to make it work with our mfn type.
+#undef mfn_to_page
+#define mfn_to_page(_mfn) (frame_table + mfn_x(_mfn))
+
+// Override page_to_mfn from asm/page.h, which was #include'd above,
+// in order to make it work with our mfn type.
+#undef page_to_mfn
+#define page_to_mfn(_pg) (_mfn((_pg) - frame_table))
+
+// Override mfn_valid from asm/page.h, which was #include'd above,
+// in order to make it work with our mfn type.
+#undef mfn_valid
+#define mfn_valid(_mfn) (mfn_x(_mfn) < max_page)
+
+// Provide mfn_t-aware versions of common xen functions
+static inline void *
+sh2_map_domain_page(mfn_t mfn)
+{
+    /* XXX Using the monitor-table as a map will happen here  */
+    return map_domain_page(mfn_x(mfn));
+}
+
+static inline void 
+sh2_unmap_domain_page(void *p) 
+{
+    /* XXX Using the monitor-table as a map will happen here  */
+    unmap_domain_page(p);
+}
+
+static inline void *
+sh2_map_domain_page_global(mfn_t mfn)
+{
+    /* XXX Using the monitor-table as a map will happen here  */
+    return map_domain_page_global(mfn_x(mfn));
+}
+
+static inline void 
+sh2_unmap_domain_page_global(void *p) 
+{
+    /* XXX Using the monitor-table as a map will happen here  */
+    unmap_domain_page_global(p);
+}
+
+static inline int
+sh2_mfn_is_dirty(struct domain *d, mfn_t gmfn)
+/* Is this guest page dirty?  Call only in log-dirty mode. */
+{
+    unsigned long pfn;
+    ASSERT(shadow2_mode_log_dirty(d));
+    ASSERT(d->arch.shadow_dirty_bitmap != NULL);
+
+    /* We /really/ mean PFN here, even for non-translated guests. */
+    pfn = get_gpfn_from_mfn(mfn_x(gmfn));
+    if ( likely(VALID_M2P(pfn))
+         && likely(pfn < d->arch.shadow_dirty_bitmap_size) 
+         && test_bit(pfn, d->arch.shadow_dirty_bitmap) )
+        return 1;
+
+    return 0;
+}
+
+static inline int
+sh2_mfn_is_a_page_table(mfn_t gmfn)
+{
+    struct page_info *page = mfn_to_page(gmfn);
+    struct domain *owner;
+    unsigned long type_info;
+
+    if ( !valid_mfn(gmfn) )
+        return 0;
+
+    owner = page_get_owner(page);
+    if ( owner && shadow2_mode_refcounts(owner) 
+         && (page->count_info & PGC_page_table) )
+        return 1; 
+
+    type_info = page->u.inuse.type_info & PGT_type_mask;
+    return type_info && (type_info <= PGT_l4_page_table);
+}
+
+
+/**************************************************************************/
+/* Shadow-page refcounting. See comment in shadow2-common.c about the  
+ * use of struct page_info fields for shadow pages */
+
+void sh2_destroy_shadow(struct vcpu *v, mfn_t smfn);
+
+/* Increase the refcount of a shadow page.  Arguments are the mfn to refcount, 
+ * and the physical address of the shadow entry that holds the ref (or zero
+ * if the ref is held by something else) */
+static inline void sh2_get_ref(mfn_t smfn, paddr_t entry_pa)
+{
+    u32 x, nx;
+    struct page_info *page = mfn_to_page(smfn);
+
+    ASSERT(mfn_valid(smfn));
+
+    x = page->count_info & PGC_SH2_count_mask;
+    nx = x + 1;
+
+    if ( unlikely(nx & ~PGC_SH2_count_mask) )
+    {
+        SHADOW2_PRINTK("shadow ref overflow, gmfn=%" PRtype_info " smfn=%lx\n",
+                       page->u.inuse.type_info, mfn_x(smfn));
+        domain_crash_synchronous();
+    }
+    
+    /* Guarded by the shadow lock, so no need for atomic update */
+    page->count_info &= ~PGC_SH2_count_mask;
+    page->count_info |= nx;
+
+    /* We remember the first shadow entry that points to each shadow. */
+    if ( entry_pa != 0 && page->up == 0 ) 
+        page->up = entry_pa;
+}
+
+
+/* Decrease the refcount of a shadow page.  As for get_ref, takes the
+ * physical address of the shadow entry that held this reference. */
+static inline void sh2_put_ref(struct vcpu *v, mfn_t smfn, paddr_t entry_pa)
+{
+    u32 x, nx;
+    struct page_info *page = mfn_to_page(smfn);
+
+    ASSERT(mfn_valid(smfn));
+    ASSERT(page_get_owner(page) == NULL);
+
+    /* If this is the entry in the up-pointer, remove it */
+    if ( entry_pa != 0 && page->up == entry_pa ) 
+        page->up = 0;
+
+    x = page->count_info & PGC_SH2_count_mask;
+    nx = x - 1;
+
+    if ( unlikely(x == 0) ) 
+    {
+        SHADOW2_PRINTK("shadow ref underflow, smfn=%lx oc=%08x t=%" 
+                       PRtype_info "\n",
+                       mfn_x(smfn),
+                       page->count_info & PGC_SH2_count_mask,
+                       page->u.inuse.type_info);
+        domain_crash_synchronous();
+    }
+
+    /* Guarded by the shadow lock, so no need for atomic update */
+    page->count_info &= ~PGC_SH2_count_mask;
+    page->count_info |= nx;
+
+    if ( unlikely(nx == 0) ) 
+        sh2_destroy_shadow(v, smfn);
+}
+
+
+/* Pin a shadow page: take an extra refcount and set the pin bit. */
+static inline void sh2_pin(mfn_t smfn)
+{
+    struct page_info *page;
+    
+    ASSERT(mfn_valid(smfn));
+    page = mfn_to_page(smfn);
+    if ( !(page->count_info & PGC_SH2_pinned) ) 
+    {
+        sh2_get_ref(smfn, 0);
+        page->count_info |= PGC_SH2_pinned;
+    }
+}
+
+/* Unpin a shadow page: unset the pin bit and release the extra ref. */
+static inline void sh2_unpin(struct vcpu *v, mfn_t smfn)
+{
+    struct page_info *page;
+    
+    ASSERT(mfn_valid(smfn));
+    page = mfn_to_page(smfn);
+    if ( page->count_info & PGC_SH2_pinned )
+    {
+        page->count_info &= ~PGC_SH2_pinned;
+        sh2_put_ref(v, smfn, 0);
+    }
+}
+
+/**************************************************************************/
+/* CPU feature support querying */
+
+static inline int
+guest_supports_superpages(struct vcpu *v)
+{
+    return hvm_guest(v) && (hvm_get_guest_ctrl_reg(v, 4) & X86_CR4_PSE);
+}
+
+static inline int
+guest_supports_nx(struct vcpu *v)
+{
+    if ( !hvm_guest(v) )
+        return cpu_has_nx;
+
+    // XXX - fix this!
+    return 1;
+}
+
+/**************************************************************************/
+/* Guest physmap (p2m) support */
+
+/* Read our own P2M table, checking in the linear pagetables first to be
+ * sure that we will succeed.  Call this function if you expect it to
+ * fail often, as it avoids page faults.  If you expect to succeed, use
+ * vcpu_gfn_to_mfn, which copy_from_user()s the entry */
+static inline mfn_t
+vcpu_gfn_to_mfn_nofault(struct vcpu *v, unsigned long gfn)
+{
+    unsigned long entry_addr = (unsigned long) &phys_to_machine_mapping[gfn];
+#if CONFIG_PAGING_LEVELS >= 4
+    l4_pgentry_t *l4e;
+    l3_pgentry_t *l3e;
+#endif
+    l2_pgentry_t *l2e;
+    l1_pgentry_t *l1e;
+
+    ASSERT(current == v);
+    if ( !shadow2_vcpu_mode_translate(v) )
+        return _mfn(gfn);
+
+#if CONFIG_PAGING_LEVELS > 2
+    if ( gfn > (RO_MPT_VIRT_END - RO_MPT_VIRT_START) / sizeof(l1_pgentry_t) ) 
+        /* This pfn is higher than the p2m map can hold */
+        return _mfn(INVALID_MFN);
+#endif
+    
+    /* Walk the linear pagetables.  Note that this is *not* the same as 
+     * the walk in sh2_gfn_to_mfn_foreign, which is walking the p2m map */
+#if CONFIG_PAGING_LEVELS >= 4
+    l4e = __linear_l4_table + l4_linear_offset(entry_addr);
+    if ( !(l4e_get_flags(*l4e) & _PAGE_PRESENT) ) return _mfn(INVALID_MFN);
+    l3e = __linear_l3_table + l3_linear_offset(entry_addr);
+    if ( !(l3e_get_flags(*l3e) & _PAGE_PRESENT) ) return _mfn(INVALID_MFN);
+#endif
+    l2e = __linear_l2_table + l2_linear_offset(entry_addr);
+    if ( !(l2e_get_flags(*l2e) & _PAGE_PRESENT) ) return _mfn(INVALID_MFN);
+    l1e = __linear_l1_table + l1_linear_offset(entry_addr);
+    if ( !(l1e_get_flags(*l1e) & _PAGE_PRESENT) ) return _mfn(INVALID_MFN);
+
+    /* Safe to look at this part of the table */
+    if ( l1e_get_flags(phys_to_machine_mapping[gfn])  & _PAGE_PRESENT )
+        return _mfn(l1e_get_pfn(phys_to_machine_mapping[gfn]));
+    
+    return _mfn(INVALID_MFN);
+}
+
+
+#endif /* _XEN_SHADOW2_PRIVATE_H */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */

xen/include/asm-x86/shadow2-types.h

@@ -0,0 +1,705 @@
+/******************************************************************************
+ * include/asm-x86/shadow2-types.h
+ * 
+ * Parts of this code are Copyright (c) 2006 by XenSource Inc.
+ * Parts of this code are Copyright (c) 2006 by Michael A Fetterman
+ * Parts based on earlier work by Michael A Fetterman, Ian Pratt et al.
+ * 
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#ifndef _XEN_SHADOW2_TYPES_H
+#define _XEN_SHADOW2_TYPES_H
+
+// Map a shadow page
+static inline void *
+map_shadow_page(mfn_t smfn)
+{
+    // XXX -- Possible optimization/measurement question for 32-bit and PAE
+    //        hypervisors:
+    //        How often is this smfn already available in the shadow linear
+    //        table?  Might it be worth checking that table first,
+    //        presumably using the reverse map hint in the page_info of this
+    //        smfn, rather than calling map_domain_page()?
+    //
+    return sh2_map_domain_page(smfn);
+}
+
+// matching unmap for map_shadow_page()
+static inline void
+unmap_shadow_page(void *p)
+{
+    sh2_unmap_domain_page(p);
+}
+
+/* 
+ * Define various types for handling pagetabels, based on these options:
+ * SHADOW_PAGING_LEVELS : Number of levels of shadow pagetables
+ * GUEST_PAGING_LEVELS  : Number of levels of guest pagetables
+ */
+
+#if (CONFIG_PAGING_LEVELS < SHADOW_PAGING_LEVELS) 
+#error Cannot have more levels of shadow pagetables than host pagetables
+#endif
+
+#if (SHADOW_PAGING_LEVELS < GUEST_PAGING_LEVELS) 
+#error Cannot have more levels of guest pagetables than shadow pagetables
+#endif
+
+#if SHADOW_PAGING_LEVELS == 2
+#define SHADOW_L1_PAGETABLE_ENTRIES    1024
+#define SHADOW_L2_PAGETABLE_ENTRIES    1024
+#define SHADOW_L1_PAGETABLE_SHIFT        12
+#define SHADOW_L2_PAGETABLE_SHIFT        22
+#endif
+
+#if SHADOW_PAGING_LEVELS == 3
+#define SHADOW_L1_PAGETABLE_ENTRIES     512
+#define SHADOW_L2_PAGETABLE_ENTRIES     512
+#define SHADOW_L3_PAGETABLE_ENTRIES       4
+#define SHADOW_L1_PAGETABLE_SHIFT        12
+#define SHADOW_L2_PAGETABLE_SHIFT        21
+#define SHADOW_L3_PAGETABLE_SHIFT        30
+#endif
+
+#if SHADOW_PAGING_LEVELS == 4
+#define SHADOW_L1_PAGETABLE_ENTRIES     512
+#define SHADOW_L2_PAGETABLE_ENTRIES     512
+#define SHADOW_L3_PAGETABLE_ENTRIES     512
+#define SHADOW_L4_PAGETABLE_ENTRIES     512
+#define SHADOW_L1_PAGETABLE_SHIFT        12
+#define SHADOW_L2_PAGETABLE_SHIFT        21
+#define SHADOW_L3_PAGETABLE_SHIFT        30
+#define SHADOW_L4_PAGETABLE_SHIFT        39
+#endif
+
+/* Types of the shadow page tables */
+typedef l1_pgentry_t shadow_l1e_t;
+typedef l2_pgentry_t shadow_l2e_t;
+#if SHADOW_PAGING_LEVELS >= 3
+typedef l3_pgentry_t shadow_l3e_t;
+#if SHADOW_PAGING_LEVELS >= 4
+typedef l4_pgentry_t shadow_l4e_t;
+#endif
+#endif
+
+/* Access functions for them */
+static inline paddr_t shadow_l1e_get_paddr(shadow_l1e_t sl1e)
+{ return l1e_get_paddr(sl1e); }
+static inline paddr_t shadow_l2e_get_paddr(shadow_l2e_t sl2e)
+{ return l2e_get_paddr(sl2e); }
+#if SHADOW_PAGING_LEVELS >= 3
+static inline paddr_t shadow_l3e_get_paddr(shadow_l3e_t sl3e)
+{ return l3e_get_paddr(sl3e); }
+#if SHADOW_PAGING_LEVELS >= 4
+static inline paddr_t shadow_l4e_get_paddr(shadow_l4e_t sl4e)
+{ return l4e_get_paddr(sl4e); }
+#endif
+#endif
+
+static inline mfn_t shadow_l1e_get_mfn(shadow_l1e_t sl1e)
+{ return _mfn(l1e_get_pfn(sl1e)); }
+static inline mfn_t shadow_l2e_get_mfn(shadow_l2e_t sl2e)
+{ return _mfn(l2e_get_pfn(sl2e)); }
+#if SHADOW_PAGING_LEVELS >= 3
+static inline mfn_t shadow_l3e_get_mfn(shadow_l3e_t sl3e)
+{ return _mfn(l3e_get_pfn(sl3e)); }
+#if SHADOW_PAGING_LEVELS >= 4
+static inline mfn_t shadow_l4e_get_mfn(shadow_l4e_t sl4e)
+{ return _mfn(l4e_get_pfn(sl4e)); }
+#endif
+#endif
+
+static inline u32 shadow_l1e_get_flags(shadow_l1e_t sl1e)
+{ return l1e_get_flags(sl1e); }
+static inline u32 shadow_l2e_get_flags(shadow_l2e_t sl2e)
+{ return l2e_get_flags(sl2e); }
+#if SHADOW_PAGING_LEVELS >= 3
+static inline u32 shadow_l3e_get_flags(shadow_l3e_t sl3e)
+{ return l3e_get_flags(sl3e); }
+#if SHADOW_PAGING_LEVELS >= 4
+static inline u32 shadow_l4e_get_flags(shadow_l4e_t sl4e)
+{ return l4e_get_flags(sl4e); }
+#endif
+#endif
+
+static inline shadow_l1e_t
+shadow_l1e_remove_flags(shadow_l1e_t sl1e, u32 flags)
+{ l1e_remove_flags(sl1e, flags); return sl1e; }
+
+static inline shadow_l1e_t shadow_l1e_empty(void) 
+{ return l1e_empty(); }
+static inline shadow_l2e_t shadow_l2e_empty(void) 
+{ return l2e_empty(); }
+#if SHADOW_PAGING_LEVELS >= 3
+static inline shadow_l3e_t shadow_l3e_empty(void) 
+{ return l3e_empty(); }
+#if SHADOW_PAGING_LEVELS >= 4
+static inline shadow_l4e_t shadow_l4e_empty(void) 
+{ return l4e_empty(); }
+#endif
+#endif
+
+static inline shadow_l1e_t shadow_l1e_from_mfn(mfn_t mfn, u32 flags)
+{ return l1e_from_pfn(mfn_x(mfn), flags); }
+static inline shadow_l2e_t shadow_l2e_from_mfn(mfn_t mfn, u32 flags)
+{ return l2e_from_pfn(mfn_x(mfn), flags); }
+#if SHADOW_PAGING_LEVELS >= 3
+static inline shadow_l3e_t shadow_l3e_from_mfn(mfn_t mfn, u32 flags)
+{ return l3e_from_pfn(mfn_x(mfn), flags); }
+#if SHADOW_PAGING_LEVELS >= 4
+static inline shadow_l4e_t shadow_l4e_from_mfn(mfn_t mfn, u32 flags)
+{ return l4e_from_pfn(mfn_x(mfn), flags); }
+#endif
+#endif
+
+#define shadow_l1_table_offset(a) l1_table_offset(a)
+#define shadow_l2_table_offset(a) l2_table_offset(a)
+#define shadow_l3_table_offset(a) l3_table_offset(a)
+#define shadow_l4_table_offset(a) l4_table_offset(a)
+
+/**************************************************************************/
+/* Access to the linear mapping of shadow page tables. */
+
+/* Offsets into each level of the linear mapping for a virtual address. */
+#define shadow_l1_linear_offset(_a)                                           \
+        (((_a) & VADDR_MASK) >> SHADOW_L1_PAGETABLE_SHIFT)
+#define shadow_l2_linear_offset(_a)                                           \
+        (((_a) & VADDR_MASK) >> SHADOW_L2_PAGETABLE_SHIFT)
+#define shadow_l3_linear_offset(_a)                                           \
+        (((_a) & VADDR_MASK) >> SHADOW_L3_PAGETABLE_SHIFT)
+#define shadow_l4_linear_offset(_a)                                           \
+        (((_a) & VADDR_MASK) >> SHADOW_L4_PAGETABLE_SHIFT)
+
+/* Where to find each level of the linear mapping.  For PV guests, we use 
+ * the shadow linear-map self-entry as many times as we need.  For HVM 
+ * guests, the shadow doesn't have a linear-map self-entry so we must use 
+ * the monitor-table's linear-map entry N-1 times and then the shadow-map 
+ * entry once. */
+#define __sh2_linear_l1_table ((shadow_l1e_t *)(SH_LINEAR_PT_VIRT_START))
+#define __sh2_linear_l2_table ((shadow_l2e_t *)                               \
+    (__sh2_linear_l1_table + shadow_l1_linear_offset(SH_LINEAR_PT_VIRT_START)))
+
+// shadow linear L3 and L4 tables only exist in 4 level paging...
+#if SHADOW_PAGING_LEVELS == 4
+#define __sh2_linear_l3_table ((shadow_l3e_t *)                               \
+    (__sh2_linear_l2_table + shadow_l2_linear_offset(SH_LINEAR_PT_VIRT_START)))
+#define __sh2_linear_l4_table ((shadow_l4e_t *)                               \
+    (__sh2_linear_l3_table + shadow_l3_linear_offset(SH_LINEAR_PT_VIRT_START)))
+#endif
+
+#define sh2_linear_l1_table(v) ({ \
+    ASSERT(current == (v)); \
+    __sh2_linear_l1_table; \
+})
+
+#define sh2_linear_l2_table(v) ({ \
+    ASSERT(current == (v)); \
+    ((shadow_l2e_t *) \
+     (hvm_guest(v) ? __linear_l1_table : __sh2_linear_l1_table) + \
+     shadow_l1_linear_offset(SH_LINEAR_PT_VIRT_START)); \
+})
+
+// shadow linear L3 and L4 tables only exist in 4 level paging...
+#if SHADOW_PAGING_LEVELS == 4
+#define sh2_linear_l3_table(v) ({ \
+    ASSERT(current == (v)); \
+    ((shadow_l3e_t *) \
+     (hvm_guest(v) ? __linear_l2_table : __sh2_linear_l2_table) + \
+      shadow_l2_linear_offset(SH_LINEAR_PT_VIRT_START)); \
+})
+
+// we use l4_pgentry_t instead of shadow_l4e_t below because shadow_l4e_t is
+// not defined for when xen_levels==4 & shadow_levels==3...
+#define sh2_linear_l4_table(v) ({ \
+    ASSERT(current == (v)); \
+    ((l4_pgentry_t *) \
+     (hvm_guest(v) ? __linear_l3_table : __sh2_linear_l3_table) + \
+      shadow_l3_linear_offset(SH_LINEAR_PT_VIRT_START)); \
+})
+#endif
+
+#if GUEST_PAGING_LEVELS == 2
+
+#include <asm/page-guest32.h>
+
+#define GUEST_L1_PAGETABLE_ENTRIES     1024
+#define GUEST_L2_PAGETABLE_ENTRIES     1024
+#define GUEST_L1_PAGETABLE_SHIFT         12
+#define GUEST_L2_PAGETABLE_SHIFT         22
+
+/* Type of the guest's frame numbers */
+TYPE_SAFE(u32,gfn)
+#define INVALID_GFN ((u32)(-1u))
+#define SH2_PRI_gfn "05x"
+
+/* Types of the guest's page tables */
+typedef l1_pgentry_32_t guest_l1e_t;
+typedef l2_pgentry_32_t guest_l2e_t;
+
+/* Access functions for them */
+static inline paddr_t guest_l1e_get_paddr(guest_l1e_t gl1e)
+{ return l1e_get_paddr_32(gl1e); }
+static inline paddr_t guest_l2e_get_paddr(guest_l2e_t gl2e)
+{ return l2e_get_paddr_32(gl2e); }
+
+static inline gfn_t guest_l1e_get_gfn(guest_l1e_t gl1e)
+{ return _gfn(l1e_get_paddr_32(gl1e) >> PAGE_SHIFT); }
+static inline gfn_t guest_l2e_get_gfn(guest_l2e_t gl2e)
+{ return _gfn(l2e_get_paddr_32(gl2e) >> PAGE_SHIFT); }
+
+static inline u32 guest_l1e_get_flags(guest_l1e_t gl1e)
+{ return l1e_get_flags_32(gl1e); }
+static inline u32 guest_l2e_get_flags(guest_l2e_t gl2e)
+{ return l2e_get_flags_32(gl2e); }
+
+static inline guest_l1e_t guest_l1e_add_flags(guest_l1e_t gl1e, u32 flags)
+{ l1e_add_flags_32(gl1e, flags); return gl1e; }
+static inline guest_l2e_t guest_l2e_add_flags(guest_l2e_t gl2e, u32 flags)
+{ l2e_add_flags_32(gl2e, flags); return gl2e; }
+
+static inline guest_l1e_t guest_l1e_from_gfn(gfn_t gfn, u32 flags)
+{ return l1e_from_pfn_32(gfn_x(gfn), flags); }
+static inline guest_l2e_t guest_l2e_from_gfn(gfn_t gfn, u32 flags)
+{ return l2e_from_pfn_32(gfn_x(gfn), flags); }
+
+#define guest_l1_table_offset(a) l1_table_offset_32(a)
+#define guest_l2_table_offset(a) l2_table_offset_32(a)
+
+/* The shadow types needed for the various levels. */
+#define PGC_SH2_l1_shadow  PGC_SH2_l1_32_shadow
+#define PGC_SH2_l2_shadow  PGC_SH2_l2_32_shadow
+#define PGC_SH2_fl1_shadow PGC_SH2_fl1_32_shadow
+
+#else /* GUEST_PAGING_LEVELS != 2 */
+
+#if GUEST_PAGING_LEVELS == 3
+#define GUEST_L1_PAGETABLE_ENTRIES      512
+#define GUEST_L2_PAGETABLE_ENTRIES      512
+#define GUEST_L3_PAGETABLE_ENTRIES        4
+#define GUEST_L1_PAGETABLE_SHIFT         12
+#define GUEST_L2_PAGETABLE_SHIFT         21
+#define GUEST_L3_PAGETABLE_SHIFT         30
+#else /* GUEST_PAGING_LEVELS == 4 */
+#define GUEST_L1_PAGETABLE_ENTRIES      512
+#define GUEST_L2_PAGETABLE_ENTRIES      512
+#define GUEST_L3_PAGETABLE_ENTRIES      512
+#define GUEST_L4_PAGETABLE_ENTRIES      512
+#define GUEST_L1_PAGETABLE_SHIFT         12
+#define GUEST_L2_PAGETABLE_SHIFT         21
+#define GUEST_L3_PAGETABLE_SHIFT         30
+#define GUEST_L4_PAGETABLE_SHIFT         39
+#endif
+
+/* Type of the guest's frame numbers */
+TYPE_SAFE(unsigned long,gfn)
+#define INVALID_GFN ((unsigned long)(-1ul))
+#define SH2_PRI_gfn "05lx"
+
+/* Types of the guest's page tables */
+typedef l1_pgentry_t guest_l1e_t;
+typedef l2_pgentry_t guest_l2e_t;
+typedef l3_pgentry_t guest_l3e_t;
+#if GUEST_PAGING_LEVELS >= 4
+typedef l4_pgentry_t guest_l4e_t;
+#endif
+
+/* Access functions for them */
+static inline paddr_t guest_l1e_get_paddr(guest_l1e_t gl1e)
+{ return l1e_get_paddr(gl1e); }
+static inline paddr_t guest_l2e_get_paddr(guest_l2e_t gl2e)
+{ return l2e_get_paddr(gl2e); }
+static inline paddr_t guest_l3e_get_paddr(guest_l3e_t gl3e)
+{ return l3e_get_paddr(gl3e); }
+#if GUEST_PAGING_LEVELS >= 4
+static inline paddr_t guest_l4e_get_paddr(guest_l4e_t gl4e)
+{ return l4e_get_paddr(gl4e); }
+#endif
+
+static inline gfn_t guest_l1e_get_gfn(guest_l1e_t gl1e)
+{ return _gfn(l1e_get_paddr(gl1e) >> PAGE_SHIFT); }
+static inline gfn_t guest_l2e_get_gfn(guest_l2e_t gl2e)
+{ return _gfn(l2e_get_paddr(gl2e) >> PAGE_SHIFT); }
+static inline gfn_t guest_l3e_get_gfn(guest_l3e_t gl3e)
+{ return _gfn(l3e_get_paddr(gl3e) >> PAGE_SHIFT); }
+#if GUEST_PAGING_LEVELS >= 4
+static inline gfn_t guest_l4e_get_gfn(guest_l4e_t gl4e)
+{ return _gfn(l4e_get_paddr(gl4e) >> PAGE_SHIFT); }
+#endif
+
+static inline u32 guest_l1e_get_flags(guest_l1e_t gl1e)
+{ return l1e_get_flags(gl1e); }
+static inline u32 guest_l2e_get_flags(guest_l2e_t gl2e)
+{ return l2e_get_flags(gl2e); }
+static inline u32 guest_l3e_get_flags(guest_l3e_t gl3e)
+{ return l3e_get_flags(gl3e); }
+#if GUEST_PAGING_LEVELS >= 4
+static inline u32 guest_l4e_get_flags(guest_l4e_t gl4e)
+{ return l4e_get_flags(gl4e); }
+#endif
+
+static inline guest_l1e_t guest_l1e_add_flags(guest_l1e_t gl1e, u32 flags)
+{ l1e_add_flags(gl1e, flags); return gl1e; }
+static inline guest_l2e_t guest_l2e_add_flags(guest_l2e_t gl2e, u32 flags)
+{ l2e_add_flags(gl2e, flags); return gl2e; }
+static inline guest_l3e_t guest_l3e_add_flags(guest_l3e_t gl3e, u32 flags)
+{ l3e_add_flags(gl3e, flags); return gl3e; }
+#if GUEST_PAGING_LEVELS >= 4
+static inline guest_l4e_t guest_l4e_add_flags(guest_l4e_t gl4e, u32 flags)
+{ l4e_add_flags(gl4e, flags); return gl4e; }
+#endif
+
+static inline guest_l1e_t guest_l1e_from_gfn(gfn_t gfn, u32 flags)
+{ return l1e_from_pfn(gfn_x(gfn), flags); }
+static inline guest_l2e_t guest_l2e_from_gfn(gfn_t gfn, u32 flags)
+{ return l2e_from_pfn(gfn_x(gfn), flags); }
+static inline guest_l3e_t guest_l3e_from_gfn(gfn_t gfn, u32 flags)
+{ return l3e_from_pfn(gfn_x(gfn), flags); }
+#if GUEST_PAGING_LEVELS >= 4
+static inline guest_l4e_t guest_l4e_from_gfn(gfn_t gfn, u32 flags)
+{ return l4e_from_pfn(gfn_x(gfn), flags); }
+#endif
+
+#define guest_l1_table_offset(a) l1_table_offset(a)
+#define guest_l2_table_offset(a) l2_table_offset(a)
+#define guest_l3_table_offset(a) l3_table_offset(a)
+#define guest_l4_table_offset(a) l4_table_offset(a)
+
+/* The shadow types needed for the various levels. */
+#if GUEST_PAGING_LEVELS == 3
+#define PGC_SH2_l1_shadow  PGC_SH2_l1_pae_shadow
+#define PGC_SH2_fl1_shadow PGC_SH2_fl1_pae_shadow
+#define PGC_SH2_l2_shadow  PGC_SH2_l2_pae_shadow
+#define PGC_SH2_l2h_shadow PGC_SH2_l2h_pae_shadow
+#define PGC_SH2_l3_shadow  PGC_SH2_l3_pae_shadow
+#else
+#define PGC_SH2_l1_shadow  PGC_SH2_l1_64_shadow
+#define PGC_SH2_fl1_shadow PGC_SH2_fl1_64_shadow
+#define PGC_SH2_l2_shadow  PGC_SH2_l2_64_shadow
+#define PGC_SH2_l3_shadow  PGC_SH2_l3_64_shadow
+#define PGC_SH2_l4_shadow  PGC_SH2_l4_64_shadow
+#endif
+
+#endif /* GUEST_PAGING_LEVELS != 2 */
+
+#define VALID_GFN(m) (m != INVALID_GFN)
+
+static inline int
+valid_gfn(gfn_t m)
+{
+    return VALID_GFN(gfn_x(m));
+}
+
+#if GUEST_PAGING_LEVELS == 2
+#define PGC_SH2_guest_root_type PGC_SH2_l2_32_shadow
+#elif GUEST_PAGING_LEVELS == 3
+#define PGC_SH2_guest_root_type PGC_SH2_l3_pae_shadow
+#else
+#define PGC_SH2_guest_root_type PGC_SH2_l4_64_shadow
+#endif
+
+/* Translation between mfns and gfns */
+static inline mfn_t
+vcpu_gfn_to_mfn(struct vcpu *v, gfn_t gfn)
+{
+    return sh2_vcpu_gfn_to_mfn(v, gfn_x(gfn));
+} 
+
+static inline gfn_t
+mfn_to_gfn(struct domain *d, mfn_t mfn)
+{
+    return _gfn(sh2_mfn_to_gfn(d, mfn));
+}
+
+static inline paddr_t
+gfn_to_paddr(gfn_t gfn)
+{
+    return ((paddr_t)gfn_x(gfn)) << PAGE_SHIFT;
+}
+
+/* Type used for recording a walk through guest pagetables.  It is
+ * filled in by the pagetable walk function, and also used as a cache
+ * for later walks.  
+ * Any non-null pointer in this structure represents a mapping of guest
+ * memory.  We must always call walk_init() before using a walk_t, and 
+ * call walk_unmap() when we're done. 
+ * The "Effective l1e" field is used when there isn't an l1e to point to, 
+ * but we have fabricated an l1e for propagation to the shadow (e.g., 
+ * for splintering guest superpages into many shadow l1 entries).  */
+typedef struct shadow2_walk_t walk_t;
+struct shadow2_walk_t 
+{
+    unsigned long va;           /* Address we were looking for */
+#if GUEST_PAGING_LEVELS >= 3
+#if GUEST_PAGING_LEVELS >= 4
+    guest_l4e_t *l4e;           /* Pointer to guest's level 4 entry */
+#endif
+    guest_l3e_t *l3e;           /* Pointer to guest's level 3 entry */
+#endif
+    guest_l2e_t *l2e;           /* Pointer to guest's level 2 entry */
+    guest_l1e_t *l1e;           /* Pointer to guest's level 1 entry */
+    guest_l1e_t eff_l1e;        /* Effective level 1 entry */
+#if GUEST_PAGING_LEVELS >= 3
+#if GUEST_PAGING_LEVELS >= 4
+    mfn_t l4mfn;                /* MFN that the level 4 entry is in */
+#endif
+    mfn_t l3mfn;                /* MFN that the level 3 entry is in */
+#endif
+    mfn_t l2mfn;                /* MFN that the level 2 entry is in */
+    mfn_t l1mfn;                /* MFN that the level 1 entry is in */
+};
+
+
+/* X86 error code bits:
+ * These bits certainly ought to be defined somewhere other than here,
+ * but until that place is determined, here they sit.
+ *
+ * "PFEC" == "Page Fault Error Code"
+ */
+#define X86_PFEC_PRESENT            1  /* 0 == page was not present */
+#define X86_PFEC_WRITE_FAULT        2  /* 0 == reading, 1 == writing */
+#define X86_PFEC_SUPERVISOR_FAULT   4  /* 0 == supervisor-mode, 1 == user */
+#define X86_PFEC_RESERVED_BIT_FAULT 8  /* 1 == reserved bits set in pte */
+#define X86_PFEC_INSN_FETCH_FAULT  16  /* 0 == normal, 1 == instr'n fetch */
+
+/* macros for dealing with the naming of the internal function names of the
+ * shadow code's external entry points.
+ */
+#define INTERNAL_NAME(name) \
+    SHADOW2_INTERNAL_NAME(name, SHADOW_PAGING_LEVELS, GUEST_PAGING_LEVELS)
+
+/* macros for renaming the primary entry points, so that they are more
+ * easily distinguished from a debugger
+ */
+#define sh2_page_fault              INTERNAL_NAME(sh2_page_fault)
+#define sh2_invlpg                  INTERNAL_NAME(sh2_invlpg)
+#define sh2_gva_to_gpa              INTERNAL_NAME(sh2_gva_to_gpa)
+#define sh2_gva_to_gfn              INTERNAL_NAME(sh2_gva_to_gfn)
+#define sh2_update_cr3              INTERNAL_NAME(sh2_update_cr3)
+#define sh2_remove_write_access     INTERNAL_NAME(sh2_remove_write_access)
+#define sh2_remove_all_mappings     INTERNAL_NAME(sh2_remove_all_mappings)
+#define sh2_remove_l1_shadow        INTERNAL_NAME(sh2_remove_l1_shadow)
+#define sh2_remove_l2_shadow        INTERNAL_NAME(sh2_remove_l2_shadow)
+#define sh2_remove_l3_shadow        INTERNAL_NAME(sh2_remove_l3_shadow)
+#define sh2_map_and_validate_gl4e   INTERNAL_NAME(sh2_map_and_validate_gl4e)
+#define sh2_map_and_validate_gl3e   INTERNAL_NAME(sh2_map_and_validate_gl3e)
+#define sh2_map_and_validate_gl2e   INTERNAL_NAME(sh2_map_and_validate_gl2e)
+#define sh2_map_and_validate_gl2he  INTERNAL_NAME(sh2_map_and_validate_gl2he)
+#define sh2_map_and_validate_gl1e   INTERNAL_NAME(sh2_map_and_validate_gl1e)
+#define sh2_destroy_l4_shadow       INTERNAL_NAME(sh2_destroy_l4_shadow)
+#define sh2_destroy_l3_shadow       INTERNAL_NAME(sh2_destroy_l3_shadow)
+#define sh2_destroy_l3_subshadow    INTERNAL_NAME(sh2_destroy_l3_subshadow)
+#define sh2_unpin_all_l3_subshadows INTERNAL_NAME(sh2_unpin_all_l3_subshadows)
+#define sh2_destroy_l2_shadow       INTERNAL_NAME(sh2_destroy_l2_shadow)
+#define sh2_destroy_l1_shadow       INTERNAL_NAME(sh2_destroy_l1_shadow)
+#define sh2_unhook_32b_mappings     INTERNAL_NAME(sh2_unhook_32b_mappings)
+#define sh2_unhook_pae_mappings     INTERNAL_NAME(sh2_unhook_pae_mappings)
+#define sh2_unhook_64b_mappings     INTERNAL_NAME(sh2_unhook_64b_mappings)
+#define shadow2_entry               INTERNAL_NAME(shadow2_entry)
+#define sh2_detach_old_tables       INTERNAL_NAME(sh2_detach_old_tables)
+#define sh2_x86_emulate_write       INTERNAL_NAME(sh2_x86_emulate_write)
+#define sh2_x86_emulate_cmpxchg     INTERNAL_NAME(sh2_x86_emulate_cmpxchg)
+#define sh2_x86_emulate_cmpxchg8b   INTERNAL_NAME(sh2_x86_emulate_cmpxchg8b)
+#define sh2_audit_l1_table          INTERNAL_NAME(sh2_audit_l1_table)
+#define sh2_audit_fl1_table         INTERNAL_NAME(sh2_audit_fl1_table)
+#define sh2_audit_l2_table          INTERNAL_NAME(sh2_audit_l2_table)
+#define sh2_audit_l3_table          INTERNAL_NAME(sh2_audit_l3_table)
+#define sh2_audit_l4_table          INTERNAL_NAME(sh2_audit_l4_table)
+#define sh2_guess_wrmap             INTERNAL_NAME(sh2_guess_wrmap)
+#define sh2_clear_shadow_entry      INTERNAL_NAME(sh2_clear_shadow_entry)
+
+/* sh2_make_monitor_table only depends on the number of shadow levels */
+#define sh2_make_monitor_table                          \
+        SHADOW2_INTERNAL_NAME(sh2_make_monitor_table,   \
+                              SHADOW_PAGING_LEVELS,     \
+                              SHADOW_PAGING_LEVELS)
+#define sh2_destroy_monitor_table                               \
+        SHADOW2_INTERNAL_NAME(sh2_destroy_monitor_table,        \
+                              SHADOW_PAGING_LEVELS,             \
+                              SHADOW_PAGING_LEVELS)
+
+
+#if GUEST_PAGING_LEVELS == 3
+/*
+ * Accounting information stored in the shadow of PAE Guest L3 pages.
+ * Because these "L3 pages" are only 32-bytes, it is inconvenient to keep
+ * various refcounts, etc., on the page_info of their page.  We provide extra
+ * bookkeeping space in the shadow itself, and this is the structure
+ * definition for that bookkeeping information.
+ */
+struct pae_l3_bookkeeping {
+    u32 vcpus;                  /* bitmap of which vcpus are currently storing
+                                 * copies of this 32-byte page */
+    u32 refcount;               /* refcount for this 32-byte page */
+    u8 pinned;                  /* is this 32-byte page pinned or not? */
+};
+
+// Convert a shadow entry pointer into a pae_l3_bookkeeping pointer.
+#define sl3p_to_info(_ptr) ((struct pae_l3_bookkeeping *)         \
+                            (((unsigned long)(_ptr) & ~31) + 32))
+
+static void sh2_destroy_l3_subshadow(struct vcpu *v, 
+                                     shadow_l3e_t *sl3e);
+
+/* Increment a subshadow ref
+ * Called with a pointer to the subshadow, and the mfn of the
+ * *first* page of the overall shadow. */
+static inline void sh2_get_ref_l3_subshadow(shadow_l3e_t *sl3e, mfn_t smfn)
+{
+    struct pae_l3_bookkeeping *bk = sl3p_to_info(sl3e);
+
+    /* First ref to the subshadow takes a ref to the full shadow */
+    if ( bk->refcount == 0 ) 
+        sh2_get_ref(smfn, 0);
+    if ( unlikely(++(bk->refcount) == 0) )
+    {
+        SHADOW2_PRINTK("shadow l3 subshadow ref overflow, smfn=%" SH2_PRI_mfn " sh=%p\n", 
+                       mfn_x(smfn), sl3e);
+        domain_crash_synchronous();
+    }
+}
+
+/* Decrement a subshadow ref.
+ * Called with a pointer to the subshadow, and the mfn of the
+ * *first* page of the overall shadow.  Calling this may cause the 
+ * entire shadow to disappear, so the caller must immediately unmap 
+ * the pointer after calling. */ 
+static inline void sh2_put_ref_l3_subshadow(struct vcpu *v, 
+                                            shadow_l3e_t *sl3e,
+                                            mfn_t smfn)
+{
+    struct pae_l3_bookkeeping *bk;
+
+    bk = sl3p_to_info(sl3e);
+
+    ASSERT(bk->refcount > 0);
+    if ( --(bk->refcount) == 0 )
+    {
+        /* Need to destroy this subshadow */
+        sh2_destroy_l3_subshadow(v, sl3e);
+        /* Last ref to the subshadow had a ref to the full shadow */
+        sh2_put_ref(v, smfn, 0);
+    }
+}
+
+/* Pin a subshadow 
+ * Called with a pointer to the subshadow, and the mfn of the
+ * *first* page of the overall shadow. */
+static inline void sh2_pin_l3_subshadow(shadow_l3e_t *sl3e, mfn_t smfn)
+{
+    struct pae_l3_bookkeeping *bk = sl3p_to_info(sl3e);
+
+#if 0
+    debugtrace_printk("%s smfn=%05lx offset=%ld\n",
+                      __func__, mfn_x(smfn),
+                      ((unsigned long)sl3e & ~PAGE_MASK) / 64);
+#endif
+
+    if ( !bk->pinned )
+    {
+        bk->pinned = 1;
+        sh2_get_ref_l3_subshadow(sl3e, smfn);
+    }
+}
+
+/* Unpin a sub-shadow. 
+ * Called with a pointer to the subshadow, and the mfn of the
+ * *first* page of the overall shadow.  Calling this may cause the 
+ * entire shadow to disappear, so the caller must immediately unmap 
+ * the pointer after calling. */ 
+static inline void sh2_unpin_l3_subshadow(struct vcpu *v, 
+                                          shadow_l3e_t *sl3e,
+                                          mfn_t smfn)
+{
+    struct pae_l3_bookkeeping *bk = sl3p_to_info(sl3e);
+
+#if 0
+    debugtrace_printk("%s smfn=%05lx offset=%ld\n",
+                      __func__, mfn_x(smfn),
+                      ((unsigned long)sl3e & ~PAGE_MASK) / 64);
+#endif
+
+    if ( bk->pinned )
+    {
+        bk->pinned = 0;
+        sh2_put_ref_l3_subshadow(v, sl3e, smfn);
+    }
+}
+
+#endif /* GUEST_PAGING_LEVELS == 3 */
+
+#if SHADOW_PAGING_LEVELS == 3
+#define MFN_FITS_IN_HVM_CR3(_MFN) !(mfn_x(_MFN) >> 20)
+#endif
+
+#if SHADOW_PAGING_LEVELS == 2
+#define SH2_PRI_pte "08x"
+#else /* SHADOW_PAGING_LEVELS >= 3 */
+#ifndef __x86_64__
+#define SH2_PRI_pte "016llx"
+#else
+#define SH2_PRI_pte "016lx"
+#endif
+#endif /* SHADOW_PAGING_LEVELS >= 3 */
+
+#if GUEST_PAGING_LEVELS == 2
+#define SH2_PRI_gpte "08x"
+#else /* GUEST_PAGING_LEVELS >= 3 */
+#ifndef __x86_64__
+#define SH2_PRI_gpte "016llx"
+#else
+#define SH2_PRI_gpte "016lx"
+#endif
+#endif /* GUEST_PAGING_LEVELS >= 3 */
+
+static inline u32
+accumulate_guest_flags(walk_t *gw)
+{
+    u32 accumulated_flags;
+
+    // We accumulate the permission flags with bitwise ANDing.
+    // This works for the PRESENT bit, RW bit, and USER bit.
+    // For the NX bit, however, the polarity is wrong, so we accumulate the
+    // inverse of the NX bit.
+    //
+    accumulated_flags =  guest_l1e_get_flags(gw->eff_l1e) ^ _PAGE_NX_BIT;
+    accumulated_flags &= guest_l2e_get_flags(*gw->l2e) ^ _PAGE_NX_BIT;
+
+    // Note that PAE guests do not have USER or RW or NX bits in their L3s.
+    //
+#if GUEST_PAGING_LEVELS == 3
+    accumulated_flags &=
+        ~_PAGE_PRESENT | (guest_l3e_get_flags(*gw->l3e) & _PAGE_PRESENT);
+#elif GUEST_PAGING_LEVELS >= 4
+    accumulated_flags &= guest_l3e_get_flags(*gw->l3e) ^ _PAGE_NX_BIT;
+    accumulated_flags &= guest_l4e_get_flags(*gw->l4e) ^ _PAGE_NX_BIT;
+#endif
+
+    // Finally, revert the NX bit back to its original polarity
+    accumulated_flags ^= _PAGE_NX_BIT;
+
+    return accumulated_flags;
+}
+
+#endif /* _XEN_SHADOW2_TYPES_H */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */

xen/include/asm-x86/shadow2.h

@@ -0,0 +1,627 @@
+/******************************************************************************
+ * include/asm-x86/shadow2.h
+ * 
+ * Parts of this code are Copyright (c) 2006 by XenSource Inc.
+ * Parts of this code are Copyright (c) 2006 by Michael A Fetterman
+ * Parts based on earlier work by Michael A Fetterman, Ian Pratt et al.
+ * 
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#ifndef _XEN_SHADOW2_H
+#define _XEN_SHADOW2_H
+
+#include <public/dom0_ops.h> 
+#include <xen/sched.h>
+#include <xen/perfc.h>
+#include <asm/flushtlb.h>
+
+/* Shadow PT operation mode : shadow-mode variable in arch_domain. */
+
+#define SHM2_shift 10
+/* We're in one of the shadow modes */
+#define SHM2_enable    (DOM0_SHADOW2_CONTROL_FLAG_ENABLE << SHM2_shift)
+/* Refcounts based on shadow tables instead of guest tables */
+#define SHM2_refcounts (DOM0_SHADOW2_CONTROL_FLAG_REFCOUNT << SHM2_shift)
+/* Enable log dirty mode */
+#define SHM2_log_dirty (DOM0_SHADOW2_CONTROL_FLAG_LOG_DIRTY << SHM2_shift)
+/* Xen does p2m translation, not guest */
+#define SHM2_translate (DOM0_SHADOW2_CONTROL_FLAG_TRANSLATE << SHM2_shift)
+/* Xen does not steal address space from the domain for its own booking;
+ * requires VT or similar mechanisms */
+#define SHM2_external  (DOM0_SHADOW2_CONTROL_FLAG_EXTERNAL << SHM2_shift)
+
+#define shadow2_mode_enabled(_d)   ((_d)->arch.shadow2_mode)
+#define shadow2_mode_refcounts(_d) ((_d)->arch.shadow2_mode & SHM2_refcounts)
+#define shadow2_mode_log_dirty(_d) ((_d)->arch.shadow2_mode & SHM2_log_dirty)
+#define shadow2_mode_translate(_d) ((_d)->arch.shadow2_mode & SHM2_translate)
+#define shadow2_mode_external(_d)  ((_d)->arch.shadow2_mode & SHM2_external)
+
+/* Xen traps & emulates all reads of all page table pages:
+ *not yet supported
+ */
+#define shadow2_mode_trap_reads(_d) ({ (void)(_d); 0; })
+
+// flags used in the return value of the shadow_set_lXe() functions...
+#define SHADOW2_SET_CHANGED            0x1
+#define SHADOW2_SET_FLUSH              0x2
+#define SHADOW2_SET_ERROR              0x4
+#define SHADOW2_SET_L3PAE_RECOPY       0x8
+
+// How do we tell that we have a 32-bit PV guest in a 64-bit Xen?
+#ifdef __x86_64__
+#define pv_32bit_guest(_v) 0 // not yet supported
+#else
+#define pv_32bit_guest(_v) !hvm_guest(v)
+#endif
+
+/* The shadow2 lock.
+ *
+ * This lock is per-domain.  It is intended to allow us to make atomic
+ * updates to the software TLB that the shadow tables provide.
+ * 
+ * Specifically, it protects:
+ *   - all changes to shadow page table pages
+ *   - the shadow hash table
+ *   - the shadow page allocator 
+ *   - all changes to guest page table pages; if/when the notion of
+ *     out-of-sync pages is added to this code, then the shadow lock is
+ *     protecting all guest page table pages which are not listed as
+ *     currently as both guest-writable and out-of-sync...
+ *     XXX -- need to think about this relative to writable page tables.
+ *   - all changes to the page_info->tlbflush_timestamp
+ *   - the page_info->count fields on shadow pages
+ *   - the shadow dirty bit array and count
+ *   - XXX
+ */
+#ifndef CONFIG_SMP
+#error shadow2.h currently requires CONFIG_SMP
+#endif
+
+#define shadow2_lock_init(_d)                                   \
+    do {                                                        \
+        spin_lock_init(&(_d)->arch.shadow2_lock);               \
+        (_d)->arch.shadow2_locker = -1;                         \
+        (_d)->arch.shadow2_locker_function = "nobody";          \
+    } while (0)
+
+#define shadow2_lock_is_acquired(_d)                            \
+    (current->processor == (_d)->arch.shadow2_locker)
+
+#define shadow2_lock(_d)                                                 \
+    do {                                                                 \
+        if ( unlikely((_d)->arch.shadow2_locker == current->processor) ) \
+        {                                                                \
+            printk("Error: shadow2 lock held by %s\n",                   \
+                   (_d)->arch.shadow2_locker_function);                  \
+            BUG();                                                       \
+        }                                                                \
+        spin_lock(&(_d)->arch.shadow2_lock);                             \
+        ASSERT((_d)->arch.shadow2_locker == -1);                         \
+        (_d)->arch.shadow2_locker = current->processor;                  \
+        (_d)->arch.shadow2_locker_function = __func__;                   \
+    } while (0)
+
+#define shadow2_unlock(_d)                                              \
+    do {                                                                \
+        ASSERT((_d)->arch.shadow2_locker == current->processor);        \
+        (_d)->arch.shadow2_locker = -1;                                 \
+        (_d)->arch.shadow2_locker_function = "nobody";                  \
+        spin_unlock(&(_d)->arch.shadow2_lock);                          \
+    } while (0)
+
+/* 
+ * Levels of self-test and paranoia
+ * XXX should go in config files somewhere?  
+ */
+#define SHADOW2_AUDIT_HASH           0x01  /* Check current hash bucket */
+#define SHADOW2_AUDIT_HASH_FULL      0x02  /* Check every hash bucket */
+#define SHADOW2_AUDIT_ENTRIES        0x04  /* Check this walk's shadows */
+#define SHADOW2_AUDIT_ENTRIES_FULL   0x08  /* Check every shadow */
+#define SHADOW2_AUDIT_ENTRIES_MFNS   0x10  /* Check gfn-mfn map in shadows */
+#define SHADOW2_AUDIT_P2M            0x20  /* Check the p2m table */
+
+#ifdef NDEBUG
+#define SHADOW2_AUDIT                   0
+#define SHADOW2_AUDIT_ENABLE            0
+#else
+#define SHADOW2_AUDIT                0x15  /* Basic audit of all except p2m. */
+#define SHADOW2_AUDIT_ENABLE         shadow2_audit_enable
+extern int shadow2_audit_enable;
+#endif
+
+/* 
+ * Levels of optimization
+ * XXX should go in config files somewhere?  
+ */
+#define SH2OPT_WRITABLE_HEURISTIC  0x01  /* Guess at RW PTEs via linear maps */
+#define SH2OPT_EARLY_UNSHADOW      0x02  /* Unshadow l1s on fork or exit */
+
+#define SHADOW2_OPTIMIZATIONS      0x03
+
+
+/* With shadow pagetables, the different kinds of address start 
+ * to get get confusing.
+ * 
+ * Virtual addresses are what they usually are: the addresses that are used 
+ * to accessing memory while the guest is running.  The MMU translates from 
+ * virtual addresses to machine addresses. 
+ * 
+ * (Pseudo-)physical addresses are the abstraction of physical memory the
+ * guest uses for allocation and so forth.  For the purposes of this code, 
+ * we can largely ignore them.
+ *
+ * Guest frame numbers (gfns) are the entries that the guest puts in its
+ * pagetables.  For normal paravirtual guests, they are actual frame numbers,
+ * with the translation done by the guest.  
+ * 
+ * Machine frame numbers (mfns) are the entries that the hypervisor puts
+ * in the shadow page tables.
+ *
+ * Elsewhere in the xen code base, the name "gmfn" is generally used to refer
+ * to a "machine frame number, from the guest's perspective", or in other
+ * words, pseudo-physical frame numbers.  However, in the shadow code, the
+ * term "gmfn" means "the mfn of a guest page"; this combines naturally with
+ * other terms such as "smfn" (the mfn of a shadow page), gl2mfn (the mfn of a
+ * guest L2 page), etc...
+ */
+
+/* With this defined, we do some ugly things to force the compiler to
+ * give us type safety between mfns and gfns and other integers.
+ * TYPE_SAFE(int foo) defines a foo_t, and _foo() and foo_x() functions 
+ * that translate beween int and foo_t.
+ * 
+ * It does have some performance cost because the types now have 
+ * a different storage attribute, so may not want it on all the time. */
+#ifndef NDEBUG
+#define TYPE_SAFETY 1
+#endif
+
+#ifdef TYPE_SAFETY
+#define TYPE_SAFE(_type,_name)                                  \
+typedef struct { _type _name; } _name##_t;                      \
+static inline _name##_t _##_name(_type n) { return (_name##_t) { n }; } \
+static inline _type _name##_x(_name##_t n) { return n._name; }
+#else
+#define TYPE_SAFE(_type,_name)                                          \
+typedef _type _name##_t;                                                \
+static inline _name##_t _##_name(_type n) { return n; }                 \
+static inline _type _name##_x(_name##_t n) { return n; }
+#endif
+
+TYPE_SAFE(unsigned long,mfn)
+#define SH2_PRI_mfn "05lx"
+
+static inline int
+valid_mfn(mfn_t m)
+{
+    return VALID_MFN(mfn_x(m));
+}
+
+static inline mfn_t
+pagetable_get_mfn(pagetable_t pt)
+{
+    return _mfn(pagetable_get_pfn(pt));
+}
+
+static inline pagetable_t
+pagetable_from_mfn(mfn_t mfn)
+{
+    return pagetable_from_pfn(mfn_x(mfn));
+}
+
+static inline int
+shadow2_vcpu_mode_translate(struct vcpu *v)
+{
+    // Returns true if this VCPU needs to be using the P2M table to translate
+    // between GFNs and MFNs.
+    //
+    // This is true of translated HVM domains on a vcpu which has paging
+    // enabled.  (HVM vcpu's with paging disabled are using the p2m table as
+    // its paging table, so no translation occurs in this case.)
+    //
+    return v->vcpu_flags & VCPUF_shadow2_translate;
+}
+
+
+/**************************************************************************/
+/* Mode-specific entry points into the shadow code */
+
+struct x86_emulate_ctxt;
+struct shadow2_entry_points {
+    int           (*page_fault            )(struct vcpu *v, unsigned long va,
+                                            struct cpu_user_regs *regs);
+    int           (*invlpg                )(struct vcpu *v, unsigned long va);
+    unsigned long (*gva_to_gpa            )(struct vcpu *v, unsigned long va);
+    unsigned long (*gva_to_gfn            )(struct vcpu *v, unsigned long va);
+    void          (*update_cr3            )(struct vcpu *v);
+    int           (*map_and_validate_gl1e )(struct vcpu *v, mfn_t gmfn,
+                                            void *new_guest_entry, u32 size);
+    int           (*map_and_validate_gl2e )(struct vcpu *v, mfn_t gmfn,
+                                            void *new_guest_entry, u32 size);
+    int           (*map_and_validate_gl2he)(struct vcpu *v, mfn_t gmfn,
+                                            void *new_guest_entry, u32 size);
+    int           (*map_and_validate_gl3e )(struct vcpu *v, mfn_t gmfn,
+                                            void *new_guest_entry, u32 size);
+    int           (*map_and_validate_gl4e )(struct vcpu *v, mfn_t gmfn,
+                                            void *new_guest_entry, u32 size);
+    void          (*detach_old_tables     )(struct vcpu *v);
+    int           (*x86_emulate_write     )(struct vcpu *v, unsigned long va,
+                                            void *src, u32 bytes,
+                                            struct x86_emulate_ctxt *ctxt);
+    int           (*x86_emulate_cmpxchg   )(struct vcpu *v, unsigned long va,
+                                            unsigned long old, 
+                                            unsigned long new,
+                                            unsigned int bytes,
+                                            struct x86_emulate_ctxt *ctxt);
+    int           (*x86_emulate_cmpxchg8b )(struct vcpu *v, unsigned long va,
+                                            unsigned long old_lo, 
+                                            unsigned long old_hi, 
+                                            unsigned long new_lo,
+                                            unsigned long new_hi,
+                                            struct x86_emulate_ctxt *ctxt);
+    mfn_t         (*make_monitor_table    )(struct vcpu *v);
+    void          (*destroy_monitor_table )(struct vcpu *v, mfn_t mmfn);
+#if SHADOW2_OPTIMIZATIONS & SH2OPT_WRITABLE_HEURISTIC
+    int           (*guess_wrmap           )(struct vcpu *v, 
+                                            unsigned long vaddr, mfn_t gmfn);
+#endif
+    /* For outsiders to tell what mode we're in */
+    unsigned int shadow_levels;
+    unsigned int guest_levels;
+};
+
+static inline int shadow2_guest_paging_levels(struct vcpu *v)
+{
+    ASSERT(v->arch.shadow2 != NULL);
+    return v->arch.shadow2->guest_levels;
+}
+
+/**************************************************************************/
+/* Entry points into the shadow code */
+
+/* Turning on shadow2 test mode */
+int shadow2_test_enable(struct domain *d);
+
+/* Handler for shadow control ops: enabling and disabling shadow modes, 
+ * and log-dirty bitmap ops all happen through here. */
+int shadow2_control_op(struct domain *d, 
+                       dom0_shadow_control_t *sc,
+                       XEN_GUEST_HANDLE(dom0_op_t) u_dom0_op);
+
+/* Call when destroying a domain */
+void shadow2_teardown(struct domain *d);
+
+/* Call once all of the references to the domain have gone away */
+void shadow2_final_teardown(struct domain *d);
+
+
+/* Mark a page as dirty in the bitmap */
+void sh2_do_mark_dirty(struct domain *d, mfn_t gmfn);
+static inline void mark_dirty(struct domain *d, unsigned long gmfn)
+{
+    if ( shadow2_mode_log_dirty(d) )
+    {
+        shadow2_lock(d);
+        sh2_do_mark_dirty(d, _mfn(gmfn));
+        shadow2_unlock(d);
+    }
+}
+
+/* Internal version, for when the shadow lock is already held */
+static inline void sh2_mark_dirty(struct domain *d, mfn_t gmfn)
+{
+    ASSERT(shadow2_lock_is_acquired(d));
+    if ( shadow2_mode_log_dirty(d) )
+        sh2_do_mark_dirty(d, gmfn);
+}
+
+static inline int
+shadow2_fault(unsigned long va, struct cpu_user_regs *regs)
+/* Called from pagefault handler in Xen, and from the HVM trap handlers
+ * for pagefaults.  Returns 1 if this fault was an artefact of the
+ * shadow code (and the guest should retry) or 0 if it is not (and the
+ * fault should be handled elsewhere or passed to the guest). */
+{
+    struct vcpu *v = current;
+    perfc_incrc(shadow2_fault);
+    return v->arch.shadow2->page_fault(v, va, regs);
+}
+
+static inline int
+shadow2_invlpg(struct vcpu *v, unsigned long va)
+/* Called when the guest requests an invlpg.  Returns 1 if the invlpg
+ * instruction should be issued on the hardware, or 0 if it's safe not
+ * to do so. */
+{
+    return v->arch.shadow2->invlpg(v, va);
+}
+
+static inline unsigned long
+shadow2_gva_to_gpa(struct vcpu *v, unsigned long va)
+/* Called to translate a guest virtual address to what the *guest*
+ * pagetables would map it to. */
+{
+    return v->arch.shadow2->gva_to_gpa(v, va);
+}
+
+static inline unsigned long
+shadow2_gva_to_gfn(struct vcpu *v, unsigned long va)
+/* Called to translate a guest virtual address to what the *guest*
+ * pagetables would map it to. */
+{
+    return v->arch.shadow2->gva_to_gfn(v, va);
+}
+
+static inline void
+shadow2_update_cr3(struct vcpu *v)
+/* Updates all the things that are derived from the guest's CR3. 
+ * Called when the guest changes CR3. */
+{
+    shadow2_lock(v->domain);
+    v->arch.shadow2->update_cr3(v);
+    shadow2_unlock(v->domain);
+}
+
+
+/* Should be called after CR3 is updated.
+ * Updates vcpu->arch.cr3 and, for HVM guests, vcpu->arch.hvm_vcpu.cpu_cr3.
+ * 
+ * Also updates other state derived from CR3 (vcpu->arch.guest_vtable,
+ * shadow_vtable, etc).
+ *
+ * Uses values found in vcpu->arch.(guest_table and guest_table_user), and
+ * for HVM guests, arch.monitor_table and hvm's guest CR3.
+ *
+ * Update ref counts to shadow tables appropriately.
+ * For PAE, relocate L3 entries, if necessary, into low memory.
+ */
+static inline void update_cr3(struct vcpu *v)
+{
+    unsigned long cr3_mfn=0;
+
+    if ( shadow2_mode_enabled(v->domain) )
+    {
+        shadow2_update_cr3(v);
+        return;
+    }
+
+#if CONFIG_PAGING_LEVELS == 4
+    if ( !(v->arch.flags & TF_kernel_mode) )
+        cr3_mfn = pagetable_get_pfn(v->arch.guest_table_user);
+    else
+#endif
+        cr3_mfn = pagetable_get_pfn(v->arch.guest_table);
+
+    /* Update vcpu->arch.cr3 */
+    BUG_ON(cr3_mfn == 0);
+    make_cr3(v, cr3_mfn);
+}
+
+extern void sh2_update_paging_modes(struct vcpu *v);
+
+/* Should be called to initialise paging structures if the paging mode
+ * has changed, and when bringing up a VCPU for the first time. */
+static inline void shadow2_update_paging_modes(struct vcpu *v)
+{
+    ASSERT(shadow2_mode_enabled(v->domain));
+    shadow2_lock(v->domain);
+    sh2_update_paging_modes(v);
+    shadow2_unlock(v->domain);
+}
+
+static inline void
+shadow2_detach_old_tables(struct vcpu *v)
+{
+    v->arch.shadow2->detach_old_tables(v);
+}
+
+static inline mfn_t
+shadow2_make_monitor_table(struct vcpu *v)
+{
+    return v->arch.shadow2->make_monitor_table(v);
+}
+
+static inline void
+shadow2_destroy_monitor_table(struct vcpu *v, mfn_t mmfn)
+{
+    v->arch.shadow2->destroy_monitor_table(v, mmfn);
+}
+
+/* Validate a pagetable change from the guest and update the shadows. */
+extern int shadow2_validate_guest_entry(struct vcpu *v, mfn_t gmfn,
+                                        void *new_guest_entry);
+
+/* Update the shadows in response to a pagetable write from a HVM guest */
+extern void shadow2_validate_guest_pt_write(struct vcpu *v, mfn_t gmfn, 
+                                            void *entry, u32 size);
+
+/* Remove all writeable mappings of a guest frame from the shadows.
+ * Returns non-zero if we need to flush TLBs. 
+ * level and fault_addr desribe how we found this to be a pagetable;
+ * level==0 means we have some other reason for revoking write access. */
+extern int shadow2_remove_write_access(struct vcpu *v, mfn_t readonly_mfn,
+                                       unsigned int level,
+                                       unsigned long fault_addr);
+
+/* Remove all mappings of the guest mfn from the shadows. 
+ * Returns non-zero if we need to flush TLBs. */
+extern int shadow2_remove_all_mappings(struct vcpu *v, mfn_t target_mfn);
+
+void
+shadow2_remove_all_shadows_and_parents(struct vcpu *v, mfn_t gmfn);
+/* This is a HVM page that we thing is no longer a pagetable.
+ * Unshadow it, and recursively unshadow pages that reference it. */
+
+/* Remove all shadows of the guest mfn. */
+extern void sh2_remove_shadows(struct vcpu *v, mfn_t gmfn, int all);
+static inline void shadow2_remove_all_shadows(struct vcpu *v, mfn_t gmfn)
+{
+    sh2_remove_shadows(v, gmfn, 1);
+}
+
+/* Add a page to a domain */
+void
+shadow2_guest_physmap_add_page(struct domain *d, unsigned long gfn,
+                               unsigned long mfn);
+
+/* Remove a page from a domain */
+void
+shadow2_guest_physmap_remove_page(struct domain *d, unsigned long gfn,
+                                  unsigned long mfn);
+
+/*
+ * Definitions for the shadow2_flags field in page_info.
+ * These flags are stored on *guest* pages...
+ * Bits 1-13 are encodings for the shadow types.
+ */
+#define PGC_SH2_type_to_index(_type) ((_type) >> PGC_SH2_type_shift)
+#define SH2F_page_type_mask \
+    (((1u << PGC_SH2_type_to_index(PGC_SH2_max_shadow + 1u)) - 1u) - \
+     ((1u << PGC_SH2_type_to_index(PGC_SH2_min_shadow)) - 1u))
+
+#define SH2F_L1_32   (1u << PGC_SH2_type_to_index(PGC_SH2_l1_32_shadow))
+#define SH2F_FL1_32  (1u << PGC_SH2_type_to_index(PGC_SH2_fl1_32_shadow))
+#define SH2F_L2_32   (1u << PGC_SH2_type_to_index(PGC_SH2_l2_32_shadow))
+#define SH2F_L1_PAE  (1u << PGC_SH2_type_to_index(PGC_SH2_l1_pae_shadow))
+#define SH2F_FL1_PAE (1u << PGC_SH2_type_to_index(PGC_SH2_fl1_pae_shadow))
+#define SH2F_L2_PAE  (1u << PGC_SH2_type_to_index(PGC_SH2_l2_pae_shadow))
+#define SH2F_L2H_PAE (1u << PGC_SH2_type_to_index(PGC_SH2_l2h_pae_shadow))
+#define SH2F_L3_PAE  (1u << PGC_SH2_type_to_index(PGC_SH2_l3_pae_shadow))
+#define SH2F_L1_64   (1u << PGC_SH2_type_to_index(PGC_SH2_l1_64_shadow))
+#define SH2F_FL1_64  (1u << PGC_SH2_type_to_index(PGC_SH2_fl1_64_shadow))
+#define SH2F_L2_64   (1u << PGC_SH2_type_to_index(PGC_SH2_l2_64_shadow))
+#define SH2F_L3_64   (1u << PGC_SH2_type_to_index(PGC_SH2_l3_64_shadow))
+#define SH2F_L4_64   (1u << PGC_SH2_type_to_index(PGC_SH2_l4_64_shadow))
+
+/* Used for hysteresis when automatically unhooking mappings on fork/exit */
+#define SH2F_unhooked_mappings (1u<<31)
+
+/* 
+ * Allocation of shadow pages 
+ */
+
+/* Return the minumum acceptable number of shadow pages a domain needs */
+unsigned int shadow2_min_acceptable_pages(struct domain *d);
+
+/* Set the pool of shadow pages to the required number of MB.
+ * Input will be rounded up to at least min_acceptable_shadow_pages().
+ * Returns 0 for success, 1 for failure. */
+unsigned int shadow2_set_allocation(struct domain *d, 
+                                    unsigned int megabytes,
+                                    int *preempted);
+
+/* Return the size of the shadow2 pool, rounded up to the nearest MB */
+static inline unsigned int shadow2_get_allocation(struct domain *d)
+{
+    unsigned int pg = d->arch.shadow2_total_pages;
+    return ((pg >> (20 - PAGE_SHIFT))
+            + ((pg & ((1 << (20 - PAGE_SHIFT)) - 1)) ? 1 : 0));
+}
+
+/*
+ * Linked list for chaining entries in the shadow hash table. 
+ */
+struct shadow2_hash_entry {
+    struct shadow2_hash_entry *next;
+    mfn_t smfn;                 /* MFN of the shadow */
+#ifdef _x86_64_ /* Shorten 'n' so we don't waste a whole word on storing 't' */
+    unsigned long n:56;         /* MFN of guest PT or GFN of guest superpage */
+#else
+    unsigned long n;            /* MFN of guest PT or GFN of guest superpage */
+#endif
+    unsigned char t;            /* shadow type bits, or 0 for empty */
+};
+
+#define SHADOW2_HASH_BUCKETS 251
+/* Other possibly useful primes are 509, 1021, 2039, 4093, 8191, 16381 */
+
+
+#if SHADOW2_OPTIMIZATIONS & SH2OPT_CACHE_WALKS
+/* Optimization: cache the results of guest walks.  This helps with MMIO
+ * and emulated writes, which tend to issue very similar walk requests
+ * repeatedly.  We keep the results of the last few walks, and blow
+ * away the cache on guest cr3 write, mode change, or page fault. */
+
+#define SH2_WALK_CACHE_ENTRIES 4
+
+/* Rather than cache a guest walk, which would include mapped pointers 
+ * to pages, we cache what a TLB would remember about the walk: the 
+ * permissions and the l1 gfn */
+struct shadow2_walk_cache {
+    unsigned long va;           /* The virtual address (or 0 == unused) */
+    unsigned long gfn;          /* The gfn from the effective l1e   */
+    u32 permissions;            /* The aggregated permission bits   */
+};
+#endif
+
+
+/**************************************************************************/
+/* Guest physmap (p2m) support */
+
+/* Walk another domain's P2M table, mapping pages as we go */
+extern mfn_t
+sh2_gfn_to_mfn_foreign(struct domain *d, unsigned long gpfn);
+
+
+/* General conversion function from gfn to mfn */
+static inline mfn_t
+sh2_gfn_to_mfn(struct domain *d, unsigned long gfn)
+{
+    if ( !shadow2_mode_translate(d) )
+        return _mfn(gfn);
+    else if ( likely(current->domain == d) )
+        return _mfn(get_mfn_from_gpfn(gfn));
+    else
+        return sh2_gfn_to_mfn_foreign(d, gfn);
+}
+
+// vcpu-specific version of gfn_to_mfn().  This is where we hide the dirty
+// little secret that, for hvm guests with paging disabled, nearly all of the
+// shadow code actually think that the guest is running on *untranslated* page
+// tables (which is actually domain->phys_table).
+//
+static inline mfn_t
+sh2_vcpu_gfn_to_mfn(struct vcpu *v, unsigned long gfn)
+{ 
+    if ( !shadow2_vcpu_mode_translate(v) )
+        return _mfn(gfn);
+    if ( likely(current->domain == v->domain) )
+        return _mfn(get_mfn_from_gpfn(gfn));
+    return sh2_gfn_to_mfn_foreign(v->domain, gfn);
+}
+
+static inline unsigned long
+sh2_mfn_to_gfn(struct domain *d, mfn_t mfn)
+{
+    if ( shadow2_mode_translate(d) )
+        return get_gpfn_from_mfn(mfn_x(mfn));
+    else
+        return mfn_x(mfn);
+}
+
+
+
+#endif /* _XEN_SHADOW2_H */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
+      

xen/include/asm-x86/shadow_64.h

@@ -1,587 +0,0 @@
-/******************************************************************************
- * include/asm-x86/shadow_64.h
- * 
- * Copyright (c) 2005 Michael A Fetterman
- * Based on an earlier implementation by Ian Pratt et al
- * 
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- * 
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-/*
- * Jun Nakajima <jun.nakajima@intel.com>
- * Chengyuan Li <chengyuan.li@intel.com>
- *
- * Extended to support 64-bit guests.
- */
-#ifndef _XEN_SHADOW_64_H
-#define _XEN_SHADOW_64_H
-#include <asm/shadow.h>
-#include <asm/shadow_ops.h>
-#include <asm/hvm/hvm.h>
-
-/*
- * The naming convention of the shadow_ops:
- * MODE_<pgentry size>_<guest paging levels>_HANDLER
- */
-extern struct shadow_ops MODE_64_2_HANDLER;
-extern struct shadow_ops MODE_64_3_HANDLER;
-extern struct shadow_ops MODE_64_PAE_HANDLER;
-#if CONFIG_PAGING_LEVELS == 4
-extern struct shadow_ops MODE_64_4_HANDLER;
-#endif
-
-#if CONFIG_PAGING_LEVELS == 3
-#define L4_PAGETABLE_SHIFT      39
-#define L4_PAGETABLE_ENTRIES    (1<<PAGETABLE_ORDER)
-typedef struct { intpte_t l4; } l4_pgentry_t;
-#define is_guest_l4_slot(_s) (1)
-#endif
-
-#define READ_FAULT  0
-#define WRITE_FAULT 1
-
-#define ERROR_P     1
-#define ERROR_W     2
-#define ERROR_U     4
-#define ERROR_I     (1 << 4)
-
-#define X86_64_SHADOW_DEBUG 0
-
-#if X86_64_SHADOW_DEBUG
-#define ESH_LOG(_f, _a...)              \
-        printk(_f, ##_a)
-#else
-#define ESH_LOG(_f, _a...) ((void)0)
-#endif
-
-#define L_MASK  0xff
-
-#define PAE_PAGING_LEVELS   3
-
-#define ROOT_LEVEL_64   PAGING_L4
-#define ROOT_LEVEL_32   PAGING_L2
-
-#define DIRECT_ENTRY    (4UL << 16)
-#define SHADOW_ENTRY    (2UL << 16)
-#define GUEST_ENTRY     (1UL << 16)
-
-#define GET_ENTRY   (2UL << 8)
-#define SET_ENTRY   (1UL << 8)
-
-#define PAGETABLE_ENTRIES    (1<<PAGETABLE_ORDER)
-
-/* For 32-bit VMX guest to allocate shadow L1 & L2*/
-#define SL1_ORDER   1
-#define SL2_ORDER   2
-
-typedef struct { intpte_t lo; } pgentry_64_t;
-#define shadow_level_to_type(l)    (l << 29)
-#define shadow_type_to_level(t)    (t >> 29)
-
-#define entry_get_value(_x)         ((_x).lo)
-#define entry_get_pfn(_x)           \
-      (((_x).lo & (PADDR_MASK&PAGE_MASK)) >> PAGE_SHIFT)
-#define entry_get_paddr(_x)          (((_x).lo & (PADDR_MASK&PAGE_MASK)))
-#define entry_get_flags(_x)         (get_pte_flags((_x).lo))
-
-#define entry_empty()           ((pgentry_64_t) { 0 })
-#define entry_from_pfn(pfn, flags)  \
-    ((pgentry_64_t) { ((intpte_t)(pfn) << PAGE_SHIFT) | put_pte_flags(flags) })
-#define entry_from_page(page, flags) (entry_from_pfn(page_to_mfn(page),(flags)))
-#define entry_add_flags(x, flags)    ((x).lo |= put_pte_flags(flags))
-#define entry_remove_flags(x, flags) ((x).lo &= ~put_pte_flags(flags))
-#define entry_has_changed(x,y,flags) \
-        ( !!(((x).lo ^ (y).lo) & ((PADDR_MASK&PAGE_MASK)|put_pte_flags(flags))) )
-
-/******************************************************************************/
-/*
- * The macro and inlines are for 32-bit PAE guest 
- */
-#define PAE_PDPT_RESERVED   0x1e6 /* [8:5], [2,1] */
-
-#define PAE_SHADOW_SELF_ENTRY   259
-#define PAE_L3_PAGETABLE_ENTRIES   4
-
-/******************************************************************************/
-static inline int  table_offset_64(unsigned long va, int level)
-{
-    switch(level) {
-        case 1:
-            return  (((va) >> L1_PAGETABLE_SHIFT) & (L1_PAGETABLE_ENTRIES - 1));
-        case 2:
-            return  (((va) >> L2_PAGETABLE_SHIFT) & (L2_PAGETABLE_ENTRIES - 1));
-        case 3:
-            return  (((va) >> L3_PAGETABLE_SHIFT) & (L3_PAGETABLE_ENTRIES - 1));
-#if CONFIG_PAGING_LEVELS == 3
-        case 4:
-            return PAE_SHADOW_SELF_ENTRY;
-#endif
-
-#if CONFIG_PAGING_LEVELS >= 4
-#ifndef GUEST_PGENTRY_32
-#ifndef GUEST_32PAE
-        case 4:
-            return  (((va) >> L4_PAGETABLE_SHIFT) & (L4_PAGETABLE_ENTRIES - 1));
-#else
-        case 4:
-            return PAE_SHADOW_SELF_ENTRY;
-#endif
-#else
-        case 4:
-            return PAE_SHADOW_SELF_ENTRY;
-#endif
-#endif
-        default:
-            return -1;
-    }
-}
-
-/*****************************************************************************/
-
-#if defined( GUEST_32PAE )
-static inline int guest_table_offset_64(unsigned long va, int level, unsigned int index)
-{
-    switch(level) {
-        case 1:
-            return  (((va) >> L1_PAGETABLE_SHIFT) & (L1_PAGETABLE_ENTRIES - 1));
-        case 2:
-            return  (((va) >> L2_PAGETABLE_SHIFT) & (L2_PAGETABLE_ENTRIES - 1));
-        case 3:
-            return  (index * 4 + ((va) >> L3_PAGETABLE_SHIFT));
-#if CONFIG_PAGING_LEVELS == 3
-        case 4:
-            return PAE_SHADOW_SELF_ENTRY;
-#endif
-
-#if CONFIG_PAGING_LEVELS >= 4
-#ifndef GUEST_PGENTRY_32
-        case 4:
-            return  (((va) >> L4_PAGETABLE_SHIFT) & (L4_PAGETABLE_ENTRIES - 1));
-#else
-        case 4:
-            return PAE_SHADOW_SELF_ENTRY;
-#endif
-#endif
-        default:
-            return -1;
-    }
-}
-
-#define SH_GUEST_32PAE 1
-#else 
-#define guest_table_offset_64(va, level, index) \
-            table_offset_64((va),(level))
-#define SH_GUEST_32PAE 0
-#endif
-
-/********************************************************************************/
-
-static inline void free_out_of_sync_state(struct domain *d)
-{
-    struct out_of_sync_entry *entry;
-
-    // NB: Be careful not to call something that manipulates this list
-    //     while walking it.  Remove one item at a time, and always
-    //     restart from start of list.
-    //
-    while ( (entry = d->arch.out_of_sync) )
-    {
-        d->arch.out_of_sync = entry->next;
-        release_out_of_sync_entry(d, entry);
-
-        entry->next = d->arch.out_of_sync_free;
-        d->arch.out_of_sync_free = entry;
-    }
-}
-
-static inline int __entry(
-    struct vcpu *v, unsigned long va, pgentry_64_t *e_p, u32 flag)
-{
-    int i;
-    pgentry_64_t *le_e;
-    pgentry_64_t *le_p = NULL;
-    pgentry_64_t *phys_vtable = NULL;
-    unsigned long mfn;
-    int index;
-    u32 level = flag & L_MASK;
-    struct domain *d = v->domain;
-    int root_level;
-    unsigned int base_idx;
-
-    base_idx = get_cr3_idxval(v);
-
-    if ( flag & SHADOW_ENTRY )
-    {
-        root_level =  ROOT_LEVEL_64;
-        index = table_offset_64(va, root_level);
-        le_e = (pgentry_64_t *)&v->arch.shadow_vtable[index];
-    }
-    else if ( flag & GUEST_ENTRY )
-    {
-        root_level = v->domain->arch.ops->guest_paging_levels;
-        if ( root_level == PAGING_L3 )
-            index = guest_table_offset_64(va, PAGING_L3, base_idx);
-        else
-            index = guest_table_offset_64(va, root_level, base_idx);
-        le_e = (pgentry_64_t *)&v->arch.guest_vtable[index];
-    }
-    else /* direct mode */
-    {
-        root_level = PAE_PAGING_LEVELS;
-        index = table_offset_64(va, root_level);
-        phys_vtable = (pgentry_64_t *)map_domain_page(
-            pagetable_get_pfn(v->domain->arch.phys_table));
-        le_e = &phys_vtable[index];
-    }
-
-    /*
-     * If it's not external mode, then mfn should be machine physical.
-     */
-    for ( i = root_level - level; i > 0; i-- )
-    {
-        if ( unlikely(!(entry_get_flags(*le_e) & _PAGE_PRESENT)) )
-        {
-            if ( le_p )
-                unmap_domain_page(le_p);
-
-            if ( phys_vtable )
-                unmap_domain_page(phys_vtable);
-
-            return 0;
-        }
-
-        mfn = entry_get_pfn(*le_e);
-        if ( (flag & GUEST_ENTRY) && shadow_mode_translate(d) )
-            mfn = get_mfn_from_gpfn(mfn);
-
-        if ( le_p )
-            unmap_domain_page(le_p);
-        le_p = (pgentry_64_t *)map_domain_page(mfn);
-
-        if ( flag & SHADOW_ENTRY )
-            index = table_offset_64(va, (level + i - 1));
-        else
-            index = guest_table_offset_64(va, (level + i - 1), base_idx);
-        le_e = &le_p[index];
-    }
-
-    if ( flag & SET_ENTRY )
-        *le_e = *e_p;
-    else
-        *e_p = *le_e;
-
-    if ( le_p )
-        unmap_domain_page(le_p);
-
-    if ( phys_vtable )
-        unmap_domain_page(phys_vtable);
-
-    return 1;
-}
-
-static inline int __rw_entry(
-    struct vcpu *v, unsigned long va, void *e_p, u32 flag)
-{
-    pgentry_64_t *e = (pgentry_64_t *)e_p;
-
-    if (e) {
-        return __entry(v, va, e, flag);
-    }
-
-    return 0;
-}
-
-#define __shadow_set_l4e(v, va, value) \
-  __rw_entry(v, va, value, SHADOW_ENTRY | SET_ENTRY | PAGING_L4)
-#define __shadow_get_l4e(v, va, sl4e) \
-  __rw_entry(v, va, sl4e, SHADOW_ENTRY | GET_ENTRY | PAGING_L4)
-#define __shadow_set_l3e(v, va, value) \
-  __rw_entry(v, va, value, SHADOW_ENTRY | SET_ENTRY | PAGING_L3)
-#define __shadow_get_l3e(v, va, sl3e) \
-  __rw_entry(v, va, sl3e, SHADOW_ENTRY | GET_ENTRY | PAGING_L3)
-#define __shadow_set_l2e(v, va, value) \
-  __rw_entry(v, va, value, SHADOW_ENTRY | SET_ENTRY | PAGING_L2)
-#define __shadow_get_l2e(v, va, sl2e) \
-  __rw_entry(v, va, sl2e, SHADOW_ENTRY | GET_ENTRY | PAGING_L2)
-#define __shadow_set_l1e(v, va, value) \
-  __rw_entry(v, va, value, SHADOW_ENTRY | SET_ENTRY | PAGING_L1)
-#define __shadow_get_l1e(v, va, sl1e) \
-  __rw_entry(v, va, sl1e, SHADOW_ENTRY | GET_ENTRY | PAGING_L1)
-
-#define __guest_set_l4e(v, va, value) \
-  __rw_entry(v, va, value, GUEST_ENTRY | SET_ENTRY | PAGING_L4)
-#define __guest_get_l4e(v, va, gl4e) \
-  __rw_entry(v, va, gl4e, GUEST_ENTRY | GET_ENTRY | PAGING_L4)
-#define __guest_set_l3e(v, va, value) \
-  __rw_entry(v, va, value, GUEST_ENTRY | SET_ENTRY | PAGING_L3)
-#define __guest_get_l3e(v, va, sl3e) \
-  __rw_entry(v, va, gl3e, GUEST_ENTRY | GET_ENTRY | PAGING_L3)
-
-#define __direct_set_l3e(v, va, value) \
-  __rw_entry(v, va, value, DIRECT_ENTRY | SET_ENTRY | PAGING_L3)
-#define __direct_get_l3e(v, va, sl3e) \
-  __rw_entry(v, va, sl3e, DIRECT_ENTRY | GET_ENTRY | PAGING_L3)
-#define __direct_set_l2e(v, va, value) \
-  __rw_entry(v, va, value, DIRECT_ENTRY | SET_ENTRY | PAGING_L2)
-#define __direct_get_l2e(v, va, sl2e) \
-  __rw_entry(v, va, sl2e, DIRECT_ENTRY | GET_ENTRY | PAGING_L2)
-#define __direct_set_l1e(v, va, value) \
-  __rw_entry(v, va, value, DIRECT_ENTRY | SET_ENTRY | PAGING_L1)
-#define __direct_get_l1e(v, va, sl1e) \
-  __rw_entry(v, va, sl1e, DIRECT_ENTRY | GET_ENTRY | PAGING_L1)
-
-
-static inline int  __guest_set_l2e(
-    struct vcpu *v, unsigned long va, void *value, int size)
-{
-    switch(size) {
-        case 4:
-            // 32-bit guest
-            {
-                l2_pgentry_32_t *l2va;
-
-                l2va = (l2_pgentry_32_t *)v->arch.guest_vtable;
-                if (value)
-                    l2va[l2_table_offset_32(va)] = *(l2_pgentry_32_t *)value;
-                return 1;
-            }
-        case 8:
-            return __rw_entry(v, va, value, GUEST_ENTRY | SET_ENTRY | PAGING_L2);
-        default:
-            BUG();
-            return 0;
-    }
-    return 0;
-}
-
-#define __guest_set_l2e(v, va, value) \
-    __guest_set_l2e(v, (unsigned long)va, value, sizeof(*value))
-
-static inline int  __guest_get_l2e(
-  struct vcpu *v, unsigned long va, void *gl2e, int size)
-{
-    switch(size) {
-        case 4:
-            // 32-bit guest
-            {
-                l2_pgentry_32_t *l2va;
-                l2va = (l2_pgentry_32_t *)v->arch.guest_vtable;
-                if (gl2e)
-                    *(l2_pgentry_32_t *)gl2e = l2va[l2_table_offset_32(va)];
-                return 1;
-            }
-        case 8:
-            return __rw_entry(v, va, gl2e, GUEST_ENTRY | GET_ENTRY | PAGING_L2);
-        default:
-            BUG();
-            return 0;
-    }
-    return 0;
-}
-
-#define __guest_get_l2e(v, va, gl2e) \
-    __guest_get_l2e(v, (unsigned long)va, gl2e, sizeof(*gl2e))
-
-static inline int  __guest_set_l1e(
-  struct vcpu *v, unsigned long va, void *value, int size)
-{
-    switch(size) {
-        case 4:
-            // 32-bit guest
-            {
-                l2_pgentry_32_t gl2e;
-                l1_pgentry_32_t *l1va;
-                unsigned long l1mfn;
-
-                if (!__guest_get_l2e(v, va, &gl2e))
-                    return 0;
-                if (unlikely(!(l2e_get_flags_32(gl2e) & _PAGE_PRESENT)))
-                    return 0;
-
-                l1mfn = get_mfn_from_gpfn(
-                  l2e_get_pfn(gl2e));
-
-                l1va = (l1_pgentry_32_t *)map_domain_page(l1mfn);
-                if (value)
-                    l1va[l1_table_offset_32(va)] = *(l1_pgentry_32_t *)value;
-                unmap_domain_page(l1va);
-
-                return 1;
-            }
-
-        case 8:
-            return __rw_entry(v, va, value, GUEST_ENTRY | SET_ENTRY | PAGING_L1);
-        default:
-            BUG();
-            return 0;
-    }
-    return 0;
-}
-
-#define __guest_set_l1e(v, va, value) \
-     __guest_set_l1e(v, (unsigned long)va, value, sizeof(*value))
-
-static inline int  __guest_get_l1e(
-  struct vcpu *v, unsigned long va, void *gl1e, int size)
-{
-    switch(size) {
-        case 4:
-            // 32-bit guest
-            {
-                l2_pgentry_32_t gl2e;
-                l1_pgentry_32_t *l1va;
-                unsigned long l1mfn;
-
-                if (!(__guest_get_l2e(v, va, &gl2e)))
-                    return 0;
-
-
-                if (unlikely(!(l2e_get_flags_32(gl2e) & _PAGE_PRESENT)))
-                    return 0;
-
-
-                l1mfn = get_mfn_from_gpfn(
-                  l2e_get_pfn(gl2e));
-                l1va = (l1_pgentry_32_t *) map_domain_page(l1mfn);
-                if (gl1e)
-                    *(l1_pgentry_32_t *)gl1e = l1va[l1_table_offset_32(va)];
-                unmap_domain_page(l1va);
-                return 1;
-            }
-        case 8:
-            // 64-bit guest
-            return __rw_entry(v, va, gl1e, GUEST_ENTRY | GET_ENTRY | PAGING_L1);
-        default:
-            BUG();
-            return 0;
-    }
-    return 0;
-}
-
-#define __guest_get_l1e(v, va, gl1e) \
-    __guest_get_l1e(v, (unsigned long)va, gl1e, sizeof(*gl1e))
-
-static inline void entry_general(
-  struct domain *d,
-  pgentry_64_t *gle_p,
-  pgentry_64_t *sle_p,
-  unsigned long smfn, u32 level)
-
-{
-    pgentry_64_t gle = *gle_p;
-    pgentry_64_t sle;
-
-    sle = entry_empty();
-    if ( (entry_get_flags(gle) & _PAGE_PRESENT) && (smfn != 0) )
-    {
-        if ((entry_get_flags(gle) & _PAGE_PSE) && level == PAGING_L2) {
-            sle = entry_from_pfn(smfn, entry_get_flags(gle));
-            entry_remove_flags(sle, _PAGE_PSE);
-
-            if ( shadow_mode_log_dirty(d) ||
-                 !(entry_get_flags(gle) & _PAGE_DIRTY) )
-            {
-                pgentry_64_t *l1_p;
-                int i;
-
-                l1_p =(pgentry_64_t *)map_domain_page(smfn);
-                for (i = 0; i < L1_PAGETABLE_ENTRIES; i++)
-                {
-                    if ( mfn_is_page_table(entry_get_pfn(l1_p[i])) )
-                        entry_remove_flags(l1_p[i], _PAGE_RW);
-                }
-
-                unmap_domain_page(l1_p);
-            }
-        } else {
-            if (d->arch.ops->guest_paging_levels <= PAGING_L3
-                    && level == PAGING_L3) {
-                sle = entry_from_pfn(smfn, entry_get_flags(gle));
-            } else {
-
-                sle = entry_from_pfn(
-                  smfn,
-                  (entry_get_flags(gle) | _PAGE_RW | _PAGE_ACCESSED) & ~_PAGE_AVAIL);
-                entry_add_flags(gle, _PAGE_ACCESSED);
-            }
-        }
-        // XXX mafetter: Hmm...
-        //     Shouldn't the dirty log be checked/updated here?
-        //     Actually, it needs to be done in this function's callers.
-        //
-        *gle_p = gle;
-    }
-
-    if ( entry_get_value(sle) || entry_get_value(gle) )
-        SH_VVLOG("%s: gpde=%lx, new spde=%lx", __func__,
-          entry_get_value(gle), entry_get_value(sle));
-
-    *sle_p = sle;
-}
-
-static inline void entry_propagate_from_guest(
-  struct domain *d, pgentry_64_t *gle_p, pgentry_64_t *sle_p, u32 level)
-{
-    pgentry_64_t gle = *gle_p;
-    unsigned long smfn = 0;
-
-    if ( entry_get_flags(gle) & _PAGE_PRESENT ) {
-        if ((entry_get_flags(gle) & _PAGE_PSE) && level == PAGING_L2) {
-            smfn =  __shadow_status(d, entry_get_pfn(gle), PGT_fl1_shadow);
-        } else {
-            smfn =  __shadow_status(d, entry_get_pfn(gle), 
-              shadow_level_to_type((level -1 )));
-        }
-    }
-    entry_general(d, gle_p, sle_p, smfn, level);
-
-}
-
-static int inline
-validate_entry_change(
-  struct domain *d,
-  pgentry_64_t *new_gle_p,
-  pgentry_64_t *shadow_le_p,
-  u32 level)
-{
-    pgentry_64_t old_sle, new_sle;
-    pgentry_64_t new_gle = *new_gle_p;
-
-    old_sle = *shadow_le_p;
-    entry_propagate_from_guest(d, &new_gle, &new_sle, level);
-
-    ESH_LOG("old_sle: %lx, new_gle: %lx, new_sle: %lx\n",
-      entry_get_value(old_sle), entry_get_value(new_gle),
-      entry_get_value(new_sle));
-
-    if ( ((entry_get_value(old_sle) | entry_get_value(new_sle)) & _PAGE_PRESENT) &&
-      entry_has_changed(old_sle, new_sle, _PAGE_PRESENT) )
-    {
-        perfc_incrc(validate_entry_changes);
-
-        if ( (entry_get_flags(new_sle) & _PAGE_PRESENT) &&
-          !get_shadow_ref(entry_get_pfn(new_sle)) )
-            BUG();
-        if ( entry_get_flags(old_sle) & _PAGE_PRESENT )
-            put_shadow_ref(entry_get_pfn(old_sle));
-    }
-
-    *shadow_le_p = new_sle;
-
-    return 1;
-}
-
-#endif
-
-

xen/include/asm-x86/shadow_ops.h

@@ -1,138 +0,0 @@
-/******************************************************************************
- * include/asm-x86/shadow_ops.h
- * 
- * Copyright (c) 2005 Michael A Fetterman
- * Based on an earlier implementation by Ian Pratt et al
- * 
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- * 
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-#ifndef _XEN_SHADOW_OPS_H
-#define _XEN_SHADOW_OPS_H
-
-#define PAGING_L4      4UL
-#define PAGING_L3      3UL
-#define PAGING_L2      2UL
-#define PAGING_L1      1UL
-
-#define PAE_CR3_ALIGN       5
-#define PAE_CR3_IDX_MASK    0x7f
-
-#if defined( GUEST_PGENTRY_32 )
-
-#define GUEST_L1_PAGETABLE_ENTRIES     L1_PAGETABLE_ENTRIES_32
-#define GUEST_L2_PAGETABLE_ENTRIES     L2_PAGETABLE_ENTRIES_32
-#define GUEST_ROOT_PAGETABLE_ENTRIES   ROOT_PAGETABLE_ENTRIES_32
-#define GUEST_L2_PAGETABLE_SHIFT       L2_PAGETABLE_SHIFT_32
-
-#define guest_l1_pgentry_t      l1_pgentry_32_t
-#define guest_l2_pgentry_t      l2_pgentry_32_t
-#define guest_root_pgentry_t    l2_pgentry_32_t
-
-#define guest_l1e_get_paddr     l1e_get_paddr_32
-#define guest_l2e_get_paddr     l2e_get_paddr_32
-
-#define guest_get_pte_flags     get_pte_flags_32
-#define guest_put_pte_flags     put_pte_flags_32
-
-#define guest_l1e_get_flags     l1e_get_flags_32
-#define guest_l2e_get_flags     l2e_get_flags_32
-#define guest_root_get_flags          l2e_get_flags_32
-#define guest_root_get_intpte         l2e_get_intpte
-
-#define guest_l1e_empty         l1e_empty_32
-#define guest_l2e_empty         l2e_empty_32
-
-#define guest_l1e_from_pfn      l1e_from_pfn_32
-#define guest_l2e_from_pfn      l2e_from_pfn_32
-
-#define guest_l1e_from_paddr    l1e_from_paddr_32
-#define guest_l2e_from_paddr    l2e_from_paddr_32
-
-#define guest_l1e_from_page     l1e_from_page_32
-#define guest_l2e_from_page     l2e_from_page_32
-
-#define guest_l1e_add_flags     l1e_add_flags_32
-#define guest_l2e_add_flags     l2e_add_flags_32
-
-#define guest_l1e_remove_flag   l1e_remove_flags_32
-#define guest_l2e_remove_flag   l2e_remove_flags_32
-
-#define guest_l1e_has_changed   l1e_has_changed_32
-#define guest_l2e_has_changed   l2e_has_changed_32
-#define root_entry_has_changed  l2e_has_changed_32
-
-#define guest_l1_table_offset   l1_table_offset_32
-#define guest_l2_table_offset   l2_table_offset_32
-
-#define guest_linear_l1_table   linear_pg_table_32
-#define guest_linear_l2_table   linear_l2_table_32
-
-#define guest_va_to_l1mfn       va_to_l1mfn_32
-
-#else
-
-#define GUEST_L1_PAGETABLE_ENTRIES      L1_PAGETABLE_ENTRIES
-#define GUEST_L2_PAGETABLE_ENTRIES      L2_PAGETABLE_ENTRIES
-#define GUEST_ROOT_PAGETABLE_ENTRIES    ROOT_PAGETABLE_ENTRIES
-#define GUEST_L2_PAGETABLE_SHIFT        L2_PAGETABLE_SHIFT
-
-#define guest_l1_pgentry_t      l1_pgentry_t
-#define guest_l2_pgentry_t      l2_pgentry_t
-#define guest_root_pgentry_t    l4_pgentry_t
-
-#define guest_l1e_get_paddr     l1e_get_paddr
-#define guest_l2e_get_paddr     l2e_get_paddr
-
-#define guest_get_pte_flags     get_pte_flags
-#define guest_put_pte_flags     put_pte_flags
-
-#define guest_l1e_get_flags     l1e_get_flags
-#define guest_l2e_get_flags     l2e_get_flags
-#define guest_root_get_flags    l4e_get_flags
-#define guest_root_get_intpte   l4e_get_intpte
-
-#define guest_l1e_empty         l1e_empty
-#define guest_l2e_empty         l2e_empty
-
-#define guest_l1e_from_pfn      l1e_from_pfn
-#define guest_l2e_from_pfn      l2e_from_pfn
-
-#define guest_l1e_from_paddr    l1e_from_paddr
-#define guest_l2e_from_paddr    l2e_from_paddr
-
-#define guest_l1e_from_page     l1e_from_page
-#define guest_l2e_from_page     l2e_from_page
-
-#define guest_l1e_add_flags     l1e_add_flags
-#define guest_l2e_add_flags     l2e_add_flags
-
-#define guest_l1e_remove_flag   l1e_remove_flags
-#define guest_l2e_remove_flag   l2e_remove_flags
-
-#define guest_l1e_has_changed   l1e_has_changed
-#define guest_l2e_has_changed   l2e_has_changed
-#define root_entry_has_changed  l4e_has_changed
-
-#define guest_l1_table_offset   l1_table_offset
-#define guest_l2_table_offset   l2_table_offset
-
-#define guest_linear_l1_table   linear_pg_table
-#define guest_linear_l2_table   linear_l2_table
-
-#define guest_va_to_l1mfn       va_to_l1mfn
-#endif
-
-#endif /* _XEN_SHADOW_OPS_H */

xen/include/asm-x86/shadow_public.h

@@ -1,61 +0,0 @@
-/******************************************************************************
- * include/asm-x86/shadow_public.h
- * 
- * Copyright (c) 2005 Michael A Fetterman
- * Based on an earlier implementation by Ian Pratt et al
- * 
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- * 
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-#ifndef _XEN_SHADOW_PUBLIC_H
-#define _XEN_SHADOW_PUBLIC_H
-
-#if CONFIG_PAGING_LEVELS >= 3
-#define MFN_PINNED(_x) (mfn_to_page(_x)->u.inuse.type_info & PGT_pinned)
-
-extern void shadow_sync_and_drop_references(
-      struct domain *d, struct page_info *page);
-extern void shadow_drop_references(
-      struct domain *d, struct page_info *page);
-
-extern int shadow_set_guest_paging_levels(struct domain *d, int levels);
-
-extern void release_out_of_sync_entry(
-    struct domain *d, struct out_of_sync_entry *entry);
-
-struct shadow_ops {
-    unsigned long guest_paging_levels; /* guest paging levels */
-    void (*invlpg)(struct vcpu *v, unsigned long va);
-    int  (*fault)(unsigned long va, struct cpu_user_regs *regs);
-    void (*update_pagetables)(struct vcpu *v);
-    void (*sync_all)(struct domain *d);
-    int  (*remove_all_write_access)(struct domain *d,
-             unsigned long readonly_gpfn, unsigned long readonly_gmfn);
-    int  (*do_update_va_mapping)(unsigned long va, l1_pgentry_t val, struct vcpu *v);
-    struct out_of_sync_entry *
-         (*mark_mfn_out_of_sync)(struct vcpu *v, unsigned long gpfn,
-                              unsigned long mfn);
-    int  (*is_out_of_sync)(struct vcpu *v, unsigned long va);
-    unsigned long (*gva_to_gpa)(unsigned long gva);
-};
-#endif
-
-#if CONFIG_PAGING_LEVELS >= 4
-extern void shadow_l4_normal_pt_update(struct domain *d,
-                                       unsigned long pa, l4_pgentry_t l4e,
-                                       struct domain_mmap_cache *cache);
-#endif
-
-#endif

xen/include/asm-x86/x86_32/page-2level.h

@@ -46,6 +46,7 @@ typedef l2_pgentry_t root_pgentry_t;
  *  12-bit flags = (pte[11:0])
  */
 
+#define _PAGE_NX_BIT            0U
 #define _PAGE_NX                0U
 
 /* Extract flags into 12-bit integer, or turn 12-bit flags into a pte mask. */

xen/include/asm-x86/x86_32/page-3level.h

@@ -59,7 +59,8 @@ typedef l3_pgentry_t root_pgentry_t;
  *  32-bit flags = (pte[63:44],pte[11:0])
  */
 
-#define _PAGE_NX (cpu_has_nx ? (1<<31) : 0)
+#define _PAGE_NX_BIT (1U<<31)
+#define _PAGE_NX     (cpu_has_nx ? _PAGE_NX_BIT : 0)
 
 /* Extract flags into 32-bit integer, or turn 32-bit flags into a pte mask. */
 #define get_pte_flags(x) (((int)((x) >> 32) & ~0xFFF) | ((int)(x) & 0xFFF))

xen/include/asm-x86/x86_64/page.h

@@ -44,6 +44,8 @@ typedef l4_pgentry_t root_pgentry_t;
 /* Given a virtual address, get an entry offset into a linear page table. */
 #define l1_linear_offset(_a) (((_a) & VADDR_MASK) >> L1_PAGETABLE_SHIFT)
 #define l2_linear_offset(_a) (((_a) & VADDR_MASK) >> L2_PAGETABLE_SHIFT)
+#define l3_linear_offset(_a) (((_a) & VADDR_MASK) >> L3_PAGETABLE_SHIFT)
+#define l4_linear_offset(_a) (((_a) & VADDR_MASK) >> L4_PAGETABLE_SHIFT)
 
 #define is_guest_l1_slot(_s) (1)
 #define is_guest_l2_slot(_t, _s) (1)
@@ -70,7 +72,8 @@ typedef l4_pgentry_t root_pgentry_t;
 #define put_pte_flags(x) (((intpte_t)((x) & ~0xFFF) << 40) | ((x) & 0xFFF))
 
 /* Bit 23 of a 24-bit flag mask. This corresponds to bit 63 of a pte.*/
-#define _PAGE_NX (cpu_has_nx ? (1U<<23) : 0U)
+#define _PAGE_NX_BIT (1U<<23)
+#define _PAGE_NX     (cpu_has_nx ? _PAGE_NX_BIT : 0U)
 
 #define L1_DISALLOW_MASK BASE_DISALLOW_MASK
 #define L2_DISALLOW_MASK BASE_DISALLOW_MASK

xen/include/public/dom0_ops.h

@@ -262,6 +262,18 @@ DEFINE_XEN_GUEST_HANDLE(dom0_sched_id_t);
 #define DOM0_SHADOW_CONTROL_OP_CLEAN       11
 #define DOM0_SHADOW_CONTROL_OP_PEEK        12
 
+/* Shadow2 operations */
+#define DOM0_SHADOW2_CONTROL_OP_GET_ALLOCATION   30
+#define DOM0_SHADOW2_CONTROL_OP_SET_ALLOCATION   31
+#define DOM0_SHADOW2_CONTROL_OP_ENABLE           32
+
+/* Mode flags for Shadow2 enable op */
+#define DOM0_SHADOW2_CONTROL_FLAG_ENABLE    (1 << 0)
+#define DOM0_SHADOW2_CONTROL_FLAG_REFCOUNT  (1 << 1)
+#define DOM0_SHADOW2_CONTROL_FLAG_LOG_DIRTY (1 << 2)
+#define DOM0_SHADOW2_CONTROL_FLAG_TRANSLATE (1 << 3)
+#define DOM0_SHADOW2_CONTROL_FLAG_EXTERNAL  (1 << 4)
+
 struct dom0_shadow_control_stats {
     uint32_t fault_count;
     uint32_t dirty_count;
@@ -277,7 +289,9 @@ struct dom0_shadow_control {
     uint32_t       op;
     XEN_GUEST_HANDLE(ulong) dirty_bitmap;
     /* IN/OUT variables. */
-    uint64_t       pages;        /* size of buffer, updated with actual size */
+    uint64_t       pages;    /* size of buffer, updated with actual size */
+    uint32_t       mb;       /* Shadow2 memory allocation in MB */
+    uint32_t       mode;     /* Shadow2 mode to enable */
     /* OUT variables. */
     struct dom0_shadow_control_stats stats;
 };

xen/include/xen/domain_page.h

@@ -26,6 +26,13 @@ extern void *map_domain_page(unsigned long pfn);
  */
 extern void unmap_domain_page(void *va);
 
+/* 
+ * Convert a VA (within a page previously mapped in the context of the
+ * currently-executing VCPU via a call to map_domain_pages()) to a machine 
+ * address 
+ */
+extern paddr_t mapped_domain_page_to_maddr(void *va);
+
 /*
  * Similar to the above calls, except the mapping is accessible in all
  * address spaces (not just within the VCPU that created the mapping). Global
@@ -98,6 +105,7 @@ domain_mmap_cache_destroy(struct domain_mmap_cache *cache)
 
 #define map_domain_page(pfn)                maddr_to_virt((pfn)<<PAGE_SHIFT)
 #define unmap_domain_page(va)               ((void)(va))
+#define mapped_domain_page_to_maddr(va)     (virt_to_maddr(va))
 
 #define map_domain_page_global(pfn)         maddr_to_virt((pfn)<<PAGE_SHIFT)
 #define unmap_domain_page_global(va)        ((void)(va))
@@ -112,4 +120,9 @@ struct domain_mmap_cache {
 
 #endif /* !CONFIG_DOMAIN_PAGE */
 
+#define HERE_I_AM \
+do { \
+    printk("HERE I AM: %s %s %d\n", __func__, __FILE__, __LINE__); \
+} while (0)
+
 #endif /* __XEN_DOMAIN_PAGE_H__ */

xen/include/xen/lib.h

@@ -18,7 +18,7 @@ extern void __bug(char *file, int line) __attribute__((noreturn));
 #ifndef NDEBUG
 #define ASSERT(_p)                                                      \
     do {                                                                \
-        if ( !(_p) )                                                    \
+        if ( unlikely(!(_p)) )                                          \
         {                                                               \
             printk("Assertion '%s' failed, line %d, file %s\n", #_p ,   \
                    __LINE__, __FILE__);                                 \
@@ -41,7 +41,7 @@ struct domain;
 void cmdline_parse(char *cmdline);
 
 #ifndef NDEBUG
-extern int debugtrace_send_to_console;
+extern void debugtrace_toggle(void);
 extern void debugtrace_dump(void);
 extern void debugtrace_printk(const char *fmt, ...);
 #else

xen/include/xen/list.h

@@ -161,6 +161,16 @@ static __inline__ void list_splice(struct list_head *list, struct list_head *hea
 	for (pos = (head)->next, n = pos->next; pos != (head); \
 		pos = n, n = pos->next)
 
+/**
+ * list_for_each_backwards_safe	-	iterate backwards over a list safe against removal of list entry
+ * @pos:	the &struct list_head to use as a loop counter.
+ * @n:		another &struct list_head to use as temporary storage
+ * @head:	the head for your list.
+ */
+#define list_for_each_backwards_safe(pos, n, head) \
+	for (pos = (head)->prev, n = pos->prev; pos != (head); \
+		pos = n, n = pos->prev)
+
 /**
  * list_for_each_entry	-	iterate over list of given type
  * @pos:	the type * to use as a loop counter.

xen/include/xen/sched.h

@@ -376,9 +376,12 @@ extern struct domain *domain_list;
  /* VCPU is paused by the hypervisor? */
 #define _VCPUF_paused          11
 #define VCPUF_paused           (1UL<<_VCPUF_paused)
- /* VCPU is blocked awaiting an event to be consumed by Xen. */
+/* VCPU is blocked awaiting an event to be consumed by Xen. */
 #define _VCPUF_blocked_in_xen  12
 #define VCPUF_blocked_in_xen   (1UL<<_VCPUF_blocked_in_xen)
+ /* HVM vcpu thinks CR0.PG == 0 */
+#define _VCPUF_shadow2_translate 13
+#define VCPUF_shadow2_translate  (1UL<<_VCPUF_shadow2_translate)
 
 /*
  * Per-domain flags (domain_flags).