Give each domain some memory below 4GB. This solves the "PGD's must be below 4GB" for the initial page tables. I'm not sure we'll stick with this approach, but this is good enough for the time being.

PAE should be a *lot* more robust on systems that actually have more than 4GB thanks to all the various patches that went in today. I find it astounding that it ever appeared to work at all!

Signed-off-by: ian@xensource.com

tools/libxc/xc_domain.c

@@ -262,13 +262,16 @@ int xc_domain_setmaxmem(int xc_handle,
 
 int xc_domain_memory_increase_reservation(int xc_handle,
                                           u32 domid, 
-                                          unsigned int mem_kb)
+                                          unsigned long mem_kb,
+                                          unsigned int extent_order,
+                                          unsigned int address_bits)
 {
     int err;
     unsigned int npages = mem_kb / (PAGE_SIZE/1024);
     struct xen_memory_reservation reservation = {
         .nr_extents   = npages,
-        .extent_order = 0,
+        .extent_order = extent_order,
+        .address_bits = address_bits,
         .domid        = domid
     };
 
@@ -277,6 +280,8 @@ int xc_domain_memory_increase_reservation(int xc_handle,
         return 0;
 
     if (err > 0) {
+        fprintf(stderr,"Failed alocation for dom %d : %d pages order %d addr_bits %d\n",
+                                 domid, npages, extent_order, address_bits);
         errno = ENOMEM;
         err = -1;
     }

tools/libxc/xc_linux_build.c

@@ -270,6 +270,11 @@ static int setup_pg_tables_64(int xc_handle, u32 dom,
 }
 #endif
 
+static int compare (const void * a, const void * b)
+{
+  return ( *(long*)a - *(long*)b );
+}
+
 #ifdef __ia64__
 #include <asm/fpu.h> /* for FPSR_DEFAULT */
 static int setup_guest(int xc_handle,
@@ -483,6 +488,9 @@ static int setup_guest(int xc_handle,
         goto error_out;
     }
 
+    qsort( page_array, nr_pages, sizeof(*page_array), compare );
+
+
     (load_funcs.loadimage)(image, image_size, xc_handle, dom, page_array,
                            &dsi);
 

tools/libxc/xc_linux_restore.c

@@ -149,9 +149,9 @@ int xc_linux_restore(int xc_handle, int io_fd, u32 dom, unsigned long nr_pfns,
     }
 
     err = xc_domain_memory_increase_reservation(xc_handle, dom,
-                                                nr_pfns * PAGE_SIZE / 1024);
+                                                nr_pfns * PAGE_SIZE / 1024, 0, 0); //FIX ME
     if (err != 0) {
-        ERR("Failed to increate reservation by %lx\n", 
+        ERR("Failed to increase reservation by %lx\n", 
             nr_pfns * PAGE_SIZE / 1024); 
         errno = ENOMEM;
         goto out;

tools/libxc/xenctrl.h

@@ -387,7 +387,9 @@ int xc_domain_setmaxmem(int xc_handle,
 
 int xc_domain_memory_increase_reservation(int xc_handle,
                                           u32 domid, 
-                                          unsigned int mem_kb);
+                                          unsigned long mem_kb,
+                                          unsigned int extent_order,
+                                          unsigned int address_bits);
 
 typedef dom0_perfc_desc_t xc_perfc_desc_t;
 /* IMPORTANT: The caller is responsible for mlock()'ing the @desc array. */

tools/python/xen/lowlevel/xc/xc.c

@@ -841,14 +841,16 @@ static PyObject *pyxc_domain_memory_increase_reservation(PyObject *self,
 
     u32 dom;
     unsigned long mem_kb;
+    unsigned int extent_order = 0 , address_bits = 0;
 
-    static char *kwd_list[] = { "dom", "mem_kb", NULL };
+    static char *kwd_list[] = { "dom", "mem_kb", "extent_order", "address_bits", NULL };
 
-    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "ii", kwd_list, 
-                                      &dom, &mem_kb) )
+    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "ii|ii", kwd_list, 
+                                      &dom, &mem_kb, &extent_order, &address_bits) )
         return NULL;
 
-    if ( xc_domain_memory_increase_reservation(xc->xc_handle, dom, mem_kb) )
+    if ( xc_domain_memory_increase_reservation(xc->xc_handle, dom, 
+                                     mem_kb, extent_order, address_bits) )
         return PyErr_SetFromErrno(xc_error);
     
     Py_INCREF(zero);

tools/python/xen/xend/image.py

@@ -159,7 +159,11 @@ class ImageHandler:
         xc.domain_setmaxmem(dom, mem_kb)
 
         try:
-            xc.domain_memory_increase_reservation(dom, mem_kb)
+            # Give the domain some memory below 4GB
+            lmem_kb = 4096
+            xc.domain_memory_increase_reservation(dom, min(lmem_kb,mem_kb), 0, 32)
+            if mem_kb > lmem_kb:
+                xc.domain_memory_increase_reservation(dom, mem_kb-lmem_kb, 0, 0)
         except:
             xc.domain_destroy(dom)
             raise

xen/common/memory.c

@@ -52,7 +52,7 @@ increase_reservation(
         if ( unlikely((page = alloc_domheap_pages(
             d, extent_order, flags)) == NULL) )
         {
-            DPRINTK("Could not allocate a frame\n");
+            DPRINTK("Could not allocate a frame id=%d %d flags=%x\n", d->domain_id, extent_order, flags);
             return i;
         }