Improved "PGDs must be under 4GB" handling for PAE.

Signed-off-by: ian@xensource.com

tools/libxc/xc_domain.c

@@ -262,28 +262,66 @@ int xc_domain_setmaxmem(int xc_handle,
 
 int xc_domain_memory_increase_reservation(int xc_handle,
                                           u32 domid, 
-                                          unsigned long mem_kb,
+                                          unsigned long nr_extents,
                                           unsigned int extent_order,
-                                          unsigned int address_bits)
+                                          unsigned int address_bits,
+					  unsigned long *extent_start)
 {
     int err;
-    unsigned int npages = mem_kb / (PAGE_SIZE/1024);
     struct xen_memory_reservation reservation = {
-        .nr_extents   = npages,
-        .extent_order = extent_order,
+        .extent_start = extent_start, /* may be NULL */
+        .nr_extents   = nr_extents,
+        .extent_order = extent_order,  
         .address_bits = address_bits,
         .domid        = domid
     };
 
     err = xc_memory_op(xc_handle, XENMEM_increase_reservation, &reservation);
-    if (err == npages)
+    if (err == nr_extents)
         return 0;
 
     if (err > 0) {
-        fprintf(stderr,"Failed alocation for dom %d : %d pages order %d addr_bits %d\n",
-                                 domid, npages, extent_order, address_bits);
+        fprintf(stderr,"Failed alocation for dom %d : %ld pages order %d addr_bits %d\n",
+                                 domid, nr_extents, extent_order, address_bits);
         errno = ENOMEM;
         err = -1;
     }
     return err;
 }
+
+int xc_domain_memory_decrease_reservation(int xc_handle,
+                                          u32 domid, 
+                                          unsigned long nr_extents,
+                                          unsigned int extent_order,
+					  unsigned long *extent_start)
+{
+    int err;
+    struct xen_memory_reservation reservation = {
+        .extent_start = extent_start, 
+        .nr_extents   = nr_extents,
+        .extent_order = extent_order,  
+        .address_bits = 0,
+        .domid        = domid
+    };
+
+    if (extent_start == NULL)
+    {
+        fprintf(stderr,"decrease_reservation extent_start is NULL!\n");
+        errno = EINVAL;
+        err = -1;
+	goto out;
+    }
+
+    err = xc_memory_op(xc_handle, XENMEM_increase_reservation, &reservation);
+    if (err == nr_extents)
+        return 0;
+
+    if (err > 0) {
+        fprintf(stderr,"Failed de-alocation for dom %d : %ld pages order %d\n",
+                                 domid, nr_extents, extent_order);
+        errno = EBUSY;
+        err = -1;
+    }
+out:
+    return err;
+}

tools/libxc/xc_linux_build.c

@@ -136,11 +136,24 @@ static int setup_pg_tables_pae(int xc_handle, u32 dom,
 
     /* First allocate page for page dir. */
     ppt_alloc = (vpt_start - dsi_v_start) >> PAGE_SHIFT;
+
+    if ( page_array[ppt_alloc] > 0xfffff )
+    {
+	unsigned long nmfn;
+	nmfn = xc_make_page_below_4G( xc_handle, dom, page_array[ppt_alloc] );
+	if ( nmfn == 0 )
+	{
+	    fprintf(stderr, "Couldn't get a page below 4GB :-(\n");
+	    goto error_out;
+	}
+	page_array[ppt_alloc] = nmfn;
+    }
+
     alloc_pt(l3tab, vl3tab);
     vl3e = &vl3tab[l3_table_offset_pae(dsi_v_start)];
     ctxt->ctrlreg[3] = l3tab;
 
-    if(l3tab>0xfffff000)
+    if(l3tab>0xfffff000ULL)
     {
         fprintf(stderr,"L3TAB = %llx above 4GB!\n",l3tab);
         goto error_out;

tools/libxc/xc_linux_restore.c

@@ -149,7 +149,7 @@ int xc_linux_restore(int xc_handle, int io_fd, u32 dom, unsigned long nr_pfns,
     }
 
     err = xc_domain_memory_increase_reservation(xc_handle, dom,
-                                                nr_pfns * PAGE_SIZE / 1024, 0, 0); //FIX ME
+                                                nr_pfns, 0, 0, NULL);
     if (err != 0) {
         ERR("Failed to increase reservation by %lx\n", 
             nr_pfns * PAGE_SIZE / 1024); 

tools/libxc/xc_private.c

@@ -427,3 +427,21 @@ int xc_version(int xc_handle, int cmd, void *arg)
 {
     return do_xen_version(xc_handle, cmd, arg);
 }
+
+unsigned long xc_make_page_below_4G(int xc_handle, u32 domid, 
+				    unsigned long mfn)
+{
+    unsigned long new_mfn;
+    if ( xc_domain_memory_decrease_reservation( 
+	xc_handle, domid, 1, 0, &mfn ) != 1 )
+    {
+	fprintf(stderr,"xc_make_page_below_4G decrease failed. mfn=%lx\n",mfn);
+	return 0;
+    }
+    if ( xc_domain_memory_increase_reservation( xc_handle, domid, 1, 0, 32, &new_mfn ) != 1 )
+    {
+	fprintf(stderr,"xc_make_page_below_4G increase failed. mfn=%lx\n",mfn);
+	return 0;
+    }
+    return new_mfn;
+}

tools/libxc/xenctrl.h

@@ -387,9 +387,19 @@ int xc_domain_setmaxmem(int xc_handle,
 
 int xc_domain_memory_increase_reservation(int xc_handle,
                                           u32 domid, 
-                                          unsigned long mem_kb,
+                                          unsigned long nr_extents,
                                           unsigned int extent_order,
-                                          unsigned int address_bits);
+                                          unsigned int address_bits,
+					  unsigned long *extent_start);
+
+int xc_domain_memory_decrease_reservation(int xc_handle,
+                                          u32 domid, 
+                                          unsigned long nr_extents,
+                                          unsigned int extent_order,
+					  unsigned long *extent_start);
+
+unsigned long xc_make_page_below_4G(int xc_handle, u32 domid, 
+				    unsigned long mfn);
 
 typedef dom0_perfc_desc_t xc_perfc_desc_t;
 /* IMPORTANT: The caller is responsible for mlock()'ing the @desc array. */

tools/python/xen/lowlevel/xc/xc.c

@@ -842,6 +842,7 @@ static PyObject *pyxc_domain_memory_increase_reservation(PyObject *self,
     u32 dom;
     unsigned long mem_kb;
     unsigned int extent_order = 0 , address_bits = 0;
+    unsigned long nr_extents;
 
     static char *kwd_list[] = { "dom", "mem_kb", "extent_order", "address_bits", NULL };
 
@@ -849,8 +850,12 @@ static PyObject *pyxc_domain_memory_increase_reservation(PyObject *self,
                                       &dom, &mem_kb, &extent_order, &address_bits) )
         return NULL;
 
+    /* round down to nearest power of 2. Assume callers using extent_order>0
+       know what they are doing */
+    nr_extents = (mem_kb / (XC_PAGE_SIZE/1024)) >> extent_order;
     if ( xc_domain_memory_increase_reservation(xc->xc_handle, dom, 
-                                     mem_kb, extent_order, address_bits) )
+					       nr_extents, extent_order, 
+					       address_bits, NULL) )
         return PyErr_SetFromErrno(xc_error);
     
     Py_INCREF(zero);

tools/python/xen/xend/image.py

@@ -160,8 +160,9 @@ class ImageHandler:
 
         try:
             # Give the domain some memory below 4GB
-            lmem_kb = 4096
-            xc.domain_memory_increase_reservation(dom, min(lmem_kb,mem_kb), 0, 32)
+            lmem_kb = 0
+            if lmem_kb > 0:
+                xc.domain_memory_increase_reservation(dom, min(lmem_kb,mem_kb), 0, 32)
             if mem_kb > lmem_kb:
                 xc.domain_memory_increase_reservation(dom, mem_kb-lmem_kb, 0, 0)
         except: