Loop retrying when ballooning out, even when the dom0-min-mem setting means

that there is not sufficient memory available at the moment.  Memory may be
about to be freed up by the system after a domain destruction (i.e. the memory
is being scrubbed asynchronously, and will be released soon).

Closes bug #407, bug #429.

Signed-off-by: Ewan Mellor <ewan@xensource.com>

tools/python/xen/xend/balloon.py

@@ -30,52 +30,95 @@ from XendError import VmError
 PROC_XEN_BALLOON = "/proc/xen/balloon"
 BALLOON_OUT_SLACK = 1 # MiB.  We need this because the physinfo details are
                       # rounded.
+RETRY_LIMIT = 10
+##
+# The time to sleep between retries grows linearly, using this value (in
+# seconds).  When the system is lightly loaded, memory should be scrubbed and
+# returned to the system very quickly, whereas when it is loaded, the system
+# needs idle time to get the scrubbing done.  This linear growth accommodates
+# such requirements.
+SLEEP_TIME_GROWTH = 0.1
 
 
 def free(required):
     """Balloon out memory from the privileged domain so that there is the
     specified required amount (in KiB) free.
     """
-    
-    xc = xen.lowlevel.xc.xc()
+
+    # We check whether there is enough free memory, and if not, instruct dom0
+    # to balloon out to free some up.  Memory freed by a destroyed domain may
+    # not appear in the free_memory field immediately, because it needs to be
+    # scrubbed before it can be released to the free list, which is done
+    # asynchronously by Xen; ballooning is asynchronous also.  No matter where
+    # we expect the free memory to come from, therefore, we need to wait for
+    # it to become available.
+    #
+    # We are not allowed to balloon below dom0_min_mem, or if dom0_min_mem
+    # is 0, we cannot balloon at all.  Memory can still become available
+    # through a rebooting domain, however.
+    #
+    # Eventually, we time out (presumably because there really isn't enough
+    # free memory).
+    #
+    # We don't want to set the memory target (triggering a watch) when that
+    # has already been done, but we do want to respond to changing memory
+    # usage, so we recheck the required alloc each time around the loop, but
+    # track the last used value so that we don't trigger too many watches.
+
+    need_mem = (required + 1023) / 1024 + BALLOON_OUT_SLACK
+
     xroot = XendRoot.instance()
+    xc = xen.lowlevel.xc.xc()
 
     try:
-        free_mem = xc.physinfo()['free_memory']
-        need_mem = (required + 1023) / 1024 + BALLOON_OUT_SLACK
-
-        log.debug("Balloon: free %d; need %d.", free_mem, need_mem)
-        
-        if free_mem >= need_mem:
-            return
-
         dom0_min_mem = xroot.get_dom0_min_mem()
-        if dom0_min_mem == 0:
-            raise VmError('Not enough free memory and dom0_min_mem is 0.')
 
-        dom0_alloc = _get_dom0_alloc()
-        dom0_new_alloc = dom0_alloc - (need_mem - free_mem)
-        if dom0_new_alloc < dom0_min_mem:
+        retries = 0
+        sleep_time = SLEEP_TIME_GROWTH
+        last_new_alloc = None
+        while retries < RETRY_LIMIT:
+            free_mem = xc.physinfo()['free_memory']
+
+            if free_mem >= need_mem:
+                log.debug("Balloon: free %d; need %d; done.", free_mem,
+                          need_mem)
+                return
+
+            if retries == 0:
+                log.debug("Balloon: free %d; need %d.", free_mem, need_mem)
+
+            if dom0_min_mem > 0:
+                dom0_alloc = _get_dom0_alloc()
+                new_alloc = dom0_alloc - (need_mem - free_mem)
+
+                if (new_alloc >= dom0_min_mem and
+                    new_alloc != last_new_alloc):
+                    log.debug("Balloon: setting dom0 target to %d.",
+                              new_alloc)
+                    dom0 = XendDomain.instance().privilegedDomain()
+                    dom0.setMemoryTarget(new_alloc)
+                    last_new_alloc = new_alloc
+                    # Continue to retry, waiting for ballooning.
+
+            time.sleep(sleep_time)
+            retries += 1
+            sleep_time += SLEEP_TIME_GROWTH
+
+        # Not enough memory; diagnose the problem.
+        if dom0_min_mem == 0:
+            raise VmError(('Not enough free memory and dom0_min_mem is 0, so '
+                           'I cannot release any more.  I need %d MiB but '
+                           'only have %d.') %
+                          (need_mem, free_mem))
+        elif new_alloc >= dom0_min_mem:
             raise VmError(
                 ('I need %d MiB, but dom0_min_mem is %d and shrinking to '
                  '%d MiB would leave only %d MiB free.') %
                 (need_mem, dom0_min_mem, dom0_min_mem,
-                 free_mem + (dom0_alloc - dom0_min_mem)))
+                 free_mem + dom0_alloc - dom0_min_mem))
+        else:
+            raise VmError('The privileged domain did not balloon!')
 
-        dom0 = XendDomain.instance().privilegedDomain()
-        dom0.setMemoryTarget(dom0_new_alloc)
-
-        timeout = 20 # 2 sec
-        while timeout > 0:
-            time.sleep(0.1)
-
-            free_mem = xc.physinfo()['free_memory']
-            if free_mem >= need_mem:
-                return
-
-            timeout -= 1
-
-        raise VmError('The privileged domain did not balloon!')
     finally:
         del xc