tools/xenballoond: add tmem capability to directed/self-ballooning

and monitor tool

This patch adds tmem support to the largely unknown/unused
xenballoond scripts that implement both self-ballooning and
a foundation for directed-ballooning.  Tmem and automated
ballooning are highly complementary in that, when ballooning
is over-aggressive, paging and swapping can increase noticably.
Precache preserves evicted pages that may be needed again soon
(thus eliminating disk reads) and preswap provides memory-based
swapping that occurs if ballooning is insufficiently responsive
to a sudden increase in activity and memory demand (thus
eliminating disk writes and reads).

There are two changes in this patch:

1) The xenballoond service is a convenient place to implement
   userland "preswap shrinking".**
2) The xenballoon-monitor script is a convenient place to
   report (and view with "watch -d") the frequent memory
   rebalancing that results from tmem usage on a busy system.

Note that for best results (and for the monitor script to
work), the xenstore-* tools should be installed on each guest.

(** In a disk-based swap device, stale pages are often left
  on-disk even after they are no longer needed or valid; they
  are simply overwritten if/when the disk blocks are needed
  again, which may be a very long time.  Preswap behaves
  much like a disk, but uses precious pages of memory that
  count against a guest's memory allocation; thus stale
  pages are very undesirable.  Preswap shrinking periodically
  attempts to remove stale pages from preswap by using a
  sysfs interface created by the linux-side tmem patch.)

Signed-off-by: Dan Magenheimer <dan.magenheimer@oracle.com>

tools/xenballoon/xenballoon-monitor

@@ -2,6 +2,7 @@
 #
 # xenballoon-monitor - monitor certain stats from xenballoond
 #   (run in dom0 with "watch -d xenballoon-monitor" for xentop-like output)
+# updated 090610 to include tmem stats
 #
 # Copyright (C) 2009 Oracle Corporation and/or its affiliates.
 # All rights reserved
@@ -9,10 +10,12 @@
 #
 # Hint: Use "xm sched-credit -d 0 -w 2000" to watch on heavily loaded machines
 #
-echo "id   mem-kb  tgt-kb  commit   swapin  swapout      pgin     pgout active(sec)"
+TMEMTMP=$(/bin/mktemp -q /tmp/xenballoon-monitor.XXXXXX)
+echo "id   mem-kb  tgt-kb  commit  swapin swapout    pgin    pgout  preswap  precache"
 for i in `xenstore-list /local/domain`; do
  if [ "$i" -ne 0 ]; then
- tot=0; tgt=0; sin=0; sout=0; pgin=0; pgout=0; cmt=0; up=0; idle=0; act=0;
+ tot=0; tgt=0; sin=0; sout=0; pgin=0; pgout=0; cmt=0; up=0; idle=0;
+ act=0; preswap=0; precache=0
  if xenstore-exists /local/domain/$i/memory/meminfo; then
   tot=`xenstore-read /local/domain/$i/memory/meminfo | grep MemTotal \
    | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'`
@@ -23,21 +26,34 @@ for i in `xenstore-list /local/domain`; do
   tgt=`xenstore-read /local/domain/$i/memory/selftarget`
  fi
  if xenstore-exists /local/domain/$i/memory/vmstat; then
-  sin=`xenstore-read /local/domain/$i/memory/vmstat | grep pswpin \
- 	| cut -d" " -f2`
-  sout=`xenstore-read /local/domain/$i/memory/vmstat | grep pswpout \
- 	| cut -d" " -f2`
-  pgin=`xenstore-read /local/domain/$i/memory/vmstat | grep pgpgin \
- 	| cut -d" " -f2`
-  pgout=`xenstore-read /local/domain/$i/memory/vmstat | grep pgout \
-  	| cut -d" " -f2`
+  sin=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \
+	| grep pswpin | cut -d" " -f2)
+  sout=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \
+	| grep pswpout | cut -d" " -f2)
+  pgin=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \
+	| grep pgpgin | cut -d" " -f2)
+  pgout=$(xenstore-read /local/domain/$i/memory/vmstat | tr '\\\n' '\n' \
+	| grep pgout | cut -d" " -f2)
  fi
- if xenstore-exists /local/domain/$i/memory/uptime; then
-  up=`xenstore-read /local/domain/$i/memory/uptime | cut -d" " -f1`
-  idle=`xenstore-read /local/domain/$i/memory/uptime | cut -d" " -f2`
-  act=`echo $up - $idle | bc -iq`
+ xm tmem-list --all --long > $TMEMTMP
+ precache=`grep "C=CI:$i" $TMEMTMP | sed 's/C=CI.*Ec:\([0-9][0-9]*\).*/\1/'`
+ if xenstore-exists /local/domain/$i/memory/preswap; then
+  preswap=`xenstore-read /local/domain/$i/memory/preswap`
+  printf "%2d %8d%8d%8d%7d%8d%9d%9d%9d%9d\n" $i $tot $tgt $cmt $sin $sout $pgin $pgout $preswap $precache
+ else
+  printf "%2d %8d%8d%8d%9d%9d%10d%10d\n" $i $tot $tgt $cmt $sin $sout $pgin $pgout
  fi
- printf "%2d %8d%8d%8d%9d%9d%10d%10d%10.2f\n" $i $tot $tgt $cmt $sin $sout $pgin $pgout $act
  fi
 done
-echo Free memory: `xm info | grep free | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'` MB
+echo -n Free memory: `xm info | grep free | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'` MiB "  "
+tmem_free_pages=`grep "G=" $TMEMTMP | sed 's/G=.*Ta:\([0-9][0-9]*\).*/\1/'`
+if [ ! -z "$tmem_free_pages" ]; then
+ let "tmem_free_mb=$tmem_free_pages/256"
+ echo -n Idle tmem: $tmem_free_mb MiB "  "
+fi
+tmem_eph_pages=`grep "G=" $TMEMTMP | sed 's/G=.*Ec:\([0-9][0-9]*\).*/\1/'`
+if [ ! -z "$tmem_eph_pages" ]; then
+ let "tmem_eph_mb=$tmem_eph_pages/256"
+ echo -n Ephemeral tmem: $tmem_eph_mb MiB
+fi
+echo ""

tools/xenballoon/xenballoon.conf

@@ -89,3 +89,48 @@ XENBALLOON_SEND_VMSTAT=1
 # If xenbus is enabled, whether selfballooning or directed ballooning,
 # place the result of 'cat /proc/uptime" on xenbus at memory/uptime
 XENBALLOON_SEND_UPTIME=1
+
+## Type: boolean
+## Default: false
+#
+# If tmem is running, pages swapped to a swap disk may instead go
+# into preswap.  These pages may become stale (i.e. no longer need
+# to be saved because, e.g., the process owning them has gone away)
+# and stale pages use precious precious domain-allocated memory.
+# Periodically try to reduce preswap to squeeze out stale pages
+XENBALLOON_PRESWAP_SHRINK=false
+
+## Type: string
+## Default: "/proc/sys/vm/preswap"
+## (change to /sys/proc/vm/preswap in later kernels)
+#
+# If tmem and preswap are running, reading this file gives the number of
+# pages currently in preswap.  Writing it with N invokes the preswap_shrink
+# routine to reduce preswap to N pages.
+XENBALLOON_PRESWAP_SYSFILE=/proc/sys/vm/preswap
+
+## Type: integer (must be > 0)
+## Default: 20
+#
+# If tmem and preswap are running, attempts to reduce number of pages
+# currently in preswap.  For a value of n, 1/n of the pages will
+# be attempted to be shrunk.
+# If xenbus is enabled, may be overridden by {memory/preswaphysteresis}
+XENBALLOON_PRESWAP_HYSTERESIS=20
+
+## Type: integer (must be > 0)
+## Default: 10
+#
+# Number of ballooning intervals where preswap size remains at N pages
+# before preswap shrinking is attempted.  Also if shrinking fails to
+# shrink to the target, counter resets to this value.
+# If xenbus is enabled, may be overridden by {memory/preswapinertia}
+XENBALLOON_PRESWAP_INERTIA=10
+
+## Type: integer (0 or 1)
+## Default: 1
+#
+# If xenbus is enabled, whether selfballooning or directed ballooning,
+# place the size of preswap (in pages) on xenbus at memory/preswap
+XENBALLOON_SEND_PRESWAP=1
+

tools/xenballoon/xenballoond

@@ -5,9 +5,10 @@
 # Written by: Dan Magenheimer <dan.magenheimer@oracle.com>
 #
 # xenballoond - In-guest engine for Xen memory ballooning
-# Version: 080630
+# Original version: 080630
+# Updated 0906XX: add tmem preswap auto-shrinking
 #
-# Two "policies" are implemented:
+# Two self-ballooning "policies" are implemented:
 # - Selfballooning: Adjust memory periodically, with no (or little) input
 #     from domain0.  Target memory is determined solely by the
 #     Committed_AS line in /proc/meminfo, but parameters may adjust
@@ -149,6 +150,75 @@ send_memory_stats() {
 	fi
 }
 
+
+curpreswappages() {
+	pages=$(cat $XENBALLOON_PRESWAP_SYSFILE)
+	RETVAL=$pages
+	return  # value returned in RETVAL in pages
+}
+
+preswaphysteresis() {
+	RETVAL=$XENBALLOON_PRESWAP_HYSTERESIS
+	if [ $xenstore_enabled = "true" ]; then
+		if xenstore-exists memory/preswaphysteresis ; then
+			RETVAL=`xenstore-read memory/preswaphysteresis`
+		fi
+	fi
+	return
+}
+
+preswapinertia() {
+	RETVAL=$XENBALLOON_PRESWAP_INERTIA
+	if [ $xenstore_enabled = "true" ]; then
+		if xenstore-exists memory/preswapinertia ; then
+			RETVAL=`xenstore-read memory/preswapinertia`
+		fi
+	fi
+	return
+}
+
+send_preswap_stats() {
+	if [ ! $xenstore_enabled = "true" ]; then
+		return
+	fi
+	curpreswappages
+	preswap_pgs=$RETVAL
+	if [ $XENBALLOON_SEND_PRESWAP ]; then
+		xenstore-write memory/preswap "$preswap_pgs"
+	fi
+}
+
+shrink_preswap() {
+	if [ "$XENBALLOON_PRESWAP_SHRINK" = "false" ]; then
+		return
+	fi
+	if [ ! -f "$XENBALLOON_PRESWAP_SYSFILE" ]; then
+		return
+	fi
+	curpreswappages
+	preswaplast=$preswapnow
+	preswapnow=$RETVAL
+	if [ $preswapnow -eq 0 -o $preswapnow -ne $preswaplast ]; then
+		preswapinertia
+		preswapinertiacounter=$RETVAL
+		return
+	fi
+	let "preswapinertiacounter=$preswapinertiacounter-1"
+	if [ $preswapinertiacounter -ne 0 ]; then
+		return
+	fi
+	preswaphysteresis
+	preswaphys=$RETVAL
+	if [ $preswaphys -eq 0 ]; then
+		return
+	fi
+	let "tgtpreswappages=$(( $preswapnow - \
+				( $preswapnow / $preswaphys ) ))"
+	preswapinertia
+	preswapinertiacounter=$RETVAL
+	echo $tgtpreswappages > "$XENBALLOON_PRESWAP_SYSFILE"
+}
+
 if [ ! -f /proc/xen/balloon ]; then
 	echo "$0: no balloon driver installed"
 	exit 0
@@ -165,6 +235,7 @@ else
 	echo "$0: missing /usr/bin/xenstore-* tools, disabling directed ballooning"
 	xenstore_enabled=false
 fi
+preswapnow=0
 
 . /etc/sysconfig/xenballoon.conf
 
@@ -194,7 +265,9 @@ do
 		fi
 		interval=$XENBALLOON_INTERVAL
 	fi
+	shrink_preswap
 	send_memory_stats
+	send_preswap_stats
 	if [ $xenstore_enabled = "true" ]; then
 		if xenstore-exists memory/interval ; then
 			interval=`xenstore-read memory/interval`

tools/xenballoon/xenballoond.init

@@ -2,7 +2,7 @@
 #
 # xenballoond	Script to start and stop Xen ballooning daemon.
 #
-# Copyright (C) 2008 Oracle Corporation and/or its affiliates.
+# Copyright (C) 2009 Oracle Corporation and/or its affiliates.
 # All rights reserved.
 # Written by: Dan Magenheimer <dan.magenheimer@oracle.com>
 #