xenballoond (memory overcommit) scripts

See the README and conf files for more info.

Signed-off-by: Dan Magenheimer <dan.magenheimer@oracle.com>

tools/xenballoon/xenballoon-monitor

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# xenballoon-monitor - monitor certain stats from xenballoond
+#   (run in dom0 with "watch -d xenballoon-monitor" for xentop-like output)
+#
+# Copyright (C) 2009 Oracle Corporation and/or its affiliates.
+# All rights reserved
+# Written by: Dan Magenheimer <dan.magenheimer@oracle.com>
+#
+# Hint: Use "xm sched-credit -d 0 -w 2000" to watch on heavily loaded machines
+#
+echo "id   mem-kb  tgt-kb  commit   swapin  swapout      pgin     pgout active(sec)"
+for i in `xenstore-list /local/domain`; do
+ if [ "$i" -ne 0 ]; then
+ tot=0; tgt=0; sin=0; sout=0; pgin=0; pgout=0; cmt=0; up=0; idle=0; act=0;
+ if xenstore-exists /local/domain/$i/memory/meminfo; then
+  tot=`xenstore-read /local/domain/$i/memory/meminfo | grep MemTotal \
+   | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'`
+  cmt=`xenstore-read /local/domain/$i/memory/meminfo | grep Committed_AS \
+   | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'`
+ fi
+ if xenstore-exists /local/domain/$i/memory/selftarget; then
+  tgt=`xenstore-read /local/domain/$i/memory/selftarget`
+ fi
+ if xenstore-exists /local/domain/$i/memory/vmstat; then
+  sin=`xenstore-read /local/domain/$i/memory/vmstat | grep pswpin \
+ 	| cut -d" " -f2`
+  sout=`xenstore-read /local/domain/$i/memory/vmstat | grep pswpout \
+ 	| cut -d" " -f2`
+  pgin=`xenstore-read /local/domain/$i/memory/vmstat | grep pgpgin \
+ 	| cut -d" " -f2`
+  pgout=`xenstore-read /local/domain/$i/memory/vmstat | grep pgout \
+  	| cut -d" " -f2`
+ fi
+ if xenstore-exists /local/domain/$i/memory/uptime; then
+  up=`xenstore-read /local/domain/$i/memory/uptime | cut -d" " -f1`
+  idle=`xenstore-read /local/domain/$i/memory/uptime | cut -d" " -f2`
+  act=`echo $up - $idle | bc -iq`
+ fi
+ printf "%2d %8d%8d%8d%9d%9d%10d%10d%10.2f\n" $i $tot $tgt $cmt $sin $sout $pgin $pgout $act
+ fi
+done
+echo Free memory: `xm info | grep free | sed 's/[^1-9]*\([1-9][0-9]*\).*/\1/'` MB

tools/xenballoon/xenballoon.conf

@@ -0,0 +1,91 @@
+## Path: System/xen
+## Description: xen domain start/stop on boot
+## Type: string
+## Default: 
+
+# NOTE: "xenbus is enabled" means not only that /proc/xen/xenbus exists
+# but also that /usr/bin/xenstore-* tools are installed.
+
+## Type: boolean
+## Default: false
+#
+# If XENBALLOON_SELF is true, selfballooning will occur, meaning the
+# balloon driver will grow and shrink according to available memory.
+# If xenbus is enabled, may be overridden by {memory/selfballoon}==0
+# If false but xenballoond is able to communicate with domain0 via
+# xenbus, balloon targets will be set by domain0
+# 
+XENBALLOON_SELF=false
+
+## Type: integer (must be > 0)
+## Default: 1
+#
+# If self-ballooning, number of seconds between checks/adjustments.
+# If xenbus is enabled, may be overridden by {memory/interval}
+XENBALLOON_SELF_INTERVAL=1
+
+## Type: integer (must be > 0)
+## Default: 1
+#
+# If NOT self-ballooning but xenbus is enabled, number of seconds between
+# checks/adjustments. May be overridden by {memory/interval}
+XENBALLOON_INTERVAL=1
+
+## Type: integer (must be > 0)
+## Default: 10
+#
+# When current > target, reduces rate at which target memory is ballooned
+# out.  For a value of n, 1/n of the difference will be ballooned.
+# This value applies both to selfballooning and directed ballooning.
+# May be overridden by {memory/downhysteresis}
+XENBALLOON_AUTO_DOWNHYSTERESIS=10
+
+## Type: integer (must be > 0)
+## Default: 1
+#
+# When current < target, reduces rate at which target memory is reclaimed
+# (if available).  For a value of n, 1/n of the difference will be ballooned.
+# This value applies both to selfballooning and directed ballooning.
+# May be overridden by {memory/uphysteresis}
+XENBALLOON_AUTO_UPHYSTERESIS=1
+
+## Type: integer (must be >= 0)
+## Default: 0
+#
+# In order to avoid ballooning so much memory that a guest experiences
+# out-of-memory errors (OOMs), memory will not be ballooned out below
+# a minimum target, in MB.  If this value is 0 (default), an heuristic
+# based on the maximum amount of memory will be used.  (The heuristic
+# provides the same minimum as recent versions of the balloon driver but
+# early versions of the balloon driver did not enforce a minimum.)
+XENBALLOON_MINMEM=0
+
+## Type: string
+## Default: "/var/run/xenballoon-maxmem"
+#
+# Location where memory high-water mark is stored; if a guest supports
+# hot-add memory, maxmem might increase across time and the minimum
+# target heuristic is based on max memory. NOTE: Reboot after changing
+# this variable, else overballooning may occur.
+XENBALLOON_MAXMEMFILE=/var/run/xenballoon-maxmem
+
+## Type: integer (0 or 1)
+## Default: 1
+#
+# If xenbus is enabled, whether selfballooning or directed ballooning,
+# place the result of 'cat /proc/meminfo" on xenbus at memory/meminfo
+XENBALLOON_SEND_MEMINFO=1
+
+## Type: integer (0 or 1)
+## Default: 1
+#
+# If xenbus is enabled, whether selfballooning or directed ballooning,
+# place the result of 'cat /proc/vmstat" on xenbus at memory/vmstat
+XENBALLOON_SEND_VMSTAT=1
+
+## Type: integer (0 or 1)
+## Default: 1
+#
+# If xenbus is enabled, whether selfballooning or directed ballooning,
+# place the result of 'cat /proc/uptime" on xenbus at memory/uptime
+XENBALLOON_SEND_UPTIME=1

tools/xenballoon/xenballoond

@@ -0,0 +1,205 @@
+#!/bin/bash
+#
+# Copyright (C) 2008 Oracle Corporation and/or its affiliates.
+# All rights reserved.
+# Written by: Dan Magenheimer <dan.magenheimer@oracle.com>
+#
+# xenballoond - In-guest engine for Xen memory ballooning
+# Version: 080630
+#
+# Two "policies" are implemented:
+# - Selfballooning: Adjust memory periodically, with no (or little) input
+#     from domain0.  Target memory is determined solely by the
+#     Committed_AS line in /proc/meminfo, but parameters may adjust
+#     the rate at which the target is achieved.
+# - Directed ballooning: Adjust memory solely as directed by domain0
+#
+# Under some circumstances, "output" may also be generated; the contents
+# of /proc/meminfo and /proc/vmstat may be periodically placed on xenbus.
+#
+# If xenbus is running and the /usr/bin/xenstore-* tools are installed,
+# "xenbus is enabled".
+#
+# Parameters are documented in /etc/sysconfig/xenballoon.conf. Although 
+# some are not used with directed ballooning, all must be set properly.
+# If xenbus is enabled, some of these parameters may be overridden by values
+# set by domain0 via xenbus.
+
+minmb() {
+	RETVAL=$XENBALLOON_MINMEM
+	if [ $RETVAL -ne 0 ]; then
+		return $RETVAL
+	fi
+	kb=`cat $XENBALLOON_MAXMEMFILE`
+	let "mb=$kb/1024"
+	let "pages=$kb/4"
+	# this algorithm from drivers/xen/balloon/balloon.c:minimum_target()
+	# which was added to balloon.c in 2008 to avoid ballooning too small
+	# it is unnecessary here except to accomodate pre-2008 balloon drivers
+	# note that ranges are adjusted because a VM with "memory=1024"
+	# gets somewhat less than 1024MB
+	if [ $mb -lt 125 ]; then
+		let RETVAL="$(( 8 + ($pages >> 9) ))"
+	elif [ $mb -lt 500 ]; then
+		let RETVAL="$(( 40 + ($pages >> 10) ))"
+	elif [ $mb -lt 2000 ]; then
+		let RETVAL="$(( 104 + ($pages >> 11) ))"
+	else
+		let RETVAL="$(( 296 + ($pages >> 13) ))"
+	fi
+	return	# value returned in RETVAL in mB
+}
+
+curkb() {
+	kb=`grep MemTotal /proc/meminfo | sed 's/  */ /' | \
+		cut -f2 -d' '`
+	RETVAL=$kb
+	return  # value returned in RETVAL in kB
+}
+
+downhysteresis() {
+	RETVAL=$XENBALLOON_AUTO_DOWNHYSTERESIS
+	if [ $xenstore_enabled = "true" ]; then
+		if xenstore-exists memory/downhysteresis ; then
+			RETVAL=`xenstore-read memory/downhysteresis`
+		fi
+	fi
+	return
+}
+
+uphysteresis() {
+	RETVAL=$XENBALLOON_AUTO_UPHYSTERESIS
+	if [ $xenstore_enabled = "true" ]; then
+		if xenstore-exists memory/uphysteresis ; then
+			RETVAL=`xenstore-read memory/uphysteresis`
+		fi
+	fi
+	return
+}
+
+selfballoon_eval() {
+	if [ $xenstore_enabled = "true" ]; then
+		if xenstore-exists memory/selfballoon; then
+			RETVAL=`xenstore-read memory/selfballoon`
+			if [ $RETVAL -eq 1 ]; then
+				selfballoon_enabled=true
+				return
+			fi
+		fi
+	fi
+	selfballoon_enabled=$XENBALLOON_SELF
+	return
+}
+
+selftarget() {
+	tgtkb=`grep Committed_AS /proc/meminfo | sed 's/  */ /' | cut -f2 -d' '`
+	minmb
+	let "minbytes=$RETVAL*1024*1024"
+	let "tgtbytes=$tgtkb*1024"
+	if [ $tgtbytes -lt $minbytes ]; then
+		let "tgtbytes=$minbytes"
+	fi
+	RETVAL=$tgtbytes  # value returned in RETVAL in bytes
+	return
+}
+
+# $1 == 1 means use selftarget, else target in kB
+balloon_to_target() {
+	if [ "$1" -eq 1 ]; then
+		selftarget
+		tgtbytes=$RETVAL
+	else
+		let "tgtbytes=$(( $1 * 1024 ))"
+	fi
+	curkb
+	let "curbytes=$RETVAL*1024"
+	if [ $curbytes -gt $tgtbytes ]; then
+		downhysteresis
+		downhys=$RETVAL
+		if [ $downhys -ne 0 ]; then
+			let "tgtbytes=$(( $curbytes - \
+				( ( $curbytes - $tgtbytes ) / $downhys ) ))"
+		fi
+	else if [ $curbytes -lt $tgtbytes ]; then
+		uphysteresis
+		uphys=$RETVAL
+		let "tgtbytes=$(( $curbytes + \
+				( ( $tgtbytes - $curbytes ) / $uphys ) ))"
+		fi
+	fi
+	echo $tgtbytes > /proc/xen/balloon
+	if [ $xenstore_enabled = "true" ]; then
+		let "tgtkb=$(( $tgtbytes/1024 ))"
+		xenstore-write memory/selftarget $tgtkb
+	fi
+}
+
+send_memory_stats() {
+	if [ ! $xenstore_enabled = "true" ]; then
+		return
+	fi
+	if [ $XENBALLOON_SEND_MEMINFO ]; then
+		xenstore-write memory/meminfo "`cat /proc/meminfo`"
+	fi
+	if [ $XENBALLOON_SEND_VMSTAT ]; then
+		xenstore-write memory/vmstat "`cat /proc/vmstat`"
+	fi
+	if [ $XENBALLOON_SEND_UPTIME ]; then
+		xenstore-write memory/uptime "`cat /proc/uptime`"
+	fi
+}
+
+if [ ! -f /proc/xen/balloon ]; then
+	echo "$0: no balloon driver installed"
+	exit 0
+fi
+if [ ! -f /proc/meminfo ]; then
+	echo "$0: can't read /proc/meminfo"
+	exit 0
+fi
+xenstore_enabled=true
+if [ -f /usr/bin/xenstore-exists -a -f /usr/bin/xenstore-read -a \
+     -f /usr/bin/xenstore-write ]; then
+	xenstore_enabled=true
+else
+	echo "$0: missing /usr/bin/xenstore-* tools, disabling directed ballooning"
+	xenstore_enabled=false
+fi
+
+. /etc/sysconfig/xenballoon.conf
+
+while true;
+do
+	# handle special case for PV domains with hot-add memory
+	if [ ! -f $XENBALLOON_MAXMEMFILE ]; then
+		maxkb=0
+	else
+		maxkb=`cat $XENBALLOON_MAXMEMFILE`
+	fi
+	curkb=`grep MemTotal /proc/meminfo | sed 's/  */ /' | cut -f2 -d' '`
+	if [ $curkb -gt $maxkb ]; then
+		echo $curkb > $XENBALLOON_MAXMEMFILE
+	fi
+	interval=$XENBALLOON_INTERVAL
+	# do self-ballooning
+	selfballoon_eval
+	if [ $selfballoon_enabled = "true" ]; then
+		balloon_to_target 1
+		interval=$XENBALLOON_SELF_INTERVAL
+	# or do directed ballooning
+	elif [ $xenstore_enabled = "true" ]; then
+		if xenstore-exists memory/target ; then
+			tgtkb=`xenstore-read memory/target`
+			balloon_to_target $tgtkb
+		fi
+		interval=$XENBALLOON_INTERVAL
+	fi
+	send_memory_stats
+	if [ $xenstore_enabled = "true" ]; then
+		if xenstore-exists memory/interval ; then
+			interval=`xenstore-read memory/interval`
+		fi
+	fi
+	sleep $interval
+done &
+

tools/xenballoon/xenballoond.README

@@ -0,0 +1,82 @@
+Xenballoond.README
+Preliminary version 0.1, 2008/06/30
+
+Copyright (C) 2008 Oracle Corporation and/or its affiliates.
+All rights reserved.
+Written by Dan Magenheimer <dan.magenheimer@oracle.com>
+
+INTRODUCTION
+
+Xenballoond runs in guest domains and both implements selfballooning and
+provides metrics to dom0 for (future) directed ballooning.  Both capabilities
+provide a foundation for basic "memory overcommit" functionality.
+
+With selfballooning enabled, xenballoond uses the Committed_AS value found
+in /proc/meminfo as a first approximation of how much memory is required
+by the guest and feeds this statistic back to the balloon driver to inflate
+or deflate the balloon as required to achieve the target guest memory size.
+Hysteresis parameters may be adjusted to rate-limit balloon inflation
+and deflation.
+
+If configured, certain selfballooning parameters -- including notably
+enabling/disabling of self-ballooning -- can be controlled from domain0.
+(These are fully documented in xenballoon.conf.)
+
+If configured, the following guest statistics are sent back to domain0:
+- /proc/meminfo
+- /proc/vmstat
+- /proc/uptime
+In a future release, some of these values will be used by a policy module
+in domain0 to control guest balloon size and provide memory balancing
+across all guests on a given system.
+
+Note that no page sharing (content-based or otherwise) is implemented
+and no VMM-based swapping is necessary.
+
+For more information, see:
+http://www.xen.org/files/xensummitboston08/MemoryOvercommit-XenSummit2008.pdf
+http://wiki.xensource.com/xenwiki/Open_Topics_For_Discussion?action=AttachFile&do=get&target=Memory+Overcommit.pdf
+
+INSTALLATION AND DEPLOYMENT
+
+In this preliminary release:
+- directed ballooning is not implemented, though a monitor is provided
+- only Redhat-based guests are supported
+
+Guest prerequisites to use xenballoond:
+- each guest must be configured with adequate[1] swap space
+- each guest must have the balloon driver installed (/proc/xen/balloon exists) 
+- if directed ballooning (or monitoring) is desired, xenstore tools must be
+  installed in each guest in /usr/bin [2]
+
+[1] for best results, for a guest that is configured with maxmem=N and
+    requires Z MB of swap space without xenballoond, available swap should
+    be increased to N+Z MB when xenballoond is running
+[2] specifically xenstore-read, xenstore-exists, and xenstore-write must
+    be installed.  Binaries can be obtained, for example, by building
+    xen-vvv.gz/tools in a guest-binary-compatible development tree
+
+Instructions to install/deploy xenballoond (in Redhat-based system):
+- in each guest:
+  - ensure pre-requisites are met (see above)
+  - place xenballoon.conf in /etc/sysconfig
+  - place xenballoond in /usr/sbin
+  - copy xenballoond.init to /etc/rc.d/init.d/xenballoond (note file rename)
+  - edit /etc/sysconfig/xenballoond.conf as desired (especially note that
+    selfballooning defaults as off)
+  - start xenballoond with "service xenballoond start", and/or configure
+    xenballoond to start at init (e.g. "chkconfig xenballoond on")
+- in domain0:
+  - if monitoring is desired, xenballoon-monitor may be installed in /usr/sbin
+- note that certain xenballoond.conf variables may be overridden by domain0
+  if xenstore is running in the guest; these are fully documented in
+  xenballoond.conf
+
+TODO:
+080630 modifications to support SUSE-based and debian-based guests
+080630 domain0 ballooning policy module
+080630 experiment with more aggressive (optionally) memory minimum targets
+080630 BUG: xenballoond doesn't properly record the fact that it's running;
+       e.g. flipping between run levels 5 and 3 launches additional daemons
+080630 BUG: reports of possible incompatibilites between ballooning and
+       save/restore/migrate have not been duplicated

tools/xenballoon/xenballoond.init

@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# xenballoond	Script to start and stop Xen ballooning daemon.
+#
+# Copyright (C) 2008 Oracle Corporation and/or its affiliates.
+# All rights reserved.
+# Written by: Dan Magenheimer <dan.magenheimer@oracle.com>
+#
+# chkconfig: 2345 98 01
+# description: Starts and stops the Xen control daemon.
+### BEGIN INIT INFO
+# Provides:          xenballoond
+# Required-Start:    $syslog $remote_fs
+# Should-Start:
+# Required-Stop:     $syslog $remote_fs
+# Should-Stop:
+# Default-Start:     3 4 5
+# Default-Stop:      0 1 2 6
+# Default-Enabled:   yes
+# Short-Description: Start/stop xend
+# Description:       Starts and stops the Xen ballooning daemon.
+### END INIT INFO
+
+# Source function library
+. /etc/init.d/functions
+
+#don't use in domain0
+[ -f /proc/xen/capabilities ] && \
+	grep -q "control_d" /proc/xen/capabilities && exit 0
+
+if [ -f /etc/sysconfig/xenballoon.conf ]; then
+	. /etc/sysconfig/xenballoon.conf
+fi
+
+# Check that balloon driver is present
+[ ! -f /proc/xen/balloon ] && exit 0
+
+# Record original memory (in kB)
+[ -z "$XENBALLOON_MAXMEMFILE" ] && exit 0
+let maxmem=`grep MemTotal /proc/meminfo | sed 's/  */ /' | cut -f2 -d' '`
+if [ -f "$XENBALLOON_MAXMEMFILE" ]; then
+	let oldmax=`cat $XENBALLOON_MAXMEMFILE`
+	if [ $oldmax -gt $maxmem ]; then
+		let maxmem=oldmax
+	fi
+fi
+echo $maxmem > $XENBALLOON_MAXMEMFILE
+
+RETVAL=0
+prog="xenballoond"
+
+start() {
+        # Start daemons.
+        echo -n $"Starting $prog: "
+        daemon xenballoond $OPTIONS
+	RETVAL=$?
+        echo
+	return $RETVAL
+}
+
+stop() {
+        echo -n $"Shutting down $prog: "
+	killproc xenballoond
+	RETVAL=$?
+        echo
+	return $RETVAL
+}
+
+# See how we were called.
+case "$1" in
+  start)
+	start
+        ;;
+  stop)
+	stop
+        ;;
+  status)
+	status xenballoond
+	RETVAL=$?
+	;;
+  restart|reload)
+	stop
+	start
+	RETVAL=$?
+	;;
+  *)
+        echo $"Usage: $0 {start|stop|restart|status}"
+        exit 1
+esac
+
+exit $RETVAL