diff --git a/MAINTAINERS b/MAINTAINERS index f5c9dc218d..4258e48c1b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -260,8 +260,15 @@ M: Shriram Rajagopalan M: Yang Hongyang S: Maintained F: docs/README.remus +F: tools/libxc/xc_domain_save.c +F: tools/libxc/xc_domain_restore.c F: tools/blktap2/drivers/block-remus.c F: tools/blktap2/drivers/hashtable* +F: tools/libxl/libxl_remus_* +F: tools/libxl/libxl_netbuffer.c +F: tools/libxl/libxl_nonetbuffer.c +F: tools/hotplug/Linux/remus-netbuf-setup +F: tools/hotplug/Linux/block-drbd-probe SCHEDULING M: George Dunlap diff --git a/README b/README index 81bf938e3c..78c5db24e5 100644 --- a/README +++ b/README @@ -73,6 +73,10 @@ disabled at compile time: * markdown * figlet (for generating the traditional Xen start of day banner) * systemd daemon development files + * Development install of libnl3 (e.g., libnl-3-200, + libnl-3-dev, etc). Required if network buffering is desired + when using Remus with libxl. See tools/remus/README for detailed + information. Second, you need to acquire a suitable kernel for use in domain 0. If possible you should use a kernel provided by your OS distributor. If diff --git a/config/Tools.mk.in b/config/Tools.mk.in index 974e28e892..a69b8466ab 100644 --- a/config/Tools.mk.in +++ b/config/Tools.mk.in @@ -43,6 +43,9 @@ PTHREAD_LIBS := @PTHREAD_LIBS@ PTYFUNCS_LIBS := @PTYFUNCS_LIBS@ +LIBNL3_LIBS := @LIBNL3_LIBS@ +LIBNL3_CFLAGS := @LIBNL3_CFLAGS@ + # Download GIT repositories via HTTP or GIT's own protocol? # GIT's protocol is faster and more robust, when it works at all (firewalls # may block it). We make it the default, but if your GIT repository downloads @@ -62,6 +65,7 @@ CONFIG_BLKTAP1 := @blktap1@ CONFIG_BLKTAP2 := @blktap2@ CONFIG_VTPM := @vtpm@ CONFIG_QEMUU_EXTRA_ARGS:= @EXTRA_QEMUU_CONFIGURE_ARGS@ +CONFIG_REMUS_NETBUF := @remus_netbuf@ CONFIG_SYSTEMD := @systemd@ SYSTEMD_CFLAGS := @SYSTEMD_CFLAGS@ diff --git a/docs/README.remus b/docs/README.remus index 9fa00fefc9..20783c93da 100644 --- a/docs/README.remus +++ b/docs/README.remus @@ -2,3 +2,19 @@ Remus provides fault tolerance for virtual machines by sending continuous checkpoints to a backup, which will activate if the target VM fails. See the website at http://wiki.xen.org/wiki/Remus for details. + +Using Remus with libxl on Xen 4.5 and higher: + To enable network buffering, you need libnl 3.2.8 + or higher along with the development headers and command line utilities. + If your distro does not have the appropriate libnl3 version, you can find + the latest source tarball of libnl3 at http://www.carisma.slowglass.com/~tgr/libnl/ + +Disk replication: + VMs protected by Remus need to use DRBD based disk backends. Specifically, you + need a compile and install a custom version of DRBD, that is available publicly + at https://github.com/rshriram/remus-drbd + This code is based on DRBD 8.3.11 and uses a new replication protocol (named + protocol D) for asynchronous disk checkpoint replication. A protected VM's DRBD + disks on the primary and backup hosts need to be configured to use protocol D + as the replication protocol. An example resource configuration file can be found + in the aforementioned github repository. diff --git a/docs/man/xl.conf.pod.5 b/docs/man/xl.conf.pod.5 index 7c43bde4fa..8ae19bb5f3 100644 --- a/docs/man/xl.conf.pod.5 +++ b/docs/man/xl.conf.pod.5 @@ -105,6 +105,12 @@ Configures the default gateway device to set for virtual network devices. Default: C +=item B + +Configures the default script used by Remus to setup network buffering. + +Default: C + =item B Configures the default output format used by xl when printing "machine diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1 index f9bc812c04..362e92faa1 100644 --- a/docs/man/xl.pod.1 +++ b/docs/man/xl.pod.1 @@ -436,7 +436,7 @@ Enable Remus HA for domain. By default B relies on ssh as a transport mechanism between the two hosts. N.B: Remus support in xl is still in experimental (proof-of-concept) phase. - There is no support for network or disk buffering at the moment. + Disk replication support is limited to DRBD disks. B @@ -446,11 +446,6 @@ B Checkpoint domain memory every MS milliseconds (default 200ms). -=item B<-b> - -Replicate memory checkpoints to /dev/null (blackhole). -Generally useful for debugging. - =item B<-u> Disable memory checkpoint compression. @@ -465,6 +460,29 @@ If empty, run instead of ssh xl migrate-receive -r [-e]. On the new host, do not wait in the background (on ) for the death of the domain. See the corresponding option of the I subcommand. +=item B<-N> I + +Use to setup network buffering instead of the +default script (/etc/xen/scripts/remus-netbuf-setup). + +=item B<-F> + +Run Remus in unsafe mode. Use this option with caution as failover may +not work as intended. + +=item B<-b> + +Replicate memory checkpoints to /dev/null (blackhole). +Generally useful for debugging. Requires enabling unsafe mode. + +=item B<-n> + +Disable network output buffering. Requires enabling unsafe mode. + +=item B<-d> + +Disable disk replication. Requires enabling unsafe mode. + =back =item B I diff --git a/docs/misc/xenstore-paths.markdown b/docs/misc/xenstore-paths.markdown index ea67536841..d94ea9d6b0 100644 --- a/docs/misc/xenstore-paths.markdown +++ b/docs/misc/xenstore-paths.markdown @@ -393,6 +393,10 @@ The guest's virtual time offset from UTC in seconds. The device model version for a domain. +#### /libxl/$DOMID/remus/netbuf/$DEVID/ifb = STRING [n,INTERNAL] + +ifb device used by Remus to buffer network output from the associated vif. + [BLKIF]: http://xenbits.xen.org/docs/unstable/hypercall/include,public,io,blkif.h.html [FBIF]: http://xenbits.xen.org/docs/unstable/hypercall/include,public,io,fbif.h.html [HVMPARAMS]: http://xenbits.xen.org/docs/unstable/hypercall/include,public,hvm,params.h.html diff --git a/tools/configure.ac b/tools/configure.ac index 4f45418783..cfa4dd674f 100644 --- a/tools/configure.ac +++ b/tools/configure.ac @@ -320,6 +320,22 @@ esac # Checks for header files. AC_CHECK_HEADERS([yajl/yajl_version.h sys/eventfd.h valgrind/memcheck.h utmp.h]) +# Check for libnl3 >=3.2.8. If present enable remus network buffering. +PKG_CHECK_MODULES(LIBNL3, [libnl-3.0 >= 3.2.8 libnl-route-3.0 >= 3.2.8], + [libnl3_lib="y"], [libnl3_lib="n"]) + +AS_IF([test "x$libnl3_lib" = "xn" ], [ + AC_MSG_WARN([Disabling support for Remus network buffering. + Please install libnl3 libraries, command line tools and devel + headers - version 3.2.8 or higher]) + AC_SUBST(remus_netbuf, [n]) + ],[ + AC_SUBST(remus_netbuf, [y]) +]) + +AC_SUBST(LIBNL3_LIBS) +AC_SUBST(LIBNL3_CFLAGS) + fi # ! $rump AX_AVAILABLE_SYSTEMD() diff --git a/tools/hotplug/Linux/Makefile b/tools/hotplug/Linux/Makefile index d5a9ed2201..5317feff26 100644 --- a/tools/hotplug/Linux/Makefile +++ b/tools/hotplug/Linux/Makefile @@ -16,6 +16,7 @@ XEN_SCRIPTS += vif-nat XEN_SCRIPTS += vif-openvswitch XEN_SCRIPTS += vif2 XEN_SCRIPTS += vif-setup +XEN_SCRIPTS-$(CONFIG_REMUS_NETBUF) += remus-netbuf-setup XEN_SCRIPTS += block XEN_SCRIPTS += block-enbd block-nbd XEN_SCRIPTS-$(CONFIG_BLKTAP1) += blktap @@ -23,6 +24,7 @@ XEN_SCRIPTS += xen-hotplug-cleanup XEN_SCRIPTS += external-device-migrate XEN_SCRIPTS += vscsi XEN_SCRIPTS += block-iscsi +XEN_SCRIPTS += block-drbd-probe XEN_SCRIPTS += $(XEN_SCRIPTS-y) SUBDIRS-$(CONFIG_SYSTEMD) += systemd diff --git a/tools/hotplug/Linux/block-drbd-probe b/tools/hotplug/Linux/block-drbd-probe new file mode 100755 index 0000000000..247a9d098d --- /dev/null +++ b/tools/hotplug/Linux/block-drbd-probe @@ -0,0 +1,87 @@ +#! /bin/bash +# +# Copyright (C) 2014 FUJITSU LIMITED +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of version 2.1 of the GNU Lesser General Public +# License as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# Usage: +# block-drbd-probe devicename +# +# Return value: +# 0: the device is drbd device +# 1: the device is not drbd device +# 2: unkown error +# 3: the drbd device does not use protocol D +# 4: the drbd device is not ready + +set -e + +drbd_res= + +function get_res_name() +{ + local drbd_dev=$1 + local drbd_dev_list=($(drbdadm sh-dev all)) + local drbd_res_list=($(drbdadm sh-resource all)) + local temp_drbd_dev temp_drbd_res + local found=0 + + for temp_drbd_dev in ${drbd_dev_list[@]}; do + if [[ "$temp_drbd_dev" == "$drbd_dev" ]]; then + found=1 + break + fi + done + + if [[ $found -eq 0 ]]; then + return 1 + fi + + for temp_drbd_res in ${drbd_res_list[@]}; do + temp_drbd_dev=$(drbdadm sh-dev $temp_drbd_res) + if [[ "$temp_drbd_dev" == "$drbd_dev" ]]; then + drbd_res="$temp_drbd_res" + return 0 + fi + done + + # OOPS + return 2 +} + +get_res_name $1 +rc=$? +if [[ $rc -ne 0 ]]; then + exit $rc +fi + +# check protocol +drbdsetup $1 show | grep -q "protocol D;" +if [[ $? -ne 0 ]]; then + exit 3 +fi + +# check connect status +state=$(drbdadm cstate "$drbd_res") +if [[ "$state" != "Connected" ]]; then + exit 4 +fi + +# check role +role=$(drbdadm role "$drbd_res") +if [[ "$role" != "Primary/Secondary" ]]; then + exit 4 +fi + +exit 0 diff --git a/tools/hotplug/Linux/remus-netbuf-setup b/tools/hotplug/Linux/remus-netbuf-setup new file mode 100644 index 0000000000..87dfa69778 --- /dev/null +++ b/tools/hotplug/Linux/remus-netbuf-setup @@ -0,0 +1,230 @@ +#!/bin/bash +#============================================================================ +# ${XEN_SCRIPT_DIR}/remus-netbuf-setup +# +# Script for attaching a network buffer to the specified vif (in any mode). +# The hotplugging system will call this script when starting remus via libxl +# API, libxl_domain_remus_start. +# +# Usage: +# remus-netbuf-setup (setup|teardown) +# +# Environment vars: +# vifname vif interface name (required). +# XENBUS_PATH path in Xenstore, where the REMUS_IFB device details will be +# stored or read from (required). +# (libxl passes /libxl//remus/netbuf/) +# REMUS_IFB ifb interface to be cleaned up (required). [for teardown op only] + +# Written to the store: (setup operation) +# XENBUS_PATH/ifb= the REMUS_IFB device serving +# as the intermediate buffer through which the interface's network output +# can be controlled. +# + +# Remus network buffering requirements: + +# We need to buffer (queue) egress traffic from every vif attached to +# the guest and release the buffers when the checkpoint associated +# with them has been committed at the backup host. We achieve this +# with the help of the plug queuing discipline (sch_plug module). +# Simply put, Remus' network buffering imposes traffic +# shaping on the guest's vif(s). + +# Limitations and Workarounds: + +# Egress traffic from a vif appears as ingress traffic to dom0. Linux +# supports policing (dropping packets) but not traffic shaping +# (queuing packets) on ingress traffic. The standard workaround to +# this limitation is to attach an ingress qdisc to the guest vif, +# redirect all egress traffic from the guest to an intermediate +# queuing interface, and apply egress rules to it. The IFB +# (Intermediate Functional Block) device serves the purpose of an +# intermediate queuing interface. +# + +# The following commands install a network buffer on a +# guest's vif (vif1.0) using an IFB device (ifb0): +# +# ip link set dev ifb0 up +# tc qdisc add dev vif1.0 ingress +# tc filter add dev vif1.0 parent ffff: proto ip \ +# prio 10 u32 match u32 0 0 action mirred egress redirect dev ifb0 +# nl-qdisc-add --dev=ifb0 --parent root plug +# nl-qdisc-add --dev=ifb0 --parent root --update plug --limit=10000000 +# (10MB limit on buffer) +# +# So order of operations when installing a network buffer on vif1.0 +# 1. find a free ifb and bring up the device +# 2. redirect traffic from vif1.0 to ifb: +# 2.1 add ingress qdisc to vif1.0 (to capture outgoing packets from guest) +# 2.2 use tc filter command with actions mirred egress + redirect +# 3. install plug_qdisc on ifb device, with which we can buffer/release +# guest's network output from vif1.0 +# +# Note: +# 1. If the setup process fails, the script's cleanup is limited to removing the +# ingress qdisc on the guest vif, so that its traffic can flow normally. +# The chosen ifb device is not torn down. Libxl has to execute the +# teardown op to remove other qdiscs and subsequently free the IFB device. +# +# 2. The teardown op may be invoked multiple times by libxl. + +#============================================================================ + +# Unlike other vif scripts, vif-common is not needed here as it executes vif +#specific setup code such as renaming. +dir=$(dirname "$0") +. "$dir/xen-hotplug-common.sh" + +findCommand "$@" + +if [ "$command" != "setup" -a "$command" != "teardown" ] +then + echo "Invalid command: $command" + log err "Invalid command: $command" + exit 1 +fi + +evalVariables "$@" + +: ${vifname:?} +: ${XENBUS_PATH:?} + +check_libnl_tools() { + if ! command -v nl-qdisc-list > /dev/null 2>&1; then + fatal "Unable to find nl-qdisc-list tool" + fi + if ! command -v nl-qdisc-add > /dev/null 2>&1; then + fatal "Unable to find nl-qdisc-add tool" + fi + if ! command -v nl-qdisc-delete > /dev/null 2>&1; then + fatal "Unable to find nl-qdisc-delete tool" + fi +} + +# We only check for modules. We don't load them. +# User/Admin is supposed to load ifb during boot time, +# ensuring that there are enough free ifbs in the system. +# Other modules will be loaded automatically by tc commands. +check_modules() { + for m in ifb sch_plug sch_ingress act_mirred cls_u32 + do + if ! modinfo $m > /dev/null 2>&1; then + fatal "Unable to find $m kernel module" + fi + done +} + +#return 0 if the ifb is free +check_ifb() { + local installed=`nl-qdisc-list -d $1` + [ -n "$installed" ] && return 1 + + for domid in `xenstore-list "/local/domain" 2>/dev/null || true` + do + [ $domid -eq 0 ] && continue + xenstore-exists "/libxl/$domid/remus/netbuf" || continue + for devid in `xenstore-list "/libxl/$domid/remus/netbuf" 2>/dev/null || true` + do + local path="/libxl/$domid/remus/netbuf/$devid/ifb" + xenstore-exists $path || continue + local ifb=`xenstore-read "$path" 2>/dev/null || true` + [ "$ifb" = "$1" ] && return 1 + done + done + + return 0 +} + +setup_ifb() { + + for ifb in `ifconfig -a -s|egrep ^ifb|cut -d ' ' -f1` + do + check_ifb "$ifb" || continue + REMUS_IFB="$ifb" + break + done + + if [ -z "$REMUS_IFB" ] + then + fatal "Unable to find a free ifb device for $vifname" + fi + + #not using xenstore_write that automatically exits on error + #because we need to cleanup + xenstore_write "$XENBUS_PATH/ifb" "$REMUS_IFB" + do_or_die ip link set dev "$REMUS_IFB" up +} + +redirect_vif_traffic() { + local vif=$1 + local ifb=$2 + + do_or_die tc qdisc add dev "$vif" ingress + + tc filter add dev "$vif" parent ffff: proto ip prio 10 \ + u32 match u32 0 0 action mirred egress redirect dev "$ifb" >/dev/null 2>&1 + + if [ $? -ne 0 ] + then + do_without_error tc qdisc del dev "$vif" ingress + fatal "Failed to redirect traffic from $vif to $ifb" + fi +} + +add_plug_qdisc() { + local vif=$1 + local ifb=$2 + + nl-qdisc-add --dev="$ifb" --parent root plug >/dev/null 2>&1 + if [ $? -ne 0 ] + then + do_without_error tc qdisc del dev "$vif" ingress + fatal "Failed to add plug qdisc to $ifb" + fi + + #set ifb buffering limit in bytes. Its okay if this command fails + nl-qdisc-add --dev="$ifb" --parent root \ + --update plug --limit=10000000 >/dev/null 2>&1 || true +} + +teardown_netbuf() { + local vif=$1 + local ifb=$2 + + #Check if the XENBUS_PATH/ifb exists and has IFB name same as REMUS_IFB. + #Otherwise, if the teardown op is called multiple times, then we may end + #up freeing another domain's allocated IFB inside the if loop. + xenstore-exists "$XENBUS_PATH/ifb" && \ + local ifb2=`xenstore-read "$XENBUS_PATH/ifb" 2>/dev/null || true` + + if [[ "$ifb2" && "$ifb2" == "$ifb" ]]; then + do_without_error ip link set dev "$ifb" down + do_without_error nl-qdisc-delete --dev="$ifb" --parent root plug >/dev/null 2>&1 + xenstore-rm -t "$XENBUS_PATH/ifb" 2>/dev/null || true + fi + do_without_error tc qdisc del dev "$vif" ingress + xenstore-rm -t "$XENBUS_PATH/hotplug-status" 2>/dev/null || true + xenstore-rm -t "$XENBUS_PATH/hotplug-error" 2>/dev/null || true +} + +case "$command" in + setup) + check_libnl_tools + check_modules + + claim_lock "pickifb" + setup_ifb + redirect_vif_traffic "$vifname" "$REMUS_IFB" + add_plug_qdisc "$vifname" "$REMUS_IFB" + release_lock "pickifb" + + success + ;; + teardown) + teardown_netbuf "$vifname" "$REMUS_IFB" + ;; +esac + +log debug "Successful remus-netbuf-setup $command for $vifname, ifb $REMUS_IFB." diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index 496a269e7f..a6c3b0eb52 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -21,11 +21,17 @@ endif LIBXL_LIBS = LIBXL_LIBS = $(LDLIBS_libxenctrl) $(LDLIBS_libxenguest) $(LDLIBS_libxenstore) $(LDLIBS_libblktapctl) $(PTYFUNCS_LIBS) $(LIBUUID_LIBS) +ifeq ($(CONFIG_REMUS_NETBUF),y) +LIBXL_LIBS += $(LIBNL3_LIBS) +endif CFLAGS_LIBXL += $(CFLAGS_libxenctrl) CFLAGS_LIBXL += $(CFLAGS_libxenguest) CFLAGS_LIBXL += $(CFLAGS_libxenstore) CFLAGS_LIBXL += $(CFLAGS_libblktapctl) +ifeq ($(CONFIG_REMUS_NETBUF),y) +CFLAGS_LIBXL += $(LIBNL3_CFLAGS) +endif CFLAGS_LIBXL += -Wshadow LIBXL_LIBS-$(CONFIG_ARM) += -lfdt @@ -43,6 +49,15 @@ LIBXL_OBJS-y += libxl_blktap2.o else LIBXL_OBJS-y += libxl_noblktap2.o endif + +ifeq ($(CONFIG_REMUS_NETBUF),y) +LIBXL_OBJS-y += libxl_netbuffer.o +else +LIBXL_OBJS-y += libxl_nonetbuffer.o +endif + +LIBXL_OBJS-y += libxl_remus_device.o libxl_remus_disk_drbd.o + LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index 3735f55dbe..9f629c48af 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -782,6 +782,10 @@ out: return ptr; } +static void libxl__remus_setup_done(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc); +static void libxl__remus_setup_failed(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc); static void remus_failover_cb(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); @@ -800,6 +804,22 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info, goto out; } + libxl_defbool_setdefault(&info->allow_unsafe, false); + libxl_defbool_setdefault(&info->blackhole, false); + libxl_defbool_setdefault(&info->compression, true); + libxl_defbool_setdefault(&info->netbuf, true); + libxl_defbool_setdefault(&info->diskbuf, true); + + if (!libxl_defbool_val(info->allow_unsafe) && + (libxl_defbool_val(info->blackhole) || + !libxl_defbool_val(info->netbuf) || + !libxl_defbool_val(info->diskbuf))) { + LOG(ERROR, "Unsafe mode must be enabled to replicate to /dev/null," + "disable network buffering and disk replication"); + goto out; + } + + GCNEW(dss); dss->ao = ao; dss->callback = remus_failover_cb; @@ -813,16 +833,62 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info, assert(info); - /* TBD: Remus setup - i.e. attach qdisc, enable disk buffering, etc */ + /* Convenience aliases */ + libxl__remus_devices_state *const rds = &dss->rds; + + if (libxl_defbool_val(info->netbuf)) { + if (!libxl__netbuffer_enabled(gc)) { + LOG(ERROR, "Remus: No support for network buffering"); + goto out; + } + rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF); + } + + if (libxl_defbool_val(info->diskbuf)) + rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD); + + rds->ao = ao; + rds->domid = domid; + rds->callback = libxl__remus_setup_done; /* Point of no return */ - libxl__domain_suspend(egc, dss); + libxl__remus_devices_setup(egc, rds); return AO_INPROGRESS; out: return AO_ABORT(rc); } +static void libxl__remus_setup_done(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (!rc) { + libxl__domain_suspend(egc, dss); + return; + } + + LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d", + dss->domid, rc); + rds->callback = libxl__remus_setup_failed; + libxl__remus_devices_teardown(egc, rds); +} + +static void libxl__remus_setup_failed(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) + LOG(ERROR, "Remus: failed to teardown device after setup failed" + " for guest with domid %u, rc %d", dss->domid, rc); + + dss->callback(egc, dss, rc); +} + static void remus_failover_cb(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { @@ -832,10 +898,6 @@ static void remus_failover_cb(libxl__egc *egc, * backup died or some network error occurred preventing us * from sending checkpoints. */ - - /* TBD: Remus cleanup - i.e. detach qdisc, release other - * resources. - */ libxl__ao_complete(egc, ao, rc); } diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h index 9ae0fccc6b..2700cc1eab 100644 --- a/tools/libxl/libxl.h +++ b/tools/libxl/libxl.h @@ -647,6 +647,12 @@ typedef struct libxl__ctx libxl_ctx; */ #define LIBXL_HAVE_BUILDINFO_SERIAL_LIST 1 +/* + * LIBXL_HAVE_REMUS + * If this is defined, then libxl supports remus. + */ +#define LIBXL_HAVE_REMUS 1 + typedef uint8_t libxl_mac[6]; #define LIBXL_MAC_FMT "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx" #define LIBXL_MAC_FMTLEN ((2*6)+5) /* 6 hex bytes plus 5 colons */ diff --git a/tools/libxl/libxl_device.c b/tools/libxl/libxl_device.c index 342544644e..4b51ded820 100644 --- a/tools/libxl/libxl_device.c +++ b/tools/libxl/libxl_device.c @@ -453,6 +453,7 @@ void libxl__prepare_ao_device(libxl__ao *ao, libxl__ao_device *aodev) /* We init this here because we might call device_hotplug_done * without actually calling any hotplug script */ libxl__async_exec_init(&aodev->aes); + libxl__ev_child_init(&aodev->child); } /* multidev */ @@ -479,15 +480,12 @@ void libxl__multidev_begin(libxl__ao *ao, libxl__multidev *multidev) multidev->preparation = libxl__multidev_prepare(multidev); } -static void multidev_one_callback(libxl__egc *egc, libxl__ao_device *aodev); - -libxl__ao_device *libxl__multidev_prepare(libxl__multidev *multidev) { +void libxl__multidev_prepare_with_aodev(libxl__multidev *multidev, + libxl__ao_device *aodev) { STATE_AO_GC(multidev->ao); - libxl__ao_device *aodev; - GCNEW(aodev); aodev->multidev = multidev; - aodev->callback = multidev_one_callback; + aodev->callback = libxl__multidev_one_callback; libxl__prepare_ao_device(ao, aodev); if (multidev->used >= multidev->allocd) { @@ -495,11 +493,19 @@ libxl__ao_device *libxl__multidev_prepare(libxl__multidev *multidev) { GCREALLOC_ARRAY(multidev->array, multidev->allocd); } multidev->array[multidev->used++] = aodev; +} + +libxl__ao_device *libxl__multidev_prepare(libxl__multidev *multidev) { + STATE_AO_GC(multidev->ao); + libxl__ao_device *aodev; + + GCNEW(aodev); + libxl__multidev_prepare_with_aodev(multidev, aodev); return aodev; } -static void multidev_one_callback(libxl__egc *egc, libxl__ao_device *aodev) +void libxl__multidev_one_callback(libxl__egc *egc, libxl__ao_device *aodev) { STATE_AO_GC(aodev->ao); libxl__multidev *multidev = aodev->multidev; @@ -523,7 +529,7 @@ void libxl__multidev_prepared(libxl__egc *egc, libxl__multidev *multidev, int rc) { multidev->preparation->rc = rc; - multidev_one_callback(egc, multidev->preparation); + libxl__multidev_one_callback(egc, multidev->preparation); } /******************************************************************************/ diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c index bd2184176f..d63ae1bbd3 100644 --- a/tools/libxl/libxl_dom.c +++ b/tools/libxl/libxl_dom.c @@ -920,8 +920,6 @@ static void domain_suspend_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); static void domain_suspend_callback_common_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int ok); -static void remus_domain_suspend_callback_common_done(libxl__egc *egc, - libxl__domain_suspend_state *dss, int ok); /*----- complicated callback, called by xc_domain_save -----*/ @@ -1583,6 +1581,14 @@ static void domain_suspend_callback_common_done(libxl__egc *egc, } /*----- remus callbacks -----*/ +static void remus_domain_suspend_callback_common_done(libxl__egc *egc, + libxl__domain_suspend_state *dss, int ok); +static void remus_devices_postsuspend_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); +static void remus_devices_preresume_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); static void libxl__remus_domain_suspend_callback(void *data) { @@ -1597,32 +1603,77 @@ static void libxl__remus_domain_suspend_callback(void *data) static void remus_domain_suspend_callback_common_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int ok) { - /* REMUS TODO: Issue disk and network checkpoint reqs. */ + if (!ok) + goto out; + + libxl__remus_devices_state *const rds = &dss->rds; + rds->callback = remus_devices_postsuspend_cb; + libxl__remus_devices_postsuspend(egc, rds); + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} + +static void remus_devices_postsuspend_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) +{ + int ok = 0; + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + + if (rc) + goto out; + + ok = 1; + +out: libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); } static void libxl__remus_domain_resume_callback(void *data) { - int ok = 0; libxl__save_helper_state *shs = data; libxl__egc *egc = shs->egc; libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); STATE_AO_GC(dss->ao); - /* Resumes the domain and the device model */ - if (libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1)) + libxl__remus_devices_state *const rds = &dss->rds; + rds->callback = remus_devices_preresume_cb; + libxl__remus_devices_preresume(egc, rds); +} + +static void remus_devices_preresume_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) +{ + int ok = 0; + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) + goto out; + + /* Resumes the domain and the device model */ + rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1); + if (rc) goto out; - /* REMUS TODO: Deal with disk. Start a new network output buffer */ ok = 1; + out: - libxl__xc_domain_saverestore_async_callback_done(egc, shs, ok); + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); } /*----- remus asynchronous checkpoint callback -----*/ static void remus_checkpoint_dm_saved(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); +static void remus_devices_commit_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev, + const struct timeval *requested_abs); static void libxl__remus_domain_checkpoint_callback(void *data) { @@ -1642,10 +1693,73 @@ static void libxl__remus_domain_checkpoint_callback(void *data) static void remus_checkpoint_dm_saved(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { - /* REMUS TODO: Wait for disk and memory ack, release network buffer */ - /* REMUS TODO: make this asynchronous */ - assert(!rc); /* REMUS TODO handle this error properly */ - usleep(dss->interval * 1000); + /* Convenience aliases */ + libxl__remus_devices_state *const rds = &dss->rds; + + STATE_AO_GC(dss->ao); + + if (rc) { + LOG(ERROR, "Failed to save device model. Terminating Remus.."); + goto out; + } + + rds->callback = remus_devices_commit_cb; + libxl__remus_devices_commit(egc, rds); + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void remus_devices_commit_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + + STATE_AO_GC(dss->ao); + + if (rc) { + LOG(ERROR, "Failed to do device commit op." + " Terminating Remus.."); + goto out; + } + + /* + * At this point, we have successfully checkpointed the guest and + * committed it at the backup. We'll come back after the checkpoint + * interval to checkpoint the guest again. Until then, let the guest + * continue execution. + */ + + /* Set checkpoint interval timeout */ + rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout, + remus_next_checkpoint, + dss->interval); + + if (rc) + goto out; + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev, + const struct timeval *requested_abs) +{ + libxl__domain_suspend_state *dss = + CONTAINER_OF(ev, *dss, checkpoint_timeout); + + STATE_AO_GC(dss->ao); + + /* + * Time to checkpoint the guest again. We return 1 to libxc + * (xc_domain_save.c). in order to continue executing the infinite loop + * (suspend, checkpoint, resume) in xc_domain_save(). + */ libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1); } @@ -1695,7 +1809,7 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss) if (r_info != NULL) { dss->interval = r_info->interval; - if (r_info->compression) + if (libxl_defbool_val(r_info->compression)) dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS; } @@ -1860,6 +1974,10 @@ static void save_device_model_datacopier_done(libxl__egc *egc, dss->save_dm_callback(egc, dss, our_rc); } +static void remus_teardown_done(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); + static void domain_suspend_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { @@ -1874,6 +1992,34 @@ static void domain_suspend_done(libxl__egc *egc, xc_suspend_evtchn_release(CTX->xch, CTX->xce, domid, dss->guest_evtchn.port, &dss->guest_evtchn_lockfd); + if (!dss->remus) { + remus_teardown_done(egc, &dss->rds, rc); + return; + } + + /* + * With Remus, if we reach this point, it means either + * backup died or some network error occurred preventing us + * from sending checkpoints. Teardown the network buffers and + * release netlink resources. This is an async op. + */ + LOG(WARN, "Remus: Domain suspend terminated with rc %d," + " teardown Remus devices...", rc); + dss->rds.callback = remus_teardown_done; + libxl__remus_devices_teardown(egc, &dss->rds); +} + +static void remus_teardown_done(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) + LOG(ERROR, "Remus: failed to teardown device for guest with domid %u," + " rc %d", dss->domid, rc); + dss->callback(egc, dss, rc); } diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index f61673c9ea..b87c5e2189 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -2150,6 +2150,8 @@ struct libxl__ao_device { libxl__async_exec_state aes; /* If we need to update JSON config */ bool update_json; + /* for asynchronous execution of synchronous-only syscalls etc. */ + libxl__ev_child child; }; /* @@ -2161,20 +2163,41 @@ struct libxl__ao_device { * Then zero or more times * libxl__multidev_prepare * libxl__initiate_device_{remove/addition} - * (or some other thing which will eventually call aodev->callback) + * (or some other thing which will eventually call + * aodev->callback or libxl__multidev_one_callback) * Finally, once * libxl__multidev_prepared - * which will result (perhaps reentrantly) in one call to callback(). + * which will result (perhaps reentrantly) in one call to + * multidev->callback(). */ /* Starts preparing to add/remove a bunch of devices. */ _hidden void libxl__multidev_begin(libxl__ao *ao, libxl__multidev*); -/* Prepares to add/remove one of many devices. Returns a libxl__ao_device - * which has had libxl__prepare_ao_device called, and which has also - * had ->callback set. The user should not mess with aodev->callback. */ +/* Prepares to add/remove one of many devices. + * Calls libxl__prepare_ao_device on libxl__ao_device argument provided and + * also sets the aodev->callback (to libxl__multidev_one_callback) + * The user should not mess with aodev->callback. + */ +_hidden void libxl__multidev_prepare_with_aodev(libxl__multidev*, + libxl__ao_device*); + +/* A wrapper function around libxl__multidev_prepare_with_aodev. + * Allocates a libxl__ao_device and prepares it for addition/removal. + * Returns the newly allocated libxl__ao_dev. + */ _hidden libxl__ao_device *libxl__multidev_prepare(libxl__multidev*); +/* Indicates to multidev that this one device has been processed. + * Normally the multidev user does not need to touch this function, as + * multidev_prepare will name it in aodev->callback. However, if you + * want to do something more complicated you can set aodev->callback + * yourself to something else, so long as you eventually call + * libxl__multidev_one_callback. + */ +_hidden void libxl__multidev_one_callback(libxl__egc *egc, + libxl__ao_device *aodev); + /* Notifies the multidev machinery that we have now finished preparing * and initiating devices. multidev->callback may then be called as * soon as there are no prepared but not completed operations @@ -2563,6 +2586,182 @@ typedef struct libxl__save_helper_state { * marshalling and xc callback functions */ } libxl__save_helper_state; +/*----- remus device related state structure -----*/ +/* + * The abstract Remus device layer exposes a common + * set of API to [external] libxl for manipulating devices attached to + * a guest protected by Remus. The device layer also exposes a set of + * [internal] interfaces that every device type must implement. + * + * The following API are exposed to libxl: + * + * One-time configuration operations: + * +libxl__remus_devices_setup + * > Enable output buffering for NICs, setup disk replication, etc. + * +libxl__remus_devices_teardown + * > Disable output buffering and disk replication; teardown any + * associated external setups like qdiscs for NICs. + * + * Operations executed every checkpoint (in order of invocation): + * +libxl__remus_devices_postsuspend + * +libxl__remus_devices_preresume + * +libxl__remus_devices_commit + * + * Each device type needs to implement the interfaces specified in + * the libxl__remus_device_instance_ops if it wishes to support Remus. + * + * The high-level control flow through the Remus device layer is shown below: + * + * xl remus + * |-> libxl_domain_remus_start + * |-> libxl__remus_devices_setup + * |-> Per-checkpoint libxl__remus_devices_[postsuspend,preresume,commit] + * ... + * |-> On backup failure, network error or other internal errors: + * libxl__remus_devices_teardown + */ + +typedef struct libxl__remus_device libxl__remus_device; +typedef struct libxl__remus_devices_state libxl__remus_devices_state; +typedef struct libxl__remus_device_instance_ops libxl__remus_device_instance_ops; + +/* + * Interfaces to be implemented by every device subkind that wishes to + * support Remus. Functions must be implemented unless otherwise + * stated. Many of these functions are asynchronous. They call + * dev->aodev.callback when done. The actual implementations may be + * synchronous and call dev->aodev.callback directly (as the last + * thing they do). + */ +struct libxl__remus_device_instance_ops { + /* the device kind this ops belongs to... */ + libxl__device_kind kind; + + /* + * Checkpoint operations. May be NULL, meaning the op is not + * implemented and the caller should treat them as a no-op (and do + * nothing when checkpointing). + * Asynchronous. + */ + + void (*postsuspend)(libxl__egc *egc, libxl__remus_device *dev); + void (*preresume)(libxl__egc *egc, libxl__remus_device *dev); + void (*commit)(libxl__egc *egc, libxl__remus_device *dev); + + /* + * setup() and teardown() are refer to the actual remus device. + * Asynchronous. + * teardown is called even if setup fails. + */ + /* + * setup() should first determines whether the subkind matches the specific + * device. If matched, the device will then be managed with this set of + * subkind operations. + * Yields 0 if the device successfully set up. + * REMUS_DEVOPS_DOES_NOT_MATCH if the ops does not match the device. + * any other rc indicates failure. + */ + void (*setup)(libxl__egc *egc, libxl__remus_device *dev); + void (*teardown)(libxl__egc *egc, libxl__remus_device *dev); +}; + +int init_subkind_nic(libxl__remus_devices_state *rds); +void cleanup_subkind_nic(libxl__remus_devices_state *rds); +int init_subkind_drbd_disk(libxl__remus_devices_state *rds); +void cleanup_subkind_drbd_disk(libxl__remus_devices_state *rds); + +typedef void libxl__remus_callback(libxl__egc *, + libxl__remus_devices_state *, int rc); + +/* + * State associated with a remus invocation, including parameters + * passed to the remus abstract device layer by the remus + * save/restore machinery. + */ +struct libxl__remus_devices_state { + /*---- must be set by caller of libxl__remus_device_(setup|teardown) ----*/ + + libxl__ao *ao; + uint32_t domid; + libxl__remus_callback *callback; + int device_kind_flags; + + /*----- private for abstract layer only -----*/ + + int num_devices; + /* + * this array is allocated before setup the remus devices by the + * remus abstract layer. + * devs may be NULL, means there's no remus devices that has been set up. + * the size of this array is 'num_devices', which is the total number + * of libxl nic devices and disk devices(num_nics + num_disks). + */ + libxl__remus_device **devs; + + libxl_device_nic *nics; + int num_nics; + libxl_device_disk *disks; + int num_disks; + + libxl__multidev multidev; + + /*----- private for concrete (device-specific) layer only -----*/ + + /* private for nic device subkind ops */ + char *netbufscript; + struct nl_sock *nlsock; + struct nl_cache *qdisc_cache; + + /* private for drbd disk subkind ops */ + char *drbd_probe_script; +}; + +/* + * Information about a single device being handled by remus. + * Allocated by the remus abstract layer. + */ +struct libxl__remus_device { + /*----- shared between abstract and concrete layers -----*/ + /* + * if this is true, that means the subkind ops match the device + */ + bool matched; + + /*----- set by remus device abstruct layer -----*/ + /* libxl__device_* which this remus device related to */ + const void *backend_dev; + libxl__device_kind kind; + libxl__remus_devices_state *rds; + libxl__ao_device aodev; + + /*----- private for abstract layer only -----*/ + + /* + * Control and state variables for the asynchronous callback + * based loops which iterate over device subkinds, and over + * individual devices. + */ + int ops_index; + const libxl__remus_device_instance_ops *ops; + + /*----- private for concrete (device-specific) layer -----*/ + + /* concrete device's private data */ + void *concrete_data; +}; + +/* the following 5 APIs are async ops, call rds->callback when done */ +_hidden void libxl__remus_devices_setup(libxl__egc *egc, + libxl__remus_devices_state *rds); +_hidden void libxl__remus_devices_teardown(libxl__egc *egc, + libxl__remus_devices_state *rds); +_hidden void libxl__remus_devices_postsuspend(libxl__egc *egc, + libxl__remus_devices_state *rds); +_hidden void libxl__remus_devices_preresume(libxl__egc *egc, + libxl__remus_devices_state *rds); +_hidden void libxl__remus_devices_commit(libxl__egc *egc, + libxl__remus_devices_state *rds); +_hidden int libxl__netbuffer_enabled(libxl__gc *gc); /*----- Domain suspend (save) state structure -----*/ @@ -2602,6 +2801,8 @@ struct libxl__domain_suspend_state { libxl__ev_xswatch guest_watch; libxl__ev_time guest_timeout; const char *dm_savefile; + libxl__remus_devices_state rds; + libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */ int interval; /* checkpoint interval (for Remus) */ libxl__save_helper_state shs; libxl__logdirty_switch logdirty; diff --git a/tools/libxl/libxl_netbuffer.c b/tools/libxl/libxl_netbuffer.c new file mode 100644 index 0000000000..edc6843253 --- /dev/null +++ b/tools/libxl/libxl_netbuffer.c @@ -0,0 +1,517 @@ +/* + * Copyright (C) 2014 + * Author Shriram Rajagopalan + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" + +#include +#include +#include +#include +#include +#include +#include + +typedef struct libxl__remus_device_nic { + int devid; + + const char *vif; + const char *ifb; + struct rtnl_qdisc *qdisc; +} libxl__remus_device_nic; + +int libxl__netbuffer_enabled(libxl__gc *gc) +{ + return 1; +} + +int init_subkind_nic(libxl__remus_devices_state *rds) +{ + int rc, ret; + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + + STATE_AO_GC(rds->ao); + + rds->nlsock = nl_socket_alloc(); + if (!rds->nlsock) { + LOG(ERROR, "cannot allocate nl socket"); + rc = ERROR_FAIL; + goto out; + } + + ret = nl_connect(rds->nlsock, NETLINK_ROUTE); + if (ret) { + LOG(ERROR, "failed to open netlink socket: %s", + nl_geterror(ret)); + rc = ERROR_FAIL; + goto out; + } + + /* get list of all qdiscs installed on network devs. */ + ret = rtnl_qdisc_alloc_cache(rds->nlsock, &rds->qdisc_cache); + if (ret) { + LOG(ERROR, "failed to allocate qdisc cache: %s", + nl_geterror(ret)); + rc = ERROR_FAIL; + goto out; + } + + if (dss->remus->netbufscript) { + rds->netbufscript = libxl__strdup(gc, dss->remus->netbufscript); + } else { + rds->netbufscript = GCSPRINTF("%s/remus-netbuf-setup", + libxl__xen_script_dir_path()); + } + + rc = 0; + +out: + return rc; +} + +void cleanup_subkind_nic(libxl__remus_devices_state *rds) +{ + STATE_AO_GC(rds->ao); + + /* free qdisc cache */ + if (rds->qdisc_cache) { + nl_cache_clear(rds->qdisc_cache); + nl_cache_free(rds->qdisc_cache); + rds->qdisc_cache = NULL; + } + + /* close & free nlsock */ + if (rds->nlsock) { + nl_close(rds->nlsock); + nl_socket_free(rds->nlsock); + rds->nlsock = NULL; + } +} + +/*----- setup() and teardown() -----*/ + +/* helper functions */ + +/* + * If the device has a vifname, then use that instead of + * the vifX.Y format. + * it must ONLY be used for remus because if driver domains + * were in use it would constitute a security vulnerability. + */ +static const char *get_vifname(libxl__remus_device *dev, + const libxl_device_nic *nic) +{ + const char *vifname = NULL; + const char *path; + int rc; + + STATE_AO_GC(dev->rds->ao); + + /* Convenience aliases */ + const uint32_t domid = dev->rds->domid; + + path = GCSPRINTF("%s/backend/vif/%d/%d/vifname", + libxl__xs_get_dompath(gc, 0), domid, nic->devid); + rc = libxl__xs_read_checked(gc, XBT_NULL, path, &vifname); + if (!rc && !vifname) { + vifname = libxl__device_nic_devname(gc, domid, + nic->devid, + nic->nictype); + } + + return vifname; +} + +static void free_qdisc(libxl__remus_device_nic *remus_nic) +{ + if (remus_nic->qdisc == NULL) + return; + + nl_object_put((struct nl_object *)(remus_nic->qdisc)); + remus_nic->qdisc = NULL; +} + +static int init_qdisc(libxl__remus_devices_state *rds, + libxl__remus_device_nic *remus_nic) +{ + int rc, ret, ifindex; + struct rtnl_link *ifb = NULL; + struct rtnl_qdisc *qdisc = NULL; + + STATE_AO_GC(rds->ao); + + /* Now that we have brought up REMUS_IFB device with plug qdisc for + * this vif, so we need to refill the qdisc cache. + */ + ret = nl_cache_refill(rds->nlsock, rds->qdisc_cache); + if (ret) { + LOG(ERROR, "cannot refill qdisc cache: %s", nl_geterror(ret)); + rc = ERROR_FAIL; + goto out; + } + + /* get a handle to the REMUS_IFB interface */ + ret = rtnl_link_get_kernel(rds->nlsock, 0, remus_nic->ifb, &ifb); + if (ret) { + LOG(ERROR, "cannot obtain handle for %s: %s", remus_nic->ifb, + nl_geterror(ret)); + rc = ERROR_FAIL; + goto out; + } + + ifindex = rtnl_link_get_ifindex(ifb); + if (!ifindex) { + LOG(ERROR, "interface %s has no index", remus_nic->ifb); + rc = ERROR_FAIL; + goto out; + } + + /* Get a reference to the root qdisc installed on the REMUS_IFB, by + * querying the qdisc list we obtained earlier. The netbufscript + * sets up the plug qdisc as the root qdisc, so we don't have to + * search the entire qdisc tree on the REMUS_IFB dev. + + * There is no need to explicitly free this qdisc as its just a + * reference from the qdisc cache we allocated earlier. + */ + qdisc = rtnl_qdisc_get_by_parent(rds->qdisc_cache, ifindex, TC_H_ROOT); + if (qdisc) { + const char *tc_kind = rtnl_tc_get_kind(TC_CAST(qdisc)); + /* Sanity check: Ensure that the root qdisc is a plug qdisc. */ + if (!tc_kind || strcmp(tc_kind, "plug")) { + LOG(ERROR, "plug qdisc is not installed on %s", remus_nic->ifb); + rc = ERROR_FAIL; + goto out; + } + remus_nic->qdisc = qdisc; + } else { + LOG(ERROR, "Cannot get qdisc handle from ifb %s", remus_nic->ifb); + rc = ERROR_FAIL; + goto out; + } + + rc = 0; + +out: + if (ifb) + rtnl_link_put(ifb); + + if (rc && qdisc) + nl_object_put((struct nl_object *)qdisc); + + return rc; +} + +/* callbacks */ + +static void netbuf_setup_script_cb(libxl__egc *egc, + libxl__async_exec_state *aes, + int status); +static void netbuf_teardown_script_cb(libxl__egc *egc, + libxl__async_exec_state *aes, + int status); + +/* + * the script needs the following env & args + * $vifname + * $XENBUS_PATH (/libxl//remus/netbuf//) + * $REMUS_IFB (for teardown) + * setup/teardown as command line arg. + */ +static void setup_async_exec(libxl__remus_device *dev, char *op) +{ + int arraysize, nr = 0; + char **env = NULL, **args = NULL; + libxl__remus_device_nic *remus_nic = dev->concrete_data; + libxl__remus_devices_state *rds = dev->rds; + libxl__async_exec_state *aes = &dev->aodev.aes; + + STATE_AO_GC(rds->ao); + + /* Convenience aliases */ + char *const script = libxl__strdup(gc, rds->netbufscript); + const uint32_t domid = rds->domid; + const int dev_id = remus_nic->devid; + const char *const vif = remus_nic->vif; + const char *const ifb = remus_nic->ifb; + + arraysize = 7; + GCNEW_ARRAY(env, arraysize); + env[nr++] = "vifname"; + env[nr++] = libxl__strdup(gc, vif); + env[nr++] = "XENBUS_PATH"; + env[nr++] = GCSPRINTF("%s/remus/netbuf/%d", + libxl__xs_libxl_path(gc, domid), dev_id); + if (!strcmp(op, "teardown") && ifb) { + env[nr++] = "REMUS_IFB"; + env[nr++] = libxl__strdup(gc, ifb); + } + env[nr++] = NULL; + assert(nr <= arraysize); + + arraysize = 3; nr = 0; + GCNEW_ARRAY(args, arraysize); + args[nr++] = script; + args[nr++] = op; + args[nr++] = NULL; + assert(nr == arraysize); + + aes->ao = dev->rds->ao; + aes->what = GCSPRINTF("%s %s", args[0], args[1]); + aes->env = env; + aes->args = args; + aes->timeout_ms = LIBXL_HOTPLUG_TIMEOUT * 1000; + aes->stdfds[0] = -1; + aes->stdfds[1] = -1; + aes->stdfds[2] = -1; + + if (!strcmp(op, "teardown")) + aes->callback = netbuf_teardown_script_cb; + else + aes->callback = netbuf_setup_script_cb; +} + +/* setup() and teardown() */ + +static void nic_setup(libxl__egc *egc, libxl__remus_device *dev) +{ + int rc; + libxl__remus_device_nic *remus_nic; + const libxl_device_nic *nic = dev->backend_dev; + + STATE_AO_GC(dev->rds->ao); + + /* + * thers's no subkind of nic devices, so nic ops is always matched + * with nic devices + */ + dev->matched = true; + + GCNEW(remus_nic); + dev->concrete_data = remus_nic; + remus_nic->devid = nic->devid; + remus_nic->vif = get_vifname(dev, nic); + if (!remus_nic->vif) { + rc = ERROR_FAIL; + goto out; + } + + setup_async_exec(dev, "setup"); + rc = libxl__async_exec_start(gc, &dev->aodev.aes); + if (rc) + goto out; + + return; + +out: + dev->aodev.rc = rc; + dev->aodev.callback(egc, &dev->aodev); +} + +/* + * In return, the script writes the name of REMUS_IFB device (during setup) + * to be used for output buffering into XENBUS_PATH/ifb + */ +static void netbuf_setup_script_cb(libxl__egc *egc, + libxl__async_exec_state *aes, + int status) +{ + libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes); + libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev); + libxl__remus_device_nic *remus_nic = dev->concrete_data; + libxl__remus_devices_state *rds = dev->rds; + const char *out_path_base, *hotplug_error = NULL; + int rc; + + STATE_AO_GC(rds->ao); + + /* Convenience aliases */ + const uint32_t domid = rds->domid; + const int devid = remus_nic->devid; + const char *const vif = remus_nic->vif; + const char **const ifb = &remus_nic->ifb; + + /* + * we need to get ifb first because it's needed for teardown + */ + rc = libxl__xs_read_checked(gc, XBT_NULL, + GCSPRINTF("%s/remus/netbuf/%d/ifb", + libxl__xs_libxl_path(gc, domid), + devid), + ifb); + if (rc) + goto out; + + if (!(*ifb)) { + LOG(ERROR, "Cannot get ifb dev name for domain %u dev %s", + domid, vif); + rc = ERROR_FAIL; + goto out; + } + + out_path_base = GCSPRINTF("%s/remus/netbuf/%d", + libxl__xs_libxl_path(gc, domid), devid); + + rc = libxl__xs_read_checked(gc, XBT_NULL, + GCSPRINTF("%s/hotplug-error", out_path_base), + &hotplug_error); + if (rc) + goto out; + + if (hotplug_error) { + LOG(ERROR, "netbuf script %s setup failed for vif %s: %s", + rds->netbufscript, vif, hotplug_error); + rc = ERROR_FAIL; + goto out; + } + + if (status) { + rc = ERROR_FAIL; + goto out; + } + + LOG(DEBUG, "%s will buffer packets from vif %s", *ifb, vif); + rc = init_qdisc(rds, remus_nic); + +out: + aodev->rc = rc; + aodev->callback(egc, aodev); +} + +static void nic_teardown(libxl__egc *egc, libxl__remus_device *dev) +{ + int rc; + STATE_AO_GC(dev->rds->ao); + + setup_async_exec(dev, "teardown"); + + rc = libxl__async_exec_start(gc, &dev->aodev.aes); + if (rc) + goto out; + + return; + +out: + dev->aodev.rc = rc; + dev->aodev.callback(egc, &dev->aodev); +} + +static void netbuf_teardown_script_cb(libxl__egc *egc, + libxl__async_exec_state *aes, + int status) +{ + int rc; + libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes); + libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev); + libxl__remus_device_nic *remus_nic = dev->concrete_data; + + if (status) + rc = ERROR_FAIL; + else + rc = 0; + + free_qdisc(remus_nic); + + aodev->rc = rc; + aodev->callback(egc, aodev); +} + +/*----- checkpointing APIs -----*/ + +/* The value of buffer_op, not the value passed to kernel */ +enum { + tc_buffer_start, + tc_buffer_release +}; + +/* API implementations */ + +static int remus_netbuf_op(libxl__remus_device_nic *remus_nic, + libxl__remus_devices_state *rds, + int buffer_op) +{ + int rc, ret; + + STATE_AO_GC(rds->ao); + + if (buffer_op == tc_buffer_start) + ret = rtnl_qdisc_plug_buffer(remus_nic->qdisc); + else + ret = rtnl_qdisc_plug_release_one(remus_nic->qdisc); + + if (ret) { + rc = ERROR_FAIL; + goto out; + } + + ret = rtnl_qdisc_add(rds->nlsock, remus_nic->qdisc, NLM_F_REQUEST); + if (ret) { + rc = ERROR_FAIL; + goto out; + } + + rc = 0; + +out: + if (rc) + LOG(ERROR, "Remus: cannot do netbuf op %s on %s:%s", + ((buffer_op == tc_buffer_start) ? + "start_new_epoch" : "release_prev_epoch"), + remus_nic->ifb, nl_geterror(ret)); + return rc; +} + +static void nic_postsuspend(libxl__egc *egc, libxl__remus_device *dev) +{ + int rc; + libxl__remus_device_nic *remus_nic = dev->concrete_data; + + STATE_AO_GC(dev->rds->ao); + + rc = remus_netbuf_op(remus_nic, dev->rds, tc_buffer_start); + + dev->aodev.rc = rc; + dev->aodev.callback(egc, &dev->aodev); +} + +static void nic_commit(libxl__egc *egc, libxl__remus_device *dev) +{ + int rc; + libxl__remus_device_nic *remus_nic = dev->concrete_data; + + STATE_AO_GC(dev->rds->ao); + + rc = remus_netbuf_op(remus_nic, dev->rds, tc_buffer_release); + + dev->aodev.rc = rc; + dev->aodev.callback(egc, &dev->aodev); +} + +const libxl__remus_device_instance_ops remus_device_nic = { + .kind = LIBXL__DEVICE_KIND_VIF, + .setup = nic_setup, + .teardown = nic_teardown, + .postsuspend = nic_postsuspend, + .commit = nic_commit, +}; + +/* + * Local variables: + * mode: C + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/tools/libxl/libxl_nonetbuffer.c b/tools/libxl/libxl_nonetbuffer.c new file mode 100644 index 0000000000..3c659c29f4 --- /dev/null +++ b/tools/libxl/libxl_nonetbuffer.c @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2014 + * Author Shriram Rajagopalan + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" + +int libxl__netbuffer_enabled(libxl__gc *gc) +{ + return 0; +} + +int init_subkind_nic(libxl__remus_devices_state *rds) +{ + return 0; +} + +void cleanup_subkind_nic(libxl__remus_devices_state *rds) +{ + return; +} + +static void nic_setup(libxl__egc *egc, libxl__remus_device *dev) +{ + STATE_AO_GC(dev->rds->ao); + + dev->aodev.rc = ERROR_FAIL; + dev->aodev.callback(egc, &dev->aodev); +} + +const libxl__remus_device_instance_ops remus_device_nic = { + .kind = LIBXL__DEVICE_KIND_VIF, + .setup = nic_setup, +}; + +/* + * Local variables: + * mode: C + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/tools/libxl/libxl_remus_device.c b/tools/libxl/libxl_remus_device.c new file mode 100644 index 0000000000..a6cb7f61ca --- /dev/null +++ b/tools/libxl/libxl_remus_device.c @@ -0,0 +1,327 @@ +/* + * Copyright (C) 2014 FUJITSU LIMITED + * Author: Yang Hongyang + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" + +extern const libxl__remus_device_instance_ops remus_device_nic; +extern const libxl__remus_device_instance_ops remus_device_drbd_disk; +static const libxl__remus_device_instance_ops *remus_ops[] = { + &remus_device_nic, + &remus_device_drbd_disk, + NULL, +}; + +/*----- helper functions -----*/ + +static int init_device_subkind(libxl__remus_devices_state *rds) +{ + /* init device subkind-specific state in the libxl ctx */ + int rc; + STATE_AO_GC(rds->ao); + + if (libxl__netbuffer_enabled(gc)) { + rc = init_subkind_nic(rds); + if (rc) goto out; + } + + rc = init_subkind_drbd_disk(rds); + if (rc) goto out; + + rc = 0; +out: + return rc; +} + +static void cleanup_device_subkind(libxl__remus_devices_state *rds) +{ + /* cleanup device subkind-specific state in the libxl ctx */ + STATE_AO_GC(rds->ao); + + if (libxl__netbuffer_enabled(gc)) + cleanup_subkind_nic(rds); + + cleanup_subkind_drbd_disk(rds); +} + +/*----- setup() and teardown() -----*/ + +/* callbacks */ + +static void all_devices_setup_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc); +static void device_setup_iterate(libxl__egc *egc, + libxl__ao_device *aodev); +static void devices_teardown_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc); + +/* remus device setup and teardown */ + +static libxl__remus_device* remus_device_init(libxl__egc *egc, + libxl__remus_devices_state *rds, + libxl__device_kind kind, + void *libxl_dev) +{ + libxl__remus_device *dev = NULL; + + STATE_AO_GC(rds->ao); + GCNEW(dev); + dev->backend_dev = libxl_dev; + dev->kind = kind; + dev->rds = rds; + + return dev; +} + +static void remus_devices_setup(libxl__egc *egc, + libxl__remus_devices_state *rds); + +void libxl__remus_devices_setup(libxl__egc *egc, libxl__remus_devices_state *rds) +{ + int i, rc; + + STATE_AO_GC(rds->ao); + + rc = init_device_subkind(rds); + if (rc) + goto out; + + rds->num_devices = 0; + rds->num_nics = 0; + rds->num_disks = 0; + + if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VIF)) + rds->nics = libxl_device_nic_list(CTX, rds->domid, &rds->num_nics); + + if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VBD)) + rds->disks = libxl_device_disk_list(CTX, rds->domid, &rds->num_disks); + + if (rds->num_nics == 0 && rds->num_disks == 0) + goto out; + + GCNEW_ARRAY(rds->devs, rds->num_nics + rds->num_disks); + + for (i = 0; i < rds->num_nics; i++) { + rds->devs[rds->num_devices++] = remus_device_init(egc, rds, + LIBXL__DEVICE_KIND_VIF, + &rds->nics[i]); + } + + for (i = 0; i < rds->num_disks; i++) { + rds->devs[rds->num_devices++] = remus_device_init(egc, rds, + LIBXL__DEVICE_KIND_VBD, + &rds->disks[i]); + } + + remus_devices_setup(egc, rds); + + return; + +out: + rds->callback(egc, rds, rc); +} + +static void remus_devices_setup(libxl__egc *egc, + libxl__remus_devices_state *rds) +{ + int i, rc; + + STATE_AO_GC(rds->ao); + + libxl__multidev_begin(ao, &rds->multidev); + rds->multidev.callback = all_devices_setup_cb; + for (i = 0; i < rds->num_devices; i++) { + libxl__remus_device *dev = rds->devs[i]; + dev->ops_index = -1; + libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev); + + dev->aodev.rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED; + dev->aodev.callback = device_setup_iterate; + device_setup_iterate(egc,&dev->aodev); + } + + rc = 0; + libxl__multidev_prepared(egc, &rds->multidev, rc); +} + + +static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev) +{ + libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev); + EGC_GC; + + if (aodev->rc != ERROR_REMUS_DEVICE_NOT_SUPPORTED && + aodev->rc != ERROR_REMUS_DEVOPS_DOES_NOT_MATCH) + /* might be success or disaster */ + goto out; + + do { + dev->ops = remus_ops[++dev->ops_index]; + if (!dev->ops) { + libxl_device_nic * nic = NULL; + libxl_device_disk * disk = NULL; + uint32_t domid; + int devid; + if (dev->kind == LIBXL__DEVICE_KIND_VIF) { + nic = (libxl_device_nic *)dev->backend_dev; + domid = nic->backend_domid; + devid = nic->devid; + } else if (dev->kind == LIBXL__DEVICE_KIND_VBD) { + disk = (libxl_device_disk *)dev->backend_dev; + domid = disk->backend_domid; + devid = libxl__device_disk_dev_number(disk->vdev, NULL, NULL); + } else { + LOG(ERROR,"device kind not handled by remus: %s", + libxl__device_kind_to_string(dev->kind)); + aodev->rc = ERROR_FAIL; + goto out; + } + LOG(ERROR,"device not handled by remus" + " (device=%s:%"PRId32"/%"PRId32")", + libxl__device_kind_to_string(dev->kind), + domid, devid); + aodev->rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED; + goto out; + } + } while (dev->ops->kind != dev->kind); + + /* found the next ops_index to try */ + assert(dev->aodev.callback == device_setup_iterate); + dev->ops->setup(egc,dev); + return; + + out: + libxl__multidev_one_callback(egc,aodev); +} + +static void all_devices_setup_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc) +{ + STATE_AO_GC(multidev->ao); + + /* Convenience aliases */ + libxl__remus_devices_state *const rds = + CONTAINER_OF(multidev, *rds, multidev); + + rds->callback(egc, rds, rc); +} + +void libxl__remus_devices_teardown(libxl__egc *egc, + libxl__remus_devices_state *rds) +{ + int i; + libxl__remus_device *dev; + + STATE_AO_GC(rds->ao); + + libxl__multidev_begin(ao, &rds->multidev); + rds->multidev.callback = devices_teardown_cb; + for (i = 0; i < rds->num_devices; i++) { + dev = rds->devs[i]; + if (!dev->ops || !dev->matched) + continue; + + libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev); + dev->ops->teardown(egc,dev); + } + + libxl__multidev_prepared(egc, &rds->multidev, 0); +} + +static void devices_teardown_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc) +{ + int i; + + STATE_AO_GC(multidev->ao); + + /* Convenience aliases */ + libxl__remus_devices_state *const rds = + CONTAINER_OF(multidev, *rds, multidev); + + /* clean nic */ + for (i = 0; i < rds->num_nics; i++) + libxl_device_nic_dispose(&rds->nics[i]); + free(rds->nics); + rds->nics = NULL; + rds->num_nics = 0; + + /* clean disk */ + for (i = 0; i < rds->num_disks; i++) + libxl_device_disk_dispose(&rds->disks[i]); + free(rds->disks); + rds->disks = NULL; + rds->num_disks = 0; + + cleanup_device_subkind(rds); + + rds->callback(egc, rds, rc); +} + +/*----- checkpointing APIs -----*/ + +/* callbacks */ + +static void devices_checkpoint_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc); + +/* API implementations */ + +#define define_remus_checkpoint_api(api) \ +void libxl__remus_devices_##api(libxl__egc *egc, \ + libxl__remus_devices_state *rds) \ +{ \ + int i; \ + libxl__remus_device *dev; \ + \ + STATE_AO_GC(rds->ao); \ + \ + libxl__multidev_begin(ao, &rds->multidev); \ + rds->multidev.callback = devices_checkpoint_cb; \ + for (i = 0; i < rds->num_devices; i++) { \ + dev = rds->devs[i]; \ + if (!dev->matched || !dev->ops->api) \ + continue; \ + libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);\ + dev->ops->api(egc,dev); \ + } \ + \ + libxl__multidev_prepared(egc, &rds->multidev, 0); \ +} + +define_remus_checkpoint_api(postsuspend); + +define_remus_checkpoint_api(preresume); + +define_remus_checkpoint_api(commit); + +static void devices_checkpoint_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc) +{ + STATE_AO_GC(multidev->ao); + + /* Convenience aliases */ + libxl__remus_devices_state *const rds = + CONTAINER_OF(multidev, *rds, multidev); + + rds->callback(egc, rds, rc); +} diff --git a/tools/libxl/libxl_remus_disk_drbd.c b/tools/libxl/libxl_remus_disk_drbd.c new file mode 100644 index 0000000000..3215f931dc --- /dev/null +++ b/tools/libxl/libxl_remus_disk_drbd.c @@ -0,0 +1,258 @@ +/* + * Copyright (C) 2014 FUJITSU LIMITED + * Author Lai Jiangshan + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" + +/*** drbd implementation ***/ +const int DRBD_SEND_CHECKPOINT = 20; +const int DRBD_WAIT_CHECKPOINT_ACK = 30; + +typedef struct libxl__remus_drbd_disk { + int ctl_fd; + int ackwait; +} libxl__remus_drbd_disk; + +int init_subkind_drbd_disk(libxl__remus_devices_state *rds) +{ + STATE_AO_GC(rds->ao); + + rds->drbd_probe_script = GCSPRINTF("%s/block-drbd-probe", + libxl__xen_script_dir_path()); + + return 0; +} + +void cleanup_subkind_drbd_disk(libxl__remus_devices_state *rds) +{ + return; +} + +/*----- helper functions, for async calls -----*/ +static void drbd_async_call(libxl__egc *egc, + libxl__remus_device *dev, + void func(libxl__remus_device *), + libxl__ev_child_callback callback) +{ + int pid = -1, rc; + libxl__ao_device *aodev = &dev->aodev; + STATE_AO_GC(dev->rds->ao); + + /* Fork and call */ + pid = libxl__ev_child_fork(gc, &aodev->child, callback); + if (pid == -1) { + LOG(ERROR, "unable to fork"); + rc = ERROR_FAIL; + goto out; + } + + if (!pid) { + /* child */ + func(dev); + /* notreached */ + abort(); + } + + return; + +out: + aodev->rc = rc; + aodev->callback(egc, aodev); +} + +/*----- match(), setup() and teardown() -----*/ + +/* callbacks */ +static void match_async_exec_cb(libxl__egc *egc, + libxl__async_exec_state *aes, + int status); + +/* implementations */ + +static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev); + +static void drbd_setup(libxl__egc *egc, libxl__remus_device *dev) +{ + STATE_AO_GC(dev->rds->ao); + + match_async_exec(egc, dev); +} + +static void match_async_exec(libxl__egc *egc, libxl__remus_device *dev) +{ + int arraysize, nr = 0, rc; + const libxl_device_disk *disk = dev->backend_dev; + libxl__async_exec_state *aes = &dev->aodev.aes; + STATE_AO_GC(dev->rds->ao); + + /* setup env & args */ + arraysize = 1; + GCNEW_ARRAY(aes->env, arraysize); + aes->env[nr++] = NULL; + assert(nr <= arraysize); + + arraysize = 3; + nr = 0; + GCNEW_ARRAY(aes->args, arraysize); + aes->args[nr++] = dev->rds->drbd_probe_script; + aes->args[nr++] = disk->pdev_path; + aes->args[nr++] = NULL; + assert(nr <= arraysize); + + aes->ao = dev->rds->ao; + aes->what = GCSPRINTF("%s %s", aes->args[0], aes->args[1]); + aes->timeout_ms = LIBXL_HOTPLUG_TIMEOUT * 1000; + aes->callback = match_async_exec_cb; + aes->stdfds[0] = -1; + aes->stdfds[1] = -1; + aes->stdfds[2] = -1; + + rc = libxl__async_exec_start(gc, aes); + if (rc) + goto out; + + return; + +out: + dev->aodev.rc = rc; + dev->aodev.callback(egc, &dev->aodev); +} + +static void match_async_exec_cb(libxl__egc *egc, + libxl__async_exec_state *aes, + int status) +{ + int rc; + libxl__ao_device *aodev = CONTAINER_OF(aes, *aodev, aes); + libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev); + libxl__remus_drbd_disk *drbd_disk; + const libxl_device_disk *disk = dev->backend_dev; + + STATE_AO_GC(aodev->ao); + + if (status) { + rc = ERROR_REMUS_DEVOPS_DOES_NOT_MATCH; + goto out; + } + + /* ops matched */ + dev->matched = true; + + GCNEW(drbd_disk); + dev->concrete_data = drbd_disk; + drbd_disk->ackwait = 0; + drbd_disk->ctl_fd = open(disk->pdev_path, O_RDONLY); + if (drbd_disk->ctl_fd < 0) { + rc = ERROR_FAIL; + goto out; + } + + rc = 0; + +out: + aodev->rc = rc; + aodev->callback(egc, aodev); +} + +static void drbd_teardown(libxl__egc *egc, libxl__remus_device *dev) +{ + libxl__remus_drbd_disk *drbd_disk = dev->concrete_data; + STATE_AO_GC(dev->rds->ao); + + close(drbd_disk->ctl_fd); + dev->aodev.rc = 0; + dev->aodev.callback(egc, &dev->aodev); +} + +/*----- checkpointing APIs -----*/ + +/* callbacks */ +static void checkpoint_async_call_done(libxl__egc *egc, + libxl__ev_child *child, + pid_t pid, int status); + +/* API implementations */ + +/* this op will not wait and block, so implement as sync op */ +static void drbd_postsuspend(libxl__egc *egc, libxl__remus_device *dev) +{ + STATE_AO_GC(dev->rds->ao); + + libxl__remus_drbd_disk *rdd = dev->concrete_data; + + if (!rdd->ackwait) { + if (ioctl(rdd->ctl_fd, DRBD_SEND_CHECKPOINT, 0) <= 0) + rdd->ackwait = 1; + } + + dev->aodev.rc = 0; + dev->aodev.callback(egc, &dev->aodev); +} + + +static void drbd_preresume_async(libxl__remus_device *dev); + +static void drbd_preresume(libxl__egc *egc, libxl__remus_device *dev) +{ + STATE_AO_GC(dev->rds->ao); + + drbd_async_call(egc, dev, drbd_preresume_async, checkpoint_async_call_done); +} + +static void drbd_preresume_async(libxl__remus_device *dev) +{ + libxl__remus_drbd_disk *rdd = dev->concrete_data; + int ackwait = rdd->ackwait; + + if (ackwait) { + ioctl(rdd->ctl_fd, DRBD_WAIT_CHECKPOINT_ACK, 0); + ackwait = 0; + } + + _exit(ackwait); +} + +static void checkpoint_async_call_done(libxl__egc *egc, + libxl__ev_child *child, + pid_t pid, int status) +{ + int rc; + libxl__ao_device *aodev = CONTAINER_OF(child, *aodev, child); + libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev); + libxl__remus_drbd_disk *rdd = dev->concrete_data; + + STATE_AO_GC(aodev->ao); + + if (!WIFEXITED(status)) { + rc = ERROR_FAIL; + goto out; + } + + rdd->ackwait = WEXITSTATUS(status); + rc = 0; + +out: + aodev->rc = rc; + aodev->callback(egc, aodev); +} + +const libxl__remus_device_instance_ops remus_device_drbd_disk = { + .kind = LIBXL__DEVICE_KIND_VBD, + .setup = drbd_setup, + .teardown = drbd_teardown, + .postsuspend = drbd_postsuspend, + .preresume = drbd_preresume, +}; diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index 7f9e7c7f55..bbb03e216a 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -61,6 +61,8 @@ libxl_error = Enumeration("error", [ (-15, "LOCK_FAIL"), (-16, "JSON_CONFIG_EMPTY"), (-17, "DEVICE_EXISTS"), + (-18, "REMUS_DEVOPS_DOES_NOT_MATCH"), + (-19, "REMUS_DEVICE_NOT_SUPPORTED"), ], value_namespace = "") libxl_domain_type = Enumeration("domain_type", [ @@ -609,8 +611,12 @@ libxl_sched_credit_params = Struct("sched_credit_params", [ libxl_domain_remus_info = Struct("domain_remus_info",[ ("interval", integer), - ("blackhole", bool), - ("compression", bool), + ("allow_unsafe", libxl_defbool), + ("blackhole", libxl_defbool), + ("compression", libxl_defbool), + ("netbuf", libxl_defbool), + ("netbufscript", string), + ("diskbuf", libxl_defbool), ]) libxl_event_type = Enumeration("event_type", [ diff --git a/tools/libxl/xl.c b/tools/libxl/xl.c index 4c5a5ee472..f0143063ee 100644 --- a/tools/libxl/xl.c +++ b/tools/libxl/xl.c @@ -44,6 +44,7 @@ char *default_vifscript = NULL; char *default_bridge = NULL; char *default_gatewaydev = NULL; char *default_vifbackend = NULL; +char *default_remus_netbufscript = NULL; enum output_format default_output_format = OUTPUT_FORMAT_JSON; int claim_mode = 1; bool progress_use_cr = 0; @@ -176,6 +177,9 @@ static void parse_global_config(const char *configfile, if (!xlu_cfg_get_long (config, "claim_mode", &l, 0)) claim_mode = l; + xlu_cfg_replace_string (config, "remus.default.netbufscript", + &default_remus_netbufscript, 0); + xlu_cfg_destroy(config); } diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h index 6a6a0f918d..6c7aa8ecdf 100644 --- a/tools/libxl/xl.h +++ b/tools/libxl/xl.h @@ -171,6 +171,7 @@ extern char *default_vifscript; extern char *default_bridge; extern char *default_gatewaydev; extern char *default_vifbackend; +extern char *default_remus_netbufscript; extern char *blkdev_start; enum output_format { diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c index d205f96a22..7912e06411 100644 --- a/tools/libxl/xl_cmdimpl.c +++ b/tools/libxl/xl_cmdimpl.c @@ -7495,18 +7495,29 @@ int main_remus(int argc, char **argv) memset(&r_info, 0, sizeof(libxl_domain_remus_info)); /* Defaults */ r_info.interval = 200; - r_info.blackhole = 0; - r_info.compression = 1; + libxl_defbool_setdefault(&r_info.blackhole, false); - SWITCH_FOREACH_OPT(opt, "bui:s:e", NULL, "remus", 2) { + SWITCH_FOREACH_OPT(opt, "Fbundi:s:N:e", NULL, "remus", 2) { case 'i': r_info.interval = atoi(optarg); break; + case 'F': + libxl_defbool_set(&r_info.allow_unsafe, true); + break; case 'b': - r_info.blackhole = 1; + libxl_defbool_set(&r_info.blackhole, true); break; case 'u': - r_info.compression = 0; + libxl_defbool_set(&r_info.compression, false); + break; + case 'n': + libxl_defbool_set(&r_info.netbuf, false); + break; + case 'N': + r_info.netbufscript = optarg; + break; + case 'd': + libxl_defbool_set(&r_info.diskbuf, false); break; case 's': ssh_command = optarg; @@ -7519,7 +7530,10 @@ int main_remus(int argc, char **argv) domid = find_domain(argv[optind]); host = argv[optind + 1]; - if (r_info.blackhole) { + if (!r_info.netbufscript) + r_info.netbufscript = default_remus_netbufscript; + + if (libxl_defbool_val(r_info.blackhole)) { send_fd = open("/dev/null", O_RDWR, 0644); if (send_fd < 0) { perror("failed to open /dev/null"); @@ -7556,13 +7570,19 @@ int main_remus(int argc, char **argv) /* Point of no return */ rc = libxl_domain_remus_start(ctx, &r_info, domid, send_fd, recv_fd, 0); - /* If we are here, it means backup has failed/domain suspend failed. - * Try to resume the domain and exit gracefully. + /* check if the domain exists. User may have xl destroyed the + * domain to force failover + */ + if (libxl_domain_info(ctx, 0, domid)) { + fprintf(stderr, "Remus: Primary domain has been destroyed.\n"); + close(send_fd); + return 0; + } + + /* If we are here, it means remus setup/domain suspend/backup has + * failed. Try to resume the domain and exit gracefully. * TODO: Split-Brain check. */ - fprintf(stderr, "remus sender: libxl_domain_suspend failed" - " (rc=%d)\n", rc); - if (rc == ERROR_GUEST_TIMEDOUT) fprintf(stderr, "Failed to suspend domain at primary.\n"); else { diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c index dd15947507..f93ee4fae0 100644 --- a/tools/libxl/xl_cmdtable.c +++ b/tools/libxl/xl_cmdtable.c @@ -495,13 +495,20 @@ struct cmd_spec cmd_table[] = { "Enable Remus HA for domain", "[options] []", "-i MS Checkpoint domain memory every MS milliseconds (def. 200ms).\n" - "-b Replicate memory checkpoints to /dev/null (blackhole)\n" "-u Disable memory checkpoint compression.\n" "-s Use instead of ssh. String will be passed\n" " to sh. If empty, run instead of \n" " ssh xl migrate-receive -r [-e]\n" "-e Do not wait in the background (on ) for the death\n" - " of the domain." + " of the domain.\n" + "-N Use netbufscript to setup network buffering instead of the\n" + " default script (/etc/xen/scripts/remus-netbuf-setup).\n" + "-F Enable unsafe configurations [-b|-n|-d flags]. Use this option\n" + " with caution as failover may not work as intended.\n" + "-b Replicate memory checkpoints to /dev/null (blackhole).\n" + " Works only in unsafe mode.\n" + "-n Disable network output buffering. Works only in unsafe mode.\n" + "-d Disable disk replication. Works only in unsafe mode." }, #endif { "devd",