blktap2: a completely rewritten blktap implementation

Benefits to blktap2 over the old version of blktap:

* Isolation from xenstore - Blktap devices are now created directly on
   the linux dom0 command line, rather than being spawned in response
   to XenStore events.  This is handy for debugging, makes blktap
   generally easier to work with, and is a step toward a generic
   user-level block device implementation that is not Xen-specific.

* Improved tapdisk infrastructure: simpler request forwarding, new
   request scheduler, request merging, more efficient use of AIO.

* Improved tapdisk error handling and memory management.  No
   allocations on the block data path, IO retry logic to protect
   guests
   transient block device failures.  This has been tested and is known
   to work on weird environments such as NFS soft mounts.

* Pause and snapshot of live virtual disks (see xmsnap script).

* VHD support.  The VHD code in this release has been rigorously
   tested, and represents a very mature implementation of the VHD
   image
   format.

* No more duplication of mechanism with blkback.  The blktap kernel
   module has changed dramatically from the original blktap.  Blkback
   is now always used to talk to Xen guests, blktap just presents a
   Linux gendisk that blkback can export.  This is done while
   preserving the zero-copy data path from domU to physical device.

These patches deprecate the old blktap code, which can hopefully be
removed from the tree completely at some point in the future.

Signed-off-by: Jake Wires <jake.wires@citrix.com>
Signed-off-by: Dutch Meyer <dmeyer@cs.ubc.ca>

tools/python/xen/xend/XendDomainInfo.py

@@ -27,6 +27,7 @@ Author: Mike Wray <mike.wray@hp.com>
 import logging
 import time
 import threading
+import thread
 import re
 import copy
 import os
@@ -534,6 +535,25 @@ class XendDomainInfo:
         
         @raise XendError: Failed pausing a domain
         """
+        try:
+            bepath="/local/domain/0/backend/"
+            if(self.domid):
+                
+                dev =  xstransact.List(bepath + 'vbd' + "/%d" % (self.domid,))
+                for x in dev:
+                    path = self.getDeviceController('vbd').readBackend(x, 'params')
+                    if path and path.startswith('/dev/xen/blktap-2'):
+                        #Figure out the sysfs path.
+                        pattern = re.compile('/dev/xen/blktap-2/tapdev(\d+)$')
+                        ctrlid = pattern.search(path)
+                        ctrl = '/sys/class/blktap2/blktap' + ctrlid.group(1)            
+                        #pause the disk
+                        f = open(ctrl + '/pause', 'w')
+                        f.write('pause');
+                        f.close()
+        except Exception, ex:
+            log.warn('Could not pause blktap disk.');
+
         try:
             xc.domain_pause(self.domid)
             self._stateSet(DOM_STATE_PAUSED)
@@ -546,6 +566,26 @@ class XendDomainInfo:
         
         @raise XendError: Failed unpausing a domain
         """
+        try:
+            bepath="/local/domain/0/backend/"
+            if(self.domid):
+                dev =  xstransact.List(bepath + "vbd" + "/%d" % (self.domid,))
+                for x in dev:
+                    path = self.getDeviceController('vbd').readBackend(x, 'params')
+                    if path and path.startswith('/dev/xen/blktap-2'):
+                        #Figure out the sysfs path.
+                        pattern = re.compile('/dev/xen/blktap-2/tapdev(\d+)$')
+                        ctrlid = pattern.search(path)
+                        ctrl = '/sys/class/blktap2/blktap' + ctrlid.group(1)
+                        #unpause the disk
+                        if(os.path.exists(ctrl + '/resume')):                  
+                            f = open(ctrl + '/resume', 'w');
+                            f.write('resume');
+                            f.close();
+
+        except Exception, ex:
+            log.warn('Could not unpause blktap disk: %s' % str(ex));
+
         try:
             xc.domain_unpause(self.domid)
             self._stateSet(DOM_STATE_RUNNING)
@@ -1171,6 +1211,15 @@ class XendDomainInfo:
 
         rc = None
         if self.domid is not None:
+            
+            #new blktap implementation may need a sysfs write after everything is torn down.
+            dev = self.getDeviceController(deviceClass).convertToDeviceNumber(devid)
+            path = self.getDeviceController(deviceClass).readBackend(dev, 'params')                
+            if path and path.startswith('/dev/xen/blktap-2'):
+                frontpath = self.getDeviceController(deviceClass).frontendPath(dev)
+                backpath = xstransact.Read(frontpath, "backend")
+                thread.start_new_thread(self.getDeviceController(deviceClass).finishDeviceCleanup, (backpath, path))
+
             rc = self.getDeviceController(deviceClass).destroyDevice(devid, force)
             if not force and rm_cfg:
                 # The backend path, other than the device itself,

tools/python/xen/xend/server/BlktapController.py

@@ -1,5 +1,6 @@
 # Copyright (c) 2005, XenSource Ltd.
-
+import string, re
+import subprocess
 
 from xen.xend.server.blkif import BlkifController
 from xen.xend.XendLogging import log
@@ -7,6 +8,11 @@ from xen.xend.XendLogging import log
 phantomDev = 0;
 phantomId = 0;
 
+TAPDISK_SYSFS   = '/sys/class/blktap2'
+TAPDISK_BINARY  = '/usr/sbin/tapdisk2'
+TAPDISK_DEVICE  = '/dev/xen/blktap-2/tapdev'
+TAPDISK_CONTROL = TAPDISK_SYSFS + '/blktap'
+
 blktap_disk_types = [
     'aio',
     'sync',
@@ -14,10 +20,33 @@ blktap_disk_types = [
     'ram',
     'qcow',
     'qcow2',
-
+    'vhd',
     'ioemu',
     'tapdisk',
     ]
+ 
+def doexec(args, inputtext=None):
+    """Execute a subprocess, then return its return code, stdout and stderr"""
+    proc = subprocess.Popen(args,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,close_fds=True)
+    (stdout,stderr) = proc.communicate(inputtext)
+    rc = proc.returncode
+    return (rc,stdout,stderr)
+
+def parseDeviceString(device):
+    if device.find('/dev') == -1:
+        raise Exception, 'invalid tap device: ' + device
+
+    pattern = re.compile(TAPDISK_DEVICE + '(\d+)$')
+    groups  = pattern.search(device)
+    if not groups:
+        raise Exception, 'malformed tap device: ' + device
+
+    minor   = groups.group(1)
+    control = TAPDISK_CONTROL + minor
+
+    return minor, device, control
+
+
 
 class BlktapController(BlkifController):
     def __init__(self, vm):
@@ -86,3 +115,24 @@ class BlktapController(BlkifController):
 
         return (devid, back, front)
 
+    def createDevice(self, config):
+
+        uname = config.get('uname', '')        
+        (typ, subtyp, params, file) = string.split(uname, ':', 3)
+        if typ in ('tap'):
+            if subtyp in ('tapdisk'):                                          
+                if params in ('ioemu', 'qcow2', 'vmdk', 'sync'):
+                    log.warn('WARNING: using deprecated blktap module');
+                    return BlkifController.createDevice(self, config);
+
+        cmd = [ TAPDISK_BINARY, '-n', '%s:%s' % (params, file) ]
+        (rc,stdout,stderr) = doexec(cmd)
+
+        minor, device, control = parseDeviceString(stdout)
+
+        #modify the configuration to attach as a vbd, now that the
+        #device is configured.  Then continue to create the device
+        config.update({'uname' : 'phy:' + device.rstrip()})
+        self.deviceClass='vbd'
+
+        return BlkifController.createDevice(self, config);

tools/python/xen/xend/server/DevController.py

@@ -27,8 +27,8 @@ from xen.xend.server.DevConstants import *
 
 from xen.xend.xenstore.xstransact import xstransact, complete
 from xen.xend.xenstore.xswatch import xswatch
-
-import os
+import xen.xend.server.DevConstants
+import os, re
 
 xoptions = XendOptions.instance()
 
@@ -238,6 +238,34 @@ class DevController:
             # xstransact.Remove(self.devicePath()) ?? Below is the same ?
             self.vm._removeVm("device/%s/%d" % (self.deviceClass, dev))
 
+    # The new blocktap implementation requires a sysfs signal to close
+    # out disks.  This function is called from a thread when the
+    # domain is detached from the disk.
+    def finishDeviceCleanup(self, backpath, path):
+        """Perform any device specific cleanup
+
+        @backpath backend xenstore path.
+        @path frontend device path
+
+        """
+        
+        if path and path.startswith('/dev/xen/blktap-2'):
+            
+            #Figure out what we're going to wait on.
+            self.waitForBackend_destroy(backpath)            
+
+            #Figure out the sysfs path.
+            pattern = re.compile('/dev/xen/blktap-2/tapdev(\d+)$')
+            ctrlid = pattern.search(path)
+            ctrl = '/sys/class/blktap2/blktap' + ctrlid.group(1)
+            
+            #Close out the disk
+            f = open(ctrl + '/remove', 'w')
+            f.write('remove');
+            f.close()
+
+        return
+
     def configurations(self, transaction = None):
         return map(lambda x: self.configuration(x, transaction), self.deviceIDs(transaction))