xend: Add support for URI ('file:' and 'data:' scheme) for PV/kernel

and PV/ramdisk

Add support for 'file:' and 'data:' URI schemes for the parameters
'PV/kernel' and 'PV/ramdisk' in the VM.create() call. The 'data:'
scheme handling enables using a file which is stored inside the
management system (from where the XenAPI call is send) as kernel or
ramdisk.

Notes:
o all included: a detailed description can be found in the xenapi
documentation
o bumped up the version of the API document to 1.0.8 (because of
(minimal) interface extension)
o Future enhancements (like http:, ftp: schemes) fit seamlessly into
the current design / classes
o Unittest cases and xm-test case included

Signed-off-by: Andreas Florath <xen@flonatel.org>

tools/python/xen/util/fileuri.py

@@ -0,0 +1,156 @@
+#============================================================================
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#============================================================================
+# Copyright (C) 2009 flonatel GmbH & Co. KG
+#============================================================================
+
+import logging
+import os
+import base64
+import tempfile
+import stat
+from xen.xend.XendLogging import log
+from xen.util import mkdir
+
+#
+# This functions and classes can be used where a filename is expected -
+# especially in the xenapi.VM.create() for PV_kernel and PV_ramdisk.
+#
+# The functions have a backward compatibility mode, i.e. when there is
+# no appropriate scheme detected, the data is seens as a path to a
+# (local) file.
+#
+
+class scheme_error(Exception):
+    def __init__(self, value):
+        self.value = value
+    def __str__(self):
+        return repr(self.value)
+
+# Data scheme (as defined in RFC 2397):
+#  data:application/octet-stream;base64,<base64 encoded data>
+# It looks that there is currently no general purpose implementation
+# available (in python) for this URL scheme - so the very basic is
+# done here.
+#
+# Limitations
+# o Only base64 is currently supported
+class scheme_data:
+
+    def encode(data, mediatype = 'application/octet-stream', 
+               encoding = 'base64'):
+        # XXX Limit this to base64 for current implementation
+        if encoding!='base64':
+            raise scheme_error("invalid encoding")
+        return 'data:' + mediatype + ";" + encoding \
+            + "," + base64.b64encode(data)
+    encode = staticmethod(encode)
+
+    # Private method: parse encoded data
+    def parse(encoded_data):
+        if not isinstance(encoded_data, str):
+            raise scheme_error("encoded data has wrong type")
+        if not encoded_data.startswith('data:'):
+            raise scheme_error("'data:' scheme declaration missing")
+        comma = encoded_data.find(',', 5)
+        if comma == -1:
+            raise scheme_error("data separator (comma) is missing")
+        # Cut off the media type and encoding
+        mtenc = encoded_data[5:comma]
+        if len(mtenc)==0:
+            raise scheme_error("encoding is empty")
+        # XXX Limit to base64 encoding
+        if not mtenc.endswith(';base64'):
+            raise scheme_error("encoding is not base64")
+        mediatype = mtenc[:-7]
+        return (mediatype, 'base64', comma+1)
+    parse = staticmethod(parse)
+
+    # Stores the data in a local file and returns the filename
+    # and a flag if this file in temporary only and must be deleted
+    # after starting the VM.
+    def decode(encoded_data):
+        mkdir.parents("/var/run/xend/boot/", stat.S_IRWXU)
+        mediatype, encoding, data_start = scheme_data.parse(encoded_data)
+        fd, filename = tempfile.mkstemp(
+            prefix="data_uri_file.", dir="/var/run/xend/boot")
+        # Because of python 2.3 support, there is a need to nest these
+        # (see http://www.python.org/doc/2.3/ref/try.html)
+        try:
+            try:
+                os.write(fd, base64.b64decode(encoded_data[data_start:]))
+            except TypeError, se:
+                raise scheme_error("failed to decode as base64")
+        finally:
+            os.close(fd)
+        return filename, True
+    decode = staticmethod(decode)
+
+    # Utility function which reads in the given (local) file and
+    # creates a data scheme from this.
+    def create_from_file(filename):
+        try:
+            f = open(filename, "r")
+            d = f.read()
+            f.close()
+            return scheme_data.encode(d)
+        except IOError:
+            raise scheme_error("file does not exists")
+    create_from_file = staticmethod(create_from_file)
+
+
+# File Scheme
+# This class supports absolut paths only.
+class scheme_file:
+
+    def encode(filename):
+        if len(filename) == 0:
+            raise scheme_error("filename is empty")
+        if filename[0] != '/':
+            raise scheme_error("filename is not absolut")
+        return 'file://' + filename
+    encode = staticmethod(encode)
+
+    def decode(encoded_data):
+        if not encoded_data.startswith("file://"):
+            raise scheme_error("no file:// scheme found")
+        path = encoded_data[7:]
+        if len(path)==0:
+            raise scheme_error("path is empty")
+        if path[0]!='/':
+            raise scheme_error("path is not absolute")
+        return path, False
+    decode = staticmethod(decode)
+
+
+class scheme_set:
+
+    def __init__(self):
+        self.schemes = [scheme_data, scheme_file]
+
+    # log_decode_exception flags whether a specific uri schema
+    # decoding exception should be logged or not (default: False - do
+    # not log).
+    def decode(self, uri, log_decode_exception=False):
+        for scheme in self.schemes:
+            try:
+                # If this passes, it is the correct scheme
+                return scheme.decode(uri)
+            except scheme_error, se:
+                if log_decode_exception:
+                    log.debug("Decode throws an error: '%s'" % se)
+        return uri, False
+
+schemes = scheme_set()
+        

tools/python/xen/xend/XendConfig.py

@@ -41,6 +41,7 @@ from xen.util.pci import pci_opts_list_from_sxp, pci_convert_sxp_to_dict
 from xen.xend.XendSXPDev import dev_dict_to_sxp
 from xen.util import xsconstants
 from xen.util import auxbin
+import xen.util.fileuri
 
 log = logging.getLogger("xend.XendConfig")
 log.setLevel(logging.WARN)
@@ -338,6 +339,8 @@ class XendConfig(dict):
             # output from xc.domain_getinfo
             self._dominfo_to_xapi(dominfo, update_mem = True)
 
+        self.handle_fileuris()
+
         log.debug('XendConfig.init: %s' % scrub_password(self))
 
         # validators go here
@@ -1999,10 +2002,14 @@ class XendConfig(dict):
             self['_temp_kernel'] = sxp.child_value(image_sxp, 'kernel','')
             self['_temp_ramdisk'] = sxp.child_value(image_sxp, 'ramdisk','')
             self['_temp_args'] = kernel_args
+            self['use_tmp_kernel'] = True
+            self['use_tmp_ramdisk'] = True
         else:
             self['PV_kernel'] = sxp.child_value(image_sxp, 'kernel','')
             self['PV_ramdisk'] = sxp.child_value(image_sxp, 'ramdisk','')
             self['PV_args'] = kernel_args
+            self['use_tmp_kernel'] = False
+            self['use_tmp_ramdisk'] = False
 
         self['superpages'] = sxp.child_value(image_sxp, 'superpages',0)
 
@@ -2072,4 +2079,12 @@ class XendConfig(dict):
                 opts = pci_opts_list_from_sxp(dev)
                 pci.append([domain, bus, slot, func, vdevfn, opts])
         self['platform']['pci'] = pci
+ 
+    def handle_fileuris(self):
+        for arg in [('PV_kernel', 'use_tmp_kernel'), 
+                    ('PV_ramdisk', 'use_tmp_ramdisk')]:
+            if arg[0] in self and self[arg[0]]!='':
+                self[arg[0]], self[arg[1]] \
+                    = xen.util.fileuri.schemes.decode(self[arg[0]])
+                log.debug("fileuri '%s' = '%s'" % (arg[0], self[arg[0]][:100]))
 

tools/python/xen/xend/XendDomainInfo.py

@@ -2662,7 +2662,7 @@ class XendDomainInfo:
 
             self._createDevices()
 
-            self.image.cleanupBootloading()
+            self.image.cleanupTmpImages()
 
             self.info['start_time'] = time.time()
 
@@ -2670,12 +2670,12 @@ class XendDomainInfo:
         except VmError, exn:
             log.exception("XendDomainInfo.initDomain: exception occurred")
             if self.image:
-                self.image.cleanupBootloading()
+                self.image.cleanupTmpImages()
             raise exn
         except RuntimeError, exn:
             log.exception("XendDomainInfo.initDomain: exception occurred")
             if self.image:
-                self.image.cleanupBootloading()
+                self.image.cleanupTmpImages()
             raise VmError(str(exn))
 
 

tools/python/xen/xend/image.py

@@ -89,6 +89,8 @@ class ImageHandler:
         self.vm = vm
 
         self.bootloader = False
+        self.use_tmp_kernel = False
+        self.use_tmp_ramdisk = False
         self.kernel = None
         self.ramdisk = None
         self.cmdline = None
@@ -106,6 +108,12 @@ class ImageHandler:
             self.kernel = vmConfig['PV_kernel']
             self.cmdline = vmConfig['PV_args']
             self.ramdisk = vmConfig['PV_ramdisk']
+        # There a code-paths where use_tmp_xxx is not set at all; but if
+        # this is set, the variable itself is a boolean.
+        if 'use_tmp_kernel' in vmConfig and vmConfig['use_tmp_kernel']:
+            self.use_tmp_kernel = True
+        if 'use_tmp_ramdisk' in vmConfig and vmConfig['use_tmp_ramdisk']:
+            self.use_tmp_ramdisk = True
         self.vm.storeVm(("image/ostype", self.ostype),
                         ("image/kernel", self.kernel),
                         ("image/cmdline", self.cmdline),
@@ -135,12 +143,12 @@ class ImageHandler:
         if 'cpuid_check' in vmConfig:
             self.cpuid_check = vmConfig['cpuid_check']
 
-    def cleanupBootloading(self):
-        if self.bootloader:
+    def cleanupTmpImages(self):
+        if self.use_tmp_kernel:
             self.unlink(self.kernel)
+        if self.use_tmp_ramdisk:
             self.unlink(self.ramdisk)
 
-
     def unlink(self, f):
         if not f: return
         try:
@@ -659,8 +667,6 @@ class ImageHandler:
                 transformed[sinput] = t
             self.cpuid_check = transformed
 
-
-
 class LinuxImageHandler(ImageHandler):
 
     ostype = "linux"
@@ -1028,3 +1034,4 @@ def findImageHandlerClass(image):
         return _handlers[arch.type][image_type]
     except KeyError:
         raise VmError('unknown image type: ' + image_type)
+