xend: Add support for URI ('file:' and 'data:' scheme) for PV/kernel and PV/ramdisk

Add support for 'file:' and 'data:' URI schemes for the parameters
'PV/kernel' and 'PV/ramdisk' in the VM.create() call.  The 'data:'
scheme handling enables using a file which is stored inside the
management system (from where the XenAPI call is send) as kernel or
ramdisk.

Notes:
o all included: a detailed description can be found in the xenapi
documentation
o bumped up the version of the API document to 1.0.8 (because of
(minimal) interface extension)
o Future enhancements (like http:, ftp: schemes) fit seamlessly into
the current design / classes

Signed-off-by: Andreas Florath <xen@flonatel.org>

tools/python/xen/util/fileuri.py

@@ -0,0 +1,273 @@
+#============================================================================
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#============================================================================
+# Copyright (C) 2009 flonatel GmbH & Co. KG
+#============================================================================
+
+import logging
+import os
+import base64
+import tempfile
+import stat
+from xen.xend.XendLogging import log
+from xen.util import mkdir
+# The following are needed for unit-testing only
+import unittest
+
+#
+# This functions and classes can be used where a filename is expected -
+# especially in the xenapi.VM.create() for PV_kernel and PV_ramdisk.
+#
+# The functions have a backward compatibility mode, i.e. when there is
+# no appropriate scheme detected, the data is seens as a path to a
+# (local) file.
+#
+
+class scheme_error(Exception):
+    def __init__(self, value):
+        self.value = value
+    def __str__(self):
+        return repr(self.value)
+
+# Data scheme (as defined in RFC 2397):
+#  data:application/octet-stream;base64,<base64 encoded data>
+# It looks that there is currently no general purpose implementation
+# available (in python) for this URL scheme - so the very basic is
+# done here.
+#
+# Limitations
+# o Only base64 is currently supported
+class scheme_data:
+
+    @staticmethod
+    def encode(data, mediatype = 'application/octet-stream', 
+               encoding = 'base64'):
+        # XXX Limit this to base64 for current implementation
+        if encoding!='base64':
+            raise scheme_error("invalid encoding")
+        return 'data:' + mediatype + ";" + encoding \
+            + "," + base64.b64encode(data)
+
+    # Private method: parse encoded data
+    @staticmethod
+    def parse(encoded_data):
+        if not isinstance(encoded_data, str):
+            raise scheme_error("encoded data has wrong type")
+        if not encoded_data.startswith('data:'):
+            raise scheme_error("'data:' scheme declaration missing")
+        comma = encoded_data.find(',', 5)
+        if comma == -1:
+            raise scheme_error("data separator (comma) is missing")
+        # Cut off the media type and encoding
+        mtenc = encoded_data[5:comma]
+        if len(mtenc)==0:
+            raise scheme_error("encoding is empty")
+        # XXX Limit to base64 encoding
+        if not mtenc.endswith(';base64'):
+            raise scheme_error("encoding is not base64")
+        mediatype = mtenc[:-7]
+        return (mediatype, 'base64', comma+1)
+
+    # Stores the data in a local file and returns the filename
+    # and a flag if this file in temporary only and must be deleted
+    # after starting the VM.
+    @staticmethod
+    def decode(encoded_data):
+        mkdir.parents("/var/run/xend/boot/", stat.S_IRWXU)
+        mediatype, encoding, data_start = scheme_data.parse(encoded_data)
+        fd, filename = tempfile.mkstemp(
+            prefix="data_uri_file.", dir="/var/run/xend/boot")
+        os.write(fd, base64.b64decode(encoded_data[data_start:]))
+        os.close(fd)
+        return filename, True
+
+    # Utility function which reads in the given (local) file and
+    # creates a data scheme from this.
+    @staticmethod
+    def create_from_file(filename):
+        try:
+            f = open(filename, "r")
+            d = f.read()
+            f.close()
+            return scheme_data.encode(d)
+        except IOError:
+            raise scheme_error("file does not exists")
+
+class scheme_data_unit_tests(unittest.TestCase):
+
+    def check_basic_encoding(self):
+        "scheme_data - basic encoding"
+        sd = scheme_data.encode('Hello!')
+        self.assertEqual(sd, 'data:application/octet-stream;base64,SGVsbG8h')
+
+    def check_encoding_with_given_mediatype(self):
+        "scheme_data - encoding with given media name"
+        sd = scheme_data.encode('Hello!', 'application/x-my-linux-kernel')
+        self.assertEqual(sd,
+              'data:application/x-my-linux-kernel;base64,SGVsbG8h')
+
+    def check_parse_01(self):
+        "scheme_data - parsing of None"
+        self.assertRaises(scheme_error, scheme_data.parse, None)
+
+    def check_parse_02(self):
+        "scheme_data - parsing of empty string"
+        self.assertRaises(scheme_error, scheme_data.parse, "")
+
+    def check_parse_03(self):
+        "scheme_data - parsing of unstructured data"
+        self.assertRaises(scheme_error, scheme_data.parse, "akskdjdfhezezu")
+
+    def check_parse_04(self):
+        "scheme_data - data: is not at the first place"
+        self.assertRaises(scheme_error, scheme_data.parse, 'ggdata:sossm')
+
+    def check_parse_05(self):
+        "scheme_data - no comma in data"
+        self.assertRaises(scheme_error, scheme_data.parse, 'data:sossm')
+
+    def check_parse_06(self):
+        "scheme_data - encoding is empty"
+        self.assertRaises(scheme_error, scheme_data.parse, 'data:,')
+
+    def check_parse_07(self):
+        "scheme_data - unknown encoding"
+        self.assertRaises(scheme_error, scheme_data.parse,
+                          'data:somemediatype;unknown,')
+
+    def check_parse_08(self):
+        "scheme_data - parse ok - empty data"
+        mediatype, encoding, data_start = scheme_data.parse(
+            'data:somemedia;base64,')
+        self.assertEqual(mediatype, 'somemedia')
+        self.assertEqual(encoding, 'base64')
+        self.assertEqual(data_start, 22)
+
+    def check_parse_09(self):
+        "scheme_data - parse ok - some data"
+        mediatype, encoding, data_start = scheme_data.parse(
+            'data:somemedia;base64,HereComesTheSun')
+        self.assertEqual(mediatype, 'somemedia')
+        self.assertEqual(encoding, 'base64')
+        self.assertEqual(data_start, 22)
+
+    def check_cff_file_does_not_exist(self):
+        "scheme_data - create from file - non existent file"
+        self.assertRaises(scheme_error, scheme_data.create_from_file,
+                          "/there/is/hopefully/no/file/like/this")
+
+    def check_cff_ok(self):
+        "scheme_data - create from file - ok"
+        tmppath = "/tmp/scheme_data_check_cff_ok"
+        f = open(tmppath, "w")
+        f.write("huhuhu")
+        f.close()
+        d = scheme_data.create_from_file(tmppath)
+        os.unlink(tmppath)
+        self.assertEqual(d, "data:application/octet-stream;base64,aHVodWh1")
+
+# File Scheme
+# This class supports absolut paths only.
+class scheme_file:
+
+    @staticmethod
+    def encode(filename):
+        if len(filename) == 0:
+            raise scheme_error("filename is empty")
+        if filename[0] != '/':
+            raise scheme_error("filename is not absolut")
+        return 'file://' + filename
+
+    @staticmethod
+    def decode(encoded_data):
+        if not encoded_data.startswith("file://"):
+            raise scheme_error("no file:// scheme found")
+        path = encoded_data[7:]
+        if len(path)==0:
+            raise scheme_error("path is empty")
+        if path[0]!='/':
+            raise scheme_error("path is not absolute")
+        return path, False
+
+class scheme_file_unit_tests(unittest.TestCase):
+
+    def check_encode_empty_filename(self):
+        "scheme_file - encode empty filename"
+        self.assertRaises(scheme_error, scheme_file.encode, "")
+
+    def check_encode_relative_filename(self):
+        "scheme_file - encode relative filename"
+        self.assertRaises(scheme_error, scheme_file.encode, "../there")
+
+    def check_encode_absolut_filename(self):
+        "scheme_file - encode absolut filename"
+        self.assertEqual(
+            scheme_file.encode("/here/and/there/again"),
+            'file:///here/and/there/again')
+
+    def check_decode_01(self):
+        "scheme_file - decode empty data"
+        self.assertRaises(scheme_error, scheme_file.decode, "")
+
+    def check_decode_02(self):
+        "scheme_file - decode data with no file:// at the beginning (1)"
+        self.assertRaises(scheme_error, scheme_file.decode,
+                          "phonehome://bbbb")
+
+    def check_decode_03(self):
+        "scheme_file - decode data with no file:// at the beginning (2)"
+        self.assertRaises(scheme_error, scheme_file.decode,
+                          "file:/bbbb")
+
+    def check_decode_04(self):
+        "scheme_file - decode empty path"
+        self.assertRaises(scheme_error, scheme_file.decode,
+                          "file://")
+
+    def check_decode_05(self):
+        "scheme_file - decode empty relative path"
+        self.assertRaises(scheme_error, scheme_file.decode,
+                          "file://somewhere")
+
+    def check_decode_06(self):
+        "scheme_file - decode ok"
+        path, tmp_file = scheme_file.decode("file:///boot/vmlinuz")
+        self.assertEqual(path, "/boot/vmlinuz")
+        self.assertEqual(tmp_file, False)
+
+class scheme_set:
+
+    def __init__(self):
+        self.schemes = [scheme_data, scheme_file]
+
+    def decode(self, uri):
+        for scheme in self.schemes:
+            try:
+                # If this passes, it is the correct scheme
+                return scheme.decode(uri)
+            except scheme_error, se:
+                log.debug("Decode throws an error: '%s'" % se)
+        return uri, False
+        
+schemes = scheme_set()
+
+
+def suite():
+    return unittest.TestSuite(
+        [unittest.makeSuite(scheme_data_unit_tests, 'check_'),
+         unittest.makeSuite(scheme_file_unit_tests, 'check_'),])
+
+if __name__ == "__main__":
+    testresult = unittest.TextTestRunner(verbosity=3).run(suite())
+

tools/python/xen/xend/XendConfig.py

@@ -41,6 +41,7 @@ from xen.util.pci import pci_opts_list_from_sxp, pci_convert_sxp_to_dict
 from xen.xend.XendSXPDev import dev_dict_to_sxp
 from xen.util import xsconstants
 from xen.util import auxbin
+import xen.util.fileuri
 
 log = logging.getLogger("xend.XendConfig")
 log.setLevel(logging.WARN)
@@ -338,6 +339,8 @@ class XendConfig(dict):
             # output from xc.domain_getinfo
             self._dominfo_to_xapi(dominfo, update_mem = True)
 
+        self.handle_fileuris()
+
         log.debug('XendConfig.init: %s' % scrub_password(self))
 
         # validators go here
@@ -1999,10 +2002,14 @@ class XendConfig(dict):
             self['_temp_kernel'] = sxp.child_value(image_sxp, 'kernel','')
             self['_temp_ramdisk'] = sxp.child_value(image_sxp, 'ramdisk','')
             self['_temp_args'] = kernel_args
+            self['use_tmp_kernel'] = True
+            self['use_tmp_ramdisk'] = True
         else:
             self['PV_kernel'] = sxp.child_value(image_sxp, 'kernel','')
             self['PV_ramdisk'] = sxp.child_value(image_sxp, 'ramdisk','')
             self['PV_args'] = kernel_args
+            self['use_tmp_kernel'] = False
+            self['use_tmp_ramdisk'] = False
 
         self['superpages'] = sxp.child_value(image_sxp, 'superpages',0)
 
@@ -2076,3 +2083,11 @@ class XendConfig(dict):
                 pci.append([domain, bus, slot, func, vdevfn, opts])
         self['platform']['pci'] = pci
 
+ 
+    def handle_fileuris(self):
+        for arg in [('PV_kernel', 'use_tmp_kernel'), 
+                    ('PV_ramdisk', 'use_tmp_ramdisk')]:
+            if self[arg[0]] != None:
+                self[arg[0]], self[arg[1]] \
+                    = xen.util.fileuri.schemes.decode(self[arg[0]])
+                log.debug("fileuri '%s' = '%s'" % (arg[0], self[arg[0]][:100]))

tools/python/xen/xend/XendDomainInfo.py

@@ -2634,7 +2634,7 @@ class XendDomainInfo:
 
             self._createDevices()
 
-            self.image.cleanupBootloading()
+            self.image.cleanupTmpImages()
 
             self.info['start_time'] = time.time()
 
@@ -2642,12 +2642,12 @@ class XendDomainInfo:
         except VmError, exn:
             log.exception("XendDomainInfo.initDomain: exception occurred")
             if self.image:
-                self.image.cleanupBootloading()
+                self.image.cleanupTmpImages()
             raise exn
         except RuntimeError, exn:
             log.exception("XendDomainInfo.initDomain: exception occurred")
             if self.image:
-                self.image.cleanupBootloading()
+                self.image.cleanupTmpImages()
             raise VmError(str(exn))
 
 

tools/python/xen/xend/image.py

@@ -89,6 +89,8 @@ class ImageHandler:
         self.vm = vm
 
         self.bootloader = False
+        self.use_tmp_kernel = False
+        self.use_tmp_ramdisk = False
         self.kernel = None
         self.ramdisk = None
         self.cmdline = None
@@ -106,6 +108,10 @@ class ImageHandler:
             self.kernel = vmConfig['PV_kernel']
             self.cmdline = vmConfig['PV_args']
             self.ramdisk = vmConfig['PV_ramdisk']
+        if vmConfig['use_tmp_kernel']:
+            self.use_tmp_ramdisk = True
+        if vmConfig['use_tmp_ramdisk']:
+            self.use_tmp_kernel = True
         self.vm.storeVm(("image/ostype", self.ostype),
                         ("image/kernel", self.kernel),
                         ("image/cmdline", self.cmdline),
@@ -135,9 +141,10 @@ class ImageHandler:
         if 'cpuid_check' in vmConfig:
             self.cpuid_check = vmConfig['cpuid_check']
 
-    def cleanupBootloading(self):
-        if self.bootloader:
+    def cleanupTmpImages(self):
+        if self.use_tmp_kernel:
             self.unlink(self.kernel)
+        if self.use_tmp_ramdisk:
             self.unlink(self.ramdisk)