Plumb the new PIF and network implementations in through the XendAPI class.

Use the new stateful records to save host, SR, network, and PIF configuration.
Add extra functionality to the SR class, to calculate capacities et al.

By Alastair Tse <atse@xensource.com>.

Signed-off-by: Ewan Mellor <ewan@xensource.com>

tools/python/xen/xend/XendAPI.py

@@ -266,6 +266,29 @@ def valid_sr(func):
 
     return check_sr_ref
 
+
+def valid_pif(func):
+    """Decorator to verify if sr_ref is valid before calling
+    method.
+
+    @param func: function with params: (self, session, sr_ref)
+    @rtype: callable object
+    """
+    def check_pif_ref(self, session, pif_ref, *args, **kwargs):
+        xennode = XendNode.instance()
+        if type(pif_ref) == type(str()) and pif_ref in xennode.pifs:
+            return func(self, session, pif_ref, *args, **kwargs)
+        else:
+            return xen_api_error(['PIF_HANDLE_INVALID', pif_ref])
+
+    # make sure we keep the 'api' attribute
+    if hasattr(func, 'api'):
+        check_pif_ref.api = func.api
+        
+    return check_pif_ref
+
+
+
 # -----------------------------
 # Bridge to Legacy XM API calls
 # -----------------------------
@@ -477,16 +500,81 @@ class XendAPI:
 
     # Xen API: Class Network
     # ----------------------------------------------------------------
-    # TODO: NOT IMPLEMENTED
 
-    Network_attr_ro = ['VIFs']
+    Network_attr_ro = ['VIFs', 'PIFs']
     Network_attr_rw = ['name_label',
                        'name_description',
-                       'NIC',
-                       'VLAN',
                        'default_gateway',
                        'default_netmask']
 
+    # Xen API: Class PIF
+    # ----------------------------------------------------------------
+
+    PIF_attr_ro = ['io_read_kbs',
+                   'io_write_kbs']
+    PIF_attr_rw = ['name',
+                   'network',
+                   'host',
+                   'MAC',
+                   'MTU',
+                   'VLAN']
+
+    PIF_attr_inst = PIF_attr_rw
+
+    # object methods
+    def PIF_get_record(self, session, pif_ref):
+        node = XendNode.instance()
+        return xen_api_success(node.pifs[pif_ref].get_record())
+
+    def PIF_get_all(self, session):
+        return xen_api_success(XendNode.instance().pifs.keys())
+
+    def PIF_set_name(self, session, pif_ref, name):
+        node = XendNode.instance()
+        pif = node.pifs.get(pif_ref)
+        if pif:
+            pif.set_name(name)
+        return xen_api_void()        
+
+    def PIF_set_mac(self, session, pif_ref, mac):
+        node = XendNode.instance()
+        pif = node.pifs.get(pif_ref)
+        if pif:
+            pif.set_mac(mac)
+        return xen_api_void()
+
+    def PIF_set_mtu(self, session, pif_ref, mtu):
+        node = XendNode.instance()
+        pif = node.pifs.get(pif_ref)
+        if pif:
+            pif.set_mtu(mtu)
+        return xen_api_void()    
+
+    def PIF_get_mac(self, session, pif_ref):
+        node = XendNode.instance()
+        return xen_api_success(node.pifs[pif_ref].get_mac())
+
+    def PIF_get_mtu(self, session, pif_ref):
+        node = XendNode.instance()
+        return xen_api_success(node.pifs[pif_ref].get_mtu())
+
+    def PIF_get_vlan(self, session, pif_ref):
+        node = XendNode.instance()
+        return xen_api_success(node.pifs[pif_ref].get_vlan())
+
+    def PIF_get_name(self, session, pif_ref):
+        node = XendNode.instance()
+        return xen_api_success(node.pifs[pif_ref].get_name())
+
+    def PIF_get_io_read_kbs(self, session, pif_ref):
+        node = XendNode.instance()
+        return xen_api_success(node.pifs[pif_ref].get_io_read_kbs())
+
+    def PIF_get_io_write_kbs(self, session, pif_ref):
+        node = XendNode.instance()
+        return xen_api_success(node.pifs[pif_ref].get_io_write_kbs())    
+    
+
     # Xen API: Class VM
     # ----------------------------------------------------------------        
 
@@ -1189,9 +1277,16 @@ class XendAPI:
         return xen_api_error(XEND_ERROR_UNSUPPORTED)
 
     def VDI_set_sharable(self, session, vdi_ref, value):
-        return xen_api_todo()
+        sr = XendNode.instance().get_sr()
+        image = sr.xen_api_get_by_uuid(vdi_ref)
+        image.sharable = bool(value)
+        return xen_api_success_void()
+    
     def VDI_set_read_only(self, session, vdi_ref, value):
-        return xen_api_todo()
+        sr = XendNode.instance().get_sr()
+        image = sr.xen_api_get_by_uuid(vdi_ref)
+        image.read_only = bool(value)
+        return xen_api_success_void()
 
     # Object Methods
     def VDI_snapshot(self, session, vdi_ref):
@@ -1383,17 +1478,7 @@ class XendAPI:
     
     def SR_get_record(self, session, sr_ref):
         sr = XendNode.instance().get_sr()
-        return xen_api_success({
-            'uuid': sr.uuid,
-            'name_label': sr.name_label,
-            'name_description': sr.name_description,
-            'VDIs': sr.list_images(),
-            'virtual_allocation': sr.used_space_bytes(),
-            'physical_utilisation': sr.used_space_bytes(),
-            'physical_size': sr.total_space_bytes(),
-            'type': sr.type,
-            'location': sr.location
-            })
+        return xen_api_success(sr.get_record())
 
     # Attribute acceess
     def SR_get_VDIs(self, session, sr_ref):
@@ -1431,11 +1516,13 @@ class XendAPI:
     def SR_set_name_label(self, session, sr_ref, value):
         sr = XendNode.instance().get_sr()
         sr.name_label = value
+        XendNode.instance().save()
         return xen_api_success_void()
     
     def SR_set_name_description(self, session, sr_ref, value):
         sr = XendNode.instance().get_sr()
         sr.name_description = value
+        XendNode.instance().save()        
         return xen_api_success_void()
 
 
@@ -1454,7 +1541,8 @@ def _decorate():
         'VIF': (valid_vif, session_required, catch_typeerror),
         'VDI': (valid_vdi, session_required, catch_typeerror),
         'VTPM':(valid_vtpm, session_required, catch_typeerror),
-        'SR':  (valid_sr, session_required, catch_typeerror)}
+        'SR':  (valid_sr, session_required, catch_typeerror),
+        'PIF': (valid_pif, session_required, catch_typeerror)}
 
     # Cheat methods
     # -------------

tools/python/xen/xend/XendNode.py

@@ -21,32 +21,128 @@ import socket
 import xen.lowlevel.xc
 from xen.xend import uuid
 from xen.xend.XendError import XendError
+from xen.xend.XendRoot import instance as xendroot
 from xen.xend.XendStorageRepository import XendStorageRepository
+from xen.xend.XendLogging import log
+from xen.xend.XendPIF import *
+from xen.xend.XendNetwork import *
+from xen.xend.XendStateStore import XendStateStore
 
 class XendNode:
     """XendNode - Represents a Domain 0 Host."""
     
     def __init__(self):
-        self.xc = xen.lowlevel.xc.xc()
-        self.uuid = uuid.createString()
-        self.cpus = {}
-        self.name = socket.gethostname()
-        self.desc = ""
-        self.sr = XendStorageRepository()
+        """Initalises the state of all host specific objects such as
+
+        * Host
+        * Host_CPU
+        * PIF
+        * Network
+        * Storage Repository
+        """
         
+        self.xc = xen.lowlevel.xc.xc()
+        self.state_store = XendStateStore(xendroot().get_xend_state_path())
+
+        # load host state from XML file
+        saved_host = self.state_store.load_state('host')
+        if saved_host and len(saved_host.keys()) == 1:
+            self.uuid = saved_host.keys()[0]
+            host = saved_host[self.uuid]
+            self.name = host.get('name_label', socket.gethostname())
+            self.desc = host.get('name_description', '')
+            self.cpus = {}
+        else:
+            self.uuid = uuid.createString()
+            self.name = socket.gethostname()
+            self.desc = ''
+            self.cpus = {}
+            
+        # load CPU UUIDs
+        saved_cpus = self.state_store.load_state('cpu')
+        for cpu_uuid, cpu in saved_cpus.items():
+            self.cpus[cpu_uuid] = cpu
+
+        # verify we have enough cpus here
         physinfo = self.physinfo_dict()
         cpu_count = physinfo['nr_cpus']
         cpu_features = physinfo['hw_caps']
-        
-        for i in range(cpu_count):
-            # construct uuid by appending extra bit on the host.
-            # since CPUs belong to a host.
-            cpu_uuid = self.uuid + '-%04d' % i
-            cpu_info = {'uuid': cpu_uuid,
-                        'host': self.uuid,
-                        'number': i,
-                        'features': cpu_features}
-            self.cpus[cpu_uuid] = cpu_info
+
+        # If the number of CPUs don't match, we should just reinitialise 
+        # the CPU UUIDs.
+        if cpu_count != len(self.cpus):
+            self.cpus = {}
+            for i in range(cpu_count):
+                cpu_uuid = uuid.createString()
+                cpu_info = {'uuid': cpu_uuid,
+                            'host': self.uuid,
+                            'number': i,
+                            'features': cpu_features}
+                self.cpus[cpu_uuid] = cpu_info
+
+        self.pifs = {}
+        self.networks = {}
+
+        # initialise PIFs
+        saved_pifs = self.state_store.load_state('pif')
+        if saved_pifs:
+            for pif_uuid, pif in saved_pifs.items():
+                self.pifs[pif_uuid] = XendPIF(pif_uuid,
+                                              pif['name'],
+                                              pif['MTU'],
+                                              pif['MAC'])
+        else:
+            for name, mtu, mac in linux_get_phy_ifaces():
+                pif_uuid = uuid.createString()
+                pif = XendPIF(pif_uuid, name, mtu, mac)
+                self.pifs[pif_uuid] = pif
+
+        # initialise networks
+        saved_networks = self.state_store.load_state('network')
+        if saved_networks:
+            for net_uuid, network in saved_networks.items():
+                self.networks[net_uuid] = XendNetwork(net_uuid,
+                                network.get('name_label'),
+                                network.get('name_description', ''),
+                                network.get('default_gateway', ''),
+                                network.get('default_netmask', ''))
+                
+                for pif_uuid in network.get('PIFs', {}).keys():
+                    pif = self.pifs.get(pif_uuid)
+                    if pif:
+                        self.networks.pifs[pif_uuid] = pif
+        else:
+            gateway, netmask = linux_get_default_network()
+            net_uuid = uuid.createString()
+            net = XendNetwork(net_uuid, 'net0', '', gateway, netmask)
+            self.networks[net_uuid] = net
+
+        # initialise storage
+        saved_sr = self.state_store.load_state('sr')
+        if saved_sr and len(saved_sr) == 1:
+            sr_uuid = saved_sr.keys()[0]
+            self.sr = XendStorageRepository(sr_uuid)
+        else:
+            sr_uuid = uuid.createString()
+            self.sr = XendStorageRepository(sr_uuid)
+        self.save()
+
+    def save(self):
+        # save state
+        host_record = {self.uuid: {'name_label':self.name,
+                                   'name_description':self.desc}}
+        self.state_store.save_state('host',host_record)
+        self.state_store.save_state('cpu', self.cpus)
+        pif_records = dict([(k, v.get_record())
+                            for k, v in self.pifs.items()])
+        self.state_store.save_state('pif', pif_records)
+        net_records = dict([(k, v.get_record())
+                            for k, v in self.networks.items()])
+        self.state_store.save_state('network', net_records)
+
+
+        sr_record = {self.sr.uuid: self.sr.get_record()}
+        self.state_store.save_state('sr', sr_record)
 
     def shutdown(self):
         return 0
@@ -56,7 +152,8 @@ class XendNode:
 
     def notify(self, _):
         return 0
-    
+
+        
     #
     # Ref validation
     #
@@ -100,6 +197,9 @@ class XendNode:
     def set_description(self, new_desc):
         self.desc = new_desc
 
+    def get_uuid(self):
+        return self.uuid
+
     #
     # Host CPU Functions
     #

tools/python/xen/xend/XendStorageRepository.py

@@ -24,13 +24,17 @@ import logging
 import os
 import stat
 import threading
+import re
+import sys
+import struct
 
 from xen.util import mkdir
 from xen.xend import uuid
 from xen.xend.XendError import XendError
 from xen.xend.XendVDI import *
 
-XEND_STORAGE_MAX_IGNORE = -1
+
+XEND_STORAGE_NO_MAXIMUM = sys.maxint
 XEND_STORAGE_DIR = "/var/lib/xend/storage/"
 XEND_STORAGE_QCOW_FILENAME = "%s.qcow"
 XEND_STORAGE_VDICFG_FILENAME = "%s.vdi.xml"
@@ -41,8 +45,17 @@ MB = 1024 * 1024
 log = logging.getLogger("xend.XendStorageRepository")
 
 
-class DeviceInvalidError(Exception):
-    pass
+def qcow_virtual_size(qcow_file):
+    """Read the first 32 bytes of the QCoW header to determine its size.
+
+    See: http://www.gnome.org/~markmc/qcow-image-format.html.
+    """
+    try:
+        qcow_header = open(qcow_file, 'rb').read(32)
+        parts = struct.unpack('>IIQIIQ', qcow_header)
+        return parts[-1]
+    except IOError:
+        return -1
 
 class XendStorageRepository:
     """A simple file backed QCOW Storage Repository.
@@ -54,11 +67,13 @@ class XendStorageRepository:
     The actual images are created in the format <uuid>.img and <uuid>.qcow.
     """
     
-    def __init__(self, storage_dir = XEND_STORAGE_DIR,
-                 storage_max = XEND_STORAGE_MAX_IGNORE):
+    def __init__(self, uuid,
+                 sr_type = "qcow_file",
+                 name_label = "Local",
+                 name_description = "Xend Storage Repository",
+                 location = XEND_STORAGE_DIR,
+                 storage_max = XEND_STORAGE_NO_MAXIMUM):
         """
-        @keyword storage_dir: Where the images will be stored.
-        @type    storage_dir: string
         @keyword storage_max: Maximum disk space to use in bytes.
         @type    storage_max: int
 
@@ -67,71 +82,78 @@ class XendStorageRepository:
         @type    images: dictionary by image uuid.
         @ivar    lock:   lock to provide thread safety.
         """
-        
-        self.storage_dir = storage_dir
-        self.storage_max = storage_max
-        self.storage_free = 0
-        self.images = {}
 
         # XenAPI Parameters
-        self.uuid = self._sr_uuid()
-        self.type = "qcow-file"
-        self.location = self.storage_dir
-        self.name_label = "Local"
-        self.name_description = "Xend Storage Repository"
+        self.uuid = uuid
+        self.type = sr_type
+        self.location = location
+        self.name_label = name_label
+        self.name_description = name_description
+        self.images = {}
+
+        self.storage_max = storage_max
+        self.storage_free = 0
+        self.storage_used = 0
+        self.storage_alloc = 0
 
         self.lock = threading.RLock()
-        self._refresh()        
+        self._refresh()
 
-    def _sr_uuid(self):
-        uuid_file = os.path.join(XEND_STORAGE_DIR, 'uuid')
-        try:
-            if uuid_file and os.path.exists(uuid_file):
-                return open(uuid_file, 'r').read().strip()
-            else:
-                new_uuid = uuid.createString()
-                open(uuid_file, 'w').write(new_uuid + '\n')
-                return new_uuid
-        except IOError:
-            log.exception("Failed to determine SR UUID")
-
-        return uuid.createString()
+    def get_record(self):
+        retval = {'uuid': self.uuid,
+                  'name_label': self.name_label,
+                  'name_description': self.name_description,
+                  'virtual_allocation': self.storage_alloc,
+                  'physical_utilisation': self.storage_used,
+                  'physical_size': self.storage_max,
+                  'type': self.type,
+                  'location': self.location,
+                  'VDIs': self.images.keys()}
+        
+        if self.storage_max == XEND_STORAGE_NO_MAXIMUM:
+            stfs = os.statvfs(self.location)
+            retval['physical_size'] = stfs.f_blocks * stfs.f_frsize
 
+        return retval
+        
     def _refresh(self):
         """Internal function that refreshes the state of the disk and
         updates the list of images available.
         """
         self.lock.acquire()
         try:
-            mkdir.parents(XEND_STORAGE_DIR, stat.S_IRWXU)
+            mkdir.parents(self.location, stat.S_IRWXU)
 
             # scan the directory and populate self.images
-            total_used = 0
+            virtual_alloc = 0
+            physical_used = 0
             seen_images = []
-            for filename in os.listdir(XEND_STORAGE_DIR):
+            for filename in os.listdir(self.location):
                 if filename[-5:] == XEND_STORAGE_QCOW_FILENAME[-5:]:
                     image_uuid = filename[:-5]
                     seen_images.append(image_uuid)
+
+                    qcow_file = XEND_STORAGE_QCOW_FILENAME % image_uuid
+                    cfg_file = XEND_STORAGE_VDICFG_FILENAME % image_uuid
+                    qcow_path = os.path.join(self.location, qcow_file)
+                    cfg_path = os.path.join(self.location, cfg_file)
+                    
+                    phys_size = os.stat(qcow_path).st_size
+                    virt_size = qcow_virtual_size(qcow_path)
                     
                     # add this image if we haven't seen it before
                     if image_uuid not in self.images:
-                        qcow_file = XEND_STORAGE_QCOW_FILENAME % image_uuid
-                        cfg_file = XEND_STORAGE_VDICFG_FILENAME % image_uuid
-                        qcow_path = os.path.join(XEND_STORAGE_DIR, qcow_file)
-                        cfg_path = os.path.join(XEND_STORAGE_DIR, cfg_file)
-
-                        qcow_size = os.stat(qcow_path).st_size
-
-                        # TODO: no way to stat virtual size of qcow
                         vdi = XendQCOWVDI(image_uuid, self.uuid,
                                           qcow_path, cfg_path,
-                                          qcow_size, qcow_size) 
+                                          virt_size, phys_size)
                         
                         if cfg_path and os.path.exists(cfg_path):
                             vdi.load_config(cfg_path)
                         
                         self.images[image_uuid] = vdi
-                        total_used += qcow_size
+
+                    physical_used += phys_size
+                    virtual_alloc += virt_size
 
             # remove images that aren't valid
             for image_uuid in self.images.keys():
@@ -142,11 +164,14 @@ class XendStorageRepository:
                         pass
                     del self.images[image_uuid]
 
+            self.storage_alloc = virtual_alloc
+            self.storage_used = physical_used
+
             # update free storage if we have to track that
-            if self.storage_max != XEND_STORAGE_MAX_IGNORE:
-                self.storage_free = self.storage_max - total_used
-            else:
+            if self.storage_max == XEND_STORAGE_NO_MAXIMUM:
                 self.storage_free = self._get_free_space()
+            else:
+                self.storage_free = self.storage_max - self.storage_alloc
                         
         finally:
             self.lock.release()
@@ -158,7 +183,7 @@ class XendStorageRepository:
 
         @rtype: int
         """
-        stfs = os.statvfs(self.storage_dir)
+        stfs = os.statvfs(self.location)
         return stfs.f_bavail * stfs.f_frsize
 
     def _has_space_available_for(self, size_bytes):
@@ -167,22 +192,19 @@ class XendStorageRepository:
 
         @rtype: bool
         """
-        if self.storage_max != -1:
-            return self.storage_free
+        if self.storage_max != XEND_STORAGE_NO_MAXIMUM:
+            return self.storage_free > size_bytes
         
         bytes_free = self._get_free_space()
-        try:
-            if size_bytes < bytes_free:
-                return True
-        except DeviceInvalidError:
-            pass
+        if size_bytes < bytes_free:
+            return True
         return False
 
     def _create_image_files(self, desired_size_bytes):
         """Create an image and return its assigned UUID.
 
-        @param desired_size_kb: Desired image size in KB.
-        @type  desired_size_kb: int
+        @param desired_size_bytes: Desired image size in bytes
+        @type  desired_size_bytes: int
         @rtype: string
         @return: uuid
 
@@ -194,7 +216,7 @@ class XendStorageRepository:
                 raise XendError("Not enough space")
 
             image_uuid = uuid.createString()
-            qcow_path = os.path.join(XEND_STORAGE_DIR,
+            qcow_path = os.path.join(self.location,
                                      XEND_STORAGE_QCOW_FILENAME % image_uuid)
             
             if qcow_path and os.path.exists(qcow_path):
@@ -268,10 +290,7 @@ class XendStorageRepository:
         """
         self.lock.acquire()
         try:
-            if self.storage_max != XEND_STORAGE_MAX_IGNORE:
-                return self.storage_max
-            else:
-                return self.free_space_bytes() + self.used_space_bytes()
+            return self.storage_max
         finally:
             self.lock.release()
             
@@ -315,7 +334,7 @@ class XendStorageRepository:
 
             # save configuration to file
             cfg_filename =  XEND_STORAGE_VDICFG_FILENAME % image_uuid
-            cfg_path = os.path.join(XEND_STORAGE_DIR, cfg_filename)
+            cfg_path = os.path.join(self.location, cfg_filename)
             image.save_config(cfg_path)
             
         except Exception, e:
@@ -327,10 +346,10 @@ class XendStorageRepository:
 
         return image_uuid
         
-    def xen_api_get_by_label(self, label):
+    def xen_api_get_by_name_label(self, label):
         self.lock.acquire()
         try:
-            for image_uuid, val in self.images.values():
+            for image_uuid, val in self.images.items():
                 if val.name_label == label:
                     return image_uuid
             return None