[PYGRUB] Plumb bootloader I/O through xenconsole.

- xend forwards console traffic between the running bootloader and a pty
   which it writes to the store so the xenconsole client can see it.
 - the xenconsole client handles the domain's console pty changing.
 - xm create no longer runs the bootloader.
 - pygrub gets '-i' option to explicitly request an interactive session.
 - xend unlocks the domain list during bootloading so that "xm console"
   can see the domain and attach to its console pty.

Signed-off-by: Tim Deegan <Tim.Deegan@xensource.com>

tools/python/xen/xend/XendBootloader.py

@@ -12,7 +12,7 @@
 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 #
 
-import os, select, errno, stat
+import os, select, errno, stat, signal
 import random
 import shlex
 from xen.xend import sxp
@@ -21,12 +21,15 @@ from xen.util import mkdir
 from XendLogging import log
 from XendError import VmError
 
-def bootloader(blexec, disk, quiet = False, blargs = '', kernel = '',
+import pty, ptsname, termios, fcntl
+
+def bootloader(blexec, disk, dom, quiet = False, blargs = '', kernel = '',
                ramdisk = '', kernel_args = ''):
     """Run the boot loader executable on the given disk and return a
     config image.
     @param blexec  Binary to use as the boot loader
     @param disk Disk to run the boot loader on.
+    @param dom DomainInfo representing the domain being booted.
     @param quiet Run in non-interactive mode, just booting the default.
     @param blargs Arguments to pass to the bootloader."""
     
@@ -50,7 +53,33 @@ def bootloader(blexec, disk, quiet = False, blargs = '', kernel = '',
                 raise
         break
 
-    child = os.fork()
+    # We need to present the bootloader's tty as a pty slave that xenconsole
+    # can access.  Since the bootloader itself needs a pty slave, 
+    # we end up with a connection like this:
+    #
+    # xenconsole -- (slave pty1 master) <-> (master pty2 slave) -- bootloader
+    #
+    # where we copy characters between the two master fds, as well as
+    # listening on the bootloader's fifo for the results.
+
+    # Termios runes for very raw access to the pty master fds.
+    attr = [ 0, 0, termios.CS8 | termios.CREAD | termios.CLOCAL,
+             0, 0, 0, [0] * 32 ]
+
+    (m1, s1) = pty.openpty()
+    termios.tcsetattr(m1, termios.TCSANOW, attr)
+    fcntl.fcntl(m1, fcntl.F_SETFL, os.O_NDELAY);
+    os.close(s1)
+    slavename = ptsname.ptsname(m1)
+    dom.storeDom("console/tty", slavename)
+
+    # Release the domain lock here, because we definitely don't want 
+    # a stuck bootloader to deny service to other xend clients.
+    from xen.xend import XendDomain
+    domains = XendDomain.instance()
+    domains.domains_lock.release()
+    
+    (child, m2) = pty.fork()
     if (not child):
         args = [ blexec ]
         if kernel:
@@ -74,6 +103,11 @@ def bootloader(blexec, disk, quiet = False, blargs = '', kernel = '',
             pass
         os._exit(1)
 
+    # record that this domain is bootloading
+    dom.bootloader_pid = child
+
+    termios.tcsetattr(m2, termios.TCSANOW, attr)
+    fcntl.fcntl(m2, fcntl.F_SETFL, os.O_NDELAY);
     while True:
         try:
             r = os.open(fifo, os.O_RDONLY)
@@ -82,17 +116,53 @@ def bootloader(blexec, disk, quiet = False, blargs = '', kernel = '',
                 continue
         break
     ret = ""
+    inbuf=""; outbuf="";
     while True:
-        select.select([r], [], [])
-        s = os.read(r, 1024)
-        ret = ret + s
-        if len(s) == 0:
-            break
-        
+        sel = select.select([r, m1, m2], [m1, m2], [])
+        try: 
+            if m1 in sel[0]:
+                s = os.read(m1, 1)
+                inbuf += s
+            if m2 in sel[1] and len(inbuf) != 0:
+                os.write(m2, inbuf[0])
+                inbuf = inbuf[1:]
+        except OSError, e:
+            if e.errno == errno.EIO:
+                pass
+        try:
+            if m2 in sel[0]:
+                s = os.read(m2, 1)
+                outbuf += s
+            if m1 in sel[1] and len(outbuf) != 0:
+                os.write(m1, outbuf[0])
+                outbuf = outbuf[1:]
+        except OSError, e:
+            if e.errno == errno.EIO:
+                pass
+        if r in sel[0]:
+            s = os.read(r, 1)
+            ret = ret + s
+            if len(s) == 0:
+                break
+    del inbuf
+    del outbuf
     os.waitpid(child, 0)
     os.close(r)
+    os.close(m2)
+    os.close(m1)
     os.unlink(fifo)
 
+    # Re-acquire the lock to cover the changes we're about to make
+    # when we return to domain creation.
+    domains.domains_lock.acquire()    
+
+    if dom.bootloader_pid is None:
+        msg = "Domain was died while the bootloader was running."
+        log.error(msg)
+        raise VmError, msg        
+        
+    dom.bootloader_pid = None
+
     if len(ret) == 0:
         msg = "Boot loader didn't return any data!"
         log.error(msg)
@@ -103,3 +173,12 @@ def bootloader(blexec, disk, quiet = False, blargs = '', kernel = '',
     pin.input_eof()
     blcfg = pin.val
     return blcfg
+
+
+def bootloader_tidy(dom):
+    if hasattr(dom, "bootloader_pid") and dom.bootloader_pid is not None:
+        pid = dom.bootloader_pid
+        dom.bootloader_pid = None
+        os.kill(pid, signal.SIGKILL)
+
+

tools/python/xen/xend/XendDomain.py

@@ -115,7 +115,6 @@ class XendDomain:
                 
                 dom0info['name'] = DOM0_NAME
                 dom0 = XendDomainInfo.recreate(dom0info, True)
-                self._add_domain(dom0)
             except IndexError:
                 raise XendError('Unable to find Domain 0')
             
@@ -172,7 +171,6 @@ class XendDomain:
                 if dom['domid'] != DOM0_ID:
                     try:
                         new_dom = XendDomainInfo.recreate(dom, False)
-                        self._add_domain(new_dom)
                     except Exception:
                         log.exception("Failed to create reference to running "
                                       "domain id: %d" % dom['domid'])
@@ -397,7 +395,6 @@ class XendDomain:
             elif domid not in self.domains and dom['dying'] != 1:
                 try:
                     new_dom = XendDomainInfo.recreate(dom, False)
-                    self._add_domain(new_dom)                    
                 except VmError:
                     log.exception("Unable to recreate domain")
                     try:
@@ -416,10 +413,10 @@ class XendDomain:
         running_domids = [d['domid'] for d in running if d['dying'] != 1]
         for domid, dom in self.domains.items():
             if domid not in running_domids and domid != DOM0_ID:
-                self._remove_domain(dom, domid)
+                self.remove_domain(dom, domid)
 
 
-    def _add_domain(self, info):
+    def add_domain(self, info):
         """Add a domain to the list of running domains
         
         @requires: Expects to be protected by the domains_lock.
@@ -434,7 +431,7 @@ class XendDomain:
         if info.get_uuid() in self.managed_domains:
             self._managed_domain_register(info)
 
-    def _remove_domain(self, info, domid = None):
+    def remove_domain(self, info, domid = None):
         """Remove the domain from the list of running domains
         
         @requires: Expects to be protected by the domains_lock.
@@ -473,7 +470,6 @@ class XendDomain:
         try:
             security.refresh_ssidref(config)
             dominfo = XendDomainInfo.restore(config)
-            self._add_domain(dominfo)
             return dominfo
         finally:
             self.domains_lock.release()
@@ -848,7 +844,6 @@ class XendDomain:
                                            os.open(chkpath, os.O_RDONLY),
                                            dominfo,
                                            paused = start_paused)
-                    self._add_domain(dominfo)
                     os.unlink(chkpath)
                 except OSError, ex:
                     raise XendError("Failed to read stored checkpoint file")
@@ -873,7 +868,6 @@ class XendDomain:
             self._refresh()
 
             dominfo = XendDomainInfo.create(config)
-            self._add_domain(dominfo)
             self.domain_sched_credit_set(dominfo.getDomid(),
                                          dominfo.getWeight(),
                                          dominfo.getCap())
@@ -893,7 +887,6 @@ class XendDomain:
             self._refresh()
 
             dominfo = XendDomainInfo.create_from_dict(config_dict)
-            self._add_domain(dominfo)
             self.domain_sched_credit_set(dominfo.getDomid(),
                                          dominfo.getWeight(),
                                          dominfo.getCap())
@@ -950,7 +943,6 @@ class XendDomain:
                                  POWER_STATE_NAMES[dominfo.state])
             
             dominfo.start(is_managed = True)
-            self._add_domain(dominfo)
         finally:
             self.domains_lock.release()
         dominfo.waitForDevices()
@@ -983,7 +975,7 @@ class XendDomain:
                          (dominfo.getName(), dominfo.info.get('uuid')))
 
                 self._managed_domain_unregister(dominfo)
-                self._remove_domain(dominfo)
+                self.remove_domain(dominfo)
                 XendDevices.destroy_device_state(dominfo)
             except Exception, ex:
                 raise XendError(str(ex))

tools/python/xen/xend/XendDomainInfo.py

@@ -41,7 +41,7 @@ from xen.xend import balloon, sxp, uuid, image, arch, osdep
 from xen.xend import XendOptions, XendNode, XendConfig
 
 from xen.xend.XendConfig import scrub_password
-from xen.xend.XendBootloader import bootloader
+from xen.xend.XendBootloader import bootloader, bootloader_tidy
 from xen.xend.XendError import XendError, VmError
 from xen.xend.XendDevices import XendDevices
 from xen.xend.xenstore.xstransact import xstransact, complete
@@ -101,7 +101,6 @@ def create_from_dict(config_dict):
         log.exception('Domain construction failed')
         vm.destroy()
         raise
-
     return vm
 
 def recreate(info, priv):
@@ -187,6 +186,11 @@ def recreate(info, priv):
 
     vm._registerWatches()
     vm.refreshShutdown(xeninfo)
+
+    # register the domain in the list 
+    from xen.xend import XendDomain
+    XendDomain.instance().add_domain(vm)
+
     return vm
 
 
@@ -1338,6 +1342,9 @@ class XendDomainInfo:
         # Set maximum number of vcpus in domain
         xc.domain_max_vcpus(self.domid, int(self.info['vcpus_number']))
 
+        # register the domain in the list 
+        from xen.xend import XendDomain
+        XendDomain.instance().add_domain(self)
 
     def _introduceDomain(self):
         assert self.domid is not None
@@ -1436,6 +1443,7 @@ class XendDomainInfo:
         try:
             self.unwatchShutdown()
             self._releaseDevices()
+            bootloader_tidy(self)
 
             if self.image:
                 try:
@@ -1536,6 +1544,9 @@ class XendDomainInfo:
         except:
             log.exception("XendDomainInfo.destroy: xc.domain_destroy failed.")
 
+        from xen.xend import XendDomain
+        XendDomain.instance().remove_domain(self)
+
         self.cleanupDomain()
 
 
@@ -1631,7 +1642,7 @@ class XendDomainInfo:
                     fn = BOOTLOADER_LOOPBACK_DEVICE
 
                 try:
-                    blcfg = bootloader(blexec, fn, True,
+                    blcfg = bootloader(blexec, fn, self, False,
                                        bootloader_args, kernel, ramdisk, args)
                 finally:
                     if mounted:

tools/python/xen/xm/create.py

@@ -24,6 +24,7 @@ import os.path
 import sys
 import socket
 import re
+import time
 import xmlrpclib
 
 from xen.xend import sxp
@@ -709,26 +710,6 @@ def configure_hvm(config_image, vals):
             config_image.append([a, vals.__dict__[a]])
     config_image.append(['vncpasswd', vals.vncpasswd])
 
-def run_bootloader(vals, config_image):
-    if not os.access(vals.bootloader, os.F_OK):
-        err("Bootloader '%s' does not exist" % vals.bootloader)
-    if not os.access(vals.bootloader, os.X_OK):
-        err("Bootloader '%s' isn't executable" % vals.bootloader)
-    if len(vals.disk) < 1:
-        err("No disks configured and boot loader requested")
-    (uname, dev, mode, backend) = vals.disk[0]
-    file = blkif.blkdev_uname_to_file(uname)
-
-    if vals.bootentry:
-        warn("The bootentry option is deprecated.  Use bootargs and pass "
-             "--entry= directly.")
-        vals.bootargs = "--entry=%s" %(vals.bootentry,)
-
-    kernel = sxp.child_value(config_image, 'kernel')
-    ramdisk = sxp.child_value(config_image, 'ramdisk')
-    args = sxp.child_value(config_image, 'args')
-    return bootloader(vals.bootloader, file, not vals.console_autoconnect,
-                      vals.bootargs, kernel, ramdisk, args)
 
 def make_config(vals):
     """Create the domain configuration.
@@ -771,14 +752,11 @@ def make_config(vals):
         if vals.bootloader == "pygrub":
             vals.bootloader = osdep.pygrub_path
 
-        # if a kernel is specified, we're using the bootloader
-        # non-interactively, and need to let xend run it so we preserve the
-        # real kernel choice.
-        if not vals.kernel:
-            config_image = run_bootloader(vals, config_image)
         config.append(['bootloader', vals.bootloader])
         if vals.bootargs:
             config.append(['bootloader_args', vals.bootargs])
+        else: 
+            config.append(['bootloader_args', '-q'])        
     config.append(['image', config_image])
 
     config_devs = []
@@ -1266,9 +1244,28 @@ def main(argv):
         if not create_security_check(config):
             raise security.ACMError('Security Configuration prevents domain from starting')
         else:
-            dom = make_domain(opts, config)
             if opts.vals.console_autoconnect:
-                console.execConsole(dom)        
-             
+                cpid = os.fork() 
+                if cpid != 0:
+                    for i in range(10):
+                        # Catch failure of the create process 
+                        time.sleep(1)
+                        (p, rv) = os.waitpid(cpid, os.WNOHANG)
+                        if os.WIFEXITED(rv):
+                            if os.WEXITSTATUS(rv) != 0:
+                                sys.exit(os.WEXITSTATUS(rv))
+                        try:
+                            # Acquire the console of the created dom
+                            name = sxp.child_value(config, 'name', -1)
+                            dom = server.xend.domain(name)
+                            domid = int(sxp.child_value(dom, 'domid', '-1'))
+                            console.execConsole(domid)
+                        except:
+                            pass
+                    print("Could not start console\n");
+                    sys.exit(0)
+            dom = make_domain(opts, config)
+
+
 if __name__ == '__main__':
     main(sys.argv)