[PYGRUB] Plumb bootloader I/O through xenconsole.

- xend forwards console traffic between the running bootloader and a pty
   which it writes to the store so the xenconsole client can see it.
 - the xenconsole client handles the domain's console pty changing.
 - xm create no longer runs the bootloader.
 - pygrub gets '-i' option to explicitly request an interactive session.
 - xend unlocks the domain list during bootloading so that "xm console"
   can see the domain and attach to its console pty.

Signed-off-by: Tim Deegan <Tim.Deegan@xensource.com>

tools/console/client/main.c

@@ -71,6 +71,43 @@ static void usage(const char *program) {
 	       , program);
 }
 
+static int get_pty_fd(struct xs_handle *xs, char *path, int seconds)
+/* Check for a pty in xenstore, open it and return its fd.
+ * Assumes there is already a watch set in the store for this path. */
+{
+	struct timeval tv;
+	fd_set watch_fdset;
+	int xs_fd = xs_fileno(xs), pty_fd = -1;
+	int start, now;
+	unsigned int len = 0;
+	char *pty_path, **watch_paths;;
+
+	start = now = time(NULL);
+	do {
+		tv.tv_usec = 0;
+		tv.tv_sec = (start + seconds) - now;
+		FD_ZERO(&watch_fdset);
+		FD_SET(xs_fd, &watch_fdset);
+		if (select(xs_fd + 1, &watch_fdset, NULL, NULL, &tv)) {
+			/* Read the watch to drain the buffer */
+			watch_paths = xs_read_watch(xs, &len);
+			free(watch_paths);
+			/* We only watch for one thing, so no need to 
+			 * disambiguate: just read the pty path */
+			pty_path = xs_read(xs, XBT_NULL, path, &len);
+			if (pty_path != NULL) {
+				pty_fd = open(pty_path, O_RDWR | O_NOCTTY);
+				if (pty_fd == -1) 
+					err(errno, "Could not open tty `%s'", 
+					    pty_path);
+				free(pty_path);
+			}
+		}
+	} while (pty_fd == -1 && (now = time(NULL)) < start + seconds);
+	return pty_fd;
+}
+
+
 /* don't worry too much if setting terminal attributes fail */
 static void init_term(int fd, struct termios *old)
 {
@@ -91,18 +128,22 @@ static void restore_term(int fd, struct termios *old)
 	tcsetattr(fd, TCSAFLUSH, old);
 }
 
-static int console_loop(int fd)
+static int console_loop(int fd, struct xs_handle *xs, char *pty_path)
 {
-	int ret;
+	int ret, xs_fd = xs_fileno(xs), max_fd;
 
 	do {
 		fd_set fds;
 
 		FD_ZERO(&fds);
 		FD_SET(STDIN_FILENO, &fds);
-		FD_SET(fd, &fds);
+		max_fd = STDIN_FILENO;
+		FD_SET(xs_fd, &fds);
+		if (xs_fd > max_fd) max_fd = xs_fd;
+		if (fd != -1) FD_SET(fd, &fds);
+		if (fd > max_fd) max_fd = fd;
 
-		ret = select(fd + 1, &fds, NULL, NULL, NULL);
+		ret = select(max_fd + 1, &fds, NULL, NULL, NULL);
 		if (ret == -1) {
 			if (errno == EINTR || errno == EAGAIN) {
 				continue;
@@ -110,6 +151,16 @@ static int console_loop(int fd)
 			return -1;
 		}
 
+		if (FD_ISSET(xs_fileno(xs), &fds)) {
+			int newfd = get_pty_fd(xs, pty_path, 0);
+			close(fd);
+                        if (newfd == -1) 
+				/* Console PTY has become invalid */
+				return 0;
+			fd = newfd;
+			continue;
+		}
+
 		if (FD_ISSET(STDIN_FILENO, &fds)) {
 			ssize_t len;
 			char msg[60];
@@ -128,12 +179,13 @@ static int console_loop(int fd)
 			}
 
 			if (!write_sync(fd, msg, len)) {
-				perror("write() failed");
-				return -1;
+				close(fd);
+				fd = -1;
+				continue;
 			}
 		}
 
-		if (FD_ISSET(fd, &fds)) {
+		if (fd != -1 && FD_ISSET(fd, &fds)) {
 			ssize_t len;
 			char msg[512];
 
@@ -143,7 +195,9 @@ static int console_loop(int fd)
 				    (errno == EINTR || errno == EAGAIN)) {
 					continue;
 				}
-				return -1;
+				close(fd);
+				fd = -1;
+				continue;
 			}
 
 			if (!write_sync(STDOUT_FILENO, msg, len)) {
@@ -168,12 +222,10 @@ int main(int argc, char **argv)
 		{ 0 },
 
 	};
-	char *str_pty, *path;
-	int spty;
-	unsigned int len = 0;
+	char *path;
+	int spty, xsfd;
 	struct xs_handle *xs;
 	char *end;
-	time_t now;
 
 	while((ch = getopt_long(argc, argv, sopt, lopt, &opt_ind)) != -1) {
 		switch(ch) {
@@ -213,7 +265,6 @@ int main(int argc, char **argv)
 	if (path == NULL)
 		err(ENOMEM, "realloc");
 	strcat(path, "/console/tty");
-	str_pty = xs_read(xs, XBT_NULL, path, &len);
 
 	/* FIXME consoled currently does not assume domain-0 doesn't have a
 	   console which is good when we break domain-0 up.  To keep us
@@ -224,38 +275,24 @@ int main(int argc, char **argv)
 		exit(EINVAL);
 	}
 
+	/* Set a watch on this domain's console pty */
+	if (!xs_watch(xs, path, ""))
+		err(errno, "Can't set watch for console pty");
+	xsfd = xs_fileno(xs);
+
 	/* Wait a little bit for tty to appear.  There is a race
-	   condition that occurs after xend creates a domain.  This
-	   code might be running before consoled has noticed the new
-	   domain and setup a pty for it.
-
-	   A xenstore watch would slightly improve responsiveness but
-	   a timeout would still be needed since we don't want to
-	   block forever if given an invalid domain or worse yet, a
-	   domain that someone else has connected to. */
-
-	now = time(0);
-	while (str_pty == NULL && (now + 5) > time(0)) {
-		struct timeval tv = { 0, 250000 };
-		select(0, NULL, NULL, NULL, &tv); /* pause briefly */
-
-		str_pty = xs_read(xs, XBT_NULL, path, &len);
-	}
-
-	if (str_pty == NULL) {
+	   condition that occurs after xend creates a domain.  This code
+	   might be running before consoled has noticed the new domain
+	   and setup a pty for it. */ 
+        spty = get_pty_fd(xs, path, 5);
+	if (spty == -1) {
 		err(errno, "Could not read tty from store");
 	}
 
-	spty = open(str_pty, O_RDWR | O_NOCTTY);
-	if (spty == -1) {
-		err(errno, "Could not open tty `%s'", str_pty);
-	}
-	free(str_pty);
-	free(path);
-
 	init_term(STDIN_FILENO, &attr);
-	console_loop(spty);
+	console_loop(spty, xs, path);
 	restore_term(STDIN_FILENO, &attr);
 
+	free(path);
 	return 0;
  }

tools/misc/xend

@@ -44,7 +44,7 @@ for p in ['python%s' % sys.version[:3], 'python']:
         if os.path.exists(os.path.join(d, AUXBIN)):
             sys.path.append(d)
             import xen.util.auxbin
-            libpath = xen.util.auxbin.libpath()
+            libpath = os.path.join(xen.util.auxbin.libpath(), p)
             sys.path = sys.path[:-1]
             sys.path.append(libpath)
             break

tools/pygrub/src/pygrub

@@ -201,7 +201,9 @@ class Grub:
             enable_cursor(False)
             self.entry_win = curses.newwin(10, 74, 2, 1)
             self.text_win = curses.newwin(10, 70, 12, 5)
-            
+            curses.def_prog_mode()
+        
+        curses.reset_prog_mode()
         self.screen.clear()
         self.screen.refresh()
 
@@ -549,11 +551,12 @@ if __name__ == "__main__":
     sel = None
     
     def usage():
-        print >> sys.stderr, "Usage: %s [-q|--quiet] [--output=] [--kernel=] [--ramdisk=] [--args=] [--entry=] <image>" %(sys.argv[0],)
+        print >> sys.stderr, "Usage: %s [-q|--quiet] [-i|--interactive] [--output=] [--kernel=] [--ramdisk=] [--args=] [--entry=] <image>" %(sys.argv[0],)
 
     try:
-        opts, args = getopt.gnu_getopt(sys.argv[1:], 'qh::',
-                                   ["quiet", "help", "output=", "entry=", "kernel=", "ramdisk=", "args=",
+        opts, args = getopt.gnu_getopt(sys.argv[1:], 'qih::',
+                                   ["quiet", "interactive", "help", "output=",
+                                    "entry=", "kernel=", "ramdisk=", "args=",
                                     "isconfig"])
     except getopt.GetoptError:
         usage()
@@ -579,6 +582,8 @@ if __name__ == "__main__":
     for o, a in opts:
         if o in ("-q", "--quiet"):
             interactive = False
+        elif o in ("-i", "--interactive"):
+            interactive = True
         elif o in ("-h", "--help"):
             usage()
             sys.exit()

tools/python/xen/xend/XendBootloader.py

@@ -12,7 +12,7 @@
 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 #
 
-import os, select, errno, stat
+import os, select, errno, stat, signal
 import random
 import shlex
 from xen.xend import sxp
@@ -21,12 +21,15 @@ from xen.util import mkdir
 from XendLogging import log
 from XendError import VmError
 
-def bootloader(blexec, disk, quiet = False, blargs = '', kernel = '',
+import pty, ptsname, termios, fcntl
+
+def bootloader(blexec, disk, dom, quiet = False, blargs = '', kernel = '',
                ramdisk = '', kernel_args = ''):
     """Run the boot loader executable on the given disk and return a
     config image.
     @param blexec  Binary to use as the boot loader
     @param disk Disk to run the boot loader on.
+    @param dom DomainInfo representing the domain being booted.
     @param quiet Run in non-interactive mode, just booting the default.
     @param blargs Arguments to pass to the bootloader."""
     
@@ -50,7 +53,33 @@ def bootloader(blexec, disk, quiet = False, blargs = '', kernel = '',
                 raise
         break
 
-    child = os.fork()
+    # We need to present the bootloader's tty as a pty slave that xenconsole
+    # can access.  Since the bootloader itself needs a pty slave, 
+    # we end up with a connection like this:
+    #
+    # xenconsole -- (slave pty1 master) <-> (master pty2 slave) -- bootloader
+    #
+    # where we copy characters between the two master fds, as well as
+    # listening on the bootloader's fifo for the results.
+
+    # Termios runes for very raw access to the pty master fds.
+    attr = [ 0, 0, termios.CS8 | termios.CREAD | termios.CLOCAL,
+             0, 0, 0, [0] * 32 ]
+
+    (m1, s1) = pty.openpty()
+    termios.tcsetattr(m1, termios.TCSANOW, attr)
+    fcntl.fcntl(m1, fcntl.F_SETFL, os.O_NDELAY);
+    os.close(s1)
+    slavename = ptsname.ptsname(m1)
+    dom.storeDom("console/tty", slavename)
+
+    # Release the domain lock here, because we definitely don't want 
+    # a stuck bootloader to deny service to other xend clients.
+    from xen.xend import XendDomain
+    domains = XendDomain.instance()
+    domains.domains_lock.release()
+    
+    (child, m2) = pty.fork()
     if (not child):
         args = [ blexec ]
         if kernel:
@@ -74,6 +103,11 @@ def bootloader(blexec, disk, quiet = False, blargs = '', kernel = '',
             pass
         os._exit(1)
 
+    # record that this domain is bootloading
+    dom.bootloader_pid = child
+
+    termios.tcsetattr(m2, termios.TCSANOW, attr)
+    fcntl.fcntl(m2, fcntl.F_SETFL, os.O_NDELAY);
     while True:
         try:
             r = os.open(fifo, os.O_RDONLY)
@@ -82,17 +116,53 @@ def bootloader(blexec, disk, quiet = False, blargs = '', kernel = '',
                 continue
         break
     ret = ""
+    inbuf=""; outbuf="";
     while True:
-        select.select([r], [], [])
-        s = os.read(r, 1024)
-        ret = ret + s
-        if len(s) == 0:
-            break
-        
+        sel = select.select([r, m1, m2], [m1, m2], [])
+        try: 
+            if m1 in sel[0]:
+                s = os.read(m1, 1)
+                inbuf += s
+            if m2 in sel[1] and len(inbuf) != 0:
+                os.write(m2, inbuf[0])
+                inbuf = inbuf[1:]
+        except OSError, e:
+            if e.errno == errno.EIO:
+                pass
+        try:
+            if m2 in sel[0]:
+                s = os.read(m2, 1)
+                outbuf += s
+            if m1 in sel[1] and len(outbuf) != 0:
+                os.write(m1, outbuf[0])
+                outbuf = outbuf[1:]
+        except OSError, e:
+            if e.errno == errno.EIO:
+                pass
+        if r in sel[0]:
+            s = os.read(r, 1)
+            ret = ret + s
+            if len(s) == 0:
+                break
+    del inbuf
+    del outbuf
     os.waitpid(child, 0)
     os.close(r)
+    os.close(m2)
+    os.close(m1)
     os.unlink(fifo)
 
+    # Re-acquire the lock to cover the changes we're about to make
+    # when we return to domain creation.
+    domains.domains_lock.acquire()    
+
+    if dom.bootloader_pid is None:
+        msg = "Domain was died while the bootloader was running."
+        log.error(msg)
+        raise VmError, msg        
+        
+    dom.bootloader_pid = None
+
     if len(ret) == 0:
         msg = "Boot loader didn't return any data!"
         log.error(msg)
@@ -103,3 +173,12 @@ def bootloader(blexec, disk, quiet = False, blargs = '', kernel = '',
     pin.input_eof()
     blcfg = pin.val
     return blcfg
+
+
+def bootloader_tidy(dom):
+    if hasattr(dom, "bootloader_pid") and dom.bootloader_pid is not None:
+        pid = dom.bootloader_pid
+        dom.bootloader_pid = None
+        os.kill(pid, signal.SIGKILL)
+
+

tools/python/xen/xend/XendDomain.py

@@ -115,7 +115,6 @@ class XendDomain:
                 
                 dom0info['name'] = DOM0_NAME
                 dom0 = XendDomainInfo.recreate(dom0info, True)
-                self._add_domain(dom0)
             except IndexError:
                 raise XendError('Unable to find Domain 0')
             
@@ -172,7 +171,6 @@ class XendDomain:
                 if dom['domid'] != DOM0_ID:
                     try:
                         new_dom = XendDomainInfo.recreate(dom, False)
-                        self._add_domain(new_dom)
                     except Exception:
                         log.exception("Failed to create reference to running "
                                       "domain id: %d" % dom['domid'])
@@ -397,7 +395,6 @@ class XendDomain:
             elif domid not in self.domains and dom['dying'] != 1:
                 try:
                     new_dom = XendDomainInfo.recreate(dom, False)
-                    self._add_domain(new_dom)                    
                 except VmError:
                     log.exception("Unable to recreate domain")
                     try:
@@ -416,10 +413,10 @@ class XendDomain:
         running_domids = [d['domid'] for d in running if d['dying'] != 1]
         for domid, dom in self.domains.items():
             if domid not in running_domids and domid != DOM0_ID:
-                self._remove_domain(dom, domid)
+                self.remove_domain(dom, domid)
 
 
-    def _add_domain(self, info):
+    def add_domain(self, info):
         """Add a domain to the list of running domains
         
         @requires: Expects to be protected by the domains_lock.
@@ -434,7 +431,7 @@ class XendDomain:
         if info.get_uuid() in self.managed_domains:
             self._managed_domain_register(info)
 
-    def _remove_domain(self, info, domid = None):
+    def remove_domain(self, info, domid = None):
         """Remove the domain from the list of running domains
         
         @requires: Expects to be protected by the domains_lock.
@@ -473,7 +470,6 @@ class XendDomain:
         try:
             security.refresh_ssidref(config)
             dominfo = XendDomainInfo.restore(config)
-            self._add_domain(dominfo)
             return dominfo
         finally:
             self.domains_lock.release()
@@ -848,7 +844,6 @@ class XendDomain:
                                            os.open(chkpath, os.O_RDONLY),
                                            dominfo,
                                            paused = start_paused)
-                    self._add_domain(dominfo)
                     os.unlink(chkpath)
                 except OSError, ex:
                     raise XendError("Failed to read stored checkpoint file")
@@ -873,7 +868,6 @@ class XendDomain:
             self._refresh()
 
             dominfo = XendDomainInfo.create(config)
-            self._add_domain(dominfo)
             self.domain_sched_credit_set(dominfo.getDomid(),
                                          dominfo.getWeight(),
                                          dominfo.getCap())
@@ -893,7 +887,6 @@ class XendDomain:
             self._refresh()
 
             dominfo = XendDomainInfo.create_from_dict(config_dict)
-            self._add_domain(dominfo)
             self.domain_sched_credit_set(dominfo.getDomid(),
                                          dominfo.getWeight(),
                                          dominfo.getCap())
@@ -950,7 +943,6 @@ class XendDomain:
                                  POWER_STATE_NAMES[dominfo.state])
             
             dominfo.start(is_managed = True)
-            self._add_domain(dominfo)
         finally:
             self.domains_lock.release()
         dominfo.waitForDevices()
@@ -983,7 +975,7 @@ class XendDomain:
                          (dominfo.getName(), dominfo.info.get('uuid')))
 
                 self._managed_domain_unregister(dominfo)
-                self._remove_domain(dominfo)
+                self.remove_domain(dominfo)
                 XendDevices.destroy_device_state(dominfo)
             except Exception, ex:
                 raise XendError(str(ex))

tools/python/xen/xend/XendDomainInfo.py

@@ -41,7 +41,7 @@ from xen.xend import balloon, sxp, uuid, image, arch, osdep
 from xen.xend import XendOptions, XendNode, XendConfig
 
 from xen.xend.XendConfig import scrub_password
-from xen.xend.XendBootloader import bootloader
+from xen.xend.XendBootloader import bootloader, bootloader_tidy
 from xen.xend.XendError import XendError, VmError
 from xen.xend.XendDevices import XendDevices
 from xen.xend.xenstore.xstransact import xstransact, complete
@@ -101,7 +101,6 @@ def create_from_dict(config_dict):
         log.exception('Domain construction failed')
         vm.destroy()
         raise
-
     return vm
 
 def recreate(info, priv):
@@ -187,6 +186,11 @@ def recreate(info, priv):
 
     vm._registerWatches()
     vm.refreshShutdown(xeninfo)
+
+    # register the domain in the list 
+    from xen.xend import XendDomain
+    XendDomain.instance().add_domain(vm)
+
     return vm
 
 
@@ -1338,6 +1342,9 @@ class XendDomainInfo:
         # Set maximum number of vcpus in domain
         xc.domain_max_vcpus(self.domid, int(self.info['vcpus_number']))
 
+        # register the domain in the list 
+        from xen.xend import XendDomain
+        XendDomain.instance().add_domain(self)
 
     def _introduceDomain(self):
         assert self.domid is not None
@@ -1436,6 +1443,7 @@ class XendDomainInfo:
         try:
             self.unwatchShutdown()
             self._releaseDevices()
+            bootloader_tidy(self)
 
             if self.image:
                 try:
@@ -1536,6 +1544,9 @@ class XendDomainInfo:
         except:
             log.exception("XendDomainInfo.destroy: xc.domain_destroy failed.")
 
+        from xen.xend import XendDomain
+        XendDomain.instance().remove_domain(self)
+
         self.cleanupDomain()
 
 
@@ -1631,7 +1642,7 @@ class XendDomainInfo:
                     fn = BOOTLOADER_LOOPBACK_DEVICE
 
                 try:
-                    blcfg = bootloader(blexec, fn, True,
+                    blcfg = bootloader(blexec, fn, self, False,
                                        bootloader_args, kernel, ramdisk, args)
                 finally:
                     if mounted:

tools/python/xen/xm/create.py

@@ -24,6 +24,7 @@ import os.path
 import sys
 import socket
 import re
+import time
 import xmlrpclib
 
 from xen.xend import sxp
@@ -709,26 +710,6 @@ def configure_hvm(config_image, vals):
             config_image.append([a, vals.__dict__[a]])
     config_image.append(['vncpasswd', vals.vncpasswd])
 
-def run_bootloader(vals, config_image):
-    if not os.access(vals.bootloader, os.F_OK):
-        err("Bootloader '%s' does not exist" % vals.bootloader)
-    if not os.access(vals.bootloader, os.X_OK):
-        err("Bootloader '%s' isn't executable" % vals.bootloader)
-    if len(vals.disk) < 1:
-        err("No disks configured and boot loader requested")
-    (uname, dev, mode, backend) = vals.disk[0]
-    file = blkif.blkdev_uname_to_file(uname)
-
-    if vals.bootentry:
-        warn("The bootentry option is deprecated.  Use bootargs and pass "
-             "--entry= directly.")
-        vals.bootargs = "--entry=%s" %(vals.bootentry,)
-
-    kernel = sxp.child_value(config_image, 'kernel')
-    ramdisk = sxp.child_value(config_image, 'ramdisk')
-    args = sxp.child_value(config_image, 'args')
-    return bootloader(vals.bootloader, file, not vals.console_autoconnect,
-                      vals.bootargs, kernel, ramdisk, args)
 
 def make_config(vals):
     """Create the domain configuration.
@@ -771,14 +752,11 @@ def make_config(vals):
         if vals.bootloader == "pygrub":
             vals.bootloader = osdep.pygrub_path
 
-        # if a kernel is specified, we're using the bootloader
-        # non-interactively, and need to let xend run it so we preserve the
-        # real kernel choice.
-        if not vals.kernel:
-            config_image = run_bootloader(vals, config_image)
         config.append(['bootloader', vals.bootloader])
         if vals.bootargs:
             config.append(['bootloader_args', vals.bootargs])
+        else: 
+            config.append(['bootloader_args', '-q'])        
     config.append(['image', config_image])
 
     config_devs = []
@@ -1266,9 +1244,28 @@ def main(argv):
         if not create_security_check(config):
             raise security.ACMError('Security Configuration prevents domain from starting')
         else:
-            dom = make_domain(opts, config)
             if opts.vals.console_autoconnect:
-                console.execConsole(dom)        
-             
+                cpid = os.fork() 
+                if cpid != 0:
+                    for i in range(10):
+                        # Catch failure of the create process 
+                        time.sleep(1)
+                        (p, rv) = os.waitpid(cpid, os.WNOHANG)
+                        if os.WIFEXITED(rv):
+                            if os.WEXITSTATUS(rv) != 0:
+                                sys.exit(os.WEXITSTATUS(rv))
+                        try:
+                            # Acquire the console of the created dom
+                            name = sxp.child_value(config, 'name', -1)
+                            dom = server.xend.domain(name)
+                            domid = int(sxp.child_value(dom, 'domid', '-1'))
+                            console.execConsole(domid)
+                        except:
+                            pass
+                    print("Could not start console\n");
+                    sys.exit(0)
+            dom = make_domain(opts, config)
+
+
 if __name__ == '__main__':
     main(sys.argv)