libxl: Asynchronous/long-running operation infrastructure

Provide a new set of machinery for writing public libxl functions
which may take a long time.  The application gets to decide whether
they want the function to be synchronous, or whether they'd prefer to
get a callback, or an event, when the operation is complete.

User(s) of this machinery will be introduced in later patch(es).

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
Committed-by: Ian Jackson <Ian.Jackson@eu.citrix.com>

tools/libxl/libxl.h

@@ -237,6 +237,59 @@ enum {
     ERROR_BUFFERFULL = -13,
 };
 
+
+/*
+ * Some libxl operations can take a long time.  These functions take a
+ * parameter to control their concurrency:
+ *     libxl_asyncop_how *ao_how
+ *
+ * If ao_how==NULL, the function will be synchronous.
+ *
+ * If ao_how!=NULL, the function will set the operation going, and if
+ * this is successful will return 0.  In this case the zero error
+ * response does NOT mean that the operation was successful; it just
+ * means that it has been successfully started.  It will finish later,
+ * perhaps with an error.
+ *
+ * If ao_how->callback!=NULL, the callback will be called when the
+ * operation completes.  The same rules as for libxl_event_hooks
+ * apply, including the reentrancy rules and the possibility of
+ * "disaster", except that libxl calls ao_how->callback instead of
+ * libxl_event_hooks.event_occurs.  (See libxl_event.h.)
+ *
+ * If ao_how->callback==NULL, a libxl_event will be generated which
+ * can be obtained from libxl_event_wait or libxl_event_check.  The
+ * event will have type OPERATION_COMPLETE (which is not used
+ * elsewhere).
+ *
+ * Note that it is possible for an asynchronous operation which is to
+ * result in a callback to complete during its initiating function
+ * call.  In this case the initiating function will return 0
+ * indicating the at the operation is "in progress", even though by
+ * the time it returns the operation is complete and the callback has
+ * already happened.
+ *
+ * The application must set and use ao_how->for_event (which will be
+ * copied into libxl_event.for_user) or ao_how->for_callback (passed
+ * to the callback) to determine which operation finished, and it must
+ * of course check the rc value for errors.
+ *
+ * *ao_how does not need to remain valid after the initiating function
+ * returns.
+ *
+ * Callbacks may occur on any thread in which the application calls
+ * libxl.
+ */
+
+typedef struct {
+    void (*callback)(libxl_ctx *ctx, int rc, void *for_callback);
+    union {
+        libxl_ev_user for_event; /* used if callback==NULL */
+        void *for_callback; /* passed to callback */
+    } u;
+} libxl_asyncop_how;
+
+
 #define LIBXL_VERSION 0
 
 typedef struct {

tools/libxl/libxl_event.c

@@ -771,10 +771,21 @@ static void egc_run_callbacks(libxl__egc *egc)
 {
     EGC_GC;
     libxl_event *ev, *ev_tmp;
+
     LIBXL_TAILQ_FOREACH_SAFE(ev, &egc->occurred_for_callback, link, ev_tmp) {
         LIBXL_TAILQ_REMOVE(&egc->occurred_for_callback, ev, link);
         CTX->event_hooks->event_occurs(CTX->event_hooks_user, ev);
     }
+
+    libxl__ao *ao, *ao_tmp;
+    LIBXL_TAILQ_FOREACH_SAFE(ao, &egc->aos_for_callback,
+                             entry_for_callback, ao_tmp) {
+        LIBXL_TAILQ_REMOVE(&egc->aos_for_callback, ao, entry_for_callback);
+        ao->how.callback(CTX, ao->rc, ao->how.u.for_callback);
+        ao->notified = 1;
+        if (!ao->in_initiator)
+            libxl__ao__destroy(CTX, ao);
+    }
 }
 
 void libxl__egc_cleanup(libxl__egc *egc)
@@ -1061,6 +1072,183 @@ int libxl_event_wait(libxl_ctx *ctx, libxl_event **event_r,
     return rc;
 }
 
+
+
+/*
+ * The two possible state flow of an ao:
+ *
+ * Completion before initiator return:
+ *
+ *     Initiator thread                       Possible other threads
+ *
+ *   * ao_create allocates memory and
+ *     initialises the struct
+ *
+ *   * the initiator function does its
+ *     work, setting up various internal
+ *     asynchronous operations -----------> * asynchronous operations
+ *                                            start to take place and
+ *                                            might cause ao completion
+ *                                                |
+ *   * initiator calls ao_inprogress              |
+ *     - if synchronous, run event loop           |
+ *       until the ao completes                   |
+ *                              - ao completes on some thread
+ *                              - completing thread releases the lock
+ *                     <--------------'
+ *     - ao_inprogress takes the lock
+ *     - destroy the ao
+ *
+ *
+ * Completion after initiator return (asynch. only):
+ *
+ *
+ *     Initiator thread                       Possible other threads
+ *
+ *   * ao_create allocates memory and
+ *     initialises the struct
+ *
+ *   * the initiator function does its
+ *     work, setting up various internal
+ *     asynchronous operations -----------> * asynchronous operations
+ *                                            start to take place and
+ *                                            might cause ao completion
+ *                                                |
+ *   * initiator calls ao_inprogress              |
+ *     - observes event not yet done,             |
+ *     - returns to caller                        |
+ *                                                |
+ *                              - ao completes on some thread
+ *                              - generate the event or call the callback
+ *                              - destroy the ao
+ */
+
+void libxl__ao__destroy(libxl_ctx *ctx, libxl__ao *ao)
+{
+    if (!ao) return;
+    if (ao->poller) libxl__poller_put(ctx, ao->poller);
+    ao->magic = LIBXL__AO_MAGIC_DESTROYED;
+    libxl__free_all(&ao->gc);
+    free(ao);
+}
+
+void libxl__ao_abort(libxl__ao *ao)
+{
+    AO_GC;
+    assert(ao->magic == LIBXL__AO_MAGIC);
+    assert(ao->in_initiator);
+    assert(!ao->complete);
+    libxl__ao__destroy(CTX, ao);
+}
+
+void libxl__ao_complete(libxl__egc *egc, libxl__ao *ao, int rc)
+{
+    assert(ao->magic == LIBXL__AO_MAGIC);
+    assert(!ao->complete);
+    ao->complete = 1;
+    ao->rc = rc;
+
+    if (ao->poller) {
+        assert(ao->in_initiator);
+        libxl__poller_wakeup(egc, ao->poller);
+    } else if (ao->how.callback) {
+        LIBXL_TAILQ_INSERT_TAIL(&egc->aos_for_callback, ao, entry_for_callback);
+    } else {
+        libxl_event *ev;
+        ev = NEW_EVENT(egc, OPERATION_COMPLETE, ao->domid);
+        if (ev) {
+            ev->for_user = ao->how.u.for_event;
+            ev->u.operation_complete.rc = ao->rc;
+            libxl__event_occurred(egc, ev);
+        }
+        ao->notified = 1;
+    }
+    if (!ao->in_initiator && ao->notified)
+        libxl__ao__destroy(libxl__gc_owner(&egc->gc), ao);
+}
+
+libxl__ao *libxl__ao_create(libxl_ctx *ctx, uint32_t domid,
+                            const libxl_asyncop_how *how)
+{
+    libxl__ao *ao;
+
+    ao = calloc(1, sizeof(*ao));
+    if (!ao) goto out;
+
+    ao->magic = LIBXL__AO_MAGIC;
+    ao->in_initiator = 1;
+    ao->poller = 0;
+    ao->domid = domid;
+    LIBXL_INIT_GC(ao->gc, ctx);
+
+    if (how) {
+        ao->how = *how;
+    } else {
+        ao->poller = libxl__poller_get(ctx);
+        if (!ao->poller) goto out;
+    }
+    return ao;
+
+ out:
+    if (ao) libxl__ao__destroy(ctx, ao);
+    return NULL;
+}
+
+int libxl__ao_inprogress(libxl__ao *ao)
+{
+    AO_GC;
+    int rc;
+
+    assert(ao->magic == LIBXL__AO_MAGIC);
+    assert(ao->in_initiator);
+
+    if (ao->poller) {
+        /* Caller wants it done synchronously. */
+        /* We use a fresh gc, so that we can free things
+         * each time round the loop. */
+        libxl__egc egc;
+        LIBXL_INIT_EGC(egc,CTX);
+
+        for (;;) {
+            assert(ao->magic == LIBXL__AO_MAGIC);
+
+            if (ao->complete) {
+                rc = ao->rc;
+                ao->notified = 1;
+                break;
+            }
+
+            rc = eventloop_iteration(&egc,ao->poller);
+            if (rc) {
+                /* Oh dear, this is quite unfortunate. */
+                LIBXL__LOG(CTX, LIBXL__LOG_ERROR, "Error waiting for"
+                           " event during long-running operation (rc=%d)", rc);
+                sleep(1);
+                /* It's either this or return ERROR_I_DONT_KNOW_WHETHER
+                 * _THE_THING_YOU_ASKED_FOR_WILL_BE_DONE_LATER_WHEN
+                 * _YOU_DIDNT_EXPECT_IT, since we don't have any kind of
+                 * cancellation ability. */
+            }
+
+            CTX_UNLOCK;
+            libxl__egc_cleanup(&egc);
+            CTX_LOCK;
+        }
+    } else {
+        rc = 0;
+    }
+
+    ao->in_initiator = 0;
+
+    if (ao->notified) {
+        assert(ao->complete);
+        libxl__ao__destroy(CTX,ao);
+    }
+
+    return rc;
+}
+
+
 /*
  * Local variables:
  * mode: C

tools/libxl/libxl_internal.h

@@ -114,6 +114,7 @@ _hidden void libxl__log(libxl_ctx *ctx, xentoollog_level msglevel, int errnoval,
 
 typedef struct libxl__gc libxl__gc;
 typedef struct libxl__egc libxl__egc;
+typedef struct libxl__ao libxl__ao;
 
 typedef struct libxl__ev_fd libxl__ev_fd;
 typedef void libxl__ev_fd_callback(libxl__egc *egc, libxl__ev_fd *ev,
@@ -218,6 +219,10 @@ struct libxl__poller {
      * releasing the ctx lock and going into poll; when it comes out
      * of poll it will take the poller off the pollers_event list.
      *
+     * A thread which is waiting for completion of a synchronous ao
+     * will allocate a poller and record it in the ao, so that other
+     * threads can wake it up.
+     *
      * When a thread is done with a poller it should put it onto
      * pollers_idle, where it can be reused later.
      *
@@ -324,6 +329,21 @@ struct libxl__egc {
     /* for event-generating functions only */
     struct libxl__gc gc;
     struct libxl__event_list occurred_for_callback;
+    LIBXL_TAILQ_HEAD(, libxl__ao) aos_for_callback;
+};
+
+#define LIBXL__AO_MAGIC              0xA0FACE00ul
+#define LIBXL__AO_MAGIC_DESTROYED    0xA0DEAD00ul
+
+struct libxl__ao {
+    uint32_t magic;
+    unsigned in_initiator:1, complete:1, notified:1;
+    int rc;
+    libxl__gc gc;
+    libxl_asyncop_how how;
+    libxl__poller *poller;
+    uint32_t domid;
+    LIBXL_TAILQ_ENTRY(libxl__ao) entry_for_callback;
 };
 
 #define LIBXL_INIT_GC(gc,ctx) do{               \
@@ -1108,6 +1128,7 @@ libxl__device_model_version_running(libxl__gc *gc, uint32_t domid);
 #define LIBXL_INIT_EGC(egc,ctx) do{                     \
         LIBXL_INIT_GC((egc).gc,ctx);                    \
         LIBXL_TAILQ_INIT(&(egc).occurred_for_callback); \
+        LIBXL_TAILQ_INIT(&(egc).aos_for_callback);      \
     } while(0)
 
 _hidden void libxl__egc_cleanup(libxl__egc *egc);
@@ -1124,6 +1145,92 @@ _hidden void libxl__egc_cleanup(libxl__egc *egc);
 #define EGC_FREE           libxl__egc_cleanup(egc)
 
 
+/*
+ * Machinery for asynchronous operations ("ao")
+ *
+ * All "slow" functions (includes anything that might block on a
+ * guest or an external script) need to use the asynchronous
+ * operation ("ao") machinery.  The function should take a parameter
+ * const libxl_asyncop_how *ao_how and must start with a call to
+ * AO_INITIATOR_ENTRY.  These functions MAY NOT be called from
+ * outside libxl, because they can cause reentrancy callbacks.
+ *
+ * Lifecycle of an ao:
+ *
+ * - Created by libxl__ao_create (or the AO_CREATE convenience macro).
+ *
+ * - After creation, can be used by code which implements
+ *   the operation as follows:
+ *      - the ao's gc, for allocating memory for the lifetime
+ *        of the operation (possibly with the help of the AO_GC
+ *        macro to introduce the gc into scope)
+ *      - the ao itself may be passed about to sub-functions
+ *        so that they can stash it away etc.
+ *      - in particular, the ao pointer must be stashed in some
+ *        per-operation structure which is also passed as a user
+ *        pointer to the internal event generation request routines
+ *        libxl__evgen_FOO, so that at some point a CALLBACK will be
+ *        made when the operation is complete.
+ *
+ * - If initiation is successful, the initiating function needs
+ *   to run libxl__ao_inprogress right before unlocking and
+ *   returning, and return whatever it returns (AO_INPROGRESS macro).
+ *
+ * - If the initiation is unsuccessful, the initiating function must
+ *   call libxl__ao_abort before unlocking and returning whatever
+ *   error code is appropriate (AO_ABORT macro).
+ *
+ * - Later, some callback function, whose callback has been requested
+ *   directly or indirectly, should call libxl__ao_complete (with the
+ *   ctx locked, as it will generally already be in any event callback
+ *   function).  This must happen exactly once for each ao (and not if
+ *   the ao has been destroyed, obviously), and it may not happen
+ *   until libxl__ao_inprogress has been called on the ao.
+ *
+ * - Note that during callback functions, two gcs are available:
+ *    - The one in egc, whose lifetime is only this callback
+ *    - The one in ao, whose lifetime is the asynchronous operation
+ *   Usually callback function should use CONTAINER_OF
+ *   to obtain its own structure, containing a pointer to the ao,
+ *   and then use the gc from that ao.
+ */
+
+#define AO_CREATE(ctx, domid, ao_how)                           \
+    libxl__ctx_lock(ctx);                                       \
+    libxl__ao *ao = libxl__ao_create(ctx, domid, ao_how);       \
+    if (!ao) { libxl__ctx_unlock(ctx); return ERROR_NOMEM; }    \
+    AO_GC;
+
+#define AO_INPROGRESS ({                                        \
+        libxl_ctx *ao__ctx = libxl__gc_owner(&ao->gc);          \
+        int ao__rc = libxl__ao_inprogress(ao);                  \
+        libxl__ctx_unlock(ao__ctx); /* gc is now invalid */     \
+        (ao__rc);                                               \
+   })
+
+#define AO_ABORT(rc) ({                                         \
+        libxl_ctx *ao__ctx = libxl__gc_owner(&ao->gc);          \
+        assert(rc);                                             \
+        libxl__ao_abort(ao);                                    \
+        libxl__ctx_unlock(ao__ctx); /* gc is now invalid */     \
+        (rc);                                                   \
+    })
+
+#define AO_GC                                   \
+    libxl__gc *const gc = &ao->gc
+
+
+/* All of these MUST be called with the ctx locked.
+ * libxl__ao_inprogress MUST be called with the ctx locked exactly once. */
+_hidden libxl__ao *libxl__ao_create(libxl_ctx*, uint32_t domid,
+                                    const libxl_asyncop_how*);
+_hidden int libxl__ao_inprogress(libxl__ao *ao);
+_hidden void libxl__ao_abort(libxl__ao *ao);
+_hidden void libxl__ao_complete(libxl__egc *egc, libxl__ao *ao, int rc);
+
+/* For use by ao machinery ONLY */
+_hidden void libxl__ao__destroy(libxl_ctx*, libxl__ao *ao);
+
 /*
  * Convenience macros.
  */

tools/libxl/libxl_types.idl

@@ -397,6 +397,7 @@ libxl_event_type = Enumeration("event_type", [
     (1, "DOMAIN_SHUTDOWN"),
     (2, "DOMAIN_DESTROY"),
     (3, "DISK_EJECT"),
+    (4, "OPERATION_COMPLETE"),
     ])
 
 libxl_ev_user = UInt(64)
@@ -420,4 +421,7 @@ libxl_event = Struct("event",[
                                         ("vdev", string),
                                         ("disk", libxl_device_disk),
                                  ])),
+           ("operation_complete", Struct(None, [
+                                        ("rc", integer),
+                                 ])),
            ]))])