tools/libxl: Write checkpoint records into the stream

when signalled to do so by libxl__remus_domain_checkpoint_callback()

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Acked-by: Ian Campbell <Ian.Campbell@citrix.com>
CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
CC: Wei Liu <wei.liu2@citrix.com>

---
v4: Adjustments in line with review for earlier patches in the series
v3: Corrections to comments

This patch has changed substantially in v2 as a result of changes earlier in
the series.  No behavioural difference from v1.

tools/libxl/libxl_dom.c

@@ -1937,8 +1937,8 @@ out:
 
 /*----- remus asynchronous checkpoint callback -----*/
 
-static void remus_checkpoint_dm_saved(libxl__egc *egc,
-                                      libxl__domain_suspend_state *dss, int rc);
+static void remus_checkpoint_stream_written(
+    libxl__egc *egc, libxl__stream_write_state *sws, int rc);
 static void remus_devices_commit_cb(libxl__egc *egc,
                                     libxl__remus_devices_state *rds,
                                     int rc);
@@ -1953,17 +1953,14 @@ static void libxl__remus_domain_checkpoint_callback(void *data)
     libxl__egc *egc = shs->egc;
     STATE_AO_GC(dss->ao);
 
-    /* This would go into tailbuf. */
-    if (dss->hvm) {
-        libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved);
-    } else {
-        remus_checkpoint_dm_saved(egc, dss, 0);
-    }
+    libxl__stream_write_start_checkpoint(egc, &dss->sws);
 }
 
-static void remus_checkpoint_dm_saved(libxl__egc *egc,
-                                      libxl__domain_suspend_state *dss, int rc)
+static void remus_checkpoint_stream_written(
+    libxl__egc *egc, libxl__stream_write_state *sws, int rc)
 {
+    libxl__domain_suspend_state *dss = CONTAINER_OF(sws, *dss, sws);
+
     /* Convenience aliases */
     libxl__remus_devices_state *const rds = &dss->rds;
 
@@ -2113,6 +2110,7 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
         callbacks->suspend = libxl__remus_domain_suspend_callback;
         callbacks->postcopy = libxl__remus_domain_resume_callback;
         callbacks->checkpoint = libxl__remus_domain_checkpoint_callback;
+        dss->sws.checkpoint_callback = remus_checkpoint_stream_written;
     } else
         callbacks->suspend = libxl__domain_suspend_callback;
 

tools/libxl/libxl_internal.h

@@ -2968,9 +2968,13 @@ struct libxl__stream_write_state {
     void (*completion_callback)(libxl__egc *egc,
                                 libxl__stream_write_state *sws,
                                 int rc);
+    void (*checkpoint_callback)(libxl__egc *egc,
+                                libxl__stream_write_state *sws,
+                                int rc);
     /* Private */
     int rc;
     bool running;
+    bool in_checkpoint;
     libxl__save_helper_state shs;
 
     /* Main stream-writing data. */
@@ -2987,6 +2991,9 @@ struct libxl__stream_write_state {
 _hidden void libxl__stream_write_init(libxl__stream_write_state *stream);
 _hidden void libxl__stream_write_start(libxl__egc *egc,
                                        libxl__stream_write_state *stream);
+_hidden void
+libxl__stream_write_start_checkpoint(libxl__egc *egc,
+                                     libxl__stream_write_state *stream);
 _hidden void libxl__stream_write_abort(libxl__egc *egc,
                                        libxl__stream_write_state *stream,
                                        int rc);

tools/libxl/libxl_stream_write.c

@@ -22,6 +22,8 @@
  * Entry points from outside:
  *  - libxl__stream_write_start()
  *     - Start writing a stream from the start.
+ *  - libxl__stream_write_start_checkpoint()
+ *     - Write the records which form a checkpoint into a stream.
  *
  * In normal operation, there are two tasks running at once; this
  * stream processing, and the libxl-save-helper.  check_all_finished()
@@ -39,6 +41,12 @@
  *  - Toolstack record
  *  - if (hvm), Qemu record
  *  - End record
+ *
+ * For checkpointed stream, there is a second loop which is triggered by a
+ * save-helper checkpoint callback.  It writes:
+ *  - Toolstack record
+ *  - if (hvm), Qemu record
+ *  - Checkpoint end record
  */
 
 /* Success/error/cleanup handling. */
@@ -48,6 +56,9 @@ static void stream_complete(libxl__egc *egc,
                             libxl__stream_write_state *stream, int rc);
 static void stream_done(libxl__egc *egc,
                         libxl__stream_write_state *stream);
+static void checkpoint_done(libxl__egc *egc,
+                            libxl__stream_write_state *stream,
+                            int rc);
 static void check_all_finished(libxl__egc *egc,
                                libxl__stream_write_state *stream, int rc);
 
@@ -72,6 +83,12 @@ static void emulator_record_done(libxl__egc *egc,
 static void write_end_record(libxl__egc *egc,
                              libxl__stream_write_state *stream);
 
+/* Event chain unique to checkpointed streams. */
+static void write_checkpoint_end_record(libxl__egc *egc,
+                                        libxl__stream_write_state *stream);
+static void checkpoint_end_record_done(libxl__egc *egc,
+                                       libxl__stream_write_state *stream);
+
 /*----- Helpers -----*/
 
 static void write_done(libxl__egc *egc,
@@ -140,6 +157,7 @@ void libxl__stream_write_init(libxl__stream_write_state *stream)
 {
     stream->rc = 0;
     stream->running = false;
+    stream->in_checkpoint = false;
     FILLZERO(stream->dc);
     stream->record_done_callback = NULL;
     FILLZERO(stream->emu_dc);
@@ -184,6 +202,16 @@ void libxl__stream_write_start(libxl__egc *egc,
     stream_complete(egc, stream, rc);
 }
 
+void libxl__stream_write_start_checkpoint(libxl__egc *egc,
+                                          libxl__stream_write_state *stream)
+{
+    assert(stream->running);
+    assert(!stream->in_checkpoint);
+    stream->in_checkpoint = true;
+
+    write_toolstack_record(egc, stream);
+}
+
 void libxl__stream_write_abort(libxl__egc *egc,
                                libxl__stream_write_state *stream, int rc)
 {
@@ -290,8 +318,12 @@ static void toolstack_record_done(libxl__egc *egc,
 
     if (dss->type == LIBXL_DOMAIN_TYPE_HVM)
         write_emulator_record(egc, stream);
-    else
-        write_end_record(egc, stream);
+    else {
+        if (stream->in_checkpoint)
+            write_checkpoint_end_record(egc, stream);
+        else
+            write_end_record(egc, stream);
+    }
 }
 
 static void write_emulator_record(libxl__egc *egc,
@@ -397,7 +429,10 @@ static void emulator_record_done(libxl__egc *egc,
     free(stream->emu_body);
     stream->emu_body = NULL;
 
-    write_end_record(egc, stream);
+    if (stream->in_checkpoint)
+        write_checkpoint_end_record(egc, stream);
+    else
+        write_end_record(egc, stream);
 }
 
 static void write_end_record(libxl__egc *egc,
@@ -412,6 +447,24 @@ static void write_end_record(libxl__egc *egc,
                 &rec, NULL, stream_success);
 }
 
+static void write_checkpoint_end_record(libxl__egc *egc,
+                                        libxl__stream_write_state *stream)
+{
+    struct libxl__sr_rec_hdr rec;
+
+    FILLZERO(rec);
+    rec.type = REC_TYPE_CHECKPOINT_END;
+
+    setup_write(egc, stream, "checkpoint end record",
+                &rec, NULL, checkpoint_end_record_done);
+}
+
+static void checkpoint_end_record_done(libxl__egc *egc,
+                                       libxl__stream_write_state *stream)
+{
+    checkpoint_done(egc, stream, 0);
+}
+
 /*----- Success/error/cleanup handling. -----*/
 
 static void stream_success(libxl__egc *egc, libxl__stream_write_state *stream)
@@ -424,6 +477,18 @@ static void stream_complete(libxl__egc *egc,
 {
     assert(stream->running);
 
+    if (stream->in_checkpoint) {
+        assert(rc);
+
+        /*
+         * If an error is encountered while in a checkpoint, pass it
+         * back to libxc.  The failure will come back around to us via
+         * libxl__xc_domain_save_done()
+         */
+        checkpoint_done(egc, stream, rc);
+        return;
+    }
+
     if (!stream->rc)
         stream->rc = rc;
     stream_done(egc, stream);
@@ -442,6 +507,16 @@ static void stream_done(libxl__egc *egc,
     check_all_finished(egc, stream, stream->rc);
 }
 
+static void checkpoint_done(libxl__egc *egc,
+                            libxl__stream_write_state *stream,
+                            int rc)
+{
+    assert(stream->in_checkpoint);
+
+    stream->in_checkpoint = false;
+    stream->checkpoint_callback(egc, stream, rc);
+}
+
 static void check_all_finished(libxl__egc *egc,
                                libxl__stream_write_state *stream,
                                int rc)