docs: document/mark-up SCHEDOP_*

The biggest subtlety here is there additional argument when op ==
SCHEDOP_shutdown and reason == SHUTDOWN_suspend and its interpretation by
xc_domain_{save,restore}. Add some clarifying comments to libxc as well.

This patch moves some structs around but there is no functional change
other than improved documentation.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Ian Jackson <ian.jackson@eu.citrix.com>
Committed-by: Ian Campbell <ian.campbell@citrix.com>

tools/libxc/xc_domain_restore.c

@@ -1967,8 +1967,14 @@ int xc_domain_restore(xc_interface *xch, int io_fd, uint32_t dom,
         if ( i == 0 )
         {
             /*
-             * Uncanonicalise the suspend-record frame number and poke
-             * resume record.
+             * Uncanonicalise the start info frame number and poke in
+             * updated values into the start info itself.
+             *
+             * The start info MFN is the 3rd argument to the
+             * HYPERVISOR_sched_op hypercall when op==SCHEDOP_shutdown
+             * and reason==SHUTDOWN_suspend, it is canonicalised in
+             * xc_domain_save and therefore the PFN is found in the
+             * edx register.
              */
             pfn = GET_FIELD(ctxt, user_regs.edx);
             if ( (pfn >= dinfo->p2m_size) ||

tools/libxc/xc_domain_save.c

@@ -1868,7 +1868,14 @@ int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iter
         goto out;
     }
 
-    /* Canonicalise the suspend-record frame number. */
+    /*
+     * Canonicalise the start info frame number.
+     *
+     * The start info MFN is the 3rd argument to the
+     * HYPERVISOR_sched_op hypercall when op==SCHEDOP_shutdown and
+     * reason==SHUTDOWN_suspend and is therefore found in the edx
+     * register.
+     */
     mfn = GET_FIELD(&ctxt, user_regs.edx);
     if ( !MFN_IS_IN_PSEUDOPHYS_MAP(mfn) )
     {

xen/include/public/sched.h

@@ -1,8 +1,8 @@
 /******************************************************************************
  * sched.h
- * 
+ *
  * Scheduler state interactions
- * 
+ *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to
  * deal in the Software without restriction, including without limitation the
@@ -29,21 +29,34 @@
 
 #include "event_channel.h"
 
+/*
+ * `incontents 150 sched Guest Scheduler Operations
+ *
+ * The SCHEDOP interface provides mechanisms for a guest to interact
+ * with the scheduler, including yield, blocking and shutting itself
+ * down.
+ */
+
 /*
  * The prototype for this hypercall is:
- *  long sched_op(int cmd, void *arg)
+ * ` long HYPERVISOR_sched_op(enum sched_op cmd, void *arg, ...)
+ *
  * @cmd == SCHEDOP_??? (scheduler operation).
  * @arg == Operation-specific extra argument(s), as described below.
- * 
+ * ...  == Additional Operation-specific extra arguments, described below.
+ *
  * Versions of Xen prior to 3.0.2 provided only the following legacy version
  * of this hypercall, supporting only the commands yield, block and shutdown:
  *  long sched_op(int cmd, unsigned long arg)
  * @cmd == SCHEDOP_??? (scheduler operation).
  * @arg == 0               (SCHEDOP_yield and SCHEDOP_block)
  *      == SHUTDOWN_* code (SCHEDOP_shutdown)
- * This legacy version is available to new guests as sched_op_compat().
+ *
+ * This legacy version is available to new guests as:
+ * ` long HYPERVISOR_sched_op_compat(enum sched_op cmd, unsigned long arg)
  */
 
+/* ` enum sched_op { // SCHEDOP_* => struct sched_* */
 /*
  * Voluntarily yield the CPU.
  * @arg == NULL.
@@ -61,21 +74,57 @@
 
 /*
  * Halt execution of this domain (all VCPUs) and notify the system controller.
- * @arg == pointer to sched_shutdown structure.
+ * @arg == pointer to sched_shutdown_t structure.
+ *
+ * If the sched_shutdown_t reason is SHUTDOWN_suspend then this
+ * hypercall takes an additional extra argument which should be the
+ * MFN of the guest's start_info_t.
+ *
+ * In addition, which reason is SHUTDOWN_suspend this hypercall
+ * returns 1 if suspend was cancelled or the domain was merely
+ * checkpointed, and 0 if it is resuming in a new domain.
  */
 #define SCHEDOP_shutdown    2
-struct sched_shutdown {
-    unsigned int reason; /* SHUTDOWN_* */
-};
-typedef struct sched_shutdown sched_shutdown_t;
-DEFINE_XEN_GUEST_HANDLE(sched_shutdown_t);
 
 /*
  * Poll a set of event-channel ports. Return when one or more are pending. An
  * optional timeout may be specified.
- * @arg == pointer to sched_poll structure.
+ * @arg == pointer to sched_poll_t structure.
  */
 #define SCHEDOP_poll        3
+
+/*
+ * Declare a shutdown for another domain. The main use of this function is
+ * in interpreting shutdown requests and reasons for fully-virtualized
+ * domains.  A para-virtualized domain may use SCHEDOP_shutdown directly.
+ * @arg == pointer to sched_remote_shutdown_t structure.
+ */
+#define SCHEDOP_remote_shutdown        4
+
+/*
+ * Latch a shutdown code, so that when the domain later shuts down it
+ * reports this code to the control tools.
+ * @arg == sched_shutdown_t, as for SCHEDOP_shutdown.
+ */
+#define SCHEDOP_shutdown_code 5
+
+/*
+ * Setup, poke and destroy a domain watchdog timer.
+ * @arg == pointer to sched_watchdog_t structure.
+ * With id == 0, setup a domain watchdog timer to cause domain shutdown
+ *               after timeout, returns watchdog id.
+ * With id != 0 and timeout == 0, destroy domain watchdog timer.
+ * With id != 0 and timeout != 0, poke watchdog timer and set new timeout.
+ */
+#define SCHEDOP_watchdog    6
+/* ` } */
+
+struct sched_shutdown {
+    unsigned int reason; /* SHUTDOWN_* => enum sched_shutdown_reason */
+};
+typedef struct sched_shutdown sched_shutdown_t;
+DEFINE_XEN_GUEST_HANDLE(sched_shutdown_t);
+
 struct sched_poll {
     XEN_GUEST_HANDLE(evtchn_port_t) ports;
     unsigned int nr_ports;
@@ -84,36 +133,13 @@ struct sched_poll {
 typedef struct sched_poll sched_poll_t;
 DEFINE_XEN_GUEST_HANDLE(sched_poll_t);
 
-/*
- * Declare a shutdown for another domain. The main use of this function is
- * in interpreting shutdown requests and reasons for fully-virtualized
- * domains.  A para-virtualized domain may use SCHEDOP_shutdown directly.
- * @arg == pointer to sched_remote_shutdown structure.
- */
-#define SCHEDOP_remote_shutdown        4
 struct sched_remote_shutdown {
     domid_t domain_id;         /* Remote domain ID */
-    unsigned int reason;       /* SHUTDOWN_xxx reason */
+    unsigned int reason;       /* SHUTDOWN_* => enum sched_shutdown_reason */
 };
 typedef struct sched_remote_shutdown sched_remote_shutdown_t;
 DEFINE_XEN_GUEST_HANDLE(sched_remote_shutdown_t);
 
-/*
- * Latch a shutdown code, so that when the domain later shuts down it
- * reports this code to the control tools.
- * @arg == as for SCHEDOP_shutdown.
- */
-#define SCHEDOP_shutdown_code 5
-
-/*
- * Setup, poke and destroy a domain watchdog timer.
- * @arg == pointer to sched_watchdog structure.
- * With id == 0, setup a domain watchdog timer to cause domain shutdown
- *               after timeout, returns watchdog id.
- * With id != 0 and timeout == 0, destroy domain watchdog timer.
- * With id != 0 and timeout != 0, poke watchdog timer and set new timeout.
- */
-#define SCHEDOP_watchdog    6
 struct sched_watchdog {
     uint32_t id;                /* watchdog ID */
     uint32_t timeout;           /* timeout */
@@ -126,11 +152,13 @@ DEFINE_XEN_GUEST_HANDLE(sched_watchdog_t);
  * software to determine the appropriate action. For the most part, Xen does
  * not care about the shutdown code.
  */
+/* ` enum sched_shutdown_reason { */
 #define SHUTDOWN_poweroff   0  /* Domain exited normally. Clean up and kill. */
 #define SHUTDOWN_reboot     1  /* Clean up, kill, and then restart.          */
 #define SHUTDOWN_suspend    2  /* Clean up, save suspend info, kill.         */
 #define SHUTDOWN_crash      3  /* Tell controller we've crashed.             */
 #define SHUTDOWN_watchdog   4  /* Restart because watchdog time expired.     */
+/* ` } */
 
 #endif /* __XEN_PUBLIC_SCHED_H__ */