Merge branch 'pin' of https://github.com/jgross1/xen into staging

* 'pin' of https://github.com/jgross1/xen:
  libxl: add force option for xl vcpu-pin
  libxl: print message how to recover from xl cpupool-cpu-remove errors
  libxc: do some retries in xc_cpupool_removecpu() for EBUSY case

All patches have Acked-by and Reviewed-by tags.

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>

docs/man/xl.pod.1

@@ -689,7 +689,7 @@ after B<vcpu-set>, go to B<SEE ALSO> section for information.
 Lists VCPU information for a specific domain.  If no domain is
 specified, VCPU information for all domains will be provided.
 
-=item B<vcpu-pin> I<domain-id> I<vcpu> I<cpus hard> I<cpus soft>
+=item B<vcpu-pin> [I<-f|--force>] I<domain-id> I<vcpu> I<cpus hard> I<cpus soft>
 
 Set hard and soft affinity for a I<vcpu> of <domain-id>. Normally VCPUs
 can float between available CPUs whenever Xen deems a different run state
@@ -716,6 +716,11 @@ leaving its hard affinity untouched. On the othe hand:
 will set both hard and soft affinity, the former to pCPUs 3 and 4, the
 latter to pCPUs 6,7,8, and 9.
 
+Specifying I<-f> or I<--force> will remove a temporary pinning done by the
+operating system (normally this should be done by the operating system).
+In case a temporary pinning is active for a vcpu the affinity of this vcpu
+can't be changed without this option.
+
 =item B<vm-list>
 
 Prints information about guests. This list excludes information about

tools/libxc/xc_cpupool.c

@@ -20,6 +20,7 @@
  */
 
 #include <stdarg.h>
+#include <unistd.h>
 #include "xc_private.h"
 
 static int do_sysctl_save(xc_interface *xch, struct xen_sysctl *sysctl)
@@ -137,17 +138,34 @@ int xc_cpupool_addcpu(xc_interface *xch,
     return do_sysctl_save(xch, &sysctl);
 }
 
+/*
+ * The hypervisor might return EBUSY when trying to remove a cpu from a
+ * cpupool when a domain running in this cpupool has pinned a vcpu
+ * temporarily. Do some retries in this case, perhaps the situation
+ * cleans up.
+ */
+#define NUM_RMCPU_BUSY_RETRIES 5
+
 int xc_cpupool_removecpu(xc_interface *xch,
                          uint32_t poolid,
                          int cpu)
 {
+    unsigned retries;
+    int err;
     DECLARE_SYSCTL;
 
     sysctl.cmd = XEN_SYSCTL_cpupool_op;
     sysctl.u.cpupool_op.op = XEN_SYSCTL_CPUPOOL_OP_RMCPU;
     sysctl.u.cpupool_op.cpupool_id = poolid;
     sysctl.u.cpupool_op.cpu = (cpu < 0) ? XEN_SYSCTL_CPUPOOL_PAR_ANY : cpu;
-    return do_sysctl_save(xch, &sysctl);
+    for ( retries = 0; retries < NUM_RMCPU_BUSY_RETRIES; retries++ ) {
+        err = do_sysctl_save(xch, &sysctl);
+        if ( err < 0 && errno == EBUSY )
+            sleep(1);
+        else
+            break;
+    }
+    return err;
 }
 
 int xc_cpupool_movedomain(xc_interface *xch,

tools/libxl/libxl.c

@@ -5317,18 +5317,20 @@ err:
     return NULL;
 }
 
-int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid,
+static int libxl__set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid,
-                           const libxl_bitmap *cpumap_hard,
+                                   uint32_t vcpuid,
-                           const libxl_bitmap *cpumap_soft)
+                                   const libxl_bitmap *cpumap_hard,
+                                   const libxl_bitmap *cpumap_soft,
+                                   unsigned flags)
 {
     GC_INIT(ctx);
     libxl_bitmap hard, soft;
-    int rc, flags = 0;
+    int rc;
 
     libxl_bitmap_init(&hard);
     libxl_bitmap_init(&soft);
 
-    if (!cpumap_hard && !cpumap_soft) {
+    if (!cpumap_hard && !cpumap_soft && !flags) {
         rc = ERROR_INVAL;
         goto out;
     }
@@ -5343,7 +5345,7 @@ int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid,
             goto out;
 
         libxl__bitmap_copy_best_effort(gc, &hard, cpumap_hard);
-        flags = XEN_VCPUAFFINITY_HARD;
+        flags |= XEN_VCPUAFFINITY_HARD;
     }
     if (cpumap_soft) {
         rc = libxl_cpu_bitmap_alloc(ctx, &soft, 0);
@@ -5394,6 +5396,23 @@ int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid,
     return rc;
 }
 
+int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid,
+                           const libxl_bitmap *cpumap_hard,
+                           const libxl_bitmap *cpumap_soft)
+{
+    return libxl__set_vcpuaffinity(ctx, domid, vcpuid, cpumap_hard,
+                                   cpumap_soft, 0);
+}
+
+int libxl_set_vcpuaffinity_force(libxl_ctx *ctx, uint32_t domid,
+                                 uint32_t vcpuid,
+                                 const libxl_bitmap *cpumap_hard,
+                                 const libxl_bitmap *cpumap_soft)
+{
+    return libxl__set_vcpuaffinity(ctx, domid, vcpuid, cpumap_hard,
+                                   cpumap_soft, XEN_VCPUAFFINITY_FORCE);
+}
+
 int libxl_set_vcpuaffinity_all(libxl_ctx *ctx, uint32_t domid,
                                unsigned int max_vcpus,
                                const libxl_bitmap *cpumap_hard,

tools/libxl/libxl.h

@@ -153,6 +153,12 @@
  */
 #define LIBXL_HAVE_VCPUINFO_SOFT_AFFINITY 1
 
+/*
+ * LIBXL_HAVE_SET_VCPUAFFINITY_FORCE indicates that the
+ * libxl_set_vcpuaffinity_force() library call is available.
+ */
+#define LIBXL_HAVE_SET_VCPUAFFINITY_FORCE 1
+
 /*
  * LIBXL_HAVE_DEVICE_DISK_DIRECT_IO_SAFE indicates that a
  * 'direct_io_safe' field (of boolean type) is present in
@@ -1798,6 +1804,10 @@ int libxl_get_physinfo(libxl_ctx *ctx, libxl_physinfo *physinfo);
 int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid,
                            const libxl_bitmap *cpumap_hard,
                            const libxl_bitmap *cpumap_soft);
+int libxl_set_vcpuaffinity_force(libxl_ctx *ctx, uint32_t domid,
+                                 uint32_t vcpuid,
+                                 const libxl_bitmap *cpumap_hard,
+                                 const libxl_bitmap *cpumap_soft);
 int libxl_set_vcpuaffinity_all(libxl_ctx *ctx, uint32_t domid,
                                unsigned int max_vcpus,
                                const libxl_bitmap *cpumap_hard,

tools/libxl/xl_cmdimpl.c

@@ -5641,6 +5641,10 @@ int main_vcpulist(int argc, char **argv)
 
 int main_vcpupin(int argc, char **argv)
 {
+    static struct option opts[] = {
+        {"force", 0, 0, 'f'},
+        COMMON_LONG_OPTS
+    };
     libxl_vcpuinfo *vcpuinfo;
     libxl_bitmap cpumap_hard, cpumap_soft;;
     libxl_bitmap *soft = &cpumap_soft, *hard = &cpumap_hard;
@@ -5653,12 +5657,17 @@ int main_vcpupin(int argc, char **argv)
     const char *vcpu, *hard_str, *soft_str;
     char *endptr;
     int opt, nb_cpu, nb_vcpu, rc = EXIT_FAILURE;
+    bool force = false;
 
     libxl_bitmap_init(&cpumap_hard);
     libxl_bitmap_init(&cpumap_soft);
 
-    SWITCH_FOREACH_OPT(opt, "", NULL, "vcpu-pin", 3) {
+    SWITCH_FOREACH_OPT(opt, "f", opts, "vcpu-pin", 3) {
-        /* No options */
+    case 'f':
+        force = true;
+        break;
+    default:
+        break;
     }
 
     domid = find_domain(argv[optind]);
@@ -5673,6 +5682,10 @@ int main_vcpupin(int argc, char **argv)
             fprintf(stderr, "Error: Invalid argument %s as VCPU.\n", vcpu);
             goto out;
         }
+        if (force) {
+            fprintf(stderr, "Error: --force and 'all' as VCPU not allowed.\n");
+            goto out;
+        }
         vcpuid = -1;
     }
 
@@ -5734,7 +5747,14 @@ int main_vcpupin(int argc, char **argv)
         goto out;
     }
 
-    if (vcpuid != -1) {
+    if (force) {
+        if (libxl_set_vcpuaffinity_force(ctx, domid, vcpuid, hard, soft)) {
+            fprintf(stderr, "Could not set affinity for vcpu `%ld'.\n",
+                    vcpuid);
+            goto out;
+        }
+    }
+    else if (vcpuid != -1) {
         if (libxl_set_vcpuaffinity(ctx, domid, vcpuid, hard, soft)) {
             fprintf(stderr, "Could not set affinity for vcpu `%ld'.\n",
                     vcpuid);
@@ -8013,8 +8033,10 @@ int main_cpupoolcpuremove(int argc, char **argv)
         goto out;
     }
 
-    if (libxl_cpupool_cpuremove_cpumap(ctx, poolid, &cpumap))
+    if (libxl_cpupool_cpuremove_cpumap(ctx, poolid, &cpumap)) {
-        fprintf(stderr, "some cpus may not have been removed from %s\n", pool);
+        fprintf(stderr, "Some cpus may have not or only partially been removed from '%s'.\n", pool);
+        fprintf(stderr, "If a cpu can't be added to another cpupool, add it to '%s' again and retry.\n", pool);
+    }
 
     rc = EXIT_SUCCESS;
 

tools/libxl/xl_cmdtable.c

@@ -222,7 +222,8 @@ struct cmd_spec cmd_table[] = {
     { "vcpu-pin",
       &main_vcpupin, 1, 1,
       "Set which CPUs a VCPU can use",
-      "<Domain> <VCPU|all> <Hard affinity|-|all> <Soft affinity|-|all>",
+      "[option] <Domain> <VCPU|all> <Hard affinity|-|all> <Soft affinity|-|all>",
+      "-f, --force        undo an override pinning done by the kernel",
     },
     { "vcpu-set",
       &main_vcpuset, 0, 1,