libxl: add force option for xl vcpu-pin

In order to be able to undo a vcpu pin override in case of a kernel
driver error add a flag "-f" to the "xl vcpu-pin" command forcing the
hypervisor to undo the override.

Cc: Ian Jackson <ian.jackson@eu.citrix.com>
Cc: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Cc: Wei Liu <wei.liu2@citrix.com>
Signed-off-by: Juergen Gross <jgross@suse.com>
Reviewed-by: Dario Faggioli <dario.faggioli@citrix.com>
Acked-by: Wei Liu <wei.liu2@citrix.com>

docs/man/xl.pod.1

@@ -689,7 +689,7 @@ after B<vcpu-set>, go to B<SEE ALSO> section for information.
 Lists VCPU information for a specific domain.  If no domain is
 specified, VCPU information for all domains will be provided.
 
-=item B<vcpu-pin> I<domain-id> I<vcpu> I<cpus hard> I<cpus soft>
+=item B<vcpu-pin> [I<-f|--force>] I<domain-id> I<vcpu> I<cpus hard> I<cpus soft>
 
 Set hard and soft affinity for a I<vcpu> of <domain-id>. Normally VCPUs
 can float between available CPUs whenever Xen deems a different run state
@@ -716,6 +716,11 @@ leaving its hard affinity untouched. On the othe hand:
 will set both hard and soft affinity, the former to pCPUs 3 and 4, the
 latter to pCPUs 6,7,8, and 9.
 
+Specifying I<-f> or I<--force> will remove a temporary pinning done by the
+operating system (normally this should be done by the operating system).
+In case a temporary pinning is active for a vcpu the affinity of this vcpu
+can't be changed without this option.
+
 =item B<vm-list>
 
 Prints information about guests. This list excludes information about

tools/libxl/libxl.c

@@ -5322,18 +5322,20 @@ err:
     return NULL;
 }
 
-int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid,
-                           const libxl_bitmap *cpumap_hard,
-                           const libxl_bitmap *cpumap_soft)
+static int libxl__set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid,
+                                   uint32_t vcpuid,
+                                   const libxl_bitmap *cpumap_hard,
+                                   const libxl_bitmap *cpumap_soft,
+                                   unsigned flags)
 {
     GC_INIT(ctx);
     libxl_bitmap hard, soft;
-    int rc, flags = 0;
+    int rc;
 
     libxl_bitmap_init(&hard);
     libxl_bitmap_init(&soft);
 
-    if (!cpumap_hard && !cpumap_soft) {
+    if (!cpumap_hard && !cpumap_soft && !flags) {
         rc = ERROR_INVAL;
         goto out;
     }
@@ -5348,7 +5350,7 @@ int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid,
             goto out;
 
         libxl__bitmap_copy_best_effort(gc, &hard, cpumap_hard);
-        flags = XEN_VCPUAFFINITY_HARD;
+        flags |= XEN_VCPUAFFINITY_HARD;
     }
     if (cpumap_soft) {
         rc = libxl_cpu_bitmap_alloc(ctx, &soft, 0);
@@ -5399,6 +5401,23 @@ int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid,
     return rc;
 }
 
+int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid,
+                           const libxl_bitmap *cpumap_hard,
+                           const libxl_bitmap *cpumap_soft)
+{
+    return libxl__set_vcpuaffinity(ctx, domid, vcpuid, cpumap_hard,
+                                   cpumap_soft, 0);
+}
+
+int libxl_set_vcpuaffinity_force(libxl_ctx *ctx, uint32_t domid,
+                                 uint32_t vcpuid,
+                                 const libxl_bitmap *cpumap_hard,
+                                 const libxl_bitmap *cpumap_soft)
+{
+    return libxl__set_vcpuaffinity(ctx, domid, vcpuid, cpumap_hard,
+                                   cpumap_soft, XEN_VCPUAFFINITY_FORCE);
+}
+
 int libxl_set_vcpuaffinity_all(libxl_ctx *ctx, uint32_t domid,
                                unsigned int max_vcpus,
                                const libxl_bitmap *cpumap_hard,

tools/libxl/libxl.h

@@ -153,6 +153,12 @@
  */
 #define LIBXL_HAVE_VCPUINFO_SOFT_AFFINITY 1
 
+/*
+ * LIBXL_HAVE_SET_VCPUAFFINITY_FORCE indicates that the
+ * libxl_set_vcpuaffinity_force() library call is available.
+ */
+#define LIBXL_HAVE_SET_VCPUAFFINITY_FORCE 1
+
 /*
  * LIBXL_HAVE_DEVICE_DISK_DIRECT_IO_SAFE indicates that a
  * 'direct_io_safe' field (of boolean type) is present in
@@ -1798,6 +1804,10 @@ int libxl_get_physinfo(libxl_ctx *ctx, libxl_physinfo *physinfo);
 int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid,
                            const libxl_bitmap *cpumap_hard,
                            const libxl_bitmap *cpumap_soft);
+int libxl_set_vcpuaffinity_force(libxl_ctx *ctx, uint32_t domid,
+                                 uint32_t vcpuid,
+                                 const libxl_bitmap *cpumap_hard,
+                                 const libxl_bitmap *cpumap_soft);
 int libxl_set_vcpuaffinity_all(libxl_ctx *ctx, uint32_t domid,
                                unsigned int max_vcpus,
                                const libxl_bitmap *cpumap_hard,

tools/libxl/xl_cmdimpl.c

@@ -5639,6 +5639,10 @@ int main_vcpulist(int argc, char **argv)
 
 int main_vcpupin(int argc, char **argv)
 {
+    static struct option opts[] = {
+        {"force", 0, 0, 'f'},
+        COMMON_LONG_OPTS
+    };
     libxl_vcpuinfo *vcpuinfo;
     libxl_bitmap cpumap_hard, cpumap_soft;;
     libxl_bitmap *soft = &cpumap_soft, *hard = &cpumap_hard;
@@ -5651,12 +5655,17 @@ int main_vcpupin(int argc, char **argv)
     const char *vcpu, *hard_str, *soft_str;
     char *endptr;
     int opt, nb_cpu, nb_vcpu, rc = EXIT_FAILURE;
+    bool force = false;
 
     libxl_bitmap_init(&cpumap_hard);
     libxl_bitmap_init(&cpumap_soft);
 
-    SWITCH_FOREACH_OPT(opt, "", NULL, "vcpu-pin", 3) {
-        /* No options */
+    SWITCH_FOREACH_OPT(opt, "f", opts, "vcpu-pin", 3) {
+    case 'f':
+        force = true;
+        break;
+    default:
+        break;
     }
 
     domid = find_domain(argv[optind]);
@@ -5671,6 +5680,10 @@ int main_vcpupin(int argc, char **argv)
             fprintf(stderr, "Error: Invalid argument %s as VCPU.\n", vcpu);
             goto out;
         }
+        if (force) {
+            fprintf(stderr, "Error: --force and 'all' as VCPU not allowed.\n");
+            goto out;
+        }
         vcpuid = -1;
     }
 
@@ -5732,7 +5745,14 @@ int main_vcpupin(int argc, char **argv)
         goto out;
     }
 
-    if (vcpuid != -1) {
+    if (force) {
+        if (libxl_set_vcpuaffinity_force(ctx, domid, vcpuid, hard, soft)) {
+            fprintf(stderr, "Could not set affinity for vcpu `%ld'.\n",
+                    vcpuid);
+            goto out;
+        }
+    }
+    else if (vcpuid != -1) {
         if (libxl_set_vcpuaffinity(ctx, domid, vcpuid, hard, soft)) {
             fprintf(stderr, "Could not set affinity for vcpu `%ld'.\n",
                     vcpuid);

tools/libxl/xl_cmdtable.c

@@ -222,7 +222,8 @@ struct cmd_spec cmd_table[] = {
     { "vcpu-pin",
       &main_vcpupin, 1, 1,
       "Set which CPUs a VCPU can use",
-      "<Domain> <VCPU|all> <Hard affinity|-|all> <Soft affinity|-|all>",
+      "[option] <Domain> <VCPU|all> <Hard affinity|-|all> <Soft affinity|-|all>",
+      "-f, --force        undo an override pinning done by the kernel",
     },
     { "vcpu-set",
       &main_vcpuset, 0, 1,