From 1ef6beea187bca8d11152b6c7d987b2b9450f936 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Thu, 24 Mar 2016 18:44:50 +0100 Subject: [PATCH 1/3] libxc: do some retries in xc_cpupool_removecpu() for EBUSY case The hypervisor might return EBUSY when trying to remove a cpu from a cpupool when a domain running in this cpupool has pinned a vcpu temporarily. Do some retries in this case, perhaps the situation cleans up. Cc: Ian Jackson Cc: Stefano Stabellini Cc: Wei Liu Signed-off-by: Juergen Gross Reviewed-by: Dario Faggioli Acked-by: Wei Liu --- tools/libxc/xc_cpupool.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/libxc/xc_cpupool.c b/tools/libxc/xc_cpupool.c index c42273e541..261b9c956e 100644 --- a/tools/libxc/xc_cpupool.c +++ b/tools/libxc/xc_cpupool.c @@ -20,6 +20,7 @@ */ #include +#include #include "xc_private.h" static int do_sysctl_save(xc_interface *xch, struct xen_sysctl *sysctl) @@ -137,17 +138,34 @@ int xc_cpupool_addcpu(xc_interface *xch, return do_sysctl_save(xch, &sysctl); } +/* + * The hypervisor might return EBUSY when trying to remove a cpu from a + * cpupool when a domain running in this cpupool has pinned a vcpu + * temporarily. Do some retries in this case, perhaps the situation + * cleans up. + */ +#define NUM_RMCPU_BUSY_RETRIES 5 + int xc_cpupool_removecpu(xc_interface *xch, uint32_t poolid, int cpu) { + unsigned retries; + int err; DECLARE_SYSCTL; sysctl.cmd = XEN_SYSCTL_cpupool_op; sysctl.u.cpupool_op.op = XEN_SYSCTL_CPUPOOL_OP_RMCPU; sysctl.u.cpupool_op.cpupool_id = poolid; sysctl.u.cpupool_op.cpu = (cpu < 0) ? XEN_SYSCTL_CPUPOOL_PAR_ANY : cpu; - return do_sysctl_save(xch, &sysctl); + for ( retries = 0; retries < NUM_RMCPU_BUSY_RETRIES; retries++ ) { + err = do_sysctl_save(xch, &sysctl); + if ( err < 0 && errno == EBUSY ) + sleep(1); + else + break; + } + return err; } int xc_cpupool_movedomain(xc_interface *xch, From 4e293401f75aeba23410c5a3761761311ffe7ee9 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Thu, 24 Mar 2016 18:44:50 +0100 Subject: [PATCH 2/3] libxl: print message how to recover from xl cpupool-cpu-remove errors An error occurring when calling "xl cpupool-cpu-remove" might leave the system in a state where a cpu is neither completely free nor in a cpupool. This can easily be repaired by adding the cpu via "xl cpupool-cpu-add" to the cpupool where it was removed from before. Print a message telling this the user in case of an error. Cc: Ian Jackson Cc: Stefano Stabellini Cc: Wei Liu Signed-off-by: Juergen Gross Reviewed-by: Dario Faggioli Acked-by: Wei Liu --- tools/libxl/xl_cmdimpl.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c index a3610fca74..b7b57ef8f7 100644 --- a/tools/libxl/xl_cmdimpl.c +++ b/tools/libxl/xl_cmdimpl.c @@ -8011,8 +8011,10 @@ int main_cpupoolcpuremove(int argc, char **argv) goto out; } - if (libxl_cpupool_cpuremove_cpumap(ctx, poolid, &cpumap)) - fprintf(stderr, "some cpus may not have been removed from %s\n", pool); + if (libxl_cpupool_cpuremove_cpumap(ctx, poolid, &cpumap)) { + fprintf(stderr, "Some cpus may have not or only partially been removed from '%s'.\n", pool); + fprintf(stderr, "If a cpu can't be added to another cpupool, add it to '%s' again and retry.\n", pool); + } rc = EXIT_SUCCESS; From 61a50dfa9d2d779c807e6de72d12786ec917ae63 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Thu, 24 Mar 2016 18:44:50 +0100 Subject: [PATCH 3/3] libxl: add force option for xl vcpu-pin In order to be able to undo a vcpu pin override in case of a kernel driver error add a flag "-f" to the "xl vcpu-pin" command forcing the hypervisor to undo the override. Cc: Ian Jackson Cc: Stefano Stabellini Cc: Wei Liu Signed-off-by: Juergen Gross Reviewed-by: Dario Faggioli Acked-by: Wei Liu --- docs/man/xl.pod.1 | 7 ++++++- tools/libxl/libxl.c | 31 +++++++++++++++++++++++++------ tools/libxl/libxl.h | 10 ++++++++++ tools/libxl/xl_cmdimpl.c | 26 +++++++++++++++++++++++--- tools/libxl/xl_cmdtable.c | 3 ++- 5 files changed, 66 insertions(+), 11 deletions(-) diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1 index dc6213e428..4d4b333d73 100644 --- a/docs/man/xl.pod.1 +++ b/docs/man/xl.pod.1 @@ -689,7 +689,7 @@ after B, go to B section for information. Lists VCPU information for a specific domain. If no domain is specified, VCPU information for all domains will be provided. -=item B I I I I +=item B [I<-f|--force>] I I I I Set hard and soft affinity for a I of . Normally VCPUs can float between available CPUs whenever Xen deems a different run state @@ -716,6 +716,11 @@ leaving its hard affinity untouched. On the othe hand: will set both hard and soft affinity, the former to pCPUs 3 and 4, the latter to pCPUs 6,7,8, and 9. +Specifying I<-f> or I<--force> will remove a temporary pinning done by the +operating system (normally this should be done by the operating system). +In case a temporary pinning is active for a vcpu the affinity of this vcpu +can't be changed without this option. + =item B Prints information about guests. This list excludes information about diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index 3471c4c038..d984d7e4de 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -5322,18 +5322,20 @@ err: return NULL; } -int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid, - const libxl_bitmap *cpumap_hard, - const libxl_bitmap *cpumap_soft) +static int libxl__set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, + uint32_t vcpuid, + const libxl_bitmap *cpumap_hard, + const libxl_bitmap *cpumap_soft, + unsigned flags) { GC_INIT(ctx); libxl_bitmap hard, soft; - int rc, flags = 0; + int rc; libxl_bitmap_init(&hard); libxl_bitmap_init(&soft); - if (!cpumap_hard && !cpumap_soft) { + if (!cpumap_hard && !cpumap_soft && !flags) { rc = ERROR_INVAL; goto out; } @@ -5348,7 +5350,7 @@ int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid, goto out; libxl__bitmap_copy_best_effort(gc, &hard, cpumap_hard); - flags = XEN_VCPUAFFINITY_HARD; + flags |= XEN_VCPUAFFINITY_HARD; } if (cpumap_soft) { rc = libxl_cpu_bitmap_alloc(ctx, &soft, 0); @@ -5399,6 +5401,23 @@ int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid, return rc; } +int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid, + const libxl_bitmap *cpumap_hard, + const libxl_bitmap *cpumap_soft) +{ + return libxl__set_vcpuaffinity(ctx, domid, vcpuid, cpumap_hard, + cpumap_soft, 0); +} + +int libxl_set_vcpuaffinity_force(libxl_ctx *ctx, uint32_t domid, + uint32_t vcpuid, + const libxl_bitmap *cpumap_hard, + const libxl_bitmap *cpumap_soft) +{ + return libxl__set_vcpuaffinity(ctx, domid, vcpuid, cpumap_hard, + cpumap_soft, XEN_VCPUAFFINITY_FORCE); +} + int libxl_set_vcpuaffinity_all(libxl_ctx *ctx, uint32_t domid, unsigned int max_vcpus, const libxl_bitmap *cpumap_hard, diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h index f61bc4b063..83d7cd3623 100644 --- a/tools/libxl/libxl.h +++ b/tools/libxl/libxl.h @@ -153,6 +153,12 @@ */ #define LIBXL_HAVE_VCPUINFO_SOFT_AFFINITY 1 +/* + * LIBXL_HAVE_SET_VCPUAFFINITY_FORCE indicates that the + * libxl_set_vcpuaffinity_force() library call is available. + */ +#define LIBXL_HAVE_SET_VCPUAFFINITY_FORCE 1 + /* * LIBXL_HAVE_DEVICE_DISK_DIRECT_IO_SAFE indicates that a * 'direct_io_safe' field (of boolean type) is present in @@ -1798,6 +1804,10 @@ int libxl_get_physinfo(libxl_ctx *ctx, libxl_physinfo *physinfo); int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid, const libxl_bitmap *cpumap_hard, const libxl_bitmap *cpumap_soft); +int libxl_set_vcpuaffinity_force(libxl_ctx *ctx, uint32_t domid, + uint32_t vcpuid, + const libxl_bitmap *cpumap_hard, + const libxl_bitmap *cpumap_soft); int libxl_set_vcpuaffinity_all(libxl_ctx *ctx, uint32_t domid, unsigned int max_vcpus, const libxl_bitmap *cpumap_hard, diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c index b7b57ef8f7..438e9146f8 100644 --- a/tools/libxl/xl_cmdimpl.c +++ b/tools/libxl/xl_cmdimpl.c @@ -5639,6 +5639,10 @@ int main_vcpulist(int argc, char **argv) int main_vcpupin(int argc, char **argv) { + static struct option opts[] = { + {"force", 0, 0, 'f'}, + COMMON_LONG_OPTS + }; libxl_vcpuinfo *vcpuinfo; libxl_bitmap cpumap_hard, cpumap_soft;; libxl_bitmap *soft = &cpumap_soft, *hard = &cpumap_hard; @@ -5651,12 +5655,17 @@ int main_vcpupin(int argc, char **argv) const char *vcpu, *hard_str, *soft_str; char *endptr; int opt, nb_cpu, nb_vcpu, rc = EXIT_FAILURE; + bool force = false; libxl_bitmap_init(&cpumap_hard); libxl_bitmap_init(&cpumap_soft); - SWITCH_FOREACH_OPT(opt, "", NULL, "vcpu-pin", 3) { - /* No options */ + SWITCH_FOREACH_OPT(opt, "f", opts, "vcpu-pin", 3) { + case 'f': + force = true; + break; + default: + break; } domid = find_domain(argv[optind]); @@ -5671,6 +5680,10 @@ int main_vcpupin(int argc, char **argv) fprintf(stderr, "Error: Invalid argument %s as VCPU.\n", vcpu); goto out; } + if (force) { + fprintf(stderr, "Error: --force and 'all' as VCPU not allowed.\n"); + goto out; + } vcpuid = -1; } @@ -5732,7 +5745,14 @@ int main_vcpupin(int argc, char **argv) goto out; } - if (vcpuid != -1) { + if (force) { + if (libxl_set_vcpuaffinity_force(ctx, domid, vcpuid, hard, soft)) { + fprintf(stderr, "Could not set affinity for vcpu `%ld'.\n", + vcpuid); + goto out; + } + } + else if (vcpuid != -1) { if (libxl_set_vcpuaffinity(ctx, domid, vcpuid, hard, soft)) { fprintf(stderr, "Could not set affinity for vcpu `%ld'.\n", vcpuid); diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c index b14b881b52..7880a9c6b9 100644 --- a/tools/libxl/xl_cmdtable.c +++ b/tools/libxl/xl_cmdtable.c @@ -222,7 +222,8 @@ struct cmd_spec cmd_table[] = { { "vcpu-pin", &main_vcpupin, 1, 1, "Set which CPUs a VCPU can use", - " ", + "[option] ", + "-f, --force undo an override pinning done by the kernel", }, { "vcpu-set", &main_vcpuset, 0, 1,