diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h index a8477c9add..d63cd11dd9 100644 --- a/tools/libxl/libxl.h +++ b/tools/libxl/libxl.h @@ -339,6 +339,31 @@ typedef struct libxl__ctx libxl_ctx; #endif #endif +/* + * LIBXL_HAVE_BUILDINFO_VCPU_AFFINITY_ARRAYS + * + * If this is defined, then the libxl_domain_build_info structure will + * contain two arrays of libxl_bitmap-s, with all the necessary information + * to set the hard affinity (vcpu_hard_affinity) and the soft affinity + * (vcpu_soft_affinity) of the VCPUs. + * + * Note that, if the vcpu_hard_affinity array is used, libxl will ignore + * the content of the cpumap field of libxl_domain_build_info. That is to + * say, if the array is allocated and used by the caller, it is it and + * only it that determines the hard affinity of the domain's VCPUs. + * + * The number of libxl_bitmap-s in the arrays should be equal to the + * maximum number of VCPUs of the domain. If there only are N elements in + * an array, with N smaller the the maximum number of VCPUs, the hard or + * soft affinity (depending on which array we are talking about) will be + * set only for the first N VCPUs. The other VCPUs will just have affinity, + * both hard and soft, with all the host PCPUs. + * Each bitmap should be big enough to accommodate the maximum number of + * PCPUs of the host. + */ +/* to be uncommented when soft array added */ +/* #define LIBXL_HAVE_BUILDINFO_VCPU_AFFINITY_ARRAYS 1 */ + /* * LIBXL_HAVE_BUILDINFO_USBDEVICE_LIST * diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c index 49bfa66404..015ccb31fe 100644 --- a/tools/libxl/libxl_dom.c +++ b/tools/libxl/libxl_dom.c @@ -249,7 +249,7 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid, * libxl_domain_set_nodeaffinity() that enacts the actual placement. */ if (libxl_defbool_val(info->numa_placement)) { - if (info->cpumap.size) { + if (info->cpumap.size || info->num_vcpu_hard_affinity) { LOG(ERROR, "Can run NUMA placement only if no vcpu " "affinity is specified explicitly"); return ERROR_INVAL; @@ -271,10 +271,23 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid, } if (info->nodemap.size) libxl_domain_set_nodeaffinity(ctx, domid, &info->nodemap); - if (info->cpumap.size) + /* As mentioned in libxl.h, vcpu_hard_array takes precedence */ + if (info->num_vcpu_hard_affinity) { + int i; + + for (i = 0; i < info->num_vcpu_hard_affinity; i++) { + if (libxl_set_vcpuaffinity(ctx, domid, i, + &info->vcpu_hard_affinity[i], + NULL)) { + LOG(ERROR, "setting affinity failed on vcpu `%d'", i); + return ERROR_FAIL; + } + } + } else if (info->cpumap.size) libxl_set_vcpuaffinity_all(ctx, domid, info->max_vcpus, &info->cpumap, NULL); + if (xc_domain_setmaxmem(ctx->xch, domid, info->target_memkb + LIBXL_MAXMEM_CONSTANT) < 0) { LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "Couldn't set max memory"); diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index 37df854e20..5607ea7142 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -303,6 +303,7 @@ libxl_domain_build_info = Struct("domain_build_info",[ ("avail_vcpus", libxl_bitmap), ("cpumap", libxl_bitmap), ("nodemap", libxl_bitmap), + ("vcpu_hard_affinity", Array(libxl_bitmap, "num_vcpu_hard_affinity")), ("numa_placement", libxl_defbool), ("tsc_mode", libxl_tsc_mode), ("max_memkb", MemKB), diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c index 4d5e0d5428..e2d3ad066c 100644 --- a/tools/libxl/xl_cmdimpl.c +++ b/tools/libxl/xl_cmdimpl.c @@ -88,9 +88,6 @@ xlchild children[child_max]; static const char *common_domname; static int fd_lock = -1; -/* Stash for specific vcpu to pcpu mappping */ -static int *vcpu_to_pcpu; - static const char savefileheader_magic[32]= "Xen saved domain, xl format\n \0 \r"; @@ -703,7 +700,7 @@ static void parse_config_data(const char *config_source, XLU_Config *config; XLU_ConfigList *cpus, *vbds, *nics, *pcis, *cvfbs, *cpuids, *vtpms; XLU_ConfigList *ioports, *irqs, *iomem; - int num_ioports, num_irqs, num_iomem; + int num_ioports, num_irqs, num_iomem, num_cpus; int pci_power_mgmt = 0; int pci_msitranslate = 0; int pci_permissive = 0; @@ -800,43 +797,32 @@ static void parse_config_data(const char *config_source, if (!xlu_cfg_get_long (config, "maxvcpus", &l, 0)) b_info->max_vcpus = l; - if (!xlu_cfg_get_list (config, "cpus", &cpus, 0, 1)) { - int n_cpus = 0; + if (!xlu_cfg_get_list (config, "cpus", &cpus, &num_cpus, 1)) { + int j = 0; - if (libxl_cpu_bitmap_alloc(ctx, &b_info->cpumap, 0)) { - fprintf(stderr, "Unable to allocate cpumap\n"); - exit(1); - } + /* Silently ignore values corresponding to non existing vcpus */ + if (num_cpus > b_info->max_vcpus) + num_cpus = b_info->max_vcpus; - /* Prepare the array for single vcpu to pcpu mappings */ - vcpu_to_pcpu = xmalloc(sizeof(int) * b_info->max_vcpus); - memset(vcpu_to_pcpu, -1, sizeof(int) * b_info->max_vcpus); + b_info->vcpu_hard_affinity = xmalloc(num_cpus * sizeof(libxl_bitmap)); - /* - * Idea here is to let libxl think all the domain's vcpus - * have cpu affinity with all the pcpus on the list. - * It is then us, here in xl, that matches each single vcpu - * to its pcpu (and that's why we need to stash such info in - * the vcpu_to_pcpu array now) after the domain has been created. - * Doing it like this saves the burden of passing to libxl - * some big array hosting the single mappings. Also, using - * the cpumap derived from the list ensures memory is being - * allocated on the proper nodes anyway. - */ - libxl_bitmap_set_none(&b_info->cpumap); - while ((buf = xlu_cfg_get_listitem(cpus, n_cpus)) != NULL) { + while ((buf = xlu_cfg_get_listitem(cpus, j)) != NULL && j < num_cpus) { i = atoi(buf); - if (!libxl_bitmap_cpu_valid(&b_info->cpumap, i)) { - fprintf(stderr, "cpu %d illegal\n", i); + + libxl_bitmap_init(&b_info->vcpu_hard_affinity[j]); + if (libxl_cpu_bitmap_alloc(ctx, + &b_info->vcpu_hard_affinity[j], 0)) { + fprintf(stderr, "Unable to allocate cpumap for vcpu %d\n", j); exit(1); } - libxl_bitmap_set(&b_info->cpumap, i); - if (n_cpus < b_info->max_vcpus) - vcpu_to_pcpu[n_cpus] = i; - n_cpus++; - } + libxl_bitmap_set_none(&b_info->vcpu_hard_affinity[j]); + libxl_bitmap_set(&b_info->vcpu_hard_affinity[j], i); - /* We have a cpumap, disable automatic placement */ + j++; + } + b_info->num_vcpu_hard_affinity = num_cpus; + + /* We have a list of cpumaps, disable automatic placement */ libxl_defbool_set(&b_info->numa_placement, false); } else if (!xlu_cfg_get_string (config, "cpus", &buf, 0)) { @@ -2183,33 +2169,6 @@ start: if ( ret ) goto error_out; - /* If single vcpu to pcpu mapping was requested, honour it */ - if (vcpu_to_pcpu) { - libxl_bitmap vcpu_cpumap; - - ret = libxl_cpu_bitmap_alloc(ctx, &vcpu_cpumap, 0); - if (ret) - goto error_out; - for (i = 0; i < d_config.b_info.max_vcpus; i++) { - - if (vcpu_to_pcpu[i] != -1) { - libxl_bitmap_set_none(&vcpu_cpumap); - libxl_bitmap_set(&vcpu_cpumap, vcpu_to_pcpu[i]); - } else { - libxl_bitmap_set_any(&vcpu_cpumap); - } - if (libxl_set_vcpuaffinity(ctx, domid, i, &vcpu_cpumap, NULL)) { - fprintf(stderr, "setting affinity failed on vcpu `%d'.\n", i); - libxl_bitmap_dispose(&vcpu_cpumap); - free(vcpu_to_pcpu); - ret = ERROR_FAIL; - goto error_out; - } - } - libxl_bitmap_dispose(&vcpu_cpumap); - free(vcpu_to_pcpu); vcpu_to_pcpu = NULL; - } - ret = libxl_userdata_store(ctx, domid, "xl", config_data, config_len); if (ret) {