tools: CMDs and APIs for Cache Monitoring Technology

Introduced some new xl commands to enable/disable Cache Monitoring
Technology(CMT) feature.

The following two commands is to attach/detach the CMT feature
to/from a certain domain.

$ xl psr-cmt-attach domid
$ xl psr-cmt-detach domid

This command is to display the CMT information, such as L3 cache
occupancy.

$ xl psr-cmt-show cache_occupancy <domid>

Signed-off-by: Dongxiao Xu <dongxiao.xu@intel.com>
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
Acked-by: Wei Liu <wei.liu2@citrix.com>

docs/man/xl.pod.1

@@ -1440,6 +1440,31 @@ Load FLASK policy from the given policy file. The initial policy is provided to
 the hypervisor as a multiboot module; this command allows runtime updates to the
 policy. Loading new security policy will reset runtime changes to device labels.
 
+=head1 Cache Monitoring Technology
+
+Intel Haswell and later server platforms offer monitoring capability in each
+logical processor to measure specific platform shared resource metric, for
+example, L3 cache occupancy. In Xen implementation, the monitoring granularity
+is domain level. To monitor a specific domain, just attach the domain id with
+the monitoring service. When the domain doesn't need to be monitored any more,
+detach the domain id from the monitoring service.
+
+=over 4
+
+=item B<psr-cmt-attach> [I<domain-id>]
+
+attach: Attach the platform shared resource monitoring service to a domain.
+
+=item B<psr-cmt-detach> [I<domain-id>]
+
+detach: Detach the platform shared resource monitoring service from a domain.
+
+=item B<psr-cmt-show> [I<psr-monitor-type>] [I<domain-id>]
+
+Show monitoring data for a certain domain or all domains. Current supported
+monitor types are:
+ - "cache-occupancy": showing the L3 cache occupancy.
+
 =back
 
 =head1 TO BE DOCUMENTED

tools/libxc/Makefile

@@ -36,6 +36,7 @@ CTRL_SRCS-y       += xc_kexec.c
 CTRL_SRCS-y       += xtl_core.c
 CTRL_SRCS-y       += xtl_logger_stdio.c
 CTRL_SRCS-y       += xc_resource.c
+CTRL_SRCS-$(CONFIG_X86) += xc_psr.c
 CTRL_SRCS-$(CONFIG_X86) += xc_pagetab.c
 CTRL_SRCS-$(CONFIG_Linux) += xc_linux.c xc_linux_osdep.c
 CTRL_SRCS-$(CONFIG_FreeBSD) += xc_freebsd.c xc_freebsd_osdep.c

tools/libxc/include/xenctrl.h

@@ -2679,6 +2679,25 @@ struct xc_resource_op {
 typedef struct xc_resource_op xc_resource_op_t;
 int xc_resource_op(xc_interface *xch, uint32_t nr_ops, xc_resource_op_t *ops);
 
+#if defined(__i386__) || defined(__x86_64__)
+enum xc_psr_cmt_type {
+    XC_PSR_CMT_L3_OCCUPANCY,
+};
+typedef enum xc_psr_cmt_type xc_psr_cmt_type;
+int xc_psr_cmt_attach(xc_interface *xch, uint32_t domid);
+int xc_psr_cmt_detach(xc_interface *xch, uint32_t domid);
+int xc_psr_cmt_get_domain_rmid(xc_interface *xch, uint32_t domid,
+    uint32_t *rmid);
+int xc_psr_cmt_get_total_rmid(xc_interface *xch, uint32_t *total_rmid);
+int xc_psr_cmt_get_l3_upscaling_factor(xc_interface *xch,
+    uint32_t *upscaling_factor);
+int xc_psr_cmt_get_l3_cache_size(xc_interface *xch, uint32_t cpu,
+    uint32_t *l3_cache_size);
+int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid,
+    uint32_t cpu, uint32_t psr_cmt_type, uint64_t *monitor_data);
+int xc_psr_cmt_enabled(xc_interface *xch);
+#endif
+
 #endif /* XENCTRL_H */
 
 /*

tools/libxc/xc_msr_x86.h

@@ -0,0 +1,36 @@
+/*
+ * xc_msr_x86.h
+ *
+ * MSR definition macros
+ *
+ * Copyright (C) 2014      Intel Corporation
+ * Author Dongxiao Xu <dongxiao.xu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#ifndef XC_MSR_X86_H
+#define XC_MSR_X86_H
+
+#define MSR_IA32_CMT_EVTSEL     0x00000c8d
+#define MSR_IA32_CMT_CTR        0x00000c8e
+
+#endif
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */

tools/libxc/xc_psr.c

@@ -0,0 +1,215 @@
+/*
+ * xc_psr.c
+ *
+ * platform shared resource related API functions.
+ *
+ * Copyright (C) 2014      Intel Corporation
+ * Author Dongxiao Xu <dongxiao.xu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "xc_private.h"
+#include "xc_msr_x86.h"
+
+#define IA32_CMT_CTR_ERROR_MASK         (0x3ull << 62)
+
+#define EVTID_L3_OCCUPANCY             0x1
+
+int xc_psr_cmt_attach(xc_interface *xch, uint32_t domid)
+{
+    DECLARE_DOMCTL;
+
+    domctl.cmd = XEN_DOMCTL_psr_cmt_op;
+    domctl.domain = (domid_t)domid;
+    domctl.u.psr_cmt_op.cmd = XEN_DOMCTL_PSR_CMT_OP_ATTACH;
+
+    return do_domctl(xch, &domctl);
+}
+
+int xc_psr_cmt_detach(xc_interface *xch, uint32_t domid)
+{
+    DECLARE_DOMCTL;
+
+    domctl.cmd = XEN_DOMCTL_psr_cmt_op;
+    domctl.domain = (domid_t)domid;
+    domctl.u.psr_cmt_op.cmd = XEN_DOMCTL_PSR_CMT_OP_DETACH;
+
+    return do_domctl(xch, &domctl);
+}
+
+int xc_psr_cmt_get_domain_rmid(xc_interface *xch, uint32_t domid,
+                                    uint32_t *rmid)
+{
+    int rc;
+    DECLARE_DOMCTL;
+
+    domctl.cmd = XEN_DOMCTL_psr_cmt_op;
+    domctl.domain = (domid_t)domid;
+    domctl.u.psr_cmt_op.cmd = XEN_DOMCTL_PSR_CMT_OP_QUERY_RMID;
+
+    rc = do_domctl(xch, &domctl);
+
+    if ( !rc )
+        *rmid = domctl.u.psr_cmt_op.data;
+
+    return rc;
+}
+
+int xc_psr_cmt_get_total_rmid(xc_interface *xch, uint32_t *total_rmid)
+{
+    static int val = 0;
+    int rc;
+    DECLARE_SYSCTL;
+
+    if ( val )
+    {
+        *total_rmid = val;
+        return 0;
+    }
+
+    sysctl.cmd = XEN_SYSCTL_psr_cmt_op;
+    sysctl.u.psr_cmt_op.cmd = XEN_SYSCTL_PSR_CMT_get_total_rmid;
+    sysctl.u.psr_cmt_op.flags = 0;
+
+    rc = xc_sysctl(xch, &sysctl);
+    if ( !rc )
+        val = *total_rmid = sysctl.u.psr_cmt_op.u.data;
+
+    return rc;
+}
+
+int xc_psr_cmt_get_l3_upscaling_factor(xc_interface *xch,
+                                            uint32_t *upscaling_factor)
+{
+    static int val = 0;
+    int rc;
+    DECLARE_SYSCTL;
+
+    if ( val )
+    {
+        *upscaling_factor = val;
+        return 0;
+    }
+
+    sysctl.cmd = XEN_SYSCTL_psr_cmt_op;
+    sysctl.u.psr_cmt_op.cmd =
+        XEN_SYSCTL_PSR_CMT_get_l3_upscaling_factor;
+    sysctl.u.psr_cmt_op.flags = 0;
+
+    rc = xc_sysctl(xch, &sysctl);
+    if ( !rc )
+        val = *upscaling_factor = sysctl.u.psr_cmt_op.u.data;
+
+    return rc;
+}
+
+int xc_psr_cmt_get_l3_cache_size(xc_interface *xch, uint32_t cpu,
+                                      uint32_t *l3_cache_size)
+{
+    static int val = 0;
+    int rc;
+    DECLARE_SYSCTL;
+
+    if ( val )
+    {
+        *l3_cache_size = val;
+        return 0;
+    }
+
+    sysctl.cmd = XEN_SYSCTL_psr_cmt_op;
+    sysctl.u.psr_cmt_op.cmd =
+        XEN_SYSCTL_PSR_CMT_get_l3_cache_size;
+    sysctl.u.psr_cmt_op.flags = 0;
+    sysctl.u.psr_cmt_op.u.l3_cache.cpu = cpu;
+
+    rc = xc_sysctl(xch, &sysctl);
+    if ( !rc )
+        val = *l3_cache_size= sysctl.u.psr_cmt_op.u.data;
+
+    return rc;
+}
+
+int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid,
+    uint32_t cpu, xc_psr_cmt_type type, uint64_t *monitor_data)
+{
+    xc_resource_op_t op;
+    xc_resource_entry_t entries[2];
+    uint32_t evtid;
+    int rc;
+
+    switch ( type )
+    {
+    case XC_PSR_CMT_L3_OCCUPANCY:
+        evtid = EVTID_L3_OCCUPANCY;
+        break;
+    default:
+        return -1;
+    }
+
+    entries[0].u.cmd = XEN_RESOURCE_OP_MSR_WRITE;
+    entries[0].idx = MSR_IA32_CMT_EVTSEL;
+    entries[0].val = (uint64_t)rmid << 32 | evtid;
+    entries[0].rsvd = 0;
+
+    entries[1].u.cmd = XEN_RESOURCE_OP_MSR_READ;
+    entries[1].idx = MSR_IA32_CMT_CTR;
+    entries[1].val = 0;
+    entries[1].rsvd = 0;
+
+    op.cpu = cpu;
+    op.nr_entries = 2;
+    op.entries = entries;
+
+    rc = xc_resource_op(xch, 1, &op);
+    if ( rc < 0 )
+        return rc;
+
+    if ( op.result !=2 || entries[1].val & IA32_CMT_CTR_ERROR_MASK )
+        return -1;
+
+    *monitor_data = entries[1].val;
+
+    return 0;
+}
+
+int xc_psr_cmt_enabled(xc_interface *xch)
+{
+    static int val = -1;
+    int rc;
+    DECLARE_SYSCTL;
+
+    if ( val >= 0 )
+        return val;
+
+    sysctl.cmd = XEN_SYSCTL_psr_cmt_op;
+    sysctl.u.psr_cmt_op.cmd = XEN_SYSCTL_PSR_CMT_enabled;
+    sysctl.u.psr_cmt_op.flags = 0;
+
+    rc = do_sysctl(xch, &sysctl);
+    if ( !rc )
+    {
+        val = sysctl.u.psr_cmt_op.u.data;
+        return val;
+    }
+
+    return 0;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */

tools/libxl/Makefile

@@ -58,7 +58,7 @@ endif
 
 LIBXL_OBJS-y += libxl_remus_device.o libxl_remus_disk_drbd.o
 
-LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o
+LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o
 LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o
 
 ifeq ($(CONFIG_NetBSD),y)

tools/libxl/libxl.h

@@ -659,6 +659,15 @@ typedef uint8_t libxl_mac[6];
 #define LIBXL_MAC_BYTES(mac) mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]
 void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, libxl_mac *src);
 
+#if defined(__i386__) || defined(__x86_64__)
+/*
+ * LIBXL_HAVE_PSR_CMT
+ *
+ * If this is defined, the Cache Monitoring Technology feature is supported.
+ */
+#define LIBXL_HAVE_PSR_CMT 1
+#endif
+
 typedef char **libxl_string_list;
 void libxl_string_list_dispose(libxl_string_list *sl);
 int libxl_string_list_length(const libxl_string_list *sl);
@@ -1400,6 +1409,18 @@ bool libxl_ms_vm_genid_is_zero(const libxl_ms_vm_genid *id);
 void libxl_ms_vm_genid_copy(libxl_ctx *ctx, libxl_ms_vm_genid *dst,
                             libxl_ms_vm_genid *src);
 
+#ifdef LIBXL_HAVE_PSR_CMT
+int libxl_psr_cmt_attach(libxl_ctx *ctx, uint32_t domid);
+int libxl_psr_cmt_detach(libxl_ctx *ctx, uint32_t domid);
+int libxl_psr_cmt_domain_attached(libxl_ctx *ctx, uint32_t domid);
+int libxl_psr_cmt_enabled(libxl_ctx *ctx);
+int libxl_psr_cmt_get_total_rmid(libxl_ctx *ctx, uint32_t *total_rmid);
+int libxl_psr_cmt_get_l3_cache_size(libxl_ctx *ctx, uint32_t socketid,
+    uint32_t *l3_cache_size);
+int libxl_psr_cmt_get_cache_occupancy(libxl_ctx *ctx, uint32_t domid,
+    uint32_t socketid, uint32_t *l3_cache_occupancy);
+#endif
+
 /* misc */
 
 /* Each of these sets or clears the flag according to whether the

tools/libxl/libxl_psr.c

@@ -0,0 +1,217 @@
+/*
+ * Copyright (C) 2014      Intel Corporation
+ * Author Dongxiao Xu <dongxiao.xu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+#include "libxl_internal.h"
+
+
+#define IA32_QM_CTR_ERROR_MASK         (0x3ul << 62)
+
+static void libxl__psr_cmt_log_err_msg(libxl__gc *gc, int err)
+{
+    char *msg;
+
+    switch (err) {
+    case ENOSYS:
+        msg = "unsupported operation";
+        break;
+    case ENODEV:
+        msg = "CMT is not supported in this system";
+        break;
+    case EEXIST:
+        msg = "CMT is already attached to this domain";
+        break;
+    case ENOENT:
+        msg = "CMT is not attached to this domain";
+        break;
+    case EUSERS:
+        msg = "no free RMID available";
+        break;
+    case ESRCH:
+        msg = "invalid domain ID";
+        break;
+    case EFAULT:
+        msg = "failed to exchange data with Xen";
+        break;
+    default:
+        msg = "unknown error";
+        break;
+    }
+
+    LOGE(ERROR, "%s", msg);
+}
+
+static int libxl__pick_socket_cpu(libxl__gc *gc, uint32_t socketid)
+{
+    int i, nr_cpus;
+    libxl_cputopology *topology;
+    int cpu = ERROR_FAIL;
+
+    topology = libxl_get_cpu_topology(CTX, &nr_cpus);
+    if (!topology)
+        return ERROR_FAIL;
+
+    for (i = 0; i < nr_cpus; i++)
+        if (topology[i].socket == socketid) {
+            cpu = i;
+            break;
+        }
+
+    libxl_cputopology_list_free(topology, nr_cpus);
+    return cpu;
+}
+
+int libxl_psr_cmt_attach(libxl_ctx *ctx, uint32_t domid)
+{
+    GC_INIT(ctx);
+    int rc;
+
+    rc = xc_psr_cmt_attach(ctx->xch, domid);
+    if (rc < 0) {
+        libxl__psr_cmt_log_err_msg(gc, errno);
+        rc = ERROR_FAIL;
+    }
+
+    GC_FREE;
+    return rc;
+}
+
+int libxl_psr_cmt_detach(libxl_ctx *ctx, uint32_t domid)
+{
+    GC_INIT(ctx);
+    int rc;
+
+    rc = xc_psr_cmt_detach(ctx->xch, domid);
+    if (rc < 0) {
+        libxl__psr_cmt_log_err_msg(gc, errno);
+        rc = ERROR_FAIL;
+    }
+
+    GC_FREE;
+    return rc;
+}
+
+int libxl_psr_cmt_domain_attached(libxl_ctx *ctx, uint32_t domid)
+{
+    int rc;
+    uint32_t rmid;
+
+    rc = xc_psr_cmt_get_domain_rmid(ctx->xch, domid, &rmid);
+    if (rc < 0)
+        return 0;
+
+    return !!rmid;
+}
+
+int libxl_psr_cmt_enabled(libxl_ctx *ctx)
+{
+    return xc_psr_cmt_enabled(ctx->xch);
+}
+
+int libxl_psr_cmt_get_total_rmid(libxl_ctx *ctx, uint32_t *total_rmid)
+{
+    GC_INIT(ctx);
+    int rc;
+
+    rc = xc_psr_cmt_get_total_rmid(ctx->xch, total_rmid);
+    if (rc < 0) {
+        libxl__psr_cmt_log_err_msg(gc, errno);
+        rc = ERROR_FAIL;
+    }
+
+    GC_FREE;
+    return rc;
+}
+
+int libxl_psr_cmt_get_l3_cache_size(libxl_ctx *ctx, uint32_t socketid,
+                                         uint32_t *l3_cache_size)
+{
+    GC_INIT(ctx);
+
+    int rc;
+    int cpu = libxl__pick_socket_cpu(gc, socketid);
+
+    if (cpu < 0) {
+        LOGE(ERROR, "failed to get socket cpu");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    rc = xc_psr_cmt_get_l3_cache_size(ctx->xch, cpu, l3_cache_size);
+    if (rc < 0) {
+        libxl__psr_cmt_log_err_msg(gc, errno);
+        rc = ERROR_FAIL;
+    }
+
+out:
+    GC_FREE;
+    return rc;
+}
+
+int libxl_psr_cmt_get_cache_occupancy(libxl_ctx *ctx, uint32_t domid,
+    uint32_t socketid, uint32_t *l3_cache_occupancy)
+{
+    GC_INIT(ctx);
+
+    unsigned int rmid;
+    uint32_t upscaling_factor;
+    uint64_t monitor_data;
+    int cpu, rc;
+    xc_psr_cmt_type type;
+
+    rc = xc_psr_cmt_get_domain_rmid(ctx->xch, domid, &rmid);
+    if (rc < 0 || rmid == 0) {
+        LOGE(ERROR, "fail to get the domain rmid, "
+            "or domain is not attached with platform QoS monitoring service");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    cpu = libxl__pick_socket_cpu(gc, socketid);
+    if (cpu < 0) {
+        LOGE(ERROR, "failed to get socket cpu");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    type = XC_PSR_CMT_L3_OCCUPANCY;
+    rc = xc_psr_cmt_get_data(ctx->xch, rmid, cpu, type, &monitor_data);
+    if (rc < 0) {
+        LOGE(ERROR, "failed to get monitoring data");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    rc = xc_psr_cmt_get_l3_upscaling_factor(ctx->xch, &upscaling_factor);
+    if (rc < 0) {
+        LOGE(ERROR, "failed to get L3 upscaling factor");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    *l3_cache_occupancy = upscaling_factor * monitor_data / 1024;
+    rc = 0;
+out:
+    GC_FREE;
+    return rc;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */

tools/libxl/libxl_types.idl

@@ -652,3 +652,7 @@ libxl_event = Struct("event",[
                                  ])),
            ("domain_create_console_available", None),
            ]))])
+
+libxl_psr_cmt_type = Enumeration("psr_cmt_type", [
+    (1, "CACHE_OCCUPANCY"),
+    ])

tools/libxl/xl.h

@@ -111,6 +111,11 @@ int main_loadpolicy(int argc, char **argv);
 int main_remus(int argc, char **argv);
 #endif
 int main_devd(int argc, char **argv);
+#ifdef LIBXL_HAVE_PSR_CMT
+int main_psr_cmt_attach(int argc, char **argv);
+int main_psr_cmt_detach(int argc, char **argv);
+int main_psr_cmt_show(int argc, char **argv);
+#endif
 
 void help(const char *command);
 

tools/libxl/xl_cmdimpl.c

@@ -7626,6 +7626,166 @@ out:
     return ret;
 }
 
+#ifdef LIBXL_HAVE_PSR_CMT
+static void psr_cmt_print_domain_cache_occupancy(libxl_dominfo *dominfo,
+                                                    uint32_t nr_sockets)
+{
+    char *domain_name;
+    uint32_t socketid;
+    uint32_t l3_cache_occupancy;
+
+    if (!libxl_psr_cmt_domain_attached(ctx, dominfo->domid))
+        return;
+
+    domain_name = libxl_domid_to_name(ctx, dominfo->domid);
+    printf("%-40s %5d", domain_name, dominfo->domid);
+    free(domain_name);
+
+    for (socketid = 0; socketid < nr_sockets; socketid++) {
+        if ( !libxl_psr_cmt_get_cache_occupancy(ctx, dominfo->domid,
+                 socketid, &l3_cache_occupancy) )
+            printf("%13u KB", l3_cache_occupancy);
+    }
+
+    printf("\n");
+}
+
+static int psr_cmt_show_cache_occupancy(uint32_t domid)
+{
+    uint32_t i, socketid, nr_sockets, total_rmid;
+    uint32_t l3_cache_size;
+    libxl_physinfo info;
+    int rc, nr_domains;
+
+    if (!libxl_psr_cmt_enabled(ctx)) {
+        fprintf(stderr, "CMT is disabled in the system\n");
+        return -1;
+    }
+
+    libxl_physinfo_init(&info);
+    rc = libxl_get_physinfo(ctx, &info);
+    if (rc < 0) {
+        fprintf(stderr, "Failed getting physinfo, rc: %d\n", rc);
+        libxl_physinfo_dispose(&info);
+        return -1;
+    }
+    nr_sockets = info.nr_cpus / info.threads_per_core / info.cores_per_socket;
+    libxl_physinfo_dispose(&info);
+
+    rc = libxl_psr_cmt_get_total_rmid(ctx, &total_rmid);
+    if (rc < 0) {
+        fprintf(stderr, "Failed to get max RMID value\n");
+        return -1;
+    }
+
+    printf("Total RMID: %d\n", total_rmid);
+
+    /* Header */
+    printf("%-40s %5s", "Name", "ID");
+    for (socketid = 0; socketid < nr_sockets; socketid++)
+        printf("%14s %d", "Socket", socketid);
+    printf("\n");
+
+    /* Total L3 cache size */
+    printf("%-46s", "Total L3 Cache Size");
+    for (socketid = 0; socketid < nr_sockets; socketid++) {
+        rc = libxl_psr_cmt_get_l3_cache_size(ctx, socketid, &l3_cache_size);
+        if (rc < 0) {
+            fprintf(stderr, "Failed to get system l3 cache size for socket:%d\n",
+                            socketid);
+            return -1;
+        }
+        printf("%13u KB", l3_cache_size);
+    }
+    printf("\n");
+
+    /* Each domain */
+    if (domid != INVALID_DOMID) {
+        libxl_dominfo dominfo;
+        if (!libxl_domain_info(ctx, &dominfo, domid)) {
+            fprintf(stderr, "Failed to get domain info for %d\n", domid);
+            return -1;
+        }
+        psr_cmt_print_domain_cache_occupancy(&dominfo, nr_sockets);
+    }
+    else
+    {
+        libxl_dominfo *list;
+        if (!(list = libxl_list_domain(ctx, &nr_domains))) {
+            fprintf(stderr, "Failed to get domain info for domain list.\n");
+            return -1;
+        }
+        for (i = 0; i < nr_domains; i++)
+            psr_cmt_print_domain_cache_occupancy(list + i, nr_sockets);
+        libxl_dominfo_list_free(list, nr_domains);
+    }
+    return 0;
+}
+
+int main_psr_cmt_attach(int argc, char **argv)
+{
+    uint32_t domid;
+    int opt, ret = 0;
+
+    SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-attach", 1) {
+        /* No options */
+    }
+
+    domid = find_domain(argv[optind]);
+    ret = libxl_psr_cmt_attach(ctx, domid);
+
+    return ret;
+}
+
+int main_psr_cmt_detach(int argc, char **argv)
+{
+    uint32_t domid;
+    int opt, ret = 0;
+
+    SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-detach", 1) {
+        /* No options */
+    }
+
+    domid = find_domain(argv[optind]);
+    ret = libxl_psr_cmt_detach(ctx, domid);
+
+    return ret;
+}
+
+int main_psr_cmt_show(int argc, char **argv)
+{
+    int opt, ret = 0;
+    uint32_t domid;
+    libxl_psr_cmt_type type;
+
+    SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-show", 1) {
+        /* No options */
+    }
+
+    libxl_psr_cmt_type_from_string(argv[optind], &type);
+
+    if (optind + 1 >= argc)
+        domid = INVALID_DOMID;
+    else if (optind + 1 == argc - 1)
+        domid = find_domain(argv[optind + 1]);
+    else {
+        help("psr-cmt-show");
+        return 2;
+    }
+
+    switch (type) {
+    case LIBXL_PSR_CMT_TYPE_CACHE_OCCUPANCY:
+        ret = psr_cmt_show_cache_occupancy(domid);
+        break;
+    default:
+        help("psr-cmt-show");
+        return 2;
+    }
+
+    return ret;
+}
+#endif
+
 /*
  * Local variables:
  * mode: C

tools/libxl/xl_cmdtable.c

@@ -517,6 +517,25 @@ struct cmd_spec cmd_table[] = {
       "[options]",
       "-F                      Run in the foreground",
     },
+#ifdef LIBXL_HAVE_PSR_CMT
+    { "psr-cmt-attach",
+      &main_psr_cmt_attach, 0, 1,
+      "Attach Cache Monitoring Technology service to a domain",
+      "<Domain>",
+    },
+    { "psr-cmt-detach",
+      &main_psr_cmt_detach, 0, 1,
+      "Detach Cache Monitoring Technology service from a domain",
+      "<Domain>",
+    },
+    { "psr-cmt-show",
+      &main_psr_cmt_show, 0, 1,
+      "Show Cache Monitoring Technology information",
+      "<PSR-CMT-Type> <Domain>",
+      "Available monitor types:\n"
+      "\"cache_occupancy\":         Show L3 cache occupancy\n",
+    },
+#endif
 };
 
 int cmdtable_len = sizeof(cmd_table)/sizeof(struct cmd_spec);