diff --git a/buildconfigs/linux-defconfig_xen0_x86_32 b/buildconfigs/linux-defconfig_xen0_x86_32 index d63ec9498b..e098035f66 100644 --- a/buildconfigs/linux-defconfig_xen0_x86_32 +++ b/buildconfigs/linux-defconfig_xen0_x86_32 @@ -548,7 +548,7 @@ CONFIG_MEGARAID_NEWGEN=y # CONFIG_MEGARAID_LEGACY is not set # CONFIG_MEGARAID_SAS is not set CONFIG_SCSI_SATA=y -# CONFIG_SCSI_SATA_AHCI is not set +CONFIG_SCSI_SATA_AHCI=y # CONFIG_SCSI_SATA_SVW is not set CONFIG_SCSI_ATA_PIIX=y # CONFIG_SCSI_SATA_MV is not set @@ -713,7 +713,7 @@ CONFIG_E1000=y CONFIG_SK98LIN=y # CONFIG_VIA_VELOCITY is not set CONFIG_TIGON3=y -# CONFIG_BNX2 is not set +CONFIG_BNX2=y # # Ethernet (10000 Mbit) diff --git a/buildconfigs/linux-defconfig_xen0_x86_64 b/buildconfigs/linux-defconfig_xen0_x86_64 index 603e0a82d9..48ba626c4b 100644 --- a/buildconfigs/linux-defconfig_xen0_x86_64 +++ b/buildconfigs/linux-defconfig_xen0_x86_64 @@ -498,7 +498,7 @@ CONFIG_MEGARAID_NEWGEN=y # CONFIG_MEGARAID_LEGACY is not set # CONFIG_MEGARAID_SAS is not set CONFIG_SCSI_SATA=y -# CONFIG_SCSI_SATA_AHCI is not set +CONFIG_SCSI_SATA_AHCI=y # CONFIG_SCSI_SATA_SVW is not set CONFIG_SCSI_ATA_PIIX=y # CONFIG_SCSI_SATA_MV is not set @@ -663,7 +663,7 @@ CONFIG_E1000=y CONFIG_SK98LIN=y # CONFIG_VIA_VELOCITY is not set CONFIG_TIGON3=y -# CONFIG_BNX2 is not set +CONFIG_BNX2=y # # Ethernet (10000 Mbit) diff --git a/buildconfigs/mk.linux-2.6-xen b/buildconfigs/mk.linux-2.6-xen index 272a58479d..a2452f35b2 100644 --- a/buildconfigs/mk.linux-2.6-xen +++ b/buildconfigs/mk.linux-2.6-xen @@ -9,6 +9,10 @@ LINUX_DIR = build-linux-$(LINUX_VER)-$(EXTRAVERSION)_$(XEN_TARGET_ARCH) IMAGE_TARGET ?= vmlinuz INSTALL_BOOT_PATH ?= $(DESTDIR) +ifeq ($(XEN_TARGET_ARCH),ia64) +INSTALL_BOOT_PATH := $(DESTDIR)/boot +endif + LINUX_VER3 := $(LINUX_SERIES).$(word 3, $(subst ., ,$(LINUX_VER))) include buildconfigs/Rules.mk diff --git a/linux-2.6-xen-sparse/arch/i386/kernel/sysenter.c b/linux-2.6-xen-sparse/arch/i386/kernel/sysenter.c index f300bd159e..1631c182c9 100644 --- a/linux-2.6-xen-sparse/arch/i386/kernel/sysenter.c +++ b/linux-2.6-xen-sparse/arch/i386/kernel/sysenter.c @@ -56,7 +56,9 @@ static void *syscall_page; int __init sysenter_setup(void) { - syscall_page = (void *)get_zeroed_page(GFP_ATOMIC); + void *page = (void *)get_zeroed_page(GFP_ATOMIC); + + syscall_page = page; #ifdef CONFIG_XEN if (boot_cpu_has(X86_FEATURE_SEP)) { @@ -70,16 +72,16 @@ int __init sysenter_setup(void) } #endif - if (boot_cpu_has(X86_FEATURE_SEP)) { - memcpy(syscall_page, - &vsyscall_sysenter_start, - &vsyscall_sysenter_end - &vsyscall_sysenter_start); + if (!boot_cpu_has(X86_FEATURE_SEP)) { + memcpy(page, + &vsyscall_int80_start, + &vsyscall_int80_end - &vsyscall_int80_start); return 0; } - memcpy(syscall_page, - &vsyscall_int80_start, - &vsyscall_int80_end - &vsyscall_int80_start); + memcpy(page, + &vsyscall_sysenter_start, + &vsyscall_sysenter_end - &vsyscall_sysenter_start); return 0; } diff --git a/tools/ioemu/hw/cirrus_vga.c b/tools/ioemu/hw/cirrus_vga.c index 4a5631195c..8467b403a9 100644 --- a/tools/ioemu/hw/cirrus_vga.c +++ b/tools/ioemu/hw/cirrus_vga.c @@ -3010,11 +3010,44 @@ static CPUWriteMemoryFunc *cirrus_mmio_write[3] = { cirrus_mmio_writel, }; +void cirrus_stop_acc(CirrusVGAState *s) +{ + if (s->map_addr){ + int error; + s->map_addr = 0; + error = unset_vram_mapping(s->cirrus_lfb_addr, + s->cirrus_lfb_end); + fprintf(stderr, "cirrus_stop_acc:unset_vram_mapping.\n"); + + munmap(s->vram_ptr, VGA_RAM_SIZE); + } +} + +void cirrus_restart_acc(CirrusVGAState *s) +{ + if (s->cirrus_lfb_addr && s->cirrus_lfb_end) { + void *vram_pointer, *old_vram; + fprintf(stderr, "cirrus_vga_load:re-enable vga acc.lfb_addr=0x%lx, lfb_end=0x%lx.\n", + s->cirrus_lfb_addr, s->cirrus_lfb_end); + vram_pointer = set_vram_mapping(s->cirrus_lfb_addr ,s->cirrus_lfb_end); + if (!vram_pointer){ + fprintf(stderr, "cirrus_vga_load:NULL vram_pointer\n"); + } else { + old_vram = vga_update_vram((VGAState *)s, vram_pointer, + VGA_RAM_SIZE); + qemu_free(old_vram); + s->map_addr = s->cirrus_lfb_addr; + s->map_end = s->cirrus_lfb_end; + } + } +} + /* load/save state */ static void cirrus_vga_save(QEMUFile *f, void *opaque) { CirrusVGAState *s = opaque; + uint8_t vga_acc; qemu_put_be32s(f, &s->latch); qemu_put_8s(f, &s->sr_index); @@ -3049,11 +3082,20 @@ static void cirrus_vga_save(QEMUFile *f, void *opaque) qemu_put_be32s(f, &s->hw_cursor_y); /* XXX: we do not save the bitblt state - we assume we do not save the state when the blitter is active */ + + vga_acc = (!!s->map_addr); + qemu_put_8s(f, &vga_acc); + qemu_put_be64s(f, (uint64_t*)&s->cirrus_lfb_addr); + qemu_put_be64s(f, (uint64_t*)&s->cirrus_lfb_end); + qemu_put_buffer(f, s->vram_ptr, VGA_RAM_SIZE); + if (vga_acc) + cirrus_stop_acc(s); } static int cirrus_vga_load(QEMUFile *f, void *opaque, int version_id) { CirrusVGAState *s = opaque; + uint8_t vga_acc = 0; if (version_id != 1) return -EINVAL; @@ -3092,6 +3134,14 @@ static int cirrus_vga_load(QEMUFile *f, void *opaque, int version_id) qemu_get_be32s(f, &s->hw_cursor_x); qemu_get_be32s(f, &s->hw_cursor_y); + qemu_get_8s(f, &vga_acc); + qemu_get_be64s(f, (uint64_t*)&s->cirrus_lfb_addr); + qemu_get_be64s(f, (uint64_t*)&s->cirrus_lfb_end); + qemu_get_buffer(f, s->vram_ptr, VGA_RAM_SIZE); + if (vga_acc){ + cirrus_restart_acc(s); + } + /* force refresh */ s->graphic_mode = -1; cirrus_update_bank_ptr(s, 0); @@ -3297,6 +3347,8 @@ void pci_cirrus_vga_init(PCIBus *bus, DisplayState *ds, uint8_t *vga_ram_base, ds, vga_ram_base, vga_ram_offset, vga_ram_size); cirrus_init_common(s, device_id, 1); + register_savevm("cirrus_vga_pci", 0, 1, generic_pci_save, generic_pci_load, d); + /* setup memory space */ /* memory #0 LFB */ /* memory #1 memory-mapped I/O */ diff --git a/tools/ioemu/hw/ide.c b/tools/ioemu/hw/ide.c index 532afbbffd..ef203a2f8a 100644 --- a/tools/ioemu/hw/ide.c +++ b/tools/ioemu/hw/ide.c @@ -2512,6 +2512,9 @@ void pci_piix3_ide_init(PCIBus *bus, BlockDriverState **hd_table, int devfn) pic_set_irq_new, isa_pic, 15); ide_init_ioport(&d->ide_if[0], 0x1f0, 0x3f6); ide_init_ioport(&d->ide_if[2], 0x170, 0x376); + + register_savevm("ide_pci", 0, 1, generic_pci_save, generic_pci_load, d); + #ifdef DMA_MULTI_THREAD dma_create_thread(); #endif //DMA_MULTI_THREAD diff --git a/tools/ioemu/hw/pci.c b/tools/ioemu/hw/pci.c index 77737c8615..fd2dcc9d93 100644 --- a/tools/ioemu/hw/pci.c +++ b/tools/ioemu/hw/pci.c @@ -40,6 +40,8 @@ target_phys_addr_t pci_mem_base; static int pci_irq_index; static PCIBus *first_bus; +static void pci_update_mappings(PCIDevice *d); + PCIBus *pci_register_bus(pci_set_irq_fn set_irq, void *pic, int devfn_min) { PCIBus *bus; @@ -71,6 +73,7 @@ int generic_pci_load(QEMUFile* f, void *opaque, int version_id) return -EINVAL; qemu_get_buffer(f, s->config, 256); + pci_update_mappings(s); return 0; } diff --git a/tools/ioemu/target-i386-dm/helper2.c b/tools/ioemu/target-i386-dm/helper2.c index ae84b9ed23..dc3394d5b7 100644 --- a/tools/ioemu/target-i386-dm/helper2.c +++ b/tools/ioemu/target-i386-dm/helper2.c @@ -546,6 +546,7 @@ int main_loop(void) { extern int vm_running; extern int shutdown_requested; + extern int suspend_requested; CPUState *env = cpu_single_env; int evtchn_fd = xc_evtchn_fd(xce_handle); @@ -563,12 +564,24 @@ int main_loop(void) qemu_system_reset(); reset_requested = 0; } + if (suspend_requested) { + fprintf(logfile, "device model received suspend signal!\n"); + break; + } } /* Wait up to 10 msec. */ main_loop_wait(10); } - destroy_hvm_domain(); + if (!suspend_requested) + destroy_hvm_domain(); + else { + char qemu_file[20]; + sprintf(qemu_file, "/tmp/xen.qemu-dm.%d", domid); + if (qemu_savevm(qemu_file) < 0) + fprintf(stderr, "qemu save fail.\n"); + } + return 0; } diff --git a/tools/ioemu/vl.c b/tools/ioemu/vl.c index 0e3a50e517..f5a79ad7e5 100644 --- a/tools/ioemu/vl.c +++ b/tools/ioemu/vl.c @@ -4441,6 +4441,11 @@ int qemu_loadvm(const char *filename) qemu_fseek(f, cur_pos + record_len, SEEK_SET); } fclose(f); + + /* del tmp file */ + if (unlink(filename) == -1) + fprintf(stderr, "delete tmp qemu state file failed.\n"); + ret = 0; the_end: if (saved_vm_running) @@ -5027,6 +5032,7 @@ typedef struct QEMUResetEntry { static QEMUResetEntry *first_reset_entry; int reset_requested; int shutdown_requested; +int suspend_requested; static int powerdown_requested; void qemu_register_reset(QEMUResetHandler *func, void *opaque) @@ -5808,6 +5814,14 @@ int set_mm_mapping(int xc_handle, uint32_t domid, return 0; } +void suspend(int sig) +{ + fprintf(logfile, "suspend sig handler called with requested=%d!\n", suspend_requested); + if (sig != SIGUSR1) + fprintf(logfile, "suspend signal dismatch, get sig=%d!\n", sig); + suspend_requested = 1; +} + #if defined(__i386__) || defined(__x86_64__) static struct map_cache *mapcache_entry; static unsigned long nr_buckets; @@ -6714,6 +6728,26 @@ int main(int argc, char **argv) vm_start(); } } + + /* register signal for the suspend request when save */ + { + struct sigaction act; + sigset_t set; + act.sa_handler = suspend; + act.sa_flags = SA_RESTART; + sigemptyset(&act.sa_mask); + + sigaction(SIGUSR1, &act, NULL); + + /* control panel mask some signals when spawn qemu, need unmask here*/ + sigemptyset(&set); + sigaddset(&set, SIGUSR1); + sigaddset(&set, SIGTERM); + if (sigprocmask(SIG_UNBLOCK, &set, NULL) == -1) + fprintf(stderr, "unblock signal fail, possible issue for HVM save!\n"); + + } + main_loop(); quit_timers(); return 0; diff --git a/tools/libfsimage/common/Makefile b/tools/libfsimage/common/Makefile index 9a62e9bbb5..6efce44f02 100644 --- a/tools/libfsimage/common/Makefile +++ b/tools/libfsimage/common/Makefile @@ -1,7 +1,7 @@ XEN_ROOT = ../../.. include $(XEN_ROOT)/tools/Rules.mk -MAJOR = 1.1 +MAJOR = 1.0 MINOR = 0 CFLAGS += -Werror -Wp,-MD,.$(@F).d diff --git a/tools/libxc/Makefile b/tools/libxc/Makefile index f92cde836d..8a563d00e0 100644 --- a/tools/libxc/Makefile +++ b/tools/libxc/Makefile @@ -27,7 +27,7 @@ GUEST_SRCS-y += xg_private.c GUEST_SRCS-$(CONFIG_X86) += xc_linux_build.c GUEST_SRCS-$(CONFIG_IA64) += xc_linux_build.c GUEST_SRCS-$(CONFIG_MIGRATE) += xc_linux_restore.c xc_linux_save.c -GUEST_SRCS-$(CONFIG_HVM) += xc_hvm_build.c +GUEST_SRCS-$(CONFIG_HVM) += xc_hvm_build.c xc_hvm_restore.c xc_hvm_save.c -include $(XEN_TARGET_ARCH)/Makefile diff --git a/tools/libxc/xc_domain.c b/tools/libxc/xc_domain.c index 82aef99ccc..3c42aba129 100644 --- a/tools/libxc/xc_domain.c +++ b/tools/libxc/xc_domain.c @@ -235,6 +235,50 @@ int xc_domain_getinfolist(int xc_handle, return ret; } +/* get info from hvm guest for save */ +int xc_domain_hvm_getcontext(int xc_handle, + uint32_t domid, + hvm_domain_context_t *hvm_ctxt) +{ + int rc; + DECLARE_DOMCTL; + + domctl.cmd = XEN_DOMCTL_gethvmcontext; + domctl.domain = (domid_t)domid; + set_xen_guest_handle(domctl.u.hvmcontext.ctxt, hvm_ctxt); + + if ( (rc = mlock(hvm_ctxt, sizeof(*hvm_ctxt))) != 0 ) + return rc; + + rc = do_domctl(xc_handle, &domctl); + + safe_munlock(hvm_ctxt, sizeof(*hvm_ctxt)); + + return rc; +} + +/* set info to hvm guest for restore */ +int xc_domain_hvm_setcontext(int xc_handle, + uint32_t domid, + hvm_domain_context_t *hvm_ctxt) +{ + int rc; + DECLARE_DOMCTL; + + domctl.cmd = XEN_DOMCTL_sethvmcontext; + domctl.domain = domid; + set_xen_guest_handle(domctl.u.hvmcontext.ctxt, hvm_ctxt); + + if ( (rc = mlock(hvm_ctxt, sizeof(*hvm_ctxt))) != 0 ) + return rc; + + rc = do_domctl(xc_handle, &domctl); + + safe_munlock(hvm_ctxt, sizeof(*hvm_ctxt)); + + return rc; +} + int xc_vcpu_getcontext(int xc_handle, uint32_t domid, uint32_t vcpu, diff --git a/tools/libxc/xc_hvm_restore.c b/tools/libxc/xc_hvm_restore.c new file mode 100644 index 0000000000..7092d5cea8 --- /dev/null +++ b/tools/libxc/xc_hvm_restore.c @@ -0,0 +1,360 @@ +/****************************************************************************** + * xc_hvm_restore.c + * + * Restore the state of a HVM guest. + * + * Copyright (c) 2003, K A Fraser. + * Copyright (c) 2006 Intel Corperation + * rewriten for hvm guest by Zhai Edwin + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA 02111-1307 USA. + * + */ + +#include +#include + +#include "xg_private.h" +#include "xg_save_restore.h" + +#include +#include +#include + +/* max mfn of the whole machine */ +static unsigned long max_mfn; + +/* virtual starting address of the hypervisor */ +static unsigned long hvirt_start; + +/* #levels of page tables used by the currrent guest */ +static unsigned int pt_levels; + +/* total number of pages used by the current guest */ +static unsigned long max_pfn; + +/* A table mapping each PFN to its new MFN. */ +static xen_pfn_t *p2m = NULL; + +static ssize_t +read_exact(int fd, void *buf, size_t count) +{ + int r = 0, s; + unsigned char *b = buf; + + while (r < count) { + s = read(fd, &b[r], count - r); + if ((s == -1) && (errno == EINTR)) + continue; + if (s <= 0) { + break; + } + r += s; + } + + return (r == count) ? 1 : 0; +} + +int xc_hvm_restore(int xc_handle, int io_fd, + uint32_t dom, unsigned long nr_pfns, + unsigned int store_evtchn, unsigned long *store_mfn, + unsigned int console_evtchn, unsigned long *console_mfn, + unsigned int pae, unsigned int apic) +{ + DECLARE_DOMCTL; + + /* The new domain's shared-info frame number. */ + unsigned long shared_info_frame; + + /* A copy of the CPU context of the guest. */ + vcpu_guest_context_t ctxt; + + char *region_base; + + unsigned long buf[PAGE_SIZE/sizeof(unsigned long)]; + + xc_dominfo_t info; + unsigned int rc = 1, n, i; + uint32_t rec_len, nr_vcpus; + hvm_domain_context_t hvm_ctxt; + unsigned long long v_end, memsize; + unsigned long shared_page_nr; + + unsigned long mfn, pfn; + unsigned int prev_pc, this_pc; + int verify = 0; + + /* Types of the pfns in the current region */ + unsigned long region_pfn_type[MAX_BATCH_SIZE]; + + /* hvm guest mem size (Mb) */ + memsize = (unsigned long long)*store_mfn; + v_end = memsize << 20; + + DPRINTF("xc_hvm_restore:dom=%d, nr_pfns=0x%lx, store_evtchn=%d, *store_mfn=%ld, console_evtchn=%d, *console_mfn=%ld, pae=%u, apic=%u.\n", + dom, nr_pfns, store_evtchn, *store_mfn, console_evtchn, *console_mfn, pae, apic); + + max_pfn = nr_pfns; + + if(!get_platform_info(xc_handle, dom, + &max_mfn, &hvirt_start, &pt_levels)) { + ERROR("Unable to get platform info."); + return 1; + } + + DPRINTF("xc_hvm_restore start: max_pfn = %lx, max_mfn = %lx, hvirt_start=%lx, pt_levels=%d\n", + max_pfn, + max_mfn, + hvirt_start, + pt_levels); + + if (mlock(&ctxt, sizeof(ctxt))) { + /* needed for build dom0 op, but might as well do early */ + ERROR("Unable to mlock ctxt"); + return 1; + } + + + p2m = malloc(max_pfn * sizeof(xen_pfn_t)); + + if (p2m == NULL) { + ERROR("memory alloc failed"); + errno = ENOMEM; + goto out; + } + + /* Get the domain's shared-info frame. */ + domctl.cmd = XEN_DOMCTL_getdomaininfo; + domctl.domain = (domid_t)dom; + if (xc_domctl(xc_handle, &domctl) < 0) { + ERROR("Could not get information on new domain"); + goto out; + } + shared_info_frame = domctl.u.getdomaininfo.shared_info_frame; + + if(xc_domain_setmaxmem(xc_handle, dom, PFN_TO_KB(max_pfn)) != 0) { + errno = ENOMEM; + goto out; + } + + for ( i = 0; i < max_pfn; i++ ) + p2m[i] = i; + for ( i = HVM_BELOW_4G_RAM_END >> PAGE_SHIFT; i < max_pfn; i++ ) + p2m[i] += HVM_BELOW_4G_MMIO_LENGTH >> PAGE_SHIFT; + + /* Allocate memory for HVM guest, skipping VGA hole 0xA0000-0xC0000. */ + rc = xc_domain_memory_populate_physmap( + xc_handle, dom, (max_pfn > 0xa0) ? 0xa0 : max_pfn, + 0, 0, &p2m[0x00]); + if ( (rc == 0) && (max_pfn > 0xc0) ) + rc = xc_domain_memory_populate_physmap( + xc_handle, dom, max_pfn - 0xc0, 0, 0, &p2m[0xc0]); + if ( rc != 0 ) + { + PERROR("Could not allocate memory for HVM guest.\n"); + goto out; + } + + + /**********XXXXXXXXXXXXXXXX******************/ + if (xc_domain_getinfo(xc_handle, dom, 1, &info) != 1) { + ERROR("Could not get domain info"); + return 1; + } + + domctl.cmd = XEN_DOMCTL_getdomaininfo; + domctl.domain = (domid_t)dom; + if (xc_domctl(xc_handle, &domctl) < 0) { + ERROR("Could not get information on new domain"); + goto out; + } + + for ( i = 0; i < max_pfn; i++) + p2m[i] = i; + + prev_pc = 0; + + n = 0; + while (1) { + + int j; + + this_pc = (n * 100) / max_pfn; + if ( (this_pc - prev_pc) >= 5 ) + { + PPRINTF("\b\b\b\b%3d%%", this_pc); + prev_pc = this_pc; + } + + if (!read_exact(io_fd, &j, sizeof(int))) { + ERROR("HVM restore Error when reading batch size"); + goto out; + } + + PPRINTF("batch %d\n",j); + + if (j == -1) { + verify = 1; + DPRINTF("Entering page verify mode\n"); + continue; + } + + if (j == 0) + break; /* our work here is done */ + + if (j > MAX_BATCH_SIZE) { + ERROR("Max batch size exceeded. Giving up."); + goto out; + } + + if (!read_exact(io_fd, region_pfn_type, j*sizeof(unsigned long))) { + ERROR("Error when reading region pfn types"); + goto out; + } + + region_base = xc_map_foreign_batch( + xc_handle, dom, PROT_WRITE, region_pfn_type, j); + + for ( i = 0; i < j; i++ ) + { + void *page; + + pfn = region_pfn_type[i]; + if ( pfn > max_pfn ) + { + ERROR("pfn out of range"); + goto out; + } + + if ( pfn >= 0xa0 && pfn < 0xc0) { + ERROR("hvm restore:pfn in vga hole"); + goto out; + } + + + mfn = p2m[pfn]; + + /* In verify mode, we use a copy; otherwise we work in place */ + page = verify ? (void *)buf : (region_base + i*PAGE_SIZE); + + if (!read_exact(io_fd, page, PAGE_SIZE)) { + ERROR("Error when reading page (%x)", i); + goto out; + } + + if (verify) { + + int res = memcmp(buf, (region_base + i*PAGE_SIZE), PAGE_SIZE); + + if (res) { + + int v; + + DPRINTF("************** pfn=%lx mfn=%lx gotcs=%08lx " + "actualcs=%08lx\n", pfn, p2m[pfn], + csum_page(region_base + i*PAGE_SIZE), + csum_page(buf)); + + for (v = 0; v < 4; v++) { + + unsigned long *p = (unsigned long *) + (region_base + i*PAGE_SIZE); + if (buf[v] != p[v]) + DPRINTF(" %d: %08lx %08lx\n", v, buf[v], p[v]); + } + } + } + + } /* end of 'batch' for loop */ + munmap(region_base, j*PAGE_SIZE); + n+= j; /* crude stats */ + + }/*while 1*/ + +/* xc_set_hvm_param(xc_handle, dom, HVM_PARAM_APIC_ENABLED, apic);*/ + xc_set_hvm_param(xc_handle, dom, HVM_PARAM_PAE_ENABLED, pae); + xc_set_hvm_param(xc_handle, dom, HVM_PARAM_STORE_EVTCHN, store_evtchn); + + if ( v_end > HVM_BELOW_4G_RAM_END ) + shared_page_nr = (HVM_BELOW_4G_RAM_END >> PAGE_SHIFT) - 1; + else + shared_page_nr = (v_end >> PAGE_SHIFT) - 1; + + xc_set_hvm_param(xc_handle, dom, HVM_PARAM_STORE_PFN, shared_page_nr-1); + xc_set_hvm_param(xc_handle, dom, HVM_PARAM_BUFIOREQ_PFN, shared_page_nr-2); + xc_set_hvm_param(xc_handle, dom, HVM_PARAM_IOREQ_PFN, shared_page_nr); + + /* caculate the store_mfn , wrong val cause hang when introduceDomain */ + *store_mfn = (v_end >> PAGE_SHIFT) - 2; + DPRINTF("hvm restore:calculate new store_mfn=0x%lx,v_end=0x%llx..\n", *store_mfn, v_end); + + /* restore hvm context including pic/pit/shpage */ + if (!read_exact(io_fd, &rec_len, sizeof(uint32_t))) { + ERROR("error read hvm context size!\n"); + goto out; + } + if (rec_len != sizeof(hvm_ctxt)) { + ERROR("hvm context size dismatch!\n"); + goto out; + } + + if (!read_exact(io_fd, &hvm_ctxt, sizeof(hvm_ctxt))) { + ERROR("error read hvm context!\n"); + goto out; + } + + if (( rc = xc_domain_hvm_setcontext(xc_handle, dom, &hvm_ctxt))) { + ERROR("error set hvm context!\n"); + goto out; + } + + if (!read_exact(io_fd, &nr_vcpus, sizeof(uint32_t))) { + ERROR("error read nr vcpu !\n"); + goto out; + } + DPRINTF("hvm restore:get nr_vcpus=%d.\n", nr_vcpus); + + for (i =0; i < nr_vcpus; i++) { + if (!read_exact(io_fd, &rec_len, sizeof(uint32_t))) { + ERROR("error read vcpu context size!\n"); + goto out; + } + if (rec_len != sizeof(ctxt)) { + ERROR("vcpu context size dismatch!\n"); + goto out; + } + + if (!read_exact(io_fd, &(ctxt), sizeof(ctxt))) { + ERROR("error read vcpu context.\n"); + goto out; + } + + if ( (rc = xc_vcpu_setcontext(xc_handle, dom, i, &ctxt)) ) { + ERROR("Could not set vcpu context, rc=%d", rc); + goto out; + } + } + + rc = 0; + goto out; + + out: + if ( (rc != 0) && (dom != 0) ) + xc_domain_destroy(xc_handle, dom); + free(p2m); + + DPRINTF("Restore exit with rc=%d\n", rc); + + return rc; +} diff --git a/tools/libxc/xc_hvm_save.c b/tools/libxc/xc_hvm_save.c new file mode 100644 index 0000000000..aa6ad5a930 --- /dev/null +++ b/tools/libxc/xc_hvm_save.c @@ -0,0 +1,727 @@ +/****************************************************************************** + * xc_hvm_save.c + * + * Save the state of a running HVM guest. + * + * Copyright (c) 2003, K A Fraser. + * Copyright (c) 2006 Intel Corperation + * rewriten for hvm guest by Zhai Edwin + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA 02111-1307 USA. + * + */ + +#include +#include +#include +#include +#include + +#include "xc_private.h" +#include "xg_private.h" +#include "xg_save_restore.h" + +/* +** Default values for important tuning parameters. Can override by passing +** non-zero replacement values to xc_hvm_save(). +** +** XXX SMH: should consider if want to be able to override MAX_MBIT_RATE too. +** +*/ +#define DEF_MAX_ITERS 29 /* limit us to 30 times round loop */ +#define DEF_MAX_FACTOR 3 /* never send more than 3x nr_pfns */ + +/* max mfn of the whole machine */ +static unsigned long max_mfn; + +/* virtual starting address of the hypervisor */ +static unsigned long hvirt_start; + +/* #levels of page tables used by the currrent guest */ +static unsigned int pt_levels; + +/* total number of pages used by the current guest */ +static unsigned long max_pfn; + +/* +** During (live) save/migrate, we maintain a number of bitmaps to track +** which pages we have to send, to fixup, and to skip. +*/ + +#define BITS_PER_LONG (sizeof(unsigned long) * 8) +#define BITMAP_SIZE ((max_pfn + BITS_PER_LONG - 1) / 8) + +#define BITMAP_ENTRY(_nr,_bmap) \ + ((unsigned long *)(_bmap))[(_nr)/BITS_PER_LONG] + +#define BITMAP_SHIFT(_nr) ((_nr) % BITS_PER_LONG) + +static inline int test_bit (int nr, volatile void * addr) +{ + return (BITMAP_ENTRY(nr, addr) >> BITMAP_SHIFT(nr)) & 1; +} + +static inline void clear_bit (int nr, volatile void * addr) +{ + BITMAP_ENTRY(nr, addr) &= ~(1UL << BITMAP_SHIFT(nr)); +} + +static inline int permute( int i, int nr, int order_nr ) +{ + /* Need a simple permutation function so that we scan pages in a + pseudo random order, enabling us to get a better estimate of + the domain's page dirtying rate as we go (there are often + contiguous ranges of pfns that have similar behaviour, and we + want to mix them up. */ + + /* e.g. nr->oder 15->4 16->4 17->5 */ + /* 512MB domain, 128k pages, order 17 */ + + /* + QPONMLKJIHGFEDCBA + QPONMLKJIH + GFEDCBA + */ + + /* + QPONMLKJIHGFEDCBA + EDCBA + QPONM + LKJIHGF + */ + + do { i = ((i>>(order_nr-10)) | ( i<<10 ) ) & ((1<= nr ); /* this won't ever loop if nr is a power of 2 */ + + return i; +} + +static uint64_t tv_to_us(struct timeval *new) +{ + return (new->tv_sec * 1000000) + new->tv_usec; +} + +static uint64_t llgettimeofday(void) +{ + struct timeval now; + gettimeofday(&now, NULL); + return tv_to_us(&now); +} + +static uint64_t tv_delta(struct timeval *new, struct timeval *old) +{ + return ((new->tv_sec - old->tv_sec)*1000000 ) + + (new->tv_usec - old->tv_usec); +} + + +#define RATE_IS_MAX() (0) +#define ratewrite(_io_fd, _buf, _n) write((_io_fd), (_buf), (_n)) +#define initialize_mbit_rate() + +static inline ssize_t write_exact(int fd, void *buf, size_t count) +{ + if(write(fd, buf, count) != count) + return 0; + return 1; +} + +static int print_stats(int xc_handle, uint32_t domid, int pages_sent, + xc_shadow_op_stats_t *stats, int print) +{ + static struct timeval wall_last; + static long long d0_cpu_last; + static long long d1_cpu_last; + + struct timeval wall_now; + long long wall_delta; + long long d0_cpu_now, d0_cpu_delta; + long long d1_cpu_now, d1_cpu_delta; + + gettimeofday(&wall_now, NULL); + + d0_cpu_now = xc_domain_get_cpu_usage(xc_handle, 0, /* FIXME */ 0)/1000; + d1_cpu_now = xc_domain_get_cpu_usage(xc_handle, domid, /* FIXME */ 0)/1000; + + if ( (d0_cpu_now == -1) || (d1_cpu_now == -1) ) + DPRINTF("ARRHHH!!\n"); + + wall_delta = tv_delta(&wall_now,&wall_last)/1000; + + if (wall_delta == 0) wall_delta = 1; + + d0_cpu_delta = (d0_cpu_now - d0_cpu_last)/1000; + d1_cpu_delta = (d1_cpu_now - d1_cpu_last)/1000; + + if (print) + DPRINTF( + "delta %lldms, dom0 %d%%, target %d%%, sent %dMb/s, " + "dirtied %dMb/s %" PRId32 " pages\n", + wall_delta, + (int)((d0_cpu_delta*100)/wall_delta), + (int)((d1_cpu_delta*100)/wall_delta), + (int)((pages_sent*PAGE_SIZE)/(wall_delta*(1000/8))), + (int)((stats->dirty_count*PAGE_SIZE)/(wall_delta*(1000/8))), + stats->dirty_count); + + d0_cpu_last = d0_cpu_now; + d1_cpu_last = d1_cpu_now; + wall_last = wall_now; + + return 0; +} + +static int analysis_phase(int xc_handle, uint32_t domid, int max_pfn, + unsigned long *arr, int runs) +{ + long long start, now; + xc_shadow_op_stats_t stats; + int j; + + start = llgettimeofday(); + + for (j = 0; j < runs; j++) { + int i; + + xc_shadow_control(xc_handle, domid, XEN_DOMCTL_SHADOW_OP_CLEAN, + arr, max_pfn, NULL, 0, NULL); + DPRINTF("#Flush\n"); + for ( i = 0; i < 40; i++ ) { + usleep(50000); + now = llgettimeofday(); + xc_shadow_control(xc_handle, domid, XEN_DOMCTL_SHADOW_OP_PEEK, + NULL, 0, NULL, 0, &stats); + + DPRINTF("now= %lld faults= %"PRId32" dirty= %"PRId32"\n", + ((now-start)+500)/1000, + stats.fault_count, stats.dirty_count); + } + } + + return -1; +} + +static int suspend_and_state(int (*suspend)(int), int xc_handle, int io_fd, + int dom, xc_dominfo_t *info, + vcpu_guest_context_t *ctxt) +{ + int i = 0; + + if (!(*suspend)(dom)) { + ERROR("Suspend request failed"); + return -1; + } + + retry: + + if (xc_domain_getinfo(xc_handle, dom, 1, info) != 1) { + ERROR("Could not get domain info"); + return -1; + } + + if ( xc_vcpu_getcontext(xc_handle, dom, 0 /* XXX */, ctxt)) + ERROR("Could not get vcpu context"); + + + if (info->shutdown && info->shutdown_reason == SHUTDOWN_suspend) + return 0; // success + + if (info->paused) { + // try unpausing domain, wait, and retest + xc_domain_unpause( xc_handle, dom ); + + ERROR("Domain was paused. Wait and re-test."); + usleep(10000); // 10ms + + goto retry; + } + + + if( ++i < 100 ) { + ERROR("Retry suspend domain."); + usleep(10000); // 10ms + goto retry; + } + + ERROR("Unable to suspend domain."); + + return -1; +} + +int xc_hvm_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters, + uint32_t max_factor, uint32_t flags, int (*suspend)(int)) +{ + xc_dominfo_t info; + + int rc = 1, i, last_iter, iter = 0; + int live = (flags & XCFLAGS_LIVE); + int debug = (flags & XCFLAGS_DEBUG); + int sent_last_iter, skip_this_iter; + + /* The new domain's shared-info frame number. */ + unsigned long shared_info_frame; + + /* A copy of the CPU context of the guest. */ + vcpu_guest_context_t ctxt; + + /* A table containg the type of each PFN (/not/ MFN!). */ + unsigned long *pfn_type = NULL; + unsigned long *pfn_batch = NULL; + + /* A copy of hvm domain context */ + hvm_domain_context_t hvm_ctxt; + + /* Live mapping of shared info structure */ + shared_info_t *live_shinfo = NULL; + + /* base of the region in which domain memory is mapped */ + unsigned char *region_base = NULL; + + uint32_t nr_pfns, rec_size, nr_vcpus; + unsigned long *page_array = NULL; + + /* power of 2 order of max_pfn */ + int order_nr; + + /* bitmap of pages: + - that should be sent this iteration (unless later marked as skip); + - to skip this iteration because already dirty; */ + unsigned long *to_send = NULL, *to_skip = NULL; + + xc_shadow_op_stats_t stats; + + unsigned long total_sent = 0; + + DPRINTF("xc_hvm_save:dom=%d, max_iters=%d, max_factor=%d, flags=0x%x, live=%d, debug=%d.\n", + dom, max_iters, max_factor, flags, + live, debug); + + /* If no explicit control parameters given, use defaults */ + if(!max_iters) + max_iters = DEF_MAX_ITERS; + if(!max_factor) + max_factor = DEF_MAX_FACTOR; + + initialize_mbit_rate(); + + if(!get_platform_info(xc_handle, dom, + &max_mfn, &hvirt_start, &pt_levels)) { + ERROR("HVM:Unable to get platform info."); + return 1; + } + + if (xc_domain_getinfo(xc_handle, dom, 1, &info) != 1) { + ERROR("HVM:Could not get domain info"); + return 1; + } + nr_vcpus = info.nr_online_vcpus; + + if (mlock(&ctxt, sizeof(ctxt))) { + ERROR("HVM:Unable to mlock ctxt"); + return 1; + } + + /* Only have to worry about vcpu 0 even for SMP */ + if (xc_vcpu_getcontext(xc_handle, dom, 0, &ctxt)) { + ERROR("HVM:Could not get vcpu context"); + goto out; + } + shared_info_frame = info.shared_info_frame; + + /* A cheesy test to see whether the domain contains valid state. */ + if (ctxt.ctrlreg[3] == 0) + { + ERROR("Domain is not in a valid HVM guest state"); + goto out; + } + + /* cheesy sanity check */ + if ((info.max_memkb >> (PAGE_SHIFT - 10)) > max_mfn) { + ERROR("Invalid HVM state record -- pfn count out of range: %lu", + (info.max_memkb >> (PAGE_SHIFT - 10))); + goto out; + } + + /* Map the shared info frame */ + if(!(live_shinfo = xc_map_foreign_range(xc_handle, dom, PAGE_SIZE, + PROT_READ, shared_info_frame))) { + ERROR("HVM:Couldn't map live_shinfo"); + goto out; + } + + max_pfn = live_shinfo->arch.max_pfn; + + DPRINTF("saved hvm domain info:max_memkb=0x%lx, max_mfn=0x%lx, nr_pages=0x%lx\n", info.max_memkb, max_mfn, info.nr_pages); + + /* nr_pfns: total pages excluding vga acc mem + * max_pfn: nr_pfns + 0x20 vga hole(0xa0~0xc0) + * getdomaininfo.tot_pages: all the allocated pages for this domain + */ + if (live) { + ERROR("hvm domain doesn't support live migration now.\n"); + goto out; + + if (xc_shadow_control(xc_handle, dom, + XEN_DOMCTL_SHADOW_OP_ENABLE_LOGDIRTY, + NULL, 0, NULL, 0, NULL) < 0) { + ERROR("Couldn't enable shadow mode"); + goto out; + } + + /* excludes vga acc mem */ + nr_pfns = info.nr_pages - 0x800; + + last_iter = 0; + DPRINTF("hvm domain live migration debug start: logdirty enable.\n"); + } else { + /* This is a non-live suspend. Issue the call back to get the + domain suspended */ + + last_iter = 1; + + /* suspend hvm domain */ + if (suspend_and_state(suspend, xc_handle, io_fd, dom, &info, &ctxt)) { + ERROR("HVM Domain appears not to have suspended"); + goto out; + } + nr_pfns = info.nr_pages; + DPRINTF("after suspend hvm domain nr_pages=0x%x.\n", nr_pfns); + } + + DPRINTF("after 1st handle hvm domain nr_pfns=0x%x, nr_pages=0x%lx, max_memkb=0x%lx, live=%d.\n", + nr_pfns, + info.nr_pages, + info.max_memkb, + live); + + nr_pfns = info.nr_pages; + + /*XXX: caculate the VGA hole*/ + max_pfn = nr_pfns + 0x20; + + skip_this_iter = 0;/*XXX*/ + /* pretend we sent all the pages last iteration */ + sent_last_iter = max_pfn; + + /* calculate the power of 2 order of max_pfn, e.g. + 15->4 16->4 17->5 */ + for (i = max_pfn-1, order_nr = 0; i ; i >>= 1, order_nr++) + continue; + + /* Setup to_send / to_fix and to_skip bitmaps */ + to_send = malloc(BITMAP_SIZE); + to_skip = malloc(BITMAP_SIZE); + + if (!to_send ||!to_skip) { + ERROR("Couldn't allocate to_send array"); + goto out; + } + + memset(to_send, 0xff, BITMAP_SIZE); + + if (lock_pages(to_send, BITMAP_SIZE)) { + ERROR("Unable to lock to_send"); + return 1; + } + + /* (to fix is local only) */ + if (lock_pages(to_skip, BITMAP_SIZE)) { + ERROR("Unable to lock to_skip"); + return 1; + } + + analysis_phase(xc_handle, dom, max_pfn, to_skip, 0); + + /* get all the HVM domain pfns */ + if ( (page_array = (unsigned long *) malloc (sizeof(unsigned long) * max_pfn)) == NULL) { + ERROR("HVM:malloc fail!\n"); + goto out; + } + + for ( i = 0; i < max_pfn; i++) + page_array[i] = i; + + + /* We want zeroed memory so use calloc rather than malloc. */ + pfn_type = calloc(MAX_BATCH_SIZE, sizeof(*pfn_type)); + pfn_batch = calloc(MAX_BATCH_SIZE, sizeof(*pfn_batch)); + + if ((pfn_type == NULL) || (pfn_batch == NULL)) { + ERROR("failed to alloc memory for pfn_type and/or pfn_batch arrays"); + errno = ENOMEM; + goto out; + } + + if (lock_pages(pfn_type, MAX_BATCH_SIZE * sizeof(*pfn_type))) { + ERROR("Unable to lock"); + goto out; + } + + /* Start writing out the saved-domain record. */ + if (!write_exact(io_fd, &max_pfn, sizeof(unsigned long))) { + ERROR("write: max_pfn"); + goto out; + } + + while(1) { + + unsigned int prev_pc, sent_this_iter, N, batch; + + iter++; + sent_this_iter = 0; + skip_this_iter = 0; + prev_pc = 0; + N=0; + + DPRINTF("Saving HVM domain memory pages: iter %d 0%%", iter); + + while( N < max_pfn ){ + + unsigned int this_pc = (N * 100) / max_pfn; + + if ((this_pc - prev_pc) >= 5) { + DPRINTF("\b\b\b\b%3d%%", this_pc); + prev_pc = this_pc; + } + + /* slightly wasteful to peek the whole array evey time, + but this is fast enough for the moment. */ + if (!last_iter && xc_shadow_control( + xc_handle, dom, XEN_DOMCTL_SHADOW_OP_PEEK, + to_skip, max_pfn, NULL, 0, NULL) != max_pfn) { + ERROR("Error peeking HVM shadow bitmap"); + goto out; + } + + + /* load pfn_type[] with the mfn of all the pages we're doing in + this batch. */ + for (batch = 0; batch < MAX_BATCH_SIZE && N < max_pfn ; N++) { + + int n = permute(N, max_pfn, order_nr); + + if (debug) { + DPRINTF("%d pfn= %08lx mfn= %08lx %d \n", + iter, (unsigned long)n, page_array[n], + test_bit(n, to_send)); + } + + if (!last_iter && test_bit(n, to_send)&& test_bit(n, to_skip)) + skip_this_iter++; /* stats keeping */ + + if (!((test_bit(n, to_send) && !test_bit(n, to_skip)) || + (test_bit(n, to_send) && last_iter))) + continue; + + if (n >= 0xa0 && n < 0xc0) { +/* DPRINTF("get a vga hole pfn= %x.\n", n);*/ + continue; + } + /* + ** we get here if: + ** 1. page is marked to_send & hasn't already been re-dirtied + ** 2. (ignore to_skip in last iteration) + */ + + pfn_batch[batch] = n; + pfn_type[batch] = page_array[n]; + + batch++; + } + + if (batch == 0) + goto skip; /* vanishingly unlikely... */ + + /* map_foreign use pfns now !*/ + if ((region_base = xc_map_foreign_batch( + xc_handle, dom, PROT_READ, pfn_batch, batch)) == 0) { + ERROR("map batch failed"); + goto out; + } + + /* write num of pfns */ + if(!write_exact(io_fd, &batch, sizeof(unsigned int))) { + ERROR("Error when writing to state file (2)"); + goto out; + } + + /* write all the pfns */ + if(!write_exact(io_fd, pfn_batch, sizeof(unsigned long)*batch)) { + ERROR("Error when writing to state file (3)"); + goto out; + } + + if (ratewrite(io_fd, region_base, PAGE_SIZE * batch) != PAGE_SIZE * batch) { + ERROR("ERROR when writting to state file (4)"); + goto out; + } + + + sent_this_iter += batch; + + munmap(region_base, batch*PAGE_SIZE); + + } /* end of this while loop for this iteration */ + + skip: + + total_sent += sent_this_iter; + + DPRINTF("\r %d: sent %d, skipped %d, ", + iter, sent_this_iter, skip_this_iter ); + + if (last_iter) { + print_stats( xc_handle, dom, sent_this_iter, &stats, 1); + + DPRINTF("Total pages sent= %ld (%.2fx)\n", + total_sent, ((float)total_sent)/max_pfn ); + } + + if (last_iter && debug){ + int minusone = -1; + memset(to_send, 0xff, BITMAP_SIZE); + debug = 0; + DPRINTF("Entering debug resend-all mode\n"); + + /* send "-1" to put receiver into debug mode */ + if(!write_exact(io_fd, &minusone, sizeof(int))) { + ERROR("Error when writing to state file (6)"); + goto out; + } + + continue; + } + + if (last_iter) break; + + if (live) { + + + if( + ((sent_this_iter > sent_last_iter) && RATE_IS_MAX()) || + (iter >= max_iters) || + (sent_this_iter+skip_this_iter < 50) || + (total_sent > max_pfn*max_factor) ) { + + DPRINTF("Start last iteration for HVM domain\n"); + last_iter = 1; + + if (suspend_and_state(suspend, xc_handle, io_fd, dom, &info, + &ctxt)) { + ERROR("Domain appears not to have suspended"); + goto out; + } + + DPRINTF("SUSPEND shinfo %08lx eip %08lx edx %08lx\n", + info.shared_info_frame, + (unsigned long)ctxt.user_regs.eip, + (unsigned long)ctxt.user_regs.edx); + } + + if (xc_shadow_control(xc_handle, dom, + XEN_DOMCTL_SHADOW_OP_CLEAN, to_send, + max_pfn, NULL, 0, &stats) != max_pfn) { + ERROR("Error flushing shadow PT"); + goto out; + } + + sent_last_iter = sent_this_iter; + + print_stats(xc_handle, dom, sent_this_iter, &stats, 1); + + } + + + } /* end of while 1 */ + + + DPRINTF("All HVM memory is saved\n"); + + /* Zero terminate */ + i = 0; + if (!write_exact(io_fd, &i, sizeof(int))) { + ERROR("Error when writing to state file (6)"); + goto out; + } + + /* save hvm hypervisor state including pic/pit/shpage */ + if (mlock(&hvm_ctxt, sizeof(hvm_ctxt))) { + ERROR("Unable to mlock ctxt"); + return 1; + } + + if (xc_domain_hvm_getcontext(xc_handle, dom, &hvm_ctxt)){ + ERROR("HVM:Could not get hvm context"); + goto out; + } + + rec_size = sizeof(hvm_ctxt); + if (!write_exact(io_fd, &rec_size, sizeof(uint32_t))) { + ERROR("error write hvm ctxt size"); + goto out; + } + + if ( !write_exact(io_fd, &hvm_ctxt, sizeof(hvm_ctxt)) ) { + ERROR("write HVM info failed!\n"); + } + + /* save vcpu/vmcs context */ + if (!write_exact(io_fd, &nr_vcpus, sizeof(uint32_t))) { + ERROR("error write nr vcpus"); + goto out; + } + + /*XXX: need a online map to exclude down cpu */ + for (i = 0; i < nr_vcpus; i++) { + + if (xc_vcpu_getcontext(xc_handle, dom, i, &ctxt)) { + ERROR("HVM:Could not get vcpu context"); + goto out; + } + + rec_size = sizeof(ctxt); + DPRINTF("write %d vcpucontext of total %d.\n", i, nr_vcpus); + if (!write_exact(io_fd, &rec_size, sizeof(uint32_t))) { + ERROR("error write vcpu ctxt size"); + goto out; + } + + if (!write_exact(io_fd, &(ctxt), sizeof(ctxt)) ) { + ERROR("write vmcs failed!\n"); + goto out; + } + } + + /* Success! */ + rc = 0; + + out: + + if (live) { + if(xc_shadow_control(xc_handle, dom, + XEN_DOMCTL_SHADOW_OP_OFF, + NULL, 0, NULL, 0, NULL) < 0) { + DPRINTF("Warning - couldn't disable shadow mode"); + } + } + + free(page_array); + + free(pfn_type); + free(pfn_batch); + free(to_send); + free(to_skip); + + return !!rc; +} diff --git a/tools/libxc/xenctrl.h b/tools/libxc/xenctrl.h index 71d000ec08..a5df681e3e 100644 --- a/tools/libxc/xenctrl.h +++ b/tools/libxc/xenctrl.h @@ -312,6 +312,30 @@ int xc_domain_getinfolist(int xc_handle, unsigned int max_domains, xc_domaininfo_t *info); +/** + * This function returns information about the context of a hvm domain + * @parm xc_handle a handle to an open hypervisor interface + * @parm domid the domain to get information from + * @parm hvm_ctxt a pointer to a structure to store the execution context of the + * hvm domain + * @return 0 on success, -1 on failure + */ +int xc_domain_hvm_getcontext(int xc_handle, + uint32_t domid, + hvm_domain_context_t *hvm_ctxt); + +/** + * This function will set the context for hvm domain + * + * @parm xc_handle a handle to an open hypervisor interface + * @parm domid the domain to set the hvm domain context for + * @parm hvm_ctxt pointer to the the hvm context with the values to set + * @return 0 on success, -1 on failure + */ +int xc_domain_hvm_setcontext(int xc_handle, + uint32_t domid, + hvm_domain_context_t *hvm_ctxt); + /** * This function returns information about the execution context of a * particular vcpu of a domain. diff --git a/tools/libxc/xenguest.h b/tools/libxc/xenguest.h index 6d4576026d..7b5a2af0a4 100644 --- a/tools/libxc/xenguest.h +++ b/tools/libxc/xenguest.h @@ -11,6 +11,7 @@ #define XCFLAGS_LIVE 1 #define XCFLAGS_DEBUG 2 +#define XCFLAGS_HVM 4 /** @@ -25,6 +26,13 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters, uint32_t max_factor, uint32_t flags /* XCFLAGS_xxx */, int (*suspend)(int domid)); +/** + * This function will save a hvm domain running unmodified guest. + * @return 0 on success, -1 on failure + */ +int xc_hvm_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters, + uint32_t max_factor, uint32_t flags /* XCFLAGS_xxx */, + int (*suspend)(int domid)); /** * This function will restore a saved domain running Linux. @@ -42,6 +50,18 @@ int xc_linux_restore(int xc_handle, int io_fd, uint32_t dom, unsigned long *store_mfn, unsigned int console_evtchn, unsigned long *console_mfn); +/** + * This function will restore a saved hvm domain running unmodified guest. + * + * @parm store_mfn pass mem size & returned with the mfn of the store page + * @return 0 on success, -1 on failure + */ +int xc_hvm_restore(int xc_handle, int io_fd, uint32_t dom, + unsigned long nr_pfns, unsigned int store_evtchn, + unsigned long *store_mfn, unsigned int console_evtchn, + unsigned long *console_mfn, + unsigned int pae, unsigned int apic); + /** * This function will create a domain for a paravirtualized Linux * using file names pointing to kernel and ramdisk diff --git a/tools/python/setup.py b/tools/python/setup.py index 56dd3e4a0b..7b25025d9d 100644 --- a/tools/python/setup.py +++ b/tools/python/setup.py @@ -30,6 +30,13 @@ xs = Extension("xs", libraries = libraries, sources = [ "xen/lowlevel/xs/xs.c" ]) +scf = Extension("scf", + extra_compile_args = extra_compile_args, + include_dirs = include_dirs + [ "xen/lowlevel/scf" ], + library_dirs = library_dirs, + libraries = libraries, + sources = [ "xen/lowlevel/scf/scf.c" ]) + acm = Extension("acm", extra_compile_args = extra_compile_args, include_dirs = include_dirs + [ "xen/lowlevel/acm" ], @@ -37,6 +44,10 @@ acm = Extension("acm", libraries = libraries, sources = [ "xen/lowlevel/acm/acm.c" ]) +modules = [ xc, xs, acm ] +if os.uname()[0] == 'SunOS': + modules.append(scf) + setup(name = 'xen', version = '3.0', description = 'Xen', @@ -56,7 +67,7 @@ setup(name = 'xen', 'xen.xm.tests' ], ext_package = "xen.lowlevel", - ext_modules = [ xc, xs, acm ] + ext_modules = modules ) os.chdir('logging') diff --git a/tools/python/xen/lowlevel/scf/scf.c b/tools/python/xen/lowlevel/scf/scf.c new file mode 100644 index 0000000000..a73ac20419 --- /dev/null +++ b/tools/python/xen/lowlevel/scf/scf.c @@ -0,0 +1,156 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include + +#include +#include + +#define XEND_FMRI "svc:/system/xen/xend:default" +#define XEND_PG "config" + +static PyObject *scf_exc; + +static void * +scf_exception(const char *err, const char *value) +{ + int scferr = scf_error(); + const char *scfstrerr = scf_strerror(scferr); + PyObject *obj = Py_BuildValue("(isss)", scferr, err, scfstrerr, value); + PyErr_SetObject(scf_exc, obj); + return (NULL); +} + +static PyObject * +pyscf_get_bool(PyObject *o, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = { "name", NULL }; + scf_simple_prop_t *prop; + uint8_t *val; + char *name; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s", kwlist, &name)) + return (NULL); + + prop = scf_simple_prop_get(NULL, XEND_FMRI, XEND_PG, name); + + if (prop == NULL) + return (scf_exception("scf_simple_prop_get() failed", name)); + + if ((val = scf_simple_prop_next_boolean(prop)) == NULL) + return (scf_exception("scf_simple_prop_next_boolean() failed", + name)); + + if (*val) { + scf_simple_prop_free(prop); + Py_INCREF(Py_True); + return (Py_True); + } + + scf_simple_prop_free(prop); + Py_INCREF(Py_False); + return (Py_False); +} + +static PyObject * +pyscf_get_int(PyObject *o, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = { "name", NULL }; + scf_simple_prop_t *prop; + PyObject *obj; + int64_t *val; + char *name; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s", kwlist, &name)) + return (NULL); + + prop = scf_simple_prop_get(NULL, XEND_FMRI, XEND_PG, name); + + if (prop == NULL) + return (scf_exception("scf_simple_prop_get() failed", name)); + + if ((val = scf_simple_prop_next_integer(prop)) == NULL) + return (scf_exception("scf_simple_prop_next_integer() failed", + name)); + + obj = PyInt_FromLong((long)*val); + scf_simple_prop_free(prop); + return (obj); +} + +static PyObject * +pyscf_get_string(PyObject *o, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = { "name", NULL }; + scf_simple_prop_t *prop; + PyObject *obj; + char *name; + char *str; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s", kwlist, &name)) + return (NULL); + + prop = scf_simple_prop_get(NULL, XEND_FMRI, XEND_PG, name); + + if (prop == NULL) + return (scf_exception("scf_simple_prop_get() failed", name)); + + if ((str = scf_simple_prop_next_astring(prop)) == NULL) { + scf_simple_prop_free(prop); + return (scf_exception("scf_simple_prop_next_astring() failed", + name)); + } + + obj = PyString_FromString(str); + scf_simple_prop_free(prop); + return (obj); +} + +PyDoc_STRVAR(pyscf_get_bool__doc__, + "get_bool(name) - get the value of the named boolean property"); +PyDoc_STRVAR(pyscf_get_int__doc__, + "get_int(name) - get the value of the named integer property"); +PyDoc_STRVAR(pyscf_get_string__doc__, + "get_string(name) - get the value of the named string property"); + +static struct PyMethodDef pyscf_module_methods[] = { + { "get_bool", (PyCFunction) pyscf_get_bool, + METH_VARARGS|METH_KEYWORDS, pyscf_get_bool__doc__ }, + { "get_int", (PyCFunction) pyscf_get_int, + METH_VARARGS|METH_KEYWORDS, pyscf_get_int__doc__ }, + { "get_string", (PyCFunction) pyscf_get_string, + METH_VARARGS|METH_KEYWORDS, pyscf_get_string__doc__ }, + { NULL, NULL, 0, NULL } +}; + +PyMODINIT_FUNC +initscf(void) +{ + PyObject *m; + m = Py_InitModule("scf", pyscf_module_methods); + + scf_exc = PyErr_NewException("scf.error", NULL, NULL); + Py_INCREF(scf_exc); + PyModule_AddObject(m, "error", scf_exc); + PyModule_AddIntConstant(m, "SCF_ERROR_NOT_FOUND", SCF_ERROR_NOT_FOUND); +} diff --git a/tools/python/xen/lowlevel/xc/xc.c b/tools/python/xen/lowlevel/xc/xc.c index eaa8fe0846..fb6ee675d9 100644 --- a/tools/python/xen/lowlevel/xc/xc.c +++ b/tools/python/xen/lowlevel/xc/xc.c @@ -160,6 +160,20 @@ static PyObject *pyxc_domain_destroy(XcObject *self, PyObject *args) return dom_op(self, args, xc_domain_destroy); } +static PyObject *pyxc_domain_shutdown(XcObject *self, PyObject *args) +{ + uint32_t dom, reason; + + if (!PyArg_ParseTuple(args, "ii", &dom, &reason)) + return NULL; + + if (xc_domain_shutdown(self->xc_handle, dom, reason) != 0) + return pyxc_error_to_exception(); + + Py_INCREF(zero); + return zero; +} + static PyObject *pyxc_vcpu_setaffinity(XcObject *self, PyObject *args, @@ -1029,6 +1043,14 @@ static PyMethodDef pyxc_methods[] = { " dom [int]: Identifier of domain to be destroyed.\n\n" "Returns: [int] 0 on success; -1 on error.\n" }, + { "domain_shutdown", + (PyCFunction)pyxc_domain_shutdown, + METH_VARARGS, "\n" + "Shutdown a domain.\n" + " dom [int, 0]: Domain identifier to use.\n" + " reason [int, 0]: Reason for shutdown.\n" + "Returns: [int] 0 on success; -1 on error.\n" }, + { "vcpu_setaffinity", (PyCFunction)pyxc_vcpu_setaffinity, METH_VARARGS | METH_KEYWORDS, "\n" diff --git a/tools/python/xen/util/xmlrpclib2.py b/tools/python/xen/util/xmlrpclib2.py index 3ba1b259b1..ac95d4cf39 100644 --- a/tools/python/xen/util/xmlrpclib2.py +++ b/tools/python/xen/util/xmlrpclib2.py @@ -256,6 +256,7 @@ class UnixXMLRPCRequestHandler(XMLRPCRequestHandler): class UnixXMLRPCServer(TCPXMLRPCServer): address_family = socket.AF_UNIX + allow_address_reuse = True def __init__(self, addr, allowed, xenapi, logRequests = 1): mkdir.parents(os.path.dirname(addr), stat.S_IRWXU, True) diff --git a/tools/python/xen/web/httpserver.py b/tools/python/xen/web/httpserver.py index 2326a2b875..14965e418f 100644 --- a/tools/python/xen/web/httpserver.py +++ b/tools/python/xen/web/httpserver.py @@ -294,8 +294,6 @@ class HttpServer: backlog = 5 - closed = False - def __init__(self, root, interface, port=8080): self.root = root self.interface = interface @@ -303,6 +301,7 @@ class HttpServer: # ready indicates when we are ready to begin accept connections # it should be set after a successful bind self.ready = False + self.closed = False def run(self): self.bind() @@ -317,7 +316,6 @@ class HttpServer: def stop(self): self.close() - def bind(self): self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) flags = fcntl.fcntl(self.socket.fileno(), fcntl.F_GETFD) @@ -334,7 +332,10 @@ class HttpServer: def close(self): self.closed = True + self.ready = False try: + # shutdown socket explicitly to allow reuse + self.socket.shutdown(socket.SHUT_RDWR) self.socket.close() except: pass @@ -345,6 +346,9 @@ class HttpServer: def getResource(self, req): return self.root.getRequestResource(req) + def shutdown(self): + self.close() + class UnixHttpServer(HttpServer): diff --git a/tools/python/xen/xend/Vifctl.py b/tools/python/xen/xend/Vifctl.py index 9320930b27..ba2105098c 100644 --- a/tools/python/xen/xend/Vifctl.py +++ b/tools/python/xen/xend/Vifctl.py @@ -20,7 +20,7 @@ """ import os -import XendRoot +import XendOptions def network(op): @@ -30,7 +30,7 @@ def network(op): """ if op not in ['start', 'stop']: raise ValueError('Invalid operation: ' + op) - script = XendRoot.instance().get_network_script() + script = XendOptions.instance().get_network_script() if script: script.insert(1, op) os.spawnv(os.P_WAIT, script[0], script) diff --git a/tools/python/xen/xend/XendCheckpoint.py b/tools/python/xen/xend/XendCheckpoint.py index 106e20fb3d..1aa2d0bc88 100644 --- a/tools/python/xen/xend/XendCheckpoint.py +++ b/tools/python/xen/xend/XendCheckpoint.py @@ -22,11 +22,14 @@ from xen.xend.XendConfig import XendConfig from xen.xend.XendConstants import * SIGNATURE = "LinuxGuestRecord" +QEMU_SIGNATURE = "QemuDeviceModelRecord" +dm_batch = 512 XC_SAVE = "xc_save" XC_RESTORE = "xc_restore" sizeof_int = calcsize("i") +sizeof_unsigned_int = calcsize("I") sizeof_unsigned_long = calcsize("L") @@ -69,6 +72,11 @@ def save(fd, dominfo, network, live, dst): "could not write guest state file: config len") write_exact(fd, config, "could not write guest state file: config") + image_cfg = dominfo.info.get('image', {}) + hvm = image_cfg.has_key('hvm') + + if hvm: + log.info("save hvm domain") # xc_save takes three customization parameters: maxit, max_f, and # flags the last controls whether or not save is 'live', while the # first two further customize behaviour when 'live' save is @@ -76,7 +84,7 @@ def save(fd, dominfo, network, live, dst): # libxenguest; see the comments and/or code in xc_linux_save() for # more information. cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd), - str(dominfo.getDomid()), "0", "0", str(int(live)) ] + str(dominfo.getDomid()), "0", "0", str(int(live) | (int(hvm) << 2)) ] log.debug("[xc_save]: %s", string.join(cmd)) def saveInputHandler(line, tochild): @@ -90,12 +98,29 @@ def save(fd, dominfo, network, live, dst): log.info("Domain %d suspended.", dominfo.getDomid()) dominfo.migrateDevices(network, dst, DEV_MIGRATE_STEP3, domain_name) + #send signal to device model for save + if hvm: + log.info("release_devices for hvm domain") + dominfo._releaseDevices(True) tochild.write("done\n") tochild.flush() log.debug('Written done') forkHelper(cmd, fd, saveInputHandler, False) + # put qemu device model state + if hvm: + write_exact(fd, QEMU_SIGNATURE, "could not write qemu signature") + qemu_fd = os.open("/tmp/xen.qemu-dm.%d" % dominfo.getDomid(), os.O_RDONLY) + while True: + buf = os.read(qemu_fd, dm_batch) + if len(buf): + write_exact(fd, buf, "could not write device model state") + else: + break + os.close(qemu_fd) + os.remove("/tmp/xen.qemu-dm.%d" % dominfo.getDomid()) + dominfo.destroyDomain() try: dominfo.setName(domain_name) @@ -149,19 +174,49 @@ def restore(xd, fd, dominfo = None, paused = False): nr_pfns = (dominfo.getMemoryTarget() + 3) / 4 + # if hvm, pass mem size to calculate the store_mfn + hvm = 0 + apic = 0 + pae = 0 + image_cfg = dominfo.info.get('image', {}) + hvm = image_cfg.has_key('hvm') + if hvm: + # the 'memory' in config has been removed + hvm = dominfo.info['memory_static_min'] + apic = dominfo.info['image']['hvm'].get('apic', 0) + pae = dominfo.info['image']['hvm'].get('pae', 0) + log.info("restore hvm domain %d, mem=%d, apic=%d, pae=%d", + dominfo.domid, hvm, apic, pae) + try: - l = read_exact(fd, sizeof_unsigned_long, - "not a valid guest state file: pfn count read") + if hvm: + l = read_exact(fd, sizeof_unsigned_int, + "not a valid hvm guest state file: pfn count read") + nr_pfns = unpack("I", l)[0] # native sizeof int + else: + l = read_exact(fd, sizeof_unsigned_long, + "not a valid guest state file: pfn count read") + max_pfn = unpack("L", l)[0] # native sizeof long if max_pfn > 16*1024*1024: # XXX raise XendError( "not a valid guest state file: pfn count out of range") - balloon.free(xc.pages_to_kib(nr_pfns)) + shadow = dominfo.info['shadow_memory'] + log.debug("restore:shadow=0x%x, _static_max=0x%x, _static_min=0x%x, " + "nr_pfns=0x%x.", dominfo.info['shadow_memory'], + dominfo.info['memory_static_max'], + dominfo.info['memory_static_min'], nr_pfns) + + + balloon.free(xc.pages_to_kib(nr_pfns) + shadow * 1024) + + shadow_cur = xc.shadow_mem_control(dominfo.getDomid(), shadow) + dominfo.info['shadow_memory'] = shadow_cur cmd = map(str, [xen.util.auxbin.pathTo(XC_RESTORE), fd, dominfo.getDomid(), max_pfn, - store_port, console_port]) + store_port, console_port, hvm, pae, apic]) log.debug("[xc_restore]: %s", string.join(cmd)) handler = RestoreInputHandler() @@ -171,10 +226,30 @@ def restore(xd, fd, dominfo = None, paused = False): if handler.store_mfn is None or handler.console_mfn is None: raise XendError('Could not read store/console MFN') - os.read(fd, 1) # Wait for source to close connection dominfo.waitForDevices() # Wait for backends to set up if not paused: dominfo.unpause() + + # get qemu state and create a tmp file for dm restore + if hvm: + qemu_signature = read_exact(fd, len(QEMU_SIGNATURE), + "invalid device model signature read") + if qemu_signature != QEMU_SIGNATURE: + raise XendError("not a valid device model state: found '%s'" % + qemu_signature) + qemu_fd = os.open("/tmp/xen.qemu-dm.%d" % dominfo.getDomid(), + os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + while True: + buf = os.read(fd, dm_batch) + if len(buf): + write_exact(qemu_fd, buf, + "could not write dm state to tmp file") + else: + break + os.close(qemu_fd) + + + os.read(fd, 1) # Wait for source to close connection dominfo.completeRestore(handler.store_mfn, handler.console_mfn) diff --git a/tools/python/xen/xend/XendConfig.py b/tools/python/xen/xend/XendConfig.py index fd9afb0b3a..1098417542 100644 --- a/tools/python/xen/xend/XendConfig.py +++ b/tools/python/xen/xend/XendConfig.py @@ -78,6 +78,25 @@ def scrub_password(data): else: return data +# +# CPU fields: +# +# vcpus_number -- the maximum number of vcpus that this domain may ever have. +# aka XendDomainInfo.getVCpuCount(). +# vcpus -- the legacy configuration name for above. +# max_vcpu_id -- vcpus_number - 1. This is given to us by Xen. +# +# cpus -- the list of pCPUs available to each vCPU. +# +# vcpu_avail: a bitmap telling the guest domain whether it may use each of +# its VCPUs. This is translated to +# /cpu//availability = {online,offline} for use +# by the guest domain. +# online_vpcus -- the number of VCPUs currently up, as reported by Xen. This +# is changed by changing vcpu_avail, and waiting for the +# domain to respond. +# + # Mapping from XendConfig configuration keys to the old # legacy configuration keys that map directly. @@ -185,7 +204,7 @@ LEGACY_CFG_TYPES = { 'uuid': str, 'name': str, 'vcpus': int, - 'vcpu_avail': int, + 'vcpu_avail': long, 'memory': int, 'shadow_memory': int, 'maxmem': int, @@ -355,9 +374,6 @@ class XendConfig(dict): 'cpu_weight': 256, 'cpu_cap': 0, 'vcpus_number': 1, - 'online_vcpus': 1, - 'max_vcpu_id': 0, - 'vcpu_avail': 1, 'console_refs': [], 'vif_refs': [], 'vbd_refs': [], @@ -389,7 +405,7 @@ class XendConfig(dict): event) def _vcpus_sanity_check(self): - if self.get('vcpus_number') != None: + if 'vcpus_number' in self and 'vcpu_avail' not in self: self['vcpu_avail'] = (1 << self['vcpus_number']) - 1 def _uuid_sanity_check(self): @@ -405,7 +421,7 @@ class XendConfig(dict): def _dominfo_to_xapi(self, dominfo): self['domid'] = dominfo['domid'] self['online_vcpus'] = dominfo['online_vcpus'] - self['max_vcpu_id'] = dominfo['max_vcpu_id'] + self['vcpus_number'] = dominfo['max_vcpu_id'] + 1 self['memory_dynamic_min'] = (dominfo['mem_kb'] + 1023)/1024 self['memory_dynamic_max'] = (dominfo['maxmem_kb'] + 1023)/1024 self['cpu_time'] = dominfo['cpu_time']/1e9 @@ -636,9 +652,6 @@ class XendConfig(dict): self['memory_dynamic_max'] = self['memory_static_max'] self['memory_dynamic_min'] = self['memory_static_min'] - # make sure max_vcpu_id is set correctly - self['max_vcpu_id'] = self['vcpus_number'] - 1 - # set device references in the configuration self['devices'] = cfg.get('devices', {}) @@ -720,13 +733,11 @@ class XendConfig(dict): _set_cfg_if_exists('on_xend_stop') _set_cfg_if_exists('on_xend_start') _set_cfg_if_exists('vcpu_avail') - _set_cfg_if_exists('max_vcpu_id') # needed for vcpuDomDetails _set_cfg_if_exists('cpu_weight') _set_cfg_if_exists('cpu_cap') # Parse and store runtime configuration _set_cfg_if_exists('start_time') - _set_cfg_if_exists('online_vcpus') _set_cfg_if_exists('cpu_time') _set_cfg_if_exists('shutdown_reason') _set_cfg_if_exists('up_time') @@ -1115,19 +1126,17 @@ class XendConfig(dict): # configuration log.debug("update_with_image_sxp(%s)" % scrub_password(image_sxp)) - kernel_args = "" + # user-specified args must come last: previous releases did this and + # some domU kernels rely upon the ordering. + kernel_args = sxp.child_value(image_sxp, 'args', '') # attempt to extract extra arguments from SXP config arg_ip = sxp.child_value(image_sxp, 'ip') if arg_ip and not re.search(r'ip=[^ ]+', kernel_args): - kernel_args += 'ip=%s ' % arg_ip + kernel_args = 'ip=%s ' % arg_ip + kernel_args arg_root = sxp.child_value(image_sxp, 'root') if arg_root and not re.search(r'root=', kernel_args): - kernel_args += 'root=%s ' % arg_root - - # user-specified args must come last: previous releases did this and - # some domU kernels rely upon the ordering. - kernel_args += sxp.child_value(image_sxp, 'args', '') + kernel_args = 'root=%s ' % arg_root + kernel_args if bootloader: self['_temp_using_bootloader'] = '1' diff --git a/tools/python/xen/xend/XendDomain.py b/tools/python/xen/xend/XendDomain.py index 4257057398..05d669de08 100644 --- a/tools/python/xen/xend/XendDomain.py +++ b/tools/python/xen/xend/XendDomain.py @@ -32,7 +32,7 @@ import threading import xen.lowlevel.xc -from xen.xend import XendRoot, XendCheckpoint, XendDomainInfo +from xen.xend import XendOptions, XendCheckpoint, XendDomainInfo from xen.xend.PrettyPrint import prettyprint from xen.xend.XendConfig import XendConfig from xen.xend.XendError import XendError, XendInvalidDomain, VmError @@ -51,7 +51,7 @@ from xen.util import mkdir, security from xen.xend import uuid xc = xen.lowlevel.xc.xc() -xroot = XendRoot.instance() +xoptions = XendOptions.instance() __all__ = [ "XendDomain" ] @@ -214,7 +214,7 @@ class XendDomain: @rtype: String @return: Path. """ - dom_path = xroot.get_xend_domains_path() + dom_path = xoptions.get_xend_domains_path() if domuuid: dom_path = os.path.join(dom_path, domuuid) return dom_path @@ -361,7 +361,7 @@ class XendDomain: def _setDom0CPUCount(self): """Sets the number of VCPUs dom0 has. Retreived from the - Xend configuration, L{XendRoot}. + Xend configuration, L{XendOptions}. @requires: Expects to be protected by domains_lock. @rtype: None @@ -369,7 +369,7 @@ class XendDomain: dom0 = self.privilegedDomain() # get max number of vcpus to use for dom0 from config - target = int(xroot.get_dom0_vcpus()) + target = int(xoptions.get_dom0_vcpus()) log.debug("number of vcpus to use is %d", target) # target == 0 means use all processors @@ -1164,7 +1164,7 @@ class XendDomain: dominfo.checkLiveMigrateMemory() if port == 0: - port = xroot.get_xend_relocation_port() + port = xoptions.get_xend_relocation_port() try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((dst, port)) diff --git a/tools/python/xen/xend/XendDomainInfo.py b/tools/python/xen/xend/XendDomainInfo.py index f80c3f5be7..27cba0049c 100644 --- a/tools/python/xen/xend/XendDomainInfo.py +++ b/tools/python/xen/xend/XendDomainInfo.py @@ -38,7 +38,7 @@ from xen.util.blkif import blkdev_uname_to_file from xen.util import security from xen.xend import balloon, sxp, uuid, image, arch, osdep -from xen.xend import XendRoot, XendNode, XendConfig +from xen.xend import XendOptions, XendNode, XendConfig from xen.xend.XendConfig import scrub_password from xen.xend.XendBootloader import bootloader @@ -54,31 +54,12 @@ MIGRATE_TIMEOUT = 30.0 BOOTLOADER_LOOPBACK_DEVICE = '/dev/xvdp' xc = xen.lowlevel.xc.xc() -xroot = XendRoot.instance() +xoptions = XendOptions.instance() log = logging.getLogger("xend.XendDomainInfo") #log.setLevel(logging.TRACE) -# -# There are a number of CPU-related fields: -# -# vcpus: the number of virtual CPUs this domain is configured to use. -# vcpu_avail: a bitmap telling the guest domain whether it may use each of -# its VCPUs. This is translated to -# /cpu//availability = {online,offline} for use -# by the guest domain. -# cpumap: a list of bitmaps, one for each VCPU, giving the physical -# CPUs that that VCPU may use. -# cpu: a configuration setting requesting that VCPU 0 is pinned to -# the specified physical CPU. -# -# vcpus and vcpu_avail settings persist with the VM (i.e. they are persistent -# across save, restore, migrate, and restart). The other settings are only -# specific to the domain, so are lost when the VM moves. -# - - def create(config): """Creates and start a VM using the supplied configuration. @@ -451,6 +432,16 @@ class XendDomainInfo: self._removeVm('xend/previous_restart_time') self.storeDom("control/shutdown", reason) + ## shutdown hypercall for hvm domain desides xenstore write + image_cfg = self.info.get('image', {}) + hvm = image_cfg.has_key('hvm') + if hvm: + for code in DOMAIN_SHUTDOWN_REASONS.keys(): + if DOMAIN_SHUTDOWN_REASONS[code] == reason: + break + xc.domain_shutdown(self.domid, code) + + def pause(self): """Pause domain @@ -614,7 +605,7 @@ class XendDomainInfo: ['name', self.info['name_label']], ['vcpu_count', self.info['vcpus_number']]] - for i in range(0, self.info['max_vcpu_id']+1): + for i in range(0, self.info['vcpus_number']): info = xc.vcpu_getinfo(self.domid, i) sxpr.append(['vcpu', @@ -739,7 +730,7 @@ class XendDomainInfo: 'domid': str(self.domid), 'vm': self.vmpath, 'name': self.info['name_label'], - 'console/limit': str(xroot.get_console_limit() * 1024), + 'console/limit': str(xoptions.get_console_limit() * 1024), 'memory/target': str(self.info['memory_static_min'] * 1024) } @@ -898,8 +889,9 @@ class XendDomainInfo: self._writeDom(self._vcpuDomDetails()) else: self.info['vcpus_number'] = vcpus - self.info['online_vcpus'] = vcpus xen.xend.XendDomain.instance().managed_config_save(self) + log.info("Set VCPU count on domain %s to %d", self.info['name_label'], + vcpus) def getLabel(self): return security.get_security_info(self.info, 'label') @@ -976,7 +968,7 @@ class XendDomainInfo: self.info['name_label'], self.domid) self._writeVm(LAST_SHUTDOWN_REASON, 'crash') - if xroot.get_enable_dump(): + if xoptions.get_enable_dump(): try: self.dumpCore() except XendError: @@ -1228,8 +1220,11 @@ class XendDomainInfo: if self.image: self.image.createDeviceModel() - def _releaseDevices(self): + def _releaseDevices(self, suspend = False): """Release all domain's devices. Nothrow guarantee.""" + if suspend and self.image: + self.image.destroy(suspend) + return while True: t = xstransact("%s/device" % self.dompath) @@ -1381,7 +1376,7 @@ class XendDomainInfo: # this is done prior to memory allocation to aide in memory # distribution for NUMA systems. if self.info['cpus'] is not None and len(self.info['cpus']) > 0: - for v in range(0, self.info['max_vcpu_id']+1): + for v in range(0, self.info['vcpus_number']): xc.vcpu_setaffinity(self.domid, v, self.info['cpus']) # Use architecture- and image-specific calculations to determine @@ -1395,6 +1390,7 @@ class XendDomainInfo: self.info['shadow_memory'] * 1024, self.info['memory_static_max'] * 1024) + log.debug("_initDomain:shadow_memory=0x%x, memory_static_max=0x%x, memory_static_min=0x%x.", self.info['shadow_memory'], self.info['memory_static_max'], self.info['memory_static_min'],) # Round shadow up to a multiple of a MiB, as shadow_mem_control # takes MiB and we must not round down and end up under-providing. shadow = ((shadow + 1023) / 1024) * 1024 @@ -1494,6 +1490,16 @@ class XendDomainInfo: self.console_mfn = console_mfn self._introduceDomain() + image_cfg = self.info.get('image', {}) + hvm = image_cfg.has_key('hvm') + if hvm: + self.image = image.create(self, + self.info, + self.info['image'], + self.info['devices']) + if self.image: + self.image.createDeviceModel(True) + self.image.register_shutdown_watch() self._storeDomDetails() self._registerWatches() self.refreshShutdown() @@ -2028,8 +2034,8 @@ class XendDomainInfo: # TODO: spec says that key is int, however, python does not allow # non-string keys to dictionaries. vcpu_util = {} - if 'max_vcpu_id' in self.info and self.domid != None: - for i in range(0, self.info['max_vcpu_id']+1): + if 'vcpus_number' in self.info and self.domid != None: + for i in range(0, self.info['vcpus_number']): info = xc.vcpu_getinfo(self.domid, i) vcpu_util[str(i)] = info['cpu_time']/1000000000.0 diff --git a/tools/python/xen/xend/XendNode.py b/tools/python/xen/xend/XendNode.py index 932c05d207..30953f6a53 100644 --- a/tools/python/xen/xend/XendNode.py +++ b/tools/python/xen/xend/XendNode.py @@ -24,7 +24,7 @@ from xen.util import Brctl from xen.xend import uuid from xen.xend.XendError import XendError, NetworkAlreadyConnected -from xen.xend.XendRoot import instance as xendroot +from xen.xend.XendOptions import instance as xendoptions from xen.xend.XendStorageRepository import XendStorageRepository from xen.xend.XendLogging import log from xen.xend.XendPIF import * @@ -45,7 +45,7 @@ class XendNode: """ self.xc = xen.lowlevel.xc.xc() - self.state_store = XendStateStore(xendroot().get_xend_state_path()) + self.state_store = XendStateStore(xendoptions().get_xend_state_path()) # load host state from XML file saved_host = self.state_store.load_state('host') diff --git a/tools/python/xen/xend/XendRoot.py b/tools/python/xen/xend/XendOptions.py similarity index 73% rename from tools/python/xen/xend/XendRoot.py rename to tools/python/xen/xend/XendOptions.py index 640c96f5f8..a51ecb670a 100644 --- a/tools/python/xen/xend/XendRoot.py +++ b/tools/python/xen/xend/XendOptions.py @@ -20,7 +20,7 @@ Creates the servers and handles configuration. Other classes get config variables by importing this module, -using instance() to get a XendRoot instance, and then +using instance() to get a XendOptions instance, and then the config functions (e.g. get_xend_port()) to get configured values. """ @@ -33,14 +33,11 @@ import sys from xen.xend import sxp, osdep, XendLogging from xen.xend.XendError import XendError -class XendRoot: - """Root of the management classes.""" +if os.uname()[0] == 'SunOS': + from xen.lowlevel import scf - """Default path to the config file.""" - config_default = "/etc/xen/xend-config.sxp" - - """Environment variable used to override config_default.""" - config_var = "XEND_CONFIG" +class XendOptions: + """Configuration options.""" """Where network control scripts live.""" network_script_dir = osdep.scripts_dir @@ -75,10 +72,10 @@ class XendRoot: xend_relocation_address_default = '' """Default port xend serves HTTP at. """ - xend_port_default = '8000' + xend_port_default = 8000 """Default port xend serves relocation at. """ - xend_relocation_port_default = '8002' + xend_relocation_port_default = 8002 xend_relocation_hosts_allow_default = '' @@ -92,9 +89,9 @@ class XendRoot: """Default path the unix-domain server listens at.""" xend_unix_path_default = '/var/lib/xend/xend-socket' - dom0_min_mem_default = '0' + dom0_min_mem_default = 0 - dom0_vcpus_default = '0' + dom0_vcpus_default = 0 vncpasswd_default = None @@ -107,16 +104,17 @@ class XendRoot: """Default xend management state storage.""" xend_state_path_default = '/var/lib/xend/state' - components = {} + """Default type of backend network interfaces""" + netback_type = osdep.netback_type + + """Default script to configure a backend network interface""" + vif_script = osdep.vif_script def __init__(self): - self.config_path = None - self.config = None self.configure() - def _logError(self, fmt, *args): - """Logging function to log to stderr. We use this for XendRoot log + """Logging function to log to stderr. We use this for XendOptions log messages because they may be logged before the logger has been configured. Other components can safely use the logger. """ @@ -125,81 +123,25 @@ class XendRoot: def configure(self): self.set_config() - XendLogging.init(self.get_config_value("logfile", + XendLogging.init(self.get_config_string("logfile", self.logfile_default), - self.get_config_value("loglevel", + self.get_config_string("loglevel", self.loglevel_default)) - def set_config(self): - """If the config file exists, read it. If not, ignore it. - - The config file is a sequence of sxp forms. - """ - self.config_path = os.getenv(self.config_var, self.config_default) - if os.path.exists(self.config_path): - try: - fin = file(self.config_path, 'rb') - try: - config = sxp.parse(fin) - finally: - fin.close() - if config is None: - config = ['xend-config'] - else: - config.insert(0, 'xend-config') - self.config = config - except Exception, ex: - self._logError('Reading config file %s: %s', - self.config_path, str(ex)) - raise - else: - self._logError('Config file does not exist: %s', - self.config_path) - self.config = ['xend-config'] - - def get_config(self, name=None): - """Get the configuration element with the given name, or - the whole configuration if no name is given. - - @param name: element name (optional) - @return: config or none - """ - if name is None: - val = self.config - else: - val = sxp.child(self.config, name) - return val - - def get_config_value(self, name, val=None): - """Get the value of an atomic configuration element. - - @param name: element name - @param val: default value (optional, defaults to None) - @return: value - """ - return sxp.child_value(self.config, name, val=val) + raise NotImplementedError() def get_config_bool(self, name, val=None): - v = string.lower(str(self.get_config_value(name, val))) - if v in ['yes', 'y', '1', 'on', 'true', 't']: - return True - if v in ['no', 'n', '0', 'off', 'false', 'f']: - return False - raise XendError("invalid xend config %s: expected bool: %s" % (name, v)) - + raise NotImplementedError() + def get_config_int(self, name, val=None): - v = self.get_config_value(name, val) - try: - return int(v) - except Exception: - raise XendError("invalid xend config %s: expected int: %s" % (name, v)) + raise NotImplementedError() + + def get_config_string(self, name, val=None): + raise NotImplementedError() def get_xen_api_server(self): - """Get the Xen-API server configuration. - """ - return self.get_config_value('xen-api-server', - self.xen_api_server_default) + raise NotImplementedError() def get_xend_http_server(self): """Get the flag indicating whether xend should run an http server. @@ -232,7 +174,7 @@ class XendRoot: self.xend_relocation_port_default) def get_xend_relocation_hosts_allow(self): - return self.get_config_value("xend-relocation-hosts-allow", + return self.get_config_string("xend-relocation-hosts-allow", self.xend_relocation_hosts_allow_default) def get_xend_address(self): @@ -241,7 +183,7 @@ class XendRoot: If this is set to 'localhost' only the localhost will be able to connect to the HTTP port. """ - return self.get_config_value('xend-address', self.xend_address_default) + return self.get_config_string('xend-address', self.xend_address_default) def get_xend_relocation_address(self): """Get the address xend listens at for its relocation server port. @@ -249,7 +191,7 @@ class XendRoot: If this is set to 'localhost' only the localhost will be able to connect to the relocation port. """ - return self.get_config_value('xend-relocation-address', self.xend_relocation_address_default) + return self.get_config_string('xend-relocation-address', self.xend_relocation_address_default) def get_xend_unix_server(self): """Get the flag indicating whether xend should run a unix-domain server. @@ -259,23 +201,23 @@ class XendRoot: def get_xend_unix_path(self): """Get the path the xend unix-domain server listens at. """ - return self.get_config_value("xend-unix-path", self.xend_unix_path_default) + return self.get_config_string("xend-unix-path", self.xend_unix_path_default) def get_xend_domains_path(self): """ Get the path for persistent domain configuration storage """ - return self.get_config_value("xend-domains-path", self.xend_domains_path_default) + return self.get_config_string("xend-domains-path", self.xend_domains_path_default) def get_xend_state_path(self): """ Get the path for persistent domain configuration storage """ - return self.get_config_value("xend-state-path", self.xend_state_path_default) + return self.get_config_string("xend-state-path", self.xend_state_path_default) def get_network_script(self): """@return the script used to alter the network configuration when Xend starts and stops, or None if no such script is specified.""" - s = self.get_config_value('network-script') + s = self.get_config_string('network-script') if s: result = s.split(" ") @@ -286,13 +228,13 @@ class XendRoot: def get_external_migration_tool(self): """@return the name of the tool to handle virtual TPM migration.""" - return self.get_config_value('external-migration-tool', self.external_migration_tool_default) + return self.get_config_string('external-migration-tool', self.external_migration_tool_default) def get_enable_dump(self): return self.get_config_bool('enable-dump', 'no') def get_vif_script(self): - return self.get_config_value('vif-script', 'vif-bridge') + return self.get_config_string('vif-script', self.vif_script) def get_dom0_min_mem(self): return self.get_config_int('dom0-min-mem', self.dom0_min_mem_default) @@ -304,19 +246,128 @@ class XendRoot: return self.get_config_int('console-limit', 1024) def get_vnclisten_address(self): - return self.get_config_value('vnc-listen', self.xend_vnc_listen_default) + return self.get_config_string('vnc-listen', self.xend_vnc_listen_default) def get_vncpasswd_default(self): - return self.get_config_value('vncpasswd', + return self.get_config_string('vncpasswd', self.vncpasswd_default) +class XendOptionsFile(XendOptions): + + """Default path to the config file.""" + config_default = "/etc/xen/xend-config.sxp" + + """Environment variable used to override config_default.""" + config_var = "XEND_CONFIG" + + def set_config(self): + """If the config file exists, read it. If not, ignore it. + + The config file is a sequence of sxp forms. + """ + self.config_path = os.getenv(self.config_var, self.config_default) + if os.path.exists(self.config_path): + try: + fin = file(self.config_path, 'rb') + try: + config = sxp.parse(fin) + finally: + fin.close() + if config is None: + config = ['xend-config'] + else: + config.insert(0, 'xend-config') + self.config = config + except Exception, ex: + self._logError('Reading config file %s: %s', + self.config_path, str(ex)) + raise + else: + self._logError('Config file does not exist: %s', + self.config_path) + self.config = ['xend-config'] + + def get_config_value(self, name, val=None): + """Get the value of an atomic configuration element. + + @param name: element name + @param val: default value (optional, defaults to None) + @return: value + """ + return sxp.child_value(self.config, name, val=val) + + def get_config_bool(self, name, val=None): + v = string.lower(str(self.get_config_value(name, val))) + if v in ['yes', 'y', '1', 'on', 'true', 't']: + return True + if v in ['no', 'n', '0', 'off', 'false', 'f']: + return False + raise XendError("invalid xend config %s: expected bool: %s" % (name, v)) + + def get_config_int(self, name, val=None): + v = self.get_config_value(name, val) + try: + return int(v) + except Exception: + raise XendError("invalid xend config %s: expected int: %s" % (name, v)) + + def get_config_string(self, name, val=None): + return self.get_config_value(name, val) + + def get_xen_api_server(self): + """Get the Xen-API server configuration. + """ + return self.get_config_value('xen-api-server', + self.xen_api_server_default) + +if os.uname()[0] == 'SunOS': + class XendOptionsSMF(XendOptions): + + def set_config(self): + pass + + def get_config_bool(self, name, val=None): + try: + return scf.get_bool(name) + except scf.error, e: + if e[0] == scf.SCF_ERROR_NOT_FOUND: + return val + else: + raise XendError("option %s: %s:%s" % (name, e[1], e[2])) + + def get_config_int(self, name, val=None): + try: + return scf.get_int(name) + except scf.error, e: + if e[0] == scf.SCF_ERROR_NOT_FOUND: + return val + else: + raise XendError("option %s: %s:%s" % (name, e[1], e[2])) + + def get_config_string(self, name, val=None): + try: + return scf.get_string(name) + except scf.error, e: + if e[0] == scf.SCF_ERROR_NOT_FOUND: + return val + else: + raise XendError("option %s: %s:%s" % (name, e[1], e[2])) + + def get_xen_api_server(self): + # When the new server is a supported configuration, we should + # expand this. + return [["unix"]] + def instance(): - """Get an instance of XendRoot. + """Get an instance of XendOptions. Use this instead of the constructor. """ global inst try: inst except: - inst = XendRoot() + if os.uname()[0] == 'SunOS': + inst = XendOptionsSMF() + else: + inst = XendOptionsFile() return inst diff --git a/tools/python/xen/xend/XendProtocol.py b/tools/python/xen/xend/XendProtocol.py index f09f6762d3..34af4a4b2b 100644 --- a/tools/python/xen/xend/XendProtocol.py +++ b/tools/python/xen/xend/XendProtocol.py @@ -24,7 +24,7 @@ import types from encode import * from xen.xend import sxp -from xen.xend import XendRoot +from xen.xend import XendOptions DEBUG = 0 @@ -34,7 +34,7 @@ HTTP_ACCEPTED = 202 HTTP_NO_CONTENT = 204 -xroot = XendRoot.instance() +xoptions = XendOptions.instance() class XendError(RuntimeError): @@ -218,7 +218,7 @@ class UnixXendClientProtocol(HttpXendClientProtocol): def __init__(self, path=None): if path is None: - path = xroot.get_xend_unix_path() + path = xoptions.get_xend_unix_path() self.path = path def makeConnection(self, _): diff --git a/tools/python/xen/xend/balloon.py b/tools/python/xen/xend/balloon.py index 3c167f3b27..e7c2693ab3 100644 --- a/tools/python/xen/xend/balloon.py +++ b/tools/python/xen/xend/balloon.py @@ -22,7 +22,7 @@ import time import xen.lowlevel.xc import XendDomain -import XendRoot +import XendOptions from XendLogging import log from XendError import VmError @@ -107,11 +107,11 @@ def free(need_mem): # usage, so we recheck the required alloc each time around the loop, but # track the last used value so that we don't trigger too many watches. - xroot = XendRoot.instance() + xoptions = XendOptions.instance() xc = xen.lowlevel.xc.xc() try: - dom0_min_mem = xroot.get_dom0_min_mem() * 1024 + dom0_min_mem = xoptions.get_dom0_min_mem() * 1024 retries = 0 sleep_time = SLEEP_TIME_GROWTH diff --git a/tools/python/xen/xend/image.py b/tools/python/xen/xend/image.py index 5147e1fef0..5076d03080 100644 --- a/tools/python/xen/xend/image.py +++ b/tools/python/xen/xend/image.py @@ -173,7 +173,7 @@ class ImageHandler: """Build the domain. Define in subclass.""" raise NotImplementedError() - def createDeviceModel(self): + def createDeviceModel(self, restore = False): """Create device model for the domain (define in subclass if needed).""" pass @@ -377,11 +377,12 @@ class HVMImageHandler(ImageHandler): # xm config file def parseDeviceModelArgs(self, imageConfig, deviceConfig): dmargs = [ 'boot', 'fda', 'fdb', 'soundhw', - 'localtime', 'serial', 'stdvga', 'isa', 'vcpus', + 'localtime', 'serial', 'stdvga', 'isa', 'acpi', 'usb', 'usbdevice', 'keymap' ] - ret = [] hvmDeviceConfig = imageConfig['hvm']['devices'] - + + ret = ['-vcpus', str(self.vm.getVCpuCount())] + for a in dmargs: v = hvmDeviceConfig.get(a) @@ -461,14 +462,14 @@ class HVMImageHandler(ImageHandler): vnclisten = imageConfig.get('vnclisten') if not(vnclisten): - vnclisten = (xen.xend.XendRoot.instance(). + vnclisten = (xen.xend.XendOptions.instance(). get_vnclisten_address()) if vnclisten: ret += ['-vnclisten', vnclisten] vncpasswd = vncpasswd_vmconfig if vncpasswd is None: - vncpasswd = (xen.xend.XendRoot.instance(). + vncpasswd = (xen.xend.XendOptions.instance(). get_vncpasswd_default()) if vncpasswd is None: raise VmError('vncpasswd is not set up in ' + @@ -478,7 +479,7 @@ class HVMImageHandler(ImageHandler): return ret - def createDeviceModel(self): + def createDeviceModel(self, restore = False): if self.pid: return # Execute device model. @@ -487,6 +488,8 @@ class HVMImageHandler(ImageHandler): args = args + ([ "-d", "%d" % self.vm.getDomid(), "-m", "%s" % (self.getRequiredInitialReservation() / 1024)]) args = args + self.dmargs + if restore: + args = args + ([ "-loadvm", "/tmp/xen.qemu-dm.%d" % self.vm.getDomid() ]) env = dict(os.environ) if self.display: env['DISPLAY'] = self.display @@ -505,12 +508,16 @@ class HVMImageHandler(ImageHandler): self.register_reboot_feature_watch() self.pid = self.vm.gatherDom(('image/device-model-pid', int)) - def destroy(self): + def destroy(self, suspend = False): self.unregister_shutdown_watch() self.unregister_reboot_feature_watch(); if self.pid: try: - os.kill(self.pid, signal.SIGKILL) + sig = signal.SIGKILL + if suspend: + log.info("use sigusr1 to signal qemu %d", self.pid) + sig = signal.SIGUSR1 + os.kill(self.pid, sig) except OSError, exn: log.exception(exn) try: diff --git a/tools/python/xen/xend/osdep.py b/tools/python/xen/xend/osdep.py index 1dd3c7c1dd..2136465ea7 100644 --- a/tools/python/xen/xend/osdep.py +++ b/tools/python/xen/xend/osdep.py @@ -33,9 +33,19 @@ _pygrub_path = { "SunOS": "/usr/lib/xen/bin/pygrub" } +_netback_type = { + "SunOS": "SUNW_mac" +} + +_vif_script = { + "SunOS": "vif-vnic" +} + def _get(var, default=None): return var.get(os.uname()[0], default) scripts_dir = _get(_scripts_dir, "/etc/xen/scripts") xend_autorestart = _get(_xend_autorestart) pygrub_path = _get(_pygrub_path, "/usr/bin/pygrub") +netback_type = _get(_netback_type, "netfront") +vif_script = _get(_vif_script, "vif-bridge") diff --git a/tools/python/xen/xend/server/DevController.py b/tools/python/xen/xend/server/DevController.py index 6ac7885036..7e411a8027 100644 --- a/tools/python/xen/xend/server/DevController.py +++ b/tools/python/xen/xend/server/DevController.py @@ -19,7 +19,7 @@ from threading import Event import types -from xen.xend import sxp, XendRoot +from xen.xend import sxp, XendOptions from xen.xend.XendError import VmError from xen.xend.XendLogging import log @@ -50,7 +50,7 @@ xenbusState = { 'Closed' : 6, } -xroot = XendRoot.instance() +xoptions = XendOptions.instance() xenbusState.update(dict(zip(xenbusState.values(), xenbusState.keys()))) @@ -324,7 +324,7 @@ class DevController: Make sure that the migration has finished and only then return from the call. """ - tool = xroot.get_external_migration_tool() + tool = xoptions.get_external_migration_tool() if tool: log.info("Calling external migration tool for step %d" % step) fd = os.popen("%s -type %s -step %d -host %s -domname %s" % @@ -341,7 +341,7 @@ class DevController: """ Recover from device migration. The given step was the last one that was successfully executed. """ - tool = xroot.get_external_migration_tool() + tool = xoptions.get_external_migration_tool() if tool: log.info("Calling external migration tool") fd = os.popen("%s -type %s -step %d -host %s -domname %s -recover" % diff --git a/tools/python/xen/xend/server/SrvRoot.py b/tools/python/xen/xend/server/SrvRoot.py index 51b4c4126f..bd194f3a38 100644 --- a/tools/python/xen/xend/server/SrvRoot.py +++ b/tools/python/xen/xend/server/SrvRoot.py @@ -25,7 +25,7 @@ class SrvRoot(SrvDir): """Server sub-components. Each entry is (name, class), where 'name' is the entry name and 'class' is the name of its class. """ - #todo Get this list from the XendRoot config. + #todo Get this list from the XendOptions config. subdirs = [ ('node', 'SrvNode' ), ('domain', 'SrvDomainDir' ), diff --git a/tools/python/xen/xend/server/SrvServer.py b/tools/python/xen/xend/server/SrvServer.py index 82df82e0f8..d4aae088e6 100644 --- a/tools/python/xen/xend/server/SrvServer.py +++ b/tools/python/xen/xend/server/SrvServer.py @@ -48,7 +48,7 @@ from threading import Thread from xen.web.httpserver import HttpServer, UnixHttpServer -from xen.xend import XendNode, XendRoot, XendAPI +from xen.xend import XendNode, XendOptions, XendAPI from xen.xend import Vifctl from xen.xend.XendLogging import log from xen.xend.XendClient import XEN_API_SOCKET @@ -57,7 +57,7 @@ from xen.web.SrvDir import SrvDir from SrvRoot import SrvRoot from XMLRPCServer import XMLRPCServer -xroot = XendRoot.instance() +xoptions = XendOptions.instance() class XendServers: @@ -65,6 +65,7 @@ class XendServers: def __init__(self, root): self.servers = [] self.root = root + self.running = False self.cleaningUp = False self.reloadingConfig = False @@ -79,6 +80,7 @@ class XendServers: server.shutdown() except: pass + self.running = False def reloadConfig(self, signum = 0, frame = None): log.debug("SrvServer.reloadConfig()") @@ -107,13 +109,12 @@ class XendServers: if server.ready: continue - thread = Thread(target=server.run, name=server.__class__.__name__) - if isinstance(server, HttpServer): - thread.setDaemon(True) + thread = Thread(target=server.run, + name=server.__class__.__name__) + thread.setDaemon(True) thread.start() threads.append(thread) - # check for when all threads have initialized themselves and then # close the status pipe @@ -143,47 +144,32 @@ class XendServers: status.close() status = None - # Interruptible Thread.join - Python Bug #1167930 - # Replaces: for t in threads: t.join() - # Reason: The above will cause python signal handlers to be - # blocked so we're not able to catch SIGTERM in any - # way for cleanup - runningThreads = threads - while len(runningThreads) > 0: - try: - for t in threads: - t.join(1.0) - runningThreads = [t for t in threads - if t.isAlive() and not t.isDaemon()] - if self.cleaningUp and len(runningThreads) > 0: - log.debug("Waiting for %s." % - [x.getName() for x in runningThreads]) - except: - pass - + # loop to keep main thread alive until it receives a SIGTERM + self.running = True + while self.running: + time.sleep(100000000) + if self.reloadingConfig: log.info("Restarting all XML-RPC and Xen-API servers...") self.cleaningUp = False self.reloadingConfig = False - xroot.set_config() - new_servers = [x for x in self.servers - if isinstance(x, HttpServer)] - self.servers = new_servers + xoptions.set_config() + self.servers = [] _loadConfig(self, self.root, True) else: break def _loadConfig(servers, root, reload): - if not reload and xroot.get_xend_http_server(): + if xoptions.get_xend_http_server(): servers.add(HttpServer(root, - xroot.get_xend_address(), - xroot.get_xend_port())) - if not reload and xroot.get_xend_unix_server(): - path = xroot.get_xend_unix_path() + xoptions.get_xend_address(), + xoptions.get_xend_port())) + if xoptions.get_xend_unix_server(): + path = xoptions.get_xend_unix_path() log.info('unix path=' + path) servers.add(UnixHttpServer(root, path)) - api_cfg = xroot.get_xen_api_server() + api_cfg = xoptions.get_xen_api_server() if api_cfg: try: addrs = [(str(x[0]).split(':'), @@ -218,10 +204,10 @@ def _loadConfig(servers, root, reload): except TypeError, exn: log.error('Xen-API server configuration %s is invalid.', api_cfg) - if xroot.get_xend_tcp_xmlrpc_server(): + if xoptions.get_xend_tcp_xmlrpc_server(): servers.add(XMLRPCServer(XendAPI.AUTH_PAM, False, True)) - if xroot.get_xend_unix_xmlrpc_server(): + if xoptions.get_xend_unix_xmlrpc_server(): servers.add(XMLRPCServer(XendAPI.AUTH_PAM, False)) diff --git a/tools/python/xen/xend/server/XMLRPCServer.py b/tools/python/xen/xend/server/XMLRPCServer.py index 05646fcea6..130da9aea8 100644 --- a/tools/python/xen/xend/server/XMLRPCServer.py +++ b/tools/python/xen/xend/server/XMLRPCServer.py @@ -179,21 +179,24 @@ class XMLRPCServer: # Custom runloop so we can cleanup when exiting. # ----------------------------------------------------------------- try: - self.server.socket.settimeout(1.0) while self.running: self.server.handle_request() finally: - self.cleanup() + self.shutdown() def cleanup(self): - log.debug("XMLRPCServer.cleanup()") + log.debug('XMLRPCServer.cleanup()') try: - self.server.socket.close() + if hasattr(self, 'server'): + # shutdown socket explicitly to allow reuse + self.server.socket.shutdown(socket.SHUT_RDWR) + self.server.socket.close() except Exception, exn: log.exception(exn) pass def shutdown(self): self.running = False - self.ready = False - + if self.ready: + self.ready = False + self.cleanup() diff --git a/tools/python/xen/xend/server/netif.py b/tools/python/xen/xend/server/netif.py index c229ed7b20..bc755e9c9d 100644 --- a/tools/python/xen/xend/server/netif.py +++ b/tools/python/xen/xend/server/netif.py @@ -24,10 +24,10 @@ import os import random import re -from xen.xend import XendRoot +from xen.xend import XendOptions from xen.xend.server.DevController import DevController -xroot = XendRoot.instance() +xoptions = XendOptions.instance() def randomMAC(): """Generate a random MAC address. @@ -138,8 +138,8 @@ class NetifController(DevController): def getDeviceDetails(self, config): """@see DevController.getDeviceDetails""" - script = os.path.join(xroot.network_script_dir, - config.get('script', xroot.get_vif_script())) + script = os.path.join(xoptions.network_script_dir, + config.get('script', xoptions.get_vif_script())) typ = config.get('type') bridge = config.get('bridge') mac = config.get('mac') @@ -150,9 +150,8 @@ class NetifController(DevController): devid = self.allocateDeviceID() - # The default type is 'netfront'. if not typ: - typ = 'netfront' + typ = xoptions.netback_type if not mac: mac = randomMAC() @@ -190,7 +189,7 @@ class NetifController(DevController): (script, ip, bridge, mac, typ, vifname, rate, uuid) = devinfo if script: - network_script_dir = xroot.network_script_dir + os.sep + network_script_dir = xoptions.network_script_dir + os.sep result['script'] = script.replace(network_script_dir, "") if ip: result['ip'] = ip diff --git a/tools/python/xen/xend/server/relocate.py b/tools/python/xen/xend/server/relocate.py index 1b7ed157e9..3cdd43a5eb 100644 --- a/tools/python/xen/xend/server/relocate.py +++ b/tools/python/xen/xend/server/relocate.py @@ -24,7 +24,7 @@ from xen.web import protocol, tcp, unix from xen.xend import sxp from xen.xend import XendDomain -from xen.xend import XendRoot +from xen.xend import XendOptions from xen.xend.XendError import XendError from xen.xend.XendLogging import log @@ -114,15 +114,15 @@ class RelocationProtocol(protocol.Protocol): def listenRelocation(): - xroot = XendRoot.instance() - if xroot.get_xend_unix_server(): + xoptions = XendOptions.instance() + if xoptions.get_xend_unix_server(): path = '/var/lib/xend/relocation-socket' unix.UnixListener(path, RelocationProtocol) - if xroot.get_xend_relocation_server(): - port = xroot.get_xend_relocation_port() - interface = xroot.get_xend_relocation_address() + if xoptions.get_xend_relocation_server(): + port = xoptions.get_xend_relocation_port() + interface = xoptions.get_xend_relocation_address() - hosts_allow = xroot.get_xend_relocation_hosts_allow() + hosts_allow = xoptions.get_xend_relocation_hosts_allow() if hosts_allow == '': hosts_allow = None else: diff --git a/tools/python/xen/xend/server/tests/test_controllers.py b/tools/python/xen/xend/server/tests/test_controllers.py index aa57f6321c..dd2677500d 100644 --- a/tools/python/xen/xend/server/tests/test_controllers.py +++ b/tools/python/xen/xend/server/tests/test_controllers.py @@ -2,9 +2,9 @@ import os import re import unittest -import xen.xend.XendRoot +import xen.xend.XendOptions -xen.xend.XendRoot.XendRoot.config_default = '/dev/null' +xen.xend.XendOptions.XendOptions.config_default = '/dev/null' from xen.xend.server import netif @@ -13,7 +13,7 @@ FAKE_DOMID = 42 FAKE_DEVID = 63 -xroot = xen.xend.XendRoot.instance() +xoptions = xen.xend.XendOptions.instance() class test_controllers(unittest.TestCase): @@ -36,8 +36,8 @@ class test_controllers(unittest.TestCase): self.assertEqual(backdets['handle'], str(FAKE_DEVID)) self.assertEqual(backdets['script'], - os.path.join(xroot.network_script_dir, - xroot.get_vif_script())) + os.path.join(xoptions.network_script_dir, + xoptions.get_vif_script())) self.assertValidMac(backdets['mac'], expectedMac) self.assertEqual(frontdets['handle'], str(FAKE_DEVID)) diff --git a/tools/python/xen/xend/server/tpmif.py b/tools/python/xen/xend/server/tpmif.py index c2a5f1f64b..b977110a43 100644 --- a/tools/python/xen/xend/server/tpmif.py +++ b/tools/python/xen/xend/server/tpmif.py @@ -20,7 +20,7 @@ """Support for virtual TPM interfaces.""" -from xen.xend import XendRoot +from xen.xend import XendOptions from xen.xend.XendLogging import log from xen.xend.XendError import XendError from xen.xend.XendConstants import DEV_MIGRATE_TEST, VTPM_DELETE_SCRIPT @@ -29,7 +29,7 @@ from xen.xend.server.DevController import DevController import os import re -xroot = XendRoot.instance() +xoptions = XendOptions.instance() def destroy_vtpmstate(name): if os.path.exists(VTPM_DELETE_SCRIPT): @@ -88,7 +88,7 @@ class TPMifController(DevController): def migrate(self, deviceConfig, network, dst, step, domName): """@see DevContoller.migrate""" if network: - tool = xroot.get_external_migration_tool() + tool = xoptions.get_external_migration_tool() if tool != '': log.info("Request to network-migrate device to %s. step=%d.", dst, step) @@ -116,7 +116,7 @@ class TPMifController(DevController): def recover_migrate(self, deviceConfig, network, dst, step, domName): """@see DevContoller.recover_migrate""" if network: - tool = xroot.get_external_migration_tool() + tool = xoptions.get_external_migration_tool() if tool != '': log.info("Request to recover network-migrated device. last good step=%d.", step) diff --git a/tools/python/xen/xend/server/vfbif.py b/tools/python/xen/xend/server/vfbif.py index 816dcd5a6f..dfa94c2989 100644 --- a/tools/python/xen/xend/server/vfbif.py +++ b/tools/python/xen/xend/server/vfbif.py @@ -52,7 +52,7 @@ class VfbifController(DevController): if config.has_key("vncpasswd"): passwd = config["vncpasswd"] else: - passwd = xen.xend.XendRoot.instance().get_vncpasswd_default() + passwd = xen.xend.XendOptions.instance().get_vncpasswd_default() if passwd: self.vm.storeVm("vncpasswd", passwd) log.debug("Stored a VNC password for vfb access") @@ -66,7 +66,7 @@ class VfbifController(DevController): elif config.has_key("vncdisplay"): args += ["--vncport", "%d" % (5900 + int(config["vncdisplay"]))] vnclisten = config.get("vnclisten", - xen.xend.XendRoot.instance().get_vnclisten_address()) + xen.xend.XendOptions.instance().get_vnclisten_address()) args += [ "--listen", vnclisten ] spawn_detached(args[0], args + std_args, os.environ) elif t == "sdl": diff --git a/tools/python/xen/xm/create.py b/tools/python/xen/xm/create.py index e0fe857885..fb42f44c64 100644 --- a/tools/python/xen/xm/create.py +++ b/tools/python/xen/xm/create.py @@ -190,6 +190,10 @@ gopts.var('vcpus', val='VCPUS', fn=set_int, default=1, use="# of Virtual CPUS in domain.") +gopts.var('vcpu_avail', val='VCPUS', + fn=set_long, default=None, + use="Bitmask for virtual CPUs to make available immediately.") + gopts.var('cpu_cap', val='CAP', fn=set_int, default=None, use="""Set the maximum amount of cpu. @@ -740,7 +744,7 @@ def make_config(vals): map(add_conf, ['name', 'memory', 'maxmem', 'shadow_memory', 'restart', 'on_poweroff', - 'on_reboot', 'on_crash', 'vcpus', 'features', + 'on_reboot', 'on_crash', 'vcpus', 'vcpu_avail', 'features', 'on_xend_start', 'on_xend_stop']) if vals.uuid is not None: diff --git a/tools/python/xen/xm/main.py b/tools/python/xen/xm/main.py index 87a0602e2d..95990f8f62 100644 --- a/tools/python/xen/xm/main.py +++ b/tools/python/xen/xm/main.py @@ -693,12 +693,15 @@ def parse_doms_info(info): up_time = time.time() - start_time return { - 'domid' : get_info('domid', str, ''), - 'name' : get_info('name', str, '??'), + 'domid' : get_info('domid', str, ''), + 'name' : get_info('name', str, '??'), 'mem' : get_info('memory_dynamic_min', int, 0), - 'vcpus' : get_info('online_vcpus', int, 0), - 'state' : get_info('state', str, ''), - 'cpu_time' : get_info('cpu_time', float, 0), + 'state' : get_info('state', str, ''), + 'cpu_time' : get_info('cpu_time', float, 0.0), + # VCPUs is the number online when the VM is up, or the number + # configured otherwise. + 'vcpus' : get_info('online_vcpus', int, + get_info('vcpus', int, 0)), 'up_time' : up_time, 'seclabel' : security.get_security_printlabel(info), } diff --git a/tools/python/xen/xm/opts.py b/tools/python/xen/xm/opts.py index 7dd28bfa4e..1b80ceeff8 100644 --- a/tools/python/xen/xm/opts.py +++ b/tools/python/xen/xm/opts.py @@ -571,6 +571,14 @@ def set_int(opt, k, v): opt.opts.err('Invalid value: ' + str(v)) opt.set(v) +def set_long(opt, k, v): + """Set an option to a long integer value.""" + try: + v = long(v) + except: + opt.opts.err('Invalid value: ' + str(v)) + opt.set(v) + def set_float(opt, k, v): """Set an option to a float value.""" try: diff --git a/tools/python/xen/xm/tests/test_create.py b/tools/python/xen/xm/tests/test_create.py index ce8e192413..a25ebb1fb0 100644 --- a/tools/python/xen/xm/tests/test_create.py +++ b/tools/python/xen/xm/tests/test_create.py @@ -3,9 +3,9 @@ import os.path import tempfile import unittest -import xen.xend.XendRoot +import xen.xend.XendOptions -xen.xend.XendRoot.XendRoot.config_default = '/dev/null' +xen.xend.XendOptions.XendOptions.config_default = '/dev/null' import xen.xm.create diff --git a/tools/xcutils/xc_restore.c b/tools/xcutils/xc_restore.c index c8b2b89331..8e26d9bb70 100644 --- a/tools/xcutils/xc_restore.c +++ b/tools/xcutils/xc_restore.c @@ -19,12 +19,13 @@ int main(int argc, char **argv) { unsigned int xc_fd, io_fd, domid, nr_pfns, store_evtchn, console_evtchn; + unsigned int hvm, pae, apic; int ret; unsigned long store_mfn, console_mfn; - if (argc != 6) + if (argc != 9) errx(1, - "usage: %s iofd domid nr_pfns store_evtchn console_evtchn", + "usage: %s iofd domid nr_pfns store_evtchn console_evtchn hvm pae apic", argv[0]); xc_fd = xc_interface_open(); @@ -36,9 +37,19 @@ main(int argc, char **argv) nr_pfns = atoi(argv[3]); store_evtchn = atoi(argv[4]); console_evtchn = atoi(argv[5]); + hvm = atoi(argv[6]); + pae = atoi(argv[7]); + apic = atoi(argv[8]); + + if (hvm) { + /* pass the memsize to xc_hvm_restore to find the store_mfn */ + store_mfn = hvm; + ret = xc_hvm_restore(xc_fd, io_fd, domid, nr_pfns, store_evtchn, + &store_mfn, console_evtchn, &console_mfn, pae, apic); + } else + ret = xc_linux_restore(xc_fd, io_fd, domid, nr_pfns, store_evtchn, + &store_mfn, console_evtchn, &console_mfn); - ret = xc_linux_restore(xc_fd, io_fd, domid, nr_pfns, store_evtchn, - &store_mfn, console_evtchn, &console_mfn); if (ret == 0) { printf("store-mfn %li\n", store_mfn); printf("console-mfn %li\n", console_mfn); diff --git a/tools/xcutils/xc_save.c b/tools/xcutils/xc_save.c index c9be3454d4..0c9530da4d 100644 --- a/tools/xcutils/xc_save.c +++ b/tools/xcutils/xc_save.c @@ -51,7 +51,10 @@ main(int argc, char **argv) max_f = atoi(argv[4]); flags = atoi(argv[5]); - ret = xc_linux_save(xc_fd, io_fd, domid, maxit, max_f, flags, &suspend); + if (flags & XCFLAGS_HVM) + ret = xc_hvm_save(xc_fd, io_fd, domid, maxit, max_f, flags, &suspend); + else + ret = xc_linux_save(xc_fd, io_fd, domid, maxit, max_f, flags, &suspend); xc_interface_close(xc_fd); diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index e23c85bbaa..a8c6010771 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -573,6 +573,7 @@ int arch_set_info_guest( else { hvm_load_cpu_guest_regs(v, &v->arch.guest_context.user_regs); + hvm_load_cpu_context(v, &v->arch.guest_context.hvmcpu_ctxt); } if ( test_bit(_VCPUF_initialised, &v->vcpu_flags) ) diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c index fdc102c1a5..79efff08e4 100644 --- a/xen/arch/x86/domctl.c +++ b/xen/arch/x86/domctl.c @@ -322,8 +322,10 @@ void arch_get_info_guest(struct vcpu *v, vcpu_guest_context_u c) if ( is_hvm_vcpu(v) ) { - if ( !IS_COMPAT(v->domain) ) + if ( !IS_COMPAT(v->domain) ) { hvm_store_cpu_guest_regs(v, &c.nat->user_regs, c.nat->ctrlreg); + hvm_save_cpu_context(v, &c.nat->hvmcpu_ctxt); + } #ifdef CONFIG_COMPAT else { diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c index febc278c23..9a46f7c555 100644 --- a/xen/arch/x86/hvm/hvm.c +++ b/xen/arch/x86/hvm/hvm.c @@ -149,11 +149,19 @@ int hvm_domain_initialise(struct domain *d) void hvm_domain_destroy(struct domain *d) { + HVMStateEntry *se, *dse; pit_deinit(d); rtc_deinit(d); pmtimer_deinit(d); hpet_deinit(d); + se = d->arch.hvm_domain.first_se; + while (se) { + dse = se; + se = se->next; + xfree(dse); + } + if ( d->arch.hvm_domain.shared_page_va ) unmap_domain_page_global( (void *)d->arch.hvm_domain.shared_page_va); @@ -189,6 +197,9 @@ int hvm_vcpu_initialise(struct vcpu *v) rtc_init(v, RTC_PORT(0), RTC_IRQ); pmtimer_init(v, ACPI_PM_TMR_BLK_ADDRESS); hpet_init(v); + + /* init hvm sharepage */ + shpage_init(v->domain, get_sp(v->domain)); /* Init guest TSC to start from zero. */ hvm_set_guest_time(v, 0); diff --git a/xen/arch/x86/hvm/i8254.c b/xen/arch/x86/hvm/i8254.c index 1ab512d2ee..cef7fc9a46 100644 --- a/xen/arch/x86/hvm/i8254.c +++ b/xen/arch/x86/hvm/i8254.c @@ -207,11 +207,11 @@ static inline void pit_load_count(PITChannelState *s, int channel, int val) switch (s->mode) { case 2: /* create periodic time */ - create_periodic_time(&s->pt, period, 0, 0, pit_time_fired, s); + create_periodic_time(current, &s->pt, period, 0, 0, pit_time_fired, s); break; case 1: /* create one shot time */ - create_periodic_time(&s->pt, period, 0, 1, pit_time_fired, s); + create_periodic_time(current, &s->pt, period, 0, 1, pit_time_fired, s); #ifdef DEBUG_PIT printk("HVM_PIT: create one shot time.\n"); #endif @@ -356,6 +356,154 @@ void pit_stop_channel0_irq(PITState * pit) destroy_periodic_time(&s->pt); } +#ifdef HVM_DEBUG_SUSPEND +static void pit_info(PITState *pit) +{ + PITChannelState *s; + int i; + + for(i = 0; i < 3; i++) { + printk("*****pit channel %d's state:*****\n", i); + s = &pit->channels[i]; + printk("pit 0x%x.\n", s->count); + printk("pit 0x%x.\n", s->latched_count); + printk("pit 0x%x.\n", s->count_latched); + printk("pit 0x%x.\n", s->status_latched); + printk("pit 0x%x.\n", s->status); + printk("pit 0x%x.\n", s->read_state); + printk("pit 0x%x.\n", s->write_state); + printk("pit 0x%x.\n", s->write_latch); + printk("pit 0x%x.\n", s->rw_mode); + printk("pit 0x%x.\n", s->mode); + printk("pit 0x%x.\n", s->bcd); + printk("pit 0x%x.\n", s->gate); + printk("pit %"PRId64"\n", s->count_load_time); + + if (s->pt) { + struct periodic_time *pt = s->pt; + printk("pit channel %d has a periodic timer:\n", i); + printk("pt %d.\n", pt->enabled); + printk("pt %d.\n", pt->one_shot); + printk("pt %d.\n", pt->irq); + printk("pt %d.\n", pt->first_injected); + + printk("pt %d.\n", pt->pending_intr_nr); + printk("pt %d.\n", pt->period); + printk("pt %"PRId64"\n", pt->period_cycles); + printk("pt %"PRId64"\n", pt->last_plt_gtime); + } + } + +} +#else +static void pit_info(PITState *pit) +{ +} +#endif + +static void pit_save(hvm_domain_context_t *h, void *opaque) +{ + struct domain *d = opaque; + PITState *pit = &d->arch.hvm_domain.pl_time.vpit; + PITChannelState *s; + struct periodic_time *pt; + int i, pti = -1; + + pit_info(pit); + + for(i = 0; i < 3; i++) { + s = &pit->channels[i]; + hvm_put_32u(h, s->count); + hvm_put_16u(h, s->latched_count); + hvm_put_8u(h, s->count_latched); + hvm_put_8u(h, s->status_latched); + hvm_put_8u(h, s->status); + hvm_put_8u(h, s->read_state); + hvm_put_8u(h, s->write_state); + hvm_put_8u(h, s->write_latch); + hvm_put_8u(h, s->rw_mode); + hvm_put_8u(h, s->mode); + hvm_put_8u(h, s->bcd); + hvm_put_8u(h, s->gate); + hvm_put_64u(h, s->count_load_time); + + if (s->pt.enabled && pti == -1) + pti = i; + } + + pt = &pit->channels[pti].pt; + + /* save the vcpu for pt */ + hvm_put_32u(h, pt->vcpu->vcpu_id); + + /* save guest time */ + hvm_put_8u(h, pti); + hvm_put_32u(h, pt->pending_intr_nr); + hvm_put_64u(h, pt->last_plt_gtime); + +} + +static int pit_load(hvm_domain_context_t *h, void *opaque, int version_id) +{ + struct domain *d = opaque; + PITState *pit = &d->arch.hvm_domain.pl_time.vpit; + PITChannelState *s; + int i, pti, vcpu_id; + u32 period; + + if (version_id != 1) + return -EINVAL; + + for(i = 0; i < 3; i++) { + s = &pit->channels[i]; + s->count = hvm_get_32u(h); + s->latched_count = hvm_get_16u(h); + s->count_latched = hvm_get_8u(h); + s->status_latched = hvm_get_8u(h); + s->status = hvm_get_8u(h); + s->read_state = hvm_get_8u(h); + s->write_state = hvm_get_8u(h); + s->write_latch = hvm_get_8u(h); + s->rw_mode = hvm_get_8u(h); + s->mode = hvm_get_8u(h); + s->bcd = hvm_get_8u(h); + s->gate = hvm_get_8u(h); + s->count_load_time = hvm_get_64u(h); + } + + vcpu_id = hvm_get_32u(h); + + pti = hvm_get_8u(h); + if ( pti < 0 || pti > 2) { + printk("pit load get a wrong channel %d when HVM resume.\n", pti); + return -EINVAL; + } + + s = &pit->channels[pti]; + period = DIV_ROUND((s->count * 1000000000ULL), PIT_FREQ); + + printk("recreate periodic timer %d in mode %d, freq=%d.\n", pti, s->mode, period); + switch (s->mode) { + case 2: + /* create periodic time */ + create_periodic_time(d->vcpu[vcpu_id], &s->pt, period, 0, 0, pit_time_fired, s); + break; + case 1: + /* create one shot time */ + create_periodic_time(d->vcpu[vcpu_id], &s->pt, period, 0, 1, pit_time_fired, s); + break; + default: + printk("pit mode %"PRId8" should not use periodic timer!\n", s->mode); + return -EINVAL; + } + s->pt.pending_intr_nr = hvm_get_32u(h); + s->pt.last_plt_gtime = hvm_get_64u(h); + + pit_info(pit); + + return 0; +} + static void pit_reset(void *opaque) { PITState *pit = opaque; @@ -383,6 +531,7 @@ void pit_init(struct vcpu *v, unsigned long cpu_khz) s++; s->pt.vcpu = v; s++; s->pt.vcpu = v; + hvm_register_savevm(v->domain, "xen_hvm_i8254", PIT_BASE, 1, pit_save, pit_load, v->domain); register_portio_handler(v->domain, PIT_BASE, 4, handle_pit_io); /* register the speaker port */ register_portio_handler(v->domain, 0x61, 1, handle_speaker_io); diff --git a/xen/arch/x86/hvm/intercept.c b/xen/arch/x86/hvm/intercept.c index 3de58812c3..c7b56c0111 100644 --- a/xen/arch/x86/hvm/intercept.c +++ b/xen/arch/x86/hvm/intercept.c @@ -29,6 +29,8 @@ #include #include #include +#include +#include extern struct hvm_mmio_handler hpet_mmio_handler; @@ -155,6 +157,307 @@ static inline void hvm_mmio_access(struct vcpu *v, } } +/* save/restore support */ +#define HVM_FILE_MAGIC 0x54381286 +#define HVM_FILE_VERSION 0x00000001 + +int hvm_register_savevm(struct domain *d, + const char *idstr, + int instance_id, + int version_id, + SaveStateHandler *save_state, + LoadStateHandler *load_state, + void *opaque) +{ + HVMStateEntry *se, **pse; + + if ( (se = xmalloc(struct HVMStateEntry)) == NULL ){ + printk("allocat hvmstate entry fail.\n"); + return -1; + } + + strncpy(se->idstr, idstr, HVM_SE_IDSTR_LEN); + + se->instance_id = instance_id; + se->version_id = version_id; + se->save_state = save_state; + se->load_state = load_state; + se->opaque = opaque; + se->next = NULL; + + /* add at the end of list */ + pse = &d->arch.hvm_domain.first_se; + while (*pse != NULL) + pse = &(*pse)->next; + *pse = se; + return 0; +} + +int hvm_save(struct vcpu *v, hvm_domain_context_t *h) +{ + uint32_t len, len_pos, cur_pos; + uint32_t eax, ebx, ecx, edx; + HVMStateEntry *se; + char *chgset; + + if (!is_hvm_vcpu(v)) { + printk("hvm_save only for hvm guest!\n"); + return -1; + } + + memset(h, 0, sizeof(hvm_domain_context_t)); + hvm_put_32u(h, HVM_FILE_MAGIC); + hvm_put_32u(h, HVM_FILE_VERSION); + + /* save xen changeset */ + chgset = strrchr(XEN_CHANGESET, ' '); + if ( chgset ) + chgset++; + else + chgset = XEN_CHANGESET; + + len = strlen(chgset); + hvm_put_8u(h, len); + hvm_put_buffer(h, chgset, len); + + /* save cpuid */ + cpuid(1, &eax, &ebx, &ecx, &edx); + hvm_put_32u(h, eax); + + for(se = v->domain->arch.hvm_domain.first_se; se != NULL; se = se->next) { + /* ID string */ + len = strnlen(se->idstr, HVM_SE_IDSTR_LEN); + hvm_put_8u(h, len); + hvm_put_buffer(h, se->idstr, len); + + hvm_put_32u(h, se->instance_id); + hvm_put_32u(h, se->version_id); + + /* record size */ + len_pos = hvm_ctxt_tell(h); + hvm_put_32u(h, 0); + + se->save_state(h, se->opaque); + + cur_pos = hvm_ctxt_tell(h); + len = cur_pos - len_pos - 4; + hvm_ctxt_seek(h, len_pos); + hvm_put_32u(h, len); + hvm_ctxt_seek(h, cur_pos); + + } + + h->size = hvm_ctxt_tell(h); + hvm_ctxt_seek(h, 0); + + if (h->size >= HVM_CTXT_SIZE) { + printk("hvm_domain_context overflow when hvm_save! need %"PRId32" bytes for use.\n", h->size); + return -1; + } + + return 0; + +} + +static HVMStateEntry *find_se(struct domain *d, const char *idstr, int instance_id) +{ + HVMStateEntry *se; + + for(se = d->arch.hvm_domain.first_se; se != NULL; se = se->next) { + if (!strncmp(se->idstr, idstr, HVM_SE_IDSTR_LEN) && + instance_id == se->instance_id){ + return se; + } + } + return NULL; +} + +int hvm_load(struct vcpu *v, hvm_domain_context_t *h) +{ + uint32_t len, rec_len, rec_pos, magic, instance_id, version_id; + uint32_t eax, ebx, ecx, edx; + HVMStateEntry *se; + char idstr[HVM_SE_IDSTR_LEN]; + xen_changeset_info_t chgset; + char *cur_chgset; + int ret; + + if (!is_hvm_vcpu(v)) { + printk("hvm_load only for hvm guest!\n"); + return -1; + } + + if (h->size >= HVM_CTXT_SIZE) { + printk("hvm_load fail! seems hvm_domain_context overflow when hvm_save! need %"PRId32" bytes.\n", h->size); + return -1; + } + + hvm_ctxt_seek(h, 0); + + magic = hvm_get_32u(h); + if (magic != HVM_FILE_MAGIC) { + printk("HVM restore magic dismatch!\n"); + return -1; + } + + magic = hvm_get_32u(h); + if (magic != HVM_FILE_VERSION) { + printk("HVM restore version dismatch!\n"); + return -1; + } + + /* check xen change set */ + cur_chgset = strrchr(XEN_CHANGESET, ' '); + if ( cur_chgset ) + cur_chgset++; + else + cur_chgset = XEN_CHANGESET; + + len = hvm_get_8u(h); + if (len > 20) { /*typical length is 18 -- "revision number:changeset id" */ + printk("wrong change set length %d when hvm restore!\n", len); + return -1; + } + + hvm_get_buffer(h, chgset, len); + chgset[len] = '\0'; + if (strncmp(cur_chgset, chgset, len + 1)) + printk("warnings: try to restore hvm guest(%s) on a different changeset %s.\n", + chgset, cur_chgset); + + + if ( !strcmp(cur_chgset, "unavailable") ) + printk("warnings: try to restore hvm guest when changeset is unavailable.\n"); + + + /* check cpuid */ + cpuid(1, &eax, &ebx, &ecx, &edx); + ebx = hvm_get_32u(h); + /*TODO: need difine how big difference is acceptable */ + if (ebx != eax) + printk("warnings: try to restore hvm guest(0x%"PRIx32") " + "on a different type processor(0x%"PRIx32").\n", + ebx, + eax); + + while(1) { + if (hvm_ctxt_end(h)) { + break; + } + + /* ID string */ + len = hvm_get_8u(h); + if (len > HVM_SE_IDSTR_LEN) { + printk("wrong HVM save entry idstr len %d!", len); + return -1; + } + + hvm_get_buffer(h, idstr, len); + idstr[len] = '\0'; + + instance_id = hvm_get_32u(h); + version_id = hvm_get_32u(h); + + rec_len = hvm_get_32u(h); + rec_pos = hvm_ctxt_tell(h); + + se = find_se(v->domain, idstr, instance_id); + if (se == NULL) { + printk("warnings: hvm load can't find device %s's instance %d!\n", + idstr, instance_id); + } else { + ret = se->load_state(h, se->opaque, version_id); + if (ret < 0) + printk("warnings: loading state fail for device %s instance %d!\n", + idstr, instance_id); + } + + + /* make sure to jump end of record */ + if ( hvm_ctxt_tell(h) - rec_pos != rec_len) { + printk("wrong hvm record size, maybe some dismatch between save&restore handler!\n"); + } + hvm_ctxt_seek(h, rec_pos + rec_len); + } + + return 0; +} + +int arch_gethvm_ctxt( + struct vcpu *v, struct hvm_domain_context *c) +{ + if ( !is_hvm_vcpu(v) ) + return -1; + + return hvm_save(v, c); + +} + +int arch_sethvm_ctxt( + struct vcpu *v, struct hvm_domain_context *c) +{ + return hvm_load(v, c); +} + +#ifdef HVM_DEBUG_SUSPEND +static void shpage_info(shared_iopage_t *sh) +{ + + vcpu_iodata_t *p = &sh->vcpu_iodata[0]; + ioreq_t *req = &p->vp_ioreq; + printk("*****sharepage_info******!\n"); + printk("vp_eport=%d\n", p->vp_eport); + printk("io packet: " + "state:%x, pvalid: %x, dir:%x, port: %"PRIx64", " + "data: %"PRIx64", count: %"PRIx64", size: %"PRIx64"\n", + req->state, req->data_is_ptr, req->dir, req->addr, + req->data, req->count, req->size); +} +#else +static void shpage_info(shared_iopage_t *sh) +{ +} +#endif + +static void shpage_save(hvm_domain_context_t *h, void *opaque) +{ + /* XXX:no action required for shpage save/restore, since it's in guest memory + * keep it for debug purpose only */ + +#if 0 + struct shared_iopage *s = opaque; + /* XXX:smp */ + struct ioreq *req = &s->vcpu_iodata[0].vp_ioreq; + + shpage_info(s); + + hvm_put_buffer(h, (char*)req, sizeof(struct ioreq)); +#endif +} + +static int shpage_load(hvm_domain_context_t *h, void *opaque, int version_id) +{ + struct shared_iopage *s = opaque; +#if 0 + /* XXX:smp */ + struct ioreq *req = &s->vcpu_iodata[0].vp_ioreq; + + if (version_id != 1) + return -EINVAL; + + hvm_get_buffer(h, (char*)req, sizeof(struct ioreq)); + + +#endif + shpage_info(s); + return 0; +} + +void shpage_init(struct domain *d, shared_iopage_t *sp) +{ + hvm_register_savevm(d, "xen_hvm_shpage", 0x10, 1, shpage_save, shpage_load, sp); +} + int hvm_buffered_io_intercept(ioreq_t *p) { struct vcpu *v = current; diff --git a/xen/arch/x86/hvm/rtc.c b/xen/arch/x86/hvm/rtc.c index 07e79d7c3c..c8c76fbc3d 100644 --- a/xen/arch/x86/hvm/rtc.c +++ b/xen/arch/x86/hvm/rtc.c @@ -62,7 +62,7 @@ static void rtc_timer_update(RTCState *s) #ifdef DEBUG_RTC printk("HVM_RTC: period = %uns\n", period); #endif - create_periodic_time(&s->pt, period, RTC_IRQ, 0, rtc_periodic_cb, s); + create_periodic_time(current, &s->pt, period, RTC_IRQ, 0, rtc_periodic_cb, s); } else destroy_periodic_time(&s->pt); diff --git a/xen/arch/x86/hvm/vioapic.c b/xen/arch/x86/hvm/vioapic.c index 6f68a85b77..7e23726b29 100644 --- a/xen/arch/x86/hvm/vioapic.c +++ b/xen/arch/x86/hvm/vioapic.c @@ -473,11 +473,143 @@ void vioapic_update_EOI(struct domain *d, int vector) spin_unlock(&hvm_irq->lock); } +#ifdef HVM_DEBUG_SUSPEND +static void ioapic_info(struct vioapic *s) +{ + int i; + printk("*****ioapic state:*****\n"); + printk("ioapic 0x%x.\n", s->ioregsel); + printk("ioapic 0x%x.\n", s->id); + printk("ioapic 0x%lx.\n", s->base_address); + for (i = 0; i < VIOAPIC_NUM_PINS; i++) { + printk("ioapic redirtbl[%d]:0x%"PRIx64"\n", i, s->redirtbl[i].bits); + } + +} +static void hvmirq_info(struct hvm_irq *hvm_irq) +{ + int i; + printk("*****hvmirq state:*****\n"); + for (i = 0; i < BITS_TO_LONGS(32*4); i++) + printk("hvmirq pci_intx[%d]:0x%lx.\n", i, hvm_irq->pci_intx[i]); + + for (i = 0; i < BITS_TO_LONGS(16); i++) + printk("hvmirq isa_irq[%d]:0x%lx.\n", i, hvm_irq->isa_irq[i]); + + for (i = 0; i < BITS_TO_LONGS(1); i++) + printk("hvmirq callback_irq_wire[%d]:0x%lx.\n", i, hvm_irq->callback_irq_wire[i]); + + printk("hvmirq callback_via_type:0x%x.\n", hvm_irq->callback_via_type); + printk("hvmirq callback_via:0x%x.\n", hvm_irq->callback_via.gsi); + + + for (i = 0; i < 4; i++) + printk("hvmirq pci_link_route[%d]:0x%"PRIx8".\n", i, hvm_irq->pci_link_route[i]); + + for (i = 0; i < 4; i++) + printk("hvmirq pci_link_assert_count[%d]:0x%"PRIx8".\n", i, hvm_irq->pci_link_assert_count[i]); + + for (i = 0; i < VIOAPIC_NUM_PINS; i++) + printk("hvmirq gsi_assert_count[%d]:0x%"PRIx8".\n", i, hvm_irq->gsi_assert_count[i]); + + printk("hvmirq round_robin_prev_vcpu:0x%"PRIx8".\n", hvm_irq->round_robin_prev_vcpu); +} +#else +static void ioapic_info(struct vioapic *s) +{ +} +static void hvmirq_info(struct hvm_irq *hvm_irq) +{ +} +#endif + +static void ioapic_save(hvm_domain_context_t *h, void *opaque) +{ + int i; + struct domain *d = opaque; + struct vioapic *s = domain_vioapic(d); + struct hvm_irq *hvm_irq = &d->arch.hvm_domain.irq; + + ioapic_info(s); + hvmirq_info(hvm_irq); + + /* save iopaic state*/ + hvm_put_32u(h, s->ioregsel); + hvm_put_32u(h, s->id); + hvm_put_64u(h, s->base_address); + for (i = 0; i < VIOAPIC_NUM_PINS; i++) { + hvm_put_64u(h, s->redirtbl[i].bits); + } + + /* save hvm irq state */ + hvm_put_buffer(h, (char*)hvm_irq->pci_intx, 16); + hvm_put_buffer(h, (char*)hvm_irq->isa_irq, 2); + hvm_put_32u(h, hvm_irq->callback_via_asserted); + hvm_put_32u(h, hvm_irq->callback_via_type); + hvm_put_32u(h, hvm_irq->callback_via.gsi); + + for (i = 0; i < 4; i++) + hvm_put_8u(h, hvm_irq->pci_link_route[i]); + + for (i = 0; i < 4; i++) + hvm_put_8u(h, hvm_irq->pci_link_assert_count[i]); + + for (i = 0; i < VIOAPIC_NUM_PINS; i++) + hvm_put_8u(h, hvm_irq->gsi_assert_count[i]); + + hvm_put_8u(h, hvm_irq->round_robin_prev_vcpu); + +} + +static int ioapic_load(hvm_domain_context_t *h, void *opaque, int version_id) +{ + int i; + struct domain *d = opaque; + struct vioapic *s = domain_vioapic(d); + struct hvm_irq *hvm_irq = &d->arch.hvm_domain.irq; + + if (version_id != 1) + return -EINVAL; + + /* restore ioapic state */ + s->ioregsel = hvm_get_32u(h); + s->id = hvm_get_32u(h); + s->base_address = hvm_get_64u(h); + for (i = 0; i < VIOAPIC_NUM_PINS; i++) { + s->redirtbl[i].bits = hvm_get_64u(h); + } + + /* restore irq state */ + hvm_get_buffer(h, (char*)hvm_irq->pci_intx, 16); + hvm_get_buffer(h, (char*)hvm_irq->isa_irq, 2); + hvm_irq->callback_via_asserted = hvm_get_32u(h); + hvm_irq->callback_via_type = hvm_get_32u(h); + hvm_irq->callback_via.gsi = hvm_get_32u(h); + + for (i = 0; i < 4; i++) + hvm_irq->pci_link_route[i] = hvm_get_8u(h); + + for (i = 0; i < 4; i++) + hvm_irq->pci_link_assert_count[i] = hvm_get_8u(h); + + for (i = 0; i < VIOAPIC_NUM_PINS; i++) + hvm_irq->gsi_assert_count[i] = hvm_get_8u(h); + + hvm_irq->round_robin_prev_vcpu = hvm_get_8u(h); + + ioapic_info(s); + hvmirq_info(hvm_irq); + + return 0; +} + void vioapic_init(struct domain *d) { struct vioapic *vioapic = domain_vioapic(d); int i; + hvm_register_savevm(d, "xen_hvm_ioapic", 0, 1, ioapic_save, ioapic_load, d); + memset(vioapic, 0, sizeof(*vioapic)); for ( i = 0; i < VIOAPIC_NUM_PINS; i++ ) vioapic->redirtbl[i].fields.mask = 1; diff --git a/xen/arch/x86/hvm/vlapic.c b/xen/arch/x86/hvm/vlapic.c index b1b42650f4..4be3f55e1e 100644 --- a/xen/arch/x86/hvm/vlapic.c +++ b/xen/arch/x86/hvm/vlapic.c @@ -659,7 +659,7 @@ static void vlapic_write(struct vcpu *v, unsigned long address, uint64_t period = APIC_BUS_CYCLE_NS * (uint32_t)val * vlapic->timer_divisor; vlapic_set_reg(vlapic, APIC_TMICT, val); - create_periodic_time(&vlapic->pt, period, vlapic->pt.irq, + create_periodic_time(current, &vlapic->pt, period, vlapic->pt.irq, vlapic_lvtt_period(vlapic), NULL, vlapic); HVM_DBG_LOG(DBG_LEVEL_VLAPIC, @@ -795,6 +795,77 @@ static int vlapic_reset(struct vlapic *vlapic) return 1; } +#ifdef HVM_DEBUG_SUSPEND +static void lapic_info(struct vlapic *s) +{ + printk("*****lapic state:*****\n"); + printk("lapic 0x%"PRIx64".\n", s->apic_base_msr); + printk("lapic 0x%x.\n", s->disabled); + printk("lapic 0x%x.\n", s->timer_divisor); + printk("lapic 0x%x.\n", s->timer_pending_count); +} +#else +static void lapic_info(struct vlapic *s) +{ +} +#endif + +static void lapic_save(hvm_domain_context_t *h, void *opaque) +{ + struct vlapic *s = opaque; + + lapic_info(s); + + hvm_put_64u(h, s->apic_base_msr); + hvm_put_32u(h, s->disabled); + hvm_put_32u(h, s->timer_divisor); + + /*XXX: need this?*/ + hvm_put_32u(h, s->timer_pending_count); + + hvm_put_buffer(h, (char*)s->regs, 0x3f0); + +} + +static int lapic_load(hvm_domain_context_t *h, void *opaque, int version_id) +{ + struct vlapic *s = opaque; + struct vcpu *v = vlapic_vcpu(s); + unsigned long tmict; + + if (version_id != 1) + return -EINVAL; + + s->apic_base_msr = hvm_get_64u(h); + s->disabled = hvm_get_32u(h); + s->timer_divisor = hvm_get_32u(h); + + /*XXX: need this?*/ + s->timer_pending_count = hvm_get_32u(h); + + hvm_get_buffer(h, (char*)s->regs, 0x3f0); + + /* rearm the actiemr if needed */ + tmict = vlapic_get_reg(s, APIC_TMICT); + if (tmict > 0) { + uint64_t period = APIC_BUS_CYCLE_NS * (uint32_t)tmict * s->timer_divisor; + + create_periodic_time(v, &s->pt, period, s->pt.irq, + vlapic_lvtt_period(s), NULL, s); + + printk("lapic_load to rearm the actimer:" + "bus cycle is %uns, " + "saved tmict count %lu, period %"PRIu64"ns\n", + APIC_BUS_CYCLE_NS, tmict, period); + + } + + + lapic_info(s); + + return 0; +} + int vlapic_init(struct vcpu *v) { struct vlapic *vlapic = vcpu_vlapic(v); @@ -813,6 +884,7 @@ int vlapic_init(struct vcpu *v) vlapic->regs = map_domain_page_global(page_to_mfn(vlapic->regs_page)); memset(vlapic->regs, 0, PAGE_SIZE); + hvm_register_savevm(v->domain, "xen_hvm_lapic", v->vcpu_id, 1, lapic_save, lapic_load, vlapic); vlapic_reset(vlapic); vlapic->apic_base_msr = MSR_IA32_APICBASE_ENABLE | APIC_DEFAULT_PHYS_BASE; diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c index a665367c6d..77abecec63 100644 --- a/xen/arch/x86/hvm/vmx/vmx.c +++ b/xen/arch/x86/hvm/vmx/vmx.c @@ -363,6 +363,299 @@ static inline void __restore_debug_registers(struct vcpu *v) /* DR7 is loaded from the VMCS. */ } +static int __get_instruction_length(void); +int vmx_vmcs_save(struct vcpu *v, struct vmcs_data *c) +{ + unsigned long inst_len; + + inst_len = __get_instruction_length(); + c->eip = __vmread(GUEST_RIP); + +#ifdef HVM_DEBUG_SUSPEND + printk("vmx_vmcs_save: inst_len=0x%lx, eip=0x%"PRIx64".\n", + inst_len, c->eip); +#endif + + c->esp = __vmread(GUEST_RSP); + c->eflags = __vmread(GUEST_RFLAGS); + + c->cr0 = v->arch.hvm_vmx.cpu_shadow_cr0; + c->cr3 = v->arch.hvm_vmx.cpu_cr3; + c->cr4 = v->arch.hvm_vmx.cpu_shadow_cr4; + +#ifdef HVM_DEBUG_SUSPEND + printk("vmx_vmcs_save: cr3=0x%"PRIx64", cr0=0x%"PRIx64", cr4=0x%"PRIx64".\n", + c->cr3, + c->cr0, + c->cr4); +#endif + + c->idtr_limit = __vmread(GUEST_IDTR_LIMIT); + c->idtr_base = __vmread(GUEST_IDTR_BASE); + + c->gdtr_limit = __vmread(GUEST_GDTR_LIMIT); + c->gdtr_base = __vmread(GUEST_GDTR_BASE); + + c->cs_sel = __vmread(GUEST_CS_SELECTOR); + c->cs_limit = __vmread(GUEST_CS_LIMIT); + c->cs_base = __vmread(GUEST_CS_BASE); + c->cs_arbytes = __vmread(GUEST_CS_AR_BYTES); + + c->ds_sel = __vmread(GUEST_DS_SELECTOR); + c->ds_limit = __vmread(GUEST_DS_LIMIT); + c->ds_base = __vmread(GUEST_DS_BASE); + c->ds_arbytes = __vmread(GUEST_DS_AR_BYTES); + + c->es_sel = __vmread(GUEST_ES_SELECTOR); + c->es_limit = __vmread(GUEST_ES_LIMIT); + c->es_base = __vmread(GUEST_ES_BASE); + c->es_arbytes = __vmread(GUEST_ES_AR_BYTES); + + c->ss_sel = __vmread(GUEST_SS_SELECTOR); + c->ss_limit = __vmread(GUEST_SS_LIMIT); + c->ss_base = __vmread(GUEST_SS_BASE); + c->ss_arbytes = __vmread(GUEST_SS_AR_BYTES); + + c->fs_sel = __vmread(GUEST_FS_SELECTOR); + c->fs_limit = __vmread(GUEST_FS_LIMIT); + c->fs_base = __vmread(GUEST_FS_BASE); + c->fs_arbytes = __vmread(GUEST_FS_AR_BYTES); + + c->gs_sel = __vmread(GUEST_GS_SELECTOR); + c->gs_limit = __vmread(GUEST_GS_LIMIT); + c->gs_base = __vmread(GUEST_GS_BASE); + c->gs_arbytes = __vmread(GUEST_GS_AR_BYTES); + + c->tr_sel = __vmread(GUEST_TR_SELECTOR); + c->tr_limit = __vmread(GUEST_TR_LIMIT); + c->tr_base = __vmread(GUEST_TR_BASE); + c->tr_arbytes = __vmread(GUEST_TR_AR_BYTES); + + c->ldtr_sel = __vmread(GUEST_LDTR_SELECTOR); + c->ldtr_limit = __vmread(GUEST_LDTR_LIMIT); + c->ldtr_base = __vmread(GUEST_LDTR_BASE); + c->ldtr_arbytes = __vmread(GUEST_LDTR_AR_BYTES); + + c->sysenter_cs = __vmread(GUEST_SYSENTER_CS); + c->sysenter_esp = __vmread(GUEST_SYSENTER_ESP); + c->sysenter_eip = __vmread(GUEST_SYSENTER_EIP); + + return 1; +} + +int vmx_vmcs_restore(struct vcpu *v, struct vmcs_data *c) +{ + unsigned long mfn, old_base_mfn; + + vmx_vmcs_enter(v); + + __vmwrite(GUEST_RIP, c->eip); + __vmwrite(GUEST_RSP, c->esp); + __vmwrite(GUEST_RFLAGS, c->eflags); + + v->arch.hvm_vmx.cpu_shadow_cr0 = c->cr0; + __vmwrite(CR0_READ_SHADOW, v->arch.hvm_vmx.cpu_shadow_cr0); + +#ifdef HVM_DEBUG_SUSPEND + printk("vmx_vmcs_restore: cr3=0x%"PRIx64", cr0=0x%"PRIx64", cr4=0x%"PRIx64".\n", + c->cr3, + c->cr0, + c->cr4); +#endif + + if (!vmx_paging_enabled(v)) { + printk("vmx_vmcs_restore: paging not enabled."); + goto skip_cr3; + } + + if (c->cr3 == v->arch.hvm_vmx.cpu_cr3) { + /* + * This is simple TLB flush, implying the guest has + * removed some translation or changed page attributes. + * We simply invalidate the shadow. + */ + mfn = gmfn_to_mfn(v->domain, c->cr3 >> PAGE_SHIFT); + if (mfn != pagetable_get_pfn(v->arch.guest_table)) { + goto bad_cr3; + } + } else { + /* + * If different, make a shadow. Check if the PDBR is valid + * first. + */ + HVM_DBG_LOG(DBG_LEVEL_VMMU, "CR3 c->cr3 = %"PRIx64"", c->cr3); + /* current!=vcpu as not called by arch_vmx_do_launch */ + mfn = gmfn_to_mfn(v->domain, c->cr3 >> PAGE_SHIFT); + if( !mfn_valid(mfn) || !get_page(mfn_to_page(mfn), v->domain)) { + goto bad_cr3; + } + old_base_mfn = pagetable_get_pfn(v->arch.guest_table); + v->arch.guest_table = pagetable_from_pfn(mfn); + if (old_base_mfn) + put_page(mfn_to_page(old_base_mfn)); + /* + * arch.shadow_table should now hold the next CR3 for shadow + */ + v->arch.hvm_vmx.cpu_cr3 = c->cr3; + } + + skip_cr3: +#if defined(__x86_64__) + if (vmx_long_mode_enabled(v)) { + unsigned long vm_entry_value; + vm_entry_value = __vmread(VM_ENTRY_CONTROLS); + vm_entry_value |= VM_ENTRY_IA32E_MODE; + __vmwrite(VM_ENTRY_CONTROLS, vm_entry_value); + } +#endif + + __vmwrite(GUEST_CR4, (c->cr4 | VMX_CR4_HOST_MASK)); + v->arch.hvm_vmx.cpu_shadow_cr4 = c->cr4; + __vmwrite(CR4_READ_SHADOW, v->arch.hvm_vmx.cpu_shadow_cr4); + + __vmwrite(GUEST_IDTR_LIMIT, c->idtr_limit); + __vmwrite(GUEST_IDTR_BASE, c->idtr_base); + + __vmwrite(GUEST_GDTR_LIMIT, c->gdtr_limit); + __vmwrite(GUEST_GDTR_BASE, c->gdtr_base); + + __vmwrite(GUEST_CS_SELECTOR, c->cs_sel); + __vmwrite(GUEST_CS_LIMIT, c->cs_limit); + __vmwrite(GUEST_CS_BASE, c->cs_base); + __vmwrite(GUEST_CS_AR_BYTES, c->cs_arbytes); + + __vmwrite(GUEST_DS_SELECTOR, c->ds_sel); + __vmwrite(GUEST_DS_LIMIT, c->ds_limit); + __vmwrite(GUEST_DS_BASE, c->ds_base); + __vmwrite(GUEST_DS_AR_BYTES, c->ds_arbytes); + + __vmwrite(GUEST_ES_SELECTOR, c->es_sel); + __vmwrite(GUEST_ES_LIMIT, c->es_limit); + __vmwrite(GUEST_ES_BASE, c->es_base); + __vmwrite(GUEST_ES_AR_BYTES, c->es_arbytes); + + __vmwrite(GUEST_SS_SELECTOR, c->ss_sel); + __vmwrite(GUEST_SS_LIMIT, c->ss_limit); + __vmwrite(GUEST_SS_BASE, c->ss_base); + __vmwrite(GUEST_SS_AR_BYTES, c->ss_arbytes); + + __vmwrite(GUEST_FS_SELECTOR, c->fs_sel); + __vmwrite(GUEST_FS_LIMIT, c->fs_limit); + __vmwrite(GUEST_FS_BASE, c->fs_base); + __vmwrite(GUEST_FS_AR_BYTES, c->fs_arbytes); + + __vmwrite(GUEST_GS_SELECTOR, c->gs_sel); + __vmwrite(GUEST_GS_LIMIT, c->gs_limit); + __vmwrite(GUEST_GS_BASE, c->gs_base); + __vmwrite(GUEST_GS_AR_BYTES, c->gs_arbytes); + + __vmwrite(GUEST_TR_SELECTOR, c->tr_sel); + __vmwrite(GUEST_TR_LIMIT, c->tr_limit); + __vmwrite(GUEST_TR_BASE, c->tr_base); + __vmwrite(GUEST_TR_AR_BYTES, c->tr_arbytes); + + __vmwrite(GUEST_LDTR_SELECTOR, c->ldtr_sel); + __vmwrite(GUEST_LDTR_LIMIT, c->ldtr_limit); + __vmwrite(GUEST_LDTR_BASE, c->ldtr_base); + __vmwrite(GUEST_LDTR_AR_BYTES, c->ldtr_arbytes); + + __vmwrite(GUEST_SYSENTER_CS, c->sysenter_cs); + __vmwrite(GUEST_SYSENTER_ESP, c->sysenter_esp); + __vmwrite(GUEST_SYSENTER_EIP, c->sysenter_eip); + + vmx_vmcs_exit(v); + + shadow_update_paging_modes(v); + return 0; + + bad_cr3: + gdprintk(XENLOG_ERR, "Invalid CR3 value=0x%"PRIx64"", c->cr3); + vmx_vmcs_exit(v); + return -EINVAL; +} + +#ifdef HVM_DEBUG_SUSPEND +static void dump_msr_state(struct vmx_msr_state *m) +{ + int i = 0; + printk("**** msr state ****\n"); + printk("shadow_gs=0x%lx, flags=0x%lx, msr_items:", m->shadow_gs, m->flags); + for (i = 0; i < VMX_MSR_COUNT; i++) + printk("0x%lx,", m->msrs[i]); + printk("\n"); +} +#else +static void dump_msr_state(struct vmx_msr_state *m) +{ +} +#endif + +void vmx_save_cpu_state(struct vcpu *v, struct hvmcpu_context *ctxt) +{ + struct vmcs_data *data = &ctxt->data; + struct vmx_msr_state *guest_state = &v->arch.hvm_vmx.msr_state; + unsigned long guest_flags = guest_state->flags; + int i = 0; + + data->shadow_gs = guest_state->shadow_gs; + data->vmxassist_enabled = v->arch.hvm_vmx.vmxassist_enabled; + /* save msrs */ + data->flags = guest_flags; + for (i = 0; i < VMX_MSR_COUNT; i++) + data->msr_items[i] = guest_state->msrs[i]; + + dump_msr_state(guest_state); +} + +void vmx_load_cpu_state(struct vcpu *v, struct hvmcpu_context *ctxt) +{ + int i = 0; + struct vmcs_data *data = &ctxt->data; + struct vmx_msr_state *guest_state = &v->arch.hvm_vmx.msr_state; + + /* restore msrs */ + guest_state->flags = data->flags; + for (i = 0; i < VMX_MSR_COUNT; i++) + guest_state->msrs[i] = data->msr_items[i]; + + guest_state->shadow_gs = data->shadow_gs; + + /*XXX:no need to restore msrs, current!=vcpu as not called by arch_vmx_do_launch */ +/* vmx_restore_guest_msrs(v);*/ + + v->arch.hvm_vmx.vmxassist_enabled = data->vmxassist_enabled; + + dump_msr_state(guest_state); +} + +void vmx_save_vmcs_ctxt(struct vcpu *v, struct hvmcpu_context *ctxt) +{ + struct vmcs_data *data = &ctxt->data; + + vmx_save_cpu_state(v, ctxt); + + vmx_vmcs_enter(v); + + vmx_vmcs_save(v, data); + + vmx_vmcs_exit(v); + +} + +void vmx_load_vmcs_ctxt(struct vcpu *v, struct hvmcpu_context *ctxt) +{ + vmx_load_cpu_state(v, ctxt); + + if (vmx_vmcs_restore(v, &ctxt->data)) { + printk("vmx_vmcs restore failed!\n"); + domain_crash(v->domain); + } + + /* only load vmcs once */ + ctxt->valid = 0; + +} + /* * DR7 is saved and restored on every vmexit. Other debug registers only * need to be restored if their value is going to affect execution -- i.e., @@ -721,6 +1014,9 @@ static void vmx_setup_hvm_funcs(void) hvm_funcs.store_cpu_guest_regs = vmx_store_cpu_guest_regs; hvm_funcs.load_cpu_guest_regs = vmx_load_cpu_guest_regs; + hvm_funcs.save_cpu_ctxt = vmx_save_vmcs_ctxt; + hvm_funcs.load_cpu_ctxt = vmx_load_vmcs_ctxt; + hvm_funcs.paging_enabled = vmx_paging_enabled; hvm_funcs.long_mode_enabled = vmx_long_mode_enabled; hvm_funcs.pae_enabled = vmx_pae_enabled; diff --git a/xen/arch/x86/hvm/vpic.c b/xen/arch/x86/hvm/vpic.c index edd7e6b4ac..4f36dd3bdf 100644 --- a/xen/arch/x86/hvm/vpic.c +++ b/xen/arch/x86/hvm/vpic.c @@ -378,6 +378,87 @@ static int vpic_intercept_elcr_io(ioreq_t *p) return 1; } +#ifdef HVM_DEBUG_SUSPEND +static void vpic_info(struct vpic *s) +{ + printk("*****pic state:*****\n"); + printk("pic 0x%x.\n", s->irr); + printk("pic 0x%x.\n", s->imr); + printk("pic 0x%x.\n", s->isr); + printk("pic 0x%x.\n", s->irq_base); + printk("pic 0x%x.\n", s->init_state); + printk("pic 0x%x.\n", s->priority_add); + printk("pic 0x%x.\n", s->readsel_isr); + printk("pic 0x%x.\n", s->poll); + printk("pic 0x%x.\n", s->auto_eoi); + printk("pic 0x%x.\n", s->rotate_on_auto_eoi); + printk("pic 0x%x.\n", s->special_fully_nested_mode); + printk("pic 0x%x.\n", s->special_mask_mode); + printk("pic 0x%x.\n", s->elcr); + printk("pic 0x%x.\n", s->int_output); + printk("pic 0x%x.\n", s->is_master); +} +#else +static void vpic_info(struct vpic *s) +{ +} +#endif + +static void vpic_save(hvm_domain_context_t *h, void *opaque) +{ + struct vpic *s = opaque; + + vpic_info(s); + + hvm_put_8u(h, s->irr); + hvm_put_8u(h, s->imr); + hvm_put_8u(h, s->isr); + hvm_put_8u(h, s->irq_base); + hvm_put_8u(h, s->init_state); + hvm_put_8u(h, s->priority_add); + hvm_put_8u(h, s->readsel_isr); + + hvm_put_8u(h, s->poll); + hvm_put_8u(h, s->auto_eoi); + + hvm_put_8u(h, s->rotate_on_auto_eoi); + hvm_put_8u(h, s->special_fully_nested_mode); + hvm_put_8u(h, s->special_mask_mode); + + hvm_put_8u(h, s->elcr); + hvm_put_8u(h, s->int_output); +} + +static int vpic_load(hvm_domain_context_t *h, void *opaque, int version_id) +{ + struct vpic *s = opaque; + + if (version_id != 1) + return -EINVAL; + + s->irr = hvm_get_8u(h); + s->imr = hvm_get_8u(h); + s->isr = hvm_get_8u(h); + s->irq_base = hvm_get_8u(h); + s->init_state = hvm_get_8u(h); + s->priority_add = hvm_get_8u(h); + s->readsel_isr = hvm_get_8u(h); + + s->poll = hvm_get_8u(h); + s->auto_eoi = hvm_get_8u(h); + + s->rotate_on_auto_eoi = hvm_get_8u(h); + s->special_fully_nested_mode = hvm_get_8u(h); + s->special_mask_mode = hvm_get_8u(h); + + s->elcr = hvm_get_8u(h); + s->int_output = hvm_get_8u(h); + + vpic_info(s); + + return 0; +} + void vpic_init(struct domain *d) { struct vpic *vpic; @@ -387,12 +468,14 @@ void vpic_init(struct domain *d) memset(vpic, 0, sizeof(*vpic)); vpic->is_master = 1; vpic->elcr = 1 << 2; + hvm_register_savevm(d, "xen_hvm_i8259", 0x20, 1, vpic_save, vpic_load, vpic); register_portio_handler(d, 0x20, 2, vpic_intercept_pic_io); register_portio_handler(d, 0x4d0, 1, vpic_intercept_elcr_io); /* Slave PIC. */ vpic++; memset(vpic, 0, sizeof(*vpic)); + hvm_register_savevm(d, "xen_hvm_i8259", 0xa0, 1, vpic_save, vpic_load, vpic); register_portio_handler(d, 0xa0, 2, vpic_intercept_pic_io); register_portio_handler(d, 0x4d1, 1, vpic_intercept_elcr_io); } diff --git a/xen/arch/x86/hvm/vpt.c b/xen/arch/x86/hvm/vpt.c index 12e8999582..8287289172 100644 --- a/xen/arch/x86/hvm/vpt.c +++ b/xen/arch/x86/hvm/vpt.c @@ -195,7 +195,7 @@ void pt_reset(struct vcpu *v) } } -void create_periodic_time(struct periodic_time *pt, uint64_t period, +void create_periodic_time(struct vcpu *v, struct periodic_time *pt, uint64_t period, uint8_t irq, char one_shot, time_cb *cb, void *data) { destroy_periodic_time(pt); @@ -209,7 +209,7 @@ void create_periodic_time(struct periodic_time *pt, uint64_t period, period = 900000; /* force to 0.9ms */ } pt->period = period; - pt->vcpu = current; + pt->vcpu = v; pt->last_plt_gtime = hvm_get_guest_time(pt->vcpu); pt->irq = irq; pt->period_cycles = (u64)period * cpu_khz / 1000000L; @@ -218,7 +218,7 @@ void create_periodic_time(struct periodic_time *pt, uint64_t period, pt->cb = cb; pt->priv = data; - list_add(&pt->list, ¤t->arch.hvm_vcpu.tm_list); + list_add(&pt->list, &v->arch.hvm_vcpu.tm_list); set_timer(&pt->timer, pt->scheduled); } diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index 406941516d..49501411e6 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -154,6 +154,15 @@ l2_pgentry_t *compat_idle_pg_table_l2 = NULL; #define l3_disallow_mask(d) L3_DISALLOW_MASK #endif +static void queue_deferred_ops(struct domain *d, unsigned int ops) +{ + if ( d == current->domain ) + this_cpu(percpu_mm_info).deferred_ops |= ops; + else + BUG_ON(!test_bit(_DOMF_paused, &d->domain_flags) || + !cpus_empty(d->domain_dirty_cpumask)); +} + void __init init_frametable(void) { unsigned long nr_pages, page_step, i, mfn; @@ -416,8 +425,7 @@ void invalidate_shadow_ldt(struct vcpu *v) } /* Dispose of the (now possibly invalid) mappings from the TLB. */ - ASSERT(v->processor == smp_processor_id()); - this_cpu(percpu_mm_info).deferred_ops |= DOP_FLUSH_TLB | DOP_RELOAD_LDT; + queue_deferred_ops(v->domain, DOP_FLUSH_TLB | DOP_RELOAD_LDT); } @@ -945,7 +953,8 @@ static int create_pae_xen_mappings(struct domain *d, l3_pgentry_t *pl3e) } #else memcpy(&pl2e[COMPAT_L2_PAGETABLE_FIRST_XEN_SLOT(d)], - &compat_idle_pg_table_l2[l2_table_offset(HIRO_COMPAT_MPT_VIRT_START)], + &compat_idle_pg_table_l2[ + l2_table_offset(HIRO_COMPAT_MPT_VIRT_START)], COMPAT_L2_PAGETABLE_XEN_SLOTS(d) * sizeof(*pl2e)); #endif unmap_domain_page(pl2e); @@ -1561,7 +1570,7 @@ void free_page_type(struct page_info *page, unsigned long type) * (e.g., update_va_mapping()) or we could end up modifying a page * that is no longer a page table (and hence screw up ref counts). */ - this_cpu(percpu_mm_info).deferred_ops |= DOP_FLUSH_ALL_TLBS; + queue_deferred_ops(owner, DOP_FLUSH_ALL_TLBS); if ( unlikely(shadow_mode_enabled(owner)) ) { @@ -1759,24 +1768,14 @@ int new_guest_cr3(unsigned long mfn) int okay; unsigned long old_base_mfn; - if ( is_hvm_domain(d) && !hvm_paging_enabled(v) ) - return 0; - #ifdef CONFIG_COMPAT if ( IS_COMPAT(d) ) { - l4_pgentry_t l4e = l4e_from_pfn(mfn, _PAGE_PRESENT|_PAGE_RW|_PAGE_USER|_PAGE_ACCESSED); - - if ( shadow_mode_refcounts(d) ) - { - okay = get_page_from_pagenr(mfn, d); - old_base_mfn = l4e_get_pfn(l4e); - if ( okay && old_base_mfn ) - put_page(mfn_to_page(old_base_mfn)); - } - else - okay = mod_l4_entry(__va(pagetable_get_paddr(v->arch.guest_table)), - l4e, 0); + okay = shadow_mode_refcounts(d) + ? 0 /* Old code was broken, but what should it be? */ + : mod_l4_entry(__va(pagetable_get_paddr(v->arch.guest_table)), + l4e_from_pfn(mfn, (_PAGE_PRESENT|_PAGE_RW| + _PAGE_USER|_PAGE_ACCESSED)), 0); if ( unlikely(!okay) ) { MEM_LOG("Error while installing new compat baseptr %lx", mfn); @@ -1789,41 +1788,13 @@ int new_guest_cr3(unsigned long mfn) return 1; } #endif - if ( shadow_mode_refcounts(d) ) + okay = shadow_mode_refcounts(d) + ? get_page_from_pagenr(mfn, d) + : get_page_and_type_from_pagenr(mfn, PGT_root_page_table, d); + if ( unlikely(!okay) ) { - okay = get_page_from_pagenr(mfn, d); - if ( unlikely(!okay) ) - { - MEM_LOG("Error while installing new baseptr %lx", mfn); - return 0; - } - } - else - { - okay = get_page_and_type_from_pagenr(mfn, PGT_root_page_table, d); - if ( unlikely(!okay) ) - { - /* Switch to idle pagetable: this VCPU has no active p.t. now. */ - MEM_LOG("New baseptr %lx: slow path via idle pagetables", mfn); - old_base_mfn = pagetable_get_pfn(v->arch.guest_table); - v->arch.guest_table = pagetable_null(); - update_cr3(v); - write_cr3(__pa(idle_pg_table)); - if ( old_base_mfn != 0 ) - put_page_and_type(mfn_to_page(old_base_mfn)); - - /* Retry the validation with no active p.t. for this VCPU. */ - okay = get_page_and_type_from_pagenr(mfn, PGT_root_page_table, d); - if ( !okay ) - { - /* Failure here is unrecoverable: the VCPU has no pagetable! */ - MEM_LOG("Fatal error while installing new baseptr %lx", mfn); - domain_crash(d); - ASSERT(v->processor == smp_processor_id()); - this_cpu(percpu_mm_info).deferred_ops = 0; - return 0; - } - } + MEM_LOG("Error while installing new baseptr %lx", mfn); + return 0; } invalidate_shadow_ldt(v); @@ -1831,7 +1802,7 @@ int new_guest_cr3(unsigned long mfn) old_base_mfn = pagetable_get_pfn(v->arch.guest_table); v->arch.guest_table = pagetable_from_pfn(mfn); - update_cr3(v); /* update shadow_table and cr3 fields of vcpu struct */ + update_cr3(v); write_ptbase(v); diff --git a/xen/arch/x86/mm/shadow/common.c b/xen/arch/x86/mm/shadow/common.c index 115ad74d72..b821222634 100644 --- a/xen/arch/x86/mm/shadow/common.c +++ b/xen/arch/x86/mm/shadow/common.c @@ -2184,10 +2184,11 @@ int sh_remove_all_mappings(struct vcpu *v, mfn_t gmfn) expected_count = (page->count_info & PGC_allocated) ? 1 : 0; if ( (page->count_info & PGC_count_mask) != expected_count ) { - /* Don't complain if we're in HVM and there's one extra mapping: - * The qemu helper process has an untyped mapping of this dom's RAM */ + /* Don't complain if we're in HVM and there are some extra mappings: + * The qemu helper process has an untyped mapping of this dom's RAM + * and the HVM restore program takes another. */ if ( !(shadow_mode_external(v->domain) - && (page->count_info & PGC_count_mask) <= 2 + && (page->count_info & PGC_count_mask) <= 3 && (page->u.inuse.type_info & PGT_count_mask) == 0) ) { SHADOW_ERROR("can't find all mappings of mfn %lx: " diff --git a/xen/arch/x86/mm/shadow/multi.c b/xen/arch/x86/mm/shadow/multi.c index 0546cb1a66..8b8b8585a7 100644 --- a/xen/arch/x86/mm/shadow/multi.c +++ b/xen/arch/x86/mm/shadow/multi.c @@ -1627,7 +1627,7 @@ sh_make_shadow(struct vcpu *v, mfn_t gmfn, u32 shadow_type) default: /* Do nothing */ break; } } - + shadow_promote(v, gmfn, shadow_type); set_shadow_status(v, gmfn, shadow_type, smfn); diff --git a/xen/common/domain.c b/xen/common/domain.c index 28530c40d5..de7892d6b5 100644 --- a/xen/common/domain.c +++ b/xen/common/domain.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #ifdef CONFIG_COMPAT @@ -505,6 +506,14 @@ int set_info_guest(struct domain *d, if ( rc == 0 ) rc = arch_set_info_guest(v, c); + /*XXX: hvm smp guest restore support */ + if ( rc == 0 && + v->vcpu_id != 0 && + is_hvm_vcpu(v) && + test_and_clear_bit(_VCPUF_down, &v->vcpu_flags) ) { + vcpu_wake(v); + } + domain_unpause(d); xfree(c.nat); diff --git a/xen/common/domctl.c b/xen/common/domctl.c index b9cd57e949..4494c84494 100644 --- a/xen/common/domctl.c +++ b/xen/common/domctl.c @@ -215,6 +215,39 @@ ret_t do_domctl(XEN_GUEST_HANDLE(xen_domctl_t) u_domctl) } break; + case XEN_DOMCTL_sethvmcontext: + { + struct hvm_domain_context *c; + struct domain *d; + struct vcpu *v; + + ret = -ESRCH; + if ( (d = find_domain_by_id(op->domain)) == NULL ) + break; + + ret = -ENOMEM; + if ( (c = xmalloc(struct hvm_domain_context)) == NULL ) + goto sethvmcontext_out; + + v = d->vcpu[0]; + + ret = -EFAULT; + +#ifndef CONFIG_COMPAT + if ( copy_from_guest(c, op->u.hvmcontext.ctxt, 1) != 0 ) + goto sethvmcontext_out; + + ret = arch_sethvm_ctxt(v, c); +#endif + + xfree(c); + + sethvmcontext_out: + put_domain(d); + + } + break; + case XEN_DOMCTL_pausedomain: { struct domain *d = find_domain_by_id(op->domain); @@ -552,6 +585,46 @@ ret_t do_domctl(XEN_GUEST_HANDLE(xen_domctl_t) u_domctl) } break; + case XEN_DOMCTL_gethvmcontext: + { + struct hvm_domain_context *c; + struct domain *d; + struct vcpu *v; + + ret = -ESRCH; + if ( (d = find_domain_by_id(op->domain)) == NULL ) + break; + + ret = -ENOMEM; + if ( (c = xmalloc(struct hvm_domain_context)) == NULL ) + goto gethvmcontext_out; + + v = d->vcpu[0]; + + ret = -ENODATA; + if ( !test_bit(_VCPUF_initialised, &v->vcpu_flags) ) + goto gethvmcontext_out; + + ret = 0; + if (arch_gethvm_ctxt(v, c) == -1) + ret = -EFAULT; + +#ifndef CONFIG_COMPAT + if ( copy_to_guest(op->u.hvmcontext.ctxt, c, 1) ) + ret = -EFAULT; + + xfree(c); +#endif + + if ( copy_to_guest(u_domctl, op, 1) ) + ret = -EFAULT; + + gethvmcontext_out: + put_domain(d); + + } + break; + case XEN_DOMCTL_getvcpuinfo: { struct domain *d; diff --git a/xen/include/asm-x86/hvm/domain.h b/xen/include/asm-x86/hvm/domain.h index 63159504ae..35cc998585 100644 --- a/xen/include/asm-x86/hvm/domain.h +++ b/xen/include/asm-x86/hvm/domain.h @@ -27,6 +27,20 @@ #include #include +typedef void SaveStateHandler(hvm_domain_context_t *h, void *opaque); +typedef int LoadStateHandler(hvm_domain_context_t *h, void *opaque, int version_id); + +#define HVM_SE_IDSTR_LEN 32 +typedef struct HVMStateEntry { + char idstr[HVM_SE_IDSTR_LEN]; + int instance_id; + int version_id; + SaveStateHandler *save_state; + LoadStateHandler *load_state; + void *opaque; + struct HVMStateEntry *next; +} HVMStateEntry; + struct hvm_domain { unsigned long shared_page_va; unsigned long buffered_io_va; @@ -44,6 +58,9 @@ struct hvm_domain { spinlock_t pbuf_lock; uint64_t params[HVM_NR_PARAMS]; + + struct hvm_domain_context *hvm_ctxt; + HVMStateEntry *first_se; }; #endif /* __ASM_X86_HVM_DOMAIN_H__ */ diff --git a/xen/include/asm-x86/hvm/hvm.h b/xen/include/asm-x86/hvm/hvm.h index 5051ab276b..95a1cfbe9f 100644 --- a/xen/include/asm-x86/hvm/hvm.h +++ b/xen/include/asm-x86/hvm/hvm.h @@ -79,6 +79,13 @@ struct hvm_function_table { struct vcpu *v, struct cpu_user_regs *r, unsigned long *crs); void (*load_cpu_guest_regs)( struct vcpu *v, struct cpu_user_regs *r); + + /* save and load hvm guest cpu context for save/restore */ + void (*save_cpu_ctxt)( + struct vcpu *v, struct hvmcpu_context *ctxt); + void (*load_cpu_ctxt)( + struct vcpu *v, struct hvmcpu_context *ctxt); + /* * Examine specifics of the guest state: * 1) determine whether paging is enabled, @@ -157,6 +164,35 @@ hvm_load_cpu_guest_regs(struct vcpu *v, struct cpu_user_regs *r) hvm_funcs.load_cpu_guest_regs(v, r); } +void hvm_set_guest_time(struct vcpu *v, u64 gtime); +u64 hvm_get_guest_time(struct vcpu *v); + +static inline void +hvm_save_cpu_context( + struct vcpu *v, struct hvmcpu_context *ctxt) +{ + hvm_funcs.save_cpu_ctxt(v, ctxt); + + /* save guest time */ + ctxt->gtime = hvm_get_guest_time(v); + + /* set valid flag to recover whole vmcs when restore */ + ctxt->valid = 0x55885588; +} + +static inline void +hvm_load_cpu_context( + struct vcpu *v, struct hvmcpu_context *ctxt) +{ + if ( ctxt->valid != 0x55885588) + return; + + hvm_funcs.load_cpu_ctxt(v, ctxt); + + /* restore guest time*/ + hvm_set_guest_time(v, ctxt->gtime); +} + static inline int hvm_paging_enabled(struct vcpu *v) { @@ -222,8 +258,6 @@ hvm_get_segment_register(struct vcpu *v, enum x86_segment seg, void hvm_cpuid(unsigned int input, unsigned int *eax, unsigned int *ebx, unsigned int *ecx, unsigned int *edx); void hvm_stts(struct vcpu *v); -void hvm_set_guest_time(struct vcpu *v, u64 gtime); -u64 hvm_get_guest_time(struct vcpu *v); void hvm_migrate_timers(struct vcpu *v); void hvm_do_resume(struct vcpu *v); diff --git a/xen/include/asm-x86/hvm/support.h b/xen/include/asm-x86/hvm/support.h index 721492412a..2274578aa1 100644 --- a/xen/include/asm-x86/hvm/support.h +++ b/xen/include/asm-x86/hvm/support.h @@ -121,6 +121,133 @@ extern unsigned int opt_hvm_debug_level; #define TRACE_VMEXIT(index, value) \ current->arch.hvm_vcpu.hvm_trace_values[index] = (value) +/* save/restore support */ + +//#define HVM_DEBUG_SUSPEND + +extern int hvm_register_savevm(struct domain *d, + const char *idstr, + int instance_id, + int version_id, + SaveStateHandler *save_state, + LoadStateHandler *load_state, + void *opaque); + +static inline void hvm_ctxt_seek(hvm_domain_context_t *h, unsigned int pos) +{ + h->cur = pos; +} + +static inline uint32_t hvm_ctxt_tell(hvm_domain_context_t *h) +{ + return h->cur; +} + +static inline int hvm_ctxt_end(hvm_domain_context_t *h) +{ + return (h->cur >= h->size || h->cur >= HVM_CTXT_SIZE); +} + +static inline void hvm_put_byte(hvm_domain_context_t *h, unsigned int i) +{ + if (h->cur >= HVM_CTXT_SIZE) { + h->cur++; + return; + } + h->data[h->cur++] = (char)i; +} + +static inline void hvm_put_8u(hvm_domain_context_t *h, uint8_t b) +{ + hvm_put_byte(h, b); +} + +static inline void hvm_put_16u(hvm_domain_context_t *h, uint16_t b) +{ + hvm_put_8u(h, b >> 8); + hvm_put_8u(h, b); +} + +static inline void hvm_put_32u(hvm_domain_context_t *h, uint32_t b) +{ + hvm_put_16u(h, b >> 16); + hvm_put_16u(h, b); +} + +static inline void hvm_put_64u(hvm_domain_context_t *h, uint64_t b) +{ + hvm_put_32u(h, b >> 32); + hvm_put_32u(h, b); +} + +static inline void hvm_put_buffer(hvm_domain_context_t *h, const char *buf, int len) +{ + memcpy(&h->data[h->cur], buf, len); + h->cur += len; +} + + +static inline char hvm_get_byte(hvm_domain_context_t *h) +{ + if (h->cur >= HVM_CTXT_SIZE) { + printk("hvm_get_byte overflow.\n"); + return -1; + } + + if (h->cur >= h->size) { + printk("hvm_get_byte exceed data area.\n"); + return -1; + } + + return h->data[h->cur++]; +} + +static inline uint8_t hvm_get_8u(hvm_domain_context_t *h) +{ + return hvm_get_byte(h); +} + +static inline uint16_t hvm_get_16u(hvm_domain_context_t *h) +{ + uint16_t v; + v = hvm_get_8u(h) << 8; + v |= hvm_get_8u(h); + + return v; +} + +static inline uint32_t hvm_get_32u(hvm_domain_context_t *h) +{ + uint32_t v; + v = hvm_get_16u(h) << 16; + v |= hvm_get_16u(h); + + return v; +} + +static inline uint64_t hvm_get_64u(hvm_domain_context_t *h) +{ + uint64_t v; + v = (uint64_t)hvm_get_32u(h) << 32; + v |= hvm_get_32u(h); + + return v; +} + +static inline void hvm_get_buffer(hvm_domain_context_t *h, char *buf, int len) +{ + memcpy(buf, &h->data[h->cur], len); + h->cur += len; +} + +extern int hvm_save(struct vcpu*, hvm_domain_context_t *h); +extern int hvm_load(struct vcpu*, hvm_domain_context_t *h); + +extern int arch_sethvm_ctxt(struct vcpu *v, struct hvm_domain_context *c); +extern int arch_gethvm_ctxt(struct vcpu *v, struct hvm_domain_context *c); + +extern void shpage_init(struct domain *d, shared_iopage_t *sp); + extern int hvm_enabled; int hvm_copy_to_guest_phys(paddr_t paddr, void *buf, int size); diff --git a/xen/include/asm-x86/hvm/vpt.h b/xen/include/asm-x86/hvm/vpt.h index d9ad3f70df..37809f9b34 100644 --- a/xen/include/asm-x86/hvm/vpt.h +++ b/xen/include/asm-x86/hvm/vpt.h @@ -152,7 +152,7 @@ void pt_update_irq(struct vcpu *v); struct periodic_time *is_pt_irq(struct vcpu *v, int vector, int type); void pt_intr_post(struct vcpu *v, int vector, int type); void pt_reset(struct vcpu *v); -void create_periodic_time(struct periodic_time *pt, uint64_t period, +void create_periodic_time(struct vcpu *v, struct periodic_time *pt, uint64_t period, uint8_t irq, char one_shot, time_cb *cb, void *data); void destroy_periodic_time(struct periodic_time *pt); diff --git a/xen/include/public/arch-x86/xen.h b/xen/include/public/arch-x86/xen.h index 34cbdb7c59..1b83e4a4af 100644 --- a/xen/include/public/arch-x86/xen.h +++ b/xen/include/public/arch-x86/xen.h @@ -108,6 +108,70 @@ DEFINE_XEN_GUEST_HANDLE(trap_info_t); typedef uint64_t tsc_timestamp_t; /* RDTSC timestamp */ +/* + * World vmcs state + */ +struct vmcs_data { + uint64_t eip; /* execution pointer */ + uint64_t esp; /* stack pointer */ + uint64_t eflags; /* flags register */ + uint64_t cr0; + uint64_t cr3; /* page table directory */ + uint64_t cr4; + uint32_t idtr_limit; /* idt */ + uint64_t idtr_base; + uint32_t gdtr_limit; /* gdt */ + uint64_t gdtr_base; + uint32_t cs_sel; /* cs selector */ + uint32_t cs_limit; + uint64_t cs_base; + uint32_t cs_arbytes; + uint32_t ds_sel; /* ds selector */ + uint32_t ds_limit; + uint64_t ds_base; + uint32_t ds_arbytes; + uint32_t es_sel; /* es selector */ + uint32_t es_limit; + uint64_t es_base; + uint32_t es_arbytes; + uint32_t ss_sel; /* ss selector */ + uint32_t ss_limit; + uint64_t ss_base; + uint32_t ss_arbytes; + uint32_t fs_sel; /* fs selector */ + uint32_t fs_limit; + uint64_t fs_base; + uint32_t fs_arbytes; + uint32_t gs_sel; /* gs selector */ + uint32_t gs_limit; + uint64_t gs_base; + uint32_t gs_arbytes; + uint32_t tr_sel; /* task selector */ + uint32_t tr_limit; + uint64_t tr_base; + uint32_t tr_arbytes; + uint32_t ldtr_sel; /* ldtr selector */ + uint32_t ldtr_limit; + uint64_t ldtr_base; + uint32_t ldtr_arbytes; + uint32_t sysenter_cs; + uint64_t sysenter_esp; + uint64_t sysenter_eip; + /* msr for em64t */ + uint64_t shadow_gs; + uint64_t flags; + /* same size as VMX_MSR_COUNT */ + uint64_t msr_items[6]; + uint64_t vmxassist_enabled; +}; +typedef struct vmcs_data vmcs_data_t; + +struct hvmcpu_context { + uint32_t valid; + struct vmcs_data data; + uint64_t gtime; +}; + /* * The following is all CPU context. Note that the fpu_ctxt block is filled * in by FXSAVE if the CPU has feature FXSR; otherwise FSAVE is used. @@ -154,6 +218,7 @@ struct vcpu_guest_context { #endif #endif unsigned long vm_assist; /* VMASST_TYPE_* bitmap */ + struct hvmcpu_context hvmcpu_ctxt; /* whole vmcs region */ #ifdef __x86_64__ /* Segment base addresses. */ uint64_t fs_base; diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h index d9b23d5b75..0a75ef7a20 100644 --- a/xen/include/public/domctl.h +++ b/xen/include/public/domctl.h @@ -386,6 +386,21 @@ struct xen_domctl_settimeoffset { }; typedef struct xen_domctl_settimeoffset xen_domctl_settimeoffset_t; DEFINE_XEN_GUEST_HANDLE(xen_domctl_settimeoffset_t); + +#define HVM_CTXT_SIZE 6144 +typedef struct hvm_domain_context { + uint32_t cur; + uint32_t size; + uint8_t data[HVM_CTXT_SIZE]; +} hvm_domain_context_t; +DEFINE_XEN_GUEST_HANDLE(hvm_domain_context_t); + +#define XEN_DOMCTL_gethvmcontext 33 +#define XEN_DOMCTL_sethvmcontext 34 +typedef struct xen_domctl_hvmcontext { + XEN_GUEST_HANDLE(hvm_domain_context_t) ctxt; /* IN/OUT */ +} xen_domctl_hvmcontext_t; +DEFINE_XEN_GUEST_HANDLE(xen_domctl_hvmcontext_t); #define XEN_DOMCTL_real_mode_area 26 struct xen_domctl_real_mode_area { @@ -423,6 +438,7 @@ struct xen_domctl { struct xen_domctl_arch_setup arch_setup; struct xen_domctl_settimeoffset settimeoffset; struct xen_domctl_real_mode_area real_mode_area; + struct xen_domctl_hvmcontext hvmcontext; uint8_t pad[128]; } u; }; diff --git a/xen/include/xlat.lst b/xen/include/xlat.lst index 950f60b2f4..84f1a43fe2 100644 --- a/xen/include/xlat.lst +++ b/xen/include/xlat.lst @@ -8,6 +8,8 @@ ? vcpu_time_info xen.h ! cpu_user_regs arch-x86/xen-@arch@.h ! trap_info arch-x86/xen.h +! hvmcpu_context arch-x86/xen.h +! vmcs_data arch-x86/xen.h ! vcpu_guest_context arch-x86/xen.h ? acm_getdecision acm_ops.h ! ctl_cpumap domctl.h