diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index f7e0e7856c..41f78179ba 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -1924,7 +1924,9 @@ int domain_relinquish_resources(struct domain *d) pci_release_devices(d); /* Tear down paging-assistance stuff. */ - paging_teardown(d); + ret = paging_teardown(d); + if ( ret ) + return ret; /* Drop the in-use references to page-table bases. */ for_each_vcpu ( d, v ) diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c index d1517c47be..70da5750cd 100644 --- a/xen/arch/x86/domctl.c +++ b/xen/arch/x86/domctl.c @@ -64,6 +64,9 @@ long arch_do_domctl( ret = paging_domctl(d, &domctl->u.shadow_op, guest_handle_cast(u_domctl, void)); + if ( ret == -ERESTART ) + return hypercall_create_continuation(__HYPERVISOR_domctl, + "h", u_domctl); copyback = 1; } break; diff --git a/xen/arch/x86/mm/hap/hap.c b/xen/arch/x86/mm/hap/hap.c index abf3d7a42e..101d1a1545 100644 --- a/xen/arch/x86/mm/hap/hap.c +++ b/xen/arch/x86/mm/hap/hap.c @@ -577,8 +577,7 @@ int hap_domctl(struct domain *d, xen_domctl_shadow_op_t *sc, paging_unlock(d); if ( preempted ) /* Not finished. Set up to re-run the call. */ - rc = hypercall_create_continuation(__HYPERVISOR_domctl, "h", - u_domctl); + rc = -ERESTART; else /* Finished. Return the new allocation */ sc->mb = hap_get_allocation(d); diff --git a/xen/arch/x86/mm/paging.c b/xen/arch/x86/mm/paging.c index 455000de65..c8c4918fc4 100644 --- a/xen/arch/x86/mm/paging.c +++ b/xen/arch/x86/mm/paging.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -116,26 +117,46 @@ static void paging_free_log_dirty_page(struct domain *d, mfn_t mfn) d->arch.paging.free_page(d, mfn_to_page(mfn)); } -void paging_free_log_dirty_bitmap(struct domain *d) +static int paging_free_log_dirty_bitmap(struct domain *d, int rc) { mfn_t *l4, *l3, *l2; int i4, i3, i2; - if ( !mfn_valid(d->arch.paging.log_dirty.top) ) - return; - paging_lock(d); - l4 = map_domain_page(mfn_x(d->arch.paging.log_dirty.top)); + if ( !mfn_valid(d->arch.paging.log_dirty.top) ) + { + paging_unlock(d); + return 0; + } - for ( i4 = 0; i4 < LOGDIRTY_NODE_ENTRIES; i4++ ) + if ( !d->arch.paging.preempt.vcpu ) + { + memset(&d->arch.paging.preempt.log_dirty, 0, + sizeof(d->arch.paging.preempt.log_dirty)); + ASSERT(rc <= 0); + d->arch.paging.preempt.log_dirty.done = -rc; + } + else if ( d->arch.paging.preempt.vcpu != current || + d->arch.paging.preempt.op != XEN_DOMCTL_SHADOW_OP_OFF ) + { + paging_unlock(d); + return -EBUSY; + } + + l4 = map_domain_page(mfn_x(d->arch.paging.log_dirty.top)); + i4 = d->arch.paging.preempt.log_dirty.i4; + i3 = d->arch.paging.preempt.log_dirty.i3; + rc = 0; + + for ( ; i4 < LOGDIRTY_NODE_ENTRIES; i4++, i3 = 0 ) { if ( !mfn_valid(l4[i4]) ) continue; l3 = map_domain_page(mfn_x(l4[i4])); - for ( i3 = 0; i3 < LOGDIRTY_NODE_ENTRIES; i3++ ) + for ( ; i3 < LOGDIRTY_NODE_ENTRIES; i3++ ) { if ( !mfn_valid(l3[i3]) ) continue; @@ -148,20 +169,54 @@ void paging_free_log_dirty_bitmap(struct domain *d) unmap_domain_page(l2); paging_free_log_dirty_page(d, l3[i3]); + l3[i3] = _mfn(INVALID_MFN); + + if ( i3 < LOGDIRTY_NODE_ENTRIES - 1 && hypercall_preempt_check() ) + { + d->arch.paging.preempt.log_dirty.i3 = i3 + 1; + d->arch.paging.preempt.log_dirty.i4 = i4; + rc = -ERESTART; + break; + } } unmap_domain_page(l3); + if ( rc ) + break; paging_free_log_dirty_page(d, l4[i4]); + l4[i4] = _mfn(INVALID_MFN); + + if ( i4 < LOGDIRTY_NODE_ENTRIES - 1 && hypercall_preempt_check() ) + { + d->arch.paging.preempt.log_dirty.i3 = 0; + d->arch.paging.preempt.log_dirty.i4 = i4 + 1; + rc = -ERESTART; + break; + } } unmap_domain_page(l4); - paging_free_log_dirty_page(d, d->arch.paging.log_dirty.top); - d->arch.paging.log_dirty.top = _mfn(INVALID_MFN); - ASSERT(d->arch.paging.log_dirty.allocs == 0); - d->arch.paging.log_dirty.failed_allocs = 0; + if ( !rc ) + { + paging_free_log_dirty_page(d, d->arch.paging.log_dirty.top); + d->arch.paging.log_dirty.top = _mfn(INVALID_MFN); + + ASSERT(d->arch.paging.log_dirty.allocs == 0); + d->arch.paging.log_dirty.failed_allocs = 0; + + rc = -d->arch.paging.preempt.log_dirty.done; + d->arch.paging.preempt.vcpu = NULL; + } + else + { + d->arch.paging.preempt.vcpu = current; + d->arch.paging.preempt.op = XEN_DOMCTL_SHADOW_OP_OFF; + } paging_unlock(d); + + return rc; } int paging_log_dirty_enable(struct domain *d, bool_t log_global) @@ -187,15 +242,25 @@ int paging_log_dirty_enable(struct domain *d, bool_t log_global) return ret; } -int paging_log_dirty_disable(struct domain *d) +static int paging_log_dirty_disable(struct domain *d, bool_t resuming) { - int ret; + int ret = 1; + + if ( !resuming ) + { + domain_pause(d); + /* Safe because the domain is paused. */ + ret = d->arch.paging.log_dirty.disable_log_dirty(d); + ASSERT(ret <= 0); + } - domain_pause(d); - /* Safe because the domain is paused. */ - ret = d->arch.paging.log_dirty.disable_log_dirty(d); if ( !paging_mode_log_dirty(d) ) - paging_free_log_dirty_bitmap(d); + { + ret = paging_free_log_dirty_bitmap(d, ret); + if ( ret == -ERESTART ) + return ret; + } + domain_unpause(d); return ret; @@ -335,7 +400,9 @@ int paging_mfn_is_dirty(struct domain *d, mfn_t gmfn) /* Read a domain's log-dirty bitmap and stats. If the operation is a CLEAN, * clear the bitmap and stats as well. */ -int paging_log_dirty_op(struct domain *d, struct xen_domctl_shadow_op *sc) +static int paging_log_dirty_op(struct domain *d, + struct xen_domctl_shadow_op *sc, + bool_t resuming) { int rv = 0, clean = 0, peek = 1; unsigned long pages = 0; @@ -343,9 +410,22 @@ int paging_log_dirty_op(struct domain *d, struct xen_domctl_shadow_op *sc) unsigned long *l1 = NULL; int i4, i3, i2; - domain_pause(d); + if ( !resuming ) + domain_pause(d); paging_lock(d); + if ( !d->arch.paging.preempt.vcpu ) + memset(&d->arch.paging.preempt.log_dirty, 0, + sizeof(d->arch.paging.preempt.log_dirty)); + else if ( d->arch.paging.preempt.vcpu != current || + d->arch.paging.preempt.op != sc->op ) + { + paging_unlock(d); + ASSERT(!resuming); + domain_unpause(d); + return -EBUSY; + } + clean = (sc->op == XEN_DOMCTL_SHADOW_OP_CLEAN); PAGING_DEBUG(LOGDIRTY, "log-dirty %s: dom %u faults=%u dirty=%u\n", @@ -374,17 +454,15 @@ int paging_log_dirty_op(struct domain *d, struct xen_domctl_shadow_op *sc) goto out; } - pages = 0; l4 = paging_map_log_dirty_bitmap(d); + i4 = d->arch.paging.preempt.log_dirty.i4; + i3 = d->arch.paging.preempt.log_dirty.i3; + pages = d->arch.paging.preempt.log_dirty.done; - for ( i4 = 0; - (pages < sc->pages) && (i4 < LOGDIRTY_NODE_ENTRIES); - i4++ ) + for ( ; (pages < sc->pages) && (i4 < LOGDIRTY_NODE_ENTRIES); i4++, i3 = 0 ) { l3 = (l4 && mfn_valid(l4[i4])) ? map_domain_page(mfn_x(l4[i4])) : NULL; - for ( i3 = 0; - (pages < sc->pages) && (i3 < LOGDIRTY_NODE_ENTRIES); - i3++ ) + for ( ; (pages < sc->pages) && (i3 < LOGDIRTY_NODE_ENTRIES); i3++ ) { l2 = ((l3 && mfn_valid(l3[i3])) ? map_domain_page(mfn_x(l3[i3])) : NULL); @@ -419,18 +497,51 @@ int paging_log_dirty_op(struct domain *d, struct xen_domctl_shadow_op *sc) } if ( l2 ) unmap_domain_page(l2); + + if ( i3 < LOGDIRTY_NODE_ENTRIES - 1 && hypercall_preempt_check() ) + { + d->arch.paging.preempt.log_dirty.i4 = i4; + d->arch.paging.preempt.log_dirty.i3 = i3 + 1; + rv = -ERESTART; + break; + } } if ( l3 ) unmap_domain_page(l3); + + if ( !rv && i4 < LOGDIRTY_NODE_ENTRIES - 1 && + hypercall_preempt_check() ) + { + d->arch.paging.preempt.log_dirty.i4 = i4 + 1; + d->arch.paging.preempt.log_dirty.i3 = 0; + rv = -ERESTART; + } + if ( rv ) + break; } if ( l4 ) unmap_domain_page(l4); - if ( pages < sc->pages ) - sc->pages = pages; + if ( !rv ) + d->arch.paging.preempt.vcpu = NULL; + else + { + d->arch.paging.preempt.vcpu = current; + d->arch.paging.preempt.op = sc->op; + d->arch.paging.preempt.log_dirty.done = pages; + } paging_unlock(d); + if ( rv ) + { + /* Never leave the domain paused for other errors. */ + ASSERT(rv == -ERESTART); + return rv; + } + + if ( pages < sc->pages ) + sc->pages = pages; if ( clean ) { /* We need to further call clean_dirty_bitmap() functions of specific @@ -441,6 +552,7 @@ int paging_log_dirty_op(struct domain *d, struct xen_domctl_shadow_op *sc) return rv; out: + d->arch.paging.preempt.vcpu = NULL; paging_unlock(d); domain_unpause(d); @@ -504,12 +616,6 @@ void paging_log_dirty_init(struct domain *d, d->arch.paging.log_dirty.clean_dirty_bitmap = clean_dirty_bitmap; } -/* This function fress log dirty bitmap resources. */ -static void paging_log_dirty_teardown(struct domain*d) -{ - paging_free_log_dirty_bitmap(d); -} - /************************************************/ /* CODE FOR PAGING SUPPORT */ /************************************************/ @@ -553,6 +659,7 @@ void paging_vcpu_init(struct vcpu *v) int paging_domctl(struct domain *d, xen_domctl_shadow_op_t *sc, XEN_GUEST_HANDLE_PARAM(void) u_domctl) { + bool_t resuming = 0; int rc; if ( unlikely(d == current->domain) ) @@ -575,6 +682,19 @@ int paging_domctl(struct domain *d, xen_domctl_shadow_op_t *sc, return -EINVAL; } + if ( d->arch.paging.preempt.vcpu ) + { + if ( d->arch.paging.preempt.vcpu != current || + d->arch.paging.preempt.op != sc->op ) + { + printk(XENLOG_G_DEBUG + "%pv: Paging op %#x on Dom%u with unfinished prior op %#x\n", + current, sc->op, d->domain_id, d->arch.paging.preempt.op); + return -EBUSY; + } + resuming = 1; + } + rc = xsm_shadow_control(XSM_HOOK, d, sc->op); if ( rc ) return rc; @@ -598,13 +718,13 @@ int paging_domctl(struct domain *d, xen_domctl_shadow_op_t *sc, case XEN_DOMCTL_SHADOW_OP_OFF: if ( paging_mode_log_dirty(d) ) - if ( (rc = paging_log_dirty_disable(d)) != 0 ) + if ( (rc = paging_log_dirty_disable(d, resuming)) != 0 ) return rc; break; case XEN_DOMCTL_SHADOW_OP_CLEAN: case XEN_DOMCTL_SHADOW_OP_PEEK: - return paging_log_dirty_op(d, sc); + return paging_log_dirty_op(d, sc, resuming); } /* Here, dispatch domctl to the appropriate paging code */ @@ -615,18 +735,24 @@ int paging_domctl(struct domain *d, xen_domctl_shadow_op_t *sc, } /* Call when destroying a domain */ -void paging_teardown(struct domain *d) +int paging_teardown(struct domain *d) { + int rc; + if ( hap_enabled(d) ) hap_teardown(d); else shadow_teardown(d); /* clean up log dirty resources. */ - paging_log_dirty_teardown(d); + rc = paging_free_log_dirty_bitmap(d, 0); + if ( rc == -ERESTART ) + return rc; /* Move populate-on-demand cache back to domain_list for destruction */ p2m_pod_empty_cache(d); + + return rc; } /* Call once all of the references to the domain have gone away */ diff --git a/xen/arch/x86/mm/shadow/common.c b/xen/arch/x86/mm/shadow/common.c index 3c803b6190..9f2882f6a8 100644 --- a/xen/arch/x86/mm/shadow/common.c +++ b/xen/arch/x86/mm/shadow/common.c @@ -3705,8 +3705,7 @@ int shadow_domctl(struct domain *d, paging_unlock(d); if ( preempted ) /* Not finished. Set up to re-run the call. */ - rc = hypercall_create_continuation( - __HYPERVISOR_domctl, "h", u_domctl); + rc = -ERESTART; else /* Finished. Return the new allocation */ sc->mb = shadow_get_allocation(d); diff --git a/xen/common/domain.c b/xen/common/domain.c index 1952070bbf..5d86e6cf1b 100644 --- a/xen/common/domain.c +++ b/xen/common/domain.c @@ -615,7 +615,6 @@ int domain_kill(struct domain *d) { if ( rc == -ERESTART ) rc = -EAGAIN; - BUG_ON(rc != -EAGAIN); break; } if ( sched_move_domain(d, cpupool0) ) diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h index 83329ed3bc..87ded497ea 100644 --- a/xen/include/asm-x86/domain.h +++ b/xen/include/asm-x86/domain.h @@ -186,6 +186,20 @@ struct paging_domain { struct hap_domain hap; /* log dirty support */ struct log_dirty_domain log_dirty; + + /* preemption handling */ + struct { + struct vcpu *vcpu; + unsigned int op; + union { + struct { + unsigned long done:PADDR_BITS - PAGE_SHIFT; + unsigned long i4:PAGETABLE_ORDER; + unsigned long i3:PAGETABLE_ORDER; + } log_dirty; + }; + } preempt; + /* alloc/free pages from the pool for paging-assistance structures * (used by p2m and log-dirty code for their tries) */ struct page_info * (*alloc_page)(struct domain *d); diff --git a/xen/include/asm-x86/paging.h b/xen/include/asm-x86/paging.h index 9b8f8de753..89322e47e8 100644 --- a/xen/include/asm-x86/paging.h +++ b/xen/include/asm-x86/paging.h @@ -132,9 +132,6 @@ struct paging_mode { /***************************************************************************** * Log dirty code */ -/* free log dirty bitmap resource */ -void paging_free_log_dirty_bitmap(struct domain *d); - /* get the dirty bitmap for a specific range of pfns */ void paging_log_dirty_range(struct domain *d, unsigned long begin_pfn, @@ -144,9 +141,6 @@ void paging_log_dirty_range(struct domain *d, /* enable log dirty */ int paging_log_dirty_enable(struct domain *d, bool_t log_global); -/* disable log dirty */ -int paging_log_dirty_disable(struct domain *d); - /* log dirty initialization */ void paging_log_dirty_init(struct domain *d, int (*enable_log_dirty)(struct domain *d, @@ -206,7 +200,7 @@ int paging_domctl(struct domain *d, xen_domctl_shadow_op_t *sc, XEN_GUEST_HANDLE_PARAM(void) u_domctl); /* Call when destroying a domain */ -void paging_teardown(struct domain *d); +int paging_teardown(struct domain *d); /* Call once all of the references to the domain have gone away */ void paging_final_teardown(struct domain *d);