diff --git a/xen/common/domain.c b/xen/common/domain.c index 376351b528..163f7fc966 100644 --- a/xen/common/domain.c +++ b/xen/common/domain.c @@ -615,6 +615,18 @@ static int __init cf_check parse_dom0_param(const char *s) } custom_param("dom0", parse_dom0_param); +static void domain_pending_scrub_free(struct domain *d) +{ + rspin_lock(&d->page_alloc_lock); + if ( d->pending_scrub ) + { + FREE_DOMHEAP_PAGES(d->pending_scrub, d->pending_scrub_order); + d->pending_scrub_order = 0; + d->pending_scrub_index = 0; + } + rspin_unlock(&d->page_alloc_lock); +} + /* * Release resources held by a domain. There may or may not be live * references to the domain, and it may or may not be fully constructed. @@ -676,6 +688,7 @@ static int domain_teardown(struct domain *d) /* Trivial teardown, not long-running enough to need a preemption check. */ domain_llc_coloring_free(d); + domain_pending_scrub_free(d); PROGRESS(gnttab_mappings): rc = gnttab_release_mappings(d); @@ -719,6 +732,7 @@ static void _domain_destroy(struct domain *d) { BUG_ON(!d->is_dying); BUG_ON(atomic_read(&d->refcnt) != DOMAIN_DESTROYED); + ASSERT(!d->pending_scrub); XVFREE(d->console); @@ -1678,6 +1692,15 @@ int domain_unpause_by_systemcontroller(struct domain *d) */ if ( new == 0 && !d->creation_finished ) { + if ( d->pending_scrub ) + { + printk(XENLOG_ERR + "%pd: cannot be started with pending unscrubbed pages, destroying\n", + d); + domain_crash(d); + domain_pending_scrub_free(d); + return -EBUSY; + } d->creation_finished = true; arch_domain_creation_finished(d); } diff --git a/xen/common/memory.c b/xen/common/memory.c index 10becf7c1f..9240a6005e 100644 --- a/xen/common/memory.c +++ b/xen/common/memory.c @@ -159,6 +159,73 @@ static void increase_reservation(struct memop_args *a) a->nr_done = i; } +/* + * Temporary storage for a domain assigned page that's not been fully scrubbed. + * Stored pages must be domheap ones. + * + * The stashed page can be freed at any time by Xen, the caller must pass the + * order and NUMA node requirement to the fetch function to ensure the + * currently stashed page matches it's requirements. + */ +static void stash_allocation(struct domain *d, struct page_info *page, + unsigned int order, unsigned int scrub_index) +{ + rspin_lock(&d->page_alloc_lock); + + /* + * Drop the passed page in preference for the already stashed one. This + * interface is designed to be used for single-threaded domain creation. + */ + if ( d->pending_scrub || d->is_dying ) + free_domheap_pages(page, order); + else + { + d->pending_scrub_index = scrub_index; + d->pending_scrub_order = order; + d->pending_scrub = page; + } + + rspin_unlock(&d->page_alloc_lock); +} + +static struct page_info *get_stashed_allocation(struct domain *d, + unsigned int order, + nodeid_t node, + unsigned int *scrub_index) +{ + struct page_info *page = NULL; + + rspin_lock(&d->page_alloc_lock); + + /* + * If there's a pending page to scrub check if it satisfies the current + * request. If it doesn't free it and return NULL. + */ + if ( d->pending_scrub ) + { + if ( d->pending_scrub_order == order && + (node == NUMA_NO_NODE || node == page_to_nid(d->pending_scrub)) ) + { + page = d->pending_scrub; + *scrub_index = d->pending_scrub_index; + } + else + free_domheap_pages(d->pending_scrub, d->pending_scrub_order); + + /* + * The caller now owns the page or it has been freed, clear stashed + * information. Prevent concurrent usages of get_stashed_allocation() + * from returning the same page to different contexts. + */ + d->pending_scrub_index = 0; + d->pending_scrub_order = 0; + d->pending_scrub = NULL; + } + + rspin_unlock(&d->page_alloc_lock); + return page; +} + static void populate_physmap(struct memop_args *a) { struct page_info *page; @@ -275,7 +342,19 @@ static void populate_physmap(struct memop_args *a) } else { - page = alloc_domheap_pages(d, a->extent_order, a->memflags); + unsigned int scrub_start = 0; + unsigned int memflags = + a->memflags | (d->creation_finished ? 0 + : MEMF_no_scrub); + nodeid_t node = + (a->memflags & MEMF_exact_node) ? MEMF_get_node(a->memflags) + : NUMA_NO_NODE; + + page = get_stashed_allocation(d, a->extent_order, node, + &scrub_start); + + if ( !page ) + page = alloc_domheap_pages(d, a->extent_order, memflags); if ( unlikely(!page) ) { @@ -286,6 +365,30 @@ static void populate_physmap(struct memop_args *a) goto out; } + if ( memflags & MEMF_no_scrub ) + { + unsigned int dirty_cnt = 0; + + /* Check if there's anything to scrub. */ + for ( j = scrub_start; j < (1U << a->extent_order); j++ ) + { + if ( !test_and_clear_bit(_PGC_need_scrub, + &page[j].count_info) ) + continue; + + scrub_one_page(&page[j], true); + + if ( (j + 1) != (1U << a->extent_order) && + !(++dirty_cnt & 0xff) && + hypercall_preempt_check() ) + { + a->preempted = 1; + stash_allocation(d, page, a->extent_order, j + 1); + goto out; + } + } + } + if ( unlikely(a->memflags & MEMF_no_tlbflush) ) { for ( j = 0; j < (1U << a->extent_order); j++ ) diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c index 2ee249ac36..588b5b99cb 100644 --- a/xen/common/page_alloc.c +++ b/xen/common/page_alloc.c @@ -792,7 +792,7 @@ static void page_list_add_scrub(struct page_info *pg, unsigned int node, # define scrub_page_cold clear_page_cold #endif -static void scrub_one_page(const struct page_info *pg, bool cold) +void scrub_one_page(const struct page_info *pg, bool cold) { void *ptr; diff --git a/xen/include/xen/mm.h b/xen/include/xen/mm.h index 426362adb2..d80bfba6d3 100644 --- a/xen/include/xen/mm.h +++ b/xen/include/xen/mm.h @@ -145,6 +145,16 @@ unsigned long avail_node_heap_pages(unsigned int nodeid); #define alloc_domheap_page(d,f) (alloc_domheap_pages(d,0,f)) #define free_domheap_page(p) (free_domheap_pages(p,0)) +/* Free an allocation, and zero the pointer to it. */ +#define FREE_DOMHEAP_PAGES(p, o) do { \ + void *_ptr_ = (p); \ + (p) = NULL; \ + free_domheap_pages(_ptr_, o); \ +} while ( false ) +#define FREE_DOMHEAP_PAGE(p) FREE_DOMHEAP_PAGES(p, 0) + +void scrub_one_page(const struct page_info *pg, bool cold); + int online_page(mfn_t mfn, uint32_t *status); int offline_page(mfn_t mfn, int broken, uint32_t *status); int query_page_offline(mfn_t mfn, uint32_t *status); diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h index 91d6a49daf..735d5b76b4 100644 --- a/xen/include/xen/sched.h +++ b/xen/include/xen/sched.h @@ -661,6 +661,11 @@ struct domain /* Pointer to console settings; NULL for system domains. */ struct domain_console *console; + + /* Pointer to allocated domheap page that possibly needs scrubbing. */ + struct page_info *pending_scrub; + unsigned int pending_scrub_order; + unsigned int pending_scrub_index; } __aligned(PAGE_SIZE); static inline struct page_list_head *page_to_list(