xen-project/xen
1
0 Fork 0
mirror of https://github.com/xen-project/xen.git synced 2026-10-28 02:01:24 +01:00
Code Issues Activity
Files
41548c5472a3ea514cb0173f2576096970358263
xen/xen/arch/x86/mm/mem_sharing.c
T
Tamas K Lengyel 41548c5472 mem_sharing: VM forking 
VM forking is the process of creating a domain with an empty memory space and a
parent domain specified from which to populate the memory when necessary. For
the new domain to be functional the VM state is copied over as part of the fork
operation (HVM params, hap allocation, etc).

Signed-off-by: Tamas K Lengyel <tamas.lengyel@intel.com>
Reviewed-by: Roger Pau Monné <roger.pau@citrix.com>
Acked-by: Jan Beulich <jbeulich@suse.com>
Acked-by: Julien Grall <jgrall@amazon.com>
2020-04-08 12:59:58 +02:00

2072 lines
55 KiB
C
Raw Blame History

/******************************************************************************
* arch/x86/mm/mem_sharing.c
*
* Memory sharing support.
*
* Copyright (c) 2011 GridCentric, Inc. (Adin Scannell & Andres Lagar-Cavilla)
* Copyright (c) 2009 Citrix Systems, Inc. (Grzegorz Milos)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; If not, see <http://www.gnu.org/licenses/>.
*/
#include <xen/types.h>
#include <xen/domain_page.h>
#include <xen/event.h>
#include <xen/spinlock.h>
#include <xen/rwlock.h>
#include <xen/mm.h>
#include <xen/grant_table.h>
#include <xen/sched.h>
#include <xen/rcupdate.h>
#include <xen/guest_access.h>
#include <xen/vm_event.h>
#include <asm/page.h>
#include <asm/string.h>
#include <asm/p2m.h>
#include <asm/altp2m.h>
#include <asm/atomic.h>
#include <asm/event.h>
#include <asm/hap.h>
#include <asm/hvm/hvm.h>
#include <xsm/xsm.h>
#include "mm-locks.h"
static shr_handle_t next_handle = 1;
typedef struct pg_lock_data {
int mm_unlock_level;
unsigned short recurse_count;
} pg_lock_data_t;
static DEFINE_PER_CPU(pg_lock_data_t, __pld);
/* Reverse map defines */
#define RMAP_HASHTAB_ORDER 0
#define RMAP_HASHTAB_SIZE \
((PAGE_SIZE << RMAP_HASHTAB_ORDER) / sizeof(struct list_head))
#define RMAP_USES_HASHTAB(page) \
((page)->sharing->hash_table.flag == NULL)
#define RMAP_HEAVY_SHARED_PAGE RMAP_HASHTAB_SIZE
/*
* A bit of hysteresis. We don't want to be mutating between list and hash
* table constantly.
*/
#define RMAP_LIGHT_SHARED_PAGE (RMAP_HEAVY_SHARED_PAGE >> 2)
#if MEM_SHARING_AUDIT
static LIST_HEAD(shr_audit_list);
static DEFINE_SPINLOCK(shr_audit_lock);
static DEFINE_RCU_READ_LOCK(shr_audit_read_lock);
/* RCU delayed free of audit list entry */
static void _free_pg_shared_info(struct rcu_head *head)
{
xfree(container_of(head, struct page_sharing_info, rcu_head));
}
static void audit_add_list(struct page_info *page)
{
INIT_LIST_HEAD(&page->sharing->entry);
spin_lock(&shr_audit_lock);
list_add_rcu(&page->sharing->entry, &shr_audit_list);
spin_unlock(&shr_audit_lock);
}
/* Removes from the audit list and cleans up the page sharing metadata. */
static void page_sharing_dispose(struct page_info *page)
{
/* Unlikely given our thresholds, but we should be careful. */
if ( unlikely(RMAP_USES_HASHTAB(page)) )
free_xenheap_pages(page->sharing->hash_table.bucket,
RMAP_HASHTAB_ORDER);
spin_lock(&shr_audit_lock);
list_del_rcu(&page->sharing->entry);
spin_unlock(&shr_audit_lock);
INIT_RCU_HEAD(&page->sharing->rcu_head);
call_rcu(&page->sharing->rcu_head, _free_pg_shared_info);
}
#else
#define audit_add_list(p) ((void)0)
static void page_sharing_dispose(struct page_info *page)
{
/* Unlikely given our thresholds, but we should be careful. */
if ( unlikely(RMAP_USES_HASHTAB(page)) )
free_xenheap_pages(page->sharing->hash_table.bucket,
RMAP_HASHTAB_ORDER);
xfree(page->sharing);
}
#endif /* MEM_SHARING_AUDIT */
/*
* Private implementations of page_lock/unlock to bypass PV-only
* sanity checks not applicable to mem-sharing.
*
* _page_lock is used in memory sharing to protect addition (share) and removal
* (unshare) of (gfn,domain) tupples to a list of gfn's that the shared page is
* currently backing.
* Nesting may happen when sharing (and locking) two pages.
* Deadlock is avoided by locking pages in increasing order.
* All memory sharing code paths take the p2m lock of the affected gfn before
* taking the lock for the underlying page. We enforce ordering between
* page_lock and p2m_lock using an mm-locks.h construct.
*
* TODO: Investigate if PGT_validated is necessary.
*/
static bool _page_lock(struct page_info *page)
{
unsigned long x, nx;
do {
while ( (x = page->u.inuse.type_info) & PGT_locked )
cpu_relax();
nx = x + (1 | PGT_locked);
if ( !(x & PGT_validated) ||
!(x & PGT_count_mask) ||
!(nx & PGT_count_mask) )
return false;
} while ( cmpxchg(&page->u.inuse.type_info, x, nx) != x );
return true;
}
static void _page_unlock(struct page_info *page)
{
unsigned long x, nx, y = page->u.inuse.type_info;
do {
x = y;
ASSERT((x & PGT_count_mask) && (x & PGT_locked));
nx = x - (1 | PGT_locked);
/* We must not drop the last reference here. */
ASSERT(nx & PGT_count_mask);
} while ( (y = cmpxchg(&page->u.inuse.type_info, x, nx)) != x );
}
static bool mem_sharing_page_lock(struct page_info *pg)
{
bool rc;
pg_lock_data_t *pld = &(this_cpu(__pld));
page_sharing_mm_pre_lock();
rc = _page_lock(pg);
if ( rc )
{
preempt_disable();
page_sharing_mm_post_lock(&pld->mm_unlock_level,
&pld->recurse_count);
}
return rc;
}
static void mem_sharing_page_unlock(struct page_info *pg)
{
pg_lock_data_t *pld = &(this_cpu(__pld));
page_sharing_mm_unlock(pld->mm_unlock_level,
&pld->recurse_count);
preempt_enable();
_page_unlock(pg);
}
static shr_handle_t get_next_handle(void)
{
/* Get the next handle get_page style */
uint64_t x, y = next_handle;
do {
x = y;
}
while ( (y = cmpxchg(&next_handle, x, x + 1)) != x );
return x + 1;
}
static atomic_t nr_saved_mfns = ATOMIC_INIT(0);
static atomic_t nr_shared_mfns = ATOMIC_INIT(0);
/*
* Reverse map
*
* Every shared frame keeps a reverse map (rmap) of <domain, gfn> tuples that
* this shared frame backs. For pages with a low degree of sharing, a O(n)
* search linked list is good enough. For pages with higher degree of sharing,
* we use a hash table instead.
*/
typedef struct gfn_info
{
unsigned long gfn;
domid_t domain;
struct list_head list;
} gfn_info_t;
static void rmap_init(struct page_info *page)
{
/* We always start off as a doubly linked list. */
INIT_LIST_HEAD(&page->sharing->gfns);
}
/* Exceedingly simple "hash function" */
#define HASH(domain, gfn) \
(((gfn) + (domain)) % RMAP_HASHTAB_SIZE)
/*
* Conversions. Tuned by the thresholds. Should only happen twice
* (once each) during the lifetime of a shared page.
*/
static inline int rmap_list_to_hash_table(struct page_info *page)
{
unsigned int i;
struct list_head *pos, *tmp, *b =
alloc_xenheap_pages(RMAP_HASHTAB_ORDER, 0);
if ( b == NULL )
return -ENOMEM;
for ( i = 0; i < RMAP_HASHTAB_SIZE; i++ )
INIT_LIST_HEAD(b + i);
list_for_each_safe ( pos, tmp, &page->sharing->gfns )
{
gfn_info_t *gfn_info = list_entry(pos, gfn_info_t, list);
struct list_head *bucket = b + HASH(gfn_info->domain, gfn_info->gfn);
list_del(pos);
list_add(pos, bucket);
}
page->sharing->hash_table.bucket = b;
page->sharing->hash_table.flag = NULL;
return 0;
}
static void rmap_hash_table_to_list(struct page_info *page)
{
unsigned int i;
struct list_head *bucket = page->sharing->hash_table.bucket;
INIT_LIST_HEAD(&page->sharing->gfns);
for ( i = 0; i < RMAP_HASHTAB_SIZE; i++ )
{
struct list_head *pos, *tmp, *head = bucket + i;
list_for_each_safe ( pos, tmp, head )
{
list_del(pos);
list_add(pos, &page->sharing->gfns);
}
}
free_xenheap_pages(bucket, RMAP_HASHTAB_ORDER);
}
/* Generic accessors to the rmap */
static unsigned long rmap_count(const struct page_info *pg)
{
unsigned long count;
unsigned long t = read_atomic(&pg->u.inuse.type_info);
count = t & PGT_count_mask;
if ( t & PGT_locked )
count--;
return count;
}
/*
* The page type count is always decreased after removing from the rmap.
* Use a convert flag to avoid mutating the rmap if in the middle of an
* iterator, or if the page will be soon destroyed anyways.
*/
static void rmap_del(gfn_info_t *gfn_info, struct page_info *page, int convert)
{
if ( RMAP_USES_HASHTAB(page) && convert &&
(rmap_count(page) <= RMAP_LIGHT_SHARED_PAGE) )
rmap_hash_table_to_list(page);
/* Regardless of rmap type, same removal operation */
list_del(&gfn_info->list);
}
/* The page type count is always increased before adding to the rmap. */
static void rmap_add(gfn_info_t *gfn_info, struct page_info *page)
{
struct list_head *head;
if ( !RMAP_USES_HASHTAB(page) &&
(rmap_count(page) >= RMAP_HEAVY_SHARED_PAGE) )
/*
* The conversion may fail with ENOMEM. We'll be less efficient,
* but no reason to panic.
*/
(void)rmap_list_to_hash_table(page);
head = (RMAP_USES_HASHTAB(page)
? page->sharing->hash_table.bucket + HASH(gfn_info->domain,
gfn_info->gfn)
: &page->sharing->gfns);
INIT_LIST_HEAD(&gfn_info->list);
list_add(&gfn_info->list, head);
}
static gfn_info_t *rmap_retrieve(uint16_t domain_id, unsigned long gfn,
struct page_info *page)
{
gfn_info_t *gfn_info;
struct list_head *le, *head;
head = (RMAP_USES_HASHTAB(page)
? page->sharing->hash_table.bucket + HASH(domain_id, gfn)
: &page->sharing->gfns);
list_for_each ( le, head )
{
gfn_info = list_entry(le, gfn_info_t, list);
if ( (gfn_info->gfn == gfn) && (gfn_info->domain == domain_id) )
return gfn_info;
}
/* Nothing was found */
return NULL;
}
/*
* The iterator hides the details of how the rmap is implemented. This
* involves splitting the list_for_each_safe macro into two steps.
*/
struct rmap_iterator {
struct list_head *curr;
struct list_head *next;
unsigned int bucket;
};
static void rmap_seed_iterator(struct page_info *page, struct rmap_iterator *ri)
{
ri->curr = (RMAP_USES_HASHTAB(page)
? page->sharing->hash_table.bucket
: &page->sharing->gfns);
ri->next = ri->curr->next;
ri->bucket = 0;
}
static gfn_info_t *rmap_iterate(struct page_info *page,
struct rmap_iterator *ri)
{
struct list_head *head = (RMAP_USES_HASHTAB(page)
? page->sharing->hash_table.bucket + ri->bucket
: &page->sharing->gfns);
retry:
if ( ri->next == head)
{
if ( RMAP_USES_HASHTAB(page) )
{
ri->bucket++;
if ( ri->bucket >= RMAP_HASHTAB_SIZE )
/* No more hash table buckets */
return NULL;
head = page->sharing->hash_table.bucket + ri->bucket;
ri->curr = head;
ri->next = ri->curr->next;
goto retry;
}
else
/* List exhausted */
return NULL;
}
ri->curr = ri->next;
ri->next = ri->curr->next;
return list_entry(ri->curr, gfn_info_t, list);
}
static gfn_info_t *mem_sharing_gfn_alloc(struct page_info *page,
struct domain *d, unsigned long gfn)
{
gfn_info_t *gfn_info = xmalloc(gfn_info_t);
if ( gfn_info == NULL )
return NULL;
gfn_info->gfn = gfn;
gfn_info->domain = d->domain_id;
rmap_add(gfn_info, page);
/* Increment our number of shared pges. */
atomic_inc(&d->shr_pages);
return gfn_info;
}
static void mem_sharing_gfn_destroy(struct page_info *page, struct domain *d,
gfn_info_t *gfn_info)
{
/* Decrement the number of pages. */
atomic_dec(&d->shr_pages);
/* Free the gfn_info structure. */
rmap_del(gfn_info, page, 1);
xfree(gfn_info);
}
static struct page_info *mem_sharing_lookup(unsigned long mfn)
{
struct page_info *page;
unsigned long t;
if ( !mfn_valid(_mfn(mfn)) )
return NULL;
page = mfn_to_page(_mfn(mfn));
if ( page_get_owner(page) != dom_cow )
return NULL;
/*
* Count has to be at least two, because we're called
* with the mfn locked (1) and this is supposed to be
* a shared page (1).
*/
t = read_atomic(&page->u.inuse.type_info);
ASSERT((t & PGT_type_mask) == PGT_shared_page);
ASSERT((t & PGT_count_mask) >= 2);
ASSERT(SHARED_M2P(get_gpfn_from_mfn(mfn)));
return page;
}
static int audit(void)
{
#if MEM_SHARING_AUDIT
int errors = 0;
unsigned long count_expected;
unsigned long count_found = 0;
struct list_head *ae;
count_expected = atomic_read(&nr_shared_mfns);
rcu_read_lock(&shr_audit_read_lock);
list_for_each_rcu ( ae, &shr_audit_list )
{
struct page_sharing_info *pg_shared_info;
unsigned long nr_gfns = 0;
struct page_info *pg;
mfn_t mfn;
gfn_info_t *g;
struct rmap_iterator ri;
pg_shared_info = list_entry(ae, struct page_sharing_info, entry);
pg = pg_shared_info->pg;
mfn = page_to_mfn(pg);
/* If we can't lock it, it's definitely not a shared page */
if ( !mem_sharing_page_lock(pg) )
{
gdprintk(XENLOG_ERR,
"mfn %lx in audit list, but cannot be locked (%lx)!\n",
mfn_x(mfn), pg->u.inuse.type_info);
errors++;
continue;
}
/* Check if the MFN has correct type, owner and handle. */
if ( (pg->u.inuse.type_info & PGT_type_mask) != PGT_shared_page )
{
gdprintk(XENLOG_ERR,
"mfn %lx in audit list, but not PGT_shared_page (%lx)!\n",
mfn_x(mfn), pg->u.inuse.type_info & PGT_type_mask);
errors++;
continue;
}
/* Check the page owner. */
if ( page_get_owner(pg) != dom_cow )
{
gdprintk(XENLOG_ERR, "mfn %lx shared, but wrong owner (%pd)!\n",
mfn_x(mfn), page_get_owner(pg));
errors++;
}
/* Check the m2p entry */
if ( !SHARED_M2P(get_gpfn_from_mfn(mfn_x(mfn))) )
{
gdprintk(XENLOG_ERR, "mfn %lx shared, but wrong m2p entry (%lx)!\n",
mfn_x(mfn), get_gpfn_from_mfn(mfn_x(mfn)));
errors++;
}
/* Check we have a list */
if ( (!pg->sharing) || rmap_count(pg) == 0 )
{
gdprintk(XENLOG_ERR, "mfn %lx shared, but empty gfn list!\n",
mfn_x(mfn));
errors++;
continue;
}
/* We've found a page that is shared */
count_found++;
/* Check if all GFNs map to the MFN, and the p2m types */
rmap_seed_iterator(pg, &ri);
while ( (g = rmap_iterate(pg, &ri)) != NULL )
{
struct domain *d;
p2m_type_t t;
mfn_t o_mfn;
d = get_domain_by_id(g->domain);
if ( d == NULL )
{
gdprintk(XENLOG_ERR,
"Unknown dom: %d, for PFN=%lx, MFN=%lx\n",
g->domain, g->gfn, mfn_x(mfn));
errors++;
continue;
}
o_mfn = get_gfn_query_unlocked(d, g->gfn, &t);
if ( !mfn_eq(o_mfn, mfn) )
{
gdprintk(XENLOG_ERR, "Incorrect P2M for %pd, PFN=%lx."
"Expecting MFN=%lx, got %lx\n",
d, g->gfn, mfn_x(mfn), mfn_x(o_mfn));
errors++;
}
if ( t != p2m_ram_shared )
{
gdprintk(XENLOG_ERR,
"Incorrect P2M type for %pd, PFN=%lx MFN=%lx."
"Expecting t=%d, got %d\n",
d, g->gfn, mfn_x(mfn), p2m_ram_shared, t);
errors++;
}
put_domain(d);
nr_gfns++;
}
/* The type count has an extra ref because we have locked the page */
if ( (nr_gfns + 1) != (pg->u.inuse.type_info & PGT_count_mask) )
{
gdprintk(XENLOG_ERR, "Mismatched counts for MFN=%lx."
"nr_gfns in list %lu, in type_info %lx\n",
mfn_x(mfn), nr_gfns,
(pg->u.inuse.type_info & PGT_count_mask));
errors++;
}
mem_sharing_page_unlock(pg);
}
rcu_read_unlock(&shr_audit_read_lock);
if ( count_found != count_expected )
{
gdprintk(XENLOG_ERR, "Expected %ld shared mfns, found %ld.",
count_expected, count_found);
errors++;
}
return errors;
#else
return -EOPNOTSUPP;
#endif
}
int mem_sharing_notify_enomem(struct domain *d, unsigned long gfn,
bool allow_sleep)
{
struct vcpu *v = current;
int rc;
vm_event_request_t req = {
.reason = VM_EVENT_REASON_MEM_SHARING,
.vcpu_id = v->vcpu_id,
.u.mem_sharing.gfn = gfn,
.u.mem_sharing.p2mt = p2m_ram_shared,
};
if ( (rc = __vm_event_claim_slot(
d, d->vm_event_share, allow_sleep)) < 0 )
return rc;
if ( v->domain == d )
{
req.flags = VM_EVENT_FLAG_VCPU_PAUSED;
vm_event_vcpu_pause(v);
}
vm_event_put_request(d, d->vm_event_share, &req);
return 0;
}
unsigned int mem_sharing_get_nr_saved_mfns(void)
{
return atomic_read(&nr_saved_mfns);
}
unsigned int mem_sharing_get_nr_shared_mfns(void)
{
return atomic_read(&nr_shared_mfns);
}
/* Functions that change a page's type and ownership */
static int page_make_sharable(struct domain *d,
struct page_info *page,
int expected_refcnt)
{
bool_t drop_dom_ref;
spin_lock(&d->page_alloc_lock);
if ( d->is_dying )
{
spin_unlock(&d->page_alloc_lock);
return -EBUSY;
}
/* Change page type and count atomically */
if ( !get_page_and_type(page, d, PGT_shared_page) )
{
spin_unlock(&d->page_alloc_lock);
return -EINVAL;
}
/* Check it wasn't already sharable and undo if it was */
if ( (page->u.inuse.type_info & PGT_count_mask) != 1 )
{
spin_unlock(&d->page_alloc_lock);
put_page_and_type(page);
return -EEXIST;
}
/*
* Check if the ref count is 2. The first from PGC_allocated, and
* the second from get_page_and_type at the top of this function.
*/
if ( page->count_info != (PGC_allocated | (2 + expected_refcnt)) )
{
spin_unlock(&d->page_alloc_lock);
/* Return type count back to zero */
put_page_and_type(page);
return -E2BIG;
}
page_set_owner(page, dom_cow);
drop_dom_ref = !domain_adjust_tot_pages(d, -1);
page_list_del(page, &d->page_list);
spin_unlock(&d->page_alloc_lock);
if ( drop_dom_ref )
put_domain(d);
return 0;
}
static int page_make_private(struct domain *d, struct page_info *page)
{
unsigned long expected_type;
if ( !get_page(page, dom_cow) )
return -EINVAL;
spin_lock(&d->page_alloc_lock);
if ( d->is_dying )
{
spin_unlock(&d->page_alloc_lock);
put_page(page);
return -EBUSY;
}
expected_type = (PGT_shared_page | PGT_validated | PGT_locked | 2);
if ( page->u.inuse.type_info != expected_type )
{
spin_unlock(&d->page_alloc_lock);
put_page(page);
return -EEXIST;
}
mem_sharing_page_unlock(page);
/* Drop the final typecount */
put_page_and_type(page);
/* Change the owner */
ASSERT(page_get_owner(page) == dom_cow);
page_set_owner(page, d);
if ( domain_adjust_tot_pages(d, 1) == 1 )
get_knownalive_domain(d);
page_list_add_tail(page, &d->page_list);
spin_unlock(&d->page_alloc_lock);
put_page(page);
return 0;
}
static struct page_info *__grab_shared_page(mfn_t mfn)
{
struct page_info *pg = NULL;
if ( !mfn_valid(mfn) )
return NULL;
pg = mfn_to_page(mfn);
/*
* If the page is not validated we can't lock it, and if it's
* not validated it's obviously not shared.
*/
if ( !mem_sharing_page_lock(pg) )
return NULL;
if ( mem_sharing_lookup(mfn_x(mfn)) == NULL )
{
mem_sharing_page_unlock(pg);
return NULL;
}
return pg;
}
static int debug_mfn(mfn_t mfn)
{
struct page_info *page;
int num_refs;
if ( (page = __grab_shared_page(mfn)) == NULL)
{
gdprintk(XENLOG_ERR, "Invalid MFN=%lx\n", mfn_x(mfn));
return -EINVAL;
}
gdprintk(XENLOG_ERR,
"Debug page: MFN=%lx is ci=%lx, ti=%lx, owner_id=%pd\n",
mfn_x(page_to_mfn(page)), page->count_info,
page->u.inuse.type_info, page_get_owner(page));
/* -1 because the page is locked and that's an additional type ref */
num_refs = ((int) (page->u.inuse.type_info & PGT_count_mask)) - 1;
mem_sharing_page_unlock(page);
return num_refs;
}
static int debug_gfn(struct domain *d, gfn_t gfn)
{
p2m_type_t p2mt;
mfn_t mfn;
int num_refs;
mfn = get_gfn_query(d, gfn_x(gfn), &p2mt);
gdprintk(XENLOG_ERR, "Debug for %pd, gfn=%" PRI_gfn "\n",
d, gfn_x(gfn));
num_refs = debug_mfn(mfn);
put_gfn(d, gfn_x(gfn));
return num_refs;
}
static int debug_gref(struct domain *d, grant_ref_t ref)
{
int rc;
uint16_t status;
gfn_t gfn;
rc = mem_sharing_gref_to_gfn(d->grant_table, ref, &gfn, &status);
if ( rc )
{
gdprintk(XENLOG_ERR, "Asked to debug [%pd,gref=%u]: error %d.\n",
d, ref, rc);
return rc;
}
gdprintk(XENLOG_ERR, "==> Grant [%pd,ref=%d], status=%x. ",
d, ref, status);
return debug_gfn(d, gfn);
}
static int nominate_page(struct domain *d, gfn_t gfn,
int expected_refcnt, shr_handle_t *phandle)
{
struct p2m_domain *hp2m = p2m_get_hostp2m(d);
p2m_type_t p2mt;
p2m_access_t p2ma;
mfn_t mfn;
struct page_info *page = NULL; /* gcc... */
int ret;
*phandle = 0UL;
mfn = get_gfn_type_access(hp2m, gfn_x(gfn), &p2mt, &p2ma, 0, NULL);
/* Check if mfn is valid */
ret = -EINVAL;
if ( !mfn_valid(mfn) )
goto out;
/* Return the handle if the page is already shared */
if ( p2m_is_shared(p2mt) )
{
struct page_info *pg = __grab_shared_page(mfn);
if ( !pg )
BUG();
*phandle = pg->sharing->handle;
ret = 0;
mem_sharing_page_unlock(pg);
goto out;
}
/* Check p2m type */
if ( !p2m_is_sharable(p2mt) )
goto out;
page = mfn_to_page(mfn);
if ( !page || is_special_page(page) )
goto out;
/* Check if there are mem_access/remapped altp2m entries for this page */
if ( altp2m_active(d) )
{
unsigned int i;
struct p2m_domain *ap2m;
mfn_t amfn;
p2m_type_t ap2mt;
p2m_access_t ap2ma;
altp2m_list_lock(d);
for ( i = 0; i < MAX_ALTP2M; i++ )
{
ap2m = d->arch.altp2m_p2m[i];
if ( !ap2m )
continue;
amfn = __get_gfn_type_access(ap2m, gfn_x(gfn), &ap2mt, &ap2ma,
0, NULL, false);
if ( mfn_valid(amfn) && (!mfn_eq(amfn, mfn) || ap2ma != p2ma) )
{
altp2m_list_unlock(d);
goto out;
}
}
altp2m_list_unlock(d);
}
/* Try to convert the mfn to the sharable type */
ret = page_make_sharable(d, page, expected_refcnt);
if ( ret )
goto out;
/*
* Now that the page is validated, we can lock it. There is no
* race because we're holding the p2m entry, so no one else
* could be nominating this gfn.
*/
ret = -ENOENT;
if ( !mem_sharing_page_lock(page) )
goto out;
/* Initialize the shared state */
ret = -ENOMEM;
if ( !(page->sharing = xmalloc(struct page_sharing_info)) )
{
/* Making a page private atomically unlocks it */
BUG_ON(page_make_private(d, page));
goto out;
}
page->sharing->pg = page;
rmap_init(page);
/* Create the handle */
page->sharing->handle = get_next_handle();
/* Create the local gfn info */
if ( !mem_sharing_gfn_alloc(page, d, gfn_x(gfn)) )
{
xfree(page->sharing);
page->sharing = NULL;
BUG_ON(page_make_private(d, page));
goto out;
}
/* Change the p2m type, should never fail with p2m locked. */
BUG_ON(p2m_change_type_one(d, gfn_x(gfn), p2mt, p2m_ram_shared));
/* Account for this page. */
atomic_inc(&nr_shared_mfns);
/* Update m2p entry to SHARED_M2P_ENTRY */
set_gpfn_from_mfn(mfn_x(mfn), SHARED_M2P_ENTRY);
*phandle = page->sharing->handle;
audit_add_list(page);
mem_sharing_page_unlock(page);
ret = 0;
out:
put_gfn(d, gfn_x(gfn));
return ret;
}
static int share_pages(struct domain *sd, gfn_t sgfn, shr_handle_t sh,
struct domain *cd, gfn_t cgfn, shr_handle_t ch)
{
struct page_info *spage, *cpage, *firstpg, *secondpg;
gfn_info_t *gfn;
struct domain *d;
int ret = -EINVAL;
mfn_t smfn, cmfn;
p2m_type_t smfn_type, cmfn_type;
struct two_gfns tg;
struct rmap_iterator ri;
unsigned long put_count = 0;
get_two_gfns(sd, sgfn, &smfn_type, NULL, &smfn,
cd, cgfn, &cmfn_type, NULL, &cmfn, 0, &tg, true);
/*
* This tricky business is to avoid two callers deadlocking if
* grabbing pages in opposite client/source order.
*/
if ( mfn_eq(smfn, cmfn) )
{
/*
* The pages are already the same. We could return some
* kind of error here, but no matter how you look at it,
* the pages are already 'shared'. It possibly represents
* a big problem somewhere else, but as far as sharing is
* concerned: great success!
*/
ret = 0;
goto err_out;
}
if ( mfn_x(smfn) < mfn_x(cmfn) )
{
ret = XENMEM_SHARING_OP_S_HANDLE_INVALID;
spage = firstpg = __grab_shared_page(smfn);
if ( spage == NULL )
goto err_out;
ret = XENMEM_SHARING_OP_C_HANDLE_INVALID;
cpage = secondpg = __grab_shared_page(cmfn);
if ( cpage == NULL )
{
mem_sharing_page_unlock(spage);
goto err_out;
}
}
else
{
ret = XENMEM_SHARING_OP_C_HANDLE_INVALID;
cpage = firstpg = __grab_shared_page(cmfn);
if ( cpage == NULL )
goto err_out;
ret = XENMEM_SHARING_OP_S_HANDLE_INVALID;
spage = secondpg = __grab_shared_page(smfn);
if ( spage == NULL )
{
mem_sharing_page_unlock(cpage);
goto err_out;
}
}
ASSERT(smfn_type == p2m_ram_shared);
ASSERT(cmfn_type == p2m_ram_shared);
/* Check that the handles match */
if ( spage->sharing->handle != sh )
{
ret = XENMEM_SHARING_OP_S_HANDLE_INVALID;
mem_sharing_page_unlock(secondpg);
mem_sharing_page_unlock(firstpg);
goto err_out;
}
if ( cpage->sharing->handle != ch )
{
ret = XENMEM_SHARING_OP_C_HANDLE_INVALID;
mem_sharing_page_unlock(secondpg);
mem_sharing_page_unlock(firstpg);
goto err_out;
}
/* Merge the lists together */
rmap_seed_iterator(cpage, &ri);
while ( (gfn = rmap_iterate(cpage, &ri)) != NULL)
{
/*
* Get the source page and type, this should never fail:
* we are under shr lock, and got a successful lookup.
*/
BUG_ON(!get_page_and_type(spage, dom_cow, PGT_shared_page));
/*
* Move the gfn_info from client list to source list.
* Don't change the type of rmap for the client page.
*/
rmap_del(gfn, cpage, 0);
rmap_add(gfn, spage);
put_count++;
d = get_domain_by_id(gfn->domain);
BUG_ON(!d);
BUG_ON(set_shared_p2m_entry(d, gfn->gfn, smfn));
put_domain(d);
}
ASSERT(list_empty(&cpage->sharing->gfns));
BUG_ON(!put_count);
/* Clear the rest of the shared state */
page_sharing_dispose(cpage);
cpage->sharing = NULL;
mem_sharing_page_unlock(secondpg);
mem_sharing_page_unlock(firstpg);
/* Free the client page */
put_page_alloc_ref(cpage);
while ( put_count-- )
put_page_and_type(cpage);
/* We managed to free a domain page. */
atomic_dec(&nr_shared_mfns);
atomic_inc(&nr_saved_mfns);
ret = 0;
err_out:
put_two_gfns(&tg);
return ret;
}
/*
* This function is intended to be used for plugging a "hole" in the client's
* physmap with a shared memory entry. Unfortunately the definition of a "hole"
* is currently ambigious. There are two cases one can run into a "hole":
* 1) there is no pagetable entry at all
* 2) there is a pagetable entry with a type that passes p2m_is_hole
*
* The intended use-case for this function is case 1.
*
* During 1) the mem_access being returned is p2m_access_n and that is
* incorrect to be applied to the new entry being added the client physmap,
* thus we make use of the p2m->default_access instead.
* When 2) is true it is possible that the existing pagetable entry also has
* a mem_access permission set, which could be p2m_access_n. Since we can't
* differentiate whether we are in case 1) or 2), we default to using the
* access permission defined as default for the p2m, thus in
* case 2) overwriting any custom mem_access permission the user may have set
* on a hole page. Custom mem_access permissions being set on a hole are
* unheard of but technically possible.
*
* TODO: to properly resolve this issue implement differentiation between the
* two "hole" types.
*/
static
int add_to_physmap(struct domain *sd, unsigned long sgfn, shr_handle_t sh,
struct domain *cd, unsigned long cgfn, bool lock)
{
struct page_info *spage;
int ret = -EINVAL;
mfn_t smfn, cmfn;
p2m_type_t smfn_type, cmfn_type;
struct gfn_info *gfn_info;
struct p2m_domain *p2m = p2m_get_hostp2m(cd);
struct two_gfns tg;
get_two_gfns(sd, _gfn(sgfn), &smfn_type, NULL, &smfn,
cd, _gfn(cgfn), &cmfn_type, NULL, &cmfn, 0, &tg, lock);
/* Get the source shared page, check and lock */
ret = XENMEM_SHARING_OP_S_HANDLE_INVALID;
spage = __grab_shared_page(smfn);
if ( spage == NULL )
goto err_out;
ASSERT(smfn_type == p2m_ram_shared);
/* Check that the handles match */
if ( spage->sharing->handle != sh )
goto err_unlock;
/*
* Make sure the target page is a hole in the physmap. These are typically
* p2m_mmio_dm, but also accept p2m_invalid and paged out pages. See the
* definition of p2m_is_hole in p2m.h.
*/
if ( !p2m_is_hole(cmfn_type) )
{
ret = XENMEM_SHARING_OP_C_HANDLE_INVALID;
goto err_unlock;
}
/* This is simpler than regular sharing */
BUG_ON(!get_page_and_type(spage, dom_cow, PGT_shared_page));
if ( !(gfn_info = mem_sharing_gfn_alloc(spage, cd, cgfn)) )
{
put_page_and_type(spage);
ret = -ENOMEM;
goto err_unlock;
}
ret = p2m_set_entry(p2m, _gfn(cgfn), smfn, PAGE_ORDER_4K,
p2m_ram_shared, p2m->default_access);
/* Tempted to turn this into an assert */
if ( ret )
{
mem_sharing_gfn_destroy(spage, cd, gfn_info);
put_page_and_type(spage);
}
else
{
/*
* There is a chance we're plugging a hole where a paged out
* page was.
*/
if ( p2m_is_paging(cmfn_type) && (cmfn_type != p2m_ram_paging_out) )
{
atomic_dec(&cd->paged_pages);
/*
* Further, there is a chance this was a valid page.
* Don't leak it.
*/
if ( mfn_valid(cmfn) )
{
struct page_info *cpage = mfn_to_page(cmfn);
if ( !get_page(cpage, cd) )
{
domain_crash(cd);
ret = -EOVERFLOW;
goto err_unlock;
}
put_page_alloc_ref(cpage);
put_page(cpage);
}
}
}
atomic_inc(&nr_saved_mfns);
err_unlock:
mem_sharing_page_unlock(spage);
err_out:
if ( lock )
put_two_gfns(&tg);
return ret;
}
/*
* A note on the rationale for unshare error handling:
* 1. Unshare can only fail with ENOMEM. Any other error conditions BUG_ON()'s
* 2. We notify a potential dom0 helper through a vm_event ring. But we
* allow the notification to not go to sleep. If the event ring is full
* of ENOMEM warnings, then it's on the ball.
* 3. We cannot go to sleep until the unshare is resolved, because we might
* be buried deep into locks (e.g. something -> copy_to_user -> __hvm_copy)
* 4. So, we make sure we:
* 4.1. return an error
* 4.2. do not corrupt shared memory
* 4.3. do not corrupt guest memory
* 4.4. let the guest deal with it if the error propagation will reach it
*/
int __mem_sharing_unshare_page(struct domain *d,
unsigned long gfn,
bool destroy)
{
p2m_type_t p2mt;
mfn_t mfn;
struct page_info *page, *old_page;
int last_gfn;
gfn_info_t *gfn_info = NULL;
mfn = get_gfn(d, gfn, &p2mt);
/* Has someone already unshared it? */
if ( !p2m_is_shared(p2mt) )
{
put_gfn(d, gfn);
return 0;
}
page = __grab_shared_page(mfn);
if ( page == NULL )
{
gdprintk(XENLOG_ERR, "Domain p2m is shared, but page is not: %lx\n",
gfn);
BUG();
}
gfn_info = rmap_retrieve(d->domain_id, gfn, page);
if ( unlikely(gfn_info == NULL) )
{
gdprintk(XENLOG_ERR, "Could not find gfn_info for shared gfn: %lx\n",
gfn);
BUG();
}
/*
* Do the accounting first. If anything fails below, we have bigger
* bigger fish to fry. First, remove the gfn from the list.
*/
last_gfn = rmap_count(page) == 1;
if ( last_gfn )
{
/*
* Clean up shared state. Get rid of the <domid, gfn> tuple
* before destroying the rmap.
*/
mem_sharing_gfn_destroy(page, d, gfn_info);
page_sharing_dispose(page);
page->sharing = NULL;
atomic_dec(&nr_shared_mfns);
}
else
atomic_dec(&nr_saved_mfns);
/*
* If the GFN is getting destroyed drop the references to MFN
* (possibly freeing the page), and exit early.
*/
if ( destroy )
{
if ( !last_gfn )
mem_sharing_gfn_destroy(page, d, gfn_info);
mem_sharing_page_unlock(page);
if ( last_gfn )
put_page_alloc_ref(page);
put_page_and_type(page);
put_gfn(d, gfn);
return 0;
}
if ( last_gfn )
{
/* Making a page private atomically unlocks it */
BUG_ON(page_make_private(d, page) != 0);
goto private_page_found;
}
old_page = page;
page = alloc_domheap_page(d, 0);
if ( !page )
{
/* Undo dec of nr_saved_mfns, as the retry will decrease again. */
atomic_inc(&nr_saved_mfns);
mem_sharing_page_unlock(old_page);
put_gfn(d, gfn);
/*
* Caller is responsible for placing an event
* in the ring.
*/
return -ENOMEM;
}
copy_domain_page(page_to_mfn(page), page_to_mfn(old_page));
BUG_ON(set_shared_p2m_entry(d, gfn, page_to_mfn(page)));
mem_sharing_gfn_destroy(old_page, d, gfn_info);
mem_sharing_page_unlock(old_page);
put_page_and_type(old_page);
private_page_found:
if ( p2m_change_type_one(d, gfn, p2m_ram_shared, p2m_ram_rw) )
{
gdprintk(XENLOG_ERR, "Could not change p2m type d %pd gfn %lx.\n",
d, gfn);
BUG();
}
/* Update m2p entry */
set_gpfn_from_mfn(mfn_x(page_to_mfn(page)), gfn);
/*
* Now that the gfn<->mfn map is properly established,
* marking dirty is feasible
*/
paging_mark_dirty(d, page_to_mfn(page));
/* We do not need to unlock a private page */
put_gfn(d, gfn);
return 0;
}
int relinquish_shared_pages(struct domain *d)
{
int rc = 0;
struct mem_sharing_domain *msd = &d->arch.hvm.mem_sharing;
struct p2m_domain *p2m = p2m_get_hostp2m(d);
unsigned long gfn, count = 0;
if ( p2m == NULL )
return 0;
p2m_lock(p2m);
for ( gfn = msd->next_shared_gfn_to_relinquish;
gfn <= p2m->max_mapped_pfn; gfn++ )
{
p2m_access_t a;
p2m_type_t t;
mfn_t mfn;
int set_rc;
if ( !atomic_read(&d->shr_pages) )
break;
mfn = p2m->get_entry(p2m, _gfn(gfn), &t, &a, 0, NULL, NULL);
if ( mfn_valid(mfn) && p2m_is_shared(t) )
{
/* Does not fail with ENOMEM given "destroy" is set to true */
BUG_ON(__mem_sharing_unshare_page(d, gfn, true));
/*
* Clear out the p2m entry so no one else may try to
* unshare. Must succeed: we just read the old entry and
* we hold the p2m lock.
*/
set_rc = p2m->set_entry(p2m, _gfn(gfn), INVALID_MFN, PAGE_ORDER_4K,
p2m_invalid, p2m_access_rwx, -1);
ASSERT(!set_rc);
count += 0x10;
}
else
++count;
/* Preempt every 2MiB (shared) or 32MiB (unshared) - arbitrary. */
if ( count >= 0x2000 )
{
if ( hypercall_preempt_check() )
{
msd->next_shared_gfn_to_relinquish = gfn + 1;
rc = -ERESTART;
break;
}
count = 0;
}
}
p2m_unlock(p2m);
return rc;
}
static int range_share(struct domain *d, struct domain *cd,
struct mem_sharing_op_range *range)
{
int rc = 0;
shr_handle_t sh, ch;
unsigned long start = range->opaque ?: range->first_gfn;
while ( range->last_gfn >= start )
{
/*
* We only break out if we run out of memory as individual pages may
* legitimately be unsharable and we just want to skip over those.
*/
rc = nominate_page(d, _gfn(start), 0, &sh);
if ( rc == -ENOMEM )
break;
if ( !rc )
{
rc = nominate_page(cd, _gfn(start), 0, &ch);
if ( rc == -ENOMEM )
break;
if ( !rc )
{
/* If we get here this should be guaranteed to succeed. */
rc = share_pages(d, _gfn(start), sh, cd, _gfn(start), ch);
ASSERT(!rc);
}
}
/* Check for continuation if it's not the last iteration. */
if ( range->last_gfn >= ++start && hypercall_preempt_check() )
{
rc = 1;
break;
}
}
range->opaque = start;
/*
* The last page may fail with -EINVAL, and for range sharing we don't
* care about that.
*/
if ( range->last_gfn < start && rc == -EINVAL )
rc = 0;
return rc;
}
static inline int mem_sharing_control(struct domain *d, bool enable)
{
if ( enable )
{
if ( unlikely(!is_hvm_domain(d)) )
return -EOPNOTSUPP;
if ( unlikely(!hap_enabled(d)) )
return -ENODEV;
if ( unlikely(is_iommu_enabled(d)) )
return -EXDEV;
}
d->arch.hvm.mem_sharing.enabled = enable;
return 0;
}
/*
* Forking a page only gets called when the VM faults due to no entry being
* in the EPT for the access. Depending on the type of access we either
* populate the physmap with a shared entry for read-only access or
* fork the page if its a write access.
*
* The client p2m is already locked so we only need to lock
* the parent's here.
*/
int mem_sharing_fork_page(struct domain *d, gfn_t gfn, bool unsharing)
{
int rc = -ENOENT;
shr_handle_t handle;
struct domain *parent = d->parent;
struct p2m_domain *p2m;
unsigned long gfn_l = gfn_x(gfn);
mfn_t mfn, new_mfn;
p2m_type_t p2mt;
struct page_info *page;
if ( !mem_sharing_is_fork(d) )
return -ENOENT;
if ( !unsharing )
{
/* For read-only accesses we just add a shared entry to the physmap */
while ( parent )
{
if ( !(rc = nominate_page(parent, gfn, 0, &handle)) )
break;
parent = parent->parent;
}
if ( !rc )
{
/* The client's p2m is already locked */
p2m = p2m_get_hostp2m(parent);
p2m_lock(p2m);
rc = add_to_physmap(parent, gfn_l, handle, d, gfn_l, false);
p2m_unlock(p2m);
if ( !rc )
return 0;
}
}
/*
* If it's a write access (ie. unsharing) or if adding a shared entry to
* the physmap failed we'll fork the page directly.
*/
p2m = p2m_get_hostp2m(d);
parent = d->parent;
while ( parent )
{
mfn = get_gfn_query(parent, gfn_l, &p2mt);
/* We can't fork grant memory from the parent, only regular ram */
if ( mfn_valid(mfn) && p2m_is_ram(p2mt) )
break;
put_gfn(parent, gfn_l);
parent = parent->parent;
}
if ( !parent )
return -ENOENT;
if ( !(page = alloc_domheap_page(d, 0)) )
{
put_gfn(parent, gfn_l);
return -ENOMEM;
}
new_mfn = page_to_mfn(page);
copy_domain_page(new_mfn, mfn);
set_gpfn_from_mfn(mfn_x(new_mfn), gfn_l);
put_gfn(parent, gfn_l);
return p2m->set_entry(p2m, gfn, new_mfn, PAGE_ORDER_4K, p2m_ram_rw,
p2m->default_access, -1);
}
static int bring_up_vcpus(struct domain *cd, struct domain *d)
{
unsigned int i;
int ret = -EINVAL;
if ( d->max_vcpus != cd->max_vcpus ||
(ret = cpupool_move_domain(cd, d->cpupool)) )
return ret;
for ( i = 0; i < cd->max_vcpus; i++ )
{
if ( !d->vcpu[i] || cd->vcpu[i] )
continue;
if ( !vcpu_create(cd, i) )
return -EINVAL;
}
domain_update_node_affinity(cd);
return 0;
}
static int copy_vcpu_settings(struct domain *cd, const struct domain *d)
{
unsigned int i;
struct p2m_domain *p2m = p2m_get_hostp2m(cd);
int ret = -EINVAL;
for ( i = 0; i < cd->max_vcpus; i++ )
{
const struct vcpu *d_vcpu = d->vcpu[i];
struct vcpu *cd_vcpu = cd->vcpu[i];
mfn_t vcpu_info_mfn;
if ( !d_vcpu || !cd_vcpu )
continue;
/* Copy & map in the vcpu_info page if the guest uses one */
vcpu_info_mfn = d_vcpu->vcpu_info_mfn;
if ( !mfn_eq(vcpu_info_mfn, INVALID_MFN) )
{
mfn_t new_vcpu_info_mfn = cd_vcpu->vcpu_info_mfn;
/* Allocate & map the page for it if it hasn't been already */
if ( mfn_eq(new_vcpu_info_mfn, INVALID_MFN) )
{
gfn_t gfn = mfn_to_gfn(d, vcpu_info_mfn);
unsigned long gfn_l = gfn_x(gfn);
struct page_info *page;
if ( !(page = alloc_domheap_page(cd, 0)) )
return -ENOMEM;
new_vcpu_info_mfn = page_to_mfn(page);
set_gpfn_from_mfn(mfn_x(new_vcpu_info_mfn), gfn_l);
ret = p2m->set_entry(p2m, gfn, new_vcpu_info_mfn,
PAGE_ORDER_4K, p2m_ram_rw,
p2m->default_access, -1);
if ( ret )
return ret;
ret = map_vcpu_info(cd_vcpu, gfn_l,
PAGE_OFFSET(d_vcpu->vcpu_info));
if ( ret )
return ret;
}
copy_domain_page(new_vcpu_info_mfn, vcpu_info_mfn);
}
/*
* TODO: to support VMs with PV interfaces copy additional
* settings here, such as PV timers.
*/
}
return 0;
}
static int fork_hap_allocation(struct domain *cd, struct domain *d)
{
int rc;
bool preempted;
unsigned long mb = hap_get_allocation(d);
if ( mb == hap_get_allocation(cd) )
return 0;
paging_lock(cd);
rc = hap_set_allocation(cd, mb << (20 - PAGE_SHIFT), &preempted);
paging_unlock(cd);
return preempted ? -ERESTART : rc;
}
static void copy_tsc(struct domain *cd, struct domain *d)
{
uint32_t tsc_mode;
uint32_t gtsc_khz;
uint32_t incarnation;
uint64_t elapsed_nsec;
tsc_get_info(d, &tsc_mode, &elapsed_nsec, &gtsc_khz, &incarnation);
/* Don't bump incarnation on set */
tsc_set_info(cd, tsc_mode, elapsed_nsec, gtsc_khz, incarnation - 1);
}
static int copy_special_pages(struct domain *cd, struct domain *d)
{
mfn_t new_mfn, old_mfn;
struct p2m_domain *p2m = p2m_get_hostp2m(cd);
static const unsigned int params[] =
{
HVM_PARAM_STORE_PFN,
HVM_PARAM_IOREQ_PFN,
HVM_PARAM_BUFIOREQ_PFN,
HVM_PARAM_CONSOLE_PFN
};
unsigned int i;
int rc;
for ( i = 0; i < ARRAY_SIZE(params); i++ )
{
p2m_type_t t;
uint64_t value = 0;
struct page_info *page;
if ( hvm_get_param(d, params[i], &value) || !value )
continue;
old_mfn = get_gfn_query_unlocked(d, value, &t);
new_mfn = get_gfn_query_unlocked(cd, value, &t);
/* Allocate the page and map it in if it's not present */
if ( mfn_eq(new_mfn, INVALID_MFN) )
{
if ( !(page = alloc_domheap_page(cd, 0)) )
return -ENOMEM;
new_mfn = page_to_mfn(page);
set_gpfn_from_mfn(mfn_x(new_mfn), value);
rc = p2m->set_entry(p2m, _gfn(value), new_mfn, PAGE_ORDER_4K,
p2m_ram_rw, p2m->default_access, -1);
if ( rc )
return rc;
}
copy_domain_page(new_mfn, old_mfn);
}
old_mfn = _mfn(virt_to_mfn(d->shared_info));
new_mfn = _mfn(virt_to_mfn(cd->shared_info));
copy_domain_page(new_mfn, old_mfn);
return 0;
}
static int copy_settings(struct domain *cd, struct domain *d)
{
int rc;
if ( (rc = copy_vcpu_settings(cd, d)) )
return rc;
if ( (rc = hvm_copy_context_and_params(cd, d)) )
return rc;
if ( (rc = copy_special_pages(cd, d)) )
return rc;
copy_tsc(cd, d);
return rc;
}
static int fork(struct domain *cd, struct domain *d)
{
int rc = -EBUSY;
if ( !cd->controller_pause_count )
return rc;
if ( !cd->parent )
{
if ( !get_domain(d) )
{
ASSERT_UNREACHABLE();
return -EBUSY;
}
domain_pause(d);
cd->max_pages = d->max_pages;
cd->parent = d;
}
/* This is preemptible so it's the first to get done */
if ( (rc = fork_hap_allocation(cd, d)) )
goto done;
if ( (rc = bring_up_vcpus(cd, d)) )
goto done;
rc = copy_settings(cd, d);
done:
if ( rc && rc != -ERESTART )
{
domain_unpause(d);
put_domain(d);
cd->parent = NULL;
}
return rc;
}
int mem_sharing_memop(XEN_GUEST_HANDLE_PARAM(xen_mem_sharing_op_t) arg)
{
int rc;
xen_mem_sharing_op_t mso;
struct domain *d;
rc = -EFAULT;
if ( copy_from_guest(&mso, arg, 1) )
return rc;
if ( mso.op == XENMEM_sharing_op_audit )
return audit();
rc = rcu_lock_live_remote_domain_by_id(mso.domain, &d);
if ( rc )
return rc;
rc = xsm_mem_sharing(XSM_DM_PRIV, d);
if ( rc )
goto out;
if ( !mem_sharing_enabled(d) && (rc = mem_sharing_control(d, true)) )
return rc;
switch ( mso.op )
{
case XENMEM_sharing_op_nominate_gfn:
{
shr_handle_t handle;
rc = nominate_page(d, _gfn(mso.u.nominate.u.gfn), 0, &handle);
mso.u.nominate.handle = handle;
}
break;
case XENMEM_sharing_op_nominate_gref:
{
grant_ref_t gref = mso.u.nominate.u.grant_ref;
gfn_t gfn;
shr_handle_t handle;
rc = mem_sharing_gref_to_gfn(d->grant_table, gref, &gfn, NULL);
if ( rc < 0 )
goto out;
rc = nominate_page(d, gfn, 3, &handle);
mso.u.nominate.handle = handle;
}
break;
case XENMEM_sharing_op_share:
{
gfn_t sgfn, cgfn;
struct domain *cd;
shr_handle_t sh, ch;
rc = rcu_lock_live_remote_domain_by_id(mso.u.share.client_domain,
&cd);
if ( rc )
goto out;
rc = xsm_mem_sharing_op(XSM_DM_PRIV, d, cd, mso.op);
if ( rc )
{
rcu_unlock_domain(cd);
goto out;
}
if ( !mem_sharing_enabled(cd) )
{
rcu_unlock_domain(cd);
rc = -EINVAL;
goto out;
}
if ( XENMEM_SHARING_OP_FIELD_IS_GREF(mso.u.share.source_gfn) )
{
grant_ref_t gref =
XENMEM_SHARING_OP_FIELD_GET_GREF(mso.u.share.source_gfn);
rc = mem_sharing_gref_to_gfn(d->grant_table, gref, &sgfn,
NULL);
if ( rc < 0 )
{
rcu_unlock_domain(cd);
goto out;
}
}
else
sgfn = _gfn(mso.u.share.source_gfn);
if ( XENMEM_SHARING_OP_FIELD_IS_GREF(mso.u.share.client_gfn) )
{
grant_ref_t gref =
XENMEM_SHARING_OP_FIELD_GET_GREF(mso.u.share.client_gfn);
rc = mem_sharing_gref_to_gfn(cd->grant_table, gref, &cgfn,
NULL);
if ( rc < 0 )
{
rcu_unlock_domain(cd);
goto out;
}
}
else
cgfn = _gfn(mso.u.share.client_gfn);
sh = mso.u.share.source_handle;
ch = mso.u.share.client_handle;
rc = share_pages(d, sgfn, sh, cd, cgfn, ch);
rcu_unlock_domain(cd);
}
break;
case XENMEM_sharing_op_add_physmap:
{
unsigned long sgfn, cgfn;
struct domain *cd;
shr_handle_t sh;
rc = rcu_lock_live_remote_domain_by_id(mso.u.share.client_domain,
&cd);
if ( rc )
goto out;
rc = xsm_mem_sharing_op(XSM_DM_PRIV, d, cd, mso.op);
if ( rc )
{
rcu_unlock_domain(cd);
goto out;
}
if ( !mem_sharing_enabled(cd) )
{
rcu_unlock_domain(cd);
rc = -EINVAL;
goto out;
}
if ( XENMEM_SHARING_OP_FIELD_IS_GREF(mso.u.share.source_gfn) )
{
/* Cannot add a gref to the physmap */
rcu_unlock_domain(cd);
rc = -EINVAL;
goto out;
}
sgfn = mso.u.share.source_gfn;
sh = mso.u.share.source_handle;
cgfn = mso.u.share.client_gfn;
rc = add_to_physmap(d, sgfn, sh, cd, cgfn, true);
rcu_unlock_domain(cd);
}
break;
case XENMEM_sharing_op_range_share:
{
unsigned long max_sgfn, max_cgfn;
struct domain *cd;
rc = -EINVAL;
if ( mso.u.range._pad[0] || mso.u.range._pad[1] ||
mso.u.range._pad[2] )
goto out;
/*
* We use opaque for the hypercall continuation value.
* Ideally the user sets this to 0 in the beginning but
* there is no good way of enforcing that here, so we just check
* that it's at least in range.
*/
if ( mso.u.range.opaque &&
(mso.u.range.opaque < mso.u.range.first_gfn ||
mso.u.range.opaque > mso.u.range.last_gfn) )
goto out;
rc = rcu_lock_live_remote_domain_by_id(mso.u.range.client_domain,
&cd);
if ( rc )
goto out;
/*
* We reuse XENMEM_sharing_op_share XSM check here as this is
* essentially the same concept repeated over multiple pages.
*/
rc = xsm_mem_sharing_op(XSM_DM_PRIV, d, cd,
XENMEM_sharing_op_share);
if ( rc )
{
rcu_unlock_domain(cd);
goto out;
}
if ( !mem_sharing_enabled(cd) )
{
rcu_unlock_domain(cd);
rc = -EINVAL;
goto out;
}
/*
* Sanity check only, the client should keep the domains paused for
* the duration of this op.
*/
if ( !atomic_read(&d->pause_count) ||
!atomic_read(&cd->pause_count) )
{
rcu_unlock_domain(cd);
rc = -EINVAL;
goto out;
}
max_sgfn = domain_get_maximum_gpfn(d);
max_cgfn = domain_get_maximum_gpfn(cd);
if ( max_sgfn < mso.u.range.first_gfn ||
max_sgfn < mso.u.range.last_gfn ||
max_cgfn < mso.u.range.first_gfn ||
max_cgfn < mso.u.range.last_gfn )
{
rcu_unlock_domain(cd);
rc = -EINVAL;
goto out;
}
rc = range_share(d, cd, &mso.u.range);
rcu_unlock_domain(cd);
if ( rc > 0 )
{
if ( __copy_to_guest(arg, &mso, 1) )
rc = -EFAULT;
else
rc = hypercall_create_continuation(__HYPERVISOR_memory_op,
"lh", XENMEM_sharing_op,
arg);
}
else
mso.u.range.opaque = 0;
}
break;
case XENMEM_sharing_op_debug_gfn:
rc = debug_gfn(d, _gfn(mso.u.debug.u.gfn));
break;
case XENMEM_sharing_op_debug_gref:
rc = debug_gref(d, mso.u.debug.u.gref);
break;
case XENMEM_sharing_op_fork:
{
struct domain *pd;
rc = -EINVAL;
if ( mso.u.fork.pad[0] || mso.u.fork.pad[1] || mso.u.fork.pad[2] )
goto out;
rc = rcu_lock_live_remote_domain_by_id(mso.u.fork.parent_domain,
&pd);
if ( rc )
goto out;
rc = -EINVAL;
if ( pd->max_vcpus != d->max_vcpus )
{
rcu_unlock_domain(pd);
goto out;
}
if ( !mem_sharing_enabled(pd) && (rc = mem_sharing_control(pd, true)) )
{
rcu_unlock_domain(pd);
goto out;
}
rc = fork(d, pd);
if ( rc == -ERESTART )
rc = hypercall_create_continuation(__HYPERVISOR_memory_op,
"lh", XENMEM_sharing_op,
arg);
rcu_unlock_domain(pd);
break;
}
default:
rc = -ENOSYS;
break;
}
if ( !rc && __copy_to_guest(arg, &mso, 1) )
rc = -EFAULT;
out:
rcu_unlock_domain(d);
return rc;
}
int mem_sharing_domctl(struct domain *d, struct xen_domctl_mem_sharing_op *mec)
{
int rc;
switch ( mec->op )
{
case XEN_DOMCTL_MEM_SHARING_CONTROL:
rc = mem_sharing_control(d, mec->u.enable);
break;
default:
rc = -ENOSYS;
break;
}
return rc;
}
Reference in new issue View Git Blame Copy Permalink