Page offline support in Xen side

This patch add support to offline a page. The basical idea is, when a
page is assigned, it will be marked offline pending and be moved out of
buddy when freed, when a page is free, it will be moved out of buddy directly.

One notice after this change is, now the page->count_info is not
always 0, especially for shadow page, since the PGC_offlining bit may be set.

Signed-off-by: Wang, Shane <shane.wang@intel.com>
Signed-off-by: Jiang, Yunhong <yunhong.jiang@intel.com>

xen/common/page_alloc.c

@@ -35,6 +35,7 @@
 #include <xen/perfc.h>
 #include <xen/numa.h>
 #include <xen/nodemask.h>
+#include <public/sysctl.h>
 #include <asm/page.h>
 #include <asm/numa.h>
 #include <asm/flushtlb.h>
@@ -74,6 +75,11 @@ static DEFINE_SPINLOCK(page_scrub_lock);
 PAGE_LIST_HEAD(page_scrub_list);
 static unsigned long scrub_pages;
 
+/* Offlined page list, protected by heap_lock */
+PAGE_LIST_HEAD(page_offlined_list);
+
+/* Broken page list, protected by heap_lock */
+PAGE_LIST_HEAD(page_broken_list);
 /*********************
  * ALLOCATION BITMAP
  *  One bit per page of memory. Bit set => page is allocated.
@@ -421,12 +427,92 @@ static struct page_info *alloc_heap_pages(
     return pg;
 }
 
+/*
+ * Remove any offlined page in the buddy poined by head
+ */
+static int reserve_offlined_page(struct page_info *head)
+{
+    unsigned int node = phys_to_nid(page_to_maddr(head));
+    int zone = page_to_zone(head), i, head_order = PFN_ORDER(head), count = 0;
+    struct page_info *cur_head;
+    int cur_order;
+
+    ASSERT(spin_is_locked(&heap_lock));
+
+    cur_head = head;
+
+    page_list_del(head, &heap(node, zone, head_order));
+
+    while ( cur_head < (head + (1 << head_order)) )
+    {
+        struct page_info *pg;
+        int next_order;
+
+        if (test_bit(_PGC_offlined, &cur_head->count_info))
+        {
+            cur_head++;
+            continue;
+        }
+
+        next_order = cur_order = 0;
+
+        while (cur_order < head_order)
+        {
+            next_order = cur_order + 1;
+
+            if ( (cur_head + (1 << next_order)) >= (head + ( 1 << head_order)))
+                goto merge;
+
+            for (i = (1 << cur_order), pg = cur_head + (1 << cur_order);
+              i < (1 << next_order);
+              i++, pg ++)
+                if (test_bit(_PGC_offlined, &pg->count_info))
+                    break;
+            if (i == ( 1 << next_order))
+            {
+                cur_order = next_order;
+                continue;
+            }
+            else
+            {
+                /*
+                 * We don't need considering merge outside the head_order
+                 */
+merge:
+                page_list_add_tail(cur_head, &heap(node, zone, cur_order));
+                PFN_ORDER(cur_head) = cur_order;
+                cur_head += (1 << cur_order);
+                break;
+            }
+        }
+    }
+
+    for (cur_head = head; cur_head < head + ( 1UL << head_order); cur_head++)
+    {
+        if (!test_bit(_PGC_offlined, &cur_head->count_info))
+            continue;
+
+        avail[node][zone] --;
+
+        map_alloc(page_to_mfn(cur_head), 1);
+
+        if (test_bit(_PGC_broken, &cur_head->count_info))
+            page_list_add_tail(cur_head, &page_broken_list);
+        else
+            page_list_add_tail(cur_head, &page_offlined_list);
+
+        count ++;
+    }
+
+    return count;
+}
+
 /* Free 2^@order set of pages. */
 static void free_heap_pages(
     struct page_info *pg, unsigned int order)
 {
     unsigned long mask;
-    unsigned int i, node = phys_to_nid(page_to_maddr(pg));
+    unsigned int i, node = phys_to_nid(page_to_maddr(pg)), tainted = 0;
     unsigned int zone = page_to_zone(pg);
 
     ASSERT(order <= MAX_ORDER);
@@ -446,7 +532,14 @@ static void free_heap_pages(
          *     in its pseudophysical address space).
          * In all the above cases there can be no guest mappings of this page.
          */
-        pg[i].count_info = 0;
+        ASSERT(!(pg[i].count_info & PGC_offlined));
+        pg[i].count_info &= PGC_offlining | PGC_broken;
+        if (pg[i].count_info & PGC_offlining)
+        {
+            pg[i].count_info &= ~PGC_offlining;
+            pg[i].count_info |= PGC_offlined;
+            tainted = 1;
+        }
 
         /* If a page has no owner it will need no safety TLB flush. */
         pg[i].u.free.need_tlbflush = (page_get_owner(&pg[i]) != NULL);
@@ -481,7 +574,7 @@ static void free_heap_pages(
                 break;
             page_list_del(pg + mask, &heap(node, zone, order));
         }
-        
+
         order++;
 
         /* After merging, pg should remain in the same node. */
@@ -491,9 +584,251 @@ static void free_heap_pages(
     PFN_ORDER(pg) = order;
     page_list_add_tail(pg, &heap(node, zone, order));
 
+    if (tainted)
+        reserve_offlined_page(pg);
+
     spin_unlock(&heap_lock);
 }
 
+
+/*
+ * Following possible status for a page:
+ * free and Online; free and offlined; free and offlined and broken;
+ * assigned and online; assigned and offlining; assigned and offling and broken
+ *
+ * Following rules applied for page offline:
+ * Once a page is broken, it can't be assigned anymore
+ * A page will be offlined only if it is free
+ * return original count_info
+ *
+ */
+static unsigned long mark_page_offline(struct page_info *pg, int broken)
+{
+    unsigned long nx, x, y = pg->count_info;
+
+    ASSERT(page_is_ram_type(page_to_mfn(pg), RAM_TYPE_CONVENTIONAL));
+    /*
+     * Caller gurantee the page will not be reassigned during this process
+     */
+    ASSERT(spin_is_locked(&heap_lock));
+
+    do {
+        nx = x = y;
+
+        if ( ((x & PGC_offlined_broken) == PGC_offlined_broken) )
+            return y;
+        /* PGC_offlined means it is free pages */
+        if (x & PGC_offlined)
+        {
+            if (broken && !(nx & PGC_broken))
+                nx |= PGC_broken;
+            else
+                return y;
+        }
+        /* It is not offlined, not reserved page */
+        else if ( allocated_in_map(page_to_mfn(pg)) )
+            nx |= PGC_offlining;
+        else
+            nx |= PGC_offlined;
+
+        if (broken)
+            nx |= PGC_broken;
+    } while ( (y = cmpxchg(&pg->count_info, x, nx)) != x );
+
+    return y;
+}
+
+static int reserve_heap_page(struct page_info *pg)
+{
+    struct page_info *head = NULL;
+    unsigned int i, node = phys_to_nid(page_to_maddr(pg));
+    unsigned int zone = page_to_zone(pg);
+
+    /* get the header */
+    for ( i = 0; i <= MAX_ORDER; i++ )
+    {
+        struct page_info *tmp;
+
+        if ( page_list_empty(&heap(node, zone, i)) )
+            continue;
+
+        page_list_for_each_safe(head, tmp, &heap(node, zone, i))
+        {
+            if ( (head <= pg) &&
+                 (head + (1UL << i) > pg) )
+                return reserve_offlined_page(head);
+        }
+    }
+
+    return -EINVAL;
+
+}
+
+/*
+ * offline one page
+ */
+int offline_page(unsigned long mfn, int broken, uint32_t *status)
+{
+    unsigned long old_info = 0;
+    struct domain *owner;
+    int ret = 0;
+    struct page_info *pg;
+
+    if (mfn > max_page)
+    {
+        dprintk(XENLOG_WARNING,
+                "try to offline page out of range %lx\n", mfn);
+        return -EINVAL;
+    }
+
+    *status = 0;
+    pg = mfn_to_page(mfn);
+
+
+#if defined(__x86_64__)
+     /* Xen's txt mfn in x86_64 is reserved in e820 */
+    if ( is_xen_fixed_mfn(mfn) )
+#elif defined(__i386__)
+    if ( is_xen_heap_mfn(mfn) )
+#endif
+    {
+        *status = PG_OFFLINE_XENPAGE | PG_OFFLINE_FAILED |
+          (DOMID_XEN << PG_OFFLINE_OWNER_SHIFT);
+        return -EPERM;
+    }
+
+    /*
+     * N.B. xen's txt in x86_64 is marked reserved and handled already
+     *  Also kexec range is reserved
+     */
+     if (!page_is_ram_type(mfn, RAM_TYPE_CONVENTIONAL))
+     {
+        *status = PG_OFFLINE_FAILED | PG_OFFLINE_NOT_CONV_RAM;
+        return -EINVAL;
+     }
+
+    spin_lock(&heap_lock);
+
+    old_info = mark_page_offline(pg, broken);
+
+    if ( !allocated_in_map(mfn) )
+    {
+        /* Free pages are reserve directly */
+        reserve_heap_page(pg);
+        *status = PG_OFFLINE_OFFLINED;
+    }
+    else if (test_bit(_PGC_offlined, &pg->count_info))
+    {
+        *status = PG_OFFLINE_OFFLINED;
+    }
+    else if ((owner = page_get_owner_and_reference(pg)))
+    {
+            *status = PG_OFFLINE_OWNED | PG_OFFLINE_PENDING |
+              (owner->domain_id << PG_OFFLINE_OWNER_SHIFT);
+            /* Release the reference since it will not be allocated anymore */
+            put_page(pg);
+    }
+    else if ( old_info & PGC_xen_heap)
+    {
+        *status = PG_OFFLINE_XENPAGE | PG_OFFLINE_PENDING |
+          (DOMID_XEN << PG_OFFLINE_OWNER_SHIFT);
+    }
+    else
+    {
+        /*
+         * assign_pages does not hold heap_lock, so small window that the owner
+         * may be set later, but please notice owner will only change from
+         * NULL to be set, not verse, since page is offlining now.
+         * No windows If called from #MC handler, since all CPU are in softirq
+         * If called from user space like CE handling, tools can wait some time
+         * before call again.
+         */
+        *status = PG_OFFLINE_ANONYMOUS | PG_OFFLINE_FAILED |
+                  (DOMID_INVALID << PG_OFFLINE_OWNER_SHIFT );
+    }
+
+    if (broken)
+        *status |= PG_OFFLINE_BROKEN;
+
+    spin_unlock(&heap_lock);
+
+    return ret;
+}
+
+/*
+ * Online the memory.
+ *   The caller should make sure end_pfn <= max_page,
+ *   if not, expand_pages() should be called prior to online_page().
+ */
+unsigned int online_page(unsigned long mfn, uint32_t *status)
+{
+    struct page_info *pg;
+    int ret = 0, free = 0;
+
+    if ( mfn > max_page )
+    {
+        dprintk(XENLOG_WARNING, "call expand_pages() first\n");
+        return -EINVAL;
+    }
+
+    pg = mfn_to_page(mfn);
+
+    *status = 0;
+
+    spin_lock(&heap_lock);
+
+    if ( unlikely(is_page_broken(pg)) )
+    {
+        ret = -EINVAL;
+        *status = PG_ONLINE_FAILED |PG_ONLINE_BROKEN;
+    }
+    else if (pg->count_info & PGC_offlined)
+    {
+        clear_bit(_PGC_offlined, &pg->count_info);
+        page_list_del(pg, &page_offlined_list);
+        *status = PG_ONLINE_ONLINED;
+        free = 1;
+    }
+    else if (pg->count_info & PGC_offlining)
+    {
+        clear_bit(_PGC_offlining, &pg->count_info);
+        *status = PG_ONLINE_ONLINED;
+    }
+    spin_unlock(&heap_lock);
+
+    if (free)
+        free_heap_pages(pg, 0);
+
+    return ret;
+}
+
+int query_page_offline(unsigned long mfn, uint32_t *status)
+{
+    struct page_info *pg;
+
+    if ( (mfn > max_page) || !page_is_ram_type(mfn, RAM_TYPE_CONVENTIONAL) )
+    {
+        dprintk(XENLOG_WARNING, "call expand_pages() first\n");
+        return -EINVAL;
+    }
+
+    *status = 0;
+    spin_lock(&heap_lock);
+
+    pg = mfn_to_page(mfn);
+
+    if (pg->count_info & PGC_offlining)
+        *status |= PG_OFFLINE_STATUS_OFFLINE_PENDING;
+    if (pg->count_info & PGC_broken)
+        *status |= PG_OFFLINE_STATUS_BROKEN;
+    if (pg->count_info & PGC_offlined)
+        *status |= PG_OFFLINE_STATUS_OFFLINED;
+
+    spin_unlock(&heap_lock);
+
+    return 0;
+}
+
 /*
  * Hand the specified arbitrary page range to the specified heap zone
  * checking the node_id of the previous page.  If they differ and the

xen/common/sysctl.c

@@ -233,6 +233,61 @@ long do_sysctl(XEN_GUEST_HANDLE(xen_sysctl_t) u_sysctl)
     }
     break;
 
+    case XEN_SYSCTL_page_offline_op:
+    {
+        uint32_t *status, *ptr;
+        unsigned long pfn;
+
+        ptr = status = xmalloc_bytes( sizeof(uint32_t) *
+                                (op->u.page_offline.end -
+                                  op->u.page_offline.start + 1));
+        if (!status)
+        {
+            dprintk(XENLOG_WARNING, "Out of memory for page offline op\n");
+            ret = -ENOMEM;
+            break;
+        }
+
+        memset(status, PG_OFFLINE_INVALID, sizeof(uint32_t) *
+                      (op->u.page_offline.end - op->u.page_offline.start + 1));
+
+        for ( pfn = op->u.page_offline.start;
+              pfn <= op->u.page_offline.end;
+              pfn ++ )
+        {
+            switch (op->u.page_offline.cmd)
+            {
+                /* Shall revert her if failed, or leave caller do it? */
+                case sysctl_page_offline:
+                    ret = offline_page(pfn, 0, ptr++);
+                    break;
+                case sysctl_page_online:
+                    ret = online_page(pfn, ptr++);
+                    break;
+                case sysctl_query_page_offline:
+                    ret = query_page_offline(pfn, ptr++);
+                    break;
+                default:
+                    gdprintk(XENLOG_WARNING, "invalid page offline op %x\n",
+                            op->u.page_offline.cmd);
+                    ret = -EINVAL;
+                    break;
+            }
+
+            if (ret)
+                break;
+        }
+
+        if (copy_to_guest(op->u.page_offline.status, status,
+                          op->u.page_offline.end - op->u.page_offline.start + 1))
+        {
+            ret = -EFAULT;
+            break;
+        }
+        xfree(status);
+    }
+    break;
+
     default:
         ret = arch_do_sysctl(op, u_sysctl);
         break;

xen/include/asm-x86/mm.h

@@ -198,8 +198,25 @@ struct page_info
  /* 3-bit PAT/PCD/PWT cache-attribute hint. */
 #define PGC_cacheattr_base PG_shift(6)
 #define PGC_cacheattr_mask PG_mask(7, 6)
+
+  /* Page is broken? */
+ #define _PGC_broken         PG_shift(7)
+ #define PGC_broken          PG_mask(1, 7)
+  /* Page is offline pending ? */
+ #define _PGC_offlining      PG_shift(8)
+ #define PGC_offlining       PG_mask(1, 8)
+  /* Page is offlined */
+ #define _PGC_offlined       PG_shift(9)
+ #define PGC_offlined        PG_mask(1, 9)
+ #define PGC_offlined_broken (PGC_offlined | PGC_broken)
+
+ #define is_page_offlining(page)          ((page)->count_info & PGC_offlining)
+ #define is_page_offlined(page)          ((page)->count_info & PGC_offlined)
+ #define is_page_broken(page)           ((page)->count_info & PGC_broken)
+ #define is_page_online(page)           (!is_page_offlined(page))
+
  /* Count of references to this frame. */
-#define PGC_count_width   PG_shift(6)
+#define PGC_count_width   PG_shift(9)
 #define PGC_count_mask    ((1UL<<PGC_count_width)-1)
 
 #if defined(__i386__)
@@ -209,9 +226,13 @@ struct page_info
     (_mfn < paddr_to_pfn(xenheap_phys_end));            \
 })
 #else
+extern unsigned long allocator_bitmap_end;
 #define is_xen_heap_page(page) ((page)->count_info & PGC_xen_heap)
 #define is_xen_heap_mfn(mfn) \
     (__mfn_valid(mfn) && is_xen_heap_page(__mfn_to_page(mfn)))
+#define is_xen_fixed_mfn(mfn) \
+    ( (mfn << PAGE_SHIFT) >= __pa(&_start) &&    \
+          (mfn << PAGE_SHIFT) <= allocator_bitmap_end )
 #endif
 
 #if defined(__i386__)

xen/include/public/sysctl.h

@@ -359,6 +359,54 @@ struct xen_sysctl_pm_op {
     };
 };
 
+#define XEN_SYSCTL_page_offline_op        14
+struct xen_sysctl_page_offline_op {
+    /* IN: range of page to be offlined */
+#define sysctl_page_offline     1
+#define sysctl_page_online      2
+#define sysctl_query_page_offline  3
+    uint32_t cmd;
+    uint32_t start;
+    uint32_t end;
+    /* OUT: result of page offline request */
+    /*
+     * bit 0~15: result flags
+     * bit 16~31: owner
+     */
+    XEN_GUEST_HANDLE(uint32) status;
+};
+
+#define PG_OFFLINE_STATUS_MASK    (0xFFUL)
+
+/* The result is invalid, i.e. HV does not handle it */
+#define PG_OFFLINE_INVALID   (0x1UL << 0)
+
+#define PG_OFFLINE_OFFLINED  (0x1UL << 1)
+#define PG_OFFLINE_PENDING   (0x1UL << 2)
+#define PG_OFFLINE_FAILED    (0x1UL << 3)
+
+#define PG_ONLINE_FAILED     PG_OFFLINE_FAILED
+#define PG_ONLINE_ONLINED    PG_OFFLINE_OFFLINED
+
+#define PG_OFFLINE_STATUS_OFFLINED              (0x1UL << 1)
+#define PG_OFFLINE_STATUS_ONLINE                (0x1UL << 2)
+#define PG_OFFLINE_STATUS_OFFLINE_PENDING       (0x1UL << 3)
+#define PG_OFFLINE_STATUS_BROKEN                (0x1UL << 4)
+
+#define PG_OFFLINE_MISC_MASK    (0xFFUL << 4)
+
+/* only valid when PG_OFFLINE_FAILED */
+#define PG_OFFLINE_XENPAGE   (0x1UL << 8)
+#define PG_OFFLINE_DOM0PAGE  (0x1UL << 9)
+#define PG_OFFLINE_ANONYMOUS (0x1UL << 10)
+#define PG_OFFLINE_NOT_CONV_RAM   (0x1UL << 11)
+#define PG_OFFLINE_OWNED     (0x1UL << 12)
+
+#define PG_OFFLINE_BROKEN    (0x1UL << 13)
+#define PG_ONLINE_BROKEN     PG_OFFLINE_BROKEN
+
+#define PG_OFFLINE_OWNER_SHIFT 16
+
 struct xen_sysctl {
     uint32_t cmd;
     uint32_t interface_version; /* XEN_SYSCTL_INTERFACE_VERSION */
@@ -375,6 +423,7 @@ struct xen_sysctl {
         struct xen_sysctl_get_pmstat        get_pmstat;
         struct xen_sysctl_cpu_hotplug       cpu_hotplug;
         struct xen_sysctl_pm_op             pm_op;
+        struct xen_sysctl_page_offline_op   page_offline;
         uint8_t                             pad[128];
     } u;
 };

xen/include/public/xen.h

@@ -354,6 +354,9 @@ typedef uint16_t domid_t;
  */
 #define DOMID_XEN  (0x7FF2U)
 
+/* DOMID_INVALID is used to identity invalid domid */
+#define DOMID_INVALID (0x7FFFU)
+
 /*
  * Send an array of these to HYPERVISOR_mmu_update().
  * NB. The fields are natural pointer/address size for this architecture.

xen/include/xen/mm.h

@@ -60,6 +60,9 @@ unsigned long avail_domheap_pages_region(
 unsigned long avail_domheap_pages(void);
 #define alloc_domheap_page(d,f) (alloc_domheap_pages(d,0,f))
 #define free_domheap_page(p)  (free_domheap_pages(p,0))
+unsigned int online_page(unsigned long mfn, uint32_t *status);
+int offline_page(unsigned long mfn, int broken, uint32_t *status);
+int query_page_offline(unsigned long mfn, uint32_t *status);
 
 void scrub_heap_pages(void);