PV-on-HVM: More save/restore fixes.

Signed-off-by: Keir Fraser <keir@xensource.com>

linux-2.6-xen-sparse/drivers/xen/core/gnttab.c

@@ -60,9 +60,6 @@ static grant_ref_t gnttab_free_head;
 static DEFINE_SPINLOCK(gnttab_list_lock);
 
 static struct grant_entry *shared;
-#ifndef CONFIG_XEN
-static unsigned long resume_frames;
-#endif
 
 static struct gnttab_free_callback *gnttab_free_callback_list;
 
@@ -514,6 +511,8 @@ int gnttab_suspend(void)
 
 #include <platform-pci.h>
 
+static unsigned long resume_frames;
+
 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 {
 	struct xen_add_to_physmap xatp;
@@ -543,23 +542,17 @@ int gnttab_resume(void)
 	if (max_nr_gframes < nr_gframes)
 		return -ENOSYS;
 
-	resume_frames = alloc_xen_mmio(PAGE_SIZE * max_nr_gframes);
+	if (!resume_frames) {
+		resume_frames = alloc_xen_mmio(PAGE_SIZE * max_nr_gframes);
+		shared = ioremap(resume_frames, PAGE_SIZE * max_nr_gframes);
+		if (shared == NULL) {
+			printk("error to ioremap gnttab share frames\n");
+			return -1;
+		}
+	}
 
 	gnttab_map(0, nr_gframes - 1);
 
-	shared = ioremap(resume_frames, PAGE_SIZE * max_nr_gframes);
-	if (shared == NULL) {
-		printk("error to ioremap gnttab share frames\n");
-		return -1;
-	}
-
-	return 0;
-}
-
-int gnttab_suspend(void)
-{
-	iounmap(shared);
-	resume_frames = 0;
 	return 0;
 }
 

unmodified_drivers/linux-2.6/platform-pci/evtchn.c

@@ -28,8 +28,10 @@
  * IN THE SOFTWARE.
  */
 
+#include <linux/config.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
+#include <linux/spinlock.h>
 #include <xen/evtchn.h>
 #include <xen/interface/hvm/ioreq.h>
 #include <xen/features.h>
@@ -41,29 +43,37 @@
 
 void *shared_info_area;
 
-static DEFINE_MUTEX(irq_evtchn_mutex);
-
 #define is_valid_evtchn(x)	((x) != 0)
 #define evtchn_from_irq(x)	(irq_evtchn[irq].evtchn)
 
 static struct {
+	spinlock_t lock;
 	irqreturn_t(*handler) (int, void *, struct pt_regs *);
 	void *dev_id;
 	int evtchn;
 	int close:1; /* close on unbind_from_irqhandler()? */
 	int inuse:1;
+	int in_handler:1;
 } irq_evtchn[256];
 static int evtchn_to_irq[NR_EVENT_CHANNELS] = {
 	[0 ...  NR_EVENT_CHANNELS-1] = -1 };
 
-static int find_unbound_irq(void)
+static DEFINE_SPINLOCK(irq_alloc_lock);
+
+static int alloc_xen_irq(void)
 {
 	static int warned;
 	int irq;
 
-	for (irq = 0; irq < ARRAY_SIZE(irq_evtchn); irq++)
-		if (!irq_evtchn[irq].inuse)
-			return irq;
+	spin_lock(&irq_alloc_lock);
+
+	for (irq = 0; irq < ARRAY_SIZE(irq_evtchn); irq++) {
+		if (irq_evtchn[irq].inuse) 
+			continue;
+		irq_evtchn[irq].inuse = 1;
+		spin_unlock(&irq_alloc_lock);
+		return irq;
+	}
 
 	if (!warned) {
 		warned = 1;
@@ -71,9 +81,18 @@ static int find_unbound_irq(void)
 		       "increase irq_evtchn[] size in evtchn.c.\n");
 	}
 
+	spin_unlock(&irq_alloc_lock);
+
 	return -ENOSPC;
 }
 
+static void free_xen_irq(int irq)
+{
+	spin_lock(&irq_alloc_lock);
+	irq_evtchn[irq].inuse = 0;
+	spin_unlock(&irq_alloc_lock);
+}
+
 int irq_to_evtchn_port(int irq)
 {
 	return irq_evtchn[irq].evtchn;
@@ -93,8 +112,7 @@ void unmask_evtchn(int port)
 	shared_info_t *s = shared_info_area;
 	vcpu_info_t *vcpu_info;
 
-	preempt_disable();
-	cpu = smp_processor_id();
+	cpu = get_cpu();
 	vcpu_info = &s->vcpu_info[cpu];
 
 	/* Slow path (hypercall) if this is a non-local port.  We only
@@ -103,7 +121,7 @@ void unmask_evtchn(int port)
 		evtchn_unmask_t op = { .port = port };
 		(void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask,
 						  &op);
-		preempt_enable();
+		put_cpu();
 		return;
 	}
 
@@ -121,7 +139,8 @@ void unmask_evtchn(int port)
 		if (!vcpu_info->evtchn_upcall_mask)
 			force_evtchn_callback();
 	}
-	preempt_enable();
+
+	put_cpu();
 }
 EXPORT_SYMBOL(unmask_evtchn);
 
@@ -135,20 +154,19 @@ int bind_listening_port_to_irqhandler(
 	struct evtchn_alloc_unbound alloc_unbound;
 	int err, irq;
 
-	mutex_lock(&irq_evtchn_mutex);
-
-	irq = find_unbound_irq();
-	if (irq < 0) {
-		mutex_unlock(&irq_evtchn_mutex);
+	irq = alloc_xen_irq();
+	if (irq < 0)
 		return irq;
-	}
+
+	spin_lock_irq(&irq_evtchn[irq].lock);
 
 	alloc_unbound.dom        = DOMID_SELF;
 	alloc_unbound.remote_dom = remote_domain;
 	err = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
 					  &alloc_unbound);
 	if (err) {
-		mutex_unlock(&irq_evtchn_mutex);
+		spin_unlock_irq(&irq_evtchn[irq].lock);
+		free_xen_irq(irq);
 		return err;
 	}
 
@@ -156,13 +174,13 @@ int bind_listening_port_to_irqhandler(
 	irq_evtchn[irq].dev_id  = dev_id;
 	irq_evtchn[irq].evtchn  = alloc_unbound.port;
 	irq_evtchn[irq].close   = 1;
-	irq_evtchn[irq].inuse   = 1;
 
 	evtchn_to_irq[alloc_unbound.port] = irq;
 
 	unmask_evtchn(alloc_unbound.port);
 
-	mutex_unlock(&irq_evtchn_mutex);
+	spin_unlock_irq(&irq_evtchn[irq].lock);
+
 	return irq;
 }
 EXPORT_SYMBOL(bind_listening_port_to_irqhandler);
@@ -176,34 +194,34 @@ int bind_caller_port_to_irqhandler(
 {
 	int irq;
 
-	mutex_lock(&irq_evtchn_mutex);
-
-	irq = find_unbound_irq();
-	if (irq < 0) {
-		mutex_unlock(&irq_evtchn_mutex);
+	irq = alloc_xen_irq();
+	if (irq < 0)
 		return irq;
-	}
+
+	spin_lock_irq(&irq_evtchn[irq].lock);
 
 	irq_evtchn[irq].handler = handler;
 	irq_evtchn[irq].dev_id  = dev_id;
 	irq_evtchn[irq].evtchn  = caller_port;
 	irq_evtchn[irq].close   = 0;
-	irq_evtchn[irq].inuse   = 1;
 
 	evtchn_to_irq[caller_port] = irq;
 
 	unmask_evtchn(caller_port);
 
-	mutex_unlock(&irq_evtchn_mutex);
+	spin_unlock_irq(&irq_evtchn[irq].lock);
+
 	return irq;
 }
 EXPORT_SYMBOL(bind_caller_port_to_irqhandler);
 
 void unbind_from_irqhandler(unsigned int irq, void *dev_id)
 {
-	int evtchn = evtchn_from_irq(irq);
+	int evtchn;
 
-	mutex_lock(&irq_evtchn_mutex);
+	spin_lock_irq(&irq_evtchn[irq].lock);
+
+	evtchn = evtchn_from_irq(irq);
 
 	if (is_valid_evtchn(evtchn)) {
 		evtchn_to_irq[irq] = -1;
@@ -216,21 +234,28 @@ void unbind_from_irqhandler(unsigned int irq, void *dev_id)
 
 	irq_evtchn[irq].handler = NULL;
 	irq_evtchn[irq].evtchn  = 0;
-	irq_evtchn[irq].inuse   = 0;
 
-	mutex_unlock(&irq_evtchn_mutex);
+	spin_unlock_irq(&irq_evtchn[irq].lock);
+
+	while (irq_evtchn[irq].in_handler)
+		cpu_relax();
+
+	free_xen_irq(irq);
 }
 EXPORT_SYMBOL(unbind_from_irqhandler);
 
 void notify_remote_via_irq(int irq)
 {
-	int evtchn = evtchn_from_irq(irq);
+	int evtchn;
+
+	evtchn = evtchn_from_irq(irq);
 	if (is_valid_evtchn(evtchn))
 		notify_remote_via_evtchn(evtchn);
 }
 EXPORT_SYMBOL(notify_remote_via_irq);
 
-irqreturn_t evtchn_interrupt(int irq, void *dev_id, struct pt_regs *regs)
+static irqreturn_t evtchn_interrupt(int irq, void *dev_id,
+				    struct pt_regs *regs)
 {
 	unsigned int l1i, port;
 	/* XXX: All events are bound to vcpu0 but irq may be redirected. */
@@ -249,13 +274,30 @@ irqreturn_t evtchn_interrupt(int irq, void *dev_id, struct pt_regs *regs)
 		while ((l2 = s->evtchn_pending[l1i] & ~s->evtchn_mask[l1i])) {
 			port = (l1i * BITS_PER_LONG) + __ffs(l2);
 			synch_clear_bit(port, &s->evtchn_pending[0]);
+
 			irq = evtchn_to_irq[port];
-			if ((irq >= 0) &&
-			    ((handler = irq_evtchn[irq].handler) != NULL))
-				handler(irq, irq_evtchn[irq].dev_id, regs);
-			else
-				printk(KERN_WARNING "unexpected event channel "
-				       "upcall on port %d!\n", port);
+			if (irq < 0)
+				continue;
+
+			spin_lock(&irq_evtchn[irq].lock);
+			handler = irq_evtchn[irq].handler;
+			dev_id  = irq_evtchn[irq].dev_id;
+			if (unlikely(handler == NULL)) {
+				printk("Xen IRQ%d (port %d) has no handler!\n",
+				       irq, port);
+				spin_unlock(&irq_evtchn[irq].lock);
+				continue;
+			}
+			irq_evtchn[irq].in_handler = 1;
+			spin_unlock(&irq_evtchn[irq].lock);
+
+			local_irq_enable();
+			handler(irq, irq_evtchn[irq].dev_id, regs);
+			local_irq_disable();
+
+			spin_lock(&irq_evtchn[irq].lock);
+			irq_evtchn[irq].in_handler = 0;
+			spin_unlock(&irq_evtchn[irq].lock);
 		}
 	}
 
@@ -268,16 +310,6 @@ void force_evtchn_callback(void)
 }
 EXPORT_SYMBOL(force_evtchn_callback);
 
-void irq_suspend(void)
-{
-	mutex_lock(&irq_evtchn_mutex);
-}
-
-void irq_suspend_cancel(void)
-{
-	mutex_unlock(&irq_evtchn_mutex);
-}
-
 void irq_resume(void)
 {
 	int evtchn, irq;
@@ -289,6 +321,16 @@ void irq_resume(void)
 
 	for (irq = 0; irq < ARRAY_SIZE(irq_evtchn); irq++)
 		irq_evtchn[irq].evtchn = 0;
-
-	mutex_unlock(&irq_evtchn_mutex);
+}
+
+int xen_irq_init(struct pci_dev *pdev)
+{
+	int irq;
+
+	for (irq = 0; irq < ARRAY_SIZE(irq_evtchn); irq++)
+		spin_lock_init(&irq_evtchn[irq].lock);
+
+	return request_irq(pdev->irq, evtchn_interrupt,
+			   SA_SHIRQ | SA_SAMPLE_RANDOM | SA_INTERRUPT,
+			   "xen-platform-pci", pdev);
 }

unmodified_drivers/linux-2.6/platform-pci/machine_reboot.c

@@ -1,24 +1,81 @@
 #include <linux/config.h>
+#include <linux/stop_machine.h>
+#include <xen/evtchn.h>
+#include <xen/gnttab.h>
 #include <xen/xenbus.h>
 #include "platform-pci.h"
 #include <asm/hypervisor.h>
 
-int __xen_suspend(int fast_suspend)
+/*
+ * Spinning prevents, for example, APs touching grant table entries while
+ * the shared grant table is not mapped into the address space imemdiately
+ * after resume.
+ */
+static void ap_suspend(void *_ap_spin)
+{
+	int *ap_spin = _ap_spin;
+
+	BUG_ON(!irqs_disabled());
+
+	while (*ap_spin) {
+		cpu_relax();
+		HYPERVISOR_yield();
+	}
+}
+
+static int bp_suspend(void)
 {
 	int suspend_cancelled;
 
-	xenbus_suspend();
-	platform_pci_suspend();
+	BUG_ON(!irqs_disabled());
 
 	suspend_cancelled = HYPERVISOR_shutdown(SHUTDOWN_suspend);
 
-	if (suspend_cancelled) {
-		platform_pci_suspend_cancel();
-		xenbus_suspend_cancel();
-	} else {
+	if (!suspend_cancelled) {
 		platform_pci_resume();
-		xenbus_resume();
+		gnttab_resume();
+		irq_resume();
 	}
 
+	return suspend_cancelled;
+}
+
+int __xen_suspend(int fast_suspend)
+{
+	int err, suspend_cancelled, ap_spin;
+
+	xenbus_suspend();
+
+	preempt_disable();
+
+	/* Prevent any races with evtchn_interrupt() handler. */
+	disable_irq(xen_platform_pdev->irq);
+
+	ap_spin = 1;
+	smp_mb();
+
+	err = smp_call_function(ap_suspend, &ap_spin, 0, 0);
+	if (err < 0) {
+		preempt_enable();
+		xenbus_suspend_cancel();
+		return err;
+	}
+
+	local_irq_disable();
+	suspend_cancelled = bp_suspend();
+	local_irq_enable();
+
+	smp_mb();
+	ap_spin = 0;
+
+	enable_irq(xen_platform_pdev->irq);
+
+	preempt_enable();
+
+	if (!suspend_cancelled)
+		xenbus_resume();
+	else
+		xenbus_suspend_cancel();
+
 	return 0;
 }

unmodified_drivers/linux-2.6/platform-pci/platform-pci.c

@@ -40,7 +40,6 @@
 #include <xen/interface/hvm/params.h>
 #include <xen/features.h>
 #include <xen/evtchn.h>
-#include <xen/gnttab.h>
 #ifdef __ia64__
 #include <asm/xen/xencomm.h>
 #endif
@@ -62,6 +61,8 @@ MODULE_AUTHOR("ssmith@xensource.com");
 MODULE_DESCRIPTION("Xen platform PCI device");
 MODULE_LICENSE("GPL");
 
+struct pci_dev *xen_platform_pdev;
+
 static unsigned long shared_info_frame;
 static uint64_t callback_via;
 
@@ -89,8 +90,6 @@ static int __devinit init_xen_info(void)
 	if (shared_info_area == NULL)
 		panic("can't map shared info\n");
 
-	gnttab_init();
-
 	return 0;
 }
 
@@ -199,8 +198,10 @@ static int set_callback_via(uint64_t via)
 	return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
 }
 
+int xen_irq_init(struct pci_dev *pdev);
 int xenbus_init(void);
 int xen_reboot_init(void);
+int gnttab_init(void);
 
 static int __devinit platform_pci_init(struct pci_dev *pdev,
 				       const struct pci_device_id *ent)
@@ -209,6 +210,10 @@ static int __devinit platform_pci_init(struct pci_dev *pdev,
 	long ioaddr, iolen;
 	long mmio_addr, mmio_len;
 
+	if (xen_platform_pdev)
+		return -EBUSY;
+	xen_platform_pdev = pdev;
+
 	i = pci_enable_device(pdev);
 	if (i)
 		return i;
@@ -249,9 +254,10 @@ static int __devinit platform_pci_init(struct pci_dev *pdev,
 	if ((ret = init_xen_info()))
 		goto out;
 
-	if ((ret = request_irq(pdev->irq, evtchn_interrupt,
-			       SA_SHIRQ | SA_SAMPLE_RANDOM,
-			       "xen-platform-pci", pdev)))
+	if ((ret = gnttab_init()))
+		goto out;
+
+	if ((ret = xen_irq_init(pdev)))
 		goto out;
 
 	if ((ret = set_callback_via(callback_via)))
@@ -292,18 +298,6 @@ static struct pci_driver platform_driver = {
 
 static int pci_device_registered;
 
-void platform_pci_suspend(void)
-{
-	gnttab_suspend();
-	irq_suspend();
-}
-
-void platform_pci_suspend_cancel(void)
-{
-	irq_suspend_cancel();
-	gnttab_resume();
-}
-
 void platform_pci_resume(void)
 {
 	struct xen_add_to_physmap xatp;
@@ -319,12 +313,8 @@ void platform_pci_resume(void)
 	if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
 		BUG();
 
-	irq_resume();
-
 	if (set_callback_via(callback_via))
 		printk("platform_pci_resume failure!\n");
-
-	gnttab_resume();
 }
 
 static int __init platform_pci_module_init(void)

unmodified_drivers/linux-2.6/platform-pci/platform-pci.h

@@ -22,16 +22,11 @@
 #ifndef _XEN_PLATFORM_PCI_H
 #define _XEN_PLATFORM_PCI_H
 
-#include <linux/interrupt.h>
+#include <linux/pci.h>
 
 unsigned long alloc_xen_mmio(unsigned long len);
-int gnttab_init(void);
-irqreturn_t evtchn_interrupt(int irq, void *dev_id, struct pt_regs *regs);
-void irq_suspend(void);
-void irq_suspend_cancel(void);
-
-void platform_pci_suspend(void);
-void platform_pci_suspend_cancel(void);
 void platform_pci_resume(void);
 
+extern struct pci_dev *xen_platform_pdev;
+
 #endif /* _XEN_PLATFORM_PCI_H */