x86, cpu hotplug: Synchronise vcpu state earlier during cpu offline.

Needs to happen before non-idle VCPU is fully descheduled after CPU is
removed from cpu_online_map. Else sync_vcpu_execstate() doesn't work
properly.

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>

xen/arch/x86/domain.c

@@ -1442,7 +1442,8 @@ void context_switch(struct vcpu *prev, struct vcpu *next)
 
     set_current(next);
 
-    if ( (per_cpu(curr_vcpu, cpu) == next) || is_idle_vcpu(next) )
+    if ( (per_cpu(curr_vcpu, cpu) == next) ||
+         (is_idle_vcpu(next) && cpu_online(cpu)) )
     {
         local_irq_enable();
     }

xen/arch/x86/smpboot.c

@@ -997,17 +997,13 @@ static int __devinit do_boot_cpu(int apicid, int cpu)
 	return boot_error;
 }
 
-static void idle_task_exit(void)
-{
-	/* Give up lazy state borrowed by this idle vcpu */
-	__sync_lazy_execstate();
-}
-
 void cpu_exit_clear(void)
 {
 	int cpu = raw_smp_processor_id();
 
-	idle_task_exit();
+	/* Previous non-idle state should be synchronised already. */
+	if (__sync_lazy_execstate())
+		BUG();
 
 	cpucount --;
 	cpu_uninit();
@@ -1302,6 +1298,13 @@ int __cpu_disable(void)
 
 	remove_siblinginfo(cpu);
 
+	/*
+	 * If we are running the idle vcpu, sync last non-idle vcpu's state
+	 * before changing cpu_online_map. If we are running non-idle vcpu,
+	 * we will synchronously sync the state in context_switch() later.
+	 */
+	__sync_lazy_execstate();
+
 	/* It's now safe to remove this processor from the online map */
 	cpu_clear(cpu, cpu_online_map);
 	fixup_irqs();