bitkeeper revision 1.911.1.5 (40acee35tmC494sW3OcQEIN4ymTS1w)

Nasty temporary mechanism that allows dom0 to block after issuing a stop on another domain.

tools/xc/lib/xc_domain.c

@@ -46,6 +46,7 @@ int xc_domain_stop(int xc_handle,
     dom0_op_t op;
     op.cmd = DOM0_STOPDOMAIN;
     op.u.stopdomain.domain = (domid_t)domid;
+    op.u.stopdomain.sync = 0; // async
     return do_dom0_op(xc_handle, &op);
 }    
 

tools/xc/lib/xc_linux_save.c

@@ -82,6 +82,67 @@ inline void set_bit ( int nr, volatile void * addr)
 	(1 << (nr % (sizeof(unsigned long)*8) ) );
 }
 
+long long tv_to_us( struct timeval *new )
+{
+    return (new->tv_sec * 1000000) + new->tv_usec;
+}
+
+long long tvdelta( struct timeval *new, struct timeval *old )
+{
+    return ((new->tv_sec - old->tv_sec)*1000000 ) + 
+	(new->tv_usec - old->tv_usec);
+}
+
+int track_cpu_usage_dom0( int xc_handle, int print )
+{
+    static struct timeval wall_last;
+    static long long      cpu_last;
+
+    struct timeval        wall_now;
+    long long             cpu_now, wall_delta, cpu_delta;
+
+    gettimeofday(&wall_now, NULL);
+
+    cpu_now = xc_domain_get_cpu_usage( xc_handle, 0 )/1000;
+
+    wall_delta = tvdelta(&wall_now,&wall_last)/1000;
+    cpu_delta  = (cpu_now - cpu_last)/1000;
+
+    if(print)
+	printf("Dom0  : wall delta %lldms, cpu delta %lldms    : %d%%\n",
+	   wall_delta, cpu_delta, (cpu_delta*100)/wall_delta);
+
+    cpu_last  = cpu_now;
+    wall_last = wall_now;	
+
+    return 0;
+}
+
+int track_cpu_usage_target( int xc_handle, u64 domid, int print )
+{
+    static struct timeval wall_last;
+    static long long      cpu_last;
+
+    struct timeval        wall_now;
+    long long             cpu_now, wall_delta, cpu_delta;
+
+    gettimeofday(&wall_now, NULL);
+
+    cpu_now = xc_domain_get_cpu_usage( xc_handle, domid )/1000;
+
+    wall_delta = tvdelta(&wall_now,&wall_last)/1000;
+    cpu_delta  = (cpu_now - cpu_last)/1000;
+
+    if(print)
+	printf("Target: wall delta %lldms, cpu delta %lldms    : %d%%\n",
+	   wall_delta, cpu_delta, (cpu_delta*100)/wall_delta);
+
+    cpu_last  = cpu_now;
+    wall_last = wall_now;	
+
+    return 0;
+}
+
 
 int xc_linux_save(int xc_handle,
                   u64 domid, 
@@ -95,10 +156,11 @@ int xc_linux_save(int xc_handle,
     int verbose = flags & XCFLAGS_VERBOSE;
     int live = flags & XCFLAGS_LIVE;
     int debug = flags & XCFLAGS_DEBUG;
-    int sent_last_iter, sent_this_iter, skip_this_iter, max_iters;
-
-    /* Remember if we stopped the guest, so we can restart it on exit. */
-    int we_stopped_it = 0;
+    int sent_last_iter, sent_this_iter, skip_this_iter;
+    
+    /* Important tuning parameters */
+    int max_iters  = 29; // limit us to 30 times round loop
+    int max_factor = 3;  // never send more than 3x nr_pfns 
 
     /* The new domain's shared-info frame number. */
     unsigned long shared_info_frame;
@@ -153,39 +215,16 @@ int xc_linux_save(int xc_handle,
     }
 
     /* Ensure that the domain exists, and that it is stopped. */
-    for ( ; ; )
+
+    if ( xc_domain_stop_sync( xc_handle, domid, &op, &ctxt ) )
     {
-        op.cmd = DOM0_GETDOMAININFO;
-        op.u.getdomaininfo.domain = (domid_t)domid;
-        op.u.getdomaininfo.ctxt = &ctxt;
-        if ( (do_dom0_op(xc_handle, &op) < 0) || 
-             ((u64)op.u.getdomaininfo.domain != domid) )
-        {
-            PERROR("Could not get info on domain");
-            goto out;
-        }
-
-        memcpy(name, op.u.getdomaininfo.name, sizeof(name));
-        shared_info_frame = op.u.getdomaininfo.shared_info_frame;
-
-        if ( op.u.getdomaininfo.state == DOMSTATE_STOPPED )
-            break;
-
-        we_stopped_it = 1;
-
-        op.cmd = DOM0_STOPDOMAIN;
-        op.u.stopdomain.domain = (domid_t)domid;
-        if ( do_dom0_op(xc_handle, &op) != 0 )
-        {
-            we_stopped_it = 0;
-            PERROR("Stopping target domain failed");
-            goto out;
-        }
-
-        usleep(1000); // 1ms
-	printf("Sleep for 1ms\n");
+	PERROR("Could not sync stop domain");
+	goto out;
     }
 
+    memcpy(name, op.u.getdomaininfo.name, sizeof(name));
+    shared_info_frame = op.u.getdomaininfo.shared_info_frame;
+
     /* A cheesy test to see whether the domain contains valid state. */
     if ( ctxt.pt_base == 0 )
     {
@@ -292,7 +331,6 @@ int xc_linux_save(int xc_handle,
 
 	last_iter = 0;
 	sent_last_iter = 1<<20; // 4GB's worth of pages
-	max_iters = 29; // limit us to 30 times round loop
     }
     else
 	last_iter = 1;
@@ -384,8 +422,11 @@ int xc_linux_save(int xc_handle,
         goto out;
     }
 
-    /* Now write out each data page, canonicalising page tables as we go... */
+    track_cpu_usage_dom0(xc_handle, 0);
+    track_cpu_usage_target( xc_handle, domid, 0);
 
+    /* Now write out each data page, canonicalising page tables as we go... */
+    
     while(1)
     {
 	unsigned int prev_pc, batch, sent_this_iter;
@@ -615,6 +656,10 @@ int xc_linux_save(int xc_handle,
 
 	verbose_printf("\b\b\b\b100%% (pages sent= %d, skipped= %d )\n", 
 		       sent_this_iter, skip_this_iter );
+
+	track_cpu_usage_dom0(xc_handle, 1);
+	track_cpu_usage_target( xc_handle, domid, 1);
+
 	
 	if ( last_iter )
 	{
@@ -649,12 +694,12 @@ int xc_linux_save(int xc_handle,
 		 // ( sent_this_iter > (sent_last_iter * 0.95) ) ||		 
 		 (iter >= max_iters) || 
 		 (sent_this_iter+skip_this_iter < 10) || 
-		 (total_sent > nr_pfns*2) )
+		 (total_sent > nr_pfns*max_factor) )
 	    {
 		DPRINTF("Start last iteration\n");
 		last_iter = 1;
 
-		xc_domain_stop_sync( xc_handle, domid );
+		xc_domain_stop_sync( xc_handle, domid, &op, NULL );
 
 	    } 
 

tools/xc/lib/xc_private.c

@@ -376,51 +376,67 @@ int finish_mmu_updates(int xc_handle, mmu_t *mmu)
 
 /* this function is a hack until we get proper synchronous domain stop */
 
-int xc_domain_stop_sync( int xc_handle, domid_t domid )
+int xc_domain_stop_sync( int xc_handle, domid_t domid,
+			 dom0_op_t *op, full_execution_context_t *ctxt)
 {
-    dom0_op_t op;
     int i;
-    
 
-    op.cmd = DOM0_STOPDOMAIN;
-    op.u.stopdomain.domain = (domid_t)domid;
-    if ( do_dom0_op(xc_handle, &op) != 0 )
-    {
-	PERROR("Stopping target domain failed");
-	goto out;
-    }
-
-    usleep(100); // 100us
+    printf("Sleep:");
 
     for(i=0;;i++)
-    {
-	if (i>0)
-	    if (i==1) printf("Sleep.");
-	    else printf(".");
+    {    
 
-        op.cmd = DOM0_GETDOMAININFO;
-        op.u.getdomaininfo.domain = (domid_t)domid;
-        op.u.getdomaininfo.ctxt = NULL;
-        if ( (do_dom0_op(xc_handle, &op) < 0) || 
-             ((u64)op.u.getdomaininfo.domain != domid) )
+	op->cmd = DOM0_STOPDOMAIN;
+	op->u.stopdomain.domain = (domid_t)domid;
+	op->u.stopdomain.sync = 1;
+	do_dom0_op(xc_handle, op);
+	/* can't trust return code due to sync stop hack :-(( */
+
+       
+        op->cmd = DOM0_GETDOMAININFO;
+        op->u.getdomaininfo.domain = (domid_t)domid;
+        op->u.getdomaininfo.ctxt = ctxt;
+        if ( (do_dom0_op(xc_handle, op) < 0) || 
+             ((u64)op->u.getdomaininfo.domain != domid) )
         {
             PERROR("Could not get info on domain");
             goto out;
         }
 
-        if ( op.u.getdomaininfo.state == DOMSTATE_STOPPED )
+        if ( op->u.getdomaininfo.state == DOMSTATE_STOPPED )
 	{
 	    printf("Domain %lld stopped\n",domid);
             return 0;
 	}
 	
-	usleep(1000);
+	printf(".");
+
+	//usleep(1000);
     }
 
+    printf("\n");
+
 out:
     return -1;    
 }
 
+long long  xc_domain_get_cpu_usage( int xc_handle, domid_t domid )
+{
+    dom0_op_t op;
+
+    op.cmd = DOM0_GETDOMAININFO;
+    op.u.getdomaininfo.domain = (domid_t)domid;
+    op.u.getdomaininfo.ctxt = NULL;
+    if ( (do_dom0_op(xc_handle, &op) < 0) || 
+	 ((u64)op.u.getdomaininfo.domain != domid) )
+    {
+	PERROR("Could not get info on domain");
+	return -1;
+    }
+    return op.u.getdomaininfo.cpu_time;
+}
+
+
 /**********************************************************************/
 
 // this is shared between save and restore, and may be useful.

tools/xc/lib/xc_private.h

@@ -248,6 +248,9 @@ void * mfn_mapper_queue_entry(mfn_mapper_t *t, int offset,
 
 /*********************/
 
-int xc_domain_stop_sync( int xc_handle, domid_t dom );
+int xc_domain_stop_sync( int xc_handle, domid_t dom, 
+			 dom0_op_t *op, full_execution_context_t *ctxt );
+
+long long  xc_domain_get_cpu_usage( int xc_handle, domid_t domid );
 
 #endif /* __XC_PRIVATE_H__ */

xen/common/dom0_ops.c

@@ -102,6 +102,20 @@ long do_dom0_op(dom0_op_t *u_dom0_op)
     case DOM0_STOPDOMAIN:
     {
         ret = stop_other_domain(op->u.stopdomain.domain);
+	
+	/* This is grim, but helps for live migrate. It's also unsafe
+	   in the strict sense as we're not explicitly setting a
+	   timeout, but dom0 is bound to have other timers going off to
+	   wake us back up. 
+	   We go to sleep so that the other domain can stop quicker, hence
+	   we have less total down time in a migrate.
+	 */
+	if( ret == 0 && op->u.stopdomain.sync == 1 )
+	{
+	    extern long do_block( void );
+	    printk("T\n");
+	    do_block(); // Yuk...
+	}
     }
     break;
 

xen/common/domain.c

@@ -266,6 +266,16 @@ void stop_domain(void)
     unlazy_fpu(current);
     wmb(); /* All CPUs must see saved info in state TASK_STOPPED. */
     set_current_state(TASK_STOPPED);
+
+    /* OK, this is grim, but helps speed up live migrate. When a domain stops,
+       kick Dom0 */
+    {
+	struct task_struct *p;
+	printk("S\n");
+	guest_schedule_to_run( p = find_domain_by_id(0ULL) );
+	put_task_struct(p);
+    }
+
     __enter_scheduler();
 }
 

xen/common/schedule.c

@@ -27,6 +27,9 @@
 #include <xen/perfc.h>
 #include <xen/sched-if.h>
 #include <hypervisor-ifs/sched_ctl.h>
+
+#undef  TRACE_BUFFER
+
 #include <xen/trace.h>
 
 /*#define WAKEUP_HISTO*/
@@ -216,7 +219,7 @@ void wake_up(struct task_struct *p)
 /* 
  * Block the currently-executing domain until a pertinent event occurs.
  */
-static long do_block(void)
+long do_block(void)
 {
     ASSERT(current->domain != IDLE_DOMAIN_ID);
     current->shared_info->vcpu_data[0].evtchn_upcall_mask = 0;

xen/include/hypervisor-ifs/dom0_ops.h

@@ -74,6 +74,9 @@ typedef struct dom0_stopdomain_st
 {
     /* IN parameters. */
     domid_t domain;
+    /* hack to indicate that you want to wait for other domain -- replace
+       with proper sychronous stop soon! */
+    int     sync;  
 } dom0_stopdomain_t;
 
 #define DOM0_GETDOMAININFO    12