bitkeeper revision 1.1159.57.1 (412f374873CWRyXQT5wTVtsSUl961g)

Use hypercall batching where possible.

linux-2.6.8.1-xen-sparse/arch/xen/i386/kernel/ldt.c

@@ -22,8 +22,10 @@
 #ifdef CONFIG_SMP /* avoids "defined but not used" warnig */
 static void flush_ldt(void *null)
 {
-	if (current->active_mm)
+	if (current->active_mm) {
 		load_LDT(&current->active_mm->context);
+		flush_page_update_queue();
+	}
 }
 #endif
 
@@ -90,6 +92,7 @@ static inline int copy_ldt(mm_context_t *new, mm_context_t *old)
 	memcpy(new->ldt, old->ldt, old->size*LDT_ENTRY_SIZE);
 	make_pages_readonly(new->ldt, (new->size * LDT_ENTRY_SIZE) /
 			    PAGE_SIZE);
+	flush_page_update_queue();
 	return 0;
 }
 

linux-2.6.8.1-xen-sparse/arch/xen/i386/kernel/traps.c

@@ -967,6 +967,7 @@ void __init trap_init(void)
 	set_call_gate(&default_ldt[0],lcall7);
 	set_call_gate(&default_ldt[4],lcall27);
 	__make_page_readonly(&default_ldt[0]);
+	xen_flush_page_update_queue();
 
 	/*
 	 * Should be a barrier for any external CPU state.

linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/hypervisor.c

@@ -28,7 +28,7 @@ static spinlock_t update_lock = SPIN_LOCK_UNLOCKED;
 #define QUEUE_SIZE 2048
 #define pte_offset_kernel pte_offset
 #else
-#define QUEUE_SIZE 1
+#define QUEUE_SIZE 128
 #endif
 
 static mmu_update_t update_queue[QUEUE_SIZE];
@@ -152,6 +152,12 @@ static inline void increment_index(void)
     if ( unlikely(idx == QUEUE_SIZE) ) __flush_page_update_queue();
 }
 
+static inline void increment_index_and_flush(void)
+{
+    idx++;
+    __flush_page_update_queue();
+}
+
 void queue_l1_entry_update(pte_t *ptr, unsigned long val)
 {
     unsigned long flags;
@@ -271,6 +277,126 @@ void queue_machphys_update(unsigned long mfn, unsigned long pfn)
     spin_unlock_irqrestore(&update_lock, flags);
 }
 
+/* queue and flush versions of the above */
+void xen_l1_entry_update(pte_t *ptr, unsigned long val)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+#if MMU_UPDATE_DEBUG > 3
+    DEBUG_disallow_pt_read((unsigned long)ptr);
+#endif
+    update_queue[idx].ptr = virt_to_machine(ptr);
+    update_queue[idx].val = val;
+    increment_index_and_flush();
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_l2_entry_update(pmd_t *ptr, unsigned long val)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+    update_queue[idx].ptr = virt_to_machine(ptr);
+    update_queue[idx].val = val;
+    increment_index_and_flush();
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_pt_switch(unsigned long ptr)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+    update_queue[idx].ptr  = phys_to_machine(ptr);
+    update_queue[idx].ptr |= MMU_EXTENDED_COMMAND;
+    update_queue[idx].val  = MMUEXT_NEW_BASEPTR;
+    increment_index_and_flush();
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_tlb_flush(void)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+    update_queue[idx].ptr  = MMU_EXTENDED_COMMAND;
+    update_queue[idx].val  = MMUEXT_TLB_FLUSH;
+    increment_index_and_flush();
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_invlpg(unsigned long ptr)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+    update_queue[idx].ptr  = MMU_EXTENDED_COMMAND;
+    update_queue[idx].ptr |= ptr & PAGE_MASK;
+    update_queue[idx].val  = MMUEXT_INVLPG;
+    increment_index_and_flush();
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_pgd_pin(unsigned long ptr)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+    update_queue[idx].ptr  = phys_to_machine(ptr);
+    update_queue[idx].ptr |= MMU_EXTENDED_COMMAND;
+    update_queue[idx].val  = MMUEXT_PIN_L2_TABLE;
+    increment_index_and_flush();
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_pgd_unpin(unsigned long ptr)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+    update_queue[idx].ptr  = phys_to_machine(ptr);
+    update_queue[idx].ptr |= MMU_EXTENDED_COMMAND;
+    update_queue[idx].val  = MMUEXT_UNPIN_TABLE;
+    increment_index_and_flush();
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_pte_pin(unsigned long ptr)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+    update_queue[idx].ptr  = phys_to_machine(ptr);
+    update_queue[idx].ptr |= MMU_EXTENDED_COMMAND;
+    update_queue[idx].val  = MMUEXT_PIN_L1_TABLE;
+    increment_index_and_flush();
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_pte_unpin(unsigned long ptr)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+    update_queue[idx].ptr  = phys_to_machine(ptr);
+    update_queue[idx].ptr |= MMU_EXTENDED_COMMAND;
+    update_queue[idx].val  = MMUEXT_UNPIN_TABLE;
+    increment_index_and_flush();
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_set_ldt(unsigned long ptr, unsigned long len)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+    update_queue[idx].ptr  = MMU_EXTENDED_COMMAND | ptr;
+    update_queue[idx].val  = MMUEXT_SET_LDT | (len << MMUEXT_CMD_SHIFT);
+    increment_index_and_flush();
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_machphys_update(unsigned long mfn, unsigned long pfn)
+{
+    unsigned long flags;
+    spin_lock_irqsave(&update_lock, flags);
+    update_queue[idx].ptr = (mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
+    update_queue[idx].val = pfn;
+    increment_index_and_flush();
+    spin_unlock_irqrestore(&update_lock, flags);
+}
+
 #ifdef CONFIG_XEN_PHYSDEV_ACCESS
 
 unsigned long allocate_empty_lowmem_region(unsigned long pages)

linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/init.c

@@ -125,6 +125,7 @@ static void __init page_table_range_init (unsigned long start, unsigned long end
 	}
 }
 
+/* NOTE: caller must call flush_page_update_queue() */
 void __init wrprotect_bootpt(pgd_t *pgd, void *page, int set)
 {
 	pmd_t *pmd;
@@ -141,6 +142,7 @@ void __init wrprotect_bootpt(pgd_t *pgd, void *page, int set)
 	    pte_val_ma(*pte) | _PAGE_RW);
 }
 
+/* NOTE: caller must call flush_page_update_queue() */
 static void __init protect_bootpt_entries(pgd_t *spgd, pgd_t *dpgd, int set,
     int pmdupdate, int pmdset)
 {
@@ -163,7 +165,6 @@ static void __init protect_bootpt_entries(pgd_t *spgd, pgd_t *dpgd, int set,
 			}
 		}
 	}
-	flush_page_update_queue();
 }
 
 static inline int is_kernel_text(unsigned long addr)
@@ -575,14 +576,13 @@ void __init paging_init(void)
 	protect_bootpt_entries((pgd_t *)start_info.pt_base, swapper_pg_dir,
 	    1, 1, 1);
 	queue_pgd_pin(__pa(swapper_pg_dir));
-	flush_page_update_queue();
 	load_cr3(swapper_pg_dir);
-	__flush_tlb_all();
+	__flush_tlb_all(); /* implicit flush */
 	queue_pgd_unpin(__pa(start_info.pt_base));
-	flush_page_update_queue();
 	protect_bootpt_entries((pgd_t *)start_info.pt_base, swapper_pg_dir,
 	    0, 1, 0);
 	wrprotect_bootpt((pgd_t *)start_info.pt_base, swapper_pg_dir, 0);
+	flush_page_update_queue();
 
 #ifdef CONFIG_X86_PAE
 	/*

linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/pgtable.c

@@ -180,6 +180,7 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
 	if (pte) {
 		clear_page(pte);
 		__make_page_readonly(pte);
+		xen_flush_page_update_queue();
 	}
 	return pte;
 }
@@ -204,7 +205,10 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
 #ifdef CONFIG_HIGHPTE
 		if (pte < highmem_start_page)
 #endif
-		__make_page_readonly(phys_to_virt(page_to_pseudophys(pte)));
+		{
+			__make_page_readonly(phys_to_virt(page_to_pseudophys(pte)));
+			flush_page_update_queue();
+		}
 	}
 	return pte;
 }

linux-2.6.8.1-xen-sparse/drivers/xen/blkfront/blkfront.c

@@ -1263,7 +1263,7 @@ void blkif_completion(blkif_request_t *req)
 	{
 	    unsigned long pfn = req->frame_and_sects[i] >> PAGE_SHIFT;
 	    unsigned long mfn = phys_to_machine_mapping[pfn];
-	    queue_machphys_update(mfn, pfn);
+	    xen_machphys_update(mfn, pfn);
 	}
 	break;
     }

linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/mmu_context.h

@@ -39,7 +39,7 @@ static inline void switch_mm(struct mm_struct *prev,
 		cpu_set(cpu, next->cpu_vm_mask);
 
 		/* Re-load page tables */
-		load_cr3_noflush(next->pgd);
+		load_cr3(next->pgd);
 
 		/*
 		 * load the LDT, if the LDT is different:
@@ -56,7 +56,7 @@ static inline void switch_mm(struct mm_struct *prev,
 			/* We were in lazy tlb mode and leave_mm disabled 
 			 * tlb flush IPI delivery. We must reload %cr3.
 			 */
-			load_cr3_noflush(next->pgd);
+			load_cr3(next->pgd);
 			load_LDT_nolock(&next->context, cpu);
 		}
 	}

linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgalloc.h

@@ -33,6 +33,7 @@ static inline void pte_free_kernel(pte_t *pte)
 {
 	free_page((unsigned long)pte);
 	__make_page_writable(pte);
+	flush_page_update_queue();
 }
 
 static inline void pte_free(struct page *pte)
@@ -43,6 +44,7 @@ static inline void pte_free(struct page *pte)
 	{
 		__make_page_writable(phys_to_virt(page_to_pseudophys(pte)));
 		__free_page(pte);
+		flush_page_update_queue();
 	}
 }
 

linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgtable-2level.h

@@ -27,14 +27,14 @@ static inline int pgd_present(pgd_t pgd)	{ return 1; }
 #define set_pte(pteptr, pteval) (*(pteptr) = pteval)
 #define set_pte_atomic(pteptr, pteval) (*(pteptr) = pteval)
 #else
-#define set_pte(pteptr, pteval) queue_l1_entry_update(pteptr, (pteval).pte_low)
-#define set_pte_atomic(pteptr, pteval) queue_l1_entry_update(pteptr, (pteval).pte_low)
+#define set_pte(pteptr, pteval) xen_l1_entry_update(pteptr, (pteval).pte_low)
+#define set_pte_atomic(pteptr, pteval) xen_l1_entry_update(pteptr, (pteval).pte_low)
 #endif
 /*
  * (pmds are folded into pgds so this doesn't get actually called,
  * but the define is needed for a generic inline function.)
  */
-#define set_pmd(pmdptr, pmdval) queue_l2_entry_update((pmdptr), (pmdval).pmd)
+#define set_pmd(pmdptr, pmdval) xen_l2_entry_update((pmdptr), (pmdval).pmd)
 #define set_pgd(pgdptr, pgdval) ((void)0)
 
 #define pgd_page(pgd) \

linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgtable.h

@@ -252,7 +252,7 @@ static inline int ptep_test_and_clear_dirty(pte_t *ptep)
 	pte_t pte = *ptep;
 	int ret = pte_dirty(pte);
 	if (ret)
-		queue_l1_entry_update(ptep, pte_mkclean(pte).pte_low);
+		xen_l1_entry_update(ptep, pte_mkclean(pte).pte_low);
 	return ret;
 }
 
@@ -261,7 +261,7 @@ static inline int ptep_test_and_clear_young(pte_t *ptep)
 	pte_t pte = *ptep;
 	int ret = pte_young(pte);
 	if (ret)
-		queue_l1_entry_update(ptep, pte_mkold(pte).pte_low);
+		xen_l1_entry_update(ptep, pte_mkold(pte).pte_low);
 	return ret;
 }
 
@@ -275,7 +275,7 @@ static inline void ptep_mkdirty(pte_t *ptep)
 {
 	pte_t pte = *ptep;
 	if (!pte_dirty(pte))
-		queue_l1_entry_update(ptep, pte_mkdirty(pte).pte_low);
+		xen_l1_entry_update(ptep, pte_mkdirty(pte).pte_low);
 }
 
 /*
@@ -318,7 +318,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 	pmd_t p = *(xp);					\
 	set_pmd(xp, __pmd(0));					\
 	__make_page_writable((void *)pmd_page_kernel(p));	\
-	/* XXXcl queue */ \
+	xen_flush_page_update_queue();				\
 } while (0)
 
 #ifndef CONFIG_DISCONTIGMEM
@@ -418,12 +418,13 @@ extern pte_t *lookup_address(unsigned long address);
 #define update_mmu_cache(vma,address,pte) do { } while (0)
 #define  __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
 
-#if 1
+#if 0
 #define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
 	do {								  \
 		if (__dirty) {						  \
 			queue_l1_entry_update((__ptep), (__entry).pte_low); \
-			flush_tlb_page(__vma, __address);		  \
+			flush_tlb_page(__vma, __address);                 \
+			xen_flush_page_update_queue();                    \
 		}							  \
 	} while (0)
 #else
@@ -434,8 +435,7 @@ extern pte_t *lookup_address(unsigned long address);
 			    xen_flush_page_update_queue();                \
 			    HYPERVISOR_update_va_mapping(address>>PAGE_SHIFT, entry, UVMF_INVLPG); \
 			} else {                                          \
-                            queue_l1_entry_update((__ptep), (__entry).pte_low); \
-			    xen_flush_page_update_queue();                \
+                            xen_l1_entry_update((__ptep), (__entry).pte_low); \
 			}                                                 \
 		}							  \
 	} while (0)
@@ -449,6 +449,7 @@ extern pte_t *lookup_address(unsigned long address);
 #define __pte_to_swp_entry(pte)		((swp_entry_t) { (pte).pte_low })
 #define __swp_entry_to_pte(x)		((pte_t) { (x).val })
 
+/* NOTE: make_page* callers must call flush_page_update_queue() */
 static inline void __make_page_readonly(void *va)
 {
 	pgd_t *pgd = pgd_offset_k((unsigned long)va);
@@ -474,7 +475,6 @@ static inline void make_page_readonly(void *va)
 	if ( (unsigned long)va >= VMALLOC_START )
 		__make_page_readonly(machine_to_virt(
 			*(unsigned long *)pte&PAGE_MASK));
-	/* XXXcl queue */
 }
 
 static inline void make_page_writable(void *va)
@@ -486,7 +486,6 @@ static inline void make_page_writable(void *va)
 	if ( (unsigned long)va >= VMALLOC_START )
 		__make_page_writable(machine_to_virt(
 			*(unsigned long *)pte&PAGE_MASK));
-	/* XXXcl queue */
 }
 
 static inline void make_pages_readonly(void *va, unsigned int nr)
@@ -496,7 +495,6 @@ static inline void make_pages_readonly(void *va, unsigned int nr)
 		make_page_readonly(va);
 		va = (void *)((unsigned long)va + PAGE_SIZE);
 	}
-	/* XXXcl queue */
 }
 
 static inline void make_pages_writable(void *va, unsigned int nr)
@@ -506,7 +504,6 @@ static inline void make_pages_writable(void *va, unsigned int nr)
 		make_page_writable(va);
 		va = (void *)((unsigned long)va + PAGE_SIZE);
 	}
-	/* XXXcl queue */
 }
 
 static inline unsigned long arbitrary_virt_to_phys(void *va)

linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/processor.h

@@ -181,12 +181,6 @@ static inline unsigned int cpuid_edx(unsigned int op)
 }
 
 #define load_cr3(pgdir) do {				\
-	queue_pt_switch(__pa(pgdir));			\
-	flush_page_update_queue();			\
-	cur_pgd = pgdir;	/* XXXsmp */		\
-} while (/* CONSTCOND */0)
-
-#define load_cr3_noflush(pgdir) do {			\
 	queue_pt_switch(__pa(pgdir));			\
 	cur_pgd = pgdir;	/* XXXsmp */		\
 } while (/* CONSTCOND */0)

linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/tlbflush.h

@@ -6,8 +6,7 @@
 #include <asm/processor.h>
 
 #define __flush_tlb() do {						\
-	queue_tlb_flush();						\
-	xen_flush_page_update_queue();					\
+	xen_tlb_flush();						\
 } while (/*CONSTCOND*/0)
 
 /*
@@ -16,8 +15,7 @@
  */
 #define __flush_tlb_global()						\
 	do {								\
-		queue_tlb_flush();					\
-		xen_flush_page_update_queue();				\
+		xen_tlb_flush();					\
 	} while (0)
 
 extern unsigned long pgkern_mask;
@@ -33,8 +31,7 @@ extern unsigned long pgkern_mask;
 #define cpu_has_invlpg	(boot_cpu_data.x86 > 3)
 
 #define __flush_tlb_single(addr) do {					\
-	queue_invlpg(addr);						\
-	xen_flush_page_update_queue();					\
+	xen_invlpg(addr);						\
 } while (/* CONSTCOND */0)
 
 # define __flush_tlb_one(addr) __flush_tlb_single(addr)

linux-2.6.8.1-xen-sparse/include/asm-xen/hypervisor.h

@@ -58,6 +58,17 @@ void queue_pte_pin(unsigned long ptr);
 void queue_pte_unpin(unsigned long ptr);
 void queue_set_ldt(unsigned long ptr, unsigned long bytes);
 void queue_machphys_update(unsigned long mfn, unsigned long pfn);
+void xen_l1_entry_update(pte_t *ptr, unsigned long val);
+void xen_l2_entry_update(pmd_t *ptr, unsigned long val);
+void xen_pt_switch(unsigned long ptr);
+void xen_tlb_flush(void);
+void xen_invlpg(unsigned long ptr);
+void xen_pgd_pin(unsigned long ptr);
+void xen_pgd_unpin(unsigned long ptr);
+void xen_pte_pin(unsigned long ptr);
+void xen_pte_unpin(unsigned long ptr);
+void xen_set_ldt(unsigned long ptr, unsigned long bytes);
+void xen_machphys_update(unsigned long mfn, unsigned long pfn);
 #define MMU_UPDATE_DEBUG 0
 
 #if MMU_UPDATE_DEBUG > 0