[PAE] Fix support for pgdirs above 4GB. Requires an interface change so that %cr3

is extended to include high-order address bits at bottom of %cr3 value.
Guests who understand this interface change publish the fact by setting the
option 'PAE=yes[extended-cr3]' in their __xen_guest section.
Signed-off-by: Keir Fraser <keir@xensource.com>

linux-2.6-xen-sparse/arch/i386/kernel/head-xen.S

@@ -173,7 +173,7 @@ ENTRY(cpu_gdt_table)
 	.ascii	         "|pae_pgdir_above_4gb"
 	.ascii	         "|supervisor_mode_kernel"
 #ifdef CONFIG_X86_PAE
-	.ascii	",PAE=yes"
+	.ascii	",PAE=yes[extended-cr3]"
 #else
 	.ascii	",PAE=no"
 #endif

linux-2.6-xen-sparse/drivers/xen/core/smpboot.c

@@ -208,7 +208,7 @@ void cpu_initialize_context(unsigned int cpu)
 	ctxt.failsafe_callback_cs  = __KERNEL_CS;
 	ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
 
-	ctxt.ctrlreg[3] = virt_to_mfn(swapper_pg_dir) << PAGE_SHIFT;
+	ctxt.ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));
 #else /* __x86_64__ */
 	ctxt.user_regs.cs = __KERNEL_CS;
 	ctxt.user_regs.esp = idle->thread.rsp0 - sizeof(struct pt_regs);
@@ -220,7 +220,7 @@ void cpu_initialize_context(unsigned int cpu)
 	ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
 	ctxt.syscall_callback_eip  = (unsigned long)system_call;
 
-	ctxt.ctrlreg[3] = virt_to_mfn(init_level4_pgt) << PAGE_SHIFT;
+	ctxt.ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(init_level4_pgt));
 
 	ctxt.gs_base_kernel = (unsigned long)(cpu_pda(cpu));
 #endif

linux-2.6-xen-sparse/include/asm-i386/mach-xen/asm/system.h

@@ -116,10 +116,12 @@ __asm__ __volatile__ ("movw %%dx,%1\n\t" \
 	__asm__ ( \
 		"movl %%cr3,%0\n\t" \
 		:"=r" (__dummy)); \
-	machine_to_phys(__dummy); \
+	__dummy = xen_cr3_to_pfn(__dummy); \
+	mfn_to_pfn(__dummy) << PAGE_SHIFT; \
 })
 #define write_cr3(x) ({						\
-	maddr_t __dummy = phys_to_machine(x);			\
+	unsigned int __dummy = pfn_to_mfn((x) >> PAGE_SHIFT);	\
+	__dummy = xen_pfn_to_cr3(__dummy);			\
 	__asm__ __volatile__("movl %0,%%cr3": :"r" (__dummy));	\
 })
 

xen/arch/x86/dom0_ops.c

@@ -467,7 +467,7 @@ void arch_getdomaininfo_ctxt(
     if ( hvm_guest(v) )
         c->flags |= VGCF_HVM_GUEST;
 
-    c->ctrlreg[3] = pagetable_get_paddr(v->arch.guest_table);
+    c->ctrlreg[3] = xen_pfn_to_cr3(pagetable_get_pfn(v->arch.guest_table));
 
     c->vm_assist = v->domain->vm_assist;
 }

xen/arch/x86/domain.c

@@ -259,7 +259,7 @@ int arch_set_info_guest(
     struct vcpu *v, struct vcpu_guest_context *c)
 {
     struct domain *d = v->domain;
-    unsigned long phys_basetab = INVALID_MFN;
+    unsigned long cr3_pfn;
     int i, rc;
 
     if ( !(c->flags & VGCF_HVM_GUEST) )
@@ -322,12 +322,8 @@ int arch_set_info_guest(
 
     if ( !(c->flags & VGCF_HVM_GUEST) )
     {
-        phys_basetab = c->ctrlreg[3];
-        phys_basetab =
-            (gmfn_to_mfn(d, phys_basetab >> PAGE_SHIFT) << PAGE_SHIFT) |
-            (phys_basetab & ~PAGE_MASK);
-
-        v->arch.guest_table = pagetable_from_paddr(phys_basetab);
+        cr3_pfn = gmfn_to_mfn(d, xen_cr3_to_pfn(c->ctrlreg[3]));
+        v->arch.guest_table = pagetable_from_pfn(cr3_pfn);
     }
 
     if ( (rc = (int)set_gdt(v, c->gdt_frames, c->gdt_ents)) != 0 )
@@ -342,7 +338,7 @@ int arch_set_info_guest(
     }
     else if ( shadow_mode_refcounts(d) )
     {
-        if ( !get_page(mfn_to_page(phys_basetab>>PAGE_SHIFT), d) )
+        if ( !get_page(mfn_to_page(cr3_pfn), d) )
         {
             destroy_gdt(v);
             return -EINVAL;
@@ -350,7 +346,7 @@ int arch_set_info_guest(
     }
     else
     {
-        if ( !get_page_and_type(mfn_to_page(phys_basetab>>PAGE_SHIFT), d,
+        if ( !get_page_and_type(mfn_to_page(cr3_pfn), d,
                                 PGT_base_page_table) )
         {
             destroy_gdt(v);

xen/arch/x86/traps.c

@@ -1034,8 +1034,8 @@ static int emulate_privileged_op(struct cpu_user_regs *regs)
             break;
             
         case 3: /* Read CR3 */
-            *reg = pfn_to_paddr(mfn_to_gmfn(v->domain,
-                                    pagetable_get_pfn(v->arch.guest_table)));
+            *reg = xen_pfn_to_cr3(mfn_to_gmfn(
+                v->domain, pagetable_get_pfn(v->arch.guest_table)));
             break;
 
         case 4: /* Read CR4 */
@@ -1085,7 +1085,7 @@ static int emulate_privileged_op(struct cpu_user_regs *regs)
         case 3: /* Write CR3 */
             LOCK_BIGLOCK(v->domain);
             cleanup_writable_pagetable(v->domain);
-            (void)new_guest_cr3(gmfn_to_mfn(v->domain, paddr_to_pfn(*reg)));
+            (void)new_guest_cr3(gmfn_to_mfn(v->domain, xen_cr3_to_pfn(*reg)));
             UNLOCK_BIGLOCK(v->domain);
             break;
 

xen/include/public/arch-x86_32.h

@@ -158,6 +158,14 @@ struct vcpu_guest_context {
 typedef struct vcpu_guest_context vcpu_guest_context_t;
 DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t);
 
+/*
+ * Page-directory addresses above 4GB do not fit into architectural %cr3.
+ * When accessing %cr3, or equivalent field in vcpu_guest_context, guests
+ * must use the following accessor macros to pack/unpack valid MFNs.
+ */
+#define xen_pfn_to_cr3(pfn) (((unsigned)(pfn) << 12) | ((unsigned)(pfn) >> 20))
+#define xen_cr3_to_pfn(cr3) (((unsigned)(cr3) >> 12) | ((unsigned)(cr3) << 20))
+
 struct arch_shared_info {
     unsigned long max_pfn;                  /* max pfn that appears in table */
     /* Frame containing list of mfns containing list of mfns containing p2m. */

xen/include/public/arch-x86_64.h

@@ -234,6 +234,9 @@ struct vcpu_guest_context {
 typedef struct vcpu_guest_context vcpu_guest_context_t;
 DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t);
 
+#define xen_pfn_to_cr3(pfn) ((unsigned long)(pfn) << 12)
+#define xen_cr3_to_pfn(cr3) ((unsigned long)(cr3) >> 12)
+
 struct arch_shared_info {
     unsigned long max_pfn;                  /* max pfn that appears in table */
     /* Frame containing list of mfns containing list of mfns containing p2m. */