x2APIC: Improve x2APIC suspend/resume

x2apic depends on interrupt remapping, so it should disable interrupt
remapping behind x2apic disabling. And also this patch wraps
__enable_x2apic to get rid of duplicated code.

Signed-off-by: Weidong Han <weidong.han@intel.com>

xen/arch/x86/apic.c

@@ -492,9 +492,21 @@ static void apic_pm_activate(void)
     apic_pm_state.active = 1;
 }
 
-static void resume_x2apic(void)
+static void __enable_x2apic(void)
 {
     uint64_t msr_content;
+
+    rdmsrl(MSR_IA32_APICBASE, msr_content);
+    if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
+    {
+        msr_content |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
+        msr_content = (uint32_t)msr_content;
+        wrmsrl(MSR_IA32_APICBASE, msr_content);
+    }
+}
+
+static void resume_x2apic(void)
+{
     struct IO_APIC_route_entry **ioapic_entries = NULL;
 
     ASSERT(x2apic_enabled);
@@ -516,14 +528,7 @@ static void resume_x2apic(void)
     mask_IO_APIC_setup(ioapic_entries);
 
     iommu_enable_IR();
-
-    rdmsrl(MSR_IA32_APICBASE, msr_content);
-    if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
-    {
-        msr_content |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
-        msr_content = (uint32_t)msr_content;
-        wrmsrl(MSR_IA32_APICBASE, msr_content);
-    }
+    __enable_x2apic();
 
     restore_IO_APIC_setup(ioapic_entries);
     unmask_8259A();
@@ -739,9 +744,10 @@ int lapic_suspend(void)
     apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
     apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
     apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
-    
+
     local_irq_save(flags);
     disable_local_APIC();
+    iommu_disable_IR();
     local_irq_restore(flags);
     return 0;
 }
@@ -1041,16 +1047,8 @@ static void enable_bsp_x2apic(void)
 
     if ( !x2apic_preenabled )
     {
-        uint64_t msr_content;
-        rdmsrl(MSR_IA32_APICBASE, msr_content);
-        if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
-        {
-            msr_content |= MSR_IA32_APICBASE_ENABLE |
-                           MSR_IA32_APICBASE_EXTD;
-            msr_content = (uint32_t)msr_content;
-            wrmsrl(MSR_IA32_APICBASE, msr_content);
-            printk("x2APIC mode enabled.\n");
-        }
+        __enable_x2apic();
+        printk("x2APIC mode enabled.\n");
     }
 
 restore_out:
@@ -1064,20 +1062,12 @@ out:
 
 static void enable_ap_x2apic(void)
 {
-    uint64_t msr_content;
-
     ASSERT(smp_processor_id() != 0);
 
     /* APs only enable x2apic when BSP did so. */
     BUG_ON(!x2apic_enabled);
 
-    rdmsrl(MSR_IA32_APICBASE, msr_content);
-    if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
-    {
-        msr_content |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
-        msr_content = (uint32_t)msr_content;
-        wrmsrl(MSR_IA32_APICBASE, msr_content);
-    }
+    __enable_x2apic();
 }
 
 void enable_x2apic(void)

xen/drivers/passthrough/vtd/intremap.c

@@ -872,6 +872,24 @@ int iommu_enable_IR(void)
     return 0;
 }
 
+/*
+ * This function is used to disable Interrutp remapping when
+ * suspend local apic
+ */
+void iommu_disable_IR(void)
+{
+    struct acpi_drhd_unit *drhd;
+
+    if ( !iommu_supports_eim() )
+        return;
+
+    for_each_drhd_unit ( drhd )
+        disable_intremap(drhd->iommu);
+
+    for_each_drhd_unit ( drhd )
+        disable_qinval(drhd->iommu);
+}
+
 /*
  * Check if interrupt remapping is enabled or not
  * return 1: enabled

xen/drivers/passthrough/vtd/iommu.c

@@ -2127,10 +2127,11 @@ static void vtd_suspend(void)
 
         iommu_disable_translation(iommu);
 
-        if ( iommu_intremap )
-            disable_intremap(iommu);
-
-        if ( iommu_qinval )
+        /* If interrupt remapping is enabled, queued invalidation
+         * will be disabled following interupt remapping disabling
+         * in local apic suspend
+         */
+        if ( !iommu_intremap && iommu_qinval )
             disable_qinval(iommu);
     }
 }

xen/include/xen/iommu.h

@@ -59,6 +59,7 @@ struct iommu {
 int iommu_setup(void);
 int iommu_supports_eim(void);
 int iommu_enable_IR(void);
+void iommu_disable_IR(void);
 int intremap_enabled(void);
 
 int iommu_add_device(struct pci_dev *pdev);