The VMCS control bits in the current tree are not optimal. Attached is

a patch to improve the VMCS control bits setting. It also adds checks to
detect any non-optimal or incompatible settings for the VMCS control
bits based on MSRs, and fixes a bug associated with vmcs region freeing.

Signed-off-by: Nitin A Kamble <nitin.a.kamble@intel.com>
Signed-off-by: Jun Nakajima <jun.nakajima@intel.com>
Signed-off-by: Xin Li <xin.b.li@intel.com>

xen/arch/x86/vmx.c

@@ -262,6 +262,20 @@ extern long evtchn_send(int lport);
 extern long do_block(void);
 void do_nmi(struct cpu_user_regs *, unsigned long);
 
+static int check_vmx_controls(ctrls, msr)
+{   
+    u32 vmx_msr_low, vmx_msr_high; 
+
+    rdmsr(msr, vmx_msr_low, vmx_msr_high);
+    if (ctrls < vmx_msr_low || ctrls > vmx_msr_high) {
+        printk("Insufficient VMX capability 0x%x, "
+               "msr=0x%x,low=0x%8x,high=0x%x\n", 
+               ctrls, msr, vmx_msr_low, vmx_msr_high);
+        return 0;
+    }
+    return 1;
+}
+
 int start_vmx(void)
 {
     struct vmcs_struct *vmcs;
@@ -292,6 +306,19 @@ int start_vmx(void)
               IA32_FEATURE_CONTROL_MSR_ENABLE_VMXON, 0);
     }
 
+    if (!check_vmx_controls(MONITOR_PIN_BASED_EXEC_CONTROLS, 
+            MSR_IA32_VMX_PINBASED_CTLS_MSR))
+        return 0;
+    if (!check_vmx_controls(MONITOR_CPU_BASED_EXEC_CONTROLS, 
+            MSR_IA32_VMX_PROCBASED_CTLS_MSR))
+        return 0;
+    if (!check_vmx_controls(MONITOR_VM_EXIT_CONTROLS, 
+            MSR_IA32_VMX_EXIT_CTLS_MSR))
+        return 0;
+    if (!check_vmx_controls(MONITOR_VM_ENTRY_CONTROLS, 
+            MSR_IA32_VMX_ENTRY_CTLS_MSR))
+        return 0;
+
     set_in_cr4(X86_CR4_VMXE);   /* Enable VMXE */
 
     if (!(vmcs = alloc_vmcs())) {
@@ -965,7 +992,7 @@ static int vmx_set_cr0(unsigned long value)
             set_bit(VMX_CPU_STATE_LMA_ENABLED,
               &d->arch.arch_vmx.cpu_state);
             __vmread(VM_ENTRY_CONTROLS, &vm_entry_value);
-            vm_entry_value |= VM_ENTRY_CONTROLS_IA_32E_MODE;
+            vm_entry_value |= VM_ENTRY_CONTROLS_IA32E_MODE;
             __vmwrite(VM_ENTRY_CONTROLS, vm_entry_value);
 
 #if CONFIG_PAGING_LEVELS >= 4 
@@ -1028,7 +1055,7 @@ static int vmx_set_cr0(unsigned long value)
                 clear_bit(VMX_CPU_STATE_LMA_ENABLED,
                           &d->arch.arch_vmx.cpu_state);
                 __vmread(VM_ENTRY_CONTROLS, &vm_entry_value);
-                vm_entry_value &= ~VM_ENTRY_CONTROLS_IA_32E_MODE;
+                vm_entry_value &= ~VM_ENTRY_CONTROLS_IA32E_MODE;
                 __vmwrite(VM_ENTRY_CONTROLS, vm_entry_value);
             }
         }

xen/arch/x86/vmx_vmcs.c

@@ -55,25 +55,22 @@ void free_vmcs(struct vmcs_struct *vmcs)
 {
     int order;
 
-    order = (vmcs_size >> PAGE_SHIFT) - 1;
+    order = get_order(vmcs_size);
     free_xenheap_pages(vmcs, order);
 }
 
 static inline int construct_vmcs_controls(void)
 {
     int error = 0;
-        
+
     error |= __vmwrite(PIN_BASED_VM_EXEC_CONTROL, 
                        MONITOR_PIN_BASED_EXEC_CONTROLS);
 
     error |= __vmwrite(CPU_BASED_VM_EXEC_CONTROL, 
                        MONITOR_CPU_BASED_EXEC_CONTROLS);
-#if defined (__x86_64__)
-    error |= __vmwrite(VM_EXIT_CONTROLS, 
-      MONITOR_VM_EXIT_CONTROLS | VM_EXIT_CONTROLS_IA_32E_MODE);
-#else
+
     error |= __vmwrite(VM_EXIT_CONTROLS, MONITOR_VM_EXIT_CONTROLS);
-#endif
+
     error |= __vmwrite(VM_ENTRY_CONTROLS, MONITOR_VM_ENTRY_CONTROLS);
 
     return error;

xen/include/asm-x86/msr.h

@@ -94,6 +94,11 @@
 
 /* MSRs & bits used for VMX enabling */
 #define MSR_IA32_VMX_BASIC_MSR                  0x480
+#define MSR_IA32_VMX_PINBASED_CTLS_MSR          0x481
+#define MSR_IA32_VMX_PROCBASED_CTLS_MSR         0x482
+#define MSR_IA32_VMX_EXIT_CTLS_MSR              0x483
+#define MSR_IA32_VMX_ENTRY_CTLS_MSR             0x484
+#define MSR_IA32_VMX_MISC_MSR                   0x485
 #define IA32_FEATURE_CONTROL_MSR                0x3a
 #define IA32_FEATURE_CONTROL_MSR_LOCK           0x1
 #define IA32_FEATURE_CONTROL_MSR_ENABLE_VMXON   0x4

xen/include/asm-x86/vmx.h

@@ -46,32 +46,61 @@ extern unsigned int cpu_rev;
 #define MONITOR_PIN_BASED_EXEC_CONTROLS_RESERVED_VALUE         0x00000016
 
 #define MONITOR_PIN_BASED_EXEC_CONTROLS       \
+    ( \
     MONITOR_PIN_BASED_EXEC_CONTROLS_RESERVED_VALUE |   \
     PIN_BASED_EXT_INTR_MASK |   \
-    PIN_BASED_NMI_EXITING
+    PIN_BASED_NMI_EXITING \
+    )
 
 #define MONITOR_CPU_BASED_EXEC_CONTROLS_RESERVED_VALUE         0x0401e172
 
-#define MONITOR_CPU_BASED_EXEC_CONTROLS \
+#define _MONITOR_CPU_BASED_EXEC_CONTROLS \
+    ( \
     MONITOR_CPU_BASED_EXEC_CONTROLS_RESERVED_VALUE |    \
     CPU_BASED_HLT_EXITING | \
     CPU_BASED_INVDPG_EXITING | \
     CPU_BASED_MWAIT_EXITING | \
     CPU_BASED_MOV_DR_EXITING | \
-    CPU_BASED_UNCOND_IO_EXITING | \
+    CPU_BASED_UNCOND_IO_EXITING \
+    )
+
+#define MONITOR_CPU_BASED_EXEC_CONTROLS_IA32E_MODE \
+    ( \
     CPU_BASED_CR8_LOAD_EXITING | \
-    CPU_BASED_CR8_STORE_EXITING
+    CPU_BASED_CR8_STORE_EXITING \
+    )
 
-#define MONITOR_VM_EXIT_CONTROLS_RESERVED_VALUE                0x0003edff
+#define MONITOR_VM_EXIT_CONTROLS_RESERVED_VALUE   0x0003edff
 
-#define VM_EXIT_CONTROLS_IA_32E_MODE		0x00000200
+#define MONITOR_VM_EXIT_CONTROLS_IA32E_MODE       0x00000200
 
-#define MONITOR_VM_EXIT_CONTROLS                \
+#define _MONITOR_VM_EXIT_CONTROLS                \
+    ( \
     MONITOR_VM_EXIT_CONTROLS_RESERVED_VALUE |\
-    VM_EXIT_ACK_INTR_ON_EXIT
+    VM_EXIT_ACK_INTR_ON_EXIT \
+    )
+
+#if defined (__x86_64__)
+#define MONITOR_CPU_BASED_EXEC_CONTROLS \
+    ( \
+    _MONITOR_CPU_BASED_EXEC_CONTROLS | \
+    MONITOR_CPU_BASED_EXEC_CONTROLS_IA32E_MODE \
+    )
+#define MONITOR_VM_EXIT_CONTROLS \
+    ( \
+    _MONITOR_VM_EXIT_CONTROLS | \
+    MONITOR_VM_EXIT_CONTROLS_IA32E_MODE  \
+    )
+#else
+#define MONITOR_CPU_BASED_EXEC_CONTROLS \
+    _MONITOR_CPU_BASED_EXEC_CONTROLS 
+
+#define MONITOR_VM_EXIT_CONTROLS \
+    _MONITOR_VM_EXIT_CONTROLS
+#endif
 
 #define VM_ENTRY_CONTROLS_RESERVED_VALUE        0x000011ff
-#define VM_ENTRY_CONTROLS_IA_32E_MODE           0x00000200
+#define VM_ENTRY_CONTROLS_IA32E_MODE            0x00000200
 #define MONITOR_VM_ENTRY_CONTROLS       VM_ENTRY_CONTROLS_RESERVED_VALUE 
 /*
  * Exit Reasons