x86/shstk: Fix possible crash while enabling Shadow Stacks

Between writing MSR_S_CET.SHSTK_EN and establishing SSP using RSTORSSP or
SETSSBSY, there must not be any CALL or RET instructions.

Building Xen with -fno-early-inlining causes wrmsr() to become out-of-line,
and thus suffers #DF during boot.

Make the MSR and shstk helpers always_inline.  They ought to have been this
way all along.

Fixes: b60ab42db2 ("x86/shstk: Activate Supervisor Shadow Stacks")
Fixes: 3037d8b7eb ("x86/boot: Use RSTORSSP to establish SSP")
Reported-by: Stefano Stabellini <sstabellini@kernel.org>
Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Teddy Astie <teddy.astie@vates.tech>
Reviewed-by: Roger Pau Monné <roger@xenproject.org>

xen/arch/x86/include/asm/msr.h

@@ -47,7 +47,7 @@ static always_inline uint64_t rdmsr(unsigned int msr)
        val = a__ | ((u64)b__<<32); \
 } while(0)
 
-static inline void wrmsr(unsigned int msr, uint64_t val)
+static always_inline void wrmsr(unsigned int msr, uint64_t val)
 {
     uint32_t lo = val, hi = val >> 32;
 
@@ -56,7 +56,7 @@ static inline void wrmsr(unsigned int msr, uint64_t val)
 #define wrmsrl(msr, val) wrmsr(msr, val)
 
 /* Non-serialising WRMSR, when available.  Falls back to a serialising WRMSR. */
-static inline void wrmsrns(uint32_t msr, uint64_t val)
+static always_inline void wrmsrns(uint32_t msr, uint64_t val)
 {
     uint32_t lo = val, hi = val >> 32;
 
@@ -72,7 +72,7 @@ static inline void wrmsrns(uint32_t msr, uint64_t val)
 }
 
 /* rdmsr with exception handling */
-static inline int rdmsr_safe(unsigned int msr, uint64_t *val)
+static always_inline int rdmsr_safe(unsigned int msr, uint64_t *val)
 {
     uint64_t lo, hi;
 
@@ -113,7 +113,7 @@ static inline int rdmsr_safe(unsigned int msr, uint64_t *val)
 }
 
 /* wrmsr with exception handling */
-static inline int wrmsr_safe(unsigned int msr, uint64_t val)
+static always_inline int wrmsr_safe(unsigned int msr, uint64_t val)
 {
     uint32_t lo = val, hi = val >> 32;
 

xen/arch/x86/include/asm/shstk.h

@@ -15,7 +15,7 @@
  */
 #define SSP_NO_SHSTK 1
 
-static inline unsigned long rdssp(void)
+static always_inline unsigned long rdssp(void)
 {
     unsigned long ssp;
 
@@ -24,7 +24,7 @@ static inline unsigned long rdssp(void)
     return ssp;
 }
 
-static inline void wrss(unsigned long val, unsigned long *ptr)
+static always_inline void wrss(unsigned long val, unsigned long *ptr)
 {
     asm ( "wrssq %[val], %[ptr]"
           : [ptr] "=m" (*ptr)

xen/arch/x86/setup.c

@@ -915,6 +915,11 @@ static void __init noreturn reinit_bsp_stack(void)
          */
         local_irq_disable();
 
+        /*
+         * There must be no CALL or RET instructions between writing
+         * MSR_S_CET.SHSTK_EN and establishing SSP with RSTORSSP or SETSSBSY
+         * below.  All helpers must be forced inline.
+         */
         wrmsrl(MSR_S_CET, xen_msr_s_cet_value());
 
         /*