x86/32on64: misc adjustments to call gate emulation

- There's no 32-bit displacement in 16-bit addressing mode.
- It is wrong to ASSERT() anything on parts of an instruction fetched
  from guest memory.
- The two scaling bits of a SIB byte don't affect whether there is a
  scaled index register or not.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
master commit: ee1cc4bfdc
master date: 2016-09-01 15:23:46 +0200

xen/arch/x86/traps.c

@@ -2839,7 +2839,7 @@ static void emulate_gate_op(struct cpu_user_regs *regs)
                             sib = insn_fetch(u8, base, eip, limit);
 
                             modrm = (modrm & ~7) | (sib & 7);
-                            if ( (sib >>= 3) != 4 )
+                            if ( ((sib >>= 3) & 7) != 4 )
                                 opnd_off = *(unsigned long *)
                                     decode_register(sib & 7, regs, 0);
                             opnd_off <<= sib >> 3;
@@ -2899,7 +2899,10 @@ static void emulate_gate_op(struct cpu_user_regs *regs)
                         opnd_off += insn_fetch(s8, base, eip, limit);
                         break;
                     case 0x80:
-                        opnd_off += insn_fetch(s32, base, eip, limit);
+                        if ( ad_bytes > 2 )
+                            opnd_off += insn_fetch(s32, base, eip, limit);
+                        else
+                            opnd_off += insn_fetch(s16, base, eip, limit);
                         break;
                     }
                     if ( ad_bytes == 4 )
@@ -2936,8 +2939,7 @@ static void emulate_gate_op(struct cpu_user_regs *regs)
 #define ad_default ad_bytes
     opnd_sel = insn_fetch(u16, base, opnd_off, limit);
 #undef ad_default
-    ASSERT((opnd_sel & ~3) == regs->error_code);
-    if ( dpl < (opnd_sel & 3) )
+    if ( (opnd_sel & ~3) != regs->error_code || dpl < (opnd_sel & 3) )
     {
         do_guest_trap(TRAP_gp_fault, regs, 1);
         return;