x86: warn about insns exceeding the 15-byte limit

Such insns will cause #UD when an attempt to execute them is made.

See also http://www.sandpile.org/x86/opc_enc.htm.

gas/ChangeLog

@@ -1,3 +1,13 @@
+2019-07-01  Jan Beulich  <jbeulich@suse.com>
+
+	* tc-i386.c (encoding_length): New.
+	(output_insn): Use it.
+	* testsuite/gas/i386/oversized16.l,
+	testsuite/gas/i386/oversized16.s,
+	testsuite/gas/i386/oversized64.l,
+	testsuite/gas/i386/oversized64.s: New.
+	* testsuite/gas/i386/i386.exp: Run new tests.
+
 2019-06-27  H.J. Lu  <hongjiu.lu@intel.com>
 
 	PR binutils/24719

gas/config/tc-i386.c

@@ -8123,6 +8123,25 @@ x86_cleanup (void)
 }
 #endif
 
+static unsigned int
+encoding_length (const fragS *start_frag, offsetT start_off,
+		 const char *frag_now_ptr)
+{
+  unsigned int len = 0;
+
+  if (start_frag != frag_now)
+    {
+      const fragS *fr = start_frag;
+
+      do {
+	len += fr->fr_fix;
+	fr = fr->fr_next;
+      } while (fr && fr != frag_now);
+    }
+
+  return len - start_off + (frag_now_ptr - frag_now->fr_literal);
+}
+
 static void
 output_insn (void)
 {
@@ -8400,6 +8419,19 @@ output_insn (void)
 
       if (i.imm_operands)
 	output_imm (insn_start_frag, insn_start_off);
+
+      /*
+       * frag_now_fix () returning plain abs_section_offset when we're in the
+       * absolute section, and abs_section_offset not getting updated as data
+       * gets added to the frag breaks the logic below.
+       */
+      if (now_seg != absolute_section)
+	{
+	  j = encoding_length (insn_start_frag, insn_start_off, frag_more (0));
+	  if (j > 15)
+	    as_warn (_("instruction length of %u bytes exceeds the limit of 15"),
+		     j);
+	}
     }
 
 #ifdef DEBUG386

gas/testsuite/gas/i386/i386.exp

@@ -130,6 +130,7 @@ if [expr ([istarget "i*86-*-*"] ||  [istarget "x86_64-*-*"]) && [gas_32_check]]
     run_dump_test "nops-7"
     run_dump_test "addr16"
     run_dump_test "addr32"
+    run_list_test "oversized16" "-al"
     run_dump_test "sse4_1"
     run_dump_test "sse4_1-intel"
     run_dump_test "sse4_2"
@@ -722,6 +723,7 @@ if [expr ([istarget "i*86-*-*"] || [istarget "x86_64-*-*"]) && [gas_64_check]] t
     run_list_test "x86-64-suffix-bad"
     run_list_test "unspec64" ""
     run_list_test "prefix64" "-al"
+    run_list_test "oversized64" "-al"
     run_dump_test "x86-64-fxsave"
     run_dump_test "x86-64-fxsave-intel"
     run_dump_test "x86-64-arch-1"

gas/testsuite/gas/i386/oversized16.l

@@ -0,0 +1,41 @@
+.*: Assembler messages:
+.*:6: Warning: instruction length.*
+.*:7: Warning: instruction length.*
+.*:8: Warning: instruction length.*
+.*:10: Warning: instruction length.*
+.*:11: Warning: instruction length.*
+GAS LISTING .*
+
+
+[ 	]*1[ 	]+\.text
+[ 	]*2[ 	]+\.intel_syntax noprefix
+[ 	]*3[ 	]+\.code16
+[ 	]*4[ 	]+
+[ 	]*5[ 	]+long16:
+[ 	]*6[ 	]+0000 26678FEA[ 	]+lwpins	eax, es:\[eax\*8\], 0x11223344
+\*\*\*\*  Warning: instruction length of 16 bytes exceeds the limit of 15
+[ 	]*6[ 	]+781204C5[ 	]*
+[ 	]*6[ 	]+00000000[ 	]*
+[ 	]*6[ 	]+44332211[ 	]*
+[ 	]*7[ 	]+0010 26678FEA[ 	]+lwpval	eax, es:\[eax\*4\], 0x11223344
+\*\*\*\*  Warning: instruction length of 16 bytes exceeds the limit of 15
+[ 	]*7[ 	]+78120C85[ 	]*
+[ 	]*7[ 	]+00000000[ 	]*
+[ 	]*7[ 	]+44332211[ 	]*
+[ 	]*8[ 	]+0020 26678FEA[ 	]+bextr	eax, es:\[eax\*2\], 0x11223344
+\*\*\*\*  Warning: instruction length of 16 bytes exceeds the limit of 15
+[ 	]*8[ 	]+78100445[ 	]*
+[ 	]*8[ 	]+00000000[ 	]*
+[ 	]*8[ 	]+44332211[ 	]*
+[ 	]*9[ 	]+
+[ 	]*10[ 	]+0030 266766F2[ 	]+xacquire lock add dword ptr es:\[eax\*2\], 0x11223344
+\*\*\*\*  Warning: instruction length of 16 bytes exceeds the limit of 15
+[ 	]*10[ 	]+F0810445[ 	]*
+[ 	]*10[ 	]+00000000[ 	]*
+[ 	]*10[ 	]+44332211[ 	]*
+[ 	]*11[ 	]+0040 266766F3[ 	]+xrelease lock sub dword ptr es:\[eax\*2\], 0x11223344
+\*\*\*\*  Warning: instruction length of 16 bytes exceeds the limit of 15
+[ 	]*11[ 	]+F0812C45[ 	]*
+[ 	]*11[ 	]+00000000[ 	]*
+[ 	]*11[ 	]+44332211[ 	]*
+#pass

gas/testsuite/gas/i386/oversized16.s

@@ -0,0 +1,11 @@
+	.text
+	.intel_syntax noprefix
+	.code16
+
+long16:
+	lwpins	eax, es:[eax*8], 0x11223344
+	lwpval	eax, es:[eax*4], 0x11223344
+	bextr	eax, es:[eax*2], 0x11223344
+
+	xacquire lock add dword ptr es:[eax*2], 0x11223344
+	xrelease lock sub dword ptr es:[eax*2], 0x11223344

gas/testsuite/gas/i386/oversized64.l

@@ -0,0 +1,40 @@
+.*: Assembler messages:
+.*:5: Warning: instruction length.*
+.*:6: Warning: instruction length.*
+.*:7: Warning: instruction length.*
+.*:9: Warning: instruction length.*
+.*:10: Warning: instruction length.*
+GAS LISTING .*
+
+
+[ 	]*1[ 	]+\.text
+[ 	]*2[ 	]+\.intel_syntax noprefix
+[ 	]*3[ 	]+
+[ 	]*4[ 	]+long64:
+[ 	]*5[ 	]+0000 64678FEA[ 	]+lwpins	rax, fs:\[eax\*8\], 0x11223344
+\*\*\*\*  Warning: instruction length of 16 bytes exceeds the limit of 15
+[ 	]*5[ 	]+F81204C5[ 	]*
+[ 	]*5[ 	]+00000000[ 	]*
+[ 	]*5[ 	]+44332211[ 	]*
+[ 	]*6[ 	]+0010 64678FEA[ 	]+lwpval	eax, fs:\[eax\*4\], 0x11223344
+\*\*\*\*  Warning: instruction length of 16 bytes exceeds the limit of 15
+[ 	]*6[ 	]+78120C85[ 	]*
+[ 	]*6[ 	]+00000000[ 	]*
+[ 	]*6[ 	]+44332211[ 	]*
+[ 	]*7[ 	]+0020 64678FEA[ 	]+bextr	rax, fs:\[eax\*2\], 0x11223344
+\*\*\*\*  Warning: instruction length of 16 bytes exceeds the limit of 15
+[ 	]*7[ 	]+F8100445[ 	]*
+[ 	]*7[ 	]+00000000[ 	]*
+[ 	]*7[ 	]+44332211[ 	]*
+[ 	]*8[ 	]+
+[ 	]*9[ 	]+0030 6567F2F0[ 	]+xacquire lock add qword ptr gs:\[eax\*8\], 0x11223344
+\*\*\*\*  Warning: instruction length of 16 bytes exceeds the limit of 15
+[ 	]*9[ 	]+488104C5[ 	]*
+[ 	]*9[ 	]+00000000[ 	]*
+[ 	]*9[ 	]+44332211[ 	]*
+[ 	]*10[ 	]+0040 6567F3F0[ 	]+xrelease lock sub qword ptr gs:\[eax\*8\], 0x11223344
+\*\*\*\*  Warning: instruction length of 16 bytes exceeds the limit of 15
+[ 	]*10[ 	]+48812CC5[ 	]*
+[ 	]*10[ 	]+00000000[ 	]*
+[ 	]*10[ 	]+44332211[ 	]*
+#pass

gas/testsuite/gas/i386/oversized64.s

@@ -0,0 +1,10 @@
+	.text
+	.intel_syntax noprefix
+
+long64:
+	lwpins	rax, fs:[eax*8], 0x11223344
+	lwpval	eax, fs:[eax*4], 0x11223344
+	bextr	rax, fs:[eax*2], 0x11223344
+
+	xacquire lock add qword ptr gs:[eax*8], 0x11223344
+	xrelease lock sub qword ptr gs:[eax*8], 0x11223344