x86-64/Intel: issue diagnostic for out of range displacement

... rather than silently dropping it altogether.
i386_finalize_displacement() expects baseindex to already be set, so
the respective statement needs to be moved up. This then also allows a
subsequent conditional to be simplified.

For this to not regress on 32-bit addressing, break out address size
guessing from i386_index_check(), invoking the new function earlier so
that i386_finalize_displacement() has i.prefix[ADDR_PREFIX] available.
i386_addressing_mode () in turn needs i.base_reg / i.index_reg set
earlier.

gas/ChangeLog

@@ -1,3 +1,14 @@
+2017-11-13  Jan Beulich  <jbeulich@suse.com>
+
+	* config/tc-i386.c (i386_index_check): Break out ...
+	(i386_addressing_mode): ... this new function.
+	* config/tc-i386-intel.c (i386_intel_operand): Do base/index
+	swapping and the setting of .baseindex earlier. Call
+	i386_addressing_mode.
+	* testsuite/gas/i386/x86-64-inval.s: Add out of range
+	displacement case.
+	* testsuite/gas/i386/x86-64-inval.l: Adjust expectations.
+
 2017-11-09  Jim Wilson  <jimw@sifive.com>
 
 	* testsuite/gas/elf/dwarf2-10.l: Accept optional line number in error.

gas/config/tc-i386-intel.c

@@ -876,18 +876,41 @@ i386_intel_operand (char *operand_string, int got_a_float)
 	  return 0;
 	}
 
+      /* Swap base and index in 16-bit memory operands like
+	 [si+bx]. Since i386_index_check is also used in AT&T
+	 mode we have to do this here.  */
+      if (intel_state.base
+	  && intel_state.index
+	  && intel_state.base->reg_type.bitfield.reg16
+	  && intel_state.index->reg_type.bitfield.reg16
+	  && intel_state.base->reg_num >= 6
+	  && intel_state.index->reg_num < 6)
+	{
+	  i.base_reg = intel_state.index;
+	  i.index_reg = intel_state.base;
+	}
+      else
+	{
+	  i.base_reg = intel_state.base;
+	  i.index_reg = intel_state.index;
+	}
+
+      if (i.base_reg || i.index_reg)
+	i.types[this_operand].bitfield.baseindex = 1;
+
       expP = &disp_expressions[i.disp_operands];
       memcpy (expP, &exp, sizeof(exp));
       resolve_expression (expP);
 
       if (expP->X_op != O_constant
 	  || expP->X_add_number
-	  || (!intel_state.base
-	      && !intel_state.index))
+	  || !i.types[this_operand].bitfield.baseindex)
 	{
 	  i.op[this_operand].disps = expP;
 	  i.disp_operands++;
 
+	  i386_addressing_mode ();
+
 	  if (flag_code == CODE_64BIT)
 	    {
 	      i.types[this_operand].bitfield.disp32 = 1;
@@ -927,9 +950,6 @@ i386_intel_operand (char *operand_string, int got_a_float)
 	    return 0;
 	}
 
-      if (intel_state.base || intel_state.index)
-	i.types[this_operand].bitfield.baseindex = 1;
-
       if (intel_state.seg)
 	{
 	  expP = symbol_get_value_expression (intel_state.seg);
@@ -956,25 +976,6 @@ i386_intel_operand (char *operand_string, int got_a_float)
 	    }
 	}
 
-      /* Swap base and index in 16-bit memory operands like
-	 [si+bx]. Since i386_index_check is also used in AT&T
-	 mode we have to do that here.  */
-      if (intel_state.base
-	  && intel_state.index
-	  && intel_state.base->reg_type.bitfield.reg16
-	  && intel_state.index->reg_type.bitfield.reg16
-	  && intel_state.base->reg_num >= 6
-	  && intel_state.index->reg_num < 6)
-	{
-	  i.base_reg = intel_state.index;
-	  i.index_reg = intel_state.base;
-	}
-      else
-	{
-	  i.base_reg = intel_state.base;
-	  i.index_reg = intel_state.index;
-	}
-
       if (!i386_index_check (operand_string))
 	return 0;
 

gas/config/tc-i386.c

@@ -8633,13 +8633,13 @@ i386_finalize_displacement (segT exp_seg ATTRIBUTE_UNUSED, expressionS *exp,
   return ret;
 }
 
-/* Make sure the memory operand we've been dealt is valid.
-   Return 1 on success, 0 on a failure.  */
+/* Return the active addressing mode, taking address override and
+   registers forming the address into consideration.  Update the
+   address override prefix if necessary.  */
 
-static int
-i386_index_check (const char *operand_string)
+static enum flag_code
+i386_addressing_mode (void)
 {
-  const char *kind = "base/index";
   enum flag_code addr_mode;
 
   if (i.prefix[ADDR_PREFIX])
@@ -8688,6 +8688,18 @@ i386_index_check (const char *operand_string)
 #endif
     }
 
+  return addr_mode;
+}
+
+/* Make sure the memory operand we've been dealt is valid.
+   Return 1 on success, 0 on a failure.  */
+
+static int
+i386_index_check (const char *operand_string)
+{
+  const char *kind = "base/index";
+  enum flag_code addr_mode = i386_addressing_mode ();
+
   if (current_templates->start->opcode_modifier.isstring
       && !current_templates->start->opcode_modifier.immext
       && (current_templates->end[-1].opcode_modifier.isstring

gas/testsuite/gas/i386/x86-64-inval.l

@@ -109,6 +109,7 @@
 .*:115: Error: .*
 .*:116: Error: .*
 .*:117: Error: .*
+.*:118: Error: .*
 GAS LISTING .*
 
 
@@ -235,3 +236,4 @@ GAS LISTING .*
 [ 	]*115[ 	]+jmpd \[r8\]	\# 32-bit data size not allowed
 [ 	]*116[ 	]+jmpd \[rax\]	\# 32-bit data size not allowed
 [ 	]*117[ 	]+jmpq \[ax\]	\# no 16-bit addressing
+[ 	]*[1-9][0-9]*[ 	]+mov eax,\[rax\+0x876543210\] \# out of range displacement

gas/testsuite/gas/i386/x86-64-inval.s

@@ -115,3 +115,4 @@ movnti word ptr [rax], ax
 	jmpd [r8]	# 32-bit data size not allowed
 	jmpd [rax]	# 32-bit data size not allowed
 	jmpq [ax]	# no 16-bit addressing
+	mov eax,[rax+0x876543210] # out of range displacement