x86: honor quoted figure braces in i386_att_operand()

When AVX512 support was added, symbol quotation was not paid attention
to. Just like the (base,index,scale) specifier gets parsed from the end
of the expression, the {...} also wants parsing from the end; in no case
is the first { found a guarantee of a masking or broadcasting specifier.

gas/ChangeLog

@@ -1,3 +1,10 @@
+2021-06-07  Jan Beulich  <jbeulich@suse.com>
+
+	* config/tc-i386.c (i386_att_operand): Re-write handling of
+	vector operations operand suffix.
+	* testsuite/gas/i386/unary.s: Add more cases.
+	* testsuite/gas/i386/unary.d: Adjust expectations.
+
 2021-06-07  Jan Beulich  <jbeulich@suse.com>
 
 	* config/tc-i386.c (parse_operands): Reduce scope of

gas/config/tc-i386.c

@@ -11488,7 +11488,6 @@ i386_att_operand (char *operand_string)
       /* Start and end of displacement string expression (if found).  */
       char *displacement_string_start;
       char *displacement_string_end;
-      char *vop_start;
 
     do_memory_reference:
       if (i.mem_operands == 1 && !maybe_adjust_templates ())
@@ -11509,18 +11508,43 @@ i386_att_operand (char *operand_string)
       base_string = op_string + strlen (op_string);
 
       /* Handle vector operations.  */
-      vop_start = strchr (op_string, '{');
-      if (vop_start && vop_start < base_string)
-	{
-	  if (check_VecOperations (vop_start) == NULL)
-	    return 0;
-	  base_string = vop_start;
-	}
-
       --base_string;
       if (is_space_char (*base_string))
 	--base_string;
 
+      if (*base_string == '}')
+	{
+	  char *vop_start = NULL;
+
+	  while (base_string-- > op_string)
+	    {
+	      if (*base_string == '"')
+		break;
+	      if (*base_string != '{')
+		continue;
+
+	      vop_start = base_string;
+
+	      --base_string;
+	      if (is_space_char (*base_string))
+		--base_string;
+
+	      if (*base_string != '}')
+		break;
+
+	      vop_start = NULL;
+	    }
+
+	  if (!vop_start)
+	    {
+	      as_bad (_("unbalanced figure braces"));
+	      return 0;
+	    }
+
+	  if (check_VecOperations (vop_start) == NULL)
+	    return 0;
+	}
+
       /* If we only have a displacement, set-up for it to be parsed later.  */
       displacement_string_start = op_string;
       displacement_string_end = base_string + 1;

gas/testsuite/gas/i386/quoted.d

@@ -13,6 +13,8 @@ Disassembly of section .text:
 [ 	]*[a-f0-9]+:[ 	]*a1 00 00 00 00[	 ]+mov    0x0,%eax	[a-f0-9]+: (R_386_|dir)32	x\(y
 [ 	]*[a-f0-9]+:[ 	]*a1 00 00 00 00[	 ]+mov    0x0,%eax	[a-f0-9]+: (R_386_|dir)32	x\)y
 [ 	]*[a-f0-9]+:[ 	]*a1 00 00 00 00[	 ]+mov    0x0,%eax	[a-f0-9]+: (R_386_|dir)32	x\?y
+[ 	]*[a-f0-9]+:[ 	]*a1 00 00 00 00[	 ]+mov    0x0,%eax	[a-f0-9]+: (R_386_|dir)32	x\{y
+[ 	]*[a-f0-9]+:[ 	]*a1 00 00 00 00[	 ]+mov    0x0,%eax	[a-f0-9]+: (R_386_|dir)32	x\{z\}
 [ 	]*[a-f0-9]+:[ 	]*ff 15 00 00 00 00[	 ]+call   \*0x0	[a-f0-9]+: (R_386_|dir)32	x\(y\)
 [ 	]*[a-f0-9]+:[ 	]*26 ff 15 00 00 00 00[	 ]+call   \*%es:0x0	[a-f0-9]+: (R_386_|dir)32	x\(y\)
 [ 	]*[a-f0-9]+:[ 	]*26 ff 15 00 00 00 00[	 ]+call   \*%es:0x0	[a-f0-9]+: (R_386_|dir)32	x\(y\)

gas/testsuite/gas/i386/quoted.s

@@ -8,6 +8,8 @@ quoted:
 	mov	"x(y", %eax
 	mov	"x)y", %eax
 	mov	"x?y", %eax
+	mov	"x{y", %eax
+	mov	"x{z}", %eax
 
 	call	*"x(y)"
 	call	*%es:"x(y)"