Lexer: require a space (or other non-ident-char) after a directive, such as %token.

src/lexer.mll

@@ -289,9 +289,13 @@ let mk_stretch pos1 pos2 parenthesize monsters =
 
 (* OCaml's reserved words. *)
 
-let reserved =
+let table words =
   let table = Hashtbl.create 149 in
-  List.iter (fun word -> Hashtbl.add table word ()) [
+  List.iter (fun word -> Hashtbl.add table word ()) words;
+  table
+
+let reserved =
+  table [
     "and";
     "as";
     "assert";
@@ -348,9 +352,33 @@ let reserved =
     "lsl";
     "lsr";
     "asr";
-  ];
+  ]
+
+(* ------------------------------------------------------------------------ *)
+
+(* Menhir's percent-directives. *)
+
+let table directives =
+  let table = Hashtbl.create 149 in
+  List.iter (fun (word, token) -> Hashtbl.add table word token) directives;
   table
 
+let directives =
+  table [
+    "token", TOKEN;
+    "type", TYPE;
+    "left", LEFT;
+    "right", RIGHT;
+    "nonassoc", NONASSOC;
+    "start", START;
+    "prec", PREC;
+    "public", PUBLIC;
+    "parameter", PARAMETER;
+    "inline", INLINE;
+    "attribute", PERCENTATTRIBUTE;
+    "on_error_reduce", ON_ERROR_REDUCE;
+  ]
+
 }
 
 (* ------------------------------------------------------------------------ *)
@@ -392,30 +420,9 @@ let syntaxerror =
 (* The lexer. *)
 
 rule main = parse
-| "%token"
-    { TOKEN }
-| "%type"
-    { TYPE }
-| "%left"
-    { LEFT }
-| "%right"
-    { RIGHT }
-| "%nonassoc"
-    { NONASSOC }
-| "%start"
-    { START }
-| "%prec"
-    { PREC }
-| "%public"
-    { PUBLIC }
-| "%parameter"
-    { PARAMETER }
-| "%inline"
-    { INLINE }
-| "%attribute"
-    { PERCENTATTRIBUTE }
-| "%on_error_reduce"
-    { ON_ERROR_REDUCE }
+| "%" (identchar+ as directive)
+    { try Hashtbl.find directives directive
+      with Not_found -> error2 lexbuf "unknown directive: %s." directive }
 | "%%"
     { (* The token [PERCENTPERCENT] carries a stretch that contains
          everything that follows %% in the input file. This string

test/static/bad/no_space_after_directive.exp

+File "no_space_after_directive.mly", line 1, characters 0-7:
+Error: unknown directive: tokenA.

test/static/bad/no_space_after_directive.mly

+%tokenA
+%start<unit> foo
+  (* This was accepted until 2020/06/06! *)
+
+%%
+
+foo:
+  A {}

test/static/src/dune.auto

@@ -8776,6 +8776,16 @@
 
 (rule (alias no-tokens) (action (diff ../bad/no-tokens.exp no-tokens.out)))
 
+(rule (target no_space_after_directive.out)
+  (deps ../bad/no_space_after_directive.mly)
+  (action
+    (with-outputs-to no_space_after_directive.out
+      (chdir ../bad (with-accepted-exit-codes (not 0) (run menhir %{deps}))))))
+
+(rule (alias no_space_after_directive)
+  (action
+    (diff ../bad/no_space_after_directive.exp no_space_after_directive.out)))
+
 (rule (target non-terminal-is-expected.out)
   (deps ../bad/non-terminal-is-expected.mly)
   (action
@@ -9802,6 +9812,7 @@
     (alias no-start-symbol)
     (alias no-tokens)
     (alias no_future)
+    (alias no_space_after_directive)
     (alias non-terminal-is-expected)
     (alias non-terminal-multiple-branch-groups)
     (alias non-terminal-multiple-definitions)