lexer.mll 18.4 KB
Newer Older
1
2
{

POTTIER Francois's avatar
POTTIER Francois committed
3
4
5
open Lexing
open Parser
open Positions
6
7
8
9
10
11
12
13
14
15
16
open Keyword

(* ------------------------------------------------------------------------ *)

(* Short-hands. *)

let error1 pos =
  Error.error (Positions.one pos)

let error2 lexbuf =
  Error.error (Positions.two lexbuf.lex_start_p lexbuf.lex_curr_p)
POTTIER Francois's avatar
POTTIER Francois committed
17

POTTIER Francois's avatar
POTTIER Francois committed
18
19
(* ------------------------------------------------------------------------ *)

POTTIER Francois's avatar
POTTIER Francois committed
20
21
22
23
24
25
26
27
28
29
(* This wrapper saves the current lexeme start, invokes its argument,
   and restores it. This allows transmitting better positions to the
   parser. *)

let savestart lexbuf f =
  let startp = lexbuf.lex_start_p in
  let token = f lexbuf in
  lexbuf.lex_start_p <- startp;
  token

POTTIER Francois's avatar
POTTIER Francois committed
30
31
(* ------------------------------------------------------------------------ *)

POTTIER Francois's avatar
POTTIER Francois committed
32
33
34
(* Extracts a chunk out of the source file. *)

let chunk ofs1 ofs2 =
35
  let contents = InputFile.get_file_contents() in
POTTIER Francois's avatar
POTTIER Francois committed
36
37
38
  let len = ofs2 - ofs1 in
  String.sub contents ofs1 len

POTTIER Francois's avatar
POTTIER Francois committed
39
40
(* ------------------------------------------------------------------------ *)

POTTIER Francois's avatar
POTTIER Francois committed
41
42
43
44
45
46
47
(* Overwrites an old character with a new one at a specified
   offset in a [bytes] buffer. *)

let overwrite content offset c1 c2 =
  assert (Bytes.get content offset = c1);
  Bytes.set content offset c2

POTTIER Francois's avatar
POTTIER Francois committed
48
49
50
51
(* ------------------------------------------------------------------------ *)

(* Keyword recognition and construction. *)

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
(* A monster is a spot where we have identified a keyword in concrete syntax.
   We describe a monster as an object with the following methods: *)

type monster = {

  (* The position of the monster. *)
  pos: Positions.t;

  (* This method is passed an array of (optional) names for the producers,
     that is, the elements of the production's right-hand side. It may
     perform some checks and is allowed to fail. *)
  check: string option array -> unit;

  (* This method transforms the keyword (in place) into a conventional
     OCaml identifier. This is done by replacing '$', '(', and ')' with
     '_'. Bloody. The arguments are [ofs1] and [content]. [ofs1] is the
     offset where [content] begins in the source file. *)
  transform: int -> bytes -> unit;

  (* This is the keyword, in abstract syntax. *)
  keyword: keyword option;

}

(* ------------------------------------------------------------------------ *)

(* The [$syntaxerror] monster. *)

let syntaxerror pos : monster =
  let check _ = ()
  and transform ofs1 content =
    (* [$syntaxerror] is replaced with
       [(raise _eRR)]. Same length. *)
    let pos = start_of_position pos in
    let ofs = pos.pos_cnum - ofs1 in
    let source = "(raise _eRR)" in
    Bytes.blit_string source 0 content ofs (String.length source)
  and keyword =
    Some SyntaxError
  in
  { pos; check; transform; keyword }

(* ------------------------------------------------------------------------ *)

(* We check that every [$i] is within range. Also, we forbid using [$i]
   when a producer has been given a name; this is bad style and may be
   a mistake. (Plus, this simplies our life, as we rewrite [$i] to [_i],
   and we would have to rewrite it to a different identifier otherwise.) *)

let check_dollar pos i producers =
  if not (0 <= i - 1 && i - 1 < Array.length producers) then
    Error.error [pos] "$%d refers to a nonexistent symbol." i
  else
    producers.(i - 1) |> Option.iter (fun x ->
      Error.error [pos] "please do not say: $%d. Instead, say: %s." i x
    )

(* We check that every reference to a producer [x] in a position keyword,
   such as [$startpos(x)], exists. *)

let check_producer pos x producers =
  if not (List.mem (Some x) (Array.to_list producers)) then
    Error.error [pos] "%s refers to a nonexistent symbol." x

(* ------------------------------------------------------------------------ *)

(* The [$i] monster. *)

let dollar pos i : monster =
  let check = check_dollar pos i
  and transform ofs1 content =
    (* [$i] is replaced with [_i]. Thus, it is no longer a keyword. *)
    let pos = start_of_position pos in
    let ofs = pos.pos_cnum - ofs1 in
    overwrite content ofs '$' '_'
  and keyword =
    None
  in
  { pos; check; transform; keyword }

(* ------------------------------------------------------------------------ *)

(* The position-keyword monster. The most horrible of all. *)

let position pos
  (where : string)
  (flavor : string)
  (i : string option) (x : string option)
=
  let none _ = () in
  let where, ofslpar (* offset of the opening parenthesis, if there is one *) =
    match where with
POTTIER Francois's avatar
POTTIER Francois committed
144
145
146
    | "symbolstart" -> WhereSymbolStart, 15
    | "start"       -> WhereStart,        9
    | "end"         -> WhereEnd,          7
147
    | _       -> assert false
POTTIER Francois's avatar
POTTIER Francois committed
148
149
150
151
152
153
154
155
156
157
  in
  let () =
    match where, i, x with
    | WhereSymbolStart, Some _, _
    | WhereSymbolStart, _, Some _ ->
        Error.error [pos] "$symbolstart%s does not take a parameter." flavor
    | _, _, _ ->
        ()
  in
  let flavor =
158
159
160
161
162
163
164
165
166
167
168
169
    match flavor with
    | "pos"   -> FlavorPosition
    | "ofs"   -> FlavorOffset
    | _       -> assert false
  in
  let subject, check =
    match i, x with
    | Some i, None ->
        let ii = int_of_string i in (* cannot fail *)
        if ii = 0 && where = WhereEnd then
          (* [$endpos($0)] *)
          Before, none
POTTIER Francois's avatar
POTTIER Francois committed
170
        else
171
172
173
174
175
176
177
178
179
180
181
182
          (* [$startpos($i)] is rewritten to [$startpos(_i)]. *)
          RightNamed ("_" ^ i), check_dollar pos ii
    | None, Some x ->
        (* [$startpos(x)] *)
        RightNamed x, check_producer pos x
    | None, None ->
        (* [$startpos] *)
        Left, none
    | Some _, Some _ ->
        assert false
  in
  let transform ofs1 content =
POTTIER Francois's avatar
POTTIER Francois committed
183
184
185
    let pos = start_of_position pos in
    let ofs = pos.pos_cnum - ofs1 in
    overwrite content ofs '$' '_';
186
187
188
189
190
191
    let ofslpar = ofs + ofslpar in
    match i, x with
    | None, Some x ->
        overwrite content ofslpar '(' '_';
        overwrite content (ofslpar + 1 + String.length x) ')' '_'
    | Some i, None ->
POTTIER Francois's avatar
POTTIER Francois committed
192
        overwrite content ofslpar '(' '_';
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
        overwrite content (ofslpar + 1) '$' '_';
        overwrite content (ofslpar + 2 + String.length i) ')' '_'
    | _, _ ->
        ()
  in
  let keyword =
    Some (Position (subject, where, flavor))
  in
  { pos; check; transform; keyword }

(* ------------------------------------------------------------------------ *)

(* In an OCaml header, there should be no monsters. This is just a sanity
   check. *)

let no_monsters monsters =
  match monsters with
POTTIER Francois's avatar
POTTIER Francois committed
210
211
  | [] ->
      ()
212
213
214
215
216
  | monster :: _ ->
      Error.error [monster.pos]
        "a Menhir keyword cannot be used in an OCaml header."

(* ------------------------------------------------------------------------ *)
POTTIER Francois's avatar
POTTIER Francois committed
217
218
219

(* Creates a stretch. *)

POTTIER Francois's avatar
POTTIER Francois committed
220
let mk_stretch pos1 pos2 parenthesize monsters =
POTTIER Francois's avatar
POTTIER Francois committed
221
222
223
224
  (* Read the specified chunk of the file. *)
  let ofs1 = pos1.pos_cnum
  and ofs2 = pos2.pos_cnum in
  let raw_content : string = chunk ofs1 ofs2 in
225
  (* Transform the monsters, if there are any. (This explicit test
POTTIER Francois's avatar
POTTIER Francois committed
226
227
     allows saving one string copy and keeping just one live copy.) *)
  let content : string =
228
    match monsters with
229
    | [] ->
POTTIER Francois's avatar
POTTIER Francois committed
230
231
232
        raw_content
    | _ :: _ ->
        let content : bytes = Bytes.of_string raw_content in
233
        List.iter (fun monster -> monster.transform ofs1 content) monsters;
POTTIER Francois's avatar
POTTIER Francois committed
234
235
236
237
238
239
240
241
242
243
244
        Bytes.unsafe_to_string content
  in
  (* Add whitespace so that the column numbers match those of the source file.
     If requested, add parentheses so that the semantic action can be inserted
     into other code without ambiguity. *)
  let content =
    if parenthesize then
      (String.make (pos1.pos_cnum - pos1.pos_bol - 1) ' ') ^ "(" ^ content ^ ")"
    else
      (String.make (pos1.pos_cnum - pos1.pos_bol) ' ') ^ content
  in
245
  Stretch.({
246
    stretch_filename = InputFile.get_input_file_name();
247
248
249
250
251
252
    stretch_linenum = pos1.pos_lnum;
    stretch_linecount = pos2.pos_lnum - pos1.pos_lnum;
    stretch_content = content;
    stretch_raw_content = raw_content;
    stretch_keywords = Misc.map_opt (fun monster -> monster.keyword) monsters
  })
POTTIER Francois's avatar
POTTIER Francois committed
253

POTTIER Francois's avatar
POTTIER Francois committed
254
255
(* ------------------------------------------------------------------------ *)

256
(* OCaml's reserved words. *)
POTTIER Francois's avatar
POTTIER Francois committed
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319

let reserved =
  let table = Hashtbl.create 149 in
  List.iter (fun word -> Hashtbl.add table word ()) [
    "and";
    "as";
    "assert";
    "begin";
    "class";
    "constraint";
    "do";
    "done";
    "downto";
    "else";
    "end";
    "exception";
    "external";
    "false";
    "for";
    "fun";
    "function";
    "functor";
    "if";
    "in";
    "include";
    "inherit";
    "initializer";
    "lazy";
    "let";
    "match";
    "method";
    "module";
    "mutable";
    "new";
    "object";
    "of";
    "open";
    "or";
    "parser";
    "private";
    "rec";
    "sig";
    "struct";
    "then";
    "to";
    "true";
    "try";
    "type";
    "val";
    "virtual";
    "when";
    "while";
    "with";
    "mod";
    "land";
    "lor";
    "lxor";
    "lsl";
    "lsr";
    "asr";
  ];
  table

320
321
}

POTTIER Francois's avatar
POTTIER Francois committed
322
323
324
325
(* ------------------------------------------------------------------------ *)

(* Patterns. *)

326
327
328
329
330
331
332
333
334
335
let newline = ('\010' | '\013' | "\013\010")

let whitespace = [ ' ' '\t' ';' ]

let lowercase = ['a'-'z' '\223'-'\246' '\248'-'\255' '_']

let uppercase = ['A'-'Z' '\192'-'\214' '\216'-'\222']

let identchar = ['A'-'Z' 'a'-'z' '_' '\192'-'\214' '\216'-'\246' '\248'-'\255' '0'-'9'] (* '\'' forbidden *)

POTTIER Francois's avatar
POTTIER Francois committed
336
let poskeyword =
337
  '$'
POTTIER Francois's avatar
POTTIER Francois committed
338
  (("symbolstart" | "start" | "end") as where)
339
340
  (("pos" | "ofs") as flavor)
  ( '(' ( '$' (['0'-'9']+ as i) | ((lowercase identchar*) as x)) ')')?
341
342
343
344
345
346
347

let previouserror =
  "$previouserror"

let syntaxerror =
  "$syntaxerror"

POTTIER Francois's avatar
POTTIER Francois committed
348
349
350
351
(* ------------------------------------------------------------------------ *)

(* The lexer. *)

352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
rule main = parse
| "%token"
    { TOKEN }
| "%type"
    { TYPE }
| "%left"
    { LEFT }
| "%right"
    { RIGHT }
| "%nonassoc"
    { NONASSOC }
| "%start"
    { START }
| "%prec"
    { PREC }
| "%public"
    { PUBLIC }
| "%parameter"
    { PARAMETER }
| "%inline"
    { INLINE }
373
374
| "%on_error_reduce"
    { ON_ERROR_REDUCE }
375
| "%%"
376
    { (* The token [PERCENTPERCENT] carries a stretch that contains
377
         everything that follows %% in the input file. This string
378
379
380
381
382
383
384
385
386
         must be created lazily. The parser decides (based on the
         context) whether this stretch is needed. If it is indeed
         needed, then constructing this stretch drives the lexer
         to the end of the file. *)
      PERCENTPERCENT (lazy (
        let openingpos = lexeme_end_p lexbuf in
        let closingpos = finish lexbuf in
        mk_stretch openingpos closingpos false []
      )) }
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
| ":"
    { COLON }
| ","
    { COMMA }
| "="
    { EQUAL }
| "("
    { LPAREN }
| ")"
    { RPAREN }
| "|"
    { BAR }
| "?"
    { QUESTION }
| "*"
    { STAR }
| "+"
    { PLUS }
| (lowercase identchar *) as id
    { if Hashtbl.mem reserved id then
407
        error2 lexbuf "this is an OCaml reserved word."
408
      else
409
        LID (with_pos (cpos lexbuf) id)
410
411
412
413
414
    }
| (uppercase identchar *) as id
    { UID (with_pos (cpos lexbuf) id) }
| "//" [^ '\010' '\013']* newline (* skip C++ style comment *)
| newline
415
    { new_line lexbuf; main lexbuf }
416
417
418
419
420
421
422
423
424
425
| whitespace+
    { main lexbuf }
| "/*"
    { comment (lexeme_start_p lexbuf) lexbuf; main lexbuf }
| "(*"
    { ocamlcomment (lexeme_start_p lexbuf) lexbuf; main lexbuf }
| "<"
    { savestart lexbuf (ocamltype (lexeme_end_p lexbuf)) }
| "%{"
    { savestart lexbuf (fun lexbuf ->
426
427
        let openingpos = lexeme_start_p lexbuf in
        let stretchpos = lexeme_end_p lexbuf in
428
429
        let closingpos, monsters = action true openingpos [] lexbuf in
        no_monsters monsters;
430
        HEADER (mk_stretch stretchpos closingpos false [])
431
432
433
      ) }
| "{"
    { savestart lexbuf (fun lexbuf ->
434
435
        let openingpos = lexeme_start_p lexbuf in
        let stretchpos = lexeme_end_p lexbuf in
436
        let closingpos, monsters = action false openingpos [] lexbuf in
437
        ACTION (
438
          fun (producers : string option array) ->
439
            List.iter (fun monster -> monster.check producers) monsters;
440
            let stretch = mk_stretch stretchpos closingpos true monsters in
441
442
            Action.from_stretch stretch
        )
443
444
445
446
      ) }
| eof
    { EOF }
| _
447
    { error2 lexbuf "unexpected character(s)." }
448

POTTIER Francois's avatar
POTTIER Francois committed
449
450
(* ------------------------------------------------------------------------ *)

451
452
453
454
(* Skip C style comments. *)

and comment openingpos = parse
| newline
455
    { new_line lexbuf; comment openingpos lexbuf }
456
457
458
459
460
461
462
| "*/"
    { () }
| eof
    { error1 openingpos "unterminated comment." }
| _
    { comment openingpos lexbuf }

POTTIER Francois's avatar
POTTIER Francois committed
463
464
(* ------------------------------------------------------------------------ *)

465
(* Collect an O'Caml type delimited by angle brackets. Angle brackets can
466
467
   appear as part of O'Caml function types and variant types, so we must
   recognize them and *not* treat them as a closing bracket. *)
468
469
470

and ocamltype openingpos = parse
| "->"
471
| "[>"
472
473
    { ocamltype openingpos lexbuf }
| '>'
474
    { OCAMLTYPE (Stretch.Declared (mk_stretch openingpos (lexeme_start_p lexbuf) true [])) }
475
476
477
| "(*"
    { ocamlcomment (lexeme_start_p lexbuf) lexbuf; ocamltype openingpos lexbuf }
| newline
478
    { new_line lexbuf; ocamltype openingpos lexbuf }
479
| eof
480
    { error1 openingpos "unterminated OCaml type." }
481
482
483
| _
    { ocamltype openingpos lexbuf }

POTTIER Francois's avatar
POTTIER Francois committed
484
485
(* ------------------------------------------------------------------------ *)

486
487
488
489
(* Collect O'Caml code delimited by curly brackets. The monsters that are
   encountered along the way are accumulated in the list [monsters]. Nested
   curly brackets must be properly counted. Nested parentheses are also kept
   track of, so as to better report errors when they are not balanced. *)
490

491
and action percent openingpos monsters = parse
492
| '{'
493
494
    { let _, monsters = action false (lexeme_end_p lexbuf) monsters lexbuf in
      action percent openingpos monsters lexbuf }
495
496
497
498
| ("}" | "%}") as delimiter
    { match percent, delimiter with
      | true, "%}"
      | false, "}" ->
499
500
          (* This is the delimiter we were instructed to look for. *)
          lexeme_start_p lexbuf, monsters
501
      | _, _ ->
502
503
          (* This is not it. *)
          error1 openingpos "unbalanced opening brace."
504
505
    }
| '('
506
507
508
509
510
    { let _, monsters = parentheses (lexeme_end_p lexbuf) monsters lexbuf in
      action percent openingpos monsters lexbuf }
| '$' (['0'-'9']+ as i)
    { let monster = dollar (cpos lexbuf) (int_of_string i) in
      action percent openingpos (monster :: monsters) lexbuf }
511
| poskeyword
512
513
    { let monster = position (cpos lexbuf) where flavor i x in
      action percent openingpos (monster :: monsters) lexbuf }
514
| previouserror
515
    { error2 lexbuf "$previouserror is no longer supported." }
516
| syntaxerror
517
518
    { let monster = syntaxerror (cpos lexbuf) in
      action percent openingpos (monster :: monsters) lexbuf }
519
520
| '"'
    { string (lexeme_start_p lexbuf) lexbuf;
521
      action percent openingpos monsters lexbuf }
522
523
| "'"
    { char lexbuf;
524
      action percent openingpos monsters lexbuf }
525
526
| "(*"
    { ocamlcomment (lexeme_start_p lexbuf) lexbuf;
527
      action percent openingpos monsters lexbuf }
528
| newline
529
    { new_line lexbuf;
530
      action percent openingpos monsters lexbuf }
531
532
533
534
| ')'
| eof
    { error1 openingpos "unbalanced opening brace." }
| _
535
    { action percent openingpos monsters lexbuf }
536

POTTIER Francois's avatar
POTTIER Francois committed
537
538
(* ------------------------------------------------------------------------ *)

539
and parentheses openingpos monsters = parse
540
| '('
541
542
    { let _, monsters = parentheses (lexeme_end_p lexbuf) monsters lexbuf in
      parentheses openingpos monsters lexbuf }
543
| ')'
544
    { lexeme_start_p lexbuf, monsters }
545
| '{'
546
547
548
549
550
    { let _, monsters = action false (lexeme_end_p lexbuf) monsters lexbuf in
      parentheses openingpos monsters lexbuf }
| '$' (['0'-'9']+ as i)
    { let monster = dollar (cpos lexbuf) (int_of_string i) in
      parentheses openingpos (monster :: monsters) lexbuf }
551
| poskeyword
552
553
    { let monster = position (cpos lexbuf) where flavor i x in
      parentheses openingpos (monster :: monsters) lexbuf }
554
| previouserror
555
    { error2 lexbuf "$previouserror is no longer supported." }
556
| syntaxerror
557
558
    { let monster = syntaxerror (cpos lexbuf) in
      parentheses openingpos (monster :: monsters) lexbuf }
559
| '"'
560
    { string (lexeme_start_p lexbuf) lexbuf; parentheses openingpos monsters lexbuf }
561
| "'"
562
    { char lexbuf; parentheses openingpos monsters lexbuf }
563
| "(*"
564
    { ocamlcomment (lexeme_start_p lexbuf) lexbuf; parentheses openingpos monsters lexbuf }
565
| newline
566
    { new_line lexbuf; parentheses openingpos monsters lexbuf }
567
568
569
570
| '}'
| eof
    { error1 openingpos "unbalanced opening parenthesis." }
| _
571
    { parentheses openingpos monsters lexbuf }
572

POTTIER Francois's avatar
POTTIER Francois committed
573
574
(* ------------------------------------------------------------------------ *)

575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
(* Skip O'Caml comments. Comments can be nested and can contain
   strings or characters, which must be correctly analyzed. (A string
   could contain begin-of-comment or end-of-comment sequences, which
   must be ignored; a character could contain a begin-of-string
   sequence.) *)

and ocamlcomment openingpos = parse
| "*)"
    { () }
| "(*"
    { ocamlcomment (lexeme_start_p lexbuf) lexbuf; ocamlcomment openingpos lexbuf }
| '"'
    { string (lexeme_start_p lexbuf) lexbuf; ocamlcomment openingpos lexbuf }
| "'"
    { char lexbuf; ocamlcomment openingpos lexbuf }
| newline
591
    { new_line lexbuf; ocamlcomment openingpos lexbuf }
592
| eof
593
    { error1 openingpos "unterminated OCaml comment." }
594
595
596
| _
    { ocamlcomment openingpos lexbuf }

POTTIER Francois's avatar
POTTIER Francois committed
597
598
(* ------------------------------------------------------------------------ *)

599
600
601
(* Skip O'Caml strings. *)

and string openingpos = parse
POTTIER Francois's avatar
POTTIER Francois committed
602
| '"'
603
604
605
   { () }
| '\\' newline
| newline
606
   { new_line lexbuf; string openingpos lexbuf }
607
608
609
610
| '\\' _
   (* Upon finding a backslash, skip the character that follows,
      unless it is a newline. Pretty crude, but should work. *)
   { string openingpos lexbuf }
POTTIER Francois's avatar
POTTIER Francois committed
611
| eof
612
   { error1 openingpos "unterminated OCaml string." }
613
614
615
| _
   { string openingpos lexbuf }

POTTIER Francois's avatar
POTTIER Francois committed
616
617
(* ------------------------------------------------------------------------ *)

618
619
620
621
622
623
(* Skip O'Caml characters. A lone quote character is legal inside
   a comment, so if we don't recognize the matching closing quote,
   we simply abandon. *)

and char = parse
| '\\'? newline "'"
624
   { new_line lexbuf }
625
626
627
628
629
| [^ '\\' '\''] "'"
| '\\' _ "'"
| '\\' ['0'-'9'] ['0'-'9'] ['0'-'9'] "'"
| '\\' 'x' ['0'-'9' 'a'-'f' 'A'-'F'] ['0'-'9' 'a'-'f' 'A'-'F'] "'"
| ""
POTTIER Francois's avatar
POTTIER Francois committed
630
   { () }
631

POTTIER Francois's avatar
POTTIER Francois committed
632
633
(* ------------------------------------------------------------------------ *)

634
635
636
(* Read until the end of the file. This is used after finding a %%
   that marks the end of the grammar specification. We update the
   current position as we go. This allows us to build a stretch
637
   for the postlude. *)
638
639
640

and finish = parse
| newline
641
    { new_line lexbuf; finish lexbuf }
642
643
644
645
| eof
    { lexeme_start_p lexbuf }
| _
    { finish lexbuf }