LRijkstra.ml 22 KB
Newer Older
1
open Grammar
2
module Q = LowIntegerPriorityQueue
3
module W = Terminal.Word(struct end) (* TEMPORARY wrap side effect in functor *)
4

5 6 7 8
(* Throughout, we ignore the [error] pseudo-token completely. We consider that
   it never appears on the input stream. Hence, any state whose incoming
   symbol is [error] is considered unreachable. *)

9 10
(* ------------------------------------------------------------------------ *)

11 12 13 14 15 16 17 18
(* We begin with a number of auxiliary functions that provide information
   about the LR(1) automaton. These functions could perhaps be moved to a
   separate module. We keep them here, for the moment, because they are not
   used anywhere else. *)

(* [reductions s z] is the list of reductions permitted in state [s] when the
   lookahead symbol is [z]. This is a list of zero or one elements. This does
   not take default reductions into account. *)
19

20
let reductions s z : Production.index list =
21 22 23 24 25
  try
    TerminalMap.find z (Lr1.reductions s)
  with Not_found ->
    []

26 27 28
(* [has_reduction s z] tells whether state [s] is willing to reduce some
   production (and if so, which one) when the lookahead symbol is [z]. It
   takes a possible default reduction into account. *)
29 30 31 32 33 34 35 36 37 38 39 40 41

let has_reduction s z : Production.index option =
  match Invariant.has_default_reduction s with
  | Some (prod, _) ->
      Some prod
  | None ->
      match reductions s z with
      | prod :: prods ->
          assert (prods = []);
          Some prod
      | [] ->
          None

42 43
(* [causes_an_error s z] tells whether state [s] will initiate an error on the
   lookahead symbol [z]. *)
44

45
let causes_an_error s z : bool =
46 47 48 49 50 51 52
  match Invariant.has_default_reduction s with
  | Some _ ->
      false
  | None ->
      reductions s z = [] &&
      not (SymbolMap.mem (Symbol.T z) (Lr1.transitions s))

53 54 55
(* [foreach_terminal f] applies the function [f] to every terminal symbol in
   turn, except [error]. *)

56 57 58 59 60 61
let foreach_terminal f =
  Terminal.iter (fun t ->
    if not (Terminal.equal t Terminal.error) then
      f t
  )

62 63 64 65 66
(* [foreach_terminal_not_causing_an_error s f] applies the function [f] to
   every terminal symbol [z] such that [causes_an_error s z] is false. This
   could be implemented in a naive manner using [foreach_terminal] and
   [causes_an_error]. This implementation is slightly more efficient. *)

67 68 69
let foreach_terminal_not_causing_an_error s f =
  match Invariant.has_default_reduction s with
  | Some _ ->
70
      (* There is a default reduction. No symbol causes an error. *)
71 72
      foreach_terminal f
  | None ->
73 74 75 76 77
      (* Enumerate every terminal symbol [z] for which there is a
         reduction. *)
      TerminalMap.iter (fun z _ ->
        if not (Terminal.equal z Terminal.error) then
          f z
78
      ) (Lr1.reductions s);
79 80
      (* Enumerate every terminal symbol [z] for which there is a
         transition. *)
81 82
      SymbolMap.iter (fun sym _ ->
        match sym with
83 84 85
        | Symbol.T z ->
            if not (Terminal.equal z Terminal.error) then
              f z
86 87 88 89
        | Symbol.N _ ->
            ()
      ) (Lr1.transitions s)

90 91
(* ------------------------------------------------------------------------ *)

POTTIER Francois's avatar
POTTIER Francois committed
92 93 94 95 96 97 98 99 100 101 102 103
(* Suppose [s] is a state that carries an outgoing edge labeled with a
   non-terminal symbol [nt]. We are interested in finding out how this edge
   can be taken. In order to do that, we must determine how, by starting in
   [s], one can follow a path that corresponds to (the right-hand side of) a
   production [prod] associated with [nt]. There are in general several such
   productions. The paths that they determine in the automaton form a "star".
   We represent the star rooted at [s] as a trie. For every state [s], the
   star rooted at [s] is constructed in advance, before the algorithm runs.
   While the algorithm runs, a point in the trie (that is, a sub-trie) tells
   us where we come form, where we are, and which production(s) we are hoping
   to reduce in the future. *)

104 105 106
module Trie : sig

  type trie
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122

  (* [star s] creates a (new) trie whose source is [s], populated with its
     branches. (There is one branch for every production [prod] associated
     with every non-terminal symbol [nt] for which [s] carries an outgoing
     edge.) If the star turns out to be trivial (i.e., without any branches)
     then [None] is returned. *)
  val star: Lr1.node -> trie option

  (* Every (sub-)trie has a unique identity. (One can think of it as its
     address.) [compare] compares the identity of two tries. This can be
     used, e.g., to set up a map whose keys are tries. *)
  val compare: trie -> trie -> int

  (* [source t] returns the source state of the (sub-)trie [t]. This is
     the root of the star of which [t] is a sub-trie. In other words, this
     tells us "where we come from". *)
123
  val source: trie -> Lr1.node
124 125 126 127
    
  (* [target t] returns the current state of the (sub-)trie [t]. This is
     the root of the sub-trie [t]. In other words, this tells us "where
     we are". *)
128 129
  val target: trie -> Lr1.node

130 131 132 133 134
  (* [accepts prod t] tells whether the current state of the trie [t] is
     the end of a branch associated with production [prod]. If so, this
     means that we have successfully followed a path that corresponds to
     the right-hand side of production [prod]. *)
  val accepts: Production.index -> trie -> bool
135

136 137 138 139 140 141 142 143
  (* [step sym t] is the immediate sub-trie of [t] along the symbol [sym].
     This function raises [Not_found] if [t] has no child labeled [sym]. *)
  val step: Symbol.t -> trie -> trie

  (* [verbose()] outputs debugging & performance information. *)
  val verbose: unit -> unit

end = struct
144

145 146
  (* A trie has the following structure. *)

147
  type trie = {
148 149 150
    (* A unique identity, used by [compare]. The trie construction code
       ensures that these numbers are indeed unique: see [fresh], [insert],
       [star]. *)
151
    identity: int;
152
    (* The root state of this star: "where we come from". *)
153
    source: Lr1.node;
154
    (* The current state, i.e., the root of this sub-trie: "where we are". *)
155
    target: Lr1.node;
156 157 158
    (* The productions that we can reduce in the current state. In other
       words, if this list is nonempty, then the current state is the end
       of one (or several) branches. It can nonetheless have children. *)
159
    productions: Production.index list;
160 161
    (* The children, or sub-tries. *)
    transitions: trie SymbolMap.t
162
  }
163

164
  (* This counter is used by [mktrie] to produce unique identities. *)
165 166
  let c = ref 0

167
  (* This smart constructor creates a new trie with a unique identity. *)
168
  let mktrie source target productions transitions =
169
    let identity = Misc.postincrement c in
170
    { identity; source; target; productions; transitions }
171

172 173
  (* TEMPORARY could insert this branch only if viable -- leads to 12600 instead of 12900 in ocaml.mly --lalr *)

174 175
  (* [insert] logically consumes its argument [t], which should no
     longer be used. *)
176
  let rec insert w prod t =
177 178
    match w with
    | [] ->
179 180 181 182
        (* We consume (update) the trie [t], so there is no need to allocate a
           new stamp. (Of course we could allocate a new stamp, but I prefer
           to be precise.) *)
        { t with productions = prod :: t.productions }
183
    | a :: w ->
184
        (* Check if there is a transition labeled [a] out of [t.target]. If
185 186 187 188
           there is, we add a child to the trie [t]. If there isn't, then it
           must have been removed by conflict resolution. (Indeed, it must be
           present in a canonical automaton.) The trie remains unchanged in
           this case. *)
189
        match SymbolMap.find a (Lr1.transitions t.target) with
190
        | successor ->
191 192 193 194 195 196 197 198
            (* Find our child at [a], or create it. *)
            let t' =
              try
                SymbolMap.find a t.transitions
              with Not_found ->
                mktrie t.source successor [] SymbolMap.empty
            in
            (* Update the child [t']. *)
199
            let t' = insert w prod t' in
200 201
            (* Update [t]. Again, no need to allocate a new stamp. *)
            { t with transitions = SymbolMap.add a t' t.transitions }
202 203 204
        | exception Not_found ->
            t

205 206 207 208 209
  (* [insert prod t] inserts a new branch, corresponding to production
     [prod], into the trie [t]. This function consumes its argument,
     which should no longer be used afterwards. *)
  let insert prod t =
    let w = Array.to_list (Production.rhs prod) in
210
    insert w prod t
211

212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
  (* [fresh s] creates a new empty trie whose source is [s]. *)
  let fresh source =
    mktrie source source [] SymbolMap.empty

  let star s =
    SymbolMap.fold (fun sym _ accu ->
      match sym with
      | Symbol.T _ ->
          accu
      | Symbol.N nt ->
          Production.foldnt nt accu insert
    ) (Lr1.transitions s) (fresh s)

  let nontrivial t =
    not (t.productions = [] && SymbolMap.is_empty t.transitions)

  let star s =
    let t = star s in
    if nontrivial t then
      Some t
    else
      None

235 236 237 238 239 240
  let source t =
    t.source

  let target t =
    t.target

241 242 243
  let accepts prod t =
    List.mem prod t.productions

244
  let step a t =
245
    SymbolMap.find a t.transitions (* careful: may raise [Not_found] *)
246

247 248
  let compare t1 t2 =
    Pervasives.compare (t1.identity : int) t2.identity
249

250
  let verbose () =
251
    Printf.fprintf stderr "Cumulated star size: %d\n%!" !c
252

253 254
end

255
type fact = {
256
  future: Trie.trie;
257
  word: W.word;
258
  lookahead: Terminal.t
259 260
}

261
let source fact =
262
  Trie.source fact.future
263 264

let target fact =
265
  Trie.target fact.future
266

267 268 269 270
let q =
  Q.create()

let add fact =
271 272
  (* assert (not (causes_an_error (target fact) fact.lookahead)); *)

273 274
  (* The length of the word serves as the priority of this fact. *)
  Q.add q fact (W.length fact.word)
POTTIER Francois's avatar
POTTIER Francois committed
275 276
    (* In principle, there is no need to insert the fact into the queue
       if [T] already stores a comparable fact. *)
277 278

let init s =
279 280 281 282 283 284 285 286 287 288 289
  match Trie.star s with
  | Some trie ->
      foreach_terminal_not_causing_an_error s (fun z ->
        add {
          future = trie;
          word = W.epsilon;
          lookahead = z
        }
      )
  | None ->
      ()
290 291

module T : sig
292 293

  (* [register fact] registers the fact [fact]. It returns [true] if this fact
294
     is new, i.e., no fact concerning the same quintuple of [source], [future],
295 296 297 298 299
     [target], [a], and [z] was previously known. *)
  val register: fact -> bool

  (* [query target z f] enumerates all known facts whose target state is [target]
     and whose lookahead assumption is [z]. *)
300
  val query: Lr1.node -> Terminal.t -> (fact -> unit) -> unit
301

302
  val verbose: unit -> unit
303

304
end = struct
305

306 307 308 309 310 311 312 313 314 315 316
  (* This module implements a set of facts. Two facts are considered equal
     (for the purposes of this set) if they have the same [future], [a], and
     [z] fields. The [word] is not considered. Indeed, we are not interested
     in keeping track of several words that produce the same effect. Only the
     shortest such word is of interest. *)

  (* We need to query the set of facts in two ways. In [register], we need to
     test whether a fact is in the set. In [query], we need to find all facts
     that match a pair [target, z]. For this reason, we use a two-level table.
     The first level is a matrix indexed by [target] and [z]. At the second
     level, we find sets of facts. *)
317 318 319
(**)

  module M =
320
    MySet.Make(struct
321 322 323
      type t = fact
      let compare fact1 fact2 =
        let c = Trie.compare fact1.future fact2.future in
324
        if c <> 0 then c else
325 326
        let a1 = W.first fact1.word fact1.lookahead
        and a2 = W.first fact2.word fact2.lookahead in
327 328
        Terminal.compare a1 a2
    end)
329

330 331 332 333 334
  let table = (* a pretty large table... *)
    Array.make (Lr1.n * Terminal.n) M.empty

  let index target z =
    Terminal.n * (Lr1.number target) + Terminal.t2i z
335

336 337
  let count = ref 0

338
  let register fact =
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357
    let target = target fact in
    let z = fact.lookahead in
    let i = index target z in
    let m = table.(i) in
    (* We crucially rely on the fact that [M.add] guarantees not to
       change the set if an ``equal'' fact already exists. Thus, a
       later, longer path is ignored in favor of an earlier, shorter
       path. *)
    let m' = M.add fact m in
    m != m' && begin
      incr count;
      table.(i) <- m';
      true
    end

  let query target z f =
    let i = index target z in
    let m = table.(i) in
    M.iter f m
358

359
  let verbose () =
360 361
    Printf.fprintf stderr "T stores %d facts.\n%!" !count

362 363 364 365 366 367 368 369 370
end

(* The module [E] is in charge of recording the non-terminal edges that we have
   discovered, or more precisely, the conditions under which these edges can be
   taken. *)

module E : sig

  (* [register s nt w z] records that, in state [s], the outgoing edge labeled
371 372 373
     [nt] can be taken by consuming the word [w], if the next symbol is [z].
     It returns [true] if this information is new. *)
  val register: Lr1.node -> Nonterminal.t -> W.word -> Terminal.t -> bool
374 375 376 377

  (* [query s nt a z] answers whether, in state [s], the outgoing edge labeled
     [nt] can be taken by consuming some word [w], under the assumption that
     the next symbol is [z], and under the constraint that the first symbol of
378 379
     [w.z] is [a]. *)
  val query: Lr1.node -> Nonterminal.t -> Terminal.t -> Terminal.t -> (W.word -> unit) -> unit
380

381
  val verbose: unit -> unit
382

383 384 385 386 387
end = struct

  (* For now, we implement a mapping of [s, nt, a, z] to [w]. *)

  module M =
388 389 390
    MySet.Make(struct
      type t = Nonterminal.t * Terminal.t * Terminal.t * W.word
      let compare (nt1, a1, z1, _w1) (nt2, a2, z2, _w2) =
391 392 393 394 395 396 397
        let c = Nonterminal.compare nt1 nt2 in
        if c <> 0 then c else
        let c = Terminal.compare a1 a2 in
        if c <> 0 then c else
        Terminal.compare z1 z2
    end)

398 399
  let table =
    Array.make Lr1.n M.empty
400

401 402
  let count = ref 0

403
  let register s nt w z =
404 405
    let i = Lr1.number s in
    let m = table.(i) in
406
    let a = W.first w z in
407
    let m' = M.add (nt, a, z, w) m in
408 409 410 411 412
    m != m' && begin
      incr count;
      table.(i) <- m';
      true
    end
413

414
  let query s nt a z f =
415 416
    let i = Lr1.number s in
    let m = table.(i) in
417 418 419
    let dummy = W.epsilon in
    match M.find (nt, a, z, dummy) m with
    | (_, _, _, w) -> f w
420
    | exception Not_found -> ()
421

422
  let verbose () =
423 424
    Printf.fprintf stderr "E stores %d facts.\n%!" !count

425 426
end

427
let new_edge s nt w z =
POTTIER Francois's avatar
POTTIER Francois committed
428 429 430 431
  (*
  Printf.fprintf stderr "Considering reduction on %s in state %d\n"
    (Terminal.print z) (Lr1.number s);
  *)
432
  if E.register s nt w z then
433
    let sym = Symbol.N nt in
434
    T.query s (W.first w z) (fun fact ->
435
      assert (Terminal.equal fact.lookahead (W.first w z));
436
      match Trie.step sym fact.future with
437
      | future ->
438
          if not (causes_an_error (Trie.target future) z) then
439 440 441 442 443 444 445
            add {
              future;
              word = W.append fact.word w;
              lookahead = z
            }
      | exception Not_found ->
          ()
446
    )
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466

(* [consequences fact] is invoked when we discover a new fact (i.e., one that
   was not previously known). It studies the consequences of this fact. These
   consequences are of two kinds:

   - As in Dijkstra's algorithm, the new fact can be viewed as a newly
   discovered vertex. We study its (currently known) outgoing edges,
   and enqueue new facts in the priority queue.

   - Sometimes, a fact can also be viewed as a newly discovered edge.
   This is the case when the word from [fact.source] to [fact.target]
   represents a production of the grammar and [fact.target] is willing
   to reduce this production. We record the existence of this edge,
   and re-inspect any previously discovered vertices which are
   interested in this outgoing edge.
*)
(**)

let consequences fact =

467 468 469
  let target = target fact in

  (* 1. View [fact] as a vertex. Examine the transitions out of [target]. *)
470
  
471
  SymbolMap.iter (fun sym s' ->
472
    match Trie.step sym fact.future, sym with
473 474 475
    | exception Not_found -> ()
    | future, Symbol.T t ->

476
        (* 1a. There is a transition labeled [t] out of [target]. If
477 478 479 480 481 482 483 484
           the lookahead assumption [fact.lookahead] is compatible with [t],
           then we derive a new fact, where one more edge has been taken. We
           enqueue this new fact for later examination. *)
        (**)

        if Terminal.equal fact.lookahead t then
          let word = W.append fact.word (W.singleton t) in
          (* assert (Lr1.Node.compare future.Trie.target s' = 0); *)
485
          foreach_terminal_not_causing_an_error s' (fun z ->
486
            add { future; word; lookahead = z }
487
          )
488 489 490

    | future, Symbol.N nt ->

491
        (* 1b. There is a transition labeled [nt] out of [target]. We
492 493 494 495 496 497 498 499 500
           need to know how this nonterminal edge can be taken. We query for a
           word [w] that allows us to take this edge. The answer depends on
           the terminal symbol [z] that comes *after* this word: we try all
           such symbols. Furthermore, we need the first symbol of [w.z] to
           satisfy the lookahead assumption [fact.lookahead], so the answer
           also depends on this assumption. *)
        (**)

        foreach_terminal_not_causing_an_error s' (fun z ->
501
          E.query target nt fact.lookahead z (fun w ->
502 503 504 505 506 507 508 509
            assert (Terminal.equal fact.lookahead (W.first w z));
            add {
              future;
              word = W.append fact.word w;
              lookahead = z
            }
          )
        )
510

511
  ) (Lr1.transitions target);
512 513

  (* 2. View [fact] as a possible edge. This is possible if the path from
514 515
     [fact.source] to [target] represents a production [prod] and
     [target] is willing to reduce this production. We check that
516 517 518 519 520
     [fact.future] accepts [epsilon]. This guarantees that reducing [prod]
     takes us all the way back to [fact.source]. Thus, this production gives
     rise to an edge labeled [nt] -- the left-hand side of [prod] -- out of
     [fact.source]. This edge is subject to the lookahead assumption
     [fact.lookahead], so we record that. *)
521 522
  (**)

523
  match has_reduction target fact.lookahead with
524
  | Some prod when Trie.accepts prod fact.future ->
525
      new_edge (source fact) (Production.nt prod) fact.word fact.lookahead
526 527 528
  | _ ->
      ()

529
let level = ref 0
530

POTTIER Francois's avatar
POTTIER Francois committed
531 532
let done_with_level () =
  Printf.fprintf stderr "Done with level %d.\n" !level;
533 534
  T.verbose();
  E.verbose();
POTTIER Francois's avatar
POTTIER Francois committed
535 536
  Printf.fprintf stderr "Q stores %d facts.\n%!" (Q.cardinal q)

537
let discover fact =
538
  if T.register fact then begin
539
    if W.length fact.word > ! level then begin
POTTIER Francois's avatar
POTTIER Francois committed
540
      done_with_level();
541 542
      level := W.length fact.word;
    end;
543
    consequences fact
544
  end
545

POTTIER Francois's avatar
POTTIER Francois committed
546
let () =
547
  Lr1.iter init;
548
  Trie.verbose();
549
  Q.repeat q discover;
550
  Time.tick "Running LRijkstra";
POTTIER Francois's avatar
POTTIER Francois committed
551
  done_with_level()
552

553 554
(* ------------------------------------------------------------------------ *)

555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584
(* The following code validates the fact that an error can be triggered in
   state [s'] by beginning in the initial state [s] and reading the
   sequence of terminal symbols [w]. We use this for debugging purposes. *)

let fail msg =
  Printf.fprintf stderr "coverage: internal error: %s.\n%!" msg;
  false

open ReferenceInterpreter

let validate s s' w : bool =
  match
    ReferenceInterpreter.check_error_path (Lr1.nt_of_entry s) (W.elements w)
  with
  | OInputReadPastEnd ->
      fail "input was read past its end"
  | OInputNotFullyConsumed ->
      fail "input was not fully consumed"
  | OUnexpectedAccept ->
      fail "input was unexpectedly accepted"
  | OK state ->
      Lr1.Node.compare state s' = 0 ||
      fail (
        Printf.sprintf "error occurred in state %d instead of %d"
          (Lr1.number state)
          (Lr1.number s')
      )

(* ------------------------------------------------------------------------ *)

585 586 587 588 589 590 591
(* We now wish to determine, given a state [s'] and a terminal symbol [z], a
   minimal path that takes us from some entry state to state [s'] with [z] as
   the next (unconsumed) symbol. *)

(* This can be formulated as a search for a shortest path in a graph. The
   graph is not just the automaton, though. It is a (much) larger graph whose
   vertices are pairs [s, z] and whose edges are obtained by querying the
592
   module [E] above. *)
593

594
let forward () =
595

596
  let module A = Astar.Make(struct
597

598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618
    (* A vertex is a pair [s, z].
       [z] cannot be the [error] token. *)
    type node =
        Lr1.node * Terminal.t

    let equal (s'1, z1) (s'2, z2) =
      Lr1.Node.compare s'1 s'2 = 0 && Terminal.compare z1 z2 = 0

    let hash (s, z) =
      Hashtbl.hash (Lr1.number s, z)

    (* An edge is labeled with a word. *)
    type label =
      W.word

    (* Forward search from every [s, z], where [s] is an initial state. *)
    let sources f =
      foreach_terminal (fun z ->
        ProductionMap.iter (fun _ s ->
          f (s, z)
        ) Lr1.entry
619 620
      )

621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645
    let successors (s, z) edge =
      assert (not (Terminal.equal z Terminal.error));
      SymbolMap.iter (fun sym s' ->
        match sym with
        | Symbol.T t ->
            if Terminal.equal z t then
              let w = W.singleton t in
              foreach_terminal (fun z ->
                edge w 1 (s', z)
              )
        | Symbol.N nt ->
           foreach_terminal (fun z' ->
             E.query s nt z z' (fun w ->
               edge w (W.length w) (s', z')
             )
           )
      ) (Lr1.transitions s)

    let estimate _ =
      0

  end) in

  (* Search forward. *)

646
  Printf.fprintf stderr "Forward search:\n%!";
647
  let seen = ref Lr1.NodeSet.empty in
648
  let _, _ = A.search (fun ((s', z), path) ->
649 650 651 652 653 654 655 656 657 658 659 660 661 662
    if causes_an_error s' z && not (Lr1.NodeSet.mem s' !seen) then begin
      seen := Lr1.NodeSet.add s' !seen;
      (* An error can be triggered in state [s'] by beginning in the initial
         state [s] and reading the sequence of terminal symbols [w]. *)
      let (s, _), ws = A.reverse path in
      let w = List.fold_right W.append ws (W.singleton z) in
      Printf.fprintf stderr
        "An error can be reached from state %d to state %d:\n%!"
        (Lr1.number s)
        (Lr1.number s');
      Printf.fprintf stderr "%s\n%!" (W.print w);
      assert (validate s s' w)
    end
  ) in
663 664 665
  Printf.fprintf stderr "Reachable (forward): %d states\n%!"
    (Lr1.NodeSet.cardinal !seen);
  !seen
666

667
(* TEMPORARY the code in this module should run only if --coverage is set *)
668 669

let () =
670 671
  let f = forward() in
  Time.tick "Forward search";
672
  ignore f
673 674 675 676 677 678

(* TODO:
  subject to --coverage
  write to .coverage file
  collect performance data, correlated with star size and alphabet size; draw a graph
*)