Commit e190f6a7 authored by POTTIER Francois's avatar POTTIER Francois
Browse files

Impose a timeout on positive tests, and modify [make data] to ignore a test that timed out.

parent 0dfba3a3
......@@ -44,26 +44,32 @@ test:
# in parallel on a heavily-loaded machine. Furthermore, this data is
# machine-dependent. The data is written to analysis/data.csv.
# A test that caused a TIMEOUT is excluded from the data.
# On MacOS, we require gsed instead of sed.
SED=$(shell if [[ "$$OSTYPE" == "darwin"* ]] ; then echo gsed ; else echo sed ; fi)
.PHONY: data
data: test
@ echo "Collecting data (using $(SED))..." && \
( \
echo "name,terminals,nonterminals,lr0states,lr1states,lr1time" && \
cd _build/default/test/static/src && \
for timings in *.out.timings ; do \
echo "name,terminals,nonterminals,lr0states,lr1states,lr1time" > analysis/data.csv && \
successful=0 && timedout=0 && \
for timings in _build/default/test/static/src/*.out.timings ; do \
name=$${timings%.out.timings} ; \
out=$$name.out ; \
terminals=`$(SED) -n -e "s/^Grammar has \([0-9]\+\) terminal symbols./\1/p" $$out` ; \
nonterminals=`$(SED) -n -e "s/^Grammar has \([0-9]\+\) nonterminal symbols, among which [0-9]\+ start symbols./\1/p" $$out` ; \
lr0states=`$(SED) -n -e "s/^Built an LR(0) automaton with \([0-9]\+\) states./\1/p" $$out` ; \
lr1states=`$(SED) -n -e "s/^Built an LR(1) automaton with \([0-9]\+\) states./\1/p" $$out` ; \
lr1time=`$(SED) -n -e "s/^Construction of the LR(1) automaton: \(.*\)s/\1/p" $$timings` ; \
echo "$$name,$$terminals,$$nonterminals,$$lr0states,$$lr1states,$$lr1time" ; \
done \
) > analysis/data.csv
if grep --quiet "TIMEOUT after" $$out ; then \
((timedout++)) ; \
else \
((successful++)) ; \
terminals=`$(SED) -n -e "s/^Grammar has \([0-9]\+\) terminal symbols./\1/p" $$out` ; \
nonterminals=`$(SED) -n -e "s/^Grammar has \([0-9]\+\) nonterminal symbols, among which [0-9]\+ start symbols./\1/p" $$out` ; \
lr0states=`$(SED) -n -e "s/^Built an LR(0) automaton with \([0-9]\+\) states./\1/p" $$out` ; \
lr1states=`$(SED) -n -e "s/^Built an LR(1) automaton with \([0-9]\+\) states./\1/p" $$out` ; \
lr1time=`$(SED) -n -e "s/^Construction of the LR(1) automaton: \(.*\)s/\1/p" $$timings` ; \
echo "$$name,$$terminals,$$nonterminals,$$lr0states,$$lr1states,$$lr1time" >> analysis/data.csv ; \
fi \
done && \
echo "$$successful successful tests; $$timedout timed out tests."
clean::
@ rm -f analysis/data.csv
......
This diff is collapsed.
......@@ -96,12 +96,21 @@ let atoms =
let rec print ppf = function
| A s ->
Format.pp_print_string ppf s
| L l ->
Format.fprintf ppf "@[<2>(%a)@]"
(Format.pp_print_list ~pp_sep:Format.pp_print_space print) l
| Lnewline l ->
Format.fprintf ppf "@[<v 2>(%a)@]"
(Format.pp_print_list ~pp_sep:Format.pp_print_space print) l
| L es ->
Format.fprintf ppf "@[<2>(%a)@]" print_list es
| Lnewline es ->
Format.fprintf ppf "@[<v 2>(%a)@]" print_list es
and print_list ppf es =
Format.pp_print_list ~pp_sep:Format.pp_print_space print ppf es
let show_list es =
let ppf = Format.str_formatter in
List.iter (fun e ->
Format.fprintf ppf " ";
print ppf e
) es;
Format.flush_str_formatter()
let print sexp =
Format.printf "@[<v>%a@,@]" print sexp;
......@@ -176,12 +185,33 @@ let extra source id =
else
atoms extra
(* The Menhir command. *)
(* This command is meant to be used inside a rule. *)
let menhir base flags =
L(A"run" :: A"menhir" :: base @ flags @ [A"%{deps}"])
(* The Menhir command, for use inside a rule, with an optional timeout. *)
(* If the [timeout] parameter is [true], we assume that a [timeout] command
exists on the system, and we use it to limit Menhir's execution time. This
is normally not necessary, but can be useful when testing experimental
extensions of Menhir. This should be used for positive tests only. *)
let threshold =
60 (* in seconds *)
let menhir_args base flags =
base @ flags @ A"%{deps}" :: []
let menhir (timeout : bool) base flags =
if timeout then
(* We must use a [system] action. *)
let command =
sprintf
"timeout %d %%{bin:menhir}%s || echo 'TIMEOUT after %d seconds.'"
threshold
(show_list (menhir_args base flags))
threshold
in
L[A"system"; A (sprintf "\"%s\"" command)]
else
(* We can use a [run] action. *)
L(A"run" :: A"menhir" :: menhir_args base flags)
(* Constructing (and printing) a pair of rules to run Menhir and compare its
output against an expected-output file.
......@@ -201,13 +231,14 @@ let menhir base flags =
let run_and_compare id positive source basenames outputs expected flags =
let output = hd outputs in
let timeout = positive in (* set up a timeout for positive tests *)
(* Run Menhir. *)
print (rule
outputs
(source // mlys basenames)
(redirect output (chdir source (
possibly_expecting_failure positive (
menhir (base basenames) flags
menhir timeout (base basenames) flags
)))));
(* Check that the output coincides with what was expected. *)
print (phony id (
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment