ce-bench: change oracles directory. Add possibility to xfail tests for ce

59a18fe3 · Sylvain Dailler · 25411b40 · 59a18fe3 · 59a18fe3 · 59a18fe3
Commit 59a18fe3 authored Oct 24, 2018 by Sylvain Dailler
101 changed files
--- a/bench/ce-bench
+++ b/bench/ce-bench
@@ -26,28 +26,43 @@ if test "$files" = "" ; then
    files="$dir/ce/*.mlw"
 fi

+is_xfail=false

-# $1 = prover, $2 = file
+# TODO this function should disappear as counterexamples should eventually get
+# more reproducible.
+decide_fail () {
+    case "$1" in
+        # Inconsistent results between two runs
+        "bench/ce/algebraic_type_Z3,4.6.0_SP")
+            is_xfail=true;;
+        # Inconsistent results between two runs
+        "bench/ce/algebraic_type_Z3,4.6.0_WP")
+            is_xfail=true;;
+        # Inconsistent results between two runs
+        "bench/ce/ref_mono_Z3,4.6.0_WP")
+            is_xfail=true;;
+    esac
+}
+
+# $1 = prover, $2 = dir, $3 = filename, $4 = true for WP; false for SP
 run () {
+    is_xfail=false
    printf "  $2 ($1)... "
-    f="$2_$1"
-    printf "WP  $2 ($1)... "
+    file_path="$2/$3"
+    if $4; then
+        f="${file_path}_$1_WP"
+        oracle_file="$2/oracles/$3_$1_WP.oracle"
+        wp_sp=""
        echo "Weakest Precondition" > "$f.out"
-    $dir/../bin/why3prove.opt --debug print_model_attrs -P "$1,counterexamples" --timelimit 1 -a split_vc "$2.mlw" | \
-        # This ad hoc sed removes any timing information from counterexamples output.
-        # Counterexamples in JSON format cannot match this regexp.
-        sed 's/ ([0-9]\+\.[0-9]\+s)//' | \
-        # This ad hoc sed removes diff between Timeout and Unknown (unknown)
-        # when running from platform with different speed.
-        sed -r 's/(Timeout|Unknown \(unknown\))/Timeout or Unknown/' | \
-        # ad hoc sed to remove the directory of the stdlib that can be found
-        # inside labels attribute [@at:'Old:loc:/home/.../stdlib/array.mlw
-        # TODO temporary as this comes from incorrect locations generated in
-        # a[i] <- x like terms.
-        sed -r 's/\:loc\:.*\]/\:loc\:location\]/' >> "$f.out"
-    printf "SP  $2 ($1)... "
-    echo "Strongest Postcondition" >> "$f.out"
-    $dir/../bin/why3prove.opt --debug print_model_attrs --debug vc_sp -P "$1,counterexamples" --timelimit 1 -a split_vc "$2.mlw" | \
+        printf "Weakest Precondition  ${file_path} ($1)... "
+    else
+        f="${file_path}_$1_SP"
+        oracle_file="$2/oracles/$3_$1_SP.oracle"
+        wp_sp=" --debug vc_sp"
+        echo "Strongest Postcondition" > "$f.out"
+        printf "Strongest Postcondition  ${file_path} ($1)... "
+    fi
+    $dir/../bin/why3prove.opt --debug print_model_attrs${wp_sp} -P "$1,counterexamples" --timelimit 1 -a split_vc "${file_path}.mlw" | \
        # This ad hoc sed removes any timing information from counterexamples output.
        # Counterexamples in JSON format cannot match this regexp.
        sed 's/ ([0-9]\+\.[0-9]\+s)//' | \
@@ -59,27 +74,35 @@ run () {
        # TODO temporary as this comes from incorrect locations generated in
        # a[i] <- x like terms.
        sed -r 's/\:loc\:.*\]/\:loc\:location\]/' >> "$f.out"
-    if cmp "$f.oracle" "$f.out" > /dev/null 2>&1 ; then
+    if cmp "${oracle_file}" "$f.out" > /dev/null 2>&1 ; then
        echo "ok"
    else
        if $updateoracle; then
-            echo "Updating oracle for $2, prover $1"
-            mv "$f.out" "$f.oracle"
+            echo "Updating oracle for ${file_path}, prover $1"
+            mv "$f.out" "${oracle_file}"
        else
            echo "FAILED!"
            echo "diff is the following:"
-            diff "$f.oracle" "$f.out"
+            echo "${f}"
+            diff "${oracle_file}" "$f.out"
+            decide_fail ${f}
+            if $is_xfail; then
+                echo "Failed but bench is ok because asserted as an XFAIL."
+            else
                success=false
            fi
        fi
+    fi
 }

 for file in $files; do
    filedir=`dirname $file`
    filebase=`basename $file .mlw`
    printf "Running provers on $filedir/$filebase.mlw\n";
-    run CVC4,1.5 $filedir/$filebase
-    run Z3,4.6.0 $filedir/$filebase
+    run CVC4,1.5 $filedir $filebase true
+    run CVC4,1.5 $filedir $filebase false
+    run Z3,4.6.0 $filedir $filebase true
+    run Z3,4.6.0 $filedir $filebase false
 done
 if $success; then
    exit 0

--- a/bench/ce/algebraic_type_CVC4,1.5.oracle
+++ b/bench/ce/algebraic_type_CVC4,1.5.oracle
-Weakest Precondition
-bench/ce/algebraic_type.mlw M G: Timeout or Unknown
-Counter-example model:File algebraic_type.mlw:
-Line 6:
-x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
-"val" : {"apply" : "Integer" , "list" : [{"type" : "Integer" ,
-"val" : "0" }] } }
-
-bench/ce/algebraic_type.mlw M g2: Timeout or Unknown
-Counter-example model:File algebraic_type.mlw:
-Line 10:
-x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
-"val" : {"apply" : "B" , "list" : [{"type" : "Integer" ,
-"val" : "0" }] } }
-
-bench/ce/algebraic_type.mlw M g4: Timeout or Unknown
-Counter-example model:File algebraic_type.mlw:
-Line 12:
-x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
-"val" : {"apply" : "A" ,
-"list" : [] } }
-
-bench/ce/algebraic_type.mlw M g5: Timeout or Unknown
-Counter-example model:File algebraic_type.mlw:
-Line 16:
-x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
-"val" : {"apply" : "Au" , "list" : [{"type" : "Integer" , "val" : "0" },
-{"type" : "Integer" ,
-"val" : "1" }] } }
-
-bench/ce/algebraic_type.mlw M g1: Timeout or Unknown
-Counter-example model:File algebraic_type.mlw:
-Line 25:
-l, [[@introduced], [@model_trace:l]] = {"type" : "Apply" ,
-"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "-1" },
-{"type" : "Apply" , "val" : {"apply" : "Nil" ,
-"list" : [] } }] } }
-
-bench/ce/algebraic_type.mlw M g7: Timeout or Unknown
-Counter-example model:File algebraic_type.mlw:
-Line 27:
-l, [[@introduced], [@model_trace:l]] = {"type" : "Apply" ,
-"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "0" },
-{"type" : "Apply" , "val" : {"apply" : "Nil" ,
-"list" : [] } }] } }
-
-bench/ce/algebraic_type.mlw M g: Timeout or Unknown
 Strongest Postcondition
 bench/ce/algebraic_type.mlw M G: Timeout or Unknown
 Counter-example model:File algebraic_type.mlw:

--- a/bench/ce/oracles/algebraic_type_CVC4,1.5_WP.oracle
+++ b/bench/ce/oracles/algebraic_type_CVC4,1.5_WP.oracle
+Weakest Precondition
+bench/ce/algebraic_type.mlw M G: Timeout or Unknown
+Counter-example model:File algebraic_type.mlw:
+Line 6:
+x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
+"val" : {"apply" : "Integer" , "list" : [{"type" : "Integer" ,
+"val" : "0" }] } }
+
+bench/ce/algebraic_type.mlw M g2: Timeout or Unknown
+Counter-example model:File algebraic_type.mlw:
+Line 10:
+x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
+"val" : {"apply" : "B" , "list" : [{"type" : "Integer" ,
+"val" : "0" }] } }
+
+bench/ce/algebraic_type.mlw M g4: Timeout or Unknown
+Counter-example model:File algebraic_type.mlw:
+Line 12:
+x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
+"val" : {"apply" : "A" ,
+"list" : [] } }
+
+bench/ce/algebraic_type.mlw M g5: Timeout or Unknown
+Counter-example model:File algebraic_type.mlw:
+Line 16:
+x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
+"val" : {"apply" : "Au" , "list" : [{"type" : "Integer" , "val" : "0" },
+{"type" : "Integer" ,
+"val" : "1" }] } }
+
+bench/ce/algebraic_type.mlw M g1: Timeout or Unknown
+Counter-example model:File algebraic_type.mlw:
+Line 25:
+l, [[@introduced], [@model_trace:l]] = {"type" : "Apply" ,
+"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "-1" },
+{"type" : "Apply" , "val" : {"apply" : "Nil" ,
+"list" : [] } }] } }
+
+bench/ce/algebraic_type.mlw M g7: Timeout or Unknown
+Counter-example model:File algebraic_type.mlw:
+Line 27:
+l, [[@introduced], [@model_trace:l]] = {"type" : "Apply" ,
+"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "0" },
+{"type" : "Apply" , "val" : {"apply" : "Nil" ,
+"list" : [] } }] } }
+
+bench/ce/algebraic_type.mlw M g: Timeout or Unknown
--- a/bench/ce/algebraic_type_Z3,4.6.0.oracle
+++ b/bench/ce/algebraic_type_Z3,4.6.0.oracle
-Weakest Precondition
-bench/ce/algebraic_type.mlw M G: Unknown (sat)
-Counter-example model:File algebraic_type.mlw:
-Line 6:
-x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
-"val" : {"apply" : "Integer" , "list" : [{"type" : "Integer" ,
-"val" : "0" }] } }
-
-bench/ce/algebraic_type.mlw M g2: Unknown (sat)
-Counter-example model:File algebraic_type.mlw:
-Line 10:
-x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
-"val" : {"apply" : "B" , "list" : [{"type" : "Integer" ,
-"val" : "0" }] } }
-
-bench/ce/algebraic_type.mlw M g4: Unknown (sat)
-Counter-example model:File algebraic_type.mlw:
-Line 12:
-x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
-"val" : {"apply" : "A" ,
-"list" : [] } }
-
-bench/ce/algebraic_type.mlw M g5: Unknown (sat)
-Counter-example model:File algebraic_type.mlw:
-Line 16:
-x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
-"val" : {"apply" : "Au" , "list" : [{"type" : "Integer" , "val" : "1" },
-{"type" : "Integer" ,
-"val" : "0" }] } }
-
-bench/ce/algebraic_type.mlw M g1: Timeout or Unknown
-Counter-example model:File algebraic_type.mlw:
-Line 25:
-l, [[@introduced], [@model_trace:l]] = {"type" : "Apply" ,
-"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "17" },
-{"type" : "Apply" , "val" : {"apply" : "Cons" ,
-"list" : [{"type" : "Integer" , "val" : "164" }, {"type" : "Apply" ,
-"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "162" },
-{"type" : "Apply" , "val" : {"apply" : "Cons" ,
-"list" : [{"type" : "Integer" , "val" : "160" }, {"type" : "Apply" ,
-"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "158" },
-{"type" : "Apply" , "val" : {"apply" : "Nil" ,
-"list" : [] } }] } }] } }] } }] } }] } }
-
-bench/ce/algebraic_type.mlw M g7: Timeout or Unknown
-Counter-example model:File algebraic_type.mlw:
-Line 27:
-l, [[@introduced], [@model_trace:l]] = {"type" : "Apply" ,
-"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "21" },
-{"type" : "Apply" , "val" : {"apply" : "Cons" ,
-"list" : [{"type" : "Integer" , "val" : "10" }, {"type" : "Apply" ,
-"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "9" },
-{"type" : "Apply" , "val" : {"apply" : "Cons" ,
-"list" : [{"type" : "Integer" , "val" : "7" }, {"type" : "Apply" ,
-"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "8" },
-{"type" : "Apply" , "val" : {"apply" : "Nil" ,
-"list" : [] } }] } }] } }] } }] } }] } }
-
-bench/ce/algebraic_type.mlw M g: Timeout or Unknown
 Strongest Postcondition
 bench/ce/algebraic_type.mlw M G: Unknown (sat)
 Counter-example model:File algebraic_type.mlw:
@@ -93,11 +34,11 @@ Line 25:
 l, [[@introduced], [@model_trace:l]] = {"type" : "Apply" ,
 "val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "17" },
 {"type" : "Apply" , "val" : {"apply" : "Cons" ,
-"list" : [{"type" : "Integer" , "val" : "164" }, {"type" : "Apply" ,
-"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "162" },
-{"type" : "Apply" , "val" : {"apply" : "Cons" ,
 "list" : [{"type" : "Integer" , "val" : "160" }, {"type" : "Apply" ,
 "val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "158" },
+{"type" : "Apply" , "val" : {"apply" : "Cons" ,
+"list" : [{"type" : "Integer" , "val" : "156" }, {"type" : "Apply" ,
+"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "154" },
 {"type" : "Apply" , "val" : {"apply" : "Nil" ,
 "list" : [] } }] } }] } }] } }] } }] } }


--- a/bench/ce/oracles/algebraic_type_Z3,4.6.0_WP.oracle
+++ b/bench/ce/oracles/algebraic_type_Z3,4.6.0_WP.oracle
+Weakest Precondition
+bench/ce/algebraic_type.mlw M G: Unknown (sat)
+Counter-example model:File algebraic_type.mlw:
+Line 6:
+x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
+"val" : {"apply" : "Integer" , "list" : [{"type" : "Integer" ,
+"val" : "0" }] } }
+
+bench/ce/algebraic_type.mlw M g2: Unknown (sat)
+Counter-example model:File algebraic_type.mlw:
+Line 10:
+x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
+"val" : {"apply" : "B" , "list" : [{"type" : "Integer" ,
+"val" : "0" }] } }
+
+bench/ce/algebraic_type.mlw M g4: Unknown (sat)
+Counter-example model:File algebraic_type.mlw:
+Line 12:
+x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
+"val" : {"apply" : "A" ,
+"list" : [] } }
+
+bench/ce/algebraic_type.mlw M g5: Unknown (sat)
+Counter-example model:File algebraic_type.mlw:
+Line 16:
+x, [[@introduced], [@model_trace:x]] = {"type" : "Apply" ,
+"val" : {"apply" : "Au" , "list" : [{"type" : "Integer" , "val" : "1" },
+{"type" : "Integer" ,
+"val" : "0" }] } }
+
+bench/ce/algebraic_type.mlw M g1: Timeout or Unknown
+Counter-example model:File algebraic_type.mlw:
+Line 25:
+l, [[@introduced], [@model_trace:l]] = {"type" : "Apply" ,
+"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "17" },
+{"type" : "Apply" , "val" : {"apply" : "Cons" ,
+"list" : [{"type" : "Integer" , "val" : "165" }, {"type" : "Apply" ,
+"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "163" },
+{"type" : "Apply" , "val" : {"apply" : "Cons" ,
+"list" : [{"type" : "Integer" , "val" : "161" }, {"type" : "Apply" ,
+"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "159" },
+{"type" : "Apply" , "val" : {"apply" : "Nil" ,
+"list" : [] } }] } }] } }] } }] } }] } }
+
+bench/ce/algebraic_type.mlw M g7: Timeout or Unknown
+Counter-example model:File algebraic_type.mlw:
+Line 27:
+l, [[@introduced], [@model_trace:l]] = {"type" : "Apply" ,
+"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "21" },
+{"type" : "Apply" , "val" : {"apply" : "Cons" ,
+"list" : [{"type" : "Integer" , "val" : "10" }, {"type" : "Apply" ,
+"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "9" },
+{"type" : "Apply" , "val" : {"apply" : "Cons" ,
+"list" : [{"type" : "Integer" , "val" : "7" }, {"type" : "Apply" ,
+"val" : {"apply" : "Cons" , "list" : [{"type" : "Integer" , "val" : "8" },
+{"type" : "Apply" , "val" : {"apply" : "Nil" ,
+"list" : [] } }] } }] } }] } }] } }] } }
+
+bench/ce/algebraic_type.mlw M g: Timeout or Unknown
--- a/bench/ce/oracles/array_mono_CVC4,1.5_SP.oracle
+++ b/bench/ce/oracles/array_mono_CVC4,1.5_SP.oracle
+Strongest Postcondition
+bench/ce/array_mono.mlw Array VC array: Valid
+bench/ce/array_mono.mlw A VC f1: Timeout or Unknown
+Counter-example model:File array_mono.mlw:
+Line 35:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
+"val" : [{"others" : {"type" : "Integer" , "val" : "0" } }] } },
+{"field" : "length" , "value" : {"type" : "Integer" ,
+"val" : "0" } }] } }
+Line 36:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
+"val" : [{"others" : {"type" : "Integer" , "val" : "0" } }] } },
+{"field" : "length" , "value" : {"type" : "Integer" ,
+"val" : "0" } }] } }
+
+bench/ce/array_mono.mlw A VC f2: Valid
+bench/ce/array_mono.mlw A VC f2: Timeout or Unknown
+Counter-example model:File array_mono.mlw:
+Line 38:
+a, [[@introduced], [@model_trace:a],
+[@at:'Old:loc:location],
+[@at:'Old:loc:location] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
+"val" : [{"indice" : "0" , "value" : {"type" : "Integer" , "val" : "42" } },
+{"indice" : "1" , "value" : {"type" : "Integer" , "val" : "42" } },
+{"others" : {"type" : "Integer" , "val" : "0" } }] } }, {"field" : "length" ,
+"value" : {"type" : "Integer" ,
+"val" : "2" } }] } }
+Line 40:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
+"val" : [{"indice" : "0" , "value" : {"type" : "Integer" , "val" : "42" } },
+{"indice" : "1" , "value" : {"type" : "Integer" , "val" : "42" } },
+{"others" : {"type" : "Integer" , "val" : "0" } }] } }, {"field" : "length" ,
+"value" : {"type" : "Integer" ,
+"val" : "2" } }] } }
+
--- a/bench/ce/array_mono_CVC4,1.5.oracle
+++ b/bench/ce/array_mono_CVC4,1.5.oracle
@@ -45,42 +45,3 @@ a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
 "value" : {"type" : "Integer" ,
 "val" : "2" } }] } }

-Strongest Postcondition
-bench/ce/array_mono.mlw Array VC array: Valid
-bench/ce/array_mono.mlw A VC f1: Timeout or Unknown
-Counter-example model:File array_mono.mlw:
-Line 35:
-a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
-"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
-"val" : [{"others" : {"type" : "Integer" , "val" : "0" } }] } },
-{"field" : "length" , "value" : {"type" : "Integer" ,
-"val" : "0" } }] } }
-Line 36:
-a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
-"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
-"val" : [{"others" : {"type" : "Integer" , "val" : "0" } }] } },
-{"field" : "length" , "value" : {"type" : "Integer" ,
-"val" : "0" } }] } }
-
-bench/ce/array_mono.mlw A VC f2: Valid
-bench/ce/array_mono.mlw A VC f2: Timeout or Unknown
-Counter-example model:File array_mono.mlw:
-Line 38:
-a, [[@introduced], [@model_trace:a],
-[@at:'Old:loc:location],
-[@at:'Old:loc:location] = {"type" : "Record" ,
-"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
-"val" : [{"indice" : "0" , "value" : {"type" : "Integer" , "val" : "42" } },
-{"indice" : "1" , "value" : {"type" : "Integer" , "val" : "42" } },
-{"others" : {"type" : "Integer" , "val" : "0" } }] } }, {"field" : "length" ,
-"value" : {"type" : "Integer" ,
-"val" : "2" } }] } }
-Line 40:
-a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
-"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
-"val" : [{"indice" : "0" , "value" : {"type" : "Integer" , "val" : "42" } },
-{"indice" : "1" , "value" : {"type" : "Integer" , "val" : "42" } },
-{"others" : {"type" : "Integer" , "val" : "0" } }] } }, {"field" : "length" ,
-"value" : {"type" : "Integer" ,
-"val" : "2" } }] } }
-
--- a/bench/ce/oracles/array_mono_Z3,4.6.0_SP.oracle
+++ b/bench/ce/oracles/array_mono_Z3,4.6.0_SP.oracle
+Strongest Postcondition
+bench/ce/array_mono.mlw Array VC array: Valid
+bench/ce/array_mono.mlw A VC f1: Timeout or Unknown
+Counter-example model:File array_mono.mlw:
+Line 35:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
+"val" : [{"others" : {"type" : "Integer" , "val" : "0" } }] } },
+{"field" : "length" , "value" : {"type" : "Integer" ,
+"val" : "0" } }] } }
+Line 36:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
+"val" : [{"others" : {"type" : "Integer" , "val" : "0" } }] } },
+{"field" : "length" , "value" : {"type" : "Integer" ,
+"val" : "0" } }] } }
+
+bench/ce/array_mono.mlw A VC f2: Valid
+bench/ce/array_mono.mlw A VC f2: Timeout or Unknown
+Counter-example model:File array_mono.mlw:
+Line 38:
+a, [[@introduced], [@model_trace:a],
+[@at:'Old:loc:location],
+[@at:'Old:loc:location] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
+"val" : [{"indice" : "0" , "value" : {"type" : "Integer" , "val" : "42" } },
+{"others" : {"type" : "Integer" , "val" : "4" } }] } }, {"field" : "length" ,
+"value" : {"type" : "Integer" ,
+"val" : "2" } }] } }
+Line 40:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
+"val" : [{"indice" : "0" , "value" : {"type" : "Integer" , "val" : "42" } },
+{"others" : {"type" : "Integer" , "val" : "4" } }] } }, {"field" : "length" ,
+"value" : {"type" : "Integer" ,
+"val" : "2" } }] } }
+
--- a/bench/ce/array_mono_Z3,4.6.0.oracle
+++ b/bench/ce/array_mono_Z3,4.6.0.oracle
@@ -42,40 +42,3 @@ a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
 "value" : {"type" : "Integer" ,
 "val" : "2" } }] } }

-Strongest Postcondition
-bench/ce/array_mono.mlw Array VC array: Valid
-bench/ce/array_mono.mlw A VC f1: Timeout or Unknown
-Counter-example model:File array_mono.mlw:
-Line 35:
-a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
-"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
-"val" : [{"others" : {"type" : "Integer" , "val" : "0" } }] } },
-{"field" : "length" , "value" : {"type" : "Integer" ,
-"val" : "0" } }] } }
-Line 36:
-a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
-"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
-"val" : [{"others" : {"type" : "Integer" , "val" : "0" } }] } },
-{"field" : "length" , "value" : {"type" : "Integer" ,
-"val" : "0" } }] } }
-
-bench/ce/array_mono.mlw A VC f2: Valid
-bench/ce/array_mono.mlw A VC f2: Timeout or Unknown
-Counter-example model:File array_mono.mlw:
-Line 38:
-a, [[@introduced], [@model_trace:a],
-[@at:'Old:loc:location],
-[@at:'Old:loc:location] = {"type" : "Record" ,
-"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
-"val" : [{"indice" : "0" , "value" : {"type" : "Integer" , "val" : "42" } },
-{"others" : {"type" : "Integer" , "val" : "4" } }] } }, {"field" : "length" ,
-"value" : {"type" : "Integer" ,
-"val" : "2" } }] } }
-Line 40:
-a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
-"val" : {"Field" : [{"field" : "elts" , "value" : {"type" : "Array" ,
-"val" : [{"indice" : "0" , "value" : {"type" : "Integer" , "val" : "42" } },
-{"others" : {"type" : "Integer" , "val" : "4" } }] } }, {"field" : "length" ,
-"value" : {"type" : "Integer" ,
-"val" : "2" } }] } }
-
--- a/bench/ce/oracles/array_records_CVC4,1.5_SP.oracle
+++ b/bench/ce/oracles/array_records_CVC4,1.5_SP.oracle
+Strongest Postcondition
+bench/ce/array_records.mlw Array_records VC var_overwrite: Timeout or Unknown
+Counter-example model:File array_records.mlw:
+Line 23:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "first_value" ,
+"value" : {"type" : "Integer" , "val" : "3" } }, {"field" : "flag" ,
+"value" : {"type" : "Boolean" , "val" : true } }, {"field" : "second_value" ,
+"value" : {"type" : "Integer" , "val" : "5" } }] } }
+i, [[@introduced],
+[@model_trace:i]] = {"type" : "Integer" ,
+"val" : "-1" }
+Line 27:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "first_value" ,
+"value" : {"type" : "Integer" , "val" : "3" } }, {"field" : "flag" ,
+"value" : {"type" : "Boolean" , "val" : true } }, {"field" : "second_value" ,
+"value" : {"type" : "Integer" , "val" : "5" } }] } }
+i, [[@introduced],
+[@model_trace:i]] = {"type" : "Integer" ,
+"val" : "-1" }
+
+bench/ce/array_records.mlw Array_records VC var_overwrite: Timeout or Unknown
+Counter-example model:File array_records.mlw:
+Line 23:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "first_value" ,
+"value" : {"type" : "Integer" , "val" : "3" } }, {"field" : "flag" ,
+"value" : {"type" : "Boolean" , "val" : true } }, {"field" : "second_value" ,
+"value" : {"type" : "Integer" , "val" : "5" } }] } }
+i, [[@introduced],
+[@model_trace:i]] = {"type" : "Integer" ,
+"val" : "-1" }
+Line 27:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "first_value" ,
+"value" : {"type" : "Integer" , "val" : "3" } }, {"field" : "flag" ,
+"value" : {"type" : "Boolean" , "val" : true } }, {"field" : "second_value" ,
+"value" : {"type" : "Integer" , "val" : "5" } }] } }
+i, [[@introduced],
+[@model_trace:i]] = {"type" : "Integer" ,
+"val" : "-1" }
+
+bench/ce/array_records.mlw Array_records VC var_overwrite: Timeout or Unknown
+Counter-example model:File array_records.mlw:
+Line 23:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "first_value" ,
+"value" : {"type" : "Integer" , "val" : "3" } }, {"field" : "flag" ,
+"value" : {"type" : "Boolean" , "val" : true } }, {"field" : "second_value" ,
+"value" : {"type" : "Integer" , "val" : "5" } }] } }
+i, [[@introduced],
+[@model_trace:i]] = {"type" : "Integer" ,
+"val" : "-1" }
+Line 27:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "first_value" ,
+"value" : {"type" : "Integer" , "val" : "3" } }, {"field" : "flag" ,
+"value" : {"type" : "Boolean" , "val" : true } }, {"field" : "second_value" ,
+"value" : {"type" : "Integer" , "val" : "5" } }] } }
+i, [[@introduced],
+[@model_trace:i]] = {"type" : "Integer" ,
+"val" : "-1" }
+
+bench/ce/array_records.mlw Array_records VC var_overwrite: Timeout or Unknown
+Counter-example model:File array_records.mlw:
+Line 23:
+a, [[@introduced], [@model_trace:a],
+[@at:'Old:loc:location],
+[@at:'Old:loc:location] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "first_value" ,
+"value" : {"type" : "Integer" , "val" : "3" } }, {"field" : "flag" ,
+"value" : {"type" : "Boolean" , "val" : true } }, {"field" : "second_value" ,
+"value" : {"type" : "Integer" , "val" : "5" } }] } }
+i, [[@introduced],
+[@model_trace:i]] = {"type" : "Integer" ,
+"val" : "-1" }
+Line 28:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "first_value" ,
+"value" : {"type" : "Integer" , "val" : "3" } }, {"field" : "flag" ,
+"value" : {"type" : "Boolean" , "val" : true } }, {"field" : "second_value" ,
+"value" : {"type" : "Integer" , "val" : "5" } }] } }
+i, [[@introduced],
+[@model_trace:i]] = {"type" : "Integer" ,
+"val" : "-1" }
+
+bench/ce/array_records.mlw Array_records VC var_overwrite: Timeout or Unknown
+Counter-example model:File array_records.mlw:
+Line 23:
+a, [[@introduced], [@model_trace:a],
+[@at:'Old:loc:location],
+[@at:'Old:loc:location] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "first_value" ,
+"value" : {"type" : "Integer" , "val" : "3" } }, {"field" : "flag" ,
+"value" : {"type" : "Boolean" , "val" : true } }, {"field" : "second_value" ,
+"value" : {"type" : "Integer" , "val" : "5" } }] } }
+i, [[@introduced],
+[@model_trace:i]] = {"type" : "Integer" ,
+"val" : "-1" }
+Line 28:
+a, [[@introduced], [@model_trace:a]] = {"type" : "Record" ,
+"val" : {"Field" : [{"field" : "first_value" ,