should-to-tap.sh 8.92 KB
Newer Older
1
#!/usr/bin/env bash
Mikaël Salson's avatar
Mikaël Salson committed
2 3 4 5 6 7 8 9 10 11 12 13

if [ $# -eq 0 -o "$1" == "-h" -o "$1" == "--help" ]; then
    echo "Usage: $0 <file.should_get>

This script takes as input a .should_get file. Don't know what it is? Read this!
In the .should_get format, every line starting with a # is a comment
every line starting with a $ is a description of the test
non empty lines, must be of the following form:
<info>:<regexp>
where <regexp> is a regular expression passed to grep 
(beware to escape backslashes).
info is the number of times this regular expression should occur in the file.
14
the info can be prefixed by a letter either s or f or e (they can be mixed):
Mikaël Salson's avatar
Mikaël Salson committed
15 16
s: if the test fails, we skip it
f: even if the test fails, the script will not exit with an error code.
17 18
e: the pattern must be searched exactly, regexp-specific characters will 
   automatically be escaped if they're not already escaped.
Mikaël Salson's avatar
Mikaël Salson committed
19 20 21 22

The script must contain a !LAUNCH: line stating what command line to be
launched (the working directory is the directory where the input file is).  A
line starting with !LOG: is the filename used for redirecting STDOUT from the
23 24
command line. By default it is the input should_get filename where the extension 
is replaced by .log. If !OUTPUT_FILE: is provided then the program is assumed to
Mikaël Salson's avatar
Mikaël Salson committed
25 26 27 28
produce a file whose filename is given after !OUTPUT_FILE:. This file will be
parsed by the script. The !LOG: file won't be used. By default, output files
are produced in the working directory, to change this behavior, specify an
option after the option !OUTPUT_DIR:
29 30
By default spaces can be replaced by any whitespaces. You can override this by
specifying !IGNORE_WHITESPACES: 0
Mikaël Salson's avatar
Mikaël Salson committed
31

32 33 34 35 36
* Exit code
  The exit code of the launched command line is also checked. By default, an
  exit code of 0 is expected. The .should_get file can specify an option
  !EXIT_CODE: indacting another expected value.

37 38 39 40 41 42
* Requirements
  Sometimes, to launch a test some requirements must be met. If the requirements
  are not met we may want to skip the test. To do so, specify in the file
  an option !REQUIRES: which will be followed by a command that is supposed
  to exit with the error code 0. If the error code is different from 0 all the
  tests in the file will be skipped.
43

Mikaël Salson's avatar
Mikaël Salson committed
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
* Environment
** Debug
   If the environment variable DEBUG is defined, then some debug information 
   is output.
** Launcher
   If one wants to use a launcher that will execute the program, then 
   environment variable LAUNCHER must be defined, with the program name to 
   use as a launcher (eg. valgrind).
   A .should_get file may specify an option !NO_LAUNCHER: to tell that this
   must not be launched using a launcher. Also, in the !LAUNCH option, 
   $LAUNCHER may be specified if the launcher must not be put at the start
   of the command line, but somewhere else.

* Output

The output is in TAP format and consists of a file whose name is the same
as the input file, where .should_get is replaced by .tap
" >&2
    exit 1
fi

debug() {
    if [ ! -z "$DEBUG" ]; then
67
        echo $* >&2
Mikaël Salson's avatar
Mikaël Salson committed
68 69 70 71 72 73 74 75
    fi
}

file=$1
DIR=$(dirname $file)
BASE=$(basename $file)
cd "$DIR"

76
VIDJIL_DIR=../../../
77
DATA_DIR=../../../data/
Mikaël Salson's avatar
Mikaël Salson committed
78 79 80
OUTPUT_DIR=.
TAP_FILE=${BASE%.*}.tap
LOG_FILE=${BASE%.*}.log
81
EXPECTED_EXIT_CODE=0
82
EXIT_CODE=
Mikaël Salson's avatar
Mikaël Salson committed
83 84 85
OUTPUT_FILE=
FILE_TO_GREP=
NO_LAUNCHER=
86
REQUIRE=
87
IGNORE_WHITESPACES=1
88
SEPARATOR_LINE="========================================================================"
Mikaël Salson's avatar
Mikaël Salson committed
89 90 91 92 93 94

TMP_TAP_FILE=$(mktemp tap.XXXX)

{
nb_tests=0
# Count number of tests to be performed
95
nb_tests=`grep -Ec '^[^$#!]' $BASE`
96
nb_tests=$((nb_tests+1))
Mikaël Salson's avatar
Mikaël Salson committed
97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112

echo "1.."$nb_tests
test_nb=1
error=0
not_ok=$?
line_nb=1
failed_lines=()
launched=0                      # Did we launch the program yet?
cmd=
while read line; do
    if [ ! -z "$line" ]; then
        if  [ ${line:0:1} == "!" ]; then
            line=${line:1}
            type=${line%%:*}
            if [ "$type" == "LAUNCH" ]; then
                eval cmd=\"${line#*:}\"
113 114
            elif [ "$type" == "EXIT_CODE" ]; then
                EXPECTED_EXIT_CODE=${line#*:}
Mikaël Salson's avatar
Mikaël Salson committed
115 116 117 118 119 120 121 122
            elif [ "$type" == "LOG" ]; then
                eval LOG_FILE=\"${line#*:}\"
            elif [ "$type" == "OUTPUT_FILE" ]; then
                eval OUTPUT_FILE=\"${line#*:}\"
            elif [ "$type" == "OUTPUT_DIR" ]; then
                eval OUTPUT_DIR=\"${line#*:}\"
            elif [ "$type" == "NO_LAUNCHER" ]; then
                NO_LAUNCHER=1
123 124
            elif [ "$type" == "IGNORE_WHITESPACES" ]; then
                IGNORE_WHITESPACES=${line#*:}
125 126 127 128
            elif [ "$type" == "REQUIRES" ]; then
                REQUIRE=${line#*:}
            else
                echo "Unknown option $type" >&2
Mikaël Salson's avatar
Mikaël Salson committed
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
            fi
        elif [ ${line:0:1} == '$' ]; then
            msg=${line:1}
        else
                # This is not a comment
            if [ ${line:0:1} != '#' ]; then
                if [ $launched -eq 0 ]; then
                    if [ -z "$cmd" ]; then
                        echo "Error: you must specify a !LAUNCH: line in $file" >&2
                        exit 2
                    fi
                    # Should we use a launcher?
                    if [ ! -z "$LAUNCHER" -a -z "$NO_LAUNCHER" ]; then
                        # Yes, we should.
                        # Do we need to specify the place where LAUNCHER should be?
144
                        if [[ "$cmd" != *'$LAUNCHER'* && "$cmd" != *"$LAUNCHER"* ]]; then
145
                            cmd="$LAUNCHER ""$cmd"
Mikaël Salson's avatar
Mikaël Salson committed
146 147 148 149 150
                        fi
                    fi
                    echo "Launching '$cmd'" >&2
                    if [ -z "$OUTPUT_FILE" ]; then
                        eval $cmd > $LOG_FILE
151
                        EXIT_CODE=$?
Mikaël Salson's avatar
Mikaël Salson committed
152 153 154
                        FILE_TO_GREP=$LOG_FILE
                    else
                        eval $cmd > /dev/null
155
                        EXIT_CODE=$?
Mikaël Salson's avatar
Mikaël Salson committed
156 157 158 159 160 161 162 163 164 165 166
                        FILE_TO_GREP=$OUTPUT_FILE
                    fi
                    launched=1

                    TAP_FILE=$OUTPUT_DIR/$TAP_FILE
                    LOG_FILE=$OUTPUT_DIR/$LOG_FILE
                    echo "==>" $TAP_FILE >&2
                fi

                skip=0
                know_to_fail=0
167
                exact=0
Mikaël Salson's avatar
Mikaël Salson committed
168

Mathieu Giraud's avatar
Mathieu Giraud committed
169 170
                pattern=$(cut -d: -f2- <<< "$line")
                nb_hits=$(cut -d: -f1 <<< "$line")
Mikaël Salson's avatar
Mikaël Salson committed
171

172 173 174 175 176 177 178 179 180 181 182 183
                # Escape special characters for sed
                pattern=$(sed -e 's/[/&]/\\&/g' <<< $pattern)

                while ! [ "${nb_hits:0:1}" -eq "${nb_hits:0:1}" ] 2> /dev/null; do
                    case ${nb_hits:0:1} in
                        "s") 
                            skip=1;;  # We skip the test if it fails
                        "f") 
                            know_to_fail=1;; # We know the test fails, but don't fail globally
                        "e")
                            # Exact: protect any character that may be part of
                            # a regex
184
                            pattern=$(sed -r 's/([^\\])(\.|\||\-|\+|\*|\[|\]|\(|\)|\{|\})/\1\\\2/g' <<< $pattern);;
185
                    esac
Mikaël Salson's avatar
Mikaël Salson committed
186
                    nb_hits=${nb_hits:1}
187 188
                done

189 190 191 192
                if ! eval $REQUIRE > /dev/null 2> /dev/null; then 
                    skip=1
                fi

193 194
                # Replace whitespaces if needed
                if [ $IGNORE_WHITESPACES -ne 0 ]; then
195
                    pattern=$(sed -r 's/[[:space:]]+/[[:space:]]+/g' <<< $pattern)
Mikaël Salson's avatar
Mikaël Salson committed
196
                fi
197 198 199

                debug "Grepping \"$pattern\" in $FILE_TO_GREP"
                if [ $(sed -rn "/$pattern/p" < $FILE_TO_GREP | wc -l) -eq $nb_hits -o $skip -eq 1 ]; then
Mikaël Salson's avatar
Mikaël Salson committed
200 201 202 203 204 205 206 207
                    if [ $know_to_fail -eq 1 ]; then
                        echo "Warning: test $test_nb should have failed, but has not!" >&2
                    fi
                    echo -n "ok"
                else
                    echo -n "not ok"
                    if [ $know_to_fail -eq 0 ]; then
                        error=1
208 209
			echo >&2; echo >&2; echo $SEPARATOR_LINE >&2
			echo "$file failed:" >&2
210
			echo "$line" >&2
211 212 213
			echo $SEPARATOR_LINE >&2
			cat $FILE_TO_GREP >&2
			echo $SEPARATOR_LINE >&2; echo >&2; echo >&2
Mikaël Salson's avatar
Mikaël Salson committed
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228
                    fi
                fi
                echo -n " "$test_nb" "
                if [ $skip -eq 1 ]; then
                    echo -n "# SKIP "
                fi
                if [ $know_to_fail -eq 1 ]; then
                    echo -n "# TODO "
                fi
                echo "- " $msg
                test_nb=$((test_nb+1))
            fi
        fi
    fi
    line_nb=$((line_nb+1))
229

Mikaël Salson's avatar
Mikaël Salson committed
230
done < $BASE
231 232

# Check exit code
233 234
if [ $EXIT_CODE -eq $EXPECTED_EXIT_CODE ]; then
    echo "ok $test_nb -  Exit code $EXIT_CODE"
235
else
236
    echo -n "not ok $test_nb "
237 238 239 240 241

    if ! eval $REQUIRE > /dev/null 2> /dev/null; then
        echo -n "# SKIP "
    else

242 243 244
    error=1

    echo >&2; echo >&2; echo $SEPARATOR_LINE >&2
245
    echo "error: exit code $EXIT_CODE (expected $EXPECTED_EXIT_CODE)" >&2
246 247 248
    echo $SEPARATOR_LINE >&2
    cat $FILE_TO_GREP >&2
    echo $SEPARATOR_LINE >&2;  echo >&2; echo >&2
249 250

    fi
251
    echo "-  Exit code $EXIT_CODE"
252 253
fi

Mikaël Salson's avatar
Mikaël Salson committed
254 255 256 257 258
} > $TMP_TAP_FILE

mv $TMP_TAP_FILE $TAP_FILE
echo >&2
exit $error