should-to-tap.sh 8.9 KB
Newer Older
1
#!/usr/bin/env bash
Mikaël Salson's avatar
Mikaël Salson committed
2 3 4 5 6 7 8 9 10 11 12 13

if [ $# -eq 0 -o "$1" == "-h" -o "$1" == "--help" ]; then
    echo "Usage: $0 <file.should_get>

This script takes as input a .should_get file. Don't know what it is? Read this!
In the .should_get format, every line starting with a # is a comment
every line starting with a $ is a description of the test
non empty lines, must be of the following form:
<info>:<regexp>
where <regexp> is a regular expression passed to grep 
(beware to escape backslashes).
info is the number of times this regular expression should occur in the file.
14
the info can be prefixed by a letter either s or f or e (they can be mixed):
Mikaël Salson's avatar
Mikaël Salson committed
15 16
s: if the test fails, we skip it
f: even if the test fails, the script will not exit with an error code.
17 18
e: the pattern must be searched exactly, regexp-specific characters will 
   automatically be escaped if they're not already escaped.
Mikaël Salson's avatar
Mikaël Salson committed
19 20 21 22

The script must contain a !LAUNCH: line stating what command line to be
launched (the working directory is the directory where the input file is).  A
line starting with !LOG: is the filename used for redirecting STDOUT from the
23 24
command line. By default it is the input should_get filename where the extension 
is replaced by .log. If !OUTPUT_FILE: is provided then the program is assumed to
Mikaël Salson's avatar
Mikaël Salson committed
25 26 27 28
produce a file whose filename is given after !OUTPUT_FILE:. This file will be
parsed by the script. The !LOG: file won't be used. By default, output files
are produced in the working directory, to change this behavior, specify an
option after the option !OUTPUT_DIR:
29 30
By default spaces can be replaced by any whitespaces. You can override this by
specifying !IGNORE_WHITESPACES: 0
Mikaël Salson's avatar
Mikaël Salson committed
31

32 33 34 35 36
* Exit code
  The exit code of the launched command line is also checked. By default, an
  exit code of 0 is expected. The .should_get file can specify an option
  !EXIT_CODE: indacting another expected value.

37 38 39 40 41 42
* Requirements
  Sometimes, to launch a test some requirements must be met. If the requirements
  are not met we may want to skip the test. To do so, specify in the file
  an option !REQUIRES: which will be followed by a command that is supposed
  to exit with the error code 0. If the error code is different from 0 all the
  tests in the file will be skipped.
43

Mikaël Salson's avatar
Mikaël Salson committed
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
* Environment
** Debug
   If the environment variable DEBUG is defined, then some debug information 
   is output.
** Launcher
   If one wants to use a launcher that will execute the program, then 
   environment variable LAUNCHER must be defined, with the program name to 
   use as a launcher (eg. valgrind).
   A .should_get file may specify an option !NO_LAUNCHER: to tell that this
   must not be launched using a launcher. Also, in the !LAUNCH option, 
   $LAUNCHER may be specified if the launcher must not be put at the start
   of the command line, but somewhere else.

* Output

The output is in TAP format and consists of a file whose name is the same
as the input file, where .should_get is replaced by .tap
" >&2
    exit 1
fi

debug() {
    if [ ! -z "$DEBUG" ]; then
67
        echo $* >&2
Mikaël Salson's avatar
Mikaël Salson committed
68 69 70 71 72 73 74 75
    fi
}

file=$1
DIR=$(dirname $file)
BASE=$(basename $file)
cd "$DIR"

76
VIDJIL_DIR=../../../
Mikaël Salson's avatar
Mikaël Salson committed
77 78 79
OUTPUT_DIR=.
TAP_FILE=${BASE%.*}.tap
LOG_FILE=${BASE%.*}.log
80
EXPECTED_EXIT_CODE=0
81
EXIT_CODE=
Mikaël Salson's avatar
Mikaël Salson committed
82 83 84
OUTPUT_FILE=
FILE_TO_GREP=
NO_LAUNCHER=
85
REQUIRE=
86
IGNORE_WHITESPACES=1
87
SEPARATOR_LINE="========================================================================"
Mikaël Salson's avatar
Mikaël Salson committed
88 89 90 91 92 93

TMP_TAP_FILE=$(mktemp tap.XXXX)

{
nb_tests=0
# Count number of tests to be performed
94
nb_tests=`grep -Ec '^[^$#!]' $BASE`
95
nb_tests=$((nb_tests+1))
Mikaël Salson's avatar
Mikaël Salson committed
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111

echo "1.."$nb_tests
test_nb=1
error=0
not_ok=$?
line_nb=1
failed_lines=()
launched=0                      # Did we launch the program yet?
cmd=
while read line; do
    if [ ! -z "$line" ]; then
        if  [ ${line:0:1} == "!" ]; then
            line=${line:1}
            type=${line%%:*}
            if [ "$type" == "LAUNCH" ]; then
                eval cmd=\"${line#*:}\"
112 113
            elif [ "$type" == "EXIT_CODE" ]; then
                EXPECTED_EXIT_CODE=${line#*:}
Mikaël Salson's avatar
Mikaël Salson committed
114 115 116 117 118 119 120 121
            elif [ "$type" == "LOG" ]; then
                eval LOG_FILE=\"${line#*:}\"
            elif [ "$type" == "OUTPUT_FILE" ]; then
                eval OUTPUT_FILE=\"${line#*:}\"
            elif [ "$type" == "OUTPUT_DIR" ]; then
                eval OUTPUT_DIR=\"${line#*:}\"
            elif [ "$type" == "NO_LAUNCHER" ]; then
                NO_LAUNCHER=1
122 123
            elif [ "$type" == "IGNORE_WHITESPACES" ]; then
                IGNORE_WHITESPACES=${line#*:}
124 125 126 127
            elif [ "$type" == "REQUIRES" ]; then
                REQUIRE=${line#*:}
            else
                echo "Unknown option $type" >&2
Mikaël Salson's avatar
Mikaël Salson committed
128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
            fi
        elif [ ${line:0:1} == '$' ]; then
            msg=${line:1}
        else
                # This is not a comment
            if [ ${line:0:1} != '#' ]; then
                if [ $launched -eq 0 ]; then
                    if [ -z "$cmd" ]; then
                        echo "Error: you must specify a !LAUNCH: line in $file" >&2
                        exit 2
                    fi
                    # Should we use a launcher?
                    if [ ! -z "$LAUNCHER" -a -z "$NO_LAUNCHER" ]; then
                        # Yes, we should.
                        # Do we need to specify the place where LAUNCHER should be?
143
                        if [[ "$cmd" != *'$LAUNCHER'* && "$cmd" != *"$LAUNCHER"* ]]; then
144
                            cmd="$LAUNCHER ""$cmd"
Mikaël Salson's avatar
Mikaël Salson committed
145 146 147 148 149
                        fi
                    fi
                    echo "Launching '$cmd'" >&2
                    if [ -z "$OUTPUT_FILE" ]; then
                        eval $cmd > $LOG_FILE
150
                        EXIT_CODE=$?
Mikaël Salson's avatar
Mikaël Salson committed
151 152 153
                        FILE_TO_GREP=$LOG_FILE
                    else
                        eval $cmd > /dev/null
154
                        EXIT_CODE=$?
Mikaël Salson's avatar
Mikaël Salson committed
155 156 157 158 159 160 161 162 163 164 165
                        FILE_TO_GREP=$OUTPUT_FILE
                    fi
                    launched=1

                    TAP_FILE=$OUTPUT_DIR/$TAP_FILE
                    LOG_FILE=$OUTPUT_DIR/$LOG_FILE
                    echo "==>" $TAP_FILE >&2
                fi

                skip=0
                know_to_fail=0
166
                exact=0
Mikaël Salson's avatar
Mikaël Salson committed
167

Mathieu Giraud's avatar
Mathieu Giraud committed
168 169
                pattern=$(cut -d: -f2- <<< "$line")
                nb_hits=$(cut -d: -f1 <<< "$line")
Mikaël Salson's avatar
Mikaël Salson committed
170

171 172 173 174 175 176 177 178 179 180 181 182
                # Escape special characters for sed
                pattern=$(sed -e 's/[/&]/\\&/g' <<< $pattern)

                while ! [ "${nb_hits:0:1}" -eq "${nb_hits:0:1}" ] 2> /dev/null; do
                    case ${nb_hits:0:1} in
                        "s") 
                            skip=1;;  # We skip the test if it fails
                        "f") 
                            know_to_fail=1;; # We know the test fails, but don't fail globally
                        "e")
                            # Exact: protect any character that may be part of
                            # a regex
183
                            pattern=$(sed -r 's/([^\\])(\.|\||\-|\+|\*|\[|\]|\(|\)|\{|\})/\1\\\2/g' <<< $pattern);;
184
                    esac
Mikaël Salson's avatar
Mikaël Salson committed
185
                    nb_hits=${nb_hits:1}
186 187
                done

188 189 190 191
                if ! eval $REQUIRE > /dev/null 2> /dev/null; then 
                    skip=1
                fi

192 193
                # Replace whitespaces if needed
                if [ $IGNORE_WHITESPACES -ne 0 ]; then
194
                    pattern=$(sed -r 's/[[:space:]]+/[[:space:]]+/g' <<< $pattern)
Mikaël Salson's avatar
Mikaël Salson committed
195
                fi
196 197 198

                debug "Grepping \"$pattern\" in $FILE_TO_GREP"
                if [ $(sed -rn "/$pattern/p" < $FILE_TO_GREP | wc -l) -eq $nb_hits -o $skip -eq 1 ]; then
Mikaël Salson's avatar
Mikaël Salson committed
199 200 201 202 203 204 205 206
                    if [ $know_to_fail -eq 1 ]; then
                        echo "Warning: test $test_nb should have failed, but has not!" >&2
                    fi
                    echo -n "ok"
                else
                    echo -n "not ok"
                    if [ $know_to_fail -eq 0 ]; then
                        error=1
207 208
			echo >&2; echo >&2; echo $SEPARATOR_LINE >&2
			echo "$file failed:" >&2
209
			echo "$line" >&2
210 211 212
			echo $SEPARATOR_LINE >&2
			cat $FILE_TO_GREP >&2
			echo $SEPARATOR_LINE >&2; echo >&2; echo >&2
Mikaël Salson's avatar
Mikaël Salson committed
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
                    fi
                fi
                echo -n " "$test_nb" "
                if [ $skip -eq 1 ]; then
                    echo -n "# SKIP "
                fi
                if [ $know_to_fail -eq 1 ]; then
                    echo -n "# TODO "
                fi
                echo "- " $msg
                test_nb=$((test_nb+1))
            fi
        fi
    fi
    line_nb=$((line_nb+1))
228

Mikaël Salson's avatar
Mikaël Salson committed
229
done < $BASE
230 231

# Check exit code
232 233
if [ $EXIT_CODE -eq $EXPECTED_EXIT_CODE ]; then
    echo "ok $test_nb -  Exit code $EXIT_CODE"
234
else
235
    echo -n "not ok $test_nb "
236 237 238 239 240

    if ! eval $REQUIRE > /dev/null 2> /dev/null; then
        echo -n "# SKIP "
    else

241 242 243
    error=1

    echo >&2; echo >&2; echo $SEPARATOR_LINE >&2
244
    echo "error: exit code $EXIT_CODE (expected $EXPECTED_EXIT_CODE)" >&2
245 246 247
    echo $SEPARATOR_LINE >&2
    cat $FILE_TO_GREP >&2
    echo $SEPARATOR_LINE >&2;  echo >&2; echo >&2
248 249

    fi
250
    echo "-  Exit code $EXIT_CODE"
251 252
fi

Mikaël Salson's avatar
Mikaël Salson committed
253 254 255 256 257
} > $TMP_TAP_FILE

mv $TMP_TAP_FILE $TAP_FILE
echo >&2
exit $error