should.py 30.2 KB
Newer Older
1 2
#!/usr/bin/env python3

3
# should -- Test command-line applications through .should files
4 5 6 7 8 9 10 11 12 13 14
#
# Copyright (C) 2018 by CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
# Contributors:
#     Mathieu Giraud <mathieu.giraud@vidjil.org>
#     Mikaël Salson <mikael.salson@vidjil.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
15
# "should" is distributed in the hope that it will be useful,
16 17 18 19 20
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
21
# along with "should". If not, see <http://www.gnu.org/licenses/>
22

23 24
import sys

25 26
if not (sys.version_info >= (3, 4)):
    print("Python >= 3.4 required")
27 28
    sys.exit(1)

Mathieu Giraud's avatar
Mathieu Giraud committed
29 30 31
__version_info__ = ('1','0','0')
__version__ = '.'.join(__version_info__)

32
import re
33
import argparse
34
import subprocess
35
import time
Mathieu Giraud's avatar
Mathieu Giraud committed
36
import random
37
import os.path
38
from collections import defaultdict, OrderedDict
39
import xml.etree.ElementTree as ET
40
import datetime
41
import tempfile
42

43 44 45 46
# Make sure the output is in utf8
sys.stdout = open(sys.stdout.fileno(), mode='w', encoding='utf8', buffering=1)
sys.stderr = open(sys.stderr.fileno(), mode='w', encoding='utf8', buffering=1)

47
DEFAULT_CFG = 'should.cfg'
48 49
RETRY = 'should.retry'
RETRY_FLAG = '--retry'
50

51
TOKEN_COMMENT = '#'
52
TOKEN_DIRECTIVE = '!'
53 54
TOKEN_NAME = '$'
TOKEN_TEST = ':'
55
RE_TEST = re.compile('^(\S)*[:]')
56

57
DIRECTIVE_REQUIRES = '!REQUIRES:'
58
DIRECTIVE_NO_LAUNCHER = '!NO_LAUNCHER:'
59 60
DIRECTIVE_SCRIPT = '!LAUNCH:'
DIRECTIVE_OPTIONS = '!OPTIONS:'
61
DIRECTIVE_SOURCE = '!OUTPUT_FILE:'
62
DIRECTIVE_EXIT_CODE = '!EXIT_CODE:'
63

64
VAR_LAUNCHER = '$LAUNCHER'
65
VAR_EXTRA = '$EXTRA'
66

67
MOD_TODO = 'f'
68
MOD_REGEX = 'r'
69
MOD_COUNT_ALL = 'w'
70
MOD_IGNORE_CASE = 'i'
Mathieu Giraud's avatar
Mathieu Giraud committed
71
MOD_BLANKS = 'b'
72
MOD_MULTI_LINES = 'l'
73
MOD_KEEP_LEADING_TRAILING_SPACES = 'z'
74

75 76 77
MOD_MORE_THAN = '>'
MOD_LESS_THAN = '<'

78
TIMEOUT = 120
79 80
SHOW_ELAPSED_TIME_ABOVE = 1.0

81
RE_MODIFIERS = re.compile('^(\D*)(\d*)(\D*)$')
82

83 84
OUT_LOG = '.log'
OUT_TAP = '.tap'
85
OUT_XML = 'should.xml'
86

87
LINE = '-' * 40
88
ENDLINE_CHARS = '\r\n'
89
CONTINUATION_CHAR = '\\'
90 91
MAX_HALF_DUMP_LINES = 45
MAX_DUMP_LINES = 2*MAX_HALF_DUMP_LINES + 10
92

93 94
SKIP = 'SKIP'

95
TODO = 'TODO'
96
TODO_PASSED = 'TODO_PASSED'
97

98 99
STATUS = {
    None: 'not run',
100
    False: 'failed',
101
    True: 'ok',
102
    SKIP: 'skip',
103
    TODO: 'TODO',
104
    TODO_PASSED: 'TODO-but-ok',
105 106
}

107
WARN_STATUS = [False, SKIP, TODO, TODO_PASSED]
108

109
STATUS_TAP = {
110
    None: 'not run',
111 112
    False: 'not ok',
    True: 'ok',
113
    SKIP: 'ok # SKIP',
114 115 116 117
    TODO: 'not ok # TODO',
    TODO_PASSED: 'ok # TODO',
}

118 119 120 121
STATUS_XML = STATUS.copy()
STATUS_XML[False] = 'failure'
STATUS_XML[SKIP] = 'skipped'

122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138

# Simple colored output

CSIm = '\033[%sm'

class ANSI:
    RESET = 0
    BRIGHT = 1
    BLACK = 30
    RED = 31
    GREEN = 32
    YELLOW = 33
    BLUE = 34
    MAGENTA = 35
    CYAN = 36
    WHITE = 37

139 140 141
def color(col, text, colorize = True):
    if not colorize:
        return text
142 143
    return CSIm % ANSI.BRIGHT + CSIm % col + text + CSIm % ANSI.RESET

144 145 146 147
STATUS_COLORS = {
    None: ANSI.BLUE,
    False: ANSI.RED,
    True: ANSI.GREEN,
148 149 150
    SKIP: ANSI.BLUE,
    TODO: ANSI.BLUE,
    TODO_PASSED: ANSI.BLUE,
151
}
152

153 154
# Modifier parser

155
MODIFIERS = [
156
    (MOD_TODO, 'todo', 'consider that the test should fail'),
157 158
    (MOD_REGEX, 'regex', 'consider as a regular expression'),
    (MOD_COUNT_ALL, 'count-all', 'count all occurrences, even on a same line'),
159
    (MOD_IGNORE_CASE, 'ignore-case', 'ignore case changes'),
160
    (MOD_BLANKS, 'blanks', "ignore whitespace differences as soon as there is at least one space. Implies 'r'"),
161 162
    (MOD_MULTI_LINES, 'multi-lines', 'search on all the output rather than on every line'),
    (MOD_KEEP_LEADING_TRAILING_SPACES, 'ltspaces', 'keep leading and trailing spaces'),
163 164 165

    (MOD_MORE_THAN, 'more-than', 'requires that the expression occurs strictly more than the given number'),
    (MOD_LESS_THAN, 'less-than', 'requires that the expression occurs strictly less than the given number'),
166
]
167

168 169 170 171 172 173 174 175 176 177 178 179 180 181 182


class ArgParser(argparse.ArgumentParser):

    def convert_arg_line_to_args(self, l):
        '''
        More flexible argument parsing from configuration file:
          - ignore leading and trailing spaces
          - allow empty lines
        '''
        ll = l.strip()
        return [ ll ] if ll else [ ]


class ModifierParser(ArgParser):
183 184 185

    def parse_modifiers(self, modifiers):
        mods, unknown = self.parse_known_args(['-' + mod for mod in modifiers])
186 187
        for m in unknown:
            sys.stderr.write("! Unknown modifier '%s'\n" % m[1])
188 189 190
        return mods

parser_mod = ModifierParser()
191
parser_mod.help = 'modifiers (uppercase letters cancel previous modifiers)\n'
192

193 194
for (mod_char, mod_long, mod_help) in MODIFIERS:
    parser_mod.add_argument('-' + mod_char, '--' + mod_long, action='store_true', help=mod_help)
195

196 197 198 199 200 201 202 203
    if mod_char.upper() != mod_char:
        parser_mod.add_argument('-' + mod_char.upper(), dest=mod_long.replace('-', '_'), action='store_const', const=False, default=False,
                                help='back to default, overriding any previous -%s' % mod_char)
        help_upper = '/%s' % mod_char.upper()
    else:
        help_upper = '  '

    parser_mod.help += '  %s%s %s\n' % (mod_char, help_upper, mod_help)
204

205 206
# Main argument parser

207 208 209
parser = ArgParser(description='Test command-line applications through .should files',
                   fromfile_prefix_chars='@',
                   epilog='''Example:
210
  %(prog)s demo/hello.should''',
211 212
                                 formatter_class=argparse.RawTextHelpFormatter)

Mathieu Giraud's avatar
Mathieu Giraud committed
213 214 215
parser.add_argument('--version', action='version',
                    version='%(prog)s {version}'.format(version=__version__))

216
options = ArgParser(fromfile_prefix_chars='@') # Can be used in !OPTIONS: directive
217 218

for p in (parser, options):
219
    p.add_argument('--cd', metavar='PATH', help='directory from which to run the test commands')
220
    p.add_argument('--cd-same', action='store_true', help='run the test commands from the same directory as the .should files')
Mathieu Giraud's avatar
Mathieu Giraud committed
221
    p.add_argument('--launcher', metavar='CMD', default='', help='launcher preceding each command (or replacing %s)' % VAR_LAUNCHER)
Mathieu Giraud's avatar
Mathieu Giraud committed
222
    p.add_argument('--extra', metavar='ARG', default='', help='extra argument after the first word of each command (or replacing %s)' % VAR_EXTRA)
223 224
    p.add_argument('--mod', metavar='MODIFIERS', action='append', help='global ' + parser_mod.help)
    p.add_argument('--var', metavar='NAME=value', action='append', help='variable definition (then use $NAME in .should files)')
225

Mathieu Giraud's avatar
Mathieu Giraud committed
226 227
parser.add_argument('--shuffle', action='store_true', help='shuffle the tests')

228 229 230 231
output = parser.add_argument_group('output options')

output.add_argument('--log', action='append_const', dest='output', const=OUT_LOG, help='stores the output into .log files')
output.add_argument('--tap', action='append_const', dest='output', const=OUT_TAP, help='outputs .tap files')
232
output.add_argument('--xml', action='append_const', dest='output', const=OUT_XML, help='outputs JUnit-like XML into %s' % OUT_XML)
233 234
output.add_argument('-v', '--verbose', action='count', help='increase verbosity', default=1)
output.add_argument('-q', '--quiet', action='store_const', dest='verbose', const=0, help='verbosity to zero')
235

236
parser.add_argument('file', metavar='should-file', nargs='+', help='''input files (.should)''')
237
parser.add_argument(RETRY_FLAG, action='store_true', help='launch again the last failed or warned tests')
238

239 240
class ShouldException(BaseException):
    pass
241

242

243
def write_to_file(f, what):
244 245
    print('==> %s' % f)
    with open(f, 'w', encoding='utf-8') as ff:
246 247
        ff.write(what)

248 249 250 251 252 253 254 255


# Command pre-processing

def pre_process(cmd):

    cc = cmd.split(' ')

256 257 258
    if not VAR_EXTRA in cmd:
        cc = [cc[0], VAR_EXTRA] + cc[1:]

259 260 261 262 263 264
    if not VAR_LAUNCHER in cmd:
        cc = [VAR_LAUNCHER] + cc

    return ' '.join(cc)


265 266 267 268 269
# Variables definition and expansion

def populate_variables(var):
    '''
    >>> populate_variables(['ab=cd', 'ef=xyz'])
Mathieu Giraud's avatar
Mathieu Giraud committed
270
    [('$ef', 'xyz'), ('$ab', 'cd')]
271 272 273 274 275 276 277 278
    '''

    variables = []

    if var:
        for v in var:
            try:
                key, var = v.split('=')
279
                variables = [('$' + key, var)] + variables
280 281 282 283 284 285

            except IOError:
                raise ShouldException('Error in parsing variable definition: ' + v)
    return variables


286 287 288 289 290
def print_variables(variables):
    for (k, v) in variables:
        print('%s=%s' % (k, v))
    print('')

291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308
def replace_variables(s, variables):
    '''
    >>> replace_variables('hello', None)
    'hello'

    >>> replace_variables('hello', [('hell', 'w'), ('o', 'orld')])
    'world'

    >>> replace_variables('xyz xyz', [('y', 'abc')])
    'xabcz xabcz'
    '''

    if variables:
        for (key, val) in variables:
            s = s.replace(key, val)
    return s


309 310 311 312
class OrderedDefaultListDict(OrderedDict):
    def __missing__(self, key):
        self[key] = value = []
        return value
313

314
class Stats():
Mathieu Giraud's avatar
Mathieu Giraud committed
315 316
    '''
    >>> s = Stats('foo')
317
    >>> s.up(2)
Mathieu Giraud's avatar
Mathieu Giraud committed
318 319
    >>> list(s.keys())
    [2]
320 321
    >>> s[2]
    [1]
Mathieu Giraud's avatar
Mathieu Giraud committed
322 323

    >>> t = Stats()
324 325
    >>> t.up(2, 'hello')
    >>> t.up(3)
Mathieu Giraud's avatar
Mathieu Giraud committed
326 327 328 329 330 331

    >>> u = s + t
    >>> sorted(u.keys())
    [2, 3]
    >>> list(s.keys())
    [2]
332 333 334

    >>> sorted(u.items())
    [(2, [1, 'hello']), (3, [1])]
Mathieu Giraud's avatar
Mathieu Giraud committed
335
    '''
336

337
    def __init__(self, item=''):
338
        self.stats = OrderedDefaultListDict()
339
        self.item = item
340 341 342 343

    def __getitem__(self, key):
        return self.stats[key]

344 345 346
    def up(self, key, data=1):
        self.stats[key].append(data)

347 348 349
    def __setitem__(self, key, value):
        self.stats[key] = value

350 351 352
    def keys(self):
        return self.stats.keys()

353 354 355 356 357 358 359 360 361
    def items(self):
        return self.stats.items()

    def values(self):
        return self.stats.values()

    def total(self):
        return sum(map(len, self.stats.values()))

362 363
    def __add__(self, other):
        result = Stats(self.item)
364 365
        for data in (self, other):
            for key in data.keys():
366
                result.stats[key] += data.stats[key]
367 368
        return result

369 370 371 372
    def str_status(self, status, colorize=True):
        s = '==> '
        s += STATUS[status]
        s += ' - '
373
        s += ' '.join(['%s:%d' % (STATUS[key], len(val)) for (key, val) in self.items()] + ['total:%s' % self.total()])
374
        if self.item:
375
            s += ' ' + self.item + ('s' if self.total() > 1 else '')
376
        return color(STATUS_COLORS[status], s, colorize)
377 378 379



380

381
class TestCaseAbstract:
382 383 384 385 386 387
    def __init__(self):
        raise NotImplemented

    def str_additional_status(self, verbose=False):
        return ''

388
    def str_status(self, verbose=False, names=STATUS, colorize=True):
389
        s = ''
390
        s += color(STATUS_COLORS[self.status], names[self.status], colorize)
391 392 393 394
        s += self.str_additional_status(verbose)

        return s

395 396 397 398 399 400 401
    def xml(self):
        x = ET.Element('testcase', {'name': self.name, 'status': STATUS_XML[self.status]})
        if self.status in WARN_STATUS:
            x.append(ET.Element(STATUS_XML[self.status],
                                {'message': repr(self) + '\n' + self.str_status(names=STATUS_XML, colorize = False)}))
        return x

402
    def tap(self, names=STATUS_TAP, colorize=False):
403 404 405
        s = []

        if self.status is not None:
406
            s.append(self.str_status(names=names, colorize=colorize))
407 408 409 410 411 412

        if self.name:
            s.append(self.name)

        return ' - '.join(s)

413 414 415
    def str(self, colorize):
        return self.tap(names=STATUS, colorize=colorize)

416
    def __str__(self):
417
        return self.str(colorize=True)
418

419 420
    def __repr__(self):
        raise NotImplemented
421

422
class ExternalTestCase(TestCaseAbstract):
423 424 425 426 427 428 429
    def __init__(self, name, status, info=''):
        self.name = name
        self.status = status
        self.info = info

    def str_additional_status(self, verbose = False):
        s = ''
430
        if self.status in WARN_STATUS or verbose:
431 432 433 434 435 436
            s += ' (%s)' % self.info
        return s

    def test(self, *args, **kwargs):
        pass

437 438 439 440
    def __repr__(self):
        return self.info


441
class TestCase(TestCaseAbstract):
442
    '''
443
    >>> test = TestCase('', 'hello')
444 445
    >>> repr(test)
    ':hello'
446

447
    >>> test.str_status(colorize=False)
448 449
    'not run'

450
    >>> test.test(['world'])
451 452
    False

453 454 455 456 457
    >>> test.status
    False

    >>> test.test(['hello'])
    True
458

459

460
    >>> test = TestCase('3', 'hello')
461 462
    >>> repr(test)
    '3:hello'
463 464 465

    >>> test.test(['hello'])
    False
466 467
    >>> test.count
    1
468
    >>> print(test.str(colorize=False))
469
    failed (1/3)
470 471
    >>> test.tap()
    'not ok (1/3)'
472 473 474 475 476

    >>> test.test(['hello'] * 3)
    True


477
    >>> TestCase('r2', ' e.*o ').test(['hello', 'ello', 'world'])
478 479
    True

480
    >>> TestCase('z1', ' e').test(['hello', 'h ello'])
481 482
    True

483
    >>> TestCase('rl', 'e.*o').test(['hel', 'lo'])
484 485
    True

486
    >>> TestCase('', 'e o').test(['e  o'])
Mathieu Giraud's avatar
Mathieu Giraud committed
487 488
    False

489
    >>> TestCase('f', 'e o').test(['e  o'])
490 491
    'TODO'

492
    >>> TestCase('b', 'e o').test(['e  o'])
Mathieu Giraud's avatar
Mathieu Giraud committed
493 494
    True

495
    >>> TestCase('b', 'e    o').test(['e  o'])
496 497
    True

498
    >>> TestCase('w2', 'o').test(['hello world'])
499 500
    True

501
    >>> TestCase('wW2', 'o').test(['hello world'])
502 503
    False

504
    >>> TestCase('wr2', 'a.c').test(['bli abc axc bla'])
505 506
    True

507

508
    >>> repr(TestCase('x3y', 'hello'))
509
    'xy3:hello'
510

511
    >>> print(TestCase('1x2', 'hello')) # doctest: +IGNORE_EXCEPTION_DETAIL
512 513 514
    Traceback (most recent call last):
     ...
    ShouldException: Error in parsing modifiers: 1x2
515 516 517
    '''

    def __init__(self, modifiers, expression, name=''):
Mathieu Giraud's avatar
Mathieu Giraud committed
518 519
        self.name = name
        self.status = None
520
        self.count = '?'
521 522 523 524 525 526 527 528

        # Extract self.expected_count from modifiers
        m = RE_MODIFIERS.match(modifiers)
        if not m:
            raise ShouldException('Error in parsing modifiers: ' + modifiers)
        self.modifiers = m.group(1) + m.group(3)
        self.expected_count = int(m.group(2)) if m.group(2) else None

529
        # Parse modifiers
530
        self.mods = parser_mod.parse_modifiers(self.modifiers)
531

532
        self.expression = expression if self.mods.ltspaces else expression.strip()
Mathieu Giraud's avatar
Mathieu Giraud committed
533
        if self.mods.blanks:
534 535
            while '  ' in self.expression:
                self.expression = self.expression.replace('  ', ' ')
Mathieu Giraud's avatar
Mathieu Giraud committed
536 537
            self.expression = self.expression.replace(' ', '\s+')
            self.mods.regex = True
538 539 540 541 542 543 544

        self.regex = None
        if self.mods.regex:
            if self.mods.ignore_case:
                self.regex = re.compile(self.expression, re.IGNORECASE)
            else:
                self.regex = re.compile(self.expression)
545

546
    def test(self, lines, variables=None, verbose=0):
547
        if self.mods.multi_lines:
548
            lines = [' '.join([l.rstrip(ENDLINE_CHARS) for l in lines])]
549

550 551
        expression_var = replace_variables(self.expression, variables)

552 553 554
        if not self.regex and self.mods.ignore_case:
            expression_var = expression_var.upper()

555
        self.count = 0
556
        for l in lines:
557
            if self.regex:
558 559 560
                if self.mods.count_all:
                    self.count += len(self.regex.findall(l))
                elif self.regex.search(l):
561
                    self.count += 1
562 563 564 565 566
            else:
                if self.mods.ignore_case:
                    l = l.upper()
                if expression_var in l:
                    self.count += l.count(expression_var) if self.mods.count_all else 1
567

568 569 570 571 572 573 574 575
        if self.expected_count is None:
            self.status = (self.count > 0)
        elif self.mods.less_than:
            self.status = (self.count < self.expected_count)
        elif self.mods.more_than:
            self.status = (self.count > self.expected_count)
        else:
            self.status = (self.count == self.expected_count)
576

577
        if self.mods.todo:
578
            self.status = [TODO, TODO_PASSED][self.status]
579

580 581
        if verbose > 0:
            print('')
582
            print(self.str_status(True) + ' ' + repr(self))
583

584
        return self.status
585

586
    def str_additional_status(self, verbose=False):
587 588
        s = ''

589
        if self.status in WARN_STATUS or verbose:
590 591 592
            s += ' (%s/%s%s)' % (self.count,
                                 MOD_LESS_THAN if self.mods.less_than else MOD_MORE_THAN if self.mods.more_than else '',
                                 self.expected_count if self.expected_count is not None else '+')
593 594 595

        return s

596 597 598
    def __repr__(self):
        return '%s%s:%s' % (self.modifiers, self.expected_count if self.expected_count is not None else '', self.expression)

599

600
class TestSuite():
601
    '''
602
    >>> s = TestSuite()
603 604 605
    >>> s.load(['echo "hello"', '$My test', ':hello'])
    >>> print(s)
    echo "hello"
606
    My test
607 608

    >>> s.test()
609
    True
610 611
    >>> s.tests[0].status
    True
612

613
    >>> s2 = TestSuite('r')
614
    >>> s2.variables.append(("$LAUNCHER", ""))
Mathieu Giraud's avatar
Mathieu Giraud committed
615
    >>> s2.variables.append(("$EXTRA", ""))
616
    >>> s2.load(['echo "hello"', '$ A nice test', ':e.*o'])
617
    >>> s2.test(verbose = 1, colorize = False)   # doctest: +NORMALIZE_WHITESPACE
618
    echo "hello"
619 620
      stdout --> 1 lines
      stderr --> 0 lines
621
    ok - A nice test
622
    ok - Exit code is 0
623
    True
624

625
    >>> s2.str_status(colorize = False)
626
    '==> ok - ok:2 total:2 tests'
627 628

    >>> print(s2.tap())   # doctest: +NORMALIZE_WHITESPACE
629
    1..2
630
    ok - A nice test
631
    ok - Exit code is 0
632 633
    '''

634 635
    def __init__(self, modifiers = '', cd = None, name = ''):
        self.name = name
636 637
        self.requires = True
        self.requires_cmd = None
638
        self.requires_stderr = []
639 640
        self.cmds = []
        self.tests = []
641 642
        self.stdin = []
        self.stdout = []
643
        self.test_lines = []
644
        self.status = None
645
        self.modifiers = modifiers
646
        self.variables = []
647
        self.stats = Stats('test')
648
        self.source = None
649
        self.cd = cd
650
        self.use_launcher = True
651
        self.expected_exit_code = 0
652
        self.elapsed_time = None
653 654 655

    def load(self, should_lines):
        name = ''
656
        this_cmd_continues = False
657 658
        for l in should_lines:

659
            l = l.lstrip().rstrip(ENDLINE_CHARS)
660 661 662 663 664 665 666
            if not l:
                continue

            # Comment
            if l.startswith(TOKEN_COMMENT):
                continue

667 668 669 670 671
            # Directive -- Requires
            if l.startswith(DIRECTIVE_REQUIRES):
                self.requires_cmd = l[len(DIRECTIVE_REQUIRES):].strip()
                continue

672 673 674 675 676
            # Directive -- No launcher
            if l.startswith(DIRECTIVE_NO_LAUNCHER):
                self.use_launcher = False
                continue

677 678
            # Directive -- Source
            if l.startswith(DIRECTIVE_SOURCE):
679
                self.source = os.path.join(self.cd if self.cd else '', l[len(DIRECTIVE_SOURCE):].strip())
680 681
                continue

682 683 684
            # Directive -- Options
            if l.startswith(DIRECTIVE_OPTIONS):
                opts, unknown = options.parse_known_args(l[len(DIRECTIVE_OPTIONS):].split())
685
                self.variables = populate_variables(opts.var) + self.variables
686 687
                if opts.mod:
                    self.modifiers += ''.join(opts.mod)
688 689
                continue

690 691 692 693 694
            # Directive -- Exit code
            if l.startswith(DIRECTIVE_EXIT_CODE):
                self.expected_exit_code = int(l[len(DIRECTIVE_EXIT_CODE):].strip())
                continue

695 696
            # Name
            if l.startswith(TOKEN_NAME):
Mathieu Giraud's avatar
Mathieu Giraud committed
697
                name = l[1:].strip()
698 699
                continue

700 701 702 703 704 705 706 707 708
            # Directive -- Command
            if l.startswith(DIRECTIVE_SCRIPT):
                l = l[len(DIRECTIVE_SCRIPT):]

            # Directive -- Others
            if l.startswith(TOKEN_DIRECTIVE):
                sys.stderr.write('! Unknown directive: %s\n' % l)
                continue

709
            # Test
710
            if RE_TEST.search(l):
711 712
                pos = l.find(TOKEN_TEST)
                modifiers, expression = l[:pos], l[pos+1:]
713
                self.tests.append(TestCase(self.modifiers + modifiers, expression, name))
714 715 716
                continue

            # Command
717
            l = l.strip()
718 719 720 721 722 723 724 725 726 727 728
            next_cmd_continues = l.endswith(CONTINUATION_CHAR)
            if next_cmd_continues:
                l = l[:-1]

            if this_cmd_continues:
                self.cmds[-1] += l
            else:
                self.cmds.append(l)

            this_cmd_continues = next_cmd_continues

729

730 731 732 733 734
    def print_stderr(self, colorize=True):
        print('  stdout --> %s lines' % len(self.stdout))
        print('  stderr --> %s lines' % len(self.stderr))
        print(color(ANSI.CYAN, ''.join(self.stderr), colorize))

735 736 737
    def skip_all(self, reason, verbose=1):
        if verbose > 0:
            print('Skipping tests: %s' % reason)
738 739
        for test in self.tests:
            test.status = SKIP
740
            self.stats.up(test.status)
741 742
        self.status = SKIP

743
    def test(self, variables=[], verbose=0, colorize=True):
744

745
        self.variables_all = self.variables + variables
746
        if verbose > 1:
747
            print_variables(self.variables_all)
748 749

        def cmd_variables_cd(cmd):
750
            cmd = replace_variables(cmd, self.variables_all)
751 752 753 754 755 756
            if self.cd:
                cmd = 'cd %s ; ' % self.cd + cmd
            if verbose > 0:
                print(color(ANSI.MAGENTA, cmd, colorize))
            return cmd

757 758
        self.status = True

759
        if self.requires_cmd:
760 761
            requires_cmd = cmd_variables_cd(self.requires_cmd)
            p = subprocess.Popen(requires_cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
762 763
            self.requires = (p.wait() == 0)
            self.requires_stderr = [l.decode(errors='replace') for l in p.stderr.readlines()]
764
            if verbose > 0:
765
                print(color(ANSI.CYAN, ''.join(self.requires_stderr), colorize))
766 767

            if not self.requires:
768
                self.skip_all('Condition is not met: %s' % self.requires_cmd, verbose)
769
                return self.status
770

771
        if not self.use_launcher:
772
            if replace_variables(VAR_LAUNCHER, self.variables_all):
773
                self.skip_all('%s while %s is given' % (DIRECTIVE_NO_LAUNCHER, VAR_LAUNCHER), verbose)
774 775
                return self.status

776 777
        start_time = time.time()

778
        cmd = ' ; '.join(map(pre_process, self.cmds))
779 780
        cmd = cmd_variables_cd(cmd)

781 782
        f_stdout = tempfile.TemporaryFile()
        f_stderr = tempfile.TemporaryFile()
783
        p = subprocess.Popen([cmd], shell=True,
784
                             stdout=f_stdout, stderr=f_stderr,
785
                             close_fds=True)
786 787

        try:
788
            self.exit_code = p.wait(TIMEOUT)
789
            self.tests.append(ExternalTestCase('Exit code is %d' % self.expected_exit_code, self.exit_code == self.expected_exit_code, str(self.exit_code)))
790
        except subprocess.TimeoutExpired:
791
            self.exit_code = None
792
            self.tests.append(ExternalTestCase('Exit code is %d' % self.expected_exit_code, SKIP, 'timeout after %s seconds' % TIMEOUT))
793

794 795 796 797 798 799
        f_stdout.seek(0)
        f_stderr.seek(0)
        self.stdout = [l.decode(errors='replace') for l in f_stdout.readlines()]
        self.stderr = [l.decode(errors='replace') for l in f_stderr.readlines()]
        f_stdout.close()
        f_stderr.close()
800

Mathieu Giraud's avatar
Mathieu Giraud committed
801 802 803 804 805 806 807

        if verbose > 0:
            self.print_stderr(colorize)

        self.test_lines = open(self.source).readlines() if self.source else self.stdout

        for test in self.tests:
808
            test.test(self.test_lines, variables=self.variables_all, verbose=verbose-1)
809
            self.stats.up(test.status)
Mathieu Giraud's avatar
Mathieu Giraud committed
810

811 812
            # When a test fails, the file fails
            if test.status is False:
Mathieu Giraud's avatar
Mathieu Giraud committed
813 814
                self.status = False

815