should.py 30.7 KB
Newer Older
1 2
#!/usr/bin/env python3

3
# should -- Test command-line applications through .should files
4 5 6 7 8 9 10 11 12 13 14
#
# Copyright (C) 2018 by CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
# Contributors:
#     Mathieu Giraud <mathieu.giraud@vidjil.org>
#     Mikaël Salson <mikael.salson@vidjil.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
15
# "should" is distributed in the hope that it will be useful,
16 17 18 19 20
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
21
# along with "should". If not, see <http://www.gnu.org/licenses/>
22

23 24
import sys

25 26
if not (sys.version_info >= (3, 4)):
    print("Python >= 3.4 required")
27 28
    sys.exit(1)

29 30 31
__version_info__ = ('1','0','0')
__version__ = '.'.join(__version_info__)

32
import re
33
import argparse
34
import subprocess
35
import time
36
import random
37
import os.path
38
from collections import defaultdict, OrderedDict
39
import xml.etree.ElementTree as ET
40
import datetime
41
import tempfile
42

43 44 45 46
# Make sure the output is in utf8
sys.stdout = open(sys.stdout.fileno(), mode='w', encoding='utf8', buffering=1)
sys.stderr = open(sys.stderr.fileno(), mode='w', encoding='utf8', buffering=1)

47
DEFAULT_CFG = 'should.cfg'
48 49
RETRY = 'should.retry'
RETRY_FLAG = '--retry'
50

51
TOKEN_COMMENT = '#'
52
TOKEN_DIRECTIVE = '!'
53 54
TOKEN_NAME = '$'
TOKEN_TEST = ':'
55
RE_TEST = re.compile('^(\S)*[:]')
56

57
DIRECTIVE_REQUIRES = '!REQUIRES:'
58
DIRECTIVE_NO_LAUNCHER = '!NO_LAUNCHER:'
59
DIRECTIVE_SCRIPT = '!LAUNCH:'
60
DIRECTIVE_NO_EXTRA = '!NO_EXTRA:'
61
DIRECTIVE_OPTIONS = '!OPTIONS:'
62
DIRECTIVE_SOURCE = '!OUTPUT_FILE:'
63
DIRECTIVE_EXIT_CODE = '!EXIT_CODE:'
64

65
VAR_LAUNCHER = '$LAUNCHER'
66
VAR_EXTRA = '$EXTRA'
67

68
MOD_TODO = 'f'
69
MOD_REGEX = 'r'
70
MOD_COUNT_ALL = 'w'
71
MOD_IGNORE_CASE = 'i'
72
MOD_BLANKS = 'b'
73
MOD_MULTI_LINES = 'l'
74
MOD_KEEP_LEADING_TRAILING_SPACES = 'z'
75

76 77 78
MOD_MORE_THAN = '>'
MOD_LESS_THAN = '<'

79
TIMEOUT = 120
80 81
SHOW_ELAPSED_TIME_ABOVE = 1.0

82
RE_MODIFIERS = re.compile('^(\D*)(\d*)(\D*)$')
83

84 85
OUT_LOG = '.log'
OUT_TAP = '.tap'
86
OUT_XML = 'should.xml'
87

88
LINE = '-' * 40
89
ENDLINE_CHARS = '\r\n'
90
CONTINUATION_CHAR = '\\'
91 92
MAX_HALF_DUMP_LINES = 45
MAX_DUMP_LINES = 2*MAX_HALF_DUMP_LINES + 10
93

94 95
SKIP = 'SKIP'

96
TODO = 'TODO'
97
TODO_PASSED = 'TODO_PASSED'
98

99 100
STATUS = {
    None: 'not run',
101
    False: 'failed',
102
    True: 'ok',
103
    SKIP: 'skip',
104
    TODO: 'TODO',
105
    TODO_PASSED: 'TODO-but-ok',
106 107
}

108
WARN_STATUS = [False, SKIP, TODO, TODO_PASSED]
109

110
STATUS_TAP = {
111
    None: 'not run',
112 113
    False: 'not ok',
    True: 'ok',
114
    SKIP: 'ok # SKIP',
115 116 117 118
    TODO: 'not ok # TODO',
    TODO_PASSED: 'ok # TODO',
}

119 120 121 122
STATUS_XML = STATUS.copy()
STATUS_XML[False] = 'failure'
STATUS_XML[SKIP] = 'skipped'

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139

# Simple colored output

CSIm = '\033[%sm'

class ANSI:
    RESET = 0
    BRIGHT = 1
    BLACK = 30
    RED = 31
    GREEN = 32
    YELLOW = 33
    BLUE = 34
    MAGENTA = 35
    CYAN = 36
    WHITE = 37

140 141 142
def color(col, text, colorize = True):
    if not colorize:
        return text
143 144
    return CSIm % ANSI.BRIGHT + CSIm % col + text + CSIm % ANSI.RESET

145 146 147 148
STATUS_COLORS = {
    None: ANSI.BLUE,
    False: ANSI.RED,
    True: ANSI.GREEN,
149 150 151
    SKIP: ANSI.BLUE,
    TODO: ANSI.BLUE,
    TODO_PASSED: ANSI.BLUE,
152
}
153

154 155
# Modifier parser

156
MODIFIERS = [
157
    (MOD_TODO, 'todo', 'consider that the test should fail'),
158 159
    (MOD_REGEX, 'regex', 'consider as a regular expression'),
    (MOD_COUNT_ALL, 'count-all', 'count all occurrences, even on a same line'),
160
    (MOD_IGNORE_CASE, 'ignore-case', 'ignore case changes'),
161
    (MOD_BLANKS, 'blanks', "ignore whitespace differences as soon as there is at least one space. Implies 'r'"),
162 163
    (MOD_MULTI_LINES, 'multi-lines', 'search on all the output rather than on every line'),
    (MOD_KEEP_LEADING_TRAILING_SPACES, 'ltspaces', 'keep leading and trailing spaces'),
164 165 166

    (MOD_MORE_THAN, 'more-than', 'requires that the expression occurs strictly more than the given number'),
    (MOD_LESS_THAN, 'less-than', 'requires that the expression occurs strictly less than the given number'),
167
]
168

169 170 171 172 173 174 175 176 177 178 179 180 181 182 183


class ArgParser(argparse.ArgumentParser):

    def convert_arg_line_to_args(self, l):
        '''
        More flexible argument parsing from configuration file:
          - ignore leading and trailing spaces
          - allow empty lines
        '''
        ll = l.strip()
        return [ ll ] if ll else [ ]


class ModifierParser(ArgParser):
184 185 186

    def parse_modifiers(self, modifiers):
        mods, unknown = self.parse_known_args(['-' + mod for mod in modifiers])
187 188
        for m in unknown:
            sys.stderr.write("! Unknown modifier '%s'\n" % m[1])
189 190 191
        return mods

parser_mod = ModifierParser()
192
parser_mod.help = 'modifiers (uppercase letters cancel previous modifiers)\n'
193

194 195
for (mod_char, mod_long, mod_help) in MODIFIERS:
    parser_mod.add_argument('-' + mod_char, '--' + mod_long, action='store_true', help=mod_help)
196

197 198 199 200 201 202 203 204
    if mod_char.upper() != mod_char:
        parser_mod.add_argument('-' + mod_char.upper(), dest=mod_long.replace('-', '_'), action='store_const', const=False, default=False,
                                help='back to default, overriding any previous -%s' % mod_char)
        help_upper = '/%s' % mod_char.upper()
    else:
        help_upper = '  '

    parser_mod.help += '  %s%s %s\n' % (mod_char, help_upper, mod_help)
205

206 207
# Main argument parser

208 209 210
parser = ArgParser(description='Test command-line applications through .should files',
                   fromfile_prefix_chars='@',
                   epilog='''Example:
211
  %(prog)s demo/hello.should''',
212 213
                                 formatter_class=argparse.RawTextHelpFormatter)

214 215 216
parser.add_argument('--version', action='version',
                    version='%(prog)s {version}'.format(version=__version__))

217
options = ArgParser(fromfile_prefix_chars='@') # Can be used in !OPTIONS: directive
218 219

for p in (parser, options):
220
    p.add_argument('--cd', metavar='PATH', help='directory from which to run the test commands')
221
    p.add_argument('--cd-same', action='store_true', help='run the test commands from the same directory as the .should files')
222
    p.add_argument('--launcher', metavar='CMD', default='', help='launcher preceding each command (or replacing %s)' % VAR_LAUNCHER)
Mathieu Giraud's avatar
Mathieu Giraud committed
223
    p.add_argument('--extra', metavar='ARG', default='', help='extra argument after the first word of each command (or replacing %s)' % VAR_EXTRA)
224 225
    p.add_argument('--mod', metavar='MODIFIERS', action='append', help='global ' + parser_mod.help)
    p.add_argument('--var', metavar='NAME=value', action='append', help='variable definition (then use $NAME in .should files)')
226
    p.add_argument('--timeout', type=int, default = TIMEOUT, help = 'Delay (in seconds) after which the task is stopped (default: %(default)d)')
227

228 229
parser.add_argument('--shuffle', action='store_true', help='shuffle the tests')

230 231 232 233
output = parser.add_argument_group('output options')

output.add_argument('--log', action='append_const', dest='output', const=OUT_LOG, help='stores the output into .log files')
output.add_argument('--tap', action='append_const', dest='output', const=OUT_TAP, help='outputs .tap files')
234
output.add_argument('--xml', action='append_const', dest='output', const=OUT_XML, help='outputs JUnit-like XML into %s' % OUT_XML)
235 236
output.add_argument('-v', '--verbose', action='count', help='increase verbosity', default=1)
output.add_argument('-q', '--quiet', action='store_const', dest='verbose', const=0, help='verbosity to zero')
237

238
parser.add_argument('file', metavar='should-file', nargs='+', help='''input files (.should)''')
239
parser.add_argument(RETRY_FLAG, action='store_true', help='launch again the last failed or warned tests')
240

241 242
class ShouldException(BaseException):
    pass
243

244

245
def write_to_file(f, what):
246 247
    print('==> %s' % f)
    with open(f, 'w', encoding='utf-8') as ff:
248 249
        ff.write(what)

250 251 252 253 254 255 256 257


# Command pre-processing

def pre_process(cmd):

    cc = cmd.split(' ')

258 259 260
    if not VAR_EXTRA in cmd:
        cc = [cc[0], VAR_EXTRA] + cc[1:]

261 262 263 264 265 266
    if not VAR_LAUNCHER in cmd:
        cc = [VAR_LAUNCHER] + cc

    return ' '.join(cc)


267 268 269 270 271
# Variables definition and expansion

def populate_variables(var):
    '''
    >>> populate_variables(['ab=cd', 'ef=xyz'])
272
    [('$ef', 'xyz'), ('$ab', 'cd')]
273 274 275 276 277 278 279 280
    '''

    variables = []

    if var:
        for v in var:
            try:
                key, var = v.split('=')
281
                variables = [('$' + key, var)] + variables
282 283 284 285 286 287

            except IOError:
                raise ShouldException('Error in parsing variable definition: ' + v)
    return variables


288 289 290 291 292
def print_variables(variables):
    for (k, v) in variables:
        print('%s=%s' % (k, v))
    print('')

293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
def replace_variables(s, variables):
    '''
    >>> replace_variables('hello', None)
    'hello'

    >>> replace_variables('hello', [('hell', 'w'), ('o', 'orld')])
    'world'

    >>> replace_variables('xyz xyz', [('y', 'abc')])
    'xabcz xabcz'
    '''

    if variables:
        for (key, val) in variables:
            s = s.replace(key, val)
    return s


311 312 313 314
class OrderedDefaultListDict(OrderedDict):
    def __missing__(self, key):
        self[key] = value = []
        return value
315

316
class Stats():
317 318
    '''
    >>> s = Stats('foo')
319
    >>> s.up(2)
320 321
    >>> list(s.keys())
    [2]
322 323
    >>> s[2]
    [1]
324 325

    >>> t = Stats()
326 327
    >>> t.up(2, 'hello')
    >>> t.up(3)
328 329 330 331 332 333

    >>> u = s + t
    >>> sorted(u.keys())
    [2, 3]
    >>> list(s.keys())
    [2]
334 335 336

    >>> sorted(u.items())
    [(2, [1, 'hello']), (3, [1])]
337
    '''
338

339
    def __init__(self, item=''):
340
        self.stats = OrderedDefaultListDict()
341
        self.item = item
342 343 344 345

    def __getitem__(self, key):
        return self.stats[key]

346 347 348
    def up(self, key, data=1):
        self.stats[key].append(data)

349 350 351
    def __setitem__(self, key, value):
        self.stats[key] = value

352 353 354
    def keys(self):
        return self.stats.keys()

355 356 357 358 359 360 361 362 363
    def items(self):
        return self.stats.items()

    def values(self):
        return self.stats.values()

    def total(self):
        return sum(map(len, self.stats.values()))

364 365
    def __add__(self, other):
        result = Stats(self.item)
366 367
        for data in (self, other):
            for key in data.keys():
368
                result.stats[key] += data.stats[key]
369 370
        return result

371 372 373 374
    def str_status(self, status, colorize=True):
        s = '==> '
        s += STATUS[status]
        s += ' - '
375
        s += ' '.join(['%s:%d' % (STATUS[key], len(val)) for (key, val) in self.items()] + ['total:%s' % self.total()])
376
        if self.item:
377
            s += ' ' + self.item + ('s' if self.total() > 1 else '')
378
        return color(STATUS_COLORS[status], s, colorize)
379 380 381



382

383
class TestCaseAbstract:
384 385 386 387 388 389
    def __init__(self):
        raise NotImplemented

    def str_additional_status(self, verbose=False):
        return ''

390
    def str_status(self, verbose=False, names=STATUS, colorize=True):
391
        s = ''
392
        s += color(STATUS_COLORS[self.status], names[self.status], colorize)
393 394 395 396
        s += self.str_additional_status(verbose)

        return s

397 398 399 400 401 402 403
    def xml(self):
        x = ET.Element('testcase', {'name': self.name, 'status': STATUS_XML[self.status]})
        if self.status in WARN_STATUS:
            x.append(ET.Element(STATUS_XML[self.status],
                                {'message': repr(self) + '\n' + self.str_status(names=STATUS_XML, colorize = False)}))
        return x

404
    def tap(self, names=STATUS_TAP, colorize=False):
405 406 407
        s = []

        if self.status is not None:
408
            s.append(self.str_status(names=names, colorize=colorize))
409 410 411 412 413 414

        if self.name:
            s.append(self.name)

        return ' - '.join(s)

415 416 417
    def str(self, colorize):
        return self.tap(names=STATUS, colorize=colorize)

418
    def __str__(self):
419
        return self.str(colorize=True)
420

421 422
    def __repr__(self):
        raise NotImplemented
423

424
class ExternalTestCase(TestCaseAbstract):
425 426 427 428 429 430 431
    def __init__(self, name, status, info=''):
        self.name = name
        self.status = status
        self.info = info

    def str_additional_status(self, verbose = False):
        s = ''
432
        if self.status in WARN_STATUS or verbose:
433 434 435 436 437 438
            s += ' (%s)' % self.info
        return s

    def test(self, *args, **kwargs):
        pass

439 440 441 442
    def __repr__(self):
        return self.info


443
class TestCase(TestCaseAbstract):
444
    '''
445
    >>> test = TestCase('', 'hello')
446 447
    >>> repr(test)
    ':hello'
448

449
    >>> test.str_status(colorize=False)
450 451
    'not run'

452
    >>> test.test(['world'])
453 454
    False

455 456 457 458 459
    >>> test.status
    False

    >>> test.test(['hello'])
    True
460

461

462
    >>> test = TestCase('3', 'hello')
463 464
    >>> repr(test)
    '3:hello'
465 466 467

    >>> test.test(['hello'])
    False
468 469
    >>> test.count
    1
470
    >>> print(test.str(colorize=False))
471
    failed (1/3)
472 473
    >>> test.tap()
    'not ok (1/3)'
474 475 476 477 478

    >>> test.test(['hello'] * 3)
    True


479
    >>> TestCase('r2', ' e.*o ').test(['hello', 'ello', 'world'])
480 481
    True

482
    >>> TestCase('z1', ' e').test(['hello', 'h ello'])
483 484
    True

485
    >>> TestCase('rl', 'e.*o').test(['hel', 'lo'])
486 487
    True

488
    >>> TestCase('', 'e o').test(['e  o'])
489 490
    False

491
    >>> TestCase('f', 'e o').test(['e  o'])
492 493
    'TODO'

494
    >>> TestCase('b', 'e o').test(['e  o'])
495 496
    True

497
    >>> TestCase('b', 'e    o').test(['e  o'])
498 499
    True

500
    >>> TestCase('w2', 'o').test(['hello world'])
501 502
    True

503
    >>> TestCase('wW2', 'o').test(['hello world'])
504 505
    False

506
    >>> TestCase('wr2', 'a.c').test(['bli abc axc bla'])
507 508
    True

509

510
    >>> repr(TestCase('x3y', 'hello'))
511
    'xy3:hello'
512

513
    >>> print(TestCase('1x2', 'hello')) # doctest: +IGNORE_EXCEPTION_DETAIL
514 515 516
    Traceback (most recent call last):
     ...
    ShouldException: Error in parsing modifiers: 1x2
517 518 519
    '''

    def __init__(self, modifiers, expression, name=''):
520 521
        self.name = name
        self.status = None
522
        self.count = '?'
523 524 525 526 527 528 529 530

        # Extract self.expected_count from modifiers
        m = RE_MODIFIERS.match(modifiers)
        if not m:
            raise ShouldException('Error in parsing modifiers: ' + modifiers)
        self.modifiers = m.group(1) + m.group(3)
        self.expected_count = int(m.group(2)) if m.group(2) else None

531
        # Parse modifiers
532
        self.mods = parser_mod.parse_modifiers(self.modifiers)
533

534
        self.expression = expression if self.mods.ltspaces else expression.strip()
535
        if self.mods.blanks:
536 537
            while '  ' in self.expression:
                self.expression = self.expression.replace('  ', ' ')
538 539
            self.expression = self.expression.replace(' ', '\s+')
            self.mods.regex = True
540 541 542 543 544 545 546

        self.regex = None
        if self.mods.regex:
            if self.mods.ignore_case:
                self.regex = re.compile(self.expression, re.IGNORECASE)
            else:
                self.regex = re.compile(self.expression)
547

548
    def test(self, lines, variables=None, verbose=0):
549
        if self.mods.multi_lines:
550
            lines = [' '.join([l.rstrip(ENDLINE_CHARS) for l in lines])]
551

552 553
        expression_var = replace_variables(self.expression, variables)

554 555 556
        if not self.regex and self.mods.ignore_case:
            expression_var = expression_var.upper()

557
        self.count = 0
558
        for l in lines:
559
            if self.regex:
560 561 562
                if self.mods.count_all:
                    self.count += len(self.regex.findall(l))
                elif self.regex.search(l):
563
                    self.count += 1
564 565 566 567 568
            else:
                if self.mods.ignore_case:
                    l = l.upper()
                if expression_var in l:
                    self.count += l.count(expression_var) if self.mods.count_all else 1
569

570 571 572 573 574 575 576 577
        if self.expected_count is None:
            self.status = (self.count > 0)
        elif self.mods.less_than:
            self.status = (self.count < self.expected_count)
        elif self.mods.more_than:
            self.status = (self.count > self.expected_count)
        else:
            self.status = (self.count == self.expected_count)
578

579
        if self.mods.todo:
580
            self.status = [TODO, TODO_PASSED][self.status]
581

582 583
        if verbose > 0:
            print('')
584
            print(self.str_status(True) + ' ' + repr(self))
585

586
        return self.status
587

588
    def str_additional_status(self, verbose=False):
589 590
        s = ''

591
        if self.status in WARN_STATUS or verbose:
592 593 594
            s += ' (%s/%s%s)' % (self.count,
                                 MOD_LESS_THAN if self.mods.less_than else MOD_MORE_THAN if self.mods.more_than else '',
                                 self.expected_count if self.expected_count is not None else '+')
595 596 597

        return s

598 599 600
    def __repr__(self):
        return '%s%s:%s' % (self.modifiers, self.expected_count if self.expected_count is not None else '', self.expression)

601

602
class TestSuite():
603
    '''
604
    >>> s = TestSuite()
605 606 607
    >>> s.load(['echo "hello"', '$My test', ':hello'])
    >>> print(s)
    echo "hello"
608
    My test
609 610

    >>> s.test()
611
    True
612 613
    >>> s.tests[0].status
    True
614

615
    >>> s2 = TestSuite('r')
616
    >>> s2.variables.append(("$LAUNCHER", ""))
617
    >>> s2.variables.append(("$EXTRA", ""))
618
    >>> s2.load(['echo "hello"', '$ A nice test', ':e.*o'])
619
    >>> s2.test(verbose = 1, colorize = False)   # doctest: +NORMALIZE_WHITESPACE
620
    echo "hello"
621 622
      stdout --> 1 lines
      stderr --> 0 lines
623
    ok - A nice test
624
    ok - Exit code is 0
625
    True
626

627
    >>> s2.str_status(colorize = False)
628
    '==> ok - ok:2 total:2 tests'
629 630

    >>> print(s2.tap())   # doctest: +NORMALIZE_WHITESPACE
631
    1..2
632
    ok - A nice test
633
    ok - Exit code is 0
634 635
    '''

636
    def __init__(self, modifiers = '', cd = None, name = '', timeout = TIMEOUT):
637
        self.name = name
638 639
        self.requires = True
        self.requires_cmd = None
640
        self.requires_stderr = []
641 642
        self.cmds = []
        self.tests = []
643 644
        self.stdin = []
        self.stdout = []
645
        self.test_lines = []
646
        self.status = None
647
        self.modifiers = modifiers
648
        self.variables = []
649
        self.stats = Stats('test')
650
        self.source = None
651
        self.cd = cd
652
        self.use_launcher = True
653
        self.expected_exit_code = 0
654
        self.elapsed_time = None
655
        self.timeout = timeout
656 657 658

    def load(self, should_lines):
        name = ''
659
        this_cmd_continues = False
660 661
        for l in should_lines:

662
            l = l.lstrip().rstrip(ENDLINE_CHARS)
663 664 665 666 667 668 669
            if not l:
                continue

            # Comment
            if l.startswith(TOKEN_COMMENT):
                continue

670 671 672 673 674
            # Directive -- Requires
            if l.startswith(DIRECTIVE_REQUIRES):
                self.requires_cmd = l[len(DIRECTIVE_REQUIRES):].strip()
                continue

675 676 677 678 679
            # Directive -- No launcher
            if l.startswith(DIRECTIVE_NO_LAUNCHER):
                self.use_launcher = False
                continue

680 681 682 683 684
            # Directive -- No extra options
            if l.startswith(DIRECTIVE_NO_EXTRA):
                self.variables = [(VAR_EXTRA, '')] + self.variables
                continue

685 686
            # Directive -- Source
            if l.startswith(DIRECTIVE_SOURCE):
687
                self.source = os.path.join(self.cd if self.cd else '', l[len(DIRECTIVE_SOURCE):].strip())
688 689
                continue

690 691 692
            # Directive -- Options
            if l.startswith(DIRECTIVE_OPTIONS):
                opts, unknown = options.parse_known_args(l[len(DIRECTIVE_OPTIONS):].split())
693
                self.variables = populate_variables(opts.var) + self.variables
694 695
                if opts.mod:
                    self.modifiers += ''.join(opts.mod)
696 697
                continue

698 699 700 701 702
            # Directive -- Exit code
            if l.startswith(DIRECTIVE_EXIT_CODE):
                self.expected_exit_code = int(l[len(DIRECTIVE_EXIT_CODE):].strip())
                continue

703 704
            # Name
            if l.startswith(TOKEN_NAME):
705
                name = l[1:].strip()
706 707
                continue

708 709 710 711 712 713 714 715 716
            # Directive -- Command
            if l.startswith(DIRECTIVE_SCRIPT):
                l = l[len(DIRECTIVE_SCRIPT):]

            # Directive -- Others
            if l.startswith(TOKEN_DIRECTIVE):
                sys.stderr.write('! Unknown directive: %s\n' % l)
                continue

717
            # Test
718
            if RE_TEST.search(l):
719 720
                pos = l.find(TOKEN_TEST)
                modifiers, expression = l[:pos], l[pos+1:]
721
                self.tests.append(TestCase(self.modifiers + modifiers, expression, name))
722 723 724
                continue

            # Command
725
            l = l.strip()
726 727 728 729 730 731 732 733 734 735 736
            next_cmd_continues = l.endswith(CONTINUATION_CHAR)
            if next_cmd_continues:
                l = l[:-1]

            if this_cmd_continues:
                self.cmds[-1] += l
            else:
                self.cmds.append(l)

            this_cmd_continues = next_cmd_continues

737

738 739 740 741 742
    def print_stderr(self, colorize=True):
        print('  stdout --> %s lines' % len(self.stdout))
        print('  stderr --> %s lines' % len(self.stderr))
        print(color(ANSI.CYAN, ''.join(self.stderr), colorize))

743 744 745
    def skip_all(self, reason, verbose=1):
        if verbose > 0:
            print('Skipping tests: %s' % reason)
746 747
        for test in self.tests:
            test.status = SKIP
748
            self.stats.up(test.status)
749 750
        self.status = SKIP

751
    def test(self, variables=[], verbose=0, colorize=True):
752

753
        self.variables_all = self.variables + variables
754
        if verbose > 1:
755
            print_variables(self.variables_all)
756 757

        def cmd_variables_cd(cmd):
758
            cmd = replace_variables(cmd, self.variables_all)
759 760 761 762 763 764
            if self.cd:
                cmd = 'cd %s ; ' % self.cd + cmd
            if verbose > 0:
                print(color(ANSI.MAGENTA, cmd, colorize))
            return cmd

765 766
        self.status = True

767
        if self.requires_cmd:
768 769
            requires_cmd = cmd_variables_cd(self.requires_cmd)
            p = subprocess.Popen(requires_cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
770 771
            self.requires = (p.wait() == 0)
            self.requires_stderr = [l.decode(errors='replace') for l in p.stderr.readlines()]
772
            if verbose > 0:
773
                print(color(ANSI.CYAN, ''.join(self.requires_stderr), colorize))
774 775

            if not self.requires:
776
                self.skip_all('Condition is not met: %s' % self.requires_cmd, verbose)
777
                return self.status
778

779
        if not self.use_launcher:
780
            if replace_variables(VAR_LAUNCHER, self.variables_all):
781
                self.skip_all('%s while %s is given' % (DIRECTIVE_NO_LAUNCHER, VAR_LAUNCHER), verbose)
782 783
                return self.status

784 785
        start_time = time.time()

786
        cmd = ' ; '.join(map(pre_process, self.cmds))
787 788
        cmd = cmd_variables_cd(cmd)

789 790
        f_stdout = tempfile.TemporaryFile()
        f_stderr = tempfile.TemporaryFile()
791
        p = subprocess.Popen([cmd], shell=True,
792
                             stdout=f_stdout, stderr=f_stderr,
793
                             close_fds=True)
794 795

        try:
796
            self.exit_code = p.wait(self.timeout)
797
            self.tests.append(ExternalTestCase('Exit code is %d' % self.expected_exit_code, self.exit_code == self.expected_exit_code, str(self.exit_code)))
798
        except subprocess.TimeoutExpired:
799
            self.exit_code = None
800
            self.tests.append(ExternalTestCase('Exit code is %d' % self.expected_exit_code, SKIP, 'timeout after %s seconds' % self.timeout))
801
            p.kill()
802

803 804 805 806 807 808
        f_stdout.seek(0)
        f_stderr.seek(0)
        self.stdout = [l.decode(errors='replace') for l in f_stdout.readlines()]
        self.stderr = [l.decode(errors='replace') for l in f_stderr.readlines()]
        f_stdout.close()
        f_stderr.close()
809

Mathieu Giraud's avatar
Mathieu Giraud committed
810 811 812 813 814 815 816

        if verbose > 0:
            self.print_stderr(colorize)

        self.test_lines = open(self.source).readlines() if self.source else self.stdout

        for test in self.tests:
817
            test.test(self.test_lines, variables=self.variables_all, verbose=verbose-1)
818
            self.stats.up(test.status)
Mathieu Giraud's avatar
Mathieu Giraud committed
819

820 821
            # When a test fails, the file fails
            if test.status is False:
Mathieu Giraud's avatar
Mathieu Giraud committed
822 823
                self.status = False

824 825 826 827
            # When the file is not failing, we may report a more sublte status
            if test.status in WARN_STATUS and self.status is True:
                self.status = test.status

828
            if verbose > 0 or test.status in WARN_STATUS:
829
                print(test.str(colorize))
Mathieu Giraud's avatar
Mathieu Giraud committed
830 831 832 833 834 835 836

        if self.status is False and verbose <= 0:
            print(color(ANSI.MAGENTA, cmd, colorize))
            self.print_stderr(colorize)

        if self.status is False or verbose > 1:
            print(LINE)
837 838 839 840 841 842 843
            if len(self.test_lines) <= MAX_DUMP_LINES:
                print(''.join(self.test_lines), end='')
            else:
                print(''.join(self.test_lines[:MAX_HALF_DUMP_LINES]), end='')
                print(color(ANSI.MAGENTA, '... %d other lines ...' % (len(self.test_lines) - 2*MAX_HALF_DUMP_LINES), colorize))
                print(''.join(self.test_lines[-MAX_HALF_DUMP_LINES:]), end='')

Mathieu Giraud's avatar
Mathieu Giraud committed
844
            print(LINE)
845

846 847
        self.elapsed_time = time.time() - start_time

848 849
        return self.status

850
    def str_status(self, colorize=True):
851
        return self.stats.str_status(self.status, colorize)
852

853 854 855 856 857 858 859 860 861
    def xml(self):
        x = ET.Element('testsuite',
                       {'id': self.name,
                        'name': self.name,
                        'cmd': str(self.cmds),
                        'tests': str(self.stats.total()),
                        'failures': str(len(self.stats[False])),
                        'skipped': str(len(self.stats[SKIP])),
                        'time': self.str_elapsed_time(tag=''),
862
                        'timestamp': datetime.datetime.now().isoformat()
863 864 865
                       })
        for test in self.tests:
            x.append(test.xml())
866 867 868 869 870 871

        v = ET.Element('properties')
        for (key, val) in self.variables_all:
            v.append(ET.Element('property', {'name': key, 'value': val}))
        x.append(v)

872 873
        return x

874 875 876
    def tap(self):
        s = ''
        s += '1..%d' % len(self.tests) + '\n'
877
        s += '\n'.join(map(TestCase.tap, self.tests))
878 879 880
        s += '\n'
        return s

881 882
    def str_elapsed_time(self, tag='s'):
        return ('%.2f%s' % (self.elapsed_time, tag)) if self.elapsed_time is not None else ''
883

884 885 886 887
    def __str__(self):
        s = ''
        s += '\n'.join(self.cmds)
        s += '\n'
888
        s += self.str_elapsed_time()
889
        s += '\n'.join(map(str,self.tests))
890 891 892
        if self.status is not None:
            s += '\n'
            s += self.str_status()
893 894 895
        return s


896 897
class FileSet():

898
    def __init__(self, files, modifiers = '', timeout = TIMEOUT):
899
        self.files = files
900
        self.sets = []
901 902
        self.modifiers = modifiers
        self.status = None
903
        self.stats = Stats('file')
904
        self.stats_tests = Stats('test')
905
        self.timeout = timeout
906

907 908 909
    def __len__(self):
        return len(self.files)

910
    def test(self, variables=None, cd=None, cd_same=False, output=None, verbose=0):
911
        self.status = True
912

913 914
        try:
          for f in self.files:
915 916
            if verbose > 0:
                print(f)
917
            cd_f = os.path.dirname(f) if cd_same else cd
918
            s = TestSuite(self.modifiers, cd_f, name = f, timeout = self.timeout)
919
            self.sets.append(s)
920
            s.load(open(f))
921

922
            s.test(variables, verbose - 1)
923
            self.stats.up(s.status, f)
924 925
            if not s.status:
                self.status = False
926

927 928
            self.stats_tests += s.stats

929 930
            filename_without_ext = os.path.splitext(f)[0]

931
            if output and OUT_LOG in output:
932
                write_to_file(filename_without_ext + OUT_LOG, ''.join(s.test_lines))
933 934

            if output and OUT_TAP in output:
935
                write_to_file(filename_without_ext + OUT_TAP, s.tap())
936

937 938 939
            if verbose > 0 or s.status is False:
                if not verbose:
                    print(f, end=' ')
940
                if s.elapsed_time:
941 942
                    if s.elapsed_time >= SHOW_ELAPSED_TIME_ABOVE:
                        print(s.str_elapsed_time())
943
                print(s.str_status())
944
                print('')
945 946
        except KeyboardInterrupt:
            print('==== interrupted ====\n')
947

948 949 950
        if output and OUT_XML in output:
            self.xml().write(OUT_XML)

951 952
        print('Summary', end=' ')
        print(self.stats.str_status(self.status))
953 954
        print('Summary', end=' ')
        print(self.stats_tests.str_status(self.status))
955

956 957 958 959 960 961
        for sta in self.stats.keys():
            if sta == True:
                continue
            print('files with %s:' % color(STATUS_COLORS[sta], STATUS[sta]))
            for f in self.stats[sta]:
                print('  ' + f)
962

963
        return self.status
964

965 966
    def xml(self):
        x = ET.Element('testsuites',
967
                       {'id': 'Test at %s' % datetime.datetime.now().isoformat(),
968 969 970 971 972 973 974 975 976 977
                        'name': 'tested by should',
                        'tests': str(self.stats_tests.total()),
                        'failures': str(len(self.stats_tests[False])),
                       })

        for s in self.sets:
            x.append(s.xml())

        return ET.ElementTree(x)

978
    def write_retry(self, argv, argv_remove, verbose=1):
979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001
        '''
        If there were tests in WARN_STATUS, write the RETRY file
        with, non-file arguments and WARN_STATUS files.
        '''

        # cmd = [sys.executable, sys.argv[0]]

        files = []

        for sta in WARN_STATUS:
            files += self.stats[sta]

        if not files:
            return

        args = []
        for arg in argv:
            if arg not in argv_remove:
                args.append(arg)

        with open(RETRY, 'w') as f:
            f.write('\n'.join(args + files) + '\n')

1002 1003 1004
        if verbose > 0:
            print('%s %s will relaunch these tests.' % (sys.argv[0], RETRY_FLAG))

1005 1006 1007 1008 1009 1010
def read_retry():
    try:
        return [l.rstrip() for l in open(RETRY).readlines()]
    except:
        return []

1011

1012
if __name__ == '__main__':
Mathieu Giraud's avatar
Mathieu Giraud committed
1013
    argv = (['@' + DEFAULT_CFG] + sys.argv[1:]) if os.path.exists(DEFAULT_CFG) else sys.argv[1:]
1014 1015 1016 1017 1018 1019 1020 1021 1022 1023

    if RETRY_FLAG in argv:
        retry = read_retry()
        argv += retry
        if retry:
            print(color(ANSI.BLUE, "Retrying previous failed or warned tests"))
        else:
            print(color(ANSI.RED, "Nothing to retry"))
            sys.exit(2)

1024
    args = parser.parse_args(argv)
1025
    variables = populate_variables(args.var)
1026
    variables.append((VAR_LAUNCHER, args.launcher))
Mathieu Giraud's avatar
Mathieu Giraud committed
1027
    variables.append((VAR_EXTRA, args.extra))
1028 1029

    if args.verbose>0:
1030
        print_variables(variables)
1031

1032 1033 1034 1035
    if args.shuffle:
        print("Shuffling test files")
        random.shuffle(args.file)

1036
    fs = FileSet(args.file, timeout = args.timeout, modifiers=''.join(args.mod if args.mod else []))
1037
    status = fs.test(variables = variables, cd = args.cd, cd_same = args.cd_same, output = args.output, verbose = args.verbose)
1038

1039 1040
    if len(fs) > 1:
        retry = fs.write_retry(sys.argv[1:], args.file, verbose = args.verbose)
1041

1042
    sys.exit(0 if status else 1)
1043 1044