Commit 348ba98b authored by Mikaël Salson's avatar Mikaël Salson

should-vdj-to-tap.py: Count the number of bugged assignations

Each bugged sequence must be tagged correspondingly. However the benefit is that
we don't need to update a counter without knowing what sequence has changed.

There is still a (minor) drawback: only the count is actually checked. Therefore
we could have some newly failed sequences balancing sequences not failing anymore.
However in the output we will see the failed sequences that should have not.

This is another benefit: the output is now much clearer as we only see sequences
that have failed (and should have not) or sequences that succeeded (and should have not).
parent 87a31996
......@@ -10,8 +10,6 @@ OTHER_SRC=$(wildcard unit-tests/*.cpp)
LIB=../core/vidjil.a ../lib/lib.a
SHOULD=$(wildcard should-get-tests/*.should-get) $(wildcard bugs/*.should-get)
SHOULD_LOG=$(SHOULD:.should-get=.tap)
SHOULD_VDJ_EXPECTED_FAILS=--expected-fails 82
SHOULD_LOCUS_EXPECTED_FAILS=--expected-fails 2
SHOULD_VDJ=$(wildcard should-vdj-tests/*.should-vdj.fa)
SHOULD_VDJ_VDJ=$(SHOULD_VDJ:.should-vdj.fa=.1.vdj)
SHOULD_LOCUS=$(wildcard should-vdj-tests/*.should-locus.fa)
......@@ -68,7 +66,7 @@ shouldvdj_if_python:
fi
shouldlocus: vidjil
$(SHOULD_VDJ_TO_TAP) -2q -r $(SHOULD_LOCUS_EXPECTED_FAILS) $(SHOULD_VDJ) $(SHOULD_LOCUS)
$(SHOULD_VDJ_TO_TAP) -2q -r $(SHOULD_VDJ) $(SHOULD_LOCUS)
shouldlocus_and_vdj: vidjil
@echo "*** Launching .should-vdj-fa tests..."
......@@ -77,19 +75,19 @@ shouldlocus_and_vdj: vidjil
@echo "*** All .should-vdj.fa tests passed"
shouldvdj_no_relaunch:
$(SHOULD_VDJ_TO_TAP) $(SHOULD_VDJ_EXPECTED_FAILS) $(SHOULD_VDJ_VDJ)
$(SHOULD_VDJ_TO_TAP) $(SHOULD_VDJ_VDJ)
shouldvdj:
$(SHOULD_VDJ_TO_TAP) $(SHOULD_VDJ_EXPECTED_FAILS) $(SHOULD_VDJ)
$(SHOULD_VDJ_TO_TAP) $(SHOULD_VDJ)
shouldvdj_with_rc:
$(SHOULD_VDJ_TO_TAP) $(SHOULD_VDJ_EXPECTED_FAILS) -r $(SHOULD_VDJ)
$(SHOULD_VDJ_TO_TAP) -r $(SHOULD_VDJ)
shouldvdj_with_rc_merged:
cat $(SHOULD_VDJ) > should-vdj-tests/should-vdj.merged.fa
$(SHOULD_VDJ_TO_TAP) -r should-vdj-tests/should-vdj.merged.fa
# When the global test suite is passing, individual failed tests (counted in SHOULD_*_EXPECTED_FAILS)
# When the global test suite is passing, individual failed tests (tagged with BUG or BUG-LOCUS in the *.should-vdj.fa files)
# can be marked as 'TODO' to make continuous integration happy
mark_failed_tests_as_todo:
-sed -e "s/^\(not ok [0-9]*\) /\1 # TODO ##/" -i".bak" */*.tap
......
......@@ -42,7 +42,6 @@ parser.add_argument('--ignore_cdr3', '-3', action='store_true', help='ignore CDR
parser.add_argument('--after-two', '-2', action='store_true', help='compare only the right part of the pattern after two underscores (locus code)')
parser.add_argument('--revcomp', '-r', action='store_true', help='duplicate the tests on reverse-complemented files')
parser.add_argument('--directory', '-d', default='../..', help='base directory where Vidjil is. This value is used by the default -p and -q values (%(default)s)')
parser.add_argument('--expected-fails', '-e', type=int, default=0, help='number of expected non-TODO failed tests -- do not exit with error for this number')
parser.add_argument('--verbose', '-v', action='store_true')
......@@ -67,10 +66,10 @@ def special_keywords(after_two):
global_failed = 0
global_stats = defaultdict(int)
global_stats_bug = defaultdict(int)
global_stats_failed = defaultdict(int)
global_stats_todo = defaultdict(int)
def should_pattern_to_regex(p):
'''
Converts patterns such as the following ones into a Python regex.
......@@ -236,6 +235,9 @@ def should_result_to_tap(should_pattern, result, tap_id):
globals()['global_stats_todo'][locus] += 1
else:
globals()['global_failed'] += 1
if (args.after_two and 'BUG-LOCUS' in should_pattern)\
or (not args.after_two and 'BUG' in should_pattern):
globals()['global_stats_bug'][locus] += 1
tap += 'not '
......@@ -332,17 +334,18 @@ if __name__ == '__main__':
print
print "=== Summary, should-vdj tests ===" + (' (only locus)' if args.after_two else '')
print " tested passed failed (todo)"
print " tested passed bug failed (todo)"
for locus in sorted(global_stats):
print " %-5s %4d %4d %4d %4s" % (locus, global_stats[locus], global_stats[locus] - global_stats_failed[locus], global_stats_failed[locus],
print " %-5s %4d %4d %4d %4d %4s" % (locus, global_stats[locus], global_stats[locus] - global_stats_failed[locus], global_stats_bug[locus], global_stats_failed[locus],
("(%d)" % global_stats_todo[locus] if global_stats_todo[locus] else ''))
print " ===== %4d %4d %4d %4s" % (sum(global_stats.values()), sum(global_stats.values()) - sum(global_stats_failed.values()), sum(global_stats_failed.values()),
print " ===== %4d %4d %4d %4d %4s" % (sum(global_stats.values()), sum(global_stats.values()) - sum(global_stats_failed.values()), sum(global_stats_bug.values()), sum(global_stats_failed.values()),
"(%d)" % sum(global_stats_todo.values()))
print
if args.revcomp:
args.expected_fails *= 2
if not global_failed == args.expected_fails:
print "! We were expecting %s non-TODO failed tests, but there are %s such failures." % (args.expected_fails, global_failed)
global_bug = sum(global_stats_bug.values())
if global_bug < global_failed:
print "! We were expecting %s failed tests, but there are %s such failures." % (global_bug, global_failed)
sys.exit(1)
if global_bug > global_failed:
print "! There were less failed sequences that expected. Please update the files accordingly!"
sys.exit(2)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment