v4games
/
assimp
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make regression test runner include "expected failure, ok" cases in its statistics. This previously caused discrepancy between the runner and the test generator.

pull/515/head
Alexander Gessler 2015-03-28 13:43:52 +01:00 committed by Alexander Gessler
parent d717c4b2dd
commit c07cddff9e
1 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
test/regression/run.py
View File

@ -65,7 +65,7 @@ import utils
# ------------------------------------------------------------------------------- # -------------------------------------------------------------------------------
EXPECTED_FAILURE_NOT_MET, DATABASE_LENGTH_MISMATCH, \ EXPECTED_FAILURE_NOT_MET, DATABASE_LENGTH_MISMATCH, \
DATABASE_VALUE_MISMATCH, IMPORT_FAILURE, \ DATABASE_VALUE_MISMATCH, IMPORT_FAILURE, \
FILE_NOT_READABLE, COMPARE_SUCCESS = range(6) FILE_NOT_READABLE, COMPARE_SUCCESS, EXPECTED_FAILURE = range(7)
messages = collections.defaultdict(lambda: "<unknown", { messages = collections.defaultdict(lambda: "<unknown", {
EXPECTED_FAILURE_NOT_MET: EXPECTED_FAILURE_NOT_MET:
@ -88,7 +88,10 @@ messages = collections.defaultdict(lambda: "<unknown", {
COMPARE_SUCCESS: COMPARE_SUCCESS:
"""Results match archived reference dump in database\n\ """Results match archived reference dump in database\n\
\tNumber of bytes compared: {0}""" \tNumber of bytes compared: {0}""",
EXPECTED_FAILURE:
"""Expected failure was met.""",
}) })
outfilename_output = "run_regression_suite_output.txt" outfilename_output = "run_regression_suite_output.txt"
@ -211,6 +214,10 @@ def process_dir(d, outfile_results, zipin, result):
result.fail(fullpath, outfile_expect, pppreset, EXPECTED_FAILURE_NOT_MET) result.fail(fullpath, outfile_expect, pppreset, EXPECTED_FAILURE_NOT_MET)
outfile_results.write("Expected import to fail\n") outfile_results.write("Expected import to fail\n")
continue continue
elif failure and r:
result.ok(fullpath, pppreset, EXPECTED_FAILURE)
outfile_results.write("Failed as expected, skipping.\n")
continue
with open(outfile_expect, "wb") as s: with open(outfile_expect, "wb") as s:
s.write(input_expected) s.write(input_expected)