diff --git a/test/regression/db.zip b/test/regression/db.zip index c3a6b85ca..7cd9a51d2 100644 Binary files a/test/regression/db.zip and b/test/regression/db.zip differ diff --git a/test/regression/gen_db.py b/test/regression/gen_db.py index 3f1b0df06..c543cc642 100644 --- a/test/regression/gen_db.py +++ b/test/regression/gen_db.py @@ -92,6 +92,10 @@ def process_dir(d, outfile, file_filter): outfile.flush() if subprocess.call(cmd, stdout=outfile, stderr=outfile, shell=False): print("Failure processing " + fullp) + + # spit out an empty file to indicate that this failure is expected + with open(outf,'wb') as f: + pass return num diff --git a/test/regression/run.py b/test/regression/run.py index 403e329ca..01cefdb07 100644 --- a/test/regression/run.py +++ b/test/regression/run.py @@ -124,8 +124,8 @@ class results: def report_results(self): """Write results to ../results/run_regression_suite_failures.txt""" - print("\n" + ('='*60) + "\n" + "SUCCESS: {0}\r\nFAILURE: {1}".format( - len(self.success), len(self.failures)) + + print("\n" + ('='*60) + "\n" + "SUCCESS: {0}\nFAILURE: {1}\nPercentage good: {2}".format( + len(self.success), len(self.failures), len(self.success)/(len(self.success)+len(self.failures)) ) + "\n" + ('='*60) + "\n") with open(os.path.join('..', 'results',outfilename_failur), "wt") as f: @@ -185,15 +185,18 @@ def process_dir(d, outfile_results, zipin, result): if r and not failure: result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r) continue - elif failure: + elif failure and not r: result.fail(fullpath, outfile_expect, pppreset, EXPECTED_FAILURE_NOT_MET) continue with open(outfile_expect, "wb") as s: s.write(input_expected) - - with open(outfile_actual, "rb") as s: - input_actual = s.read() + + try: + with open(outfile_actual, "rb") as s: + input_actual = s.read() + except IOError: + continue if len(input_expected) != len(input_actual): result.fail(fullpath, outfile_expect, pppreset, DATABASE_LENGTH_MISMATCH,