From fa0ec40b2491184ea044afeec718e485c42d84e8 Mon Sep 17 00:00:00 2001 From: Alexander Gessler Date: Wed, 4 Feb 2015 14:09:10 +0100 Subject: [PATCH] Regression suite: 2.7 compatibility, drop old logic to find assimp binary and require it to be explicitly specified. Process files in a stable order to make results diffable. --- test/regression/gen_db.py | 29 +++++++++++---- test/regression/run.py | 40 +++++++++++++++----- test/regression/utils.py | 77 ++++----------------------------------- 3 files changed, 60 insertions(+), 86 deletions(-) diff --git a/test/regression/gen_db.py b/test/regression/gen_db.py index ef10a298d..d405af9db 100644 --- a/test/regression/gen_db.py +++ b/test/regression/gen_db.py @@ -40,8 +40,15 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --------------------------------------------------------------------------- -"""Generate the regression database db.zip from the files in the -/test/models directory. Older databases are overwritten with no prompt. +""" +Generate the regression database db.zip from the files in the /test/models +directory. Older databases are overwritten with no prompt but can be restored +using Git as needed. + +Use --help for usage. + +On Windows, use ``py run.py `` to make sure command line parameters +are forwarded to the script. """ import sys @@ -52,9 +59,14 @@ import zipfile import settings import utils -usage = """gen_db [-i=...] [-e=...] [-p] [-n] +usage = """gen_db [assimp_binary] [-i=...] [-e=...] [-p] [-n] + +The assimp_cmd (or assimp) binary to use is specified by the first +command line argument and defaults to ``assimp``. + +To build, set ``ASSIMP_BUILD_ASSIMP_TOOLS=ON`` in CMake. If generating +configs for an IDE, make sure to build the assimp_cmd project. -(lists of file extensions are comma delimited, i.e. `3ds,lwo,x`) -i,--include: List of file extensions to update dumps for. If omitted, all file extensions are updated except those in `exclude`. @@ -66,6 +78,8 @@ usage = """gen_db [-i=...] [-e=...] [-p] [-n] Dont' change anything. -n,--nozip: Don't pack to ZIP archive. Keep all dumps in individual files. + +(lists of file extensions are comma delimited, i.e. `3ds,lwo,x`) """ # ------------------------------------------------------------------------------- @@ -87,7 +101,7 @@ def process_dir(d, outfile, file_filter): outf = os.path.join(os.getcwd(), settings.database_name, utils.hashing(fullp, pp)) - cmd = [utils.assimp_bin_path,"dump",fullp,outf,"-b","-s","-l"] + pp.split() + cmd = [ assimp_bin_path, "dump", fullp, outf, "-b", "-s", "-l" ] + pp.split() outfile.write("assimp dump "+"-"*80+"\n") outfile.flush() if subprocess.call(cmd, stdout=outfile, stderr=outfile, shell=False): @@ -158,7 +172,8 @@ def gen_db(ext_list,outfile): # ------------------------------------------------------------------------------- if __name__ == "__main__": - utils.find_assimp_or_die() + assimp_bin_path = sys.argv[1] if len(sys.argv) > 1 else 'assimp' + def clean(f): f = f.strip("* \'") return "."+f if f[:1] != '.' else f @@ -184,7 +199,7 @@ if __name__ == "__main__": outfile = open(os.path.join("..", "results", "gen_regression_db_output.txt"), "w") if ext_list is None: - (ext_list, err) = subprocess.Popen([utils.assimp_bin_path, "listext"], + (ext_list, err) = subprocess.Popen([assimp_bin_path, "listext"], stdout=subprocess.PIPE).communicate() ext_list = str(ext_list).lower().split(";") diff --git a/test/regression/run.py b/test/regression/run.py index 6d776b362..73172cf04 100644 --- a/test/regression/run.py +++ b/test/regression/run.py @@ -41,8 +41,16 @@ # --------------------------------------------------------------------------- """ -Run the regression test suite using the settings from settings.py. +Run the regression test suite using settings from settings.py. +The assimp_cmd (or assimp) binary to use is specified by the first +command line argument and defaults to ``assimp``. + +To build, set ``ASSIMP_BUILD_ASSIMP_TOOLS=ON`` in CMake. If generating +configs for an IDE, make sure to build the assimp_cmd project. + +On Windows, use ``py run.py `` to make sure the command +line parameter is forwarded to the script. """ import sys @@ -124,8 +132,11 @@ class results: def report_results(self): """Write results to ../results/run_regression_suite_failures.txt""" + count_success = len(self.success) + count_fail = len(self.failures) + percent_good = float(count_success) / (count_success + count_fail) print("\n" + ('='*60) + "\n" + "SUCCESS: {0}\nFAILURE: {1}\nPercentage good: {2}".format( - len(self.success), len(self.failures), len(self.success)/(len(self.success)+len(self.failures)) ) + + count_success, count_fail, percent_good) + "\n" + ('='*60) + "\n") with open(os.path.join('..', 'results',outfilename_failur), "wt") as f: @@ -138,7 +149,7 @@ class results: + " for more details\n\n") # ------------------------------------------------------------------------------- -def mkoutputdir_andgetpath(fullpath, myhash, app): +def prepare_output_dir(fullpath, myhash, app): outfile = os.path.join(settings.results, "tmp", os.path.split(fullpath)[1] + "_" + myhash) try: os.mkdir(outfile) @@ -154,7 +165,7 @@ def process_dir(d, outfile_results, zipin, result): shellparams = {'stdout':outfile_results, 'stderr':outfile_results, 'shell':False} print("Processing directory " + d) - for f in os.listdir(d): + for f in sorted(os.listdir(d)): fullpath = os.path.join(d, f) if os.path.isdir(fullpath) and not f == ".svn": process_dir(fullpath, outfile_results, zipin, result) @@ -167,13 +178,16 @@ def process_dir(d, outfile_results, zipin, result): for pppreset in settings.pp_configs_to_test: filehash = utils.hashing(fullpath, pppreset) failure = False + try: input_expected = zipin.open(filehash, "r").read() # empty dump files indicate 'expected import failure' if not len(input_expected): failure = True except KeyError: - #print("Didn't find "+fullpath+" (Hash is "+filehash+") in database") + # TODO(acgessler): Keep track of this and report as error in the end. + print("Didn't find "+fullpath+" (Hash is "+filehash+") in database. Outdated "+\ + "regression database? Use gen_db.zip to re-generate.") continue # Ignore extensions via settings.py configured list @@ -184,13 +198,18 @@ def process_dir(d, outfile_results, zipin, result): print("-"*60 + "\n " + os.path.realpath(fullpath) + " pp: " + pppreset) - outfile_actual = mkoutputdir_andgetpath(fullpath, filehash, "ACTUAL") - outfile_expect = mkoutputdir_andgetpath(fullpath, filehash, "EXPECT") + outfile_actual = prepare_output_dir(fullpath, filehash, "ACTUAL") + outfile_expect = prepare_output_dir(fullpath, filehash, "EXPECT") outfile_results.write("assimp dump "+"-"*80+"\n") outfile_results.flush() - command = [utils.assimp_bin_path,"dump",fullpath,outfile_actual,"-b","-s","-l"]+pppreset.split() + command = [assimp_bin_path, + "dump", + fullpath, outfile_actual, "-b", "-s", "-l" ] +\ + pppreset.split() + r = subprocess.call(command, **shellparams) + print(r) if r and not failure: result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r) @@ -216,7 +235,7 @@ def process_dir(d, outfile_results, zipin, result): outfile_results.write("assimp cmpdump "+"-"*80+"\n") outfile_results.flush() - command = [utils.assimp_bin_path,'cmpdump',outfile_actual,outfile_expect] + command = [ assimp_bin_path, 'cmpdump', outfile_actual, outfile_expect ] if subprocess.call(command, **shellparams) != 0: result.fail(fullpath, outfile_expect, pppreset, DATABASE_VALUE_MISMATCH) continue @@ -235,7 +254,6 @@ def del_folder_with_contents(folder): # ------------------------------------------------------------------------------- def run_test(): - utils.find_assimp_or_die() tmp_target_path = os.path.join(settings.results, "tmp") try: os.mkdir(tmp_target_path) @@ -261,6 +279,8 @@ def run_test(): # ------------------------------------------------------------------------------- if __name__ == "__main__": + assimp_bin_path = sys.argv[1] if len(sys.argv) > 1 else 'assimp' + print('Using assimp binary: ' + assimp_bin_path) run_test() # vim: ai ts=4 sts=4 et sw=4 diff --git a/test/regression/utils.py b/test/regression/utils.py index 73c8336ca..8e358e9b0 100644 --- a/test/regression/utils.py +++ b/test/regression/utils.py @@ -40,7 +40,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # --------------------------------------------------------------------------- -"""Shared stuff for the gen_db and run scripts """ +"""Shared stuff for the gen_db and run scripts""" # ------------------------------------------------------------------------------- def hashing(file,pp): @@ -51,75 +51,14 @@ def hashing(file,pp): and platforms, so we implement the hashing manually. """ - def myhash(instring): - # sdbm hash - res = 0 - for t in instring: - res = (ord(t) + (res<<6) + (res<<16) - res) % 2**32 - return res + file = file.replace('\\','/')+":"+pp + # SDBM hash + res = 0 + for t in file: + res = (ord(t) + (res<<6) + (res<<16) - res) % 2**32 - return hex(myhash(file.replace('\\','/')+":"+pp)) + # Python 2.7 normalization: strip 'L' suffix. + return hex(res).rstrip('L') -assimp_bin_path = None -# ------------------------------------------------------------------------------- -def find_assimp_or_die(): - """Find assimp_cmd's binary for the current platform. - - The path to the binary is stored in assimp_bin_path, the process - is aborted if it can't be found. - - """ - - import os - import platform - import sys - - def locate_file(f_list): - for f in f_list: - try: - fl = open(f,"rb") - except IOError: - continue - fl.close() - return f - return None - - global assimp_bin_path - if os.name == "nt": - search_x86 = [ - os.path.join("assimp.exe"), - os.path.join("..","..","bin","assimpcmd_release-dll_Win32","assimp.exe"), - os.path.join("..","..","bin","x86","assimp"), - os.path.join("..","..","bin","Release","assimp.exe") - ] - if platform.machine() == "x86": - search = search_x86 - else: # amd64, hopefully - search = [ - os.path.join("..","..","bin","assimpcmd_release-dll_x64","assimp.exe"), - os.path.join("..","..","bin","x64","assimp") - ] - # x64 platform does not guarantee a x64 build. Also look for x86 as last paths. - search += search_x86 - - assimp_bin_path = locate_file(search) - if assimp_bin_path is None: - print("Can't locate assimp_cmd binary") - print("Looked in", search) - sys.exit(-5) - - print("Located assimp/assimp_cmd binary from", assimp_bin_path) - elif os.name == "posix": - #search = [os.path.join("..","..","bin","gcc","assimp"), - # os.path.join("/usr","local","bin",'assimp')] - assimp_bin_path = "assimp" - print("Taking system-wide assimp binary") - else: - print("Unsupported operating system") - sys.exit(-5) - -if __name__ == '__main__': - find_assimp_or_die() - # vim: ai ts=4 sts=4 et sw=4