Regression suite: 2.7 compatibility, drop old logic to find assimp binary and require it to be explicitly specified. Process files in a stable order to make results diffable.

pull/459/head
Alexander Gessler 2015-02-04 14:09:10 +01:00 committed by Alex Gessler
parent 27a686210e
commit fa0ec40b24
3 changed files with 60 additions and 86 deletions

View File

@ -40,8 +40,15 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
"""Generate the regression database db.zip from the files in the <root> """
/test/models directory. Older databases are overwritten with no prompt. Generate the regression database db.zip from the files in the <root>/test/models
directory. Older databases are overwritten with no prompt but can be restored
using Git as needed.
Use --help for usage.
On Windows, use ``py run.py <arguments>`` to make sure command line parameters
are forwarded to the script.
""" """
import sys import sys
@ -52,9 +59,14 @@ import zipfile
import settings import settings
import utils import utils
usage = """gen_db [-i=...] [-e=...] [-p] [-n] usage = """gen_db [assimp_binary] [-i=...] [-e=...] [-p] [-n]
The assimp_cmd (or assimp) binary to use is specified by the first
command line argument and defaults to ``assimp``.
To build, set ``ASSIMP_BUILD_ASSIMP_TOOLS=ON`` in CMake. If generating
configs for an IDE, make sure to build the assimp_cmd project.
(lists of file extensions are comma delimited, i.e. `3ds,lwo,x`)
-i,--include: List of file extensions to update dumps for. If omitted, -i,--include: List of file extensions to update dumps for. If omitted,
all file extensions are updated except those in `exclude`. all file extensions are updated except those in `exclude`.
@ -66,6 +78,8 @@ usage = """gen_db [-i=...] [-e=...] [-p] [-n]
Dont' change anything. Dont' change anything.
-n,--nozip: Don't pack to ZIP archive. Keep all dumps in individual files. -n,--nozip: Don't pack to ZIP archive. Keep all dumps in individual files.
(lists of file extensions are comma delimited, i.e. `3ds,lwo,x`)
""" """
# ------------------------------------------------------------------------------- # -------------------------------------------------------------------------------
@ -87,7 +101,7 @@ def process_dir(d, outfile, file_filter):
outf = os.path.join(os.getcwd(), settings.database_name, outf = os.path.join(os.getcwd(), settings.database_name,
utils.hashing(fullp, pp)) utils.hashing(fullp, pp))
cmd = [utils.assimp_bin_path,"dump",fullp,outf,"-b","-s","-l"] + pp.split() cmd = [ assimp_bin_path, "dump", fullp, outf, "-b", "-s", "-l" ] + pp.split()
outfile.write("assimp dump "+"-"*80+"\n") outfile.write("assimp dump "+"-"*80+"\n")
outfile.flush() outfile.flush()
if subprocess.call(cmd, stdout=outfile, stderr=outfile, shell=False): if subprocess.call(cmd, stdout=outfile, stderr=outfile, shell=False):
@ -158,7 +172,8 @@ def gen_db(ext_list,outfile):
# ------------------------------------------------------------------------------- # -------------------------------------------------------------------------------
if __name__ == "__main__": if __name__ == "__main__":
utils.find_assimp_or_die() assimp_bin_path = sys.argv[1] if len(sys.argv) > 1 else 'assimp'
def clean(f): def clean(f):
f = f.strip("* \'") f = f.strip("* \'")
return "."+f if f[:1] != '.' else f return "."+f if f[:1] != '.' else f
@ -184,7 +199,7 @@ if __name__ == "__main__":
outfile = open(os.path.join("..", "results", "gen_regression_db_output.txt"), "w") outfile = open(os.path.join("..", "results", "gen_regression_db_output.txt"), "w")
if ext_list is None: if ext_list is None:
(ext_list, err) = subprocess.Popen([utils.assimp_bin_path, "listext"], (ext_list, err) = subprocess.Popen([assimp_bin_path, "listext"],
stdout=subprocess.PIPE).communicate() stdout=subprocess.PIPE).communicate()
ext_list = str(ext_list).lower().split(";") ext_list = str(ext_list).lower().split(";")

View File

@ -41,8 +41,16 @@
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
""" """
Run the regression test suite using the settings from settings.py. Run the regression test suite using settings from settings.py.
The assimp_cmd (or assimp) binary to use is specified by the first
command line argument and defaults to ``assimp``.
To build, set ``ASSIMP_BUILD_ASSIMP_TOOLS=ON`` in CMake. If generating
configs for an IDE, make sure to build the assimp_cmd project.
On Windows, use ``py run.py <path to assimp>`` to make sure the command
line parameter is forwarded to the script.
""" """
import sys import sys
@ -124,8 +132,11 @@ class results:
def report_results(self): def report_results(self):
"""Write results to ../results/run_regression_suite_failures.txt""" """Write results to ../results/run_regression_suite_failures.txt"""
count_success = len(self.success)
count_fail = len(self.failures)
percent_good = float(count_success) / (count_success + count_fail)
print("\n" + ('='*60) + "\n" + "SUCCESS: {0}\nFAILURE: {1}\nPercentage good: {2}".format( print("\n" + ('='*60) + "\n" + "SUCCESS: {0}\nFAILURE: {1}\nPercentage good: {2}".format(
len(self.success), len(self.failures), len(self.success)/(len(self.success)+len(self.failures)) ) + count_success, count_fail, percent_good) +
"\n" + ('='*60) + "\n") "\n" + ('='*60) + "\n")
with open(os.path.join('..', 'results',outfilename_failur), "wt") as f: with open(os.path.join('..', 'results',outfilename_failur), "wt") as f:
@ -138,7 +149,7 @@ class results:
+ " for more details\n\n") + " for more details\n\n")
# ------------------------------------------------------------------------------- # -------------------------------------------------------------------------------
def mkoutputdir_andgetpath(fullpath, myhash, app): def prepare_output_dir(fullpath, myhash, app):
outfile = os.path.join(settings.results, "tmp", os.path.split(fullpath)[1] + "_" + myhash) outfile = os.path.join(settings.results, "tmp", os.path.split(fullpath)[1] + "_" + myhash)
try: try:
os.mkdir(outfile) os.mkdir(outfile)
@ -154,7 +165,7 @@ def process_dir(d, outfile_results, zipin, result):
shellparams = {'stdout':outfile_results, 'stderr':outfile_results, 'shell':False} shellparams = {'stdout':outfile_results, 'stderr':outfile_results, 'shell':False}
print("Processing directory " + d) print("Processing directory " + d)
for f in os.listdir(d): for f in sorted(os.listdir(d)):
fullpath = os.path.join(d, f) fullpath = os.path.join(d, f)
if os.path.isdir(fullpath) and not f == ".svn": if os.path.isdir(fullpath) and not f == ".svn":
process_dir(fullpath, outfile_results, zipin, result) process_dir(fullpath, outfile_results, zipin, result)
@ -167,13 +178,16 @@ def process_dir(d, outfile_results, zipin, result):
for pppreset in settings.pp_configs_to_test: for pppreset in settings.pp_configs_to_test:
filehash = utils.hashing(fullpath, pppreset) filehash = utils.hashing(fullpath, pppreset)
failure = False failure = False
try: try:
input_expected = zipin.open(filehash, "r").read() input_expected = zipin.open(filehash, "r").read()
# empty dump files indicate 'expected import failure' # empty dump files indicate 'expected import failure'
if not len(input_expected): if not len(input_expected):
failure = True failure = True
except KeyError: except KeyError:
#print("Didn't find "+fullpath+" (Hash is "+filehash+") in database") # TODO(acgessler): Keep track of this and report as error in the end.
print("Didn't find "+fullpath+" (Hash is "+filehash+") in database. Outdated "+\
"regression database? Use gen_db.zip to re-generate.")
continue continue
# Ignore extensions via settings.py configured list # Ignore extensions via settings.py configured list
@ -184,13 +198,18 @@ def process_dir(d, outfile_results, zipin, result):
print("-"*60 + "\n " + os.path.realpath(fullpath) + " pp: " + pppreset) print("-"*60 + "\n " + os.path.realpath(fullpath) + " pp: " + pppreset)
outfile_actual = mkoutputdir_andgetpath(fullpath, filehash, "ACTUAL") outfile_actual = prepare_output_dir(fullpath, filehash, "ACTUAL")
outfile_expect = mkoutputdir_andgetpath(fullpath, filehash, "EXPECT") outfile_expect = prepare_output_dir(fullpath, filehash, "EXPECT")
outfile_results.write("assimp dump "+"-"*80+"\n") outfile_results.write("assimp dump "+"-"*80+"\n")
outfile_results.flush() outfile_results.flush()
command = [utils.assimp_bin_path,"dump",fullpath,outfile_actual,"-b","-s","-l"]+pppreset.split() command = [assimp_bin_path,
"dump",
fullpath, outfile_actual, "-b", "-s", "-l" ] +\
pppreset.split()
r = subprocess.call(command, **shellparams) r = subprocess.call(command, **shellparams)
print(r)
if r and not failure: if r and not failure:
result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r) result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r)
@ -216,7 +235,7 @@ def process_dir(d, outfile_results, zipin, result):
outfile_results.write("assimp cmpdump "+"-"*80+"\n") outfile_results.write("assimp cmpdump "+"-"*80+"\n")
outfile_results.flush() outfile_results.flush()
command = [utils.assimp_bin_path,'cmpdump',outfile_actual,outfile_expect] command = [ assimp_bin_path, 'cmpdump', outfile_actual, outfile_expect ]
if subprocess.call(command, **shellparams) != 0: if subprocess.call(command, **shellparams) != 0:
result.fail(fullpath, outfile_expect, pppreset, DATABASE_VALUE_MISMATCH) result.fail(fullpath, outfile_expect, pppreset, DATABASE_VALUE_MISMATCH)
continue continue
@ -235,7 +254,6 @@ def del_folder_with_contents(folder):
# ------------------------------------------------------------------------------- # -------------------------------------------------------------------------------
def run_test(): def run_test():
utils.find_assimp_or_die()
tmp_target_path = os.path.join(settings.results, "tmp") tmp_target_path = os.path.join(settings.results, "tmp")
try: try:
os.mkdir(tmp_target_path) os.mkdir(tmp_target_path)
@ -261,6 +279,8 @@ def run_test():
# ------------------------------------------------------------------------------- # -------------------------------------------------------------------------------
if __name__ == "__main__": if __name__ == "__main__":
assimp_bin_path = sys.argv[1] if len(sys.argv) > 1 else 'assimp'
print('Using assimp binary: ' + assimp_bin_path)
run_test() run_test()
# vim: ai ts=4 sts=4 et sw=4 # vim: ai ts=4 sts=4 et sw=4

View File

@ -40,7 +40,7 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
"""Shared stuff for the gen_db and run scripts """ """Shared stuff for the gen_db and run scripts"""
# ------------------------------------------------------------------------------- # -------------------------------------------------------------------------------
def hashing(file,pp): def hashing(file,pp):
@ -51,75 +51,14 @@ def hashing(file,pp):
and platforms, so we implement the hashing manually. and platforms, so we implement the hashing manually.
""" """
def myhash(instring): file = file.replace('\\','/')+":"+pp
# sdbm hash # SDBM hash
res = 0 res = 0
for t in instring: for t in file:
res = (ord(t) + (res<<6) + (res<<16) - res) % 2**32 res = (ord(t) + (res<<6) + (res<<16) - res) % 2**32
return res
return hex(myhash(file.replace('\\','/')+":"+pp)) # Python 2.7 normalization: strip 'L' suffix.
return hex(res).rstrip('L')
assimp_bin_path = None
# -------------------------------------------------------------------------------
def find_assimp_or_die():
"""Find assimp_cmd's binary for the current platform.
The path to the binary is stored in assimp_bin_path, the process
is aborted if it can't be found.
"""
import os
import platform
import sys
def locate_file(f_list):
for f in f_list:
try:
fl = open(f,"rb")
except IOError:
continue
fl.close()
return f
return None
global assimp_bin_path
if os.name == "nt":
search_x86 = [
os.path.join("assimp.exe"),
os.path.join("..","..","bin","assimpcmd_release-dll_Win32","assimp.exe"),
os.path.join("..","..","bin","x86","assimp"),
os.path.join("..","..","bin","Release","assimp.exe")
]
if platform.machine() == "x86":
search = search_x86
else: # amd64, hopefully
search = [
os.path.join("..","..","bin","assimpcmd_release-dll_x64","assimp.exe"),
os.path.join("..","..","bin","x64","assimp")
]
# x64 platform does not guarantee a x64 build. Also look for x86 as last paths.
search += search_x86
assimp_bin_path = locate_file(search)
if assimp_bin_path is None:
print("Can't locate assimp_cmd binary")
print("Looked in", search)
sys.exit(-5)
print("Located assimp/assimp_cmd binary from", assimp_bin_path)
elif os.name == "posix":
#search = [os.path.join("..","..","bin","gcc","assimp"),
# os.path.join("/usr","local","bin",'assimp')]
assimp_bin_path = "assimp"
print("Taking system-wide assimp binary")
else:
print("Unsupported operating system")
sys.exit(-5)
if __name__ == '__main__':
find_assimp_or_die()
# vim: ai ts=4 sts=4 et sw=4 # vim: ai ts=4 sts=4 et sw=4