321 lines
12 KiB
Python
Executable File
321 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- Coding: UTF-8 -*-
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Open Asset Import Library (ASSIMP)
|
|
# ---------------------------------------------------------------------------
|
|
#
|
|
# Copyright (c) 2006-2020, ASSIMP Development Team
|
|
#
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use of this software in source and binary forms,
|
|
# with or without modification, are permitted provided that the following
|
|
# conditions are met:
|
|
#
|
|
# * Redistributions of source code must retain the above
|
|
# copyright notice, this list of conditions and the
|
|
# following disclaimer.
|
|
#
|
|
# * Redistributions in binary form must reproduce the above
|
|
# copyright notice, this list of conditions and the
|
|
# following disclaimer in the documentation and/or other
|
|
# materials provided with the distribution.
|
|
#
|
|
# * Neither the name of the ASSIMP team, nor the names of its
|
|
# contributors may be used to endorse or promote products
|
|
# derived from this software without specific prior
|
|
# written permission of the ASSIMP Development Team.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
"""
|
|
Run the regression test suite using settings from settings.py.
|
|
|
|
The assimp_cmd (or assimp) binary to use is specified by the first
|
|
command line argument and defaults to ``assimp``.
|
|
|
|
To build, set ``ASSIMP_BUILD_ASSIMP_TOOLS=ON`` in CMake. If generating
|
|
configs for an IDE, make sure to build the assimp_cmd project.
|
|
|
|
On Windows, use ``py run.py <path to assimp>`` to make sure the command
|
|
line parameter is forwarded to the script.
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
import subprocess
|
|
import zipfile
|
|
import collections
|
|
import multiprocessing
|
|
|
|
import settings
|
|
import utils
|
|
|
|
# -------------------------------------------------------------------------------
|
|
EXPECTED_FAILURE_NOT_MET, DATABASE_LENGTH_MISMATCH, \
|
|
DATABASE_VALUE_MISMATCH, IMPORT_FAILURE, \
|
|
FILE_NOT_READABLE, COMPARE_SUCCESS, EXPECTED_FAILURE = range(7)
|
|
|
|
messages = collections.defaultdict(lambda: "<unknown", {
|
|
EXPECTED_FAILURE_NOT_MET:
|
|
"""Unexpected success during import\n\
|
|
\tReturn code was 0""",
|
|
|
|
DATABASE_LENGTH_MISMATCH:
|
|
"""Database mismatch: lengths don't match\n\
|
|
\tExpected: {0} Actual: {1}""",
|
|
|
|
DATABASE_VALUE_MISMATCH:
|
|
"""Database mismatch: """,
|
|
|
|
IMPORT_FAILURE:
|
|
"""Unexpected failure during import\n\
|
|
\tReturn code was {0}""",
|
|
|
|
FILE_NOT_READABLE:
|
|
"""Unexpected failure reading file""",
|
|
|
|
COMPARE_SUCCESS:
|
|
"""Results match archived reference dump in database\n\
|
|
\tNumber of bytes compared: {0}""",
|
|
|
|
EXPECTED_FAILURE:
|
|
"""Expected failure was met.""",
|
|
})
|
|
|
|
outfilename_output = "run_regression_suite_output.txt"
|
|
outfilename_failur = "run_regression_suite_failures.csv"
|
|
Environment = {}
|
|
|
|
# -------------------------------------------------------------------------------
|
|
class results:
|
|
|
|
""" Handle formatting of results"""
|
|
|
|
def __init__(self, zipin):
|
|
"""Init, given a ZIPed database """
|
|
self.failures = []
|
|
self.success = []
|
|
self.zipin = zipin
|
|
|
|
|
|
def fail(self, failfile, filename_expect, pp, msg, *args):
|
|
"""
|
|
Report failure of a sub-test
|
|
|
|
File f failed a test for pp config pp, failure notice is msg,
|
|
*args is format()ting args for msg
|
|
|
|
"""
|
|
print("[FAILURE] " + messages[msg].format(*args))
|
|
self.failures.append((failfile, filename_expect, pp))
|
|
|
|
|
|
def ok(self, f, pp, msg, *args):
|
|
"""
|
|
Report success of a sub-test
|
|
|
|
File f passed the test, msg is a happy success note,
|
|
*args is format()ing args for msg.
|
|
|
|
"""
|
|
print("[SUCCESS] " + messages[msg].format(*args))
|
|
self.success.append(f)
|
|
|
|
|
|
def report_results(self):
|
|
"""Write results to ../results/run_regression_suite_failures.txt"""
|
|
|
|
count_success = len(self.success)
|
|
count_fail = len(self.failures)
|
|
percent_good = float(count_success) / (count_success + count_fail)
|
|
print("\n" + ('='*60) + "\n" + "SUCCESS: {0}\nFAILURE: {1}\nPercentage good: {2}".format(
|
|
count_success, count_fail, percent_good) +
|
|
"\n" + ('='*60) + "\n")
|
|
|
|
with open(os.path.join('..', 'results',outfilename_failur), "wt") as f:
|
|
f.write("ORIGINAL FILE;EXPECTED DUMP\n")
|
|
f.writelines(map(
|
|
lambda x: x[0] + ' ' + x[2] + ";" + x[1] + "\n", self.failures))
|
|
|
|
if self.failures:
|
|
print("\nSee " + settings.results + "\\" + outfilename_failur
|
|
+ " for more details\n\n")
|
|
|
|
def hasFailures( self ):
|
|
""" Return True, if any failures there. """
|
|
return 0 != len( self.failures )
|
|
|
|
# -------------------------------------------------------------------------------
|
|
def setEnvVar( var, value ):
|
|
print ( "set var " + var +" to" + value)
|
|
Environment[ var ] = value
|
|
|
|
# -------------------------------------------------------------------------------
|
|
def getEnvVar( var ):
|
|
if var in Environment:
|
|
return Environment[ var ]
|
|
else:
|
|
print ( "Error: cannot find " + var )
|
|
return ""
|
|
|
|
# -------------------------------------------------------------------------------
|
|
def prepare_output_dir(fullpath, myhash, app):
|
|
outfile = os.path.join(settings.results, "tmp", os.path.split(fullpath)[1] + "_" + myhash)
|
|
try:
|
|
os.mkdir(outfile)
|
|
except OSError:
|
|
pass
|
|
|
|
outfile = os.path.join(outfile, app)
|
|
return outfile
|
|
|
|
# -------------------------------------------------------------------------------
|
|
def process_dir(d, outfile_results, zipin, result ):
|
|
shellparams = {'stdout':outfile_results, 'stderr':outfile_results, 'shell':False}
|
|
|
|
print("Processing directory " + d)
|
|
all = ""
|
|
for f in sorted(os.listdir(d)):
|
|
fullpath = os.path.join(d, f)
|
|
if os.path.isdir(fullpath) and not f[:1] == '.':
|
|
process_dir(fullpath, outfile_results, zipin, result)
|
|
continue
|
|
|
|
if f in settings.files_to_ignore or os.path.splitext(f)[1] in settings.exclude_extensions:
|
|
print("Ignoring " + f)
|
|
return
|
|
|
|
for pppreset in settings.pp_configs_to_test:
|
|
filehash = utils.hashing(fullpath, pppreset)
|
|
failure = False
|
|
|
|
try:
|
|
input_expected = zipin.open(filehash, "r").read()
|
|
# empty dump files indicate 'expected import failure'
|
|
if not len(input_expected):
|
|
failure = True
|
|
except KeyError:
|
|
# TODO(acgessler): Keep track of this and report as error in the end.
|
|
print("Didn't find "+fullpath+" (Hash is "+filehash+") in database. Outdated "+\
|
|
"regression database? Use gen_db.zip to re-generate.")
|
|
continue
|
|
|
|
print("-"*60 + "\n " + os.path.realpath(fullpath) + " pp: " + pppreset)
|
|
|
|
outfile_actual = prepare_output_dir(fullpath, filehash, "ACTUAL")
|
|
outfile_expect = prepare_output_dir(fullpath, filehash, "EXPECT")
|
|
outfile_results.write("assimp dump "+"-"*80+"\n")
|
|
outfile_results.flush()
|
|
assimp_bin_path = getEnvVar("assimp_path")
|
|
command = [assimp_bin_path,
|
|
"dump",
|
|
fullpath, outfile_actual, "-b", "-s", "-l" ] +\
|
|
pppreset.split()
|
|
print( "command = " + str( command ) )
|
|
r = subprocess.call(command, **shellparams)
|
|
outfile_results.flush()
|
|
|
|
if r and not failure:
|
|
result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r)
|
|
outfile_results.write("Failed to import\n")
|
|
continue
|
|
elif failure and not r:
|
|
result.fail(fullpath, outfile_expect, pppreset, EXPECTED_FAILURE_NOT_MET)
|
|
outfile_results.write("Expected import to fail\n")
|
|
continue
|
|
elif failure and r:
|
|
result.ok(fullpath, pppreset, EXPECTED_FAILURE)
|
|
outfile_results.write("Failed as expected, skipping.\n")
|
|
continue
|
|
|
|
with open(outfile_expect, "wb") as s:
|
|
s.write(input_expected)
|
|
|
|
try:
|
|
with open(outfile_actual, "rb") as s:
|
|
input_actual = s.read()
|
|
except IOError:
|
|
continue
|
|
|
|
outfile_results.write("Expected data length: {0}\n".format(len(input_expected)))
|
|
outfile_results.write("Actual data length: {0}\n".format(len(input_actual)))
|
|
failed = False
|
|
if len(input_expected) != len(input_actual):
|
|
result.fail(fullpath, outfile_expect, pppreset, DATABASE_LENGTH_MISMATCH,
|
|
len(input_expected), len(input_actual))
|
|
# Still compare the dumps to see what the difference is
|
|
failed = True
|
|
|
|
outfile_results.write("assimp cmpdump "+"-"*80+"\n")
|
|
outfile_results.flush()
|
|
command = [ assimp_bin_path, 'cmpdump', outfile_actual, outfile_expect ]
|
|
if subprocess.call(command, **shellparams) != 0:
|
|
if not failed:
|
|
result.fail(fullpath, outfile_expect, pppreset, DATABASE_VALUE_MISMATCH)
|
|
continue
|
|
|
|
result.ok(fullpath, pppreset, COMPARE_SUCCESS, len(input_expected))
|
|
|
|
# -------------------------------------------------------------------------------
|
|
def del_folder_with_contents(folder):
|
|
for root, dirs, files in os.walk(folder, topdown=False):
|
|
for name in files:
|
|
os.remove(os.path.join(root, name))
|
|
for name in dirs:
|
|
os.rmdir(os.path.join(root, name))
|
|
|
|
|
|
# -------------------------------------------------------------------------------
|
|
def run_test():
|
|
tmp_target_path = os.path.join(settings.results, "tmp")
|
|
try:
|
|
print( "try to make " + tmp_target_path )
|
|
os.mkdir(tmp_target_path)
|
|
except OSError as oerr:
|
|
# clear contents if tmp folder exists already
|
|
del_folder_with_contents(tmp_target_path)
|
|
|
|
try:
|
|
zipin = zipfile.ZipFile(settings.database_name + ".zip",
|
|
"r", zipfile.ZIP_STORED)
|
|
except IOError:
|
|
print("Regression database ", settings.database_name,
|
|
".zip was not found")
|
|
return
|
|
|
|
res = results(zipin)
|
|
with open(os.path.join(settings.results, outfilename_output), "wt") as outfile:
|
|
for tp in settings.model_directories:
|
|
process_dir(tp, outfile, zipin, res)
|
|
|
|
res.report_results()
|
|
if res.hasFailures():
|
|
return 1
|
|
|
|
return 0
|
|
|
|
# -------------------------------------------------------------------------------
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) > 1:
|
|
assimp_bin_path = sys.argv[1]
|
|
else:
|
|
assimp_bin_path = 'assimp'
|
|
setEnvVar("assimp_path", assimp_bin_path)
|
|
print('Using assimp binary: ' + assimp_bin_path)
|
|
sys.exit( run_test() )
|
|
|
|
# vim: ai ts=4 sts=4 et sw=4
|