From 8b27163c20db3dbf1dc03933396407bdf5b121b4 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 29 Jun 2015 22:35:31 -0600 Subject: [PATCH 1/7] Fixed: Ensure failing tests fail travis --- .travis.sh | 16 ++++++++++++++++ .travis.yml | 16 +--------------- 2 files changed, 17 insertions(+), 15 deletions(-) create mode 100755 .travis.sh diff --git a/.travis.sh b/.travis.sh new file mode 100755 index 000000000..9f1ea37e9 --- /dev/null +++ b/.travis.sh @@ -0,0 +1,16 @@ +function generate() +{ + cmake -G "Unix Makefiles" -DASSIMP_ENABLE_BOOST_WORKAROUND=YES -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD +} + +if [ $ANDROID ]; then + ant -v -Dmy.dir=${TRAVIS_BUILD_DIR} -f ${TRAVIS_BUILD_DIR}/port/jassimp/build.xml ndk-jni +else + generate \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && (cd test/unit; ../../bin/unit) \ + && (cd test/regression; chmod 755 run.py; ./run.py) \ + && (cd test/regression; chmod 755 result_checker.py; ./result_checker.py) +fi diff --git a/.travis.yml b/.travis.yml index ccc6ff815..66b53f686 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,18 +28,4 @@ install: - if [ $ANDROID ]; then wget -c http://dl.google.com/android/ndk/android-ndk-${PV}-${PLATF}.tar.bz2 && tar xf android-ndk-${PV}-${PLATF}.tar.bz2 ; fi script: - - if [ $ANDROID ]; then - ant -v -Dmy.dir=${TRAVIS_BUILD_DIR} -f ${TRAVIS_BUILD_DIR}/port/jassimp/build.xml ndk-jni ; - else - cmake -G "Unix Makefiles" -DASSIMP_ENABLE_BOOST_WORKAROUND=YES -DASSIMP_NO_EXPORT=$TRAVIS_NO_EXPORT -DBUILD_SHARED_LIBS=$SHARED_BUILD ; - make ; - sudo make install ; - sudo ldconfig ; - cd test/unit ; - ../../bin/unit ; - cd ../regression ; - chmod 755 run.py ; - ./run.py ; - chmod 755 result_checker.py ; - ./result_checker.py; - fi + - . ./.travis.sh From 08169a693430011408ecdf5c00358e841ffb3a20 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 30 Jun 2015 07:26:58 -0600 Subject: [PATCH 2/7] Fixed: Use CMake to identify directory containing test models --- test/CMakeLists.txt | 2 ++ test/unit/utColladaExportCamera.cpp | 2 +- test/unit/utColladaExportLight.cpp | 2 +- test/unit/utExport.cpp | 2 +- test/unit/utImporter.cpp | 16 ++++++++-------- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4bd3ff9c6..a93bcf91f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -54,6 +54,8 @@ add_executable( unit ${TEST_SRCS} ) +add_definitions(-DASSIMP_TEST_MODELS_DIR="${CMAKE_CURRENT_LIST_DIR}/models") + SET_PROPERTY( TARGET assimp PROPERTY DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) add_dependencies( unit gtest ) diff --git a/test/unit/utColladaExportCamera.cpp b/test/unit/utColladaExportCamera.cpp index 0ebdb0163..8735c626e 100644 --- a/test/unit/utColladaExportCamera.cpp +++ b/test/unit/utColladaExportCamera.cpp @@ -43,7 +43,7 @@ TEST_F(ColladaExportCamera, testExportCamera) { const char* file = "cameraExp.dae"; - const aiScene* pTest = im->ReadFile("../test/models/Collada/cameras.dae",0); + const aiScene* pTest = im->ReadFile(ASSIMP_TEST_MODELS_DIR "/Collada/cameras.dae",0); ASSERT_TRUE(pTest!=NULL); ASSERT_TRUE(pTest->HasCameras()); diff --git a/test/unit/utColladaExportLight.cpp b/test/unit/utColladaExportLight.cpp index bac35954c..ee17b77e0 100644 --- a/test/unit/utColladaExportLight.cpp +++ b/test/unit/utColladaExportLight.cpp @@ -43,7 +43,7 @@ TEST_F(ColladaExportLight, testExportLight) { const char* file = "cameraExp.dae"; - const aiScene* pTest = im->ReadFile("../test/models/Collada/lights.dae",0); + const aiScene* pTest = im->ReadFile(ASSIMP_TEST_MODELS_DIR "/Collada/lights.dae",0); ASSERT_TRUE(pTest!=NULL); ASSERT_TRUE(pTest->HasLights()); diff --git a/test/unit/utExport.cpp b/test/unit/utExport.cpp index ac2a081fc..f34429338 100644 --- a/test/unit/utExport.cpp +++ b/test/unit/utExport.cpp @@ -14,7 +14,7 @@ public: ex = new Assimp::Exporter(); im = new Assimp::Importer(); - pTest = im->ReadFile("../../test/models/X/test.x",0); + pTest = im->ReadFile(ASSIMP_TEST_MODELS_DIR "/X/test.x",0); } virtual void TearDown() diff --git a/test/unit/utImporter.cpp b/test/unit/utImporter.cpp index d6090fafe..0b300b9b4 100644 --- a/test/unit/utImporter.cpp +++ b/test/unit/utImporter.cpp @@ -219,13 +219,13 @@ TEST_F(ImporterTest, testMultipleReads) aiProcess_OptimizeMeshes | aiProcess_OptimizeGraph; - EXPECT_TRUE(pImp->ReadFile("../test/models/X/test.x",flags)); - //EXPECT_TRUE(pImp->ReadFile("../../test/models/X/dwarf.x",flags)); # is in nonbsd - EXPECT_TRUE(pImp->ReadFile("../test/models/X/Testwuson.X",flags)); - EXPECT_TRUE(pImp->ReadFile("../test/models/X/anim_test.x",flags)); - //EXPECT_TRUE(pImp->ReadFile("../../test/models/X/dwarf.x",flags)); # is in nonbsd + EXPECT_TRUE(pImp->ReadFile(ASSIMP_TEST_MODELS_DIR "/X/test.x",flags)); + //EXPECT_TRUE(pImp->ReadFile(ASSIMP_TEST_MODELS_DIR "/X/dwarf.x",flags)); # is in nonbsd + EXPECT_TRUE(pImp->ReadFile(ASSIMP_TEST_MODELS_DIR "/X/Testwuson.X",flags)); + EXPECT_TRUE(pImp->ReadFile(ASSIMP_TEST_MODELS_DIR "/X/anim_test.x",flags)); + //EXPECT_TRUE(pImp->ReadFile(ASSIMP_TEST_MODELS_DIR "/X/dwarf.x",flags)); # is in nonbsd - EXPECT_TRUE(pImp->ReadFile("../test/models/X/anim_test.x",flags)); - EXPECT_TRUE(pImp->ReadFile("../test/models/X/BCN_Epileptic.X",flags)); - //EXPECT_TRUE(pImp->ReadFile("../../test/models/X/dwarf.x",flags)); # is in nonbsd + EXPECT_TRUE(pImp->ReadFile(ASSIMP_TEST_MODELS_DIR "/X/anim_test.x",flags)); + EXPECT_TRUE(pImp->ReadFile(ASSIMP_TEST_MODELS_DIR "/X/BCN_Epileptic.X",flags)); + //EXPECT_TRUE(pImp->ReadFile(ASSIMP_TEST_MODELS_DIR "/X/dwarf.x",flags)); # is in nonbsd } From 267ff0b93cad258908ae5928517f9577ddfaab66 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 30 Jun 2015 19:10:37 -0600 Subject: [PATCH 3/7] Refactor: Rename README to README.txt --- test/regression/{README => README.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/regression/{README => README.txt} (100%) diff --git a/test/regression/README b/test/regression/README.txt similarity index 100% rename from test/regression/README rename to test/regression/README.txt From ac23795778cce46ade32390ce0a52b1de2ebd505 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 30 Jun 2015 19:10:54 -0600 Subject: [PATCH 4/7] Refactor: Use LF line endings --- test/regression/README.txt | 190 ++++++++++++++++++------------------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/test/regression/README.txt b/test/regression/README.txt index 907d1ecf7..4a9778e21 100644 --- a/test/regression/README.txt +++ b/test/regression/README.txt @@ -1,95 +1,95 @@ -Assimp Regression Test Suite -============================ - -1) How does it work? ---------------------------------------------------------------------------------- -run.py checks all model in the /test/models* folders and compares the result -against a regression database provided with assimp (db.zip). A few failures -are totally fine (see sections 7+). You need to worry if a huge -majority of all files in a particular format (or post-processing configuration) -fails as this might be a sign of a recent regression in assimp's codebase or -gross incompatibility with your system or compiler. - -2) What do I need? ---------------------------------------------------------------------------------- - - You need Python installed (2.7+, 3.x). On Windows, run the scripts using "py". - - You need to build the assimp command line tool (ASSIMP_BUILD_ASSIMP_TOOLS - CMake build flag). Both run.py and gen_db.py take the full path to the binary - as first command line parameter. - -3) How to add more test files? ---------------------------------------------------------------------------------- -Use the following procedure: - - Verify the correctness of your assimp build - run the regression suite. - DO NOT continue if more tests fail than usual. - - Add your additional test files to /test/models/, where - is the file type (typically the file extension). - - If you test file does not meet the BSD license requirements, add it to - /test/models-nonbsd/ so people know to be careful with it. - - Rebuild the regression database: - "gen_db.py -ixyz" where .xyz is the file extension of the new file. - - Run the regression suite again. There should be no new failures and the new - file should not be among the failures. - - Include the db.zip file with your Pull Request. Travis CI enforces a passing - regression suite (with offenders whitelisted as a last resort). - -4) I made a change/fix/patch to a loader, how to update the database? ---------------------------------------------------------------------------------- - - Rebuild the regression database using "gen_db.py -ixyz" - where .xyz is the file extension for which the loader was patched. - - Run the regression suite again. There should be no new failures and the new - file should not be among the failures. - - Include the db.zip file with your Pull Request. Travis CI enforces a passing - regression suite (with offenders whitelisted as a last resort). - -5) How to add my whole model repository to the database? ---------------------------------------------------------------------------------- - -Edit the reg_settings.py file and add the path to your repository to -<>. Then, rebuild the database. - -6) So what is actually tested? ---------------------------------------------------------------------------------- -The regression database includes mini dumps of the aiScene data structure, i.e. -the scene hierarchy plus the sizes of all data arrays MUST match. Floating-point -data buffers, such as vertex positions are handled less strictly: min, max and -average values are stored with low precision. This takes hardware- or -compiler-specific differences in floating-point computations into account. -Generally, almost all significant regressions will be detected while the -number of false positives is relatively low. - -7) The test suite fails, what do do? ---------------------------------------------------------------------------------- -Get back to /test/results and look at regression_suite_failures.txt. -It contains a list of all files which failed the test. Failing dumps are copied to -root>/test/results/tmp. Both an EXPECTED and an ACTUAL file is produced per test. -The output of "assimp cmpdump" is written to regressions_suite_output.txt. Grep -for the file name in question and locate the log for the failed comparison. It -contains a full trace of which scene elements have been compared before, which -makes it reasonably easy to locate the offending field. - -8) fp:fast vs fp:precise fails the test suite (same for gcc equivalents) ---------------------------------------------------------------------------------- -As mentioned above, floating-point inaccuracies between differently optimized -builds are not considered regressions and all float comparisons done by the test -suite involve an epsilon to accomodate. However compiler settings that allow -compilers to perform non-IEEE754 compliant optimizations can cause arbitrary -failures in the test suite. Even if the compiler is configured to be IEE754 -comformant, there is lots of code in assimp that leaves the compiler a choice -and different compilers make different choices (for example the precision of -float intermediaries is implementation-specified). - - - - - - - - - - - - - - - +Assimp Regression Test Suite +============================ + +1) How does it work? +--------------------------------------------------------------------------------- +run.py checks all model in the /test/models* folders and compares the result +against a regression database provided with assimp (db.zip). A few failures +are totally fine (see sections 7+). You need to worry if a huge +majority of all files in a particular format (or post-processing configuration) +fails as this might be a sign of a recent regression in assimp's codebase or +gross incompatibility with your system or compiler. + +2) What do I need? +--------------------------------------------------------------------------------- + - You need Python installed (2.7+, 3.x). On Windows, run the scripts using "py". + - You need to build the assimp command line tool (ASSIMP_BUILD_ASSIMP_TOOLS + CMake build flag). Both run.py and gen_db.py take the full path to the binary + as first command line parameter. + +3) How to add more test files? +--------------------------------------------------------------------------------- +Use the following procedure: + - Verify the correctness of your assimp build - run the regression suite. + DO NOT continue if more tests fail than usual. + - Add your additional test files to /test/models/, where + is the file type (typically the file extension). + - If you test file does not meet the BSD license requirements, add it to + /test/models-nonbsd/ so people know to be careful with it. + - Rebuild the regression database: + "gen_db.py -ixyz" where .xyz is the file extension of the new file. + - Run the regression suite again. There should be no new failures and the new + file should not be among the failures. + - Include the db.zip file with your Pull Request. Travis CI enforces a passing + regression suite (with offenders whitelisted as a last resort). + +4) I made a change/fix/patch to a loader, how to update the database? +--------------------------------------------------------------------------------- + - Rebuild the regression database using "gen_db.py -ixyz" + where .xyz is the file extension for which the loader was patched. + - Run the regression suite again. There should be no new failures and the new + file should not be among the failures. + - Include the db.zip file with your Pull Request. Travis CI enforces a passing + regression suite (with offenders whitelisted as a last resort). + +5) How to add my whole model repository to the database? +--------------------------------------------------------------------------------- + +Edit the reg_settings.py file and add the path to your repository to +<>. Then, rebuild the database. + +6) So what is actually tested? +--------------------------------------------------------------------------------- +The regression database includes mini dumps of the aiScene data structure, i.e. +the scene hierarchy plus the sizes of all data arrays MUST match. Floating-point +data buffers, such as vertex positions are handled less strictly: min, max and +average values are stored with low precision. This takes hardware- or +compiler-specific differences in floating-point computations into account. +Generally, almost all significant regressions will be detected while the +number of false positives is relatively low. + +7) The test suite fails, what do do? +--------------------------------------------------------------------------------- +Get back to /test/results and look at regression_suite_failures.txt. +It contains a list of all files which failed the test. Failing dumps are copied to +root>/test/results/tmp. Both an EXPECTED and an ACTUAL file is produced per test. +The output of "assimp cmpdump" is written to regressions_suite_output.txt. Grep +for the file name in question and locate the log for the failed comparison. It +contains a full trace of which scene elements have been compared before, which +makes it reasonably easy to locate the offending field. + +8) fp:fast vs fp:precise fails the test suite (same for gcc equivalents) +--------------------------------------------------------------------------------- +As mentioned above, floating-point inaccuracies between differently optimized +builds are not considered regressions and all float comparisons done by the test +suite involve an epsilon to accomodate. However compiler settings that allow +compilers to perform non-IEEE754 compliant optimizations can cause arbitrary +failures in the test suite. Even if the compiler is configured to be IEE754 +comformant, there is lots of code in assimp that leaves the compiler a choice +and different compilers make different choices (for example the precision of +float intermediaries is implementation-specified). + + + + + + + + + + + + + + + From 1f28662687bed67a92991de0b4a71c2adc7b9d40 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 30 Jun 2015 19:11:53 -0600 Subject: [PATCH 5/7] Implemented: Add editor config rules for *.txt files --- .editorconfig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.editorconfig b/.editorconfig index 0b8dc48bb..d95a7e3dc 100644 --- a/.editorconfig +++ b/.editorconfig @@ -15,3 +15,8 @@ insert_final_newline = true trim_trailing_whitespace = true indent_size = 4 indent_style = space + +[*.txt] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true From 90dd01c8486e603e4f79e4e2736e215ac3752857 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 30 Jun 2015 19:12:32 -0600 Subject: [PATCH 6/7] Fixed: Correct path location --- test/regression/README.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/regression/README.txt b/test/regression/README.txt index 4a9778e21..b0e79e1f8 100644 --- a/test/regression/README.txt +++ b/test/regression/README.txt @@ -62,7 +62,7 @@ number of false positives is relatively low. --------------------------------------------------------------------------------- Get back to /test/results and look at regression_suite_failures.txt. It contains a list of all files which failed the test. Failing dumps are copied to -root>/test/results/tmp. Both an EXPECTED and an ACTUAL file is produced per test. +/test/results/tmp. Both an EXPECTED and an ACTUAL file is produced per test. The output of "assimp cmpdump" is written to regressions_suite_output.txt. Grep for the file name in question and locate the log for the failed comparison. It contains a full trace of which scene elements have been compared before, which From e958d3d42d2b572021e9c29d87dc3461126112b4 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 30 Jun 2015 19:49:02 -0600 Subject: [PATCH 7/7] Fixed: Use status from result_checker.py as status of regression test --- .travis.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.sh b/.travis.sh index 9f1ea37e9..aa53a64e7 100755 --- a/.travis.sh +++ b/.travis.sh @@ -11,6 +11,6 @@ else && sudo make install \ && sudo ldconfig \ && (cd test/unit; ../../bin/unit) \ - && (cd test/regression; chmod 755 run.py; ./run.py) \ - && (cd test/regression; chmod 755 result_checker.py; ./result_checker.py) + && (cd test/regression; chmod 755 run.py; ./run.py; \ + chmod 755 result_checker.py; ./result_checker.py) fi