From 27bcddfb1aa2c08f1976d833440e66e5ad3ae71f Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 8 Feb 2022 20:43:47 +0100 Subject: [PATCH 1/7] Migrate more importers to compression class --- code/AssetLib/Blender/BlenderLoader.cpp | 50 +++++-------------------- code/AssetLib/FBX/FBXParser.cpp | 20 ++++++---- code/AssetLib/X/XFileParser.cpp | 25 ++++++++----- code/AssetLib/XGL/XGLLoader.cpp | 4 +- code/Common/Compression.cpp | 14 ++++--- code/Common/Compression.h | 11 +++++- 6 files changed, 58 insertions(+), 66 deletions(-) diff --git a/code/AssetLib/Blender/BlenderLoader.cpp b/code/AssetLib/Blender/BlenderLoader.cpp index 3d9e8017b..70fccae04 100644 --- a/code/AssetLib/Blender/BlenderLoader.cpp +++ b/code/AssetLib/Blender/BlenderLoader.cpp @@ -66,11 +66,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // zlib is needed for compressed blend files #ifndef ASSIMP_BUILD_NO_COMPRESSED_BLEND -# ifdef ASSIMP_BUILD_NO_OWN_ZLIB +#include "Common/Compression.h" +/* #ifdef ASSIMP_BUILD_NO_OWN_ZLIB # include # else # include "../contrib/zlib/zlib.h" -# endif +# endif*/ #endif namespace Assimp { @@ -141,7 +142,7 @@ void BlenderImporter::SetupProperties(const Importer * /*pImp*/) { void BlenderImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSystem *pIOHandler) { #ifndef ASSIMP_BUILD_NO_COMPRESSED_BLEND - std::vector uncompressed; + std::vector uncompressed; #endif FileDatabase file; @@ -159,7 +160,6 @@ void BlenderImporter::InternReadFile(const std::string &pFile, #ifdef ASSIMP_BUILD_NO_COMPRESSED_BLEND ThrowException("BLENDER magic bytes are missing, is this file compressed (Assimp was built without decompression support)?"); #else - if (magic[0] != 0x1f || static_cast(magic[1]) != 0x8b) { ThrowException("BLENDER magic bytes are missing, couldn't find GZIP header either"); } @@ -173,42 +173,12 @@ void BlenderImporter::InternReadFile(const std::string &pFile, stream->Seek(0L, aiOrigin_SET); std::shared_ptr reader = std::shared_ptr(new StreamReaderLE(stream)); - // build a zlib stream - z_stream zstream; - zstream.opaque = Z_NULL; - zstream.zalloc = Z_NULL; - zstream.zfree = Z_NULL; - zstream.data_type = Z_BINARY; - - // http://hewgill.com/journal/entries/349-how-to-decompress-gzip-stream-with-zlib - inflateInit2(&zstream, 16 + MAX_WBITS); - - zstream.next_in = reinterpret_cast(reader->GetPtr()); - zstream.avail_in = (uInt)reader->GetRemainingSize(); - - size_t total = 0l; - - // TODO: be smarter about this, decompress directly into heap buffer - // and decompress the data .... do 1k chunks in the hope that we won't kill the stack -#define MYBLOCK 1024 - Bytef block[MYBLOCK]; - int ret; - do { - zstream.avail_out = MYBLOCK; - zstream.next_out = block; - ret = inflate(&zstream, Z_NO_FLUSH); - - if (ret != Z_STREAM_END && ret != Z_OK) { - ThrowException("Failure decompressing this file using gzip, seemingly it is NOT a compressed .BLEND file"); - } - const size_t have = MYBLOCK - zstream.avail_out; - total += have; - uncompressed.resize(total); - memcpy(uncompressed.data() + total - have, block, have); - } while (ret != Z_STREAM_END); - - // terminate zlib - inflateEnd(&zstream); + size_t total = 0; + Compression compression; + if (compression.open(Compression::Format::Binary)) { + total = compression.decompress((unsigned char *)reader->GetPtr(), reader->GetRemainingSize(), uncompressed); + compression.close(); + } // replace the input stream with a memory stream stream.reset(new MemoryIOStream(reinterpret_cast(uncompressed.data()), total)); diff --git a/code/AssetLib/FBX/FBXParser.cpp b/code/AssetLib/FBX/FBXParser.cpp index ef26f5322..1cbe8e833 100644 --- a/code/AssetLib/FBX/FBXParser.cpp +++ b/code/AssetLib/FBX/FBXParser.cpp @@ -46,11 +46,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef ASSIMP_BUILD_NO_FBX_IMPORTER -#ifdef ASSIMP_BUILD_NO_OWN_ZLIB -# include +//#ifdef ASSIMP_BUILD_NO_OWN_ZLIB +#include "Common/Compression.h" +/*# include #else # include "../contrib/zlib/zlib.h" -#endif +#endif*/ #include "FBXTokenizer.h" #include "FBXParser.h" @@ -571,8 +572,11 @@ void ReadBinaryDataArray(char type, uint32_t count, const char*& data, const cha else if(encmode == 1) { // zlib/deflate, next comes ZIP head (0x78 0x01) // see http://www.ietf.org/rfc/rfc1950.txt - - z_stream zstream; + Compression compress; + if (compress.open(Compression::Format::Binary)) { + compress.decompress(data, comp_len, buff); + } + /* z_stream zstream; zstream.opaque = Z_NULL; zstream.zalloc = Z_NULL; zstream.zfree = Z_NULL; @@ -581,9 +585,9 @@ void ReadBinaryDataArray(char type, uint32_t count, const char*& data, const cha // http://hewgill.com/journal/entries/349-how-to-decompress-gzip-stream-with-zlib if(Z_OK != inflateInit(&zstream)) { ParseError("failure initializing zlib"); - } + }*/ - zstream.next_in = reinterpret_cast( const_cast(data) ); + /* zstream.next_in = reinterpret_cast(const_cast(data)); zstream.avail_in = comp_len; zstream.avail_out = static_cast(buff.size()); @@ -595,7 +599,7 @@ void ReadBinaryDataArray(char type, uint32_t count, const char*& data, const cha } // terminate zlib - inflateEnd(&zstream); + inflateEnd(&zstream);*/ } #ifdef ASSIMP_BUILD_DEBUG else { diff --git a/code/AssetLib/X/XFileParser.cpp b/code/AssetLib/X/XFileParser.cpp index adcb7c8f4..e867c3f11 100644 --- a/code/AssetLib/X/XFileParser.cpp +++ b/code/AssetLib/X/XFileParser.cpp @@ -60,11 +60,12 @@ using namespace Assimp::Formatter; #ifndef ASSIMP_BUILD_NO_COMPRESSED_X -#ifdef ASSIMP_BUILD_NO_OWN_ZLIB +/* #ifdef ASSIMP_BUILD_NO_OWN_ZLIB #include #else #include "../contrib/zlib/zlib.h" -#endif +#endif*/ +#include "Common/Compression.h" // Magic identifier for MSZIP compressed data #define MSZIP_MAGIC 0x4B43 @@ -72,13 +73,13 @@ using namespace Assimp::Formatter; // ------------------------------------------------------------------------------------------------ // Dummy memory wrappers for use with zlib -static void *dummy_alloc(void * /*opaque*/, unsigned int items, unsigned int size) { +/* static void *dummy_alloc(void * opaque, unsigned int items, unsigned int size) { return ::operator new(items *size); } -static void dummy_free(void * /*opaque*/, void *address) { +static void dummy_free(void * opaque, void *address) { return ::operator delete(address); -} +}*/ #endif // !! ASSIMP_BUILD_NO_COMPRESSED_X @@ -171,15 +172,16 @@ XFileParser::XFileParser(const std::vector &pBuffer) : * /////////////////////////////////////////////////////////////////////// */ + Compression compression; // build a zlib stream - z_stream stream; + /* z_stream stream; stream.opaque = nullptr; stream.zalloc = &dummy_alloc; stream.zfree = &dummy_free; stream.data_type = (mIsBinaryFormat ? Z_BINARY : Z_ASCII); // initialize the inflation algorithm - ::inflateInit2(&stream, -MAX_WBITS); + ::inflateInit2(&stream, -MAX_WBITS);*/ // skip unknown data (checksum, flags?) mP += 6; @@ -213,7 +215,12 @@ XFileParser::XFileParser(const std::vector &pBuffer) : // Allocate storage and terminating zero and do the actual uncompressing uncompressed.resize(est_out + 1); char *out = &uncompressed.front(); - while (mP + 3 < mEnd) { + + if (compression.open(mIsBinaryFormat ? Compression::Format::Binary : Compression::Format::ASCII)) { + compression.decompress(mP, std::distance(mP, mEnd-3), uncompressed); + compression.close(); + } + /* while (mP + 3 < mEnd) { uint16_t ofs = *((uint16_t *)mP); AI_SWAP2(ofs); mP += 4; @@ -242,7 +249,7 @@ XFileParser::XFileParser(const std::vector &pBuffer) : } // terminate zlib - ::inflateEnd(&stream); + ::inflateEnd(&stream);*/ // ok, update pointers to point to the uncompressed file data mP = &uncompressed[0]; diff --git a/code/AssetLib/XGL/XGLLoader.cpp b/code/AssetLib/XGL/XGLLoader.cpp index a711598f9..6c92f03b2 100644 --- a/code/AssetLib/XGL/XGLLoader.cpp +++ b/code/AssetLib/XGL/XGLLoader.cpp @@ -112,7 +112,7 @@ const aiImporterDesc *XGLImporter::GetInfo() const { // Imports the given file into the given scene structure. void XGLImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSystem *pIOHandler) { #ifndef ASSIMP_BUILD_NO_COMPRESSED_XGL - std::vector uncompressed; + std::vector uncompressed; #endif m_scene = pScene; @@ -132,7 +132,7 @@ void XGLImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy Compression c; size_t total = 0l; - if (c.open()) { + if (c.open(Compression::Format::Binary)) { // skip two extra bytes, zgl files do carry a crc16 upfront (I think) raw_reader->IncPtr(2); total = c.decompress((unsigned char *)raw_reader->GetPtr(), raw_reader->GetRemainingSize(), uncompressed); diff --git a/code/Common/Compression.cpp b/code/Common/Compression.cpp index 2e3defa8c..e5d1323b7 100644 --- a/code/Common/Compression.cpp +++ b/code/Common/Compression.cpp @@ -70,7 +70,7 @@ Compression::~Compression() { delete mImpl; } -bool Compression::open() { +bool Compression::open(Format format) { ai_assert(mImpl != nullptr); if (mImpl->mOpen) { @@ -81,7 +81,11 @@ bool Compression::open() { mImpl->mZSstream.opaque = Z_NULL; mImpl->mZSstream.zalloc = Z_NULL; mImpl->mZSstream.zfree = Z_NULL; - mImpl->mZSstream.data_type = Z_BINARY; + if (format == Format::Binary) { + mImpl->mZSstream.data_type = Z_BINARY; + } else { + mImpl->mZSstream.data_type = Z_ASCII; + } // raw decompression without a zlib or gzip header inflateInit2(&mImpl->mZSstream, -MAX_WBITS); @@ -90,12 +94,12 @@ bool Compression::open() { return mImpl->mOpen; } -constexpr size_t MYBLOCK = 1024; +constexpr size_t MYBLOCK = 32786; -size_t Compression::decompress(unsigned char *data, size_t in, std::vector &uncompressed) { +size_t Compression::decompress(const void *data, size_t in, std::vector &uncompressed) { ai_assert(mImpl != nullptr); - mImpl->mZSstream.next_in = reinterpret_cast(data); + mImpl->mZSstream.next_in = (Bytef*)(data); mImpl->mZSstream.avail_in = (uInt)in; Bytef block[MYBLOCK] = {}; diff --git a/code/Common/Compression.h b/code/Common/Compression.h index 20c92cb60..cb5c00fbd 100644 --- a/code/Common/Compression.h +++ b/code/Common/Compression.h @@ -48,6 +48,13 @@ namespace Assimp { /// @brief This class provides the decompression of zlib-compressed data. class Compression { public: + enum class Format { + Binary = 0, + ASCII, + + NumFormats, + InvalidFormat + }; /// @brief The class constructor. Compression(); @@ -56,7 +63,7 @@ public: /// @brief Will open the access to the compression. /// @return true if close was successful, false if not. - bool open(); + bool open(Format format); /// @brief Will return the open state. /// @return true if the access is opened, false if not. @@ -70,7 +77,7 @@ public: /// @param[in] data The data to decompress /// @param[in] in The size of the data. /// @param[out uncompressed A std::vector containing the decompressed data. - size_t decompress(unsigned char *data, size_t in, std::vector &uncompressed); + size_t decompress(const void *data, size_t in, std::vector &uncompressed); private: struct impl; From 23b43d1825af6df3d49742a59679e9a0efcd9f89 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 8 Feb 2022 21:43:14 +0100 Subject: [PATCH 2/7] Fix X-Importer decompress algorithm --- code/AssetLib/X/XFileParser.cpp | 80 +++++++-------------------------- code/Common/Compression.cpp | 25 +++++++++++ code/Common/Compression.h | 2 + 3 files changed, 43 insertions(+), 64 deletions(-) diff --git a/code/AssetLib/X/XFileParser.cpp b/code/AssetLib/X/XFileParser.cpp index e867c3f11..60dfb8d56 100644 --- a/code/AssetLib/X/XFileParser.cpp +++ b/code/AssetLib/X/XFileParser.cpp @@ -5,8 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2022, assimp team - - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -60,26 +58,11 @@ using namespace Assimp::Formatter; #ifndef ASSIMP_BUILD_NO_COMPRESSED_X -/* #ifdef ASSIMP_BUILD_NO_OWN_ZLIB -#include -#else -#include "../contrib/zlib/zlib.h" -#endif*/ #include "Common/Compression.h" // Magic identifier for MSZIP compressed data -#define MSZIP_MAGIC 0x4B43 -#define MSZIP_BLOCK 32786 - -// ------------------------------------------------------------------------------------------------ -// Dummy memory wrappers for use with zlib -/* static void *dummy_alloc(void * opaque, unsigned int items, unsigned int size) { - return ::operator new(items *size); -} - -static void dummy_free(void * opaque, void *address) { - return ::operator delete(address); -}*/ +constexpr unsigned int MSZIP_MAGIC = 0x4B43; +constexpr size_t MSZIP_BLOCK = 32786l; #endif // !! ASSIMP_BUILD_NO_COMPRESSED_X @@ -172,17 +155,6 @@ XFileParser::XFileParser(const std::vector &pBuffer) : * /////////////////////////////////////////////////////////////////////// */ - Compression compression; - // build a zlib stream - /* z_stream stream; - stream.opaque = nullptr; - stream.zalloc = &dummy_alloc; - stream.zfree = &dummy_free; - stream.data_type = (mIsBinaryFormat ? Z_BINARY : Z_ASCII); - - // initialize the inflation algorithm - ::inflateInit2(&stream, -MAX_WBITS);*/ - // skip unknown data (checksum, flags?) mP += 6; @@ -211,45 +183,25 @@ XFileParser::XFileParser(const std::vector &pBuffer) : P1 += ofs; est_out += MSZIP_BLOCK; // one decompressed block is 32786 in size } - + // Allocate storage and terminating zero and do the actual uncompressing + Compression compression; uncompressed.resize(est_out + 1); char *out = &uncompressed.front(); - if (compression.open(mIsBinaryFormat ? Compression::Format::Binary : Compression::Format::ASCII)) { - compression.decompress(mP, std::distance(mP, mEnd-3), uncompressed); + while (mP + 3 < mEnd) { + uint16_t ofs = *((uint16_t *)mP); + AI_SWAP2(ofs); + mP += 4; + + if (mP + ofs > mEnd + 2) { + throw DeadlyImportError("X: Unexpected EOF in compressed chunk"); + } + out += compression.decompressBlock(mP, ofs, out, MSZIP_BLOCK); + mP += ofs; + } compression.close(); } - /* while (mP + 3 < mEnd) { - uint16_t ofs = *((uint16_t *)mP); - AI_SWAP2(ofs); - mP += 4; - - if (mP + ofs > mEnd + 2) { - throw DeadlyImportError("X: Unexpected EOF in compressed chunk"); - } - - // push data to the stream - stream.next_in = (Bytef *)mP; - stream.avail_in = ofs; - stream.next_out = (Bytef *)out; - stream.avail_out = MSZIP_BLOCK; - - // and decompress the data .... - int ret = ::inflate(&stream, Z_SYNC_FLUSH); - if (ret != Z_OK && ret != Z_STREAM_END) - throw DeadlyImportError("X: Failed to decompress MSZIP-compressed data"); - - ::inflateReset(&stream); - ::inflateSetDictionary(&stream, (const Bytef *)out, MSZIP_BLOCK - stream.avail_out); - - // and advance to the next offset - out += MSZIP_BLOCK - stream.avail_out; - mP += ofs; - } - - // terminate zlib - ::inflateEnd(&stream);*/ // ok, update pointers to point to the uncompressed file data mP = &uncompressed[0]; @@ -507,7 +459,7 @@ void XFileParser::ParseDataObjectSkinWeights(Mesh *pMesh) { bone.mWeights.reserve(numWeights); for (unsigned int a = 0; a < numWeights; a++) { - BoneWeight weight; + BoneWeight weight = {}; weight.mVertex = ReadInt(); bone.mWeights.push_back(weight); } diff --git a/code/Common/Compression.cpp b/code/Common/Compression.cpp index e5d1323b7..6b990a47c 100644 --- a/code/Common/Compression.cpp +++ b/code/Common/Compression.cpp @@ -108,6 +108,7 @@ size_t Compression::decompress(const void *data, size_t in, std::vector &u do { mImpl->mZSstream.avail_out = MYBLOCK; mImpl->mZSstream.next_out = block; + ret = inflate(&mImpl->mZSstream, Z_NO_FLUSH); if (ret != Z_STREAM_END && ret != Z_OK) { @@ -123,6 +124,30 @@ size_t Compression::decompress(const void *data, size_t in, std::vector &u return total; } +size_t Compression::decompressBlock(const void *data, size_t in, char *out, size_t availableOut) { + + // push data to the stream + mImpl->mZSstream.next_in = (Bytef *)data; + mImpl->mZSstream.avail_in = (uInt)in; + mImpl->mZSstream.next_out = (Bytef *)out; + mImpl->mZSstream.avail_out = (uInt)availableOut; + + // and decompress the data .... + int ret = ::inflate(&mImpl->mZSstream, Z_SYNC_FLUSH); + if (ret != Z_OK && ret != Z_STREAM_END) + throw DeadlyImportError("X: Failed to decompress MSZIP-compressed data"); + + ::inflateReset(&mImpl->mZSstream); + ::inflateSetDictionary(&mImpl->mZSstream, (const Bytef *)out, (uInt)availableOut - mImpl->mZSstream.avail_out); + + + + ::inflateReset(&mImpl->mZSstream); + ::inflateSetDictionary(&mImpl->mZSstream, (const Bytef *)out, (uInt)availableOut - mImpl->mZSstream.avail_out); + + return availableOut - (size_t)mImpl->mZSstream.avail_out; +} + bool Compression::isOpen() const { ai_assert(mImpl != nullptr); diff --git a/code/Common/Compression.h b/code/Common/Compression.h index cb5c00fbd..3938acb7f 100644 --- a/code/Common/Compression.h +++ b/code/Common/Compression.h @@ -79,6 +79,8 @@ public: /// @param[out uncompressed A std::vector containing the decompressed data. size_t decompress(const void *data, size_t in, std::vector &uncompressed); + size_t decompressBlock(const void *data, size_t in, char *out, size_t availableOut); + private: struct impl; impl *mImpl; From 3e09d462faf72e7d31e7d0c82f413e5736e7c92b Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 12 Feb 2022 09:25:45 +0100 Subject: [PATCH 3/7] Fix compression for fbx --- code/AssetLib/Blender/BlenderLoader.cpp | 2 +- code/AssetLib/FBX/FBXParser.cpp | 41 ++++++-------- code/AssetLib/X/XFileParser.cpp | 5 +- code/AssetLib/XGL/XGLLoader.cpp | 8 +-- code/Common/Compression.cpp | 73 ++++++++++++++++++------- code/Common/Compression.h | 12 +++- test/unit/utFBXImporterExporter.cpp | 2 +- 7 files changed, 88 insertions(+), 55 deletions(-) diff --git a/code/AssetLib/Blender/BlenderLoader.cpp b/code/AssetLib/Blender/BlenderLoader.cpp index 70fccae04..5d3757ca1 100644 --- a/code/AssetLib/Blender/BlenderLoader.cpp +++ b/code/AssetLib/Blender/BlenderLoader.cpp @@ -175,7 +175,7 @@ void BlenderImporter::InternReadFile(const std::string &pFile, size_t total = 0; Compression compression; - if (compression.open(Compression::Format::Binary)) { + if (compression.open(Compression::Format::Binary, Compression::FlushMode::Finish) ) { total = compression.decompress((unsigned char *)reader->GetPtr(), reader->GetRemainingSize(), uncompressed); compression.close(); } diff --git a/code/AssetLib/FBX/FBXParser.cpp b/code/AssetLib/FBX/FBXParser.cpp index 1cbe8e833..2c0ff95b1 100644 --- a/code/AssetLib/FBX/FBXParser.cpp +++ b/code/AssetLib/FBX/FBXParser.cpp @@ -4,7 +4,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2022, assimp team - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -48,10 +47,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //#ifdef ASSIMP_BUILD_NO_OWN_ZLIB #include "Common/Compression.h" -/*# include -#else -# include "../contrib/zlib/zlib.h" -#endif*/ +//# include +//#else +//# include "../contrib/zlib/zlib.h" +//#endif #include "FBXTokenizer.h" #include "FBXParser.h" @@ -116,9 +115,7 @@ namespace Assimp { namespace FBX { // ------------------------------------------------------------------------------------------------ -Element::Element(const Token& key_token, Parser& parser) -: key_token(key_token) -{ +Element::Element(const Token& key_token, Parser& parser) : key_token(key_token) { TokenPtr n = nullptr; do { n = parser.AdvanceToNextToken(); @@ -211,8 +208,7 @@ Scope::Scope(Parser& parser,bool topLevel) } // ------------------------------------------------------------------------------------------------ -Scope::~Scope() -{ +Scope::~Scope() { for(ElementMap::value_type& v : elements) { delete v.second; } @@ -528,9 +524,7 @@ void ReadBinaryDataArrayHead(const char*& data, const char* end, char& type, uin // ------------------------------------------------------------------------------------------------ // read binary data array, assume cursor points to the 'compression mode' field (i.e. behind the header) void ReadBinaryDataArray(char type, uint32_t count, const char*& data, const char* end, - std::vector& buff, - const Element& /*el*/) -{ + std::vector& buff, const Element& /*el*/) { BE_NCONST uint32_t encmode = SafeParse(data, end); AI_SWAP4(encmode); data += 4; @@ -572,11 +566,12 @@ void ReadBinaryDataArray(char type, uint32_t count, const char*& data, const cha else if(encmode == 1) { // zlib/deflate, next comes ZIP head (0x78 0x01) // see http://www.ietf.org/rfc/rfc1950.txt - Compression compress; - if (compress.open(Compression::Format::Binary)) { + Compression compress; + if (compress.open(Compression::Format::Binary, Compression::FlushMode::Finish)) { compress.decompress(data, comp_len, buff); + compress.close(); } - /* z_stream zstream; + /* z_stream zstream = {}; zstream.opaque = Z_NULL; zstream.zalloc = Z_NULL; zstream.zfree = Z_NULL; @@ -585,9 +580,9 @@ void ReadBinaryDataArray(char type, uint32_t count, const char*& data, const cha // http://hewgill.com/journal/entries/349-how-to-decompress-gzip-stream-with-zlib if(Z_OK != inflateInit(&zstream)) { ParseError("failure initializing zlib"); - }*/ + } - /* zstream.next_in = reinterpret_cast(const_cast(data)); + zstream.next_in = reinterpret_cast(const_cast(data)); zstream.avail_in = comp_len; zstream.avail_out = static_cast(buff.size()); @@ -705,7 +700,6 @@ void ParseVectorDataArray(std::vector& out, const Element& el) } } - // ------------------------------------------------------------------------------------------------ // read an array of color4 tuples void ParseVectorDataArray(std::vector& out, const Element& el) @@ -790,8 +784,7 @@ void ParseVectorDataArray(std::vector& out, const Element& el) // ------------------------------------------------------------------------------------------------ // read an array of float2 tuples -void ParseVectorDataArray(std::vector& out, const Element& el) -{ +void ParseVectorDataArray(std::vector& out, const Element& el) { out.resize( 0 ); const TokenList& tok = el.Tokens(); if(tok.empty()) { @@ -835,8 +828,7 @@ void ParseVectorDataArray(std::vector& out, const Element& el) out.push_back(aiVector2D(static_cast(d[0]), static_cast(d[1]))); } - } - else if (type == 'f') { + } else if (type == 'f') { const float* f = reinterpret_cast(&buff[0]); for (unsigned int i = 0; i < count2; ++i, f += 2) { out.push_back(aiVector2D(f[0],f[1])); @@ -869,8 +861,7 @@ void ParseVectorDataArray(std::vector& out, const Element& el) // ------------------------------------------------------------------------------------------------ // read an array of ints -void ParseVectorDataArray(std::vector& out, const Element& el) -{ +void ParseVectorDataArray(std::vector& out, const Element& el) { out.resize( 0 ); const TokenList& tok = el.Tokens(); if(tok.empty()) { diff --git a/code/AssetLib/X/XFileParser.cpp b/code/AssetLib/X/XFileParser.cpp index 60dfb8d56..4731710d7 100644 --- a/code/AssetLib/X/XFileParser.cpp +++ b/code/AssetLib/X/XFileParser.cpp @@ -188,7 +188,7 @@ XFileParser::XFileParser(const std::vector &pBuffer) : Compression compression; uncompressed.resize(est_out + 1); char *out = &uncompressed.front(); - if (compression.open(mIsBinaryFormat ? Compression::Format::Binary : Compression::Format::ASCII)) { + if (compression.open(mIsBinaryFormat ? Compression::Format::Binary : Compression::Format::ASCII, Compression::FlushMode::SyncFlush)) { while (mP + 3 < mEnd) { uint16_t ofs = *((uint16_t *)mP); AI_SWAP2(ofs); @@ -238,8 +238,9 @@ void XFileParser::ParseFile() { while (running) { // read name of next object std::string objectName = GetNextToken(); - if (objectName.length() == 0) + if (objectName.length() == 0) { break; + } // parse specific object if (objectName == "template") diff --git a/code/AssetLib/XGL/XGLLoader.cpp b/code/AssetLib/XGL/XGLLoader.cpp index 6c92f03b2..966d547ac 100644 --- a/code/AssetLib/XGL/XGLLoader.cpp +++ b/code/AssetLib/XGL/XGLLoader.cpp @@ -130,13 +130,13 @@ void XGLImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy #else std::unique_ptr raw_reader(new StreamReaderLE(stream)); - Compression c; + Compression compression; size_t total = 0l; - if (c.open(Compression::Format::Binary)) { + if (compression.open(Compression::Format::Binary, Compression::FlushMode::NoFlush)) { // skip two extra bytes, zgl files do carry a crc16 upfront (I think) raw_reader->IncPtr(2); - total = c.decompress((unsigned char *)raw_reader->GetPtr(), raw_reader->GetRemainingSize(), uncompressed); - c.close(); + total = compression.decompress((unsigned char *)raw_reader->GetPtr(), raw_reader->GetRemainingSize(), uncompressed); + compression.close(); } // replace the input stream with a memory stream stream.reset(new MemoryIOStream(reinterpret_cast(uncompressed.data()), total)); diff --git a/code/Common/Compression.cpp b/code/Common/Compression.cpp index 6b990a47c..f6f0db4f4 100644 --- a/code/Common/Compression.cpp +++ b/code/Common/Compression.cpp @@ -54,9 +54,12 @@ namespace Assimp { struct Compression::impl { bool mOpen; z_stream mZSstream; - + FlushMode mFlushMode; impl() : - mOpen(false) {} + mOpen(false), + mFlushMode(Compression::FlushMode::NoFlush) { + // empty + } }; Compression::Compression() : @@ -70,7 +73,7 @@ Compression::~Compression() { delete mImpl; } -bool Compression::open(Format format) { +bool Compression::open(Format format, FlushMode flush) { ai_assert(mImpl != nullptr); if (mImpl->mOpen) { @@ -81,6 +84,7 @@ bool Compression::open(Format format) { mImpl->mZSstream.opaque = Z_NULL; mImpl->mZSstream.zalloc = Z_NULL; mImpl->mZSstream.zfree = Z_NULL; + mImpl->mFlushMode = flush; if (format == Format::Binary) { mImpl->mZSstream.data_type = Z_BINARY; } else { @@ -88,7 +92,7 @@ bool Compression::open(Format format) { } // raw decompression without a zlib or gzip header - inflateInit2(&mImpl->mZSstream, -MAX_WBITS); + inflateInit(&mImpl->mZSstream); mImpl->mOpen = true; return mImpl->mOpen; @@ -96,30 +100,62 @@ bool Compression::open(Format format) { constexpr size_t MYBLOCK = 32786; +static int getFlushMode(Compression::FlushMode flush) { + int z_flush = 0; + switch (flush) { + case Compression::FlushMode::NoFlush: + z_flush = Z_NO_FLUSH; + break; + case Compression::FlushMode::SyncFlush: + z_flush = Z_SYNC_FLUSH; + break; + case Compression::FlushMode::Finish: + z_flush = Z_FINISH; + break; + default: + ai_assert(false); + break; + } + + return z_flush; +} + size_t Compression::decompress(const void *data, size_t in, std::vector &uncompressed) { ai_assert(mImpl != nullptr); mImpl->mZSstream.next_in = (Bytef*)(data); mImpl->mZSstream.avail_in = (uInt)in; - Bytef block[MYBLOCK] = {}; int ret = 0; size_t total = 0l; - do { - mImpl->mZSstream.avail_out = MYBLOCK; - mImpl->mZSstream.next_out = block; - - ret = inflate(&mImpl->mZSstream, Z_NO_FLUSH); + + const int flushMode = getFlushMode(mImpl->mFlushMode); + if (flushMode == Z_FINISH) { + mImpl->mZSstream.avail_out = static_cast(uncompressed.size()); + mImpl->mZSstream.next_out = reinterpret_cast(&*uncompressed.begin()); + ret = inflate(&mImpl->mZSstream, Z_FINISH); if (ret != Z_STREAM_END && ret != Z_OK) { throw DeadlyImportError("Compression", "Failure decompressing this file using gzip."); - } - const size_t have = MYBLOCK - mImpl->mZSstream.avail_out; - total += have; - uncompressed.resize(total); - ::memcpy(uncompressed.data() + total - have, block, have); - } while (ret != Z_STREAM_END); + total = mImpl->mZSstream.avail_out; + } else { + do { + Bytef block[MYBLOCK] = {}; + mImpl->mZSstream.avail_out = MYBLOCK; + mImpl->mZSstream.next_out = block; + + ret = inflate(&mImpl->mZSstream, flushMode); + + if (ret != Z_STREAM_END && ret != Z_OK) { + throw DeadlyImportError("Compression", "Failure decompressing this file using gzip."); + } + const size_t have = MYBLOCK - mImpl->mZSstream.avail_out; + total += have; + uncompressed.resize(total); + ::memcpy(uncompressed.data() + total - have, block, have); + } while (ret != Z_STREAM_END); + } return total; } @@ -140,11 +176,6 @@ size_t Compression::decompressBlock(const void *data, size_t in, char *out, size ::inflateReset(&mImpl->mZSstream); ::inflateSetDictionary(&mImpl->mZSstream, (const Bytef *)out, (uInt)availableOut - mImpl->mZSstream.avail_out); - - - ::inflateReset(&mImpl->mZSstream); - ::inflateSetDictionary(&mImpl->mZSstream, (const Bytef *)out, (uInt)availableOut - mImpl->mZSstream.avail_out); - return availableOut - (size_t)mImpl->mZSstream.avail_out; } diff --git a/code/Common/Compression.h b/code/Common/Compression.h index 3938acb7f..aa552ab60 100644 --- a/code/Common/Compression.h +++ b/code/Common/Compression.h @@ -55,6 +55,16 @@ public: NumFormats, InvalidFormat }; + + enum class FlushMode { + NoFlush = 0, + SyncFlush, + Finish, + + NumModes, + InvalidMode + }; + /// @brief The class constructor. Compression(); @@ -63,7 +73,7 @@ public: /// @brief Will open the access to the compression. /// @return true if close was successful, false if not. - bool open(Format format); + bool open(Format format, FlushMode flush); /// @brief Will return the open state. /// @return true if the access is opened, false if not. diff --git a/test/unit/utFBXImporterExporter.cpp b/test/unit/utFBXImporterExporter.cpp index 679f6ded3..9b8f2fa0d 100644 --- a/test/unit/utFBXImporterExporter.cpp +++ b/test/unit/utFBXImporterExporter.cpp @@ -53,7 +53,7 @@ using namespace Assimp; class utFBXImporterExporter : public AbstractImportExportBase { public: - virtual bool importerTest() { + bool importerTest() override { Assimp::Importer importer; const aiScene *scene = importer.ReadFile(ASSIMP_TEST_MODELS_DIR "/FBX/spider.fbx", aiProcess_ValidateDataStructure); return nullptr != scene; From 52b6c4f7c098adc84ffee41ef45849cc4d9611c4 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sun, 13 Feb 2022 18:44:22 +0100 Subject: [PATCH 4/7] Fix broken importer and add some review findings2 --- code/AssetLib/Blender/BlenderLoader.cpp | 2 +- code/AssetLib/FBX/FBXParser.cpp | 2 +- code/AssetLib/X/XFileParser.cpp | 2 +- code/AssetLib/XGL/XGLLoader.cpp | 2 +- code/Common/Compression.cpp | 13 ++++++------- code/Common/Compression.h | 16 +++++++++++----- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/code/AssetLib/Blender/BlenderLoader.cpp b/code/AssetLib/Blender/BlenderLoader.cpp index 5d3757ca1..c67879be5 100644 --- a/code/AssetLib/Blender/BlenderLoader.cpp +++ b/code/AssetLib/Blender/BlenderLoader.cpp @@ -175,7 +175,7 @@ void BlenderImporter::InternReadFile(const std::string &pFile, size_t total = 0; Compression compression; - if (compression.open(Compression::Format::Binary, Compression::FlushMode::Finish) ) { + if (compression.open(Compression::Format::Binary, Compression::FlushMode::NoFlush, 16 + MAX_WBITS)) { total = compression.decompress((unsigned char *)reader->GetPtr(), reader->GetRemainingSize(), uncompressed); compression.close(); } diff --git a/code/AssetLib/FBX/FBXParser.cpp b/code/AssetLib/FBX/FBXParser.cpp index 2c0ff95b1..c453647e0 100644 --- a/code/AssetLib/FBX/FBXParser.cpp +++ b/code/AssetLib/FBX/FBXParser.cpp @@ -567,7 +567,7 @@ void ReadBinaryDataArray(char type, uint32_t count, const char*& data, const cha // zlib/deflate, next comes ZIP head (0x78 0x01) // see http://www.ietf.org/rfc/rfc1950.txt Compression compress; - if (compress.open(Compression::Format::Binary, Compression::FlushMode::Finish)) { + if (compress.open(Compression::Format::Binary, Compression::FlushMode::Finish,0)) { compress.decompress(data, comp_len, buff); compress.close(); } diff --git a/code/AssetLib/X/XFileParser.cpp b/code/AssetLib/X/XFileParser.cpp index 4731710d7..90a257dfa 100644 --- a/code/AssetLib/X/XFileParser.cpp +++ b/code/AssetLib/X/XFileParser.cpp @@ -188,7 +188,7 @@ XFileParser::XFileParser(const std::vector &pBuffer) : Compression compression; uncompressed.resize(est_out + 1); char *out = &uncompressed.front(); - if (compression.open(mIsBinaryFormat ? Compression::Format::Binary : Compression::Format::ASCII, Compression::FlushMode::SyncFlush)) { + if (compression.open(mIsBinaryFormat ? Compression::Format::Binary : Compression::Format::ASCII, Compression::FlushMode::SyncFlush, -MAX_WBITS)) { while (mP + 3 < mEnd) { uint16_t ofs = *((uint16_t *)mP); AI_SWAP2(ofs); diff --git a/code/AssetLib/XGL/XGLLoader.cpp b/code/AssetLib/XGL/XGLLoader.cpp index 966d547ac..4fbee6eea 100644 --- a/code/AssetLib/XGL/XGLLoader.cpp +++ b/code/AssetLib/XGL/XGLLoader.cpp @@ -132,7 +132,7 @@ void XGLImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy Compression compression; size_t total = 0l; - if (compression.open(Compression::Format::Binary, Compression::FlushMode::NoFlush)) { + if (compression.open(Compression::Format::Binary, Compression::FlushMode::NoFlush, -MAX_WBITS)) { // skip two extra bytes, zgl files do carry a crc16 upfront (I think) raw_reader->IncPtr(2); total = compression.decompress((unsigned char *)raw_reader->GetPtr(), raw_reader->GetRemainingSize(), uncompressed); diff --git a/code/Common/Compression.cpp b/code/Common/Compression.cpp index f6f0db4f4..86f1e9ec0 100644 --- a/code/Common/Compression.cpp +++ b/code/Common/Compression.cpp @@ -43,11 +43,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#ifdef ASSIMP_BUILD_NO_OWN_ZLIB -#include -#else -#include "../contrib/zlib/zlib.h" -#endif namespace Assimp { @@ -73,7 +68,7 @@ Compression::~Compression() { delete mImpl; } -bool Compression::open(Format format, FlushMode flush) { +bool Compression::open(Format format, FlushMode flush, int windowBits) { ai_assert(mImpl != nullptr); if (mImpl->mOpen) { @@ -92,7 +87,11 @@ bool Compression::open(Format format, FlushMode flush) { } // raw decompression without a zlib or gzip header - inflateInit(&mImpl->mZSstream); + if (windowBits == 0) { + inflateInit(&mImpl->mZSstream); + } else { + inflateInit2(&mImpl->mZSstream, windowBits); + } mImpl->mOpen = true; return mImpl->mOpen; diff --git a/code/Common/Compression.h b/code/Common/Compression.h index aa552ab60..05beed7f7 100644 --- a/code/Common/Compression.h +++ b/code/Common/Compression.h @@ -41,6 +41,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once +#ifdef ASSIMP_BUILD_NO_OWN_ZLIB +#include +#else +#include "../contrib/zlib/zlib.h" +#endif + #include namespace Assimp { @@ -49,20 +55,20 @@ namespace Assimp { class Compression { public: enum class Format { + InvalidFormat = -1, Binary = 0, ASCII, - NumFormats, - InvalidFormat + NumFormats }; enum class FlushMode { + InvalidFormat = -1, NoFlush = 0, SyncFlush, Finish, - NumModes, - InvalidMode + NumModes }; /// @brief The class constructor. @@ -73,7 +79,7 @@ public: /// @brief Will open the access to the compression. /// @return true if close was successful, false if not. - bool open(Format format, FlushMode flush); + bool open(Format format, FlushMode flush, int windowBits); /// @brief Will return the open state. /// @return true if the access is opened, false if not. From c718500c55c9ae96e4338a7b9e900ba9a88b9ab1 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Mon, 14 Feb 2022 20:25:18 +0100 Subject: [PATCH 5/7] Fix last review findings and finish windows bits --- code/AssetLib/Blender/BlenderLoader.cpp | 2 +- code/AssetLib/FBX/FBXParser.cpp | 26 +----------------- code/AssetLib/X/XFileParser.cpp | 6 ++--- code/AssetLib/XGL/XGLLoader.cpp | 12 ++++----- code/Common/Compression.cpp | 18 +++++++++---- code/Common/Compression.h | 35 +++++++++++++++++-------- 6 files changed, 48 insertions(+), 51 deletions(-) diff --git a/code/AssetLib/Blender/BlenderLoader.cpp b/code/AssetLib/Blender/BlenderLoader.cpp index c67879be5..b3591b4f1 100644 --- a/code/AssetLib/Blender/BlenderLoader.cpp +++ b/code/AssetLib/Blender/BlenderLoader.cpp @@ -175,7 +175,7 @@ void BlenderImporter::InternReadFile(const std::string &pFile, size_t total = 0; Compression compression; - if (compression.open(Compression::Format::Binary, Compression::FlushMode::NoFlush, 16 + MAX_WBITS)) { + if (compression.open(Compression::Format::Binary, Compression::FlushMode::NoFlush, 16 + Compression::MaxWBits)) { total = compression.decompress((unsigned char *)reader->GetPtr(), reader->GetRemainingSize(), uncompressed); compression.close(); } diff --git a/code/AssetLib/FBX/FBXParser.cpp b/code/AssetLib/FBX/FBXParser.cpp index c453647e0..e20377a3c 100644 --- a/code/AssetLib/FBX/FBXParser.cpp +++ b/code/AssetLib/FBX/FBXParser.cpp @@ -567,34 +567,10 @@ void ReadBinaryDataArray(char type, uint32_t count, const char*& data, const cha // zlib/deflate, next comes ZIP head (0x78 0x01) // see http://www.ietf.org/rfc/rfc1950.txt Compression compress; - if (compress.open(Compression::Format::Binary, Compression::FlushMode::Finish,0)) { + if (compress.open(Compression::Format::Binary, Compression::FlushMode::Finish, 0)) { compress.decompress(data, comp_len, buff); compress.close(); } - /* z_stream zstream = {}; - zstream.opaque = Z_NULL; - zstream.zalloc = Z_NULL; - zstream.zfree = Z_NULL; - zstream.data_type = Z_BINARY; - - // http://hewgill.com/journal/entries/349-how-to-decompress-gzip-stream-with-zlib - if(Z_OK != inflateInit(&zstream)) { - ParseError("failure initializing zlib"); - } - - zstream.next_in = reinterpret_cast(const_cast(data)); - zstream.avail_in = comp_len; - - zstream.avail_out = static_cast(buff.size()); - zstream.next_out = reinterpret_cast(&*buff.begin()); - const int ret = inflate(&zstream, Z_FINISH); - - if (ret != Z_STREAM_END && ret != Z_OK) { - ParseError("failure decompressing compressed data section"); - } - - // terminate zlib - inflateEnd(&zstream);*/ } #ifdef ASSIMP_BUILD_DEBUG else { diff --git a/code/AssetLib/X/XFileParser.cpp b/code/AssetLib/X/XFileParser.cpp index 90a257dfa..65f4023a7 100644 --- a/code/AssetLib/X/XFileParser.cpp +++ b/code/AssetLib/X/XFileParser.cpp @@ -117,13 +117,13 @@ XFileParser::XFileParser(const std::vector &pBuffer) : mIsBinaryFormat = true; compressed = true; } else - ThrowException("Unsupported xfile format '", mP[8], mP[9], mP[10], mP[11], "'"); + ThrowException("Unsupported x-file format '", mP[8], mP[9], mP[10], mP[11], "'"); // float size mBinaryFloatSize = (unsigned int)(mP[12] - 48) * 1000 + (unsigned int)(mP[13] - 48) * 100 + (unsigned int)(mP[14] - 48) * 10 + (unsigned int)(mP[15] - 48); if (mBinaryFloatSize != 32 && mBinaryFloatSize != 64) - ThrowException("Unknown float size ", mBinaryFloatSize, " specified in xfile header."); + ThrowException("Unknown float size ", mBinaryFloatSize, " specified in x-file header."); // The x format specifies size in bits, but we work in bytes mBinaryFloatSize /= 8; @@ -188,7 +188,7 @@ XFileParser::XFileParser(const std::vector &pBuffer) : Compression compression; uncompressed.resize(est_out + 1); char *out = &uncompressed.front(); - if (compression.open(mIsBinaryFormat ? Compression::Format::Binary : Compression::Format::ASCII, Compression::FlushMode::SyncFlush, -MAX_WBITS)) { + if (compression.open(mIsBinaryFormat ? Compression::Format::Binary : Compression::Format::ASCII, Compression::FlushMode::SyncFlush, -Compression::MAX_WBITS)) { while (mP + 3 < mEnd) { uint16_t ofs = *((uint16_t *)mP); AI_SWAP2(ofs); diff --git a/code/AssetLib/XGL/XGLLoader.cpp b/code/AssetLib/XGL/XGLLoader.cpp index 4fbee6eea..154af9854 100644 --- a/code/AssetLib/XGL/XGLLoader.cpp +++ b/code/AssetLib/XGL/XGLLoader.cpp @@ -53,8 +53,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include -#include -#include +//#include +//#include using namespace Assimp; @@ -132,14 +132,14 @@ void XGLImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy Compression compression; size_t total = 0l; - if (compression.open(Compression::Format::Binary, Compression::FlushMode::NoFlush, -MAX_WBITS)) { + if (compression.open(Compression::Format::Binary, Compression::FlushMode::NoFlush, -Compression::MaxWBits)) { // skip two extra bytes, zgl files do carry a crc16 upfront (I think) raw_reader->IncPtr(2); total = compression.decompress((unsigned char *)raw_reader->GetPtr(), raw_reader->GetRemainingSize(), uncompressed); compression.close(); } // replace the input stream with a memory stream - stream.reset(new MemoryIOStream(reinterpret_cast(uncompressed.data()), total)); + stream.reset(new MemoryIOStream(reinterpret_cast(uncompressed.data()), total)); #endif } @@ -200,7 +200,7 @@ void XGLImporter::ReadWorld(XmlNode &node, TempScope &scope) { if (!nd) { ThrowException("failure reading "); } - if (!nd->mName.length) { + if (nd->mName.length == 0) { nd->mName.Set("WORLD"); } @@ -784,4 +784,4 @@ aiColor3D XGLImporter::ReadCol3(XmlNode &node) { return aiColor3D(v.x, v.y, v.z); } -#endif +#endif // ASSIMP_BUILD_NO_XGL_IMPORTER diff --git a/code/Common/Compression.cpp b/code/Common/Compression.cpp index 86f1e9ec0..1249746b4 100644 --- a/code/Common/Compression.cpp +++ b/code/Common/Compression.cpp @@ -43,15 +43,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include - namespace Assimp { struct Compression::impl { bool mOpen; z_stream mZSstream; FlushMode mFlushMode; + impl() : mOpen(false), + mZSstream(), mFlushMode(Compression::FlushMode::NoFlush) { // empty } @@ -97,8 +98,6 @@ bool Compression::open(Format format, FlushMode flush, int windowBits) { return mImpl->mOpen; } -constexpr size_t MYBLOCK = 32786; - static int getFlushMode(Compression::FlushMode flush) { int z_flush = 0; switch (flush) { @@ -119,15 +118,19 @@ static int getFlushMode(Compression::FlushMode flush) { return z_flush; } +constexpr size_t MYBLOCK = 32786; + size_t Compression::decompress(const void *data, size_t in, std::vector &uncompressed) { ai_assert(mImpl != nullptr); + if (data == nullptr || in == 0) { + return 0l; + } mImpl->mZSstream.next_in = (Bytef*)(data); mImpl->mZSstream.avail_in = (uInt)in; int ret = 0; size_t total = 0l; - const int flushMode = getFlushMode(mImpl->mFlushMode); if (flushMode == Z_FINISH) { mImpl->mZSstream.avail_out = static_cast(uncompressed.size()); @@ -160,6 +163,10 @@ size_t Compression::decompress(const void *data, size_t in, std::vector &u } size_t Compression::decompressBlock(const void *data, size_t in, char *out, size_t availableOut) { + ai_assert(mImpl != nullptr); + if (data == nullptr || in == 0 || out == nullptr || availableOut == 0) { + return 0l; + } // push data to the stream mImpl->mZSstream.next_in = (Bytef *)data; @@ -169,8 +176,9 @@ size_t Compression::decompressBlock(const void *data, size_t in, char *out, size // and decompress the data .... int ret = ::inflate(&mImpl->mZSstream, Z_SYNC_FLUSH); - if (ret != Z_OK && ret != Z_STREAM_END) + if (ret != Z_OK && ret != Z_STREAM_END) { throw DeadlyImportError("X: Failed to decompress MSZIP-compressed data"); + } ::inflateReset(&mImpl->mZSstream); ::inflateSetDictionary(&mImpl->mZSstream, (const Bytef *)out, (uInt)availableOut - mImpl->mZSstream.avail_out); diff --git a/code/Common/Compression.h b/code/Common/Compression.h index 05beed7f7..fa0fc9e40 100644 --- a/code/Common/Compression.h +++ b/code/Common/Compression.h @@ -54,21 +54,25 @@ namespace Assimp { /// @brief This class provides the decompression of zlib-compressed data. class Compression { public: - enum class Format { - InvalidFormat = -1, - Binary = 0, - ASCII, + static const int MaxWBits = MAX_WBITS; - NumFormats + /// @brief Describes the format data type + enum class Format { + InvalidFormat = -1, ///< Invalid enum type. + Binary = 0, ///< Binary format. + ASCII, ///< ASCII format. + + NumFormats ///< The number of supported formats. }; + /// @brief The supported flush mode, used for blocked access. enum class FlushMode { - InvalidFormat = -1, - NoFlush = 0, - SyncFlush, - Finish, + InvalidFormat = -1, ///< Invalid enum type. + NoFlush = 0, ///< No flush, will be done on inflate end. + SyncFlush, ///< Synced flush mode. + Finish, ///< Finish mode, all in once, no block access. - NumModes + NumModes ///< The number of supported modes. }; /// @brief The class constructor. @@ -78,6 +82,9 @@ public: ~Compression(); /// @brief Will open the access to the compression. + /// @param[in] format The format type + /// @param[in] flush The flush mode. + /// @param[in] windowBits The windows history working size, shall be between 8 and 15. /// @return true if close was successful, false if not. bool open(Format format, FlushMode flush, int windowBits); @@ -89,12 +96,18 @@ public: /// @return true if close was successful, false if not. bool close(); - /// @brief Will decompress the data buffer. + /// @brief Will decompress the data buffer in one step. /// @param[in] data The data to decompress /// @param[in] in The size of the data. /// @param[out uncompressed A std::vector containing the decompressed data. size_t decompress(const void *data, size_t in, std::vector &uncompressed); + /// @brief Will decompress the data buffer block-wise. + /// @param[in] data The compressed data + /// @param[in] in The size of the data buffer + /// @param[out] out The output buffer + /// @param[out] availableOut The upper limit of the output buffer. + /// @return The size of the decompressed data buffer. size_t decompressBlock(const void *data, size_t in, char *out, size_t availableOut); private: From 7c13b16d308a29fb87ff3de5af7ad2c960b8afb0 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Mon, 14 Feb 2022 20:51:06 +0100 Subject: [PATCH 6/7] Add missing flush modes supported by zlib --- code/AssetLib/X/XFileParser.cpp | 15 +++++++++------ code/Common/Compression.cpp | 6 ++++++ code/Common/Compression.h | 2 ++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/code/AssetLib/X/XFileParser.cpp b/code/AssetLib/X/XFileParser.cpp index 65f4023a7..d22b99f5f 100644 --- a/code/AssetLib/X/XFileParser.cpp +++ b/code/AssetLib/X/XFileParser.cpp @@ -188,7 +188,8 @@ XFileParser::XFileParser(const std::vector &pBuffer) : Compression compression; uncompressed.resize(est_out + 1); char *out = &uncompressed.front(); - if (compression.open(mIsBinaryFormat ? Compression::Format::Binary : Compression::Format::ASCII, Compression::FlushMode::SyncFlush, -Compression::MAX_WBITS)) { + if (compression.open(mIsBinaryFormat ? Compression::Format::Binary : Compression::Format::ASCII, + Compression::FlushMode::SyncFlush, -Compression::MaxWBits)) { while (mP + 3 < mEnd) { uint16_t ofs = *((uint16_t *)mP); AI_SWAP2(ofs); @@ -243,11 +244,11 @@ void XFileParser::ParseFile() { } // parse specific object - if (objectName == "template") + if (objectName == "template") { ParseDataObjectTemplate(); - else if (objectName == "Frame") + } else if (objectName == "Frame") { ParseDataObjectFrame(nullptr); - else if (objectName == "Mesh") { + } else if (objectName == "Mesh") { // some meshes have no frames at all Mesh *mesh = new Mesh; ParseDataObjectMesh(mesh); @@ -286,11 +287,13 @@ void XFileParser::ParseDataObjectTemplate() { while (running) { std::string s = GetNextToken(); - if (s == "}") + if (s == "}") { break; + } - if (s.length() == 0) + if (s.length() == 0) { ThrowException("Unexpected end of file reached while parsing template definition"); + } } } diff --git a/code/Common/Compression.cpp b/code/Common/Compression.cpp index 1249746b4..3bf306dee 100644 --- a/code/Common/Compression.cpp +++ b/code/Common/Compression.cpp @@ -104,6 +104,12 @@ static int getFlushMode(Compression::FlushMode flush) { case Compression::FlushMode::NoFlush: z_flush = Z_NO_FLUSH; break; + case Compression::FlushMode::Block: + z_flush = Z_BLOCK; + break; + case Compression::FlushMode::Tree: + z_flush = Z_TREES; + break; case Compression::FlushMode::SyncFlush: z_flush = Z_SYNC_FLUSH; break; diff --git a/code/Common/Compression.h b/code/Common/Compression.h index 97e3f9bcd..edf1d232f 100644 --- a/code/Common/Compression.h +++ b/code/Common/Compression.h @@ -70,6 +70,8 @@ public: enum class FlushMode { InvalidFormat = -1, ///< Invalid enum type. NoFlush = 0, ///< No flush, will be done on inflate end. + Block, ///< Assists in combination of compress. + Tree, ///< Assists in combination of compress and returns if stream is finish. SyncFlush, ///< Synced flush mode. Finish, ///< Finish mode, all in once, no block access. From 5d8e5fd130c8d6180d9a55d5d1bf80b4c739f737 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 15 Feb 2022 16:10:33 +0100 Subject: [PATCH 7/7] Update XFileParser.cpp Fix typo --- code/AssetLib/X/XFileParser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/AssetLib/X/XFileParser.cpp b/code/AssetLib/X/XFileParser.cpp index d22b99f5f..558b97958 100644 --- a/code/AssetLib/X/XFileParser.cpp +++ b/code/AssetLib/X/XFileParser.cpp @@ -181,7 +181,7 @@ XFileParser::XFileParser(const std::vector &pBuffer) : // and advance to the next offset P1 += ofs; - est_out += MSZIP_BLOCK; // one decompressed block is 32786 in size + est_out += MSZIP_BLOCK; // one decompressed block is 327861 in size } // Allocate storage and terminating zero and do the actual uncompressing