From 27bcddfb1aa2c08f1976d833440e66e5ad3ae71f Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Tue, 8 Feb 2022 20:43:47 +0100 Subject: [PATCH] Migrate more importers to compression class --- code/AssetLib/Blender/BlenderLoader.cpp | 50 +++++-------------------- code/AssetLib/FBX/FBXParser.cpp | 20 ++++++---- code/AssetLib/X/XFileParser.cpp | 25 ++++++++----- code/AssetLib/XGL/XGLLoader.cpp | 4 +- code/Common/Compression.cpp | 14 ++++--- code/Common/Compression.h | 11 +++++- 6 files changed, 58 insertions(+), 66 deletions(-) diff --git a/code/AssetLib/Blender/BlenderLoader.cpp b/code/AssetLib/Blender/BlenderLoader.cpp index 3d9e8017b..70fccae04 100644 --- a/code/AssetLib/Blender/BlenderLoader.cpp +++ b/code/AssetLib/Blender/BlenderLoader.cpp @@ -66,11 +66,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // zlib is needed for compressed blend files #ifndef ASSIMP_BUILD_NO_COMPRESSED_BLEND -# ifdef ASSIMP_BUILD_NO_OWN_ZLIB +#include "Common/Compression.h" +/* #ifdef ASSIMP_BUILD_NO_OWN_ZLIB # include # else # include "../contrib/zlib/zlib.h" -# endif +# endif*/ #endif namespace Assimp { @@ -141,7 +142,7 @@ void BlenderImporter::SetupProperties(const Importer * /*pImp*/) { void BlenderImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSystem *pIOHandler) { #ifndef ASSIMP_BUILD_NO_COMPRESSED_BLEND - std::vector uncompressed; + std::vector uncompressed; #endif FileDatabase file; @@ -159,7 +160,6 @@ void BlenderImporter::InternReadFile(const std::string &pFile, #ifdef ASSIMP_BUILD_NO_COMPRESSED_BLEND ThrowException("BLENDER magic bytes are missing, is this file compressed (Assimp was built without decompression support)?"); #else - if (magic[0] != 0x1f || static_cast(magic[1]) != 0x8b) { ThrowException("BLENDER magic bytes are missing, couldn't find GZIP header either"); } @@ -173,42 +173,12 @@ void BlenderImporter::InternReadFile(const std::string &pFile, stream->Seek(0L, aiOrigin_SET); std::shared_ptr reader = std::shared_ptr(new StreamReaderLE(stream)); - // build a zlib stream - z_stream zstream; - zstream.opaque = Z_NULL; - zstream.zalloc = Z_NULL; - zstream.zfree = Z_NULL; - zstream.data_type = Z_BINARY; - - // http://hewgill.com/journal/entries/349-how-to-decompress-gzip-stream-with-zlib - inflateInit2(&zstream, 16 + MAX_WBITS); - - zstream.next_in = reinterpret_cast(reader->GetPtr()); - zstream.avail_in = (uInt)reader->GetRemainingSize(); - - size_t total = 0l; - - // TODO: be smarter about this, decompress directly into heap buffer - // and decompress the data .... do 1k chunks in the hope that we won't kill the stack -#define MYBLOCK 1024 - Bytef block[MYBLOCK]; - int ret; - do { - zstream.avail_out = MYBLOCK; - zstream.next_out = block; - ret = inflate(&zstream, Z_NO_FLUSH); - - if (ret != Z_STREAM_END && ret != Z_OK) { - ThrowException("Failure decompressing this file using gzip, seemingly it is NOT a compressed .BLEND file"); - } - const size_t have = MYBLOCK - zstream.avail_out; - total += have; - uncompressed.resize(total); - memcpy(uncompressed.data() + total - have, block, have); - } while (ret != Z_STREAM_END); - - // terminate zlib - inflateEnd(&zstream); + size_t total = 0; + Compression compression; + if (compression.open(Compression::Format::Binary)) { + total = compression.decompress((unsigned char *)reader->GetPtr(), reader->GetRemainingSize(), uncompressed); + compression.close(); + } // replace the input stream with a memory stream stream.reset(new MemoryIOStream(reinterpret_cast(uncompressed.data()), total)); diff --git a/code/AssetLib/FBX/FBXParser.cpp b/code/AssetLib/FBX/FBXParser.cpp index ef26f5322..1cbe8e833 100644 --- a/code/AssetLib/FBX/FBXParser.cpp +++ b/code/AssetLib/FBX/FBXParser.cpp @@ -46,11 +46,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef ASSIMP_BUILD_NO_FBX_IMPORTER -#ifdef ASSIMP_BUILD_NO_OWN_ZLIB -# include +//#ifdef ASSIMP_BUILD_NO_OWN_ZLIB +#include "Common/Compression.h" +/*# include #else # include "../contrib/zlib/zlib.h" -#endif +#endif*/ #include "FBXTokenizer.h" #include "FBXParser.h" @@ -571,8 +572,11 @@ void ReadBinaryDataArray(char type, uint32_t count, const char*& data, const cha else if(encmode == 1) { // zlib/deflate, next comes ZIP head (0x78 0x01) // see http://www.ietf.org/rfc/rfc1950.txt - - z_stream zstream; + Compression compress; + if (compress.open(Compression::Format::Binary)) { + compress.decompress(data, comp_len, buff); + } + /* z_stream zstream; zstream.opaque = Z_NULL; zstream.zalloc = Z_NULL; zstream.zfree = Z_NULL; @@ -581,9 +585,9 @@ void ReadBinaryDataArray(char type, uint32_t count, const char*& data, const cha // http://hewgill.com/journal/entries/349-how-to-decompress-gzip-stream-with-zlib if(Z_OK != inflateInit(&zstream)) { ParseError("failure initializing zlib"); - } + }*/ - zstream.next_in = reinterpret_cast( const_cast(data) ); + /* zstream.next_in = reinterpret_cast(const_cast(data)); zstream.avail_in = comp_len; zstream.avail_out = static_cast(buff.size()); @@ -595,7 +599,7 @@ void ReadBinaryDataArray(char type, uint32_t count, const char*& data, const cha } // terminate zlib - inflateEnd(&zstream); + inflateEnd(&zstream);*/ } #ifdef ASSIMP_BUILD_DEBUG else { diff --git a/code/AssetLib/X/XFileParser.cpp b/code/AssetLib/X/XFileParser.cpp index adcb7c8f4..e867c3f11 100644 --- a/code/AssetLib/X/XFileParser.cpp +++ b/code/AssetLib/X/XFileParser.cpp @@ -60,11 +60,12 @@ using namespace Assimp::Formatter; #ifndef ASSIMP_BUILD_NO_COMPRESSED_X -#ifdef ASSIMP_BUILD_NO_OWN_ZLIB +/* #ifdef ASSIMP_BUILD_NO_OWN_ZLIB #include #else #include "../contrib/zlib/zlib.h" -#endif +#endif*/ +#include "Common/Compression.h" // Magic identifier for MSZIP compressed data #define MSZIP_MAGIC 0x4B43 @@ -72,13 +73,13 @@ using namespace Assimp::Formatter; // ------------------------------------------------------------------------------------------------ // Dummy memory wrappers for use with zlib -static void *dummy_alloc(void * /*opaque*/, unsigned int items, unsigned int size) { +/* static void *dummy_alloc(void * opaque, unsigned int items, unsigned int size) { return ::operator new(items *size); } -static void dummy_free(void * /*opaque*/, void *address) { +static void dummy_free(void * opaque, void *address) { return ::operator delete(address); -} +}*/ #endif // !! ASSIMP_BUILD_NO_COMPRESSED_X @@ -171,15 +172,16 @@ XFileParser::XFileParser(const std::vector &pBuffer) : * /////////////////////////////////////////////////////////////////////// */ + Compression compression; // build a zlib stream - z_stream stream; + /* z_stream stream; stream.opaque = nullptr; stream.zalloc = &dummy_alloc; stream.zfree = &dummy_free; stream.data_type = (mIsBinaryFormat ? Z_BINARY : Z_ASCII); // initialize the inflation algorithm - ::inflateInit2(&stream, -MAX_WBITS); + ::inflateInit2(&stream, -MAX_WBITS);*/ // skip unknown data (checksum, flags?) mP += 6; @@ -213,7 +215,12 @@ XFileParser::XFileParser(const std::vector &pBuffer) : // Allocate storage and terminating zero and do the actual uncompressing uncompressed.resize(est_out + 1); char *out = &uncompressed.front(); - while (mP + 3 < mEnd) { + + if (compression.open(mIsBinaryFormat ? Compression::Format::Binary : Compression::Format::ASCII)) { + compression.decompress(mP, std::distance(mP, mEnd-3), uncompressed); + compression.close(); + } + /* while (mP + 3 < mEnd) { uint16_t ofs = *((uint16_t *)mP); AI_SWAP2(ofs); mP += 4; @@ -242,7 +249,7 @@ XFileParser::XFileParser(const std::vector &pBuffer) : } // terminate zlib - ::inflateEnd(&stream); + ::inflateEnd(&stream);*/ // ok, update pointers to point to the uncompressed file data mP = &uncompressed[0]; diff --git a/code/AssetLib/XGL/XGLLoader.cpp b/code/AssetLib/XGL/XGLLoader.cpp index a711598f9..6c92f03b2 100644 --- a/code/AssetLib/XGL/XGLLoader.cpp +++ b/code/AssetLib/XGL/XGLLoader.cpp @@ -112,7 +112,7 @@ const aiImporterDesc *XGLImporter::GetInfo() const { // Imports the given file into the given scene structure. void XGLImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSystem *pIOHandler) { #ifndef ASSIMP_BUILD_NO_COMPRESSED_XGL - std::vector uncompressed; + std::vector uncompressed; #endif m_scene = pScene; @@ -132,7 +132,7 @@ void XGLImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy Compression c; size_t total = 0l; - if (c.open()) { + if (c.open(Compression::Format::Binary)) { // skip two extra bytes, zgl files do carry a crc16 upfront (I think) raw_reader->IncPtr(2); total = c.decompress((unsigned char *)raw_reader->GetPtr(), raw_reader->GetRemainingSize(), uncompressed); diff --git a/code/Common/Compression.cpp b/code/Common/Compression.cpp index 2e3defa8c..e5d1323b7 100644 --- a/code/Common/Compression.cpp +++ b/code/Common/Compression.cpp @@ -70,7 +70,7 @@ Compression::~Compression() { delete mImpl; } -bool Compression::open() { +bool Compression::open(Format format) { ai_assert(mImpl != nullptr); if (mImpl->mOpen) { @@ -81,7 +81,11 @@ bool Compression::open() { mImpl->mZSstream.opaque = Z_NULL; mImpl->mZSstream.zalloc = Z_NULL; mImpl->mZSstream.zfree = Z_NULL; - mImpl->mZSstream.data_type = Z_BINARY; + if (format == Format::Binary) { + mImpl->mZSstream.data_type = Z_BINARY; + } else { + mImpl->mZSstream.data_type = Z_ASCII; + } // raw decompression without a zlib or gzip header inflateInit2(&mImpl->mZSstream, -MAX_WBITS); @@ -90,12 +94,12 @@ bool Compression::open() { return mImpl->mOpen; } -constexpr size_t MYBLOCK = 1024; +constexpr size_t MYBLOCK = 32786; -size_t Compression::decompress(unsigned char *data, size_t in, std::vector &uncompressed) { +size_t Compression::decompress(const void *data, size_t in, std::vector &uncompressed) { ai_assert(mImpl != nullptr); - mImpl->mZSstream.next_in = reinterpret_cast(data); + mImpl->mZSstream.next_in = (Bytef*)(data); mImpl->mZSstream.avail_in = (uInt)in; Bytef block[MYBLOCK] = {}; diff --git a/code/Common/Compression.h b/code/Common/Compression.h index 20c92cb60..cb5c00fbd 100644 --- a/code/Common/Compression.h +++ b/code/Common/Compression.h @@ -48,6 +48,13 @@ namespace Assimp { /// @brief This class provides the decompression of zlib-compressed data. class Compression { public: + enum class Format { + Binary = 0, + ASCII, + + NumFormats, + InvalidFormat + }; /// @brief The class constructor. Compression(); @@ -56,7 +63,7 @@ public: /// @brief Will open the access to the compression. /// @return true if close was successful, false if not. - bool open(); + bool open(Format format); /// @brief Will return the open state. /// @return true if the access is opened, false if not. @@ -70,7 +77,7 @@ public: /// @param[in] data The data to decompress /// @param[in] in The size of the data. /// @param[out uncompressed A std::vector containing the decompressed data. - size_t decompress(unsigned char *data, size_t in, std::vector &uncompressed); + size_t decompress(const void *data, size_t in, std::vector &uncompressed); private: struct impl;