From 9335cc30abf5a55a3df7fb99827c23b74b273f96 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sun, 6 Feb 2022 20:42:58 +0100 Subject: [PATCH 1/2] INtroduce compression class to encapsulate compression via zlib --- code/AssetLib/Assbin/AssbinFileWriter.cpp | 4 +- code/AssetLib/XGL/XGLLoader.cpp | 69 ++++------- code/AssetLib/XGL/XGLLoader.h | 2 +- code/CMakeLists.txt | 2 + code/Common/Base64.cpp | 30 +++-- code/Common/Compression.cpp | 141 ++++++++++++++++++++++ code/Common/Compression.h | 80 ++++++++++++ code/Common/IOSystem.cpp | 2 +- code/Common/material.cpp | 1 - code/Common/simd.cpp | 2 - include/assimp/Base64.hpp | 4 + 11 files changed, 270 insertions(+), 67 deletions(-) create mode 100644 code/Common/Compression.cpp create mode 100644 code/Common/Compression.h diff --git a/code/AssetLib/Assbin/AssbinFileWriter.cpp b/code/AssetLib/Assbin/AssbinFileWriter.cpp index 72fbfdcb1..1d16f179e 100644 --- a/code/AssetLib/Assbin/AssbinFileWriter.cpp +++ b/code/AssetLib/Assbin/AssbinFileWriter.cpp @@ -182,6 +182,8 @@ inline size_t Write(IOStream *stream, const aiVertexWeight &v) { return t + Write(stream, v.mWeight); } +constexpr size_t MatrixSize = 64; + // ----------------------------------------------------------------------------------- // Serialize a mat4x4 template <> @@ -192,7 +194,7 @@ inline size_t Write(IOStream *stream, const aiMatrix4x4 &m) { } } - return 64; + return MatrixSize; } // ----------------------------------------------------------------------------------- diff --git a/code/AssetLib/XGL/XGLLoader.cpp b/code/AssetLib/XGL/XGLLoader.cpp index b003f852a..8932d6d41 100644 --- a/code/AssetLib/XGL/XGLLoader.cpp +++ b/code/AssetLib/XGL/XGLLoader.cpp @@ -44,9 +44,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef ASSIMP_BUILD_NO_XGL_IMPORTER #include "XGLLoader.h" +#include "Common/Compression.h" + #include #include - #include #include #include @@ -73,6 +74,7 @@ const char *LogFunctions::Prefix() { static auto prefix = "XGL: "; return prefix; } + } // namespace Assimp static const aiImporterDesc desc = { @@ -137,46 +139,14 @@ void XGLImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy #else std::unique_ptr raw_reader(new StreamReaderLE(stream)); - // build a zlib stream - z_stream zstream; - zstream.opaque = Z_NULL; - zstream.zalloc = Z_NULL; - zstream.zfree = Z_NULL; - zstream.data_type = Z_BINARY; - - // raw decompression without a zlib or gzip header - inflateInit2(&zstream, -MAX_WBITS); - - // skip two extra bytes, zgl files do carry a crc16 upfront (I think) - raw_reader->IncPtr(2); - - zstream.next_in = reinterpret_cast(raw_reader->GetPtr()); - zstream.avail_in = (uInt) raw_reader->GetRemainingSize(); - - size_t total = 0l; - - // TODO: be smarter about this, decompress directly into heap buffer - // and decompress the data .... do 1k chunks in the hope that we won't kill the stack -#define MYBLOCK 1024 - Bytef block[MYBLOCK]; - int ret; - do { - zstream.avail_out = MYBLOCK; - zstream.next_out = block; - ret = inflate(&zstream, Z_NO_FLUSH); - - if (ret != Z_STREAM_END && ret != Z_OK) { - ThrowException("Failure decompressing this file using gzip, seemingly it is NOT a compressed .XGL file"); - } - const size_t have = MYBLOCK - zstream.avail_out; - total += have; - uncompressed.resize(total); - memcpy(uncompressed.data() + total - have, block, have); - } while (ret != Z_STREAM_END); - - // terminate zlib - inflateEnd(&zstream); - + Compression c; + size_t total = 0l; + if (c.open()) { + // skip two extra bytes, zgl files do carry a crc16 upfront (I think) + raw_reader->IncPtr(2); + total = c.decompress((unsigned char *)raw_reader->GetPtr(), raw_reader->GetRemainingSize(), uncompressed); + c.close(); + } // replace the input stream with a memory stream stream.reset(new MemoryIOStream(reinterpret_cast(uncompressed.data()), total)); #endif @@ -291,7 +261,8 @@ aiNode *XGLImporter::ReadObject(XmlNode &node, TempScope &scope) { const std::string &s = ai_stdStrToLower(child.name()); if (s == "mesh") { const size_t prev = scope.meshes_linear.size(); - if (ReadMesh(child, scope)) { + bool empty; + if (ReadMesh(child, scope, empty)) { const size_t newc = scope.meshes_linear.size(); for (size_t i = 0; i < newc - prev; ++i) { meshes.push_back(static_cast(i + prev)); @@ -475,12 +446,12 @@ aiMesh *XGLImporter::ToOutputMesh(const TempMaterialMesh &m) { } // ------------------------------------------------------------------------------------------------ -bool XGLImporter::ReadMesh(XmlNode &node, TempScope &scope) { +bool XGLImporter::ReadMesh(XmlNode &node, TempScope &scope, bool &empty) { TempMesh t; std::map bymat; const unsigned int mesh_id = ReadIDAttr(node); - + bool empty_mesh = true; for (XmlNode &child : node.children()) { const std::string &s = ai_stdStrToLower(child.name()); @@ -539,6 +510,9 @@ bool XGLImporter::ReadMesh(XmlNode &node, TempScope &scope) { mid = ResolveMaterialRef(sub_child, scope); } } + if (has[0] || has[1] || has[2]) { + empty_mesh = false; + } if (mid == ~0u) { ThrowException("missing material index"); @@ -590,6 +564,11 @@ bool XGLImporter::ReadMesh(XmlNode &node, TempScope &scope) { scope.meshes.insert(std::pair(mesh_id, m)); } } + if (empty_mesh) { + LogWarn("Mesh is empty, skipping."); + empty = empty_mesh; + return false; + } // no id == not a reference, insert this mesh right *here* return mesh_id == ~0u; @@ -759,7 +738,7 @@ aiVector2D XGLImporter::ReadVec2(XmlNode &node) { std::string val; XmlParser::getValueAsString(node, val); const char *s = val.c_str(); - ai_real v[2]; + ai_real v[2] = {}; for (int i = 0; i < 2; ++i) { if (!SkipSpaces(&s)) { LogError("unexpected EOL, failed to parse vec2"); diff --git a/code/AssetLib/XGL/XGLLoader.h b/code/AssetLib/XGL/XGLLoader.h index 903f114cf..ae7ccddc2 100644 --- a/code/AssetLib/XGL/XGLLoader.h +++ b/code/AssetLib/XGL/XGLLoader.h @@ -186,7 +186,7 @@ private: void ReadLighting(XmlNode &node, TempScope &scope); aiLight *ReadDirectionalLight(XmlNode &node); aiNode *ReadObject(XmlNode &node, TempScope &scope); - bool ReadMesh(XmlNode &node, TempScope &scope); + bool ReadMesh(XmlNode &node, TempScope &scope, bool &empty); void ReadMaterial(XmlNode &node, TempScope &scope); aiVector2D ReadVec2(XmlNode &node); aiVector3D ReadVec3(XmlNode &node); diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index 7ce933585..5936d6d06 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -166,6 +166,8 @@ SET( Logging_SRCS SOURCE_GROUP(Logging FILES ${Logging_SRCS}) SET( Common_SRCS + Common/Compression.cpp + Common/Compression.h Common/BaseImporter.cpp Common/BaseProcess.cpp Common/BaseProcess.h diff --git a/code/Common/Base64.cpp b/code/Common/Base64.cpp index 2916cae7f..3e9c47405 100644 --- a/code/Common/Base64.cpp +++ b/code/Common/Base64.cpp @@ -3,7 +3,7 @@ Open Asset Import Library (assimp) --------------------------------------------------------------------------- -Copyright (c) 2006-2021, assimp team +Copyright (c) 2006-2022, assimp team All rights reserved. @@ -57,7 +57,7 @@ static const uint8_t tableDecodeBase64[128] = { 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0 }; -static const char* tableEncodeBase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; +static const char *tableEncodeBase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; static inline char EncodeChar(uint8_t b) { return tableEncodeBase64[size_t(b)]; @@ -104,17 +104,16 @@ void Encode(const uint8_t *in, size_t inLength, std::string &out) { } } -void Encode(const std::vector& in, std::string &out) { - Encode (in.data (), in.size (), out); +void Encode(const std::vector &in, std::string &out) { + Encode(in.data(), in.size(), out); } -std::string Encode (const std::vector& in) { +std::string Encode(const std::vector &in) { std::string encoded; - Encode (in, encoded); + Encode(in, encoded); return encoded; } - size_t Decode(const char *in, size_t inLength, uint8_t *&out) { if (inLength % 4 != 0) { throw DeadlyImportError("Invalid base64 encoded data: \"", std::string(in, std::min(size_t(32), inLength)), "\", length:", inLength); @@ -159,23 +158,22 @@ size_t Decode(const char *in, size_t inLength, uint8_t *&out) { return outLength; } -size_t Decode(const std::string& in, std::vector& out) { - uint8_t* outPtr = nullptr; - size_t decodedSize = Decode (in.data (), in.size (), outPtr); +size_t Decode(const std::string &in, std::vector &out) { + uint8_t *outPtr = nullptr; + size_t decodedSize = Decode(in.data(), in.size(), outPtr); if (outPtr == nullptr) { return 0; } - out.assign (outPtr, outPtr + decodedSize); + out.assign(outPtr, outPtr + decodedSize); delete[] outPtr; return decodedSize; } -std::vector Decode (const std::string& in) { +std::vector Decode(const std::string &in) { std::vector result; - Decode (in, result); + Decode(in, result); return result; } -} - -} +} // namespace Base64 +} // namespace Assimp diff --git a/code/Common/Compression.cpp b/code/Common/Compression.cpp new file mode 100644 index 000000000..2e3defa8c --- /dev/null +++ b/code/Common/Compression.cpp @@ -0,0 +1,141 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2022, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior + written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ + +#include "Compression.h" +#include +#include + +#ifdef ASSIMP_BUILD_NO_OWN_ZLIB +#include +#else +#include "../contrib/zlib/zlib.h" +#endif + +namespace Assimp { + +struct Compression::impl { + bool mOpen; + z_stream mZSstream; + + impl() : + mOpen(false) {} +}; + +Compression::Compression() : + mImpl(new impl) { + // empty +} + +Compression::~Compression() { + ai_assert(mImpl != nullptr); + + delete mImpl; +} + +bool Compression::open() { + ai_assert(mImpl != nullptr); + + if (mImpl->mOpen) { + return false; + } + + // build a zlib stream + mImpl->mZSstream.opaque = Z_NULL; + mImpl->mZSstream.zalloc = Z_NULL; + mImpl->mZSstream.zfree = Z_NULL; + mImpl->mZSstream.data_type = Z_BINARY; + + // raw decompression without a zlib or gzip header + inflateInit2(&mImpl->mZSstream, -MAX_WBITS); + mImpl->mOpen = true; + + return mImpl->mOpen; +} + +constexpr size_t MYBLOCK = 1024; + +size_t Compression::decompress(unsigned char *data, size_t in, std::vector &uncompressed) { + ai_assert(mImpl != nullptr); + + mImpl->mZSstream.next_in = reinterpret_cast(data); + mImpl->mZSstream.avail_in = (uInt)in; + + Bytef block[MYBLOCK] = {}; + int ret = 0; + size_t total = 0l; + do { + mImpl->mZSstream.avail_out = MYBLOCK; + mImpl->mZSstream.next_out = block; + ret = inflate(&mImpl->mZSstream, Z_NO_FLUSH); + + if (ret != Z_STREAM_END && ret != Z_OK) { + throw DeadlyImportError("Compression", "Failure decompressing this file using gzip."); + + } + const size_t have = MYBLOCK - mImpl->mZSstream.avail_out; + total += have; + uncompressed.resize(total); + ::memcpy(uncompressed.data() + total - have, block, have); + } while (ret != Z_STREAM_END); + + return total; +} + +bool Compression::isOpen() const { + ai_assert(mImpl != nullptr); + + return mImpl->mOpen; +} + +bool Compression::close() { + ai_assert(mImpl != nullptr); + + if (!mImpl->mOpen) { + return false; + } + + inflateEnd(&mImpl->mZSstream); + mImpl->mOpen = false; + + return true; +} + +} // namespace Assimp diff --git a/code/Common/Compression.h b/code/Common/Compression.h new file mode 100644 index 000000000..20c92cb60 --- /dev/null +++ b/code/Common/Compression.h @@ -0,0 +1,80 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2022, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior + written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ + +#pragma once + +#include + +namespace Assimp { + +/// @brief This class provides the decompression of zlib-compressed data. +class Compression { +public: + /// @brief The class constructor. + Compression(); + + /// @brief The class destructor. + ~Compression(); + + /// @brief Will open the access to the compression. + /// @return true if close was successful, false if not. + bool open(); + + /// @brief Will return the open state. + /// @return true if the access is opened, false if not. + bool isOpen() const; + + /// @brief Will close the decompress access. + /// @return true if close was successful, false if not. + bool close(); + + /// @brief Will decompress the data buffer. + /// @param[in] data The data to decompress + /// @param[in] in The size of the data. + /// @param[out uncompressed A std::vector containing the decompressed data. + size_t decompress(unsigned char *data, size_t in, std::vector &uncompressed); + +private: + struct impl; + impl *mImpl; +}; + +} // namespace Assimp diff --git a/code/Common/IOSystem.cpp b/code/Common/IOSystem.cpp index 77305d193..1e63827ba 100644 --- a/code/Common/IOSystem.cpp +++ b/code/Common/IOSystem.cpp @@ -3,7 +3,7 @@ Open Asset Import Library (assimp) --------------------------------------------------------------------------- -Copyright (c) 2006-2019, assimp team +Copyright (c) 2006-2022, assimp team All rights reserved. diff --git a/code/Common/material.cpp b/code/Common/material.cpp index 8f791a313..a7541d4f1 100644 --- a/code/Common/material.cpp +++ b/code/Common/material.cpp @@ -4,7 +4,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2022, assimp team - All rights reserved. Redistribution and use of this software in source and binary forms, diff --git a/code/Common/simd.cpp b/code/Common/simd.cpp index 9eb547f9a..0dd437d26 100644 --- a/code/Common/simd.cpp +++ b/code/Common/simd.cpp @@ -5,8 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2022, assimp team - - All rights reserved. Redistribution and use of this software in source and binary forms, diff --git a/include/assimp/Base64.hpp b/include/assimp/Base64.hpp index 894eb449b..ee319aceb 100644 --- a/include/assimp/Base64.hpp +++ b/include/assimp/Base64.hpp @@ -50,6 +50,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace Assimp { namespace Base64 { +/// @brief Will encode the given +/// @param in +/// @param inLength +/// @param out void Encode(const uint8_t *in, size_t inLength, std::string &out); void Encode(const std::vector& in, std::string &out); std::string Encode(const std::vector& in); From eb5a7938e03862c9bbcb77dec6ac7032b26c97e8 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sun, 6 Feb 2022 21:16:04 +0100 Subject: [PATCH 2/2] Remove dead code --- code/AssetLib/XGL/XGLLoader.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/code/AssetLib/XGL/XGLLoader.cpp b/code/AssetLib/XGL/XGLLoader.cpp index 8932d6d41..a711598f9 100644 --- a/code/AssetLib/XGL/XGLLoader.cpp +++ b/code/AssetLib/XGL/XGLLoader.cpp @@ -58,15 +58,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. using namespace Assimp; -// zlib is needed for compressed XGL files -#ifndef ASSIMP_BUILD_NO_COMPRESSED_XGL -# ifdef ASSIMP_BUILD_NO_OWN_ZLIB -# include -# else -# include -# endif -#endif - namespace Assimp { // this has to be in here because LogFunctions is in ::Assimp template <> @@ -120,8 +111,8 @@ const aiImporterDesc *XGLImporter::GetInfo() const { // ------------------------------------------------------------------------------------------------ // Imports the given file into the given scene structure. void XGLImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSystem *pIOHandler) { -#ifndef ASSIMP_BUILD_NO_COMPRESSED_XGL - std::vector uncompressed; + #ifndef ASSIMP_BUILD_NO_COMPRESSED_XGL + std::vector uncompressed; #endif m_scene = pScene;