From e7211790fb6307ae0e73a44a5529b709877d2f89 Mon Sep 17 00:00:00 2001 From: Tom spot Callaway Date: Mon, 3 May 2021 13:27:52 -0400 Subject: [PATCH 01/34] PBR material support --- code/AssetLib/FBX/FBXConverter.cpp | 53 ++++++++++++++++++++++++++++- code/AssetLib/FBX/FBXProperties.cpp | 2 +- include/assimp/material.h | 12 +++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/code/AssetLib/FBX/FBXConverter.cpp b/code/AssetLib/FBX/FBXConverter.cpp index cb033a651..177fa636c 100644 --- a/code/AssetLib/FBX/FBXConverter.cpp +++ b/code/AssetLib/FBX/FBXConverter.cpp @@ -2126,7 +2126,12 @@ void FBXConverter::SetShadingPropertiesCommon(aiMaterial *out_mat, const Propert const aiColor3D &Emissive = GetColorPropertyFromMaterial(props, "Emissive", ok); if (ok) { out_mat->AddProperty(&Emissive, 1, AI_MATKEY_COLOR_EMISSIVE); - } + } else { + const aiColor3D &emissiveColor = GetColorPropertyFromMaterial(props, "Maya|emissive", ok); + if (ok) { + out_mat->AddProperty(&emissiveColor, 1, AI_MATKEY_COLOR_EMISSIVE); + } + } const aiColor3D &Ambient = GetColorPropertyFromMaterial(props, "Ambient", ok); if (ok) { @@ -2207,6 +2212,52 @@ void FBXConverter::SetShadingPropertiesCommon(aiMaterial *out_mat, const Propert if (ok) { out_mat->AddProperty(&DispFactor, 1, "$mat.displacementscaling", 0, 0); } + + // PBR material information + const aiColor3D &baseColor = GetColorPropertyFromMaterial(props, "Maya|base_color", ok); + if (ok) { + out_mat->AddProperty(&baseColor, 1, AI_MATKEY_BASE_COLOR); + } + + const float useColorMap = PropertyGet(props, "Maya|use_color_map", ok); + if (ok) { + out_mat->AddProperty(&useColorMap, 1, AI_MATKEY_USE_COLOR_MAP); + } + + const float useMetallicMap = PropertyGet(props, "Maya|use_metallic_map", ok); + if (ok) { + out_mat->AddProperty(&useMetallicMap, 1, AI_MATKEY_USE_METALLIC_MAP); + } + + const float metallicFactor = PropertyGet(props, "Maya|metallic", ok); + if (ok) { + out_mat->AddProperty(&metallicFactor, 1, AI_MATKEY_METALLIC_FACTOR); + } + + const float useRoughnessMap = PropertyGet(props, "Maya|use_roughness_map", ok); + if (ok) { + out_mat->AddProperty(&useRoughnessMap, 1, AI_MATKEY_USE_ROUGHNESS_MAP); + } + + const float roughnessFactor = PropertyGet(props, "Maya|roughness", ok); + if (ok) { + out_mat->AddProperty(&roughnessFactor, 1, AI_MATKEY_ROUGHNESS_FACTOR); + } + + const float useEmissiveMap = PropertyGet(props, "Maya|use_emissive_map", ok); + if (ok) { + out_mat->AddProperty(&useEmissiveMap, 1, AI_MATKEY_USE_EMISSIVE_MAP); + } + + const float emissiveIntensity = PropertyGet(props, "Maya|emissive_intensity", ok); + if (ok) { + out_mat->AddProperty(&emissiveIntensity, 1, AI_MATKEY_EMISSIVE_INTENSITY); + } + + const float useAOMap = PropertyGet(props, "Maya|use_ao_map", ok); + if (ok) { + out_mat->AddProperty(&useAOMap, 1, AI_MATKEY_USE_AO_MAP); + } } void FBXConverter::SetShadingPropertiesRaw(aiMaterial *out_mat, const PropertyTable &props, const TextureMap &_textures, const MeshGeometry *const mesh) { diff --git a/code/AssetLib/FBX/FBXProperties.cpp b/code/AssetLib/FBX/FBXProperties.cpp index 1e4cd0ead..1a5ebffd1 100644 --- a/code/AssetLib/FBX/FBXProperties.cpp +++ b/code/AssetLib/FBX/FBXProperties.cpp @@ -131,7 +131,7 @@ Property* ReadTypedProperty(const Element& element) ParseTokenAsFloat(*tok[6])) ); } - else if (!strcmp(cs,"double") || !strcmp(cs,"Number") || !strcmp(cs,"Float") || !strcmp(cs,"FieldOfView") || !strcmp( cs, "UnitScaleFactor" ) ) { + else if (!strcmp(cs,"double") || !strcmp(cs,"Number") || !strcmp(cs,"float") || !strcmp(cs,"Float") || !strcmp(cs,"FieldOfView") || !strcmp( cs, "UnitScaleFactor" ) ) { checkTokenCount(tok, 5); return new TypedProperty(ParseTokenAsFloat(*tok[4])); } diff --git a/include/assimp/material.h b/include/assimp/material.h index f3daa62dc..08c0491c0 100644 --- a/include/assimp/material.h +++ b/include/assimp/material.h @@ -920,6 +920,18 @@ extern "C" { #define AI_MATKEY_SHADER_PRIMITIVE "?sh.ps", 0, 0 #define AI_MATKEY_SHADER_COMPUTE "?sh.cs", 0, 0 +// --------------------------------------------------------------------------- +// PBR material support +#define AI_MATKEY_USE_COLOR_MAP "$mat.useColorMap", 0, 0 +#define AI_MATKEY_BASE_COLOR "$clr.base", 0, 0 +#define AI_MATKEY_USE_METALLIC_MAP "$mat.useMetallicMap", 0, 0 +#define AI_MATKEY_METALLIC_FACTOR "$mat.metallicFactor", 0, 0 +#define AI_MATKEY_USE_ROUGHNESS_MAP "$mat.useRoughnessMap", 0, 0 +#define AI_MATKEY_ROUGHNESS_FACTOR "$mat.roughnessFactor", 0, 0 +#define AI_MATKEY_USE_EMISSIVE_MAP "$mat.useEmissiveMap", 0, 0 +#define AI_MATKEY_EMISSIVE_INTENSITY "$mat.emissiveIntensity", 0, 0 +#define AI_MATKEY_USE_AO_MAP "$mat.useAOMap", 0, 0 + // --------------------------------------------------------------------------- // Pure key names for all texture-related properties //! @cond MATS_DOC_FULL From f91b439f79cc106f08678a265562191c67b184cf Mon Sep 17 00:00:00 2001 From: Tom spot Callaway Date: Mon, 3 May 2021 13:40:31 -0400 Subject: [PATCH 02/34] preserve UV Stream names in FBX files --- code/AssetLib/Assxml/AssxmlFileWriter.cpp | 7 +++++-- code/AssetLib/FBX/FBXConverter.cpp | 2 ++ include/assimp/mesh.h | 4 ++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/code/AssetLib/Assxml/AssxmlFileWriter.cpp b/code/AssetLib/Assxml/AssxmlFileWriter.cpp index 6d876b3aa..24fda5955 100644 --- a/code/AssetLib/Assxml/AssxmlFileWriter.cpp +++ b/code/AssetLib/Assxml/AssxmlFileWriter.cpp @@ -598,8 +598,11 @@ static void WriteDump(const char *pFile, const char *cmd, const aiScene *scene, if (!mesh->mTextureCoords[a]) break; - ioprintf(io, "\t\t \n", mesh->mNumVertices, - a, mesh->mNumUVComponents[a]); + ioprintf(io, "\t\t \n", + mesh->mNumVertices, + a, + mesh->mTextureCoordsNames[a].C_Str(), + mesh->mNumUVComponents[a]); if (!shortened) { if (mesh->mNumUVComponents[a] == 3) { diff --git a/code/AssetLib/FBX/FBXConverter.cpp b/code/AssetLib/FBX/FBXConverter.cpp index cb033a651..4778f9b07 100644 --- a/code/AssetLib/FBX/FBXConverter.cpp +++ b/code/AssetLib/FBX/FBXConverter.cpp @@ -1126,6 +1126,8 @@ unsigned int FBXConverter::ConvertMeshSingleMaterial(const MeshGeometry &mesh, c *out_uv++ = aiVector3D(v.x, v.y, 0.0f); } + out_mesh->mTextureCoordsNames[i] = mesh.GetTextureCoordChannelName(i); + out_mesh->mNumUVComponents[i] = 2; } diff --git a/include/assimp/mesh.h b/include/assimp/mesh.h index 427dba008..989ed3800 100644 --- a/include/assimp/mesh.h +++ b/include/assimp/mesh.h @@ -674,6 +674,10 @@ struct aiMesh { */ C_STRUCT aiVector3D *mTextureCoords[AI_MAX_NUMBER_OF_TEXTURECOORDS]; + /** Vertex stream names. + */ + C_STRUCT aiString mTextureCoordsNames[AI_MAX_NUMBER_OF_TEXTURECOORDS]; + /** Specifies the number of components for a given UV channel. * Up to three channels are supported (UVW, for accessing volume * or cube maps). If the value is 2 for a given channel n, the From 8ad9c937f171bd716bbd5692af1c501acbaded88 Mon Sep 17 00:00:00 2001 From: Krishty Date: Tue, 4 May 2021 19:10:24 +0200 Subject: [PATCH 03/34] enabled debug information in MSVC release build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No effect on runtime speed/size. Slightly slower link time, but debugging experience improves by a million times. - /Zi – Store debug information in a .pdb file, not directly in the DLL/EXE - /DEBUG:FULL – generate debug information during link - /PDBALTPATH:%_PDB% – do not store the file system path of the .pdb, just the filename and hash (no disclose paths on distribution) - /OPT:REF /OPT:ICF – remove unreferenced functions and fold identical functions (this was enabled before, but requires explicit enabling if /DEBUG:FULL is specified) --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 97a3641f5..f1b60936d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -268,6 +268,8 @@ ELSEIF(MSVC) ADD_COMPILE_OPTIONS(/wd4351) ENDIF() SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /D_DEBUG /Zi /Od") + SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi") + SET(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG:FULL /PDBALTPATH:%_PDB% /OPT:REF /OPT:ICF") ELSEIF (CMAKE_CXX_COMPILER_ID MATCHES "Clang" ) IF(NOT ASSIMP_HUNTER_ENABLED) SET(CMAKE_CXX_STANDARD 11) From 2a126f9f62a8174448a5ac31d94a3534eefd909f Mon Sep 17 00:00:00 2001 From: Krishty Date: Mon, 3 May 2021 21:46:53 +0200 Subject: [PATCH 04/34] reduced Ogre string bloat The Ogre importer used std::string where a string literal would have been sufficient. Saves another 600 B of code and data. --- code/AssetLib/Ogre/OgreBinarySerializer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/AssetLib/Ogre/OgreBinarySerializer.cpp b/code/AssetLib/Ogre/OgreBinarySerializer.cpp index 0fc18feb9..68b1cf1ed 100644 --- a/code/AssetLib/Ogre/OgreBinarySerializer.cpp +++ b/code/AssetLib/Ogre/OgreBinarySerializer.cpp @@ -55,9 +55,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace Assimp { namespace Ogre { -const std::string MESH_VERSION_1_8 = "[MeshSerializer_v1.8]"; -const std::string SKELETON_VERSION_1_8 = "[Serializer_v1.80]"; -const std::string SKELETON_VERSION_1_1 = "[Serializer_v1.10]"; +static constexpr auto MESH_VERSION_1_8 = "[MeshSerializer_v1.8]"; +static constexpr auto SKELETON_VERSION_1_8 = "[Serializer_v1.80]"; +static constexpr auto SKELETON_VERSION_1_1 = "[Serializer_v1.10]"; const unsigned short HEADER_CHUNK_ID = 0x1000; From 7b6dab5e209257a17b4ca3bcc7834e2c8cc402b1 Mon Sep 17 00:00:00 2001 From: Krishty Date: Mon, 3 May 2021 21:52:48 +0200 Subject: [PATCH 05/34] reduced DXF string bloat The DXF importer defined a global std::string constant, only to convert it back to a C string on use. This commit defines the constant as a C string right away, thus saving 340 B of code and data. --- code/AssetLib/DXF/DXFLoader.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/code/AssetLib/DXF/DXFLoader.cpp b/code/AssetLib/DXF/DXFLoader.cpp index d4a6be4ad..b42d497f4 100644 --- a/code/AssetLib/DXF/DXFLoader.cpp +++ b/code/AssetLib/DXF/DXFLoader.cpp @@ -63,11 +63,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. using namespace Assimp; // AutoCAD Binary DXF -const std::string AI_DXF_BINARY_IDENT = std::string("AutoCAD Binary DXF\r\n\x1a\0"); -const size_t AI_DXF_BINARY_IDENT_LEN = 24u; +static constexpr char AI_DXF_BINARY_IDENT[] = "AutoCAD Binary DXF\r\n\x1a"; +static constexpr size_t AI_DXF_BINARY_IDENT_LEN = sizeof AI_DXF_BINARY_IDENT; // default vertex color that all uncolored vertices will receive -const aiColor4D AI_DXF_DEFAULT_COLOR(aiColor4D(0.6f, 0.6f, 0.6f, 0.6f)); +static const aiColor4D AI_DXF_DEFAULT_COLOR(aiColor4D(0.6f, 0.6f, 0.6f, 0.6f)); // color indices for DXF - 16 are supported, the table is // taken directly from the DXF spec. @@ -156,10 +156,10 @@ void DXFImporter::InternReadFile( const std::string& filename, aiScene* pScene, } // Check whether this is a binary DXF file - we can't read binary DXF files :-( - char buff[AI_DXF_BINARY_IDENT_LEN+1] = {0}; + char buff[AI_DXF_BINARY_IDENT_LEN] = {0}; file->Read(buff,AI_DXF_BINARY_IDENT_LEN,1); - if (0 == strncmp(AI_DXF_BINARY_IDENT.c_str(),buff,AI_DXF_BINARY_IDENT_LEN)) { + if (0 == memcmp(AI_DXF_BINARY_IDENT,buff,AI_DXF_BINARY_IDENT_LEN)) { throw DeadlyImportError("DXF: Binary files are not supported at the moment"); } @@ -549,7 +549,7 @@ void DXFImporter::ParseEntities(DXF::LineReader& reader, DXF::FileData& output) ++reader; } - ASSIMP_LOG_VERBOSE_DEBUG_F( "DXF: got ", block.lines.size()," polylines and ", block.insertions.size(), + ASSIMP_LOG_VERBOSE_DEBUG_F( "DXF: got ", block.lines.size()," polylines and ", block.insertions.size(), " inserted blocks in ENTITIES" ); } From f3c18556d1331e43ac5828fa82bf10fee966df96 Mon Sep 17 00:00:00 2001 From: Krishty Date: Tue, 4 May 2021 22:03:44 +0200 Subject: [PATCH 06/34] reduced OpenGEX string bloat The OpenGEX importer defined a few global std::string constants, only to convert them back to C strings on use. This commit defines them as C strings from the beginning. strncmp() was used to compare these strings to other strings, but the length limit was set to string length, which made it equivalent to strcmp(), just slower. Fixed that as well. --- code/AssetLib/OpenGEX/OpenGEXImporter.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/code/AssetLib/OpenGEX/OpenGEXImporter.cpp b/code/AssetLib/OpenGEX/OpenGEXImporter.cpp index b29aeeeb1..7ee278521 100644 --- a/code/AssetLib/OpenGEX/OpenGEXImporter.cpp +++ b/code/AssetLib/OpenGEX/OpenGEXImporter.cpp @@ -735,22 +735,22 @@ enum MeshAttribute { TexCoord }; -static const std::string PosToken = "position"; -static const std::string ColToken = "color"; -static const std::string NormalToken = "normal"; -static const std::string TexCoordToken = "texcoord"; +constexpr auto PosToken = "position"; +constexpr auto ColToken = "color"; +constexpr auto NormalToken = "normal"; +constexpr auto TexCoordToken = "texcoord"; //------------------------------------------------------------------------------------------------ static MeshAttribute getAttributeByName(const char *attribName) { ai_assert(nullptr != attribName); - if (0 == strncmp(PosToken.c_str(), attribName, PosToken.size())) { + if (0 == strcmp(PosToken, attribName)) { return Position; - } else if (0 == strncmp(ColToken.c_str(), attribName, ColToken.size())) { + } else if (0 == strcmp(ColToken, attribName)) { return Color; - } else if (0 == strncmp(NormalToken.c_str(), attribName, NormalToken.size())) { + } else if (0 == strcmp(NormalToken, attribName)) { return Normal; - } else if (0 == strncmp(TexCoordToken.c_str(), attribName, TexCoordToken.size())) { + } else if (0 == strcmp(TexCoordToken, attribName)) { return TexCoord; } From b57ce004f805f133cf731311c3c8a6f08b0d7db9 Mon Sep 17 00:00:00 2001 From: Krishty Date: Tue, 4 May 2021 09:45:26 +0200 Subject: [PATCH 07/34] reduced FBX string bloat The FBX importer used two std::strings where string literals would have been sufficient. --- code/AssetLib/FBX/FBXMeshGeometry.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/AssetLib/FBX/FBXMeshGeometry.cpp b/code/AssetLib/FBX/FBXMeshGeometry.cpp index 2bca8dff2..4f87c5a2c 100644 --- a/code/AssetLib/FBX/FBXMeshGeometry.cpp +++ b/code/AssetLib/FBX/FBXMeshGeometry.cpp @@ -604,15 +604,15 @@ void MeshGeometry::ReadVertexDataTangents(std::vector& tangents_out, } // ------------------------------------------------------------------------------------------------ -static const std::string BinormalIndexToken = "BinormalIndex"; -static const std::string BinormalsIndexToken = "BinormalsIndex"; +static const char * BinormalIndexToken = "BinormalIndex"; +static const char * BinormalsIndexToken = "BinormalsIndex"; void MeshGeometry::ReadVertexDataBinormals(std::vector& binormals_out, const Scope& source, const std::string& MappingInformationType, const std::string& ReferenceInformationType) { const char * str = source.Elements().count( "Binormals" ) > 0 ? "Binormals" : "Binormal"; - const char * strIdx = source.Elements().count( "Binormals" ) > 0 ? BinormalsIndexToken.c_str() : BinormalIndexToken.c_str(); + const char * strIdx = source.Elements().count( "Binormals" ) > 0 ? BinormalsIndexToken : BinormalIndexToken; ResolveVertexDataArray(binormals_out,source,MappingInformationType,ReferenceInformationType, str, strIdx, From 153b890b02efaea35f5a73d3bb10f7576bce57a1 Mon Sep 17 00:00:00 2001 From: "Max Vollmer (Microsoft Havok)" <60260460+ms-maxvollmer@users.noreply.github.com> Date: Wed, 5 May 2021 14:09:43 +0100 Subject: [PATCH 08/34] Prevent accessing nullpointers --- code/AssetLib/glTF2/glTF2Importer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/AssetLib/glTF2/glTF2Importer.cpp b/code/AssetLib/glTF2/glTF2Importer.cpp index f34b1b451..e6265946b 100644 --- a/code/AssetLib/glTF2/glTF2Importer.cpp +++ b/code/AssetLib/glTF2/glTF2Importer.cpp @@ -1263,7 +1263,7 @@ aiMeshMorphAnim *CreateMeshMorphAnim(glTF2::Asset&, Node &node, AnimationSampler static const float kMillisecondsFromSeconds = 1000.f; - if (nullptr != samplers.weight) { + if (samplers.weight && samplers.weight->input && samplers.weight->output) { float *times = nullptr; samplers.weight->input->ExtractData(times); float *values = nullptr; From 2b9d88c9434b13a3c0824bb5b402eb2e83857b0b Mon Sep 17 00:00:00 2001 From: ywang Date: Thu, 6 May 2021 15:10:06 -0700 Subject: [PATCH 09/34] support basis universal --- code/AssetLib/glTF2/glTF2Asset.h | 2 ++ code/AssetLib/glTF2/glTF2Asset.inl | 2 ++ code/AssetLib/glTF2/glTF2AssetWriter.inl | 13 +++++++++ code/AssetLib/glTF2/glTF2Exporter.cpp | 36 ++++++++++++++++++++---- code/AssetLib/glTF2/glTF2Importer.cpp | 6 ++++ 5 files changed, 53 insertions(+), 6 deletions(-) diff --git a/code/AssetLib/glTF2/glTF2Asset.h b/code/AssetLib/glTF2/glTF2Asset.h index fad5cba83..f2c0369d6 100644 --- a/code/AssetLib/glTF2/glTF2Asset.h +++ b/code/AssetLib/glTF2/glTF2Asset.h @@ -1118,11 +1118,13 @@ public: bool KHR_materials_transmission; bool KHR_draco_mesh_compression; bool FB_ngon_encoding; + bool KHR_texture_basisu; } extensionsUsed; //! Keeps info about the required extensions struct RequiredExtensions { bool KHR_draco_mesh_compression; + bool KHR_texture_basisu; } extensionsRequired; AssetMetadata asset; diff --git a/code/AssetLib/glTF2/glTF2Asset.inl b/code/AssetLib/glTF2/glTF2Asset.inl index 77537028f..42d3f060e 100644 --- a/code/AssetLib/glTF2/glTF2Asset.inl +++ b/code/AssetLib/glTF2/glTF2Asset.inl @@ -1121,6 +1121,7 @@ inline Image::Image() : } inline void Image::Read(Value &obj, Asset &r) { + //basisu: no need to handle .ktx2, .basis, load as is if (!mDataLength) { Value *curUri = FindString(obj, "uri"); if (nullptr != curUri) { @@ -2101,6 +2102,7 @@ inline void Asset::ReadExtensionsUsed(Document &doc) { CHECK_EXT(KHR_materials_clearcoat); CHECK_EXT(KHR_materials_transmission); CHECK_EXT(KHR_draco_mesh_compression); + CHECK_EXT(KHR_texture_basisu); #undef CHECK_EXT } diff --git a/code/AssetLib/glTF2/glTF2AssetWriter.inl b/code/AssetLib/glTF2/glTF2AssetWriter.inl index 01a28d4b7..bf7dbbb2e 100644 --- a/code/AssetLib/glTF2/glTF2AssetWriter.inl +++ b/code/AssetLib/glTF2/glTF2AssetWriter.inl @@ -250,6 +250,7 @@ namespace glTF2 { inline void Write(Value& obj, Image& img, AssetWriter& w) { + //basisu: no need to handle .ktx2, .basis, write as is if (img.bufferView) { obj.AddMember("bufferView", img.bufferView->index, w.mAl); obj.AddMember("mimeType", Value(img.mimeType, w.mAl).Move(), w.mAl); @@ -892,10 +893,22 @@ namespace glTF2 { if (this->mAsset.extensionsUsed.FB_ngon_encoding) { exts.PushBack(StringRef("FB_ngon_encoding"), mAl); } + + if (this->mAsset.extensionsUsed.KHR_texture_basisu) { + exts.PushBack(StringRef("KHR_texture_basisu"), mAl); + } } if (!exts.Empty()) mDoc.AddMember("extensionsUsed", exts, mAl); + + //basisu extensionRequired + Value extsReq; + extsReq.SetArray(); + if (this->mAsset.extensionsUsed.KHR_texture_basisu) { + extsReq.PushBack(StringRef("KHR_texture_basisu"), mAl); + mDoc.AddMember("extensionsRequired", extsReq, mAl); + } } template diff --git a/code/AssetLib/glTF2/glTF2Exporter.cpp b/code/AssetLib/glTF2/glTF2Exporter.cpp index 51aef013d..e039bf88a 100644 --- a/code/AssetLib/glTF2/glTF2Exporter.cpp +++ b/code/AssetLib/glTF2/glTF2Exporter.cpp @@ -494,7 +494,6 @@ void glTF2Exporter::GetMatTexProp(const aiMaterial* mat, float& prop, const char void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTextureType tt, unsigned int slot = 0) { - if (mat->GetTextureCount(tt) > 0) { aiString tex; @@ -507,6 +506,7 @@ void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTe texture = mAsset->textures.Get(it->second); } + bool useBasisUniversal = false; if (!texture) { std::string texId = mAsset->FindUniqueID("", "texture"); texture = mAsset->textures.Create(texId); @@ -519,18 +519,42 @@ void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTe aiTexture* curTex = mScene->mTextures[atoi(&path[1])]; texture->source->name = curTex->mFilename.C_Str(); - - // The asset has its own buffer, see Image::SetData - texture->source->SetData(reinterpret_cast(curTex->pcData), curTex->mWidth, *mAsset); - + + //basisu: embedded ktx2, bu if (curTex->achFormatHint[0]) { std::string mimeType = "image/"; - mimeType += (memcmp(curTex->achFormatHint, "jpg", 3) == 0) ? "jpeg" : curTex->achFormatHint; + if(memcmp(curTex->achFormatHint, "jpg", 3) == 0) + mimeType += "jpeg"; + else if(memcmp(curTex->achFormatHint, "ktx", 3) == 0) { + useBasisUniversal = true; + mimeType += "ktx2"; + } + else if(memcmp(curTex->achFormatHint, "bu", 3) == 0) { + useBasisUniversal = true; + mimeType += "basis"; + } + else + mimeType += curTex->achFormatHint; texture->source->mimeType = mimeType; } + + // The asset has its own buffer, see Image::SetData + //basisu: "image/ktx2", "image/basis" as is + texture->source->SetData(reinterpret_cast(curTex->pcData), curTex->mWidth, *mAsset); } else { texture->source->uri = path; + if(texture->source->uri.find(".ktx2")!=std::string::npos || + texture->source->uri.find(".basis")!=std::string::npos) + { + useBasisUniversal = true; + } + } + + //basisu + if(useBasisUniversal) { + mAsset->extensionsUsed.KHR_texture_basisu = true; + mAsset->extensionsRequired.KHR_texture_basisu = true; } GetTexSampler(mat, texture, tt, slot); diff --git a/code/AssetLib/glTF2/glTF2Importer.cpp b/code/AssetLib/glTF2/glTF2Importer.cpp index ab1f01bf8..b109891cb 100644 --- a/code/AssetLib/glTF2/glTF2Importer.cpp +++ b/code/AssetLib/glTF2/glTF2Importer.cpp @@ -1476,6 +1476,12 @@ void glTF2Importer::ImportEmbeddedTextures(glTF2::Asset &r) { if (strcmp(ext, "jpeg") == 0) { ext = "jpg"; } + else if(strcmp(ext, "ktx2") == 0) { //basisu + ext = "ktx"; + } + else if(strcmp(ext, "basis") == 0) { //basisu + ext = "bu"; + } size_t len = strlen(ext); if (len <= 3) { From 964778cac1a98fd333d00ab18f43577ff69ce39b Mon Sep 17 00:00:00 2001 From: Carsten Rudolph <18394207+crud89@users.noreply.github.com> Date: Fri, 7 May 2021 17:30:26 +0200 Subject: [PATCH 10/34] Add AI_CONFIG_EXPORT_BLOB_NAME export property. --- include/assimp/config.h.in | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/include/assimp/config.h.in b/include/assimp/config.h.in index d78568da7..8ea82482f 100644 --- a/include/assimp/config.h.in +++ b/include/assimp/config.h.in @@ -1066,6 +1066,23 @@ enum aiComponent */ #define AI_CONFIG_EXPORT_POINT_CLOUDS "EXPORT_POINT_CLOUDS" +/** + * @brief Specifies the blob name, assimp uses for exporting. + * + * Some formats require auxiliary files to be written, that need to be linked back into + * the original file. For example, OBJ files export materials to a separate MTL file and + * use the `mtllib` keyword to reference this file. + * + * When exporting blobs using #ExportToBlob, assimp does not know the name of the blob + * file and thus outputs `mtllib $blobfile.mtl`, which might not be desired, since the + * MTL file might be called differently. + * + * This property can be used to give the exporter a hint on how to use the magic + * `$blobfile` keyword. If the exporter detects the keyword and is provided with a name + * for the blob, it instead uses this name. + */ +#define AI_CONFIG_EXPORT_BLOB_NAME "EXPORT_BLOB_NAME" + /** * @brief Specifies a gobal key factor for scale, float value */ From be85f238f4dd96f1182678d23e997b2e675484a7 Mon Sep 17 00:00:00 2001 From: Carsten Rudolph <18394207+crud89@users.noreply.github.com> Date: Fri, 7 May 2021 17:30:58 +0200 Subject: [PATCH 11/34] Add optional blob base name to blob IO system. --- include/assimp/BlobIOSystem.h | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/include/assimp/BlobIOSystem.h b/include/assimp/BlobIOSystem.h index 4e3d5c2a3..081ccf32a 100644 --- a/include/assimp/BlobIOSystem.h +++ b/include/assimp/BlobIOSystem.h @@ -194,8 +194,14 @@ class BlobIOSystem : public IOSystem { friend class BlobIOStream; typedef std::pair BlobEntry; + public: - BlobIOSystem() { + BlobIOSystem() : + baseName{} { + } + + BlobIOSystem(const std::string &baseName) : + baseName(baseName) { } virtual ~BlobIOSystem() { @@ -207,27 +213,32 @@ public: public: // ------------------------------------------------------------------- const char *GetMagicFileName() const { - return AI_BLOBIO_MAGIC; + return baseName.empty() ? AI_BLOBIO_MAGIC : baseName.c_str(); } // ------------------------------------------------------------------- aiExportDataBlob *GetBlobChain() { + const auto magicName = std::string(this->GetMagicFileName()); + const bool hasBaseName = baseName.empty(); + // one must be the master aiExportDataBlob *master = nullptr, *cur; + for (const BlobEntry &blobby : blobs) { - if (blobby.first == AI_BLOBIO_MAGIC) { + if (blobby.first == magicName) { master = blobby.second; + master->name.Set(hasBaseName ? blobby.first : ""); break; } } + if (!master) { ASSIMP_LOG_ERROR("BlobIOSystem: no data written or master file was not closed properly."); return nullptr; } - master->name.Set(""); - cur = master; + for (const BlobEntry &blobby : blobs) { if (blobby.second == master) { continue; @@ -236,9 +247,14 @@ public: cur->next = blobby.second; cur = cur->next; - // extract the file extension from the file written - const std::string::size_type s = blobby.first.find_first_of('.'); - cur->name.Set(s == std::string::npos ? blobby.first : blobby.first.substr(s + 1)); + if (hasBaseName) { + cur->name.Set(blobby.first); + } + else { + // extract the file extension from the file written + const std::string::size_type s = blobby.first.find_first_of('.'); + cur->name.Set(s == std::string::npos ? blobby.first : blobby.first.substr(s + 1)); + } } // give up blob ownership @@ -283,6 +299,7 @@ private: } private: + std::string baseName; std::set created; std::vector blobs; }; From 8ff52c0f89fe52d52b6fc0ac6f111c9355616139 Mon Sep 17 00:00:00 2001 From: Carsten Rudolph <18394207+crud89@users.noreply.github.com> Date: Fri, 7 May 2021 17:31:30 +0200 Subject: [PATCH 12/34] Pass base name from export properties to the IO system. --- code/Common/Exporter.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/code/Common/Exporter.cpp b/code/Common/Exporter.cpp index 20cbb05d6..ebcc955df 100644 --- a/code/Common/Exporter.cpp +++ b/code/Common/Exporter.cpp @@ -343,9 +343,11 @@ const aiExportDataBlob* Exporter::ExportToBlob( const aiScene* pScene, const cha delete pimpl->blob; pimpl->blob = nullptr; } + + auto baseName = pProperties ? pProperties->GetPropertyString(AI_CONFIG_EXPORT_BLOB_NAME, AI_BLOBIO_MAGIC) : AI_BLOBIO_MAGIC; std::shared_ptr old = pimpl->mIOSystem; - BlobIOSystem* blobio = new BlobIOSystem(); + BlobIOSystem *blobio = new BlobIOSystem(baseName); pimpl->mIOSystem = std::shared_ptr( blobio ); if (AI_SUCCESS != Export(pScene,pFormatId,blobio->GetMagicFileName(), pPreprocessing, pProperties)) { From 1b33dd1965dd0a58b20fa4e270d1809abb920c58 Mon Sep 17 00:00:00 2001 From: Carsten Rudolph <18394207+crud89@users.noreply.github.com> Date: Fri, 7 May 2021 17:31:38 +0200 Subject: [PATCH 13/34] Document AI_CONFIG_EXPORT_BLOB_NAME. --- include/assimp/cexport.h | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/include/assimp/cexport.h b/include/assimp/cexport.h index 2e84b1f30..44843af0e 100644 --- a/include/assimp/cexport.h +++ b/include/assimp/cexport.h @@ -205,16 +205,22 @@ struct aiExportDataBlob { void *data; /** Name of the blob. An empty string always - indicates the first (and primary) blob, - which contains the actual file data. - Any other blobs are auxiliary files produced - by exporters (i.e. material files). Existence - of such files depends on the file format. Most - formats don't split assets across multiple files. - - If used, blob names usually contain the file - extension that should be used when writing - the data to disc. + * indicates the first (and primary) blob, + * which contains the actual file data. + * Any other blobs are auxiliary files produced + * by exporters (i.e. material files). Existence + * of such files depends on the file format. Most + * formats don't split assets across multiple files. + * + * If used, blob names usually contain the file + * extension that should be used when writing + * the data to disc. + * + * The blob names generated can be influenced by + * setting the #AI_CONFIG_EXPORT_BLOB_NAME export + * property to the name that is used for the master + * blob. All other names are typically derived from + * the base name, by the file format exporter. */ C_STRUCT aiString name; From a19b708144b91d6936339b40f93ab7540ee25689 Mon Sep 17 00:00:00 2001 From: ywang Date: Fri, 7 May 2021 16:27:23 -0700 Subject: [PATCH 14/34] support both ktx and ktx2 --- code/AssetLib/glTF2/glTF2Exporter.cpp | 8 ++++++-- code/AssetLib/glTF2/glTF2Importer.cpp | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/code/AssetLib/glTF2/glTF2Exporter.cpp b/code/AssetLib/glTF2/glTF2Exporter.cpp index e039bf88a..751508225 100644 --- a/code/AssetLib/glTF2/glTF2Exporter.cpp +++ b/code/AssetLib/glTF2/glTF2Exporter.cpp @@ -526,10 +526,14 @@ void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTe if(memcmp(curTex->achFormatHint, "jpg", 3) == 0) mimeType += "jpeg"; else if(memcmp(curTex->achFormatHint, "ktx", 3) == 0) { + useBasisUniversal = true; + mimeType += "ktx"; + } + else if(memcmp(curTex->achFormatHint, "kx2", 3) == 0) { useBasisUniversal = true; mimeType += "ktx2"; } - else if(memcmp(curTex->achFormatHint, "bu", 3) == 0) { + else if(memcmp(curTex->achFormatHint, "bu", 2) == 0) { useBasisUniversal = true; mimeType += "basis"; } @@ -544,7 +548,7 @@ void glTF2Exporter::GetMatTex(const aiMaterial* mat, Ref& texture, aiTe } else { texture->source->uri = path; - if(texture->source->uri.find(".ktx2")!=std::string::npos || + if(texture->source->uri.find(".ktx")!=std::string::npos || texture->source->uri.find(".basis")!=std::string::npos) { useBasisUniversal = true; diff --git a/code/AssetLib/glTF2/glTF2Importer.cpp b/code/AssetLib/glTF2/glTF2Importer.cpp index b109891cb..db5da8813 100644 --- a/code/AssetLib/glTF2/glTF2Importer.cpp +++ b/code/AssetLib/glTF2/glTF2Importer.cpp @@ -1476,8 +1476,8 @@ void glTF2Importer::ImportEmbeddedTextures(glTF2::Asset &r) { if (strcmp(ext, "jpeg") == 0) { ext = "jpg"; } - else if(strcmp(ext, "ktx2") == 0) { //basisu - ext = "ktx"; + else if(strcmp(ext, "ktx2") == 0) { //basisu: ktx remains + ext = "kx2"; } else if(strcmp(ext, "basis") == 0) { //basisu ext = "bu"; From 859b32c04581644a9f54c6f18860dd7927a1d127 Mon Sep 17 00:00:00 2001 From: Jason C Date: Fri, 7 May 2021 22:32:32 -0400 Subject: [PATCH 15/34] [Logger] Log a notification instead of silently dropping long log messages. Logs a notification instead of silently dropping long log messages, which can complicate debugging. This way, if you don't see a message you expect to see, you'll immediately know why. The *correct* approach would be to eliminate length filtering here entirely and use `snprintf` appropriately (also there's a tiny -- probably negligible -- performance hit here in calling `strlen` regardless of whether or not the verbosity level matches). Failing that, the second best option is to copy and truncate messages here. However, for now, this should be OK. --- code/Common/DefaultLogger.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/code/Common/DefaultLogger.cpp b/code/Common/DefaultLogger.cpp index aa13ca5ce..3f6d2d7ed 100644 --- a/code/Common/DefaultLogger.cpp +++ b/code/Common/DefaultLogger.cpp @@ -169,7 +169,7 @@ void Logger::debug(const char *message) { // sometimes importers will include data from the input file // (i.e. node names) in their messages. if (strlen(message) > MAX_LOG_MESSAGE_LENGTH) { - return; + return OnDebug(""); } return OnDebug(message); } @@ -181,7 +181,7 @@ void Logger::verboseDebug(const char *message) { // sometimes importers will include data from the input file // (i.e. node names) in their messages. if (strlen(message) > MAX_LOG_MESSAGE_LENGTH) { - return; + return OnVerboseDebug(""); } return OnVerboseDebug(message); } @@ -191,7 +191,7 @@ void Logger::info(const char *message) { // SECURITY FIX: see above if (strlen(message) > MAX_LOG_MESSAGE_LENGTH) { - return; + return OnInfo(""); } return OnInfo(message); } @@ -201,7 +201,7 @@ void Logger::warn(const char *message) { // SECURITY FIX: see above if (strlen(message) > MAX_LOG_MESSAGE_LENGTH) { - return; + return OnWarn(""); } return OnWarn(message); } @@ -210,7 +210,7 @@ void Logger::warn(const char *message) { void Logger::error(const char *message) { // SECURITY FIX: see above if (strlen(message) > MAX_LOG_MESSAGE_LENGTH) { - return; + return OnError(""); } return OnError(message); } From 27135bd3e7ec444bb7000a2d8be179a52f77a5cf Mon Sep 17 00:00:00 2001 From: ogjamesfranco Date: Sat, 15 May 2021 15:27:24 -0400 Subject: [PATCH 16/34] changed the assimp output directory vars to cached vars --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7027e3300..7b9d6fc55 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -334,9 +334,9 @@ INCLUDE (FindPkgMacros) INCLUDE (PrecompiledHeader) # Set Assimp project output directory variables. -SET(ASSIMP_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin") -SET(ASSIMP_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin") -SET(ASSIMP_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib") +SET(ASSIMP_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin" CACHE STRING "Path for runtime output files") +SET(ASSIMP_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin" CACHE STRING "Path for library output files") +SET(ASSIMP_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/lib" CACHE STRING "Path for archive output files") # Macro used to set the output directories of a target to the # respective Assimp output directories. From 1f32743f8b455082432adb249426f17c39dfa756 Mon Sep 17 00:00:00 2001 From: dlyr Date: Sat, 22 May 2021 00:56:01 +0200 Subject: [PATCH 17/34] Fix camera fov comment since full fov is stored --- include/assimp/camera.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/assimp/camera.h b/include/assimp/camera.h index d7324d10d..6a7acadbb 100644 --- a/include/assimp/camera.h +++ b/include/assimp/camera.h @@ -137,7 +137,7 @@ struct aiCamera */ C_STRUCT aiVector3D mLookAt; - /** Half horizontal field of view angle, in radians. + /** Horizontal field of view angle, in radians. * * The field of view angle is the angle between the center * line of the screen and the left or right border. From 28e34878cb9d85627cce81ed20dcb6f8b493d83b Mon Sep 17 00:00:00 2001 From: Jagoon <5785026+jagoon@users.noreply.github.com> Date: Sat, 22 May 2021 23:20:34 +0900 Subject: [PATCH 18/34] Fix fbx exporter bug if root node contains meshes. --- code/AssetLib/FBX/FBXExporter.cpp | 32 +++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/code/AssetLib/FBX/FBXExporter.cpp b/code/AssetLib/FBX/FBXExporter.cpp index e519f7e77..b24360cd5 100644 --- a/code/AssetLib/FBX/FBXExporter.cpp +++ b/code/AssetLib/FBX/FBXExporter.cpp @@ -541,10 +541,17 @@ void FBXExporter::WriteReferences () // (before any actual data is written) // --------------------------------------------------------------- -size_t count_nodes(const aiNode* n) { - size_t count = 1; +size_t count_nodes(const aiNode* n, const aiNode* root) { + size_t count; + if (n == root) { + count = n->mNumMeshes; // (not counting root node) + } else if (n->mNumMeshes > 1) { + count = n->mNumMeshes + 1; + } else { + count = 1; + } for (size_t i = 0; i < n->mNumChildren; ++i) { - count += count_nodes(n->mChildren[i]); + count += count_nodes(n->mChildren[i], root); } return count; } @@ -714,7 +721,7 @@ void FBXExporter::WriteDefinitions () // Model / FbxNode // <~~ node hierarchy - count = int32_t(count_nodes(mScene->mRootNode)) - 1; // (not counting root node) + count = int32_t(count_nodes(mScene->mRootNode, mScene->mRootNode)); if (count) { n = FBX::Node("ObjectType", "Model"); n.AddChild("Count", count); @@ -2625,17 +2632,14 @@ void FBXExporter::WriteModelNodes( ], new_node_uid ); - // write model node - FBX::Node m("Model"); + + aiNode new_node; // take name from mesh name, if it exists - std::string name = mScene->mMeshes[node->mMeshes[i]]->mName.C_Str(); - name += FBX::SEPARATOR + "Model"; - m.AddProperties(new_node_uid, name, "Mesh"); - m.AddChild("Version", int32_t(232)); - FBX::Node p("Properties70"); - p.AddP70enum("InheritType", 1); - m.AddChild(p); - m.Dump(outstream, binary, 1); + new_node.mName = mScene->mMeshes[node->mMeshes[i]]->mName; + // write model node + WriteModelNode( + outstream, binary, &new_node, new_node_uid, "Mesh", transform_chain + ); } } From b7b3c6db7e3ba84814dcec1394787e40d3ff7e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Verdon?= Date: Sat, 22 May 2021 16:57:07 +0200 Subject: [PATCH 19/34] Fixing GCC 4.9 compilation issues --- code/AssetLib/glTF2/glTF2Asset.inl | 2 ++ include/assimp/TinyFormatter.h | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/code/AssetLib/glTF2/glTF2Asset.inl b/code/AssetLib/glTF2/glTF2Asset.inl index 8a793c144..256fc8931 100644 --- a/code/AssetLib/glTF2/glTF2Asset.inl +++ b/code/AssetLib/glTF2/glTF2Asset.inl @@ -58,7 +58,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma clang diagnostic ignored "-Wsign-compare" #elif defined(__GNUC__) #pragma GCC diagnostic push +#if (__GNUC__ > 4) #pragma GCC diagnostic ignored "-Wbool-compare" +#endif #pragma GCC diagnostic ignored "-Wsign-compare" #endif diff --git a/include/assimp/TinyFormatter.h b/include/assimp/TinyFormatter.h index ace20be5c..112f19013 100644 --- a/include/assimp/TinyFormatter.h +++ b/include/assimp/TinyFormatter.h @@ -88,9 +88,17 @@ public: underlying << sin; } + // Same problem as the copy constructor below, but with root cause is that stream move + // is not permitted on older GCC versions. Small performance impact on those platforms. +#if defined(__GNUC__) && (__GNUC__ == 4 && __GNUC_MINOR__ <= 9) + basic_formatter(basic_formatter&& other) { + underlying << (string)other; + } +#else basic_formatter(basic_formatter&& other) : underlying(std::move(other.underlying)) { } +#endif // The problem described here: // https://sourceforge.net/tracker/?func=detail&atid=1067632&aid=3358562&group_id=226462 From f96e3cde2d299aa1bc1fd5126d55fbc4aec5cc73 Mon Sep 17 00:00:00 2001 From: Jagoon <5785026+jagoon@users.noreply.github.com> Date: Sun, 23 May 2021 00:06:05 +0900 Subject: [PATCH 20/34] Fix transform chain is applied twice --- code/AssetLib/FBX/FBXExporter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/AssetLib/FBX/FBXExporter.cpp b/code/AssetLib/FBX/FBXExporter.cpp index b24360cd5..f7beee2e0 100644 --- a/code/AssetLib/FBX/FBXExporter.cpp +++ b/code/AssetLib/FBX/FBXExporter.cpp @@ -2638,7 +2638,7 @@ void FBXExporter::WriteModelNodes( new_node.mName = mScene->mMeshes[node->mMeshes[i]]->mName; // write model node WriteModelNode( - outstream, binary, &new_node, new_node_uid, "Mesh", transform_chain + outstream, binary, &new_node, new_node_uid, "Mesh", std::vector>() ); } } From 799384f2b85b8d3ee9c5b9e18bbe532b4dc7c63c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Verdon?= Date: Sat, 22 May 2021 17:36:39 +0200 Subject: [PATCH 21/34] Adding the required c flag to compile zip files using gcc 4.9 --- code/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index ebc3e0116..d3cb6e923 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -1137,6 +1137,9 @@ ELSE() TARGET_COMPILE_OPTIONS(assimp PRIVATE -Werror) ENDIF() +# adds C_FLAGS required to compile zip.c on old GCC 4.x compiler +TARGET_COMPILE_FEATURES(assimp PUBLIC c_std_99) + TARGET_INCLUDE_DIRECTORIES ( assimp PUBLIC $ $ From 5468dd667e41069da6f6ca97141c953838107eef Mon Sep 17 00:00:00 2001 From: Evangel Date: Wed, 26 May 2021 18:36:56 +1000 Subject: [PATCH 22/34] Fix bug in aiMetadata constructor that overwrites an array of one of aiString, aiVector3D, or aiMetadata with the first entry aiMetadata copy constructor calls aiMetadata::Get on the copied from aiMetadata using the const aiString &key version. When this is called on the metadata of an array type, this overwrites all entries with the first entry. This is due to the key of all entries in an array being the name of the array. ie, in a glTF2 file with an extension: "Extension" : [ "Value1", "Value2", "Value3" ] the aiMetadata struct for the "Extension" entry will have 3 entries with key/value pairs as: "Extension"/"Value1" "Extension"/"Value2" "Extension"/"Value3" So when the copy constructor calls the key based aiMetadata::Get, it will find "Value1" for all three entries. This change simply replaces the key based aiMetadata::Get with the index based aiMetadata::Get --- include/assimp/metadata.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/assimp/metadata.h b/include/assimp/metadata.h index fdc88be56..39ac386e4 100644 --- a/include/assimp/metadata.h +++ b/include/assimp/metadata.h @@ -202,17 +202,17 @@ struct aiMetadata { } break; case AI_AISTRING: { aiString v; - rhs.Get(mKeys[i], v); + rhs.Get(i, v); mValues[i].mData = new aiString(v); } break; case AI_AIVECTOR3D: { aiVector3D v; - rhs.Get(mKeys[i], v); + rhs.Get(i, v); mValues[i].mData = new aiVector3D(v); } break; case AI_AIMETADATA: { aiMetadata v; - rhs.Get(mKeys[i], v); + rhs.Get(i, v); mValues[i].mData = new aiMetadata(v); } break; #ifndef SWIG From 2f4fba070364ec3ceb34104e884ee9bddcbd8acf Mon Sep 17 00:00:00 2001 From: Evangel Date: Wed, 26 May 2021 19:11:19 +1000 Subject: [PATCH 23/34] Static cast i back to unsigned int because MSVC complains otherwise. i will never be bigger than an unsigned int since that's what mNumProperties is to begin with. --- include/assimp/metadata.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/assimp/metadata.h b/include/assimp/metadata.h index 39ac386e4..551a9aba4 100644 --- a/include/assimp/metadata.h +++ b/include/assimp/metadata.h @@ -156,7 +156,7 @@ struct aiMetadata { #ifdef __cplusplus - /** + /** * @brief The default constructor, set all members to zero by default. */ aiMetadata() AI_NO_EXCEPT @@ -202,17 +202,17 @@ struct aiMetadata { } break; case AI_AISTRING: { aiString v; - rhs.Get(i, v); + rhs.Get(static_cast(i), v); mValues[i].mData = new aiString(v); } break; case AI_AIVECTOR3D: { aiVector3D v; - rhs.Get(i, v); + rhs.Get(static_cast(i), v); mValues[i].mData = new aiVector3D(v); } break; case AI_AIMETADATA: { aiMetadata v; - rhs.Get(i, v); + rhs.Get(static_cast(i), v); mValues[i].mData = new aiMetadata(v); } break; #ifndef SWIG From c33a4b26346fd1cdaf7133dad06fcc29f47990aa Mon Sep 17 00:00:00 2001 From: Carsten Rudolph <18394207+crud89@users.noreply.github.com> Date: Thu, 27 May 2021 10:10:55 +0200 Subject: [PATCH 24/34] Fixed base name check. --- include/assimp/BlobIOSystem.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/include/assimp/BlobIOSystem.h b/include/assimp/BlobIOSystem.h index 081ccf32a..0abf166bc 100644 --- a/include/assimp/BlobIOSystem.h +++ b/include/assimp/BlobIOSystem.h @@ -197,7 +197,7 @@ class BlobIOSystem : public IOSystem { public: BlobIOSystem() : - baseName{} { + baseName{AI_BLOBIO_MAGIC} { } BlobIOSystem(const std::string &baseName) : @@ -213,13 +213,13 @@ public: public: // ------------------------------------------------------------------- const char *GetMagicFileName() const { - return baseName.empty() ? AI_BLOBIO_MAGIC : baseName.c_str(); + return baseName.c_str(); } // ------------------------------------------------------------------- aiExportDataBlob *GetBlobChain() { const auto magicName = std::string(this->GetMagicFileName()); - const bool hasBaseName = baseName.empty(); + const bool hasBaseName = baseName != AI_BLOBIO_MAGIC; // one must be the master aiExportDataBlob *master = nullptr, *cur; @@ -249,8 +249,7 @@ public: if (hasBaseName) { cur->name.Set(blobby.first); - } - else { + } else { // extract the file extension from the file written const std::string::size_type s = blobby.first.find_first_of('.'); cur->name.Set(s == std::string::npos ? blobby.first : blobby.first.substr(s + 1)); From 59467b204a6f099ed13ab7234bee08c1d89d24e6 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 27 May 2021 15:50:28 +0200 Subject: [PATCH 25/34] Create tech_debt.md --- .github/ISSUE_TEMPLATE/tech_debt.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/tech_debt.md diff --git a/.github/ISSUE_TEMPLATE/tech_debt.md b/.github/ISSUE_TEMPLATE/tech_debt.md new file mode 100644 index 000000000..2cd84d975 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/tech_debt.md @@ -0,0 +1,25 @@ +--- +name: Technical debt +about: Create a report to help us to fix and detect tech debts +title: '' +labels: '' +assignees: '' + +--- + +**Describe the technical debt** +A clear and concise description of what the tech debt is about. + +**Better solution** +A clear and concise description of what you would expect. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. From 919ae69fe8208524301689371e43a5581b8889c6 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 28 May 2021 08:44:40 +0200 Subject: [PATCH 26/34] Update tech_debt.md --- .github/ISSUE_TEMPLATE/tech_debt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/tech_debt.md b/.github/ISSUE_TEMPLATE/tech_debt.md index 2cd84d975..a1172d932 100644 --- a/.github/ISSUE_TEMPLATE/tech_debt.md +++ b/.github/ISSUE_TEMPLATE/tech_debt.md @@ -2,7 +2,7 @@ name: Technical debt about: Create a report to help us to fix and detect tech debts title: '' -labels: '' +labels: 'Techdebt' assignees: '' --- From 3c51ff771cfa001e96b78382e299788250647ef8 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 28 May 2021 09:34:03 +0200 Subject: [PATCH 27/34] Update bug_report.md - Add the bug label - Make platform config more easy --- .github/ISSUE_TEMPLATE/bug_report.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index dd84ea782..0f9a5f105 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,8 +1,8 @@ --- name: Bug report about: Create a report to help us improve -title: '' -labels: '' +title: 'Bug:' +labels: 'Bug' assignees: '' --- @@ -23,16 +23,10 @@ A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. -**Desktop (please complete the following information):** +**Platform (please complete the following information):** - OS: [e.g. iOS] - Browser [e.g. chrome, safari] - Version [e.g. 22] -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - **Additional context** Add any other context about the problem here. From 2559befaca621a94f6e911493519730d42f17775 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 28 May 2021 10:26:03 +0200 Subject: [PATCH 28/34] Update feature_request.md - Add label Feature-Request --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index bbcbbe7d6..87302a0f0 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request about: Suggest an idea for this project title: '' -labels: '' +labels: 'Feature-Request' assignees: '' --- From 7534b149cff5ef287fce50da4f17cebf4c5a3608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20M=C3=B6ller?= Date: Fri, 28 May 2021 11:55:46 +0200 Subject: [PATCH 29/34] fix non skipped CR in header parsing --- code/AssetLib/Ply/PlyParser.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/AssetLib/Ply/PlyParser.cpp b/code/AssetLib/Ply/PlyParser.cpp index 4b416d1a1..59cb6b976 100644 --- a/code/AssetLib/Ply/PlyParser.cpp +++ b/code/AssetLib/Ply/PlyParser.cpp @@ -419,7 +419,8 @@ bool PLY::DOM::ParseHeader(IOStreamBuffer &streamBuffer, std::vector if (PLY::Element::ParseElement(streamBuffer, buffer, &out)) { // add the element to the list of elements alElements.push_back(out); - } else if (TokenMatch(buffer, "end_header", 10)) { + } else if ( TokenMatch(buffer, "end_header\r", 11) || //checks for header end with /r/n ending + TokenMatch(buffer, "end_header", 10)) { //checks for /n ending, if it doesn't end with /r/n // we have reached the end of the header break; } else { From 77ce4080b689d4fcee3435813c3811dfa5781682 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sun, 30 May 2021 21:54:04 +0200 Subject: [PATCH 30/34] fix viewer in case of unknown primitives. --- code/AssetLib/DXF/DXFLoader.cpp | 2 -- tools/assimp_view/assimp_view.cpp | 7 +++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/code/AssetLib/DXF/DXFLoader.cpp b/code/AssetLib/DXF/DXFLoader.cpp index 5d32ed121..49d572b0b 100644 --- a/code/AssetLib/DXF/DXFLoader.cpp +++ b/code/AssetLib/DXF/DXFLoader.cpp @@ -5,8 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2021, assimp team - - All rights reserved. Redistribution and use of this software in source and binary forms, diff --git a/tools/assimp_view/assimp_view.cpp b/tools/assimp_view/assimp_view.cpp index 260b22941..5ab7c53ad 100644 --- a/tools/assimp_view/assimp_view.cpp +++ b/tools/assimp_view/assimp_view.cpp @@ -489,7 +489,7 @@ int CreateAssetData() { nidx = 3; break; default: - ai_assert(false); + CLogWindow::Instance().WriteLine("Unknown primitiv type"); break; }; @@ -500,8 +500,7 @@ int CreateAssetData() { // check whether we can use 16 bit indices if (numIndices >= 65536) { // create 32 bit index buffer - if (FAILED(g_piDevice->CreateIndexBuffer(4 * - numIndices, + if (FAILED(g_piDevice->CreateIndexBuffer(4 * numIndices, D3DUSAGE_WRITEONLY | dwUsage, D3DFMT_INDEX32, D3DPOOL_DEFAULT, @@ -523,7 +522,7 @@ int CreateAssetData() { } else { // create 16 bit index buffer if (FAILED(g_piDevice->CreateIndexBuffer(2 * - numIndices, +numIndices, D3DUSAGE_WRITEONLY | dwUsage, D3DFMT_INDEX16, D3DPOOL_DEFAULT, From 84a2e1fc922542c3c60dff94c5db682f7ab95711 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 3 Jun 2021 20:43:37 +0200 Subject: [PATCH 31/34] Update unity plugin to trilib2 - closes https://github.com/assimp/assimp/issues/3872 --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index f2ea4b094..949d60966 100644 --- a/Readme.md +++ b/Readme.md @@ -42,7 +42,7 @@ Take a look into the https://github.com/assimp/assimp/blob/master/Build.md file. * [.NET](https://bitbucket.org/Starnick/assimpnet/src/master/) * [Pascal](port/AssimpPascal/Readme.md) * [Javascript (Alpha)](https://github.com/makc/assimp2json) -* [Unity 3d Plugin](https://www.assetstore.unity3d.com/en/#!/content/91777) +* [Unity 3d Plugin](https://ricardoreis.net/trilib-2/) * [JVM](https://github.com/kotlin-graphics/assimp) Full jvm port (current [status](https://github.com/kotlin-graphics/assimp/wiki/Status)) * [HAXE-Port](https://github.com/longde123/assimp-haxe) The Assimp-HAXE-port. * [Rust](https://github.com/jkvargas/russimp) From b4fc41bc094574400b0a688ec1381888e40ee99c Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 3 Jun 2021 22:52:10 +0200 Subject: [PATCH 32/34] Use corret attribute name - closes https://github.com/assimp/assimp/issues/3887 --- port/PyAssimp/pyassimp/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/port/PyAssimp/pyassimp/core.py b/port/PyAssimp/pyassimp/core.py index 37beac886..35ad882b3 100644 --- a/port/PyAssimp/pyassimp/core.py +++ b/port/PyAssimp/pyassimp/core.py @@ -211,7 +211,7 @@ def _init(self, target = None, parent = None): else: # starts with 'm' but not iterable - setattr(target, name, obj) + setattr(target, m, obj) logger.debug("Added " + name + " as self." + name + " (type: " + str(type(obj)) + ")") if _is_init_type(obj): From cc912f09f7cf1d0d7cbdff5d6d6617d5af904c3a Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 3 Jun 2021 23:16:20 +0200 Subject: [PATCH 33/34] update pugi_xml to 1.11 --- contrib/pugixml/readme.txt | 10 +- contrib/pugixml/src/pugiconfig.hpp | 21 +- contrib/pugixml/src/pugixml.cpp | 638 +++++++++++++++++++---------- contrib/pugixml/src/pugixml.hpp | 69 +++- 4 files changed, 496 insertions(+), 242 deletions(-) diff --git a/contrib/pugixml/readme.txt b/contrib/pugixml/readme.txt index 5beb08a90..bfb1875cb 100644 --- a/contrib/pugixml/readme.txt +++ b/contrib/pugixml/readme.txt @@ -1,7 +1,7 @@ -pugixml 1.9 - an XML processing library +pugixml 1.11 - an XML processing library -Copyright (C) 2006-2018, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com) -Report bugs and download new versions at http://pugixml.org/ +Copyright (C) 2006-2020, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com) +Report bugs and download new versions at https://pugixml.org/ This is the distribution of pugixml, which is a C++ XML processing library, which consists of a DOM-like interface with rich traversal/modification @@ -13,8 +13,6 @@ automatically during parsing/saving). The distribution contains the following folders: - contrib/ - various contributions to pugixml - docs/ - documentation docs/samples - pugixml usage examples docs/quickstart.html - quick start guide @@ -28,7 +26,7 @@ The distribution contains the following folders: This library is distributed under the MIT License: -Copyright (c) 2006-2018 Arseny Kapoulkine +Copyright (c) 2006-2019 Arseny Kapoulkine Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation diff --git a/contrib/pugixml/src/pugiconfig.hpp b/contrib/pugixml/src/pugiconfig.hpp index 065b3c8bb..405a66b65 100644 --- a/contrib/pugixml/src/pugiconfig.hpp +++ b/contrib/pugixml/src/pugiconfig.hpp @@ -1,8 +1,8 @@ /** - * pugixml parser - version 1.9 + * pugixml parser - version 1.11 * -------------------------------------------------------- - * Copyright (C) 2006-2018, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com) - * Report bugs and download new versions at http://pugixml.org/ + * Copyright (C) 2006-2020, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com) + * Report bugs and download new versions at https://pugixml.org/ * * This library is distributed under the MIT License. See notice at the end * of this file. @@ -30,10 +30,8 @@ // #define PUGIXML_NO_EXCEPTIONS // Set this to control attributes for public classes/functions, i.e.: -//#ifdef _WIN32 -//#define PUGIXML_API __declspec(dllexport) // to export all public symbols from DLL -//#define PUGIXML_CLASS __declspec(dllimport) // to import all classes from DLL -//#endif +// #define PUGIXML_API __declspec(dllexport) // to export all public symbols from DLL +// #define PUGIXML_CLASS __declspec(dllimport) // to import all classes from DLL // #define PUGIXML_FUNCTION __fastcall // to set calling conventions to all public functions to fastcall // In absence of PUGIXML_CLASS/PUGIXML_FUNCTION definitions PUGIXML_API is used instead @@ -42,16 +40,19 @@ // #define PUGIXML_MEMORY_OUTPUT_STACK 10240 // #define PUGIXML_MEMORY_XPATH_PAGE_SIZE 4096 +// Tune this constant to adjust max nesting for XPath queries +// #define PUGIXML_XPATH_DEPTH_LIMIT 1024 + // Uncomment this to switch to header-only version -#define PUGIXML_HEADER_ONLY +// #define PUGIXML_HEADER_ONLY // Uncomment this to enable long long support -//#define PUGIXML_HAS_LONG_LONG +// #define PUGIXML_HAS_LONG_LONG #endif /** - * Copyright (c) 2006-2018 Arseny Kapoulkine + * Copyright (c) 2006-2020 Arseny Kapoulkine * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation diff --git a/contrib/pugixml/src/pugixml.cpp b/contrib/pugixml/src/pugixml.cpp index 2afff09dd..efdcdf699 100644 --- a/contrib/pugixml/src/pugixml.cpp +++ b/contrib/pugixml/src/pugixml.cpp @@ -1,8 +1,8 @@ /** - * pugixml parser - version 1.9 + * pugixml parser - version 1.11 * -------------------------------------------------------- - * Copyright (C) 2006-2018, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com) - * Report bugs and download new versions at http://pugixml.org/ + * Copyright (C) 2006-2020, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com) + * Report bugs and download new versions at https://pugixml.org/ * * This library is distributed under the MIT License. See notice at the end * of this file. @@ -378,7 +378,7 @@ PUGI__NS_BEGIN static PUGI__UNSIGNED_OVERFLOW unsigned int hash(const void* key) { - unsigned int h = static_cast(reinterpret_cast(key)); + unsigned int h = static_cast(reinterpret_cast(key) & 0xffffffff); // MurmurHash3 32-bit finalizer h ^= h >> 16; @@ -1861,7 +1861,7 @@ PUGI__NS_BEGIN enum chartypex_t { ctx_special_pcdata = 1, // Any symbol >= 0 and < 32 (except \t, \r, \n), &, <, > - ctx_special_attr = 2, // Any symbol >= 0 and < 32 (except \t), &, <, >, " + ctx_special_attr = 2, // Any symbol >= 0 and < 32, &, <, ", ' ctx_start_symbol = 4, // Any symbol > 127, a-z, A-Z, _ ctx_digit = 8, // 0-9 ctx_symbol = 16 // Any symbol > 127, a-z, A-Z, 0-9, _, -, . @@ -1869,10 +1869,10 @@ PUGI__NS_BEGIN static const unsigned char chartypex_table[256] = { - 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 2, 3, 3, 2, 3, 3, // 0-15 + 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 3, 2, 3, 3, // 0-15 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // 16-31 - 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 16, 16, 0, // 32-47 - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 3, 0, 3, 0, // 48-63 + 0, 0, 2, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 16, 16, 0, // 32-47 + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 3, 0, 1, 0, // 48-63 0, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, // 64-79 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 0, 0, 0, 0, 20, // 80-95 @@ -2709,7 +2709,7 @@ PUGI__NS_BEGIN { PUGI__STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20 && parse_trim_pcdata == 0x0800); - switch (((optmask >> 4) & 3) | ((optmask >> 9) & 4)) // get bitmask for flags (eol escapes trim) + switch (((optmask >> 4) & 3) | ((optmask >> 9) & 4)) // get bitmask for flags (trim eol escapes); this simultaneously checks 3 options from assertion above { case 0: return strconv_pcdata_impl::parse; case 1: return strconv_pcdata_impl::parse; @@ -2878,7 +2878,7 @@ PUGI__NS_BEGIN { PUGI__STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20 && parse_wconv_attribute == 0x40 && parse_wnorm_attribute == 0x80); - switch ((optmask >> 4) & 15) // get bitmask for flags (wconv wnorm eol escapes) + switch ((optmask >> 4) & 15) // get bitmask for flags (wnorm wconv eol escapes); this simultaneously checks 4 options from assertion above { case 0: return strconv_attribute_impl::parse_simple; case 1: return strconv_attribute_impl::parse_simple; @@ -3903,7 +3903,7 @@ PUGI__NS_BEGIN xml_encoding encoding; }; - PUGI__FN void text_output_escaped(xml_buffered_writer& writer, const char_t* s, chartypex_t type) + PUGI__FN void text_output_escaped(xml_buffered_writer& writer, const char_t* s, chartypex_t type, unsigned int flags) { while (*s) { @@ -3930,7 +3930,17 @@ PUGI__NS_BEGIN ++s; break; case '"': - writer.write('&', 'q', 'u', 'o', 't', ';'); + if (flags & format_attribute_single_quote) + writer.write('"'); + else + writer.write('&', 'q', 'u', 'o', 't', ';'); + ++s; + break; + case '\'': + if (flags & format_attribute_single_quote) + writer.write('&', 'a', 'p', 'o', 's', ';'); + else + writer.write('\''); ++s; break; default: // s is not a usual symbol @@ -3938,7 +3948,8 @@ PUGI__NS_BEGIN unsigned int ch = static_cast(*s++); assert(ch < 32); - writer.write('&', '#', static_cast((ch / 10) + '0'), static_cast((ch % 10) + '0'), ';'); + if (!(flags & format_skip_control_chars)) + writer.write('&', '#', static_cast((ch / 10) + '0'), static_cast((ch % 10) + '0'), ';'); } } } @@ -3949,7 +3960,7 @@ PUGI__NS_BEGIN if (flags & format_no_escapes) writer.write_string(s); else - text_output_escaped(writer, s, type); + text_output_escaped(writer, s, type, flags); } PUGI__FN void text_output_cdata(xml_buffered_writer& writer, const char_t* s) @@ -4063,6 +4074,7 @@ PUGI__NS_BEGIN PUGI__FN void node_output_attributes(xml_buffered_writer& writer, xml_node_struct* node, const char_t* indent, size_t indent_length, unsigned int flags, unsigned int depth) { const char_t* default_name = PUGIXML_TEXT(":anonymous"); + const char_t enquotation_char = (flags & format_attribute_single_quote) ? '\'' : '"'; for (xml_attribute_struct* a = node->first_attribute; a; a = a->next_attribute) { @@ -4078,12 +4090,12 @@ PUGI__NS_BEGIN } writer.write_string(a->name ? a->name + 0 : default_name); - writer.write('=', '"'); + writer.write('=', enquotation_char); if (a->value) text_output(writer, a->value, ctx_special_attr, flags); - writer.write('"'); + writer.write(enquotation_char); } } @@ -4423,6 +4435,9 @@ PUGI__NS_BEGIN while (sit && sit != sn) { + // loop invariant: dit is inside the subtree rooted at dn + assert(dit); + // when a tree is copied into one of the descendants, we need to skip that subtree to avoid an infinite loop if (sit != dn) { @@ -4452,9 +4467,14 @@ PUGI__NS_BEGIN sit = sit->parent; dit = dit->parent; + + // loop invariant: dit is inside the subtree rooted at dn while sit is inside sn + assert(sit == sn || dit); } while (sit != sn); } + + assert(!sit || dit == dn->parent); } PUGI__FN void node_copy_attribute(xml_attribute_struct* da, xml_attribute_struct* sa) @@ -4653,19 +4673,19 @@ PUGI__NS_BEGIN } template - PUGI__FN bool set_value_convert(String& dest, Header& header, uintptr_t header_mask, float value) + PUGI__FN bool set_value_convert(String& dest, Header& header, uintptr_t header_mask, float value, int precision) { char buf[128]; - PUGI__SNPRINTF(buf, "%.9g", value); + PUGI__SNPRINTF(buf, "%.*g", precision, double(value)); return set_value_ascii(dest, header, header_mask, buf); } template - PUGI__FN bool set_value_convert(String& dest, Header& header, uintptr_t header_mask, double value) + PUGI__FN bool set_value_convert(String& dest, Header& header, uintptr_t header_mask, double value, int precision) { char buf[128]; - PUGI__SNPRINTF(buf, "%.17g", value); + PUGI__SNPRINTF(buf, "%.*g", precision, value); return set_value_ascii(dest, header, header_mask, buf); } @@ -4688,6 +4708,7 @@ PUGI__NS_BEGIN char_t* buffer = 0; size_t length = 0; + // coverity[var_deref_model] if (!impl::convert_buffer(buffer, length, buffer_encoding, contents, size, is_mutable)) return impl::make_parse_result(status_out_of_memory); // delete original buffer if we performed a conversion @@ -4960,7 +4981,12 @@ PUGI__NS_BEGIN #if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__) || (defined(__MINGW32__) && (!defined(__STRICT_ANSI__) || defined(__MINGW64_VERSION_MAJOR))) PUGI__FN FILE* open_file_wide(const wchar_t* path, const wchar_t* mode) { +#if defined(PUGI__MSVC_CRT_VERSION) && PUGI__MSVC_CRT_VERSION >= 1400 + FILE* file = 0; + return _wfopen_s(&file, path, mode) == 0 ? file : 0; +#else return _wfopen(path, mode); +#endif } #else PUGI__FN char* convert_path_heap(const wchar_t* str) @@ -5004,6 +5030,16 @@ PUGI__NS_BEGIN } #endif + PUGI__FN FILE* open_file(const char* path, const char* mode) + { +#if defined(PUGI__MSVC_CRT_VERSION) && PUGI__MSVC_CRT_VERSION >= 1400 + FILE* file = 0; + return fopen_s(&file, path, mode) == 0 ? file : 0; +#else + return fopen(path, mode); +#endif + } + PUGI__FN bool save_file_impl(const xml_document& doc, FILE* file, const char_t* indent, unsigned int flags, xml_encoding encoding) { if (!file) return false; @@ -5329,14 +5365,28 @@ namespace pugi { if (!_attr) return false; - return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs); + return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, default_double_precision); + } + + PUGI__FN bool xml_attribute::set_value(double rhs, int precision) + { + if (!_attr) return false; + + return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, precision); } PUGI__FN bool xml_attribute::set_value(float rhs) { if (!_attr) return false; - return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs); + return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, default_float_precision); + } + + PUGI__FN bool xml_attribute::set_value(float rhs, int precision) + { + if (!_attr) return false; + + return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, precision); } PUGI__FN bool xml_attribute::set_value(bool rhs) @@ -6046,6 +6096,27 @@ namespace pugi return true; } + PUGI__FN bool xml_node::remove_attributes() + { + if (!_root) return false; + + impl::xml_allocator& alloc = impl::get_allocator(_root); + if (!alloc.reserve()) return false; + + for (xml_attribute_struct* attr = _root->first_attribute; attr; ) + { + xml_attribute_struct* next = attr->next_attribute; + + impl::destroy_attribute(attr, alloc); + + attr = next; + } + + _root->first_attribute = 0; + + return true; + } + PUGI__FN bool xml_node::remove_child(const char_t* name_) { return remove_child(child(name_)); @@ -6064,6 +6135,27 @@ namespace pugi return true; } + PUGI__FN bool xml_node::remove_children() + { + if (!_root) return false; + + impl::xml_allocator& alloc = impl::get_allocator(_root); + if (!alloc.reserve()) return false; + + for (xml_node_struct* cur = _root->first_child; cur; ) + { + xml_node_struct* next = cur->next_sibling; + + impl::destroy_node(cur, alloc); + + cur = next; + } + + _root->first_child = 0; + + return true; + } + PUGI__FN xml_parse_result xml_node::append_buffer(const void* contents, size_t size, unsigned int options, xml_encoding encoding) { // append_buffer is only valid for elements/documents @@ -6164,16 +6256,9 @@ namespace pugi PUGI__FN xml_node xml_node::first_element_by_path(const char_t* path_, char_t delimiter) const { - xml_node found = *this; // Current search context. + xml_node context = path_[0] == delimiter ? root() : *this; - if (!_root || !path_[0]) return found; - - if (path_[0] == delimiter) - { - // Absolute path; e.g. '/foo/bar' - found = found.root(); - ++path_; - } + if (!context._root) return xml_node(); const char_t* path_segment = path_; @@ -6183,19 +6268,19 @@ namespace pugi while (*path_segment_end && *path_segment_end != delimiter) ++path_segment_end; - if (path_segment == path_segment_end) return found; + if (path_segment == path_segment_end) return context; const char_t* next_segment = path_segment_end; while (*next_segment == delimiter) ++next_segment; if (*path_segment == '.' && path_segment + 1 == path_segment_end) - return found.first_element_by_path(next_segment, delimiter); + return context.first_element_by_path(next_segment, delimiter); else if (*path_segment == '.' && *(path_segment+1) == '.' && path_segment + 2 == path_segment_end) - return found.parent().first_element_by_path(next_segment, delimiter); + return context.parent().first_element_by_path(next_segment, delimiter); else { - for (xml_node_struct* j = found._root->first_child; j; j = j->next_sibling) + for (xml_node_struct* j = context._root->first_child; j; j = j->next_sibling) { if (j->name && impl::strequalrange(j->name, path_segment, static_cast(path_segment_end - path_segment))) { @@ -6490,14 +6575,28 @@ namespace pugi { xml_node_struct* dn = _data_new(); - return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false; + return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, default_float_precision) : false; + } + + PUGI__FN bool xml_text::set(float rhs, int precision) + { + xml_node_struct* dn = _data_new(); + + return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, precision) : false; } PUGI__FN bool xml_text::set(double rhs) { xml_node_struct* dn = _data_new(); - return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false; + return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, default_double_precision) : false; + } + + PUGI__FN bool xml_text::set(double rhs, int precision) + { + xml_node_struct* dn = _data_new(); + + return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, precision) : false; } PUGI__FN bool xml_text::set(bool rhs) @@ -6873,8 +6972,7 @@ namespace pugi { reset(); - for (xml_node cur = proto.first_child(); cur; cur = cur.next_sibling()) - append_copy(cur); + impl::node_copy_tree(_root, proto._root); } PUGI__FN void xml_document::_create() @@ -7104,7 +7202,7 @@ namespace pugi reset(); using impl::auto_deleter; // MSVC7 workaround - auto_deleter file(fopen(path_, "rb"), impl::close_file); + auto_deleter file(impl::open_file(path_, "rb"), impl::close_file); return impl::load_file_impl(static_cast(_root), file.data, options, encoding, &_buffer); } @@ -7187,7 +7285,7 @@ namespace pugi PUGI__FN bool xml_document::save_file(const char* path_, const char_t* indent, unsigned int flags, xml_encoding encoding) const { using impl::auto_deleter; // MSVC7 workaround - auto_deleter file(fopen(path_, (flags & format_save_file_text) ? "w" : "wb"), impl::close_file); + auto_deleter file(impl::open_file(path_, (flags & format_save_file_text) ? "w" : "wb"), impl::close_file); return impl::save_file_impl(*this, file.data, indent, flags, encoding); } @@ -7331,14 +7429,14 @@ PUGI__NS_BEGIN } }; - template void swap(T& lhs, T& rhs) + template inline void swap(T& lhs, T& rhs) { T temp = lhs; lhs = rhs; rhs = temp; } - template I min_element(I begin, I end, const Pred& pred) + template PUGI__FN I min_element(I begin, I end, const Pred& pred) { I result = begin; @@ -7349,17 +7447,20 @@ PUGI__NS_BEGIN return result; } - template void reverse(I begin, I end) + template PUGI__FN void reverse(I begin, I end) { - while (end - begin > 1) swap(*begin++, *--end); + while (end - begin > 1) + swap(*begin++, *--end); } - template I unique(I begin, I end) + template PUGI__FN I unique(I begin, I end) { // fast skip head - while (end - begin > 1 && *begin != *(begin + 1)) begin++; + while (end - begin > 1 && *begin != *(begin + 1)) + begin++; - if (begin == end) return begin; + if (begin == end) + return begin; // last written element I write = begin++; @@ -7377,7 +7478,7 @@ PUGI__NS_BEGIN return write + 1; } - template void insertion_sort(T* begin, T* end, const Pred& pred) + template PUGI__FN void insertion_sort(T* begin, T* end, const Pred& pred) { if (begin == end) return; @@ -7399,16 +7500,19 @@ PUGI__NS_BEGIN } } - template I median3(I first, I middle, I last, const Pred& pred) + template inline I median3(I first, I middle, I last, const Pred& pred) { - if (pred(*middle, *first)) swap(middle, first); - if (pred(*last, *middle)) swap(last, middle); - if (pred(*middle, *first)) swap(middle, first); + if (pred(*middle, *first)) + swap(middle, first); + if (pred(*last, *middle)) + swap(last, middle); + if (pred(*middle, *first)) + swap(middle, first); return middle; } - template void partition3(T* begin, T* end, T pivot, const Pred& pred, T** out_eqbeg, T** out_eqend) + template PUGI__FN void partition3(T* begin, T* end, T pivot, const Pred& pred, T** out_eqbeg, T** out_eqend) { // invariant: array is split into 4 groups: = < ? > (each variable denotes the boundary between the groups) T* eq = begin; @@ -7435,7 +7539,7 @@ PUGI__NS_BEGIN *out_eqend = gt; } - template void sort(I begin, I end, const Pred& pred) + template PUGI__FN void sort(I begin, I end, const Pred& pred) { // sort large chunks while (end - begin > 16) @@ -7464,6 +7568,41 @@ PUGI__NS_BEGIN // insertion sort small chunk insertion_sort(begin, end, pred); } + + PUGI__FN bool hash_insert(const void** table, size_t size, const void* key) + { + assert(key); + + unsigned int h = static_cast(reinterpret_cast(key)); + + // MurmurHash3 32-bit finalizer + h ^= h >> 16; + h *= 0x85ebca6bu; + h ^= h >> 13; + h *= 0xc2b2ae35u; + h ^= h >> 16; + + size_t hashmod = size - 1; + size_t bucket = h & hashmod; + + for (size_t probe = 0; probe <= hashmod; ++probe) + { + if (table[bucket] == 0) + { + table[bucket] = key; + return true; + } + + if (table[bucket] == key) + return false; + + // hash collision, quadratic probing + bucket = (bucket + probe + 1) & hashmod; + } + + assert(false && "Hash table is full"); // unreachable + return false; + } PUGI__NS_END // Allocator used for AST and evaluation stacks @@ -8053,15 +8192,6 @@ PUGI__NS_BEGIN } }; - struct duplicate_comparator - { - bool operator()(const xpath_node& lhs, const xpath_node& rhs) const - { - if (lhs.attribute()) return rhs.attribute() ? lhs.attribute() < rhs.attribute() : true; - else return rhs.attribute() ? false : lhs.node() < rhs.node(); - } - }; - PUGI__FN double gen_nan() { #if defined(__STDC_IEC_559__) || ((FLT_RADIX - 0 == 2) && (FLT_MAX_EXP - 0 == 128) && (FLT_MANT_DIG - 0 == 24)) @@ -8069,7 +8199,7 @@ PUGI__NS_BEGIN typedef uint32_t UI; // BCC5 workaround union { float f; UI i; } u; u.i = 0x7fc00000; - return u.f; + return double(u.f); #else // fallback const volatile double zero = 0.0; @@ -8849,12 +8979,42 @@ PUGI__NS_BEGIN _end = pos; } - void remove_duplicates() + void remove_duplicates(xpath_allocator* alloc) { - if (_type == xpath_node_set::type_unsorted) - sort(_begin, _end, duplicate_comparator()); + if (_type == xpath_node_set::type_unsorted && _end - _begin > 2) + { + xpath_allocator_capture cr(alloc); - _end = unique(_begin, _end); + size_t size_ = static_cast(_end - _begin); + + size_t hash_size = 1; + while (hash_size < size_ + size_ / 2) hash_size *= 2; + + const void** hash_data = static_cast(alloc->allocate(hash_size * sizeof(void**))); + if (!hash_data) return; + + memset(hash_data, 0, hash_size * sizeof(const void**)); + + xpath_node* write = _begin; + + for (xpath_node* it = _begin; it != _end; ++it) + { + const void* attr = it->attribute().internal_object(); + const void* node = it->node().internal_object(); + const void* key = attr ? attr : node; + + if (key && hash_insert(hash_data, hash_size, key)) + { + *write++ = *it; + } + } + + _end = write; + } + else + { + _end = unique(_begin, _end); + } } xpath_node_set::type_t type() const @@ -9611,7 +9771,7 @@ PUGI__NS_BEGIN { xpath_context c(*it, i, size); - if (expr->eval_number(c, stack) == i) + if (expr->eval_number(c, stack) == static_cast(i)) { *last++ = *it; @@ -9635,11 +9795,11 @@ PUGI__NS_BEGIN double er = expr->eval_number(c, stack); - if (er >= 1.0 && er <= size) + if (er >= 1.0 && er <= static_cast(size)) { size_t eri = static_cast(er); - if (er == eri) + if (er == static_cast(eri)) { xpath_node r = last[eri - 1]; @@ -10083,6 +10243,7 @@ PUGI__NS_BEGIN bool once = (axis == axis_attribute && _test == nodetest_name) || (!_right && eval_once(axis_type, eval)) || + // coverity[mixed_enums] (_right && !_right->_next && _right->_test == predicate_constant_one); xpath_node_set_raw ns; @@ -10115,7 +10276,7 @@ PUGI__NS_BEGIN // child, attribute and self axes always generate unique set of nodes // for other axis, if the set stayed sorted, it stayed unique because the traversal algorithms do not visit the same node twice if (axis != axis_child && axis != axis_attribute && axis != axis_self && ns.type() == xpath_node_set::type_unsorted) - ns.remove_duplicates(); + ns.remove_duplicates(stack.temp); return ns; } @@ -10275,35 +10436,38 @@ PUGI__NS_BEGIN if (_rettype == xpath_type_boolean) return _data.variable->get_boolean(); + + // variable needs to be converted to the correct type, this is handled by the fallthrough block below + break; } - // fallthrough default: - { - switch (_rettype) - { - case xpath_type_number: - return convert_number_to_boolean(eval_number(c, stack)); - - case xpath_type_string: - { - xpath_allocator_capture cr(stack.result); - - return !eval_string(c, stack).empty(); - } - - case xpath_type_node_set: - { - xpath_allocator_capture cr(stack.result); - - return !eval_node_set(c, stack, nodeset_eval_any).empty(); - } - - default: - assert(false && "Wrong expression for return type boolean"); // unreachable - return false; - } + ; } + + // none of the ast types that return the value directly matched, we need to perform type conversion + switch (_rettype) + { + case xpath_type_number: + return convert_number_to_boolean(eval_number(c, stack)); + + case xpath_type_string: + { + xpath_allocator_capture cr(stack.result); + + return !eval_string(c, stack).empty(); + } + + case xpath_type_node_set: + { + xpath_allocator_capture cr(stack.result); + + return !eval_node_set(c, stack, nodeset_eval_any).empty(); + } + + default: + assert(false && "Wrong expression for return type boolean"); // unreachable + return false; } } @@ -10410,36 +10574,38 @@ PUGI__NS_BEGIN if (_rettype == xpath_type_number) return _data.variable->get_number(); + + // variable needs to be converted to the correct type, this is handled by the fallthrough block below + break; } - // fallthrough default: - { - switch (_rettype) - { - case xpath_type_boolean: - return eval_boolean(c, stack) ? 1 : 0; - - case xpath_type_string: - { - xpath_allocator_capture cr(stack.result); - - return convert_string_to_number(eval_string(c, stack).c_str()); - } - - case xpath_type_node_set: - { - xpath_allocator_capture cr(stack.result); - - return convert_string_to_number(eval_string(c, stack).c_str()); - } - - default: - assert(false && "Wrong expression for return type number"); // unreachable - return 0; - } - + ; } + + // none of the ast types that return the value directly matched, we need to perform type conversion + switch (_rettype) + { + case xpath_type_boolean: + return eval_boolean(c, stack) ? 1 : 0; + + case xpath_type_string: + { + xpath_allocator_capture cr(stack.result); + + return convert_string_to_number(eval_string(c, stack).c_str()); + } + + case xpath_type_node_set: + { + xpath_allocator_capture cr(stack.result); + + return convert_string_to_number(eval_string(c, stack).c_str()); + } + + default: + assert(false && "Wrong expression for return type number"); // unreachable + return 0; } } @@ -10596,7 +10762,7 @@ PUGI__NS_BEGIN double first = round_nearest(_right->eval_number(c, stack)); if (is_nan(first)) return xpath_string(); // NaN - else if (first >= s_length + 1) return xpath_string(); + else if (first >= static_cast(s_length + 1)) return xpath_string(); size_t pos = first < 1 ? 1 : static_cast(first); assert(1 <= pos && pos <= s_length + 1); @@ -10620,12 +10786,12 @@ PUGI__NS_BEGIN double last = first + round_nearest(_right->_next->eval_number(c, stack)); if (is_nan(first) || is_nan(last)) return xpath_string(); - else if (first >= s_length + 1) return xpath_string(); + else if (first >= static_cast(s_length + 1)) return xpath_string(); else if (first >= last) return xpath_string(); else if (last < 1) return xpath_string(); size_t pos = first < 1 ? 1 : static_cast(first); - size_t end = last >= s_length + 1 ? s_length + 1 : static_cast(last); + size_t end = last >= static_cast(s_length + 1) ? s_length + 1 : static_cast(last); assert(1 <= pos && pos <= end && end <= s_length + 1); const char_t* rbegin = s.c_str() + (pos - 1); @@ -10694,34 +10860,37 @@ PUGI__NS_BEGIN if (_rettype == xpath_type_string) return xpath_string::from_const(_data.variable->get_string()); + + // variable needs to be converted to the correct type, this is handled by the fallthrough block below + break; } - // fallthrough default: - { - switch (_rettype) - { - case xpath_type_boolean: - return xpath_string::from_const(eval_boolean(c, stack) ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false")); - - case xpath_type_number: - return convert_number_to_string(eval_number(c, stack), stack.result); - - case xpath_type_node_set: - { - xpath_allocator_capture cr(stack.temp); - - xpath_stack swapped_stack = {stack.temp, stack.result}; - - xpath_node_set_raw ns = eval_node_set(c, swapped_stack, nodeset_eval_first); - return ns.empty() ? xpath_string() : string_value(ns.first(), stack.result); - } - - default: - assert(false && "Wrong expression for return type string"); // unreachable - return xpath_string(); - } + ; } + + // none of the ast types that return the value directly matched, we need to perform type conversion + switch (_rettype) + { + case xpath_type_boolean: + return xpath_string::from_const(eval_boolean(c, stack) ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false")); + + case xpath_type_number: + return convert_number_to_string(eval_number(c, stack), stack.result); + + case xpath_type_node_set: + { + xpath_allocator_capture cr(stack.temp); + + xpath_stack swapped_stack = {stack.temp, stack.result}; + + xpath_node_set_raw ns = eval_node_set(c, swapped_stack, nodeset_eval_first); + return ns.empty() ? xpath_string() : string_value(ns.first(), stack.result); + } + + default: + assert(false && "Wrong expression for return type string"); // unreachable + return xpath_string(); } } @@ -10735,16 +10904,16 @@ PUGI__NS_BEGIN xpath_stack swapped_stack = {stack.temp, stack.result}; - xpath_node_set_raw ls = _left->eval_node_set(c, swapped_stack, eval); - xpath_node_set_raw rs = _right->eval_node_set(c, stack, eval); + xpath_node_set_raw ls = _left->eval_node_set(c, stack, eval); + xpath_node_set_raw rs = _right->eval_node_set(c, swapped_stack, eval); // we can optimize merging two sorted sets, but this is a very rare operation, so don't bother - rs.set_type(xpath_node_set::type_unsorted); + ls.set_type(xpath_node_set::type_unsorted); - rs.append(ls.begin(), ls.end(), stack.result); - rs.remove_duplicates(); + ls.append(rs.begin(), rs.end(), stack.result); + ls.remove_duplicates(stack.temp); - return rs; + return ls; } case ast_filter: @@ -10843,13 +11012,18 @@ PUGI__NS_BEGIN return ns; } + + // variable needs to be converted to the correct type, this is handled by the fallthrough block below + break; } - // fallthrough default: - assert(false && "Wrong expression for return type node set"); // unreachable - return xpath_node_set_raw(); + ; } + + // none of the ast types that return the value directly matched, but conversions to node set are invalid + assert(false && "Wrong expression for return type node set"); // unreachable + return xpath_node_set_raw(); } void optimize(xpath_allocator* alloc) @@ -10863,6 +11037,7 @@ PUGI__NS_BEGIN if (_next) _next->optimize(alloc); + // coverity[var_deref_model] optimize_self(alloc); } @@ -10871,13 +11046,14 @@ PUGI__NS_BEGIN // Rewrite [position()=expr] with [expr] // Note that this step has to go before classification to recognize [position()=1] if ((_type == ast_filter || _type == ast_predicate) && + _right && // workaround for clang static analyzer (_right is never null for ast_filter/ast_predicate) _right->_type == ast_op_equal && _right->_left->_type == ast_func_position && _right->_right->_rettype == xpath_type_number) { _right = _right->_right; } // Classify filter/predicate ops to perform various optimizations during evaluation - if (_type == ast_filter || _type == ast_predicate) + if ((_type == ast_filter || _type == ast_predicate) && _right) // workaround for clang static analyzer (_right is never null for ast_filter/ast_predicate) { assert(_test == predicate_default); @@ -10893,8 +11069,8 @@ PUGI__NS_BEGIN // The former is a full form of //foo, the latter is much faster since it executes the node test immediately // Do a similar kind of rewrite for self/descendant/descendant-or-self axes // Note that we only rewrite positionally invariant steps (//foo[1] != /descendant::foo[1]) - if (_type == ast_step && (_axis == axis_child || _axis == axis_self || _axis == axis_descendant || _axis == axis_descendant_or_self) && _left && - _left->_type == ast_step && _left->_axis == axis_descendant_or_self && _left->_test == nodetest_type_node && !_left->_right && + if (_type == ast_step && (_axis == axis_child || _axis == axis_self || _axis == axis_descendant || _axis == axis_descendant_or_self) && + _left && _left->_type == ast_step && _left->_axis == axis_descendant_or_self && _left->_test == nodetest_type_node && !_left->_right && is_posinv_step()) { if (_axis == axis_child || _axis == axis_descendant) @@ -10906,7 +11082,9 @@ PUGI__NS_BEGIN } // Use optimized lookup table implementation for translate() with constant arguments - if (_type == ast_func_translate && _right->_type == ast_string_constant && _right->_next->_type == ast_string_constant) + if (_type == ast_func_translate && + _right && // workaround for clang static analyzer (_right is never null for ast_func_translate) + _right->_type == ast_string_constant && _right->_next->_type == ast_string_constant) { unsigned char* table = translate_table_generate(alloc, _right->_data.string, _right->_next->_data.string); @@ -10919,6 +11097,8 @@ PUGI__NS_BEGIN // Use optimized path for @attr = 'value' or @attr = $value if (_type == ast_op_equal && + _left && _right && // workaround for clang static analyzer and Coverity (_left and _right are never null for ast_op_equal) + // coverity[mixed_enums] _left->_type == ast_step && _left->_axis == axis_attribute && _left->_test == nodetest_name && !_left->_left && !_left->_right && (_right->_type == ast_string_constant || (_right->_type == ast_variable && _right->_rettype == xpath_type_string))) { @@ -10978,6 +11158,14 @@ PUGI__NS_BEGIN } }; + static const size_t xpath_ast_depth_limit = + #ifdef PUGIXML_XPATH_DEPTH_LIMIT + PUGIXML_XPATH_DEPTH_LIMIT + #else + 1024 + #endif + ; + struct xpath_parser { xpath_allocator* _alloc; @@ -10990,6 +11178,8 @@ PUGI__NS_BEGIN char_t _scratch[32]; + size_t _depth; + xpath_ast_node* error(const char* message) { _result->error = message; @@ -11006,6 +11196,11 @@ PUGI__NS_BEGIN return 0; } + xpath_ast_node* error_rec() + { + return error("Exceeded maximum allowed query depth"); + } + void* alloc_node() { return _alloc->allocate(sizeof(xpath_ast_node)); @@ -11361,6 +11556,8 @@ PUGI__NS_BEGIN return error("Unrecognized function call"); _lexer.next(); + size_t old_depth = _depth; + while (_lexer.current() != lex_close_brace) { if (argc > 0) @@ -11370,6 +11567,9 @@ PUGI__NS_BEGIN _lexer.next(); } + if (++_depth > xpath_ast_depth_limit) + return error_rec(); + xpath_ast_node* n = parse_expression(); if (!n) return 0; @@ -11382,6 +11582,8 @@ PUGI__NS_BEGIN _lexer.next(); + _depth = old_depth; + return parse_function(function, argc, args); } @@ -11398,10 +11600,15 @@ PUGI__NS_BEGIN xpath_ast_node* n = parse_primary_expression(); if (!n) return 0; + size_t old_depth = _depth; + while (_lexer.current() == lex_open_square_brace) { _lexer.next(); + if (++_depth > xpath_ast_depth_limit) + return error_rec(); + if (n->rettype() != xpath_type_node_set) return error("Predicate has to be applied to node set"); @@ -11417,6 +11624,8 @@ PUGI__NS_BEGIN _lexer.next(); } + _depth = old_depth; + return n; } @@ -11568,12 +11777,17 @@ PUGI__NS_BEGIN xpath_ast_node* n = alloc_node(ast_step, set, axis, nt_type, nt_name_copy); if (!n) return 0; + size_t old_depth = _depth; + xpath_ast_node* last = 0; while (_lexer.current() == lex_open_square_brace) { _lexer.next(); + if (++_depth > xpath_ast_depth_limit) + return error_rec(); + xpath_ast_node* expr = parse_expression(); if (!expr) return 0; @@ -11590,6 +11804,8 @@ PUGI__NS_BEGIN last = pred; } + _depth = old_depth; + return n; } @@ -11599,11 +11815,16 @@ PUGI__NS_BEGIN xpath_ast_node* n = parse_step(set); if (!n) return 0; + size_t old_depth = _depth; + while (_lexer.current() == lex_slash || _lexer.current() == lex_double_slash) { lexeme_t l = _lexer.current(); _lexer.next(); + if (++_depth > xpath_ast_depth_limit) + return error_rec(); + if (l == lex_double_slash) { n = alloc_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0); @@ -11614,6 +11835,8 @@ PUGI__NS_BEGIN if (!n) return 0; } + _depth = old_depth; + return n; } @@ -11799,6 +12022,9 @@ PUGI__NS_BEGIN { _lexer.next(); + if (++_depth > xpath_ast_depth_limit) + return error_rec(); + xpath_ast_node* rhs = parse_path_or_unary_expression(); if (!rhs) return 0; @@ -11844,13 +12070,22 @@ PUGI__NS_BEGIN // | MultiplicativeExpr 'mod' UnaryExpr xpath_ast_node* parse_expression(int limit = 0) { + size_t old_depth = _depth; + + if (++_depth > xpath_ast_depth_limit) + return error_rec(); + xpath_ast_node* n = parse_path_or_unary_expression(); if (!n) return 0; - return parse_expression_rec(n, limit); + n = parse_expression_rec(n, limit); + + _depth = old_depth; + + return n; } - xpath_parser(const char_t* query, xpath_variable_set* variables, xpath_allocator* alloc, xpath_parse_result* result): _alloc(alloc), _lexer(query), _query(query), _variables(variables), _result(result) + xpath_parser(const char_t* query, xpath_variable_set* variables, xpath_allocator* alloc, xpath_parse_result* result): _alloc(alloc), _lexer(query), _query(query), _variables(variables), _result(result), _depth(0) { } @@ -11859,6 +12094,8 @@ PUGI__NS_BEGIN xpath_ast_node* n = parse_expression(); if (!n) return 0; + assert(_depth == 0); + // check if there are unparsed tokens left if (_lexer.current() != lex_eof) return error("Incorrect query"); @@ -12013,74 +12250,61 @@ namespace pugi size_t size_ = static_cast(end_ - begin_); - if (size_ <= 1) + // use internal buffer for 0 or 1 elements, heap buffer otherwise + xpath_node* storage = (size_ <= 1) ? _storage : static_cast(impl::xml_memory::allocate(size_ * sizeof(xpath_node))); + + if (!storage) { - // deallocate old buffer - if (_begin != &_storage) impl::xml_memory::deallocate(_begin); - - // use internal buffer - if (begin_ != end_) _storage = *begin_; - - _begin = &_storage; - _end = &_storage + size_; - _type = type_; + #ifdef PUGIXML_NO_EXCEPTIONS + return; + #else + throw std::bad_alloc(); + #endif } - else - { - // make heap copy - xpath_node* storage = static_cast(impl::xml_memory::allocate(size_ * sizeof(xpath_node))); - if (!storage) - { - #ifdef PUGIXML_NO_EXCEPTIONS - return; - #else - throw std::bad_alloc(); - #endif - } + // deallocate old buffer + if (_begin != _storage) + impl::xml_memory::deallocate(_begin); + // size check is necessary because for begin_ = end_ = nullptr, memcpy is UB + if (size_) memcpy(storage, begin_, size_ * sizeof(xpath_node)); - // deallocate old buffer - if (_begin != &_storage) impl::xml_memory::deallocate(_begin); - - // finalize - _begin = storage; - _end = storage + size_; - _type = type_; - } + _begin = storage; + _end = storage + size_; + _type = type_; } #ifdef PUGIXML_HAS_MOVE PUGI__FN void xpath_node_set::_move(xpath_node_set& rhs) PUGIXML_NOEXCEPT { _type = rhs._type; - _storage = rhs._storage; - _begin = (rhs._begin == &rhs._storage) ? &_storage : rhs._begin; + _storage[0] = rhs._storage[0]; + _begin = (rhs._begin == rhs._storage) ? _storage : rhs._begin; _end = _begin + (rhs._end - rhs._begin); rhs._type = type_unsorted; - rhs._begin = &rhs._storage; - rhs._end = rhs._begin; + rhs._begin = rhs._storage; + rhs._end = rhs._storage; } #endif - PUGI__FN xpath_node_set::xpath_node_set(): _type(type_unsorted), _begin(&_storage), _end(&_storage) + PUGI__FN xpath_node_set::xpath_node_set(): _type(type_unsorted), _begin(_storage), _end(_storage) { } - PUGI__FN xpath_node_set::xpath_node_set(const_iterator begin_, const_iterator end_, type_t type_): _type(type_unsorted), _begin(&_storage), _end(&_storage) + PUGI__FN xpath_node_set::xpath_node_set(const_iterator begin_, const_iterator end_, type_t type_): _type(type_unsorted), _begin(_storage), _end(_storage) { _assign(begin_, end_, type_); } PUGI__FN xpath_node_set::~xpath_node_set() { - if (_begin != &_storage) + if (_begin != _storage) impl::xml_memory::deallocate(_begin); } - PUGI__FN xpath_node_set::xpath_node_set(const xpath_node_set& ns): _type(type_unsorted), _begin(&_storage), _end(&_storage) + PUGI__FN xpath_node_set::xpath_node_set(const xpath_node_set& ns): _type(type_unsorted), _begin(_storage), _end(_storage) { _assign(ns._begin, ns._end, ns._type); } @@ -12095,7 +12319,7 @@ namespace pugi } #ifdef PUGIXML_HAS_MOVE - PUGI__FN xpath_node_set::xpath_node_set(xpath_node_set&& rhs) PUGIXML_NOEXCEPT: _type(type_unsorted), _begin(&_storage), _end(&_storage) + PUGI__FN xpath_node_set::xpath_node_set(xpath_node_set&& rhs) PUGIXML_NOEXCEPT: _type(type_unsorted), _begin(_storage), _end(_storage) { _move(rhs); } @@ -12104,7 +12328,7 @@ namespace pugi { if (this == &rhs) return *this; - if (_begin != &_storage) + if (_begin != _storage) impl::xml_memory::deallocate(_begin); _move(rhs); @@ -12771,7 +12995,7 @@ namespace pugi #endif /** - * Copyright (c) 2006-2018 Arseny Kapoulkine + * Copyright (c) 2006-2020 Arseny Kapoulkine * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation diff --git a/contrib/pugixml/src/pugixml.hpp b/contrib/pugixml/src/pugixml.hpp index 1775600f1..7e2ce7776 100644 --- a/contrib/pugixml/src/pugixml.hpp +++ b/contrib/pugixml/src/pugixml.hpp @@ -1,8 +1,8 @@ /** - * pugixml parser - version 1.9 + * pugixml parser - version 1.11 * -------------------------------------------------------- - * Copyright (C) 2006-2018, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com) - * Report bugs and download new versions at http://pugixml.org/ + * Copyright (C) 2006-2020, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com) + * Report bugs and download new versions at https://pugixml.org/ * * This library is distributed under the MIT License. See notice at the end * of this file. @@ -12,8 +12,9 @@ */ #ifndef PUGIXML_VERSION -// Define version macro; evaluates to major * 100 + minor so that it's safe to use in less-than comparisons -# define PUGIXML_VERSION 190 +// Define version macro; evaluates to major * 1000 + minor * 10 + patch so that it's safe to use in less-than comparisons +// Note: pugixml used major * 100 + minor * 10 + patch format up until 1.9 (which had version identifier 190); starting from pugixml 1.10, the minor version number is two digits +# define PUGIXML_VERSION 1110 #endif // Include user configuration file (this can define various configuration macros) @@ -110,6 +111,15 @@ # endif #endif +// If C++ is 2011 or higher, use 'nullptr' +#ifndef PUGIXML_NULL +# if __cplusplus >= 201103 +# define PUGIXML_NULL nullptr +# else +# define PUGIXML_NULL 0 +# endif +#endif + // Character interface macros #ifdef PUGIXML_WCHAR_MODE # define PUGIXML_TEXT(t) L ## t @@ -252,10 +262,19 @@ namespace pugi // Don't output empty element tags, instead writing an explicit start and end tag even if there are no children. This flag is off by default. const unsigned int format_no_empty_element_tags = 0x80; + // Skip characters belonging to range [0; 32) instead of "&#xNN;" encoding. This flag is off by default. + const unsigned int format_skip_control_chars = 0x100; + + // Use single quotes ' instead of double quotes " for enclosing attribute values. This flag is off by default. + const unsigned int format_attribute_single_quote = 0x200; + // The default set of formatting flags. // Nodes are indented depending on their depth in DOM tree, a default declaration is output if document has none. const unsigned int format_default = format_indent; + const int default_double_precision = 17; + const int default_float_precision = 9; + // Forward declarations struct xml_attribute_struct; struct xml_node_struct; @@ -403,7 +422,9 @@ namespace pugi bool set_value(long rhs); bool set_value(unsigned long rhs); bool set_value(double rhs); + bool set_value(double rhs, int precision); bool set_value(float rhs); + bool set_value(float rhs, int precision); bool set_value(bool rhs); #ifdef PUGIXML_HAS_LONG_LONG @@ -569,10 +590,16 @@ namespace pugi bool remove_attribute(const xml_attribute& a); bool remove_attribute(const char_t* name); + // Remove all attributes + bool remove_attributes(); + // Remove specified child bool remove_child(const xml_node& n); bool remove_child(const char_t* name); + // Remove all children + bool remove_children(); + // Parses buffer as an XML document fragment and appends all nodes as children of the current node. // Copies/converts the buffer, so it may be deleted or changed after the function returns. // Note: append_buffer allocates memory that has the lifetime of the owning document; removing the appended nodes does not immediately reclaim that memory. @@ -643,15 +670,15 @@ namespace pugi #ifndef PUGIXML_NO_XPATH // Select single node by evaluating XPath query. Returns first node from the resulting node set. - xpath_node select_node(const char_t* query, xpath_variable_set* variables = 0) const; + xpath_node select_node(const char_t* query, xpath_variable_set* variables = PUGIXML_NULL) const; xpath_node select_node(const xpath_query& query) const; // Select node set by evaluating XPath query - xpath_node_set select_nodes(const char_t* query, xpath_variable_set* variables = 0) const; + xpath_node_set select_nodes(const char_t* query, xpath_variable_set* variables = PUGIXML_NULL) const; xpath_node_set select_nodes(const xpath_query& query) const; // (deprecated: use select_node instead) Select single node by evaluating XPath query. - PUGIXML_DEPRECATED xpath_node select_single_node(const char_t* query, xpath_variable_set* variables = 0) const; + PUGIXML_DEPRECATED xpath_node select_single_node(const char_t* query, xpath_variable_set* variables = PUGIXML_NULL) const; PUGIXML_DEPRECATED xpath_node select_single_node(const xpath_query& query) const; #endif @@ -754,7 +781,9 @@ namespace pugi bool set(long rhs); bool set(unsigned long rhs); bool set(double rhs); + bool set(double rhs, int precision); bool set(float rhs); + bool set(float rhs, int precision); bool set(bool rhs); #ifdef PUGIXML_HAS_LONG_LONG @@ -1192,7 +1221,7 @@ namespace pugi public: // Construct a compiled object from XPath expression. // If PUGIXML_NO_EXCEPTIONS is not defined, throws xpath_exception on compilation errors. - explicit xpath_query(const char_t* query, xpath_variable_set* variables = 0); + explicit xpath_query(const char_t* query, xpath_variable_set* variables = PUGIXML_NULL); // Constructor xpath_query(); @@ -1251,11 +1280,12 @@ namespace pugi }; #ifndef PUGIXML_NO_EXCEPTIONS - -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning( disable: 4275 ) -#endif + #if defined(_MSC_VER) + // C4275 can be ignored in Visual C++ if you are deriving + // from a type in the Standard C++ Library + #pragma warning(push) + #pragma warning(disable: 4275) + #endif // XPath exception class class PUGIXML_CLASS xpath_exception: public std::exception { @@ -1272,10 +1302,11 @@ namespace pugi // Get parse result const xpath_parse_result& result() const; }; + #if defined(_MSC_VER) + #pragma warning(pop) + #endif #endif -#ifdef _MSC_VER -# pragma warning(pop) -#endif + // XPath node class (either xml_node or xml_attribute) class PUGIXML_CLASS xpath_node { @@ -1379,7 +1410,7 @@ namespace pugi private: type_t _type; - xpath_node _storage; + xpath_node _storage[1]; xpath_node* _begin; xpath_node* _end; @@ -1443,7 +1474,7 @@ namespace std #endif /** - * Copyright (c) 2006-2018 Arseny Kapoulkine + * Copyright (c) 2006-2020 Arseny Kapoulkine * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation From bf0f8d4c1b2f53a457b4e1834abf7f841ef30300 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Thu, 3 Jun 2021 23:29:53 +0200 Subject: [PATCH 34/34] Update pugiconfig.hpp --- contrib/pugixml/src/pugiconfig.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pugixml/src/pugiconfig.hpp b/contrib/pugixml/src/pugiconfig.hpp index 405a66b65..03f854bf1 100644 --- a/contrib/pugixml/src/pugiconfig.hpp +++ b/contrib/pugixml/src/pugiconfig.hpp @@ -44,7 +44,7 @@ // #define PUGIXML_XPATH_DEPTH_LIMIT 1024 // Uncomment this to switch to header-only version -// #define PUGIXML_HEADER_ONLY +#define PUGIXML_HEADER_ONLY // Uncomment this to enable long long support // #define PUGIXML_HAS_LONG_LONG