From 971ba308b38b9c42617331dd37383ac948eb7506 Mon Sep 17 00:00:00 2001 From: Mike Samsonov Date: Wed, 23 Jan 2019 17:07:44 +0000 Subject: [PATCH] Merged PR 2811682: buffer grow changes and large files support Buffer grow changes: The exporting of relatevely large data could take a few days, because reallocation of a buffer every time for a few new bytes is overkill. I've introduced standard capacity member for the buffer and growth it by 1.5 times every time. That helps a lot, descrease exporting to a minute (from a few days). Large file support: glTF is a json file, all lengths and offsets don't have a type, just numbers, but code was always reading it as uint32, this doesn't work for files bigger than int32 (almost all files we have as an example). So, I've changed it to be reading as size_t. Specification doesn't specify the type for it, so it's not against spec. --- code/glTF2Asset.h | 5 +++-- code/glTF2Asset.inl | 28 +++++++++++++++++++++++----- code/glTF2Exporter.cpp | 6 +++--- code/glTF2Importer.cpp | 24 ++++++++++++------------ code/glTFAsset.h | 2 +- code/glTFAsset.inl | 28 +++++++++++++++++++++++----- 6 files changed, 65 insertions(+), 28 deletions(-) diff --git a/code/glTF2Asset.h b/code/glTF2Asset.h index 897db7474..0015197c2 100644 --- a/code/glTF2Asset.h +++ b/code/glTF2Asset.h @@ -430,9 +430,9 @@ namespace glTF2 struct Accessor : public Object { Ref bufferView; //!< The ID of the bufferView. (required) - unsigned int byteOffset; //!< The offset relative to the start of the bufferView in bytes. (required) + size_t byteOffset; //!< The offset relative to the start of the bufferView in bytes. (required) ComponentType componentType; //!< The datatype of components in the attribute. (required) - unsigned int count; //!< The number of attributes referenced by this accessor. (required) + size_t count; //!< The number of attributes referenced by this accessor. (required) AttribType::Value type; //!< Specifies if the attribute is a scalar, vector, or matrix. (required) std::vector max; //!< Maximum value of each component in this attribute. std::vector min; //!< Minimum value of each component in this attribute. @@ -529,6 +529,7 @@ namespace glTF2 //std::string uri; //!< The uri of the buffer. Can be a filepath, a data uri, etc. (required) size_t byteLength; //!< The length of the buffer in bytes. (default: 0) //std::string type; //!< XMLHttpRequest responseType (default: "arraybuffer") + size_t capacity = 0; //!< The capacity of the buffer in bytes. (default: 0) Type type; diff --git a/code/glTF2Asset.inl b/code/glTF2Asset.inl index 97ad6df57..b51975c77 100755 --- a/code/glTF2Asset.inl +++ b/code/glTF2Asset.inl @@ -85,6 +85,14 @@ namespace { return val.IsString() ? (out = std::string(val.GetString(), val.GetStringLength()), true) : false; }}; + template<> struct ReadHelper { static bool Read(Value& val, uint64_t& out) { + return val.IsUint64() ? out = val.GetUint64(), true : false; + }}; + + template<> struct ReadHelper { static bool Read(Value& val, int64_t& out) { + return val.IsInt64() ? out = val.GetInt64(), true : false; + }}; + template struct ReadHelper< Nullable > { static bool Read(Value& val, Nullable& out) { return out.isPresent = ReadHelper::Read(val, out.value); }}; @@ -520,7 +528,17 @@ inline size_t Buffer::AppendData(uint8_t* data, size_t length) inline void Buffer::Grow(size_t amount) { if (amount <= 0) return; - uint8_t* b = new uint8_t[byteLength + amount]; + if (capacity >= byteLength + amount) + { + byteLength += amount; + return; + } + + // Shift operation is standard way to divide integer by 2, it doesn't cast it to float back and forth, also works for odd numbers, + // originally it would look like: static_cast(capacity * 1.5f) + capacity = std::max(capacity + (capacity >> 1), byteLength + amount); + + uint8_t* b = new uint8_t[capacity]; if (mData) memcpy(b, mData.get(), byteLength); mData.reset(b, std::default_delete()); byteLength += amount; @@ -537,8 +555,8 @@ inline void BufferView::Read(Value& obj, Asset& r) buffer = r.buffers.Retrieve(bufferVal->GetUint()); } - byteOffset = MemberOrDefault(obj, "byteOffset", 0u); - byteLength = MemberOrDefault(obj, "byteLength", 0u); + byteOffset = MemberOrDefault(obj, "byteOffset", size_t(0)); + byteLength = MemberOrDefault(obj, "byteLength", size_t(0)); byteStride = MemberOrDefault(obj, "byteStride", 0u); } @@ -553,9 +571,9 @@ inline void Accessor::Read(Value& obj, Asset& r) bufferView = r.bufferViews.Retrieve(bufferViewVal->GetUint()); } - byteOffset = MemberOrDefault(obj, "byteOffset", 0u); + byteOffset = MemberOrDefault(obj, "byteOffset", size_t(0)); componentType = MemberOrDefault(obj, "componentType", ComponentType_BYTE); - count = MemberOrDefault(obj, "count", 0u); + count = MemberOrDefault(obj, "count", size_t(0)); const char* typestr; type = ReadMember(obj, "type", typestr) ? AttribType::FromString(typestr) : AttribType::SCALAR; diff --git a/code/glTF2Exporter.cpp b/code/glTF2Exporter.cpp index 01e606ed3..655b4fc7d 100644 --- a/code/glTF2Exporter.cpp +++ b/code/glTF2Exporter.cpp @@ -156,7 +156,7 @@ static void IdentityMatrix4(mat4& o) { } inline Ref ExportData(Asset& a, std::string& meshName, Ref& buffer, - unsigned int count, void* data, AttribType::Value typeIn, AttribType::Value typeOut, ComponentType compType, bool isIndices = false) + size_t count, void* data, AttribType::Value typeIn, AttribType::Value typeOut, ComponentType compType, bool isIndices = false) { if (!count || !data) { return Ref(); @@ -176,7 +176,7 @@ inline Ref ExportData(Asset& a, std::string& meshName, Ref& bu // bufferView Ref bv = a.bufferViews.Create(a.FindUniqueID(meshName, "view")); bv->buffer = buffer; - bv->byteOffset = unsigned(offset); + bv->byteOffset = offset; bv->byteLength = length; //! The target that the WebGL buffer should be bound to. bv->byteStride = 0; bv->target = isIndices ? BufferViewTarget_ELEMENT_ARRAY_BUFFER : BufferViewTarget_ARRAY_BUFFER; @@ -768,7 +768,7 @@ void glTF2Exporter::ExportMeshes() } } - p.indices = ExportData(*mAsset, meshId, b, unsigned(indices.size()), &indices[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_UNSIGNED_INT, true); + p.indices = ExportData(*mAsset, meshId, b, indices.size(), &indices[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_UNSIGNED_INT, true); } switch (aim->mPrimitiveTypes) { diff --git a/code/glTF2Importer.cpp b/code/glTF2Importer.cpp index be3a9a96f..0eafe227f 100755 --- a/code/glTF2Importer.cpp +++ b/code/glTF2Importer.cpp @@ -412,7 +412,7 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r) Mesh::Primitive::Attributes& attr = prim.attributes; if (attr.position.size() > 0 && attr.position[0]) { - aim->mNumVertices = attr.position[0]->count; + aim->mNumVertices = static_cast(attr.position[0]->count); attr.position[0]->ExtractData(aim->mVertices); } @@ -511,10 +511,10 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r) aiFace* faces = 0; - unsigned int nFaces = 0; + size_t nFaces = 0; if (prim.indices) { - unsigned int count = prim.indices->count; + size_t count = prim.indices->count; Accessor::Indexer data = prim.indices->GetIndexer(); ai_assert(data.IsValid()); @@ -665,7 +665,7 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r) if (faces) { aim->mFaces = faces; - aim->mNumFaces = nFaces; + aim->mNumFaces = static_cast(nFaces); ai_assert(CheckValidFacesIndices(faces, nFaces, aim->mNumVertices)); } @@ -751,7 +751,7 @@ static void BuildVertexWeightMapping(Mesh::Primitive& primitive, std::vectorcount; + size_t num_vertices = attr.weight[0]->count; struct Weights { float values[4]; }; Weights* weights = nullptr; @@ -822,7 +822,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector& if (node.skin) { for (int primitiveNo = 0; primitiveNo < count; ++primitiveNo) { aiMesh* mesh = pScene->mMeshes[meshOffsets[mesh_idx]+primitiveNo]; - mesh->mNumBones = node.skin->jointNames.size(); + mesh->mNumBones = static_cast(node.skin->jointNames.size()); mesh->mBones = new aiBone*[mesh->mNumBones]; // GLTF and Assimp choose to store bone weights differently. @@ -837,7 +837,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector& std::vector> weighting(mesh->mNumBones); BuildVertexWeightMapping(node.meshes[0]->primitives[primitiveNo], weighting); - for (size_t i = 0; i < mesh->mNumBones; ++i) { + for (uint32_t i = 0; i < mesh->mNumBones; ++i) { aiBone* bone = new aiBone(); Ref joint = node.skin->jointNames[i]; @@ -854,7 +854,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector& std::vector& weights = weighting[i]; - bone->mNumWeights = weights.size(); + bone->mNumWeights = static_cast(weights.size()); if (bone->mNumWeights > 0) { bone->mWeights = new aiVertexWeight[bone->mNumWeights]; memcpy(bone->mWeights, weights.data(), bone->mNumWeights * sizeof(aiVertexWeight)); @@ -930,7 +930,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl samplers.translation->input->ExtractData(times); aiVector3D* values = nullptr; samplers.translation->output->ExtractData(values); - anim->mNumPositionKeys = samplers.translation->input->count; + anim->mNumPositionKeys = static_cast(samplers.translation->input->count); anim->mPositionKeys = new aiVectorKey[anim->mNumPositionKeys]; for (unsigned int i = 0; i < anim->mNumPositionKeys; ++i) { anim->mPositionKeys[i].mTime = times[i] * kMillisecondsFromSeconds; @@ -952,7 +952,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl samplers.rotation->input->ExtractData(times); aiQuaternion* values = nullptr; samplers.rotation->output->ExtractData(values); - anim->mNumRotationKeys = samplers.rotation->input->count; + anim->mNumRotationKeys = static_cast(samplers.rotation->input->count); anim->mRotationKeys = new aiQuatKey[anim->mNumRotationKeys]; for (unsigned int i = 0; i < anim->mNumRotationKeys; ++i) { anim->mRotationKeys[i].mTime = times[i] * kMillisecondsFromSeconds; @@ -978,7 +978,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl samplers.scale->input->ExtractData(times); aiVector3D* values = nullptr; samplers.scale->output->ExtractData(values); - anim->mNumScalingKeys = samplers.scale->input->count; + anim->mNumScalingKeys = static_cast(samplers.scale->input->count); anim->mScalingKeys = new aiVectorKey[anim->mNumScalingKeys]; for (unsigned int i = 0; i < anim->mNumScalingKeys; ++i) { anim->mScalingKeys[i].mTime = times[i] * kMillisecondsFromSeconds; @@ -1042,7 +1042,7 @@ void glTF2Importer::ImportAnimations(glTF2::Asset& r) std::unordered_map samplers = GatherSamplers(anim); - ai_anim->mNumChannels = samplers.size(); + ai_anim->mNumChannels = static_cast(samplers.size()); if (ai_anim->mNumChannels > 0) { ai_anim->mChannels = new aiNodeAnim*[ai_anim->mNumChannels]; int j = 0; diff --git a/code/glTFAsset.h b/code/glTFAsset.h index 463c69be4..92b7dd9d1 100644 --- a/code/glTFAsset.h +++ b/code/glTFAsset.h @@ -537,7 +537,7 @@ namespace glTF shared_ptr mData; //!< Pointer to the data bool mIsSpecial; //!< Set to true for special cases (e.g. the body buffer) - + size_t capacity = 0; //!< The capacity of the buffer in bytes. (default: 0) /// \var EncodedRegion_List /// List of encoded regions. std::list EncodedRegion_List; diff --git a/code/glTFAsset.inl b/code/glTFAsset.inl index f545b809c..7b7acd705 100644 --- a/code/glTFAsset.inl +++ b/code/glTFAsset.inl @@ -95,6 +95,14 @@ namespace { return out.isPresent = ReadHelper::Read(val, out.value); }}; + template<> struct ReadHelper { static bool Read(Value& val, uint64_t& out) { + return val.IsUint64() ? out = val.GetUint64(), true : false; + }}; + + template<> struct ReadHelper { static bool Read(Value& val, int64_t& out) { + return val.IsInt64() ? out = val.GetInt64(), true : false; + }}; + template inline static bool ReadValue(Value& val, T& out) { @@ -311,7 +319,7 @@ inline void Buffer::Read(Value& obj, Asset& r) " bytes, but found " + to_string(dataURI.dataLength)); } - this->mData.reset(new uint8_t[dataURI.dataLength]); + this->mData.reset(new uint8_t[dataURI.dataLength], std::default_delete()); memcpy( this->mData.get(), dataURI.data, dataURI.dataLength ); } } @@ -417,7 +425,7 @@ uint8_t* new_data; // Copy data which place after replacing part. memcpy(&new_data[pBufferData_Offset + pReplace_Count], &mData.get()[pBufferData_Offset + pBufferData_Count], pBufferData_Offset); // Apply new data - mData.reset(new_data); + mData.reset(new_data, std::default_delete()); byteLength = new_data_size; return true; @@ -434,9 +442,19 @@ inline size_t Buffer::AppendData(uint8_t* data, size_t length) inline void Buffer::Grow(size_t amount) { if (amount <= 0) return; - uint8_t* b = new uint8_t[byteLength + amount]; + if (capacity >= byteLength + amount) + { + byteLength += amount; + return; + } + + // Shift operation is standard way to divide integer by 2, it doesn't cast it to float back and forth, also works for odd numbers, + // originally it would look like: static_cast(capacity * 1.5f) + capacity = std::max(capacity + (capacity >> 1), byteLength + amount); + + uint8_t* b = new uint8_t[capacity]; if (mData) memcpy(b, mData.get(), byteLength); - mData.reset(b); + mData.reset(b, std::default_delete()); byteLength += amount; } @@ -1445,7 +1463,7 @@ inline std::string Asset::FindUniqueID(const std::string& str, const char* suffi if (it == mUsedIds.end()) return id; - char buffer[256]; + char buffer[1024]; int offset = ai_snprintf(buffer, sizeof(buffer), "%s_", id.c_str()); for (int i = 0; it != mUsedIds.end(); ++i) { ai_snprintf(buffer + offset, sizeof(buffer) - offset, "%d", i);