Merge pull request #2319 from muxanick/topic/large_gltf_files_support

glTF/glTF2 Buffer grow changes and large files support

code/glTF2Asset.h

@@ -430,9 +430,9 @@ namespace glTF2
     struct Accessor : public Object
     {
         Ref<BufferView> bufferView;  //!< The ID of the bufferView. (required)
-        unsigned int byteOffset;     //!< The offset relative to the start of the bufferView in bytes. (required)
+        size_t byteOffset;           //!< The offset relative to the start of the bufferView in bytes. (required)
         ComponentType componentType; //!< The datatype of components in the attribute. (required)
-        unsigned int count;          //!< The number of attributes referenced by this accessor. (required)
+        size_t count;                //!< The number of attributes referenced by this accessor. (required)
         AttribType::Value type;      //!< Specifies if the attribute is a scalar, vector, or matrix. (required)
         std::vector<float> max;      //!< Maximum value of each component in this attribute.
         std::vector<float> min;      //!< Minimum value of each component in this attribute.
@@ -529,6 +529,7 @@ namespace glTF2
 		//std::string uri; //!< The uri of the buffer. Can be a filepath, a data uri, etc. (required)
 		size_t byteLength; //!< The length of the buffer in bytes. (default: 0)
 		//std::string type; //!< XMLHttpRequest responseType (default: "arraybuffer")
+        size_t capacity = 0; //!< The capacity of the buffer in bytes. (default: 0)
 
 		Type type;
 

code/glTF2Asset.inl

@@ -85,6 +85,14 @@ namespace {
         return val.IsString() ? (out = std::string(val.GetString(), val.GetStringLength()), true) : false;
     }};
 
+    template<> struct ReadHelper<uint64_t> { static bool Read(Value& val, uint64_t& out) {
+        return val.IsUint64() ? out = val.GetUint64(), true : false;
+    }};
+
+    template<> struct ReadHelper<int64_t> { static bool Read(Value& val, int64_t& out) {
+        return val.IsInt64() ? out = val.GetInt64(), true : false;
+    }};
+
     template<class T> struct ReadHelper< Nullable<T> > { static bool Read(Value& val, Nullable<T>& out) {
         return out.isPresent = ReadHelper<T>::Read(val, out.value);
     }};
@@ -520,7 +528,17 @@ inline size_t Buffer::AppendData(uint8_t* data, size_t length)
 inline void Buffer::Grow(size_t amount)
 {
     if (amount <= 0) return;
-    uint8_t* b = new uint8_t[byteLength + amount];
+    if (capacity >= byteLength + amount)
+    {
+        byteLength += amount;
+        return;
+    }
+
+    // Shift operation is standard way to divide integer by 2, it doesn't cast it to float back and forth, also works for odd numbers,
+    // originally it would look like: static_cast<size_t>(capacity * 1.5f)
+    capacity = std::max(capacity + (capacity >> 1), byteLength + amount);
+
+    uint8_t* b = new uint8_t[capacity];
     if (mData) memcpy(b, mData.get(), byteLength);
     mData.reset(b, std::default_delete<uint8_t[]>());
     byteLength += amount;
@@ -537,8 +555,8 @@ inline void BufferView::Read(Value& obj, Asset& r)
         buffer = r.buffers.Retrieve(bufferVal->GetUint());
     }
 
-    byteOffset = MemberOrDefault(obj, "byteOffset", 0u);
+    byteOffset = MemberOrDefault(obj, "byteOffset", size_t(0));
-    byteLength = MemberOrDefault(obj, "byteLength", 0u);
+    byteLength = MemberOrDefault(obj, "byteLength", size_t(0));
     byteStride = MemberOrDefault(obj, "byteStride", 0u);
 }
 
@@ -553,9 +571,9 @@ inline void Accessor::Read(Value& obj, Asset& r)
         bufferView = r.bufferViews.Retrieve(bufferViewVal->GetUint());
     }
 
-    byteOffset = MemberOrDefault(obj, "byteOffset", 0u);
+    byteOffset = MemberOrDefault(obj, "byteOffset", size_t(0));
     componentType = MemberOrDefault(obj, "componentType", ComponentType_BYTE);
-    count = MemberOrDefault(obj, "count", 0u);
+    count = MemberOrDefault(obj, "count", size_t(0));
 
     const char* typestr;
     type = ReadMember(obj, "type", typestr) ? AttribType::FromString(typestr) : AttribType::SCALAR;

code/glTF2Exporter.cpp

@@ -156,7 +156,7 @@ static void IdentityMatrix4(mat4& o) {
 }
 
 inline Ref<Accessor> ExportData(Asset& a, std::string& meshName, Ref<Buffer>& buffer,
-    unsigned int count, void* data, AttribType::Value typeIn, AttribType::Value typeOut, ComponentType compType, bool isIndices = false)
+    size_t count, void* data, AttribType::Value typeIn, AttribType::Value typeOut, ComponentType compType, bool isIndices = false)
 {
     if (!count || !data) {
         return Ref<Accessor>();
@@ -176,7 +176,7 @@ inline Ref<Accessor> ExportData(Asset& a, std::string& meshName, Ref<Buffer>& bu
     // bufferView
     Ref<BufferView> bv = a.bufferViews.Create(a.FindUniqueID(meshName, "view"));
     bv->buffer = buffer;
-    bv->byteOffset = unsigned(offset);
+    bv->byteOffset = offset;
     bv->byteLength = length; //! The target that the WebGL buffer should be bound to.
     bv->byteStride = 0;
     bv->target = isIndices ? BufferViewTarget_ELEMENT_ARRAY_BUFFER : BufferViewTarget_ARRAY_BUFFER;
@@ -768,7 +768,7 @@ void glTF2Exporter::ExportMeshes()
                 }
             }
 
-			p.indices = ExportData(*mAsset, meshId, b, unsigned(indices.size()), &indices[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_UNSIGNED_INT, true);
+			p.indices = ExportData(*mAsset, meshId, b, indices.size(), &indices[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_UNSIGNED_INT, true);
 		}
 
         switch (aim->mPrimitiveTypes) {

code/glTF2Importer.cpp

@@ -412,7 +412,7 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r)
             Mesh::Primitive::Attributes& attr = prim.attributes;
 
             if (attr.position.size() > 0 && attr.position[0]) {
-                aim->mNumVertices = attr.position[0]->count;
+                aim->mNumVertices = static_cast<unsigned int>(attr.position[0]->count);
                 attr.position[0]->ExtractData(aim->mVertices);
             }
 
@@ -511,10 +511,10 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r)
 
 
             aiFace* faces = 0;
-            unsigned int nFaces = 0;
+            size_t nFaces = 0;
 
             if (prim.indices) {
-                unsigned int count = prim.indices->count;
+                size_t count = prim.indices->count;
 
                 Accessor::Indexer data = prim.indices->GetIndexer();
                 ai_assert(data.IsValid());
@@ -665,8 +665,8 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r)
 
             if (faces) {
                 aim->mFaces = faces;
-                aim->mNumFaces = nFaces;
+                aim->mNumFaces = static_cast<unsigned int>(nFaces);
-                ai_assert(CheckValidFacesIndices(faces, nFaces, aim->mNumVertices));
+                ai_assert(CheckValidFacesIndices(faces, static_cast<unsigned>(nFaces), aim->mNumVertices));
             }
 
             if (prim.material) {
@@ -751,7 +751,7 @@ static void BuildVertexWeightMapping(Mesh::Primitive& primitive, std::vector<std
         return;
     }
 
-    const int num_vertices = attr.weight[0]->count;
+    size_t num_vertices = attr.weight[0]->count;
 
     struct Weights { float values[4]; };
     Weights* weights = nullptr;
@@ -773,13 +773,13 @@ static void BuildVertexWeightMapping(Mesh::Primitive& primitive, std::vector<std
         return;
     }
 
-    for (int i = 0; i < num_vertices; ++i) {
+    for (size_t i = 0; i < num_vertices; ++i) {
         for (int j = 0; j < 4; ++j) {
             const unsigned int bone = (indices8!=nullptr) ? indices8[i].values[j] : indices16[i].values[j];
             const float weight = weights[i].values[j];
             if (weight > 0 && bone < map.size()) {
                 map[bone].reserve(8);
-                map[bone].emplace_back(i, weight);
+                map[bone].emplace_back(static_cast<unsigned int>(i), weight);
             }
         }
     }
@@ -822,7 +822,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector<unsigned int>&
         if (node.skin) {
             for (int primitiveNo = 0; primitiveNo < count; ++primitiveNo) {
                 aiMesh* mesh = pScene->mMeshes[meshOffsets[mesh_idx]+primitiveNo];
-                mesh->mNumBones = node.skin->jointNames.size();
+                mesh->mNumBones = static_cast<unsigned int>(node.skin->jointNames.size());
                 mesh->mBones = new aiBone*[mesh->mNumBones];
 
                 // GLTF and Assimp choose to store bone weights differently.
@@ -837,7 +837,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector<unsigned int>&
                 std::vector<std::vector<aiVertexWeight>> weighting(mesh->mNumBones);
                 BuildVertexWeightMapping(node.meshes[0]->primitives[primitiveNo], weighting);
 
-                for (size_t i = 0; i < mesh->mNumBones; ++i) {
+                for (uint32_t i = 0; i < mesh->mNumBones; ++i) {
                     aiBone* bone = new aiBone();
 
                     Ref<Node> joint = node.skin->jointNames[i];
@@ -854,7 +854,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector<unsigned int>&
 
                     std::vector<aiVertexWeight>& weights = weighting[i];
 
-                    bone->mNumWeights = weights.size();
+                    bone->mNumWeights = static_cast<uint32_t>(weights.size());
                     if (bone->mNumWeights > 0) {
                       bone->mWeights = new aiVertexWeight[bone->mNumWeights];
                       memcpy(bone->mWeights, weights.data(), bone->mNumWeights * sizeof(aiVertexWeight));
@@ -930,7 +930,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl
         samplers.translation->input->ExtractData(times);
         aiVector3D* values = nullptr;
         samplers.translation->output->ExtractData(values);
-        anim->mNumPositionKeys = samplers.translation->input->count;
+        anim->mNumPositionKeys = static_cast<uint32_t>(samplers.translation->input->count);
         anim->mPositionKeys = new aiVectorKey[anim->mNumPositionKeys];
         for (unsigned int i = 0; i < anim->mNumPositionKeys; ++i) {
             anim->mPositionKeys[i].mTime = times[i] * kMillisecondsFromSeconds;
@@ -952,7 +952,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl
         samplers.rotation->input->ExtractData(times);
         aiQuaternion* values = nullptr;
         samplers.rotation->output->ExtractData(values);
-        anim->mNumRotationKeys = samplers.rotation->input->count;
+        anim->mNumRotationKeys = static_cast<uint32_t>(samplers.rotation->input->count);
         anim->mRotationKeys = new aiQuatKey[anim->mNumRotationKeys];
         for (unsigned int i = 0; i < anim->mNumRotationKeys; ++i) {
             anim->mRotationKeys[i].mTime = times[i] * kMillisecondsFromSeconds;
@@ -978,7 +978,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl
         samplers.scale->input->ExtractData(times);
         aiVector3D* values = nullptr;
         samplers.scale->output->ExtractData(values);
-        anim->mNumScalingKeys = samplers.scale->input->count;
+        anim->mNumScalingKeys = static_cast<uint32_t>(samplers.scale->input->count);
         anim->mScalingKeys = new aiVectorKey[anim->mNumScalingKeys];
         for (unsigned int i = 0; i < anim->mNumScalingKeys; ++i) {
             anim->mScalingKeys[i].mTime = times[i] * kMillisecondsFromSeconds;
@@ -1042,7 +1042,7 @@ void glTF2Importer::ImportAnimations(glTF2::Asset& r)
 
         std::unordered_map<unsigned int, AnimationSamplers> samplers = GatherSamplers(anim);
 
-        ai_anim->mNumChannels = samplers.size();
+        ai_anim->mNumChannels = static_cast<uint32_t>(samplers.size());
         if (ai_anim->mNumChannels > 0) {
             ai_anim->mChannels = new aiNodeAnim*[ai_anim->mNumChannels];
             int j = 0;

code/glTFAsset.h

@@ -537,7 +537,7 @@ namespace glTF
 
 		shared_ptr<uint8_t> mData; //!< Pointer to the data
 		bool mIsSpecial; //!< Set to true for special cases (e.g. the body buffer)
-
+        size_t capacity = 0; //!< The capacity of the buffer in bytes. (default: 0)
 		/// \var EncodedRegion_List
 		/// List of encoded regions.
 		std::list<SEncodedRegion*> EncodedRegion_List;

code/glTFAsset.inl

@@ -95,6 +95,14 @@ namespace {
         return out.isPresent = ReadHelper<T>::Read(val, out.value);
     }};
 
+    template<> struct ReadHelper<uint64_t> { static bool Read(Value& val, uint64_t& out) {
+        return val.IsUint64() ? out = val.GetUint64(), true : false;
+    }};
+
+    template<> struct ReadHelper<int64_t> { static bool Read(Value& val, int64_t& out) {
+        return val.IsInt64() ? out = val.GetInt64(), true : false;
+    }};
+
     template<class T>
     inline static bool ReadValue(Value& val, T& out)
     {
@@ -311,7 +319,7 @@ inline void Buffer::Read(Value& obj, Asset& r)
                                         " bytes, but found " + to_string(dataURI.dataLength));
             }
 
-            this->mData.reset(new uint8_t[dataURI.dataLength]);
+            this->mData.reset(new uint8_t[dataURI.dataLength], std::default_delete<uint8_t[]>());
             memcpy( this->mData.get(), dataURI.data, dataURI.dataLength );
         }
     }
@@ -417,7 +425,7 @@ uint8_t* new_data;
 	// Copy data which place after replacing part.
 	memcpy(&new_data[pBufferData_Offset + pReplace_Count], &mData.get()[pBufferData_Offset + pBufferData_Count], pBufferData_Offset);
 	// Apply new data
-	mData.reset(new_data);
+	mData.reset(new_data, std::default_delete<uint8_t[]>());
 	byteLength = new_data_size;
 
 	return true;
@@ -434,9 +442,19 @@ inline size_t Buffer::AppendData(uint8_t* data, size_t length)
 inline void Buffer::Grow(size_t amount)
 {
     if (amount <= 0) return;
-    uint8_t* b = new uint8_t[byteLength + amount];
+    if (capacity >= byteLength + amount)
+    {
+        byteLength += amount;
+        return;
+    }
+
+    // Shift operation is standard way to divide integer by 2, it doesn't cast it to float back and forth, also works for odd numbers,
+    // originally it would look like: static_cast<size_t>(capacity * 1.5f)
+    capacity = std::max(capacity + (capacity >> 1), byteLength + amount);
+
+    uint8_t* b = new uint8_t[capacity];
     if (mData) memcpy(b, mData.get(), byteLength);
-    mData.reset(b);
+    mData.reset(b, std::default_delete<uint8_t[]>());
     byteLength += amount;
 }
 
@@ -1445,7 +1463,7 @@ inline std::string Asset::FindUniqueID(const std::string& str, const char* suffi
     if (it == mUsedIds.end())
         return id;
 
-    char buffer[256];
+    char buffer[1024];
     int offset = ai_snprintf(buffer, sizeof(buffer), "%s_", id.c_str());
     for (int i = 0; it != mUsedIds.end(); ++i) {
         ai_snprintf(buffer + offset, sizeof(buffer) - offset, "%d", i);