Merged PR 2811682: buffer grow changes and large files support

Buffer grow changes:
The exporting of relatevely large data could take a few days, because reallocation of a buffer every time for a few new bytes is overkill. I've introduced standard capacity member for the buffer and growth it by 1.5 times every time. That helps a lot, descrease exporting to a minute (from a few days).

Large file support:
glTF is a json file, all lengths and offsets don't have a type, just numbers, but code was always reading it as uint32, this doesn't work for files bigger than int32 (almost all files we have as an example). So, I've changed it to be reading as size_t. Specification doesn't specify the type for it, so it's not against spec.

code/glTF2Asset.h

@@ -430,9 +430,9 @@ namespace glTF2
     struct Accessor : public Object
     {
         Ref<BufferView> bufferView;  //!< The ID of the bufferView. (required)
-        unsigned int byteOffset;     //!< The offset relative to the start of the bufferView in bytes. (required)
+        size_t byteOffset;           //!< The offset relative to the start of the bufferView in bytes. (required)
         ComponentType componentType; //!< The datatype of components in the attribute. (required)
-        unsigned int count;          //!< The number of attributes referenced by this accessor. (required)
+        size_t count;                //!< The number of attributes referenced by this accessor. (required)
         AttribType::Value type;      //!< Specifies if the attribute is a scalar, vector, or matrix. (required)
         std::vector<float> max;      //!< Maximum value of each component in this attribute.
         std::vector<float> min;      //!< Minimum value of each component in this attribute.
@@ -529,6 +529,7 @@ namespace glTF2
 		//std::string uri; //!< The uri of the buffer. Can be a filepath, a data uri, etc. (required)
 		size_t byteLength; //!< The length of the buffer in bytes. (default: 0)
 		//std::string type; //!< XMLHttpRequest responseType (default: "arraybuffer")
+        size_t capacity = 0; //!< The capacity of the buffer in bytes. (default: 0)
 
 		Type type;
 

code/glTF2Asset.inl

@@ -85,6 +85,14 @@ namespace {
         return val.IsString() ? (out = std::string(val.GetString(), val.GetStringLength()), true) : false;
     }};
 
+    template<> struct ReadHelper<uint64_t> { static bool Read(Value& val, uint64_t& out) {
+        return val.IsUint64() ? out = val.GetUint64(), true : false;
+    }};
+
+    template<> struct ReadHelper<int64_t> { static bool Read(Value& val, int64_t& out) {
+        return val.IsInt64() ? out = val.GetInt64(), true : false;
+    }};
+
     template<class T> struct ReadHelper< Nullable<T> > { static bool Read(Value& val, Nullable<T>& out) {
         return out.isPresent = ReadHelper<T>::Read(val, out.value);
     }};
@@ -520,7 +528,17 @@ inline size_t Buffer::AppendData(uint8_t* data, size_t length)
 inline void Buffer::Grow(size_t amount)
 {
     if (amount <= 0) return;
-    uint8_t* b = new uint8_t[byteLength + amount];
+    if (capacity >= byteLength + amount)
+    {
+        byteLength += amount;
+        return;
+    }
+
+    // Shift operation is standard way to divide integer by 2, it doesn't cast it to float back and forth, also works for odd numbers,
+    // originally it would look like: static_cast<size_t>(capacity * 1.5f)
+    capacity = std::max(capacity + (capacity >> 1), byteLength + amount);
+
+    uint8_t* b = new uint8_t[capacity];
     if (mData) memcpy(b, mData.get(), byteLength);
     mData.reset(b, std::default_delete<uint8_t[]>());
     byteLength += amount;
@@ -537,8 +555,8 @@ inline void BufferView::Read(Value& obj, Asset& r)
         buffer = r.buffers.Retrieve(bufferVal->GetUint());
     }
 
-    byteOffset = MemberOrDefault(obj, "byteOffset", 0u);
+    byteOffset = MemberOrDefault(obj, "byteOffset", size_t(0));
-    byteLength = MemberOrDefault(obj, "byteLength", 0u);
+    byteLength = MemberOrDefault(obj, "byteLength", size_t(0));
     byteStride = MemberOrDefault(obj, "byteStride", 0u);
 }
 
@@ -553,9 +571,9 @@ inline void Accessor::Read(Value& obj, Asset& r)
         bufferView = r.bufferViews.Retrieve(bufferViewVal->GetUint());
     }
 
-    byteOffset = MemberOrDefault(obj, "byteOffset", 0u);
+    byteOffset = MemberOrDefault(obj, "byteOffset", size_t(0));
     componentType = MemberOrDefault(obj, "componentType", ComponentType_BYTE);
-    count = MemberOrDefault(obj, "count", 0u);
+    count = MemberOrDefault(obj, "count", size_t(0));
 
     const char* typestr;
     type = ReadMember(obj, "type", typestr) ? AttribType::FromString(typestr) : AttribType::SCALAR;

code/glTF2Exporter.cpp

@@ -156,7 +156,7 @@ static void IdentityMatrix4(mat4& o) {
 }
 
 inline Ref<Accessor> ExportData(Asset& a, std::string& meshName, Ref<Buffer>& buffer,
-    unsigned int count, void* data, AttribType::Value typeIn, AttribType::Value typeOut, ComponentType compType, bool isIndices = false)
+    size_t count, void* data, AttribType::Value typeIn, AttribType::Value typeOut, ComponentType compType, bool isIndices = false)
 {
     if (!count || !data) {
         return Ref<Accessor>();
@@ -176,7 +176,7 @@ inline Ref<Accessor> ExportData(Asset& a, std::string& meshName, Ref<Buffer>& bu
     // bufferView
     Ref<BufferView> bv = a.bufferViews.Create(a.FindUniqueID(meshName, "view"));
     bv->buffer = buffer;
-    bv->byteOffset = unsigned(offset);
+    bv->byteOffset = offset;
     bv->byteLength = length; //! The target that the WebGL buffer should be bound to.
     bv->byteStride = 0;
     bv->target = isIndices ? BufferViewTarget_ELEMENT_ARRAY_BUFFER : BufferViewTarget_ARRAY_BUFFER;
@@ -768,7 +768,7 @@ void glTF2Exporter::ExportMeshes()
                 }
             }
 
-			p.indices = ExportData(*mAsset, meshId, b, unsigned(indices.size()), &indices[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_UNSIGNED_INT, true);
+			p.indices = ExportData(*mAsset, meshId, b, indices.size(), &indices[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_UNSIGNED_INT, true);
 		}
 
         switch (aim->mPrimitiveTypes) {

code/glTF2Importer.cpp

@@ -412,7 +412,7 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r)
             Mesh::Primitive::Attributes& attr = prim.attributes;
 
             if (attr.position.size() > 0 && attr.position[0]) {
-                aim->mNumVertices = attr.position[0]->count;
+                aim->mNumVertices = static_cast<unsigned int>(attr.position[0]->count);
                 attr.position[0]->ExtractData(aim->mVertices);
             }
 
@@ -511,10 +511,10 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r)
 
 
             aiFace* faces = 0;
-            unsigned int nFaces = 0;
+            size_t nFaces = 0;
 
             if (prim.indices) {
-                unsigned int count = prim.indices->count;
+                size_t count = prim.indices->count;
 
                 Accessor::Indexer data = prim.indices->GetIndexer();
                 ai_assert(data.IsValid());
@@ -665,7 +665,7 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r)
 
             if (faces) {
                 aim->mFaces = faces;
-                aim->mNumFaces = nFaces;
+                aim->mNumFaces = static_cast<unsigned int>(nFaces);
                 ai_assert(CheckValidFacesIndices(faces, nFaces, aim->mNumVertices));
             }
 
@@ -751,7 +751,7 @@ static void BuildVertexWeightMapping(Mesh::Primitive& primitive, std::vector<std
         return;
     }
 
-    const int num_vertices = attr.weight[0]->count;
+    size_t num_vertices = attr.weight[0]->count;
 
     struct Weights { float values[4]; };
     Weights* weights = nullptr;
@@ -822,7 +822,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector<unsigned int>&
         if (node.skin) {
             for (int primitiveNo = 0; primitiveNo < count; ++primitiveNo) {
                 aiMesh* mesh = pScene->mMeshes[meshOffsets[mesh_idx]+primitiveNo];
-                mesh->mNumBones = node.skin->jointNames.size();
+                mesh->mNumBones = static_cast<unsigned int>(node.skin->jointNames.size());
                 mesh->mBones = new aiBone*[mesh->mNumBones];
 
                 // GLTF and Assimp choose to store bone weights differently.
@@ -837,7 +837,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector<unsigned int>&
                 std::vector<std::vector<aiVertexWeight>> weighting(mesh->mNumBones);
                 BuildVertexWeightMapping(node.meshes[0]->primitives[primitiveNo], weighting);
 
-                for (size_t i = 0; i < mesh->mNumBones; ++i) {
+                for (uint32_t i = 0; i < mesh->mNumBones; ++i) {
                     aiBone* bone = new aiBone();
 
                     Ref<Node> joint = node.skin->jointNames[i];
@@ -854,7 +854,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector<unsigned int>&
 
                     std::vector<aiVertexWeight>& weights = weighting[i];
 
-                    bone->mNumWeights = weights.size();
+                    bone->mNumWeights = static_cast<uint32_t>(weights.size());
                     if (bone->mNumWeights > 0) {
                       bone->mWeights = new aiVertexWeight[bone->mNumWeights];
                       memcpy(bone->mWeights, weights.data(), bone->mNumWeights * sizeof(aiVertexWeight));
@@ -930,7 +930,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl
         samplers.translation->input->ExtractData(times);
         aiVector3D* values = nullptr;
         samplers.translation->output->ExtractData(values);
-        anim->mNumPositionKeys = samplers.translation->input->count;
+        anim->mNumPositionKeys = static_cast<uint32_t>(samplers.translation->input->count);
         anim->mPositionKeys = new aiVectorKey[anim->mNumPositionKeys];
         for (unsigned int i = 0; i < anim->mNumPositionKeys; ++i) {
             anim->mPositionKeys[i].mTime = times[i] * kMillisecondsFromSeconds;
@@ -952,7 +952,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl
         samplers.rotation->input->ExtractData(times);
         aiQuaternion* values = nullptr;
         samplers.rotation->output->ExtractData(values);
-        anim->mNumRotationKeys = samplers.rotation->input->count;
+        anim->mNumRotationKeys = static_cast<uint32_t>(samplers.rotation->input->count);
         anim->mRotationKeys = new aiQuatKey[anim->mNumRotationKeys];
         for (unsigned int i = 0; i < anim->mNumRotationKeys; ++i) {
             anim->mRotationKeys[i].mTime = times[i] * kMillisecondsFromSeconds;
@@ -978,7 +978,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl
         samplers.scale->input->ExtractData(times);
         aiVector3D* values = nullptr;
         samplers.scale->output->ExtractData(values);
-        anim->mNumScalingKeys = samplers.scale->input->count;
+        anim->mNumScalingKeys = static_cast<uint32_t>(samplers.scale->input->count);
         anim->mScalingKeys = new aiVectorKey[anim->mNumScalingKeys];
         for (unsigned int i = 0; i < anim->mNumScalingKeys; ++i) {
             anim->mScalingKeys[i].mTime = times[i] * kMillisecondsFromSeconds;
@@ -1042,7 +1042,7 @@ void glTF2Importer::ImportAnimations(glTF2::Asset& r)
 
         std::unordered_map<unsigned int, AnimationSamplers> samplers = GatherSamplers(anim);
 
-        ai_anim->mNumChannels = samplers.size();
+        ai_anim->mNumChannels = static_cast<uint32_t>(samplers.size());
         if (ai_anim->mNumChannels > 0) {
             ai_anim->mChannels = new aiNodeAnim*[ai_anim->mNumChannels];
             int j = 0;

code/glTFAsset.h

@@ -537,7 +537,7 @@ namespace glTF
 
 		shared_ptr<uint8_t> mData; //!< Pointer to the data
 		bool mIsSpecial; //!< Set to true for special cases (e.g. the body buffer)
-
+        size_t capacity = 0; //!< The capacity of the buffer in bytes. (default: 0)
 		/// \var EncodedRegion_List
 		/// List of encoded regions.
 		std::list<SEncodedRegion*> EncodedRegion_List;

code/glTFAsset.inl

@@ -95,6 +95,14 @@ namespace {
         return out.isPresent = ReadHelper<T>::Read(val, out.value);
     }};
 
+    template<> struct ReadHelper<uint64_t> { static bool Read(Value& val, uint64_t& out) {
+        return val.IsUint64() ? out = val.GetUint64(), true : false;
+    }};
+
+    template<> struct ReadHelper<int64_t> { static bool Read(Value& val, int64_t& out) {
+        return val.IsInt64() ? out = val.GetInt64(), true : false;
+    }};
+
     template<class T>
     inline static bool ReadValue(Value& val, T& out)
     {
@@ -311,7 +319,7 @@ inline void Buffer::Read(Value& obj, Asset& r)
                                         " bytes, but found " + to_string(dataURI.dataLength));
             }
 
-            this->mData.reset(new uint8_t[dataURI.dataLength]);
+            this->mData.reset(new uint8_t[dataURI.dataLength], std::default_delete<uint8_t[]>());
             memcpy( this->mData.get(), dataURI.data, dataURI.dataLength );
         }
     }
@@ -417,7 +425,7 @@ uint8_t* new_data;
 	// Copy data which place after replacing part.
 	memcpy(&new_data[pBufferData_Offset + pReplace_Count], &mData.get()[pBufferData_Offset + pBufferData_Count], pBufferData_Offset);
 	// Apply new data
-	mData.reset(new_data);
+	mData.reset(new_data, std::default_delete<uint8_t[]>());
 	byteLength = new_data_size;
 
 	return true;
@@ -434,9 +442,19 @@ inline size_t Buffer::AppendData(uint8_t* data, size_t length)
 inline void Buffer::Grow(size_t amount)
 {
     if (amount <= 0) return;
-    uint8_t* b = new uint8_t[byteLength + amount];
+    if (capacity >= byteLength + amount)
+    {
+        byteLength += amount;
+        return;
+    }
+
+    // Shift operation is standard way to divide integer by 2, it doesn't cast it to float back and forth, also works for odd numbers,
+    // originally it would look like: static_cast<size_t>(capacity * 1.5f)
+    capacity = std::max(capacity + (capacity >> 1), byteLength + amount);
+
+    uint8_t* b = new uint8_t[capacity];
     if (mData) memcpy(b, mData.get(), byteLength);
-    mData.reset(b);
+    mData.reset(b, std::default_delete<uint8_t[]>());
     byteLength += amount;
 }
 
@@ -1445,7 +1463,7 @@ inline std::string Asset::FindUniqueID(const std::string& str, const char* suffi
     if (it == mUsedIds.end())
         return id;
 
-    char buffer[256];
+    char buffer[1024];
     int offset = ai_snprintf(buffer, sizeof(buffer), "%s_", id.c_str());
     for (int i = 0; it != mUsedIds.end(); ++i) {
         ai_snprintf(buffer + offset, sizeof(buffer) - offset, "%d", i);