v4games
/
assimp
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merged PR 2811682: buffer grow changes and large files support 

Buffer grow changes:
The exporting of relatevely large data could take a few days, because reallocation of a buffer every time for a few new bytes is overkill. I've introduced standard capacity member for the buffer and growth it by 1.5 times every time. That helps a lot, descrease exporting to a minute (from a few days).

Large file support:
glTF is a json file, all lengths and offsets don't have a type, just numbers, but code was always reading it as uint32, this doesn't work for files bigger than int32 (almost all files we have as an example). So, I've changed it to be reading as size_t. Specification doesn't specify the type for it, so it's not against spec.
pull/2319/head
Mike Samsonov 2019-01-23 17:07:44 +00:00
parent 6c4fde73e6
commit 971ba308b3
6 changed files with 65 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
code/glTF2Asset.h
View File

@ -430,9 +430,9 @@ namespace glTF2
struct Accessor : public Object
{
Ref<BufferView> bufferView; //!< The ID of the bufferView. (required)
unsigned int byteOffset; //!< The offset relative to the start of the bufferView in bytes. (required)
size_t byteOffset; //!< The offset relative to the start of the bufferView in bytes. (required)
ComponentType componentType; //!< The datatype of components in the attribute. (required)
unsigned int count; //!< The number of attributes referenced by this accessor. (required)
size_t count; //!< The number of attributes referenced by this accessor. (required)
AttribType::Value type; //!< Specifies if the attribute is a scalar, vector, or matrix. (required)
std::vector<float> max; //!< Maximum value of each component in this attribute.
std::vector<float> min; //!< Minimum value of each component in this attribute.
@ -529,6 +529,7 @@ namespace glTF2
//std::string uri; //!< The uri of the buffer. Can be a filepath, a data uri, etc. (required)
size_t byteLength; //!< The length of the buffer in bytes. (default: 0)
//std::string type; //!< XMLHttpRequest responseType (default: "arraybuffer")
size_t capacity = 0; //!< The capacity of the buffer in bytes. (default: 0)
Type type;

28
code/glTF2Asset.inl
View File

@ -85,6 +85,14 @@ namespace {
return val.IsString() ? (out = std::string(val.GetString(), val.GetStringLength()), true) : false;
}};
template<> struct ReadHelper<uint64_t> { static bool Read(Value& val, uint64_t& out) {
return val.IsUint64() ? out = val.GetUint64(), true : false;
}};
template<> struct ReadHelper<int64_t> { static bool Read(Value& val, int64_t& out) {
return val.IsInt64() ? out = val.GetInt64(), true : false;
}};
template<class T> struct ReadHelper< Nullable<T> > { static bool Read(Value& val, Nullable<T>& out) {
return out.isPresent = ReadHelper<T>::Read(val, out.value);
}};
@ -520,7 +528,17 @@ inline size_t Buffer::AppendData(uint8_t* data, size_t length)
inline void Buffer::Grow(size_t amount)
{
if (amount <= 0) return;
uint8_t* b = new uint8_t[byteLength + amount];
if (capacity >= byteLength + amount)
{
byteLength += amount;
return;
}
// Shift operation is standard way to divide integer by 2, it doesn't cast it to float back and forth, also works for odd numbers,
// originally it would look like: static_cast<size_t>(capacity * 1.5f)
capacity = std::max(capacity + (capacity >> 1), byteLength + amount);
uint8_t* b = new uint8_t[capacity];
if (mData) memcpy(b, mData.get(), byteLength);
mData.reset(b, std::default_delete<uint8_t[]>());
byteLength += amount;
@ -537,8 +555,8 @@ inline void BufferView::Read(Value& obj, Asset& r)
buffer = r.buffers.Retrieve(bufferVal->GetUint());
}
byteOffset = MemberOrDefault(obj, "byteOffset", 0u);
byteLength = MemberOrDefault(obj, "byteLength", 0u);
byteOffset = MemberOrDefault(obj, "byteOffset", size_t(0));
byteLength = MemberOrDefault(obj, "byteLength", size_t(0));
byteStride = MemberOrDefault(obj, "byteStride", 0u);
}
@ -553,9 +571,9 @@ inline void Accessor::Read(Value& obj, Asset& r)
bufferView = r.bufferViews.Retrieve(bufferViewVal->GetUint());
}
byteOffset = MemberOrDefault(obj, "byteOffset", 0u);
byteOffset = MemberOrDefault(obj, "byteOffset", size_t(0));
componentType = MemberOrDefault(obj, "componentType", ComponentType_BYTE);
count = MemberOrDefault(obj, "count", 0u);
count = MemberOrDefault(obj, "count", size_t(0));
const char* typestr;
type = ReadMember(obj, "type", typestr) ? AttribType::FromString(typestr) : AttribType::SCALAR;

6
code/glTF2Exporter.cpp
View File

@ -156,7 +156,7 @@ static void IdentityMatrix4(mat4& o) {
}
inline Ref<Accessor> ExportData(Asset& a, std::string& meshName, Ref<Buffer>& buffer,
unsigned int count, void* data, AttribType::Value typeIn, AttribType::Value typeOut, ComponentType compType, bool isIndices = false)
size_t count, void* data, AttribType::Value typeIn, AttribType::Value typeOut, ComponentType compType, bool isIndices = false)
{
if (!count || !data) {
return Ref<Accessor>();
@ -176,7 +176,7 @@ inline Ref<Accessor> ExportData(Asset& a, std::string& meshName, Ref<Buffer>& bu
// bufferView
Ref<BufferView> bv = a.bufferViews.Create(a.FindUniqueID(meshName, "view"));
bv->buffer = buffer;
bv->byteOffset = unsigned(offset);
bv->byteOffset = offset;
bv->byteLength = length; //! The target that the WebGL buffer should be bound to.
bv->byteStride = 0;
bv->target = isIndices ? BufferViewTarget_ELEMENT_ARRAY_BUFFER : BufferViewTarget_ARRAY_BUFFER;
@ -768,7 +768,7 @@ void glTF2Exporter::ExportMeshes()
}
}
p.indices = ExportData(*mAsset, meshId, b, unsigned(indices.size()), &indices[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_UNSIGNED_INT, true);
p.indices = ExportData(*mAsset, meshId, b, indices.size(), &indices[0], AttribType::SCALAR, AttribType::SCALAR, ComponentType_UNSIGNED_INT, true);
}
switch (aim->mPrimitiveTypes) {

24
code/glTF2Importer.cpp
View File

@ -412,7 +412,7 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r)
Mesh::Primitive::Attributes& attr = prim.attributes;
if (attr.position.size() > 0 && attr.position[0]) {
aim->mNumVertices = attr.position[0]->count;
aim->mNumVertices = static_cast<unsigned int>(attr.position[0]->count);
attr.position[0]->ExtractData(aim->mVertices);
}
@ -511,10 +511,10 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r)
aiFace* faces = 0;
unsigned int nFaces = 0;
size_t nFaces = 0;
if (prim.indices) {
unsigned int count = prim.indices->count;
size_t count = prim.indices->count;
Accessor::Indexer data = prim.indices->GetIndexer();
ai_assert(data.IsValid());
@ -665,7 +665,7 @@ void glTF2Importer::ImportMeshes(glTF2::Asset& r)
if (faces) {
aim->mFaces = faces;
aim->mNumFaces = nFaces;
aim->mNumFaces = static_cast<unsigned int>(nFaces);
ai_assert(CheckValidFacesIndices(faces, nFaces, aim->mNumVertices));
}
@ -751,7 +751,7 @@ static void BuildVertexWeightMapping(Mesh::Primitive& primitive, std::vector<std
return;
}
const int num_vertices = attr.weight[0]->count;
size_t num_vertices = attr.weight[0]->count;
struct Weights { float values[4]; };
Weights* weights = nullptr;
@ -822,7 +822,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector<unsigned int>&
if (node.skin) {
for (int primitiveNo = 0; primitiveNo < count; ++primitiveNo) {
aiMesh* mesh = pScene->mMeshes[meshOffsets[mesh_idx]+primitiveNo];
mesh->mNumBones = node.skin->jointNames.size();
mesh->mNumBones = static_cast<unsigned int>(node.skin->jointNames.size());
mesh->mBones = new aiBone*[mesh->mNumBones];
// GLTF and Assimp choose to store bone weights differently.
@ -837,7 +837,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector<unsigned int>&
std::vector<std::vector<aiVertexWeight>> weighting(mesh->mNumBones);
BuildVertexWeightMapping(node.meshes[0]->primitives[primitiveNo], weighting);
for (size_t i = 0; i < mesh->mNumBones; ++i) {
for (uint32_t i = 0; i < mesh->mNumBones; ++i) {
aiBone* bone = new aiBone();
Ref<Node> joint = node.skin->jointNames[i];
@ -854,7 +854,7 @@ aiNode* ImportNode(aiScene* pScene, glTF2::Asset& r, std::vector<unsigned int>&
std::vector<aiVertexWeight>& weights = weighting[i];
bone->mNumWeights = weights.size();
bone->mNumWeights = static_cast<uint32_t>(weights.size());
if (bone->mNumWeights > 0) {
bone->mWeights = new aiVertexWeight[bone->mNumWeights];
memcpy(bone->mWeights, weights.data(), bone->mNumWeights * sizeof(aiVertexWeight));
@ -930,7 +930,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl
samplers.translation->input->ExtractData(times);
aiVector3D* values = nullptr;
samplers.translation->output->ExtractData(values);
anim->mNumPositionKeys = samplers.translation->input->count;
anim->mNumPositionKeys = static_cast<uint32_t>(samplers.translation->input->count);
anim->mPositionKeys = new aiVectorKey[anim->mNumPositionKeys];
for (unsigned int i = 0; i < anim->mNumPositionKeys; ++i) {
anim->mPositionKeys[i].mTime = times[i] * kMillisecondsFromSeconds;
@ -952,7 +952,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl
samplers.rotation->input->ExtractData(times);
aiQuaternion* values = nullptr;
samplers.rotation->output->ExtractData(values);
anim->mNumRotationKeys = samplers.rotation->input->count;
anim->mNumRotationKeys = static_cast<uint32_t>(samplers.rotation->input->count);
anim->mRotationKeys = new aiQuatKey[anim->mNumRotationKeys];
for (unsigned int i = 0; i < anim->mNumRotationKeys; ++i) {
anim->mRotationKeys[i].mTime = times[i] * kMillisecondsFromSeconds;
@ -978,7 +978,7 @@ aiNodeAnim* CreateNodeAnim(glTF2::Asset& r, Node& node, AnimationSamplers& sampl
samplers.scale->input->ExtractData(times);
aiVector3D* values = nullptr;
samplers.scale->output->ExtractData(values);
anim->mNumScalingKeys = samplers.scale->input->count;
anim->mNumScalingKeys = static_cast<uint32_t>(samplers.scale->input->count);
anim->mScalingKeys = new aiVectorKey[anim->mNumScalingKeys];
for (unsigned int i = 0; i < anim->mNumScalingKeys; ++i) {
anim->mScalingKeys[i].mTime = times[i] * kMillisecondsFromSeconds;
@ -1042,7 +1042,7 @@ void glTF2Importer::ImportAnimations(glTF2::Asset& r)
std::unordered_map<unsigned int, AnimationSamplers> samplers = GatherSamplers(anim);
ai_anim->mNumChannels = samplers.size();
ai_anim->mNumChannels = static_cast<uint32_t>(samplers.size());
if (ai_anim->mNumChannels > 0) {
ai_anim->mChannels = new aiNodeAnim*[ai_anim->mNumChannels];
int j = 0;

2
code/glTFAsset.h
View File

@ -537,7 +537,7 @@ namespace glTF
shared_ptr<uint8_t> mData; //!< Pointer to the data
bool mIsSpecial; //!< Set to true for special cases (e.g. the body buffer)
size_t capacity = 0; //!< The capacity of the buffer in bytes. (default: 0)
/// \var EncodedRegion_List
/// List of encoded regions.
std::list<SEncodedRegion*> EncodedRegion_List;

28
code/glTFAsset.inl
View File

@ -95,6 +95,14 @@ namespace {
return out.isPresent = ReadHelper<T>::Read(val, out.value);
}};
template<> struct ReadHelper<uint64_t> { static bool Read(Value& val, uint64_t& out) {
return val.IsUint64() ? out = val.GetUint64(), true : false;
}};
template<> struct ReadHelper<int64_t> { static bool Read(Value& val, int64_t& out) {
return val.IsInt64() ? out = val.GetInt64(), true : false;
}};
template<class T>
inline static bool ReadValue(Value& val, T& out)
{
@ -311,7 +319,7 @@ inline void Buffer::Read(Value& obj, Asset& r)
" bytes, but found " + to_string(dataURI.dataLength));
}
this->mData.reset(new uint8_t[dataURI.dataLength]);
this->mData.reset(new uint8_t[dataURI.dataLength], std::default_delete<uint8_t[]>());
memcpy( this->mData.get(), dataURI.data, dataURI.dataLength );
}
}
@ -417,7 +425,7 @@ uint8_t* new_data;
// Copy data which place after replacing part.
memcpy(&new_data[pBufferData_Offset + pReplace_Count], &mData.get()[pBufferData_Offset + pBufferData_Count], pBufferData_Offset);
// Apply new data
mData.reset(new_data);
mData.reset(new_data, std::default_delete<uint8_t[]>());
byteLength = new_data_size;
return true;
@ -434,9 +442,19 @@ inline size_t Buffer::AppendData(uint8_t* data, size_t length)
inline void Buffer::Grow(size_t amount)
{
if (amount <= 0) return;
uint8_t* b = new uint8_t[byteLength + amount];
if (capacity >= byteLength + amount)
{
byteLength += amount;
return;
}
// Shift operation is standard way to divide integer by 2, it doesn't cast it to float back and forth, also works for odd numbers,
// originally it would look like: static_cast<size_t>(capacity * 1.5f)
capacity = std::max(capacity + (capacity >> 1), byteLength + amount);
uint8_t* b = new uint8_t[capacity];
if (mData) memcpy(b, mData.get(), byteLength);
mData.reset(b);
mData.reset(b, std::default_delete<uint8_t[]>());
byteLength += amount;
}
@ -1445,7 +1463,7 @@ inline std::string Asset::FindUniqueID(const std::string& str, const char* suffi
if (it == mUsedIds.end())
return id;
char buffer[256];
char buffer[1024];
int offset = ai_snprintf(buffer, sizeof(buffer), "%s_", id.c_str());
for (int i = 0; it != mUsedIds.end(); ++i) {
ai_snprintf(buffer + offset, sizeof(buffer) - offset, "%d", i);