Buffered stream API for big files: add next try of a prototype.^
Kim Kulling
parent
bf06ee3ed4
commit
905e48a35c
|
|
@ -171,6 +171,7 @@ SET( Common_SRCS
|
|||
Bitmap.h
|
||||
XMLTools.h
|
||||
Version.cpp
|
||||
IOStreamBuffer.h
|
||||
)
|
||||
SOURCE_GROUP(Common FILES ${Common_SRCS})
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|||
#include "ObjFileImporter.h"
|
||||
#include "ObjFileParser.h"
|
||||
#include "ObjFileData.h"
|
||||
#include "IOStreamBuffer.h"
|
||||
#include <memory>
|
||||
#include <assimp/Importer.hpp>
|
||||
#include <assimp/scene.h>
|
||||
|
|
@ -70,78 +71,6 @@ namespace Assimp {
|
|||
|
||||
using namespace std;
|
||||
|
||||
template<class T>
|
||||
struct IOStreamBuffer {
|
||||
BaseImporter *m_importer;
|
||||
IOStream *m_stream;
|
||||
size_t m_cacheSize;
|
||||
std::vector<T> m_buffer;
|
||||
size_t m_filesize;
|
||||
size_t c;
|
||||
size_t m_numBlocks;
|
||||
size_t m_sizeLastBlock;
|
||||
|
||||
IOStreamBuffer( BaseImporter *imp, IOStream *stream, size_t cache = 4096 )
|
||||
: m_importer( imp )
|
||||
, m_stream( stream )
|
||||
, m_cacheSize( cache )
|
||||
, m_buffer()
|
||||
, m_filesize( 0 )
|
||||
, m_blockIndex( 0 )
|
||||
, m_numBlocks( 0 )
|
||||
, m_sizeLastBlock( 0 ) {
|
||||
m_buffer.resize( m_cacheSize );
|
||||
m_filesize = m_stream->FileSize();
|
||||
m_numBlocks = m_filesize / m_cacheSize;
|
||||
m_sizeLastBlock = m_filesize % m_cacheSize;
|
||||
}
|
||||
|
||||
~IOStreamBuffer() {
|
||||
clear();
|
||||
}
|
||||
|
||||
void clear() {
|
||||
m_cacheSize = 4096;
|
||||
m_buffer.resize( 0 );
|
||||
}
|
||||
|
||||
bool mapPosToBlock( size_t pos, size_t &blockIdx, size_t &blockPos ) {
|
||||
if ( pos > m_filesize ) {
|
||||
return false;
|
||||
}
|
||||
blockIdx = pos / m_cacheSize;
|
||||
blockPos = pos % m_cacheSize;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void loadBlock( size_t blockIdx ) {
|
||||
size_t pos = blockIdx * m_cacheSize;
|
||||
size_t sizeToRead( m_cacheSize );
|
||||
if ( m_blockIdx == ( m_numBlocks - 1 ) ) {
|
||||
sizeToRead = m_sizeLastBlock;
|
||||
}
|
||||
|
||||
m_stream->Seek( pos <, aiOrigin_SET );
|
||||
m_stream->Read( &m_buffer[ 0 ], sizeof( T ), sizeToRead );
|
||||
m_blockIndex = blockIdx;
|
||||
|
||||
BaseImporter::ConvertToUTF8( m_buffer );
|
||||
}
|
||||
|
||||
T &operator [] (size_t pos ) {
|
||||
size_t blockIdx( 0 ), blockPos( 0 );
|
||||
if ( !mapPosToBlock( pos, blockIdx, blockPos ) ) {
|
||||
throw DeadlyImportError( "OBJ-file-access out of bounds." );
|
||||
}
|
||||
if ( m_blockIndex != blockIdx ) {
|
||||
loadBlock( blockIdx );
|
||||
}
|
||||
|
||||
return m_buffer[ blockPos ];
|
||||
}
|
||||
};
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// Default constructor
|
||||
ObjFileImporter::ObjFileImporter() :
|
||||
|
|
@ -198,8 +127,11 @@ void ObjFileImporter::InternReadFile( const std::string &file, aiScene* pScene,
|
|||
throw DeadlyImportError( "OBJ-file is too small.");
|
||||
}
|
||||
|
||||
IOStreamBuffer<char> streamedBuffer;
|
||||
streamedBuffer.open( fileStream.get() );
|
||||
|
||||
// Allocate buffer and read file into it
|
||||
TextFileToBuffer( fileStream.get(),m_Buffer);
|
||||
//TextFileToBuffer( fileStream.get(),m_Buffer);
|
||||
|
||||
// Get the model name
|
||||
std::string modelName, folderName;
|
||||
|
|
@ -222,7 +154,7 @@ void ObjFileImporter::InternReadFile( const std::string &file, aiScene* pScene,
|
|||
const unsigned int updateProgressEveryBytes = 100 * 1024;
|
||||
const unsigned int progressTotal = (3*m_Buffer.size()/updateProgressEveryBytes);
|
||||
// process all '\'
|
||||
std::vector<char> ::iterator iter = m_Buffer.begin();
|
||||
/*std::vector<char> ::iterator iter = m_Buffer.begin();
|
||||
while (iter != m_Buffer.end())
|
||||
{
|
||||
if (*iter == '\\')
|
||||
|
|
@ -241,17 +173,19 @@ void ObjFileImporter::InternReadFile( const std::string &file, aiScene* pScene,
|
|||
m_progress->UpdateFileRead(++progress, progressTotal);
|
||||
progressCounter = 0;
|
||||
}
|
||||
}
|
||||
}*/
|
||||
|
||||
// 1/3rd progress
|
||||
m_progress->UpdateFileRead(1, 3);
|
||||
|
||||
// parse the file into a temporary representation
|
||||
ObjFileParser parser(m_Buffer, modelName, pIOHandler, m_progress, file);
|
||||
ObjFileParser parser( streamedBuffer, modelName, pIOHandler, m_progress, file);
|
||||
|
||||
// And create the proper return structures out of it
|
||||
CreateDataFromImport(parser.GetModel(), pScene);
|
||||
|
||||
streamedBuffer.close();
|
||||
|
||||
// Clean up allocated storage for the next import
|
||||
m_Buffer.clear();
|
||||
|
||||
|
|
|
|||
|
|
@ -61,9 +61,9 @@ const std::string ObjFileParser::DEFAULT_MATERIAL = AI_DEFAULT_MATERIAL_NAME;
|
|||
|
||||
// -------------------------------------------------------------------
|
||||
// Constructor with loaded data and directories.
|
||||
ObjFileParser::ObjFileParser(std::vector<char> &data, const std::string &modelName, IOSystem *io, ProgressHandler* progress, const std::string &originalObjFileName) :
|
||||
m_DataIt(data.begin()),
|
||||
m_DataItEnd(data.end()),
|
||||
ObjFileParser::ObjFileParser( IOStreamBuffer<char> &streamBuffer, const std::string &modelName, IOSystem *io, ProgressHandler* progress, const std::string &originalObjFileName) :
|
||||
m_DataIt(),
|
||||
m_DataItEnd(),
|
||||
m_pModel(NULL),
|
||||
m_uiLine(0),
|
||||
m_pIO( io ),
|
||||
|
|
@ -83,7 +83,7 @@ ObjFileParser::ObjFileParser(std::vector<char> &data, const std::string &modelNa
|
|||
m_pModel->m_MaterialMap[ DEFAULT_MATERIAL ] = m_pModel->m_pDefaultMaterial;
|
||||
|
||||
// Start parsing the file
|
||||
parseFile();
|
||||
parseFile( streamBuffer );
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
|
|
@ -103,31 +103,33 @@ ObjFile::Model *ObjFileParser::GetModel() const
|
|||
|
||||
// -------------------------------------------------------------------
|
||||
// File parsing method.
|
||||
void ObjFileParser::parseFile()
|
||||
{
|
||||
if (m_DataIt == m_DataItEnd)
|
||||
return;
|
||||
|
||||
void ObjFileParser::parseFile( IOStreamBuffer<char> &streamBuffer ) {
|
||||
// only update every 100KB or it'll be too slow
|
||||
const unsigned int updateProgressEveryBytes = 100 * 1024;
|
||||
unsigned int progressCounter = 0;
|
||||
const unsigned int bytesToProcess = std::distance(m_DataIt, m_DataItEnd);
|
||||
const unsigned int bytesToProcess = streamBuffer.size();
|
||||
const unsigned int progressTotal = 3 * bytesToProcess;
|
||||
const unsigned int progressOffset = bytesToProcess;
|
||||
unsigned int processed = 0;
|
||||
|
||||
DataArrayIt lastDataIt = m_DataIt;
|
||||
//DataArrayIt lastDataIt = m_DataIt;
|
||||
|
||||
bool endOfFile( false );
|
||||
std::vector<char> buffer;
|
||||
|
||||
//while ( m_DataIt != m_DataItEnd )
|
||||
while ( streamBuffer.getNextLine( buffer ) ) {
|
||||
m_DataIt = buffer.begin();
|
||||
m_DataItEnd = buffer.end();
|
||||
|
||||
while (m_DataIt != m_DataItEnd)
|
||||
{
|
||||
// Handle progress reporting
|
||||
processed += std::distance(lastDataIt, m_DataIt);
|
||||
/*processed += std::distance(lastDataIt, m_DataIt);
|
||||
lastDataIt = m_DataIt;
|
||||
if (processed > (progressCounter * updateProgressEveryBytes))
|
||||
{
|
||||
progressCounter++;
|
||||
m_progress->UpdateFileRead(progressOffset + processed*2, progressTotal);
|
||||
}
|
||||
}*/
|
||||
|
||||
// parse line
|
||||
switch (*m_DataIt)
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|||
#include <assimp/vector2.h>
|
||||
#include <assimp/vector3.h>
|
||||
#include <assimp/mesh.h>
|
||||
#include "IOStreamBuffer.h"
|
||||
|
||||
namespace Assimp {
|
||||
|
||||
|
|
@ -72,7 +73,7 @@ public:
|
|||
|
||||
public:
|
||||
/// \brief Constructor with data array.
|
||||
ObjFileParser(std::vector<char> &Data, const std::string &strModelName, IOSystem* io, ProgressHandler* progress, const std::string &originalObjFileName);
|
||||
ObjFileParser( IOStreamBuffer<char> &streamBuffer, const std::string &strModelName, IOSystem* io, ProgressHandler* progress, const std::string &originalObjFileName);
|
||||
/// \brief Destructor
|
||||
~ObjFileParser();
|
||||
/// \brief Model getter.
|
||||
|
|
@ -80,7 +81,7 @@ public:
|
|||
|
||||
private:
|
||||
/// Parse the loaded file
|
||||
void parseFile();
|
||||
void parseFile( IOStreamBuffer<char> &streamBuffer );
|
||||
/// Method to copy the new delimited word in the current line.
|
||||
void copyNextWord(char *pBuffer, size_t length);
|
||||
/// Method to copy the new line.
|
||||
|
|
|
|||
|
|
@ -246,6 +246,20 @@ string_type trim_whitespaces(string_type str)
|
|||
return str;
|
||||
}
|
||||
|
||||
template<class T>
|
||||
bool hasLineEnd( T it, T end ) {
|
||||
bool hasLineEnd( false );
|
||||
while ( !isEndOfBuffer( it, end ) ) {
|
||||
it++;
|
||||
if ( IsLineEnd( it ) ) {
|
||||
hasLineEnd = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return hasLineEnd;
|
||||
}
|
||||
|
||||
} // Namespace Assimp
|
||||
|
||||
#endif // OBJ_TOOLS_H_INC
|
||||
|
|
|
|||
Loading…
Reference in New Issue