Buffered stream API for big files: add next try of a prototype.^

pull/1043/head
Kim Kulling 2016-10-11 12:29:40 +02:00
parent bf06ee3ed4
commit 905e48a35c
5 changed files with 47 additions and 95 deletions

View File

@ -171,6 +171,7 @@ SET( Common_SRCS
Bitmap.h
XMLTools.h
Version.cpp
IOStreamBuffer.h
)
SOURCE_GROUP(Common FILES ${Common_SRCS})

View File

@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "ObjFileImporter.h"
#include "ObjFileParser.h"
#include "ObjFileData.h"
#include "IOStreamBuffer.h"
#include <memory>
#include <assimp/Importer.hpp>
#include <assimp/scene.h>
@ -70,78 +71,6 @@ namespace Assimp {
using namespace std;
template<class T>
struct IOStreamBuffer {
BaseImporter *m_importer;
IOStream *m_stream;
size_t m_cacheSize;
std::vector<T> m_buffer;
size_t m_filesize;
size_t c;
size_t m_numBlocks;
size_t m_sizeLastBlock;
IOStreamBuffer( BaseImporter *imp, IOStream *stream, size_t cache = 4096 )
: m_importer( imp )
, m_stream( stream )
, m_cacheSize( cache )
, m_buffer()
, m_filesize( 0 )
, m_blockIndex( 0 )
, m_numBlocks( 0 )
, m_sizeLastBlock( 0 ) {
m_buffer.resize( m_cacheSize );
m_filesize = m_stream->FileSize();
m_numBlocks = m_filesize / m_cacheSize;
m_sizeLastBlock = m_filesize % m_cacheSize;
}
~IOStreamBuffer() {
clear();
}
void clear() {
m_cacheSize = 4096;
m_buffer.resize( 0 );
}
bool mapPosToBlock( size_t pos, size_t &blockIdx, size_t &blockPos ) {
if ( pos > m_filesize ) {
return false;
}
blockIdx = pos / m_cacheSize;
blockPos = pos % m_cacheSize;
return true;
}
void loadBlock( size_t blockIdx ) {
size_t pos = blockIdx * m_cacheSize;
size_t sizeToRead( m_cacheSize );
if ( m_blockIdx == ( m_numBlocks - 1 ) ) {
sizeToRead = m_sizeLastBlock;
}
m_stream->Seek( pos <, aiOrigin_SET );
m_stream->Read( &m_buffer[ 0 ], sizeof( T ), sizeToRead );
m_blockIndex = blockIdx;
BaseImporter::ConvertToUTF8( m_buffer );
}
T &operator [] (size_t pos ) {
size_t blockIdx( 0 ), blockPos( 0 );
if ( !mapPosToBlock( pos, blockIdx, blockPos ) ) {
throw DeadlyImportError( "OBJ-file-access out of bounds." );
}
if ( m_blockIndex != blockIdx ) {
loadBlock( blockIdx );
}
return m_buffer[ blockPos ];
}
};
// ------------------------------------------------------------------------------------------------
// Default constructor
ObjFileImporter::ObjFileImporter() :
@ -198,8 +127,11 @@ void ObjFileImporter::InternReadFile( const std::string &file, aiScene* pScene,
throw DeadlyImportError( "OBJ-file is too small.");
}
IOStreamBuffer<char> streamedBuffer;
streamedBuffer.open( fileStream.get() );
// Allocate buffer and read file into it
TextFileToBuffer( fileStream.get(),m_Buffer);
//TextFileToBuffer( fileStream.get(),m_Buffer);
// Get the model name
std::string modelName, folderName;
@ -222,7 +154,7 @@ void ObjFileImporter::InternReadFile( const std::string &file, aiScene* pScene,
const unsigned int updateProgressEveryBytes = 100 * 1024;
const unsigned int progressTotal = (3*m_Buffer.size()/updateProgressEveryBytes);
// process all '\'
std::vector<char> ::iterator iter = m_Buffer.begin();
/*std::vector<char> ::iterator iter = m_Buffer.begin();
while (iter != m_Buffer.end())
{
if (*iter == '\\')
@ -241,17 +173,19 @@ void ObjFileImporter::InternReadFile( const std::string &file, aiScene* pScene,
m_progress->UpdateFileRead(++progress, progressTotal);
progressCounter = 0;
}
}
}*/
// 1/3rd progress
m_progress->UpdateFileRead(1, 3);
// parse the file into a temporary representation
ObjFileParser parser(m_Buffer, modelName, pIOHandler, m_progress, file);
ObjFileParser parser( streamedBuffer, modelName, pIOHandler, m_progress, file);
// And create the proper return structures out of it
CreateDataFromImport(parser.GetModel(), pScene);
streamedBuffer.close();
// Clean up allocated storage for the next import
m_Buffer.clear();

View File

@ -61,9 +61,9 @@ const std::string ObjFileParser::DEFAULT_MATERIAL = AI_DEFAULT_MATERIAL_NAME;
// -------------------------------------------------------------------
// Constructor with loaded data and directories.
ObjFileParser::ObjFileParser(std::vector<char> &data, const std::string &modelName, IOSystem *io, ProgressHandler* progress, const std::string &originalObjFileName) :
m_DataIt(data.begin()),
m_DataItEnd(data.end()),
ObjFileParser::ObjFileParser( IOStreamBuffer<char> &streamBuffer, const std::string &modelName, IOSystem *io, ProgressHandler* progress, const std::string &originalObjFileName) :
m_DataIt(),
m_DataItEnd(),
m_pModel(NULL),
m_uiLine(0),
m_pIO( io ),
@ -83,7 +83,7 @@ ObjFileParser::ObjFileParser(std::vector<char> &data, const std::string &modelNa
m_pModel->m_MaterialMap[ DEFAULT_MATERIAL ] = m_pModel->m_pDefaultMaterial;
// Start parsing the file
parseFile();
parseFile( streamBuffer );
}
// -------------------------------------------------------------------
@ -103,31 +103,33 @@ ObjFile::Model *ObjFileParser::GetModel() const
// -------------------------------------------------------------------
// File parsing method.
void ObjFileParser::parseFile()
{
if (m_DataIt == m_DataItEnd)
return;
void ObjFileParser::parseFile( IOStreamBuffer<char> &streamBuffer ) {
// only update every 100KB or it'll be too slow
const unsigned int updateProgressEveryBytes = 100 * 1024;
unsigned int progressCounter = 0;
const unsigned int bytesToProcess = std::distance(m_DataIt, m_DataItEnd);
const unsigned int bytesToProcess = streamBuffer.size();
const unsigned int progressTotal = 3 * bytesToProcess;
const unsigned int progressOffset = bytesToProcess;
unsigned int processed = 0;
DataArrayIt lastDataIt = m_DataIt;
//DataArrayIt lastDataIt = m_DataIt;
bool endOfFile( false );
std::vector<char> buffer;
//while ( m_DataIt != m_DataItEnd )
while ( streamBuffer.getNextLine( buffer ) ) {
m_DataIt = buffer.begin();
m_DataItEnd = buffer.end();
while (m_DataIt != m_DataItEnd)
{
// Handle progress reporting
processed += std::distance(lastDataIt, m_DataIt);
/*processed += std::distance(lastDataIt, m_DataIt);
lastDataIt = m_DataIt;
if (processed > (progressCounter * updateProgressEveryBytes))
{
progressCounter++;
m_progress->UpdateFileRead(progressOffset + processed*2, progressTotal);
}
}*/
// parse line
switch (*m_DataIt)
@ -149,8 +151,8 @@ void ObjFileParser::parseFile()
}
} else if (*m_DataIt == 't') {
// read in texture coordinate ( 2D or 3D )
++m_DataIt;
getVector( m_pModel->m_TextureCoord );
++m_DataIt;
getVector( m_pModel->m_TextureCoord );
} else if (*m_DataIt == 'n') {
// Read in normal vector definition
++m_DataIt;

View File

@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <assimp/vector2.h>
#include <assimp/vector3.h>
#include <assimp/mesh.h>
#include "IOStreamBuffer.h"
namespace Assimp {
@ -72,7 +73,7 @@ public:
public:
/// \brief Constructor with data array.
ObjFileParser(std::vector<char> &Data, const std::string &strModelName, IOSystem* io, ProgressHandler* progress, const std::string &originalObjFileName);
ObjFileParser( IOStreamBuffer<char> &streamBuffer, const std::string &strModelName, IOSystem* io, ProgressHandler* progress, const std::string &originalObjFileName);
/// \brief Destructor
~ObjFileParser();
/// \brief Model getter.
@ -80,7 +81,7 @@ public:
private:
/// Parse the loaded file
void parseFile();
void parseFile( IOStreamBuffer<char> &streamBuffer );
/// Method to copy the new delimited word in the current line.
void copyNextWord(char *pBuffer, size_t length);
/// Method to copy the new line.

View File

@ -246,6 +246,20 @@ string_type trim_whitespaces(string_type str)
return str;
}
template<class T>
bool hasLineEnd( T it, T end ) {
bool hasLineEnd( false );
while ( !isEndOfBuffer( it, end ) ) {
it++;
if ( IsLineEnd( it ) ) {
hasLineEnd = true;
break;
}
}
return hasLineEnd;
}
} // Namespace Assimp
#endif // OBJ_TOOLS_H_INC