v4games
/
assimp
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Huge optimization of Q3BSPZipArchive to reduce 3DXML importing time by a factor of 15~20. (However, the zip archive is now entirely decompressed in memory, so it takes twice as much memory)

pull/261/head
Léo Terziman 2013-11-19 09:45:17 +01:00
parent 203e896dbc
commit 44f1a1fb5d
2 changed files with 44 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
code/Q3BSPZipArchive.cpp
View File

@ -49,41 +49,24 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
namespace Assimp { namespace Assimp {
namespace Q3BSP { namespace Q3BSP {
ZipFile::ZipFile(const std::string &rFileName, unzFile zipFile) : m_Name(rFileName), m_zipFile(zipFile) { ZipFile::ZipFile(size_t size) : m_Size(size) {
ai_assert(m_zipFile != NULL); ai_assert(m_Size != 0);
m_Buffer = std::malloc(m_Size);
} }
ZipFile::~ZipFile() { ZipFile::~ZipFile() {
m_zipFile = NULL; std::free(m_Buffer);
m_Buffer = NULL;
} }
size_t ZipFile::Read(void* pvBuffer, size_t pSize, size_t pCount) { size_t ZipFile::Read(void* pvBuffer, size_t pSize, size_t pCount) {
size_t bytes_read = 0; const size_t size = pSize * pCount;
assert(size <= m_Size);
if(m_zipFile != NULL) {
// search file and place file pointer there
if(unzLocateFile(m_zipFile, m_Name.c_str(), 0) == UNZ_OK) {
// get file size, etc.
unz_file_info fileInfo;
if(unzGetCurrentFileInfo(m_zipFile, &fileInfo, 0, 0, 0, 0, 0, 0) == UNZ_OK) {
const size_t size = pSize * pCount;
assert(size <= fileInfo.uncompressed_size);
// The file has EXACTLY the size of uncompressed_size. In C std::memcpy(pvBuffer, m_Buffer, size);
// you need to mark the last character with '\0', so add
// another character
if(unzOpenCurrentFile(m_zipFile) == UNZ_OK) {
if(unzReadCurrentFile(m_zipFile, pvBuffer, fileInfo.uncompressed_size) == (long int) fileInfo.uncompressed_size) {
if(unzCloseCurrentFile(m_zipFile) == UNZ_OK) {
bytes_read = fileInfo.uncompressed_size;
}
}
}
}
}
}
return bytes_read; return size;
} }
size_t ZipFile::Write(const void* /*pvBuffer*/, size_t /*pSize*/, size_t /*pCount*/) { size_t ZipFile::Write(const void* /*pvBuffer*/, size_t /*pSize*/, size_t /*pCount*/) {
@ -91,18 +74,7 @@ size_t ZipFile::Write(const void* /*pvBuffer*/, size_t /*pSize*/, size_t /*pCoun
} }
size_t ZipFile::FileSize() const { size_t ZipFile::FileSize() const {
size_t size = 0; return m_Size;
if(m_zipFile != NULL) {
if(unzLocateFile(m_zipFile, m_Name.c_str(), 0) == UNZ_OK) {
unz_file_info fileInfo;
if(unzGetCurrentFileInfo(m_zipFile, &fileInfo, 0, 0, 0, 0, 0, 0) == UNZ_OK) {
size = fileInfo.uncompressed_size;
}
}
}
return size;
} }
aiReturn ZipFile::Seek(size_t /*pOffset*/, aiOrigin /*pOrigin*/) { aiReturn ZipFile::Seek(size_t /*pOffset*/, aiOrigin /*pOrigin*/) {
@ -119,7 +91,7 @@ void ZipFile::Flush() {
// ------------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------------
// Constructor. // Constructor.
Q3BSPZipArchive::Q3BSPZipArchive(const std::string& rFile) : m_ZipFileHandle(NULL), m_ArchiveMap(), m_FileList(), m_bDirty(true) { Q3BSPZipArchive::Q3BSPZipArchive(const std::string& rFile) : m_ZipFileHandle(NULL), m_ArchiveMap() {
if (! rFile.empty()) { if (! rFile.empty()) {
m_ZipFileHandle = unzOpen(rFile.c_str()); m_ZipFileHandle = unzOpen(rFile.c_str());
if(m_ZipFileHandle != NULL) { if(m_ZipFileHandle != NULL) {
@ -131,14 +103,11 @@ Q3BSPZipArchive::Q3BSPZipArchive(const std::string& rFile) : m_ZipFileHandle(NUL
// ------------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------------
// Destructor. // Destructor.
Q3BSPZipArchive::~Q3BSPZipArchive() { Q3BSPZipArchive::~Q3BSPZipArchive() {
for( std::map<IOStream*, std::string>::iterator it(m_ArchiveMap.begin()), end(m_ArchiveMap.end()); it != end; ++it ) { for( std::map<std::string, ZipFile*>::iterator it(m_ArchiveMap.begin()), end(m_ArchiveMap.end()); it != end; ++it ) {
ZipFile *pZipFile = reinterpret_cast<ZipFile*>(it->first); delete it->second;
delete pZipFile;
} }
m_ArchiveMap.clear(); m_ArchiveMap.clear();
m_FileList.clear();
if(m_ZipFileHandle != NULL) { if(m_ZipFileHandle != NULL) {
unzClose(m_ZipFileHandle); unzClose(m_ZipFileHandle);
m_ZipFileHandle = NULL; m_ZipFileHandle = NULL;
@ -160,9 +129,9 @@ bool Q3BSPZipArchive::Exists(const char* pFile) const {
if (pFile != NULL) { if (pFile != NULL) {
std::string rFile(pFile); std::string rFile(pFile);
std::set<std::string>::const_iterator it = m_FileList.find(rFile); std::map<std::string, ZipFile*>::const_iterator it = m_ArchiveMap.find(rFile);
if(it != m_FileList.end()) { if(it != m_ArchiveMap.end()) {
exist = true; exist = true;
} }
} }
@ -187,14 +156,10 @@ IOStream *Q3BSPZipArchive::Open(const char* pFile, const char* /*pMode*/) {
IOStream* result = NULL; IOStream* result = NULL;
std::string rItem(pFile); std::map<std::string, ZipFile*>::iterator it = m_ArchiveMap.find(pFile);
std::set<std::string>::iterator it = m_FileList.find(rItem);
if(it != m_FileList.end()) { if(it != m_ArchiveMap.end()) {
ZipFile *pZipFile = new ZipFile(*it, m_ZipFileHandle); result = (IOStream*) it->second;
m_ArchiveMap[pZipFile] = rItem;
result = pZipFile;
} }
return result; return result;
@ -205,21 +170,16 @@ IOStream *Q3BSPZipArchive::Open(const char* pFile, const char* /*pMode*/) {
void Q3BSPZipArchive::Close(IOStream *pFile) { void Q3BSPZipArchive::Close(IOStream *pFile) {
ai_assert(pFile != NULL); ai_assert(pFile != NULL);
std::map<IOStream*, std::string>::iterator it = m_ArchiveMap.find(pFile); // We don't do anything in case the file would be opened again in the future
if(it != m_ArchiveMap.end()) {
ZipFile *pZipFile = reinterpret_cast<ZipFile*>((*it).first);
delete pZipFile;
m_ArchiveMap.erase(it);
}
} }
// ------------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------------
// Returns the file-list of the archive. // Returns the file-list of the archive.
void Q3BSPZipArchive::getFileList(std::vector<std::string> &rFileList) { void Q3BSPZipArchive::getFileList(std::vector<std::string> &rFileList) {
rFileList.clear(); rFileList.clear();
std::copy(m_FileList.begin(), m_FileList.end(), rFileList.begin()); for(std::map<std::string, ZipFile*>::iterator it(m_ArchiveMap.begin()), end(m_ArchiveMap.end()); it != end; ++it) {
rFileList.push_back(it->first);
}
} }
// ------------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------------
@ -228,22 +188,30 @@ bool Q3BSPZipArchive::mapArchive() {
bool success = false; bool success = false;
if(m_ZipFileHandle != NULL) { if(m_ZipFileHandle != NULL) {
if(m_bDirty) { if(m_ArchiveMap.empty()) {
m_FileList.clear();
// At first ensure file is already open // At first ensure file is already open
if(unzGoToFirstFile(m_ZipFileHandle) == UNZ_OK) { if(unzGoToFirstFile(m_ZipFileHandle) == UNZ_OK) {
// Loop over all files // Loop over all files
do { do {
char filename[FileNameSize]; char filename[FileNameSize];
unz_file_info fileInfo;
if(unzGetCurrentFileInfo(m_ZipFileHandle, NULL, filename, FileNameSize, NULL, 0, NULL, 0) == UNZ_OK) { if(unzGetCurrentFileInfo(m_ZipFileHandle, &fileInfo, filename, FileNameSize, NULL, 0, NULL, 0) == UNZ_OK) {
m_FileList.insert(filename); // The file has EXACTLY the size of uncompressed_size. In C
// you need to mark the last character with '\0', so add
// another character
if(unzOpenCurrentFile(m_ZipFileHandle) == UNZ_OK) {
std::pair<std::map<std::string, ZipFile*>::iterator, bool> result = m_ArchiveMap.insert(std::make_pair(filename, new ZipFile(fileInfo.uncompressed_size)));
if(unzReadCurrentFile(m_ZipFileHandle, result.first->second->m_Buffer, fileInfo.uncompressed_size) == (long int) fileInfo.uncompressed_size) {
if(unzCloseCurrentFile(m_ZipFileHandle) == UNZ_OK) {
// Nothing to do anymore...
}
}
}
} }
} while(unzGoToNextFile(m_ZipFileHandle) != UNZ_END_OF_LIST_OF_FILE); } while(unzGoToNextFile(m_ZipFileHandle) != UNZ_END_OF_LIST_OF_FILE);
} }
m_bDirty = false;
} }
success = true; success = true;

13
code/Q3BSPZipArchive.h
View File

@ -59,9 +59,11 @@ namespace Q3BSP {
// ------------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------------
class ZipFile : public IOStream { class ZipFile : public IOStream {
friend class Q3BSPZipArchive;
public: public:
ZipFile(const std::string &rFileName, unzFile zipFile); ZipFile(size_t size);
~ZipFile(); ~ZipFile();
@ -79,9 +81,9 @@ class ZipFile : public IOStream {
private: private:
std::string m_Name; void* m_Buffer;
unzFile m_zipFile; size_t m_Size;
}; };
// ------------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------------
@ -123,11 +125,8 @@ class Q3BSPZipArchive : public Assimp::IOSystem {
unzFile m_ZipFileHandle; unzFile m_ZipFileHandle;
std::map<IOStream*, std::string> m_ArchiveMap; std::map<std::string, ZipFile*> m_ArchiveMap;
std::set<std::string> m_FileList;
bool m_bDirty;
}; };
// ------------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------------