From 727cf0c49f552073fe022c4ec24526dc0c561df0 Mon Sep 17 00:00:00 2001 From: Ryan McCampbell Date: Mon, 8 Jul 2019 23:21:57 -0700 Subject: [PATCH 1/6] Fixed unicode on windows --- code/Common/DefaultIOSystem.cpp | 80 +++++++++------------------------ 1 file changed, 20 insertions(+), 60 deletions(-) diff --git a/code/Common/DefaultIOSystem.cpp b/code/Common/DefaultIOSystem.cpp index d40b67de3..7606d8ced 100644 --- a/code/Common/DefaultIOSystem.cpp +++ b/code/Common/DefaultIOSystem.cpp @@ -73,28 +73,14 @@ using namespace Assimp; // Tests for the existence of a file at the given path. bool DefaultIOSystem::Exists( const char* pFile) const { -#ifdef _WIN32 +#if defined(_WIN32) && !defined(WindowsStore) wchar_t fileName16[PATHLIMIT]; -#ifndef WindowsStore - bool isUnicode = IsTextUnicode(pFile, static_cast(strlen(pFile)), NULL) != 0; - if (isUnicode) { - - MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, pFile, -1, fileName16, PATHLIMIT); - struct __stat64 filestat; - if (0 != _wstat64(fileName16, &filestat)) { - return false; - } - } else { -#endif - FILE* file = ::fopen(pFile, "rb"); - if (!file) - return false; - - ::fclose(file); -#ifndef WindowsStore + MultiByteToWideChar(CP_UTF8, 0, pFile, -1, fileName16, PATHLIMIT); + struct __stat64 filestat; + if (0 != _wstat64(fileName16, &filestat)) { + return false; } -#endif #else FILE* file = ::fopen( pFile, "rb"); if( !file) @@ -112,27 +98,19 @@ IOStream* DefaultIOSystem::Open( const char* strFile, const char* strMode) ai_assert(NULL != strFile); ai_assert(NULL != strMode); FILE* file; -#ifdef _WIN32 +#if defined(_WIN32) && !defined(WindowsStore) wchar_t fileName16[PATHLIMIT]; -#ifndef WindowsStore - bool isUnicode = IsTextUnicode(strFile, static_cast(strlen(strFile)), NULL) != 0; - if (isUnicode) { - MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, strFile, -1, fileName16, PATHLIMIT); - std::string mode8(strMode); - file = ::_wfopen(fileName16, std::wstring(mode8.begin(), mode8.end()).c_str()); - } else { -#endif - file = ::fopen(strFile, strMode); -#ifndef WindowsStore - } -#endif + MultiByteToWideChar(CP_UTF8, 0, strFile, -1, fileName16, PATHLIMIT); + std::string mode8(strMode); + std::wstring mode16(mode8.begin(), mode8.end()); + file = ::_wfopen(fileName16, mode16.c_str()); #else file = ::fopen(strFile, strMode); #endif if (nullptr == file) return nullptr; - return new DefaultIOStream(file, (std::string) strFile); + return new DefaultIOStream(file, strFile); } // ------------------------------------------------------------------------------------------------ @@ -167,44 +145,26 @@ inline static void MakeAbsolutePath (const char* in, char* _out) ai_assert(in && _out); #if defined( _MSC_VER ) || defined( __MINGW32__ ) #ifndef WindowsStore - bool isUnicode = IsTextUnicode(in, static_cast(strlen(in)), NULL) != 0; - if (isUnicode) { - wchar_t out16[PATHLIMIT]; - wchar_t in16[PATHLIMIT]; - MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, in, -1, out16, PATHLIMIT); - wchar_t* ret = ::_wfullpath(out16, in16, PATHLIMIT); - if (ret) { - WideCharToMultiByte(CP_UTF8, MB_PRECOMPOSED, out16, -1, _out, PATHLIMIT, nullptr, nullptr); - } - if (!ret) { - // preserve the input path, maybe someone else is able to fix - // the path before it is accessed (e.g. our file system filter) - ASSIMP_LOG_WARN_F("Invalid path: ", std::string(in)); - strcpy(_out, in); - } - - } else { -#endif - char* ret = :: _fullpath(_out, in, PATHLIMIT); - if (!ret) { - // preserve the input path, maybe someone else is able to fix - // the path before it is accessed (e.g. our file system filter) - ASSIMP_LOG_WARN_F("Invalid path: ", std::string(in)); - strcpy(_out, in); - } -#ifndef WindowsStore + wchar_t in16[PATHLIMIT]; + wchar_t out16[PATHLIMIT]; + MultiByteToWideChar(CP_UTF8, 0, in, -1, in16, PATHLIMIT); + wchar_t* ret = ::_wfullpath(out16, in16, PATHLIMIT); + if (ret) { + WideCharToMultiByte(CP_UTF8, 0, out16, -1, _out, PATHLIMIT, nullptr, nullptr); } +#else + char* ret = ::_fullpath(_out, in, PATHLIMIT); #endif #else // use realpath char* ret = realpath(in, _out); +#endif if(!ret) { // preserve the input path, maybe someone else is able to fix // the path before it is accessed (e.g. our file system filter) ASSIMP_LOG_WARN_F("Invalid path: ", std::string(in)); strcpy(_out,in); } -#endif } // ------------------------------------------------------------------------------------------------ From 6c65b5b5496638c149907a605003f9a74f7bceb7 Mon Sep 17 00:00:00 2001 From: Ryan McCampbell Date: Tue, 9 Jul 2019 02:17:45 -0700 Subject: [PATCH 2/6] Cleaned up unicode code and removed fixed-size buffers --- code/Common/DefaultIOSystem.cpp | 74 ++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/code/Common/DefaultIOSystem.cpp b/code/Common/DefaultIOSystem.cpp index 7606d8ced..da7376809 100644 --- a/code/Common/DefaultIOSystem.cpp +++ b/code/Common/DefaultIOSystem.cpp @@ -61,12 +61,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. using namespace Assimp; -// maximum path length -// XXX http://insanecoding.blogspot.com/2007/11/pathmax-simply-isnt.html -#ifdef PATH_MAX -# define PATHLIMIT PATH_MAX -#else -# define PATHLIMIT 4096 +#ifdef _WIN32 +static std::wstring Utf8ToWide(const char* in) +{ + int size = MultiByteToWideChar(CP_UTF8, 0, in, -1, nullptr, 0); + std::wstring out(size, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, in, -1, &out[0], size); + return out; +} + +static std::string WideToUtf8(const wchar_t* in) +{ + int size = WideCharToMultiByte(CP_UTF8, 0, in, -1, nullptr, 0, nullptr, nullptr); + std::string out(size, '\0'); + WideCharToMultiByte(CP_UTF8, 0, in, -1, &out[0], size, nullptr, nullptr); + return out; +} #endif // ------------------------------------------------------------------------------------------------ @@ -74,11 +84,8 @@ using namespace Assimp; bool DefaultIOSystem::Exists( const char* pFile) const { #if defined(_WIN32) && !defined(WindowsStore) - wchar_t fileName16[PATHLIMIT]; - - MultiByteToWideChar(CP_UTF8, 0, pFile, -1, fileName16, PATHLIMIT); struct __stat64 filestat; - if (0 != _wstat64(fileName16, &filestat)) { + if (0 != _wstat64(Utf8ToWide(pFile).c_str(), &filestat)) { return false; } #else @@ -99,11 +106,7 @@ IOStream* DefaultIOSystem::Open( const char* strFile, const char* strMode) ai_assert(NULL != strMode); FILE* file; #if defined(_WIN32) && !defined(WindowsStore) - wchar_t fileName16[PATHLIMIT]; - MultiByteToWideChar(CP_UTF8, 0, strFile, -1, fileName16, PATHLIMIT); - std::string mode8(strMode); - std::wstring mode16(mode8.begin(), mode8.end()); - file = ::_wfopen(fileName16, mode16.c_str()); + file = ::_wfopen(Utf8ToWide(strFile).c_str(), Utf8ToWide(strMode).c_str()); #else file = ::fopen(strFile, strMode); #endif @@ -140,31 +143,39 @@ bool IOSystem::ComparePaths (const char* one, const char* second) const // ------------------------------------------------------------------------------------------------ // Convert a relative path into an absolute path -inline static void MakeAbsolutePath (const char* in, char* _out) +inline static std::string MakeAbsolutePath (const char* in) { - ai_assert(in && _out); -#if defined( _MSC_VER ) || defined( __MINGW32__ ) + ai_assert(in); + std::string out; +#ifdef _WIN32 #ifndef WindowsStore - wchar_t in16[PATHLIMIT]; - wchar_t out16[PATHLIMIT]; - MultiByteToWideChar(CP_UTF8, 0, in, -1, in16, PATHLIMIT); - wchar_t* ret = ::_wfullpath(out16, in16, PATHLIMIT); + wchar_t* ret = ::_wfullpath(nullptr, Utf8ToWide(in).c_str(), 0); if (ret) { - WideCharToMultiByte(CP_UTF8, 0, out16, -1, _out, PATHLIMIT, nullptr, nullptr); + out = WideToUtf8(ret); + free(ret); } #else - char* ret = ::_fullpath(_out, in, PATHLIMIT); + char* ret = ::_fullpath(nullptr, in, 0); + if (ret) { + out = ret; + free(ret); + } #endif #else // use realpath - char* ret = realpath(in, _out); + char* ret = realpath(in, nullptr); + if (ret) { + out = ret; + free(ret); + } #endif - if(!ret) { + if (!ret) { // preserve the input path, maybe someone else is able to fix // the path before it is accessed (e.g. our file system filter) ASSIMP_LOG_WARN_F("Invalid path: ", std::string(in)); - strcpy(_out,in); + out = in; } + return out; } // ------------------------------------------------------------------------------------------------ @@ -176,11 +187,8 @@ bool DefaultIOSystem::ComparePaths (const char* one, const char* second) const if( !ASSIMP_stricmp(one,second) ) return true; - char temp1[PATHLIMIT]; - char temp2[PATHLIMIT]; - - MakeAbsolutePath (one, temp1); - MakeAbsolutePath (second, temp2); + std::string temp1 = MakeAbsolutePath(one); + std::string temp2 = MakeAbsolutePath(second); return !ASSIMP_stricmp(temp1,temp2); } @@ -213,5 +221,3 @@ std::string DefaultIOSystem::absolutePath( const std::string &path ) } // ------------------------------------------------------------------------------------------------ - -#undef PATHLIMIT From 0685e415ad6b32b23eef89315bd5658d9bf021c1 Mon Sep 17 00:00:00 2001 From: Ryan McCampbell Date: Mon, 19 Aug 2019 12:00:01 -0400 Subject: [PATCH 3/6] removed extra null character in Utf8ToWide and WideToUtf8 --- code/Common/DefaultIOSystem.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/code/Common/DefaultIOSystem.cpp b/code/Common/DefaultIOSystem.cpp index da7376809..7e26ec4b2 100644 --- a/code/Common/DefaultIOSystem.cpp +++ b/code/Common/DefaultIOSystem.cpp @@ -64,16 +64,18 @@ using namespace Assimp; #ifdef _WIN32 static std::wstring Utf8ToWide(const char* in) { - int size = MultiByteToWideChar(CP_UTF8, 0, in, -1, nullptr, 0); - std::wstring out(size, L'\0'); + size_t size = MultiByteToWideChar(CP_UTF8, 0, in, -1, nullptr, 0); + // size includes terminating null; std::wstring adds null automatically + std::wstring out(size - 1, L'\0'); MultiByteToWideChar(CP_UTF8, 0, in, -1, &out[0], size); return out; } static std::string WideToUtf8(const wchar_t* in) { - int size = WideCharToMultiByte(CP_UTF8, 0, in, -1, nullptr, 0, nullptr, nullptr); - std::string out(size, '\0'); + size_t size = WideCharToMultiByte(CP_UTF8, 0, in, -1, nullptr, 0, nullptr, nullptr); + // size includes terminating null; std::string adds null automatically + std::string out(size - 1, '\0'); WideCharToMultiByte(CP_UTF8, 0, in, -1, &out[0], size, nullptr, nullptr); return out; } From a8822a2b29514700202d586a5397481de93bcbd2 Mon Sep 17 00:00:00 2001 From: Ryan McCampbell Date: Mon, 19 Aug 2019 15:36:45 -0400 Subject: [PATCH 4/6] Cleaned up formatting in DefaultIOSystem.cpp --- code/Common/DefaultIOSystem.cpp | 40 ++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/code/Common/DefaultIOSystem.cpp b/code/Common/DefaultIOSystem.cpp index 7e26ec4b2..ce5996fa8 100644 --- a/code/Common/DefaultIOSystem.cpp +++ b/code/Common/DefaultIOSystem.cpp @@ -83,36 +83,36 @@ static std::string WideToUtf8(const wchar_t* in) // ------------------------------------------------------------------------------------------------ // Tests for the existence of a file at the given path. -bool DefaultIOSystem::Exists( const char* pFile) const +bool DefaultIOSystem::Exists(const char* pFile) const { #if defined(_WIN32) && !defined(WindowsStore) struct __stat64 filestat; - if (0 != _wstat64(Utf8ToWide(pFile).c_str(), &filestat)) { + if (_wstat64(Utf8ToWide(pFile).c_str(), &filestat) != 0) { return false; } #else - FILE* file = ::fopen( pFile, "rb"); - if( !file) + FILE* file = ::fopen(pFile, "rb"); + if (!file) return false; - ::fclose( file); + ::fclose(file); #endif return true; } // ------------------------------------------------------------------------------------------------ // Open a new file with a given path. -IOStream* DefaultIOSystem::Open( const char* strFile, const char* strMode) +IOStream* DefaultIOSystem::Open(const char* strFile, const char* strMode) { - ai_assert(NULL != strFile); - ai_assert(NULL != strMode); + ai_assert(strFile != nullptr); + ai_assert(strMode != nullptr); FILE* file; #if defined(_WIN32) && !defined(WindowsStore) file = ::_wfopen(Utf8ToWide(strFile).c_str(), Utf8ToWide(strMode).c_str()); #else file = ::fopen(strFile, strMode); #endif - if (nullptr == file) + if (!file) return nullptr; return new DefaultIOStream(file, strFile); @@ -120,7 +120,7 @@ IOStream* DefaultIOSystem::Open( const char* strFile, const char* strMode) // ------------------------------------------------------------------------------------------------ // Closes the given file and releases all resources associated with it. -void DefaultIOSystem::Close( IOStream* pFile) +void DefaultIOSystem::Close(IOStream* pFile) { delete pFile; } @@ -138,14 +138,14 @@ char DefaultIOSystem::getOsSeparator() const // ------------------------------------------------------------------------------------------------ // IOSystem default implementation (ComparePaths isn't a pure virtual function) -bool IOSystem::ComparePaths (const char* one, const char* second) const +bool IOSystem::ComparePaths(const char* one, const char* second) const { - return !ASSIMP_stricmp(one,second); + return !ASSIMP_stricmp(one, second); } // ------------------------------------------------------------------------------------------------ // Convert a relative path into an absolute path -inline static std::string MakeAbsolutePath (const char* in) +inline static std::string MakeAbsolutePath(const char* in) { ai_assert(in); std::string out; @@ -182,21 +182,21 @@ inline static std::string MakeAbsolutePath (const char* in) // ------------------------------------------------------------------------------------------------ // DefaultIOSystem's more specialized implementation -bool DefaultIOSystem::ComparePaths (const char* one, const char* second) const +bool DefaultIOSystem::ComparePaths(const char* one, const char* second) const { // chances are quite good both paths are formatted identically, // so we can hopefully return here already - if( !ASSIMP_stricmp(one,second) ) + if (!ASSIMP_stricmp(one, second)) return true; std::string temp1 = MakeAbsolutePath(one); std::string temp2 = MakeAbsolutePath(second); - return !ASSIMP_stricmp(temp1,temp2); + return !ASSIMP_stricmp(temp1, temp2); } // ------------------------------------------------------------------------------------------------ -std::string DefaultIOSystem::fileName( const std::string &path ) +std::string DefaultIOSystem::fileName(const std::string& path) { std::string ret = path; std::size_t last = ret.find_last_of("\\/"); @@ -205,16 +205,16 @@ std::string DefaultIOSystem::fileName( const std::string &path ) } // ------------------------------------------------------------------------------------------------ -std::string DefaultIOSystem::completeBaseName( const std::string &path ) +std::string DefaultIOSystem::completeBaseName(const std::string& path) { std::string ret = fileName(path); std::size_t pos = ret.find_last_of('.'); - if(pos != ret.npos) ret = ret.substr(0, pos); + if (pos != std::string::npos) ret = ret.substr(0, pos); return ret; } // ------------------------------------------------------------------------------------------------ -std::string DefaultIOSystem::absolutePath( const std::string &path ) +std::string DefaultIOSystem::absolutePath(const std::string& path) { std::string ret = path; std::size_t last = ret.find_last_of("\\/"); From a16906686f498cf4360c54ce41ef7838b1fecc84 Mon Sep 17 00:00:00 2001 From: Ryan McCampbell Date: Tue, 3 Sep 2019 21:06:48 -0400 Subject: [PATCH 5/6] Fixed type conversion warnings --- code/Common/DefaultIOSystem.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/Common/DefaultIOSystem.cpp b/code/Common/DefaultIOSystem.cpp index ce5996fa8..c0d356689 100644 --- a/code/Common/DefaultIOSystem.cpp +++ b/code/Common/DefaultIOSystem.cpp @@ -64,18 +64,18 @@ using namespace Assimp; #ifdef _WIN32 static std::wstring Utf8ToWide(const char* in) { - size_t size = MultiByteToWideChar(CP_UTF8, 0, in, -1, nullptr, 0); + int size = MultiByteToWideChar(CP_UTF8, 0, in, -1, nullptr, 0); // size includes terminating null; std::wstring adds null automatically - std::wstring out(size - 1, L'\0'); + std::wstring out(static_cast(size) - 1, L'\0'); MultiByteToWideChar(CP_UTF8, 0, in, -1, &out[0], size); return out; } static std::string WideToUtf8(const wchar_t* in) { - size_t size = WideCharToMultiByte(CP_UTF8, 0, in, -1, nullptr, 0, nullptr, nullptr); + int size = WideCharToMultiByte(CP_UTF8, 0, in, -1, nullptr, 0, nullptr, nullptr); // size includes terminating null; std::string adds null automatically - std::string out(size - 1, '\0'); + std::string out(static_cast(size) - 1, '\0'); WideCharToMultiByte(CP_UTF8, 0, in, -1, &out[0], size, nullptr, nullptr); return out; } From eb762e654eb73bf31220f1879d74254fb9332803 Mon Sep 17 00:00:00 2001 From: Ryan McCampbell Date: Tue, 3 Sep 2019 23:13:44 -0400 Subject: [PATCH 6/6] Removed WindowsStore checks --- code/Common/DefaultIOSystem.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/code/Common/DefaultIOSystem.cpp b/code/Common/DefaultIOSystem.cpp index c0d356689..6fdc24dd8 100644 --- a/code/Common/DefaultIOSystem.cpp +++ b/code/Common/DefaultIOSystem.cpp @@ -85,7 +85,7 @@ static std::string WideToUtf8(const wchar_t* in) // Tests for the existence of a file at the given path. bool DefaultIOSystem::Exists(const char* pFile) const { -#if defined(_WIN32) && !defined(WindowsStore) +#ifdef _WIN32 struct __stat64 filestat; if (_wstat64(Utf8ToWide(pFile).c_str(), &filestat) != 0) { return false; @@ -107,7 +107,7 @@ IOStream* DefaultIOSystem::Open(const char* strFile, const char* strMode) ai_assert(strFile != nullptr); ai_assert(strMode != nullptr); FILE* file; -#if defined(_WIN32) && !defined(WindowsStore) +#ifdef _WIN32 file = ::_wfopen(Utf8ToWide(strFile).c_str(), Utf8ToWide(strMode).c_str()); #else file = ::fopen(strFile, strMode); @@ -150,21 +150,12 @@ inline static std::string MakeAbsolutePath(const char* in) ai_assert(in); std::string out; #ifdef _WIN32 -#ifndef WindowsStore wchar_t* ret = ::_wfullpath(nullptr, Utf8ToWide(in).c_str(), 0); if (ret) { out = WideToUtf8(ret); free(ret); } #else - char* ret = ::_fullpath(nullptr, in, 0); - if (ret) { - out = ret; - free(ret); - } -#endif -#else - // use realpath char* ret = realpath(in, nullptr); if (ret) { out = ret;