Fix: Fix utf8 encoding and first steps of irrloader.

code/AssetLib/Irr/IRRLoader.cpp

@@ -86,10 +86,6 @@ IRRImporter::IRRImporter() :
 	// empty
 }
 
-// ------------------------------------------------------------------------------------------------
-// Destructor, private as well
-IRRImporter::~IRRImporter() = default;
-
 // ------------------------------------------------------------------------------------------------
 // Returns whether the class can handle the format of the given file.
 bool IRRImporter::CanRead(const std::string &pFile, IOSystem *pIOHandler, bool /*checkSig*/) const {
@@ -836,13 +832,32 @@ void IRRImporter::GenerateGraph(Node *root, aiNode *rootOut, aiScene *scene,
 	}
 }
 
+// ------------------------------------------------------------------------------------------------
+void setupXmlTree(const std::string &filename, IOSystem *pIOHandler, pugi::xml_node &rootElement) {
+    std::unique_ptr<IOStream> file(pIOHandler->Open(filename));
+
+    // Check whether we can read from the file
+    if (file == nullptr) {
+        throw DeadlyImportError("Failed to open IRR file ", filename);
+    }
+
+    // Construct the irrXML parser
+    XmlParser st;
+    if (!st.parse(file.get())) {
+        throw DeadlyImportError("XML parse error while loading IRR file ", filename);
+    }
+    rootElement = st.getRootNode().child("irr_scene");
+}
+
 // ------------------------------------------------------------------------------------------------
 // Imports the given file into the given scene structure.
-void IRRImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSystem *pIOHandler) {
+void IRRImporter::InternReadFile(const std::string &filename, aiScene *pScene, IOSystem *pIOHandler) {
-	std::unique_ptr<IOStream> file(pIOHandler->Open(pFile));
+    pugi::xml_node rootElement;
+    setupXmlTree(filename, pIOHandler, rootElement);
+    //std::unique_ptr<IOStream> file(pIOHandler->Open(pFile));
 
 	// Check whether we can read from the file
-    if (file == nullptr) {
+    /* if (file == nullptr) {
         throw DeadlyImportError("Failed to open IRR file ", pFile);
     }
 
@@ -851,7 +866,7 @@ void IRRImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
     if (!st.parse( file.get() )) {
         throw DeadlyImportError("XML parse error while loading IRR file ", pFile);
     }
-    pugi::xml_node rootElement = st.getRootNode();
+    pugi::xml_node rootElement = st.getRootNode().child("irr_scene");*/
 
 	// The root node of the scene
 	Node *root = new Node(Node::DUMMY);
@@ -862,7 +877,7 @@ void IRRImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
 	Node *curParent = root;
 
 	// Scene-graph node we're currently working on
-	Node *curNode = nullptr;
+    Node *curNode = nullptr;
 
 	// List of output cameras
 	std::vector<aiCamera *> cameras;
@@ -872,7 +887,6 @@ void IRRImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
 
 	// Batch loader used to load external models
 	BatchLoader batch(pIOHandler);
-	//batch.SetBasePath(pFile);
 
 	cameras.reserve(5);
 	lights.reserve(5);
@@ -881,9 +895,7 @@ void IRRImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
 	unsigned int guessedAnimCnt = 0, guessedMeshCnt = 0, guessedMatCnt = 0;
 
 	// Parse the XML file
-
+	for (const pugi::xml_node &child : rootElement.children())
-	//while (reader->read())  {
-	for (pugi::xml_node child : rootElement.children())
 		switch (child.type()) {
 			case pugi::node_element:
 				if (!ASSIMP_stricmp(child.name(), "node")) {
@@ -907,47 +919,46 @@ void IRRImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
                      *  materials assigned (except lights, cameras and dummies, of course).
                      */
 					// ***********************************************************************
-					//const char *sz = reader->getAttributeValueSafe("type");
 					pugi::xml_attribute attrib = child.attribute("type");
 					Node *nd;
-					if (!ASSIMP_stricmp(attrib.name(), "mesh") || !ASSIMP_stricmp(attrib.name(), "octTree")) {
+                    if (!ASSIMP_stricmp(attrib.value(), "mesh") || !ASSIMP_stricmp(attrib.value(), "octTree")) {
 						// OctTree's and meshes are treated equally
 						nd = new Node(Node::MESH);
-					} else if (!ASSIMP_stricmp(attrib.name(), "cube")) {
+                    } else if (!ASSIMP_stricmp(attrib.value(), "cube")) {
 						nd = new Node(Node::CUBE);
 						++guessedMeshCnt;
-					} else if (!ASSIMP_stricmp(attrib.name(), "skybox")) {
+                    } else if (!ASSIMP_stricmp(attrib.value(), "skybox")) {
 						nd = new Node(Node::SKYBOX);
 						guessedMeshCnt += 6;
-					} else if (!ASSIMP_stricmp(attrib.name(), "camera")) {
+                    } else if (!ASSIMP_stricmp(attrib.value(), "camera")) {
 						nd = new Node(Node::CAMERA);
 
 						// Setup a temporary name for the camera
 						aiCamera *cam = new aiCamera();
 						cam->mName.Set(nd->name);
 						cameras.push_back(cam);
-					} else if (!ASSIMP_stricmp(attrib.name(), "light")) {
+                    } else if (!ASSIMP_stricmp(attrib.value(), "light")) {
 						nd = new Node(Node::LIGHT);
 
 						// Setup a temporary name for the light
 						aiLight *cam = new aiLight();
 						cam->mName.Set(nd->name);
 						lights.push_back(cam);
-					} else if (!ASSIMP_stricmp(attrib.name(), "sphere")) {
+                    } else if (!ASSIMP_stricmp(attrib.value(), "sphere")) {
 						nd = new Node(Node::SPHERE);
 						++guessedMeshCnt;
-					} else if (!ASSIMP_stricmp(attrib.name(), "animatedMesh")) {
+                    } else if (!ASSIMP_stricmp(attrib.value(), "animatedMesh")) {
 						nd = new Node(Node::ANIMMESH);
-					} else if (!ASSIMP_stricmp(attrib.name(), "empty")) {
+                    } else if (!ASSIMP_stricmp(attrib.value(), "empty")) {
 						nd = new Node(Node::DUMMY);
-					} else if (!ASSIMP_stricmp(attrib.name(), "terrain")) {
+                    } else if (!ASSIMP_stricmp(attrib.value(), "terrain")) {
 						nd = new Node(Node::TERRAIN);
-					} else if (!ASSIMP_stricmp(attrib.name(), "billBoard")) {
+                    } else if (!ASSIMP_stricmp(attrib.value(), "billBoard")) {
 						// We don't support billboards, so ignore them
 						ASSIMP_LOG_ERROR("IRR: Billboards are not supported by Assimp");
 						nd = new Node(Node::DUMMY);
 					} else {
-						ASSIMP_LOG_WARN("IRR: Found unknown node: ", attrib.name());
+                        ASSIMP_LOG_WARN("IRR: Found unknown node: ", attrib.value());
 
 						/*  We skip the contents of nodes we don't know.
                          *  We parse the transformation and all animators
@@ -992,10 +1003,7 @@ void IRRImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
 						++guessedAnimCnt;
 					}
 
-					/*  Parse all elements in the attributes block
+					// Parse all elements in the attributes block and process them.
-                     *  and process them.
-                     */
-					//					while (reader->read()) {
 					for (pugi::xml_node attrib : child.children()) {
 						if (attrib.type() == pugi::node_element) {
 							//if (reader->getNodeType() == EXN_ELEMENT) {
@@ -1083,10 +1091,9 @@ void IRRImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
 									if (prop.name == "FramesPerSecond" && Node::ANIMMESH == curNode->type) {
 										curNode->framesPerSecond = prop.value;
 									} else if (Node::CAMERA == curNode->type) {
-										/*  This is the vertical, not the horizontal FOV.
+										//  This is the vertical, not the horizontal FOV.
-                                    *  We need to compute the right FOV from the
+                                        //  We need to compute the right FOV from the
-                                    *  screen aspect which we don't know yet.
+                                        //  screen aspect which we don't know yet.
-                                    */
 										if (prop.name == "Fovy") {
 											cameras.back()->mHorizontalFOV = prop.value;
 										} else if (prop.name == "Aspect") {
@@ -1097,8 +1104,7 @@ void IRRImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
 											cameras.back()->mClipPlaneFar = prop.value;
 										}
 									} else if (Node::LIGHT == curNode->type) {
-										/*  Additional light information
+										// Additional light information
-                                     */
 										if (prop.name == "Attenuation") {
 											lights.back()->mAttenuationLinear = prop.value;
 										} else if (prop.name == "OuterCone") {

code/AssetLib/Irr/IRRLoader.h

@@ -65,7 +65,7 @@ namespace Assimp    {
 class IRRImporter : public BaseImporter, public IrrlichtBase {
 public:
     IRRImporter();
-    ~IRRImporter() override;
+    ~IRRImporter() override = default;
 
     // -------------------------------------------------------------------
     /** Returns whether the class can handle the format of the given file.

code/Common/BaseImporter.cpp

@@ -389,9 +389,18 @@ void BaseImporter::ConvertToUTF8(std::vector<char> &data) {
     // UTF 16 LE with BOM
     if (*((uint16_t *)&data.front()) == 0xFEFF) {
         ASSIMP_LOG_DEBUG("Found UTF-16 BOM ...");
-
+        const size_t size = data.size();
-        std::vector<unsigned char> output;
+        const uint16_t *sourceStart = (uint16_t *)data.data();
-        utf8::utf16to8(data.begin(), data.end(), back_inserter(output));
+        const size_t targetSize = size * 3; // enough to encode
+        char *targetStart = new char[targetSize];
+        std::memset(targetStart, 0, targetSize * sizeof(char));
+        utf8::utf16to8(sourceStart, sourceStart + size / 2, targetStart);        
+        std::string result(targetStart);
+        data.clear();
+        for (size_t i = 0; i < result.size(); ++i) {
+            data.push_back(result[i]);
+        }
+        delete[] targetStart;
         return;
     }
 }

contrib/pugixml/src/pugiconfig.hpp

@@ -15,7 +15,7 @@
 #define HEADER_PUGICONFIG_HPP
 
 // Uncomment this to enable wchar_t mode
-// #define PUGIXML_WCHAR_MODE
+//#define PUGIXML_WCHAR_MODE
 
 // Uncomment this to enable compact mode
 // #define PUGIXML_COMPACT

include/assimp/Base64.hpp

@@ -43,7 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef AI_BASE64_HPP_INC
 #define AI_BASE64_HPP_INC
 
-#include <stdint.h>
+#include <cstdint>
 #include <vector>
 #include <string>
 

include/assimp/BlobIOSystem.h

@@ -54,6 +54,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <assimp/DefaultLogger.hpp>
 #include <assimp/IOStream.hpp>
 #include <assimp/IOSystem.hpp>
+
 #include <cstdint>
 #include <set>
 #include <vector>
@@ -62,7 +63,9 @@ namespace Assimp {
 class BlobIOSystem;
 
 // --------------------------------------------------------------------------------------------
-/** Redirect IOStream to a blob */
+/**
+ * @brief Redirect IOStream to a blob
+ */
 // --------------------------------------------------------------------------------------------
 class BlobIOStream : public IOStream {
 public:
@@ -84,7 +87,6 @@ public:
     ///	@brief  The class destructor.
     ~BlobIOStream() override;
 
-public:
     // -------------------------------------------------------------------
     aiExportDataBlob *GetBlob() {
         aiExportDataBlob *blob = new aiExportDataBlob();
@@ -194,15 +196,14 @@ private:
 /** Redirect IOSystem to a blob */
 // --------------------------------------------------------------------------------------------
 class BlobIOSystem : public IOSystem {
-
     friend class BlobIOStream;
     typedef std::pair<std::string, aiExportDataBlob *> BlobEntry;
 
-
 public:
     /// @brief The default class constructor.
     BlobIOSystem() :
             baseName{AI_BLOBIO_MAGIC} {
+        // 
     }
 
     ///	@brief  The class constructor with the base name.
@@ -218,7 +219,6 @@ public:
         }
     }
 
-public:
     // -------------------------------------------------------------------
     const char *GetMagicFileName() const {
         return baseName.c_str();
@@ -269,7 +269,6 @@ public:
         return master;
     }
 
-public:
     // -------------------------------------------------------------------
     bool Exists(const char *pFile) const override {
         return created.find(std::string(pFile)) != created.end();

include/assimp/XmlParser.h

@@ -44,13 +44,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <assimp/ai_assert.h>
 #include <assimp/DefaultLogger.hpp>
+#include <assimp/BaseImporter.h>
 
-#include "BaseImporter.h"
 #include "IOStream.hpp"
 
 #include <pugixml.hpp>
 #include <utility>
 #include <vector>
+#include <cinttypes>
 
 namespace Assimp {
 
@@ -127,6 +128,8 @@ public:
     /// @return true, if the parsing was successful, false if not.
     bool parse(IOStream *stream);
 
+    bool parseFromBuffer(std::vector<char> &buffer);
+
     /// @brief  Will return true if a root node is there.
     /// @return true in case of an existing root.
     bool hasRoot() const;
@@ -287,22 +290,34 @@ bool TXmlParser<TNodeType>::hasNode(const std::string &name) {
 
 template <class TNodeType>
 bool TXmlParser<TNodeType>::parse(IOStream *stream) {
-    if (hasRoot()) {
-        clear();
-    }
-
     if (nullptr == stream) {
         ASSIMP_LOG_DEBUG("Stream is nullptr.");
         return false;
     }
 
     const size_t len = stream->FileSize();
+    if (len == 0) {
+        ASSIMP_LOG_DEBUG("The xml file is empty.");
+        return false;
+    }
+
     mData.resize(len + 1);
     memset(&mData[0], '\0', len + 1);
     stream->Read(&mData[0], 1, len);
 
+    return parseFromBuffer(mData);
+}
+
+template <class TNodeType>
+bool TXmlParser<TNodeType>::parseFromBuffer(std::vector<char> &buffer) {
+    if (hasRoot()) {
+        clear();
+    }
+
+    BaseImporter::ConvertToUTF8(mData);
+
     mDoc = new pugi::xml_document();
-    pugi::xml_parse_result parse_result = mDoc->load_string(&mData[0], pugi::parse_full);
+    pugi::xml_parse_result parse_result = mDoc->load_string(&buffer[0], pugi::parse_full);
     if (parse_result.status == pugi::status_ok) {
         return true;
     }