- fbx: further work on broadphase tokenization.

code/FBXTokenizer.cpp

@@ -45,9 +45,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifndef ASSIMP_BUILD_NO_FBX_IMPORTER
 
-#include "FBXTokenizer.h"
 #include "ParsingUtils.h"
 
+#include "FBXTokenizer.h"
+#include "FBXUtil.h"
+
 namespace Assimp {
 namespace FBX {
 
@@ -70,6 +72,32 @@ Token::~Token()
 }
 
 
+namespace {
+
+// process a potential data token up to 'cur', adding it to 'output_tokens'. 
+// ------------------------------------------------------------------------------------------------
+void ProcessDataToken( TokenList& output_tokens, const char*& start, const char*& end,
+					  unsigned int line, 
+					  unsigned int column, 
+					  TokenType type = TokenType_DATA)
+{
+	if (start != end) {
+		// tokens should have no whitespace in them and [start,end] should
+		// properly delimit the valid range.
+		for (const char* c = start; c != end; ++c) {
+			if (IsSpaceOrNewLine(*c)) {
+				throw DeadlyImportError(Util::AddLineAndColumn("FBX-Tokenize","unexpected whitespace in token",line,column));
+			}
+		}
+
+		output_tokens.push_back(boost::make_shared<Token>(start,end,type,line,column));
+	}
+
+	start = end = NULL;
+}
+
+}
+
 // ------------------------------------------------------------------------------------------------
 void Tokenize(TokenList& output_tokens, const char* input)
 {
@@ -80,7 +108,9 @@ void Tokenize(TokenList& output_tokens, const char* input)
 	unsigned int column = 1;
 
 	bool comment = false;
+	bool in_double_quotes = false;
 	
+	const char* token_begin = NULL, *token_end = NULL;
 	for (const char* cur = input;*cur;++cur,++column) {
 		const char c = *cur;
 
@@ -97,30 +127,54 @@ void Tokenize(TokenList& output_tokens, const char* input)
 			continue;
 		}
 
+		if(in_double_quotes) {
+			if (c == '\"') {
+				in_double_quotes = false;
+				token_end = cur;
+				if (!token_begin) {
+					token_begin = cur;
+				}
+			}
+			continue;
+		}
+
 		switch(c)
 		{
+		case '\"':
+			in_double_quotes = true;
+			continue;
+
 		case ';':
+			ProcessDataToken(output_tokens,token_begin,token_end,line,column);
 			comment = true;
 			continue;
 
 		case '{':
+			ProcessDataToken(output_tokens,token_begin,token_end, line, column);
 			output_tokens.push_back(boost::make_shared<Token>(cur,cur+1,TokenType_OPEN_BRACKET,line,column));
 			break;
 
 		case '}':
+			ProcessDataToken(output_tokens,token_begin,token_end,line,column);
 			output_tokens.push_back(boost::make_shared<Token>(cur,cur+1,TokenType_CLOSE_BRACKET,line,column));
 			break;
 		
 		case ',':
+			ProcessDataToken(output_tokens,token_begin,token_end,line,column);
 			output_tokens.push_back(boost::make_shared<Token>(cur,cur+1,TokenType_COMMA,line,column));
 			break;
-		}
 
-		
+		case ':':
-		if (IsSpaceOrNewLine(c)) {
+			ProcessDataToken(output_tokens,token_begin,token_end,line,column, TokenType_KEY);
-			//
+			break;
+		}
+		
+		if (!IsSpaceOrNewLine(c)) {
+			token_end = cur;
+			if (!token_begin) {
+				token_begin = cur;
+			}
 		}
-		// XXX parse key and data elements
 	}
 }
 

code/FBXTokenizer.h

@@ -118,7 +118,8 @@ typedef std::vector< boost::shared_ptr<Token> > TokenList;
  *  Skips over comments and generates line and column numbers.
  *
  * @param output_tokens Receives a list of all tokens in the input data.
- * @param input_buffer Textual input buffer to be processed, 0-terminated. */
+ * @param input_buffer Textual input buffer to be processed, 0-terminated.
+ * @throw DeadlyImportError if something goes wrong */
 void Tokenize(TokenList& output_tokens, const char* input);
 
 

code/FBXUtil.cpp

@@ -0,0 +1,66 @@
+/*
+Open Asset Import Library (assimp)
+----------------------------------------------------------------------
+
+Copyright (c) 2006-2012, assimp team
+All rights reserved.
+
+Redistribution and use of this software in source and binary forms, 
+with or without modification, are permitted provided that the 
+following conditions are met:
+
+* Redistributions of source code must retain the above
+  copyright notice, this list of conditions and the
+  following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the
+  following disclaimer in the documentation and/or other
+  materials provided with the distribution.
+
+* Neither the name of the assimp team, nor the names of its
+  contributors may be used to endorse or promote products
+  derived from this software without specific prior
+  written permission of the assimp team.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------
+*/
+
+/** @file  FBXUtil.cpp
+ *  @brief Implementation of internal FBX utility functions
+ */
+#include "AssimpPCH.h"
+
+#include "FBXUtil.h"
+#include "TinyFormatter.h"
+
+#ifndef ASSIMP_BUILD_NO_FBX_IMPORTER
+
+namespace Assimp {
+namespace FBX {
+namespace Util {
+
+// ------------------------------------------------------------------------------------------------
+std::string AddLineAndColumn(const std::string& prefix, const std::string& text, unsigned int line, unsigned int column)
+{
+	return static_cast<std::string>( (Formatter::format(),prefix,"(line ",line,", col ",column,") ",text) );
+}
+
+} // !Util
+} // !FBX
+} // !Assimp
+
+#endif
+

code/FBXUtil.h

@@ -0,0 +1,67 @@
+/*
+Open Asset Import Library (assimp)
+----------------------------------------------------------------------
+
+Copyright (c) 2006-2012, assimp team
+All rights reserved.
+
+Redistribution and use of this software in source and binary forms, 
+with or without modification, are permitted provided that the 
+following conditions are met:
+
+* Redistributions of source code must retain the above
+  copyright notice, this list of conditions and the
+  following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the
+  following disclaimer in the documentation and/or other
+  materials provided with the distribution.
+
+* Neither the name of the assimp team, nor the names of its
+  contributors may be used to endorse or promote products
+  derived from this software without specific prior
+  written permission of the assimp team.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------
+*/
+
+/** @file  FBXUtil.h
+ *  @brief FBX utility functions for internal use
+ */
+#ifndef INCLUDED_AI_FBX_UTIL_H
+#define INCLUDED_AI_FBX_UTIL_H
+
+#include <string>
+#include "FBXCompileConfig.h"
+
+namespace Assimp {
+namespace FBX {
+namespace Util {
+
+/** Format log/error messages using a given line location in the source file.
+ *
+ *  @param prefix Message prefix to be preprended to the location info.
+ *  @param text Message text
+ *  @param line Line index, 1-based
+ *  @param column Colum index, 1-based 
+ *  @return A string of the following format: {prefix} (line {line}, col {column}) {text}*/
+std::string AddLineAndColumn(const std::string& prefix, const std::string& text, unsigned int line, unsigned int column);
+	
+}
+}
+}
+
+#endif // ! INCLUDED_AI_FBX_UTIL_H

workspaces/vc9/assimp.vcproj

@@ -2083,6 +2083,14 @@
 						RelativePath="..\..\code\FBXTokenizer.h"
 						>
 					</File>
+					<File
+						RelativePath="..\..\code\FBXUtil.cpp"
+						>
+					</File>
+					<File
+						RelativePath="..\..\code\FBXUtil.h"
+						>
+					</File>
 				</Filter>
 			</Filter>
 			<Filter